├── caffe ├── eq1.png ├── caffe.jpeg ├── digit.png ├── my_net.png ├── solver.prototxt ├── test.prototxt ├── train.prototxt ├── data.txt └── caffelayerpython.py ├── data └── fire_theft.xls ├── pytorch ├── kernel.png └── ipytorch.ipynb ├── tensorflow ├── graph.png ├── tfdemo.py └── itensorflow.ipynb ├── .vscode ├── settings.json └── tasks.json ├── basic ├── data.txt ├── layers.py ├── linear_regression.py ├── backprop.py ├── ibackprop.ipynb └── ilinear_regression.ipynb ├── .gitignore └── README.md /caffe/eq1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/caffe/eq1.png -------------------------------------------------------------------------------- /caffe/caffe.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/caffe/caffe.jpeg -------------------------------------------------------------------------------- /caffe/digit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/caffe/digit.png -------------------------------------------------------------------------------- /caffe/my_net.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/caffe/my_net.png -------------------------------------------------------------------------------- /data/fire_theft.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/data/fire_theft.xls -------------------------------------------------------------------------------- /pytorch/kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/pytorch/kernel.png -------------------------------------------------------------------------------- /tensorflow/graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pedrodiamel/nettutorial/HEAD/tensorflow/graph.png -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | } -------------------------------------------------------------------------------- /tensorflow/tfdemo.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | 6 | 7 | print('ok!!!') 8 | -------------------------------------------------------------------------------- /caffe/solver.prototxt: -------------------------------------------------------------------------------- 1 | power : 0.75 2 | max_iter : 1000 3 | base_lr : 0.01 4 | weight_decay : 0.0005 5 | lr_policy : "inv" 6 | snapshot_prefix : "snapshot/snapshot" 7 | snapshot : 100 8 | gamma : 0.01 9 | solver_mode : CPU 10 | net : "train.prototxt" 11 | display : 50 12 | momentum : 0.95 13 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "0.1.0", 5 | "command": "python", 6 | "isShellCommand": true, 7 | "args": ["${fileBasename}"], 8 | "showOutput": "always" 9 | } -------------------------------------------------------------------------------- /caffe/test.prototxt: -------------------------------------------------------------------------------- 1 | name: "feactnet" 2 | layer { 3 | name: "data" 4 | type: "Input" 5 | top: "data" 6 | input_param { shape: { dim: 1 dim: 1 dim: 256 dim: 256 } } 7 | } 8 | 9 | layer { 10 | name: "conv1" 11 | type: "Convolution" 12 | bottom: "data" 13 | top: "conv1" 14 | convolution_param { 15 | num_output: 1 16 | pad: 1 17 | kernel_size: 3 18 | stride: 1 19 | weight_filler { 20 | type: "gaussian" 21 | std: 0.00999999977648 22 | } 23 | bias_filler { 24 | type: "constant" 25 | value: 0.0 26 | } 27 | } 28 | } 29 | layer { 30 | name: "pool1" 31 | type: "Pooling" 32 | bottom: "conv1" 33 | top: "pool1" 34 | pooling_param { 35 | pool: MAX 36 | kernel_size: 3 37 | stride: 1 38 | pad: 1 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /caffe/train.prototxt: -------------------------------------------------------------------------------- 1 | name: "feactnet" 2 | layer { 3 | name: "data" 4 | type: "Python" 5 | top: "data" 6 | top: "label" 7 | python_param { 8 | module: "caffelayerpython" 9 | layer: "pydatagenerate" 10 | param_str: "{\"batch_size\":1, \"im_shape\":50, \"mu\":0.5 }" 11 | } 12 | } 13 | layer { 14 | name: "conv1" 15 | type: "Convolution" 16 | bottom: "data" 17 | top: "conv1" 18 | convolution_param { 19 | num_output: 1 20 | pad: 1 21 | kernel_size: 3 22 | stride: 1 23 | weight_filler { 24 | type: "gaussian" 25 | std: 0.00999999977648 26 | } 27 | bias_filler { 28 | type: "constant" 29 | value: 0.0 30 | } 31 | } 32 | } 33 | layer { 34 | name: "pool1" 35 | type: "Pooling" 36 | bottom: "conv1" 37 | top: "pool1" 38 | pooling_param { 39 | pool: MAX 40 | kernel_size: 3 41 | stride: 1 42 | pad: 1 43 | } 44 | } 45 | layer { 46 | name: "loss" 47 | type: "Python" 48 | bottom: "pool1" 49 | bottom: "label" 50 | top: "loss" 51 | python_param { 52 | module: "caffelayerpython" 53 | layer: "pyimgloss" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /basic/data.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /caffe/data.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | #Folders 3 | graphs 4 | checkpoints 5 | data 6 | snapshot 7 | model 8 | out 9 | 10 | #VSCode 11 | .vscode 12 | 13 | # Byte-compiled / optimized / DLL files 14 | __pycache__/ 15 | *.py[cod] 16 | *$py.class 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Distribution / packaging 22 | .Python 23 | env/ 24 | build/ 25 | develop-eggs/ 26 | dist/ 27 | downloads/ 28 | eggs/ 29 | .eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | wheels/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | 40 | # PyInstaller 41 | # Usually these files are written by a python script from a template 42 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 43 | *.manifest 44 | *.spec 45 | 46 | # Installer logs 47 | pip-log.txt 48 | pip-delete-this-directory.txt 49 | 50 | # Unit test / coverage reports 51 | htmlcov/ 52 | .tox/ 53 | .coverage 54 | .coverage.* 55 | .cache 56 | nosetests.xml 57 | coverage.xml 58 | *.cover 59 | .hypothesis/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # celery beat schedule file 89 | celerybeat-schedule 90 | 91 | # SageMath parsed files 92 | *.sage.py 93 | 94 | # dotenv 95 | .env 96 | 97 | # virtualenv 98 | .venv 99 | venv/ 100 | ENV/ 101 | 102 | # Spyder project settings 103 | .spyderproject 104 | .spyproject 105 | 106 | # Rope project settings 107 | .ropeproject 108 | 109 | # mkdocs documentation 110 | /site 111 | 112 | # mypy 113 | .mypy_cache/ 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nettutorial 2 | 3 | Deep learning tutorial 4 | - [Basic](https://github.com/pedrodiamel/nettutorial/blob/master/ibackprop.ipynb) 5 | - [Caffe](https://github.com/pedrodiamel/nettutorial/blob/master/icaffelayer.ipynb) 6 | - [Tensorflow](https://github.com/pedrodiamel/nettutorial/blob/master/tf_linear_regression.ipynb) 7 | - [Pytorch](https://github.com/pedrodiamel/nettutorial/blob/master/ipytorch.ipynb) 8 | 9 | ## Install 10 | 11 | - install python 12 | - $ pip install jupyter 13 | - $ git clone https://github.com/pedrodiamel/nettutorial.git nettutorial 14 | - $ cd nettutorial 15 | - $ jupyter notebook 16 | 17 | 18 | ## Tutorials 19 | 20 | - backpropagation (ibackprop.ipynb) 21 | - linal regression (ilinear_regression.ipynb) 22 | - caffe layer customizer (icaffelayer.ipynb) 23 | - tensorflow 24 | - pytorch 25 | 26 | ## Links 27 | 28 | ### NeuralNet: 29 | - http://www.deeplearningbook.org/ 30 | - https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/ 31 | - https://www.youtube.com/watch?v=GlcnxUlrtek 32 | - http://neuralnetworksanddeeplearning.com/chap3.html 33 | - https://www.youtube.com/watch?v=-YRB0eFxeQA&list=PLE6Wd9FR--EfW8dtjAuPoTuPcqmOV53Fu&index=10 34 | - https://www.youtube.com/watch?v=PpFTODTztsU&list=PL6Xpj9I5qXYEcOhn7TqghAJ6NAPrNmUBH&index=8 35 | - https://stats.stackexchange.com/questions/235528/backpropagation-with-softmax-cross-entropy 36 | 37 | 38 | ### Caffe: 39 | - https://github.com/NVIDIA/caffe 40 | - https://github.com/NVIDIA/caffe/tree/caffe-0.16/examples 41 | - http://davidstutz.de/pycaffe-tools-examples-and-resources/ 42 | - http://christopher5106.github.io/deep/learning/2015/09/04/Deep-learning-tutorial-on-Caffe-Technology.html 43 | - http://www.andrewjanowczyk.com/real-time-data-augmentation-using-nvidia-digits-python-layer/ 44 | 45 | ### Tensorflow: 46 | - https://github.com/chiphuyen/stanford-tensorflow-tutorials 47 | - https://www.youtube.com/playlist?list=PL9Hr9sNUjfsmEu1ZniY0XpHSzl5uihcXZ 48 | 49 | ### Pytorch 50 | 51 | - https://github.com/ritchieng/the-incredible-pytorch 52 | - https://github.com/bharathgs/Awesome-pytorch-list 53 | 54 | -------------------------------------------------------------------------------- /basic/layers.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import numpy.matlib 5 | 6 | # Layer class 7 | class layer(object): 8 | 9 | def forward(self, param): 10 | raise NotImplementedError 11 | 12 | def backward(self, param): 13 | raise NotImplementedError 14 | 15 | # Inner layer 16 | class inner(layer): 17 | 18 | def forward(self, param): 19 | return np.dot(param['w'],param['x']); 20 | 21 | def backward(self, param): 22 | return np.dot(param['w'].T, param['dzdx']), np.dot(param['dzdx'],param['x'].T); 23 | 24 | # Sigmoid layer 25 | class sigm(layer): 26 | 27 | def forward(self, param): 28 | return self._sigm(param['x']); 29 | 30 | def backward(self, param): 31 | return self._sigm(param['x'])*(1-self._sigm(param['x']))*param['dzdx']; 32 | 33 | def _sigm(self, x): return 1.0/(1 + np.exp(-x)); 34 | 35 | 36 | # Loss layer 37 | class loss(layer): 38 | 39 | def forward(self, param): 40 | return (1.0/(2.0*len(param['y']))*np.sum((param['y']-param['x'])**2)); 41 | 42 | def backward(self, param): 43 | return (param['x']-param['y'])*param['dzdx']; 44 | 45 | 46 | # Const function 47 | def costFunc( x, w1, w2): 48 | ''' 49 | forward function 50 | Entrada: 51 | * x vector nxm. m featurs 52 | * w1, w2 weigths 53 | Return: 54 | * z4 55 | 56 | Note: b=0 57 | ''' 58 | 59 | z1 = x; 60 | z2 = inner().forward({'x':z1, 'w':w1}) 61 | z3 = inner().forward({'x':z2, 'w':w2}) 62 | z4 = sigm().forward({'x':z3}); 63 | return z4; 64 | 65 | 66 | # Gradind function 67 | def gradCostFunc(x, y, w1, w2): 68 | ''' 69 | Apply backward function 70 | Compute dervative with respect to w1 and w2 71 | ''' 72 | 73 | # forward ---> 74 | z1 = x; 75 | z2 = inner().forward({'x':z1, 'w':w1}) 76 | z3 = inner().forward({'x':z2, 'w':w2}) 77 | z4 = sigm().forward({'x':z3}); 78 | z5 = loss().forward({'x':z4, 'y':y}); 79 | 80 | E = z5; 81 | 82 | # <--- backward 83 | l5 = 1; 84 | l4 = loss().backward({'x':z4, 'y':y, 'dzdx':l5} ); 85 | l3 = sigm().backward({'x':z3, 'dzdx':l4}); 86 | l2, dEdW2 = inner().backward({'x':z2, 'w':w2, 'dzdx':l3}); 87 | _ , dEdW1 = inner().backward({'x':z1, 'w':w1, 'dzdx':l2}); 88 | 89 | 90 | return E, dEdW1, dEdW2 91 | 92 | 93 | -------------------------------------------------------------------------------- /basic/linear_regression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import numpy.matlib 4 | import layers as net 5 | 6 | def plotData(x,y): 7 | plt.plot(x,y,'or'); 8 | plt.ylabel('Profit in $10,000s'); 9 | plt.xlabel('Population of City in 10,000s'); 10 | plt.grid(1) 11 | 12 | def plotLoss(J): 13 | plt.plot(J); 14 | 15 | 16 | # Const function 17 | def costFunc( x, w1): 18 | 19 | m = x.shape[0]; 20 | z = 0; 21 | for i in range(m): 22 | 23 | z1 = x[i,:].T; 24 | z += net.inner().forward({'x':z1, 'w':w1}) 25 | #z3 = net.sigm().forward({'x':z2}); 26 | 27 | return z/m; 28 | 29 | # Gradind function 30 | def gradCostFunc(x, y, w1): 31 | 32 | m = x.shape[0]; 33 | dEdW1 = 0; E = 0; 34 | for i in range(m): 35 | 36 | # forward ---> 37 | z1 = np.matrix(x[i,:]).T; yi = np.matrix(y[i]); 38 | z2 = net.inner().forward({'x':z1, 'w':w1}) 39 | #z3 = net.sigm().forward({'x':z2}); 40 | 41 | z = net.loss().forward({'x':z2, 'y':yi}); 42 | E += z; 43 | 44 | # <--- backward 45 | l4 = [1]; 46 | l3 = net.loss().backward({'x':z2, 'y':yi, 'dzdx':l4} ); 47 | #l2 = net.sigm().backward({'x':z2, 'dzdx':l3}); 48 | _ , dEdW1_i = net.inner().backward({'x':z1, 'w':w1, 'dzdx':l3}); 49 | 50 | dEdW1 += dEdW1_i; 51 | 52 | return E/m, dEdW1/m 53 | 54 | def loss(y,y_): 55 | m = len(y); 56 | e = 0; 57 | for i in range(m): 58 | e += net.loss().forward({'x':np.matrix(y_[i]), 'y':np.matrix(y[i])}); 59 | return e/m; 60 | 61 | 62 | def gradDescent(x,y,w1, alpha, numiter): 63 | 64 | E_h = np.zeros((numiter,1)); 65 | for i in range(numiter): 66 | E, dEdW1 = gradCostFunc(X, y, w1); 67 | w1 = w1 - alpha*dEdW1; 68 | 69 | E_h[i] = E; 70 | #print('E = {} dEdW = {}'.format(E, dEdW1)); 71 | 72 | return w1, E_h; 73 | 74 | 75 | 76 | # Load data 77 | datas = np.loadtxt('data.txt', delimiter=','); 78 | Xo = datas[:,0]; y = datas[:,1]; 79 | m = len(y); 80 | 81 | # plot data 82 | # plotData(X,y); 83 | 84 | 85 | # Running Gradient Descent 86 | X = np.concatenate((np.ones((m,1)),Xo.reshape(m,1)),axis=1) 87 | # w1 = np.random.randn(1,2); 88 | w1 = np.zeros((1,2)); 89 | 90 | theta, j = gradDescent(X,y,w1, 0.01, 1500) 91 | 92 | # print theta to screen 93 | print('Theta found by gradient descent: {:.2f} {:.2f}'.format(theta[0,0], theta[0,1])); 94 | 95 | 96 | # Plot the linear fit 97 | plotData(Xo,y); 98 | plt.plot(X[:,1], X*theta.T, '-') 99 | plt.legend(['Training data', 'Linear regression']) 100 | plt.show() 101 | 102 | # Predict values for population sizes of 35,000 and 70,000 103 | predict1 = [1, 3.5] *theta.T; 104 | print('For population = 35,000, we predict a profit of {}'.format(predict1*10000)); 105 | predict2 = [1, 7] * theta.T; 106 | print('For population = 70,000, we predict a profit of {}'.format(predict2*10000)); 107 | 108 | 109 | #plotLoss(j) 110 | #plt.show() 111 | 112 | print('ok!!!!') -------------------------------------------------------------------------------- /basic/backprop.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import numpy.matlib 5 | 6 | 7 | # layer sigmoid 8 | def sigm(x): 9 | '''Sigmoide function ''' 10 | return 1/(1 + np.exp(-x)); 11 | 12 | def dsigmdx(x,dzdx): 13 | '''dSigm/dx derivative of the sigmoide function''' 14 | return sigm(x)*(1-sigm(x))*dzdx; 15 | #------ 16 | 17 | # layer softmat 18 | def softmax(x): 19 | """Soft function""" 20 | return np.exp(x) / np.sum(np.exp(x), axis = 0) 21 | 22 | def dsoftmatdx(x,dzdx): 23 | '''dSigm/dx derivative of the sigmoide function''' 24 | return softmax(x)*(1-softmax(x))*dzdx; 25 | #------ 26 | 27 | 28 | # layer inner 29 | def inner(x,w): 30 | ''' 31 | z=W*x+b 32 | note: b = 0 33 | ''' 34 | return np.dot(w,x); 35 | 36 | def dinnerdx(x,w, dzdx): 37 | ''' 38 | dinnerdx derivate with respect to x 39 | ''' 40 | return np.dot(w.T, dzdx); 41 | 42 | def dinnerdw(x,w, dzdx): 43 | ''' 44 | dinnerdw derivate with respect to w 45 | ''' 46 | return np.dot(dzdx,x.T); 47 | # --- 48 | 49 | # layer loss 50 | def loss(x,y): 51 | ''' Loss function 52 | loss = (1/2)(y-x)^2 53 | ''' 54 | return (1/(2*len(y))*np.sum((y-x)**2)); 55 | 56 | 57 | def dlossdx(x,y, dzdx): 58 | ''' 59 | dlossdx derivate with respect to x 60 | Ecuation: d/dx (1/n)\sum_x (y-x)^2 61 | ''' 62 | return (x-y)*dzdx 63 | # --- 64 | 65 | 66 | # inner inner sigm loss 67 | # z1-->|w1x+b|--(z2)-->|w2x+b|--(z3)-->|sigm(x)|--(z4)-->|E=1/n\sum((y-x)^2)| 68 | 69 | def forward( x, w1, w2): 70 | ''' 71 | forward function 72 | Entrada: 73 | * x vector nxm. m featurs 74 | * w1, w2 weigths 75 | Return: 76 | * z4 77 | 78 | Note: b=0 79 | ''' 80 | 81 | z1 = x; 82 | z2 = inner(z1,w1); 83 | z3 = inner(z2,w2); 84 | z4 = sigm(z3); 85 | return z4; 86 | 87 | 88 | def backward(x, y, w1, w2): 89 | ''' 90 | backward function 91 | Compute dervative with respect to w1 and w2 92 | ''' 93 | 94 | # forward ---> 95 | z1 = x; 96 | z2 = inner(z1,w1); 97 | z3 = inner(z2,w2); 98 | z4 = sigm(z3); 99 | 100 | E = loss(z4,y); 101 | 102 | # <--- backward 103 | l5 = 1; 104 | l4 = dlossdx(z4, y, l5) 105 | l3 = dsigmdx(z3,l4); 106 | l2 = dinnerdx(z2,w2,l3); dEdW2 = dinnerdw(z2,w2,l3); 107 | l1 = dinnerdx(z1,w1,l2); dEdW1 = dinnerdw(z1,w1,l2); 108 | 109 | # note: l1 not have need to calculated 110 | 111 | return E, dEdW1, dEdW2 112 | 113 | 114 | 115 | # data 116 | x = np.matrix([[1,2,3]]).T; 117 | y = np.matrix([1.0]); 118 | 119 | # init weigth 120 | w1 = np.matrix([[0.1, 0.2, 0.3],[0.1, 0.2, 0.3]]) 121 | w2 = np.matrix([0.8, 0.2]) 122 | 123 | # Function to minimize 124 | # J(x)_{w1,w2} 125 | y_ = forward(x,w1,w2); 126 | e = loss(y_,y); 127 | 128 | # derivate 129 | # grad J(x) 130 | E, dEdW1, dEdW2 = backward(x, y, w1, w2); 131 | 132 | 133 | # minimization with gradien decent 134 | # w^t = w^(t-1) + lr*gardJ 135 | 136 | 137 | 138 | print(E) 139 | print(dEdW1) 140 | print(dEdW2) 141 | 142 | 143 | 144 | print('ok!!!!') -------------------------------------------------------------------------------- /caffe/caffelayerpython.py: -------------------------------------------------------------------------------- 1 | 2 | import caffe 3 | import numpy as np 4 | from skimage import feature 5 | from skimage import data, color, exposure 6 | import scipy.misc 7 | 8 | class pydatagenerate(caffe.Layer): 9 | 10 | def setup(self, bottom, top): 11 | 12 | # Check top shape 13 | if len(top) != 2: raise Exception("Need to define tops (data, label)") 14 | 15 | #Check bottom shape 16 | if len(bottom) != 0: raise Exception("Do not define a bottom.") 17 | 18 | #Read parameters 19 | params = eval(self.param_str) 20 | self.batch_size = params["batch_size"] 21 | self.im_shape = params["im_shape"] 22 | self.mu = params["mu"] 23 | 24 | 25 | #Reshape top 26 | top[0].reshape(self.batch_size, 1, self.im_shape, self.im_shape) 27 | top[1].reshape(self.batch_size, 1, self.im_shape, self.im_shape) 28 | 29 | 30 | def forward(self, bottom, top): 31 | 32 | for itt in range(self.batch_size): 33 | 34 | # Use the batch loader to load the next image. 35 | im, label = self.load_next_img() 36 | 37 | #Here we could preprocess the image 38 | # ... 39 | interp = 'bilinear'; 40 | im = scipy.misc.imresize(im, (self.im_shape, self.im_shape), interp=interp); 41 | label = scipy.misc.imresize(label, (self.im_shape, self.im_shape), interp=interp); 42 | 43 | # Add directly to the top blob 44 | top[0].data[itt, ...] = im 45 | top[1].data[itt, ...] = label 46 | 47 | 48 | def load_next_img(self): 49 | 50 | # get image dummy 51 | image = color.rgb2gray(data.astronaut()) 52 | 53 | # get label 54 | label = (image < self.mu); 55 | return image, label 56 | 57 | def reshape(self, bottom, top): 58 | """ 59 | There is no need to reshape the data, since the input is of fixed size 60 | (img shape and batch size) 61 | """ 62 | pass 63 | 64 | def backward(self, bottom, top): 65 | """ 66 | This layer does not back propagate 67 | """ 68 | pass 69 | 70 | 71 | 72 | 73 | class pyimgloss(caffe.Layer): 74 | 75 | 76 | def setup(self, bottom, top): 77 | # check input pair 78 | if len(bottom) != 2: raise Exception("Need two inputs to compute distance.") 79 | 80 | def reshape(self, bottom, top): 81 | 82 | # check input dimensions match 83 | # if bottom[0].count != bottom[1].count: raise Exception("Inputs must have the same dimension.") 84 | # difference is shape of inputs 85 | self.diff = np.zeros_like(bottom[0].data, dtype=np.float32) 86 | 87 | # loss output is scalar 88 | top[0].reshape(1) 89 | 90 | 91 | def forward(self, bottom, top): 92 | ''' 93 | 1 1 94 | E = ---\sum max{0,1-x} + --- \sum max{0,x} 95 | |P| xEP |N| xEN 96 | ''' 97 | 98 | x = bottom[0].data; 99 | y = bottom[1].data; 100 | 101 | pos = (np.array(y)==1); 102 | neg = (np.array(y)!=1); 103 | 104 | # E(1,t) = ... 105 | # mean(max(0, 1 - res.x3(pos))) + ... 106 | # mean(max(0, res.x3(neg))) ; 107 | 108 | E = np.mean( self._amax(1-x[pos]) ) + np.mean( self._amax(x[neg]) ); 109 | top[0].data[...] = E; 110 | 111 | def backward(self, top, propagate_down, bottom): 112 | 113 | # dzdx3 = ... 114 | # - single(res.x3 < 1 & pos) / sum(pos(:)) + ... 115 | # + single(res.x3 > 0 & neg) / sum(neg(:)) ; 116 | 117 | x = bottom[0].data; 118 | y = bottom[1].data; 119 | 120 | pos = (np.array(y)==1); 121 | neg = (np.array(y)!=1); 122 | 123 | dzdx = -( x<1 and pos) / np.sum(pos) + ( x>1 and nos) / np.sum(nos); 124 | 125 | bottom[0].diff[...] = dzdx; 126 | bottom[1].diff[...] = -dzdx; 127 | 128 | 129 | def _amax(self, x): return x*(x>0) if x.tolist() else 0.0; 130 | 131 | -------------------------------------------------------------------------------- /tensorflow/itensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "0.12.1\n" 15 | ] 16 | } 17 | ], 18 | "source": [ 19 | "import tensorflow as tf\n", 20 | "import numpy as np\n", 21 | "\n", 22 | "print(tf.__version__)" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 10, 28 | "metadata": { 29 | "collapsed": false 30 | }, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "[16, 4]\n" 37 | ] 38 | } 39 | ], 40 | "source": [ 41 | "x = 2;\n", 42 | "y = 2;\n", 43 | "op1 = tf.add(x,y)\n", 44 | "op2 = tf.mul(x,y)\n", 45 | "op3 = tf.pow(x,op2)\n", 46 | "\n", 47 | "#sess = tf.Session()\n", 48 | "#print( sess.run(op3) )\n", 49 | "\n", 50 | "with tf.Session() as sess:\n", 51 | " print( sess.run([op3, op2 ]) )\n", 52 | "\n" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 13, 58 | "metadata": { 59 | "collapsed": false 60 | }, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "8\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "\n", 72 | "g = tf.Graph()\n", 73 | "with g.as_default():\n", 74 | " x = tf.add(3,5)\n", 75 | "\n", 76 | "sess = tf.Session( graph=g )\n", 77 | "print(sess.run(x))\n", 78 | "sess.close()\n" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 2, 84 | "metadata": { 85 | "collapsed": false 86 | }, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "5\n" 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "# tensorboard\n", 98 | "# !tensorboard --logdir=\"./graphs\"\n", 99 | "\n", 100 | "path_logs = './graphs'\n", 101 | "\n", 102 | "#\n", 103 | "# -a +-----+\n", 104 | "# | add | --\n", 105 | "# -b +-----+\n", 106 | "#\n", 107 | "\n", 108 | "a = tf.constant(2, name='a');\n", 109 | "b = tf.constant(3, name='b');\n", 110 | "x = tf.add(a,b, name='add')\n", 111 | "\n", 112 | "with tf.Session() as sess:\n", 113 | " write = tf.summary.FileWriter( path_logs, sess.graph )\n", 114 | " print( sess.run(x) )\n", 115 | " \n", 116 | "write.close()" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 6, 122 | "metadata": { 123 | "collapsed": false 124 | }, 125 | "outputs": [ 126 | { 127 | "name": "stdout", 128 | "output_type": "stream", 129 | "text": [ 130 | "[[0 2]\n", 131 | " [4 6]]\n" 132 | ] 133 | } 134 | ], 135 | "source": [ 136 | "\n", 137 | "# constant type\n", 138 | "#tf.constant(value, dtype=None, shape=None, name='Const', verify_shape=False)\n", 139 | "\n", 140 | "a = tf.constant([2, 2], name='a')\n", 141 | "b = tf.constant([[0, 1], [2, 3]], name='b')\n", 142 | "x = tf.multiply(a, b, name='dot_product')\n", 143 | "with tf.Session() as sess:\n", 144 | " print(sess.run(x))\n", 145 | "\n" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 2, 151 | "metadata": { 152 | "collapsed": false 153 | }, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "Tensor(\"y:0\", shape=(2, 3), dtype=int32)\n", 160 | "node {\n", 161 | " name: \"x\"\n", 162 | " op: \"Const\"\n", 163 | " attr {\n", 164 | " key: \"dtype\"\n", 165 | " value {\n", 166 | " type: DT_INT32\n", 167 | " }\n", 168 | " }\n", 169 | " attr {\n", 170 | " key: \"value\"\n", 171 | " value {\n", 172 | " tensor {\n", 173 | " dtype: DT_INT32\n", 174 | " tensor_shape {\n", 175 | " dim {\n", 176 | " size: 2\n", 177 | " }\n", 178 | " dim {\n", 179 | " size: 3\n", 180 | " }\n", 181 | " }\n", 182 | " int_val: 0\n", 183 | " }\n", 184 | " }\n", 185 | " }\n", 186 | "}\n", 187 | "node {\n", 188 | " name: \"y\"\n", 189 | " op: \"ZerosLike\"\n", 190 | " input: \"x\"\n", 191 | " attr {\n", 192 | " key: \"T\"\n", 193 | " value {\n", 194 | " type: DT_INT32\n", 195 | " }\n", 196 | " }\n", 197 | "}\n", 198 | "versions {\n", 199 | " producer: 17\n", 200 | "}\n", 201 | "\n", 202 | "[[0 0 0]\n", 203 | " [0 0 0]]\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "#tf.zeros(shape, dtype=tf.float32, name=None)\n", 209 | "\n", 210 | "x = tf.zeros([2, 3], tf.int32, name='x') \n", 211 | "y = tf.zeros_like(x, optimize=True, name='y')\n", 212 | "\n", 213 | "print(y)\n", 214 | "print(tf.get_default_graph().as_graph_def())\n", 215 | "\n", 216 | "with tf.Session() as sess:\n", 217 | " y = sess.run(y)\n", 218 | " print(y)" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 4, 224 | "metadata": { 225 | "collapsed": false 226 | }, 227 | "outputs": [ 228 | { 229 | "name": "stdout", 230 | "output_type": "stream", 231 | "text": [ 232 | "[ 10. 11. 12. 13.]\n", 233 | "[0 1 2 3 4]\n", 234 | "0\n", 235 | "1\n", 236 | "2\n", 237 | "3\n", 238 | "4\n" 239 | ] 240 | } 241 | ], 242 | "source": [ 243 | "\n", 244 | "with tf.Session() as sess:\n", 245 | " print(sess.run(tf.linspace(10.0, 13.0, 4)))\n", 246 | " print(sess.run(tf.range(5)))\n", 247 | " for i in np.arange(5):\n", 248 | " print(i)\n", 249 | "\n" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 6, 255 | "metadata": { 256 | "collapsed": false 257 | }, 258 | "outputs": [ 259 | { 260 | "name": "stdout", 261 | "output_type": "stream", 262 | "text": [ 263 | "[0, 1]\n" 264 | ] 265 | } 266 | ], 267 | "source": [ 268 | "t_0 = 19 \n", 269 | "x = tf.zeros_like(t_0) # ==> 0\n", 270 | "y = tf.ones_like(t_0) # ==> 1\n", 271 | "\n", 272 | "with tf.Session() as sess:\n", 273 | " print(sess.run([x, y]))" 274 | ] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": 9, 279 | "metadata": { 280 | "collapsed": false 281 | }, 282 | "outputs": [ 283 | { 284 | "name": "stdout", 285 | "output_type": "stream", 286 | "text": [ 287 | "[array([[False, False, False],\n", 288 | " [False, False, False],\n", 289 | " [False, False, False]], dtype=bool), array([[ True, True, True],\n", 290 | " [ True, True, True],\n", 291 | " [ True, True, True]], dtype=bool)]\n" 292 | ] 293 | } 294 | ], 295 | "source": [ 296 | "t_1 = ['apple', 'peach', 'banana']\n", 297 | "x = tf.zeros_like(t_1) # ==> ['' '' '']\n", 298 | "#y = tf.ones_like(t_1) # ==> TypeError: Expected string, got 1 of type 'int' instead.\n", 299 | "\n", 300 | "t_2 = [[True, False, False],\n", 301 | " [False, False, True],\n", 302 | " [False, True, False]] \n", 303 | "x = tf.zeros_like(t_2) # ==> 2x2 tensor, all elements are False\n", 304 | "y = tf.ones_like(t_2) # ==> 2x2 tensor, all elements are True\n", 305 | "with tf.Session() as sess:\n", 306 | " print(sess.run([x, y]))\n" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 2, 312 | "metadata": { 313 | "collapsed": false 314 | }, 315 | "outputs": [], 316 | "source": [ 317 | "with tf.variable_scope('meh') as scope:\n", 318 | " a = tf.get_variable('a', [10])\n", 319 | " b = tf.get_variable('b', [100])\n", 320 | "\n", 321 | "writer = tf.summary.FileWriter('./graphs', tf.get_default_graph())\n" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": 19, 327 | "metadata": { 328 | "collapsed": false 329 | }, 330 | "outputs": [ 331 | { 332 | "name": "stdout", 333 | "output_type": "stream", 334 | "text": [ 335 | "[768.0, 32.0]\n", 336 | "dz/dy = 32.0*dy/dx\n", 337 | "dz/dy = 768.0\n" 338 | ] 339 | } 340 | ], 341 | "source": [ 342 | "\n", 343 | "#y = 2*x^3\n", 344 | "#z = 3 + y^2\n", 345 | "\n", 346 | "#dz/dy = dz/dy*dy/dx\n", 347 | "#dz/dy = (2y)*(6x^2) = 12*yx^2\n", 348 | "# = 12*(2*x^3)*x^2\n", 349 | "# = 24*x^5\n", 350 | "\n", 351 | "x = tf.Variable(2.0)\n", 352 | "y = 2.0 * (x ** 3)\n", 353 | "z = 3.0 + y ** 2\n", 354 | "grad_z = tf.gradients(z, [x, y])\n", 355 | "with tf.Session() as sess:\n", 356 | " sess.run(x.initializer)\n", 357 | " print(sess.run(grad_z))\n", 358 | "\n", 359 | "x_ = 2.0\n", 360 | "print('dz/dy = {}*dy/dx'.format(4*x_**3) )\n", 361 | "print('dz/dy = {}'.format(24*x_**5) )\n" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": 25, 367 | "metadata": { 368 | "collapsed": false 369 | }, 370 | "outputs": [ 371 | { 372 | "name": "stdout", 373 | "output_type": "stream", 374 | "text": [ 375 | "Tensor(\"W/read:0\", shape=(10, 10), dtype=float32)\n", 376 | "[[-0.40563682 0.16062029 -0.71486294 0.24939488 0.04161522 0.474022\n", 377 | " 1.79872763 -0.49964508 0.16275671 -0.05247218]\n", 378 | " [ 0.74917567 -0.46427971 -0.10228646 -1.86843133 -0.9804033 0.15528598\n", 379 | " -0.81140155 0.61410964 0.74684948 0.48466322]\n", 380 | " [-0.03071053 -0.22619423 -0.19512153 -1.66248596 1.03542316 0.62709397\n", 381 | " -1.04152501 -0.07148623 -0.04174674 -0.12625839]\n", 382 | " [-1.24382877 0.52946997 -0.84973633 0.06872373 1.50142717 -0.37171298\n", 383 | " -0.01037326 -0.08588105 0.08802573 0.1017476 ]\n", 384 | " [ 0.12682524 0.64130563 0.68141538 1.28140152 -0.2666024 0.58476579\n", 385 | " -1.2755065 0.19763047 -0.69266617 0.05350485]\n", 386 | " [ 0.78631306 0.34876707 -0.12141786 -1.05309522 0.41274527 -1.4360677\n", 387 | " -0.6248036 -0.18029913 1.173823 -0.01998813]\n", 388 | " [-0.59159058 0.18904682 1.24058342 -0.20518452 -0.37875903 0.77038735\n", 389 | " 0.12067994 -1.51706517 1.31233466 -0.35130689]\n", 390 | " [-0.37181696 -0.65482563 0.49762568 -0.07201093 1.46821725 -0.53847587\n", 391 | " 0.24810202 -0.40731627 -1.76003945 0.15248844]\n", 392 | " [-0.97816843 0.9981873 0.86167103 1.19296026 0.28791907 0.81492198\n", 393 | " 0.2340035 -0.68419749 -1.5758878 1.6395185 ]\n", 394 | " [-1.86600673 -0.53029943 -1.43597567 -0.55110127 0.79476976 1.32880175\n", 395 | " 1.0182606 -0.4016479 -1.68002391 -0.33767429]]\n" 396 | ] 397 | } 398 | ], 399 | "source": [ 400 | "\n", 401 | "W = tf.Variable(tf.truncated_normal([10,10]), name='W')\n", 402 | "with tf.Session() as sess:\n", 403 | " sess.run( W.initializer )\n", 404 | " print(W)\n", 405 | " print( sess.run(W) )\n" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": 28, 411 | "metadata": { 412 | "collapsed": false 413 | }, 414 | "outputs": [ 415 | { 416 | "name": "stdout", 417 | "output_type": "stream", 418 | "text": [ 419 | "10\n", 420 | "100\n" 421 | ] 422 | } 423 | ], 424 | "source": [ 425 | "\n", 426 | "W = tf.Variable( 10, name='W' )\n", 427 | "op = W.assign( 100 )\n", 428 | "with tf.Session() as sess:\n", 429 | " sess.run(W.initializer)\n", 430 | " print(W.eval()) # << 10\n", 431 | " sess.run(op)\n", 432 | " print(W.eval()) #<< 100\n" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 30, 438 | "metadata": { 439 | "collapsed": false 440 | }, 441 | "outputs": [ 442 | { 443 | "name": "stdout", 444 | "output_type": "stream", 445 | "text": [ 446 | "4\n", 447 | "8\n", 448 | "16\n" 449 | ] 450 | } 451 | ], 452 | "source": [ 453 | "\n", 454 | "a = tf.Variable( 2, name='a' )\n", 455 | "op_times_two = a.assign( 2*a )\n", 456 | "with tf.Session() as sess:\n", 457 | " sess.run(a.initializer)\n", 458 | " sess.run(op_times_two); print(a.eval())\n", 459 | " sess.run(op_times_two); print(a.eval())\n", 460 | " sess.run(op_times_two); print(a.eval())\n", 461 | "\n" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": 3, 467 | "metadata": { 468 | "collapsed": false 469 | }, 470 | "outputs": [ 471 | { 472 | "name": "stdout", 473 | "output_type": "stream", 474 | "text": [ 475 | "Tensor(\"a_1:0\", shape=(3,), dtype=float32)\n", 476 | "Tensor(\"b_1:0\", shape=(3,), dtype=float32)\n", 477 | "Tensor(\"add_1:0\", shape=(3,), dtype=float32)\n", 478 | "[ 2. 4. 6.]\n" 479 | ] 480 | } 481 | ], 482 | "source": [ 483 | "# placeholder\n", 484 | "\n", 485 | "path_logs = './graphs'\n", 486 | "\n", 487 | "a = tf.placeholder( dtype=tf.float32, shape=(3), name='a' )\n", 488 | "print(a)\n", 489 | "\n", 490 | "b = tf.constant([1.0,2.0,3.0], dtype=tf.float32, name='b')\n", 491 | "print(b)\n", 492 | "\n", 493 | "z = a+b; # tf.add(a,b)\n", 494 | "print(z)\n", 495 | "\n", 496 | "\n", 497 | "with tf.Session() as sess:\n", 498 | " \n", 499 | " writer = tf.summary.FileWriter( path_logs, sess.graph )\n", 500 | " feed_dict = {a:[1,2,3]};\n", 501 | " print(sess.run(z, feed_dict=feed_dict))\n", 502 | "\n", 503 | "writer.close()\n" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": 9, 509 | "metadata": { 510 | "collapsed": false 511 | }, 512 | "outputs": [ 513 | { 514 | "name": "stdout", 515 | "output_type": "stream", 516 | "text": [ 517 | "2.5\n", 518 | "2.5\n" 519 | ] 520 | } 521 | ], 522 | "source": [ 523 | "\n", 524 | "x = tf.constant( [1,2,3,4], dtype=tf.float32 )\n", 525 | "y = tf.reduce_mean(x)\n", 526 | "with tf.Session() as sess:\n", 527 | " print( sess.run(y) )\n", 528 | "print( np.mean( np.array([1,2,3,4]) ) )\n" 529 | ] 530 | } 531 | ], 532 | "metadata": { 533 | "kernelspec": { 534 | "display_name": "Python [conda root]", 535 | "language": "python", 536 | "name": "conda-root-py" 537 | }, 538 | "language_info": { 539 | "codemirror_mode": { 540 | "name": "ipython", 541 | "version": 3 542 | }, 543 | "file_extension": ".py", 544 | "mimetype": "text/x-python", 545 | "name": "python", 546 | "nbconvert_exporter": "python", 547 | "pygments_lexer": "ipython3", 548 | "version": "3.5.2" 549 | } 550 | }, 551 | "nbformat": 4, 552 | "nbformat_minor": 2 553 | } 554 | -------------------------------------------------------------------------------- /basic/ibackprop.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Notebook for deep learning (basic concepts)" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook we implemented de basic neural net framework. The idea is describe the basic concepts of neural net definition and the way to implement backpropagation derivation.\n", 15 | "\n", 16 | "Some usefull references and tutorials are:\n", 17 | "\n", 18 | "- https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/\n", 19 | "- https://www.youtube.com/watch?v=GlcnxUlrtek\n", 20 | "- http://neuralnetworksanddeeplearning.com/chap3.html\n", 21 | "- https://www.youtube.com/watch?v=-YRB0eFxeQA&list=PLE6Wd9FR--EfW8dtjAuPoTuPcqmOV53Fu&index=10\n", 22 | "- https://www.youtube.com/watch?v=PpFTODTztsU&list=PL6Xpj9I5qXYEcOhn7TqghAJ6NAPrNmUBH&index=8\n", 23 | "- https://stats.stackexchange.com/questions/235528/backpropagation-with-softmax-cross-entropy\n", 24 | "\n", 25 | "\n", 26 | "Currently we are using Caffe:\n", 27 | "- http://christopher5106.github.io/deep/learning/2015/09/04/Deep-learning-tutorial-on-Caffe-Technology.html\n", 28 | "- http://www.andrewjanowczyk.com/real-time-data-augmentation-using-nvidia-digits-python-layer/\n", 29 | "\n", 30 | "\n", 31 | "\n" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 11, 37 | "metadata": { 38 | "collapsed": true 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "import numpy as np\n", 43 | "import numpy.matlib\n", 44 | "import matplotlib.pyplot as plt\n", 45 | "%matplotlib inline" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "\n", 53 | "Neural network could be seen as a function J(x), an example of a simple net is:\n", 54 | "\n", 55 | " J(x) = 1/2(y - sigm( W2*( W1*x + b1) +b2 ) )\n", 56 | "\n", 57 | "then can be defined:\n", 58 | "\n", 59 | " z_1 = x\n", 60 | " z_2 = W1*z_1 + b1\n", 61 | " z_3 = W2*z_2 + b2\n", 62 | " z_4 = sigm( z_3 )\n", 63 | " z_5 = 1/2(y-z_4)\n", 64 | "\n", 65 | "Train the net means obtain parameters {w1, w2} that minimize the function J(x).\n", 66 | "Descend gradient iterative solution update net weights such that:\n", 67 | "\n", 68 | " min_{W_i} J(x)\n", 69 | " \n", 70 | "Weights update could be done as:\n", 71 | "\n", 72 | " {W_i}^t = {W_i}^t-1 + delta * grad J(x)\n", 73 | " \n", 74 | "\n", 75 | "\n" 76 | ] 77 | }, 78 | { 79 | "cell_type": "markdown", 80 | "metadata": {}, 81 | "source": [ 82 | "## Caffe layers interface\n", 83 | "\n", 84 | "Caffe has a layer interface with functions to compute forward() and backward(). Also, the interface have two methods (not covered in this example) setup() and reshape() for automatically set the parameters and quantity of neurons in the layer depending on input data size. Caffe currently allows python layers implementation." 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 4, 90 | "metadata": { 91 | "collapsed": true 92 | }, 93 | "outputs": [], 94 | "source": [ 95 | "# Example of generic Layer class \n", 96 | "# + ---- -+\n", 97 | "# z ---> | layer | -- f(z) -->\n", 98 | "# +-------+ <-------- (dz/dx)_i+1 ------\n", 99 | "# | |\n", 100 | "# df(z)/dx_i df(z)/dW_i\n", 101 | "\n", 102 | "class layer(object):\n", 103 | " \n", 104 | " def forward(self, param):\n", 105 | " raise NotImplementedError\n", 106 | "\n", 107 | " def backward(self, param):\n", 108 | " raise NotImplementedError\n" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "Lets implement a layer for inner product. For this only forward() and backward() methods need to be defined. Backward method referes to layer derivative. Because this layer depends on input vector x and weights W, derivative respect both must be calculated. 'dzdx' parameter refers to propagated error." 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 5, 121 | "metadata": { 122 | "collapsed": true 123 | }, 124 | "outputs": [], 125 | "source": [ 126 | "# Inner layer class\n", 127 | "# Ecuation:\n", 128 | "# z = W*x + b \n", 129 | "# dz/dx_i = W*(dz/dx_{i+1}) \n", 130 | "# dz/dw_i = x*(dz/dx_{i+1}) \n", 131 | "#\n", 132 | "class inner(layer):\n", 133 | "\n", 134 | " # z = W*x + b \n", 135 | " def forward(self, param): \n", 136 | " return np.dot(param['w'],param['x']);\n", 137 | "\n", 138 | " # dz/dx_i = W*(dz/dx_{i+1}) \n", 139 | " # dz/dw_i = x*(dz/dx_{i+1})\n", 140 | " def backward(self, param):\n", 141 | " return np.dot(param['w'].T, param['dzdx']), np.dot(param['dzdx'],param['x'].T); \n", 142 | " " 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "metadata": {}, 148 | "source": [ 149 | "And for sigmoid function:" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": { 156 | "collapsed": true 157 | }, 158 | "outputs": [], 159 | "source": [ 160 | "# Sigmoid layer class\n", 161 | "# Ecuation:\n", 162 | "# z = sigm(x) = 1/(1+exp(-x))\n", 163 | "# dz/dx = sigm(x)(1-sigm(x)) *(dz/dx_i+1) \n", 164 | "#\n", 165 | "class sigm(layer):\n", 166 | " \n", 167 | " def forward(self, param): \n", 168 | " return self._sigm(param['x']);\n", 169 | "\n", 170 | " def backward(self, param):\n", 171 | " return self._sigm(param['x'])*(1.0-self._sigm(param['x']))*param['dzdx']; \n", 172 | "\n", 173 | " def _sigm(self, x): return 1.0/(1.0 + np.exp(-x));\n", 174 | " " 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 10, 180 | "metadata": {}, 181 | "outputs": [ 182 | { 183 | "data": { 184 | "text/plain": [ 185 | "" 186 | ] 187 | }, 188 | "execution_count": 10, 189 | "metadata": {}, 190 | "output_type": "execute_result" 191 | }, 192 | { 193 | "data": { 194 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VFX+x/H3Sa8kkJAASSChhJZQQ7CAhiILgiA2sLOu\n4qpYVmTVtSyrsoht9bcqymJjVbCtCIqiKJEiSAm9xdBDgBRISE8mc35/3BBCpAwwkzvl+3qeeabd\nmfkeMnxyc+655yitNUIIIdyLl9kFCCGEsD8JdyGEcEMS7kII4YYk3IUQwg1JuAshhBuScBdCCDck\n4S6EEG5Iwl0IIdyQhLsQQrghH7M+ODIyUsfHx5v18eettLSU4OBgs8toVNJmz+BpbXbV9q5duzZf\na938bNuZFu7x8fGsWbPGrI8/b+np6aSlpZldRqOSNnsGT2uzq7ZXKbXXlu2kW0YIIdyQhLsQQrgh\nCXchhHBDZ+1zV0q9C4wAcrXWSad4XgGvAVcCZcA4rXXG+RRTXV1NdnY2FRUV5/PyRhEWFsa2bdvM\nLqNRhYWFsXv3bmJjY/H19TW7HCGEDWw5oPo+8Dow6zTPDwM61F76AtNrr89ZdnY2oaGhxMfHY/zO\ncD7FxcWEhoaaXUajOnbsGFVVVWRnZ5OQkGB2OUIIG5y1W0ZrvQQ4coZNRgGztGElEK6Uank+xVRU\nVBAREeG0we6plFJEREQ49V9UQoiT2aPPPQbYX+9+du1j50WC3TnJz0UI19Ko49yVUuOB8QDR0dGk\np6ef9HxYWBjFxcWNWdI5q6mpcfoa7e14mysqKn73M3NXJSUlHtPW4zytzfZor8WqKbdAabWmzKIp\nq9aUVkNZtaa8RlNhgQqLpqLGuK6sve4c4c3V7f3s05DTsEe4HwDi6t2PrX3sd7TWM4AZACkpKbrh\nCQTbtm1zyv7sO++8k4cffpguXbo4rM/9yiuv5OOPPyY8PPykxydPnkxISAiPPPKI3T/TVsfbHBAQ\nQM+ePU2rozG56gkuF8LT2nyq9pZWWigoqSKvpJKCkkryS6pqryvJLzVuF5ZVU1RezbHyakqras7r\ns9vFRpGW1tsOrTg9e4T7PGCCUmoOxoHUIq31QTu8r9OYOXOmwz9jwYIFDv8MITxZpaWG7KPlHCys\nIKeonBVZVXxXsJGcogoOFpZzsKiCkkrLOb2nUtAkwJewQF+aBPoY17X3Q/x9CPb3qbsO9vcm2M+4\n3SIswEGtPMGWoZCzgTQgUimVDfwd8AXQWr8FLMAYBpmFMRTyj44qtjGUlpZyww03kJ2dTU1NDU89\n9RTTp0/npZdeIiUlhVmzZvHaa68RHh5O9+7d8ff35/XXX2fcuHEEBgaybt06cnNzeffdd5k1axYr\nVqygb9++vP/++wDMnj2bf/7zn2itGT58ONOmTQNOTMcQGRnJlClT+OCDD4iKiiIuLo7evR37G14I\nd2GpsbKnoIw9+aXsKai95Jexp6CUnMJyrLrhK/afdM/fx4vIEH8iQ/2JDPYjIsSPyBB/IkL8iQzx\nIyLYn6bBtQEe5EuInw9eXs55POqs4a61vvEsz2vgPrtVVCv+sW/s/ZYA7Hl++Bmf/+6772jVqhXf\nfGN8flFREdOnTwcgJyeHF154gXXr1hEaGsrAgQPp3r173WuPHj3KihUrmDdvHiNHjmT58uXMnDmT\nPn36sH79eqKionj00UdZu3YtTZs2ZciQIcydO5err7667j3Wrl3LnDlzWL9+PRaLhV69ekm4C9GA\n1prDxyrZfugYOw4Vs732sjO3hKoa6ylf46UgtmkgMeGBtAoPpLool77dOhITHkDLsEBahQXSJNDH\nbQYPmDZxmLNKTk5m4sSJPProo4wYMYL+/fvXPbdq1SouvfRSmjVrBsD1119PZmZm3fNXXXUVSimS\nk5OJjo4mOTkZgK5du7Jnzx727t1LWloazZsbE7rdfPPNLFmy5KRwX7p0KaNHjyYoKAiAkSNHOrzN\nQji7wrIqNmQXsX5fIRuyC9mwv5CC0qpTbhsTHkjb5sHERwQTHxlMfEQQ8ZHBxDUNws/nxADB9PR0\n0i5q01hNaHROG+5n28N2lMTERDIyMliwYAFPPvkkgwYNsvm1/v7+AHh5edXdPn7fYrHI2Z1C2OhA\nYTkrdxawYlcBa/ceZXd+6e+2CQv0pWOLUDq1CK29bkJidAihAfL/DJw43M2Sk5NDs2bNuOWWWwgP\nDz/pYGqfPn148MEHOXr0KKGhoXzxxRd1e+e2SE1N5YEHHiA/P5+mTZsye/Zs7r///pO2ueyyyxg3\nbhyPP/44FouF+fPnc/fdd9utfUI4o6OlVSz5LY9fsoxA33ek7KTn/X28SIoJo0dcON3jwukRG05c\ns0C36UJxBAn3BjZt2sSkSZPw8vLC19eX6dOn1w1DjImJYeLEiaSmptKsWTM6depEWFiYze/dsmVL\nnn/+eQYMGFB3QHXUqFEnbdOrVy/GjBlD9+7diYqKok+fPnZtnxDOQGtN5uESftx+mJ+25ZKx7+hJ\nBztD/X1ITWjGxe0i6JsQQaeWofh6yzyH50IZx0MbX0pKim64WMe2bdvo3LmzKfXY6uDBg7Rs2RKL\nxcLo0aO54447GD16tNllOdTxce6u8POxF08b8w2Ob7PWms0HjjF/Yw4LNh0k+2h53XO+3oq+CRH0\n7xDJxe0i6NoqDG8Hj0Jx1Z+xUmqt1jrlbNvJnvs5mjp1KkuWLKGiooIhQ4acdDBUCPF7mYeLmbc+\nh/kbc9hbcKK7JTLEjwEdoxjUOYpL20dKX7mdSbifoylTpjjlWbRCOJOSSgvzN+QwZ/V+NuwvrHs8\nMsSfEd1aMqJbS3q1buq0Y8TdgYS7EMJuNh8o4sOVe5m/Iafu1PzQAB+GJ7dkZPdW9G0b4fDuFmGQ\ncBdCXBCrVfPj9lxmLt3Fr7tPzA6eGt+MMX3iuDK5JYF+3iZW6Jkk3IUQ56XSUsOna7J5d9nuunHo\nof4+3NAnjhtTW9M+KsTkCj2bhLsQ4pxUWmr4dPV+3li8k0PHjAVcYsIDuaNfAjekxMqBUSchA0dt\ncOedd7J161aHfsaVV15JYWHh7x6fPHkyL730EgDjxo0jISGBHj160KtXL1asWHHK93rrrbeYNet0\nqyIKcX6qa6z8d+Ve0l5M56mvtnDoWAWdWoTy+k09+XlSGn/qlyDB7kRkz90GzjTl74svvsh1113H\n999/z913383GjRtPet5isfDnP//ZESUKD6W1ZtG2XKYu2Mau2u6XjtGhPDS4A3/o2kJGvDgp2XNv\noLS0lOHDh9O9e3eSkpL45JNPSEtL4/gJV7NmzSIxMZHU1FTuuusuJkyYABh71ffccw8XXXQRbdu2\nJT09nTvuuIPOnTszbty4uvefPXs2ycnJJCUl8eijj9Y9Hh8fT35+PmAMt0xMTKRfv37s2LHjlHVe\ndtllZGVlAZCWlsZDDz1ESkoKr7322kl7+2lpafzlL38hJSWFzp07s3r1aq655ho6dOjAk08+Wfd+\nH374IampqfTo0YO7776bmprzW4RAuJfNB4q46T+/ctesNezKLyUhMpg3burFtw/2Z1hySwl2J+a8\ne+6TbT+t/9zet+iMT7vKlL/z588/aV6bqqqqul9AkydPPmlbPz8/1qxZw2uvvcaoUaNYu3YtzZo1\no127dvzlL38hNzeXTz75hOXLl+Pr68u9997LRx99xG233XZu/7bCbRSVVTNt4XZmr9qH1hAe5MuD\ngzpwc982J82sKJyX84a7SZx9yt9Jkybx3HPP0bx5c9555526x8eMGXPaNh1/j+TkZLp27UrLli0B\naNu2Lfv372fZsmWsXbu2bh6b8vJyoqKizvFfTrgDrTVfbzzIP+ZvJb+kEl9vxe0Xx3P/wA6EBUl/\nuitx3nA/yx62ozj7lL/H+9wbCg4OPu+6tNbcfvvtTJ069YLrE64rv9zKuPdW83NmHgB94pvyz9HJ\ndIiWM7Jdkfx91UBOTg5BQUHccsstTJo0iYyMjLrn+vTpw/Llyzl69CgWi4UvvvjinN47NTWVn3/+\nmfz8fGpqapg9ezaXX375SdtcdtllzJ07l/LycoqLi5k/f75d2nUmgwYN4vPPPyc3NxeAI0eOsHfv\nXod/rnAOWms+X5vNk8vK+TkzjyYBPky9JplPxl8swe7CnHfP3SSeOOVvly5deO655xgyZAhWqxVf\nX1/eeOMN2rRx31VqhKGgpJK/fbmJhVsOA/CHrtE8e3USUaGOX8BZOJZM+XuOZMpf5/752IurTgd7\nLpZk5vHwpxvIL6kkxN+HsYlePHHTYI9ZAMNVf8a2Tvkr3TLnaOrUqfTo0YOkpCQSEhJkyl/hcmqs\nmld+yOT291aRX1JJ34RmfPtgf/rF+HpMsHsC6ZY5RzLlr3Bl+SWVPDRnPcuy8lEKHr4ikQkD2uPl\npdhpdnHCrpwu3LXWsvfghMzqvhP2s27fUe75MINDxyqICPbjtbE96dch0uyyhIM4VbgHBARQUFBA\nRESEBLwT0VpTUFBAQIAcZHNVX60/wKTPN1JlsdInvin/vrEXLcLk5+nOnCrcY2Njyc7OJi8vz+xS\nTquiosLjQq6iooLw8HBiY2PNLkWcI6tV8+qiTP7vJ2Oqipv7tmbyyK6y2LQHcKpw9/X1JSEhwewy\nzig9PZ2ePXuaXUaj8sQ2u4PyqhomfraeBZsO4aXg6RFduP2SePmr2EM4VbgLIeyjqKyaOz5Yzdq9\nRwn19+HfN/UkraNMKeFJJNyFcDOHiiq4/d1V7DhcTKuwAD64I1XONPVAEu5CuJFdeSXc+s4qDhSW\n0z4qhFl3pNIqPNDssoQJJNyFcBNbcoq47Z1VFJRW0SMunPfG9aFpsJ/ZZQmTSLgL4QY2Hyji5pm/\nUlReTf8Okbx1S2+C/eW/tyezaTyUUmqoUmqHUipLKfXYKZ5vrZRarJRap5TaqJS60v6lCiFOpX6w\nD+4cxczbUyTYxdnDXSnlDbwBDAO6ADcqpbo02OxJ4FOtdU9gLPCmvQsVQvzeycEezZs398bfx9vs\nsoQTsGXPPRXI0lrv0lpXAXOAUQ220UCT2tthQI79ShRCnMqWnBPBfkWXaN68uZcsgSfq2PK3Wwyw\nv979bKBvg20mA98rpe4HgoHBdqlOCHFKu/NLuf3dVXXB/sZNEuziZGedz10pdR0wVGt9Z+39W4G+\nWusJ9bZ5uPa9XlZKXQy8AyRpra0N3ms8MB4gOjq695w5c+zamMZQUlJCSEiI2WU0KmmzczlSYWXK\nygoKKjRdI7x4qHcAvl4XftapM7fZEVy1vQMGDLBpPndb9twPAHH17sfWPlbfn4ChAFrrFUqpACAS\nyK2/kdZ6BjADjMU6XHGifFed4P9CSJudx5HSKm54ewUFFZqercP58E997Xbw1Fnb7Cju3l5b/o5b\nDXRQSiUopfwwDpjOa7DNPmAQgFKqMxAAOO/sX0K4oNJKC398bxVZuSV0jA7lvXF9ZFSMOK2zhrvW\n2gJMABYC2zBGxWxRSj2jlBpZu9lE4C6l1AZgNjBOywTgQtiNpcbKhI8z2JBdRFyzQGb9KZXwIDlB\nSZyeTb/2tdYLgAUNHnu63u2twKX2LU0IAcZ8+v+Yv5XFO/JoGuTLrDv6Et3Es6adFudODq8L4eTe\nWbab/67ci5+3FzNuSyEhMtjskoQLkHAXwol9t/kQUxZsA+DF67vRJ76ZyRUJVyHhLoST2pRdxEOf\nrENreGRIIqN6xJhdknAhEu5COKH8kkru/u8aKqqtXNc7lvsGtDe7JOFiJNyFcDLVNVbu/SiDnKIK\nerYOZ8roJFkaT5wzCXchnMxzX29l1e4jRIX689YtMhGYOD8S7kI4kU/X7OeDFcbImLdu7S1DHsV5\nk3AXwkms31/Ik19uBuCZUV3p1bqpyRUJVybhLoQTKCyr4r6PMqiqsXJz39aMTW1tdknCxUm4C2Ey\nrTWPfLaBA4XldI8L5+9XdTW7JOEGJNyFMNnMpbtZtC2XJgE+vH5jT5mXXdiFfIuEMNHavUeZ9t12\nAF66vjtxzYJMrki4Cwl3IUxSWFbF/R9nYLFq/tQvgSFdW5hdknAjEu5CmEBrzcRPN5BTVEGPuHAe\nHdrJ7JKEm5FwF8IEH/yyhx+35xIW6MvrN0k/u7A/+UYJ0cgyDxcz9Vujn33atcnENpV+dmF/Eu5C\nNKJKSw0PzllPpcXKDSmxDE1qaXZJwk1JuAvRiF7+PpNtB4/RJiJIxrMLh5JwF6KR/JKVz3+W7sLb\nS/GvMT1kcWvhUBLuQjSCorJqJn62Aa3h/oHtZd4Y4XAS7kI4mNaav83dxMHa+dknyMIbohFIuAvh\nYN9sOsg3Gw8S5OfNq2N64OMt/+2E48m3TAgHyi+p5OmvtgDwxPDOtIkINrki4Skk3IVwEK01T83d\nzJHSKvq1j+QmmcZXNCIJdyEc5JtNB/l28yGC/bx5/tpkWQdVNCoJdyEcoH53zN+Gd5azUEWjk3AX\nws6kO0Y4Awl3IexMumOEM5BwF8KOpDtGOAsJdyHs6Jn5W6U7RjgFCXch7GRJZh7zNuQQ4OvF1Guk\nO0aYy6ZwV0oNVUrtUEplKaUeO802NyiltiqltiilPrZvmUI4t4rqGp76ajMADw5KlLVQhenOOi2d\nUsobeAO4AsgGViul5mmtt9bbpgPwOHCp1vqoUirKUQUL4YzeTN/J3oIyEqNDuLN/gtnlCGHTnnsq\nkKW13qW1rgLmAKMabHMX8IbW+iiA1jrXvmUK4bx25pXwVvpOAKaMTsZX5o4RTsCWb2EMsL/e/eza\nx+pLBBKVUsuVUiuVUkPtVaAQzkxrzZNfbqaqxsqYlDj6xDczuyQhABu6Zc7hfToAaUAssEQplay1\nLqy/kVJqPDAeIDo6mvT0dDt9fOMpKSlxybovhLT59H7JsbBiVyUhvtC/SYFL/zt52s/Z3dtrS7gf\nAOLq3Y+tfay+bOBXrXU1sFsplYkR9qvrb6S1ngHMAEhJSdFpaWnnWbZ50tPTccW6L4S0+dQKy6qY\n+PLPAPx9VDdGpMSdcXtn52k/Z3dvry3dMquBDkqpBKWUHzAWmNdgm7kYe+0opSIxuml22bFOIZzO\ntO92UFBaRd+EZlzXO9bscoQ4yVnDXWttASYAC4FtwKda6y1KqWeUUiNrN1sIFCiltgKLgUla6wJH\nFS2E2dbuPcLsVfvw9VZMGZ0kY9qF07Gpz11rvQBY0OCxp+vd1sDDtRch3Fp1jZUnvjTGtI+/rC3t\no0JNrkiI35MxW0Kco/eW72b7oWJaNwvi/oEdzC5HiFOScBfiHGQfLeNfP/wGwDOjuhLg621yRUKc\nmoS7EOdg8rytlFfXMLxbS9I6yonYwnlJuAtho++3HGLRtsOE+Pvw9IguZpcjxBlJuAthg9JKC5Pn\nGfO0PzIkkegmASZXJMSZSbgLYYNXF2WSU1RBckwYt14cb3Y5QpyVhLsQZ7E15xjvLt+Dl4J/jk7G\n20vGtAvnJ+EuxBlYrZon5m6ixqq57eJ4kmPDzC5JCJtIuAtxBrNX72PdvkKiQv2ZOCTR7HKEsJmE\nuxCnkVdcybRvtwPw96u6Ehrga3JFQthOwl2I05jyzVaOVVi4PLE5Vya3MLscIc6JhLsQp7C1oIa5\n63Pw9/Hi2VEyMZhwPRLuQjRQUV3DB1sqAXhgUAdaR8hi18L1SLgL0cBbP+/kcJmmfVQId/Vva3Y5\nQpwXCXch6tmdX8qbi2sXu746CT8f+S8iXJN8c4WopbXmybmbqKqx0i/Gh75tI8wuSYjzZq8FsoVw\nefM25LA8q4DwIF/GdJRhj8K1yZ67EEBRWTXPfr0VgL8N60yon4yOEa5Nwl0I4IWF28kvqaJPfFNZ\n7Fq4BQl34fEy9h3l41X78PFSTBmdjJdMDCbcgIS78GiW2sWutYa7LmtLYrQsdi3cg4S78Gjv/7KH\nbQePEds0kAdksWvhRiTchcfKKSznlR8yAWOx60A/WexauA8Jd+Gx/jF/C2VVNQxLasHATtFmlyOE\nXUm4C4+0aOthFm45TLCfN09fJYtdC/cj4S48TlmVhb/XLnb98JCOtAwLNLkiIexPwl14nFcX/caB\nwnK6tmrC7Re3MbscIRxCwl14lC05RbyzbDdeCqZek4yPt/wXEO5JvtnCY9RYNX/7cnPdYtfdYsPN\nLkkIh5FwFx7jo1/3smF/IS2aBMhi18LtSbgLj3D4WAUvfLcDgMkjZbFr4f5sCnel1FCl1A6lVJZS\n6rEzbHetUkorpVLsV6IQF27yvC2UVFoY3DmaP3SVMe3C/Z013JVS3sAbwDCgC3CjUup3A4OVUqHA\ng8Cv9i5SiAuxaOthvt18iCA/b/4xqqssdi08gi177qlAltZ6l9a6CpgDjDrFds8C04AKO9YnxAUp\nraw3pv2KRGLCZUy78Ay2hHsMsL/e/ezax+oopXoBcVrrb+xYmxAX7NVFmRwoLCcppgnjLok3uxwh\nGs0FL7OnlPICXgHG2bDteGA8QHR0NOnp6Rf68Y2upKTEJeu+EK7a5r3HanhnRQUKuK51FcuWLrH5\nta7a5gvhaW129/baEu4HgLh692NrHzsuFEgC0mv7MlsA85RSI7XWa+q/kdZ6BjADICUlRaelpZ1/\n5SZJT0/HFeu+EK7YZkuNlVem/4JVV/DHS+MZd1XXc3q9K7b5Qnlam929vbZ0y6wGOiilEpRSfsBY\nYN7xJ7XWRVrrSK11vNY6HlgJ/C7YhWhM7yzbzcbsIlqFBTBxSEezyxGi0Z013LXWFmACsBDYBnyq\ntd6ilHpGKTXS0QUKca525ZXUzdP+z2uSCfG/4N5HIVyOTd96rfUCYEGDx54+zbZpF16WEOfHatU8\n9sUmKi1WrukVQ1rHKLNLEsIUcoaqcCsf/rqXVXuOEBniz9MjZJ524bkk3IXbyD5axrRvtwPw7Kiu\nhAf5mVyREOaRcBduQWvN4//bRGntsnnDkluaXZIQppJwF27h87XZLP0tn7BAX/4x6tyGPQrhjiTc\nhcvLPVbBs19vBeDpEV2ICg0wuSIhzCfhLlya1ppHv9jIsQoLlyc255peMWd/kRAeQMJduLQ5q/ez\neEceTQJ8mHZtN5nxUYhaEu7CZe0rKOO52u6YZ69OokWYdMcIcZyEu3BJNVbNI59toLSqhuHJLRnZ\nvZXZJQnhVCTchUt6d9luVu05QvNQf569Okm6Y4RoQMJduJzMw8W8uNBYD3Xatck0C5aTlYRoSMJd\nuJQqi5WHP11PVY2VG1PjGNhJ1kMV4lQk3IVLefn7HWw+cIy4ZoE8MVzmjhHidCTchctYkpnH20t2\n4e2leHVMT5nKV4gzkHAXLiGvuJKHP90AGAtd927T1OSKhHBuEu7C6VmtmomfbSC/pJKL20bw58vb\nmV2SEE5Pwl04vXeW7WZJZh5Ng3z515geeHvJsEchzkbCXTi1TdlFvLDQmKP9xeu6y1moQthIwl04\nraKyau77OIPqGs24S+IZ3EWGPQphKwl34ZSMfvb17DtSRlJMEx4b1snskoRwKRLuwilN/3kni7bl\nEhboy/SbexPg6212SUK4FAl34XSWZ+Xz8vfG9AKvjulBXLMgkysSwvVIuAuncrConAdmr8Oq4YGB\n7RnQKcrskoRwSRLuwmlUWmq476MMCkqr6N8hkgcHJ5pdkhAuS87fFk5Ba80TX24mY18hrcICeG1s\nT8eNZ7fWQN4OyM+EIzvhyG4oK4DyQqgoAjQppeWQGQ5BERDcHEJbQmQiNO9oXPyCHVObEHYi4S6c\nwsylu/l8bTaBvt7MuC3FvtP4Wmsgew1kLYL9K+FABlSVnPElIQClp3lSeUGLbtDmEmhzKbRNA/8Q\n+9UrhB1IuAvTLd6ey9RvtwHw8g3dSYoJu/A3rbHArsWw6XP47XsoP3Ly8+GtIaorRLSDZm0hJAoC\nwiGgCShv1qxeRUqPZGOPvjQPivaf2NvPz4SD643LyjfB2x/aDYBOw6HLKAiwQ/1CXCAJd2GqrNzi\nugOoDw3uwJXJLS/sDXO3w7r/wsZPoTT3xONNEyBxKMT3g9g+EHrmE6JKQvMhrs+pn6wqhezVsHcF\n7PzR+Ksg8zvjsuCv0GUk9LwF2vQDLzmsJcwh4S5Mk19SyR3vr6G40sLw5JY8MLDD+b2R1Qo7f4KV\nbxjXx0W0h25jjbCNTAR7LcXnF2x0xbRNgwGPQ/Eh2PEtbP4C9iyFjZ8Yl8hEuOhe6D4WfAPt89lC\n2EjCXZiirMrCn95fXXcG6ovXd8PrXA+g1lhg02ew7BWjqwTANwi63QA9b4WY3vYL9DMJbQEpfzQu\nR3bDhtmw7kOjpq8fgp+eg9S7oO/dEChTFYvGIeEuGp2lxsqEj9exIbuI2KaBvDuuD0F+5/BVtNYY\nof7zC8ZoF4AmMZA6HnrdBkHNHFO4LZolwIC/wWWTYMtcWPFvOLgB0qfCijfg4vvgonukX144nIS7\naFRaa576ajM/bc+laZAvH9yRSlSojTM9ag1b58JPU6DgN+Oxpglw+aOQfB14+zqu8HPl7Qvdrjfq\n2rMMlr4Eu9KNkF85HS59APreA35y9q1wDJuO9iilhiqldiilspRSj53i+YeVUluVUhuVUj8qpdrY\nv1ThDv7vxyxmr9qPv48XM29PoV1zG4cQZq+Bd/8An40zgr1pPIx6EyasgR43Olew16cUJPSH276C\ncd8YQycrCuHHZ+D1FNjwiXHMQAg7O2u4K6W8gTeAYUAX4EalVMOVidcBKVrrbsDnwAv2LlS4vneX\n7eZfizJRCl4b25PebWzoPincD1/cCTMHwf5fjROKRvzLCPWeN4O3C/3xGd/PCPhb5xrj5I8dgC/H\nwzuDYd+vZlcn3Iwte+6pQJbWepfWugqYA4yqv4HWerHWuqz27kog1r5lClc3Z9U+nvl6KwBTRycz\nNKnFmV9QXQGLpxp7t5s+M8aS958I92dAyh3Ou6d+NkoZY+LHp8OoNyAkGg6shXeHwGd/hGM5Zlco\n3ITSWp95A6WuA4Zqre+svX8r0FdrPeE0278OHNJaP3eK58YD4wGio6N7z5kz5wLLb3wlJSWEhHjW\n2YgX2uaIQhd+AAASJElEQVQVORZmbKxEAzd38uOK+DMHc/jRjSRmTieo3Ai6w1GXsavtrVQGNN4k\nYo31c/a2lBO3/3/E7Z+Lt7UKi3cAe+Jv4kDMCLRX405z7GnfbVdt74ABA9ZqrVPOtp1d/6ZVSt0C\npACXn+p5rfUMYAZASkqKTktLs+fHN4r09HRcse4LcSFt/m7zIWZ+n4EGJv2hI/cNaH/6jUvy4Psn\njDHiAM07wYh/Ed3mEhp7DabG/TkPg8Kn4LvH8dn+Ne13vkv7klUw/BVo3beRavC877a7t9eWbpkD\nQFy9+7G1j51EKTUYeAIYqbWutE95wpV9s/EgEz7OoMaqmTCg/emD3WqFte8bXTAbPwGfABj0NNy9\n1Ji/xROEt4axH8GNnxi3D282umq+mgBlR87+eiEasCXcVwMdlFIJSik/YCwwr/4GSqmewNsYwZ57\nivcQHubLddncPzsDi1Vz9+VtmTjkNNP3Ht4K7w2D+Q8ao0jaD4Z7Vxr96z52nDzMVXQcCvf+Cv0f\nAS9fYyqFf/eGjFkyqkack7OGu9baAkwAFgLbgE+11luUUs8opUbWbvYixkR6nyml1iul5p3m7YQH\nmLNqHw9/uqFuvpjHhnZCNTxTtKoMfvg7vN3fmKkxJBquew9u/tw4EciT+QXBoKfg3hWQcJkx6dm8\n++H94ZC7zezqhIuwqc9da70AWNDgsafr3R5s57qEi3pv+W7+Md8YFfPo0E7ck9bu9xtlfg8LJkLh\nPkBBn7uMMJOzNk8W2QFum2fMbLnwcdj3C7zVDy55wDgDVk6AEmcgU9YJu7BaNVO/3VYX7E+P6PL7\nYD92ED69DT6+3gj2Fslw548w/CUJ9tNRyjjTdcJqYwiotcaYS+fNi+C3H8yuTjgxCXdxwSotNfzl\n0/W8/fMufLwUL13fnTv61etasdbAr2/D631g61fgGwxDpsBd6RDb27S6XUpgU+PkrT/9ANFJULgX\nProOPr3d+KUpRAMudHqfcEZF5dX8+b9rWbGrgGA/b6bf0pvLEpuf2OBABnz9F2NhC4COw2HYNAiP\nO/UbijOL62OcALVyujFPzda5kPWj0a3V505o5LHxwnnJnrs4b7vzS7l2+i+s2FVA81B/Prn74hPB\nXlEECybBfwYawd4kFsZ+DDd+LMF+obx9jYnH7lsFHa+EqmL49q/GFA05682uTjgJCXdxXn7OzGPU\n68vIyi2hQ1QI/7vnEmN5PK2NRSte7wOrZhjrjV5yP9z3q7EMnbCf8Di4cTaM+ciY8jhnHfxnAHz7\nKFQcM7s6YTLplhHnRGvNjCW7mPbddqwahnSJ5pUxPQjx94GCnbDgkROrIcWmGv3ELZLMLdrddR5h\nrAp1fDrhX98yjm0MmwadRzbOgiXC6Ui4C5sdq6jm8f9t4puNxgG8Bwd14MFBHfCqqYD0l2Hpy1BT\naSw0fcU/oOdtsoZoY/EPgT9MgW5jjNWfDqw1RiZ1GAJXvgRNZRZuTyPhLmyyfn8h98/OYP+RcoL9\nvHn5hh4M7RoN2+bB90/WjlkHut8IVzwLIc3P/IbCMVp2M0bUrH0PFj0Dv30Pb/SFtEfhovs886xf\nDyXhLs7IatV8u7uaL77/BYtVkxTThH/f2IsEy2744E5jQWiAqC4w7AVjYQphLi9vY+RMp6uMk582\nfwGLJhvrug55DhKHSleNB5BwF6d1oLCcx77YyNLfqgAYd0k8j6dF4b/0aVjzLmirMf564JPQa5xr\nLZzhCUKj4bp3ocfNxsilgiyYPdaY0uAP/zROIhNuS/43it/RWjNn9X6mfLONkkoLwb7w2vVdGVw8\nD9582ZjgS3lD6t2Q9pi5C1KLs2s/yJiMbc07kP487F4Cb/WHnrfAwKeMXwLC7Ui4i5PsP1LG377c\nxNLf8gEY1iWS8WoePRdNMpaFA2NkxtDnIaqzaXWKc+TjBxfdYxxw/fkFWP0fY8bJzf8zxsxfdK/Z\nFQo7k3AXgDGFwH+W7OL1xVlUVFsJD/Th7T6HSN31D1T+DmOj6GQYPNnYE5Q+W9cU1AyGPW/0yf/w\nFOxYYAyh/PVt4lpeBVWpMiGZm5BwFyzJzOPv87awO78U0Dzadi93Wj/Dd9U6AMoDogm8cgokXStD\nG91FZHvjBKg9y+GnZ2HfCtrt+gD+7ztjLvnet4OPv9lVigsg4e7BfjtczLTvdrBo22EUVm4L38xf\nA+cTkrPF2CAoEi7/K6tK23J5tyvMLVY4Rvyl8MdvYeePHPvqUZoUZ8G3k2D5a0Z3Tc9bZU/eRUm4\ne6CDReX864dMPl+bjdI1XOO3hidCvyGiNAsqMBbOuOQBSPkj+AWj09PNLlk4klLQfjAZvV4irUUp\nLJ4CuVuN+Wp+ngZ974HUO42RUcJlSLh7kNziCmYu3c0Hv+zBz1LCXT6LuSdwEeHVh6EUCG0F/R6C\nXreBb6DZ5YrGppQxlUHHK2HHN7D0FcjJgMXPwfJXjV/2qXfLxG8uQsLdAxwoLOftn3cyZ/V+omoO\n8VfvhdwcmE6ALodqoFk7uPg+Y2ic9LMKLy/ofBV0GmEMm1z2CuxKh1/+DSveMCaAS70b4vvJgXUn\nJuHuxrbkFPHe8j18vW4vl5PBDO+fuMx/I15o0EB8f7h4gjH/iBwoFQ0pBW0vNy4HMoxg3zoXts03\nLlFdIPUuSL4e/EPNrlY0IOHuZqprrCzccogPftnDob3bGeu9mCW+S4hShcYG3n7Q9RpjT71lN3OL\nFa4jphdc9w4UT4E17xlz1+RuNRZiWfgkdL3aOBO2zSWyN+8kJNzdxJ78Uv6Xkc13q7eQWraESd6/\nkOq/48QGkR2N4W3db5QzSsX5C20BAx6H/hONSeNWz4R9K2D9R8alaYIR8t1ukJkoTSbh7sKOVVTz\nzcaDLFi9g6YHFjPK+xfu99qIr28NANonENVlFPQeB60vkj0qYT8+fpB8nXEp2Fkb7rPh6G7jAOzi\n56BVL+g6GrqMkqA3gYS7iykqr+an7Yf5Zf0WAnd9z0BWM9NrC/5+FgC08ka3G4xKvh7Vabj0hQrH\ni2gHg56GAU/ArsVGyO/41hhpk5NhnAkb09sI+cShEJkoOxqNQMLdBRwqquDn7QfZnrGU0JylDFAZ\njPbKgtq1kDWKmtiL8E6+FtV1tMylLszh5Q3tBxuXqjLIWgRbvoTMhcbiIQfWwg9PQ3gb4yB+4h+M\nETcy7NYhJNydUEV1DWt2H2HjpgxqfltM+9I1DPXawhhVVhfoFi9/LPFpBCRdhUocircEunAmfkHQ\nZaRxqSqDrB9g+wLjunCvMXHZ6v+ATyC0udgI+fj+0KqnsQC4uGAS7k6gtNLC+j157N2yEsvelTQ/\nup4eKpN+6oixQW2gFwfF4dNhAIGdhuDTbiA+fsHmFS2ErfyCjC6ZLqPAWmMs5J250Fgl6uB6Y83d\n4+vu+gYbx4fi+0FcKrTsYSwhKM6ZhHsjq7FqducdY8+OTRTtXguHNhBTupVeaieXKmNRDGqHnJf7\nNKG0VT/Ckq7At8NAQpvGm1a3EHbh5Q2xKcZl4BNQkmus5rVnmXHJz4SdPxoXAOUFzTtDbG+j3z4m\nBZp3lL17G0i4O1BJpYU9+w+Qu2sDxdlb8c3bTIuyHXRkL+1V5YkNa8O8wL815S16E5bYn9AOlxIY\nmUignFwk3FlIlDHbaNK1xv3iQ0bI711u9NEf3gK5tZeMWcY23n7G0N7orhDdxbiO6moM05QDtXUk\n3C9QjVVz6EghefuzOHIgk6pDO/A9mkXTsj3EWfeTpI6d/ILa795R70gKwzrjE9Od5ol9CWh7CRHB\nkY3fACGcSWiLE0MsAarL4eAGI+iz1xijb47ugcObjEt9gU0hor0xnUZEO2jW1rgf0c4jR41JuJ9F\nRZWFgryDFOXu41jeAXK3r2Xx1s/xL8mmSUUOzWsOE6MKiTnVixVU4E+uf2sqwtrh2yqZiPZ9aJLQ\nm6bBkcgce0KchW+g0Qff+qITj1UWQ+622r36rcb14S1QfhSyVxuXhoKbQ1gchMXWXSLziuBAKDSJ\nNZ53s7+SPS7ctdYUl5ZQXJBLSWEu5UV5VBXnU11cQE1JHl6lhwmoyCW4Kp/wmgIidCExqqYuvC9q\n+IYKLHiR7xVFcUBLqsLb4R3VkfDWXYmMTyIgPI7WbvalEcJU/qHGwda41BOPaQ0lh40Tqgqy4MjO\n2ts74cguKM0zLjkZdS9JAtgyzbjj5WsEfEhzCI4yuotCok7cDm4OwZEQEA6B4eAb5PRdQDaFu1Jq\nKPAaxriNmVrr5xs87w/MAnoDBcAYrfUe+5ZqKC0uouTYESpKiigvLaK6tIjq8mNYyoupKS9CV5ZA\nZTGqqgSv6hJ8qkvxtRQTZCkixFpME11ME1VJE1s/UMExgin0jqDUL5JCHUJQTBf8IhMIbdGWyNiO\nBDSLoYW3Dy0c0WAhxNkpZXTphLYwFiCpz2qF4hwoOgBF+421gIuyyd+5nkjfcijKNvb6i3OMiy28\n/U4E/e+uw8AvxBjl4xcKfsEnbvuHGM8FhDl8FNBZw10p5Q28AVwBZAOrlVLztNZb6232J+Co1rq9\nUmosMA0Y44iCd78+iqTKdef/BgqqtTdFqgml3qGU+4RR6RtOtX84KigCr9AW+DVtRVBELOFRsYQ1\nj6OJf1DdL4P09HQuTkuzR1OEEI3By+tEdwx96x7enJ5O2vH/y9Xlxp59SZ7xF0BprnG7NNcY0VOa\nB2UFUF4IFYVgqTCeK809v5o6jYCxH11w087Elj33VCBLa70LQCk1BxgF1A/3UcDk2tufA68rpZTW\nWtuxVgAq/ZtRUBlOuQqkwiuIKu9gqn2CsPiEYPUNxlr721H5h+IV0ATvwFD8g5sSGB5JSHgUTZpF\n4R8URqRSyOFLIQRg9O2HtzYutqiuMEK+/OiJwD9+XVEEVSVQWdLguvjE/WDHn3Sozpa/SqnrgKFa\n6ztr798K9NVaT6i3zebabbJr7++s3Sa/wXuNB8YDREdH954zZ44929IoSkpKCAnxrJMqpM2ewdPa\n7KrtHTBgwFqtdcrZtmvUA6pa6xnADICUlBSd5oLdG+n1/5TzENJmz+BpbXb39toyjOMAUH/RxNja\nx065jVLKBwjDOLAqhBDCBLaE+2qgg1IqQSnlB4wF5jXYZh5we+3t64CfHNHfLoQQwjZn7ZbRWluU\nUhOAhRhDId/VWm9RSj0DrNFazwPeAf6rlMoCjmD8AhBCCGESm/rctdYLgAUNHnu63u0K4Hr7liaE\nEOJ8yamTQgjhhiTchRDCDUm4CyGEGzrrSUwO+2Cl8oC9pnz4hYkE8s+6lXuRNnsGT2uzq7a3jdb6\nrKe4mhburkoptcaWs8PcibTZM3ham929vdItI4QQbkjCXQgh3JCE+7mbYXYBJpA2ewZPa7Nbt1f6\n3IUQwg3JnrsQQrghCfcLoJSaqJTSSim3XvdDKfWiUmq7UmqjUupLpVS42TU5ilJqqFJqh1IqSyn1\nmNn1OJpSKk4ptVgptVUptUUp9aDZNTUWpZS3UmqdUuprs2txBAn386SUigOGAPvMrqUR/AAkaa27\nAZnA4ybX4xD1lpQcBnQBblRKdTG3KoezABO11l0w1n+/zwPafNyDwDazi3AUCffz9y/gr4DbH7TQ\nWn+vtbbU3l2JMae/O6pbUlJrXQUcX1LSbWmtD2qtM2pvF2OEXYy5VTmeUioWGA7MNLsWR5FwPw9K\nqVHAAa31BrNrMcEdwLdmF+EgMcD+evez8YCgO04pFQ/0BH41t5JG8SrGzpnV7EIcpVGX2XMlSqlF\nQItTPPUE8DeMLhm3cab2aq2/qt3mCYw/4x27bLtodEqpEOAL4CGt9TGz63EkpdQIIFdrvVYplWZ2\nPY4i4X4aWuvBp3pcKZUMJAAblFJgdFFkKKVStdaHGrFEuzpde49TSo0DRgCD3HiVLVuWlHQ7Silf\njGD/SGv9P7PraQSXAiOVUlcCAUATpdSHWutbTK7LrmSc+wVSSu0BUrTWrjgBkU2UUkOBV4DLtdZ5\nZtfjKLXr/2YCgzBCfTVwk9Z6i6mFOZAy9lA+AI5orR8yu57GVrvn/ojWeoTZtdib9LkLW7wOhAI/\nKKXWK6XeMrsgR6g9aHx8ScltwKfuHOy1LgVuBQbW/mzX1+7RChcne+5CCOGGZM9dCCHckIS7EEK4\nIQl3IYRwQxLuQgjhhiTchRDCDUm4CyGEG5JwF0IINyThLoQQbuj/AUH73pYPx7IMAAAAAElFTkSu\nQmCC\n", 195 | "text/plain": [ 196 | "" 197 | ] 198 | }, 199 | "metadata": {}, 200 | "output_type": "display_data" 201 | } 202 | ], 203 | "source": [ 204 | "x = np.arange(-5,5,0.01)\n", 205 | "plt.plot(x, sigm().forward({'x':x}), linewidth=2)\n", 206 | "plt.plot(x, sigm().backward({'x':x, 'dzdx':1}), linewidth=2)\n", 207 | "plt.grid(1)\n", 208 | "plt.legend(['sigmoid', 'sigmoidPrime'])" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "Now, lets define or loss function. In this case correspond to MSE. For custom loss implementation only forward and backward definition is needed. An example of custom loss layer in caffe with python is given in https://chrischoy.github.io/research/caffe-python-layer/.\n", 216 | "Tensorflow use the same idea. If you define a custom loss function with existing operations tf take derivative for you, example:\n", 217 | "\n", 218 | " loss_mse = 1/n(sum((y-y_)^2)))\n", 219 | "\n", 220 | "can be defined as:\n", 221 | "\n", 222 | " cost = tf.reduce_mean(tf.square(tf.sub(y,y_)))\n", 223 | "\n", 224 | "in other case c++ implementation must be done with the same idea (functions to do forward and backwards)." 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 21, 230 | "metadata": { 231 | "collapsed": true 232 | }, 233 | "outputs": [], 234 | "source": [ 235 | "# Loss layer class\n", 236 | "# Ecuation:\n", 237 | "# z = 1/2 (y-x)^2\n", 238 | "# dz/dx = (x-y)*(dz/dx_i+1) \n", 239 | "# note: dz/dx_n = 1 \n", 240 | "#\n", 241 | "class loss(layer):\n", 242 | " \n", 243 | " def forward(self, param): \n", 244 | " return (1.0/(2.0*len(param['y']))*np.sum((param['y']-param['x'])**2));\n", 245 | "\n", 246 | " def backward(self, param):\n", 247 | " return (param['x']-param['y'])*param['dzdx']; \n" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "With this, evaluation (forward) in previous network can be defined as:" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 22, 260 | "metadata": { 261 | "collapsed": true 262 | }, 263 | "outputs": [], 264 | "source": [ 265 | "# Create cost function neural netword\n", 266 | "# Demo\n", 267 | "# +--------+ +--------+ +--------+ +--------+\n", 268 | "# ----> | inner |--> | inner |---> | sigm | ---->(*)---> | loss |\n", 269 | "# +--------+ +--------+ +--------+ +--------+\n", 270 | "\n", 271 | "# Const function\n", 272 | "def costFunc( x, w1, w2):\n", 273 | " '''\n", 274 | " forward function\n", 275 | " Entrada:\n", 276 | " * x vector nxm. m featurs\n", 277 | " * w1, w2 weigths\n", 278 | " Return:\n", 279 | " * z4\n", 280 | " \n", 281 | " Note: b=0\n", 282 | " '''\n", 283 | "\n", 284 | " z1 = x;\n", 285 | " z2 = inner().forward({'x':z1, 'w':w1}) \n", 286 | " z3 = inner().forward({'x':z2, 'w':w2}) \n", 287 | " z4 = sigm().forward({'x':z3});\n", 288 | " return z4;\n", 289 | "\n", 290 | "\n" 291 | ] 292 | }, 293 | { 294 | "cell_type": "markdown", 295 | "metadata": {}, 296 | "source": [ 297 | "An example of network initialization and evaluation:" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 23, 303 | "metadata": {}, 304 | "outputs": [ 305 | { 306 | "name": "stdout", 307 | "output_type": "stream", 308 | "text": [ 309 | "Loss: 0.0195656069729\n" 310 | ] 311 | } 312 | ], 313 | "source": [ 314 | "# create data\n", 315 | "\n", 316 | "# data\n", 317 | "x = np.matrix([[1,2,3]], dtype=np.float64).T;\n", 318 | "y = np.matrix([1.0], dtype=np.float64);\n", 319 | "\n", 320 | "# init weights\n", 321 | "w1 = np.matrix([[0.1, 0.2, 0.3],[0.1, 0.2, 0.3]], dtype=np.float64)\n", 322 | "w2 = np.matrix([0.8, 0.2], dtype=np.float64)\n", 323 | "\n", 324 | "# evaluate function\n", 325 | "y_ = costFunc( x, w1, w2);\n", 326 | "e = loss().forward({'x':y_,'y':y});\n", 327 | "\n", 328 | "print 'Loss:',e\n", 329 | "\n" 330 | ] 331 | }, 332 | { 333 | "cell_type": "markdown", 334 | "metadata": {}, 335 | "source": [ 336 | "For forward and backward computation:" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": 15, 342 | "metadata": { 343 | "collapsed": true 344 | }, 345 | "outputs": [], 346 | "source": [ 347 | "# Gradind function\n", 348 | "def gradCostFunc(x, y, w1, w2):\n", 349 | " '''\n", 350 | " Apply backward function\n", 351 | " Compute dervative with respect to w1 and w2\n", 352 | " '''\n", 353 | " \n", 354 | " # forward --->\n", 355 | " z1 = x;\n", 356 | " z2 = inner().forward({'x':z1, 'w':w1}) \n", 357 | " z3 = inner().forward({'x':z2, 'w':w2}) \n", 358 | " z4 = sigm().forward({'x':z3});\n", 359 | " z5 = loss().forward({'x':z4, 'y':y});\n", 360 | "\n", 361 | " E = z5;\n", 362 | "\n", 363 | " # <--- backward\n", 364 | " l5 = 1;\n", 365 | " l4 = loss().backward({'x':z4, 'y':y, 'dzdx':l5} ); \n", 366 | " l3 = sigm().backward({'x':z3, 'dzdx':l4});\n", 367 | " l2, dEdW2 = inner().backward({'x':z2, 'w':w2, 'dzdx':l3});\n", 368 | " _ , dEdW1 = inner().backward({'x':z1, 'w':w1, 'dzdx':l2});\n", 369 | "\n", 370 | "\n", 371 | " return E, dEdW1, dEdW2" 372 | ] 373 | }, 374 | { 375 | "cell_type": "markdown", 376 | "metadata": {}, 377 | "source": [ 378 | "Then, for previous example, gradients of weights are:" 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": 1, 384 | "metadata": {}, 385 | "outputs": [ 386 | { 387 | "ename": "NameError", 388 | "evalue": "name 'gradCostFunc' is not defined", 389 | "output_type": "error", 390 | "traceback": [ 391 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 392 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 393 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# derivate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;31m# grad J(x)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdEdW1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdEdW2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgradCostFunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 394 | "\u001b[0;31mNameError\u001b[0m: name 'gradCostFunc' is not defined" 395 | ] 396 | } 397 | ], 398 | "source": [ 399 | "# derivate\n", 400 | "# grad J(x)\n", 401 | "E, dEdW1, dEdW2 = gradCostFunc(x, y, w1, w2);\n", 402 | "\n", 403 | "\n", 404 | "# minimization with gradien decent\n", 405 | "# w^t = m*w^(t-1) + lr*gardJ \n", 406 | "\n", 407 | "print(E)\n", 408 | "print(dEdW1)\n", 409 | "print(dEdW2)" 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "metadata": {}, 415 | "source": [ 416 | "Continue tutorial demo: lineal_regression" 417 | ] 418 | } 419 | ], 420 | "metadata": { 421 | "kernelspec": { 422 | "display_name": "Python 2", 423 | "language": "python", 424 | "name": "python2" 425 | }, 426 | "language_info": { 427 | "codemirror_mode": { 428 | "name": "ipython", 429 | "version": 2 430 | }, 431 | "file_extension": ".py", 432 | "mimetype": "text/x-python", 433 | "name": "python", 434 | "nbconvert_exporter": "python", 435 | "pygments_lexer": "ipython2", 436 | "version": "2.7.12" 437 | } 438 | }, 439 | "nbformat": 4, 440 | "nbformat_minor": 2 441 | } 442 | -------------------------------------------------------------------------------- /pytorch/ipytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 17, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "0.2.0_4\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "import torch as th\n", 18 | "from torch.autograd import Variable\n", 19 | "\n", 20 | "import numpy as np\n", 21 | "import matplotlib.pyplot as plt\n", 22 | "\n", 23 | "print( th.__version__ )" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 16, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "name": "stdout", 33 | "output_type": "stream", 34 | "text": [ 35 | "0 266230.457784815\n", 36 | "1 15956.895521402268\n", 37 | "2 3905.722384996317\n", 38 | "3 1431.7602827240112\n", 39 | "4 643.0559108923042\n", 40 | "5 359.9729471657151\n", 41 | "6 251.0651680024454\n", 42 | "7 205.39495511207852\n", 43 | "8 183.34546355389088\n", 44 | "9 170.36776456614658\n" 45 | ] 46 | }, 47 | { 48 | "data": { 49 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH8lJREFUeJzt3XuQXWW95vHv05fcu0Nu0G0SSJAU3cGBIG2IxjMw5hgY\n58yAJUicEVPnUKIFx5EZakY5c0aOeii1RkV0kDkcQQMyAiKWjIViCOoRlEuCKOaCtFykQ0JC7oHc\nuvs3f+x3d3Z3Okkn3atX772fT9Wuvfa71rv63buSfnqt9e7fUkRgZmaWpZq8B2BmZpXPYWNmZplz\n2JiZWeYcNmZmljmHjZmZZc5hY2ZmmXPYmJlZ5hw2ZmaWOYeNmZllri7vAYwUU6dOjVmzZuU9DDOz\nsrJq1arXI2La0bZz2CSzZs1i5cqVeQ/DzKysSHp5INv5NJqZmWXOYWNmZplz2JiZWeZ8zcbM7Dgd\nOHCAjo4O9u7dm/dQMjdmzBhmzJhBfX39cfV32JiZHaeOjg4aGhqYNWsWkvIeTmYigi1bttDR0cHs\n2bOPax8+jWZmdpz27t3LlClTKjpoACQxZcqUQR3BOWzMzAah0oOmaLDv02EzSDvePMBNDz/P7zu2\n5z0UM7MRy9dsBqmmBm58+I/U1YozZ5yQ93DMzEYkH9kMUsOYemZOHsuaDTvzHoqZ2SG6urqO+Ppw\nOjs7h3QcDpsh0NLUyDqHjZnl4Lvf/S7z589n3rx5fOxjH6Orq4sJEyZw7bXXctZZZ/Gb3/yGWbNm\n8alPfYq3v/3tfP/73+eZZ55hwYIFnHnmmbz//e9n27ZtAJx//vlcc801tLW1cdNNNw3pOH0abQi0\nNjeyYu1r7D3QxZj62ryHY2Y5+Oz/W82aV4f2j865b2nk+n9/xmHXr127lnvuuYfHHnuM+vp6rrrq\nKu666y7eeOMNzj33XL7yla/0bDtlyhSefvppAM4880y+8Y1vcN555/GZz3yGz372s3zta18DYP/+\n/ZnUiXTYDIHWpga6A57buIuzZvq6jZkNjxUrVrBq1Sre8Y53ALBnzx5OPPFEamtr+cAHPtBr28su\nuwyAHTt2sH37ds477zwAli5dyqWXXnrIdkPNYTMEWpsbAVi3cafDxqxKHekIJCsRwdKlS/nCF77Q\nq/3LX/4ytbW9z7KMHz9+QPsc6HbHytdshsDJk8cxblQtazfsynsoZlZFFi1axH333cemTZsA2Lp1\nKy+/fOSK/xMnTmTSpEn86le/AuDOO+/sOcrJko9shkBNjTi9qYG1niRgZsNo7ty5/OM//iOLFy+m\nu7ub+vp6br755qP2W7ZsGR//+Md58803OfXUU/n2t7+d+VgdNkOktbmRH//uVSKiar5RbGb5u+yy\nyw65zrJ79+5er1966aVer+fNm8fjjz9+yL5+8YtfDPXwemR2Gk3STEk/l7RG0mpJn0zt/yBpvaRn\n0uN9JX2uk9Qu6TlJF5S0nyPp2bTu60q/zSWNlnRPan9C0qySPkslPZ8eS7N6n0WtTQ3s3NvJhh2V\nX/3VzOxYZXnNphO4NiLmAguAqyXNTetujIh56fEgQFq3BDgDuBD4pqTiFa5bgI8Cc9LjwtR+BbAt\nIk4DbgS+lPY1GbgeOBeYD1wvaVKG77VnkoBPpZmZHSqzsImIDRHxdFreBawFph+hy0XA3RGxLyJe\nBNqB+ZKagcaIeDwiArgDuLikz7K0fB+wKB31XAAsj4itEbENWM7BgMrE6U0NAKzb6EkCZtWk8Gup\n8g32fQ7LbLR0euts4InU9AlJv5d0e8kRx3TglZJuHaltelru296rT0R0AjuAKUfYV99xXSlppaSV\nmzdvPu73By5bY1aNxowZw5YtWyo+cIr3sxkzZsxx7yPzCQKSJgA/AK6JiJ2SbgE+D0R6/grwN1mP\noz8RcStwK0BbW9ug/7W0NjX6NJpZFZkxYwYdHR0M9o/VclC8U+fxyjRsJNVTCJq7IuJ+gIh4rWT9\nPwM/Ti/XAzNLus9IbevTct/20j4dkuqAicCW1H5+nz6/GIr3dCQtzY08vPY19uzvYuwol60xq3T1\n9fXHfefKapPlbDQBtwFrI+KrJe3NJZu9H/hDWn4AWJJmmM2mMBHgyYjYAOyUtCDt8yPAj0r6FGea\nXQI8kq7rPAQsljQpnaZbnNoyNbe5ULbmj6/5uo2ZWaksj2wWApcDz0p6JrX9HfAhSfMonEZ7CfgY\nQESslnQvsIbCTLarI6JYC/sq4DvAWOAn6QGFMLtTUjuwlcJsNiJiq6TPA0+l7T4XEVszep89Wppc\ntsbMrD+ZhU1EPAr09+3GB4/Q5wbghn7aVwJv66d9L3Bp3/a07nbg9oGOdyi4bI2ZWf9cG20IuWyN\nmVn/HDZDrLW5MCOt0qdCmpkdC4fNECuWrXnVZWvMzHo4bIZYz71tfCrNzKyHw2aIFcvW+LqNmdlB\nDpshVixbs9Y10szMejhsMuCyNWZmvTlsMtDS3MhLr7/Bnv1dR9/YzKwKOGwy4LI1Zma9OWwyUFq2\nxszMHDaZOHnyOMa7bI2ZWQ+HTQaKZWt8IzUzswKHTUZamhtZ57I1ZmaAwyYzrc2NLltjZpY4bDLS\nmioJuGyNmZnDJjMuW2NmdpDDJiMuW2NmdpDDJkMuW2NmVuCwyZDL1piZFThsMuSyNWZmBQ6bDBVv\npOZTaWZW7Rw2GZo5qVC2Zp0nCZhZlXPYZMhla8zMChw2GXPZGjMzh03mXLbGzMxhkzmXrTEzc9hk\nzmVrzMwcNpnrKVvjG6mZWRXLLGwkzZT0c0lrJK2W9MnUPlnScknPp+dJJX2uk9Qu6TlJF5S0nyPp\n2bTu65KU2kdLuie1PyFpVkmfpelnPC9paVbvcyBamxpZ61tEm1kVy/LIphO4NiLmAguAqyXNBT4N\nrIiIOcCK9Jq0bglwBnAh8E1JtWlftwAfBeakx4Wp/QpgW0ScBtwIfCntazJwPXAuMB+4vjTUhlur\ny9aYWZXLLGwiYkNEPJ2WdwFrgenARcCytNky4OK0fBFwd0Tsi4gXgXZgvqRmoDEiHo/C/OE7+vQp\n7us+YFE66rkAWB4RWyNiG7CcgwE17FpdtsbMqtywXLNJp7fOBp4AToqIDWnVRuCktDwdeKWkW0dq\nm56W+7b36hMRncAOYMoR9tV3XFdKWilp5ebNm4/z3R2dy9aYWbXLPGwkTQB+AFwTEb1+26Yjldy+\n7RgRt0ZEW0S0TZs2LbOf47I1ZlbtMg0bSfUUguauiLg/Nb+WTo2Rnjel9vXAzJLuM1Lb+rTct71X\nH0l1wERgyxH2lQuXrTGzapflbDQBtwFrI+KrJaseAIqzw5YCPyppX5JmmM2mMBHgyXTKbaekBWmf\nH+nTp7ivS4BH0tHSQ8BiSZPSxIDFqS03LltjZtUsyyObhcDlwHskPZMe7wO+CLxX0vPAX6bXRMRq\n4F5gDfBT4OqIKE7fugr4FoVJA38CfpLabwOmSGoH/itpZltEbAU+DzyVHp9Lbblx2Rozq2Z1We04\nIh4FdJjViw7T5wbghn7aVwJv66d9L3DpYfZ1O3D7QMebtbnNqZLAqzuZfsLYnEdjZja8XEFgmJze\nVJiRts5f7jSzKuSwGSYTRtdx8uRxLltjZlXJYTOMWpoaXLbGzKqSw2YYuWyNmVUrh80wctkaM6tW\nDpth5LI1ZlatHDbDyGVrzKxaOWyGkcvWmFm1ctgMs9bmRta6bI2ZVRmHzTBraW5kl8vWmFmVcdgM\ns9KyNWZm1cJhM8xctsbMqpHDZpi5bI2ZVSOHTQ5ctsbMqo3DJgcuW2Nm1cZhk4PW5kaXrTGzquKw\nyUFrcUaav9xpZlXCYZODYtkah42ZVQuHTQ6KZWvWukaamVUJh01OXLbGzKqJwyYnLltjZtXEYZMT\nl60xs2risMmJy9aYWTVx2OTEZWvMrJo4bHLU2tzg6c9mVhUcNjlqaWrkxS0uW2Nmlc9hk6PW5kYi\n4DmXrTGzCuewyVGxbM06n0ozswqXWdhIul3SJkl/KGn7B0nrJT2THu8rWXedpHZJz0m6oKT9HEnP\npnVfl6TUPlrSPan9CUmzSvoslfR8eizN6j0OlsvWmFm1yPLI5jvAhf203xgR89LjQQBJc4ElwBmp\nzzcl1abtbwE+CsxJj+I+rwC2RcRpwI3Al9K+JgPXA+cC84HrJU0a+rc3eC5bY2bVIrOwiYh/AbYO\ncPOLgLsjYl9EvAi0A/MlNQONEfF4FOq63AFcXNJnWVq+D1iUjnouAJZHxNaI2AYsp//QGxFctsbM\nqkEe12w+Ien36TRb8YhjOvBKyTYdqW16Wu7b3qtPRHQCO4ApR9jXiOSyNWZWDYY7bG4BTgXmARuA\nrwzzz+9F0pWSVkpauXnz5lzG4LI1ZlYNhjVsIuK1iOiKiG7gnylcUwFYD8ws2XRGaluflvu29+oj\nqQ6YCGw5wr76G8+tEdEWEW3Tpk0bzFs7bsWyNZ4kYGaVbFjDJl2DKXo/UJyp9gCwJM0wm01hIsCT\nEbEB2ClpQboe8xHgRyV9ijPNLgEeSdd1HgIWS5qUTtMtTm0jUrFszTpPEjCzClaX1Y4lfQ84H5gq\nqYPCDLHzJc0DAngJ+BhARKyWdC+wBugEro6I4tfqr6Iws20s8JP0ALgNuFNSO4WJCEvSvrZK+jzw\nVNrucxEx0IkKuXDZGjOrdJmFTUR8qJ/m246w/Q3ADf20rwTe1k/7XuDSw+zrduD2AQ82Zy1Njfxs\nzWvs2d/F2FG1R+9gZlZmBnQaTdInJTWq4DZJT0tanPXgqoXL1phZpRvoNZu/iYidFK5/TAIuB76Y\n2aiqjMvWmFmlG2jYKD2/D7gzIlaXtNkguWyNmVW6gYbNKkk/oxA2D0lqALqzG1Z1qakRLc2NLltj\nZhVroBMErqDwRcwXIuLNVH/sr7MbVvVpaWrggd+9SkSQao2amVWMgR7ZvBN4LiK2S/ow8PcUysPY\nEGlNZWvWb9+T91DMzIbcQMPmFuBNSWcB1wJ/olAU04bIwUkCPpVmZpVnoGHTmb6dfxHwvyPiZqAh\nu2FVH5etMbNKNtBrNrskXUdhyvNfSKoB6rMbVvVx2Rozq2QDPbK5DNhH4fs2GykUt/xfmY2qSrls\njZlVqgGFTQqYu4CJkv4K2BsRvmYzxFqaGnlxyxvs2d919I3NzMrIQMvVfBB4kkItsg8CT0i6JMuB\nVSOXrTGzSjXQazb/A3hHRGwCkDQNeJjC7ZhtiMxtLkwSWLdhJ/NmnpDzaMzMhs5Ar9nUFIMm2XIM\nfW2AZkwa67I1ZlaRBnpk81NJDwHfS68vAx7MZkjVq6dsjb9rY2YVZkBhExH/TdIHgIWp6daI+GF2\nw6peLltjZpVowDdPi4gfAD/IcCxGYZLAXU/8mfXb9zBj0ri8h2NmNiSOGDaSdlG4hfMhq4CIiMZM\nRlXFSsvWOGzMrFIc8SJ/RDRERGM/jwYHTTZctsbMKpFnlI0wE0bXccoUl60xs8risBmBWppctsbM\nKovDZgRqbS6UrXlzf2feQzEzGxIOmxGopalQtuaPr+3OeyhmZkPCYTMCFcvW+FSamVUKh80IVCxb\ns85hY2YVwmEzArlsjZlVGofNCNXS1MDajTsp3I3bzKy8OWxGqNbmRnbt7WT99j15D8XMbNAyCxtJ\nt0vaJOkPJW2TJS2X9Hx6nlSy7jpJ7ZKek3RBSfs5kp5N676uVJ1S0mhJ96T2JyTNKumzNP2M5yUt\nzeo9Zqm15942PpVmZuUvyyOb7wAX9mn7NLAiIuYAK9JrJM0FlgBnpD7flFSb+twCfBSYkx7FfV4B\nbIuI04AbgS+lfU0GrgfOBeYD15eGWrk4valQI80z0sysEmQWNhHxL8DWPs0XAcvS8jLg4pL2uyNi\nX0S8CLQD8yU1A40R8XgULl7c0adPcV/3AYvSUc8FwPKI2BoR24DlHBp6I16xbM3ajQ4bMyt/w33N\n5qSI2JCWNwInpeXpwCsl23WktulpuW97rz4R0QnsAKYcYV9lp6WpwafRzKwi5DZBIB2p5DrVStKV\nklZKWrl58+Y8h9Ivl60xs0ox3GHzWjo1RnrelNrXAzNLtpuR2tan5b7tvfpIqgMmAluOsK9DRMSt\nEdEWEW3Tpk0bxNvKhsvWmFmlGO6weQAozg5bCvyopH1JmmE2m8JEgCfTKbedkhak6zEf6dOnuK9L\ngEfS0dJDwGJJk9LEgMWprey4bI2ZVYoB3xb6WEn6HnA+MFVSB4UZYl8E7pV0BfAy8EGAiFgt6V5g\nDdAJXB0RXWlXV1GY2TYW+El6ANwG3CmpncJEhCVpX1slfR54Km33uYjoO1GhLLhsjZlVCvkb6gVt\nbW2xcuXKvIdxiA/c8mtqJe79+DvzHoqZ2SEkrYqItqNt5woCI1xrs8vWmFn5c9iMcC1NLltjZuXP\nYTPCtfZMEvD3bcysfDlsRrhi2RpPEjCzcuawGeFctsbMKoHDpgy4bI2ZlTuHTRlw2RozK3cOmzLQ\n2uyyNWZW3hw2ZaC1yWVrzKy8OWzKwIxJY5kwus4z0sysbDlsykBNjTi9qcHftTGzsuWwKRMuW2Nm\n5cxhUyZctsbMypnDpky4bI2ZlTOHTZlw2RozK2cOmzLhsjVmVs4cNmWktanRZWvMrCw5bMpIS3OD\ny9aYWVly2JSRYtma5zb66MbMyovDpowUy9asc9iYWZlx2JSRYtka10gzs3LjsCkjxbI1niRgZuXG\nYVNmXLbGzMqRw6bMtDa7bI2ZlR+HTZlpaXLZGjMrPw6bMtPisjVmVoYcNmVmvMvWmFkZctiUodam\nRp9GM7OykkvYSHpJ0rOSnpG0MrVNlrRc0vPpeVLJ9tdJapf0nKQLStrPSftpl/R1SUrtoyXdk9qf\nkDRruN9jllqaG3jJZWvMrIzkeWTzbyJiXkS0pdefBlZExBxgRXqNpLnAEuAM4ELgm5JqU59bgI8C\nc9LjwtR+BbAtIk4DbgS+NAzvZ9i4bI2ZlZuRdBrtImBZWl4GXFzSfndE7IuIF4F2YL6kZqAxIh6P\nwpdO7ujTp7iv+4BFxaOeSuCyNWZWbvIKmwAelrRK0pWp7aSI2JCWNwInpeXpwCslfTtS2/S03Le9\nV5+I6AR2AFOG+k3kxWVrzKzc1OX0c98dEeslnQgsl7SudGVEhKTMvyKfgu5KgJNPPjnrHzdkampE\ni8vWmFkZyeXIJiLWp+dNwA+B+cBr6dQY6XlT2nw9MLOk+4zUtj4t923v1UdSHTAR2NLPOG6NiLaI\naJs2bdrQvLlh0uKyNWZWRoY9bCSNl9RQXAYWA38AHgCWps2WAj9Kyw8AS9IMs9kUJgI8mU657ZS0\nIF2P+UifPsV9XQI8EhX2W9lla8ysnORxGu0k4Ifpen0d8H8j4qeSngLulXQF8DLwQYCIWC3pXmAN\n0AlcHRFdaV9XAd8BxgI/SQ+A24A7JbUDWynMZqsopWVrZkwal/NozMyObNjDJiJeAM7qp30LsOgw\nfW4AbuinfSXwtn7a9wKXDnqwI1ixbM3aDTt579yTjrK1mVm+RtLUZzsGxbI161y2xszKgMOmjLls\njZmVC4dNGXPZGjMrFw6bMuayNWZWLhw2ZWxus8vWmFl5cNiUseknuGyNmZUHh00ZK5atcdiY2Ujn\nsClzLc2FGmkVViDBzCqMw6bMtTY3smtfJx3bXLbGzEYuh02Za/G9bcysDDhsylxLUwMSvm5jZiOa\nw6bMjR9dxymTXbbGzEY2h00FaHHZGjMb4Rw2FaC1udFla8xsRHPYVICW5gaXrTGzEc1hUwGKZWt8\nKs3MRiqHTQUolq3xJAEzG6kcNhXAZWvMbKRz2FQIl60xs5HMYVMhimVr7n96PTvePJD3cMzMeqnL\newA2NBacOoWGMXVc+/3fIcG/mj6RhadN5d2nTeWcUyYxpr427yGaWRWTT7sUtLW1xcqVK/MexqAc\n6Ormd69s59H213ms/XV+++ftdHYHo+tqaJs1qSd8znjLRGprlPdwzawCSFoVEW1H3c5hU1AJYdPX\n7n2dPPniFh5r38Jj7a/3FOucOLaed546hYVzCuEza8o4JIePmR27gYaNT6NVsAmj63hPy0m8p+Uk\nADbt2stv/lQInkeff52frt4IFKZOv+utU3j3nKm8661TmdYwOs9hm1kF8pFNUolHNkcSEby05U0e\nbX+dX7e/zq//tIUdewoTC1qaGnjXW6fy7jlTmD97ChNG+28SM+ufT6Mdo2oLm766uoPVr+7oud7z\n1Evb2N/ZTV2NOPvkE1h42lQWnjaVeTNPoL7WkxjNrMBhc4yqPWz62nugi1Uvb+sJn2fX7yACxo+q\n5dxTp/Scdjv9pAZf7zGrYr5mY4Mypr6252gGYPub+3n8hS0pfLbwyLpNAEydMJqFp01h4VunsnDO\nVKafMDbPYZvZCFXRRzaSLgRuAmqBb0XEFw+3rY9sjs367Xt4LB31PNb+Oq/v3g/A7KnjOeeUSZww\ntp4JY+qYMLqOhjF1TBjd93UdE8bUMX5Unadhm5Wxqj+NJqkW+CPwXqADeAr4UESs6W97h83xiwie\ne21XzxTrZ9fvYPfeTvYc6BpQ//GjanuCaMKYehpG1zF+dC0TRtf3CqZeQZXaGkpCbFSdryWZDTef\nRoP5QHtEvAAg6W7gIqDfsLHjJ4mWpkZamhq54t2ze9o7u7p5Y18Xu/Yd4I19Xezed4BdezvZva+T\n3en5kNf7Otm99wCbd+1L6w+we18n3QP4m2hUXU0KqkL4jBtVS22NqK+tSc+itkbU1dRQl5bra2qo\nrRX1NaK2pqZkG1HXq1+fdWkfdTW99138WcX+dTWiRqKmBoSoUeHzkqBGQqRncbCt5Lm0z4D69mrz\nEaONHJUcNtOBV0pedwDn5jSWqlRXW8PEcTVMHFc/qP1EBHsOdLF7bzGM+gZVIZB2p0Arrn9zfxed\n3YW+nV3ddHYHnV1BZ/fB5a7u3q87u7vTc2Uc8fcKKlL4lDypZ7mwUAi41JZWqk+fXusOs/3BnOtv\nu95j7BlXP+v7xuXRArRX32P4Of3u68irBzaeAexjYBsd86aF7Qf4B0drcyPf+NDZx7j3Y1PJYXNU\nkq4ErgQ4+eSTcx6NHY4kxo2qY9yoOk4cpp8ZEXRHoQRQV9+Q6g66uoID3YV1xW0OlIZXWj7Q1U13\nBBHQHRAU9hs9bb2fD64vtgUBdHen5759Kekbh+mb2gvvKz0T0LN88D3HIW1p25K+pZ9R6XbF/fa3\nDw67jz6viSOsG3jfvhsf2vfIf0wM5E+No12BGNg+Bv5HzTH/+XMMHWZOyn5iTyWHzXpgZsnrGamt\nR0TcCtwKhWs2wzc0G+kkUSuorXEBU7OhUMlXVJ8C5kiaLWkUsAR4IOcxmZlVpYo9somITkl/CzxE\nYerz7RGxOudhmZlVpYoNG4CIeBB4MO9xmJlVu0o+jWZmZiOEw8bMzDLnsDEzs8w5bMzMLHMOGzMz\ny1zFFuI8VpI2Ay8PYhdTgdeHaDjlzp9Fb/48evPncVAlfBanRMS0o23ksBkiklYOpPJpNfBn0Zs/\nj978eRxUTZ+FT6OZmVnmHDZmZpY5h83QuTXvAYwg/ix68+fRmz+Pg6rms/A1GzMzy5yPbMzMLHMO\nm0GSdKGk5yS1S/p03uPJk6SZkn4uaY2k1ZI+mfeY8iapVtJvJf0477HkTdIJku6TtE7SWknvzHtM\neZL0X9L/kz9I+p6kMXmPKUsOm0GQVAvcDPxbYC7wIUlz8x1VrjqBayNiLrAAuLrKPw+ATwJr8x7E\nCHET8NOIaAHOooo/F0nTgf8MtEXE2yjcBmVJvqPKlsNmcOYD7RHxQkTsB+4GLsp5TLmJiA0R8XRa\n3kXhl8n0fEeVH0kzgH8HfCvvseRN0kTgXwO3AUTE/ojYnu+oclcHjJVUB4wDXs15PJly2AzOdOCV\nktcdVPEv11KSZgFnA0/kO5JcfQ3470B33gMZAWYDm4Fvp9OK35I0Pu9B5SUi1gNfBv4MbAB2RMTP\n8h1Vthw2NuQkTQB+AFwTETvzHk8eJP0VsCkiVuU9lhGiDng7cEtEnA28AVTtNU5JkyicBZkNvAUY\nL+nD+Y4qWw6bwVkPzCx5PSO1VS1J9RSC5q6IuD/v8eRoIfAfJL1E4fTqeyR9N98h5aoD6IiI4pHu\nfRTCp1r9JfBiRGyOiAPA/cC7ch5Tphw2g/MUMEfSbEmjKFzgeyDnMeVGkiick18bEV/Nezx5iojr\nImJGRMyi8O/ikYio6L9cjyQiNgKvSDo9NS0C1uQ4pLz9GVggaVz6f7OICp8wUZf3AMpZRHRK+lvg\nIQqzSW6PiNU5DytPC4HLgWclPZPa/i4iHsxxTDZyfAK4K/1h9gLw1zmPJzcR8YSk+4CnKczi/C0V\nXk3AFQTMzCxzPo1mZmaZc9iYmVnmHDZmZpY5h42ZmWXOYWNmZplz2JhVAEnnu7K0jWQOGzMzy5zD\nxmwYSfqwpCclPSPpn9L9bnZLujHd22SFpGlp23mSHpf0e0k/TPW0kHSapIcl/U7S05LemnY/oeR+\nMXelb6abjQgOG7NhIqkVuAxYGBHzgC7gPwHjgZURcQbwS+D61OUO4FMRcSbwbEn7XcDNEXEWhXpa\nG1L72cA1FO6tdCqFig5mI4LL1ZgNn0XAOcBT6aBjLLCJwi0I7knbfBe4P93/5YSI+GVqXwZ8X1ID\nMD0ifggQEXsB0v6ejIiO9PoZYBbwaPZvy+zoHDZmw0fAsoi4rlej9D/7bHe8NaT2lSx34f/fNoL4\nNJrZ8FkBXCLpRABJkyWdQuH/4SVpm/8IPBoRO4Btkv4itV8O/DLdAbVD0sVpH6MljRvWd2F2HPyX\nj9kwiYg1kv4e+JmkGuAAcDWFG4nNT+s2UbiuA7AU+D8pTEqrJF8O/JOkz6V9XDqMb8PsuLjqs1nO\nJO2OiAl5j8MsSz6NZmZmmfORjZmZZc5HNmZmljmHjZmZZc5hY2ZmmXPYmJlZ5hw2ZmaWOYeNmZll\n7v8DrsPUGk9B9zgAAAAASUVORK5CYII=\n", 50 | "text/plain": [ 51 | "" 52 | ] 53 | }, 54 | "metadata": {}, 55 | "output_type": "display_data" 56 | } 57 | ], 58 | "source": [ 59 | "\n", 60 | "\n", 61 | "dtype = th.FloatTensor\n", 62 | "N, D_in, H, D_out = 64, 1000, 100, 10\n", 63 | "\n", 64 | "#\n", 65 | "# x_(64x1000) ----> a_64x100 ---> y_64x10 \n", 66 | "# W^1_1000x100 w^2_100x10\n", 67 | "#\n", 68 | "\n", 69 | "# randonm input\n", 70 | "x = th.randn(N, D_in).type( dtype ) \n", 71 | "y = th.randn(N, D_out).type( dtype )\n", 72 | "\n", 73 | "# random initialization\n", 74 | "w1 = th.rand( D_in, H ).type( dtype )\n", 75 | "w2 = th.rand( H, D_out).type( dtype )\n", 76 | "\n", 77 | "learning_rate = 1e-6\n", 78 | "niters = 10\n", 79 | "error = np.zeros(niters)\n", 80 | "for t in range( niters ):\n", 81 | " \n", 82 | " #Forward\n", 83 | " z1 = x\n", 84 | " z2 = z1.mm(w1)\n", 85 | " a2 = z2.clamp(min=0)\n", 86 | " z3 = a2.mm(w2)\n", 87 | " yhat = z3\n", 88 | " \n", 89 | " loss = (yhat - y).pow(2).mean()\n", 90 | " print(t, loss)\n", 91 | " error[t] = loss\n", 92 | "\n", 93 | " # Backprop to compute gradients of w1 and w2 with respect to loss\n", 94 | " grad_y_pred = 2.0 * (yhat - y)\n", 95 | " grad_w2 = a2.t().mm(grad_y_pred)\n", 96 | " grad_h_relu = grad_y_pred.mm(w2.t())\n", 97 | " grad_h = grad_h_relu.clone()\n", 98 | " grad_h[z2 < 0] = 0\n", 99 | " grad_w1 = z1.t().mm(grad_h)\n", 100 | "\n", 101 | " # Update weights using gradient descent\n", 102 | " \n", 103 | " w1 -= learning_rate * grad_w1\n", 104 | " w2 -= learning_rate * grad_w2\n", 105 | " \n", 106 | " \n", 107 | "\n", 108 | "# plot error\n", 109 | "plt.plot(error,'-', label='error')\n", 110 | "plt.xlabel('epoch')\n", 111 | "plt.ylabel('loss')\n", 112 | "plt.legend()\n", 113 | "plt.show()\n", 114 | "\n", 115 | "\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 33, 121 | "metadata": {}, 122 | "outputs": [ 123 | { 124 | "name": "stdout", 125 | "output_type": "stream", 126 | "text": [ 127 | "0 686.0169677734375\n", 128 | "1 631.2456665039062\n", 129 | "2 584.3085327148438\n", 130 | "3 544.17822265625\n", 131 | "4 508.70062255859375\n", 132 | "5 477.228759765625\n", 133 | "6 448.806640625\n", 134 | "7 422.8643493652344\n", 135 | "8 399.0734558105469\n", 136 | "9 377.01947021484375\n" 137 | ] 138 | }, 139 | { 140 | "data": { 141 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH8lJREFUeJzt3XuQXWW95vHv05fcu0Nu0G0SSJAU3cGBIG2IxjMw5hgY\n58yAJUicEVPnUKIFx5EZakY5c0aOeii1RkV0kDkcQQMyAiKWjIViCOoRlEuCKOaCtFykQ0JC7oHc\nuvs3f+x3d3Z3Okkn3atX772fT9Wuvfa71rv63buSfnqt9e7fUkRgZmaWpZq8B2BmZpXPYWNmZplz\n2JiZWeYcNmZmljmHjZmZZc5hY2ZmmXPYmJlZ5hw2ZmaWOYeNmZllri7vAYwUU6dOjVmzZuU9DDOz\nsrJq1arXI2La0bZz2CSzZs1i5cqVeQ/DzKysSHp5INv5NJqZmWXOYWNmZplz2JiZWeZ8zcbM7Dgd\nOHCAjo4O9u7dm/dQMjdmzBhmzJhBfX39cfV32JiZHaeOjg4aGhqYNWsWkvIeTmYigi1bttDR0cHs\n2bOPax8+jWZmdpz27t3LlClTKjpoACQxZcqUQR3BOWzMzAah0oOmaLDv02EzSDvePMBNDz/P7zu2\n5z0UM7MRy9dsBqmmBm58+I/U1YozZ5yQ93DMzEYkH9kMUsOYemZOHsuaDTvzHoqZ2SG6urqO+Ppw\nOjs7h3QcDpsh0NLUyDqHjZnl4Lvf/S7z589n3rx5fOxjH6Orq4sJEyZw7bXXctZZZ/Gb3/yGWbNm\n8alPfYq3v/3tfP/73+eZZ55hwYIFnHnmmbz//e9n27ZtAJx//vlcc801tLW1cdNNNw3pOH0abQi0\nNjeyYu1r7D3QxZj62ryHY2Y5+Oz/W82aV4f2j865b2nk+n9/xmHXr127lnvuuYfHHnuM+vp6rrrq\nKu666y7eeOMNzj33XL7yla/0bDtlyhSefvppAM4880y+8Y1vcN555/GZz3yGz372s3zta18DYP/+\n/ZnUiXTYDIHWpga6A57buIuzZvq6jZkNjxUrVrBq1Sre8Y53ALBnzx5OPPFEamtr+cAHPtBr28su\nuwyAHTt2sH37ds477zwAli5dyqWXXnrIdkPNYTMEWpsbAVi3cafDxqxKHekIJCsRwdKlS/nCF77Q\nq/3LX/4ytbW9z7KMHz9+QPsc6HbHytdshsDJk8cxblQtazfsynsoZlZFFi1axH333cemTZsA2Lp1\nKy+/fOSK/xMnTmTSpEn86le/AuDOO+/sOcrJko9shkBNjTi9qYG1niRgZsNo7ty5/OM//iOLFy+m\nu7ub+vp6br755qP2W7ZsGR//+Md58803OfXUU/n2t7+d+VgdNkOktbmRH//uVSKiar5RbGb5u+yy\nyw65zrJ79+5er1966aVer+fNm8fjjz9+yL5+8YtfDPXwemR2Gk3STEk/l7RG0mpJn0zt/yBpvaRn\n0uN9JX2uk9Qu6TlJF5S0nyPp2bTu60q/zSWNlnRPan9C0qySPkslPZ8eS7N6n0WtTQ3s3NvJhh2V\nX/3VzOxYZXnNphO4NiLmAguAqyXNTetujIh56fEgQFq3BDgDuBD4pqTiFa5bgI8Cc9LjwtR+BbAt\nIk4DbgS+lPY1GbgeOBeYD1wvaVKG77VnkoBPpZmZHSqzsImIDRHxdFreBawFph+hy0XA3RGxLyJe\nBNqB+ZKagcaIeDwiArgDuLikz7K0fB+wKB31XAAsj4itEbENWM7BgMrE6U0NAKzb6EkCZtWk8Gup\n8g32fQ7LbLR0euts4InU9AlJv5d0e8kRx3TglZJuHaltelru296rT0R0AjuAKUfYV99xXSlppaSV\nmzdvPu73By5bY1aNxowZw5YtWyo+cIr3sxkzZsxx7yPzCQKSJgA/AK6JiJ2SbgE+D0R6/grwN1mP\noz8RcStwK0BbW9ug/7W0NjX6NJpZFZkxYwYdHR0M9o/VclC8U+fxyjRsJNVTCJq7IuJ+gIh4rWT9\nPwM/Ti/XAzNLus9IbevTct/20j4dkuqAicCW1H5+nz6/GIr3dCQtzY08vPY19uzvYuwol60xq3T1\n9fXHfefKapPlbDQBtwFrI+KrJe3NJZu9H/hDWn4AWJJmmM2mMBHgyYjYAOyUtCDt8yPAj0r6FGea\nXQI8kq7rPAQsljQpnaZbnNoyNbe5ULbmj6/5uo2ZWaksj2wWApcDz0p6JrX9HfAhSfMonEZ7CfgY\nQESslnQvsIbCTLarI6JYC/sq4DvAWOAn6QGFMLtTUjuwlcJsNiJiq6TPA0+l7T4XEVszep89Wppc\ntsbMrD+ZhU1EPAr09+3GB4/Q5wbghn7aVwJv66d9L3Bp3/a07nbg9oGOdyi4bI2ZWf9cG20IuWyN\nmVn/HDZDrLW5MCOt0qdCmpkdC4fNECuWrXnVZWvMzHo4bIZYz71tfCrNzKyHw2aIFcvW+LqNmdlB\nDpshVixbs9Y10szMejhsMuCyNWZmvTlsMtDS3MhLr7/Bnv1dR9/YzKwKOGwy4LI1Zma9OWwyUFq2\nxszMHDaZOHnyOMa7bI2ZWQ+HTQaKZWt8IzUzswKHTUZamhtZ57I1ZmaAwyYzrc2NLltjZpY4bDLS\nmioJuGyNmZnDJjMuW2NmdpDDJiMuW2NmdpDDJkMuW2NmVuCwyZDL1piZFThsMuSyNWZmBQ6bDBVv\npOZTaWZW7Rw2GZo5qVC2Zp0nCZhZlXPYZMhla8zMChw2GXPZGjMzh03mXLbGzMxhkzmXrTEzc9hk\nzmVrzMwcNpnrKVvjG6mZWRXLLGwkzZT0c0lrJK2W9MnUPlnScknPp+dJJX2uk9Qu6TlJF5S0nyPp\n2bTu65KU2kdLuie1PyFpVkmfpelnPC9paVbvcyBamxpZ61tEm1kVy/LIphO4NiLmAguAqyXNBT4N\nrIiIOcCK9Jq0bglwBnAh8E1JtWlftwAfBeakx4Wp/QpgW0ScBtwIfCntazJwPXAuMB+4vjTUhlur\ny9aYWZXLLGwiYkNEPJ2WdwFrgenARcCytNky4OK0fBFwd0Tsi4gXgXZgvqRmoDEiHo/C/OE7+vQp\n7us+YFE66rkAWB4RWyNiG7CcgwE17FpdtsbMqtywXLNJp7fOBp4AToqIDWnVRuCktDwdeKWkW0dq\nm56W+7b36hMRncAOYMoR9tV3XFdKWilp5ebNm4/z3R2dy9aYWbXLPGwkTQB+AFwTEb1+26Yjldy+\n7RgRt0ZEW0S0TZs2LbOf47I1ZlbtMg0bSfUUguauiLg/Nb+WTo2Rnjel9vXAzJLuM1Lb+rTct71X\nH0l1wERgyxH2lQuXrTGzapflbDQBtwFrI+KrJaseAIqzw5YCPyppX5JmmM2mMBHgyXTKbaekBWmf\nH+nTp7ivS4BH0tHSQ8BiSZPSxIDFqS03LltjZtUsyyObhcDlwHskPZMe7wO+CLxX0vPAX6bXRMRq\n4F5gDfBT4OqIKE7fugr4FoVJA38CfpLabwOmSGoH/itpZltEbAU+DzyVHp9Lbblx2Rozq2Z1We04\nIh4FdJjViw7T5wbghn7aVwJv66d9L3DpYfZ1O3D7QMebtbnNqZLAqzuZfsLYnEdjZja8XEFgmJze\nVJiRts5f7jSzKuSwGSYTRtdx8uRxLltjZlXJYTOMWpoaXLbGzKqSw2YYuWyNmVUrh80wctkaM6tW\nDpth5LI1ZlatHDbDyGVrzKxaOWyGkcvWmFm1ctgMs9bmRta6bI2ZVRmHzTBraW5kl8vWmFmVcdgM\ns9KyNWZm1cJhM8xctsbMqpHDZpi5bI2ZVSOHTQ5ctsbMqo3DJgcuW2Nm1cZhk4PW5kaXrTGzquKw\nyUFrcUaav9xpZlXCYZODYtkah42ZVQuHTQ6KZWvWukaamVUJh01OXLbGzKqJwyYnLltjZtXEYZMT\nl60xs2risMmJy9aYWTVx2OTEZWvMrJo4bHLU2tzg6c9mVhUcNjlqaWrkxS0uW2Nmlc9hk6PW5kYi\n4DmXrTGzCuewyVGxbM06n0ozswqXWdhIul3SJkl/KGn7B0nrJT2THu8rWXedpHZJz0m6oKT9HEnP\npnVfl6TUPlrSPan9CUmzSvoslfR8eizN6j0OlsvWmFm1yPLI5jvAhf203xgR89LjQQBJc4ElwBmp\nzzcl1abtbwE+CsxJj+I+rwC2RcRpwI3Al9K+JgPXA+cC84HrJU0a+rc3eC5bY2bVIrOwiYh/AbYO\ncPOLgLsjYl9EvAi0A/MlNQONEfF4FOq63AFcXNJnWVq+D1iUjnouAJZHxNaI2AYsp//QGxFctsbM\nqkEe12w+Ien36TRb8YhjOvBKyTYdqW16Wu7b3qtPRHQCO4ApR9jXiOSyNWZWDYY7bG4BTgXmARuA\nrwzzz+9F0pWSVkpauXnz5lzG4LI1ZlYNhjVsIuK1iOiKiG7gnylcUwFYD8ws2XRGaluflvu29+oj\nqQ6YCGw5wr76G8+tEdEWEW3Tpk0bzFs7bsWyNZ4kYGaVbFjDJl2DKXo/UJyp9gCwJM0wm01hIsCT\nEbEB2ClpQboe8xHgRyV9ijPNLgEeSdd1HgIWS5qUTtMtTm0jUrFszTpPEjCzClaX1Y4lfQ84H5gq\nqYPCDLHzJc0DAngJ+BhARKyWdC+wBugEro6I4tfqr6Iws20s8JP0ALgNuFNSO4WJCEvSvrZK+jzw\nVNrucxEx0IkKuXDZGjOrdJmFTUR8qJ/m246w/Q3ADf20rwTe1k/7XuDSw+zrduD2AQ82Zy1Njfxs\nzWvs2d/F2FG1R+9gZlZmBnQaTdInJTWq4DZJT0tanPXgqoXL1phZpRvoNZu/iYidFK5/TAIuB76Y\n2aiqjMvWmFmlG2jYKD2/D7gzIlaXtNkguWyNmVW6gYbNKkk/oxA2D0lqALqzG1Z1qakRLc2NLltj\nZhVroBMErqDwRcwXIuLNVH/sr7MbVvVpaWrggd+9SkSQao2amVWMgR7ZvBN4LiK2S/ow8PcUysPY\nEGlNZWvWb9+T91DMzIbcQMPmFuBNSWcB1wJ/olAU04bIwUkCPpVmZpVnoGHTmb6dfxHwvyPiZqAh\nu2FVH5etMbNKNtBrNrskXUdhyvNfSKoB6rMbVvVx2Rozq2QDPbK5DNhH4fs2GykUt/xfmY2qSrls\njZlVqgGFTQqYu4CJkv4K2BsRvmYzxFqaGnlxyxvs2d919I3NzMrIQMvVfBB4kkItsg8CT0i6JMuB\nVSOXrTGzSjXQazb/A3hHRGwCkDQNeJjC7ZhtiMxtLkwSWLdhJ/NmnpDzaMzMhs5Ar9nUFIMm2XIM\nfW2AZkwa67I1ZlaRBnpk81NJDwHfS68vAx7MZkjVq6dsjb9rY2YVZkBhExH/TdIHgIWp6daI+GF2\nw6peLltjZpVowDdPi4gfAD/IcCxGYZLAXU/8mfXb9zBj0ri8h2NmNiSOGDaSdlG4hfMhq4CIiMZM\nRlXFSsvWOGzMrFIc8SJ/RDRERGM/jwYHTTZctsbMKpFnlI0wE0bXccoUl60xs8risBmBWppctsbM\nKovDZgRqbS6UrXlzf2feQzEzGxIOmxGopalQtuaPr+3OeyhmZkPCYTMCFcvW+FSamVUKh80IVCxb\ns85hY2YVwmEzArlsjZlVGofNCNXS1MDajTsp3I3bzKy8OWxGqNbmRnbt7WT99j15D8XMbNAyCxtJ\nt0vaJOkPJW2TJS2X9Hx6nlSy7jpJ7ZKek3RBSfs5kp5N676uVJ1S0mhJ96T2JyTNKumzNP2M5yUt\nzeo9Zqm15942PpVmZuUvyyOb7wAX9mn7NLAiIuYAK9JrJM0FlgBnpD7flFSb+twCfBSYkx7FfV4B\nbIuI04AbgS+lfU0GrgfOBeYD15eGWrk4valQI80z0sysEmQWNhHxL8DWPs0XAcvS8jLg4pL2uyNi\nX0S8CLQD8yU1A40R8XgULl7c0adPcV/3AYvSUc8FwPKI2BoR24DlHBp6I16xbM3ajQ4bMyt/w33N\n5qSI2JCWNwInpeXpwCsl23WktulpuW97rz4R0QnsAKYcYV9lp6WpwafRzKwi5DZBIB2p5DrVStKV\nklZKWrl58+Y8h9Ivl60xs0ox3GHzWjo1RnrelNrXAzNLtpuR2tan5b7tvfpIqgMmAluOsK9DRMSt\nEdEWEW3Tpk0bxNvKhsvWmFmlGO6weQAozg5bCvyopH1JmmE2m8JEgCfTKbedkhak6zEf6dOnuK9L\ngEfS0dJDwGJJk9LEgMWprey4bI2ZVYoB3xb6WEn6HnA+MFVSB4UZYl8E7pV0BfAy8EGAiFgt6V5g\nDdAJXB0RXWlXV1GY2TYW+El6ANwG3CmpncJEhCVpX1slfR54Km33uYjoO1GhLLhsjZlVCvkb6gVt\nbW2xcuXKvIdxiA/c8mtqJe79+DvzHoqZ2SEkrYqItqNt5woCI1xrs8vWmFn5c9iMcC1NLltjZuXP\nYTPCtfZMEvD3bcysfDlsRrhi2RpPEjCzcuawGeFctsbMKoHDpgy4bI2ZlTuHTRlw2RozK3cOmzLQ\n2uyyNWZW3hw2ZaC1yWVrzKy8OWzKwIxJY5kwus4z0sysbDlsykBNjTi9qcHftTGzsuWwKRMuW2Nm\n5cxhUyZctsbMypnDpky4bI2ZlTOHTZlw2RozK2cOmzLhsjVmVs4cNmWktanRZWvMrCw5bMpIS3OD\ny9aYWVly2JSRYtma5zb66MbMyovDpowUy9asc9iYWZlx2JSRYtka10gzs3LjsCkjxbI1niRgZuXG\nYVNmXLbGzMqRw6bMtDa7bI2ZlR+HTZlpaXLZGjMrPw6bMtPisjVmVoYcNmVmvMvWmFkZctiUodam\nRp9GM7OykkvYSHpJ0rOSnpG0MrVNlrRc0vPpeVLJ9tdJapf0nKQLStrPSftpl/R1SUrtoyXdk9qf\nkDRruN9jllqaG3jJZWvMrIzkeWTzbyJiXkS0pdefBlZExBxgRXqNpLnAEuAM4ELgm5JqU59bgI8C\nc9LjwtR+BbAtIk4DbgS+NAzvZ9i4bI2ZlZuRdBrtImBZWl4GXFzSfndE7IuIF4F2YL6kZqAxIh6P\nwpdO7ujTp7iv+4BFxaOeSuCyNWZWbvIKmwAelrRK0pWp7aSI2JCWNwInpeXpwCslfTtS2/S03Le9\nV5+I6AR2AFOG+k3kxWVrzKzc1OX0c98dEeslnQgsl7SudGVEhKTMvyKfgu5KgJNPPjnrHzdkampE\ni8vWmFkZyeXIJiLWp+dNwA+B+cBr6dQY6XlT2nw9MLOk+4zUtj4t923v1UdSHTAR2NLPOG6NiLaI\naJs2bdrQvLlh0uKyNWZWRoY9bCSNl9RQXAYWA38AHgCWps2WAj9Kyw8AS9IMs9kUJgI8mU657ZS0\nIF2P+UifPsV9XQI8EhX2W9lla8ysnORxGu0k4Ifpen0d8H8j4qeSngLulXQF8DLwQYCIWC3pXmAN\n0AlcHRFdaV9XAd8BxgI/SQ+A24A7JbUDWynMZqsopWVrZkwal/NozMyObNjDJiJeAM7qp30LsOgw\nfW4AbuinfSXwtn7a9wKXDnqwI1ixbM3aDTt579yTjrK1mVm+RtLUZzsGxbI161y2xszKgMOmjLls\njZmVC4dNGXPZGjMrFw6bMuayNWZWLhw2ZWxus8vWmFl5cNiUseknuGyNmZUHh00ZK5atcdiY2Ujn\nsClzLc2FGmkVViDBzCqMw6bMtTY3smtfJx3bXLbGzEYuh02Za/G9bcysDDhsylxLUwMSvm5jZiOa\nw6bMjR9dxymTXbbGzEY2h00FaHHZGjMb4Rw2FaC1udFla8xsRHPYVICW5gaXrTGzEc1hUwGKZWt8\nKs3MRiqHTQUolq3xJAEzG6kcNhXAZWvMbKRz2FQIl60xs5HMYVMhimVr7n96PTvePJD3cMzMeqnL\newA2NBacOoWGMXVc+/3fIcG/mj6RhadN5d2nTeWcUyYxpr427yGaWRWTT7sUtLW1xcqVK/MexqAc\n6Ormd69s59H213ms/XV+++ftdHYHo+tqaJs1qSd8znjLRGprlPdwzawCSFoVEW1H3c5hU1AJYdPX\n7n2dPPniFh5r38Jj7a/3FOucOLaed546hYVzCuEza8o4JIePmR27gYaNT6NVsAmj63hPy0m8p+Uk\nADbt2stv/lQInkeff52frt4IFKZOv+utU3j3nKm8661TmdYwOs9hm1kF8pFNUolHNkcSEby05U0e\nbX+dX7e/zq//tIUdewoTC1qaGnjXW6fy7jlTmD97ChNG+28SM+ufT6Mdo2oLm766uoPVr+7oud7z\n1Evb2N/ZTV2NOPvkE1h42lQWnjaVeTNPoL7WkxjNrMBhc4yqPWz62nugi1Uvb+sJn2fX7yACxo+q\n5dxTp/Scdjv9pAZf7zGrYr5mY4Mypr6252gGYPub+3n8hS0pfLbwyLpNAEydMJqFp01h4VunsnDO\nVKafMDbPYZvZCFXRRzaSLgRuAmqBb0XEFw+3rY9sjs367Xt4LB31PNb+Oq/v3g/A7KnjOeeUSZww\ntp4JY+qYMLqOhjF1TBjd93UdE8bUMX5Unadhm5Wxqj+NJqkW+CPwXqADeAr4UESs6W97h83xiwie\ne21XzxTrZ9fvYPfeTvYc6BpQ//GjanuCaMKYehpG1zF+dC0TRtf3CqZeQZXaGkpCbFSdryWZDTef\nRoP5QHtEvAAg6W7gIqDfsLHjJ4mWpkZamhq54t2ze9o7u7p5Y18Xu/Yd4I19Xezed4BdezvZva+T\n3en5kNf7Otm99wCbd+1L6w+we18n3QP4m2hUXU0KqkL4jBtVS22NqK+tSc+itkbU1dRQl5bra2qo\nrRX1NaK2pqZkG1HXq1+fdWkfdTW99138WcX+dTWiRqKmBoSoUeHzkqBGQqRncbCt5Lm0z4D69mrz\nEaONHJUcNtOBV0pedwDn5jSWqlRXW8PEcTVMHFc/qP1EBHsOdLF7bzGM+gZVIZB2p0Arrn9zfxed\n3YW+nV3ddHYHnV1BZ/fB5a7u3q87u7vTc2Uc8fcKKlL4lDypZ7mwUAi41JZWqk+fXusOs/3BnOtv\nu95j7BlXP+v7xuXRArRX32P4Of3u68irBzaeAexjYBsd86aF7Qf4B0drcyPf+NDZx7j3Y1PJYXNU\nkq4ErgQ4+eSTcx6NHY4kxo2qY9yoOk4cpp8ZEXRHoQRQV9+Q6g66uoID3YV1xW0OlIZXWj7Q1U13\nBBHQHRAU9hs9bb2fD64vtgUBdHen5759Kekbh+mb2gvvKz0T0LN88D3HIW1p25K+pZ9R6XbF/fa3\nDw67jz6viSOsG3jfvhsf2vfIf0wM5E+No12BGNg+Bv5HzTH/+XMMHWZOyn5iTyWHzXpgZsnrGamt\nR0TcCtwKhWs2wzc0G+kkUSuorXEBU7OhUMlXVJ8C5kiaLWkUsAR4IOcxmZlVpYo9somITkl/CzxE\nYerz7RGxOudhmZlVpYoNG4CIeBB4MO9xmJlVu0o+jWZmZiOEw8bMzDLnsDEzs8w5bMzMLHMOGzMz\ny1zFFuI8VpI2Ay8PYhdTgdeHaDjlzp9Fb/48evPncVAlfBanRMS0o23ksBkiklYOpPJpNfBn0Zs/\nj978eRxUTZ+FT6OZmVnmHDZmZpY5h83QuTXvAYwg/ix68+fRmz+Pg6rms/A1GzMzy5yPbMzMLHMO\nm0GSdKGk5yS1S/p03uPJk6SZkn4uaY2k1ZI+mfeY8iapVtJvJf0477HkTdIJku6TtE7SWknvzHtM\neZL0X9L/kz9I+p6kMXmPKUsOm0GQVAvcDPxbYC7wIUlz8x1VrjqBayNiLrAAuLrKPw+ATwJr8x7E\nCHET8NOIaAHOooo/F0nTgf8MtEXE2yjcBmVJvqPKlsNmcOYD7RHxQkTsB+4GLsp5TLmJiA0R8XRa\n3kXhl8n0fEeVH0kzgH8HfCvvseRN0kTgXwO3AUTE/ojYnu+oclcHjJVUB4wDXs15PJly2AzOdOCV\nktcdVPEv11KSZgFnA0/kO5JcfQ3470B33gMZAWYDm4Fvp9OK35I0Pu9B5SUi1gNfBv4MbAB2RMTP\n8h1Vthw2NuQkTQB+AFwTETvzHk8eJP0VsCkiVuU9lhGiDng7cEtEnA28AVTtNU5JkyicBZkNvAUY\nL+nD+Y4qWw6bwVkPzCx5PSO1VS1J9RSC5q6IuD/v8eRoIfAfJL1E4fTqeyR9N98h5aoD6IiI4pHu\nfRTCp1r9JfBiRGyOiAPA/cC7ch5Tphw2g/MUMEfSbEmjKFzgeyDnMeVGkiick18bEV/Nezx5iojr\nImJGRMyi8O/ikYio6L9cjyQiNgKvSDo9NS0C1uQ4pLz9GVggaVz6f7OICp8wUZf3AMpZRHRK+lvg\nIQqzSW6PiNU5DytPC4HLgWclPZPa/i4iHsxxTDZyfAK4K/1h9gLw1zmPJzcR8YSk+4CnKczi/C0V\nXk3AFQTMzCxzPo1mZmaZc9iYmVnmHDZmZpY5h42ZmWXOYWNmZplz2JhVAEnnu7K0jWQOGzMzy5zD\nxmwYSfqwpCclPSPpn9L9bnZLujHd22SFpGlp23mSHpf0e0k/TPW0kHSapIcl/U7S05LemnY/oeR+\nMXelb6abjQgOG7NhIqkVuAxYGBHzgC7gPwHjgZURcQbwS+D61OUO4FMRcSbwbEn7XcDNEXEWhXpa\nG1L72cA1FO6tdCqFig5mI4LL1ZgNn0XAOcBT6aBjLLCJwi0I7knbfBe4P93/5YSI+GVqXwZ8X1ID\nMD0ifggQEXsB0v6ejIiO9PoZYBbwaPZvy+zoHDZmw0fAsoi4rlej9D/7bHe8NaT2lSx34f/fNoL4\nNJrZ8FkBXCLpRABJkyWdQuH/4SVpm/8IPBoRO4Btkv4itV8O/DLdAbVD0sVpH6MljRvWd2F2HPyX\nj9kwiYg1kv4e+JmkGuAAcDWFG4nNT+s2UbiuA7AU+D8pTEqrJF8O/JOkz6V9XDqMb8PsuLjqs1nO\nJO2OiAl5j8MsSz6NZmZmmfORjZmZZc5HNmZmljmHjZmZZc5hY2ZmmXPYmJlZ5hw2ZmaWOYeNmZll\n7v8DrsPUGk9B9zgAAAAASUVORK5CYII=\n", 142 | "text/plain": [ 143 | "" 144 | ] 145 | }, 146 | "metadata": {}, 147 | "output_type": "display_data" 148 | } 149 | ], 150 | "source": [ 151 | "\n", 152 | "\n", 153 | "N, D_in, H, D_out = 64, 1000, 100, 10\n", 154 | "\n", 155 | "x = Variable(th.randn(N, D_in))\n", 156 | "y = Variable(th.randn(N, D_out), requires_grad=False)\n", 157 | "\n", 158 | "model = th.nn.Sequential(\n", 159 | " th.nn.Linear(D_in, H),\n", 160 | " th.nn.ReLU(),\n", 161 | " th.nn.Linear(H, D_out),\n", 162 | ")\n", 163 | "\n", 164 | "loss_fn = th.nn.MSELoss(size_average=False)\n", 165 | "\n", 166 | "learning_rate = 1e-4\n", 167 | "niter = 10\n", 168 | "erros = np.zeros(niter)\n", 169 | "for t in range(niter):\n", 170 | " \n", 171 | " \n", 172 | " yhat = model(x)\n", 173 | " loss = loss_fn(yhat,y)\n", 174 | " print(t,loss.data[0])\n", 175 | " erros[t] = loss.data[0]\n", 176 | " \n", 177 | " model.zero_grad()\n", 178 | " loss.backward()\n", 179 | " \n", 180 | " for param in model.parameters():\n", 181 | " param.data -= learning_rate*param.grad.data\n", 182 | " \n", 183 | " \n", 184 | "# plot error\n", 185 | "plt.plot(error,'-', label='error')\n", 186 | "plt.xlabel('epoch')\n", 187 | "plt.ylabel('loss')\n", 188 | "plt.legend()\n", 189 | "plt.show()\n" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 36, 195 | "metadata": {}, 196 | "outputs": [ 197 | { 198 | "name": "stdout", 199 | "output_type": "stream", 200 | "text": [ 201 | "0 719.0383911132812\n", 202 | "1 665.91845703125\n", 203 | "2 619.938232421875\n", 204 | "3 579.8974609375\n", 205 | "4 544.4142456054688\n", 206 | "5 513.1660766601562\n", 207 | "6 484.68170166015625\n", 208 | "7 458.2241516113281\n", 209 | "8 433.6163330078125\n", 210 | "9 410.652099609375\n" 211 | ] 212 | }, 213 | { 214 | "data": { 215 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VdUd7vHvLyEQCDMyB0kYZFIIEJB5EEVUFK0KOCCC\nCgpFtFqrvbbXtt7aqhQoIjiBKCiCI4KiiMo8JUwCAQmTBAJEZkEISdb9I9s22AABcrJzct7P8+Q5\n+6ysffLL0fCetYe1zDmHiIjIr4X5XYCIiBROCggREcmVAkJERHKlgBARkVwpIEREJFcKCBERyZUC\nQkREcqWAEBGRXCkgREQkV8X8LuBiXHLJJS4mJsbvMkREgkpiYuKPzrnK5+oX1AERExNDQkKC32WI\niAQVM9uRl346xCQiIrlSQIiISK4UECIikqugPgchInK+Tp06RUpKCidOnPC7lICLjIwkOjqaiIiI\nC9pfASEiISUlJYUyZcoQExODmfldTsA459i/fz8pKSnExsZe0GvoEJOIhJQTJ05QqVKlIh0OAGZG\npUqVLmqkpIAQkZBT1MPhFxf7e4ZkQPycnskzM9Zz+Pgpv0sRESm0QjIg1u8+zDvLfqDfhGUc/lkh\nISKSm5AMiPiYiozv14Kk1CPc84ZCQkQKl8zMzLM+P5OMjIx8rSMkAwLgqoZVGX93SzakHuGeCcs5\nckIhISIFY/LkybRu3Zq4uDgGDx5MZmYmpUuX5rHHHqNZs2YsWbKEmJgY/vCHP9CiRQumT5/O6tWr\nadOmDU2bNuWWW27h4MGDAHTp0oVHHnmE+Ph4Ro8ena91hvRlrt0aVWXcXS15aEoi97yxnLfva02Z\nyAu7XlhEgs9fPl3Pht1H8vU1G9coy/+9sckZv5+UlMR7773HokWLiIiIYMiQIUyZMoVjx45x5ZVX\nMmLEiP/0rVSpEitXrgSgadOmjBkzhs6dO/PnP/+Zv/zlL4waNQqA9PT0gMxLF7IjiF9c3bgqY+9s\nwbpdh+k/YTlHNZIQkQCaO3cuiYmJtGrViri4OObOncvWrVsJDw/n1ltvPa1vnz59ADh8+DCHDh2i\nc+fOAPTv35/58+f/T7/8FtIjiF90b1KNl+5swW/fWcm9E1cwaWBrSpfQWyNS1J3tk36gOOfo378/\nzz333GntL774IuHh4ae1RUVF5ek189rvfIX8COIXPS6vxkt3NmfNzkPcO2E5P53M35M9IiIA3bp1\n4/3332ffvn0AHDhwgB07zj77drly5ahQoQILFiwA4O233/7PaCKQ9DE5hx6XV2fMHfDbd1cxYOJy\n3hzQmiiNJEQkHzVu3Jhnn32W7t27k5WVRUREBGPHjj3nfpMmTeLBBx/k+PHj1KlTh4kTJwa8VnPO\nBfyHBEp8fLwLxImZWWtTeXjqKlpeWoGJA1opJESKkKSkJBo1auR3GQUmt9/XzBKdc/Hn2jdgh5jM\nrIGZrc7xdcTMHjGzimY2x8w2e48VcuzzlJklm9kmM7s2ULWdyw1NqzO6bxyJPxxkwJsrOJ6uw00i\nEnoCFhDOuU3OuTjnXBzQEjgOfAQ8Ccx1ztUH5nrPMbPGQF+gCdADeNnMwnN98QLQs2kNRvWJI2H7\nAQYqJEQkBBXUSepuwBbn3A6gFzDJa58E3Oxt9wKmOudOOue2AclA6wKqL1c3NqvByD5xLN92gPve\nTODn9LzdzSgihVswH1o/Hxf7exZUQPQF3vW2qzrnUr3tPUBVb7smsDPHPile22nMbJCZJZhZQlpa\nWqDq/Y9ecTUZ2SeOZdv2c9+kFQoJkSAXGRnJ/v37i3xI/LIeRGRk5AW/RsDPvppZceAm4Klff885\n58zsvP4rOedeBV6F7JPU+VLkOfSKq0mWc/xu2hruf2sFb/RvRWSEb0e/ROQiREdHk5KSQkF8wPTb\nLyvKXaiCuDznOmClc26v93yvmVV3zqWaWXVgn9e+C6iVY79or61QuKV5NFlZ8Pj7a3jgrQReuyde\nISEShCIiIi54hbVQUxCHmO7gv4eXAGYA/b3t/sAnOdr7mlkJM4sF6gPLC6C+PLu1ZTQv3NaMhck/\n8sBbCZw4pcNNIlJ0BTQgzCwKuAb4MEfzP4BrzGwzcLX3HOfcemAasAGYDQx1zhW6f4FvaxnN87c2\nZWHyjwx6O1EhISJFlm6Uu0DTVuzkiQ/W0vmyyrzSr6UON4lI0PD9RrmirnerWvzjN1cw7/s0Hpqc\nyMkMjSREpGhRQFyEvq0v5bnfXME3m9J4aPJKhYSIFCkKiIt0R+tL+fstV/D1xn0MnaKQEJGiQwGR\nD+688lKevflyvkrax9Apq0jPyPK7JBGRi6aAyCd3t6nN33o14aukvQx9Z6VCQkSCngIiH/VrG8Nf\nezVhzoa9DHt3JacyFRIiErwUEPnsnrYxPHNjY75Yv5dh76xSSIhI0FJABMC97WP5c8/GzF6/h4ff\nVUiISHBSQATIwA6xPH1DIz5ft4dHpq4mQyEhIkFGa2kG0P0d6wDw7KwkzGBUnziKhSuTRSQ4KCAC\n7P6OdXAO/t9nSZgZI3s3U0iISFBQQBSABzrVIcs5nvt8Iwb8SyEhIkFAAVFABneuS5aDf87eSJjB\niN5xhIeZ32WJiJyRAqIAPdSlLlnO8cIXmzAzXry9mUJCRAotBUQBG9q1HkB2SAAvKCREpJBSQPhg\naNd6ZGU5Rsz5HjPjn7deoXMSIlLoKCB8MqxbfbIcjPzqew4dT2fMnc0pVVz/OUSk8NDHVh8Nv7o+\nf7v5cr7ZtI8+ryxl39ETfpckIvIfCgif9WtTm9fuiSd530/cMnYxm/ce9bskERFAAVEodGtUlWmD\n25KemcWt4xazZMt+v0sSEVFAFBZXRJfjoyHtqFo2knsmLOOjVSl+lyQiIU4BUYhEVyjF+w+1o2Xt\nCjz63hrGzN2Mc87vskQkRCkgCplyJSOYNLA1tzSvyYg53/PkB99punAR8UVAA8LMypvZ+2a20cyS\nzKytmT1jZrvMbLX3dX2O/k+ZWbKZbTKzawNZW2FWolg4/+rdjGFX1eO9hJ3cNymBoydO+V2WiISY\nQI8gRgOznXMNgWZAktc+0jkX5319BmBmjYG+QBOgB/CymYUHuL5Cy8x4rHsDnr+1KYuTf+T28UtI\nPfyz32WJSAgJWECYWTmgE/AGgHMu3Tl36Cy79AKmOudOOue2AclA60DVFyx6t6rFhHtbkXLwZ24Z\nu5ik1CN+lyQiISKQI4hYIA2YaGarzOx1M4vyvjfMzNaa2QQzq+C11QR25tg/xWs7jZkNMrMEM0tI\nS0sLYPmFR6fLKjNtcFsAbh+/hPnfh8bvLSL+CmRAFANaAOOcc82BY8CTwDigDhAHpAIjzudFnXOv\nOufinXPxlStXzueSC6/GNcry0dB2RFcoycA3VzBtxc5z7yQichECGRApQIpzbpn3/H2ghXNur3Mu\n0zmXBbzGfw8j7QJq5dg/2msTT/VyJZn+YFva1q3EEx+sZcSXm3QZrIgETMACwjm3B9hpZg28pm7A\nBjOrnqPbLcA6b3sG0NfMSphZLFAfWB6o+oJVmcgIJtzbij7xtRjzdTKPTVtDeoYugxWR/Bfo6UOH\nAVPMrDiwFRgA/NvM4gAHbAcGAzjn1pvZNGADkAEMdc5lBri+oBQRHsY/br2C6AolGTHne1IPn2B8\nv5aUKxnhd2kiUoRYMB+iiI+PdwkJCX6X4auPVqXwxPtriakUxcQBrYiuUMrvkkSkkDOzROdc/Ln6\n6U7qIHdL82jeGngle4+c4JaXF/NdymG/SxKRIkIBUQS0rVuJDx5qR/HwMHq/soSvN+71uyQRKQIU\nEEVE/apl+GhoO+pVKc39kxJ4e+kOv0sSkSCngChCqpSJZOqgNnRtUIU/fbyO5z5LIisreM8xiYi/\nFBBFTFSJYrzSryX92tTmlflbGTZ1FSdO6WIwETl/gb7MVXxQLDyMv/ZqQq2KJfn7ZxvZe/gEr90T\nT4Wo4n6XJiJBRCOIIsrMGNSpLi/d2Zy1uw5z67jF7Nh/zO+yRCSIKCCKuJ5Na/DO/Vdy8Hg6v3l5\nMSt/OOh3SSISJBQQISA+piIfDmlP6chi3PHqUmav2+N3SSISBBQQISL2kig+fKgdjWuU5aEpibyx\ncJvfJYlIIaeACCGVSpfg3QfacG3javxt5gaembGeTF0GKyJnoIAIMZER4Yy9qwX3dYjlzcXbeWhy\nIj+n6zJYEflfCogQFB5m/KlnY565sTFzkvbS97Wl/PjTSb/LEpFCRgERwu5tH8srd7dk054j9Hpp\nEWtTzrZkuIiEGgVEiOvepBrvDcpe7/q2cUt4e8l2rVInIoACQoBmtcozc1gH2terxJ8+Wc+wd1fx\n08kMv8sSEZ8pIASAClHFeaN/K57o0YDPvkvlpjELSUo94ndZIuIjBYT8R1iYMaRLPd55oA0/nczg\n5rGLmJaw0++yRMQnCgj5H23qVGLWwx2Jj6nAE++v5fHpa3QprEgIUkBIriqXKcFbA6/k4W71+WBl\nCjePXUTyvp/8LktECpACQs4oPMz43TWXMWlAa9J+OkmvlxYyY81uv8sSkQKigJBz6nRZZWY93IFG\n1cvy8LurePrj77QIkUgIUEBInlQvV5J3B7VhcKc6TF76A7eNX8wP+4/7XZaIBFBAA8LMypvZ+2a2\n0cySzKytmVU0szlmttl7rJCj/1Nmlmxmm8zs2kDWJucvIjyMp65vxGv3xPPD/uPcMGYBX6zX1OEi\nRVWgRxCjgdnOuYZAMyAJeBKY65yrD8z1nmNmjYG+QBOgB/CymYUHuD65ANc0rsqshzsSe0kUg99O\n5NmZGziVmeV3WSKSzwIWEGZWDugEvAHgnEt3zh0CegGTvG6TgJu97V7AVOfcSefcNiAZaB2o+uTi\n1KpYiukPtqV/29q8vnAbfV5Zwu5DP/tdlojko0COIGKBNGCima0ys9fNLAqo6pxL9frsAap62zWB\nnHdlpXhtUkiVKBbOX3pdzpg7mrNpz1Fu+PcCvt20z++yRCSfBDIgigEtgHHOuebAMbzDSb9w2bPC\nndfMcGY2yMwSzCwhLS0t34qVC3djsxp8OqwDVctGMuDNFYz4cpMWIhIpAgIZEClAinNumff8fbID\nY6+ZVQfwHn/5yLkLqJVj/2iv7TTOuVedc/HOufjKlSsHrHg5P3Uql+ajIe3p3bIWY75O5u7Xl7Hv\n6Am/yxKRixCwgHDO7QF2mlkDr6kbsAGYAfT32voDn3jbM4C+ZlbCzGKB+sDyQNUn+a9k8XD+eVtT\nXry9Gat2HuT60QtZsmW/32WJyAUK9FVMw4ApZrYWiAP+DvwDuMbMNgNXe89xzq0HppEdIrOBoc45\n3Y0VhG5rGc0nQztQtmQx7np9KWO/SSZLh5xEgo4F8+Iw8fHxLiEhwe8y5Ax+OpnBHz/8jhlrdtP5\nssqM7BNHxajifpclEvLMLNE5F3+ufrqTWgKmdIlijO4bx7M3X86SLfu54d8LSNxx0O+yRCSPFBAS\nUGbG3W1q8+GQdhQLN/q8soTXF2zVsqYiQUABIQXi8prlmDmsI1c1rMKzs5J4cHIih38+5XdZInIW\nCggpMOVKRvBKv5Y8fUMj5ibto+eYBXyXctjvskTkDBQQUqDMjPs71uG9wW3JyHTcOm4xk5fu0CEn\nkUJIASG+aFm7ArMe7kjbupV4+uN1DJ+6mp9OZvhdlojkoIAQ31SMKs7Ee1vx+2sbMHPtbm56aSEb\n9xzxuywR8SggxFdhYcbQrvWYfP+VHPk5g5vGLGLct1s0l5NIIaCAkEKhXd1LmP1I9lVO/5y9kdvG\nL2ZL2k9+lyUS0hQQUmhcUroE4+5uwei+cWxNO8b1oxfw+oKtGk2I+CRPAWFmw82srGV7w8xWmln3\nQBcnocfM6BVXkzmPdqJj/Ut4dlYSfV5ZwrYfj/ldmkjIyesIYqBz7gjQHagA9MObZE8kEKqUjeS1\ne+L5V+9mfL/3KNeNns+bi7Zp0j+RApTXgDDv8XrgbW/mVTtLf5GLZmb8pkU0Xz7amTZ1KvHMpxu4\n47Wl/LD/uN+liYSEvAZEopl9SXZAfGFmZQCtUi8Folq5SCbe24rnb23Kht1H6DF6Pm8v3aHRhEiA\n5TUg7iN7udBWzrnjQAQwIGBVifyKmdG7VS1mP9qJlrUr8KeP19FvwjJSDmo0IRIoeQ2ItsAm59wh\nM7sbeBrQJDpS4GqWL8lbA1vz91uuYPUPh+gxagFTl/+gqTpEAiCvATEOOG5mzYDHgC3AWwGrSuQs\nzIw7r7yU2Y904oqa5Xjyw+/oP3EFqYd/9rs0kSIlrwGR4bI/ovUCXnLOjQXKBK4skXOrVbEUU+6/\nkr/1asKKbQfoPnI+0xN2ajQhkk/yGhBHzewpsi9vnWVmYWSfhxDxVViY0a9tDLMf6UijamX5/ftr\nuX9SAnuPnPC7NJGgl9eA6AOcJPt+iD1ANPBCwKoSOU+1K0UxdVAb/tyzMYu2/Ej3kfP5eNUujSZE\nLkKeAsILhSlAOTPrCZxwzukchBQqYWHGwA6xfPZwR+pVKc0j761m8NuJpB096XdpIkEpr1Nt9AaW\nA7cDvYFlZnZbIAsTuVB1Kpdm2uC2/PH6hnz7fRrdR87j0zW7/S5LJOhYXobgZrYGuMY5t897Xhn4\nyjnXLMD1nVV8fLxLSEjwswQp5JL3HeWx6WtZs/MQN1xRnb/2akKl0iX8LkvEV2aW6JyLP1e/vJ6D\nCPslHDz787KvmW03s+/MbLWZJXhtz5jZLq9ttZldn6P/U2aWbGabzOzaPNYmckb1qpThgwfb8kSP\nBszZsJfuI+cze12q32WJBIVieew328y+AN71nvcBPsvjvl2dcz/+qm2kc+7FnA1m1hjoCzQBagBf\nmdllzrnMPP4ckVwVCw9jSJd6dGtYlcemr+bBySu5qVkN/nJTEypEFfe7PJFCK68nqX8PvAo09b5e\ndc79IZ9r6QVMdc6ddM5tA5KB1vn8MySENahWho+GtOexay7j83WpdB81nzkb9vpdlkihlecFg5xz\nHzjnfud9fZTX3cgeCSSa2aAc7cPMbK2ZTTCzCl5bTWBnjj4pXptIvokID2NYt/p8MrQDl5QuwQNv\nJfC7aas5fPyU36WJFDpnDQgzO2pmR3L5OmpmeVldvoNzLg64DhhqZp3InrajDhAHpAIjzqdgMxtk\nZglmlpCWlnY+u4r8R+MaZflkaHse7lafT1bvpvuoeXyzad+5dxQJIWcNCOdcGedc2Vy+yjjnyp7r\nxZ1zu7zHfcBHQGvn3F7nXKZzLgt4jf8eRtoF1Mqxe7TX9uvXfNU5F++ci69cuXLefkuRXBQvFsbv\nrrmMj4e0p3zJ4gyYuII/vL+WIyc0mhCBAK5JbWZR3roRmFkU2avRrTOz6jm63QKs87ZnAH3NrISZ\nxQL1yb73QiSgrogux4xh7RnatS7TE3fSY+R8vt6ocxMiAQsIoCqw0LuHYjkwyzk3G3jeu/R1LdAV\neBTAW6VuGrABmA0M1RVMUlBKFAvn99c25MMh7YkqUYyBbyZw/6QV7NivtbAldOXpRrnCSjfKSSCk\nZ2Tx5uJtjP5qM6eyHIM71WFIl3qULB7ud2ki+SK/b5QTCRnFi4UxqFNdvn68CzdcUZ0xXyfTbcS3\nfPZdqib/k5CigBA5g6plIxnZJ47pD7alXKniDJmykrvfWMbmvUf9Lk2kQCggRM6hVUxFZg7rwN96\nNWHdriNcN3oBz87cwFFd7SRFnAJCJA/CvYWJvnm8C7fH1+KNRdvo+uI8PkhMIStLh52kaFJAiJyH\nilHFee43V/DJ0PZEVyjJY9PXcNv4xazbddjv0kTynQJC5AI0jS7Phw+144XbmvLDgePc+NJC/s9H\n33HwWLrfpYnkGwWEyAUKCzNuj6/F1493YUC7WKau2EnXEd8yeekOMnXYSYoABYTIRSobGcGfb2zM\nZw93pFG1sjz98TpuemkhiTsO+F2ayEVRQIjkkwbVyvDOA1fy0p3NOXAsnVvHLeF301az7+gJv0sT\nuSAKCJF8ZGb0bFqDr37XmSFd6jJzTSpXvTiP1xds5VRmlt/liZwXBYRIAESVKMYTPRryxaOdaBVT\ngWdnJXHd6AUsSv714ooihZcCQiSAYi+JYuKA1rzRP570jCzuen0ZQ6YksuvQz36XJnJOeV2TWkQu\nQrdGVWlf7xJem7+Vsd8m8/XGfQztUo8HOtUhMkKTAErhpBGESAGJjAhnWLf6zH2sC1c1rMKIOd/T\nfeR85iZp7QkpnBQQIgWsZvmSvHxXS6bcfyXFi4Vx36QEBkxczrYftfaEFC4KCBGftK93CZ8P78jT\nNzRixfaDXDtyPi98sZHj6Rl+lyYCKCBEfBURHsb9Hevw9eOd6dmsOmO/2UK3EfP4dM1urT0hvlNA\niBQCVcpE8q/ecXzwUFsqRhVn2LuruOO1pazfrUkAxT8KCJFCpGXtisz4bQeevflyNu45yg3/Xsgj\nU1ex88Bxv0uTEKQ1qUUKqcM/n2L8vC1MWLgN5+DuNrX57VX1qBhV3O/SJMjldU1qBYRIIbfn8AlG\nffU90xJ2ElW8GIM712Fgh1hKFddtTHJhFBAiRUzyvqM8P3sTX27YS5UyJXjk6svoHR9NsXAdKZbz\nk9eA0P9ZIkGiXpUyvHpPPB881JZLK5bijx99R/eR85m9LlVXPElABDQgzGy7mX1nZqvNLMFrq2hm\nc8xss/dYIUf/p8ws2cw2mdm1gaxNJFi1rF2R6Q+25bV74gkLMx6cvJJbXl7Msq37/S5NipiCGEF0\ndc7F5RjOPAnMdc7VB+Z6zzGzxkBfoAnQA3jZzDRJjUguzIxrGldl9vCOPH9rU/YcPkGfV5dy35sr\n2LTnqN/lSRHhxyGmXsAkb3sScHOO9qnOuZPOuW1AMtDah/pEgkax8DB6t6rFN4934Q89GrJ8+wF6\njJ7P49PXaMZYuWiBDggHfGVmiWY2yGur6pxL9bb3AFW97ZrAzhz7pnhtInIOJYuH81CXuix4oisP\ndKzDjDW76frit/z9syQOHU/3uzwJUoEOiA7OuTjgOmComXXK+U2XfWbtvM6umdkgM0sws4S0tLR8\nLFUk+JUvVZw/Xt+Ibx7vwk3NavDagq10ev4bxs/bwolTmX6XJ0EmoAHhnNvlPe4DPiL7kNFeM6sO\n4D3u87rvAmrl2D3aa/v1a77qnIt3zsVXrlw5kOWLBK2a5Uvy4u3N+Hx4R+JjKvKPzzfS5YVvmbZi\nJ5lZuuJJ8iZgAWFmUWZW5pdtoDuwDpgB9Pe69Qc+8bZnAH3NrISZxQL1geWBqk8kFDSsVpYJ97Zi\n6qA2VCsXyRMfrKXHqPl8tWGvLo2VcwrkCKIqsNDM1pD9D/0s59xs4B/ANWa2Gbjae45zbj0wDdgA\nzAaGOuc0JhbJB23qVOKjIe0Yd1cLMrMc97+VQO9XlpC444DfpUkhpjupRULMqcwspiXsZNRXm0k7\nepLujavyRI8G1KtSxu/SpIBoqg0ROavj6RlMWLiN8fO2cjw9g97xtXjk6suoVi7S79IkwBQQIpIn\n+386yUvfJDN56Q7Cw4yB7WMZ3Lku5UpG+F2aBIgCQkTOy84Dxxnx5SY+Xr2b8qUi+G3XetzdpjaR\nEZrQoKjRZH0icl5qVSzFqL7NmTmsA1fULMezs5LoNmIeHySm6NLYEKWAEJHTXF6zHG/fdyWT77uS\nClERPDZ9DdeMnMcnq3cpKEKMAkJEctWh/iXMGNqBl+9qQURYGMOnrqa7giKk6ByEiJxTVpZj9vo9\njP5qM5v2HqVeldI83K0+N1xRnfAw87s8OU86ByEi+SYszLj+iup8PrwjY+9sQZjBw++u4tpR85mx\nZrdGFEWUAkJE8iwszLihaXVmD+/ES3c2x8gOih6j5vPpmt1kKSiKFAWEiJy3sDCjZ9MafPFIJ8bc\n0RwHDPNGFDPXKiiKCgWEiFywsDDjxmanB8Vv31lFj9HzmbU2VUER5HSSWkTyTWaWY9Z3qYz+6nu2\npB3jsqqlGd7tMq67vBphOpldaOhOahHxTWaWY+ba3fx77ma2pB2jQdUyDL+6Pj2aKCgKAwWEiPju\nl6AYPXczW9OO0bBaGYZ3q8+1CgpfKSBEpNDILSgeubo+3RsrKPyggBCRQiczy/HpmuxDT1t/VFD4\nRQEhIoVWRmYWn67dzb/nJrPtx2M0ql6W4d3q071xVQVFAVBAiEih9+ugaFy9LMOvzg4KMwVFoCgg\nRCRoZGRmMcM79LR9/3EFRYApIEQk6GRkZvHJ6t2M+To7KJrUyD70dI2CIl8pIEQkaGVkZvGxFxQ7\ncgTF1Y10jiI/KCBEJOj9OijqVynN4M516RVXg4hwzRR0oRQQIlJkZGRmMXNtKuPnbWHjnqPUKBfJ\nfR3r0LdVLaJKFPO7vKBTaNaDMLNwM1tlZjO958+Y2S4zW+19XZ+j71Nmlmxmm8zs2kDXJiLBoVh4\nGDc3r8nnwzsy8d5WRFcsxd9mbqD9P7/mX3O+58CxdL9LLJIKInqHA0lA2RxtI51zL+bsZGaNgb5A\nE6AG8JWZXeacyyyAGkUkCJgZXRtWoWvDKiTuOMj4eVv499zNvDp/C31bXcr9HWOJrlDK7zKLjICO\nIMwsGrgBeD0P3XsBU51zJ51z24BkoHUg6xOR4NWydgVeuyeeOY92omfTGkxeuoPOL3zLo++tZuOe\nI36XVyQE+hDTKOAJIOtX7cPMbK2ZTTCzCl5bTWBnjj4pXpuIyBnVr1qGF29vxvwnunJvuxi+WL+H\nHqMWMPDNFSzfdoBgPs/qt4AFhJn1BPY55xJ/9a1xQB0gDkgFRpzn6w4yswQzS0hLS8ufYkUk6NUo\nX5I/9WzM4iev4nfXXMbqnYfo/coSbh23mDkb9mrxogsQsKuYzOw5oB+QAUSSfQ7iQ+fc3Tn6xAAz\nnXOXm9lTAM6557zvfQE845xbcqafoauYRORMfk7PZFrCTl5bsJWUgz//5xLZm5rVoHix0L5EtlBd\n5mpmXYDHnXM9zay6cy7Va38UuNI519fMmgDvkH3eoQYwF6h/tpPUCggROZeMzCxmfZfKuG+zL5Gt\nXi6S+zpgJLczAAAI/klEQVTEckfrS0P2Etm8BoQf787zZhYHOGA7MBjAObfezKYBG8gedQzVFUwi\ncrGKhYfRK64mNzWrwbffpzH+2y08OyuJMV8n079tbfq3i6FS6RJ+l1ko6UY5EQk5K384yPhvt/Dl\nhr1ERoTRJ74W93esQ62KoXGJbKE6xBQoCggRuRjJ+47yyrytfLx6F1kObmxancGd69Koetlz7xzE\nFBAiInmUevhn3liwjXeW/8Dx9Ey6NqjMg53r0jq2YpGcRVYBISJyng4dT+ftJTt4c/F29h9Lp/ml\n5Xmoc90iN4usAkJE5AL9nJ7J9MSdvDo/+xLZelVKM7hTHXrF1SwSl8gqIERELlJul8gObB9L71a1\nKFcywu/yLpgCQkQknzjnmPd9GuPnbWHp1gOUKh7O7S2j6d8uhjqVS/td3nlTQIiIBMC6XYeZuGg7\nn67ZTXpmFl0bVGZgh1g61LskaE5oKyBERAIo7ehJpizbweSlO/jxp3TqVynNgPax3NK8JiWLh/td\n3lkpIERECsDJjExmrkllwqJtrN99hPKlIrij9aXc07Y21cuV9Lu8XCkgREQKkHOOFdsPMmHhNr7c\nsAcz47rLqzGgfSwtLi1fqA4/Fea5mEREihwzo3VsRVrHVmTngeO8tWQ7U1fsZObaVJrVKs/A9jFc\nd3n1oLpMViMIEZEAOXYygw9XpjBx0Xa2/niMKmVKcE/b2tzR+lJfJwjUISYRkUIiK8sxb3MaExdt\nZ/73aRQvFsbNcTUY0D7Wl3mfdIhJRKSQCAszujaoQtcGVUjed5SJi7bzwcoUpiWk0LZOJQZ2iOWq\nhlUIL2TTeWgEISLig0PH05m6YidvLd7O7sMnuLRiKe5tF8Pt8dGUiQzsXdo6xCQiEgQyMrP4Yv1e\nJizaRuKOg5QuUYzb46O5t10MtStFBeRnKiBERILM2pRDTFy0nZlrd5OR5ejWsAoD28fStm6lfL1M\nVgEhIhKk9h05weSlO5i87AcOHEunYbUyDGgfQ6+4mkRGXPxd2goIEZEgd+JUJjPW7Gbiou0kpR6h\nQqkI7rqyNv3a1qZq2cgLfl0FhIhIEeGcY+nWA0xctI05SXsJN+PedjE83bPxBb2eLnMVESkizIy2\ndSvRtm4lfth/nElLthNdIfDzPCkgRESCyKWVSvGnCxw5nK/gmRREREQKVMADwszCzWyVmc30nlc0\nszlmttl7rJCj71Nmlmxmm8zs2kDXJiIiZ1YQI4jhQFKO508Cc51z9YG53nPMrDHQF2gC9ABeNrPC\nveqGiEgRFtCAMLNo4Abg9RzNvYBJ3vYk4OYc7VOdcyedc9uAZKB1IOsTEZEzC/QIYhTwBJCVo62q\ncy7V294DVPW2awI7c/RL8dpERMQHAQsIM+sJ7HPOJZ6pj8u+CeO8bsQws0FmlmBmCWlpaRdbpoiI\nnEEgRxDtgZvMbDswFbjKzCYDe82sOoD3uM/rvwuolWP/aK/tNM65V51z8c65+MqVKwewfBGR0Baw\ngHDOPeWci3bOxZB98vlr59zdwAygv9etP/CJtz0D6GtmJcwsFqgPLA9UfSIicnZ+3Cj3D2Camd0H\n7AB6Azjn1pvZNGADkAEMdc5lnu2FEhMTfzSzHRdRyyXAjxexf1Gi9+J0ej/+S+/F6YrC+1E7L52C\nei6mi2VmCXmZjyQU6L04nd6P/9J7cbpQej90J7WIiORKASEiIrkK9YB41e8CChG9F6fT+/Ffei9O\nFzLvR0ifgxARkTML9RGEiIicQUgGhJn18GaMTTazJ/2ux09mVsvMvjGzDWa23syG+12T3349A3Eo\nM7PyZva+mW00syQza+t3TX4ys0e9v5N1ZvaumV34up9BIOQCwpshdixwHdAYuMObSTZUZQCPOeca\nA22AoSH+fsD/zkAcykYDs51zDYFmhPD7YmY1gYeBeOfc5UA42TcBF1khFxBkzxCb7Jzb6pxLJ3sa\nkF4+1+Qb51yqc26lt32U7H8AQnaSxDPMQBySzKwc0Al4A8A5l+6cO+RvVb4rBpQ0s2JAKWC3z/UE\nVCgGhGaNPQMziwGaA8v8rcRXuc1AHKpigTRgonfI7XUzi/K7KL8453YBLwI/AKnAYefcl/5WFVih\nGBCSCzMrDXwAPOKcO+J3PX7IywzEIaYY0AIY55xrDhzDW+ArFHmrX/YiOzhrAFFmdre/VQVWKAZE\nnmaNDSVmFkF2OExxzn3odz0+OtMMxKEqBUhxzv0yonyf7MAIVVcD25xzac65U8CHQDufawqoUAyI\nFUB9M4s1s+Jkn2Sa4XNNvjEzI/sYc5Jz7l9+1+Ons8xAHJKcc3uAnWbWwGvqRvZkmqHqB6CNmZXy\n/m66UcRP2vsxm6uvnHMZZvZb4Auyr0KY4Jxb73NZfmoP9AO+M7PVXtsfnXOf+ViTFB7DgCneh6mt\nwACf6/GNc26Zmb0PrCT76r9VFPG7qnUntYiI5CoUDzGJiEgeKCBERCRXCggREcmVAkJERHKlgBAR\nkVwpIER8YmZdNGOsFGYKCBERyZUCQuQczOxuM1tuZqvN7BVvvYifzGyktzbAXDOr7PWNM7OlZrbW\nzD7y5u/BzOqZ2VdmtsbMVppZXe/lS+dYb2GKd4euSKGggBA5CzNrBPQB2jvn4oBM4C4gCkhwzjUB\n5gH/19vlLeAPzrmmwHc52qcAY51zzcievyfVa28OPEL22iR1yL6zXaRQCLmpNkTOUzegJbDC+3Bf\nEthH9nTg73l9JgMfeusnlHfOzfPaJwHTzawMUNM59xGAc+4EgPd6y51zKd7z1UAMsDDwv5bIuSkg\nRM7OgEnOuadOazT706/6XeicNSdzbGeiv0kpRHSISeTs5gK3mVkVADOraGa1yf7buc3rcyew0Dl3\nGDhoZh299n7APG+lvhQzu9l7jRJmVqpAfwuRC6BPKyJn4ZzbYGZPA1+aWRhwChhK9uI5rb3v7SP7\nPAVAf2C8FwA5Zz/tB7xiZn/1XuP2Avw1RC6IZnMVuQBm9pNzrrTfdYgEkg4xiYhIrjSCEBGRXGkE\nISIiuVJAiIhIrhQQIiKSKwWEiIjkSgEhIiK5UkCIiEiu/j95jU6o446/FgAAAABJRU5ErkJggg==\n", 216 | "text/plain": [ 217 | "" 218 | ] 219 | }, 220 | "metadata": {}, 221 | "output_type": "display_data" 222 | } 223 | ], 224 | "source": [ 225 | "class MyLayerNet(th.nn.Module):\n", 226 | " def __init__(self, D_in, H, D_out):\n", 227 | " \"\"\"\n", 228 | " In the constructor we instantiate two nn.Linear modules and assign them as\n", 229 | " member variables.\n", 230 | " \"\"\"\n", 231 | " super(MyLayerNet, self).__init__()\n", 232 | " self.linear1 = th.nn.Linear(D_in, H)\n", 233 | " self.linear2 = th.nn.Linear(H, D_out)\n", 234 | "\n", 235 | " def forward(self, x):\n", 236 | " \"\"\"\n", 237 | " In the forward function we accept a Variable of input data and we must return\n", 238 | " a Variable of output data. We can use Modules defined in the constructor as\n", 239 | " well as arbitrary operators on Variables.\n", 240 | " \"\"\"\n", 241 | " h_relu = self.linear1(x).clamp(min=0)\n", 242 | " y_pred = self.linear2(h_relu)\n", 243 | " return y_pred\n", 244 | "\n", 245 | "\n", 246 | "N, D_in, H, D_out = 64, 1000, 100, 10\n", 247 | "\n", 248 | "x = Variable( th.randn(N,D_in) )\n", 249 | "y = Variable( th.randn(N,D_out), requires_grad=False )\n", 250 | "\n", 251 | "net = MyLayerNet(D_in, H, D_out)\n", 252 | "criterior = th.nn.MSELoss(size_average=False)\n", 253 | "opt = th.optim.SGD( net.parameters(), lr=1e-4 )\n", 254 | "\n", 255 | "niter = 10\n", 256 | "error = np.zeros(niter)\n", 257 | "for t in range(niter):\n", 258 | " \n", 259 | " yhat = net(x)\n", 260 | " loss = criterior(yhat,y)\n", 261 | " print(t, loss.data[0])\n", 262 | " error[t] = loss.data[0]\n", 263 | " \n", 264 | " opt.zero_grad()\n", 265 | " loss.backward()\n", 266 | " opt.step()\n", 267 | " \n", 268 | "# plot error\n", 269 | "plt.plot(error,'-', label='error')\n", 270 | "plt.xlabel('epoch')\n", 271 | "plt.ylabel('loss')\n", 272 | "plt.legend()\n", 273 | "plt.show()\n", 274 | "\n", 275 | "\n" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": { 289 | "collapsed": true 290 | }, 291 | "outputs": [], 292 | "source": [] 293 | } 294 | ], 295 | "metadata": { 296 | "kernelspec": { 297 | "display_name": "Python 3", 298 | "language": "python", 299 | "name": "python3" 300 | }, 301 | "language_info": { 302 | "codemirror_mode": { 303 | "name": "ipython", 304 | "version": 3 305 | }, 306 | "file_extension": ".py", 307 | "mimetype": "text/x-python", 308 | "name": "python", 309 | "nbconvert_exporter": "python", 310 | "pygments_lexer": "ipython3", 311 | "version": "3.6.1" 312 | } 313 | }, 314 | "nbformat": 4, 315 | "nbformat_minor": 2 316 | } 317 | -------------------------------------------------------------------------------- /basic/ilinear_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import numpy.matlib\n", 14 | "import layers as net\n", 15 | "%matplotlib inline" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 3, 21 | "metadata": { 22 | "collapsed": true 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "# aux funtion\n", 27 | "def plotData(x,y):\n", 28 | " plt.plot(x,y,'or');\n", 29 | " plt.ylabel('Profit in $10,000s');\n", 30 | " plt.xlabel('Population of City in 10,000s');\n", 31 | " plt.grid(1)\n", 32 | "\n", 33 | "def plotLoss(J):\n", 34 | " plt.plot(J);\n", 35 | " plt.ylabel('J(x)');\n", 36 | " plt.xlabel('epoch');\n", 37 | " plt.grid(1)\n", 38 | " " 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 4, 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "data": { 48 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+8HHV97/HXOwGj4YBCYk5TNYlWpaVa0JMqVFsNjYqo\n+ON6MfYAqYi5SOnDH6XWa9peLDdXq6leqwXKhUg05xpaC5Xa9FIDEfU+ipKgQBQR5OZQKSYSQDhE\nA4TP/WNmPXs2Ozuze3b25/v5eMxjd2dndj7Zs5nvfL/z/X6+igjMzGx4zel2AGZm1l0uCMzMhpwL\nAjOzIeeCwMxsyLkgMDMbci4IzMyGnAsCM7Mh54LAzGzIuSAwMxtyh5T1wZKeBXwOGAUCuCQiPiXp\nfOBdwE/STT8UEVsafdbChQtj2bJlLcXxyCOPcNhhh7W0bzc43vL1W8yOt1z9Fi8Uj3nHjh33RcTT\nczeMiFIWYDHw4vT54cAPgGOA84HzmvmssbGxaNW2bdta3rcbHG/5+i1mx1uufos3onjMwPYocI4t\nrUYQEfcC96bPH5Z0G/CMso5nZmat6cg9AknLgBcB30xXnSvpFkkbJB3ZiRjMzKw+RcnZRyWNANcD\n6yLiSkmjwH0k9w0uABZHxJl19lsDrAEYHR0d27x5c0vHn5qaYmRkpNXwO87xlq/fYna85eq3eKF4\nzCtWrNgREctzNyzSftTqAhwKXAO8P+P9ZcDOvM/xPYLe1W/xRvRfzI63XP0Wb0T77xGU1jQkScBl\nwG0R8Ymq9YurNnszsLOsGMzMLF+Z9wheBpwOnCjpO+lyMvAxSbdKugVYAbyvxBjMzPrTxAQsWwZz\n5iSPExOlHarMXkPfAFTnrYZjBszMht7EBKxZA/v2Ja8nJ5PXAOPjbT+cRxabmfWatWunC4GKffuS\n9SVwQWBm1mvuvru59bPkgsDMrNcsWdLc+llyQWBm1mvWrYP582eumz8/WV8CFwRmZr1mfBwuuQSW\nLgUpebzkklJuFEOJvYbMzGwWxsdLO/HXco3AzGzIuSAwMxtyLgjMzIacCwIz6y8dTL0wLHyz2Mz6\nR4dTLwwL1wjMrH90OPXCsHBBYGb9o8OpF4aFCwIz6x8dTr0wLFwQmFn/6HDqhWHhgsDM+keHUy8M\nCxcEZtZfxsdh1y544onksZ2FwJB2TXX3UTMzGOquqa4RmJnBUHdNdUFgZgZD3TXVBYGZGQx111QX\nBGZmUL9rKsDU1MDfNHZBYGYG011TFyyYuX7v3uSm8QAXBi4IzMwqxsdhZOTg9QN+09gFgZlZtSG8\naeyCwMys2hDeNHZBYGZWbQjzGbkgMDOrNoT5jEpLMSHpWcDngFEggEsi4lOSjgKuAJYBu4BTI+KB\nsuIwM2va+PhAn/hrlVkjeBz4o4g4Bjge+ANJxwAfBK6NiOcB16avzcysS0orCCLi3oi4KX3+MHAb\n8AzgjcDGdLONwJvKisHMzPJ15B6BpGXAi4BvAqMRcW/61o9Jmo7MzKxLFBHlHkAaAa4H1kXElZIe\njIinVb3/QEQcWWe/NcAagNHR0bHNmze3dPypqSlG6g0Q6VGOt3z9FrPjLVe/xQvFY16xYsWOiFie\nu2FElLYAhwLXAO+vWnc7sDh9vhi4Pe9zxsbGolXbtm1red9ucLzl67eYHW+5+i3eiOIxA9ujwLm6\ntKYhSQIuA26LiE9UvXU1sDp9vhr4UlkxmJlZvjJnKHsZcDpwq6TvpOs+BHwU+DtJ7wQmgVNLjMHM\nzHKUVhBExDcAZbz9u2Ud18zMmuORxWZmQ84FgZnZkHNBYGY25FwQmJkNORcEZmZDzgWBmdmQc0Fg\nZjbkXBCYmbXbxAQsWwZz5iSPExPdjqihMkcWm5kNn4kJWLMG9u1LXk9OJq+hZye7cY3AzKyd1q6d\nLgQq9u1L1vcoFwRF9Fk1z8y66O67m1vfA1wQ5KlU8yYnIWK6mufCwMzqWbKkufU9wAVBnj6s5plZ\nF61bB/Pnz1w3f36yvke5IMjTh9U8M+ui8XG45BJYuhSk5PGSS3r2RjG4IMjXh9U8s4HWD/fsxsdh\n1y544onksYcLAXBBkK8Pq3lmA8v37ErhgiBPH1bzzAaW79mVwgPKihgf94nfrBf4nl0pXCMws/7h\ne3alcEFgZv3D9+xK4YLAzPqH79mVolBBIOk/Szo8ff6nkq6U9OJyQzMzq6PPumb2g6I1gj+LiIcl\nvRxYCVwGXFReWGZm1ilFC4ID6ePrgEsi4p+BJ5UTkpmZdVLRguAeSX8LvA3YImleE/uamVkPK3oy\nPxW4BnhNRDwIHAX8cWlRmQ2KfkiHYEMvd0CZJAEvTF++UNJRwLci4t5SIzPrd304U5UNp4Y1Akmv\nBu4AzgdOTpcPA3ek75lZFqdDsD6RVyP4FLAyInZVr5T0bGAL8GtZO0raALwe2BMRL0jXnQ+8C/hJ\nutmHImJLS5Gb9TqnQ7A+kXeP4BDgR3XW3wMcmrPv5cBJddZ/MiKOSxcXAja4nA7B+kRejWADcKOk\nzcC/p+ueBawiGUuQKSK+JmnZbAM061vr1s28RwBOh2A9qWGNICI+AowDAk5IFwHj6XutOFfSLZI2\nSDqyxc8wK99se/w4HYL1CUVEsQ2T3kJExP2FPzypEXy56h7BKHAfEMAFwOKIODNj3zXAGoDR0dGx\nzZs3Fz3sDFNTU4yMjLS0bzc43vIViXnR1q0cvX49c/fv/8W6A/Pmcft557Fn5cqyQ5yh375jx1u+\nojGvWLFiR0Qsz90wIjIXYAmwGdhD0nvozvT5ZmBZo33T/ZcBO5t9r3YZGxuLVm3btq3lfbvB8Zav\nUMxLl0Ykc2DNXJYuLTm6g/Xbd+x4y1c0ZmB7FDjH5t0svgK4iuTK/XkR8VxgMfCPaWHQFEmLq16+\nGdjZ7GeYdYR7/NgQySsIFkbEFRFRyTVERByIiM3AgkY7SvoC8G/A0ZJ+JOmdwMck3SrpFmAF8L5Z\nxm9WDvf4sSGS12toh6QLgY3M7DW0Gvh2ox0j4u11VjfsaWTWM9zjx4ZIXkFwBvBOktHEz0jX3QNc\njU/qNsgqPXvWrk2ag5YsSQoB9/ixAZTXffTRiLgoIk6KiBemy0kRcWFE7G+0r3WJk5y1jydAsSHR\nsEYg6RCSGsGbmFkj+BJwWUQ8Vm541hQnOTOzFuTdLP48cBxJ01B10rljgU3lhmZNc5Kz5rkGZZZ7\nj2AsIp5fs+5HwA2SflBSTNYqd3lsjmtQZkB+jeD+dOL6X2wnaY6ktwEPlBuaNc1dHpvjGpQZkF8Q\nrALeCuyW9IO0FvBj4C3pe9ZL1q1LujhWc5fHbK5BmQE5TUORzEPwNgBJC9J1e8sPy1riLo/NWbIk\naQ6qt95siBSegD4i9lYKAUnLJf1yeWFZy9zlsTjXoMyAJgqCGn8I/LOkK9oZjFlHOU20GVBg8vp6\nImI1gKTD2xuOWYeNj/vEb0MvtyCQ9FSSKSerB5RdExEPRsTDZQZnZmbla9g0JOkM4CbglcD8dFlB\nkozujNKjMzOz0uXdI1hLMqjs3RHx39PlbGA58Kflh2cd4dG1vaXm77Fo69ZuR2QDLq8gEMm0krWe\nSN+zXtTMib0yunZyMpmDqzK61oVBd9T5exy9fr3/HlaqvIJgHXCTpIskfShdLiZpLnIfu6I6ccVd\nOYYEp59e/MTu0bW9pc7fY+7+/f57WKny0lBvJGkGuh7Yny5fBZZHxOVlBzcQOnHFXX0MSI5TrdGJ\n3aNre4v/HtYFub2GIuIBWpif2FKNrrjb1W2x3jFqZZ1IPLq2t/jvYV3Q6oAyJN3azkAGVieu8Ip8\nVtaJxKNre0udv8eBefP897BS5U1M85ast4Bfan84A6gTV3hZx6hodGJ3fqLeUufvcftpp3GM/x5W\norwawRXAKcAbapbXA08uN7QB0Ykr7nrHUNqpq0jaBOcn6i01f489K1d2OyIbcHn3CG4B1kfEzto3\nJPnXWUQnrrh9VW9ms5BXELwXeCjjvTe3OZbB1Yl8Ns6ZY2YtypuP4OsN3tve/nDMzKzTcnsNSVok\n6bD0+VMkrZX0UUmLyw/PzMzKVqT76GZgQfr8w8BzSeYr/t9lBWVmZp2Tl310NfArwCvT528DtpPM\nW7xU0hmSfqP8MM3MrCx5N4u/CjxC0ntoAbAb+CeScQR/kL7/0/LCMzOzsuXlGpoEPg1cA2wC/iIi\n7ibJSLo3Iu6OiLoFgaQNkvZI2lm17ihJX5F0R/p4ZPv+KWYtchpuG3K59wgi4iKS5qFfiYgvp6v3\nAm/P2fVykpnNqn0QuDYingdcm7426x6n4TYrlmsoIqYiYl/V60ci4sGcfb4G3F+z+o3AxvT5RuBN\nTcRq1n5Ow22GojZlcTs/XFoGfDkiXpC+fjAinpY+F/BA5XWdfdcAawBGR0fHNm9uLQHq1NQUIyMj\nLe3bDY63fNUxv+LEE1Gd/wMhcf1113U6tLr67Tt2vOUrGvOKFSt2RMTy3A0jorQFWAbsrHr9YM37\nDxT5nLGxsWjVtm3bWt63Gxxv+WbEvHRpRNIoNHNZurRL0R2s375jx1u+ojED26PAObblNNQt2l0Z\niJY+7unw8c1mchpus2IFgaS3pD19firpIUkPS8rKQdTI1cDq9Plq4EstfIZZ+4yPJ9lZly5NMrYW\nydZqNmCK1gg+BpwSEU+NiCMi4vCIOKLRDpK+APwbcLSkH0l6J/BR4FWS7gBWpq+7x90GDYql4fZv\nxQZY7lSVqd0RcVszHxwRWd1Lf7eZzylNpdtgpcdIpdsg+GrQZvJvxQZc0RrBdklXSHp72kz0lgaz\nl/UHdxu0ovxbsQFXtEZwBLAPeHXVugCubHtEndKJuYRtMPi3YgOuUEEQEe8oO5CO68RcwjYY/Fux\nAZeXffQD6eOnJf117dKZEEviboNWlH8rNuDyagSVG8SDNxuZ5/m1ovxbsQGXN1XlP6WPGxtt17c8\nz68V5d+KDbBOjyw2mx335zdru6K9hsy6z/35zUpRNMXEy4qssyHWiSt19+c3K0XRpqFPF1xnw6hT\nk7u4P79ZKfK6j54g6Y+Ap0t6f9VyPjC3IxFab2h0xd+pK/Wsfvvuz282K3k1gicBIyT3Eg6vWh4C\n3lpuaNYz8q74O3Wl7v78ZqXIm7z++oj4MHB8RHy4avlERNzRoRg7rx97ppQZc94Vf6eu1J0y2qwU\neU1D/zN9+hlJV9cuHYiv8/pxMvOyY8674m/1Sr2VwqtIymgza0pe99HPpY/ryw6kZzS6+u3Vk07Z\nMefl2mll5K27gpr1jLx7BB9PH09Om4lmLGUH1xXd7JkyMcHxq1Y137wzm5iLXJUXueIfH09eL1mS\nHHft2sbxuyuoWc/IKwgWS/ot4BRJL5L04uqlEwF23FFHNbe+XdIr5Cfv3j2zeeecc/JP1K220Rdt\nUsprm5+YgIUL4bTTijdPuSuoWc/IKwj+HPgz4JnAJ4C/qlqGp7mokXbdpM26Qr744uyTa+XYk5PJ\nCbpakTb6Zq7KM9rmF23dmsS0d+/B+zS6wndXULOekZd07ovAFyX9WURc0KGYuuv++4uvb2c7d9aV\ncMTM19Un1+pjRySFQURyxV4kO2Ybrsqfc+mlBxcmRT5r3bqZ8YO7gpp1SaGRxRFxgaRTJK1Pl9eX\nHVjXNHOl2s527mauhCtt8LXHrhQCRXvTtOGqfN6ePa0dw11BzXpG0VxDHwHeA3wvXd4j6X+UGVjX\nNNMVsp3t3PWOW9vcU1G5ITvbY7dhgNb+RYuy38z7LHcFNesJRXMNvQ54VURsiIgNwEnAYNYKmrlS\nzbranTOn+XsF6XF/Pjo6fdyzz84+Ubejjb0NV+V3nXXWwTECLFjgK3yzPtHMfARPq3r+1HYH0lOK\nXqnWu6IGOHCgtQFd4+PcsHnz9HEvvDD7RN2udAuzvCrfs3LlwTFu2gT33edCwKxPFC0IPgJ8W9Ll\nkjYCO4DevquX9qh5xYknlpcmonJFPbdO/r129YnPOlFXjr1gwfS2T3nK7I/XCjfxmPW13IJAkoBv\nAMcDVwL/AJwQEVeUHFvrqvrHq+w0EePjyQmwnk70if/Zz6af793b++kwzKzn5BYEERHAloi4NyKu\nTpcfdyC21nVi1Gr1+IE5GV9jdXt9GUnhyvp39nrSvV6Pz6zPFJ2q8iZJvxkRN5YaTbuUPWp1YgLO\nPBMefTR5feDAwdtUt9eXlVenjH9nr+cA6vX4zPpQ0XsELwVukPRDSbdIulXSLWUGNitZPWfalSbi\n7LOnC4F6JFi9emYytjKu3Iv2HGrmCrrXcwD1enxmfahoQfAa4DnAicAbSLqOvqHVg0ralRYm35G0\nvdXPybRuHTzpSQevf+ih2TcjTEzA1FTjbSJgy5bp101cuS/aurX4SbtIz6FmU1T3eg6gXo/PrA/l\nzUfwZEnvBf6YZOzAPRExWVlmeewVEXFcRCyf5eccbHwcDj/84PWPPTb7K8ei+1efmLKu3CNmnuwn\nJjh6/friJ+0i4wCavYLu9RxAvR6fWR/KqxFsBJYDtwKvJUk21x+ycgbN9sqx6P7VJ6as8QYw82S/\ndi1z9++f+X5es0de181mr6B7fTrIXo/PrA8papOaVb8p3RoRL0yfHwJ8KyJmnX5a0v8DHgAC+NuI\nuKTONmuANQCjo6NjmzdvbuoYx69alaR0rvHz0dFk0FaVRVu38pxLL2Xenj3sX7SIu846Kxko1cTn\nVjswbx63n3fejM/4xTF276Ze4oifj44yb8+epLtrjZC4/rrrGh4zSzPfw0GxFvg+pqamGBkZaSm2\nVjUTXz3diHk2HG+5+i1eKB7zihUrdhRqdYmIzAW4qdHrVhfgGenjIuBm4HcabT82NhZN27QpYv78\niKSRJVnmz0/Wt7Jdo+0hYmQkQopYujR734hkm9p9YXrfeu9VPnPp0mLHmM2/r0nbtm1ry+d0Ur/F\n7HjL1W/xRhSPGdgeBc7JeU1Dx0p6KF0eBn6j8lzSQ7mlTHbhc0/6uAe4CnhJq5+VqV7unnq5b5pt\nQ6/XLr9pEzz8cLGRtY16NE1NcVB9YP58OPnk1uckdpZPM8vRsCCIiLkRcUS6HB4Rh1Q9P6KVA0o6\nTNLhlefAq4GdrXxWrtrcPdUnv+pJXeppdC9gNikV6rVxH3JIMip4796ZzUZz5iTdULdsKV5Y1esq\n6hQQZtZAM0nn2mUU+Iakm4FvAf8cEf+noxFUd6nMUlYvlNor9AUL4PHH62/7xBOwcWN2nJOTB/U6\narnmYGZDq+MFQUTcFRHHpsuvR0Tnu3vUaw6q1q5eKFkDuSpX6J//PDz4YOPP2LevflK7ippeRy0P\ntnLaBrOh1Y0aQedMTHD8qlUHn9waNfu0qw097+q88n699BS1DhzI7n4K0yf7VgdbuSZhNtQGtyBI\nT25P3r374JNbVrNPM9M8Vo6RdRWdd3WeVyupjavSnJTl7rtbH2zltA1mQ21wC4Ksk9t73lM/RUSz\nzUF5V9F5V+dFB6ZJ0xPR79qVXRgsWdL6YCunbTAbaoNbEGSdxNLeOTPMmTN9BVy0OSTvKjrv6rzo\nzeiImTWURif7VruKOm2D2VAb3IKgmZNYZWKZyUk47TRYuDC/QMi7im50wi6SuK6itgaQd7Jvpauo\n0zaYDbXBLQga5ffJs3cvnH46nHPOzPXNTEaTdcKGpAmptlZy2GEHZ0zNOhm3e1yAB52ZDbXBLQjq\njSyunt83TwRcfHF2H/28yWgqMdSesLNuEi9cCBs25I+ELosHnZkNrcEtCOo59dTmagkR+b185s5t\n7sTdqEmp0UhoM7OSFJ2qsv9Uuo9WT2m4ceN0yoa7705O9Hnyevk88UT25PX1LFlSf6Swb8yaWZcM\nbo0gq1fPli3TTSCN+uVX5PXyafYE7huzZtZjBrcgKNI3Pu+G8qGHTp+g620rJZlBm+Ebs2bWYwa3\nIChyBV99Uq5Hmrnt6tUz10UkzU3NpmLwjVkz6yGDWxAUbYJpNGL30UdnplnYsuXg+wpOxWBmfW5w\nC4L0av/RI6qmTXjKU7K3L9KU5FQMZjaABrcgSM2YDL4yUEw6OElckaYkp2IwswE02AXB2rUzCwKY\nbtqZnIQzz5wuDE4+eWb7f8V9901v4x4/ZjaABnccAeQ32Tz6aJKNFJKbvvXGFTzyCLzjHcnzyk3d\nSu7/SsZP3+w1sz422DWCIk02e/fmzw3w2GPTN4Td48fMBsxgFwTr1nFg3rz87Yrc7K3extM6mtkA\nGeymofFxbr/tNo7ZtCl7AvgFC2BkpPFE9pA0Gy1bltxL2LhxugZRmZAmPZ6ZWb8Z7BoBsGflyqQJ\nZ9OmZKRwtUMPhU99qnjK6snJJCOpJ4g3swEy2DWCavVu9D73uclo4QMHkh5DIyPJhDFz5mQnkstK\nVFd0gnjXJMysxwx8jWCG6hu9J58M1147Pa9ARFIIvPvdybpNm5r7bE8Qb2Z9anhqBLUqs4XVuugi\neNnLGp+gpZk1gyLJ5zwq2cx61MDXCBZt3Xpwu/w559SfYayiMhNZlhNPbD75nEclm1mPGuyCYGKC\no9evn55ecnISzjgjuepvpNGYggUL4M47m08+51HJZtajBrsgqJdiopnZxGrNn5/0MspqzpmczO4N\n5HkIzKxHDXZB0I7293pzEjdqzqn0BsoqDNo9KtldUs1slrpSEEg6SdLtku6U9MHSDtSO9vfKnMTV\nJ+68cQed6g1U6ZJa3fSVVQiZmWXoeEEgaS7wN8BrgWOAt0s6ppSDrVtHgenpG6tXmOTNbAad6Q3k\nLqlm1gbdqBG8BLgzIu6KiEeBzcAbSznSbJteGt3MbTSzGXSmN5C7pJpZGyiyRsqWdUDprcBJEXFW\n+vp04KURcW7NdmuANQCjo6Njmzdvbul4Lzn1VOb/5CcHrX9izhzm1Llx/MScOSiC/YsWcddZZyUp\nKhpYtHUrR69fP+Om9IF587j9vPNy961namqKkZGRQtsev2oVT969+6D1Px8d5YYWv69mNRNvr+i3\nmB1vufotXige84oVK3ZExPLcDSOiowvwVuDSqtenA59ptM/Y2Fi06rtr10bMnx+RtKIny/z5Ee9+\nd/31mzY1f5BNmyKWLo2QksdWPiO1bdu25o7brn9Di5qKt0f0W8yOt1z9Fm9E8ZiB7VHgvNyNpqF7\ngGdVvX5muq4Ue1aurN9t88IL29eds1tzFLhLqpm1QTdSTNwIPE/Ss0kKgFXA73UhjuSE2e8nzUH4\nN5hZV3W8IIiIxyWdC1wDzAU2RMR3yzreoq1b4ZOfdNZPM7MMXRlHEBFbIuL5EfErEVFqjoXnXHqp\nu1iamTUw2COLgXl79tR/I29GMjOzITHwBcH+RYvqvyF5BK6ZGUNQENx11lkzU0ZXRMyuecg5fsxs\nQAx8QbBn5crWp5fM4hw/ZjZABr4gANqfBsI5fsxsgAxHQdDuSWGc48fMBshwFATtHoHraSfNbIAM\nR0EA7U0D4WknzWyADE9B0E7O8WNmA6QbuYYGg3P8mNmAcI3AzGzIDW5BkA74esWJJ3rAl5lZA4PZ\nNFQZ8LVvHwJnHDUza2AwawQe8GVmVthgFgQe8GVmVthgFgQe8GVmVthgFgQe8GVmVthgFgRVA77C\nA77MzBoazIIAfpFS4vrrrpt9SgkzswE2uAWBmZkV4oLAzGzIuSAwMxtyLgjMzIacCwIzsyGnyJrY\nvYdI+gkw2eLuC4H72hhO2Rxv+fotZsdbrn6LF4rHvDQinp63UV8UBLMhaXtELO92HEU53vL1W8yO\nt1z9Fi+0P2Y3DZmZDTkXBGZmQ24YCoJLuh1Akxxv+fotZsdbrn6LF9oc88DfIzAzs8aGoUZgZmYN\nDExBIGmXpFslfUfS9jrvS9JfS7pT0i2SXtyNONNYjk7jrCwPSXpvzTavlPTTqm3+vMMxbpC0R9LO\nqnVHSfqKpDvSxyMz9l2dbnOHpNVdjvnjkr6f/s2vkvS0jH0b/n46GO/5ku6p+rufnLHvSZJuT3/P\nH+xivFdUxbpL0ncy9u3G9/ssSdskfU/SdyW9J13fk7/jBvGW/xuOiIFYgF3Awgbvnwz8CyDgeOCb\n3Y45jWsu8GOS/r7V618JfLmLcf0O8GJgZ9W6jwEfTJ9/EPjLOvsdBdyVPh6ZPj+yizG/Gjgkff6X\n9WIu8vvpYLznA+cV+M38EHgO8CTgZuCYbsRb8/5fAX/eQ9/vYuDF6fPDgR8Ax/Tq77hBvKX/hgem\nRlDAG4HPReIG4GmSFnc7KOB3gR9GRKsD5koREV8D7q9Z/UZgY/p8I/CmOru+BvhKRNwfEQ8AXwFO\nKi3QKvVijoh/jYjH05c3AM/sRCxFZHzHRbwEuDMi7oqIR4HNJH+bUjWKV5KAU4EvlB1HURFxb0Tc\nlD5/GLgNeAY9+jvOircTv+FBKggC+FdJOyStqfP+M4B/r3r9o3Rdt60i+z/PCZJulvQvkn69k0Fl\nGI2Ie9PnPwZG62zTq98zwJkktcJ68n4/nXRu2gywIaPZohe/498GdkfEHRnvd/X7lbQMeBHwTfrg\nd1wTb7VSfsOHNBtgD3t5RNwjaRHwFUnfT69gepakJwGnAP+1zts3kTQXTaXtxP8IPK+T8TUSESGp\nb7qcSVoLPA5MZGzSK7+fi4ALSP5TX0DS3HJmF+Jo1ttpXBvo2vcraQT4B+C9EfFQUnlJ9OLvuDbe\nqvWl/YYHpkYQEfekj3uAq0iqz9XuAZ5V9fqZ6bpuei1wU0Tsrn0jIh6KiKn0+RbgUEkLOx1gjd2V\n5rT0cU+dbXrue5b0+8DrgfFIG1NrFfj9dERE7I6IAxHxBPC/MuLoqe9Y0iHAW4Arsrbp1vcr6VCS\nk+pERFyZru7Z33FGvKX/hgeiIJB0mKTDK89Jbq7srNnsauAMJY4HflpVPeyWzKsoSb+Utrsi6SUk\nf6u9HYytnquBSu+J1cCX6mxzDfBqSUemzRqvTtd1haSTgA8Ap0TEvoxtivx+OqLmvtWbM+K4EXie\npGentcpVJH+bblkJfD8iflTvzW59v+n/n8uA2yLiE1Vv9eTvOCvejvyGy7wL3qmFpPfEzenyXWBt\nuv5s4OwKNaVwAAAF0klEQVT0uYC/IeltcSuwvMsxH0ZyYn9q1brqeM9N/y03k9wg+q0Ox/cF4F7g\nMZL20XcCC4BrgTuArcBR6bbLgUur9j0TuDNd3tHlmO8kaev9TrpcnG77y8CWRr+fLsX7+fT3eQvJ\nCWtxbbzp65NJepX8sJvxpusvr/xuq7bthe/35SRNbLdU/f1P7tXfcYN4S/8Ne2SxmdmQG4imITMz\na50LAjOzIeeCwMxsyLkgMDMbci4IzMyGnAsCK0TSgTSr4U5Jfy9pfps///clfSZnm1dK+q2q12dL\nOqOdcdQ55sfTTJAfr/PeayVtT7NFflvSX9XGlf67frnJY14q6Zgmtv9VSf8mab+k82rey81Sqoxs\nnOmYm7oZe9WlDLNWDncftUIkTUXESPp8AtgRMwfpzPbzf59kbMe5DbY5H5iKiPXtOm6BuH5K0s/8\nQM36F5AMRHpdRHxf0lxgTURcVLPdV0myiZaWejlNKbCUJHnaA5XvJ43pB8CrSPr93wi8PSK+V7P/\nx4D7I+KjaWFxZET8SZra5A9J+rK/FPhURLxU0lHAdpJ+9wHsAMYiSc5mfcg1AmvF14HnAkh6f1pL\n2Kl0TgVJy5TkT5+QdJukL1ZqEEpypi9Mny9PT5QzSHqDpG+mV9lbJY0qScJ1NvC+tGby20py95+X\n7nOcpBs0nbO9clX7VUl/Kelbkn4g6bfrHE/plf9OJfnc35auvxoYAXZU1lX5ALAuIr4PEElaiIvS\n/c6XdJ6kt5KcLCfSmF8n6R+rjvsqSVfVieerkpanz6ckrVOSfPAGSQclSIuIPRFxI8lAr2pFs5Rm\nZePMythbNzOnpLmSLq/6Ht9X51jWg1wQWFOU5JV5LXCrpDHgHSRXi8cD75L0onTTo4ELI+LXgIeA\nc5o4zDeA4yPiRSQnrw9ExC7gYuCTEXFcRHy9Zp/PAX8SEb9BMjL3v1W9d0hEvAR4b836ircAxwHH\nkqRL+LikxRFxCvCz9Hi1eXReQHIlnCkivkhy5TweEccBW4BflfT0dJN3ABsafQbJCPQbIuJY4GvA\nu3K2r1Y0g2ZWNs6s/bPWH0eSNvkFEfFC4LNNxGpd5ILAinqKktmntgN3k+REeTlwVUQ8EkmCvCtJ\n0hED/HtE/N/0+aZ026KeCVwj6Vbgj4GGKbglPRV4WkRcn67aSDKJSkUledcOYFmdj3g58IX0qn43\ncD3wm03EW0gk7bCfB05TMsvUCWSnFK54FPhy+jwr/rZJY2y1vfgu4DmSPq0kP85DeTtYb3BBYEVV\nroyPi4g/TJsaGqk9mVReP8707+7JGft+GvhMelX5XxpsV9T+9PEA7Uu9/l1grIX9PgucRpJw8O9j\nesKRLI/F9I28ZuMvmkEzKxtn1v5116fNRMcCXyVpxru0iViti1wQ2Gx8HXiTpPlKMh6+OV0HsETS\nCenz3yNp7oFkOr3KCfQ/ZXzuU5k+YVX3SHmYZAq/GSLip8ADVe3/p5Nc1Tfz73hb2sb9dJLaxLdy\n9vk48CFJzweQNEfS2XW2mxFzRPwH8B/An1J+00lmllJJH5H05nS7rGycWRl762bmTO/9zImIf0j/\nfV2bF9yaM0gT01iHRcRNki5n+qR5aUR8O72xezvwB5I2AN8jmXAF4MPAZZIuILlyrOd84O8lPQBc\nBzw7Xf9PwBclvZGkN0u11cDF6U3pu0ja34u6iqSZ5maSmssHIuLHjXaIiFvSm+NfSI8ZTDfhVLs8\njetnwAkR8TOSiUWeHhG3NRFjJkm/RNJkdwTwRBrXMZFMwnIuyYl7LrAhIr6b7vZCplNXfxT4O0nv\nBCZJppyE5J5GJfvlPtLvNCLuT/9+N6bb/UW67ljgs5IqF5j1JlyyHuTuo9Z2aUHw5Yh4QZdD6UlK\nxkt8OyIu62IM10TEa7p1fOstrhGYdZCkHcAjwB91Mw4XAlbNNQIzsyHnm8VmZkPOBYGZ2ZBzQWBm\nNuRcEJiZDTkXBGZmQ84FgZnZkPv//ywMQxPRQckAAAAASUVORK5CYII=\n", 49 | "text/plain": [ 50 | "" 51 | ] 52 | }, 53 | "metadata": {}, 54 | "output_type": "display_data" 55 | } 56 | ], 57 | "source": [ 58 | "# Load data\n", 59 | "datas = np.loadtxt('data.txt', delimiter=',');\n", 60 | "Xo = datas[:,0]; y = datas[:,1];\n", 61 | "m = len(y);\n", 62 | "\n", 63 | "# plot data\n", 64 | "plotData(Xo,y);" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 5, 70 | "metadata": { 71 | "collapsed": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "# Create cost function neural netword\n", 76 | "# Demo\n", 77 | "# +--------+ +--------+\n", 78 | "# ----> | inner |--> (*)---> | loss |\n", 79 | "# +--------+ +--------+\n", 80 | "\n", 81 | "# Const function\n", 82 | "def costFunc( x, w1):\n", 83 | "\n", 84 | " m = x.shape[0];\n", 85 | " z = 0.0; \n", 86 | " for i in range(m): \n", 87 | " z1 = x[i,:].T;\n", 88 | " z += net.inner().forward({'x':z1, 'w':w1}) \n", 89 | " return z/m;\n", 90 | "\n", 91 | "# Gradind function\n", 92 | "def gradCostFunc(x, y, w1):\n", 93 | " \n", 94 | " m = x.shape[0];\n", 95 | " dEdW1 = 0.0; E = 0.0;\n", 96 | " for i in range(m):\n", 97 | "\n", 98 | " # forward --->\n", 99 | " z1 = np.matrix(x[i,:]).T; yi = np.matrix(y[i]); \n", 100 | " z2 = net.inner().forward({'x':z1, 'w':w1}) \n", 101 | " z = net.loss().forward({'x':z2, 'y':yi});\n", 102 | " E += z;\n", 103 | " \n", 104 | " # <--- backward\n", 105 | " l4 = [1];\n", 106 | " l3 = net.loss().backward({'x':z2, 'y':yi, 'dzdx':l4} ); \n", 107 | " _ , dEdW1_i = net.inner().backward({'x':z1, 'w':w1, 'dzdx':l3});\n", 108 | "\n", 109 | " dEdW1 += dEdW1_i;\n", 110 | "\n", 111 | " return E/m, dEdW1/m\n", 112 | "\n", 113 | "def loss(y,y_):\n", 114 | " m = len(y);\n", 115 | " e = 0.0;\n", 116 | " for i in range(m):\n", 117 | " e += net.loss().forward({'x':np.matrix(y_[i]), 'y':np.matrix(y[i])});\n", 118 | " return e/m;\n", 119 | "\n", 120 | " \n" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 6, 126 | "metadata": { 127 | "collapsed": true 128 | }, 129 | "outputs": [], 130 | "source": [ 131 | "# Performs gradient descent to learn theta\n", 132 | "def gradDescent(x,y,w1, alpha, numiter):\n", 133 | " \n", 134 | " E_h = np.zeros((numiter,1), dtype=np.float32); \n", 135 | " for i in range(numiter):\n", 136 | " E, dEdW1 = gradCostFunc(X, y, w1);\n", 137 | " w1 = w1 - alpha*dEdW1;\n", 138 | " \n", 139 | " E_h[i] = E;\n", 140 | " #print('E = {} dEdW = {}'.format(E, dEdW1));\n", 141 | "\n", 142 | " return w1, E_h;" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 11, 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "Theta found by gradient descent: -3.63 1.17\n" 155 | ] 156 | } 157 | ], 158 | "source": [ 159 | "# Running Gradient Descent\n", 160 | "X = np.concatenate((np.ones((m,1)),Xo.reshape(m,1)),axis=1)\n", 161 | "\n", 162 | "# Wrigth initialization\n", 163 | "# w1 = np.random.randn(1,2);\n", 164 | "w1 = np.zeros((1,2), dtype=np.float32);\n", 165 | "\n", 166 | "# Some gradient descent settings\n", 167 | "iterations = 1500;\n", 168 | "alpha = 0.01;\n", 169 | "\n", 170 | "theta, j = gradDescent(X,y,w1, alpha, iterations)\n", 171 | "\n", 172 | "# print theta to screen\n", 173 | "print('Theta found by gradient descent: {:.2f} {:.2f}'.format(theta[0,0], theta[0,1]));" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 12, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "data": { 183 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGpFJREFUeJzt3X1wHPddx/H39+70ZJ+J7NhWFCeN89ROQ6BJ5QkJgSL1\nIQ2dQtuhQEMpgQbMQGEK7UybtEBpoUwLnbbAdNoUEmpKWvWBhBRDCakrJxNok9hpHhy7TpzYTW3s\n2G4Sx1IsS7r78sf+JJ9knVaSb+82u5/XzM3u/nZv96u17z63u7/bM3dHRETyq9DqAkREpLUUBCIi\nOacgEBHJOQWBiEjOKQhERHJOQSAiknMKAhGRnFMQiIjknIJARCTnSq0uYD5Wrlzpa9euXdRzR0ZG\nWLp0aWMLarC015j2+kA1NkLa64P015i2+rZu3XrY3VfFLujuqX/09fX5Yg0NDS36uc2S9hrTXp+7\namyEtNfnnv4a01YfsMXn8R6rU0MiIjmnIBARyTkFgYhIzikIRERyTkEgIpJzCgIRkZxTEIiI5Fym\ng2DTjqfZ+ORYq8sQEUm1TAfB5p2HuGP3eKvLEBFJtUwHgYiIxMt8EHirCxARSbnMB4GIiMwt00Fg\n1uoKRETSL9NBICIi8TIfBLpGICIyt0wHgc4MiYjEy3QQiIhIPAWBiEjOKQhERHIusSAws04zu8/M\nHjKzR83sw6H9XDO718x2mdlXzKw9wRqSWrWISGYkeURwHHi1u78CuAS42swuBz4OfMrdLwCeBa5L\nsAYREYmRWBB4ZDhMtoWHA68Gvh7aNwBvTqqGqI4k1y4i8uKX6DUCMyua2YPAQeBO4AngOXefCIvs\nBdYkWYOIiMzNvAkfmc2sG7gN+FPgC+G0EGZ2NvBNd794luesB9YD9PT09A0ODi54u7fsOM49e8f5\n7OvKp1J+4oaHhymX01tj2usD1dgIaa8P0l9j2uobGBjY6u7r4pYrNaMYd3/OzIaAK4BuMyuFo4Kz\ngH11nvN54PMA69at8/7+/gVvd/Pzj2L79rCY5zbT5s2bU11j2usD1dgIaa8P0l9j2uurJ8leQ6vC\nkQBm1gW8DtgBDAFvDYtdC9yeVA2gW0yIiMRJ8oigF9hgZkWiwPmqu280s+3AoJn9JfA94KakClDv\nURGReIkFgbs/DFw6S/uTwGVJbVdERBZG3ywWEcm5TAeB6f6jIiKxMh0EIiIST0EgIpJzmQ8C3WJC\nRGRumQ4CdR8VEYmX6SAQEZF4CgIRkZzLdBDozJCISLxMB4GIiMRTEIiI5Fzmg0C9R0VE5pbpIFD3\nURGReJkOAhERiZf5INCpIRGRuWU6CEznhkREYmU6CEREJJ6CQEQk57IfBLpIICIyp0wHga4QiIjE\ny3QQiIhIvMwHgc4MiYjMLdtBoHNDIiKxsh0EIiISS0EgIpJzmQ8CXSMQEZlbpoPAdJFARCRWYkFg\nZmeb2ZCZbTezR83s3aH9z81sn5k9GB5vSKoGERGJV0pw3RPAe939ATNbBmw1szvDvE+5+ycS3PYJ\nOjckIjKnxILA3fcD+8P4UTPbAaxJanuz0c1HRUTiNeUagZmtBS4F7g1Nf2BmD5vZzWa2vBk1iIjI\n7Mw92XMnZlYG7gI+6u63mlkPcJjopM1fAL3u/s5ZnrceWA/Q09PTNzg4uOBtf23nGHfsGeMfX18+\nlT8hccPDw5TL6a0x7fWBamyEtNcH6a8xbfUNDAxsdfd1sQu6e2IPoA24A3hPnflrgW1x6+nr6/PF\n+Ng3d/h5129c1HObaWhoqNUlzCnt9bmrxkZIe33u6a8xbfUBW3we79VJ9hoy4CZgh7t/sqa9t2ax\ntwDbEqshqRWLiGRIkr2GrgTeATxiZg+Gtg8A15jZJUSnhvYAv5tgDSIiEiPJXkP3MPuH8v9Mapsi\nIrJw2f5msc4NiYjEynQQiIhIPAWBiEjOZT4IdIcJEZG5ZToIdPdREZF4mQ4CERGJl/kgSPgOGiIi\nL3qZDgJ1HxURiZfpIBARkXgKAhGRnFMQiIjkXKaDQJcIRETiZToIREQkXuaDQL1HRUTmlu0gUP9R\nEZFY2Q4CERGJpSAQEck5BYGISM5lOgh0hUBEJF6mg0BEROLlIghctyAVEakr00Gg3qMiIvEyHQQi\nIhJPQSAiknO5CAJdIhARqS/TQaAfrxcRiZfpIBARkXiJBYGZnW1mQ2a23cweNbN3h/YVZnanmT0e\nhsuTqmGSzgyJiNSX5BHBBPBed78IuBx4l5ldBFwPbHL3C4FNYToR6j4qIhIvsSBw9/3u/kAYPwrs\nANYAbwI2hMU2AG9OqgYREYnXlGsEZrYWuBS4F+hx9/1h1gGgpxk1iIjI7Czp2y+YWRm4C/iou99q\nZs+5e3fN/Gfd/aTrBGa2HlgP0NPT0zc4OLjgbd++a4zbdo1z01VLKBbSe55oeHiYcrnc6jLqSnt9\noBobIe31QfprTFt9AwMDW919XeyC7p7YA2gD7gDeU9O2E+gN473Azrj19PX1+WL83bce83Pev9HH\nJyqLen6zDA0NtbqEOaW9PnfV2Ahpr889/TWmrT5gi8/jvTrJXkMG3ATscPdP1sz6BnBtGL8WuD2p\nGkREJF4pwXVfCbwDeMTMHgxtHwA+BnzVzK4DfgD8SoI1AOo+KiIyl8SCwN3vof5vw7wmqe3WUvdR\nEZF4+maxiEjOzXlEYGadwBuBnwXOBI4B24D/cPdHky9PRESSVjcIzOzDRCGwmaj//0GgE3gp8LEQ\nEu9194ebUOcp0d1HRUTqm+uI4D53/1CdeZ80s9XASxKoqWFMFwlERGLVDQJ3/w+ITg+5+2jtPDNb\n6e4HiY4SRETkRWw+F4vvN7PLJyfM7JeA/02upMZzdSAVEalrPt1Hfw242cw2E10wPh14dZJFiYhI\n88QGgbs/YmYfBb4IHAVe5e57E69MRESaIjYIzOwm4HzgJ4l6DG00s793988kXZyIiCRvPtcIHgEG\n3H23u98B/BTwymTLaix1HxURqW8+p4Y+PWP6CHBdYhU1kHqPiojEq3tEYGb/bma/YGZts8w7z8w+\nYmbvTLY8ERFJ2lxHBL8DvAf4tJk9AxwCuoC1wC7gM+7+b4lXKCIiiZrrC2UHgPcB7ws/NXkG0b2G\nHnP3Y02p7hRZ3ZufiojIpLnuNXSU6bfyt8lpMzsOPAF80N03JVqhiIgkaq4jgmX15plZEbgYuCUM\nRUTkRWpRv0fg7hV3fwj4+wbXkwh1HxURqe+UfpjG3W9sVCFJUPdREZF4+oUyEZGcy0UQ6O6jIiL1\nZToIdGZIRCRepoNARETiKQhERHIuF0Gg7qMiIvVlOgjUfVREJF6mg0BEROLlIgh0ZkhEpL7EgsDM\nbjazg2a2rabtz81sn5k9GB5vSGr7oLuPiojMR5JHBF8Arp6l/VPufkl4/GeC2xcRkXlILAjc/W7g\nmaTWLyIijdGKawR/YGYPh1NHy5uxQVf/URGRuizJN8nwy2Yb3f3iMN0DHCa6fvsXQK+7z/q7x2a2\nHlgP0NPT0zc4OLjg7f/X7nEGd47x2dcuoauU3usFw8PDlMvlVpdRV9rrA9XYCGmvD9JfY9rqGxgY\n2Oru62IXdPfEHkS/b7xtofNmPvr6+nwx/uHuJ/yc92/054+NLer5zTI0NNTqEuaU9vrcVWMjpL0+\n9/TXmLb6gC0+j/fYpp4aMrPemsm3ANvqLdtIOjEkIlJf3Z+qPFVm9mWgH1hpZnuBDwH9ZnYJ0Xvz\nHuB3k9q+iIjMT2JB4O7XzNJ8U1LbExGRxcnFN4tFRKS+XASBeo+KiNSX6SAw3X5URCRWpoNARETi\n5SMIdGpIRKSuTAeBTgyJiMTLdBCIiEg8BYGISM7lIghcFwlEROrKdBCo96iISLxMB4GIiMTLRRDo\nm8UiIvVlOgh0ZkhEJF6mg0BEROIpCEREci4XQaBLBCIi9WU6CHT3URGReJkOAhERiZeLIHD1HxUR\nqSvTQaAzQyIi8TIdBCIiEk9BICKSc7kIAl0hEBGpL9NBoEsEIiLxMh0EIiISLxdBoN6jIiL1ZTsI\n1H9URCRWYkFgZjeb2UEz21bTtsLM7jSzx8NweVLbFxGR+UnyiOALwNUz2q4HNrn7hcCmMC0iIi2U\nWBC4+93AMzOa3wRsCOMbgDcntf1ptagDqYhIXZbkfXjMbC2w0d0vDtPPuXt3GDfg2cnpWZ67HlgP\n0NPT0zc4OLjg7X/7qXH+efsYnx7oorsjvZdDhoeHKZfLrS6jrrTXB6qxEdJeH6S/xrTVNzAwsNXd\n18Uu6O6JPYC1wLaa6edmzH92Puvp6+vzxfjid/b4Oe/f6E8/f2xRz2+WoaGhVpcwp7TX564aGyHt\n9bmnv8a01Qds8Xm8xzb7Y/LTZtYLEIYHm7JVnRkSEamr2UHwDeDaMH4tcHuSG1PvURGReEl2H/0y\n8B3gZWa218yuAz4GvM7MHgdeG6ZFRKSFSkmt2N2vqTPrNUltU0REFi69XWkaSJcIRETqy3QQmO4/\nKiISK9NBICIi8XIRBLr7qIhIfZkOAnUfFRGJl+kgEBGReAoCEZGcy0UQ6O6jIiL1ZToIdIlARCRe\npoNARETi5SII1H1URKS+TAeBuo+KiMTLdBCIiEg8BYGISM7lIgh0iUBEpL5MB4HuPioiEi/TQSAi\nIvFyEQSu/qMiInVlOgjaStGpobGJaosrERFJr0wHQWepCMDouIJARKSebAdBWwiCiUqLKxERSa9M\nB0FHW/TnjY4rCERE6sl0EEweERzXqSERkbpKrS4gScuXtAPwri89wBXnnc7LzljGy85YxgWry5y3\nskxXe7HFFYqItF6mg+DclUt558XtHC6uZOeBo9z12CEmqie6kq7p7uL81WUuWFXm/NVLOX9VmQtW\nlzl9aTumO9aJSE5kOggAXnVWG/39lwJRN9Ldh0d44tAwuw4OTw3v3/0Mx2quIyzrLHHO6Ut4yYol\nnL1iCeesWMpLVkTTZ3Z3Uipm+oyaiORM5oOgVnupMHV6qFa16ux/fjQKh4PDPHl4mB8+c4zv7z/K\nndufZrxy4iiiWDDWdHdNhULvaV2c2d3JGad1ceZpnfR2d1HuyNVuFZEXuZa8Y5nZHuAoUAEm3H1d\nK+qYVAhv7mu6u/i5l66aNq9SdQ48P8pTP3qBHz7zAk898wI/CMPNOw9xaPj4ST98s6yjRG9NOPT8\nWCerlnWwstzBqmUdrA7jukYhImnQyo+uA+5+uIXbn5diTUhccf7pJ80fm6jy9POj7D8yyv4jx6Lh\nc2F4ZJTt/3eEw8Njs6673FFi1bIO2irH+Oq+rawqRwGxfGk7y5e0s3xJG91L2lm+tI3lS9qnekGJ\niDSSzmGcovZSgbPDtYR6xitVnhkZ49DR49FjOBoeDsNde4+x88BR/mf4Rxw5Nl53PZ1tBZYvaY/C\nYUlbGI+GyzpLLOtsC8No/Mdq2pa0F3UBXERmZa24IZuZ7QaeJfqpgBvd/fOzLLMeWA/Q09PTNzg4\nuKhtDQ8PUy6XT6Ha5NXWOF51hsec4XHC0BkZ96nx2vbhcWckLBv3r1gw6CpBV8nCg6lhR9HoKIZh\nzXRn0Wgvgo+P0l3umrZcZwlKhfQEy4vt3zmN0l4fpL/GtNU3MDCwdT6n3lsVBGvcfZ+ZrQbuBP7Q\n3e+ut/y6det8y5Yti9rW5s2b6e/vX1yhTXKqNbo7I2MVjo6Oc3R0gqOj4zw/OjE1Pn04vX1kbIIX\nxiocG6swMjZx0vWOuZQKRld7kY5Skc62Ah2lQs14kY62Ap1h2FEq0NlWjF2mrVigVCzQVjTai5PT\ns4+3lQqUCtH03Xfflfl/56SlvT5If41pq8/M5hUELTk15O77wvCgmd0GXAbUDQKZm5lR7ihR7ijR\ne9ri1+PuHJ+oMnI8hMN4hZHjE3zn/gd46csvZmRsgmNjFV4Yq/BCCJAXxiocn6hyfKLC8fEwnKgy\nOl5hZGSC4+NVRmvmjYbpRn/+KBi0feubUUiEgGgrFmgvRaFSKkQhUjCjVDCKNY/StPEChdo2M4rF\nMDxpWatZtkCxAMVCYaq9aEbBoGCGGTy2b5xnv7c3TJ+YV6gdLxDmTX9u7XIzn2s1zz1pfZPzC+G5\nRNMGMGP66Jjz7Eh0PcvCPIyp+WYWhieeN8lmrGvasjolmXpNDwIzWwoU3P1oGL8K+Eiz65CTmRmd\nbUU624rUXhY/8mSR/ot6GrYdd2ei6oyOT4ZIFBzHx6tMVKuMV6qMTfis4+MTzlilykSlynhlctzZ\n9eRuzjzr7Knp8Up12vh4pUqlGm236s5ExRmbqFJxj9orob0aTU8+JqpVKlWoVKvRc6snlqn9cuK8\nPfJQw/ZjIr59Z6KrrxcqTLVPD5naZTGoTEzQdtd/n7SeyXWHiJseUjO2f6Ld6rTXLn9yiM0MwNr1\nHTt2jK77hmLXN22tMcv+1Vt+gsvOXXFSHY3UiiOCHuC28EeWgC+5+3+1oA5pETOjrRh9Yl8Wv/i8\nbN68j/7+lzdobfNXrQ2XGQHiDlV3qh4t953vfpfLLvupqTZ3p+JOtRotd2L5E/OrNW3T1uceza9y\n8vqmnjvbusGJph3AHY8GPPb441x4wQVT09EwCjuf8bzJI7rJNsKytfOnbWfGtmauy4kmTt7m9OX3\n7t3LmjVnTn9ezbLU1DZZIbO0Txuvt0zMsrONHnj6OD2ru2dbpGaf1bb5rMvWTiztSL63YNODwN2f\nBF7R7O2KJKFQMNrnedH8iSUF1q5cmnBFi7d5bA/9V57b6jLmtHnzIfr7L251GXVF1wgubXUZC6Z7\nJYiI5JyCQEQk5xQEIiI5pyAQEck5BYGISM4pCEREck5BICKScwoCEZGca8lN5xbKzA4BP1jk01cC\naf/dg7TXmPb6QDU2Qtrrg/TXmLb6znH3VXELvSiC4FSY2ZZW/wJanLTXmPb6QDU2Qtrrg/TXmPb6\n6tGpIRGRnFMQiIjkXB6C4KRfP0uhtNeY9vpANTZC2uuD9NeY9vpmlflrBCIiMrc8HBGIiMgcMh0E\nZna1me00s11mdn2LajjbzIbMbLuZPWpm7w7tK8zsTjN7PAyXh3Yzs78LNT9sZq9sUp1FM/uemW0M\n0+ea2b2hjq+YWXto7wjTu8L8tU2qr9vMvm5m3zezHWZ2RQr34R+Hf+NtZvZlM+ts9X40s5vN7KCZ\nbatpW/B+M7Nrw/KPm9m1Cdf3N+Hf+WEzu83Mumvm3RDq22lmr69pT+y1PluNNfPea2ZuZivDdNP3\nYUN4+KWjrD2AIvAEcB7QDjwEXNSCOnqBV4bxZcBjwEXAXwPXh/brgY+H8TcA3yT61brLgXubVOd7\ngC8BG8P0V4G3hfHPAb8Xxn8f+FwYfxvwlSbVtwH47TDeDnSnaR8Ca4DdQFfN/vvNVu9H4FXAK4Ft\nNW0L2m/ACuDJMFwexpcnWN9VQCmMf7ymvovC67gDODe8votJv9ZnqzG0nw3cQfQdp5Wt2ocN+Rtb\nXUBifxhcAdxRM30DcEMK6rodeB2wE+gNbb3AzjB+I3BNzfJTyyVY01nAJuDVwMbwn/hwzYtxal+G\n//hXhPFSWM4Sru+08CZrM9rTtA/XAD8ML/RS2I+vT8N+BNbOeKNd0H4DrgFurGmftlyj65sx7y3A\nLWF82mt4ch8247U+W43A14l+bXEPJ4KgJfvwVB9ZPjU0+cKctDe0tUw4/L8UuBfocff9YdYBot9y\nhtbU/WngfUA1TJ8OPOfuE7PUMFVfmH8kLJ+kc4FDwD+F01f/aGZLSdE+dPd9wCeAp4D9RPtlK+na\nj5MWut9a+Vp6J9EnbOaoo+n1mdmbgH3u/tCMWampcSGyHASpYmZl4F+BP3L352vnefQRoSXdt8zs\njcBBd9/aiu3PU4no0Pyz7n4pMEJ0SmNKK/chQDjP/iai0DoTWApc3ap65qvV+20uZvZBYAK4pdW1\n1DKzJcAHgD9rdS2NkuUg2Ed0Dm/SWaGt6cysjSgEbnH3W0Pz02bWG+b3AgdDe7PrvhL4RTPbAwwS\nnR76W6DbzEqz1DBVX5h/GvCjBOuD6NPTXne/N0x/nSgY0rIPAV4L7Hb3Q+4+DtxKtG/TtB8nLXS/\nNX1/mtlvAm8E3h7CKk31nU8U+A+F181ZwANmdkaKalyQLAfB/cCFoddGO9EFuW80uwgzM+AmYIe7\nf7Jm1jeAyZ4D1xJdO5hs/43Q++By4EjNYXzDufsN7n6Wu68l2kffdve3A0PAW+vUN1n3W8PyiX6i\ndPcDwA/N7GWh6TXAdlKyD4OngMvNbEn4N5+sMTX7scZC99sdwFVmtjwc+VwV2hJhZlcTnar8RXd/\nYUbdbws9rs4FLgTuo8mvdXd/xN1Xu/va8LrZS9Qh5AAp2YcL1uqLFEk+iK7gP0bUo+CDLarhZ4gO\nvR8GHgyPNxCdD94EPA58C1gRljfgM6HmR4B1Tay1nxO9hs4jepHtAr4GdIT2zjC9K8w/r0m1XQJs\nCfvx34h6XqRqHwIfBr4PbAO+SNS7paX7Efgy0TWLcaI3rOsWs9+IztXvCo/fSri+XUTn0ydfL5+r\nWf6Dob6dwM/XtCf2Wp+txhnz93DiYnHT92EjHvpmsYhIzmX51JCIiMyDgkBEJOcUBCIiOacgEBHJ\nOQWBiEjOKQhEEmZm/Rbu6iqSRgoCEZGcUxCIBGb262Z2n5k9aGY3WvQbDcNm9imLfmdgk5mtCste\nYmbfrbln/uQ9/S8ws2+Z2UNm9oCZnR9WX7YTv6dwS/j2sUgqKAhEADN7OfCrwJXufglQAd5OdPO4\nLe7+48BdwIfCU/4ZeL+7/yTRN0gn228BPuPurwB+mugbqRDddfaPiO6pfx7RfYhEUqEUv4hILrwG\n6APuDx/Wu4huxlYFvhKW+RfgVjM7Deh297tC+wbga2a2DFjj7rcBuPsoQFjffe6+N0w/SHR/+3uS\n/7NE4ikIRCIGbHD3G6Y1mv3pjOUWe0+W4zXjFfTakxTRqSGRyCbgrWa2GqZ+1/ccotfI5N1Dfw24\nx92PAM+a2c+G9ncAd7n7UWCvmb05rKMj3LteJNX0qUQEcPftZvYnwH+bWYHoTpPvIvoRnMvCvINE\n1xEgun3z58Ib/ZPAb4X2dwA3mtlHwjp+uYl/hsii6O6jInMws2F3L7e6DpEk6dSQiEjO6YhARCTn\ndEQgIpJzCgIRkZxTEIiI5JyCQEQk5xQEIiI5pyAQEcm5/wech2zH3O5JiwAAAABJRU5ErkJggg==\n", 184 | "text/plain": [ 185 | "" 186 | ] 187 | }, 188 | "metadata": {}, 189 | "output_type": "display_data" 190 | } 191 | ], 192 | "source": [ 193 | "# Plot error\n", 194 | "plotLoss(j)\n", 195 | "plt.show()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 13, 201 | "metadata": {}, 202 | "outputs": [ 203 | { 204 | "data": { 205 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNXV+PHvARFkURRkJKAMGrdREJmRgEgEQUVQNNEY\nzKCoMUSNS2JQSEAD6kRc3iS+akR+ikuYV4zGGBUiBgXjhgooi4DiMihIIKICA7LNnN8fVT3TPV3d\nXd3T1ducz/P0M923azl0N3Wrbt17rqgqxhhjmq5m2Q7AGGNMdllFYIwxTZxVBMYY08RZRWCMMU2c\nVQTGGNPEWUVgjDFNnFUExhjTxFlFYIwxTZxVBMYY08TtFdSGReRg4DGgCFBgmqreLSKTgJ8B/3UX\n/a2qzo63rY4dO2pxcXFKcWzbto02bdqktG42WLzBy7eYLd5g5Vu84D/mRYsWfamqByZcUFUDeQCd\ngd7u83bAh0AJMAkYm8y2SktLNVXz5s1Led1ssHiDl28xW7zByrd4Vf3HDCxUH8fYwK4IVHU9sN59\nvlVEVgJdgtqfMcaY1GTkHoGIFAPHA2+5RVeJyFIRmS4i+2ciBmOMMd5EA84+KiJtgVeAClV9WkSK\ngC9x7hvcAnRW1Us91hsDjAEoKioqnTlzZkr7r66upm3btqmGn3EWb/DyLWaLN1j5Fi/4j3nQoEGL\nVLUs0XKBVgQi0gJ4Hpijqn/weL8YeF5Vj423nbKyMl24cGFE2e7du1m7di07duyIG8OOHTto1apV\nkpFnj8UbqVWrVnTt2pUWLVqkbZvz589n4MCBadte0CzeYOVbvOA/ZhHxVREE2WtIgIeAleGVgIh0\ndu8fAPwAWJ7K9teuXUu7du0oLi7G2ZW3rVu30q5du1R2kRUWbz1VZdOmTaxdu5bu3bsHsg9jTLD3\nCPoDFwKniMh77mMYcIeILBORpcAg4FepbHzHjh106NAhbiVg8puI0KFDh4RXfcYUpMpKKC6GZs2c\nv5WVge0qyF5DrwFeR+m4YwaSYZVA4bPv2DRJlZUwZgxs3+68XrPGeQ1QXp723dnIYmOMyTUTJtRX\nAiHbtzvlAbCKIEWbNm2iV69e9OrVi4MOOoguXbrUvd61a5evbVxyySV88MEHcZe57777qAzgknDu\n3Lmcc845cZdZvHgxL7zwQtr3bYxJ4LPPkitvpMCahnJOZaVTm372GRxyCFRUNOoSq0OHDrz33nsA\nTJo0ibZt2zJ27NiIZUKj9po1865vH3744YT7+cUvfpFyjI21ePFili9fztChQ7MWgzFN0iGHOM1B\nXuUBaBpXBKH2tjVrQLW+vS2AM+2PPvqIkpISysvLOeaYY1i/fj1jxoyhrKyMY445hptvvrlu2ZNO\nOon33nuPPXv20L59e8aPH8+JJ55Iv3792LhxIwATJ07kT3/6U93y48ePp0+fPhx55JG88cYbgJN3\n5Nxzz6WkpITzzjuPsrKyukoq3KxZszjyyCPp3bs3//jHP+rKFyxYQL9+/Tj++OPp378/q1ev5ttv\nv+Xmm2+msrKSXr168dRTT3kuZ4wJQEUFtG4dWda6tVMegKZREWS4vW3VqlX86le/YsWKFXTp0oUp\nU6awcOFClixZwr/+9S9WrFgRtc7mzZs5+eSTeeONN+jXrx/Tp0/33Laq8vbbb3PnnXfWVSr33HMP\nBx10ECtWrODGG2/k3XffjVpv+/bt/PznP2f27NksWrSIL774ou69o48+mldffZV3332XG2+8kYkT\nJ7LPPvtw0003UV5eznvvvcd5553nuZwxJgDl5TBtGnTrBiLO32nTArlRDE2laSjD7W2HHXYYZWX1\nYzgef/xxHnroIfbs2cMXX3zBihUrKCkpiVhnn3324YwzzmDr1q2Ulpby6quvem77hz/8IQClpaVU\nVVUB8NprrzFu3DgAjjvuOI455pio9VasWMERRxzBYYcdBkB5eTmPPfYYAN988w0XXXQRH3/8cdx/\nl9/ljDFpUF4e2IG/oaZxRRCrXS2g9rbw9LCrV6/m7rvv5uWXX2bp0qUMHTrUs1/83nvvXfe8efPm\n7Nmzx3PbLVu2TLhMsiZMmMDpp5/O8uXLeeaZZ2L22/e7nDEmvzSNiiDD7W3htmzZQrt27dh3331Z\nv349c+bMSfs++vfvz1//+lcAli1b5tn0VFJSwurVq/n0009RVR5//PG69zZv3kyXLk5i2EceeaSu\nvF27dmzdujXhcsaY/NY0KoIMt7eF6927NyUlJRx11FFcdNFF9O/fP+37uPrqq1m3bh0lJSVMnjyZ\nkpIS9ttvv4hlWrduzdSpUznjjDMoKyujc+fOde+NGzeO66+/nt69exOee+qUU05hyZIlHH/88Tz1\n1FMxlzPG5Dk/kxZk++E1Mc2KFSt8TcywZcsWX8vlilTi3b17t3777beqqvrhhx9qcXGx7t69O92h\necrE5+v3u/Yr3yYisXgbmDFDtVs3VRHn74wZjdpcvn2+qnk0MY3JnOrqagYPHsyePXtQVR544AH2\n2su+WlOAMpx6oamwo0UBaN++PYsWLcp2GMYEL15XcKsIUtY07hEYYwpDhruCNxVWERhj8keGu4I3\nFVYRGGPyRxa7gmfaffM+4sKH3mLbzvSMF4rH7hEYY/JH6D5AGhNI5hJV5fYXPmDqK/Wj97ft3EOb\nlsEequ2KoBG8Jo+eOnVqXeqGpuzZZ59lypQp2Q7DFKLycqiqgtpa5286K4EMzgoWrrZW+e3fl9H9\nN7PrKoFuHVrz7o2n0mnf4OcwtyuCNLv88ssD3X6o32+s1NY1NTU0b9485e3v2bMnLV1PR4wYwYgR\nIxq9HWMyJgtdU/fU1HLdX5fw7JL6JJA9uuzH//3se7Rr1SKQfXqxK4I0mzRpEnfddRcAAwcOZNy4\ncfTp04cjjjiiLpFcTU0N119/PSeccAI9e/bkgQceAJzxAGeddRa9e/emR48edamiq6qqOPLII7no\noos49thj+fzzzyP2WVxczLhx4+jduzdPPvkkH3/8MUOHDqW0tJQBAwawatUqAD7++GP69u1Ljx49\nmDhxYt0Vzfz58xkwYAAjRoyoS4Y3Y8YM+vTpQ69evfj5z39OTU0NNTU1XHzxxRx77LH06NGDP/7x\njwD87//+LyUlJfTs2ZORI0cCTgqKq666qi7+U045hZ49ezJ48GA+c3t4XHzxxVxzzTWceOKJHHro\noTz11FPBfCnG+JHBLMU799Rw8cNv890J/6yrBE48rAOrbhnKc1eflNFKAArkimDyc++z4ostnu+l\neoZc8p19+d1Z0Vk8k7Vnzx7efvttZs+ezeTJk5k7dy4PPfQQ++23H++88w47d+6kf//+nHbaaRx8\n8MFUVlbSpUsXvvzyS/r27Vt3Vr169WoeffRR+vbt67mfDh06sHjxYgAGDx7M1KlTOfzww3nrrbe4\n8sorefnll7n22mu59tprueCCC5g6dWrE+qFJaLp3787KlSt54okneP3112nRogVXXnkllZWVHHPM\nMaxbt47ly5cDTjZSgClTpvDpp5/SsmXLurJwV199NaNHj2b06NFMnz6da665hmeeeQaA9evX89pr\nr7Fq1SpGjBjBeeed1+jP3JiUZKBr6vZdexj14Fss/qz+/8npxxRxzwW92Xuv7J2XF0RFkMu80ka/\n+OKLLF26tO4MePPmzaxevZquXbsyefJkFixYQLNmzVi3bh0bNmwAoFu3bjErAYAf//jHgHNV8cYb\nb/CjH/2o7r2dO3cC8Oabb9YdgH/yk59EzKjWp08funfvDsBLL73EokWLOOGEEwD49ttv6dSpE2ed\ndRaffPIJV199NcOHD+e0005j27Zt9OzZk/Lycs455xzP6S/ffPNNnn76aQAuvPBCbrjhhrr3zjnn\nHJo1a0ZJSUndv9WYrAhwVrDN3+7m3Pvf4KON1XVl55V25fZze9K8mTR6+41VEBVBvDP3rVu30q5d\nuwxGE8krbbSqcs8993D66adHLPvII4+wadMmFi1aRIsWLSguLq5L9Rye2tpL6P3a2lrat2/vOUOZ\nn/VD8Y0ePZrbbrstarklS5YwZ84cpk6dyl//+lfuvvtuZs2axb///W+ee+45KioqWLZsme/9hj6f\n0H6NyZqKish7BCHV1c79gxTuE3xZvZMz//c1/rOlPmX7Jf2LuenMEkSyXwGE2D2CLDj99NO5//77\n2b17NwAffvgh27ZtY/PmzXTs2JEWLVowb9481nidnSSw77770r17d5588knAObguWbIEgL59+/K3\nv/0NgJkzZ8bcxuDBg3nqqafqpsv86quvWLNmDV9++SW1tbWce+653HrrrSxevJja2lo+//xzBg0a\nxO23387mzZuprq6O2N6JJ55Yt7/KykoGDBiQ9L/LmMCFshR36BBZvmlT0lPbLlrzNcXjZ1F269y6\nSuDawYfz6W3D+N1Zx+RUJQAFckWQLdu3b6dr1651r6+77jpf61122WVUVVXVpXM+8MADeeaZZygv\nL2fYsGH06NGDsrIyjjrqqJTiqqys5IorruDWW29l9+7djBw5kuOOO44//elPjBo1ioqKCoYOHRqV\nqjqkpKSEW2+9ldNOO43a2lpatGjBfffdxz777MMll1xCbW0tALfddhs1NTWMGjWKzZs3o6pcc801\ntG/fPmJ799xzD5dccgl33nknBx54IA8//HBK/y5jAlde7twc3rQpstxnPqP5H2zk4offiSibOPxo\nLhtwaLojTSvJh8vxsrIyXbhwYUTZypUrOfrooxOum+2moWQFGe/27dvZZ599EBFmzpzJ448/HjGJ\nfSoy8fn6/a79mj9/PgMHDkzb9oJm8QYrKt5mzcDruCjijF3w8I/31nHtzMjm2OMPac/fr0z//CPg\n/zMWkUWqWpZoObsiaEIWLVrEVVddharSvn17pk+fnu2QjMk9Sdw0fvj1T5n8XOSMgGccexD3jyoN\nKrpAWEXQhAwYMKDufoExJgavm8YN8hndNecD7p33UcRqo/t1Y/LZx2YqyrTK64pAVXPupotJr3xo\nujQFJk4+o/F/W8rMdyIHdF536hFcM/jwLASaPoFVBCJyMPAYUAQoME1V7xaRA4AngGKgCjhfVb9O\ndvutWrVi06ZNdOjQwSqDAqWqbNq0iVatgs+1YkyE8vKIG8OXPvIOL4+fFbHIreccy6i+3TIdWSCC\nvCLYA/xaVReLSDtgkYj8C7gYeElVp4jIeGA8MC7ZjXft2pW1a9fy3//+N+5yO3bsyKsDicUbqVWr\nVhE9s4zJpGF3v8qK9ZFZC+77SW+G9+ycpYiCEVhFoKrrgfXu860ishLoApwNDHQXexSYTwoVQYsW\nLepGwsYzf/58jj/++GQ3nzUWrzHZV9zg7B+g8rLv0f+7HbMQTfAyco9ARIqB44G3gCK3kgD4D07T\nkTHGZJWq0v03s6PK//GL/hx3cHuPNQpH4OMIRKQt8ApQoapPi8g3qto+7P2vVXV/j/XGAGMAioqK\nSuONhI2nurrac96AXGXxBi/fYrZ4g7V5SzXXvhF9n3HC91px+P6pp3QPkt/PeNCgQdkfRyAiLYC/\nAZWq+rRbvEFEOqvqehHpDGz0WldVpwHTwBlQluoAlbwf3JLj8i1eyL+YLd5gVO/cw7G/mwNEVgJ/\nu+JESrtFnZvmlHR/xkH2GhLgIWClqv4h7K1ngdHAFPdv44a2GmNMEjZu2UGf378UVf7yr0/m0APz\n50omnYK8IugPXAgsE5HQ2Ovf4lQAfxWRnwJrgPMDjMEYYwD4aGM1Q/7wSlT53YNac/bpg7IQUe4I\nstfQazS85qo3OKj9GmNMuHeqvuJHU9+MKn9/8um0abkX8+fPz3xQOSavRxYbY0ws/1y2nisqF0eV\nf1RxBns1twz84awiMMYUFK9EcACf3jbMshDEYBWBMaYgVMxawf979dOo8qopw7MQTX6xisAYk9fG\nPLaQF1dEz3dtFYB/VhEYY/LSqX94hdUbq6PKrQJInlUExpi84pUHCKwCaAyrCIwxecEqgOBYHypj\nTE4rHj/LsxKomjI8dyuBykooLnbmPy4udl7nMLsiMMbkpLy9AqisjJzqcs0a5zVETHaTS6wiMMbk\nFK8K4Miidsz51fezEE0KJkyInO8YnNcTJuRsRWBNQ37k2WWeMflGVT2bgM7u9R2qpgzPn0oAnHmO\nkynPAXZFkEgeXuYZky927anliIn/jCr/9alHcHW+Tgh/yCHOccKrPEdZRZBIHl7mGZPrNm/fzXE3\nvxhVfvfIXpzdq0sWIkqjiorIk0eA1q2d8hxlFUEieXiZZ0yu+mzTdr5/57yo8icv78cJxQdkIaIA\nhE4QJ0xwjhOHHOJUAjl84mgVQSJ5eJlnTK5Z/NnX/PDPb0SVzxs7kO4d2yS3scrK3D/IlpfnXkxx\nWEWQSB5e5hmTK2YtXc8v/i86FfS7N57K/m32Tn6Dds8uEFYRJJKHl3nGZNvUVz5myj9XRZWvumUo\nrVo0YkJ4u2cXCKsI/MizyzxjsuX6J5fw5KJtQGQlkLa5AOyeXSCsIjDGNNqIe19j6drNUeVpHwVs\n9+wCYRWBMSZlGU8DYffsAmEVgTEmafEqgEAng7d7doHwVRGIyI+AF1R1q4hMBHoDt6pqdHcAY0zB\nyolEcHbPLu38XhHcqKpPishJwBDgTuB+4HuBRWaMyRk5UQGYwPitCGrcv8OBaao6S0RuDSgmY0yO\nsAqgafBbEawTkQeAU4HbRaQllrnUmIJlFUDT4vdgfj4wBzhdVb8BDgCuDywqYwpFnqUw90oFXdyh\ndW7PBmYaLeEVgTijQHq4L3uIyAHA26q6PtDIjMl3eZIOQVXp/pvZUeXDehzEn8tLsxCRybS4FYGI\nnAb8GVgNrHOLuwLfFZErVTU6j6wxxpHj6RB219Ry+ITouQCuGXw41516RBYiMtmS6IrgbmCIqlaF\nF4pId2A2cHSsFUVkOnAmsFFVj3XLJgE/A/7rLvZbVY0+FTGmEORoOoQtO3bTc1L0OdxdPzqO80q7\nZiEik22JKoK9gLUe5euAFgnWfQS4F3isQfkfVfUuX9EZk89yLB3C2q+3c9Lt0XMBPP6zvvQ7rEMW\nIjK5IlFFMB14R0RmAp+7ZQcDI4GH4q2oqv8WkeLGBmhM3sqRdAhLPv+Gs+97Pap87nXf57ud2mU0\nFpOb4vYaUtXbgHJAgH7uQ4By971UXCUiS0Vkuojsn+I2jAleY3v8lJfDtGnQrRuIOH+nTcvY/YE5\n7/+H4vGzoiqBhROHUDVluFUCpo6oqr8Fnd5CqOpXvjfuXBE8H3aPoAj4ElDgFqCzql4aY90xwBiA\noqKi0pkzZ/rdbYTq6mratm2b0rrZYPEGz0/MnebO5ci77qL5zp11ZTUtW/LB2LFsHDIk6BAjJPsZ\nz6nazeOrdkWVTzu1NXs3T0Mq6ATy7TeRb/GC/5gHDRq0SFXLEi6oqjEfwCHATGAjTs+hj9znM4Hi\neOu66xcDy5N9r+GjtLRUUzVv3ryU180Gizd4vmLu1k0Voh/dugUcXTS/n/Fvnl6q3cY9H/WoqakN\nNsAG8u03kW/xqvqPGVioPo6xie4RPAH8CacpqAZARJoDP3Irg74Ja5owItJZ68cf/ABYnsz6xmRM\njvb48XLe/W+wcM3XUeU2AMz4lagi6KiqT4QXuBXCTBG5Jd6KIvI4MBDoKCJrgd8BA0WkF07TUBXw\n8xTjNiZYOdbjx8tRN/6THbtro8qtAjDJSlQRLBKRPwOPEtlraDTwbrwVVfUCj+K4PY2MyRk50uPH\ni+UBMumWqCK4CPgpMBno4patA57FDuqmkOXgBChWAZigxK0IVHUXzrwD92cmHNNolZU5dfDKazky\nAcrFL2yDF6IrAasATLokyjW0F84VwTlEXhH8A3hIVXcHG55JSp4kOTP+2BWAyZREaaj/AvTCaRoa\n5j4mA8cBM4INzSQtXpIz4y0H00R7pYIGLBW0CUyiewSlqtowDeFaYIGIfBhQTCZVedTlMSfk2BWU\n18H/O/u14vf9mjNw4MCMx2OajkRXBF+JyI9EpG45EWkmIj8Gojsum+yK1bUxh7o85pQcuIJSVc8r\ngAv7dqNqynDe+M3gjMVimq5EVwQjgduBP4tI6MDfHpjnvmdySQ53ecxJWbyC2lNTy3c95gKYOPxo\nLhtwaOD7NyZcol5DVcCPAUSkg1u2KfiwTEpysMtjTsvCoLHtu/ZQctOcqPL7y3tzRo/Oge3XmHj8\nTl4fUQGISBnwhap+EUhUJnU50uUxL2TwCmrj1h30qXgpqvxvV5xIaTdLwmuyy3dF0MDVQE8R+VBV\nf5zOgIzJmAxcQS1ft5kz73ktqnze2IF079gmbfsxpjFSqghUdTSAiFhCc5PfArqCemnlBn766MKo\n8sU3nsoBbfZO+/6MaYyEFYGI7AcMJXJA2RxV/UZVtwYZnDH55tE3qvjds+9Hla+4+XRa753qBbgx\nwUo0svginKyhL+JUAACDgN+LyGRVbTgfsTFN0qRn3+eRN6qiyj/+/TCaNwt+MhhjGiPRKcoEnEFl\n34QXulNMvkX0xPQmH1l+opRdMG0Bb34S3ZGuUSOAG3wfnUaNAhtQZgKUqCIQnLkDGqp13zO5KJkD\ne46Nrs0XPSfNYcuOPVHljU4B4fF9HHnXXXD00fZ9mMAkGllcASwWkftF5LfuYyqw2H3P+JGJfDah\nfYjAhRc6B3TV+gN7rH3mwOjafBIaBdywEkhbHiCP76P5zp32fZhAJRpQ9qiIPAucTv3N4vnAb1TV\nUkz4kYkz7ob70AYXcaEDu9f+LD+RLxnLBGrfh8mChN0Y3AP+zAzEUpjinXGnqyLw2kdDsQ4keTAl\nYzZlPBW0fR8mCxI1DcUkIsvSGUjBysQZnp9txTqQVFQ4o2nDWX6i7KWC9vg+alq2bPLfhwlWou6j\nP4z1FnBQ+sMpQJk4w4u1j5B4B3bLTxQh65PBeHwfH4waRUkT/T5MZiRqGnoCqMS751Cr9IdTgDKR\nz8ZrHyLOvYJu3RIf2C0/UfYrgHANvo+N8+dTkvkoTBOSqCJYCtylqssbviEiQ4IJqcBk4ozbzupT\n5lUBHNqxDS+PHZj5YIzJkkQVwS+BLTHe+0GaYylcmTjjtrN632pq1bMCOLNnZ+79Se8sRGRMdiXq\nPvpqnPeiM2oZk8M2f7ub4ya/GFV+2UndmXimNb6YpstP0rlOwDZV3SYi+wDXAe2Au1V1fdABGtNY\nn23azvfvnBdVfse5PTn/hIOzEJExucVPOsSZwMXANmAycCCwCvg/nAR0xuSktz/9ivMfeDOqfHyf\nVlz+Q5sL2JiQRN1HRwOHAQNFRHCmrbwDqAa6udlJ31PVpYFHaoxPTy1ay9gnl0SVzx87kOKObZg/\nf37mgzImhyW6IpiPcyWwFOgAbACewxlH8Av3/c3BhWeMf7fNXskD//4kqnzJTaexX+sWWYjImPyQ\n6GbxGhG5B5iDk3H0Z6r6mYgcAmxS1ZhDWkVkOnAmsFFVj3XLDsAZm1AMVAHnW84i01ijHnyL1z76\nMqp8dcUZtGjuY/C8peE2TZyfXEP3i8hfgFpVDY1Y2gRckGDVR4B7iZyzYDzwkqpOEZHx7utxSUdt\nDHD0jS/w7e6aqPJPbxuG05Lpg6XhNsbfnMWqWt3g9TYf6/xbRIobFJ8NDHSfP4rTtGQVgUlKWkcB\nZyIpoDE5TrRhyuJ0btypCJ4Paxr6RlXbu88F+Dr02mPdMcAYgKKiotKZM1NLgFpdXU3btm1TWjcb\nLN7YLn7B+/zjkaFtktpOeMwnn3IK4vF/QEV45eWXkw8yAPabCFa+xQv+Yx40aNAiVS1LtFzWKgL3\n9dequn+i7ZSVlenChamNX5s/fz4D82iaP4s3WrrzAEXEXFzsnbCvWzeoqkpp++lmv4lg5Vu84D9m\nEfFVEfhqGkqjDSLSWVXXi0hnYGOG92/ySEYSwWUiKaAxOc5XReCmo74d6ITTdVQAVdV9k9zfs8Bo\nYIr79x9Jrm+agIxmArWEfcb4npjmDmCEqu6nqvuqartElYCIPA68CRwpImtF5Kc4FcCpIrIaGOK+\nzp5MzCVsfMvaZDDl5U4zUG2t89erErDfiilgfpuGNqjqymQ2rKqxupfmxth+6zaYM3JqLgAv9lsx\nBc5vRbBQRJ4AngF2hgpV9elAosoE6zaYdTlfAYTYb8UUOL8Vwb7AduC0sDIF8rciyMRcwibKnppa\nvjvhn1Hle+/VjA9vPSMLEflgvxVT4PwOKLsk6EAyLhNzCZs6W3bspuek6LkAzjruO9xzwfFZiCgJ\n9lsxBS5R9tEbVPUON99Q1IADVb0msMiCZt0GM2LNpm2cfOf8qPLfnHEUPz/5sMwHlAr7rZgCl+iK\nIHSDuPBmI7Nug4F68+NNXPD/FkSVPzS6jMFHF2Uhokaw34opcImyjz7n/n00M+FkmM3zm3Z/WbCG\nG59ZHlX+wi8HcNRByQ47ySH2WzEFLNMji02BuvSRd3h5VfRA8UUTh9Chbcv07chSRhuTdlYRmEYZ\n8+I2dr0Q3Q30g1uH0nKv5undmfXnNyYQvkYWi0h/P2Wm6QiNAt5VG1n+6R1nUTXzF7R8IrVssXHF\n689vjEmZ3yuCe4DePspMgYs5COye84M/U7f+/MYEIlH30X7AicCBInJd2Fv7Amm+7je5LFYF8MjQ\nNgy8+OLMjLy1/vzGBCJR09DeQFucCqNd2GMLcF6woZlcEDMR3D3nU9XjG+dFps7UKyqc/vvhrD+/\nMY0WtyJQ1VdUdTLQV1Unhz3+oKqrMxRj5uVjpsk0xxyzArj9TKpuPzOybT7WGXm6z9TLy2HaNGfS\nGBHn77RpdqPYmEZK1DT0J1X9JXCviHiNLB4RWGTZko89U9IYc8x7ALefGV0YOuNPdeRtKl1BrT+/\nMWmX6GbxY+7fu4IOJGfkY6bJNMQcNxNocbH3SqEz/lRG3uZjhWtMgUp0j+BO9+8wt5ko4hF0cFmR\nzZ4plZX0HTky+eadRsTsazIYP23z5eXO60MOcfY7YUL8+K0rqDE5I1FF0FlETgRGiMjxItI7/JGJ\nADPugAM0nsJVAAAX9klEQVSSK08X9wy51YYNoFp/hnzllYnb/lNoo/d1EzgkUdt8ZSV07AijRjlx\nh8cfqzKwrqDG5IxEFcFNwI1AV+APwP+EPZpOc1E86bpJG+sMeerU2AfX0L7XrHEO0OE82uj31NQm\ndxM4XIzpHDvNnevEtGlT9DrxzvAzdYPZGJNQoqRzTwFPiciNqnpLhmLKrq++8l+eznbuWGfC2uAe\nffjBNXzfqk5loOqcsYe10W+q3knprXM9Nx/3JrAPhz74YHQF5mdbltrZmJzhd2KaW0RkBPB9t2i+\nqj4fXFhZlMygpXTeWI61Xy+hNviG+w5VAlVVACxd+w0j7n09avVjvrMvs64ZkPgmsA8tN0YnmvO1\nLUvtbEzO8Jtr6DbgWmCF+7hWRH4fZGBZk8ygpXS2c3vtt2FzT0johmyMff9lwRqKx8+KqgQuP/kw\nqqYMdyqBWPtM8qx8Z6dOsd9MtK0YzU3GmMzyVREAw4FTVXW6qk4HhgIebQoFIJlBS7HOdps1S/5e\ngbvfHUVF9fu9/PLYB2qPff/03JsovuG5qPkApo7qTdWU4Yw/4yjPfTZmgNYnl10WHSNAhw422MuY\nPJFMGur2QKihfL8AYskdfgctebVzA9TUpHavoLycBV26MHDgwPqy/v1jN5+4+y4e591K9+Kvvs8R\nRe0S7rMxB+uNQ4ZQcvTR1sRjTB7zWxHcBrwrIvMAwblXMD6wqNLBHbV6cpAHp9D2Ro92Dv7h0jUI\nLdaBuryc4mXtPVdZOuk09m3VonH7TYaN9jUmryWsCEREgNeAvsAJbvE4Vf1PkIE1SlhvHoFgR62W\nl8OFF3q/F1Cf+FijgD+5/Syatd4HeliTjDHGv4T3CFRVgdmqul5Vn3UfuVsJQGZGrYaPH2gW42MM\nb8dPw3iDRGMAmqHp+XfmetK9XI/PmDzjt2losYicoKrvBBpNugQ9arWyEi69FHbtcl43bBaCyB4z\njRxvEDMP0B1nRY8zgMb9O3M9B1Cux2dMHvLba+h7wAIR+VhElorIMhFZGmRgjRKrN0+60kRcfnl9\nJeBFxLlvEN5XPoUrlIR5gPyOzk3mDDrXcwDlenzG5CG/VwSnp3OnIlIFbAVqgD2qWpbO7VNREXnG\nHrJli3MQbMyZY2UlVFfHX0YVZs+uf53EFUqnuXMpfmGb5+J1SeBC/IzOTfYMOtdzAOV6fMbkobhX\nBCLSSkR+CVyPM3ZgnaquCT0aue9Bqtor7ZUAOAe4dh7dJnfvbvyZo9/1ww9Msc7cVSPO0IvHz2LY\nnn5Ri1X1+Ca6EgB/4wCSPYPO9RxAuR6fMXkoUdPQo0AZsAw4AyfZXH6IlTOosWeOftcPPzB5jeAN\nWbOG4mXt4yeCi1f5JBqdm+wZdK5PB5nr8RmTh0S9bjaG3hRZpqo93Od7AW+raqPTT4vIp8DXgAIP\nqOo0j2XGAGMAioqKSmfOnJnUPvqOHOmkdG5gR1ERCxpsq9PcuRz64IO03LiRnZ068clll7FxyJCk\nthuupmVLPhg7NmIbdfvYsIFQ4ohYA8EaJoJTEV55+eW4+4wlmc8hKlYfn0d1dTVt27ZNKbZUJROf\nl2zE3BgWb7DyLV7wH/OgQYMW+Wp1UdWYD2BxvNepPoAu7t9OwBLg+/GWLy0t1aTNmKHaurWq0wDj\nPFq3dspTWS7e8qDatq2qiGq3brHXVVUV0W7jnvd8aLdu0duF+m126+ZvH4359yVp3rx5adlOJuVb\nzBZvsPItXlX/MQML1ccxOVHT0HEissV9bAV6hp6LyJaEtUzsymed+3cj8HegT6rbiskrd49X7ptk\n29C92uVnzICtW+MmT6ubC+CG56Leq7r9TKoeHA3V1URdn7VuDcOGOTd4/U76kiheywFkjAmTaD6C\n5uneoYi0AZqp6lb3+WnAzeneD+CduyckNHF6rNTP8e4FJJFS4YtvvuXEKd7NOnVNQHvtVTexS0S+\n0WbNnG6os2f7T3cda0J4O/AbY2JIJulcuhQBf3cyV7AX8H+q+kJGI2jYpdJLI3uhzF2xgcseW+j5\nXlWPb5yDtYgztsFrdi9wrjAefTR2nGvWOL2OQgd7G2xljEmB3wFlaaOqn6jqce7jGFXNfHcPr+ag\ncI3ohTLxmWUUj58VVQm02/VtfTfQUE+fv/wFvvnGe0Mh27dD8zgXZuHNRI0ZbGVpG4xpsrJxRZA5\nlZX0/fWvYePGyGaSeM0+DaZ59OvoG1/g293RqSYuWvQ8N8+d6rwIdXsMP3v3Sk/RUE2Ns26syit0\nsE91sJVdSRjTpBVuReAe3Fp5HdxiTQsZNs2j333ESgU9/ZU/c8qC2ZGF4e36ia5KGsZVUZH4nkYy\n02yGS+eUm8aYvJPxpqGMiXVwu/Za7xQRSTYHFY+f5VkJvHnUZqqmDOeUt/7pvWLo7NzvwDSR+iuU\nqiqnUvASuuJJZbCVpW0wpkkr3CuCWAcxrxuzzZpFtqXHOQuOlQn04ztG0FxrnQP1xT9JfHbud7J6\n1ch44uUXSnVC+FSvJIwxBaFwrwiSOYjV1jp/16yBUaOgY8eom6WJ5gJoru42QhVQvLNzP4nrQhpe\nASQaF5DKhPCWtsGYJq1wrwhizSfsx6ZNzqxjr79O8b4eyd6AqrvO9r7RG6qAYp2dg3dcbdo4SfHC\nM6bGOhine1xAqlcSxpiCULgVgXsQ2/HrX9Mq1Guoujp2n/0GvEYAQ3QeoAgND9xeB+ziYu/KqWNH\nqKiIjDeTB2MbdGZMk1W4FYGX88+PP0CLOIngZv7Cux29eXOnGcbvgTvejdl4I6GNMSYghVsReHUf\nffTR+pQNn30WMc1jzArgjrOcA/0dZ3nvp7a2/h6DH3Zj1hiTYwq3IojVfXT27PqxAsXFFI+8z3P1\nuiag0M3adB3A/cwqZowxGVS4vYYS9I0vHj/LsxKomwwGoEWL+gO0V88aESczaDIsG6gxJscU7hVB\njDP44huegxjdQKNIWC7Q8nJ4/XWYOrW+SUnVaW7q3z+5A7ndmDXG5JDCvSIIO4NXnHsAXvcBqqYM\ndxLBeY3Y3bUrMmHb7NkR9xUA/0ndjDEmRxXuFUF5ObUKhy6PTgNxZFE75vzq+5GFftIsWCoGY0wB\nKtwrAuCna9pEvJ748oNU3XEWc647OTrVcqybvuHlfpYxxpg8U9AVwcSZvwdg7oOXU3X7mVz2zjP1\nTTtr1sCll9ZXBsOGRd4TCPnyy/plLBWDMaYAFW7TEHDY+wupWh5nJPCuXU42UnBu+jZs/wfYtg0u\nucR5bqkYjDEFqKCvCHw12WzalHhugN27IzOTJpvUzRhjclhhVwQVFdS0bJl4OT83e8OXsWkdjTEF\npKCbhigv54OVKymZMSN27v8OHaBt28RzA6g6B/1hwyLzFdm0jsaYPFfYVwTAxiFDnCacGTOckcLh\nWrSAu+/2vgnsZc0aZ0CZTRBvjCkghX1FEM7rRu93v+skoaupcXoMtW3rpKpu1ix2IjmvG8pgE8Qb\nY/JWwV8RRAi/0TtsGLz0Uv3kMqpOJXDFFU7ZjBnJbbsxE8QbY0wWNZ0rgoamTfMuv/9+J3dQvAO0\nSOSVgZ/kczYq2RiTowr+iqDT3LnR7fJXXuk9zWTImDHxbx6fckrk4LNQ8rl4bf42KtkYk6MKuyKo\nrOTIu+5yDuqqzt+LLnLO+uOJN6agQwf46KPkk8/ZqGRjTI4q7IpgwgSa79wZWZbMbGINtW7t9DKK\n1ZyzZk3s3kA2D4ExJkcVdkWQjvb35s2jD9zxmnNCvYFiVQbpHpVsXVKNMY2UlYpARIaKyAci8pGI\njA9sR+lofw/NSRx+4E407iBTvYFCXVLDm75iVULGGBNDxisCEWkO3AecAZQAF4hISSA7q6ggRq9/\n/7wqk/Bmnlgy0RvIuqQaY9IgG1cEfYCPVPUTVd0FzATODmRPjW16iXczN9TME6syyERvIOuSaoxJ\nA9FYI2WD2qHIecBQVb3MfX0h8D1VvarBcmOAMQBFRUWlM2fOTGl/fc4/n9b//W9UeW2zZjTzuHFc\n26wZosrOTp345LLLnBQVcXSaO5cj77or4qZ0TcuWfDB2bMJ1vVRXV9O2bVtfy/YdOZJWGzZEle8o\nKmJBip9XspKJN1fkW8wWb7DyLV7wH/OgQYMWqWpZwgVVNaMP4DzgwbDXFwL3xluntLRUU/X+hAmq\nrVurOq3ozqN1a9UrrvAunzEj+Z3MmKHarZuqiPM3lW245s2bl9x+0/VvSFFS8eaIfIvZ4g1WvsWr\n6j9mYKH6OC5no2loHXBw2OuublkgNg4Z4t1t889/Tl93zmzNUWBdUo0xaZCNFBPvAIeLSHecCmAk\n8JMsxOEcMPP9oFkI/wZjTFZlvCJQ1T0ichUwB2gOTFfV94PaX6e5c+GPf7Ssn8YYE0NWxhGo6mxV\nPUJVD1PVQHMsHPrgg9bF0hhj4ijskcVAy40bvd9INCOZMcY0EQVfEezs1Mn7DREbgWuMMTSBiuCT\nyy6LTBkdotq45iHL8WOMKRAFXxFsHDIk9eklY7EcP8aYAlLwFQGQ/jQQluPHGFNAmkZFkO5JYSzH\njzGmgDSNiiDdI3Bt2kljTAFpGhUBpDcNhE07aYwpIE2nIkgny/FjjCkg2cg1VBgsx48xpkDYFYEx\nxjRxhVsRuAO+Tj7lFBvwZYwxcRRm01BowNf27QhYxlFjjImjMK8IbMCXMcb4VpgVgQ34MsYY3wqz\nIrABX8YY41thVgQ24MsYY3wrzIogbMCX2oAvY4yJqzArAqhLKfHKyy83PqWEMcYUsMKtCIwxxvhi\nFYExxjRxVhEYY0wTZxWBMcY0cVYRGGNMEycaa2L3HCIi/wXWpLh6R+DLNIYTNIs3ePkWs8UbrHyL\nF/zH3E1VD0y0UF5UBI0hIgtVtSzbcfhl8QYv32K2eIOVb/FC+mO2piFjjGnirCIwxpgmrilUBNOy\nHUCSLN7g5VvMFm+w8i1eSHPMBX+PwBhjTHxN4YrAGGNMHAVTEYhIlYgsE5H3RGShx/siIv8rIh+J\nyFIR6Z2NON1YjnTjDD22iMgvGywzUEQ2hy1zU4ZjnC4iG0VkeVjZASLyLxFZ7f7dP8a6o91lVovI\n6CzHfKeIrHK/87+LSPsY68b9/WQw3kkisi7sex8WY92hIvKB+3sen8V4nwiLtUpE3ouxbjY+34NF\nZJ6IrBCR90XkWrc8J3/HceIN/jesqgXxAKqAjnHeHwb8ExCgL/BWtmN242oO/Aenv294+UDg+SzG\n9X2gN7A8rOwOYLz7fDxwu8d6BwCfuH/3d5/vn8WYTwP2cp/f7hWzn99PBuOdBIz18Zv5GDgU2BtY\nApRkI94G7/8PcFMOfb6dgd7u83bAh0BJrv6O48Qb+G+4YK4IfDgbeEwdC4D2ItI520EBg4GPVTXV\nAXOBUNV/A181KD4beNR9/ihwjseqpwP/UtWvVPVr4F/A0MACDeMVs6q+qKp73JcLgK6ZiMWPGJ+x\nH32Aj1T1E1XdBczE+W4CFS9eERHgfODxoOPwS1XXq+pi9/lWYCXQhRz9HceKNxO/4UKqCBR4UUQW\nicgYj/e7AJ+HvV7rlmXbSGL/5+knIktE5J8ickwmg4qhSFXXu8//AxR5LJOrnzPApThXhV4S/X4y\n6Sq3GWB6jGaLXPyMBwAbVHV1jPez+vmKSDFwPPAWefA7bhBvuEB+w3slG2AOO0lV14lIJ+BfIrLK\nPYPJWSKyNzAC+I3H24txmouq3XbiZ4DDMxlfPKqqIpI3Xc5EZAKwB6iMsUiu/H7uB27B+U99C05z\ny6VZiCNZFxD/aiBrn6+ItAX+BvxSVbc4Fy+OXPwdN4w3rDyw33DBXBGo6jr370bg7ziXz+HWAQeH\nve7qlmXTGcBiVd3Q8A1V3aKq1e7z2UALEemY6QAb2BBqTnP/bvRYJuc+ZxG5GDgTKFe3MbUhH7+f\njFDVDapao6q1wP+LEUdOfcYishfwQ+CJWMtk6/MVkRY4B9VKVX3aLc7Z33GMeAP/DRdERSAibUSk\nXeg5zs2V5Q0Wexa4SBx9gc1hl4fZEvMsSkQOcttdEZE+ON/VpgzG5uVZINR7YjTwD49l5gCnicj+\nbrPGaW5ZVojIUOAGYISqbo+xjJ/fT0Y0uG/1gxhxvAMcLiLd3avKkTjfTbYMAVap6lqvN7P1+br/\nfx4CVqrqH8Leysnfcax4M/IbDvIueKYeOL0nlriP94EJbvnlwOXucwHuw+ltsQwoy3LMbXAO7PuF\nlYXHe5X7b1mCc4PoxAzH9ziwHtiN0z76U6AD8BKwGpgLHOAuWwY8GLbupcBH7uOSLMf8EU5b73vu\nY6q77HeA2fF+P1mK9y/u73MpzgGrc8N43dfDcHqVfJzNeN3yR0K/27Blc+HzPQmniW1p2Pc/LFd/\nx3HiDfw3bCOLjTGmiSuIpiFjjDGps4rAGGOaOKsIjDGmibOKwBhjmjirCIwxpomzisD4IiI1blbD\n5SLypIi0TvP2LxaRexMsM1BETgx7fbmIXJTOODz2eaebCfJOj/fOEJGFbrbId0XkfxrG5f67vpPk\nPh8UkZIklj9KRN4UkZ0iMrbBewmzlEqMbJzumBvPjL2SpQyzJhjWfdT4IiLVqtrWfV4JLNLIQTqN\n3f7FOGM7roqzzCSgWlXvStd+fcS1GaefeU2D8mNxBiINV9VVItIcGKOq9zdYbj5ONtHAUi+7KQW6\n4SRP+zr0+bgxfQicitPv/x3gAlVd0WD9O4CvVHWKW1nsr6rj3NQmV+P0Zf8ecLeqfk9EDgAW4vS7\nV2ARUKpOcjaTh+yKwKTiVeC7ACJynXuVsFzcORVEpFic/OmVIrJSRJ4KXUGIkzO9o/u8zD1QRhCR\ns0TkLfcse66IFImThOty4FfulckAcXL3j3XX6SUiC6Q+Z3vorHa+iNwuIm+LyIciMsBjf+Ke+S8X\nJ5/7j93yZ4G2wKJQWZgbgApVXQWgTlqI+931JonIWBE5D+dgWenGPFxEngnb76ki8nePeOaLSJn7\nvFpEKsRJPrhARKISpKnqRlV9B2egVzi/WUpjZeOMlbHXMzOniDQXkUfCPsdfeezL5CCrCExSxMkr\ncwawTERKgUtwzhb7Aj8TkePdRY8E/qyqRwNbgCuT2M1rQF9VPR7n4HWDqlYBU4E/qmovVX21wTqP\nAeNUtSfOyNzfhb23l6r2AX7ZoDzkh0Av4DicdAl3ikhnVR0BfOvur2EenWNxzoRjUtWncM6cy1W1\nFzAbOEpEDnQXuQSYHm8bOCPQF6jqccC/gZ8lWD6c3wyasbJxxlo/VnkvnLTJx6pqD+DhJGI1WWQV\ngfFrH3Fmn1oIfIaTE+Uk4O+quk2dBHlP46QjBvhcVV93n89wl/WrKzBHRJYB1wNxU3CLyH5Ae1V9\nxS16FGcSlZBQ8q5FQLHHJk4CHnfP6jcArwAnJBGvL+q0w/4FGCXOLFP9iJ1SOGQX8Lz7PFb8aePG\nmGp78SfAoSJyjzj5cbYkWsHkBqsIjF+hM+Neqnq129QQT8ODSej1Hup/d61irHsPcK97VvnzOMv5\ntdP9W0P6Uq+/D5SmsN7DwCichINPav2EI7Hs1vobecnG7zeDZqxsnLHW9yx3m4mOA+bjNOM9mESs\nJousIjCN8Spwjoi0Fifj4Q/cMoBDRKSf+/wnOM094EynFzqAnhtju/tRf8AK75GyFWcKvwiquhn4\nOqz9/0Kcs/pk/h0/dtu4D8S5mng7wTp3Ar8VkSMARKSZiFzusVxEzKr6BfAFMJHgm05iZikVkdtE\n5AfucrGyccbK2OuZmdO999NMVf/m/vuyNi+4SU4hTUxjMkxVF4vII9QfNB9U1XfdG7sfAL8QkenA\nCpwJVwAmAw+JyC04Z45eJgFPisjXwMtAd7f8OeApETkbpzdLuNHAVPem9Cc47e9+/R2nmWYJzpXL\nDar6n3grqOpS9+b44+4+lfomnHCPuHF9C/RT1W9xJhY5UFVXJhFjTCJyEE6T3b5ArRtXiTqTsFyF\nc+BuDkxX1ffd1XpQn7p6CvBXEfkpsAZnyklw7mmEsl9ux/1MVfUr9/t7x13uZrfsOOBhEQmdYHpN\nuGRykHUfNWnnVgTPq+qxWQ4lJ4kzXuJdVX0oizHMUdXTs7V/k1vsisCYDBKRRcA24NfZjMMqARPO\nrgiMMaaJs5vFxhjTxFlFYIwxTZxVBMYY08RZRWCMMU2cVQTGGNPEWUVgjDFN3P8HRfTfbdCkImYA\nAAAASUVORK5CYII=\n", 206 | "text/plain": [ 207 | "" 208 | ] 209 | }, 210 | "metadata": {}, 211 | "output_type": "display_data" 212 | } 213 | ], 214 | "source": [ 215 | "# Plot the linear fit\n", 216 | "plotData(Xo,y);\n", 217 | "plt.plot(X[:,1], X*theta.T, '-')\n", 218 | "plt.legend(['Training data', 'Linear regression'])\n", 219 | "plt.show()" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 14, 225 | "metadata": {}, 226 | "outputs": [ 227 | { 228 | "name": "stdout", 229 | "output_type": "stream", 230 | "text": [ 231 | "For population = 35,000, we predict a profit of [[ 4519.7678677]]\n", 232 | "For population = 70,000, we predict a profit of [[ 45342.45012945]]\n" 233 | ] 234 | } 235 | ], 236 | "source": [ 237 | "# Predict values for population sizes of 35,000 and 70,000\n", 238 | "predict1 = [1, 3.5] *theta.T;\n", 239 | "print('For population = 35,000, we predict a profit of {}'.format(predict1*10000));\n", 240 | "predict2 = [1, 7] * theta.T;\n", 241 | "print('For population = 70,000, we predict a profit of {}'.format(predict2*10000));" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": { 248 | "collapsed": true 249 | }, 250 | "outputs": [], 251 | "source": [] 252 | } 253 | ], 254 | "metadata": { 255 | "kernelspec": { 256 | "display_name": "Python 2", 257 | "language": "python", 258 | "name": "python2" 259 | }, 260 | "language_info": { 261 | "codemirror_mode": { 262 | "name": "ipython", 263 | "version": 2 264 | }, 265 | "file_extension": ".py", 266 | "mimetype": "text/x-python", 267 | "name": "python", 268 | "nbconvert_exporter": "python", 269 | "pygments_lexer": "ipython2", 270 | "version": "2.7.12" 271 | } 272 | }, 273 | "nbformat": 4, 274 | "nbformat_minor": 2 275 | } 276 | --------------------------------------------------------------------------------