├── README.md ├── mf.py └── mf.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # Implementation of Matrix Factorization in Python 2 | 3 | The source code [mf.py](mf.py) is an implementation of the matrix factorization algorithm in Python, using stochastic gradient descent. An article with detailed explanation of the algorithm can be found at [http://albertauyeung.com/2017/04/23/python-matrix-factorization.html](http://albertauyeung.com/2017/04/23/python-matrix-factorization.html). 4 | 5 | Below is an example of using the algorithm: 6 | 7 | ```python 8 | import numpy as np 9 | from mf import MF 10 | 11 | # A rating matrix with ratings from 5 users on 4 items 12 | # zero entries are unknown values 13 | R = np.array([ 14 | [5, 3, 0, 1], 15 | [4, 0, 0, 1], 16 | [1, 1, 0, 5], 17 | [1, 0, 0, 4], 18 | [0, 1, 5, 4], 19 | ]) 20 | 21 | # Perform training and obtain the user and item matrices 22 | mf = MF(R, K=2, alpha=0.1, beta=0.01, iterations=20) 23 | training_process = mf.train() 24 | print(mf.P) 25 | print(mf.Q) 26 | print(mf.full_matrix()) 27 | 28 | # Prints the following: 29 | ''' 30 | [[ 1.45345236 0.06946249] 31 | [ 1.12922538 0.2319001 ] 32 | [-1.21051208 0.94619099] 33 | [-0.93607816 0.43182699] 34 | [-0.6919936 -0.93611985]] 35 | 36 | [[ 1.42787151 -0.20548935] 37 | [ 0.84792614 0.29530697] 38 | [ 0.18071811 -1.2672859 ] 39 | [-1.4211893 0.20465575]] 40 | 41 | [[ 4.98407556 2.99856476 3.96309763 1.01351377] 42 | [ 3.99274702 2.27661831 3.20365416 1.0125506 ] 43 | [ 1.0064803 1.00498576 2.37696737 4.98530109] 44 | [ 1.00999456 0.59175173 2.58437035 3.99597255] 45 | [ 2.26471556 1.01985428 4.9871617 3.9942251 ]] 46 | ''' 47 | ``` 48 | -------------------------------------------------------------------------------- /mf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class MF(): 5 | 6 | def __init__(self, R, K, alpha, beta, iterations): 7 | """ 8 | Perform matrix factorization to predict empty 9 | entries in a matrix. 10 | 11 | Arguments 12 | - R (ndarray) : user-item rating matrix 13 | - K (int) : number of latent dimensions 14 | - alpha (float) : learning rate 15 | - beta (float) : regularization parameter 16 | """ 17 | 18 | self.R = R 19 | self.num_users, self.num_items = R.shape 20 | self.K = K 21 | self.alpha = alpha 22 | self.beta = beta 23 | self.iterations = iterations 24 | 25 | def train(self): 26 | # Initialize user and item latent feature matrice 27 | self.P = np.random.normal(scale=1./self.K, size=(self.num_users, self.K)) 28 | self.Q = np.random.normal(scale=1./self.K, size=(self.num_items, self.K)) 29 | 30 | # Initialize the biases 31 | self.b_u = np.zeros(self.num_users) 32 | self.b_i = np.zeros(self.num_items) 33 | self.b = np.mean(self.R[np.where(self.R != 0)]) 34 | 35 | # Create a list of training samples 36 | self.samples = [ 37 | (i, j, self.R[i, j]) 38 | for i in range(self.num_users) 39 | for j in range(self.num_items) 40 | if self.R[i, j] > 0 41 | ] 42 | 43 | # Perform stochastic gradient descent for number of iterations 44 | training_process = [] 45 | for i in range(self.iterations): 46 | np.random.shuffle(self.samples) 47 | self.sgd() 48 | mse = self.mse() 49 | training_process.append((i, mse)) 50 | if (i+1) % 10 == 0: 51 | print("Iteration: %d ; error = %.4f" % (i+1, mse)) 52 | 53 | return training_process 54 | 55 | def mse(self): 56 | """ 57 | A function to compute the total mean square error 58 | """ 59 | xs, ys = self.R.nonzero() 60 | predicted = self.full_matrix() 61 | error = 0 62 | for x, y in zip(xs, ys): 63 | error += pow(self.R[x, y] - predicted[x, y], 2) 64 | return np.sqrt(error) 65 | 66 | def sgd(self): 67 | """ 68 | Perform stochastic graident descent 69 | """ 70 | for i, j, r in self.samples: 71 | # Computer prediction and error 72 | prediction = self.get_rating(i, j) 73 | e = (r - prediction) 74 | 75 | # Update biases 76 | self.b_u[i] += self.alpha * (e - self.beta * self.b_u[i]) 77 | self.b_i[j] += self.alpha * (e - self.beta * self.b_i[j]) 78 | 79 | # Create copy of row of P since we need to update it but use older values for update on Q 80 | P_i = self.P[i, :][:] 81 | 82 | # Update user and item latent feature matrices 83 | self.P[i, :] += self.alpha * (e * self.Q[j, :] - self.beta * self.P[i,:]) 84 | self.Q[j, :] += self.alpha * (e * P_i - self.beta * self.Q[j,:]) 85 | 86 | def get_rating(self, i, j): 87 | """ 88 | Get the predicted rating of user i and item j 89 | """ 90 | prediction = self.b + self.b_u[i] + self.b_i[j] + self.P[i, :].dot(self.Q[j, :].T) 91 | return prediction 92 | 93 | def full_matrix(self): 94 | """ 95 | Computer the full matrix using the resultant biases, P and Q 96 | """ 97 | return self.b + self.b_u[:,np.newaxis] + self.b_i[np.newaxis:,] + self.P.dot(self.Q.T) 98 | -------------------------------------------------------------------------------- /mf.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# # Implementation of Matrix Factorization in Python\n", 8 | "\n", 9 | "Please refer to the article at http://www.albertauyeung.com/post/python-matrix-factorization/ for the detailed explanation." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": { 16 | "collapsed": true, 17 | "deletable": true, 18 | "editable": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import matplotlib\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import numpy as np" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "metadata": { 32 | "collapsed": false, 33 | "deletable": true, 34 | "editable": true 35 | }, 36 | "outputs": [], 37 | "source": [ 38 | "class MF():\n", 39 | " \n", 40 | " def __init__(self, R, K, alpha, beta, iterations):\n", 41 | " \"\"\"\n", 42 | " Perform matrix factorization to predict empty\n", 43 | " entries in a matrix.\n", 44 | " \n", 45 | " Arguments\n", 46 | " - R (ndarray) : user-item rating matrix\n", 47 | " - K (int) : number of latent dimensions\n", 48 | " - alpha (float) : learning rate\n", 49 | " - beta (float) : regularization parameter\n", 50 | " \"\"\"\n", 51 | " \n", 52 | " self.R = R\n", 53 | " self.num_users, self.num_items = R.shape\n", 54 | " self.K = K\n", 55 | " self.alpha = alpha\n", 56 | " self.beta = beta\n", 57 | " self.iterations = iterations\n", 58 | "\n", 59 | " def train(self):\n", 60 | " # Initialize user and item latent feature matrice\n", 61 | " self.P = np.random.normal(scale=1./self.K, size=(self.num_users, self.K))\n", 62 | " self.Q = np.random.normal(scale=1./self.K, size=(self.num_items, self.K))\n", 63 | " \n", 64 | " # Initialize the biases\n", 65 | " self.b_u = np.zeros(self.num_users)\n", 66 | " self.b_i = np.zeros(self.num_items)\n", 67 | " self.b = np.mean(self.R[np.where(self.R != 0)])\n", 68 | " \n", 69 | " # Create a list of training samples\n", 70 | " self.samples = [\n", 71 | " (i, j, self.R[i, j])\n", 72 | " for i in range(self.num_users)\n", 73 | " for j in range(self.num_items)\n", 74 | " if self.R[i, j] > 0\n", 75 | " ]\n", 76 | " \n", 77 | " # Perform stochastic gradient descent for number of iterations\n", 78 | " training_process = []\n", 79 | " for i in range(self.iterations):\n", 80 | " np.random.shuffle(self.samples)\n", 81 | " self.sgd()\n", 82 | " mse = self.mse()\n", 83 | " training_process.append((i, mse))\n", 84 | " if (i+1) % 10 == 0:\n", 85 | " print(\"Iteration: %d ; error = %.4f\" % (i+1, mse))\n", 86 | " \n", 87 | " return training_process\n", 88 | "\n", 89 | " def mse(self):\n", 90 | " \"\"\"\n", 91 | " A function to compute the total mean square error\n", 92 | " \"\"\"\n", 93 | " xs, ys = self.R.nonzero()\n", 94 | " predicted = self.full_matrix()\n", 95 | " error = 0\n", 96 | " for x, y in zip(xs, ys):\n", 97 | " error += pow(self.R[x, y] - predicted[x, y], 2)\n", 98 | " return np.sqrt(error)\n", 99 | "\n", 100 | " def sgd(self):\n", 101 | " \"\"\"\n", 102 | " Perform stochastic graident descent\n", 103 | " \"\"\"\n", 104 | " for i, j, r in self.samples:\n", 105 | " # Computer prediction and error\n", 106 | " prediction = self.get_rating(i, j)\n", 107 | " e = (r - prediction)\n", 108 | " \n", 109 | " # Update biases\n", 110 | " self.b_u[i] += self.alpha * (e - self.beta * self.b_u[i])\n", 111 | " self.b_i[j] += self.alpha * (e - self.beta * self.b_i[j])\n", 112 | " \n", 113 | " # Create copy of row of P since we need to update it but use older values for update on Q\n", 114 | " P_i = self.P[i, :][:]\n", 115 | " \n", 116 | " # Update user and item latent feature matrices\n", 117 | " self.P[i, :] += self.alpha * (e * self.Q[j, :] - self.beta * self.P[i,:])\n", 118 | " self.Q[j, :] += self.alpha * (e * P_i - self.beta * self.Q[j,:])\n", 119 | "\n", 120 | " def get_rating(self, i, j):\n", 121 | " \"\"\"\n", 122 | " Get the predicted rating of user i and item j\n", 123 | " \"\"\"\n", 124 | " prediction = self.b + self.b_u[i] + self.b_i[j] + self.P[i, :].dot(self.Q[j, :].T)\n", 125 | " return prediction\n", 126 | " \n", 127 | " def full_matrix(self):\n", 128 | " \"\"\"\n", 129 | " Computer the full matrix using the resultant biases, P and Q\n", 130 | " \"\"\"\n", 131 | " return mf.b + mf.b_u[:,np.newaxis] + mf.b_i[np.newaxis:,] + mf.P.dot(mf.Q.T)\n" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 3, 137 | "metadata": { 138 | "collapsed": false, 139 | "deletable": true, 140 | "editable": true 141 | }, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "Iteration: 2 ; error = 4.8575\n", 148 | "Iteration: 12 ; error = 0.1299\n", 149 | "\n", 150 | "P x Q:\n", 151 | "[[ 4.98421651 2.99708195 3.12289161 1.00971487]\n", 152 | " [ 3.99564826 2.1268789 3.50941346 1.01124379]\n", 153 | " [ 1.01613916 0.98386613 4.5843299 4.9908544 ]\n", 154 | " [ 0.99513504 0.75011143 3.73365012 3.98862199]\n", 155 | " [ 1.61407605 1.02983448 4.98695019 3.99475012]]\n", 156 | "\n", 157 | "Global bias:\n", 158 | "2.76923076923\n", 159 | "\n", 160 | "User bias:\n", 161 | "[ 0.1810972 -0.0346873 0.11877163 -0.44968314 0.29954531]\n", 162 | "\n", 163 | "Item bias:\n", 164 | "[ 0.34310914 -0.81457993 0.851025 -0.22669804]\n" 165 | ] 166 | } 167 | ], 168 | "source": [ 169 | "R = np.array([\n", 170 | " [5, 3, 0, 1],\n", 171 | " [4, 0, 0, 1],\n", 172 | " [1, 1, 0, 5],\n", 173 | " [1, 0, 0, 4],\n", 174 | " [0, 1, 5, 4],\n", 175 | "])\n", 176 | "\n", 177 | "mf = MF(R, K=2, alpha=0.1, beta=0.01, iterations=20)\n", 178 | "training_process = mf.train()\n", 179 | "print()\n", 180 | "print(\"P x Q:\")\n", 181 | "print(mf.full_matrix())\n", 182 | "print()\n", 183 | "print(\"Global bias:\")\n", 184 | "print(mf.b)\n", 185 | "print()\n", 186 | "print(\"User bias:\")\n", 187 | "print(mf.b_u)\n", 188 | "print()\n", 189 | "print(\"Item bias:\")\n", 190 | "print(mf.b_i)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 4, 196 | "metadata": { 197 | "collapsed": false, 198 | "deletable": true, 199 | "editable": true 200 | }, 201 | "outputs": [ 202 | { 203 | "data": { 204 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6oAAAEKCAYAAAAb/6jZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//H3Z2ayAAkoSwKKFgQRRXbEBVS0dcOtrdq6\n1rVo625vvVr7a2vvbavV2muVaq1a3G/rWq24cG3RigqyL4LKIgiyrwmQbebz+2MmYRKyTCCTM0le\nz8fjPDLnzDln3hk0yfuc7zlj7i4AAAAAADJFKOgAAAAAAAAko6gCAAAAADIKRRUAAAAAkFEoqgAA\nAACAjEJRBQAAAABkFIoqAAAAACCjUFQBAAAAABmFogoAAAAAyCgUVQAAAABARokEHSBZ165dvVev\nXkHHAAAAAAA0sRkzZmxw926prJtRRbVXr16aPn160DEAAAAAAE3MzJanui5DfwEAAAAAGYWiCgAA\nAADIKBRVAAAAAEBGoagCAAAAADIKRRUAAAAAkFEoqgAAAACAjEJRBQAAAABkFIpqij5YvEETpixT\nNOZBRwEAAACAVo2imqLX563WL177ROc9/IEWrysKOg4AAAAAtFoU1RT99zcP1++/O1jLNmzX2Pvf\n1wPvfK6yiljQsQAAAACg1aGopsjM9K2hPTXpluN1yuHd9btJn+msB9/X3JVbgo4GAAAAAK0KRbWR\nuubl6IELhurP3xuhzTvK9M3xU/TriQu1sywadDQAAAAAaBUoqnvopMMKNemW43X+yAP1yHtLder9\n7+mDJRuCjgUAAAAALR5FdS90zM3Sr781UM99/yiZpAv/PFW3vzRP20rKg44GAAAAAC0WRbUJHN2n\ni9648ThdfdxB+uvHK3TSfe9q0idrg44FAAAAAC0SRbWJtMsO6/axh+qVa0dp3/bZ+v6T03XdszO1\nobg06GgAAAAA0KJQVJvYoJ776LXrR+s/Tu6ntxes1Tfue1cvzVwpdw86GgAAAAC0CBTVNMgKh3Td\niQdr4o2j1adbnm752xxd9pePtWrLzqCjAQAAAEDGo6imUd+CfD1/9dG686wB+viLTTr5vnf15Idf\nKBbj7CoAAAAA1IWimmahkOnSY3rp7ZuP0/BenfWzvy/Qd/70oRavKw46GgAAAABkJIpqM+m5b3s9\ncfkR+t15g7V4fbHG3v9vjf/XYpVHY0FHAwAAAICMQlFtRmamc4b31KSbj9dJAwp1z1uf6qwHp2je\nyq1BRwMAAACAjEFRDUC3/ByNv3CY/nTJcG0sLtU3/zhFv3ljoUrKo0FHAwAAAIDAUVQDdMqA7pp0\ny/H6zoie+tO7S3Xa/f/WR0s3Bh0LAAAAAAJFUQ1Yp3ZZ+s23B+nZq45UNOY6/5GPdMfL81RUUh50\nNAAAAAAIBEU1QxzTt6veuuk4ff/Y3npu2gqddN97emfh2qBjAQAAAECzo6hmkHbZYd1x+mF6+Yej\ntE/7LF35xHTd8NwsbSwuDToaAAAAADSbtBZVM/vCzOaZ2Wwzm57O12pNBh+wj169brRuOamf3pi/\nWt+47129MmuV3D3oaAAAAACQds1xRvUEdx/i7iOa4bVajexISDd8/WC9fsOx6tW1g27662xdMeFj\nfbVlZ9DRAAAAACCtGPqb4foV5uuFa47Rz888TB8t3aSTf/+envpouWIxzq4CAAAAaJ3SXVRd0ttm\nNsPMxqX5tVqtcMh0+ajeevvm4zT0wH30/16Zr/Mf+UhL1xcHHQ0AAAAAmlwkzfsf7e6rzKxA0iQz\nW+Tu7yWvkCiw4ySpsLBQkydPTnOklu2Kg1z9crL13KJNOvn37+qbfbN0aq8sRUIWdDQAAAAAaBLW\nXDfoMbNfSCp293vrWmfEiBE+fTr3XErFuqIS/eLVBZo4b40G7NdRd58zSIfv3ynoWAAAAABQKzOb\nkeq9i9I29NfMOphZfuVjSSdLmp+u12trCvJz9ceLhuvhi4dpXVGpzh4/RXe/uUgl5dGgowEAAADA\nXknnNaqFkt43szmSpkl63d3fTOPrtUmnHt5D/3fz8Tpn2P56aPISjb3/35q2bFPQsQAAAABgjzXb\n0N9UMPR377z/+Qbd/vJcfblpp278+sG6+aR+QUcCAAAAAEkZMvQXzW/0wV311k3H6ZxhPXX/O5/r\noclLgo4EAAAAAI2W7rv+opm1z47ot+cOUkUsprvfXKS83IguOeprQccCAAAAgJRRVFuhcMh073mD\nVVxSoZ/9fb7ycsL61tCeQccCAAAAgJQw9LeVygqHNP6iYTqqdxf9x/Nz9faCNUFHAgAAAICUUFRb\nsdyssP586Qgdvn8nXffsLE1ZvCHoSAAAAADQIIpqK5eXE9ETlx+h3l076PtPTtfMFZuDjgQAAAAA\n9aKotgH7tM/WU1eOVLf8HF32+DQtXL0t6EgAAAAAUCeKahtR0DFXT195pDrkRHTJY9O0bMP2oCMB\nAAAAQK0oqm3IAZ3b66krj5S76+JHp2rVlp1BRwIAAACA3VBU25i+BXl64oqR2razXJc8OlXri0qD\njgQAAAAA1VBU26DD9++kxy8/Ql9t3anvPT5NW3eUBx0JAAAAAKpQVNuoI3p11p8uGaHF64p0+YRp\n2lFWEXQkAAAAAJBEUW3Tju/XTX84f6hmf7lFVz81Q6UV0aAjAQAAAABFta07bWAP/fbcwfr35xt0\nw3OzVBGNBR0JAAAAQBtHUYXOHd5TvzjzML21YK1ufWGuYjEPOhIAAACANiwSdABkhstG9VZRSYV+\nN+kz5eVGdOdZA2RmQccCAAAA0AZRVFHluhP7qqi0Qo+8t1T5uRH9+JT+QUcCAAAA0AZRVFHFzHT7\naf1VVFKu8f9aovzcLF1zfJ+gYwEAAABoYyiqqMbM9N/fHKiikgrd9cYi5edGdNGRXws6FgAAAIA2\nhKKK3YRDpt9/d4h2lEX101fmKy8norOH7B90LAAAAABtBHf9Ra2ywiH98aJhOrJ3Z93ytzn6v0/W\nBh0JAAAAQBtBUUWdcrPCevTSI3T4fh31w2dn6oPFG4KOBAAAAKANoKiiXnk5EU24fKR6d+mgq56c\nrlkrNgcdCQAAAEArR1FFg/btkK2nrhyprnk5uuwvH2vh6m1BRwIAAADQilFUkZKCjrl65qoj1S4r\nrEsem6YvNmwPOhIAAACAVqreompmYTO7ubnCILMd0Lm9nr5qpGLuuujRqfpqy86gIwEAAABoheot\nqu4elXRBM2VBC9C3IF9PXjFS23aW6+LHpmpDcWnQkQAAAAC0MqkM/Z1iZg+a2bFmNqxySvUFEmdl\nZ5nZP/YiJzLI4ft30uOXH6GvtuzU9x6bpq07y4OOBAAAAKAVSaWoDpE0QNIvJf0uMd3biNe4UdLC\nxkdDJjuiV2f96ZIR+nxdka6Y8LF2lFUEHQkAAABAK9FgUXX3E2qZTkxl52bWU9Lpkh7d26DIPMf3\n66Y/nD9Us1Zs1tVPzVBpRTToSAAAAABagUhDK5hZJ0k/l3RcYtG7kn7p7ltT2P//SLpVUn49+x8n\naZwkFRYWavLkySnsFpminaTLB2TrsfkbdOEDk/TDwTkKhyzoWAAAAABasAaLqqTHJc2X9J3E/CWS\n/iLp2/VtZGZnSFrn7jPMbExd67n7I5IekaQRI0b4mDF1rooMNUbS/u8v0y//8Ykmbuise84dpBBl\nFQAAAMAeSqWo9nH3c5Lm7zSz2SlsN0rSWWY2VlKupI5m9rS7X7wnQZHZrhjdW0UlFfr9/32m/NyI\nfn7mYTKjrAIAAABovFRuprTTzEZXzpjZKEkNfoCmu9/u7j3dvZek8yX9k5Laut3w9b66anRvTfjg\nC9036bOg4wAAAABooVI5o3qNpCcT16pK0mZJl6YvEloqM9Mdpx+q4tIKPfDPxcrPjWjccX2CjgUA\nAACgham3qJpZSNIh7j7YzDpKkrtva+yLuPtkSZP3JCBaFjPTr741UMWlFfr1xEXKy8nShUceGHQs\nAAAAAC1IvUXV3WNmdqukv+1JQUXbFA6Z7vvOEG0vrdAdr8xTh5ywzh6yf9CxAAAAALQQqVyj+n9m\n9h9mdoCZda6c0p4MLVp2JKSHLh6ukb0660d/m6N3Fq4NOhIAAACAFiKVovpdSddKek/SjMQ0PZ2h\n0DrkZoX16KUjdNh+HfXDZ2bqwyUbg44EAAAAoAWot6gmrlG92N1715gOaqZ8aOHyc7P0xOUjdWDn\n9rrqiY81+8stQUcCAAAAkOHqLaruHpP0YDNlQSu1b4dsPX3VkeqSl6NLH5+mT9cUBR0JAAAAQAZL\nZejvO2Z2jplZ2tOg1SrsmKtnrjpSuVkhXfzYVH2xYXvQkQAAAABkqFSK6tWSnpdUambbzKzIzLgD\nMBrtgM7t9fSVR6oiGtNFj07V6q07g44EAAAAIAM1WFTdPd/dQ+6e7e4dE/MdmyMcWp+DC/P15BVH\natvOcl386FRtLC4NOhIAAACADFNnUTWzi5Mej6rx3HXpDIXWbWDPTnrssiO0astOfe/xadpWUh50\nJAAAAAAZpL4zqrckPX6gxnNXpCEL2pCRvTvr4YuH69M1RfrR3+YoFvOgIwEAAADIEPUVVavjcW3z\nQKONOaRAPxl7qCZ9slYPv7ck6DgAAAAAMkR9RdXreFzbPLBHLh/VS2cM6qF73/pUUxZvCDoOAAAA\ngAxQX1Htb2ZzzWxe0uPK+UOaKR9aOTPT3ecMUp9uebrhuVncCRgAAABAvUX1UElnSjoj6XHl/GHp\nj4a2okNORA9dPFwl5VH98JmZKquIBR0JAAAAQIDqLKruvry+qTlDovXrW5Cne84brFkrtuhXr38S\ndBwAAAAAAWrwc1SB5jJ2YA99/9jeeuLD5Xpl1qqg4wAAAAAICEUVGeXWU/trZK/Ouv2lefp0TVHQ\ncQAAAAAEIKWiambtzIwbKCHtssIhPXjhUOXlRnTN0zO0raQ86EgAAAAAmlmDRdXMzpQ0W9Kbifkh\nZvZquoOh7SromKvxFw7Tik079OPn58idT0MCAAAA2pJUzqj+QtJISVskyd1nS+qdxkyARvburNtP\n66+3FqzVn95bGnQcAAAAAM0olaJa7u5bayzjFBfS7srRvXX6oB767ZuL9MGSDUHHAQAAANBMUimq\nC8zsQklhMzvYzB6Q9EGacwEyM919ziD17tpBNzw3S2u2lgQdCQAAAEAzSKWoXi9pgKRSSc9K2irp\npnSGAirl5UT0p0uGa2dZVD98ZobKKmJBRwIAAACQZvUWVTMLS/qlu9/h7kckpp+6O6e20Gz6FuTr\nt+cO1swVW/TriQuDjgMAAAAgzeotqu4elTS6mbIAdTp9UA9dObq3Jnzwhf4+e1XQcQAAAACkUSSF\ndWYlPo7meUnbKxe6+0tpSwXU4rbT+mvuyi267cV56t+9ow7pnh90JAAAAABpkMo1qrmSNko6UdKZ\niemMhjYys1wzm2Zmc8xsgZnduXdR0dZlhUMaf+Ew5eVG9IOnZ6iopDzoSAAAAADSoMEzqu5++R7u\nu1TSie5ebGZZkt43szfc/aM93B+ggo65Gn/hMF3w54/04+fn6qGLh8nMgo4FAAAAoAk1eEY1cWb0\nWjP7o5k9Xjk1tJ3HFSdmsxITn7+KvTayd2fdflp/vblgjR55b2nQcQAAAAA0sVSG/j4lqbukUyS9\nK6mnpKJUdm5mYTObLWmdpEnuPnVPgwLJrhzdW2MHdtfdby7Sh0s2Bh0HAAAAQBNK5WZKfd39PDM7\n292fMLNnJf07lZ0n7ho8xMz2kfSymR3u7vOT1zGzcZLGSVJhYaEmT57cuO8AbdaZha6ZS01XPzFV\ndx6Tq31zUznuAgAAACDTpVJUK+9Ys8XMDpe0RlJBY17E3beY2b8knSppfo3nHpH0iCSNGDHCx4wZ\n05hdo43rO7BIZ4+fomeW5eq5cUcpK0xZBQAAAFq6VP6qf8TM9pX0/yS9KukTSb9taCMz65Y4kyoz\nayfpJEmL9iIrsJuDC/N19zmDNH35Zv164sKg4wAAAABoAqnc9ffRxMN3JR3UiH33kPSEmYUVL8R/\nc/d/ND4iUL8zB++nmSs26y9TvtDQA/fVWYP3CzoSAAAAgL3QYFE1s5/Vttzdf1nfdu4+V9LQPcwF\nNMpPxh6qeSu36rYX5+rQ7vk6uDA/6EgAAAAA9lAqQ3+3J01RSadJ6pXGTECjZYVDGn/RMLXPjujq\np2eoqKS84Y0AAAAAZKQGi6q7/y5p+pWkMWrcEGCgWRR2zNWDFw7V8o07dOsLc+XOx/YCAAAALdGe\n3CK1veKfpQpknKMO6qL/PPUQvTF/jR7997Kg4wAAAADYA6lcozpPUuWpqbCkbpLqvT4VCNL3jz1I\ns1Zs0V1vLtKgnp105EFdgo4EAAAAoBFSOaN6hqQzE9PJkvZz9wfTmgrYC2am3547SF/r0l7XPjtL\na7eVBB0JAAAAQCOkUlSLkqadkjqaWefKKa3pgD2Un5ulhy8eru2lFbr2mZkqj8aCjgQAAAAgRakU\n1ZmS1kv6TNLnicczEtP09EUD9k6/wnzdfe4gTV++Wb+ZuCjoOAAAAABSlEpRnSTpTHfv6u5dFB8K\n/La793Z37v6LjHbW4P102TG99PiUZfrH3K+CjgMAAAAgBakU1aPcfWLljLu/IemY9EUCmtZPxh6q\nYQfuo1tfmKvP1xYFHQcAAABAA1Ipql+Z2U/NrFdiukMSp6bQYmRHQvrjRcPVPjusa56eoeLSiqAj\nAQAAAKhHKkX1AsU/kublxFSQWAa0GN075eoPFwzVsg3b9Z8vzJW7N7wRAAAAgEA0WFTdfZO73+ju\nQyWdKOkmd9+U/mhA0zqmT1fdemp/vT5vtR57f1nQcQAAAADUoc6iamY/M7P+icc5ZvZPSYslrTWz\nbzRXQKApXX3cQTplQKF+88YiTVvG8RYAAAAgE9V3RvW7kj5NPL40sW6BpOMl/TrNuYC0MDPdc95g\nHdi5va59dqbWbSsJOhIAAACAGuorqmW+60K+UyQ95+5Rd18oKZL+aEB6dMzN0sMXD1dxSYWue3aW\nyqOxoCMBAAAASFJfUS01s8PNrJukEyS9nfRc+/TGAtLrkO75uuucgZr2xSbd/caioOMAAAAASFLf\nmdEbJb2g+B1/f+/uyyTJzMZKmtUM2YC0OnvI/pq5fLMefX+Zhh64r04f1CPoSAAAAABUT1F196mS\n+teyfKKkiekMBTSXO04/TPNWbdWtL8zRId3z1LcgP+hIAAAAQJuXyueoAq1WdiSk8RcNU25WWNc8\nPVPbSyuCjgQAAAC0eRRVtHk9OrXTAxcM1dL1xbr1xbnadQ8xAAAAAEGgqAKSjunbVT8+pb9en7ta\nj0/5Iug4AAAAQJuW0sfMmNkxknolr+/uT6YpExCIa44/SDNXbNZvJi7UoJ6ddESvzkFHAgAAANqk\nBs+omtlTku6VNFrSEYlpRJpzAc3OzPS77wxWz33b6dpnZmpdUUnQkQAAAIA2KZWhvyMkjXL3H7r7\n9YnphnQHA4LQMTdLD108XNtKynXds7NUHo0FHQkAAABoc1IpqvMldU93ECBTHNqjo37z7YGatmyT\n7nnr06DjAAAAAG1OKteodpX0iZlNk1RaudDdz0pbKiBg3xraUzOXb9Ej7y3V0AP20WkDewQdCQAA\nAGgzUimqv9iTHZvZAZKelFQoySU94u7378m+gCD89IxDNW/VVv34hbnq1z1ffbrlBR0JAAAAaBMa\nHPrr7u/WNqWw7wpJP3L3wyQdJelaMztsbwMDzSUnEtYfLxqm7EhI1zw1Q9tLK4KOBAAAALQJqdz1\n9ygz+9jMis2szMyiZratoe3cfbW7z0w8LpK0UNL+ex8ZaD777dNOD1wwVEvWF+vWF+cqGvOgIwEA\nAACtXio3U3pQ0gWSPpfUTtJVksY35kXMrJekoZKmNi4eELxRfbvq1lP76/W5q3X5hI+1dUd50JEA\nAACAVi2Va1Tl7ovNLOzuUUl/MbNZkm5PZVszy5P0oqSb3H23M7FmNk7SOEkqLCzU5MmTU80ONJv+\nki4bkK2nPlmvk+6dpBuH5mr//FSO8wAAAABoLHOvfyijmb0n6RuSHpW0RtJqSZe5++AGd26WJekf\nkt5y9/saWn/EiBE+ffr0VHIDgZixfJOueXqmdpRW6L7vDtEpA/jkJgAAACAVZjbD3Ueksm4qp4Qu\nSax3naTtkg6QdE4KIUzSY5IWplJSgZZg+Nc667XrRqtvYb6ufmqG7pv0mWJctwoAAAA0qVTu+rtc\nkknq4e53uvst7r44hX2PUrzknmhmsxPT2L3MCwSue6dc/XXcUTp3eE/94Z3PNe6pGSoq4bpVAAAA\noKmkctffMyXNlvRmYn6Imb3a0Hbu/r67m7sPcvchiWni3kcGgpebFdY95w7SnWcN0L8+Xadvjp+i\npeuLg44FAAAAtAqpDP39haSRkrZIkrvPltQ7jZmAFsHMdOkxvfT0lUdq845ynf3gFP1z0dqgYwEA\nAAAtXipFtdzdt9ZYxkV5QMLRfbro1etG6cAu7XXlE9M1/l+L1dBNygAAAADULZWiusDMLpQUNrOD\nzewBSR+kORfQovTct71euOYYnTloP93z1qe69tmZ2l5aEXQsAAAAoEVKpaheL2mApFJJz0naJumm\ndIYCWqJ22WHdf/4Q3TH2UL05f43OeegDrdi4I+hYAAAAQIvT4OeoNic+RxWtxXufrdf1z82SJD14\n4VAde3C3gBMBAAAAwWrM56jWWVQburOvu5+1B9nqRVFFa7J843aNe3KGPl9XpNtPO1RXHdtb8Y8X\nBgAAANqexhTVSD3PHS3pS8WH+05V/LNUAaToa1066KUfHqP/eH6OfjVxoRZ8tVV3nTNIuVnhoKMB\nAAAAGa2+a1S7S/qJpMMl3S/pJEkb3P1dd3+3OcIBLV2HnIj+eNEw/fiUQ/T3OV/p3Ic/0KotO4OO\nBQAAAGS0Oouqu0fd/U13v1TSUZIWS5psZtc1WzqgFTAzXXtCXz126Qgt37BDZz3wvj5aujHoWAAA\nAEDGqveuv2aWY2bflvS0pGsl/UHSy80RDGhtTuxfqFeuG6VO7bN08aNT9cQHX/B5qwAAAEAt6iyq\nZvakpA8lDZN0p7sf4e7/5e6rmi0d0Mr06ZanV64dpTGHdNPPX12g/3xxrkrKo0HHAgAAADJKfXf9\njUnanphNXskkubt3bOow3PUXbUUs5vqfdz7XH975XEMO2Ed/umS4CjvmBh0LAAAASJvG3PW3vmtU\nQ+6en5g6Jk356SipQFsSCpluOamfHr54uD5fW6QzHnhfM5ZvDjoWAAAAkBHqvUYVQHqdenh3vXzt\nKLXPDuv8Rz7U/05bEXQkAAAAIHAUVSBg/Qrz9eq1o3V0n6667aV5+ukr81RWEQs6FgAAABAYiiqQ\nATq1z9JfLjtC1xzfR09/tEIXPfqR1heVBh0LAAAACARFFcgQ4ZDpttP66w8XDNW8VVt15gPva+7K\nLUHHAgAAAJodRRXIMGcN3k8v/uAYhUOmcx/+UC/OWBl0JAAAAKBZUVSBDDRgv0567frRGn7gvvrR\n83P0y9c+UUWU61YBAADQNlBUgQzVuUO2nrxypC4f1UuPT1mm7z0+TZu2lwUdCwAAAEg7iiqQwbLC\nIf38zAG697zBmr58s8584H0t+Gpr0LEAAACAtKKoAi3AucN76vmrj1Y05jrnoQ/02pyvgo4EAAAA\npA1FFWghBh+wj167frQG7t9J1z83S3e9sUjRmAcdCwAAAGhyFFWgBemWn6NnrjpKFx91oB5+d4ku\nn/Cxtu4oDzoWAAAA0KQoqkALkx0J6b+/OVC/+fZAfbhkg84a/74+W1sUdCwAAACgyVBUgRbqgpEH\n6n/HHaUdZVF9a/wUvTl/TdCRAAAAgCZBUQVasOFf66zXrhutvoX5uubpGbpv0meKcd0qAAAAWri0\nFVUze9zM1pnZ/HS9BgCpe6dc/XXcUTpveE/94Z3PNe6pGSoq4bpVAAAAtFzpPKM6QdKpadw/gITc\nrLB+e+4g/fLsAZr86TqNvvtf+vHzc/SvT9eprCIWdDwAAACgUSLp2rG7v2dmvdK1fwDVmZm+d3Qv\nDdy/k576cLnenL9Gz89YqY65EZ10WHedPqi7RvXtqpxIOOioAAAAQL3SVlRTZWbjJI2TpMLCQk2e\nPDnYQEArcFahdFq3bC3YENbHa6J6Y+5KvThzpdpFpKEFER3RPawBXcLKDlvQUQEAAIDdmHv6bryS\nOKP6D3c/PJX1R4wY4dOnT09bHqCtKquIacqSDZo4d7Xe/mSttu4sV15ORN84tECnDeyh4/t1U24W\nZ1oBAACQPmY2w91HpLJu4GdUAaRfdiSkEw4p0AmHFOjX0Zg+WLJRE+eu1lufrNErs79Sh+ywvn5o\nocYO7K4xhxRQWgEAABAoiirQxmSFQzq+Xzcd36+b/jt6uD5aulET563WWwvW6tU5X6l9dlgn9C/Q\n6QN76IRDCtQum9IKAACA5pW2ob9m9pykMZK6Slor6efu/lh92zD0FwhORTSmqcs26fV5q/XW/DXa\nuL1M7bLCOqF/N40d2EMn9i9Q+2yObQEAAGDPNGbob1qvUW0siiqQGaIx19RlG/XGvDV6Y/4abSgu\nVW5WSGP6FWjsoB76ev8CdcihtAIAACB1FFUATSYac338xSa9MW+13pi/RuuKSpUTiQ8fPn1Q/Exr\nfm5W0DEBAACQ4SiqANIiFnNNX75ZE+et1hvzV2vttlJlR0I67uBuOn1Qd3390EJ1pLQCAACgFhRV\nAGkXi7lmrtisifPW6I35q7V6a4mywyEde3BXnTawh046rFCd2lFaAQAAEEdRBdCsYjHXrC+3VA0P\nXrVlp7LCptF946X15MMKtU/77KBjAgAAIEAUVQCBcXfNWblVE+et1utzV2vVlp2KhEyj+nbV2IHd\ndfJh3bVvB0orAABAW0NRBZAR3F3zVm3V6/NWa+K81fpy006FQ6Zj+nTR2IE9dMqA7upMaQUAAGgT\nKKoAMo67a8FX26pK6/KNOxQOmQ7q2kF9uuWpT0H8a9+CPB3ULU95fPwNAABAq0JRBZDR3F2frN6m\ntxas1cIwyX0OAAAVGklEQVTV27RkfbGWb9yhaGzXz6PuHXOrldc+3eJTYcccmVmA6QEAALAnGlNU\nOWUBoNmZmQbs10kD9utUtaysIqYVm7Zr8brtWrK+OD6tK9ZLM1epuLSiar28nIj6dKs8C5unPt06\nqG9Bng7s3EHZkVAQ3w4AAACaGEUVQEbIjoTUtyBffQvyqy13d60rKtWSdfHyunhdsZas364Pl27U\nS7NWVa0XDpm+1rm9DkoMI+5bVWTz+JgcAACAFoaiCiCjmZkKO+aqsGOujunbtdpzxaUVWlp19nV7\nVZF997N1Ko/uGkbcNS9HfRPDiJPPxO7XqZ1CIYYRAwAAZBqKKoAWKy8nokE999GgnvtUW14RjenL\nzTu1ZF2xFieGEC9ZX6zX5nylbSW7hhG3ywrroG67CmzfgvjZ2F5dOig3K9zc3w4AAAASKKoAWp1I\nOKTeXTuod9cO+oYKq5a7uzYUlyVdA7tdi9cXa8byzXp1zldV65lJB+zbvur61z7d8tS7awd175Sr\ngvxctcumxAIAAKQTRRVAm2Fm6pafo275OTrqoC7VnttZFtXSDbuuga28mdOUJRtVVhGrtm5+TqRq\nP93yc1SQn6uCjjnqlpcT/5pYtm/7LO5QDAAAsAcoqgAgqV12eLc7EUtSNOZatXmnlm3crrXbSrS+\nqLRqWldUonmrtmrdtnXaWR7dbZ9ZYVPXvBwVVJXa3KrHVV875qpbXg53LAYAAEhCUQWAeoRDpgO7\ntNeBXdrXu15xaUW8vG4r0bqqIrur0K7cvFOzVmzRxu1ltW6/T/uspBKbm1Ruq5+1zc+JcJYWAAC0\nehRVAGgCeTkR5eVE1Ltrh3rXK4/GtLG4TOuKSnYrs+u2lWp9cammLduk9cWluw05lqScSGjXMOM6\nhhx3yctWp3ZZapcVptQCAIAWiaIKAM0oKxxS90656t4pt9713F3bdlbUWmgr5xevL9aHSzdq687y\nWvcRCZk6tstSp3ZZ6pgbUcd2WfEpN0sd20XUMTfxXNLznZKez4lw0ygAABAMiioAZCAzU6f2WerU\nPksHF+bXu25JeVQbiuPldd22Um3aXqZtJeXatrNcW3eWa1tJRdXjVVt2atvO+HxZdPcztslys0KJ\n0hovsp2Sim78caTq+U41CnB+bkSRMNfdAgCAPUNRBYAWLjcrrJ77tlfPfeu/jramkvKotu0s17aS\nRKHdWVFnwd1WUq4NxWVaumF7Yt1yxbz+/eflRGo9k7ur1MYLbV5ORB1yIsrLCSsvJ0sdcsJVy7Io\nuwAAtEkUVQBoo3KzwsrNCqugY/3DkGvj7tpeFq0qrdsSxbZqvpbyu3LzDhWtjpffotKKlF4nJxJK\nKrKVpTasDjkR5edG1CE7sutxPYU3LyeinEiIa3YBAGghKKoAgEYzs6riuP8+7Rq9fUU0pqKSChWX\nxqftpTUfR3dbtr20QkUlFVpfXKovNu6IP1dSUetHA9UmErK9Kry5WWFlh0PKyQolvoaVEwkpEjIK\nMAAATYyiCgBodpFwSPt2yNa+HbL3el/RmGt7Wby0bq8qtlEVl5bXWngrC+72sgptK6nQ6q0lu7Yt\nq5A3MKS5JrP4md/K8lpbma2csiMh5UR2rVNtWY11dj2uvqyudbgmGADQmlBUAQAtWjhk8Wtec7P2\nel/urp3lURWXJBfe+OPSiqjKKmIqrYglvtacr5yi1ZaVVcT3sWl73dtWNHTBbwpCpqrCm1xwqwpz\nOFTHcyFlh6tvV7MkVz5fNZ+8Xi3PcZYZALC3KKoAACSYmdpnR9Q+O6KCZnzdimhMZdHYbkW4tEYR\nrr0gR1VaHt++tCKm0vJo1eOyGtvuLI9qy86y3ZaXVcRUmnj9ppB8ljm7xpng5FKbFY6X2nDIFAmF\nFE48js8nfw0pEjaFrMbycNLzIVNot+2q73e358LxbcOWPF9921BIVfvIClffJwAgfSiqAAAELBKO\nD91tv/cjofeKu+9WmKs9jlYvz9XXiyatV3fBLovGVFoeU0l5/DrlaMwVjbkqqr7GFI26op60PFr9\n+SY4Ab3XzFRVaLNCoarSXFliK0tvVtJ8zecrS3ZWuHopT56PJJf5pMdZ4V0Fvea8mRQyS0zxAzCh\nymWhyvldy0xJ64SSt0laJ2mf8cf1r5P8ujW/Vt9P9e05Ew+gUlqLqpmdKul+SWFJj7r7Xel8PQAA\nsOfMLHHta1j1f3pvsNxrltukEhtTvOwmL48ml9zk+eSvMUVr2zbmikZj1ebj28eXJc+XJ5XqisTz\nyfOVWSpiMZVUxPdXnryvpJy7nkvsK7Gf1i65vDZcbHeV5vrK727PhXZtH0rx9Sqfk3bft5LXU/Vl\nlfOh0K5tkzNJyfuqY9v4kYTE/ncdUKi5rWlXpuTsSvq+LGmb5O83ed6063uu/h7sOjhRPVvtBzF2\nrVO5/+rZEtGS//Wr/XdQfUn1gxjVlyfvYfd91Pzva0/2V5k/+b2p+R5Vf2+Tvnfteu/QOGkrqmYW\nljRe0kmSVkr62MxedfdP0vWaAACg9TNLnJEMB52kebm7Yi6VR2NJpbl6sS6PVp5xjq8bc1csFv/q\nlfOJ/VTur/K55Pnq61euU339htapeo1Y4nHV9xCfjyaKd+XzdW5fy+tW37/kqm2dpH3GUt9nNHHW\nvmqdmMtVuV58+6rvX/GvSs6s+OtJNd/LXf8uyd+nauwr5tptWWNv8obMVPPAQHLhryy9psqRDdVH\nO1Qv+InREbUcVPjOiJ4ad1yfYL/RJpLOM6ojJS1296WSZGb/K+lsSRRVAACARjIzhU0Kh9pYQ4ek\nXeW2WsnV7uV89+JbWYaTlscSBydqFHyvUbaTXyP5tT1pP5Xr1zx44TX249W+h137Vjxyte9z9+89\n6XHS2tWWV1tn931V22sq+0taPfn9qXr/VPeBi5oHfqptm/Qe1PU+Vtu28n1LOiCza53dc3TNy9nt\n/Wup0llU95f0ZdL8SklHpvH1AAAAgFap8kBF9YGqQOsV+M2UzGycpHGSVFhYqMmTJwcbCAAAAAAQ\nqHQW1VWSDkia75lYVo27PyLpEUkaMWKEjxkzJo2RAAAAAACZLpTGfX8s6WAz621m2ZLOl/RqGl8P\nAAAAANAKpO2MqrtXmNl1kt5S/ONpHnf3Bel6PQAAAABA65DWa1TdfaKkiel8DQAAAABA65LOob8A\nAAAAADQaRRUAAAAAkFEoqgAAAACAjGLuHnSGKma2XtLyoHPUo6ukDUGHSBFZm15LySmRNR1aSk6J\nrOnSUrK2lJwSWdOhpeSUyJouLSVrS8kpkbUpfc3du6WyYkYV1UxnZtPdfUTQOVJB1qbXUnJKZE2H\nlpJTImu6tJSsLSWnRNZ0aCk5JbKmS0vJ2lJySmQNCkN/AQAAAAAZhaIKAAAAAMgoFNXGeSToAI1A\n1qbXUnJKZE2HlpJTImu6tJSsLSWnRNZ0aCk5JbKmS0vJ2lJySmQNBNeoAgAAAAAyCmdUAQAAAAAZ\nhaKaIjM71cw+NbPFZnZb0HnqYmaPm9k6M5sfdJb6mNkBZvYvM/vEzBaY2Y1BZ6qLmeWa2TQzm5PI\nemfQmepjZmEzm2Vm/wg6S33M7Aszm2dms81setB56mNm+5jZC2a2yMwWmtnRQWeqjZkdkng/K6dt\nZnZT0LlqY2Y3J/5/mm9mz5lZbtCZ6mJmNyZyLsi097O2n/lm1tnMJpnZ54mv+waZsVIdWc9LvK8x\nM8uYu1TWkfWexM+AuWb2spntE2TGRKbacv5XIuNsM3vbzPYLMmOl+v4+MbMfmZmbWdcgstVUx/v6\nCzNblfTzdWyQGROZan1Pzez6xH+rC8zst0HlS1bHe/rXpPfzCzObHWTGSnVkHWJmH1X+zWJmI4PM\nWKmOrIPN7MPE31ivmVnHIDPuDYpqCswsLGm8pNMkHSbpAjM7LNhUdZog6dSgQ6SgQtKP3P0wSUdJ\nujaD39NSSSe6+2BJQySdamZHBZypPjdKWhh0iBSd4O5DWsBt1O+X9Ka795c0WBn6/rr7p4n3c4ik\n4ZJ2SHo54Fi7MbP9Jd0gaYS7Hy4pLOn8YFPVzswOl/R9SSMV/7c/w8z6Bpuqmgna/Wf+bZLecfeD\nJb2TmM8EE7R71vmSvi3pvWZPU78J2j3rJEmHu/sgSZ9Jur25Q9VignbPeY+7D0r8HPiHpJ81e6ra\nTVAtf5+Y2QGSTpa0orkD1WOCav9b6veVP2PdfWIzZ6rNBNXIaWYnSDpb0mB3HyDp3gBy1WaCamR1\n9+8m/c56UdJLQQSrxQTt/u//W0l3JrL+LDGfCSZo96yPSrrN3Qcq/jfAj5s7VFOhqKZmpKTF7r7U\n3csk/a/iPwQyjru/J2lT0Dka4u6r3X1m4nGR4n/47x9sqtp5XHFiNisxZeTF3WbWU9Lpiv+QQhMw\ns06SjpP0mCS5e5m7bwk2VUq+LmmJuy8POkgdIpLamVlEUntJXwWcpy6HSprq7jvcvULSu4oXq4xQ\nx8/8syU9kXj8hKRvNmuoOtSW1d0XuvunAUWqUx1Z3078NyBJH0nq2ezBaqgj57ak2Q7KkN9X9fx9\n8ntJtypDckot6m+p2nL+QNJd7l6aWGddswerRX3vqZmZpO9Ieq5ZQ9WhjqwuqfLMZCdlyO+sOrL2\n066Df5MkndOsoZoQRTU1+0v6Mml+pTK0VLVEZtZL0lBJU4NNUrfEcNrZktZJmuTumZr1fxT/hR8L\nOkgKXNLbZjbDzMYFHaYevSWtl/SXxJDqR82sQ9ChUnC+MuSXfk3uvkrxo/wrJK2WtNXd3w42VZ3m\nSzrWzLqYWXtJYyUdEHCmhhS6++rE4zWSCoMM00pdIemNoEPUxcx+ZWZfSrpImXNGdTdmdrakVe4+\nJ+gsKbouMaz68UwZUl+Lfor/zJpqZu+a2RFBB0rBsZLWuvvnQQepx02S7kn8f3WvMmNERV0WaNcJ\ntfOU+b+z6kRRRaDMLE/x4R431TgKnFHcPZoY7tFT0sjEcMCMYmZnSFrn7jOCzpKi0e4+TPEh9dea\n2XFBB6pDRNIwSQ+5+1BJ25U5QylrZWbZks6S9HzQWWqT+APvbMUPAuwnqYOZXRxsqtq5+0JJd0t6\nW9KbkmZLigYaqhE8fmv/jDlT1RqY2R2KX77yTNBZ6uLud7j7AYpnvC7oPLVJHPj5iTK4SNfwkKQ+\nil8CtFrS74KNU6eIpM6KX1b1Y0l/S5yxzGQXKEMPrCb5gaSbE/9f3azEKKsMdYWkH5rZDEn5ksoC\nzrPHKKqpWaXqRyN6JpZhL5hZluIl9Rl3z5TrEuqVGPL5L2XmdcCjJJ1lZl8oPjz9RDN7OthIdUuc\nVasclvSy4kPsM9FKSSuTzqK/oHhxzWSnSZrp7muDDlKHb0ha5u7r3b1c8euSjgk4U53c/TF3H+7u\nx0narPj1iZlsrZn1kKTE14wY+tcamNllks6QdJG3jM/3e0aZO+yvj+IHq+Ykfm/1lDTTzLoHmqoO\n7r42cdA6JunPyuzfWS8lLluapvgIq4y4SVVtEpd/fFvSX4PO0oBLtesa2ueVuf/+cvdF7n6yuw9X\n/ADAkqAz7SmKamo+lnSwmfVOnKk4X9KrAWdq0RJH9x6TtNDd7ws6T33MrFvl3R3NrJ2kkyQtCjbV\n7tz9dnfv6e69FP9v9J/unpFnqcysg5nlVz5W/EYaGXmnandfI+lLMzsksejrkj4JMFIqMv3o9ApJ\nR5lZ+8TPgq8rQ29QJUlmVpD4eqDif1A9G2yiBr2q+B9VSnz9e4BZWg0zO1XxSyvOcvcdQeepi5kd\nnDR7tjLw95Ukufs8dy9w916J31srJQ1L/MzNOJUHfxK+pQz9nSXpFUknSJKZ9ZOULWlDoInq9w1J\ni9x9ZdBBGvCVpOMTj0+UlLHDlJN+Z4Uk/VTSw8Em2nORoAO0BO5eYWbXSXpL8btTPu7uCwKOVSsz\ne07SGEldzWylpJ+7eyYOTxgl6RJJ85JuR/6TDLmLXk09JD2RuPtzSNLf3D2jP/qlBSiU9HJiNFJE\n0rPu/mawkep1vaRnEgeqlkq6POA8dUoU/5MkXR10lrq4+1Qze0HSTMWHUM6S9Eiwqer1opl1kVQu\n6dpMuplWbT/zJd2l+HC/KyUtV/wmJYGrI+smSQ9I6ibpdTOb7e6nBJcyro6st0vKkTQp8bPrI3e/\nJrCQqjPn2MSBtZji//6BZqzUgv4+qet9HWNmQxQfSv+FMuBnbB05H5f0eOLjSsokXZoJZ//r+ffP\nuPsp1PG+fl/S/YkzwCWSMuLeGnVkzTOzaxOrvCTpLwHF22uWAf/tAgAAAABQhaG/AAAAAICMQlEF\nAAAAAGQUiioAAAAAIKNQVAEAAAAAGYWiCgAAAADIKBRVAADqYWbFia+9zOzCJt73T2rMf9CU+wcA\noKWiqAIAkJpekhpVVBOfuVefakXV3Y9pZCYAAFoliioAAKm5S9KxZjbbzG42s7CZ3WNmH5vZXDO7\nWpLMbIyZ/dvMXpX0SWLZK2Y2w8wWmNm4xLK7JLVL7O+ZxLLKs7eW2Pd8M5tnZt9N2vdkM3vBzBaZ\n2TNmZpX7M7NPElnubfZ3BwCAJtTQkV4AABB3m6T/cPczJClROLe6+xFmliNpipm9nVh3mKTD3X1Z\nYv4Kd99kZu0kfWxmL7r7bWZ2nbsPqeW1vi1piKTBkromtnkv8dxQSQMkfSVpiqRRZrZQ0rck9Xd3\nN7N9mvy7BwCgGXFGFQCAPXOypO+Z2WxJUyV1kXRw4rlpSSVVkm4wszmSPpJ0QNJ6dRkt6Tl3j7r7\nWknvSjoiad8r3T0mabbiQ5K3SiqR9JiZfVvSjr3+7gAACBBFFQCAPWOSrnf3IYmpt7tXnlHdXrWS\n2RhJ35B0tLsPljRLUu5evG5p0uOopIi7V0gaKekFSWdIenMv9g8AQOAoqgAApKZIUn7S/FuSfmBm\nWZJkZv3MrEMt23WStNndd5hZf0lHJT1XXrl9Df+W9N3EdbDdJB0naVpdwcwsT1Ind58o6WbFhwwD\nANBicY0qAACpmSspmhjCO0HS/YoPu52ZuKHReknfrGW7NyVdk7iO9FPFh/9WekTSXDOb6e4XJS1/\nWdLRkuZIckm3uvuaRNGtTb6kv5tZruJnem/Zs28RAIDMYO4edAYAAAAAAKow9BcAAAAAkFEoqgAA\nAACAjEJRBQAAAABkFIoqAAAAACCjUFQBAAAAABmFogoAAAAAyCgUVQAAAABARqGoAgAAAAAyyv8H\nHhBVPeX80zgAAAAASUVORK5CYII=\n", 205 | "text/plain": [ 206 | "" 207 | ] 208 | }, 209 | "metadata": {}, 210 | "output_type": "display_data" 211 | } 212 | ], 213 | "source": [ 214 | "x = [x for x, y in training_process]\n", 215 | "y = [y for x, y in training_process]\n", 216 | "plt.figure(figsize=((16,4)))\n", 217 | "plt.plot(x, y)\n", 218 | "plt.xticks(x, x)\n", 219 | "plt.xlabel(\"Iterations\")\n", 220 | "plt.ylabel(\"Mean Square Error\")\n", 221 | "plt.grid(axis=\"y\")" 222 | ] 223 | } 224 | ], 225 | "metadata": { 226 | "kernelspec": { 227 | "display_name": "Python 3", 228 | "language": "python", 229 | "name": "python3" 230 | }, 231 | "language_info": { 232 | "codemirror_mode": { 233 | "name": "ipython", 234 | "version": 3 235 | }, 236 | "file_extension": ".py", 237 | "mimetype": "text/x-python", 238 | "name": "python", 239 | "nbconvert_exporter": "python", 240 | "pygments_lexer": "ipython3", 241 | "version": "3.4.3" 242 | } 243 | }, 244 | "nbformat": 4, 245 | "nbformat_minor": 2 246 | } 247 | --------------------------------------------------------------------------------