├── .gitignore
├── Gradient_Checking.ipynb
├── LICENSE
├── Learning_Curve_and_Other_Visualizations.ipynb
├── Linear_Regression.ipynb
├── Logistic_Regression.ipynb
├── README.md
└── Softmax_Regression.ipynb
/.gitignore:
--------------------------------------------------------------------------------
1 | .*
2 | stanford_dl_ex
3 | data
4 | *.dat
5 |
--------------------------------------------------------------------------------
/Gradient_Checking.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {
7 | "collapsed": false
8 | },
9 | "outputs": [],
10 | "source": [
11 | "%matplotlib inline"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 2,
17 | "metadata": {
18 | "collapsed": false
19 | },
20 | "outputs": [],
21 | "source": [
22 | "import matplotlib.pyplot as plt\n",
23 | "import numpy as np\n",
24 | "import scipy.optimize"
25 | ]
26 | },
27 | {
28 | "cell_type": "markdown",
29 | "metadata": {},
30 | "source": [
31 | "## Preparation (Based on Linear Regression)"
32 | ]
33 | },
34 | {
35 | "cell_type": "markdown",
36 | "metadata": {},
37 | "source": [
38 | "Prepare train and test data."
39 | ]
40 | },
41 | {
42 | "cell_type": "code",
43 | "execution_count": 3,
44 | "metadata": {
45 | "collapsed": false
46 | },
47 | "outputs": [],
48 | "source": [
49 | "data_original = np.loadtxt('stanford_dl_ex/ex1/housing.data')\n",
50 | "data = np.insert(data_original, 0, 1, axis=1)\n",
51 | "np.random.shuffle(data)\n",
52 | "train_X = data[:400, :-1]\n",
53 | "train_y = data[:400, -1]\n",
54 | "\n",
55 | "m, n = train_X.shape\n",
56 | "theta = np.random.rand(n)"
57 | ]
58 | },
59 | {
60 | "cell_type": "markdown",
61 | "metadata": {},
62 | "source": [
63 | "Define some necessary functions."
64 | ]
65 | },
66 | {
67 | "cell_type": "code",
68 | "execution_count": 4,
69 | "metadata": {
70 | "collapsed": true
71 | },
72 | "outputs": [],
73 | "source": [
74 | "def cost_function(theta, X, y): \n",
75 | " squared_errors = (X.dot(theta) - y) ** 2\n",
76 | " J = 0.5 * squared_errors.sum()\n",
77 | " return J\n",
78 | "\n",
79 | "def gradient(theta, X, y):\n",
80 | " errors = X.dot(theta) - y\n",
81 | " return errors.dot(X)"
82 | ]
83 | },
84 | {
85 | "cell_type": "markdown",
86 | "metadata": {},
87 | "source": [
88 | "## Gradient Checking"
89 | ]
90 | },
91 | {
92 | "cell_type": "markdown",
93 | "metadata": {},
94 | "source": [
95 | "Define \"step size\" (don't set it too low to avoid numerical precision issues)."
96 | ]
97 | },
98 | {
99 | "cell_type": "code",
100 | "execution_count": 5,
101 | "metadata": {
102 | "collapsed": false
103 | },
104 | "outputs": [],
105 | "source": [
106 | "epsilon = 1e-4"
107 | ]
108 | },
109 | {
110 | "cell_type": "markdown",
111 | "metadata": {},
112 | "source": [
113 | "Prepare theta step values (making use of numpy broadcasting)."
114 | ]
115 | },
116 | {
117 | "cell_type": "code",
118 | "execution_count": 6,
119 | "metadata": {
120 | "collapsed": false
121 | },
122 | "outputs": [],
123 | "source": [
124 | "mask = np.identity(theta.size)\n",
125 | "theta_plus = theta + epsilon * mask\n",
126 | "theta_minus = theta - epsilon * mask"
127 | ]
128 | },
129 | {
130 | "cell_type": "markdown",
131 | "metadata": {},
132 | "source": [
133 | "Compute diffs between theta's gradient as mathematically defined and the gradient as defined by our function above."
134 | ]
135 | },
136 | {
137 | "cell_type": "code",
138 | "execution_count": 8,
139 | "metadata": {
140 | "collapsed": false
141 | },
142 | "outputs": [],
143 | "source": [
144 | "diffs = np.empty_like(theta)\n",
145 | "for i in range(theta_plus.shape[0]):\n",
146 | " gradient_def = (\n",
147 | " (cost_function(theta_plus[i], train_X, train_y) - cost_function(theta_minus[i], train_X, train_y)) /\n",
148 | " (2 * epsilon)\n",
149 | " )\n",
150 | " gradient_lin_reg = gradient(theta, train_X, train_y)[i]\n",
151 | " diffs[i] = np.absolute(gradient_def - gradient_lin_reg)"
152 | ]
153 | },
154 | {
155 | "cell_type": "code",
156 | "execution_count": 9,
157 | "metadata": {
158 | "collapsed": false
159 | },
160 | "outputs": [
161 | {
162 | "data": {
163 | "text/plain": [
164 | "array([ 3.31414049e-06, 1.44055812e-05, 9.00169834e-06,\n",
165 | " 5.53415157e-06, 6.84440965e-06, 1.88233535e-05,\n",
166 | " 1.28877582e-05, 1.83098018e-06, 4.86033969e-06,\n",
167 | " 2.71014869e-06, 3.72529030e-07, 1.79391354e-05,\n",
168 | " 1.09821558e-05, 1.00492034e-05])"
169 | ]
170 | },
171 | "execution_count": 9,
172 | "metadata": {},
173 | "output_type": "execute_result"
174 | }
175 | ],
176 | "source": [
177 | "diffs"
178 | ]
179 | },
180 | {
181 | "cell_type": "markdown",
182 | "metadata": {},
183 | "source": [
184 | "**Lookin' good!** The smaller the values, the better.
\n",
185 | "(Any value significantly larger than 1e-4 indicates a problem.)"
186 | ]
187 | },
188 | {
189 | "cell_type": "code",
190 | "execution_count": 10,
191 | "metadata": {
192 | "collapsed": false
193 | },
194 | "outputs": [],
195 | "source": [
196 | "assert all(np.less(diffs, 1e-4))"
197 | ]
198 | },
199 | {
200 | "cell_type": "markdown",
201 | "metadata": {},
202 | "source": [
203 | "Quality check: passed with distinction."
204 | ]
205 | }
206 | ],
207 | "metadata": {
208 | "kernelspec": {
209 | "display_name": "Python 3",
210 | "language": "python",
211 | "name": "python3"
212 | },
213 | "language_info": {
214 | "codemirror_mode": {
215 | "name": "ipython",
216 | "version": 3
217 | },
218 | "file_extension": ".py",
219 | "mimetype": "text/x-python",
220 | "name": "python",
221 | "nbconvert_exporter": "python",
222 | "pygments_lexer": "ipython3",
223 | "version": "3.4.2"
224 | }
225 | },
226 | "nbformat": 4,
227 | "nbformat_minor": 0
228 | }
229 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2013 Andrew Maas
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
6 | this software and associated documentation files (the "Software"), to deal in
7 | the Software without restriction, including without limitation the rights to
8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/Linear_Regression.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {
7 | "collapsed": false
8 | },
9 | "outputs": [],
10 | "source": [
11 | "%matplotlib inline"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 2,
17 | "metadata": {
18 | "collapsed": false
19 | },
20 | "outputs": [],
21 | "source": [
22 | "import matplotlib.pyplot as plt\n",
23 | "import numpy as np\n",
24 | "import scipy.optimize\n",
25 | "import time"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "metadata": {},
31 | "source": [
32 | "Load and preprocess the data."
33 | ]
34 | },
35 | {
36 | "cell_type": "code",
37 | "execution_count": 3,
38 | "metadata": {
39 | "collapsed": false
40 | },
41 | "outputs": [],
42 | "source": [
43 | "data_original = np.loadtxt('stanford_dl_ex/ex1/housing.data')"
44 | ]
45 | },
46 | {
47 | "cell_type": "code",
48 | "execution_count": 4,
49 | "metadata": {
50 | "collapsed": false
51 | },
52 | "outputs": [],
53 | "source": [
54 | "data = np.insert(data_original, 0, 1, axis=1)"
55 | ]
56 | },
57 | {
58 | "cell_type": "code",
59 | "execution_count": 5,
60 | "metadata": {
61 | "collapsed": false
62 | },
63 | "outputs": [],
64 | "source": [
65 | "np.random.shuffle(data)"
66 | ]
67 | },
68 | {
69 | "cell_type": "markdown",
70 | "metadata": {},
71 | "source": [
72 | "Create train & test sets."
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": 6,
78 | "metadata": {
79 | "collapsed": false
80 | },
81 | "outputs": [],
82 | "source": [
83 | "train_X = data[:400, :-1]\n",
84 | "train_y = data[:400, -1]\n",
85 | "\n",
86 | "test_X = data[400:, :-1]\n",
87 | "test_y = data[400:, -1]"
88 | ]
89 | },
90 | {
91 | "cell_type": "code",
92 | "execution_count": 7,
93 | "metadata": {
94 | "collapsed": false
95 | },
96 | "outputs": [],
97 | "source": [
98 | "m, n = train_X.shape"
99 | ]
100 | },
101 | {
102 | "cell_type": "markdown",
103 | "metadata": {},
104 | "source": [
105 | "Define the cost function and how to compute the gradient.
\n",
106 | "Both are needed for the subsequent optimization procedure."
107 | ]
108 | },
109 | {
110 | "cell_type": "code",
111 | "execution_count": 8,
112 | "metadata": {
113 | "collapsed": false
114 | },
115 | "outputs": [],
116 | "source": [
117 | "def cost_function(theta, X, y): \n",
118 | " squared_errors = (X.dot(theta) - y) ** 2\n",
119 | " J = 0.5 * squared_errors.sum()\n",
120 | " return J"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": 9,
126 | "metadata": {
127 | "collapsed": false
128 | },
129 | "outputs": [],
130 | "source": [
131 | "def gradient(theta, X, y):\n",
132 | " errors = X.dot(theta) - y\n",
133 | " return errors.dot(X)"
134 | ]
135 | },
136 | {
137 | "cell_type": "markdown",
138 | "metadata": {},
139 | "source": [
140 | "Run a timed optimization and store the iteration values of the cost function (for latter investigation)."
141 | ]
142 | },
143 | {
144 | "cell_type": "code",
145 | "execution_count": 10,
146 | "metadata": {
147 | "collapsed": false
148 | },
149 | "outputs": [
150 | {
151 | "name": "stdout",
152 | "output_type": "stream",
153 | "text": [
154 | "Optimization terminated successfully.\n",
155 | " Current function value: 3909.307674\n",
156 | " Iterations: 25\n",
157 | " Function evaluations: 34\n",
158 | " Gradient evaluations: 34\n",
159 | "Optimization took 0.004139900207519531 seconds\n"
160 | ]
161 | }
162 | ],
163 | "source": [
164 | "J_history = []\n",
165 | "\n",
166 | "t0 = time.time()\n",
167 | "res = scipy.optimize.minimize(\n",
168 | " fun=cost_function,\n",
169 | " x0=np.random.rand(n),\n",
170 | " args=(train_X, train_y),\n",
171 | " method='bfgs',\n",
172 | " jac=gradient,\n",
173 | " options={'maxiter': 200, 'disp': True},\n",
174 | " callback=lambda x: J_history.append(cost_function(x, train_X, train_y)),\n",
175 | ")\n",
176 | "t1 = time.time()\n",
177 | "\n",
178 | "print('Optimization took {s} seconds'.format(s=t1 - t0))\n",
179 | "optimal_theta = res.x"
180 | ]
181 | },
182 | {
183 | "cell_type": "markdown",
184 | "metadata": {},
185 | "source": [
186 | "It's always interesting to take a more detailed look at the optimization results."
187 | ]
188 | },
189 | {
190 | "cell_type": "code",
191 | "execution_count": 11,
192 | "metadata": {
193 | "collapsed": false
194 | },
195 | "outputs": [
196 | {
197 | "data": {
198 | "text/plain": [
199 | ""
200 | ]
201 | },
202 | "execution_count": 11,
203 | "metadata": {},
204 | "output_type": "execute_result"
205 | },
206 | {
207 | "data": {
208 | "image/png": [
209 | "iVBORw0KGgoAAAANSUhEUgAAAaAAAAEPCAYAAAAEfBBiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
210 | "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+8VXWd7/HXW0FkBEHM/AnKCGZIJTn5IysxRwSm1O7k\n",
211 | "j6aUaZSphwbeZu6MP2auUHkrpzET70h3yCmkxh+jkzoTIpii5ahkiUqkoEnCUSBRFBopkM/9Y313\n",
212 | "Z5/DPufsfc7ee+2z9/v5eKzHWfu71nftz1ls9ues73et71cRgZmZWb3tlncAZmbWmpyAzMwsF05A\n",
213 | "ZmaWCycgMzPLhROQmZnlwgnIzMxyUbMEJGmkpAck/VzSCkkzU/lsSeskPZGWKUV1Lpe0WtIzkiYV\n",
214 | "lR8j6em07bqi8kGSbk3lj0o6tGjbNEmr0nJ+UfloSY+lOrdIGlirc2BmZl2r5RXQduDzEXEUcDxw\n",
215 | "saR3AgF8PSImpOUeAEnjgHOAccBk4AZJSseaC1wQEWOBsZImp/ILgE2p/Frg6nSsEcCVwLFpmSVp\n",
216 | "WKpzNXBNqvNaOoaZmdVZzRJQRKyPiOVpfSvwC+DgtFklqpwB3BwR2yNiDfAccJykA4GhEbEs7XcT\n",
217 | "cGZaPx2Yn9bvAE5J66cBiyNic0RsBpYAU1JCOxm4Pe03v+hYZmZWR3XpA5J0GDABeDQVzZD0pKQb\n",
218 | "JQ1PZQcB64qqrSNLWJ3L22hPZAcDawEiYgfwuqR9uznWCGBzROwscSwzM6ujmicgSUPIrjguSVdC\n",
219 | "c4HRwNHAy8A1tY4h8ZhDZmYNZEAtD546+O8AvhsRdwJExMai7d8C/iO9bANGFlU/hOzKpS2tdy4v\n",
220 | "1BkFvCRpADAsIjZJagMmFtUZCdwPvAoMl7Rbugo6JB2jc9xOVmZmvRARpbpYuty5JgtZP89NwLWd\n",
221 | "yg8sWv888K9pfRywHNiD7ArpeUBp22PAcemYC4HJqfwiYG5aPxe4Ja2PAH4JDAf2KaynbbcB56T1\n",
222 | "bwKfLRF7QKTlrAdrdY76wwLMzjuGRll8LnwufC56PBdRyf61vAI6EfgU8JSkJ1LZFcAnJB2dfcnz\n",
223 | "AvCZFPVKSbcBK4EdwEWRfiOyRPMdYDCwMCIWpfIbgQWSVgObyJIQEfGqpC8BP0n7fSGymxEALgVu\n",
224 | "kXQV8LN0jG6MerfEERGs6tVZMDOzkmqWgCLix5TuY7qnmzpfBr5covynwLtKlP8WOLuLY30b+HaJ\n",
225 | "8hfIrqbKcOHz8N57gYclrgKuj2BnT7XMzKxnHgmhS1MXwaMzIy66GHg/WaL7ocTonAOrt6V5B9BA\n",
226 | "luYdQANZmncADWRp3gH0V2pv5bICSRGdOtIkdifrs7qUrCnxWxG+s87MrKDUd2e3+zsB7aq7kyhx\n",
227 | "FNkDrL8GLozY9S46M7NW5ARUBT2dRImBwOXA54C/gvGvwsiZMHRP2LIN1s6JWLGwbgGbmTUAJ6Aq\n",
228 | "KPckSrwX7v13WLIv/OOQ9i3Tn4NHLnESMrNW4gRUBZWcROlPFsMPTt11y9kPw22TIvjv0vXGT/VV\n",
229 | "k5k1k0oTUE1HQmgNe+1RunzMBGCTxEbg2Y7LJw+GE/4O5o1p33/64dJ4ekpCTlxm1iycgPpsy7bS\n",
230 | "5csfAj4KHAq8Iy3jgT+FkSfAVwd13H/eGPjMN9Jt3q+RDRv0Wsdl/CQ44bpKE1dvklZvE50TpJmV\n",
231 | "ywmoz9bOgemHd0wKFz4PL14fwQ6yIYWeJxtCCADpl0uBk3Y91qBBwFFkQwntU7SMAIbB6QFf7vRv\n",
232 | "Nm8MfHaOxDvIRoPotLz7BDjhG5UkrSyJ9DbRNXaCNLPG4QTURxErFkrjgakzYMhg2Ppmlny6+zLs\n",
233 | "6qrpuZURXFRqi8Ru8MKDwAd23Tpwd+Aw4Bhg347LR/eG/9OpTXbeGPjr70o8CbxFNvTRjvb1D78f\n",
234 | "5hywa52Lb5D4N7LJBnekn0Xrx/5lx+RTqPeJKyVeAd7stGyDd59crwTZXteJy6wROAFVQfoCq+BL\n",
235 | "rOurpq7fg53S5t+U3vr8MxFcUmqLtPpB4EO7bnntV8AXyT4Du6efafntkcABu9ZhB9nzTwOAgcAg\n",
236 | "YEj762HDS9QBDj0SuAHYk2w8v6Ll9EFwVaf9542Bz39HYgmwOS2vt6+/529LJ7qpM+jm36EvicvM\n",
237 | "qs8JKAe9u2qC3iQu2PJm6fL16yN4oNQW6cULyUYn7+SF1RH8Q1fvJD1zBNlkgJ089UgEU0rXWbWU\n",
238 | "ks2RWzcAi4BhZKOavx0Ym60f8oelIzjmwxIryRLVa7Qnr7T+vj/vTeIys9pwAspJ5VdNvU1cvUla\n",
239 | "vanT23pdNUe2rYtgQakt0lOLyKZd72TFf5E9HLwPWdIaXrR+IIzYt/R7DRncdXxmVitOQP1MpYmr\n",
240 | "N0mrt1do+SfI1V+L4Odd1ZJWHkXJpsWdHpTXLAd+ELWESh+msr7J+mZGVZTsel+ncx/QJa/AGXvA\n",
241 | "h28Aroqgi342M+uJR0KoAieg5lUqccGKJ4Cvkd2s8VfAHR7p3KxyTkBV4ATUmiROAv4JeAmYEcGz\n",
242 | "OYdk1q84AVWBE1DrSiOdfw74O2AecBWMP8nPDpn1zGPBmfVBBNuBayVuBb4G970AJ78F1xfdvOBn\n",
243 | "h8yqwVdAJfgKyAqkTyyDm9+365apiyIWlny2yaxVVfrd6dtPzbr1VsnpNPzskFnfOQGZdaurB2W3\n",
244 | "djHChJmVywnIrFtr52Qz3BYrZ1QIM+uJ+4BKcB+QFWt/duioP4JfvwSPX+4bEMx25duwq8AJyEqR\n",
245 | "+Htg7wj+Nu9YzBqRb0Iwq53lwNF5B2HWLJyAzMq3HDhawlfHZlXgBGRWvjay/zOlJuszswo5AZmV\n",
246 | "KQ1Q6mY4sypxAjKrjBOQWZU4AZlVxgnIrEqcgMwq8yROQGZV4eeASvBzQNaVNF3DG8B+EWzNOx6z\n",
247 | "RuLngMxqKE3XsBJ4V96xmPV3TkBmlVsOvCfvIMz6Oycgs8r5RgSzKqhZApI0UtIDkn4uaYWkmal8\n",
248 | "hKQlklZJWixpeFGdyyWtlvSMpElF5cdIejptu66ofJCkW1P5o5IOLdo2Lb3HKknnF5WPlvRYqnOL\n",
249 | "pIG1OgfWtJyAzKqglldA24HPR8RRwPHAxZLeCVwGLImII4AfptdIGgecA4wDJgM3SCp0Zs0FLoiI\n",
250 | "scBYSZNT+QXAplR+LXB1OtYI4Erg2LTMkjQs1bkauCbVeS0dw6wSTwHvktg970DM+rOaJaCIWB8R\n",
251 | "y9P6VuAXwMHA6cD8tNt84My0fgZwc0Rsj4g1wHPAcZIOBIZGxLK0301FdYqPdQdwSlo/DVgcEZsj\n",
252 | "YjOwBJiSEtrJwO0l3t+sLBG8DqwHxuYdi1l/Vpc+IEmHAROAx4D9I2JD2rQB2D+tHwSsK6q2jixh\n",
253 | "dS5vS+Wkn2sBImIH8Lqkfbs51ghgc0TsLHEss0q4Gc6sjwbU+g0kDSG7OrkkIra0t6pBRISkej2I\n",
254 | "VNH7SJpd9HJpRCytajTW3xUeSL0l70DM8iJpIjCxt/VrmoBSB/8dwIKIuDMVb5B0QESsT81rG1N5\n",
255 | "GzCyqPohZFcubWm9c3mhzijgJUkDgGERsUlSGx1PykjgfuBVYLik3dJV0CHpGLuIiNm9+JWtdSwH\n",
256 | "Lso7CLM8pT/MlxZeS5pVSf1a3gUn4EZgZUR8o2jT3cC0tD4NuLOo/FxJe0gaTda+viwi1gNvSDou\n",
257 | "HfM84K4Sx/o42U0NAIuBSZKGS9oHOBW4N7JhHx4Azirx/maV8LNAZn1Us6F4JH0AeIjsjqHCm1wO\n",
258 | "LANuI7tyWQOcnW4UQNIVwF8AO8ia7O5N5ccA3wEGAwsjonBL9yBgAVn/0ibg3HQDA5I+DVyR3veq\n",
259 | "iJifykeTNZuMAH4GfCoitneK3UPxWLfSpHSbgHERrM87HrNGUOl3p8eCK8EJyMohcT9wdQT35h2L\n",
260 | "WSPwWHBm9eM74cz6wAnIrPecgMz6wAnIrPecgMz6wH1AJbgPyMohsQewmWxuoN/kHY9Z3twHZFYn\n",
261 | "EfwOeBYYn3csZv2RE5BZ3/h5ILNecgIy6xv3A5n1khOQWd84AZn1km9CKME3IVi5JIaTjU04LIK3\n",
262 | "8o7HLE++CcGsjiLYDPwaODzvWMz6Gycgs75zM5xZLzgBmfWdE5BZLzgBmfVdYXI6M6uAE5BZ3/lZ\n",
263 | "ILNecAIy67tfAXtJvD3vQMz6Eycgsz6KIPBVkFnFnIDMqsM3IphVyAnIrDqcgMwq5ARkVh1OQGYV\n",
264 | "8lA8JXgoHqtUmhvodWBEBG/mHY9ZHjwUj1kOPDeQWeWcgMyq50l8J5xZ2ZyAzKrH/UBmFXACMqse\n",
265 | "JyCzCvgmhBJ8E4L1hsQIYA0wPIKdOYdjVne+CcEsJxG8CrwG/GHesZj1B05AZtXlZjizMjkBmVWX\n",
266 | "E5BZmZyAzKrLCcisTE5AZtXlUbHNyuQEZFZda4C9Jd6WdyBmjc4JyKyK0txAHhHBrAxOQGbV534g\n",
267 | "szI4AZlVnxOQWRmcgMyqzwnIrAw1TUCS/kXSBklPF5XNlrRO0hNpmVK07XJJqyU9I2lSUfkxkp5O\n",
268 | "264rKh8k6dZU/qikQ4u2TZO0Ki3nF5WPlvRYqnOLpIG1PAfWklYCYyT2zDsQs0ZW6yugbwOTO5UF\n",
269 | "8PWImJCWewAkjQPOAcalOjdIKowpNBe4ICLGAmMlFY55AbAplV8LXJ2ONQK4Ejg2LbMkDUt1rgau\n",
270 | "SXVeS8cwq5oItgGrgaPyjsWskdU0AUXEj8i+5DsrNVjdGcDNEbE9ItYAzwHHSToQGBoRy9J+NwFn\n",
271 | "pvXTgflp/Q7glLR+GrA4IjZHxGZgCTAlJbSTgdvTfvOLjmVWTX4eyKwHefUBzZD0pKQbJQ1PZQcB\n",
272 | "64r2WQccXKK8LZWTfq4FiIgdwOuS9u3mWCOAzRGxs8SxzKrJ/UBmPcgjAc0FRpP953wZuKZO7+t5\n",
273 | "J6yensQJyKxbA+r9hhGxsbAu6VvAf6SXbcDIol0PIbtyaUvrncsLdUYBL0kaAAyLiE2S2oCJRXVG\n",
274 | "AvcDrwLDJe2WroIOScfYhaTZRS+XRsTS8n9Ls+xhVIndPDeQNStJE+n4XVuRuicgSQdGxMvp5ceA\n",
275 | "wh1ydwP/KunrZM1iY4FlERGS3pB0HLAMOA+YU1RnGvAo8HHgh6l8MfDl1Lwn4FTg0nSsB4CzgFtT\n",
276 | "3TtLxRkRs6v0K1sLiuAViTeAw4Bf5hyOWU2kP8yXFl5LmlVJ/ZomIEk3AycBb5O0FpgFTJR0NFmT\n",
277 | "2AvAZwAiYqWk28huYd0BXBTt07VeBHwHGAwsjIhFqfxGYIGk1cAm4Nx0rFclfQn4SdrvC+lmBIBL\n",
278 | "gVskXQX8LB3DrBYK/UBOQGYleEruEjwlt1WDxJeAiODKvGMxq4dKvzvr3gRn1jqu3g1eni699CHY\n",
279 | "sg3WzolYsTDvqMwaRVkJSNJeZB35AayLiN/UNCqzfk4aPxU++EmYux9ZMzQw/XBpPE5CZpkum+Ak\n",
280 | "DQWmk/WrvA3YQNahvz9Zf8v3gHkRsbU+odaPm+Csr6Qpi+Ce03bdMnVRxMIpu5ab9X/VbIK7E7gF\n",
281 | "+GhEbOj0JgeQjUJwF+2jD5jZ7w3tYhy4IYPrG4dZ4+oyAUVEl4klItYD/5wWM9vFlm2ly7e+Wd84\n",
282 | "zBpXuX1AI8ieyxlUKIuIh2oVlFn/t3YOTD8c5o1pL/vLF+DF6/OLyayx9HgbtqTpwEyyUQOWA8cD\n",
283 | "j0TEh2sfXj7cB2TVkN2IMGpG1uy232iYuCrirFPzjsusVir97iwnAa0A3keWdI6WdCTwlYj4WN9C\n",
284 | "bVxOQFZtEkOAp4AZEfwg73jMaqHS785yBiPdFhFvpoPvGRHPAO/obYBmrSiCrWRzT/0/iX3yjses\n",
285 | "EZSTgNZJ2ofsrrglku4G1tQ0KrMmFMEDZHeOfj3vWMwaQUVD8aSRT/cGFkXE72oVVN7cBGe1kpri\n",
286 | "ngYujsAPpFpTqXoTnKQFhfWIWBoRd+MBPM16pVNT3PCe9jdrZuU0wY0vfpHm3TmmNuGYNb8I7ieb\n",
287 | "B8tNcdbSukxAkq6QtAV4l6QthQXYSDYPj5n13qXAyRIelsdaVjm3YX81Ii6rUzwNwX1AVg8Sp5DN\n",
288 | "czU+gtdzDsesz2rxHNBuwCeB0RHxRUmjgAMiYlnfQm1cTkBWLxLfBAZGcEHesZj1VS0S0DeBncCH\n",
289 | "I+LINCzP4oj4o76F2ricgKxeJIaS3RX32QgW9bS/WSOrxYOox0XERcCbkE13DQzsZXxmViSCLcCF\n",
290 | "wD9LDMs7HrN6KicB/U7S7oUXkvYjuyIysyqI4D7gHuAf847FrJ7KSUDXA98H3i7py8DDwFdqGpVZ\n",
291 | "6/kb4FSJEpPYmTWnskZCkPRO2iee+2FE/KKmUeXMfUCWB4lT4b7vwXVPweAB2ZxCa+d4Cm/rL6o5\n",
292 | "I2qxVcAbaf+QNCoiXuxNgGbWlfED4bRB8B9Fk0FOP1waj5OQNaNy7oKbAcwiewD1rUJ5RLyrtqHl\n",
293 | "x1dAlgdpyiK4p0QT3NRFEQv9wKo1vFpcAf1P4B0Rsan3YZlZz4buWbp8yOD6xmFWH+XchPAiWfOb\n",
294 | "mdXUlm2ly7e+Wd84zOqjyysgSX+dVn8JLJX0n0BhCoaICA+kaFZVa+fA9MNh3pj2sgufhxevzy8m\n",
295 | "s9rprgluKBBkV0BrgT3SYmY1ELFioTQemDoDDjwQ9v1DeHSmb0CwZlXOTQhnR8RtPZU1E9+EYHmT\n",
296 | "2A14ATg9gifzjsesHLUYiufyMsvMrEoi2Al8Fzgv71jMaqW7PqApwFTgYElzgEJWGwpsr0NsZq1u\n",
297 | "AfCAxGUR7Mg7GLNq6+4K6CXgp8C29POnwONkk9F5uBCzGovgGWAd7aOQmDWVcvqA9oiI33W7U5Nx\n",
298 | "H5A1ComZwLERfCrvWMx6UrU+IEk/kHQWJZrpJO0l6RxJvjvHrLZuAT4iMSTvQMyqrcsrIElvBz4H\n",
299 | "fJxsCJ6XyfqBDiBLSrcC/xQRv65PqPXjKyBrJBL/CdwWwU15x2LWnarPiJoOuj9waHr5q4jY0Mv4\n",
300 | "+gUnIGskEmcD0yM4Ne9YzLpTtQQkaSvZg6il/BZ4Dvj7iLiv4igbnBOQNRKJwUAb8O4I1uUdj1lX\n",
301 | "qtYHFBFDImJoqYWsGe4zwHU9BPMvkjZIerqobISkJZJWSVosaXjRtsslrZb0jKRJReXHSHo6bbuu\n",
302 | "qHyQpFtT+aOSDi3aNi29xypJ5xeVj5b0WKpziyRPL24NLYI3gX8H/izvWMyqqZwHUXcRETsi4kmy\n",
303 | "2VK7821gcqeyy4AlEXEE8MP0GknjgHOAcanODZIKmXQucEFEjAXGSioc8wJgUyq/Frg6HWsEcCVw\n",
304 | "bFpmSRqW6lwNXJPqvJaOYdboFgDnSfjK3JpGrxJQQUR8s4ftPyL7ki92OjA/rc8HzkzrZwA3R8T2\n",
305 | "iFhD1sR3nKQDgaERsSztd1NRneJj3UH78xKnAYsjYnNEbAaWAFNSQjsZuL3E+5s1sh8BewPvyTsQ\n",
306 | "s2rpUwLqpf2LbmLYAOyf1g+CDu3b64CDS5S3pXLSz7WQXZUBr0vat5tjjQA2R8TOEscya1hpaJ4F\n",
307 | "eGgeayJ5JKDfi+wOiJ5vw6vS29XpfcxqZQHwZ1JZE0maNbw8PsgbJB0QEetT89rGVN4GjCza7xCy\n",
308 | "K5e2tN65vFBnFPCSpAHAsIjYJKkNmFhUZyRwP/AqMFzSbukq6JB0jF1Iml30cmlELK30FzWrpgie\n",
309 | "lVgL/DGwKO94zCRNpON3bUXyuAK6G5iW1qcBdxaVnytpD0mjgbHAsohYD7wh6bjUh3MecFeJY32c\n",
310 | "7KYGgMXAJEnDJe0DnArcm664HgDOKvH+HUTE7KJlaZ9/a7PqcDOcNYyIWFr8XVlp/bIeRO0tSTcD\n",
311 | "JwFvI+vvuZIsedxGduWyBjg73SiApCuAvwB2AJdExL2p/BjgO8BgYGFEzEzlg8j+Q04ANgHnphsY\n",
312 | "kPRp4IoUylURMT+VjyYb3mQE8DPgUxHRYXRvPwdkjUpiP2A1MDKCLXnHY1asJiMhtBonIGtkEncD\n",
313 | "d0T8/g5Qs4ZQiwnpzKyxuBnOmoKvgErwFZA1Mok9yebr8tA81lB8BWTW5CLYRvbg9SfzjsWsL5yA\n",
314 | "zPonD81j/Z4TkFn/9GNgCHB03oGY9ZYTkFk/5KF5rBn4JoQSfBOC9QcSRwAPkj0TtCPveMx8E4JZ\n",
315 | "i4hgFfAi2dA8Zv2OE5BZ/7YAOL/HvcwakJvgSnATnPUXEm8DngcO8dA8ljc3wZm1kAheAZYCf5pz\n",
316 | "KGYVcwIy6/98N5z1S57Yyqzfe/8OOOmD0ouPwKuvw9o5ESsW5h2VWU+cgMz6MWn8VDjha/CVgcDx\n",
317 | "Wen0w6XxOAlZo3MTnFm/NnImzBvTsWzeGBg1I594zMrnBGTWrw3ds3T5kMH1jcOsck5AZv3alm2l\n",
318 | "y7e+Wd84zCrnBGTWr62dA9Of61h24fPw4vX5xGNWPj+IWoIfRLX+JLsRYdQMePt+8PZ3wMJzfAOC\n",
319 | "5aHS704noBKcgKw/ktgd2AQcEcHGvOOx1uOREMxaVARvAQ8DH8o7FrNyOAGZNZcHgZPyDsKsHE5A\n",
320 | "Zs3FCcj6DfcBleA+IOuvJAaS9QONjmBT3vFYa3EfkFkLi2A78AjwwbxjMeuJE5BZ83EznPULTkBm\n",
321 | "zccJyPoF9wGV4D4g688k9iDrBxoZwea847HW4T4gsxYXwe+AZcCJecdi1h0nILPm5GY4a3hOQGbN\n",
322 | "yQnIGp77gEpwH5D1dxJ7Aq8AB0awJe94rDW4D8jMiGAb8FPg/XnHYtYVJyCz5vUQboazBuYEZNa8\n",
323 | "3A9kDc19QCW4D8iagcRewEZgvwj+O+94rPn1mz4gSWskPSXpCUnLUtkISUskrZK0WNLwov0vl7Ra\n",
324 | "0jOSJhWVHyPp6bTtuqLyQZJuTeWPSjq0aNu09B6rJJ1fr9/ZrJ4i+A3wJHBC3rGYlZJnE1wAEyNi\n",
325 | "QkQcm8ouA5ZExBHAD9NrJI0DzgHGAZOBGyQVsuxc4IKIGAuMlTQ5lV8AbErl1wJXp2ONAK4Ejk3L\n",
326 | "rOJEZ9Zk3AxnDSvvPqDOl2qnA/PT+nzgzLR+BnBzRGyPiDXAc8Bxkg4EhkbEsrTfTUV1io91B3BK\n",
327 | "Wj8NWBwRmyNiM7CELKmZNSMnIGtYeV8B3SfpcUnTU9n+EbEhrW8A9k/rBwHriuquAw4uUd6Wykk/\n",
328 | "1wJExA7gdUn7dnMss2b0MHBMei7IrKEMyPG9T4yIlyXtByyR9EzxxogISbndISFpdtHLpRGxNKdQ\n",
329 | "zHotgi0SK4HjyK6GzKpG0kRgYm/r55aAIuLl9PPXkr5P1h+zQdIBEbE+Na9tTLu3ASOLqh9CduXS\n",
330 | "ltY7lxfqjAJekjQAGBYRmyS10fGEjQTuLxHf7L79hmYNo9AM5wRkVZX+MF9aeC1pViX1c2mCk/QH\n",
331 | "koam9b2AScDTwN3AtLTbNODOtH43cK6kPSSNBsYCyyJiPfCGpOPSTQnnAXcV1Skc6+NkNzUALAYm\n",
332 | "SRouaR/gVODeGv2qZo3gQeBDeQdh1lleV0D7A99PN7INAL4XEYslPQ7cJukCYA1wNkBErJR0G7AS\n",
333 | "2AFcFO0PMF0EfAcYDCyMiEWp/EZggaTVZHOjnJuO9aqkLwE/Sft9Id2MYNasfgzcLLFHmqrBrCH4\n",
334 | "QdQS/CCqNRuJnwGfi+C/8o7Fmle/eRDVzOrKt2Nbw3ECMmsNTkDWcNwEV4Kb4KzZSOwLvACMiGBH\n",
335 | "3vFYc3ITnJntIoJNwK+A9+Ydi1mBE5BZ63AznDUUJyCz1uEEZA3FfUAluA/ImpHE/sCzwL4RvJV3\n",
336 | "PNZ83AdkZiVFsAF4GXhP3rGYgROQWatxM5w1DCcgs9biBGQNw31AJbgPyJqVxEFkA//uF8HOvOOx\n",
337 | "5uI+IDPrUgQvAa8CR+Udi5kTkFnrcTOcNQQnILPW4wRkDcEJyKz1PAh8SML9nJYrJyCzFhPBi8B/\n",
338 | "A0fmHYu1Nicgs9bkZjjLnROQWWt6CCcgy5kTkFlrehA4yf1AlicnILPW9EtgJzAm70CsdTkBmbWg\n",
339 | "CAL3A1nOBuQdgJnl5f9ugme/IG34FGzZBmvnRKxYmHdU1jqcgMxakDR+KnzwDJh7EHBQVjr9cGk8\n",
340 | "TkJWL26CM2tJI2fC3FEdy+aNgVEz8onHWpETkFlLGrpn6fIhg+sbh7UyJyCzlrRlW+nyrW/WNw5r\n",
341 | "ZU5AZi1p7RyY/lzHspmvwIvX5xOPtSLfhGDWgiJWLJTGA1NnZM1uOwQXj4c5A/OOzVqHZ0QtwTOi\n",
342 | "WiuS+CPgHuD0CB7JOx7rfzwjqpn1SgSPA9OA70sckXc81vycgMzs9yJYCPwdcI/E/nnHY83NCcjM\n",
343 | "OojgRmAB8AOJIXnHY83LfUAluA/IWl0aJftbwAHAGRHsyDkk6wfcB2RmfZYGK/0s2XfEXE/bYLXg\n",
344 | "BGRmJUWwHTgLeC/wv3MOx5pQSyYgSZMlPSNptaRL847HrFFFsBX4E+DPJT6ddzzWXFquD0jS7sCz\n",
345 | "wB8DbcBPgE9ExC+K9nEfUCJpYkQszTuORtDK50LiHcCD8MV/gkdOhDcOgL3XlzuFQzb69siZ2Rh0\n",
346 | "5U39UK86fX8vn4v2uhV+d0ZESy3ACcCioteXAZd12ifyjrNRFmB23jE0ytLq5wL+9q/hsh0QAbMi\n",
347 | "+3nhajhqavf1jpqa7RfRvnRfr151qvNerX0uOtYnKvpM5f2hrvcCfByYV/T6U8D1fTmJzby0+peu\n",
348 | "z0Xx7z95UfuX0qyiL6iPLIEY2vXykfs6fqGVU69edarxXq1wLqbcU+b/kajkM9WKY8G1VpujWdV0\n",
349 | "NYXDhJOBl7quN6GLKR66q1evOs36XtWOrzbTdLRiH9DxZH/JTk6vLwd2RsTVRfu01kkxM6uSqKAP\n",
350 | "qBUT0ACymxBOIftLYBmdbkIwM7Paa7kmuIjYIelzwL3A7sCNTj5mZvXXcldAZmbWGFryQdTu+CHV\n",
351 | "dpLWSHpK0hOSluUdTz1J+hdJGyQ9XVQ2QtISSaskLZY0PM8Y66WLczFb0rr02XhC0uQ8Y6wXSSMl\n",
352 | "PSDp55JWSJqZylvus9HNuSj7s+EroCLlPKTaSiS9ABwTEa/mHUu9SfogsBW4KSLelcr+AXglIv4h\n",
353 | "/XGyT0Rclmec9dDFuZgFbImIr+caXJ1JOgA4ICKWSxoC/BQ4E/g0LfbZ6OZcnE2Znw1fAXV0LPBc\n",
354 | "RKyJiO3icC04AAAEPElEQVTALcAZOceUt5YcESIifgS81qn4dGB+Wp9P9p+t6XVxLqAFPxsRsT4i\n",
355 | "lqf1rcAvgINpwc9GN+cCyvxsOAF1dDCwtuj1OtpPaCsK4D5Jj0uanncwDWD/iNiQ1jdAy0/YNkPS\n",
356 | "k5JubIUmp84kHQZMAB6jxT8bRefi0VRU1mfDCagjt0d2dGJETACmABenphgjPe7d2p+XucBo4Gjg\n",
357 | "ZeCafMOpr9TkdAdwSURsKd7Wap+NdC5uJzsXW6ngs+EE1FEbMLLo9Uiyq6CWFBEvp5+/Br5P1kTZ\n",
358 | "yjakdm8kHQhszDme3ETExkjIJq5rmc+GpIFkyWdBRNyZilvys1F0Lr5bOBeVfDacgDp6HBgr6TBJ\n",
359 | "ewDnAHfnHFMuJP2BpKFpfS9gEvB097Wa3t3AtLQ+Dbizm32bWvqSLfgYLfLZkCTgRmBlRHyjaFPL\n",
360 | "fTa6OheVfDZ8F1wnkqYA36D9IdWv5BxSLiSNJrvqgeyB5e+10rmQdDNwEvA2sjb9K4G7gNuAUcAa\n",
361 | "4OyI2JxXjPVS4lzMAiaSNbEE8ALwmaI+kKYl6QPAQ8BTtDezXU42okpLfTa6OBdXAJ+gzM+GE5CZ\n",
362 | "meXCTXBmZpYLJyAzM8uFE5CZmeXCCcjMzHLhBGRmZrlwAjIzs1w4AZnVgKSt6eehkj5R5WNf0en1\n",
363 | "w9U8vlm9OAGZ1UbhAbvRwJ9VUjFNG9+dyzu8UcSJlRzfrFE4AZnV1leBD6aJuS6RtJukr0lalkYL\n",
364 | "/ksASRMl/UjSXcCKVHZnGol8RWE0cklfBQan4y1IZYWrLaVjP50mEjy76NhLJf2bpF9I+m4hOElf\n",
365 | "TROKPSnpa3U9M9byevpLy8z65lLgf0XERwFSwtkcEcdKGgT8WNLitO8E4KiI+FV6/emIeE3SYGCZ\n",
366 | "pNsj4jJJF6dRygsKV1v/A3gP8G5gP+Ankh5K244GxpGNTvywpBOBZ4AzI+LIFNveNfj9zbrkKyCz\n",
367 | "2uo8Mdck4HxJT5DNnTICGJO2LStKPgCXSFoOPEI2MvvYHt7rA8C/poGINwIPAu8jS1DLIuKlNELx\n",
368 | "cuBQYDOwLc3Z8jHgzV7/lma94ARkVn+fi4gJaTk8Iu5L5b8p7CBpInAKcHxEHA08AezZw3GDXRNe\n",
369 | "4erot0VlbwEDI+ItsqHybwc+AizqzS9j1ltOQGa1tQUYWvT6XuCiwo0Gko6Q9Acl6u0NvBYR2yQd\n",
370 | "CRxftG17Fzcq/Ag4J/Uz7Qd8iGyU5pLTI6dpNoZHxD3AX5E135nVjfuAzGqjcOXxJPBWakr7NjAH\n",
371 | "OAz4WZpPZSPZnCmdZ9FcBHxW0krgWbJmuIJ/Bp6S9NOIOK9QLyK+L+mE9J4B/E1EbJT0TnadoTPI\n",
372 | "EuNdkvYkS1Kfr8pvblYmT8dgZma5cBOcmZnlwgnIzMxy4QRkZma5cAIyM7NcOAGZmVkunIDMzCwX\n",
373 | "TkBmZpYLJyAzM8vF/wdEM++MldDY+QAAAABJRU5ErkJggg==\n"
374 | ],
375 | "text/plain": [
376 | ""
377 | ]
378 | },
379 | "metadata": {},
380 | "output_type": "display_data"
381 | }
382 | ],
383 | "source": [
384 | "plt.plot(J_history, marker='o')\n",
385 | "plt.xlabel('Iterations')\n",
386 | "plt.ylabel('J(theta)')"
387 | ]
388 | },
389 | {
390 | "cell_type": "markdown",
391 | "metadata": {},
392 | "source": [
393 | "Now compute the Root Mean Square Error on both the train and the test set and hopefully they are similar to each other. "
394 | ]
395 | },
396 | {
397 | "cell_type": "code",
398 | "execution_count": 12,
399 | "metadata": {
400 | "collapsed": false
401 | },
402 | "outputs": [
403 | {
404 | "name": "stdout",
405 | "output_type": "stream",
406 | "text": [
407 | "RMS train error: 4.421146725707398\n",
408 | "RMS test error: 5.638168304221702\n"
409 | ]
410 | }
411 | ],
412 | "source": [
413 | "for dataset, (X, y) in (\n",
414 | " ('train', (train_X, train_y)),\n",
415 | " ('test', (test_X, test_y)),\n",
416 | "):\n",
417 | " actual_prices = y\n",
418 | " predicted_prices = X.dot(optimal_theta)\n",
419 | " print(\n",
420 | " 'RMS {dataset} error: {error}'.format(\n",
421 | " dataset=dataset,\n",
422 | " error=np.sqrt(np.mean((predicted_prices - actual_prices) ** 2))\n",
423 | " )\n",
424 | " )"
425 | ]
426 | },
427 | {
428 | "cell_type": "markdown",
429 | "metadata": {},
430 | "source": [
431 | "Finally, let's have a more intuitive look at the predictions."
432 | ]
433 | },
434 | {
435 | "cell_type": "code",
436 | "execution_count": 13,
437 | "metadata": {
438 | "collapsed": false
439 | },
440 | "outputs": [
441 | {
442 | "data": {
443 | "text/plain": [
444 | ""
445 | ]
446 | },
447 | "execution_count": 13,
448 | "metadata": {},
449 | "output_type": "execute_result"
450 | },
451 | {
452 | "data": {
453 | "image/png": [
454 | "iVBORw0KGgoAAAANSUhEUgAAAmMAAAHuCAYAAADTDgvmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
455 | "AAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xec3Vd54P/PM6Pem1Vsyd3GxhTjRrEBY0w3DgECBEIL\n",
456 | "S7JLEvjtbnbXyW52fH/ZbJbsJj9YFrLU0OIAAQyYUGyMhXFvcsHdsmVZliWr9zrz/P4436u5M1YZ\n",
457 | "jebOnfJ5v156cc+3nithzaNznvOcyEwkSZLUGm2t7oAkSdJoZjAmSZLUQgZjkiRJLWQwJkmS1EIG\n",
458 | "Y5IkSS1kMCZJktRCTQ/GImJGRHw3Ih6MiAci4qURMSsiromIRyLi6oiY0ex+SJIkDUWDMTL2aeAn\n",
459 | "mXk68CLgIeAy4JrMPBW4tmpLkiSNOtHMoq8RMR1Ykpkn9jr+EPDqzFwdEfOBxZl5WtM6IkmSNEQ1\n",
460 | "e2TsBGBNRPxDRNwVEV+MiMnAvMxcXV2zGpjX5H5IkiQNSWMG4flnAX+cmbdHxKfoNSWZmRkRzxme\n",
461 | "298xSZKkoSozoz/3NTsYWwGsyMzbq/Z3gT8DVkXE/MxcFRELgGf3d3N/v9RoFxGXZ+blre7HcOXv\n",
462 | "35Hx9+/I+PvXf/7eHRl//47MkQwiNXWaMjNXAU9FxKnVoYuB+4GrgA9Wxz4I/KCZ/ZAkSRqqmj0y\n",
463 | "BvAnwD9GxDhgKfBhoB34TkR8BFgGvGsQ+iFJkjTkND0Yy8x7gHP3c+riZr97FFvc6g4Mc4tb3YFh\n",
464 | "bnGrOzDMLW51B4axxa3uwDC3uNUdGK2aWtriSEREmjMmSZKGgyOJWwZjmnJAucpy6DBYliTpyA27\n",
465 | "YAwMAoYCg2JJkgaGG4VLkiS1kMGYJElSCxmMSZIktZDBmCRJUgsZjI0wEdEVESe2uh+SJKlvDMZa\n",
466 | "LCKWRcRFre6HJElqjVEVjEUwPoKjI5jU6r40SMBSHZIkjVIjKhiLYHIEsyKeG9xEMAv4I+APgI9H\n",
467 | "sHDg3x+XRcRjEbE5Iu6PiLc1nPtoRDzQcO4lEfEN4FjgqojYEhF/GhEXRsRTvZ67b/QsIs6LiJsj\n",
468 | "YkNErIyIz0TE2IH+LpIkaXCMmGAsgrOBfw98HHh3xHO+2/nAtOrzBOA1+3nG7AguieDNEUzvRzce\n",
469 | "Ay7IzGlADfhmRMyPiN8BOoD3V+cuBdZl5vuB5cAlmTk1M//XAZ7bWGB1L/AJYDbwcuC1wMf60VdJ\n",
470 | "kjQEjIhgrAq83kz39zkNOPUwnzER+DBwDnAe8KGIw9uhIDO/m5mrqs/fAR6tnvUR4JOZeWd1bmlm\n",
471 | "Lj+cZze8467MvC0zuzLzSeALwKv78yxJktR6IyIYo+Rc9Z6abO/VvhHYXH3eCVzX6/xcYEpDe2b1\n",
472 | "q++diPhARCypphA3AC8A5gCLgKWH86yDvOPUiPhxRDwTEZuAv6KMkkmSpGFoWO5N2VsmnREsBuqr\n",
473 | "Ep8CHu51zfoIPksJXDZksqPXY9YDu4FxVXs73cHbIUXEcZRRqouAmzMzI2IJJUh8Cjj5QN3v1d4G\n",
474 | "3QsMIqIdOKrh/N8DdwLvzsxtEfH/AO/oaz8lSdLQMiKCMYBMro/gIUo+2NOZdO7nml3AygPcvyWC\n",
475 | "K4ALgU7g2ur6vppMCazWAm0R8QHKyFgCXwL+LiJuAJYAJwG7q6nK1VX7l9VzHgEmRMSbgWuAPwfG\n",
476 | "N7xnCrAF2B4RpwH/Bnj2MPopSZKGkJEyTQlAJs9msnx/gVgf71+WyVcz+Ubm/oO2A9+bDwB/C9wM\n",
477 | "rKIEYjdU575LmU68gjLa9n26p0D/Gvgv1dTmv8vMTZSE/C8BK4CtlJG1uj8F3ls95wvAt+g5utZ7\n",
478 | "pE2SJA1hkTk0f3ZHRGbmfkpU7P+4Bpd/DpIkdTuSn4sjamRMkiRpuDEYkyRJaiGDMUmSpBYyGJMk\n",
479 | "SWohgzFJkqQWMhiTJElqIYMxSZKkFjIYkyRJaiGDsWEgIpZFxEXV5z+PiC8OwjsvjIinDn2lJEk6\n",
480 | "EiNmb8oRbt82CZn53/tyQ0R8FXgqM/+iWZ2SJElHblQFY1GL8cBsYGN25PaW9CFiTGbubcW7JUnS\n",
481 | "0DOipimjFpOjFrOiFs/d07IWs4A/Av4A+HjUYuGAvrtMJV4WEfdHxPqI+EpEjK+m+1ZExH+MiGeA\n",
482 | "L0dxWUQ8FhFrI+LbETGz4Vnvj4gnq3N/3us9l0fENxraF0TETdVG48sj4oMR8VHKZuL/MSK2RMQP\n",
483 | "q2uPjojvRcSzEfF4RPxJw3MmRsRXq77fD5w7kL8/kiRp/0bMyFjU4mzgLZQA86GoxXeyI7saLjkf\n",
484 | "mFZ9ngC8BvhGr2fMBl4OdAE3ZkduOsxuvBd4PbAduAr4L8AvgHnATOBYoB34OHAp8CpgDfAZ4LPA\n",
485 | "eyPi+cDngDcBtwF/DTQGjvumLCPiOOAnwEeB7wLTgUWZeU9EvIIyTflfq2vbqj5dCbwbWAT8IiIe\n",
486 | "zsyrgQ7gBOBEYArws8Z3SZKk5hgRI2NRizbgzXR/n9OAUw/zGROBDwPnAOcBH4paHE6wmsD/ycyn\n",
487 | "M3MD8FfA71bnuoCOzNyTmTuBPwT+S2auzMw9QA14Z0S0A+8ErsrMGzJzN/AX1f37utrw+b3ANZn5\n",
488 | "7czszMz1mXnPAa49F5iTmf8tM/dm5hPAl4D3VOd/B/irzNyYmSuAT/e6X5IkNcFIGRkLnhs4tPdq\n",
489 | "3wicQhkd2wlc1+v8XMqIUN3M6teaw+hH4+rD5cDR1ec1VWBVdzxwZUQ0Bll7KSNoC4AV9YOZuT0i\n",
490 | "1h3gfYuAx/vYt+OAoyNiQ8OxduD66vPR++m/JElqshExMpYd2Qksbjj0FPBwr2vWU6YCvwB8Ojty\n",
491 | "BT2tBxoDpu3A5sPsyrG9Pq+sv77XdcuBN2bmzIZfkzJzJfAMJcgCICImURYd7M9y4KQDnNvfO5/o\n",
492 | "9c5pmXlJdf6Z/fRfkiQ12YgIxgCyI6+n5Fp9Bfhqdjx3xWJ25K7syJXZkTv2c24LcAWwDFgKfDM7\n",
493 | "ctdhdCGAj0XEMRExC/jPwLcOcO3/Bf57RBwLEBFHRcSl1bnvApdExPkRMQ74fznwn9MVwMUR8TsR\n",
494 | "MSYiZkfEi6tzqyn5X3W3AVuqhQQTI6I9Il4QEedU578D/FlEzIiIhcCfIEmSmm6kTFMCkB357BHe\n",
495 | "vwz4an9vpwRHV1Om/H4A/DfgZTx3lKqej3V1RBwNPEsJ3H6UmQ9ExB9Vz5oM/B09pw+z/rzMXB4R\n",
496 | "bwb+FyX/axMlCLwH+DLwz9W05HWZ+faIuAT4W8rU5njgIcoiAyh5a/8XeAJ4uvp9+Hg/fy8kSVIf\n",
497 | "RebQXDAXEZmZzy1RcYDjrRYRTwAfycxftrovg2Go/jlIktQKR/JzccRMU0qSJA1HBmOSJEkt5DSl\n",
498 | "+sU/B0mSujlNKUmSNEwZjEmSJLWQwZgkSVILDcs6YxExNBPdJEmSDtOwC8ZMGpckSSOJ05SSJEkt\n",
499 | "ZDAmSZLUQgZjkiRJLWQwJkmS1EIGY5IkSS1kMCZJktRCBmOSJEktZDAmSZLUQgZjkiRJLWQwJkmS\n",
500 | "1EIGY5IkSS1kMCZJktRCBmOSJEktZDAmSZLUQgZjkiRJLWQwJkmS1EIGY5IkSS1kMCZJktRCBmOS\n",
501 | "JEktZDAmSZLUQgZjkiRJLTSm2S+IiGXAZqAT2JOZ50XELODbwHHAMuBdmbmx2X2RJEkaagZjZCyB\n",
502 | "CzPzJZl5XnXsMuCazDwVuLZqS5KkIS6CMRG8KIIXV5/HVp9ftL92q/s7HERmNvcFEU8A52TmuoZj\n",
503 | "DwGvzszVETEfWJyZp/W6LzMzmto5SZLUZxG0AR+kzGwBPAUEsLBqLwfagWOq9hPANzLpGsx+tsKR\n",
504 | "xC2DNTL2i4i4IyI+Wh2bl5mrq8+rgXmD0A9JknRk5tIdiAE8H2gcTDkDOLWhfQIwexD6NawNxvDh\n",
505 | "+Zn5TEQcBVxTjYrtk5kZEfsdnouIyxuaizNzcfO6KUmSDmEHZZClPgK0u9f53dX5ui5g5yD0a9BF\n",
506 | "xIXAhQPyrGZPU/Z4WUQHsBX4KCWPbFVELACuc5pSkqShL4KzgTdWzZ9TArPXV+2fUQZ6LqYEZT/L\n",
507 | "5K5B72QLHEnc0tRgLCImAe2ZuSUiJgNXAzXKH9K6zPxkRFwGzMjMy3rdazAmSdIQFFFGxjLLKNj+\n",
508 | "2vXPo8VQDsZOAK6smmOAf8zMv65KW3wHOJYDlLYwGJMkScPFkA3GjoTBmCRJGi6G+mpKSZIkHYDF\n",
509 | "2CRJ0gFFMB44k5Kof3fmyFwd2UpOU0qSpP2KoB34CHB0dWg18MVM9rauV0OT05SSJKkZ5tAdiEEp\n",
510 | "0m6h9gFmMCZJkg5kG9DZ0O6i1AvVAHKaUpIk7RPBAuAiyoDNr4AplCKvAVyTyb0t7N6QZWkLSZJ0\n",
511 | "xKpk/U8Ak6pDu4DPZDoadijmjEmSpIEwne5ADGA8MKtFfRk1DMYkSVLdBqBxR5xtwJoW9WXUcJpS\n",
512 | "kiTtE8FM4ALKgM1NmQZjfWHOmCRJUguZMyZJkjRMGYxJkiS1kMGYJElSCxmMSZIktZDBmCRJUgsZ\n",
513 | "jEmSJLWQwZgkSVILGYxJkiS1kMGYJElSCxmMSZIktZDBmCRJUgsZjEmSJLWQwZgkSVILGYxJkiS1\n",
514 | "kMGYJElSCxmMSZI0ykUwM4LZre7HaDWm1R2QJEmtE8FrgVdWn5dk8sMWd2nUcWRMkqRRKoJpVIFY\n",
515 | "5SURHNOq/oxWBmOSJI1e0cdjaiKDMUmSRqlMNgG3Nhy6H3i6Rd0ZtSIzW92H/YqIzEyjc0mSmiyC\n",
516 | "eZQBmlWZDM3AYIg7krjFYEySJOkIHUnc4jSlJElSCxmMSZIktZDBmCRJUgsZjEmSJLWQFfglSRpF\n",
517 | "IpgPvBZoB36dyRMt7tKoZzAmSdIoEcFY4PeAKdWhRRF8JpPNLezWqOc0pSRJo8dUugMxgLHgBuGt\n",
518 | "ZjAmSdLosQlY39DeDjzbor6oYtFXSZJGkYbNwccAN2cajA0EK/BLkiS1kBX4JUmShimDMUmSpBYy\n",
519 | "GJMkSWohgzFJkqQWMhiTJElqISvwS5KkARO1OAo4H0jg19mR6w9xy6hnaQtJkjQgohYTgT8GJleH\n",
520 | "NgP/Jztyd+t6NTgsbSFJkoaCOXQHYgDTgJkt6suwYTAmSZIGynpgZ0N7O7CxRX0ZNpymlCRJAyZq\n",
521 | "sRB4FSVn7LrsyFUt7tKgcDskSZKkFjJnTJIkaZgyGJMkSWoh64xJkqR+i1pMAM6i5IgtyY7ceYhb\n",
522 | "1Is5Y5IkqV+iFmOAjwLzqkOrgC9mR3a2rletYc6YJElqhXl0B2IA84GjWtSXYctgTJIk9dcWoHEU\n",
523 | "rBPY2qK+DFsGY5IkqV+yIzcDVwKbKMVdv5cdaTB2mMwZkyRJOkLmjEmSJA1TBmOSJEktZDAmSZLU\n",
524 | "Qk0PxiKiPSKWRMRVVXtWRFwTEY9ExNURMaPZfZAkSRqqBmNk7BPAA5TKvACXAddk5qnAtVVbkiRp\n",
525 | "VGpqMBYRC4E3A18C6isMLgW+Vn3+GvC2ZvZBkiRpKGv2yNj/B/wHoKvh2LzMXF19Xk3Pyr2SJEmj\n",
526 | "StM2Co+IS4BnM3NJRFy4v2syMyPigIXOIuLyhubizFw8oJ2UJEnqhyq2uXBAntWsoq8R8d+B9wN7\n",
527 | "gQnANOD7wLnAhZm5KiIWANdl5mn7ud+ir5IkaVgYkkVfM/PPM3NRZp4AvAf4ZWa+H/gR8MHqsg8C\n",
528 | "P2hWHyRJkoa6wawzVh+C+x/A6yLiEeCiqi1JkjQquTelJEnSERqS05SSJEk6NIMxSZKkFjIYkyRJ\n",
529 | "aqGm1RmTJEkjT9RiOvCKqnlzduTGVvZnJDCBX5Ik9UnUYhzwMWBGdWgT8LnsyF2t69XQYAK/JEka\n",
530 | "DLPpDsQAplfHdAQMxiRJUl9tAhpHwXZVx3QEnKaUJEl9FrU4llK0PYBfZkc+2eIuDQlHErcYjEmS\n",
531 | "JB0hc8YkSZKGKYMxSZKkFjIYkyRJaiGLvkqSpAOKWhwFvIqSsP/r7MjVLe7SiGMCvyRJI1wE04Ex\n",
532 | "maw75LW1OBF4HWX27NfAG4Ep1entwGeyI3c0q6/D1ZHELY6MSZI0gkXwauA11ed7MrnygNfWYiLw\n",
533 | "HmBcdeh91f/Wa4tNohR5XdGc3o5O5oxJkjRCRTCZKhCrvDiCYw9yyxS6AzGAvUBXQ3sHsH7geihw\n",
534 | "ZEySpJFsf9NmBxuI2QA8C8yt2uuA7wEvr551fXbk9gHtocwZkyRpJIvg9cArquZDwLczOeAP/6jF\n",
535 | "JOA8StB2e3bklub3cvhragX+iDiDsorieCCBZcCvM/P+/rywzx0zGJMkaUBEMI8yG7byYIGY+q8p\n",
536 | "wVhEvB/4E8oQ5W3ASsoQ5QJKxDwH+HRmfrM/Lz5kxwzGJEk6bBE8D3grJfj6VSY3t7hLo0KzVlPO\n",
537 | "BF6buf/hyYiYBnyoPy+VJEkDL4JxwDuBsdWhN0TwRCarWtgtHYI5Y5IkDWMRzAJeS1kFeQ8lGGv0\n",
538 | "j5k8OugdG2WaulF4RPzPiJgeEWMj4tqIWFtNYUqSpBaKoA34AHAGcArwNkpaUd06YHkLuqbD0JfS\n",
539 | "Fq/PzP8QEb9NSd5/O6Ui7zea2TFJknRIk4AZDe0xlJ/REyhTlfdl7ivYqiGqL8FY/ZpLgO9m5qaI\n",
540 | "GJpzm5IkjS7bgbWURXUAuykrJje1rks6XH0Jxq6KiIeAncC/iYi51WdJktRCmXRF8HXgQkrO2C2H\n",
541 | "G4hFLcYCL6akLt2bHenP+EHWpwT+iJgFbMrMzoiYDEzLzGea2jET+CVJaqqoRRvwQeC46tCzwBez\n",
542 | "I/e0rlfDU9M2Co+IGZTd2o8BMiKeBn7e7EBMkiQNipl0B2JQtkE6GniyNd0ZnQ64mjIiPgDcSRn6\n",
543 | "nEhJErwIuCsiPjgovZMkSc20g7IZeF0CW1vUl1HrYBX4HwHOy8yNvY7PBG7LzFOa2jGnKSVJarqo\n",
544 | "xfOBNwHtwLXZkXe2uEvDUtOmKQ/AlZSSJI0Q2ZEPAA+0uh+j2cGCsb8C7oyIq4EV1bFFwOuBv2x2\n",
545 | "xyRJkkaDg66mrFZRvoGSzAfwNHB1Zq5vesecppQkScPEkcQtfS1tMRsgM9f15yX9YTAmSZKGi6bs\n",
546 | "TRkRx0XEtyJiDXArcGtErKmOHd+/rkqSJKnRwTYK/zZwJbAgM0/OzJOBBcAPgG8NRuckSZJGuoOV\n",
547 | "tnj0QOUrDnZuwDrmNKUkSRommlXa4q6I+BzwNeCp6tixlG0TlvTnZZIkSerpYCNj44GPAJdStkOC\n",
548 | "spryR8CXM3NXUzvmyJgkSRommr6ashUMxiRJ0nDRzI3C3wi8je6RsRXADzPzZ/15mSRJkno62DTl\n",
549 | "p4FTgK9TpicBFgLvBx7LzI83tWOOjEmSpGGiKdOUB1oxGREBPFqVumgagzFJkjRcNGuacmdEnJeZ\n",
550 | "t/U6fh6woz8vkyRJrRW1eB7wZqAduC478s4Wd2nUO9jI2NnA3wNT6d4ofCGwGfhYZnP/8BwZkyTp\n",
551 | "uSKYClwIjANuzdz3M/rQ99ZiAvDvgbHVoQQ+mx25dqD7Odo0ZWSsCrbOi4gFNJS2yMxn+vMiSZJ0\n",
552 | "ZCII4APAUdWh50XwuUw29vERE+kOxACCMuhiMNZCB11NCVAFXz0CsIg4LTMfalqvJEnS/kyiOxCD\n",
553 | "Mjq2APocjG0EngSOq9pr6V6kpxY5ZDB2ANcAiwayI5Ik6ZB2ABuAmVV7L/Bs1GI6ZcRrXXZkRi1m\n",
554 | "AGOzI9dELcYCZ1J+5t8DfLOxnR25e7C/hHo6WM7YZw5y34cyc2pzurTv/eaMSZLUSwSzgIspo2I3\n",
555 | "c3ksAF5LmXK8H1gNXFRdfj8wGTi+aq8HPp8dzd1FZzRqVmmLLcCfArsoCX77TgF/m5mz+/PCPnfM\n",
556 | "YEySpIOqEvL/E+VnM5TRscl0T1tOqH41TmN+Izty6aB1cpRoVmmLO4DfZOaN+3nh5f15mSRJGnC9\n",
557 | "A4DG9h56/qxPSlUEDSEHGxmbBezMzO2D26V973dkTJKkQ4haXAS8qmo+SpmmvKBqPwTcCbyFEpQt\n",
558 | "zo68Y9A7OQq4UbgkSSNUBO3AWcB44N7M545sRS2OouSQrawS+OdQpixXZccQ/UE/wjQtGIuIiyhb\n",
559 | "Hz0VEccBX6bUI/kPmXl9v3rb144ZjEmSRATvo+wVDWWK8fOZbGthl7QfRxK3tB3i/P+gO+nvr4Hv\n",
560 | "AZ8APtWfl0mSpL6LYALdgRjANLprhGmEOGACf5Wkvwj4d2VvcN4APA7MBeZERAdAZtaa301JkkaH\n",
561 | "CMYAZ1NWQd5LqS02seESE/BHmENNU95KKW8xF/hgZl5aHb8xM89vasecppQkjUK9piW3AT+m1BUb\n",
562 | "D9yYyc2t6psOrFmlLaBsJvq3wG7gD6qXnQHc3Z+XSZKkA4tgHD2nJScD7ZkcrBC7hjlXU0qS1ELV\n",
563 | "tOR5lGnJe4DfpwRhdV/KZEUr+qa+a0oCf0Rc2IcXv6Y/L5UkSfu8B3g9pVbYvwKuotQK2wT8zEBs\n",
564 | "5DvYNOUlEfE3wC8o1fifoQRv84FzKPPX11W/JEnSYYpgLHByw6GJwLhM/r5FXVILHDAYy8w/jYip\n",
565 | "wG8Br6N7Ke2TwA3AX2Xm1uZ3UZKkkSmTPRFsodTwrNvQqv6oNcwZkySphSJYAFxCGRW7JZPbWtwl\n",
566 | "9YPbIUmSJLVQMyvwS5IkqYmaFoxFxISIuDUi7o6IByLir6vjsyLimoh4JCKujogZzeqDJEnSUHfI\n",
567 | "YCwiJkfEX0TEF6v2KRFxyaHuy8ydwGsy80zgRcBrIuIC4DLgmsw8Fbi2akuSJI1KfRkZ+wdKBf5X\n",
568 | "VO2VwF/15eGZub36OA5op6wQuRT4WnX8a8Db+tpZSZKkkaYvwdhJmflJSkBGZm7r68Mjoi0i7qYU\n",
569 | "r7suM+8H5mXm6uqS1cC8w+yzJEnSiHGovSkBdkXEvt3iI+IkYFdfHp6ZXcCZETEd+Hnviv2ZmREx\n",
570 | "NJdzSpIkDYK+BGOXAz8DFkbEFcD5wIcO5yWZuSki/gU4G1gdEfMzc1VELACePdB9EXF5Q3NxZi4+\n",
571 | "nPdKkjTURDABeCWlrthdbnc0PFXbRl44IM/qS52xiJgDvKxq3pKZa/t4z97M3FiNrP0cqAFvANZl\n",
572 | "5icj4jJgRmY+J4nfOmOSpJEogt8Hjq2ae4DPZ3LIn6sa2o4kbjnkyFhEvB34ZWb+uGrPiIi3ZeYP\n",
573 | "DnHrAuBrEdFGyU37RmZeGxFLgO9ExEeAZcC7+tNxSZKGmwja6Q7EAMYCi8BgbDQ75MhYRNyTmS/u\n",
574 | "dezuqmRF8zrmyJgkaQSK4I+BOVUzgS9msrKFXdIAaOrIGLC/B7f352WSJIl/BN5IyRm7zUBMfRkZ\n",
575 | "+wdKfbDPUgKzPwJmZuaHmtoxR8YkSdIw0ey9Kf+EkmD4beBbwE5KQCZJklogajExajGt1f3QwOjT\n",
576 | "aspWcGRMkjQcRXA08NvAZOAO4BHKbjOTKNOSi4/o+bU4F3gTZUDlHuAH2TFEf5iPIk3JGYuIT2fm\n",
577 | "JyLiqv2czsy8tD8vlCRphHs3ML36/CrgYqpdbIALI3gqk6X9eXDUYhzdgRjAiykB2eP9765a7WAJ\n",
578 | "/F+v/vd/8dwkfiNwSZKACCYBr6WMhC0BpjaeBmYBqxqOHcn0Yr1cVKO+LMbTEHbAP8DMvDMixgB/\n",
579 | "mJnvHcQ+SZI0nLwbOK76fCrwNKV2GMA24EbgpIb2c0bFohYBtGdH7o1aTAQuACZQpjknAL8FjAdu\n",
580 | "AG4CXlHd+uT+nqfhpS+rKW8AXpuZfdqPcqCYMyZJGg4i+K/0HK36KWUP58nAA8AmynTiJOD+TDb2\n",
581 | "uL8WzwPeDowDbgeOBhZWp/dQCsM2+jLQVV2/PDuycyC/j/qn2XXGngBuiIgfAdurY5mZf9efF0qS\n",
582 | "NMKsoLuqfhewIpOne12zpLERtTiJUmfsMeAdlMAK4OWUfLP1VXsiZZqzcR/nqdmRDwxY79VyfQnG\n",
583 | "lla/2oAplPlvc8YkSSq+DVxE+Rm5pHcgFrWYRUm6n0SZdpxH937PGygBV310q5PugQ8oif/3AvOr\n",
584 | "9ibKVoIaQfpc2iIiplNGxDY3t0v73uc0pSRp2Ita/BFwVMOhqcCWhvY6YHbD5+8DF1JyxW4BHqJM\n",
585 | "c44HfpMd2XivhohmbxR+LvAVqtUfEbER+Ehm3tGfF0qSNNJELWZScsSeoUxVnk4ZCXuE7n0o68b1\n",
586 | "al9d3TMBeDQ7cidly6RGdw10nzV09CWB/z7gY5n566p9AfC5zHxRUzvmyJgkaQiqan29kjIteS9l\n",
587 | "VOstlDSepyn5XS+pLt9S/Tq6au8BfkEphTGOEmRdZdHW4e9I4pa+BGNLMvMlvY7dlZln9eeFfe6Y\n",
588 | "wZgkaQiIWkymFG6dSimw+kJKCQsoOV4T6S7qCiUnbHVD+2eUUbJJwJLsyKejFm2UUhZ7mtx9DZJm\n",
589 | "r6b8VUR8Hvinqv3u6thZAJnp0KkkaST7HeD46vNJlKBsK11tQVtXO2X145qG67f3vJ0N2ZG3NB7I\n",
590 | "juyiTE1KfRoZW8xBVk9m5msGuE/19zoyJkkadFUB1pdQcqUfBD5CY57XlgVHseKlp9I1ZgxTVq9m\n",
591 | "0Y1fp63rHErVgSeBX9G9F+Wd2ZE/HezvoMHX1GnKVjEYkyS1QtTircDZVXM3pfzEvKqdXPGDCRx9\n",
592 | "18mM2zKOpy54hgff/lkuj2WU6cq11aiXRplmT1NKkjSanNHweRyltMQy6jljj/zWO3jkt+5vuGZi\n",
593 | "VW7CkhPqF4MxSdKoFrUYC7yKkvt1H7CRrXNPZs+k8Ux9ZhNjdj3L5bmBsnpyOXArZTUlwGbKVKbU\n",
594 | "bwZjkqTR7reB51efX8iv/vNSxm09lfGbx7P2+Y9x05/OoyTxQ9mm6EuUkbIpwNJMtg56jzWi9KXo\n",
595 | "62Tg3wHHZuZHI+IU4HmZ+eOm906SpCaIWkykFGldD5zQeIrNC1/Bnf+6sVLAO+neG3IW8MJMbh2c\n",
596 | "nmo06MvI2D8AdwKvqNorge8CBmOSpGEnanEqZaRrLKVI6xq2zD+D3ZMnMG3lRjYev77XLTt7tXcj\n",
597 | "DaC+BGMnZea7IuI9AJm5LcJFjpKkYetNlEAM4Bju/MgaNh2/kAkbxrPqzF0sfePXgDdT9oJcCtwG\n",
598 | "vIOSzP8opeq+NGD6EoztioiJ9UZEnATsal6XJElqqrYerafPO527/qAxCX8K8D+BCfV8sAj+JzDe\n",
599 | "/DA1Q19YmPtXAAAgAElEQVSCscspWzksjIgrgPOBDzWxT5IkDaioxRRKkLUWuJZSlLUNWMvDv7WF\n",
600 | "Uraiblcme6E78MpkD2VfSWnA9anoa0TMAV5WNW/JzLVN7RUWfZUk9U+1kfeFwEzgAUoQ9U7KAMRq\n",
601 | "4KvABEoAtorL82jgPdWxR4FvZdI56B3XsNbsjcIvAO7OzK0R8X7KFhGfzswn+/PCPnfMYEyS1AdR\n",
602 | "ixmU8hSzKDW/JlI2865rhx7B1S+yI2/o8YygjTINuaPJ3dUIdSRxS9uhL+Hvge0R8WJKiYulwNf7\n",
603 | "8zJJkgZK1KKeavNbwHGUka7zKOk0jWYe6lmZdBmIqVX6kjO2NzO7IuJtwGcz80sR8ZFmd0ySNLpF\n",
604 | "LcZmR+6pPp8IzAGeoAwkvBeYHrV4jDIi1mhbr/ZPgbMoI2RrKOWapCGjL9OU11MS+D9M2f5hDWXa\n",
605 | "8oUHvfFIO+Y0pSSNSlGLo4D3ATMowddS4OLq9B5gBzCt4Zb1dAdkncA/AqdT5YxlR94VtZhGSeBf\n",
606 | "Uw/wpIHU7JyxBZR/gdyWmb+OiGOB12Tm1/rzwj53zGBMkkaMqEUbMAnYlh2ZUYs5wGxgZXbklqjF\n",
607 | "MZSRr+XApfSsij8d2NTQnkNZFVl3A6VC/izg0ezIp5/z/uAkSnC3NJONDe3HMns8W+qXpgZjrWIw\n",
608 | "JkkjQ9RiLvB7lNGs1cAtwCWUacMdlI23Xw0Epbr9XkrgVjcFetT3egpYVH3eCXw5O3LNAd8fvBJ4\n",
609 | "bcP19wNnV+0dwBcz6V11XzoszR4Z2wrULxpHqVq8NTOnHfiuI2cwJknDR9RiOmXEajUl4LkImE+Z\n",
610 | "ZjwOOLnh8t4jXbOBdQ3ttdWxoBQZv7J63mzgEeB7rHjp8eyevIiJ6+/l80s6KRXy66spbwLeTndp\n",
611 | "i1Ord9YdRUm5qftlJtf3+8tLHFnccsgE/syc0vCiNsrw8csOfIckabiLWpwNHEMZhbobeDlwdNW+\n",
612 | "h7KC8WjgSeA+4F2Uf6zvoEw1Pq961Ik8t7RE71GA3gn391OCrjnA8uzIDRHsqdqPA8dW7xtHyQ3b\n",
613 | "U/UV4Bzg3IZ3nFv1qzFPbMsh3i8Nqn5NU0bE3Zl5ZhP60/gOR8YkaZBELcZTcqg2UupJvrHh9EpK\n",
614 | "4FW3lTJ1WDcRepSF6D3StYYyatVOGTW7qnp+KboK/0L3SNYy7n3fd/j+N19OCb4epQRd9f7UK+E3\n",
615 | "TmP2ft9cSg5Z3X3AQsroWH3k7Hco06b3A1dm0vXc3xWp75o9TfmOhmYbZZ791Zn58v68sM8dMxiT\n",
616 | "pCMStYgqWb6NshrxWEpgtZgyy3Es8DRwPfBuunOztgHzGh7Ve1qvd3smsKGhvYfujbihBF/LqvtW\n",
617 | "Zkdujlq0U4K4ktAfTOKo38xmzQtWAW+gjHDV9R5ZmwU9crw20726ci8lgDu9of3lTJ6JIDK7R+V6\n",
618 | "t6Uj0exg7Kt0D/fupfwH9cXMfPZA9wwEgzFJ6p+oxUTK9j7HUnK4ltEzvaR3sDSekptV1zv46b13\n",
619 | "4wrKSFPdtZRiq1MpI1TfAF5ElTOWHXn7c/oYnEwZwXqiev/vVv+7oXp/Y17yVHpOLT4MnERJtdkA\n",
620 | "fIWSl1ZWU5bA64WUIPGRTFb1fr800FxNKUnaJ2rxRnoGX72DmUONbD1ESbBfRMn/upay2vGYqv0r\n",
621 | "SlL80ZScrseqavhTgC3ZkQfd1zGCc4G3VM1OSiA46cB3cC3wAsq0ZUngL9dPB57N7BFISi3R7JGx\n",
622 | "RcD/Bi6oDl0PfCIzV/TnhX3umMGYJPVZVSh1EWUk7KWUkam6pKxMrHuMMrJUP3Y9ZVpwEiX36+vZ\n",
623 | "kc8MaP+CyZSgbw3wAboT7uG50463UYK0oygjXbcOZF+kZmjqakrgHyjVjN9Vtd9XHXtdf14oSToy\n",
624 | "UYug7L94IiUB/lHK381jKIHXLZRgpr1qf5eSBH8cJUfsDuB4SvD2dHbk0qjFzZSRp3XZkdsPu09l\n",
625 | "2nEB8GQmyyM4hTJNuYzu7YvGU/K7ehdZvan6LuMoCwhutBCrRpO+jIzdk5kvPtSxAe+YI2OStF9R\n",
626 | "i/OANzccaoMeqwGXUfZjXAis2l9F+iN6f9AOXEgZ3VpGSfq/tDqdlNWL9ZG5LsoKysZpyN9QVm7O\n",
627 | "pWx19H1KIDYdWJvJ7oHsrzQYmj0yti4i3g9cQRnSfg89t6GQJA2uo3u1p1BGnOq2Z0eupkxZNsNr\n",
628 | "6E5dOZHys2Rv1Q5Kgdb6z4k2StDWmJO2PZPv9nrmHqz3pVGqrQ/X/D5linIV8AylNsuHm9kpSdJB\n",
629 | "Le/VvpZSfBVKAHb1QL8wglMiuDCCE+mZ7wVl+rFR73+w30QZHYMyDXnTQPdPGs5cTSlJw0DUYipl\n",
630 | "RGxtduS6qMU5dOeM3ZAd2RW1aMuOPGDx0gjaGoub1tsRtFFGsxZRylYspkxDLqRU3F9P9zQklKnJ\n",
631 | "4xva11Hy0RZU566urq9vh3QlZfRsOrAus0c1fGlEaMpqyoj4TEOz90qczMyP9+eFfe6YwZikUSZq\n",
632 | "MTY7ck/1uZ2y+nALJYj5MKVIaifwz9mRD/W4t6xWnEcJdjZFMIWSk7WWMoX4u1Q5ZMAPKRt1L6QU\n",
633 | "gX2KsgKzrrGIKjx3O6EHKQsB6tsh3WbxVI12zcoZu5PuIKwG/Fe6AzL/o5OkAVJtRfRe4LioxUbg\n",
634 | "B5RgaQ6l1MQqSiAGZYXk+ZRaYOX+YC7wIUqS/J4IrqaMdE2kBFFPU0a9oIxefRz2JckfTdlHsrEO\n",
635 | "2Un0rEPWOxjbkMkN/f7Cknro0zRlRCzJzJcMQn8a3+nImKQRK2rxQkoQtJYSYL2y4fRkYBtd7W20\n",
636 | "dXZR36ao3s5YSq3rabqnKcfRs65YfY/Jut51vHrv5dh79uMZStBW90tKnli96OsPXPEo9dTs1ZSS\n",
637 | "pMNUFWE9iRIELQXeRgm+1gH30HMj7i52TxrLtrlTmbBpB+O2TGLZa05h54wZjNm5g6N+cz0rXn4a\n",
638 | "7XuOo6ttA3d99EHgVdW9jdsS1e3t1V5KmXZsp5SauIoyLVlvf5syFVrPGbuNsr1RPWfsdqchpeZx\n",
639 | "ZEySBkDUYgoliX0DJcD5CN37P/YeaapvyF1sn9XG4xefQ+QkyE5WnHsrcx84h3Hbx9HV3sXtH3uE\n",
640 | "5eevYvKaceyYuYeucVPpWcpiLWV1/KzquT+g1CGrt79OGfk6BliVycoI5lXtZzIZ0Gr70mjUrAT+\n",
641 | "rXTnhk2k5C3UZWZOe+5dA8dgTNJQV1+9GLWYCfwryvRiUvKt5jZcehQrz5rEtrlzGLtjB8fc8iRr\n",
642 | "XnAU2+bOZuy27aw79XGu/4tg3t3T2XTsdlafOZYJ6zcz5+EpbF64g82LptBzWrG36yn7RU4HtmSy\n",
643 | "pyrMuq/dlN8ASfu4UbgkDaJqpeM7gNMpW/ssp2fOVs+NuVeeOYV1p51FW2ep7bj0dQ/SOW4cc++b\n",
644 | "zbZ527njDx9n97TGqcWNlGT8cZTg7kbKtGG9/X3KSFc9Z+xWpxGl1jJnTJIG19nA86vPMyhTkI2z\n",
645 | "B49SRsdOA9bzk8/+hp0zk6PvmMmWY3bwxEXtlFysp6rrN1KKos6nVKH/TvW8Y4E11bTiXZScrjWZ\n",
646 | "rKzuu7dp31DSoDEYkyQgajELOBXYnB35QNTidOCFlJGvxZTg6wWUwGlXr9ufBdawZ+LzaN+9ibbO\n",
647 | "H2dHrohgUnXtCcAZrD29vt3PU3SXmoBSt+saSi7Z9sx9Cfj7thDKZD09V0RKGiGcppQ06kUtZgMf\n",
648 | "BSZUhx6nBFD1v4M2UkbA6laya8p8ts6fx7itO2nffRV/s+54xm0+iT2Tt5Dt36VUsD8W2E7Z23cS\n",
649 | "3aspbwFOoQR/aynTjAesnC9p6DNnTJKOQNTifOB1DYfmU3Kx9t/eMXMvn78TFty5iG3zd7L8gvvo\n",
650 | "WWKi52pJWJnJFwa+55KGCnPGJOnIbOnVXsnKs05g64J5tO/cyfwlt7PphFPZOn8eY3buZPPRD7Hx\n",
651 | "hN1sPKFeEuJMem6OPYGewdi4ZnZe0vBmMCZp1Khqge3KjtwTtZhIqQu2BbiPzQtOYdeMV9G2Zy2/\n",
652 | "/rNfsXPm0Sy8dSc7Zuzin78zkYW3beeYW7ayY9Yulvx+7ymFxyl1vqZQVjv+M2X1Y73UhVsHSTog\n",
653 | "pykljXhVKYp3U3K0dgM/Yd3Jb2fbvFMYs3MX0fktPr/kDEpJCiijXHMaHjEPWN3Q3k4p5HpS9fmf\n",
654 | "KMn1i4CNmayqNupeRNnHsXHKU9II5DSlJB3cCymBGMA4ts/8Y1adeRJE+Ytz77h/C9zccP18yirI\n",
655 | "9qp9GyUnrN5+MJOrIpgA7G5Ivt+3eXcmWymrJCXpoAzGJA1LUYtFwEsoI1O/ptT6egmlTtf1lL0b\n",
656 | "z6Lkbj3e4+Yds6fvC8QAds6Y1Ovxq4ErKeUsNgG3U7YOOp2ysvIOgEx2Duy3kjQaGYxJGhaiFq+m\n",
657 | "5GHtoARbl9L9d9jplNIT9ZGrs9i0aD4bj59P+67dzLtnMpuOC7bNex7tu3az8bgfsP6UtzP70dl0\n",
658 | "jtvLY2/6GSVgez4luPtRJisom2bXNRZplaQBY86YpCEvanEi8IGGQ9MpI1Z1CyilJ8pfaDumncgT\n",
659 | "Fy8k28v2Q+27t/D9r9/FMXfMYvus3aw+807gESY9ey67p65n78SfZLIlgrHAXrcWknS4zBmTNOJE\n",
660 | "LcZSVjvuAKb1Ot27QOqTwFjqRVrXnr6WPZMXMGZnfS/IMeyeljxxUX2z7RMyuQLm3tH4EDfUltQK\n",
661 | "BmOShpyoxTjgw5QRL4Db2XDCODYtOon2PXuZ/fC/8PClGxi74610tW+ma8zn6Rx7JmN3vIVs28R9\n",
662 | "7/06a09/N8f++ih2Tt/Do295ku7q+oCrGyUNHU5TShpyohYvAN6578C2OZP40k17OfbGeeyetoeH\n",
663 | "3/ooXWMn0Z0jtpEyetZWtZcBS+hO6P85cCKlOOtW4OeZzyn0Kkn95nZIkoa9qhbYAkpJidnsnvwB\n",
664 | "NpxwFO2799A1tovP3bu2O9bqlSP23HYX8JfmfkkaLEM2ZywiFgFfB+ZS/pL8Qmb+74iYBXybkg+y\n",
665 | "DHhXZm5sZl8ktVbUYirwIkqwtYRSEPWlVftXwNsofyfAxkU3ct/7jmHW0uPoauvi0bcsrgKxejT2\n",
666 | "BDCe7o28H6dMQ9bbzxiISRoumjoyFhHzgfmZeXdETAHupPyF+2FgbWb+TUT8J2BmZl7W615HxqRh\n",
667 | "LGrRRhmx2k2ZKvxDyipIKKNYc+j+B2Ebz7xkERuPW0hb514mrVnLV697ggmbxrBnYie7p+0Fvkup\n",
668 | "G7YduIZS9+uchvai6vw24BqnISUNpiE7MpaZq6gSZTNza0Q8SPkL9FLg1dVlXwMWA5ft7xmShp8q\n",
669 | "EHsvcHJ1aBndgRjAi9g9eSvrT5pF++69TFw3kQ3Hn0CO6aJr7Fg2nHQ82fY42+btrq7fCTyQyf0N\n",
670 | "z9hMzwr391e/JGlYGbTVlBFxPCWZ9lZgXmbW93lbTdn3TdLIcSLdgRiU7Yh2AHsB2D1hI0svPpOu\n",
671 | "caXy/a5JG9i68FlmPzKHJHn4kgfpGncjZeRrJ3Cl046SRqpBCcaqKcrvAZ/IzC3RsAtJZmZE7Pcv\n",
672 | "2Yi4vKG5ODMXN7OfkgZMFwlsmzeVtr2dTFq3mVv/eD0TNr2JbNvFo2/8IVsWTuaknx/D3gl7ufuD\n",
673 | "K9i6YDUznpjDnsmdbF1wdyZXA1e3+otI0v5ExIXAhQPyrGavpoyIscCPgZ9m5qeqYw8BF2bmqohY\n",
674 | "AFyXmaf1us+cMWmYijO+08YJiz/FpLUvIklWveSX3PBn7Q2XTKSMeNX/AtoJfI6yHdEu4J6Gzbcl\n",
675 | "acgbsqUtogyBfQ1Yl5n/tuH431THPhkRlwEzTOCXhr4Ioj5dGMHxTNjwKjrH7mHPlGs46wvHMuPJ\n",
676 | "95Btu/jNe37GmtNfx6ylk9g7oZPNi6ZSaoHtbXjc9cC5lODrh5k8MfjfSJIGxlAOxi6g/IV7L93/\n",
677 | "Av4z4DbgO8CxHKC0hcGYNLTE+C2v5JjbfpvOsXtY8fIfcO7nPsqxN55IRrL04kc5/vrjGLNrPABb\n",
678 | "52/jp5+6A9rq/923Axvo3tbowUy+3YKvIUlNMWSDsSNhMCYNvqjFRCCzI3fGyz41h2kr3wG5lwff\n",
679 | "dh2nX/l3zFg+A4ANJ+xmxrIJRJapxPYdM+icsB1i976HXfPJf2LjCQso/xC7FrgLOINS6uI+pyEl\n",
680 | "jSQGY5L2K2oxGyA7cl3UYiJd7WcBSVvnXcBEutpeVtpdNwHnkVwAkWyfdSOPvuUPGbvjKAD2jt3O\n",
681 | "mN2T9tVUHbdlMp3jdtI5vrO0Nyed46BzQv2/2Y3snPFOrvrCDGBPJpsG83tL0mAzGJNE1OIo4FWU\n",
682 | "iOkGSgHU86rTd7L8/Feydf45AEx9+g6icyKbjz0DgNkPPwgxkXWnHg+ZzFy6mU3HzdkXbI3ZOp72\n",
683 | "zu3sml6S8CeuXcusx29n47Fn0dbVxfQn/4XlFyynfc/vkm27yLa/z3/+1sOD+PUlqaUMxqRRKGqx\n",
684 | "EHgtZYugm4C3AlMaLhlDPWF+15Q5PPnqM9kzqbTHbJ8E0cXeiTv3tbOti84Jpd22ezxBF53j9gCQ\n",
685 | "7GbRzV9k95TXQnYy/alvMG77HZRaYnuAx7NjiP5lIkmDYMhW4JfUHFGLCcD7yJhEBrR1nQJAV1sJ\n",
686 | "ntq6JrLtqJmsOrNMM05bvoGutu4H7Jq6F0jay8AX247axfa5W5j5RCnCuuqsp4jOnzPn4beS0cm2\n",
687 | "uZ/Pv3vq6qjFVUBXdmQ9N+yh5n9bSRrZHBmThokIgrKRdjv/cc5Wts7/S9aedjIZweyHn6Rtb7L2\n",
688 | "+ceREUxd+ShPXDSZOQ8fA8CqFy9n3SmPc9LVL4NIHn/d9eyc3sVJP38FRPLom37Fk6/exsKbzyDb\n",
689 | "kxUvu4Ns/0er3ktS3zhNKY0CEbwdeBEA47Y8w+v//duYuHEiADund3L7v36I0340FxLuf/ca1pyx\n",
690 | "hTkPthFdsOaMbcBXKPtDJmVPxwBOb2i3Ve1O4KFMOgf5K0rSsGUwJo1QEYyhBEmTgU80nJrElGcm\n",
691 | "8YJvzyA6gwfesY5Nx++lbJ4Npa7XXvYtf2QT8LlMdg1W3yVpNDEYk0agCM4F3kQJxu4AzqY7uAJY\n",
692 | "DcyrPj9JCb5OqtrLgB8CL6/aN2XSo7CyJGngGIxJI0wEk4A/pQRidY0B2U3AL4BTq2seAbqA51Xn\n",
693 | "H3aaUZIGj6sppREggkXAGyj/Xd5Fz0AM5i95mPe9eStdY9qZvuKW7Mgunrua8cFB6awkacA4MiYN\n",
694 | "kqjFGEoh1rGU/Vr3AC8DxrLivLv54Vf+hBdecQZte9t4+K2P0zV2Oaf98FVEZxvLXnkLv/vbG2jf\n",
695 | "e3T1uHXA5xtKTEiSWshpSmkYiFp8ADixam6i7NFY6oDtmNHFo29+Fe17ymh1RhfH3Ho3wRi62tuY\n",
696 | "/uRm2jsDeiTg/0N25JOD9w0kSQfiNKU0xEUtptAdiAHMpVTLXwfAhI2TmPFkG1uqga/xm2DqqgmM\n",
697 | "2fVsdf0YYALdwVgXsKX5PZckNZvBmDRAqqr4rwDGA3cC24FXA+Oq9q7qHJRRsQ3dN7ODBXddR/uu\n",
698 | "k8m2NmY9+jhjdm2he3ujzcB3gAspuWS/zI5c3/QvJUlqOqcppcMQtVhEGaFaRhmdOqtq3wu8E1hU\n",
699 | "XbqTEnBNq9q7gX+hOzi7nsdev5G9Ez5GdI0j277FaT+6G3glJafs5uoZF1D+0XRzdmR9lEySNMSY\n",
700 | "MyYNgqjFxZTgCGAVsI3uul47KYFXPaF+LDADWLPvAUs+fD0//MrzqnM3UGqAHVWd3UEpyurUoyQN\n",
701 | "Q+aMSU1WrYQ8v+HQMcBsoD5aNYESZNWDsT0kq1h/0ny6xrYzffmz/PqyV8O+vR7fWV2/rWpPBBZg\n",
702 | "HpgkjToGY9IBRC3mA5dQAqXbKXs21v+b6aQ7kKq7AngxJS/sJr5wx9k874cvYsyuMTx+8TrWn9pV\n",
703 | "3QelrEVjHbG9wNrmfBNJ0lBmMCYd2O9SNtYGeCOl6v1LKfs+3kcJ0N5KGRW7OTtySQSPUXLCtgPv\n",
704 | "4Jmz72943u7qHJSE/O9Rpj3HAjdkYkK+JI1CBmPSfkQt2ukOxOqeBT4JjM2OrI+KfXbfPcFLKUFb\n",
705 | "AA9TRr/GNtz/LcpU5FhgSSabKAsBJEmjmAn80gFELd4HnFI1dwL/Nzty32bbEUwBXkOZllwCvI+e\n",
706 | "U4+3AGdSgq8bM/nlYPRbkjT4TOCXmuPbwHmUnLF7GgOxyu8B86vPzwcmUYK2uicy+VkEkcnQ/FeP\n",
707 | "JKnlDMakStRiLCX4mgTcmx25mpIn9txrg/F0B2JQRsQeBo6r2k8AjwEYiEmSDsZgTOr2O8Cp1edz\n",
708 | "ohafP1CV+0x2RbAWmFMd6gR+TMkXGw88nUlXszssSRr+DMY0qkQtjgImA09TAqgzq/YDdOeHQQmo\n",
709 | "joeeKxwjmElZPbka+Cbw2qp9W2ZDgVdJkvrIYEyjRtTiZcAbKKNXqyiB1vOr06+g5HtNbLhlXY/7\n",
710 | "g5cDr6/ufwL4Zibfa3K3JUkjXNuhL5FGjIsogRSUfK/XNJybSNnM+0nKFkY/yY58sn4ygnbgdQ33\n",
711 | "nwA8r9kdliSNfI6MaTTp7NXe2qu9LDvyF4fxPBPzJUlHzJExjSY/oTsgexz4FCX3ayuwODvysQPd\n",
712 | "mEkn8HO6A7ClwCPN66okabSw6KtGlajFJErC/YbsKP/n710HrN6OYDolR2wicHsmD1bHJgBrXC0p\n",
713 | "Sao7krjFYEwjWgT7Vk9msqexDRwDvJNSV+xO4EHgHZTg6w7Kasq51aO6gC9m8sygfgFJ0rBgMCbt\n",
714 | "RwTnAW+iJN0/C/yGkrRfX005nZ6rJydRNvimumYWPVdU/jCTJU3utiRpGDqSuMWcMY1kjasn5wLv\n",
715 | "pudqyuMbrg1gakM76VljbC+woim9lCSNaq6m1Mg167HxnPWlFzFu83ieesUz3Pd723pdcT/dFfQ3\n",
716 | "AfcBL2hofwM4lzJ6dpdFXSVJzWAwphEjgjZK8DQVuJ8/eHcbG0+YQba1cebXZzPt6Su58T+dBbRT\n",
717 | "VkNeAZxMySF7BNhGmcqcBDycyTbgp634LpKk0cOcMQ1rEQQwNpPdEfw2R9/xSiatHc/T56zhj58/\n",
718 | "nnFbOukcP4bxm3YS/JTL817KasiNbuAtSRooJvBrVIhatAHnUBLvH+CKH81g4S1/ztjtU9h4/C3s\n",
719 | "mXQCJ11TNvreOW0n05+6lZOvrv8ffC/whezIZ1vTe0nSSHYkcYvTlBpOLgHOqj6/lBf801m0750F\n",
720 | "wLSn30D7zvF0TtgFwITNE9g54wFKCYspwH0GYpKkochgTENK1GIWMA3+//buPcjOur7j+Pu72exm\n",
721 | "N1cCiAEiiRQCQYjc0YpEQaUUQUsHEGVQR6Ydb9S21gszXXf6R6udtipqO8NdrFgGHETrIEFAKQgo\n",
722 | "BIRgCBfDJZdNQjYhgVw22W//eE6SzbIhm+xmf2c379dMJs/vOefsfs9vsjmffZ7v83tYAmwEjqY6\n",
723 | "EvYU6yfOYvFxb2PzmDGMX9zBxBensHbKhq0v3ue5DtYcBJubmpmwaClT5j6VbflokTciSVI/GcZU\n",
724 | "N6I9jgY+TLXkykpgIduOhL2b5947k2ysrn58+fBxjNrwLHAwAA1d62nYdAV/cseJVD1h84HfD+kb\n",
725 | "kCRpNxjGVEzt1kRnAfsAT1L1g21Z+24y3Q1HsOzoZja2jmHii8tZfOI6WlZ20vxKM0uOX8aTf3Ed\n",
726 | "f3nhAXQ37s+YVXfnd+bPi/a4G2jKtlxf6G1JkrRLbOBXMdEeF9E96gg2jWlk9Ksb2Th2HItOns6m\n",
727 | "ljGMW7qc7lHjWX3IvtWzM1lw9r08dsmW+0F2A1dnsqhU/ZIkbWEDv4anzmkzWPr2d9A9ejTNq1fT\n",
728 | "Oa2D5rWTGNU1ihfe2c3Dly7i+CubGPNKM0uPWcbvP3o30EXVUzbPICZJGgkMYxpS0R5vo7rn49M8\n",
729 | "fvmb2HfBaADWT5zA3E918vJhv6F1xWhWTVtPNjZz+7eX9nh5RyZPlqhbkqQ9xTCmIROXHXo2q07/\n",
730 | "JF0tLbS+3MHTZz5H5/RmWpe3sOjk5Sw+cQnQzIZJm6hOQ14LHAtMAp4wiEmSRiJ7xjRk4oLzbiBH\n",
731 | "Td26Y8WM+7j7nzZS3aR7DXA1MIMqfM3P5PkihUqStIvsGdPwsObAYFzHtnF34yLgFqrw9WImrwEP\n",
732 | "lilOkqQyPDKmQRPBeKp1wvYDngL+j+ZV59G6YgqvTH2U/ec1M+uGS2jpbGH5zGU8fOnluW4fj35J\n",
733 | "koY9702puhDBRcDhW3cc9MAETvruMTRuaGTtAWt46LNf5+UZr1E18C/I5IVStUqSNJgGklsadv4U\n",
734 | "qd/22W50xK3H0bihOhU+rmM8x111ciaPZHKnQUySpIphTINpXo/tbiY/+9y2YSYH/XbxkFckSVKd\n",
735 | "s4FfgyaTeyJYCewLPMNRN09g+ZGfpqt1LK3Ln2fSCz8vXaMkSfXGnjENSLTHWGAisALYDLyLWgN/\n",
736 | "tuUT0R6Tqa6WXJJtua5cpZIk7Tk28KuIaI+3AhcCTUAnsAh4W4+n/Cjbcn6J2iRJGko28KuU06mC\n",
737 | "GFTN+6f1enzakFYjSdIwZBjTYFrZa7ykSBWSJA0jhjENxC+BjbXtTuDbwMPA88CcbMvHShUmSdJw\n",
738 | "Yc+YBqTWwD8BWJFt2VW6HkmSSrCBX5IkqSBvFK4hE+3RCIwF1mRbdpeuR5Kk4c4jY+q3aI8DWX7k\n",
739 | "F+hq3Z8xqxYw+dlv8rWcDuwPPJPJS6VrlCSpBI+MaWg884HPsGHCidUgD+GJC1qoFnoFeHcE389k\n",
740 | "YaHqJEkalryaUv235sC3bBtE8MrUt/d4tAE4cqhLkiRpuDOMaaeiPap/J8+d8QjdDVWfWFfrRp49\n",
741 | "Y16vp64e4tIkSRr27BnTDtXuK/kRqntN/pFbfvC/LJ11PuOWvoXOQx9l1fTbgQ9R9YwtAH6WiU39\n",
742 | "kqS9Tt0ubRER1wB/DizLzKNr+yYD/wMcAiwEzs/MVX281jBWQASTgX2BJXwtzmXdpGNYP6mFscvX\n",
743 | "0koh6nQAAAwmSURBVPTqndmWd5WuUZKkelPPDfzXAlcA3++x78vAnMz8RkR8qTb+8h6uQ/0QwWHA\n",
744 | "BVT/Ltbxx9mTWLfviWRDAw1dXez31BOFS5QkacTZoz1jmXkv1W1yejoHuL62fT3VaS7Vh3exLaC3\n",
745 | "cN8XD6G7oUr5m1oauOMbXn0rSdIgK/HhekBmdtS2O4ADCtSgvm3abvTMWYt57U1LmbhwLB2zVrPy\n",
746 | "sKWF6pIkacQqeqQjMzMidti0FhFf6zG8JzPv2eNF7d3uAC4GxgHLgBtYfMKHWXzClvF9JYuTJKle\n",
747 | "RMRsYPagfK09fTVlREwDftqjgX8+MDszl0bEFODuzDyij9fZwF9AXPLeCazf5yAmLPpjXvnA+gga\n",
748 | "gVZgrVdKSpLUt3pu4O/LbcAlwNdrf99aoAb1IdrjEN7KRUAz8Eq0x7WZ2Qm8Urg0SZJGrD3awB8R\n",
749 | "NwL3AzMi4sWI+ATwL8D7ImIB8N7aWPXhPVRBDGAC8M6CtUiStFfYo0fGMvMjO3jojD35fbXbep+z\n",
750 | "rs8VgSVJGkG8HZJ6ugtYX9teRXVUU5Ik7UHeDmkvF+0xgWrF/Y5sy9eiPVqAicDL2ZZdZauTJGl4\n",
751 | "qNvbIQ2EYWzPi/aYzubRH6OrdSxNaztp2HxNtuXy0nVJkjTcDLerKVUvOqefTccxp7K5qYnGda8x\n",
752 | "5ZGFVPcNlSRJQ8QwNoJFEMDJwIHA88AjwCnAFGAh57znSJrXNAGwqaWVp8553XpvkiRpzzKMjWyn\n",
753 | "Ui0fAnAMcAITX5jOPs+2suKItTx42Vre8e8bGf1aE+smreOhzzxVsFZJkvZK9oyNYDFm1Sc56bvv\n",
754 | "Z8JL4+l862pe+NNuTr7iQBo2N7CpeRMPXDaHl2c0M7ZjLGsOWsWmlqszWVa6bkmShhsb+NWn+OBf\n",
755 | "XU5L5/u27mhas4mN47cdDU1u5+ab/ovqasqlmbw69FVKkjT82cCvvr39usUsOfYlNkwcT8vKVUyZ\n",
756 | "u5pFJ05mw4Rq/OZH52WyimpNMUmSVIBhbCRr3DifqQ8e0mPPPUx94DiqWx2tAH5VpjBJkrSFpylH\n",
757 | "uGiPWdSupsy2fDLaYxQwDliTbdldtjpJkkYGe8YkSZIKGkhu8d6UkiRJBRnGJEmSCjKMSZIkFWQY\n",
758 | "kyRJKsgwJkmSVJBhTJIkqSDDmCRJUkGuwD/CRHscARwEvJBt+XTpeiRJ0htz0dcRJNrjeOCDPXbd\n",
759 | "km35eKl6JEnaW7joq7Y4cidjSZJUZwxjI0vnTsaSJKnO2DM2stwJjKXWMwbcU7QaSZK0U/aMSZIk\n",
760 | "DZA9Y9oqgoigpXQdkiSpfzxNOYJEMAm4GNg3gmXADZmsKVyWJEl6Ax4ZG1nOAPatbb8JmF2uFEmS\n",
761 | "1B+GsZGleSdjSZJUZwxjI8sDwObadhfw24K1SJKkfvBqyhEmgv2AA4AlmawsXY8kSXuDgeQWw5gk\n",
762 | "SdIAubSFJEnSMGUYkyRJKsgwJkmSVJBhTJIkqSDDmCRJUkGGMUmSpIIMY5IkSQUZxiRJkgoyjEmS\n",
763 | "JBVkGJMkSSrIMCZJklSQYUySJKkgw5gkSVJBhjFJkqSCDGOSJEkFGcYkSZIKMoxJkiQVZBiTJEkq\n",
764 | "yDAmSZJUkGFMkiSpIMOYJElSQYYxSZKkggxjkiRJBRnGJEmSCjKMSZIkFWQYkyRJKsgwJkmSVJBh\n",
765 | "TJIkqSDDmCRJUkGGMUmSpIIMY5IkSQUZxiRJkgoyjEmSJBVkGJMkSSqoWBiLiDMjYn5EPB0RXypV\n",
766 | "x0gUEbNL1zCcOX8D4/wNjPO3+5y7gXH+yikSxiJiFPAd4ExgJvCRiDiyRC0j1OzSBQxzs0sXMMzN\n",
767 | "Ll3AMDe7dAHD2OzSBQxzs0sXsLcqdWTsJOCZzFyYmV3Aj4BzC9UiSZJUTKkwdhDwYo/xS7V9kiRJ\n",
768 | "e5XIzKH/phHnAWdm5qW18ceAkzPzcz2eM/SFSZIk7abMjN15XeNgF9JPi4CpPcZTqY6ObbW7b0iS\n",
769 | "JGk4KXWa8nfAYRExLSKagAuA2wrVIkmSVEyRI2OZuSkiPgv8AhgFXJ2ZfyhRiyRJUklFesYkSZJU\n",
770 | "qasV+CPiXyPiDxHxWET8OCIm9njsK7UFYudHxPtL1lnPXEx310TE1Ii4OyLmRcQTEfH52v7JETEn\n",
771 | "IhZExB0RMal0rfUqIkZFxNyI+Glt7Nz1U0RMioiba//vPRkRJzt//Vf7XJgXEY9HxA8jotn561tE\n",
772 | "XBMRHRHxeI99O5wrP3O3t4P5G7TMUldhDLgDOCozZwELgK8ARMRMqr6ymVQLxX4vIuqt9uJcTHe3\n",
773 | "dAFfyMyjgFOAz9Tm7MvAnMw8HPhlbay+XQY8CWw5zO7c9d+3gJ9n5pHAMcB8nL9+iYhpwKXAcZl5\n",
774 | "NFXLy4U4fztyLdVnQ099zpWfuX3qa/4GLbPU1eRm5pzM7K4NHwQOrm2fC9yYmV2ZuRB4hmrhWG3P\n",
775 | "xXR3UWYuzcxHa9trgT9QrXl3DnB97WnXAx8qU2F9i4iDgbOAq4AtV0A7d/1Q+y361My8Bqpe2sxc\n",
776 | "jfPXX69Q/TLVGhGNQCuwGOevT5l5L9DZa/eO5srP3F76mr/BzCx1FcZ6+STw89r2gWy/9IWLxPbN\n",
777 | "xXQHoPab9rFUP1QHZGZH7aEO4IBCZdW7/wC+CHT32Ofc9c90YHlEXBsRj0TElRExFuevXzJzJfBv\n",
778 | "wAtUIWxVZs7B+dsVO5orP3N33YAyy5CHsdr56cf7+PPBHs+5HNiYmT98gy/llQev55zspogYB9wC\n",
779 | "XJaZa3o+ltVVLs5tLxFxNrAsM+ey7ajYdpy7N9QIHAd8LzOPA16l1yk152/HIuJQ4G+AaVQffuNq\n",
780 | "C4hv5fz1Xz/mynncgcHILEO+tEVmvu+NHo+Ij1Od9ji9x+7ei8QeXNun7e10MV29XkSMpgpiN2Tm\n",
781 | "rbXdHRHx5sxcGhFTgGXlKqxb7wTOiYizgDHAhIi4Aeeuv14CXsrM39bGN1P1nCx1/vrlBOD+zHwZ\n",
782 | "ICJ+DLwD529X7Ohn1c/cfhqszFJXpykj4kyqUx7nZub6Hg/dBlwYEU0RMR04DHioRI11zsV0d1FE\n",
783 | "BHA18GRmfrPHQ7cBl9S2LwFu7f3avV1mfjUzp2bmdKrG6bsy82Kcu37JzKXAixFxeG3XGcA84Kc4\n",
784 | "f/0xHzglIlpqP8dnUF1I4vz1345+Vv3M7YfBzCx1tc5YRDwNNAEra7t+k5mfrj32VapzspuoTiX9\n",
785 | "okyV9S0i/gz4JtsW0/3nwiXVtYh4F/Br4PdsO4z8FaofnJuAtwALgfMzc1WJGoeDiDgN+LvMPCci\n",
786 | "JuPc9UtEzKK6+KEJeBb4BNXPrvPXDxHxD1Qhoht4BPgUMB7n73Ui4kbgNGA/qv6wfwR+wg7mys/c\n",
787 | "7fUxf21UnxWDklnqKoxJkiTtberqNKUkSdLexjAmSZJUkGFMkiSpIMOYJElSQYYxSZKkggxjkiRJ\n",
788 | "BRnGJNW1iFjba/zxiLiiYD0zIuK6qNxfqg5JI4dhTFK9670YYunFEU+lWij4GOCJwrVIGgEMY5KG\n",
789 | "m603Ja/d+uuuiHgsIu6MiKm1/ddFxHk9nre29veUiPh1RMyNiMdrd2AgIt4fEfdHxMMRcVNEjH3d\n",
790 | "N404NSLmAl8H/h74GfCBiPA2MZIGxDAmqd611MLT3FoYamfb0bErgGszcxbw38C3a/t3dDTtIuD2\n",
791 | "zDwWmAU8GhH7AZcDp2fm8cDDwN/2LiIz76297qnMnAnMAc7MzJMG7Z1K2is1li5AknZiXS0EARAR\n",
792 | "lwAn1IanAB+qbf8A+MZOvtZDwDURMRq4NTMfi4jZwEzg/up+0zQBffaCRUQrsKE2PAxYsMvvRpJ6\n",
793 | "8ciYpOEmdjKG6ua8DQAR0UAVsMjMe6l6vhYB10XExbXnz8nMY2t/jsrMS1/3TSN+AswFZkbEY1Q9\n",
794 | "Y7+LiPMH401J2nsZxiQNZ/cDF9a2P0rVWA+wEDi+tn0OMBogIt4CLM/Mq4CrgGOBB4A/jYhDa88Z\n",
795 | "GxGH9f5GmXkucCXw18Dngf+shbeb9sD7krQXMYxJqnd99X9t2fc54BO1I1UfBS6r7b8SOC0iHqU6\n",
796 | "lblleYz3UPWJPQKcD3wrM1cAHwdurH2d+4EZO6jl3cB9VEfXfjXA9yVJAERm6avEJUmS9l4eGZMk\n",
797 | "SSrIMCZJklSQYUySJKkgw5gkSVJBhjFJkqSCDGOSJEkFGcYkSZIK+n/M6EMn+g11QAAAAABJRU5E\n",
798 | "rkJggg==\n"
799 | ],
800 | "text/plain": [
801 | ""
802 | ]
803 | },
804 | "metadata": {},
805 | "output_type": "display_data"
806 | }
807 | ],
808 | "source": [
809 | "plt.figure(figsize=(10, 8))\n",
810 | "plt.scatter(np.arange(test_y.size), sorted(test_y), c='b', edgecolor='None', alpha=0.5, label='actual')\n",
811 | "plt.scatter(np.arange(test_y.size), sorted(test_X.dot(optimal_theta)), c='g', edgecolor='None', alpha=0.5, label='predicted')\n",
812 | "plt.legend(loc='upper left')\n",
813 | "plt.ylabel('House price ($1000s)')\n",
814 | "plt.xlabel('House #')"
815 | ]
816 | }
817 | ],
818 | "metadata": {
819 | "kernelspec": {
820 | "display_name": "Python 3",
821 | "language": "python",
822 | "name": "python3"
823 | },
824 | "language_info": {
825 | "codemirror_mode": {
826 | "name": "ipython",
827 | "version": 3
828 | },
829 | "file_extension": ".py",
830 | "mimetype": "text/x-python",
831 | "name": "python",
832 | "nbconvert_exporter": "python",
833 | "pygments_lexer": "ipython3",
834 | "version": "3.4.2"
835 | }
836 | },
837 | "nbformat": 4,
838 | "nbformat_minor": 0
839 | }
840 |
--------------------------------------------------------------------------------
/Logistic_Regression.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 17,
6 | "metadata": {
7 | "collapsed": false
8 | },
9 | "outputs": [],
10 | "source": [
11 | "%matplotlib inline"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 18,
17 | "metadata": {
18 | "collapsed": false
19 | },
20 | "outputs": [],
21 | "source": [
22 | "import scipy.optimize\n",
23 | "import time\n",
24 | "import matplotlib.pyplot as plt\n",
25 | "import numpy as np\n",
26 | "from sklearn.datasets import fetch_mldata"
27 | ]
28 | },
29 | {
30 | "cell_type": "code",
31 | "execution_count": 19,
32 | "metadata": {
33 | "collapsed": false
34 | },
35 | "outputs": [],
36 | "source": [
37 | "def normalize_features(train, test):\n",
38 | " \"\"\"Normalizes train set features to a standard normal distribution\n",
39 | " (zero mean and unit variance). The same procedure is then applied\n",
40 | " to the test set features.\n",
41 | " \"\"\"\n",
42 | " train_mean = train.mean(axis=0)\n",
43 | " # +0.1 to avoid division by zero in this specific case\n",
44 | " train_std = train.std(axis=0) + 0.1\n",
45 | " \n",
46 | " train = (train - train_mean) / train_std\n",
47 | " test = (test - train_mean) / train_std\n",
48 | " return train, test"
49 | ]
50 | },
51 | {
52 | "cell_type": "markdown",
53 | "metadata": {},
54 | "source": [
55 | "First get and preprocess the data."
56 | ]
57 | },
58 | {
59 | "cell_type": "code",
60 | "execution_count": 20,
61 | "metadata": {
62 | "collapsed": false
63 | },
64 | "outputs": [],
65 | "source": [
66 | "# get data: contains 70k samples of which the last 10k are meant for testing\n",
67 | "mnist = fetch_mldata('MNIST original', data_home='./data')\n",
68 | "\n",
69 | "# prepare for concat\n",
70 | "y_all = mnist.target[:, np.newaxis]\n",
71 | "\n",
72 | "# intercept term to be added\n",
73 | "intercept = np.ones_like(y_all)\n",
74 | "\n",
75 | "# normalize the data (zero mean and unit variance)\n",
76 | "train_normalized, test_normalized = normalize_features(\n",
77 | " mnist.data[:60000, :],\n",
78 | " mnist.data[60000:, :],\n",
79 | ")\n",
80 | "\n",
81 | "# concat intercept, X, and y so that shuffling is easier in a next step\n",
82 | "train_all = np.hstack((\n",
83 | " intercept[:60000],\n",
84 | " train_normalized,\n",
85 | " y_all[:60000],\n",
86 | "))\n",
87 | "test_all = np.hstack((\n",
88 | " intercept[60000:],\n",
89 | " test_normalized,\n",
90 | " y_all[60000:],\n",
91 | "))"
92 | ]
93 | },
94 | {
95 | "cell_type": "markdown",
96 | "metadata": {},
97 | "source": [
98 | "I don't think this randomization step is really needed in our case, but let's stick with the ufldl tutorial here."
99 | ]
100 | },
101 | {
102 | "cell_type": "code",
103 | "execution_count": 21,
104 | "metadata": {
105 | "collapsed": false
106 | },
107 | "outputs": [],
108 | "source": [
109 | "np.random.shuffle(train_all)\n",
110 | "np.random.shuffle(test_all)"
111 | ]
112 | },
113 | {
114 | "cell_type": "markdown",
115 | "metadata": {},
116 | "source": [
117 | "Now prepare the final train and test datasets. Let's only pick the data for the digits `0` and `1`."
118 | ]
119 | },
120 | {
121 | "cell_type": "code",
122 | "execution_count": 22,
123 | "metadata": {
124 | "collapsed": false
125 | },
126 | "outputs": [],
127 | "source": [
128 | "# train data\n",
129 | "train_X = train_all[np.logical_or(train_all[:, -1] == 0, train_all[:, -1] == 1), :-1]\n",
130 | "train_y = train_all[np.logical_or(train_all[:, -1] == 0, train_all[:, -1] == 1), -1]\n",
131 | "\n",
132 | "# test data\n",
133 | "test_X = test_all[np.logical_or(test_all[:, -1] == 0, test_all[:, -1] == 1), :-1] \n",
134 | "test_y = test_all[np.logical_or(test_all[:, -1] == 0, test_all[:, -1] == 1), -1]"
135 | ]
136 | },
137 | {
138 | "cell_type": "code",
139 | "execution_count": 23,
140 | "metadata": {
141 | "collapsed": false
142 | },
143 | "outputs": [],
144 | "source": [
145 | "def sigmoid(z):\n",
146 | " return 1 / (1 + np.exp(-z))"
147 | ]
148 | },
149 | {
150 | "cell_type": "code",
151 | "execution_count": 24,
152 | "metadata": {
153 | "collapsed": false
154 | },
155 | "outputs": [],
156 | "source": [
157 | "def cost_function(theta, X, y):\n",
158 | " h = sigmoid(X.dot(theta))\n",
159 | " return -sum(y * np.log(h) + (1 - y) * np.log(1 - h))"
160 | ]
161 | },
162 | {
163 | "cell_type": "code",
164 | "execution_count": 25,
165 | "metadata": {
166 | "collapsed": false
167 | },
168 | "outputs": [],
169 | "source": [
170 | "def gradient(theta, X, y):\n",
171 | " errors = sigmoid(X.dot(theta)) - y\n",
172 | " return errors.dot(X)"
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": 27,
178 | "metadata": {
179 | "collapsed": false
180 | },
181 | "outputs": [
182 | {
183 | "name": "stdout",
184 | "output_type": "stream",
185 | "text": [
186 | "Optimization took 1.385227918624878 seconds\n"
187 | ]
188 | },
189 | {
190 | "name": "stderr",
191 | "output_type": "stream",
192 | "text": [
193 | "/Users/fhartl/anaconda/envs/py34/lib/python3.4/site-packages/IPython/kernel/__main__.py:3: RuntimeWarning: divide by zero encountered in log\n",
194 | " app.launch_new_instance()\n",
195 | "/Users/fhartl/anaconda/envs/py34/lib/python3.4/site-packages/IPython/kernel/__main__.py:3: RuntimeWarning: invalid value encountered in multiply\n",
196 | " app.launch_new_instance()\n",
197 | "/Users/fhartl/anaconda/envs/py34/lib/python3.4/site-packages/IPython/kernel/__main__.py:2: RuntimeWarning: overflow encountered in exp\n",
198 | " from IPython.kernel.zmq import kernelapp as app\n"
199 | ]
200 | }
201 | ],
202 | "source": [
203 | "J_history = []\n",
204 | "\n",
205 | "t0 = time.time()\n",
206 | "res = scipy.optimize.minimize(\n",
207 | " fun=cost_function,\n",
208 | " x0=np.random.rand(train_X.shape[1]) * 0.001,\n",
209 | " args=(train_X, train_y),\n",
210 | " method='L-BFGS-B',\n",
211 | " jac=gradient,\n",
212 | " options={'maxiter': 100, 'disp': True},\n",
213 | " callback=lambda x: J_history.append(cost_function(x, train_X, train_y)),\n",
214 | ")\n",
215 | "t1 = time.time()\n",
216 | "\n",
217 | "print('Optimization took {s} seconds'.format(s=t1 - t0))\n",
218 | "optimal_theta = res.x"
219 | ]
220 | },
221 | {
222 | "cell_type": "code",
223 | "execution_count": 28,
224 | "metadata": {
225 | "collapsed": false
226 | },
227 | "outputs": [
228 | {
229 | "data": {
230 | "text/plain": [
231 | ""
232 | ]
233 | },
234 | "execution_count": 28,
235 | "metadata": {},
236 | "output_type": "execute_result"
237 | },
238 | {
239 | "data": {
240 | "image/png": [
241 | "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEPCAYAAACp/QjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
242 | "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3X2UXHWd5/H3B0JMIEHIAHkiYJYEIQZNRCMcfGgRkEQE\n",
243 | "3NkFdHTQlawOGhhcXQkzC60zI7gOrMCoqwHdDCM4UQ8QlaeQoQUUeUyAEKIJEJJA6CAGSCSBAN/9\n",
244 | "496iq7uqu6s6devWw+d1Tp2693fv/dU3fTr17ft7uooIzMzMiu2SdwBmZtZ4nBzMzKyEk4OZmZVw\n",
245 | "cjAzsxJODmZmVsLJwczMSmSWHCSNkHS3pOWSVkq6MC3vlLRB0rL0NbvomvmSVktaJem4rGIzM7OB\n",
246 | "Kct5DpJ2j4iXJA0D7gS+DHwI2BIRl/Q5dxpwNfBuYCJwK3BwRLyeWYBmZlZWps1KEfFSujkc2BXY\n",
247 | "nO6rzOknAddExI6IWAusAWZlGZ+ZmZWXaXKQtIuk5UA3cFtEPJIemifpQUlXStorLZsAbCi6fAPJ\n",
248 | "HYSZmdVZ1ncOr0fEDGB/4P2SOoDvAZOBGcBG4OKBqsgyPjMzK29YPT4kIl6Q9CvgXRHRVSiXdAXw\n",
249 | "i3T3KWBS0WX7p2W9SHLCMDMbgogo16RfVmbJQdI+wKsR8bykkcCxwNckjYuIZ9LTPgY8nG4vBq6W\n",
250 | "dAlJc9JU4J5ydVfzD7SBSeqMiM6842gF/lnWln+etVXtH9ZZ3jmMBxZK2oWk+eqqiFgq6V8lzSBp\n",
251 | "MnoC+BxARKyUtAhYCbwKnBleMtbMLBeZJYeIeBh4Z5nyvx7gmm8A38gqJjMzq4xnSFtX3gG0kK68\n",
252 | "A2gxXXkH0M4ynQSXBUnhPgczs+pU+93pOwczMyvh5GBmZiWcHMzMrISTg5mZlXByMDOzEk4OZmZW\n",
253 | "wsnBzMxKODmYmVkJJwczMyvRlMlBmn2TNH1O3nGYmbWqpkwOcOOH4chLnSDMzLLRpMkBYMEUOGBe\n",
254 | "3lGYmbWiJk4OAKNG5h2BmVkravLksHVb3hGYmbWiJk4OZzwG6y7POwozs1bUpMnhE/fC786KWHFD\n",
255 | "3pGYmbWiLJ8hnaGrb4/AicHMLCNNeufA7LwDMDNrZc2aHPaVODDvIMzMWlWzJoeb8d2DmVlmmjU5\n",
256 | "3AB4drSZWUYySw6SRki6W9JySSslXZiWj5G0RNIfJN0iaa+ia+ZLWi1plaTjBqj+FqBDYkRW8ZuZ\n",
257 | "tbPMkkNEbAc+GBEzgLcDH5T0XuBcYElEHAwsTfeRNA04FZgGHA98V1LZ+CJ4DlgBvC+r+M3M2lmm\n",
258 | "zUoR8VK6ORzYFdgMnAgsTMsXAien2ycB10TEjohYC6wBZg1QvZuWzMwykmlykLSLpOVAN3BbRDwC\n",
259 | "jI2I7vSUbmBsuj0B2FB0+QZg4gDV34g7pc3MMpHpJLiIeB2YIenNwM2SPtjneEiKgaooVyipEyQ4\n",
260 | "d39p2ccjbrymdlGbmTU/SR1Ax1Cvr8sM6Yh4QdKvgMOBbknjIuIZSeOBTelpTwGTii7bPy0rV18n\n",
261 | "QDrXYUxmgZuZNamI6AK6CvuSLqjm+ixHK+1TGIkkaSRwLLAMWAycnp52OnBdur0YOE3ScEmTganA\n",
262 | "PYN8jPsdzMwykOWdw3hgYTriaBfgqohYKmkZsEjSZ4G1wCkAEbFS0iJgJfAqcGZEDNTkBLAEuEJi\n",
263 | "ZARevtvMrEY0+PdvY5EUEaGefe4A/imCm3IMy8ysofX97hxMs86QLuZRS2ZmNdYKycH9DmZmNdYK\n",
264 | "yeFBYA+JqXkHYmbWKpo+OUQQuGnJzKymmj45pNy0ZGZWQ00/Wikp480ky22MjeCl8leambWvdhyt\n",
265 | "RAQvAPcDHxzsXDMzG1xLJIeU+x3MzGqklZLDDcAciYpvm8zMrLxWSg4rSJ4bcXDegZiZNbuWSQ7p\n",
266 | "kFaPWjIzq4GWSQ4p9zuYmdVASwxl7TnGniTPgBgfwdb6RmZm1rjacihrQQQvAvcCR+cdi5lZM2up\n",
267 | "5JC6ATctmZntlFZNDh7Sama2E1oxOTyavh+aaxRmZk2s5ZKDh7Same28lksOKQ9pNTPbCS01lLXn\n",
268 | "HEYBG4EJEWypT2RmZo2rrYeyFqRzHH4HfCjvWMzMmlFLJoeU+x3MzIaolZPDjcBsD2k1M6teZslB\n",
269 | "0iRJt0l6RNIKSWel5Z2SNkhalr5mF10zX9JqSaskHbeTIfwe2AFM38l6zMzazrAM694BnBMRyyWN\n",
270 | "Au6XtAQI4JKIuKT4ZEnTgFOBacBE4FZJB0fE60P58AhC4kaSpqWHd+YfYmbWbjK7c4iIZyJiebq9\n",
271 | "lWRy2sT0cLmmnpOAayJiR0SsBdYAs3YyDC+lYWY2BHXpc5D0FmAmyQgigHmSHpR0paS90rIJwIai\n",
272 | "yzbQk0yG6jbgcIk372Q9ZmZtJctmJQDSJqWfAWdHxFZJ3wO+nh7+B+Bi4LP9XF52EoakzqLdrojo\n",
273 | "Kntx8JLEb4BjgJ9XH72ZWXOS1AF0DPX6TJODpN1IvpT/LSKuA4iITUXHrwB+ke4+BUwqunz/tKxE\n",
274 | "RHRWEUZhSKuTg5m1jfSP5q7CvqQLqrk+y9FKAq4EVkbEt4vKxxed9jF6OosXA6dJGi5pMjAVuKcG\n",
275 | "oXhIq5lZlbK8czgK+CTwkKRladl5wMclzSBpMnoC+BxARKyUtAhYCbwKnBk1WNsjgtUSfwbeASzf\n",
276 | "2frMzNpBS66tVHoNlwEbI7gwo7DMzBqa11Yqz0tpmJlVoV3uHEYC3cCBEWzOJjIzs8blO4cyItgG\n",
277 | "3AEcm3csZmbNoC2SQ6qwlIaZmQ2iLZqVkuv4T8BvSR4ANKT1mszMmpWblfoRwePACyTLeJiZ2QAy\n",
278 | "Xz6jsVy9Erp+Ij3/FGzZDusvi1hxQ95RmZk1mrZJDtL0OdAxC34wAZiSlM49SJqOE4SZWW9t06wE\n",
279 | "k86Cf5nQu2zBFDhgXj7xmJk1rjZKDqNHlC8fNbK+cZiZNb42Sg5btpcv37qtvnGYmTW+NkoO6y+D\n",
280 | "uWt6l53xGKy7PJ94zMwaV9vMc0iunT4Hpn4Fph8F9y+FdZe7M9rM2kG1351tlRyS6xHwNHBEBE/W\n",
281 | "LjIzs8blSXCDiCBIHiL0nrxjMTNrVG2XHFJ3A7PyDsLMrFG1a3LwnYOZ2QDars8hqYM3A08Be0Xw\n",
282 | "am0iMzNrXO5zqEAELwDrgbflHYuZWSNqy+SQctOSmVk/2jk5uFPazKwf7Zwc7sHJwcysrLbskE7q\n",
283 | "YTiwGRgbwdadj8zMrHE1TIe0pEmSbpP0iKQVks5Ky8dIWiLpD5JukbRX0TXzJa2WtErScVnFBhDB\n",
284 | "K8BDwOFZfo6ZWTPKsllpB3BORLwNOAL4gqRDgXOBJRFxMLA03UfSNOBUYBpwPPBdSVk3e7lT2sys\n",
285 | "jMy+fCPimYhYnm5vBR4FJgInAgvT0xYCJ6fbJwHXRMSOiFgLrCH7PgF3SpuZlVFRcpC0h6RDJL1V\n",
286 | "0h7VfoiktwAzSb6Mx0ZEd3qoGxibbk8ANhRdtoEkmWTJndJmZmX0+wxpSaOBucBpwD4kX+QCxkp6\n",
287 | "DvgxsCC9K+iXpFHAz4GzI2KL1NMfEhEhaaAe8bLHJHUW7XZFRNdAMQzgMWAPifERbBxiHWZmDUdS\n",
288 | "B9Ax1Ov7TQ7AdcBPgI8W/aVf+NBxJM1D1wMfGiC43UgSw1URcV1a3C1pXEQ8I2k8sCktfwqYVHT5\n",
289 | "/mlZiYjoHCDuikUQ0ht3D9fXok4zs0aQ/tHcVdiXdEE112c2lFXJLcJC4LmIOKeo/H+nZd+UdC6w\n",
290 | "V0Scm3ZIX03yRT0RuBWYEn0CrNVQ1p76+BowLIK/q1WdZmaNptrvzoHuHIorHQNMBd5UKIuI2we5\n",
291 | "7Cjgk8BDkpalZfOBi4BFkj4LrAVOSetbKWkRsBJ4FTizb2LIyN3Al+rwOWZmTWPQOwdJc4GzSJp5\n",
292 | "lpMMS70rIo7OPryy8dT6zmFfYDUwJoLXa1WvmVkjyWIS3NkkTT1PRsQHSUYdvTDE+BpOBM8CfwLe\n",
293 | "mncsZmaNopLksD0itgFIGhERq2i9L1LPdzAzK1JJctggaW+S0UtLJC0m6StoJZ7vYGZWpKrRSum4\n",
294 | "2T2BmyLilayCGiSGmvY5JHVyFHBpBO+qZb1mZo2i5n0Okq4qbEdEV0QsBq4cYnyN6gFgmsSIvAMx\n",
295 | "M2sElTQrTS/ekTSMFlvJNIJtwCpgRt6xmJk1gn6Tg6TzJG0BDpO0pfAimdG8uG4R1s/deIVWMzOg\n",
296 | "snkOF0XEuXWKZ1BZ9Dkk9fIZ4JgI/qrWdZuZ5S2LeQ7nSfqUpPPTDzhAUiuO7PGzHczMUpUkh+8C\n",
297 | "RwKfSPe3pmWtZhWwn8Rf5B2ImVneKkkO74mIM4FtABHxJ2C3TKPKQQSvAfcB7847FjOzvFWSHF6R\n",
298 | "tGthR9K+0LJrELlT2syMypLD5cC1wH6SvgH8Brgw06jy45nSZmZUOENa0qH0PNRnaUQ8mmlUA8eS\n",
299 | "yWilpG4mkqw8u19E+afQmZk1o2q/OytNDrsC40ie/xAAEbFuqEHujCyTQ1I/G4D3R/B4Vp9hZlZv\n",
300 | "NX/Yj6R5wAUkk99eKzp0WPXhNYVC05KTg5m1rUqeBPe3wFsj4rmsg2kQhU7pn+QdiJlZXirpkF4H\n",
301 | "vJh1IA3EndJm1vb67XOQ9D/SzWnAIcAvgcIy3RERl2QfXtm4su5z2BN4Gtg7gh1ZfY6ZWT3Vss9h\n",
302 | "NEnn8zpgPTA8fbW0CF6UWEvSp/JAzuGYmeWi3+QQEZ0Akk6JiEXFxySdknFceSs0LTk5mFlbqqTP\n",
303 | "YX6FZa3E/Q5m1tb6vXOQNBuYA0yUdBlQaKsaDS3fFn83MC/vIMzM8jLQncPTwP3A9vT9fpKF6RYD\n",
304 | "H66kckk/lNQt6eGisk5JGyQtS1+zi47Nl7Ra0ipJxw3lH1QjK4AD085pM7O2U8nDfoZHxCsDntT/\n",
305 | "te8jWeL7XyPisLTsAmBL39FOkqYBV5OsijoRuBU4OCJe73NepqOVej6HO4HzI/iPrD/LzCxrNXvY\n",
306 | "j6RfSfqvlGl6krSHpFMl3TBQ5RFxB7C5XPVlyk4CromIHRGxFlhDvu3+7ncws7Y1ULPSZ0iGc94n\n",
307 | "6WFJt0hakjYR3QccCpw+xM+dJ+lBSVdK2istmwBsKDpnA8kdRF6cHMysbQ00lHUTcD5wvqSxwIHp\n",
308 | "oScjonsnPvN7wNfT7X8ALgY+218Y5QoldRbtdkVE107E05+7SWIzM2s6kjqAjqFeP9Bopa30/+X8\n",
309 | "Mkmzz99HxK3VfGCadAr1XAH8It19CphUdOr+aVm5Ojqr+cwhWgsMl5gYUT4OM7NGlf7R3FXYT/t7\n",
310 | "K9Zvs1JEjIqI0eVeJMt3fw64tNqAJY0v2v0YUBjJtBg4TdJwSZOBqSRNO7lIn+dwN25aMrM2VMmq\n",
311 | "rCUi4lXgQUmXD3SepGuADwD7SFpPsvR3h6QZJHclT5AkGSJipaRFwErgVeDMqORhE9m6h2SF1mtz\n",
312 | "jsPMrK4qethPI6nXUNbkszge+J8RHF2PzzMzy0omT4JrJHVODmNI+h72juj1oCMzs6ZSs3kOBhH8\n",
313 | "CegmWbLczKxtODkMzvMdzKztODkMrvDYUDOztuHkMDjfOZhZ23GH9KCfxwjgOWCfCLbV63PNzGrJ\n",
314 | "HdI1FsF2krkX78w7FjOzenFyqIyblsysrTg5VMbLaJhZW3FyqExhGQ0zs7bg5FCZPwBjJPbNOxAz\n",
315 | "s3pwcqhABK8D9+KmJTNrE04OlXOntJm1DSeHyrlT2szahpND5e4FZknUbQKemVlenBwqFMFG4M/A\n",
316 | "QXnHYmaWNSeH6rhpyczagpNDdTzfwczagpNDdXznYGZtwauyVvXZjCJ5MtzeEbySRwxmZkNR7Xfn\n",
317 | "sCyDaT3T3w+zX4Nn75K6n4X1l0WsuCHvqMzMas3JoULS9Dlw5KXwrdG8sXz33IOk6ThBmFmrcZ9D\n",
318 | "xSadBQum9C5bMAUOmJdPPGZm2ck0OUj6oaRuSQ8XlY2RtETSHyTdImmvomPzJa2WtErScVnGVr3R\n",
319 | "I8qXjxpZ3zjMzLKX9Z3Dj4Dj+5SdCyyJiIOBpek+kqYBpwLT0mu+K6mB7my2bC9fvtWPDjWzlpPp\n",
320 | "l29E3AFs7lN8IrAw3V4InJxunwRcExE7ImItsIaGGja6/jKYu6Z32ZlPw7rL84nHzCw7eXRIj42I\n",
321 | "7nS7Gxibbk8Afld03gZgYj0DG0jEihuk6cCceUlT0vA94NOT4Lu35x2bmVmt5TpaKSJC0kATLcoe\n",
322 | "k9RZtNsVEV21jKvfYJJRSW+MTJL4f8A3gLPq8flmZpWS1AF0DPX6PJJDt6RxEfGMpPHAprT8KWBS\n",
323 | "0Xn7p2UlIqIz2xAr9iXgYYlFEdyZdzBmZgXpH81dhX1JF1RzfR4dvouB09Pt04HrispPkzRc0mRg\n",
324 | "KslaRg0rgj8BXwCulPCoJTNrGZkunyHpGuADwD4k/QvnA9cDi4ADgLXAKRHxfHr+ecB/A14Fzo6I\n",
325 | "m8vUmdvyGf2R+HdgbQRfzTsWM7Nyqv3u9NpKNSCxH/AQ8NEI7s07HjOzvqr97mygeQTNK4JNwDnA\n",
326 | "DyWG5x2PmdnOcnKonZ8AjwPn5R2ImdnOcrNSDUlMAJYDx0TwUN7xmJkVuFkpRxE8TbIcyI8kr3hr\n",
327 | "Zs3LyaH2fgQ8B3w570DMzIbKzUoZkDgQuA94XwSr8o7HzMzNSg0ggieBTpLRS7vmHI6ZWdWcHLLz\n",
328 | "PZLJfF/MOxAzs2q5WSlDElOBu4BZETyedzxm1r7crNRAIlgNXAhcIdEUCc3MDJwc6uHbwB7A3LwD\n",
329 | "MTOrlJuV6kDibSRL574zgvU5h2NmbcjNSg0ogkeAS4Hvu3nJzJqBk0P9fBMYD3wy70DMzAbjZqU6\n",
330 | "kpgJS/8DLlsObxJs2Q7rL0sfP2pmlplqvzu9/k9dTR8PxwZc39FTNvcgaTpOEGbWSNysVFeTzoL/\n",
331 | "s3fvsgVT4IB5+cRjZlaek0NdjR5RvnyUnz9tZg3FyaGutmwvX/7nfsrNzPLh5FBX6y+DuWt6l335\n",
332 | "Jfj8WIm9y19jZlZ/Hq1UZ9L0OUkfw6iRsHUbbPwOLDsamAOcEMGaweowM6tWtd+dTg4NQuJzwNeA\n",
333 | "UyK4Pe94zKy1eIZ0k4rg+yQT5H4q8emcwzGzNpfbnYOktcCLwGvAjoiYJWkM8O/AgcBa4JSIeL7P\n",
334 | "dS1551AgcQjwS+CnwN9F8HrOIZlZC2imO4cAOiJiZkTMSsvOBZZExMHA0nS/raSPFT0COIrkLmKP\n",
335 | "nEMyszaUd7NS3yx2IrAw3V4InFzfcBpDBH8EjgW2AL+WmJBzSGbWZvK+c7hV0n2SCs86GBsR3el2\n",
336 | "NzA2n9DyF8HLwGeAnwF3S7wz55DMrI3kubbSURGxUdK+wBJJq4oPRkRIKtshIqmzaLcrIrqyCzM/\n",
337 | "EQRwkcRq4GaJuRFcl3dcZtb4JHUAHUO+vhGGskq6ANhK8rS0joh4RtJ44LaIOKTPuS3dId0fiXcB\n",
338 | "1wGXwvRHknWaRo/wyq5mVommWJVV0u7ArhGxRdIewHEkY/wXA6eTPPvgdPBfyQUR3CdxBNz0a/jw\n",
339 | "PnDxnj1HvbKrmdVWLncOkiYD16a7w4AfR8SF6VDWRcABtOlQ1sFIJyyBXx5TemTOTRE3zK5/RGbW\n",
340 | "DJriziEingBmlCn/E1Dmi8967L5b+XKv7GpmtZP3UFarWn8ru05+t8Q5XsDPzGrByaHplFvZ9YzH\n",
341 | "IM4HDgcel7gieSSpmdnQNMRopWq0e58DlFvZdd3lhc5oif2AzwKfB54CvgP8LJ03YWZtyquyGgAS\n",
342 | "w4CPAF8A3g5cCXw/gnXJ8elzPBzWrH04OVgJibcCfwN8CrgDLrgXnv508vzqgrlr4K6znSDMWpOT\n",
343 | "g/UrXcTvr+Ar/wzfGl16hofDmrWqZlqV1eosgj9H8AN48oHyZ4wdK7FrfaMys0bk5NCW+hsOu+8U\n",
344 | "oFvixxKfkBhT17DMrGE4ObSl/obD3nAayeTEXwOnAmsl7pA4V+IwqWeJdWn6HGn2TdIpXcn79Dn1\n",
345 | "/BeYWbbc59CmBhoO23MOI0hWdfwIcALJHxO/gn/shnWfgh8c1HO2O7TNGpk7pC0T6V3DocBH4Evz\n",
346 | "4ZIyM7HdoW3WqJpibSVrPumzJVYCK6UNHwE+UHrWu46R+E3hvPT1KLA+vb4Xz7Uwa1xODjYE/XVo\n",
347 | "P3g78HVgWvo6IX0fJfEovZLG6fvBkfP7zLWoeulxJxizbDg52BCsvwzmHtT7i/2Mx+CxiyP4NUmH\n",
348 | "9hvSxQAPpSdpHA3jPgDfHNG73gVT4NMXSrwMPA1sBF4od9eR1Dt9Dhx5qROMWe05OVjVIlbcIE0H\n",
349 | "5gzYod1zPpuB36YvAKQnuijbNPUX44C/Ayakr2EST0Ov18bk/bCzeycGSPbnzAMq+nKvRYJxcrFW\n",
350 | "5ORgQ5J++e3EF2B/TVOPPhDBG53aEqOA8fQki8JrJhx4cPk6Dj9a4nFgW/ra3v/7ez8K/3dy7+sX\n",
351 | "TIG//KrEncCW/u5ckvhqc/di1micHCwn/TVNrbu8+KwItgKr01cv0oM3AR8urXvFb4EzgBHAyPQ1\n",
352 | "os97ut3ff4FDZ5Hcpewm8Sz095r5Nzt792LWiJwcLBfVNk2V11+CWf2tCB6rpAbpseOByaVHHuiK\n",
353 | "YLbESGDffl6TYdyk8jXvvVfl/w6zxuPkYLnZ2aapbBNMcgcTwTZgXfoqIa3Yn7J3LwceLvEAcD2w\n",
354 | "GFg+UPOUWaPxJDhre5XMFh/42r59Dmc8BsvOgftfAE5KX8NJksRioCuCV2r/LzHrn2dIm9XZYMml\n",
355 | "aHb5ienrUOAWkruKGyPY7BFPljUnB7MGJzGOZL2qk4AO+OUTsGQCXLpPz1nVr1XlBGMDafrkIOl4\n",
356 | "4NvArsAVEfHNPsedHKxlSOwOn7oTrppZevRvu+Hb1wKbKD9a6o8RvJrUU3ZIbVUJxsmltTX12kqS\n",
357 | "dgX+BTgGeAq4V9LiiHg038hal6SOiOjKO45WMJSfZQQvSS+/WP7otueAFcB+JDPL+46WGiPxIvAs\n",
358 | "fHg/uLjPCKkFU+ATnRIbSBLMG8mkNPbGmW3eU8eL42DPZ5yk8tFQyQGYBayJiLUAkn5Ccuvt5JCd\n",
359 | "DqAr5xhaRQdD+ln2NyFw/boIvtPfVelT+8YA+8LmH5M8i6OPSVOBq0kSzBiJF0juOjb1fn/Pfyk/\n",
360 | "X+PkL0vcBbwCvNxfckniqdVs80IdnUDnO/JIUrVNcvnWMVSNlhwmAuuL9jcA78kpFrM6qWxCYF8R\n",
361 | "vEbaxCRt7C5/1sO/K8w4751M2K/3+55vLn/9Ye8FniAZbfUmJY0SrwAvp6+i7eP3h3/es/f1C6bA\n",
362 | "WQskfkHpDPUys9bf3lk+Sf3lV9PFG18DXk/f+77S8ncfC0deMtQkVfskl18dO6PRkkNjdYCY1UE9\n",
363 | "5mskn9OTTEhWx32DtOoIkmVJ+rh/aZ/lTIaRJor0Nbzn/Y8LgcNL63hlC/AgvWep70vprPURMGlK\n",
364 | "6fUAh8wCbiPpi9wlfe/7SsuP2w3+qU/b+oIp8L9+JfF64cdRdDB6v580rPz1f/+LdFHIKHNdn+2P\n",
365 | "7gEXDiut47zr0qbAcnX0ieuEveCiN5XWUZ/Z9w3VIS3pCKAzIo5P9+cDrxd3SktqnIDNzJpI045W\n",
366 | "kjQM+D3wIZJ1be4BPu4OaTOz+mqoZqWIeFXSF4GbSW4Tr3RiMDOrv4a6czAzs8awS94BVErS8ZJW\n",
367 | "SVot6at5x9PsJK2V9JCkZZLuyTueZiPph5K6JT1cVDZG0hJJf5B0iySvzFqhfn6enZI2pL+jy9IJ\n",
368 | "sjYISZMk3SbpEUkrJJ2Vllf1+9kUyaFoctzxJJOBPi7p0HyjanoBdETEzIiYlXcwTehHJL+Pxc4F\n",
369 | "lkTEwcDSdN8qU+7nGcAl6e/ozIi4KYe4mtEO4JyIeBtwBPCF9Puyqt/PpkgOFE2Oi4gdQGFynO0c\n",
370 | "L0MyRBFxB7C5T/GJwMJ0eyFwcl2DamL9/DzBv6NVi4hnImJ5ur2VZBLxRKr8/WyW5FBuctzEnGJp\n",
371 | "FQHcKuk+SXPzDqZFjI2IwmS0bmBsnsG0iHmSHpR0pZvpqifpLcBM4G6q/P1sluTgXvPaOyoiZgKz\n",
372 | "SW4735d3QK0kkpEe/r3dOd8jeUrfDGAjcHG+4TQXSaOAnwNnR8SW4mOV/H42S3J4Cih+HOMkkrsH\n",
373 | "G6KI2Ji+PwtcS9J0ZzunW9I4AEnjSdYtsiGKiE2RAq7Av6MVk7QbSWK4KiKuS4ur+v1sluRwHzBV\n",
374 | "0lskDQdOJXmilg2BpN0ljU639wCOAx4e+CqrwGLg9HT7dOC6Ac61QaRfYAUfw7+jFZEk4EpgZUR8\n",
375 | "u+hQVb+fTTPPQdJsep7zcGVEXJhzSE1L0mSSuwVIJkL+2D/P6ki6BvgAsA9J++35JE92WwQcAKwF\n",
376 | "TomI5/OKsZmU+XleQLLK7QyS5o8ngM8VtZlbPyS9F7gdeIiepqP5JCtOVPz72TTJwczM6qdZmpXM\n",
377 | "zKyOnBzMzKyEk4OZmZVwcjAzsxJODmZmVsLJwczMSjg5WFuRtDV9P1DSx2tc93l99n9Ty/rN6snJ\n",
378 | "wdpNYWLPZOAT1VyYPsZ2IPN7fVDEUdXUb9ZInBysXV0EvC99iMzZknaR9C1J96SrgP53AEkdku6Q\n",
379 | "dD2wIi27Ll3NdkVhRVtJFwEj0/quSssKdylK6344fcDSKUV1d0n6qaRHJf1bIThJF6UPa3lQ0rfq\n",
380 | "+pMxo8GeIW1WR18FvhwRHwVIk8HzETFL0puAOyXdkp47E3hbRDyZ7n8mIjZLGgncI+lnEXGupC+k\n",
381 | "K90WFO5S/jPwDuDtwL7AvZJuT4/NIHmA1UbgN5KOAlYBJ0fEIWlse2bw7zcbkO8crF31fYjMccBf\n",
382 | "S1oG/A4YA0xJj91TlBgAzpa0HLiLZIXgqYN81nuBq9MFRjcBvwbeTZI87omIp9OVR5cDBwLPA9vT\n",
383 | "Zxh8DNg25H+l2RA5OZj1+GLRIykPiohb0/I/F06Q1AF8CDgiImYAy4ARg9QblCajwl3Fy0VlrwG7\n",
384 | "RcRrJMtT/ww4AfDjMa3unBysXW0BRhft3wycWeh0lnSwpN3LXLcnsDkitks6hOQZvQU7+um0vgM4\n",
385 | "Ne3X2Bd4P8kKmWUfgZkuo75XRNwIfImkScqsrtznYO2m8Bf7g8BrafPQj4DLgLcAD6Tr4W8ieYZA\n",
386 | "3ydm3QR6/VU+AAAAcklEQVR8XtJK4PckTUsFPwAeknR/RHyqcF1EXCvpyPQzA/hKRGxKH/red1nk\n",
387 | "IEla10saQZJAzqnJv9ysCl6y28zMSrhZyczMSjg5mJlZCScHMzMr4eRgZmYlnBzMzKyEk4OZmZVw\n",
388 | "cjAzsxJODmZmVuL/A/+0Tfs1CW9cAAAAAElFTkSuQmCC\n"
389 | ],
390 | "text/plain": [
391 | ""
392 | ]
393 | },
394 | "metadata": {},
395 | "output_type": "display_data"
396 | }
397 | ],
398 | "source": [
399 | "plt.plot(J_history, marker='o')\n",
400 | "plt.xlabel('Iterations')\n",
401 | "plt.ylabel('J(theta)')"
402 | ]
403 | },
404 | {
405 | "cell_type": "code",
406 | "execution_count": 29,
407 | "metadata": {
408 | "collapsed": false
409 | },
410 | "outputs": [],
411 | "source": [
412 | "def accuracy(theta, X, y):\n",
413 | " correct = np.sum(np.equal(y, (sigmoid(X.dot(theta))) > 0.5))\n",
414 | " return correct / y.size"
415 | ]
416 | },
417 | {
418 | "cell_type": "code",
419 | "execution_count": 30,
420 | "metadata": {
421 | "collapsed": false
422 | },
423 | "outputs": [
424 | {
425 | "name": "stdout",
426 | "output_type": "stream",
427 | "text": [
428 | "Training accuracy: 1.0\n",
429 | "Test accuracy: 0.9995271867612293\n"
430 | ]
431 | }
432 | ],
433 | "source": [
434 | "print('Training accuracy: {acc}'.format(acc=accuracy(res.x, train_X, train_y)))\n",
435 | "print('Test accuracy: {acc}'.format(acc=accuracy(res.x, test_X, test_y)))"
436 | ]
437 | },
438 | {
439 | "cell_type": "markdown",
440 | "metadata": {},
441 | "source": [
442 | "### Looking good, right? Well, look closer...\n",
443 | "I actually had to use the **`L-BFGS-B`** optimization method for it to work.
\n",
444 | "Had I used the expected **`BFGS`** method, **nan** and **inf** values due to `log(0)` would have made trouble.
\n",
445 | "Why? I can think of two reasons:\n",
446 | "1. Even if being multiplied with `0`, the `log(0)` expression is still evaluated by numpy. And unfortunately: `0 * np.nan = np.nan`.\n",
447 | "2. Floating point arithmetic limits which don't exist in Mathematics.\n",
448 | "\n",
449 | "One way to counteract those issues is to substitute the troubling values:"
450 | ]
451 | },
452 | {
453 | "cell_type": "code",
454 | "execution_count": 31,
455 | "metadata": {
456 | "collapsed": false
457 | },
458 | "outputs": [],
459 | "source": [
460 | "def safe_log(x, nan_substitute=-1e+4):\n",
461 | " l = np.log(x)\n",
462 | " l[np.logical_or(np.isnan(l), np.isinf(l))] = nan_substitute\n",
463 | " return l"
464 | ]
465 | },
466 | {
467 | "cell_type": "code",
468 | "execution_count": 32,
469 | "metadata": {
470 | "collapsed": false
471 | },
472 | "outputs": [],
473 | "source": [
474 | "def cost_function_safe(theta, X, y):\n",
475 | " h = sigmoid(X.dot(theta))\n",
476 | " return -sum(y * safe_log(h) + (1 - y) * safe_log(1 - h))"
477 | ]
478 | },
479 | {
480 | "cell_type": "code",
481 | "execution_count": 33,
482 | "metadata": {
483 | "collapsed": false
484 | },
485 | "outputs": [
486 | {
487 | "name": "stdout",
488 | "output_type": "stream",
489 | "text": [
490 | "Warning: Maximum number of iterations has been exceeded.\n",
491 | " Current function value: 0.001535\n",
492 | " Iterations: 100\n",
493 | " Function evaluations: 114\n",
494 | " Gradient evaluations: 114\n",
495 | "Optimization took 5.0561230182647705 seconds\n"
496 | ]
497 | },
498 | {
499 | "name": "stderr",
500 | "output_type": "stream",
501 | "text": [
502 | "/Users/fhartl/anaconda/envs/py34/lib/python3.4/site-packages/IPython/kernel/__main__.py:2: RuntimeWarning: overflow encountered in exp\n",
503 | " from IPython.kernel.zmq import kernelapp as app\n",
504 | "/Users/fhartl/anaconda/envs/py34/lib/python3.4/site-packages/IPython/kernel/__main__.py:2: RuntimeWarning: divide by zero encountered in log\n",
505 | " from IPython.kernel.zmq import kernelapp as app\n"
506 | ]
507 | }
508 | ],
509 | "source": [
510 | "J_history = []\n",
511 | "\n",
512 | "t0 = time.time()\n",
513 | "res = scipy.optimize.minimize(\n",
514 | " fun=cost_function_safe,\n",
515 | " x0=np.random.rand(train_X.shape[1]) * 0.001,\n",
516 | " args=(train_X, train_y),\n",
517 | " method='BFGS',\n",
518 | " jac=gradient,\n",
519 | " options={'maxiter': 100, 'disp': True},\n",
520 | " callback=lambda x: J_history.append(cost_function_safe(x, train_X, train_y)),\n",
521 | ")\n",
522 | "t1 = time.time()\n",
523 | "\n",
524 | "print('Optimization took {s} seconds'.format(s=t1 - t0))\n",
525 | "optimal_theta = res.x"
526 | ]
527 | },
528 | {
529 | "cell_type": "markdown",
530 | "metadata": {},
531 | "source": [
532 | "
\n",
533 | "Notice that the above optimization procedure doesn't converge due to the substitutions (which doesn't allow the gradients to further improve (= get smaller) at some point). Therefore, it used all 100 allowed iterations."
534 | ]
535 | },
536 | {
537 | "cell_type": "code",
538 | "execution_count": 34,
539 | "metadata": {
540 | "collapsed": false
541 | },
542 | "outputs": [
543 | {
544 | "data": {
545 | "text/plain": [
546 | ""
547 | ]
548 | },
549 | "execution_count": 34,
550 | "metadata": {},
551 | "output_type": "execute_result"
552 | },
553 | {
554 | "data": {
555 | "image/png": [
556 | "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEPCAYAAABcA4N7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
557 | "AAALEgAACxIB0t1+/AAAH7lJREFUeJzt3Xu4XXVh5vHvSzACBo2p9iQkoUSSCCG0opVLBYkWQ5LB\n",
558 | "gNYJ2KedFCHzzIMKOq1jwnSatM8UwV68MMPMSMFJmZJOaitGhZDIeIq1YryAXGIkoaYQNCdeUIkC\n",
559 | "JvDOH2tts3PYZ5+Tk7PXPnvv9/M85znrttf67ZUn6z2/y1pLtomIiBjKEe0uQEREjG8JioiIaCpB\n",
560 | "ERERTSUoIiKiqQRFREQ0laCIiIimWhoUkq6S9ICkByVdVS6bImmzpIclbZI0uW77VZK2S9omaWEr\n",
561 | "yxYRESPTsqCQNB+4HHgt8GvABZJOBFYCm23PBe4q55E0D7gYmAcsAm6QlBpPRESbtfJCfBLwZdtP\n",
562 | "234W+Efgt4ClwNpym7XAReX0hcA62/ts7wR2AKe3sHwRETECrQyKB4FzyqamY4AlwAygz/ZAuc0A\n",
563 | "0FdOHwfsqvv8LmB6C8sXEREjcGSrdmx7m6TrgE3AT4H7gGcHbWNJzZ4hkueLRES0WcuCAsD2zcDN\n",
564 | "AJL+lKKWMCBpqu3dkqYBe8rNHwdm1n18RrnsIMMES0REDMG2RvO5lgaFpF+2vUfS8cBbgTOBWcBy\n",
565 | "4Lry923l5huAWyX9JUWT0xxgS6P9jvbLdhtJa2yvaXc5xoOciwNyLg7IuTjgcP7IbmlQAJ+Q9EvA\n",
566 | "PuAK2z+WdC2wXtJlwE5gGYDtrZLWA1uB/eX2qT1ERLRZq5ueXt9g2Q+B84bY/hrgmlaWKSIiDk3u\n",
567 | "U+hs/e0uwDjS3+4CjCP97S7AONLf7gJ0A3Va644kp48iIuLQHM61MzWKiIhoKkERERFNJSgiIqKp\n",
568 | "BEVERDSVoIiIiKYSFBER0VSCIiIimkpQREREUwmKiIhoKkERERFNdWRQSIs3SvOXtLscERG9oCOD\n",
569 | "Au44H876SMIiIqL1OjQoAG6cDce/u92liIjodh0cFACTjm53CSIiul1Lg0LSKkkPSXpA0q2SXihp\n",
570 | "iqTNkh6WtEnS5EHbb5e0TdLC4Y+w96lWlj8iIloYFJJOAFYAr7Z9KjABuARYCWy2PRe4q5xH0jzg\n",
571 | "YmAesAi4QVKT8l3+CDx6favKHxERhVbWKH5C8a7sYyQdCRwDfAdYCqwtt1kLXFROXwiss73P9k5g\n",
572 | "B3B6410v2Qj3XGk/eHvLSh8REUAL35lt+4eS/gJ4FHgKuNP2Zkl9tgfKzQaAvnL6OOCeul3sAqY3\n",
573 | "3vfti1tU7IiIGKRlQSHpROA9wAnAj4G/k/Q79dvYtqRm72JtuE7SmrrZftv9h1XYiIguI2kBsGAs\n",
574 | "9tWyoAB+Hfhn2z8AkPQPwFnAbklTbe+WNA3YU27/ODCz7vMzymXPY3tNy0odEdEFyj+g+2vzklaP\n",
575 | "dl+t7KPYBpwp6WhJAs4DtgKfBpaX2ywHbiunNwCXSJooaRYwB9jSwvJFRMQItLKP4huS/hr4KvAc\n",
576 | "8HXgY8CxwHpJlwE7gWXl9lslracIk/3AFbabNUtFREQF1GnXYkm2rXaXIyKikxzOtbPD78yOiIhW\n",
577 | "S1BERERTCYqIiGgqQREREU0lKCIioqmODAqppTcKRkREnY4MCmBiuwsQEdErOjUoXtjuAkRE9IpO\n",
578 | "DYrUKCIiKpKgiIiIpjo1KNL0FBFRkU4NitQoIiIq0qlBkRpFRERFOjUoUqOIiKhIpwZFahQRERXp\n",
579 | "1KBIjSIioiItDQpJr5R0b93PjyVdKWmKpM2SHpa0SdLkus+skrRd0jZJC4fYdYIiIqIiLQ0K29+y\n",
580 | "fZrt04DXAD8DPgmsBDbbngvcVc4jaR5wMTAPWATcIKlRGdP0FBFRkSqbns4Ddth+DFgKrC2XrwUu\n",
581 | "KqcvBNbZ3md7J7ADOL3BvlKjiIioSJVBcQmwrpzusz1QTg8AfeX0ccCuus/sAqY32FdqFBERFank\n",
582 | "cd2SJgJvBt4/eJ1tS3KTjzdYt+ht0p2vKGf6bfePQTEjIrqGpAXAgrHYV1XvdVgMfM3298r5AUlT\n",
583 | "be+WNA3YUy5/HJhZ97kZ5bJBNt5p899bWN6IiI5W/gHdX5uXtHq0+6qq6entHGh2AtgALC+nlwO3\n",
584 | "1S2/RNJESbOAOcCWBvtLH0VEREVaXqOQ9CKKjuwVdYuvBdZLugzYCSwDsL1V0npgK7AfuMJ2o2ap\n",
585 | "BEVEREXU+Do8fhX9GV5t8yftLktERKeQZNsazWdzZ3ZERDTVqUGR4bERERXp1KBIjSIioiIJioiI\n",
586 | "aKpTgyJNTxERFenUoEiNIiKiIp0aFKlRRERUpFODIjWKiIiKdGpQpEYREVGRTg2K1CgiIiqSoIiI\n",
587 | "iKY6NSjS9BQRUZFODYrUKCIiKtKpQZEaRURERTo1KFKjiIioSKcGRWoUEREVaXlQSJos6ROSvilp\n",
588 | "q6QzJE2RtFnSw5I2SZpct/0qSdslbZO0cIjdpkYREVGRKmoUHwFut30y8KvANmAlsNn2XOCuch5J\n",
589 | "84CLgXnAIuAGSY3KmKCIiKhIS4NC0kuAc2zfDGB7v+0fA0uBteVma4GLyukLgXW299neCewATm+w\n",
590 | "6zQ9RURUpNU1ilnA9yR9XNLXJd0o6UVAn+2BcpsBoK+cPg7YVff5XcD0BvudKDGqd79GRMShObKC\n",
591 | "/b8aeJftr0j6MGUzU41tS3KTfTRYtxq45o+l/c8B/bb7x6zEERFdQNICYMGY7Mtudo0+zJ1LU4Ev\n",
592 | "2Z5Vzp8NrAJeAbzB9m5J04DP2z5J0koA29eW228EVtv+ct0+DX4KeLnNT1tW+IiILiLJtkfVEtPS\n",
593 | "pifbu4HHJM0tF50HPAR8GlheLlsO3FZObwAukTRR0ixgDrClwa6fIR3aERGVaHXTE8C7gb+RNBF4\n",
594 | "BLgUmACsl3QZsBNYBmB7q6T1wFZgP3CFG1d5fk46tCMiKtHSpqdWKJueHgPOtnm03eWJiOgE47bp\n",
595 | "qYVSo4iIqEinBkX6KCIiKtKpQZEaRURERTo5KFKjiIioQKcGRZqeIiIq0qlBkaaniIiKdGpQpEYR\n",
596 | "EVGRTg2K1CgiIirSyUGRGkVERAU6NSieITWKiIhKdGpQpEYREVGRTg2KdGZHRFSkU4MindkRERXp\n",
597 | "1KBIjSIioiKdGhSpUUREVGREQSHpRZJOkvRKSS86lANI2inpfkn3StpSLpsiabOkhyVtkjS5bvtV\n",
598 | "krZL2iZp4RC7TWd2RERFhnzDnaRjgRXAJcDLgAFAQJ+kHwB/A9xoe+8wxzCwwPYP65atBDbb/qCk\n",
599 | "95fzKyXNAy4G5gHTgc9Jmmv7uUH7fAZ4yUi/ZEREjF6zGsVtwJPAm22/wvZZts+0PQu4APgp8KkR\n",
600 | "HmfwW5WWAmvL6bXAReX0hcA62/ts7wR2AKc32F+aniIiKjJkjcL2bzZZtxv4WPkzHFPUDJ4F/pft\n",
601 | "G4E+2wPl+gGgr5w+Drin7rO7KGoWg6UzOyKiIkMGRT1JU4A51P0Vb/vuER7jdba/K+nlwGZJ2+pX\n",
602 | "2nbxHuwhNVqXGkVEREWGDQpJK4ArgRnAfcCZwJeAN47kALa/W/7+nqRPUjQlDUiaanu3pGnAnnLz\n",
603 | "x4GZdR+fUS4b5LwL4ITZ0k1rgH7b/SMpS0REr5C0AFgwJvuym/0xD5IeBF4LfMn2qySdBHzA9luG\n",
604 | "3bl0DDDB9pPlaKlNwB8D5wE/sH2dpJXAZNu1zuxbKcJkOvA5YLbrClnUPrwMuNjmbaP50hERvUaS\n",
605 | "bQ/uLx6RkTQ9PW37KUlIOsr2NkmvHOH++4BPSqod629sb5L0VWC9pMuAncAyANtbJa0HtgL7gSvc\n",
606 | "OMkyPDYioiIjCYpdkl5KMQpqs6QnKC7uw7L9beBVDZb/kKJW0egz1wDXDLPrdGZHRFRk2KCwXRu6\n",
607 | "ukZSP/BiYGMrCzUC6cyOiKjIsHdmS7qlNm273/YG4KaWlmp4qVFERFRkJI/wmF8/I+lI4DWtKc6I\n",
608 | "pUYREVGRIYNC0tWSngROlfRk7YdiKOuGykrYWDqzIyIqMpLhsdfaXllReYZVDo89GbjN5qR2lyci\n",
609 | "ohMczvDYkTQ9XS3pdyX9UXmw4yU1ev5SldL0FBFRkZEExQ3AWcBvl/N7y2XtlM7siIiKjOQ+ijNs\n",
610 | "nybpXijugZD0ghaXazipUUREVGQkNYqfS5pQmykf7jf4/RBVS2d2RERFRhIU1wOfBH5Z0jXAF4EP\n",
611 | "tLRUw3uG1CgiIiox7KgnAEknA7X3U9xl+5stLVXzshg8AXgWOMJu+BjyiIioczijnkYaFBOAqRR9\n",
612 | "Ggaw/ehoDni4al9WYh9wjM2+dpQjIqKTtPTpsZLeDaymuNHu2bpVp47mgGOo1qGdoIiIaKGRjHp6\n",
613 | "D/BK2z9odWEOUYbIRkRUYCSd2Y8CP2l1QUYhQ2QjIiowZI1C0u+Xk/8C9Ev6DMXFGYpXXf9lqws3\n",
614 | "jAyRjYioQLMaxbHAJIoaxWaKi/Kk8ufYkR5A0gRJ90r6dDk/RdJmSQ9L2iRpct22qyRtl7RN0sJh\n",
615 | "dp0hshERFRiyRmF7DYCkZbbX16+TtOwQjnEVxatNa+GyEths+4OS3l/O196XfTEwj/J92ZLm2h7q\n",
616 | "5r7UKCIiKjCSPopVI1z2PJJmAEuAvwJqw7KWAmvL6bVA7Q16FwLrbO+zvRPYATR7+GA6syMiKtCs\n",
617 | "j2IxxUV+uqSPcuBCfywjH5L6IeB9FK9PremzPVBODwB95fRxwD112+2iqFkMJZ3ZEREVaFaj+A7w\n",
618 | "NeDp8vfXgK9SvLTo/OF2LOkCYI/tezkQMgdxcbdfszv+mq1LjSIiogLN+ii+AXxD0q22fz7Udk38\n",
619 | "BrBU0hLgKODF5fu3ByRNtb1b0jSKG/kAHgdm1n1+RrnseSStgXfOgK9cLm2R7f5RlC8iomtJWgAs\n",
620 | "GJN9DfUID0mfBf438FnbPxu07kXABcBy20uGPYh0LvAHtt8s6YPAD2xfJ2klMNl2rTP7Vop+ienA\n",
621 | "54DZHlTAukd4fBb4HzafOcTvHBHRc1r1CI9LgXcBfyzpWeC7FE1ItWc+/V9g+SEcq3bBvxZYL+ky\n",
622 | "YCewDMD2VknrKUZI7QeuGBwSg6TpKSKiAiN9KGAf8Cvl7L/WdUZXrq5G8bfAp2zWtassERGdoiU1\n",
623 | "Ckl7GaIzWdIzFMNX/9D250Zz4DGQGkVERAWadWZPGmqdpCOBUyj6FE5pQblGIsNjIyIqMJIb7p7H\n",
624 | "9v5yVNT1Y1yeQ5E7syMiKjCqoKix/T/HqiCjkGc9RURU4LCCos1So4iIqEAnB0U6syMiKtDJQZHO\n",
625 | "7IiICnRyUKRGERFRgU4OitQoIiIq0OlBkRpFRESLdXJQZHhsREQFOjkoUqOIiKhAJwdFOrMjIirQ\n",
626 | "yUGRzuyIiAp0elCkRhER0WKdHBTpzI6IqEDLgkLSUZK+LOk+SVslfaBcPkXSZkkPS9okaXLdZ1ZJ\n",
627 | "2i5pm6SFwxwiNYqIiAq0LChsPw28wfargF8F3iDpbGAlsNn2XOCucp7yndkXA/OARcANkpqVL53Z\n",
628 | "EREVaGnTk+2flZMTgQnAE8BSYG25fC1wUTl9IbDO9j7bOyneoHd6k92nMzsiogItDQpJR0i6DxgA\n",
629 | "Pm/7IaCv7p3bA0BfOX0csKvu47uA6U12nxpFREQFhnwV6liw/RzwKkkvAe6U9IZB6y2p4Xu5a5s0\n",
630 | "WihpDcycDL/dJ123wHb/mBU6IqILSFoALBiLfbU0KGps/1jSZ4HXAAOSptreLWkasKfc7HFgZt3H\n",
631 | "ZpTLGu1vjcR0YJl9bX8Lix4R0ZHKP6D7a/OSVo92X60c9fSy2ogmSUcDbwLuBTYAy8vNlgO3ldMb\n",
632 | "gEskTZQ0C5gDbGm87/lLYOkt8F9eLi3eWMxHREQrtLJGMQ1YW45cOgK4xfZdku4F1ku6DNgJLAOw\n",
633 | "vVXSemArsB+4wvYQzVJnfQRunF3OnA8rTpTmYz94ewu/T0RET9KQ1+JxqujTaFTmJRvt2xdXXqCI\n",
634 | "iA4gybY1ms928p3Zg0w6ut0liIjoRl0UFHufancJIiK6UYcGxYodB89f/gg8en17yhIR0d06tI/i\n",
635 | "lH8Dx78b5syDp5+BL74nHdkREUM7nD6KjgyK2peVeAfwepvfa2+pIiLGt17uzH4EmD3sVhERMWqd\n",
636 | "HhQ7gBPbXYiIiG7W6UHxXeAlEpPaXZCIiG7V0UFh8xzwL8Ar2l2WiIhu1dFBUUo/RUREC3VDUKSf\n",
637 | "IiKihbohKFKjiIhooW4IitQoIiJaqBuCIjWKiIgW6ug7s4t5jgT2Ai+xeaZ9JYuIGL96+c5sbPYD\n",
638 | "u4AT2lyUiIiu1NKgkDRT0uclPSTpQUlXlsunSNos6WFJm2qvTC3XrZK0XdI2SQtHeKj0U0REtEir\n",
639 | "axT7gPfaPgU4E3inpJOBlcBm23OBu8p5JM0DLgbmAYuAG8pXqQ4n/RQRES3S0qCwvdv2feX0XuCb\n",
640 | "wHRgKbC23GwtcFE5fSGwzvY+2zspagqnj+BQj5AaRURES1TWRyHpBOA04MtAn+2BctUA0FdOH0fR\n",
641 | "31CziyJYhrOD1CgiIlriyCoOImkS8PfAVbaflA50vNt28TKiIT1vnaQ1dbP94NQoIiLqSFoALBiL\n",
642 | "fbU8KCS9gCIkbrF9W7l4QNJU27slTQP2lMsfB2bWfXxGuewgttccfIyzL4Jz50jb++HJp+Gxj+aN\n",
643 | "dxHRy2z3A/21eUmrR7uvVo96EnATsNX2h+tWbQCWl9PLgdvqll8iaaKkWcAcYEvzY8xfAif/Gfzp\n",
644 | "EbD+XLjjfDjrI8XyiIg4XC294U7S2cDdwP0caEJaRXHxXw8cD+wEltn+UfmZq4F3APspmqruHLTP\n",
645 | "QTfcLd5YhMNgSzbaty8e468UEdGRDueGu5Y2Pdn+J4autZw3xGeuAa4Z+VGOParx8klHj3wfEREx\n",
646 | "lI6/M7vok2hk71PVliMiojt1QVA89lFYsePgZZc/Ao9e357yRER0l45/KGCxbP4SOP7d8Jrz4IEv\n",
647 | "wI4/z6iniIgDDqePoiuC4sA6bgc+Zv9iFFVERNDjT48d5MsUz5SKiIgx0m1BcQ8JioiIMdVtTU8v\n",
648 | "BR4FXlq+pyIiIkjT0y/YPEHxIMFT2l2WiIhu0VVBUUo/RUTEGOrGoEg/RUTEGOrWoDij3YWIiOgW\n",
649 | "lbyPolqvmQnnz5W+/U/wo7155HhExOHpqqAo7tA+6y/hmgnA64qlK06U5pOwiIgYnS5repp5Jdw4\n",
650 | "6JWoN84uHu8RERGj0WVBkUeOR0SMtS4LijxyPCJirLX6Vag3SxqQ9EDdsimSNkt6WNImSZPr1q2S\n",
651 | "tF3SNkkLD/2IjR45viKPHI+IOAytfhXqOcBe4K9tn1ou+yDwfdsflPR+4KW2V0qaB9wKvBaYDnwO\n",
652 | "mGv7uUH7bHob+oFHjk86GmbMh1ess9+VPoqI6Gnj+jHjkk4APl0XFNuAc20PSJoK9Ns+SdIq4Dnb\n",
653 | "15XbbQTW2L5n0P5G/GUlXgN3bYIPfR2OeUHRNJXhshHRe8btO7OH0Gd7oJweAPrK6eMobpar2UVR\n",
654 | "szgM8/tg0UT4TN37uTNcNiLiULT1PgrbltSsStNwnaQ1dbP9tvsbf3zmlfDnkw5eduNsWPJuIEER\n",
655 | "EV1L0gJgwVjsqx1BMSBpqu3dkqYBe8rljwMz67abUS57HttrRnaoDJeNiN5U/gHdX5uXtHq0+2rH\n",
656 | "8NgNwPJyejn84rWlG4BLJE2UNAuYA2w5vENluGxExOFq9fDYdcA/A6+U9JikS4FrgTdJehh4YzmP\n",
657 | "7a3AemArcAdwhQ+7p73RcNnLM1w2IuIQdNUb7hpvXxsuO20avGw23HGxff9nW1nGiIjxZlwPjx1r\n",
658 | "o/2yEoI7vgW37oVnfpKhshHRSzpteGybzF8MbzgWbplzYFmGykZEDKfLnvXUzMwr4fqpB+bvBvpm\n",
659 | "wytvkRZvLJqoIiJisB6qUdQPlb0buBP4U4ApwPmpXURENNZDNYr6obKbKEOiTt5bERHRSA8FRf1Q\n",
660 | "2UYVqbuBI86QlvWnKSoi4oCeGfVUfLY2VPaIM+AzLz2w5qCmqHL+L34GEx6Bp76T0VER0ekyPPaQ\n",
661 | "9zF/CZz1kQOvTf1D4L+WaxMaEdF9Mjz2ENkP3i7Np3g44KSj4dlTKTq1Obj/ohYanzoGOLX4Sad3\n",
662 | "RPSWngwKoHahvx1AWrwROL9YU39KGnV6/+5s2LVWWvZQbtqLiF7Qs0FxsMc+CitOLJqi9tctH3x6\n",
663 | "ajWMO14GnFs2S50jvTXNUhHRtRIUDG6K2j8d3nEi3HzMwaEBaZaKiF7Uk53Zwx+jNjpq/3SYUYYG\n",
664 | "wJryBw7uAIciODYB3/wh/Owr8NiXYOZZxY1+aaKKiPbKqKeWHq8WGpOOhidPKZudODg0Go2UumUf\n",
665 | "3PiCA/MZORUR7ZOgqOzY9cNq62sUg2sXhzTcNjWPiGi5rhoeK2kR8GFgAvBXtq9rc5F+Yei+jMGn\n",
666 | "caiRU4P7Ne4+FW5546CaR9k5/v2fw1GCyU+WAZJAiYi2GFdBIWkC8N+A8yjel/0VSRtsf7O9JTvg\n",
667 | "4GG185cUoXHEGUDdnd5DjZwaPNx2EweHRC1E7j61QVNWg0D5jd1w5BNDB8ruSQfW1U83264zQ0jS\n",
668 | "gvIdwT0v5+KAnIuxMa6CAjgd2GF7J4CkvwUuBMZNUNSrhUYRGCvq7vReCKwo+yiaDbcdKkRGGiiX\n",
669 | "vgKOo3Gg1Dd5NexDabBdzVtPlRbufn7AjDRs2rHd7F+SXv/99pQv52L8nrOci9offyO+sDVie9z8\n",
670 | "AG8Dbqyb/x3g+kHbuN3lbFz2U5bA4jvg3/YXv09ZXfx+0/1w6U/Bhv/s4nftp35+9RDTg+frP3NO\n",
671 | "k/0NNT3cun80XD3MdG3+8p+Pn+1+r03HzbkY3+es189F7efy7Ydz7RzVh1oYFL/VqUExshCpD43B\n",
672 | "/6AjvbCvrps+d9B2q0cwPdy6ww2bdm031qF5KOGaczF+z1mvn4v6H0Z97RxXo54knQmssb2onF8F\n",
673 | "POe6Dm1J46fAEREdxN0wPFbSkcC3gN8EvgNsAd7ucdSZHRHRa8ZVZ7bt/ZLeRdG7OgG4KSEREdFe\n",
674 | "46pGERER409HvQpV0iJJ2yRtl/T+dpenKpJmSvq8pIckPSjpynL5FEmbJT0saZOkye0ua1UkTZB0\n",
675 | "r6RPl/M9eS4kTZb0CUnflLRV0hk9fC5Wlf9HHpB0q6QX9sq5kHSzpAFJD9QtG/K7l+dqe3k9XTjc\n",
676 | "/jsmKOpuxlsEzAPeLunk9paqMvuA99o+BTgTeGf53VcCm23PBe4q53vFVcBWoFYl7tVz8RHgdtsn\n",
677 | "A78KbKMHz4WkE4AVwKttn0rRdH0JvXMuPk5xbazX8LtLmgdcTHEdXQTcIKlpFnRMUFB3M57tfUDt\n",
678 | "ZryuZ3u37fvK6b0UNyBOB5YCa8vN1gIXtaeE1ZI0A1gC/BVQG8XRc+dC0kuAc2zfDEUfn+0f04Pn\n",
679 | "AvgJxR9Ux5SDYo6hGBDTE+fC9heAJwYtHuq7Xwiss73Pxc3NOyiur0PqpKCYDjxWN7+rXNZTyr+c\n",
680 | "TgO+DPTZHihXDQB9bSpW1T4EvA94rm5ZL56LWcD3JH1c0tcl3SjpRfTgubD9Q+AvgEcpAuJHtjfT\n",
681 | "g+eizlDf/TiK62fNsNfSTgqKnu91lzQJ+HvgKttP1q9zMSqh68+RpAuAPbbv5UBt4iC9ci4oRi2+\n",
682 | "GrjB9quBnzKoaaVXzoWkE4H3ACdQXAgnSfqd+m165Vw0MoLv3vS8dFJQPA7MrJufycGp2NUkvYAi\n",
683 | "JG6xfVu5eEDS1HL9NGBPu8pXod8Alkr6NrAOeKOkW+jNc7EL2GX7K+X8JyiCY3cPnotfB/7Z9g9s\n",
684 | "7wf+ATiL3jwXNUP9nxh8LZ1RLhtSJwXFV4E5kk6QNJGiM2ZDm8tUCUkCbgK22v5w3aoNwPJyejlw\n",
685 | "2+DPdhvbV9ueaXsWRWfl/7P9u/TmudgNPCZpbrnoPOAh4NP02Lmg6MQ/U9LR5f+X8ygGO/TiuagZ\n",
686 | "6v/EBuASSRMlzQLmUNzcPKSOuo9C0mIOvKviJtsfaHORKiHpbIrHvN7PgSriKop/3PXA8cBOYJnt\n",
687 | "H7WjjO0g6Vzg920vlTSFHjwXkn6NolN/IvAIcCnF/49ePBf/ieKC+BzwdeBy4Fh64FxIWgecC7yM\n",
688 | "oj/ij4BPMcR3l3Q18A6Kx1tfZfvOpvvvpKCIiIjqdVLTU0REtEGCIiIimkpQREREUwmKiIhoKkER\n",
689 | "ERFNJSgiIqKpBEX0HEl7y9+/IuntY7zvqwfNf3Es9x/RDgmK6EW1m4dmAb99KB8sn0zazKqDDmS/\n",
690 | "7lD2HzEeJSiil10LnFO+AOkqSUdI+jNJWyR9Q9K/B5C0QNIXJH0KeLBcdpukr5YvklpRLrsWOLrc\n",
691 | "3y3lslrtReW+H5B0v6Rldfvul/R35cuH/k+tcJKuLV/E8w1Jf1bpmYmoM67emR1RsfcDf2D7zQBl\n",
692 | "MPzI9umSXgj8k6RN5banAafY/tdy/lLbT0g6Gtgi6RO2V0p6p+3T6o5Rq728Ffg1ipcLvRz4iqS7\n",
693 | "y3WvoniJzHeBL0p6HcWziy6yfVJZthe34PtHjEhqFNHLBj+mfCHw7yTdC9wDTAFml+u21IUEwFWS\n",
694 | "7gO+RPEkzjnDHOts4FYX9gD/CLyWIki22P5O+Sjo+4BfAX4EPC3pJklvAZ4a9beMOEwJioiDvcv2\n",
695 | "aeXPibY/Vy7/aW0DSQuA3wTOtP0q4F7gqGH2a54fTLXaxjN1y54FXmD7WYq3jn0CuADYOJovEzEW\n",
696 | "EhTRy56keLpozZ3AFbUOa0lzJR3T4HMvBp6w/bSkkyjeY16zb4gO7y8AF5f9IC8HXk/x9N+GL18q\n",
697 | "31Q32fYdwH+kaLaKaIv0UUQvqv0l/w3g2bIJ6ePARynekPb18p0Ge4C3lNvXP2Z5I/AfJG0FvkXR\n",
698 | "/FTzMeB+SV8r35NhANuflHRWeUwD77O9R9LJPP/tYqYIsE9JOooiTN47Jt88YhTymPGIiGgqTU8R\n",
699 | "EdFUgiIiIppKUERERFMJioiIaCpBERERTSUoIiKiqQRFREQ0laCIiIim/j/aN/SGIk9sfgAAAABJ\n",
700 | "RU5ErkJggg==\n"
701 | ],
702 | "text/plain": [
703 | ""
704 | ]
705 | },
706 | "metadata": {},
707 | "output_type": "display_data"
708 | }
709 | ],
710 | "source": [
711 | "plt.plot(J_history, marker='o')\n",
712 | "plt.xlabel('Iterations')\n",
713 | "plt.ylabel('J(theta)')"
714 | ]
715 | },
716 | {
717 | "cell_type": "code",
718 | "execution_count": 35,
719 | "metadata": {
720 | "collapsed": false
721 | },
722 | "outputs": [
723 | {
724 | "name": "stdout",
725 | "output_type": "stream",
726 | "text": [
727 | "Training accuracy: 1.0\n",
728 | "Test accuracy: 0.9985815602836879\n"
729 | ]
730 | }
731 | ],
732 | "source": [
733 | "print('Training accuracy: {acc}'.format(acc=accuracy(res.x, train_X, train_y)))\n",
734 | "print('Test accuracy: {acc}'.format(acc=accuracy(res.x, test_X, test_y)))"
735 | ]
736 | }
737 | ],
738 | "metadata": {
739 | "kernelspec": {
740 | "display_name": "Python 3",
741 | "language": "python",
742 | "name": "python3"
743 | },
744 | "language_info": {
745 | "codemirror_mode": {
746 | "name": "ipython",
747 | "version": 3
748 | },
749 | "file_extension": ".py",
750 | "mimetype": "text/x-python",
751 | "name": "python",
752 | "nbconvert_exporter": "python",
753 | "pygments_lexer": "ipython3",
754 | "version": "3.4.2"
755 | }
756 | },
757 | "nbformat": 4,
758 | "nbformat_minor": 0
759 | }
760 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## Stanford Unsupervised Feature Learning and Deep Learning Tutorial (new version) with Python
2 | Tutorial Website:
3 | http://deeplearning.stanford.edu/tutorial/
4 |
5 | Good way of viewing the ipython notebooks of this repo:
6 | http://nbviewer.ipython.org/
7 |
8 | To simplify things, the links to all notebooks are listed below:
9 | - [Linear Regression](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Linear_Regression.ipynb)
10 | - [Logistic Regression](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Logistic_Regression.ipynb)
11 | - [Gradient Checking](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Gradient_Checking.ipynb)
12 | - [Leanring Curve and Other Visualizations](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Learning_Curve_and_Other_Visualizations.ipynb)
13 | - [Softmax Regression](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Softmax_Regression.ipynb)
14 |
--------------------------------------------------------------------------------
/Softmax_Regression.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 32,
6 | "metadata": {
7 | "collapsed": true
8 | },
9 | "outputs": [],
10 | "source": [
11 | "%matplotlib inline"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 33,
17 | "metadata": {
18 | "collapsed": true
19 | },
20 | "outputs": [],
21 | "source": [
22 | "import scipy.optimize\n",
23 | "import time\n",
24 | "import matplotlib.pyplot as plt\n",
25 | "import numpy as np\n",
26 | "from sklearn.datasets import fetch_mldata"
27 | ]
28 | },
29 | {
30 | "cell_type": "markdown",
31 | "metadata": {},
32 | "source": [
33 | "# Data\n",
34 | "First get and preprocess the data. This time, we will use the complete data set and not just the samples for the numbers **`0`** and **`1`**."
35 | ]
36 | },
37 | {
38 | "cell_type": "markdown",
39 | "metadata": {},
40 | "source": [
41 | "Get data: contains 70k samples of which the last 10k are meant for testing"
42 | ]
43 | },
44 | {
45 | "cell_type": "code",
46 | "execution_count": 34,
47 | "metadata": {
48 | "collapsed": true
49 | },
50 | "outputs": [],
51 | "source": [
52 | "mnist = fetch_mldata('MNIST original', data_home='./data')"
53 | ]
54 | },
55 | {
56 | "cell_type": "markdown",
57 | "metadata": {},
58 | "source": [
59 | "Prepare for concat"
60 | ]
61 | },
62 | {
63 | "cell_type": "code",
64 | "execution_count": 35,
65 | "metadata": {
66 | "collapsed": true
67 | },
68 | "outputs": [],
69 | "source": [
70 | "y_all = mnist.target[:, np.newaxis]"
71 | ]
72 | },
73 | {
74 | "cell_type": "markdown",
75 | "metadata": {},
76 | "source": [
77 | "Intercept term to be added"
78 | ]
79 | },
80 | {
81 | "cell_type": "code",
82 | "execution_count": 36,
83 | "metadata": {
84 | "collapsed": true
85 | },
86 | "outputs": [],
87 | "source": [
88 | "intercept = np.ones_like(y_all)"
89 | ]
90 | },
91 | {
92 | "cell_type": "markdown",
93 | "metadata": {},
94 | "source": [
95 | "Before the next step, we need to define this util function which normalizes the data."
96 | ]
97 | },
98 | {
99 | "cell_type": "code",
100 | "execution_count": 37,
101 | "metadata": {
102 | "collapsed": true
103 | },
104 | "outputs": [],
105 | "source": [
106 | "def normalize_features(train, test):\n",
107 | " \"\"\"Normalizes train set features to a standard normal distribution\n",
108 | " (zero mean and unit variance). The same procedure is then applied\n",
109 | " to the test set features.\n",
110 | " \"\"\"\n",
111 | " train_mean = train.mean(axis=0)\n",
112 | " # +0.1 to avoid division by zero in this specific case\n",
113 | " train_std = train.std(axis=0) + 0.1\n",
114 | " \n",
115 | " train = (train - train_mean) / train_std\n",
116 | " test = (test - train_mean) / train_std\n",
117 | " return train, test"
118 | ]
119 | },
120 | {
121 | "cell_type": "markdown",
122 | "metadata": {},
123 | "source": [
124 | "Now, normalize the data (zero mean and unit variance)"
125 | ]
126 | },
127 | {
128 | "cell_type": "code",
129 | "execution_count": 38,
130 | "metadata": {
131 | "collapsed": true
132 | },
133 | "outputs": [],
134 | "source": [
135 | "train_normalized, test_normalized = normalize_features(\n",
136 | " mnist.data[:60000, :],\n",
137 | " mnist.data[60000:, :],\n",
138 | ")"
139 | ]
140 | },
141 | {
142 | "cell_type": "markdown",
143 | "metadata": {},
144 | "source": [
145 | "Concat **`intercept`**, **`X`**, and **`y`** so that shuffling is easier in a next step"
146 | ]
147 | },
148 | {
149 | "cell_type": "code",
150 | "execution_count": 39,
151 | "metadata": {
152 | "collapsed": true
153 | },
154 | "outputs": [],
155 | "source": [
156 | "train_all = np.hstack((\n",
157 | " intercept[:60000],\n",
158 | " train_normalized,\n",
159 | " y_all[:60000],\n",
160 | "))\n",
161 | "test_all = np.hstack((\n",
162 | " intercept[60000:],\n",
163 | " test_normalized,\n",
164 | " y_all[60000:],\n",
165 | "))"
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "Shuffle the data. As mentioned in the [Logistic_Regression](http://nbviewer.ipython.org/github/HaFl/ufldl-tutorial-python/blob/master/Logistic_Regression.ipynb) notebook already, I don't think it's needed, but let's stick with the ufldl tutorial here."
173 | ]
174 | },
175 | {
176 | "cell_type": "code",
177 | "execution_count": 40,
178 | "metadata": {
179 | "collapsed": false
180 | },
181 | "outputs": [],
182 | "source": [
183 | "np.random.shuffle(train_all)\n",
184 | "np.random.shuffle(test_all)"
185 | ]
186 | },
187 | {
188 | "cell_type": "markdown",
189 | "metadata": {},
190 | "source": [
191 | "Finally, get train and test data sets"
192 | ]
193 | },
194 | {
195 | "cell_type": "code",
196 | "execution_count": 41,
197 | "metadata": {
198 | "collapsed": true
199 | },
200 | "outputs": [],
201 | "source": [
202 | "train_X = train_all[:, :-1]\n",
203 | "train_y = train_all[:, -1]\n",
204 | "\n",
205 | "test_X = test_all[:, :-1] \n",
206 | "test_y = test_all[:, -1]"
207 | ]
208 | },
209 | {
210 | "cell_type": "markdown",
211 | "metadata": {},
212 | "source": [
213 | "# Softmax Regression"
214 | ]
215 | },
216 | {
217 | "cell_type": "markdown",
218 | "metadata": {},
219 | "source": [
220 | "Define some helpful variables and initial random theta values for all classes."
221 | ]
222 | },
223 | {
224 | "cell_type": "code",
225 | "execution_count": 42,
226 | "metadata": {
227 | "collapsed": false
228 | },
229 | "outputs": [],
230 | "source": [
231 | "m, n = train_X.shape\n",
232 | "k = np.unique(train_y).size\n",
233 | "theta = np.random.rand(n, k) * 0.001"
234 | ]
235 | },
236 | {
237 | "cell_type": "markdown",
238 | "metadata": {},
239 | "source": [
240 | "This **`indicator_mask`** will come in handy when computing the gradient later on."
241 | ]
242 | },
243 | {
244 | "cell_type": "code",
245 | "execution_count": 43,
246 | "metadata": {
247 | "collapsed": false
248 | },
249 | "outputs": [],
250 | "source": [
251 | "indicator_mask = np.zeros((train_X.shape[0], theta.shape[1]), dtype=np.bool)\n",
252 | "for i, idx in enumerate(train_y):\n",
253 | " indicator_mask[i][idx] = True"
254 | ]
255 | },
256 | {
257 | "cell_type": "markdown",
258 | "metadata": {},
259 | "source": [
260 | "This is a helper function to keep the code DRY. It computes the probabilities of all classes for all samples."
261 | ]
262 | },
263 | {
264 | "cell_type": "code",
265 | "execution_count": 44,
266 | "metadata": {
267 | "collapsed": false
268 | },
269 | "outputs": [],
270 | "source": [
271 | "def probs(theta, X, y):\n",
272 | " if theta.ndim == 1:\n",
273 | " theta = theta.reshape((theta.size / k, k))\n",
274 | " values = np.exp(X.dot(theta)) \n",
275 | " sums = np.sum(values, axis=1)\n",
276 | " return (values.T / sums).T"
277 | ]
278 | },
279 | {
280 | "cell_type": "markdown",
281 | "metadata": {},
282 | "source": [
283 | "The cost function of Softmax Regression. We could actually use the **`indicator_mask`** here instead of the loop at the end, but that would be computational overkill."
284 | ]
285 | },
286 | {
287 | "cell_type": "code",
288 | "execution_count": 45,
289 | "metadata": {
290 | "collapsed": false
291 | },
292 | "outputs": [],
293 | "source": [
294 | "def cost_function(theta, X, y):\n",
295 | " log_probs = np.log(probs(theta, X, y))\n",
296 | " cost = 0\n",
297 | " for i in range(m):\n",
298 | " cost -= log_probs[i][y[i]]\n",
299 | " return cost"
300 | ]
301 | },
302 | {
303 | "cell_type": "markdown",
304 | "metadata": {},
305 | "source": [
306 | "The gradient function of Softmax Regression."
307 | ]
308 | },
309 | {
310 | "cell_type": "code",
311 | "execution_count": 46,
312 | "metadata": {
313 | "collapsed": true
314 | },
315 | "outputs": [],
316 | "source": [
317 | "def gradient(theta, X, y):\n",
318 | " gradient_matrix = -X.T.dot(indicator_mask - probs(theta, X, y))\n",
319 | " return gradient_matrix.flatten()"
320 | ]
321 | },
322 | {
323 | "cell_type": "markdown",
324 | "metadata": {},
325 | "source": [
326 | "Alright, let's run the optimization. 100 iterations are enough here."
327 | ]
328 | },
329 | {
330 | "cell_type": "code",
331 | "execution_count": 47,
332 | "metadata": {
333 | "collapsed": false
334 | },
335 | "outputs": [
336 | {
337 | "name": "stdout",
338 | "output_type": "stream",
339 | "text": [
340 | "Optimization took 195.31115102767944 seconds\n"
341 | ]
342 | }
343 | ],
344 | "source": [
345 | "J_history = []\n",
346 | "\n",
347 | "t0 = time.time()\n",
348 | "res = scipy.optimize.minimize(\n",
349 | " fun=cost_function,\n",
350 | " x0=theta,\n",
351 | " args=(train_X, train_y),\n",
352 | " method='L-BFGS-B',\n",
353 | " jac=gradient,\n",
354 | " options={'maxiter': 100, 'disp': True},\n",
355 | " callback=lambda x: J_history.append(cost_function(x, train_X, train_y)),\n",
356 | ")\n",
357 | "t1 = time.time()\n",
358 | "\n",
359 | "print('Optimization took {s} seconds'.format(s=t1 - t0))\n",
360 | "optimal_theta = res.x.reshape((theta.size / k, k))"
361 | ]
362 | },
363 | {
364 | "cell_type": "markdown",
365 | "metadata": {},
366 | "source": [
367 | "Plot the evolution of **`J`** (to make sure we did the right thing)."
368 | ]
369 | },
370 | {
371 | "cell_type": "code",
372 | "execution_count": 48,
373 | "metadata": {
374 | "collapsed": false
375 | },
376 | "outputs": [
377 | {
378 | "data": {
379 | "text/plain": [
380 | ""
381 | ]
382 | },
383 | "execution_count": 48,
384 | "metadata": {},
385 | "output_type": "execute_result"
386 | },
387 | {
388 | "data": {
389 | "image/png": [
390 | "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEPCAYAAACOU4kjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
391 | "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3X2UVfV97/H3RxFEIRpqisiDcnVMRGwktqKJiZOaEqAp\n",
392 | "apZXSG8JKyE0XaTG5vamFZtbuW3TxHSpUbt0rSqJSKtXE6tgi8SHZGqSGzI1QUUJCaaiMAgkGBGi\n",
393 | "JhC/94/9O5nNOHM4nNn7nJk5n9das2bv7344v7OXzpff41ZEYGZmVqTDml0AMzMbepxczMyscE4u\n",
394 | "ZmZWOCcXMzMrnJOLmZkVzsnFzMwKV3pykbRZ0pOS1knqTLGlkram2DpJs3LnL5G0SdJGSTNy8bMk\n",
395 | "rU/Hrs/FR0i6K8XXSjqx7O9kZmbVNaLmEkB7REyLiLNzsWtTbFpEPAAgaQowF5gCzARukqR0zc3A\n",
396 | "wohoA9okzUzxhcCuFL8OuLoB38nMzKpoVLOYaoxdCNwZEfsiYjPwDDBd0jhgdER0pvNuBy5K23OA\n",
397 | "5Wn7HuCCwkptZmZ1aVTN5WFJj0lalItfJukJScskHZtiJwBbc+dsBcb3Eu9KcdLvLQARsR/YLWlM\n",
398 | "Cd/DzMxq1Ijk8q6ImAbMAj4h6d1kTVyTgTOBF4BrGlAOMzNrkGFlf0BEvJB+/0TSvcDZEfHNynFJ\n",
399 | "twL3p90uYGLu8glkNZautN0zXrlmErBN0jDgmIh4MV8GSV5AzcysDhHRWxfGQZWaXCQdBRweEXsk\n",
400 | "HQ3MAP6PpOMjYns67WJgfdpeBdwh6Vqy5q42oDMiQtLLkqYDncB84IbcNQuAtcAlwCO9laXeBzTU\n",
401 | "SFoaEUubXY6BwM+im59FNz+Lbv35h3nZNZexwL1pwNcw4F8i4kFJt0s6k6w/5lng4wARsUHS3cAG\n",
402 | "YD+wOLqXbV4M3AaMBFZHxJoUXwaskLQJ2AXMK/k7mZnZQZSaXCLiWbJ+lZ7xD1e55u+Bv+8l/j3g\n",
403 | "jF7ivwAu7V9JzcysSJ6h33o6ml2AAaSj2QUYQDqaXYABpKPZBRgK1AovC5MU7nMxMzs0/fnb6ZqL\n",
404 | "mZkVrmWSizRrjTR1drPLYWbWClomucAD74dzr3eCMTMrXwslF4BbToFJlzW7FGZmQ12LJReAUSOb\n",
405 | "XQIzs6GuBZPL3lebXQIzs6GuxZLLx34Mz9/Y7FKYmQ11pS9cOXDMXgPP3xjx1Opml8TMbKjzJEoz\n",
406 | "M+uVJ1GamdmA4uRiZmaFc3IxM7PCObmYmVnhnFzMzKxwpScXSZslPSlpnaTOFBsj6SFJP5L0oKRj\n",
407 | "c+cvkbRJ0kZJM3LxsyStT8euz8VHSLorxddKOrHs72RmZtU1ouYSQHtETIuIs1PsCuChiDiV7J33\n",
408 | "VwBImgLMBaYAM4GblN6RDNwMLIyINqBN0swUXwjsSvHrgKsb8J3MzKyKRjWL9RwnPQdYnraXAxel\n",
409 | "7QuBOyNiX0RsBp4BpksaB4yOiM503u25a/L3uge4oPjim5nZoWhUzeVhSY9JWpRiYyNiR9reAYxN\n",
410 | "2ycAW3PXbgXG9xLvSnHS7y0AEbEf2C1pTOHfwszMataI5V/eFREvSHoL8JCkjfmDERGSSl8mQDri\n",
411 | "b2D/62m3IyI6yv5MM7PBRFI70F7EvUpPLhHxQvr9E0n3AmcDOyQdHxHbU5PXznR6FzAxd/kEshpL\n",
412 | "V9ruGa9cMwnYJmkYcExEvPjGkuy7NoKXCvtiZmZDTPpHd0dlX9JV9d6r1GYxSUdJGp22jwZmAOuB\n",
413 | "VcCCdNoC4L60vQqYJ2m4pMlAG9AZEduBlyVNTx3884GVuWsq97qEbIBAb44u7puZmVk1ZddcxgL3\n",
414 | "pgFfw4B/iYgHJT0G3C1pIbAZuBQgIjZIuhvYAOwHFkf3ypqLgduAkcDqiFiT4suAFZI2AbuAeX2U\n",
415 | "xcnFzKxBWmZVZIhpETze7LKYmQ0WXhW5Nq65mJk1iJOLmZkVzsnFzMwK5+RiZmaFc3IxM7PCObmY\n",
416 | "mVnhnFzMzKxwTi5mZla4VkouRzW7AGZmraKVkotrLmZmDeLkYmZmhXNyMTOzwjm5mJlZ4ZxczMys\n",
417 | "cE4uZmZWOCcXMzMrXOnJRdLhktZJuj/tL5W0NcXWSZqVO3eJpE2SNkqakYufJWl9OnZ9Lj5C0l0p\n",
418 | "vlbSiVWK4uRiZtYgjai5XE722uLKKy8DuDYipqWfBwAkTQHmAlOAmcBNSu9HBm4GFkZEG9AmaWaK\n",
419 | "LwR2pfh1wNVVyuFJlGZmDVJqcpE0AZgN3ApUEoVy23kXAndGxL6I2Aw8A0yXNA4YHRGd6bzbgYvS\n",
420 | "9hxgedq+B7igSnGOknr9XDMzK1jZNZfrgE8Dr+diAVwm6QlJyyQdm+InAFtz520FxvcS70px0u8t\n",
421 | "ABGxH9gtaUwfZfkFMLIf38XMzGpUWnKR9AFgZ0Ss48Cays3AZOBM4AXgmrLKcKC/CjhuaerzaW/M\n",
422 | "Z5qZDR6S2tPfyKWSlvbnXsMKKlNv3gnMkTQbOBJ4k6TbI+LDlRMk3Qrcn3a7gIm56yeQ1Vi60nbP\n",
423 | "eOWaScA2ScOAYyLixd6L89mfwGdvimBzP7+XmdmQFBEdQEdlX9JV9d6rtJpLRFwZERMjYjIwD/h6\n",
424 | "RHw49aFUXAysT9urgHmShkuaDLQBnRGxHXhZ0vTUwT8fWJm7ZkHavgR4pEqRXsEjxszMGqLMmkue\n",
425 | "6B4t9gVJb0/7zwIfB4iIDZLuJhtZth9YHBGVaxYDt5H1mayOiDUpvgxYIWkTsIssifXl5zi5mJk1\n",
426 | "hLr/fg9dkgLiUWBpBN9odnnMzAYDSRERdY2ybaUZ+j/Hc13MzBqi1ZKLm8XMzBrAycXMzArn5GJm\n",
427 | "ZoVzcjEzs8K1UnLxPBczswZppeTimouZWYM4uZiZWeGcXMzMrHCtllw8idLMrAFaLbm45mJm1gBO\n",
428 | "LmZmVjgnFzMzK5yTi5mZFa6VkosnUZqZNUgrJRfXXMzMGqT05CLpcEnrJN2f9sdIekjSjyQ9KOnY\n",
429 | "3LlLJG2StFHSjFz8LEnr07Hrc/ERku5K8bWSTqxSFCcXM7MGaUTN5XKyVxdXXnl5BfBQRJxK9s77\n",
430 | "KwAkTQHmAlOAmcBNkipvQLsZWBgRbUCbpJkpvhDYleLXAVdXKccvgMMljijsm5mZWa9KTS6SJgCz\n",
431 | "gVuBSqKYAyxP28uBi9L2hcCdEbEvIjYDzwDTJY0DRkdEZzrv9tw1+XvdA1zQV1kiCDyR0sysIcqu\n",
432 | "uVwHfBp4PRcbGxE70vYOYGzaPgHYmjtvKzC+l3hXipN+bwGIiP3AbkljqpTHTWNmZg0wrKwbS/oA\n",
433 | "sDMi1klq7+2ciAhJ0duxEsqzFD49Ar5yhbT5XyOioxGfa2Y2WKS/1e1F3Ku05AK8E5gjaTZwJPAm\n",
434 | "SSuAHZKOj4jtqclrZzq/C5iYu34CWY2lK233jFeumQRskzQMOCYiXuytMBGxVOJi+MKXIni8oO9o\n",
435 | "ZjZkpH90d1T2JV1V771KaxaLiCsjYmJETAbmAV+PiPnAKmBBOm0BcF/aXgXMkzRc0mSgDeiMiO3A\n",
436 | "y5Kmpw7++cDK3DWVe11CNkCgGjeLmZk1QJk1l54qzV+fB+6WtBDYDFwKEBEbJN1NNrJsP7A4IirX\n",
437 | "LAZuA0YCqyNiTYovA1ZI2gTsIkti1XgipZlZA6j77/fQJSl177ASuC2Ce5tdJjOzga7yt7Oea1tp\n",
438 | "hj64WczMrCGcXMzMrHCtmFw8idLMrGStmFxcczEzK5mTi5mZFc7JxczMCtfIeS5NJU2dDb+zCN78\n",
439 | "G9IPToUtN0Q8tbrZ5TIzG4paJrnAudfDLaekneNh0cnSVJxgzMyK1zKTKLsXCMibvSZi9ayGF8jM\n",
440 | "bBDwJMq6jRrZ7BKYmQ1FLZ5c9r7a7BKYmQ1FLZRcFj1z4P7HfgzP39icspiZDW0t1Ody+u/DSX8G\n",
441 | "Z70P/vNr8PyN7sw3M+tbf/pcWia5VB6QxF5gXAR7mlwsM7MBzR36h2YXcFyzC2FmNpTVlFwkHS3p\n",
442 | "bZLeKqmmGe6SjpT0XUmPS9og6XMpvlTSVknr0s+s3DVLJG2StFHSjFz8LEnr07Hrc/ERku5K8bWS\n",
443 | "TqyhaD8FfqOW72BmZvXpcxKlpNHAIrK3Ox4H7AAEjJW0C/gX4JaI2Nvb9RHxmqT3RsQr6f3235J0\n",
444 | "HtmEk2sj4toenzcFmAtMAcYDD0tqS2+jvBlYGBGdklZLmpneRrkQ2BURbZLmAldz8LdR/hTXXMzM\n",
445 | "SlWt5nIfsAf4g4j4bxFxbkScExGTgQ+QrdO1ssr1RMQraXM4cDjws7TfWxvehcCdEbEvIjYDzwDT\n",
446 | "JY0DRkdEZzrvduCitD0HWJ627wEuqFaexM1iZmYl6zO5RMQFEXFLROzo5dj2iPiniKj6x1zSYZIe\n",
447 | "J6v1fCMink6HLpP0hKRlko5NsROArbnLt5LVYHrGu1Kc9HtLKtN+YLekMdXKhJvFzMxKV2ufyxhJ\n",
448 | "0yW9p/JTy3UR8XpEnAlMAN4jqZ2siWsycCbwAnBNfUWvm2suZmYlO+jClZIWAZ8kSxCPA+cA3wF+\n",
449 | "t9YPiYjdkv4d+O2I6Mjd+1bg/rTbBUzMXTaBrMbSlbZ7xivXTAK2pX6dYyLixT6+x9Js6/d/B+bu\n",
450 | "h/m1Ft/MrCWkCkB7EfeqpeZyOXA28FxEvBeYBuw+2EWSjqs0eUkaCfwesE7S8bnTLgbWp+1VwDxJ\n",
451 | "wyVNBtqAzojYDrycak4iyworc9csSNuXAI/0VZ6IWBoRS+Hfbof5v6jhe5uZtZSI6Kj8rcz+Xtav\n",
452 | "liX3X4uIVyUh6ciI2CjprTVcNw5YLukwsiS2IiIekXS7pDPJRo09C3wcICI2SLob2ADsBxZH9wzP\n",
453 | "xcBtwEhgdRopBrAMWCFpE1lz18FGioGbxczMSnfQGfqS7gM+QlaDuYBsxNewiJhdfvGK0WOG/pnA\n",
454 | "8gje3uRimZkNaA1b/iW1x70JWBMRv6znA5uhR3KZCKyN+PWIMzMz60Wpy79IWlHZTu1xq8iaowar\n",
455 | "XcBxUq9zbczMrAC1dOhPze+kUVlnlVOc8kXwClmfTk3L2JiZ2aHrM7lIulLSHuAMSXsqP8BOslFa\n",
456 | "g5k79c3MSlRLh/7nI+KKBpWnFD3bDSW+D/xxBI81sVhmZgNa2UvuXylpvqS/Th82SdLZ9XzYAOIl\n",
457 | "YMzMSlRLcrkJOBf4w7S/N8UGMzeLmZmVqJZJlNMjYpqkdQAR8aKkI0ouV9lcczEzK1EtNZdfSjq8\n",
458 | "siPpLcDr5RWpIVxzMTMrUS3J5UbgXuA3Jf098G3gc6WWqnx+YZiZWYkO2iwWEf8s6Xt0v4jrwoj4\n",
459 | "QbnFKp2bxczMSlRLnwvAj4CX0/khaVJEPF9esUrnZjEzsxLV8j6Xy4CryCZP/ip36IyyCtUArrmY\n",
460 | "mZWolprLnwFvjYhdZRemgVxzMTMrUS0d+s+TNYkNJT/Fi1eamZWmz5qLpD9Pm/8FdEj6N6CyzH5E\n",
461 | "xLVlF64sEbwi8TpwFPDzZpfHzGyoqVZzGQ2MIqu5PAQMT/uj0rGqJB0p6buSHpe0QdLnUnyMpIck\n",
462 | "/UjSg5VXIadjSyRtkrRR0oxc/CxJ69Ox63PxEZLuSvG1kk48hO/upjEzs5L0WXOpvD9Z0qURcXf+\n",
463 | "mKRLD3bjiHhN0nsj4pW0TP+3JJ0HzAEeiogvSPpL4ArgCklTgLnAFGA88LCktvSq45uBhRHRKWm1\n",
464 | "pJnpVccLgV0R0SZpLnA1tb3qGLrnujxX4/lmZlajWvpcltQYe4OIeCVtDgcOJ3tF8hxgeYovBy5K\n",
465 | "2xcCd0bEvojYDDwDTJc0DhgdEZ3pvNtz1+TvdQ/dc3Fq4RFjZmYlqdbnMguYDYyXdAP8uvN7NLCv\n",
466 | "lptLOgz4PnAycHNEPC1pbETsSKfsAMam7ROAtbnLt5LVYPal7YquFCf93gIQEfsl7ZY0JiJerKF4\n",
467 | "bhYzMytJtaHI24DvkdUovkeWXALYA3yqlptHxOvAmZKOAb4m6b09joek6i+UKY9rLmZmJanW5/IE\n",
468 | "8ISkOyLil32dV4uI2C3p38lej7xD0vERsT01ee1Mp3UBE3OXTSCrsXSl7Z7xyjWTgG2pX+eYvmot\n",
469 | "kpbmdjsgXHMxM8uR1A60F3Gvaq85/ndJ/51eEpCkoyXNlbS6yvXHVUaCSRoJ/B6wjuwVyQvSaQuA\n",
470 | "+9L2KmCepOGSJgNtQGdEbAdeljRdkoD5wMrcNZV7XQI80ld5ImJp7qcDL15pZnaAiOjI/63sz72q\n",
471 | "NYt9BPhT4P9I+hXwAlnT2PHpurvo/sPem3HA8tTvchiwIiIeSe+FuVvSQmAzcGn6Uhsk3Q1sAPYD\n",
472 | "i6P7HcyLgduAkcDqNFIMYBmwQtImsj6UWkeKgZvFzMxKo+6/31VOksYClTkkz+U65AeF3t4DLTED\n",
473 | "+IsI3tekYpmZDWi9/e2sVbXRYnvJOvB7O/YLsqHCn4mIh+v54AHANRczs5JU69Af1dex1Hl+OnBH\n",
474 | "+j0I/Y/fgolvk/6rA/a8BltuiHiqzz4kMzOrXa3vczlAROwnG0l2Y8HlaQhp6mx45/+Gzx8JnJ9F\n",
475 | "F50sTcUJxsys/2rqcxnserYbSrPWwAPvf+OZs9dErJ7VyLKZmQ1U/elzqWX5lyFo9JG9x0eNbGw5\n",
476 | "zMyGphZNLnte6z2+99XGlsPMbGhq0eSy5QZY9MyBsY/9GJ4flH1IZmYDTUv2uWSxqbPh7C/Asb8B\n",
477 | "Gx+H5290Z76ZWbf+9Lm0bHLJ4rQDfxfBeY0vlZnZwOYO/fr9EHhrswthZjbUtHpy2Q4MlzxT38ys\n",
478 | "SC2dXCIIXHsxMytcSyeXxMnFzKxgTi5OLmZmhXNygY04uZiZFcrJJau5vK3ZhTAzG0pKTS6SJkr6\n",
479 | "hqSnJT0l6ZMpvlTSVknr0s+s3DVLJG2StFHSjFz8LEnr07Hrc/ERku5K8bWSTuTQPANMlupbIdrM\n",
480 | "zN6o7JrLPuBTEXE6cA7wCUmnkb2E7NqImJZ+HgCQNAWYC0wBZgI3SapM4LkZWBgRbUCbpJkpvhDY\n",
481 | "leLXAVcfSgEjeJXsFc6T+/NFzcysW6nJJSK2R8TjaXsv8ANgfDrc26zPC4E7I2JfRGwmq1VMlzQO\n",
482 | "GB0Rnem824GL0vYcYHnavge4oI6iut/FzKxADetzkXQSMA1Ym0KXSXpC0jJJx6bYCcDW3GVbyZJR\n",
483 | "z3gX3UlqPLAFfv0Ss92Sxhxi8TxizMysQA3pZ5A0CvgqcHlE7JV0M/A36fDfAteQNW+VWYalud2O\n",
484 | "iOjI7f8QeEeZn29mNtBJagfai7hX6clF0hFkzVX/HBH3AUTEztzxW4H7024XMDF3+QSyGktX2u4Z\n",
485 | "r1wzCdgmaRhwTES82LMcEbG0SjF/CHyo9m9lZjb0pH90d1T2JV1V773KHi0mYBmwISK+mIuPy512\n",
486 | "MbA+ba8C5kkaLmky0AZ0RsR24GVJ09M95wMrc9csSNuXAI/UUVT3uZiZFajsmsu7gD8CnpS0LsWu\n",
487 | "BD4k6UyyUWPPAh8HiIgNku4GNgD7gcXR/U6AxcBtwEhgdUSsSfFlwApJm4BdwLxDL+bUM2HOGOnZ\n",
488 | "b8FLe2HLDX63i5lZ/Vr6fS7Zsamz4dzr4ZZTuqOLnoHvXO4EY2atzO9z6ZeJnzwwsUC2P+my5pTH\n",
489 | "zGzwc3Jh9JG9x0eNbGw5zMyGDicX9rzWe3zvq40th5nZ0OHkwpYbsj6WvI/9GJ6/sTnlMTMb/Fq+\n",
490 | "Qz87PnV21scy9Wx44TlY9xl35ptZq+tPh76TywHnsQT4zQg+1YBimZkNaB4tVpz/AM5vdiHMzAY7\n",
491 | "11wOOI/hZBMxJ0Swu/ySmZkNXK65FCSCXwKdwHnNLouZ2WDm5PJGj+KmMTOzfnFyeaP/AN7T7EKY\n",
492 | "mQ1m7nN5w7mMBH4CHB/B3nJLZmY2cPWnz6UhLwsbXKa+F2b8EnZ/W9r2gldINjM7dE4uOd0rJF/7\n",
493 | "ZuDNwG/BopOlqTjBmJnVzn0uB/AKyWZmRXByOYBXSDYzK0LZrzmeKOkbkp6W9JSkT6b4GEkPSfqR\n",
494 | "pAclHZu7ZomkTZI2SpqRi58laX06dn0uPkLSXSm+VtKJ9ZfYKySbmRWh7JrLPuBTEXE6cA7wCUmn\n",
495 | "AVcAD0XEqWTvvL8CQNIUYC4wBZgJ3CSpMlLhZmBhRLQBbZJmpvhCYFeKXwdcXX9xe1sh+fJdXiHZ\n",
496 | "zOzQNHQosqT7gH9MP+dHxA5JxwMdEfE2SUuA1yPi6nT+GmAp8Bzw9Yg4LcXnAe0R8SfpnKsi4ruS\n",
497 | "hgEvRMRbenzuIQxFrqyQPGokvH4YvPsd8PVOGHFYVrPx6DEzaw2DYiiypJOAacB3gbERsSMd2gGM\n",
498 | "TdsnAGtzl20FxpPVgLbm4l0pTvq9BSAi9kvaLWlMRLxYTzlT4lidlXnqbJh0B6x8b/cZHj1mZnYw\n",
499 | "DUkukkYB9wCXR8Se7pYuiIiQVHr1SdLS3G5HRHQc/KqJn4RrjzkwdsspMPsyUgIyMxsqJLUD7UXc\n",
500 | "q/TkIukIssSyIiLuS+Edko6PiO2SxgE7U7wLmJi7fAJZjaUrbfeMV66ZBGxLzWLH9FZriYilh156\n",
501 | "jx4zs9aR/tHdUdmXdFW99yp7tJiAZcCGiPhi7tAqYEHaXgDcl4vPkzRc0mSgDeiMiO3Ay5Kmp3vO\n",
502 | "B1b2cq9LyAYIFMSjx8zM6lFqh76k88hWGX4SqHzQErJl7e8mq3FsBi6NiJfSNVcCHwX2kzWjfS3F\n",
503 | "zwJuA0YCqyOiMqx5BLCCrD9nFzAvIjb3KEddnVLdM/YrEysfBa57DbQJXt3mzn0zG8r8muOD6NcD\n",
504 | "+vXosf3j4cS3wi3Du48uega+c7kTjJkNRU4uB9GfB9R9j1lr4IH3v/HI7DURq2f1595mZgOR30TZ\n",
505 | "EO7cNzOrlZNLzXp27j8KfAb41RnSrDVZ85mZmYGX3D8EW26ARSdnnfuPAl8DPgswBni/J1eamXVz\n",
506 | "n8sh3afSuX/YdPi3Nx949FHgcz+F0U97mRgzGwoGxfIvQ0FlaRjp0g7g/O4jlZrMA8d1x12TMbPW\n",
507 | "5T6XuvTsf3mQ1ESW45eMmVnrcnKpS8+l+XtWACud/Uef685+M2tF7nOp+575pfn3nJ6axOjR2Z/2\n",
508 | "r3kFDv+xZ/Wb2WDiSZQHUUZyOfD++WViPgP8XTrSM9GAZ/Wb2WDh5HIQZSeX7DMqNZmjz4WvpGX6\n",
509 | "84mmwqPKzGxwcHI5iEYkl+7Pyi8TszT9VLjJzMwGDy//MqDkO/v39ziWH1VWSTQrj4J/PQOWvB9O\n",
510 | "/or0wSc9CMDMBjvPcylYxFOrpalkb6vcPx4+ejJ86ajsaP5x95VoOCP78TwZMxu8nFxKUJlsCZW+\n",
511 | "mNm5UWWkUWV9JZqK+afA1uXSpU/D9lFwpODYPe6nMbPBwMmlZG9MNIvSqLJ8k1lv82QqM/4fPb+X\n",
512 | "fpp3Sx90P42ZDVhlv+b4S5J2SFqfiy2VtFXSuvQzK3dsiaRNkjZKmpGLnyVpfTp2fS4+QtJdKb5W\n",
513 | "0ollfp/+ypLAdy6H2Wugcz189JXsSLW+mWr9NA+8H069RZrxPenSDvfVmNlAUXbN5cvAjcDtuVgA\n",
514 | "10bEtfkTJU0B5gJTgPHAw5LaIhvOdjOwMCI6Ja2WNDMi1gALgV0R0SZpLnA1MK/k79QvvTeZVeub\n",
515 | "qdZ89ihw2gnw2RO69yu1mp/+0k1pZtYspSaXiPimpJN6OdTb0LYLgTsjYh+wWdIzwHRJzwGjI6Iz\n",
516 | "nXc7cBGwBpgDXJXi9wD/WGDxS1db30y15rO+ajWPnvHGyZsfPEOasd3JxswaoVl9LpdJ+jDwGPDn\n",
517 | "EfEScAKwNnfOVrIazL60XdGV4qTfWwAiYr+k3ZLGRMSLZX+BovXdNzMD+CuyRNGz+ayvWs2h1HBe\n",
518 | "3QZbvgMTz83etunEY2b914zkcjPwN2n7b4FryJq3SiVpaW63IyI6yv7Meh04nHnUSNh+NMwQMPzA\n",
519 | "5rO+ajW11nA4I6vlrPhduOWI7uNuWjNrRZLagfYi7tXw5BIROyvbkm4F7k+7XcDE3KkTyGosXWm7\n",
520 | "Z7xyzSRgm6RhwDF91VoiYmkR5W+UfE0m78Dms+1Hw0fHw5fGHZhoaq3hVPbziaXWpjUPjzYbatI/\n",
521 | "ujsq+5Ku6vPkg2h4cpE0LiJeSLsXA5WRZKuAOyRdS9bc1QZ0RkRIelnSdKATmA/ckLtmAVlz2iXA\n",
522 | "Iw36Gk3TM+n0Pigg35QG1ftt6mla63UZGzezmdmvlZpcJN1J9mbG4yRtIet8b5d0JtmosWeBjwNE\n",
523 | "xAZJdwMbyP4aLo7uhc8WA7cBI4HVaaQYwDJghaRNwC4G+EixMvQ9KKDSlHbs3r5rOFBf01phzWxO\n",
524 | "QmZDlBeubBHdqzbvHw8Tcv02jwIr9mXJIL+Kc88VnZfSvQhnfrvneX29cqC32k7lcyv7lUU8qyUh\n",
525 | "N8eZNUp//nZ6hn6L6LuGs/dVeH4tzD6n9qa1IprZau3reZTumtChNMflk5ATklmjObm0oL4GC0Ct\n",
526 | "TWvVhkfX2sxWTxKqtTmuWo0JDjIwodYE5SY9syqcXOwAh554eg6PngEsSs1d1Uaw1ZOEah311p+B\n",
527 | "CbUkqPx5lf1a+pVqTVauddng5+RiNatteHStzWz1JKFam+PqGZhQ2a8lQfW3Sa/epAZ917qKqIHV\n",
528 | "c54TnvXOycX6rb5mtnqSUK3NcbXO+ak3QfW3Sa/oWlcRNbB6zqsYaAmv9POcTGvg5GKlqpZ48mpL\n",
529 | "QrU2x+WTUK0DE6odK7pJr+haV9lJra/zYOAlvLLPg/4n00GRQNN59XNysQGh1iSU13dzXG+DEQ42\n",
530 | "MAFqT1D9bdIrutZVdlIbTAlvoNceqx0biOfVP4PDycUGrXoSEtRTS+qZoPrbpFfveUUnq/6eBwMv\n",
531 | "4Q302mO1YwPxvPo5uVjLqTcp9aXYZFVPravspNbXeTDwEt5Arz0WcY9Gnlc/Jxezfio6WVVTfR5S\n",
532 | "WUntYOcNpIQ30GuPRdyjkefVz8u/mFm/dC8tVEk8R+YSz5E9EtSkcwb3eduPhlNSMq11eaNajw3E\n",
533 | "80S9fzudXMzMDkH/k+lAT6D58x6Y6eRShZOLmdmh68/fzsOKLoyZmZmTi5mZFa7U5CLpS5J2SFqf\n",
534 | "i42R9JCkH0l6UNKxuWNLJG2StFHSjFz8LEnr07Hrc/ERku5K8bWSTizz+5iZWW3Krrl8GZjZI3YF\n",
535 | "8FBEnEpwB7ZuAAAGxUlEQVT2WuIrACRNAeYCU9I1N0mqtPXdDCyMiDagTVLlnguBXSl+HXB1mV9m\n",
536 | "KJDU3uwyDBR+Ft38LLr5WRSj1OQSEd8EftYjPAdYnraXAxel7QuBOyNiX0RsBp4BpksaB4yOiM50\n",
537 | "3u25a/L3uge4oPAvMfS0N7sAA0h7swswgLQ3uwADSHuzCzAUNKPPZWxE7EjbO4CxafsEYGvuvK3A\n",
538 | "+F7iXSlO+r0FICL2A7sljSmp3GZmVqOmduhHNg566I+FNjNrMc1Y/mWHpOMjYntq8tqZ4l3AxNx5\n",
539 | "E8hqLF1pu2e8cs0kYJukYcAxEfFibx8qyUkskXRVs8swUPhZdPOz6OZn0X/NSC6rgAVkne8LgPty\n",
540 | "8TskXUvW3NUGdEZESHpZ0nSgE5gP3NDjXmuBS8gGCLyBJ1CamTVWqTP0Jd0JnA8cR9a/8tfASuBu\n",
541 | "shrHZuDSiHgpnX8l8FGyldMuj4ivpfhZwG3ASGB1RHwyxUcAK4BpwC5gXhoMYGZmTdQSy7+YmVlj\n",
542 | "DekZ+pJmpgmZmyT9ZbPL00iSJkr6hqSnJT0lqVLb63MS61An6XBJ6yTdn/Zb8llIOlbSVyX9QNIG\n",
543 | "SdNb+FksSf+PrJd0R5qY3RLPoqhJ7n0ZsslF0uHAP5JNyJwCfEjSac0tVUPtAz4VEacD5wCfSN+/\n",
544 | "10msLeJyYAPdIxRb9VlcT9a8fBrwW8BGWvBZSDoJWAS8IyLOAA4H5tE6z6K/k9yr5o8hm1yAs4Fn\n",
545 | "ImJzROwD/i/ZRM2WEBHbI+LxtL0X+AHZQIm+JrEOaZImALOBW+l+MXjLPQtJxwDvjogvQTY/LCJ2\n",
546 | "04LPAniZ7B9hR6XRpkcB22iRZ1HAJPezq91/KCeXX0+wTCqTMltO+hfaNOC79D2Jdai7Dvg08Hou\n",
547 | "1orPYjLwE0lflvR9SbdIOpoWfBZp2sI1wPNkSeWliHiIFnwWOYc6yb1PQzm5eKQCIGkU2dI4l0fE\n",
548 | "nvyxVpnEKukDwM6IWEd3reUArfIsyKYfvAO4KSLeAfycHs0+rfIsJJ0M/BlwEtkfz1GS/ih/Tqs8\n",
549 | "i97U8N2rPpehnFx6TsqcyIGZd8iTdARZYlkREZX5RDskHZ+O5yexDmXvBOZIeha4E/hdSStozWex\n",
550 | "FdgaEf+Z9r9Klmy2t+Cz+G3g/0XErrR81L8C59Kaz6Kir/8nepvk3lXtRkM5uTxGtoLySZKGk3VG\n",
551 | "rWpymRomrSi9DNgQEV/MHapMPIUDJ7EOWRFxZURMjIjJZB22X4+I+bTms9gObJF0agq9D3gauJ8W\n",
552 | "exZkAxnOkTQy/f/yPrIBH634LCr6+n9iFTBP0nBJk0mT3KvdaEjPc5E0C/gi2SiQZRHxuSYXqWEk\n",
553 | "nQc8CjxJd/V1Cdl/EL1OYm0Fks4H/jwi5qRFTlvuWUh6O9nAhuHAj4GPkP0/0orP4i/I/oi+Dnwf\n",
554 | "+BgwmhZ4FkVNcu/z/kM5uZiZWXMM5WYxMzNrEicXMzMrnJOLmZkVzsnFzMwK5+RiZmaFc3IxM7PC\n",
555 | "ObmYHYSkven3iZI+VPC9r+yx/+0i72/WLE4uZgdXmQw2GfjDQ7kwrbZbzZIDPijiXYdyf7OBysnF\n",
556 | "rHafB96dXjh2uaTDJP2DpE5JT0j6YwBJ7ZK+KWkl8FSK3SfpsfTitkUp9nlgZLrfihSr1JKU7r1e\n",
557 | "0pOSLs3du0PSV9LLvv65UjhJn08vvnpC0j809MmY9XCwf1WZWbe/BP5XRPwBQEomL0XE2ZJGAN+S\n",
558 | "9GA6dxpwekQ8l/Y/EhE/kzQS6JT01Yi4QtInImJa7jMqtaQPAm8ne5nXW4D/lPRoOnYm2UubXgC+\n",
559 | "LeldZOtkXRQRb0tle1MJ39+sZq65mNWu53L9M4APS1oHrAXGAKekY525xAJwuaTHge+QrS7bdpDP\n",
560 | "Og+4IzI7gf8Afocs+XRGxLa0JPrjwInAS8BrkpZJuhh4te5vaVYAJxez/vnTiJiWfk6OiIdT/OeV\n",
561 | "EyS1AxcA50TEmcA64MiD3Dd4YzKr1Gp+kYv9CjgiIn5F9mbArwIfANbU82XMiuLkYla7PWQr5lZ8\n",
562 | "DVhc6bSXdKqko3q57k3AzyLiNUlvA87JHdvXR6f/N4G5qV/nLcB7yFa07vVlZ+ltksdGxAPA/yRr\n",
563 | "UjNrGve5mB1cpcbwBPCr1Lz1ZeAGsrcYfj+9D2QncHE6P7/c+BrgTyRtAH5I1jRW8U/Ak5K+l94x\n",
564 | "EwARca+kc9NnBvDpiNgp6TTe+AbAIEt6KyUdSZaAPlXINzerk5fcNzOzwrlZzMzMCufkYmZmhXNy\n",
565 | "MTOzwjm5mJlZ4ZxczMyscE4uZmZWOCcXMzMrnJOLmZkV7v8DX2qxSUZpL/8AAAAASUVORK5CYII=\n"
566 | ],
567 | "text/plain": [
568 | ""
569 | ]
570 | },
571 | "metadata": {},
572 | "output_type": "display_data"
573 | }
574 | ],
575 | "source": [
576 | "plt.plot(J_history, marker='o')\n",
577 | "plt.xlabel('Iterations')\n",
578 | "plt.ylabel('J(theta)')"
579 | ]
580 | },
581 | {
582 | "cell_type": "markdown",
583 | "metadata": {},
584 | "source": [
585 | "Even the **`optimal_theta`** still results in a relatively high cost value **`J`**. That's not necessarily a bad sign, but rather the consequence of the model being more complex than e.g., Logistic Regression."
586 | ]
587 | },
588 | {
589 | "cell_type": "code",
590 | "execution_count": 49,
591 | "metadata": {
592 | "collapsed": true
593 | },
594 | "outputs": [],
595 | "source": [
596 | "def accuracy(theta, X, y):\n",
597 | " correct = np.sum(np.argmax(probs(theta, X, y), axis=1) == y)\n",
598 | " return correct / y.size"
599 | ]
600 | },
601 | {
602 | "cell_type": "code",
603 | "execution_count": 50,
604 | "metadata": {
605 | "collapsed": false
606 | },
607 | "outputs": [
608 | {
609 | "name": "stdout",
610 | "output_type": "stream",
611 | "text": [
612 | "Training accuracy: 0.9435833333333333\n",
613 | "Test accuracy: 0.9235\n"
614 | ]
615 | }
616 | ],
617 | "source": [
618 | "print('Training accuracy: {acc}'.format(acc=accuracy(res.x, train_X, train_y)))\n",
619 | "print('Test accuracy: {acc}'.format(acc=accuracy(res.x, test_X, test_y)))"
620 | ]
621 | }
622 | ],
623 | "metadata": {
624 | "kernelspec": {
625 | "display_name": "Python 3",
626 | "language": "python",
627 | "name": "python3"
628 | },
629 | "language_info": {
630 | "codemirror_mode": {
631 | "name": "ipython",
632 | "version": 3
633 | },
634 | "file_extension": ".py",
635 | "mimetype": "text/x-python",
636 | "name": "python",
637 | "nbconvert_exporter": "python",
638 | "pygments_lexer": "ipython3",
639 | "version": "3.4.2"
640 | }
641 | },
642 | "nbformat": 4,
643 | "nbformat_minor": 0
644 | }
645 |
--------------------------------------------------------------------------------