├── Calibration.ipynb
├── Data Science Team.ipynb
├── Spark-Pandas-Differences.ipynb
├── TensorFlow Tutorial.ipynb
├── Tuning Neural Networks.ipynb
└── anomaly detection with t-sne.ipynb


/Calibration.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Using scikit-learn calibration"
  8 |    ]
  9 |   },
 10 |   {
 11 |    "cell_type": "code",
 12 |    "execution_count": 21,
 13 |    "metadata": {
 14 |     "collapsed": true
 15 |    },
 16 |    "outputs": [],
 17 |    "source": [
 18 |     "import pandas as pd\n",
 19 |     "from sklearn.preprocessing import LabelEncoder\n",
 20 |     "from sklearn.cross_validation import train_test_split\n",
 21 |     "from sklearn.ensemble import RandomForestClassifier, BaggingClassifier \n",
 22 |     "from sklearn.metrics import log_loss\n",
 23 |     "from sklearn.calibration import CalibratedClassifierCV"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "markdown",
 28 |    "metadata": {},
 29 |    "source": [
 30 |     "# Import Data (Kaggle OTTO challenge)"
 31 |    ]
 32 |   },
 33 |   {
 34 |    "cell_type": "code",
 35 |    "execution_count": 22,
 36 |    "metadata": {
 37 |     "collapsed": false
 38 |    },
 39 |    "outputs": [
 40 |     {
 41 |      "data": {
 42 |       "text/html": [
 43 |        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
 44 |        "<table border=\"1\" class=\"dataframe\">\n",
 45 |        "  <thead>\n",
 46 |        "    <tr style=\"text-align: right;\">\n",
 47 |        "      <th></th>\n",
 48 |        "      <th>feat_1</th>\n",
 49 |        "      <th>feat_2</th>\n",
 50 |        "      <th>feat_3</th>\n",
 51 |        "      <th>feat_4</th>\n",
 52 |        "      <th>feat_5</th>\n",
 53 |        "      <th>feat_6</th>\n",
 54 |        "      <th>feat_7</th>\n",
 55 |        "      <th>feat_8</th>\n",
 56 |        "      <th>feat_9</th>\n",
 57 |        "      <th>feat_10</th>\n",
 58 |        "      <th>...</th>\n",
 59 |        "      <th>feat_84</th>\n",
 60 |        "      <th>feat_85</th>\n",
 61 |        "      <th>feat_86</th>\n",
 62 |        "      <th>feat_87</th>\n",
 63 |        "      <th>feat_88</th>\n",
 64 |        "      <th>feat_89</th>\n",
 65 |        "      <th>feat_90</th>\n",
 66 |        "      <th>feat_91</th>\n",
 67 |        "      <th>feat_92</th>\n",
 68 |        "      <th>feat_93</th>\n",
 69 |        "    </tr>\n",
 70 |        "  </thead>\n",
 71 |        "  <tbody>\n",
 72 |        "    <tr>\n",
 73 |        "      <th>0</th>\n",
 74 |        "      <td> 1</td>\n",
 75 |        "      <td> 0</td>\n",
 76 |        "      <td> 0</td>\n",
 77 |        "      <td> 0</td>\n",
 78 |        "      <td> 0</td>\n",
 79 |        "      <td> 0</td>\n",
 80 |        "      <td> 0</td>\n",
 81 |        "      <td> 0</td>\n",
 82 |        "      <td> 0</td>\n",
 83 |        "      <td> 0</td>\n",
 84 |        "      <td>...</td>\n",
 85 |        "      <td>  0</td>\n",
 86 |        "      <td> 1</td>\n",
 87 |        "      <td> 0</td>\n",
 88 |        "      <td> 0</td>\n",
 89 |        "      <td> 0</td>\n",
 90 |        "      <td> 0</td>\n",
 91 |        "      <td> 0</td>\n",
 92 |        "      <td> 0</td>\n",
 93 |        "      <td> 0</td>\n",
 94 |        "      <td> 0</td>\n",
 95 |        "    </tr>\n",
 96 |        "    <tr>\n",
 97 |        "      <th>1</th>\n",
 98 |        "      <td> 0</td>\n",
 99 |        "      <td> 0</td>\n",
100 |        "      <td> 0</td>\n",
101 |        "      <td> 0</td>\n",
102 |        "      <td> 0</td>\n",
103 |        "      <td> 0</td>\n",
104 |        "      <td> 0</td>\n",
105 |        "      <td> 1</td>\n",
106 |        "      <td> 0</td>\n",
107 |        "      <td> 0</td>\n",
108 |        "      <td>...</td>\n",
109 |        "      <td>  0</td>\n",
110 |        "      <td> 0</td>\n",
111 |        "      <td> 0</td>\n",
112 |        "      <td> 0</td>\n",
113 |        "      <td> 0</td>\n",
114 |        "      <td> 0</td>\n",
115 |        "      <td> 0</td>\n",
116 |        "      <td> 0</td>\n",
117 |        "      <td> 0</td>\n",
118 |        "      <td> 0</td>\n",
119 |        "    </tr>\n",
120 |        "    <tr>\n",
121 |        "      <th>2</th>\n",
122 |        "      <td> 0</td>\n",
123 |        "      <td> 0</td>\n",
124 |        "      <td> 0</td>\n",
125 |        "      <td> 0</td>\n",
126 |        "      <td> 0</td>\n",
127 |        "      <td> 0</td>\n",
128 |        "      <td> 0</td>\n",
129 |        "      <td> 1</td>\n",
130 |        "      <td> 0</td>\n",
131 |        "      <td> 0</td>\n",
132 |        "      <td>...</td>\n",
133 |        "      <td>  0</td>\n",
134 |        "      <td> 0</td>\n",
135 |        "      <td> 0</td>\n",
136 |        "      <td> 0</td>\n",
137 |        "      <td> 0</td>\n",
138 |        "      <td> 0</td>\n",
139 |        "      <td> 0</td>\n",
140 |        "      <td> 0</td>\n",
141 |        "      <td> 0</td>\n",
142 |        "      <td> 0</td>\n",
143 |        "    </tr>\n",
144 |        "    <tr>\n",
145 |        "      <th>3</th>\n",
146 |        "      <td> 1</td>\n",
147 |        "      <td> 0</td>\n",
148 |        "      <td> 0</td>\n",
149 |        "      <td> 1</td>\n",
150 |        "      <td> 6</td>\n",
151 |        "      <td> 1</td>\n",
152 |        "      <td> 5</td>\n",
153 |        "      <td> 0</td>\n",
154 |        "      <td> 0</td>\n",
155 |        "      <td> 1</td>\n",
156 |        "      <td>...</td>\n",
157 |        "      <td> 22</td>\n",
158 |        "      <td> 0</td>\n",
159 |        "      <td> 1</td>\n",
160 |        "      <td> 2</td>\n",
161 |        "      <td> 0</td>\n",
162 |        "      <td> 0</td>\n",
163 |        "      <td> 0</td>\n",
164 |        "      <td> 0</td>\n",
165 |        "      <td> 0</td>\n",
166 |        "      <td> 0</td>\n",
167 |        "    </tr>\n",
168 |        "    <tr>\n",
169 |        "      <th>4</th>\n",
170 |        "      <td> 0</td>\n",
171 |        "      <td> 0</td>\n",
172 |        "      <td> 0</td>\n",
173 |        "      <td> 0</td>\n",
174 |        "      <td> 0</td>\n",
175 |        "      <td> 0</td>\n",
176 |        "      <td> 0</td>\n",
177 |        "      <td> 0</td>\n",
178 |        "      <td> 0</td>\n",
179 |        "      <td> 0</td>\n",
180 |        "      <td>...</td>\n",
181 |        "      <td>  0</td>\n",
182 |        "      <td> 1</td>\n",
183 |        "      <td> 0</td>\n",
184 |        "      <td> 0</td>\n",
185 |        "      <td> 0</td>\n",
186 |        "      <td> 0</td>\n",
187 |        "      <td> 1</td>\n",
188 |        "      <td> 0</td>\n",
189 |        "      <td> 0</td>\n",
190 |        "      <td> 0</td>\n",
191 |        "    </tr>\n",
192 |        "  </tbody>\n",
193 |        "</table>\n",
194 |        "<p>5 rows × 93 columns</p>\n",
195 |        "</div>"
196 |       ],
197 |       "text/plain": [
198 |        "   feat_1  feat_2  feat_3  feat_4  feat_5  feat_6  feat_7  feat_8  feat_9  \\\n",
199 |        "0       1       0       0       0       0       0       0       0       0   \n",
200 |        "1       0       0       0       0       0       0       0       1       0   \n",
201 |        "2       0       0       0       0       0       0       0       1       0   \n",
202 |        "3       1       0       0       1       6       1       5       0       0   \n",
203 |        "4       0       0       0       0       0       0       0       0       0   \n",
204 |        "\n",
205 |        "   feat_10   ...     feat_84  feat_85  feat_86  feat_87  feat_88  feat_89  \\\n",
206 |        "0        0   ...           0        1        0        0        0        0   \n",
207 |        "1        0   ...           0        0        0        0        0        0   \n",
208 |        "2        0   ...           0        0        0        0        0        0   \n",
209 |        "3        1   ...          22        0        1        2        0        0   \n",
210 |        "4        0   ...           0        1        0        0        0        0   \n",
211 |        "\n",
212 |        "   feat_90  feat_91  feat_92  feat_93  \n",
213 |        "0        0        0        0        0  \n",
214 |        "1        0        0        0        0  \n",
215 |        "2        0        0        0        0  \n",
216 |        "3        0        0        0        0  \n",
217 |        "4        1        0        0        0  \n",
218 |        "\n",
219 |        "[5 rows x 93 columns]"
220 |       ]
221 |      },
222 |      "execution_count": 22,
223 |      "metadata": {},
224 |      "output_type": "execute_result"
225 |     }
226 |    ],
227 |    "source": [
228 |     "X = pd.read_csv('../train.csv')\n",
229 |     "X = X.drop('id', axis=1)\n",
230 |     "\n",
231 |     "# Extract target\n",
232 |     "# Encode it to make it manageable by ML algo\n",
233 |     "y = X.target.values\n",
234 |     "y = LabelEncoder().fit_transform(y)\n",
235 |     "\n",
236 |     "# Remove target from train, else it's too easy ...\n",
237 |     "X = X.drop('target', axis=1)\n",
238 |     "\n",
239 |     "X.head(5)"
240 |    ]
241 |   },
242 |   {
243 |    "cell_type": "markdown",
244 |    "metadata": {},
245 |    "source": [
246 |     "# Split Train / Test"
247 |    ]
248 |   },
249 |   {
250 |    "cell_type": "code",
251 |    "execution_count": 23,
252 |    "metadata": {
253 |     "collapsed": false
254 |    },
255 |    "outputs": [],
256 |    "source": [
257 |     "Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.20, random_state=36)"
258 |    ]
259 |   },
260 |   {
261 |    "cell_type": "markdown",
262 |    "metadata": {},
263 |    "source": [
264 |     "# Train and apply a Random Forest (without calibration)"
265 |    ]
266 |   },
267 |   {
268 |    "cell_type": "code",
269 |    "execution_count": 24,
270 |    "metadata": {
271 |     "collapsed": false
272 |    },
273 |    "outputs": [
274 |     {
275 |      "name": "stdout",
276 |      "output_type": "stream",
277 |      "text": [
278 |       "0.60\n"
279 |      ]
280 |     }
281 |    ],
282 |    "source": [
283 |     "clf = RandomForestClassifier(n_estimators=250, n_jobs=-1)\n",
284 |     "# we use a BaggingClassifier to make 5 predictions, and average\n",
285 |     "# beacause that's what CalibratedClassifierCV do behind the scene\n",
286 |     "# and we want to compare things fairly\n",
287 |     "clfbag = BaggingClassifier(clf, n_estimators=5)\n",
288 |     "clfbag.fit(Xtrain, ytrain)\n",
289 |     "ypreds = clfbag.predict_proba(Xtest)\n",
290 |     "print \"%.2f\" % log_loss(ytest, ypreds, eps=1e-15, normalize=True)"
291 |    ]
292 |   },
293 |   {
294 |    "cell_type": "markdown",
295 |    "metadata": {},
296 |    "source": [
297 |     "# Train and apply a Random Forest (with calibration)"
298 |    ]
299 |   },
300 |   {
301 |    "cell_type": "code",
302 |    "execution_count": 25,
303 |    "metadata": {
304 |     "collapsed": false
305 |    },
306 |    "outputs": [
307 |     {
308 |      "name": "stdout",
309 |      "output_type": "stream",
310 |      "text": [
311 |       "0.49\n"
312 |      ]
313 |     }
314 |    ],
315 |    "source": [
316 |     "clf = RandomForestClassifier(n_estimators=250, n_jobs=-1)\n",
317 |     "# in our case, 'isotonic' works better than default 'sigmoid'\n",
318 |     "calibrated_clf = CalibratedClassifierCV(clf, method='isotonic', cv=5)\n",
319 |     "calibrated_clf.fit(Xtrain, ytrain)\n",
320 |     "ypreds = calibrated_clf.predict_proba(Xtest)\n",
321 |     "print \"%.2f\" % log_loss(ytest, ypreds, eps=1e-15, normalize=True)"
322 |    ]
323 |   },
324 |   {
325 |    "cell_type": "markdown",
326 |    "metadata": {},
327 |    "source": [
328 |     "# We highly improved performance with calibration !"
329 |    ]
330 |   }
331 |  ],
332 |  "metadata": {
333 |   "kernelspec": {
334 |    "display_name": "Python 2",
335 |    "language": "python",
336 |    "name": "python2"
337 |   },
338 |   "language_info": {
339 |    "codemirror_mode": {
340 |     "name": "ipython",
341 |     "version": 2
342 |    },
343 |    "file_extension": ".py",
344 |    "mimetype": "text/x-python",
345 |    "name": "python",
346 |    "nbconvert_exporter": "python",
347 |    "pygments_lexer": "ipython2",
348 |    "version": "2.7.9"
349 |   }
350 |  },
351 |  "nbformat": 4,
352 |  "nbformat_minor": 0
353 | }
354 | 


--------------------------------------------------------------------------------
/Spark-Pandas-Differences.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# Some Differences Between Pandas And Spark Dataframes"
  8 |    ]
  9 |   },
 10 |   {
 11 |    "cell_type": "markdown",
 12 |    "metadata": {},
 13 |    "source": [
 14 |     "### Find the associated blog post here : https://medium.com/@chris_bour"
 15 |    ]
 16 |   },
 17 |   {
 18 |    "cell_type": "markdown",
 19 |    "metadata": {},
 20 |    "source": [
 21 |     "*To launch a Spark-compatible notebook, you can run in a shell :*\n",
 22 |     "\n",
 23 |     "*IPYTHON_OPTS=\"notebook\" --packages com.databricks:spark-csv_2.10:1.0.3 pyspark*"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "code",
 28 |    "execution_count": 1,
 29 |    "metadata": {
 30 |     "collapsed": false
 31 |    },
 32 |    "outputs": [],
 33 |    "source": [
 34 |     "from pyspark.sql import SQLContext\n",
 35 |     "import pandas as pd"
 36 |    ]
 37 |   },
 38 |   {
 39 |    "cell_type": "code",
 40 |    "execution_count": 2,
 41 |    "metadata": {
 42 |     "collapsed": true
 43 |    },
 44 |    "outputs": [],
 45 |    "source": [
 46 |     "sqlContext = SQLContext(sc)"
 47 |    ]
 48 |   },
 49 |   {
 50 |    "cell_type": "markdown",
 51 |    "metadata": {},
 52 |    "source": [
 53 |     "## Reading"
 54 |    ]
 55 |   },
 56 |   {
 57 |    "cell_type": "markdown",
 58 |    "metadata": {},
 59 |    "source": [
 60 |     "With Pandas, you easily read CSV file with read_csv().\n",
 61 |     "\n",
 62 |     "CSV is not supported natively by Spark. You have to use a separate library : spark-csv."
 63 |    ]
 64 |   },
 65 |   {
 66 |    "cell_type": "code",
 67 |    "execution_count": 3,
 68 |    "metadata": {
 69 |     "collapsed": false
 70 |    },
 71 |    "outputs": [],
 72 |    "source": [
 73 |     "pandasDF = pd.read_csv('train.csv')\n",
 74 |     "sparkDF = sqlContext.read.format('com.databricks.spark.csv') \\\n",
 75 |     "                                .options(header='true').load('train.csv')"
 76 |    ]
 77 |   },
 78 |   {
 79 |    "cell_type": "markdown",
 80 |    "metadata": {},
 81 |    "source": [
 82 |     "## Counting"
 83 |    ]
 84 |   },
 85 |   {
 86 |    "cell_type": "markdown",
 87 |    "metadata": {},
 88 |    "source": [
 89 |     "*sparkDF.count()* and *pandasDF.count()* are not the exactly the same.\n",
 90 |     "\n",
 91 |     "The first one returns the number of rows.\n",
 92 |     "\n",
 93 |     "The second one returns the number of non NA/null observations for each column."
 94 |    ]
 95 |   },
 96 |   {
 97 |    "cell_type": "code",
 98 |    "execution_count": 4,
 99 |    "metadata": {
100 |     "collapsed": false
101 |    },
102 |    "outputs": [
103 |     {
104 |      "data": {
105 |       "text/plain": [
106 |        "PassengerId    891\n",
107 |        "Survived       891\n",
108 |        "Pclass         891\n",
109 |        "Name           891\n",
110 |        "Sex            891\n",
111 |        "Age            714\n",
112 |        "SibSp          891\n",
113 |        "Parch          891\n",
114 |        "Ticket         891\n",
115 |        "Fare           891\n",
116 |        "Cabin          204\n",
117 |        "Embarked       889\n",
118 |        "dtype: int64"
119 |       ]
120 |      },
121 |      "execution_count": 4,
122 |      "metadata": {},
123 |      "output_type": "execute_result"
124 |     }
125 |    ],
126 |    "source": [
127 |     "pandasDF.count()"
128 |    ]
129 |   },
130 |   {
131 |    "cell_type": "code",
132 |    "execution_count": 5,
133 |    "metadata": {
134 |     "collapsed": false
135 |    },
136 |    "outputs": [
137 |     {
138 |      "data": {
139 |       "text/plain": [
140 |        "891"
141 |       ]
142 |      },
143 |      "execution_count": 5,
144 |      "metadata": {},
145 |      "output_type": "execute_result"
146 |     }
147 |    ],
148 |    "source": [
149 |     "sparkDF.count()"
150 |    ]
151 |   },
152 |   {
153 |    "cell_type": "markdown",
154 |    "metadata": {},
155 |    "source": [
156 |     "*sparkDF.shape* does not exist."
157 |    ]
158 |   },
159 |   {
160 |    "cell_type": "code",
161 |    "execution_count": 6,
162 |    "metadata": {
163 |     "collapsed": false
164 |    },
165 |    "outputs": [
166 |     {
167 |      "name": "stdout",
168 |      "output_type": "stream",
169 |      "text": [
170 |       "(891, 12)\n"
171 |      ]
172 |     }
173 |    ],
174 |    "source": [
175 |     "print pandasDF.shape"
176 |    ]
177 |   },
178 |   {
179 |    "cell_type": "code",
180 |    "execution_count": 7,
181 |    "metadata": {
182 |     "collapsed": false
183 |    },
184 |    "outputs": [
185 |     {
186 |      "ename": "AttributeError",
187 |      "evalue": "'DataFrame' object has no attribute 'shape'",
188 |      "output_type": "error",
189 |      "traceback": [
190 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
191 |       "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
192 |       "\u001b[0;32m<ipython-input-7-1206bc91d8d1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0msparkDF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
193 |       "\u001b[0;32m/usr/local/Cellar/apache-spark/1.4.0/libexec/python/pyspark/sql/dataframe.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    716\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    717\u001b[0m             raise AttributeError(\n\u001b[0;32m--> 718\u001b[0;31m                 \"'%s' object has no attribute '%s'\" % (self.__class__.__name__, name))\n\u001b[0m\u001b[1;32m    719\u001b[0m         \u001b[0mjc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    720\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mColumn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
194 |       "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'shape'"
195 |      ]
196 |     }
197 |    ],
198 |    "source": [
199 |     "print sparkDF.shape()"
200 |    ]
201 |   },
202 |   {
203 |    "cell_type": "markdown",
204 |    "metadata": {},
205 |    "source": [
206 |     "## Viewing"
207 |    ]
208 |   },
209 |   {
210 |    "cell_type": "markdown",
211 |    "metadata": {},
212 |    "source": [
213 |     "Pandas *.head()* is beautiful. Spark *.head()* is ugly. Prefer *.show()*"
214 |    ]
215 |   },
216 |   {
217 |    "cell_type": "code",
218 |    "execution_count": 8,
219 |    "metadata": {
220 |     "collapsed": false
221 |    },
222 |    "outputs": [
223 |     {
224 |      "data": {
225 |       "text/html": [
226 |        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
227 |        "<table border=\"1\" class=\"dataframe\">\n",
228 |        "  <thead>\n",
229 |        "    <tr style=\"text-align: right;\">\n",
230 |        "      <th></th>\n",
231 |        "      <th>PassengerId</th>\n",
232 |        "      <th>Survived</th>\n",
233 |        "      <th>Pclass</th>\n",
234 |        "      <th>Name</th>\n",
235 |        "      <th>Sex</th>\n",
236 |        "      <th>Age</th>\n",
237 |        "      <th>SibSp</th>\n",
238 |        "      <th>Parch</th>\n",
239 |        "      <th>Ticket</th>\n",
240 |        "      <th>Fare</th>\n",
241 |        "      <th>Cabin</th>\n",
242 |        "      <th>Embarked</th>\n",
243 |        "    </tr>\n",
244 |        "  </thead>\n",
245 |        "  <tbody>\n",
246 |        "    <tr>\n",
247 |        "      <th>0</th>\n",
248 |        "      <td>1</td>\n",
249 |        "      <td>0</td>\n",
250 |        "      <td>3</td>\n",
251 |        "      <td>Braund, Mr. Owen Harris</td>\n",
252 |        "      <td>male</td>\n",
253 |        "      <td>22</td>\n",
254 |        "      <td>1</td>\n",
255 |        "      <td>0</td>\n",
256 |        "      <td>A/5 21171</td>\n",
257 |        "      <td>7.2500</td>\n",
258 |        "      <td>NaN</td>\n",
259 |        "      <td>S</td>\n",
260 |        "    </tr>\n",
261 |        "    <tr>\n",
262 |        "      <th>1</th>\n",
263 |        "      <td>2</td>\n",
264 |        "      <td>1</td>\n",
265 |        "      <td>1</td>\n",
266 |        "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
267 |        "      <td>female</td>\n",
268 |        "      <td>38</td>\n",
269 |        "      <td>1</td>\n",
270 |        "      <td>0</td>\n",
271 |        "      <td>PC 17599</td>\n",
272 |        "      <td>71.2833</td>\n",
273 |        "      <td>C85</td>\n",
274 |        "      <td>C</td>\n",
275 |        "    </tr>\n",
276 |        "    <tr>\n",
277 |        "      <th>2</th>\n",
278 |        "      <td>3</td>\n",
279 |        "      <td>1</td>\n",
280 |        "      <td>3</td>\n",
281 |        "      <td>Heikkinen, Miss. Laina</td>\n",
282 |        "      <td>female</td>\n",
283 |        "      <td>26</td>\n",
284 |        "      <td>0</td>\n",
285 |        "      <td>0</td>\n",
286 |        "      <td>STON/O2. 3101282</td>\n",
287 |        "      <td>7.9250</td>\n",
288 |        "      <td>NaN</td>\n",
289 |        "      <td>S</td>\n",
290 |        "    </tr>\n",
291 |        "    <tr>\n",
292 |        "      <th>3</th>\n",
293 |        "      <td>4</td>\n",
294 |        "      <td>1</td>\n",
295 |        "      <td>1</td>\n",
296 |        "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
297 |        "      <td>female</td>\n",
298 |        "      <td>35</td>\n",
299 |        "      <td>1</td>\n",
300 |        "      <td>0</td>\n",
301 |        "      <td>113803</td>\n",
302 |        "      <td>53.1000</td>\n",
303 |        "      <td>C123</td>\n",
304 |        "      <td>S</td>\n",
305 |        "    </tr>\n",
306 |        "  </tbody>\n",
307 |        "</table>\n",
308 |        "</div>"
309 |       ],
310 |       "text/plain": [
311 |        "   PassengerId  Survived  Pclass  \\\n",
312 |        "0            1         0       3   \n",
313 |        "1            2         1       1   \n",
314 |        "2            3         1       3   \n",
315 |        "3            4         1       1   \n",
316 |        "\n",
317 |        "                                                Name     Sex  Age  SibSp  \\\n",
318 |        "0                            Braund, Mr. Owen Harris    male   22      1   \n",
319 |        "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female   38      1   \n",
320 |        "2                             Heikkinen, Miss. Laina  female   26      0   \n",
321 |        "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female   35      1   \n",
322 |        "\n",
323 |        "   Parch            Ticket     Fare Cabin Embarked  \n",
324 |        "0      0         A/5 21171   7.2500   NaN        S  \n",
325 |        "1      0          PC 17599  71.2833   C85        C  \n",
326 |        "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
327 |        "3      0            113803  53.1000  C123        S  "
328 |       ]
329 |      },
330 |      "execution_count": 8,
331 |      "metadata": {},
332 |      "output_type": "execute_result"
333 |     }
334 |    ],
335 |    "source": [
336 |     "pandasDF.head(4)"
337 |    ]
338 |   },
339 |   {
340 |    "cell_type": "code",
341 |    "execution_count": 9,
342 |    "metadata": {
343 |     "collapsed": false
344 |    },
345 |    "outputs": [
346 |     {
347 |      "data": {
348 |       "text/plain": [
349 |        "[Row(PassengerId=u'1', Survived=u'0', Pclass=u'3', Name=u'Braund, Mr. Owen Harris', Sex=u'male', Age=u'22', SibSp=u'1', Parch=u'0', Ticket=u'A/5 21171', Fare=u'7.25', Cabin=u'', Embarked=u'S'),\n",
350 |        " Row(PassengerId=u'2', Survived=u'1', Pclass=u'1', Name=u'Cumings, Mrs. John Bradley (Florence Briggs Thayer)', Sex=u'female', Age=u'38', SibSp=u'1', Parch=u'0', Ticket=u'PC 17599', Fare=u'71.2833', Cabin=u'C85', Embarked=u'C'),\n",
351 |        " Row(PassengerId=u'3', Survived=u'1', Pclass=u'3', Name=u'Heikkinen, Miss. Laina', Sex=u'female', Age=u'26', SibSp=u'0', Parch=u'0', Ticket=u'STON/O2. 3101282', Fare=u'7.925', Cabin=u'', Embarked=u'S'),\n",
352 |        " Row(PassengerId=u'4', Survived=u'1', Pclass=u'1', Name=u'Futrelle, Mrs. Jacques Heath (Lily May Peel)', Sex=u'female', Age=u'35', SibSp=u'1', Parch=u'0', Ticket=u'113803', Fare=u'53.1', Cabin=u'C123', Embarked=u'S')]"
353 |       ]
354 |      },
355 |      "execution_count": 9,
356 |      "metadata": {},
357 |      "output_type": "execute_result"
358 |     }
359 |    ],
360 |    "source": [
361 |     "sparkDF.head(4)"
362 |    ]
363 |   },
364 |   {
365 |    "cell_type": "code",
366 |    "execution_count": 10,
367 |    "metadata": {
368 |     "collapsed": false
369 |    },
370 |    "outputs": [
371 |     {
372 |      "name": "stdout",
373 |      "output_type": "stream",
374 |      "text": [
375 |       "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
376 |       "|PassengerId|Survived|Pclass|                Name|   Sex|Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
377 |       "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
378 |       "|          1|       0|     3|Braund, Mr. Owen ...|  male| 22|    1|    0|       A/5 21171|   7.25|     |       S|\n",
379 |       "|          2|       1|     1|Cumings, Mrs. Joh...|female| 38|    1|    0|        PC 17599|71.2833|  C85|       C|\n",
380 |       "|          3|       1|     3|Heikkinen, Miss. ...|female| 26|    0|    0|STON/O2. 3101282|  7.925|     |       S|\n",
381 |       "|          4|       1|     1|Futrelle, Mrs. Ja...|female| 35|    1|    0|          113803|   53.1| C123|       S|\n",
382 |       "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
383 |       "\n"
384 |      ]
385 |     }
386 |    ],
387 |    "source": [
388 |     "sparkDF.show(4)"
389 |    ]
390 |   },
391 |   {
392 |    "cell_type": "markdown",
393 |    "metadata": {},
394 |    "source": [
395 |     "*.tail()* exists with Pandas, but not with Spark"
396 |    ]
397 |   },
398 |   {
399 |    "cell_type": "code",
400 |    "execution_count": 11,
401 |    "metadata": {
402 |     "collapsed": false
403 |    },
404 |    "outputs": [
405 |     {
406 |      "data": {
407 |       "text/html": [
408 |        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
409 |        "<table border=\"1\" class=\"dataframe\">\n",
410 |        "  <thead>\n",
411 |        "    <tr style=\"text-align: right;\">\n",
412 |        "      <th></th>\n",
413 |        "      <th>PassengerId</th>\n",
414 |        "      <th>Survived</th>\n",
415 |        "      <th>Pclass</th>\n",
416 |        "      <th>Name</th>\n",
417 |        "      <th>Sex</th>\n",
418 |        "      <th>Age</th>\n",
419 |        "      <th>SibSp</th>\n",
420 |        "      <th>Parch</th>\n",
421 |        "      <th>Ticket</th>\n",
422 |        "      <th>Fare</th>\n",
423 |        "      <th>Cabin</th>\n",
424 |        "      <th>Embarked</th>\n",
425 |        "    </tr>\n",
426 |        "  </thead>\n",
427 |        "  <tbody>\n",
428 |        "    <tr>\n",
429 |        "      <th>886</th>\n",
430 |        "      <td>887</td>\n",
431 |        "      <td>0</td>\n",
432 |        "      <td>2</td>\n",
433 |        "      <td>Montvila, Rev. Juozas</td>\n",
434 |        "      <td>male</td>\n",
435 |        "      <td>27</td>\n",
436 |        "      <td>0</td>\n",
437 |        "      <td>0</td>\n",
438 |        "      <td>211536</td>\n",
439 |        "      <td>13.00</td>\n",
440 |        "      <td>NaN</td>\n",
441 |        "      <td>S</td>\n",
442 |        "    </tr>\n",
443 |        "    <tr>\n",
444 |        "      <th>887</th>\n",
445 |        "      <td>888</td>\n",
446 |        "      <td>1</td>\n",
447 |        "      <td>1</td>\n",
448 |        "      <td>Graham, Miss. Margaret Edith</td>\n",
449 |        "      <td>female</td>\n",
450 |        "      <td>19</td>\n",
451 |        "      <td>0</td>\n",
452 |        "      <td>0</td>\n",
453 |        "      <td>112053</td>\n",
454 |        "      <td>30.00</td>\n",
455 |        "      <td>B42</td>\n",
456 |        "      <td>S</td>\n",
457 |        "    </tr>\n",
458 |        "    <tr>\n",
459 |        "      <th>888</th>\n",
460 |        "      <td>889</td>\n",
461 |        "      <td>0</td>\n",
462 |        "      <td>3</td>\n",
463 |        "      <td>Johnston, Miss. Catherine Helen \"Carrie\"</td>\n",
464 |        "      <td>female</td>\n",
465 |        "      <td>NaN</td>\n",
466 |        "      <td>1</td>\n",
467 |        "      <td>2</td>\n",
468 |        "      <td>W./C. 6607</td>\n",
469 |        "      <td>23.45</td>\n",
470 |        "      <td>NaN</td>\n",
471 |        "      <td>S</td>\n",
472 |        "    </tr>\n",
473 |        "    <tr>\n",
474 |        "      <th>889</th>\n",
475 |        "      <td>890</td>\n",
476 |        "      <td>1</td>\n",
477 |        "      <td>1</td>\n",
478 |        "      <td>Behr, Mr. Karl Howell</td>\n",
479 |        "      <td>male</td>\n",
480 |        "      <td>26</td>\n",
481 |        "      <td>0</td>\n",
482 |        "      <td>0</td>\n",
483 |        "      <td>111369</td>\n",
484 |        "      <td>30.00</td>\n",
485 |        "      <td>C148</td>\n",
486 |        "      <td>C</td>\n",
487 |        "    </tr>\n",
488 |        "    <tr>\n",
489 |        "      <th>890</th>\n",
490 |        "      <td>891</td>\n",
491 |        "      <td>0</td>\n",
492 |        "      <td>3</td>\n",
493 |        "      <td>Dooley, Mr. Patrick</td>\n",
494 |        "      <td>male</td>\n",
495 |        "      <td>32</td>\n",
496 |        "      <td>0</td>\n",
497 |        "      <td>0</td>\n",
498 |        "      <td>370376</td>\n",
499 |        "      <td>7.75</td>\n",
500 |        "      <td>NaN</td>\n",
501 |        "      <td>Q</td>\n",
502 |        "    </tr>\n",
503 |        "  </tbody>\n",
504 |        "</table>\n",
505 |        "</div>"
506 |       ],
507 |       "text/plain": [
508 |        "     PassengerId  Survived  Pclass                                      Name  \\\n",
509 |        "886          887         0       2                     Montvila, Rev. Juozas   \n",
510 |        "887          888         1       1              Graham, Miss. Margaret Edith   \n",
511 |        "888          889         0       3  Johnston, Miss. Catherine Helen \"Carrie\"   \n",
512 |        "889          890         1       1                     Behr, Mr. Karl Howell   \n",
513 |        "890          891         0       3                       Dooley, Mr. Patrick   \n",
514 |        "\n",
515 |        "        Sex  Age  SibSp  Parch      Ticket   Fare Cabin Embarked  \n",
516 |        "886    male   27      0      0      211536  13.00   NaN        S  \n",
517 |        "887  female   19      0      0      112053  30.00   B42        S  \n",
518 |        "888  female  NaN      1      2  W./C. 6607  23.45   NaN        S  \n",
519 |        "889    male   26      0      0      111369  30.00  C148        C  \n",
520 |        "890    male   32      0      0      370376   7.75   NaN        Q  "
521 |       ]
522 |      },
523 |      "execution_count": 11,
524 |      "metadata": {},
525 |      "output_type": "execute_result"
526 |     }
527 |    ],
528 |    "source": [
529 |     "pandasDF.tail(5)"
530 |    ]
531 |   },
532 |   {
533 |    "cell_type": "code",
534 |    "execution_count": 12,
535 |    "metadata": {
536 |     "collapsed": false
537 |    },
538 |    "outputs": [
539 |     {
540 |      "ename": "AttributeError",
541 |      "evalue": "'DataFrame' object has no attribute 'tail'",
542 |      "output_type": "error",
543 |      "traceback": [
544 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
545 |       "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
546 |       "\u001b[0;32m<ipython-input-12-92d96a598db0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msparkDF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtail\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
547 |       "\u001b[0;32m/usr/local/Cellar/apache-spark/1.4.0/libexec/python/pyspark/sql/dataframe.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    716\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    717\u001b[0m             raise AttributeError(\n\u001b[0;32m--> 718\u001b[0;31m                 \"'%s' object has no attribute '%s'\" % (self.__class__.__name__, name))\n\u001b[0m\u001b[1;32m    719\u001b[0m         \u001b[0mjc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    720\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mColumn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
548 |       "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'tail'"
549 |      ]
550 |     }
551 |    ],
552 |    "source": [
553 |     "sparkDF.tail(5)"
554 |    ]
555 |   },
556 |   {
557 |    "cell_type": "markdown",
558 |    "metadata": {},
559 |    "source": [
560 |     "## Inferring types"
561 |    ]
562 |   },
563 |   {
564 |    "cell_type": "markdown",
565 |    "metadata": {},
566 |    "source": [
567 |     "With Pandas, you rarely have to bother with types : they are inferred. \n",
568 |     "\n",
569 |     "With Spark DataFrames, default types are assumed to be “strings”."
570 |    ]
571 |   },
572 |   {
573 |    "cell_type": "code",
574 |    "execution_count": 13,
575 |    "metadata": {
576 |     "collapsed": false
577 |    },
578 |    "outputs": [
579 |     {
580 |      "data": {
581 |       "text/plain": [
582 |        "PassengerId      int64\n",
583 |        "Survived         int64\n",
584 |        "Pclass           int64\n",
585 |        "Name            object\n",
586 |        "Sex             object\n",
587 |        "Age            float64\n",
588 |        "SibSp            int64\n",
589 |        "Parch            int64\n",
590 |        "Ticket          object\n",
591 |        "Fare           float64\n",
592 |        "Cabin           object\n",
593 |        "Embarked        object\n",
594 |        "dtype: object"
595 |       ]
596 |      },
597 |      "execution_count": 13,
598 |      "metadata": {},
599 |      "output_type": "execute_result"
600 |     }
601 |    ],
602 |    "source": [
603 |     "pandasDF.dtypes"
604 |    ]
605 |   },
606 |   {
607 |    "cell_type": "code",
608 |    "execution_count": 14,
609 |    "metadata": {
610 |     "collapsed": false
611 |    },
612 |    "outputs": [
613 |     {
614 |      "data": {
615 |       "text/plain": [
616 |        "StructType(List(StructField(PassengerId,StringType,true),StructField(Survived,StringType,true),StructField(Pclass,StringType,true),StructField(Name,StringType,true),StructField(Sex,StringType,true),StructField(Age,StringType,true),StructField(SibSp,StringType,true),StructField(Parch,StringType,true),StructField(Ticket,StringType,true),StructField(Fare,StringType,true),StructField(Cabin,StringType,true),StructField(Embarked,StringType,true)))"
617 |       ]
618 |      },
619 |      "execution_count": 14,
620 |      "metadata": {},
621 |      "output_type": "execute_result"
622 |     }
623 |    ],
624 |    "source": [
625 |     "sparkDF.schema"
626 |    ]
627 |   },
628 |   {
629 |    "cell_type": "markdown",
630 |    "metadata": {},
631 |    "source": [
632 |     "With Spark and Pandas, you can change type with *.astype()*"
633 |    ]
634 |   },
635 |   {
636 |    "cell_type": "code",
637 |    "execution_count": 15,
638 |    "metadata": {
639 |     "collapsed": false
640 |    },
641 |    "outputs": [
642 |     {
643 |      "data": {
644 |       "text/plain": [
645 |        "StructType(List(StructField(PassengerId,StringType,true),StructField(Survived,StringType,true),StructField(Pclass,StringType,true),StructField(Name,StringType,true),StructField(Sex,StringType,true),StructField(SibSp,StringType,true),StructField(Parch,StringType,true),StructField(Ticket,StringType,true),StructField(Fare,StringType,true),StructField(Cabin,StringType,true),StructField(Embarked,StringType,true),StructField(Age2,FloatType,true)))"
646 |       ]
647 |      },
648 |      "execution_count": 15,
649 |      "metadata": {},
650 |      "output_type": "execute_result"
651 |     }
652 |    ],
653 |    "source": [
654 |     "sparkDF = sparkDF.withColumn('Age2', sparkDF['Age'].astype(\"float\"))\n",
655 |     "sparkDF = sparkDF.drop('Age')\n",
656 |     "sparkDF.schema"
657 |    ]
658 |   },
659 |   {
660 |    "cell_type": "markdown",
661 |    "metadata": {},
662 |    "source": [
663 |     "## Describing"
664 |    ]
665 |   },
666 |   {
667 |    "cell_type": "markdown",
668 |    "metadata": {},
669 |    "source": [
670 |     "Pandas and Spark *.describe()* give slightly different results for two reasons :\n",
671 |     "\n",
672 |     "* In Pandas, NaN values are excluded. In Spark, NaN values make that computation of mean and standard deviation fail. \n",
673 |     "\n",
674 |     "* Standard deviation is not computed in the same way. Unbiased (or corrected) standard deviation by default in Pandas, and uncorrected standard deviation in Spark. The difference is the use of N-1 instead of N on the denominator."
675 |    ]
676 |   },
677 |   {
678 |    "cell_type": "code",
679 |    "execution_count": 16,
680 |    "metadata": {
681 |     "collapsed": false
682 |    },
683 |    "outputs": [
684 |     {
685 |      "data": {
686 |       "text/html": [
687 |        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
688 |        "<table border=\"1\" class=\"dataframe\">\n",
689 |        "  <thead>\n",
690 |        "    <tr style=\"text-align: right;\">\n",
691 |        "      <th></th>\n",
692 |        "      <th>Survived</th>\n",
693 |        "      <th>Age</th>\n",
694 |        "      <th>Fare</th>\n",
695 |        "      <th>Pclass</th>\n",
696 |        "    </tr>\n",
697 |        "  </thead>\n",
698 |        "  <tbody>\n",
699 |        "    <tr>\n",
700 |        "      <th>count</th>\n",
701 |        "      <td>891.000000</td>\n",
702 |        "      <td>714.000000</td>\n",
703 |        "      <td>891.000000</td>\n",
704 |        "      <td>891.000000</td>\n",
705 |        "    </tr>\n",
706 |        "    <tr>\n",
707 |        "      <th>mean</th>\n",
708 |        "      <td>0.383838</td>\n",
709 |        "      <td>29.699118</td>\n",
710 |        "      <td>32.204208</td>\n",
711 |        "      <td>2.308642</td>\n",
712 |        "    </tr>\n",
713 |        "    <tr>\n",
714 |        "      <th>std</th>\n",
715 |        "      <td>0.486592</td>\n",
716 |        "      <td>14.526497</td>\n",
717 |        "      <td>49.693429</td>\n",
718 |        "      <td>0.836071</td>\n",
719 |        "    </tr>\n",
720 |        "    <tr>\n",
721 |        "      <th>min</th>\n",
722 |        "      <td>0.000000</td>\n",
723 |        "      <td>0.420000</td>\n",
724 |        "      <td>0.000000</td>\n",
725 |        "      <td>1.000000</td>\n",
726 |        "    </tr>\n",
727 |        "    <tr>\n",
728 |        "      <th>25%</th>\n",
729 |        "      <td>0.000000</td>\n",
730 |        "      <td>20.125000</td>\n",
731 |        "      <td>7.910400</td>\n",
732 |        "      <td>2.000000</td>\n",
733 |        "    </tr>\n",
734 |        "    <tr>\n",
735 |        "      <th>50%</th>\n",
736 |        "      <td>0.000000</td>\n",
737 |        "      <td>28.000000</td>\n",
738 |        "      <td>14.454200</td>\n",
739 |        "      <td>3.000000</td>\n",
740 |        "    </tr>\n",
741 |        "    <tr>\n",
742 |        "      <th>75%</th>\n",
743 |        "      <td>1.000000</td>\n",
744 |        "      <td>38.000000</td>\n",
745 |        "      <td>31.000000</td>\n",
746 |        "      <td>3.000000</td>\n",
747 |        "    </tr>\n",
748 |        "    <tr>\n",
749 |        "      <th>max</th>\n",
750 |        "      <td>1.000000</td>\n",
751 |        "      <td>80.000000</td>\n",
752 |        "      <td>512.329200</td>\n",
753 |        "      <td>3.000000</td>\n",
754 |        "    </tr>\n",
755 |        "  </tbody>\n",
756 |        "</table>\n",
757 |        "</div>"
758 |       ],
759 |       "text/plain": [
760 |        "         Survived         Age        Fare      Pclass\n",
761 |        "count  891.000000  714.000000  891.000000  891.000000\n",
762 |        "mean     0.383838   29.699118   32.204208    2.308642\n",
763 |        "std      0.486592   14.526497   49.693429    0.836071\n",
764 |        "min      0.000000    0.420000    0.000000    1.000000\n",
765 |        "25%      0.000000   20.125000    7.910400    2.000000\n",
766 |        "50%      0.000000   28.000000   14.454200    3.000000\n",
767 |        "75%      1.000000   38.000000   31.000000    3.000000\n",
768 |        "max      1.000000   80.000000  512.329200    3.000000"
769 |       ]
770 |      },
771 |      "execution_count": 16,
772 |      "metadata": {},
773 |      "output_type": "execute_result"
774 |     }
775 |    ],
776 |    "source": [
777 |     "pandasDF[['Survived', 'Age', 'Fare', 'Pclass']].describe()"
778 |    ]
779 |   },
780 |   {
781 |    "cell_type": "code",
782 |    "execution_count": 17,
783 |    "metadata": {
784 |     "collapsed": false
785 |    },
786 |    "outputs": [
787 |     {
788 |      "name": "stdout",
789 |      "output_type": "stream",
790 |      "text": [
791 |       "+-------+------------------+----+-----------------+------------------+\n",
792 |       "|summary|          Survived| Age|             Fare|            Pclass|\n",
793 |       "+-------+------------------+----+-----------------+------------------+\n",
794 |       "|  count|               891| 891|              891|               891|\n",
795 |       "|   mean|0.3838383838383838| NaN|32.20420796857464| 2.308641975308642|\n",
796 |       "| stddev|0.4863193178670999| NaN| 49.6655344447741|0.8356019334795164|\n",
797 |       "|    min|                 0|0.42|              0.0|                 1|\n",
798 |       "|    max|                 1|80.0|         512.3292|                 3|\n",
799 |       "+-------+------------------+----+-----------------+------------------+\n",
800 |       "\n"
801 |      ]
802 |     }
803 |    ],
804 |    "source": [
805 |     "sqlContext.createDataFrame(pandasDF[['Survived', 'Age', 'Fare', 'Pclass']]).describe().show()"
806 |    ]
807 |   },
808 |   {
809 |    "cell_type": "code",
810 |    "execution_count": 18,
811 |    "metadata": {
812 |     "collapsed": false
813 |    },
814 |    "outputs": [
815 |     {
816 |      "name": "stdout",
817 |      "output_type": "stream",
818 |      "text": [
819 |       "+-------+------------------+\n",
820 |       "|summary|              Age2|\n",
821 |       "+-------+------------------+\n",
822 |       "|  count|               714|\n",
823 |       "|   mean| 29.69911764704046|\n",
824 |       "| stddev|14.516321150855628|\n",
825 |       "|    min|              0.42|\n",
826 |       "|    max|              80.0|\n",
827 |       "+-------+------------------+\n",
828 |       "\n"
829 |      ]
830 |     }
831 |    ],
832 |    "source": [
833 |     "# This only shows numerical data types, see above\n",
834 |     "sparkDF.describe().show()"
835 |    ]
836 |   },
837 |   {
838 |    "cell_type": "markdown",
839 |    "metadata": {},
840 |    "source": [
841 |     "## Wrangling"
842 |    ]
843 |   },
844 |   {
845 |    "cell_type": "markdown",
846 |    "metadata": {},
847 |    "source": [
848 |     "In Spark, you cannot create new columns with the '[ ]' operator. \n",
849 |     "\n",
850 |     "You have to use *.withColumn()*"
851 |    ]
852 |   },
853 |   {
854 |    "cell_type": "code",
855 |    "execution_count": 19,
856 |    "metadata": {
857 |     "collapsed": false
858 |    },
859 |    "outputs": [],
860 |    "source": [
861 |     "pandasDF2 = pandasDF.copy()\n",
862 |     "pandasDF2['Fare2'] = pandasDF['Fare'] + 2"
863 |    ]
864 |   },
865 |   {
866 |    "cell_type": "code",
867 |    "execution_count": 20,
868 |    "metadata": {
869 |     "collapsed": false
870 |    },
871 |    "outputs": [
872 |     {
873 |      "data": {
874 |       "text/html": [
875 |        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
876 |        "<table border=\"1\" class=\"dataframe\">\n",
877 |        "  <thead>\n",
878 |        "    <tr style=\"text-align: right;\">\n",
879 |        "      <th></th>\n",
880 |        "      <th>Fare</th>\n",
881 |        "      <th>Fare2</th>\n",
882 |        "    </tr>\n",
883 |        "  </thead>\n",
884 |        "  <tbody>\n",
885 |        "    <tr>\n",
886 |        "      <th>0</th>\n",
887 |        "      <td>7.2500</td>\n",
888 |        "      <td>9.2500</td>\n",
889 |        "    </tr>\n",
890 |        "    <tr>\n",
891 |        "      <th>1</th>\n",
892 |        "      <td>71.2833</td>\n",
893 |        "      <td>73.2833</td>\n",
894 |        "    </tr>\n",
895 |        "    <tr>\n",
896 |        "      <th>2</th>\n",
897 |        "      <td>7.9250</td>\n",
898 |        "      <td>9.9250</td>\n",
899 |        "    </tr>\n",
900 |        "    <tr>\n",
901 |        "      <th>3</th>\n",
902 |        "      <td>53.1000</td>\n",
903 |        "      <td>55.1000</td>\n",
904 |        "    </tr>\n",
905 |        "    <tr>\n",
906 |        "      <th>4</th>\n",
907 |        "      <td>8.0500</td>\n",
908 |        "      <td>10.0500</td>\n",
909 |        "    </tr>\n",
910 |        "  </tbody>\n",
911 |        "</table>\n",
912 |        "</div>"
913 |       ],
914 |       "text/plain": [
915 |        "      Fare    Fare2\n",
916 |        "0   7.2500   9.2500\n",
917 |        "1  71.2833  73.2833\n",
918 |        "2   7.9250   9.9250\n",
919 |        "3  53.1000  55.1000\n",
920 |        "4   8.0500  10.0500"
921 |       ]
922 |      },
923 |      "execution_count": 20,
924 |      "metadata": {},
925 |      "output_type": "execute_result"
926 |     }
927 |    ],
928 |    "source": [
929 |     "pandasDF2[['Fare', 'Fare2']].head(5)"
930 |    ]
931 |   },
932 |   {
933 |    "cell_type": "code",
934 |    "execution_count": 21,
935 |    "metadata": {
936 |     "collapsed": true
937 |    },
938 |    "outputs": [],
939 |    "source": [
940 |     "sparkDF = sparkDF.withColumn('Fare2', sparkDF['Fare'] + 2)"
941 |    ]
942 |   },
943 |   {
944 |    "cell_type": "code",
945 |    "execution_count": 22,
946 |    "metadata": {
947 |     "collapsed": false
948 |    },
949 |    "outputs": [
950 |     {
951 |      "name": "stdout",
952 |      "output_type": "stream",
953 |      "text": [
954 |       "+-------+-------+\n",
955 |       "|   Fare|  Fare2|\n",
956 |       "+-------+-------+\n",
957 |       "|   7.25|   9.25|\n",
958 |       "|71.2833|73.2833|\n",
959 |       "|  7.925|  9.925|\n",
960 |       "|   53.1|   55.1|\n",
961 |       "|   8.05|  10.05|\n",
962 |       "+-------+-------+\n",
963 |       "\n"
964 |      ]
965 |     }
966 |    ],
967 |    "source": [
968 |     "sparkDF[['Fare', 'Fare2']].show(5)"
969 |    ]
970 |   }
971 |  ],
972 |  "metadata": {
973 |   "kernelspec": {
974 |    "display_name": "Python 2",
975 |    "language": "python",
976 |    "name": "python2"
977 |   },
978 |   "language_info": {
979 |    "codemirror_mode": {
980 |     "name": "ipython",
981 |     "version": 2
982 |    },
983 |    "file_extension": ".py",
984 |    "mimetype": "text/x-python",
985 |    "name": "python",
986 |    "nbconvert_exporter": "python",
987 |    "pygments_lexer": "ipython2",
988 |    "version": "2.7.10"
989 |   }
990 |  },
991 |  "nbformat": 4,
992 |  "nbformat_minor": 0
993 | }
994 | 


--------------------------------------------------------------------------------
/TensorFlow Tutorial.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "metadata": {},
  6 |    "source": [
  7 |     "# A TensorFlow tutorial"
  8 |    ]
  9 |   },
 10 |   {
 11 |    "cell_type": "code",
 12 |    "execution_count": 1,
 13 |    "metadata": {
 14 |     "collapsed": false
 15 |    },
 16 |    "outputs": [],
 17 |    "source": [
 18 |     "%matplotlib inline\n",
 19 |     "from matplotlib import pyplot\n",
 20 |     "import numpy as np\n",
 21 |     "import pandas as pd\n",
 22 |     "from sklearn import preprocessing\n",
 23 |     "from sklearn.model_selection import train_test_split\n",
 24 |     "from sklearn.linear_model import LogisticRegression\n",
 25 |     "from sklearn.ensemble import RandomForestClassifier\n",
 26 |     "from sklearn.metrics import log_loss\n",
 27 |     "import tensorflow as tf"
 28 |    ]
 29 |   },
 30 |   {
 31 |    "cell_type": "code",
 32 |    "execution_count": 2,
 33 |    "metadata": {
 34 |     "collapsed": false
 35 |    },
 36 |    "outputs": [],
 37 |    "source": [
 38 |     "# data source = https://www.kaggle.com/c/leaf-classification\n",
 39 |     "data = pd.read_csv('01-DATA/train.csv')"
 40 |    ]
 41 |   },
 42 |   {
 43 |    "cell_type": "code",
 44 |    "execution_count": 3,
 45 |    "metadata": {
 46 |     "collapsed": false
 47 |    },
 48 |    "outputs": [
 49 |     {
 50 |      "data": {
 51 |       "text/plain": [
 52 |        "(990, 194)"
 53 |       ]
 54 |      },
 55 |      "execution_count": 3,
 56 |      "metadata": {},
 57 |      "output_type": "execute_result"
 58 |     }
 59 |    ],
 60 |    "source": [
 61 |     "data.shape"
 62 |    ]
 63 |   },
 64 |   {
 65 |    "cell_type": "code",
 66 |    "execution_count": 4,
 67 |    "metadata": {
 68 |     "collapsed": false
 69 |    },
 70 |    "outputs": [
 71 |     {
 72 |      "data": {
 73 |       "text/html": [
 74 |        "<div>\n",
 75 |        "<table border=\"1\" class=\"dataframe\">\n",
 76 |        "  <thead>\n",
 77 |        "    <tr style=\"text-align: right;\">\n",
 78 |        "      <th></th>\n",
 79 |        "      <th>id</th>\n",
 80 |        "      <th>species</th>\n",
 81 |        "      <th>margin1</th>\n",
 82 |        "      <th>margin2</th>\n",
 83 |        "      <th>margin3</th>\n",
 84 |        "      <th>margin4</th>\n",
 85 |        "      <th>margin5</th>\n",
 86 |        "      <th>margin6</th>\n",
 87 |        "      <th>margin7</th>\n",
 88 |        "      <th>margin8</th>\n",
 89 |        "      <th>...</th>\n",
 90 |        "      <th>texture55</th>\n",
 91 |        "      <th>texture56</th>\n",
 92 |        "      <th>texture57</th>\n",
 93 |        "      <th>texture58</th>\n",
 94 |        "      <th>texture59</th>\n",
 95 |        "      <th>texture60</th>\n",
 96 |        "      <th>texture61</th>\n",
 97 |        "      <th>texture62</th>\n",
 98 |        "      <th>texture63</th>\n",
 99 |        "      <th>texture64</th>\n",
100 |        "    </tr>\n",
101 |        "  </thead>\n",
102 |        "  <tbody>\n",
103 |        "    <tr>\n",
104 |        "      <th>0</th>\n",
105 |        "      <td>1</td>\n",
106 |        "      <td>Acer_Opalus</td>\n",
107 |        "      <td>0.007812</td>\n",
108 |        "      <td>0.023438</td>\n",
109 |        "      <td>0.023438</td>\n",
110 |        "      <td>0.003906</td>\n",
111 |        "      <td>0.011719</td>\n",
112 |        "      <td>0.009766</td>\n",
113 |        "      <td>0.027344</td>\n",
114 |        "      <td>0.0</td>\n",
115 |        "      <td>...</td>\n",
116 |        "      <td>0.007812</td>\n",
117 |        "      <td>0.000000</td>\n",
118 |        "      <td>0.002930</td>\n",
119 |        "      <td>0.002930</td>\n",
120 |        "      <td>0.035156</td>\n",
121 |        "      <td>0.0</td>\n",
122 |        "      <td>0.0</td>\n",
123 |        "      <td>0.004883</td>\n",
124 |        "      <td>0.000000</td>\n",
125 |        "      <td>0.025391</td>\n",
126 |        "    </tr>\n",
127 |        "    <tr>\n",
128 |        "      <th>1</th>\n",
129 |        "      <td>2</td>\n",
130 |        "      <td>Pterocarya_Stenoptera</td>\n",
131 |        "      <td>0.005859</td>\n",
132 |        "      <td>0.000000</td>\n",
133 |        "      <td>0.031250</td>\n",
134 |        "      <td>0.015625</td>\n",
135 |        "      <td>0.025391</td>\n",
136 |        "      <td>0.001953</td>\n",
137 |        "      <td>0.019531</td>\n",
138 |        "      <td>0.0</td>\n",
139 |        "      <td>...</td>\n",
140 |        "      <td>0.000977</td>\n",
141 |        "      <td>0.000000</td>\n",
142 |        "      <td>0.000000</td>\n",
143 |        "      <td>0.000977</td>\n",
144 |        "      <td>0.023438</td>\n",
145 |        "      <td>0.0</td>\n",
146 |        "      <td>0.0</td>\n",
147 |        "      <td>0.000977</td>\n",
148 |        "      <td>0.039062</td>\n",
149 |        "      <td>0.022461</td>\n",
150 |        "    </tr>\n",
151 |        "    <tr>\n",
152 |        "      <th>2</th>\n",
153 |        "      <td>3</td>\n",
154 |        "      <td>Quercus_Hartwissiana</td>\n",
155 |        "      <td>0.005859</td>\n",
156 |        "      <td>0.009766</td>\n",
157 |        "      <td>0.019531</td>\n",
158 |        "      <td>0.007812</td>\n",
159 |        "      <td>0.003906</td>\n",
160 |        "      <td>0.005859</td>\n",
161 |        "      <td>0.068359</td>\n",
162 |        "      <td>0.0</td>\n",
163 |        "      <td>...</td>\n",
164 |        "      <td>0.154300</td>\n",
165 |        "      <td>0.000000</td>\n",
166 |        "      <td>0.005859</td>\n",
167 |        "      <td>0.000977</td>\n",
168 |        "      <td>0.007812</td>\n",
169 |        "      <td>0.0</td>\n",
170 |        "      <td>0.0</td>\n",
171 |        "      <td>0.000000</td>\n",
172 |        "      <td>0.020508</td>\n",
173 |        "      <td>0.002930</td>\n",
174 |        "    </tr>\n",
175 |        "    <tr>\n",
176 |        "      <th>3</th>\n",
177 |        "      <td>5</td>\n",
178 |        "      <td>Tilia_Tomentosa</td>\n",
179 |        "      <td>0.000000</td>\n",
180 |        "      <td>0.003906</td>\n",
181 |        "      <td>0.023438</td>\n",
182 |        "      <td>0.005859</td>\n",
183 |        "      <td>0.021484</td>\n",
184 |        "      <td>0.019531</td>\n",
185 |        "      <td>0.023438</td>\n",
186 |        "      <td>0.0</td>\n",
187 |        "      <td>...</td>\n",
188 |        "      <td>0.000000</td>\n",
189 |        "      <td>0.000977</td>\n",
190 |        "      <td>0.000000</td>\n",
191 |        "      <td>0.000000</td>\n",
192 |        "      <td>0.020508</td>\n",
193 |        "      <td>0.0</td>\n",
194 |        "      <td>0.0</td>\n",
195 |        "      <td>0.017578</td>\n",
196 |        "      <td>0.000000</td>\n",
197 |        "      <td>0.047852</td>\n",
198 |        "    </tr>\n",
199 |        "    <tr>\n",
200 |        "      <th>4</th>\n",
201 |        "      <td>6</td>\n",
202 |        "      <td>Quercus_Variabilis</td>\n",
203 |        "      <td>0.005859</td>\n",
204 |        "      <td>0.003906</td>\n",
205 |        "      <td>0.048828</td>\n",
206 |        "      <td>0.009766</td>\n",
207 |        "      <td>0.013672</td>\n",
208 |        "      <td>0.015625</td>\n",
209 |        "      <td>0.005859</td>\n",
210 |        "      <td>0.0</td>\n",
211 |        "      <td>...</td>\n",
212 |        "      <td>0.096680</td>\n",
213 |        "      <td>0.000000</td>\n",
214 |        "      <td>0.021484</td>\n",
215 |        "      <td>0.000000</td>\n",
216 |        "      <td>0.000000</td>\n",
217 |        "      <td>0.0</td>\n",
218 |        "      <td>0.0</td>\n",
219 |        "      <td>0.000000</td>\n",
220 |        "      <td>0.000000</td>\n",
221 |        "      <td>0.031250</td>\n",
222 |        "    </tr>\n",
223 |        "  </tbody>\n",
224 |        "</table>\n",
225 |        "<p>5 rows × 194 columns</p>\n",
226 |        "</div>"
227 |       ],
228 |       "text/plain": [
229 |        "   id                species   margin1   margin2   margin3   margin4  \\\n",
230 |        "0   1            Acer_Opalus  0.007812  0.023438  0.023438  0.003906   \n",
231 |        "1   2  Pterocarya_Stenoptera  0.005859  0.000000  0.031250  0.015625   \n",
232 |        "2   3   Quercus_Hartwissiana  0.005859  0.009766  0.019531  0.007812   \n",
233 |        "3   5        Tilia_Tomentosa  0.000000  0.003906  0.023438  0.005859   \n",
234 |        "4   6     Quercus_Variabilis  0.005859  0.003906  0.048828  0.009766   \n",
235 |        "\n",
236 |        "    margin5   margin6   margin7  margin8    ...      texture55  texture56  \\\n",
237 |        "0  0.011719  0.009766  0.027344      0.0    ...       0.007812   0.000000   \n",
238 |        "1  0.025391  0.001953  0.019531      0.0    ...       0.000977   0.000000   \n",
239 |        "2  0.003906  0.005859  0.068359      0.0    ...       0.154300   0.000000   \n",
240 |        "3  0.021484  0.019531  0.023438      0.0    ...       0.000000   0.000977   \n",
241 |        "4  0.013672  0.015625  0.005859      0.0    ...       0.096680   0.000000   \n",
242 |        "\n",
243 |        "   texture57  texture58  texture59  texture60  texture61  texture62  \\\n",
244 |        "0   0.002930   0.002930   0.035156        0.0        0.0   0.004883   \n",
245 |        "1   0.000000   0.000977   0.023438        0.0        0.0   0.000977   \n",
246 |        "2   0.005859   0.000977   0.007812        0.0        0.0   0.000000   \n",
247 |        "3   0.000000   0.000000   0.020508        0.0        0.0   0.017578   \n",
248 |        "4   0.021484   0.000000   0.000000        0.0        0.0   0.000000   \n",
249 |        "\n",
250 |        "   texture63  texture64  \n",
251 |        "0   0.000000   0.025391  \n",
252 |        "1   0.039062   0.022461  \n",
253 |        "2   0.020508   0.002930  \n",
254 |        "3   0.000000   0.047852  \n",
255 |        "4   0.000000   0.031250  \n",
256 |        "\n",
257 |        "[5 rows x 194 columns]"
258 |       ]
259 |      },
260 |      "execution_count": 4,
261 |      "metadata": {},
262 |      "output_type": "execute_result"
263 |     }
264 |    ],
265 |    "source": [
266 |     "data.head()"
267 |    ]
268 |   },
269 |   {
270 |    "cell_type": "code",
271 |    "execution_count": 5,
272 |    "metadata": {
273 |     "collapsed": false
274 |    },
275 |    "outputs": [
276 |     {
277 |      "data": {
278 |       "text/plain": [
279 |        "99"
280 |       ]
281 |      },
282 |      "execution_count": 5,
283 |      "metadata": {},
284 |      "output_type": "execute_result"
285 |     }
286 |    ],
287 |    "source": [
288 |     "nb_classes = len(data['species'].unique())\n",
289 |     "nb_classes"
290 |    ]
291 |   },
292 |   {
293 |    "cell_type": "code",
294 |    "execution_count": 6,
295 |    "metadata": {
296 |     "collapsed": true
297 |    },
298 |    "outputs": [],
299 |    "source": [
300 |     "le = preprocessing.LabelEncoder()"
301 |    ]
302 |   },
303 |   {
304 |    "cell_type": "code",
305 |    "execution_count": 7,
306 |    "metadata": {
307 |     "collapsed": false
308 |    },
309 |    "outputs": [],
310 |    "source": [
311 |     "y = le.fit_transform(data['species'])"
312 |    ]
313 |   },
314 |   {
315 |    "cell_type": "code",
316 |    "execution_count": 8,
317 |    "metadata": {
318 |     "collapsed": true
319 |    },
320 |    "outputs": [],
321 |    "source": [
322 |     "X = data.drop('id', 1).drop('species', 1)"
323 |    ]
324 |   },
325 |   {
326 |    "cell_type": "code",
327 |    "execution_count": 9,
328 |    "metadata": {
329 |     "collapsed": false
330 |    },
331 |    "outputs": [],
332 |    "source": [
333 |     "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=487)"
334 |    ]
335 |   },
336 |   {
337 |    "cell_type": "markdown",
338 |    "metadata": {},
339 |    "source": [
340 |     "# Logistic Regression"
341 |    ]
342 |   },
343 |   {
344 |    "cell_type": "code",
345 |    "execution_count": 10,
346 |    "metadata": {
347 |     "collapsed": false
348 |    },
349 |    "outputs": [
350 |     {
351 |      "name": "stdout",
352 |      "output_type": "stream",
353 |      "text": [
354 |       "4.23374595993\n"
355 |      ]
356 |     }
357 |    ],
358 |    "source": [
359 |     "clf = LogisticRegression()\n",
360 |     "clf.fit(X_train, y_train)\n",
361 |     "print log_loss(y_test, clf.predict_proba(X_test))"
362 |    ]
363 |   },
364 |   {
365 |    "cell_type": "markdown",
366 |    "metadata": {},
367 |    "source": [
368 |     "# Random Forest"
369 |    ]
370 |   },
371 |   {
372 |    "cell_type": "code",
373 |    "execution_count": 11,
374 |    "metadata": {
375 |     "collapsed": false
376 |    },
377 |    "outputs": [
378 |     {
379 |      "name": "stdout",
380 |      "output_type": "stream",
381 |      "text": [
382 |       "1.80428024212\n"
383 |      ]
384 |     }
385 |    ],
386 |    "source": [
387 |     "clf = RandomForestClassifier()\n",
388 |     "clf.fit(X_train, y_train)\n",
389 |     "preds = clf.predict(X_test)\n",
390 |     "print log_loss(y_test, clf.predict_proba(X_test))"
391 |    ]
392 |   },
393 |   {
394 |    "cell_type": "markdown",
395 |    "metadata": {
396 |     "collapsed": true
397 |    },
398 |    "source": [
399 |     "# TensorFlow"
400 |    ]
401 |   },
402 |   {
403 |    "cell_type": "code",
404 |    "execution_count": 12,
405 |    "metadata": {
406 |     "collapsed": false
407 |    },
408 |    "outputs": [],
409 |    "source": [
410 |     "graph = tf.Graph()\n",
411 |     "with graph.as_default():\n",
412 |     "    X_train_tf = tf.constant(np.array(X_train).astype(np.float32))\n",
413 |     "    y_train_tf = tf.constant((np.arange(nb_classes) == y_train[:,None]).astype(np.float32))\n",
414 |     "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
415 |     "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
416 |     "    \n",
417 |     "    weights = tf.Variable(\n",
418 |     "        tf.truncated_normal([X.shape[1], nb_classes]))\n",
419 |     "    biases = tf.Variable(tf.zeros([nb_classes]))\n",
420 |     "\n",
421 |     "    logits = tf.matmul(X_train_tf, weights) + biases\n",
422 |     "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
423 |     "    optimizer = tf.train.GradientDescentOptimizer(0.8).minimize(loss)\n",
424 |     "    \n",
425 |     "    train_prediction = tf.nn.softmax(logits)\n",
426 |     "    test_prediction = tf.nn.softmax(tf.matmul(X_test_tf, weights) + biases)"
427 |    ]
428 |   },
429 |   {
430 |    "cell_type": "code",
431 |    "execution_count": 13,
432 |    "metadata": {
433 |     "collapsed": false
434 |    },
435 |    "outputs": [
436 |     {
437 |      "name": "stdout",
438 |      "output_type": "stream",
439 |      "text": [
440 |       "Train Loss at step 0: 4.624413\n",
441 |       "Valid Loss: 4.631759\n",
442 |       "Train Loss at step 1000: 4.137475\n",
443 |       "Valid Loss: 4.195310\n",
444 |       "Train Loss at step 2000: 3.732263\n",
445 |       "Valid Loss: 3.823207\n",
446 |       "Train Loss at step 3000: 3.384432\n",
447 |       "Valid Loss: 3.503140\n",
448 |       "Train Loss at step 4000: 3.086440\n",
449 |       "Valid Loss: 3.228689\n",
450 |       "Train Loss at step 5000: 2.829750\n",
451 |       "Valid Loss: 2.991925\n",
452 |       "Train Loss at step 6000: 2.606733\n",
453 |       "Valid Loss: 2.785802\n",
454 |       "Train Loss at step 7000: 2.411375\n",
455 |       "Valid Loss: 2.604877\n",
456 |       "Train Loss at step 8000: 2.239062\n",
457 |       "Valid Loss: 2.445024\n",
458 |       "Train Loss at step 9000: 2.086184\n",
459 |       "Valid Loss: 2.303040\n",
460 |       "Train Loss at step 10000: 1.949856\n",
461 |       "Valid Loss: 2.176355\n"
462 |      ]
463 |     }
464 |    ],
465 |    "source": [
466 |     "num_steps = 10001\n",
467 |     "train_loss = []\n",
468 |     "test_loss = []\n",
469 |     "\n",
470 |     "with tf.Session(graph=graph) as session:\n",
471 |     "    tf.global_variables_initializer().run()\n",
472 |     "    for step in range(num_steps):\n",
473 |     "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
474 |     "        if (step % 1000 == 0):\n",
475 |     "            train_loss.append(l)\n",
476 |     "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
477 |     "            print('Train Loss at step %d: %f' % (step, l))\n",
478 |     "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
479 |    ]
480 |   },
481 |   {
482 |    "cell_type": "code",
483 |    "execution_count": 14,
484 |    "metadata": {
485 |     "collapsed": false
486 |    },
487 |    "outputs": [
488 |     {
489 |      "data": {
490 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VEXbwOHfkwohQCgSpITQBAEloBRRIICAgGADjAUI\nvKIiiIrlswEWfO2iqK+KKMUCiKIioKBAKIL00Jv0Il0INSHJfH+czZ5sCiQkW5I893XtRc7M7NnZ\nybJPzsyZGTHGoJRSSqXx83YFlFJK+RYNDEoppVxoYFBKKeVCA4NSSikXGhiUUkq50MCglFLKhdsD\ng4jsEpE1IrJaRJZlU2aUiGwTkXgRiXJ3nZRSSmUvwAOvkQpEG2P+zSpTRDoBNY0xtUWkGfAp0NwD\n9VJKKZUFT3QlySVe5zZgAoAxZilQWkTCPVAvpZRSWfBEYDDA7yKyXET6Z5FfGdib7ni/I00ppZQX\neKIr6UZjzD8icgVWgNhkjFnkgddVSil1GdweGIwx/zj+PSIiPwJNgfSBYT9QNd1xFUeaCxHRRZ2U\nUuoyGGMkN+Xd2pUkIiEiEur4uQTQAVifodg0oLejTHPghDHmUFbnM8bowxiGDx/u9Tr4ykPbQttC\n2+Lij8vh7iuGcOBHx1/7AcA3xpjZIvIQYIwxo40xM0Wks4j8DZwB+rq5TgXerl27vF0Fn6FtYdO2\nsGlb5I1bA4MxZieQaV6CMeazDMeD3FkPpZRSOacznwug2NhYb1fBZ2hb2LQtbNoWeSOX2wflaSJi\nCkpdlVLKV4gIxpcGn5V7xMXFebsKPkPbwuaNtoiMjERE9OEDj8jIyHz7vXpiHoNSqpDavXv3Zd/5\novKXSK4uCi5+roLyS9WuJKV8j6ObwtvVUGT/u9CuJKWUUnmmgaEA0n51m7aFTdtC5RcNDEoplY0B\nAwbw2muvXdZz27Rpw5dffpnPNfIMHWNQSl02Xx5jqF69Ol988QVt27b1yuu3adOGXr160a9fP4+8\nno4xKKVUHqWkpHi7Cj5LA0MBpH3JNm0Lm7aFrXfv3uzZs4euXbtSqlQp3nnnHXbv3o2fnx9ffvkl\n1apVo127dgD07NmTK6+8kjJlyhAdHc3GjRud5+nbty/Dhg0DYP78+VStWpX33nuP8PBwKleuzLhx\n43JUH2MMI0aMIDIykooVKxIbG0tCQgIAiYmJ9OrVi/Lly1OmTBmaNWvGkSNHABg3bhw1a9akVKlS\n1KxZk4kTJ+ZjK2VPA4NSyi1E8veRGxMmTCAiIoLp06eTkJDAU0895cxbsGABmzdvZtasWQB07tyZ\n7du3c/jwYRo3bsx9992X7XkPHjzIqVOnOHDgAGPGjGHgwIGcPHnykvUZO3YsEyZMYP78+ezYsYNT\np07x6KOPAjB+/HgSEhLYv38/x48f59NPP6V48eKcPXuWxx57jFmzZpGQkMDixYuJisq09JxbaGAo\ngKKjo71dBZ+hbWHTtsgsY5+7iPDyyy9TvHhxgoODAWtdpZCQEAIDAxk2bBhr1qzh1KlTWZ4vKCiI\noUOH4u/vT6dOnQgNDWXLli2XrMe3337LkCFDqFatGiEhIbz++utMmjSJ1NRUAgMDOXbsGFu3bkVE\naNSoEaGhoQD4+/uzbt06zp8/T3h4OFdffXUeWyRnNDAopYqUKlWqOH9OTU3l2WefpVatWoSFhVG9\nenVEhKNHj2b53HLlyuHnZ39thoSEcPr06Uu+5oEDB6hWrZrzuFq1aly4cIFDhw7Rq1cvOnbsSExM\nDFWqVOHZZ58lJSWFkJAQJk+ezCeffMKVV15J165dcxSE8oMGhgJI+5Jt2hY2X2sLY/L3kVvZLRGR\nPv3bb7/ll19+Ye7cuZw4cYJdu3blaYOb7FSqVIndu3c7j3fv3k1gYCDh4eEEBAQwdOhQNmzYwOLF\ni/nll1+YMGECAO3bt2f27NkcPHiQOnXq0L9//3ytV3Y0MCilCqWKFSuyY8cOl7SMX/inTp0iODiY\nMmXKcObMGZ577rl8XXMozT333MPIkSPZtWsXp0+f5oUXXiAmJgY/Pz/i4uJYv349qamphIaGEhgY\niJ+fH4cPH2batGmcPXuWwMBAQkND8ff3z/e6ZUUDQwGkfck2bQubtoWrZ599lldffZWyZcvy3nvv\nAZmvInr37k1ERASVK1emQYMGtGjRIlevcbEgkj6vX79+9OrVi1atWlGzZk1CQkIYNWoUYA1od+/e\nndKlS1O/fn3n/IfU1FTee+89KleuTPny5VmwYAGffPJJrup3uQrUBLfzF84THBDs7aoopRx8eYJb\nUVNkJ7g1fv9mjpw54u1qeJ2v9SV7k7aFTdtC5ZcCFRg2nllEvfebsO7QOm9XRSmlCq0C1ZXES9bP\nwYQyJeZbutbp6tU6KVXUaVeS7yiyXUkkWpM+EjlNt4m38eait/RDqZRS+axABYaolUvg30jrQAzP\nzvk/+vwUS2Jyolfr5Wnal2zTtrBpW6j84pHAICJ+IrJKRKZlkddaRE448leJyIvZnefPHxvQdscy\n2N3SmfbV2glEj2vDodOH3FR7pZQqWjwyxiAiTwDXAaWMMd0y5LUGnsyYnsU5jDGGpCS4v08SU84+\nAo2/cOZXLRXBL/dMo2HFhu54C0qpLOgYg+8oUGMMIlIF6AyMuVixnJ4vKAgmfh3Ew5U/h9/eg1Tr\nLexN2MONX97IT5t/yluFlVKqiPNEV9JI4GngYn9W3CAi8SIyQ0TqXeqE/v7wv4+FF29+Ar6dDudL\nAXDmwhnumHwHry98vVD/FaN9yTZtC5u2Rf5I23chTYMGDViwYEGOymbk5+eXaVmOgiDAnScXkS7A\nIWNMvIhEk/WVwUogwhhzVkQ6AT8BV2V1vtjYWCIjIwEICwujXbsoypbtxJDXl0DLm6HkP1Adnp/7\nPHPnzeXpG5+mQ7sOgP2fJm3ZAD0uHMdpfKU+3jyOj4/3WvsXNumXs1i/fn2Oy+Ymz13i4uKcGwil\nfV/mllvHGETkv8D9QDJQHCgJTDXG9L7Ic3YC1xljjmdIz3bP53HjoN+gY5ju3aF6nDO9eZXm/Hj3\nj1QMrZjXt6KUykJhHGOYP38+vXr1Ys+ePXku6+fnx99//02NGjXyu5qZFJgxBmPM88aYCGNMDSAG\nmJsxKIhIeLqfm2IFq+PkQmwsTP26HIGTZ8GKB53pf+37i6afN2X1P6vz9D6UUgXLW2+9RY8ePVzS\nHnvsMR5//HHA2jKzXr16lCpVilq1ajF69Ohsz1W9enXmzp0LwPnz54mNjaVs2bI0aNCA5cuX57hO\nCQkJ9O7dmwoVKlC9enVee+01Z9727duJjo4mLCyMChUqcM899zjznnjiCcLDwyldujQNGzZ02XrU\nXdzalZQdEXkIMMaY0UB3ERkAXADOAXdfzjlvvx1+mxFEt9s+5cyR+tDxCfBLZW/CXm4aexNf3fEV\nd159Zz6+C++Ji4vTlTQdtC1svtYW8nL+dqOY4Tm/MomJieGVV17hzJkzlChRgtTUVKZMmcLPP/8M\nQHh4ODNnziQyMpKFCxdyyy230LRp00tunfnSSy+xc+dOdu7cyenTp7nllltyXKdBgwZx6tQpdu3a\nxZEjR+jQoQOVKlWib9++DB06lI4dOxIXF0dSUhIrVqwAYPbs2SxatIi///6bkiVLsmXLFsLCwnL8\nmpfLYxPcjDHz025JNcZ85ggKGGM+NsY0MMY0Msa0MMYsvdzXaNsW5s0Vyv09GL75Fc6XBuDshbPc\n9d1djFgwotBd9iqlMouIiKBx48b8+OOPAMyZM4cSJUrQpEkTADp16uTsf2/ZsiUdOnRg4cKFlzzv\nlClTePHFFyldujSVK1dm8ODBOapPamoqkydP5o033iAkJIRq1arx5JNP8tVXXwEQGBjI7t272b9/\nP0FBQc7lvwMDAzl16hQbN27EGEOdOnUIDw+/2EvliwI18zknmjSBhQuh8vkOMOYvOFbLmTd03lDu\nm3of5y6c82IN886X/ir0Nm0Lm7aFq3vuuYeJEycCMHHiRO69915n3q+//soNN9xAuXLlKFOmDL/+\n+mu223mmd+DAAZetQdNv13kxR48eJTk5mYiICJfn7t+/H7C6vlJTU2natCnXXHMNY8eOBaBNmzYM\nGjSIgQMHEh4ezsMPP5yjrUTzLG0bO19/WFXNuV27jKld2xiKHzP0bmt4CeejyegmZn/C/lydTymV\nWW7/X3rSkSNHTEhIiNm3b58JCwszmzdvNsYYk5iYaEJCQszUqVNNSkqKMcaY22+/3QwdOtQYY0xc\nXJypWrWq8zyRkZFmzpw5xhhjqlevbmbNmuXMGz16tEvZjETEbN++3aSkpJjg4GCzadMmZ95nn31m\n2rRpk+k5ixYtMsWKFTPbt2/P9H6io6PNsGHDsnyt7H4XjvRcfd8WuiuGNNWqwaJFEFWnLHz9Gywf\n4MxbfmA5TT9vysoDK71Yw8tX2G8VzA1tC5u2havy5cvTunVr+vbtS40aNahTpw4ASUlJJCUlUb58\nefz8/Pj111+ZPXt2js7Zs2dPXn/9dU6cOMG+ffv46KOPcvQ8Pz8/evbsyQsvvMDp06fZvXs3I0eO\npFevXgB8//33zquHsLAw/Pz88PPzY8WKFSxbtozk5GSKFy9OsWLF8PNz/9d2oQ0MABUqQFwctLwx\nEGb8D2Z8DKnWnqn7T+2n5diWTNkwxbuVVEq5zb333sucOXO47777nGmhoaGMGjWKHj16ULZsWSZN\nmsRtt92W7TnSz0UYPnw4ERERVK9enVtuuYXevbO98z7Tc0eNGkVISAg1atSgVatW3H///fTt2xeA\n5cuX06xZM0qVKsXtt9/OqFGjiIyMJCEhgf79+1O2bFmqV69O+fLlefrppy+3OXKsQO3HcLl1PXcO\nevaE6dOBGn9Ajx5Q/IQz/6XWLzGs9TCvTEZRqiArjPMYCqr8nMdQJAIDwIUL0LcvfPMNUG4r3NMV\nym915ves35Oxt40lJDAkH2qrVNGggcF3FJgJbr4kMBAmTIBHHwWOXWXdsbT9Zmf+dxu+o9XYVuxP\n2O+9SuaQ9iXbtC1s2hYqvxSZwADg5wcffADDhwPny1hzHZYOcuav/GclTT5vwvL9OZ/NqJRShU2R\n6UrKaNQoeOwxx8H1n0LnQeCXAkCxgGKMvW0sMQ1i8u31lCqMtCvJd+gYQz756itr3CElBag+F7m7\nO6bYv878F1u+yMttXsZPitSFlVI5poHBd+gYQz7p1Qt+/BGCg4GdbTGjl+F3vK4zf8TCEfSc0pMz\nSWe8V8ksaF+yTdvCpm2h8kuRDgwAXbvCrFlQsiRwvBapo5fgt6OjM/+HTT/QcmxL9p7c671KKuWj\nqlWrhojowwceOV2eIyeKdFdSeqtWwS23wJEjgF8yfrc8RWrTD5z54SXC+TnmZ5pVaea2OiilVH7T\nMYY82rIF2reHvY6LA7nuc6TrI6SSDECwfzCfd/2cXg17ubUeSimVX3SMIY/q1IE//4S6jmEGs7I/\nqWP/IIRyACSmJNL7p970/rE3CYkJXqun9iXbtC1s2hY2bYu80cCQQdWqsGABXHedI2F3a85+sIxy\nqfWcZb5a+xVRn0bx554/vVNJpZRyI+1KykZCAtx2m7UIHwBBp6j92KNsKzHeWcZP/Hih5QsMbTWU\nQP9Aj9VNKaVySscY8tn583D33TBtmp3WJPY7ttV5iBOJ9iJ8zSo34+s7v6ZW2VpZnEUppbxHxxjy\nWbFi8MMP0KePnbZ8XE/KTVlLkyuinWlL9y8l6tMoxq4e65HJPtp/atO2sGlb2LQt8kYDwyUEBMCX\nX8KQIXba9lVV2fB/f3BP+TcJ9LO6kM5cOEO/af3oMaUHx84e81JtlVIq77QrKRcmTICHH7b2d0jT\n87FVxNe8l63HtzjTKpWsxITbJ9CuRjsv1FIppWw6xuAB69bBXXfBtm12WpMWZ6kz+Em+3vypS9kn\nb3iS19q+RnBAsIdrqZRSFh1j8IBrroHly+GOO+y05YtD+G3QJ7xy9TSuCLnCmf7ukndpNqYZG49s\nzNc6aP+pTdvCpm1h07bIG48EBhHxE5FVIjItm/xRIrJNROJFJMoTdcqL0qWtQel33gF/awtpjh6F\n4TFd6Xt+LR1r3uIsu+bQGq4bfR0fLftIV6FUShUIHulKEpEngOuAUsaYbhnyOgGDjDFdRKQZ8IEx\npnkW5/CJrqSMFiywbmk9eNBO69TZ0Pqpjxn+51MkpiQ60zvX7syX3b4kPDTcCzVVShVFPtmVJCJV\ngM7AmGyK3AZMADDGLAVKi0iB+eZs1QpWr7b+TfPrTOHTfoOYcNNKrg2/1pk+c9tMrvnkGqZvne6F\nmiqlVM54oitpJPA0kN2f+5WB9Gta73ekFRgVK8KcOfDMM3barl3Qq2N9HpRlPNHcvtf1yNkjdJ3Y\nlUdmPMLZC2cv6/W0/9SmbWHTtrBpW+RNgDtPLiJdgEPGmHgRiQZydTmTUWxsLJGRkQCEhYURFRVF\ndHQ0YH8QvHW8aFEcnTpB8+bRxMZCQkIcSUkw6OFo+vR5l/9GV+LdZa9zLNya4/DJlE+YPns6Pz/7\nM42ubOT1+hfU4zS+Uh9vHsfHx/tUfbx5HB8f71P18eRxXFwc48aNA3B+X+aWW8cYROS/wP1AMlAc\nKAlMNcb0TlfmU2CeMWay43gz0NoYcyjDuXxyjCEr27ZB9+6wdq2ddu21MObbY7yx4UGmbprqTA/0\nC2RE2xE81eIp3UJUKZXvfHoeg4i0Bp7MYvC5MzDQMfjcHHi/IA0+Z+fsWRgwwJoUl6ZUKRg3zvBv\n5FgG/zqYMxfsLUPbRLZh/O3jqVq6qhdqq5QqrHxy8DkrIvKQiDwIYIyZCewUkb+Bz4BHvFGn/BYS\nAuPGwWefQVCQlZaQAHfeKWyZ1I/lD6ymaeWmzvLzds3j2k+vZcqGKZc8d8ZulKJM28KmbWHTtsgb\njwUGY8z8tKsFY8xnxpjR6fIGGWNqGWMaGmNWeapO7iYCDz4IixdD+q6+t96CAT1r832nRbzY8kVn\nF9KJ8yfo+X1PYn+K5VTiKe9UWilV5OmSGB5y/Dj06gUzZ9ppFSvCd9+BVFtErx97sevELmdejTI1\n+PqOr7mh6g2er6xSqtAoMF1JRVHZsvDLLzBihHUlAdakuDZtYOmUm1j9YDz3X3u/s/yOf3fQcmxL\nXop7ieTUZC/VWilVFGlg8CA/P3jhBZg1C8qXt9JSUuCpp+A/95fmo7Zf8e2d31I6uLSVZ1J4ef7L\ntBrbih3/7nCeR/tPbdoWNm0Lm7ZF3mhg8IL27WHVKmie7t6rqVOhSRNowD2seXgNrarZU6mX7FtC\nw08bMj5+vK63pJRyOx1j8KKkJOtq4cMP7bTixa07me69L4W3/nyLYXHDXLqSetTrwWe3fkaZ4mW8\nUGOlVEHj0/MY8qowBoY0kybBAw/AGXtaAw8/DO+/D+uOreC+qfex9dhWZ16VUlX4pMsn3HrVrV6o\nrVKqINHB5wIqJgaWLYO6de20Tz+Fm26C8knXs+rBVTzY+EFn3r41++g6sSt3TL6D3Sd2e6HGvkP7\nkm3aFjZti7zRwOAj6tWzgsPdd9tpK1bAddfBwrkl+KzrZ/x090+UDynvzP9p80/U+1893lz0Jkkp\nSV6otVKqMNKuJB9jDHz0EQwZAsmOoQURGDoUhg2DfxOP8uwfz/LF6i9cnnd1+av5X5f/ER0Z7flK\nK6V8lo4xFCKLF0PPnrB/v53WoQN88411q+vivYsZMGMAaw+tdXne/dfez9vt36ZiaEUP11gp5Yt0\njKEQadHCuqW1XTs7bfZsaNwY/ve/OFpUbcHKB1cysuNIQoNCnWW+Xvs1dT+qy8fLPiYlNcULNfcs\n7Uu2aVvYtC3yRgODD6tQwZoM98ILdtrevTB4MLzxBpAawOPNH2fzwM3cXd8enDiZeJJBvw6i6Zim\nLNu/zPMVV0oVaNqVVEBMn26ttXTihJ3WpAl8+SU0aGAd/779dwbOHMi249ucZQThoese4r/t/qtz\nH5QqgnSMoZDbudO6a2n5cjstKMgalH7mGQgMhMTkRN5e/DavLXyN88nnneWuCLmCt9u/Te+GvRHJ\n00Z6SqkCRMcYCrnq1a1B6f7945x7PCQlwYsvWstrrF0LwQHBvNjqRTY8soHOtTs7n3vk7BFif46l\n9bjWrD+83kvvIP9pX7JN28KmbZE3GhgKmIAAuPdeWL0amtr7/LBqFVx/PbzyCly4YC3bPf2e6Uzt\nOZWqpexd4RbuWUjUp1E8PftpTied9sI7UEr5Ou1KKsCSk2HkSGuOQ2KinR4VBWPHWv8CnE46zavz\nX+W9v95zWXepSqkqfHDLB9xR9w7tXlKqkNIxhiJq82bo1w+WLLHTAgLg+eetO5rSup02HN7AIzMf\nYcHuBS7P71SrEx92+pCaZWt6sNZKKU/QMYYiImP/ad26sHAhvPsuFCtmpSUnW91K119vdTMB1K9Q\nn7g+cUy4fQIVSlRwPv/Xv3+l/v/q88r8V1wGrAsC7Uu2aVvYtC3yRgNDIeHvby2jsWaNtfhemnXr\nrLGIF1+0uptEhF4Ne7F54GYeuf4RBOsPicSURIbHDeeaT65h9vbZXnoXSilfoF1JhVBqqrXe0rPP\nwrlzdnr9+tbYQ5MmdtqKAysYMGMAKw6scDlHj3o9GNlxJJVLVfZQrZVS7qBjDMrF33/Df/4DC9IN\nKfj5wdNPw0sv2d1OKakpjF45mufmPMfJxJPOsqFBobwc/TKPNn2UQP9Az1ZeKZUvdIyhiMhp/2mt\nWjBvnrVDXIkSVlpqKrz5JjRqBH/9ZaX5+/kzoMkAtgzaQu+GvZ3PP510midnP8l1o6/jzz1/5vO7\nyB/al2zTtrBpW+SNWwODiASLyFIRWS0i60RkeBZlWovICRFZ5Xi86M46FTV+fjBokDX5rU0bO33z\nZrjxRuvqIa27KTw0nPG3jyeuTxz1rqjnLLvu8DpuGnsT/X7ux5EzRzz8DpRSnub2riQRCTHGnBUR\nf+BPYLAxZlm6/NbAk8aYbpc4j3Yl5VFqKowebQWD0+nmtl11lTX20KKFnZaUksT7f73Py/Nf5uyF\ns870MsXK8MbNb/BA4wfwE73gVMrX+WRXkjEm7VslGAgAsvp219lVHuDnZ+0lvX493Hyznb51q3Un\n05AhcNbx2wryD+KZG59h08BN3FH3DmfZf8//y0PTH6LFFy1YeWClh9+BUsoT3B4YRMRPRFYDB4Hf\njTHLsyh2g4jEi8gMEamXRb5KJ6/9p9WqWXs7jB4NJUtaacZYs6gbNrTmRKSJKB3B1LunMv2e6VQP\nq+5MX7p/Kdd/fj09p/Rk89HNeapPXmhfsk3bwqZtkTcB7n4BY0wq0EhESgE/iUg9Y8zGdEVWAhGO\n7qZOwE/AVVmdKzY2lsjISADCwsKIiooiOjoasD8Iepyz4/nz46hdG9avj+bBB2HWLCv/77+jad0a\n7rgjjgcegE6drPIlDpTgk/qf8Kf/n7z555sk/W3tMT2FKfyw6Qfa+7UnNiqWmFtjPPp+0ni7PX3h\nOD4+3qfq483j+Ph4n6qPJ4/j4uIYN24cgPP7Mrc8eruqiAwFzhhj3rtImZ3AdcaY4xnSdYzBTYyx\nxhieeAISEuz0GjWs/R5at3Ytv/XYVp6b8xxTN011SQ/0C+Sh6x7ihVYv6NaiSvkInxtjEJHyIlLa\n8XNxoD2wOUOZ8HQ/N8UKVi5BQbmXiLXW0oYN0NleqZsdOyA62rqryWWwutxV/NDzB5Y9sIwONTs4\n0y+kXuCj5R9R44MaPPfHcxw/p79GpQoid48xXAnME5F4YCkwyxgzU0QeEpEHHWW6i8h6xzjE+8Dd\n2Z1MWTJ2o+SXKlWsneLGjYOwMDv944/hmmtg7lzX8k0qN2HW/bOI6xPHjVVvdKafSz7HG3++QfUP\nqjNiwQi3Lu/trrYoiLQtbNoWeePWwGCMWWeMaWyMiTLGXGuMec2R/pkxZrTj54+NMQ2MMY2MMS2M\nMUvdWSd1cSLQp4919XDrrXb6rl3Qrp11V1P67iaA1pGtWdh3ITPunUFUxShnekJiAkPnDaXGBzV4\n/6/3C9wCfUoVVbokhsqWMfDNNzB4MPz7r50eEQFjxkD79pmfk2pS+X7j9wydN5Stx7a65FUpVYXh\nrYfTp2EfXWJDKQ/RtZKUWxw8CAMGwE8/uabHxsJ//wtXXpn5OcmpyXy15itemv8Se07uccmrXbY2\nr7R5hZ71e+okOaXczOcGn5V7eLr/tGJFmDoVJk6EcuXs9HHjoHZtKzikX8UVIMAvgL6N+rJ10FZG\n3TLKZf+Hbce3cc8P99Dos0b8suUX8hLwtS/Zpm1h07bIGw0MKkdEICbGGnu46y47/cwZa5e4unVh\n0iSr+ym94IBgHm32KDsG7+C/bf9LWDF7VHvtobV0m9SNFl+2YN7OeR56J0qpS8lRV5KIPAaMBU4B\nY4BGwLPGGI/t6KJdSb5l9mxrCY0NG1zTW7SwZlA3bZr1806cP8E7i9/h/b/e58yFMy55N9e4mdfa\nvkbTytk8WSmVa24bYxCRNcaYhiLSEXgIGAp8ZYxpfHlVzT0NDL4nOdkahB46FI4edc277z54/XWo\nWjXr5x46fYjXF73OJys+ISklySXvtjq38WqbV7km/Bo31VyposOdYwxpJ+2MFRA2oAvfeY2v9J8G\nBFi3r27bBk89BYHpbjT65huoUweGD7e6mzIKDw3n/VveZ9uj2/hPo//gL/7OvJ+3/EzDTxty/9T7\n+fv43xetg6+0hS/QtrBpW+RNTgPDShGZjRUYZolISSDVfdVSBUlYGLz9NmzcCHfYC7Fy7hy88oq1\nrPeECday3xlFlI5gTLcxbBy4kZgGMc50g+Gbdd9w9cdX8/D0h9mfsN8D70QpBTnvSvIDooAdxpgT\nIlIWqGKMWevuCqarg3YlFRBxcdb4w+rVrunXX2+NP9x0U/bPjT8Yz9B5Q5m+dbpLerB/MAObDOTZ\nm57lihJX5H+llSqk3DnGcCMQb4w5IyL3A42BD4wxuy+vqrmngaFgSUmxrhKef96aB5Fejx7W9qLV\nq2f9XIBIUzIAAAAgAElEQVTFexfz/Jznmb97vkt6aFAoQ5oPYcgNQyhdrLQbaq5U4eLOMYZPgLMi\n0hB4EtgOTMhl/VQ+KQj9p/7+0LevtQnQCy9AsWJ23pQpcPXV8NxzmZfXSNOiagvm9ZnH7Ptn06RS\nE2f66aTTvLLgFWqMqsHbf77Nb3/85uZ3UnAUhM+Fp2hb5E1OA0Oy48/124CPjDEfAyXdVy1VWJQs\nCSNGWHtMx9hDCCQmwhtvWBPkPv/cusLISERoX7M9Sx9Yyo93/0j9K+o7846fO84zfzzDvT/cy5uL\n3uTfc/9mPoFS6rLktCtpPvAb0A9oCRwG1hhjPHY/oXYlFQ5Lllj7PizNsFTitdda4w9t22b/3JTU\nFCaun8jwuOHs+HeHS16JwBI80PgBHm/+OJFhkflfcaUKKHeOMVQE7gWWG2MWikgEEG2M8Vh3kgaG\nwiM11Vpe49lnYd8+17xu3eCdd6wriewkpSTx5eoveXXBqxw4dcAlz0/86F6vO0/e8KROlFMKN44x\nGGMOAt8ApUXkVuC8J4OCclXQ+0/9/KwJcFu2WLezhoTYedOmQf361l1N/2bTOxTkH8TD1z/MjsE7\n+L/K/+fSxZRqUvluw3c0G9OMlmNb8vPmn0k1RePO6oL+uchP2hZ5k6PAICI9gWVAD6AnsFREuruz\nYqrwCwmxZk1v3WrtAZHmwgWrW6l2bfjoI+s4K8EBwdxS6xbWDVjHb/f9xs01bnbJX7RnEbdPvp26\nH9Xl0xWfcvbCWTe+G6UKjxwviQG0N8YcdhxfAfxhjGno5vqlr4N2JRVyK1ZY4w+LFrmmX301vPsu\ndOp06XOsObiGd5e8y8T1E0lOTXbJK1e8HAObDGRg04Euq70qVZi5c4xhXfqBZseENx18VvnOGPjh\nB3jmGdi50zWvY0crQNSvn/Vz09uXsI8Pl37IZys/42TiSZe8YP9gejfszZAbhlC3fN18rL1Svsed\n8xh+E5FZIhIrIrHADGBmbiuo8kdh7j8Vge7dreU13nzTut01zaxZ0LAhDBxoL9qXXVtUKVWFN9u/\nyd4n9jKy40iqla7mzEtMSeTzVZ9z9cdXc+u3txK3Ky5Pe0L4isL8ucgtbYu8yeng89PAaOBax2O0\nMeb/3FkxVbQVK2ZdNWzbBg8+aA1YgzXf4X//g1q14L33sh9/SFMyuCSPN3+cvwf/zeTuk10mywHM\n2DaDNuPbcP3n1/Ptum+5kHKJEypVBOjWnqpAWLvWulNpzhzX9Fq1rAHse++1Vnu9FGMMi/Ys4t0l\n7zJtyzQMrp+pqqWq8njzx3mg8QOUCi6Vj+9AKe/I9zEGETkFZFVAAGOM8dj/HA0MyhiYPt1a4nvr\nVte8GjWsdZl69YKgoJydb+uxrYxcMpJxa8ZxPvm8S16p4FL0b9yfx5o9RtXS2WwqoVQB4LbBZ1+g\ngcEWFxdHdHS0t6vhNUlJVnfSyy/DiRNxQLQzLyLCmjjXrx8EB+fsfEfOHOGTFZ/w0bKPOHL2iEte\ngF8APev35MkbnqTxlR7bl+qyFPXPRXraFjZ3Dj5fFhEJFpGlIrJaRNaJyPBsyo0SkW0iEi8iUe6s\nkyr4goLg8cetu5b69oWyZe28PXvgkUesK4hRo6w9IS7lihJXMKz1MHY/vpvRt46mTrk6zrzk1GS+\nXfct142+jrbj2zJj64wiM2FOFV1uv2IQkRBjzFkR8Qf+BAYbY5aly+8EDDLGdBGRZljLeTfP4jx6\nxaCydOqUdQXx7rtwxPUPfsLDra6nhx+G0NCcnS/VpDJz20zeWfxOpmW/Aa4ufzVDbhjC/dfeT7GA\nYlmcQSnf4dNdSSISAiwABhhjlqdL/xSYZ4yZ7DjehLUO06EMz9fAoC7qzBkYPRreeivzHhDlylmD\n14MGQalcjIytOLCCd5e8y5QNU0gxrkvAVihRgUFNBjGgyQDKh5TPh3egVP7zua4ksCbDichq4CDw\ne/qg4FAZ2JvueL8jTWVD79G2pW+LEiWsmdM7d1pLaVSpYpc7dszaF6JaNWtsIrt1mDK6vtL1TLxr\nItsHb2dI8yGUDLInVhw+c5hhccOIGBlB/2n9WbZ/mVfnQ+jnwqZtkTc5uMEvb4wxqUAjESkF/CQi\n9YwxGy/nXLGxsURGRgIQFhZGVFSUc4Ap7YOgx0XrOE36/GLFoH79OL74Anbtiub112HXLiv/xIlo\nXnoJ3nwzjjvvhPffj6Z8+Uu/3s74nXQN7sqwJ4bx+arPefObNzl65ihUh3PJ5xgzdQxjpo7hmmbX\n0L9xf6qdqEap4FIebY/4+Hiv/z585Tg+Pt6n6uPJ47i4OMaNGwfg/L7MLY/elSQiQ4Ezxpj30qVl\n7EraDLTWriSVXy5cgG++gf/+15owl16JEjBggDUOER6ei3OmXOC7Dd/xzpJ3iD8Ynyk/2D+Y7vW6\n80DjB2hdrTUiubqSVyrf+NwYg4iUBy4YY06KSHFgFvCGMWZmujKdgYGOwefmwPs6+KzcITkZvvvO\n2lFu0ybXvGLFrBnWzzwDlXPRkWmMYfHexYxZPYbJ6ydzLjnzbVC1y9bmP43+Q2xULOGhuYg+SuUD\nXwwM1wDjscYy/IDJxpjXROQhrAlyox3lPgJuAc4AfY0xq7I4lwYGhzi9R9vpctoiNdVaqG/ECGtG\ndXpBQfCf/8D//Z81HpEbJ8+fZOL6iXy+6nNW/ZPpI0yAXwBdr+pK/8b96VCzA/5+/rl7gUvQz4VN\n28Lmc4EhP2lgsOmH3paXtkhNhV9+gVdfhZUrXfMCAqw9Ip57DmrWzP25V/2zijGrxvDNum9ISEzI\nlF+1VFX6NepHv0b9iCgdcVn1z0g/FzZtC5sGBqUugzHw22/WbnJ//eWa5+9v7Tb3/PNQp07Wz7+Y\nsxfOMmXDFMasHsOiPYsy5QtCx1od6d+4P12v6kqgf+BlvgulsqaBQak8MMZapO/VV2HBAtc8Ebj7\nbuuW1wYNLu/8m45s4ovVXzB+zXiOnj2aKb9CiQrENozlP43/w1Xlrrq8F1EqA5+cx6DyX8ZbNYuy\n/GwLEbj5Zpg/33rcnG6nUGNg0iS45hq46y5YvTr357/6iqt5p8M77HtiH991/472Ndq75B8+c5i3\nFr9FnY/qED0umq/Xfs25CzlY08NBPxc2bYu80cCgVBZatYLff4fFi6FzZ9e8qVOhcWPo2hWWLcv6\n+RcTHBBMj/o9mN1rNjsG7+DFli9SqWQllzLzd8+n14+9qPReJR6d+ShrD63N5mxK5T/tSlIqB1as\nsO5i+vnnzHkdO1orurZubV11XI7k1GR++/s3xqwaw/St0zMtvwHQpFIT+jfuT0yDGEoGl8ziLEpl\npmMMSrnZ2rVWgPj+e6t7Kb1rrrHWYrrvPmvi3OU6cOoA4+PHM2b1GHb8uyNTfonAEsQ0iKF/4/40\nrdxUJ8+pi9LAUETorXg2b7XFxo3WTOqJE63bXtMLC7P2gxg40Fr++3KlmlTm7ZzHmNVjmLppKkkp\nSZnKNKjQgP6N+3P/tfezdula/Vw46P8Rmw4+K+Uh9erB11/D5s3WkhrprxBOnLD2o65VyxqHmD07\nc/DICT/xo12Ndky8ayL7h+xnZMeR1LuinkuZ9YfX89hvj1Hp3Uq8Ov9Vpm+dnmUAUSo39IpBqXxw\n4gSMH2+t6vr335nzr7rK6mbq0yd3y35nZIzhr31/8fmqz5m8YTJnL5zNVKZMsTLcdfVdxDSIIToy\nOt9nWKuCRbuSlPKy1FSYNQs+/BB+/TVzfmioFRwGDYK6dfP2WgmJCUxcN5Exq8ew4sCKLMuElwin\nR70e3HPNPTSv0hw/0U6CokYDQxGh/ac2X26LbdusneW+/BISMq+KQfv28Oij1u2w/nn8o37NwTW8\n+c2bLPZfzO6Tu7MsE1E6grvr301MgxgaVWxUqAetfflz4Wk6xqCUD6ldG0aOhP37rQBRz3V4gN9/\nh27drHLvvAPHj1/+azWs2JAHr3uQnY/tZHG/xQxuOpiKoRVdyuw5uYe3F7/NdaOvo+7HdRk+bzib\njmzK5oyqKNMrBqU8xBiYN8/qZpo2LfOAdPHi1q2ujz4K116b99dLSU1hwe4FTFw/kR82/cDxc1lH\nnmvDryWmfgwxDWKoXqZ63l9Y+RTtSlKqgNi1Cz75BMaMyfpKoVUrK0Dcfru10mteJaUk8ceOP5i0\nfhI/bv6R00mnsyzXrHIzYhrE0LN+z0yzsVXBpIGhiND+U1tBb4tz56y5EB9+CPGZN4KjShV4+GHo\n3x8qVLj4uXLaFucunGPmtplM2jCJ6Vuncz75fKYygtA6sjUx9WO4q95dlA8pn8N35BsK+uciP+kY\ng1IFTPHi1mS4Vatg4ULo2dN1IHrfPnjxRaha1bqbafnyfHjNwOLcVe8upvSYwuGnDvP1HV9z61W3\nEuBnX5oYDHG74nh4xsNUfKcinb7pxPj48Zw8fzLvFVA+T68YlPIx+/fDZ59Zj8OHM+c3a2Z1M3Xv\nDsHB+fe6x88dZ+qmqUxaP4l5u+aRajLPygv2D6Zz7c7ENIjh1qtuJSQwJP8qoNxCu5KUKkQSE2HK\nFKubKatVXMPDrX2qH34YKuXzcMDB0wf5fuP3TFw/kcV7F2dZpkRgCbrV6cY9De6hQ80OBAfkY5RS\n+UYDQxGh/ae2otIWy5ZZs6onT4akDCteBARYe0S0aBHHo49GX/YKr9nZfWI33234jkkbJmW5lzVA\nWLEw7qx7JzENYmhTvY1Lt5Q3FJXPRU5oYCgi9ENvK2ptcfgwjB5t3dF04EDG3Dhq146mTx/o1Qsi\n8mcraRdbjm5h8obJTFw/kc1HN2dZpkyxMnS5qgvdrupGx1odKRWchzVALlNR+1xcjAYGpYqICxfg\np5+sbqaFCzPni0CbNhAbC3fembdlwLNijGHd4XVMWj+JSesnsfPEzizLBfoF0qZ6G7pd1Y2udboS\nUdoN0UpdlAYGpYqgNWvg44+trUdPncqcHxpqDVTHxkLLluCXz/ciGmNYtn8Zk9ZP4vtN37MvYV+2\nZRuGN6RbnW50q9ONxlc21rWbPEADQxGhl8k2bQvbb7/Fcfx4NOPGwR9/ZN5ICCAyEnr3th41a+Z/\nHYwxxB+MZ9qWaUzbOi3bMQmASiUr0fWqrnSr04221dtSLKBYvtVDPxc2nwsMIlIFmACEA6nA58aY\nURnKtAZ+BtK2qppqjBmRxbk0MDjoh96mbWFL3xb79ln7RYwfb+0ZkZWWLa25ET165G0p8IvZe3Iv\n07dOZ9rWaczdOTfbvSJKBJagQ80OdKvTjS61u3BFiSvy9Lr6ubD5YmCoCFQ0xsSLSCiwErjNGLM5\nXZnWwJPGmG6XOJcGBqVyyRjrjqbx462upn//zVymeHG44w6rq6lt27yv9JqdU4mnmL19NtO2TmPG\n1hkcO3csy3KC0KJqC2eXU51ydQr1SrDu5nOBIdOLifwEfGiMmZMurTXwlDGm6yWeq4FBqTxITIRf\nfoFx4+C33yAlJXOZypWtO5r69Mn7fhEXk5yazJK9S5xdTluPbc22bO2ytZ1BokXVFl6/Fbag8enA\nICKRQBzQwBhzOl16a+AHYB+wH3jaGLMxi+drYHDQy2SbtoUtN21x8CB8+60VJNaty7pMs2ZWgIiJ\ngTJl8q2aWdpydIszSCzeuzjLWdcAZYuXpUvtLnSr042ONTtSMrhkluX0c2Hz2cDg6EaKA141xvyc\nRV6qMeasiHQCPjDGXJXFOUyfPn2IjIwEICwsjKioKOcvPy4uDqBIHKf97Cv18eZxWpqv1Mebx/Hx\n8Tz++OO5en7r1tHEx8Nrr8Xx+++QkGDlW/9dAaIJCoIbboijY0d4+uloAgLc+36OnDnCuxPfZcne\nJawMXsmZC2cg7W7YtFXBd0KAfwDt2rSjW51ulD9UngqhFZzne//994v098O4ceMAiIyM5OWXX/a9\nwCAiAcB04FdjzAc5KL8TuM4YczxDul4xKOVGFy5Y25GOGwfTp1vHGYWHw/33W1cS11zj/jqdTz5P\n3K4462piyzT2n9qfbdmoilF0u8q+FVbHJSw+ecUgIhOAo8aYIdnkhxtjDjl+bgp8Z4yJzKKcBgal\nPOToUWuwetw4WLky6zKNGlkB4t574Yq83USUI8YYVh9c7QwSqw+uzrbslaFX0qFmB9rXaM/NNW4m\nPDTc/RX0UT4XGETkRmABsA4wjsfzQDXAGGNGi8hAYABwATgHPGGMWZrFuTQwOMRp/6mTtoXNXW2x\nYYN1V9NXX1ljExkFBECXLlaQ6NIFgoLyvQpZ2nNyj3Ur7BbrVtgLqekucXZidzth7VLXoUYH2tds\nT8uIlhQPLO6ZSvoAnwsM+UkDg02/DG3aFjZ3t0VysrVP9fjx1nIciYmZy5QrZ82L6N4dWrfOn93n\nciIhMcG6FXbLNGZsm8HxjcddAkN6wf7B3BRxk/OKomHFhoV6BrYGBqWUR5w4Ya30On48LFmSdZly\n5az5Ed27W/MjAgM9U7eU1BRWHFjB7zt+5/cdv7N472KSU5OzLX9FyBW0q9HOeUVRpVQVz1TUQzQw\nKKU8butWmDDBeuzdm3WZMmXgttusIHHzzfm7wdClnEo8xfzd8/l9uxUoNh3ddNHydcvXpX2N9rSv\n0Z7oyOhsb4ktKDQwFBHafWLTtrB5uy1SU2HRIvjhB/j++6yWBbeUKgXdullBokMHa+Z1frtYW+xL\n2McfO/5g9vbZ/LHjD46cPZLteQL8Arihyg1WoKjZnusrXV/gJthpYCgivP0F4Eu0LWy+1BapqfDX\nX3aQ2LMn63KhoXDrrVaQ6NQJQvJpp9CctkWqSWXtobX8vv13Zu+YzcLdC0lMyWLwxCGsWBhtq7d1\nXlHULOuGlQjzmQYGpZTPMQZWrLACxJQpsDPrrRsICYHOna0g0aWLFTQ87dyFcyzas4jfd/zO7O2z\nWXNozUXL1yhTwxkk2lZvS5nibp4ifhk0MCilfJoxEB9vB4lt27IuV6wY3HKLFSRuvRVKl/ZsPdMc\nOn2IOTvnOAPFgVPZ9I8BfuJHk0pNnN1Ozas0J8jfQ/fuXoQGhiLCl7oMvE3bwlbQ2sIYWL/eDhKb\nshkTDgqyxiK6d7fGJnKybpM72sIYw6ajm5yD2HG74qzlOrIRGhTKjVVvpFW1VrSq1oomlZoQHODB\nUXcHDQxFREH7AnAnbQtbQW+LjRutIPH999kv7BcQYN3V1L27dZdT+fJZl/NEWySlJLFk7xLnbbEr\nDqzIdvE/sOZPNKvSjFYRVqC4oeoNhAa5v79MA4NSqlDYutUeuF6VzSZw/v7Wvtbdu1vzJSpU8Gwd\nMzp+7jhzd851XlFktw92Gn/xp/GVjZ1XFDdF3ETZ4mXzvV4aGJRShc6OHXaQWLYs6zJ+ftCqlR0k\nKlXybB2zsvPfnSzcs5AFuxewYPcCth3PZkAlnQYVGjivKFpWa0mlknl/IxoYioiC3mWQn7QtbEWh\nLXbvhqlTrSCxeHHWZUSgXr047rsvmi5drFVgfWGh1YOnD7JwtyNQ7FnAukPrMFz8O61mmZrOK4pW\n1VpRPax6rleN1cBQRBSFL4Cc0rawFbW22LcPfvzRChILF1qD2bY4IBqAKlWs22A7d4Z27bxzG2xW\n/j33L3/u/dN5RbHyn5UXXboDoFLJSlaQcFxVXH3F1Zdc50kDg1KqSPrnHztIzJ9vTbDLSlAQREdb\nQaJLF6hVy6PVvKgzSWf4a99fziuKv/b9xfnk8xd9TtniZWkZ0dJ5RRFVMSrTzGwNDEqpIu/IEWtP\n6xkzYNYsa8G/7NSubQWIzp2tMQpPruF0KYnJiaz8Z6XzimLRnkWcSjp10eeEBoXSomoL5xVFk8pN\nKB5YXANDUVDUugwuRtvCpm1hS2uL5GRr9deZM61Akd1tsAAlSli3wqYFisqVPVffnEhJTWHNoTXW\nOMUeK1gcPXv0os8J9g8mcWhirgNDwVoNSimlciEgAFq2tB6vv26t/jpzpvX44w84e9Yue+YM/Pyz\n9QBo2NAOEs2bW7fHepO/n3V7a+MrG/NY88cwxrD56GbnnU/zd89nX8I+l+dcbN2ni9ErBqVUkXT+\nvDUekXY1sX179mXLloWOHa1A0bFj9hPrvMkYw+6Tu13ufNp6bCu8hHYlKaVUbhljrds0Y4YVKObP\nhwsXsi7r5wfNmtlXE1FRvnE7bFYOnj7IlSWv1MBQFGhfsk3bwqZtYctrW5w6BXPm2IEiu70lAK68\n0r7L6eaboaSP7etzOXclFd6NTpVS6jKVLAm33w6ff27Nl1i9GkaMgBYtrCuG9P75B774Au6809rO\n9OabYeRI2LIl49yKgkOvGJRSKheOHbNug50xw7ot9vjx7MtGRFiT6tq2tR7eWKpD5zEopZQHpaTA\n0qX2AHZ8/MXL161rB4roaGtQ2900MBQR2pds07awaVvYvNUW+/fDr7/at8Oeush8NBFo1MgKEu3a\nwU03uWe5Dp8bYxCRKiIyV0Q2iMg6ERmcTblRIrJNROJFJMqddVJKKXepXBkeeMBa6O/YMWuhvxEj\nrOXBM86qNsZaUvydd6z9rsuUseZbDB8OCxZA4uVNQcgXbr1iEJGKQEVjTLyIhAIrgduMMZvTlekE\nDDLGdBGRZsAHxpjmWZxLrxiUUgXWuXPWLOw5c2DuXFi+3OqKyk7x4lagSLuiaNTo8ibZ+XxXkoj8\nBHxojJmTLu1TYJ4xZrLjeBMQbYw5lOG5GhiUUoXGyZPWqrBpgWLt2ouXDwuzxiXSBrLr1cvZ/Amf\n60pKT0QigShgaYasysDedMf7HWkqG3Fxcd6ugs/QtrBpW9gKQluULg233mrd2rpmDRw6BJMnw4MP\nZr3q64kT8NNPMHgwNGhgzZ+4917rVtmdF98sLtc8slaSoxvpe+AxY8zpyz1PbGwskZGRAISFhREV\nFeUcYEr7IOhx0TpO4yv18eZxfHy8T9XHm8fxjtuDfKU+OT3u2TOanj2t40OH4Ny5aObMgV9/jePY\nMUjbYwKs/IkTo5k40TquWBG6dInmyivj2Lp1HMWL4/y+zC23dyWJSAAwHfjVGPNBFvkZu5I2A621\nK0kppSzGWPtgp3U7zZt38fkTAPXrW11OH37og2MMIjIBOGqMGZJNfmdgoGPwuTnwvg4+K6VU9lJT\nre6ntECxYIG1OmzWfGyMQURuBO4D2orIahFZJSK3iMhDIvIggDFmJrBTRP4GPgMecWedCoOM3ShF\nmbaFTdvCVtjbws/PukvpqaesORPHj8OiRfDyy9C6tbVTXV64dYzBGPMncMkbrIwxg9xZD6WUKsyC\nguDGG63HsGHWPhN//mldUbz5Zu7PpzOflVKqEPPp21WVUkoVDBoYCqDC3n+aG9oWNm0Lm7ZF3mhg\nUEop5ULHGJRSqhDTMQallFJ5poGhANL+U5u2hU3bwqZtkTcaGJRSSrnQMQallCrEdIxBKaVUnmlg\nKIC0/9SmbWHTtrBpW+SNBgallFIudIxBKaUKMR1jUEoplWcaGAog7T+1aVvYtC1s2hZ5o4FBKaWU\nCx1jUEqpQkzHGJRSSuWZBoYCSPtPbdoWNm0Lm7ZF3mhgUEop5ULHGJRSqhDTMQallFJ55tbAICJf\niMghEVmbTX5rETkhIqscjxfdWZ/CQvtPbdoWNm0Lm7ZF3rj7imEs0PESZRYYYxo7HiPcXJ9CIT4+\n3ttV8BnaFjZtC5u2Rd64NTAYYxYB/16iWK76vhScOHHC21XwGdoWNm0Lm7ZF3vjCGMMNIhIvIjNE\npJ63K6OUUkVdgJdffyUQYYw5KyKdgJ+Aq7xcJ5+3a9cub1fBZ2hb2LQtbNoWeeP221VFpBrwizHm\n2hyU3QlcZ4w5nkWe3quqlFKXIbe3q3riikHIZhxBRMKNMYccPzfFClSZggLk/o0ppZS6PG4NDCLy\nLRANlBORPcBwIAgwxpjRQHcRGQBcAM4Bd7uzPkoppS6twMx8Vkop5Rm+cFfSJYnILSKyWUS2isj/\nebs+3iIiVURkrohsEJF1IjLY23XyJhHxc0yMnObtunibiJQWkSkissnx+Wjm7Tp5g4g8ISLrRWSt\niHwjIkHerpMnZTWpWETKiMhsEdkiIrNEpPSlzuPzgUFE/ICPsCbK1QfuEZG63q2V1yQDQ4wx9YEb\ngIFFuC0AHgM2ersSPuIDYKYx5mqgIbDJy/XxOBGpBDwKNHbc7BIAxHi3Vh6X1aTiZ4E/jDF1gLnA\nc5c6ic8HBqApsM0Ys9sYcwGYBNzm5Tp5hTHmoDEm3vHzaaz//JW9WyvvEJEqQGdgjLfr4m0iUgpo\naYwZC2CMSTbGJHi5Wt7iD5QQkQAgBDjg5fp4VDaTim8Dxjt+Hg/cfqnzFITAUBnYm+54H0X0yzA9\nEYkEooCl3q2J14wEngZ0kAyqA0dFZKyja220iBT3dqU8zRhzAHgX2APsB04YY/7wbq18QoW0uz+N\nMQeBCpd6QkEIDCoDEQkFvgcec1w5FCki0gU45Lh6yvZ26CIkAGgMfGyMaQycxeo+KFJEJAzrr+Nq\nQCUgVETu9W6tfNIl/5gqCIFhPxCR7riKI61Iclwifw98ZYz52dv18ZIbgW4isgOYCLQRkQlerpM3\n7QP2GmNWOI6/xwoURc3NwA5jzHFjTAowFWjh5Tr5gkMiEg4gIhWBw5d6QkEIDMuBWiJSzXGHQQxQ\nlO9C+RLYaIz5wNsV8RZjzPPGmAhjTA2sz8NcY0xvb9fLWxzdBHtFJG05mXYUzUH5PUBzESkmIoLV\nDkVuEJ7MV9HTgFjHz32AS/5B6e21ki7JGJMiIoOA2ViB7AtjTFH8ZSMiNwL3AetEZDXWJeHzxpjf\nvFsz5QMGA9+ISCCwA+jr5fp4nDFmmYh8D6zGmjS7Ghjt3Vp5VjaTit8ApohIP2A30POS59EJbkop\npdIrCF1JSimlPEgDg1JKKRcaGJRSSrnQwKCUUsqFBgallFIuNDAopZRyoYFBKQ8QkdYi8ou366FU\nTvXBgW4AAAGuSURBVGhgUMpzdNKQKhA0MCiVjojcJyJLHauUfuLYDOiUiLzn2ADmdxEp5ygbJSJL\nRCReRH5I2wBFRGo6ysWLyAoRqe44fcl0m+l85bU3qdQlaGBQysGx6dHdQAvHKqWpWEuQhADLjDEN\ngAVYywyAtbb908aYKGB9uvRvgA8d6S2AfxzpUVhLV9QDaoqILvCmfJLPr5WklAe1w1qVdLljEbZi\nwCGsAPGdo8zXwA+OzXFKOzZGAStIfOdYEr2yMWYagDEmCcA6HcuMMf84juOBSGCxB96XUrmigUEp\nmwDjjTEvuCSKDM1QzqQrnxuJ6X5OQf//KR+lXUlK2eYA3UXkCnBuoh6BtV1kd0eZ+4BFjq0zjztW\nvAXoBcx3bJy0V0Ruc5wjqCjupqYKNv2LRSkHY8wmEXkRmC0ifkASMAg4AzR1XDkcwhqHAGtt+88c\nX/zpl7ruBYwWkVcc5+iR1cu5750olTe67LZSlyAip4wxJb1dD6U8RbuSlLo0/etJFSl6xaCUUsqF\nXjEopZRyoYFBKaWUCw0MSimlXGhgUEop5UIDg1JKKRcaGJRSSrn4f+O7exYSL/ZXAAAAAElFTkSu\nQmCC\n",
491 |       "text/plain": [
492 |        "<matplotlib.figure.Figure at 0x11c3f8810>"
493 |       ]
494 |      },
495 |      "metadata": {},
496 |      "output_type": "display_data"
497 |     }
498 |    ],
499 |    "source": [
500 |     "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
501 |     "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
502 |     "pyplot.grid()\n",
503 |     "pyplot.legend()\n",
504 |     "pyplot.xlabel(\"epoch\")\n",
505 |     "pyplot.ylabel(\"loss\")\n",
506 |     "pyplot.show()"
507 |    ]
508 |   },
509 |   {
510 |    "cell_type": "markdown",
511 |    "metadata": {},
512 |    "source": [
513 |     "# Mini Batch"
514 |    ]
515 |   },
516 |   {
517 |    "cell_type": "markdown",
518 |    "metadata": {},
519 |    "source": [
520 |     "#### Mini Batch allows a faster gradient descent"
521 |    ]
522 |   },
523 |   {
524 |    "cell_type": "code",
525 |    "execution_count": 15,
526 |    "metadata": {
527 |     "collapsed": true
528 |    },
529 |    "outputs": [],
530 |    "source": [
531 |     "batch_size = 64\n",
532 |     "\n",
533 |     "graph = tf.Graph()\n",
534 |     "with graph.as_default():\n",
535 |     "    X_train_tf = tf.placeholder(tf.float32,\n",
536 |     "                                    shape=(batch_size, X.shape[1]))\n",
537 |     "    y_train_tf = tf.placeholder(tf.float32, shape=(batch_size, nb_classes))\n",
538 |     "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
539 |     "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
540 |     "    \n",
541 |     "    weights = tf.Variable(\n",
542 |     "        tf.truncated_normal([X.shape[1], nb_classes]))\n",
543 |     "    biases = tf.Variable(tf.zeros([nb_classes]))\n",
544 |     "\n",
545 |     "    logits = tf.matmul(X_train_tf, weights) + biases\n",
546 |     "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
547 |     "    optimizer = tf.train.GradientDescentOptimizer(0.8).minimize(loss)\n",
548 |     "    \n",
549 |     "    train_prediction = tf.nn.softmax(logits)\n",
550 |     "    test_prediction = tf.nn.softmax(tf.matmul(X_test_tf, weights) + biases)"
551 |    ]
552 |   },
553 |   {
554 |    "cell_type": "code",
555 |    "execution_count": 16,
556 |    "metadata": {
557 |     "collapsed": false
558 |    },
559 |    "outputs": [
560 |     {
561 |      "name": "stdout",
562 |      "output_type": "stream",
563 |      "text": [
564 |       "Train Loss at step 0: 4.617716\n",
565 |       "Valid Loss: 4.640052\n",
566 |       "Train Loss at step 4000: 3.103014\n",
567 |       "Valid Loss: 3.245315\n",
568 |       "Train Loss at step 8000: 2.139842\n",
569 |       "Valid Loss: 2.453359\n",
570 |       "Train Loss at step 12000: 1.759780\n",
571 |       "Valid Loss: 1.968372\n",
572 |       "Train Loss at step 16000: 1.318464\n",
573 |       "Valid Loss: 1.650348\n",
574 |       "Train Loss at step 20000: 1.119166\n",
575 |       "Valid Loss: 1.427687\n",
576 |       "Train Loss at step 24000: 0.949450\n",
577 |       "Valid Loss: 1.267724\n",
578 |       "Train Loss at step 28000: 0.779175\n",
579 |       "Valid Loss: 1.146158\n",
580 |       "Train Loss at step 32000: 0.690034\n",
581 |       "Valid Loss: 1.052394\n",
582 |       "Train Loss at step 36000: 0.688905\n",
583 |       "Valid Loss: 0.978355\n",
584 |       "Train Loss at step 40000: 0.530511\n",
585 |       "Valid Loss: 0.917245\n",
586 |       "Train Loss at step 44000: 0.473384\n",
587 |       "Valid Loss: 0.867600\n",
588 |       "Train Loss at step 48000: 0.526159\n",
589 |       "Valid Loss: 0.825346\n",
590 |       "Train Loss at step 52000: 0.430732\n",
591 |       "Valid Loss: 0.789898\n",
592 |       "Train Loss at step 56000: 0.343112\n",
593 |       "Valid Loss: 0.759255\n",
594 |       "Train Loss at step 60000: 0.403765\n",
595 |       "Valid Loss: 0.732507\n",
596 |       "Train Loss at step 64000: 0.368342\n",
597 |       "Valid Loss: 0.709447\n",
598 |       "Train Loss at step 68000: 0.287409\n",
599 |       "Valid Loss: 0.688575\n"
600 |      ]
601 |     }
602 |    ],
603 |    "source": [
604 |     "num_steps = 70001\n",
605 |     "train_loss = []\n",
606 |     "test_loss = []\n",
607 |     "\n",
608 |     "with tf.Session(graph=graph) as session:\n",
609 |     "    tf.global_variables_initializer().run()\n",
610 |     "    for step in range(num_steps):\n",
611 |     "        # Pick an offset within the training data, which has been randomized.\n",
612 |     "        # Note: we could use better randomization across epochs.\n",
613 |     "        offset = (step * batch_size) % (len(y_train) - batch_size)\n",
614 |     "        # Generate a minibatch.\n",
615 |     "        batch_data = np.array(X_train)[offset:(offset + batch_size), :]\n",
616 |     "        batch_labels = y_train[offset:(offset + batch_size)]\n",
617 |     "        batch_labels = (np.arange(nb_classes) == batch_labels[:,None]).astype(np.float32)\n",
618 |     "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
619 |     "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
620 |     "        # and the value is the numpy array to feed to it.\n",
621 |     "        feed_dict = {X_train_tf : batch_data, y_train_tf : batch_labels}\n",
622 |     "        _, l, predictions = session.run(\n",
623 |     "              [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
624 |     "        if (step % 4000 == 0):\n",
625 |     "            train_loss.append(l)\n",
626 |     "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
627 |     "            print('Train Loss at step %d: %f' % (step, l))\n",
628 |     "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
629 |    ]
630 |   },
631 |   {
632 |    "cell_type": "code",
633 |    "execution_count": 17,
634 |    "metadata": {
635 |     "collapsed": false
636 |    },
637 |    "outputs": [
638 |     {
639 |      "data": {
640 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEPCAYAAABBUX+lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Wl4FFX69/Hv6YQQkhDCGpYACasKyKKA4BZEVnXckBGV\nTUcHRtx1xAVx/jM6jo6oqA+KCoKjoCgoOiAoEBBhBJR934LsBEhICCEhyXlenE5nh0ro5YTcn+uq\nq7uqq7t+IeFU9V1V5yitNUIIIS58rkAHEEII4R/S4AshRCUhDb4QQlQS0uALIUQlIQ2+EEJUEtLg\nCyFEJRHs6w0opRKBE0AucEZr3cXX2xRCCFGczxt8TEMfr7VO9sO2hBBClMIfJR3lp+0IIYQ4C380\nxBr4QSm1Uil1vx+2J4QQogT+KOlcqbU+qJSqi2n4N2utl/phu0IIIQrweYOvtT7ofkxSSs0CugCF\nGnyllHToI4QQZaS1VmVZ36clHaVUmFIqwv08HOgNbChpXa21VdPYsWMDnkEyXTiZbM0lmSpupvLw\n9RF+NDDLfQQfDHyqtZ7v4216RWJiYqAjFCOZnLExE9iZSzI5Y2Om8vBpg6+13g108OU2hBBCOCOX\nS5Zi2LBhgY5QjGRyxsZMYGcuyeSMjZnKQ5W3FuTVEEppG3IIIURFoZRC23TStiJLSEgIdIRiJJMz\nNmYCO3OVlik2NhallEwWTLGxsV77ffvjOnwhRAWzZ8+ecl8JIrxLqTIdxJ/9s2z4pUpJRwi7uMsF\ngY4hKP13ISUdIYQQpZIGvxQVqd4aSJLJORtz2ZhJ+I40+EKISmfkyJG89NJL5Xpvjx49mDRpkpcT\n+YfU8IUQxdhcw4+Li+Ojjz7iuuuuC8j2e/ToweDBg7n33nv9sj2p4QshRClycnICHcFa0uCXwsba\npmRyxsZMYGcuGzOdzZAhQ/j999+56aabiIyM5N///jd79uzB5XIxadIkmjZtSs+ePQEYOHAgDRo0\noGbNmsTHx7Np0ybP5wwfPpwXXngBgMWLF9O4cWPGjRtHdHQ0jRo14uOPP3aUR2vNP/7xD2JjY6lf\nvz7Dhg0jNTUVgMzMTAYPHkydOnWoWbMmXbt2JSkpCYCPP/6Y5s2bExkZSfPmzZk2bZoX/5VKZ02D\nfzDtYKAjCCEcUMq7U1lMnTqVJk2a8N1335GamsqTTz7peW3JkiVs2bKFefPmAdC/f3927tzJkSNH\n6NSpE3fffXepn3vo0CHS0tI4cOAAH374IQ8++CAnTpw4Z57JkyczdepUFi9ezK5du0hLS+Ohhx4C\nYMqUKaSmprJ//36OHz/Oe++9R7Vq1Th16hSPPPII8+bNIzU1lWXLltGhg5+6HAt0F5/u2pR+fdGH\nWghhB9M0lPaad6eyio2N1QsWLPDMJyYmapfLpRMTE0t9T3JyslZK6dTUVK211sOGDdNjxozRWmud\nkJCgw8LCdE5Ojmf9evXq6V9++aXEz4qPj9cfffSR1lrrnj176gkTJnhe27p1qw4JCdE5OTl60qRJ\n+sorr9Tr1q0r9P709HRds2ZNPXPmTJ2RkXHOn7e034V7eZnaWmuO8D9d+W2gIwghKrCYmBjP89zc\nXEaPHk2LFi2IiooiLi4OpRRHjx4t8b21a9fG5cpvDsPCwjh58uQ5t3ngwAGaNm3qmW/atClnzpzh\n8OHDDB48mD59+nDnnXcSExPD6NGjycnJISwsjM8//5wJEybQoEEDbrrpJrZu3XoeP7lz1jT4a0/+\nwOns04GO4WFjbVMyOWNjJrAzV3kyefsYv6xK62qg4PLPPvuMb7/9loULF5KSkkJiYuJ5DRxSmoYN\nG7Jnzx7P/J49e6hSpQrR0dEEBwczZswYNm7cyLJly/j222+ZOnUqAL169WL+/PkcOnSI1q1bc//9\n/hnu25oGP8d1igW7FgU6hhDCcvXr12fXrl2FlhVtyNPS0qhatSo1a9YkPT2dZ555xqt90uQZNGgQ\nb7zxBomJiZw8eZLnnnuOO++8E5fLRUJCAhs2bCA3N5eIiAiqVKmCy+XiyJEjzJ49m1OnTlGlShUi\nIiIICgryeraSWNPgA0xd/l2gI3jEx8cHOkIxkskZGzOBnblszHQuo0eP5u9//zu1atVi3LhxQPGj\n/iFDhtCkSRMaNWpE27Zt6d69e5m2cbadQ8HX7r33XgYPHsw111xD8+bNCQsLY/z48YA5ETxgwABq\n1KhBmzZtPNfv5+bmMm7cOBo1akSdOnVYsmQJEyZMKFO+8rLmxitehBqqMclj9vhkTyyEcM7mG68q\nmwv2xqsTei/rj6wPdAzgwqm3+ppkcs7GXDZmEr5jVYMP8NUGuVpHCCF8waqSDsBF4Vew+cnlAc0j\nRGUnJR17XJglnVwTZcvJXziSfiTAYYQQ4sJjT4O/90rzqDRzts8JbBbsrG1KJmdszAR25rIxk/Ad\naxr8oJ03eZ5/sdaeyzOFEOJCYU0Nv9sfNrG80yUAVFURnHj2KFWDqwY4mRCVk9Tw7XFB1vBvveoi\nON4MgEx9kiV7lgQ4kRBCXFisafD79FGwLb+sM3trYC/PtLG2KZmcsTET2JnLxky+ktfvfZ62bduy\nZEnJB5ZF1y3K5XIV696hIrCmwW/XDmom3eiZn7nhO/lKKYTwqoJ38W/YsIFrrrnG0bplec1m1jT4\nSkH/NtdAZnUADmTsZlPSpnO8y3ds7GNEMjljYyawM5eNmSqCinowak2DD9C/Twjs6OOZ/26bXK0j\nhMj36quvcscddxRa9sgjj/Doo48CZujASy65hMjISFq0aMHEiRNL/ay4uDgWLlwIwOnTpxk2bBi1\natWibdu2rFy50nGm1NRUhgwZQr169YiLi+Oll17yvLZz507i4+OJioqiXr16DBo0yPPaY489RnR0\nNDVq1KB9+/aFhmD0lWCfb6EMrr8eePUmaPMlAF9v/panr3o6IFkSEhKsO/qRTM7YmAnszFWeTOpv\n3i1n6LHOj5bvvPNO/u///o/09HTCw8PJzc1lxowZfPPNNwBER0czZ84cYmNj+emnn+jbty9dunQ5\n5xCCL774Irt372b37t2cPHmSvn37Os40atQo0tLSSExMJCkpid69e9OwYUOGDx/OmDFj6NOnDwkJ\nCWRlZbFq1SoA5s+fz9KlS9mxYwfVq1dn69atREVFOd5meVl1hF+vHrSr1g/cVxqtOLCco6dKHqFG\nCFH5NGnShE6dOjFr1iwAFixYQHh4OJ07dwagX79+xMbGAnD11VfTu3dvfvrpp3N+7owZM3j++eep\nUaMGjRo14uGHH3aUJzc3l88//5xXXnmFsLAwmjZtyhNPPMEnn3wCQJUqVdizZw/79+8nJCTE001z\nlSpVSEtLY9OmTWitad26NdHR0WX95ygzqxp8gBvi68LebgDkksvc7XMDksO2IzGQTE7ZmAnszGVj\npnMZNGgQ06ZNA2DatGncddddntfmzp1Lt27dqF27NjVr1mTu3LmlDmtY0IEDBwoNkVhw2MKzOXr0\nKNnZ2TRp0qTQe/fv3w+YElRubi5dunShXbt2TJ48GYAePXowatQoHnzwQaKjoxkxYoSjIRXPW1kH\nwfXFRIFBehcu1JqrXta8iOZF9MAZA885yK8Qwrsoz+jifpKUlKTDwsL0vn37dFRUlN6yZYvWWuvM\nzEwdFhamZ86c6RmQ/JZbbik0WHnjxo09n1NwMPS4uDg9b948z2sTJ04stG5RSim9c+dOnZOTo6tW\nrao3b97see3999/XPXr0KPaepUuX6tDQUL1z585iP098fLx+4YUXStxWab8LKvIg5nm6d4dqe/Ov\nx5+z7XuycrL8nsPG65MlkzM2ZgI7c9mY6Vzq1KnDtddey/Dhw2nWrBmtW7cGICsri6ysLOrUqYPL\n5WLu3LnMnz/f0WcOHDiQf/7zn6SkpLBv3z7eeecdR+9zuVwMHDiQ5557jpMnT7Jnzx7eeOMNBg8e\nDMCXX37pOdqPiorC5XLhcrlYtWoVK1asIDs7m2rVqhEaGlpoEHVfsa7Br1oVerRtAynmK9XJM6ks\n/X1pgFMJIWxy1113sWDBAu6++27PsoiICMaPH88dd9xBrVq1mD59OjfffHOpn1HwWvqxY8fSpEkT\n4uLi6Nu3L0OGDDnr9gu+d/z48YSFhdGsWTOuueYa7rnnHoYPHw7AypUr6dq1K5GRkdxyyy2MHz+e\n2NhYUlNTuf/++6lVqxZxcXHUqVOHp556qrz/HI75pS8dpZQLWAXs01r/oYTXdcEcb78ND899CLqa\nveyjXR/ljb5v+DynEMKQvnTsURH70nkEcHyRae/ewLb8u26/3fat/PEJIcR58nmDr5SKAfoDHzp9\nT6tW0CQ3HrLCAdiZvJOtx7b6JmApbKxtSiZnbMwEduayMZPwHX8c4b8BPAU4PkRXCvpcXxV29vYs\nk7tuhRDi/Pj0Tlul1A3AYa31GqVUPFBqvWnYsGGeGyaioqJo1KgDrLwJLp4Fu+GT2Z/wZPcngfyj\nkrxriH01n8df26uI8/Hx8VblyVPwDtJA57F5/my/P2GPvN9JQkICiYmJ5f4cn560VUq9DNwDZAPV\ngOrATK31kCLr6aI5kpOhdtPD6McbgNIEqSCOPHWEWtVq+SyvEMKQk7b2qDAnbbXWz2qtm2itmwF3\nAguLNvalqVkTuraJhv1dAMjROXy/43sfpi3MxqMcyeSMjZnAzlw2ZhK+Y911+AX16UOhq3Wkji+E\nfzRt2hSllEwWTE67eXDCmjFtS8qxbBlcedtaGGl6uosKjeLIk0eoElTF3xGFEMIq1pV0zleXLhB5\n+lI4YYYaSzmdwrK9ywKcSgghKiarG/zgYLi+pyp2E5Y/2FjblEzO2JgJ7MwlmZyxMVN5WN3gQ/G7\nbqWOL4QQ5WN1DR9g925o1uo0/LU2hJwCYNuobbSs3dKfEYUQwioXXA0fIC4OWsaFwq7rPcvkKF8I\nIcrO+gYfSrg8c7vvG3wba3aSyRkbM4GduSSTMzZmKo8K0eD37g1sv8Ezv2TPEk6cPhG4QEIIUQFZ\nX8MHOHkSatWCM8Mvh4a/AvD5gM8Z2GagvyIKIYRVLsgaPkBEhBn6kK35Qx/66/JMIYS4UFSIBh+K\nX545Z/sccnJzfLY9G2t2kskZGzOBnbkkkzM2ZiqPCtPg9+kDHOwEqQ0BOJ5xnOX7lgc2lBBCVCAV\nooYPkJsL0dFwtNsDcNkHADx95dO8cv0r/ogohBBWuWBr+AAuF/TqhdTxhRCinCpMgw/uOv7unnAm\nFIBNSZvYlbzLJ9uysWYnmZyxMRPYmUsyOWNjpvKoeA3+mTDT6LvJXbdCCOFMhanh52nXDjaEvgc3\njgSgV7NezB8835fxhBDCOhd0DT9P0cszExITSM1MDVwgIYSoICpcg9+nD5AaAwfNKFhncs/ww84f\nvL4dG2t2kskZGzOBnbkkkzM2ZiqPCtfgX301hIYC2+RqHSGEKIsKV8MHc5Q/f+MKuL8rAHXD6nLw\niYMEuYJ8FVEIIaxSKWr44K7jH7gcTkYDkHQqiRX7VwQ2lBBCWK5CNvh9+gDaBdvyu0z29uWZNtbs\nJJMzNmYCO3NJJmdszFQeFbLBb9MGGjRA6vhCCFEGFbKGDzBsGEyZdtKMdRucBUDiI4k0jWrqg4RC\nCGGXSlPDB3dZJysCdl/nWSZ33QohROkqbIN//fWgFD4b69bGmp1kcsbGTGBnLsnkjI2ZyqPCNvh1\n60KnThRq8BfuXsjJrJOBCyWEEBarsDV8gGefhX/+Exh5KUSvB2DWH2dxy0W3eDmhEELYpVLV8MFd\nx4dCfeRP2zAtMGGEEMJyFbrB79YNwsOBzbd7ls3YOIN1h9ed92fbWLOTTM7YmAnszCWZnLExU3lU\n6AY/JAR69MCMdbvV1PI1mucXPh/YYEIIYaEKXcMHeOcdeOghIHodjOgAynzOz/f+TPfG3b2YUggh\n7FHpavjg7lcH4PClBG8e5Fn+7IJnsWFnJoQQtqjwDX7LlhAba55n//g3glQwAIv3LOaHXeXvJ9/G\nmp1kcsbGTGBnLsnkjI2ZyqPCN/hKFTjKP96CS7Pv87wmR/lCCJHPpzV8pVRVYAkQAgQDX2qt/1bC\neuWu4QPMmgW33WaeN2i1n+QhLTidfRqAGXfMYMAlA8r92UIIYSPravha60ygh9a6I9AB6KeU6uLt\n7fTvD/XqmecHtzWiV+RDntfGLBpDdm62tzcphBAVjs9LOlrrU+6nVTFH+V7/SlG1KowYkT9/6Kun\niawaCcCWo1v4ZO0nZf5MG2t2kskZGzOBnbkkkzM2ZioPnzf4SimXUmo1cAj4QWu90hfbGTECqlQx\nz1curs2gpk96Xntx8YtkZmf6YrNCCFFh+O06fKVUJPA1MEprvanIa+dVw89zzz3w6afm+Z1D0ljQ\ntjlJp5IAeKvvWzzc9eHz3oYQQtigPDV8v954pZQaA6RrrccVWa6HDh1KrPv6yqioKDp06EB8fDyQ\n/3XqXPPh4fF06QKQQFAQjJ27lheWPQq7oUZoDfaN30dESITjz5N5mZd5mbdlPu95YmIiAFOmTClz\ng4/W2mcTUAeo4X5eDXPFTv8S1tPe0q2b1mCm58Zm6MbjGmteRPMi+u+L/+74cxYtWuS1TN4imZyx\nMZPWduaSTM7YmMndbpapTfZ1Db8BsEgptQb4BZintZ7jyw0+XKBq88GEUJ678kXP/GvLXuN4xnFf\nbl4IIaxV4fvSKerMGYiLg/37zfzkKdm8ktqWrce2AvDX7n/lX73+5ZVtCSFEoFh3HX4gVKkCf/lL\n/vw744P5vx5/98yPXzGeA2kHApBMCCEC64Jr8AEeeABCQ83zX3+FBsm306lBJwBOZ5/mH0v+cc7P\nKHiixBaSyRkbM4GduSSTMzZmKo8LssGvUwfuvjt//u3xLl6+7mXP/Ae/fcDO4zsDkEwIIQLngqvh\n51m3Dtq3N8+DgmDXLs3ghfEs2bMEgLvb3c1/bvuPV7cphBD+IjX8Ai69FNyXsZKTAxMmKP7Z85+e\n1z9b/xnrD68PTDghhAiAC7bBB3jkkfznEydCh9rdubFVgaEQF5U+FKKNNTvJ5IyNmcDOXJLJGRsz\nlccF3eDfdFP+4CjHj5tuF1667iXP67O3zmb53uWBCSeEEH52wdbw87z+Ojzp7ketbVtT27975l1M\n2zANgPjYeBYOWYhSZbtDWQghAsn6vnRKDeHDBj8lBWJiID3dzC9YAE067ODidy/29JM//5759Gre\nyyfbF0IIX5CTtiWIioKhQ/Pn33oLWtRqwX0dCwyFuLD4UIg21uwkkzM2ZgI7c0kmZ2zMVB4XfIMP\n8FD+AFh8+y3s2gVjrhlDaLC5O2vVgVXM3DwzQOmEEMI/HJV0lFKPAJOBNOBDoCMwWms93yshfFjS\nydO3L8ybZ54/9hiMGwdPzX+Kfy//NwAX1bmIDSM3EOQK8mkOIYTwBl+WdO7VWqcCvYGawGDglTLm\nC6iCl2h+9BGkpcHoq0ZTPaQ64B4KcV3Zh0IUQoiKwmmDn7cX6Q98orXeWGBZhdCnD7RqZZ6npsKU\nKVA7rDZPds8fCnFswljPUIg21uwkkzM2ZgI7c0kmZ2zMVB5OG/xflVLzMQ3+PKVUdSDXd7G8z+Uq\n3Ff+229Dbi48dsVj1A2rC8DvJ35n4q8TA5RQCCF8y2kN3wV0AHZprVOUUrWAGK31Oq+E8EMNH0wZ\nJybGHOEDzJkD/frBm/97k8fmPQZAvfB67Hx4JxEhET7PI4QQ5eXLGn43YKu7sb8HeB44UdaAgVa9\nOtyXfzUmb71lHkdcPoLGkY0BOJJ+hLf+91YA0gkhhG85bfAnAKeUUu2BJ4CdwFSfpfKhUaMg76ba\nefNgyxYIDQ5l7LVjPeu8tuw1Zs+bHaCEpbOxjiiZnLMxl2RyxsZM5eG0wc9211xuBt7RWr8LVPdd\nLN9p1gz+8If8+bffNo9DOwylde3WAJzIPMH0DdMDkE4IIXzHaQ1/MfA9cC9wNXAEWKu1bueVEH6q\n4edZtAiuu848Dw+HffvMHbkzNs5g4JcDAagWXI3Vf15N6zqt/ZZLCCGc8mUN/49AJuZ6/ENADPBa\nGfNZIz7edKQGpo+djz4yz2+/JH8oxIzsDPp/1p+k9KTAhBRCCC9z1OC7G/lPgRpKqRuB01rrClnD\nB1PDL3gj1jvvmEFSXMrFBzd9QFiVMNgNu5J3ccvnt3A6+3TgwhZgYx1RMjlnYy7J5IyNmcrDUYOv\nlBoIrADuAAYCvyilBvgymK/dfTfUrm2eJyaaPnYAOjXoxGe3feZZb9neZQz7ehi5ukLddiCEEMU4\nreGvBXpprY+45+sCP2qt23slhJ9r+HmefRb+6R71MD7e1PbzFLw2H+C5q5/jH9f9w78BhRCiFL6s\n4bvyGnu3Y2V4r7VGjjQDnAMkJJjBUfI80vURHuz8oGf+pZ9eYvLqyf4NKIQQXuS00f5eKTVPKTVM\nKTUM+C8wx3ex/KNxY7j99vz5twrcb7V48WLe7Psm/Vr08yx74LsHWLh7oR8TFmZjHVEyOWdjLsnk\njI2ZysPpSdungInApe5potb6aV8G85eCJ28//RSOHs2fD3YF8/mAz7k0+lIAsnOzue3z29ictNnP\nKYUQ4vxd8EMcnovW0KULrFpl5l96ydT2C9qXuo+uH3blQNoBAGKjYvnfff8jOiLaz2mFEMLweg1f\nKZWmlEotYUpTSqWeX1w7KFW4F83/9//gzJnC68RExvDtoG8JrxIOQGJKIjdPv5mMMxl+TCqEEOfn\nrA2+1rq61jqyhKm61jrSXyF9beBAiHYfrO/fDzNnFq/ZdWrQiekDpuNS5p/sl/2/MPTroX69XNPG\nOqJkcs7GXJLJGRszlUeFv9LGG6pWNVfs5HmrlM4yb2x1I2/2edMzP2PTDJ5b8JyP0wkhhHdU+hp+\nnsOHzVU7eeWcFSugc+eS131k7iOMXzHeM//BTR/wp05/8kNKIYQwfHkd/gUvOhruvDN/vrSjfIBx\nfcZxU6ubPPMjvhvBDzt/8GE6IYQ4f9LgF1DwEs3p0xNYvrzk9YJcQXx2+2d0rN8RgBydw4AZA9h4\nZKNP89lYR5RMztmYSzI5Y2Om8pAGv4DLLoOrrjLPc3KgZ0+YO7fkdSNCIvjuru+IiYwBIDUzlRs+\nu4FDJw/5Ka0QQpSN1PCL2LDB9JWf5O4VOTgYJk+Ge+4pef21h9Zy1eSrOJl1EoDODTuTMCzB9Lgp\nhBA+Yl0NXykVo5RaqJTaqJRar5R6+NzvCqy2beHnnyE21sxnZ8PgwTBuXMnrt6/fni8GfOG5XHPl\ngZXcM/Me6V1TCGEdX5d0soHHtdZtMAOhP6iUusjH2zxvLVvCa68l0K7AeF5PPAGjR5s7c4vq17If\n7/R7xzM/a8ssnv7B+z1P2FhHlEzO2ZhLMjljY6by8GmDr7U+pLVe435+EtgMNPLlNr2lTh1YsiS/\npg/wr3/Bn/5kjvqLGtl5JI9f8bhn/t/L/817q97zQ1IhhHDGbzV8pVQskAC0dTf+BV+zpoZfVEYG\n/PGP+QOkgBkEffp0qFat8Lo5uTnc/sXtfLP1GwCCVBDf3fUdfVv09WNiIURlYF0NP49SKgL4Enik\naGNvu2rVTFcLw4fnL5s9G3r3hpSUwusGuYL49LZPuazBZYC5XHPgjIGsPbTWj4mFEKJkPj/CV0oF\nA98Bc7XWJd7OpJTSQ4cOJdZ9pjQqKooOHToQHx8P5NfP/Dm/Zs0aHn30Uc+81jBvXjz/+heYLyrQ\nrl08338P27YVfv9Xc75i5H9HklTPXOoTui+Uv3b/K38b/rfzype3LBD/HqXNF80W6DwAb775ZsD/\nfkqaz1tmSx75/TmfL9oeBCJP3vPExEQApkyZUuYjfLTWPp2AqcC4c6yjbbNo0aISl48bp7U5dWum\n2Fitt24tvt66Q+t05D8jNS/imR6a85DOzM70eqZAkkzO2ZhLMjljYyZ3u1mm9tinR/hKqSuBJcB6\nQLunZ7XW3xdZT/syh7f95z+mxJN38rZuXXOD1mWXFV5vzaE1DPhiADuTd3qWdWnUhS8GfEHTqKZ+\nTCyEuNCUp4YvN16V09y5ZnjEDHeX+BER8PXX5u7cgk6cPsHwb4Yza8ssz7Ja1Wrxn1v/Q7+W/RBC\niPKw9qRtRVSwblaSfv1gwQKoWdPMnzwJ/fvDjBmF16sRWoOvBn7F671fJ9gVDMDxjOP0/6w/zy14\njuzcEq7xLGemQJBMztmYSzI5Y2Om8pAG/zx06wZLl0KM6U6HrCxzCeeECYXXU0rxeLfHWTxsMY2q\n59+G8PLSl+n9SW/pf0cI4RdS0vGC33+HPn1gy5b8ZWPHmkkV+cKVlJ7EPbPuYf7O+Z5l9SPq8/mA\nz7mm6TV+SiyEqOikpBMgTZrATz9B1675y/72N3jwQdPrZkF1w+sy5645vHjtiyjM7+rQyUNcN+U6\n/rX0X9IHjxDCZ6TBL0VZa3Z16sCPP5oj/TwTJsCgQZCZWXjdIFcQY+PHMu+eedQJqwOYm7RGLxjN\nzdNvJjkj2SuZ/EEyOWdjLsnkjI2ZykMafC+KiDB34Q4alL9sxgy45RZT3y+qV/NerP7zaro37u5Z\n9t227+g0sROrDqzyQ2IhRGUiNXwfyM2Fxx8vPEzigAEwbZrpX7+oMzlneGbBM7y+/HXPspCgEN7s\n8yYjLh+BKnoiQAhR6cl1+BbR2py0/fvf85cNGwYffQSuUr5Xzdo8i2HfDCM1M9WzbFDbQUy8aSIR\nIRG+DSyEqFDkpK0XnW/NTilz4tbd/QYAH38Mjz1Wcp/6ALdefCu/PfCbZ6xcgGkbptH5g85sPLLR\nyjqiZHLOxlySyRkbM5WHNPg+pJQZKeu++/KXjR8PL7xQ+nua12rOsvuW8UCnBzzLthzdQpcPuzB/\nx3wutG9CQgj/kZKOH+TkwF13wRdf5C979VV46qmzv++TtZ8w4r8jOHXmlGfZ1U2u5pXrXyl0olcI\nUflIDd9iWVlw660wZ07+svfegz//+ezv23hkI7d/cTtbj20ttPwPrf/Ay9e9TJt6bXyQVghhO6nh\ne5G3a3ZUCpmbAAAah0lEQVQhIfDll3DttfnLRo6ETz89+/va1GvDyvtXMqrzKIL2BHmWz946m0vf\nu5Th3wzn9xO/ezVrWdhY27QxE9iZSzI5Y2Om8pAG34+qVTNDJXbpYua1hqFD4Ztvzv6+6lWr83b/\nt5l661TuaneXZ3muzuXjNR/T8u2WPD7vcY6eOurD9EKIik5KOgFw/Lg50t+wwcyHhMB//wvXX+/s\n/WsOreGZBc/w/Y5CwwoQWTWSp7o/xaNXPCqXcQpxgZMafgVy8CBccw3s2GHmw8Lghx+gexnOxSYk\nJjD6x9H8sv+XQsujw6MZc80Y7r/sfkKCQryYWghhC6nhe5Gva3YNGpi+d/K6Vj51yvSnv2aN80zx\nsfEsv285MwfO5KI6F3mWH04/zKi5o7j43YuZtn6aTztks7G2aWMmsDOXZHLGxkzlIQ1+ADVtahr9\nunXN/IkT0Lt34W6Wz0Upxa0X38r6kev58KYPC/W3vyt5F3fNvIvLJ17OvB3z5Bp+ISo5KelYYO1a\niI+HlBQzHxNjuluOjS37Z2WcyeDdle/y8k8vk3y6cK+b8bHxvNLzFbrGdC3l3UKIikJq+BXY8uXQ\nqxekp5v55s1No9+gQfk+L+V0Cq/+/Cpv/u9NMrIzCr1228W38fzVz9OxQcdS3i2EsJ3U8L3I3zW7\nbt3M5Zkh7nOsO3eaHcCxY+XLFBUaxcs9X2bHwzv482V/JkjlX8M/c/NMOk3sRKf3O/HuindL7X/f\nCRtrmzZmAjtzSSZnbMxUHtLgW6RnT9N/fpC7bd640QyWnpp69vedTcPqDXnvxvfY9OAmBrYZWOi1\n1YdWM2ruKBq83oBBXw3ih50/yIhbQlzApKRjoc8+g3vuye9V85prYO5cc+nm+fr1wK+8vvx1Zm6e\nSWZOZrHXm9RowvAOwxnWYRixUbHnv0EhhE9IDf8C8v77MGJE/ny/fvD11/kln/OVnJHMtA3TmLR6\nEr8e/LXEdXrG9eS+jvdx68W3Ehoc6p0NCyG8Qmr4XhTomt2f/wyvvZY/P3cu9OqVwIkT3vn8mtVq\n8pfOf2HVA6tY8+c1PNzlYWpVq1VonQW7F3DXzLto8HoDRs0ZxW8Hfyt2aWeg/51KYmMmsDOXZHLG\nxkzlIQ2+xZ58Ep5/Pn9+yRJz7f6YMXDUi93mtK/fnrf6vcWBxw/wxYAv6NuiL4r8A4eU0ym8u/Jd\nLpt4GR3f78j4X8Zz7NSxs3yiEMJGUtKxnNZm1Kzx4wsvDwsz3wKeeAIaNSr5vedj74m9TFk7hUmr\nJ7E7ZXex10OCQrjlolsY1n4YPZv1lC4chPAzqeFfoLSGTz6Bl16CbdsKv1alihkr9+mnzbX73par\nc1mcuJhJaybx5aYvOZ19utg6ESER9GrWixta3kC/lv1oWL2h94MIIQqRBt+LEhISiI+PD3SMQhYs\nSODYsXheftncnVuQywV33gnPPANt2/pm+ymnU5i+YTqTVk9i5YGVZuFuIK7weh3rd+SGljdwQ6sb\n6NywM0GuoGKf5Us2/u7AzlySyRkbM8lJ2wtcUBAMHAirV8N335mbtfLk5prLOdu1g1tugZUrvb/9\nqNAoRlw+ghX3r2DdiHU8dsVj1I+oX2y91YdW84+f/kG3j7pR//X6DJ41mOkbpp/XDV5CiPMnR/gV\nmNbmRO5LL5mulYu6/np47jnT974q03FAWTJothzdwpztc/jv9v/y0+8/kZ2bXeK6LuWie+Pu5ui/\n5Q20rdcW5atgQlzgpKRTia1cCS+/bK7VL6pbN9Pw9+/vu4Y/z4nTJ/hh1w/M2T6HOdvncDj9cKnr\nNo5sTP+W/bmh5Q1cF3cd4SHhvg0nxAVEGnwvsrFm5yTTxo3wyiswbRrk5BR+rX17U+MfMCC/+wZf\nZsrVufx28DfP0f/K/SvRlPx7rhpUle6Nu3umK2KuKHZfgDcyBZKNuSSTMzZmKk+DH+yrMCIw2rQx\nV/T87W/w6qsweTJkZZnX1q41J3ZbtoTHH4e774bq1X2XxaVcXN7wci5veDkvXPsCR9KP8P2O7/nv\n9v8yb8c8TmTm30WWmZPJosRFLEpc5Fl2UZ2L6B7TnW6Nu9G9cXcuqnMRLiWnnYQoLznCv8Dt3w/j\nxsF775lRtQqKiDB99owYYY7+/elMzhmW7V3mOfrfmLTxnO+JCo3iipgrPDuBro26Ur2qD/dYQlhM\nSjqiVEePmpu33n47f6CVgrp1Mw3/HXdAtWr+z7cvdR/L9i5j2d5lLN+3nN8O/lbqyd88LuWibb22\ndI8xZaBujbvRvGZzOREsKgXrGnyl1EfAjcBhrfWlZ1nPugbfxpqdNzKlpsKUKTBhAmzeXPz1mjXN\njVwjRkCrVv7JVJKMMxmsOrCK5fuWe3YCR9KPnPN9dcPq0jKtJdf3uJ520e24NPpSmtds7vd7AUpy\nof5NeZtkcsbGGv5k4G1gqo+3IxyKjISHHoJRo8yIWu+9B19+CWfOmNeTk+GNN8x03XWm4b/5Zu/1\n0ulUtSrVuLrp1Vzd9GrAXP65K3mXp/FftncZ64+sL9Z/f9KpJJJ+T2LZkmX5nxVcjTb12tCuntkB\n5D3WDa/r159JiEDzeUlHKdUU+LaiHeFXJkeOmJO7778Pu4t3m0N0NPzpT3D//abzNlukZaaxYv8K\nz05g+b7lpJwuoV5Viujw6EI7gHbR7bik7iXSFbSoEKwr6YA0+BVJbq65geu992D2bDNfkFLmWv6R\nI6FvX+9d2uktuTqXLUe3sOrAKtYfXs+6I+tYf3g9B08edPwZLuWiVe1Wnp1Am7ptaFW7Fc1rNZcd\ngbBKhW7whw4dSmxsLABRUVF06NDBUzPL64van/Nr1qzh0UcfDdj2S5rPW+aP7SUlwcaN8XzwARw4\nkLf9ePdjAvXqwcMPx3PxxQnUqsU5P8+f82+++Wahv59vvv+GXcm7cMW5WH9kPUuXLCUxJZHMxu4R\nv/K+1eT1CVTKfNMOTWlZuyXh+8KJqRFD/179aVmrJXvW7CE4KNiq35/T+aLZAp0Hiv/+Ap3HlvYg\n73liYiIAU6ZMqbgNvm1H+AkWnqQJRKbsbPj2W3PUP39+8dddrgRuuy2ekSOhRw/f38nrhJN/p5zc\nHHYl72L9kfWFvg3sOL6j1JvDShPsCqZZzWa0rNWSVrVb5T/WbklMZIzn3gH5m3JGMjlj6xF+LKbB\nb3eWdaxr8EVxO3bAxIkwaRIcK2H8k9atzUneoUPN1T4VUXpWOpuSNrHu8DrWH1nP1mNb2XZsG4kp\nieUa4D00OJQWtVrQqnYrYmvE0jSqKU1qNKFpDfNYq1otuYxUlIt1Db5S6jNMHaA2cBgYq7WeXMJ6\n0uBXIJmZ8NVX5tLOpUuLvx4aau7oHTECunSx46j/fGVmZ7I7ZTfbjm1j+7Ht5vG4edyftr/cnxte\nJdzsAKKa0iSySbEdQqPIRgS75IZ4UZx1Db7jEBY2+DZ+hbMx0+TJCfz6azxTp0JaWvHXO3Y0J3kH\nDTJ39vqDv/+d0rPS2XF8R6GdQN7j0VMFxqIsYeyAc3EpF42qNyq0I2gc2ZhGkY1oWL0hDas3pF54\nvXLvFGz8m5JMzth4Hb64wMXFwfDh+R22TZhg+uvPs3o1PPCAGZ938GBz1O+rAVoCJTwknPb129O+\nfvH+KZIzktl+fDs7ju9g0aJFBDUL4vcTv7PnxB72pOwh/Uz6WT87V+eyN3Uve1P3lrqOS7mIDo/2\n7ABKm+qE1ZG+iCo5OcIXXqW16ap5wgSYPh1OFx8RkauuMkf9t98OVav6P6MttNYkn042O4CUPZ4d\nQcEdwtm6ly6rYFcwDSIaFNoJRIdHUy+8HvXC61E3vK7neY2qNeTcguWkpCOscvw4TJ1qrvDZurX4\n63XqwL33msHYmzXzf76K4HT2afae2FtoZ7D3xF4OnjzIgbQDHEg7QNKpJK9vt4qrSqEdQL3wetQN\nK31exjLwP2nwvcjGml1FzaQ1LFpkGv5Zs8ylnkX16WPq/FdcYbpvdp1H5cHGfyfwXa6snCwOnTzk\n2QEUnfJ2Dsczjhd/cznOK5SkWnA1aofVpna12p7HWtVqFZovtDysNjVDa5bYx5GNvz8bM0kNX1hJ\nKdMvz3XXwcGD5rLOiRPh99/z15k3z0wAUVHm6p6uXfMf60q3N6UKCQqhSY0mNKnR5KzrZZzJKLZj\nWPnzSqq1rMaRU0c4km6mpPQk0rJKOAN/ts/OzmBf6j72pe5z/B6FIio0qtiO4NS2UySQQFRoFDVD\na5rHajULzUeEREjJqRzkCF8ERE4OzJljjvrnzjXfAs4mLq7wDqBjx8B041xZZJzJIOlUUqGdQN7z\nojuHI+lHyMzJ9Gu+IBVEVGhUiTuDgjuJyKqRRFaNpEbVGp7neVOVoCp+zextUtIRFdLu3fDZZ7Bs\nGfzyS8k3dRUVHGwGbcnbAXTtarpzPp9SkCgfrTVpWWkczzjOsVPHOJZxjGOnjpl59/NjGcWXl6Wj\nO18IDQ4tdWdQdEdRvWp1qodUL/QYERJB9RDzGIjut6XB9yIba3aVIZPWsGsXrFhhGv9ffjGXdmY6\nOICsUQM6d4batRNo3Trec8OXUpz1+dlea9oUbrwRwsLO/2erDL+/ssjOzSY5I5ljGccK7SxW/ryS\n2pfUJuV0CimnU0g+nWweM5I986fOnDr3BrzpHOc6wqqEFdsReHYOBZ5HhERQvWp1+rboS2xU7HlF\nkhq+qPCUgubNzTRokFmWlQXr1uXvAH75BbZtK/7eEyfgxx+9nykyEv74R3NFUdeuF8adwzYIdgVT\nN7xusXEJYlNiz7kTysrJ4sTpEyXuDArOp2alkpppphOnT3iep2amkqNzvPaznDpzilNnTjm+jPab\nO7857wa/POQIX1RIycnmev+CO4GjR8/9vvN10UXmRrPBg6FBA99vT/iG1pqM7IwSdwSpmamcyCy+\nLC0rjbTMNNKy0jiZdbLQ87JaOGQhPeJ6nNfPICUdUWlpDYmJ+Uf/eX35a51/QjjvedH50l7LyjIn\nlLdvL769oCDo1880/jfe6P8RwYQ9cnUu6VnpxXYEBR9PZp0stOzpK5+mea3m57Xd8jT4aK0DPpkY\ndlm0aFGgIxQjmZzxZqbcXK2XLtX63nu1jogouFvIn+rU0frRR7Veu9Z/ubxFMjljYyZ3u1mmtlau\naRDiLJSCK6+Ejz4y9xB8/DFce23hdY4ehTffNFcNXX45vPuuuctYCNtISUeIcti50zT+U6bA3hL6\nNQsJgVtvNSWf66+3bzhIUfFJDV8IP8vJgQULzCDws2aVfPloTAwMGWK+GbRoAU2amPsIAik9Hfbt\ng3r1Ku5gNZWdNPheJNdMOyOZ8iUnmy6iJ0+GVatKWiMBiCc42Fzfn3f5acGpWTMI90I/ZKdPm5PY\nedPu3YUfkzz9rSVwySXxdO9uSldXXml2SoG89FT+ppyR6/CFCKCaNeEvfzHT+vWm4f/Pfwo2rkZ2\ntikJ7dxZ8ufUr28a/xYtiu8Qatc2jXFWlumLqKTGfPduOHTIee5Nm8z04Ydmvm5d6N4dz07gssvM\nKGai4pMjfCF8KCvL9Bk0e7a5vHPnTnPyt7wiI6F6dThw4Nz9D51NlSrQsCHs319y76UFhYSYRv/K\nK/N3BNHR5d+28A4p6QhRAZw6ZbqPyDvKLzglJp67AXYiKAgaNzadzsXGFn9s0MCsc+qUuYFt2TL4\n+WfzmJx87s9v0SL/G0D37nDJJdKPkb9Jg+9FNtbsJJMzNmYCZ7mys81VPyXtDHbuhJPumzqVMieD\nCzbiBZ/HxDg7MVw0U26uGazm55/zdwAldWNRVI0apiO7vKlz5/LfiWzj78/GTFLDF6KCCw42jXZc\nnLmcsyCtzfmAkydNg+6Lu3tdLrj4YjP96U9mWVKSafjzvgWsWlX8aqQTJ+CHH8yUJyam8E7gsstM\nScoXzpwx5y62bzc7qO3bzTmO6tXNlUh165rHvClvPjKycvWNJEf4QogyycyE337L/xbw88/FT0yX\nRCmzI+ncOX8ncOmlzndcOTmmEc9r0POmbdtMKSynHH2hhYQU3wmUNN+okfnGYtPOQUo6Qgi/09oc\nXa9cabq1XrECfv0VMjLO/d6QEDOYTV4ZqEsX0xV10QZ9+3Zz3iMry/c/T2nq1IFOnczUsaN5bNYs\ncOcupMH3IhtrdpLJGRszgZ25fJUpOxs2bszfAaxcaS5VzevU7hypgLJnatzYjIecN8XFmZPSSUlw\n5Ej+VHA+Pd3pp5ecKTLSNP55O4BOnaB1a//cWCc1fCGEFfJGJGvfHu6/3yxLTzeD2RTcCezaVbbP\nrV+/cKPeqpV5bN68fIPUnGuHkDdt3WrWLSo1FRYvNlOeatXMz11wJ9CmDVStWvZ83iZH+EKIgDl6\ntHApaNUqU4svqVFv0cJ3J33PJTfX7Jx++83stH77zUxOx2CoUsU0+nk7gD59zM9zPqSkI4QQfqK1\n6Y+o6E5g//5zv3fixPxvPuVVngZfbpUoRUJCQqAjFCOZnLExE9iZSzI5U1Impcx5g5tvhhdfNHdT\n79sHhw/D99/Dyy/DgAHmxG5RnTr5PHKJpIYvhBBeVK+eKdn06ZO/LCUF1qzJ/zbQtm1gsklJRwgh\nKiAp6QghhCiVNPilqCh1xECTTM7ZmEsyOWNjpvKQBl8IISoJqeELIUQFJDV8IYQQpfJ5g6+U6quU\n2qKU2qaUetrX2/MWG2t2kskZGzOBnbkkkzM2ZioPnzb4SikX8A7QB2gDDFJKXeTLbXrLmjVrAh2h\nGMnkjI2ZwM5ckskZGzOVh6+P8LsA27XWe7TWZ4DpwM0+3qZXpKSkBDpCMZLJGRszgZ25JJMzNmYq\nD183+I2AvQXm97mXCSGE8DM5aVuKxMTEQEcoRjI5Y2MmsDOXZHLGxkzl4dPLMpVSVwAvaq37uudH\nA1pr/a8i68k1mUIIUUZWdY+slAoCtgI9gYPACmCQ1nqzzzYqhBCiRD7tLVNrnaOUGgXMx5SPPpLG\nXgghAsOKO22FEEL4XkBP2tp2U5ZSKkYptVAptVEptV4p9XCgM+VRSrmUUr8ppWYHOksepVQNpdQM\npdRm979ZVwsyPaaU2qCUWqeU+lQpFRKADB8ppQ4rpdYVWFZTKTVfKbVVKTVPKVXDklyvun9/a5RS\nXyml/DqIYEmZCrz2hFIqVylVy4ZMSqmH3P9W65VSrwQ6k1KqvVJquVJqtVJqhVLq8nN9TsAafEtv\nysoGHtdatwG6AQ9akCnPI8CmQIco4i1gjtb6YqA9ENBynVKqIfAQ0ElrfSmmZHlnAKJMxvxdFzQa\n+FFr3RpYCDzj91Ql55oPtNFadwC24/9cJWVCKRUD9AL2+DkPlJBJKRUP3AS001q3A/4d6EzAq8BY\nrXVHYCzw2rk+JJBH+NbdlKW1PqS1XuN+fhLTgAX8vgH3H39/4MNAZ8njPhK8Wms9GUBrna21Tg1w\nLIAgIFwpFQyEAQf8HUBrvRRILrL4ZmCK+/kU4Ba/hqLkXFrrH7XWue7Z/wExgc7k9gbwlD+z5Ckl\n00jgFa11tnsdh8OX+zRTLpD3TTEKOOdouoFs8K2+KUspFQt0AH4JbBIg/4/fphMuccBRpdRkd6lp\nolKqWiADaa0PAK8Dv2P++FO01j8GMlMB9bTWh8EcWAD1ApynJPcCcwMdQin1B2Cv1np9oLMU0Aq4\nRin1P6XUIiflEz94DPi3Uup3zNH+Ob+dyY1XJVBKRQBfAo+4j/QDmeUG4LD7m4dyTzYIBjoB72qt\nOwGnMGWLgFFKRWGOpJsCDYEIpdRdgcx0FjbtvFFKPQec0Vp/FuAc1YBnMSUKz+IAxSkoGKiptb4C\n+CvwRYDzgPnW8YjWugmm8Z90rjcEssHfDzQpMB+Dg68kvuYuBXwJfKK1/ibQeYArgT8opXYB04Ae\nSqmpAc4E5hvZXq31Kvf8l5gdQCBdD+zSWh/XWucAM4HuAc6U57BSKhpAKVUfOBLgPB5KqWGYkqEN\nO8fmQCywVim1G9Mu/KqUCvQ3or2Yvye01iuBXKVU7cBGYqjW+mt3pi8xZfKzCmSDvxJooZRq6r6S\n4k7AhitQJgGbtNZvBToIgNb6Wa11E611M8y/0UKt9RALch0G9iqlWrkX9STwJ5V/B65QSoUqpZQ7\nU6BOJBf9NjYbGOZ+PhQI1MFEoVxKqb6YcuEftNaZgc6ktd6gta6vtW6mtY7DHFh01Fr7ewdZ9Pf3\nNXAdgPtvvorW+liAM+1XSl3rztQT2HbOT9BaB2wC+mLuxN0OjA5kFneeK4EcYA2wGvgN6BvoXAXy\nXQvMDnSOAnnaY3bcazBHPzUsyDQW08ivw5wcrRKADJ9hThZnYnZCw4GawI/uv/f5QJQlubZjroT5\nzT39v0BnKvL6LqBWoDNhSjqfAOuBVcC1FmTq7s6yGliO2TGe9XPkxishhKgk5KStEEJUEtLgCyFE\nJSENvhBCVBLS4AshRCUhDb4QQlQS0uALIUQlIQ2+EOdBKXWtUurbQOcQwglp8IU4f3Izi6gQpMEX\nlYJS6m6l1C/unj0nuAeUSVNKjXMPmPJDXt8oSqkO7oEl8gYFqeFe3ty93hql1CqlVJz746sXGAjm\nk4D9kEKcgzT44oLnHsTmj0B3bXr2zAXuxvSXv0Jr3RZYQn4PjVOAp7QZFGRDgeWfAm+7l3cHDrqX\ndwAeBi4BmiulbOmwTYhCfDqIuRCW6InpyXOlu1O1UOAwpuHP6+b2P0DeEH81tBlwAkzj/4W7y+xG\nWuvZAFrrLADzcazQWh90z6/B9Pa4zA8/lxBlIg2+qAwUMEVr/VyhhUqNKbKeLrB+WRTsZTIH+X8l\nLCUlHVEZLAAGKKXqgmdA8SaY4RAHuNe5G1iqzTCNx5VSV7qXDwYWazMQzl6l1M3uzwgJ9AhfQpSV\nHImIC57WerNS6nlgvlLKBWQBo4B0oIv7SP8wps4Ppr/6990N+i5MV7RgGv+JSqn/c3/GHSVtznc/\niRDnR7pHFpWWUipNa1090DmE8Bcp6YjKTI52RKUiR/hCCFFJyBG+EEJUEtLgCyFEJSENvhBCVBLS\n4AshRCUhDb4QQlQS0uALIUQl8f8B9XBZG/NbaxkAAAAASUVORK5CYII=\n",
641 |       "text/plain": [
642 |        "<matplotlib.figure.Figure at 0x12014d3d0>"
643 |       ]
644 |      },
645 |      "metadata": {},
646 |      "output_type": "display_data"
647 |     }
648 |    ],
649 |    "source": [
650 |     "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
651 |     "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
652 |     "pyplot.grid()\n",
653 |     "pyplot.legend()\n",
654 |     "pyplot.xlabel(\"epoch\")\n",
655 |     "pyplot.ylabel(\"loss\")\n",
656 |     "pyplot.show()"
657 |    ]
658 |   },
659 |   {
660 |    "cell_type": "markdown",
661 |    "metadata": {},
662 |    "source": [
663 |     "# Mini Batch with Relu and 256 hidden layers"
664 |    ]
665 |   },
666 |   {
667 |    "cell_type": "markdown",
668 |    "metadata": {},
669 |    "source": [
670 |     "#### Increasing network complexity and introducing non-linearities should improve performance "
671 |    ]
672 |   },
673 |   {
674 |    "cell_type": "code",
675 |    "execution_count": 18,
676 |    "metadata": {
677 |     "collapsed": false
678 |    },
679 |    "outputs": [],
680 |    "source": [
681 |     "batch_size = 128\n",
682 |     "train_loss = []\n",
683 |     "test_loss = []\n",
684 |     "\n",
685 |     "graph = tf.Graph()\n",
686 |     "with graph.as_default():\n",
687 |     "    X_train_tf = tf.placeholder(tf.float32,\n",
688 |     "                                    shape=(batch_size, X.shape[1]))\n",
689 |     "    y_train_tf = tf.placeholder(tf.float32, shape=(batch_size, nb_classes))\n",
690 |     "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
691 |     "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
692 |     "    \n",
693 |     "    weights1 = tf.Variable(\n",
694 |     "        tf.truncated_normal([X.shape[1], 256]))\n",
695 |     "    biases1 = tf.Variable(tf.zeros([256]))\n",
696 |     "    \n",
697 |     "    weights2 = tf.Variable(\n",
698 |     "        tf.truncated_normal([256, nb_classes]))\n",
699 |     "    biases2 = tf.Variable(tf.zeros([nb_classes]))\n",
700 |     "\n",
701 |     "    h1 = tf.nn.relu(tf.matmul(X_train_tf, weights1) + biases1)\n",
702 |     "    logits = tf.matmul(h1, weights2) + biases2\n",
703 |     "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
704 |     "    optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)\n",
705 |     "    \n",
706 |     "    train_prediction = tf.nn.softmax(logits)\n",
707 |     "    test_prediction_h1 = tf.nn.relu(tf.matmul(X_test_tf, weights1) + biases1)\n",
708 |     "    test_prediction = tf.nn.softmax(tf.matmul(test_prediction_h1, weights2) + biases2)"
709 |    ]
710 |   },
711 |   {
712 |    "cell_type": "code",
713 |    "execution_count": 19,
714 |    "metadata": {
715 |     "collapsed": false
716 |    },
717 |    "outputs": [
718 |     {
719 |      "name": "stdout",
720 |      "output_type": "stream",
721 |      "text": [
722 |       "Train Loss at step 0: 9.716219\n",
723 |       "Valid Loss: 8.543947\n",
724 |       "Train Loss at step 1000: 1.845988\n",
725 |       "Valid Loss: 2.275506\n",
726 |       "Train Loss at step 2000: 0.903404\n",
727 |       "Valid Loss: 1.362468\n",
728 |       "Train Loss at step 3000: 0.411469\n",
729 |       "Valid Loss: 1.036987\n",
730 |       "Train Loss at step 4000: 0.260850\n",
731 |       "Valid Loss: 0.879940\n",
732 |       "Train Loss at step 5000: 0.185567\n",
733 |       "Valid Loss: 0.794446\n",
734 |       "Train Loss at step 6000: 0.166375\n",
735 |       "Valid Loss: 0.734401\n",
736 |       "Train Loss at step 7000: 0.136283\n",
737 |       "Valid Loss: 0.694401\n",
738 |       "Train Loss at step 8000: 0.100913\n",
739 |       "Valid Loss: 0.662252\n",
740 |       "Train Loss at step 9000: 0.101982\n",
741 |       "Valid Loss: 0.641285\n",
742 |       "Train Loss at step 10000: 0.071903\n",
743 |       "Valid Loss: 0.620439\n",
744 |       "Train Loss at step 11000: 0.074838\n",
745 |       "Valid Loss: 0.603872\n",
746 |       "Train Loss at step 12000: 0.061306\n",
747 |       "Valid Loss: 0.589687\n",
748 |       "Train Loss at step 13000: 0.064904\n",
749 |       "Valid Loss: 0.579660\n",
750 |       "Train Loss at step 14000: 0.047869\n",
751 |       "Valid Loss: 0.569570\n",
752 |       "Train Loss at step 15000: 0.071436\n",
753 |       "Valid Loss: 0.559656\n",
754 |       "Train Loss at step 16000: 0.040468\n",
755 |       "Valid Loss: 0.553105\n",
756 |       "Train Loss at step 17000: 0.067016\n",
757 |       "Valid Loss: 0.547071\n",
758 |       "Train Loss at step 18000: 0.032301\n",
759 |       "Valid Loss: 0.541087\n",
760 |       "Train Loss at step 19000: 0.036903\n",
761 |       "Valid Loss: 0.535128\n"
762 |      ]
763 |     }
764 |    ],
765 |    "source": [
766 |     "num_steps = 20000\n",
767 |     "\n",
768 |     "with tf.Session(graph=graph) as session:\n",
769 |     "    tf.global_variables_initializer().run()\n",
770 |     "    for step in range(num_steps):\n",
771 |     "        # Pick an offset within the training data, which has been randomized.\n",
772 |     "        # Note: we could use better randomization across epochs.\n",
773 |     "        offset = (step * batch_size) % (len(y_train) - batch_size)\n",
774 |     "        # Generate a minibatch.\n",
775 |     "        batch_data = np.array(X_train)[offset:(offset + batch_size), :]\n",
776 |     "        batch_labels = y_train[offset:(offset + batch_size)]\n",
777 |     "        batch_labels = (np.arange(nb_classes) == batch_labels[:,None]).astype(np.float32)\n",
778 |     "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
779 |     "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
780 |     "        # and the value is the numpy array to feed to it.\n",
781 |     "        feed_dict = {X_train_tf : batch_data, y_train_tf : batch_labels}\n",
782 |     "        _, l, predictions = session.run(\n",
783 |     "              [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
784 |     "        if (step % 1000 == 0):\n",
785 |     "            train_loss.append(l)\n",
786 |     "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
787 |     "            print('Train Loss at step %d: %f' % (step, l))\n",
788 |     "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
789 |    ]
790 |   },
791 |   {
792 |    "cell_type": "code",
793 |    "execution_count": 20,
794 |    "metadata": {
795 |     "collapsed": false
796 |    },
797 |    "outputs": [
798 |     {
799 |      "data": {
800 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VNX5+PHPM9kXSCBAgGASNkEWDahQUCEuKGit2p9Q\nUbFgi0ulrrXaBZfWfmu1pRbbolbZ3Ev9un61AmoEpCpWURBQWQLIEtaQjezn98edzJJMYJLcyZ2Z\nPO/X677mnnvv3HlyCPPknHPvuWKMQSmllGrgcjoApZRS4UUTg1JKKT+aGJRSSvnRxKCUUsqPJgal\nlFJ+NDEopZTyE9LEICIJIvKRiHwmIutE5N5mjpsrIt+IyFoRyQtlTEoppY4tNpQnN8ZUicjZxpgK\nEYkBPhCRt4wxHzccIyKTgP7GmIEiMhp4DPhOKONSSinVvJB3JRljKtyrCViJqPEddZcAi93HfgSk\niUhmqONSSikVWMgTg4i4ROQzYC+wzBizptEhWcBOn/Iu9zallFIOaI8WQ70xZgTQBxgtIkNC/ZlK\nKaVaL6RjDL6MMSUi8h4wEdjgs2sXcIJPuY97mx8R0UmdlFKqFYwx0pLjQ31VUjcRSXOvJwETgE2N\nDnsNuMZ9zHeAYmNMUaDzGWNatMSe8Re4D7gPbnh1VovfH83Lvffe63gM0bRofWpdhuvSGqFuMfQC\nFomICysJvWiMeVNErgeMMeYJd/lCEdkMlAMz7PrwOJKoda+XVR2167RRobCw0OkQoorWp320Lp0X\n6stV1wEjA2x/vFF5Vig+P06SaEgH5dWaGJRSKhhRfedzvCR51is0MfiZPn260yFEFa1P+2hdOi+6\nE4Mr2bNeUaOJwVd+fr7TIUQVrU/7aF06L6oTQ4LLt8VQcYwjO56CggKnQ4gqHbU+c3NzERFdwmDJ\nzc217d+13S5XdUJCjDcxHK3VFoNSdtu+fXurr3xR9hJp0RWpxxTVLYZETQzN0ua6vbQ+VTSJ6sSQ\nFOtNDFV1mhiUUioYUZ4YvIPPlZoY/HTUPvFQ0fpU0SS6E0Ocb4tBB5+VUi1z44038rvf/a5V7z37\n7LOZP3++zRG1j6gefE6O9yaGaqMtBl/aJ24vrc/w07dvX5566inOOeecVp9j3rx5NkYUOaK6xZAc\nn+hZrzFV1Jt6B6NRSoWTuro6p0MIW1GdGBITBGq8yaGyttLBaMKL9onbS+szvFxzzTXs2LGDiy++\nmM6dO/PHP/6R7du343K5mD9/Pjk5OZx77rkATJkyhV69etGlSxfy8/PZsME7+fOMGTO45557AHj/\n/fc54YQTmDNnDpmZmWRlZbFw4cKg4jHG8MADD5Cbm0vPnj2ZPn06JSUlAFRVVTFt2jS6detGly5d\nGD16NPv37wdg4cKF9O/fn86dO9O/f3+ef/55G2upedGdGBKBWp9LVvXuZ6XajYi9S0ssXryY7Oxs\n3njjDUpKSvjZz37m2bdixQo2bdrE22+/DcCFF17Ili1b2LdvHyNHjuSqq65q9rx79+6ltLSU3bt3\n8+STT3LTTTdx5MiR48azYMECFi9ezPvvv8/WrVspLS3lpz/9KQCLFi2ipKSEXbt2cejQIR577DGS\nkpKoqKjglltu4e2336akpITVq1eTl5fXsopopahODAkJQI3vtBg6AN1A+8TtpfUZnhrffCci3H//\n/SQlJZGQkABYczMlJycTFxfHPffcw+eff05paWnA88XHxzN79mxiYmKYNGkSqampfPXVV8eN47nn\nnuP2228nJyeH5ORkfv/73/PCCy9QX19PXFwcBw8e5Ouvv0ZEGDFiBKmpqQDExMSwbt06KisryczM\n5KSTTmpjjQSnAyQGvclNKeXVp08fz3p9fT133303AwYMID09nb59+yIiHDhwIOB7MzIycLm8X5vJ\nycmUlZUd9zN3795NTk6Op5yTk0NNTQ1FRUVMmzaNCy64gCuuuII+ffpw9913U1dXR3JyMi+++CLz\n5s2jV69eXHzxxUElITtEdWLQrqTmaZ+4vbQ+mzLG3qWlmpsiwnf7c889x+uvv867775LcXExhYWF\nbXrATXN69+7N9u3bPeXt27cTFxdHZmYmsbGxzJ49my+//JLVq1fz+uuvs3jxYgAmTJjA0qVL2bt3\nL4MGDWLmzJm2xtWcqE4M2mJQquPq2bMnW7du9dvW+Au/tLSUhIQEunTpQnl5Ob/4xS9snXOowdSp\nU/nzn/9MYWEhZWVl/OpXv+KKK67A5XJRUFDA+vXrqa+vJzU1lbi4OFwuF/v27eO1116joqKCuLg4\nUlNTiYmJsT22QKI/MWiLISDtE7eX1mf4ufvuu/ntb39L165dmTNnDtC0FXHNNdeQnZ1NVlYWw4YN\nY+zYsS36jGMlEd991157LdOmTWPcuHH079+f5ORk5s6dC1gD2pdffjlpaWkMHTqUs88+m2nTplFf\nX8+cOXPIysqiW7durFixot3uq5BImRlRRExLY33ySZhZcCEMfAuAN6a+wUUnXhSK8JTqkEREZ1cN\nE839W7i3t6gZFP0tBu1KCkj7xO2l9amiSfQnBu1KUkqpFonqxJCYiLYYmqF94vbS+lTRJKoTg7YY\nlFKq5aI/MeidzwFpn7i9tD5VNInqxKBdSUop1XJRnRi0K6l52iduL61PFU2iPzFoi0EppVok+hOD\nthgC0j5xe2l9Ro+G5y40GDZsGCtWrAjq2MZcLleTaTkiQVQ/2rPxGENFrQ4+K6WOz3c6i/Xr1wd9\nbEv2hbOQthhEpI+IvCsiX4rIOhG5OcAx40WkWEQ+dS+/tuvzG1+VpC0GL+0Tt5fWpwokUqcLCXVX\nUi1wuzFmKDAGuElEBgc4boUxZqR7ecCuD2/SlaRjDEp1CA899BCTJ0/223bLLbdw6623AtYjM4cM\nGULnzp0ZMGAATzzxRLPn6tu3L++++y4AlZWVTJ8+na5duzJs2DDWrFkTdEwlJSVcc8019OjRg759\n+/K73/3Os2/Lli3k5+eTnp5Ojx49mDp1qmffbbfdRmZmJmlpaZxyyil+jx4NlZB2JRlj9gJ73etl\nIrIRyAI2NTo0JO2tJperaovBo6CgQP/KtZHWZ1Nyv73/rc29wf/1fcUVV/Cb3/yG8vJyUlJSqK+v\nZ8mSJbz66qsAZGZm8uabb5Kbm8vKlSuZOHEio0aNOu6jM++77z62bdvGtm3bKCsrY+LEiUHHNGvW\nLEpLSyksLGT//v2cf/759O7dmxkzZjB79mwuuOACCgoKqK6u5pNPPgFg6dKlrFq1is2bN9OpUye+\n+uor0tPTg/7M1mq3wWcRyQXygI8C7B4jImtF5P9EZIhdnxkbiw4+K9UBZWdnM3LkSF5++WUA3nnn\nHVJSUjj99NMBmDRpErm5uQCcddZZnH/++axcufK4512yZAm//vWvSUtLIysri5tvbtI7HlB9fT0v\nvvgiDz74IMnJyeTk5HDHHXfw9NNPAxAXF8f27dvZtWsX8fHxnum/4+LiKC0tZcOGDRhjGDRoEJmZ\nmS2tjhZrl8QgIqnAv4BbjDGNn4P3XyDbGJMH/BV4xb7PhXiXNzGUV+vgcwP969ZeWp/hZ+rUqTz/\n/PMAPP/881x55ZWefW+99RZjxowhIyODLl268NZbbzX7OE9fu3fv9ns0qO/jOo/lwIED1NbWkp2d\n7ffeXbt2AVbXV319PaNGjWL48OEsWLAAgLPPPptZs2Zx0003kZmZyQ033BDUo0TbKuRXJYlILFZS\neNoY82rj/b6Jwhjzloj8XUS6GmMONT52+vTpniyfnp5OXl6e5z9kw+WCjcsJrhOoBtgGh/Z5T9nc\n8VrWspaDLx9LS7p+QmHy5Mn87Gc/Y9euXbz88st8+OGHAFRXV3P55ZfzzDPPcMkll+ByubjsssuC\nGiju1asXO3fu5KSTTgLwe1znsXTr1s3TKhg8eLDnvVlZWYDVtdUwzvHBBx9w3nnnMX78ePr168es\nWbOYNWsWBw4cYPLkyTz88MPcf//9zX5WQUEBCxcuBPB8X7ZYw/NNQ7UAi4E5x9if6bM+Cihs5jjT\nGt367jLch+E+TI8/9GzVOaLRe++953QIUaWj1mdr/1+2l0mTJpkJEyaYkSNHeraVlpaa2NhYs2LF\nCmOMMW+++aZJTk42s2fPNsYYU1BQYE444QTP8bm5ueadd94xxhhz1113mfz8fHP48GGzc+dOc/LJ\nJ/sd25iImC1bthhjjJk2bZr5/ve/b0pLS01hYaEZPHiwmT9/vjHGmCVLlphvv/3WGGPM+vXrTXJy\nstm2bZtZs2aN+eijj0xNTY0pKyszEydONPfdd1/Az2ru38K9vUXf26G+XPUM4CrgHBH5zH056kQR\nuV5ErnMfdrmIrBeRz4BHgB/YGUNijF6VpFRHdeWVV/LOO+9w1VVXebalpqYyd+5cJk+eTNeuXXnh\nhRe45JJLmj2H770I9957L9nZ2fTt25eJEydyzTXXHPPzfd87d+5ckpOT6devH+PGjePqq69mxowZ\nAKxZs4bRo0fTuXNnLr30UubOnUtubi4lJSXMnDmTrl270rdvX7p168add97Z2uoIWlQ/2hNg4EmV\nbL7CSg5xrniqZ1fZHZpSHZY+2jN86KM9WyAxLgHcdVJTX01dfZ3DESmlVHiL/sSQIDqRXgA6t4+9\ntD5VNIn+xJCI3suglFItEPWJQafeDkyvu7eX1qeKJh0jMWiLQSmlghb1iUEf7xmY9onbS+tTRZOo\nfh4DNG0xVNTotBhK2SUnJydinzkQbYKdniMYHSMx6DMZmtA+cXt11PosLCx0OgQVAtqVpJRSyk/U\nJwYdfA5M+8TtpfVpH61L53WMxKAtBqWUClrUJ4bGN7jp4LOlo/aJh4rWp320Lp0X9YkhIQGo6uQp\nF5UVOReMUkpFgI6RGPZ6n+O6aucq54IJI9qPay+tT/toXTov6hNDYiKwfbynvHrnamrqapwLSCml\nwlzUJ4aEBOBINhzuC1hjDJ/s/sTZoMKA9uPaS+vTPlqXzusYiQGg0NtqeH/7+84Eo5RSEaDjJIbt\nmhh8aT+uvbQ+7aN16byoTwyJie4VnxbDqh2rqK2vdSYgpZQKc1GfGDwthuJcEqtOAKCsuoxP93zq\nXFBhQPtx7aX1aR+tS+d1nMSA0PmwT3dSoXYnKaVUIFGfGDxdSUDKfh1naKD9uPbS+rSP1qXzoj4x\neFsMkLDHmxhW7lhJXX2dAxEppVR4E2OM0zEERURMa2LdsAGGDrXWBw02lPw4iz1lewD473X/ZWSv\nkXaGqZRSYUVEMMa06GlKUd9i8O1Kqq4SxufqOINSSh1L1CcG366kykoYn6PjDKD9uHbT+rSP1qXz\nOlRiqKryTwwrd6yk3tQ7EJVSSoWvqB9jKCuDTu5Zt1NSoLTU0PNPPdlXvg+Az2/4nJMzT7YzVKWU\nChs6xhBA4xaDiPh3J+k4g1JK+QlpYhCRPiLyroh8KSLrROTmZo6bKyLfiMhaEckLdExrxcaCuHNl\nbS3U1ek4A2g/rt20Pu2jdem8ULcYaoHbjTFDgTHATSIy2PcAEZkE9DfGDASuBx6zMwAR/yuTqqrw\nvzJp+/tESneaUkq1h3YdYxCRV4BHjTHv+Gx7DHjPGPOiu7wRyDfGFDV6b6vGGAC6dIHiYmv90CFI\nS6+nx8M9OHj0IADrb1zP0B5DW3VupZQKZ2E9xiAiuUAe8FGjXVnATp/yLvc22zS+ZNUlLsbljPNs\n66jdSUopFUhse3yIiKQC/wJuMcaUtfY806dPJzc3F4D09HTy8vI8MzE29EsGKltdSVa5qsra3/tg\nb9gG9LUSw5DyIc2+PxrLjzzySND1p2Wtz/Ys+44xhEM8kVYuKChg4cKFAJ7vy5YKeVeSiMQCbwBv\nGWP+EmB/466kTcB4O7uSBg2Cr7+21jdtsspr965lxOMjAMhMyWTPHXsQaVFrK6IVFBR4fqlU22l9\n2kfr0l7h2pU0H9gQKCm4vQZcAyAi3wGKGyeFtmrclQQwvMdw0hPTASgqL+Lrg1/b+ZFhT//j2Uvr\n0z5al84L9eWqZwBXAeeIyGci8qmITBSR60XkOgBjzJvANhHZDDwO/MTuOBpflQQQ44rhrOyzPNt1\nnEEppSwhTQzGmA+MMTHGmDxjzAhjzEhjzL+NMY8bY57wOW6WMWaAMeYUY4ztj1ZrfJNbg458P4Nv\nP65qO61P+2hdOi/q73yGwF1JQJOZVvV+BqWU6iCJIVBXEkBezzw6xVsTKe0q3cXWw1vbOTLnaD+u\nvbQ+7aN16bwOkRia60qKdcVyVo6OMyillK8Olxh8u5Kg444zaD+uvbQ+7aN16bwOkRia60oC/8RQ\nUFjQPgEppVQY6xCJobmuJICRvUaSEpcCwI4jOygsLmy/wByk/bj20vq0j9al8zpEYkhJ8a4XNbp1\nLi4mjjOyz/CU9fkMSqmOrkMkhjyfJzx88EHT/R1xnEH7ce2l9WkfrUvndYjEcJb3wiP+8x/rgT2+\nOmJiUEqp5kT9M58b5OTAjh3W+scfw+mne/dV11WT/mA6R2uPArDztp306dynLeEqpVRYCNdJ9MKC\nb6th5Ur/ffEx8Yw5YYynrOMMSqmOTBODW0frTtJ+XHtpfdpH69J5HTIxrFoFjXulOlpiUEqp5nSY\nMQZjoHt3OGg95pkNG+Ckk7z7K2srSX8wnao660aH3bfvplenXm0JWSmlHKdjDMcgAmee6S037k5K\njE1kdJ/RnvKK7SvaKTKllAovHSYxgH930ooA3/sdaXoM7ce1l9anfbQunddhE0OgAej83HzPuo4z\nKKU6qg4zxgBQUwNdukB5uVXevh2ys737K2oqSH8wnZr6GgCKflZEj5QebfpMpZRyko4xHEdcHIzx\n3q7QpNWQHJfMqKxRnrKOMyilOqIOlRighfczRPGNbtqPay+tT/toXTpPE0Mjfs+B1nEGpVQH1KHG\nGAAqKiA93RpvADhwADIyvPvLqstIfzCdOlNn7b/zABnJGQHOpJRS4U/HGIKQnAynnuotr1rlvz81\nPpXTep/mKa/cEaBZoZRSUazDJQbQcQbQfly7aX3aR+vSeR0+MQS80U3HGZRSHVhQYwwicguwACgF\nngRGAHcbY5aGNjy/GGwZYwA4dMg7rhATA8XFkJrq3V9SVUKXP3Sh3tQjCIfuOkR6Yrotn62UUu0p\nlGMM1xpjSoDzgS7ANODBFsYXNrp2heHDrfW6OvjwQ//9nRM6M6LnCAAMhlU7Gg1EKKVUFAs2MTRk\nmwuBp40xX/psi0gtGWeIxnmTtB/XXlqf9tG6dF6wieG/IrIUKzG8LSKdgPrjvUlEnhKRIhH5opn9\n40WkWEQ+dS+/Dj70ttH7GZRSKrBgxxhcQB6w1RhTLCJdgT7GmIBf+D7vOxMoAxYbY04OsH88cIcx\n5ntBxGDbGAPArl3Qx/1Y56Qka5whPt67//DRw2Q8lIHB4BIXh+86TOeEzrZ9vlJKtYdQjjGMAb5y\nJ4WrgV8DR473JmPMKuDwcQ5zpEsqKwv69rXWjx6FTz/1398lqQun9DwFgHpTzwc7PmjnCJVSyhnB\nJoZ5QIWInALcAWwBFtsUwxgRWSsi/yciQ2w6Z1A68nOgtR/XXlqf9tG6dF5skMfVGmOMiFwC/NUY\n85SI/MiGz/8vkG2MqRCRScArwInNHTx9+nRyc3MBSE9PJy8vj/z8fMD7y9SSco8eAFb55ZcLOP10\n//0ZRd6pMF57+zUmxk5s0+eFU3nt2rVhFU+kl7U+tRwu5YKCAhYuXAjg+b5sqWDHGN4H/g1cC5wF\n7AM+N8YMD+K9OcDrgcYYAhy7DTjVGHMowD5bxxgAvvoKBg+21rt0seZNcvm0oQ5UHKD7w90BiHXF\nUnxXMSnxKbbGoJRSoRTKMYYfAFVY9zPsBfoADwcbF82MI4hIps/6KKxE1SQphMqJJ+JuNcDhw/Dl\nl/77uyV3Y1iPYQDU1teyeufq9gpNKaUcE1RicCeDZ4E0EfkuUGmMOe4Yg4g8B6wGThSRHSIyQ0Su\nF5Hr3IdcLiLrReQz4BGsBNRuRDruOEND01PZQ+vTPlqXzgsqMYjIFOBjYDIwBfhIRC4/3vuMMVca\nY3obYxKMMdnGmAXGmMeNMU+49//NGDPMGDPCGDPWGPNRW36Y1uioiUEppZoT7BjD58AEY8w+d7k7\nsNwYc0qI4/ONwfYxBrAuU22YhjsrC3butFoSDYrKiuj5p54AxMfEU3xXMUlxSbbHoZRSoRDKMQZX\nQ1JwO9iC94a1U06BTp2s9V27oLDQf39maiaDu1kj1NV11Xz4baOJlZRSKsoE++X+bxF5W0Smi8h0\n4P+AN0MXVvuJiYGxY73ljjJvkvbj2kvr0z5al84LdvD5TuAJ4GT38oQx5q5QBtaedJxBKaW8Otwz\nnwNZsQLGu7/7TzzRur/B1+7S3WTNyQIgISaBQ3cdIjkuOSSxKKWUnWwfYxCRUhEpCbCUikhJ28IN\nH6NGeSfQ+/prKCry39+7U29OzLBuyK6qq2L2u7PbOUKllGo/x0wMxphOxpjOAZZOxpiomWo0MdFK\nDg1WBXguzx1j7vCs//nDP0f8WIP249pL69M+WpfOi4ori+xwvHGGmSNnMnHARMB6qtsPX/khRyqP\nO8GsUkpFHB1jcHvrLbjwQmt95Ej473+bHrO7dDfD5w3n0FFr1o4fnvJDFl66MGQxKaVUW4XyPoao\nN3as98a2tWuhJMAISu9OvZl30TxPedHni3h548vtFKFSSrUPTQxuaWnWzW4A9fXwn/8EPm7K0Clc\nOfxKT/m6N65jb9nedojQXtqPay+tT/toXTpPE4OP440zNPjrpL+S1cm6fPVAxQFmvj6TSOmSU0qp\n49ExBh9LlsCUKdb6uHHw/jHuZVu+dTkTnp7gKf/j4n/w45E/Dml8SinVUq0ZY9DE4GPvXujVy1pP\nSIAjR6zX5tz81s08+vGjAKTGp/L5DZ/Tr0u/kMaolFItoYPPbdSzJwwcaK1XVcGaNcc+/sHzHmRQ\nxiAAyqrLuObla6irrwtxlPbQflx7aX3aR+vSeZoYGgl2nAEgOS6Zpy97mhiJAeCDnR/wx9V/DGF0\nSikVetqV1MjChTBjhrU+aRK8GcQcsvcX3M99798HQJwrjk+u+4STM4/7iGullAo5HWOwwZYtMGCA\ntd65Mxw6ZE3NfSw1dTWcMf8M1uy2+p6G9xjOmplrSIg9xgCFUkq1Ax1jsEG/ft4B6JISWLfu+O+J\ni4nj6cueJinWerLbun3ruOe9e0IYZdtpP669tD7to3XpPE0MjYi0bJyhwaBug3howkOe8sOrH2bl\n9iDfrJRSYUS7kgL461/hpz+11idPhn/+M7j31Zt6Jj4zkWVblwHQN70vn9/wOZ0SOoUoUqWUOjbt\nSrJJ4xZDsPnIJS4WXLKA9MR0ALYVb+O2t28LQYRKKRU6mhgCGDYM0q3vdvbuhc2bg39vVucs/n7h\n3z3lpz57ite+es3mCNtO+3HtpfVpH61L52liCCAmBs44w1sOdpyhwdThU/nB0B94yjNfn8n+8v02\nRaeUUqGliaEZrRmA9vX3i/5O7069AdhXvo/r3rgurCbay8/PdzqEqKL1aR+tS+dpYmhGWxND16Su\nzP/efE/5lU2vsOjzRTZEppRSoaWJoRmnnWY9Cxqsm9727Gn5OS4YcAE/Oe0nnvLNb91MYXGhPQG2\nkfbj2kvr0z5al87TxNCM+HgYPdpbbk2rAeChCQ8xsKs1M19pdSnTX5lOvam3IUKllAqNkCYGEXlK\nRIpE5ItjHDNXRL4RkbUikhfKeFqqrd1JACnxKX4T7b2//X3ufe9ex5OD9uPaS+vTPlqXzgt1i2EB\ncEFzO0VkEtDfGDMQuB54LMTxtIgdiQFgdJ/R/PKsX3rKD6x8gInPTGRXya42RKeUUqER0sRgjFkF\nHD7GIZcAi93HfgSkiUhmKGNqiTFjwOWuoS++gOLi1p9r9rjZjMsZ5ykv27qM4fOG888vg7yt2mba\nj2svrU/7aF06z+kxhixgp095l3tbWOjUCUaOtNaNgQ8+aP254mLiWHr1Un4+9ucI1t3physP84N/\n/YBpL0/jSOURGyJWSqm2i3U6gJaYPn06ubm5AKSnp5OXl+fpj2z4K8Pu8lln5fPJJwAFPPssXHRR\n2873hwl/4KITL2LKw1MoKiuCvvDMF8+w9J2l/OLMX3DrFbeG9Ofx7b8tKCgIef11lHLDtnCJJ5LL\n+fn5YRVPpJULCgpYuHAhgOf7sqVCPomeiOQArxtjmjy5RkQeA94zxrzoLm8CxhtjigIc226T6Pl6\n+WX4/vet9bFj29Zq8HWk8gg3//tmFn++2LNNEO4YcwcPnPOAPstBKWWLcJ1ET9xLIK8B1wCIyHeA\n4kBJwUlnnuldX7MGjh6157xpiWksunQRSyYvoWtSVwAMhj/+54+c/o/TWVcUxIMg2qDhLwxlD61P\n+2hdOi/Ul6s+B6wGThSRHSIyQ0SuF5HrAIwxbwLbRGQz8Djwk2OczhHdu8PgwdZ6TQ18/LG95798\nyOWsu3EdF/T3Xry1bt86TvvHafxp9Z8cv6xVKdXx6PMYgnDddfCPf1jrv/0t/PrX9n+GMYa/rfkb\ndy67k8raSs/2/Nx8Fl26iOy0bPs/VCkV9cK1Kyni2XU/w7GICLNGzeKz6z/j1F6nerYXFBZw8ryT\nefaLZ8NqEj6lVPTSxBAE38SwejXU1obuswZ3G8zqH63mV2f9CpdY/zxHqo5w9ctXM/WlqRw6esiW\nz9F+XHtpfdpH69J5mhiCkJMDJ5xgrZeVwdtvh/bz4mPieeCcB1g5YyX9uvTzbH/xyxc5ed7JLNuy\nLLQBKKU6NB1jCNIdd8CcOdZ6fj689177fG5pVSm3vX0bT332lN/2M044gxl5M5gydIo+U1op1azW\njDFoYgjSjh3Qrx/U1VnlNWusqbnbyyubXmHm6zM5UHHAb3tyXDKTh0xmRt4MxuWMQ6RF//5KqSin\ng88hlJ0NV1zhLT/8cPt+/qWDL2X9jeu5YtgVxLq8N6xX1FSw6PNF5C/KZ+CjA3lgxQPsPLLzGGey\naD+uvbSSRn9iAAAVR0lEQVQ+7aN16TxNDC1w553e9X/9C7Zubd/Pz0zN5Pn/9zy7bt/Fn87/E0O7\nD/Xbv+XwFma/N5ucR3K44JkLeGH9C36XviqlVDC0K6mFzj8flrnHfmfNgkcfdS4WYwyf7P6EBWsX\n8Pz65ymubDr9a3piOlOHTeXaEddyaq9TtatJqQ5GxxjawbJlVnIASEqyxh66dXM2JoDK2kpe2fQK\n8z+bz/KtyzE0rathPYYxI28GV598NT1SejgQpVKqvWliaAfGWFNxr11rle+/H+65x9mYGttxZAeL\nP1/MgrUL2Hq4aX9XrCuWU6tOZep3pzKh/wRO6naStiTayHdmVdU2Wpf20sTQTp59Fq6+2lrv3h22\nb7daD+Gm3tSzcvtKFqxdwJINS6ioqfDu3Ab0tVZ7pfbivH7nMaHfBM7tdy69O/V2JN5Ipl9m9tG6\ntJcmhnZSUwP9+8NO98U/8+bBDTc4G9PxlFaVsmTDEuZ/Np8Pdh577vAh3YdwXt/zmNB/AuNzxut9\nEkpFME0M7eiRR+C226z1AQNg0yaIiXE2pmBtO7yNpVuWsmzrMt7d9i6HK5t/+mqsK5bRWaOZ0G8C\n5/U7j1FZo4iLiWvHaJVSbaGJoR2VlVnTZDQ8B/qll7wP9IkEDc31uvo6Ptv7Gcu2LGP5tuWs2rGK\n6rrqZt+XGp9Kfm4+E/pN4Dt9vsPQ7kNJiU9px8jDk3Z/2Efr0l6tSQwR9WjPcJKaCjfeCL//vVV+\n+GG47DKItDHcGFcMp/U+jdN6n8YvzvoFFTUVrNqxiuVbl7N863I+2/uZ3/Fl1WW88fUbvPH1G4D1\n1Ln+XfszvMdwTs482fPar0s/YlwR0oRSSvnRFkMb7NkDublQ7f4De+VK/ye+RYP95ft5d9u7LN+6\nnGVbl7H9yPag3pcUm8SwHsO8CSPTeu2WHAbX9irVgWhXkgN+/GN4yj2/3fe+B6++6mw8oWSMYcvh\nLSzfupyCwgK+KPqCrw9+TZ2pC/ocPVN7eloWw3sMZ1C3QQzoOoCMpAy9ZFapENDE4ICNG2HIEP9y\nw6NAw5ld/biVtZVs3L+RdfvW8UXRF57XvWV7W3Se9MR0BnQdwMCuAz2vAzMGRkzS0H5x+2hd2kvH\nGBxw0klw8cXw+utW+U9/8j4GtCNIjE1kRK8RjOg1wm/7/vL9rNu3jnVFVqL4Yt8XfLnvS47WHg14\nnuLKYj7Z/Qmf7P6kyb5ITxpKRRptMdhg5UoYN85aj4+3bnjr2dPZmMJRXX0dWw5v8SSLDQc28M3B\nb9h8aDPlNeWtOmen+E70TO3pt2SmZDbZ1iOlh15mqzok7UpyiDEwZgx89JFV/uUv4Xe/czamSGKM\nYW/ZXjYf2sw3h76xksXhzW1OGo1lJGU0SRiBEklGcobnsapKRTpNDA566SW4/HJrPT3dmlyvUxjf\nMBwp/bgNSeObQ1aSCFXS8BUjMfRI6WEljVR30kjp6V92L2kJaYhIxNRnJNC6tJeOMTjo0kutO6A3\nb7ZuenvqKbj1VqejinwiQq9OvejVqRfjcsb57TPGcLjyMHvL9lJUVsTesr3epdy7XlRWxL7yfQFn\nnA2kztSxp2wPe8r2HPfY+Jh4eqb2JGFnArk7c+ma1JWMpAwykjP8XrsmdfWspyWmaYtEhTVtMdho\n3jz4yU+s9exsK0nEabd2WKitr+VAxQG/5FFUVsSesj0UlRf5bTvWFCF2cInLL4E0rHdJ7EJ6YnrA\npUuStS81PlWTimoR7Upy2NGjVkI44H4s87PPwpVXOhuTarmq2iqKyov8WiF+ycNnvay6rF1jc4mL\ntIS0pkkjIZ3OCZ1JjU8lJT6FlLgUz3pqfCopcSl+6w374mPi2zV+1f40MYSB+++H++6z1vPy4NNP\nw3OaDO3HtUd5dTlF5UUsXb6U3BG5HKw4yMGjBz2vh44e8isfrDhIaXWp02F7xLnimk0knnJc4O3H\nek9SbFKrLyPW30176RhDGLjpJvjDH6zWw9q18M47cN55TkelQiUlPoV+8f0Y3H0w+QPyg3pPdV01\nh48ebpJAiiuL/ZbDlYebbLO7hVJTX+M5t50E8SSPxq0WTwJpZl/htkJKepWQEJNAYmwiCbEJfuuJ\nsYkkxCR41mNd+jVmN20xhMCsWfC3v1nr558Pb7/tbDwqetTU1XCk6kjTJHL0MCVVJZTXlFNeXU5Z\ndRnlNd7XJtvc5ZZMZxKuXOJqkiwa1hte42Pim2xLiAlifzP7jrctnJJVWHYlichE4BHABTxljPlD\no/3jgVeBhmdQ/q8x5oEA54mYxLB1KwwcCPX1VnntWjjlFGdjUqoxYwzVddV+yaJx4mi2fJz3VNZW\nOv3jOcolrmaTSHxMfJME5Let0fty03P50cgftTqWsEsMIuICvgbOBXYDa4ArjDGbfI4ZD9xhjPne\ncc4VMYkBYMoUWLLEWr/6anj6aWfjaUz7ce2l9emvrr7O01JpnER8E0jj9fKacrat3UbaoDQqayup\nqquyXmur/NZ999Wbeqd/3JAa02cMq3+0utXvD8cxhlHAN8aY7QAi8gJwCbCp0XFhODzbNnfe6U0M\nL7wA//M/1oN9lOoIYlwxdE7oTOeEzi1+b0F6y5JsbX1tk4TRkEgCvVbXVR/zmOq6au+25rYfY1tV\nbVXQ98wEIyE2wbZzBSvULYb/B1xgjLnOXb4aGGWMudnnmPHAS8C3wC7gTmPMhgDniqgWA0B+Prz/\nvrV+++3WBHtKqehXW1973GQU7LbstGxmjJjR6ljCscUQjP8C2caYChGZBLwCnOhwTLa4805vYnji\nCZg925ouQykV3WJdscTGx5JCZD72NtSJYReQ7VPu497mYYwp81l/S0T+LiJdjTGHGp9s+vTp5Obm\nApCenk5eXp6nyVlQUAAQVuWkJBgyJJ8NG6CsrICf/xyeeCI84nvkkUfCvv4iqaz1aV+5YT1c4om0\nckFBAQsXLgTwfF+2VKi7kmKAr7AGn/cAHwNTjTEbfY7JNMYUuddHAf80xuQGOFfEdSUBLFgA115r\nrffsCYWFkND+XYZNFOhgqa20Pu2jdWmvsLsqCTyXq/4F7+WqD4rI9YAxxjwhIjcBNwI1wFHgNmPM\nRwHOE5GJoaoK+va1ng8N1uR6DYlCKaVCLSwTg10iNTGAdSf03Xdb6yedBOvXg0vnQVNKtYPWJAb9\nemoH118PqanW+saN8OabzsYD3j5JZQ+tT/toXTpPE0M7SE+H667zlmfNsh4HqpRS4Ui7ktrJzp3W\ng3yqq62yCNxyi/UI0ORkZ2NTSkUv7UoKYyecAM8/D53dN4IaA488Yk3N/cEHzsamlFK+NDG0o+9/\nH778EiZO9G775hs46yy44w5rqu72ov249tL6tI/WpfM0MbSzPn2swecnn/RvPcyZY7Ue/vMfZ+NT\nSikdY3DQzp3w4x/D0qXebS6XNa/Sb34DSUnOxaaUig56H0MEMsZqPdxxB5T6PPFx8GBYuBBGj3Ys\nNKVUFNDB5wgkAjNnwrp1/o8A3bQJxo6Fu+6CyhA880T7ce2l9WkfrUvnaWIIEzk5VpfSY495b4ar\nr4eHHoKRI2HNGmfjU0p1HNqVFIYKC+FHP4J33/Vuc7ng5z+H++4Lj0n4lFKRQccYokh9PTz+uPVM\nh/Jy7/ahQ62xh9NOcyw0pVQE0TGGKOJywY03WmMPvjMQf/kljBoF55xjdTvt29e682s/rr20Pu2j\ndek8TQxhrm9feOcdePRR79QZxsB771mJo1cvOPdcq3Wxf7+zsSqlooN2JUWQLVvgpz+Ff//bSg6N\nuVxw9tkwZQpcdhl0797+MSqlwouOMXQQu3fD//4v/POfsGpV4CQRE+OfJLp1a/84lVLO0zGGDqJ3\nb2vq7hUr4NtvYe5cOPNM/2Pq6mD5cmu675494fzzrRvpDh609ms/rr20Pu2jdek8bTFEkV274KWX\nrJZEczO2xsRYYxI5OQWMHZtP9+7Qo4d30Wk4WkefU2wfrUt7aVeS8vj2W2+SWL06+PelptIkWTQs\njbdnZuojSpUKd5oYVEA7d3qThJ2zt8bHW1dN9evnv/Tvb21vuINbKeUcTQzquHbuhNdegxUrCkhI\nyGffPvyWmhr7PqtHD2+iaJw4evWKrtaGdn/YR+vSXq1JDLGhCkaFpxNOgJtusu6gbvx/zxg4csS6\nH6JxwmhYGvbt3esdyG5Ow3s+/LDpvoQEq1UxeLB3OekkGDQI0tJs+3GVUq2gLQbVaiUlsG0bbN1q\nLVu2eNcLC1vf+ujVy5sofJNGVpY1G61SKnjalaTCRl2ddZWUb7LwTSDHa20EkpLinygGD7aeiAfW\n3FLGWIvvejDljAyri6tbN008KvpoYlBBc7of98gR2LwZvvoKNm60nj+xaRN8/TVUVzsTU2qq/ziI\n77hITs6xZ7V1uj6jidalvXSMQUWMtDQ49VRr8VVXZ3VPNSSKTZusxLFxIxw+HNqYysrgiy+spTER\na3wm0BVYubnW0/cOHLDir621XoNZb3gVsZ4BnpZmLenp1lVfHZH+/ec8bTGoiGCM9cXbkCgaXg8e\ntL5UfReXK/htYA2kb9liJYZwkphoJQjfZBHotWE9MdFKNI2XmprA2wMtyclWl1pGhrX4ricmtu3n\nqa2FoiKri3HXLmtql4Z136WiArKzraQ7YID16rukpNhTv2D9Xh06BNu3w44d3uXbb606HTTI6rIc\nNMhqNcbE2PfZ7UW7kpRqJWOsJNN4EL1h2blT/5JNSWmaLBrWG17T0qwr1wJ9+e/da43vtFXPnv6J\nwjd5ZGT4jxNVV1tf8r5f+o2TQEVFcJ+bkAADB3oTRcProEFWay9chWViEJGJwCNY8zI9ZYz5Q4Bj\n5gKTgHJgujFmbYBjNDHYSPtxW6aqyvpCaTyIvnWr9eVSU2PdFxIba/1VGROD33rjcuN9xlhXeRUX\nW+MvR45YXUwdUwGQ36p3du5sJYr4eOvfZc+e9knoDVfS+SaMAQOsf+fqaqvV1rC0tJyVBVOntj62\nsBtjEBEX8FfgXGA3sEZEXjXGbPI5ZhLQ3xgzUERGA48B3wllXArWrl2riaEFEhLgxBOtJZBHHlnL\nrbfm2/Z5xlhP7mtIEr4Jo7n1ykqIi7O+jBpeW7LExFjdaQcPWt12Bw/6r9fWtv3n6t7d+qLr3dt6\nbbz07g3z56/lu9/NZ8sWK/lu3oxn/XiXQZeUwKeftiym1FSrmyg721pycqxYDh60uiy/+sp6LSpq\n/hx79ljLe++17LODMX582xJDa4R68HkU8I0xZjuAiLwAXAJs8jnmEmAxgDHmIxFJE5FMY8wx/hlU\nWxUXFzsdQlSxuz5FrC+s1FTrS8ppDS2axsnCd/3AAStBZWQ0/bLPyrL+qg7meeUVFcUMGQJDhjTd\nV1dndev5JgvfBOL7GFywxpF69/Z+6fsuDckgLS24y5SLi60k0ZAoGl43bw7tlXR2zkYQrFAnhixg\np0/5W6xkcaxjdrm3aWJQKkyIeAe6+/VzLo6YGOsqsNxcOO88/33GWHfab9litW6ys62EFBdnz2en\np8Po0dbiq7bWask0Thrbtln1FhdndW3FxXkX3/Kx9sXFOVPferlqB1VYWOh0CFFF69M+ra1LEWvG\n38xMe+M5nthYazxhwAC46KL2/exQCengs4h8B7jPGDPRXb4bML4D0CLyGPCeMeZFd3kTML5xV5KI\n6MizUkq1QlgNPgNrgAEikgPsAa4AGg+jvAbcBLzoTiTFgcYXWvqDKaWUap2QJgZjTJ2IzAKW4r1c\ndaOIXG/tNk8YY94UkQtFZDPW5aozQhmTUkqpY4uYG9yUUkq1j4h4VIqITBSRTSLytYjc5XQ8kU5E\nCkXkcxH5TEQ+djqeSCIiT4lIkYh84bOti4gsFZGvRORtEdEnSgSpmfq8V0S+FZFP3ctEJ2OMFCLS\nR0TeFZEvRWSdiNzs3t7i38+wTww+N8ldAAwFporIYGejinj1QL4xZoQxpvHlw+rYFmD9Lvq6G1hu\njBkEvAv8ot2jilyB6hNgjjFmpHv5d3sHFaFqgduNMUOBMcBN7u/KFv9+hn1iwOcmOWNMDdBwk5xq\nPSEy/u3DjjFmFdB4ntdLgEXu9UXApe0aVARrpj7B+h1VLWCM2dswnZAxpgzYCPShFb+fkfDlEOgm\nuTC4FzSiGWCZiKwRkZlOBxMFejRcSWeM2Qv0cDieaDBLRNaKyJPaNddyIpIL5AEfApkt/f2MhMSg\n7HeGMWYkcCFWc/NMpwOKMnpFR9v8HehnjMkD9gJzHI4noohIKvAv4BZ3y6Hx7+Nxfz8jITHsArJ9\nyn3c21QrGWP2uF/3Ay/TdJoS1TJFIpIJICI9gX0OxxPRjDH7faZS/gdwupPxRBIRicVKCk8bY151\nb27x72ckJAbPTXIiEo91k9xrDscUsUQk2f0XBSKSApwPrHc2qogj+PeBvwZMd6//EHi18RvUMfnV\np/vLq8H30d/PlpgPbDDG/MVnW4t/PyPiPgb35Wp/wXuT3IMOhxSxRKQvVivBYN3g+KzWZ/BE5Dms\nhwVkYE30eC/wCrAEOAHYDkwxxuj0tUFopj7PxuofrwcKget1tuXjE5EzgBXAOqz/3wb4JfAx8E9a\n8PsZEYlBKaVU+4mEriSllFLtSBODUkopP5oYlFJK+dHEoJRSyo8mBqWUUn40MSillPKjiUGpdiAi\n40XkdafjUCoYmhiUaj9605CKCJoYlPIhIleJyEfuB8TMExGXiJSKyBwRWS8iy0Qkw31snoj8xz0L\n6EsNs4CKSH/3cWtF5BP33eYAnURkiYhsFJGnHfshlToOTQxKubkfavIDYKx79tl64CogGfjYGDMM\na8qBe91vWQTc6Z4FdL3P9meBR93bxwJ73NvzgJuBIUB/ERkb+p9KqZaLdToApcLIucBIYI2ICJCI\nNX9PPdZcMwDPAC+JSGcgzf2gGbCSxD/dExRmGWNeAzDGVANYp+PjhpltRWQtkAusboefS6kW0cSg\nlJcAi4wxv/LbKDK70XHG5/iWqPJZr0P//6kwpV1JSnm9A1wuIt3B8xD1bCAGuNx9zFXAKmNMCXDI\nPaMlwDTgffeDUXaKyCXuc8SLSFK7/hRKtZH+xaKUmzFmo4j8GlgqIi6gGpgFlAOj3C2HIqxxCLDm\ntn/c/cW/FZjh3j4NeEJEfuM+x+RAHxe6n0SpttFpt5U6DhEpNcZ0cjoOpdqLdiUpdXz615PqULTF\noJRSyo+2GJRSSvnRxKCUUsqPJgallFJ+NDEopZTyo4lBKaWUH00MSiml/Px/iaVSf1zaGtEAAAAA\nSUVORK5CYII=\n",
801 |       "text/plain": [
802 |        "<matplotlib.figure.Figure at 0x1206ac490>"
803 |       ]
804 |      },
805 |      "metadata": {},
806 |      "output_type": "display_data"
807 |     }
808 |    ],
809 |    "source": [
810 |     "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
811 |     "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
812 |     "pyplot.grid()\n",
813 |     "pyplot.legend()\n",
814 |     "pyplot.ylim(0, 3.0)\n",
815 |     "pyplot.xlabel(\"epoch\")\n",
816 |     "pyplot.ylabel(\"loss\")\n",
817 |     "pyplot.show()"
818 |    ]
819 |   },
820 |   {
821 |    "cell_type": "markdown",
822 |    "metadata": {
823 |     "collapsed": true
824 |    },
825 |    "source": [
826 |     "#### Train loss is far lower than valid loss : we probably overfit, but still valid loss is not so bad ;)"
827 |    ]
828 |   },
829 |   {
830 |    "cell_type": "code",
831 |    "execution_count": null,
832 |    "metadata": {
833 |     "collapsed": true
834 |    },
835 |    "outputs": [],
836 |    "source": []
837 |   }
838 |  ],
839 |  "metadata": {
840 |   "kernelspec": {
841 |    "display_name": "Python 2",
842 |    "language": "python",
843 |    "name": "python2"
844 |   },
845 |   "language_info": {
846 |    "codemirror_mode": {
847 |     "name": "ipython",
848 |     "version": 2
849 |    },
850 |    "file_extension": ".py",
851 |    "mimetype": "text/x-python",
852 |    "name": "python",
853 |    "nbconvert_exporter": "python",
854 |    "pygments_lexer": "ipython2",
855 |    "version": "2.7.11"
856 |   }
857 |  },
858 |  "nbformat": 4,
859 |  "nbformat_minor": 0
860 | }
861 | 


--------------------------------------------------------------------------------