├── .gitignore ├── Convolutional Neural Network ├── Convolution.PNG ├── ConvolutionalNeuralNetwork.ipynb └── Forward.JPG ├── LICENSE ├── Neural Network ├── NeuralNetwork.ipynb └── network.JPG ├── README.md └── Recurrent Neural Network ├── LSTM.JPG ├── LSTMForward.JPG └── RecurrentNeuralNet(LSTM).ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | Recurrent\ Neural\ Network -------------------------------------------------------------------------------- /Convolutional Neural Network/Convolution.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navjindervirdee/neural-networks/6c1a6e13ffc38110863bae530aa4f148d15510ed/Convolutional Neural Network/Convolution.PNG -------------------------------------------------------------------------------- /Convolutional Neural Network/Forward.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navjindervirdee/neural-networks/6c1a6e13ffc38110863bae530aa4f148d15510ed/Convolutional Neural Network/Forward.JPG -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Navjinder Virdee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Neural Network/NeuralNetwork.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Implementing Neural Network from Scratch.\n", 8 | "Neural Networks are really powerful algorithms used for classification.
\n", 9 | "Dataset = Iris_Dataset
\n", 10 | "Link = http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html
\n", 11 | "\n", 12 | "### Import required libraries" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 324, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "from sklearn import datasets #for dataset\n", 24 | "import numpy as np #for maths\n", 25 | "import matplotlib.pyplot as plt #for plotting" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": {}, 31 | "source": [ 32 | "### Get Dataset" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 325, 38 | "metadata": {}, 39 | "outputs": [ 40 | { 41 | "name": "stdout", 42 | "output_type": "stream", 43 | "text": [ 44 | "Data Shape = (150, 4)\n", 45 | "Target Shape = (150, 1)\n", 46 | "Classes : [0 1 2]\n", 47 | "Sample data : [ 5.9 3.2 4.8 1.8] , Target = [1]\n" 48 | ] 49 | } 50 | ], 51 | "source": [ 52 | "iris = datasets.load_iris() #load the dataset\n", 53 | "data = iris.data #get features \n", 54 | "target = iris.target #get labels\n", 55 | "\n", 56 | "shape = data.shape #shape of data\n", 57 | "\n", 58 | "#convert into numpy array\n", 59 | "data = np.array(data).reshape(shape[0],shape[1])\n", 60 | "target = np.array(target).reshape(shape[0],1)\n", 61 | "\n", 62 | "#print shape\n", 63 | "print(\"Data Shape = {}\".format(data.shape))\n", 64 | "print(\"Target Shape = {}\".format(target.shape))\n", 65 | "print('Classes : {}'.format(np.unique(target)))\n", 66 | "print('Sample data : {} , Target = {}'.format(data[70],target[70]))" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Define Parameters and Hyperparameters\n", 74 | "\n", 75 | "\n", 76 | "\n", 77 | "\n" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "One hidden layer Neural Network.\n", 85 | " ![](https://github.com/navjindervirdee/neural-networks/blob/master/Neural%20Network/network.JPG?raw=true)", 86 | "\n", 87 | "Input Units = 4
\n", 88 | "Hidden Units = 8
\n", 89 | "Output Units = 3
" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 326, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "#HYPERPARAMETERS\n", 99 | "\n", 100 | "#num of target labels\n", 101 | "num_classes = len(np.unique(target))\n", 102 | "\n", 103 | "#define layer_neurons\n", 104 | "input_units = 4 #neurons in input layer\n", 105 | "hidden_units = 8 #neurons in hidden layer\n", 106 | "output_units = 3 #neurons in output layer\n", 107 | "\n", 108 | "#define hyper-parameters\n", 109 | "learning_rate = 0.03\n", 110 | "\n", 111 | "#regularization parameter\n", 112 | "beta = 0.00001\n", 113 | "\n", 114 | "#num of iterations\n", 115 | "iters = 4001" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "### Dimesions of Parameters\n", 123 | "Shape of layer1_weights (Wxh) = (4,8)
\n", 124 | "Shape of layer1_biasess (Bh) = (8,1)
\n", 125 | "Shape of layer2_weights (Why) = (8,3)
\n", 126 | "Shape of layer2_biasess (By) = (3,1)
" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 327, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "#PARAMETERS\n", 138 | "\n", 139 | "#initialize parameters i.e weights\n", 140 | "def initialize_parameters():\n", 141 | " #initial values should have zero mean and 0.1 standard deviation\n", 142 | " mean = 0 #mean of parameters \n", 143 | " std = 0.03 #standard deviation\n", 144 | " \n", 145 | " layer1_weights = np.random.normal(mean,std,(input_units,hidden_units)) \n", 146 | " layer1_biases = np.ones((hidden_units,1)) \n", 147 | " layer2_weights = np.random.normal(mean,std,(hidden_units,output_units))\n", 148 | " layer2_biases = np.ones((output_units,1))\n", 149 | " \n", 150 | " parameters = dict()\n", 151 | " parameters['layer1_weights'] = layer1_weights\n", 152 | " parameters['layer1_biases'] = layer1_biases\n", 153 | " parameters['layer2_weights'] = layer2_weights\n", 154 | " parameters['layer2_biases'] = layer2_biases\n", 155 | " \n", 156 | " return parameters" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "### Activation Function\n", 164 | "\n", 165 | "**Sigmoid**\n", 166 | "\n", 167 | "![](https://upload.wikimedia.org/wikipedia/commons/8/88/Logistic-curve.svg)", 168 | "\n" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 328, 174 | "metadata": { 175 | "collapsed": true 176 | }, 177 | "outputs": [], 178 | "source": [ 179 | "#activation function\n", 180 | "def sigmoid(X):\n", 181 | " return 1/(1+np.exp((-1)*X))\n", 182 | "\n", 183 | "#softmax function for output\n", 184 | "def softmax(X):\n", 185 | " exp_X = np.exp(X)\n", 186 | " exp_X_sum = np.sum(exp_X,axis=1).reshape(-1,1)\n", 187 | " exp_X = (exp_X/exp_X_sum)\n", 188 | " return exp_X" 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": {}, 194 | "source": [ 195 | "### Define Utility Functions\n", 196 | "#### 1. Forward Propagation\n", 197 | "---- Logits = matmul(X,Wxh) + Bh
\n", 198 | "---- A = sigmoid(logits)
\n", 199 | "---- logits = matmul(A,Why) + By
\n", 200 | "---- output = softmax(logits)
\n", 201 | "\n", 202 | "Store output and A in cache to use it in backward propagation
\n", 203 | "\n", 204 | "#### 2. Backward Propagation\n", 205 | "---- Error_output = output - train_labels
\n", 206 | "---- Error_activation = (matmul(error_output,Why.T))(A)(1-A)
\n", 207 | "---- dWhy = (matmul(A.T,error_output))/m
\n", 208 | "---- dWxh = (matmul(train_dataset.T,error_activation))/m
\n", 209 | "\n", 210 | "m = len(train_dataset)
\n", 211 | "Store derivatives in derivatives dict\n", 212 | "\n", 213 | "#### 3. Update Parameters\n", 214 | "---- Wxh = Wxh - learning_rate(dWxh + beta*Wxh)
\n", 215 | "---- Why = Why - learning_rate(dWhy + beta*Why)
\n", 216 | "\n", 217 | "#### 4. Calculate Loss and Accuracy\n", 218 | "---- Loss = (-1(Y log(prediction)) + (1-Y) (log(1-predictions))) + beta * (sum(Wxh^2) + sum(Why^2)))/m
\n", 219 | "---- Accuracy = sum(Y==predictions)/m " 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 329, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "#forward propagation\n", 229 | "def forward_propagation(train_dataset,parameters):\n", 230 | " cache = dict() #to store the intermediate values for backward propagation\n", 231 | " m = len(train_dataset) #number of training examples\n", 232 | " \n", 233 | " #get the parameters\n", 234 | " layer1_weights = parameters['layer1_weights']\n", 235 | " layer1_biases = parameters['layer1_biases']\n", 236 | " layer2_weights = parameters['layer2_weights']\n", 237 | " layer2_biases = parameters['layer2_biases']\n", 238 | " \n", 239 | " #forward prop\n", 240 | " logits = np.matmul(train_dataset,layer1_weights) + layer1_biases.T\n", 241 | " activation1 = np.array(sigmoid(logits)).reshape(m,hidden_units)\n", 242 | " activation2 = np.array(np.matmul(activation1,layer2_weights) + layer2_biases.T).reshape(m,output_units)\n", 243 | " output = np.array(softmax(activation2)).reshape(m,num_classes)\n", 244 | " \n", 245 | " #fill in the cache\n", 246 | " cache['output'] = output\n", 247 | " cache['activation1'] = activation1\n", 248 | " \n", 249 | " return cache,output\n", 250 | "\n", 251 | "#backward propagation\n", 252 | "def backward_propagation(train_dataset,train_labels,parameters,cache):\n", 253 | " derivatives = dict() #to store the derivatives\n", 254 | " \n", 255 | " #get stuff from cache\n", 256 | " output = cache['output']\n", 257 | " activation1 = cache['activation1']\n", 258 | " \n", 259 | " #get parameters\n", 260 | " layer1_weights = parameters['layer1_weights']\n", 261 | " layer2_weights = parameters['layer2_weights']\n", 262 | " \n", 263 | " #calculate errors\n", 264 | " error_output = output - train_labels\n", 265 | " error_activation1 = np.matmul(error_output,layer2_weights.T)\n", 266 | " error_activation1 = np.multiply(error_activation1,activation1)\n", 267 | " error_activation1 = np.multiply(error_activation1,1-activation1)\n", 268 | " \n", 269 | " \n", 270 | " #calculate partial derivatives\n", 271 | " partial_derivatives2 = np.matmul(activation1.T,error_output)/len(train_dataset)\n", 272 | " partial_derivatives1 = np.matmul(train_dataset.T,error_activation1)/len(train_dataset)\n", 273 | " \n", 274 | " #store the derivatives\n", 275 | " derivatives['partial_derivatives1'] = partial_derivatives1\n", 276 | " derivatives['partial_derivatives2'] = partial_derivatives2\n", 277 | " \n", 278 | " return derivatives\n", 279 | "\n", 280 | "\n", 281 | "#update the parameters\n", 282 | "def update_parameters(derivatives,parameters):\n", 283 | " #get the parameters\n", 284 | " layer1_weights = parameters['layer1_weights']\n", 285 | " layer2_weights = parameters['layer2_weights']\n", 286 | " \n", 287 | " #get the derivatives\n", 288 | " partial_derivatives1 = derivatives['partial_derivatives1']\n", 289 | " partial_derivatives2 = derivatives['partial_derivatives2']\n", 290 | " \n", 291 | " #update the derivatives\n", 292 | " layer1_weights -= (learning_rate*(partial_derivatives1 + beta*layer1_weights))\n", 293 | " layer2_weights -= (learning_rate*(partial_derivatives2 + beta*layer2_weights))\n", 294 | " \n", 295 | " #update the dict\n", 296 | " parameters['layer1_weights'] = layer1_weights\n", 297 | " parameters['layer2_weights'] = layer2_weights\n", 298 | " \n", 299 | " return parameters\n", 300 | " \n", 301 | "#calculate the loss and accuracy\n", 302 | "def cal_loss_accuray(train_labels,predictions,parameters):\n", 303 | " #get the parameters\n", 304 | " layer1_weights = parameters['layer1_weights']\n", 305 | " layer2_weights = parameters['layer2_weights']\n", 306 | " \n", 307 | " #cal loss and accuracy\n", 308 | " loss = -1*np.sum(np.multiply(np.log(predictions),train_labels) + np.multiply(np.log(1-predictions),(1-train_labels)))/len(train_labels) + np.sum(layer1_weights**2)*beta/len(train_labels) + np.sum(layer2_weights**2)*beta/len(train_labels)\n", 309 | " accuracy = np.sum(np.argmax(train_labels,axis=1)==np.argmax(predictions,axis=1))\n", 310 | " accuracy /= len(train_dataset)\n", 311 | " \n", 312 | " return loss,accuracy" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "### Train Function\n", 320 | "\n", 321 | "1. Initialize Parameters\n", 322 | "2. Forward Propagation\n", 323 | "3. Backward Propagation\n", 324 | "4. Calculate Loss and Accuracy\n", 325 | "5. Update the parameters\n", 326 | "\n", 327 | "Repeat the steps 2-5 for the given number of iterations" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 330, 333 | "metadata": { 334 | "collapsed": true 335 | }, 336 | "outputs": [], 337 | "source": [ 338 | "#Implementation of 3 layer Neural Network\n", 339 | "\n", 340 | "#training function\n", 341 | "def train(train_dataset,train_labels,iters=2):\n", 342 | " #To store loss after every iteration.\n", 343 | " J = []\n", 344 | " \n", 345 | " #WEIGHTS\n", 346 | " global layer1_weights\n", 347 | " global layer1_biases\n", 348 | " global layer2_weights\n", 349 | " global layer2_biases\n", 350 | " \n", 351 | " #initialize the parameters\n", 352 | " parameters = initialize_parameters()\n", 353 | " \n", 354 | " layer1_weights = parameters['layer1_weights']\n", 355 | " layer1_biases = parameters['layer1_biases']\n", 356 | " layer2_weights = parameters['layer2_weights']\n", 357 | " layer2_biases = parameters['layer2_biases']\n", 358 | " \n", 359 | " #to store final predictons after training\n", 360 | " final_output = []\n", 361 | " \n", 362 | " for j in range(iters):\n", 363 | " #forward propagation\n", 364 | " cache,output = forward_propagation(train_dataset,parameters)\n", 365 | " \n", 366 | " #backward propagation\n", 367 | " derivatives = backward_propagation(train_dataset,train_labels,parameters,cache)\n", 368 | " \n", 369 | " #calculate the loss and accuracy\n", 370 | " loss,accuracy = cal_loss_accuray(train_labels,output,parameters)\n", 371 | " \n", 372 | " #update the parameters\n", 373 | " parameters = update_parameters(derivatives,parameters)\n", 374 | " \n", 375 | " #append loss\n", 376 | " J.append(loss)\n", 377 | " \n", 378 | " #update final output\n", 379 | " final_output = output\n", 380 | " \n", 381 | " #print accuracy and loss\n", 382 | " if(j%500==0):\n", 383 | " print(\"Step %d\"%j)\n", 384 | " print(\"Loss %f\"%loss)\n", 385 | " print(\"Accuracy %f%%\"%(accuracy*100))\n", 386 | " \n", 387 | " return J,final_output" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": 331, 393 | "metadata": {}, 394 | "outputs": [], 395 | "source": [ 396 | "#shuffle the dataset\n", 397 | "z = list(zip(data,target))\n", 398 | "np.random.shuffle(z)\n", 399 | "data,target = zip(*z)\n", 400 | "\n", 401 | "#make train_dataset and train_labels\n", 402 | "train_dataset = np.array(data).reshape(-1,4)\n", 403 | "train_labels = np.zeros([train_dataset.shape[0],num_classes])\n", 404 | "\n", 405 | "#one-hot encoding\n", 406 | "for i,label in enumerate(target):\n", 407 | " train_labels[i,label] = 1\n", 408 | "\n", 409 | "#normalizations\n", 410 | "for i in range(input_units):\n", 411 | " mean = train_dataset[:,i].mean()\n", 412 | " std = train_dataset[:,i].std()\n", 413 | " train_dataset[:,i] = (train_dataset[:,i]-mean)/std" 414 | ] 415 | }, 416 | { 417 | "cell_type": "code", 418 | "execution_count": 332, 419 | "metadata": {}, 420 | "outputs": [ 421 | { 422 | "name": "stdout", 423 | "output_type": "stream", 424 | "text": [ 425 | "Step 0\n", 426 | "Loss 1.911767\n", 427 | "Accuracy 33.333333%\n", 428 | "Step 500\n", 429 | "Loss 1.772284\n", 430 | "Accuracy 76.666667%\n", 431 | "Step 1000\n", 432 | "Loss 0.934929\n", 433 | "Accuracy 93.333333%\n", 434 | "Step 1500\n", 435 | "Loss 0.648656\n", 436 | "Accuracy 93.333333%\n", 437 | "Step 2000\n", 438 | "Loss 0.504929\n", 439 | "Accuracy 96.000000%\n", 440 | "Step 2500\n", 441 | "Loss 0.413799\n", 442 | "Accuracy 96.666667%\n", 443 | "Step 3000\n", 444 | "Loss 0.348909\n", 445 | "Accuracy 96.666667%\n", 446 | "Step 3500\n", 447 | "Loss 0.301780\n", 448 | "Accuracy 96.000000%\n", 449 | "Step 4000\n", 450 | "Loss 0.267607\n", 451 | "Accuracy 97.333333%\n" 452 | ] 453 | } 454 | ], 455 | "source": [ 456 | "#train data\n", 457 | "J,final_output = train(train_dataset,train_labels,iters=4001)" 458 | ] 459 | }, 460 | { 461 | "cell_type": "markdown", 462 | "metadata": {}, 463 | "source": [ 464 | "#### Reached an Accuracy of 97%" 465 | ] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "metadata": {}, 470 | "source": [ 471 | "### Plot the loss vs iteration graph" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 333, 477 | "metadata": {}, 478 | "outputs": [ 479 | { 480 | "data": { 481 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8XXWd//HXJ3uzNGmapUu6QlpaullC2UtBEAoIoigg\nAo44UBR1dHREnZ86Mw8cZtRRcRktOw7LqIgwrFJQSsvWtLSlpStt2qZb0nRJ0yXN8vn9cU/rpSTt\nTZubc+/N+/l43EfO/Z5z7v3c80jyvud7zvkec3dERESOJi3sAkREJDkoMEREJCYKDBERiYkCQ0RE\nYqLAEBGRmCgwREQkJgoMkQ6YWZOZjQy7DpFEosCQhGNmNWZ2QTD9WTObE+f3+6uZfT66zd3z3X1N\nPN836v2vCT6zHdaeYWZ1ZnZZ8PzbZrY2CLNaM/vfI7zmoW0o0l0UGJLSzCwj7Bpi8CegCDj3sPaL\nAQeeN7MbgeuBC9w9H6gCXurRKqXXU2BIwjKzMcCvgTOCb9U7g/ZsM/uRma03s61m9msz6xPMmxZ8\n+/6mmW0B7jezfmb2tJnVm9mOYLoiWP4O4BzgF8F7/CJodzM7MZguNLOHgvXXmdk/m1laMO+zZjYn\nqGdHsAcwPeozfNbM1pjZ7mDedYd/TnffD/wOuOGwWTcAj7h7K3Aq8IK7vxess8XdZx7jdv17M1tt\nZtvN7CkzGxS0m5n9JNiraTSzd8xsXDDvEjN7N/gcG83s68fy3pLcFBiSsNx9GTADeD3oIioKZt0J\njAImAScCg4HvRq06ACgGhgE3E/k9vz94PhTYB/wieI/vAK8CtwXvcVsHpfwcKARGEtkLuAH4u6j5\npwErgBLgP4F7g3++ecBdwHR3LwDOBBZ28nEfBK6KCr5C4KNBO8AbwA1m9g0zqzKz9E433BGY2fnA\nvwOfAgYC64DHgtkfAaYS2baFwTINwbx7gVuCzzEOePlY3l+SmwJDkkrQz38z8FV33+7uu4EfANdE\nLdYOfM/dm919n7s3uPvj7r43WP4OPtj909n7pQev/S133+3uNcCPiXQPHbTO3e929zYi/+AHAuVR\ntYwzsz7uvtndl3b0Pu4+F9gKXBk0fQpY6e4Lg/n/A3wJuAh4Bagzs2/G8hkOcx1wn7svcPdm4FtE\n9uCGAy1AAXASYO6+zN03B+u1AGPNrK+773D3Bcfw3pLkFBiSbEqBXGC+me0MuqmeD9oPqg+6eQAw\ns1wz+03QndQIzAaKYvyWXgJkEvkmftA6Ins1B205OOHue4PJfHffA1xNZC9ps5k9Y2YnHeG9HuJv\n3VLXB88PcfeH3f0CIsc7ZgD/ZmYXxfAZog2K/izu3kRkL2Kwu79MZM/rl0QCaaaZ9Q0W/QRwCbDO\nzF4xszO6+L6SAhQYkugOH055G5EupZPdvSh4FAYHgjtb5x+B0cBp7t6XSLcLgHWy/OHv10KkO+ug\nocDGmIp3f8HdLySy17EcuPsIi/8W+HDwz/h04OFOXrPF3X8PLCbSPdQVm4j6LEG3WX+Cz+Pud7n7\nKcBYIl1T3wja57n7FUAZkYP0v+vi+0oKUGBIotsKVJhZFoC7txP5p/sTMysDMLPBR/mmXUAkZHaa\nWTHwvQ7eo8NrLoJupt8Bd5hZgZkNA74G/M/RCjezcjO7Ivin3Aw0Eemi6lDQ3TUHeBR40d0P7bkE\nB88vDWpICw6snwy8eYQSMs0sJ+qREbz235nZJDPLJtKd96a715jZqWZ2mpllAnuA/UC7mWWZ2XVm\nVujuLUDjkT6HpC4FhiS6l4GlwBYz2xa0fRNYDbwRdDHNIrIH0ZmfAn2I7C28QaQLK9rPiBxw3mFm\nd3Ww/peI/ANdQ+Qf+iPAfTHUnkYkXDYB24kcN7n1KOs8SGQP4KHD2huBbwPrgZ1EDq7f6u5Hukbl\nWSJBefDxfXefBfw/4HFgM3ACfzv+05dIGO8g0m3VAPwwmHc9UBNs7xlEjoVIL2O6gZKIiMRCexgi\nIhITBYaIiMREgSEiIjFRYIiISEySYWC2mJWUlPjw4cPDLkNEJGnMnz9/m7uXHn3JFAuM4cOHU11d\nHXYZIiJJw8zWHX2pCHVJiYhITBQYIiISEwWGiIjERIEhIiIxUWCIiEhMFBgiIhITBYaIiMQkpa7D\nOFZ3vbSKQUV9GFSUQ9+cTPKzM8jNSic3O4M+memkp9nRX0REJMX1+sBobm3jvrlr2bm3pdNlcjLT\nyM0KQiQr/dB0XnYGZQXZDCzMYUBhJHAqywooLcjuwU8gItIzen1gZGeks+CfL2TNtibqdx9g174W\n9rW0sqe5jX0H2thzoPXQz73NbeyNatu+Zy/z1+1g+54D73vNkvxsxgws4PSR/TnrxBLGDy7UXoqI\nJL2UuoFSVVWVhzE0yP6WNrbs2k/tjn2s2LqbZZsbWbJxF8u37AYiAfKxSYP4ZNUQRg8o6PH6REQ6\nY2bz3b0qpmUVGPGzramZuau38ew7m3l5eR0tbc4FY8r5hwsqGTe4MOzyREQUGIlo+54DPPzGOu6Z\ns5bG/S3ceMZwvn7RaPKze32voIiEqCuBodNqe0hxXhZf+nAlc755HjeeMZwHX6/hsrteZUXQbSUi\nkugUGD2sICeT719+Mv978xnsOdDGlb+ay19W1IVdlojIUSkwQjJlRDFPf+lsRpTkcctD85n17taw\nSxIROSIFRojK++bwyOdP56SBBXzh4QXMX7c97JJERDqlwAhZYW4mD31uCoOKcrj5ofls2L437JJE\nRDqkwEgARblZ3HPjqRxobecrj71Na1t72CWJiHyAAiNBnFiWz799bBwL1u/k16+8F3Y5IiIfoMBI\nIFdMGsRHJw7ip7NWsbpOp9uKSGJRYCQQM+N7Hx1Ln6x0/uX/3iWVLqoUkeSnwEgwJfnZfO3CUby6\naht/1qm2IpJAFBgJ6PrTh3FCaR4//vMK2tu1lyEiiUGBkYAy0tP4ygWjWLm1iWfe2Rx2OSIiQBwD\nw8zuM7M6M1vSyfxvmNnC4LHEzNrMrDiYV2Nm7wTzEnM0wTi7dPxAKsvy+dlLq7SXISIJIZ57GA8A\nF3c2091/6O6T3H0S8C3gFXePvtT5vGB+TKMoppr0NOO2809kdV2TxpoSkYQQt8Bw99lArGNdXAs8\nGq9aktUl4wcyoG8O98+tCbsUEZHwj2GYWS6RPZHHo5odmGVm883s5qOsf7OZVZtZdX19fTxL7XGZ\n6Wlcf8Yw5qzexsqtui5DRMIVemAAHwXmHtYddXbQVTUd+KKZTe1sZXef6e5V7l5VWloa71p73LVT\nhpKdkcaDr9WEXYqI9HKJEBjXcFh3lLtvDH7WAU8AU0KoKyEU52Vx6fiBPLVwE/sOtIVdjoj0YqEG\nhpkVAucCT0a15ZlZwcFp4CNAh2da9RZXVVWwu7mVF5ZuCbsUEenF4nla7aPA68BoM6s1s5vMbIaZ\nzYha7Ergz+6+J6qtHJhjZouAt4Bn3P35eNWZDE4f0Z+Kfn34/fwNYZciIr1YRrxe2N2vjWGZB4ic\nfhvdtgaYGJ+qklNamnHVKRX87KVV1O7YS0W/3LBLEpFeKBGOYUgMPjG5Anf4v0W68ltEwqHASBJD\ninOZOKSIZ97ZFHYpItJLKTCSyGXjB7JkYyPrGvYcfWERkW6mwEgi08cPANCAhCISCgVGEqnol8uk\nIUU8q8AQkRAoMJLMZRMi3VI129QtJSI9S4GRZC46OdItNWuZ7sYnIj1LgZFkhhTnMrq8QIEhIj1O\ngZGEPjymjHk1O9i1ryXsUkSkF1FgJKEPjymnrd15ZWVqDecuIolNgZGEJg0pon9eFi+pW0pEepAC\nIwmlpxnnnVTGX1fU09rWHnY5ItJLKDCS1AVjyti1r4XqdTvCLkVEegkFRpI6p7KUzHTjLyvqwi5F\nRHoJBUaSysvOoGpYMbNXbgu7FBHpJRQYSWzqqFKWbW6krnF/2KWISC+gwEhiU0eVADB7lfYyRCT+\nFBhJbMyAvpTkZzNb12OISA9QYCSxtDRjamUJc1Zvo73dwy5HRFKcAiPJTR1VyvY9B1iyaVfYpYhI\nilNgJLmzKyPHMV5ZoW4pEYmvuAWGmd1nZnVmtqST+dPMbJeZLQwe342ad7GZrTCz1WZ2e7xqTAUl\n+dmMH1zI7FUKDBGJr3juYTwAXHyUZV5190nB418BzCwd+CUwHRgLXGtmY+NYZ9KbOqqEBet30rhf\no9eKSPzELTDcfTaw/RhWnQKsdvc17n4AeAy4oluLSzFTK0tpa3deW90QdikiksLCPoZxppktNrPn\nzOzkoG0wsCFqmdqgrUNmdrOZVZtZdX197+yWmTysH/nZGRruXETiKszAWAAMdfcJwM+BPx3Li7j7\nTHevcveq0tLSbi0wWWSmp3HGCf2ZvbIed51eKyLxEVpguHujuzcF088CmWZWAmwEhkQtWhG0yRGc\nO6qUjTv3sWbbnrBLEZEUFVpgmNkAM7NgekpQSwMwD6g0sxFmlgVcAzwVVp3J4txRkb0rnV4rIvGS\nEa8XNrNHgWlAiZnVAt8DMgHc/dfAVcCtZtYK7AOu8Uh/SquZ3Qa8AKQD97n70njVmSqGFOcysiSP\n2avq+dzZI8IuR0RSUNwCw92vPcr8XwC/6GTes8Cz8agrlU0dVcpj89azv6WNnMz0sMsRkRQT9llS\n0o3OHVXK/pZ25tUcy9nMIiJHpsBIIaeNLCYrPU2j14pIXCgwUkhuVganjuin6zFEJC4UGCnm3FGl\nrNzaxOZd+8IuRURSjAIjxUwNTq9Vt5SIdDcFRooZXV5Aed9sZq/UbVtFpHspMFKMmTG1spRXV9XT\n2tYedjkikkIUGCno3NGlNO5vZVGt7sInIt1HgZGCzj6xhDRDZ0uJSLdSYKSgotwsJg4p0oFvEelW\nCowUNbWylEW1O9mx50DYpYhIilBgpKipo0pxh1dX62wpEekeCowUNWlIEUW5mfx1eV3YpYhIilBg\npKj0NOP80WW8vKJOp9eKSLdQYKSwC8eWs3NvC9XrdoRdioikAAVGCjtnVClZ6WnMendr2KWISApQ\nYKSw/OwMzjyxPy8u20rkZoYiIsdOgZHiLhhTzrqGvayuawq7FBFJcgqMFHfBmHIAXlymbikROT4K\njBQ3oDCHCRWFvKjjGCJynBQYvcCFY8pZuGEndbv3h12KiCSxuAWGmd1nZnVmtqST+deZ2WIze8fM\nXjOziVHzaoL2hWZWHa8ae4uLxg3AHZ5fsiXsUkQkicVzD+MB4OIjzF8LnOvu44F/A2YeNv88d5/k\n7lVxqq/XGFVewKjyfJ5evDnsUkQkicUtMNx9NrD9CPNfc/eDV5S9AVTEqxaByyYMYl7NdrY2qltK\nRI5NohzDuAl4Luq5A7PMbL6Z3XykFc3sZjOrNrPq+noN592ZSycMxB2e0V6GiByj0APDzM4jEhjf\njGo+290nAdOBL5rZ1M7Wd/eZ7l7l7lWlpaVxrjZ5nVCaz5iBfXl68aawSxGRJBVqYJjZBOAe4Ap3\nbzjY7u4bg591wBPAlHAqTC2XTRjIgvU72bhzX9iliEgSCi0wzGwo8EfgendfGdWeZ2YFB6eBjwAd\nnmklXXPp+IEAPKtuKRE5BvE8rfZR4HVgtJnVmtlNZjbDzGYEi3wX6A/86rDTZ8uBOWa2CHgLeMbd\nn49Xnb3J8JI8JlQU8se3N4ZdiogkoYx4vbC7X3uU+Z8HPt9B+xpg4gfXkO5w1SkVfPfJpSzdtIuT\nBxWGXY6IJJHQD3pLz7p84iCy0tP4w/zasEsRkSSjwOhlinKzuHBsOU8u3MSBVt2JT0Rip8Doha46\npYLtew7wsu73LSJdoMDohc6pLKGsIFvdUiLSJQqMXigjPY0rJw/mLyvq2LJLQ4WISGxiCgwzO8HM\nsoPpaWb2ZTMrim9pEk/XTRlGuzuPvLU+7FJEJEnEuofxONBmZicSGVV2CPBI3KqSuBvaP5fzRpfx\nyJvrdfBbRGISa2C0u3srcCXwc3f/BjAwfmVJT7j+jGFsa2rm+aW6T4aIHF2sgdFiZtcCNwJPB22Z\n8SlJesq5laUM65/Lb1+vCbsUEUkCsQbG3wFnAHe4+1ozGwH8Nn5lSU9ISzM+c9ow5tXs4N1NjWGX\nIyIJLqbAcPd33f3L7v6omfUDCtz9P+Jcm/SAT1UNITcrnbtfXRN2KSKS4GI9S+qvZtbXzIqBBcDd\nZvZf8S1NekJhbiafnjKUpxZtYsP2vWGXIyIJLNYuqUJ3bwQ+Djzk7qcBF8SvLOlJN50zgjSDe+es\nDbsUEUlgsQZGhpkNBD7F3w56S4oYWNiHKyYN5rF562loag67HBFJULEGxr8CLwDvufs8MxsJrIpf\nWdLTZpw7kv0t7Tz4Wk3YpYhIgor1oPfv3X2Cu98aPF/j7p+Ib2nSk04sK+Cik8u5f24NO/ceCLsc\nEUlAsR70rjCzJ8ysLng8bmYV8S5OetbXLhxN04FWfjNbZ0yJyAfF2iV1P/AUMCh4/F/QJilk9IAC\nLp84iAfm1lC3W4MSisj7xRoYpe5+v7u3Bo8HgNI41iUh+YcLRnGgrZ3//ut7YZciIgkm1sBoMLPP\nmFl68PgM0BDPwiQcI0ryuGpyBQ+/sV7XZYjI+8QaGJ8jckrtFmAzcBXw2TjVJCH7ygWVpKXBnc8v\nD7sUEUkgsZ4ltc7dL3f3Uncvc/ePAUc8S8rM7gsOkC/pZL6Z2V1mttrMFpvZ5Kh5F5vZimDe7V36\nRHLcBhX14ZapJ/DM4s28tXZ72OWISII4njvufe0o8x8ALj7C/OlAZfC4GfhvADNLB34ZzB8LXGtm\nY4+jTjkGM849gYGFOfzr00tpb/ewyxGRBHA8gWFHmunus4EjfT29gsgwI+7ubwBFwdXkU4DVwbUe\nB4DHgmWlB/XJSuf26SexZGOj7v0tIsDxBcbxfu0cDGyIel4btHXW3iEzu9nMqs2sur6+/jhLkmiX\nTxzE5KFF3Pn8crbv0cV8Ir3dEQPDzHabWWMHj91ErscInbvPdPcqd68qLdWZvt3JzPjBx8fTuK+F\nO55ZFnY5IhKyIwaGuxe4e98OHgXunnGc772RyL3BD6oI2jprlxCcNKAvt5w7kscX1DJ39bawyxGR\nEB1Pl9Txegq4IThb6nRgl7tvBuYBlWY2wsyygGuCZSUkXzq/kuH9c/n2E++wv6Ut7HJEJCRxCwwz\nexR4HRhtZrVmdpOZzTCzGcEizwJrgNXA3cAXANy9FbiNyOi4y4DfufvSeNUpR5eTmc4PrhzPuoa9\n/OiFFWGXIyIhOd5upU65+7VHme/AFzuZ9yyRQJEEceaJJXzm9KHcM2ct559UxpknloRdkoj0sDC7\npCTJfOeSsYwsyePrv1/Ern0tYZcjIj1MgSEx65OVzk+unkTd7ma++2SHF/CLSApTYEiXTBxSxJc/\nXMmTCzfxxwW6oE+kN1FgSJd9YdoJnDaimO88sYQVW3aHXY6I9BAFhnRZRnoaP//0h8jPyeDWh+fT\n1Nwadkki0gMUGHJMygpy+Pm1H6Jm2x6++YfFRE56E5FUpsCQY3b6yP5846KTeOadzczUfcBFUp4C\nQ47LLVNHcsn4Adz5/HJmvbs17HJEJI4UGHJc0tKMH39yEuMGFfKVx95m2ebGsEsSkThRYMhx65OV\nzt03VJGfk8HnH6ymfndz2CWJSBwoMKRbDCjM4Z4bTqVhTzM3PThPZ06JpCAFhnSb8RWF/OLaySzd\n1Mgtv62muVUj24qkEgWGdKsLxpbzH5+YwNzVDXz1fxfSpvuBi6SMuI1WK73XVadUsGPPAe54dhlF\nuUu442PjMDviLeBFJAkoMCQu/n7qSLbvPcB///U9stLT+N5Hxyo0RJKcAkPi5p8uGs2B1nbunbMW\nQKEhkuQUGBI3ZsY/XzoGQKEhkgIUGBJXh4eGu/O9j55MWppCQyTZKDAk7g6GRprB3a+uZde+Fn74\nyYlkpuskPZFkosCQHmFmfPuSMRTlZvHDF1awa18Lv7ruFPpkpYddmojESF/xpMeYGV8870R+cOV4\nXllZz2fufZNde3VvcJFkEdfAMLOLzWyFma02s9s7mP8NM1sYPJaYWZuZFQfzaszsnWBedTzrlJ71\n6dOG8stPT+ad2l184tevsb5hb9gliUgM4hYYZpYO/BKYDowFrjWzsdHLuPsP3X2Su08CvgW84u7b\noxY5L5hfFa86JRzTxw/koZumsK2pmSt+OYe31m4/+koiEqp47mFMAVa7+xp3PwA8BlxxhOWvBR6N\nYz2SYE4f2Z8nvnAW/XKzuO6eN3h8fm3YJYnIEcQzMAYDG6Ke1wZtH2BmucDFwONRzQ7MMrP5ZnZz\nZ29iZjebWbWZVdfX13dD2dKTRpTk8cQXzuLU4cX84+8X8e/PLaO1rT3sskSkA4ly0PujwNzDuqPO\nDrqqpgNfNLOpHa3o7jPdvcrdq0pLS3uiVulmhbmZPPi5KVx32lB+88oarr/3Ld1TQyQBxTMwNgJD\nop5XBG0duYbDuqPcfWPwsw54gkgXl6SozPQ07rhyPD/65EQWrN/BZT9/lfnrdFxDJJHEMzDmAZVm\nNsLMsoiEwlOHL2RmhcC5wJNRbXlmVnBwGvgIsCSOtUqCuOqUCp74wlnkZKZz9W/e4P65kavDRSR8\ncQsMd28FbgNeAJYBv3P3pWY2w8xmRC16JfBnd98T1VYOzDGzRcBbwDPu/ny8apXEMnZQX5667Wym\njS7jX/7vXW56sJptTeqiEgmbpdK3t6qqKq+u1iUbqcLdefC1Gn7w3HL65mTyo09OYNrosrDLEkkp\nZjY/1ksXEuWgt8gHmBmfPWsET912Fv3zsvjs/fP4/lNL2d+iW7+KhEGBIQnvpAF9efK2s/jsmcN5\n4LUaLrnrVaprdEBcpKcpMCQp5GSm8/3LT+a3N02huaWdT/7mdb7/1FL2HmgNuzSRXkOBIUnlnMpS\n/vzVqdxw+jAeeK2Gi346m7mrt4VdlkivoMCQpJOXncG/XDGO391yBhlpaVx3z5v8w2NvU9e4P+zS\nRFKaAkOS1pQRxTz3lXP48vkn8uw7Wzj/x69w75y1GlpEJE4UGJLUcjLT+dpHRvPnr07llGH9+Len\n3+XSu+bw5pqGsEsTSTkKDEkJw0vyeODvTuU3159CU3MrV898g1t+W82a+qawSxNJGbpFq6QMM+Oi\nkwcwtbKUu19dw29eeY+Xls3mutOG8uUPV9I/PzvsEkWSmq70lpRVv7uZn85ayWPzNpCbmc6MaSfw\nubNG6D7iIlG6cqW3AkNS3uq6Ju58bjmzlm2lJD+bGeeO5DOnDyMnU8EhosAQ6UB1zXZ+Mmslc1c3\nUFqQza3nnsCnTxuq4JBeTYEhcgRvrmngJ7NW8saa7ZQVZHPrtBO4+tQh5GbpkJ70PgoMkRi8/l4D\nP3lxJW/VbKdfbiY3nDGcG88cTnFeVtilifQYBYZIF1TXbOfXr7zHrGV15GSmcXXVED5/zkiGFOeG\nXZpI3CkwRI7Bqq27+c3sNTy5cCPtDtPHDeCzZw7nlGH9MLOwyxOJCwWGyHHYvGsf981Zy2PzNrB7\nfysnD+rLjWcO5/KJg3SAXFKOAkOkG+xpbuVPCzfy4Gs1rNzaRL/cTK4+dSifOX0oFf3UXSWpQYEh\n0o3cndfXNPDQa+v487tbAJg6qpRrTh3C+SeVk5WhEXYkeSkwROJk4859PPrmev4wv5Ytjfspyc/i\n45Mr+FTVEE4syw+7PJEuU2CIxFlbuzN7ZT2PzVvPS8vqaG13qob141OnDmH6uAEU5GSGXaJITBIm\nMMzsYuBnQDpwj7vfedj8acCTwNqg6Y/u/q+xrNsRBYaEoX53M39cUMv/Vm9gTf0esjPSuGBMOVdM\nGsS00WXqspKElhCBYWbpwErgQqAWmAdc6+7vRi0zDfi6u1/W1XU7osCQMLk7b2/YyZNvb+TpxZtp\n2HOAotxMLhk/kI9NGkzVsH6kpen0XEksXQmMeI6FMAVY7e5rgqIeA64AjvhPvxvWFQmFmTF5aD8m\nD+3HP182ljmrt/Hk2xt5YsFGHnlzPYOL+nDphIFMHzeASUOKdG2HJJ14BsZgYEPU81rgtA6WO9PM\nFgMbiextLO3CupjZzcDNAEOHDu2GskWOX2Z6GueNLuO80WXsaW7lxXe38uTCjdw/dy0zZ69hYGEO\nF48bwCXjB3LKUO15SHIIe7S1BcBQd28ys0uAPwGVXXkBd58JzIRIl1T3lyhyfPKyM/jYhwbzsQ8N\nZte+Fl5atpXnlmzh4TfXc//cGkoLsrn45AFMHzeAKSOKyUjXMQ9JTPEMjI3AkKjnFUHbIe7eGDX9\nrJn9ysxKYllXJBkV9snk45Mr+PjkCpqaW3l5eR3PL9nMH+bX8ts31lHYJ5Npo0s5/6Qypo0qozBX\nZ1tJ4ohnYMwDKs1sBJF/9tcAn45ewMwGAFvd3c1sCpF7jDcAO4+2rkiyy8/O4PKJg7h84iD2HWjj\nlZV1zFpWx1+W1/Hkwk2kpxlVw/pxwZhyPjymjJGlus5DwhXv02ovAX5K5NTY+9z9DjObAeDuvzaz\n24BbgVZgH/A1d3+ts3WP9n46S0pSQVu7s6h2Jy8t28pLy+pYvmU3ACNK8jj/pDLOHVXKlBHFGtdK\nukVCnFYbBgWGpKLaHXt5eXkdLy2r4/X3GjjQ1k5WRhqnjShmamUp54wqYXR5gc66kmOiwBBJUfsO\ntPHm2gZeXbWN2SvrWVXXBEBpQTbnVJYwtbKUsytLKMnPDrlSSRaJch2GiHSzPlnpTBtdxrTRZUBk\nKPZXV23j1VXb+MvyOv64IHJuyEkDCjh9ZH9OH9mf00YU0093EZRuoD0MkRTR3u4s3dTI7FX1vP5e\nA9XrtrO/pR0zOGlAX04fWXwoQIpyFSASoS4pEeFAazuLa3fy+nsNvLG2geqaHTS3vj9AThtRzORh\n/SgryAm7XAmJAkNEPqC5tY3FtbsiAbKmgfnrIgECMLQ4l6ph/ThleD+qhhVTWZavq897CQWGiBxV\nc2sbSzY2Mn/dduav28H8dTvY1nQAgIKcDCYP7XcoRCYNKSI3S4c8U5EOeovIUWVnpHPKsH6cMqwf\nEBltd117wWrEAAAMNUlEQVTDXqqD8Ji/bjs/frEegPQ0Y1R5AZOGFDKxoogJFUWMKs/XMCa9jPYw\nRKRTu/a2sGDDDubX7GBR7U4W1+5i174WAHIy0xg3qJCJQ4qYUFHIpCFFDC3O1fUgSUZdUiISFwf3\nQhbV7mTRhl0srt3Jkk272N8SORZSlJvJhIoixg3qy8mDCjl5UF+GFufqeEgCU5eUiMSFmTG8JI/h\nJXlcMWkwAK1t7azc2hTsgexk4YZdzJy9htb2yJfR/OwMxgwsYOzASIiMHdSXyvJ8sjM0tEmy0R6G\niHS75tY2Vm1tYummXby7qZGlmxpZtrmRPQfaAMhIMyrLD4ZIX04aUMCoAQW6Qj0E2sMQkVBlZ6Qz\nbnAh4wYXHmprb3fWbd/7vhB5ZWU9jy+oPbRM/7wsRpUXMHpAQfAzn8ryAvrmaJj3RKDAEJEekZZm\njCjJY0RJHpdNGHSovW73flZuaWLF1t2s3LKbFVt38/vqDYf2RgAGFeZQGRUko8rzGVmaT362/oX1\nJG1tEQlVWUEOZQU5nF1Zcqitvd3ZuHMfK7dGAmTV1iZWbNnN62saOBBcbAhQ3jebkSX5nFCWx8iS\nfEaW5nFCaT6Di/roQHscKDBEJOGkpRlDinMZUpzLh8eUH2pvbWunpmEvq+uaWLOtiffq9rBmWxNP\nLdxE4/7WQ8tlZ6QxoiQSHieU5jGyNJ8RJXkM75+nuxgeBwWGiCSNjPQ0TizL58Sy99990N1p2HOA\n9+qaWLNtD2vqm3ivfg9LN+3iuSWbaY86t6ewTybD+ucytDiX4f3zGNo/l2HFuQzrn0dZQbb2TI5A\ngSEiSc/MKMnPpiQ/m9NG9n/fvObWNtY37GXNtj2sb9jLuu17WNewl8W1u3huyRbaotIkJzONocW5\nDC3OY1j/XIb3z6WiOJeKoj4M7ten1w+P0rs/vYikvOyMdCrLC6gsL/jAvJa2djbt3Me6hr2s276X\n9Q17qGnYy/qGvcxZXX/ogsSD+uVmMrhfHyqKchncrw+DgyAZXNSHIf1y6dsnI6WvdFdgiEivlZme\nxrD+eQzrn/eBee5O/e5mNuzYR+2OvWzcuY+NO/ZRu2Mfq+ubeGVlPfta2t63Tn52xvtCZEBhDgML\ncxjQN4cBhZFHMu+lJG/lIiJxZGaU9c2hrG/OoQEao7k72/ccOBQkG3dGwqQ2mK6u2f6+A/EH9c3J\nCMKjDwP6Zgc/I8FSHvwsys1MyD0VBYaIyDEwM/rnZ9M/P5sJFUUdLrP3QCtbdu1nS+P+9/8Mppdv\nbqS+qZnDB9zIzkijrG92cMpxNqUF2ZQVRJ6XHnzeN5v+edmk9+BB+rgGhpldDPwMSAfucfc7D5t/\nHfBNwIDdwK3uviiYVxO0tQGtsV66LiKSKHKzMhhZGrnIsDMtbe3U725m8679bI0Klq2N+6lrbGZV\nXRNzV2/rcG8lzaB/fjbD++fy+xlnxvOjAHEMDDNLB34JXAjUAvPM7Cl3fzdqsbXAue6+w8ymAzOB\n06Lmn+fu2+JVo4hI2DLT0xhU1IdBRX2OuNz+ljbqdzdTt7uZ+t3N1O/ef+h5T4nnHsYUYLW7rwEw\ns8eAK4BDgeHur0Ut/wZQEcd6RESSVk5m+qGLGcMSz9tlDQY2RD2vDdo6cxPwXNRzB2aZ2Xwzu7mz\nlczsZjOrNrPq+vr64ypYREQ6lxAHvc3sPCKBcXZU89nuvtHMyoAXzWy5u88+fF13n0mkK4uqqqrU\nGatdRCTBxHMPYyMwJOp5RdD2PmY2AbgHuMLdGw62u/vG4Gcd8ASRLi4REQlJPANjHlBpZiPMLAu4\nBngqegEzGwr8Ebje3VdGteeZWcHBaeAjwJI41ioiIkcRty4pd281s9uAF4icVnufuy81sxnB/F8D\n3wX6A78KLlI5ePpsOfBE0JYBPOLuz8erVhEROTrdolVEpBfryi1a49klJSIiKUSBISIiMUmpLikz\nqwfWHePqJUAiXlWuurpGdXWN6uqaVKxrmLuXxrJgSgXG8TCz6kQcr0p1dY3q6hrV1TW9vS51SYmI\nSEwUGCIiEhMFxt/MDLuATqiurlFdXaO6uqZX16VjGCIiEhPtYYiISEwUGCIiEpNeHxhmdrGZrTCz\n1WZ2ewjvX2Nm75jZQjOrDtqKzexFM1sV/OwXtfy3glpXmNlF3VjHfWZWZ2ZLotq6XIeZnRJ8ntVm\ndpcd553sO6nr+2a2MdhmC83skhDqGmJmfzGzd81sqZl9JWgPdZsdoa5Qt5mZ5ZjZW2a2KKjrX4L2\nsLdXZ3WF/jsWvGa6mb1tZk8Hz8P9m3T3XvsgMijie8BIIAtYBIzt4RpqgJLD2v4TuD2Yvh34j2B6\nbFBjNjAiqD29m+qYCkwGlhxPHcBbwOlE7tP+HDA9DnV9H/h6B8v2ZF0DgcnBdAGwMnj/ULfZEeoK\ndZsFr5EfTGcCbwavHfb26qyu0H/Hgtf8GvAI8HQi/E329j2MQ7eRdfcDwMHbyIbtCuDBYPpB4GNR\n7Y+5e7O7rwVW0033CfHIzam2H08dZjYQ6Ovub3jkN/WhqHW6s67O9GRdm919QTC9G1hG5I6SoW6z\nI9TVmZ6qy929KXiaGTyc8LdXZ3V1psd+x8ysAriUyP2Cot8/tO3V2wOjq7eRjYeObkVb7u6bg+kt\nRIZ7h56vt6t1DA6me6K+L5nZ4qDL6uBueSh1mdlw4ENEvp0mzDY7rC4IeZsF3SsLgTrgRXdPiO3V\nSV0Q/u/YT4F/Atqj2kLdXr09MBLB2e4+CZgOfNHMpkbPDL4VhH7uc6LUEfhvIt2Ik4DNwI/DKsTM\n8oHHgX9w98boeWFusw7qCn2buXtb8LteQeTb77jD5oeyvTqpK9TtZWaXAXXuPr+zZcLYXr09MGK6\njWw8ece3ot0a7EoS/KwLFu/pertax8ZgOq71ufvW4I+8Hbibv3XL9WhdZpZJ5J/yw+7+x6A59G3W\nUV2Jss2CWnYCfwEuJgG2V0d1JcD2Ogu43MxqiHSVn29m/0PI26u3B8ZRbyMbT9b5rWifAm4MFrsR\neDKYfgq4xsyyzWwEUEnkgFa8dKmOYFe50cxOD87EuCFqnW5z8A8mcCV/u31vj9UVvM69wDJ3/6+o\nWaFus87qCnubmVmpmRUF032AC4HlhL+9Oqwr7O3l7t9y9wp3H07k/9LL7v4Zwv6bPNaj5anyAC4h\ncibJe8B3evi9RxI5s2ERsPTg+xO5be1LwCpgFlActc53glpX0A1nYUS97qNEdr1biPRz3nQsdQBV\nRP643gN+QTCaQDfX9VvgHWBx8IcyMIS6zibSHbAYWBg8Lgl7mx2hrlC3GTABeDt4/yXAd4/1d72H\n6gr9dyzqdafxt7OkQt1eGhpERERi0tu7pEREJEYKDBERiYkCQ0REYqLAEBGRmCgwREQkJgoMkYCZ\nNQU/h5vZp7v5tb992PPXuvP1RXqCAkPkg4YDXQoMM8s4yiLvCwx3P7OLNYmEToEh8kF3AudY5D4I\nXw0Gp/uhmc0LBqO7BcDMppnZq2b2FPBu0PanYCDJpQcHkzSzO4E+wes9HLQd3Jux4LWXWOSeBVdH\nvfZfzewPZrbczB4OrtTFzO60yP0uFpvZj3p860ivdbRvRSK90e1E7oVwGUDwj3+Xu59qZtnAXDP7\nc7DsZGCcR4aUBvicu28PhpmYZ2aPu/vtZnabRwa4O9zHiQxwNxEoCdaZHcz7EHAysAmYC5xlZsuI\nDFVxkrv7wWEtRHqC9jBEju4jwA0WGQL7TSLDM1QG896KCguAL5vZIuANIoPBVXJkZwOPemSgu63A\nK8CpUa9d65EB8BYS6SrbBewH7jWzjwN7j/vTicRIgSFydAZ8yd0nBY8R7n5wD2PPoYXMpgEXAGe4\n+0QiYxTlHMf7NkdNtwEZ7t5KZOTUPwCXAc8fx+uLdIkCQ+SDdhO5velBLwC3BsOGY2ajgtGFD1cI\n7HD3vWZ2EpHbYh7UcnD9w7wKXB0cJyklckvaTkcgtsh9Lgrd/Vngq0S6skR6hI5hiHzQYqAt6Fp6\nAPgZke6gBcGB53o6vs3l88CM4DjDCiLdUgfNBBab2QJ3vy6q/QngDCIjFjvwT+6+JQicjhQAT5pZ\nDpE9n68d20cU6TqNVisiIjFRl5SIiMREgSEiIjFRYIiISEwUGCIiEhMFhoiIxESBISIiMVFgiIhI\nTP4/QsdovYcuFO4AAAAASUVORK5CYII=\n", 482 | "text/plain": [ 483 | "" 484 | ] 485 | }, 486 | "metadata": {}, 487 | "output_type": "display_data" 488 | } 489 | ], 490 | "source": [ 491 | "#plot loss graph\n", 492 | "plt.plot(list(range(1,len(J))),J[1:])\n", 493 | "plt.xlabel('Iterations')\n", 494 | "plt.ylabel('Loss')\n", 495 | "plt.title('Iterations VS Loss')\n", 496 | "plt.show()" 497 | ] 498 | } 499 | ], 500 | "metadata": { 501 | "kernelspec": { 502 | "display_name": "Python 3", 503 | "language": "python", 504 | "name": "python3" 505 | }, 506 | "language_info": { 507 | "codemirror_mode": { 508 | "name": "ipython", 509 | "version": 3 510 | }, 511 | "file_extension": ".py", 512 | "mimetype": "text/x-python", 513 | "name": "python", 514 | "nbconvert_exporter": "python", 515 | "pygments_lexer": "ipython3", 516 | "version": "3.6.1" 517 | } 518 | }, 519 | "nbformat": 4, 520 | "nbformat_minor": 2 521 | } 522 | -------------------------------------------------------------------------------- /Neural Network/network.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navjindervirdee/neural-networks/6c1a6e13ffc38110863bae530aa4f148d15510ed/Neural Network/network.JPG -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Neural-Networks 2 | ### **Implemented Convolutional Neural Network, LSTM Neural Network and Neural Network From Scratch.** 3 | 4 | ## **Convolutional Neural Network** 5 | 6 | In machine learning, a convolutional neural network (CNN, or ConvNet) is a class of deep, feed-forward artificial neural networks that has successfully been applied to analyzing visual imagery. 7 | 8 | CNNs use a variation of multilayer perceptrons designed to require minimal preprocessing. They are also known as shift invariant or space invariant artificial neural networks (SIANN), based on their shared-weights architecture and translation invariance characteristics. 9 | 10 | CNNs use relatively little pre-processing compared to other image classification algorithms. This means that the network learns the filters that in traditional algorithms were hand-engineered. This independence from prior knowledge and human effort in feature design is a major advantage. 11 | 12 | They have applications in image and video recognition, recommender systems and natural language processing. 13 | 14 | --> **Implemented Convolutional Neural Network from SCRATCH!** 15 | * **Dataset** : 42,000 Images of Hand Written Digtis (0-9). Each digit has around 3500-4000 Images. 16 | * **Network Architecture** : 17 | ![](https://github.com/navjindervirdee/neural-networks/blob/master/Convolutional%20Neural%20Network/Forward.JPG?raw=true) 18 | 19 | * **Layers INFO** : 20 | 1. 2 Convolution Layers 21 | 2. 2 Fully Connected Layers 22 | 23 | * **Filter Size** : 24 | 1. 3x3x1x4 Filters used in Conv Layer 1 25 | 2. 3x3x4x16 Filters used in Conv Layee 2 26 | 27 | * **Actvation Function** : 28 | 1. Sigmoid 29 | 2. Softmax 30 | 31 | * **Data Distribution** : 32 | 1. Train Set = 30,000 Images 33 | 2. Cross Valid Set = 6000 Images 34 | 3. Test Set = 6000 Images 35 | 36 | * **Functions Implemented using Numpy** : 37 | 1. Initialize Parameters 38 | 2. Forward Propagation 39 | 3. Cal Loss And Accuracy 40 | 4. Backward Propagation 41 | 5. Update Parameters 42 | 43 | * **Libraries Used** : 44 | 1. Numpy 45 | 2. Pandas 46 | 3. Matplot 47 | 48 | * **Results** : 49 | 1. Train Accuracy : 87% 50 | 2. Valid Accuracy : 87% 51 | 3. Test Accuracy : 87% 52 | 53 | * Learned about the implementation details of forward propagation and backward propagation algorithms used in CNN's and how different activation functions affect the training accuracy. 54 | 55 | ## **LSTM Neural Network**: 56 | 57 | Long short-term memory (LSTM) units (or blocks) are a building unit for layers of a recurrent neural network (RNN). A RNN composed of LSTM units is often called an LSTM network. A common LSTM unit is composed of a cell, an input gate, an output gate and a forget gate. The cell is responsible for "remembering" values over arbitrary time intervals; hence the word "memory" in LSTM. Each of the three gates can be thought of as a "conventional" artificial neuron, as in a multi-layer (or feedforward) neural network: that is, they compute an activation (using an activation function) of a weighted sum. Intuitively, they can be thought as regulators of the flow of values that goes through the connections of the LSTM; hence the denotation "gate". There are connections between these gates and the cell. 58 | 59 | The expression long short-term refers to the fact that LSTM is a model for the short-term memory which can last for a long period of time. An LSTM is well-suited to classify, process and predict time series given time lags of unknown size and duration between important events. LSTMs were developed to deal with the exploding and vanishing gradient problem when training traditional RNNs. 60 | 61 | --> **Implemented LSTM Neural Network from SCRATCH!** 62 | * **Dataset** : US Baby's First Names 63 | * **Network Architecture** : 64 | ![](https://github.com/navjindervirdee/neural-networks/blob/master/Recurrent%20Neural%20Network/LSTMForward.JPG?raw=true) 65 | 66 | * **Actvation Function** : 67 | 1. Sigmoid 68 | 2. Softmax 69 | 3. Tanh 70 | 71 | * **Functions Implemented using Numpy** : 72 | 1. Initialize Parameters 73 | 2. Forward Propagation 74 | 3. Cal Loss And Accuracy and Perplexity 75 | 4. Backward Propagation 76 | 5. Update Parameters 77 | 6. Update Embeddings 78 | 79 | * **Libraries Used** : 80 | 1. Numpy 81 | 2. Pandas 82 | 3. Matplot 83 | 84 | * **Results** : 85 | 1. Perplexity : 1.05 86 | 2. Accuracy : 85% 87 | 3. **Awesome names predicted like -- Donald, Margaret, Harris, Isabella, William** 88 | * Learned about the implementation details of forward propagation and backward propagation through time algorithms used in RNN's and how different activation functions affect the training accuracy. 89 | 90 | ## **Neural Network**: 91 | 92 | Artificial neural networks (ANNs) or connectionist systems are computing systems vaguely inspired by the biological neural networks that constitute animal brains. Such systems "learn" (i.e. progressively improve performance on) tasks by considering examples, generally without task-specific programming. 93 | 94 | An ANN is based on a collection of connected units or nodes called artificial neurons. Each connection (a simplified version of a synapse) between artificial neurons can transmit a signal from one to another. The artificial neuron that receives the signal can process it and then signal artificial neurons connected to it. 95 | 96 | **Implemented 1-hidden layer neural network for the classification of Iris Dataset. Learned about the detailed implementation of forward and backward propagation algorithms used in the neural network. Learned ways to easily compute gradients for any function.** 97 | 98 | * **Dataset** - Iris Dataset 99 | * **Network Architecture** : 100 | ![](https://github.com/navjindervirdee/neural-networks/blob/master/Neural%20Network/network.JPG?raw=true) 101 | 102 | * **Actvation Function** : 103 | 1. Sigmoid 104 | 2. Softmax 105 | 106 | * **Functions Implemented using Numpy** : 107 | 1. Initialize Parameters 108 | 2. Forward Propagation 109 | 3. Cal Loss And Accuracy 110 | 4. Backward Propagation 111 | 5. Update Parameters 112 | -------------------------------------------------------------------------------- /Recurrent Neural Network/LSTM.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navjindervirdee/neural-networks/6c1a6e13ffc38110863bae530aa4f148d15510ed/Recurrent Neural Network/LSTM.JPG -------------------------------------------------------------------------------- /Recurrent Neural Network/LSTMForward.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navjindervirdee/neural-networks/6c1a6e13ffc38110863bae530aa4f148d15510ed/Recurrent Neural Network/LSTMForward.JPG -------------------------------------------------------------------------------- /Recurrent Neural Network/RecurrentNeuralNet(LSTM).ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "collapsed": true 7 | }, 8 | "source": [ 9 | "# Implementing LSTM Neural Network from Scratch\n", 10 | "**Dataset = Us Baby's First Names
\n", 11 | "Link = https://www.kaggle.com/chr7stos/us-names-babies-and-presindent-names/data**" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "### Import Required Libraries" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": { 25 | "collapsed": true 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "import numpy as np #for maths\n", 30 | "import pandas as pd #for data manipulation\n", 31 | "import matplotlib.pyplot as plt #for visualization" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "### Load the data" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 2, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "#data \n", 48 | "path = r'E:\\Study n Work\\Neural Networks\\Recurrent Neural Network\\Data\\Names\\NationalNames.csv'\n", 49 | "data = pd.read_csv(path)\n", 50 | "\n", 51 | "#get names from the dataset\n", 52 | "data['Name'] = data['Name']\n", 53 | "\n", 54 | "#get first 10000 names\n", 55 | "data = np.array(data['Name'][:10000]).reshape(-1,1)\n", 56 | "\n", 57 | "#covert the names to lowee case\n", 58 | "data = [x.lower() for x in data[:,0]]\n", 59 | "\n", 60 | "data = np.array(data).reshape(-1,1)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 3, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "name": "stdout", 70 | "output_type": "stream", 71 | "text": [ 72 | "Data Shape = (10000, 1)\n", 73 | "\n", 74 | "Lets see some names : \n", 75 | "[['anna']\n", 76 | " ['emma']\n", 77 | " ['elizabeth']\n", 78 | " ['minnie']\n", 79 | " ['margaret']\n", 80 | " ['ida']\n", 81 | " ['alice']\n", 82 | " ['bertha']\n", 83 | " ['sarah']]\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "print(\"Data Shape = {}\".format(data.shape))\n", 89 | "print()\n", 90 | "print(\"Lets see some names : \")\n", 91 | "print(data[1:10])" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Transform the names to equal length by adding -- > ('.') dots" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 4, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "#to store the transform data\n", 108 | "transform_data = np.copy(data)\n", 109 | "\n", 110 | "#find the max length name\n", 111 | "max_length = 0\n", 112 | "for index in range(len(data)):\n", 113 | " max_length = max(max_length,len(data[index,0]))\n", 114 | "\n", 115 | "#make every name of max length by adding '.'\n", 116 | "for index in range(len(data)):\n", 117 | " length = (max_length - len(data[index,0]))\n", 118 | " string = '.'*length\n", 119 | " transform_data[index,0] = ''.join([transform_data[index,0],string])" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 5, 125 | "metadata": {}, 126 | "outputs": [ 127 | { 128 | "name": "stdout", 129 | "output_type": "stream", 130 | "text": [ 131 | "Transformed Data\n", 132 | "[['anna........']\n", 133 | " ['emma........']\n", 134 | " ['elizabeth...']\n", 135 | " ['minnie......']\n", 136 | " ['margaret....']\n", 137 | " ['ida.........']\n", 138 | " ['alice.......']\n", 139 | " ['bertha......']\n", 140 | " ['sarah.......']]\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "print(\"Transformed Data\")\n", 146 | "print(transform_data[1:10])" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "### Lets Make Vocabulary" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 6, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "Vocab size = 27\n", 166 | "Vocab = {'y', 'c', 'd', 'x', 'e', 'o', 'g', 'f', 'w', 'i', 'z', 'u', 'l', 'q', '.', 's', 'm', 'j', 'a', 'k', 'b', 'v', 'n', 'p', 't', 'r', 'h'}\n" 167 | ] 168 | } 169 | ], 170 | "source": [ 171 | "#to store the vocabulary\n", 172 | "vocab = list()\n", 173 | "for name in transform_data[:,0]:\n", 174 | " vocab.extend(list(name))\n", 175 | "\n", 176 | "vocab = set(vocab)\n", 177 | "vocab_size = len(vocab)\n", 178 | "\n", 179 | "print(\"Vocab size = {}\".format(len(vocab)))\n", 180 | "print(\"Vocab = {}\".format(vocab))" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "### Map characters to ids and ids to characters" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 7, 193 | "metadata": {}, 194 | "outputs": [ 195 | { 196 | "name": "stdout", 197 | "output_type": "stream", 198 | "text": [ 199 | "a-18, 22-n\n" 200 | ] 201 | } 202 | ], 203 | "source": [ 204 | "#map char to id and id to chars\n", 205 | "char_id = dict()\n", 206 | "id_char = dict()\n", 207 | "\n", 208 | "for i,char in enumerate(vocab):\n", 209 | " char_id[char] = i\n", 210 | " id_char[i] = char\n", 211 | "\n", 212 | "print('a-{}, 22-{}'.format(char_id['a'],id_char[22]))" 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "### Make the Train dataset\n", 220 | "\n", 221 | "Example - names - [['mary.'], ['anna.']
\n", 222 | "m - [0,0,0,1,0,0]
\n", 223 | "a - [0,0,1,0,0,0]
\n", 224 | "r - [0,1,0,0,0,0]
\n", 225 | "y - [0,0,0,0,1,0]
\n", 226 | "**.** - [1,0,0,0,0,0]
\n", 227 | "\n", 228 | "'mary.' = [[0,0,0,1,0,0], [0,0,1,0,0,0], [0,1,0,0,0,0], [0,0,0,0,1,0], [1,0,0,0,0,0]]
\n", 229 | "'anna.' = [[0,0,1,0,0,0], [0,0,0,0,0,1], [0,0,0,0,0,1], [0,0,1,0,0,0], [1,0,0,0,0,0]]
\n", 230 | "\n", 231 | "batch_dataset = [ [[0,0,0,1,0,0],[0,0,1,0,0,0]] , [[0,0,1,0,0,0], [0,0,0,0,0,1]], [[0,1,0,0,0,0], [0,0,0,0,0,1]], [[0,0,0,0,1,0], [0,0,1,0,0,0]] , [ [1,0,0,0,0,0], [1,0,0,0,0,0]] ]" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 8, 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "# list of batches of size = 20\n", 241 | "train_dataset = []\n", 242 | "\n", 243 | "batch_size = 20\n", 244 | "\n", 245 | "#split the trasnform data into batches of 20\n", 246 | "for i in range(len(transform_data)-batch_size+1):\n", 247 | " start = i*batch_size\n", 248 | " end = start+batch_size\n", 249 | " \n", 250 | " #batch data\n", 251 | " batch_data = transform_data[start:end]\n", 252 | " \n", 253 | " if(len(batch_data)!=batch_size):\n", 254 | " break\n", 255 | " \n", 256 | " #convert each char of each name of batch data into one hot encoding\n", 257 | " char_list = []\n", 258 | " for k in range(len(batch_data[0][0])):\n", 259 | " batch_dataset = np.zeros([batch_size,len(vocab)])\n", 260 | " for j in range(batch_size):\n", 261 | " name = batch_data[j][0]\n", 262 | " char_index = char_id[name[k]]\n", 263 | " batch_dataset[j,char_index] = 1.0\n", 264 | " \n", 265 | " #store the ith char's one hot representation of each name in batch_data\n", 266 | " char_list.append(batch_dataset)\n", 267 | " \n", 268 | " #store each char's of every name in batch dataset into train_dataset\n", 269 | " train_dataset.append(char_list)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "metadata": {}, 275 | "source": [ 276 | "### Hyperparameters" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 9, 282 | "metadata": { 283 | "collapsed": true 284 | }, 285 | "outputs": [], 286 | "source": [ 287 | "#number of input units or embedding size\n", 288 | "input_units = 100\n", 289 | "\n", 290 | "#number of hidden neurons\n", 291 | "hidden_units = 256\n", 292 | "\n", 293 | "#number of output units i.e vocab size\n", 294 | "output_units = vocab_size\n", 295 | "\n", 296 | "#learning rate\n", 297 | "learning_rate = 0.005\n", 298 | "\n", 299 | "#beta1 for V parameters used in Adam Optimizer\n", 300 | "beta1 = 0.90\n", 301 | "\n", 302 | "#beta2 for S parameters used in Adam Optimizer\n", 303 | "beta2 = 0.99" 304 | ] 305 | }, 306 | { 307 | "cell_type": "markdown", 308 | "metadata": {}, 309 | "source": [ 310 | "### Activation Functions\n", 311 | "* **Sigmoid = 1/(1+exp(-X))**
\n", 312 | "* **Tanh = (exp(X) - exp(-X)) / (exp(X) + exp(X)) **\n", 313 | "* **Softmax = exp(X)/(sum(exp(X),1))**\n", 314 | "
\n", 315 | "![](https://i.stack.imgur.com/o0JA0.png)\n", 316 | "\n" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 10, 322 | "metadata": { 323 | "collapsed": true 324 | }, 325 | "outputs": [], 326 | "source": [ 327 | "#Activation Functions\n", 328 | "#sigmoid\n", 329 | "def sigmoid(X):\n", 330 | " return 1/(1+np.exp(-X))\n", 331 | "\n", 332 | "#tanh activation\n", 333 | "def tanh_activation(X):\n", 334 | " return np.tanh(X)\n", 335 | "\n", 336 | "#softmax activation\n", 337 | "def softmax(X):\n", 338 | " exp_X = np.exp(X)\n", 339 | " exp_X_sum = np.sum(exp_X,axis=1).reshape(-1,1)\n", 340 | " exp_X = exp_X/exp_X_sum\n", 341 | " return exp_X\n", 342 | "\n", 343 | "#derivative of tanh\n", 344 | "def tanh_derivative(X):\n", 345 | " return 1-(X**2)" 346 | ] 347 | }, 348 | { 349 | "cell_type": "markdown", 350 | "metadata": {}, 351 | "source": [ 352 | "### Initialize Parameters\n", 353 | "Embeddings Size = 100
\n", 354 | "Hidden Units = 256
\n", 355 | "Total INPUT Weights = 100 + 256 = 356
\n", 356 | "\n", 357 | "* **LSTM CELL Weights **
\n", 358 | " * Forget Gate Weights = {356,256}
\n", 359 | " * Input Gate Weights = {356,256}
\n", 360 | " * Gate Gate Weights = {356,256}
\n", 361 | " * Output Gate Weights = {356,256}
\n", 362 | "
\n", 363 | "\n", 364 | "* **Output CELL Weights **
\n", 365 | " * Output Weights = {256,27}
\n", 366 | "\n", 367 | "Store these weights in parameters dictionary!" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": 11, 373 | "metadata": { 374 | "collapsed": true 375 | }, 376 | "outputs": [], 377 | "source": [ 378 | "#initialize parameters\n", 379 | "def initialize_parameters():\n", 380 | " #initialize the parameters with 0 mean and 0.01 standard deviation\n", 381 | " mean = 0\n", 382 | " std = 0.01\n", 383 | " \n", 384 | " #lstm cell weights\n", 385 | " forget_gate_weights = np.random.normal(mean,std,(input_units+hidden_units,hidden_units))\n", 386 | " input_gate_weights = np.random.normal(mean,std,(input_units+hidden_units,hidden_units))\n", 387 | " output_gate_weights = np.random.normal(mean,std,(input_units+hidden_units,hidden_units))\n", 388 | " gate_gate_weights = np.random.normal(mean,std,(input_units+hidden_units,hidden_units))\n", 389 | " \n", 390 | " #hidden to output weights (output cell)\n", 391 | " hidden_output_weights = np.random.normal(mean,std,(hidden_units,output_units))\n", 392 | " \n", 393 | " parameters = dict()\n", 394 | " parameters['fgw'] = forget_gate_weights\n", 395 | " parameters['igw'] = input_gate_weights\n", 396 | " parameters['ogw'] = output_gate_weights\n", 397 | " parameters['ggw'] = gate_gate_weights\n", 398 | " parameters['how'] = hidden_output_weights\n", 399 | " \n", 400 | " return parameters\n" 401 | ] 402 | }, 403 | { 404 | "cell_type": "markdown", 405 | "metadata": {}, 406 | "source": [ 407 | "### LSTM CELL\n", 408 | "\n", 409 | "![](https://github.com/navjindervirdee/neural-networks/blob/master/Recurrent%20Neural%20Network/LSTM.JPG?raw=true)
\n", 410 | "\n", 411 | "**Equations**\n", 412 | "* fa = sigmoid(Wf x [xt,at-1])
\n", 413 | "* ia = sigmoid(Wi x [xt,at-1])
\n", 414 | "* ga = tanh(Wg x [xt,at-1]) \n", 415 | "* oa = sigmoid(Wo x [xt,at-1]) \n", 416 | "* ct = (fa x ct-1) + (ia x ga) \n", 417 | "* at = oa x tanh(ct) " 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": 12, 423 | "metadata": { 424 | "collapsed": true 425 | }, 426 | "outputs": [], 427 | "source": [ 428 | "#single lstm cell\n", 429 | "def lstm_cell(batch_dataset, prev_activation_matrix, prev_cell_matrix, parameters):\n", 430 | " #get parameters\n", 431 | " fgw = parameters['fgw']\n", 432 | " igw = parameters['igw']\n", 433 | " ogw = parameters['ogw']\n", 434 | " ggw = parameters['ggw']\n", 435 | " \n", 436 | " #concat batch data and prev_activation matrix\n", 437 | " concat_dataset = np.concatenate((batch_dataset,prev_activation_matrix),axis=1)\n", 438 | " \n", 439 | " #forget gate activations\n", 440 | " fa = np.matmul(concat_dataset,fgw)\n", 441 | " fa = sigmoid(fa)\n", 442 | " \n", 443 | " #input gate activations\n", 444 | " ia = np.matmul(concat_dataset,igw)\n", 445 | " ia = sigmoid(ia)\n", 446 | " \n", 447 | " #output gate activations\n", 448 | " oa = np.matmul(concat_dataset,ogw)\n", 449 | " oa = sigmoid(oa)\n", 450 | " \n", 451 | " #gate gate activations\n", 452 | " ga = np.matmul(concat_dataset,ggw)\n", 453 | " ga = tanh_activation(ga)\n", 454 | " \n", 455 | " #new cell memory matrix\n", 456 | " cell_memory_matrix = np.multiply(fa,prev_cell_matrix) + np.multiply(ia,ga)\n", 457 | " \n", 458 | " #current activation matrix\n", 459 | " activation_matrix = np.multiply(oa, tanh_activation(cell_memory_matrix))\n", 460 | " \n", 461 | " #lets store the activations to be used in back prop\n", 462 | " lstm_activations = dict()\n", 463 | " lstm_activations['fa'] = fa\n", 464 | " lstm_activations['ia'] = ia\n", 465 | " lstm_activations['oa'] = oa\n", 466 | " lstm_activations['ga'] = ga\n", 467 | " \n", 468 | " return lstm_activations,cell_memory_matrix,activation_matrix" 469 | ] 470 | }, 471 | { 472 | "cell_type": "markdown", 473 | "metadata": {}, 474 | "source": [ 475 | "### Output Cell\n", 476 | "\n", 477 | "Equations \n", 478 | "* ot = W x at\n", 479 | "* ot = softmax(ot)" 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": 13, 485 | "metadata": { 486 | "collapsed": true 487 | }, 488 | "outputs": [], 489 | "source": [ 490 | "def output_cell(activation_matrix,parameters):\n", 491 | " #get hidden to output parameters\n", 492 | " how = parameters['how']\n", 493 | " \n", 494 | " #get outputs \n", 495 | " output_matrix = np.matmul(activation_matrix,how)\n", 496 | " output_matrix = softmax(output_matrix)\n", 497 | " \n", 498 | " return output_matrix" 499 | ] 500 | }, 501 | { 502 | "cell_type": "markdown", 503 | "metadata": {}, 504 | "source": [ 505 | "### Get corresponding embeddings for the batch dataset" 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "execution_count": 14, 511 | "metadata": { 512 | "collapsed": true 513 | }, 514 | "outputs": [], 515 | "source": [ 516 | "def get_embeddings(batch_dataset,embeddings):\n", 517 | " embedding_dataset = np.matmul(batch_dataset,embeddings)\n", 518 | " return embedding_dataset" 519 | ] 520 | }, 521 | { 522 | "cell_type": "markdown", 523 | "metadata": {}, 524 | "source": [ 525 | "### Forward Propagation\n", 526 | "![](https://github.com/navjindervirdee/neural-networks/blob/master/Recurrent%20Neural%20Network/LSTMForward.JPG?raw=true)\n", 527 | "\n", 528 | "Function returns the intermediate ativations in the respective caches:\n", 529 | "* LSTM Cache :- All lstm cell activation in every cell (fa,ia,ga,oa)\n", 530 | "* Activation Cache : All activation (a0,a1,a2..)\n", 531 | "* Cell Cache : All cell activations (c0,c1,c2..\n", 532 | "* Embedding cache : Embeddings of each batch (e0,e1,e2..)\n", 533 | "* Output Cache : All output (o1,o2,o3... )" 534 | ] 535 | }, 536 | { 537 | "cell_type": "code", 538 | "execution_count": 15, 539 | "metadata": { 540 | "collapsed": true 541 | }, 542 | "outputs": [], 543 | "source": [ 544 | "#forward propagation\n", 545 | "def forward_propagation(batches,parameters,embeddings):\n", 546 | " #get batch size\n", 547 | " batch_size = batches[0].shape[0]\n", 548 | " \n", 549 | " #to store the activations of all the unrollings.\n", 550 | " lstm_cache = dict() #lstm cache\n", 551 | " activation_cache = dict() #activation cache \n", 552 | " cell_cache = dict() #cell cache\n", 553 | " output_cache = dict() #output cache\n", 554 | " embedding_cache = dict() #embedding cache \n", 555 | " \n", 556 | " #initial activation_matrix(a0) and cell_matrix(c0)\n", 557 | " a0 = np.zeros([batch_size,hidden_units],dtype=np.float32)\n", 558 | " c0 = np.zeros([batch_size,hidden_units],dtype=np.float32)\n", 559 | " \n", 560 | " #store the initial activations in cache\n", 561 | " activation_cache['a0'] = a0\n", 562 | " cell_cache['c0'] = c0\n", 563 | " \n", 564 | " #unroll the names\n", 565 | " for i in range(len(batches)-1):\n", 566 | " #get first first character batch\n", 567 | " batch_dataset = batches[i]\n", 568 | " \n", 569 | " #get embeddings \n", 570 | " batch_dataset = get_embeddings(batch_dataset,embeddings)\n", 571 | " embedding_cache['emb'+str(i)] = batch_dataset\n", 572 | " \n", 573 | " #lstm cell\n", 574 | " lstm_activations,ct,at = lstm_cell(batch_dataset,a0,c0,parameters)\n", 575 | " \n", 576 | " #output cell\n", 577 | " ot = output_cell(at,parameters)\n", 578 | " \n", 579 | " #store the time 't' activations in caches\n", 580 | " lstm_cache['lstm' + str(i+1)] = lstm_activations\n", 581 | " activation_cache['a'+str(i+1)] = at\n", 582 | " cell_cache['c' + str(i+1)] = ct\n", 583 | " output_cache['o'+str(i+1)] = ot\n", 584 | " \n", 585 | " #update a0 and c0 to new 'at' and 'ct' for next lstm cell\n", 586 | " a0 = at\n", 587 | " c0 = ct\n", 588 | " \n", 589 | " return embedding_cache,lstm_cache,activation_cache,cell_cache,output_cache" 590 | ] 591 | }, 592 | { 593 | "cell_type": "markdown", 594 | "metadata": {}, 595 | "source": [ 596 | "### Calculate the Loss, Perplexity, and Accuracy\n", 597 | "**Loss**\n", 598 | "* Loss at time t = -sum(Y x log(d) + (1-Y) x log(1-pred)))/m\n", 599 | "* Overall Loss = **∑**(Loss(t)) sum of all losses at each time step 't'\n", 600 | "\n", 601 | "**Perplexity **\n", 602 | "* Probability Product = **∏**(prob(pred_char)) for each char in name\n", 603 | "* Perplexity = (1/probability_product) ^ (1/n) where n in number of chars in name\n", 604 | "\n", 605 | "**Accuracy**\n", 606 | "* Accuracy(t) = (Y==predictions,axis=1) for all time steps\n", 607 | "* Accuracy = ((**∑**Acc(t))/batch_size)/n for all time steps, n is number of chars in name" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": 16, 613 | "metadata": { 614 | "collapsed": true 615 | }, 616 | "outputs": [], 617 | "source": [ 618 | "#calculate loss, perplexity and accuracy\n", 619 | "def cal_loss_accuracy(batch_labels,output_cache):\n", 620 | " loss = 0 #to sum loss for each time step\n", 621 | " acc = 0 #to sum acc for each time step \n", 622 | " prob = 1 #probability product of each time step predicted char\n", 623 | " \n", 624 | " #batch size\n", 625 | " batch_size = batch_labels[0].shape[0]\n", 626 | " \n", 627 | " #loop through each time step\n", 628 | " for i in range(1,len(output_cache)+1):\n", 629 | " #get true labels and predictions\n", 630 | " labels = batch_labels[i]\n", 631 | " pred = output_cache['o'+str(i)]\n", 632 | " \n", 633 | " prob = np.multiply(prob,np.sum(np.multiply(labels,pred),axis=1).reshape(-1,1))\n", 634 | " loss += np.sum((np.multiply(labels,np.log(pred)) + np.multiply(1-labels,np.log(1-pred))),axis=1).reshape(-1,1)\n", 635 | " acc += np.array(np.argmax(labels,1)==np.argmax(pred,1),dtype=np.float32).reshape(-1,1)\n", 636 | " \n", 637 | " #calculate perplexity loss and accuracy\n", 638 | " perplexity = np.sum((1/prob)**(1/len(output_cache)))/batch_size\n", 639 | " loss = np.sum(loss)*(-1/batch_size)\n", 640 | " acc = np.sum(acc)/(batch_size)\n", 641 | " acc = acc/len(output_cache)\n", 642 | " \n", 643 | " return perplexity,loss,acc" 644 | ] 645 | }, 646 | { 647 | "cell_type": "markdown", 648 | "metadata": {}, 649 | "source": [ 650 | "### Calculate Output Cell Errors for each time step\n", 651 | "* Output Error Cache :- to store output error for each time step\n", 652 | "* Activation Error Cache : to store activation error for each time step" 653 | ] 654 | }, 655 | { 656 | "cell_type": "code", 657 | "execution_count": 17, 658 | "metadata": { 659 | "collapsed": true 660 | }, 661 | "outputs": [], 662 | "source": [ 663 | "#calculate output cell errors\n", 664 | "def calculate_output_cell_error(batch_labels,output_cache,parameters):\n", 665 | " #to store the output errors for each time step\n", 666 | " output_error_cache = dict()\n", 667 | " activation_error_cache = dict()\n", 668 | " how = parameters['how']\n", 669 | " \n", 670 | " #loop through each time step\n", 671 | " for i in range(1,len(output_cache)+1):\n", 672 | " #get true and predicted labels\n", 673 | " labels = batch_labels[i]\n", 674 | " pred = output_cache['o'+str(i)]\n", 675 | " \n", 676 | " #calculate the output_error for time step 't'\n", 677 | " error_output = pred - labels\n", 678 | " \n", 679 | " #calculate the activation error for time step 't'\n", 680 | " error_activation = np.matmul(error_output,how.T)\n", 681 | " \n", 682 | " #store the output and activation error in dict\n", 683 | " output_error_cache['eo'+str(i)] = error_output\n", 684 | " activation_error_cache['ea'+str(i)] = error_activation\n", 685 | " \n", 686 | " return output_error_cache,activation_error_cache" 687 | ] 688 | }, 689 | { 690 | "cell_type": "markdown", 691 | "metadata": {}, 692 | "source": [ 693 | "### Calculate Single LSTM CELL Error" 694 | ] 695 | }, 696 | { 697 | "cell_type": "code", 698 | "execution_count": 18, 699 | "metadata": { 700 | "collapsed": true 701 | }, 702 | "outputs": [], 703 | "source": [ 704 | "#calculate error for single lstm cell\n", 705 | "def calculate_single_lstm_cell_error(activation_output_error,next_activation_error,next_cell_error,parameters,lstm_activation,cell_activation,prev_cell_activation):\n", 706 | " #activation error = error coming from output cell and error coming from the next lstm cell\n", 707 | " activation_error = activation_output_error + next_activation_error\n", 708 | " \n", 709 | " #output gate error\n", 710 | " oa = lstm_activation['oa']\n", 711 | " eo = np.multiply(activation_error,tanh_activation(cell_activation))\n", 712 | " eo = np.multiply(np.multiply(eo,oa),1-oa)\n", 713 | " \n", 714 | " #cell activation error\n", 715 | " cell_error = np.multiply(activation_error,oa)\n", 716 | " cell_error = np.multiply(cell_error,tanh_derivative(tanh_activation(cell_activation)))\n", 717 | " #error also coming from next lstm cell \n", 718 | " cell_error += next_cell_error\n", 719 | " \n", 720 | " #input gate error\n", 721 | " ia = lstm_activation['ia']\n", 722 | " ga = lstm_activation['ga']\n", 723 | " ei = np.multiply(cell_error,ga)\n", 724 | " ei = np.multiply(np.multiply(ei,ia),1-ia)\n", 725 | " \n", 726 | " #gate gate error\n", 727 | " eg = np.multiply(cell_error,ia)\n", 728 | " eg = np.multiply(eg,tanh_derivative(ga))\n", 729 | " \n", 730 | " #forget gate error\n", 731 | " fa = lstm_activation['fa']\n", 732 | " ef = np.multiply(cell_error,prev_cell_activation)\n", 733 | " ef = np.multiply(np.multiply(ef,fa),1-fa)\n", 734 | " \n", 735 | " #prev cell error\n", 736 | " prev_cell_error = np.multiply(cell_error,fa)\n", 737 | " \n", 738 | " #get parameters\n", 739 | " fgw = parameters['fgw']\n", 740 | " igw = parameters['igw']\n", 741 | " ggw = parameters['ggw']\n", 742 | " ogw = parameters['ogw']\n", 743 | " \n", 744 | " #embedding + hidden activation error\n", 745 | " embed_activation_error = np.matmul(ef,fgw.T)\n", 746 | " embed_activation_error += np.matmul(ei,igw.T)\n", 747 | " embed_activation_error += np.matmul(eo,ogw.T)\n", 748 | " embed_activation_error += np.matmul(eg,ggw.T)\n", 749 | " \n", 750 | " input_hidden_units = fgw.shape[0]\n", 751 | " hidden_units = fgw.shape[1]\n", 752 | " input_units = input_hidden_units - hidden_units\n", 753 | " \n", 754 | " #prev activation error\n", 755 | " prev_activation_error = embed_activation_error[:,input_units:]\n", 756 | " \n", 757 | " #input error (embedding error)\n", 758 | " embed_error = embed_activation_error[:,:input_units]\n", 759 | " \n", 760 | " #store lstm error\n", 761 | " lstm_error = dict()\n", 762 | " lstm_error['ef'] = ef\n", 763 | " lstm_error['ei'] = ei\n", 764 | " lstm_error['eo'] = eo\n", 765 | " lstm_error['eg'] = eg\n", 766 | " \n", 767 | " return prev_activation_error,prev_cell_error,embed_error,lstm_error" 768 | ] 769 | }, 770 | { 771 | "cell_type": "markdown", 772 | "metadata": {}, 773 | "source": [ 774 | "### Calculate Output Cell Derivatives for each time step" 775 | ] 776 | }, 777 | { 778 | "cell_type": "code", 779 | "execution_count": 19, 780 | "metadata": { 781 | "collapsed": true 782 | }, 783 | "outputs": [], 784 | "source": [ 785 | "#calculate output cell derivatives\n", 786 | "def calculate_output_cell_derivatives(output_error_cache,activation_cache,parameters):\n", 787 | " #to store the sum of derivatives from each time step\n", 788 | " dhow = np.zeros(parameters['how'].shape)\n", 789 | " \n", 790 | " batch_size = activation_cache['a1'].shape[0]\n", 791 | " \n", 792 | " #loop through the time steps \n", 793 | " for i in range(1,len(output_error_cache)+1):\n", 794 | " #get output error\n", 795 | " output_error = output_error_cache['eo' + str(i)]\n", 796 | " \n", 797 | " #get input activation\n", 798 | " activation = activation_cache['a'+str(i)]\n", 799 | " \n", 800 | " #cal derivative and summing up!\n", 801 | " dhow += np.matmul(activation.T,output_error)/batch_size\n", 802 | " \n", 803 | " return dhow" 804 | ] 805 | }, 806 | { 807 | "cell_type": "markdown", 808 | "metadata": {}, 809 | "source": [ 810 | "### Calculate LSTM CELL Derivatives for each time step" 811 | ] 812 | }, 813 | { 814 | "cell_type": "code", 815 | "execution_count": 20, 816 | "metadata": { 817 | "collapsed": true 818 | }, 819 | "outputs": [], 820 | "source": [ 821 | "#calculate derivatives for single lstm cell\n", 822 | "def calculate_single_lstm_cell_derivatives(lstm_error,embedding_matrix,activation_matrix):\n", 823 | " #get error for single time step\n", 824 | " ef = lstm_error['ef']\n", 825 | " ei = lstm_error['ei']\n", 826 | " eo = lstm_error['eo']\n", 827 | " eg = lstm_error['eg']\n", 828 | " \n", 829 | " #get input activations for this time step\n", 830 | " concat_matrix = np.concatenate((embedding_matrix,activation_matrix),axis=1)\n", 831 | " \n", 832 | " batch_size = embedding_matrix.shape[0]\n", 833 | " \n", 834 | " #cal derivatives for this time step\n", 835 | " dfgw = np.matmul(concat_matrix.T,ef)/batch_size\n", 836 | " digw = np.matmul(concat_matrix.T,ei)/batch_size\n", 837 | " dogw = np.matmul(concat_matrix.T,eo)/batch_size\n", 838 | " dggw = np.matmul(concat_matrix.T,eg)/batch_size\n", 839 | " \n", 840 | " #store the derivatives for this time step in dict\n", 841 | " derivatives = dict()\n", 842 | " derivatives['dfgw'] = dfgw\n", 843 | " derivatives['digw'] = digw\n", 844 | " derivatives['dogw'] = dogw\n", 845 | " derivatives['dggw'] = dggw\n", 846 | " \n", 847 | " return derivatives" 848 | ] 849 | }, 850 | { 851 | "cell_type": "markdown", 852 | "metadata": {}, 853 | "source": [ 854 | "### Backward Propagation\n", 855 | "\n", 856 | "* Apply chain rule and calculate the errors for each time step\n", 857 | "* Store the deivatives in **derivatives** dict" 858 | ] 859 | }, 860 | { 861 | "cell_type": "code", 862 | "execution_count": 21, 863 | "metadata": { 864 | "collapsed": true 865 | }, 866 | "outputs": [], 867 | "source": [ 868 | "#backpropagation\n", 869 | "def backward_propagation(batch_labels,embedding_cache,lstm_cache,activation_cache,cell_cache,output_cache,parameters):\n", 870 | " #calculate output errors \n", 871 | " output_error_cache,activation_error_cache = calculate_output_cell_error(batch_labels,output_cache,parameters)\n", 872 | " \n", 873 | " #to store lstm error for each time step\n", 874 | " lstm_error_cache = dict()\n", 875 | " \n", 876 | " #to store embeding errors for each time step\n", 877 | " embedding_error_cache = dict()\n", 878 | " \n", 879 | " # next activation error \n", 880 | " # next cell error \n", 881 | " #for last cell will be zero\n", 882 | " eat = np.zeros(activation_error_cache['ea1'].shape)\n", 883 | " ect = np.zeros(activation_error_cache['ea1'].shape)\n", 884 | " \n", 885 | " #calculate all lstm cell errors (going from last time-step to the first time step)\n", 886 | " for i in range(len(lstm_cache),0,-1):\n", 887 | " #calculate the lstm errors for this time step 't'\n", 888 | " pae,pce,ee,le = calculate_single_lstm_cell_error(activation_error_cache['ea'+str(i)],eat,ect,parameters,lstm_cache['lstm'+str(i)],cell_cache['c'+str(i)],cell_cache['c'+str(i-1)])\n", 889 | " \n", 890 | " #store the lstm error in dict\n", 891 | " lstm_error_cache['elstm'+str(i)] = le\n", 892 | " \n", 893 | " #store the embedding error in dict\n", 894 | " embedding_error_cache['eemb'+str(i-1)] = ee\n", 895 | " \n", 896 | " #update the next activation error and next cell error for previous cell\n", 897 | " eat = pae\n", 898 | " ect = pce\n", 899 | " \n", 900 | " \n", 901 | " #calculate output cell derivatives\n", 902 | " derivatives = dict()\n", 903 | " derivatives['dhow'] = calculate_output_cell_derivatives(output_error_cache,activation_cache,parameters)\n", 904 | " \n", 905 | " #calculate lstm cell derivatives for each time step and store in lstm_derivatives dict\n", 906 | " lstm_derivatives = dict()\n", 907 | " for i in range(1,len(lstm_error_cache)+1):\n", 908 | " lstm_derivatives['dlstm'+str(i)] = calculate_single_lstm_cell_derivatives(lstm_error_cache['elstm'+str(i)],embedding_cache['emb'+str(i-1)],activation_cache['a'+str(i-1)])\n", 909 | " \n", 910 | " #initialize the derivatives to zeros \n", 911 | " derivatives['dfgw'] = np.zeros(parameters['fgw'].shape)\n", 912 | " derivatives['digw'] = np.zeros(parameters['igw'].shape)\n", 913 | " derivatives['dogw'] = np.zeros(parameters['ogw'].shape)\n", 914 | " derivatives['dggw'] = np.zeros(parameters['ggw'].shape)\n", 915 | " \n", 916 | " #sum up the derivatives for each time step\n", 917 | " for i in range(1,len(lstm_error_cache)+1):\n", 918 | " derivatives['dfgw'] += lstm_derivatives['dlstm'+str(i)]['dfgw']\n", 919 | " derivatives['digw'] += lstm_derivatives['dlstm'+str(i)]['digw']\n", 920 | " derivatives['dogw'] += lstm_derivatives['dlstm'+str(i)]['dogw']\n", 921 | " derivatives['dggw'] += lstm_derivatives['dlstm'+str(i)]['dggw']\n", 922 | " \n", 923 | " return derivatives,embedding_error_cache" 924 | ] 925 | }, 926 | { 927 | "cell_type": "markdown", 928 | "metadata": {}, 929 | "source": [ 930 | "### Adam Optimizer\n", 931 | "Using Exponentially Weighted Averages
\n", 932 | "* Vdw = beta1 x Vdw + (1-beta1) x (dw) \n", 933 | "* Sdw = beta2 x Sdw + (1-beta2) x dw^2\n", 934 | "* W = W - learning_rate x ( Vdw/ (sqrt(Sdw)+1e-7) )" 935 | ] 936 | }, 937 | { 938 | "cell_type": "code", 939 | "execution_count": 22, 940 | "metadata": { 941 | "collapsed": true 942 | }, 943 | "outputs": [], 944 | "source": [ 945 | "#update the parameters using adam optimizer\n", 946 | "#adam optimization\n", 947 | "def update_parameters(parameters,derivatives,V,S,t):\n", 948 | " #get derivatives\n", 949 | " dfgw = derivatives['dfgw']\n", 950 | " digw = derivatives['digw']\n", 951 | " dogw = derivatives['dogw']\n", 952 | " dggw = derivatives['dggw']\n", 953 | " dhow = derivatives['dhow']\n", 954 | " \n", 955 | " #get parameters\n", 956 | " fgw = parameters['fgw']\n", 957 | " igw = parameters['igw']\n", 958 | " ogw = parameters['ogw']\n", 959 | " ggw = parameters['ggw']\n", 960 | " how = parameters['how']\n", 961 | " \n", 962 | " #get V parameters\n", 963 | " vfgw = V['vfgw']\n", 964 | " vigw = V['vigw']\n", 965 | " vogw = V['vogw']\n", 966 | " vggw = V['vggw']\n", 967 | " vhow = V['vhow']\n", 968 | " \n", 969 | " #get S parameters\n", 970 | " sfgw = S['sfgw']\n", 971 | " sigw = S['sigw']\n", 972 | " sogw = S['sogw']\n", 973 | " sggw = S['sggw']\n", 974 | " show = S['show']\n", 975 | " \n", 976 | " #calculate the V parameters from V and current derivatives\n", 977 | " vfgw = (beta1*vfgw + (1-beta1)*dfgw)\n", 978 | " vigw = (beta1*vigw + (1-beta1)*digw)\n", 979 | " vogw = (beta1*vogw + (1-beta1)*dogw)\n", 980 | " vggw = (beta1*vggw + (1-beta1)*dggw)\n", 981 | " vhow = (beta1*vhow + (1-beta1)*dhow)\n", 982 | " \n", 983 | " #calculate the S parameters from S and current derivatives\n", 984 | " sfgw = (beta2*sfgw + (1-beta2)*(dfgw**2))\n", 985 | " sigw = (beta2*sigw + (1-beta2)*(digw**2))\n", 986 | " sogw = (beta2*sogw + (1-beta2)*(dogw**2))\n", 987 | " sggw = (beta2*sggw + (1-beta2)*(dggw**2))\n", 988 | " show = (beta2*show + (1-beta2)*(dhow**2))\n", 989 | " \n", 990 | " #update the parameters\n", 991 | " fgw = fgw - learning_rate*((vfgw)/(np.sqrt(sfgw) + 1e-6))\n", 992 | " igw = igw - learning_rate*((vigw)/(np.sqrt(sigw) + 1e-6))\n", 993 | " ogw = ogw - learning_rate*((vogw)/(np.sqrt(sogw) + 1e-6))\n", 994 | " ggw = ggw - learning_rate*((vggw)/(np.sqrt(sggw) + 1e-6))\n", 995 | " how = how - learning_rate*((vhow)/(np.sqrt(show) + 1e-6))\n", 996 | " \n", 997 | " #store the new weights\n", 998 | " parameters['fgw'] = fgw\n", 999 | " parameters['igw'] = igw\n", 1000 | " parameters['ogw'] = ogw\n", 1001 | " parameters['ggw'] = ggw\n", 1002 | " parameters['how'] = how\n", 1003 | " \n", 1004 | " #store the new V parameters\n", 1005 | " V['vfgw'] = vfgw \n", 1006 | " V['vigw'] = vigw \n", 1007 | " V['vogw'] = vogw \n", 1008 | " V['vggw'] = vggw\n", 1009 | " V['vhow'] = vhow\n", 1010 | " \n", 1011 | " #store the s parameters\n", 1012 | " S['sfgw'] = sfgw \n", 1013 | " S['sigw'] = sigw \n", 1014 | " S['sogw'] = sogw \n", 1015 | " S['sggw'] = sggw\n", 1016 | " S['show'] = show\n", 1017 | " \n", 1018 | " return parameters,V,S " 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "markdown", 1023 | "metadata": {}, 1024 | "source": [ 1025 | "### Update the embeddings" 1026 | ] 1027 | }, 1028 | { 1029 | "cell_type": "code", 1030 | "execution_count": 23, 1031 | "metadata": { 1032 | "collapsed": true 1033 | }, 1034 | "outputs": [], 1035 | "source": [ 1036 | "#update the Embeddings\n", 1037 | "def update_embeddings(embeddings,embedding_error_cache,batch_labels):\n", 1038 | " #to store the embeddings derivatives\n", 1039 | " embedding_derivatives = np.zeros(embeddings.shape)\n", 1040 | " \n", 1041 | " batch_size = batch_labels[0].shape[0]\n", 1042 | " \n", 1043 | " #sum the embedding derivatives for each time step\n", 1044 | " for i in range(len(embedding_error_cache)):\n", 1045 | " embedding_derivatives += np.matmul(batch_labels[i].T,embedding_error_cache['eemb'+str(i)])/batch_size\n", 1046 | " \n", 1047 | " #update the embeddings\n", 1048 | " embeddings = embeddings - learning_rate*embedding_derivatives\n", 1049 | " return embeddings" 1050 | ] 1051 | }, 1052 | { 1053 | "cell_type": "markdown", 1054 | "metadata": {}, 1055 | "source": [ 1056 | "### Functions to Initialize the V and S parameters for Adam Optimizer" 1057 | ] 1058 | }, 1059 | { 1060 | "cell_type": "code", 1061 | "execution_count": 24, 1062 | "metadata": { 1063 | "collapsed": true 1064 | }, 1065 | "outputs": [], 1066 | "source": [ 1067 | "def initialize_V(parameters):\n", 1068 | " Vfgw = np.zeros(parameters['fgw'].shape)\n", 1069 | " Vigw = np.zeros(parameters['igw'].shape)\n", 1070 | " Vogw = np.zeros(parameters['ogw'].shape)\n", 1071 | " Vggw = np.zeros(parameters['ggw'].shape)\n", 1072 | " Vhow = np.zeros(parameters['how'].shape)\n", 1073 | " \n", 1074 | " V = dict()\n", 1075 | " V['vfgw'] = Vfgw\n", 1076 | " V['vigw'] = Vigw\n", 1077 | " V['vogw'] = Vogw\n", 1078 | " V['vggw'] = Vggw\n", 1079 | " V['vhow'] = Vhow\n", 1080 | " return V\n", 1081 | "\n", 1082 | "def initialize_S(parameters):\n", 1083 | " Sfgw = np.zeros(parameters['fgw'].shape)\n", 1084 | " Sigw = np.zeros(parameters['igw'].shape)\n", 1085 | " Sogw = np.zeros(parameters['ogw'].shape)\n", 1086 | " Sggw = np.zeros(parameters['ggw'].shape)\n", 1087 | " Show = np.zeros(parameters['how'].shape)\n", 1088 | " \n", 1089 | " S = dict()\n", 1090 | " S['sfgw'] = Sfgw\n", 1091 | " S['sigw'] = Sigw\n", 1092 | " S['sogw'] = Sogw\n", 1093 | " S['sggw'] = Sggw\n", 1094 | " S['show'] = Show\n", 1095 | " return S" 1096 | ] 1097 | }, 1098 | { 1099 | "cell_type": "markdown", 1100 | "metadata": {}, 1101 | "source": [ 1102 | "### Train Function\n", 1103 | "\n", 1104 | "1. Initialize Parameters\n", 1105 | "2. Forward Propagation\n", 1106 | "3. Calculate Loss, Perplexity and Accuracy\n", 1107 | "4. Backward Propagation\n", 1108 | "5. Update the Parameters and Embeddings\n", 1109 | "\n", 1110 | "Batch Size = 20\n", 1111 | "Repeat the steps 2-5 for each batch!" 1112 | ] 1113 | }, 1114 | { 1115 | "cell_type": "code", 1116 | "execution_count": 25, 1117 | "metadata": { 1118 | "collapsed": true 1119 | }, 1120 | "outputs": [], 1121 | "source": [ 1122 | "#train function\n", 1123 | "def train(train_dataset,iters=1000,batch_size=20):\n", 1124 | " #initalize the parameters\n", 1125 | " parameters = initialize_parameters()\n", 1126 | " \n", 1127 | " #initialize the V and S parameters for Adam\n", 1128 | " V = initialize_V(parameters)\n", 1129 | " S = initialize_S(parameters)\n", 1130 | " \n", 1131 | " #generate the random embeddings\n", 1132 | " embeddings = np.random.normal(0,0.01,(len(vocab),input_units))\n", 1133 | " \n", 1134 | " #to store the Loss, Perplexity and Accuracy for each batch\n", 1135 | " J = []\n", 1136 | " P = []\n", 1137 | " A = []\n", 1138 | " \n", 1139 | " \n", 1140 | " for step in range(iters):\n", 1141 | " #get batch dataset\n", 1142 | " index = step%len(train_dataset)\n", 1143 | " batches = train_dataset[index]\n", 1144 | " \n", 1145 | " #forward propagation\n", 1146 | " embedding_cache,lstm_cache,activation_cache,cell_cache,output_cache = forward_propagation(batches,parameters,embeddings)\n", 1147 | " \n", 1148 | " #calculate the loss, perplexity and accuracy\n", 1149 | " perplexity,loss,acc = cal_loss_accuracy(batches,output_cache)\n", 1150 | " \n", 1151 | " #backward propagation\n", 1152 | " derivatives,embedding_error_cache = backward_propagation(batches,embedding_cache,lstm_cache,activation_cache,cell_cache,output_cache,parameters)\n", 1153 | " \n", 1154 | " #update the parameters\n", 1155 | " parameters,V,S = update_parameters(parameters,derivatives,V,S,step)\n", 1156 | " \n", 1157 | " #update the embeddings\n", 1158 | " embeddings = update_embeddings(embeddings,embedding_error_cache,batches)\n", 1159 | " \n", 1160 | " \n", 1161 | " J.append(loss)\n", 1162 | " P.append(perplexity)\n", 1163 | " A.append(acc)\n", 1164 | " \n", 1165 | " #print loss, accuracy and perplexity\n", 1166 | " if(step%1000==0):\n", 1167 | " print(\"For Single Batch :\")\n", 1168 | " print('Step = {}'.format(step))\n", 1169 | " print('Loss = {}'.format(round(loss,2)))\n", 1170 | " print('Perplexity = {}'.format(round(perplexity,2)))\n", 1171 | " print('Accuracy = {}'.format(round(acc*100,2)))\n", 1172 | " print()\n", 1173 | " \n", 1174 | " return embeddings, parameters,J,P,A" 1175 | ] 1176 | }, 1177 | { 1178 | "cell_type": "markdown", 1179 | "metadata": {}, 1180 | "source": [ 1181 | "### Let's Train\n", 1182 | "* Will take around 5-10 mins on CPU" 1183 | ] 1184 | }, 1185 | { 1186 | "cell_type": "code", 1187 | "execution_count": 26, 1188 | "metadata": {}, 1189 | "outputs": [ 1190 | { 1191 | "name": "stdout", 1192 | "output_type": "stream", 1193 | "text": [ 1194 | "For Single Batch :\n", 1195 | "Step = 0\n", 1196 | "Loss = 47.05\n", 1197 | "Perplexity = 27.0\n", 1198 | "Accuracy = 2.27\n", 1199 | "\n", 1200 | "For Single Batch :\n", 1201 | "Step = 1000\n", 1202 | "Loss = 11.94\n", 1203 | "Perplexity = 2.08\n", 1204 | "Accuracy = 75.0\n", 1205 | "\n", 1206 | "For Single Batch :\n", 1207 | "Step = 2000\n", 1208 | "Loss = 8.49\n", 1209 | "Perplexity = 1.63\n", 1210 | "Accuracy = 83.18\n", 1211 | "\n", 1212 | "For Single Batch :\n", 1213 | "Step = 3000\n", 1214 | "Loss = 8.0\n", 1215 | "Perplexity = 1.59\n", 1216 | "Accuracy = 82.27\n", 1217 | "\n", 1218 | "For Single Batch :\n", 1219 | "Step = 4000\n", 1220 | "Loss = 8.04\n", 1221 | "Perplexity = 1.58\n", 1222 | "Accuracy = 83.64\n", 1223 | "\n", 1224 | "For Single Batch :\n", 1225 | "Step = 5000\n", 1226 | "Loss = 8.08\n", 1227 | "Perplexity = 1.58\n", 1228 | "Accuracy = 84.09\n", 1229 | "\n", 1230 | "For Single Batch :\n", 1231 | "Step = 6000\n", 1232 | "Loss = 7.86\n", 1233 | "Perplexity = 1.56\n", 1234 | "Accuracy = 84.09\n", 1235 | "\n", 1236 | "For Single Batch :\n", 1237 | "Step = 7000\n", 1238 | "Loss = 7.85\n", 1239 | "Perplexity = 1.56\n", 1240 | "Accuracy = 84.09\n", 1241 | "\n", 1242 | "For Single Batch :\n", 1243 | "Step = 8000\n", 1244 | "Loss = 7.83\n", 1245 | "Perplexity = 1.56\n", 1246 | "Accuracy = 84.55\n", 1247 | "\n" 1248 | ] 1249 | } 1250 | ], 1251 | "source": [ 1252 | "embeddings,parameters,J,P,A = train(train_dataset,iters=8001)" 1253 | ] 1254 | }, 1255 | { 1256 | "cell_type": "markdown", 1257 | "metadata": {}, 1258 | "source": [ 1259 | "### Let's Plot some graphs\n", 1260 | "\n", 1261 | "* Plotted average loss of 30 batches, average perplexity of 30 batches, and average accuracy of 30 batches." 1262 | ] 1263 | }, 1264 | { 1265 | "cell_type": "code", 1266 | "execution_count": 27, 1267 | "metadata": {}, 1268 | "outputs": [ 1269 | { 1270 | "data": { 1271 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4XOWV+PHvmdFIo94ly5YsdxuDwTY2pjcDARJayCYB\nQghk0zYJySabhGyy6WFhk5CGk10IBMKPEjqEEoptMKYY995tyeq9jtqU9/fHvXMtGUkeG43K6Hye\nZx7P3GnnamDOvO95ixhjUEopNX65RjoApZRSI0sTgVJKjXOaCJRSapzTRKCUUuOcJgKllBrnNBEo\npdQ4p4lAqTFERIyIzBjpOFRs0USgxjQRKRGRi0bovQtE5F4RqRSRdhE5ICIPiMickYhHqeOliUCp\n4yAi2cA7QBJwDpAKLATeBC4e4DlxwxagUsdAE4GKWSLyBRHZJyKNIvK8iEy0j4uI/FZEakWkVUS2\nishJ9n2Xi8gOEWkTkQoR+Y8BXv7fgVbgRmPMfmNpNsb81RjzR/u1pthdOZ8XkUPACvv4EyJSLSIt\nIrJKRE7sFfMDIvK/IvKaHcObIlJ8xHtfJCJ7RaRZRJaJiAzxn06NM5oIVEwSkQuB/wY+CRQApcBj\n9t2XAOcCs4B0+zEN9n33AV8yxqQCJ2F/effjIuAZY0wognDOA04APmLffhmYCeQBG4CHj3j8DcDP\ngRxgUz/3fwxYDJxsx/4RlPoQNBGoWHUDcL8xZoMxphv4PnCGiEwB/FhdOXMAMcbsNMZU2c/zA3NF\nJM0Y02SM2TDA6+cA1eEbInKl/Qu9TURePeKxPzHG+IwxnQDGmPuNMW12XD8BThGR9F6Pf9EYs8q+\n/wd23EW97r/Dbn0cAlYC84/xb6NUH5oIVKyaiNUKAMAY0471q3+SMWYFcDewDKgVkXtEJM1+6LXA\n5UCp3S1zxgCv34DV0gi//vPGmAysLqP4Ix5bFr4iIm4RuUNE9otIK1Bi35XT3+PtuBvt8wmr7nW9\nA0gZIEalIqKJQMWqSsDpWxeRZCAbqAAwxvzBGHMqMBeri+g79vG1xpirsLptngUeH+D1lwNXi0gk\n/w/1XuL3euAqrK6ldGBKOMRej3F+/YtICpBln49SUaGJQMUCj4h4e13igEeBm0VkvogkALcDa4wx\nJSKyWESWiIgH8AFdQEhE4kXkBhFJN8b4sYrBA9UA7gIygYdEZLpdgE7l6N00qUA3VosiyY7rSJeL\nyNkiEo9VK3jPGFPWz+OUGhKaCFQseAno7HX5iTHmdeC/gKeAKmA68Gn78WnAvUATVvdRA/Ar+74b\ngRK72+bLWLWGDzDG1AOnYyWR1UAbVmE3FfjKILH+zX7PCmAH8F4/j3kE+DFWl9CpwGcGO3mlPizR\njWmUGj1E5AGg3Bjzw5GORY0f2iJQSqlxThOBUkqNc9o1pJRS41zUWgT26I33RWSziGwXkZ/ax7Ps\n6fN77X8zoxWDUkqpo4tai8Be/yTZGNNuD9NbDXwD+DjQaIy5Q0RuAzKNMd8b7LVycnLMlClTohKn\nUkrFqvXr19cbY3KP9riorYZorAzTbt/02BeDNZnmfPv4g8AbwKCJYMqUKaxbty4qcSqlVKwSkdKj\nPyrKxWJ7Ov0moBZ4zRizBsjvta5LNZA/wHO/KCLrRGRdXV1dNMNUSqlxLaqJwBgTNMbMBwqB08JL\n/fa639B3+n3v++4xxiwyxizKzT1qy0YppdRxGpbho8aYZqxVEi8FakSkAKwdnrBaC0oppUZINEcN\n5YpIhn09EWvXpl3A88BN9sNuAp6LVgxKKaWOLppb5xUAD4qIGyvhPG6MeUFE3gUeF5HPY6258sko\nxqCUUuooojlqaAuwoJ/jDcDSaL2vUkqpY6NLTCil1DgX04lg+c4a/vzG/pEOQymlRrWYTgRv7K7j\nnlWaCJRSajAxnQji3EIgqIvqKaXUYGI7EbiEQEgTgVJKDSa2E4HbRSA00JazSimlIMYTgccl+IMG\n3XNBKaUGFtOJIM5tnV5Qu4eUUmpAMZ4IBEDrBEopNYiYTgQel3V6mgiUUmpgMZ0I3C67RRDUgrFS\nSg0kphOBx+4a8utcAqWUGlBMJ4JwsViHkCql1MBiOxE4XUPaIlBKqYHEdCLw2C0Cv9YIlFJqQDGd\nCMLFYp1HoJRSA4vpRKDFYqWUOrqYTgRxLi0WK6XU0cR2ItAWgVJKHVVMJ4JwsVgnlCml1MBiOhFo\nsVgppY4uphOBUyzWRKCUUgOK6UTgFIu1a0gppQYU24lAi8VKKXVUMZ0IPLrWkFJKHVVMJwItFiul\n1NHFdCIIb0yjXUNKKTWwmE4EzlaVWixWSqkBjYtEoMNHlVJqYDGdCDw6fFQppY4qphOB263FYqWU\nOpqYTgRaLFZKqaOL6USgxWKllDq62E4ELi0WK6XU0cR0IhAR4lyiLQKllBpETCcCsGYXB7RFoJRS\nA4r5ROBxuwhosVgppQYU84kgzi266JxSSg0iaolARIpEZKWI7BCR7SLyDfv4T0SkQkQ22ZfLoxUD\nWHsS6PBRpZQaWFwUXzsAfNsYs0FEUoH1IvKafd9vjTG/juJ7OzxuLRYrpdRgopYIjDFVQJV9vU1E\ndgKTovV+A9FisVJKDW5YagQiMgVYAKyxD31dRLaIyP0ikhnN9/a4XZoIlFJqEEdNBCKSJyLXiMhX\nReQWETlNRCJOICKSAjwFfNMY0wr8GZgGzMdqMfxmgOd9UUTWici6urq6SN/uA3QegVJKDW7AL3QR\nuUBEXgFeBC4DCoC5wA+BrSLyUxFJG+zFRcSDlQQeNsY8DWCMqTHGBI0xIeBe4LT+nmuMuccYs8gY\nsyg3N/d4zg2AOLcWi5VSajCD1QguB75gjDl05B0iEgd8DLgY64v+A0REgPuAncaYu3odL7DrBwDX\nANuOM/aIeHT4qFJKDWrARGCM+c4g9wWAZ4/y2mcBN2K1HjbZx/4TuE5E5gMGKAG+dCwBHyura0hb\nBEopNZCjjhqyx///FWgD/oJV9L3NGPPqYM8zxqwGpJ+7XjqOOI9bnMulLQKllBpEJEXfW+wi7yVA\nJtav/DuiGtUQinNri0AppQYTSSII/6q/HHjIGLOd/n/pj0pxbpcuQ62UUoOIJBGsF5FXsRLBK/Ys\n4THT1+LR4aNKKTWoSGYWfx5rzP8BY0yHiGQDN0c3rKGjXUNKKTW4SFoEBmv+wK327WTAG7WIhlic\ny4Vfi8VKKTWgSBLBn4AzgOvs223AsqhFNMTi3EJQawRKKTWgSLqGlhhjForIRgBjTJOIxEc5riET\n59KNaZRSajCRtAj8IuLG6iJCRHIZS8Vit+DXYrFSSg0okkTwB+AZIE9EfgmsBm6PalRDyNqhTFsE\nSik1kKN2DRljHhaR9cBSrPkDVxtjdkY9siFi7VCmLQKllBpIpBvT7AVaw48Xkcn9LUY3GsW5tFis\nlFKDiWStoa8DPwZqgCBWq8AAJ0c3tKER59ZisVJKDSaSFsE3gNnGmIZoBxMNHrfoPAKllBpEJMXi\nMqAl2oFES5zLhTFo95BSSg1gwBaBiHzLvnoAeENEXgS6w/f33mxmNItzW+vj+YMh3C73CEejlFKj\nz2BdQ6n2v4fsS7x9GVPiXFYi0BaBUkr1b7Adyn46nIFEi9djtQI6eoIkJ0Q6SEoppcaPo9YIROQ1\nEcnodTvT3tR+TMhOsRoxjb6eEY5EKaVGp0iKxbnGmObwDWNME5AXvZCGVnZyAgAN7d1HeaRSSo1P\nkSSCoIhMDt8QkWLsdYfGgtxUq0VQp4lAKaX6FUmn+Q+A1SLyJtZksnOAL0U1qiF0uEWgXUNKKdWf\nSNYa+qeILAROtw990xhTH92whk56oge3S2jwaYtAKaX6E0mxeLkxpt4Y84J9qReR5cMR3FBwuYTs\n5HhtESil1AAGm1DmBZKAHBHJxOoWAkgDJg1DbEMmOyWBeq0RKKVUvwbrGvoS8E1gIrCew4mgFbg7\nynENqZyUeOq1RaCUUv0abELZ74Hfi8jXjTF/HMaYhlxOSgIlDb6RDkMppUalSIrFfxSRk4C5gLfX\n8b9FM7ChlJ0cT32btgiUUqo/kexH8GPgfKxE8BJwGdZ2lWMnEaQk0OkP0tETIClel5lQSqneIplQ\n9gmsbSqrjTE3A6cA6VGNaojl2MtM6MghpZT6oEgSQacxJgQERCQNqAWKohvW0MpJsSaV1bbpyCGl\nlDpSJP0k6+xF5+7FGj3UDrwb1aiGWF6alQjq2rpGOBKllBp9IikW/5t99X9F5J9AmjFmS3TDGlr5\naVaNu6ZVWwRKKXWkiCqnIvJx4GysxeZWA2MqEWQlxRPnEmpatUWglFJHimSJiT8BXwa2AtuAL4nI\nsmgHNpRcLiEvNUFbBEop1Y9IWgQXAicYYwyAiDwIbI9qVFGQl+alVmsESin1AZGMGtoHTO51u8g+\nNqbkpyVo15BSSvVjsEXn/oFVE0gFdorI+/btJcD7wxPe0MlP8/LegcaRDkMppUadwbqGfj1sUQyD\n/DQvLZ1+uvxBZ0N7pZRSgy869+ZwBhJtean2pLLWbiZnJ41wNEopNXpEUiM4LiJSJCIrRWSHiGwX\nkW/Yx7NE5DUR2Wv/mxmtGHpz5hJowVgppfqIWiIAAsC3jTFzsba5/KqIzAVuA5YbY2YCy+3bURdO\nBJXNncPxdkopNWZELREYY6qMMRvs623ATqydza4CHrQf9iBwdbRi6K04OwmXwP7a9uF4O6WUGjMG\nTAQiki4id4jILhFpFJEGEdlpH8s4ljcRkSnAAmANkG+MqbLvqgbyB3jOF0VknYisq6urO5a365fX\n42ZKTjK7a9q45YG13L1i74d+TaWUigWDtQgeB5qA840xWcaYbOAC+9jjkb6BiKQATwHfNMa09r7P\nnqRm+nueMeYeY8wiY8yi3NzcSN9uULPzU1lX0sSKXbWsOahDSZVSCgZPBFOMMXcaY6rDB4wx1caY\nO4HiSF5cRDxYSeBhY8zT9uEaESmw7y/AWtZ6WMyekEqDz9qToLpFi8ZKKQWDJ4JSEfmuiDhdNyKS\nLyLfA8qO9sIiIsB9wE5jzF297noeuMm+fhPw3LGHfXxm56c61zURKKWUZbBE8CkgG3jTrhE0Am8A\nWcAnI3jts4AbgQtFZJN9uRy4A7hYRPYCF9m3h8XsCYcTQVt3gPbuwHC9tVJKjVqDTShrAr5nX46Z\nMWY1IAPcvfR4XvPDKs5OZs6EVPLSvKzaU0d1Sxcz8lJGIhSllBo1Bh0+KiKnichi+/pcEfmW/at+\nTHK7hH9+81y+ct50AF2ETimlGHzRuR8DlwFxIvIa1mJzK4HbRGSBMeaXwxTjkCtItyaXVWmdQCml\nBl107hPAfCABa7x/oTGmVUR+jTUfYMwmggnp4a0rNREopdRgXUMBY0zQGNMB7A/PATDGdAKhYYku\nSrweNxlJHqpadLkJpZQaLBH0iEh4mc5TwwdFJJ0xnggAJqR5qW7RrSuVUmqwrqFzjTHdAMaY3l/8\nHg7PAxizCtK9ugCdUkoxSIsgnAT6OV5vjNkavZCGR1FWEuVNHVS1dPKXtw5gb8mslFLjTjSXoR7V\nijKTaO0KcO+qg/zixZ1srWgZ6ZCUUmpEDLb6aMJwBjLcirISAVi521rqaMWuYVvySCmlRpXBWgTv\nAojIQ8MUy7AqzLTq4AfrfQCs3P3hl7pWSqmxaLBicbyIXA+cKSIfP/LOXquJjklFWYf3LU70uNlS\n3kx9ezc5KTHdEFJKqQ8YrEXwZeAcIAO44ojLx6IfWnSlJ3pI81p58CMn5mMM7K3R3cuUUuPPYIvO\nrQZWi8g6Y8x9wxjTsCnKSmJ7ZSuLp2bx7KZKmjt6RjokpZQadpGMGnpIRG4VkSfty9ftDWfGvCK7\nTrB4ShYATR3+kQxHKaVGxGA1grA/YU0i+5N9+0bgz8C/Riuo4XLurFw6/UEnITTZLYKXt1aRn+5l\n4eTMkQxPKaWGRSSJYLEx5pRet1eIyOZoBTScrl8ymeuXTAYgIc7ldA395B/bmTcpg7/ctGgkw1NK\nqWERSddQUESmh2+IyDQgGL2QRkZWcjxNHX78wRC1bd26IJ1SatyIpEXwHWCliBzA2nGsGLg5qlGN\ngIykeJo7eqhr68YY3atAKTV+HDURGGOWi8hMYLZ9aPdA6xCNZZlJHpo6/E4CaPT10OUP4vW4Rzgy\npZSKrojWGjLGdBtjttiXmEsCAJlJ8TR19PTZrEZbBUqp8WDcLjp3pIwkD829WgSA1gmUUuOCJgJb\npl0jqOq1R0FVc/8tgkAwpMtWK6VixlETgYgs7OcyXUQiKTSPGRlJHkIG9tS2MyEtvLl9J92BILuq\nW53HBUOGE370T3747LaRClUppYZUJC2CPwHvAfcA92KtSvoEsFtELolibMMqMykegJ1VrRRnJ5GV\nHE9lSxd/X1vGpb97i4fXlALw3KYK/EHDw2sOjWS4Sik1ZCJJBJXAAmPMImPMqcAC4ABwMfA/0Qxu\nOGUmW6tm1LV1U5DuZUKal6rmTsoaOwD4r2e3Udrg448r9gGwqFhnHSulYkMk3TuzjDHbwzeMMTtE\nZI4x5oCIRDG04ZVhtwgAJmYk0t4doLyp01mWOmRgU1mzs39Be3dgROJUSqmhFkmLYLuI/FlEzrMv\nfwJ22DuYxcwqbRPTrR3LspLjufGMYgrSE6lq6aK2rZvcVCsZbC23trN0u4S2LisRBIIhdle3jUzQ\nSik1BCJJBJ8D9gHftC8H7GN+4IJoBTbcJqR7eeu7F7DmP5dSkJ5IQYaXlk4/hxo7OKEgDRHYVmkl\ngum5ybR1WTnwsbVlXP6Ht2j06RLWSqmxKZKuocuAu40xv+nnvpjayaX3rmUF6dbIoYP1PpZMzSIr\nKZ7tFdbooem5KeyrbccYw6ayZoIhQ3VLF1nJ8f2+rlJKjWaRtAiuAPaIyEMi8rFYGzY6kAK7qwgg\nLzWB3NQE2uy6wPTcFEIGOnqC7Ki0kkODLyYnXCulxoGjJgJjzM3ADKwho9cB+0XkL9EObKRN7JUI\nclMTyLPnFiTEuSjIsK43+nrYW2vVBxratWtIKTU2RfTr3hjjF5GXAQMkAlcTAxvTDCY//fAm9rmp\nCeTZBeP8NC+pXmuo6YZDTfiD1gzj+narRfC71/fgcbu4/rTJZGpXkVJqDIhkZvFlIvIAsBe4FvgL\nMCHKcY24hDg3OSnWF3luqtdJBHmpCaTam96vOdjoPL7R10Nbl5/fvb6XX72ym5/+wxpx2xMIOaON\nlFJqNIqkRvBZ4FlgtjHmc8aYl4wx42IQfbhOkNerRZCXlkBqgpUI1pU04vW4yElJoKG9h5L6Due5\n1fYqps9sLOeqZaudFoNSSo02kdQIrjPGPBteflpEzhaRZdEPbeSFRw71rhHkpR7uGtpf52NyVhK5\nqQk0+Lo52GBNNpuak+wMJz1Y30HIQG2rJgKl1OgU0eqjIrJARH4lIiXAz4FdUY1qlJiZn0JBuhev\nx92nRZBidw0FQ4ZJGYnkpMRT395DiT3reEFRBo0+a55Bpb2aaXg/ZKWUGm0GLBaLyCysUULXAfXA\n3wExxsTMJLKj+fqFM7npzCkATM5KwuMWpuemODUCgMLMJNq6/JQ0+DhY72NiupeCDC9NHT2EQsbZ\n06CpI2YmYSulYsxgo4Z2AW8BHzPG7AMQkX8flqhGCa/H7WxVmZfm5a3vXkh+WgKhXlsRTMpMpK7N\nRUN7DwfrfUzNTSYrOYFgyNDWFaDS3tOgSVsESqlRarCuoY8DVVgb198rIkuxNq+PiIjcLyK1IrKt\n17GfiEiFiGyyL5cff+jDb0K6FxHB7RJS7IJxYWYi2SnxdPQE2VnVypTsZLLCK5m2dzlF4yZdgkIp\nNUoNmAjsAvGngTnASqx1hvLsBegi2YfgAeDSfo7/1hgz3768dDxBjwbhRDApI5Fse75AdyDE1Byr\nRQCwq7qNoN180K4hpdRoFcmoIZ8x5hFjzBVAIbAR+F4Ez1sFNB7tcWNVuE5QmJlEdvLhyWeXnjSB\nLHtJ6+2Vh3c2669Y3B0IUtemo4mUUiNrwEQgIilHHjPGNBlj7jHGLB3oMRH4uohssbuOBtzdRUS+\nKCLrRGRdXV3dcbxNdKV640iIc5GTEs+SaVl89YLprPnPpRRmJpFlT0TbVmFNJPN6XE6N4LH3D7G3\nxlqW4qf/2MHiX76uexsopUbUYC2C50TkNyJyrogkhw+KyDQRuUVEXqH/rp/B/BmYBszHqj/0t6Ip\nAHbCWWSMWZSbm3uMbxN9aYkeJmUmIiKkej185yNzyLfnGhzZIpg9IY3GDj8dPQFue3orf3vX2vZy\nzYEGAF7eWjUCZ6CUUpbBagRLgeXAl7A2p2kRkQbg/wEFwE3GmCeP5c2MMTXGmKAxJoS1//Fpxx/6\nyPrG0pn87MqT+r0vMd5NosdNo6+HzCQPk7OSaO7o4UCdNc+gwp5bMCXbyq9PbSgfnqCVUqofgy46\nZxdzh6ygKyIFxpjwz99rgG2DPX40WzB58D2LU7xxdPqDLD0hn+R4N02+HvbXWds3VDSF5xZY3UXv\nHWik0dej+xkopUZE1PYWEJFHgfOBHBEpB34MnC8i87FWMS3Bam3EpHAR+KPzCthU1kxrV8DZ0rKi\nuRNjDE0dftwuIRgy1LV1ayJQSo2IqCUCY8x1/Ry+L1rvN1qdNSOHUnsNovWlTYC18X1rZ4CG9m6m\n5iSzr7bdGVW0YlcNs/JTKcxMGvA1lVJqKEW01pA6dk98+QzuufFU4uNczr4E60qb8LitOXmljT5a\nuwJMzbHqBM2dfroDQb74t/U88HbJSIWtlBqHItmPYLqIJNjXzxeRW0UkI/qhjW2Lp2RxyYnWtg25\nKdY8g2DIsKg4Czg8omianQhaOvwcrPcRCBmdfKaUGlaRtAieAoIiMgO4BygCHolqVDHm9GnZfPaM\nYgCuXjARgK32HINpueEWQQ97aqxicmuXJgKl1PCJJBGE7I1orgH+aIz5DtbwURUhl0v42VUnsecX\nl/HJRUUkxLmcXcsKM5OIcwktnX5nollrpyYCpdTwiaRY7BeR64CbgCvsY57ohRS74uOsvDspM9EZ\nQZSVHE9GkofmDj8N7VbBuLVLZxorpYZPJC2Cm4EzgF8aYw6KyFTgoeiGFdsWFGXSEwwBViJIT/TQ\n3OlnT23fFkEwZOjo0aSglIquSBad22GMudUY86i9NlCqMebOYYgtZp09M9u5nplkJYK61m5KG6w9\nj8OJYNnKfZz7P2/QEwiNSJxKqfEhklFDb4hImohkARuAe0XkruiHFrvOnJ4DQGpCHPFxLjKS4tlS\n0UwwZJiWm0xbd4BgyPDcpgrq27vZXN7sPLel048/qIlBKTV0IukaSjfGtGJtVPM3Y8wS4KLohhXb\n8tO8zMhLceYXZCR66PJbX+6Liq2lKzaXN7PfXpto9d56AIwxXPq7Vdyz6sAIRK2UilWRJII4ESkA\nPgm8EOV4xo1bl87kc/Z+yOlJVu3dJXBKkTVF48n11kJ0kzISeXuflQhqWrupaumipN43/AErpWJW\nJKOGfga8ArxtjFkrItOAvdENK/ZdecpE53pGotUymJKd7Ew+e21HDbPyU7johHzuWXUAX3eAfbXW\nPINmHV6qlBpCR00ExpgngCd63T4AXBvNoMabDLtFMCMvhbREe7/jtm4WT8nk5MIMAiHDgTqfs3pp\niyYCpdQQiqRYXCgiz9gb0deKyFMiUjgcwY0X4UQwKz+VNO/hKRpTspMpykoEoKypw2kRtOgSFEqp\nIRRJjeCvwPPARPvyD/uYGiLpditgZn6KUy8AmJKTTFGWtQppWWOvRKAtAqXUEIokEeQaY/5qjAnY\nlweA0bd35Bi2YHIm1yyYxLkzc0nzHu6tm5qTTJrXQ3qih7KmDqdrqLmzZ6RCVUrFoEgSQYOIfEZE\n3PblM0BDtAMbT9ITPfz2U/PJTI4nOT4Ol7VStbOVZVFWItsqWqlt6ybNG0eXP0SXPziCESulYkkk\nieAWrKGj1Vgbzn8C+FwUYxrXXC4h1eshJSGOnBRrNFFRZhKbyqxJZRfOyQOs2cehkBmxOJVSsSOS\nJSZKjTFXGmNyjTF5xpir0VFDUZWWGEdxdhIiVtMgXCdI88Zx/mwrEXznyS2cfecKfN2H1yJq7fJj\njCYHpdSxOd4dyr41pFGoPhZPyeK8WYfLMEWZ1sihc2blkm23Et7cU0dlSxc/em47YI0kWvLL5Ty8\n5tDwB6yUGtOONxHIkEah+rjrk/P57qVznNuT7VrB+bNynRFGYU9vLKc7EGRHVSud/iAPvlMyYKvg\nvQMNfVoQSikFx58ItP9hGJ05PZufXnkiV5wy0ZmFDDC3IA1joLa1mz32pjZ7a9vZcKjpA69R19bN\ndfe+x1MbyoctbqXU2DBgIhCRNhFp7efShjWfQA0Tj9vFTWdOwetx92kRnDXDWs66qqWL3TVtpHrj\n8HpcvLilGoBAMORMPitt8GGMlRAisXpvPQ+9VzrEZ6KUGo0GTATGmFRjTFo/l1RjTCRrFKkoSPXG\nYdeQOXOGtZx1VUsne6rbOKEgjem5KRyot+Yb3Lf6IBf85g2CIUNZk7XXQVNHZHMQHnm/lGUr9g39\nCSilRp3j7RpSI8TlEtK8HhLiXCycbC1ZHW4RzM5PZUp2srPBzdv7G2j09VDX1s2hhk4AmiJcnqKl\n06/1BKXGCU0EY1BGkoepOcmkJ1rzDTYeaqKtK8CsCakUZydR1tiBPxhii72hTWVLp9MiiHSdouYO\nP76egA5HVWoc0C6eMWjh5ExnsllBupe37I1r5hakkhDnIhAyvLu/gWb7S7+quYuyxmPrGmru8BMy\n0OkPkhSv/5koFcv0//Ax6Lefmu9cn5DuZW9tO0nxbuZNyiC8i+Vzmyqdx1S1dFLeZHUNNfdqEWyv\nbOH1HbV846KZH3iP8L7J7d0BTQRKxTjtGhrjCtK9gDUJLT7OxZRsaxby0xvLSYhz4fW4ONTYQVVL\nOBEcbhH8fW0Zv319DzWtXfzHE5upb7dGFPmDIdrs+oCvW9c0UirWaSIY4yakW7OOw0NJc1OtHc6M\ngYvm5jMm6e8iAAAfY0lEQVQxI5H1pU2EDEzOSsLXE6QnYDUbDth7Ir+0tYon15c7W2K29lrmWgvG\nSsU+TQRjXKG9/MSZ062hpCJCpr2nwS+vPomJ6YnsqGoFrFYDHG4VhJe13njIKirXtHYBffc7aB8g\nEby8tYq7XtszpOeilBoZ2vk7xl15ykTyUhM4aVK6c+z5r52Nx+0iIymegnQvxsCENC/nzc7lqQ3l\nfOXhDczMS6Gqxfri31hmzUSubrG6hnrvidzR038ieG5TJe/sr+dbF8+KKM59te1Mz012FtJTSo0e\n2iIY47wet7MiaVhRVhIT7NpBQYbVYrj0pAlkJVkjjdaXNvHY2jLn8WWNVv2gps1uEXT0bhH0XyOo\nb++mtSsQ0b4IB+t9XHTXm7y5py7S01JKDSNNBDEuvHLp5fMKnL2RB1LT8sGuoYFqBHV2YTmSJSsq\nm61EU2aPXFJKjS7aNRTjrjhlIhlJ8SyekklF8+EvYo9bnALywXqraFxt1wh6jywaKBHU2wmgtq3b\n2S9hII0+6/Ua23WLTaVGI20RxDivx83Fc/PtIrLVNZSZ5OETpxZx0qR0p9gM1iqmxpg+NYL+isUd\nPQF8PVaXUCQtgvAktkgnsymlhpe2CMaRpHg38W4X84sy+PlVJxI0hh88sw2w9kUua+yk0ddDc4ef\n1IQ4AiHTb4ugvu3wF3qdXVcYTJPPSiwNPk0ESo1G2iIYR0SEG88o5volxcS5XSTEuZmQZhWVFxRZ\nC9jVtHbT2uknPclDckLcB4rF9e3dlDd3OLePpUXQ6LMeu6emzdk/YTBv7qnjqmVv4w9Pl1ZKRYW2\nCMaZ//rY3D6389OsCWgLJmfw/OZKalq7aO70k57oIc4VoLXTz69e2cUzGyqYnJ3E2pImJmZ4nefX\nHpEIfvf6Hk4uTOfCOfnOsXCNoMGuEfzgma0APPHlMwEwxvDG7jrOnpmDx334t8nag41sLmumprWL\nwszB6xDHo6XDz6q9dVxxim6vocY3bRGMcxfMyeMTpxY6Q1Afff8QGw81kZuaQHJCHC9urWLZyv1M\ny02htq2bjESPM9w0JyWeV3fUcOGv36C0wce2ihZ+9/pefv7CTkKhw6uWHm4RWP8eauygotcIog2H\nmrn5gbV91kcCaPAdLkhHw9/XHeLrj26kNoLuLaViWdQSgYjcLyK1IrKt17EsEXlNRPba/2ZG6/1V\nZAozk/j1v5zClOwkPr24iNd31pCZFM/3LzuB5ASrwej1uHjwltNY8e3zufmsKc5z50xIo9HXw4F6\nH3e8vIs/v7kfsOYNrLaXq4C+xeKeQIjatm5q27qdZLHmYANgzW8A+PUru/nvl3dSb7cgalsjTwTf\neWIzD75T8oHjxhiu/fM7vLDlcLIJJ7TeNY8P41ev7GJtSeOQvJZSwymaLYIHgEuPOHYbsNwYMxNY\nbt9Wo4CIcMe1J7P+hxfzz2+ey+wJqaTYiWB2fipulzUj+NRia5mKrOR4Ur3W/ZlJHl7eVs2LW6q4\n+awpZCfH89jaQ85rh4vF/qBhb20bxkAgZHhjTy3ff3or7+63EsFGe6/lZzdV8NqOGhrCcxXaB04E\nDe3d3PbUFtq7rcltz2ysYPmuWrZVtPDEusOT5uraullf2uS8F0D5UXZte2dfPR/57aoBZ1f31h0I\nsmzlfp4+jj2ht1W06L4PakRFrUZgjFklIlOOOHwVcL59/UHgDeB70YpBHbvM5HjnelK8G7B++YfN\nL8ogziXkpMTT1mV9QS67YSFv76tnWk4KV86fSHtXgBW7aunoCbC/1kejr4eclHjq23vYVtHivNY9\nqw7w3gHrF7TbJeypaaOmtYvypk4SPW4CQevLsa61b9fNj5/bRncgxB3Xnswbu+t4bG0Z583KpSAj\nkUDIUNncyf2rD/LCliquXVhIa5ff2ZgnvKwG4MyrGGg00zv7G9hd08aemnbmF2UM+ncLt14ONXYM\n+rgjbSpr5uplb/P4l87gtKlZx/RcpYbKcNcI8o0xVfb1aiB/oAeKyBdFZJ2IrKur06UJRkKnPVdg\nTkGqcywx3s3C4kwmZyXzkytP5Dsfmc0Z07L5zkfmcO2phXjcLuZPzqDB18N3n9zCFXevptMfZFpu\nCgDbKlqd19pQ2uxc/+i8AkIGnlxv/aLu9AedL+reNYLOniB/X1fGs5sq6A4EKbW/eDeVNztJprK5\nk9LGDnqCIV7ZXs3Cn7/G8p21zn1gdRWF92hoHKDFEf5S31/b3ud4R0+A1q6+O72FR0+Fu5vCuvxB\n7np1N509QV7YUulM3gsrsW+XNPQ9rtRwGrFisbHawgO2h40x9xhjFhljFuXm5g5jZCos/EU5e0Jq\nn+P/95lT+c2/nMKMvBS+esGMDywkFx6K+sKWKufYjDw7EVQebhH0BEPMzk/lhII0bl06AxF4+L1S\n5/6gXUPoPUT1zT11dPlDdPlDbDrUTKn9Bbq57HAi6OgJstNecfWlbdWEzOGNesItguYOPx12omsc\nYPtOJxHU9U0Etz66iavvfttZzrt3jJXNnQRDhrf21vHj57axem89f1ixj39ur+Ibj23i16/s7vNa\n4XhqWiIvWD/0Xik/+8eOfu97fG1ZnzpIS4efWx5Y6+xHcaTOniD/+cxWZy8KNT4NdyKoEZECAPvf\n2mF+f3UMwjWA3l1DYHUfpQ+ybtGs/BQSPe4+x2Y4LYIW0rxx2CUHLps3gZe/cQ4z8lI5f1Yulf18\nIYYLy99+fDP/88ouUu3nv7O/gdIG68t6a3kLm8tbnFpG+Es+vMdCuHXR0umnoyfgJDk4PL8BrJZC\nRXMnLR1+Z3vPXdVt/PfLO6lo7qS+vZuVu2s5UO/j70fUIMCqfVS1dPLsxkoefLfUSUgvbqkmGDK8\nuaeOnkCIYMjQ0ul3lv6uao0sERhjuGfVfh55v9RJlL397vU93LvqgHN7/aFGVuyq5b0DDR94LMC6\n0kYeWXOIFbsi+19x5e7a41o88IUtldRGeI5q+A13IngeuMm+fhPw3DC/vzoGd1+/kGXXLySrV90g\nEnFuFycXWstiJ8RZ/4kVZycR73bhDxqKspKcDXSm2wkC4MYzigGYlHF42Yvs5Hhq27pYW9LIUxvK\nOVjv45oFk5g3KZ139tdzqLGDNG8cPrsVcMa07D6xNPbT/1/Z3EWFPSnOJX0f86WH1nPWHSv4+J/f\ndmoHK3fX8n9vHuDBd0p4cUsVwZBhak4yd6/Y64x86j0EtayxkwP1ViviLTsRrbK/PNu7A6w52MBd\nr+1m6W/ecBLSkS2CB94+yFl3rCBgT6br7AlijGFfbTtljZ10+UPsqGzliXVlTqG5uaOHypYuShoO\n1ykO2dfDS4wfKdyVFd6k6Gh+8+pu7nx514D3P/b+IR5Zc6jPseaOHr72yEYefLckovdQwy+aw0cf\nBd4FZotIuYh8HrgDuFhE9gIX2bfVKDUh3ctHTy44rufeeEYxXzhnKn+8bgEAcyem8bULZwDWxLJ8\ne0ZzuMsI4LxZecyZkMoVp0wkzv5lf0JBGvXtPTy/uRKvx8X6H17Mj684kXNm5rK+tIlGXw9XzZ9E\nvNvFhXPy+PEVcxlIeBZ17z2cZ+SlUN3SxQ+e2cqqPXW8uqOGoqxE9ttfjMXZSYQH9LyyvZrH15Ux\nZ0IqXzlvOjWt3aw/1MSPntvW54u0rKnDqQVssIfE9gRDxLutrUOf3VjJY++XUd/ew/pSq1hefcSv\n5Re3VlHR3MnOqjaafD0s+sVrPLXBGhEV9sPntvGdJ7ewvrSJzz+wloftL+CWTj9NzpwN6zyrB+ga\nCnd/HTii+8sY02c58rCq5i4O1vv6zBMBWLmrliZfD/e8dYA/LN/bZxRU+D1K6o+tkD6QVXvq+M2r\nu4/+QJsxhu88sZnVe+uP/uBxKmqJwBhznTGmwBjjMcYUGmPuM8Y0GGOWGmNmGmMuMsbooOsY9bGT\nJ/KDj87lkhMnUHLHRylIT+Qr509n6Zw8bl06k/w0LyIwNSfZeY7bJbx06zncdtkcJ1GcUJBKMGR4\nakM5F87JIys5HrdLuGr+RMLfRWfNyGb9f13EfTctYnpuCvF2KyTcsgj/Gx6VU9XcxfsHG0lP9DAl\nO5mNZc08vOYQX3xoHQC/uHqeE9MF9kS74uwkShs62F7ZyufPnspi+7V++Mw2/vZuKS9trWJabjIu\ngS3lzTTbX6KBXl+Y0/NSuHZhIU9tKHdaG03246p7tQjauwPOrnHvlzSyrrQJX0+QJ9eX8fK2aqbl\nJiNi1UUAHninhOW7avn963ud1zho107CX8JVR7Q42rsD7Kttc0ZTHaj38Ze3DjhdSHf8cxen3f46\nW8sP13S6/EEafD10+oNO4qpv76bR18PND6zl7pX7KKn3Ud3a1Wel23AMRxbKB7PhUBNfe2SD0yLq\n7dH3D7Fs5b4+NZpH3z/kFN6P1NTh54n15Ty7qSKi935zj1XfOVb//fJO/vr2wWN+3migM4vVsPG4\nXdz3ucVcv2QyZ07P5oLZeXiPqCW47JZAgb2xzrxCa9hmdyDEpxdPdh43Mz/V6X6anJVMqteDiOBy\nCQXpXlIT4phn79r2MbtVs2Sa9eX9jy2VvLqjxprzkBLv/OLv8oeYNymd82blOgnq+iWTOW9WLsuu\nX4gInFqcybULC5mSnUROSjy77TWTAiHDxPRECtITeWN33z70PLsbbM6EVH7w0ROYnptMQbqVCK2/\ni9Dg66E7EOR/39zP957aQiBkcLuEdSWNrLNbDe8dsJbc+OzpxUzNPpxAX9pqFeV7giGn9hIuoofr\nHNWtXfz6ld2s3FVLXVs3n/jzO1z++9VOgf1gvY9fvrSTP67YS0m9j/tXH6Q7EOLfHlnvzKOo6dVq\nOVDnY+WuWpbcvpzXdlQD8NymCic5ry9tIhAMsbOq9XCLoMEX8XyJ/3tzPy9sqWJLRQt3vLyL2rYu\nSup9NLR3s7umjZCBQ43WObZ0+Pn+01sH7HoK/y32HjH6ayAvbLbqO0eODAt7ZM0hrvjj6g+0ip5c\nV/6B2fHHq6XDz/2rDw7b/BJNBGpE3HzWVO7/3OIB7y/ISCQ53s1lJ03g7usX8MZ/nM+5s/qOHvvs\nGVPITPL0aVWAtcfC1NxkirKslsBl8wq497OLuHZhITkpCby1t56CdC9fPHeaszT3hDQvs/JTuH6J\nlWyumj+R4uwkZuWn8uAtp3HSpHT+8tlF3H39AlwuQURYVNx33H9eagJnTM92up3C6zgtPcFqVczM\nTyEpPo5nv3oWz371LKbYX+ZzC6xi/Itbqrjj5V28uKUKr8fFpSdOYG1JI+tLmshJseLMSYnn06dN\ndkZyZSfHEzLW7G+ARcVZuAQO1ndgjOnza3zZG/v47et7+OGzW9lb205PMERpQwfJ8W6CIYMxsPZg\nE7e/tBOP28Wd186jrLHTaSX0blUcqG/nzT11BEPG2e0uPJdCBN4/2Mi3n9jMZb9/i+ftL8eOnmCf\nEWDrSxuZ/7NX2Wsn02Ur9/HPbVW0dPpZuctKpnev2Mf/vrmfpzdUcN297/HdJ7c4AwTC3XF7a9v6\n3O6t999gX00b3YEg3YGg/fh2bnlgrZMsw8LneeSw4bDlO2vYWtHC5vLDw59bu/w0+Ho4UNce8Ze3\nPxji8XVlfVo2YU9uKOdnL+xge2VrP88cerronBqVrltcxLxJaXjcLj52cv+Lwn3i1EKuWTDJGSkU\n9ourT8IfNGyraCElIY7pucnOhLDvXjqbyuZOrl1YSFJ8nFMIP3VKJsuuX+i8xq0XzuSrF8zo87pL\nT+g77eX0aVn8c3s1V5wykX9sriQ3NYFLT5rAk+vLcbuEs2fk2l1a+UzPTeGq+ZMASPV6SPV6mJWf\nwsF6HwsmZ7K5vIVvPb6ZSRmJfOm8aYgICW4XL26tosHXw7+ePZXKli4utFtRp0/LZsOhJj61qIg/\nrNjHxxcW8s6+es6ZmUNVaycl9T7q261unMwkj9MFtaW8hW0VLdxy1lQeff8Qvp4gZ87I4bUdNSTE\nuegOhHh1Rw3/evZUrpo/if96djtrDjQyvyizz/pQ+2vbWW/PBA93YwHEu12cWpzp1CvAGnXlEggZ\nKyHl2d1+j75fRnOHnz+/uZ/br5nH717fQ0F6Iv92/nR6giE8bnFGMz21vpyqli6qW7ucFly4qyn8\nSz9coA97bUcN3358k/O5+XqCfPqe94hzCU98+Ux++eJOVuyqxetx8acbTnWeV2nXU/bVtrNg8uFV\ncL7wt3VMzkpyhkCv3FXLgsmZ7KpudebctHYFaOrwRzTA4vlNlXz3yS0kxLkobejgtKlZpCTEUd/e\nzQ47AZQ0+PrsRx4tmgjUqHTmjBzOnJFz1McdmQQAiu1f2tNykll6Qh6p3sNDXT+5qKjPY7PtX9rz\nC/vOHHa5BBcffO3erl9SzMLiTNq6Ak4imF+UwdyCNLoCQWbmp9jxJHHx3A/OnZw9IY1Xttf0mbV8\n57Unc/ZM67z9wRBv7q3jxS1VLJmazUW9XuOzZxRzw5LJrDnYCCv2ceb0bH525Ym4XcL7JY28s7+e\n21/aCcDiKVm8uqPGeW7IWMX8g/U+lu+q5YLZeazcVcu/njOV+1YfpCcQ4qYzp+D1uDmlKJ2XtlXx\n4LslZCdbLZwZeSlsqWhxvqwAMpI8NHf4mZ6Xwr9dMJ3CzETOnpnDvW8dYFtFK6cUZbDxUDOlDR0s\nmZZNlz/IK9uqiXe7eH5TJRfMzsMftH69//LFnczKT2FyVhKv2xMBw1/24SQg0qtFUGPdV97USWVz\nJ26X0BMI8a3HNzmfTVg4aT2xrozlu2qZlpvMS1ur2VTWzPyiDIwxzqTDfXXtNLR3c9Wyt/nplSey\nclct8XEuZ2jyit21LJ6axY33vc+SXrPCD9a3k5Vs3fZ1B/jC39bx9Qtncsb0bB5fW8aBeh+3XTaH\nZzZaNYsn15fz1t56LpmbT3t3gK3lLUy061oD1T2GmiYCFbNcLumTBPoz2d5m83iWd4iPc3FyYQZd\n/iAfPbmAc2flIiL86YaF+HoCZCTFW1+OvYbI9vaRE/PZeKiJ82fncnJhOp89Y4qTBMCqqfzh0wu4\n5awpLJzcd31GESHOLZw5PZv7P7eI82blOUnxE6cWsmzlPucL8KwZOc5oqMlZSXjj3BRnJ3PWjByW\n76rl5MJ0Xrj1bKblpOCz958o6vV3WbbSWkywormT9EQP84synBngM/JS2Ffbzvmzclm1t565BWmc\nMzOXc2Za3Xg7KlvZVtHK6dOy2VrewoF6H796ZRd/fbuEjp4gt18zjx8+u5Wf/mM7YC1r0tYd4L6r\n57G2pJHXd9YyKz+FPTXtpHnjaO0KEOcSTpqU3qtFYHUNGQMf/9M7xLmFRcWZBIKG/LQEalq7mZ2f\n6tRzAL7/9FYmpHl54ktncNadK3hmQznzizJo7vDT5be6avbXtvPqjhrKmzr51Su7CYQMATsJnD87\nlzd21/H1RzcCWAnZdqDO56zJ9cr2at7Z30Cjr4eXbj2Hv75Twq7qVi45MZ+399cjAm/Zo5nWlTbR\n2ROk0x90Yj04RCOtjkYTgRrXTi3O4q3vXnDUfZcH4/W4+3QrTelVs7jtsjkDPu/Eiek89PklADz/\ntbP7fYzbJc6XSn9EpM/eDwBXzZ/EVfMn0dzRQ3lTJ3FuK0GcUpjBXZ+c7xSpP31aESneOE6cmObM\nDv/JlSf2ea0lU7NZtnK/071UkO7l3y+exf66dvbXtnPj6cX8+PntzMxP5cvnT3daDWFnzsjh/1Yd\nYGpOMgsnZ/L3tYdo6wowpyCVWfmpfHJRIct31li/znOS+fw5U/F1B5xukofeLeV7l87h8w+u45xZ\nuZTU+/DbM9LDQ2n31bYzMy+FvbXtzmim8qZObjqjmJ5giEffL+PkwnSaOnrITU3AJcLWiha+dfEs\nslMSOGdmLq/uqOHjCwud+oDX42Jfbbsz6mtXdZv997YSzv9cezK/X76X5zdVckJBGjurWslOjqe1\ny+8kqC5/kGc3VRLvdrGruo3H1paxu7oVY+ALD65DgM+cXszf3rVm0x8550XEal0MB00Eatz7MElg\nNMtIiicjKZ62Lj9ej4sl07KdobUASfFxH+gqO9JZM3K489p5FGYmccNf1lCQ7mVSRiJPfflM2nsC\nNLb3EOcSFkzO+MAMdICzpmfzo4/N5fJ5BSwoyuCjf1xNfJyL+29a7NQKbjh9Mst31bKwOJMblhQ7\nz507MY33/nMpoZDho/MK+NSiInJSEvAHQ6w52MDf13Vz5d2rqWrp4qsXTHe6j6blJlNS7+OWs6ey\nqayZR98vozg7ibs+OZ/MZA/763y8vLWKa08tBOCSufm8tqOGq//0Nh6X9fc5fVo2q/bUUdHcSXZy\nPA2+HvJSE5iem0JNWxd5aV5+ec08fnH1STy9oYJvP7GZabnJNPp6OFjv44l1ZXz3qS0AfOW86byy\nvZrbX9pJyECaN44GXw+3Lp3JJXPz+du7pZw7K9eZdDg1J5mD9T4WFWc681miTROBUjEu1evhjf+4\nwJnNfSzcLuFTiycTChkmZSQyK98areRyCWleD2leDxt/dPGAXXBxbhe3nD0VsIb83n/TYoLGOEkA\nrImEn1pUxL8sKuz3NVwuYdkNC/scm5abTE8gxFt765kzIZXL5xXwzIYKJmUmcue1J7O3tp3i7GRS\nEuKYlJHIaVOzne6/Eyemc2WvXemWnpBPnEswWENwAT6zpJi9Ne1UtXTyg4+ewLce38wpRRn89MoT\nnRoBWC2y06dbs9mLs5PJSo7n/YONbDzUTFFmEgXpXq5fMpmclAR+9sIOXAL/+5lTeXNPHbdeOAO3\nS/jRx+Zy2bwJXPLbVWQnx/OZ04u5Z9V+LpiTx9qS3bR0+Add0mUoyFhYB33RokVm3bp1Ix2GUuNa\nS4cfb7y11/VotPFQE5lJ8X265iK1tbyF9u4A1937ntWV8/NLcbmE7kCQeLeLmx9Yy8cXFvZJIL39\n4oUdXDgnj4ykeD75f+/S3h3g/31+iVPzaenwc9rtrzM9N4WXvnFOv6+xbOU+0hM93LBkMsGQYcWu\nWr740Hqe/epZR10GfSAist4Ys+hoj9MWgVIqItH+VfphLZh8/BsezitMxxjjLCkSntgYTnoP3Hza\noM//Ya+9wB+8ZTHrS5s4a8bhda/Skzzcfs08MpMH/hv2Hq4c5xam56UwZ0Iq3f7ggM8ZKtoiUEop\n2+s7amjr9nPNgv67qcYabREopdQxuqif+R7jgS4xoZRS45wmAqWUGuc0ESil1DiniUAppcY5TQRK\nKTXOaSJQSqlxThOBUkqNc5oIlFJqnBsTM4tFpA4oPc6n5wD1QxjOaDQezhHGx3nqOcaG0XKOxcaY\n3KM9aEwkgg9DRNZFMsV6LBsP5wjj4zz1HGPDWDtH7RpSSqlxThOBUkqNc+MhEdwz0gEMg/FwjjA+\nzlPPMTaMqXOM+RqBUkqpwY2HFoFSSqlBaCJQSqlxLqYTgYhcKiK7RWSfiNw20vEMFREpEZGtIrJJ\nRNbZx7JE5DUR2Wv/e/z79o0AEblfRGpFZFuvYwOek4h83/5cd4vIR0Ym6mMzwDn+REQq7M9yk4hc\n3uu+sXiORSKyUkR2iMh2EfmGfTxmPstBznHsfpbGmJi8AG5gPzANiAc2A3NHOq4hOrcSIOeIY/8D\n3GZfvw24c6TjPMZzOhdYCGw72jkBc+3PMwGYan/O7pE+h+M8x58A/9HPY8fqORYAC+3rqcAe+1xi\n5rMc5BzH7GcZyy2C04B9xpgDxpge4DHgqhGOKZquAh60rz8IXD2CsRwzY8wqoPGIwwOd01XAY8aY\nbmPMQWAf1uc9qg1wjgMZq+dYZYzZYF9vA3YCk4ihz3KQcxzIqD/HWE4Ek4CyXrfLGfzDGksM8LqI\nrBeRL9rH8o0xVfb1aiAWNl8d6Jxi7bP9uohssbuOwl0mY/4cRWQKsABYQ4x+lkecI4zRzzKWE0Es\nO9sYMx+4DPiqiJzb+05jtUdjalxwLJ6T7c9Y3ZfzgSrgNyMbztAQkRTgKeCbxpjW3vfFymfZzzmO\n2c8ylhNBBVDU63ahfWzMM8ZU2P/WAs9gNTNrRKQAwP63duQiHDIDnVPMfLbGmBpjTNAYEwLu5XCX\nwZg9RxHxYH1BPmyMedo+HFOfZX/nOJY/y1hOBGuBmSIyVUTigU8Dz49wTB+aiCSLSGr4OnAJsA3r\n3G6yH3YT8NzIRDikBjqn54FPi0iCiEwFZgLvj0B8H1r4y9F2DdZnCWP0HEVEgPuAncaYu3rdFTOf\n5UDnOKY/y5GuVkfzAlyOVdHfD/xgpOMZonOahjUCYTOwPXxeQDawHNgLvA5kjXSsx3hej2I1p/1Y\nfaifH+ycgB/Yn+tu4LKRjv9DnONDwFZgC9YXRsEYP8ezsbp9tgCb7MvlsfRZDnKOY/az1CUmlFJq\nnIvlriGllFIR0ESglFLjnCYCpZQa5zQRKKXUOKeJQCmlxjlNBEopNc5pIlBKqXFOE4FSx0FEFtuL\ni3nt2d7bReSkkY5LqeOhE8qUOk4i8gvACyQC5caY/x7hkJQ6LpoIlDpO9hpWa4Eu4ExjTHCEQ1Lq\nuGjXkFLHLxtIwdqlyjvCsSh13LRFoNRxEpHnsXa+m4q1wNjXRjgkpY5L3EgHoNRYJCKfBfzGmEdE\nxA28IyIXGmNWjHRsSh0rbREopdQ4pzUCpZQa5zQRKKXUOKeJQCmlxjlNBEopNc5pIlBKqXFOE4FS\nSo1zmgiUUmqc+/9pFmYIjkmNOgAAAABJRU5ErkJggg==\n", 1272 | "text/plain": [ 1273 | "" 1274 | ] 1275 | }, 1276 | "metadata": {}, 1277 | "output_type": "display_data" 1278 | }, 1279 | { 1280 | "data": { 1281 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8nFW9x/HPb7I1W9OmTbo33Sm0pQtllUWoInBRQFFB\nQVC06PWKXL2i6L1Xr4rXXbmgCAKCrCKLICBbZSlQCm2hG91buqbZm6RJs87v/vFMQijJzDTNZP2+\nX6+8knnmmef8TqadX845zznH3B0RERm4Qj0dgIiI9CwlAhGRAU6JQERkgFMiEBEZ4JQIREQGOCUC\nEZEBTolABgQze8HMvtgF11lrZh/sgpB6hJldbmYv93Qc0rsoEUiPMbN3zOyAme03syIzu8PMsno6\nrmjcfYa7vwBgZj8ws7sP53pm9mEze97Mqs2szMzeMrNvm9mgLglYJA5KBNLTPuruWcA8YD7wn4d6\nATNL7vKouoGZfRJ4ELgXKHD3YcCngbHAuA5e0yfrKr2bEoH0Cu6+G/gHMBPAzHLM7DYzKzSz3Wb2\nYzNLijx3uZm9Yma/MbMy4Adtjt1oZpVmtt7MFnRUnpl9wczWmVmFmT1tZgWR4yeZWamZjYs8nh05\nZ3rk8Ttm9iEzOwv4LvDpSItmpZl90syWH1TON8zs0XbKN+DXwA/d/Y/uXh75PWxw96+5+6bIeT8w\nswfN7G4zqwIuN7PjzGyJme2L/H5uNLPUNtd2M7vKzLZG6vILMwsdVP4vI/XaZmZnH9q7Jf2NEoH0\nCpEP3nOANyOH7gCagCnAXOBMoG0f//HAVmAEcF2bY1uA4cD3gYfNLLedss4j+BD/OJAHLAbuA3D3\nV4GbgTvNLB24G/gvd1/f9hru/hTwE+Av7p7l7rOBx4CJZnZkm1MvBf7cTpWPIPjL/6Fov5eI8wha\nDkOAe4Bm4N8j9TwRWAD860GvuYCghTUv8vovtHnueGBD5PU/B26LJCYZoJQIpKf9zcz2AS8DLwI/\nMbMRBEnhanevcfdi4DfARW1et8fdb3D3Jnc/EDlWDPzW3Rvd/S8EH3b/0k6ZXwb+193XuXsTwQf6\nnJZWAfADIAd4HdgN/C6eirh7PfAX4BIAM5sBTAAeb+f04ZHve1sOmNn9kb/ya83s0jbnLnH3v7l7\n2N0PuPtyd38tUvd3CBLXaQdd/2fuXu7uO4DfAhe3eW57pBXSDNwJjCJIqDJAKRFITzvf3Ye4e4G7\n/2vkQ70ASAEKIx+M+wg+7PLbvG5nO9fa7e9dRXE7MLqd8wqA69tcuxwwYAyAuzcStEhmAr866Jqx\n3Al8JvIX9qXAA5EEcbCyyPdRLQfc/SJ3HwKsAJLanPueuprZNDN73Mz2RrqLfsK7iaW91xz8e2hN\nPu5eG/mxVw/SS2IpEUhvtBOoB4ZHksQQdx/s7jPanNPeh/OYg7o4xgN7Orj+lW2uPcTd0yPdQpjZ\nGIKupT8BvzKztA7ifF8M7v4a0ACcAnwGuKuD124gaG18vIPno5VzE7AemOrugwm6uQ7u2mk72NzR\n70EEUCKQXsjdC4FnCD6EB5tZyMwmm9nB3R8HyweuMrOUyB05RwJPtnPeH4BrI103LQPTn4z8bASt\ngduAK4BC4EcdlFcETDh4IJZgTOBGoNHd271n393DwDeB75vZl8xsqAWmErubJhuoAvZHBrG/0s45\n34pccxzwdYIuK5F2KRFIb/U5IBV4G6ggGCwdFfUVsBSYCpQSDCBf6O5lB5/k7o8APwPuj3StrAFa\n7py5iiCh/FekS+jzwOfN7JR2yvtr5HuZma1oc/wugm6lqHMMIuMYnyIYU9gZifsB4JY2127PfxC0\nNqqBP9L+h/yjwHLgLeAJgsQm0i7TxjTSH5jZ5cAX3f3kXhBLOsHA9byW20C7uXwn6Dba3N1lS9+k\nFoFI1/sK8EZPJAGRztAsRZEuZGbvEAzcnt/DoYjETV1DIiIDXMK6hszsdjMrNrM1Bx3/WmT6/1oz\n+3miyhcRkfgksmvoDoJb6Fqn15vZ6QTT3We7e72Z5Xfw2vcYPny4T5gwIRExioj0W8uXLy9197xY\n5yUsEbj7S2Y24aDDXwF+2jLTMrJ0QEwTJkxg2bJlXRugiEg/Z2bb4zmvu+8amgacYmZLzexFMzu2\noxPNbKGZLTOzZSUlJd0YoojIwNLdiSAZyAVOAL4FPNDRqofufou7z3f3+Xl5MVs2IiLSSd2dCHYB\nD3vgdSDM+xfLEhGRbtTdieBvwOkQrKBIsIRAaTfHICIibSRssNjM7gM+CAw3s10EqzneDtweuaW0\nAbjsEJf4FRGRLpbIu4Yu7uCpSxJVpoiIHDqtNSQiMsD160SwaF0Rv39BCzCKiETTrxPBixtL+ONL\nW3s6DBGRXq1fJ4KQGWENRYuIRNWvE4EZhHVTkohIVP06EYTMUB4QEYmunycCtQhERGLp54nAlAhE\nRGLo14nANFgsIhJTv04EIQOtYCEiEl0/TwRqEYiIxNLPE4EGi0VEYunXicAit4+qe0hEpGP9OhGE\nIpufKQ+IiHSsnyeC4Lu6h0REOta/E0EkE2jAWESkY/06EZhaBCIiMfXrRKAxAhGR2Pp5Igi+NysT\niIh0qJ8ngpYxAiUCEZGOJCwRmNntZlZsZmvaee6bZuZmNjxR5UObrqFwIksREenbEtkiuAM46+CD\nZjYOOBPYkcCyAd0+KiISj4QlAnd/CShv56nfANcACf90fvf2USUCEZGOdOsYgZmdB+x295VxnLvQ\nzJaZ2bKSkpLOlgdoHoGISDTdlgjMLAP4LvDf8Zzv7re4+3x3n5+Xl9epMlu6hrTWkIhIx7qzRTAZ\nmAisNLN3gLHACjMbmagCQ2oRiIjElNxdBbn7aiC/5XEkGcx399JElanBYhGR2BJ5++h9wBLgCDPb\nZWZXJKqsKDEASgQiItHEbBGYWT7wAWA0cABYAyxzj353vrtfHOP5CfGH2TlaYkJEJLYOE4GZnQ58\nB8gF3gSKgUHA+cBkM3sQ+JW7V3VHoJ2hriERkdiitQjOAb7k7u+b+GVmycC5wIeBhxIU22HTYLGI\nSGwdJgJ3/1aU55qAvyUkoi6kZahFRGKLOVhsZl83s8EWuM3MVpjZmd0R3OF6d4xAiUBEpCPx3DX0\nhcg4wJnAUOBS4KcJjaqLqGtIRCS2eBJBpIOFc4C73H1tm2O9mgaLRURiiycRLDezZwgSwdNmlg30\niYWdW+cR9IloRUR6Rjwzi68A5gBb3b3WzIYBn09sWF1DLQIRkdjiaRE4cBRwVeRxJsF8gl5PO5SJ\niMQWTyL4PXAi0DJTuBr4XcIi6kJJIQ0Wi4jEEk/X0PHuPs/M3gRw9wozS01wXF1C8whERGKLp0XQ\naGZJRHYUM7M8+shgseYRiIjEFk8i+D/gESDfzK4DXgZ+ktCouojmEYiIxBaza8jd7zGz5cACgvkD\n57v7uoRH1gVa7xpSJhAR6VC8G9NsAqpazjez8e0tRtfbaM9iEZHY4tmP4GvA94EioJmgVeDA0YkN\n7fBpz2IRkdjiaRF8HTjC3csSHUxXC+n2URGRmOIZLN4JVCY6kETQzGIRkdii7VD2jciPW4EXzOwJ\noL7leXf/dYJjO2zas1hEJLZoLYLsyNcO4Fkgtc2x7FgXNrPbzazYzNa0OfYLM1tvZqvM7BEzG3J4\n4UenPYtFRGKLtkPZ/xzmte8AbgT+3ObYs8C17t5kZj8DrgW+fZjldEhdQyIiscWzQ9mzbf9yN7Oh\nZvZ0rNe5+0tA+UHHnolscwnwGjD2EOM9JJpQJiISWzyDxXnuvq/lgbtXAPldUPYXgH909KSZLTSz\nZWa2rKSkpFMFaK0hEZHY4kkEzWY2vuWBmRUQWXeos8zse0ATcE9H57j7Le4+393n5+XldaocrTUk\nIhJbPPMIvge8bGYvEkwmOwW4srMFmtnlwLnAAk/wJ7S6hkREYotnraGnzGwecELk0NXuXtqZwszs\nLOAa4DR3r+3MNQ5Fy2BxszKBiEiH4hksXuTupe7+eOSr1MwWxfG6+4AlwBFmtsvMriC4iygbeNbM\n3jKzPxx2DaLHAGiMQEQkmmgTygYBGcBwMxtK0C0EMBgYE+vC7n5xO4dv60yQndWyQ5nygIhIx6J1\nDV0JXA2MBpbzbiKoIvjLvtfTPAIRkdiiTSi7HrjezL7m7jd0Y0xdRoPFIiKxxTNYfIOZzQSOAga1\nOf7njl/VO2gegYhIbPHsR/B94IMEieBJ4GyC7Sp7fSLQPAIRkdjimVB2IcE2lXvd/fPAbCAnoVF1\nEXUNiYjEFk8iOODuYaDJzAYDxcC4xIbVNTRYLCISWzwzi5dFFp37I8HdQ/sJ5gf0etqzWEQktngG\ni/818uMfzOwpYLC7r0psWF1DexaLiMQWT4sAM/s4cDLBYnMvA30kEURaBGoSiIh0KJ4lJn4PfBlY\nDawBrjSz3yU6sK6gwWIRkdjiaRGcARzZslKomd0JrE1oVF3EImlOg8UiIh2L566hzcD4No/HRY71\netqzWEQktmiLzv2dYEwgG1hnZq9HHh8PvN494R0e3T4qIhJbtK6hX3ZbFAmiMQIRkdiiLTr3YncG\nkghaa0hEJLZ4xgj6LN0+KiIS28BIBMoDIiId6ueJIPiuriERkY51mAjMLMfMfmpm682s3MzKzGxd\n5NiQ7gyys8wMMy0xISISTbQWwQNABfBBd89192HA6ZFjD3RHcF0hZKauIRGRKKIlggnu/jN339ty\nwN33uvvPgIJYFzaz282s2MzWtDmWa2bPmtmmyPehhxd+bCFT15CISDTREsF2M7vGzEa0HDCzEWb2\nbWBnHNe+AzjroGPfARa5+1RgUeRxQplaBCIiUUVLBJ8GhgEvRsYIyoEXgFzgU7Eu7O4vAeUHHT4P\nuDPy853A+Yca8KEKaYxARCSqaBPKKoBvR766ygh3L4z8vBcY0dGJZrYQWAgwfvz4jk6LKRgjUCIQ\nEelI1NtHzew4Mzs28vNRZvYNMzunKwqOrGba4Se0u9/i7vPdfX5eXl6ny9FgsYhIdNEWnfs+cDaQ\nbGbPEiw29zzwHTOb6+7XdaK8IjMb5e6FZjaKYP/jhDINFouIRBVt0bkLgTlAGkE3zlh3rzKzXwJL\ngc4kgseAy4CfRr4/2olrHJKQmZahFhGJIlrXUJO7N7t7LbDF3asA3P0AEI51YTO7j2CT+yPMbJeZ\nXUGQAD5sZpuAD0UeJ5RuHxURiS5ai6DBzDIiieCYloNmlkMcicDdL+7gqQWHFuLh0WCxiEh00RLB\nqe5eD+DubT/4Uwi6dfoEzSMQEYku2u2j9R0cLwVKExZRF9M8AhGR6Pr16qMQ6RqK2ZElIjJwRVt9\nNK07A0mUkEGzWgQiIh2K1iJYAmBmd3VTLAlhGiwWEYkq2mBxqpl9BjjJzD5+8JPu/nDiwuo6oRCa\nRyAiEkW0RPBl4LPAEOCjBz3nQJ9IBElqEYiIRBXtrqGXgZfNbJm739aNMXUprTUkIhJdtBZBi7vM\n7Crg1MjjF4E/uHtj4sLqOlprSEQkungSwe8JJpH9PvL4UuAm4IuJCqorBWsNKRGIiHQknkRwrLvP\nbvP4n2a2MlEBdTXNIxARiS6eCWXNZja55YGZTQKaExdS11LXkIhIdPG0CL4FPG9mWwEj2Lj+8wmN\nqgtpsFhEJLqYicDdF5nZVOCIyKENHa1D1BsF8wiUCUREOhJPi6BlAbpVCY4lIbQMtYhIdP1+0Tkt\nQy0iEl2/TwTaoUxEJLqYXUNmNq+dw5XAdndv6vqQupb2LBYRiS7eCWXzCMYIDJgJrAVyzOwr7v5M\nAuM7bGoRiIhEF0/X0B5grrvPd/djgLnAVuDDwM87U6iZ/buZrTWzNWZ2n5kN6sx14ixLiUBEJIp4\nEsE0d1/b8sDd3wamu/vWzhRoZmOAq4D57j4TSAIu6sy14hG0CBJ1dRGRvi+erqG1ZnYTcH/k8aeB\ntyM7mHV24blkIN3MGoEMglZHQoTMaGrWGhMiIh2Jp0VwObAZuDrytTVyrBE4/VALdPfdwC+BHUAh\nUJnIcQbNIxARiS6eFsHZwI3u/qt2ntt/qAWa2VDgPGAisA/4q5ld4u53H3TeQmAhwPjx4w+1mFah\nkOYRiIhEE0+L4KPARjO7y8zONbO4ZiNH8SFgm7uXRPY0eBg46eCT3P2WyAD1/Ly8vE4XFjItMSEi\nEk3MRODunwemAH8FLga2mNmth1HmDuAEM8swMwMWAOsO43pRadE5EZHo4l1rqNHM/kGwV3E6cD6d\n3JjG3Zea2YPACqAJeBO4pTPXiofmEYiIRBfPzOKzCe4U+iDwAnAr8KnDKdTdvw98/3CuES+tNSQi\nEl08LYLPAX8BruxLy0+30BiBiEh08exHcHHbx2Z2MnCxu381YVF1Id0+KiISXVxjBGY2F/gM8Elg\nG8GdPn2CBotFRKLrMBGY2TSCu4QuBkoJuofM3Q95EllP0p7FIiLRRWsRrAcWA+e6+2YIFovrlqi6\nkJahFhGJLto8go8TLAHxvJn90cwWECxD3afo9lERkeg6TATu/jd3vwiYDjxPsM5QvpndZGZndleA\nh0uDxSIi0cUzs7jG3e91948CYwkmgH074ZF1ETMjrMVHRUQ61GEiMLOsg4+5e0VkDaAFHZ3T22ge\ngYhIdNFaBI+a2a/M7FQzy2w5aGaTzOwLZvY0cFbiQzw8un1URCS6Du8acvcFZnYOcCXwgcjy0U3A\nBuAJ4DJ339s9YXZeKKTBYhGRaKJOKHP3J4EnuymWhNCexSIi0cWzH0Gfpj2LRUSi6/eJIEktAhGR\nqPp9IghuH1UiEBHpSMxEELlzaEZ3BJMIWmJCRCS6eFoE64BbzGypmX3ZzHISHVRX0hITIiLRxTOz\n+FZ3/wDBBjUTgFVmdq+Z9YlVSEMhzSMQEYkmrjECM0siWHNoOsGS1CuBb5jZ/QmMrUtoGWoRkeji\n2bP4N8C5wD+Bn7j765GnfmZmGxIZXFfQGIGISHTx7FC2CvhPd69p57njOlOomQ0BbgVmAg58wd2X\ndOZasWiMQEQkuni6hi45OAmY2SIAd6/sZLnXA0+5+3RgNsGAdEJoGWoRkeiibVU5CMgAhkfWGWrZ\nlGYwMKazBUbuOjoVuBzA3RuAhs5eL47yNFgsIhJFtK6hKwk2oxkNrGhzvAq48TDKnAiUAH8ys9nA\ncuDr7bQ6FgILAcaPH9/pwkKR9OXumPW5DdZERBIu2g5l17v7ROA/3H1im6/Z7n44iSAZmAfc5O5z\ngRrgO+2Uf4u7z3f3+Xl5eZ0uLBT58FerQESkfdG6hs5w938Cu83s4wc/7+4Pd7LMXcAud18aefwg\n7SSCrtLSIgi7k9T3tlwWEUm4aF1DpxHcMvrRdp5zoFOJwN33mtlOMzvC3TcAC4C3O3OteFhri0BN\nAhGR9kTbmOb7ke+fT0C5XwPuMbNUYCuQiDKAd7uGlAdERNoXz6Jzd7VdX8jMClpuH+0sd38r0v9/\ntLuf7+4Vh3O9aFq6hpo1SCAi0q545hG8DCw1s3PM7EvAs8BvExtW1wmpa0hEJKqYM4vd/WYzWws8\nT7DO0Ny+sFdxi+SkIBE0NisRiIi0J56uoUuB2wlWH70DeDJy/3+fkJedBkBRVV0PRyIi0jvF0zX0\nCeBkd7/P3a8Fvgzcmdiwus6onHQA9lYqEYiItCeerqHzD3r8upl1arG5njB6yCAA9lQe6OFIRER6\np3i6hqaZ2SIzWxN5fDRwTcIj6yJ5WWmETC0CEZGOxNM19EfgWqARwN1XARclMqiulJwUYsTgQezZ\np0QgItKeeBJBRpvNaFo0JSKYRBmVM4hCdQ2JiLQrnkRQamaTCZaVwMwuBAoTGlUXG5WTrq4hEZEO\nxLND2VeBW4DpZrYb2AZcktCoutionEEsWl+kpahFRNoRz11DW4EPmVkmEHL36sSH1bVGDUmnrjFM\n5YFGhmSk9nQ4IiK9SrRlqL/RwXEA3P3XCYqpy43KCW4h3b3vgBKBiMhBorUIsrstigSbODwTgC0l\nNcwYnRPjbBGRgSXaMtT/052BJNKkvEySQsamoj7XqyUiknDxTCibZGZ/N7MSMys2s0fNbFJ3BNdV\n0pKTKBiWwaai/T0diohIrxPP7aP3Ag8Aowg2sv8rcF8ig0qEqflZbCxWi0BE5GDxTii7y92bIl93\nA4MSHVhXmzYim+1ltdQ3Nfd0KCIivUo8ieAfZvYdM5sQ2Z3sGoKlqHPNLDfRAXaVqSOyaQ4720pr\nejoUEZFeJZ4JZZ+KfL/yoOMXEcw27hPjBUeMCG6CWrWrkvrGMLmZqYzLzejhqEREel7URGBmIeAS\nd3+lm+JJmGkjshg7NJ2/vLGT9YVVfGDKcG753PyeDktEpMdF7Rpy9zBwYyIKNrMkM3vTzB5PxPXb\nKY/z5oxm+fYKahqaWbGjAtc+xiIicY0RLDKzT1jXL9LzdWBdF18zqvPmjAEgLTlE6f4GdpZrRVIR\nkXgSwZUEt4w2mFmVmVWbWdXhFGpmY4F/AW49nOscqmkjsvnR+TP59afmAPDmzoruLF5EpFeKZ9G5\nRCw18VuCXc46vLaZLQQWAowfP77LCr70hAKamsNkpCaxYntFaytBRGSgimdmsZnZJWb2X5HH4w5n\nz2IzOxcodvfl0c5z91vcfb67z8/Ly+tsce1KTgoxZ9wQlm1Xi0BEJJ6uod8DJwKfiTzeD/zuMMr8\nAPAxM3sHuB84w8zuPozrdcrxE4fxdmEVK3fu4zfPbqQ5rIFjERmY4kkEx7v7V4E6AHevADq9lrO7\nX+vuY919AsFchH+6e7dvdHPSlGG4w8K7lnH9ok08vGJXd4cgItIrxJMIGs0siXe3qswDwgmNqhvM\nHjuE9JQkiqrqAfj1sxupa9TyEyIy8MSTCP4PeATIN7PrgJeBn3RF4e7+gruf2xXXOlSpySGOnRis\nkHHNWUdQWFnHki1lPRGKiEiPipkI3P0egjt8/pdg0/rz3f2viQ6sOyw8ZRJXnTGFz504ATNYuWsf\nADvKainbX9/D0YmIdI9oW1UOAr4MTAFWAze7e1N3BdYdTp46nJOnDgeCZapX7txHQ1OYj9/0KnPG\n5XDrZcf2cIQiIokXrUVwJzCfIAmcDfyyWyLqIbPHDmHlrkqeXruX0v31vLSxlOq6xp4OS0Qk4aIl\ngqPc/RJ3vxm4EDi1m2LqEUePG0J5TQO/eW4j6SlJNDSHeWFDyfvOC4edXz2zgSdXF/ZAlCIiXS9a\nImj9c7i/dQm1Z+64IQBsLanhmrOOYHhWKk+v3cvaPZV88c5l3LBoEw1NYX721Hpu+Odmfvz42z0c\nsYhI14i2xMTsNmsKGZAeeWyAu/vghEfXjWaMHsxNn53HxLxMpo8czLrCKp5as5fB6Sk8t66I59YV\nMTJnELcs3gpAKBSswVdd18irW8r4yIyRPRm+iEinddgicPckdx8c+cp29+Q2P/erJADBMtVnzxrF\n9JFB1U6ZmkdVXRMPr9jFB6YMIyXJuPmlrbjD8RNz2bPvAA1NYX7z7CauvGs5W0r293ANREQ6J555\nBAPSB6YMxwzqGsMsmD6Co8cOYXPxflKTQnxszmjCDhuLqvnrsp0AvL6tHIBdFbU8tWYvhZXvLnGt\nfQ9EpDdTIuhAbmYqM0fnAMFtpsdOCCafHT02h2mRbS9/+9wmquubSE0K8ca2cipqGjjrt4v58t3L\n+a+/rQHgH6sLmfPDZ9lX29AzFRERiUGJIIrz545h9tgcpuZncdzEoQAcOzGXgshex8+tK2L6yGxO\nn57H6++U88Cyneyvb2L2uCG8uWMf7s49S3dQeaCRZe9opVMR6Z2UCKK44uSJPPpvJ2NmnDBpGAum\n53P+nDHkZaeRnpIEwMfmjObYCbnsqjjALS9t5biJuVw4bwxlNQ2s3FXJq1tKAVi+Q4lARHonJYI4\nZaQmc9vlx3LEyGzMjIJhQavgo0eP5sNHjaBgWAY1DU18+bRJzBob3Ir6kyfWEXbIy05jeWTvg53l\nta2L2+3ed4B/vWc5Z/32JfbX9/s7dEWkl4q5Q5m07/iJuYzMGcS4SDfRi986vfW5+qZmUpKM198p\n57gJucwck8M9S7ezq6KWD//mRT55zDh+dP5MfvX0Bp5cvReANbsrOWHSMG5dvJUTJg1j5picHqmX\niAw8ahF00v+cN5M/Xd7+WkRpyUlMHzkYM/jvjx7FMQVDqW8K86U/L6euMczDK3ZRXdfIS5tKOGny\nMADWF1axtWQ/P35iHX965Z1urImIDHRqERwGM+vwua+dMYWS/fXMHJPDlPwsTpk6nMWbSpk3fggr\nduzjuifWUbq/ge+eM5b1e6tZv7ea8tpgMve6wqoOrysi0tXUIkiQM2eM5LPHFwAwKCWJ2y8/ll9+\ncjZ/+vxxzB0/hPvfCOYfnDI1jyNGZLOusIpH39oNwObi/TQ2h7n5xS1ccutSdlXUAtDUHObs6xdz\nxyvbeqZSItIvKRF0k5SkEBceM5ac9BRu/dx85hcM5ZSpw8nLTmP6qGxW7qpke1ktp03Lo6E5zDcf\nWMn//mM9S7aWccHvX6WipoElW8tYV1jFovXFPV0dEelH1DXUA4ZlpfHgV04iHA5mHB8ZWdZi9tgc\nrjnrCF7cWMJjK/fwkRkj+PJpk7ng969y3xs72FJcA8DKncEchWhdUyIi8VIi6EEtC9edOHkYU/Kz\nuO6CWUzNzyYlyWhsdr6+YBpHjR7MSZOH8adX3qG2vonBg5Kpqmvi+Q3FJIVCnDJlOKGQUdfYzLce\nXEV6Soivnj6FgmGZ7ynL3XlidSEnTxnOkIzUmLFV1DSwv76p9a4oEem/ur1ryMzGmdnzZva2ma01\ns693dwy9zbjcDJ77xmnMHJNDanKI2WOHcMb0fI4aHbQUvnjKREqq68nLTuO6C2YB8KU/L+ey21/n\nwj+8SlNzmNte3sbfV+7h0bf28P3H1gLwzNq9fPXeFYTDzrLtFfzbvW9y+8vxjS/892Nr+cytryWm\nwiLSq/REi6AJ+Ka7rzCzbGC5mT3r7lrgP+LPVxxHqE23zxnTR7Dom6cxYVgm7k56ShIHGpu5+Ljx\n3Pf6Dv5DQc40AAARhklEQVS8ZDs3vbCFDx05gqkjsrjlpa1sLdnPdx9ZTen+Bj573HjufX0HAK9s\nKeMbkeu+uLGER1bs4tefmtPaOoGg9fDK5lLKaxoor2kgNzN2C0JE+q5ubxG4e6G7r4j8XA2sA8Z0\ndxy9WUZqMoMiS1i0mJyXRVLISE4K8dHZo7jsxAJ+fP5Mxudm8MPH36Y57Fx7znQ+PncMzWHn4j++\nRllNAxmpSVy/aBNPrdlLZmoSK3fu4/Vt5azfW8WPHn+bv721h8dW7uHTNy9hc3E1AJuK91NeEyyS\n19GtrPe/voPvPbI6rvo0NIX5/QubqTygrT9FeqMevWvIzCYAc4GlPRlHX/PzC2fzP+fNJClkXHna\nJMzgN5+ezeS8LKaOyG6dwPbDj83ggrljWLqtnGFZqfzgYzNoCjufunkJ51y/mM3FwR4K1zy4iqXb\nyrl+0WYAXtta1lpWe4kgHHb+b9Em7lm6gx1ltTHjXbSuiJ8/tYH7Iq2SygONrUmhoSnM4k0l1Dc1\nH/bvRUQ6p8cGi80sC3gIuNrd3/dpY2YLgYUA48eP7+bo+o7PHl/AWTNGMiwrrfXYPV88HrNghvOO\nslqaw85VC6aSm5nK9/62hrysNEYMTqO2oZlpI7J5bOUehmSk8MSqPZx51AgeeXM3o3MG0RR2lmwp\nY0vJfp5as5cRgwdRUdvAkaMGs6eyDoDfPb+ZlGRjzrih1DY0saviADNGD+bco0eTFOlueubtIgCe\nXruXK0+dxCW3LiUpZDz0lZP497+8xROrCxkzJJ2bLz2GmWNyeHD5LtydKflZ3Lt0B9ddMIvU5Hf/\nZnF33N8dbC/bX99a/8WbShiVk86U/Kz3/a7qGpvf09JasaOC217exq8/NZu05KT3nQ9B0spJT4nr\nvdheVkN+9iDSU9u/lkhvZT2xaYqZpQCPA0+7+69jnT9//nxftmxZ4gMbAF7dUsr43AzGDEmnKeys\n2V3Jf/5tDT/7xNF87vbXKa9pICXJ+O45R/LixhJe2FBCalKIs2aOpPJAI03hMK9sLiMrLZlJeZms\n2lX5nusnh4ymsDO/YCifmj+OV7aU8vz6YuqawsGez5+YxbcfCrqUzpk1kidX7+WyEwt45u0iQhaU\n+7X7VpCSFGL6yGB+xXUXzOSzxxewq6KWVzaX8ocXt5I9KJm7v3g8/1hdyLcfWs3PLzya4VmpXHHn\nMiYNz+TK0yazdGs5nzl+HD96fB1XLZjCNx9YycXHjeeas6YTDjvn3vAybxdW8YdL5nHWzFGtdWj5\nP3HjPzdzwz838+BXTmRIeipjhqaTFDI2FlVT19jMrDE5rbfwFlfVceovnufMo0byfxfPpaiqjk1F\n+zlhUi4vby7lpMnDeXFjCZPyMpmcl0V1XSPF1fUMzUjlphc2s/DUyeRlpxGP9XurqDrQxHETc9/3\nXG1DEyGz1oTX1Bxm8aZSPjBl+HuSadu6vrVzH7PG5JCcFLuDoOV3o1uX+wYzW+7u82Oe192JwIJ/\nQXcC5e5+dTyvUSLoHtV1jawrrGbCsAzyBw/il09v4MbnN3P9RXM4b04wjOPu/Pa5TeRmpjJi8CBu\n+Ocmrr9oDk1hZ2hGKsOz0nh4xS6ufXg1TeF3/2196yNH8IunN5AcMvKy06iobaCuMcx5c0Zz/UVz\nWb2rkk/e/Cp1jWGGZaZSFhmjSE0KkZuZynlzR3Pb4m00hZ1JeZnsLK9lSn42uytqqWlobo0tNzOV\n0v3vbgKUlhyivinc+jg5ZNx2+bG8sa2cG5/fTFLIOG5CLgXDMkhLDrF8RwUb9lYzZkg6u/cdoLHZ\nGZKRwr7aRqaPzCbszsaioEtt2ogsvvHhYN7H/vom/r5yDwB3XXEcP/z722wq3s9HZozg6bVFFAzL\nYHtZLWnJwcTC59cXU1xdzwmThvHy5lL+ZdYofvfZeWwu3s/aPZU8ubqQ7WW13HDxXJrCzjUPruLs\nWSPJSEniJ/9YTzjsXP2hqazdU8Un5o3lxuc3c9q0PP66bCeD01N46CsnkZmWzO+e38wvnt7ANz88\njYuOG09SyMhMS2LN7ioONDSzsaiaHz7+NgtPncSUvCzGDE1n+shsXthQwvbyWo6dMJSTpwzHzAiH\nnc/f8QZhd26//Fgqaht4as1ePnnMOB5asYtZY3J45M3dNIXDfO+co9i9r5bXt1VwxvR8vnrvCs6f\nM5pLT5zQ+l64O2U1DQyPtOYqahoYlJJEemoSr2wu5aWNJVz9oWns3lfLxOFZrCusYmhmKttLa1i9\nu5KFp05qjev2V7Zx8tThrVvNtlXf1MwP//42HzpyBKdPz39P+ZUHGt93O/WOslrW7qnk7FmjDr4U\nAPtqG9hZfoBZY4OFIVtamsu3l5OZltxuDO1xdzYV72dqflZrYm35PA477CivZeLwzGiXiKk3J4KT\ngcXAaqDlf+h33f3Jjl6jRNAz9tU2sK6wmhMjC+Mdipc3lbJ+bxULjhzBCxuKufSEAu5csp3Vu/Zx\n3pwxPPN2EYvWFfH01acyNHJX0p59B3huXRHzxg/l509v4PVtZdxw8Ty+fv+b1DY08y+zRvG1BVOY\nlp/NovXF/PDxtZRU13PPF4/n7td2MG5oOpecWMBlt79B2f565o4fwtNri7hqwVRufnELXz19Crcu\n3kpVXbDk9xnT85kwLJPbX9lGanIIA6aNyOakKcNYtK6YipoGvn3WdP7z0TWcP2c0K3bsY2hGCh+b\nPZq0lCR++o/1rYPqAB86cgRv7dxH6f56zGBYZhql++uZPTaHtXuquGDuGKrrmoLuqyHpVNc1UlRV\nz9ih6eyqOMD43Ax2lAdjLoMHJZOSFKLyQCMhM0KhYNtUgJMmD6Owso5tpTWYgXuQMBuaw2SnJVPT\n0MTE4ZmMHZrBki1lOE5KUoimZqfZnaSQ0dAmOWanJVPdZhn0lmu2WDA9n7qmZlKTQjy/oQSA06bl\nsbGomsLKutak2fb1SWatfwhkpia1Juv/OHMaWWnJvLa1nLWFlewsP8CRowbzwSPyuGvJdtJTkzhq\n1GBe3BiUM2JwGkVV9UzNz2JLyX4y05Kpj7Qurzh5IufMGskTq/Zy+yvbGDMknUtOKGBv5QGyB6Xw\n5s4KdpYfYPSQQby2tZz0lCQKhmWwr7aReQVDWPZOBcXV9UwcnslXPjiZUTmDyEpL5hsPrGRbaQ3f\nOXs6Oekp3PziFj4+b2zr73J7WS2FlXWcNi2PjNQknnm7iLNmjuTZtUWkJBl3fOE4puVn88qWUnZX\nHGDptnKmjcjiIzNG8sibu9lVcYBzZgU/L95UylkzRlJYVceM0YPZVlLD3qo6xudm8OLGEn50/kwu\nPaHgkP//vfte9NJE0BlKBP1PY3OY+qYwWWntD1MVV9dRXBUs2tfYHKaitoG8rLT3dEk0h539dU3k\nZLy3D7+ytpFmd7LSktleVsPUEdmtf7XtLA/+2isYlsmRowbzTmkN335oFd/6yBEcUzC09frNYaeu\nsZnMtGSamsPtdpvsKKvlqbWFfPiokTyyYhcXHTee5nAwcW9UZIny2xZv47oLZpKSFCIzUteW/3PL\nt1dw6+Jt/Oj8mdz68lZ2VRxg9tgcPnhEPuNzMyiraeDu17ZzoKGZL506iY1F1WSlJTO/YCi7Kg6w\nZGsZx0/M5c5Xt3P5SRNYtr2cGaNzWLO7kr+8sZP65jBDM1JYeOokPnfb68wrGMqxE4bS0BTmmIKh\nVNQ2smhdMf997lH86tkNnDR5GHWNYSoPNHL6EflMzs/ktsXb+O2iTYzPzWB7WQ2nTstj9tgh/HnJ\nO+RnD+LUacP54+JtfPb48WSkJjFjdA75g9N4YUMJIwcPwgx+/MQ6rj17Oku2lLUujzJxeCaThmcy\ne9wQnltXxKpdlcwem0N6ahLF1fV8ZMZIxg5N5+dPbeDco0fx+KpCTp46nHdKa3CH6SOzefjN3a3v\nxYLp+by0qYTG5uD26vqmZo4cNZistGSWbivnomPHsXhTsEnUpLxMtpbUcEzBUI4Ymc0zbxexcue+\n1muZwdFjh7Qey89Oo7i6nvSUJHIzU0lLCXHmUSN5fNUe6pvCTB+ZzeJNpYzLTSdkxvay2tbEDDAu\nN53dFQdoaSBnD0qmuq6JrLRkzpiez2Mr95CfHfzRkJacxOD0ZIqq6pk+MpsNRdXcftmx72nJHAol\nAhFptWffAfKy00iJYxzgYI3NYVKSQhRX1zF4UMr7bm3eWV7L2KHpHY4b1DY0kZEaJMENe6sJu3Pk\nqPd2nxRX1ZGbmfq+hNuylEpzOGjJuDthh5DB2j1VrbdIHzN+KMu2V5CaHGJW5I+HljjfKa2hYFgG\nBxqbSUkKve93EA47i9YXk5ocYmvJfoZkpHDOrFEs2VJGRmoyxxQM5aHlu5g1NofpkY2pDo7x6bV7\nmTE6h8y0ZB5cvpPCyjrOPXo043MzyMtOo2x/PYvWFzNhWCZHjR7M9rIapuRnkZacxPq9VUwYlsm2\n0hrSU5IYmpHKzopapuRn8YunN3DVgqlx37BwMCUCEZEBLt5EoNVHRUQGOCUCEZEBTolARGSAUyIQ\nERnglAhERAY4JQIRkQFOiUBEZIBTIhARGeD6xIQyMysBtnfy5cOB0i4MpzcaCHWEgVFP1bF/6C11\nLHD3vFgn9YlEcDjMbFk8M+v6soFQRxgY9VQd+4e+Vkd1DYmIDHBKBCIiA9xASAS39HQA3WAg1BEG\nRj1Vx/6hT9Wx348RiIhIdAOhRSAiIlEoEYiIDHD9OhGY2VlmtsHMNpvZd3o6nq5iZu+Y2Woze8vM\nlkWO5ZrZs2a2KfJ9aE/HeSjM7HYzKzazNW2OdVgnM7s28r5uMLOP9EzUh6aDOv7AzHZH3su3zOyc\nNs/1xTqOM7PnzextM1trZl+PHO8372WUOvbd99Ld++UXkARsASYBqcBK4KiejquL6vYOMPygYz8H\nvhP5+TvAz3o6zkOs06nAPGBNrDoBR0XezzRgYuR9TurpOnSyjj8A/qOdc/tqHUcB8yI/ZwMbI3Xp\nN+9llDr22feyP7cIjgM2u/tWd28A7gfO6+GYEuk84M7Iz3cC5/dgLIfM3V8Cyg863FGdzgPud/d6\nd98GbCZ4v3u1DurYkb5ax0J3XxH5uRpYB4yhH72XUerYkV5fx/6cCMYAO9s83kX0N6svceA5M1tu\nZgsjx0a4e2Hk573AiJ4JrUt1VKf+9t5+zcxWRbqOWrpM+nwdzWwCMBdYSj99Lw+qI/TR97I/J4L+\n7GR3nwOcDXzVzE5t+6QH7dF+dV9wf6xTxE0E3ZdzgELgVz0bTtcwsyzgIeBqd69q+1x/eS/bqWOf\nfS/7cyLYDYxr83hs5Fif5+67I9+LgUcImplFZjYKIPK9uOci7DId1anfvLfuXuTuze4eBv7Iu10G\nfbaOZpZC8AF5j7s/HDncr97L9urYl9/L/pwI3gCmmtlEM0sFLgIe6+GYDpuZZZpZdsvPwJnAGoK6\nXRY57TLg0Z6JsEt1VKfHgIvMLM3MJgJTgdd7IL7D1vLhGHEBwXsJfbSOZmbAbcA6d/91m6f6zXvZ\nUR379HvZ06PVifwCziEY0d8CfK+n4+miOk0iuANhJbC2pV7AMGARsAl4Dsjt6VgPsV73ETSnGwn6\nUK+IVifge5H3dQNwdk/Hfxh1vAtYDawi+MAY1cfreDJBt88q4K3I1zn96b2MUsc++15qiQkRkQGu\nP3cNiYhIHJQIREQGOCUCEZEBTolARGSAUyIQERnglAhERAY4JQIRkQFOiUCkE8zs2MjiYoMis73X\nmtnMno5LpDM0oUykk8zsx8AgIB3Y5e7/28MhiXSKEoFIJ0XWsHoDqANOcvfmHg5JpFPUNSTSecOA\nLIJdqgb1cCwinaYWgUgnmdljBDvfTSRYYOzfejgkkU5J7ukARPoiM/sc0Oju95pZEvCqmZ3h7v/s\n6dhEDpVaBCIiA5zGCEREBjglAhGRAU6JQERkgFMiEBEZ4JQIREQGOCUCEZEBTolARGSA+3+l37pT\nzHcQJwAAAABJRU5ErkJggg==\n", 1282 | "text/plain": [ 1283 | "" 1284 | ] 1285 | }, 1286 | "metadata": {}, 1287 | "output_type": "display_data" 1288 | }, 1289 | { 1290 | "data": { 1291 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd8XNWVwPHfGY006r24d2xjgzHYtEAooUMIgRRqSF3C\nLmEJG5IN2XSyKUt6QiCEFFJoSSAhQOjg0HG3ce+WLKu30UjTz/7x3oxGsiSPwSq2zvfz0cd6b957\nc0eW7nn33PvuFVXFGGOM2R/PSBfAGGPMocEChjHGmLRYwDDGGJMWCxjGGGPSYgHDGGNMWixgGGOM\nSYsFDGMMACLyOxH51kiXw4xeFjDMIUdEXhSRVhHxjXRZhoo4PiMia0SkS0Tq3M99xUiXzYxdFjDM\nIUVEpgHvBhR43zC/t3cY3+6nwGeBzwFlwETgy8D5/R3sBhj7ezZDyn7BzKHmWuB14HfAR1NfEJEc\nEfmBiOwSkXYReVlEctzXThWRV0WkTUSqReRj7v4XReRTKdf4mIi8nLKtInKDiGwBtrj7fuJeo0NE\nlovIu1OOzxCRL4nINhHxu69PFpE7ROQHfcr7qIjc3PcDishs4D+AK1T1GVXtVtWYqr6sqh9LOe5F\nEflfEXkF6AJmiMjHRWSD+97bReTTKcefISI1bvmaRGSniFzd5+1LRORx9/w3RGRmOv8pZmywgGEO\nNdcCf3K/zhORqpTXvg8sAt4FlAJfAOIiMhX4J/AzoAJYCKw6gPd8P3AiMM/dXupeoxS4D/iziGS7\nr/0XcCVwIVAIfAKnMr8XuDLRChCRcuBs9/y+3gNUq+qyNMr2EeA6oADYBTQA73Xf++PAj0TkuJTj\nxwHlOC2WjwJ3i8iclNevAL4BlABbgf9NowxmjLCAYQ4ZInIqMBV4SFWXA9uAq9zXPDiV802quse9\nI39VVUPuMc+q6v2qGlHVZlU9kIDxHVVtUdVuAFX9o3uNqKr+APABiUr3U8CXVXWTOla7x74JtANn\nucddAbyoqvX9vF85UNfns9e4raOgGwATfqeq69yyRFT1cVXd5r73EuBpnBReqq+oash9/XHgwymv\nPaKqb6pqFCcoLzyAn5M5zFnAMIeSjwJPq2qTu30fPWmpciAbJ4j0NXmA/emqTt0QkVvctE+7iLQB\nRe777++97gWucb+/BvjDAMc1A+NTd6jqJPc9fIAMUrYLROR1EWlxy3ZhStkAWlU1kLK9C5iQsp0a\nqLqA/AHKaMYgCxjmkOD2RXwYON0dMVQH3AwcIyLHAE1AEOgv5149wH6AAJCbsj2un2OSUzq7/RVf\ncMtSoqrFOC2HRCU+2Hv9EbjELe+RwN8GOO55YJKILB7g9YHK5gP+ipOaq3LL9gS9A0yJiOSlbE8B\natN4H2MsYJhDxvuBGE4/wkL360jgJeBaVY0DvwF+KCIT3M7nk91K9E/A2SLyYRHxikiZiCRSLauA\ny0QkV0RmAZ/cTzkKgCjQCHhF5Ks4/QUJ9wC3icgR7silBSJSBqCqNTj9H38A/ppIcfWlqpuAXwIP\niMg5bmd+Bk7fzGCycFogjUBURC4Azu3nuG+ISJYb/N4L/Hk/1zUGsIBhDh0fBX6rqrtVtS7xBfwc\nuNod8noLsBanUm4Bvgd4VHU3Tmrmc+7+VcAx7nV/BISBepyU0Z/2U46ngCeBzTjpnCC900I/BB7C\n6TvoAH4N5KS8fi9wNAOnoxJuwBla+0O3zDXAbcDlwO7+TlBVP/Cf7vu34vTdPNrnsDr3tVqcz3q9\nqm7cT1mMAUBsASVjho+InIaTmpqqw/zHJyJnAH90+0OMOWDWwjBmmIhIJnATcM9wBwtjDgYLGMYM\nAxE5EmjDGf304xEujjFvi6WkjDHGpMVaGMYYY9IynJOpDbny8nKdNm3aSBfDGGMOGcuXL29S1Yp0\njj2sAsa0adNYtiyd6XeMMcYAiMiudI+1lJQxxpi0WMAwxhiTFgsYxhhj0mIBwxhjTFosYBhjjEmL\nBQxjjDFpsYBhjDEmLRYwzJi3bGcL62rbR7oYxox6FjDMYe/ffr+M2x5bP+DrX3pkLd9+YsMwlmh0\naQ2Eae4MHZRrxePKy1uaOJznqIvE4vzHn5azqrptpIsy7CxgmFFpR1Ng/wcB62rbae+KDPi6qvL6\n9mbW7hm4BVHXHmR3S9eg77NidytPrN2bVpnSEY3FufH+lazc3XrQrvl23fzQKm68f+VBudZzGxu4\n5tdvsPIwrkw31fl5Ym0d/9rcONJFGXYWMMyo8/r2Zs78/ou8NUglD06l+8E7X+NXL20f8JiWQBh/\nMEpLINzv68FIjI5glNq2IJFYfMDr/HLJNr741zXE44PfOW9v7OTTf1hGVzi6z2vxuBKMxADYVO/n\nH6treXFTT6VT3dLFfW/0u5jekFFVVuxqTTtAJ+xu7uq3FbHCDYB724IHpXwHy6rqNn70zOaDcq31\nezsAaDpIrbKVu1v5+6o9B+VaQ21IA4aInC8im0Rkq4h8sZ/Xi0TkHyKyWkTWicjH0z3XHL4Slc6e\ntn6XvE6qbQvSHYnR4A/y1+U1/PS5Lfscs7PZqQgHSrk0+p39sbjuU8nF48qtD6/lhY0NtAYidASj\nbGvsZPmulgHL9Mz6ep5aV99vuuLe13ZyynefJxqLJ4NhY0q57ntzN196ZC1tXf0HtyWbG7nqV68T\njjqBrTsce8epn9r2IB3BKA3+EIFQlDU1+28ZVLd0ccb3X+Dxflpcq93PnahMtzZ0Ut/R83ONxuL8\nfdWe/QZecILZR379Bt/55ztPF9714jZ+8twWWge4cejPDX9awV+X1+yzf32tEzCaO9O/1kBC0Rg3\n3r+SWx9em9bPpD8N/iBNnaFhSQMOWcBwF62/A7gAmAdcKSLz+hx2A7BeVY8BzgB+4C5On865ZpR4\ncOluXt3a9I6u8edl1dzjthQ27PUD7PePe1eLEwzauiJ87s+r+eEzm/e5s9/R5KSa2rojxFL+IONx\n5f+e3JgMTkAyLaWq/HPtXh5cVs39b+7m209soNWtxD/359V84M7XBrwj39rQCcDmOv8+rz35Vh3N\ngTCtXZFkiqzRH6IjGMEfjCTff6BA+eq2Jl7d1szSnS0s3dnCkV99kvvfdJYTf3R1LX94befAP6wB\nbHArv1hc+elzW3j/Ha9Q1z5462BdbTtxhWU7e6fT4nFlbY3zuZo6Q7QEwlz6i1f4n0fWJo95dkM9\nNz2wite3Nyf3RWPxflt3r25r5qUtTby2refYUDR2wJ8xEovzivv7ubm+5/9lS71/wHRme3eEx9fu\n5bE1tfu8lggYjYO0MOrag0RTPtOz6+u5+p7Xe+0D+OPru6lp7aYrHKO61fn/X76rtdfv6v787Lmt\nnPWDJYhI2ue8XUPZwjgB2Kqq21U1DDwAXNLnGAUKxPmk+TiL3UfTPNeMEj99biu/eWXnAZ8Xjyuf\nuncpT6+r4/ev7eKBpU7lt94dsdTaFeH9d7zCn5dV93v+ruaeYJDw6tZmorE4j6/ZSyyu7HQrdlVo\n7QqzYncrn7lvBRvqOvjFi9u4a0lPOitRYb+xo4V//9MKbn3YqegmleTQ6lYsa9wKcYtb8cTj2uvO\ncIsbMDbVdyb3Vbd0sbXBnwxOzYEQa/f0pDU+c99Kbn5wFTWJgNHaEzBUlWvueYOv/O0tGjucCure\nV3fyid8tBeCVbU38/Pkt/Of9K/nmY+sPqEKtaw+yJiXtt2RzI3GFZ9bX8aG7XmVjXUe/5212P1ui\n4gRo74rw2Nq9+EPR5Of60TOb8QejvLG9JVkBrqp23m9nc0+f0Rf+uoar73kjuR2MxHhw6e5ky2JH\nUwBV5Ym1ezn2m88MmAraUu/nxvtXEgj1vmlYsas1Wa5EwGjvjvC+n7/Cj57tP02V+P9N3LwkdIdj\nyZRUotW6urqtV2Bp6gxx+u0vcP/Snt/b3766g1e2NrO6pufn/cz6er77zw1MLs1Jvtcb25v5wJ2v\n9huowPl9e/KtOtpTfud3NgeYWpbb7/EH21BObz4RSP1LrwFO7HPMz4FHgVqgALhcVeMiks65AIjI\ndcB1AFOmTDk4JTcHJBiJsbd98PRRf9bVdvDshgbyfF62NXaS7/PSHY4l796rW7tYVd1GZYGPDy2e\nnDxvd3MXNz+0inFF2YBTWZXn+2jqDPHCpgb2tHXztUfX8ftPnJBMSQFs3Ovnml87FdPMinwANuzt\nqfQSAWO3W5lNLctlb3uQ1q7IPmmiHU0Blmxu5LMPrOQTp0znxrOOQFXZlmhhpNzJfvLepexs6iIS\ncyrN+o5Q8n0b/SFqWrt73VGmtjBe2NTAy1ubeHlrE+8+ohyAp9fXk+/zMrE4h/r2IGtq2sj3eekM\nRdmw18/CycXJ83c2BSjJzaIoN7NX+YORGCd95zkAMjxCLK5sdFtFP3xmM61dEV7Z2szccYX7/L8l\nPtv6vR3E44rHI3zuz6t5dkM9ADmZGdS2BXl1WxMTirKpbQ+yvraDoycVsXZPW/L/FiAcjfP0uno6\nQ1Hq2oNkZggfuPNVdjZ3kZXh4dgpxazc3UZLIMxja2rpCsdYsauVc+ePS5Zn+a5W2rvDvLatmX+s\nrmXh5GI+eep0wKnU731tJxkeIdvrYZNb9sfW1NIdifX6/0+VOK6uI0hLIExpXhZPravj039YDkBu\nVgZNbkrqS4+sZf3eDioLsjlhein/2txIKBpndXUbHzlpKs2dIV7f7qQxX9naxKKpJQQjMf7roVUc\nOb6QX127mJO+8xyb6vw0B5wg9Pr2Fi5ZOHGfcv11RQ2f/8sappblcu/HT2BaeR67mrs4JuX/fCiN\ndKf3ecAqYAKwEPi5iOz7GzoIVb1bVRer6uKKirTWADEHWXckxt79pDH6s2RzAwAvbWmiKxyjIxhh\nU72fRN2ZSOusq+1gdXVbMu31wNLdLN/VyuNrnBx6W3eYzpBzx/XUujrufHEbALuaA+xsDpCTmQHA\n7U9vSr73i5sakt9neIQppbnctWQbF/30JapbuxCBZ24+nQuOGsfuli6icaUw20uW10OBz8vSnS18\n6t6ltHZFeHp9PapKXUcQfyhKdqaHzXV+VJWtDX4213cSTklFvLmjmXA0TlWhj/qOII1+J32T6Jjf\n09pNMBLj64+u4+YHVwPg83po9IfI8Dhphy+cP4fTZlewYW8H1S3dXLJwAkCvPojfvLyDs364hFsf\nWcOetm7+vmpPMhhta+xpAR0/raTX/0uiNVXd0sXK3a08vKKmVz/ElvpORKAzFGV3Sxerq9t4dkM9\n710wns+dM5tFU0tYsbuVSEz56LumAfDGjmZUNdlCq3aD84rdrXS6d//Pbazn5a1N7Gzu4o6rjmPj\nbedz43tmueUN8NIW5/9/dcpnDEVjfOa+Fdx430pecAcQ/Pql7ckU16d+v4wn1tZx9YlTmDehkNXV\n7dz54jbufXWn81ncAH/Xkm2c88MlbG/cN6WYCCp/eG0X5fk+LlownsuOm0h7d4QVu1tZV9tBhgi3\n/Hk14WicJe7oqUQr5Ym36ojFlZLcTF52P8OSzY34g1E+d+4cqgqzmVaWx/q97fzzrToAlu7s6Ser\nbuniR89s5qt/f4sfPrOZWZX5NPlD/PT5LYSjcWpau5g2TC2MoQwYe4DJKduT3H2pPg48rI6twA5g\nbprnmlFA1Rn50xIIJ0cA9RUIRftNlSRGCCUqymAknsztF/i8ybu8PW3d/MefVvA/f3sLVeXR1b2b\n602dYYKROOfNrwKcu0JwWgw7m7qSd9xra9qYVZlPhkd6pWIq8n14M5yKeF1tB8t2tjKuMJssr4ey\nPF+yfLecN4d/3vRujhxfyPMbG4jElDPmVLCutp3P/Xk1J3/neQDOmluFPxSltj3IU+ucu+7vfeBo\nPn/eHABW7nYqvMXTSpOtjlS7Wrr4t98v497XdnLqrHIuO24ioWicnc0BLjt2Ir+6djHXnDiVmRV5\nBMLOz/WceVWU52fx7IYGbn9qI0+s3cs3H1tPXlYGz21o4Mb7VnDTA6s494dLaPSHkn0tt39wAT++\n/Fgy3c9flNPTEtnd0sWn/7Cc/3poNR+481XC0TjRWJztTZ2cMtNp7bxV61TAJbmZfPcDC7jxrCOo\nLPDhDzpB4KQZZUwvz+P7T2/i3B/9K7l/d0sX/3n/Sr7xj/V4PcK4wmye39DA1oZOPAJnz6vE4xGm\nluUB8MjKPfiDUTxCrwEFDy6tZm97kEA4xtaGThZPLaG2Pej0F3WGWLm7jZvPns03LzmK2VUFrN3T\nzvee3Mjm+k5mVuTREgjzl+U1fPefG9na2MkVd79OayDMxjp/Ms2zYW8He9q6eWVbE1efOIU7rjou\n2fL6xQvbyM708KPLF7K7pYs/L69ODrfdXN/JXUu28bW/v8XccQVcfvyUZIB8bM1eSnIzedfMMgDm\nVBXw/MYGGv0h5o0vZGtDJy2BMA0dQT78y9f42fNb+MvyGva2B/nmJfN574IJPPlWHVsanBusxM9p\nqA1lwFgKHCEi00UkC7gCJ/2UajdwFoCIVAFzgO1pnmtGgUhMky2CgVoZV9/zBl9/tPeDcx3BCCur\n25hR0fsXPZHSmTOuIFm5gBM0alq7WL6rlZrW7uR5HiGZznnP3Eqe+uxp/PZjxzOzIo/Xt7fQGYpy\n0gznjzKuMG98IZNLckgdUFJZ6OPrF8/nfDfNsXRnCxOLnbxyWX5W8rgJRTnMrMhnRkUecXWC2sdP\nmU5c4eEVPfczFx8zHnBaEk+s3cvCycVcfvwU/v30mXiEZFA8tp80QlFOJs9vbOClLU3cdslR3HH1\ncZw5pxJwAurEkhzOmVeFxyPMrMxPnjd/QhELJhXzr82N3PHCNv7jTyuYXJrDz646jlA0zordbVx6\n7EQC4RhPrN3LNrdift/CCYwryqaywEnvffyUaUwszmH+hEJWVbfR4A9x5pwKalq7+cJfVnP9H5cT\niSkXLRhPltfDK1ubWLK5kfcumEC+z8lwlxf4kuWaUZHHv58xk9OOqEgG8oWTi3lrTzuPrq5lw94O\njp9Wynnzq3hlWxMb9nYwtSwPn9dpFU4uycUj8MjKGjI8woVHj2dNdTv1HUF2NAX4wdObOWFaKRPc\n9OQXL5jLxOIcHlxazRs7nLv0U91U3pHjnUr+c+fM5sVbzuCrF88H4Mt/W8vMijz+/OmTafCH+P1r\nu9hc7+fkGWVUFvhYX9vBP1bXogofXDTJ+Yz5zmd8bmM9Z82t4r0LxrNgUhHf+Md6WrsinDqrnO5I\njO8/tYlTj6jgwetO5pRZZUTjyqtbm3huQz3nHzWezAynCl4wuYhITDl9dgW3XjgXgGvueYMzvv8i\n7d0RHv3Mqaz+2rm8/N9n8q6Zzk1EVzjGPS/tADj0WxiqGgU+AzwFbAAeUtV1InK9iFzvHnYb8C4R\nWQs8B/y3qjYNdO5QldWk76Fl1Uz74uPJTrdgSsthb5/RPR3BCKrKjqYAL21p5LE1tVz8s5f5r4dW\nsbamnVhcuebEqb3O2dkcIN/npaowe5/3jsSUv7nj1W+75CgA5qTk2EtysyjL93Hm3EqmlOYmK+aT\n3bs4gFmV+cxw+y8SQaGywMdpsyv4zmVHAxCNK5NK3ICR1xMwSvKcu+/p5U6wOmF6KcdPK8Hrpom+\nfNGRXHH8ZM6ZN45Zlfl87e/rWFfbwYcWO5WMxyOU5jl339mZHuaMK0heOzvTk7xmIn1xxfFOI3tC\ncc/PoiKlMp7lfo7KAh8VBT4WTXVSS9efPpOJxTl86/1Hc+qscsrzfeRkZvD1i+dz5PhC/rZqD1sb\nO5lSmpusmMe7Fe6FR4/nlS++h3fNLEu2rK47bSaLp5bwt1W1LN3ZikfgxOmlnDW3kgeXVtMdifGe\nuZXJcpW7QbaiwEdBdiYfXjyZu69dzAPXncQt587mnHlVyZuMn191LN/9wNGcPLOcYMRJ58xMuYnI\n8nqYVJJLMBLnA8dN5LTZFfhDUU789nOc+f0Xiavyfx9cwFUnTmFcYTYLJxfz4cWTeXlrEw8urSY3\nK4MFk4oAuPTYidx1zXHccOYsppXnMdf9+Qcjca49eRqLpzmf6Y4XttLaFWHBpGIWTCpmZXUbb+5o\nYVZlPpNLc3t9RlU4cUYpIsLnz5tDcU4mnz9vDp89+4jk79L1p8+gKDeT46aUkOERfvHiNrrCMc6d\nV5X8nJ84ZTqP3Xgqv/v48Rw/rZQZ5XnE4soHF03i/n87iaMmFpGZ4fwsAI6fVsrk0pzk38OUw6DT\nG1V9Aniiz767Ur6vBc5N91wz8u5/03mwbHV1G6fNriAY7gkYtSktjN3NXZz9oyXcefVxtHdHaO+O\n8IOnN7OrOcDaPe143CGA5x81ju89uZGQ+2zBzqYAJXmZyco5y+thYnEOTf4Q/lCUFzY2MqEom1Nm\nlXP3RxbR3h3h839ZA0BpSuWe+MMGOHJ8AcW5mbR1RZhVmU9Hd4TncVokf3h9FxXu3XVJXlay83xi\nImDk91TQxbnO9RMB4+SZZeRmeTl5ZhnZmRl86t0zksfecOZMbn5wNfMnFHLF8T2DMcrysmjqDDGh\nOCdZ+edkZnDcVKdz9+iJRTyzvp7z5o/D6959ji/KSZ6faAmAE/B8Xg/zJzhB8+OnTOPdR5SzYFIx\nX7xgbvK4r108j0gsTlFuJpcsnMB3/7mRXXlZHDelp++iqigbEZji/tympPz85o0v5MdXLGTjXj/v\nmVtJOBYnOzODS4+dyD/fqsPn9SRbcdBz9z2jvHfrcf6EIuZPKEqOADqiMp/3LnD6XhKtk0hMe7Wc\nwGmlNHWGuOXcOXg8wtlHVnHMpCLqOoJctGA808rz+I8zZnHdaTPxZnj40OJJ3LlkK0s2N3LGnIrk\nXXyez8v5R41P+Vn6KMj2EorEed8xTjlueM8sXtrSxMdPmcaHFk8iEIry7IZ66tqDyWNSPyM4lTfA\nu4+o4M3/ORtwbpYAqgp9nDi9LPn+iZZbVoaHE2eUJq+RnZnBUROLkt8/f8sZDMbjET5xynS+8Y/1\n5GZlUJFSnqE0pAHDHH5mlOezcncba/e0OwEj0tOZm9rCeGZDPeFonDd39HTe7WgK8MFFk/jL8hoe\nX7OX4txMxhdl86HFk2jrivDYmr1Ut3Zz1IRCStzKubLAx28/djz1HUEuv/t19rR1c9psZ3DDufPH\n8eq2nuc/SlICRqLCm1yaQ0F2JqV5WbR1RTiiMp82t1N3zrgCvnTh3OQfPMDsqnyaOkPJO7nUlFSi\nTCdOL+O8+VXJyu43Hzt+n5/TxQsmsKMxwMXHTEh2VCevV+9U9ok/8kklOVx94lQWTSlJDrG84Oje\nFZtHnJRaagvD4xG+dOGRHFHlVLC5WV4WTNo3zXVxSkX3oUWTuPPFbbQEwsxKqZjPmltJpkfIdgcI\nJALuxOIcinIzKcrNTP5Msj3OMWfMqaQ0L4tjJxeTk5WRvFaijImWXF+T3eu8+4ieQSpl+T6OqMxn\nS0NnsuWU8OWLjqS9O0Kl2+q856OL97mmxyNkuT/nCcU5PPqZU/nZ81u57Lh9RxoliAhnzqkkP9ub\n/N05bkoJa75+bvLncMJ053ejOxJLtuCc8jrHF2R7mVNVQF+F2ZmcMK2U0+dU9Pr/Xzy1lDU17Sye\nVkJu1jurfi8/fjI/eW4L4wqzh+UZDLCAYQ6Qe7OW7HhMTUmltjBe2OiMQtrY5wG2j548jRc2NtAc\nCLNgUhEiwrfefzRbG/w85j47UZqX1StgTCvPY3yxcwes6tyZJhTn9FTopbk93ycqtzlVzt13WV4W\nu5q7mFqWR5fbKpo7roDFKcECYHZVAa9ua+7pw3ArEpGeDuGi3Ex++ZGeSitxB5vKm+Hhv86ds8/+\nsmSQyKUkNys5QuvCo8dz4dHjCUZiZHg8nObm3RPXqirMZm97sFfAAJKjkNJVlu/jfy46ki/8ZU0y\nJQNw2XGTuOy4ScntRMBN5P37k+X18NCnT6Ywu3c1krj7nlnRf0fsnHEFXHDUOK44YXKv/SdML3UC\nRp8WxqzKfSvk/ZldVcDPrjx2v8f9tJ9jEsECYP6EQnKzMugKxzguJWDk+7z4vB4WTy3B4+m/sn7o\n+pP32XfC9BJ+88qOXsHy7crN8vKTK45leEKFwwKG2UfiOYd7rl3c664doKPb6YheubuVe17azrwJ\nPRVKrdvCCISivLHDeTp3kxswMjOE0rwsjppYyDGTi3l+Y0OvCqswu2d0TkleVjK9lEjB+LwZVBVk\nU9cR7B0w3OcLPAKFKSN8eio85z0ml+bSFY6R5fVwzORiXrzlDKaV71uhJY6f5o46SVTwRTmZve4U\n365EAJpUkoPHIyyaWpK8iwWnskpNfSSML3ICRnl+1j6vHagPLZrEpJIcFk8tHfCYiSU55GVlcOyU\nwcf3963cE/s+9q5pXJjSSkqVnZnBndcs2mf/JQsn8lZtR6++nZHmzfCwaGoJa/e090qxiQj/dc7s\nflt0gzltdgVXnziFDywauOVzIE6fPbyPEljAMPtYs6eN5btak2mnVIncbFNnmG89voEr3bvEaWW5\nztQVW5v4w+u7iMSUnMyM5MiY//vgAqaU5iIiHDPJCRipHdaplX1pblYyEKTeUU8uzXECRtW+AaPY\nvVtPmFmZx7nzqrjAzVl//X3zCaWkz/oLFgCXHjuJqWV5yU7EvKwMfF5PssXzTiUCRqIF89Cn970L\n7c/44hyKmwLJTup3QkR418zyQY/xeTN48rOnUVl44LnxzAwPX3/f/AM+74Tppfz9hlMO+Lyh9rWL\n59HoD+/Tkvj06TMP+Fq5WV7+99KjD1bRhp0FDLOPRMom9entu5ZsIxSJ0xGMsGBSEZUFPp7d0JCc\nvO9/LprHrQ+v4ap73sAj8KUL57J0ZyvPrHeeQzh1VkWy8j9lVhk/eY5ed68+r4esDA/hWJzS/NQW\nRkrAKMll6c5WZlX03IHmZGaQleGhpM+TzD5vBndf25M2KszOhH0HXu0jq08HrohQnu9LBqZ3KtFi\nSXSqp+vf3j2j16ia4ZA6cGAsm1VZwKzK/R83FljAMPvocp+8rU2ZvfXhFTWoQjgWZ+HkYr73gQXM\n/cqTNLrryj7+AAAgAElEQVTTI0wozubOaxbx4qYGLj12IrMqC9jV3DPpXGqFu3haKcu/fE6vdJeI\nUJiTSVNniNLcLMYVZuOR3sMFLzh6PB6P9JrmQsTZLs07OC2A/syoyNun7+DtOmlGKafNrhi0b6A/\nCycX95ryw5iRYAHD7CPQp4URjys7m7vIzcpAcO7WfV4PmRlCk9vCyM7MYP6Eol4jjhKVbFFO5j4d\nw337RgAKc7w0dYYoycuisjCbf950Wq8c+Tnzqjinn7vsCUXZTCg+sDv2A/HLjyxKDgN+p2ZU5PP7\nT5xwUK5lzHCzgGH20Z0MGE4Lo7a9m3A0TjgadzuXvYgIeT5vcubQ1JElCYmAUZbm3X+i4ztxfLqd\nn3deswifd+gmLXinwx+NOVzYX4LZR8BdU2JPazc33Lei10Ncce2p2PN93uQzDdn9VNiJEU7pposS\nHd/9tT4GM5StC2NMDwsYBnAW4IlE43xg0aRkC2N7U4DtTYF9hpMWpASMhMFaGOkGgMR4/tKDNCLJ\nGHNwWcAwgDMldF1HkMuOm5jsw0jou/pXYY7za5NuwEg3JVWUk4lHes+YaowZPSxgGMBZ1Ke+I8Tu\nlq7kKClwhq12R2IU5WQmJxxMpqTcFkFWhqffh9rK87PwCGmPMDp3/jiyvJ4Bn5w1xoyskV5AyYwC\nsbgm1yd+Y0dL8olocKaeyM3K6LXITqKvIc9tYfgy+/818nkz+PVHj+cjJ0/t9/W+Tp9dwdcuPvAH\nvowxw8NaGGNYfUeQygIfzZ2hZNrpzR0tdIWjLJpSwvSKPK49eSqnHVFOZWE2b+5ooSMYTfY1FLgB\no790VMKZc+2JJ2MOFxYwxohv/mM9sXicb7jrSGxv7OTsHy7h7o8sTq49kZuVwbKdLXgzPEwsyeHb\n7hQGiVFIFQU+OoLRZKd3XjJgWEPVmLFgv3/pIuIRkWNF5CIReY+I2C3jIej17c08ua4uub18Vytx\ndfbXp6yEVtsWpCsU7ffZg0RfREF2707vnEFaGMaYw8eALQwRmQn8N3A2sAVoxJmNZ7aIdAG/BO5V\n1fhA1zCjR0sgTH1HiJZAmNK8LNbVOgvbr6puS07EN298Ia9ua6axM0Ru1r5BoDzfh8/rSaagEoFj\nsJSUMebwMVhK6lvAncCnVbXXuEq3lXEV8BHg3qErnjkYVJWWLmfOp411HbxrZjlvucuXvlXbzvHT\nS/EIzHafrI7EtN8WxokzygikjKBKpqQOwgyqxpjRb8CAoapXDvJaA/DjISmROegC4RhhdwnUjXv9\nnDi9jPV7O6gs8NHgD/HylibK83291tHur4XxkZOm8pGTekY85e9nlJQx5vCSTh/Gh0SkwP3+KyLy\nsIgcN/RFMwdLayCc/H5TnZ8dTQG6wjGuON5Zy2LtnnbGFWX3esCuv4DRV34ao6SMMYePdG4Nv6Kq\nfhE5FTgL+DVOqsqMcsGI88R2sxswMjzCxrqO5Cp4584fx6mznIV0wtF4r4Xt05lwL/HgnnV6GzM2\npBMwEvNEXATcraqPAzbZzyhX1x5kwdef5s0dLckWxlETCtnWGEgupTq5JJc7rzmOU2aVcdWJUyjJ\n65mSI8+3/yCQl2XDao0ZS9J5DmOPiPwSOAf4noj4sCfER72a1i7CsThratoodifzO3pSEatr2tlY\n5yc3KyM5TfmfPnVS8ryCbC/+YP/DavuyUVLGjC3pVPwfBp4CzlPVNqAU+PyQlsq8Y/6gM5qpuqUr\n2cJYMNFZsW3pzhbGF2Uj/SwKlEhLWR+GMaav/QYMVe0CGoBT3V1RnOcyzCjWEXQmCqxu7aY5ECYz\nQ5g73hk2u7ula8A1JBId3+kEjDyfF5H0jjXGHPr2m3cQka8Bi4E5wG+BTOCPwClDWzTzTnS6z0vs\nbumiIt9HSW4Wk0p6FkIaX5Td73ll+YmAsf+UVJbXw51XL+LYKbbWtDFjQTopqUuB9wEBAFWtBdJb\nO9OMmNSUVHMgRGleFiW5mckRTeOLBmhhHEBKCuD8o8b1en7DGHP4SidghN0nvRVARPKGtkjmYPC7\nKalQNM6mej+leVmICBNLnEAxobj/Sr7cTUnl+WxeSmNMb+kEjIfcUVLFIvJvwLPAr9K5uIicLyKb\nRGSriHyxn9c/LyKr3K+3RCQmIqXuaztFZK372rID+VAGOoM9U3hUt3Qn19We5AaMgVoYp8wq58w5\nFbbqnTFmH/u9jVTV74vIOUAHTj/GV1X1mf2dJyIZwB04w3FrgKUi8qiqrk+59u3A7e7xFwM3q2pL\nymXOVNWmA/lAY9lja2pZW9POrRceiT8YRQQSs4Cd4j6gN7F48BbGiTPKOHFG2bCU1xhzaEnreQpV\nfUZVP6+qt6QTLFwnAFtVdbuqhoEHgEsGOf5K4P40rz3mqSqPrq5NzhEF8NflNfz65R2EojE6glGm\nl+eRmSGcfWRVchqQOeMKyMnMGHCUlDHGDCSduaQuE5EtItIuIh0i4heRjjSuPRGoTtmucff19x65\nwPnAX1N2K/CsiCwXkesGKd91IrJMRJY1NjamUazDw6vbmvnP+1fyf09uTO7b0RQgGle2NnTSGYpQ\nnudj1VfP5VfXLko+c3HlCVN47nOnpzUKyhhjUqXTwvg/4H2qWqSqhapaoKqFB7kcFwOv9ElHnaqq\nC4ELgBtE5LT+TlTVu1V1saourqioOMjFGv2W7moFnLmgqludKT/W13bgD0YpyPa6z0r0PKCXmeGx\n1oUx5m1JJ2DUq+qGt3HtPcDklO1J7r7+XEGfdJSq7nH/bQAewUlxGVc45qSialq6AOd5i8S63Bv2\n+ukMRZNTdxhjzMEw2Ip7l7nfLhORB4G/AaHE66r68H6uvRQ4QkSm4wSKK3AWXer7PkXA6cA1Kfvy\nAI87S24ecC7wzbQ+0RjRHe49E+2OpgDgzBy7fm87/mA0OZusMcYcDIPVKBenfN+FU2knKDBowFDV\nqIh8BmceqgzgN6q6TkSud1+/yz30UuBpVQ2knF4FPOKmUrzAfar6ZBqfZ8zoCsdSvo+yvbETgPcc\nWcnLW5roCkcpyLahscaYg2ewFfc+/k4vrqpPAE/02XdXn+3fAb/rs287cMw7ff/DWXe45zmLLfWd\n7GgKUJaXxckzynh8zV4AS0kZYw6qdEZJ3SsixSnbJSLym6Etltmf7khPC2NTvbOK3vTyPI6Z1DOv\nU4E9rW2MOYjS6fRe4E5rDoCqtgLHDl2RTDpSU1I1LV3UdwQZV5TNnHE903xZSsoYczClEzA8IlKS\n2HCn7rBb1xGwsynAmzuckcfd4Rg+r4eqQh91HUEa/CGqCrPJ8nrwuKNo862FYYw5iNIJGD8AXhOR\n20TkNuBV3Ok8zPD62fNbufH+FYDTwsjJyqCqMJttjQG6wjEqC5yZZmdU5AM2gaAx5uBKZwGl3wOX\nAfXu12XuPjPM2rrCNPpDRGNxusIxcjOdgLGuth0gOc34OfOqAFvYyBhzcKWzgNIfVPUjwPp+9plh\n1BGMEFfn2YvuSNRtYfgIRpyH+BItjFvOncNpR1RwzGRb2MgYc/Ckk5Kan7rhzkK7aGiKYwaTWBSp\noSNEdzhGbpaXcSmLF1W632d4hJNn2oyzxpiDa8CAISK3iogfWJAy6aAfZ33vvw9bCU1SR7ezKFKD\nP9irDyOhqtA3UkUzxowBAwYMVf2OqhYAt6dMOligqmWqeuswltG4ki0Mf4juSIzclICRk5lho6KM\nMUMqnU7vW92H9U4QkdMSX8NRuLGuMxTlxG8/y4ubGojFFX+oJyXVFY6Rk5nBuCInYFQV+nrNSmuM\nMQdbOp3enwJuwpltdhVwEvAa8J6hLZp5bkM99R0hfvXSdo6dknwUhgZ/kO5ESqrACRiVBf2voGeM\nMQdLOp3eNwHHA7tU9Uycp7zbBj/FHAzPrK8HYN74wmT/BfROSRXmeMnO9FBp/RfGmCGWTtI7qKpB\nEUFEfKq6UUTmDHnJxrhgJMbzGxvc7+PJ/gtwAkZXOEpulrM40qdOnWFDaI0xQy6dgFHjTj74N+AZ\nEWkFdg1tsczGOn9yvqhAKEpH0GlhTCjKpr49SDASJyfTeTDvlvMsfhtjht5+A4aqXup++3UReQEo\nAmxtiiGWmoLqDEWTLYyZlfm8tKUJgBx7ktsYM4zS6cNARI4Tkf8EFgA1qhoe2mKZRIAoyPYSCEeT\nAWT+hKLkMTb1hzFmOKWzHsZXgXuBMqAc+K2IfHmoCzbWJVJQ44uy6QzF8LvbJ0zvGS2VSEkZY8xw\nSKcP42rgGFUNAojId3GG135rKAs21iUCxLiiHGrbuulwWxyLppYmj8nNsgf1jDHDJ52UVC2QOsjf\nB+wZmuKY7nCMLfV+/MEoHnEmFAyEoviDEXIyMyjK6VkUyVJSxpjhNOAtqoj8DFCgHVgnIs+42+cA\nbw5P8caW7nCM4//3WTpDUa46cQr5Pi/5Pi+doSgd3dHkGt0F2V78wah1ehtjhtVgOY1l7r/LgUdS\n9r84ZKUZ43783GY63ek/djYFKMjOJN/nTQ6rLXRbF1PLcnlrTwc2EYgxZjgNGDBU9d7hLIiBXU1d\nye93NAUozs0iz+clrtDoDyVbGJ86dQaffXAVE4pzRqqoxpgxyHpNR5FE6wJgb3uQyaW55PuctNPu\nli6OmugMqX3/sRO58OjxZHnTGhVtjDEHhdU4o4g/GGF6eV5yuzDbmxwJ1eAPMb6oZ+yBBQtjzHCz\nWmcU8YeiTC3LTW4XZGeSl7LGRWrAMMaY4TbYKCkv8EngUmCCu3sPzmp7v1bVyEDnmrenMxilqiCb\n7EwPwUicwmxvr0WRxhVZn4UxZuQM1sL4A7AQ+Dpwofv1DeAY4I9DXrIxQFX58bOb2dkUAJw+jIJs\nL2V5zlTlTgujZ+jsBGthGGNG0GCd3otUdXaffTXA6yKyeQjLNGY0+kP8+Nkt/PqlHaz62rl0hWPk\nZ3spL/Cxp62bgn1aGBYwjDEjZ7AWRouIfEhEkseIiEdELgda07m4iJwvIptEZKuIfLGf1z8vIqvc\nr7dEJCYipemcezhodycU9Ieiye/zfV7K87KA/vowLCVljBk5gwWMK4APAvUisllEtgB1wGXua4MS\nkQzgDuACYB5wpYjMSz1GVW9X1YWquhC4FViiqi3pnHs4aE+ZwvzZDc7qegXZXsrznZRUYY43GTCK\nczPtyW5jzIga7MG9ncDlACJS5u5rPoBrnwBsVdXt7jUeAC4B1g9w/JXA/W/z3ENSasD4x+paAPJ9\nmZTlp7Qw3CAxrtDSUcaYkTXosFoRGSci49xA4RGRy0RkfprXnghUp2zXuPv6e59c4Hzgr2/j3OtE\nZJmILGtsbEyzaKNDW5cTMLK8Hjbs9QOQn+2lLD/R6e3Fm+EhO9NjT3UbY0bcgAFDRD4NvIbTyf3v\nwGPARcDDIvLJg1yOi4FXVLXlQE9U1btVdbGqLq6oqDjIxTq4YnGlwR8kFlegp4Uxp6qAps4Q4PRh\nTC/PxSM9rYoJRTnMqswfmUIbY4xrsFFSnwHmAzk4a3jPUtU6ESkBXgB+vZ9r7wEmp2xPYuBp0a+g\nJx11oOceMv738Q385pUdTC7NYcktZyYDxhGV+azd0w44rYrjphSz5PNnJlsVD11/Mnm29oUxZoQN\nlpKKqGqXm47apqp1AKraijPN+f4sBY4QkekikoUTFB7te5CIFAGn4zwQeEDnHmqqW53JBatbumnr\njtDeHaEg28v44p7+iXyfFxFhcmnPE9/l+T7r8DbGjLjBAoaKSGK1nosSO0Ukez/nOSerRnFaKU8B\nG4CHVHWdiFwvItenHHop8LSqBvZ3bpqfadSIxZUv/nUNG+s6AOdJ7oRGf4iO7ghFOZm9OrTzs60l\nYYwZnQarnS7FbUmoak3K/jLgc+lcXFWfAJ7os++uPtu/A36XzrmHmqbOEA8srWZicQ5zxxU6T3L7\nvPhDURr9IdrcgFGVEjAs9WSMGa0GbCmo6m73Tr/v/j2q+uzQFuvw4HdbFHUdQcCZ+mOaOxttY2eQ\n9j4BIy8rgwyPLYtkjBmdbLbaIRRw17eoTwkYienLG/2hZMBITPlh6ShjzGhmAWMIJRZESrYwglGq\nCn3kZGbQ0OEEjOLcTMrysvAIveaNMsaY0Waw5zCec//93vAV5/CSDBjtIaKxON2RGPm+TCoKfDT4\nQ7R3Oet0ezM8VBT4yM/O3M8VjTFm5Ax2SzteRN4FvM+dmqNXcl1VVwxpyQ4DiVFRzQGngxuctFNF\ngY/q1i7CsThFOU6QmFqaR2GOtTCMMaPXYDXUV4Gv4Dw098M+rynwnqEq1OEi0cJQhR3umhcFPi+V\nBT5e3toEkAwYP7z8GLweyxAaY0avwSYf/AvwFxH5iqreNoxlOmwkAgbA1oZOoKeFkRhBlQgYk0py\n972AMcaMIvvNgajqbSLyPuA0d9eLqvrY0Bbr8JAaMLa5ASPP56XCnVwQYP6EomEvlzHGvB37zYGI\nyHeAm3CmFl8P3CQi3x7qgh2qWgJhXtvmzAIfCEWTz1VsbXRbGD5vcpqPK0+YnBxma4wxo106vawX\nAQtVNQ4gIvcCK4EvDWXBDlW/e2UHv3hxG+u/eT6dwSjjCrNp9IeSKamCbC+XLJyIPxjl38+YOcKl\nNcaY9KU7LKcYSEw9bjmUQdR3hIjGlZZA2JkKJNuLz+thu9vpnedz+jBuPqfvcunGGDO6pRMwvgOs\nFJEXcIbWngYclmtsv11ratqYUJxDeb6P5kAYcOaR6gxFyfd5KSjyJgOGPZxnjDlUpTPr7P3AScDD\nOCvinayqDw51wQ4VqsrVv3qDu17cBjjPXAA0doYIhKLk+bxMLevpp7CAYYw5VKVVe6nqXg6D9SiG\nQmNnCH8oSoPfCRQtiRaG39k/qTSXaWXOkNlcm1zQGHMIsyfF3qHqlm4AWrucQNHSmUhJhQmEouRn\neZnqjoTKs9aFMeYQZgHjHapxV9Fr7QoTisbwu89eNHWG6AxGyc/2Ms1NSRVYwDDGHML2W4OJSGk/\nu/2qGhmC8hxyqlvcgBGI0Bro+ZE0+EMEwjHyfF4mleSQ4RGbvtwYc0hLp4WxAmgENgNb3O93isgK\nEVk0lIU7FCRSUm1d4WSHN8BuN5AU+LxkZniYVJJjq+kZYw5p6QSMZ4ALVbVcVcuAC4DHgP8AfjGU\nhRuNmjpDvL69Obld7aakAuEYde3OuhcluZnsTHnuAuCGM2dxzUlTh7m0xhhz8KQTME5S1acSG6r6\nNM7Q2tcB38CnHZ5uemAlV9z9Ou1dTvopETAAtrnTf8wZV0B7ynTmAB9ePJmLFowf5tIaY8zBk07A\n2Csi/y0iU92vLwD1IpIBxIe4fKNOozt8dunOFqKxOLVtQWZUOJ3aiek/5o4rTB6fmI3WGGMOdekE\njKtw1sT4m/s12d2XAXx46Io2Os2qzAfg9e3NNAfCxOLKvPFOgNjWGCDDI0wqyQFgSmkuJ83ob8yA\nMcYcetLphZ2iqjcO8NrWg1mYQ0E4qgC8vqOZDy2eDJCccXZbYycluVm8Z24lq2va+ep75+HzZoxY\nWY0x5mBKp4XxAxHZICK3ichRQ16iUa4z5PRNrK/tYE+b03+RmPqjrSvC5NIcZlTk87Mrj6WiYMx1\n8RhjDmPpzCV1JnAmznDaX4rIWhH58pCXbJRKLIoUVydoAMmpPwBOnlE2IuUyxpihltaT3qpap6o/\nBa4HVuGs9z0mdQajVBU6LYcdTU4LI7Ulccqs8hEplzHGDLV0Vtw7UkS+LiJrgZ8Br+J0go9JnaEo\nU0qdFsXOZudZi9SRUIumloxIuYwxZqil08L4DdAKnKeqZ6jqnaraMMTlGrX8wSiT3YCxyw0YBdk9\nASM70zq5jTGHp/2OklLVk1O3RWQycIWq3r6/c0XkfOAnOENw71HV7/ZzzBnAj4FMoElVT3f37wT8\nQAyIquri/b3fUAtH44SicSaXOAGjqTNMQbaXDI+w7Mtn47Wpy40xh7G0JjcSkQrgQ8CVwATgkTTO\nyQDuAM4BaoClIvKoqq5POaYYZ3qR81V1t4hU9rnMmaralNYnGQYBt8O7JDeTgmwv/mA0mY4qz7cR\nUcaYw9uAAUNECoDLcB7Sm42z4t50VU23/+IEYKuqbnev9wBwCbA+5ZirgIdVdTfAaE91JUZI5Wdn\nUpqXhT8YpTDbnuQ2xowNg/VhNACfAL4FzFDVzwHhA7j2RKA6ZbvG3ZdqNlAiIi+KyHIRuTblNQWe\ndfdfN9CbiMh1IrJMRJY1NjYeQPEOnD/oBgyfl5LcLMCm/jDGjB2DBYxbcSYX/AVwq4jMHIL39wKL\ngIuA84CviMhs97VTVXUhzuy4N4jIaf1dQFXvVtXFqrq4oqJiCIrYI9HCKMj2UppnAcMYM7YMGDBU\n9ceqehJOGgmceaQmuBMRzh7ovBR7cOadSpjk7ktVAzylqgG3r+JfwDHu++9x/23A6TM5IY33HFKJ\np7ythWGMGYvSedJ7u6p+W1WPBhYDhcATaVx7KXCEiEwXkSzgCuDRPsf8HThVRLwikgucCGwQkTy3\nDwURyQPOBd5K+1MNkWRKKttLaZ4TKIpyLWAYY8aGwTq9RVU1dZ+qvgX8j/vV7zEpx0ZF5DPAUzjD\nan+jqutE5Hr39btUdYOIPAmswZkq/R5VfUtEZgCPiEiijPep6pPv9MO+U8mUlM9LiaWkjDFjzGDD\nal8Qkb8Cf0+MYgJwWwunAh8FXgB+N9AFVPUJ+rRGVPWuPtu3A7f32bcdNzU1Gnzwzle59LiJdKa2\nMNyUVKEFDGPMGDFYwDgfZ5TU/SIyHWgDsnFaC08DP1bVlUNfxJEVjsZZtquVqqJsZpTn4RHIycyw\nTm9jzJgzYMBQ1SDOCKlfiEgmUA50q2rbcBVuNGgJOCOJq1u6qMj3ke/zIiJMdBdJmlCUPZLFM8aY\nYZPubLURVd071oIFQFOnsyTrruYuOkPR5LxR8ycU8czNp9lkg8aYMSOtqUHGskTAaO+OsGFvB+X5\nWcnXjqgqGKliGWPMsEurhTGWNXf2PNy+rraDY6dYi8IYMzalsx7GjSIyZmvJ5kCo1/ZxloIyxoxR\n6bQwqnBmmn1IRM4X9+GIsaK5M0xGyrTl1mdhjBmr0nnS+8vAEcCvgY8BW0Tk20M0t9So09gZYlxh\nNqV5WYwrzLZRUcaYMSutTm9VVRGpA+qAKFAC/EVEnlHVLwxlAUdac2eY8vwsppTlUZ6fxRhrYBlj\nTNJ+A4aI3ARcCzQB9wCfV9WIiHiALcDhHTACISoLsvnZlceOdFGMMWZEpdPCKAUuU9VdqTtVNS4i\n7x2aYo0ezZ1hjhxXONLFMMaYEZdOp/c/gZbEhogUisiJAKq6YagKNhqoqpOSKrDlV40xJp2AcSfQ\nmbLd6e477HUEo4RjccrysvZ/sDHGHObSCRi9pjBX1Thj5AnxPa3dAEwozhnhkhhjzMhLJ2BsF5H/\nFJFM9+smYPtQF2w02N3SBcCU0twRLokxxoy8dALG9cC7cJZXrcFZFe+6oSzUaFHtBozJFjCMMWb/\nqSV3Te0rhqEso87uli6KcjJtzQtjjCG95zCygU8C83EWUAJAVT8xhOUaFXa1dDG1zFoXxhgD6aWk\n/gCMA84DlgCTAP9QFmq0qG7psnSUMca40gkYs1T1K0BAVe8FLsLpxzisxeJKTWuXdXgbY4wrnYAR\ncf9tE5GjgCKgcuiKNDrsbe8mElMLGMYY40rneYq73fUwvgw8CuQDXxnSUo0Cu5ttSK0xxqQaNGC4\nEwx2qGor8C9gxrCUahTY1ug83D6zIn+ES2KMMaPDoCkp96nuw3o22oFsawyQl5VBVaHNI2WMMZBe\nH8azInKLiEwWkdLE15CXbIRta+xkRkW+rX9hjDGudPowLnf/vSFln3KYp6e2NwZYPM2WYzXGmIR0\nnvSePhwFGU26wzH2tHVzecXkkS6KMcaMGuk86X1tf/tV9fcHvzijw46mAAAzKvJGuCTGGDN6pNOH\ncXzK17uBrwPvS+fiInK+iGwSka0i8sUBjjlDRFaJyDoRWXIg5w6VxAipGeU2QsoYYxLSSUndmLot\nIsXAA/s7T0QygDuAc3BmuV0qIo+q6vo+1/oFcL6q7haRynTPHUpb6v14xFoYxhiTKp0WRl8BIJ1+\njROAraq6XVXDOEHmkj7HXAU8rKq7ITkzbrrnDplN9X6mleeRnZkxXG9pjDGjXjp9GP/AGRUFToCZ\nBzyUxrUnAtUp24m1NFLNBjJF5EWgAPiJ2zeSzrmJ8l2Huz7HlClT0ijW/m2u7+TI8QUH5VrGGHO4\nSGdY7fdTvo8Cu1S15iC+/yLgLCAHeE1EXj+QC6jq3cDdAIsXL9b9HL5fwUiMnc0BLlk44Z1eyhhj\nDivpBIzdwF5VDQKISI6ITFPVnfs5bw+QOi51krsvVQ3QrKoBICAi/wKOcffv79whsaW+E1WYU2Ut\nDGOMSZVOH8afgXjKdszdtz9LgSNEZLqIZOGs2vdon2P+DpwqIl4RycVJO21I89whsaneWepj9jgL\nGMYYkyqdFobX7XgGQFXDbiU+KFWNishngKeADOA3qrpORK53X79LVTeIyJPAGpygdI+qvgXQ37kH\n+uHejs31frK8HqbaLLXGGNNLOgGjUUTep6qPAojIJUBTOhdX1SeAJ/rsu6vP9u3A7emcOxw21fmZ\nVZGPN+PtDCAzxpjDVzoB43rgTyLyc3e7Buj36e/DweZ6PyfPKBvpYhhjzKiTzoN724CTRCTf3e4c\n8lKNkPbuCHvbg9Z/YYwx/dhv3kVEvi0ixaraqaqdIlIiIt8ajsINty1uh7eNkDLGmH2lk6i/QFXb\nEhvu6nsXDl2RRo6NkDLGmIGlEzAyRCS57JyI5ACH5TJ0m+v85Pu8TCjKHumiGGPMqJNOp/efgOdE\n5Lfu9seBw3Jq853NXUwvz7NV9owxph/pdHp/T0RWA2e7u25T1aeGtlgjo649yJQye/7CGGP6k9bD\nBqs1DMYAAAf+SURBVKr6pKreoqq34EzhcccQl2tE1HUEGVdo6ShjjOlPOikpRORY4Ergw8AO4OGh\nLNRICEZitHdHGGf9F8YY068BA4aIzMYJElfiPNn9ICCqeuYwlW1Y1bUHAayFYYwxAxishbEReAl4\nr6puBRCRm4elVCOgrsMNGNbCMMaYfg3Wh3EZsBd4QUR+JSJnAYft8KF6N2BUWQvDGGP6NWDAUNW/\nqeoVwFzgBeCzQKWI3Cki5w5XAYdLMiVlLQxjjOnXfkdJqWpAVe9T1YtxFjJaCfz3kJdsmO1tD1Lg\n85LvS2scgDHGjDkHNIe3qraq6t2qetZQFWik1HcEqbLWhTHGDMgWfXDZMxjGGDM4Cxiu9q4IxbmZ\nI10MY4wZtSxguDpDUQqyrf/CGGMGYgHDFQhFycuygGGMMQOxgAHE40ogHCPPRkgZY8yALGAAgXAU\nwFJSxhgzCAsYQCAUA7AWhjHGDMICBtD5/+3dT4hd5RnH8e+vM2ZSJsFWU2NI0naELKpFo6RZSJBW\nqdVQSAtdzMosCqFFxS5cRATJQpAWWrqwKCkVRGqz0egsbItpCxYEzSgxf0xTxzRihmiigiaDGTPp\n08V5p7kMc2aOd+b4zjn394HLPfc9Z2aehwfOM+977r1n8gLghmFmNhc3DOBcmmGsGOjLHImZ2dLl\nhkHxDimAFQP+HIaZWRk3DODs+aJhDHqGYWZWyg2DzhmGr2GYmZVxw+DS22p90dvMrFytDUPSHZKO\nSRqTtHOW/d+V9LGkA+nxUMe+E5IOpfHROuOcXpLyDMPMrFxtZ0hJfcDvgO8DJ4H9kkYi4s0Zh/4z\nIn5Y8mu+FxEf1BXjtInJKfq/JAb6PeEyMytT5xlyMzAWEccj4jNgD7Ctxr/XtYnJKQYH+pFaewda\nM7MFq7NhrAXe7Xh9Mo3NdLOkg5L+LOm6jvEA9kl6TdKOsj8iaYekUUmjZ86c6SrQs5NTXo4yM5tH\n7rPk68DXI+KcpK3Ac8CGtG9LRIxLugp4UdK/IuKlmb8gInYDuwE2bdoU3QQx4YZhZjavOmcY48D6\njtfr0tj/RcQnEXEubb8AXCZpVXo9np5PA3splrhqMTF50Z/BMDObR50NYz+wQdKQpGXAMDDSeYCk\nq5UuHEjanOL5UNKgpJVpfBC4HThcV6Bn0zUMMzMrV9tZMiKmJN0D/BXoA56IiCOSfpb2Pw78BPi5\npCngU2A4IkLSamBv6iX9wNMR8Ze6Yp2YnGLtV3w/bzOzudT6b3VaZnphxtjjHduPAo/O8nPHgRvq\njK2T77ZnZjY/f/AAOHfeS1JmZvNxwwBu+9ZVXL/u8txhmJktaf63Gvjt8I25QzAzW/I8wzAzs0rc\nMMzMrBI3DDMzq8QNw8zMKnHDMDOzStwwzMysEjcMMzOrxA3DzMwqUURXt5BYkiSdAd7p8sdXAbXf\nDjYz59gOvZAj9EaeSyHHb0TE16oc2KqGsRCSRiNiU+446uQc26EXcoTeyLNpOXpJyszMKnHDMDOz\nStwwLtmdO4AvgHNsh17IEXojz0bl6GsYZmZWiWcYZmZWiRuGmZlV0vMNQ9Idko5JGpO0M3c8i0XS\nCUmHJB2QNJrGrpD0oqS30vNXc8f5eUl6QtJpSYc7xkrzkvRAqu0xST/IE/XnU5LjLknjqZ4HJG3t\n2NfEHNdL+oekNyUdkXRfGm9NLefIsbm1jIiefQB9wNvANcAy4A3g2txxLVJuJ4BVM8Z+BexM2zuB\nX+aOs4u8bgFuAg7PlxdwbarpADCUat2XO4cuc9wF3D/LsU3NcQ1wU9peCfw75dKaWs6RY2Nr2esz\njM3AWEQcj4jPgD3Atswx1Wkb8GTafhL4UcZYuhIRLwEfzRguy2sbsCciJiPiP8AYRc2XtJIcyzQ1\nx1MR8XraPgscBdbSolrOkWOZJZ9jrzeMtcC7Ha9PMndBmySAfZJek7Qjja2OiFNp+z1gdZ7QFl1Z\nXm2r772SDqYlq+mlmsbnKOmbwI3AK7S0ljNyhIbWstcbRpttiYiNwJ3A3ZJu6dwZxRy4de+pbmte\nwGMUS6cbgVPAr/OGszgkrQCeAX4REZ907mtLLWfJsbG17PWGMQ6s73i9Lo01XkSMp+fTwF6Kqe37\nktYApOfT+SJcVGV5taa+EfF+RFyMiP8Cv+fSUkVjc5R0GcWJ9I8R8WwablUtZ8uxybXs9YaxH9gg\naUjSMmAYGMkc04JJGpS0cnobuB04TJHb9nTYduD5PBEuurK8RoBhSQOShoANwKsZ4luw6ZNo8mOK\nekJDc5Qk4A/A0Yj4Tceu1tSyLMdG1zL3VffcD2ArxbsX3gYezB3PIuV0DcW7Ld4AjkznBVwJ/A14\nC9gHXJE71i5y+xPFNP4CxRrvT+fKC3gw1fYYcGfu+BeQ41PAIeAgxYllTcNz3EKx3HQQOJAeW9tU\nyzlybGwt/dUgZmZWSa8vSZmZWUVuGGZmVokbhpmZVeKGYWZmlbhhmJlZJW4YZmZWiRuGmZlV4oZh\nVhNJ30lfMLc8ffr+iKRv547LrFv+4J5ZjSQ9DCwHvgycjIhHModk1jU3DLMape8o2w+cB26OiIuZ\nQzLrmpekzOp1JbCC4o5ryzPHYrYgnmGY1UjSCMWdHIcovmTunswhmXWtP3cAZm0l6S7gQkQ8LakP\neFnSrRHx99yxmXXDMwwzM6vE1zDMzKwSNwwzM6vEDcPMzCpxwzAzs0rcMMzMrBI3DDMzq8QNw8zM\nKvkfECH6Wvr/m5EAAAAASUVORK5CYII=\n", 1292 | "text/plain": [ 1293 | "" 1294 | ] 1295 | }, 1296 | "metadata": {}, 1297 | "output_type": "display_data" 1298 | } 1299 | ], 1300 | "source": [ 1301 | "avg_loss = list()\n", 1302 | "avg_acc = list()\n", 1303 | "avg_perp = list()\n", 1304 | "i = 0\n", 1305 | "while(i