├── .gitignore ├── .whitesource ├── LICENSE ├── README.md ├── _config.yml └── labs ├── activation_functions.ipynb ├── batchnorm_adaptive_opt.ipynb ├── classifier_xor.ipynb ├── eda_titanic.ipynb ├── exploring_attention.ipynb ├── exploring_rnn.ipynb ├── hello_gpu.ipynb ├── hello_mnist.ipynb ├── image_classification.ipynb ├── learning_representations.ipynb ├── linear_regression_sgd.ipynb ├── lm_horizon.ipynb ├── loading_vis_nlp.ipynb ├── nmt_on_tpu.ipynb ├── normalization_initialization.ipynb ├── text_classification_cnn.ipynb ├── transfer_learning.ipynb ├── word_embeddings.ipynb └── working_with_images.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "generalSettings": { 3 | "shouldScanRepo": true 4 | }, 5 | "checkRunSettings": { 6 | "vulnerableCheckRunConclusionLevel": "failure" 7 | } 8 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 OpenSUTD 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning Workshop 2 | 3 | Introductory Machine Learning and Deep Learning Workshop 4 | 5 | ![GitHub last commit](https://img.shields.io/github/last-commit/OpenSUTD/machine-learning-workshop.svg) ![GitHub](https://img.shields.io/github/license/OpenSUTD/machine-learning-workshop.svg) ![](https://img.shields.io/github/repo-size/OpenSUTD/machine-learning-workshop.svg) 6 | 7 | ## Notebooks 8 | 9 | >**Running the Notebooks** 10 | > 11 | >Do note that you'll get Colab's usual warning about running an untrusted notebook. That is expected behavior for opening a third-party notebook on Colab. Just click past that. 12 | 13 | ### Deep Learning Fundamentals 14 | 15 | | Notebook Title | Link | 16 | | ------------------------------------ | ---- | 17 | | Linear Regression with SGD | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/linear_regression_sgd.ipynb) | 18 | | Activation Functions | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/activation_functions.ipynb) | 19 | | Normalization & Initialization | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/normalization_initialization.ipynb) | 20 | | BatchNorm & Adaptive Optimizers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/batchnorm_adaptive_opt.ipynb) | 21 | | XOR Classifier | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/classifier_xor.ipynb) | 22 | | Hello MNIST | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/hello_mnist.ipynb) | 23 | | DL as Representation Learning | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/learning_representations.ipynb) | 24 | 25 | 26 | ### Computer Vision 27 | 28 | | Notebook Title | Link | 29 | | ------------------------------------ | ---- | 30 | | Working with Images | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/working_with_images.ipynb) | 31 | | Image Classification | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/image_classification.ipynb) | 32 | | Transfer Learning | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/transfer_learning.ipynb) | 33 | 34 | 35 | ### Natural Language 36 | 37 | | Notebook Title | Link | 38 | | ------------------------------------ | ---- | 39 | | Loading and Visualising NLP Data | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/loading_vis_nlp.ipynb) | 40 | | Word Embeddings | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/word_embeddings.ipynb) | 41 | | Text Classification with CNN | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/text_classification_cnn.ipynb) | 42 | | Exploring RNNs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/exploring_rnn.ipynb) | 43 | | Deep Dive into Self-Attention | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/exploring_attention.ipynb) | 44 | | Transfer Learning in NLP | WIP | 45 | 46 | 47 | ### Hardware Accelerators 48 | 49 | | Notebook Title | Link | 50 | | ------------------------------------ | ---- | 51 | | Hello GPU | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/hello_gpu.ipynb) | 52 | | Neural Machine Translation on TPUs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/OpenSUTD/machine-learning-workshop/blob/master/labs/nmt_on_tpu.ipynb) | 53 | | TF Performance Optimizations | WIP | 54 | 55 | 56 | ## Other Materials 57 | 58 | * [Workshop Slides](https://drive.google.com/open?id=1yFUOCdwcvAwMPjRhd37u85h9b78TXS4u) 59 | 60 | ## Contributing 61 | 62 | Any relevant contributions are welcome! Feel free to open a pull request. 63 | 64 | **How To Contribute** 65 | 66 | You can fork this repository into your own GitHub account, then clone it into your Google Drive folder on your local computer. This way, you'll be able to push commits to your own fork, and submit a Pull Request to this repository. 67 | 68 | **Simple Requirements** 69 | 70 | 1. Only use "common" libraries 71 | 2. Design to run on Google Colaboratory 72 | 3. Make an effort to keep code simple and understandable 73 | 74 | **Core Maintainer** 75 | 76 | * Timothy Liu / [@tlkh](https://github.com/tlkh) 77 | 78 | **Other Authors** 79 | 80 | * Soh Jun De / [@SolsticeDante](https://github.com/SolsticeDante) 81 | * Aiden Chia / [@aidenchia](https://github.com/aidenchia) 82 | * Rishabh Anand / [@rish-16](https://github.com/rish-16) 83 | 84 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /labs/classifier_xor.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"classifier_xor.ipynb","provenance":[],"collapsed_sections":["twLvQhoqw7h9"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"twLvQhoqw7h9","colab_type":"text"},"source":["## Setup"]},{"cell_type":"code","metadata":{"id":"L-q9WB-9wzWA","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":0},"outputId":"0f9cc7e8-0663-4cc3-9013-e51d440e1dce","executionInfo":{"status":"ok","timestamp":1578411791195,"user_tz":-480,"elapsed":52681,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}}},"source":["!pip install tensorflow==2.0 -Uq"],"execution_count":1,"outputs":[{"output_type":"stream","text":["\u001b[K |████████████████████████████████| 86.3MB 68kB/s \n","\u001b[K |████████████████████████████████| 3.8MB 41.8MB/s \n","\u001b[K |████████████████████████████████| 450kB 33.1MB/s \n","\u001b[K |████████████████████████████████| 81kB 8.9MB/s \n","\u001b[31mERROR: tensorboard 2.0.2 has requirement grpcio>=1.24.3, but you'll have grpcio 1.15.0 which is incompatible.\u001b[0m\n","\u001b[31mERROR: google-colab 1.0.0 has requirement google-auth~=1.4.0, but you'll have google-auth 1.10.0 which is incompatible.\u001b[0m\n","\u001b[?25h"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"u_mJYXACA3ui","colab_type":"code","colab":{}},"source":["from IPython.display import clear_output\n","import matplotlib.pyplot as plt\n","from matplotlib.animation import *\n","import numpy as np\n","import tensorflow.compat.v2 as tf"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"XQcNrDJHw-J-","colab_type":"text"},"source":["# Neural XOR Gate in TF 2.0"]},{"cell_type":"markdown","metadata":{"id":"8OTYZYsAxsXm","colab_type":"text"},"source":["## Data"]},{"cell_type":"code","metadata":{"id":"HCo6oXvjBMmG","colab_type":"code","colab":{}},"source":["inputs = [[0, 0],\n"," [0, 1],\n"," [1, 0],\n"," [1, 1]]\n","\n","outputs = [0, 1, 1, 0]\n","\n","inputs, outputs = np.asarray(inputs, dtype=\"float\"), np.asarray(outputs, dtype=\"float\")"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"UiURpaRfxwOj","colab_type":"code","colab":{}},"source":["x_1 = np.arange(-1.0, 2.1, 0.1).tolist()\n","x_2 = np.arange(-1.0, 2.1, 0.1).tolist()\n","xv, yv = np.meshgrid(x_1, x_2)\n","x = np.stack([xv, yv], axis=-1)\n","x = x.reshape((len(x_1)*len(x_2), 2))"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"qziVvm1uxtSd","colab_type":"text"},"source":["## Model"]},{"cell_type":"code","metadata":{"id":"IYflO_58BnPg","colab_type":"code","colab":{}},"source":["model = tf.keras.models.Sequential([\n"," tf.keras.layers.Dense(8, activation=\"tanh\"),\n"," tf.keras.layers.Dense(1, activation=\"sigmoid\")\n","])\n","\n","model.compile(loss=\"binary_crossentropy\",\n"," optimizer=\"sgd\")\n","\n","loss = model.evaluate(inputs, outputs, verbose=0)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"vAZ_vCsLxpdj","colab_type":"text"},"source":["## Fitting with SGD"]},{"cell_type":"code","metadata":{"id":"oJvjtLK1yekT","colab_type":"code","outputId":"ec9ca41d-517d-40f6-d86f-1dbb160f9033","executionInfo":{"status":"ok","timestamp":1578411996051,"user_tz":-480,"elapsed":435,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":390}},"source":["moviewriter = FFMpegWriter(fps=10)\n","fig, ax = plt.subplots(figsize=(6,6))\n","moviewriter.setup(fig, 'xor.mp4', dpi=100)\n","while loss > 0.05:\n"," preds = model.predict(x).reshape((len(x_1), len(x_2)))\n"," ax.cla()\n"," CS = ax.contour(x_1, x_2, preds, levels=[0.1,0.5,0.9], colors=['r', 'k', 'b'])\n"," ax.clabel(CS, inline=1, fontsize=10)\n"," ax.scatter([0,1], [0,1], label=\"0\", color=\"r\")\n"," ax.scatter([0,1], [1,0], label=\"1\", color=\"b\")\n"," ax.legend()\n"," plt.title(\"XOR Decision Boundary, loss=\" + str(loss))\n"," moviewriter.grab_frame()\n"," model.fit(inputs, outputs, batch_size=1, epochs=3, verbose=0)\n"," loss = round(model.evaluate(inputs, outputs, verbose=0), 2)\n"," print(\"Loss:\", loss)\n"," clear_output(wait=True)\n","moviewriter.finish()"],"execution_count":6,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYQAAAF1CAYAAADoc51vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd3gU1deA3wsJvUtv0hFBQEABKSKo\nCCJYaAoiIEWRIkVUyg8QEBDhEwQpEpoUpYogXQm9GEITBOkm9GKAFEKSPd8fdwMBA2mbmd3kvs8z\nT7Izs3POzs7OmXvuKUpEMBgMBoMhjd0KGAwGg8E9MAbBYDAYDIAxCAaDwWBwYgyCwWAwGABjEAwG\ng8HgxBgEg8FgMADGIBgsRCkVrJQqEcc+dZRSx6zSyQ6UUsWUUqKU8rJB9lCl1Dyr5Ro8A2MQ3BCl\nVBal1BmlVJsY67Iqpf5RSjWPse45pdTvSqlbSqkbSqmVSqknY2yvp5RyOG/Et5RSx5RSHR4hN/pG\nFexcLimlVimlXnLF5xKRLCJyKo59topIWVfIi8lDPtt3SilvV8syxI1SqrJSaq9SKtT5t/Ij9s2l\nlFqulApRSp1VSr3zwPY8SqkFzt/Av0qp+cn/CVImxiC4ISISDHQFvlFK5XGu/grwE5ElAEqpmsB6\nYAVQECgOHAC2P/AUfl5EsgDZgN7A90qpuG64OZzvqQRsAJYrpdq75MPZT/RnewqoCXxksz4Jwo5R\nhatRSqVDX7fzgJzAHGCFc31sTAbuAPmANsAUpVT5GNuXAReBokBe4OtkUj3lIyJmcdMFmA0sBOoB\n14D8MbZtBb6L5T1rgLnO/+sBgQ9svwy0eIi8YoAAXg+s7wdcAtI4XxcElgJXgNNAzxj7pgUGACeB\nW8BeoIhzmwClnP83Bo449zkH9ItNZ6Ac4AsEAYeBpg+cn8nAr87j7AZKxvezoY3s9HjK8gU6xXjd\nHtgW47UAHwDHne+fDKgY5+Rr4CpwCm2E7uoCdAD+cn6GU0DXGMetBwQCn6Jvej8AfwKvxdjH23ns\np+NxTQ0F5sV43dT5WYOcn7FcjG2fOr+bW8AxoIFz/bOAH3DTeV2MT+B1/bLzuCrGun+AV2LZNzPa\nGJSJse4HYHSMY50B0tr9e00JixkhuDe90TeEJegb5kUApVQm4DlgcSzvWQT8x8WjlEqjlGoK5AZO\nJFCPZegnr7JKqTTASvRopBDQAPhYKdXQuW8f4G30DT8b0BEIjeWYPugbX1agAvB7LDp7O2Wtd8rv\nAcx/YITTGhiGftI8AYyMzwdSShUEGgK7EiArLpoAzwAVgZbO4wN0dm57GqgGNH/gfZed27OhjcP/\nKaWqxNieH8gFPA50AeYCbWNsbwxcEJF9CdAVpVQZ9APHx0AeYDWwUimVzvm5uwPPOL+jhugbL8AE\nYIKIZANKoq+56GMGPWL5zLlbeeCgOO/oTg461z9IGSBSRP6Ose5AjH1roI3VHKXUNaXUH0qp5xNy\nHgz3MAbBjRGRf9FPb5nQN+VocqG/uwuxvO0C+qYfTUGlVBAQBiwH+iT0xgGcjyH3GSCPiHwhIndE\nzwl8j74xA3QCBonIMdEcEJFrsRwzAnhSKZVNRP4VEf9Y9qkBZEE/Dd4Rkd+BVWiDE81yEdkjIpHA\nfOChvmgnV53n4xwQgja28ZUVF6NFJEhE/gE2xdClJfCNiASIyHVgVMw3icivInLSeb42o41SnRi7\nOIAhIhIuImFoV0tjpVQ25/Z30U/NCaUV8KuIbBCRCPQoJiP6YSMKSI/+jrxF5IyInHS+LwIopZTK\nLSLBIrIrxmfJ8YhltHO3LMCNB3S5AWSNRccs6JHIw/YtjB4lbEIbznFo91NuDAnGGAQ3RinVFu3q\n2AiMibHpX/RNokAsbyuAdh9Ec15EcqCfPicC9ROhSiHn3+vop9SCMZ/80C6ifM59iqDdRXHxFvrJ\n9qxSarNzTuRBCgIBIuKIse5sDH1Au1GiCUXfQB5Fbuf5yARsB9YlQFZcPEyXgkDAA8e9i1KqkVJq\nl1LquvN8NuZ+o35FRG5HvxCR807d31JK5QAaoY1hQikYUxfnZw8AConICfTIYShwWSn1o3NUBfA+\n+sn9qPOJvEkC5Qajr8eYZEO7phK6bxhwRkR8RCRCRH50foZaCdTJgDEIbotSKi/wf2h3Q1egpVKq\nDoCIhAA7gRaxvLUl8NuDK0UkHO0Tfkop9XoC1XkD7dY4hv6xnX7gyS+riDR27huAdiM8EhH5Q0Sa\nod0zPxPD7RCD80ARp5sqmqLop/sk4XzSng3UcD5NxiUrBG1EosmfAHEX0IYy5nEBUEqlR8/HfA3k\ncxqr1YCKqW4sx5yDdhu1AHaKSGLOyXm0gY/WRTn1PAcgIgtEpLZzH8H5UCIix0XkbfR3NwZYopTK\n7DxG8COWAU5Rh4GKTnnRVHSuf5C/AS+lVOkY6yrF2Pcg/z0/poRzIjEGwX2ZBPwsIptE5ALQHx0h\nlN65/TPgPaVUT2dIak6l1Ah05Myw2A4oInfQQ+r/xUcBpVQ+pVR3YAjwufMJcg9wSyn1qVIqo1Iq\nrVKqglLqGefbZgDDlVKllaaiUuqxB46bTinVRimV3emquIke8TzIbvSTdn+llLdSqh7wGvBjfPSP\n47OlR7taLqIn7OOStR94UymVSSlVCv2UHF8WAT2VUoWVUjnR31006dCumStApFKqEdoFEhc/A1WA\nXug5hZif7Uw8o8IWAa8qpRo451D6AuHADqVUWaVUfed5uo1+Enc4j99WKZXHeT0EOY/lgLuhxQ9b\nvnTu64t2SfVUSqV3XmMQyzyS8+FnGfCFUiqzUqoW0Ix7LrLlQE6l1HvOa7E52o20PR6f3/AgVs9i\nmyXuBXgd/fSW44H1vwMjY7yujf5xBaNvqr8CFWJsr8d/o4wyoV1Kr8Uitxj66SoY/UR8Gf20+soD\n+xVET0ZeRLuvdgEvOrelBQaho49uAX8AhZ3bBCiFvgmudb73pnOf2rHpjJ483Iz2Gx8B3oixbTYw\n4lGf9yGfLRh9I9uMnjSNj6zcaN/+LfTNZij/jTIqFZtugBd6tHfNeV4ejDL6CB2tE4S+0f0Y472P\n+kwznN9Tlhjr0jl1fOIh7xnK/VFGbzg/6w3nZy/vXF8Rp/FHuwpXAQWd2+Y5r41g9JP664m4xp9G\nR6CFAf7EiJBCuyDXxHidC20AQ9DRSO88cKw6wCGnPn5AHbt/w566RIfFGQwGD0Mp9T90OGbbGOtq\nAx+JdukYDAnCGASDwQNRSuUC9gHvisgWu/UxpAySPIeglCqilNqklDqilDqslOoVyz5KKTVRKXVC\nKXXwgRhrg8GQAJRSndGT92uMMTC4kiSPEJRSBYACIuKvlMqK9gu+LiJHYuzTGJ3o0xiojk5qqZ4k\nwQaDwWBwKUkeIYjIBXEmFYnILXQK/oOx283Q5RREdBJLDqchMRgMBoOb4NKwU6VUMXT0wO4HNhXi\n/sScQBKW8GMwGAyGZMZllROVUlnQCTYfi8iDqebxPUYXdK0WMmfOXPWJJ55wlXoGQ4rn1CkICoLy\n5SF9+rj3dyXHjx/n1q1blCtXjowZM1orPDYuX4aAAChcGPLli3v/FMTevXuvikieuPeMBVfErqKr\nLa5D18mJbfs04O0Yr4+h5x0eesyqVauKwWCIH2vXioDIiBHWy540aZIAMnHiROuFx8bevSLp0ok0\naSLicNitjeWgy+Tbk4fgTD+fA1wXkY8fss+r6MqJ0ZPKE0Xk2Ucdt1q1auLn55ck3QyG1MDt2/DU\nU5AmDRw8aO3o4PDhw1SrVo369euzatUq7q9GYQO3bkGVKhAWBvv3Q+7UV+NOKbVXRKol5r2ucBnV\nQpcAOKSU2u9cNwBnvRYRmYrOdm2MLk8cii7xazAYXMDYsXDiBKxfb60xCA8P55133iFbtmzMnDnT\nfmMgAh98oH1nvr6p0hgklSQbBBHZxv2FuGLbR/CwzlQGgydw6hR8+SW0bAkvuaTRafwZPHgwBw8e\nZNWqVeRzBz/9rFmwYAF88QXUqRP3/ob/4PHt+AyG1IoI9OgBXl4wfry1sjdv3szXX39N165defXV\nV60VHhtHjkD37lC/PgwYEPf+cRAREUFgYCC3b9+Oe2ebyJAhA4ULF8bb23VtwY1BMBg8lBUrYPVq\nGDcOClkYxH3jxg3atWtHyZIl+fprN2hfHBYGrVpBliwwbx6kTZvkQwYGBpI1a1aKFStmvyssFkSE\na9euERgYSPHixV12XGMQDAYPJCQEevXSk8k9elgru1evXgQGBrJ9+3ayZImrH5EFDB4Mf/4Ja9dC\nAdfku96+fdttjQGAUorHHnuMK1euuPS4xiAYDB5I377wzz+wdSu40GMQJ0uXLmXOnDkMHjyYGjVq\nWCf4YRw5AhMmQKdO0LBh3PsnAHc1BtEkh36mQY7B4GHMmQPTpsGnn0Lt2tbJvXDhAl27dqVq1aoM\nHjzYOsEPI3oSJWtWGDUq7v09jLVr11K2bFlKlSrF6NGj436DCzAGwWDwIP76Cz78EOrVgxEjrJMr\nIrz//vuEhIQwb948l05kJpp58+D332H48BQXYhoVFcVHH33EmjVrOHLkCAsXLuTIkSNxvzGJGINg\nMHgI4eHw9tuQObOOrvSy0OE7depU1qxZw9ixY3GLkjLnzkHPnlCrls49sJv586FYMZ0dWKyYfp0E\n9uzZQ6lSpShRogTp0qWjdevWrFixwiWqPgpjEAwGD2HAADhwAGbOdNncabw4duwYffv2pWHDhnz0\nkRukE4lAly7aQs6a5ZKooiQxf77W5+xZrdvZs/p1EozCuXPnKFKkyN3XhQsX5ty5c67Q9pEYg2Aw\neADr1ulcg48+gtdes05uREQEbdu2JWPGjO6RjQwwe7aOtx09GkqXtlsbGDgQQkPvXxcaqtd7GCbK\nyGBwcy5fhvfe01VMx461Vvbw4cPx8/Nj8eLFFCxY0FrhsREQAB9/DHXr6kQ0d+CffxK2Ph4UKlSI\ngIB7HQMCAwMpZEGyiRkhGAxujAh07KjLWi9YAFZWlt61axcjR46kXbt2NG/e3DrBD0NEh5dGRWlX\nURo3uX0VLZqw9fHgmWee4fjx45w+fZo7d+7w448/0rRp00QfL764yRk1GAyxMXky/PqrHhlUrGid\n3ODgYNq2bUuRIkWYOHGidYIfxYwZuoLfV19BiRJ2a3OPkSMhU6b712XKpNcnEi8vLyZNmkTDhg0p\nV64cLVu2pHz58klUNB4ktm52ci+mH4IhtXPwoEj69CKNG1tf1r9z586ilJItW7ZYK/hhnDkjkiWL\nSIMGIlFRyS7uyJEjCXvDvHkijz8uopT+O29ecqj1H2LTkyT0QzBzCAaDGxIWBu+8AzlyaO+IlXO5\nv/zyC99//z2ffvopddyhaqjDof1mAD4+7uMqikmbNnrxcIxBMBjckP79dXmeNWsgb17r5F66dIlO\nnTpRqVIlhg0bZp3gRzFtmk5Amz4dHn/cbm1SNMYgGAxuxqpVMGmSDqZ55RXr5IoIHTt25ObNm2za\ntIn0Vjdmjo1Tp+CTT3Sdok6d7NYmxWMMgsHgRty4oXOaKla0vjzP9OnTWb16NRMnTrRmAjMuIiOh\nbVudeDZjhrV+s1SKMQgGgxsxeDBcvAi//AIZMlgn99SpU/Tt25eXXnqJ7u4S3z9sGOzcCT/+CIUL\n261NqsANZ2cMhtSJv78OM+3WDaolqkV64nA4HLRv3560adPi4+PjHtnImzfrsM0OHXTzG4MlGINg\nMLgBN2/qqKK8ea2tYgowYcIEtm7dysSJE++rn2Mbd+5A585QsiS4Sw6EDXTs2JG8efNSoUIFy2Qa\ng2Aw2Ex0NvKJE7BwoQ41tYqjR48yYMAAmjZtSrt27awT/CgmTIDjx+Hbb3VbzFRK+/btWbt2raUy\njUEwGGxm/HhYulRPIterZ53cyMhI3nvvPTJlysS0adPcw1V04QJ88QU0aWJtiFUScXH1awDq1q1L\nrly5kn6gBGAmlQ0GG9myRXc+e/NN6NfPWtlfffUVe/bs4aeffiJ//vzWCn8Yn3+uXUb/9392axJv\noqtfRxc8ja5+DZ6Xq2ZGCAaDTZw/Dy1bale51dnIBw4cYOjQobRq1YqWLVtaJ/hR7N6t+4P27g2l\nStmtTbxJQdWvzQjBYLCDiAhtDG7dgt9+g2zZrJN9584d2rVrR65cuZg8ebJ1gh+Fw6H7Ixco4HF3\n0mSofm0bxiAYDDbQvz9s365LWludA/bFF19w8OBBfvnlFx577DFrhT+MOXPgjz9g7lzImtVubRJE\n0aLaTRTbek/DuIwMBotZtAi++UY/EL/9trWy9+zZw6hRo+jQoQOvWdl67VHcuKHnDmrW9DynO8lS\n/RqAt99+m5o1a3Ls2DEKFy6Mj49P0g4YD8wIwWCwkCtX4MMPoXp1+Ppra2VH9zgoVKgQ/+cuk7Yi\nOhPvyhVdxMkdK5nGQbQNGzhQu4mKFtXGIKm2beHChUlXLoEYg2AwWEj//joJzccH0qWzVnavXr04\nceIEvr6+ZM+e3VrhD2POHO03++ILa9OzXUwKqX5tXEYGg1Vs2aL7w/frZ/28wZIlS5g5cyYDBgyg\nbt261gp/GMeOwUcf6eSLAQPs1saAMQgGgyXcuaNdRcWK6QJ2VhIYGEjnzp2pXr06Q4YMsVb4wwgP\nh9atdZPoefN0RVOD7RiXkcFgAePHw5EjsHLlfycgk5PownURERHMnz8fb29v64Q/iv79Yf9+Xda1\nUCG7tYkVEXGP7O2HoLtluhYzQjAYkpkzZ7SL/I03dEUGK5k4cSK//fYbEyZMoGTJktYKfxgrV+qi\ndT17grtEOj1AhgwZuHbtWrLcdF2BiHDt2jUyuLhGunLXD1ytWjXx8/OzWw2DIUmI6Huery/89RdY\nWUz08OHDVK1alYYNG/Lzzz+7x9PuuXNQqZI+Ebt2gTt0ZYuFiIgIAgMDuX37tt2qPJQMGTJQuHDh\n/4z6lFJ7RSRRM/TGZWQwJCM//wy//grjxllrDO7cuUPbtm3Jli0b33//vXsYg6goHYpz+7ZueuOm\nxgDA29ub4sWL262G5RiDYDAkE4GB0LWrbofZs6e1socOHcr+/ftZsWIFefPmtVb4wxg9Wje+mTUL\nypa1WxtDLJg5BIMhGQgPh+bNISwMfvoJvCx89Nq2bRtjxoyhU6dONG3a1DrBj+LECT2R0rIlvPee\n3doYHoIZIRgMyUCfPrp455Il8MQT1sm9desW7dq1o1ixYowfP946wY9CRFcwTZdO1+xwB/eVIVaM\nQTAYXMwvv8B330HfvvDWW9bK7tWrF2fPnmXLli1kdZciccuX67IUY8fqaqYGt8W4jAwGF3LxIrz/\nPlSuDF9+aa3spUuXMmvWLAYMGECtWrWsFf4wbtzQVfwqVYJevezWxhAHZoRgMLiI6N7IwcG6i5aV\ntYrOnz9Ply5dqFatGv/73/+sExwXAwfqtpjLl4O7JMUZHooxCAaDi5gyBdas0b3hn3zSOrnR2ci3\nb992r2zkXbu076x7d3j2Wbu1McQDYxAMBhfw1196zuCVV3S9NiuZNGkSGzZsYOrUqZQpU8Za4Q8j\nIkI3Fi5YEEaMsFsbQzwxBsFgSCJ37kDbtpAli/W9kQ8fPkz//v1p0qQJXaI7u7sD48fDoUM6M8/K\n/qCGJGEMgsGQRIYMAX9/7SbPn986ueHh4bRp04bs2bPj4+PjHtnIAKdOwbBhunhTs2Z2a2NIAMYg\nGAxJYMsWGDMGOnWC11+3VvagQYM4cOAAK1eudJ9sZBFd59vLS0+mGDwKl4SdKqVmKqUuK6X+fMj2\nekqpG0qp/c7FjcIgDIbEceMGvPsulCgBVnek3LRpE+PGjeODDz6gidUlVB/FggWwfr2OuXXTstaG\nh+OSaqdKqbpAMDBXRCrEsr0e0E9E4n3lmmqnBnenbVtdo23bNqhRwzq5165do1KlSmTOnBl/f38y\nZ85snfBHcf26TssuUQK2bzdNb2zC9mqnIrJFKVXMFccyGDyBn37SuQZDh1prDESErl27cvnyZXbt\n2uU+xgB0b9Dr12HjRmMMPBQrM5VrKqUOKKXWKKVi7SirlOqilPJTSvlduXLFQtUMhvhz/bpOvq1W\nTeddWcm8efNYunQpI0aMoEqVKtYKfxTLl+sQq/79dXlXg0fisgY5zhHCqoe4jLIBDhEJVko1BiaI\nSOlHHc+4jAzuyvvvw5w5sHevrshgFQEBATz11FM89dRT+Pr6ktZdnsLPn4ennoLixWHHDmtTtA3/\nISkuI0tGCCJyU0SCnf+vBryVUrmtkG0wuJIffoCZM/WDsJXGwOFw0LFjRyIjI5k9e7b7GAOHA9q3\n101vrK7XYfgv4eFJerslBkEplV85g6SVUs865V6zQrbB4Cr8/XXybb16urS/lUyZMoWNGzcybtw4\n9+mNDDBhAmzYoMOsTNMbe/njjyTXTHFVlNFCoB6QG7gEDAG8AURkqlKqO/AhEAmEAX1EZMejjmlc\nRgZ34to1qFpVd4HcuxesDPs/fvw4lSpV4vnnn2f16tXuk4B25oyOKmrYUGcku4teqQ0RmDRJ104p\nUAD1zz+2Rxm9Hcf2ScAkV8gyGKwmKgrefluXtt661VpjEBkZyXvvvUf69OmZMWOG+xgD0H6ztGlh\n8mRjDOwiKEhPai1bBq+9BrNnw2OPJfpwJlPZYIiDgQO1V8THB555xlrZY8eOZefOnSxYsIBC7pTo\ntWULLF6sS1QULmy3NqkTPz/dkjQgAL7+WrfpS6JhdlmUkasxLiODO7BkCbRoAR98oMtbW8mBAwd4\n5plneP311/npp5/cZ3QQFaVjbq9dg6NHIVMmuzVKXcR0EeXPr5Niata8u9n2xDSDISVy5IgOoKlR\nQ8+dWkl4eDjt2rUjV65cfPfdd+5jDEDnG+zfDwsXGmNgNTduaBfR0qXw6qs6/jkJLqIHMQbBYIiF\nGzd0sbosWfRvz+poyqFDh3Lw4EFWrlxJ7txuFKF986b2odWqBa1a2a1N6mLvXu0iOntW96fu0wfS\nuDZQ1BgEg+EBolthnj4NmzbpHi9Wsn37dr766ivef/999ypcB7rW9+XL8OuvZiLZKsLDtQEYPlxH\nNGzZAs89lyyijEEwGB5g8WIdtDFmDNSuba3soKAg3nnnHYoVK8b48eOtFR4XGzbAN9/o8tbVEuWi\nNiSUbdugc2c9V9OihZ7IcqGL6EGsrGVkMLg90XWKqlbVI3IrERG6dOnC+fPnWbBgAdncqdPYlSvw\n3ntQrpyOaDEkLzdv6l6sdepAWBisXg2LFiWrMQAzQjAY7qNfPx08s3697vFiJbNmzWLx4sWMGjWK\n6tWrWyv8UYjoicxr12DNGjORnNz8+qsOazt3Dnr10j2ps2SxRLQZIRgMTjZuvFew08o6RQAnTpyg\nZ8+evPDCC/Tv399a4XExdSqsXKl9aFafmNTElSvwzjvQpAlkz64LBX7zjWXGAEwegsEAQGioLtiZ\nNi0cOAAZM1onOzIykjp16nD06FEOHjxIkSJFrBMeF4cP6/mCevX0k6uLo1oM6BHY/Pnw8cf3org+\n/zzRoW0mD8FgSCJDhuje8L6+1hoDgC+//JJdu3axcOFC9zIGt2/rmh1Zs+qSCMYYuJ6zZ7V7aO1a\nnfAyYwaUj7VdjCWYb9iQ6tmwAcaP18Eczz9vrezdu3fzxRdf0KZNG1q3bm2t8Lj47DM4dEgbg3z5\n7NYmZREVBRMn6pv/1q36/23bbDUG4OYuoz17/MxDiSFZOXlS1ycqVAh27rTUXUtwcDBPP/004eHh\nHDx4kBw5clgnPC7WrIHGjXXI1cSJdmuTsvjnH2jTRhuAV17RczSPP+6yw7t9g5zEcPq0LtXhpvbK\nkAIIDtbZyKCrN1tpDAD69u3LyZMnmTt3rnsZg0uXdM2OChXgq6/s1iZlsXy5npg/cECXnVi92qXG\nIKm4rUHw8tIT7OZ6NCQHIvqed+SIDu+2uufMypUrmT59Op988gn16tWzVvijEIEOHfTk5sKFkCGD\n3RqlDG7fhu7d4c03oVQp2LcP2rVzv2xvEXHLpWrVqvL22yIgMmeOGAwuZexYfW2NG2e97AsXLkie\nPHmkUqVKcvv2besVeBQLF+oTM3Gi3ZqkHH7/XaRcOX1e+/QRCQ9PVnGAnyTyvmv7jf9hS9WqVSU8\nXKR+fRFvb5FNm1x6zgypmIMHRdKlE3njDRGHw1rZUVFR0rBhQ8mQIYP8+eef1gqPi7AwkccfF6lc\nWSQy0m5tPJ+AAJGWLfVttnhxkTVrLBGbFIPgti4j0GG4S5dC6dLwxhvw1192a2TwdO7c0SP1HDlg\n2jTrR+wTJ05k3bp1jB8/nvI2R5T8hwkTdBjkuHE6IcOQOO7c0Ul8TzwBv/yimwgdPqwnkN2dxFqS\n5F6qVq161+KdPi2SL582spcuucaKGlInAwfqB7YVK6yXvW/fPkmXLp00bdpUHFYPTeLi0iWRrFlF\nXnvNbk08m3XrRMqU0RdZs2Yip05ZrgIpdYQQTbFiOnP+4kVo2lTXejIYEsru3TBqlJ5MbtrUWtmh\noaG8/fbbPPbYY/j4+LhXwxuAoUP1D2vsWLs18UyuXdO9Cho2BIdDRw/9/DMUL263ZgnCIwwC6Fjx\nBQtgzx549119zg2G+BIaql1FhQvr6DWr6dOnD8eOHeOHH35wr4Y3oEOtpk3TGbNly9qtjeexbRtU\nrqwNwPDhOpmvUSO7tUoUHmMQQMeMjxun5xU++8xubQyexGefwd9/66Tb7Nmtlf3zzz8zbdo0Pvnk\nExo0aGCt8PjwySe6PMWQIXZr4llERcHIkbrOU/r0uhjdoEGeHaqbWF9Tci8x5xBi4nCIdOumXXRT\npybB0WZINWzcqK+Xnj2tlx0YGCi5cuUSHTWXvOGGiWL9en1yxo61WxPP4sIFkQYN9Llr3Vrkxg27\nNboLSZhDcOvSFQ+rdhoZCc2awbp1sGqVZ0zeG+zhxg1dxTRTJvD3t7aUf1RUFC+99BK7d+9m3759\nlClTxjrh8SEqCp5+Wqds//WXfso1xM369dpvfeuWLuvx/vtulWCWIktXPAovL/jpJ/1Db9FCZ4Eb\nDLHRqxecPw9z51rf1+Wrr7EvzI4AACAASURBVL5i06ZNfPvtt+5nDEA3fzh0SIdIGmMQNxERuix1\nw4aQJw/88Qd06uRWxiDJJHZokdzLw1xGMQkMFClUSC8BAQkaVRlSAcuX6xH9oEHWy965c6ekTZtW\nWrdu7X4hpiIiN2+K5M8v8txz1mfneSJnzojUrKkvqM6dRUJC7NbooZDSw04fRqFCumfHzZvaaF+7\nZrdGBnfh4kXo0kV7RAYPtlb2zZs3eeeddyhSpAhTp051vxBT0MlSFy/qKA131M9diIiA//s/7Y74\n80/48UeYPj3FthH1aIMAunDgihVw4oTuPBcSYrdGBrsJC9NzTCEh2lWUyMZTiaZHjx6cPXuW+fPn\nk93qkKb4sG2bbgDRtatuymKInT17oEoV6NMHateG/fuhVSu7tUpWPN4gALzwgjbce/bAW2/pzHFD\n6kScxTr/+EN3JaxQwVr5P/74I3PnzmXw4ME899xz1gqPD8HB8N57Otvz66/t1sY9CQvTjbVr1oSg\nIF2y+tdfoUQJuzVLfhLra0ruJT5zCA8yY8a9KLCoqAS/3ZACGDJEXwNjxlgv++zZs5I9e3apUaOG\nREREWK9AfPjwQxGlRLZssVsT92T7dpGyZfVF1KmTSFCQ3RoliGXLlqXcaqeJYfRo/ak++sjMlaU2\n5s/X332HDtZ/95GRkfL8889LlixZ5MSJE9YKjy/r1ukT1Lev3Zq4HyEhIr17a2NZtKjOz/Agrl27\nJq1atRLAGISYOBwi/frpTzZ0aKIOYfBAduwQSZ9epG7dZC83HyujRo0SQGbPnm298Pjw7786HK9c\nOV3m2nCPLVtESpXSN40PP9QRWB7E+vXrpWDBguLl5SUjRowwBuFBHA6R9u31p/v220QfxuAhnDkj\nkjevSMmSIleuWC/fz89PvLy8pEWLFu4ZYioi0q6dSNq0In/8Ybcm7kNwsEiPHnpUULy4bmTjQYSE\nhEiPHj0EkHLlysnevXtFRIxBiI2ICJGmTfV3vWBBkg5lcGNu3hSpUEEke3aRv/6yXn5wcLCUKVNG\nChcuLNeuXbNegfiwZo3+qf/vf3Zr4j78/rs2AqCNwq1bdmuUIPz8/OSJJ54QQHr16iWhoaF3txmD\n8BBCQ7ULwctLZO3aJB/O4GZERoq8+qp+8N2wwR4dunbtKkop+d1dny4jIkSefFK7RNyxlpLV3Lql\n3UKgh5SbN9utUYKIiIiQ4cOHi5eXlxQqVEg2xHLhG4PwCIKCRF5/XeTvv11yOIMb0bu3voKnTLFH\n/s8//yyA9O/f3x4F4sPUqfokLV1qtyb2s3u3NoxK6YvHjbONY+P48eNSs2ZNAaR169Zy/fr1WPcz\nBsGQ6pg2TV+9vXrZI//8+fOSO3duqVKlintWMRXRFTjz5hWpUyd1h9xFRIgMH66HkkWKiPj62q1R\ngnA4HDJ9+nTJnDmz5MiRQxbE4QM3BsGQqti1S7sBGzWypxd8ZGSkNGjQQDJmzCh/2TFxEV8++UT/\nxPfssVsT+9iwQaR8ebmboPTvv3ZrlCAuXbokzZo1E0AaNGggAfEo2mYMggsICzPReJ7Av/+KFCsm\n8vjj9v22hw4dKoD4+PjYo0B8+PNPbTXff99uTezhxAnd0xhESpTQlQ49bJS0YsUKyZs3r6RLl07G\njRsnUfHMtjUGIQmEhGj36gsv6P7iCxdaItaQCBwOkebN9X1u5057dPD19ZU0adJI27Zt3TfE1OHQ\n0RS5ctkTh2snN2+KfPaZSLp0Ipkzi4wa5XFPejdu3JCOHTsKIJUrV5ZDhw4l6P3GICSSK1dExo/X\n4alLl4r4++vR5dGjyS7akAjGjhXbylKIiFy9elUKFSokpUuXllvuHKY4d64+UdOn262JdURFicye\nrUt6g8h774mcO2e3VgnG19dXHn/8cUmTJo0MHDgwUfNTxiAkgtu3Rb75RqRLl/tdrHXqmNwdd+SH\nH/TV2qKFPXWqHA6HvPbaa+Lt7X03Acgt+fdfPZFcvXrqKei1b5/Is8/qC6R6dR1N5GGEhYVJ3759\nRSklpUqVkh07diT6WMYgJIJ160QaNtQlD0RE7twR+fFHXc8qtfyOPIU1a7SbqH59bcjtYOLEiQLI\nN998Y48C8eWjj0TSpNHD3ZROVJTIuHHaPZQ/v35q8MAfr7+/v5QvX14A+fDDDyU4ODhJxzMGIYFE\nRoq0aiUya5Z+HR4u8ttvOjR50iTtgvXA6ypFsnu3SKZMIpUr29fH3N/fX9KlSydNmjRx33kDERE/\nPx1j36OH3ZokP+fPi7z0kr6FvfGGyNWrdmuUYCIiImTkyJHi7e0tBQoUkDVr1rjkuEkxCF5Wldl2\nJ5SCDBl0MySAhQt1a9ls2aBjR71dKd2D/NYt3aj98cft1Tk1cuwYvPoq5MsHa9bo78dqgoODad26\nNblz52bWrFnu2f0M9MX64YeQNy8MH263NsnLL7/oH2poKEybBp07e1zXtxMnTtCuXTt27txJy5Yt\n+e6773jsscfsVss1IwRgJnAZ+PMh2xUwETgBHASqxHXM5HYZ/fmnyFNPiTz/vEjbtrqXQsx5wogI\n/eDRoYNI1ap6ns4QN/Pm6ZBQpfTfefMSd5xz5/T78+QROX7chQomkPbt27t3aYpoojOS589P2nFc\n9QUmByEhIh98oD/n00/bU7wqiTgcDvHx8Yl3klliwG6XEVAXqPIIg9AYWOM0DDWA3XEd04ooo6tX\nRc6e1TWPoonua9Kvn66Yev26yKFDuoDa9u3JrpJHM2+edu/ovmV6yZQp4feUf//VxjpLFu0FsYv5\n8+cLIIMHD7ZPifhw6ZJIzpw6djopLi1XfYHJwb59unQ36B+nXZNJSeD69evSokULAeSFF16IV5JZ\nYrDdIGgdKPYIgzANeDvG62NAgUcdz8rEtP/7v/82kBo6VGTFinuvR4wQOXbMMpU8kscfv/9eEr08\n/nj8jxEWpkPovb3t7VFy4sQJyZo1q9SuXdt9u59F0769PmFHjiTtOK74Al1NRITI11/rieMCBeyr\nYphENm/eLEWKFBEvLy8ZPXq0RCZjir0nGIRVQO0Yr38DqsWyXxfAD/ArWrRoMp2u/3L9unYhidwb\nLfTurV1JEREiS5boh5Ok/t5SOkrFfj9RKn7vdzh0dQFXeD6SQlhYmDz99NOSM2dOOXv2rH2KxIdt\n2/QJ+/TTpB8rqV+gq/H1FalYUevQrJlHJtmFh4fL559/LkopKVmypOyxoIxIijEIMRc7ahkdPXr/\njWjCBJEmTfTv4eef9brTp/U+69ZZrp7bk9QHzP/9T+8/alRyahk3nTp1EkBWrVplryJxEREhUqmS\nLtiWxFBFEXGfEcLFiyLvvKNlFy0qsnixx5WdEBE5evSoVK1aVQB5//335aZFndg8wSC4tcsomuvX\nden40aNFLlzQ8e8FCoj88ovI5cvatVS7tg71LlFCZzkb7pEUF3R04pkd/ZBjMmvWLAFkwIAB9ikR\nX779Vp+0xYtdczy75xAcDh3dkTOndhH9738eV6JaRE8cT5kyRTJmzCi5cuWSpRaXHvcEg/DqA5PK\ne+I6nl3VTs+d06UsOnbULqNff9WTnMOG6eKR0SODY8f0zcvDyqQkO4kJUtm6Vf/+69Wzt4fLgQMH\nJEOGDFK/fv1k9fG6hIsXdZu4F190rQW1K8ro6FEd8gd6EskDI4hEdHXSJk2aCCAvv/yynLOhfIbt\nBgFYCFwAIoBA4H3gA+AD53YFTAZOAoficheJjQZB5N5NPnqEN2uWSN++2l0bTd++OrnNkDSOHxd5\n7DGRMmVE7OxAGRQUJKVKlZICBQrIxYsX7VMkvnTvrtO3PfTGeR9//KGfCHLk0CMED80K/fXXXyVv\n3rySPn16mTBhQryrk7oa2w1Ccizu0g/h1i3tpv3xx3vrli7VyaAHDujXFy9qF5MhYRw7JlK4sDYI\nduYaOBwOefPNNyVt2rSydetW+xSJLwEB+gbaqZPdmsTNnTtx7xMZKTJ4sMf+iEJCQuTDDz8UQCpW\nrJjg6qSuJikGIVVmKieEs2ehWDFo1Uq/XrIEdu+GUqV0Bu2QIbBtG9y+DfXqwciRdmrrORw5Ag0a\n6ATb337T59Muxo0bx7Jlyxg3bhy1a9e2T5H4MmoUOBwwcKDdmjycwEAYNgwyZYL334eyZSF9ej0z\n8WBWcdq08MUX9uiZRPz9/WnTpg1Hjx6lb9++jBgxggwZMtitVqIxBiEOSpSAEyegf3+4fFlfzzVq\nwLPPwowZsHkzzJ0LuXND/frQsCHUrWu31u7NgQPw4ovg7Q2+vvDkk/bp4uvry6effkrz5s3p3bu3\nfYrEl4AAfeF16KCfVNyR4GBo2RJeeQXSpYNvv9UGoV8/jysx8TCioqIYO3YsgwcPJl++fGzcuJEG\nDRrYrVbSSezQIrkXd3EZiYgEBenIo6FDdVJoRIROVGvZUkcfiWj/94sv3nMjGWJnzx4dRFK4sMjf\nf9urS2BgoOTNm1eeeOIJy0ICk8wHH+gktDNn7Nbk4fz5p8hbb917vXWrziP47Tf92gNDSGNy5swZ\nqVu3rgDSokULuWbn5FcsYOYQrGXCBF1989Il/TosTEcfde/ukbkzlrFtm0i2bCLFi4ucOmWvLuHh\n4VKzZk3JkiWLHPGUjMMzZ7Qx6NrVbk3uJyhIZPJkkbVr9evISJ3JGZ3+f/OmyJQpIm3aeLwxmDdv\nnmTLlk2yZs0qc+bMccvqt0kxCGlsHqB4JOfOwfjxurDkrVuwbx/4+OhRce7cdmvnnmzapN1p+fLB\nli1QvLi9+vTt25edO3cyc+ZMypUrZ68y8eXLL/XfAQPs1SMmx45B1aq6XPCgQXre4PJl7R6aPFnv\nkzUr1K6t/x46ZK++iSQoKIh33nmHtm3bUqFCBQ4cOEC7du3ct/ptIjEGIRFcvqwnlwMDYfFifd1X\nrgzdu9utmXuybBk0aqRLiG/eDIUL26vPrFmzmDRpEr1796ZFixb2KhNfjh7VTx2dO0PRonZrc48/\n/tBzBVOmwA8/6Cek+fP1DyJDBpg0Se9XpgycOqUnmT2M7du3U7lyZRYtWsTw4cPZvHkzxe1+okkm\njEFIBNOm6VFChw5w8iQ0bgyff663idirm7vx/ffQogU8/bQeGRQoYK8+u3fv5oMPPqBBgwZ89dVX\n9iqTEPr31zfTIUPs1gTu3Ln3f9GicPw43LwJTzwBL7wAFy/qye+PPtJD6c2bYd06Pdkc3YTEA4iM\njOSLL76gbt26pEmThu3btzNo0CC8vNw3Fuf69SQeILG+puRe3HkOQUS7QqMnlGOuM2gcDpHhw/Us\nVaNGrim1k1TOnz8vBQsWlOLFi8tVT+qw9dtv4hZFnubPF3nmGZGVK++t8/cX6dNHZNky/To8XH/x\n0XVdfHx0ueqnn/aoSqXHjh2T6tWrCyBt2rSRG3a164snDofIggU6tw8zqex+OBw6Gz81EhWlJ9hB\nl/+IT25ScnP79m2pUaOGZM6cWQ4ePGi3OvEnMlJHMDz+uL11UvbvF3n1Vd22csiQe7Xgg4NFxo4V\nGTDg3roVK7Th8EAcDodMnjxZMmbMKDlz5pSFCxfarVKcXL2qIx5BpEYNYxDckhkzdEDIxImpa+QQ\nFCTy5pv6yurTxz2qEDgcDunYsaMAsmTJErvVSRizZumTmQydtRLE1as6k/jcOZF33xWZPftebRd/\nf51p3LGjfr1xo/7fwwrTBQYGyssvvyyANGzYUAIDA+1WKU5WrRLJn1/fa778UofEG4Pghly/rovk\ngUiLFvY1iLeSvXt1Fdi0abXHwF0M4bfffiuADBo0yG5VEkZwsC63W7269SczuilQTIse/f/8+SKd\nO4vs3Hlv26VLIs2b63rxBQvqUsEexIIFCyRHjhySKVMm+e6779wynDQmN2/qrwB0d8H9++9tMwbB\nTXE4RMaM0TfIMmVEPMlTkRAcDh2Gni6dTjiLWQTQbjZt2iRp06aV1157zbZiY4lm6FD9E7Wyd+uV\nK7q++4ABOnMw+sb44A2ye3eRkSPv9weGhuqiVG5+M43JtWvXpFWrVgJIjRo15G+7syXjwebNOpcn\nTRrdF+nBbqLGILg5vr56WJcxo8jcuXZr41qCgvQICEQaN3avxLwzZ85I7ty5pVy5cm4/Kfgfzp3T\nvQhatLBO5q1bugb5Z5/peYLWrUVmzrx/n2ijeuiQyIcf6hT+mjU9csJszZo1UqBAAfHy8pIRI0a4\nfavUsDA9P6+USMmSD3/wMgbBA7hwQf/WQKRLl5TRR8HfX1+YadPq+4I7PYAHBwdL5cqVJXv27HLM\nE5thd+igh1wnT1on88gR3ZMgmsWLdVZ0dHTQg19w6dK6W9vEiZap6Apu3bolH3zwgQBSvnx58ff3\nt1ulONm7V6R8eX3/+OADbbsfhjEIHkJEhH74Ah044udnt0aJw+EQ+e47fb8qVEiXqnEnIiMj5fXX\nX5c0adLI6tWr7VYn4ezbpx8D+/WzXnbt2vdKUFy8KPLNNzo6IOYdKLqzWcuWj74zuSG7d++WUqVK\niVJK+vbtK2Fu/mQWFqYHa15e8Z+aMQbBw1i5UruQ0qTR1t7Kh8CkcvHiPRfRK6+4l4somj59+ggg\nEyZMsFuVxNGggW4S8e+/1sqNitJtOXv0uNe6btMm/friRT1xvGOHNgjuEEucACIjI2X06NHi5eUl\nRYsWFV9fX7tVipNdu/TcI+gyUNevx+99xiB4IEFB+nfm7a0Nw5tvap+gu87HRUToon7ZsmmdR41y\nLxdRNOPHjxdAunfvbrcqiePvv/XPcswYe+QfPqwnladNu7eucmVdWG/BgnsVSz2I06dPS506de5W\nJ70e3zurTTgc2gvn7a3TT6Lb9sYXYxA8mHPndEBHzpz626heXeSnn+5F/dnNyZM68bRUKa3fyy/f\nyz9yNxYuXCiAvPXWW+7fE/lhjBihT3RAgD3yHQ6R1at1YtmGDfoppW5dkdOn7dEnCTgcDpkzZ45k\nzZrVrauTxuTmTd2aF3QEb2JslzEIKYDgYB26GX3jLVpUZNw4PZKwmqtX9RzBc89pXUDPNS5b5r4j\nmI0bN4q3t7fUrVvX7f3Cj6RiRZFatayR9agvc8ECkfffF6lQQWT5cmv0cSFXr16V5s2bCyB16tSR\n0x5g0P78U6RsWe0xSMoI3BiEFERUlM78f/55/e1kzSrSs6fOSEzOkW5oqMiiRTqZzttbyy5fXkcP\nnT2bfHJdwb59+yRr1qxSoUIF+ddqv7sr+esvfeKTe+7j6FGR997TkUyPwsPmCaJZt26dFChQQLy9\nvWXUqFEeMVqcN09HGefLp6dtkoIxCCkUPz89mRR9gwaRJ5/UGYqzZyc9BygqSuT333WVgWzZ9PEL\nFtTBLfv3u+9oICanTp2S/PnzS5EiRSTALjeLqxg2TEcXnTuXPMf/80+dW5AmjU6K6dfPM77keBIa\nGio9e/YUQMqVK+cR4aS3b+vAEhCpU0fk/PmkHzMpBkHp97sf1apVEz8/P7vVcAtCQnTZ+e3b9bJz\nJwQF6W1588Jzz0GtWnqpUkU3rr90SfdtiF5ivo7+/9w5+Pdf3bfkrbegbVuoV0/3PPcErly5Qq1a\ntbh69Srbtm3jSTubM7uCKlV0iett21x73KNHdfOapUshSxZdlrpPH33xpBD27dtH27ZtOXLkCD17\n9mT06NFkzJjRbrUeyZkzujS8nx988onuf+SKytpKqb0iUi1Rb06sJUnu5cknn/TMhCILiIrSiaJT\np4q0a6eTw6JHEErd+//BJWtWPUfx3HMir7+uc45+/NHjapCJiE48e/bZZyVDhgyyzZ1qZSSWs2f1\nl/TVV6475qVLIt266czBbNl0ATpPKvsdD6KiomTMmDHi7e0tBQsWlPXr19utUrxYsUIHkmTP7vop\nGlKiyyhDhgxSpEgROePOzcTdiAsXRJYuFRk0SE9I+fjofIc9e3TEYGio3Rq6jjt37kjjxo0lTZo0\nstwDJzxj5dtv9c/RFQ9BoaH6IsiaVRuDFNrsOzAwUBo0aHA3sswTelycO3cvj6dyZZETJ1wvI0Ua\nhHLlykn27NmlVKlScuHCBdeeMYPH4nA4pEOHDgLI1KlT7VbHdbz0kg4xSQpRUXp2smhR/dNu2tQj\nawzFh+XLl0uuXLkkU6ZMMmPGDLcPJ42MFJk0SQ/U0qfXodwPFqVzFSnSIFStWlV27NghmTNnlgoV\nKniE9TckPwMHDhRAhgwZYrcqriMoSEcO9O+f+GNs3ixSrZr+SVepkvRQFTclJCREunbtKoBUqVJF\njnqAwdu3T6d1gMiLL+pgkOQkxRoEER1fnj59eqlWrZpnhxQakszYsWMFkM6dO7v9E2GCWLxY/xS3\nbEn4e/fsEXntNf3+woV1OV13TCF3Afv27ZMnnnhCAOnfv7+ER5fXcFMiInTSadq0Innz6jYSVly2\nKdogiIisXLlSvL295dlnnzVGIRXicDjujgxatmzp9mWKE8z27fqnOHDgo/eLjBQ5dUpnEo8fr91M\noGcnR4zwzOiAeBAVFSXjx4+XdOnSSYECBWSDB/RmjlnduEOH5M0hepAUbxBERH755Rfx9vaWZ555\nxhiFVERUVJR069bt7sjAE5KMEsV77+mSlgcP6gqifn76kXLwYD0L+dRTIhkyyH1hY4UK6ZpH0a0s\nUyAXLlyQhg0bCiDNmjWTKx4wOb5li250lzGjyJw51stPFQZBxBiF1MadO3fk7bffvusiSFFuoge5\nelUkd+7/3vTTpNGxwk2a6ESy77/X9cY94MaYVFatWiV58uSRDBkyyJQpU9z++3c4RMaO1S6i0qXt\n65CYagyCiHYflSpVyiNqkxgST0hIiDRu3FgAGT16tN3qWMO6dSLt2+tu6UuX6sqjyRWK4saEhoZK\n9+7dBZCKFSvK4cOH7VYpTs6fF2nWTN9R33rL3h7qqcogiOgnR0PKJSgoSOrUqSNKKZkWswyzIcWz\nf/9+efLJJwWQXr16uX2hwqgoXQgyOpx0/Hj7q4EkxSC4IFHaery9ve1WwZBMXL58mYYNG3L48GEW\nLlxIq1at7FbJYAEOh4MJEybw2WefkStXLtauXUvDhg3tVuuRHDoEXbrArl1Qvz5MnQqlS9utVdJI\nY7cCyUV4eLjdKhgSyD///EOdOnU4duwYv/zyizEGqYTz58/TsGFD+vTpQ6NGjTh48KBbG4PQUPjs\nM1166sQJmDsXNm70fGMAeOYI4VGICN26dSM0NJQbN26wZMkSvFxRMcqQrBw9epSXXnqJW7dusWHD\nBmrVqmW3SgYLuHDhAhUrViQsLIzp06fTqVMnlFJ2q/VIPv8cJk6EDh3gq68gd267NXIdKWqEEBgY\nyIABAzh79izffPMNBQsWZODAgURGRtqtmuER7N27lzp16nDnzh02b95sjEEKQ7u1Y6dAgQL07t0b\nf39/Onfu7PbGALRB2LQJZs5MWcYAUpBBCA0NZdGiRQQFBTF79mxy5sxJrVq1CAgIMCMEN+b333/n\nhRdeIHPmzGzbto1KlSrZrZLBRVy4cAHg7k3+YYZh4MCBlC1b1jK9Hsbp07ryeFTUo/fLn1+XiU+J\npJg75b59+9i7dy89evQgb968XLlyhX/++YcGDRoAEBUVRVpPKfSfCoiKimLs2LEMGjSIsmXLsm7d\nOgoXLmy3WgYX8dFHH/HXX39RqVIlmjZtygsvvODWT/9Tp8L48VCsGDzxBNSuDS1b2q2V9aSYEcK4\nceMoXrw4NWrUQETw8/MjICCAggULAtw1BocOHeLw4cN2qprqCQgI4MUXX+Tzzz/nzTffZOfOncYY\npCDmzp3LyZMnWbJkCfny5WPevHnMmjULeLT7yC4cDt14askSWLtWN5patAhWrdLb3VDlZCNFGISI\niAhy5sxJ586dAdi4cSO+vr4UKVKERo0a3d3v1KlT/PDDD9SsWZNRo0bZpW6qZvHixVSqVIk//viD\nWbNm8dNPP5EtWza71TK4kNOnT9OoUSNy5cpF165deeONN1i8eDH79+9HKeV2RuHmTfj7b7hzB9Kk\ngVdegTfegOnTISAA3Hhg43JShEHw9vamaNGivPTSSwwYMIAxY8aQL1++uwYC4ObNm2zcuJF06dKx\nbNky/Pz86Ny5swlPtYhbt27RsWNHWrZsSenSpdm/fz/t27d3azeCIX6EhYWxadMm/v77bwCee+45\nZsyYQVBQ0N25vBdffJE5c+bgcDjc4jv/6ScdLnr8OOTIAe+8A4MH623Zs+u8gsqVYd06e/W0mhRh\nEACGDBnC6NGjyZMnD5MnT6ZPnz7kzp0bh8MB6NGBj48P4eHhvPjiiyxdupQWLVpw48YNmzVP+ezZ\ns4enn36aOXPmMGjQILZt20apUqXsVsvgAg4cOECJEiXw8fGhcePGLFu2jEqVKtGsWTOGDBkCQM6c\nOalQoQKhoaGEhYXZrDH07w/Dh8Phw9CmjTYOTZpAyZIwdKjep0ABSJ9e9x9PTaQYgwDw5ptv0rt3\nb5RSTJo0CYfDQZo0+iNWrlyZIUOGsGnTJmbMmAHAyy+/TN68eYmMjLwbEWFwHVFRUYwcOZLnnnuO\niIgIfH19GT58uMk0TyGICBs3buTzzz9n3rx5jBw5kq1bt7Js2TLeeOMNzp8/z4QJEwB49tlnOXny\nJBEREbbqfPMmnDwJO3bAmDHQsyfs2QNbtui8gs2b9QQzaFfRlSupaw4hxUQZxaRMmTKcP3+eiIgI\nbt68SZ48eQBo3LgxoEMdQV/Qp0+fpm/fvjgcDs6dO8eKFSsoVKiQbbqnFM6ePcu7777L1q1bad26\nNVOmTCFHjhx2q2VwAcHBwXh7e5M+fXrCwsLYt28fAK1atSIyMpJ9+/ZRokQJBg4cyJtvvklISAjL\nli2jWrVqZMmSxVbds2WDyEiYNQt69YK2bXXm8dGjUK3aPSOxbh0cOwYrV5o5hBRBvXr18PLy4ptv\nvrlvAtnhcLBz505EJxfMWwAAIABJREFUhLVr1zJlyhSKFCnCihUreOutt5gyZQpRcQUiGx6Kw+Fg\n9uzZVKpUif379zN37lwWLFhgjEEKYd68eXTq1Il//vkHgIYNG5IrVy62bNkCQLNmzciaNSuHDx+m\ncuXKLFq0iCeffJKuXbsydepUW3KCPv4YJk/WriGA1q31ZPHBg/r1a6/BjRvaKDz7rI4uGjsWjhzR\nbqTUhEsMglLqFaXUMaXUCaXUZ7Fsb6+UuqKU2u9cOrlCblykTZuWjz/+mK1bt/L666/j4+NDt27d\n6NatGzdu3MDX15e8efMybNgwALJmzcqRI0dMvkIi2bp1K9WrV6dDhw6UL1+e/fv38+6777rFJKIh\n6XTo0IEZM2bw6aefUtpZuKdQoULkyZOHTZs2cebMGbJkyULt2rWZOXMmUVFRVKtWjddff/2+AA+r\nuHoVXngBwsOhfHn99D9lCrz0Enh76xv/xYt6vqBKFfjlF/2+3LkhtU5xJdkgKKXSApOBRsCTwNtK\nqSdj2fUnEansXGYkVW58yZMnD6tXr6ZOnTrcvn2bL7/8kjZt2rBhwwYiIyNp0qQJOXPmJCAggOPH\nj9O2bVsAjhw5gq+vLydPnrRKVY/l1KlTNG/enLp163LhwgXmzp3L1q1bKVGihN2qGVzEli1bOHz4\nMD169ODpp5/m/PnzXLt2jYIFC9KqVSuCg4MZP348ANmzZ6d8+fKEhITYqnNICJQooY1AvXp6Irl7\ndz06aN4crl/Xo4egID13UK5c6poviA1XjN+eBU6IyCkApdSPQDPgiAuO7TL69u173+u5c+dSr149\nypUrR1RUFCtXriRjxoyUKFGC9evX06tXL+rXr8+yZcuYNm0aTZs2tUlz9+XGjRuMHDmSCRMm4OXl\nxbBhw+jXrx+ZMmWyWzWDi6lUqRJdunRh165d+Pv74+vrS6lSpQgICODXX3/l008/pVevXjRu3JiD\nBw/yzTff2J5fcuGCngc4dw4KFdJP/jVrwgcfwO7d8OST0Lu3LmGdNi0MGGCruu5BYhspRC9Ac2BG\njNfvApMe2Kc9cAE4CCwBisR13Ec1yEkqISEh8u6778rRo0dFRGTq1KnSp08f+eGHH2THjh1Sr149\nWbRokYiILFq0SDp27JhsungioaGhMnbsWHnsscdEKSXt27eXwMBAu9UyuJDY2lX+/fff8tFHH0nV\nqlXlyJEjEhwcLO3bt5emTZve3efEiRMSFBRkpap3iVY5purduulOZh9+KFKlisiRIyItW4rMnHlv\nn5TWkpokNMixalJ5JVBMRCoCG4A5se2klOqilPJTSvlduXIl2ZTJlCkTNWrUoE6dOnTq1ImpU6fy\n0ksv8fzzzzNz5kwaNWpEixYtiIiI4MKFCxQtWhRwz7R7K7lz5w6TJ0+mZMmSfPLJJ1StWvVuxrGJ\nzEo5LF++nGHDhrFmzZr7KgWXLl2aHj168Pvvv1OuXDkyZ87M559/Tq5cue4meJYsWZLs2bNbrvO8\neboUtb+/jiKKZvJkXZ20Vi2YPVu7hUqX1iOGaLJmtVxdt8UVBuEcUCTG68LOdXcRkWsiEp0SPAOo\nGtuBRGS6iFQTkWrRoaLJRbdu3Vi7di3Nmzdn1apVvPLKK/z4448EBATQv39/AK5cucLFixfJlSsX\nQKqdHP3333+ZOnUqZcqUoXv37pQsWZLNmzezbt06qlaN9as0eCjff/89vXv3Jm/evIwaNYrRo0dz\n7Nixu9vLli17nyto9OjRZM6cmfTp09uhLqAzjMeMgYgIffMfP/7+hLLq1XUC2lNP6czk337T9YsM\nsZDYoUX0gp6HOAUUB9IBB4DyD+xTIMb/bwC74jpucrqMYiMkJEQaNWoku3fvFhGRq1evio+PjzRt\n2lQuXbokIrEPo1MqYWFhsmTJEnnjjTckXbp0Aki1atVk7dq1qeo8pCYcDof069dP1q9fLyIiu3fv\nli+++EI++eQTuX379t39oqKiZMeOHVK7dm3p3LmzREVF2aWy3Lkj0rGjyNmz+vWmTSKffSYybNj9\n+4WHi6xfL1K6tMjUqZaraSnY6TISkUigO7AO+AtYJCKHlVJfKKWiZ2J7KqUOK6UOAD3RcwpuRaZM\nmShYsCD+/v7cvHmTBQsWsHbtWnr27EnevHndpgZLcuJwONi0aROdOnUif/78NG/enB07dtCtWzf+\n+OMP9uzZQ8OGDVP8eUitKKXIkCHD3cqkzz77LPXr1+fOnTv8FB3ED6RJk4bw8HA+/fRTpk+ffrca\ngB14e+toou++06/r1dN1iC5fhp9/vn+/8uV1Y5uuXW1R1TNIrCVJ7sXqEYKIiJ+fn1StWlXq1asn\nn3/+uWzYsOGh+3bo0EF8fHwkNDTUQg1dj8PhkP3798snn3wihQsXFkCyZMki7dq1k3Xr1klERITd\nKhos5Pjx49K+fXtZunSpiOiR87fffisjR46UqKgomThxouzbt89mLe9n926RNm1EfH316+vXRUaN\nEpk0SSQkRGTECBFn/EiqgCSMEGy/8T9sscMgRPPXX389cvv169elQoUKAkiuXLmkf//+cvr0aWuU\ncwGhoaGyefNmGT58+N3P4eXlJU2aNJGFCxdKSEiI3SoabCIyMlJ8fHykQ4cOcujQIRER2bZtm7z8\n8ssiIrJp0yYbtYudsDCRCRNEOne+5zpavlxHE4mI+Pvbp5sdJMUgKP1+96NatWri5+dnqw4i8lD3\niIiwefNmJk2axM8//4zD4eCFF16gVq1a1KhRg+rVq/PYY49ZrHHsXLp0ie3bt99d/P397xYZq1Wr\nFm3atKFFixbkTmkNYg2JIigoiO+//55ly5Yxf/58Jk+ezPXr15k+fbrbFia8eFEXpfP1hTlz4H//\ng5w59QSzjR4tW1BK7RWRaol6rzEISScgIIBp06axevVqDh48eLcWUunSpalevTo1atSgRo0aVKxY\nMdl/UA6Hg6NHj95nAE6cOAFA+vTpeeaZ/2/vvMOkKrI+/NYMcYagiAqIgn6ACgbEWSUYkGAAQclZ\noiiKmFEEJMgSFhcWdxFFhAUJShSQJFGQsDAgOWclKDkzMzD1/XF6FHGACd1dt3vO+zz9dLo99btz\nu+/vVtWpc/5GuXLlKFeuHGXLlvWMaSneo0uXLhw4cICDBw8yZswYoqOjXUu6Ju3by8rj06dh9GjX\natyghuAhzpw5Q2xsLMuWLfv9dvDgQQCyZctGTEwMpUuX5v777yd37tzkzJmTHDlykDNnzt9v0dHR\nyU7UxcXFcfDgQfbv38+BAwfYv3//77ek53v37uXkyZOApO1IOvmXK1eOUqVKOQ0PVEKPhIQET/UK\n9uyB2267egbSCxfAQQ49z6CG4GGstezdu/dPBrFq1Sri4+Ov+rno6OjfDSJLliz8+uuvHD58+C/b\nZcqUifz585M/f34KFCjALbfcwoMPPki5cuUoWrSoRgQpyWKt5fPPP+d///sfQ4cO9fz3JD4eeveG\nHj3kyr92bdeKvEt6DCED+2hwMMZQqFAhChUqRL169QC50t+1axenT5/m1KlTnDp16k+PL38eHx/P\no48+SoECBX6/JRlA3rx5nYb9KaHHmTNnaN26NaNHj+bpp5/m/PnzZM+e3bWsK3LgANSqBUuXQoMG\n8PjjrhWFL2oIDsiaNSt33XWXaxlKBmTz5s3UqlWLzZs306NHDzp06ODpC4qlS8UMTp6UegZ167pW\nFN6oIShKBmHcuHG0aNGC7Nmz8/3331OxYkXXkq7KF1/Aq6/CrbfC99/DPfe4VhT+ePfSQFEUv5CQ\nkMBbb71F3bp1uffee1m1apWnzeDkSalv3Lq1FLhZsULNIFioIShKGHPgwAEqVKhA//79adeuHQsW\nLKBgwYKuZV2RefMkCd2IEdCxI0yfDr7ckkoQ0CEjRQlTFi9eTO3atX/PzdWgQQPXkq7I2bPw/vvw\n739DsWKweDGULu1aVcZDewhKeGKtrE7KgFhr+eSTTyhfvjw5cuRg2bJlnjaDpUuhZEkxg3bt4Kef\n1AxcoT0EJbSJj4cdO2DTJti8+Y/7zZvFEF55BT7+GDwcVulPLg0prV69OiNGjHBSsCYlxMVBly7Q\nt69MHM+bJ3MGijvUEJTQwFrYulWK4W7a9MeJf/t28KUKAaBgQSmL1bw5nD8veZF/+EFWM913nzv9\nQWD79u3UrFmT9evXez6k9Kef4IUXYP16aNUK/vlPcFyCWUENQfEqiYmwYQMsXCgn9IUL/yiDlTmz\n1EEsUUKWrN51l9zuvPOv9RBr14amTeGhh6TG4muvXT3vQYgydepUmjRpQmRkJDNmzOCpp55yLSlZ\nEhJkxXH37nDjjTBtGlSp4lqVkoQaguINEhNhzRqpYLJwISxaBEePynu33gqVK8Njj0lx3KJFxRRS\nwpNPwtq10KIFvP46zJolxXUDXKI1WFy8eJGuXbvSo0cPSpUqxYQJEyhcuLBrWcmyaZP0CmJjZcXx\nf/6jEUSeI615swN9c1kPQUkHx4+nfNuzZ60dM0YS199wg5TnAGuLFJG6iP/9r7W7dlnrj5KdiYnW\nDhwof/+tt9L/9zzAyZMnbfXq1S1gmzdv7uliTVOmWJsjhxzmsWNdqwlvSEc9BO0hKP5h1Cjo2RNu\nvllqFNauDZGRcoq/fIhmzx746CMYN05WIeXLB1WrSi/giSfgllv8r88YqFNHlr56OA4/pezdu5dq\n1aqxYcMGPvnkE9q2bevJBHXWyvxA+/ZQqhRMnhyYw6v4BzUEJW3s2QP580OWLFLAdv58qVCSJQsM\nGCCB5c2bJ//ZzJlh/HioWVPGEB5/PDhVTDZulPvixQPfVgBZsWIF1apV4/z588yYMYPKlSu7lpQs\n8fHQpg0MHSrXB8OHQ1SUa1XK1fBmCILiTayVJaQ33iihIR07yuvHj8u4/6OPwt/+Bs2awZQpEvaZ\n3FVrgQJiIsOGSY8gWJEwYWAIEydO5PHHHycqKoqlS5d61gwOH5YO39Ch0LmzJKZTM/A+agjKlTl5\nUm6XPh8+HFauhNmzZeJ3wgS46Sa5bd0qJ/fixSXuf+FC+VxyNTeyZAnOPlzKxo0ShRSCQ0bWWj7+\n+GNq167N/fffz7Jly7j77rtdy0qWjRvh4YclQnjUKIko8mj0q3IZepiUv7JtGzz9NNxwg1zpJxER\nIXGDSXH/L78smce2bJHon9mz5fWcOSXmf/Xq4Gu/Ghs3ill5cKz9aiQkJPDyyy/z7rvvUqdOHebN\nm8dNN93kWlayzJwJZcrAmTMSLdywoWtFSmpQQ1D+Snw8tGwpYZoHD8KRI/L69u0yJLRjhzx/8EEx\nh59/lrPAd9/J67lzw6FDUusQvHMC3rAh5IaLTpw4QdWqVRk8eDAffPABY8aM8WQxG2sl9UTVqnD7\n7bB8ufQSlNBCDSGjsnQpNGkCnTrBzp1/fq9wYYnIqVhRrv7375fX8+WDrFn/MIQiRaR47fnz8NRT\nYhjz58OpU7IEtVgx2c4LZVqPHJGFbSFkCHv27KFcuXLMnz+foUOH8ve//92TK48TEiRDSLt2UK0a\n/PjjH9cCSmjhvW+XEnh++w3695dhHWuhQwfYtUvesxaio+XxQw/JyX7PHnmeP79c/m3ZIifX7Nkl\nfURcnDzu21cmnYsVk2xlSYbghR7Cpk1yX6KEWx0pZPny5Tz88MPs27ePWbNm0fxKEVuOOXpURhc/\n+wzeew8mToQcOVyrUtKKGkJGZN062L0b3n1XzODee2X1Lvxx8rZW5hDy5ZOTadKw0bPPwr59ssx0\n61bZ/vbb5b3nn5eQ0wMHoE8fuO66YO/ZldmwQe5DoIcwYcIEypcvT1RUFEuWLKFChQquJSXL6tUy\nLLRokXx9evfWyeNQRw9fRiRvXrnaP3NGLudiYmRYaPt2eT9pzTBA2bJiBlmzyvP8+SVF5dmzUKOG\nzB088sgff9urGco2bpSez623ulZyRZIiierUqUPJkiU9HUn02WeSovrsWclS2rSpa0WKP1BDyIjk\nyCErin/6SZ7fdptMBK9bJ8+N+aOnUKSIlK0qXVpO/iBX2X36yFX3u++6CSFNLbGx0hPy6CWstZYP\nPvjg90iiuXPnejKSKDFRDnmbNrKEZPXqP18PKKGNN38dSmDJn18Why1aJM/z5IETJ2QtAcC5c2II\nW7ZA3boSt//++5IYLolMIbTI/fhxCYqvVMm1kmSx1tKxY0d69+5N69atPRtJlJAgPYGPP5YMIN99\nFzY5AhUfIfSrVvxGVJSki3jnHZlDyJdPMo0WLCgLz+LiJAKpSJE/JmNDmblzJTzWgymhrbV06tSJ\nXr160bp1awYNGuTJSKLTpyX9xKxZ0KMHfPCBN2IFFP+ihhCuHD8ui8peeCH59594Ah54QHoAK1ZI\nmugCBWRi2KMVttLMrFkyt+GxwHhrLR9++CE9e/akVatWnjWDQ4dkfcHKlTBkiCxRUcITNYRwY+9e\n+Ne/4Isv5LIuJubKkTWDBsmvPDpaxtch/MzAWjGEihVTXkMhSHTr1o0ePXrQsmVLPv/8c0+awa5d\n0rH6+WeYNAmqV3etSAkk3vsGKqknPl5+rdWqwR13yJLR55+XSeOrhVlmziyTxUlmEI5s2SIm6bHh\nom7dutGtWzeaN2/O4MGDPWkGq1dLkNnhwzBnjppBRkB7CKHMhg2SMXTECOnXFyggiefbtPF0eGVQ\nSZoI95AhdO/ena5du9KsWTOGDBniSTOYP1+uKXLlkimYEFi+ofgBNYRQ48QJySU8dKhEzmTOLJdu\nLVrIPEAoRf8Eg5kzZcW0R8pK9ujRgy5duvDCCy941gzGjYPGjSWmYNaskEwOq6QRPXuEAtZKKumh\nQ+XXeu6cpGDo109+uRr7lzznz0vKzVatXCsBoGfPnnTu3JkmTZowdOhQIiMjXUv6E9ZKRpN33pGh\noilTtOZxRkMNwcvs3y85AYYOlYRyuXJJIHiLFjJZrHF/V2fRIjFPDwwX9e7dm44dO9KoUSOGDRvm\nOTO4cEGS2376KdSqBV99JemplIyFGoIXWbZM6hNPmyZLQ8uXh65dpeSklp1KObNmySrq8uWdyujb\nty8dOnSgYcOGDB8+3HNmcPasJLedPl1WIWtOooyLGoKXOHJEFop98YWsGn7vPekNFCniWlnoYa1E\nXj3++B/ZWx0wevRo2rdvT7169TxpBnFxkpJq9myJQn75ZdeKFJeoIXiBxERZIdy+PRw7Bm+/LQnk\ncuZ0rSx0+fFHqfPQubMzCWvWrKFly5Y89thjjBgxgkwem/BPSIB69eD77+HLL+XaQ8nYeOsbmhFZ\nu1aqiyxeLGUoBw0K73UBweKTT+D662UsxAHHjx+nVq1a5MmTh3HjxpHFYwkAL16U6ajJk+VfpWag\ngC5Mc8epU9ITKFVKFk8NGyaRRGoG6WfPHqnU0rq1k+Eiay3NmjVjz549jB071nNZSxMT4aWXYMwY\n6NULXnvNtSLFK2gPIdgkJso6gnfflUIzrVvLBPINN7hWFj785z8SgfXqq06a79u3L5MnT+Zf//oX\n5cqVc6LhSlgLb74pQ0SdOkkSW0X5HWutJ28PPvigDTsWLbL2b3+T8jMPPGDt0qWuFYUfp05Zmzu3\ntfXqOWl+/vz5NiIiwtatW9cmJiY60XA1OnSQr9+bb1rrQXmKHwBibRrPuzpkFAy2bZPg7kcflbUF\nw4dLwZbSpV0rCz/++19Zzf3mm0Fvev/+/dSrV49ixYoxZMgQjMfWifz97zJE1Lo1/POfuoxF+Ss6\nZBRIjhyBjz6CgQOlBOVHH8Fbb+lagkCRmCg1nUuXDnqq64SEBOrWrcuZM2eYP38+OT0WIfavf8kQ\nUePGEregZqAkh196CMaYp40xW4wx240xfxmVNMZkNcZ843v/f8aYwv5o17PExcklWJEiknm0RQup\nV9ypU/ibwahRkjcoIkLuR40KXtvTp8v/+Y03gtemj/fee4/FixczZMgQinssE9zw4dJhqllTYheu\ntujM5eFTPEBax5qSbkAksAO4A8gCrAGKX7bNK8Bnvsf1gW+u9XdDcg4hMdHab76x9vbbZaD2mWes\nXb/etargMXKktVFRsu9Jt6goeT0YVKxobcGC1sbHB6c9H2PHjrWAfe2114LabkqYMcPayEj518TF\nXX1b14dP8Q+kYw7BH4ZQBph1yfMOQIfLtpkFlPE9zgQcBszV/m7IGcKWLdaWKSP/0vvus/b7710r\nCj6FCv35bJJ0K1Qo8G2vXStt9e4d+LYuYevWrTZHjhy2dOnSNu5aZ9wgs3q1tdHR1pYsae2JE9fe\n3uXhU/xHegzBH0NGtwA/X/L8F99ryW5jrb0AnAD+EmdpjGltjIk1xsQeOnTID9KCxNdfw4MPwtat\nEs+3ahVUruxaVfDZuzd1r/uToUMlb1EQM5taa3n55ZeJjIxk7Nixnlp8Fh8v1VNz5pSRtFy5rv0Z\nl4dP8QaeijKy1g621sZYa2NuDIWUzufPSzGaBg3gvvukQlmLFuCxfDVB47bbUve6v0hIgNGjpS5E\nENdzjBkzhnnz5tGrVy9u9VhBol69ZBH8559D/vwp+4yrw6d4B38Ywj7g0l9DQd9ryW5jjMkE5AaO\n+KFtd2zbJtEsn30mOYgWLNAqZX//+18nzaOi5PVAMmsW/PabXBIHiRMnTvDWW28RExND69atg9Zu\nSli7Fnr0gIYNU1f20tXhUzxEWseakm7InMBO4Hb+mFQucdk2r/LnSeWx1/q7np5D+Ppra3PmtDZP\nHmunTnWtxluMHCmDzsbIfTBmJOvUsTZv3qBOJrdt29ZGRETY2NjYoLWZEhISrH3wQWtvusnaw4dT\n/3kXh0/xL7icVJb2qQJsRaKNOvpe6w5U9z3OBowDtgPLgTuu9Tc9aQjnzlnbpo3828qUsXbPHteK\nlKNHrc2a1dp27YLWZGxsrI2IiLBt27YNWpsppWdP+XqOG+daieKK9BiCkc97j5iYGBsbG+taxh9s\n3w5168o8wTvvSP6hzJldq1IGD5ZMbbGxMrEfYBITEylTpgx79uxhy5Yt5M6dO+BtppRNm6BkSRkm\nGjfOtRrFFcaYldbamLR8Vlcqp4Rx46BlSylgP2UKVKvmWpGSxIgRULy4ZI0NAqNHj2b58uWMGDHC\nU2Zw8aLEM+TMKbn9FCUteCrKyHPExUHbttIzKF5cegdqBt5hxw6pI/HCC0HJxXD27Fk6dOhATEwM\njRo1Cnh7qWHAAKm8+skncPPNrtUooYr2EK7Epk3QqJGYwFtvSRyfh+LMFaQSvDFynIJAv379+OWX\nXxg9ejQRHio6vG0bdOwoQ0UNGrhWo4Qy3vlWewVrJRldqVKyImfyZMlLpGbgLayV4aKKFaFgwYA3\nd+DAAXr37k2tWrV49NFHA95eSklMlNHMbNk0aZ2SftQQLuX0abnMatsWypeHdetSF8itBI/Fi2HX\nrqCtPejatSvx8fH06dMnKO2llCFDYNEi6N8fChRwrUYJdXTIKIlDh6BqVVi5UgZkX3tNL7e8zNSp\nEuX1/PMBb+r06dOMHDmSpk2b8n//938Bby81DBoEMTFSH1lR0osaAsDu3fDkk/DzzzBpkvYKQoGZ\nM+GRRySsJsBMmjSJs2fP0tRjZ92NG2H1arl+0WsXxR/okNHatVC2LBw+DHPmqBmEAvv3y3F7+umg\nNDdy5EgKFy5M2bJlg9JeShkzRuoW1K3rWokSLmRsQ/jhBylrGRkpA7EeK4iuXIFZs+T+qacC3tSB\nAweYM2cOjRo18lRkkbWSz69iRciXz7UaJVzwzjc82EycKCeUW26BJUugRAnXipSUMnOmnAXvuy/g\nTX399dckJibSuHHjgLeVGpYvh507JYGdoviLjGkIn30GtWtLaOmPP2qW0lDi4kWYPVuGi4IwcD5y\n5EhiYmK46667At5Wapg+XYaLgjCnrmQgMpYhWAtdu0oNg6pVZc4gTx7XqpTUsGIFHDsWlPmDjRs3\nsmrVKs/1DkCuY0qWhOuuc61ECScyjiGcPw9NmkC3btC8uUQThXvB+3Bk5kzpGVSqFPCmRo4cSWRk\nJPXr1w94W6khIUHSVDzyiGslSriRMcJOf/tN+tZLl0q1jw4dNE4vVJk5Ex56KOCV0RITExk1ahSV\nK1fmZo8lB1q9Gs6eVUNQ/E/49xDWr5cTyOrVkrX0gw/UDEKVI0dkyCgIw0U//vgje/fu9exwEWhQ\nnOJ/wruHsGSJnDxy5ICFC2VJpxK6TJ8uyXuCEG46ZswYoqOjed6Ds7ZLlsDtt2uqCsX/hK8hrFwJ\nzzwj4Ynz5gUlAZoSQKyFfv3gzjvh4YcD3tzy5cspV64c0dHRAW8rtZw/Dx4qxaCEEeE5ZLRunaSi\nyJMH5s5VMwgHvv9ehv3ee0/iLQPIhQsX2LBhA/fee29A20kruXPDyZOuVSjhSPgZwpYtEoGSPbuY\nga4xCA9695ZFhEGofbB9+3bi4uI8awi5cqkhKIEhvAxh1y5Zyw9iBnfc4VaP4h+WLYMFC+Dtt4NS\nl2LdunUAnjWE3LnhxAkZRVMUfxI+hvDLL1ChApw7JytZ77zTtSLFX/TpA9dfDy++GJTm1q1bR0RE\nBMWLFw9Ke6klVy5ZixAX51qJEm6EhyEcPCg9g6NHJfFZEHLcKEFi0yb49lupT5EjR1CaXLt2LcWK\nFSNbtmxBaS+1JE0onzjhVocSfoS+Ifz2G1SuDPv2wYwZGloabvzjHzIf9NprQWty3bp1nh0uAukh\ngM4jKP4ntA3hwAEpdbljB0yZInUNlPDh559h5Eho1Qry5g1Kk6dPn2bnzp2eNgTtISiBInTXIfz8\ns8wZHDwoPYPHH3etSPE3/fvLzOnbbwetyQ0bNgBwn4eHHbWHoASK0DSEXbvEDI4elfj0MmVcK1L8\nzZEjMHiwJPwvVChoza5duxbwboQRqCEogSP0DOHgQXjiCfk1zJ2rcwbhyuDBcOYMtG8f1GY3bdpE\n9uzZKVy4cFA2cd2ZAAAUCUlEQVTbTQ1JKa9/+82tDiX8CK05hHPnJGvpoUMSWqpmEJ5YCyNGSHnT\ne+4JatOJiYlkzpzZU+UyL6dQIcifX34CiuJPvPutvxxrpY7B8uUwahQ8+KBrRUqgWLMGNm8Oyqrk\ny4mKiuLs2bNBbzc1RETAc8/J1NmZM67VKOFE6BhC167wzTeSwsCDGSgVPzJ6NGTKJGVOg0xUVBQX\nLlwgISEh6G2nhiZNxAyGDXOtRAknQsMQRo2C7t2lh/Duu67VKIEkMRHGjJG05QEugpMcUb4qel7v\nJZQtK7EU/frBhQuu1SjhgvcNYckSaNFCwko/+0yL24Q7ixZJGpKGDZ00HyqGAHJttGsXTJzoWokS\nLnjbEHbvluGh226DCROCkthMcczo0RAdDdWrO2k+lAyhenUoWlQWc2uiO8UfeNcQLl6EZ5+VLF7f\nfedk+EAJMvHxUub0+efFFBwQSoYQGQnvvCO1oBYscK1GCQe8awg7d0ptg/HjNXNpRmHmTDh2zNlw\nEYSWIQC88ALcdBP07etaiRIOeNcQoqNh4MA/6hso4c/o0ZKzqHJlZxJCzRCyZZO8fzNmSKFARUkP\n3jWEAgWgdWvXKpRgceqUJCisWxcyZ3YmI9QMAaBNG4iKgo8/dq1ECXW8awhKxmLqVFmJ3qCBUxm5\nfImCjhw54lRHarjhBkkIO3o0bN3qWo0SyqghKN5g8mTIl895CvOiRYuSO3duFi5c6FRHaunQQcpG\nvPWWayVKKKOGoLgnLk4GwatVk7wMDomMjKRChQrMnj0bG0KxnPnyQefOMG2a/CsVJS2oISju+eEH\nmUNwtPbgcipVqsTevXvZsWOHaymp4vXXZV3Cm29KBK+ipBY1BMU9U6bIrKhHIsoq+6KcZodYOtEs\nWaSm0JYt8J//uFajhCJqCIpbrBVDePJJGQT3AEWKFOG2225jzpw5rqWkmqpV4ZlnoFs3rZegpB41\nBMUtq1dLOdTnnnOt5HeMMVSuXJl58+Zx8eJF13JSTb9+cPYsdOzoWokSaqghKG755htJWFi1qmsl\nf6JSpUocP36clStXupaSau66C9q1gy+/lLQWipJS0mUIxpg8xpjZxphtvvvrr7DdRWPMat9tSnra\nVMKI336Twe7ateHGG12r+RMVKlQACMlhI4APP5R/6euva+I7JeWkt4fwPjDXWlsUmOt7nhznrLUl\nfTdvhJIo7unZE86fh48+cq3kL9x0002ULFkyZA0hd2759y5eDF9/7VqNEiqk1xCeA4b7Hg8HtJSZ\nkjL27IFBg6BZM88mL6xUqRKLFy8OqTQWl9KsGZQqJXUTtNSmkhLSawg3W2sP+B4fBG6+wnbZjDGx\nxphlxpgrmoYxprVvu9hDhw6lU5riaZLSc3bp4lbHVXjqqaeIj49n0qRJrqWkichI+OQT2LdPJ5iV\nlGGutRrTGDMHyJfMWx2B4dba6y7Z9pi19i/zCMaYW6y1+4wxdwDzgIrW2quu+omJibGxsbEp2Qcl\n1Dh5Em65BWrWhOHDr729IxITE7n//vu5cOEC69evJzIy0rWkNPHaazJVM20aVKniWo0SaIwxK621\nMWn57DV7CNbaStbae5K5TQZ+Ncbk94nIDyQb+Wyt3ee73wksAB5Ii1glTPjqKzh9Gl591bWSqxIR\nEUHnzp3ZvHkz48ePdy0nzfTpAyVLQqNGsH27azWKl7lmD+GqHzamL3DEWtvbGPM+kMda2/6yba4H\nzlpr44wxeYGlwHPW2o1X+9vaQwhTrIUSJSBHDli+3LWaa5KYmMi9996LMYa1a9cS4TjXUlrZtQti\nYiSr/NKl8u9XwpOA9hCuQW+gsjFmG1DJ9xxjTIwxZohvm7uBWGPMGmA+0PtaZqCEMfPnw6ZNnu8d\nJBEREUGnTp3YsGEDEyZMcC0nzdx+u0QbbdwILVtqKKqSPOnqIQQS7SGEKTVrwsKF8MsvUu4rBLh4\n8SL33HMPmTNnZvXq1SHbSwAZPnr/fZnTf+cd12qUQOCyh6AoKefnn6XuQatWIWMGICmxO3XqxLp1\n6/j2229dy0kX7dvLOsD33oO5c12rUbyGGoISPD7/XMYqXn7ZtZJUU79+fYoVK0b37t1JTEx0LSfN\nGAPDhsHdd0O9erIcRFGSUENQgkNcHAweLEVwChd2rSbVJPUS1qxZw5QpoZ19JUcOmDQJLlyAGjWk\ncqmigBqCEizGj4dDh0JmMjk5GjRoQJEiRejevXtIVVNLjqJFYeRI+Okn6bCF+O4ofkINQQkOAwdC\nsWJQqZJrJWkmU6ZMdOzYkZ9++onvvvvOtZx08+yz0LUrjBghh0dRNMpICTwbNsA990g5rzfecK0m\nXVy4cIG7774bgDVr1hAVFeVYUfpITITnn5c6zD/8AGXLulakpBeNMlK8zahRklinYUPXStJNpkyZ\nGDx4MNu3b6djGCQIioiQHkKhQlCnjlZZy+ioISiBJTERRo+GypXhpptcq/ELTzzxBK+88goDBgzg\nxx9/dC0n3Vx3HUyYAEePQv36MtmsZEzUEJTAsmSJxDY2auRaiV/p06cPhQoVokWLFiGbHvtS7r8f\nPvtMFpJ37uxajeIKNQQlsIwaBVFRMlAdRuTIkYMvv/ySbdu20TlMzqBNm0Lr1tC7t/QYlIyHGoIS\nOOLjYexYeO65sMymVqFCBdq0aUP//v1ZsmSJazl+YcAAKF0amjQBjenIeKghKIFj1iwZmA6z4aJL\n6dOnD7fddhvNmzfnXBis8MqWDb79VqZ7qlWTbCNKxkENQQkco0ZB3rzw5JOulQSMnDlz8uWXX7J1\n61Y+/PBD13L8ws03SzGds2fFFE6dcq1ICRZqCEpgOHUKpkyBunUhc2bXagJKxYoVeemll+jXrx9L\nly51LccvlCgho33r10ODBnDxomtFSjBQQ1ACw6RJkiQnjIeLLuUf//gHBQsWDJuhI4CnnoJ//1t6\nC2+/7VqNEgzUEJTAMGqUVGUpU8a1kqCQK1cuhgwZwpYtW8Jm6AigTRtZXD5gAHz6qWs1SqBRQ1D8\nz5kzMG+eLH01xrWaoFG5cmVeeuklPv74Y2bMmOFajt/4+GPJe9SunaxTUMIXNQTF/yxZIstdK1Rw\nrSTo9O/fn/vuu4/GjRuzJ0yKDURGSoevWDHx+F27XCtSAoUaguJ/fvhBziIZMFNa9uzZGT9+PBcu\nXKBOnTrExcW5luQXcuWSYncXL8qyktOnXStSAoEaguJfrJXooocfhpw5XatxQtGiRRk2bBgrVqzg\n7TCajS1aFL75RpLXNmsmaaqU8EINQfEvy5fDunWSByEDU7NmTd5++20GDhzImDFjXMvxG08+CX37\nSmqLHj1cq1H8jRqC4l8GD4boaAlez+D06tWLRx55hFatWrF+/XrXcvzGm2/CCy9Aly6yqlkJH9QQ\nFP9x8iR8/bWYQQYdLrqUzJkzM3bsWHLlykWNGjU4fvy4a0l+wRj4/HN46CHJebRunWtFir9QQ1D8\nx+jRku/gxRddK/EM+fPnZ/z48ezevZvGjRuTGCYD79myydrDnDmhalXYv9+1IsUfqCEo/uOLLySx\n/t/+5lqJpyhXrhwDBgxg2rRpdO/e3bUcv1GggKxiPnpU1ilozqPQRw1B8Q8rV8KqVdI7yECL0VJK\nmzZtaNq0Kd26dWPq1Kmu5fiNBx6AceNg7VpJW6XV1kIbNQTFPwwZAtmzZ5jcRanFGMOgQYMoVaoU\njRs3ZsuWLa4l+Y1nnoFBg2DmTHjlFYk8VkITNQQl/VgLU6fKuMF117lW41myZ8/OxIkTyZo1K9Wq\nVePYsWOuJfmNF1+EDz6QUcMBA1yrUdKKGoKSfnbvhn37oHx510o8T6FChZg4cSK7d++mTp06JCQk\nuJbkNz76CGrUkMyoYZTKKUOhhqCkn4UL5f6xx9zqCBEeeeQRBg8ezNy5c3njjTdcy/EbERHw1Vdw\n331Qvz5s3OhakZJa1BCU9LNwIeTJA8WLu1YSMjRr1oz27dvz6aefMnDgQNdy/EZ0tGQuyZ5dqq0d\nPuxakZIa1BCU9LNwITz6qFwiKimmZ8+eVK9enddff53Zs2e7luM3br1VEuHt2we1akF8vGtFSkrR\nX7CSPvbvh+3bdbgoDURGRjJy5EhKlChBnTp12Lx5s2tJfuPhh2HoULlWePVVjTwKFdQQlPSxaJHc\nqyGkiZw5czJlypTfI4+OHj3qWpLfaNgQOnaUiGSNPAoN1BCU9LFwIeTIASVLulYSshQqVIhJkyax\nd+9eateuHVaRR927Q82aGnkUKqghKOlj4UIoVw4yZXKtJKQpW7YsX375JfPnz+eVV17BhskYS0QE\njBghGU008sj7qCEoaefIEVi/XoeL/ETjxo3p1KkTQ4YM4Z///KdrOX4jOlommaOioEoVOHDAtSLl\nSqghKGln8WK5V0PwG926daNOnTq0b9+eb8Oo2MCtt8pi9sOHJTuqJsLzJmoIStpJqrZ+991udYQR\nERERDB8+nIceeoiGDRuyYsUK15L8RkzMH4nw6tfXRHheRA1BSTtJuXg0f5FfyZ49O5MnTyZfvnw8\n++yz7N6927Ukv/HMMzBwIEyfDu3aaTiq11BDUNLOsWOQOzdERrpWEnbcfPPNTJ8+nfj4eKpUqRI2\n1dYAXnoJ2reXDKn9+rlWo1yKGoKSdo4dg+uvd60ibLnrrrv49ttv2b59OzVr1iQ+jJb89uoFderA\nO+/AhAmu1ShJqCEoaefoUTWEAPP4448zdOhQ5s+fz4svvhhW4ajDh0OZMtC4MSxb5lqRAmoISno4\ndkyS2ikBpXHjxnTr1o0RI0bw0UcfuZbjN7Jnl3DUAgWgenXYudO1IkUNQUk7OmQUNDp37kzTpk3p\n0qULX331lWs5fuPGG2UF88WLskYhjDJ3hCTpMgRjTB1jzAZjTKIxJuYq2z1tjNlijNlujHk/PW0q\nHkKHjIKGMYbBgwdToUIFWrZsyYIFC1xL8hvFisG330oUc40aEBfnWlHGJb09hPVATWDhlTYwxkQC\nA4FngOJAA2OMJs4PdazVHkKQyZIlCxMmTKBo0aLUqFGDTZs2uZbkNx59FIYNk0woLVtqOKor0mUI\n1tpN1tprVQt/CNhurd1prY0HvgaeS0+7igc4d04S3escQlC57rrrmDZtGlmzZqVKlSr8+uuvriX5\njYYNoUcPGDUKunRxrSZjEow5hFuAny95/ovvNSWUOXdOlp4WLuxaSYajcOHCfPfddxw+fJg5c+a4\nluNXPvgAWrSAb76BM2dcq8l4mGuFsRlj5gD5knmro7V2sm+bBcA71trYZD5fG3jaWtvK97wJ8LC1\ntm0y27YGWvue3oMMSYUreYFwLjCo+xfahPP+hfO+Adxprc2Zlg9eM2extbZSWv7wJewDbr3keUHf\na8m1NRgYDGCMibXWXnGiOtTR/QttdP9Cl3DeN5D9S+tngzFktAIoaoy53RiTBagPTAlCu4qiKEoq\nSG/YaQ1jzC9AGWCaMWaW7/UCxpjpANbaC0BbYBawCRhrrd2QPtmKoiiKv0lXmStr7SRgUjKv7weq\nXPJ8OjA9lX9+cHq0hQC6f6GN7l/oEs77BunYv2tOKiuKoigZA01doSiKogAeMoRwT4NhjMljjJlt\njNnmu092ia8x5qIxZrXv5vnJ92sdD2NMVmPMN773/2eMKRx8lWknBfvXzBhz6JJj1sqFzrRgjBlq\njPnNGJNseLcRPvHt+1pjTKlga0wPKdi/8saYE5ccuw+DrTGtGGNuNcbMN8Zs9J03X09mm9QfP2ut\nJ27A3cCdwAIg5grbRAI7gDuALMAaoLhr7Sncv38A7/sevw/0ucJ2p11rTcU+XfN4AK8An/ke1we+\nca3bz/vXDPiPa61p3L/HgFLA+iu8XwWYARigNPA/15r9vH/lge9c60zjvuUHSvke5wS2JvPdTPXx\n80wPwYZ/GozngOG+x8OB5x1q8RcpOR6X7vd4oKIxxgRRY3oI5e/bNbHWLgSull/0OWCEFZYB1xlj\n8gdHXfpJwf6FLNbaA9baVb7Hp5AIzsszQKT6+HnGEFJIKKfBuNlae8D3+CBw8xW2y2aMiTXGLDPG\neN00UnI8ft/GSgjyCeCGoKhLPyn9vtXydcnHG2NuTeb9UCWUf28ppYwxZo0xZoYxpoRrMWnBNwz7\nAPC/y95K9fFLV9hpaklJGoxQ5mr7d+kTa601xlwpvKuQtXafMeYOYJ4xZp21doe/tSp+Yyowxlob\nZ4x5CekNVXCsSUkZq5Df22ljTBXgW6CoY02pwhiTA5gAvGGtPZnevxdUQ7BBTIPhgqvtnzHmV2NM\nfmvtAV+37bcr/I19vvudvhxRDyDj2F4kJccjaZtfjDGZgNzAkeDISzfX3D9r7aX7MgSZKwoXPP17\nSy+XnkCttdONMZ8aY/Jaa0Miz5ExJjNiBqOstROT2STVxy/UhoxCOQ3GFKCp73FT4C89ImPM9caY\nrL7HeYFywMagKUw9KTkel+53bWCe9c14hQDX3L/LxmSrI2O54cIU4AVftEpp4MQlw54hjzEmX9J8\nljHmIeR8GBIXKz7dXwKbrLX9rrBZ6o+f69nyS2bEayBjXHHAr8As3+sFgOmXzZxvRa6aO7rWnYr9\nuwGYC2wD5gB5fK/HAEN8j8sC65BolnVAS9e6U7BffzkeQHeguu9xNmAcsB1YDtzhWrOf968XsMF3\nzOYDd7nWnIp9GwMcABJ8v72WwMvAy773DVLcaofv+5hs9J9XbynYv7aXHLtlQFnXmlOxb48AFlgL\nrPbdqqT3+OlKZUVRFAUIvSEjRVEUJUCoISiKoiiAGoKiKIriQw1BURRFAdQQFEVRFB9qCIqiKAqg\nhqAoiqL4UENQFEVRAPh/dYf6uJnTbYAAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"SVmI2xtK4KZC","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]} -------------------------------------------------------------------------------- /labs/eda_titanic.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Lab 0A - EDA on Titanic.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":["iwLiwKhfXPk0"],"toc_visible":true},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"iwLiwKhfXPk0","colab_type":"text"},"source":["## Setup"]},{"cell_type":"code","metadata":{"colab_type":"code","id":"LF7fKrycl3uJ","colab":{}},"source":["import pandas as pd\n","\n","import matplotlib.pyplot as plt\n","import seaborn\n","\n","import tensorflow as tf\n","\n","%matplotlib inline\n","%config InlineBackend.figure_format = 'retina'"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"Ffy0K-fYmF1E"},"source":["# Exploratory Data Analysis (EDA)\n","\n","Exploratory Data Analysis (EDA) is a term we use to describe the intial process of exploring and trying to better understand a dataset. It is a crucial early step that we need to take when approaching a new problem.\n","\n","This notebook presents a simple way of tackling EDA on a tabular dataset. Very often, the precise steps will differ depending on your problem and dataset, but this notebook aims to provide a \"feel\" of what it is like to do EDA.\n","\n","**Titanic Dataset**\n","\n","![Titanic (source image for Kaggle)](https://storage.googleapis.com/kaggle-competitions/kaggle/3136/logos/header.png)\n","\n","The `titanic.csv` file contains data for 891 of the real [Titanic](https://en.wikipedia.org/wiki/Sinking_of_the_RMS_Titanic) passengers. Each row represents one person. The columns describe different attributes about the person. We can use this small dataset to develop toy models to predict, based on the attributes of a person, the likelihood of someone to survive the Titantic disaster."]},{"cell_type":"markdown","metadata":{"id":"KGYX7wN-VAeP","colab_type":"text"},"source":["## Loading the Data"]},{"cell_type":"markdown","metadata":{"id":"U6FC1oZSVC4a","colab_type":"text"},"source":["First, we will need to download the dataset. The [`tf.keras.utils.get_file`](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) function is very useful since it caches the downloaded files automatically. It can also be used to download and extract archives (such as `zip` files)."]},{"cell_type":"code","metadata":{"id":"Je_nf80jT0SI","colab_type":"code","outputId":"aebf5379-2202-4bb5-b70d-3b7949006107","executionInfo":{"status":"ok","timestamp":1564239492571,"user_tz":-480,"elapsed":3501,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["DATA_URL = \"https://deeplearning-mat.s3-ap-southeast-1.amazonaws.com/titanic.csv\"\n","DATA_DIR = tf.keras.utils.get_file(\"titanic.csv\", DATA_URL, cache_subdir='datasets')\n","\n","print(\"Dataset present at:\", DATA_DIR)"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Dataset present at: /root/.keras/datasets/titanic.csv\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"wbD2KwYNVhBA","colab_type":"text"},"source":["Next, we can use the popular [Pandas](https://pandas.pydata.org/) dataframe library to load the CSV file into a `Dataframe` object and display the data in a nice tabular format."]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239492572,"user_tz":-480,"elapsed":3494,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"YGAGobyQl5sD","outputId":"aff02fd5-4153-4901-dbf7-26d7aa097901","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["# use Pandas to load the csv file\n","\n","df = pd.read_csv(DATA_DIR)\n","\n","print(\"df is of type\", type(df))"],"execution_count":3,"outputs":[{"output_type":"stream","text":["df is of type \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"wfy3ywthX3xx","colab_type":"code","outputId":"346f604f-9f09-4540-d535-e801419bffee","executionInfo":{"status":"ok","timestamp":1564239492573,"user_tz":-480,"elapsed":3488,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":306}},"source":["df.info()"],"execution_count":4,"outputs":[{"output_type":"stream","text":["\n","RangeIndex: 891 entries, 0 to 890\n","Data columns (total 12 columns):\n","PassengerId 891 non-null int64\n","Survived 891 non-null int64\n","Pclass 891 non-null int64\n","Name 891 non-null object\n","Sex 891 non-null object\n","Age 714 non-null float64\n","SibSp 891 non-null int64\n","Parch 891 non-null int64\n","Ticket 891 non-null object\n","Fare 891 non-null float64\n","Cabin 204 non-null object\n","Embarked 889 non-null object\n","dtypes: float64(2), int64(5), object(5)\n","memory usage: 83.6+ KB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"I_KJPPuqVqCa","colab_type":"code","outputId":"92213186-9379-4e80-b020-a902eebfa470","executionInfo":{"status":"ok","timestamp":1564239492576,"user_tz":-480,"elapsed":3482,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["# look at the top few rows of the dataset\n","\n","df.head()"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
PassengerIdSurvivedPclassNameSexAgeSibSpParchTicketFareCabinEmbarked
0103Braund, Mr. Owen Harrismale22.010A/5 211717.2500NaNS
1211Cumings, Mrs. John Bradley (Florence Briggs Th...female38.010PC 1759971.2833C85C
2313Heikkinen, Miss. Lainafemale26.000STON/O2. 31012827.9250NaNS
3411Futrelle, Mrs. Jacques Heath (Lily May Peel)female35.01011380353.1000C123S
4503Allen, Mr. William Henrymale35.0003734508.0500NaNS
\n","
"],"text/plain":[" PassengerId Survived Pclass ... Fare Cabin Embarked\n","0 1 0 3 ... 7.2500 NaN S\n","1 2 1 1 ... 71.2833 C85 C\n","2 3 1 3 ... 7.9250 NaN S\n","3 4 1 1 ... 53.1000 C123 S\n","4 5 0 3 ... 8.0500 NaN S\n","\n","[5 rows x 12 columns]"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"OKMLWyLiUYRf","colab_type":"code","outputId":"3e92e266-e929-47cd-ee39-746bfe942382","executionInfo":{"status":"ok","timestamp":1564239492577,"user_tz":-480,"elapsed":3477,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["# look at the last few rows of the dataset\n","\n","df.tail()"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
PassengerIdSurvivedPclassNameSexAgeSibSpParchTicketFareCabinEmbarked
88688702Montvila, Rev. Juozasmale27.00021153613.00NaNS
88788811Graham, Miss. Margaret Edithfemale19.00011205330.00B42S
88888903Johnston, Miss. Catherine Helen \"Carrie\"femaleNaN12W./C. 660723.45NaNS
88989011Behr, Mr. Karl Howellmale26.00011136930.00C148C
89089103Dooley, Mr. Patrickmale32.0003703767.75NaNQ
\n","
"],"text/plain":[" PassengerId Survived Pclass ... Fare Cabin Embarked\n","886 887 0 2 ... 13.00 NaN S\n","887 888 1 1 ... 30.00 B42 S\n","888 889 0 3 ... 23.45 NaN S\n","889 890 1 1 ... 30.00 C148 C\n","890 891 0 3 ... 7.75 NaN Q\n","\n","[5 rows x 12 columns]"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"markdown","metadata":{"id":"ut8NszMi7ZEQ","colab_type":"text"},"source":["The first thing we notice: **there are missing (`NaN`) values!**\n","\n","This is a huge deal since now we must decide how to deal with that, since obviously missing some data columns in our training set, and very possibly test set and real world, will result in problems for our ML algorithm."]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"p8CTz7qvqkjE"},"source":["## Search for Missing Data\n","\n","Missing data is a real problem that is relatively common to encounter in real world datasets.\n","\n","We can use `isnull()` on the `Dataframe` to check for missing data. `isnull()` will return a new `Dataframe` object, which will show `True` values where there is missing data, and `False` where data is present."]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239492577,"user_tz":-480,"elapsed":3471,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"-wnHsqMxotCf","outputId":"11e378f0-c07a-4d9c-b416-0be90267eac0","colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["df.isnull().tail()"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
PassengerIdSurvivedPclassNameSexAgeSibSpParchTicketFareCabinEmbarked
886FalseFalseFalseFalseFalseFalseFalseFalseFalseFalseTrueFalse
887FalseFalseFalseFalseFalseFalseFalseFalseFalseFalseFalseFalse
888FalseFalseFalseFalseFalseTrueFalseFalseFalseFalseTrueFalse
889FalseFalseFalseFalseFalseFalseFalseFalseFalseFalseFalseFalse
890FalseFalseFalseFalseFalseFalseFalseFalseFalseFalseTrueFalse
\n","
"],"text/plain":[" PassengerId Survived Pclass Name ... Ticket Fare Cabin Embarked\n","886 False False False False ... False False True False\n","887 False False False False ... False False False False\n","888 False False False False ... False False True False\n","889 False False False False ... False False False False\n","890 False False False False ... False False True False\n","\n","[5 rows x 12 columns]"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"markdown","metadata":{"id":"UuKgiVuI71rL","colab_type":"text"},"source":["In the above dataframe, everywhere with `True` values have missing/`NaN` values."]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239492578,"user_tz":-480,"elapsed":3466,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"vWIOwpVApFwS","outputId":"6996c2c8-2eee-4d69-9ac5-05a90d11d2b8","colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["for col in df.columns:\n"," col_has_null = df[col].isnull().values.any()\n"," if col_has_null:\n"," print(\"Column\", col, \"has null value is:\", col_has_null)"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Column Age has null value is: True\n","Column Cabin has null value is: True\n","Column Embarked has null value is: True\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"FWNM7V81XVWw","colab_type":"text"},"source":["We observe from the output above that the following columns have missing data:\n","\n","* `Age`\n","* `Cabin`\n","* `Embarked`"]},{"cell_type":"markdown","metadata":{"id":"UsxL62p38LiN","colab_type":"text"},"source":["### \"Fixing\" the Missing Data\n","\n","Now, we shall \"fix\" the dataset by adding in values that might be sensible. Of course, another approach might be to remove the columns with missing data, or the rows with missing data. The best approach would vary depending on the type of problem, and the extent of missing data."]},{"cell_type":"code","metadata":{"colab_type":"code","id":"ihopIciiphHk","colab":{}},"source":["# fill missing ages/fares with mean value\n","\n","df[\"Age\"].fillna(df[\"Age\"].mean(), inplace=True)\n","df[\"Fare\"].fillna(df[\"Fare\"].mean(), inplace=True)\n","\n","# fill cabin and embarked with \"unknown\" category\n","\n","df[\"Cabin\"].fillna(\"\", inplace=True)\n","df[\"Embarked\"].fillna(\"\", inplace=True)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"hUXMRlsBYfmv","colab_type":"text"},"source":["## Data Exploration"]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239492582,"user_tz":-480,"elapsed":3459,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"PmZfjwa0ma3T","outputId":"c0282c49-9793-4f25-f538-fbb61cd8d690","colab":{"base_uri":"https://localhost:8080/","height":297}},"source":["df.describe()"],"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
PassengerIdSurvivedPclassAgeSibSpParchFare
count891.000000891.000000891.000000891.000000891.000000891.000000891.000000
mean446.0000000.3838382.30864229.6991180.5230080.38159432.204208
std257.3538420.4865920.83607113.0020151.1027430.80605749.693429
min1.0000000.0000001.0000000.4200000.0000000.0000000.000000
25%223.5000000.0000002.00000022.0000000.0000000.0000007.910400
50%446.0000000.0000003.00000029.6991180.0000000.00000014.454200
75%668.5000001.0000003.00000035.0000001.0000000.00000031.000000
max891.0000001.0000003.00000080.0000008.0000006.000000512.329200
\n","
"],"text/plain":[" PassengerId Survived Pclass ... SibSp Parch Fare\n","count 891.000000 891.000000 891.000000 ... 891.000000 891.000000 891.000000\n","mean 446.000000 0.383838 2.308642 ... 0.523008 0.381594 32.204208\n","std 257.353842 0.486592 0.836071 ... 1.102743 0.806057 49.693429\n","min 1.000000 0.000000 1.000000 ... 0.000000 0.000000 0.000000\n","25% 223.500000 0.000000 2.000000 ... 0.000000 0.000000 7.910400\n","50% 446.000000 0.000000 3.000000 ... 0.000000 0.000000 14.454200\n","75% 668.500000 1.000000 3.000000 ... 1.000000 0.000000 31.000000\n","max 891.000000 1.000000 3.000000 ... 8.000000 6.000000 512.329200\n","\n","[8 rows x 7 columns]"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239492941,"user_tz":-480,"elapsed":3813,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"Ca4KtBwzmeoG","outputId":"457dd2f9-84a3-4df9-c5cd-a340ce11a74c","colab":{"base_uri":"https://localhost:8080/","height":297}},"source":["ratio = df.Survived.sum()/df.Survived.count()\n","print(\"Survival ratio:\", ratio)\n","\n","seaborn.countplot(x='Survived', data=df);"],"execution_count":11,"outputs":[{"output_type":"stream","text":["Survival ratio: 0.3838383838383838\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAw0AAAIPCAYAAAAxXjJfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xu4pVV9H/DvT6YBQRnwEjVFixq5\nNMEbPFEhESStwXiB6NjQxIjXPF6IN7BJFSPaaG2DVYFEW1MZlCRgoWpJxNiKiApeAFs0TkSUUUlQ\nw20Id8HVP/Z7kuPmnDVnZvaZffaZz+d5zrNmr3et9a49jw+e76x3vataawEAAFjMvaY9AQAAYGUT\nGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoA\nAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACArjXTnsCOqKquSrJ7ko1TngoAAKvb3kluaq09fFsGERqm\nY/d73/ve99t///3vN+2JAACwem3YsCG33XbbNo8jNEzHxv333/9+l1566bTnAQDAKnbggQfmsssu\n27it49jTAAAAdAkNAABAl9AAAAB0CQ0AAECX0AAAAHQJDQAAQJfQAAAAdAkNAABAl9AAAAB0CQ0A\nAECX0AAAAHQJDQAAQJfQAAAAdAkNAABAl9AAAAB0CQ0AAECX0AAAAHQJDQAAQNeaaU+A6Tnw9R+c\n9hSAGXHpHz5/2lMAYIqsNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA\n0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAl\nNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQA\nAABdQgMAANA1c6GhqjZWVVvk5/uL9Dm4qj5eVddX1W1VdXlVvaaqdurc5xlVdUFVbaqqm6vqi1V1\nzPJ9MwAAWJnWTHsCW2lTkncvUH/zeEVVHZnknCS3JzkryfVJnpnkXUkOSfLcBfocm+SUJNclOSPJ\nnUnWJVlfVQe01o6fzNcAAICVb1ZDw42ttRM316iqdk/y/iR3JzmstXbJUP+mJOcnWVdVR7fWzpzX\nZ+8kJ2UULg5qrW0c6t+a5MtJjquqc1prF0/yCwEAwEo1c48nbaF1SR6Y5My5wJAkrbXbk5wwfHz5\nWJ8XJdk5yalzgWHoc0OStw8fX7ZcEwYAgJVmVlcadq6q5yV5WJJbklye5MLW2t1j7Q4fyk8sMMaF\nSW5NcnBV7dxau2MJfc4bawMAAKverIaGByf50FjdVVX1wtbaZ+bV7TuUV4wP0Fq7q6quSvJzSR6R\nZMMS+lxTVbck2auqdm2t3dqbZFVdusil/Xr9AABgJZnFx5NOS/LLGQWH3ZIckOS/Jtk7yXlV9Zh5\nbdcO5aZFxpqr32Mr+qxd5DoAAKwqM7fS0Fp7y1jV15K8rKpuTnJckhOT/Nr2ntdCWmsHLlQ/rEA8\nfjtPBwAAtsosrjQs5n1D+eR5dZtbFZirv3Er+iy2EgEAAKvKagoNfz+Uu82r+8ZQ7jPeuKrWJHl4\nkruSfHuJfR4yjH/15vYzAADAarGaQsMTh3J+ADh/KI9YoP2Tk+ya5KJ5b07aXJ+njbUBAIBVb6ZC\nQ1XtX1W7LVC/d5JTh49nzLt0dpJrkxxdVQfNa79Lkj8YPr53bLjTktyR5Nhh3Lk+eyZ5w/DxfQEA\ngB3ErG2E/vWMTmS+MMl3kvxDkkcmeXqSXZJ8PKPTnJMkrbWbquqlGYWHC6rqzIxOen5WRq9WPTvJ\nWfNv0Fq7qqpen+TkJJdU1VlJ7szooLi9krzTadAAAOxIZi00fDqjX/Yfl+SQjPYX3Jjkcxmd2/Ch\n1lqb36G19tGqOjTJG5M8J6NwcWWS1yU5ebz90OeUqtqY5Pgkz89oRebrSU5orZ2+PF8NAABWppkK\nDcPBbZ/ZbMN79vt8kl/dwj7nJjl3S+8FAACrzUztaQAAALY/oQEAAOgSGgAAgC6hAQAA6BIaAACA\nLqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6h\nAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEA\nAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADo\nEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIa\nAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAA\ngC6hAQAA6BIaAACALqEBAADoEhoAAICumQ8NVfW8qmrDz0sWafOMqrqgqjZV1c1V9cWqOmYz4x5T\nVV8a2m8a+j9jeb4FAACsXDMdGqrqoUlOTXJzp82xSc5N8vNJzkjy/iQ/k2R9VZ20SJ+TkqxP8pCh\n/RlJDkhy7jAeAADsMGY2NFRVJTktyXVJ3rdIm72TnJTk+iQHtdZe2Vp7bZJHJ/lWkuOq6kljfQ5O\nctxw/dGttde21l6Z5MBhnJOGcQEAYIcws6EhyauSHJ7khUluWaTNi5LsnOTU1trGucrW2g1J3j58\nfNlYn7nPbxvazfXZmOSPhvFeuI1zBwCAmTGToaGq9k/yjiTvaa1d2Gl6+FB+YoFr54212ZY+AACw\naq2Z9gS2VFWtSfKhJN9N8obNNN93KK8Yv9Bau6aqbkmyV1Xt2lq7tap2S/LPk9zcWrtmgfG+OZT7\nLHGuly5yab+l9AcAgJVg5kJDkt9P8rgkv9hau20zbdcO5aZFrm9KstvQ7tYltk+SPZY2VQAAmH0z\nFRqq6gkZrS68s7V28bTnszmttQMXqh9WIB6/nacDAABbZWb2NAyPJX0wo0eN3rTEbnMrA2sXuT6+\nsrDU9jcu8f4AADDzZiY0JLlPRnsJ9k9y+7wD3VqSNw9t3j/UvXv4/I2hvMcehKp6SEaPJl3dWrs1\nSVprtyT52yT3Ga6Pe9RQ3mOPBAAArFaz9HjSHUn++yLXHp/RPofPZRQU5h5dOj/JIUmOmFc352nz\n2sx3fpLfGvqctsQ+AACwas1MaBg2Pb9koWtVdWJGoeH01tqfzLt0WpJ/l+TYqjpt7qyGqtoz//Tm\npfGD4d6XUWh4Y1V9dO6shuFAt1dmFF7GwwQAAKxaMxMatkZr7aqqen2Sk5NcUlVnJbkzyboke2WB\nDdWttYuq6r8keV2Sy6vq7CQ/leTXk9wvye/MPygOAABWu1UdGpKktXZKVW1McnyS52e0j+PrSU5o\nrZ2+SJ/jquqrGa0s/HaSHye5LMkfttb+YrtMHAAAVohVERpaaycmObFz/dwk527hmOuTrN+GaQEA\nwKowS29PAgAApkBoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gA\nAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAA\nuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqE\nBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBrzbQnAACz5LtvPWDaUwBmxMN+/6vTnsLEWGkAAAC6\nhAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQG\nAACgS2gAAAC6hAYAAKBroqGhqh5WVbtvps19q+phk7wvAACwfCa90nBVkldvps2rhnYAAMAMmHRo\nqOEHAABYJaaxp+HBSW6Zwn0BAICtsGZbB6iq549VPXaBuiTZKcnDkjwvyVe39b4AAMD2sc2hIcn6\nJG34c0ty5PAzbu6xpVuTvGUC9wUAALaDSYSGFw5lJflAko8m+dgC7e5Ocl2Si1trN07gvgAAwHaw\nzaGhtXb63J+r6pgkH22tfXBbxwUAAFaGSaw0/KPW2lMmOR4AADB9ToQGAAC6Jh4aqurQqvqLqvph\nVf2oqu5e4OeubRj/P1XVp6rqe1V1W1VdX1Vfqao3V9X9F+lzcFV9fGh7W1VdXlWvqaqdOvd5RlVd\nUFWbqurmqvri8PgVAADsUCb6eFJVPT2jjdA7Jflukm8k2eqAsIjXJrksyf9O8sMkuyV5YpITk/x2\nVT2xtfa9eXM6Msk5SW5PclaS65M8M8m7khyS5LkLfI9jk5yS0cbtM5LcmWRdkvVVdUBr7fgJfycA\nAFixJhoaMvrF/UdJnt5a++SEx56ze2vt9vHKqnpbkjck+fdJXjHU7Z7k/Rm9uemw1tolQ/2bkpyf\nZF1VHd1aO3PeOHsnOSmjcHFQa23jUP/WJF9OclxVndNau3iZvh8AAKwok3486eeTnLWMgSELBYbB\nh4fyUfPq1iV5YJIz5wLDvDFOGD6+fGycFyXZOcmpc4Fh6HNDkrcPH1+2VZMHAIAZNOnQcHNG/0I/\nDc8cysvn1R0+lJ9YoP2FGR00d3BV7bzEPueNtQEAgFVv0o8nfSrJkyY85oKq6vgk90myNslBSX4x\no8DwjnnN9h3KK8b7t9buqqqrkvxckkck2bCEPtdU1S1J9qqqXVtrt25mjpcucmm/Xj8AAFhJJh0a\nfjfJl6rqhCRva621CY8/3/FJHjTv8yeSvKC19vfz6tYO5aZFxpir32ML++w2tOuGBgAAWA0mHRre\nnOSvk7wlyYuq6v8muXGBdq219uJtuVFr7cFJUlUPSnJwRisMX6mqZ7TWLtuWsSeltXbgQvXDCsTj\nt/N0AABgq0w6NLxg3p/3Hn4W0pJsU2j4x4Fa+0GSj1TVZRk9UvTBjDZkJ/+0WrB2ob7z6ucHm01J\nHjBcu67TZ7GVCAAAWFUmHRoePuHxlqy19p2q+nqSx1bVA1pr12Z0TsRBSfZJ8hP7C6pqTUbzvSvJ\nt+dd+kZGoWGfJBeP9XlIRo8mXb25/QwAALBaTDQ0tNa+M8nxtsLPDOXdQ3l+kt9MckSSPx9r++Qk\nuya5sLV2x7z68zM69O2IjIWGJE+b1wYAAHYIk37l6rKqqn2q6h6PGlXVvYbD3X46yUXDmQpJcnaS\na5McXVUHzWu/S5I/GD6+d2y405LckeTY4aC3uT57ZnR4XJK8b9u/DQAAzIaJrjRU1cOW2ra19t2t\nuMWvJvmPVfW5JFdltOfgQUkOzei1qd9P8tJ597ipql6aUXi4oKrOzOgciWdl9GrVs5OcNTavq6rq\n9UlOTnJJVZ2V5M6MDorbK8k7nQYNAMCOZNJ7GjZmtMl5c9pW3vv/JPnZjM5keFxGr0q9JaMN0B9K\ncnJr7ScOl2utfbSqDk3yxiTPSbJLkiuTvG5of4/5ttZOqaqNGb3W9fkZrch8PckJrbXTt2LeAAAw\nsyYdGj6YhUPDHkkem+RfJLkgyVbtfWitfS3JsVvR7/MZrVJsSZ9zk5y7pfcCAIDVZtIboV+w2LWq\nuleSNyV5WZJjJnlfAABg+Wy3jdCttR+31t6S0SNM79he9wUAALbNNN6edFGSp07hvgAAwFaYRmi4\nX0YHpAEAADNgu4aGqvpXSX49yde2530BAICtN+lzGhY7KXlNkocmmTvH4a2TvC8AALB8Jv3K1cMW\nqW9JbkjyV0lOaq0tFi4AAIAVZtKvXJ3GHgkAAGAZ+SUfAADomvTjST+hqu6b0WnQm1prNy3nvQAA\ngOUx8ZWGqlpTVb9XVVcmuTGjw9xuqKorh/plDSoAAMBkTfrtST+V5BNJDs1o8/P3klyT5CFJ9k7y\ntiRHVNVTW2t3TvLeAADA8pj0SsPrMnqD0l8m2b+1tndr7Umttb2T7Jvk3CS/NLQDAABmwKRDw29k\ndHDbUa21b86/0Fr7VpJnJ/nrJL854fsCAADLZNKh4WeTnNda+/FCF4f685I8csL3BQAAlsmkQ8Od\nSe6zmTa7JfnRhO8LAAAsk0mHhsuTrKuqBy50saoekGRdkv834fsCAADLZNKh4dQkD0zypap6cVU9\noqruXVUPr6oXJvnicP3UCd8XAABYJhN95Wpr7cNV9dgkv5fkvy3QpJL859bahyd5XwAAYPlM/KC1\n1tobqup/JXlxksclWZtkU5KvJPlAa+3iSd8TAABYPstyOnNr7QtJvrAcYwMAANvXRPc0VNVzq+r8\nqvqZRa7/86r6VFU9e5L3BQAAls+kN0K/JMkerbW/W+hia+1vM3pc6SUTvi8AALBMJh0aDkhyyWba\nfDnJoyd8XwAAYJlMOjTcL8kPN9PmuiQPmPB9AQCAZTLp0HBtkkdtps2jktw44fsCAADLZNKh4fNJ\nnlVV+y10sar2T3Jkks9O+L4AAMAymXRoOCmj17h+rqpeVVX7VNVuQ/nqjMLCTkM7AABgBkz6ROgv\nV9UrkvxRkncNP/PdneTlrbUvTvK+AADA8lmOE6HfX1WfS/KKJE9IskdGexi+kOS9rbUNk74nAACw\nfJbrROgNSX5nOcYGAAC2r0nvaQAAAFYZoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAu\noQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEB\nAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEBAADoEhoAAIAuoQEAAOgSGgAAgC6hAQAA\n6BIaAACArpkKDVV1/6p6SVV9pKqurKrbqmpTVX2uql5cVQt+n6o6uKo+XlXXD30ur6rXVNVOnXs9\no6ouGMa/uaq+WFXHLN+3AwCAlWnNtCewhZ6b5L1Jrkny6STfTfKgJM9O8idJnlZVz22ttbkOVXVk\nknOS3J7krCTXJ3lmknclOWQY8ydU1bFJTklyXZIzktyZZF2S9VV1QGvt+OX6ggAAsNLMWmi4Ismz\nkvxla+3Hc5VV9YYkX0rynIwCxDlD/e5J3p/k7iSHtdYuGerflOT8JOuq6ujW2pnzxto7yUkZhYuD\nWmsbh/q3JvlykuOq6pzW2sXL+k0BAGCFmKnHk1pr57fWzp0fGIb67yd53/DxsHmX1iV5YJIz5wLD\n0P72JCcMH18+dpsXJdk5yalzgWHoc0OStw8fX7Zt3wQAAGbHTIWGzfjRUN41r+7wofzEAu0vTHJr\nkoOraucl9jlvrA0AAKx6qyI0VNWaJM8fPs7/ZX/fobxivE9r7a4kV2X0iNYjltjnmiS3JNmrqnbd\nxmkDAMBMmLU9DYt5R5KfT/Lx1tpfzatfO5SbFuk3V7/HFvbZbWh3a29SVXXpIpf26/UDAICVZOZX\nGqrqVUmOS/I3SX5rytMBAIBVZ6ZXGoZXo74nydeT/HJr7fqxJnOrBWuzsLn6G8f6PGC4dl2nz2Ir\nEf+otXbgQvXDCsTjN9cfAABWgpldaaiq12R0lsLXkjxleIPSuG8M5T4L9F+T5OEZbZz+9hL7PCSj\nR5Oubq11H00CAIDVYiZDQ1X9bkaHs/3fjALDDxdpev5QHrHAtScn2TXJRa21O5bY52ljbQAAYNWb\nudAwHMz2jiSXZvRI0rWd5mcnuTbJ0VV10LwxdknyB8PH9471OS3JHUmOHQ56m+uzZ5I3DB/fFwAA\n2EHM1J6GqjomyVszOuH5s0leVVXjzTa21tYnSWvtpqp6aUbh4YKqOjOjk56fldGrVc9Octb8zq21\nq6rq9UlOTnJJVZ2V5M6MDorbK8k7nQYNAMCOZKZCQ0Z7EJJkpySvWaTNZ5Ksn/vQWvtoVR2a5I1J\nnpNklyRXJnldkpNba218gNbaKVW1McnxGZ3/cK+MNluf0Fo7fSLfBAAAZsRMhYbW2olJTtyKfp9P\n8qtb2OfcJOdu6b0AAGC1mbk9DQAAwPYlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAl\nNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQA\nAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAA\nXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1C\nAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMA\nANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQJTQAAABdQgMAANAlNAAAAF1CAwAA0CU0AAAAXUIDAADQ\nJTQAAABdQgMAANA1c6GhqtZV1SlV9dmquqmqWlWdsZk+B1fVx6vq+qq6raour6rXVNVOnT7PqKoL\nqmpTVd1cVV+sqmMm/40AAGBlWzPtCWyFE5I8JsnNSa5Osl+vcVUdmeScJLcnOSvJ9UmemeRdSQ5J\n8twF+hyb5JQk1yU5I8mdSdYlWV9VB7TWjp/UlwEAgJVu5lYakrw2yT5Jdk/y8l7Dqto9yfuT3J3k\nsNbai1trr0/y2CQXJ1lXVUeP9dk7yUkZhYuDWmuvbK29Nsmjk3wryXFV9aSJfiMAAFjBZi40tNY+\n3Vr7ZmutLaH5uiQPTHJma+2SeWPcntGKRXLP4PGiJDsnObW1tnFenxuSvH34+LKtnD4AAMycmQsN\nW+jwofzEAtcuTHJrkoOraucl9jlvrA0AAKx6s7inYUvsO5RXjF9ord1VVVcl+bkkj0iyYQl9rqmq\nW5LsVVW7ttZu7d28qi5d5FJ3HwYAAKwkq32lYe1Qblrk+lz9HlvRZ+0i1wEAYFVZ7SsNU9VaO3Ch\n+mEF4vHbeToAALBVVvtKw+ZWBebqb9yKPoutRAAAwKqy2kPDN4Zyn/ELVbUmycOT3JXk20vs85Ak\nuyW5enP7GQAAYLVY7aHh/KE8YoFrT06ya5KLWmt3LLHP08baAADAqrfaQ8PZSa5NcnRVHTRXWVW7\nJPmD4eN7x/qcluSOJMcOB73N9dkzyRuGj+9bpvkCAMCKM3MboavqqCRHDR8fPJRPqqr1w5+vba0d\nnySttZuq6qUZhYcLqurMjE56flZGr1Y9O8lZ88dvrV1VVa9PcnKSS6rqrCR3ZnRQ3F5J3tlau3i5\nvh8AAKw0Mxcakjw2yTFjdY8YfpLkO0mOn7vQWvtoVR2a5I1JnpNklyRXJnldkpMXOlm6tXZKVW0c\nxnl+RisyX09yQmvt9Il+GwAAWOFmLjS01k5McuIW9vl8kl/dwj7nJjl3S/oAAMBqtNr3NAAAANtI\naAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gA\nAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAA\nuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqE\nBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYA\nAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACg\nS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hIZFVNVeVfWBqvq7qrqjqjZW\n1buras9pzw0AALanNdOewEpUVY9MclGSn07ysSR/k+QXkrw6yRFVdUhr7bopThEAALYbKw0L++OM\nAsOrWmtHtdZ+r7V2eJJ3Jdk3ydumOjsAANiOhIYxwyrDU5NsTPJHY5ffnOSWJL9VVbtt56kBAMBU\nCA339JSh/GRr7cfzL7TW/iHJ55PsmuSJ23tiAAAwDULDPe07lFcscv2bQ7nPdpgLAABMnY3Q97R2\nKDctcn2ufo/NDVRVly5y6TEbNmzIgQceuKVzm6gNf2svN7A0B57/nmlPYcW485orpz0FYEb81Mem\n+7tekmzYsCFJ9t7WcYSG6bj7tttu23TZZZdtnPZEYMx+Q/k3U50FK85lP/jOtKcAK5n/drKway6b\n9gySUWC4aVsHERruaW4lYe0i1+fqb9zcQK216cdL2AJzq2P+twuwdP7byY7AnoZ7+sZQLrZn4VFD\nudieBwAAWFWEhnv69FA+tap+4u+nqu6b5JAktyb5wvaeGAAATIPQMKa19q0kn8zo+a9Xjl1+S5Ld\nknyotXbLdp4aAABMhT0NC3tFkouSnFxVv5xkQ5InZHSGwxVJ3jjFuQEAwHZlpWEBw2rDQUnWZxQW\njkvyyCTvSfLE1pp3lQIAsMOo1tq05wAAAKxgVhoAAIAuoQEAAOgSGgAAgC6hAQAA6BIaAACALqEB\nAADoEhoAAIAuoQFIVe1VVR+oqr+rqjuqamNVvbuq9pz23ABWoqpaV1WnVNVnq+qmqmpVdca05wXL\nZc20JwBMV1U9MslFSX46yceS/E2SX0jy6iRHVNUhTkEHuIcTkjwmyc1Jrk6y33SnA8vLSgPwxxkF\nhle11o5qrf1ea+3wJO9Ksm+St011dgAr02uT7JNk9yQvn/JcYNlVa23acwCmZFhluDLJxiSPbK39\neN61+ya5Jkkl+enW2i1TmSTACldVhyX5dJI/ba09b8rTgWVhpQF2bE8Zyk/ODwxJ0lr7hySfT7Jr\nkidu74kBACuH0AA7tn2H8opFrn9zKPfZDnMBAFYooQF2bGuHctMi1+fq99gOcwEAViihAQAA6BIa\nYMc2t5KwdpHrc/U3boe5AAArlNAAO7ZvDOViexYeNZSL7XkAAHYAQgPs2D49lE+tqp/478HwytVD\nktya5Avbe2IAwMohNMAOrLXkZz6FAAAGeElEQVT2rSSfTLJ3kleOXX5Lkt2SfMgZDQCwY3O4G+zg\nhgPeLsroVOiPJdmQ5AkZneFwRZKDW2vXTW+GACtPVR2V5Kjh44OT/EqSbyf57FB3bWvt+GnMDZaD\n0ACkqh6a5K1Jjkhy/4xOgv5Ikre01m6Y5twAVqKqOjHJmztNvtNa23v7zAaWn9AAAAB02dMAAAB0\nCQ0AAECX0AAAAHQJDQAAQJfQAAAAdAkNAABAl9AAAAB0CQ0AAECX0AAAAHQJDQAAQJfQAAAAdAkN\nAMycqnpBVbWqesG05zJnJc4JYFKEBgBSVTtV1Uur6jNVdX1V/aiqflhVl1fVn1TVs6Y9RwCmZ820\nJwDAdFXVTkn+IskRSW5M8pdJrk7yU0l+LslvJNkvyf+a1hwX8JEkX0hyzbQnArAjEBoA+LcZBYb/\nl+TQ1tqm+ReratckT5jGxBYzzHHTZhsCMBEeTwLg4KFcPx4YkqS1dmtr7dNzn6vqxOHZ/cPG21bV\n3sO19WP164f6R1TV7wyPPd1WVRdU1dHDtXctNLmq2rmqbqiqa6pqzVD3E/sHqmqXqrpxeKRqwX8Q\nq6r3Dn2eMVa/3zC/71XVnVX1g6r6s6rad5Fxfraq/scwp1uq6qKqevpCbQFWC6EBgOuGcp/tcK/3\nJPkPSb46/PnzST6a0arBbyzyC/+RSfZI8qettbsWGrS1dnuSs5I8MMnTxq9X1c5Jfj3JD5J8Yl79\nEUkuS/KbSb6c5N1JPpXk2Um+VFWPHxvnURk9FrUuycXDd7h6+A7PXspfAMAs8ngSAP8zye8meVlV\n3Tej/QKXtta+swz3enySx7XWrppfWVVnJfntjB6T+ouxPscM5embGXv9MMYxSc4du/asJHsm+S9z\nwaOq9kzy50luTfLk1trX583n5zMKB38yzHnOHyW5f5LXtNbeM6/9kRkFB4BVyUoDwA6utfaVJM/L\n6F/hn5fknCQbq+q6qvpIVT1zgrf7z+OBYTAXCI6ZX1lVD07yK0m+0lr7am/g1trFSa5I8syqut/Y\n5YWCx/MzWsF48/zAMIz1tSTvT/K4qvqXw1z2SvKvk1yV5NSx9h9L8pne/ABmmZUGANJa+3BVfSTJ\nU5L8YpLHDeVRSY6qqg8meUFrrW3jrb60yP0vqqq5X/j3bK3dMFz6zSQ7ZbSKsBSnJ3lbkqOT/HGS\nVNWD8k/B4/J5bZ80lI+pqhMXGGvuca39k3w9o7+TJPlca+3uBdpfkOTQJc4TYKYIDQAkSVprP0ry\nyeFn7lWsz0nygYz+Vf4j2fZHcL7fuTb/F/73DnXHJPlRkj9b4vgfzGjPxDEZQkNGwWNN7vl40/2H\n8qWbGfM+Q7l2KH+wSLvedwOYaR5PAmBBrbW7W2sfTjL3VqPDh/LHQ7nQPzztsblhO9c+NIx9TJJU\n1eOSHJDk4621a5c456uTnJ/kF6pqv6F6seAx96aox7TWqvNz+lj7By1y+wcvZY4As0hoAGBz/mEo\nayjnHh166AJtD9ram7TWvpfRL/xPGF53utQN0OPWD+UxVfXYJI9Ocl5r7e/H2n1hKH9pieN+ZSh/\ncViFGXfYlkwSYJYIDQA7uKr6t1X1r6vqHv+fMGxEnnt858KhnNuX8ML5r0itqocm+f1tnM76oXxx\nRofOXZt7vk1pc/5nkpsy2tT9grFx5zstoxOw31xVvzB+saruNf8simEV438neXiSY8faHhn7GYBV\nzJ4GAJ6Q5NVJvl9Vn8vo7UDJ6Jfjpye5d5KPJTk7SVprX6yqC5M8OaOzDM7P6JGdZyb5qyy8ArFU\nH8noF/7XJPlnSU4Z9losWWvttqr6HxkFj1dkdA7FXy7Q7rqqWjfc8wtV9akkf53RI1QPzWij9P2T\n7DKv2yszOp/h3VX11IxO0f7ZJL+W0WteJ/mmKYAVQ2gA4J1JvpnkX2X0KM+vZPSL8nUZvRHoz5L8\n2dibk45M8odD+TtD/3+X0Sbqf7O1E2mt3TrvF/5kyx9NmrN+GOOfJfnz1tqdi9zvU1X16CTHZ/S9\nfynJnUn+LqNHpc4Za//Nqnpikndk9Pd1WJLLM3rL1AMjNACrVG372/MAAIDVzJ4GAACgS2gAAAC6\nhAYAAKBLaAAAALqEBgAAoEtoAAAAuoQGAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoEtoAAAAuoQG\nAACgS2gAAAC6hAYAAKBLaAAAALqEBgAAoOv/AxbtNV3itBiLAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"tags":[],"image/png":{"width":390,"height":263}}}]},{"cell_type":"markdown","metadata":{"colab_type":"text","id":"-kRnCezfnMeZ"},"source":["## Creative a Naive Hypothesis\n","\n","What if we just assumed that females lived, and males died?"]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1564239493472,"user_tz":-480,"elapsed":4337,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh4.googleusercontent.com/-dGSoF3PTUms/AAAAAAAAAAI/AAAAAAAAEjo/onM3a4Ivxls/s64/photo.jpg","userId":"03413426750796061565"}},"id":"bcqRN18ZnH-y","outputId":"634f86a4-b2d0-49d2-99cd-d26dee9ffd98","colab":{"base_uri":"https://localhost:8080/","height":368}},"source":["seaborn.catplot(x='Sex', col='Survived', kind='count', data=df);"],"execution_count":12,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABakAAAK+CAYAAACozpJnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3X+0b3Vd5/HXW25KYoKa44/EQVPB\n8UeKrHEkE3LK0VRwFCcyE/zV8gcqKVajmGTZOIWZSoNrdORCWmAaGv6qRiJC8WcaGuaPgpRCTS7g\nCCg/fM8f333yeDzn3nPv/d77ud97H4+1ztr3u/dn7/353rsWa58n+7u/1d0BAAAAAIARbjZ6AgAA\nAAAA7LlEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQG\nAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagC2WVUdW1Vd\nVceOnsuSXXFOAACMtyteJ+6KcwIYQaQG2EVU1V5V9cyq+quq2lRVN1TV16rqoqp6U1UdMXqObJuq\nOrSq3jv9u143/ZseX1V7jZ4bAMC2cO26+6mqH6iqF1TVaVX1qaq6fgrozxg9N2D3t2H0BACYXeQn\neXeSRya5Ksl7klyW5OZJ7pPkSUkOSvKno+a4hrOTfDjJ5aMnsquqqiOTvCPJt5KclWRTkscmeU2S\nH0/yxHGzAwDYeq5dd1v7JPm96c9fTfKVJPuPmw6wJxGpAXYNP5fZRf7fJjmsu69evrGqbpnkwSMm\ntjnTPK/e4sA9VFXdOskbk9yU5PDu/vi0/mVJzk1yVFUd3d1nDpwmAMDWcu26e7o2yc8k+VR3X15V\nJyV5+dgpAXsKj/sA2DUcOi03rrzIT5Luvra7/3L5uqo6afr43eErx1fVAdO2jSvWb5zW372qnjd9\nHPO6qjqvqo6etr1mtQlW1S2q6sqquryqNkzrvucZelW1d1VdNX3Uc9X/EVpVp077PGbF+oOm+X15\n+mjhV6vqD6vqwDWOc4+q+uNpTtdU1Yeq6tGrjR3oqCS3T3LmUqBOku7+VpITp5fPHjExAIDt4Np1\nN7x27e7ru/t93e1Oc2CnE6kBdg1XTMt77aTzvTbJbyT59PTnDyZ5Z2Z3ljxpjYv0I5Psl+St3X3j\naged4utZmYXZR63cXlW3SPKzmX188P3L1j8yyd8k+fkkH8vsY4YfSPL4JB+tqoNXHOeemX1U86gk\nF07v4bLpPTx+PX8BO8nDp+X7V9l2fmZ3qxw6/b0AACwK166757UrwDAe9wGwa/iTJL+S5FlV9UOZ\nPS/vE939TzvofAcneWB3X7J8ZVWdleQXM/v45rtX7HPMtDx9C8feOB3jmCTnrNh2RJLbJPndpV8W\nquo2Sf4os2D7sO6+eNl87pvZBf2bpjkv+f0kt0tyfHe/dtn4IzO72F+3qjogybFbs09mdw1duo5x\nS3fSfH7lhu6+saouyey5jXdP8tmtnAMAwCiuXXfPa1eAYURqgF1Ad3+yqp6c2V0VT55+UlWbMrvj\n9s3dvfKieXv89sqL/Mnp+e5F+r9d6FfVHZP8lySf7O5Pb+7A3X1hVX0+yWOr6rbdvWnZ5tV+WXhK\nZne5HLf8In861meq6o1Jjq+q/9DdF1fVXZL8dJJLkpyyYvy7quqvkhy2uTmucEC2/ll75yW5dB3j\n9p2Waz37cGn9flt5fgCAYVy77rbXrgDDiNQAu4jufltVnZ3kJ5M8NMkDp+Xjkjyuqs5Icmx39xxO\n99E15vChZRfpt+nuK6dNP59kr8zuNFmP05O8MsnRSf5XklTVHfLdXxYuWjb2IdPyx2r25SwrLX2M\n9N5JLs7s7yVJLujum1YZf1624kK/u89LUusdDwCAa9e4dgWYK5EaYBfS3Tck+fPpJ1W1V5InJHlz\nZndtnJ2t/EjgGr6ymW3LL9JPndYdk+SGJH+4zuOfkdlzA4/JdKGf2S8LG/L9H7m83bR85haOeatp\nuXR38lfXGLe597azLd0pve8a25fWX7UT5gIAMFeuXTdrEa9dAYYRqQF2YdPdFm+rqvslOTGzL+Jb\nutD/zrRc7b/lW3p8xObuaPmDfPci/dSqemCS+yV5V3d/fZ3zvqyqzk3yU1V1UHf/fdb+ZWEp5P7Y\nirtU1rI0/g5rbL/jeua4ZAc/1+9zSQ7J7I6aT6w474Ykd0tyY5J/3MrzAwDscly7rmqRrl0BhhGp\nARbD/5uWyz/at/Rxxv1XGX/Itp6ou7+87CL9wKz/S2dW2pjkp5IcM32pzf2T/Gl3/+uKcR/O7I6b\nn0iyngv9T07Lh1bVXqt8bPLwrZznAdlxz/U7N7O7cB6Z2RfsLPewJLdMcn53f3srzw8AsCtz7fpd\ni3TtCjDMzUZPAICkqn6uqn66qr7vv8vTF78sfZzw/GWblp7N99Tprtyl8fsn+bXtnNLGafn0JD+X\n5Ov5/m9M35I/SfKNzL5I59gVx13utMwed/HyqvqPKzdW1c2q6vCl1919WZK/yOwu5ONWjD0yW/fF\nM+nu87q7tvLnvHUe/u2Z/d0dXVX/9stXVe2d5Denl6eutiMAwK7Ktetue+0KMIw7qQF2DQ9O8oIk\nX6mqCzL79u9kdjH76CQ/mORdmUXPJEl3f6Sqzs/sjtyPTneQ3CHJY5P8WVa/S2W9zs7sIv34JD+Q\n5PXTMwfXrbuvq6o/zuyXheckuSLJe1YZd0VVHTWd88NV9YEkf5fZxzr3z+zLaW6XZO9luz03yYVJ\nfq+qHpHkb5PcI8l/TXJOZn8Hw3X3N6rqmZn9u51XVWcm2ZTkiCQHTuvPGjhFAIBt4dp1N7x2TZKq\n+tUkB00vHzAtn1pVD53+fEF3v2nnzwzY3YnUALuGVyf5QmYfMbx/Zt8kvndmF8fnZfYsvD9c5dvR\nj0zyO9PyedMxfjmzL6/5b9s6me6+dtlFerL1H5dcsnE6xg8k+aPuvn6N832gqu6f5ITM3vtPJLk+\nyb9k9siMd6wY/4Wq+k9JXpXZ39nhmX3c8nFJbp9d6EK/u99ZVYcleWlmHw3dO8kXk7wwyevm9I33\nAAA7k2vX3fTaNbPH1K28u/vQ6WeJSA3MXfndGAAAAACAUTyTGgAAAACAYURqAAAAAACGEakBAAAA\nABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAA\nAACGEakHqKq3VtVbR88DAAC2xLUrAAA72obRE9hDHXTwwQcfnORJoycCAMAuqUZPYBnXrgAAbM52\nX7u6kxoAAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAA\nAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAA\nAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoA\nAAAAgGFEagAAAAAAhhGpAQAAAAAYRqQGAAAAAGAYkRoAAAAAgGFEagAAAAAAhhGpAQAAAAAYZsPo\nCTDGg158xugpALuQT/zOU0ZPAQAAANhDuZMaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABg\nGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAA\nGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAA\nAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAA\nAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAA\nAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEA\nAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoA\nAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEa\nAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEak\nBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYR\nqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBh\nRGoAAAAAAIZZ+EhdVU+uqp5+nrHGmMdU1XlVdXVVfbOqPlJVx2zhuMdU1Uen8VdP+z9mx7wLAAAA\nAIA900JH6qraP8kpSb65mTHHJTknyX2TvCXJG5PcOcnGqjp5jX1OTrIxyZ2m8W9Jcr8k50zHAwAA\nAABgDhY2UldVJTktyRVJ3rDGmAOSnJxkU5JDuvu53f1LSe6f5B+SvKiqHrJin0OTvGjafv/u/qXu\nfm6SB03HOXk6LgAAAAAA22lhI3WS5yd5eJKnJrlmjTFPS3KLJKd096VLK7v7yiS/Nb181op9ll6/\nchq3tM+lSX5/Ot5Tt3PuAAAAAABkQSN1Vd07yauSvLa7z9/M0IdPy/evsu19K8Zszz4AAAAAAGyD\nDaMnsLWqakOSP0jypSQv2cLwA6fl51du6O7Lq+qaJHepqlt297VVtU+SH0nyze6+fJXjfWFa3mud\nc/3EGpsOWs/+AACws7h2BQBglIWL1El+LckDkzy0u6/bwth9p+XVa2y/Osk+07hr1zk+SfZb31QB\nAAAAANichYrUVfXgzO6efnV3Xzh6PlvS3Q9abf10l8rBO3k6AACwJteuAACMsjDPpJ4e83FGZo/u\neNk6d1u683nfNbavvHN6veOvWuf5AQAAAADYjIWJ1EluldmzoO+d5FtV1Us/SV4+jXnjtO73ptef\nm5bf9wzpqrpTZo/6uKy7r02S7r4myT8nudW0faV7Tsvve8Y1AAAAAABbb5Ee9/HtJP9njW0HZ/ac\n6gsyC9NLjwI5N8mPJ3nksnVLHrVszHLnJvmFaZ/T1rkPAAAAAADbYGEi9fQlic9YbVtVnZRZpD69\nu9+0bNNpSX45yXFVdVp3XzqNv01mz7ZOkjesONwbMovUL62qd3b3ldM+ByR5bmaxfGW8BgAAAABg\nGyxMpN4W3X1JVb04yeuSfLyqzkpyfZKjktwlq3wBY3d/qKp+N8kLk1xUVW9PcvMkP5vktkmetxS7\nAQAAAADYPrt1pE6S7n59VV2a5IQkT8nsOdwXJzmxu09fY58XVdWnM7tz+heTfCfJ3yT5ne5+906Z\nOAAAAADAHmC3iNTdfVKSkzaz/Zwk52zlMTcm2bgd0wIAAAAAYAtuNnoCAAAAAADsuURqAAAAAACG\nEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACA\nYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAA\nYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAA\nABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAA\nAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAA\nAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYA\nAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakB\nAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURq\nAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiR\nGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhG\npAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACG\nEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACA\nYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAA\nYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAA\nABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYJiFi9RV9T+r6gNV9eWq\nuq6qNlXVJ6vq5VV1uzX2ObSq3juNva6qLqqq46tqr82c5zFVdV5VXV1V36yqj1TVMTvunQEAAAAA\n7HkWLlIn+aUk+yT5iySvTfLWJDcmOSnJRVW1//LBVXVkkvOTPCzJ2UlOSXLzJK9JcuZqJ6iq45Kc\nk+S+Sd6S5I1J7pxkY1WdPPd3BAAAAACwh9owegLb4Nbd/a2VK6vqlUlekuS/J3nOtO7WmQXmm5Ic\n3t0fn9a/LMm5SY6qqqO7+8xlxzkgyclJNiU5pLsvnda/IsnHkryoqt7R3RfuqDcIAAAAALCnWLg7\nqVcL1JO3Tct7Llt3VJLbJzlzKVAvO8aJ08tnrzjO05LcIskpS4F62ufKJL81vXzWNk0eAAAAAIDv\nsXCRejMeOy0vWrbu4dPy/auMPz/JtUkOrapbrHOf960YAwAAAADAdljEx30kSarqhCS3SrJvkkOS\nPDSzQP2qZcMOnJafX7l/d99YVZckuU+Suyf57Dr2ubyqrklyl6q6ZXdfu4U5fmKNTQdtbj8AANjZ\nXLsCADDKwkbqJCckucOy1+9Pcmx3/+uydftOy6vXOMbS+v22cp99pnGbjdQAAAAAAGzewkbq7r5j\nklTVHZIcmtkd1J+sqsd0998Mndykux+02vrpLpWDd/J0AABgTa5dAQAYZeGfSd3dX+3us5M8Isnt\nkpyxbPPS3dD7ft+O37v+qm3YZ607rQEAAAAAWKeFj9RLuvufklyc5D5V9cPT6s9Ny3utHF9VG5Lc\nLcmNSf5x2abN7XOnzB71cdmWnkcNAAAAAMCW7TaRenLnaXnTtDx3Wj5ylbEPS3LLJB/q7m8vW7+5\nfR61YgwAAAAAANthoSJ1Vd2rqr7vMRxVdbOqemWSf5dZdL5y2vT2JF9PcnRVHbJs/N5JfnN6eeqK\nw52W5NtJjquqA5btc5skL5levmH73w0AAAAAAIv2xYk/k+R/VNUFSS5JckWSOyQ5LMndk3wlyTOX\nBnf3N6rqmZnF6vOq6swkm5IckeTAaf1Zy0/Q3ZdU1YuTvC7Jx6vqrCTXJzkqyV2SvLq7L9yh7xIA\nAAAAYA+xaJH6/ya5R5KHJnlgkv2SXJPk80n+IMnrunvT8h26+51VdViSlyZ5QpK9k3wxyQun8b3y\nJN39+qq6NMkJSZ6S2R3nFyc5sbtP3zFvDQAAAABgz7NQkbq7P5PkuG3Y74OZ3YW9Nfuck+ScrT0X\nAAAAAADrt1DPpAYAAAAAYPciUgMAAAAAMIxIDQAAAADAMCI1AAAAAADDiNQAAAAAAAwjUgMAAAAA\nMIxIDQAAAADAMCI1AAAAAADDiNQAAAAAAAwjUgMAAAAAMIxIDQAAAADAMCI1AAAAAADDiNQAAAAA\nAAwjUgMAAAAAMIxIDQAAAADAMCI1AAAAAADDiNQAAAAAAAwjUgMAAAAAMIxIDQAAAADAMCI1AAAA\nAADDiNQAAAAAAAwjUgMAAAAAMIxIDQAAAADAMCI1AAAAAADDiNQAAAAAAAwjUgMAAAAAMIxIDQAA\nAADAMCI1AAAAAADDiNQAAAAAAAwjUgMAAAAAMIxIDQAAAADAMHON1FV116q69RbG/FBV3XWe5wUA\nAAAAYDHN+07qS5K8YAtjnj+NAwAAAABgDzfvSF3TDwAAAAAAbNGIZ1LfMck1A84LAAAAAMAuZsP2\nHqCqnrJi1QNWWZckeyW5a5InJ/n09p4XAAAAAIDFt92ROsnGJD39uZMcOf2stPQYkGuT/PoczgsA\nAAAAwIKbR6R+6rSsJG9O8s4k71pl3E1JrkhyYXdfNYfzAgAAAACw4LY7Unf36Ut/rqpjkryzu8/Y\n3uMCAAAAALD7m8ed1P+mu39ynscDAAAAAGD3drPREwAAAAAAYM8190hdVYdV1bur6mtVdUNV3bTK\nz43zPi8AAAAAAItnro/7qKpHZ/bFiXsl+VKSzyURpAEAAAAAWNVcI3WSk5LckOTR3f3ncz42AAAA\nAAC7mXk/7uO+Sc4SqAEAAAAAWI95R+pvJtk052MCAAAAALCbmnek/kCSh8z5mAAAAAAA7KbmHal/\nJcmPVtWJVVVzPjYAAAAAALuZeX9x4suT/F2SX0/ytKr6VJKrVhnX3f30OZ8bAAAAAIAFM+9Ifeyy\nPx8w/aymk4jUAAAAAAB7uHlH6rvN+XgAAAAAAOzG5hqpu/uf5nk8AAAAAAB2b/P+4kQAAAAAAFi3\nud5JXVV3Xe/Y7v7SPM8NAAAAAMDimfczqS/N7EsRt6R3wLkBAAAAAFgw8w7FZ2T1SL1fkgck+fdJ\nzkvi2dUAAAAAAMz9ixOPXWtbVd0sycuSPCvJMfM8LwAAAAAAi2mnfXFid3+nu389s0eCvGpnnRcA\nAAAAgF3XTovUy3woySMGnBcAAAAAgF3MiEh92yT7DDgvAAAAAAC7mJ0aqavqp5L8bJLP7MzzAgAA\nAACwa5rrFydW1bmbOc/+Se46vX7FPM8LAAAAAMBimmukTnL4Gus7yZVJ/izJyd29VswGAAAAAGAP\nMtdI3d0jnnENAAAAAMCCEpUBAAAAABhm3o/7+B5V9UNJ9ktydXd/Y0eeCwAAAACAxTP3O6mrakNV\n/WpVfTHJVUkuTXJlVX1xWr9DwzgAAAAAAItjrsG4qm6e5P1JDsvsyxK/nOTyJHdKckCSVyZ5ZFU9\noruvn+e5AQAAAABYPPO+k/qFSQ5P8p4k9+7uA7r7Id19QJIDk5yT5CemcQAAAAAA7OHmHamflOQz\nSR7X3V9YvqG7/yHJ45P8XZKfn/N5AQAAAABYQPOO1PdI8r7u/s5qG6f170vyo3M+LwAAAAAAC2je\nX2J4fZJbbWHMPklumPN5AQAAAPYIX3rF/UZPAdhF3PXXPj16CnMx7zupL0pyVFXdfrWNVfXDSY5K\n8rdzPi8AAAAAAAto3pH6lCS3T/LRqnp6Vd29qn6wqu5WVU9N8pFp+ylzPi8AAAAAAAtoro/76O63\nVdUDkvxqkv+9ypBK8tvd/bZ5nhcAAAAAgMU072dSp7tfUlV/muTpSR6YZN8kVyf5ZJI3d/eF8z4n\nAAAAAACLae6ROkm6+8NJPrwjjg0AAAAAwO5jrs+krqonVtW5VXXnNbb/SFV9oKoeP8/zAgAAAACw\nmOb9xYnPSLJfd//Lahu7+58ze/zHM+Z8XgAAAAAAFtC8I/X9knx8C2M+luT+cz4vAAAAAAALaN6R\n+rZJvraFMVck+eE5nxcAAAAAgAU070j99ST33MKYeya5as7nBQAAAABgAc07Un8wyRFVddBqG6vq\n3kmOTPLXcz4vAAAAAAALaN6R+uQkG5JcUFXPr6p7VdU+0/IFmcXpvaZxAAAAAADs4TbM82Dd/bGq\nek6S30/ymulnuZuSPLu7PzLP8wIAAAAAsJjmGqmTpLvfWFUXJHlOkgcn2S+zZ1B/OMmp3f3ZeZ8T\nAAAAAIDFNPdInSRTiH7ejjg2AAAAAAC7j3k/kxoAAAAAANZNpAYAAAAAYBiRGgAAAACAYURqAAAA\nAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAA\nAACAYURqAAAAAACGEakBAAAAABhGpAYAAAAAYBiRGgAAAACAYURqAAAAAACGEakBAAAAABhGpAYA\nAAAAYJiFitRVdbuqekZVnV1VX6yq66rq6qq6oKqeXlWrvp+qOrSq3ltVm6Z9Lqqq46tqr82c6zFV\ndd50/G9W1Ueq6pgd9+4AAAAAAPY8G0ZPYCs9McmpSS5P8pdJvpTkDkken+RNSR5VVU/s7l7aoaqO\nTPKOJN9KclaSTUkem+Q1SX58Oub3qKrjkrw+yRVJ3pLk+iRHJdlYVffr7hN21BsEAAAAANiTLFqk\n/nySI5K8p7u/s7Syql6S5KNJnpBZsH7HtP7WSd6Y5KYkh3f3x6f1L0tybpKjquro7j5z2bEOSHJy\nZjH7kO6+dFr/iiQfS/KiqnpHd1+4Q98pAAAAAMAeYKEe99Hd53b3OcsD9bT+K0neML08fNmmo5Lc\nPsmZS4F6Gv+tJCdOL5+94jRPS3KLJKcsBeppnyuT/Nb08lnb904AAAAAAEgWLFJvwQ3T8sZl6x4+\nLd+/yvjzk1yb5NCqusU693nfijEAAAAAAGyHRXvcx6qqakOSp0wvl8flA6fl51fu0903VtUlSe6T\n5O5JPruOfS6vqmuS3KWqbtnd125hXp9YY9NBm9sPAAB2NteuAACMsrvcSf2qJPdN8t7u/rNl6/ed\nllevsd/S+v22YZ9919gOAAAAAMA6Lfyd1FX1/CQvSvL3SX5h8HS+R3c/aLX1010qB+/k6QAAwJpc\nuwIAMMpC30ldVccleW2Si5O9lWJGAAAZ9UlEQVT8ZHdvWjFkS3c9L62/ahv2WetOawAAAAAA1mlh\nI3VVHZ/k9Uk+k1mg/soqwz43Le+1yv4bktwtsy9a/Md17nOnJPskuWxLz6MGAAAAAGDLFjJSV9Wv\nJHlNkk9lFqi/tsbQc6flI1fZ9rAkt0zyoe7+9jr3edSKMQAAAAAAbIeFi9RV9bLMvijxE0n+c3d/\nfTPD357k60mOrqpDlh1j7yS/Ob08dcU+pyX5dpLjquqAZfvcJslLppdv2I63AAAAAADAZKG+OLGq\njknyiiQ3JfnrJM+vqpXDLu3ujUnS3d+oqmdmFqvPq6ozk2xKckSSA6f1Zy3fubsvqaoXJ3ldko9X\n1VlJrk9yVJK7JHl1d1+4Y94hAAAAAMCeZaEidWbPkE6SvZIcv8aYv0qycelFd7+zqg5L8tIkT0iy\nd5IvJnlhktd1d688QHe/vqouTXJCkqdkdsf5xUlO7O7T5/JOAAAAAABYrEjd3SclOWkb9vtgkp/Z\nyn3OSXLO1p4LAAAAAID1W7hnUgMAAAAAsPsQqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAA\nAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAA\nAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAA\nAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEA\nAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoA\nAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEa\nAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEak\nBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYR\nqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBh\nRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABg\nGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAA\nGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAA\nAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAA\nAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAAAABgGJEaAAAAAIBhRGoAAAAAAIYRqQEAAAAAGEakBgAA\nAABgGJEaAAAAAIBhRGoA4P+3d+/RupT1fcC/P0FAiIKiLgW04AVFTRQFjXcxjRIvFCtpNInGGm29\nraAGWhMxgqldrubCRY1JtIiU0NClMda2WqxCk3iXYzQVCIkRbyBGEJB7gF//eGeHnZd9ztnnnA3P\nvnw+a80a3nlmnnnmcNazf/t75p0BAACAYYTUAAAAAAAMI6QGAAAAAGCYnUcPAAAAYEsed+zpo4cA\nrCLn/dZLRw8BgBXmTmoAAAAAAIYRUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMs+ZC6qo6qqreWVV/\nXlVXV1VX1RlbOeZJVfW/quqKqrq+qr5aVa+vqp22cMzzqurcqrqqqq6pqs9X1S+t/BUBAAAAAGxc\nO48ewHY4Lsmjk1yT5DtJHr6lnavqXyT5UJIbkpyV5Iokz09yYpInJ/nZJY55XZJ3Jrk8yRlJbkpy\nVJLTqurHu/uYlboYAAAAAICNbM3dSZ3kDUkOTHKPJK/e0o5VdY8k701yS5JndPcvd/exSR6T5LNJ\njqqqF80ds3+S384szD6ku1/b3W9I8hNJvp7kV6vqiSt6RQAAAAAAG9SaC6m7+5zu/pvu7mXsflSS\n+yT54+7+0qI+bsjsjuzk9kH3y5PsmuRd3X3xomN+mOQ/Th9ftZ3DBwAAAABgkTUXUm+jZ07rjy/R\n9mdJrkvypKradZnHfGxuHwAAAAAAdsBafCb1tnjYtL5ovqG7b66qbyR5ZJIHJblgGcdcWlXXJtmv\nqnbv7uu2dPKqOm8zTVt8jjYAANzZ1K4AAIyy3u+k3nNaX7WZ9oXte23HMXtuph0AAAAAgGVa73dS\nD9Xdj1tq+3SXymPv5OEAAMBmqV0BABhlvd9JvbW7nhe2X7kdx2zuTmsAAAAAAJZpvYfUfz2tD5xv\nqKqdkxyQ5OYkf7fMY+6fZI8k39na86gBAAAAANi69R5Sf2paH75E29OS7J7kM9194zKP+Zm5fQAA\nAAAA2AHrPaT+YJIfJHlRVR2ysLGqdkvyH6aP75k75v1Jbkzyuqraf9Ex90zy69PH37+DxgsAAAAA\nsKGsuRcnVtWRSY6cPt5vWj+xqk6b/vsH3X1MknT31VX1yszC6nOr6o+TXJHkiCQPm7aftbj/7v5G\nVR2b5JQkX6qqs5LclOSoJPsl+Z3u/uwddX0AAAAAABvJmgupkzwmyS/NbXvQtCTJN5Mcs9DQ3X9a\nVU9P8uYkL0yyW5K/TfLGJKd0d8+foLvfWVUXT/28NLM7zs9Pclx3f2BFrwYAAAAAYANbcyF1dx+f\n5PhtPObTSZ6zjcd8NMlHt+UYAAAAAAC2zXp/JjUAAAAAAKuYkBoAAAAAgGGE1AAAAAAADCOkBgAA\nAABgGCE1AAAAAADDCKkBAAAAABhGSA0AAAAAwDBCagAAAAAAhhFSAwAAAAAwjJAaAAAAAIBhhNQA\nAAAAAAwjpAYAAAAAYBghNQAAAAAAwwipAQAAAAAYZufRAwCA1eBbb/vx0UMAVpEH/sZfjR4CAABs\nGO6kBgAAAABgGCE1AAAAAADDCKkBAAAAABhGSA0AAAAAwDBCagAAAAAAhhFSAwAAAAAwjJAaAAAA\nAIBhhNQAAAAAAAwjpAYAAAAAYBghNQAAAAAAwwipAQAAAAAYRkgNAAAAAMAwQmoAAAAAAIYRUgMA\nAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QGAAAAAGAYITUAAAAAAMMIqQEAAAAAGEZIDQAAAADAMEJq\nAAAAAACGEVIDAAAAADCMkBoAAAAAgGGE1AAAAAAADCOkBgAAAABgGCE1AAAAAADDCKkBAAAAABhG\nSA0AAAAAwDBCagAAAAAAhhFSAwAAAAAwjJAaAAAAAIBhhNQAAAAAAAwjpAYAAAAAYBghNQAAAAAA\nwwipAQAAAAAYRkgNAAAAAMAwQmoAAAAAAIYRUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QGAAAA\nAGAYITUAAAAAAMMIqQEAAAAAGEZIDQAAAADAMEJqAAAAAACGEVIDAAAAADCMkBoAAAAAgGGE1AAA\nAAAADCOkBgAAAABgGCE1AAAAAADDCKkBAAAAABhGSA0AAAAAwDBCagAAAAAAhhFSAwAAAAAwjJAa\nAAAAAIBhhNQAAAAAAAwjpAYAAAAAYBghNQAAAAAAwwipAQAAAAAYRkgNAAAAAMAwQmoAAAAAAIYR\nUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QGAAAAAGAYITUAAAAAAMMIqQEAAAAAGEZIDQAAAADA\nMEJqAAAAAACGEVIDAAAAADCMkBoAAAAAgGGE1AAAAAAADCOkBgAAAABgGCE1AAAAAADDCKkBAAAA\nABhGSA0AAAAAwDBCagAAAAAAhhFSAwAAAAAwjJAaAAAAAIBhhNQAAAAAAAwjpAYAAAAAYBghNQAA\nAAAAwwipAQAAAAAYRkgNAAAAAMAwQmoAAAAAAIYRUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QG\nAAAAAGAYITUAAAAAAMMIqTejqvarqlOr6pKqurGqLq6qk6rqnqPHBgAAAACwXuw8egCrUVU9OMln\nktw3yUeSXJjk8UmOTnJ4VT25uy8fOEQAAAAAgHXBndRL+73MAupf6e4ju/tN3f3MJCcmeViStw8d\nHQAAAADAOiGknjPdRf2sJBcnefdc81uTXJvkJVW1x508NAAAAACAdUdIfXuHTeuzu/vWxQ3d/aMk\nn06ye5KfvLMHBgAAAACw3lR3jx7DqlJVv5XkmCTHdPfvLNH+riSvTfKa7n7PVvo6bzNNj77b3e62\n00EHHbTD491eF3zXI7WB2xy0796jhzDcTZeeP3oIwCqyy/0fMfT8mzZtOrO7f+HOPKfaFVgr1K5q\nV+A2o+vWZGVqVy9OvL09p/VVm2lf2L7XDpzjluuvv/6qTZs2XbwDfcBKePi0vnDoKBhu02XfHD0E\nWA3Midzm0k2jR7CaqF1ZLczTJFG7wsScyMw6qVuF1Heg7n7c6DHAlizcMeXvKoA5EfzdZ7UzTwPc\nxpzIeuOZ1Le3cKf0nptpX9h+5Z0wFgAAAACAdU1IfXt/Pa0P3Ez7Q6f1RXfCWAAAAAAA1jUh9e2d\nM62fVVX/5M+nqu6e5MlJrkvyuTt7YAAAAAAA642Qek53fz3J2Un2T/LaueYTkuyR5L9097V38tAA\nAAAAANYdL05c2muSfCbJKVX1U0kuSPKEJIdl9piPNw8cGwAAAADAulHdPXoMq1JVPSDJ25IcnmTv\nJJcm+XCSE7r7hyPHBgAAAACwXgipAQAAAAAYxjOpAQAAAAAYRkgNAAAAAMAwQmoAAAAAAIYRUgMA\nAAAAMIyQGgAAAACAYYTUwBZV1f5V1VV12uixABtDVf1KVZ1fVddP88/rR49pW1XVadPY9x89FoCN\nQt0KjKB2hZWx8+gBAAAsqKoXJTk5yZeTnJTkxiSfGzooAABYgtoVVo6QGgBYTZ63sO7uS4aOBAAA\ntkztCivE4z4AgNVknyRR5AMAsAaoXWGFCKlhDVj8fL2qenBVfbCqLq+qH1XV2VX1qGm/+1TVH1bV\npVV1Q1V9saoOm+trn6r6jar6dFV9r6puqqpLqurMqnrENo5r96r6tar6y6q6tqquqarPVtWLV/L6\ngfWvqo6vqk5y2PS5F5ZF+zx8mge/Pc1dl01z18OW6G/huXoHVNXrpucE3lBVF1fVr1dVTfv9bFV9\nYZrDvl9V76qquy3R35FVdUZVXTTte21VnTc9g3Cb6qmqesI0jy/Mwd+uqj+oqn22+Q8OYJVRtwIb\ngdpV7crK87gPWFv2T/L5JBckOW36/IIk51bVE5N8PMnVSc5Kcq8kL0rysao6sLu/NfXxtCRvSnJO\nkg8luSbJQ5McleSIqnpyd39lawOpqr2SfCrJwUk2JTk1s3/4enaSM6vqkd193I5fMrBBnDutX5bk\nnyU5YXFjVR2e5E+S3DXJR5P8bZL9kvzLJM+tqsO6e9MS/f52kmdMx5yd5Igkb0+yS1VdkeQdSf40\nyZ8n+ekkr02yU5JXz/XzjiS3ZjYHfzfJnkmemdkzCA9N8pLlXGRVvTzJH2b2vML/nuTbmc3Br0jy\n/Kr6yUXzNcBatn/UrcD6de60flnUrmpXVkZ3WyyWVb5kVtT3tLx5ru0t0/Yrkvx+krssanvJ1Hbi\nom33TXL3Jc7x6MwK/49t5tynzW0/bdr+7+a275bZLx23JnnM6D87i8WytpbMCv6e23bPJD9M8oMk\nj5hre9Q0d22a274wR12cZN9F2/ea+rk2yd8nOWhR265Jzs+sCL/vXH8PXmKsd0nygek8T9jM+fdf\ntO3AJDdl9kvKvnP7/1SSW5J8ePT/A4vFYtmRRd1qsVg20qJ2VbtaVm7xuA9YWy7O7F9EF/vAtN41\nybHdfeuitjOT3JzkMQsbuvv73f2j+Y57dhfKp5IcVlV33dIgqmrvJL+Y5Evd/Z/m+rkhyb9PUkl+\nfhnXBLA1L82sQH9rd5+/uKG7/1+S9yY5eDNf/f7N7v7uov2vzOwukN2TvKe7L1jUdmNmd/TtkuSg\nufN8fb7jab49efr47GVcx6szu5vm6MVjmvr65DSu51fV3ZfRF8Bqd3HUrcDGpHaF7eBxH7C2/GV3\n3zK3beEFDRfNF/HdfUtVXZbZ14r+UVU9N8mrkhyS5N65/Vxw7ySXbmEch2b2laKuquOXaF/4ZeGg\nJdoAttUTp/WjNzPnHDitD8rsbpLFvrTE/gvz5nlLtC0U4PPz5t5Jjk3ynCQPSrLH3HH7LtHXvIXr\neHpVHbpE+30zm1sP3MzYANYSdSuwUaldYTsIqWFtuWp+Q3ffPL1D4XZtk5tzW/Gdqjo6yUmZff3o\nE0m+leS6zL7ac2RmX5/cdSvj2HtaHzotm/NjW+kHYDkW5pxXbmW/peacpebGm5fRtnje3CvJF5Mc\nkOQLSU7P7KvqN2d2l8zR2fq8mdx2HcduZT9zJ7AeqFuBjUrtCttBSA0bSFXtnOT4JN9L8tjuvnSu\n/YlLHbeEhR+OJ3b3G1duhABLWphzHt3dXx1w/ldkVuSf0N3HL26Y5s2jl9nPwnXs2d1Xr9zwANYf\ndSuwhqldYTt4JjVsLPfO7F9OP7NEof9jSR67zH6+kNkLZp66ssMDWNLnpvWoOech0/pDS7Q9fRv6\nGX0dAGuJuhVYq0bXfGpX1iQhNWws38/sK5KPm4r7JMn0wpmTM/tlYKu6+/tJ/ijJIVX1lqraaX6f\nqnpwVR2wMsMGNrj3J7kyyVur6vHzjVV1l6p6xh14/oun9T85R1UdnOTXtqGfdyX5hyQnVtWB841V\ntUtV+SUAYEbdCqxValfYDh73ARtId99aVackeVOSv6qqj2T2JuDDktwryTnTfy/H65I8NMnbkryk\nqv4iyWVJ9snsBRCHJnlxkm+s6EUAG053X15VRyX5cJLPVdUnk3wts2eSPiCzl7rsnWS3O2gIp2f2\nLL6TquqwJH+T2fz3vCR/kuTnltNJd19YVS9PcmqSr1XVx5NclNkzBB+Y2V0qf5/k4St+BQBrjLoV\nWKvUrrB9hNSw8bwlsx8kr0jybzN7ztQnkhyX5ITldtLdV1fV05P8myQ/n+SFmf2QvSyzH4JvmPoF\n2GHd/cmq+okkxyR5dmZF8U2Zve38U1n664wrde5LprtE3pHkKdP5L0zymiT/J8ss9Ke+zqiqryT5\n1czClWcluTaz6/hgkrNWdvQAa5q6FViT1K6w7aq7R48BAAAAAIANyjOpAQAAAAAYRkgNAAAAAMAw\nQmoAAAAAAIYRUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QGAAAAAGAYITUAAAAAAMMIqQEAAAAA\nGEZIDQAAAADAMEJqAAAAAACGEVIDAAAAADCMkBqAHVZVO1XVK6vq/1bVFVX1D1X1/ar6alW9r6qO\nGD1GAABI1K4Aq1F19+gxALCGVdVOSf5HksOTXJnkfyb5TpJdkjwyyVOTbOrupwwbJAAARO0KsFrt\nPHoAAKx5L86syP9Kkqd391WLG6tq9yRPGDEwAACYo3YFWIU87gOAHfWkaX3afJGfJN19XXefM7+9\nql5cVedU1ZVVdUNVXVBVx1XVrnP7nVxVXVW/u0Qfvzy1faKq/EwDAGBr1K4Aq5BJEYAddfm0PnC5\nB1TVqUnOTPKQJB9K8u4kVyT5zSQfr6rF3/Q5NsmmJK+vqucu6uORSU5J8r0kv9jdt+7IRQAAsCGo\nXQFWIc+kBmCHVNXBST6f2SOk/ijJh5Oc193f3Mz+L0vy/mm/X+ju6xe1HZ/krUle390nL9r+kMyK\n/RuTPCbJD5N8IclBSZ7V3Z9c8QsDAGDdUbsCrE5CagB2WFX9qyQnJ7nfos1XJPmzJKd290cX7fvl\nJI9Kcp/uvnKun52SXJbk77r78XNtL0ryX6c+v57kXyd5e3cft/JXBADAeqV2BVh9hNQArIiqumuS\nw5I8JcnB03qvqfn0JC9Lcrck1yT5QZLf20xXr0xyj+6++xLneG+SV0wf/yLJM7r7lhW6BAAANgi1\nK8DqIqQG4A4x3VnywiSnJtkjyQuSfDHJd5ZzfHfXEn0+O8nHp48/193/bWVGCwDARqZ2BRjLixMB\nuEN09y1TIX7itOmZSRbeoP7l7q4tLfP9VdW9k/znJNdNy4lVdZ8741oAAFjf1K4AYwmpAbij/Wha\nV3dfk+RrSR5ZVfdabgdVVUk+kGTfJEdPyz5JTp/aAABgJahdAQYQUgOwQ6rqxVX101V1u58pVXW/\nzJ7Tl8xeGpMkv5tklySnVtVeSxxzz6p67NzmNyZ5TpKzuvt93f2+JGclOTzJsSt0KQAArHNqV4DV\nyTOpAdghVXVSZneHfC+zF8J8Y2o6IMlzM3vhzEeSvKCnHzpV9e4kr8nsLer/O8m3ktxrOuZpSd7f\n3a+a9j00yaczex7gwd191bT9Hkm+nOSBSZ7a3Z+7wy8WAIA1Te0KsDoJqQHYIVX1gCRHJPnnSR6R\n5P5JdktyeWaF+JlJzuzuW+eOe16SVyV5fGZvUr8is4L/7CRndPeFVbXn1Md+SZ7S3V+Y6+OQzH4J\nuCSzXwKuvKOuEwCAtU/tCrA6CakBAAAAABjGM6kBAAAAABhGSA0AAAAAwDBCagAAAAAAhhFSAwAA\nAAAwjJAaAAAAAIBhhNQAAAAAAAwjpAYAAAAAYBghNQAAAAAAwwipAQAAAAAYRkgNAAAAAMAwQmoA\nAAAAAIYRUgMAAAAAMIyQGgAAAACAYYTUAAAAAAAMI6QGAAAAAGAYITUAAAAAAMMIqQEAAAAAGOb/\nA+qJC/RLNZUNAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"tags":[],"image/png":{"width":724,"height":351}}}]}]} -------------------------------------------------------------------------------- /labs/exploring_attention.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"exploring_attention.ipynb","provenance":[],"collapsed_sections":["64Sbg_A_dcFn"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"64Sbg_A_dcFn","colab_type":"text"},"source":["## Setup"]},{"cell_type":"code","metadata":{"id":"16C1jXJGdaMi","colab_type":"code","colab":{}},"source":["!pip install tensorflow>=2.0 -Uq"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"PWi4tGjHh0Rj","colab_type":"code","colab":{}},"source":["import tensorflow"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"SSYyg4Usdibk","colab_type":"code","colab":{}},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","%matplotlib inline\n","%config InlineBackend.figure_format = \"retina\"\n","\n","import tensorflow.compat.v2 as tf"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bn447DUtdkK1","colab_type":"text"},"source":["Helper functions for prettier displays"]},{"cell_type":"code","metadata":{"id":"TKpheNmkdhAg","colab_type":"code","colab":{}},"source":["def plot_weights(array):\n"," \"\"\"\n"," Plot attention weights\n"," \"\"\"\n"," try:\n"," array = array.numpy()\n"," except:\n"," pass\n"," sns.heatmap(array[0,:].T)\n"," plt.title(\"Attention Map\")\n"," plt.xlabel(\"Query\")\n"," plt.ylabel(\"Key\")\n"," plt.show()\n"," \n","def print_outputs(output):\n"," \"\"\"\n"," Print value vector (V) or self-attention output\n"," \"\"\"\n"," try:\n"," output = output.numpy()\n"," except:\n"," pass\n"," print(\"Output:\")\n"," print(str(output[0]))\n","\n","def causal_attention_mask(size):\n"," mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n"," return mask"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PDH_E3CedpuO","colab_type":"text"},"source":["# Exploring Self-Attention\n","\n","In this notebook, we take a deep dive into the self-attention mechanism and see what it is doing.\n","\n","We will explore an implementation of attention in TensorFlow 2.0 and some toy examples of self-attention.\n","\n","![Self-attention mechanism](https://github.com/NVAITC/dnn_animations/raw/master/gif/self_attention.gif)"]},{"cell_type":"markdown","metadata":{"id":"IFj-0EcIor5d","colab_type":"text"},"source":["## Attention Mechanism"]},{"cell_type":"markdown","metadata":{"id":"YRFeU0aDmmjO","colab_type":"text"},"source":["The attention mechanism takes in three vectors (query `Q`, key `K` and value `V`) and returns the product of `V` and an attention map.\n","\n","We show a simple implementation of the **dot-product attention** mechanism as shown by the equation below. There are other methods of calculating attention scores, but dot-product is by far the most common method.\n","\n","Note the optional the scaling of the dot-product by $\\sqrt{H}$ (square-root of model dimension) in order to prevent the value of the dot-product from being too large. This is a problem for high dimensions.\n","\n","$\\text{attention}(Q,K,V) = \\text{softmax}(\\frac{QK^T}{\\sqrt{H}})V $\n","\n","In addition to returning the output, we will also return the **attention map** so that we can visualize it and understand how attention works."]},{"cell_type":"code","metadata":{"id":"vRfrnhdFdq_o","colab_type":"code","colab":{}},"source":["def attention(query, key, value, mask, scale=False):\n"," matmul_qk = tf.matmul(query, key, transpose_b=True)\n"," if scale:\n"," H = tf.cast(tf.shape(key)[-1], matmul_qk.dtype)\n"," matmul_qk = matmul_qk / tf.math.sqrt(H)\n"," # add the mask to zero out padding tokens\n"," mask = tf.cast(mask, matmul_qk.dtype)\n"," matmul_qk += (mask * -1e9)\n"," # softmax is normalized on the last axis (seq_len)\n"," attention_weights = tf.nn.softmax(matmul_qk, axis=-1)\n"," output = tf.matmul(attention_weights, value)\n"," return output, attention_weights"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"7wR2o2BTWiMi","colab_type":"text"},"source":["You may have noticed a `mask` in the implementation above.\n","\n","Sometimes, it is not desirable to allow the attention mechanism to attend to all positions. Hence, we use can use an **attention mask** to prevent attention to specific positions. In positions where the mask has a value of `1`, the attention weight will be set to effectively zero."]},{"cell_type":"markdown","metadata":{"id":"NZU0DEsbeBbo","colab_type":"text"},"source":["## `K`, `Q` and `V` vectors"]},{"cell_type":"markdown","metadata":{"id":"hdmaBfeofsiQ","colab_type":"text"},"source":["The **query** vector, `Q`, is a representation of the information that is relevant to the current token.\n","\n","During the training process, the model learns to transform the input token into the `Q` vector via a fully-connected (dense) layer.\n","\n","Below, we create a toy example of what the `Q` matrix might look like for a sequence of five tokens."]},{"cell_type":"code","metadata":{"id":"VcCJ_6_EgOVl","colab_type":"code","colab":{}},"source":["Q = tf.constant([[[0,10,0],\n"," [0,10,0],\n"," [0,10,0],\n"," [10,0,0],\n"," [0,0,10]]], dtype=tf.float32)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Zbtt5knzgMyN","colab_type":"text"},"source":["The **key** vector, `K`, is a representation of the type of information that is contained inside the current token. \n","\n","Same as the `Q` vector, this representation is learnt during the training process."]},{"cell_type":"code","metadata":{"id":"P5vkiR_Jebrx","colab_type":"code","colab":{}},"source":["K = tf.constant([[[10,0,0],\n"," [0,10,0],\n"," [0,0,10],\n"," [0,0,10],\n"," [0,0,10]]], dtype=tf.float32)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"k6mKIYoigSre","colab_type":"code","outputId":"9777cf6e-41de-4a2b-d996-6a642ea30e29","executionInfo":{"status":"ok","timestamp":1577283160885,"user_tz":-480,"elapsed":6103,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["print(\"K and Q have the same shape:\", Q.shape)"],"execution_count":8,"outputs":[{"output_type":"stream","text":["K and Q have the same shape: (1, 5, 3)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"_OM7sbjFeM9_","colab_type":"text"},"source":["The **value** vector, `V`, represents the semantic meaning encoded in the input token. \n","\n","Here, we are using a sequence of integer values to represent the input sequence.\n","\n","The values reflect the indices of the individual elements (element 0 has value `0.0`).\n","\n","We can use this as a proxy to identify which elements of the sequence are attended to later."]},{"cell_type":"code","metadata":{"id":"P-yy_e4Vdx5Z","colab_type":"code","outputId":"4e7e2375-5de4-4c5b-bd6f-645303f2ff70","executionInfo":{"status":"ok","timestamp":1577283160886,"user_tz":-480,"elapsed":6092,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["V = np.arange(int(K.shape[1]), dtype=\"float32\").reshape((1,K.shape[1],1))\n","\n","print(\"Sequence:\", V.tolist())\n","print(\"Sequence length:\", V.shape[1])\n","print(\"Sequence shape:\", V.shape)"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Sequence: [[[0.0], [1.0], [2.0], [3.0], [4.0]]]\n","Sequence length: 5\n","Sequence shape: (1, 5, 1)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"_hH_MWBDhKvS","colab_type":"text"},"source":["## Performing Self-Attention"]},{"cell_type":"code","metadata":{"id":"ldp-YdUEpyh2","colab_type":"code","colab":{}},"source":["# causal attention mask - explained later\n","causal_mask = causal_attention_mask(K.shape[1])\n","# for now we will use no mask (no positions are masked)\n","no_mask = np.zeros_like(causal_mask)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ikqbf8J0hKIB","colab_type":"code","outputId":"9fdd9023-ebd5-4eec-8a17-9082ee7741ef","executionInfo":{"status":"ok","timestamp":1577283160887,"user_tz":-480,"elapsed":6082,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["output, attn_w = attention(Q, K, V, no_mask)\n","\n","print(\"Output shape:\", output.shape)\n","print(\"Attention Map shape:\", attn_w.shape)"],"execution_count":11,"outputs":[{"output_type":"stream","text":["Output shape: (1, 5, 1)\n","Attention Map shape: (1, 5, 5)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"VMlQJzfUjOtx","colab_type":"text"},"source":["The shape of `output` is same as the input `V`, while the attention map has shape `(seq_len, seq_len)`."]},{"cell_type":"markdown","metadata":{"id":"MqxERduUiSoz","colab_type":"text"},"source":["### Interpreting the Attention Map"]},{"cell_type":"markdown","metadata":{"id":"wuxHU2hSjnxJ","colab_type":"text"},"source":["We can visualize the attention map to inteprete what it is trying to do."]},{"cell_type":"code","metadata":{"id":"NoM2fqOghkG_","colab_type":"code","outputId":"71fad678-bb6a-43e8-8cd2-1d9b609a9557","executionInfo":{"status":"ok","timestamp":1577283161583,"user_tz":-480,"elapsed":6769,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":294}},"source":["plot_weights(attn_w)"],"execution_count":12,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAtUAAAIqCAYAAADrd7anAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de5x1ZVn/8c9XBEWwBzBRgRJFDpr+\nQiFF8QCiSGpGeChLVOxkaKhpWZoinitTwNL8qYhnzUzJn6WY4gGIDuQp4ySIInhEQAREYK7fH2uN\nDMPsmXnm3vPsvZ/1efvar/Xste59r2uYR7jmmuu+V6oKSZIkSWt3i0kHIEmSJM06k2pJkiSpkUm1\nJEmS1MikWpIkSWpkUi1JkiQ1MqmWJEmSGplUS5IkSY1MqiVJkqRGJtWSJElSI5NqSZIkqZFJtSRJ\nktTIpFqSJElqZFItSZIkNTKplqSNlKT6166TjkWSNB1MqiUtKcmhC5LHT6xi/N5JXpLkqSuMe3Y/\nbtcxhTo2SQ7oYzt00rGMU5JPL/hefj/JliuMf9aC8ZXkgE0UqiTNLJNqSaM8ZcGfH5pk5xXG7w0c\nDTx1hXHP7sftuubI1s8BdLGtlFSf07+uW++A1sHtgEetMOYpK1yXJC1iUi3pZpL8LF3idRXwHrp/\nVxw+0aCmSFXt1b8unnQsG+kb/fHJowYk+QXg3sDXN0lEkrSZMKmWtJQnAlsC/wS8qT9n9XL2nQRc\nCTwqyQ4jxsx/n9+1aUKSpM2DSbWkpcwnVu8GPkdX4dwryX2XGpykgLf1bx+yqB+3FvQqF3Dnftwp\ni8Z8eol5t03ygiT/meSKJD9Ocl6S45P83IhY5vuHn5pk6/6+5yS5Jsl3k7wvye6LPrNrH9vR81//\nEl/Drgu/3uUWKibZLcmbklzQx3xZks8m+Z0kW4wr7jW4GvggsBXw60vEcAvgt/q371xuoiT3SfLq\nJKcm+UaSa5Nc2n8dy32dL+m/zhOT3CLJc5J8MclV/ef/adTfM0maZibVkm6i//X/PsClwMlVVcB7\n+8ujqtXfAX7Y//m6/v3C10+AH/V/nuvHXbZozA8WxXF34H+AVwD7ArcBrgfuBvwh8MUk+y/zpfwM\ncBpdonxnoIDb0yWTZyTZbcHYG/oYrurf/3iJr+GGZe61MO5H93H/HnCXfq5tgAcBbwY+lmSbMcW9\nFvPJ8lItIA8DdgJOB85fYZ6TgecD+9P1aV8N7AA8hO7r/Kckt1zm8wH+AXgtcA+6vzc7AL8CnJ7k\nZkm/JE0zk2pJi80nzn9fVfML8d7dH38jyVaLP1BVdwSe1b89varuuOh1elW9ph93UT/usEVjDpuf\nL8kG4J/pksoPAL8I3LqqtgV2o+vz3h74YJLtRnwdx/RjDqFLarcFHgx8ky55e9WC+C/qY3tNf+r9\nS3wNF7GCPuF9H3Br4DPAXlW1HXBb4PeBa+kS1+OWmWbVca/RKXTfg/2WqHzPJ9rvWMU8J9O1Cd2p\nqrapqu37WA8Hvg08EnjOMp//VeAxwB8BP9P/c7ob8AlgC+BtY/gBQpI2GZNqST/V/8r+Sf3b98yf\nr6ovA1/mxkrievtjut1B3ltVT6iqL1XVDX0sF1TVbwEfA+4A/M6IOW4FPLyqPl5VN1TVXFV9jm73\nEYDHLPUDQqMX0CXC5wOPrKpz+pivrar/CxzVj3takrtNIu7+Nw/z/dI/XXyaZFvg1+gS//evYp7f\nrKr3VdW3F5y7qqreBTyhP3XkMlNsAI6uqtdV1TX958+nS7TPAbYG/mzVX5gkTZhJtaSFHg7ciW7n\nh9MWXZuvVm+KBYvz9/jrZcbMJ/0PH3H9H6rqq0uc/ye6lopb0VVGxyJJgMf2b19XVVcvMewtwMV0\nrQ+PGzHVpoh7vhL9pD5u+nhuA3ykqi5vmbz/IeByYNckO40YdjVw7BKf/TE3ft8fuyA+SZpqJtWS\nFnpqf3xvX9Fc6L10Sd0vJ7n9egXQL0DcpX/7z0m+vdSLG1sollywCPznUif7lpbv9m+3H1vgcFe6\n6it0LRZL3XsO+HT/9j4j5ln3uKvq7P4+d6Hr9YaNa/0AIMnjk3y4X6h4zcKFncB8W86opPq/quqq\nEdc+0x+362OUpKm33CISSQPS9zH/av/2PYuvV9U3knyOrr/3N1m+L7jFnRb8ecdVjL/NiPNXLvOZ\nH/fHZZ8suJEW/qCx3P7V31xi/EKbKu53AL8EHJ7ka3QPvvke8C8rfbBfgPj3dO0i864Fvs+NCzpv\nT1e4GbUoc7l/Rguv3R64YKWYJGnSrFRLmvfrdAvsAL60xJZyRZdQw/q2gCz899L2VZUVXruuYyxr\ndeuVh0zc++h23Hg88Lt0LSnvrarrV/HZ36VLqK+m6xP/uaq6dVXdfn5hJ3BJP9b2DUmDYFItad7G\nJMr3TnKvdYrjOwv+/PPrdI/18L0Ff14u7vnWlu8tM2bdVdX36arSG4A/7U8vuzf1Ao/vjy+rqtdX\n1TcXXuwXvP7sCnOMagtZfG2i/5wkabVMqiXRb632gP7t3nQ9u6NeH+nHLU7C5/efXqkyuey4qvoa\nNybWv7yK8MdptV/DUi6gW5wHcOBSA/qHqxzQv/3vNdxj3Ob7p7cE/req/muVn5v/weDzI67vz8rV\n+n2TjGrdeUh/vBz42ipjkqSJMqmWBDcuUvtiVX2xqi4f9aLbNxrgtxY9NW/+4S+j9o3emHEn9sfn\nJdl51KB0Vrrfxljt13Az/cLOf+zfPmtEwvg7wM50Cz4/sMT1Te0jwF/R7bbxpyuMXeiK/niz31b0\n/dYvX8Uc23Dj3uYLP38rur2rodsJZfGCWUmaSibV0sD1W5bN71f8j8uN7X2Erhf3jsAjFpz/Sn+8\nR5L7LfP5+XFPTDKqmvlqusrvz9I9Xe8JSbZeEPPPJ/k9umrvoauIebXmY3vgGh8J/kq6pzLuBHw0\nyZ7QJYpJfhc4vh/31n5P5omqqp9U1Z9U1fOq6iMrf+KnPtEfX5TkV+d/uEqyF93fj/ty49MpR7kC\neFmSZ81/b5PcFTgJuDvdosxXb0RMkjRRJtWSDqB7ciHAB1ca3FerP9W/fcqC8+cBn6XbVeiMJJcm\nubB/7bdgirf2x8cDVyS5qB/zvkX3eARwFl1/8vuBK5N8P8nVdPtov4muVWWclcxP0z24ZQfgnCTf\nXfA17LL8R3/68JIn0iWEBwBnJ7mMbkeP/0u3x/QnufFBLrPqNXT/nH4G+DBwTZIr6L5fDweeTrcT\nyHJOott7+1i6vweX9XM+gm4HkSOm4QcPSVotk2pJ84nxuVX1lWVH3mg++X7MovaLw4A30PXBbkuX\nrN+ZBf21VfUpup0jPgNcQ9cOcWe6yjcLxn0VuDfdU/lOAS6jW1R3PfAluiT1Udz4dMBm/V7QB9Et\n2LuYrod8/mtY1RakfcX3XsCbgQvptvy7GjgV+D3gEcvszzwTquoHwH7AG7lxi8Br6BLsh1TViauZ\nhu4Hqz+iS8a3ovse/z/gAVX1vmU+K0lTJ7arSZI2lSQvAY4G3l5VT51sNJI0PlaqJUmSpEYm1ZIk\nSVIjk2pJkiSpkUm1JEmS1MiFipIkSVIjK9WSJEmaOkkel+T1ST6X5IdJKsmatlFNskuSE5JckuTa\n/vkDxybZflzxrmrfVUmSJGkT+3PgF4Ef0e2Jv9daJkmyG3A6sCPdg6fOpnvy67OAQ5LsX1WXtgZr\npVqSJEnT6DnAHnRPb/2DhnneQJdQH1VVh1bVn1bVQ4HXAXsCr2iOFHuqJUmSNOWSHED3dN13V9WT\nNuJzuwFfpXvC7W5VNbfg2m2BbwEBdmx92q2VakmSJG2uDuyPJy9MqAGq6krgNOA2wH6tN7KnWpIk\nSWOX5MxR16pqn00Uxp798dwR188DDqZrM/lky41MqlfpllvtbJ+MpMG65pLPTToErdLWOz1o0iFo\nI1z/k4sz6RgArvv+BeuR5/z3Osy5sTb0xytGXJ8/v13rjUyqJUmSNHabsBo9FUyqJUmShm7uhklH\nsF7mK9EbRlyfP395641cqChJkqTN1Tn9cY8R13fvj6N6rlfNSrUkSdLQ3XRjjM3JKf3x4CS3WGJL\nvf2Bq4EzWm9kpVqSJGno5ubG/9qEkmyZZK9+X+qfqqrzgZOBXYFnLPrYMcA2wDtb96gGK9WSJEma\nQkkOBQ7t396xP94/yYn9n79fVc/r/7wzcBbwdboEeqEj6R5TfnySg/px96Pbw/pc4IXjiNekWpIk\naeBqOts/9gaesujcXfsXdAn081hBVZ2fZF/gpcAhwCPpnqR4HHBMVV02jmB9TPkquU+1pCFzn+rZ\n4T7Vs2Va9qn+ySVfGXues9VOvzAVX9umYqVakiRp6DZxD/TmyKRakiRp6Kaz/WOmuPuHJEmS1MhK\ntSRJ0tBtvk9U3GSsVEuSJEmNrFRLkiQNnT3VzaxUS5IkSY2sVEuSJA2dW+o1M6mWJEkauCl9ouJM\nsf1DkiRJamSlWpIkaehs/2hmpVqSJElqZKVakiRp6OypbmZSLUmSNHQ+UbGZ7R+SJElSIyvVkiRJ\nQ2f7RzMr1ZIkSVIjK9WSJElD55Z6zUyqJUmShs72j2a2f0iSJEmNrFRLkiQNne0fzaxUS5IkSY2s\nVEuSJA1clQ9/aWVSLUmSNHQuVGxm+4ckSZLUyEq1JEnS0LlQsZmVakmSJKmRlWpJkqShs6e6mZVq\nSZIkqZGVakmSpKGbc0u9VjObVCfZGtgP2APYrj99OXAucEZVXTOp2CRJkmaK7R/NZi6pTrI98Arg\ncOA2I4ZdneQdwJ9X1WWbLDhJkiQN0kwl1Um2A04D9gKuAj4BnAdc0Q/ZAOwO7A/8AXBgkvtX1RVL\nTCdJkiRwS70xmKmkGjiaLqF+HXB0Vf1oqUFJtgVeCjwbeDHw3NVMnuTMUde22HKnjQ5WkiRJwzBr\nu38cCnyqqp47KqEGqKofVdUfAZ8GDttUwUmSJM2kmhv/a2BmrVJ9J+C9GzH+DOABqx1cVfuMunbL\nrXaujbivJEnS7LD9o9msVaovBfbciPF37z8jSZIkrZtZS6o/Dhya5MiVBiZ5JvAY4GPrHpUkSdIs\nm5sb/2tgZq3940XAo4DXJ3kucDLdvtQLd//YAzgY2BX4Lt1CRUmSJGndzFRSXVUXJ7k/8Ebg4cDv\nA4t7ndMfTwaOrKqLN2GIkiRJM6fKJyq2mqmkGqCqLgAekeSuwIF0PdYb+stXAOcAp/TjJEmStJIB\ntmuM28wl1fP6pNnEWZIkSRM3s0m1JEmSxmSA+0qP26zt/iFJkiRNHSvVkiRJQ2dPdTMr1ZIkSVIj\nK9WSJElDZ091M5NqSZKkobP9o5ntH5IkSVIjK9WSJElDZ/tHMyvVkiRJUiMr1ZIkSUNnT3Uzk2pJ\nkqShM6luZvuHJEmS1MhKtSRJ0tC5ULGZlWpJkiSpkZVqSZKkobOnuplJtSRJ0tDZ/tHM9g9JkiSp\nkZVqSZKkobP9o5mVakmSJKmRlWpJkqShs6e6mZVqSZIkqZGVakmSpKGzp7qZSbUkSdLQmVQ3s/1D\nkiRJamSlWpIkaeiqJh3BzLNSLUmSJDWyUi1JkjR09lQ3M6mWJEkaOpPqZrZ/SJIkSY2sVEuSJA2d\nT1RsZqVakiRJUyvJLklOSHJJkmuTXJjk2CTbb+Q8D0xyUv/5Hyf5RpJ/TnLIOOK0Ui1JkjR0U9pT\nnWQ34HRgR+Ak4GzgvsCzgEOS7F9Vl65inj8A3gBcBXwI+CawC3AY8MtJ/ryqXtESq0m1JEnS0E3v\nPtVvoEuoj6qq18+fTPJa4DnAK4CnLzdBki2BVwE/BvapqnMWXHsl8HnghUleU1XXrjVQ2z8kSZI0\ndfoq9cHAhcDfLrp8NF3V+fAk26ww1Q7ABuDchQk1QFWdBZwLbA1s2xKvSbUkSdLQzc2N/9XuwP54\nctVNV1JW1ZXAacBtgP1WmOe7wPeAPZLsvvBCkj2A3YEvrKaNZDm2f6zSNZd8btIhSNLEbL3TgyYd\ngqQZk+TMUdeqap9VTLFnfzx3xPXz6CrZewCfXOZeleQZwLuAM5N8CLgE2Bn4NeArwG+sIp5lmVRL\nkiQN3XQuVNzQH68YcX3+/HYrTVRVH0hyCfBe4MkLLn0HeBtwwVqDnGdSLUmSpLFbZTV6k0jyJODN\nwD8CLwO+DtwZeBHwN8BDgCe03MOkWpIkaeim8+Ev85XoDSOuz5+/fLlJ+r7pE4AvAYcv6M8+O8nh\ndG0mj09yQFV9eq3BulBRkiRp4Gquxv4ag/mdOvYYcX1+0eGonut5BwNbAp9ZYsHjHPDZ/m1TZd2k\nWpIkSdPolP54cJKb5KxJbgvsD1wNnLHCPLfqj7cfcX3+/E/WEuQ8k2pJkqShm8It9arqfOBkYFfg\nGYsuHwNsA7yzqq6aP5lkryR7LRo7v4Xb45L8n4UXkuwNPA4o4FMt8dpTLUmSpGl1JN1jyo9PchBw\nFnA/uj2szwVeuGj8Wf0x8yeq6j+SvA04AvjPfku9r9Ml64cCWwHHVtVXWgI1qZYkSRq66VyoSFWd\nn2Rf4KXAIcAjgW8BxwHHVNVlq5zqt+l6p58KPAK4LfBD4FTgzVX1vtZYTaolSZKGbjwLC9dFVV1E\nV2VezdiMOF/Aif1rXdhTLUmSJDWyUi1JkjR00/lExZlipVqSJElqZKVakiRp6KxUNzOpliRJGrqa\n3oWKs8L2D0mSJKmRlWpJkqShs/2jmZVqSZIkqZGVakmSpKGb4oe/zAor1ZIkSVIjK9WSJElDV/ZU\ntzKpliRJGjrbP5rZ/iFJkiQ1slItSZI0cOWWes2sVEuSJEmNrFRLkiQNnT3VzUyqJUmShs7dP5rZ\n/iFJkiQ1slItSZI0dLZ/NLNSLUmSJDWyUi1JkjR0bqnXzKRakiRp6Gz/aGb7hyRJktTISrUkSdLQ\nuaVeMyvVkiRJUiMr1ZIkSUNnT3UzK9WSJElSIyvVkiRJA1duqdfMpFqSJGnobP9oZvuHJEmS1MhK\ntSRJ0tBZqW5mpVqSJElqtNkn1Un+Ksn5k45DkiRpatXc+F8DM4T2j58Fdl3NwCRnjrr2k++Zl0uS\npM2U7R/NNvtKtSRJkrTeZq5SneQdG/mRB6x2YFXtM+radd+/wB/hJEnSZqmsVDebuaQaeBJQQDbi\nM/5NkSRJ0rqZxaT6SuCbwJGrHP+nwMHrF44kSdKMs1LdbBaT6i8Cv1hVn1nN4CRPXd9wJEmSZpyP\nKW82iwsVvwBsm2S3SQciSZIkwWxWqj8DPAjYBVjNPncfBi5cz4AkSZJmmu0fzWYuqa6qDwIf3Ijx\nJwEnrV9EkiRJGrqZS6olSZI0Zlaqm81iT7UkSZI0VaxUS5IkDVyVlepWJtWSJElDZ/tHM9s/JEmS\npEZWqiVJkobOSnUzK9WSJElSIyvVkiRJA1dWqpuZVEuSJA2dSXUz2z8kSZKkRlaqJUmShm5u0gHM\nPivVkiRJUiMr1ZIkSQPnQsV2JtWSJElDZ1LdzPYPSZIkqZGVakmSpKFzoWIzK9WSJElSIyvVkiRJ\nA+dCxXZWqiVJkqRGVqolSZKGzp7qZibVkiRJA2f7RzvbPyRJkqRGVqolSZKGzvaPZlaqJUmSpEZW\nqiVJkgaurFQ3M6mWJEkaOpPqZrZ/SJIkSY2sVEuSJA2c7R/trFRLkiRJjaxUS5IkDZ2V6mZWqiVJ\nkgau5sb/GpckuyQ5IcklSa5NcmGSY5Nsv4a57pPkPUm+2c/1nSSfSfLk1jitVEuSJGkqJdkNOB3Y\nETgJOBu4L/As4JAk+1fVpauc65nAccBlwEeBi4EdgHsCjwTe0RKrSbUkSdLATfFCxTfQJdRHVdXr\n508meS3wHOAVwNNXmiTJwcDxwCeAx1XVlYuub9kaqO0fkiRJmjp9lfpg4ELgbxddPhq4Cjg8yTar\nmO6vgGuA31ycUANU1XVt0VqpliRJGrwprVQf2B9PrrpphFV1ZZLT6JLu/YBPjpokyT2B/wN8GPhB\nkgOBfYACvgCcsnj+tTCpXqWtd3rQpEOQJGlFx9zpgEmHoFlUGfuUSc4cebuqfVYxxZ798dwR18+j\nS6r3YJmkGvil/vhd4NPAgxdd/3KSw6rqq6uIaSTbPyRJkjSNNvTHK0Zcnz+/3Qrz7NgffxvYFXhU\nP/cewLuAewEfTbLVmiPFSrUkSdLgrUf7xyqr0ZvCfBF5C+A3qurf+vc/7LfS2wvYF3gs8N7Wm0iS\nJEnTZL4SvWHE9fnzl68wz/z1by9IqAGoqqLbqg+6rfrWzEq1JEnSwNXc+Huqx+Cc/rjHiOu798dR\nPdeL5xmVfF/WH7deZVxLslItSZKkaXRKfzw4yU1y1iS3BfYHrgbOWGGeM+i239t1xPZ79+yPX2uI\n1aRakiRp6KbxMeVVdT5wMt3iwmcsunwMsA3wzqq6av5kkr2S7LVonquBtwK3Bl6eJAvG3wt4KnA9\n8A8t8dr+IUmSNHC1DlvqjcmRdI8pPz7JQcBZwP3o9rA+F3jhovFn9cfFX9CL6LbSezZw/36P6zsA\nh9El28/uk/g1s1ItSZKkqdQnuvsCJ9Il088FdgOOA/arqktXOc8PgQcBrwR2AJ4JPBo4FXhEVR3X\nGquVakmSpIGb0icqAlBVFwFHrHLsyJJ7Vf2IrrK9uLo9FlaqJUmSpEZWqiVJkgZuSrfUmykm1ZIk\nSQNXNekIZp/tH5IkSVIjK9WSJEkDZ/tHOyvVkiRJUiMr1ZIkSQNnpbqdSbUkSdLAuVCxne0fkiRJ\nUiMr1ZIkSQNn+0c7K9WSJElSIyvVkiRJA1dlpbqVlWpJkiSpkZVqSZKkgau5SUcw+0yqJUmSBm7O\n9o9mtn9IkiRJjaxUS5IkDZwLFdtZqZYkSZIaWamWJEkaOB/+0s6kWpIkaeCqJh3B7Btr+0eS1ybZ\nY5xzSpIkSdNu3D3VzwbOSvKpJL+eZMsxzy9JkqQxq7mM/TU0406qnwh8BngI8B7gm0lenWS3Md9H\nkiRJmhpjTaqr6v1V9VBgT+C1/ek/Ac5JcnKSw5JsMc57SpIkqc1cZeyvoVmXLfWq6qtV9cfALnTV\n688CBwEfAC5K8vIkd16Pe0uSJGnjVGXsr6FZ132qq+q6BdXrBwCXAHcA/gw4P8mHk+y9njFIkiRJ\n623dH/6SZP8k7wBOAXYGfgD8LXAG8BjgP5I8br3jkCRJ0tKqxv8amnXZpzrJBuDJwO8B9wAC/Dvw\nRuD9VXVtP+4BdC0hxwD/sB6xSJIkSettrEl1kvvTJdJPALYGrgbeCryxqj6/eHxVnZ7kbXSLGSVJ\nkjQBQ1xYOG7jrlSf1h/PBv4OOLGqfrjCZy4GvjPmOCRJkqRNZtxJ9T/QVaVPWe0HquqNdG0hkiRJ\nmoAh7tYxbmNNqqvqCeOcb5QkvwYcAFwPfKyqPjFi3FOAp/S7j0iSJGkJQ1xYOG7rslARIMndgLsD\n21bVe8c0Z4D3A4+lW/wI8OwkHwWeXFWXL/rIrnRPd5QkSZLWzdiT6iT3At4C7Lvg9Hv7aw8BPgI8\nsao+uobpjwAeB1xE17N9HfAU4NHAqUkeWlXfbYj9zFHXtthyp7VOK0mSNNVcqNhurPtU99XpzwK/\nQLcX9ccXDfkc8CO6xHgtjgAuB36pql5VVa8B9qZ7JPo9gH9N8rNrnFuSJElak3FXql8C3Aq4b1X9\nT5KjgUfMX6yquSSnA/db4/z3Av5hYTW6qm4AnpfkG8CxdIn1gVV12cZOXlX7jLp2y612tttIkiRt\nllyo2G7cSfVBwIeq6n+WGfONftxabMWI7feq6vgkc8DxwCeSPGyN95AkSRoU2z/ajfsx5TvQ9Tsv\nJ3TV7LW4GPj5URer6m+APwLuQ9d6smGN95EkSZJWbdyV6u8Cu60w5h7AN9c4/5eBA5cbUFXHJrkV\n8Crg3mu8jyRJ0mDY49pu3JXqU4BfSbL7UheT7AM8DDh5jfP/M7BTkkctN6iq/gI4mnXcMlCSJEma\nN+6k81V0O3t8LsmLgDsCJNkTeDDwUrrdP16zxvn/EdgCuGqlgVX1sn7x4q5rvJckSdIg2FPdbtxP\nVDwryeOBd9PtIw1dD/X/9scfAo+rqgvXOP8PgDdtxPi3r+U+kiRJQ+LuH+3G3h5RVR9Ncle6PaX3\nA24HXAGcAby1qr4/7ntKkiRJk7QuPcd9RfmvR11PcveqOms97i1JkqSNMzfpADYDzQsVk7x8I8ff\nDfjX1vtKkiRJ02IcleoXJLm4qt640sAkPw98kn4BoyRJkiavsKe61Ti21PsacHySw5YblGRnui33\nfg545RjuK0mSJE2FcSTVjwB+ALw7yYOXGpBkR7oK9V2A11XVi8ZwX0mSJI3BXI3/NTTNSXVVfRV4\nJHA98OEk91p4PckOdD3UewB/V1XPbb2nJEmSxmeOjP01NGN5omJVnQkcBtwG+Je+d5okG4BPAPcE\n3l5VR47jfpIkSdI0GdtjyqvqE3R7U98J+HiSXYGPAfcG3g88bVz3kiRJ0vgUGftraMb9RMX39v3T\nrwPOBrYC/gl4UlUNsLtGkiRJQ7AeT1Q8LskdgefTVaofX1U3jPs+kiRJGg8f/tKuOalOMiphLrqd\nQX6c3OxXAFVV6/I0R0mSJG2cIbZrjNs4Etu1fBf8zkmSJGmz0ZxUV9XYFjtKkiRp07P9o50JsSRJ\nktTIvmZJkqSBs1LdzqRakiRp4Fyo2M72D0mSJKmRlWpJkqSBm7NQ3cxKtSRJktTISrUkSdLAzdlT\n3cxKtSRJktTISrUkSdLA1aQD2AyYVEuSJA2c+1S3s/1DkiRJamSlWpIkaeDm4kLFVlaqJUmSpEZW\nqiVJkgbOhYrtTKolSZIGzoWK7Wz/kCRJkhqZVEuSJA3cXMb/GpckuyQ5IcklSa5NcmGSY5Ns3zDn\ng5PckKSSvHwccdr+IUmSpKmUZDfgdGBH4CTgbOC+wLOAQ5LsX1WXbuSctwXeDlwNbDuuWK1US5Ik\nDdwcGftrTN5Al1AfVVWHVtWfVtVDgdcBewKvWMOcxwEbgFeNK0gwqZYkSRq8WodXq75KfTBwIfC3\niy4fDVwFHJ5km42Y81eBI4CjgEvGEOZPmVRLkiRpGh3YH0+uqptsUFJVVwKnAbcB9lvNZEl2BN4M\nfLiq3jXOQMGeakmSpMEb539DcMIAABdUSURBVMLCeUnOHHWtqvZZxRR79sdzR1w/j66SvQfwyVXM\n92a6gvLTVzF2o5lUS5IkaRpt6I9XjLg+f367lSZK8jTgMcCvV9V3xhDbzZhUS5IkDdx6PPxlldXo\ndZdkV+BY4ANV9ffrdR97qiVJkjSN5ivRG0Zcnz9/+QrznABcAxw5jqBGMamWJEkauGnc/QM4pz/u\nMeL67v1xVM/1vPvQbcv3vf5hL5WkgLf111/Yn/twS7C2f0iSJA3ceixUHINT+uPBSW6xcAeQ/gEu\n+9M9wOWMFeZ5B90uIYvtDjwY+AJwJvD5lmBNqiVJkjR1qur8JCfT7fDxDOD1Cy4fA2wDvKmqrpo/\nmWSv/rNnL5jnqKXmT/JUuqT6o1X1563xmlRLkiQN3HosVByTI+keU358koOAs4D70e1hfS7wwkXj\nz+qPm7z2bk+1JEmSplJVnQ/sC5xIl0w/F9iN7lHj+1XVpZOL7qasVEuSJA3cFFeqqaqL6B4tvpqx\nq65QV9WJdMn6WJhUS5IkDdzqU1GNYvuHJEmS1MhKtSRJ0sBNc/vHrLBSLUmSJDWyUi1JkjRwVqrb\nmVRLkiQN3JgeKz5otn9IkiRJjaxUS5IkDdycW+o1s1ItSZIkNbJSLUmSNHAuVGxnpVqSJElqZKVa\nkiRp4KxUtzOpliRJGji31Gtn+4ckSZLUyEq1JEnSwLmlXjsr1ZIkSVIjK9WSJEkD50LFdibVkiRJ\nA+dCxXa2f0iSJEmNrFRLkiQN3Jy16mZWqiVJkqRGVqolSZIGzoWK7UyqJUmSBs7mj3a2f0iSJEmN\nrFRLkiQNnO0f7axUS5IkSY2sVEuSJA3cXCYdwewzqZYkSRo496luZ/uHJEmS1MhKtSRJ0sBZp263\n2VWqk+yQZMOk45AkSdJwzFxSnWTnJH+T5ONJ/jLJ7frzeyf5EvA94AdJPptkr8lGK0mSNP3m1uE1\nNDPV/pFkB+AMYOf+1MOBhyV5BPBR4HbAF4GdgAcC/5rknlV1+STilSRJ0jDMVFINPJMuoX4l8PfA\nY4CXAicCVwL3r6pvACR5BfBnwB8CL1vN5EnOHHVtiy13aolbkiRparn7R7tZa/84FPjPqvrzqvpS\nVb0cOBU4BHj+fELd+3PgAuDRE4hTkiRpZtQ6vIZm1irVdwbetejcfwH7A6cvPFlVleQzwGGrnbyq\n9hl17ZZb7TzEvx+SJElahVlLqrcGrlp07gqAqvreEuO/A2yz3kFJkiTNsiEuLBy3WWv/+D6w46Jz\nVwHfHTH+doCLFCVJkrSuZi2pPhe4x8ITVfWaqrrTiPF3Ab657lFJkiTNsDlq7K+hmbWk+kxgnyRb\nrTQwyc/Qbat36rpHJUmSNMNcqNhuppLqqnp+Vd2qqn6yiuF3oNtS703rHJYkSZIGbtYWKq5aVZ0H\nHDfpOCRJkqadCxXbzVSlWpIkSZpGm22lWpIkSatTg+yCHi+TakmSpIGz/aOd7R+SJElSIyvVkiRJ\nAzfEfaXHzUq1JEmS1MhKtSRJ0sBZp25npVqSJElqZKVakiRp4OypbmdSLUmSNHBuqdfO9g9JkiSp\nkZVqSZKkgfOJiu2sVEuSJEmNrFRLkiQNnD3V7UyqJUmSBs72j3a2f0iSJEmNrFRLkiQNnO0f7axU\nS5IkSY2sVEuSJA3cXNlT3cqkWpIkaeBMqdvZ/iFJkiQ1slItSZI0cHPWqptZqZYkSZIaWamWJEka\nOB/+0s5KtSRJktTISrUkSdLA+fCXdibVkiRJA+dCxXa2f0iSJEmNrFRLkiQNnAsV21mpliRJkhqZ\nVEuSJA3c3Dq8xiXJLklOSHJJkmuTXJjk2CTbr/Lz2yT5rSTvSXJ2kquSXJnkv5I8N8lW44jT9g9J\nkqSBq5rO9o8kuwGnAzsCJwFnA/cFngUckmT/qrp0hWkeBLwL+AFwCvBhYHvgMcBrgMOSHFRVP26J\n1aRakiRJ0+oNdAn1UVX1+vmTSV4LPAd4BfD0Feb4NvAk4ANV9ZMFczwP+DTwAOAZwF+3BGr7hyRJ\n0sDNUWN/teqr1AcDFwJ/u+jy0cBVwOFJtllunqr6QlW9e2FC3Z+/khsT6QNa4zWpliRJ0jQ6sD+e\nXFU3adPuE+LTgNsA+zXc47r+eH3DHIDtH5IkSYO3Hk9UTHLmqGtVtc8qptizP5474vp5dJXsPYBP\nblx0P/W0/vixNX7+p0yqJUmSBm5K96ne0B+vGHF9/vx2a5k8yTOBQ4AvACesZY6FTKolSZI0dqus\nRk9EksOAY+kWMT62qq5b4SMrMqmWJEkauHEsLFwH85XoDSOuz5+/fGMmTXIo8D7gu8CBVXXB2sK7\nKRcqSpIkaRqd0x/3GHF99/44quf6ZpI8HvgA8B3gIVV1zgofWTUr1ZIkSQM3pQ9/OaU/HpzkFgt3\nAElyW2B/4GrgjNVMluS3gLcDFzPGCvU8K9WSJEmaOlV1PnAysCvdw1kWOgbYBnhnVV01fzLJXkn2\nWjxXkqcA7wC+ATx43Ak1WKmWJEkavPXYUm9MjqR7TPnxSQ4CzgLuR7eH9bnACxeNP6s/Zv5EkgPp\ndve4BV31+4gkiz7G5VV1bEugJtWSJEkDN6Vb6lFV5yfZF3gp3fZ3jwS+BRwHHFNVl61imjtzY3fG\n00aM+TrdbiBrZlItSZKkqVVVFwFHrHLszUrQVXUicOJ4o7o5k2pJkqSBm9It9WaKCxUlSZKkRlaq\nJUmSBm5Kt9SbKSbVkiRJA2f7RzvbPyRJkqRGVqolSZIGblq31JslVqolSZKkRlaqJUmSBm7OhYrN\nTKolSZIGzpS6ne0fkiRJUiMr1ZIkSQPnlnrtrFRLkiRJjaxUS5IkDZyV6nZWqiVJkqRGVqolSZIG\nrtxSr5lJtSRJ0sDZ/tHO9g9JkiSpkZVqSZKkgSsr1c2sVEuSJEmNrFRLkiQNnAsV25lUS5IkDZwL\nFdvZ/iFJkiQ1slItSZI0cLZ/tLNSLUmSJDWyUi1JkjRw9lS3M6mWJEkaOPepbmf7hyRJktTISrUk\nSdLAzblQsZmVakmSJKnRZpVUJ3ltksdPOg5JkqRZUuvwv6HZrJJq4NnAwycdhCRJkoZlZnqqkzxt\nlUP3XDi2qk5Yp5AkSZI2C/ZUt5uZpBp4C6z4u4QCHti/0r83qZYkSVrGENs1xm2WkmqAHwF/B1y9\nxLUALwb+G/jIWiZPcuaoa1tsudNappQkSdIAzFJS/WTg9cBhwBFVderiAUleDPx3VR2zqYOTJEma\nVbZ/tJuZpLqq3pXkM8DbgE8nORZ4YVVdO8Z77DPq2i232tm/bZIkSVrSTO3+UVUXVdXDgOcCRwKf\nT3LfCYclSZI009xSr91MJdXzquo4YB+63urTkrwqyZYTDkuSJGkmzVWN/TU0M5lUA1TVWcD9gFcB\nzwM+z8q7g0iSJEljN7NJNUBV3VBVL6bbQm9Luh1AJEmStBFs/2g3MwsVl1NV/57kHsC2wNgWLkqS\nJEmrsVkk1dBVrYErJh2HJEnSrKmam3QIM2+zSaolSZK0NnMDbNcYt5nuqZYkSZKmgZVqSZKkgasB\nboE3blaqJUmSpEZWqiVJkgbOnup2VqolSZKkRlaqJUmSBs6e6nYm1ZIkSQM3Z1LdzPYPSZIkqZGV\nakmSpIErFyo2s1ItSZIkNbJSLUmSNHAuVGxnUi1JkjRw7lPdzvYPSZIkqZGVakmSpIGz/aOdlWpJ\nkiSpkZVqSZKkgfPhL+1MqiVJkgbO9o92tn9IkiRJjaxUS5IkDZxb6rWzUi1JkiQ1slItSZI0cPZU\nt7NSLUmSJDWyUi1JkjRwbqnXzqRakiRp4MqFis1s/5AkSZIaWamWJEkaONs/2lmpliRJkhpZqZYk\nSRo4t9RrZ1ItSZI0cC5UbGf7hyRJkqZWkl2SnJDkkiTXJrkwybFJtt/IeXboP3dhP88l/by7jCNO\nK9WSJEkDN63tH0l2A04HdgROAs4G7gs8Czgkyf5Vdekq5rldP88ewKeA9wF7AUcAj0py/6q6oCVW\nK9WSJEmaVm+gS6iPqqpDq+pPq+qhwOuAPYFXrHKeV9Il1K+tqoP6eQ6lS8537O/TJNP6k8m0ueVW\nO/sPSpI09Y650wGTDkEb4YVff3cmHQPAluuQ51z3k4ubvra+Sv1V4EJgt6qaW3DttsC3gAA7VtVV\ny8yzLfBdYA64U1VdueDaLYALgDv391hztdpKtSRJ0sDVOrzG4MD+ePLChBqgT4xPA24D7LfCPPsB\nWwOnLUyo+3nmgI8vut+a2FMtSZKksUty5qhrVbXPKqbYsz+eO+L6ecDBdG0dn2ych36eNTOpXqXr\nG3+FMY3m/7Kv8i+2Jsjv1ezwezU7/F7NFr9f62s98pzlkupV2tAfrxhxff78dptonmWZVEuSJGns\nhvYDkD3VkiRJmkbzFeQNI67Pn798E82zLJNqSZIkTaNz+uOoXufd++OoXulxz7Msk2pJkiRNo1P6\n48H91nc/1W+ptz9wNXDGCvOcAVwD7N9/buE8t6Bb7LjwfmtiUi1JkqSpU1XnAycDuwLPWHT5GGAb\n4J0L96hOsleSvRbN8yPgnf34lyya55n9/B9vfaKiCxUlSZI0rY6ke7z48UkOAs4C7ke3p/S5wAsX\njT+rPy7ezeQFwAHAHyXZG/gP4O7Ar9I9GGZx0r7RfKKiJEmSplaSnwNeChwC3I7uSYofAo6pqssW\njS2AqrrZFoFJdgCOBg4F7gRcCvwL8OKq+mZznCbVkiRJUht7qiVJkqRGJtWSJElSI5NqSZIkqZFJ\ntSRJktTIpFqSJElqZFItSZIkNTKpHqAkuyQ5IcklSa5NcmGSY5NsP+nYdKMkj0vy+iSfS/LDJJXk\nXZOOSzeV5HZJfifJh5J8Nck1Sa5IcmqS3178aF1NXpK/SPLJJBf1368fJPl8kqOT3G7S8Wm0JE/q\n/11YSX5n0vFIC7lP9cAk2Y3uyUQ7AicBZwP3pXsy0TnA/lV16eQi1LwkXwB+EfgR8E1gL+DdVfWk\niQamm0jydOCNdA8jOAX4BnAH4DBgA/BB4PHlv2ynRpKfAP8N/C/dk9S2AfYD9gUuAfarqosmF6GW\n0j8A5MvAFsC2wO9W1VsmG5V0Ix9TPjxvoEuoj6qq18+fTPJa4DnAK4CnTyg23dRz6JLprwIPoUvY\nNH3OBR4DfLSq5uZPJnkB3WNwH0uXYH9wMuFpCT9TVT9efDLJK+geZfxndI9G1pRIEuBtdE/A+0fg\neZONSLo5fy05IH2V+mDgQuBvF10+GrgKODzJNps4NC2hqk6pqvOscE63qvpUVX1kYULdn/828Hf9\n2wM2eWAaaamEuvf3/XH3TRWLVu0o4KHAEXT/rZKmjkn1sBzYH09eIgG4EjgNuA3dr0EltbuuP14/\n0Si0Wr/SH7800Sh0E0nuDrwaOK6qPjvpeKRRbP8Ylj3747kjrp9HV8neA/jkJolI2kwluSXw5P7t\nxyYZi5aW5Hl0vbkb6PqpH0iXUL96knHpRv3/j95Jt1bhBRMOR1qWSfWwbOiPV4y4Pn9+u00Qi7S5\nezVwT+Cfq+rjkw5GS3oe3aLSeR8DnlpV35tQPLq5FwP3Bh5YVddMOhhpObZ/SNKYJTkKeC7d7jqH\nTzgcjVBVd6yqAHekW0x6V+DzSe4z2cgEkOR+dNXpv66qf5t0PNJKTKqHZb4SvWHE9fnzl2+CWKTN\nUpJnAsfRbdd2YFX9YMIhaQVV9Z2q+hBd+9vtgHdMOKTB69s+3kHXrviiCYcjrYpJ9bCc0x/3GHF9\nfsX7qJ5rSctI8mzg9cD/0CXU355wSNoIVfV1uh+GfiHJz046noHblu6/VXcHfrzggS9Ft1sVwJv7\nc8dOLEppAXuqh2V+n+ODk9xi0Z66twX2B64GzphEcNIsS/J8uj7qLwAPr6rvTzgkrc1O/fGGiUah\na4G3jrh2H7o+61PpikW2hmgqmFQPSFWdn+Rkul9xPoOuojbvGLqnir2pqtwDVNoISV4EvBQ4EzjY\nlo/plWQP4DtVdcWi87cAXkb3cKzTq+qyScSnTr8occnHkCd5CV1S/XafqKhpYlI9PEfSPab8+CQH\nAWcB96Pbw/pc4IUTjE0LJDkUOLR/e8f+eP8kJ/Z//n5V+VSxCUvyFLqE+gbgc8BR3cPfbuLCqjpx\nE4empT0SeFWSU4Gv0T2h7w50Ty29K/Bt4HcnF56kWWVSPTB9tXpfuiTgELr/wHyLbmHVMVZnpsre\nwFMWnbtr/wL4Oj6qdxrcpT9uATx7xJjPACdukmi0kn8F7ka3J/W96bYQvYquqPBO4Hh/0yBpLeIT\nkCVJkqQ27v4hSZIkNTKpliRJkhqZVEuSJEmNTKolSZKkRibVkiRJUiOTakmSJKmRSbUkSZLUyKRa\nkiRJamRSLUmSJDUyqZYkSZIamVRLkiRJjUyqJUmSpEYm1ZIGI8m+Sd6W5IIk1yT5YZIvJvmLJHec\ndHySpNmVqpp0DJK0rpIEeDXwJ8D1wCeALwNbAQ8A7gv8CHhiVf2/ScUpSZpdJtWSNntJXgwcA1wI\nPLqqvrLo+mOBdwFbAA+qqn/f5EFKkmaa7R+SNmtJdgVeBFwHPGZxQg1QVR8EngNsCfzdgs++JEkl\nOWCpeftrJy5x7TZJ/izJF5JcleRHSf4tyROXGHtAP89Lktw3yUeT/KA/t1uSi/o2lW1HfH2v78c+\nbrX/TCRJ42dSLWlzdwRwS+BDVfXlZca9BfgWsHeS/dZ6syTbAacCrwRuAE4A3g7cHnhPkpeP+Oj9\ngc8Bt17wmWuANwO3BZZKyLcGngR8GzhprTFLktqZVEva3D2wP/7rcoOq6nrglP7tgxvudyxwb+D5\nVbVPVf1hVT0DuCfwceAFSfZe4nMHA0dV1UFV9byqempVXUKXVF8P/P4Sn/l1YDvghKq6riFmSVIj\nk2pJm7s79ceLVjF2fswua7lRktvRVY7/q6r+cuG1qvox8HwgwG8u8fEvVNWbFp+sqm8BHwb2SbLP\nosu/D8zRJd6SpAm65aQDkKQpdOs1fu6X6BY7VpKXLHF9y/549yWu/ccy874BeBxdEv17AEnuBewH\n/EtVXbjGeCVJY2JSLWlz9226JPbnVjF2fsz31niv2/XHX+pfoyy16PDbowZX1SlJzgKemOS5VXUl\nfXIN3Ky6LUna9Gz/kLS5O7U/Pmy5QUm2AA7o357ZH+f641IFiO2WOHdFf3xdVWWZ14FLfHal/U3/\nji4Z/60FCxQvBtxXW5KmgEm1pM3d2+gW+v1akl9YZtzTgJ2AHwAf689d1h+XqnLvu8S5/6BLxB+0\ntlCX9XbgaroK9fwCxbdW1Q3rcC9J0kYyqZa0WauqrwEvp+tn/qck91g8JsmhwHH92+dX1dX9n+f7\nnI9IcssF438OePES9/ou8G5g3yQv6qvfi++1W5K7rOHruAJ4D93OIi+n267PBYqSNCXsqZY0BC8F\ntgH+GPhiko8DX6FLtB8A3K8f95dV9Zb5D1XVvyf5LN0We/+R5FPAHYBfodseb6kK9jOB3ft7Hp7k\nVOA7dFXwu9P1Wj8R+Noavo43AL8D7Ax8pKq+uYY5JEnrwKRa0mavqgr4kyQfAJ4BPISux/pW/ZBv\nAU+uqqX2sv5V4K/64x8C5wF/ApwMPGGJe/0wyUPo2jR+E3gs3W4i3+k/+xzgE2v8Oj6f5AvA3rhA\nUZKmSrr/1kjS8CS5Ld1CxnsAj6+qD084pGX18V5C1/d9l6qaW+EjkqRNxJ5qSYPVb033aLot9N6f\n5JAJh7SSP6DbAeQNJtSSNF2sVEsavCS/CPwa3e4ax1bVTyYc0k8l2UCXTO8M/C5dlXrP/gcCSdKU\nMKmWpCmWZFe6RY3X0u2f/YdV9d+TjEmSdHMm1ZIkSVIje6olSZKkRibVkiRJUiOTakmSJKmRSbUk\nSZLUyKRakiRJamRSLUmSJDUyqZYkSZIamVRLkiRJjUyqJUmSpEYm1ZIkSVIjk2pJkiSpkUm1JEmS\n1MikWpIkSWr0/wGHW4tAVJOF3wAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{"tags":[],"image/png":{"width":362,"height":277}}}]},{"cell_type":"markdown","metadata":{"id":"i1iia78EjsMo","colab_type":"text"},"source":["Moving horizontally across the map (x-axis), we see that positions `0`, `1` and `2` perform queries that attend to position `1`.\n","\n","Looking at our `Q` vector, we see that the `Q` vector at positions `0`, `1`, `2` have the same value of `Q=[0,10,0]`. Comparing with all the possible key values in `K`, we find a direct match in index `1` (`K=[0,10,0]`).\n","\n","Hence, we can see that the attention mechanism behaves conceptually similar to a dictionary lookup or database query, in that it searches for a key that matches the query. In the case of the attention mechanism, the \"quality\" of the match is given by the value of the dot product between `Q` and `V`, also referred to as the attention score. In our toy examples, all our `Q` and `K` vectors are large and octogonal to each other, resulting in either a very strong or a very weak match.\n","\n","Likewise, we can see that:\n","\n","* position `3` (`Q=[10,0,0]`) attends to position `0` (`K=[10,0,0]`)\n","* position `4` (`Q=[0,0,10]`) gives equal attention to positions `2`, `3` and `4` (`K=[0,0,10]`)."]},{"cell_type":"markdown","metadata":{"id":"tKHqkiQApDjw","colab_type":"text"},"source":["### Interpreting the Outputs"]},{"cell_type":"markdown","metadata":{"id":"KTERN2NGYodF","colab_type":"text"},"source":["Here, we have the output from the attention mechanism: an array of five values."]},{"cell_type":"code","metadata":{"id":"T08S6iDphoEe","colab_type":"code","outputId":"5d3f596e-b2bc-48c6-9e78-e732076a9bd7","executionInfo":{"status":"ok","timestamp":1577283161585,"user_tz":-480,"elapsed":6762,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":119}},"source":["print_outputs(output)"],"execution_count":13,"outputs":[{"output_type":"stream","text":["Output:\n","[[1.00e+00]\n"," [1.00e+00]\n"," [1.00e+00]\n"," [3.78e-43]\n"," [3.00e+00]]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"2FbIS5u7pFtT","colab_type":"text"},"source":["Remember how our `V` vector contains a sequence of values from `0.0` to `4.0`?\n","\n","The output of the self-attention operation shows that:\n","\n","* Positions `0`, `1`, `2` retained information mostly from position `1`\n","* Position `3` retained information mostly from position `0`\n","* Position `4` retained information from a even mix of position `2`, `3`, `4`\n"]},{"cell_type":"markdown","metadata":{"id":"AvkjgqpfeGp8","colab_type":"text"},"source":["## Causal Masking"]},{"cell_type":"markdown","metadata":{"id":"6vAB1YMsrmek","colab_type":"text"},"source":["Sometimes, we do not want the attention mechanism to be able to \"look into the future\". An example of this is in the decoder portion of a language model. At each position, we do not want to allow the attention mechanism to look into positions on the right (\"future words\"), but only attend to positions on the left (\"past words\").\n","\n","In order to facilitate this, we introduce the concept of a **causal attention mask**."]},{"cell_type":"code","metadata":{"id":"hJek5-X1dx70","colab_type":"code","outputId":"f596ea26-28f9-4473-d123-d5aea52fc3e5","executionInfo":{"status":"ok","timestamp":1577283161587,"user_tz":-480,"elapsed":6751,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":119}},"source":["causal_mask = causal_attention_mask(K.shape[1])\n","\n","causal_mask"],"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"markdown","metadata":{"id":"XC6oOEdRZXeL","colab_type":"text"},"source":["Now, we will use \"softer\" values for our `Q` and `K` vectors to get a more spread-out attention map.\n","\n","This will allow us to see the masked-out areas of the map more easily."]},{"cell_type":"code","metadata":{"id":"731ToBkJtRij","colab_type":"code","colab":{}},"source":["Q = tf.constant([[[0.1,1.0,0.1],\n"," [0.1,1.0,0.1],\n"," [0.1,1.0,0.1],\n"," [1.0,0.1,0.1],\n"," [0.1,0.1,1.0]]], dtype=tf.float32)\n","\n","K = tf.constant([[[1.0,0.1,0.1],\n"," [0.1,1.0,0.1],\n"," [0.1,0.1,1.0],\n"," [0.1,0.1,1.0],\n"," [0.1,0.1,1.0]]], dtype=tf.float32)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Q4AShLBotAvU","colab_type":"code","outputId":"ff344738-57ea-43ea-a82c-40a7e86b8d19","executionInfo":{"status":"ok","timestamp":1577283161966,"user_tz":-480,"elapsed":7121,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":294}},"source":["output, attn_w = attention(Q, K, V, causal_mask)\n","plot_weights(attn_w)"],"execution_count":16,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAtUAAAIqCAYAAADrd7anAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde7x29Zz/8ddbyqHMXSFURqQDw0zU\nKHIokQZDk8OMIcowY2LC8BvGKSHMjKEyTj8kZwZD/MyQIYdqGjM5jumkRERIJZWk/fn9sdbWbrev\nvfe9v9e+r2vf6/X0uB7rvtb6Xt/12e1bffZnf77flapCkiRJ0srdaNIBSJIkSWudSbUkSZLUyKRa\nkiRJamRSLUmSJDUyqZYkSZIamVRLkiRJjUyqJUmSpEYm1ZIkSVIjk2pJkiSpkUm1JEmS1MikWpIk\nSWpkUi1JkiQ1MqmWJEmSGplUS9J6SlL9a4dJxyJJmg4m1ZIWlOTAOcnjZ5YxfrckL01yyBLjntWP\n22FMoY5Nkn362A6cdCzjlOTzc76XP02y6RLjnzlnfCXZZwOFKklrlkm1pFGeNOfPD0yy3RLjdwOO\nAA5ZYtyz+nE7rDiy1bMPXWxLJdVn9a9rVjugVXBL4GFLjHnSEtclSfOYVEu6gSS3oku8rgDeR/fv\nioMnGtQUqapd+9cPJh3Levpef3ziqAFJfge4B/DdDRKRJG0kTKolLeRxwKbAx4G39OesXq59JwCX\nAw9LsvWIMbPf5/dsmJAkaeNgUi1pIbOJ1XuBL9FVOHdNcq+FBicp4B392wfM68etOb3KBdyhH3fS\nvDGfX2DeLZK8IMl/JbksyS+TnJPk2CS3HxHLbP/wIUlu1t/3rCRXJflxkg8k2WneZ3boYzti9utf\n4GvYYe7Xu9hCxSQ7JnlLkvP6mC9J8sUkT0myybjiXoErgY8AmwF/vEAMNwIe379992ITJblnklcn\nOTnJ95JcneTi/utY7Ot8af91Hp/kRkmeneTrSa7oP//xUX/PJGmamVRLup7+1/+7AxcDJ1ZVAe/v\nL4+qVl8E/Lz/8zX9+7mvXwG/6P8804+7ZN6Yn82L4y7A/wBHAXsANwd+DdwZ+Cvg60n2XuRL+S3g\nFLpE+Q5AAbemSyZPS7LjnLHX9jFc0b//5QJfw7WL3Gtu3A/v4/5z4I79XJsD9wPeCnwqyeZjinsl\nZpPlhVpAHgRsC5wKnLvEPCcCzwP2puvTvhLYGngA3df58SQ3XuTzAT4MvBa4K93fm62BPwROTXKD\npF+SpplJtaT5ZhPnf66q2YV47+2Pf5Jks/kfqKrbAs/s355aVbed9zq1ql7Tj7ugH3fQvDEHzc6X\nZB3wr3RJ5YeA3wNuWlVbADvS9XlvBXwkyZYjvo4j+zEH0CW1WwD3B75Pl7y9ak78F/SxvaY/9cEF\nvoYLWEKf8H4AuCnwBWDXqtoSuAXwF8DVdInrMYtMs+y4V+gkuu/BXgtUvmcT7XctY54T6dqEbldV\nm1fVVn2sBwM/Ah4KPHuRzz8SeATw18Bv9f+c7gx8BtgEeMcYfoCQpA3GpFrSb/S/sn9C//Z9s+er\n6pvAN7mukrja/g/d7iDvr6rHVtU3quraPpbzqurxwKeA2wBPGTHHTYAHV9Wnq+raqpqpqi/R7T4C\n8IiFfkBo9AK6RPhc4KFVdVYf89VV9X+Bw/txT05y50nE3f/mYbZf+jeLT5NsAfwRXeL/wWXM86dV\n9YGq+tGcc1dU1XuAx/anDltkinXAEVX1uqq6qv/8uXSJ9lnAzYC/XfYXJkkTZlItaa4HA7ej2/nh\nlHnXZqvVG2LB4uw9/nGRMbNJ/4NHXP9wVX17gfMfp2upuAldZXQskgR4VP/2dVV15QLD3gb8gK71\n4dEjptoQcc9Wop/Qx00fz82BT1TVpS2T9z8EXArskGTbEcOuBI5e4LO/5Lrv+6PmxCdJU82kWtJc\nh/TH9/cVzbneT5fU/UGSW69WAP0CxO37t/+a5EcLvbiuhWLBBYvAfy10sm9p+XH/dquxBQ53oqu+\nQtdisdC9Z4DP92/vOWKeVY+7qs7s73NHul5vWL/WDwCSPCbJx/qFilfNXdgJzLbljEqq/7uqrhhx\n7Qv9ccs+RkmaeostIpE0IH0f8yP7t++bf72qvpfkS3T9vX/K4n3BLW4358/bLGP8zUecv3yRz/yy\nPy76ZMH1NPcHjcX2r/7+AuPn2lBxvwv4feDgJN+he/DNT4B/W+qD/QLEf6ZrF5l1NfBTrlvQeWu6\nws2oRZmL/TOae+3WwHlLxSRJk2alWtKsP6ZbYAfwjQW2lCu6hBpWtwVk7r+XtqqqLPHaYRVjWamb\nLj1k4j5At+PGY4Cn0rWkvL+qfr2Mzz6VLqG+kq5P/PZVddOquvXswk7gwn6s7RuSBsGkWtKs9UmU\n75Hk7qsUx0Vz/vzbq3SP1fCTOX9eLO7Z1pafLDJm1VXVT+mq0uuA5/enF92beo7H9MeXV9Xrq+r7\ncy/2C15vtcQco9pC5l+b6D8nSVouk2pJ9Fur3ad/uxtdz+6o1yf6cfOT8Nn9p5eqTC46rqq+w3WJ\n9R8sI/xxWu7XsJDz6BbnAey70ID+4Sr79G+/soJ7jNts//SmwP9W1X8v83OzPxh8dcT1vVm6Wr9H\nklGtOw/oj5cC31lmTJI0USbVkuC6RWpfr6qvV9Wlo150+0YDPH7eU/NmH/4yat/o9Rl3fH98bpLt\nRg1KZ6n7rY/lfg030C/s/Jf+7TNHJIxPAbajW/D5oQWub2ifAP6BbreN5y8xdq7L+uMNflvR91u/\nYhlzbM51e5vP/fxN6Pauhm4nlPkLZiVpKplUSwPXb1k2u1/xvyw2tvcJul7c2wIPmXP+W/3xrkn2\nXOTzs+Mel2RUNfPVdJXfW9E9Xe+xSW42J+bfTvLndNXeA5cR83LNxnbfFT4S/JV0T2XcFvhkkl2g\nSxSTPBU4th/39n5P5omqql9V1d9U1XOr6hNLf+I3PtMfX5zkkbM/XCXZle7vx7247umUo1wGvDzJ\nM2e/t0nuBJwA3IVuUear1yMmSZook2pJ+9A9uRDgI0sN7qvVn+vfPmnO+XOAL9LtKnRakouTnN+/\n9pozxdv742OAy5Jc0I/5wLx7PAQ4g64/+YPA5Ul+muRKun2030LXqjLOSubn6R7csjVwVpIfz/ka\ntl/8o795eMnj6BLCfYAzk1xCt6PH/6XbY/qzXPcgl7XqNXT/nH4L+BhwVZLL6L5fDwaeRrcTyGJO\noNt7+2i6vweX9HM+hG4HkUOn4QcPSVouk2pJs4nx2VX1rUVHXmc2+X7EvPaLg4A30vXBbkGXrN+B\nOf21VfU5up0jvgBcRdcOcQe6yjdzxn0buAfdU/lOAi6hW1T3a+AbdEnqw7ju6YDN+r2g96NbsPcD\nuh7y2a9hWVuQ9hXfuwNvBc6n2/LvSuBk4M+BhyyyP/OaUFU/A/YC3sR1WwReRZdgP6Cqjl/ONHQ/\nWP01XTK+Gd33+P8B96mqDyzyWUmaOrFdTZK0oSR5KXAE8M6qOmSy0UjS+FipliRJkhqZVEuSJEmN\nTKolSZKkRibVkiRJUiMXKkqSJEmNrFRLkiRp6iR5dJLXJ/lSkp8nqSQr2kY1yfZJjktyYZKr++cP\nHJ1kq3HFu6x9VyVJkqQN7EXA7wG/oNsTf9eVTJJkR+BUYBu6B0+dSffk12cCByTZu6oubg3WSrUk\nSZKm0bOBneme3vqXDfO8kS6hPryqDqyq51fVA4HXAbsARzVHij3VkiRJmnJJ9qF7uu57q+oJ6/G5\nHYFv0z3hdseqmplz7RbAD4EA27Q+7dZKtSRJkjZW+/bHE+cm1ABVdTlwCnBzYK/WG9lTLUmSpLFL\ncvqoa1W1+wYKY5f+ePaI6+cA+9O1mXy25UYm1ct0zU/Ps09mjXjJHi+adAhapgvql5MOQct00NU3\nnXQIWqaX1HmTDkHr4X8uOi2TjgFWLc/5yirMub7W9cfLRlyfPb9l641MqiVJkjR2G7AaPRVMqiVJ\nkoZu5tpJR7BaZivR60Zcnz1/aeuNXKgoSZKkjdVZ/XHnEdd36o+jeq6XzUq1JEnS0F1/Y4yNyUn9\ncf8kN1pgS729gSuB01pvZKVakiRp6GZmxv/agJJsmmTXfl/q36iqc4ETgR2Ap8/72JHA5sC7W/eo\nBivVkiRJmkJJDgQO7N/etj/eO8nx/Z9/WlXP7f+8HXAG8F26BHquw+geU35skv36cXvS7WF9NvDC\nccRrUi1JkjRwNZ3tH7sBT5p37k79C7oE+rksoarOTbIH8DLgAOChdE9SPAY4sqouGUewJtWSJEma\nOlX1UuClyxx7Pt3jxkddvwA4dBxxjWJSLUmSNHQbuAd6Y2RSLUmSNHTT2f6xprj7hyRJktTISrUk\nSdLQbbxPVNxgrFRLkiRJjaxUS5IkDZ091c2sVEuSJEmNrFRLkiQNnVvqNTOpliRJGrgpfaLimmL7\nhyRJktTISrUkSdLQ2f7RzEq1JEmS1MhKtSRJ0tDZU93MpFqSJGnofKJiM9s/JEmSpEZWqiVJkobO\n9o9mVqolSZKkRlaqJUmShs4t9ZqZVEuSJA2d7R/NbP+QJEmSGlmpliRJGjrbP5pZqZYkSZIaWamW\nJEkauCof/tLKpFqSJGnoXKjYzPYPSZIkqZGVakmSpKFzoWIzK9WSJElSIyvVkiRJQ2dPdTMr1ZIk\nSVIjK9WSJElDN+OWeq3WbFKd5GbAXsDOwJb96UuBs4HTquqqScUmSZK0ptj+0WzNJdVJtgKOAg4G\nbj5i2JVJ3gW8qKou2WDBSZIkaZDWVFKdZEvgFGBX4ArgM8A5wGX9kHXATsDewF8C+ya5d1VdtsB0\nkiRJArfUG4M1lVQDR9Al1K8DjqiqXyw0KMkWwMuAZwEvAZ6znMmTnD7q2q9+cu56BytJkqRhWGu7\nfxwIfK6qnjMqoQaoql9U1V8DnwcO2lDBSZIkrUk1M/7XwKy1SvXtgPevx/jTgPssd3BV7T7q2jU/\nPa/W476SJElrh+0fzdZapfpiYJf1GH+X/jOSJEnSqllrSfWngQOTHLbUwCTPAB4BfGrVo5IkSVrL\nZmbG/xqYtdb+8WLgYcDrkzwHOJFuX+q5u3/sDOwP7AD8mG6hoiRJkrRq1lRSXVU/SHJv4E3Ag4G/\nAOb3Oqc/nggcVlU/2IAhSpIkrTlVPlGx1ZpKqgGq6jzgIUnuBOxL12O9rr98GXAWcFI/TpIkSUsZ\nYLvGuK25pHpWnzSbOEuSJGni1mxSLUmSpDEZ4L7S47bWdv+QJEmSpo6VakmSpKGzp7qZlWpJkiSp\nkZVqSZKkobOnuplJtSRJ0tDZ/tHM9g9JkiSpkZVqSZKkobP9o5mVakmSJKmRlWpJkqShs6e6mUm1\nJEnS0JlUN7P9Q5IkSWpkpVqSJGnoXKjYzEq1JEmS1MhKtSRJ0tDZU93MpFqSJGnobP9oZvuHJEmS\n1MhKtSRJ0tDZ/tHMSrUkSZLUyEq1JEnS0NlT3cxKtSRJktTISrUkSdLQ2VPdzKRakiRp6Eyqm9n+\nIUmSJDWyUi1JkjR0VZOOYM2zUi1JkiQ1slItSZI0dPZUNzOpliRJGjqT6ma2f0iSJEmNrFRLkiQN\nnU9UbGalWpIkSVMryfZJjktyYZKrk5yf5OgkW63nPPdNckL/+V8m+V6Sf01ywDjitFItSZI0dFPa\nU51kR+BUYBvgBOBM4F7AM4EDkuxdVRcvY56/BN4IXAF8FPg+sD1wEPAHSV5UVUe1xGpSLUmSNHTT\nu0/1G+kS6sOr6vWzJ5O8Fng2cBTwtMUmSLIp8Crgl8DuVXXWnGuvBL4KvDDJa6rq6pUGavuHJEmS\npk5fpd4fOB94w7zLR9BVnQ9OsvkSU20NrAPOnptQA1TVGcDZwM2ALVriNamWJEkaupmZ8b/a7dsf\nT6y6/krKqrocOAW4ObDXEvP8GPgJsHOSneZeSLIzsBPwteW0kSzG9o9lutm295t0CFqmS552j0mH\noGW64JNT++tGzfOcmasmHYKW61eTDkDqJDl91LWq2n0ZU+zSH88ecf0cukr2zsBnF7lXJXk68B7g\n9CQfBS4EtgP+CPgW8CfLiGdRJtWSJElDN50LFdf1x8tGXJ89v+VSE1XVh5JcCLwfeOKcSxcB7wDO\nW2mQs0yqJUmSNHbLrEZvEEmeALwV+Bfg5cB3gTsALwb+CXgA8NiWe5hUS5IkDd10PvxlthK9bsT1\n2fOXLjZJ3zd9HPAN4OA5/dlnJjmYrs3kMUn2qarPrzRYFypKkiQNXM3U2F9jMLtTx84jrs8uOhzV\ncz1rf2BT4AsLLHicAb7Yv22qrJtUS5IkaRqd1B/3T3K9nDXJLYC9gSuB05aY5yb98dYjrs+eb1rm\na1ItSZI0dFO4pV5VnQucCOwAPH3e5SOBzYF3V9UVsyeT7Jpk13ljv9QfH53kd+deSLIb8GiggM+1\nxGtPtSRJkqbVYXSPKT82yX7AGcCedHtYnw28cN74M/pjZk9U1ZeTvAM4FPivfku979Il6wcCmwFH\nV9W3WgI1qZYkSRq66VyoSFWdm2QP4GXAAcBDgR8CxwBHVtUly5zqz+h6pw8BHgLcAvg5cDLw1qr6\nQGusJtWSJElDN56Fhauiqi6gqzIvZ2xGnC/g+P61KuypliRJkhpZqZYkSRq66Xyi4ppipVqSJElq\nZKVakiRp6KxUNzOpliRJGrqa3oWKa4XtH5IkSVIjK9WSJElDZ/tHMyvVkiRJUiMr1ZIkSUM3xQ9/\nWSusVEuSJEmNrFRLkiQNXdlT3cqkWpIkaehs/2hm+4ckSZLUyEq1JEnSwJVb6jWzUi1JkiQ1slIt\nSZI0dPZUNzOpliRJGjp3/2hm+4ckSZLUyEq1JEnS0Nn+0cxKtSRJktTISrUkSdLQuaVeM5NqSZKk\nobP9o5ntH5IkSVIjK9WSJElD55Z6zaxUS5IkSY2sVEuSJA2dPdXNrFRLkiRJjaxUS5IkDVy5pV4z\nk2pJkqShs/2jme0fkiRJUiMr1ZIkSUNnpbqZlWpJkiSp0UafVCf5hyTnTjoOSZKkqVUz438NzBDa\nP24F7LCcgUlOH3Vtk023HVc8kiRJ08X2j2YbfaVakiRJWm1rrlKd5F3r+ZH7LHdgVe0+6tqNN9vO\nH+EkSdJGqaxUN1tzSTXwBKCArMdn/JsiSZKkVbMWk+rLge8Dhy1z/POB/VcvHEmSpDXOSnWztZhU\nfx34var6wnIGJzlkdcORJEla43xMebO1uFDxa8AWSXacdCCSJEkSrM1K9ReA+wHbA8vZf/pjwPmr\nGZAkSdKaZvtHszWXVFfVR4CPrMf4E4ATVi8iSZIkDd2aS6olSZI0Zlaqm63FnmpJkiRpqlipliRJ\nGrgqK9WtTKolSZKGzvaPZrZ/SJIkSY2sVEuSJA2dlepmVqolSZKkRlaqJUmSBq6sVDczqZYkSRo6\nk+pmtn9IkiRJjaxUS5IkDd3MpANY+6xUS5IkSY2sVEuSJA2cCxXbmVRLkiQNnUl1M9s/JEmSpEZW\nqiVJkobOhYrNrFRLkiRJjaxUS5IkDZwLFdtZqZYkSZIaWamWJEkaOnuqm5lUS5IkDZztH+1s/5Ak\nSZIaWamWJEkaOts/mlmpliRJkhpZqZYkSRq4slLdzKRakiRp6Eyqm9n+IUmSJDWyUi1JkjRwtn+0\ns1ItSZIkNbJSLUmSNHRWqptZqZYkSRq4mhn/a1ySbJ/kuCQXJrk6yflJjk6y1QrmumeS9yX5fj/X\nRUm+kOSJrXFaqZYkSdJUSrIjcCqwDXACcCZwL+CZwAFJ9q6qi5c51zOAY4BLgE8CPwC2Bu4GPBR4\nV0usJtWSJEkDN8ULFd9Il1AfXlWvnz2Z5LXAs4GjgKctNUmS/YFjgc8Aj66qy+dd37Q1UNs/JEmS\nNHX6KvX+wPnAG+ZdPgK4Ajg4yebLmO4fgKuAP52fUANU1TVt0VqpliRJGrwprVTv2x9PrLp+hFV1\neZJT6JLuvYDPjpokyd2A3wU+Bvwsyb7A7kABXwNOmj//SphUa6Oz1Zu/OukQtEx/crs9Jx2Clu2q\nSQegZdrrZttPOgStRZWxT5nk9JG3q9p9GVPs0h/PHnH9HLqkemcWSaqB3++PPwY+D9x/3vVvJjmo\nqr69jJhGsv1DkiRJ02hdf7xsxPXZ81suMc82/fHPgB2Ah/Vz7wy8B7g78Mkkm604UqxUS5IkDd5q\ntH8ssxq9IcwWkTcB/qSq/qN///N+K71dgT2ARwHvb72JJEmSNE1mK9HrRlyfPX/pEvPMXv/RnIQa\ngKoquq36oNuqb8WsVEuSJA1czYy/p3oMzuqPO4+4vlN/HNVzPX+eUcn3Jf3xZsuMa0FWqiVJkjSN\nTuqP+ye5Xs6a5BbA3sCVwGlLzHMa3fZ7O4zYfu9u/fE7DbGaVEuSJA3dND6mvKrOBU6kW1z49HmX\njwQ2B95dVVfMnkyya5Jd581zJfB24KbAK5Jkzvi7A4cAvwY+3BKv7R+SJEkDV6uwpd6YHEb3mPJj\nk+wHnAHsSbeH9dnAC+eNP6M/zv+CXky3ld6zgHv3e1zfBjiILtl+Vp/Er5iVakmSJE2lPtHdAzie\nLpl+DrAjcAywV1VdvMx5fg7cD3glsDXwDODhwMnAQ6rqmNZYrVRLkiQN3JQ+URGAqroAOHSZY0eW\n3KvqF3SV7fnV7bGwUi1JkiQ1slItSZI0cFO6pd6aYlItSZI0cFWTjmDts/1DkiRJamSlWpIkaeBs\n/2hnpVqSJElqZKVakiRp4KxUtzOpliRJGjgXKraz/UOSJElqZKVakiRp4Gz/aGelWpIkSWpkpVqS\nJGngqqxUt7JSLUmSJDWyUi1JkjRwNTPpCNY+k2pJkqSBm7H9o5ntH5IkSVIjK9WSJEkD50LFdlaq\nJUmSpEZWqiVJkgbOh7+0M6mWJEkauKpJR7D2jbX9I8lrk+w8zjklSZKkaTfunupnAWck+VySP06y\n6ZjnlyRJ0pjVTMb+GppxJ9WPA74APAB4H/D9JK9OsuOY7yNJkiRNjbEm1VX1wap6ILAL8Nr+9N8A\nZyU5MclBSTYZ5z0lSZLUZqYy9tfQrMqWelX17ar6P8D2dNXrLwL7AR8CLkjyiiR3WI17S5Ikaf1U\nZeyvoVnVfaqr6po51ev7ABcCtwH+Fjg3yceS7LaaMUiSJEmrbdUf/pJk7yTvAk4CtgN+BrwBOA14\nBPDlJI9e7TgkSZK0sKrxv4ZmVfapTrIOeCLw58BdgQD/CbwJ+GBVXd2Puw9dS8iRwIdXIxZJkiRp\ntY01qU5yb7pE+rHAzYArgbcDb6qqr84fX1WnJnkH3WJGSZIkTcAQFxaO27gr1af0xzOBNwPHV9XP\nl/jMD4CLxhyHJEmStMGMO6n+MF1V+qTlfqCq3kTXFiJJkqQJGOJuHeM21qS6qh47zvlGSfJHwD7A\nr4FPVdVnRox7EvCkfvcRSZIkLWCICwvHbVUWKgIkuTNwF2CLqnr/mOYM8EHgUXSLHwGeleSTwBOr\n6tJ5H9mB7umOkiRJ0qoZe1Kd5O7A24A95px+f3/tAcAngMdV1SdXMP2hwKOBC+h6tq8BngQ8HDg5\nyQOr6scNsZ8+6tomm2670mklSZKmmgsV2411n+q+Ov1F4Hfo9qL+9LwhXwJ+QZcYr8ShwKXA71fV\nq6rqNcBudI9Evyvw70lutcK5JUmSpBUZd6X6pcBNgHtV1f8kOQJ4yOzFqppJciqw5wrnvzvw4bnV\n6Kq6Fnhuku8BR9Ml1vtW1SXrO3lV7T7q2o03285uI0mStFFyoWK7cSfV+wEfrar/WWTM9/pxK7EZ\nI7bfq6pjk8wAxwKfSfKgFd5DkiRpUGz/aDfux5RvTdfvvJjQVbNX4gfAb4+6WFX/BPw1cE+61pN1\nK7yPJEmStGzjrlT/GNhxiTF3Bb6/wvm/Cey72ICqOjrJTYBXAfdY4X0kSZIGwx7XduOuVJ8E/GGS\nnRa6mGR34EHAiSuc/1+BbZM8bLFBVfV3wBGs4paBkiRJ0qxxJ52votvZ40tJXgzcFiDJLsD9gZfR\n7f7xmhXO/y/AJsAVSw2sqpf3ixd3WOG9JEmSBsGe6nbjfqLiGUkeA7yXbh9p6Hqo/7c//hx4dFWd\nv8L5fwa8ZT3Gv3Ml95EkSRoSd/9oN/b2iKr6ZJI70e0pvRdwS+Ay4DTg7VX103HfU5IkSZqkVek5\n7ivK/zjqepK7VNUZq3FvSZIkrZ+ZSQewEWheqJjkFes5/s7Av7feV5IkSZoW46hUvyDJD6rqTUsN\nTPLbwGfpFzBKkiRp8gp7qluNY0u97wDHJjlosUFJtqPbcu/2wCvHcF9JkiRpKowjqX4I8DPgvUnu\nv9CAJNvQVajvCLyuql48hvtKkiRpDGZq/K+haU6qq+rbwEOBXwMfS3L3udeTbE3XQ70z8Oaqek7r\nPSVJkjQ+M2Tsr6EZyxMVq+p04CDg5sC/9b3TJFkHfAa4G/DOqjpsHPeTJEmSpsnYHlNeVZ+h25v6\ndsCnk+wAfAq4B/BB4MnjupckSZLGp8jYX0Mz7icqvr/vn34dcCawGfBx4AlVNcDuGkmSJA3BajxR\n8ZgktwWeR1epfkxVXTvu+0iSJGk8fPhLu+akOsmohLnodgb5ZXKDXwFUVa3K0xwlSZK0fobYrjFu\n40hsV/Jd8DsnSZKkjUZzUl1VY1vsKEmSpA3P9o92JsSSJElSI/uaJUmSBs5KdTuTakmSpIFzoWI7\n2z8kSZKkRlaqJUmSBm7GQnUzK9WSJElSIyvVkiRJAzdjT3UzK9WSJElSIyvVkiRJA1eTDmAjYFIt\nSZI0cO5T3c72D0mSJKmRlWpJkqSBm4kLFVtZqZYkSZIaWamWJEkaOBcqtjOpliRJGjgXKraz/UOS\nJElqZFItSZI0cDMZ/2tckmyf5LgkFya5Osn5SY5OslXDnPdPcm2SSvKKccRp+4ckSZKmUpIdgVOB\nbYATgDOBewHPBA5IsndVXbyec94CeCdwJbDFuGK1Ui1JkjRwM2TsrzF5I11CfXhVHVhVz6+qBwKv\nA3YBjlrBnMcA64BXjStIMKmWJEkavFqFV6u+Sr0/cD7whnmXjwCuAA5Osvl6zPlI4FDgcODCMYT5\nGybVkiRJmkb79scTq+p6G2ozvMgAABejSURBVJRU1eXAKcDNgb2WM1mSbYC3Ah+rqveMM1Cwp1rS\nBH3gh/856RC0TA++ze9OOgRJq2icCwtnJTl91LWq2n0ZU+zSH88ecf0cukr2zsBnlzHfW+kKyk9b\nxtj1ZlItSZKkabSuP1424vrs+S2XmijJk4FHAH9cVReNIbYbMKmWJEkauNV4+Msyq9GrLskOwNHA\nh6rqn1frPvZUS5IkaRrNVqLXjbg+e/7SJeY5DrgKOGwcQY1iUi1JkjRw07j7B3BWf9x5xPWd+uOo\nnutZ96Tblu8n/cNeKkkB7+ivv7A/97GWYG3/kCRJGrjVWKg4Bif1x/2T3GjuDiD9A1z2pnuAy2lL\nzPMuul1C5tsJuD/wNeB04KstwZpUS5IkaepU1blJTqTb4ePpwOvnXD4S2Bx4S1VdMXsyya79Z8+c\nM8/hC82f5BC6pPqTVfWi1nhNqiVJkgZuNRYqjslhdI8pPzbJfsAZwJ50e1ifDbxw3vgz+uMGr73b\nUy1JkqSpVFXnAnsAx9Ml088BdqR71PheVXXx5KK7PivVkiRJAzfFlWqq6gK6R4svZ+yyK9RVdTxd\nsj4WJtWSJEkDt/xUVKPY/iFJkiQ1slItSZI0cNPc/rFWWKmWJEmSGlmpliRJGjgr1e1MqiVJkgZu\nTI8VHzTbPyRJkqRGVqolSZIGbsYt9ZpZqZYkSZIaWamWJEkaOBcqtrNSLUmSJDWyUi1JkjRwVqrb\nmVRLkiQNnFvqtbP9Q5IkSWpkpVqSJGng3FKvnZVqSZIkqZGVakmSpIFzoWI7k2pJkqSBc6FiO9s/\nJEmSpEZWqiVJkgZuxlp1MyvVkiRJUiMr1ZIkSQPnQsV2JtWSJEkDZ/NHO9s/JEmSpEZWqiVJkgbO\n9o92VqolSZKkRlaqJUmSBm4mk45g7TOpliRJGjj3qW5n+4ckSZLUyEq1JEnSwFmnbrfRVaqTbJ1k\n3aTjkCRJ0nCsuaQ6yXZJ/inJp5P8fZJb9ud3S/IN4CfAz5J8Mcmuk41WkiRp+s2swmto1lT7R5Kt\ngdOA7fpTDwYelOQhwCeBWwJfB7YF7gv8e5K7VdWlk4hXkiRJw7CmkmrgGXQJ9SuBfwYeAbwMOB64\nHLh3VX0PIMlRwN8CfwW8fDmTJzl91LVNNt22JW5JkqSp5e4f7dZa+8eBwH9V1Yuq6htV9QrgZOAA\n4HmzCXXvRcB5wMMnEKckSdKaUavwGpq1Vqm+A/Ceeef+G9gbOHXuyaqqJF8ADlru5FW1+6hrN95s\nuyH+/ZAkSdIyrLWk+mbAFfPOXQZQVT9ZYPxFwOarHZQkSdJaNsSFheO21to/fgpsM+/cFcCPR4y/\nJeAiRUmSJK2qtZZUnw3cde6JqnpNVd1uxPg7At9f9agkSZLWsBlq7K+hWWtJ9enA7kk2W2pgkt+i\n21bv5FWPSpIkaQ1zoWK7NZVUV9XzquomVfWrZQy/Dd2Wem9Z5bAkSZI0cGttoeKyVdU5wDGTjkOS\nJGnauVCx3ZqqVEuSJEnTaKOtVEuSJGl5apBd0ONlUi1JkjRwtn+0s/1DkiRJamSlWpIkaeCGuK/0\nuFmpliRJkhpZqZYkSRo469TtrFRLkiRJjaxUS5IkDZw91e1MqiVJkgbOLfXa2f4hSZIkNbJSLUmS\nNHA+UbGdlWpJkiSpkZVqSZKkgbOnup1JtSRJ0sDZ/tHO9g9JkiSpkZVqSZKkgbP9o52VakmSJKmR\nlWpJkqSBmyl7qluZVEuSJA2cKXU72z8kSZKkRlaqJUmSBm7GWnUzK9WSJElSIyvVkiRJA+fDX9pZ\nqZYkSZIaWamWJEkaOB/+0s6kWpIkaeBcqNjO9g9JkiSpkZVqSZKkgXOhYjsr1ZIkSVIjk2pJkqSB\nm1mF17gk2T7JcUkuTHJ1kvOTHJ1kq2V+fvMkj0/yviRnJrkiyeVJ/jvJc5JsNo44bf+QJEkauKrp\nbP9IsiNwKrANcAJwJnAv4JnAAUn2rqqLl5jmfsB7gJ8BJwEfA7YCHgG8BjgoyX5V9cuWWE2qJUmS\nNK3eSJdQH15Vr589meS1wLOBo4CnLTHHj4AnAB+qql/NmeO5wOeB+wBPB/6xJVDbPyRJkgZuhhr7\nq1Vfpd4fOB94w7zLRwBXAAcn2Xyxearqa1X13rkJdX/+cq5LpPdpjdekWpIkSdNo3/54YlVdr027\nT4hPAW4O7NVwj2v6468b5gBs/5AkLcNnLvrGpEPQMh2y7b0nHYLWoNV4omKS00ddq6rdlzHFLv3x\n7BHXz6GrZO8MfHb9ovuNJ/fHT63w879hUi1JkjRwU7pP9br+eNmI67Pnt1zJ5EmeARwAfA04biVz\nzGVSLUmSpLFbZjV6IpIcBBxNt4jxUVV1zRIfWZJJtSRJ0sCNY2HhKpitRK8bcX32/KXrM2mSA4EP\nAD8G9q2q81YW3vW5UFGSJEnT6Kz+uPOI6zv1x1E91zeQ5DHAh4CLgAdU1VlLfGTZrFRLkiQN3JQ+\n/OWk/rh/khvN3QEkyS2AvYErgdOWM1mSxwPvBH7AGCvUs6xUS5IkaepU1bnAicAOdA9nmetIYHPg\n3VV1xezJJLsm2XX+XEmeBLwL+B5w/3En1GClWpIkafBWY0u9MTmM7jHlxybZDzgD2JNuD+uzgRfO\nG39Gf8zsiST70u3ucSO66vehSeZ9jEur6uiWQE2qJUmSBm5Kt9Sjqs5NsgfwMrrt7x4K/BA4Bjiy\nqi5ZxjR34LrujCePGPNdut1AVsykWpIkSVOrqi4ADl3m2BuUoKvqeOD48UZ1QybVkiRJAzelW+qt\nKS5UlCRJkhpZqZYkSRq4Kd1Sb00xqZYkSRo42z/a2f4hSZIkNbJSLUmSNHDTuqXeWmKlWpIkSWpk\npVqSJGngZlyo2MykWpIkaeBMqdvZ/iFJkiQ1slItSZI0cG6p185KtSRJktTISrUkSdLAWaluZ6Va\nkiRJamSlWpIkaeDKLfWamVRLkiQNnO0f7Wz/kCRJkhpZqZYkSRq4slLdzEq1JEmS1MhKtSRJ0sC5\nULGdSbUkSdLAuVCxne0fkiRJUiMr1ZIkSQNn+0c7K9WSJElSIyvVkiRJA2dPdTuTakmSpIFzn+p2\ntn9IkiRJjaxUS5IkDdyMCxWbWamWJEmSGm1USXWS1yZ5zKTjkCRJWktqFf43NBtVUg08C3jwpIOQ\nJEnSsKyZnuokT17m0F3mjq2q41YpJEmSpI2CPdXt1kxSDbwNlvxdQgH37V/p35tUS5IkLWKI7Rrj\ntpaSaoBfAG8GrlzgWoCXAF8BPrGSyZOcPuraJptuu5IpJUmSNABrKal+IvB64CDg0Ko6ef6AJC8B\nvlJVR27o4CRJktYq2z/arZmkuqrek+QLwDuAzyc5GnhhVV09xnvsPurajTfbzr9tkiRJWtCa2v2j\nqi6oqgcBzwEOA76a5F4TDkuSJGlNc0u9dmsqqZ5VVccAu9P1Vp+S5FVJNp1wWJIkSWvSTNXYX0Oz\nJpNqgKo6A9gTeBXwXOCrLL07iCRJkjR2azapBqiqa6vqJXRb6G1KtwOIJEmS1oPtH+3WzELFxVTV\nfya5K7AFMLaFi5IkSdJybBRJNXRVa+CyScchSZK01lTNTDqENW+jSaolSZK0MjMDbNcYtzXdUy1J\nkiRNAyvVkiRJA1cD3AJv3KxUS5IkSY2sVEuSJA2cPdXtrFRLkiRJjaxUS5IkDZw91e1MqiVJkgZu\nxqS6me0fkiRJUiMr1ZIkSQNXLlRsZqVakiRJamSlWpIkaeBcqNjOpFqSJGng3Ke6ne0fkiRJUiMr\n1ZIkSQNn+0c7K9WSJElSIyvVkiRJA+fDX9qZVEuSJA2c7R/tbP+QJEmSGlmpliRJGji31GtnpVqS\nJElqZKVakiRp4OypbmelWpIkSWpkpVqSJGng3FKvnUm1JEnSwJULFZvZ/iFJkiQ1slItSZI0cLZ/\ntLNSLUmSJDWyUi1JkjRwbqnXzqRakiRp4Fyo2M72D0mSJE2tJNsnOS7JhUmuTnJ+kqOTbLWe82zd\nf+78fp4L+3m3H0ecVqolSZIGblrbP5LsCJwKbAOcAJwJ3At4JnBAkr2r6uJlzHPLfp6dgc8BHwB2\nBQ4FHpbk3lV1XkusVqolSZI0rd5Il1AfXlUHVtXzq+qBwOuAXYCjljnPK+kS6tdW1X79PAfSJefb\n9Pdpkmn9yWTa3Hiz7fwHJUmaeodse+9Jh6D18LbzP5xJxwCw6SrkOdf86gdNX1tfpf42cD6wY1XN\nzLl2C+CHQIBtquqKRebZAvgxMAPcrqoun3PtRsB5wB36e6y4Wm2lWpIkaeBqFV5jsG9/PHFuQg3Q\nJ8anADcH9lpinr2AmwGnzE2o+3lmgE/Pu9+K2FMtSZKksUty+qhrVbX7MqbYpT+ePeL6OcD+dG0d\nn22ch36eFTOpXqZfN/4KYxrN/mVf5l9sTZDfq7XD79Xa4fdqbfH7tbpWI89ZLKlepnX98bIR12fP\nb7mB5lmUSbUkSZLGbmg/ANlTLUmSpGk0W0FeN+L67PlLN9A8izKpliRJ0jQ6qz+O6nXeqT+O6pUe\n9zyLMqmWJEnSNDqpP+7fb333G/2WensDVwKnLTHPacBVwN795+bOcyO6xY5z77ciJtWSJEmaOlV1\nLnAisAPw9HmXjwQ2B949d4/qJLsm2XXePL8A3t2Pf+m8eZ7Rz//p1icqulBRkiRJ0+owuseLH5tk\nP+AMYE+6PaXPBl44b/wZ/XH+biYvAPYB/jrJbsCXgbsAj6R7MMz8pH29+URFSZIkTa0ktwdeBhwA\n3JLuSYofBY6sqkvmjS2AqrrBFoFJtgaOAA4EbgdcDPwb8JKq+n5znCbVkiRJUht7qiVJkqRGJtWS\nJElSI5NqSZIkqZFJtSRJktTIpFqSJElqZFItSZIkNTKpHqAk2yc5LsmFSa5Ocn6So5NsNenYdJ0k\nj07y+iRfSvLzJJXkPZOOS9eX5JZJnpLko0m+neSqJJclOTnJn81/tK4mL8nfJflskgv679fPknw1\nyRFJbjnp+DRakif0/y6sJE+ZdDzSXO5TPTBJdqR7MtE2wAnAmcC96J5MdBawd1VdPLkINSvJ14Df\nA34BfB/YFXhvVT1hooHpepI8DXgT3cMITgK+B9wGOAhYB3wEeEz5L9upkeRXwFeA/6V7ktrmwF7A\nHsCFwF5VdcHkItRC+geAfBPYBNgCeGpVvW2yUUnX8THlw/NGuoT68Kp6/ezJJK8Fng0cBTxtQrHp\n+p5Nl0x/G3gAXcKm6XM28Ajgk1U1M3syyQvoHoP7KLoE+yOTCU8L+K2q+uX8k0mOonuU8d/SPRpZ\nUyJJgHfQPQHvX4DnTjYi6Yb8teSA9FXq/YHzgTfMu3wEcAVwcJLNN3BoWkBVnVRV51jhnG5V9bmq\n+sTchLo//yPgzf3bfTZ4YBppoYS698/9cacNFYuW7XDggcChdP+tkqaOSfWw7NsfT1wgAbgcOAW4\nOd2vQSW1u6Y//nqiUWi5/rA/fmOiUeh6ktwFeDVwTFV9cdLxSKPY/jEsu/THs0dcP4eukr0z8NkN\nEpG0kUpyY+CJ/dtPTTIWLSzJc+l6c9fR9VPfly6hfvUk49J1+v8fvZturcILJhyOtCiT6mFZ1x8v\nG3F99vyWGyAWaWP3auBuwL9W1acnHYwW9Fy6RaWzPgUcUlU/mVA8uqGXAPcA7ltVV006GGkxtn9I\n0pglORx4Dt3uOgdPOByNUFW3raoAt6VbTHon4KtJ7jnZyASQZE+66vQ/VtV/TDoeaSkm1cMyW4le\nN+L67PlLN0As0kYpyTOAY+i2a9u3qn424ZC0hKq6qKo+Stf+dkvgXRMOafD6to930bUrvnjC4UjL\nYlI9LGf1x51HXJ9d8T6q51rSIpI8C3g98D90CfWPJhyS1kNVfZfuh6HfSXKrScczcFvQ/bfqLsAv\n5zzwpeh2qwJ4a3/u6IlFKc1hT/WwzO5zvH+SG83bU/cWwN7AlcBpkwhOWsuSPI+uj/prwIOr6qcT\nDkkrs21/vHaiUehq4O0jrt2Trs/6ZLpika0hmgom1QNSVecmOZHuV5xPp6uozTqS7qlib6kq9wCV\n1kOSFwMvA04H9rflY3ol2Rm4qKoum3f+RsDL6R6OdWpVXTKJ+NTpFyUu+BjyJC+lS6rf6RMVNU1M\nqofnMLrHlB+bZD/gDGBPuj2szwZeOMHYNEeSA4ED+7e37Y/3TnJ8/+efVpVPFZuwJE+iS6ivBb4E\nHN49/O16zq+q4zdwaFrYQ4FXJTkZ+A7dE/puQ/fU0jsBPwKeOrnwJK1VJtUD01er96BLAg6g+w/M\nD+kWVh1pdWaq7AY8ad65O/UvgO/io3qnwR374ybAs0aM+QJw/AaJRkv5d+DOdHtS34NuC9Er6IoK\n7waO9TcNklYiPgFZkiRJauPuH5IkSVIjk2pJkiSpkUm1JEmS1MikWpIkSWpkUi1JkiQ1MqmWJEmS\nGplUS5IkSY1MqiVJkqRGJtWSJElSI5NqSZIkqZFJtSRJktTIpFqSJElqZFItaTCS7JHkHUnOS3JV\nkp8n+XqSv0ty20nHJ0lau1JVk45BklZVkgCvBv4G+DXwGeCbwGbAfYB7Ab8AHldV/29ScUqS1i6T\nakkbvSQvAY4EzgceXlXfmnf9UcB7gE2A+1XVf27wICVJa5rtH5I2akl2AF4MXAM8Yn5CDVBVHwGe\nDWwKvHnOZ1+apJLss9C8/bXjF7h28yR/m+RrSa5I8osk/5HkcQuM3aef56VJ7pXkk0l+1p/bMckF\nfZvKFiO+vtf3Yx+93H8mkqTxM6mWtLE7FLgx8NGq+uYi494G/BDYLcleK71Zki2Bk4FXAtcCxwHv\nBG4NvC/JK0Z89N7Al4CbzvnMVcBbgVsACyXkNwOeAPwIOGGlMUuS2plUS9rY3bc//vtig6rq18BJ\n/dv7N9zvaOAewPOqaveq+quqejpwN+DTwAuS7LbA5/YHDq+q/arquVV1SFVdSJdU/xr4iwU+88fA\nlsBxVXVNQ8ySpEYm1ZI2drfrjxcsY+zsmO1XcqMkt6SrHP93Vf393GtV9UvgeUCAP13g41+rqrfM\nP1lVPwQ+BuyeZPd5l/8CmKFLvCVJE3TjSQcgSVPopiv83O/TLXasJC9d4Pqm/fEuC1z78iLzvhF4\nNF0S/ecASe4O7AX8W1Wdv8J4JUljYlItaWP3I7ok9vbLGDs75icrvNct++Pv969RFlp0+KNRg6vq\npCRnAI9L8pyqupw+uQZuUN2WJG14tn9I2tid3B8ftNigJJsA+/RvT++PM/1xoQLElgucu6w/vq6q\nsshr3wU+u9T+pm+mS8YfP2eB4g8A99WWpClgUi1pY/cOuoV+f5TkdxYZ92RgW+BnwKf6c5f0x4Wq\n3HsscO7LdIn4/VYW6qLeCVxJV6GeXaD49qq6dhXuJUlaTybVkjZqVfUd4BV0/cwfT3LX+WOSHAgc\n0799XlVd2f95ts/50CQ3njP+9sBLFrjXj4H3AnskeXFf/Z5/rx2T3HEFX8dlwPvodhZ5Bd12fS5Q\nlKQpYU+1pCF4GbA58H+Aryf5NPAtukT7PsCe/bi/r6q3zX6oqv4zyRfpttj7cpLPAbcB/pBue7yF\nKtjPAHbq73lwkpOBi+iq4Heh67V+HPCdFXwdbwSeAmwHfKKqvr+COSRJq8CkWtJGr6oK+JskHwKe\nDjyArsf6Jv2QHwJPrKqF9rJ+JPAP/fGvgHOAvwFOBB67wL1+nuQBdG0afwo8im43kYv6zz4b+MwK\nv46vJvkasBsuUJSkqZLuvzWSNDxJbkG3kPGuwGOq6mMTDmlRfbwX0vV937GqZpb4iCRpA7GnWtJg\n9VvTPZxuC70PJjlgwiEt5S/pdgB5owm1JE0XK9WSBi/J7wF/RLe7xtFV9asJh/QbSdbRJdPbAU+l\nq1Lv0v9AIEmaEibVkjTFkuxAt6jxarr9s/+qqr4yyZgkSTdkUi1JkiQ1sqdakiRJamRSLUmSJDUy\nqZYkSZIamVRLkiRJjUyqJUmSpEYm1ZIkSVIjk2pJkiSpkUm1JEmS1MikWpIkSWpkUi1JkiQ1MqmW\nJEmSGplUS5IkSf+/3ToWAAAAABjkbz2IvUXRJNUAADAF+OWdiDx7yHgAAAAASUVORK5CYII=\n","text/plain":["
"]},"metadata":{"tags":[],"image/png":{"width":362,"height":277}}}]},{"cell_type":"markdown","metadata":{"id":"AzKOzG5xuNVi","colab_type":"text"},"source":["We observe that in the attention map, position `n` is not allowed to attend to keys at position `>n`. For example, position `2` cannot attend to position `3`. The effect of this can also be seen on the output:"]},{"cell_type":"code","metadata":{"id":"RPbwjWCptHPE","colab_type":"code","outputId":"f16417d6-e98f-4899-db99-8b83f7089d4b","executionInfo":{"status":"ok","timestamp":1577283161969,"user_tz":-480,"elapsed":7073,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":119}},"source":["print_outputs(output)"],"execution_count":17,"outputs":[{"output_type":"stream","text":["Output:\n","[[0. ]\n"," [0.6921095]\n"," [1. ]\n"," [1.1433128]\n"," [2.4281611]]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"2gMVyyfYujw2","colab_type":"text"},"source":["We can see that now, position `0` cannot attend to position `1` at all. However, position `1` and `2` are able to.\n","\n","In the same way, position `4` can attend to all the other positions."]}]} -------------------------------------------------------------------------------- /labs/hello_gpu.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"hello_gpu.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"yKcbbrwlarek","colab_type":"text"},"source":["# Hello, GPU!\n","\n","The graphics processing unit (GPU) is a powerful hardware acceleration card originally developed for 3D graphics. It is also ideal for various parallelizable workloads, common in machine learning and deep learning.\n","\n","There are a couple of possible GPUs that you might get assigned (for **free**!) on Google Colab. Roughly ranked in order of performance, they are:\n","\n","* P100\n","* T4 (rare)\n","* P4\n","* K80\n","\n","These server-grade GPUs (even the K80) are more powerful than most laptop or even desktop GPUs. The K80 is roughly equivalent to a GTX 1060 in training performance (~4 TFLOPS), but has much more memory (VRAM). The fastest card on the list, the P100, is slightly faster than a GTX 1080 Ti and has faster and more memory (16GB HBM2). The T4 can match a GTX 1080 Ti by using mixed precision.\n","\n","You can use the `nvidia-smi` command to check which GPU you are allocated.\n","\n"]},{"cell_type":"code","metadata":{"id":"apV6X-9Karem","colab_type":"code","outputId":"e2df0338-5680-4e52-f1e0-82a5f01edb32","executionInfo":{"status":"ok","timestamp":1577644034251,"user_tz":-480,"elapsed":8993,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":323}},"source":["!nvidia-smi"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Sun Dec 29 18:27:10 2019 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 440.44 Driver Version: 418.67 CUDA Version: 10.1 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","|===============================+======================+======================|\n","| 0 Tesla P100-PCIE... Off | 00000000:00:04.0 Off | 0 |\n","| N/A 40C P0 26W / 250W | 0MiB / 16280MiB | 0% Default |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: GPU Memory |\n","| GPU PID Type Process name Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"JxPZB-VE691Z","colab_type":"text"},"source":["You might see an error: `\n","NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running. `\n","\n","If you see the above error, please go to **Runtime > Change runtime type > Hardware accelerator** and select **GPU**."]},{"cell_type":"markdown","metadata":{"id":"HXZhBsEj6JUg","colab_type":"text"},"source":["You can also install and use other utilities like `gpustat` in the Colab environment"]},{"cell_type":"code","metadata":{"id":"tS5MdBSBaret","colab_type":"code","outputId":"38299c2d-bf26-413e-ef41-e447f2a9f6b1","executionInfo":{"status":"ok","timestamp":1577644050344,"user_tz":-480,"elapsed":25076,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":323}},"source":["!pip install gpustat\n","!gpustat"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting gpustat\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/b4/69/d8c849715171aeabd61af7da080fdc60948b5a396d2422f1f4672e43d008/gpustat-0.6.0.tar.gz (78kB)\n","\r\u001b[K |████▏ | 10kB 19.4MB/s eta 0:00:01\r\u001b[K |████████▍ | 20kB 4.2MB/s eta 0:00:01\r\u001b[K |████████████▋ | 30kB 5.7MB/s eta 0:00:01\r\u001b[K |████████████████▊ | 40kB 5.9MB/s eta 0:00:01\r\u001b[K |█████████████████████ | 51kB 4.8MB/s eta 0:00:01\r\u001b[K |█████████████████████████▏ | 61kB 5.3MB/s eta 0:00:01\r\u001b[K |█████████████████████████████▍ | 71kB 5.8MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 81kB 4.1MB/s \n","\u001b[?25hRequirement already satisfied: six>=1.7 in /usr/local/lib/python3.6/dist-packages (from gpustat) (1.12.0)\n","Requirement already satisfied: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.6/dist-packages (from gpustat) (7.352.0)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from gpustat) (5.4.8)\n","Collecting blessings>=1.6\n"," Downloading https://files.pythonhosted.org/packages/03/74/489f85a78247609c6b4f13733cbf3ba0d864b11aa565617b645d6fdf2a4a/blessings-1.7-py3-none-any.whl\n","Building wheels for collected packages: gpustat\n"," Building wheel for gpustat (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for gpustat: filename=gpustat-0.6.0-cp36-none-any.whl size=12620 sha256=bd8bab9960b4b512117613843fbc6d61124afcaf3bcf96e681b6d702e2fd1898\n"," Stored in directory: /root/.cache/pip/wheels/48/b4/d5/fb5b7f1d040f2ff20687e3bad6867d63155dbde5a7c10f4293\n","Successfully built gpustat\n","Installing collected packages: blessings, gpustat\n","Successfully installed blessings-1.7 gpustat-0.6.0\n","\u001b[1m\u001b[37mbc94c57c4335 \u001b[m Sun Dec 29 18:27:25 2019 \u001b[1m\u001b[30m418.67\u001b[m\n","\u001b[36m[0]\u001b[m \u001b[34mTesla P100-PCIE-16GB\u001b[m |\u001b[31m 40'C\u001b[m, \u001b[32m 0 %\u001b[m | \u001b[36m\u001b[1m\u001b[33m 0\u001b[m / \u001b[33m16280\u001b[m MB |\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"qZq7W5AW6OM9","colab_type":"text"},"source":["Here's an example of how you can check your GPU device via TensorFlow."]},{"cell_type":"code","metadata":{"id":"kXpFDB8darex","colab_type":"code","outputId":"4f266793-23f9-4907-8e95-8bb11fe64f22","executionInfo":{"status":"ok","timestamp":1577644053252,"user_tz":-480,"elapsed":27978,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":550}},"source":["import tensorflow as tf\n","from tensorflow.python.client import device_lib\n","\n","device_lib.list_local_devices()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["

\n","The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.
\n","We recommend you upgrade now \n","or ensure your notebook will continue to use TensorFlow 1.x via the %tensorflow_version 1.x magic:\n","more info.

\n"],"text/plain":[""]},"metadata":{"tags":[]}},{"output_type":"execute_result","data":{"text/plain":["[name: \"/device:CPU:0\"\n"," device_type: \"CPU\"\n"," memory_limit: 268435456\n"," locality {\n"," }\n"," incarnation: 12901138985227046086, name: \"/device:XLA_CPU:0\"\n"," device_type: \"XLA_CPU\"\n"," memory_limit: 17179869184\n"," locality {\n"," }\n"," incarnation: 6107419359804291040\n"," physical_device_desc: \"device: XLA_CPU device\", name: \"/device:XLA_GPU:0\"\n"," device_type: \"XLA_GPU\"\n"," memory_limit: 17179869184\n"," locality {\n"," }\n"," incarnation: 897582802262704042\n"," physical_device_desc: \"device: XLA_GPU device\", name: \"/device:GPU:0\"\n"," device_type: \"GPU\"\n"," memory_limit: 15956161332\n"," locality {\n"," bus_id: 1\n"," links {\n"," }\n"," }\n"," incarnation: 16757169888688836939\n"," physical_device_desc: \"device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\"]"]},"metadata":{"tags":[]},"execution_count":3}]}]} -------------------------------------------------------------------------------- /labs/linear_regression_sgd.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"linear_regression_sgd.ipynb","provenance":[],"collapsed_sections":["g6VpDXLOSmJr"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"cells":[{"cell_type":"markdown","metadata":{"id":"g6VpDXLOSmJr","colab_type":"text"},"source":["## Setup\n","\n","This section downloads, installs and imports some Python libraries we are using in this notebook."]},{"cell_type":"code","metadata":{"id":"wLkD5IpCSxDq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":0},"outputId":"ef729807-fe6f-498c-e315-45ae7e619d5a","executionInfo":{"status":"ok","timestamp":1578131833196,"user_tz":-480,"elapsed":49271,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}}},"source":["!pip3 install tensorflow>=2.0 gpustat -Uq"],"execution_count":1,"outputs":[{"output_type":"stream","text":["\u001b[31mERROR: tensorboard 2.0.2 has requirement grpcio>=1.24.3, but you'll have grpcio 1.15.0 which is incompatible.\u001b[0m\n","\u001b[31mERROR: google-colab 1.0.0 has requirement google-auth~=1.4.0, but you'll have google-auth 1.10.0 which is incompatible.\u001b[0m\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","id":"zNbGLsDSUe3W","outputId":"8ce7e9fd-6030-45a8-c14b-dc623a15912d","executionInfo":{"status":"ok","timestamp":1578131834816,"user_tz":-480,"elapsed":50875,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":0}},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n","\n","import tensorflow\n","print(\"TensorFlow version:\", tensorflow.__version__)\n","\n","import tensorflow.compat.v2 as tf\n","from tensorflow.keras import layers\n","\n","from random import random, randint\n","from IPython.display import clear_output"],"execution_count":2,"outputs":[{"output_type":"stream","text":["TensorFlow version: 2.0.0\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"r_mTM8H7A2kA","colab_type":"text"},"source":["# Linear Regression"]},{"cell_type":"markdown","metadata":{"id":"0HTjhbkDA-A8","colab_type":"text"},"source":["## Generate synthetic data"]},{"cell_type":"code","metadata":{"id":"mqQ20tRMA3_N","colab_type":"code","outputId":"fab41646-b4d0-439f-b62d-a3966e95f909","executionInfo":{"status":"ok","timestamp":1578131834817,"user_tz":-480,"elapsed":50866,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["A = random()\n","C = random()\n","\n","X = np.linspace(0.0, 1.0, num=100, endpoint=True)\n","noise = np.random.rand(100)/40\n","Y = A * X + C + noise\n","\n","plt.scatter(X, Y)\n","plt.show()"],"execution_count":3,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAeK0lEQVR4nO3dcYxdZ5nf8e/Pk0kyWQKTracqmTjY\nICeQEGTDVZbKKl2yBLug2hapaoNQk4o2guICZhut0aIFGa0wRE3YqpHApVFppdRmWRTNKqwsdp2U\nNm1Yj2VvWBu8OIYlHpCYxZ6glQczdp7+cc+dnLlzzr3nzpw7d+65v49kee6554zfi8Pjd573eZ9X\nEYGZmVXXml4PwMzMusuB3sys4hzozcwqzoHezKziHOjNzCruml4PoNnatWtj/fr1vR6GmVlfOX78\n+N9FxFjWe6su0K9fv57JycleD8PMrK9I+tu895y6MTOrOAd6M7OKc6A3M6s4B3ozs4pzoDczqzgH\nejOzinOgNzOrOAd6M7OKc6A3M6s4B3ozs4pzoDczqzgHejOzinOgNzOruELdKyVtA/4IGAK+GhEH\nmt5/AHgYmEou/eeI+Gry3lXge8n1n0TE9hLGbWbWc0+emOLhI2f46cwsN4+O8NDW29m5eTz3epFn\nu0ER0foGaQj4G+Be4DxwDHh/RJxO3fMAUIuIPRnP/31EvKrogGq1WrhNsZmtdk+emOJT3/wes3NX\n568JiNTvzdfHk4AOLHp2ZHiIz7/vriUHe0nHI6KW9V6RGf3dwNmIOJd8s0PADuB0y6fMzCqoMROf\nmpld9F40/d58fWpmlk9983tcP7xmQZAHmJ27ysNHznRlVl8kRz8OvJh6fT651uw+Sc9L+oakdanr\n10ualPScpJ3LGayZWS81ZvFZQb6o2bmrXLw0l/neT5fxfVspazH2T4H1EfEW4NvA11LvvS75ceID\nwJckvaH5YUkPJv8YTE5PT5c0JDOzcj185MyimXiZbh4d6cr3LRLop4D0DP0WXll0BSAifhERl5OX\nXwXelnpvKvn9HPAMsLn5D4iIgxFRi4ja2FjmkYdmZj3XrRk31HP0jfx92YoE+mPARkkbJF0L7AYm\n0jdIem3q5Xbg+8n1myRdl3y9FtiCc/tm1qdazbiV87r5epbx0ZFlLcS203YxNiKuSNoDHKFeXvl4\nRJyStB+YjIgJ4GOStgNXgAvAA8njbwK+Iull6v+oHEhX65iZ9ZOHtt6eWy0DtCy1zMvrC3h23z1d\nHXfb8sqV5vJKM1tt0jXvrxkZRoKZS3Md1b9vOXA0M9iPj46UEuiXW15pZjYQsjYxwcKa95nZOUaG\nh3h016aOUi15Pw10Ky+f5kBvZsbiDVBl17w37l2p3bBpDvRmZmSXTs7OXc0tp1xKBc7OzeMrEtib\nOdCb2UBrt1iap1s1793gQG9mAyurX00RK5VbL4sDvZkNhKyF1qXsdB1fwdx6WRzozayy0mmZdEfJ\nxkLrUoJ8t2veu8GB3swqqTkt07xjaHbuKkMSVzP2Eo2ODHP5yss9KYXsBgd6M6ukImmZqxGMDA8t\nCuif3X7n/PdY6VLIbnCgN7NKKlr+eN01a7h+eE3mTtd+DezNHOjNrJJuHh0pVDK51J2u/cSHg5tZ\nJT209XZGhocWXMvrJNnY6VpVDvRmVilPnphiy4Gj7D18kuuuWcNNNwwj6hUzj+7alBvsu9lrvtec\nujGzymiutMlKy+Ttgu2nna6d8ozezPpSY+a+Yd9TbDlwdL5mPq8BWUNWSqefSyeL8IzezPpOXqfJ\nIg3IetlFslcc6M2sb7RqQNZqA1RzWqZXXSR7xYHezPpCkQZkeRugqpyWKaJQjl7SNklnJJ2VtC/j\n/QckTUs6mfz6N6n37pf0w+TX/WUO3swGR9EGZM2VNt08dLtftJ3RSxoCHgPuBc4DxyRNZBzyfTgi\n9jQ9+5vAZ4Aa9VYTx5NnL5YyejOrvE77xQ/CBqhOFZnR3w2cjYhzEfFr4BCwo+D33wp8OyIuJMH9\n28C2pQ3VzAZNI13T6aEgVd8A1akigX4ceDH1+nxyrdl9kp6X9A1J6zp81sxskXbpmuYyybQqb4Dq\nVFl19H8KrI+It1CftX+tk4clPShpUtLk9PR0SUMys37XKlg38u/jORudqrwBqlNFqm6mgHWp17ck\n1+ZFxC9SL78KfDH17G83PftM8x8QEQeBgwC1Wm1xbZSZVV76BKjXjAwjLe4h39B8AEhzNY4rbRYq\nMqM/BmyUtEHStcBuYCJ9g6TXpl5uB76ffH0EeLekmyTdBLw7uWZmNi+diw/qC6oXL81l3tscxHdu\nHp+f2bvSJlvbGX1EXJG0h3qAHgIej4hTkvYDkxExAXxM0nbgCnABeCB59oKkz1H/xwJgf0Rc6MLn\nMLM+sZyzW/POax20DVCdUmTsIuulWq0Wk5OTvR6GmXVB1qan9FmurQj40YH3dmtofU/S8YioZb3n\npmZmtmKyZu5Fp5peXF06t0Aws67rdNNTMy+uLo8DvZktS1bOPZ0vL9KjptloUnWTdY6rdc6B3syW\nLK9dMCxsB9xJkG8unbTlc47ezJasyEEfrTY9NR/r5xRNdzjQm1nHGqc75eXc08E9bxG1cYar69+7\nz+WVZtaRojn3Rs07ZO9cdVAvl8srzaw0RXPu6Xy9d672lhdjzawjnXSFbOTrn913jwN7D3lGb2Yd\n6XTj0tTMLFsOHOXJE1Ptb7aucKA3s448tPX2RX3gR4aHuOmG4dxnGmkcB/vecKA3s47kdYv8zD+/\ns+VBID71qXecozezBdrtdIXW3SJbtTrwqU+94UBvZvNa7XQFCv8DkFdj78ZkveFAb2bz8na6fuLw\nyQXthLNaHaQ9tPV2n/q0ijhHb2bzWqVWmrdWtsq5+9Sn1cUzejObd/PoSEethFv9w+BTn1YPz+jN\nbF5W6WQrzrn3B8/ozWxeurVwu5m9c+79o9CMXtI2SWcknZW0r8V990kKSbXk9XpJs5JOJr++XNbA\nzaw7dm4e59l99/ClXZsWze4bbYWdc+8vbWf0koaAx4B7gfPAMUkTEXG66b4bgY8D3236Fi9ExKaS\nxmtmJWlXL5+e3bcqqbTVr0jq5m7gbEScA5B0CNgBnG6673PAF4CHSh2hmZUuq15+7+GTfOLwyfn2\nwo3FVAf2/lck0I8DL6Zenwd+K32DpLcC6yLiKUnNgX6DpBPAL4FPR8T/bv4DJD0IPAhw6623djB8\nM2sna+aeVS9ftEbe+s+yq24krQEeAX434+2fAbdGxGbgk8ATkl7dfFNEHIyIWkTUxsbGljskM0s0\nZu5TM7MErwTxdgut7ktTLUUC/RSwLvX6luRaw43Am4FnJP0YeDswIakWEZcj4hcAEXEceAG4rYyB\nm1l7eTtdh9R8Wuti7ktTHUUC/TFgo6QNkq4FdgMTjTcj4qWIWBsR6yNiPfAcsD0iJiWNJYu5SHo9\nsBE4V/qnMLNMecH6akTbennXyFdH20AfEVeAPcAR4PvA1yPilKT9kra3efwdwPOSTgLfAD4cEReW\nO2gzK6ZVsL7umjXzPeSb5/euka8WHw5uVmHtDvJuHNINLqPsd60OB/fOWLMKa7fT1We6DgYHerM+\nky6XfM3IMBLMXJpr2yN+w76nFnWgBC+6DgIHerM+0pyKmZmdm3+v3SEheZ0pvehafQ70Zn2gMYsv\nUv/+2YlTXL7y8qJTou572zh/cnzKh4EMIAd6s1UqHdzTpzu1k57lN8zOXeXpH0zz+ffd5UXXAeRA\nb7YKNadoyqiN++nMrHvXDCgfPGK2CmXtaF0u5+IHlwO92SpUtBJmdGR4ftNTK87FDzanbsxWWLs+\n8ND+7NbGRqfGc1sOHM29f9y5+IHnQG+2AvIWVvNaAj+09fZFO1obz2UF7qz7m/8xsMHlQG/WZe0W\nVhu7U5dzupNPg7JWHOjNuqzIwmpWTr7TChlX1FgeB3qzEmXl34ssrK6R2LDvqUItDcw65UBvVpKs\nc1g/9c3vMXrDMBcvLd7ElHY16SKb19LAwd6Ww+WVZiXJO80pgkWHfDT6v7c76clH+lkZHOjNSpKX\nopmZnZs/5EPUq2Ye3bWJHx94Ly8XOA/C3SVtuZy6MVuCrFx8q9r3mdk5RoaHeHTXpgVpmHb18o17\nzJbDM3qzDjVy8VMzswSv5NLf+caxluewZqVhHtp6e8tnvKPVylAo0EvaJumMpLOS9rW47z5JIamW\nuvap5LkzkraWMWizXnjyxBRbDhzlE4dPZubiG90hx1vMwJvTMDs3j88/I15padBI8XjDk5WhbepG\n0hDwGHAvcB44JmkiIk433Xcj8HHgu6lrdwC7gTuBm4E/l3RbRJTbrcmsy9qdvQoLu0PmtSTISsO4\n/t26rciM/m7gbESci4hfA4eAHRn3fQ74AvCr1LUdwKGIuBwRPwLOJt/PrK8U2fSUDuJZKRmnYaxX\nigT6ceDF1OvzybV5kt4KrIuIpzp9Nnn+QUmTkianp6cLDdxsJbWrfGkO4s0pGadhrJeWXXUjaQ3w\nCPDAUr9HRBwEDgLUarUyzlgwK1Wr6pi87pBOydhqUSTQTwHrUq9vSa413Ai8GXhG9c0f/wiYkLS9\nwLNmfcHdIa2fFQn0x4CNkjZQD9K7gQ803oyIl4C1jdeSngH+Q0RMSpoFnpD0CPXF2I3AX5Y3fLOV\n4e6Q1s/aBvqIuCJpD3AEGAIej4hTkvYDkxEx0eLZU5K+DpwGrgAfdcWN9au8VEyRg0TMeklRYAv2\nSqrVajE5OdnrYZgVklV26ZSO9YKk4xFRy3rPLRDMmnQyQ89rZNZ8kIhZLznQm6XktRqG7FbBeWWX\nbkRmq4l73ZiltJqhZ8lrOOZGZLaaONDbwGr0rtmw7ym2HDjKkyemcmfiUzOz8/ekeQes9QOnbmyg\nNPLvUzOziFcO6p6amWXv4ZOLDu5Oy0rjuOzS+oEDvQ2M5vx7c1AvUn+WtdDqHbC22jl1YwOjSGOy\nIrzQav3Ggd4GRlkB2gut1m+curG+1knNe5Fj+9JGR4a5fOXlRZuhvNBq/cYzeutbeUf6NVfGNGRV\nyKjp94aR4SE+u/1Otxq2SnALBOsLWTP3RvVMltGRYSSYuTS3YKaf9xOA+9VYv2vVAsGB3la9vH4y\nnSysNkop83rHm/W7VoHeqRtb9fJ2qw6pOeGSL10v3yq9Y1ZFDvS26uVVy1yNWJRzL6JVSwOzKnKg\nt1Wr0aIgL7nYWBwdX0K5o2vhbZA40NuqlK6oydIoc9y5eZxn993Dl3Zt6mh271p4GySuo7dVqdUu\n1qwF1eaeM69Jqm4uXppb0NMGXAtvg8eB3lalvNSKgGf33ZP5no/6M8tWKNBL2gb8EfUzY78aEQea\n3v8w8FHgKvD3wIMRcVrSeuD7QGPl67mI+HA5Q7cqagTlvLz8zaMjHQduNx2zQdc20EsaAh4D7gXO\nA8ckTUTE6dRtT0TEl5P7twOPANuS916IiE3lDtuqKKtePm1keIh3vnGsoxOgzKzYYuzdwNmIOBcR\nvwYOATvSN0TEL1Mvf4NiHV/NFmiXl//8++7i6R9Md3QClJkVC/TjwIup1+eTawtI+qikF4AvAh9L\nvbVB0glJ/0vSP8n6AyQ9KGlS0uT09HQHw7cqaZeX37l53Ge0mi1BaeWVEfFYRLwB+D3g08nlnwG3\nRsRm4JPAE5JenfHswYioRURtbGysrCFZnyly/qrPaDXrXJFAPwWsS72+JbmW5xCwEyAiLkfEL5Kv\njwMvALctbahWVY2NUY3j/dKaSyF9RqtZ54pU3RwDNkraQD3A7wY+kL5B0saI+GHy8r3AD5PrY8CF\niLgq6fXARuBcWYO31a9dhUzW8X6tGpD5jFazzrUN9BFxRdIe4Aj18srHI+KUpP3AZERMAHskvQuY\nAy4C9yePvwPYL2kOeBn4cERc6MYHsdWnOYinK2SA3DbDjSDfab28mWVzm2LrmkY6plnWyU3NBPzo\nwHu7ODqzamnVptg7Y610jXRNXp+amdm5tt/Di6tm5XGgt1K12/RUhBdXzcrlQG+laDeLbxgZHuL6\n4TVcvJQ9q/cJUGblc6C3ZSs6i28EcSDzaEAfvG3WHQ70tmytWhc0jKcO9G60Eb5+eM2iw7vNrHwO\n9NaRdF18uud7K1nNyGZm5xgZHuLRXZsc4M26zIHeMmVtdAIWBet20jP5vGZkDvRm3eVAb4vkbXS6\nfnhN4Wqa5pz73sMnM+9zMzKz7vOZsbZI3uy7XYqmodFSOD1TdzMys95xoLdFljPLbrQuaE7HuBmZ\nWe84dTPgsnLxN4+OLKl1QavA7WZkZr3jXjcDLKv+fWR4iPveNs6fHJ/KrHMHFlXduETSrPfc68Yy\n5eXin/7BNJ9/312LAvrewycd0M36kHP0A6zVsXw7N4/z7L57eHTXJi5feZmLl+YIXqnAefJEq7Nn\nzGw1caAfYEUqYVrVv5tZf3CgH2BFKmF8GLdZ/3OOfkC0OtKvVSVMXgWO69/N+oerbios3Tq4cQ5r\nQ1a3yCJtD/KeNbPealV1Uyh1I2mbpDOSzkral/H+hyV9T9JJSf9H0h2p9z6VPHdG0talfwwr4skT\nU2w5cJT1+55i7+GT87Px5n/Om/PsjVLLqZnZBYuuAJ9/312Mj44gsne9mtnq1jZ1I2kIeAy4FzgP\nHJM0ERGnU7c9ERFfTu7fDjwCbEsC/m7gTuBm4M8l3RYRSz9+yHI118W3+1ktnWdvteiatdPVzPpH\nkRn93cDZiDgXEb8GDgE70jdExC9TL3+DV2LMDuBQRFyOiB8BZ5PvZ11QpC98WjrP7kVXs+oqEujH\ngRdTr88n1xaQ9FFJLwBfBD7W4bMPSpqUNDk9PV107Nakk6DcXF3jpmNm1VVaeWVEPBYRbwB+D/h0\nh88ejIhaRNTGxsbKGtLAaOTl26VqlPw+mpzutPfwSbYcOMqTJ6bcdMyswooE+ilgXer1Lcm1PIeA\nnUt81jqUXkTN0gju46MjPLprE1/K2ekKXnQ1q6oidfTHgI2SNlAP0ruBD6RvkLQxIn6YvHwv0Ph6\nAnhC0iPUF2M3An9ZxsCtrlVefjyjLn7LgaNedDUbMG0DfURckbQHOAIMAY9HxClJ+4HJiJgA9kh6\nFzAHXATuT549JenrwGngCvBRV9yUKy8vL+DZffcUvt+LrmbVVWhnbER8C/hW07U/SH398RbP/iHw\nh0sdoLXW6c5V73Q1GzxugbDKpXerZvV/f2jr7Zk7V/MWUTu938z6n5uarWLNu1VnZueWvYi6c/O4\nF13NBox73axiWw4cza2maWic0Wpmg80nTPWBrIZiRRZIvYhqZu040K8CzT1qGmmZ0RuGuXhpruWz\nXkQ1s3aco18F8hqKRbBot2qaF1HNrAgH+h5qtC7Iy8O/NDu3YOF0dGSYm24Ynv+6uY2BmVkWp256\npDldk+Xm0RF2bh5fVBGTl+oBXD1jZot4Rt8j7VoKt0rL+MBuM+uEZ/Q90qpaJqtHTZFnXYFjZlkc\n6Luk1WHckN+KoBHkHz5yhr2HT/rAbjNbNqduuiDv/NX0gmle//d3vnFsyc+6AsfMsjjQl6hRRfOJ\nwyfb5tDzWhE8/YPpJT/rhVgzy+LUTUmKVNE059CzKmr2Hj655GfNzLI40JekyMHcN4+OLDl37/y7\nmS2VUzclaVfx4vy7mfWKA31JWs24nX83s15y6mYJstIveQd6pIO08+9m1guFZvSStkk6I+mspH0Z\n739S0mlJz0v6C0mvS713VdLJ5NdEmYPvhbzSSWh/AEir4/3MzLql7Yxe0hDwGHAvcB44JmkiIk6n\nbjsB1CLikqSPAF8EdiXvzUbEppLH3TOt2g88u++eljNxH+NnZr1QJHVzN3A2Is4BSDoE7ADmA31E\nPJ26/zngg2UOcjVZSvuB5nNfrx9es+DcV6dpzKybiqRuxoEXU6/PJ9fyfAj4s9Tr6yVNSnpO0s6s\nByQ9mNwzOT09XWBIvZOXZgnIbBecde7rr+Ze5tFdm9r+BGBmVoZSq24kfRCoAQ+nLr8uOcfwA8CX\nJL2h+bmIOBgRtYiojY2NlTmk0mWVPzZklUu606SZ9VqRQD8FrEu9viW5toCkdwG/D2yPiMuN6xEx\nlfx+DngG2LyM8fZcuvwxS3MQd6dJM+u1IoH+GLBR0gZJ1wK7gQXVM5I2A1+hHuR/nrp+k6Trkq/X\nAltI5fb71c7N4zy77x6U8346iLvSxsx6rW2gj4grwB7gCPB94OsRcUrSfknbk9seBl4F/HFTGeWb\ngElJfwU8DRxoqtbpiUbzsQ37nlrWMXxFgrh3uppZrxXaMBUR3wK+1XTtD1Jfvyvnuf8L3LWcAZat\n1TF8QMs+NM2KlEs2nu/k+5qZlUkR0esxLFCr1WJycrJr3z/vMO7RkWEuX3m55c7WLO2alJmZrQRJ\nx5PCl0UGrgVC3iLozOzcomvphdW8YO52BWa22g1cU7NOF0GnZmbZe/hky46TZmar2cAF+qzF0bzq\nmYbm5Jbr4M2snwxc6ia9ODo1M4tYHMiLcB28mfWLgZvRwyt18OOjI0sK8uA6eDPrH5Wb0XdSBbPU\nWbnr4M2sn1RqRp/XKz5v4TRvVj46Mpybx/eJT2bWbyo1o2/VQCwrMOdtePrs9jvnv5/r482s31Uq\n0OelYqZmZtly4OiiYN1u16oDu5lVQaV2xubtem1oVNiMe4ZuZhXTamdspXL0rXrFwytllN70ZGaD\npFKBvl2v+DRvejKzQVGpQA8La+Tb8aYnMxsElQv0De3SOOBNT2Y2GCpVdZPWrtWBNz2Z2aCobKCH\nhS2E3TfezAZVpQN9mvvGm9mgKpSjl7RN0hlJZyXty3j/k5JOS3pe0l9Iel3qvfsl/TD5dX+Zgzcz\ns/baBnpJQ8BjwD8D7gDeL+mOpttOALWIeAvwDeCLybO/CXwG+C3gbuAzkm4qb/hmZtZOkRn93cDZ\niDgXEb8GDgE70jdExNMRcSl5+RxwS/L1VuDbEXEhIi4C3wa2lTN0MzMrokigHwdeTL0+n1zL8yHg\nzzp5VtKDkiYlTU5PTxcYkpmZFVVqHb2kDwI14OFOnouIgxFRi4ja2NhYmUMyMxt4RQL9FLAu9fqW\n5NoCkt4F/D6wPSIud/KsmZl1T5FAfwzYKGmDpGuB3cBE+gZJm4GvUA/yP0+9dQR4t6SbkkXYdyfX\nzMxshbSto4+IK5L2UA/QQ8DjEXFK0n5gMiImqKdqXgX8sSSAn0TE9oi4IOlz1P+xANgfERe68knM\nzCxTpfrRm5kNqoHpR29mZos50JuZVZwDvZlZxTnQm5lV3KpbjJU0DfxtCd9qLfB3JXyffuHPW23+\nvNVV1md9XURk7jhddYG+LJIm81agq8ift9r8eatrJT6rUzdmZhXnQG9mVnFVDvQHez2AFebPW23+\nvNXV9c9a2Ry9mZnVVXlGb2ZmONCbmVVe3wf6AgeXXyfpcPL+dyWtX/lRlmc5B7X3o3afN3XffZJC\nUt+W5BX5rJL+ZfL3e0rSEys9xjIV+G/5VklPSzqR/Pf8nl6MsyySHpf0c0l/nfO+JP2n5H+P5yW9\ntbQ/PCL69hf1tskvAK8HrgX+Crij6Z5/B3w5+Xo3cLjX4+7y530ncEPy9Ueq/nmT+24EvkP9vOJa\nr8fdxb/bjcAJ4Kbk9T/s9bi7/HkPAh9Jvr4D+HGvx73Mz/wO4K3AX+e8/x7qx7AKeDvw3bL+7H6f\n0bc9uDx5/bXk628Av6OkaX4fWs5B7f2oyN8vwOeALwC/WsnBlazIZ/23wGMRcREgFh7y02+KfN4A\nXp18/Rrgpys4vtJFxHeAVudx7AD+e9Q9B4xKem0Zf3a/B/oih4/P3xMRV4CXgH+wIqMr33IOau9H\nbT9v8uPtuoh4aiUH1gVF/m5vA26T9Kyk5yRtW7HRla/I5/0s8EFJ54FvAf9+ZYbWM53+/7uwtidM\nWX9KHdT+T3s9lm6RtAZ4BHigx0NZKddQT9/8NvWf1L4j6a6ImOnpqLrn/cB/i4j/KOkfA/9D0psj\n4uVeD6zf9PuMvsjh4/P3SLqG+o+Av1iR0ZVvOQe196N2n/dG4M3AM5J+TD2vOdGnC7JF/m7PAxMR\nMRcRPwL+hnrg70dFPu+HgK8DRMT/A66n3gCsqgr9/3sp+j3Qtz24PHl9f/L1vwCORrLy0YeWc1B7\nP2r5eSPipYhYGxHrI2I99TWJ7RHRj2dRFvlv+Unqs3kkraWeyjm3koMsUZHP+xPgdwAkvYl6oJ9e\n0VGurAngXyXVN28HXoqIn5Xxjfs6dRPFDi7/r9R/5DtLfSFkd+9GvDwFP2/mQe09G/QyFPy8lVDw\nsx4B3i3pNHAVeCgi+vKn04Kf93eB/yJpL/WF2Qf6eJKGpP9J/R/qtcm6w2eAYYCI+DL1dYj3AGeB\nS8C/Lu3P7uP/3czMrIB+T92YmVkbDvRmZhXnQG9mVnEO9GZmFedAb2ZWcQ70ZmYV50BvZlZx/x+A\nQwZyfIAWegAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"uvoIcqD9P8wt","colab_type":"text"},"source":["## Fit a Linear Regression Model"]},{"cell_type":"code","metadata":{"id":"oNJGbUeMBwmk","colab_type":"code","outputId":"895fc9f6-8a59-4a81-9ab3-5c870844c28f","executionInfo":{"status":"ok","timestamp":1578131840070,"user_tz":-480,"elapsed":9,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":540}},"source":["model = tf.keras.models.Sequential([\n"," layers.Dense(1),\n","])\n","\n","model.compile(loss=\"mae\",\n"," optimizer=\"sgd\")\n","\n","reg_X = np.linspace(-0.1, 1.1, num=100, endpoint=True)\n","\n","loss_list = []\n","\n","loss = model.evaluate(X, Y, batch_size=100, verbose=0)\n","loss_list.append(loss)\n","\n","while loss > 0.007:\n"," clear_output(wait=True)\n"," model.fit(X, Y, epochs=1, batch_size=5, verbose=0)\n"," loss = model.evaluate(X, Y, batch_size=25, verbose=0)\n"," loss_list.append(loss)\n","\n"," weights = model.layers[0].get_weights()\n"," gradient = weights[0][0][0]\n"," bias = weights[1][0]\n"," \n"," reg_Y = gradient * reg_X + bias\n","\n"," fig, axs = plt.subplots(2, figsize=(10,8))\n"," fig.suptitle(\"Linear regression with SGD\")\n"," axs[0].plot(reg_X, reg_Y, color=\"r\")\n"," axs[0].plot(reg_X, reg_Y, color=\"r\")\n"," axs[0].scatter(X, Y, alpha=0.5)\n"," title = \"grad: \" + str(round(gradient, 1)) + \" bias: \" + str(round(bias, 1))\n"," axs[0].set_title(title)\n"," axs[1].plot(loss_list, color=\"r\")\n"," axs[1].set_title(\"Loss over time\")\n"," plt.show()"],"execution_count":4,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAlkAAAILCAYAAADbvom7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3yV5f3/8dfnnJOdEAhhhEAARTCK\nYBUFFw4cKO4tztoW/aqttbV2Klpra62/2jraitZq3UrdA0VFUUEUkI2EPUMIgUxyMq/fH/dJODmc\nhJAQEuD9fDx8mHOfe1x37Peb9+Man8ucc4iIiIjI7uVr7waIiIiI7IsUskRERETagEKWiIiISBtQ\nyBIRERFpAwpZIiIiIm1AIUtERESkDShkiXRQZnaCmS1p73bsrczsSjP7sKM+38xOMrN1e7JNIrJn\nKWSJtDMzW2Vmp0Yed8597pwb1B5t2hc45553zp3eUZ5vZs7MBrT0fmZ2qJl9aGZbzKzQzGaZ2Vlh\n36eY2V9D/3sqM7M1ZjbRzIZHtKHMzErNrMDMPjazy1r+liLSFIUsEWnAzAK745zd/UzhbWAy0BPo\nDvwEKAYwszjgE+Aw4GygE5ANvAScGXGfoc65ZGAQ8DTwqJmN3wPtF9nvKGSJdFCRw0mhHorbzWye\nmRWZ2ctmFh/2/dlmNifUyzHNzIaEffcrM1tuZiVmtsjMLgj77joz+9LMHjKzAuDuKG25O9Qr8pyZ\nFQPXmZkv7L4FZvaKmaWFXXONma0OfXdneI/drt7PzOJD5xaE3u8bM+sR1v4VoXdbaWZXhh3/Iqw9\nx4auKwr9+9iw7z41s3tDv4eSUI9ReiP/XT4zs4tCPx8X6h0aE/o8yszmRD7fzKaGLp8b6kW6LOx+\nPzezTWaWa2bfb+SZ6UB/4AnnXGXony+dc3XvdzXQGzjfObfAOVfjnCtzzk10zu3w3xPAObfZOfcs\n8H/Ar82sa7TzRKTlFLJE9i6XAqPx/uAOAa4DMLPvAU8BNwBdgceBt0I9HADLgROAVOAe4Dkzywi7\n73BgBdADuK+RZ58HTAQ6A88DPwbOB04EegFbgcdC7TkE+AdwJZARem5mS+8HXBu6R5/Q+90IlJtZ\nEvAwcKZzLgU4FpgT2fBQWHs3dG5X4K/AuxHBYizwfbxeoljg9kZ+D58BJ4V+PhHv9zYy7PNnkRc4\n5+q+H+qcS3bOvRz63JPtv5sfAI+ZWZcozywAluH9dzu/LmCGORX4wDlX1kibm/ImEACObsG1ItIE\nhSyRvcvDzrkNzrkteMNHh4eOjwMed87NCPViPANUACMAnHOvhq6rDf2BX0rDP6obnHOPOOeqnXPl\njTx7unPujdA9yvGCzm+dc+uccxV4PWAXh4b+Lgbeds594ZyrBO4CIjdK3ZX7VeGFowGh95vlnCsO\n3acWGGxmCc65XOfcwihtHwMsdc49G3rHF4HvgHPCzvmPcy4n1JZXwn63kT7DC1Pghas/hX2OGrKa\nUAX83jlX5Zx7DyjFG8ZrwHmbzJ4MrAL+H5BrZlPN7KDQKenAxrrzzezwUI9fse1k8YRzrgrYDKQ1\ndZ6I7DqFLJG9y8awn7cByaGf+wI/D/1hLTSzQrxen15QP3Q3J+y7wXh/mOusbcazI8/pC7weds/F\nQA1eb1iv8POdc9vwemNaer9ngQ+Al8xsg5k9YGYxoZ6by/ACWq6ZvWtmB0dpey9gdcSx1TTsXWvs\ndxtpOjAw1Jt0OPBfoE9oSO9oYGoj10VT4Jyrbs5zQ+HzFufcgXi/q7LQs8H73WaEnTvHOdcZuBCI\n2+FmYcwsBugGbNmFdotIMyhkiewb1gL3Oec6h/2T6Jx70cz6Ak8AtwBdQ398FwAWdn1kL1M0kees\nxRumC39mvHNuPZCLN0cIADNLwOuJatH9Qj099zjnDsEbEjwbuAbAOfeBc+40vJDxXehdI23ACybh\nsoD1zXjvho32AuMs4FZgQainbhrwM2C5c27zrt6zBW1YizeUOjh06GPg9NDw6a46D6gGvt5NzROR\nEIUskY4hJjS5u+6fXV1t9wRwo5kNN0+SmY0xsxQgCS/Q5AOEJlcPbuJezfUv4L5QiMPMupnZeaHv\nJgLnhCabx+IN/Vn02+z8fmZ2spkdZmZ+vBV1VUCtmfUws/NC4aICb7itNsq938PrfRprZoHQxPND\ngHda+O6f4YXWuqHBTyM+R5MHHNCSh5lZFzO7x8wGhBYIpAPXA1+FTvkvXrB93cwGm5nfvEURw5q4\nZ1pokcBjwJ+dc5E9jSLSSgpZIh3De0B52D9378rFzrmZwI+AR/EmjC8jNCneObcIbx7PdLw/9IcB\nX+6GNv8deAv40MxK8P7gDw89cyHeRPaX8P74lwKb8ILQLt8Pb4L4RLyAtRgvzDyL9//DfobXU7UF\nb07U/0XeOBQgzgZ+jje0dgdwdit6nT4DUtg+NBj5OZq7gWdCw6GX7uLzKoF+wEd4v4MFeL/L6wCc\nc0G8OVuL8Cb4FwNLgKPwFkuEm2tmpXj/G/khcJtz7q5dbI+ININ58ylFRNqOmSUDhcBBzrmV7d0e\nEZE9QT1ZItImzOwcM0sMDeU9CMzHWx0nIrJfUMgSkbZyHt4w3gbgIOByp65zEdmPaLhQREREpA2o\nJ0tERESkDShkiYiIiLQBhSwRERGRNqCQJSIiItIGFLJERERE2oBCloiIiEgbUMgSERERaQMKWSIi\nIiJtQCFLREREpA0oZImIiIi0AYUsERERkTagkCUiIiLSBhSyRERERNqAQpaIiIhIG1DIEhEREWkD\nClkiIiIibUAhS0RERKQNKGSJiIiItAGFLBEREZE2oJAlIiIi0gYUskRERETagEKWiIiISBtQyBIR\nERFpAwpZIiIiIm1AIUtERESkDShkiYiIiLQBhSwRERGRNqCQJSIiItIGFLJEZI8ys7vN7LmO3g4z\nW2hmJ+3BJonIPkYhS0Q6NDMbZWbfmdk2M5tiZn2bOHeKmeWbWbGZzTWz81r6XOfcoc65T1t6fXOY\nWZyZPRVq70Yz+1kT515rZrNC564zswfMLNCW7ROR1lHIEpEWa+s/8maWDrwG3AmkATOBl5u45FYg\nwznXCRgHPGdmGW3Zxla6GzgI6AucDNxhZqMbOTcR+CmQDgwHRgG374E2ikgLKWSJSANmdoSZfWtm\nJWb2qpm9bGZ/CH13UqgX5ZdmthH4j5l1MbN3Qj1IW0M/9w67X38z+yx0v8l4IaG5LgQWOudedc4F\n8ULJUDM7ONrJzrl5zrnquo9ADNCnifvHh96vxMxmm9nQsHavMrNTQz8fbWbTzazQzHLN7FEziw19\nZ2b2kJltCvUyzTezwc18v2uBe51zW51zi4EngOsaebd/Ouc+d85VOufWA88DxzXzOSLSDhSyRKRe\nKDi8DjyN13P0InBBxGk9Q9/1xest8gH/CX3OAsqBR8POfwGYhReu7sULFuHPnGdmYxtp0qHA3LoP\nzrkyYHnoeGPv8I6ZBYEZwKd4vV+NOQ94NfQ+LwBvmFlMlPNqgNtC73AMXi/STaHvTgdGAgOBVOBS\noCDUlrFmNq+RdnYBMsLfL/Rzo+8WYSSwsJnnikg7UMgSkXAjgADwsHOuyjn3GvB1xDm1wHjnXIVz\nrtw5V+Cc+59zbptzrgS4DzgRwMyygKOAO0PnTwXeDr+Zc26Ic+6FRtqTDBRFHCsCUhp7Aefc2aHv\nzwI+dM7VNvG+s5xzE51zVcBfgfjQ7yDynrOcc18556qdc6uAx+veEagKPe9gwJxzi51zuaHrXnDO\nDWni3erep1nvVsfMrgeGAQ/u7FwRaT8KWSISrhew3jnnwo6tjTgnPzR0B4CZJZrZ42a22syKgalA\nZzPzh+63NdQDVWf1LrSnFOgUcawTUNLURaGA+D5wupmd28Spa8OuqQXWhdrcgJkNDPWQbQy94x8J\nDXs65z7B67l7DNhkZhPMLLLN0ZSGvQ9hPzf5bmZ2PvAn4Ezn3OZmPEdE2olCloiEywUyzczCjkXO\naXIRn38ODAKGhyacjwwdt9D9uphZUtj5WbvQnoVA+DypJOBAmj9MFgid35j6dzMzH9Ab2BDlvH8C\n3wEHhd7xN3jvB4Bz7mHn3JHAIXjDhr/YWcOcc1vxfj9Dww4PpYl3C02KfwI4xzk3f2fPEJH2pZAl\nIuGm480/usXMAqESCEfv5JoUvHlYhWaWBoyv+8I5txpvTtQ9ZhZrZscD5+xCe14HBpvZRWYWD9wF\nzHPOfRd5opkdbGZnmlmCmcWY2VV4ge+zJu5/pJldGFol+VOgAviqkXcsBkpDk+7/L+y5R5nZ8NBc\nrjIgiDek2hz/BX4XWjxwMPAjvPlwOzCzU/Amu1/knIscwhWRDkghS0TqOecq8Vb0/QAoBK4C3sEL\nH435G5AAbMYLKJMivh+LV3JgC14A+2/4l6Gin1c20p584CK8eV5bQ/e5POzaf5nZv+o+4q0+3ATk\n45VzuMw5N7uJtr8JXBa699XAhaH5WZFuD71HCV5PUngZiU6hY1vxhkILgL+E2nelmTXV6zYebyL/\narww+Bfn3KTQtVlmVhqa1wZeGYtU4L3Q8VIze7+Je4tIO7OGUy9ERBoysxnAv5xz/2nvtoiI7E3U\nkyUiDZjZiWbWMzRceC0whB17p0REZCe0JYOIRBoEvAIkASuAi+tKEoiISPNpuFBERESkDWi4UERE\nRKQNdLjhwvT0dNevX7/2boaIiIjITs2aNWuzc65btO86XMjq168fM2c2tdWYiIiISMdgZo3uYqHh\nQhEREZE2oJAlIiIi0gYUskRERETaQKtClpmNNrMlZrbMzH7VyDmXmtmi0NYZL7TmeSIiIiJ7ixZP\nfDczP/AYcBqwDvjGzN5yzi0KO+cg4NfAcc65rWbWvbUNFhEREdkbtKYn62hgmXNuRWhT2ZeA8yLO\n+RHwmHNuK4BzblMrniciIiLSfNXV7fr41oSsTGBt2Od1oWPhBgIDzexLM/vKzEZHu5GZjTOzmWY2\nMz8/vxVNEhERkf3amjXwwx9C165w2GHt2pS2rpMVAA4CTgJ6A1PN7DDnXGH4Sc65CcAEgGHDhmmf\nHxEREWm+4mK47z54/nlYv947FgjA8OHt2qzWhKz1QJ+wz71Dx8KtA2Y456qAlWaWgxe6vmnFc0VE\nRGR/V10NDz8MEyZATg44B2YwZAj8+Mdw/fXga98iCq0JWd8AB5lZf7xwdTkwNuKcN4ArgP+YWTre\n8OGKVjxTRERE9le1tfDqq/DggzB7tvcZ4IAD4Pvfh9tvh/j49m1jmBaHLOdctZndAnwA+IGnnHML\nzez3wEzn3Fuh7043s0VADfAL51zB7mi4iIiI7CemToV77/X+XVnpHevRAy67DO68E9LT27d9jTDn\nOtYUqGHDhjntXSgiIrKfW7wY7rkH3nsPSkq8Y506wZgx3vGDDmrf9oWY2Szn3LBo33W4DaJFRERk\nP7VxI/z+9zBxItRVG4iLgzPO8Hqsjjuufdu3ixSyREREpP1s2wYPPABPPw2rV3vH/H5vZeDtt8PF\nF7dr81pDIUtERET2rNpaePxxeOwxWLRo+8rA7Gy46Sa48UavBMNebu9/AxEREdk7vPmm12v19dfb\nq7H36QPXXgu//CUkJ7dv+3YzhSwRERFpO998A3ffDZ98AsGgd6xrV7jwQu94r17t2bo2pZAlIiIi\nu9fKlTB+PLz9NhSGNnlJSvKC1T33wODB7du+PUQhS0RERFpvyxb4wx/gpZcgN9c7FhMDJ58Mv/0t\njBrVvu1rBwpZIiIi0jKVlfDXv8K//w3LlnnHfD448kj46U9h7Nh239qmPSlkiYiISPPV1sKzz8Lf\n/gbz5m3f2uagg+BHP4Jbb4XY2PZtYwehkCUiIiI7N3ky3HcfTJsGVVXesV69vN6q3/4WOndu3/Z1\nQApZIiIiEt28ed4KwA8/hLIy71jnznDeed4E9r5927V5HZ1CloiIiGy3bp0XoF57zZvMDpCQ4O0Z\nOH48HHVU+7ZvL6KQJSIiso9ZnFvEpAV5rC8sJ7NzAqMH9yA7I7XRc/rG1nL5x8/S7Y2JsHatd0Ig\n4O0V+Ktfwdlnb79mck6T95XtzDnX3m1oYNiwYW7mzJnt3QwREZG9RnhgivUbecUV9ElLJCU+QEmw\nmqLyKk7N7kZOXln9OflbyrhgxpsM/+AV0tauxABnxsbeB7Lgwmvo/YtbyM7s0uAZE6auJDUhpsF9\nx43sv18HLTOb5ZwbFu079WSJiIjsxcLDT0ZqPFNz8ikJVtMzNQ6fxZCaEMOW0goe+WQ5I/p14cQ5\nUxj03OMcuDYHv6vFAVu7Z/Lm4afz3hljGTG4jxegvljNuJG++gA1aUEeqQne/YD6f09akLdfh6ym\nKGSJiIjsxSLDT1WNIznOz7JNZaQnxwOQOnsGf3j73xy+cj6BqkocUJTcmenDRrHq/37OlK1QUVlN\nZa3DZ1YfzMa/tYistEQyOyewKLeIg3t2avDslPgA6wvL9/Qr7zUUskRERPZi6wvLyUiNr/+cHB+g\norKalNXLOfPFezng66nElnsrAysSk1l23Kn88+SrWdUlk8pax+ldulKam0fAB53ivaCWXxIkJ6+U\nqtpahvdPo6i8irUF5STG+OmXvn0T55JgNZmdE/bsC+9FFLJERET2YpmdEygqryI1IYb4rQX89I2H\nyf5iEp1LCzGgOiaWmQOP4ovLb8AdfwIASSVBSldsITk+QK1zxPiNkmA1h/X2hv2W5ZeBQXpyXH3P\n1sAeySzJK6VLUlyDOVmXHdW7Hd++Y1PIEhER2YuNPiCFVb+6hxFT36bzpg0YUGM+1h00mCljrqZg\n9LkMzEhh9eJ8UsurSIkPEBvwk9U1kV6p8eQWBTm0VyfyiiuI8fupdY4tpZX4fTCge1L9c/qmJ7Gt\nqobUhJj61YWXHdVb87GaoJAlIiLSgTSr/ML6rax74BEGv/4sB69bTrZzOIxNmf1ZdO4V9Pz1z8ju\n05Vrwq45oFtyg/veMXpQg/uGPzctOZaMTnH1c7rAGxo8tFcqt502sK1/BfsMlXAQERHpIHZaJuGt\nt9j2hz8RN+sb/LU1OKC4a09mnDCGvvePZ9CgPnumHVJPJRxEREQ6qPAepDVbtpHRKa5BmYSMZQvg\nnFupmj+DmMoKEoCy5FRWnHAGX111M2XdelJUXsXCNeUMGrR72pSdkcq4kf0b9HxpaHDXKWSJiIi0\nk8gaV3PWFFK0rZLeRXmM+d8EDpj+MXFlJRhQGZdAzrGn8pcRl7Om9wCG9etSP5zXFqUUsjNSFapa\nSSFLRESkjexsflV4javY0mJ+8v6/OO6byaQVF3gT2P0B5g/4HhNPv4ou53tb2xStKMDKqxrUwVIp\nhY5JIUtERGQ3aWx7m4zUeIrKq5gwdWWDeU25m4sY8/HLDHn/VTpvWIMBtWYs6zWApVf+kK+PPYtP\ncjZz3IC0+mcM6JbE7NWFbC6toNY5lVLowBSyREREdoPmbG8DMGleLtmT34S//Y3758zFF9raprBX\nFgvOvJT/jbyIDeXOq7SeFMcJA7oSE/DXP6dbSjwDeySzsaSC3KKg5kt1YApZIiIiu8HOtrfpPXsa\nF73wT3otnA011QDUdO/BZ0edxowrbiSmWzolwWqsvIp7Ttve21UX3oD6lX5+v497zj1EwaqDU8gS\nERHZDaJtb9N9VQ6Xv/cUR+d8Q2ywHAcEE1PIOXYUr5/3IxIHDWBgjyTi8soaXcWnlX57L4UsERGR\n3aBue5uMbVs45r+PMu7zD0gsLcKAqtg4co48gRfP/D5rDhxMn7REUuIDFJVX8dHi/J3Wn9JKv72T\nQpaIiEgL1U10z99YwGlv/JshU94hbWuetzLQ52fpgCF8dfEPmHfEiWR2TqCyJEifgL9BHSzwhhoV\novY9ClkiIiItsHhtAYt/dz9XffIa6etWYjhqMfKyBvD5GZex7qKrGD0kg2vCwtPtr84lLbnhn962\nqHElHYNCloiISBSN1riaOBH+8hcGzZxJdq23MrC4RyYLT7+QT86+mqQuje/vVzekWNeDBapxtS9T\nyBIREaHpGlcps7+G2x+jdsm3+CoqACjrlMayE0fz1VW3EOzSFYAE55rslRo9uMcOKwVV42rfpZAl\nIiL7vWg1rjqtWckF01/g4NlfELetFIDKhCRiLzsfxo/nyXX+Xe6V0krB/UurQpaZjQb+DviBJ51z\n90d8fx3wF2B96NCjzrknW/NMERGR3SHaxsw9gsUMf+qfXP/xO3Qu3YoB1TGxrDriOL4aeyMfpQ/i\n0F6prF+wvbeL0ErB5vZKaaXg/qPFIcvM/MBjwGnAOuAbM3vLObco4tSXnXO3tKKNIiIiu1V4z1Vm\nAhz62hOMnjmJnltyvZWB5mNZ1sFMPOlS4sdeAT4fK/NLWbeplN5dvCHEkmA1tc5RVV1DblG1eqVk\nB63pyToaWOacWwFgZi8B5wGRIUtERKRDmTQvl5OnvsEx771I+uqlmHM4YH2Pviy98Eo+Pukipq8p\nJjk+wEgzSsqryMkrZVDP5AblF/p2TSI1IabRie6yf2tNyMoE1oZ9XgcMj3LeRWY2EsgBbnPOrY08\nwczGAeMAsrKyWtEkERGRJlYGvvMO/PnP/GT6dPw1NTigNL0nM0eO4eFhF1Icm8hZh2UQCFaT1TWR\nXqnx9fsD9umaQFbXpAbPUfkFaUpbT3x/G3jROVdhZjcAzwCnRJ7knJsATAAYNmyYa+M2iYjIPixy\nEnvi/G+p+dU/qFn4Nf6KIA7YltSJ+UefwoIf3kZZt54A9MkvbbDp8h2jBzUY+ntoco7KL8guaU3I\nWg/0Cfvcm+0T3AFwzhWEfXwSeKAVzxMREdmpSQvyyCrexOn//icHTv+EuNJiDCiPjWfV8FP45ppb\nmNm5L9+uLeQISybLuWZtuqzyC7KrWhOyvgEOMrP+eOHqcmBs+AlmluGcyw19PBdY3IrniYiIADsO\nBw7skcSq5bkc8tQjXD31PboWb/YmsPsDrDvsKJ458Qqm9PseZw/pBUD/0H1yiyuICfibNWld5Rdk\nV7U4ZDnnqs3sFuADvBIOTznnFprZ74GZzrm3gJ+Y2blANbAFuG43tFlERPYj0QLVR4vzQysDffR7\n+h8c9uFEztq8HgNqzVjWawBzL7yGtWdfAj4fn87LpWtSTIP79k1PIjbGz4OXDG12W1R+QXZFq+Zk\nOefeA96LOHZX2M+/Bn7dmmeIiMi+pdFJ6Y2cGz6/qqi8ikcmL2Xs8i8446OX6LZsMT7nbW2Tl57J\n0nMuZfLpl/Plum0NVgYGfEZGanyDe2s+lbQ1VXwXEZE9JlpomjB1JeNG9o8atCYtyCM1IYbUhBh6\nz/mK4c//g98umEVMTTUOKOuSzodDT+Gts64jLzaJ0w/pSRfgqJg45q4rqp/E/uNRB/LR4nyKyqs0\nn0r2GIUsERHZY8JDE1D/70kL8qKGrOp58xj7+gT6zfqS2OA2AMoSkpmSfQxrb/0lxRl9mL6igOLy\nKlLjtw8HxscEOP2Qng3qVx3QLVnzqWSPUsgSEZE2Ezk0uCi3iIN7dmpwTrCqmhkrC+rPGZNey8DH\n/wYTJ3J7QQEGVMXGseLoE/nqypv5Kv1AcjaVMqJzV1Kco2dKHLmF5QzqkUxtaKVgtF4qzaeSPU0h\nS0RE2kS0ocG1BeUkxvjpl54MQH5JkG9WbiWdSi5640WyP3yD1M0bvRv4/ZQfeTQvnnQFa0eeVj/M\n5y+v4senHEhOXhnrC8vp3y2ZMwb3qP+sXirpKBSyRESkTUQbGhzYI5kleaV0SYojJQB9X/wP46a+\nTt9NazAczoz8rAOZf95VjPrbXST6fBwb0RtWF6DGRDwv8rNIe1PIEhGRNrG+sHyHFX1905MYNG0y\nFz7+It1zFuCv9ba2Ke7ei0Wnnc/My37E+kpj7roi3v/f/PrVh9obUPZGClkiItImMjsn1G9Dk7Fw\nFiOe+weZ874mpqrSO6FbN+YOP5UPL76BQK8MoG74cAvJ8YFmrT4U6cgUskREpNl2pcbVOYmlFN39\nWw799nPiy8sACMYnUj76LDr9+Y+QnU1sbhEbpq4kNVRaYeGGYhwwOLMTPrOdrj4U6ch87d0AERHZ\nO9RNZC8qr2rQy7Q4t6j+nCULljP7/GsoSe3KgccfyZHTJhGoqiTnsOG8/OCzrFyxgU5vvQ7Z2cD2\nrWpSE2LILQpSVeMYfkAX0pO3DzOmxAdYX1i+x99XpLXUkyUiIs0SbSL7ltIK7p04m6umvcbwKa8z\nMC+0tY3Px4YBh/LxmVdy1C9vJDuzC43NqgovrfDQ5ByKyqsafK/K7LK3UsgSEZFmaTCRvbaWvm++\nxHlvPEe/jSvwOYcD1nbrzfwxl5Fz6fUQCFBUXkXBonyyM7s06xmjB/dgwtSVAKrMLns9hSwREWmW\nzM4JpH/xCSe/9m8yFs/BH9raZkvnbiw//XzuP+ICqhOTSYgNMCLg/XnZ1aG+uuFDVWaXfYFCloiI\nNG32bLjnHm79cDK+YDkOCKak8unBx/Lc6Ovo972DSU+OJ7CigJrKaoqD24f7WjLUp8rssq9QyBIR\nkR2tWQPjx8Obb8LWrQD4EhMpOeMs3jjvR8xN78+aLdvI6BRXP0l9QLckZqzwyi80tb2NyP5CIUtE\nRABY8t1qiu+6l4M+epvUrZswgEAARo6EX/0KzjyTFODq0D/hqw1T4gPEBvxkdU2kV2o8uUVBDfXJ\nfk8hS0Rkf1ZZCY88QsU//sXAFcswwJmxsd8g3j35ElaedREVzsgMJDA6t6hBYIo2f+qO0YMUqkRC\nFLJERPYT9YVEt5Rx8twpnPLusyQsmAe1tcQCW3v2ZtHoi5l18ffJq8Ab+ttYysiB3RqtvK75UyKN\nU8gSEdkPLM4t4uN/vsp5b/6brEWzCFRX4YCqbt2JuXIs4wefR2qfDHxmACxbV0BynJ/KmlpVXhdp\nIYUsEZF92cKFcPfdHPjue2SXbwOgIimFpSPP5JWzr2dhYney0hK9SewFZfRLTwagNFhNwAed4mPq\nb6XK6yK7RiFLRGRfs3Ej3HMPTJwImzcDYDGxrBh2AjOuupmNh3yP/JIgs1cXUlVawfD+aVRW1TB7\nTSEAWV2TiPEbJcFqDuu9vZyLaCsAACAASURBVNdKlddFdo1ClojIvqC0FB54AP77X1i92jvm98OI\nEXDHHTyafChF5VX1w37L8svAID05Dp8Z/bt5PVi5xRXEBPwc2qsTecUVxPj9Kscg0kIKWSIiHVz9\nhPXQCr7Rg3sAMGnOerL+9xwnfvQqXdcsx5wDMzjkELj5Zhg3zivBAIwOlVsAb9hvS2klfh8M6J5U\n/5y+6UnExvh58JKhUZ+rcgwiu0YhS0SkA6urRZWaEENGajxF5VVMvu9xznr/WW5dtRhfbQ0O2Jre\nk5qrr6bb7++C5GQvIE1Z0SCYhZdbSEuObVBIFHYcDtTKQZHWUcgSEWlnkT1GA3skkZNXxvrC8vqq\n6oNWLmDE8/+gz9wZBCorcEB5aheWHn8GX119C7kJnUlNiOG2UMCKDGZ15RduO21g/TPDC4lqOFBk\n91PIEhFpR5GBaGV+Ka/NXscRWZ05LLiZ0RMeZMSiaSQHywCoTEhi6tBjePaMaznk1GPr7xMsLmfG\nyoIGwaxu/lW08gvaiFmk7SlkiYi0o0kL8khNiKkPQqW5edz23lOcPOcTOhdvwYAqf4D5Bx/Jdz+4\nlXVDh/Ppkk0AHBK6R35JkG9WbiU5PkBGajxz1hRStK2S5PhA/XBgtPILGg4UaVsKWSIi7Wh9YTmZ\nCTDsxX9z2KSJpOauxYAa85F30GCmjLmSZ3qPoAo467AMSsqrSEuKxWdWP9S3cEMxDhic2QmfGWnJ\nsRSXV7FsU1l9yFL5BZE9TyFLRGQ3irYSMGpvUW0tPPMMd/zhAdJXLsHnHA7Y0D2Ld0eczZTTL+Xo\ngRkADMwvZWNJRf2my3eMHgRQ/5yqGsfwA7rUB6oB3ZKYvbqQzaUVKr8g0o4UskREWiE8VMX6jbzi\nCvqkJe4w4bw+aL3/Ptx/P0yfDlVVdAOKunRj/knnMOeqG1ka9PPt2kKOSEupD0h+v497zj1kh7BW\n9/mhyTkUlVfVH++WEs/AHskNgpnmW4nseQpZIiKN2FmvVOSk9ak5+ZQEq+mZGofPts+zmvnGFLI/\neBomT4Zt3tY2dOkC55+P3X03uTGpzAw9p3+3BM4Y3KN+dWFzAtLowT0a1MBqKpiJyJ5jzrn2bkMD\nw4YNczNnzmzvZojIfi48QIWXOAjvlarrQaoLU5MX5RHwQUJsgFOTKxnx7CMc+OVkEkqLvQnscfEE\nTzqF/J/9kjet586HFHexvc0aphSR3crMZjnnhkX7Tj1ZIiJRRK76i1YGYX1hORmp24t5ptcGOfed\nZzjxmw9IL9qMAdU+PwsPHErOtTcxb+hxrC4ow7fB6JNW1fiQYgtopaBIx6OQJSISRWSAAkIr+Yp4\naHJOfT2qmmAF5017gyHvvMRP163EgFoz8vsP4qVjzueNoadzzEHppCfHkwpsKasEYHCmF4iihTcR\n2TcoZInIfqupIbbMzgkNhgIBVm8uY93WcnqnxnPit58w5OUn6b3yO/yu1tvapkdvphw7hnmXXk+Z\nP45FG4oZ3qdTg61rKqtr8dYRbhethpWI7P1aFbLMbDTwd8APPOmcu7+R8y4CJgJHOec04UpE2l1T\nW89kZ6RGnUwe9+Xn/G3qCwz47lsC1VU4oCQ1jS++dwpfXXUzXbJ6MXpwDy5qZNUfQGzAt0NbVMNK\nZN/U4pBlZn7gMeA0YB3wjZm95ZxbFHFeCnArMKM1DRURaY3IXqvNJcEm51zVbTsz471pfO/Jh8ie\nO43Ycm9rm4rEZJaecAbTr/kxW3v1JbcoyIOXDN3hmdGCWmQhUdWwEtl3taYn62hgmXNuBYCZvQSc\nByyKOO9e4M/AL1rxLBGRFovWa/X5sgKOG5AGbB8OrB+227gRfv97sidOJDs/3/syPp6Vw07g44vH\nUXLE0fXXlJRXNdoLFW1/wMhCoqphJbLvak3IygTWhn1eBwwPP8HMjgD6OOfeNbNGQ5aZjQPGAWRl\nZbWiSSIiO4q2UrBLYgyLNpTQfZAXkALBbQx+YQJHfPImbNrgXej3w/Dh8ItfwEUXEcwtYuHUlaTu\nQi9UY6v+FKpE9n1tNvHdzHzAX4Hrdnauc24CMAG8Ollt1SYR2T9FWymYnZHCtKWbGfDas4yY9DLp\nq5dhzuHMIDsbbroJbrwRAoGwa3bsmVIvlIg0pjUhaz3QJ+xz79CxOinAYOBTMwPoCbxlZudq8ruI\n7EmRKwUPmPYRF704gfuWLsBfW+NNYO+WQcXYK+n2h/GQnNzovVSPSkSaqzUh6xvgIDPrjxeuLgfG\n1n3pnCsC0us+m9mnwO0KWCKyp40e3INJ/36LMW88Qf/5XxNTWYEDarqkwaWXYHfdRadevVicW8Rz\n0zeoarqI7BYtDlnOuWozuwX4AK+Ew1POuYVm9ntgpnPurd3VSBGRFlm+HMaPJ/uddzi4qAgDKuIT\nyDn+DPy/H8+BJx9Tf+rOSjqIiOyqVs3Jcs69B7wXceyuRs49qTXPEhFpSl2JhoJ1uVz49lMc9uUH\nxGzK876MjcVOOQV++1viTjmFgVGub842OiIiu0IV30Vkr7d4dT7Lfv0Hrp3yOl02rg1tbeOjfMhQ\nEn5xO4wdC74di4CGa2wbHVViF5GWUsgSkQ4v2vY31Nay+m+PM3ji0wxanUO28zarKczsy/yzLuO1\n4y9gQ7kjKy6RzI+X7XR+VbRtdFSJXURaQyFLRDqc8FAV6zfyiivok5ZIRmo86V9+it3yLwYsn092\nTTUOKOjUlU+/dwqLf/ATknt2J78kyHerC6mqrWV4/7Rmza+KVp1dldhFpDUUskSkQ4mcgD41J5/0\nFUu45MvnGTj3K2IqynFAWUIyq48/lenX/IQPy+IpLq8idZufEcCy/DIwSE+Ow2fWrPlVqoElIrub\nQpaIdCh1E9B7lRZwzD8f4UdTPyBlWwkGVMXFs3zEyfztmMtZmjWIMw7NAGBASZDZqwvZXFpBrXNs\nKa3E74MB3ZPq79uc+VWqgSUiu5NClojsVtHmTzU7uBQXM+Sx+znqy/dJ2bwRA6p9fhYdMISXT7mC\nLpdeCMCaJZuIC7usW0o8A3sks7GkgtyiIGnJsWR0iiM9eftEds2vEpE9TSFLRHabFtWaqq6GRx+F\nxx+HJUsYFdraZnP/QXx7/lV8euzZzFhVSHJ8gJHOURKsJi0pFp8ZRWF7CPr9Pu459xCyM1Lr21G0\nC3sMiojsbuZcx9oqcNiwYW7mTBWFF+mIdtZL9dDknB1W6K3ML2VjSQVZaYkNVgau+MfTHPbyk/Re\n+R2+2lrv5H792HTJWP5y6NkkdelUH5BWF5TRKzWeihq3/R7QZFta1aMmItJMZjbLOTcs2nfqyRKR\nZmlOL1Vkran8kiA5eaX1q/xSZ35FzU8fY1DOt2RXV3kT2FPT+OrIU5hx5U1sTUkjs3MCJ/VIIiev\nrD4g3TF6UNSA1FRo0vwqEWlvClki0izNqYgeWWtqWX4ZmZvW8IPJTzPszq+ILS8DoCw+keXHn8b0\nq3/M0s69mLFiC8nbAozs7YW3jxbnazsbEdnrKWSJSKPCh9wWbShmaJ9OwPahwMgVe3W1phILCzht\n4uNc98l7dC4r9Cawx8Sy6sjjeeTYS1l4wJD6lYHLVhSQHOensqa22eUWRET2BgpZIhJV5PDg0rwS\nZqzYyjEHWv2qvQYr9rZtI3vC3/jzk08Rs24NBtSYj9UDBjNv7DiWH38GAMsiVgaWBqsJ+KBTfOPh\nTURkb6SQJSJRRQ4PHtqrEzNWbGHB+mJGDozzVuyVVXDDog/gZ/+BhQvBOWLN4OCD4aabyDl/LBOm\nrSU1IYaURlYGxviNkmA1h/Xe3mulcgsisi9QyBKRek0ND3ZLieeo/l2Yu66ILh9P4vJ3nqHXknlY\nTY23Z2DXHuScfgGdx/+GQYP6AJANjBsZaLDK747Rg4DtKwMP7dWJvOIKYvx+akNBTOUWRGRfoJAl\nIsDOhwd7fDePMc88TP/5XxNTWQFAdZc0vv7eSUy76hYss5cXkOYUMK5Tp/r5VI2t8muq3IK2sxGR\nfYFClogA0YcHV32zkFGv3s+xC6cRV+ZtbVObmAhnXwj33MMjubENVhO2dNK6yi2IyL5IIUtEgO01\nrmJLixnx/D8YNOUdkrbkeysD/QHWDh2O+82v6XvpeduvWTy3QV0s0KR1EZE6Clki+7G6YbrczUUc\n/urTnDHtbdLz1no9VuYj94Bs5l10LaPv/wVZPt8O10fWxQJNWhcRqaOQJbKP2NVtZBav38rs+x7l\nskkv0HNVDj5XiwM29+jDorMv5bPRV7K1xhg3sj9ECViwvS4WoD0CRUQiaO9CkX1A+KT18LATXjW9\nLoQlfvYJZ739HzIWziZQU+1tbZPWnSUnj+GV069mpYtvsM/gzuZKaY9AEdmfae9CkX1QeLhZs2Ub\nGZ3iGp2AvvzjL6n69V3cPG86sRXlOKA0Ppmc409izvU/pTjDK7nQ3TlqioI8eMnQZrdDk9ZFRKJT\nyBLZC0WWW5izppCibZUkxwfqq7H3LNnM4X//O1z+GQds2YIBVXHxLB9xMjPG3sTL1hOAkzK6199X\n86lERHYfhSyRvUDkkNzmkmCDcgtpybEUl1exZk0+586YSPZHb5CSvxEDCARYNehwvr1iHKuPHVV/\nz+zicqYt31JfeV3zqUREdi+FLJEOLrLXqqi8is+XFXDcgDQgBqqruerrNzn8nRfps8nbM9CZsS7z\nQGaeM5YvT7qANYVBMjrF0S/svvExAY4f0JXUhBgVARURaQMKWSIdQGRP1cAeSeTklTU636pLvJ/O\n77zJ5dP+R4+lC/DVeisD87tmMPuU85h2/nWsr/TRJy2RjPgAlTWO2WsKAcjqmhR1YryIiOxeClki\nbWxnq+8ie6pW5pfy2ux1HJHVmayuSQ3mWw1ZuYDhL/yTW+bOIKa6CgdsS01j4bGn89b5P+LKc45k\ndEYqiyfn4AurX9W/WzIAucUVxAT86rUSEdkDFLJE2lC0ob4JU1c26EGK3M5mY0kFSXEBNhZX0C89\nmUPLcjn/jScYvvgrEoPbAKhISGLu8FP49vu3sqBTLzI7J3BlWHirq94erm96ErEx/l1aOSgiIi2n\nkCXShiIDVLS9/SIDUWmwmsyKIs793384dcFnJBZ6KwMr/TGsOuJYPrl4HPMPGMq4kf25rpGeKFVi\nFxFpfwpZIm0oWo9S5N5+dYGoi7+Go155kivfnUi3glxvaxufj42DhvD+6WOZlH08WenJZHZOYNxO\nCn6qEruISPtTyBJppabmXO20R6m2lsvmToJHHiFj7XLMORywulsfZp15Kasu+z4l1VBUXsU9uzBJ\nPTsjlXEj+zdol+ZgiYjsWQpZIq2wszlXjfUojVk9k3W3jSXjuzn0qqnxKrCn9+Tr48/iu2tvpt8B\nPViTV0ZuKwKSKrGLiLQvhSyRVtjZnKvwHiVmzeSaN56g75yv8FcEcUAwJZXFw0fx5gXjuPyCEYzK\nSKWuXOiY9nklERHZTRSyRFphp3OuVq8m+667yH7rLSj06lRVxiewZMQoZl73EwoOOBgAK69qMBle\nRET2fgpZImGaKgoarcZVtDlXlZu3cPnrT8Kt70NurncwJgZOOgl+8xt+U9idjNR4fGb110ROhhcR\nkb2frzUXm9loM1tiZsvM7FdRvr/RzOab2Rwz+8LMDmnN80TaUt38qqLyqvqioPe/v4RVm0sbzLda\nnFtUf83owT0oKq+ipKiMI155kmuuPY1fXXkswyY+Re3GjeQdmM36v/8LgkGYMgVOO43MzgmUBKsb\nPFvlFURE9j0t7skyMz/wGHAasA74xszecs4tCjvtBefcv0Lnnwv8FRjdivaK7FbhPVeR29dEFgXd\nocZVbS3ZH7/NH//yIHHz5+Nz3tY2m7r3Zt7oi1lwyfUU1fq87WvySup7wFReQURk/9Ca4cKjgWXO\nuRUAZvYScB5QH7Kcc8Vh5ycBrhXPE9mtIlcGhm9fk54cT2mwmpQ4P8XBqvprUuIDJHw+Bf50I3z5\nJVRWkgDQsydccQX/OO5y8mOStk+ED10XPt9K5RVERPYPrQlZmcDasM/rgOGRJ5nZzcDPgFjglGg3\nMrNxwDiArKysVjRJpHGR8602lwQbrAxMS46luLyKZZvKSE+OJzk+QHFovlXXlTmM+O8j9J35OXEV\n5TigIimF4LkX0PmBP0H//gCsfHUuGfEN/88q2nwrlVcQEdn3tfnEd+fcY8BjZjYW+B1wbZRzJgAT\nAIYNG6beLtntotWz+nxZAccNSAO8kDWgWxKzVxeyubSCWucYWF3MUS88wqjFX5BUWoQBwUAsCw8/\nnjnX/Zil/Q/1hgLj08gOPUfb2YiISJ3WhKz1QJ+wz71DxxrzEvDPVjxPpMWi1bPqkhjDog0ldB/k\nBaBuKfEMTvVxzKtPccqfJtG5IC+0tY2fVQOH8vKoseSecCr90pO9e4TdW/OtREQkUmtC1jfAQWbW\nHy9cXQ6MDT/BzA5yzi0NfRwDLEWkHUSrZ5WdkcK05VsoLinn+I9eZcjbL5K+bgUGYAaDB8NNN+G7\n4Qb6+3xsfnUuWTvZh1DzrUREpE6LQ5ZzrtrMbgE+APzAU865hWb2e2Cmc+4t4BYzOxWoArYSZahQ\nZE+INoz3va8/5qbXn6b38oX4ar2VgVW9s4j94fXwi19AYuJO7xFtKFDzrUREBFo5J8s59x7wXsSx\nu8J+vrU19xdpyq4UDq0bxuu9+FtGvfo4ved9TUxVJQ6wbt3gkkuwO+8ktmfPRp+noUAREdkV5lzH\nmmc+bNgwN3PmzPZuhnRw4RPZU+IDrN5cxrdrCzkiqzNZXZPqA9C4kf3JLt0E48dT8/Y7+EtLAKhM\nSCJ46ml0+vMfITt7J09r+NzwYBdZAV5ERPYvZjbLOTcs2nfaVkc6pJ2FmciJ7JGFQ3tUljD62UfI\nuvEDKCwAwB8bC6eeCnfeSezIkcS2oF0aChQRkeZSyJIOJ1q5hQlTV3q9UqGAEzmRvTRYTRerYvSb\n/+X8bz+g08Z13spA88GwYfCzn8Fll4GvVTtJiYiINJtClnQ40cot1B2vC1n1k9Dj/Bz6wUQueOUZ\nsjYsx+ccDtjauz9fn34Jiy++ltvO1JaZIiKy5ylkSYcQPjy4aEMxQ/t0oq5IKOxYKuGijXOpuu+P\n9Fs2H39NDQ7I75TO3BPP5rvv30KBP57VBWX0Kq/m9lfnav6UiIjscQpZ0u4ihweX5pUwY8VWjjnQ\nSE/2hgRLgtUcvmk5nPM7+Phjssq9rW2CKanMO+pk5l3/U3odNqB+dWGsq8FnRkzAT1pyIOqQo4iI\nSFtSyJJ2Fzk8eGivTsxYsYUF64sZk7qZo555hEFffUJCWWi/8cREOP98bPx4Eg4/nOFs3zRzTOjf\nD03OITbgb3LIUUREpC0pZMlut6tlDiInsWdaJXd+/jSDP32XrsWbvQnsgQDrDjuK9875PttOGrXL\n94ToGzWLiIi0FYUsabXwUBXrN/KKK+iTltjoysBImZ0TKCkq48QPX2LIe6/Qef2q0MpAww4/nPXX\njePB7sNJTYrzioA2857aqFlERNqTQpa0SuR8qqk5+ZQEq+mZGofPYpoepquthRdf5MYH/kLc/Pn4\nnLe1zdaevZl24vkc+Kffkd2/B69MziE1LDA1Z+hP1dlFRKS9KWRJq0TOp6qqcSTH+Vm2qax+0nrk\nMN2qV9/G/vgnMhfMJFBdRTxQ3a07c44fzRtjvk+XrF4NhgNbMvSnjZpFRKS9KWRJq0QGoOT4ABWV\n1RQHq+qPlQSrGVy4Di6+l5pJk+hXVgZAMCmFhcedwdsXjOOiS0dyREYqR0R5RkuH/lSdXURE2pNC\nljRpZ5PYIwPQgG5JzFixheT4APEFmxj2zCNkf/khSSWFANTGxrH8yBOYec2PycseCkB1eZWG/kRE\nZJ+jkCWNas72NpEBKKmygp9MfY6R09+l8+aNGOD8fuyYY+CXv+TXlf3ISI3HZ1b/HA39iYjIvkgh\nSxrVnO1tsjNSGXdsH3L//HcOefN5eqxbieHADA49FG6+Gbvhhvo9AzMn52joT0RE9gsKWdKoaBPO\ng1XVzFhZwPrCckYumMqp7zxL9tzZZNfUeCf07QvXXQd33OEVDSV6iQfSEjX0JyIi+zSFLGlU5Hyr\n/JIgpVO+4A+fPMvQZd8SqKrEAdVduxK45BIYPx569mxwj8ghx5JgNbXOUVVdQ25RtYb+RERkn6WQ\nJY2qm2/VJXctp738GH2nTyEp6K0MrExIYskxo/jokhuoHJTNbacNjHqPaEOOfbsmkZoQ0+g1IiIi\n+wKFLIlu82ay77+XB158iUD+Jgyo8gdYethw5lxzM+uGersF1jpHbhOT1rW9jYiI7K8UsvYhu7pn\n4A6CQfh//w+eegpWrAAgxueDI4+En/+cR9O+R1FFzS5NWtf2NiIisr/ytXcDZPeom/tUVF7VoNzC\n4tyiBuc8NDmH21+dy0OTc7zvamvhP/+Bww+HpCT43e+8gDVwIDz4IJSXw8yZcMUVjB6SQVF5FUXl\nVdQ6V//z6ME9Gm3X6ME9dvkaERGRfYF6svYROyu3EDkBPf2LT4i59Qnc0nlYdbV3k8xMNp9/MS+P\nuoqV1TFeb1hBOdkZsUDL6lWpxpWIiOyvFLL2ETub+zRpQR6D1i/ltFf/Rda304ipCOKA8qROrDhl\nFK+fP47yHr3IK66gjz+BjKRA1OKjLalXpRpXIiKyP1LI2kdEm/u0enMZlavX8PXJd/HDr6eQvK3E\nm8AeF8+yY0bx7gU/5H+uB6cc3J2U+AAzc/IpCVbTMzUOn8VELT4qIiIizaOQtY8I396ma02Qg596\nmKumvkt6cQEGVPv8zO0/hDlX3sDWkacC8OmSTXRh+9BiVY0jOc7Psk1lpCd7vWJaCSgiItIyClkd\n1K6uFMzulsQvF79L7L+fpMu6VRhQa0b+AQfz7QXX8NmIM5mxqpDk+AAjnaMkWM3WbVUcNyCt/h7J\n8QEqKqspDlbVH9NKQBERkZZRyOqAmrMxM+CtDHz5ZfjrX2H2bHrW1nrHDziA944+iyVX/hDivIDU\nDTiqvzF3XRG5RUEyOydwwoCuxAT89bcb0C2JGSu2kBwfoDYUxLTljYiISMsoZHVA0VYKbimtYPxb\ni8hKS2T46nmc+dZTJM+YBpWV3kU9esBll8Gdd0J6OkvqNmIOu298TIDTD+lZX2m9LsyBNywYG/CT\n1TWRXqnx9UFMKwFFRERaRiGrA4pcKZhfEmTbvIXc9P6/OXbpN8SWbwOgJiUF/0UXwT33wEEHNbhH\n+BytxjZijlZe4Y7RgxSqREREdgOFrA6obqVgRnkhI559lH6fTaJTaaE3gT0mllVHHs9Hl9zAioOG\n0C0lnvVztpG5KqfBvK3m1qdSeQUREZG2oZDV0WzbxlXvP4n/2efosjkXA2rMx+K+h/DtpT9g02ln\nA5BXXM60ZQWccnD3RudtKUCJiIi0H4WsjqC6Gp54Ah57DBYtoptzODMK+g5g6qiLeeWIs+iRlkS/\n9OT6SxbnltAlsfEK7yIiItK+FLLa0+uvw1/+Al9/DTU13rE+feDaa7Ff/pL05GQuBLLD9iWsm18V\nWX4BVNNKRESkI1HI2tOmT4d774UpUyAY9I6lp8NFF8Fdd0GvXjtcEm1+VWT5BVBNKxERkY6kVSHL\nzEYDfwf8wJPOufsjvv8Z8EOgGsgHrnfOrW7NM/dKy5fD+PHwzjtQVOQdS072gtXdd8PgwTu9ReT8\nqsjyC6ppJSIi0rH4WnqhmfmBx4AzgUOAK8zskIjTvgWGOeeGABOBB1r6vL3Oli1w222QkQEDBsDz\nz0N5OYwaBR9/DCUlMHFiswJWNHW9W6kJMeQWBUlNiNmxWKmIiIi0m9b0ZB0NLHPOrQAws5eA84BF\ndSc456aEnf8VcFUrntfxBYPw0EPw5JOwYoV3zOeDI4/0AtcVV3ifdxOtHhQREem4WhOyMoG1YZ/X\nAcObOP8HwPuteF7HVFsLzzwDDz8M8+Z5nwEGDoRx4+DHP4bY2PZto4iIiOxxe2Tiu5ldBQwDTmzk\n+3HAOICsrKw90aTWmzwZ7rsPpk2DqtCGyr16wZVXwm9+A507t2/7RERE/n979x1eVZX1cfy7SKQX\nHUCBUBVkKCMwBikKUgSF1zIOoDCCo2KbEQvq2IdRfNWxV0ZlEAsWRGw4dgURFZEIooIFBKQKiIhY\nIbDeP/bNm5gJ5Ibk5tyb+/s8Tx5yzzncrJwA/tx7n7UlUqUJWauBJgVeN44d+xUzOwy4HDjU3X8p\n6o3cfTwwHiA7O9tLUVNiffhhWMD+yivwY9jahr32gmOOCQvYmzWLtDwRERFJHqUJWXOBVmbWghCu\nhgJ/KniBmXUC7gWOcPf1pfha0VmxAsaODT2tvvkmHKtWDY4+OmzGnJ0dbX0iIiKSlHY7ZLl7rpmN\nAl4mtHCY6O4LzWwskOPu04AbgZrAE2YGsMLdjy6DuhPru+/guutg0iRYHRucy8yEHj3gkktg4MBo\n6xMREZGkV6o1We7+AvBCoWNjCnx+WGnev1zl5sJdd8E998Dnn4M7mMHvfgfnnAOnnFKmTwaKiIhI\nxZbeHd937IAnnoCbboL58/O3tmnRIoSqCy+EqlWjrVFERERSUnqGrDffDFvbzJoFv8TW4u+9Nxx/\nfNjapl69aOsTERGRlJd+IWv2bDg01kmidm344x/hqqugVato6xIREZEKJf1CVrducOKJcNppcMgh\nUVcjIiIiFVT6hSwIHdpFREREEkiPy4mIiIgkgEKWiIiISAIoZImIiIgkgEKWiIiISAIoZImIiIgk\ngEKWiIiISAIoZImIiIgkgLl71DX8ipltAL6Muo4kUg/4OuoiKhjd07Kl+1n2dE/Llu5n2dM9zdfM\n3esXdSLpQpb8mpnluHt21HVUJLqnZUv3s+zpnpYt3c+yp3saH00XioiIiCSAQpaIiIhIAihkJb/x\nURdQAemeli3dz7KnRarLIAAAIABJREFUe1q2dD/Lnu5pHLQmS0RERCQBNJIlIiIikgAKWUnCzI4w\ns8/MbImZXVLE+Spm9njs/Bwza17+VaaOOO7n+Wa2yMw+NLPXzaxZFHWmkuLuaYHrBpmZm5mePNqF\neO6nmR0X+3O60MweLe8aU00cf++bmtkMM5sf+7s/MIo6U4WZTTSz9Wb28U7Om5ndEbvfH5rZ78u7\nxmSnkJUEzCwDGAcMANoCw8ysbaHLRgKb3L0lcCtwfflWmTrivJ/zgWx3PwCYCtxQvlWmljjvKWZW\nCzgXmFO+FaaWeO6nmbUCLgUOdvd2wHnlXmgKifPP6BXAFHfvBAwF/lW+VaacB4AjdnF+ANAq9nE6\ncHc51JRSFLKSw0HAEndf6u5bgcnAMYWuOQZ4MPb5VKCvmVk51phKir2f7j7D3X+MvXwXaFzONaaa\neP6MAlxN+B+An8uzuBQUz/08DRjn7psA3H19OdeYauK5pw7Ujn1eB1hTjvWlHHd/E/hmF5ccAzzk\nwbvAnmbWsHyqSw0KWckhC1hZ4PWq2LEir3H3XGAzULdcqks98dzPgkYCLya0otRX7D2NTRU0cffn\ny7OwFBXPn9H9gf3N7G0ze9fMdjWiIPHd0yuB4Wa2CngBOLt8SquwSvpvbdrJjLoAkSiZ2XAgGzg0\n6lpSmZlVAm4BToq4lIokkzAN04sw0vqmmf3O3b+NtKrUNgx4wN1vNrNuwCQza+/uO6IuTComjWQl\nh9VAkwKvG8eOFXmNmWUShro3lkt1qSee+4mZHQZcDhzt7r+UU22pqrh7WgtoD7xhZsuBrsA0LX7f\nqXj+jK4Cprn7NndfBnxOCF1StHju6UhgCoC7zwaqEvbgk90T17+16UwhKznMBVqZWQszq0xYkDmt\n0DXTgD/HPh8MTHc1OduZYu+nmXUC7iUELK11Kd4u76m7b3b3eu7e3N2bE9a5He3uOdGUm/Ti+Tv/\nDGEUCzOrR5g+XFqeRaaYeO7pCqAvgJm1IYSsDeVaZcUyDTgx9pRhV2Czu6+NuqhkounCJODuuWY2\nCngZyAAmuvtCMxsL5Lj7NOA+wtD2EsJCxKHRVZzc4ryfNwI1gSdizw+scPejIys6ycV5TyVOcd7P\nl4H+ZrYI2A78zd01er0Tcd7TC4B/m9lowiL4k/Q/qztnZo8Rgn692Dq2fwB7ALj7PYR1bQOBJcCP\nwMnRVJq81PFdREREJAE0XSgiIiKSAApZIiIiIgmgkCUiIiKSAApZIiIiIgmgkCUiIiKSAApZIiIi\nIgmgkCUiIiKSAApZIiIiIgmgkCUiUkpm9r2Z7Rt1HSKSXBSyRGS3mdny2EbbacPM3jCzUwsec/ea\n7q59BUXkVxSyRER2wswyoq5BRFKXQpaIJISZnWZmS8zsGzObZmaNYsfNzG41s/Vm9p2ZfWRm7WPn\nBprZIjPbYmarzezCnbx3JTO7wsy+jL3PQ2ZWJ3buxdhGwQWvX2Bmf4x9/lszezVW12dmdlyB6x4w\ns7vN7AUz+wHoXeh9rgF6AHfFpgjvih13M2tZ4D3+FavjezN728wamNltZrbJzD41s04F3rORmT1p\nZhvMbJmZnVPqmy8iSUEhS0TKnJn1Aa4DjgMaAl8Ck2On+wM9gf2BOrFrNsbO3Qec4e61gPbA9J18\niZNiH72BfYGawF2xc48BwwrU0hZoBjxvZjWAV4FHgb2BocC/Ytfk+RNwDVALeKvgF3X3y4FZwKjY\nFOGvwlwBxwFXAPWAX4DZwLzY66nALbHaKgHPAQuALKAvcJ6ZHb6T9xWRFKKQJSKJcAIw0d3nufsv\nwKVANzNrDmwjBJjfAubun7j72tjv2wa0NbPa7r7J3eft4v1vcfel7v597P2Hmlkm8DTQ0cyaFbj2\nqVgdRwLL3f1+d8919/nAk8CQAu/9rLu/7e473P3n3fz+n3b392O//2ngZ3d/yN23A48DeSNZnYH6\n7j7W3bfG1nX9mxD+RCTFKWSJSCI0IoxeARALQhuBLHefThh1GgesN7PxZlY7dukgYCDwpZnNNLNu\n8bx/7PNMYB933wI8T35QGQY8Evu8GdDFzL7N+yCEsAYF3mvlbn3Hv7auwOc/FfG6ZoF6GhWq5zJg\nnzKoQUQippAlIomwhhAgAIhN09UFVgO4+x3ufiDQljBt+LfY8bnufgxhKu8ZYEo87w80BXLJDzOP\nAcNiIa0qMCN2fCUw0933LPBR093/UuC9vJjvrbjzJbESWFaonlruPrAMv4aIREQhS0RKaw8zq1rg\nI5MQck42s45mVgW4Fpjj7svNrLOZdTGzPYAfgJ+BHWZW2cxOMLM67r4N+A7YsZOv+Rgw2sxamFnN\n2Ps/7u65sfMvEELY2NjxvPf5D7C/mY0wsz1iH53NrE0Jvt91hHVgZeE9YIuZXWxm1cwsw8zam1nn\nMnp/EYmQQpaIlNYLhCmwvI8r3f014O+E9U5rgf3In76rTVh3tIkwzbcRuDF2bgSw3My+A84kTOUV\nZSIwCXgTWEYIamfnnYytv3oKOIywyD3v+BbCwvuhhNGwr4DrgSol+H5vBwbHnhS8owS/77/E1mgd\nCXSMfR9fAxMIDwSISIoz97Ic+RYRERER0EiWiIiISEIoZImIiIgkgEKWiIiISAIoZImIiIgkQGbU\nBRRWr149b968edRliIiIiBTr/fff/9rd6xd1LulCVvPmzcnJyYm6DBEREZFimdmXOzun6UIRERGR\nBFDIEhEREUkAhSwRERGRBFDIEhEREUkAhSwRERGRBFDIEhEREUmA9AtZ27bBkCEwa1bUlYiIiEgF\nln4ha80a+OAD6NULrr4atm+PuiIRERGpgNIvZDVrBvPmwdChMGYM9O8Pa9dGXZWIiIhUMOkXsgBq\n1YKHH4aJE+Hdd6FDB3jppairEhERkQokPUMWgBmcfDLk5ECDBjBgAFx0UVizJSIiIlJK6Ruy8rRp\nA3PmwJlnwo03Qo8esGxZ1FWJiIhIilPIAqhWDe6+G6ZMgU8+gU6dYOrUqKsSERGRFKaQVdCQIeHJ\nw9atw+d/+Qv89FPUVYmIiEgKUsgqrEWL0EPrb3+De+6Brl3D6JaIiIhICShkFaVyZbjhBnjhhdBX\nKzsb7r8f3KOuTERERFKEQtauDBgACxZAly5wyikwYgRs2RJ1VSIiIpICFLKK06gRvPoqjB0Ljz0G\nBx4I8+dHXZWIiIgkOYWseGRkwN//DjNmwI8/hnVad96p6UMRERHZKYWskujZMzx92L8/nHMOHHss\nfPNN1FWJiIhIElLIKql69WDaNLj11rAwvmNHeOutqKsSERGRJKOQtTvM4Lzz4J13wpOIvXrBNdfA\n9u1RVyYiIiJJQiGrNLKzYd680Lj0iivg8MPhq6+irkpERESSgEJWadWuDY8+ChMmhJGtDh3glVei\nrkpEREQippBVFsxg5EiYOxfq1w8jWpdcAtu2RV2ZiIiIREQhqyy1awfvvQennw7XXw+HHgpffhl1\nVSIiIhIBhayyVr063HsvTJ4MCxeGpw+feirqqkRERKScKWQlyvHHh87wLVvCoEEwahT8/HPUVYmI\niEg5iStkmdkRZvaZmS0xs0uKOF/FzB6PnZ9jZs1jx/cwswfN7CMz+8TMLi3b8pPcvvvC22/DBRfA\nuHGhU/xnn0VdlYiIiJSDYkOWmWUA44ABQFtgmJm1LXTZSGCTu7cEbgWujx0fAlRx998BBwJn5AWw\ntFG5Mtx0E/znP7BqVdj78MEHo65KREREEiyekayDgCXuvtTdtwKTgWMKXXMMkJccpgJ9zcwAB2qY\nWSZQDdgKfFcmlaea//kfWLAg9NY66SQ48UT4/vuoqxIREZEEiSdkZQErC7xeFTtW5DXungtsBuoS\nAtcPwFpgBXCTu//XZn9mdrqZ5ZhZzoYNG0r8TaSMrCx4/XX4xz/gkUfCqNYHH0RdlYiIiCRAohe+\nHwRsBxoBLYALzGzfwhe5+3h3z3b37Pr16ye4pIhlZMCVV4aw9f33YZ3WuHHgHnVlIiIiUobiCVmr\ngSYFXjeOHSvymtjUYB1gI/An4CV33+bu64G3gezSFl0h9OoVRrH69g1PHg4aBJs2RV2ViIiIlJF4\nQtZcoJWZtTCzysBQYFqha6YBf459PhiY7u5OmCLsA2BmNYCuwKdlUXiFUL8+PPcc3HxzWBjfsWPY\nmkdERERSXrEhK7bGahTwMvAJMMXdF5rZWDM7OnbZfUBdM1sCnA/ktXkYB9Q0s4WEsHa/u39Y1t9E\nSqtUCc4/P7R6yMiAnj3hn/+EHTuirkxERERKwTzJ1gJlZ2d7Tk5O1GVEY/PmsCXPlCnQrx9MmgT7\n7BN1VSIiIrITZva+uxe5FEod35NJnTphO57x42HWLOjQAV57LeqqREREZDcoZCUbMzjtNJg7F+rW\nhf794fLLITc36spERESkBBSyklX79iFonXIKXHstHHoorFgRdVUiIiISJ4WsZFa9OkyYAI8+Ch99\nFJ4+fOaZqKsSERGROChkpYJhw2DevLDh9LHHwjnnwC+/RF2ViIiI7IJCVqpo2TK0eRg9Gu68E7p1\ng88/j7oqERER2QmFrFRSpQrccktoYPrll2Hvw4cfjroqERERKYJCVio68khYsAA6dYIRI+Ckk8I+\niCIiIpI0FLJSVePGMH06/P3v8NBDkJ0NH6qZvoiISLJQyEplmZkwdmxoWLp5Mxx0ENx9NyRZF38R\nEZF0pJBVEfTpE6YPe/eGv/4VhgyBb7+NuioREZG0ppBVUey9Nzz/PNx4Izz7LDRrBiNHwowZ2mxa\nREQkAgpZFUmlSnDhhTBnTuinNWVKGOVq1gwuvlhrtkRERMqRQlZF9PvfwwMPwLp1YcPpjh1D64cO\nHeCAA+CGG2DlyqirFBERqdAUsiqy6tXh+ONDX621a2HcOKhZM4xqNWsW1nDdd5/Wb4mIiCSAQla6\nqFcvLIp/5x1YsgSuvBJWr4ZTT4UGDWDw4LAvorbrERERKRMKWelov/1gzBj47DN47z044wx4882w\njqthQzjzTJg1SwvmRURESkEhK52ZQefOcPvtsGYNvPgiDBwIkyZBz55hQ+rLL4dFi6KuVEREJOUo\nZEmQmQlHHBH2Qly3LgStNm3gn/+Edu3CYvpbbglhTERERIqlkCX/rWZNGD48jGytXg233QYZGXDB\nBdCkCfTrBw8+CFu2RF2piIhI0lLIkl1r0ADOPRfmzoVPPw3Th198ETal3mcfGDoU/vMf2LYt6kpF\nRESSikKWxK9167BX4hdfhKcUTz457Jt41FHQqBGMGgWzZ2vvRBERERSyZHeYQbduoe/WmjUwbRr0\n7Rt6bnXvDq1awT/+AZ9/HnWlIiIikVHIktKpXDmMZE2eHBbM338/tGgBV18dRr4OOgjuuCOcExER\nSSMKWVJ2atcOa7VefTVs23PTTWGt1rnnQlYWDBgAjzwCP/wQdaUiIiIJp5AliZGVFZ5GnD8fPv4Y\nLroo9NsaPjwsmB8xAl56CXJzo65UREQkIRSyJPHatYNrr4Vly2DmTDjhhPBE4oABIYydd154elEL\n5kVEpAJRyJLyU6lS6CR/773w1Vfw1FPQowfcfXdYu/Xb34a1XEuXRl2piIhIqSlkSTSqVAl7JU6d\nGgLXv/8d9k0cMybsrdi9O/zrX/D111FXKiIislsUsiR6e+0Fp54Kb7wBX34ZtvL57js466wQvI46\nCh5/HH78MepKRURE4qaQJcmlaVO4+GL46CP44AMYPRrmzQud5Rs0yG+Aun171JWKiIjskkKWJCcz\n6NABbrgBVqyA11+HwYPDOq5+/UIYu+wyWLIk6kpFRESKpJAlyS8jA/r0gYkTw/qtKVOgUye4/vrQ\nXb53b3j4Yfjpp6grFRER+X9xhSwzO8LMPjOzJWZ2SRHnq5jZ47Hzc8yseYFzB5jZbDNbaGYfmVnV\nsitf0k61ajBkSGgBsWIFXHNN+HXEiLB+66yzwvSiiIhIxIoNWWaWAYwDBgBtgWFm1rbQZSOBTe7e\nErgVuD72ezOBh4Ez3b0d0AvYVmbVS3rLygpThosXw/TpcOSRYf/EAw8MI13jxsGmTVFXKSIiaSqe\nkayDgCXuvtTdtwKTgWMKXXMM8GDs86lAXzMzoD/wobsvAHD3je6uFctStipVyp8yXLsW7rorrOka\nNQoaNQrNT2fMgB07oq5URETSSDwhKwtYWeD1qtixIq9x91xgM1AX2B9wM3vZzOaZ2UVFfQEzO93M\ncswsZ8OGDSX9HkTy7bVX/pTh++/DKafA88+HNV2tWoXpxdWro65SRETSQKIXvmcChwAnxH491sz6\nFr7I3ce7e7a7Z9evXz/BJUna+P3vw5Th2rVhlKtpU7jiivDrkUfCM8+EDaxFREQSIJ6QtRpoUuB1\n49ixIq+JrcOqA2wkjHq96e5fu/uPwAvA70tbtEiJVKuWP2W4eHHowzVvXug436RJ2Lz6s8+irlJE\nRCqYeELWXKCVmbUws8rAUGBaoWumAX+OfT4YmO7uDrwM/M7MqsfC16HAorIpXWQ3tGwZNqtesQKe\new66doVbbgn7JvboAQ88AD/8EHWVIiJSARQbsmJrrEYRAtMnwBR3X2hmY83s6Nhl9wF1zWwJcD5w\nSez3bgJuIQS1D4B57v582X8bIiWUmZk/ZbhqVei5tX596CjfsCGccQa89x64R12piIikKPMk+49I\ndna25+TkRF2GpCN3eOut0AZiypTQ3PR3v4ORI2H4cKhbN+oKRUQkyZjZ++6eXdQ5dXwXyWOWP2W4\ndi3ccw9UrQrnnRdaQQwdCq++qlYQIiISF4UskaLUqZM/ZbhgAZx5ZghY/fvDvvvC2LGwcmXx7yMi\nImlLIUukOAccALffHvprPfZY6Lf1j39As2ZwxBEwdSps3Rp1lSIikmQUskTiVbVq/pTh0qWh59bC\nhWEvxawsuOACWKSHZ0VEJFDIEtkdLVqEKcPly+HFF+HQQ+HOO6FdO+jWLSye37Il6ipFRCRCClki\npZGRkT9luHo13HwzbN4Mp54aWkGMHAmzZ6sVhIhIGlLIEikr9evD+eeHKcR33glTi48/Dt27hxGu\nm28G7c0pIpI2FLJEyppZmDKcMCG0gpgwAfbcEy68MKzdGjw4TDFu3x51pSIikkAKWSKJVKtWmDJ8\n550wwnX22TBzJgwcCM2bw5gxsGxZ1FWKiEgCKGSJlJe2bcOU4erV8MQT0L49/O//hr5b/frB5Mnw\n889RVykiImVEIUukvFWunD9luHw5XHUVLF4Mw4aF6cRzz4UPP4y6ShERKSWFLJEoNW0apgyXLg39\nt/r1C9v5dOgQPq6+Gj79NOoqRURkNyhkiSSDSpXgsMPClOGaNXDHHVC7dugs36ZN2Kh67Fg1OxUR\nSSHmSda/Jzs723NycqIuQyQ5rFkDTz4Z1nC99Vbot9W2begyP2RIaA0hIiKRMbP33T27yHMKWSIp\nYu1aeOqpELjefDMErjZtwvquIUPCQnqzqKsUEUkrClkiFc1XX8HTT4fANXMm7NgBrVvnB64DDlDg\nEhEpBwpZIhXZunUhcE2dCjNmhMDVqlUIW4MHQ8eOClwiIgmikCWSLjZsyA9c06eHrvItW+aPcHXq\npMAlIlKGFLJE0tHXX8Mzz4QpxddfD4Fr333zA9eBBypwiYiUkkKWSLrbuBGefTYErtdeg9zcsK1P\nXuDq3FmBS0RkN+wqZKlPlkg6qFsXTjkldJlftw7uvz+0grj9dujSJQSuCy6Ad98NTy2KiEipaSRL\nJJ1t2gTTpoU1XK+8Alu3QpMmYYRr8GDo2jU0ShURkSJpulBEird5c37geumlELgaN4ZBg8KUYrdu\nClwiIoVoulBEilenDowYEdZubdgADz8cFsffcw8cckgY4TrnHJg1K7SJEBGRXVLIEpH/Vrs2nHBC\neDpx/Xp49NGwduvf/4aePcMI16hRoRHq9u1RVysikpQUskRk12rXhmHDwpY+69eHTay7d4eJE6FX\nrxC4zjorNEJV4BIR+X8KWSISv1q14Pjjw7qt9evh8cehRw944AHo0wcaNYK//CU0Qs3NjbpaEZFI\nKWSJyO6pWROOOw6mTAmB64knwsjWQw9B374hcJ1xRn5fLhGRNKOQJSKlV6NGaPnw+ONh0fyTT4ag\n9cgj0K8fNGgAp50W2kRs2xZ1tSIi5UIhS0TKVvXq8Mc/wmOP5e+lePjhIYAdfngIXCNHwquvag2X\niFRoClkikjjVqsEf/hBGtNavD+0hBg4Ma7r694cWLWDMGFi6NOpKRUTKnEKWiJSPqlXh6KNh0qT8\nNVzt28M118B++4WF8w8/DD/+GHWlIiJlQiFLRMpflSphDdcLL8CXX4agtWJFaIbasCGceSa89572\nURSRlBZXyDKzI8zsMzNbYmaXFHG+ipk9Hjs/x8yaFzrf1My+N7MLy6ZsEakwGjeGyy6DxYtDc9Nj\njw2jXV26hJGum28OI18iIimm2JBlZhnAOGAA0BYYZmZtC102Etjk7i2BW4HrC52/BXix9OWKSIVl\nFrrJP/AArF0busvXqQMXXghZWSF8Pfec2kGISMqIZyTrIGCJuy91963AZOCYQtccAzwY+3wq0NfM\nDMDM/gAsAxaWTckiUuHVrg2nngrvvAOLFsHo0TB7dljT1aQJXHwxfPpp1FWKiOxSPCErC1hZ4PWq\n2LEir3H3XGAzUNfMagIXA1ft6guY2elmlmNmORs2bIi3dhFJB23awA03wMqV4enELl3CFGKbNnDw\nwXDffbBlS9RVioj8l0QvfL8SuNXdv9/VRe4+3t2z3T27fv36CS5JRFLSHnuEkaxnnoFVq+DGG2HT\npjDi1aABnHwyzJqlxfIikjTiCVmrgSYFXjeOHSvyGjPLBOoAG4EuwA1mthw4D7jMzEaVsmYRSXcN\nGoS1WgsXhmnEE04IXeZ79oT994frroPVhf+ZEhEpX/GErLlAKzNrYWaVgaHAtELXTAP+HPt8MDDd\ngx7u3tzdmwO3Ade6+11lVLuIpDsz6NoVxo8Pi+UffDAskr/sMmjaNL/x6S+/RF2piKShYkNWbI3V\nKOBl4BNgirsvNLOxZnZ07LL7CGuwlgDnA//V5kFEJKFq1IATT4Q33gjtIC69FD76CIYMCcHrvPPg\nww+jrlJE0oh5kq1fyM7O9pycnKjLEJGKYPt2eO01mDgxrOXauhUOPBBOOQWGDYO99oq6QhFJcWb2\nvrtnF3VOHd9FpOLKyMjfnHrNGrjjjtBn66yzQmf5P/0phLAdO6KuVEQqIIUsEUkPdevC2WfDBx/A\nvHlw2mnw0kvQrx/suy9ceSUsXx51lSJSgShkiUj66dQJ7rwzjG5NngytW8PYsdCiBRx2GDz6KPz0\nU9RVikiKU8gSkfRVtSocfzy8/HIYxRo7FpYuDS0hGjaEv/4VcnLUe0tEdotClogIhJYPf/87LFkC\n06fDUUeFfRQ7d4YOHeDWW0E7UohICShkiYgUVKkS9O4NkyaF3lv33gvVq8P554dWEIMGwfPPa6Nq\nESmWQpaIyM7UqQOnnw7vvgsffwznnBO27jnyyDDydeml8PnnUVcpIklKIUtEJB7t2sFNN4V9E59+\nGrKzw/6JrVtDjx5w//3w/S63aRWRNKOQJSJSEpUrwx/+ANOmwcqVcP31Ya3WKaeEPRVHjoS339Zi\neRFRyBIR2W0NG8JFF8Enn4RgNXQoTJkChxwCv/0tXHttCGIikpYUskRESssMuneHCRPCYvn774d9\n9oHLL4dmzULD00cegR9/jLpSESlHClkiImWpZk046SR480344gsYMya0hRg+PH86cdYsTSeKpAGF\nLBGRRMnbrueLL+CNN2Dw4DCd2LMntGwJV10Fy5ZFXaWIJIhClohIolWqBIceChMnwldfwUMPhS18\nrroqBLFevcIU45YtUVcqImVIIUtEpDzVqAEjRsBrr4WtfP73f2H16vynE088EV5/HXbsiLpSESkl\nhSwRkag0bRoWx3/+eXg6cfhwePbZsEl1ixZwxRWweHHUVYrIblLIEhGJWt7TiffeG6YTH3sM2raF\n666D/feHgw+G8ePh22+jrlRESkAhS0QkmVSrFvptvfhifrPTb7+FM84IfbmGDYOXXoLt26OuVESK\noZAlIpKsGjUKzU4//hjeey+0f3j5ZRgwAJo0gYsvhkWLoq5SRHZCIUtEJNmZQefOcNddodnp1Klh\n78Sbbw57KnbuDOPGwcaNUVcqIgUoZImIpJIqVWDQoLB34urVcMstsG0bjBoVphMHDYLnngvHRCRS\nClkiIqlqn31g9Gj44AOYPx/OOit0kz/6aGjcGM4/HxYsiLpKkbSlkCUiUhF07Ai33hpGt559NmxS\nfddd4XjHjnDbbbB+fdRViqQVhSwRkYpkjz3CSNaTT4b1W3feGY6NHg1ZWeHcU0/BL79EXalIhaeQ\nJSJSUdWtG9ZqzZ0bnlAcPRpycsK6rUaNwrmcHG1WLZIgClkiIumgXTu44QZYsSL04OrXDyZMCE8m\ntm8fzq1ZE3WVIhWKQpaISDrJzIQjjoDJk0N3+XvugTp1Qs+tJk1CD67Jk+Gnn6KuVCTlKWSJiKSr\nPfcMneTfeQc+/RQuuQQWLgxd5Rs2zD+n6USR3aKQJSIi0Lo1XHMNLF8Or70GRx0FkyaFfRPzzq1Y\nEXWVIilFIUtERPJVqgR9+4aAtW4dTJwYRrWuuAKaN4fDDgvnfvgh6kpFkp5CloiIFK1WLTj5ZJg5\nE774AsaMgaVL4cQToUEDOO64sKZr8WJNKYoUwTzJ/mJkZ2d7Tk5O1GWIiEhRduyAt96Chx6Cl14K\nzU8hdJjv0yeMgvXpE16LpAEze9/ds4s6l1nexYiISAqrVAl69gwf7rBkCUyfDq+/Di+8EMIXQKtW\n+aGrVy+oXz/SskWiENdIlpkdAdwOZAAT3P2fhc5XAR4CDgQ2Ase7+3Iz6wf8E6gMbAX+5u7Td/W1\nNJIlIpKiduyLCdEtAAAJYElEQVQITU9ffz0Er5kzYcuWcO6AA/JHuXr2hNq1o61VpIzsaiSr2JBl\nZhnA50A/YBUwFxjm7osKXPNX4AB3P9PMhgLHuvvxZtYJWOfua8ysPfCyu2ft6uspZImIVBC5uaGj\n/PTp4ePtt+HnnyEjA7Kz80NX9+5QrVrU1YrsltKGrG7Ale5+eOz1pQDufl2Ba16OXTPbzDKBr4D6\nXuDNzcwIo1wN3X2nm2YpZImIVFA//wyzZ+eHrjlzYPt2qFw5BK280NW5c9hvUSQF7CpkxfN0YRaw\nssDrVbFjRV7j7rnAZqBuoWsGAfOKClhmdrqZ5ZhZzoYNG+IoSUREUk7VqtC7N1x9dRjV2rQJnn8e\nzj4bNm8OTy8efDDstRcMHAg33wzz54dpSJEUVC4L382sHXA90L+o8+4+HhgPYSSrPGoSEZGI1aoV\nwtTAgeH1xo3wxhv5C+lffDEc/81vQjjr0yd8tG4NZpGVLRKveELWaqBJgdeNY8eKumZVbLqwDmFq\nEDNrDDwNnOjuX5S6YhERqZjq1oVBg8IHhPYQM2bkh64nnwzHGzXKD1x9+kCzZtHVLLIL8YSsuUAr\nM2tBCFNDgT8VumYa8GdgNjAYmO7ubmZ7As8Dl7j722VXtoiIVHhZWTB8ePhwD41Q89ZzvfIKPPxw\nuG6//fIDV+/esM8+0dYtEhNvC4eBwG2EFg4T3f0aMxsL5Lj7NDOrCkwCOgHfAEPdfamZXQFcCiwu\n8Hb93X39zr6WFr6LiEix3MNm1nmjXG+8Ad99F861b58fug49NGyELZIgpXq6sLwpZImISInl5oZF\n8nmh66234KefQvPUAw/MD12HHALVq0ddrVQgClkiIpJefvkltIjIa4z67rshiO2xB3Trlh+6unQJ\nLSREdpNCloiIpLfvvw9tI/JC17x5YcqxenXo0SN/C6COHUOzVJE4KWSJiIgUtGlTfruI6dNhUWwT\nkz33DHst5oWuNm3ULkJ2SSFLRERkV9auzW8XMX06LFsWju+zT3hiMa9P1377KXTJryhkiYiIlMSy\nZSFs5QWvtWvD8caN81tF9OkDTZtGW6dETiFLRERkd7nD55/nh64ZM+Drr8O5/fbLD1y9e0ODBtHW\nKuVOIUtERKSs7NgBH3+cP8o1c2bYexHCGq680NWrV+hiLxWaQpaIiEiibN8eenTlha5Zs+CHH8K5\nDh3yR7l69oQ6daKtVcqcQpaIiEh52bYN5s7ND11vvx36dhVujHrwwVCjRtTVSikpZImIiETl559D\nM9S8NV0FG6N26ZI/vdi1K1StGnW1UkIKWSIiIsnihx/C6FZe6MrJCeu8qlaF7t3zpxc7dw5BTJKa\nQpaIiEiy2rwZ3nwzf3pxwYJwvEaN/G70vXtDp07qRp+EFLJERERSxddfhycW80LXJ5+E43XqhCcW\n86YX27UL67wkUgpZIiIiqWrt2vwtgGbMgC++CMfr1ft1N/r991c3+ggoZImIiFQUK1b8egugVavC\n8YYN859c7N0bWrSIts40oZAlIiJSEbmHka2CoWv9+nCuefNfd6PPyoq01IpKIUtERCQduIc1XAW3\nANq0KZzbf//8wNWrF+y9d6SlVhQKWSIiIuloxw748MP80DVzJmzZEs61awetWkGjRmGUq/Cvdepo\njVccFLJEREQkNEF9//0QuGbNCuu71qyBb77572urVSs6fBX8tVGjtG+gqpAlIiIiO/fTT+EpxtWr\nQ+gq6tfVq0P3+sJ+85viw9jee1fYHl+7ClmZ5V2MiIiIJJlq1WDffcPHzrjDt98WHb7yPv/oI/jq\nqzBNWVBGBjRoUHwYq2BTlApZIiIiUjwz2Guv8NGu3c6vy80NTzjubFRs8eKwNixvQX5B1avvOoRl\nZYVWFSkyRamQJSIiImUnMzN/vdau/Pjjrqco58wJvxY1RVm3bvFhrH79yKcoFbJERESk/FWvDvvt\nFz52xj2MeO1sndiaNWGvx3Xrip6iPPZYeOKJxH4fu6CQJSIiIsnJLCys/81voH37nV+XmxuCVuEQ\n1rRp+dVaBIUsERERSW2ZmWGKMCsLOneOupr/p+27RURERBJAIUtEREQkARSyRERERBJAIUtEREQk\nARSyRERERBJAIUtEREQkARSyRERERBLA3D3qGn7FzDYAX5bDl6oHfF0OX0cSQz+/1KefYerTzzD1\n6WdYes3cvX5RJ5IuZJUXM8tx9+yo65Ddo59f6tPPMPXpZ5j69DNMLE0XioiIiCSAQpaIiIhIAqRz\nyBofdQFSKvr5pT79DFOffoapTz/DBErbNVkiIiIiiZTOI1kiIiIiCaOQJSIiIpIAaReyzOwIM/vM\nzJaY2SVR1yMlY2ZNzGyGmS0ys4Vmdm7UNUnJmVmGmc03s/9EXYuUnJntaWZTzexTM/vEzLpFXZOU\njJmNjv0b+rGZPWZmVaOuqSJKq5BlZhnAOGAA0BYYZmZto61KSigXuMDd2wJdgbP0M0xJ5wKfRF2E\n7LbbgZfc/bdAB/SzTClmlgWcA2S7e3sgAxgabVUVU1qFLOAgYIm7L3X3rcBk4JiIa5IScPe17j4v\n9vkWwj/uWdFWJSVhZo2B/wEmRF2LlJyZ1QF6AvcBuPtWd/822qpkN2QC1cwsE6gOrIm4ngop3UJW\nFrCywOtV6D/QKcvMmgOdgDnRViIldBtwEbAj6kJkt7QANgD3x6Z8J5hZjaiLkvi5+2rgJmAFsBbY\n7O6vRFtVxZRuIUsqCDOrCTwJnOfu30Vdj8THzI4E1rv7+1HXIrstE/g9cLe7dwJ+ALS+NYWY2V6E\nWZwWQCOghpkNj7aqiindQtZqoEmB141jxySFmNkehID1iLs/FXU9UiIHA0eb2XLCdH0fM3s42pKk\nhFYBq9w9bwR5KiF0Seo4DFjm7hvcfRvwFNA94poqpHQLWXOBVmbWwswqExb6TYu4JikBMzPCWpBP\n3P2WqOuRknH3S929sbs3J/z9m+7u+j/oFOLuXwErzax17FBfYFGEJUnJrQC6mln12L+pfdHDCwmR\nGXUB5cndc81sFPAy4WmKie6+MOKypGQOBkYAH5nZB7Fjl7n7CxHWJJJuzgYeif3P6lLg5IjrkRJw\n9zlmNhWYR3hiez7aXichtK2OiIiISAKk23ShiIiISLlQyBIRERFJAIUsERERkQRQyBIRERFJAIUs\nERERkQRQyBIRERFJAIUsERERkQT4P7xSihlyLZ3GAAAAAElFTkSuQmCC\n","text/plain":["
"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"6gBIbd9eiEs0","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]} -------------------------------------------------------------------------------- /labs/transfer_learning.ipynb: -------------------------------------------------------------------------------- 1 | {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"transfer_learning.ipynb","provenance":[{"file_id":"1z1U96GN66FkYS-vVtx3fl-Lyvr_TiS47","timestamp":1561810535270},{"file_id":"1bqE06VfVpNXtL0We-fbBEFO50BW450AJ","timestamp":1555169281814}],"collapsed_sections":["h7R_FcIPA1Ko"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"h7R_FcIPA1Ko","colab_type":"text"},"source":["## Setup"]},{"cell_type":"code","metadata":{"id":"HTqhOE9jWQRc","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":0},"outputId":"d77d4d0d-d43d-4823-bd7b-1a7c253aad39","executionInfo":{"status":"ok","timestamp":1577688609465,"user_tz":-480,"elapsed":7336,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}}},"source":["!pip uninstall tensorflow -yq\n","!pip install tensorflow-gpu>=2.0 gpustat -qU"],"execution_count":1,"outputs":[{"output_type":"stream","text":["\u001b[33mWARNING: Skipping tensorflow as it is not installed.\u001b[0m\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"nOug94VRWWd0","colab_type":"code","colab":{}},"source":["import time\n","import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","import cv2\n","\n","import tensorflow\n","\n","import tensorflow.compat.v2 as tf\n","import tensorflow.keras.layers as layers\n","import tensorflow_datasets as tfds"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"oEi36ou7KySx","colab_type":"text"},"source":["# Transfer Learning with Keras"]},{"cell_type":"markdown","metadata":{"id":"WYijcApeY5R4","colab_type":"text"},"source":["In this notebook, we are going to be doing image classification on the [Kaggle \"Cats vs Dogs\" dataset](https://www.kaggle.com/c/dogs-vs-cats).\n","\n","However, we are not going to be training the model from scratch. Instead, we will be leveraging **transfer learning** to reduce the amount of time needed to train a very high accuracy model on a relatively small dataset. Transfer learning works by taking a model that has been trained on a much larger dataset (e.g. ImageNet), and then fine-tuning it for another task. This works because much of the model learns features that apply to the same type images in general (e.g. RGB photographs). \n","\n","This notebook demonstrates a typical example of transfer learning where we take a model that has been trained on ImageNet, and fine-tune it for our own task."]},{"cell_type":"code","metadata":{"id":"KEiB77wqK1vy","colab_type":"code","outputId":"2ca764f6-7af1-4786-ce7e-0486d88893d9","executionInfo":{"status":"ok","timestamp":1577688615874,"user_tz":-480,"elapsed":13723,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":53}},"source":["!gpustat"],"execution_count":3,"outputs":[{"output_type":"stream","text":["\u001b[1m\u001b[37m7875b60ed81f \u001b[m Mon Dec 30 06:50:13 2019 \u001b[1m\u001b[30m418.67\u001b[m\n","\u001b[36m[0]\u001b[m \u001b[34mTesla P100-PCIE-16GB\u001b[m |\u001b[31m 43'C\u001b[m, \u001b[32m 0 %\u001b[m | \u001b[36m\u001b[1m\u001b[33m 0\u001b[m / \u001b[33m16280\u001b[m MB |\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"AZieFWCpLUtO","colab_type":"text"},"source":["## Construct Input Pipeline"]},{"cell_type":"markdown","metadata":{"id":"eVgB1lohZu2l","colab_type":"text"},"source":["This section demonstrates how to build a high-performance input pipeline for feeding in training and evaluation data into the model."]},{"cell_type":"markdown","metadata":{"id":"z4O-FO1UaYeY","colab_type":"text"},"source":["### Load Dataset using TensorFlow Datasets\n","\n","\"Cats vs Dogs\" is not a standard research dataset, and is not found in `tf.keras.datasets` like the more common MNIST or CIFAR10. Fortunately, **TensorFlow Datasets** not only contains this dataset, but also many others. \n","\n","When we load datasets with TensorFlow Datasets, we get [`tf.data.Dataset`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/data/Dataset) objects. This means we automatically take advantage of high performance `tf.data` primitives to create highly-optimized, efficient input pipelines that result in better utilization of our hardware accelerators, such as GPUs or TPUs that might otherwise be constantly \"waiting\" for data from the CPU. This is important because hardware accelerators for deep learning are so fast, the speed at which we can feed images into the accelerator is usually a significant bottleneck. \n","\n","Using `tf.data` is a best practice (especially for reading large datasets). You can read more about using tf.data at the following links:\n","\n","* https://www.tensorflow.org/beta/guide/data\n","* https://www.tensorflow.org/guide/performance/datasets"]},{"cell_type":"code","metadata":{"id":"kf0vStKoaYCH","colab_type":"code","outputId":"a079377c-f0a4-4e66-ddaa-20ce8e74f442","executionInfo":{"status":"ok","timestamp":1577688616376,"user_tz":-480,"elapsed":14214,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["# define train-validation-test split\n","splits = tfds.Split.TRAIN.subsplit(weighted=(8, 1, 1))\n","\n","# load dataset with corresponding split\n","(raw_train, raw_validation, raw_test), info = tfds.load(\"cats_vs_dogs\",\n"," split=list(splits),\n"," with_info=True,\n"," as_supervised=True)\n","\n","num_examples = info.splits[\"train\"].num_examples\n","num_train = int(num_examples * 0.8)\n","num_val = int(num_examples * 0.1)\n","num_test = int(num_examples * 0.1)\n","\n","print(\"Split:\", num_train, num_val, num_test)"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Split: 18609 2326 2326\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"WOy7hAFtcnaW","colab_type":"text"},"source":["### Build `tf.data` Pipeline"]},{"cell_type":"code","metadata":{"id":"6Ob-z76OwFQL","colab_type":"code","colab":{}},"source":["BATCH_SIZE = 64\n","IMG_SIZE = (256, 256)\n","\n","@tf.function\n","def format_example(image, label):\n"," \"\"\"\n"," This function will run as part of a tf.data pipeline.\n"," It is reponsible for resizing and normalizing the input images.\n"," \"\"\"\n"," image = tf.cast(image, tf.float32)\n"," image = tf.image.resize(image, IMG_SIZE)\n"," image = (image/127.5) - 1\n"," return image, label"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"CMD5_KhndBJK","colab_type":"text"},"source":["**Input Pipeline for Train Images**"]},{"cell_type":"code","metadata":{"id":"iMO6uEZPKmc-","colab_type":"code","colab":{}},"source":["train = raw_train.shuffle(4096)\n","train = train.repeat(count=-1)\n","train = train.map(format_example, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","train = train.batch(BATCH_SIZE)\n","train = train.prefetch(tf.data.experimental.AUTOTUNE)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"tgvLwbffdL9-","colab_type":"text"},"source":["**Input Pipeline for Validation and Test Images**\n","\n","We remove some unnecessary steps (like shuffling) and increase the batch size for faster inferencing."]},{"cell_type":"code","metadata":{"id":"bPxsljSkdMKC","colab_type":"code","colab":{}},"source":["val = raw_validation.repeat(count=-1)\n","val = val.map(format_example, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","val = val.batch(BATCH_SIZE*2)\n","val = val.prefetch(tf.data.experimental.AUTOTUNE)\n","\n","test = raw_test.repeat(count=-1)\n","test = test.map(format_example, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","test = test.batch(BATCH_SIZE*2)\n","test = test.prefetch(tf.data.experimental.AUTOTUNE)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"AUB3aSlaN0iH","colab_type":"text"},"source":["## Construct the Model"]},{"cell_type":"code","metadata":{"id":"7h0pyAF0L_EX","colab_type":"code","colab":{}},"source":["# you could also use other models like the following ResNet 50:\n","# from tensorflow.keras.applications.resnet50 import ResNet50\n","\n","from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"UmMHAwp688E-","colab_type":"code","outputId":"3644fe1d-49e5-47c6-967d-7544b548b7b2","executionInfo":{"status":"ok","timestamp":1577688619574,"user_tz":-480,"elapsed":17330,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":73}},"source":["input_layer = layers.Input(shape=(IMG_SIZE[0],IMG_SIZE[1],3,))\n","\n","base = MobileNetV2(input_tensor=input_layer, include_top=False, weights='imagenet')\n","\n","base.trainable = False\n","\n","x = base.output\n","x = layers.GlobalAveragePooling2D()(x)\n","x = layers.Dropout(0.2)(x)\n","preds = layers.Dense(2, activation=\"softmax\")(x)\n","\n","model = tf.keras.models.Model(inputs=input_layer, outputs=preds)"],"execution_count":9,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/keras_applications/mobilenet_v2.py:294: UserWarning: `input_shape` is undefined or non-square, or `rows` is not in [96, 128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n"," warnings.warn('`input_shape` is undefined or non-square, '\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"u90VRx_5NiS6","colab_type":"code","colab":{}},"source":["opt = tf.keras.optimizers.Adam()\n","\n","model.compile(loss=\"sparse_categorical_crossentropy\",\n"," optimizer=opt,\n"," metrics=[\"accuracy\"])"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"CpB3PSWgqTLw","colab_type":"text"},"source":["## Train Model"]},{"cell_type":"code","metadata":{"id":"2zkA41zrd4YJ","colab_type":"code","outputId":"6bd9ba3c-24a6-44ca-979b-1d710057b602","executionInfo":{"status":"ok","timestamp":1577688728522,"user_tz":-480,"elapsed":126258,"user":{"displayName":"Timothy Liu","photoUrl":"https://lh3.googleusercontent.com/a-/AAuE7mCS8LwI0k0t2vkujQUBYfPPAo7627gWEnmKxe1ocA=s64","userId":"03413426750796061565"}},"colab":{"base_uri":"https://localhost:8080/","height":125}},"source":["train_steps = num_train//BATCH_SIZE\n","valid_steps = num_val//(BATCH_SIZE*2)\n","\n","model.fit(train, validation_data=val,\n"," validation_steps=valid_steps,\n"," steps_per_epoch=train_steps,\n"," epochs=2)"],"execution_count":11,"outputs":[{"output_type":"stream","text":["Train for 290 steps, validate for 18 steps\n","Epoch 1/2\n","290/290 [==============================] - 58s 201ms/step - loss: 0.1507 - accuracy: 0.9396 - val_loss: 0.1010 - val_accuracy: 0.9614\n","Epoch 2/2\n","290/290 [==============================] - 50s 173ms/step - loss: 0.0894 - accuracy: 0.9657 - val_loss: 0.1019 - val_accuracy: 0.9640\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":11}]}]} --------------------------------------------------------------------------------