├── git
├── 06_Train_GPT-2
    └── homework
    │   ├── Ghamarani
    │       └── checkpoint.pdf
    │   └── nima_kaveh
    │       ├── figure
    │           ├── 2.1.png
    │           ├── 2.2.png
    │           ├── 2.3.png
    │           ├── 3.png
    │           ├── 4.png
    │           ├── 5.1.png
    │           ├── 5.2.png
    │           ├── 6.png
    │           ├── 7.1.png
    │           └── 7.2.png
    │       └── Train_GPT-2.pdf
├── README.md
├── 05_Train_Eval_Gen_Functions
    └── 1_Train_Eval_Gen_General.ipynb
└── 08_Evaluation
    └── Evaluation.ipynb


/git:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/Ghamarani/checkpoint.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/Ghamarani/checkpoint.pdf


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/2.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/2.1.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/2.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/2.2.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/2.3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/2.3.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/3.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/4.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/5.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/5.1.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/5.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/5.2.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/6.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/6.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/7.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/7.1.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/figure/7.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/figure/7.2.png


--------------------------------------------------------------------------------
/06_Train_GPT-2/homework/nima_kaveh/Train_GPT-2.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/howsam/Building-a-ChatGPT-like-Model-from-Scratch/HEAD/06_Train_GPT-2/homework/nima_kaveh/Train_GPT-2.pdf


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Building-a-ChatGPT-like-Model-from-Scratch
2 | This repository contains a step-by-step implementation of a ChatGPT-like model from scratch using PyTorch. It covers everything from tokenization and transformer architecture to training and fine-tuning on conversational data. The course is designed for those who want to deeply understand how large language models work and build their own! 🚀
3 | 


--------------------------------------------------------------------------------
/05_Train_Eval_Gen_Functions/1_Train_Eval_Gen_General.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |   "nbformat": 4,
   3 |   "nbformat_minor": 0,
   4 |   "metadata": {
   5 |     "colab": {
   6 |       "provenance": [],
   7 |       "collapsed_sections": [
   8 |         "VrFCH9hpEyip",
   9 |         "3in1e9BksgIh",
  10 |         "xv32RDGrJhqv",
  11 |         "w_a3OXnSeV0z",
  12 |         "RwaY_YcgRayy",
  13 |         "mf-QHos1nK3D",
  14 |         "RTql4Ftiunfr",
  15 |         "ZnBQs-C8PNrY",
  16 |         "lAoYRsvKN_8l",
  17 |         "WTVewgG-m7nq",
  18 |         "ih6sV9ljndzW",
  19 |         "LCzTHf8iitEt",
  20 |         "Q1Gx0mlvVJsV",
  21 |         "W0QNbC0YPCKZ",
  22 |         "ACjVH6bBpAEV",
  23 |         "G9HgVWslPGsH",
  24 |         "6kSRpX11o6Dm",
  25 |         "o_5f69nwPtY2",
  26 |         "CgpgXUYbuwHn"
  27 |       ],
  28 |       "gpuType": "T4",
  29 |       "include_colab_link": true
  30 |     },
  31 |     "kernelspec": {
  32 |       "name": "python3",
  33 |       "display_name": "Python 3"
  34 |     },
  35 |     "language_info": {
  36 |       "name": "python"
  37 |     },
  38 |     "accelerator": "GPU"
  39 |   },
  40 |   "cells": [
  41 |     {
  42 |       "cell_type": "markdown",
  43 |       "metadata": {
  44 |         "id": "view-in-github",
  45 |         "colab_type": "text"
  46 |       },
  47 |       "source": [
  48 |         "<a href=\"https://colab.research.google.com/github/howsam/Building-a-ChatGPT-like-Model-from-Scratch/blob/main/1_Train_Eval_Gen_General.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
  49 |       ]
  50 |     },
  51 |     {
  52 |       "cell_type": "markdown",
  53 |       "metadata": {
  54 |         "id": "-uXkcYhkIxS-"
  55 |       },
  56 |       "source": [
  57 |         "#  <font color='#FFE15D'><b>💎 Train, Evaluate, and Generate Functions (General)</b></font>"
  58 |       ]
  59 |     },
  60 |     {
  61 |       "cell_type": "markdown",
  62 |       "metadata": {
  63 |         "id": "VrFCH9hpEyip"
  64 |       },
  65 |       "source": [
  66 |         "# 🔴 **Environment Setup**"
  67 |       ]
  68 |     },
  69 |     {
  70 |       "cell_type": "markdown",
  71 |       "metadata": {
  72 |         "id": "3in1e9BksgIh"
  73 |       },
  74 |       "source": [
  75 |         "## 🟠 Change the font size of the output cells"
  76 |       ]
  77 |     },
  78 |     {
  79 |       "cell_type": "code",
  80 |       "execution_count": null,
  81 |       "metadata": {
  82 |         "colab": {
  83 |           "base_uri": "https://localhost:8080/"
  84 |         },
  85 |         "id": "1_nYkVog8SUR",
  86 |         "outputId": "f02c5706-3dfe-4b6d-f0e5-0ce5920432b2"
  87 |       },
  88 |       "outputs": [
  89 |         {
  90 |           "output_type": "stream",
  91 |           "name": "stdout",
  92 |           "text": [
  93 |             "Salam Howsam!\n"
  94 |           ]
  95 |         }
  96 |       ],
  97 |       "source": [
  98 |         "print('Salam Howsam!')"
  99 |       ]
 100 |     },
 101 |     {
 102 |       "cell_type": "code",
 103 |       "execution_count": null,
 104 |       "metadata": {
 105 |         "id": "BmMM0EfKsSiO"
 106 |       },
 107 |       "outputs": [],
 108 |       "source": [
 109 |         "from IPython.display import HTML\n",
 110 |         "shell = get_ipython()\n",
 111 |         "\n",
 112 |         "def adjust_font_size():\n",
 113 |         "  display(HTML('''<style>\n",
 114 |         "    body {\n",
 115 |         "      font-size: 24px;\n",
 116 |         "    }\n",
 117 |         "  '''))\n",
 118 |         "\n",
 119 |         "if adjust_font_size not in shell.events.callbacks['pre_execute']:\n",
 120 |         "  shell.events.register('pre_execute', adjust_font_size)"
 121 |       ]
 122 |     },
 123 |     {
 124 |       "cell_type": "code",
 125 |       "execution_count": null,
 126 |       "metadata": {
 127 |         "colab": {
 128 |           "base_uri": "https://localhost:8080/"
 129 |         },
 130 |         "id": "10N1yUE88XRW",
 131 |         "outputId": "cf14ac4b-4923-4aee-a1fb-8d08cbb83596"
 132 |       },
 133 |       "outputs": [
 134 |         {
 135 |           "output_type": "display_data",
 136 |           "data": {
 137 |             "text/plain": [
 138 |               "<IPython.core.display.HTML object>"
 139 |             ],
 140 |             "text/html": [
 141 |               "<style>\n",
 142 |               "    body {\n",
 143 |               "      font-size: 24px;\n",
 144 |               "    }\n",
 145 |               "  "
 146 |             ]
 147 |           },
 148 |           "metadata": {}
 149 |         },
 150 |         {
 151 |           "output_type": "stream",
 152 |           "name": "stdout",
 153 |           "text": [
 154 |             "Salam Howsam!\n"
 155 |           ]
 156 |         }
 157 |       ],
 158 |       "source": [
 159 |         "print('Salam Howsam!')"
 160 |       ]
 161 |     },
 162 |     {
 163 |       "cell_type": "markdown",
 164 |       "metadata": {
 165 |         "id": "xv32RDGrJhqv"
 166 |       },
 167 |       "source": [
 168 |         "## 🟠 `pip`"
 169 |       ]
 170 |     },
 171 |     {
 172 |       "cell_type": "code",
 173 |       "source": [
 174 |         "# !pip install -q datasets torchmetrics"
 175 |       ],
 176 |       "metadata": {
 177 |         "colab": {
 178 |           "base_uri": "https://localhost:8080/"
 179 |         },
 180 |         "id": "cnvXOLwGJkHV",
 181 |         "outputId": "1f7de8c6-a8fe-4f2b-d924-dbee884d6bd1"
 182 |       },
 183 |       "execution_count": null,
 184 |       "outputs": [
 185 |         {
 186 |           "output_type": "display_data",
 187 |           "data": {
 188 |             "text/plain": [
 189 |               "<IPython.core.display.HTML object>"
 190 |             ],
 191 |             "text/html": [
 192 |               "<style>\n",
 193 |               "    body {\n",
 194 |               "      font-size: 24px;\n",
 195 |               "    }\n",
 196 |               "  "
 197 |             ]
 198 |           },
 199 |           "metadata": {}
 200 |         }
 201 |       ]
 202 |     },
 203 |     {
 204 |       "cell_type": "markdown",
 205 |       "metadata": {
 206 |         "id": "w_a3OXnSeV0z"
 207 |       },
 208 |       "source": [
 209 |         "# 🔴 **Import**"
 210 |       ]
 211 |     },
 212 |     {
 213 |       "cell_type": "code",
 214 |       "execution_count": null,
 215 |       "metadata": {
 216 |         "id": "vhlVJEkJeTsV",
 217 |         "colab": {
 218 |           "base_uri": "https://localhost:8080/",
 219 |           "height": 17
 220 |         },
 221 |         "outputId": "f432d61e-2a55-4f70-830b-c4b2e41c214b"
 222 |       },
 223 |       "outputs": [
 224 |         {
 225 |           "output_type": "display_data",
 226 |           "data": {
 227 |             "text/plain": [
 228 |               "<IPython.core.display.HTML object>"
 229 |             ],
 230 |             "text/html": [
 231 |               "<style>\n",
 232 |               "    body {\n",
 233 |               "      font-size: 24px;\n",
 234 |               "    }\n",
 235 |               "  "
 236 |             ]
 237 |           },
 238 |           "metadata": {}
 239 |         }
 240 |       ],
 241 |       "source": [
 242 |         "import time\n",
 243 |         "import math\n",
 244 |         "import random\n",
 245 |         "from tqdm import tqdm\n",
 246 |         "from dataclasses import dataclass\n",
 247 |         "\n",
 248 |         "import numpy as np\n",
 249 |         "import matplotlib.pyplot as plt\n",
 250 |         "import seaborn as sns\n",
 251 |         "sns.set()\n",
 252 |         "\n",
 253 |         "from datasets import load_dataset\n",
 254 |         "from tokenizers import Tokenizer\n",
 255 |         "\n",
 256 |         "import torch\n",
 257 |         "from torch.utils.data import Dataset, DataLoader\n",
 258 |         "from torch import nn\n",
 259 |         "from torch.nn import functional as F\n",
 260 |         "\n",
 261 |         "from torchmetrics.aggregation import MeanMetric"
 262 |       ]
 263 |     },
 264 |     {
 265 |       "cell_type": "markdown",
 266 |       "metadata": {
 267 |         "id": "RwaY_YcgRayy"
 268 |       },
 269 |       "source": [
 270 |         "# 🔴 **Utils**"
 271 |       ]
 272 |     },
 273 |     {
 274 |       "cell_type": "code",
 275 |       "source": [
 276 |         "def prepare_data(tokens, seq_len):\n",
 277 |         "    # Trim tokens so that total length is divisible by seq_len\n",
 278 |         "    n_tokens = (tokens.shape[0] // seq_len) * seq_len\n",
 279 |         "    tokens = tokens[:n_tokens]\n",
 280 |         "    # Reshape to 2D tensor\n",
 281 |         "    return tokens.view(-1, seq_len)"
 282 |       ],
 283 |       "metadata": {
 284 |         "colab": {
 285 |           "base_uri": "https://localhost:8080/",
 286 |           "height": 17
 287 |         },
 288 |         "id": "VRLHzZMYKmLL",
 289 |         "outputId": "10c8ac29-87ca-45fa-c4cd-e3141a5600fd"
 290 |       },
 291 |       "execution_count": null,
 292 |       "outputs": [
 293 |         {
 294 |           "output_type": "display_data",
 295 |           "data": {
 296 |             "text/plain": [
 297 |               "<IPython.core.display.HTML object>"
 298 |             ],
 299 |             "text/html": [
 300 |               "<style>\n",
 301 |               "    body {\n",
 302 |               "      font-size: 24px;\n",
 303 |               "    }\n",
 304 |               "  "
 305 |             ]
 306 |           },
 307 |           "metadata": {}
 308 |         }
 309 |       ]
 310 |     },
 311 |     {
 312 |       "cell_type": "code",
 313 |       "execution_count": null,
 314 |       "metadata": {
 315 |         "id": "PpKbTUEIRayz",
 316 |         "colab": {
 317 |           "base_uri": "https://localhost:8080/",
 318 |           "height": 17
 319 |         },
 320 |         "outputId": "66bfa5a3-8dc3-4acb-cc81-059818b342f7"
 321 |       },
 322 |       "outputs": [
 323 |         {
 324 |           "output_type": "display_data",
 325 |           "data": {
 326 |             "text/plain": [
 327 |               "<IPython.core.display.HTML object>"
 328 |             ],
 329 |             "text/html": [
 330 |               "<style>\n",
 331 |               "    body {\n",
 332 |               "      font-size: 24px;\n",
 333 |               "    }\n",
 334 |               "  "
 335 |             ]
 336 |           },
 337 |           "metadata": {}
 338 |         }
 339 |       ],
 340 |       "source": [
 341 |         "def num_trainable_params(model):\n",
 342 |         "  nums = sum(p.numel() for p in model.parameters() if p.requires_grad)/1e6\n",
 343 |         "  return nums"
 344 |       ]
 345 |     },
 346 |     {
 347 |       "cell_type": "code",
 348 |       "source": [
 349 |         "# Benchmarking function\n",
 350 |         "def calculate_time(model, x, num_runs=10):\n",
 351 |         "    torch.cuda.synchronize()\n",
 352 |         "    start = time.time()\n",
 353 |         "    for _ in range(num_runs):\n",
 354 |         "        model(x)\n",
 355 |         "    torch.cuda.synchronize()\n",
 356 |         "    return (time.time() - start) / num_runs"
 357 |       ],
 358 |       "metadata": {
 359 |         "id": "G6LDfZvmOLcI",
 360 |         "colab": {
 361 |           "base_uri": "https://localhost:8080/",
 362 |           "height": 17
 363 |         },
 364 |         "outputId": "8a3ea7ca-29d5-41c7-acf2-bd55f109e0b9"
 365 |       },
 366 |       "execution_count": null,
 367 |       "outputs": [
 368 |         {
 369 |           "output_type": "display_data",
 370 |           "data": {
 371 |             "text/plain": [
 372 |               "<IPython.core.display.HTML object>"
 373 |             ],
 374 |             "text/html": [
 375 |               "<style>\n",
 376 |               "    body {\n",
 377 |               "      font-size: 24px;\n",
 378 |               "    }\n",
 379 |               "  "
 380 |             ]
 381 |           },
 382 |           "metadata": {}
 383 |         }
 384 |       ]
 385 |     },
 386 |     {
 387 |       "cell_type": "markdown",
 388 |       "metadata": {
 389 |         "id": "mf-QHos1nK3D"
 390 |       },
 391 |       "source": [
 392 |         "# 🔴 **Init**"
 393 |       ]
 394 |     },
 395 |     {
 396 |       "cell_type": "code",
 397 |       "execution_count": null,
 398 |       "metadata": {
 399 |         "colab": {
 400 |           "base_uri": "https://localhost:8080/",
 401 |           "height": 48
 402 |         },
 403 |         "id": "2Xb8YR2VdmXp",
 404 |         "outputId": "b01e9187-339e-4fe3-ca8b-e1f1d4836fc0"
 405 |       },
 406 |       "outputs": [
 407 |         {
 408 |           "output_type": "display_data",
 409 |           "data": {
 410 |             "text/plain": [
 411 |               "<IPython.core.display.HTML object>"
 412 |             ],
 413 |             "text/html": [
 414 |               "<style>\n",
 415 |               "    body {\n",
 416 |               "      font-size: 24px;\n",
 417 |               "    }\n",
 418 |               "  "
 419 |             ]
 420 |           },
 421 |           "metadata": {}
 422 |         },
 423 |         {
 424 |           "output_type": "execute_result",
 425 |           "data": {
 426 |             "text/plain": [
 427 |               "'cuda'"
 428 |             ]
 429 |           },
 430 |           "metadata": {},
 431 |           "execution_count": 9
 432 |         }
 433 |       ],
 434 |       "source": [
 435 |         "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
 436 |         "device"
 437 |       ]
 438 |     },
 439 |     {
 440 |       "cell_type": "markdown",
 441 |       "metadata": {
 442 |         "id": "RTql4Ftiunfr"
 443 |       },
 444 |       "source": [
 445 |         "# 🔴 **Dataset**"
 446 |       ]
 447 |     },
 448 |     {
 449 |       "cell_type": "code",
 450 |       "source": [
 451 |         "dataset = load_dataset(\"roneneldan/TinyStories\")\n",
 452 |         "dataset"
 453 |       ],
 454 |       "metadata": {
 455 |         "colab": {
 456 |           "base_uri": "https://localhost:8080/",
 457 |           "height": 325
 458 |         },
 459 |         "id": "am_JqZ3XKuxs",
 460 |         "outputId": "0d7b4a52-c0db-4f46-a68e-d83c8d1d679d"
 461 |       },
 462 |       "execution_count": null,
 463 |       "outputs": [
 464 |         {
 465 |           "output_type": "display_data",
 466 |           "data": {
 467 |             "text/plain": [
 468 |               "<IPython.core.display.HTML object>"
 469 |             ],
 470 |             "text/html": [
 471 |               "<style>\n",
 472 |               "    body {\n",
 473 |               "      font-size: 24px;\n",
 474 |               "    }\n",
 475 |               "  "
 476 |             ]
 477 |           },
 478 |           "metadata": {}
 479 |         },
 480 |         {
 481 |           "output_type": "execute_result",
 482 |           "data": {
 483 |             "text/plain": [
 484 |               "DatasetDict({\n",
 485 |               "    train: Dataset({\n",
 486 |               "        features: ['text'],\n",
 487 |               "        num_rows: 2119719\n",
 488 |               "    })\n",
 489 |               "    validation: Dataset({\n",
 490 |               "        features: ['text'],\n",
 491 |               "        num_rows: 21990\n",
 492 |               "    })\n",
 493 |               "})"
 494 |             ]
 495 |           },
 496 |           "metadata": {},
 497 |           "execution_count": 10
 498 |         }
 499 |       ]
 500 |     },
 501 |     {
 502 |       "cell_type": "code",
 503 |       "source": [
 504 |         "tokenizer = Tokenizer.from_file(\"bpe-tokenizer_tinystories.json\")\n",
 505 |         "tokenizer"
 506 |       ],
 507 |       "metadata": {
 508 |         "colab": {
 509 |           "base_uri": "https://localhost:8080/",
 510 |           "height": 48
 511 |         },
 512 |         "id": "ZAkt3gbSKxoS",
 513 |         "outputId": "06bbb223-e1f2-4bdd-8902-314ecea8c4b7"
 514 |       },
 515 |       "execution_count": null,
 516 |       "outputs": [
 517 |         {
 518 |           "output_type": "display_data",
 519 |           "data": {
 520 |             "text/plain": [
 521 |               "<IPython.core.display.HTML object>"
 522 |             ],
 523 |             "text/html": [
 524 |               "<style>\n",
 525 |               "    body {\n",
 526 |               "      font-size: 24px;\n",
 527 |               "    }\n",
 528 |               "  "
 529 |             ]
 530 |           },
 531 |           "metadata": {}
 532 |         },
 533 |         {
 534 |           "output_type": "execute_result",
 535 |           "data": {
 536 |             "text/plain": [
 537 |               "<tokenizers.Tokenizer at 0x15d11145c30>"
 538 |             ]
 539 |           },
 540 |           "metadata": {},
 541 |           "execution_count": 11
 542 |         }
 543 |       ]
 544 |     },
 545 |     {
 546 |       "cell_type": "code",
 547 |       "source": [
 548 |         "# Load tokens from pytorch file\n",
 549 |         "train_token_ids = torch.load('tokenized-train-samples_vocab-10k.pt')\n",
 550 |         "valid_token_ids = torch.load('tokenized-valid-samples_vocab-10k.pt')\n",
 551 |         "\n",
 552 |         "print(\"📊 Number of Tokens\")\n",
 553 |         "print(f\"🔹 Train: {len(train_token_ids):,} tokens\")\n",
 554 |         "print(f\"🔹 Valid: {len(valid_token_ids):,} tokens\")"
 555 |       ],
 556 |       "metadata": {
 557 |         "colab": {
 558 |           "base_uri": "https://localhost:8080/",
 559 |           "height": 109
 560 |         },
 561 |         "id": "yYC8mmvaKzds",
 562 |         "outputId": "3f8f15ab-1226-4566-90b2-a7f09d9d1095"
 563 |       },
 564 |       "execution_count": null,
 565 |       "outputs": [
 566 |         {
 567 |           "output_type": "display_data",
 568 |           "data": {
 569 |             "text/plain": [
 570 |               "<IPython.core.display.HTML object>"
 571 |             ],
 572 |             "text/html": [
 573 |               "<style>\n",
 574 |               "    body {\n",
 575 |               "      font-size: 24px;\n",
 576 |               "    }\n",
 577 |               "  "
 578 |             ]
 579 |           },
 580 |           "metadata": {}
 581 |         },
 582 |         {
 583 |           "output_type": "stream",
 584 |           "name": "stdout",
 585 |           "text": [
 586 |             "📊 Number of Tokens\n",
 587 |             "🔹 Train: 464,965,814 tokens\n",
 588 |             "🔹 Valid: 4,673,588 tokens\n"
 589 |           ]
 590 |         }
 591 |       ]
 592 |     },
 593 |     {
 594 |       "cell_type": "code",
 595 |       "source": [
 596 |         "class TinyStoriesDataset(Dataset):\n",
 597 |         "\n",
 598 |         "    def __init__(self, data, seq_len):\n",
 599 |         "        self.seq_len = seq_len\n",
 600 |         "        self.data = prepare_data(data, seq_len+1)\n",
 601 |         "\n",
 602 |         "    def __len__(self):\n",
 603 |         "        return self.data.shape[0]\n",
 604 |         "\n",
 605 |         "    def __getitem__(self, idx):\n",
 606 |         "        sample = self.data[idx]\n",
 607 |         "        return sample.long()#[:-1], sample[1:]"
 608 |       ],
 609 |       "metadata": {
 610 |         "colab": {
 611 |           "base_uri": "https://localhost:8080/",
 612 |           "height": 17
 613 |         },
 614 |         "id": "Qoi9WubZK1P2",
 615 |         "outputId": "e401f8c3-4433-4d19-ba70-09a08b8b3563"
 616 |       },
 617 |       "execution_count": null,
 618 |       "outputs": [
 619 |         {
 620 |           "output_type": "display_data",
 621 |           "data": {
 622 |             "text/plain": [
 623 |               "<IPython.core.display.HTML object>"
 624 |             ],
 625 |             "text/html": [
 626 |               "<style>\n",
 627 |               "    body {\n",
 628 |               "      font-size: 24px;\n",
 629 |               "    }\n",
 630 |               "  "
 631 |             ]
 632 |           },
 633 |           "metadata": {}
 634 |         }
 635 |       ]
 636 |     },
 637 |     {
 638 |       "cell_type": "markdown",
 639 |       "source": [
 640 |         "# 🔴 **Model**"
 641 |       ],
 642 |       "metadata": {
 643 |         "id": "ZnBQs-C8PNrY"
 644 |       }
 645 |     },
 646 |     {
 647 |       "cell_type": "markdown",
 648 |       "source": [
 649 |         "## 🟠 Multi Head Attention"
 650 |       ],
 651 |       "metadata": {
 652 |         "id": "lAoYRsvKN_8l"
 653 |       }
 654 |     },
 655 |     {
 656 |       "cell_type": "code",
 657 |       "source": [
 658 |         "class MultiHeadAttention(nn.Module):\n",
 659 |         "\n",
 660 |         "    def __init__(self, config):\n",
 661 |         "        super().__init__()\n",
 662 |         "        self.n_embd = config.n_embd\n",
 663 |         "        self.n_head = config.n_head\n",
 664 |         "        self.head_size = self.n_embd // self.n_head\n",
 665 |         "\n",
 666 |         "        self.qkv_proj = nn.Linear(self.n_embd, 3*self.n_embd, bias=False)\n",
 667 |         "        self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)\n",
 668 |         "        self.c_proj.residual = True\n",
 669 |         "\n",
 670 |         "    def forward(self, x):\n",
 671 |         "        B, T, C = x.shape\n",
 672 |         "        # QKV linear\n",
 673 |         "        q, k, v = self.qkv_proj(x).view(B, T, 3*self.n_head, self.head_size).transpose(1, 2).chunk(3, dim=-3)\n",
 674 |         "        # Scaled Dot Product Attention using pytorch\n",
 675 |         "        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)\n",
 676 |         "        # Reshape and final projection\n",
 677 |         "        y = y.transpose(1, 2).contiguous().view(B, T, C)\n",
 678 |         "        y = self.c_proj(y)\n",
 679 |         "        return y"
 680 |       ],
 681 |       "metadata": {
 682 |         "id": "gnxYy1oIQcY-",
 683 |         "colab": {
 684 |           "base_uri": "https://localhost:8080/"
 685 |         },
 686 |         "outputId": "08943fdb-637b-4f46-9ee7-2ff0aaed5de2"
 687 |       },
 688 |       "execution_count": null,
 689 |       "outputs": [
 690 |         {
 691 |           "output_type": "display_data",
 692 |           "data": {
 693 |             "text/plain": [
 694 |               "<IPython.core.display.HTML object>"
 695 |             ],
 696 |             "text/html": [
 697 |               "<style>\n",
 698 |               "    body {\n",
 699 |               "      font-size: 24px;\n",
 700 |               "    }\n",
 701 |               "  "
 702 |             ]
 703 |           },
 704 |           "metadata": {}
 705 |         }
 706 |       ]
 707 |     },
 708 |     {
 709 |       "cell_type": "markdown",
 710 |       "source": [
 711 |         "## 🟠 Feed Forward (MLP)"
 712 |       ],
 713 |       "metadata": {
 714 |         "id": "WTVewgG-m7nq"
 715 |       }
 716 |     },
 717 |     {
 718 |       "cell_type": "code",
 719 |       "source": [
 720 |         "class FeedForward(nn.Module):\n",
 721 |         "\n",
 722 |         "    def __init__(self, config):\n",
 723 |         "        super().__init__()\n",
 724 |         "        self.n_embd = config.n_embd\n",
 725 |         "        self.f_expnd = config.f_expnd\n",
 726 |         "\n",
 727 |         "        self.up_proj = nn.Linear(self.n_embd, int(self.f_expnd*self.n_embd), bias=False)\n",
 728 |         "        self.down_proj = nn.Linear(int(self.f_expnd*self.n_embd), self.n_embd, bias=False)\n",
 729 |         "        self.down_proj.residual = True\n",
 730 |         "\n",
 731 |         "    def forward(self, x):\n",
 732 |         "        return self.down_proj(F.gelu(self.up_proj(x)))"
 733 |       ],
 734 |       "metadata": {
 735 |         "id": "BTcx4J5Lm66z",
 736 |         "colab": {
 737 |           "base_uri": "https://localhost:8080/"
 738 |         },
 739 |         "outputId": "1913f8f8-64b0-4c63-a56c-08ab06ededb9"
 740 |       },
 741 |       "execution_count": null,
 742 |       "outputs": [
 743 |         {
 744 |           "output_type": "display_data",
 745 |           "data": {
 746 |             "text/plain": [
 747 |               "<IPython.core.display.HTML object>"
 748 |             ],
 749 |             "text/html": [
 750 |               "<style>\n",
 751 |               "    body {\n",
 752 |               "      font-size: 24px;\n",
 753 |               "    }\n",
 754 |               "  "
 755 |             ]
 756 |           },
 757 |           "metadata": {}
 758 |         }
 759 |       ]
 760 |     },
 761 |     {
 762 |       "cell_type": "markdown",
 763 |       "source": [
 764 |         "## 🟠 Decoder Block"
 765 |       ],
 766 |       "metadata": {
 767 |         "id": "ih6sV9ljndzW"
 768 |       }
 769 |     },
 770 |     {
 771 |       "cell_type": "code",
 772 |       "source": [
 773 |         "class DecoderBlock(nn.Module):\n",
 774 |         "\n",
 775 |         "    def __init__(self, config):\n",
 776 |         "        super().__init__()\n",
 777 |         "        self.n_embd = config.n_embd\n",
 778 |         "        # Multi Head Attention\n",
 779 |         "        self.ln1 = nn.LayerNorm(config.n_embd)\n",
 780 |         "        self.mha = MultiHeadAttention(config)\n",
 781 |         "        # Feed Forward Neural Network\n",
 782 |         "        self.ln2 = nn.LayerNorm(config.n_embd)\n",
 783 |         "        self.mlp = FeedForward(config)\n",
 784 |         "\n",
 785 |         "    def forward(self, x):\n",
 786 |         "        x = x + self.mha(self.ln1(x))\n",
 787 |         "        x = x + self.mlp(self.ln2(x))\n",
 788 |         "        return x"
 789 |       ],
 790 |       "metadata": {
 791 |         "id": "Gq1kw31av9DR",
 792 |         "colab": {
 793 |           "base_uri": "https://localhost:8080/"
 794 |         },
 795 |         "outputId": "91d83027-7ac9-4231-8d6c-a8d86e2b3daf"
 796 |       },
 797 |       "execution_count": null,
 798 |       "outputs": [
 799 |         {
 800 |           "output_type": "display_data",
 801 |           "data": {
 802 |             "text/plain": [
 803 |               "<IPython.core.display.HTML object>"
 804 |             ],
 805 |             "text/html": [
 806 |               "<style>\n",
 807 |               "    body {\n",
 808 |               "      font-size: 24px;\n",
 809 |               "    }\n",
 810 |               "  "
 811 |             ]
 812 |           },
 813 |           "metadata": {}
 814 |         }
 815 |       ]
 816 |     },
 817 |     {
 818 |       "cell_type": "markdown",
 819 |       "source": [
 820 |         "## 🟠 GPT"
 821 |       ],
 822 |       "metadata": {
 823 |         "id": "LCzTHf8iitEt"
 824 |       }
 825 |     },
 826 |     {
 827 |       "cell_type": "code",
 828 |       "source": [
 829 |         "class GPT(nn.Module):\n",
 830 |         "\n",
 831 |         "    def __init__(self, config):\n",
 832 |         "        super().__init__()\n",
 833 |         "        self.config = config\n",
 834 |         "        self.wte = nn.Embedding(config.vocab_size, config.n_embd) # Token embedding\n",
 835 |         "        self.wpe = nn.Embedding(config.max_seq_len, config.n_embd) # Position embedding\n",
 836 |         "        self.decoders = nn.ModuleList([DecoderBlock(config) for _ in range(config.n_layer)]) # Decoders\n",
 837 |         "        self.lnf = nn.LayerNorm(config.n_embd)\n",
 838 |         "        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) # Classifier\n",
 839 |         "        self.lm_head.weight = self.wte.weight # Weight tying\n",
 840 |         "\n",
 841 |         "        self.apply(self._init_weights)\n",
 842 |         "\n",
 843 |         "    def _init_weights(self, module):\n",
 844 |         "        std = 0.02\n",
 845 |         "        if isinstance(module, nn.Linear):\n",
 846 |         "            if hasattr(module, 'residual'):\n",
 847 |         "                std *= (2*self.config.n_layer)**-0.5\n",
 848 |         "            nn.init.normal_(module.weight, mean=0.0, std=std)\n",
 849 |         "            if module.bias is not None:\n",
 850 |         "                nn.init.zeros_(module.bias)\n",
 851 |         "        elif isinstance(module, nn.Embedding):\n",
 852 |         "            nn.init.normal_(module.weight, mean=0.0, std=std)\n",
 853 |         "\n",
 854 |         "    def forward(self, idx):\n",
 855 |         "        B, T = idx.shape\n",
 856 |         "        # Token Embedding + Position Embedding\n",
 857 |         "        x = self.wte(idx) + self.wpe(torch.arange(T, device=device))\n",
 858 |         "        # Decoders\n",
 859 |         "        for decoder in self.decoders:\n",
 860 |         "            x = decoder(x)\n",
 861 |         "        # Classifier\n",
 862 |         "        x = self.lnf(x)\n",
 863 |         "        logits = self.lm_head(x)\n",
 864 |         "        return logits"
 865 |       ],
 866 |       "metadata": {
 867 |         "id": "Lmrc034JwvSS",
 868 |         "colab": {
 869 |           "base_uri": "https://localhost:8080/",
 870 |           "height": 17
 871 |         },
 872 |         "outputId": "fa55de8f-c918-4819-afe9-bcfa4a24e793"
 873 |       },
 874 |       "execution_count": null,
 875 |       "outputs": [
 876 |         {
 877 |           "output_type": "display_data",
 878 |           "data": {
 879 |             "text/plain": [
 880 |               "<IPython.core.display.HTML object>"
 881 |             ],
 882 |             "text/html": [
 883 |               "<style>\n",
 884 |               "    body {\n",
 885 |               "      font-size: 24px;\n",
 886 |               "    }\n",
 887 |               "  "
 888 |             ]
 889 |           },
 890 |           "metadata": {}
 891 |         }
 892 |       ]
 893 |     },
 894 |     {
 895 |       "cell_type": "markdown",
 896 |       "source": [
 897 |         "## 🟠 Config"
 898 |       ],
 899 |       "metadata": {
 900 |         "id": "Q1Gx0mlvVJsV"
 901 |       }
 902 |     },
 903 |     {
 904 |       "cell_type": "code",
 905 |       "source": [
 906 |         "@dataclass\n",
 907 |         "class GPTConfig:\n",
 908 |         "    vocab_size: int = 50257 # number of tokens\n",
 909 |         "    max_seq_len: int = 1024 # max sequence length\n",
 910 |         "    n_layer: int = 12 # number of layers\n",
 911 |         "    n_head: int = 12 # number of heads\n",
 912 |         "    n_embd: int = 768 # embedding dimension\n",
 913 |         "    f_expnd: int = 4 # expansion factor in mlp"
 914 |       ],
 915 |       "metadata": {
 916 |         "colab": {
 917 |           "base_uri": "https://localhost:8080/",
 918 |           "height": 17
 919 |         },
 920 |         "id": "PTeKlMqVP3qp",
 921 |         "outputId": "a884f0c3-759e-4b3f-8e57-0862c011ee43"
 922 |       },
 923 |       "execution_count": null,
 924 |       "outputs": [
 925 |         {
 926 |           "output_type": "display_data",
 927 |           "data": {
 928 |             "text/plain": [
 929 |               "<IPython.core.display.HTML object>"
 930 |             ],
 931 |             "text/html": [
 932 |               "<style>\n",
 933 |               "    body {\n",
 934 |               "      font-size: 24px;\n",
 935 |               "    }\n",
 936 |               "  "
 937 |             ]
 938 |           },
 939 |           "metadata": {}
 940 |         }
 941 |       ]
 942 |     },
 943 |     {
 944 |       "cell_type": "markdown",
 945 |       "metadata": {
 946 |         "id": "W0QNbC0YPCKZ"
 947 |       },
 948 |       "source": [
 949 |         "# 🔴 **Functions ⚙️**"
 950 |       ]
 951 |     },
 952 |     {
 953 |       "cell_type": "markdown",
 954 |       "source": [
 955 |         "## 🟠 Train ➰"
 956 |       ],
 957 |       "metadata": {
 958 |         "id": "ACjVH6bBpAEV"
 959 |       }
 960 |     },
 961 |     {
 962 |       "cell_type": "code",
 963 |       "execution_count": null,
 964 |       "metadata": {
 965 |         "id": "WniOAgk0QyRI",
 966 |         "colab": {
 967 |           "base_uri": "https://localhost:8080/",
 968 |           "height": 17
 969 |         },
 970 |         "outputId": "bffc4f9f-b6ee-408d-94d6-e159117d096e"
 971 |       },
 972 |       "outputs": [
 973 |         {
 974 |           "output_type": "display_data",
 975 |           "data": {
 976 |             "text/plain": [
 977 |               "<IPython.core.display.HTML object>"
 978 |             ],
 979 |             "text/html": [
 980 |               "<style>\n",
 981 |               "    body {\n",
 982 |               "      font-size: 24px;\n",
 983 |               "    }\n",
 984 |               "  "
 985 |             ]
 986 |           },
 987 |           "metadata": {}
 988 |         }
 989 |       ],
 990 |       "source": [
 991 |         "def train_one_epoch(model, train_loader, optimizer, epoch=None):\n",
 992 |         "    model.train()\n",
 993 |         "    loss_train = MeanMetric()\n",
 994 |         "\n",
 995 |         "    with tqdm(train_loader, unit='batch') as tepoch:\n",
 996 |         "        for inputs in tepoch:\n",
 997 |         "            if epoch is not None:\n",
 998 |         "                tepoch.set_description(f'Epoch {epoch}')\n",
 999 |         "\n",
1000 |         "            inputs = inputs.to(device)\n",
1001 |         "            logits = model(inputs[:, :-1])\n",
1002 |         "            loss = F.cross_entropy(logits.reshape(-1, logits.shape[-1]), inputs[:, 1:].flatten())\n",
1003 |         "            loss.backward()\n",
1004 |         "\n",
1005 |         "            nn.utils.clip_grad.clip_grad_norm_(model.parameters(), max_norm=1.)\n",
1006 |         "\n",
1007 |         "            optimizer.step()\n",
1008 |         "            optimizer.zero_grad()\n",
1009 |         "\n",
1010 |         "            loss_train.update(loss.item(), inputs.shape[0])\n",
1011 |         "\n",
1012 |         "            tepoch.set_postfix(loss=loss_train.compute().item())\n",
1013 |         "\n",
1014 |         "    return loss_train.compute().item()"
1015 |       ]
1016 |     },
1017 |     {
1018 |       "cell_type": "markdown",
1019 |       "metadata": {
1020 |         "id": "G9HgVWslPGsH"
1021 |       },
1022 |       "source": [
1023 |         "## 🟠 Evaluate"
1024 |       ]
1025 |     },
1026 |     {
1027 |       "cell_type": "code",
1028 |       "execution_count": null,
1029 |       "metadata": {
1030 |         "id": "uV0_67_ZQ0xf",
1031 |         "colab": {
1032 |           "base_uri": "https://localhost:8080/",
1033 |           "height": 17
1034 |         },
1035 |         "outputId": "0463ccac-afd0-4c09-a2f0-eb1a6a080e70"
1036 |       },
1037 |       "outputs": [
1038 |         {
1039 |           "output_type": "display_data",
1040 |           "data": {
1041 |             "text/plain": [
1042 |               "<IPython.core.display.HTML object>"
1043 |             ],
1044 |             "text/html": [
1045 |               "<style>\n",
1046 |               "    body {\n",
1047 |               "      font-size: 24px;\n",
1048 |               "    }\n",
1049 |               "  "
1050 |             ]
1051 |           },
1052 |           "metadata": {}
1053 |         }
1054 |       ],
1055 |       "source": [
1056 |         "def evaluate(model, valid_loader):\n",
1057 |         "    model.eval()\n",
1058 |         "    loss_eval = MeanMetric()\n",
1059 |         "\n",
1060 |         "    with torch.inference_mode():\n",
1061 |         "        for inputs in valid_loader:\n",
1062 |         "            inputs = inputs.to(device)\n",
1063 |         "\n",
1064 |         "            logits = model(inputs[:, :-1])\n",
1065 |         "\n",
1066 |         "            loss = F.cross_entropy(logits.reshape(-1, logits.shape[-1]), inputs[:, 1:].flatten())\n",
1067 |         "            loss_eval.update(loss.item(), inputs.shape[0])\n",
1068 |         "\n",
1069 |         "    return loss_eval.compute().item()"
1070 |       ]
1071 |     },
1072 |     {
1073 |       "cell_type": "markdown",
1074 |       "metadata": {
1075 |         "id": "6kSRpX11o6Dm"
1076 |       },
1077 |       "source": [
1078 |         "## 🟠 Generate"
1079 |       ]
1080 |     },
1081 |     {
1082 |       "cell_type": "code",
1083 |       "source": [
1084 |         "def generate_text(\n",
1085 |         "        model, tokenizer, xgen, max_length, temperature=1.0, top_k=50, num_return_sequences=1, device='cpu', seed=42):\n",
1086 |         "    \"\"\"\n",
1087 |         "    Generates text using top-k sampling.\n",
1088 |         "\n",
1089 |         "    Args:\n",
1090 |         "        model: The language model.\n",
1091 |         "        tokenizer: Tokenizer for decoding generated text.\n",
1092 |         "        xgen: Initial token sequence (Tensor of shape (B, T)).\n",
1093 |         "        max_length: Maximum length of generated text.\n",
1094 |         "        temperature: Sampling temperature.\n",
1095 |         "        top_k: Number of top candidates to sample from.\n",
1096 |         "        num_return_sequences: Number of generated sequences.\n",
1097 |         "        device: Device to run the model on.\n",
1098 |         "        seed: Random seed for reproducibility.\n",
1099 |         "        epoch: Current epoch number (for logging).\n",
1100 |         "    \"\"\"\n",
1101 |         "    # Initialize a random number generator for sampling\n",
1102 |         "    sample_rng = torch.Generator(device=device)\n",
1103 |         "    sample_rng.manual_seed(seed)\n",
1104 |         "\n",
1105 |         "    generated_texts = []\n",
1106 |         "\n",
1107 |         "    with torch.no_grad():  # Move outside the loop to optimize performance\n",
1108 |         "        while xgen.size(1) < max_length:\n",
1109 |         "            logits = model(xgen)  # (B, T, vocab_size)\n",
1110 |         "            logits = logits[:, -1, :]  # Take the logits at the last position (B, vocab_size)\n",
1111 |         "            probs = F.softmax(logits / temperature, dim=-1)  # Get the probabilities\n",
1112 |         "\n",
1113 |         "            # Perform top-k sampling (default k=50 in Hugging Face pipeline)\n",
1114 |         "            topk_probs, topk_indices = torch.topk(probs, top_k, dim=-1)  # (B, top_k)\n",
1115 |         "\n",
1116 |         "            # Select a token from the top-k probabilities\n",
1117 |         "            # Note: multinomial does not require the input to sum to 1\n",
1118 |         "            ix = torch.multinomial(topk_probs, 1, generator=sample_rng)  # (B, 1)\n",
1119 |         "\n",
1120 |         "            # Gather the corresponding indices from the top-k indices\n",
1121 |         "            xcol = torch.gather(topk_indices, -1, ix)  # (B, 1)\n",
1122 |         "\n",
1123 |         "            # Append the selected token to the sequence\n",
1124 |         "            xgen = torch.cat((xgen, xcol), dim=1)\n",
1125 |         "\n",
1126 |         "    # Decode and return generated sequences\n",
1127 |         "    for i in range(num_return_sequences):\n",
1128 |         "        tokens = xgen[i, :max_length].tolist()\n",
1129 |         "        decoded_text = tokenizer.decode(tokens)\n",
1130 |         "        generated_texts.append(decoded_text)\n",
1131 |         "\n",
1132 |         "    return generated_texts  # Return generated sequences as a list"
1133 |       ],
1134 |       "metadata": {
1135 |         "id": "G3lqeoYfAVWe",
1136 |         "colab": {
1137 |           "base_uri": "https://localhost:8080/",
1138 |           "height": 17
1139 |         },
1140 |         "outputId": "7c3e2063-e6e2-477a-e8a8-e5794ff87a63"
1141 |       },
1142 |       "execution_count": null,
1143 |       "outputs": [
1144 |         {
1145 |           "output_type": "display_data",
1146 |           "data": {
1147 |             "text/plain": [
1148 |               "<IPython.core.display.HTML object>"
1149 |             ],
1150 |             "text/html": [
1151 |               "<style>\n",
1152 |               "    body {\n",
1153 |               "      font-size: 24px;\n",
1154 |               "    }\n",
1155 |               "  "
1156 |             ]
1157 |           },
1158 |           "metadata": {}
1159 |         }
1160 |       ]
1161 |     },
1162 |     {
1163 |       "cell_type": "markdown",
1164 |       "metadata": {
1165 |         "id": "o_5f69nwPtY2"
1166 |       },
1167 |       "source": [
1168 |         "# 🔴 **Training Process 〽️**"
1169 |       ]
1170 |     },
1171 |     {
1172 |       "cell_type": "code",
1173 |       "source": [
1174 |         "torch.manual_seed(1337)\n",
1175 |         "\n",
1176 |         "seq_len = 128\n",
1177 |         "train_set = TinyStoriesDataset(train_token_ids, seq_len)\n",
1178 |         "valid_set = TinyStoriesDataset(valid_token_ids, seq_len)\n",
1179 |         "\n",
1180 |         "batch_size = 192\n",
1181 |         "train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, pin_memory=True)#, num_workers=4)\n",
1182 |         "valid_loader = DataLoader(valid_set, batch_size=batch_size, shuffle=False, pin_memory=True)#, num_workers=4)\n",
1183 |         "\n",
1184 |         "print(f\"📊 Number of Batches\")\n",
1185 |         "print(f\"🔹 Train: {len(train_loader):,} batches\")\n",
1186 |         "print(f\"🔹 Valid: {len(valid_loader):,} batches\")"
1187 |       ],
1188 |       "metadata": {
1189 |         "colab": {
1190 |           "base_uri": "https://localhost:8080/",
1191 |           "height": 109
1192 |         },
1193 |         "id": "9FQ8l4rrK2nW",
1194 |         "outputId": "e98e8390-16b5-492d-c4b3-09adf7d05910"
1195 |       },
1196 |       "execution_count": null,
1197 |       "outputs": [
1198 |         {
1199 |           "output_type": "display_data",
1200 |           "data": {
1201 |             "text/plain": [
1202 |               "<IPython.core.display.HTML object>"
1203 |             ],
1204 |             "text/html": [
1205 |               "<style>\n",
1206 |               "    body {\n",
1207 |               "      font-size: 24px;\n",
1208 |               "    }\n",
1209 |               "  "
1210 |             ]
1211 |           },
1212 |           "metadata": {}
1213 |         },
1214 |         {
1215 |           "output_type": "stream",
1216 |           "name": "stdout",
1217 |           "text": [
1218 |             "📊 Number of Batches\n",
1219 |             "🔹 Train: 18,773 batches\n",
1220 |             "🔹 Valid: 189 batches\n"
1221 |           ]
1222 |         }
1223 |       ]
1224 |     },
1225 |     {
1226 |       "cell_type": "code",
1227 |       "source": [
1228 |         "model = GPT(\n",
1229 |         "    GPTConfig(\n",
1230 |         "        max_seq_len=256,\n",
1231 |         "        vocab_size=10_000,\n",
1232 |         "        n_embd=128,\n",
1233 |         "        n_layer=8,\n",
1234 |         "        n_head=16\n",
1235 |         "        )\n",
1236 |         "    ).to(device)\n",
1237 |         "\n",
1238 |         "print(model)\n",
1239 |         "num_trainable_params(model)"
1240 |       ],
1241 |       "metadata": {
1242 |         "colab": {
1243 |           "base_uri": "https://localhost:8080/",
1244 |           "height": 663
1245 |         },
1246 |         "id": "i7zJv8yJEBLl",
1247 |         "outputId": "2a180137-9279-4087-dec3-f0c2934ffec7",
1248 |         "collapsed": true
1249 |       },
1250 |       "execution_count": null,
1251 |       "outputs": [
1252 |         {
1253 |           "output_type": "display_data",
1254 |           "data": {
1255 |             "text/plain": [
1256 |               "<IPython.core.display.HTML object>"
1257 |             ],
1258 |             "text/html": [
1259 |               "<style>\n",
1260 |               "    body {\n",
1261 |               "      font-size: 24px;\n",
1262 |               "    }\n",
1263 |               "  "
1264 |             ]
1265 |           },
1266 |           "metadata": {}
1267 |         },
1268 |         {
1269 |           "output_type": "stream",
1270 |           "name": "stdout",
1271 |           "text": [
1272 |             "GPT(\n",
1273 |             "  (wte): Embedding(10000, 128)\n",
1274 |             "  (wpe): Embedding(256, 128)\n",
1275 |             "  (decoders): ModuleList(\n",
1276 |             "    (0-7): 8 x DecoderBlock(\n",
1277 |             "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
1278 |             "      (mha): MultiHeadAttention(\n",
1279 |             "        (qkv_proj): Linear(in_features=128, out_features=384, bias=False)\n",
1280 |             "        (c_proj): Linear(in_features=128, out_features=128, bias=False)\n",
1281 |             "      )\n",
1282 |             "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
1283 |             "      (mlp): FeedForward(\n",
1284 |             "        (up_proj): Linear(in_features=128, out_features=512, bias=False)\n",
1285 |             "        (down_proj): Linear(in_features=512, out_features=128, bias=False)\n",
1286 |             "      )\n",
1287 |             "    )\n",
1288 |             "  )\n",
1289 |             "  (lnf): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
1290 |             "  (lm_head): Linear(in_features=128, out_features=10000, bias=False)\n",
1291 |             ")\n"
1292 |           ]
1293 |         },
1294 |         {
1295 |           "output_type": "execute_result",
1296 |           "data": {
1297 |             "text/plain": [
1298 |               "2.889984"
1299 |             ]
1300 |           },
1301 |           "metadata": {},
1302 |           "execution_count": 23
1303 |         }
1304 |       ]
1305 |     },
1306 |     {
1307 |       "cell_type": "code",
1308 |       "execution_count": null,
1309 |       "metadata": {
1310 |         "id": "ketNqGoLsK2U",
1311 |         "colab": {
1312 |           "base_uri": "https://localhost:8080/",
1313 |           "height": 17
1314 |         },
1315 |         "outputId": "a10a59cd-01ed-47da-e661-1d335aecfe73",
1316 |         "collapsed": true
1317 |       },
1318 |       "outputs": [
1319 |         {
1320 |           "output_type": "display_data",
1321 |           "data": {
1322 |             "text/plain": [
1323 |               "<IPython.core.display.HTML object>"
1324 |             ],
1325 |             "text/html": [
1326 |               "<style>\n",
1327 |               "    body {\n",
1328 |               "      font-size: 24px;\n",
1329 |               "    }\n",
1330 |               "  "
1331 |             ]
1332 |           },
1333 |           "metadata": {}
1334 |         }
1335 |       ],
1336 |       "source": [
1337 |         "learning_rate = 6e-4\n",
1338 |         "weight_decay = 0.1\n",
1339 |         "use_fused = True\n",
1340 |         "\n",
1341 |         "optimizer = torch.optim.AdamW(\n",
1342 |         "    model.parameters(),\n",
1343 |         "    lr=learning_rate,\n",
1344 |         "    betas=(0.9, 0.95),\n",
1345 |         "    weight_decay=weight_decay,\n",
1346 |         "    fused=use_fused\n",
1347 |         "    )"
1348 |       ]
1349 |     },
1350 |     {
1351 |       "cell_type": "code",
1352 |       "execution_count": null,
1353 |       "metadata": {
1354 |         "id": "zqxSVVB7JWpW",
1355 |         "colab": {
1356 |           "base_uri": "https://localhost:8080/",
1357 |           "height": 17
1358 |         },
1359 |         "outputId": "54f2199e-5806-47d9-941e-bad20f38537c"
1360 |       },
1361 |       "outputs": [
1362 |         {
1363 |           "output_type": "display_data",
1364 |           "data": {
1365 |             "text/plain": [
1366 |               "<IPython.core.display.HTML object>"
1367 |             ],
1368 |             "text/html": [
1369 |               "<style>\n",
1370 |               "    body {\n",
1371 |               "      font-size: 24px;\n",
1372 |               "    }\n",
1373 |               "  "
1374 |             ]
1375 |           },
1376 |           "metadata": {}
1377 |         }
1378 |       ],
1379 |       "source": [
1380 |         "loss_train_hist = []\n",
1381 |         "loss_valid_hist = []\n",
1382 |         "\n",
1383 |         "metric_train_hist = []\n",
1384 |         "metric_valid_hist = []\n",
1385 |         "\n",
1386 |         "best_loss_valid = torch.inf\n",
1387 |         "epoch_counter = 0"
1388 |       ]
1389 |     },
1390 |     {
1391 |       "cell_type": "code",
1392 |       "source": [
1393 |         "prompts = [\n",
1394 |         "    'In last night',\n",
1395 |         "    'Once upon',\n",
1396 |         "    'Once upon a time',\n",
1397 |         "    'One day, a little boy named TimTommy was a smart 3 year old, much smarter']\n",
1398 |         "\n",
1399 |         "num_return_sequences = 3\n",
1400 |         "sent_test = [\n",
1401 |         "    torch.tensor(tokens.ids, dtype=torch.long).unsqueeze(0).repeat(num_return_sequences, 1).to(device)\n",
1402 |         "    for tokens in tokenizer.encode_batch(prompts)]\n",
1403 |         "sent_test\n",
1404 |         "\n",
1405 |         "temperature = 0.9\n",
1406 |         "max_length = 32\n",
1407 |         "top_k = 50"
1408 |       ],
1409 |       "metadata": {
1410 |         "id": "yDapu_YBmOTv",
1411 |         "colab": {
1412 |           "base_uri": "https://localhost:8080/",
1413 |           "height": 17
1414 |         },
1415 |         "outputId": "6a789195-b835-4ac7-ed85-54ad1b1a3fcc"
1416 |       },
1417 |       "execution_count": null,
1418 |       "outputs": [
1419 |         {
1420 |           "output_type": "display_data",
1421 |           "data": {
1422 |             "text/plain": [
1423 |               "<IPython.core.display.HTML object>"
1424 |             ],
1425 |             "text/html": [
1426 |               "<style>\n",
1427 |               "    body {\n",
1428 |               "      font-size: 24px;\n",
1429 |               "    }\n",
1430 |               "  "
1431 |             ]
1432 |           },
1433 |           "metadata": {}
1434 |         }
1435 |       ]
1436 |     },
1437 |     {
1438 |       "cell_type": "code",
1439 |       "execution_count": null,
1440 |       "metadata": {
1441 |         "colab": {
1442 |           "base_uri": "https://localhost:8080/",
1443 |           "height": 1000
1444 |         },
1445 |         "id": "eVqS9SEPJWpW",
1446 |         "outputId": "c4fde068-0e55-45e0-8957-b263fe9ab97d"
1447 |       },
1448 |       "outputs": [
1449 |         {
1450 |           "output_type": "display_data",
1451 |           "data": {
1452 |             "text/plain": [
1453 |               "<IPython.core.display.HTML object>"
1454 |             ],
1455 |             "text/html": [
1456 |               "<style>\n",
1457 |               "    body {\n",
1458 |               "      font-size: 24px;\n",
1459 |               "    }\n",
1460 |               "  "
1461 |             ]
1462 |           },
1463 |           "metadata": {}
1464 |         },
1465 |         {
1466 |           "output_type": "stream",
1467 |           "name": "stderr",
1468 |           "text": [
1469 |             "Epoch 0:   0%|                                                                            | 0/18773 [00:00<?, ?batch/s]C:\\Users\\Howsam\\AppData\\Local\\Temp\\ipykernel_12048\\2308121339.py:18: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\\cb\\pytorch_1000000000000\\work\\aten\\src\\ATen\\native\\transformers\\cuda\\sdp_utils.cpp:455.)\n",
1470 |             "  y = F.scaled_dot_product_attention(q, k, v, is_causal=True)\n",
1471 |             "Epoch 0: 100%|█████████████████████████████████████████████████████| 18773/18773 [50:12<00:00,  6.23batch/s, loss=2.17]\n"
1472 |           ]
1473 |         },
1474 |         {
1475 |           "output_type": "stream",
1476 |           "name": "stdout",
1477 |           "text": [
1478 |             "Valid: Loss = 1.98\n",
1479 |             "Model Saved!\n",
1480 |             "......................................................................................................................................................\n",
1481 |             "Epoch 0 Sample 0: In last night, the birds were so surprised that the big cloud was now gone. All its friends looked around the big, scary cloud in the clouds.\n",
1482 |             "Epoch 0 Sample 1: In last night, they were dreaming of their friends. They liked to hear them laugh and laugh. One night, they heard a loud noise. It sounded\n",
1483 |             "Epoch 0 Sample 2: In last night, the princess and her princess were standing in the sun. But when the sun went down, the princess was still asleep.\n",
1484 |             "\n",
1485 |             "The\n",
1486 |             "......................................................................................................................................................\n",
1487 |             "Epoch 0 Sample 0: Once upon a time, there were two friends, Lily and Sam. They loved to play together in the snow. One day, they found a big,\n",
1488 |             "Epoch 0 Sample 1: Once upon a time, there was a girl named Lily. She loved to play outside in the park. One day, she saw a big dog and asked\n",
1489 |             "Epoch 0 Sample 2: Once upon a time, there was a little girl named Lily. She loved to play outside in the park. One day, Lily saw a big, scary\n",
1490 |             "......................................................................................................................................................\n",
1491 |             "Epoch 0 Sample 0: Once upon a time, there was a big, fluffy lion. He lived in a thick jungle in the jungle. One day, he found a big hole\n",
1492 |             "Epoch 0 Sample 1: Once upon a time there was a little girl named Lily. She had a fancy toy bear that she loved to play with. One day, Lily's friend\n",
1493 |             "Epoch 0 Sample 2: Once upon a time there was a boy named Paul. He was the happiest boy had ever been. One day he was playing in his garden when he saw\n",
1494 |             "......................................................................................................................................................\n",
1495 |             "Epoch 0 Sample 0: One day, a little boy named TimTommy was a smart 3 year old, much smarter than the most. Every day, he would use his huge hands\n",
1496 |             "Epoch 0 Sample 1: One day, a little boy named TimTommy was a smart 3 year old, much smarter. He was always eager to try new things and exciting. \n",
1497 |             "\n",
1498 |             "Epoch 0 Sample 2: One day, a little boy named TimTommy was a smart 3 year old, much smarter. He worked hard to be obedient, but every day he made\n",
1499 |             "......................................................................................................................................................\n"
1500 |           ]
1501 |         },
1502 |         {
1503 |           "output_type": "stream",
1504 |           "name": "stderr",
1505 |           "text": [
1506 |             "Epoch 1: 100%|███████████████████████████████████████████████████| 18773/18773 [1:15:36<00:00,  4.14batch/s, loss=1.95]\n"
1507 |           ]
1508 |         },
1509 |         {
1510 |           "output_type": "stream",
1511 |           "name": "stdout",
1512 |           "text": [
1513 |             "Valid: Loss = 1.942\n",
1514 |             "Model Saved!\n",
1515 |             "......................................................................................................................................................\n"
1516 |           ]
1517 |         },
1518 |         {
1519 |           "output_type": "stream",
1520 |           "name": "stderr",
1521 |           "text": [
1522 |             "Epoch 2: 100%|█████████████████████████████████████████████████████| 18773/18773 [49:51<00:00,  6.28batch/s, loss=1.93]\n"
1523 |           ]
1524 |         },
1525 |         {
1526 |           "output_type": "stream",
1527 |           "name": "stdout",
1528 |           "text": [
1529 |             "Valid: Loss = 1.925\n",
1530 |             "Model Saved!\n",
1531 |             "......................................................................................................................................................\n",
1532 |             "Epoch 2 Sample 0: In last night, a little girl was going to take a nap. She lay down in her bed, closed her eyes, and snuggled up in her bed\n",
1533 |             "Epoch 2 Sample 1: In last night, something special happened. The sky lit up in amazing happened and a few people started their day to fall asleep. Suddenly, a loud crash\n",
1534 |             "Epoch 2 Sample 2: In last night, a chubby boy named Tim went to bed up in his beds with a big smile on his face. He was happy and ready to sleep\n",
1535 |             "......................................................................................................................................................\n",
1536 |             "Epoch 2 Sample 0: Once upon a time, there were two friends, Fred and Bob. Fred had a new toy. It was a bright red balloon, and Bob wanted to\n",
1537 |             "Epoch 2 Sample 1: Once upon a time, there was a girl named Lily. She loved to play in her garden. One day, she found a big, shiny coin.\n",
1538 |             "Epoch 2 Sample 2: Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toy cars and trucks. One day, Timmy's car broke\n",
1539 |             "......................................................................................................................................................\n",
1540 |             "Epoch 2 Sample 0: Once upon a time, there was a big house with a hairy dog. The dog liked to sit on the table with the toys. One day, a\n",
1541 |             "Epoch 2 Sample 1: Once upon a time there was a little girl named Daisy. Daisy liked to wander around in the sunshine. Every day, Daisy would wander around and explore.\n",
1542 |             "Epoch 2 Sample 2: Once upon a time there was a big white pig. He was small and strong with many ears. One day he was walking in the woods and he saw\n",
1543 |             "......................................................................................................................................................\n",
1544 |             "Epoch 2 Sample 0: One day, a little boy named TimTommy was a smart 3 year old, much smarter than his own. Every day, he found a secret hidden in\n",
1545 |             "Epoch 2 Sample 1: One day, a little boy named TimTommy was a smart 3 year old, much smarter. He loved to play with his toys and his special rag,\n",
1546 |             "Epoch 2 Sample 2: One day, a little boy named TimTommy was a smart 3 year old, much smarter. He especially loved to climb trees, but the other kids loved\n",
1547 |             "......................................................................................................................................................\n"
1548 |           ]
1549 |         }
1550 |       ],
1551 |       "source": [
1552 |         "torch.manual_seed(1337)\n",
1553 |         "num_epochs = 2\n",
1554 |         "\n",
1555 |         "for epoch in range(0, num_epochs+1):\n",
1556 |         "    # Train\n",
1557 |         "    loss_train = train_one_epoch(model, train_loader, optimizer, epoch)\n",
1558 |         "\n",
1559 |         "    # Validation\n",
1560 |         "    loss_valid = evaluate(model, valid_loader)\n",
1561 |         "    print(f'Valid: Loss = {loss_valid:.4}')\n",
1562 |         "\n",
1563 |         "    loss_train_hist.append(loss_train)\n",
1564 |         "    loss_valid_hist.append(loss_valid)\n",
1565 |         "\n",
1566 |         "    if loss_valid < best_loss_valid:\n",
1567 |         "        torch.save(model, f'best-model.pt')\n",
1568 |         "        best_loss_valid = loss_valid\n",
1569 |         "        print(f'Model Saved!')\n",
1570 |         "    print(150*'.')\n",
1571 |         "\n",
1572 |         "    # Generate\n",
1573 |         "    if epoch % 3 == 0 or epoch == num_epochs:\n",
1574 |         "        model.eval()\n",
1575 |         "        for xgen in sent_test:\n",
1576 |         "            xgen = generate_text(model, tokenizer, xgen, max_length, temperature, 50, 3, device)\n",
1577 |         "            for i, text in enumerate(xgen):\n",
1578 |         "                print(f\"Epoch {epoch} Sample {i}: {text}\")\n",
1579 |         "            print(150*'.')\n",
1580 |         "\n",
1581 |         "    epoch_counter += 1"
1582 |       ]
1583 |     },
1584 |     {
1585 |       "cell_type": "code",
1586 |       "source": [
1587 |         "torch.cuda.empty_cache()"
1588 |       ],
1589 |       "metadata": {
1590 |         "id": "KvIcb5SQEzuu",
1591 |         "colab": {
1592 |           "base_uri": "https://localhost:8080/",
1593 |           "height": 17
1594 |         },
1595 |         "outputId": "38381402-ec55-4565-a28a-f1af141568b7"
1596 |       },
1597 |       "execution_count": null,
1598 |       "outputs": [
1599 |         {
1600 |           "output_type": "display_data",
1601 |           "data": {
1602 |             "text/plain": [
1603 |               "<IPython.core.display.HTML object>"
1604 |             ],
1605 |             "text/html": [
1606 |               "<style>\n",
1607 |               "    body {\n",
1608 |               "      font-size: 24px;\n",
1609 |               "    }\n",
1610 |               "  "
1611 |             ]
1612 |           },
1613 |           "metadata": {}
1614 |         }
1615 |       ]
1616 |     },
1617 |     {
1618 |       "cell_type": "markdown",
1619 |       "metadata": {
1620 |         "id": "CgpgXUYbuwHn"
1621 |       },
1622 |       "source": [
1623 |         "## 🟠 Learning Curves"
1624 |       ]
1625 |     },
1626 |     {
1627 |       "cell_type": "code",
1628 |       "execution_count": null,
1629 |       "metadata": {
1630 |         "id": "cc4iNXKQuwHn",
1631 |         "colab": {
1632 |           "base_uri": "https://localhost:8080/",
1633 |           "height": 546
1634 |         },
1635 |         "outputId": "25f71463-a274-4628-8a4a-4786a7f338ef"
1636 |       },
1637 |       "outputs": [
1638 |         {
1639 |           "output_type": "display_data",
1640 |           "data": {
1641 |             "text/plain": [
1642 |               "<IPython.core.display.HTML object>"
1643 |             ],
1644 |             "text/html": [
1645 |               "<style>\n",
1646 |               "    body {\n",
1647 |               "      font-size: 24px;\n",
1648 |               "    }\n",
1649 |               "  "
1650 |             ]
1651 |           },
1652 |           "metadata": {}
1653 |         },
1654 |         {
1655 |           "output_type": "display_data",
1656 |           "data": {
1657 |             "text/plain": [
1658 |               "<Figure size 800x600 with 1 Axes>"
1659 |             ],
1660 |             "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAIRCAYAAABDD9VwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB7hElEQVR4nO3dd5hU5f3+8feZunW2V5beVBAUwRJRI4SYGE3E2FDU2CgiMVFjvpGfYomaosZE6dgNxkRjibHEktgbiIqK9Lq99+nn98cuAyu7CMvuntmd+3VdXCxnzpz5zIfh7M3Z5zyPYZqmiYiIiIhIDLBZXYCIiIiISE9R+BURERGRmKHwKyIiIiIxQ+FXRERERGKGwq+IiIiIxAyFXxERERGJGQq/IiIiIhIzFH5FREREJGY4rC6gtzBNk3C459YDsdmMHn293kJ96Zh60z71pWPqTfvUl/apLx1Tb9rX032x2QwMw/jW/RR+91M4bFJV1dgjr+Vw2EhLS6SurolgMNwjr9kbqC8dU2/ap750TL1pn/rSPvWlY+pN+6zoS3p6Inb7t4dfDXsQERERkZih8CsiIiIiMUPhV0RERERihsKviIiIiMQMhV8RERERiRma7UFERET6lHA4TCgU7KHXMvB67fj9PkIhTXe2S1f3xW53YLN1zTVbhV8RERHpE0zTpK6uiubmhh593YoKG+Gwpjn7pq7uS3x8Eh5P+n7N5bsvCr8iIiLSJ+wKvklJabhc7oMOSfvLbjd01bcdXdUX0zTx+300NFQDkJKScVDHU/gVERGRXi8cDkWCb1KSp0df2+GwaYGLdnRlX1wuNwANDdUkJ6cd1BAI3fAmIiIivV4oFAJ2hyTpe3b93R7seG6FXxEREekzemqog/S8rvq7VfgVERERkZih8CsiIiIiMUM3vImIiIhEidtvv5mXXnphn/u8887KAz7uVVfNIC8vn3nzbu5kZX2Hwq+IiIhIlLj66uuYNeuqyJ9/8pMf8POfX8vkyVMO6rh33PFHbDb7wZbXJyj8ioiIiESJpKQkkpKS9tqWkZF5UMf1eFIO6vl9icKviIiI9FmmaWL6/d36GuGQjXAH89kaLleXzkDx4ov/4sEHl3LCCd/lpZf+xZgxR/KHP/yJd955i7/+9WE2btxAKBRi6NBhzJgxhwkTjgHaDnvYdYzLLpvJww8vp6yslKFDh/OLX1zH6NFjuqzWaKXwKyIiIn2SaZrs+N3teDdttKyGuGHD6f/rG7o0AJeUFFNeXsYDDzyOz+fj66/XcsMN1zF79s+58cbbaGxsZNmyhdx6643885//xul07nWMiopynn32aW688TacTid33XUnv/3tfJ544p99fro4zfYQZUINDez4y5+pePd9q0sRERHp/fpokPvZzy6nX78ChgwZit1u4+qrr2PatOnk5/dj+PARnH32eVRXV1FVVdnu84PBINdd93+MHn04I0cewkUXXcLOnTuorGx//75EV36jjK+okPpPVrH+888YdMP/wzlgkNUliYiI9EqGYdD/1zd0+7CHfS3j29XDHnbp379/5Ovhw0eSnJzCX//6CNu3b2PHju1s2LAOgHC44+WFBw4cHPk6MbFlnHEwGOjyWqONwm+UiR8+guSjjqJ+1Sp2LryfATfegj0x0eqyREREeiXDMDDc3bvksc1hw2bvOGR2B7c7LvL1p59+wjXXXMWxxx7P2LFH8L3vfR+v18tvfnPdPo/hcrn22maaZpfXGm007CHKGIZB/qWXE5ebQ6CigpKHlsfEB1FEREQ654knHuPII8dzxx1/5NxzL2DChGMpLS0BYiPMHiiF3yhkT0xk5PXXYTgcNH66mur/vGx1SSIiIhKlsrNz2bRpA5999inFxUX8+9/Ps3z5YgACgb4/jOFAKfxGqaShQ8g5fzoAFU//g+YNGyyuSERERKLR5ZfPZNSo0fz617/gkkvO51//epbf/OYm3G43X331hdXlRR3D1PXw/RIKhamqauyR13I4bKSlJVJV1cDORYuo/+gDHGlpDLjpFhzJnh6pIRrt6kt1dWOHNxbEKvWmfepLx9Sb9qkv7esNfQkE/FRWFpORkYfTufdY1u60rxveYllX9+Xb/o7T0xOx27/9uq6u/EYxwzDIuehinLm5BKurKVm+FHMfd22KiIiIyL4p/EY5W1w8+bPmYLhcNH35BVUvvmB1SSIiIiK9lsJvL+Au6E/2BRcCUPncMzR9vdbiikRERER6J4XfXiLl+BPwHH8CmCbFSxcRrK2xuiQRERGRXkfhtxfJPn86rn4FhOrqKF62RON/RURERA6Qwm8vYnO7yZ91JYbbTfPXa6l8/lmrSxIRERHpVRR+exlXXj45F/0MgKp//4vGL9ZYW5CIiIhIL6Lw2wt5jjmOlJNOBtOkZPlSAlVVVpckIiIi0iso/PZSWedNwz1gIKGGeoqXLsIMBq0uSURERCTqKfz2Ujani7xZc7DFx+PduIGKZ562uiQRERE5SHPnzuRnPzu/w8f/+Mc7OOus09nXAr0vvvgvJk4cH/nzWWedzgMPLOlw/wceWMJZZ52+3zWapslLL71AdXVVu68X7RR+ezFXdjY5P7sMgOpXXqLh09UWVyQiIiIH47TTfsLGjevZvHnTXo/5/X7eeOM1fvSjH2MYxn4fc9myR5k27cIuq/HTTz/h9ttvxuv1AjB58hSee+7lLjt+d1P47eWSjxpP6ve+D0DJg8sIVJRbXJGIiIh01ne/O4mkpCRefXXvMPnOO2/R2NjAqafu/1VagLS0NBISErqqxL2uOrvdcWRkZHbZ8bubw+oC5OBlnXUO3s0b8W7eTNHihfT/9Q3YnE6ryxIREbGcaZr4A907L34obBIMtv8aLqftgK7Sut1xfO97P+DVV19mxowr2zz35Zf/zYQJx2IYBrfeeiMff/whdXW1ZGRkcsopp3LFFbOx2fa+rnnWWafzwx+exmWXzQTguef+yYoVj1JeXs4xxxxLTk5em/03b97EsmUL+eyzT2lqaiQnJ5ef/vRczjlnGp98spKf/3wWAGef/WNuuGE+AHfccQvvvLMSgLq6WpYtW8y7775FTU0NI0cewqxZVzF27JFAyzCL1atXcdxxx/PUU09SW1vD6NFjuO66/2PAgEH73avOUvjtAwyHg7yZc9h26034tm6h4h9Pkn3+dKvLEhERsZRpmtz5+CdsLKy1rIZhBSn85oJxBxSATzvtJzz77FN8/vlnjB17BADV1VV8+OF73HLLHVx//S9JS0vjnnvuIyEhkffff4d7772Lww4bxQknfHefx37ttVe4557f8/OfX8uECcfw1lv/ZenShWRn5wDg9Xr55S+v5KijjmbhwuU4HA7+/e/n+ctf7ubII8dx+OFjuf32PzBv3vUsW/YIQ4YM5fXXX40cPxQK8ctfXkUg4Gf+/FtJSUnn6af/zi9+cSWLFj3AIYccBsCXX64hPj6eP/zhXpqbm/jtb+dz992/589/XnRgDe4EDXvoI5wZGeRedgUANW+8Rv3KjyyuSEREJArsf+aMGocccijDho3g1Vdfimx79dWXSU72cPTRx3LKKafy61//P4YPH0m/fgWcddZ5ZGZmsWnTxm899j/+8Te+973v89OfnsOAAQOZPv1nHH/8CZHHm5ubOfvsaVx77a8ZNGgwBQX9ufTSGQBs2rQRp9NJcrIHgNTUNNzuuDbH/+ijD1i3bi0333w7Rx01gUGDBnPNNdczZMgwVqx4LLJfMBjkxhtvZfjwEYwZcwRnnXUen3/+6cG0bb/pym8fkjTmCNJ++COqX/o3pQ8/iLv/AFw5uVaXJSIiYgnDMPjNBeO6fdiDw2HrsmEPu5x22o958MFl/OIXv8LhcPDSSy/wgx/8iISERH7603P43/9e5+9/X8HOnTvYuHEDFRXlhEKhbz3u5s0b+d73TmmzbfToMWzYsB5oGR985pln89pr/2HjxvXs3Lkj8lg4/O193Lx5I0lJSQwZMiyyzTAMxo49gg8/fD+yLT09HY8nJfLnpKQkAoHAtx6/K+jKbx+TecaZxA8fQdjrpXjxAsJ+v9UliYiIWMYwDNwuu2W/OhN8Ab7//R/i9TbzwQfvsnnzRjZsWM/pp5+B1+tl9uzLeOSRB0hMTOKUU37EggXLIsMW9qMje92w5nDsvhZaVVXJxRdP4/nnnyE9PYOf/OSnPPDAY988SIdajr33ew6Hw21ex+l07fcxu5qu/PYxht1O3szZbLvlJnw7dlD+t7+Sc9ElVpclIiIiB8DjSeGEE77Lf//7GllZORx++FgGDhzEm2++wbp1a3n++VdIT88AWm4wq6qq3K/jDh8+gs8//5RzzpkW2bZ27VeRr//zn5eora3liSf+GQmru4ZT7ArN+wr0Q4cOo6Ghns2bNzJixIjI9s8//5RBgwbv57vvXrry2wc5UtPIvWIWGAa1b71J3fvvWl2SiIiIHKDTTvsJ7777Dv/972ucdtpPAMjKygbglVdeoqSkmM8++5T/+79rCQaD+Pfjp73Tp/+Mt976LytWPMqOHdt56qm/8eabb0Qez87Oxett5o03XqWkpISPPvqA+fNvACAQaDl+fHzLtGkbNqynqampzfEnTDiWoUOHc8st/49Vq1aydesW7r7792zatJGzz+548Y6epCu/fVTiYaPIOP0nVD7/LKWPPYJ74CDc+f2sLktERET20/jxR5OcnEx1dTWTJk0B4LDDRjN37i958skVLFu2iKysLCZP/j7Z2Tl89dUX33rM73xnIvPn/5YHH1zK8uWLGTXqcM47b3pkXuGTT57MunUXcv/999LY2EBeXj6nnfYT3nnnLb766kvOOOMshg4dxnHHHc/8+b9hxow5pKTsHrvrcDi4994F3H//vfzmN9fh9/sZOfJQ/vznRYwefXj3NOoAGea+1seTiFAoTFVVY4+8lsNhIy0tkerqxg4H0O8PMxym8E9307T2S1x5+Qz4f/Oxud1dWGnP6qq+9EXqTfvUl46pN+1TX9rXG/oSCPiprCwmIyOvx8eT7uuGt1jW1X35tr/j9PRE7PZvH9SgYQ99mGGzkXvFTOwpqfiLiyh9/JF9rgUuIiIi0tcp/PZxDo+HvJmzwWaj/v33qHv7LatLEhEREbGMwm8MSBgxksypPwWgbMVjeLdvs7giEREREWso/MaItFN+SOKYsZjBIMWLFxJqbra6JBEREZEep/AbIwybjdxLr8CRnkGgrJTSRx7U+F8RERGJOQq/McSelNQy/tdup2Hlx9T+93WrSxIREelSurDTd3XV363Cb4yJHzqMrLPOAaDsySfwbtlscUUiIiIHz263A+D3+yyuRLrLrr9bu/3glqmwfJGLmpoa7rnnHv73v//R0NDAyJEjufbaaxk/fvw+nxcOh7niiis44ogjmDt3bpvHJk2aRGFhYZttp59+OnfddVeX198bpX7v+zSvX0/D6lUULVnIwBtvwZ6YaHVZIiIinWaz2YmPT6KhoRoAl8u9z2V4u1I4bBAK6YrzN3VVX0zTxO/30dBQTXx8EjbbwV27tTz8XnPNNVRWVnLPPfeQnp7OihUruOyyy/jnP//J0KFD232O1+tl3rx5vPPOOxxxxBFtHmtoaKCoqIglS5YwatSoyPa4uLjufBu9imEY5FxyKb6d2wmUl1Py0HLy5/y8x04SIiIi3cHjSQeIBOCeYrPZCIe1yMU3dXVf4uOTIn/HB8PS8Ltt2zbeffddnnjiCcaNGwfAvHnzeOutt3jhhRe4+uqr93rOJ598wrx58wgEAng8nr0eX79+PaZpMm7cuHYflxb2hETyZs1hx52/pfHT1VT/52XST/mh1WWJiIh0mmEYpKRkkJycRigU7JHXtNsNUlISqK1t0tXfPXR1X+x2x0Ff8d3F0vCblpbG0qVLGT16dGSbYRiYpkltbW27z3n77beZMmUKM2bM4Mc//vFej69bt46srCwF3/0QN3AQWeedT9njj1Lx9D+IHzKM+OHDrS5LRETkoNhsNmy2nlni2OGwERcXR3NzSEsc7yGa+2Jp+PV4PJx00klttr300kts376diRMntvuc9q4G72n9+vUkJCQwd+5cVq9eTXp6OmeeeSYXXXTRwY8RcfTM/YG71qXen/WpD1bG5Ml4N66n7oMPKF66kCG33IYjSv/j0JN96W3Um/apLx1Tb9qnvrRPfemYetO+aO6L5WN+97Rq1SpuuOEGJk+ezKRJkzp1jA0bNlBfX8+pp57KVVddxcqVK7nrrruora391uC8LzabQVpaz94U5vHE98zrXH0Vn+/YTnNhEWUPLeewm+ZhdNGPFrpDT/WlN1Jv2qe+dEy9aZ/60j71pWPqTfuisS9RE35fe+01rrvuOsaOHcs999zT6eM89NBD+Hw+kpKSABg5ciSNjY0sWrSIuXPndvrqbzhsUlfX1Om6DoTdbsPjiaeurplQqGd+VJA3aw5bbruFmtWfsuGxJ8lqZ0iJ1azoS2+h3rRPfemYetM+9aV96kvH1Jv2WdEXjyd+v640R0X4ffzxx7n99tuZMmUKd911Fy5X58fpOJ1OnE5nm20jRoygqamJ2tpa0tLSOn3snh6zEgqFe+w1HXn9yL7gQkofeoDyZ57GPWQoCYcc2iOvfaB6si+9jXrTPvWlY+pN+9SX9qkvHVNv2heNfbH8Z9srVqzgtttu44ILLuDee+89qOAbDoeZNGkSixYtarN9zZo1ZGZmHlTwjQUpx5+A5/gTwDQpXrqIYG2N1SWJiIiIdClLw++WLVu44447mDJlCjNnzqSyspLy8nLKy8upr68nFApRXl6O1+vdr+PZbDZOOeUUli9fHrlx7sknn2T58uUHNd43lmSfPx1XvwJCdXUUL12MqXkLRUREpA+xdNjDK6+8QiAQ4NVXX+XVV19t89jUqVO56qqrmDx5MnfeeSdnnnnmfh3z2muvxePxcPfdd1NSUkJBQQHz5s3jnHPO6Y630OfY3G7yZ89h22230Lzuayqff4bMM35qdVkiIiIiXcIwTVMzMu+HUChMVVVjj7yWw2EjLS2R6upGy8bJ1H34ASXLFgPQ7xfXkDh6jCV17Cka+hKt1Jv2qS8dU2/ap760T33pmHrTPiv6kp6euF83vFk+5leik+eYY0n5bst0c8XLlxKoqrS4IhEREZGDp/ArHco69zzcAwYSbmigeMkizGDPLBUpIiIi0l0UfqVDNqeLvNlzsMXH4920kYpnnrK6JBEREZGDovAr++TKyibnkssBqH7lZRpWf2JxRSIiIiKdp/Ar3yp53FGkTjkFgJKHlhMoL7e4IhEREZHOUfiV/ZL107OJGzKUcFMTRUsWEg4ErC5JRERE5IAp/Mp+MRwO8mZeiS0xEd/WLVT8429WlyQiIiJywBR+Zb85MzLIvWwGADVvvE79yo8srkhERETkwCj8ygFJGjOWtB/+CIDShx/EX1picUUiIiIi+0/hVw5Y5hlnEj98BGGvl+LFCwj7/VaXJCIiIrJfFH7lgBl2O3kzZ2NPTsa3Ywflf/ur1SWJiIiI7BeFX+kUR2oauVfMAsOg9q03qXv/XatLEhEREflWCr/SaYmHjSLj9J8AUPrYI/iKCi2uSERERGTfFH7loKSf9mMSDh2F6fdTvGgBYZ/P6pJEREREOqTwKwfFsNnIvWIm9pRU/MVFlD7+CKZpWl2WiIiISLsUfuWgOTwe8mbOBpuN+vffo+7tt6wuSURERKRdCr/SJRJGjCRz6k8BKFvxGN7t2yyuSERERGRvCr/SZdJO+SGJY8ZiBoMUL15IqLnZ6pJERERE2lD4lS5j2GzkXnoFjvQMAmWllD7yoMb/ioiISFRR+JUuZU9KIm/WlWC307DyY2r++7rVJYmIiIhEKPxKl4sfMpSss88FoPzJJ/Bu2WxxRSIiIiItFH6lW6ROnkLSuKMgFKJo8QJCjY1WlyQiIiKi8CvdwzAMcn52Gc6sLIKVlZQ8uEzjf0VERMRyCr/SbewJCeTNmoPhcND42adU/+dlq0sSERGRGKfwK90qbuAgsqZdAEDF0/+gecMGiysSERGRWKbwK90u5cTvknzMsRAOU7x0IcH6OqtLEhERkRil8CvdzjAMci78Ga7cPILV1ZQsX4oZDltdloiIiMQghV/pEba4OPJmz8FwuWj68guqXnzB6pJEREQkBin8So9x9ysg+4ILAah87hmavl5rcUUiIiISaxR+pUelHH8CnuNPANOkeOkigrU1VpckIiIiMUThV3pc9vnTcfUrIFRXR/HSxRr/KyIiIj1G4Vd6nM3tJn/2HAx3HM3rvqby+WesLklERERihMKvWMKVm0fORT8DoOqFf9H4xefWFiQiIiIxQeFXLOM55lhSvjsJgOLlSwlUVVpckYiIiPR1Cr9iqaxzz8M9YCDhhgaKlyzCDAatLklERET6MIVfsZTN6SJv9hxs8fF4N22k4pmnrC5JRERE+jCFX7GcKyubnEsuB6D6lZdpWP2JxRWJiIhIX6XwK1EhedxRpE45BYCSh5YTKC+3uCIRERHpixR+JWpk/fRs4oYMJdzURNGShYQDAatLEhERkT5G4VeihuFwkDfzSmyJifi2bqHiH3+zuiQRERHpYxR+Jao4MzLIvWwGADVvvE79xx9ZXJGIiIj0JQq/EnWSxowl/dTTACh95EH8JSUWVyQiIiJ9hcKvRKWMn0wlfsRIwl4vRYsXEPb7rS5JRERE+gCFX4lKht1O3oxZ2JM9+HfuoOyJx60uSURERPoAhV+JWo7UNPJmzALDoO7tt6h59x2rSxIREZFeTuFXolrCoYeR8eMzACh+5GGatm+3tiARERHp1RR+Jeql/+h0Eg4bhen38/Xv7ybs9VpdkoiIiPRSCr8S9QybjdzLZ+JITaN5506KH3kY0zStLktERER6IYVf6RUcHg/9Zl8JNhu1779H7dtvWl2SiIiI9EIKv9JrJI4cycDp5wNQvuJxvNu3WVyRiIiI9DYKv9Kr9Jv6E5LGjsUMBilevJBQc7PVJYmIiEgvovArvYphs5F/xQwc6RkEykopfeRBjf8VERGR/abwK72OIymZvFlXgt1Ow8qPqfnv61aXJCIiIr2Ewq/0SvFDhpJ19rkAlD/5BN4tmy2uSERERHoDhV/ptVInTyFp3FEQClG0eAGhxkarSxIREZEop/ArvZZhGOT87DKcWVkEKyspeXCZxv+KiIjIPin8Sq9mT0ggb9YcDIeDxs8+pfo/L1tdkoiIiEQxhV/p9eIGDiJr2gUAVDz9D5o3bLC4IhEREYlWCr/SJ6Sc+F2SjzkWwmGKly4kWF9ndUkiIiIShRR+pU8wDIOcC3+GKzePYHU1JcuXYobDVpclIiIiUUbhV/oMW1wcebPnYLhcNH35BVUvvmB1SSIiIhJlFH6lT3H3KyD7gosAqHzuGZrWfmVxRSIiIhJNFH6lz0k5fiKeiSeAaVK8bDHBmhqrSxIREZEoofArfVL2tOm4+hUQqqujeOkizFDI6pJEREQkCij8Sp9kc7vJnz0Hwx1H8/p1VD7/rNUliYiISBRQ+JU+y5WbR+7FlwBQ9e9/0bjmc4srEhEREasp/Eqflnz0MaScPAmA4geWEqiqtLgiERERsZLCr/R5WedMwz1wEOGGBoqXLMIMBq0uSURERCyi8Ct9ns3pJG/Wldji4/Fu2kjFP5+yuiQRERGxiMKvxARXVja5l14OQPV/XqZh9ScWVyQiIiJWUPiVmJF05FGkTjkFgJIHlxEoL7e4IhEREelpCr8SU7J+ejZxQ4YSbm6maMlCwoGA1SWJiIhID1L4lZhiOBzkzbwSW2Iivq1bqPjH36wuSURERHqQ5eG3pqaGm266iRNPPJFx48Yxbdo0Vq5c+a3PC4fDXHbZZdx33317PfbSSy9x6qmncvjhh3P66afz1ltvdUfp0ks5MzLIvWwGADVvvE79xx9ZXJGIiIj0FMvD7zXXXMNnn33GPffcw1NPPcWoUaO47LLL2LRpU4fP8Xq9/OpXv+Kdd97Z67EPPviAX/3qV5x//vk8++yzTJw4kTlz5uzzeBJ7ksaMJf3U0wAofeRB/CUlFlckIiIiPcHS8Ltt2zbeffdd5s+fz/jx4xkyZAjz5s0jJyeHF154od3nfPLJJ0ydOpXPPvsMj8ez1+PLli1jypQpTJ8+naFDh/LrX/+aUaNG8cgjj3T325FeJuMnU4kfMZKw10vR4gWE/X6rSxIREZFuZmn4TUtLY+nSpYwePTqyzTAMTNOktra23ee8/fbbTJkyhWeffZbk5OQ2j4XDYT755BOOPfbYNtuPOeaY/RpKIbHFsNvJmzELe7IH/84dlD3xuNUliYiISDdzWPniHo+Hk046qc22l156ie3btzNx4sR2n3P11Vd3eLy6ujqamprIzc1tsz07O5vi4uKDrtfh6Jn/K9jttja/S4vu6IsjM4N+s2ez/Y9/oO7tt0g65BBSj2//sxfN9Jlpn/rSMfWmfepL+9SXjqk37Yvmvlgafr9p1apV3HDDDUyePJlJkyYd8PO9Xi8ALperzXa3243P5zuo2mw2g7S0xIM6xoHyeOJ79PV6i67uS9rxR2PuOIcdTzxJyaOPkDPmUBIGDOjS1+gp+sy0T33pmHrTPvWlfepLx9Sb9kVjX6Im/L722mtcd911jB07lnvuuadTx3C73QD4vzF20+fzER9/cM0Ph03q6poO6hj7y2634fHEU1fXTCgU7pHX7A26sy9JU35I4udf0vjlF3x55x8ZctPN2OLiuvQ1upM+M+1TXzqm3rRPfWmf+tIx9aZ9VvTF44nfryvNURF+H3/8cW6//XamTJnCXXfdtdeV2/2VmppKQkICZWVlbbaXlZXtNRSiM4LBnv1Qh0LhHn/N3qC7+pJz2Qy23XoT/qIiCh96iNzLZ2AYRpe/TnfSZ6Z96kvH1Jv2qS/tU186pt60Lxr7YvlAjBUrVnDbbbdxwQUXcO+993Y6+ELLzXLjxo3jo4/aztv64YcfctRRRx1sqdLHOTwe8mbMBpuN+g/fp/btN60uSURERLqYpeF3y5Yt3HHHHUyZMoWZM2dSWVlJeXk55eXl1NfXEwqFKC8vj4zl3R+XXHIJ//73v3nooYfYtGkTf/jDH1i7di0XX3xxN74T6SsSRowkc+pZAJSveBzv9m0WVyQiIiJdydLw+8orrxAIBHj11VeZOHFim1+33347xcXFTJw4kRdffHG/jzlx4kTuuOMOnnjiCaZOncoHH3zA4sWLGTp0aDe+E+lL0k75AYljxmIGgxQvXkioqWfGeouIiEj3M0zTNK0uojcIhcJUVTX2yGs5HDbS0hKprm6MunEyVurJvoQaGth263yCVZUkHTWevFlzonr8rz4z7VNfOqbetE99aZ/60jH1pn1W9CU9PXG/bnizfMyvSDSyJyWRN+tKsNtpWLWSmjdes7okERER6QIKvyIdiB8ylKyzzwOg/O9/o3nzZosrEhERkYOl8CuyD6mTv0fSUeMhFKJ4yQJCDQ1WlyQiIiIHQeFXZB8MwyDn4ktxZmUTrKyk5MFlmGGN6RIREemtFH5FvoU9IYG82XMwHA4aP/+M6v+8bHVJIiIi0kkKvyL7IW7AQLKmTQeg4p9P0bxhvcUViYiISGco/Irsp5QTTyL5mGMhHKZoyUKC9XVWlyQiIiIHSOFXZD8ZhkHOhT/DlZtHqKaGkuVLNf5XRESkl1H4FTkAtri4lvG/LhdNX35B1YsvWF2SiIiIHACFX5ED5O5XQPYFFwFQ+dwzNK39yuKKREREZH8p/Ip0QsrxE/FMPAFMk+JliwnW1FhdkoiIiOwHhV+RTsqeNh1XvwJCdXUUL12EGQpZXZKIiIh8C4VfkU6yud3kz56D4Y6jef06Kp9/1uqSRERE5Fso/IocBFduHrkXXwJA1b//ReOazy2uSERERPZF4VfkICUffQwpJ08CoPiBpQSqKi2uSERERDqi8CvSBbLOmYZ74CDCDQ0UL1mEGQxaXZKIiIi0Q+FXpAvYnE7yZl2JLT4e76aNVPzzKatLEhERkXYo/Ip0EVdWNrmXXg5A9X9epmH1JxZXJCIiIt+k8CvShZKOPIq0KacAUPLgMvzlZRZXJCIiIntS+BXpYpk/PZu4ocMINzdTvHgh4UDA6pJERESklcKvSBczHA7yZs7GlpSEb9tWyv/+N6tLEhERkVYKvyLdwJmeQd5lMwCo/e/r1H/0ocUViYiICCj8inSbxMPHkH7qaQCUPPIQ/pISiysSERERhV+RbpTxk6nEjxiJ6fNStOh+wn6/1SWJiIjENIVfkW5k2O3kzZiNPdmDv3AnZSset7okERGRmKbwK9LNHKmp5M2YBYZB3TtvUfvuO1aXJCIiErMUfkV6QMKhh5Hx4zMAKPvro/gKd1pbkIiISIxS+BXpIek/Op2Ew0Zh+v0UL1pA2Ou1uiQREZGYo/Ar0kMMm43cy2diT03FX1JM6WOPYJqm1WWJiIjEFIVfkR7k8HjImzEbbDbqP3yf2rfftLokERGRmKLwK9LDEkaMJHPqWQCUr3gc7/ZtFlckIiISOxR+RSyQdsoPSBwzFjMYpHjxQkJNTVaXJCIiEhMUfkUsYNhs5F56BY70DAJlpZQ+8qDG/4qIiPQAhV8Ri9iTksibdSXY7TSsWknNG69ZXZKIiEifp/ArYqH4IUPJOvs8AMr//jeaN2+2uCIREZG+TeFXxGKpk79H0lHjIRSieMkCQg0NVpckIiLSZyn8iljMMAxyLr4UZ1Y2wcpKSh5chhkOW12WiIhIn6TwKxIF7AkJ5M2eg+Fw0Pj5Z1T/52WrSxIREemTFH5FokTcgIFkTZsOQMU/n6J5w3qLKxIREel7FH5FokjKiSeRfMxxEA5TtGQhwfo6q0sSERHpUxR+RaKIYRjkXHgxrtw8QjU1lCxfqvG/IiIiXUjhVyTK2OLiyJt9FYbLRdOXX1D1739ZXZKIiEifofArEoXc/fqRfcFFAFQ+/yxNa7+yuCIREZG+QeFXJEqlHD8Rz8QTwTQpXrqYYE2N1SWJiIj0egq/IlEs+/zpuAr6E6qvo3jpIsxQyOqSREREejWFX5EoZnO5yJ81B8MdR/P6dVQ+94zVJYmIiPRqCr8iUc6Vm0vuxZcAUPXiCzSu+dziikRERHovhV+RXiD56GNIOXkyAMXLlxCoqrS4IhERkd5J4Vekl8g65zzcAwcRbmykePFCzGDQ6pJERER6HYVfkV7C5nSSN+tKbPHxeDdvouLpf1hdkoiISK+j8CvSi7iyssm99HIAql99hYbVn1hckYiISO+i8CvSyyQdeRRpU04BoOTBZfjLyyyuSEREpPdQ+BXphTJ/ejZxQ4cRbm6mePFCwoGA1SWJiIj0Cgq/Ir2Q4XCQN3M2tqQkfNu2Uvq3J6wuSUREpFdQ+BXppZzpGeRdNgOA6tdfo/ztdy2uSEREJPop/Ir0YomHjyH91NMA2Hj/QnwlxRZXJCIiEt0UfkV6uYyfTCVh5CGEvV523n8/Yb/f6pJERESilsKvSC9n2O30m30lzpQUfDt3ULbicatLEhERiVoKvyJ9gDM1lRHX/RIMg7p33qL23XesLklERCQqKfyK9BGpYw4n64ypAJT99VF8hTstrkhERCT6KPyK9CGZp/+YhFGjMf1+ihctIOz1Wl2SiIhIVFH4FelDDJuN3Mtn4EhLw19STOljD2OaptVliYiIRA2FX5E+xpHsIW/GbLDZqP/wA2rfetPqkkRERKKGwq9IHxQ/fASZZ54FQPkTj+Pdvs3iikRERKKDwq9IH5X2/R+QOPYIzGCQ4kULCDU1WV2SiIiI5RR+Rfoow2Yj95LLcWRkECgvo/SRBzX+V0REYp7Cr0gfZk9KIm/mHLDbaVi1kprXX7O6JBEREUsp/Ir0cfFDhpB1znkAlP/jbzRv3mRxRSIiItZR+BWJAamTvkfSUeMhFKJ48UJCDQ1WlyQiImKJToffjz/+mE8++QSAnTt3MmPGDE4//XQWLFjQZcWJSNcwDIOciy/FmZVNsKqSkgeXYYbDVpclIiLS4zoVfp977jkuuugiXnutZfzgzTffzMcff8zAgQNZvHgxS5cu7dIiReTg2RMSyJs9B8PhoPHzz6h+5WWrSxIREelxnQq/Dz30EFOnTuX666+nsrKS9957j6uuuor777+fX/7ylzz99NNdXaeIdIG4AQPJmjYdgIpnnqJ5w3qLKxIREelZnQq/mzdv5ic/+QkAb731FqZpMnnyZAAOP/xwiouL9/tYNTU13HTTTZx44omMGzeOadOmsXLlyg73r66u5tprr2XChAlMmDCBG2+8kaZvzF86adIkRo4c2ebXdddd14l3KtL3pJx4EsnHHAfhMEVLFhKsr7O6JBERkR7j6MyTPB4PjY2NALz55pvk5+czaNAgALZv305aWtp+H+uaa66hsrKSe+65h/T0dFasWMFll13GP//5T4YOHbrX/j//+c/x+Xw8/PDD1NXVMW/ePG655RZ+//vfA9DQ0EBRURFLlixh1KhRkefFxcV15q2K9DmGYZBz4cX4tm3FX1JMyfKl9Lv6Ggyb7n8VEZG+r1Pf7Y499ljuv/9+lixZwquvvsqpp54KwCuvvMKf//xnjj/++P06zrZt23j33XeZP38+48ePZ8iQIcybN4+cnBxeeOGFvfZfvXo1H330EXfeeSejRo3iuOOO49Zbb+W5556jtLQUgPXr12OaJuPGjSMrKyvyKzk5uTNvVaRPssXFkTf7KgyXi6Yvv6Dq3/+yuiQREZEe0anwO2/ePNLS0liwYAHf+c53mDlzJgB33nkn+fn5XHvttft1nLS0NJYuXcro0aMj2wzDwDRNamtr99p/5cqVZGVltbkifPTRR2MYBqtWrQJg3bp1ZGVl4fF4OvPWRGKGu18/si+4CIDK55+lae1XFlckIiLS/To17CEtLY0HHnhgr+0rVqwgPz9/v4/j8Xg46aST2mx76aWX2L59OxMnTtxr/9LSUvLy8tpsc7lcpKamRsYZr1+/noSEBObOncvq1atJT0/nzDPP5KKLLsJ2kD/WdTh65sfCdrutze/SQn3pWGd7k3HSifg2baDmrTcpXraYIbf+FmdqajdUaA19Zjqm3rRPfWmf+tIx9aZ90dyXToVfaBlb29jYSE5ODn6/n0cffZSSkhJOOeUUJkyY0Kljrlq1ihtuuIHJkyczadKkvR5vbm7G5XLttd3tduPz+QDYsGED9fX1nHrqqVx11VWsXLmSu+66i9raWq6++upO1QVgsxmkpSV2+vmd4fHE9+jr9RbqS8c60xvPVTP5fPtWmrZuo3T5EkbfOh/Dbu+G6qyjz0zH1Jv2qS/tU186pt60Lxr70qnw+/nnn3P55ZdzzjnncN111/Hb3/6Wv//973g8HlasWMF9990Xmf1hf7322mtcd911jB07lnvuuafdfeLi4vD7/Xtt9/l8JCQkAC3TsPl8PpKSkgAYOXIkjY2NLFq0iLlz53b66m84bFJX1/TtO3YBu92GxxNPXV0zoZAWIthFfenYwfYmb9aVbLl5PnVffMn6Bx8j+6yzu6HKnqfPTMfUm/apL+1TXzqm3rTPir54PPH7daW5U+H3T3/6E0OGDOHcc8/F6/Xyr3/9i/PPP5+bbrqJm266icWLFx9Q+H388ce5/fbbmTJlCnfddVe7V3cBcnNzIwtr7OL3+6mpqSEnJwcAp9OJ0+lss8+IESNoamqitrb2gGai+KZgsGc/1KFQuMdfszdQXzrW2d7YM3PIuegSipcuouKFf+EeOpzEw8d0Q4XW0GemY+pN+9SX9qkvHVNv2heNfenUZdDPPvuM2bNn079/f95//328Xm9k3t9TTz2VDRs27PexVqxYwW233cYFF1zAvffe22HwBZgwYQIlJSVs27Ytsu3DDz8EYNy4cYTDYSZNmsSiRYvaPG/NmjVkZmYeVPAV6euSjz6GlJNb/tNavHwJgapKiysSERHpep0KvzabLRJS33zzTTweD2PGtFwlamho2O85dbds2cIdd9zBlClTmDlzJpWVlZSXl1NeXk59fT2hUIjy8nK8Xi8AY8eOZdy4cfzyl7/k888/54MPPmD+/PmcccYZ5OTkYLPZOOWUU1i+fHnkxrknn3yS5cuXH9R4X5FYkXXOebgHDiLc2Ejx4oWYwaDVJYmIiHSpTg17GD16NE899RRxcXG89NJLfPe738UwDCorK1m2bFmbqcv25ZVXXiEQCPDqq6/y6quvtnls6tSpXHXVVUyePJk777yTM888E8MwuP/++7nlllu4+OKLcbvd/OAHP+A3v/lN5HnXXnstHo+Hu+++m5KSEgoKCpg3bx7nnHNOZ96qSEyxOZ3kz5rDtltvwrt5ExVP/4Osc6dZXZaIiEiXMUzTNA/0SV999RWXX345VVVVkVXZBg0axHHHHUc4HOaBBx7Y7wDcW4RCYaqqGnvktRwOG2lpiVRXN0bdOBkrqS8d6+reNKz+hKIFfwEg78q5JI876qCPaQV9Zjqm3rRPfWmf+tIx9aZ9VvQlPT2x+254O+yww/jPf/7Dpk2bGD58eGSmhZtvvjmyspqI9F5JR44j7fs/oPo/L1P60HLc/fvjysq2uiwREZGD1ul5fpOSkhg8eDArV66kvr6etLQ0jj/++MgUYyLSu2WeeRbNmzbi3bSR4kUL6P+bedicHd+QKiIi0ht0OvwuXbqUhQsX4vP52DVywul0MmvWLObMmdNlBYqINQyHg7yZs9l263x827dR/ve/kdO6HLKIiEhv1anZHp5++mnuueceTjvtNB555BFefPFFHnnkEU4//XTuv/9+nnnmma6uU0Qs4EzPIO/yGQDU/vcN6j76wOKKREREDk6nrvw+/PDDTJs2jfnz50e2DRkyhGOOOYa4uDgeffRRpk6d2mVFioh1EkePIf1Hp1P1739R+sjDxA0YiCs3z+qyREREOqVTV363bdvG9773vXYfmzx5Mps3bz6ookQkumT8+AziRx6C6fNStGgBYZ/P6pJEREQ6pVPhNycnh507d7b72I4dO3TTm0gfY9jt5F0xC7vHg79wJ2VPPG51SSIiIp3SqfA7adIk/vKXv/Dpp5+22b569Wruu+8+Jk2a1BW1iUgUcaSmknfFLDAM6t55m9p337G6JBERkQPWqTG/c+fO5b333mPatGnk5+eTlZVFeXk5hYWFDBs2jGuvvbar6xSRKJBw6GFk/PgMKp97hrK/PkrcoEG4+xVYXZaIiMh+69SV36SkJJ566iluuukmxowZQ0JCAmPGjOGmm27i6aefJjU1tYvLFJFokf6j00kYNRrT76d40QLCXq/VJYmIiOy3/b7y+5vf/Kbd7XFxccTFxQGwZs0a1qxZg2EY3HHHHV1ToYhEFcNmI/fyGWy/dT7+kmJKH3uY3MtnYhiG1aWJiIh8q/0Ovx9++OF+H1TfBEX6Nkeyh7wZs9nxx99R/+EHxI84hNSTvmt1WSIiIt9qv8PvG2+80Z11iEgvEz98BJlnnkXFU3+n/InHiRs8mLgBA60uS0REZJ86NeZXRAQg7fs/IHHsEZjBIMWLFhBqarK6JBERkX1S+BWRTjNsNnIvuRxHRgaB8jJKH3kQ0zStLktERKRDCr8iclDsSUnkzZwDdjsNq1ZS8/prVpckIiLSIYVfETlo8UOGkHXOeQCU/+NvNG/eZHFFIiIi7VP4FZEukTrpeyQdNR5CIYoXLyTU0GB1SSIiIntR+BWRLmEYBjkXX4ozO4dgVSUlDy7DDIetLktERKQNhV8R6TL2hATyZl2J4XDQ+PlnVL/ystUliYiItKHwKyJdKm7AQLLOnw5AxTNP0bR+ncUViYiI7KbwKyJdLuWEk0g+9jgIhyleuohgXZ3VJYmIiAAKvyLSDQzDIGf6xbjy8gnV1FCybInG/4qISFRQ+BWRbmGLiyNv9hwMl4umtV9S9e9/WV2SiIiIwq+IdB93fj9yLrwYgMrnn6Vp7VcWVyQiIrFO4VdEupXnuOPxnHAimCbFSxcTrKm2uiQREYlhCr8i0u2yp03HVdCfUH0dxUsXY4ZCVpckIiIxSuFXRLqdzeUif9YcDHcczevXUfncM1aXJCIiMUrhV0R6hCs3l9yLLwGg6sUXaPj8M4srEhGRWKTwKyI9JvnoY0g5eTIAJQ8sJVBVaXFFIiISaxR+RaRHZZ1zHu6Bgwg3NlK8eCFmMGh1SSIiEkMUfkWkR9mcTvJnzcEWH4938yYqnv6H1SWJiEgMUfgVkR7nzMoi99IrAKh+9RXqP1llcUUiIhIrFH5FxBJJR44j7fs/AKD0oeX4y8ssrkhERGKBwq+IWCbzzLOIGzqMcHMzxYsWEA74rS5JRET6OIVfEbGM4XCQN3M2tqQkfNu3Uf73v1ldkoiI9HEKvyJiKWd6BnmXzwCg9r9vUPfRBxZXJCIifZnCr4hYLnH0GNJ/dDoApY88jL+k2OKKRESkr1L4FZGokPHjM4gfeQimz0vRogWEfT6rSxIRkT5I4VdEooJht5N3xSzsHg/+wp2UPfG41SWJiEgfpPArIlHDkZpK3ozZYBjUvfM2te++bXVJIiLSxyj8ikhUSTjkUDJ+MhWAsr8+hq9wp8UViYhIX6LwKyJRJ/3U00gYNRrT76do0f2EvV6rSxIRkT5C4VdEoo5hs5F7+QwcaWkESkoofexhTNO0uiwREekDFH5FJCo5kj3kzbgSbDbqP/yA2rf+Z3VJIiLSByj8ikjUih8+nMyfng1A+RN/xbttq7UFiYhIr6fwKyJRLe37PyBx7BGYwSDFixcQamqyuiQREenFFH5FJKoZhkHupVfgyMwkUF5O6cMPaPyviIh0msKviEQ9e2Ii+TOvBLudhk9WUfP6q1aXJCIivZTCr4j0CnGDh5B1znkAlP/jSZo3b7K4IhER6Y0UfkWk10id9D2SjhoPoRDFixcSamiwuiQREellFH5FpNcwDIOciy/FmZ1DsKqSkgeXYYbDVpclIiK9iMKviPQq9oQE8mZdieFw0Pj5Z1S/8rLVJYmISC+i8CsivU7cgIFknT8dgIpnnqJp/TqLKxIRkd5C4VdEeqWUE04i+djjIBymeOkignV1VpckIiK9gMKviPRKhmGQM/1iXHn5hGpqKFm2RON/RUTkWyn8ikivZYuLI2/2HAyXi6a1X1L1739ZXZKIiEQ5hV8R6dXc+f3IufBiACqff5amtV9ZXJGIiEQzhV8R6fU8xx2P54QTwTQpXrqYYE211SWJiEiUUvgVkT4he9p0XAX9CdXXUbx0MWYoZHVJIiIShRR+RaRPsLlc5M+agy0ujub166h87hmrSxIRkSik8CsifYYrN5eciy8FoOrFF6j/7DOLKxIRkWij8CsifUryhKNJnTQZgKKli/GVl1tckYiIRBOFXxHpczLPPg/3oMGEGhtZ98d7MINBq0sSEZEoofArIn2Ozekkf+aV2BISqF+3ntJ/PGl1SSIiEiUUfkWkT3JmZdHvihkAVL3yCvWfrLK4IhERiQYKvyLSZyUfOY78M34MQOlDy/GXlVlckYiIWE3hV0T6tIEXXkD8sOGEm5spXryAcMBvdUkiImIhhV8R6dNsDgcFV87BlpSEb/s2yp/8m9UliYiIhRR+RaTPc6ank3d5y/jf2v+9Qd2HH1hckYiIWEXhV0RiQuLoMaT/6HQASh99CH9xkcUViYiIFRR+RSRmZPz4DOJHHoLp81G0eCFhn8/qkkREpIcp/IpIzDDsdvKumIXd48FfuJOyJx63uiQREelhloffmpoabrrpJk488UTGjRvHtGnTWLlyZYf7V1dXc+211zJhwgQmTJjAjTfeSFNTU5t9XnrpJU499VQOP/xwTj/9dN56663ufhsi0ks4UlPJmzEbDIO6d96m9t23rS5JRER6kOXh95prruGzzz7jnnvu4amnnmLUqFFcdtllbNq0qd39f/7zn7Njxw4efvhh/vKXv/Duu+9yyy23RB7/4IMP+NWvfsX555/Ps88+y8SJE5kzZ06HxxOR2JNwyKFk/GQqAGV/fQxf4U6LKxIRkZ5iafjdtm0b7777LvPnz2f8+PEMGTKEefPmkZOTwwsvvLDX/qtXr+ajjz7izjvvZNSoURx33HHceuutPPfcc5SWlgKwbNkypkyZwvTp0xk6dCi//vWvGTVqFI888khPvz0RiWLpp55GwqjRmH4/RYvuJ+z1Wl2SiIj0AEvDb1paGkuXLmX06NGRbYZhYJomtbW1e+2/cuVKsrKyGDp0aGTb0UcfjWEYrFq1inA4zCeffMKxxx7b5nnHHHPMPodSiEjsMWw2ci+fgSMtjUBJCaWPPYxpmlaXJSIi3cxh5Yt7PB5OOumkNtteeukltm/fzsSJE/fav7S0lLy8vDbbXC4XqampFBcXU1dXR1NTE7m5uW32yc7Opri4+KDrdTh65v8Kdrutze/SQn3pmHrTvm/riyMtlYIr57D1zjuo//ADkg45hLSTJ/VkiZbRZ6Z96kv71JeOqTfti+a+WBp+v2nVqlXccMMNTJ48mUmT9v4G1NzcjMvl2mu72+3G5/Phbf2x5Tf32fX4wbDZDNLSEg/qGAfK44nv0dfrLdSXjqk37dtXX9KOPhIums7Whx+l5K+Pkz12FElDh/RgddbSZ6Z96kv71JeOqTfti8a+RE34fe2117juuusYO3Ys99xzT7v7xMXF4ff799ru8/lISEjA7XYD7LWPz+cjPv7gmh8Om9TVNX37jl3Abrfh8cRTV9dMKBTukdfsDdSXjqk37dvfvsSfNJmkT9fQ8Olqvrrzjwy55VbsCQk9WGnP02emfepL+9SXjqk37bOiLx5P/H5daY6K8Pv4449z++23M2XKFO666652r+4C5Obm8tprr7XZ5vf7qampIScnh9TUVBISEigrK2uzT1lZ2V5DITojGOzZD3UoFO7x1+wN1JeOqTft25++5FxyOd7b5hMoL6Nw+TLyZl+FYRg9VKF19Jlpn/rSPvWlY+pN+6KxL5YPxFixYgW33XYbF1xwAffee2+HwRdgwoQJlJSUsG3btsi2Dz/8EIBx48ZhGAbjxo3jo48+avO8Dz/8kKOOOqp73oCI9An2xETyZ14JdjsNn6yi5vVXrS5JRES6gaXhd8uWLdxxxx1MmTKFmTNnUllZSXl5OeXl5dTX1xMKhSgvL4+M5R07dizjxo3jl7/8JZ9//jkffPAB8+fP54wzziAnJweASy65hH//+9889NBDbNq0iT/84Q+sXbuWiy++2Mq3KiK9QNzgIWSdOw2A8n88SfOmjRZXJCIiXc3S8PvKK68QCAR49dVXmThxYptft99+O8XFxUycOJEXX3wRaJkG7f7776egoICLL76YX/ziF5x44oncfPPNkWNOnDiRO+64gyeeeIKpU6fywQcfsHjx4jbTo4mIdCT15MkkjZ8AoRDFSxYRamiwuiQREelChqmJLfdLKBSmqqqxR17L4bCRlpZIdXVj1I2TsZL60jH1pn2d7UuouZntt91MoKyUxMPHkD/3Fxg2y0eJdSl9ZtqnvrRPfemYetM+K/qSnp64Xze89a2zuYhIF7DHx5M/ew6Gw0Hjms+pfuUlq0sSEZEuovArItIOd/8BZJ9/IQAVzzxN0/p1FlckIiJdQeFXRKQDnhNOJPm470A4TPGSRQTr6qwuSUREDpLCr4hIBwzDIGf6xbjy8wnV1lCybAlmWGP6RER6M4VfEZF9sLnd5M2ag+Fy0bT2S6peeN7qkkRE5CAo/IqIfAt3fj9yLmyZK7zyX8/R+NWXFlckIiKdpfArIrIfPMcdj+eEE8E0KVm2mGBNtdUliYhIJyj8iojsp+xp03EV9CdUX0/xkkWYoZDVJYmIyAFS+BUR2U82l4v8WXOwxcXRvGE9lc89Y3VJIiJygBR+RUQOgCs3l5yLLwWg6sUXaPj8M4srEhGRA6HwKyJygJInHE3qpMkAlDywlEBlpcUViYjI/lL4FRHphMyzz8M9aDDhxkaKlyzEDAatLklERPaDwq+ISCfYnE7yZ16JLSEB7+ZNlD/9D6tLEhGR/aDwKyLSSc6sLHIvvQKAmldfof6TVRZXJCIi30bhV0TkICQdcSRpp/wAgNKHluMvK7O4IhER2ReFXxGRg5Q59Szihg4j3NxM8eIFhAN+q0sSEZEOKPyKiBwkw+Egb+aV2JKS8G3fRvmTf7O6JBER6YDCr4hIF3Cmp5N3+QwwDGr/9wZ1H35gdUkiItIOhV8RkS6SOHoM6T86DYDSRx/CX1xkcUUiIvJNCr8iIl0o48dTiT/kUEyfj6LFCwn7fFaXJCIie1D4FRHpQobNRt4VM7F7PPgLd1K24nGrSxIRkT0o/IqIdDFHSip5M2aDYVD37tvUvvO21SWJiEgrhV8RkW6QcMihZPxkKgBlKx7Dt3OHxRWJiAgo/IqIdJv0U08jYdRoTL+fosULCHubrS5JRCTmKfyKiHQTw2Yj7/KZONLSCJSUUProw5imaXVZIiIxTeE3ytQ2+LjzsVX89sEPeeG9razfUUMgGLK6LBHpJHtyMnkzrwS7nfqPPqT2zf9aXZKISExzWF2AtFVZ5+PrbdWYwIdftmxz2A0G5iYzvF8qQ/ulMLwgBU+iy9I6RWT/xQ8bTuaZZ1Hxjycp/9sK4gYPIW7gIKvLEhGJSQq/UWZIvofbrjiGzSUNfLa+jA07aqhrCrCpsI5NhXWR/bLT4hneL4VhBSkM65dCXmYiNsOwsHIR2Ze07/+A5vXraPzsU4oXL2DAjTdjT0i0uiwRkZij8BuFBuQkM/aQXL47No9AIER5TTMbdtaysbCWjTtrKaxopKy6mbLqZt79ogSAxDgHQ/u1BOHhBSkMyvPgdtotficisothGOReegXbbptPoLyc0oceJO/KqzD0n1YRkR6l8BvlDMMgOy2B7LQEjj88D4BGb8uV4I2FNWzcWcvmojoavUE+31TJ55sqAbDbDAbkJDGsXyrDC1quEKcmua18KyIxz56YSP7MK9n+u9tpWL2KmtdfJe1737e6LBGRmKLw2wslxjkZMzSDMUMzAAiGwuwoa2Djzlo2FNaycWcNNQ1+thTXs6W4nldXtswvmpkSx7CClNbhEqn0y0zEZtNVJ5GeFDd4CFnnTqN8xeOU/+NJ4gYPIX7oMKvLEhGJGQq/fYDDbmNwnofBeR6mTOiPaZpU1nrZWLgrDNeys6yBilovFbVePviyFIB4t50h+SmRscND8j3EufSREOluqSdPpnn9OhpWfkzxkkUMvOkW7ElJVpclIhITlHT6IMMwyEyNJzM1nmNH5QLQ7AuyqaglCG8srGVTUR3NvhBfbqniyy1VANgMg/7ZSZGb6IYXpJDuibPyrYj0SYZhkHPxpfi2bydQVkrJA0vJn/sLDJtmnxQR6W4KvzEi3u1g9OAMRg9uGSoRCofZWdbYcnV4Zw2bCmuprPOxrbSebaX1vL5qJwDpHndrEE5lWL8UCrITsesbtMhBs8fHkz97Dttvv5XGNZ9T/cpLpP/wR1aXJSLS5yn8xii7zcbA3GQG5iYz+agCAKrqWodKtF4d3lHaQFWdj4/qyvhobRkAbpedIXmeyE10Q/JSSIjTx0ikM9z9B5B9/oWUPvoQFc88TdzQYSSMGGl1WSIifZpSi0Ske+I42hPH0YfmAOD1B9lSVBcZN7ypqJZmX4i126pZu60aAAPol5UUCcPD+6WQkRKn6ZtE9pPnhBNp2rCO+vffaxn/O/9WHB6P1WWJiPRZCr/SoTiXg0MHpXPooHQAwmGToorGyIwSG3bWUlHrZWd5AzvLG/jv6kIAUpNcDGudUWJ4QQr9s5Nw2DVUQqQ9hmGQM/1ifNu24i8qomTZEvr98lqN/xUR6SYKv7LfbDaDguwkCrKTOPnIfgDUNPgiN9Ft2FnL9tJ6ahr8rFxXzsp15QC4HC2zUQwraLmJbmi/FBLjnFa+FZGoYnO7yZs1h+2/vYWmtV9S9cLzZPz4DKvLEhHpkxR+5aCkJrkZf0g24w/JBsAXCLG1uC4ShjcV1tLoDbJuRw3rdtREntcvM5GhrTNKDCtIITs1XkMlJKa58/uRc+HPKHlgKZX/eo64YcNJPGyU1WWJiPQ5Cr/SpdxOOyMHpDFyQBoAYdOkuLKJjTtrIsszl1Y3U1jRSGFFI299VgSAJ8HJsNYZJYYVpDAwJxmnQz/2ldjiOe47NG9YR+1bb1KybHHL+N/UNKvLEhHpUxR+pVvZDIN+mYn0y0zkpCNahkrUNfojQXhDYQ3bSuqpawrwyfpyPlnfMlSiZeGO5Micw8P6pZCmOYclBmSddwHeLZvx7dhB8ZJFFFz3awy73eqyRET6DIVf6XGeRBfjRmQxbkQWAIFgiK0l9S1huHX8cENzgA2tf94lLyOB0UMzGZCdyJA8D7npCRoqIX2OzeVqGf972800b1hPxbP/JOunZ1tdlohIn6HwK5ZzOuwML0hleEEqPwRM06S0upkNO2siN9MVVza1/toeeV5SvDMyTGJYvxQG5yXjdOgKmfR+rpxccn52KcWLF1L90r+JHz6cpDFHWF2WiEifoPArUccwDHLTE8hNT+CEMfkANDQH2FJSx47yJtZsKGdzcR0NzQE+3VjBpxsrALDbDAbl7hoqkcqwghRSEl1WvhWRTksefzTNk9ZT88ZrlCxfxsD5t+DMyLS6LBGRXk/hV3qFpHgnRw7PYtLRiVRXN+L1BdlWUr/H2OFa6hr9bCqqY1NRHa+wA4DstPjI1eHh/VLIy0zEpqES0ktknn0uzZs34du6heIlC+l//Q0YDp22RUQOhs6i0is57DaG9muZM/iUo1uGSpTXNEemV9tQWEtReSNl1c2UVTfz3hclACS4HQzdIwwPzvfgdmqohEQnm9NJ/qwr2XbrfLybN1P+1N/JPu98q8sSEenVFH6lTzAMg+y0BLLTEjj+8DwAmrwBNhbWsbGwZezw5uI6mnxB1myuZM3mSqBlqMSAnKTIMIlh/VJIS3Zb+VZE2nBmZpF76RUU3f9nal77D/HDR5B81HiryxIR6bUUfqXPSohzMmZoBmOGZgAQDIXZUdYQGSaxcWcNNQ1+thTXs6W4nldXtgyVyEyJi1wZHtovhYKsJGw2DZUQ6yQdcSRpp/yA6ldepvThB3D3H4ArO9vqskREeiWFX4kZLXMHexic52HKhP6YpkllnXePMFzLzvIGKmq9VNR6+eDLUgDi3XaG5LeE4WEFKQzO8xDv1j8d6VmZU8+ieeNGvJs2Urx4Af1/Mw+bUzd0iogcKH0Hl5hlGAaZKfFkpsRz7KhcAJp9QTYX1bVMs1ZYy6aiOpp9Ib7cUsWXW6panwf9s5MY3jpUYnhBCulagEO6meFwkDfzSrbdehO+7dso/9sT5Fx4sdVliYj0Ogq/InuIdzsYNTidUYPTAQiFwxSWN0YW39i4s4bKOh/bSxvYXtrA65/sBCAt2c3w1jHDwwtSKchOxG7T8szStZzp6eRdPoPCP/+J2jf/S/yIkXiOOdbqskREehWFX5F9sNtsDMhJZkBOMpOPKgCgqs7bZoq1HaUNVNf7+GhtGR+tLQPA7bQzJN/TGoZTGJKfQkKc/rnJwUscPYb0H51G1Qv/ovTRh4gbMABXXr7VZYmI9Br6bixygNI9cRztiePoQ3MA8PqDbCmqY2PrFGubCuto9gVZu62atduqATCAfllJkRvphhWkkJkSp+WZpVMyfjyV5o0baf56LUWLFzLghhuxuTVLiYjI/lD4FTlIcS4Hhw5K59BBLUMlwqZJUXlj5Ca6jYU1lNd42VnewM7yBv63uhCAlCRXSxDul8KwglQG5CThsGuohHw7w2Yj74qZbLvlJvyFOylb8Ti5l1xmdVkiIr2Cwq9IF7MZBgXZSRRkJ3Hykf0AqGnwtQbhWjbsrGV7aT21DX5Writn5bpyAFyOltkods03PKwghcQ4p5VvRaKYIyWVvBmz2Xn3H6h7923ih48gZeIJVpclIhL1FH5FekBqkpvxh2Qz/pCWuVn9gRBbiusiYXhTYS2N3iDrdtSwbkdN5Hn5mYmRccPD+qWQnRavoRISkXDIoWT8ZCqVz/6TshWPETdoEO6C/laXJSIS1RR+RSzgctoZOSCNkQPSgJahEiWVTa1huGVFutLqZooqGimqaOStz4oA8CQ4Gdo6o8SwghQG5iTjdGioRCxLP/U0mjesp+nLLyhavICB/28+trh4q8sSEYlaCr8iUcBmGORnJpKfmciJY1vu3K9r9LOpcPcCHFtL6qhrCrB6QwWrN1QALQt3DMpLZmT/VI48JIe8tDjiXfpnHUsMm428y2ey7dabCJSUUProw+ReMUs/IRAR6YC+S4pEKU+iiyNHZHHkiCwAAsEQW0vqd0+ztrOWhuZAy1jinbX8+/1tAOSkJ0RmlBhekEJueoKCUB9nT04mb+aV7Pjj76j/6EPiR4wk9buTrC5LRCQqKfyK9BJOh53hBakML0iFY8A0TUqrm9mws4bNRXVsLq5jR2kDpVVNlFY18c6aYgAS4xyRG+iGF6QyKDcZl9Nu7ZuRLhc/bDiZZ55FxT+epPxvK4gbPIS4gYOsLktEJOoo/Ir0UoZhkJueQG56AiePKyAtLZHthdWs214TmVliS3Edjd4gn22q5LNNlQDYbQaDcpNbxw63TLOWkuiy+N1IV0j7/g9o3rCexk9XU7x4AQNuvBl7QqLVZYmIRBWFX5E+JDnBxRHDMjliWCYAwVCYbaX1kaERGwprW8YSF9WxqaiO/3y8A4Ds1Pg2U6zlZyZi01CJXscwDHIvuZxtt80nUF5O6UMPknflVRr2IiKyB4VfkT7MYbcxND+FofkpnHJ0y1CJ8lovG1tnlNhQWEtReSNlNc2U1TTz3hclACS4HQxtDcLD+qUwJM+D26WhEr2BPTGR/Flz2PG722lYvYqa1/5D2pRTrC5LRCRqKPyKxBDDMMhOjSc7NZ7vjM4DoMkbYFNRHRt21rJxZw2bi+to8gVZs7mSNZt3D5Xon50UGTc8rF8KaclaTjdaxQ0aTNY551G24nHKn/o7cUOGEj90mNVliYhEBYVfkRiXEOfk8CEZHD4kA2gZKrGjrCEyq8TGwlqq631sLalna0k9r63cCUCGJ651zHDL1eGCrCRsNv14PVqknDyZpvXraVj5EcVLFjLwpluxJyVZXZaIiOUUfkWkDYe9ZZnlwXkepozvj2maVNZ5I0F4485adpQ3UFnnpfIrLx98VQpAnMvO0HwPw1oX4BiS5yHerVOMVQzDIOfiS/Dt2EagtJSSB5aSP/cXGDYtiiIisU3fmURknwzDIDMlnsyUeI4dlQtAsy/I5qI6NuysYVNhLZuK6vD6Q3y5tZovt1a3Pg/6Z7UMlRhWkMLwfqlkpMRZ+VZijj0+nvxZc9h+x200rvmc6pdfJP3U06wuS0TEUgq/InLA4t0ORg1OZ9TgdADCYZOd5Q0t44Zbrw5X1nnZXtbA9rIG3vikEIC0ZDfDC1Ii06z1z07CriuR3crdfwDZ50+n9JGHqHjmaeKGDiNh5CFWlyUiYhmFXxE5aDabwYCcZAbkJDP5qAIAquq8u1ejK6xlR2kD1fU+PlpbxkdrywBwO+0MyfdEwvDQ/BQS4nRa6mqeiSfStH4d9e+/R/HSxQy86RYcGWlWlyUiYgl9lxGRbpHuieNoTxxHH5oDgM8fYnNxHRt31rChsJZNhXU0+4Ks3VbN2m2tQyWAflmJDCtIZXi/FIYWpJCVEqd5ag+SYRjkTL8Y37at+IuKKFm+hIG/ut7qskRELKHwKyI9wu2yc+jANA4d2HLFMWyaFFU0tlwZ3lnLxsIaymu87CxvZGd5I/9b3TJUIiXR1TpmuGU1ugE5STjsGipxoGxuN3mz5rD9t7fQtPYryp97lvRLL7S6LBGRHqfwKyKWsBkGBVlJFGQl8d0j+wFQ2+BjY2FtZOzwtpJ6ahv9rFpXzqp15QC4HDYG5Xlaplnr1zJ+OCneaeVb6TXc+f3IufBnlDywlIrnnyNcUoSRmY0jOwdXbh6unFxNhyYifZ7Cr4hEjZQkN0eNzOaokdkA+AMhthTXtZlzuNEbZP2OGtbvqIk8Ly8joTUMpzK8IIXstHgNleiA57jv0LxxPbVv/o+qDz/a63F7UjLO3Fxcubm4cvJafs/NxZmVjeHQtwwR6f2i6ky2cOFC3n//fR577LEO99mxYwe33347n3zyCU6nkx/+8Idce+21xMfHR/aZNGkShYWFbZ53+umnc9ddd3Vb7SLS9VxOOyMHpDFywO6hEiWVTW1upCutaqK4suXXW58VA5Cc4GRYvxRGDEjlqEPzyEhyoii8W/b0i0k99lhslWXUbN6Gr7gYf0kJweoqQg31hDbW4924oe2TbDacWdmRMOzKyWsNyXnYk5P1nw0R6TWiJvw+/PDD/OUvf2HChAkd7lNfX8+0adPo378/y5cvxzAMbr/9dubMmcODDz4IQENDA0VFRSxZsoRRo0ZFnhsXp/lFRXo7m2GQn5lIfmYiJ47NB6Cuyc+m1iC8sbCWrcV11DcFWL2hgtUbKnjy9Y047TYG5iUzrF9K5EY6T4LL4ndjHcMwSDz0MNLSJhBf3UgwGAYg7PXiLy3BX1KCv6SYwK6vS0swfT4CpSUESkto/Kzt8WwJCd8IxC2h2Jmdjc0Zu30WkehkefgtLS1l3rx5rFq1isGDB+9z32eeeYaGhgYWLFhAenrL/KJ/+tOfOPnkk1m5ciXjx49n/fr1mKbJuHHj8Hg8PfEWRMRCngQXR47I4sgRWQAEgmG2ldSzobCGTYV1bCqqpbbB3zJsYmctL7c+Lyc9ofUmupaxw3kZCTF/9dIWF0fcwEHEDRzUZrtpmgSrq1vDcHEkHPtLSwhWVhJuasK7eTPezZvbHtAwcGZm4txj+IQrt+Vre0pqzPdbRKxhefj98ssvSUlJ4fnnn2fBggV7DVfY05YtWxgyZEgk+ALk5eWRlpbGRx99xPjx41m3bh1ZWVndEnwdjp65w9zeeie7XXe0t6G+dEy92c3hsHHIoDQOGZSG3W4jOTmODVsr+XpbNet31LBhZy1FFY2UVjVRWtXEO2tahkokxjsZXpDC8IKWccND8j24nHaL3033OdDPjDM7k/jsTDh8dJvtYb8ff2kp/uJifCXFrb+3hONwczOB8nIC5eU0ffF5m+fZ4uIiYdid1xKI3a3B2OZ2d82b7AT9W2qf+tIx9aZ90dwXy8PvpEmTmDRp0n7tm5WVRXl5OaFQCLu95ZtSQ0MDtbW1VFZWArB+/XoSEhKYO3cuq1evJj09nTPPPJOLLroI20GsJGWzGaSlJXb6+Z3h8cR/+04xSH3pmHrTvhGDMxkxODPy5/omP19vrWLt1iq+2lLFhu3VNDYH+HRDBZ9uqADAYTcY2i+VQwenc+igll9pnr43fOrgPzOJkJMGY9quGmeaJoGaGpoLi2guLGz9veVrb2kZYa8X79ateLdu3euIrsxMEgr6Ed8vv/VXy9eujAyMHloRUP+W2qe+dEy9aV809sXy8HsgfvSjH7F48WLuuOMOrrnmGkKhELfccguGYeD3+wHYsGED9fX1nHrqqVx11VWsXLmSu+66i9raWq6++upOv3Y4bFJX19RVb2Wf7HYbHk88dXXNhELhHnnN3kB96Zh607599WVYXjLD8pI5/biBBEOtQyV21rBhRy3rd9ZQ2+Bn3fZq1m2v5tk3NwGQnRrP8P4tV4dH9E8lPysRWy/90X3PfGZc0G8Q7n6DcAOprVvNYBB/WWnLjXZ7XCn2FxcTamzEX1GBv6KCmk/bDi42XC5cObm481qvGOfm4Wr92h7fNd9g9W+pfepLx9Sb9lnRF48nfr+uNPeq8Dtw4EDuu+8+brrpJv76178SFxfHhRdeyOjRo0lqnZvyoYcewufzRf48cuRIGhsbWbRoEXPnzj2oq7+7bgrpKaFQuMdfszdQXzqm3rRvf/oyMCeZgTnJfO+o/pimSXmtl407a9hY2LIqXWF5I2U1zZTVNPPumhIA4t0OhvbzRBbgGJLnwe3qXUMlrPnM2LBn55GQnUfC2G/UU1/fepPd7rHFgZIS/OVlmH4/vh3b8e3YvtcR7SmpbcYUu3Jbbr5zZmR26mqx/i21T33pmHrTvmjsS68KvwAnnXQSb775JuXl5SQnJxMXF8d3vvMdzjzzTACcTidOZ9sJ70eMGEFTUxO1tbWkpWk9exHZN8MwyE6NJzs1nu+MzgOgyRtgU1EdG3bWsqmwls1FLcszf7G5ii82VwEts1H0z0mK3Eg3vCCVtGTrxq/2RvbkZOKTk4kfPrzNdjMUIlBRvvtmu12huKSEUH0dodoammtraF73dZvnGQ4Hzuyc3YE4Z/eNd/bEnh3KJiLRoVeF31WrVvGnP/2JBx98kKyslju7P/roI6qrq/nOd75DOBzme9/7HmeffTazZ8+OPG/NmjVkZmYq+IpIpyXEOTl8SAaHD8kAIBQOs6OsoWU1utYFOKrrfWwrqWdbST2vrdoJQIYnLjKjxPCCFAqykrDZeudQCSsZdjuunFxcObkw9og2j4WaGvGXlLSG4eLIdG2B0pKWIRZFhfiL9r6Z2p6cHLlCvGuFu/iCfMLJg3rmTYmIJaI6/IZCIaqqqiJXeIcOHcqGDRu44447uOyyy9ixYwfXX3895513Hv379wfglFNOYfny5QwaNIhRo0bx/vvvs3z5cubNm2fxuxGRvsRuszEo18OgXA9TxrecfyprvWworIlMq7ajvIHKOi+VX3n58KtSAOJcdobmexjar+XK8JB8D/HuqD4VRz17QiLxQ4YSP2Rom+1mOEywsnKPIRS7rxqHamoI1dfTXF9P84b1bZ632W7HmZXdEor3uFLszM3FnqQFPUR6u6g+4xYXFzN58mTuvPNOzjzzTFJTU1m6dCl33nknp59+OmlpaZx33nltrvJee+21eDwe7r77bkpKSigoKGDevHmcc845Fr4TEYkFGSlxZKTkcuxhuQA0+4JsLq5rDcM1bCqqw+sP8eXWar7cWg2AYUD/rKTI1eFhBSlkeOIUsLqAYbPhzMrCmZVF4ugxbR4Le5vxl5RGgnFg1/zFpSWYfn8kJDd+45i2hMQ2cxY7W8cYO7OysX1jyJ2IRCfDNE3T6iJ6g1AoTFXVN0+D3cPhsJGWlkj1HisvifqyL+pN+6KtL+Gwyc7yht3LM++spbLOu9d+acnuSBAeXpBC/+wk7F08xVe09SZa2G2QGPZRtm4T3sKiSCD2FxcTrKrs+ImGgTMzqyUI77rpLqd1bHFKSq//z4w+Lx1Tb9pnRV/S0xP73mwPIiK9mc1mMCAnmQE5yUwaVwBAdb2PDTtrIoF4e2kD1fU+Pv66jI+/LgPA5bQxJM/DsNYFOIbme0iI01XG7mDYbLgzMklyxBM38rA2j4V9PgJlpXsMn2gJxoGSYsJeL4HyMgLlZbDmGwt6xMe33Gi3awhFXl7LUtA5OdhcWv5ZpKcp/IqIWCgt2c3Rh+Zw9KE5APj8odahEq3TrBXW0uwL8vX2Gr7eXgOAAeRnJe5enrkglawUDZXobja3G3f/Abj7D2iz3TRNQrW1bW+22zUjRUUF4eZmfFu34Nu6pe0BDQNHenrkZrs9rxo70tL19ynSTRR+RUSiiNtl59CBaRw6sGV2mrBpUlTRGBkmsamwlrKaZgrLGyksb+R/nxYBkJLoigyVGFaQwsCcZBxRuKxoX2QYBo7UVBypqSQccmibx8KBAIGyspYgXLrHFeOSEsJNjQQrKwlWVtL05Rdtj9m6oMfuQLx7KIUtru+tNCjSkxR+RUSimM0wKMhKoiArie8e2Q+A2gYfGwtbwvDGwlq2ldRT2+hn1fpyVq0vB8DpsDE4N5lhBamRm+mS4jVUoqfZnE7c/frh7tevzXbTNAk11O+enm3X2OKSYgLl5ftc0MORltY6X3He7pvvcvJw9ODyzyK9mcKviEgvk5Lk5qiR2Rw1MhsAfyDE1tblmXfNOdzoDbJ+Zy3rd9ZGnpeXkRC5OjxyQBpJybqCaBXDMHAke3Ake4gfPqLNY2Yw+I0FPUoiV41D9fUEq6sJVlfT/PXatsd0OHYv4pGTu8ccxrnYE7Sgh8guCr8iIr2cy2lnRP9URvRPBVqGSpRWNUUW4NhQWEtpVRPFlS2/3v68OPLc5AQnqUluUpPcpCW7Wr5OdpMW2eYmKcGJTeNPe4zhcLRe1c0DjmzzWKixca9A7C8pIVBW2rKgR+FO/IU79zqm3eOJXCne86qxMzMLw967luQWOVgKvyIifYzNMMjLSCQvI5ETx+YDUNfkZ1Ph7jC8raSeQDBMfVOA+qYAO8oaOjye3WaQmuTaKxSnJu/xdZKLOJe+pXQ3e2Ii8UOHET90WJvtZjhMoLICf3Hrss97LOwRqq0hVFdHc10dzevXfeOAdly7FvSIjCtu+d2enNyD70yk5+hMJSISAzwJLo4cnsWRw1uWhrfbDRxuF1t2VFFR46WmwUdNvY/qb/xe1xQgFDaprPNRWefb52vEu+17XEXeFYrbBuSUJFeXz1ksLVO0ubKycWVlw5ixbR4LNTfvdbNdoLQYf2npNxb0WN3mebbExMgV6Lj8PMLDBhFITsOWnoXhUHyQ3kufXhGRGGQYBp5EFwNyksnP6Hg8aDAUprbBT02Dj+pdoXhXQK73UdPgp7rBh88fotkXotnXMrSi49cFT2LL8Iq0yBCLPa4qt4bmBLdDU311EXt8PPZBg4kbNLjNdjMcJlhdHQm/LVO0tVw1DlZVEW5sxLtpI95NG6kDynY90WaLLOjhytljUY/cXOye3r+gh/R9Cr8iItIhh93Wumzzvm+Oa/YFIwE58nt9a2hu/XNtg5+waVLb4Ke2wc826js8nsthi4w/Tk1yRa4iR64mt4Zmp0PjVTvLsNlwZmTgzMggcdToNo+Ffb7WBTxaF/IoLSFUXkZzYWHLgh5lpQTKSmnkszbPs8XH777Rbs+xxTk52Jxa0EOig8KviIgctHi3g3i3g7x9XEUOh03qm/ytQyr8ew2x2BWaG71B/MEwZTXNlNU07/N1k+KdHY5H3nUlOVk37B0wm9tN3ICBxA0YCOxeqraqqgFfZVXL8Ini4sjY4kBJCYHKlgU9vFs2492yue0BDQNHRkabQOzKzcOZk4sjLU1Xi6VHKfyKiEiPsNkMUpLcpCS5Ibfj/fyBEDWN/j2GVuz+fVdYrq73EwyFaWgO0NAcYGd5Y4fHs9sMUpJckXCcmrx7DHJa0u4b9+Ld+pb4bVoW9EjDkZrWzoIe/siCHnsOoWhZ0KOJYEUFwYqKvRf0cLvbXiXedfNdTi42t7sn357ECP1LFxGRqOJy2slOjSc7Nb7DfUzTpNEbbHPFuGU8ctvQXNfoJxQ2qarzUfUtN+zFueykJbvJSksgOd6BJ7ElHO855MKT6NLKeR2wOV24+xXg7lfQZrtpmoTq61sW8NgViItbloIOlJdj+nz4tm/Dt33bXsd0pKXvDsQ5u8cWO9K1oId0nsKviIj0OoZhkBTvJCneSUF2Uof7BUNh6hr9ewyt8LcE5V1Xklt/NftCeP2hyFzIHb4ukNwmFLvaTPm260pyYpxu2NvFMAwcHg8OjwdGjGzzmBkMEigvi8xCsecwilBDPcHqKoLVVbD2q7bHdDpxZufgyms7PZszNw97fMf/aRIBhV8REenDHHYb6Z440j37d8NefXMAfxh2ltRRVettMx65pqHlKnJdo5+6Rj/bSju+Yc/psLUE46S9p3yLhOYkNy5nbN+wZzgcuPLyceXl7/VYqKEhsuTzrtXuAiUl+MtKMQOBjhf0SEnZexhFTh7OzEwt6CGAwq+IiEjkhr3+rTd2VVc3EgyG2+wTNk3qmwK7b9L75pRvrSG5oTlAIBimvMZLeY13n6+bGOdoc7Nem/HIrduTE1zYbLF3FdmelER8UjsLeoRCBCoq8Je2huFdy0CXlhCqrSVUW0tzbe1eC3oYDgfOby7o0Tq22J7U8U8PpO9R+BUREdkPNsMgJdFFSqKLgXS8+lkgGGoThmv2mA95z23+YJhGb5BGb5DCfdywZzNabthrO7Si7VXltOTYuWHPsNtx5eTgysmBMW0fCzU1tS7osccqd8XFLcs/BwL4i4vwFxfxzW7bk5JbQ3HbscXOrGwt6NEH6W9URESkCzkddrJS48n6lhv2mnzBPWav2H2z3p438NU1tsyNvGuc8pbijl/X7bK3Lh7i+sZ8yLt/T0nq2zfs2RMSsA8eQtzgIW22m+EwwarKNmOLd101DlZXEWqoJ7SxHu/GDW0PaLPhzMraYxhFXuTKsT05WeO6eymFXxERkR5mGAaJcU4S45z0y+r4R+6hcJi6xkCHU77tusLc7Avi84corWqitKrjG/YAPAnOdm/SaxmX3BKck+KdfSrYGa2r0jkzs0gcfXibx3Yt6BEZU7zHMArT5yNQWkqgtJTGz7+xoEdCAq7cXNx5eTQMHkg4JQN7dg7O7Gwt6BHlFH5FRESilN1mi9wkty8+f2ivJahbVtlrOx45FDapawpQ1xRge2lDh8dz2I02V47TPW7ysz3EOYw24dndB27Y++aCHruYpkmwurp1GEVxm1AcrKwk3NSEd/NmvJs3U/vuu7ufaBg4MzJ3D6NovWLsys3FnpLap/5T0Vsp/IqIiPRybpednPQEctITOtwnbJo0NAfaDq3YNdxij6vK9U0BgiGTilovFbX7vmEvwe1oM+VbZAzyHleTUxJ75w17hmHgTE/HmZ5OwqGHtXks7PcTKCttGTZRVgpV5dRv34G/uJhwczOBinICFeU0fbGmzfNscXE4c3Lb3Gy3ayloLejRcxR+RUREYoDNMPAkuPAkuBiQs68b9sLUNrZdgrq2yU+jN0RZVSNVdS1TwPkDYZp8QZp8QQorOr5hzzAgJXHvccjf/HO8295rroraXC7cBf1xF/SPLP1cXd1IIBAiVFe3xyp3xa1DKkoIVJQT9nrxbduKb9vWvY7pSE9vmZLtG1eMHWlpWtCjiyn8ioiISITTYSMzJZ7MlN037Dm+MQWcaZo0+0Jt5kHea1xy6xVl06T1az/Q8dzILqdtr1DcZlxy69XlaL5hzzAMHCkpOFJSSBh5SJvHzGAQf1kZgV2zUOwxjCLc0ECwqopgVRWs/bLtMV0uXDk5OPeYhWLXMApbnBb06AyFXxERETkghmGQEOcgIc5Bv8zEDvcLh01qG/3fuEmv7ewW1fU+mnxB/IEwpdXNlFY37/O1kxOc31g8xLXH4iEtoTkp3oktyq4iGw4H7vx83PntLOhRX992erZdN9+Vl2H6/fh27MC3Y8dez7OnpO4OxDm7Z6JwZmbqavE+KPyKiIhIt7DZjN037OV1vJ8vEGobkOt336S359XlYKhloZH6pgA7yjq+Yc9uM/YIyHuPR971tdsVHTfs2ZOTiU9OJn748DbbWxb0KN8diEt3XzEO1dURqq2hubaG5nVft3me4XC0LP+8KwzvMc7Yntjxf1ZihcKviIiIWMrttJOTlkBOWsc37Jm7btj7xgIi31w8pK4pQChsUlnnpbJu3zfsxbsdbVbT23Ooxa7fPYlO7BZdRW1Z0KPlhjjGHtHmsVBTI/6S0pZxxXuOLS4taRliUVSIv6hwr2Pak5P3CsSu3FycmVkxs6BHbLxLERER6dUMwyA5wUVygov+2R3PjRwMhalt2H2z3jfHJVe3DrfwBUI0+4I0+4IUV3Y8N7JhgCfR1c545JbQnJkaj9PtxDTN7njbHbInJBI/ZAjxQ9pZ0KOyco8hFLuvGgerqwnV19NcX0/zhvXfOKC9ZUGP1lkoIleNc3OxJ/WtBT0UfkVERKTPcNhtZKTEkZESt8/9mn3Bb0z5tscMF63baxtaVtirbfBT2+Bna8k+bthz2Dq8SW/Pq8lOR/deRTZaV6VzZmWROLrt+s9hrzdyhbhlXHFrQC4twfT7CZSUECgp2Wv5Z1tCYpub7ZytK961LOjh7Nb30x0UfkVERCTmxLsdxLsd5GXs+4a9+ib/7nHIHayy19gcwB8MU1bdTNm33LCXFO9sMx45LXmP4Rat25MSuueGPVtcHHEDBxE3cFCb7WY4TLCmevf0bK2B2F9STLCqinBTI97Nm/Bu3tT2gIaBMzOrZdhE6/CJXUtB2zPSurz+rqLwKyIiItIOm80gJclNSpIbctvfx+GwkZAUx5btVVTWetteSW4zLtlPMBSmoTlAQ3OAneXfdsOeq808yKltxiW3hOY4V9fEOMNmw5megTM9Aw4b1eaxsN9PoLR09zCK4pbxxYGSYsJeL4HyMgLlZbDm8zbPs8XHkzxiODmXzYCEjoepWEHhV0REROQguJ0tK+xleDoeamGaJo3eYJtxyLvHI+++ia+u0d96w56PyjrfPl833m1vne6t7U16kRkuktykJLkO6oY9m8uFu39/3P377/V+QrW1bW+227W4R0U54eZmaj/7nLSyMlyDFH5FREREYophGCTFO0mKd1LwLTfs1TXuccPeN8LxrvHIXn+IZl+IZl/Tvm/Yo+WGvbZXkF1t50pOdpMY5zigm9oMw8CRmoojNZWEQw5t81g4EMCsqiA5yY0/JZNgMLzfx+0JCr8iIiIiUcJht5HuiSN9H1eRYfcNe5Erx+2MR65taLmKXNvop7bRz7Z9rLDndNhahlN8c3W9NstQu3A6vn1uZJvTiaNfPxLTEvFXd7z0tVUUfkVERER6mf26Yc9sWRSk/SnfWm7iq2nw0dAcIBAMU17jpbxm33MjJ8Y5vhGId/++a8hFcqKrq99ul1L4FREREemDbIZBSqKLlEQXA0nucL9AMNTu4iGRba1/DgTDNHqDNHqD7Czv+Iqu3WaQkuRiSL9UfvaDkSS4oytuRlc1IiIiItKjnA47WanxZKXGd7iPaZo0+YK7l51uHW5Rs8fsFtV73LBXVeejqq6UHx7dn8F5nh58N99O4VdERERE9skwDBLjnCTGOSnI6viGvVA4TF1jgLomP2mpCaQlOAiFenb1u2+j8CsiIiIiXcJus5GW7CYrLZ60tESqqxuB6Aq/3bvGnoiIiIhIFFH4FREREZGYofArIiIiIjFD4VdEREREYobCr4iIiIjEDIVfEREREYkZCr8iIiIiEjMUfkVEREQkZij8ioiIiEjMUPgVERERkZih8CsiIiIiMUPhV0RERERihsKviIiIiMQMhV8RERERiRkKvyIiIiISMxR+RURERCRmKPyKiIiISMwwTNM0rS6iNzBNk3C451plt9sIhcI99nq9hfrSMfWmfepLx9Sb9qkv7VNfOqbetK+n+2KzGRiG8a37KfyKiIiISMzQsAcRERERiRkKvyIiIiISMxR+RURERCRmKPyKiIiISMxQ+BURERGRmKHwKyIiIiIxQ+FXRERERGKGwq+IiIiIxAyFXxERERGJGQq/IiIiIhIzFH5FREREJGYo/IqIiIhIzFD4FREREZGYofDbzcLhMH/5y1844YQTGDt2LJdeeinbtm3rcP/q6mquvfZaJkyYwIQJE7jxxhtpampqs89LL73EqaeeyuGHH87pp5/OW2+91d1vo1scaG82bNjAjBkzOOaYYzjuuOP4+c9/TlFRUZt9Jk2axMiRI9v8uu6667r7rXSpA+3LM888s9d7HjlyZJvnxOJn5r777mu3LyNHjuQ3v/lNZL++8JnZ08KFC7nwwgv3uU8snWf2tD+9iZXzzJ72py+xdJ7Z07f1JpbOMzU1Ndx0002ceOKJjBs3jmnTprFy5coO94/q84wp3eq+++4zjzvuOPN///ufuXbtWvPSSy81p0yZYvp8vnb3nz59unn22WebX3zxhfnee++ZJ598snn99ddHHn///ffNUaNGmY899pi5ceNG83e/+505evRoc+PGjT31lrrMgfSmqqrKPP74481f/OIX5vr16801a9aY06dPN3/4wx+aXq/XNE3TrK+vN0eOHGn+97//NcvKyiK/6urqevqtHZQD/czceeed5vTp09u857KyMjMYDJqmGbufmYaGhr16snDhQnPMmDHm2rVrTdPsO5+ZXR566CFz5MiR5vTp0/e5XyydZ3bZn97E0nlml/39zMTSeWaX/elNLJ1nLrnkEvPHP/6x+fHHH5ubNm0yb7vtNnPMmDEd/h1H83lG4bcb+Xw+88gjjzRXrFgR2VZbW2uOGTPGfOGFF/ba/5NPPjFHjBjR5i/+7bffNkeOHGmWlJSYpmmal156qfmLX/yizfPOPfdc88Ybb+ymd9E9DrQ3f//7381x48ZFvgGZpmkWFxebI0aMMN977z3TNE1z1apV5ogRI8za2trufwPd5ED7YpotJ6Tf/va3HR4zVj8z37Rt2zZz7NixbZ7fFz4zpmmaJSUl5mWXXWYeccQR5g9+8IN9frOOpfOMaR5Yb2LlPGOaB9YX04yd84xpHnhv9tRXzzNbt241R4wYYa5atSqyLRwOm1OmTDHvvffevfaP9vOMhj10o6+//prGxkaOPfbYyDaPx8Nhhx3Gxx9/vNf+K1euJCsri6FDh0a2HX300RiGwapVqwiHw3zyySdtjgdwzDHH7PNHD9HoQHtz3HHHsWDBAtxu916P1dbWArBu3TqysrLweDzdV3g3O9C+QMv7HjZsWLuPxfJn5pt+97vfMXz4cM4999zItr7wmQH48ssvSUlJ4fnnn2fs2LH73DeWzjNwYL2JlfMMHFhfIHbOM3DgvdlTXz3PpKWlsXTpUkaPHh3ZZhgGpmlG/m3sKdrPM45uf4UYVlJSAkBeXl6b7dnZ2RQXF++1f2lp6V77ulwuUlNTKS4upq6ujqamJnJzc/freNHsQHtTUFBAQUFBm21LlizB7XYzYcIEANavX09CQgJz585l9erVpKenc+aZZ3LRRRdhs/WO/+cdaF+qqqqoqKjg448/5rHHHqOmpoaxY8dy3XXXMXjw4Jj+zOxpzZo1vP766zzyyCNtPgt94TMDLeMJJ02atF/7xtJ5Bg6sN7FynoED60ssnWfgwHqzp758nvF4PJx00klttr300kts376diRMn7rV/tJ9nekfXe6nm5mag5S98T263G5/P1+7+39x3z/29Xu8BHS+aHWhvvunRRx9lxYoVXHPNNWRkZAAtN6rU19dz6qmn8sADD3Duuefy5z//mfvuu6/r30A3OdC+rF+/HgC73c7vf/97/vSnP9HU1MT5559PRUWFPjOtHn74YcaOHbvXVYa+8Jk5ULF0njlYffU8c6Bi6TxzMGLpPLNq1SpuuOEGJk+e3O5/FKL9PKMrv90oLi4OAL/fH/kawOfzER8f3+7+fr9/r+0+n4+EhITIj+K+uU9Hx4tmB9qbXUzT5M9//jOLFi1i5syZ/OxnP4s89tBDD+Hz+UhKSgJg5MiRNDY2smjRIubOndsr/od9oH059thj+eijj0hJSYlsW7BgASeffDL//Oc/OfvssyPH21MsfWaampp49dVXmT9//l6P9YXPzIGKpfNMZ/X188yBiqXzTGfF0nnmtdde47rrrmPs2LHcc8897e4T7eeZ3tXxXmbXJf+ysrI228vKyva61A+Qm5u7175+v5+amhpycnJITU0lISFhv48XzQ60NwCBQIBf/epXLF68mOuvv55rrrmmzeNOpzNyctllxIgRNDU1tTsmKRp1pi97fkMCSEhIoKCggNLS0pj/zAC8/fbbhMNhpkyZstdjfeEzc6Bi6TzTGbFwnumMWDnPdFasnGcef/xx5s6dy4knnsiyZcvaXIjYU7SfZxR+u9EhhxxCUlISH374YWRbXV0dX331FePHj99r/wkTJlBSUtJm3sRdzx03bhyGYTBu3Dg++uijNs/78MMPOeqoo7rpXXSPA+0NwPXXX8/LL7/M3XffzWWXXdbmsXA4zKRJk1i0aFGb7WvWrCEzM5O0tLSufxPd4ED7smLFCo455pjIj5AAGhoa2Lp1K8OGDYv5zwy0/Hhu1KhRe91s0lc+Mwcqls4znREL55kDFUvnmc6KhfPMihUruO2227jgggu499572x3WsEu0n2c07KEbuVwupk+fzl133UV6ejr9+vXjj3/8I7m5uUyZMoVQKERVVRXJycnExcUxduxYxo0bxy9/+UtuvvlmmpqamD9/PmeccQY5OTkAXHLJJcyYMYPDDjuME088kaeffpq1a9dy++23W/xuD8yB9uaf//wnL774Itdffz1HH3005eXlkWPt2ueUU05h+fLlDBo0iFGjRvH++++zfPly5s2bZ+E7PTAH2peTTz6Ze++9l+uvv565c+fi9Xq55557SE9PZ+rUqUDsfmZ2+frrrxkxYsRex7PZbH3iM/NtYvk8821i9TzzbWL5PPNtYvU8s2XLFu644w6mTJnCzJkzqaysjDwWFxdHQkJC7zrPdPtkajEuGAyaf/jDH8xjjz3WPOKII8wrrrjC3LFjh2maprljxw5zxIgR5tNPPx3Zv6Kiwpw7d655xBFHmMccc4w5f/78NnNOmqZpPvPMM+aUKVPMww8/3Jw6dWpk/sne5kB6c8kll5gjRoxo99eufQKBgLlw4UJz8uTJ5qhRo8xTTjnFfPLJJy17f511oJ+Zr776yrz00kvNo446yhw3bpw5d+5cs6ioqM0xY/Ezs8sPf/hD86677mr3eH3lM7OnX//6123mJY3188yevq03sXSe2dP+fGZi6Tyzp/3pjWn2/fPMokWLOvy38etf/7rXnWcM0zTN7o/YIiIiIiLW05hfEREREYkZCr8iIiIiEjMUfkVEREQkZij8ioiIiEjMUPgVERERkZih8CsiIiIiMUPhV0RERERihsKviIiIiMQMLW8sIhLF/u///o9nnnmmw8dTU1P58MMPe7AiGDlyJFdddRVz587t0dcVEekKCr8iIlEuKyuL+++/v93HHA6dxkVEDoTOmiIiUc7lcnHEEUdYXYaISJ+g8Csi0gdceOGF9OvXj8GDB/Poo4/S3NzMMcccww033ED//v0j+61Zs4Z7772XL774gkAgwNFHH821117L8OHDI/tUVlZy991387///Y/m5mYOO+wwrrnmGo466qjIPg0NDcybN49XX32VQCDACSecwPz588nIyOjR9y0icqB0w5uISC8QDAbb/WWaZmSf119/naeffpp58+Zx66238vXXX3PRRRfR1NQEwAcffMC0adMIh8Pcfvvt/Pa3v6W4uJjzzjuPTZs2AdDU1MR5553He++9x7XXXsv9999PYmIil19+eWQfgEcffZRAIMCf//xnfvnLX/LGG29wyy239GxTREQ6QVd+RUSiXGFhIaNGjWr3sauvvporr7wSaAmuTz/9NAMGDABgyJAhTJ06lWeeeYYLLriAu+++m/79+7N8+XLsdjsAEydOZMqUKdx3333ce++9PPPMM+zYsYNnn32WQw45BIDx48dzxhln8PHHHzN06FAADj/8cP7whz8AcNxxx/H555/z1ltvdWsfRES6gsKviEiUy8rKYtGiRe0+lpOTE/n6yCOPjARfgMMOO4z+/fuzcuVKpk6dypo1a5gzZ04k+AJ4PB5OPvlk3nzzTQBWrlxJQUFBJPgCuN1uXnrppTavu+cQCID+/ftTV1fX+TcpItJDFH5FRKKcy+Xi8MMP/9b9srOz99qWkZFBXV0d9fX1mKZJZmbmXvtkZmZSX18PQE1NzX6N201ISGjzZ5vN1mYIhohItNKYXxGRPqKmpmavbRUVFaSnp5OcnIxhGFRUVOy1T3l5OampqQAkJydTVVW11z6rV69mw4YNXV2yiEiPU/gVEekjVq9e3Sa4fvnll+zcuZPjjjuOhIQERo8ezYsvvkgoFIrsU19fz//+97/IMIbx48ezY8cO1q1bF9nH7/czd+5c/v73v/fcmxER6SYa9iAiEuX8fj+ffvpph4+PGDECgObmZq644gpmz55NY2Mjf/rTnxgxYgSnnXYaANdeey2XXXYZl19+OdOnTycQCLB06VL8fj9XXXUVAGeeeSaPPfYYs2fP5uqrryY9PZ2//vWveL1eLrzwwm5/ryIi3U3hV0QkypWXl3Puued2+PhTTz0FtFy1PfbYY5k3bx4AkyZN4vrrr8flcgEtszI89NBD/OUvf+Gaa67B5XIxfvx4fv/730fm+U1KSuLxxx/nD3/4A7fffjvBYJCxY8fy2GOPtbmZTkSktzJM3aEgItLr7boq+9hjj1lciYhIdNOYXxERERGJGQq/IiIiIhIzNOxBRERERGKGrvyKiIiISMxQ+BURERGRmKHwKyIiIiIxQ+FXRERERGKGwq+IiIiIxAyFXxERERGJGQq/IiIiIhIzFH5FREREJGb8f970S0m/LNgqAAAAAElFTkSuQmCC"
1661 |           },
1662 |           "metadata": {}
1663 |         }
1664 |       ],
1665 |       "source": [
1666 |         "plt.figure(figsize=(8, 6))\n",
1667 |         "\n",
1668 |         "plt.plot(range(len(loss_train_hist)), loss_train_hist, 'r-', label='Train')\n",
1669 |         "plt.plot(range(len(loss_valid_hist)), loss_valid_hist, 'b-', label='Validation')\n",
1670 |         "\n",
1671 |         "plt.xlabel('Epoch')\n",
1672 |         "plt.ylabel('loss')\n",
1673 |         "plt.grid(True)\n",
1674 |         "plt.legend();"
1675 |       ]
1676 |     }
1677 |   ]
1678 | }


--------------------------------------------------------------------------------
/08_Evaluation/Evaluation.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |   "nbformat": 4,
   3 |   "nbformat_minor": 0,
   4 |   "metadata": {
   5 |     "colab": {
   6 |       "provenance": [],
   7 |       "collapsed_sections": [
   8 |         "VrFCH9hpEyip",
   9 |         "3in1e9BksgIh",
  10 |         "d7Psbf21OXiW",
  11 |         "w_a3OXnSeV0z",
  12 |         "1GJWSuUqlhnu",
  13 |         "FnWk37Hbf1ru",
  14 |         "Eaf14FzFf1rv",
  15 |         "o_5f69nwPtY2",
  16 |         "gn6_jAdJe6wR",
  17 |         "lAoYRsvKN_8l",
  18 |         "LfdeZV28aamI",
  19 |         "mtdWrvT5vcdd",
  20 |         "6I2iyKGHdTAl",
  21 |         "ZlmhmSzLf1ru",
  22 |         "2I75f5ObfOk7",
  23 |         "LKLbB4fikXz_"
  24 |       ],
  25 |       "authorship_tag": "ABX9TyNiAmVxV7eELYcLA6f7c/Ht",
  26 |       "include_colab_link": true
  27 |     },
  28 |     "kernelspec": {
  29 |       "name": "python3",
  30 |       "display_name": "Python 3"
  31 |     },
  32 |     "language_info": {
  33 |       "name": "python"
  34 |     },
  35 |     "widgets": {
  36 |       "application/vnd.jupyter.widget-state+json": {
  37 |         "2042d8a3d97c4afe82cd6e767de74058": {
  38 |           "model_module": "@jupyter-widgets/controls",
  39 |           "model_name": "HBoxModel",
  40 |           "model_module_version": "2.0.0",
  41 |           "state": {
  42 |             "_dom_classes": [],
  43 |             "_model_module": "@jupyter-widgets/controls",
  44 |             "_model_module_version": "2.0.0",
  45 |             "_model_name": "HBoxModel",
  46 |             "_view_count": null,
  47 |             "_view_module": "@jupyter-widgets/controls",
  48 |             "_view_module_version": "2.0.0",
  49 |             "_view_name": "HBoxView",
  50 |             "box_style": "",
  51 |             "children": [
  52 |               "IPY_MODEL_76bd70def6b1496db513e64c57cc4c29",
  53 |               "IPY_MODEL_84f62a340d5c44588656de5ff1274bba",
  54 |               "IPY_MODEL_8540dc77dd334a39af9452e60e554825"
  55 |             ],
  56 |             "layout": "IPY_MODEL_a1319920d408492eb172385d02045064",
  57 |             "tabbable": null,
  58 |             "tooltip": null
  59 |           }
  60 |         },
  61 |         "76bd70def6b1496db513e64c57cc4c29": {
  62 |           "model_module": "@jupyter-widgets/controls",
  63 |           "model_name": "HTMLModel",
  64 |           "model_module_version": "2.0.0",
  65 |           "state": {
  66 |             "_dom_classes": [],
  67 |             "_model_module": "@jupyter-widgets/controls",
  68 |             "_model_module_version": "2.0.0",
  69 |             "_model_name": "HTMLModel",
  70 |             "_view_count": null,
  71 |             "_view_module": "@jupyter-widgets/controls",
  72 |             "_view_module_version": "2.0.0",
  73 |             "_view_name": "HTMLView",
  74 |             "description": "",
  75 |             "description_allow_html": false,
  76 |             "layout": "IPY_MODEL_dc8e7c9d886d4f4fa68514602ffbf170",
  77 |             "placeholder": "​",
  78 |             "style": "IPY_MODEL_960d613b9a184aeea9cb2fc8cef4ce2e",
  79 |             "tabbable": null,
  80 |             "tooltip": null,
  81 |             "value": "Generating: 100%"
  82 |           }
  83 |         },
  84 |         "84f62a340d5c44588656de5ff1274bba": {
  85 |           "model_module": "@jupyter-widgets/controls",
  86 |           "model_name": "FloatProgressModel",
  87 |           "model_module_version": "2.0.0",
  88 |           "state": {
  89 |             "_dom_classes": [],
  90 |             "_model_module": "@jupyter-widgets/controls",
  91 |             "_model_module_version": "2.0.0",
  92 |             "_model_name": "FloatProgressModel",
  93 |             "_view_count": null,
  94 |             "_view_module": "@jupyter-widgets/controls",
  95 |             "_view_module_version": "2.0.0",
  96 |             "_view_name": "ProgressView",
  97 |             "bar_style": "success",
  98 |             "description": "",
  99 |             "description_allow_html": false,
 100 |             "layout": "IPY_MODEL_be2c8eb4d84a4141ae50a3e4da72b898",
 101 |             "max": 100,
 102 |             "min": 0,
 103 |             "orientation": "horizontal",
 104 |             "style": "IPY_MODEL_df89141e1d9440e8880d54f5648e8e96",
 105 |             "tabbable": null,
 106 |             "tooltip": null,
 107 |             "value": 100
 108 |           }
 109 |         },
 110 |         "8540dc77dd334a39af9452e60e554825": {
 111 |           "model_module": "@jupyter-widgets/controls",
 112 |           "model_name": "HTMLModel",
 113 |           "model_module_version": "2.0.0",
 114 |           "state": {
 115 |             "_dom_classes": [],
 116 |             "_model_module": "@jupyter-widgets/controls",
 117 |             "_model_module_version": "2.0.0",
 118 |             "_model_name": "HTMLModel",
 119 |             "_view_count": null,
 120 |             "_view_module": "@jupyter-widgets/controls",
 121 |             "_view_module_version": "2.0.0",
 122 |             "_view_name": "HTMLView",
 123 |             "description": "",
 124 |             "description_allow_html": false,
 125 |             "layout": "IPY_MODEL_ae7ea84d05614bf9a01fd97b9f44ffc3",
 126 |             "placeholder": "​",
 127 |             "style": "IPY_MODEL_06eda88a825143de9594a9dd40c2d661",
 128 |             "tabbable": null,
 129 |             "tooltip": null,
 130 |             "value": " 100/100 [03:10&lt;00:00,  1.98s/it]"
 131 |           }
 132 |         },
 133 |         "a1319920d408492eb172385d02045064": {
 134 |           "model_module": "@jupyter-widgets/base",
 135 |           "model_name": "LayoutModel",
 136 |           "model_module_version": "2.0.0",
 137 |           "state": {
 138 |             "_model_module": "@jupyter-widgets/base",
 139 |             "_model_module_version": "2.0.0",
 140 |             "_model_name": "LayoutModel",
 141 |             "_view_count": null,
 142 |             "_view_module": "@jupyter-widgets/base",
 143 |             "_view_module_version": "2.0.0",
 144 |             "_view_name": "LayoutView",
 145 |             "align_content": null,
 146 |             "align_items": null,
 147 |             "align_self": null,
 148 |             "border_bottom": null,
 149 |             "border_left": null,
 150 |             "border_right": null,
 151 |             "border_top": null,
 152 |             "bottom": null,
 153 |             "display": null,
 154 |             "flex": null,
 155 |             "flex_flow": null,
 156 |             "grid_area": null,
 157 |             "grid_auto_columns": null,
 158 |             "grid_auto_flow": null,
 159 |             "grid_auto_rows": null,
 160 |             "grid_column": null,
 161 |             "grid_gap": null,
 162 |             "grid_row": null,
 163 |             "grid_template_areas": null,
 164 |             "grid_template_columns": null,
 165 |             "grid_template_rows": null,
 166 |             "height": null,
 167 |             "justify_content": null,
 168 |             "justify_items": null,
 169 |             "left": null,
 170 |             "margin": null,
 171 |             "max_height": null,
 172 |             "max_width": null,
 173 |             "min_height": null,
 174 |             "min_width": null,
 175 |             "object_fit": null,
 176 |             "object_position": null,
 177 |             "order": null,
 178 |             "overflow": null,
 179 |             "padding": null,
 180 |             "right": null,
 181 |             "top": null,
 182 |             "visibility": null,
 183 |             "width": null
 184 |           }
 185 |         },
 186 |         "dc8e7c9d886d4f4fa68514602ffbf170": {
 187 |           "model_module": "@jupyter-widgets/base",
 188 |           "model_name": "LayoutModel",
 189 |           "model_module_version": "2.0.0",
 190 |           "state": {
 191 |             "_model_module": "@jupyter-widgets/base",
 192 |             "_model_module_version": "2.0.0",
 193 |             "_model_name": "LayoutModel",
 194 |             "_view_count": null,
 195 |             "_view_module": "@jupyter-widgets/base",
 196 |             "_view_module_version": "2.0.0",
 197 |             "_view_name": "LayoutView",
 198 |             "align_content": null,
 199 |             "align_items": null,
 200 |             "align_self": null,
 201 |             "border_bottom": null,
 202 |             "border_left": null,
 203 |             "border_right": null,
 204 |             "border_top": null,
 205 |             "bottom": null,
 206 |             "display": null,
 207 |             "flex": null,
 208 |             "flex_flow": null,
 209 |             "grid_area": null,
 210 |             "grid_auto_columns": null,
 211 |             "grid_auto_flow": null,
 212 |             "grid_auto_rows": null,
 213 |             "grid_column": null,
 214 |             "grid_gap": null,
 215 |             "grid_row": null,
 216 |             "grid_template_areas": null,
 217 |             "grid_template_columns": null,
 218 |             "grid_template_rows": null,
 219 |             "height": null,
 220 |             "justify_content": null,
 221 |             "justify_items": null,
 222 |             "left": null,
 223 |             "margin": null,
 224 |             "max_height": null,
 225 |             "max_width": null,
 226 |             "min_height": null,
 227 |             "min_width": null,
 228 |             "object_fit": null,
 229 |             "object_position": null,
 230 |             "order": null,
 231 |             "overflow": null,
 232 |             "padding": null,
 233 |             "right": null,
 234 |             "top": null,
 235 |             "visibility": null,
 236 |             "width": null
 237 |           }
 238 |         },
 239 |         "960d613b9a184aeea9cb2fc8cef4ce2e": {
 240 |           "model_module": "@jupyter-widgets/controls",
 241 |           "model_name": "HTMLStyleModel",
 242 |           "model_module_version": "2.0.0",
 243 |           "state": {
 244 |             "_model_module": "@jupyter-widgets/controls",
 245 |             "_model_module_version": "2.0.0",
 246 |             "_model_name": "HTMLStyleModel",
 247 |             "_view_count": null,
 248 |             "_view_module": "@jupyter-widgets/base",
 249 |             "_view_module_version": "2.0.0",
 250 |             "_view_name": "StyleView",
 251 |             "background": null,
 252 |             "description_width": "",
 253 |             "font_size": null,
 254 |             "text_color": null
 255 |           }
 256 |         },
 257 |         "be2c8eb4d84a4141ae50a3e4da72b898": {
 258 |           "model_module": "@jupyter-widgets/base",
 259 |           "model_name": "LayoutModel",
 260 |           "model_module_version": "2.0.0",
 261 |           "state": {
 262 |             "_model_module": "@jupyter-widgets/base",
 263 |             "_model_module_version": "2.0.0",
 264 |             "_model_name": "LayoutModel",
 265 |             "_view_count": null,
 266 |             "_view_module": "@jupyter-widgets/base",
 267 |             "_view_module_version": "2.0.0",
 268 |             "_view_name": "LayoutView",
 269 |             "align_content": null,
 270 |             "align_items": null,
 271 |             "align_self": null,
 272 |             "border_bottom": null,
 273 |             "border_left": null,
 274 |             "border_right": null,
 275 |             "border_top": null,
 276 |             "bottom": null,
 277 |             "display": null,
 278 |             "flex": null,
 279 |             "flex_flow": null,
 280 |             "grid_area": null,
 281 |             "grid_auto_columns": null,
 282 |             "grid_auto_flow": null,
 283 |             "grid_auto_rows": null,
 284 |             "grid_column": null,
 285 |             "grid_gap": null,
 286 |             "grid_row": null,
 287 |             "grid_template_areas": null,
 288 |             "grid_template_columns": null,
 289 |             "grid_template_rows": null,
 290 |             "height": null,
 291 |             "justify_content": null,
 292 |             "justify_items": null,
 293 |             "left": null,
 294 |             "margin": null,
 295 |             "max_height": null,
 296 |             "max_width": null,
 297 |             "min_height": null,
 298 |             "min_width": null,
 299 |             "object_fit": null,
 300 |             "object_position": null,
 301 |             "order": null,
 302 |             "overflow": null,
 303 |             "padding": null,
 304 |             "right": null,
 305 |             "top": null,
 306 |             "visibility": null,
 307 |             "width": null
 308 |           }
 309 |         },
 310 |         "df89141e1d9440e8880d54f5648e8e96": {
 311 |           "model_module": "@jupyter-widgets/controls",
 312 |           "model_name": "ProgressStyleModel",
 313 |           "model_module_version": "2.0.0",
 314 |           "state": {
 315 |             "_model_module": "@jupyter-widgets/controls",
 316 |             "_model_module_version": "2.0.0",
 317 |             "_model_name": "ProgressStyleModel",
 318 |             "_view_count": null,
 319 |             "_view_module": "@jupyter-widgets/base",
 320 |             "_view_module_version": "2.0.0",
 321 |             "_view_name": "StyleView",
 322 |             "bar_color": null,
 323 |             "description_width": ""
 324 |           }
 325 |         },
 326 |         "ae7ea84d05614bf9a01fd97b9f44ffc3": {
 327 |           "model_module": "@jupyter-widgets/base",
 328 |           "model_name": "LayoutModel",
 329 |           "model_module_version": "2.0.0",
 330 |           "state": {
 331 |             "_model_module": "@jupyter-widgets/base",
 332 |             "_model_module_version": "2.0.0",
 333 |             "_model_name": "LayoutModel",
 334 |             "_view_count": null,
 335 |             "_view_module": "@jupyter-widgets/base",
 336 |             "_view_module_version": "2.0.0",
 337 |             "_view_name": "LayoutView",
 338 |             "align_content": null,
 339 |             "align_items": null,
 340 |             "align_self": null,
 341 |             "border_bottom": null,
 342 |             "border_left": null,
 343 |             "border_right": null,
 344 |             "border_top": null,
 345 |             "bottom": null,
 346 |             "display": null,
 347 |             "flex": null,
 348 |             "flex_flow": null,
 349 |             "grid_area": null,
 350 |             "grid_auto_columns": null,
 351 |             "grid_auto_flow": null,
 352 |             "grid_auto_rows": null,
 353 |             "grid_column": null,
 354 |             "grid_gap": null,
 355 |             "grid_row": null,
 356 |             "grid_template_areas": null,
 357 |             "grid_template_columns": null,
 358 |             "grid_template_rows": null,
 359 |             "height": null,
 360 |             "justify_content": null,
 361 |             "justify_items": null,
 362 |             "left": null,
 363 |             "margin": null,
 364 |             "max_height": null,
 365 |             "max_width": null,
 366 |             "min_height": null,
 367 |             "min_width": null,
 368 |             "object_fit": null,
 369 |             "object_position": null,
 370 |             "order": null,
 371 |             "overflow": null,
 372 |             "padding": null,
 373 |             "right": null,
 374 |             "top": null,
 375 |             "visibility": null,
 376 |             "width": null
 377 |           }
 378 |         },
 379 |         "06eda88a825143de9594a9dd40c2d661": {
 380 |           "model_module": "@jupyter-widgets/controls",
 381 |           "model_name": "HTMLStyleModel",
 382 |           "model_module_version": "2.0.0",
 383 |           "state": {
 384 |             "_model_module": "@jupyter-widgets/controls",
 385 |             "_model_module_version": "2.0.0",
 386 |             "_model_name": "HTMLStyleModel",
 387 |             "_view_count": null,
 388 |             "_view_module": "@jupyter-widgets/base",
 389 |             "_view_module_version": "2.0.0",
 390 |             "_view_name": "StyleView",
 391 |             "background": null,
 392 |             "description_width": "",
 393 |             "font_size": null,
 394 |             "text_color": null
 395 |           }
 396 |         }
 397 |       }
 398 |     }
 399 |   },
 400 |   "cells": [
 401 |     {
 402 |       "cell_type": "markdown",
 403 |       "metadata": {
 404 |         "id": "view-in-github",
 405 |         "colab_type": "text"
 406 |       },
 407 |       "source": [
 408 |         "<a href=\"https://colab.research.google.com/github/howsam/Building-a-ChatGPT-like-Model-from-Scratch/blob/main/Evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
 409 |       ]
 410 |     },
 411 |     {
 412 |       "cell_type": "markdown",
 413 |       "metadata": {
 414 |         "id": "-uXkcYhkIxS-"
 415 |       },
 416 |       "source": [
 417 |         "#  <font color='#FFE15D'><b>💎 Evaluation </b></font><font color='#FF0B55'></font>"
 418 |       ]
 419 |     },
 420 |     {
 421 |       "cell_type": "markdown",
 422 |       "metadata": {
 423 |         "id": "VrFCH9hpEyip",
 424 |         "jp-MarkdownHeadingCollapsed": true
 425 |       },
 426 |       "source": [
 427 |         "# 🔴 **Environment Setup**"
 428 |       ]
 429 |     },
 430 |     {
 431 |       "cell_type": "markdown",
 432 |       "metadata": {
 433 |         "id": "3in1e9BksgIh",
 434 |         "jp-MarkdownHeadingCollapsed": true
 435 |       },
 436 |       "source": [
 437 |         "## 🟠 Change the font size of the output cells"
 438 |       ]
 439 |     },
 440 |     {
 441 |       "cell_type": "code",
 442 |       "execution_count": 16,
 443 |       "metadata": {
 444 |         "id": "1_nYkVog8SUR",
 445 |         "colab": {
 446 |           "base_uri": "https://localhost:8080/"
 447 |         },
 448 |         "outputId": "a2625a11-bc54-4e43-fe1c-d68edf44cc65"
 449 |       },
 450 |       "outputs": [
 451 |         {
 452 |           "output_type": "stream",
 453 |           "name": "stdout",
 454 |           "text": [
 455 |             "Salam Howsam!\n"
 456 |           ]
 457 |         }
 458 |       ],
 459 |       "source": [
 460 |         "print('Salam Howsam!')"
 461 |       ]
 462 |     },
 463 |     {
 464 |       "cell_type": "code",
 465 |       "execution_count": 17,
 466 |       "metadata": {
 467 |         "id": "BmMM0EfKsSiO"
 468 |       },
 469 |       "outputs": [],
 470 |       "source": [
 471 |         "from IPython.display import HTML\n",
 472 |         "shell = get_ipython()\n",
 473 |         "\n",
 474 |         "def adjust_font_size():\n",
 475 |         "  display(HTML('''<style>\n",
 476 |         "    body {\n",
 477 |         "      font-size: 20px;\n",
 478 |         "    }\n",
 479 |         "  '''))\n",
 480 |         "\n",
 481 |         "if adjust_font_size not in shell.events.callbacks['pre_execute']:\n",
 482 |         "  shell.events.register('pre_execute', adjust_font_size)"
 483 |       ]
 484 |     },
 485 |     {
 486 |       "cell_type": "code",
 487 |       "execution_count": 18,
 488 |       "metadata": {
 489 |         "id": "10N1yUE88XRW",
 490 |         "colab": {
 491 |           "base_uri": "https://localhost:8080/",
 492 |           "height": 0
 493 |         },
 494 |         "outputId": "715b7354-5c76-4085-ae8f-f8d5a5a23ba9"
 495 |       },
 496 |       "outputs": [
 497 |         {
 498 |           "output_type": "display_data",
 499 |           "data": {
 500 |             "text/plain": [
 501 |               "<IPython.core.display.HTML object>"
 502 |             ],
 503 |             "text/html": [
 504 |               "<style>\n",
 505 |               "    body {\n",
 506 |               "      font-size: 20px;\n",
 507 |               "    }\n",
 508 |               "  "
 509 |             ]
 510 |           },
 511 |           "metadata": {}
 512 |         },
 513 |         {
 514 |           "output_type": "stream",
 515 |           "name": "stdout",
 516 |           "text": [
 517 |             "Salam Howsam!\n"
 518 |           ]
 519 |         }
 520 |       ],
 521 |       "source": [
 522 |         "print('Salam Howsam!')"
 523 |       ]
 524 |     },
 525 |     {
 526 |       "cell_type": "markdown",
 527 |       "metadata": {
 528 |         "jp-MarkdownHeadingCollapsed": true,
 529 |         "id": "d7Psbf21OXiW"
 530 |       },
 531 |       "source": [
 532 |         "## 🟠 `pip`"
 533 |       ]
 534 |     },
 535 |     {
 536 |       "cell_type": "code",
 537 |       "execution_count": 19,
 538 |       "metadata": {
 539 |         "id": "jv21KFRNOXiX",
 540 |         "colab": {
 541 |           "base_uri": "https://localhost:8080/",
 542 |           "height": 0
 543 |         },
 544 |         "outputId": "4480ed4a-f0f6-4309-8527-3ed985ec9aa3"
 545 |       },
 546 |       "outputs": [
 547 |         {
 548 |           "output_type": "display_data",
 549 |           "data": {
 550 |             "text/plain": [
 551 |               "<IPython.core.display.HTML object>"
 552 |             ],
 553 |             "text/html": [
 554 |               "<style>\n",
 555 |               "    body {\n",
 556 |               "      font-size: 20px;\n",
 557 |               "    }\n",
 558 |               "  "
 559 |             ]
 560 |           },
 561 |           "metadata": {}
 562 |         },
 563 |         {
 564 |           "output_type": "stream",
 565 |           "name": "stdout",
 566 |           "text": [
 567 |             "Requirement already satisfied: openai in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (1.59.9)\n",
 568 |             "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (4.2.0)\n",
 569 |             "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (1.9.0)\n",
 570 |             "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (0.27.2)\n",
 571 |             "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (0.8.2)\n",
 572 |             "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (2.10.5)\n",
 573 |             "Requirement already satisfied: sniffio in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (1.3.0)\n",
 574 |             "Requirement already satisfied: tqdm>4 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (4.66.4)\n",
 575 |             "Requirement already satisfied: typing-extensions<5,>=4.11 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from openai) (4.12.2)\n",
 576 |             "Requirement already satisfied: idna>=2.8 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
 577 |             "Requirement already satisfied: certifi in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\n",
 578 |             "Requirement already satisfied: httpcore==1.* in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from httpx<1,>=0.23.0->openai) (1.0.7)\n",
 579 |             "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
 580 |             "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
 581 |             "Requirement already satisfied: pydantic-core==2.27.2 in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (2.27.2)\n",
 582 |             "Requirement already satisfied: colorama in c:\\users\\howsam\\anaconda3\\envs\\torch-howsam-free\\lib\\site-packages (from tqdm>4->openai) (0.4.6)\n"
 583 |           ]
 584 |         }
 585 |       ],
 586 |       "source": [
 587 |         "!pip install openai"
 588 |       ]
 589 |     },
 590 |     {
 591 |       "cell_type": "markdown",
 592 |       "metadata": {
 593 |         "id": "w_a3OXnSeV0z",
 594 |         "jp-MarkdownHeadingCollapsed": true
 595 |       },
 596 |       "source": [
 597 |         "# 🔴 **Import**"
 598 |       ]
 599 |     },
 600 |     {
 601 |       "cell_type": "code",
 602 |       "execution_count": 1,
 603 |       "metadata": {
 604 |         "id": "vhlVJEkJeTsV"
 605 |       },
 606 |       "outputs": [],
 607 |       "source": [
 608 |         "import os\n",
 609 |         "import io\n",
 610 |         "import re\n",
 611 |         "import math\n",
 612 |         "import time\n",
 613 |         "import yaml\n",
 614 |         "import json\n",
 615 |         "import random\n",
 616 |         "import pathlib\n",
 617 |         "import requests\n",
 618 |         "from pprint import pprint\n",
 619 |         "from tqdm.auto import tqdm\n",
 620 |         "from collections import Counter\n",
 621 |         "from dataclasses import dataclass\n",
 622 |         "\n",
 623 |         "import numpy as np\n",
 624 |         "import pandas as pd\n",
 625 |         "import matplotlib.pyplot as plt\n",
 626 |         "\n",
 627 |         "import torch\n",
 628 |         "import torch.nn as nn\n",
 629 |         "import torch.nn.functional as F\n",
 630 |         "\n",
 631 |         "from datasets import load_dataset\n",
 632 |         "from tokenizers import Tokenizer\n",
 633 |         "\n",
 634 |         "from openai import OpenAI"
 635 |       ]
 636 |     },
 637 |     {
 638 |       "cell_type": "markdown",
 639 |       "source": [
 640 |         "# 🔴 **Functions**"
 641 |       ],
 642 |       "metadata": {
 643 |         "id": "1GJWSuUqlhnu"
 644 |       }
 645 |     },
 646 |     {
 647 |       "cell_type": "markdown",
 648 |       "source": [
 649 |         "## 🟠 Model"
 650 |       ],
 651 |       "metadata": {
 652 |         "id": "FnWk37Hbf1ru"
 653 |       }
 654 |     },
 655 |     {
 656 |       "cell_type": "code",
 657 |       "source": [
 658 |         "class MultiHeadAttention(nn.Module):\n",
 659 |         "\n",
 660 |         "    def __init__(self, config):\n",
 661 |         "        super().__init__()\n",
 662 |         "        self.n_embd = config.n_embd\n",
 663 |         "        self.n_head = config.n_head\n",
 664 |         "        self.head_size = self.n_embd // self.n_head\n",
 665 |         "\n",
 666 |         "        self.qkv_proj = nn.Linear(self.n_embd, 3*self.n_embd, bias=False)\n",
 667 |         "        self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)\n",
 668 |         "        self.c_proj.residual = True\n",
 669 |         "\n",
 670 |         "    def forward(self, x):\n",
 671 |         "        B, T, C = x.shape\n",
 672 |         "        # QKV linear\n",
 673 |         "        q, k, v = self.qkv_proj(x).view(B, T, 3*self.n_head, self.head_size).transpose(1, 2).chunk(3, dim=-3)\n",
 674 |         "        # Scaled Dot Product Attention using pytorch\n",
 675 |         "        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)\n",
 676 |         "        # Reshape and final projection\n",
 677 |         "        y = y.transpose(1, 2).contiguous().view(B, T, C)\n",
 678 |         "        y = self.c_proj(y)\n",
 679 |         "        return y"
 680 |       ],
 681 |       "metadata": {
 682 |         "id": "HwVunU8Hf1ru"
 683 |       },
 684 |       "execution_count": 2,
 685 |       "outputs": []
 686 |     },
 687 |     {
 688 |       "cell_type": "code",
 689 |       "source": [
 690 |         "class FeedForward(nn.Module):\n",
 691 |         "\n",
 692 |         "    def __init__(self, config):\n",
 693 |         "        super().__init__()\n",
 694 |         "        self.n_embd = config.n_embd\n",
 695 |         "        self.f_expnd = config.f_expnd\n",
 696 |         "\n",
 697 |         "        self.up_proj = nn.Linear(self.n_embd, int(self.f_expnd*self.n_embd), bias=False)\n",
 698 |         "        self.down_proj = nn.Linear(int(self.f_expnd*self.n_embd), self.n_embd, bias=False)\n",
 699 |         "        self.down_proj.residual = True\n",
 700 |         "\n",
 701 |         "    def forward(self, x):\n",
 702 |         "        return self.down_proj(F.gelu(self.up_proj(x)))"
 703 |       ],
 704 |       "metadata": {
 705 |         "id": "h3Sz7ydwf1ru"
 706 |       },
 707 |       "execution_count": 3,
 708 |       "outputs": []
 709 |     },
 710 |     {
 711 |       "cell_type": "code",
 712 |       "source": [
 713 |         "class DecoderBlock(nn.Module):\n",
 714 |         "\n",
 715 |         "    def __init__(self, config):\n",
 716 |         "        super().__init__()\n",
 717 |         "        self.n_embd = config.n_embd\n",
 718 |         "        # Multi Head Attention\n",
 719 |         "        self.ln1 = nn.LayerNorm(config.n_embd)\n",
 720 |         "        self.mha = MultiHeadAttention(config)\n",
 721 |         "        # Feed Forward Neural Network\n",
 722 |         "        self.ln2 = nn.LayerNorm(config.n_embd)\n",
 723 |         "        self.mlp = FeedForward(config)\n",
 724 |         "\n",
 725 |         "    def forward(self, x):\n",
 726 |         "        x = x + self.mha(self.ln1(x))\n",
 727 |         "        x = x + self.mlp(self.ln2(x))\n",
 728 |         "        return x"
 729 |       ],
 730 |       "metadata": {
 731 |         "id": "LzGmGKdvf1ru"
 732 |       },
 733 |       "execution_count": 4,
 734 |       "outputs": []
 735 |     },
 736 |     {
 737 |       "cell_type": "code",
 738 |       "source": [
 739 |         "class GPT(nn.Module):\n",
 740 |         "\n",
 741 |         "    def __init__(self, config):\n",
 742 |         "        super().__init__()\n",
 743 |         "        self.config = config\n",
 744 |         "        self.wte = nn.Embedding(config.vocab_size, config.n_embd) # Token embedding\n",
 745 |         "        self.wpe = nn.Embedding(config.max_seq_len, config.n_embd) # Position embedding\n",
 746 |         "        self.decoders = nn.ModuleList([DecoderBlock(config) for _ in range(config.n_layer)]) # Decoders\n",
 747 |         "        self.lnf = nn.LayerNorm(config.n_embd)\n",
 748 |         "        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) # Classifier\n",
 749 |         "        self.lm_head.weight = self.wte.weight # Weight tying\n",
 750 |         "\n",
 751 |         "        self.apply(self._init_weights)\n",
 752 |         "\n",
 753 |         "    def _init_weights(self, module):\n",
 754 |         "        std = 0.02\n",
 755 |         "        if isinstance(module, nn.Linear):\n",
 756 |         "            if hasattr(module, 'residual'):\n",
 757 |         "                std *= (2*self.config.n_layer)**-0.5\n",
 758 |         "            nn.init.normal_(module.weight, mean=0.0, std=std)\n",
 759 |         "            if module.bias is not None:\n",
 760 |         "                nn.init.zeros_(module.bias)\n",
 761 |         "        elif isinstance(module, nn.Embedding):\n",
 762 |         "            nn.init.normal_(module.weight, mean=0.0, std=std)\n",
 763 |         "\n",
 764 |         "    def forward(self, idx):\n",
 765 |         "        B, T = idx.shape\n",
 766 |         "        # Token Embedding + Position Embedding\n",
 767 |         "        x = self.wte(idx) + self.wpe(torch.arange(T, device=idx.device))\n",
 768 |         "        # Decoders\n",
 769 |         "        for decoder in self.decoders:\n",
 770 |         "            x = decoder(x)\n",
 771 |         "        # Classifier\n",
 772 |         "        x = self.lnf(x)\n",
 773 |         "        logits = self.lm_head(x)\n",
 774 |         "        return logits"
 775 |       ],
 776 |       "metadata": {
 777 |         "id": "7oOprH8hf1ru"
 778 |       },
 779 |       "execution_count": 5,
 780 |       "outputs": []
 781 |     },
 782 |     {
 783 |       "cell_type": "code",
 784 |       "source": [
 785 |         "@dataclass\n",
 786 |         "class GPTConfig:\n",
 787 |         "    vocab_size: int = 50257\n",
 788 |         "    max_seq_len: int = 1024\n",
 789 |         "    n_layer: int = 12\n",
 790 |         "    n_head: int = 12\n",
 791 |         "    n_embd: int = 768\n",
 792 |         "    f_expnd: int = 4"
 793 |       ],
 794 |       "metadata": {
 795 |         "id": "fQm-H53yiEP4"
 796 |       },
 797 |       "execution_count": 6,
 798 |       "outputs": []
 799 |     },
 800 |     {
 801 |       "cell_type": "markdown",
 802 |       "source": [
 803 |         "## 🟠 Generate"
 804 |       ],
 805 |       "metadata": {
 806 |         "id": "Eaf14FzFf1rv"
 807 |       }
 808 |     },
 809 |     {
 810 |       "cell_type": "code",
 811 |       "source": [
 812 |         "def generate(model, tokenizer, prompt, n_rep=5, max_seq_len=128, T=0.9, top_k=10, device='cuda', seed=42):\n",
 813 |         "    # Get the token ID for <|endoftext|>\n",
 814 |         "    eot_token_id = tokenizer.encode(\"<|endoftext|>\").ids[0]\n",
 815 |         "\n",
 816 |         "    # Tokenize the prompt and convert it to a tensor on the specified device (e.g., GPU)\n",
 817 |         "    inputs = torch.tensor(tokenizer.encode(prompt).ids, dtype=torch.int, device=device)  # Shape: [T]\n",
 818 |         "    n = inputs.shape[0]\n",
 819 |         "\n",
 820 |         "    # Repeat the input prompt n_rep times to generate multiple sequences in parallel\n",
 821 |         "    inputs = inputs.unsqueeze(0).repeat(n_rep, 1)  # Shape: [B, T] where B = n_rep\n",
 822 |         "\n",
 823 |         "    # Set the model to evaluation mode\n",
 824 |         "    model.eval()\n",
 825 |         "\n",
 826 |         "    # Initialize a random number generator for sampling\n",
 827 |         "    sample_rng = torch.Generator(device=device)\n",
 828 |         "    sample_rng.manual_seed(seed)\n",
 829 |         "\n",
 830 |         "    # Track which sequences are still active\n",
 831 |         "    is_finished = torch.zeros(n_rep, dtype=torch.bool, device=device)\n",
 832 |         "\n",
 833 |         "    # Disable gradient calculation for faster inference\n",
 834 |         "    with torch.no_grad():\n",
 835 |         "        # Continue generating tokens until reaching the maximum sequence length\n",
 836 |         "        while inputs.shape[-1] < (max_seq_len + n):\n",
 837 |         "            # Forward pass: get logits from the model\n",
 838 |         "            logits = model(inputs)  # Shape: [B, T, vocab_size]\n",
 839 |         "\n",
 840 |         "            # Apply temperature scaling and softmax to get probabilities for the next token\n",
 841 |         "            probs = torch.softmax(logits[:, -1, :] / T, dim=-1)  # Shape: [B, vocab_size]\n",
 842 |         "\n",
 843 |         "            # Select the top_k tokens with the highest probabilities\n",
 844 |         "            topk_probs, topk_indices = torch.topk(probs, k=top_k, dim=-1)  # Shape: [B, top_k]\n",
 845 |         "\n",
 846 |         "            # Sample one token from the top_k candidates based on their probabilities\n",
 847 |         "            sampled = torch.multinomial(topk_probs, 1, generator=sample_rng)  # Shape: [B, 1]\n",
 848 |         "\n",
 849 |         "            # Map the sampled indices back to the original token IDs\n",
 850 |         "            next_token = torch.gather(topk_indices, -1, sampled).squeeze(-1) # Shape: [B]\n",
 851 |         "\n",
 852 |         "            # For finished sequences, force pad with eot_token_id again to avoid changing inputs\n",
 853 |         "            next_token = torch.where(is_finished, torch.tensor(eot_token_id, device=device), next_token)\n",
 854 |         "\n",
 855 |         "            # Update finished mask\n",
 856 |         "            is_finished = is_finished | (next_token == eot_token_id)\n",
 857 |         "\n",
 858 |         "            # Append the sampled tokens to the input sequence\n",
 859 |         "            inputs = torch.cat((inputs, next_token.unsqueeze(-1)), dim=-1)  # Shape: [B, T+1]\n",
 860 |         "\n",
 861 |         "    # Decode the generated sequences back into text\n",
 862 |         "    generated_text = tokenizer.decode_batch(inputs.tolist())\n",
 863 |         "\n",
 864 |         "    # Slice off the prompt part, keep only the generated continuation\n",
 865 |         "    only_generated = [gen[len(prompt):] for gen in generated_text]\n",
 866 |         "\n",
 867 |         "    return only_generated"
 868 |       ],
 869 |       "metadata": {
 870 |         "id": "K0mV0Ksnf1rv"
 871 |       },
 872 |       "execution_count": 7,
 873 |       "outputs": []
 874 |     },
 875 |     {
 876 |       "cell_type": "markdown",
 877 |       "metadata": {
 878 |         "id": "o_5f69nwPtY2"
 879 |       },
 880 |       "source": [
 881 |         "# 🔴 **GPT-Eval**"
 882 |       ]
 883 |     },
 884 |     {
 885 |       "cell_type": "markdown",
 886 |       "metadata": {
 887 |         "id": "gn6_jAdJe6wR"
 888 |       },
 889 |       "source": [
 890 |         "## 🟠 Config"
 891 |       ]
 892 |     },
 893 |     {
 894 |       "cell_type": "code",
 895 |       "source": [
 896 |         "@dataclass\n",
 897 |         "class GeneralConfig:\n",
 898 |         "    tokenizer_path: str\n",
 899 |         "    model_path: str\n",
 900 |         "    llm_eval_name: str\n",
 901 |         "    api_key: str\n",
 902 |         "    prompts_yaml_url: str\n",
 903 |         "    n_samples: int = 10\n",
 904 |         "    max_new_tokens: int = 128\n",
 905 |         "    temperature: float = 1.0\n",
 906 |         "    device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\""
 907 |       ],
 908 |       "metadata": {
 909 |         "id": "oIkx-mzbztIg"
 910 |       },
 911 |       "execution_count": null,
 912 |       "outputs": []
 913 |     },
 914 |     {
 915 |       "cell_type": "markdown",
 916 |       "source": [
 917 |         "## 🟠 LLM Evaluator"
 918 |       ],
 919 |       "metadata": {
 920 |         "id": "lAoYRsvKN_8l"
 921 |       }
 922 |     },
 923 |     {
 924 |       "cell_type": "code",
 925 |       "source": [
 926 |         "def deepseekv3_grade(prompt: str, completion: str):\n",
 927 |         "    \"\"\"\n",
 928 |         "    Ask DeepSeek-V3 (model 'deepseek-chat') to grade a TinyStories completion.\n",
 929 |         "    Returns a list in the order [grammar, creativity, consistency, plot].\n",
 930 |         "    \"\"\"\n",
 931 |         "    delim = \"***\"\n",
 932 |         "    full_text = f\"{prompt.strip()} {delim} {completion.strip()}\"\n",
 933 |         "\n",
 934 |         "    system_msg = (\n",
 935 |         "        \"You are an English teacher grading a student's story completion. \"\n",
 936 |         "        \"Return ONLY a JSON object with keys grammar, creativity, consistency, plot \"\n",
 937 |         "        \"— each an integer 0-10.\"\n",
 938 |         "    )\n",
 939 |         "\n",
 940 |         "    user_msg = (\n",
 941 |         "        \"Here is the story. The part after the three asterisks (***) is the student's \"\n",
 942 |         "        \"completion. Evaluate and respond ONLY with the JSON described.\\n\\n\"\n",
 943 |         "        f\"{full_text}\"\n",
 944 |         "    )\n",
 945 |         "\n",
 946 |         "    response = client.chat.completions.create(\n",
 947 |         "        model = \"deepseek-chat\",\n",
 948 |         "        messages = [\n",
 949 |         "            {\"role\": \"system\", \"content\": system_msg},\n",
 950 |         "            {\"role\": \"user\",   \"content\": user_msg}\n",
 951 |         "        ],\n",
 952 |         "        temperature = 0  # deterministic grading\n",
 953 |         "    )\n",
 954 |         "\n",
 955 |         "    # Extract JSON part (remove ```json ... ```)\n",
 956 |         "    json_str = re.search(r\"\\{.*?\\}\", response.choices[0].message.content, re.DOTALL).group(0)\n",
 957 |         "\n",
 958 |         "    # Parse and reorder the values\n",
 959 |         "    grades_dict = json.loads(json_str)\n",
 960 |         "    return [grades_dict[k] for k in [\"grammar\", \"creativity\", \"consistency\", \"plot\"]]"
 961 |       ],
 962 |       "metadata": {
 963 |         "id": "DWStWyB1JxVS"
 964 |       },
 965 |       "execution_count": null,
 966 |       "outputs": []
 967 |     },
 968 |     {
 969 |       "cell_type": "markdown",
 970 |       "metadata": {
 971 |         "id": "LfdeZV28aamI"
 972 |       },
 973 |       "source": [
 974 |         "## 🟠 Evaluation"
 975 |       ]
 976 |     },
 977 |     {
 978 |       "cell_type": "code",
 979 |       "source": [
 980 |         "# 0: General config\n",
 981 |         "\n",
 982 |         "gen_cfg = GeneralConfig(\n",
 983 |         "    tokenizer_path = 'data/bpe-tokenizer_tinystories.json',\n",
 984 |         "    model_path = 'weights/model_hdim-512_layer-8_lossv129.pt',\n",
 985 |         "    llm_eval_name = 'deepseek-chat',\n",
 986 |         "    api_key = open(\"others/deepseek_api_key.txt\", \"r\").read().strip(),\n",
 987 |         "    max_new_tokens = 256,\n",
 988 |         "    prompts_yaml_url = (\n",
 989 |         "        \"https://huggingface.co/datasets/roneneldan/TinyStories/\"\n",
 990 |         "        \"resolve/main/Evaluation%20prompts.yaml\"\n",
 991 |         "    )\n",
 992 |         ")"
 993 |       ],
 994 |       "metadata": {
 995 |         "id": "cEWHIa-haIg7"
 996 |       },
 997 |       "execution_count": null,
 998 |       "outputs": []
 999 |     },
1000 |     {
1001 |       "cell_type": "code",
1002 |       "source": [
1003 |         "# 1: Load prompts\n",
1004 |         "data = requests.get(gen_cfg.prompts_yaml_url, timeout=30).text\n",
1005 |         "prompts = yaml.safe_load(io.StringIO(data))\n",
1006 |         "type(prompts), len(prompts)"
1007 |       ],
1008 |       "metadata": {
1009 |         "id": "fbSSF6_sfNRI"
1010 |       },
1011 |       "execution_count": null,
1012 |       "outputs": []
1013 |     },
1014 |     {
1015 |       "cell_type": "code",
1016 |       "source": [
1017 |         "prompts[-1]"
1018 |       ],
1019 |       "metadata": {
1020 |         "id": "mPYFf7hicIIn"
1021 |       },
1022 |       "execution_count": null,
1023 |       "outputs": []
1024 |     },
1025 |     {
1026 |       "cell_type": "code",
1027 |       "source": [
1028 |         "# 2: Load tokenizer\n",
1029 |         "tokenizer = Tokenizer.from_file(gen_cfg.tokenizer_path)"
1030 |       ],
1031 |       "metadata": {
1032 |         "id": "f6LVfiFnbt2O"
1033 |       },
1034 |       "execution_count": null,
1035 |       "outputs": []
1036 |     },
1037 |     {
1038 |       "cell_type": "code",
1039 |       "source": [
1040 |         "# 3: Load Pretrained GPT model\n",
1041 |         "\n",
1042 |         "gpt_cfg = GPTConfig(\n",
1043 |         "    vocab_size=10_000,\n",
1044 |         "    max_seq_len=1024,\n",
1045 |         "    n_layer=8,\n",
1046 |         "    n_head=16,\n",
1047 |         "    n_embd=512,\n",
1048 |         "    f_expnd=4\n",
1049 |         ")\n",
1050 |         "\n",
1051 |         "model = GPT(gpt_cfg)\n",
1052 |         "model.load_state_dict(torch.load(gen_cfg.model_path))\n",
1053 |         "model.to(gen_cfg.device).eval()"
1054 |       ],
1055 |       "metadata": {
1056 |         "id": "bQ_TnnfC-k00"
1057 |       },
1058 |       "execution_count": null,
1059 |       "outputs": []
1060 |     },
1061 |     {
1062 |       "cell_type": "code",
1063 |       "source": [
1064 |         "# 4: Call DeepSeek API\n",
1065 |         "\n",
1066 |         "client = OpenAI(api_key=gen_cfg.api_key, base_url=\"https://api.deepseek.com\")"
1067 |       ],
1068 |       "metadata": {
1069 |         "id": "pJjMmmn_wtPH"
1070 |       },
1071 |       "execution_count": null,
1072 |       "outputs": []
1073 |     },
1074 |     {
1075 |       "cell_type": "code",
1076 |       "source": [
1077 |         "# 5: Evaluate the quality of student GPT completions using the LLM teacher as a reference\n",
1078 |         "\n",
1079 |         "results = []\n",
1080 |         "\n",
1081 |         "for prompt_id, prompt in enumerate(tqdm(prompts, desc=\"Prompts\")):\n",
1082 |         "    # Generate\n",
1083 |         "    completions = generate(\n",
1084 |         "        model, tokenizer, prompt,\n",
1085 |         "        n_rep=gen_cfg.n_samples, max_seq_len=256,\n",
1086 |         "        T=gen_cfg.temperature, top_k=10\n",
1087 |         "    )\n",
1088 |         "\n",
1089 |         "    prompt_scores = []\n",
1090 |         "    for i, completion in enumerate(completions):\n",
1091 |         "        # Grade\n",
1092 |         "        grades = deepseekv3_grade(prompt, completion)  # [grammar, ...]\n",
1093 |         "        prompt_scores.append(grades)\n",
1094 |         "        time.sleep(1.2)  # polite rate-limit; tweak as needed\n",
1095 |         "    prompt_scores = np.array(prompt_scores)\n",
1096 |         "    mean_scores   = prompt_scores.mean(axis=0)\n",
1097 |         "\n",
1098 |         "    results.append({\n",
1099 |         "        \"prompt_id\": prompt_id,\n",
1100 |         "        \"prompt\": prompt,\n",
1101 |         "        \"completions\": completions,\n",
1102 |         "        \"grammar\": mean_scores[0],\n",
1103 |         "        \"creativity\":mean_scores[1],\n",
1104 |         "        \"consistency\":mean_scores[2],\n",
1105 |         "        \"plot\": mean_scores[3]\n",
1106 |         "    })"
1107 |       ],
1108 |       "metadata": {
1109 |         "id": "xkBiKNZQM_Ph"
1110 |       },
1111 |       "execution_count": null,
1112 |       "outputs": []
1113 |     },
1114 |     {
1115 |       "cell_type": "code",
1116 |       "source": [
1117 |         "df = pd.DataFrame(results)\n",
1118 |         "overall = df.mean(numeric_only=True)\n",
1119 |         "\n",
1120 |         "display(df.head())\n",
1121 |         "display(overall)"
1122 |       ],
1123 |       "metadata": {
1124 |         "id": "VZOn8KieKmv_"
1125 |       },
1126 |       "execution_count": null,
1127 |       "outputs": []
1128 |     },
1129 |     {
1130 |       "cell_type": "code",
1131 |       "source": [
1132 |         "prompt_scores"
1133 |       ],
1134 |       "metadata": {
1135 |         "id": "vpTXRbJk4-0u"
1136 |       },
1137 |       "execution_count": null,
1138 |       "outputs": []
1139 |     },
1140 |     {
1141 |       "cell_type": "code",
1142 |       "source": [
1143 |         "pprint(prompt)\n",
1144 |         "print()\n",
1145 |         "pprint(completions[1])\n",
1146 |         "print()\n",
1147 |         "print(tokenizer.encode(completions[1]))"
1148 |       ],
1149 |       "metadata": {
1150 |         "id": "vKGM-dTzqAWv"
1151 |       },
1152 |       "execution_count": null,
1153 |       "outputs": []
1154 |     },
1155 |     {
1156 |       "cell_type": "code",
1157 |       "source": [
1158 |         "torch.cuda.empty_cache()"
1159 |       ],
1160 |       "metadata": {
1161 |         "id": "kp0hFdApoSH7"
1162 |       },
1163 |       "execution_count": null,
1164 |       "outputs": []
1165 |     },
1166 |     {
1167 |       "cell_type": "markdown",
1168 |       "metadata": {
1169 |         "id": "mtdWrvT5vcdd"
1170 |       },
1171 |       "source": [
1172 |         "# 🔴 **ROUGE**"
1173 |       ]
1174 |     },
1175 |     {
1176 |       "cell_type": "markdown",
1177 |       "metadata": {
1178 |         "id": "6I2iyKGHdTAl"
1179 |       },
1180 |       "source": [
1181 |         "## 🟠 Config"
1182 |       ]
1183 |     },
1184 |     {
1185 |       "cell_type": "code",
1186 |       "source": [
1187 |         "@dataclass\n",
1188 |         "class GeneralConfig:\n",
1189 |         "    tokenizer_path: str\n",
1190 |         "    model_path: str\n",
1191 |         "    n_examples: int = 100\n",
1192 |         "    split_ratio: float = 0.4  # keep first 40 % as prompt\n",
1193 |         "    max_new_tokens: int = 256\n",
1194 |         "    n_samples: int = 10\n",
1195 |         "    temperature: float = 1.0\n",
1196 |         "    device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
1197 |         "    random_seed: int = 42\n",
1198 |         "    k_values = [2, 4, 5]  # n-gram sizes to analyse"
1199 |       ],
1200 |       "metadata": {
1201 |         "id": "hdbxMBFnwMSB"
1202 |       },
1203 |       "execution_count": 8,
1204 |       "outputs": []
1205 |     },
1206 |     {
1207 |       "cell_type": "markdown",
1208 |       "source": [
1209 |         "## 🟠 ROUGE"
1210 |       ],
1211 |       "metadata": {
1212 |         "id": "ZlmhmSzLf1ru"
1213 |       }
1214 |     },
1215 |     {
1216 |       "cell_type": "code",
1217 |       "source": [
1218 |         "def rouge_k_precision(t1, t2, k=2):\n",
1219 |         "    # Build k-gram sets\n",
1220 |         "    grams1 = set(zip(*[t1.split()[i:] for i in range(k)]))\n",
1221 |         "    grams2 = set(zip(*[t2.split()[i:] for i in range(k)]))\n",
1222 |         "    if not grams1: return 0.0\n",
1223 |         "    overlap = grams1 & grams2\n",
1224 |         "    return len(overlap) / len(grams1)"
1225 |       ],
1226 |       "metadata": {
1227 |         "id": "HghzRpZSf1rv"
1228 |       },
1229 |       "execution_count": 9,
1230 |       "outputs": []
1231 |     },
1232 |     {
1233 |       "cell_type": "code",
1234 |       "source": [
1235 |         "t1 = \"the cat sat on the mat\"\n",
1236 |         "t2 = \"the cat lay on the mat\"\n",
1237 |         "# rouge_k_precision(t1, t2, k=2)\n",
1238 |         "\n",
1239 |         "k = 2\n",
1240 |         "grams1 = set(zip(*[t1.split()[i:] for i in range(k)]))\n",
1241 |         "grams2 = set(zip(*[t2.split()[i:] for i in range(k)]))\n",
1242 |         "print(grams1)\n",
1243 |         "print(grams2)\n",
1244 |         "\n",
1245 |         "overlap = grams1 & grams2\n",
1246 |         "\n",
1247 |         "len(overlap) / len(grams1)"
1248 |       ],
1249 |       "metadata": {
1250 |         "colab": {
1251 |           "base_uri": "https://localhost:8080/",
1252 |           "height": 94
1253 |         },
1254 |         "id": "Pft4292Fj_3x",
1255 |         "outputId": "a267ff44-0527-4005-9718-1b2edb5d6d33"
1256 |       },
1257 |       "execution_count": 21,
1258 |       "outputs": [
1259 |         {
1260 |           "output_type": "display_data",
1261 |           "data": {
1262 |             "text/plain": [
1263 |               "<IPython.core.display.HTML object>"
1264 |             ],
1265 |             "text/html": [
1266 |               "<style>\n",
1267 |               "    body {\n",
1268 |               "      font-size: 20px;\n",
1269 |               "    }\n",
1270 |               "  "
1271 |             ]
1272 |           },
1273 |           "metadata": {}
1274 |         },
1275 |         {
1276 |           "output_type": "stream",
1277 |           "name": "stdout",
1278 |           "text": [
1279 |             "{('the', 'mat'), ('sat', 'on'), ('cat', 'sat'), ('on', 'the'), ('the', 'cat')}\n",
1280 |             "{('the', 'mat'), ('the', 'cat'), ('cat', 'lay'), ('on', 'the'), ('lay', 'on')}\n"
1281 |           ]
1282 |         },
1283 |         {
1284 |           "output_type": "execute_result",
1285 |           "data": {
1286 |             "text/plain": [
1287 |               "0.6"
1288 |             ]
1289 |           },
1290 |           "metadata": {},
1291 |           "execution_count": 21
1292 |         }
1293 |       ]
1294 |     },
1295 |     {
1296 |       "cell_type": "code",
1297 |       "source": [
1298 |         "def rouge_k_fmeasure(t1, t2, k=2):\n",
1299 |         "    p = rouge_k_precision(t1, t2, k)\n",
1300 |         "    r = rouge_k_precision(t2, t1, k)\n",
1301 |         "    if p + r == 0: return 0.0\n",
1302 |         "    return 2 * p * r / (p + r)"
1303 |       ],
1304 |       "metadata": {
1305 |         "id": "9HSfhcB6h4r9",
1306 |         "colab": {
1307 |           "base_uri": "https://localhost:8080/",
1308 |           "height": 17
1309 |         },
1310 |         "outputId": "294da6ef-4dbd-4575-e892-70ed3ff03cff"
1311 |       },
1312 |       "execution_count": 22,
1313 |       "outputs": [
1314 |         {
1315 |           "output_type": "display_data",
1316 |           "data": {
1317 |             "text/plain": [
1318 |               "<IPython.core.display.HTML object>"
1319 |             ],
1320 |             "text/html": [
1321 |               "<style>\n",
1322 |               "    body {\n",
1323 |               "      font-size: 20px;\n",
1324 |               "    }\n",
1325 |               "  "
1326 |             ]
1327 |           },
1328 |           "metadata": {}
1329 |         }
1330 |       ]
1331 |     },
1332 |     {
1333 |       "cell_type": "code",
1334 |       "source": [
1335 |         "def corpus_kgram_freq(dataset_texts, k):\n",
1336 |         "    counter = Counter()\n",
1337 |         "    for txt in tqdm(dataset_texts, desc=f\"Counting {k}-grams\"):\n",
1338 |         "        counter.update(zip(*[txt.split()[i:] for i in range(k)]))\n",
1339 |         "    return counter"
1340 |       ],
1341 |       "metadata": {
1342 |         "id": "dfvAOdiXiYkP",
1343 |         "colab": {
1344 |           "base_uri": "https://localhost:8080/",
1345 |           "height": 17
1346 |         },
1347 |         "outputId": "57572bea-053b-44f5-8b1b-ef68b37e14a8"
1348 |       },
1349 |       "execution_count": 23,
1350 |       "outputs": [
1351 |         {
1352 |           "output_type": "display_data",
1353 |           "data": {
1354 |             "text/plain": [
1355 |               "<IPython.core.display.HTML object>"
1356 |             ],
1357 |             "text/html": [
1358 |               "<style>\n",
1359 |               "    body {\n",
1360 |               "      font-size: 20px;\n",
1361 |               "    }\n",
1362 |               "  "
1363 |             ]
1364 |           },
1365 |           "metadata": {}
1366 |         }
1367 |       ]
1368 |     },
1369 |     {
1370 |       "cell_type": "code",
1371 |       "source": [
1372 |         "def closest_story_score(gen, corpus_texts):\n",
1373 |         "    return max(rouge_k_precision(gen, story, k=2) for story in corpus_texts)"
1374 |       ],
1375 |       "metadata": {
1376 |         "id": "GHyR5UwKicNI",
1377 |         "colab": {
1378 |           "base_uri": "https://localhost:8080/",
1379 |           "height": 17
1380 |         },
1381 |         "outputId": "3703758e-1eee-43e9-bf8d-48e81a377923"
1382 |       },
1383 |       "execution_count": 24,
1384 |       "outputs": [
1385 |         {
1386 |           "output_type": "display_data",
1387 |           "data": {
1388 |             "text/plain": [
1389 |               "<IPython.core.display.HTML object>"
1390 |             ],
1391 |             "text/html": [
1392 |               "<style>\n",
1393 |               "    body {\n",
1394 |               "      font-size: 20px;\n",
1395 |               "    }\n",
1396 |               "  "
1397 |             ]
1398 |           },
1399 |           "metadata": {}
1400 |         }
1401 |       ]
1402 |     },
1403 |     {
1404 |       "cell_type": "markdown",
1405 |       "metadata": {
1406 |         "id": "2I75f5ObfOk7"
1407 |       },
1408 |       "source": [
1409 |         "## 🟠 Evaluation"
1410 |       ]
1411 |     },
1412 |     {
1413 |       "cell_type": "code",
1414 |       "source": [
1415 |         "# 0: General config\n",
1416 |         "\n",
1417 |         "gen_cfg = GeneralConfig(\n",
1418 |         "    tokenizer_path = 'data/bpe-tokenizer_tinystories.json',\n",
1419 |         "    model_path = 'weights/model_hdim-512_layer-8_lossv129.pt',\n",
1420 |         "    n_samples = 1\n",
1421 |         ")\n",
1422 |         "\n",
1423 |         "random.seed(gen_cfg.random_seed)"
1424 |       ],
1425 |       "metadata": {
1426 |         "id": "fSLoET_-eAUN",
1427 |         "colab": {
1428 |           "base_uri": "https://localhost:8080/",
1429 |           "height": 17
1430 |         },
1431 |         "outputId": "8cfc4245-c8eb-4adf-f9c6-308c99478400"
1432 |       },
1433 |       "execution_count": 25,
1434 |       "outputs": [
1435 |         {
1436 |           "output_type": "display_data",
1437 |           "data": {
1438 |             "text/plain": [
1439 |               "<IPython.core.display.HTML object>"
1440 |             ],
1441 |             "text/html": [
1442 |               "<style>\n",
1443 |               "    body {\n",
1444 |               "      font-size: 20px;\n",
1445 |               "    }\n",
1446 |               "  "
1447 |             ]
1448 |           },
1449 |           "metadata": {}
1450 |         }
1451 |       ]
1452 |     },
1453 |     {
1454 |       "cell_type": "code",
1455 |       "source": [
1456 |         "# 1: Load TinyStories\n",
1457 |         "dataset = load_dataset(\"roneneldan/TinyStories\", split=\"train\")"
1458 |       ],
1459 |       "metadata": {
1460 |         "id": "LU6ve7PFfbSV",
1461 |         "colab": {
1462 |           "base_uri": "https://localhost:8080/",
1463 |           "height": 88
1464 |         },
1465 |         "outputId": "f55a0148-7589-4d57-af02-02fffd73758c"
1466 |       },
1467 |       "execution_count": 26,
1468 |       "outputs": [
1469 |         {
1470 |           "output_type": "display_data",
1471 |           "data": {
1472 |             "text/plain": [
1473 |               "<IPython.core.display.HTML object>"
1474 |             ],
1475 |             "text/html": [
1476 |               "<style>\n",
1477 |               "    body {\n",
1478 |               "      font-size: 20px;\n",
1479 |               "    }\n",
1480 |               "  "
1481 |             ]
1482 |           },
1483 |           "metadata": {}
1484 |         },
1485 |         {
1486 |           "output_type": "stream",
1487 |           "name": "stderr",
1488 |           "text": [
1489 |             "Using the latest cached version of the dataset since roneneldan/TinyStories couldn't be found on the Hugging Face Hub\n",
1490 |             "Found the latest cached dataset configuration 'default' at C:\\Users\\Howsam\\.cache\\huggingface\\datasets\\roneneldan___tiny_stories\\default\\0.0.0\\f54c09fd23315a6f9c86f9dc80f725de7d8f9c64 (last modified on Sat Apr 12 16:34:29 2025).\n"
1491 |           ]
1492 |         }
1493 |       ]
1494 |     },
1495 |     {
1496 |       "cell_type": "code",
1497 |       "source": [
1498 |         "# 2: Select 100 random stories\n",
1499 |         "def split_story(text, ratio=0.4):\n",
1500 |         "    words = text.strip().split()\n",
1501 |         "    cut = max(1, int(len(words) * ratio))\n",
1502 |         "    return \" \".join(words[:cut]), \" \".join(words[cut:])\n",
1503 |         "\n",
1504 |         "indices = random.sample(range(len(dataset)), gen_cfg.n_examples)\n",
1505 |         "examples = [dataset[i][\"text\"] for i in indices]\n",
1506 |         "prompts, originals = zip(*(split_story(t) for t in examples))"
1507 |       ],
1508 |       "metadata": {
1509 |         "id": "2D7HCE99gRs4",
1510 |         "colab": {
1511 |           "base_uri": "https://localhost:8080/",
1512 |           "height": 17
1513 |         },
1514 |         "outputId": "383aed7a-9b26-4f78-bbfb-78015a84a881"
1515 |       },
1516 |       "execution_count": 27,
1517 |       "outputs": [
1518 |         {
1519 |           "output_type": "display_data",
1520 |           "data": {
1521 |             "text/plain": [
1522 |               "<IPython.core.display.HTML object>"
1523 |             ],
1524 |             "text/html": [
1525 |               "<style>\n",
1526 |               "    body {\n",
1527 |               "      font-size: 20px;\n",
1528 |               "    }\n",
1529 |               "  "
1530 |             ]
1531 |           },
1532 |           "metadata": {}
1533 |         }
1534 |       ]
1535 |     },
1536 |     {
1537 |       "cell_type": "code",
1538 |       "source": [
1539 |         "prompts[0], originals[0]"
1540 |       ],
1541 |       "metadata": {
1542 |         "colab": {
1543 |           "base_uri": "https://localhost:8080/",
1544 |           "height": 120
1545 |         },
1546 |         "id": "hU1m4JMslibK",
1547 |         "outputId": "b3d0a0f5-3ea2-4dc1-8cad-4eeb026159b1"
1548 |       },
1549 |       "execution_count": 29,
1550 |       "outputs": [
1551 |         {
1552 |           "output_type": "display_data",
1553 |           "data": {
1554 |             "text/plain": [
1555 |               "<IPython.core.display.HTML object>"
1556 |             ],
1557 |             "text/html": [
1558 |               "<style>\n",
1559 |               "    body {\n",
1560 |               "      font-size: 20px;\n",
1561 |               "    }\n",
1562 |               "  "
1563 |             ]
1564 |           },
1565 |           "metadata": {}
1566 |         },
1567 |         {
1568 |           "output_type": "execute_result",
1569 |           "data": {
1570 |             "text/plain": [
1571 |               "('One day, a little boy named Tim was very excited. He saw a big gray dog in the park. The dog was playing with a ball. Tim wanted to play with the dog too. Tim went to his',\n",
1572 |               " 'mom and asked her to explain why the dog was gray. His mom said, \"The dog is gray because it has gray fur. Just like how you have brown hair, the dog has gray fur.\" Tim was happy to learn about the gray dog. He went to play with the dog and they had lots of fun together.')"
1573 |             ]
1574 |           },
1575 |           "metadata": {},
1576 |           "execution_count": 29
1577 |         }
1578 |       ]
1579 |     },
1580 |     {
1581 |       "cell_type": "code",
1582 |       "source": [
1583 |         "# 3: Load tokenizer\n",
1584 |         "tokenizer = Tokenizer.from_file(gen_cfg.tokenizer_path)"
1585 |       ],
1586 |       "metadata": {
1587 |         "id": "V72ND6JZfbSV",
1588 |         "colab": {
1589 |           "base_uri": "https://localhost:8080/",
1590 |           "height": 17
1591 |         },
1592 |         "outputId": "22e4d317-e722-4728-c969-e600d2394292"
1593 |       },
1594 |       "execution_count": 30,
1595 |       "outputs": [
1596 |         {
1597 |           "output_type": "display_data",
1598 |           "data": {
1599 |             "text/plain": [
1600 |               "<IPython.core.display.HTML object>"
1601 |             ],
1602 |             "text/html": [
1603 |               "<style>\n",
1604 |               "    body {\n",
1605 |               "      font-size: 20px;\n",
1606 |               "    }\n",
1607 |               "  "
1608 |             ]
1609 |           },
1610 |           "metadata": {}
1611 |         }
1612 |       ]
1613 |     },
1614 |     {
1615 |       "cell_type": "code",
1616 |       "source": [
1617 |         "# 4: Load Pretrained GPT model\n",
1618 |         "\n",
1619 |         "gpt_cfg = GPTConfig(\n",
1620 |         "    vocab_size=10_000,\n",
1621 |         "    max_seq_len=1024,\n",
1622 |         "    n_layer=8,\n",
1623 |         "    n_head=16,\n",
1624 |         "    n_embd=512,\n",
1625 |         "    f_expnd=4\n",
1626 |         ")\n",
1627 |         "\n",
1628 |         "model = GPT(gpt_cfg)\n",
1629 |         "model.load_state_dict(torch.load(gen_cfg.model_path))\n",
1630 |         "model.to(gen_cfg.device).eval()"
1631 |       ],
1632 |       "metadata": {
1633 |         "colab": {
1634 |           "base_uri": "https://localhost:8080/",
1635 |           "height": 531
1636 |         },
1637 |         "outputId": "d833adc3-807f-404a-8ca1-552ad05b8751",
1638 |         "id": "JkCfeKRSfbSV"
1639 |       },
1640 |       "execution_count": 31,
1641 |       "outputs": [
1642 |         {
1643 |           "output_type": "display_data",
1644 |           "data": {
1645 |             "text/plain": [
1646 |               "<IPython.core.display.HTML object>"
1647 |             ],
1648 |             "text/html": [
1649 |               "<style>\n",
1650 |               "    body {\n",
1651 |               "      font-size: 20px;\n",
1652 |               "    }\n",
1653 |               "  "
1654 |             ]
1655 |           },
1656 |           "metadata": {}
1657 |         },
1658 |         {
1659 |           "output_type": "execute_result",
1660 |           "data": {
1661 |             "text/plain": [
1662 |               "GPT(\n",
1663 |               "  (wte): Embedding(10000, 512)\n",
1664 |               "  (wpe): Embedding(1024, 512)\n",
1665 |               "  (decoders): ModuleList(\n",
1666 |               "    (0-7): 8 x DecoderBlock(\n",
1667 |               "      (ln1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
1668 |               "      (mha): MultiHeadAttention(\n",
1669 |               "        (qkv_proj): Linear(in_features=512, out_features=1536, bias=False)\n",
1670 |               "        (c_proj): Linear(in_features=512, out_features=512, bias=False)\n",
1671 |               "      )\n",
1672 |               "      (ln2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
1673 |               "      (mlp): FeedForward(\n",
1674 |               "        (up_proj): Linear(in_features=512, out_features=2048, bias=False)\n",
1675 |               "        (down_proj): Linear(in_features=2048, out_features=512, bias=False)\n",
1676 |               "      )\n",
1677 |               "    )\n",
1678 |               "  )\n",
1679 |               "  (lnf): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
1680 |               "  (lm_head): Linear(in_features=512, out_features=10000, bias=False)\n",
1681 |               ")"
1682 |             ]
1683 |           },
1684 |           "metadata": {},
1685 |           "execution_count": 31
1686 |         }
1687 |       ]
1688 |     },
1689 |     {
1690 |       "cell_type": "code",
1691 |       "source": [
1692 |         "# 5: Generate\n",
1693 |         "completions = []\n",
1694 |         "for prompt in tqdm(prompts, desc=\"Generating\"):\n",
1695 |         "    completions.append(\n",
1696 |         "        generate(\n",
1697 |         "            model, tokenizer, prompt,\n",
1698 |         "            n_rep=gen_cfg.n_samples, max_seq_len=256,\n",
1699 |         "            T=gen_cfg.temperature, top_k=10\n",
1700 |         "        )[0]\n",
1701 |         "    )"
1702 |       ],
1703 |       "metadata": {
1704 |         "id": "q-yEfgfKft33",
1705 |         "colab": {
1706 |           "base_uri": "https://localhost:8080/",
1707 |           "height": 88,
1708 |           "referenced_widgets": [
1709 |             "2042d8a3d97c4afe82cd6e767de74058",
1710 |             "76bd70def6b1496db513e64c57cc4c29",
1711 |             "84f62a340d5c44588656de5ff1274bba",
1712 |             "8540dc77dd334a39af9452e60e554825",
1713 |             "a1319920d408492eb172385d02045064",
1714 |             "dc8e7c9d886d4f4fa68514602ffbf170",
1715 |             "960d613b9a184aeea9cb2fc8cef4ce2e",
1716 |             "be2c8eb4d84a4141ae50a3e4da72b898",
1717 |             "df89141e1d9440e8880d54f5648e8e96",
1718 |             "ae7ea84d05614bf9a01fd97b9f44ffc3",
1719 |             "06eda88a825143de9594a9dd40c2d661"
1720 |           ]
1721 |         },
1722 |         "outputId": "42cc33ca-a5e9-4319-c789-b67beae63a1e"
1723 |       },
1724 |       "execution_count": 32,
1725 |       "outputs": [
1726 |         {
1727 |           "output_type": "display_data",
1728 |           "data": {
1729 |             "text/plain": [
1730 |               "<IPython.core.display.HTML object>"
1731 |             ],
1732 |             "text/html": [
1733 |               "<style>\n",
1734 |               "    body {\n",
1735 |               "      font-size: 20px;\n",
1736 |               "    }\n",
1737 |               "  "
1738 |             ]
1739 |           },
1740 |           "metadata": {}
1741 |         },
1742 |         {
1743 |           "output_type": "display_data",
1744 |           "data": {
1745 |             "text/plain": [
1746 |               "Generating:   0%|          | 0/100 [00:00<?, ?it/s]"
1747 |             ],
1748 |             "application/vnd.jupyter.widget-view+json": {
1749 |               "version_major": 2,
1750 |               "version_minor": 0,
1751 |               "model_id": "2042d8a3d97c4afe82cd6e767de74058"
1752 |             }
1753 |           },
1754 |           "metadata": {}
1755 |         },
1756 |         {
1757 |           "output_type": "stream",
1758 |           "name": "stderr",
1759 |           "text": [
1760 |             "C:\\Users\\Howsam\\AppData\\Local\\Temp\\ipykernel_16352\\2308121339.py:18: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at C:\\cb\\pytorch_1000000000000\\work\\aten\\src\\ATen\\native\\transformers\\cuda\\sdp_utils.cpp:455.)\n",
1761 |             "  y = F.scaled_dot_product_attention(q, k, v, is_causal=True)\n"
1762 |           ]
1763 |         }
1764 |       ]
1765 |     },
1766 |     {
1767 |       "cell_type": "code",
1768 |       "source": [
1769 |         "pprint(completions[0])\n",
1770 |         "print()\n",
1771 |         "pprint(originals[0])"
1772 |       ],
1773 |       "metadata": {
1774 |         "colab": {
1775 |           "base_uri": "https://localhost:8080/",
1776 |           "height": 505
1777 |         },
1778 |         "id": "tgi1IXYzNQ77",
1779 |         "outputId": "02cca3ec-1d90-496b-d5fc-2aeebbe24a70"
1780 |       },
1781 |       "execution_count": 33,
1782 |       "outputs": [
1783 |         {
1784 |           "output_type": "display_data",
1785 |           "data": {
1786 |             "text/plain": [
1787 |               "<IPython.core.display.HTML object>"
1788 |             ],
1789 |             "text/html": [
1790 |               "<style>\n",
1791 |               "    body {\n",
1792 |               "      font-size: 20px;\n",
1793 |               "    }\n",
1794 |               "  "
1795 |             ]
1796 |           },
1797 |           "metadata": {}
1798 |         },
1799 |         {
1800 |           "output_type": "stream",
1801 |           "name": "stdout",
1802 |           "text": [
1803 |             "(' mom and said, \"Mom, can I play with the gray dog?\"\\n'\n",
1804 |             " '\\n'\n",
1805 |             " 'His mom said, \"Yes, Tim. You can play with the dog.\" Tim was very happy. He '\n",
1806 |             " 'went to the park with the big gray dog. He threw the ball and the dog ran to '\n",
1807 |             " 'get it.\\n'\n",
1808 |             " '\\n'\n",
1809 |             " \"As they played, the ball hit a man's car. The car was going very fast. Tim's \"\n",
1810 |             " 'mom was scared. She said, \"Tim, we have to hurry and go home now.\" Tim '\n",
1811 |             " 'looked at the car and saw a big, red car on the road. He said, \"Mom, I '\n",
1812 |             " 'don\\'t like this car. It\\'s too fast.\"\\n'\n",
1813 |             " '\\n'\n",
1814 |             " 'Tim and his mom hurried home. Tim said, \"Mom, the gray dog is here. It '\n",
1815 |             " 'won\\'t hit the car. I\\'m glad we hurried.\" Tim smiled and hugged his mom. '\n",
1816 |             " 'They went inside and had a fun day together.')\n",
1817 |             "\n",
1818 |             "('mom and asked her to explain why the dog was gray. His mom said, \"The dog is '\n",
1819 |             " 'gray because it has gray fur. Just like how you have brown hair, the dog has '\n",
1820 |             " 'gray fur.\" Tim was happy to learn about the gray dog. He went to play with '\n",
1821 |             " 'the dog and they had lots of fun together.')\n"
1822 |           ]
1823 |         }
1824 |       ]
1825 |     },
1826 |     {
1827 |       "cell_type": "code",
1828 |       "source": [
1829 |         "# 6.1: s_i\n",
1830 |         "s_scores = [rouge_k_precision(gen, orig, k=2) for gen, orig in zip(completions, originals)]\n",
1831 |         "\n",
1832 |         "plt.hist(s_scores, bins=30)\n",
1833 |         "plt.title(\"Similarity to original ending (s_i)\")\n",
1834 |         "plt.xlabel(\"Score\"); plt.ylabel(\"Count\");"
1835 |       ],
1836 |       "metadata": {
1837 |         "id": "NP7Xdl62iFsr",
1838 |         "colab": {
1839 |           "base_uri": "https://localhost:8080/",
1840 |           "height": 470
1841 |         },
1842 |         "outputId": "294071e7-e1dc-404b-b8cc-6302613802ad"
1843 |       },
1844 |       "execution_count": 34,
1845 |       "outputs": [
1846 |         {
1847 |           "output_type": "display_data",
1848 |           "data": {
1849 |             "text/plain": [
1850 |               "<IPython.core.display.HTML object>"
1851 |             ],
1852 |             "text/html": [
1853 |               "<style>\n",
1854 |               "    body {\n",
1855 |               "      font-size: 20px;\n",
1856 |               "    }\n",
1857 |               "  "
1858 |             ]
1859 |           },
1860 |           "metadata": {}
1861 |         },
1862 |         {
1863 |           "output_type": "display_data",
1864 |           "data": {
1865 |             "text/plain": [
1866 |               "<Figure size 640x480 with 1 Axes>"
1867 |             ],
1868 |             "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAHFCAYAAAAHcXhbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxUklEQVR4nO3dd3hU1b7/8c9AwgQIhCahBRIUwdCbAvEYehEQQY+CSPGABwUhCB4holL0EMoBUWnqpQkX5HipVxRB2gUJCgiigGIBRSlRWqgBkvX7w1/mOExC2iQzK7xfzzPP41577bW/s7JDPu4y4zDGGAEAAFiqgK8LAAAAyAnCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMrPHZZ5+pa9euqly5spxOp0JDQ9W0aVMNHz7crV/z5s3VvHlzr+47rTEdDofGjBnj1f0cOXJEDodD8+fPd7Vt375dY8aM0dmzZ726r8WLF2vatGleHdNb+vbtq/Dw8GxtO3/+fDkcDh05csSrNd0oPDxcffv2zdV9ZMXmzZvlcDi0efNmV1tO5tFbWrVqpaeeesqrY6b1ezJnzhxVrFhRFy9e9Oq+YAkDWOCDDz4wBQoUMC1btjRLliwxmzdvNkuWLDHDhw83FStWdOu7f/9+s3//fq/uP60xJZnRo0d7dT9Xrlwx8fHxJiEhwdU2efJkI8kcPnzYq/vq2LGjqVKlilfH9Jbvv//efPHFF9naNiEhwcTHx5srV654uSp3VapUMX369MnVfWTFpk2bjCSzadMmV1tO5tEbVq5caZxOp/nll1+8Om5avyfXrl0z1apVMy+//LJX9wU7BPg4SwGZMmnSJEVEROjjjz9WQMB/Dtvu3btr0qRJbn0jIyO9vv/cGPPPkpOTdf36dTmdTjVp0iRX9+XPLl26pCJFiuj222/P9hi33XabbrvtNi9WZa+czKM3jB8/Xl27dlXFihW9Om5avycBAQEaMGCAXnnlFY0YMUJFihTx6j7h37jMBCucOnVKZcqUcQsyqQoUcD+Mb7wklHpKevLkyZo4caLCw8NVuHBhNW/eXIcOHdK1a9c0cuRIVahQQSEhIeratasSEhJuOmZafvvtNw0cOFCRkZEKDg5W2bJl1bJlS23dutWtX2o9kyZN0quvvqqIiAg5nU5t2rTJ4/T5mDFj9I9//EOSFBERIYfD4bqU0K9fP5UqVUqXLl3yqKVly5aqWbNmurU2b95ca9as0U8//eQa0+FwuNafPn1aAwcOVMWKFVWoUCFVrVpVo0aNUlJS0k3nINXcuXNVt25dBQUFqVSpUuratasOHjzo1qdv374KDg7WV199pbZt26pYsWJq1aqVa92Nl0fOnj3res/BwcHq2LGjfvzxR4/LfWldZmrevLlq1aqlnTt36i9/+YuKFCmiqlWrasKECUpJSXH1u3LlioYPH6569eopJCREpUqVUtOmTbVq1apMve+0GGM0c+ZM1atXT4ULF1bJkiX18MMP68cff3Trl9kaJembb75R+/btVaRIEZUpU0ZPPfWUzp8/77HvtObR4XDomWee0cKFC3XXXXepSJEiqlu3rj744AOP7VetWqU6derI6XSqatWqev311zVmzBi3YyU9e/bs0eeff65evXq5tV+6dEnPPfecIiIiXMdHo0aNtGTJkgzHTJXWZSZJ6tmzpxITE/Xee+9leizkD4QZWKFp06b67LPPNGTIEH322We6du1alseYMWOGPv30U82YMUP/9V//pW+++UadO3dWv3799Ntvv2nu3LmaNGmSPvnkE/Xv3z/L458+fVqSNHr0aK1Zs0bz5s1T1apV1bx5c7f7GFK98cYb2rhxo/71r3/po48+Uo0aNTz69O/fX4MHD5YkLV++XPHx8YqPj1eDBg0UExOjM2fOaPHixW7bHDhwQJs2bdKgQYPSrXXmzJmKiopSuXLlXGPGx8dL+uMPeosWLfTuu+9q2LBhWrNmjR5//HFNmjRJ3bp1y3Ae4uLi1K9fP9WsWVPLly/X66+/rn379qlp06b67rvv3PpevXpVDzzwgFq2bKlVq1Zp7NixaY6ZkpKizp07a/HixRoxYoRWrFihe+65R+3bt8+wnlQnTpxQz5499fjjj2v16tXq0KGDYmNjtWjRIlefpKQknT59Ws8995xWrlypJUuW6N5771W3bt307rvvZnpffzZgwAANHTpUrVu31sqVKzVz5kzt379fzZo108mTJ7Nc48mTJxUdHa2vv/5aM2fO1MKFC3XhwgU988wzma5pzZo1mj59usaNG6dly5a5AuefA9batWvVrVs3lS5dWkuXLtWkSZO0ZMkSLViwIFP7+OCDD1SwYEHdd999bu3Dhg3TrFmzNGTIEK1du1YLFy7UX//6V506dSrT9aenXLlyqlGjhtasWZPjsWAZX1/nAjLj999/N/fee6+RZCSZwMBA06xZMxMXF2fOnz/v1jc6OtpER0e7lg8fPmwkmbp165rk5GRX+7Rp04wk88ADD7htP3ToUCPJnDt3Lt0xjcn4npnr16+ba9eumVatWpmuXbt61HP77bebq1evum2Tum7evHmutpvdMxMdHW3q1avn1vb000+b4sWLe8zLjdK7Z2b27NlGkvn3v//t1j5x4kQjyaxbty7dMc+cOWMKFy5s7r//frf2n3/+2TidTvPYY4+52vr06WMkmblz53qM06dPH7fa1qxZYySZWbNmufWLi4vz+DnMmzfPY76io6ONJPPZZ5+5bR8ZGWnatWuX7vtJ/Rn269fP1K9f321dZu6ZiY+PN5LMlClT3NqPHj1qChcubJ5//vks1zhixAjjcDjM3r173fq1adPG456ZG+fRmD+O29DQUJOYmOhqO3HihClQoICJi4tztTVu3NiEhYWZpKQkV9v58+dN6dKlTWb+dHTo0MHUqFHDo71WrVrmwQcfzHD7m0nr9yRVz549TWhoaI7Gh304MwMrlC5dWlu3btXOnTs1YcIEdenSRYcOHVJsbKxq166t33//PcMx7r//frdLUnfddZckqWPHjm79Utt//vnnLNc5e/ZsNWjQQEFBQQoICFBgYKA2bNjgcYlFkh544AEFBgZmeR9/FhMTo7179+rTTz+VJCUmJmrhwoXq06ePgoODszXmxo0bVbRoUT388MNu7alP7mzYsCHdbePj43X58mWPp3zCwsLUsmXLNLd96KGHMqxpy5YtkqRHHnnErb1Hjx4ZbpuqXLlyuvvuu93a6tSpo59++smt7f3331dUVJSCg4NdP8M5c+ak+TPMyAcffCCHw6HHH39c169fd73KlSununXrepyxy0yNmzZtUs2aNVW3bl23fo899lim62rRooWKFSvmWg4NDVXZsmVd+7l48aJ27dqlBx98UIUKFXL1Cw4OVufOnTO1j2PHjqls2bIe7Xfffbc++ugjjRw5Ups3b9bly5czXXdmlC1bVgkJCbp+/bpXx4V/I8zAKo0aNdKIESP0/vvv69ixY3r22Wd15MgRj5uA01KqVCm35dR/pNNrv3LlSpZqmzp1qp5++mndc889WrZsmXbs2KGdO3eqffv2af6DXb58+SyNn5YuXbooPDxcM2bMkPTH/SIXL1686SWmjJw6dUrlypXzuC+ibNmyCggIuOnlgNR1ab23ChUqeGxbpEgRFS9ePFM1BQQEePysQkNDM9w2VenSpT3anE6n289m+fLleuSRR1SxYkUtWrRI8fHx2rlzp/72t79l+XiQ/rgkZIxRaGioAgMD3V47duzwCOGZqTH153OjtNrSk9F+zpw546r7Rpmd88uXLysoKMij/Y033tCIESO0cuVKtWjRQqVKldKDDz7ocQkyu4KCgmSMydbPC/biaSZYKzAwUKNHj9Zrr72mr7/+2tflaNGiRWrevLlmzZrl1p7WjZmSMnUTZUYKFCigQYMG6YUXXtCUKVM0c+ZMtWrVStWrV8/2mKVLl9Znn30mY4xbjan/t1umTJmbbitJx48f91h37Ngxj20zOwelS5fW9evXdfr0abdAc+LEiUxtn1mLFi1SRESEli5d6lZbZm98vlGZMmXkcDi0detWOZ1Oj/VptWWkdOnSab5vb85FyZIl5XA4PO7pycp+ypQp47qP7M+KFi2qsWPHauzYsTp58qTrLE3nzp31zTff5Lj206dPy+l0ZvvMJOzEmRlYIa0/jpJcp/4rVKiQl+WkyeFwePxx2rdvn+vG2uxKHTO90/H9+/dXoUKF1LNnT3377beZvhH0xv/jT9WqVStduHBBK1eudGtPvQE29YmjtDRt2lSFCxd2u2FVkn755Rdt3LjxptveTHR0tCRp6dKlbu3efmrF4XCoUKFCbkHmxIkT2X6aqVOnTjLG6Ndff1WjRo08XrVr187ymC1atND+/fv15ZdfurXfeCN4ThQtWlSNGjXSypUrdfXqVVf7hQsX0nzqKS01atTweGLrRqGhoerbt6969Oihb7/9Ns0n87Lqxx9/zPWPUoD/4cwMrNCuXTtVqlRJnTt3Vo0aNZSSkqK9e/dqypQpCg4OVkxMjK9LVKdOnfTKK69o9OjRio6O1rfffqtx48YpIiIiR9fvU//gvf766+rTp48CAwNVvXp11z0PJUqUUO/evTVr1ixVqVIl0/c01K5dW8uXL9esWbPUsGFDFShQQI0aNVLv3r01Y8YM9enTR0eOHFHt2rW1bds2jR8/Xvfff79at26d7pglSpTQSy+9pBdeeEG9e/dWjx49dOrUKY0dO1ZBQUEaPXp0tuagffv2ioqK0vDhw5WYmKiGDRsqPj7eFbBufDw/uzp16qTly5dr4MCBevjhh3X06FG98sorKl++fLYug0RFRenvf/+7nnjiCe3atUv33XefihYtquPHj2vbtm2qXbu2nn766SyNOXToUM2dO1cdO3bUq6++qtDQUP33f/+3V85q/Nm4cePUsWNHtWvXTjExMUpOTtbkyZMVHByc5hmXGzVv3lxz587VoUOHdOedd7ra77nnHnXq1El16tRRyZIldfDgQS1cuFBNmzbN8WfDpKSk6PPPP1e/fv1yNA7sQ5iBFV588UWtWrVKr732mo4fP66kpCSVL19erVu3VmxsrOumXV8aNWqULl26pDlz5mjSpEmKjIzU7NmztWLFijQfzc6s5s2bKzY2VgsWLNA777yjlJQUbdq0ye1zbx599FHNmjVLTz/9dKb/sMfExGj//v164YUXdO7cORljZIxRUFCQNm3apFGjRmny5Mn67bffVLFiRT333HOZCiOxsbEqW7as3njjDS1dutT1mT7jx49XtWrVsjUHBQoU0P/+7/9q+PDhmjBhgq5evaqoqCgtWrRITZo0UYkSJbI17o2eeOIJJSQkaPbs2Zo7d66qVq2qkSNH6pdffkn3sfGMvPXWW2rSpIneeustzZw5UykpKapQoYKioqI8bvbNjHLlymnLli2KiYnR008/rSJFiqhr166aPn26unTpkq0a09K+fXstW7ZML7/8sh599FGVK1dOAwcO1LFjx7Rw4cIMt+/SpYuCg4O1atUq12clSX98BtLq1av12muv6dKlS6pYsaJ69+6tUaNG5bjmzZs369y5c+rZs2eOx4JdHMYY4+siAOTM8OHDNWvWLB09ejTNmzvzq8WLF6tnz5769NNP1axZM1+Xk+9du3ZN9erVU8WKFbVu3boM+w8ePFgbNmzQ/v37vXKPWEZ69eqlH3/80fV0H24dnJkBLLZjxw4dOnRIM2fO1IABA/J1kFmyZIl+/fVX1a5dWwUKFNCOHTs0efJk3XfffQSZXNKvXz+1adNG5cuX14kTJzR79mwdPHhQr7/+eqa2f/HFF/Xuu+9q2bJlHo/6e9sPP/ygpUuXauPGjbm6H/gnwgxgsdT7DDp16qRXX33V1+XkqmLFium9997Tq6++qosXL6p8+fLq27dvvn/fvnT+/Hk999xz+u233xQYGKgGDRroww8/vOl9U3+Wej/PmTNnMtXfGKPk5OSb9ilYsGCaZ3l+/vlnTZ8+Xffee2+m9oX8hctMAAC/sHnzZrVo0eKmfebNm+fxoYwAYQYA4BfOnz+vb7/99qZ9IiIi8vXlVGQPYQYAAFiND80DAABWy/c3AKekpOjYsWMqVqxYnjwaCAAAcs4Yo/Pnz6tChQoZfn5Wvg8zx44dU1hYmK/LAAAA2XD06FFVqlTppn3yfZhJ/cj3o0ePZurbeQEAgO8lJiYqLCzM9Xf8ZvJ9mEm9tFS8eHHCDAAAlsnMLSLcAAwAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwWoCvC0DeCx+5xif7PTKho0/2CwDI3zgzAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAVvNpmPm///s/de7cWRUqVJDD4dDKlSvd1htjNGbMGFWoUEGFCxdW8+bNtX//ft8UCwAA/JJPw8zFixdVt25dTZ8+Pc31kyZN0tSpUzV9+nTt3LlT5cqVU5s2bXT+/Pk8rhQAAPirAF/uvEOHDurQoUOa64wxmjZtmkaNGqVu3bpJkhYsWKDQ0FAtXrxYAwYMyMtSAQCAn/Lbe2YOHz6sEydOqG3btq42p9Op6Ohobd++3YeVAQAAf+LTMzM3c+LECUlSaGioW3toaKh++umndLdLSkpSUlKSazkxMTF3CgQAAH7Bb8NMKofD4bZsjPFo+7O4uDiNHTs2t8uCRcJHrsn2tkcmdPRiJQCA3OC3l5nKlSsn6T9naFIlJCR4nK35s9jYWJ07d871Onr0aK7WCQAAfMtvw0xERITKlSun9evXu9quXr2qLVu2qFmzZulu53Q6Vbx4cbcXAADIv3x6menChQv6/vvvXcuHDx/W3r17VapUKVWuXFlDhw7V+PHjVa1aNVWrVk3jx49XkSJF9Nhjj/mwagAA4E98GmZ27dqlFi1auJaHDRsmSerTp4/mz5+v559/XpcvX9bAgQN15swZ3XPPPVq3bp2KFSvmq5IBAICf8WmYad68uYwx6a53OBwaM2aMxowZk3dFAQAAq/jtPTMAAACZQZgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWC/B1Abh1hI9ck+1tj0zo6MVKMs/GmgHgVsOZGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsJpfh5nr16/rxRdfVEREhAoXLqyqVatq3LhxSklJ8XVpAADATwT4uoCbmThxombPnq0FCxaoZs2a2rVrl5544gmFhIQoJibG1+UBAAA/4NdhJj4+Xl26dFHHjh0lSeHh4VqyZIl27drl48oAAIC/8OvLTPfee682bNigQ4cOSZK+/PJLbdu2Tffff7+PKwMAAP7Cr8/MjBgxQufOnVONGjVUsGBBJScn65///Kd69OiR7jZJSUlKSkpyLScmJuZFqQAAwEf8+szM0qVLtWjRIi1evFhffPGFFixYoH/9619asGBButvExcUpJCTE9QoLC8vDigEAQF7z6zDzj3/8QyNHjlT37t1Vu3Zt9erVS88++6zi4uLS3SY2Nlbnzp1zvY4ePZqHFQMAgLzm15eZLl26pAIF3PNWwYIFb/pottPplNPpzO3SAACAn/DrMNO5c2f985//VOXKlVWzZk3t2bNHU6dO1d/+9jdflwYAAPyEX4eZN998Uy+99JIGDhyohIQEVahQQQMGDNDLL7/s69IAAICf8OswU6xYMU2bNk3Tpk3zdSkAAMBP+fUNwAAAABkhzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsF+LoAIDPCR67xdQlZlpOaj0zo6MVKACB/48wMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAVstWmKlatapOnTrl0X727FlVrVo1x0UBAABkVrbCzJEjR5ScnOzRnpSUpF9//TXHRQEAAGRWQFY6r1692vXfH3/8sUJCQlzLycnJ2rBhg8LDw71WHAAAQEayFGYefPBBSZLD4VCfPn3c1gUGBio8PFxTpkzxWnEAAAAZyVKYSUlJkSRFRERo586dKlOmTK4UBQAAkFlZCjOpDh8+7O06AAAAsiVbYUaSNmzYoA0bNighIcF1xibV3Llzc1xYql9//VUjRozQRx99pMuXL+vOO+/UnDlz1LBhQ6/tAwAA2CtbYWbs2LEaN26cGjVqpPLly8vhcHi7LknSmTNnFBUVpRYtWuijjz5S2bJl9cMPP6hEiRK5sj8AAGCfbIWZ2bNna/78+erVq5e363EzceJEhYWFad68ea42npYCAAB/lq3Pmbl69aqaNWvm7Vo8rF69Wo0aNdJf//pXlS1bVvXr19c777yT6/sFAAD2yFaY6d+/vxYvXuztWjz8+OOPmjVrlqpVq6aPP/5YTz31lIYMGaJ333033W2SkpKUmJjo9gIAAPlXti4zXblyRW+//bY++eQT1alTR4GBgW7rp06d6pXiUlJS1KhRI40fP16SVL9+fe3fv1+zZs1S796909wmLi5OY8eO9cr+c1v4yDXZ3vbIhI5erAQAAHtlK8zs27dP9erVkyR9/fXXbuu8eTNw+fLlFRkZ6dZ21113admyZeluExsbq2HDhrmWExMTFRYW5rWaAACAf8lWmNm0aZO360hTVFSUvv32W7e2Q4cOqUqVKulu43Q65XQ6c7s0AADgJ7J1z0xeefbZZ7Vjxw6NHz9e33//vRYvXqy3335bgwYN8nVpAADAT2TrzEyLFi1uejlp48aN2S7ozxo3bqwVK1YoNjZW48aNU0REhKZNm6aePXt6ZXwAAGC/bIWZ1PtlUl27dk179+7V119/7fEFlDnVqVMnderUyatjAgCA/CNbYea1115Ls33MmDG6cOFCjgoCAADICq/eM/P444979XuZAAAAMuLVMBMfH6+goCBvDgkAAHBT2brM1K1bN7dlY4yOHz+uXbt26aWXXvJKYQAAAJmRrTATEhLitlygQAFVr15d48aNU9u2bb1SGAAAQGZkK8z8+VusAQAAfClbYSbV7t27dfDgQTkcDkVGRqp+/freqgsAACBTshVmEhIS1L17d23evFklSpSQMUbnzp1TixYt9N577+m2227zdp0AAABpytbTTIMHD1ZiYqL279+v06dP68yZM/r666+VmJioIUOGeLtGAACAdGXrzMzatWv1ySef6K677nK1RUZGasaMGdwADAAA8lS2zsykpKQoMDDQoz0wMFApKSk5LgoAACCzshVmWrZsqZiYGB07dszV9uuvv+rZZ59Vq1atvFYcAABARrIVZqZPn67z588rPDxct99+u+644w5FRETo/PnzevPNN71dIwAAQLqydc9MWFiYvvjiC61fv17ffPONjDGKjIxU69atvV0fAADATWXpzMzGjRsVGRmpxMRESVKbNm00ePBgDRkyRI0bN1bNmjW1devWXCkUAAAgLVkKM9OmTdOTTz6p4sWLe6wLCQnRgAEDNHXqVK8VBwAAkJEshZkvv/xS7du3T3d927ZttXv37hwXBQAAkFlZCjMnT55M85HsVAEBAfrtt99yXBQAAEBmZSnMVKxYUV999VW66/ft26fy5cvnuCgAAIDMylKYuf/++/Xyyy/rypUrHusuX76s0aNHq1OnTl4rDgAAICNZejT7xRdf1PLly3XnnXfqmWeeUfXq1eVwOHTw4EHNmDFDycnJGjVqVG7VCgAA4CFLYSY0NFTbt2/X008/rdjYWBljJEkOh0Pt2rXTzJkzFRoamiuFAgAApCXLH5pXpUoVffjhhzpz5oy+//57GWNUrVo1lSxZMjfqAwAAuKlsfQKwJJUsWVKNGzf2Zi0AAABZlq3vZgIAAPAXhBkAAGC1bF9mgm+Fj1zj6xLgp3JybByZ0NGLlQBA3uDMDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWM2qMBMXFyeHw6GhQ4f6uhQAAOAnrAkzO3fu1Ntvv606der4uhQAAOBHrAgzFy5cUM+ePfXOO++oZMmSvi4HAAD4ESvCzKBBg9SxY0e1bt3a16UAAAA/E+DrAjLy3nvv6YsvvtDOnTsz1T8pKUlJSUmu5cTExNwqDQAA+AG/DjNHjx5VTEyM1q1bp6CgoExtExcXp7Fjx+ZyZQD8RfjINdne9siEjl6sBICv+PVlpt27dyshIUENGzZUQECAAgICtGXLFr3xxhsKCAhQcnKyxzaxsbE6d+6c63X06FEfVA4AAPKKX5+ZadWqlb766iu3tieeeEI1atTQiBEjVLBgQY9tnE6nnE5nXpUIAAB8zK/DTLFixVSrVi23tqJFi6p06dIe7QAA4Nbk15eZAAAAMuLXZ2bSsnnzZl+XAAAA/AhnZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGoBvi7AduEj1/i6BORDHFf5W05+vkcmdPRiJUD+wJkZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwml+Hmbi4ODVu3FjFihVT2bJl9eCDD+rbb7/1dVkAAMCP+HWY2bJliwYNGqQdO3Zo/fr1un79utq2bauLFy/6ujQAAOAnAnxdwM2sXbvWbXnevHkqW7asdu/erfvuu89HVQEAAH/i12dmbnTu3DlJUqlSpXxcCQAA8Bd+fWbmz4wxGjZsmO69917VqlUr3X5JSUlKSkpyLScmJuZFeQAAwEesOTPzzDPPaN++fVqyZMlN+8XFxSkkJMT1CgsLy6MKAQCAL1gRZgYPHqzVq1dr06ZNqlSp0k37xsbG6ty5c67X0aNH86hKAADgC359mckYo8GDB2vFihXavHmzIiIiMtzG6XTK6XTmQXUAAMAf+HWYGTRokBYvXqxVq1apWLFiOnHihCQpJCREhQsX9nF1AADAH/j1ZaZZs2bp3Llzat68ucqXL+96LV261NelAQAAP+HXZ2aMMb4uAQAA+Dm/PjMDAACQEcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwWoCvCwCQP4SPXOPrErIsJzUfmdDRi5Vknq9qtnGufMXGubKx5j/jzAwAALAaYQYAAFiNMAMAAKxGmAEAAFYjzAAAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgNcIMAACwGmEGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQYAAFjNijAzc+ZMRUREKCgoSA0bNtTWrVt9XRIAAPATfh9mli5dqqFDh2rUqFHas2eP/vKXv6hDhw76+eeffV0aAADwA34fZqZOnap+/fqpf//+uuuuuzRt2jSFhYVp1qxZvi4NAAD4Ab8OM1evXtXu3bvVtm1bt/a2bdtq+/btPqoKAAD4kwBfF3Azv//+u5KTkxUaGurWHhoaqhMnTqS5TVJSkpKSklzL586dkyQlJibmSo0pSZdyZVzAF3Lye3Kr/S7YOFe+qjm3/v31VzbOlT/WnDquMSbDvn4dZlI5HA63ZWOMR1uquLg4jR071qM9LCwsV2oD8pOQab6uwB42zpWvarZxrnzFxrnK7ZrPnz+vkJCQm/bx6zBTpkwZFSxY0OMsTEJCgsfZmlSxsbEaNmyYazklJUWnT59W6dKl0w1At6LExESFhYXp6NGjKl68uK/L8SvMTfqYm5tjftLH3KSPuUmbMUbnz59XhQoVMuzr12GmUKFCatiwodavX6+uXbu62tevX68uXbqkuY3T6ZTT6XRrK1GiRG6WabXixYvzy5MO5iZ9zM3NMT/pY27Sx9x4yuiMTCq/DjOSNGzYMPXq1UuNGjVS06ZN9fbbb+vnn3/WU0895evSAACAH/D7MPPoo4/q1KlTGjdunI4fP65atWrpww8/VJUqVXxdGgAA8AN+H2YkaeDAgRo4cKCvy8hXnE6nRo8e7XFJDszNzTA3N8f8pI+5SR9zk3MOk5lnngAAAPyUX39oHgAAQEYIMwAAwGqEGQAAYDXCDAAAsBphJp+YOXOmIiIiFBQUpIYNG2rr1q037b9lyxY1bNhQQUFBqlq1qmbPnu22fv78+XI4HB6vK1eu5ObbyDVZmZ/jx4/rscceU/Xq1VWgQAENHTo0zX7Lli1TZGSknE6nIiMjtWLFilyqPnd5e27y07GTlblZvny52rRpo9tuu03FixdX06ZN9fHHH3v0uxWPm8zMza163Gzbtk1RUVEqXbq0ChcurBo1aui1117z6JdfjptcY2C99957zwQGBpp33nnHHDhwwMTExJiiRYuan376Kc3+P/74oylSpIiJiYkxBw4cMO+8844JDAw0//M//+PqM2/ePFO8eHFz/Phxt5eNsjo/hw8fNkOGDDELFiww9erVMzExMR59tm/fbgoWLGjGjx9vDh48aMaPH28CAgLMjh07cvndeFduzE1+OXayOjcxMTFm4sSJ5vPPPzeHDh0ysbGxJjAw0HzxxReuPrfqcZOZublVj5svvvjCLF682Hz99dfm8OHDZuHChaZIkSLmrbfecvXJL8dNbiLM5AN33323eeqpp9zaatSoYUaOHJlm/+eff97UqFHDrW3AgAGmSZMmruV58+aZkJAQr9fqC1mdnz+Ljo5O8w/2I488Ytq3b+/W1q5dO9O9e/cc1ZrXcmNu8suxk5O5SRUZGWnGjh3rWua4+Y8b54bj5j+6du1qHn/8cddyfjluchOXmSx39epV7d69W23btnVrb9u2rbZv357mNvHx8R7927Vrp127dunatWuutgsXLqhKlSqqVKmSOnXqpD179nj/DeSy7MxPZqQ3hzkZM6/l1txI9h873piblJQUnT9/XqVKlXK1cdz8Ia25kThuJGnPnj3avn27oqOjXW354bjJbYQZy/3+++9KTk72+Bbx0NBQj28bT3XixIk0+1+/fl2///67JKlGjRqaP3++Vq9erSVLligoKEhRUVH67rvvcueN5JLszE9mpDeHORkzr+XW3OSHY8cbczNlyhRdvHhRjzzyiKuN4+YPac3NrX7cVKpUSU6nU40aNdKgQYPUv39/17r8cNzkNiu+zgAZczgcbsvGGI+2jPr/ub1JkyZq0qSJa31UVJQaNGigN998U2+88Ya3ys4zWZ0fX43pC95+H/np2Mnu3CxZskRjxozRqlWrVLZsWa+M6W+8PTe3+nGzdetWXbhwQTt27NDIkSN1xx13qEePHjka81ZCmLFcmTJlVLBgQY+EnpCQ4JHkU5UrVy7N/gEBASpdunSa2xQoUECNGze26v+SpOzNT2akN4c5GTOv5dbc3MjGYycnc7N06VL169dP77//vlq3bu227lY/bm42Nze61Y6biIgISVLt2rV18uRJjRkzxhVm8sNxk9u4zGS5QoUKqWHDhlq/fr1b+/r169WsWbM0t2natKlH/3Xr1qlRo0YKDAxMcxtjjPbu3avy5ct7p/A8kp35yYz05jAnY+a13JqbG9l47GR3bpYsWaK+fftq8eLF6tixo8f6W/m4yWhubnQrHTc3MsYoKSnJtZwfjptc54u7juFdqY8Czpkzxxw4cMAMHTrUFC1a1Bw5csQYY8zIkSNNr169XP1TH81+9tlnzYEDB8ycOXM8Hs0eM2aMWbt2rfnhhx/Mnj17zBNPPGECAgLMZ599lufvL6eyOj/GGLNnzx6zZ88e07BhQ/PYY4+ZPXv2mP3797vWf/rpp6ZgwYJmwoQJ5uDBg2bChAlWPiqZG3OTX46drM7N4sWLTUBAgJkxY4bbo8Vnz5519blVj5vMzM2tetxMnz7drF692hw6dMgcOnTIzJ071xQvXtyMGjXK1Se/HDe5iTCTT8yYMcNUqVLFFCpUyDRo0MBs2bLFta5Pnz4mOjrarf/mzZtN/fr1TaFChUx4eLiZNWuW2/qhQ4eaypUrm0KFCpnbbrvNtG3b1mzfvj0v3kquyOr8SPJ4ValSxa3P+++/b6pXr24CAwNNjRo1zLJly/LgnXift+cmPx07WZmb6OjoNOemT58+bmPeisdNZubmVj1u3njjDVOzZk1TpEgRU7x4cVO/fn0zc+ZMk5yc7DZmfjlucovDmP9/5ycAAICFuGcGAABYjTADAACsRpgBAABWI8wAAACrEWYAAIDVCDMAAMBqhBkAAGA1wgwAALAaYQaATyQkJGjAgAGqXLmynE6nypUrp3bt2ik+Pt7XpQGwDN+aDcAnHnroIV27dk0LFixQ1apVdfLkSW3YsEGnT5/Olf1dvXpVhQoVypWxAfgWZ2YA5LmzZ89q27Ztmjhxolq0aKEqVaro7rvvVmxsrOsblc+ePau///3vCg0NVVBQkGrVqqUPPvjANcayZctUs2ZNOZ1OhYeHa8qUKW77CA8P16uvvqq+ffsqJCRETz75pCRp+/btuu+++1S4cGGFhYVpyJAhunjxYt69eQBeR5gBkOeCg4MVHByslStXKikpyWN9SkqKOnTooO3bt2vRokU6cOCAJkyYoIIFC0qSdu/erUceeUTdu3fXV199pTFjxuill17S/Pnz3caZPHmyatWqpd27d+ull17SV199pXbt2qlbt27at2+fli5dqm3btumZZ57Ji7cNIJfwRZMAfGLZsmV68skndfnyZTVo0EDR0dHq3r276tSpo3Xr1qlDhw46ePCg7rzzTo9te/bsqd9++03r1q1ztT3//PNas2aN9u/fL+mPMzP169fXihUrXH169+6twoUL66233nK1bdu2TdHR0bp48aKCgoJy8R0DyC2cmQHgEw899JCOHTum1atXq127dtq8ebMaNGig+fPna+/evapUqVKaQUaSDh48qKioKLe2qKgofffdd0pOTna1NWrUyK3P7t27NX/+fNeZoeDgYLVr104pKSk6fPiw998kgDzBDcAAfCYoKEht2rRRmzZt9PLLL6t///4aPXq0nnvuuZtuZ4yRw+HwaLtR0aJF3ZZTUlI0YMAADRkyxKNv5cqVs/EOAPgDwgwAvxEZGamVK1eqTp06+uWXX3To0KE0z85ERkZq27Ztbm3bt2/XnXfe6bqvJi0NGjTQ/v37dccdd3i9dgC+w2UmAHnu1KlTatmypRYtWqR9+/bp8OHDev/99zVp0iR16dJF0dHRuu+++/TQQw9p/fr1Onz4sD766COtXbtWkjR8+HBt2LBBr7zyig4dOqQFCxZo+vTpGZ7RGTFihOLj4zVo0CDt3btX3333nVavXq3BgwfnxdsGkEs4MwMgzwUHB+uee+7Ra6+9ph9++EHXrl1TWFiYnnzySb3wwguS/rhB+LnnnlOPHj108eJF3XHHHZowYYKkP86w/Pvf/9bLL7+sV155ReXLl9e4cePUt2/fm+63Tp062rJli0aNGqW//OUvMsbo9ttv16OPPprbbxlALuJpJgAAYDUuMwEAAKsRZgAAgNUIMwAAwGqEGQAAYDXCDAAAsBphBgAAWI0wAwAArEaYAQAAViPMAAAAqxFmAACA1QgzAADAaoQZAABgtf8HOoYKdZqXchkAAAAASUVORK5CYII="
1869 |           },
1870 |           "metadata": {}
1871 |         }
1872 |       ]
1873 |     },
1874 |     {
1875 |       "cell_type": "code",
1876 |       "source": [
1877 |         "# 6.2: r_i\n",
1878 |         "r_scores = []\n",
1879 |         "for i, ti in enumerate(completions):\n",
1880 |         "    others = [tj for j, tj in enumerate(completions) if j != i]\n",
1881 |         "    r_scores.append(max(rouge_k_fmeasure(ti, tj, k=2) for tj in others))\n",
1882 |         "\n",
1883 |         "plt.hist(r_scores, bins=30)\n",
1884 |         "plt.title(\"Similarity to original ending (r_i)\")\n",
1885 |         "plt.xlabel(\"Score\"); plt.ylabel(\"Count\");"
1886 |       ],
1887 |       "metadata": {
1888 |         "id": "rYmLiokdiJp3",
1889 |         "colab": {
1890 |           "base_uri": "https://localhost:8080/",
1891 |           "height": 470
1892 |         },
1893 |         "outputId": "8a012b67-e4f4-44a9-8267-ad5c11ebaa02"
1894 |       },
1895 |       "execution_count": 36,
1896 |       "outputs": [
1897 |         {
1898 |           "output_type": "display_data",
1899 |           "data": {
1900 |             "text/plain": [
1901 |               "<IPython.core.display.HTML object>"
1902 |             ],
1903 |             "text/html": [
1904 |               "<style>\n",
1905 |               "    body {\n",
1906 |               "      font-size: 20px;\n",
1907 |               "    }\n",
1908 |               "  "
1909 |             ]
1910 |           },
1911 |           "metadata": {}
1912 |         },
1913 |         {
1914 |           "output_type": "display_data",
1915 |           "data": {
1916 |             "text/plain": [
1917 |               "<Figure size 640x480 with 1 Axes>"
1918 |             ],
1919 |             "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAHFCAYAAADlrWMiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzeElEQVR4nO3dd3hUZf7//9cEwgQIhCYhYCTgKhCqFF2KJhRpERH1gwgKKijSEVwhIlIlFAvu0lYsoCzIKogsWEAEBAlKFUQUlUSiiChgQg0p9+8Pf5mvQ0LqTGZufD6ua66Luc99znmfO5MzL865Z+IwxhgBAABYLMDXBQAAABQVgQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBlb57LPP1KNHD11zzTVyOp0KDQ1Vy5YtNXr0aLd+0dHRio6O9ui+c9qmw+HQxIkTPbqfxMREORwOLVq0yNW2bds2TZw4Ub///rtH97V06VLNnj3bo9v0lAceeEARERGFWnfRokVyOBxKTEz0aE2XioiI0AMPPODVfRTEpk2b5HA4tGnTJldbUcbRU9q3b69HH33UY9u79Pduw4YNCg4O1k8//eSxfcBCBrDEmjVrTEBAgGnXrp1ZtmyZ2bRpk1m2bJkZPXq0qVGjhlvfAwcOmAMHDnh0/zltU5KZMGGCR/dz4cIFEx8fb44fP+5qmzVrlpFkEhISPLqvmJgYU7NmTY9u01O+++47s3v37kKte/z4cRMfH28uXLjg4arc1axZ0/Tr18+r+yiIjRs3Gklm48aNrraijKMnrFq1yjidTvPjjz96bJvx8fEmKSnJra1t27amb9++HtsH7FPSx3kKyLeZM2eqVq1a+vDDD1Wy5P976fbq1UszZ8506xsZGenx/Xtjm3+WkZGh9PR0OZ1O/f3vf/fqvvzZuXPnVKZMGV177bWF3sZVV12lq666yoNV2aso4+gJ06ZNU48ePVSjRo1c+50/f15BQUFyOBx5bjOn348hQ4bonnvu0dSpUxUeHl7oemEvbjnBGidOnFCVKlXcwkyWgAD3l/Klt4eybuPMmjVLM2bMUEREhEqXLq3o6GgdOnRIaWlpGjt2rKpXr66QkBD16NFDx48fz3WbOfn11181ePBgRUZGKjg4WFWrVlW7du20ZcsWt35Z9cycOVNTp05VrVq15HQ6tXHjxmy3nCZOnKh//OMfkqRatWrJ4XC4biv0799flSpV0rlz57LV0q5dO9WvX/+ytUZHR2vt2rX64YcfXNv885vJyZMnNXjwYNWoUUOlSpVS7dq1NW7cOKWmpuY6BlleffVVNW7cWEFBQapUqZJ69OihgwcPuvV54IEHFBwcrP3796tjx44qV66c2rdv71p26a2S33//3XXMwcHBiomJ0eHDh7PdgsjpllN0dLQaNGigHTt26Oabb1aZMmVUu3ZtTZ8+XZmZma5+Fy5c0OjRo9WkSROFhISoUqVKatmypd599918HXdOjDGaN2+emjRpotKlS6tixYq6++67dfjwYbd++a1Rkr7++mt17txZZcqUUZUqVfToo4/q9OnT2fad0zg6HA4NHTpUb7zxhurVq6cyZcqocePGWrNmTbb13333XTVq1EhOp1O1a9fWiy++qIkTJ+YreOzZs0eff/657r//frf2rJ/PunXr9NBDD+mqq65SmTJl8v3ayulWb7du3RQcHKyFCxfmaxu48hBoYI2WLVvqs88+0/Dhw/XZZ58pLS2twNuYO3euPv30U82dO1cvv/yyvv76a3Xr1k39+/fXr7/+qldffVUzZ87URx99pAEDBhR4+ydPnpQkTZgwQWvXrtVrr72m2rVrKzo62m1eQ5Z//vOf+vjjj/Xss8/q/fffV926dbP1GTBggIYNGyZJWrlypeLj4xUfH6+mTZtqxIgROnXqlJYuXeq2zldffaWNGzdqyJAhl6113rx5at26tapVq+baZnx8vKQ/3tTbtm2r119/XaNGjdLatWt13333aebMmbrzzjvzHIe4uDj1799f9evX18qVK/Xiiy9q3759atmypb799lu3vhcvXtTtt9+udu3a6d1339WkSZNy3GZmZqa6deumpUuXasyYMXrnnXd00003qXPnznnWk+XYsWPq06eP7rvvPq1evVpdunRRbGyslixZ4uqTmpqqkydP6vHHH9eqVau0bNkytWnTRnfeeadef/31fO/rzwYOHKiRI0eqQ4cOWrVqlebNm6cDBw6oVatW+uWXXwpc4y+//KKoqCh9+eWXmjdvnt544w2dOXNGQ4cOzXdNa9eu1Zw5czR58mStWLHCFTr/HLI++OAD3XnnnapcubKWL1+umTNnatmyZVq8eHG+9rFmzRqVKFFCt9xyS47LH3roIQUGBuqNN97Q22+/rcDAwHzXf6lSpUqpVatWWrt2baG3Acv5+p4XkF+//fabadOmjZFkJJnAwEDTqlUrExcXZ06fPu3WNyoqykRFRbmeJyQkGEmmcePGJiMjw9U+e/ZsI8ncfvvtbuuPHDnSSDLJycmX3aYxec+hSU9PN2lpaaZ9+/amR48e2eq59tprzcWLF93WyVr22muvudpym0MTFRVlmjRp4tY2aNAgU758+WzjcqnLzaFZsGCBkWT++9//urXPmDHDSDLr1q277DZPnTplSpcubbp27erWfuTIEeN0Ok3v3r1dbf369TOSzKuvvpptO/369XOrbe3atUaSmT9/vlu/uLi4bD+H1157Ldt4RUVFGUnms88+c1s/MjLSdOrU6bLHk/Uz7N+/v7nhhhvcluVnDk18fLyRZJ577jm39qSkJFO6dGnzxBNPFLjGMWPGGIfDYfbu3evW79Zbb802h+bScTTmj9dtaGioSUlJcbUdO3bMBAQEmLi4OFdbixYtTHh4uElNTXW1nT592lSuXNnk5+2jS5cupm7dutnas34+hZ3zcrnfu3HjxpmAgABz5syZQm0XduMKDaxRuXJlbdmyRTt27ND06dPVvXt3HTp0SLGxsWrYsKF+++23PLfRtWtXt9tT9erVkyTFxMS49ctqP3LkSIHrXLBggZo2baqgoCCVLFlSgYGB2rBhQ7bbLZJ0++23F+l/pZI0YsQI7d27V59++qkkKSUlRW+88Yb69eun4ODgQm3z448/VtmyZXX33Xe7tWd9omfDhg2XXTc+Pl7nz5/P9umf8PBwtWvXLsd177rrrjxr2rx5sySpZ8+ebu333ntvnutmqVatmm688Ua3tkaNGumHH35wa3vrrbfUunVrBQcHu36Gr7zySo4/w7ysWbNGDodD9913n9LT012PatWqqXHjxtmu3OWnxo0bN6p+/fpq3LixW7/evXvnu662bduqXLlyruehoaGqWrWqaz9nz57Vzp07dccdd6hUqVKufsHBwerWrVu+9nH06FFVrVr1ssvz83MviKpVqyozM1PHjh3z6HZhBwINrNO8eXONGTNGb731lo4eParHHntMiYmJ2SYG56RSpUpuz7NO1Jdrv3DhQoFqe/755zVo0CDddNNNWrFihbZv364dO3aoc+fOOn/+fLb+YWFhBdp+Trp3766IiAjNnTtX0h/zE86ePZvr7aa8nDhxQtWqVcs2T6Jq1aoqWbKkTpw4keu6Us7HVr169WzrlilTRuXLl89XTSVLlsz2swoNDc1z3SyVK1fO1uZ0Ot1+NitXrlTPnj1Vo0YNLVmyRPHx8dqxY4ceeuihAr8epD9uDxljFBoaqsDAQLfH9u3bswXx/NSY9fO5VE5tl5PXfk6dOuWq+1L5HfOsib6X44nX/59l7Sun3zVc+fiUE6wWGBioCRMm6IUXXtCXX37p63K0ZMkSRUdHa/78+W7tOU3WlJSviZV5CQgI0JAhQ/Tkk0/queee07x589S+fXvVqVOn0NusXLmyPvvsMxlj3Go8fvy40tPTVaVKlVzXlaSff/4527KjR49mWze/Y1C5cmWlp6fr5MmTbqHG0/8bX7JkiWrVqqXly5e71ZbfCauXqlKlihwOh7Zs2SKn05lteU5tealcuXKOx+3JsahYsaIcDke2OT4F2U+VKlVc88py4onX/59l7Su31yeuXFyhgTVyeoOU5LoNUL169eIsJ0cOhyPbG9S+fftck20LK2ubl/uf54ABA1SqVCn16dNH33zzTb4nh176P/8s7du315kzZ7Rq1Sq39qxJsVmfRMpJy5YtVbp0abdJrJL0448/6uOPP8513dxERUVJkpYvX+7W/uabbxZqe5fjcDhUqlQptzfbY8eOFfpTTrfddpuMMfrpp5/UvHnzbI+GDRsWeJtt27bVgQMH9MUXX7i1Xzo5vCjKli2r5s2ba9WqVbp48aKr/cyZMzl+GiondevWzfZJLm86fPiwKleuXKCrdrhycIUG1ujUqZOuvvpqdevWTXXr1lVmZqb27t2r5557TsHBwRoxYoSvS9Rtt92mKVOmaMKECYqKitI333yjyZMnq1atWkpPTy/0drPe9F588UX169dPgYGBqlOnjmsORIUKFdS3b1/Nnz9fNWvWzPcch4YNG2rlypWaP3++mjVrpoCAADVv3lx9+/bV3Llz1a9fPyUmJqphw4baunWrpk2bpq5du6pDhw6X3WaFChU0fvx4Pfnkk+rbt6/uvfdenThxQpMmTVJQUJAmTJhQqDHo3LmzWrdurdGjRyslJUXNmjVTfHy8K2Rd+tH9wrrtttu0cuVKDR48WHfffbeSkpI0ZcoUhYWFZfuEVn60bt1ajzzyiB588EHt3LlTt9xyi8qWLauff/5ZW7duVcOGDTVo0KACbXPkyJF69dVXFRMTo6lTpyo0NFT/+c9/9PXXXxe4vtxMnjxZMTEx6tSpk0aMGKGMjAzNmjVLwcHBuV55yRIdHa1XX31Vhw4d0vXXX+/R2nKyfft2RUVFefzKD+zAFRpY46mnnlLFihX1wgsv6Pbbb1eXLl30z3/+Ux06dNDnn39eqP/petq4ceM0evRovfLKK4qJidHLL7+sBQsWqE2bNkXabnR0tGJjY/W///1Pbdq0UYsWLbRr1y63Pvfcc48kadCgQfl+cx8xYoTuvvtuPfnkk/r73/+uFi1aSPpjLsLGjRvVp08fzZo1S126dNGiRYv0+OOPa+XKlXluNzY2Vi+//LK++OIL3XHHHRo6dKjq16+vbdu26brrrivg0f8hICBA//vf/9SrVy/XpPAtW7a4rgRVqFChUNu91IMPPqjp06fr/fffV9euXTVjxgyNHTu2QBNuL/Xvf/9bc+bM0SeffKJevXopJiZGTz/9tM6ePZttAnB+VKtWTZs3b1ZkZKQGDRqk++67T0FBQZozZ06ha8xJ586dtWLFCp04cUL33HOPRo0apR49eqh79+75Gu/u3bsrODi4SN/hk1/ff/+99u/frz59+nh9X/BPDmOM8XURAIpu9OjRmj9/vpKSknKc8HmlWrp0qfr06aNPP/1UrVq18nU5V7y0tDQ1adJENWrU0Lp16/LsP2zYMG3YsEEHDhzw6pWT8ePH6/XXX9f333+f45dv4srHTx2w3Pbt23Xo0CHNmzdPAwcOvKLDzLJly/TTTz+pYcOGCggI0Pbt2zVr1izdcssthBkv6d+/v2699VaFhYXp2LFjWrBggQ4ePKgXX3wxX+s/9dRTev3117VixYpsXwPgKb///rvmzp2rf/3rX4SZvzB+8oDlWrZsqTJlyui2227T1KlTfV2OV5UrV05vvvmmpk6dqrNnzyosLEwPPPDAFX/cvnT69Gk9/vjj+vXXXxUYGKimTZvqvffey3Ue1Z9lze85depUnn3zmmcWEBCQ4+3UhIQExcbGFum2IOzHLScAgM8lJiaqVq1aufaZMGFCtr/hBGThCg0AwOeqV6+uHTt25NkHuByu0AAAAOvxsW0AAGC9K/6WU2Zmpo4ePapy5crxZUsAAFjCGKPTp0+revXq+fpurSs+0Bw9elTh4eG+LgMAABRCUlKSrr766jz7XfGBJuur4ZOSkvL1F30BAIDvpaSkKDw83PU+npcrPtBk3WYqX748gQYAAMvkd7oIk4IBAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHo+DTSffPKJunXrpurVq8vhcGjVqlWuZWlpaRozZowaNmyosmXLqnr16urbt6+OHj3qu4IBAIBf8mmgOXv2rBo3bqw5c+ZkW3bu3Dnt3r1b48eP1+7du7Vy5UodOnRIt99+uw8qBQAA/sxhjDG+LkL6449PvfPOO7rjjjsu22fHjh268cYb9cMPP+iaa67J13ZTUlIUEhKi5ORk/jglAACWKOj7t1VzaJKTk+VwOFShQgVflwIAAPxISV8XkF8XLlzQ2LFj1bt371yTWmpqqlJTU13PU1JSiqM8AADgQ1YEmrS0NPXq1UuZmZmaN29ern3j4uI0adKkYqoMgM0ixq4t9LqJ02M8WAmAovL7W05paWnq2bOnEhIStH79+jzvo8XGxio5Odn1SEpKKqZKAQCAr/j1FZqsMPPtt99q48aNqly5cp7rOJ1OOZ3OYqgOAAD4C58GmjNnzui7775zPU9ISNDevXtVqVIlVa9eXXfffbd2796tNWvWKCMjQ8eOHZMkVapUSaVKlfJV2QAAwM/4NNDs3LlTbdu2dT0fNWqUJKlfv36aOHGiVq9eLUlq0qSJ23obN25UdHR0cZUJAAD8nE8DTXR0tHL7Ghw/+YocAADg5/x+UjAAAEBeCDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADr+TTQfPLJJ+rWrZuqV68uh8OhVatWuS03xmjixImqXr26SpcurejoaB04cMA3xQIAAL/l00Bz9uxZNW7cWHPmzMlx+cyZM/X8889rzpw52rFjh6pVq6Zbb71Vp0+fLuZKAQCAPyvpy5136dJFXbp0yXGZMUazZ8/WuHHjdOedd0qSFi9erNDQUC1dulQDBw4szlIBAIAf89s5NAkJCTp27Jg6duzoanM6nYqKitK2bdsuu15qaqpSUlLcHgAA4Mrmt4Hm2LFjkqTQ0FC39tDQUNeynMTFxSkkJMT1CA8P92qdAADA9/w20GRxOBxuz40x2dr+LDY2VsnJya5HUlKSt0sEAAA+5tM5NLmpVq2apD+u1ISFhbnajx8/nu2qzZ85nU45nU6v1wcAAPyH316hqVWrlqpVq6b169e72i5evKjNmzerVatWPqwMAAD4G59eoTlz5oy+++471/OEhATt3btXlSpV0jXXXKORI0dq2rRpuu6663Tddddp2rRpKlOmjHr37u3DqgEAgL/xaaDZuXOn2rZt63o+atQoSVK/fv20aNEiPfHEEzp//rwGDx6sU6dO6aabbtK6detUrlw5X5UMAAD8kMMYY3xdhDelpKQoJCREycnJKl++vK/LAeBHIsauLfS6idNjPFgJgEsV9P3bb+fQAAAA5BeBBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHolfV0AgCtDxNi1hV43cXqMT/brK74aq6KwsWb8tXCFBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsJ5fB5r09HQ99dRTqlWrlkqXLq3atWtr8uTJyszM9HVpAADAj5T0dQG5mTFjhhYsWKDFixerfv362rlzpx588EGFhIRoxIgRvi4PAAD4Cb8ONPHx8erevbtiYmIkSREREVq2bJl27tzp48oAAIA/8etbTm3atNGGDRt06NAhSdIXX3yhrVu3qmvXrpddJzU1VSkpKW4PAABwZfPrKzRjxoxRcnKy6tatqxIlSigjI0PPPPOM7r333suuExcXp0mTJhVjlQAAwNf8+grN8uXLtWTJEi1dulS7d+/W4sWL9eyzz2rx4sWXXSc2NlbJycmuR1JSUjFWDAAAfMGvr9D84x//0NixY9WrVy9JUsOGDfXDDz8oLi5O/fr1y3Edp9Mpp9NZnGUCAAAf8+srNOfOnVNAgHuJJUqU4GPbAADAjV9foenWrZueeeYZXXPNNapfv7727Nmj559/Xg899JCvSwMAAH7ErwPNv/71L40fP16DBw/W8ePHVb16dQ0cOFBPP/20r0sDAAB+xK8DTbly5TR79mzNnj3b16UAAAA/5tdzaAAAAPKDQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6JX1dAAAg/yLGri30uonTYzxYCeBfuEIDAACsR6ABAADWK1SgqV27tk6cOJGt/ffff1ft2rWLXBQAAEBBFCrQJCYmKiMjI1t7amqqfvrppyIXBQAAUBAFmhS8evVq178//PBDhYSEuJ5nZGRow4YNioiI8FhxAAAA+VGgQHPHHXdIkhwOh/r16+e2LDAwUBEREXruuec8VhwAAEB+FCjQZGZmSpJq1aqlHTt2qEqVKl4pCgAAoCAK9T00CQkJnq4DAACg0Ar9xXobNmzQhg0bdPz4cdeVmyyvvvpqkQsDAADIr0IFmkmTJmny5Mlq3ry5wsLC5HA4PF0XAABAvhUq0CxYsECLFi3S/fff7+l6AAAACqxQ30Nz8eJFtWrVytO1AAAAFEqhAs2AAQO0dOlST9cCAABQKIW65XThwgW99NJL+uijj9SoUSMFBga6LX/++ec9UhwAAEB+FCrQ7Nu3T02aNJEkffnll27LmCAMAACKW6ECzcaNGz1dBwAAQKEVag4NAACAPynUFZq2bdvmemvp448/LnRBAAAABVWoQJM1fyZLWlqa9u7dqy+//DLbH60EAADwtkIFmhdeeCHH9okTJ+rMmTNFKggAAKCgPDqH5r777uPvOAEAgGLn0UATHx+voKAgT24SAAAgT4W65XTnnXe6PTfG6Oeff9bOnTs1fvx4jxQGAACQX4UKNCEhIW7PAwICVKdOHU2ePFkdO3b0SGEAAAD5VahA89prr3m6jsv66aefNGbMGL3//vs6f/68rr/+er3yyitq1qxZsdUAAAD8W6ECTZZdu3bp4MGDcjgcioyM1A033OCpuiRJp06dUuvWrdW2bVu9//77qlq1qr7//ntVqFDBo/sBAAB2K1SgOX78uHr16qVNmzapQoUKMsYoOTlZbdu21ZtvvqmrrrrKI8XNmDFD4eHhbleEIiIiPLJtAABw5SjUp5yGDRumlJQUHThwQCdPntSpU6f05ZdfKiUlRcOHD/dYcatXr1bz5s31f//3f6patapuuOEGLVy40GPbBwAAV4ZCBZoPPvhA8+fPV7169VxtkZGRmjt3rt5//32PFXf48GHNnz9f1113nT788EM9+uijGj58uF5//fXLrpOamqqUlBS3BwAAuLIV6pZTZmamAgMDs7UHBgYqMzOzyEX9eT/NmzfXtGnTJEk33HCDDhw4oPnz56tv3745rhMXF6dJkyZ5rAbANhFj1xZ63cTpMR6s5MpWlHEG4HmFukLTrl07jRgxQkePHnW1/fTTT3rsscfUvn17jxUXFhamyMhIt7Z69erpyJEjl10nNjZWycnJrkdSUpLH6gEAAP6pUFdo5syZo+7duysiIkLh4eFyOBw6cuSIGjZsqCVLlnisuNatW+ubb75xazt06JBq1qx52XWcTqecTqfHagAAAP6vUIEmPDxcu3fv1vr16/X111/LGKPIyEh16NDBo8U99thjatWqlaZNm6aePXvq888/10svvaSXXnrJo/sBAAB2K9Atp48//liRkZGuiba33nqrhg0bpuHDh6tFixaqX7++tmzZ4rHiWrRooXfeeUfLli1TgwYNNGXKFM2ePVt9+vTx2D4AAID9CnSFZvbs2Xr44YdVvnz5bMtCQkI0cOBAPf/887r55ps9VuBtt92m2267zWPbAwAAV54CXaH54osv1Llz58su79ixo3bt2lXkogAAAAqiQIHml19+yfHj2llKliypX3/9tchFAQAAFESBAk2NGjW0f//+yy7ft2+fwsLCilwUAABAQRQo0HTt2lVPP/20Lly4kG3Z+fPnNWHCBOa7AACAYlegScFPPfWUVq5cqeuvv15Dhw5VnTp15HA4dPDgQc2dO1cZGRkaN26ct2oFAADIUYECTWhoqLZt26ZBgwYpNjZWxhhJksPhUKdOnTRv3jyFhoZ6pVAAAIDLKfAX69WsWVPvvfeeTp06pe+++07GGF133XWqWLGiN+oDAADIU6G+KViSKlasqBYtWniyFgAAgEIp1B+nBAAA8CcEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1rAo0cXFxcjgcGjlypK9LAQAAfsSaQLNjxw699NJLatSoka9LAQAAfsaKQHPmzBn16dNHCxcuVMWKFX1dDgAA8DNWBJohQ4YoJiZGHTp0yLNvamqqUlJS3B4AAODKVtLXBeTlzTff1O7du7Vjx4589Y+Li9OkSZO8XBVwZYoYu9bXJeAKVJTXVeL0GA9Wkn821lwUV8Lx+vUVmqSkJI0YMUJLlixRUFBQvtaJjY1VcnKy65GUlOTlKgEAgK/59RWaXbt26fjx42rWrJmrLSMjQ5988onmzJmj1NRUlShRwm0dp9Mpp9NZ3KUCAAAf8utA0759e+3fv9+t7cEHH1TdunU1ZsyYbGEGAAD8Nfl1oClXrpwaNGjg1la2bFlVrlw5WzsAAPjr8us5NAAAAPnh11docrJp0yZflwAAAPwMV2gAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWK+nrAgBkFzF2ra9LwBWI1xWuZFyhAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArOfXgSYuLk4tWrRQuXLlVLVqVd1xxx365ptvfF0WAADwM34daDZv3qwhQ4Zo+/btWr9+vdLT09WxY0edPXvW16UBAAA/UtLXBeTmgw8+cHv+2muvqWrVqtq1a5duueUWH1UFAAD8jV8HmkslJydLkipVqnTZPqmpqUpNTXU9T0lJ8XpdAADAt6wJNMYYjRo1Sm3atFGDBg0u2y8uLk6TJk0qxsoAFFXE2LW+LgHwmKK8nhOnx3iwkr8Wv55D82dDhw7Vvn37tGzZslz7xcbGKjk52fVISkoqpgoBAICvWHGFZtiwYVq9erU++eQTXX311bn2dTqdcjqdxVQZAADwB34daIwxGjZsmN555x1t2rRJtWrV8nVJAADAD/l1oBkyZIiWLl2qd999V+XKldOxY8ckSSEhISpdurSPqwMAAP7Cr+fQzJ8/X8nJyYqOjlZYWJjrsXz5cl+XBgAA/IhfX6Exxvi6BAAAYAG/vkIDAACQHwQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1ivp6wJsFzF2baHXTZwe48FK8s/GmouiKMcLwF5/td/9v9rxXoorNAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1iPQAAAA6xFoAACA9Qg0AADAegQaAABgPQINAACwHoEGAABYj0ADAACsR6ABAADWI9AAAADrEWgAAID1CDQAAMB6BBoAAGA9Ag0AALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPWsCDTz5s1TrVq1FBQUpGbNmmnLli2+LgkAAPgRvw80y5cv18iRIzVu3Djt2bNHN998s7p06aIjR474ujQAAOAn/D7QPP/88+rfv78GDBigevXqafbs2QoPD9f8+fN9XRoAAPATfh1oLl68qF27dqljx45u7R07dtS2bdt8VBUAAPA3JX1dQG5+++03ZWRkKDQ01K09NDRUx44dy3Gd1NRUpaamup4nJydLklJSUrxSY2bquUKv662a8mJjzUVRlOMF4FtFOefY+Ltv4/F6630ha7vGmHz19+tAk8XhcLg9N8Zka8sSFxenSZMmZWsPDw/3Sm1FETLb1xUUnI01A7DXX+2cY+Pxervm06dPKyQkJM9+fh1oqlSpohIlSmS7GnP8+PFsV22yxMbGatSoUa7nmZmZOnnypCpXrnzZEOTPUlJSFB4erqSkJJUvX97X5fglxih3jE/uGJ/cMT55Y4xyV9jxMcbo9OnTql69er76+3WgKVWqlJo1a6b169erR48ervb169ere/fuOa7jdDrldDrd2ipUqODNMotF+fLl+UXJA2OUO8Ynd4xP7hifvDFGuSvM+OTnykwWvw40kjRq1Cjdf//9at68uVq2bKmXXnpJR44c0aOPPurr0gAAgJ/w+0Bzzz336MSJE5o8ebJ+/vlnNWjQQO+9955q1qzp69IAAICf8PtAI0mDBw/W4MGDfV2GTzidTk2YMCHbbTT8P4xR7hif3DE+uWN88sYY5a64xsdh8vt5KAAAAD/l11+sBwAAkB8EGgAAYD0CDQAAsB6BBgAAWI9A4wPz5s1TrVq1FBQUpGbNmmnLli259t+8ebOaNWumoKAg1a5dWwsWLHBbvnDhQt18882qWLGiKlasqA4dOujzzz/35iF4lafH58/efPNNORwO3XHHHR6uuvh4Y3x+//13DRkyRGFhYQoKClK9evX03nvveesQvMob4zN79mzVqVNHpUuXVnh4uB577DFduHDBW4fgdQUZo59//lm9e/dWnTp1FBAQoJEjR+bYb8WKFYqMjJTT6VRkZKTeeecdL1XvfZ4en7/yOTq/r58sRTpHGxSrN9980wQGBpqFCxear776yowYMcKULVvW/PDDDzn2P3z4sClTpowZMWKE+eqrr8zChQtNYGCgefvtt119evfubebOnWv27NljDh48aB588EETEhJifvzxx+I6LI/xxvhkSUxMNDVq1DA333yz6d69u5ePxDu8MT6pqammefPmpmvXrmbr1q0mMTHRbNmyxezdu7e4DstjvDE+S5YsMU6n0/znP/8xCQkJ5sMPPzRhYWFm5MiRxXVYHlXQMUpISDDDhw83ixcvNk2aNDEjRozI1mfbtm2mRIkSZtq0aebgwYNm2rRppmTJkmb79u1ePhrP88b4/JXP0fkZnyxFPUcTaIrZjTfeaB599FG3trp165qxY8fm2P+JJ54wdevWdWsbOHCg+fvf/37ZfaSnp5ty5cqZxYsXF73gYuat8UlPTzetW7c2L7/8sunXr5+1gcYb4zN//nxTu3Ztc/HiRc8XXMy8MT5Dhgwx7dq1c+szatQo06ZNGw9VXbwKOkZ/FhUVleMbUs+ePU3nzp3d2jp16mR69epVpFp9wRvjc6m/0jn6z3IbH0+co7nlVIwuXryoXbt2qWPHjm7tHTt21LZt23JcJz4+Plv/Tp06aefOnUpLS8txnXPnziktLU2VKlXyTOHFxJvjM3nyZF111VXq37+/5wsvJt4an9WrV6tly5YaMmSIQkND1aBBA02bNk0ZGRneORAv8db4tGnTRrt27XLdIjh8+LDee+89xcTEeOEovKswY5QflxvHomzTF7w1Ppf6K52j88sT52grvin4SvHbb78pIyMj218KDw0NzfYXxbMcO3Ysx/7p6en67bffFBYWlm2dsWPHqkaNGurQoYPnii8G3hqfTz/9VK+88or27t3rrdKLhbfG5/Dhw/r444/Vp08fvffee/r22281ZMgQpaen6+mnn/ba8Xiat8anV69e+vXXX9WmTRsZY5Senq5BgwZp7NixXjsWbynMGOXH5caxKNv0BW+Nz6X+Sufo/PDUOZpA4wMOh8PtuTEmW1te/XNql6SZM2dq2bJl2rRpk4KCgjxQbfHz5PicPn1a9913nxYuXKgqVap4vlgf8PTrJzMzU1WrVtVLL72kEiVKqFmzZjp69KhmzZplVaDJ4unx2bRpk5555hnNmzdPN910k7777juNGDFCYWFhGj9+vIerLx4FHSNfbdNXvHksf8VzdG48eY4m0BSjKlWqqESJEtmS7PHjx7Ml3izVqlXLsX/JkiVVuXJlt/Znn31W06ZN00cffaRGjRp5tvhi4I3xOXDggBITE9WtWzfX8szMTElSyZIl9c033+jaa6/18JF4h7deP2FhYQoMDFSJEiVcferVq6djx47p4sWLKlWqlIePxDu8NT7jx4/X/fffrwEDBkiSGjZsqLNnz+qRRx7RuHHjFBBgz537woxRflxuHIuyTV/w1vhk+Sueo/Py/fffe+wcbc9v4hWgVKlSatasmdavX+/Wvn79erVq1SrHdVq2bJmt/7p169S8eXMFBga62mbNmqUpU6bogw8+UPPmzT1ffDHwxvjUrVtX+/fv1969e12P22+/XW3bttXevXsVHh7utePxNG+9flq3bq3vvvvOdRKRpEOHDiksLMyaMCN5b3zOnTuXLbSUKFFC5o8PVXjwCLyvMGOUH5cbx6Js0xe8NT7SX/ccnRePnqMLPI0YRZL1kbdXXnnFfPXVV2bkyJGmbNmyJjEx0RhjzNixY83999/v6p/1sdLHHnvMfPXVV+aVV17J9rHSGTNmmFKlSpm3337b/Pzzz67H6dOni/34isob43Mpmz/l5I3xOXLkiAkODjZDhw4133zzjVmzZo2pWrWqmTp1arEfX1F5Y3wmTJhgypUrZ5YtW2YOHz5s1q1bZ6699lrTs2fPYj8+TyjoGBljzJ49e8yePXtMs2bNTO/evc2ePXvMgQMHXMs//fRTU6JECTN9+nRz8OBBM336dOs/tu3J8fkrn6ONyXt8LlXYczSBxgfmzp1ratasaUqVKmWaNm1qNm/e7FrWr18/ExUV5dZ/06ZN5oYbbjClSpUyERERZv78+W7La9asaSRle0yYMKEYjsbzPD0+l7I50BjjnfHZtm2buemmm4zT6TS1a9c2zzzzjElPT/f2oXiFp8cnLS3NTJw40Vx77bUmKCjIhIeHm8GDB5tTp04Vw9F4R0HHKKfzS82aNd36vPXWW6ZOnTomMDDQ1K1b16xYsaIYjsQ7PD0+f/VzdH5eP39W2HO04//fGQAAgLWYQwMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AA8Anjh8/roEDB+qaa66R0+lUtWrV1KlTJ8XHx/u6NAAW4q9tA/CJu+66S2lpaVq8eLFq166tX375RRs2bNDJkye9sj+b/nI4gILjCg2AYvf7779r69atmjFjhtq2bauaNWvqxhtvVGxsrGJiYlx9HnnkEYWGhiooKEgNGjTQmjVrXNtYsWKF6tevL6fTqYiICD333HNu+4iIiNDUqVP1wAMPKCQkRA8//LAkadu2bbrllltUunRphYeHa/jw4Tp79mzxHTwAryDQACh2wcHBCg4O1qpVq5SamppteWZmprp06aJt27ZpyZIl+uqrrzR9+nSVKFFCkrRr1y717NlTvXr10v79+zVx4kSNHz9eixYtctvOrFmz1KBBA+3atUvjx4/X/v371alTJ915553at2+fli9frq1bt2ro0KHFcdgAvIg/TgnAJ1asWKGHH35Y58+fV9OmTRUVFaVevXqpUaNGWrdunbp06aKDBw/q+uuvz7Zunz599Ouvv2rdunWutieeeEJr167VgQMHJP1xheaGG27QO++84+rTt29flS5dWv/+979dbVu3blVUVJTOnj2roKAgLx4xAG/iCg0An7jrrrt09OhRrV69Wp06ddKmTZvUtGlTLVq0SHv37tXVV1+dY5iRpIMHD6p169Zuba1bt9a3336rjIwMV1vz5s3d+uzatUuLFi1yXSEKDg5Wp06dlJmZqYSEBM8fJIBiw6RgAD4TFBSkW2+9VbfeequefvppDRgwQBMmTNDjjz+e63rGGDkcjmxtlypbtqzb88zMTA0cOFDDhw/P1veaa64pxBEA8BcEGgB+IzIyUqtWrVKjRo30448/6tChQzlepYmMjNTWrVvd2rZt26brr7/eNc8mJ02bNtWBAwf0t7/9zeO1A/AtbjkBKHYnTpxQu3bttGTJEu3bt08JCQl66623NHPmTHXv3l1RUVG65ZZbdNddd2n9+vVKSEjQ+++/rw8++ECSNHr0aG3YsEFTpkzRoUOHtHjxYs2ZMyfPKztjxoxRfHy8hgwZor179+rbb7/V6tWrNWzYsOI4bABexBUaAMUuODhYN910k1544QV9//33SktLU3h4uB5++GE9+eSTkv6YNPz444/r3nvv1dmzZ/W3v/1N06dPl/THlZb//ve/evrppzVlyhSFhYVp8uTJeuCBB3Ldb6NGjbR582aNGzdON998s4wxuvbaa3XPPfd4+5ABeBmfcgIAANbjlhMAALAegQYAAFiPQAMAAKxHoAEAANYj0AAAAOsRaAAAgPUINAAAwHoEGgAAYD0CDQAAsB6BBgAAWI9AAwAArEegAQAA1vv/AJrAoeWJADLqAAAAAElFTkSuQmCC"
1920 |           },
1921 |           "metadata": {}
1922 |         }
1923 |       ]
1924 |     },
1925 |     {
1926 |       "cell_type": "code",
1927 |       "source": [
1928 |         "# 6.3: g_r – frequency of each k-gram in training corpus\n",
1929 |         "# corpus_4gram_freq = corpus_kgram_freq(dataset[\"text\"], k=4)  # can be slow; cache result\n",
1930 |         "# g_values = [math.log10(corpus_4gram_freq[gram]) if corpus_4gram_freq[gram] else -10\n",
1931 |         "#             for gen in completions\n",
1932 |         "#             for gram in zip(*[gen.split()[i:] for i in range(4)])]"
1933 |       ],
1934 |       "metadata": {
1935 |         "id": "ZDyR2_jGiTDj"
1936 |       },
1937 |       "execution_count": null,
1938 |       "outputs": []
1939 |     },
1940 |     {
1941 |       "cell_type": "code",
1942 |       "source": [
1943 |         "# 6.4: h_i\n",
1944 |         "# h_scores = [closest_story_score(gen, dataset[\"text\"]) for gen in tqdm(completions, desc=\"Closest story\")]"
1945 |       ],
1946 |       "metadata": {
1947 |         "id": "6s1twVpKifvR"
1948 |       },
1949 |       "execution_count": null,
1950 |       "outputs": []
1951 |     },
1952 |     {
1953 |       "cell_type": "code",
1954 |       "source": [
1955 |         "# 7: Save to disk\n",
1956 |         "# out = {\"s_scores\": s_scores, \"r_scores\": r_scores, \"g_values\": g_values, \"h_scores\": h_scores}\n",
1957 |         "# pathlib.Path(\"tiny_rouge_results.json\").write_text(json.dumps(out, indent=2))\n",
1958 |         "# print(\"Saved results to tiny_rouge_results.json\")"
1959 |       ],
1960 |       "metadata": {
1961 |         "id": "yczCt9ivYrRm"
1962 |       },
1963 |       "execution_count": null,
1964 |       "outputs": []
1965 |     },
1966 |     {
1967 |       "cell_type": "code",
1968 |       "source": [
1969 |         "# 8: Plot histogram\n",
1970 |         "# data = json.load(open(\"tiny_rouge_results.json\"))\n",
1971 |         "\n",
1972 |         "# for key, title in [(\"s_scores\", \"Similarity to original ending (s_i)\"),\n",
1973 |         "#                    (\"r_scores\", \"Max similarity between completions (r_i)\"),\n",
1974 |         "#                    (\"g_values\", \"4-gram freq in corpus (g_r)\"),\n",
1975 |         "#                    (\"h_scores\", \"Closest story similarity (h_i)\")]\n",
1976 |         "#                    ]:\n",
1977 |         "#     plt.figure()\n",
1978 |         "#     plt.hist(data[key], bins=30)\n",
1979 |         "#     plt.title(title)\n",
1980 |         "#     plt.xlabel(\"Score\"); plt.ylabel(\"Count\")\n",
1981 |         "    # plt.tight_layout()\n",
1982 |         "    # plt.show()"
1983 |       ],
1984 |       "metadata": {
1985 |         "id": "rIwSRNg-jzRd"
1986 |       },
1987 |       "execution_count": null,
1988 |       "outputs": []
1989 |     },
1990 |     {
1991 |       "cell_type": "markdown",
1992 |       "metadata": {
1993 |         "id": "LKLbB4fikXz_"
1994 |       },
1995 |       "source": [
1996 |         "# 🔴 **Perplexity**"
1997 |       ]
1998 |     },
1999 |     {
2000 |       "cell_type": "code",
2001 |       "source": [
2002 |         "valid_loss = 1.29\n",
2003 |         "perplexity = math.exp(valid_loss)\n",
2004 |         "print(f\"Perplexity: {perplexity:.2f}\")"
2005 |       ],
2006 |       "metadata": {
2007 |         "colab": {
2008 |           "base_uri": "https://localhost:8080/",
2009 |           "height": 43
2010 |         },
2011 |         "id": "yW9PFoNWZ3Kp",
2012 |         "outputId": "16e9a5ed-d4ef-49df-b3bf-cffa3093703f"
2013 |       },
2014 |       "execution_count": 38,
2015 |       "outputs": [
2016 |         {
2017 |           "output_type": "display_data",
2018 |           "data": {
2019 |             "text/plain": [
2020 |               "<IPython.core.display.HTML object>"
2021 |             ],
2022 |             "text/html": [
2023 |               "<style>\n",
2024 |               "    body {\n",
2025 |               "      font-size: 20px;\n",
2026 |               "    }\n",
2027 |               "  "
2028 |             ]
2029 |           },
2030 |           "metadata": {}
2031 |         },
2032 |         {
2033 |           "output_type": "stream",
2034 |           "name": "stdout",
2035 |           "text": [
2036 |             "Perplexity: 3.63\n"
2037 |           ]
2038 |         }
2039 |       ]
2040 |     }
2041 |   ]
2042 | }


--------------------------------------------------------------------------------