├── .gitignore ├── AC_data_input.npy ├── AC_data_output.npy ├── Applications with NN Surrogates.ipynb ├── Autodifferentiation ├── Autodifferentiation - solution.ipynb ├── Autodifferentiation.ipynb ├── engine.py └── engine_solution.py ├── Common.py ├── Cross Validation.ipynb ├── DeepONet.ipynb ├── FNO.ipynb ├── IntroToML.ipynb ├── LICENSE ├── Pinns.ipynb ├── PinnsHD.ipynb ├── PinnsInv.ipynb ├── README.md ├── antiderivative_aligned_test.npz ├── antiderivative_aligned_train.npz └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .ipynb_checkpoints/ 3 | 4 | __pycache__/ 5 | 6 | .DS_Store 7 | 8 | .idea 9 | -------------------------------------------------------------------------------- /AC_data_input.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mroberto166/CAMLab-DLSCTutorials/c2ca37b6e711e3691ed95f421ffc10b6d1ad1a6f/AC_data_input.npy -------------------------------------------------------------------------------- /AC_data_output.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mroberto166/CAMLab-DLSCTutorials/c2ca37b6e711e3691ed95f421ffc10b6d1ad1a6f/AC_data_output.npy -------------------------------------------------------------------------------- /Applications with NN Surrogates.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "Cg3aekRznPPM", 7 | "pycharm": { 8 | "name": "#%% md\n" 9 | } 10 | }, 11 | "source": [ 12 | "## Practical Applications with Neural Network Surrogates\n", 13 | "\n", 14 | "General idea of most applications: replace high fidelity model, usually correspoding to the solution of a PDE, with the network model trained with a set generate by solving the PDE $N$ times." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "outputs": [], 21 | "source": [ 22 | "import numpy as np\n", 23 | "import matplotlib.pyplot as plt\n", 24 | "import torch.optim as optim\n", 25 | "import torch\n", 26 | "from torch.utils.data import DataLoader\n", 27 | "import seaborn as sb\n", 28 | "\n", 29 | "# Adapt this import to your specific directory tree.\n", 30 | "from Common import NeuralNet, fit\n", 31 | "torch.manual_seed(42)" 32 | ], 33 | "metadata": { 34 | "collapsed": false, 35 | "pycharm": { 36 | "name": "#%%\n" 37 | } 38 | } 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": { 44 | "id": "1bnu6a6ynPPP", 45 | "pycharm": { 46 | "is_executing": false, 47 | "name": "#%%\n" 48 | } 49 | }, 50 | "outputs": [], 51 | "source": [ 52 | "def initial_condition(x, y):\n", 53 | " # Generate perturbed initial condition.\n", 54 | " ic = torch.zeros_like(x)\n", 55 | " for i in range(y.shape[0]):\n", 56 | " ic = ic + y[i] * torch.sin((2 + i) * np.pi * x) \n", 57 | " ic = ic / torch.mean(ic ** 2)**0.5\n", 58 | " return ic\n", 59 | "\n", 60 | "\n", 61 | "def solve_heat_eq(y):\n", 62 | " # http://hplgit.github.io/num-methods-for-PDEs/doc/pub/diffu/html/._diffu-solarized001.html\n", 63 | " # Typical heat equation solver.\n", 64 | " nx = 51\n", 65 | " x = torch.linspace(0,1, nx)\n", 66 | " T = 0.01\n", 67 | " ic = initial_condition(x,y)\n", 68 | "\n", 69 | " diff = 1\n", 70 | " dx = x[1] - x[0]\n", 71 | " dt = 0.5 * dx ** 2 / diff\n", 72 | "\n", 73 | " F = diff * dt / dx ** 2\n", 74 | " nt = int((T / dt))\n", 75 | " nx = x.shape[0]\n", 76 | "\n", 77 | " u_old = torch.clone(ic)\n", 78 | " u_new = torch.zeros_like(ic)\n", 79 | "\n", 80 | " for k in range(1, nt):\n", 81 | " for i in range(1, nx - 1):\n", 82 | " u_new[i] = (\n", 83 | " u_old[i] + F * (u_old[i + 1] - 2 * u_old[i] + u_old[i - 1]))\n", 84 | " u_new[0] = 0\n", 85 | " u_new[-1] = 0\n", 86 | "\n", 87 | " u_old[:] = u_new\n", 88 | "\n", 89 | " flux = -diff * (u_new[0] - u_new[1]) / dx\n", 90 | " return flux, (x, ic, u_new)\n" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 3, 96 | "metadata": { 97 | "colab": { 98 | "base_uri": "https://localhost:8080/", 99 | "height": 747 100 | }, 101 | "executionInfo": { 102 | "elapsed": 11570, 103 | "status": "ok", 104 | "timestamp": 1682413827284, 105 | "user": { 106 | "displayName": "Victor Armegioiu", 107 | "userId": "10988735183621484296" 108 | }, 109 | "user_tz": -120 110 | }, 111 | "id": "asOH7rSgnPPP", 112 | "outputId": "9de670da-ad89-4383-aca8-969e031d2f42", 113 | "pycharm": { 114 | "is_executing": false, 115 | "name": "#%%\n" 116 | } 117 | }, 118 | "outputs": [ 119 | { 120 | "name": "stdout", 121 | "output_type": "stream", 122 | "text": [ 123 | "###############################\n", 124 | "Generating Training Set\n", 125 | "###############################\n" 126 | ] 127 | }, 128 | { 129 | "ename": "KeyboardInterrupt", 130 | "evalue": "", 131 | "output_type": "error", 132 | "traceback": [ 133 | "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", 134 | "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", 135 | "Cell \u001B[0;32mIn[3], line 26\u001B[0m\n\u001B[1;32m 24\u001B[0m axs[\u001B[38;5;241m1\u001B[39m]\u001B[38;5;241m.\u001B[39mgrid(\u001B[38;5;28;01mTrue\u001B[39;00m, which\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mboth\u001B[39m\u001B[38;5;124m\"\u001B[39m, ls\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m:\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m j \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(n_samples):\n\u001B[0;32m---> 26\u001B[0m f, (x, ic, u_end) \u001B[38;5;241m=\u001B[39m \u001B[43msolve_heat_eq\u001B[49m\u001B[43m(\u001B[49m\u001B[43my\u001B[49m\u001B[43m[\u001B[49m\u001B[43mj\u001B[49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 27\u001B[0m training_set[j, :d] \u001B[38;5;241m=\u001B[39m y[j]\n\u001B[1;32m 28\u001B[0m training_set[j, \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m] \u001B[38;5;241m=\u001B[39m f\n", 136 | "Cell \u001B[0;32mIn[2], line 32\u001B[0m, in \u001B[0;36msolve_heat_eq\u001B[0;34m(y)\u001B[0m\n\u001B[1;32m 29\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m k \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(\u001B[38;5;241m1\u001B[39m, nt):\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m i \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(\u001B[38;5;241m1\u001B[39m, nx \u001B[38;5;241m-\u001B[39m \u001B[38;5;241m1\u001B[39m):\n\u001B[1;32m 31\u001B[0m u_new[i] \u001B[38;5;241m=\u001B[39m (\n\u001B[0;32m---> 32\u001B[0m \u001B[43mu_old\u001B[49m\u001B[43m[\u001B[49m\u001B[43mi\u001B[49m\u001B[43m]\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mF\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43m \u001B[49m\u001B[43m(\u001B[49m\u001B[43mu_old\u001B[49m\u001B[43m[\u001B[49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m2\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mu_old\u001B[49m\u001B[43m[\u001B[49m\u001B[43mi\u001B[49m\u001B[43m]\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mu_old\u001B[49m\u001B[43m[\u001B[49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m)\n\u001B[1;32m 33\u001B[0m u_new[\u001B[38;5;241m0\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 34\u001B[0m u_new[\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n", 137 | "\u001B[0;31mKeyboardInterrupt\u001B[0m: " 138 | ] 139 | } 140 | ], 141 | "source": [ 142 | "print(\"###############################\")\n", 143 | "print(\"Generating Training Set\")\n", 144 | "torch.manual_seed(12446)\n", 145 | "n_samples = 100\n", 146 | "sobol = False\n", 147 | "min_inputs = -5\n", 148 | "max_inputs = 5\n", 149 | "d = 2\n", 150 | "# Inputs for generating the training set are in [-5,5]\n", 151 | "if not sobol:\n", 152 | " y_rand = torch.rand((n_samples, d))\n", 153 | " y = (max_inputs - min_inputs) * y_rand + min_inputs\n", 154 | "else:\n", 155 | " sob_eng = torch.quasirandom.SobolEngine(d)\n", 156 | " sob_eng.fast_forward(1)\n", 157 | " y_sob = sob_eng.draw(n_samples)\n", 158 | " y = (max_inputs - min_inputs)*y_sob + min_inputs\n", 159 | "training_set = torch.zeros((n_samples, y.shape[1] + 1))\n", 160 | "\n", 161 | "\n", 162 | "print(\"###############################\")\n", 163 | "fig, axs = plt.subplots(2, dpi=150)\n", 164 | "axs[0].grid(True, which=\"both\", ls=\":\")\n", 165 | "axs[1].grid(True, which=\"both\", ls=\":\")\n", 166 | "for j in range(n_samples):\n", 167 | " f, (x, ic, u_end) = solve_heat_eq(y[j])\n", 168 | " training_set[j, :d] = y[j]\n", 169 | " training_set[j, -1] = f\n", 170 | " axs[0].plot(x, ic)\n", 171 | " axs[1].plot(x, u_end)\n", 172 | " \n", 173 | "axs[0].set(ylabel='u')\n", 174 | "axs[1].set(xlabel='x', ylabel='u')\n", 175 | "axs[0].set_title(\"Initial Condition\")\n", 176 | "axs[1].set_title(\"Solution at T = 0.01\")\n", 177 | "for ax in fig.get_axes():\n", 178 | " ax.label_outer()" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": { 185 | "id": "mpAgg2gKnPPQ", 186 | "pycharm": { 187 | "is_executing": false, 188 | "name": "#%%\n" 189 | } 190 | }, 191 | "outputs": [], 192 | "source": [ 193 | "# Setup dataset for training.\n", 194 | "if not sobol:\n", 195 | " inputs = y_rand\n", 196 | "else:\n", 197 | " inputs = y_sob\n", 198 | "output = training_set[:, -1].reshape(-1, 1)\n", 199 | "\n", 200 | "batch_size = inputs.shape[0]\n", 201 | "training_set_loader = DataLoader(\n", 202 | " torch.utils.data.TensorDataset(inputs, output), \n", 203 | " batch_size=batch_size,\n", 204 | " shuffle=True)" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": { 210 | "id": "BOKMPX_N0Bhw", 211 | "pycharm": { 212 | "name": "#%% md\n" 213 | } 214 | }, 215 | "source": [ 216 | "## Neural Surrogates\n", 217 | "\n", 218 | "Since actually solving PDEs from scratch for every perturbation of the initial condition is quite an expensive process, we shall instead opt for a neural approximation. \n", 219 | "\n", 220 | "\n", 221 | "This amounts to learning a neural approximant (parameterized by $\\theta$)\n", 222 | "\n", 223 | "$$\n", 224 | "\\hat{L}_\\theta: y \\mapsto \\hat{L}_\\theta^\\Delta(y), \\quad y\\in[0,1]^d\n", 225 | "$$\n", 226 | "\n", 227 | "which is trained to map $y$'s to pre-computed observables at a pre-determined number of query points (i.e. your dataset).\n" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": { 234 | "colab": { 235 | "base_uri": "https://localhost:8080/" 236 | }, 237 | "executionInfo": { 238 | "elapsed": 17824, 239 | "status": "ok", 240 | "timestamp": 1682416020638, 241 | "user": { 242 | "displayName": "Victor Armegioiu", 243 | "userId": "10988735183621484296" 244 | }, 245 | "user_tz": -120 246 | }, 247 | "id": "E6RkEnR6zjgs", 248 | "outputId": "ecc1cf4c-cf4c-4baa-a124-3754c06b2760", 249 | "pycharm": { 250 | "name": "#%%\n" 251 | } 252 | }, 253 | "outputs": [], 254 | "source": [ 255 | "# Setup model and optimizer.\n", 256 | "model = NeuralNet(input_dimension=inputs.shape[1], \n", 257 | " output_dimension=output.shape[1], \n", 258 | " n_hidden_layers=4, \n", 259 | " neurons=20, \n", 260 | " regularization_param=0.0, \n", 261 | " regularization_exp=2,\n", 262 | " retrain_seed=128)\n", 263 | "\n", 264 | "optimizer_ = optim.LBFGS(\n", 265 | " model.parameters(),\n", 266 | " lr=0.1, max_iter=1,\n", 267 | " max_eval=50000,\n", 268 | " tolerance_change=1.0 * np.finfo(float).eps\n", 269 | ")\n", 270 | "\n", 271 | "n_epochs = 2500\n", 272 | "history = fit(\n", 273 | " model, training_set_loader, n_epochs, optimizer_, p=2, verbose=False)\n", 274 | "print(\"Final Training loss: \", history[-1])" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": { 281 | "colab": { 282 | "base_uri": "https://localhost:8080/", 283 | "height": 449 284 | }, 285 | "executionInfo": { 286 | "elapsed": 534, 287 | "status": "ok", 288 | "timestamp": 1682413963824, 289 | "user": { 290 | "displayName": "Victor Armegioiu", 291 | "userId": "10988735183621484296" 292 | }, 293 | "user_tz": -120 294 | }, 295 | "id": "zA_eI7fKrwC5", 296 | "outputId": "6da3f959-7acf-4d1a-c2fc-f56c0856cd25", 297 | "pycharm": { 298 | "name": "#%%\n" 299 | } 300 | }, 301 | "outputs": [], 302 | "source": [ 303 | "plt.figure()\n", 304 | "plt.grid(True, which=\"both\", ls=\":\")\n", 305 | "plt.plot(np.arange(1, n_epochs + 1), np.log10(history), label=\"Train Loss\")\n", 306 | "plt.xlabel(\"Epochs\")\n", 307 | "plt.legend()\n", 308 | "plt.show()" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "metadata": { 315 | "colab": { 316 | "base_uri": "https://localhost:8080/" 317 | }, 318 | "executionInfo": { 319 | "elapsed": 850372, 320 | "status": "ok", 321 | "timestamp": 1682414817633, 322 | "user": { 323 | "displayName": "Victor Armegioiu", 324 | "userId": "10988735183621484296" 325 | }, 326 | "user_tz": -120 327 | }, 328 | "id": "dQdsctCCnPPQ", 329 | "outputId": "00b526d3-3ee0-4a43-e819-624e15832a10", 330 | "pycharm": { 331 | "is_executing": false, 332 | "name": "#%%\n" 333 | } 334 | }, 335 | "outputs": [], 336 | "source": [ 337 | "def generate_test_set(n_samples):\n", 338 | " torch.manual_seed(34)\n", 339 | " inputs_ = (\n", 340 | " (max_inputs - min_inputs) * torch.rand((n_samples, d)) + min_inputs)\n", 341 | " s_ = torch.zeros((n_samples, d + 1))\n", 342 | "\n", 343 | " print(\"###############################\")\n", 344 | " print(\"Generating Test Set\")\n", 345 | " for j in range(n_samples):\n", 346 | " s_[j, :d] = inputs_[j]\n", 347 | " s_[j, -1], _ = solve_heat_eq(inputs_[j])\n", 348 | "\n", 349 | " return s_\n", 350 | "\n", 351 | "test_set = generate_test_set(10000)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "metadata": { 358 | "colab": { 359 | "base_uri": "https://localhost:8080/" 360 | }, 361 | "executionInfo": { 362 | "elapsed": 261, 363 | "status": "ok", 364 | "timestamp": 1682416405878, 365 | "user": { 366 | "displayName": "Victor Armegioiu", 367 | "userId": "10988735183621484296" 368 | }, 369 | "user_tz": -120 370 | }, 371 | "id": "Id8okqJhnPPQ", 372 | "outputId": "73f0c7d1-cb62-418c-9029-e70484565ffa", 373 | "pycharm": { 374 | "is_executing": false, 375 | "name": "#%%\n" 376 | } 377 | }, 378 | "outputs": [], 379 | "source": [ 380 | "test_inputs = test_set[:, :d]\n", 381 | "test_output = test_set[:, -1]\n", 382 | "\n", 383 | "test_inputs_scaled = (test_inputs - min_inputs)/(max_inputs - min_inputs)\n", 384 | "\n", 385 | "test_pred = model(test_inputs_scaled).reshape(-1, ) \n", 386 | "err = (\n", 387 | " torch.mean(\n", 388 | " (test_output - test_pred) ** 2) / torch.mean(test_output ** 2)) ** 0.5\n", 389 | "\n", 390 | "print(\"Error Model : \", err.item())" 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": null, 396 | "metadata": { 397 | "id": "rlnXYDiMnPPQ", 398 | "outputId": "19b003b1-551c-46f8-cd15-a2e1f10b2114", 399 | "pycharm": { 400 | "is_executing": false, 401 | "name": "#%%\n" 402 | } 403 | }, 404 | "outputs": [], 405 | "source": [ 406 | "y1=torch.linspace(-5,5, 10000)\n", 407 | "fig, axs = plt.subplots(2, figsize=(8,12), dpi=200)\n", 408 | "im1 = axs[0].scatter(test_inputs[:,0], test_inputs[:,1], c=test_output)\n", 409 | "axs[0].set_xlabel(\"y1\")\n", 410 | "axs[0].set_ylabel(\"y2\")\n", 411 | "plt.colorbar(im1,ax=axs[0])\n", 412 | "axs[0].plot(y1,y1, color=\"grey\", ls=\":\")\n", 413 | "axs[0].grid(True, which=\"both\", ls=\":\")\n", 414 | "im2 = axs[1].scatter(test_inputs[:,0], test_inputs[:,1], c=test_pred.detach())\n", 415 | "axs[1].set_xlabel(\"y1\")\n", 416 | "axs[1].set_ylabel(\"y2\")\n", 417 | "plt.colorbar(im2,ax=axs[1])\n", 418 | "axs[1].plot(y1,y1, color=\"grey\", ls=\":\")\n", 419 | "axs[1].grid(True, which=\"both\", ls=\":\")\n", 420 | "axs[0].set_title(\"Exact Solution\")\n", 421 | "axs[1].set_title(\"Approximate Solution\")" 422 | ] 423 | }, 424 | { 425 | "cell_type": "markdown", 426 | "metadata": { 427 | "id": "EebF-3l1nPPR", 428 | "pycharm": { 429 | "name": "#%% md\n" 430 | } 431 | }, 432 | "source": [ 433 | "## Uncertainty Quantification\n", 434 | "\n", 435 | "In this subsection we look at the distribution of the flux values, and qualitatively compare the results we get from the surrogate approach of getting the flux vs. the finite difference method.\n", 436 | "\n", 437 | "Since different discretizations of $y$ lead to variation in the approximate distribution of $f = L(y)$, we showcase these effects via plotting the distribution corresponding to the $y$'s in the trraining set, versus randomply sampled $y$ values." 438 | ] 439 | }, 440 | { 441 | "cell_type": "code", 442 | "execution_count": null, 443 | "metadata": { 444 | "colab": { 445 | "base_uri": "https://localhost:8080/", 446 | "height": 1000 447 | }, 448 | "executionInfo": { 449 | "elapsed": 2010, 450 | "status": "ok", 451 | "timestamp": 1682418594810, 452 | "user": { 453 | "displayName": "Victor Armegioiu", 454 | "userId": "10988735183621484296" 455 | }, 456 | "user_tz": -120 457 | }, 458 | "id": "8IGf2_83nPPR", 459 | "outputId": "712c60bd-1b49-4851-bba4-021bb3f46363", 460 | "pycharm": { 461 | "is_executing": true, 462 | "name": "#%%\n" 463 | } 464 | }, 465 | "outputs": [], 466 | "source": [ 467 | "inputs_for_UQ = torch.rand((10000, 2))\n", 468 | "outputs_for_UQ = model(inputs_for_UQ).reshape(-1,).detach()\n", 469 | "\n", 470 | "plt.figure(dpi=150)\n", 471 | "sb.distplot(outputs_for_UQ, label=\"P(f) [f := NN(y), 10k random y samples]\")\n", 472 | "sb.distplot(output, label=\"P(f) [f := finite_diff(y), 100 Sobol points]\")\n", 473 | "sb.distplot(test_output,\n", 474 | " label=\"P(f) [f := finite_diff(y), 10k random y samples] \")\n", 475 | "plt.legend()\n", 476 | "plt.show()" 477 | ] 478 | }, 479 | { 480 | "cell_type": "markdown", 481 | "metadata": { 482 | "id": "kiZGwG1OnPPR", 483 | "pycharm": { 484 | "name": "#%% md\n" 485 | } 486 | }, 487 | "source": [ 488 | "## Optimal Design\n", 489 | "\n", 490 | "We want to find a value of the input variables $y$ that maximizes the flux $q$:\n", 491 | "$$\n", 492 | "y_{opt, exact} = \\arg\\max_{y\\in[0,1]^d} q(y)\n", 493 | "$$\n", 494 | "\n", 495 | "In order to solve the problem we employ the DNNOPT algorithm, that boils down to replacing the model $q(y)$ with the neural network previously trained $q^*(y)$ and solving the maximization problem:\n", 496 | "$$\n", 497 | "y_{opt} = \\arg\\max_{y\\in[0,1]^d} q^*(y) \n", 498 | "$$" 499 | ] 500 | }, 501 | { 502 | "cell_type": "code", 503 | "execution_count": null, 504 | "metadata": { 505 | "colab": { 506 | "base_uri": "https://localhost:8080/" 507 | }, 508 | "executionInfo": { 509 | "elapsed": 517, 510 | "status": "ok", 511 | "timestamp": 1682414818130, 512 | "user": { 513 | "displayName": "Victor Armegioiu", 514 | "userId": "10988735183621484296" 515 | }, 516 | "user_tz": -120 517 | }, 518 | "id": "zCmkt7tTnPPR", 519 | "outputId": "db2dbc13-9619-4367-87b4-48657d8820d6", 520 | "pycharm": { 521 | "is_executing": false, 522 | "name": "#%%\n" 523 | } 524 | }, 525 | "outputs": [], 526 | "source": [ 527 | "y_opt = torch.tensor(torch.tensor([0.5, 1.24]), requires_grad=True)\n", 528 | "y_init = torch.clone(y_opt)\n", 529 | "\n", 530 | "optimizer = optim.LBFGS(\n", 531 | " [y_opt],\n", 532 | " lr=float(0.00001),\n", 533 | " max_iter=50000,\n", 534 | " max_eval=50000,\n", 535 | " history_size=100,\n", 536 | " line_search_fn=\"strong_wolfe\",\n", 537 | " tolerance_change=1.0 * np.finfo(float).eps\n", 538 | ")\n", 539 | "\n", 540 | "optimizer.zero_grad()\n", 541 | "cost = list([0])\n", 542 | "\n", 543 | "def closure():\n", 544 | " y_tilde = ((\n", 545 | " torch.clamp(y_opt, min=min_inputs, max=max_inputs) - min_inputs) /\n", 546 | " (max_inputs - min_inputs))\n", 547 | " G = -model(y_tilde)\n", 548 | " cost[0] = G\n", 549 | " G.backward()\n", 550 | " return G\n", 551 | "\n", 552 | "\n", 553 | "optimizer.step(closure=closure)\n", 554 | "print(\"Minimizer: \", torch.clamp(y_opt, min=min_inputs, max=max_inputs))\n", 555 | "print(\"Corresponding flux values: \", \n", 556 | " model(\n", 557 | " (torch.clamp(y_opt, min=min_inputs, max=max_inputs) - min_inputs) / \n", 558 | " (max_inputs - min_inputs)\n", 559 | " )\n", 560 | " )\n", 561 | "\n", 562 | "f_opt, (x,ic,u) = solve_heat_eq(\n", 563 | " torch.clamp(y_opt, min=min_inputs, max=max_inputs))" 564 | ] 565 | }, 566 | { 567 | "cell_type": "code", 568 | "execution_count": null, 569 | "metadata": { 570 | "colab": { 571 | "base_uri": "https://localhost:8080/", 572 | "height": 652 573 | }, 574 | "executionInfo": { 575 | "elapsed": 11, 576 | "status": "ok", 577 | "timestamp": 1682416603733, 578 | "user": { 579 | "displayName": "Victor Armegioiu", 580 | "userId": "10988735183621484296" 581 | }, 582 | "user_tz": -120 583 | }, 584 | "id": "gb_Yho6YsS-r", 585 | "outputId": "923287d6-60d8-45d9-fc56-e8b05710ac8e", 586 | "pycharm": { 587 | "name": "#%%\n" 588 | } 589 | }, 590 | "outputs": [], 591 | "source": [ 592 | "plt.figure(dpi=150)\n", 593 | "plt.grid(True, which=\"both\", ls=\":\")\n", 594 | "plt.plot(x, ic.detach(),label=\"Initial Condition\")\n", 595 | "plt.plot(x, u.detach(), label=\"Solution at T=0.01\")\n", 596 | "plt.legend()" 597 | ] 598 | }, 599 | { 600 | "cell_type": "code", 601 | "execution_count": null, 602 | "metadata": { 603 | "colab": { 604 | "base_uri": "https://localhost:8080/", 605 | "height": 726 606 | }, 607 | "executionInfo": { 608 | "elapsed": 409649, 609 | "status": "ok", 610 | "timestamp": 1682417015563, 611 | "user": { 612 | "displayName": "Victor Armegioiu", 613 | "userId": "10988735183621484296" 614 | }, 615 | "user_tz": -120 616 | }, 617 | "id": "nvsEsmbMnPPR", 618 | "outputId": "83375d33-97af-4acd-89e1-85184d939e4d", 619 | "pycharm": { 620 | "is_executing": false, 621 | "name": "#%%\n" 622 | } 623 | }, 624 | "outputs": [], 625 | "source": [ 626 | "y_opt_ex = torch.tensor(torch.tensor([0.5, 1.24]), requires_grad=True)\n", 627 | "y_init = torch.clone(y_opt_ex)\n", 628 | "\n", 629 | "optimizer = optim.LBFGS(\n", 630 | " [y_opt_ex],\n", 631 | " lr=float(0.00001),\n", 632 | " max_iter=50000,\n", 633 | " max_eval=50000,\n", 634 | " history_size=100,\n", 635 | " line_search_fn=\"strong_wolfe\",\n", 636 | " tolerance_change=1.0 * np.finfo(float).eps\n", 637 | ")\n", 638 | "\n", 639 | "optimizer.zero_grad()\n", 640 | "cost = list([0])\n", 641 | "\n", 642 | "def closure():\n", 643 | " G, _ = solve_heat_eq(\n", 644 | " torch.clamp(y_opt_ex, min=min_inputs, max=max_inputs))\n", 645 | " G = -G\n", 646 | " cost[0] = G\n", 647 | " G.backward()\n", 648 | " return G\n", 649 | "\n", 650 | "\n", 651 | "optimizer.step(closure=closure)\n", 652 | "print(\"Exact Minimizer: \", torch.clamp(y_opt_ex, min=min_inputs, max=max_inputs))\n", 653 | "\n", 654 | "f_opt_ex, (x,ic,u) = solve_heat_eq(\n", 655 | " torch.clamp(y_opt_ex, min=min_inputs, max=max_inputs))\n", 656 | "\n", 657 | "plt.figure(dpi=150)\n", 658 | "plt.grid(True, which=\"both\", ls=\":\")\n", 659 | "plt.plot(x, ic.detach(),label=\"Initial Condition\")\n", 660 | "plt.plot(x, u.detach(), label=\"Solution at T=0.01\")\n", 661 | "plt.legend()" 662 | ] 663 | }, 664 | { 665 | "cell_type": "code", 666 | "execution_count": null, 667 | "metadata": { 668 | "colab": { 669 | "base_uri": "https://localhost:8080/", 670 | "height": 893 671 | }, 672 | "executionInfo": { 673 | "elapsed": 4419, 674 | "status": "ok", 675 | "timestamp": 1682417465202, 676 | "user": { 677 | "displayName": "Victor Armegioiu", 678 | "userId": "10988735183621484296" 679 | }, 680 | "user_tz": -120 681 | }, 682 | "id": "RymIKO72nPPS", 683 | "outputId": "1c3355e3-5aca-43aa-8fea-f5a8e118239d", 684 | "pycharm": { 685 | "is_executing": false, 686 | "name": "#%%\n" 687 | } 688 | }, 689 | "outputs": [], 690 | "source": [ 691 | "plt.figure(dpi=200)\n", 692 | "plt.scatter(test_inputs[:,0], test_inputs[:,1], c=test_output)\n", 693 | "plt.scatter(y_opt_ex[0].detach(), y_opt_ex[1].detach(), marker = \"o\", label = \"Exact Maximizer\")\n", 694 | "plt.scatter(y_opt[0].detach(), y_opt[1].detach(), marker = \"*\", label = \"Approximate Maximizer\")\n", 695 | "plt.colorbar()\n", 696 | "plt.xlabel(\"y1\")\n", 697 | "plt.ylabel(\"y2\")\n", 698 | "plt.legend()\n", 699 | "plt.show()" 700 | ] 701 | } 702 | ], 703 | "metadata": { 704 | "colab": { 705 | "provenance": [] 706 | }, 707 | "kernelspec": { 708 | "display_name": "Python 3 (ipykernel)", 709 | "language": "python", 710 | "name": "python3" 711 | }, 712 | "language_info": { 713 | "codemirror_mode": { 714 | "name": "ipython", 715 | "version": 3 716 | }, 717 | "file_extension": ".py", 718 | "mimetype": "text/x-python", 719 | "name": "python", 720 | "nbconvert_exporter": "python", 721 | "pygments_lexer": "ipython3", 722 | "version": "3.8.17" 723 | }, 724 | "pycharm": { 725 | "stem_cell": { 726 | "cell_type": "raw", 727 | "metadata": { 728 | "collapsed": false 729 | }, 730 | "source": [] 731 | } 732 | } 733 | }, 734 | "nbformat": 4, 735 | "nbformat_minor": 1 736 | } -------------------------------------------------------------------------------- /Autodifferentiation/Autodifferentiation - solution.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"markdown","id":"0e209abf","metadata":{"id":"0e209abf"},"source":["# Autodifferentiation tutorial\n","\n","In this tutorial, we will be implementing autodifferentiation from scratch, without using external python libraries.\n","\n","## Problem overview\n","\n","In particular, we will be defining a custom `Value` object in python.\n","\n","This object defines a scalar value (analogous to a python `float`) which can have primitive operations (e.g. add/multiply/pow) applied to it. Each primitive operation returns a new `Value` object.\n","\n","Importantly, each `Value` object keeps track of the child `Value` objects which created it, and defines a `._backward` method which computes the vector-Jacobian product of the primative operation which created it.\n","\n","This extra bookmarking allows us to define a `Value.backward()` method (similar to `PyTorch`) which recursively backpropagates gradients through the entire computational graph, accumulating gradients in the leaf `Values` of the graph.\n","\n","\n","## Part A: implement the `Value` class\n","\n","The `Value` class is defined in `engine.py`.\n","\n","There are a number of lines of code missing from this class that you must fill in:\n","\n","> **Task A.1**: Implement the primitive operations which currently raise a `NotImplementedError`\n","\n","> **Task A.2**: Implement the `Value.backward` method.\n","\n","> **Task A.3**: Test that the class is correctly implemented by running the tests defined in `engine.py`\n","\n","More hints are contained in `engine.py`.\n","\n","## Part B: use the `Value` class to solve a differentiable physics problem\n","\n","Once we have a working `Value` object, we can use it to solve any problem requiring gradients.\n","\n","In this part, we will use the `Value` object to learn how to **throw a ball** to hit a target.\n","\n","### Problem overview\n","\n","Imagine you are located at $P1 = (0,0)$ and you throw a ball with an initial velocity $v$ at an angle $\\alpha$ from the x-axis. Your goal is to hit a target at $P2 = (x_2, y_2)$. What are values of $\\alpha$ and $v$ you should use?\n","\n","We can treat this as a inverse problem, where the goal is to minimise the following loss function\n","\n","$$\n","L(v,\\alpha) = (F(v, \\alpha, x_2) - y_2)^2 ~~~~~~~(1)\n","$$\n","\n","Here $F(v, \\alpha, x_2)$ is our forward physics model, which tells us the height of the ball at $x_2$ given the initial velocity and angle of the ball.\n","\n","In this case, $F$ is given by\n","\n","$$\n","F(v, \\alpha, x_2) = - \\frac{g}{2} \\left( \\frac{x_2}{v \\cos \\alpha} \\right)^2 + x_2 \\tan \\alpha ~~~~~~~(2)\n","$$\n","\n","where $g$ is the acceleration due to gravity.\n","\n","> **Optional task B.1**: Derive (2) using Netwon's laws.\n","\n","> **Task B.2**: Write a python function that computes $F(v, \\alpha, x_2)$, where $v$ and $\\alpha$ are `Value` objects.\n","\n","> **Task B.3**: Use `Value.backward` to write a gradient descent algorithm which minimises the loss function (1), given the starting guess of $v$ and $\\alpha$ below.\n","\n","> **Task B.4**: Verify that the learned values of $\\alpha$ and $v$ hits the target by plotting the optimised trajectory.\n","\n","> **Task B.5**: What happens to the trajectory when you change the starting guess of $v$ and $\\alpha$? Why is this the case?"]},{"cell_type":"code","execution_count":null,"id":"5c638652","metadata":{"scrolled":false,"id":"5c638652","outputId":"b546f8a7-b49e-4a31-d1ea-2a9b9431e606"},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAPEAAAG0CAYAAADjKcFiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3HElEQVR4nO3de1gU9f4H8PfsLrsLwi4gyEVRVFAUFa8gaKmJYZqJt7RjoWValpalR/E5Hs3M6FceT+ap1FOBdjlaHTWPqWnkLUQxBa9geEFAuYjocpPb7vf3x8Imcdtddnd2dj+v55k/dnZm57PDvpnv3L7DMcYYCCGCJeK7AEJI21CICRE4CjEhAkchJkTgKMSECByFmBCBoxATInAUYkIEjkJMiMBRiAkROAoxIQJHIRaAsrIyrFq1CmPHjoW7uzs4jkNCQoJBn1FVVYVly5bB19cXjo6OCAsLw6FDh8xTsIWtXbsWHMehT58+jd47cuQIOI5rcjh58qRen2/t607CdwGkdUVFRXj77bfRuXNnhISE4MiRIwZ/xuzZs/H9999j0aJFCAwMREJCAsaNG4fDhw9j+PDhpi/aQnJzc/Huu++iXbt2LU732muvYciQIQ3GBQQE6LUMq193jFi9yspKlpeXxxhj7PTp0wwAi4+P13v+U6dOMQDsgw8+0I178OAB6969OwsPDzd1uRY1ffp09thjj7ERI0aw4ODgRu8fPnyYAWDfffedUZ8vhHVHzWkDjBkzptn/3oMHD0ZYWJhZliuTyeDt7W30/N9//z3EYjHmzZunGyeXyzFnzhwkJycjJyfH6M/esWMHBg4cCEdHR/Tq1Qs///wzGGMIDg7G2rVrjf5cfRw7dgzff/89PvzwQ72mLy0tRW1trUHLMOe6MxUKsQGCg4ORlZWFqqqqBuN3796NM2fO4J133mk0T01NDYqKivQaNBqNWepOTU1Fjx49oFAoGowPDQ0FAKSlpRn1uStWrMCMGTMQEhKCdevWQa1WIyYmBvv27UNubi4WLFjQaB5TrQ+1Wo2FCxfixRdfRN++fVut9fnnn4dCoYBcLseoUaPw22+/6fUdzbXuTIn2iQ0QHBwMtVqNzMxM3UEUxhhWrVqFESNGYMyYMY3mSUpKwqhRo/T6/Bs3bsDf39+UJQMA8vLy4OPj02h8/bjbt28b/JnHjx/H2rVrsWzZMrz33nsAAG9vb0ydOhWxsbGYP38+lEplo/lMtT42bdqEmzdv4ueff27xM6RSKaZMmYJx48bBw8MDly9fxrp16/DII4/gxIkTGDBgQIvzm2PdmRqF2ADBwcEAgIyMDF2Iv/vuO5w/fx7Hjx9vcp6QkBC9j2S2pcnckgcPHkAmkzUaL5fLde8basOGDXBzc8OKFSt04+oDd/XqVbzxxhtNzmeK9XH37l2sXLkSf//73+Hp6dniZ0RERCAiIkL3+qmnnsLUqVPRr18/LF++HAcOHGhxfnOsO1OjEBvg4RADgEajwVtvvYWoqKhmj1K6ubkhMjLSYjU2xdHRsdEuAABUVlbq3jeEWq3GwYMHMWHCBDg7Ozd6//nnn4eXl1eT85pifaxYsQLu7u5YuHChUfMHBARg4sSJ2LlzJ9RqNcRicbPTmnrdmQOF2ABKpRK+vr66EH/zzTdIT0/Htm3bmp2nuroaxcXFen2+p6dniz8oY/n4+ODWrVuNxufl5QEAfH19Dfq869evo7S0FAMHDmww/s6dOwCAV199tdl527o+MjMzsWXLFnz44YcNmrKVlZWoqalBVlYWFAoF3N3dW/xsPz8/VFdXo7y8vNH+7sNMve7Mgu/D40IzZswYNmjQIFZbW8sCAwNZdHR0i9PXn+LQZ7hx40aryzfmFNOSJUuYWCxmKpWqwfi1a9cyACw7O1vvz2KMsaSkJAaAJSQkNBj/6KOPMgAsPz+/2Xnbuj70mf/1119v9TtMmTKFyeVyplarW5zO1OvOHGhLbKDg4GB89tln2Lp1K65du4adO3e2OL0l94krKiqQnZ0NDw8PeHh46MZPnToV69atw5YtW7BkyRIA2quQ4uPjERYWBj8/P4OWU3/A6uLFi7px33zzDY4dOwbgj6ZmU9q6Pvr06YNdu3Y1Gr9ixQqUlpZiw4YN6N69u278nTt3Gu03nzt3Dnv27METTzwBkUh7gsZS684s+P4vIjT//ve/GQDm6enJnnnmGYstd+PGjWzNmjVs/vz5DACbPHkyW7NmDVuzZg27f/8+Y+yPrdSqVasazT9t2jQmkUjYX//6V7Z582YWERHBJBIJO3r0aIPpALARI0a0WItarWZdu3ZlUqmUrVy5kr311ltMLpezp59+mgFgs2fPZufPnzfVV9dLcxd7jBo1io0bN4698847bMuWLWzRokXMycmJKZVKdvnyZd10plh3fKEQG+jEiRMMABOLxez333+32HK7dOnSarOzpR/igwcP2JIlS5i3tzeTyWRsyJAh7MCBAw2mKS0tZQDYjBkzWq0nNTWVDR06lMlkMubm5sb+9re/MY1Gw1544QUmkUgaNbXNrbkQb9iwgYWGhjJ3d3cmkUiYj48Pe/bZZ1lmZmaD6dq67vjEMUadxxOtffv24cknn8S5c+f0uoCCWAe6YovoHD58GDNmzKAACwxtiQkRONoSEyJwFGJCBI5CTIjAUYgJETibu2JLo9Hg9u3bcHFxAcdxfJdDiN4YYygtLYWvr6/uSjJ92FyIb9++bR2XwhFipJycHHTq1Env6W0uxC4uLgC0K6Klu1MIsTYlJSXw8/PT/Yb1ZXMhrm9CKxQKCjERJEN3A+nAFiECRyEmROAoxIQInM3tE+uDMYba2lqo1Wq+SyEm4uDgYJaujYTA7kJcXV2NvLw8VFRU8F0KMSGO49CpU6cmO+6zdXYVYo1Ggxs3bkAsFsPX1xdSqZQuCLEBjDHcuXMHubm5CAwMtMotMmMM6vv3oSmvgKidE8Surib77dlViKurq6HRaODn5wcnJye+yyEm5OnpiaysLNTU1FhViNUlJVDt3o3ir75CTfYfj3xx6OwH92efhTI6GuI2ngq1ywNbhlzSRoTBGltUZcd/ReaIkSiIew81ObkN3qvJyUVB3HvIHDESZcd/bdNy6NdMiBmUHf8VOS+9BFZZCTCmHR5WN45VViLnpZfaFGQKMSEmpi4pQe5rrzUd3j+rmyb3tdegLikxankUYjs2cuRILFq0iO8ybI5q9+4/tsD6qNsiq37cZ9TyKMQCwHFci8Nbb71l1Ofu3LkTa9asMVmd9E9BexS6+KuvjJr33o7tRs1nV0enhar+uT+A9qHeK1euxJUrV3TjHj43yhiDWq2GRNL6n7a15xXxpbq6GlKplO8yjKK+f7/BUWi9MYbaPx380pfdb4kZY6ioruVl0LejUW9vb92gVCrBcZzudUZGBlxcXLB//34MGjQIMpkMv/76K65du4aJEyfCy8sLzs7OGDJkSKNn+f55y1lVVYUlS5agY8eOaNeuHcLCwnDkyJEG8yQlJWHkyJFwcnKCm5sboqKicO/ePcyePRtHjx7Fhg0bdC2ErKwsAMDRo0cRGhoKmUwGHx8fxMbGora2tkEdCxYswKJFi+Dh4YGoqCi88MILePLJJxssu6amBh06dMDnn3+u/x/YwjTllr+IyO63xA9q1Oi98ideln357Sg4SU3zJ4iNjcW6devQrVs3uLm5IScnB+PGjcPatWshk8mwbds2TJgwAVeuXEHnzp2b/IwFCxbg8uXL2L59O3x9fbFr1y6MHTsWFy5cQGBgINLS0jB69Gi88MIL2LBhAyQSCQ4fPgy1Wo0NGzbg999/R58+ffD2228D0J67vXXrFsaNG4fZs2dj27ZtyMjIwNy5cyGXyxvsBmzduhXz589HUlISAO0ziB999NEGD/neu3cvKioqMH36dJOsM3MQtbP89Qd2H2Jb8fbbb2PMmDG61+7u7ggJCdG9XrNmDXbt2oU9e/ZgwYIFjebPzs5GfHw8srOzdY/rXLJkCQ4cOID4+Hi8++67eP/99zF48GB88sknuvnqn9kMAFKpFE5OTg0ehPbJJ5/Az88P//rXv8BxHIKCgnD79m0sW7YMK1eu1J2zDwwMxPvvv9+gpp49e+LLL7/E0qVLAQDx8fGYNm2aVV9aKXZ1hUNnP+15YUO6dOc4SPw6AVczDV6m3YfY0UGMy29H8bZsUxk8eHCD12VlZXjrrbfw448/Ii8vD7W1tXjw4AGys7ObnP/ChQtQq9Xo0aNHg/FVVVVo3749ACAtLQ3Tpk0zqK709HSEh4c3uBhj2LBhKCsrQ25urq5VMGjQoEbzvvjii9iyZQuWLl2KgoIC7N+/H7/88otBy7c0juPg/uyzKIh7z+B53abPAA4fNng+uw8xx3Ema9LyqV27dg1eL1myBIcOHcK6desQEBAAR0dHTJ06FdXV1U3OX1ZWBrFYjDNnzjS6bLF+y+fo6Gie4tG4fgCIiYlBbGwskpOTceLECXTt2hWPPPKI2WowFWV0NAr/+aH+p5lEInAyGZTjxxm1PLs/sGWrkpKSMHv2bEyaNAl9+/aFt7e37kBTUwYMGAC1Wo3CwkIEBAQ0GOqbx/369UNiYmKznyGVShvd3tmrVy8kJyc3OIiXlJQEFxeXVjuDa9++PaKjoxEfH4+EhAQ8//zzenxz/okVCnT66COA47RDS+re77Rxo9HXUFOIbVRgYCB27tyJtLQ0nDt3Dn/5y1+g0Wianb5Hjx6YOXMmYmJisHPnTty4cQMpKSmIi4vDjz/+CABYvnw5Tp8+jVdeeQXnz59HRkYGPv30UxQVFQEA/P39cerUKWRlZaGoqAgajQavvPIKcnJysHDhQmRkZOCHH37AqlWr8Oabb+p1DfuLL76IrVu3Ij09HbNmzTLNyrEA50eGw2/zZnByedNhrhvHyeXw27IFzsOHGb0sCrGNWr9+Pdzc3BAREYEJEyYgKioKAwcObHGe+Ph4xMTEYPHixejZsyeio6Nx+vRp3X5rjx49cPDgQZw7dw6hoaEIDw/HDz/8oDsnvWTJEojFYvTu3Ruenp7Izs5Gx44dsW/fPqSkpCAkJAQvv/wy5syZgxUrVuj1PSIjI+Hj44OoqCjdATehcH5kOAKPHoHX8uVw8GvY6nDw6wSv5csReOxomwIM2OBTEUtKSqBUKqFSqRr1dllZWYkbN26ga9eukMvlPFVoPcLDwzF69Gi88847fJfSrLKyMnTs2BHx8fGYPHlys9NZ+99Wn/uJW/rttoS2xHaoqqoKv/32Gy5dutTgFJE10Wg0KCwsxJo1a+Dq6oqnnnqK75LahOM4SNzcIO3UERI3N5PeOin8w7LEYPv370dMTAyeeuopTJ06le9ympSdnY2uXbuiU6dOSEhI0OsyUntFa8YORUdHo8TI294sxd/fX+/LUu0dNacJETgKMSECRyEmROAoxIQIHB3YMpI5+xEmxBAUYgNZoh9hQgxBzWkDWKofYUIMQSHWkyX7Ef4zc3WUZ6radu/ezdvyCTWn9WJwP8IAcl97DYFHj5ikaW1IR3n6EHJHdKQx2hLrweh+hHf/YJLlt9RRXnl5OWbOnNlih3j+/v5Ys2YNYmJioFAoMG/ePADAv//9b91zqSZNmoT169fD1dW1wbw//PADBg4cCLlcjm7dumH16tW6Tu78/f0BAJMmTQLHcbrXxLIoxK1oSz/CxV99afZLB8vKyjBu3DgkJiYiNTUVY8eOxYQJExp1w7Nu3TqEhIQgNTUVf//735GUlISXX34Zr7/+OtLS0jBmzBisXbu2wTzHjx9HTEwMXn/9dVy+fBmbN29GQkKCbrrTp08D0N7CmJeXp3tNLIua061oSz/CNdk5UN+/D4mbm+kLqxMSEqJXh3iPPfYYFi9erHv9t7/9DU888QSWLFkCQHuv8IkTJ7B3717dNKtXr0ZsbKzuZvxu3bphzZo1WLp0KVatWgVPT08AgKura4PO8Yhl0Za4FW3tR9jc/RCXlZVhyZIl6NWrF1xdXeHs7Iz09PRGW+I/d6R35coVhIaGNhj359fnzp3D22+/DWdnZ90wd+5ceki7laEtcSva2o+wufsh1rdDvKY6omtNWVkZVq9e3eTN+NZ44729ohC3oi39CDv4dYL4TweKTO3hDvEAbfBa6hCvXs+ePRvtw/759cCBA3HlyhUEBAQ0+zkODg6NOscjlkXN6VbU9yNsDPdnnzP7pZiGdohXb+HChdi3bx/Wr1+PzMxMbN68Gfv3729Q78qVK7Ft2zasXr0aly5dQnp6OrZv396gfyx/f38kJiYiPz8f9+7dM8t3JC2zSIg//vhj+Pv7Qy6XIywsDCkpKc1Om5CQ0OhiBr6bbsro6D96LdSHSAROLocyeqJ5C4NxHeIB2g7cN23ahPXr1yMkJAQHDhzAG2+80WBdR0VFYe/evTh48CCGDBmCoUOH4p///Ce6dOmim+Yf//gHDh06BD8/PwwYMMAs35G0zOwd5e3YsQMxMTHYtGkTwsLC8OGHH+K7777DlStX0KFDh0bTJyQk4PXXX29wMQPHcfDy8tJreebqKK/+iq1WL/io64q0rd2Q8mHu3LnIyMjA8ePH+S7FYNbeUZ4+rLajvPXr12Pu3Ll4/vnn0bt3b2zatAlOTk744osvmp3n4YsZvL299Q6wOVmyH2FLWbduHc6dO4erV69i48aN2Lp1q6D6diZaZg1xdXU1zpw5g8jIyD8WKBIhMjISycnJzc5XVlaGLl26wM/PDxMnTsSlS5eanbaqqgolJSUNBnOxVD/ClpKSkoIxY8agb9++2LRpEz766CO8+OKLfJdFDGTWo9NFRUVQq9WNtqReXl7IyMhocp6ePXviiy++QL9+/aBSqbBu3TpERETg0qVLTT72Iy4uDqtXrzZL/U0RKxRwj3kObs89K/j7ib/99lu+SyAmYHVHp8PDwxETE4P+/ftjxIgR2LlzJzw9PbF58+Ymp1++fDlUKpVuyMkx4uoqI5izH2FCDGHWLbGHhwfEYjEKCgoajC8oKND7Mj0HBwcMGDAAV69ebfJ9mUwGmUxmUF3UFartsee/qVm3xFKpFIMGDWrwJD2NRoPExESEh4fr9RlqtRoXLlzQPS2+LRwcHACALhm0QfVXqP35saz2wOxXbL355puYNWsWBg8ejNDQUHz44YcoLy/XPaYyJiYGHTt2RFxcHADtE++HDh2KgIAA3L9/Hx988AFu3rxpkgMuYrEYrq6uKCwsBAA4OTlRM9gGaDQa3LlzB05OTnb5pAizf+Pp06fjzp07WLlyJfLz89G/f38cOHBAd7ArOzu7wSMu7927h7lz5yI/Px9ubm4YNGgQTpw4gd69e5uknvpmfH2QiW0QiUTo3LmzXf5TtqunIj5MrVajpqbGgpURc5JKpXo979iaGXuxh/21PeqIxWK73H8itkfY/7oIIRRiQoSOQkyIwFGICRE4CjEhAkchJkTgKMSECByFmBCBoxATInAUYkIEjkJMiMBRiAkROAoxIQJHISZE4CjEhAgchZgQgaMQEyJwFGJCBI5CTIjAUYgJETgKMSECRyEmROAoxIQIHIWYEIGjEBMicBRiQgSOQkyIwFGICRE4CjEhAkchJkTgKMSECByFmBCBoxATInAUYkIEjkJMiMBRiAkROAoxIQJHISZE4CjEhAgchZgQgaMQEyJwFGJCBI5CTIjAUYgJETgKMSECRyEmROAoxIQIHIWYEIGjEBMicBRiQgSOQkyIwFGICRE4CjEhAkchJkTgJHwXYMsYY7hbXo2bd8uRp6pEcXk1yqpqUatmEHGAo1QChVwCL4Ucfu5O8HNzhERM/1eJYSjEJqTWMKTl3EfytSL8dvMeLt5SoaisWu/5pWIRenq7oL+fK4Z2a49hAe3h6iQ1Y8XEFnCMMcZ3EaZUUlICpVIJlUoFhUJh9uVpNAzJ1+9iT9ptHLycj3sVNQ3e5zjAV+kIX1c52reTwVkugYNYBI2G4UGNGvcqqlFQUons4gpU1mgazCsWcRji74Yn+/niyX4+FGgbZ+xvl0JspHvl1fjP6Wx8cyobufce6MYr5BIMD/RAqL87+nd2Q08vFzhKxa1+nkbDkHvvAc7fuo/fsu4h6WoRMgvLdO9LJSKM6+ON2cO6or+fqzm+EuEZhbiOuUNcWFKJzceu4+tTN3VbThe5BE/288WEfj4I7epusv3anOIK7L+Yh51nbyEjv1Q3PrSrO14fHYiI7u3BcZxJlkX4RyGuY64Ql1bW4JMj1xCfdEMX3t4+CrwwvCue7OcDuUPrW1tjMcZwPleFrclZ+N+526hRa/9kQ7u5Y/kTvRBCW2abQCGuY+oQM8bw37O38N7+DBSVVQEABnZ2xeuRPfBooIfFt4T5qkpsOnoN36Rko7pW+8/k6cGdsHRsEDycZRathZgWhbiOKUOcVVSO2J3ncfJ6MQCgm0c7LB/XC5G9OvDejL11/wH+8dMV7Ey9BQBwdXLA38f3xuSBHXmvjRiHQlzHFCFmjOHLkzcRty8DD2rUkDuIsCiyB14Y1hVSiXWdxz1z8x5W7L6I9LwSAEBkrw6Im9wPni60VRYaCnGdtoa4uLwaf/3uHBIzCgEAEd3b4/+m9IOfu5OpSzWZGrUG/z5+HR8eykS1WgMPZyn+Ob0/Hgn05Ls0YgAKcZ22hDg1+x5e+fos8lSVkEpEWP5EEGaF+0MkEkbzND2vBG/sSENGfik4Dlg4KgCLInsIpn57Z+xv17rahjz69nQOnt6cjDxVJbp5tMPuV4bh+WFdBRWAXj4K7H51GGaGdQZjwEe/XMWcradRUlnT+sxEsOw+xGoNwzt7L2Ppf8+jRs0wNtgbPywYht6+5r/ayxzkDmKsndQX658OgUwiwuErdzDlkxPIvlvBd2nETOw6xJU1arz69Vl89usNAMCbY3rg02cHwkXuwHNlbTd5YCf8d34EvBQyZBaWYfKnSTife5/vsogZWCTEH3/8Mfz9/SGXyxEWFoaUlJQWp//uu+8QFBQEuVyOvn37Yt++fSavSfWgBjGfp+DApXxIxSJ89MwAvDY60KZOz/TpqMSeBcPR20eBorJqzNhyEscz7/BdFjExs4d4x44dePPNN7Fq1SqcPXsWISEhiIqKQmFhYZPTnzhxAs888wzmzJmD1NRUREdHIzo6GhcvXjRZTXfLqvDMlpNIySqGi0yCbXNC8VSIr8k+35p4KeTY8dJQPBLogYpqNeYk/IYDF/P5LouYkNmPToeFhWHIkCH417/+BQDQaDTw8/PDwoULERsb22j66dOno7y8HHv37tWNGzp0KPr3749Nmza1urzWjvDdKa3CzM9O4veCMng4S7H1hVAE+yrb8A2FobpWg0U7UrHvQj7EIg4bZvTHk/1s8x+XUFnl0enq6mqcOXMGkZGRfyxQJEJkZCSSk5ObnCc5ObnB9AAQFRXV7PRVVVUoKSlpMDSnskaNGVuS8XtBGbwUMux4KdwuAgxo74La+MxATBnYCWoNw2v/ScX/zt3muyxiAmYNcVFREdRqNby8vBqM9/LyQn5+0026/Px8g6aPi4uDUqnUDX5+fs3WI3cQY2ZYF/gq5dgxLxzdPZ0N/EbCJhZx+GBqPzw9uBM0DFi0Iw0/XaKmtdAJ/uj08uXLoVKpdENOTk6L078wvCt+euNR+Hu0s1CF1kUk4vDe5H6YPKAj1BqGBd+cxa+ZRXyXRdrArCH28PCAWCxGQUFBg/EFBQXw9vZuch5vb2+DppfJZFAoFA2G1tjCKaS2EIk4vD+1H8b19UaNmmHel78hLec+32URI5k1xFKpFIMGDUJiYqJunEajQWJiIsLDw5ucJzw8vMH0AHDo0KFmpyfGkYhF+Of0/hgeUH/U+jSyisr5LosYg5nZ9u3bmUwmYwkJCezy5cts3rx5zNXVleXn5zPGGHvuuedYbGysbvqkpCQmkUjYunXrWHp6Olu1ahVzcHBgFy5c0Gt5KpWKAWAqlcos38fWlFXWsPEfHWNdlu1lI97/hd0tq+K7JLtl7G/X7CFmjLGNGzeyzp07M6lUykJDQ9nJkyd1740YMYLNmjWrwfTffvst69GjB5NKpSw4OJj9+OOPei+LQmy4gpIHbNh7iazLsr1s6qdJrLKmlu+S7JKxv126i4kAADILSjH5kxMorarF5IEd8Y9pITZ19ZoQWOV5YiIcgV4u+OTZgRCLOOw8ewuf111PTqwfhZjoPBLoiRXjewEA3t2XTqeeBIJCTBqYHeGPaYO0F4Ms/M9Z3Lr/oPWZCK8oxKQBjuOwJroP+nRU4F5FDV756gyqatV8l0VaQCEmjcgdxPh05iAoHR1wLleFuH0ZfJdEWkAhJk3yc3fC+qdDAAAJJ7Kw/0IezxWR5lCISbNG9/LCSyO6AQCW/fc87R9bKQoxadGSx3uiv58rSiprsWh7KmrVmtZnIhZFISYtchCL8NGMAXCWSXA66x4+PXKN75LIn1CISas6t3fC2xODAQAbEjNxju54sioUYqKXSQM6YnxfH9RqGN74Ng2VNXTayVpQiIleOI7D2kl90MFFhut3yvHBT1f4LonUoRATvbk6SfF/U/oBAL5IuoGUG8U8V0QACjEx0KigDnh6cCcwpj3tRM1q/lGIicFWPNkbXgoZbhSVY/2h3/kux+5RiInBFHIHvDupLwDgs+PXcSFXxXNF9o1CTIwyupcXJoT4QsOA2J3n6SIQHlGIidFWPtkbSkcHXLpdgi+SqBMBvlCIidE8XWT42zhtJwL/PJRJ11bzhEJM2mTqoE4I9XfHgxo1Vu+5xHc5dolCTNpEJOLwzqQ+kIg4HLxcgF8yClqfiZgUhZi0WQ8vF8wZ3hUA8Naey3Tu2MIoxMQkFo4OhJdChuziCmw5dp3vcuwKhZiYhLNMgr+N7w0A+OTIVdymg1wWQyEmJjOhnw9Cu7qjskaDuP3UL5elUIiJyXAch1UTeoPjgP+du003SFgIhZiYVLCvEjOGdAYAvPPjZWg0NvWUIKtEISYm9+aYHnCWSXA+V4Xdabf4LsfmUYiJyXm6yPDKqO4AgPcPXMGDajrlZE4UYmIWLwzrio6ujsgvqaTrqs2MQkzMQu4gxl+jegIAPj1yDXfLqniuyHZRiInZPBXiiz4dFSirqsXGX67yXY7NohATsxGJOMSO1d7l9PWpm8gpruC5IttEISZmNTzQA8MDPFCjZtSVj5lQiInZLRsbBADYnXYLGfklPFdjeyjExOz6dlJifF8fMAb84yBtjU2NQkws4o0xPSDigEOXC5BGj4ExKQoxsYiADs6YPLATAOAfB+npEaZEISYW8/roQEhEHI5nFtHNESZEISYW4+fuhKeH+AEA1h+irbGpUIiJRb06KgBSsQgnrxfjxLUivsuxCRRiYlEdXR0xvW5rvOHnTJ6rsQ0UYmJx80d2h4OYw6kbxTh5/S7f5QgehZhYnK+rI54eTFtjU6EQE168MioADmIOydfv4sxNOlLdFhRiwouOro6YUnfe+F90h1ObUIgJb+aP7A4RBxy+cocej9oGFGLCmy7t2+GpEF8A2r6qiXEoxIRXr4wKAAAcuJSPq4VlPFcjTBRiwqseXi6I7OUFxoDNR6/xXY4gUYgJ7+p7xtyVegt5Knr8i6EoxIR3Azu7YWg3d9RqGL74lXrGNBSFmFiFl0Zot8bfnMqGqqKG52qEhUJMrMLIHp4I8nZBebUaX526yXc5gkIhJlaB4zjMe7QbACDhRBaqaumpEfqiEBOrMSHEF94KOe6UVuGHtNt8lyMYFGJiNRzEIjw/zB8A8Nnx62CMnqioDwoxsSrPhHWGs0yC3wvKcPT3O3yXIwgUYmJVFHIH3W2Kn9PpJr1QiInVeX6YP0QccDyzCL8XlPJdjtWjEBOr4+fuhKhgbwCgiz/0QCEmVmnO8K4AtJdiFpdX81yNdaMQE6s0qIsb+nZUoqpWg/+kZPNdjlWjEBOrxHGc7nTTl8k3UaPW8FuQFaMQE6s1vp8PPJxlyC+pxIGL+XyXY7UoxMRqySRizAzrDADYlpzFbzFWjEJMrNpfwjpDIuJwOuseLt2mfriaQiEmVs1LIccTfX0AANtO0N1NTTFriIuLizFz5kwoFAq4urpizpw5KCtruR+lkSNHguO4BsPLL79szjKJlZsV3gUAsDvtFu5X0OmmPzNriGfOnIlLly7h0KFD2Lt3L44dO4Z58+a1Ot/cuXORl5enG95//31zlkms3KAubujto0BVrQbfn8nluxyrY7YQp6en48CBA/jss88QFhaG4cOHY+PGjdi+fTtu3275NjMnJyd4e3vrBoVCYa4yiQBwHIfn6rbGX528CY2G7m56mNlCnJycDFdXVwwePFg3LjIyEiKRCKdOnWpx3q+//hoeHh7o06cPli9fjoqKimanraqqQklJSYOB2J6J/X3hIpcg624Ffr1Kj0R9mNlCnJ+fjw4dOjQYJ5FI4O7ujvz85s/5/eUvf8FXX32Fw4cPY/ny5fjyyy/x7LPPNjt9XFwclEqlbvDz8zPZdyDWw0kqwdRB2se+bEumA1wPMzjEsbGxjQ48/XnIyMgwuqB58+YhKioKffv2xcyZM7Ft2zbs2rUL16413Sfx8uXLoVKpdENOTo7RyybWbWaYtkn9S0YBdW37EImhMyxevBizZ89ucZpu3brB29sbhYWFDcbX1taiuLgY3t7eei8vLCwMAHD16lV079690fsymQwymUzvzyPCFdDBGUO7uePk9WJsT8nBG2N68F2SVTA4xJ6envD09Gx1uvDwcNy/fx9nzpzBoEGDAAC//PILNBqNLpj6SEtLAwD4+PgYWiqxQTPDumhDfDobCx4LgIOYLnUw2xro1asXxo4di7lz5yIlJQVJSUlYsGABZsyYAV9f7UO0bt26haCgIKSkpAAArl27hjVr1uDMmTPIysrCnj17EBMTg0cffRT9+vUzV6lEQKKCveHhLEVBSRUS0wtbn8EOmPXf2Ndff42goCCMHj0a48aNw/Dhw7Flyxbd+zU1Nbhy5Yru6LNUKsXPP/+Mxx9/HEFBQVi8eDGmTJmC//3vf+YskwiIVCLClLoDXNtP0y2KAMAxG+tSsKSkBEqlEiqVis4v26isonKMXHcEHAccXzoKndyc+C7JJIz97dIOBREcf492GBbQHowBO07T2QgKMRGkZ0K1tyh++1sOau28wwAKMRGkMb294N5Oe4DL3vunphATQZJJxJg8oCMAYLudN6kpxESwpg/RXmL7S0YhCksqea6GPxRiIliBXi4Y1MUNag3D92ft9xZFCjERtOl1j3z5/rdcu30AG4WYCNq4fj5wkopxvagcv928x3c5vKAQE0FzlknwZD/tdfXf2ukBLgoxEbz6pyjuPZ+HsqpanquxPAoxEbxBXdzQ1aMdHtSose9CHt/lWByFmAgex3G6Xj/ssSM9CjGxCZMHdoSIA1JuFOPm3XK+y7EoCjGxCT5KRwwP1HZW8V872xpTiInNqG9S//fsLbvq1pZCTGzG47294CKT4Nb9B0jJKua7HIuhEBObIXcQY3zdOWN7alJTiIlNmTxQ26TedyEPFdX2cc6YQkxsyhB/N3R2d0J5tRqHLhfwXY5FUIiJTeE4DtF19xnvPHuL52osg0JMbM6kuhAfz7yDwlLbv8+YQkxsTlePdhjQ2RUaBuxJa/kJnLaAQkxsUn3XPbtSbb9JTSEmNml8P19IRBwu3S7B1cJSvssxKwoxsUnu7aQY2VN7GebuVNtuUlOIic2a2F/bpN6ddsumu+6hEBObFdnLC+2kYuTee2DTXfdQiInNcpSKEdVH+yxsWz5KTSEmNq2+Sf3jhTzU2OjjXijExKYN694e7dtJUVxejV+vFvFdjllQiIlNk4hFut4w/2ejTWoKMbF5T/X3BQD8dCkfD6rVPFdjehRiYvMGdnZDR1dHlFercfhKId/lmByFmNg8juPwZEhdk/qc7TWpKcTELjwVom1SJ2YUorSyhudqTItCTOxCbx8Funm2Q3WtBgcv2VZnARRiYhc4jsOEftqt8d7zttWkphATuzGhbr/4eGYRVBW206SmEBO7EdDBBUHeLqjVMPx0KZ/vckyGQkzsiu7CDxtqUlOIiV15sm6/+MS1uygur+a5GtOgEBO74u/RDsG+CqhtqElNISZ2p/4pEbbyLGMKMbE74/tqQ2wrTWoKMbE7XdrbVpOaQkzs0ri+ttOkphATu/Rwk/qewJvUFGJil/w92qG3j7ZJLfQHr1GIid0a11fbid6+i8JuUlOIid16oq5JnXRV2NdSU4iJ3eru6YyeXi6oUTP8nC7cJjWFmNi1sXX9Uu+/KNxTTRRiYtfqQ3ws8w7Kq2p5rsY4FGJi14K8XeDf3gnVtRrBdqJHISZ2jeM43aNehNqkphATu/dEH+1R6iMZhaisEV6/1BRiYvf6dVTCRylHebUaSQJ81AuFmNg9kYjD4729AECQN0RQiAkBdPvFP6cXolZgT0+kEBMCINTfHW5ODigur8bpLGE9kJxCTAi0T0+M7KVtUh+8LKwmNYWYkDqPB2ub1AcvFYAxxnM1+qMQE1LnkUAPODqIcev+A1y6XcJ3OXqjEBNSR+4gxqM9PAAABwV0jzGFmJCHPN67vkktnP1iCjEhDxndqwPEIg4Z+aXIvlvBdzl6oRAT8hBXJymG+LsBEM5RagoxIX8ypnf9hR/C2C+mEBPyJ/WXYJ7Ouof7FdbfE6bZQrx27VpERETAyckJrq6ues3DGMPKlSvh4+MDR0dHREZGIjMz01wlEtIkP3cnBHm7QK1h+CXD+u8xNluIq6urMW3aNMyfP1/ved5//3189NFH2LRpE06dOoV27dohKioKlZWV5iqTkCaNqdsaC6I7W2Zm8fHxTKlUtjqdRqNh3t7e7IMPPtCNu3//PpPJZOw///mP3stTqVQMAFOpVMaUSwhjjLFzOfdYl2V7We+/72eVNbUWWaaxv12r2Se+ceMG8vPzERkZqRunVCoRFhaG5OTkZuerqqpCSUlJg4GQturjq0QHFxnKq9U4eb2Y73JaZDUhzs/XHs738vJqMN7Ly0v3XlPi4uKgVCp1g5+fn1nrJPZBJOIwuu6GiJ+tvEltUIhjY2PBcVyLQ0ZGhrlqbdLy5cuhUql0Q05OjkWXT2zXmN4dAACJ6dZ9Q4TEkIkXL16M2bNntzhNt27djCrE21t7bq6goAA+Pj668QUFBejfv3+z88lkMshkMqOWSUhLIrprb4i4rarE5bwSBPsq+S6pSQaF2NPTE56enmYppGvXrvD29kZiYqIutCUlJTh16pRBR7gJMRW5gxjDAz1w6HIBEtMLrTbEZtsnzs7ORlpaGrKzs6FWq5GWloa0tDSUlZXppgkKCsKuXbsAaLsOXbRoEd555x3s2bMHFy5cQExMDHx9fREdHW2uMglpUWSvuia1FZ8vNmhLbIiVK1di69atutcDBgwAABw+fBgjR44EAFy5cgUqlUo3zdKlS1FeXo558+bh/v37GD58OA4cOAC5XG6uMglp0aie2hCfy7mPwtJKdHCxvt8ix6x5j90IJSUlUCqVUKlUUCgUfJdDbMDEf/2Kc7kqvD+lH54eYr6zH8b+dq3mFBMh1uqxoLpTTVZ6QwSFmJBWjK7bL/71ahGqaq3vCREUYkJaEeyrgJdChopqNU5Z4dVbFGJCWsFxHB4L0m6NrfGuJgoxIXqoP0qdmGF9V29RiAnRw7AAD0glIuQUP8DVwrLWZ7AgCjEhemgnk2Bot/YArK9JTSEmRE+P9dRecnz4CoWYEEGqP1/8W9Y9lFTW8FzNHyjEhOipc3sndPNsh1oNw6+Z1vMwcgoxIQZ4rO4o9WEr2i+mEBNigFF154sPX7kDjcY6TjVRiAkxwBB/d7STilFUVoXLedbRnxuFmBADSCUiRARon5xoLU1qCjEhBqq/euvI73d4rkSLQkyIgUbWnS9OzbaOx7xQiAkxkK+rI3p4OUPDgGNWcKqJQkyIEXRNaiu4eotCTIgRRvTQNqmP/V7E+6kmCjEhRhjs7w4nKznVRCEmxAhSiQgR3bWnmo7yfJSaQkyIkeqPUh+9QiEmRJDq94vPZPN7VxOFmBAj+blr72pSaxhOXOXvVBOFmJA2qN8a87lfTCEmpA0efehUE18d6FGICWmDoV3bQyoR4db9B7h2h58O9CjEhLSBo1SMsK7uAICjv/OzX0whJqSN+N4vphAT0kb1+8Wnrt9FZY3ln9VEISakjQI7OMNLIUNVrQansyz/rCYKMSFtxHEcHgnUbo2P83BrIoWYEBP441ST5feLKcSEmMDwAA9wHJCRX4rCkkqLLptCTIgJuLeToo+vEoDle/ugEBNiIo8Eam9N/DXTsk1qCjEhJjK8PsRX71r0EkwKMSEmMqiLGxwdtL19ZOSXWmy5FGJCTEQmESOsm/YSzOMWbFJTiAkxIT7OF1OICTGh+oNbKTeKLXYJJoWYEBN6+BLMMzfvWWSZFGJCTIjjOAyre+CapZrUFGJCTKy+SZ1koX63KMSEmNiwuv6oL95W4V65+R+4RiEmxMQ6KOTo6eUCxoAT1+6afXkUYkLM4I+rt8x/vphCTIgZDAtoDwBIukpbYkIEKbRre0hEHLKLK5BTXGHWZVGICTEDZ5kE/f1cAZj/KDWFmBAzqT9f/CuFmBBhqg/xiWt3zfogcgoxIWbS388VTlIxisurzXprIoWYEDORSkQIrXs6xIlr5mtSU4gJMaP6q7fMeXCLQkyIGUXUnS9OuVGMGrXGLMugEBNiRr28FXBzckB5tRrncu6bZRkUYkLMSCTiEN7dvFdvUYgJMbOI+v1iMx3ckpjlUwkhOo8GeuKZUD/dI1BNjUJMiJl1bu+EuMn9zPb51JwmROAoxIQIHIWYEIGjEBMicBRiQgSOQkyIwFGICRE4s4V47dq1iIiIgJOTE1xdXfWaZ/bs2eA4rsEwduxYc5VIiE0w28Ue1dXVmDZtGsLDw/H555/rPd/YsWMRHx+vey2TycxRHiE2w2whXr16NQAgISHBoPlkMhm8vb3NUBEhtsnq9omPHDmCDh06oGfPnpg/fz7u3m35zo+qqiqUlJQ0GAixJ1YV4rFjx2Lbtm1ITEzE//3f/+Ho0aN44oknoFY3/5zXuLg4KJVK3eDn52fBigmxAswAy5YtYwBaHNLT0xvMEx8fz5RKpSGL0bl27RoDwH7++edmp6msrGQqlUo35OTkMABMpVIZtUxC+KJSqYz67Rq0T7x48WLMnj27xWm6detm8D+Slj7Lw8MDV69exejRo5ucRiaTNTj4xZi2a1BqVhOhqf/N1v+G9WVQiD09PeHpaZ57IpuSm5uLu3fvwsfHR+95Sku1XYNSs5oIVWlpKZRKpd7Tm+3odHZ2NoqLi5GdnQ21Wo20tDQAQEBAAJydnQEAQUFBiIuLw6RJk1BWVobVq1djypQp8Pb2xrVr17B06VIEBAQgKipK7+X6+voiJycHLi4u4Diu0fslJSXw8/NDTk4OFAqFSb6rPaH1Z7zW1h1jDKWlpfD19TXsg83RtmeMsVmzZjW5z3z48GHdNABYfHw8Y4yxiooK9vjjjzNPT0/m4ODAunTpwubOncvy8/NNWpex+x1Ei9af8cy17jjGDGyAC1xJSQmUSiVUKhVtSYxA68945lp3VnWKiRBiOLsLsUwmw6pVq+hyTiPR+jOeudad3TWnCbE1drclJsTWUIgJETgKMSECRyEmROAoxIQInE2G+OOPP4a/vz/kcjnCwsKQkpLS4vTfffcdgoKCIJfL0bdvX+zbt89ClVonQ9ZfQkJCoy6V5HK5Bau1HseOHcOECRPg6+sLjuOwe/fuVuc5cuQIBg4cCJlMhoCAAIM70QBsMMQ7duzAm2++iVWrVuHs2bMICQlBVFQUCgsLm5z+xIkTeOaZZzBnzhykpqYiOjoa0dHRuHjxooUrtw6Grj8AUCgUyMvL0w03b960YMXWo7y8HCEhIfj444/1mv7GjRsYP348Ro0ahbS0NCxatAgvvvgifvrpJ8MWbNKLOK1AaGgoe/XVV3Wv1Wo18/X1ZXFxcU1O//TTT7Px48c3GBcWFsZeeukls9ZprQxdf225X9yWAWC7du1qcZqlS5ey4ODgBuOmT5/OoqKiDFqWTW2Jq6urcebMGURGRurGiUQiREZGIjk5ucl5kpOTG0wPAFFRUc1Ob8uMWX8AUFZWhi5dusDPzw8TJ07EpUuXLFGu4Jnqt2dTIS4qKoJarYaXl1eD8V5eXsjPz29ynvz8fIOmt2XGrL+ePXviiy++wA8//ICvvvoKGo0GERERyM3NtUTJgtbcb6+kpAQPHjzQ+3Po+cSkTcLDwxEeHq57HRERgV69emHz5s1Ys2YNj5XZD5vaEnt4eEAsFqOgoKDB+IKCgma7wfX29jZoeltmzPr7MwcHBwwYMABXr141R4k2pbnfnkKhgKOjo96fY1MhlkqlGDRoEBITE3XjNBoNEhMTG2wtHhYeHt5gegA4dOhQs9PbMmPW35+p1WpcuHDBoC6V7JXJfnuGHnWzdtu3b2cymYwlJCSwy5cvs3nz5jFXV1ddDyHPPfcci42N1U2flJTEJBIJW7duHUtPT2erVq1iDg4O7MKFC3x9BV4Zuv5Wr17NfvrpJ3bt2jV25swZNmPGDCaXy9mlS5f4+gq8KS0tZampqSw1NZUBYOvXr2epqans5s2bjDHGYmNj2XPPPaeb/vr168zJyYn99a9/Zenp6ezjjz9mYrGYHThwwKDl2lyIGWNs48aNrHPnzkwqlbLQ0FB28uRJ3XsjRoxgs2bNajD9t99+y3r06MGkUikLDg5mP/74o4Urti6GrL9FixbppvXy8mLjxo1jZ8+e5aFq/h0+fLjJLqnq19esWbPYiBEjGs3Tv39/JpVKWbdu3XTdVRmC7icmROBsap+YEHtEISZE4CjEhAgchZgQgaMQEyJwFGJCBI5CTIjAUYgJETgKMSECRyEmROAoxIQI3P8DX0CZb83z7EUAAAAASUVORK5CYII=","text/plain":["
"]},"metadata":{},"output_type":"display_data"},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAiMAAAGlCAYAAAAoK/bpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVaklEQVR4nO3deVhUZf8G8HtmYGbYF5FNRhF3UUBBCM3UwpfKNDMLK0WtbHGr+LVolphWWJlRaZnmUlpp+bq9aqSipimF4r7hiuDCpsKwCAMzz+8PdIoEZRA4zHB/rmsu5ficc75zVObmnGeRCSEEiIiIiCQil7oAIiIiatoYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREkmIYISIiIkkxjBAREZGkGEaIiIhIUgwjREREJCmGEaImqLCwELGxsXjwwQfh6uoKmUyGJUuW1GjfPXv2YPz48fD394ednR1atmyJJ598EidPnqzfouvJ9u3bIZPJqnz9+eeft7Q/deoUhg0bBh8fH9ja2qJjx46YPn06iouL6/xcRE2FldQFEFHDy83NxfTp09GyZUsEBgZi+/btNd73o48+wq5du/DEE08gICAAmZmZmDNnDrp3744///wTXbp0qb/C69HEiRPRo0ePStvatm1b6euMjAyEhobCyckJ48ePh6urK5KSkhAbG4uUlBSsXbu2zs5F1JQwjBA1QV5eXrh8+TI8PT2xd+/eWz4YbycmJgY//vgjlEqlcVtUVBS6du2KmTNnYtmyZfVRcr3r3bs3hg4dets2S5cuRV5eHv744w/4+/sDAF544QUYDAZ8//33uHbtGlxcXOrkXERNCR/TEEmof//+1f5EHBISgrCwsHo5r0qlgqenZ6327dmzZ6UgAgDt2rWDv78/jh8/fld1rVixAt27d4eNjQ06deqELVu2QAgBf39/fPDBB3d17JooKChAeXl5tX+u1WoBAB4eHpW2e3l5QS6X33Jd7uZcRE0JwwiRhPz9/ZGWlobS0tJK29esWYOUlBS8//77t+xTVlaG3NzcGr0MBkODvA8hBLKysuDm5lbrY7zzzjsYNmwYAgMDMWvWLOj1ekRHR2Pjxo24cOECxo8ff8s+dXktRo8eDUdHR6jVavTr1w979+69pU3fvn0BAM899xwOHDiAjIwMrFixAl9//TUmTpwIOzu7Gr3XmpyLqEkRRCSZ+fPnCwDi8OHDxm0Gg0EEBASIPn36VLnPtm3bBIAavc6dO3fHGvbs2SMAiMWLF9f6fSxdulQAEAsXLqzV/jt27BAAxFtvvWXctnLlSgFAdOnSpdL2f6qLa7Fr1y7x+OOPi4ULF4q1a9eKuLg40axZM6FWq8W+fftuaT9jxgxhY2NT6dhTpkyp0fs09VxETYVMCCEaJvYQ0b/t3r0bvXr1wi+//GLsQ/Dzzz8jKioKO3fuxL333nvLPteuXUNKSkqNjn/vvfdCrVbfts3NPiOLFy/GqFGjTH4PJ06cQFhYGPz9/bFz504oFAqTjzF06FBs3boV6enpsLe3BwCkpKQgJCQEarUaaWlptzwaAer+Wtx0+vRpBAQE4L777kNCQkKlP1u2bBmWLVuGxx9/HM2aNcOGDRuwePFifPHFF1XevbmbcxE1GVKnIaKmLC8vTwAQM2bMEEIIodfrRadOnURkZGSD1XA3d0YuX74s/Pz8hEajERcvXqzV+cvLy4WDg4N4+umnK23fu3evACBefvnlWh33bg0bNkwolUpRXl5u3PbTTz8JGxsbkZGRUantqFGjhK2trcjNza2zcxE1JRxNQyQhJycneHt748SJEwCAH3/8EcePH8f3339f7T46nQ5Xr16t0fGbN29eqzsVNZGfn4+HHnoIeXl52LlzJ7y9vWt1nLNnz6KgoADdu3evtD0nJwcAMG7cuGr3rc9rodFooNPpUFRUBEdHRwDAV199hW7dusHHx6dS20GDBmHJkiXYv38/IiIianyO252LqClhGCGSmL+/P06cOAG9Xo/p06dj8ODBCAkJqbb97t270a9fvxod+9y5c/D19a2jSv9WUlKCgQMH4uTJk9iyZQs6d+5c62PdDB3/7vwaFxdX5fZ/qs9rcfbsWajVauNjIwDIysqqcuhuWVkZANR6dExV5yJqShhGiCTm7++Pb7/9Ft999x3OnDmDVatW3bZ9YGAgNm/eXKNj13b47k3FxcVIT0+Hm5ubMRTo9XpERUUhKSkJa9euRXh4+F2dw8nJCQBw5MgR47Yff/wRO3bsAFARfKpTF9ciJycHzZs3r7Tt4MGDWLduHR566CHI5X8POmzfvj02bdqEkydPon379sbtP/30E+RyOQICAgBUfd1MPRdRU8IOrEQS+/bbbzFmzBg0b94cERER+PHHHxvkvHPmzEFeXh4uXbqEr7/+GkOGDEG3bt0AABMmTICTkxO2b9+Ofv36ITY2FtOmTQMAvPrqq/j8888xcOBAPPnkk7ccd/jw4cbfy2Qy9OnT57YzvBoMBrRt2xYXL17EpEmTIJfLMXPmTAwaNAg///wzRo0ahZiYGHTt2rVO3/9N999/P2xsbNCzZ0+4u7vj2LFjmD9/PqytrZGUlIROnToZ2+7YsQP3338/mjVrhvHjx6NZs2ZYv349fv31Vzz//PNYsGABAFR53Uw9F1GTInWnFaKmbvfu3QKAUCgU4uTJkw123latWt1xGOzNobOxsbHG/fr06XPbIbQ3FRQUCABi2LBhd6xl//794p577hEqlUq4uLiIKVOmCIPBIJ599llhZWUllixZUtdv3+jzzz8XoaGhwtXVVVhZWQkvLy8xfPhwcerUqSrb//XXX+Khhx4Snp6ewtraWrRv31588MEHoqyszNimqutWm3MRNRW8M0JE9WLjxo145JFHcPDgwXq7q0FEloEPKImoXmzbtg3Dhg1jECGiO+KdESIiIpIU74wQERGRpBhGiIiISFIMI0RERCQphhEiIiKSlFnMwGowGHDp0iU4ODhAJpNJXQ4RERHVgBACBQUF8Pb2vu0Mw2YRRi5dugSNRiN1GURERFQLGRkZtyww+U9mEUYcHBwAVLwZrmhJRERkHrRaLTQajfFzvDpmEUZuPppxdHRkGCEiIjIzd+piwQ6sREREJCmGESIiIpIUwwgRERFJyiz6jNSEwWCATqeTugyqI9bW1lAoFFKXQUREDcAiwohOp8O5c+dgMBikLoXqkLOzMzw9PTm3DBGRhTP7MCKEwOXLl6FQKKDRaG47qQqZByEEiouLkZ2dDQDw8vKSuCIiIqpPZh9GysvLUVxcDG9vb9ja2kpdDtURGxsbAEB2djbc3d35yIaIyIKZ/W0EvV4PAFAqlRJXQnXtZrgsKyuTuBIiIqpPZh9GbmK/AsvDv1MioqbBYsIIERERmSeGEQvSt29fvPrqq1KXQUREZBKGEQnIZLLbvqZNm1ar465atQozZsyoszoZboiIqCGY/Wgac3T58mXj71esWIGpU6ciNTXVuM3e3t74eyEE9Ho9rKzu/Ffl6upat4XWEZ1Oxw7GRNRgyvUGXCsuQ16xDleLdLhWrEP+9TIUlJRDW1KOgpIyFJaUo7hMjxKdHtfLKl6lZQaU6Q0oN4iKX/UCAgJCAOLGsWUArOQyKBQyWMnlUMhlUFnJobZWQG0th9pKAbVSAQeVFexVVrBTWcFBbQUnG2s42yrhbGsNF9uK37vaKiGXS9s3TggBfV4eDEXFkNvZQuHsLEl/PYYRCXh6ehp/7+TkBJlMZty2fft29OvXDxs3bsQ777yDw4cPY9OmTdBoNIiJicGff/6JoqIidOrUCXFxcYiIiDAeq2/fvggKCkJ8fDwAoLS0FFOmTMFPP/2EvLw8dOnSBR999BH69u1r3GfXrl2YMmUKkpOToVKpEBoaiuXLl+O1117D77//jt9//x2ff/45AODcuXPw9fXF77//jjfeeAMHDx6Eq6srRo4ciffff98YmPr27YsuXbrAysoKy5YtQ9euXdG6dWtkZ2dj/fr1xnOXlZWhRYsWiIuLw3PPPVdfl5uILIQQAvnXy3Ax7zou55Xgcv51XMovQZa2BDkFpcgpKEV2QSmuFpnHbNwKuQyudkq42avgZq+Ep6MaXs428HJSw9NJDW8nG/i42MBOVfcf1XqtFvlr1uDqsmUoS88wbrduqYHr8OFwGjwYCkfHOj9vdSwujAghcL1ML8m5bawVdZYoJ02ahFmzZsHPzw8uLi7IyMjAww8/jA8++AAqlQrff/89Bg4ciNTUVLRs2bLKY4wfPx7Hjh3D8uXL4e3tjdWrV+PBBx/E4cOH0a5dOxw4cAAPPPAAnn32WXz++eewsrLCtm3boNfr8fnnn+PkyZPo0qULpk+fDgBo3rw5Ll68iIcffhijRo3C999/jxMnTmDMmDFQq9WVHi999913ePnll7Fr1y4AwJUrV3Dffffh8uXLxknM1q9fj+LiYkRFRdXJNSMi8yeEQHZBKc7kFOJcbhHOXylG+pVipF8tRsbVYhSUltf4WM621nC9cTfC2VYJB7XVjZd1xV0LpQI2SgXU1grYWCugslbAWiGDtUIOK3nFrze/pcsgg0wGGISAwQCUGwzQGwR0egN05QaUlBlQWq5HSZkeRaV6FJWWo1BXjsKSchSWliOvuAx51yvu1uQVlyH/ehn0BmEMUbfjaqeEj0tFMPFtZgdfNzv4uVX82sxOafLnTuHOP3Bh4kSIkpJb/qws4wKy4mYi+7N4+HzxBex732vSsWurVmFk7ty5+OSTT5CZmYnAwEB8+eWXCA0NrbJt37598fvvv9+y/eGHH8aGDRtqc/rbul6mR+epv9X5cWvi2PRI2CrrJt9Nnz4d/fv3N37t6uqKwMBA49czZszA6tWrsW7dOowfP/6W/dPT07F48WKkp6fD29sbAPD6668jISEBixcvxocffoiPP/4YISEh+Oqrr4z7+fv7G3+vVCpha2tb6U7OV199BY1Ggzlz5kAmk6Fjx464dOkS3nrrLUydOtU4A267du3w8ccfV6qpQ4cOWLp0Kd58800AwOLFi/HEE09UeixFRE2DEAIX867jVFYhUrMKcDKrAKeyKgJI4R0Ch6udEt7Oang52cDbSQ0PJzXcHdRwd1ChuYMKbvYquNhaw0rReLtFlukNuFqkqwgjhTfu6mhLcDm/BJn5Fb9eyr+OvOIyXC2qeNx06EL+LcdxVFuhnYcD2rnbo627Pdp7OKCjpwOaO6iqDCmFO/9AxosvouLZk7jlz29uEyUlyHjxRWi++aZBAonJn5wrVqxATEwM5s2bh7CwMMTHxyMyMhKpqalwd3e/pf2qVasqLWB35coVBAYG4oknnri7yi1cSEhIpa8LCwsxbdo0bNiwAZcvX0Z5eTmuX7+O9PT0Kvc/fPgw9Ho92rdvX2l7aWkpmjVrBgA4cOCAyX8Px48fR3h4eKV/5L169UJhYSEuXLhgvEsTHBx8y77PP/885s+fjzfffBNZWVn49ddfsXXrVpPOT0Tmp0xvQGpmAY5d1uLYJS2OXdbi+CVttXc55DKgpastWt/46b+lq63x5eNiCxul+c/IbK2Qw8NRDQ9H9W3baUvKcOHqdWRcq7gzlHalCGm5xTiXW4RL+dehLSlHyvlrSDl/rdJ+rnZKdPJyQCdPR3T2dkSAjxNaKg24MHFi9UHkn278+YWJE9Hu9+31/sjG5DAye/ZsjBkzBqNHjwYAzJs3Dxs2bMCiRYswadKkW9r/u1Pl8uXLYWtrW29hxMZagWPTI+vl2DU5d12xs7Or9PXrr7+OzZs3Y9asWWjbti1sbGwwdOjQalcqLiwshEKhQEpKyi1Tqd+8E3FzyvX68O/6ASA6OhqTJk1CUlISdu/ejdatW6N37971VgMRNTwhBM7kFOFgRh4OXcjDwQv5OHZZC135rQuZWsllaNPcHu087NHBwwHtPBzQ1t0OLV3toLRqvHc1GpKj2hqdva3R2fvWMFBSpkfalSKcyirEqawCnMouxMmsApzLLcLVIh12nb6CXaevGNvbygX8gkejXV4G/K+cQ6/LR25/ciEgSkqQv2YtXKNH1PVbq8SkMKLT6ZCSkoLJkycbt8nlckRERCApKalGx1i4cCGGDRtW5YdVXZDJZHX2qKQx2bVrF0aNGoXHHnsMQEXYSEtLq7Z9t27doNfrkZ2dXe0HfkBAABITE/Hee+9V+edKpdI43f5NnTp1wn//+18IIYx3R3bt2gUHBwf4+Pjc9j00a9YMgwcPxuLFi5GUlGQMtERkvq7r9DiQkYd96RU/ne9Lv4a84luXcHBUW8Hf2wmdvR3R2avip/U2ze0ZOu6C2lqBjp6O6OhZOaiUlOlxMqsAxy9rcfxyAY5czMfRS1oUl+lxxM0PR9z8cNbJ+85h5Iary5bCZcTweh1lY9Kndm5uLvR6PTw8PCpt9/DwwIkTJ+64f3JyMo4cOYKFCxfetl1paSlKS//u0KPVak0p0yK1a9cOq1atwsCBAyGTyfDuu+/CYLj1J42b2rdvj2eeeQbR0dH49NNP0a1bN+Tk5CAxMREBAQEYMGAAJk+ejK5du2Ls2LF46aWXoFQqsW3bNjzxxBNwc3ODr68v/vrrL6SlpcHe3h6urq4YO3Ys4uPjMWHCBIwfPx6pqamIjY1FTExMjVZMfv755/HII49Ar9dj5MiRdXmJiKgBFJaWY2/aVSSfu4q/zl3FoQt5KNNXvuWvspKjawsnBPg4I1BT8atvM1su8dBA1NYKBPg4I8DH2bit9MpVbHtwCE46a3DKxQcttVk1O5gQKEvPgD4vD1YuLvVTMBp4NM3ChQvRtWvXaju73hQXF1ftT+tN1ezZs/Hss8+iZ8+ecHNzw1tvvXXHkLZ48WK8//77+L//+z9cvHgRbm5uuOeee/DII48AqAgsmzZtwttvv43Q0FDY2NggLCwMTz31FICKR0MjR45E586dcf36dePQ3o0bN+KNN95AYGAgXF1d8dxzz+Gdd96p0fuIiIiAl5cX/P39jR1riajx0pUbsD/9GnadzsWuM1dwMCMP5YbK4cPTUY1gXxcEt3RB91Yu6OzlyDsejYzs+nW0KshCq4Is9M/Ya/L+hqJioB7DiEyIO/Vi+ZtOp4OtrS1WrlyJwYMHG7ePHDkSeXl5WLt2bbX7FhUVwdvbG9OnT8crr7xy2/NUdWdEo9EgPz8fjv/qRFNSUoJz586hdevWUKtv3xHI0oWHh+OBBx7A+++/L3Up1SosLESLFi2wePFiDBky5LZt+XdLJI203CL8fjIHv5/MQdKZK7dMl6BxtUFY62YIbe2Ke1o3g8bVhnc9Grnya9dwKrxnrfdvl7S7VndGtFotnJycqvz8/ieT7owolUoEBwcjMTHRGEYMBgMSExOrHF76T7/88gtKS0sxfPjwO55HpVJBpVKZUlqTVlpaisOHD+Po0aOYOHGi1OVUyWAwIDc3F59++imcnZ0xaNAgqUsiohtKy/X48+xVbD2ehe0nc3D+SnGlP3ezVyK8jRvubdsMPdu4QeNqK1GlVFsKZ2dYt9SgLOPCnUfS/JNMBmuNDxTOzvVWG1CLxzQxMTEYOXIkQkJCEBoaivj4eBQVFRk7I0ZHRxtn1fynhQsXYvDgwcZhpVR3fv31V0RHR2PQoEEYOnSo1OVUKT09Ha1bt4aPjw+WLFlSo+ntiaj+XC3SIfF4FhKPZ2PnqRwU6f6++2GtkCGklSv6dGiO+9o1RycvB975MHMymQyuw4cjK26myfu6Dh9R73//Jn8iREVFIScnB1OnTkVmZiaCgoKQkJBg7NSanp5+S0fG1NRU/PHHH9i0aVPdVE2VDB48uNF38vX19YUJTwSJqB5cyruO345m4rejmUg+dxX/7Prh7qDCA53c0a+DO3q2dYN9PUxBTtJyGjwY2Z/FV8y8WpPvx3I5ZCoVnAY/Wu+1mdRnRCq3e+bEfgWWi3+3RHfvYt51bDh0CRsOXcbBf83g6e/tiIhOHojo5AF/b0fJF22j+nfHGVhvkskAmQya+fNhf2+vWp+vXvqMEBFR45elLcH6Q5ex/tAl7E/PM26XyYCQVi6I9PdEpL8n+340Qfa974Xmm28qr03zz1By43GMTK2Gz5df3lUQMQXDCBGRBdCWlCHhcCbWHLiIpLNXjJ8vMhkQ6uuKRwK8ENnFE+4OvMvY1Nn3vhftft+O/DVrcXXZ0sqr9mp84Dp8BJweGwyFg0OD1cQwQkRkpsr1Buw4lYOVKRew5Xh2pSnXg1u5YGCAFx7u6gX3O6x/Qk2PwtERrtEj4DJiOPR5eTAUFUNuZwuFs7MknZUZRoiIzExqZgH+u+8CVu27iNzCv+dkauduj8HdWmBQoDcfwVCNyGSyivlD6nFCs5pgGLlBCNEo0iERUVUKS8ux/uAl/LQnAwcz8ozbXe2UGBzUAkO6t4C/tyO/b5FZavJhRK/VIn/NGlxdtqzyc7OWGrgOHw6nwYPrfelkIqKqCCFw6EI+fkpOx/8OXjLOBWIll+H+ju4YGuyDvh3cOfU6mb0mHUYKd/5RuUfxP5RlXEBW3ExkfxYPny++gH3ve+vsvHf6ySU2NhbTpk2rs/OZQiaTYfXq1ZWm+yeihnVdp8e6gxex9M/zOHLx7zmE/NzsMCxUgyHdfeBmz1mqyXI02TByx7HWN7aJkhJkvPgiNN98U2eB5PLly8bfr1ixAlOnTkVqaqpxm729vUnH0+l0UCqVdVIbEUnnbE4hlv55HitTLqCgpBwAoLSSY0BXLwzroUFoa1c+hiGL1CTv7em1WlyYOPHOk74AxjYXJk6Evo5mOfX09DS+nJycIJPJjF8XFRXhmWeegYeHB+zt7dGjRw9s2bKl0v6+vr6YMWMGoqOj4ejoiBdeeAEAsGDBAmg0Gtja2uKxxx7D7Nmz4fyv9QTWrl2L7t27Q61Ww8/PD++99x7Ky8uNxwWAxx57DDKZzPg1EdUfg0FgW2o2Ri5Kxv2f/o7Fu9JQUFKOVs1s8fbDHfHX5AfwWVQQwvyaMYiQxWqSd0by16yp+XS4ACAEREkJ8teshWv0iHqtrbCwEA8//DA++OADqFQqfP/99xg4cCBSU1PRsmVLY7tZs2Zh6tSpiI2NBQDs2rULL730Ej766CMMGjQIW7Zswbvvvlvp2Dt37kR0dDS++OIL9O7dG2fOnDEGmdjYWOzZswfu7u5YvHgxHnzwQSgUinp9r0RNWVFpOVamXMB3u9NwNrcIQMWcIA90dMeIcF/0buvGGVGpyWhy08ELIXAmMrLWKxe2+e23Ov3pZMmSJXj11VeRl5dXbZsuXbrgpZdeMq6M7Ovri27dumH16tXGNsOGDUNhYSHWr19v3DZ8+HCsX7/eeOyIiAg88MADmDx5srHNsmXL8Oabb+LSpUs33mbj6TPC6eDJEmVpS7Bkdxp++PM8tDcexTiorPBkDw1GhvuiZTMOySXLwengq6HPy6s0aqbGhEBZegb0eXkVY7LrSWFhIaZNm4YNGzbg8uXLKC8vx/Xr15Genl6pXUhISKWvU1NT8dhjj1XaFhoaWimcHDx4ELt27cIHH3xg3KbX61FSUoLi4mLY2vKbIFF9OZGpxfwdZ/G/g5dQpq/4Qai1mx1G9/LF4919YMeF6agJa3L/+g1FxXe/fz2Gkddffx2bN2/GrFmz0LZtW9jY2GDo0KHQ6XSV2tnZ2Zl87MLCQrz33nsYMmTILX/GOw9EdU8IgT1p1/D19tPYlppj3B7q64rne7dGRCcPPoohQhMMI3K7u/vp/273v5Ndu3Zh1KhRxrschYWFSEtLu+N+HTp0wJ49eypt+/fX3bt3R2pqKtq2bVvtcaytraHX600vnIiMhBBIPJ6Nr38/g5Tz1wAAchnwUBcvjLnPD0EaZ2kLJGpkmlwYUTg7w7qlptZ9RhT/Gp1S19q1a4dVq1Zh4MCBkMlkePfdd2EwGO6434QJE3Dfffdh9uzZGDhwILZu3Ypff/21Uv+WqVOn4pFHHkHLli0xdOhQyOVyHDx4EEeOHMH7778PoKI/SmJiInr16gWVSgUXiacIJjIneoPAr0cuY87W0ziRWQAAUCrkeDzYBy/c54fWbqbf0SRqCprc0F6ZTAbX4cNrta/r8BH1PrRu9uzZcHFxQc+ePTFw4EBERkaie/fud9yvV69emDdvHmbPno3AwEAkJCTgtddeq/T4JTIyEuvXr8emTZvQo0cP3HPPPfjss8/QqlUrY5tPP/0UmzdvhkajQbdu3erlPRJZmnK9Aav2XcB/Pvsd43/cjxOZBbBTKvBiHz/88VY/xA3pyiBCdBtNbjQNUDHPyKk+fWs+vFcuh0ylQrvft5vV1PBjxozBiRMnsHPnTqlLqRWOpqHGrlxvwLqDl/BF4imkXanoj+aotsLoXq0xupcvnG05GSE1bRxNcxsKR0f4fPFFxQyswO0DyY07IT5fftnog8isWbPQv39/2NnZ4ddff8V3332Hr776SuqyiCyO3iDwvxsh5OYcIa52Sjx3b2tEh7eCg9pa4gqJzEuTDCMAYN/7Xmi++aby2jT/DCU3QohMrYbPl1/C/t5eElRpmuTkZHz88ccoKCiAn58fvvjiCzz//PNSl0VkMQwGgYSjmZi9+SROZxcCAJxtrfHCfX4YGe7L4blEtdSk/+fY974X7X7fjvw1a3F12dLKq/ZqfOA6fAScHhsMhYODhFXW3M8//yx1CUQWSQiB7Sdz8OmmVOPCdU42N0JIT1/YM4QQ3ZUm/z9I4egI1+gRcBkxHPq8PBiKiiG3s4XC2ZnrQBAR9qRdxccJJ7AnrWKIrp1Sged7++G53q3hyMcxRHXCYsLI3fbDlclkFTOrcihro2EGfavJgp3OLsDMX1Ox5XgWgIrVc0eGt8LLfdvC1Y4dU4nqktmHkZuLuel0OtjY2EhcDdWl4uKK0QnW1vzpkxpOlrYEn20+iZ/3ZsAgAIVchidDfDDxgXbwcuL3GKL6YPZhxMrKCra2tsjJyYG1tTXk8iY3dYrFEUKguLgY2dnZcHZ25urB1CCKSsvxzY6zmL/jDErKKiYa/E9nD7z5YEe0dbeXuDoiy2b2YUQmk8HLywvnzp3D+fPnpS6H6pCzszM8PT2lLoMsnN4g8N+UC5i1KRXZBaUAgOBWLpj8UEeE+LpKXB1R02D2YQQAlEol2rVrd8ticmS+rK2teUeE6t3u07mYvv6Ycer2Vs1sMenBjniwiyc7sBM1IIsIIwAgl8s5SycR1Uj6lWJ8sPEYfjta0TnVUW2FiQ+0w4jwVlBZMQQTNTSLCSNERHdSVFqOr7afxoKd56ArN0Ahl2F4WEu8GtEeLhwhQyQZhhEisnhCCKw7eAkfbjyOLG1Fv5B727ph6sDOaO9hHpMaElkyhhEismipmQWYuvYI/jp3FQDQ0tUW7wzohP6dPdgvhKiRYBghIotUUFKG+C2nsGR3GvQGAbW1HOP6tsWY+/ygtma/EKLGhGGEiCyKEAIbDl/G9P8dMw7VjfT3wLuPdIaPi63E1RFRVRhGiMhipOUW4d21R7DzVC4AwLeZLd57tAv6tG8ucWVEdDsMI0Rk9krL9fjm97OYs+00dOUGKBVyjO3XBi/1acNHMkRmoFZzp8+dOxe+vr5Qq9UICwtDcnLybdvn5eVh3Lhx8PLygkqlQvv27bFx48ZaFUxE9E970q5iwBd/YPbmk9CVG9C7nRt+e+0+vBrRnkGEyEyYfGdkxYoViImJwbx58xAWFob4+HhERkYiNTUV7u7ut7TX6XTo378/3N3dsXLlSrRo0QLnz5+Hs7NzXdRPRE2UtqQMH/16Aj/8lQ4AcLNX4t1HOmNQoDdHyRCZGZkwcZ32sLAw9OjRA3PmzAEAGAwGaDQaTJgwAZMmTbql/bx58/DJJ5/gxIkTtV59VavVwsnJCfn5+XB0dKzVMYjIciQcycTUtUeMHVSjQjSY/HBHONty4jKixqSmn98mPabR6XRISUlBRETE3weQyxEREYGkpKQq91m3bh3Cw8Mxbtw4eHh4oEuXLvjwww+h1+tNOTUREXIKSjHuh314aVkKsgtK4edmh5/G3IOPhgYwiBCZMZMe0+Tm5kKv18PDw6PSdg8PD5w4caLKfc6ePYutW7fimWeewcaNG3H69GmMHTsWZWVliI2NrXKf0tJSlJaWGr/WarWmlElEFkYIgTUHLuK9/x1DXnEZFHIZXurjhwn3t2O/ECILUO+jaQwGA9zd3TF//nwoFAoEBwfj4sWL+OSTT6oNI3FxcXjvvffquzQiMgOZ+SV4e/VhbD2RDQDo5OWIT4YGoEsLJ4krI6K6YlIYcXNzg0KhQFZWVqXtWVlZ8PT0rHIfLy+vW5aD79SpEzIzM6HT6aBU3nprdfLkyYiJiTF+rdVqodFoTCmViMycEAKr91/EtHVHoS0ph1Ihx8QH2uLFPm1grajVQEAiaqRM+h+tVCoRHByMxMRE4zaDwYDExESEh4dXuU+vXr1w+vRpGAwG47aTJ0/Cy8uryiACACqVCo6OjpVeRNR0ZBeUYMz3KYj5+SC0JeUI8HHC+on3Yvz97RhEiCyQyf+rY2JisGDBAnz33Xc4fvw4Xn75ZRQVFWH06NEAgOjoaEyePNnY/uWXX8bVq1fxyiuv4OTJk9iwYQM+/PBDjBs3ru7eBRFZjHUHL+E/n+3AluNZsFbI8EZkB6x6uSdX1yWyYCb3GYmKikJOTg6mTp2KzMxMBAUFISEhwdipNT09HXL53xlHo9Hgt99+w2uvvYaAgAC0aNECr7zyCt566626exdEZPbyinV4Z80RrD90GQDg7+2IT58MREdP3hklsnQmzzMiBc4zQmTZtqdm482Vh5BdUAqFXIbx/dpi/P1t+UiGyMzV9POba9MQkWSu6/T4cONxLP3zPADAr7kdPnsyCIEaZ2kLI6IGxTBCRJI4cjEfE5fvx9mcIgDAqJ6+eOvBjrBRct4QoqaGYYSIGpTeILBg51l8uikVZXoBD0cVZj0RiN7tmktdGhFJhGGEiBrM5fzreG3FAfx59ioA4EF/T8QN6QoXO07lTtSUMYwQUYNIOJKJt/57CPnXy2CrVGDaQH88EeLDFXaJiGGEiOpXSZke7284hmV/pgMAAnyc8PmwbmjtZidxZUTUWDCMEFG9OZlVgAk/7kdqVgEA4MU+fvi//h2gtOKQXSL6G8MIEdU5IQRW7MlA7LqjKC03wM1ehdlPBuK+9uykSkS3YhghojpVWFqOKasPY+2BSwCA+9o3x6dPBKK5g0riyoiosWIYIaI6c+ySFuN/3IezuUVQyGV4/T8d8OJ9fpDL2UmViKrHMEJEd00IgR+T0/He/45BV26Al5MaXz7VDSG+rlKXRkRmgGGEiO5Ksa4cU1Yfwer9FwEA93d0x6dPBHLuECKqMYYRIqq109mFeHlZCk5lF0Ihl+HNyA4Y05uPZYjINAwjRFQr6w5ewqT/HkKxTg93BxXmPN0doa35WIaITMcwQkQm0ZUb8OHG41iyOw0AEO7XDF881Y2jZYio1hhGiKjGsrUlGPvDPuw9fw0AMK5fG8T07wAFH8sQ0V1gGCGiGkk+dxXjftyHnIJSOKisMDsqCP07e0hdFhFZAIYRIrotIQSW7E7DBxuOo9wg0MHDAfNGBHNtGSKqMwwjRFStkjI93l59GKv2VQzbHRTojZmPd4Wtkt86iKju8DsKEVXpUt51vLg0BYcv5kMhl2HyQx3x3L2tIZOxfwgR1S2GESK6xZ9nr2DcD/twpUgHF1trzH26O3q2dZO6LCKyUAwjRFTJ0j/P4711R1FuEOjs5YhvRgRD42ordVlEZMEYRogIAFCmN2DauqP44a90ABX9Qz56PAA2SoXElRGRpWMYISJcLdLh5WUp+OvcVchkwJuRHfFSHz/2DyGiBsEwQtTEncjU4vnv9uLCteuwV1nh82FBeKAT5w8hoobDMELUhG05loVXlu9HkU6PVs1s8W10CNp5OEhdFhE1MQwjRE2QEALf7jyHD389DiGAnm2aYe7T3eFip5S6NCJqghhGiJoYXbkB7645ghV7MwAAT4e1xHuD/GGtkEtcGRE1VQwjRE1IXrEOLy6t6KgqlwHvDOiM0b182VGViCTFMELURKTlFmH0kj04l1sEe5UVvny6G/p1cJe6LCIihhGipmBP2lW88P1eXCsuQwtnGywa1QMdPNlRlYgaB4YRIgu39sBFvPHLIej0BgT6OGHByBC4O6ilLouIyIhhhMhCCSHw5dbTmL35JAAg0t8D8VHdOKMqETU6DCNEFqhMb8CU1Yfx894LAIAX7vPDpAc7Qi5nR1UianwYRogsTGFpOcb+sA87TuZALgPee7QLRtzTSuqyiIiqVauJBebOnQtfX1+o1WqEhYUhOTm52rZLliyBTCar9FKr+byaqD5kaUvw5Lwk7DiZAxtrBRZEhzCIEFGjZ/KdkRUrViAmJgbz5s1DWFgY4uPjERkZidTUVLi7Vz1M0NHREampqcavOacBUd07lVWAkYuScSm/BG72Siwa1QMBPs5Sl0VEdEcm3xmZPXs2xowZg9GjR6Nz586YN28ebG1tsWjRomr3kclk8PT0NL48PLgIF1Fd2pN2FY9/vRuX8kvg19wOq8f2YhAhIrNhUhjR6XRISUlBRETE3weQyxEREYGkpKRq9yssLESrVq2g0Wjw6KOP4ujRo7c9T2lpKbRabaUXEVUt4chlPPPtX9CWlCO4lQv++1JPaFxtpS6LiKjGTAojubm50Ov1t9zZ8PDwQGZmZpX7dOjQAYsWLcLatWuxbNkyGAwG9OzZExcuXKj2PHFxcXBycjK+NBqNKWUSNRnfJ6Xh5R/2QVduQP/OHvjh+TAudkdEZqfeV8YKDw9HdHQ0goKC0KdPH6xatQrNmzfHN998U+0+kydPRn5+vvGVkZFR32USmRUhBD757QSmrj0KISoWu/v6me5QW3MOESIyPyZ1YHVzc4NCoUBWVlal7VlZWfD09KzRMaytrdGtWzecPn262jYqlQoqlcqU0oiajHK9Ae+sOYLleypCekz/9phwf1t2DCcis2XSnRGlUong4GAkJiYatxkMBiQmJiI8PLxGx9Dr9Th8+DC8vLxMq5SIUFKmx7gf92H5ngzIZcCHj3XFxAfaMYgQkVkzeWhvTEwMRo4ciZCQEISGhiI+Ph5FRUUYPXo0ACA6OhotWrRAXFwcAGD69Om455570LZtW+Tl5eGTTz7B+fPn8fzzz9ftOyGycNqSMoz5bi/+OncVSoUcXzwVhAe7MNQTkfkzOYxERUUhJycHU6dORWZmJoKCgpCQkGDs1Jqeng65/O8bLteuXcOYMWOQmZkJFxcXBAcHY/fu3ejcuXPdvQsiC5dTUIqRi5Jx7LIW9iorzI8ORs82blKXRURUJ2RCCCF1EXei1Wrh5OSE/Px8ODo6Sl0OUYO6cK0Yw7/9C2lXiuFmr8SS0aHo0sJJ6rKIiO6opp/fXJuGqBE7nV2A4d8mI1NbghbONlj2fBhau9lJXRYRUZ1iGCFqpA5fyMfIxcm4WqRDW3d7LHsuDJ5OXNeJiCwPwwhRI/Tn2St4/ru9KCwtR6CPExaPDoUrJzMjIgvFMELUyGw7kY2XlqWgtNyAcL9mWDAyBPYq/lclIsvF73BEjcivhy9j4vL9KNMLRHRyx5ynOasqEVk+hhGiRuK/KRfwxsqDMAhgYKA3Zj8ZCGtFva/YQEQkOYYRokZg6Z/n8e6aIwCAqBANPhzSFQo5Z1UloqaBYYRIYgt2nMUHG48DAEb19MXURzpDziBCRE0IwwiRhOZsPYVZm04CAMb1a4PX/9OB68wQUZPDMEIkASEEZm8+iS+3Vqxe/X/922PCA+0kroqISBoMI0QNTAiBuF9PYP6OswCAtx/uiBfuayNxVURE0mEYIWpAQgi8979jWLI7DQAwbWBnjOrVWtqiiIgkxjBC1EAMBoGp645g2Z/pkMmADwZ3xdNhLaUui4hIcgwjRA3AYBCYsuYIfkquCCIfPR6AJ0M0UpdFRNQoMIwQ1TODQeDt1YexfE8GZDJg1tBAPB7sI3VZRESNBsMIUT3SGwQm/fcQfkm5ALkMmP1kEAZ3ayF1WUREjQrDCFE90RsE3vrvIay8EUQ+iwrCo0EMIkRE/8YwQlQPDDfuiKxMuQCFXIb4qCAMDPSWuiwiokaJq3AR1TGDQWDyqsPGRzMMIkREt8cwQlSHbnZWXbE3oyKIDOvGIEJEdAcMI0R15Obw3eV7Mox9RAYxiBAR3RHDCFEdEEJg2v+O4qfkdOOoGXZWJSKqGYYRorskhMD7G47j+6TzkMmAT4YGcvguEZEJGEaI7oIQAh//loqFf5wDAMQ91pUTmhERmYhhhOguxG85ha+3nwEAzHjUH8NCudYMEZGpGEaIamnuttP4PPEUAODdRzpjRLivtAUREZkphhGiWlj0xzl88lsqAGDSQx3x3L2tJa6IiMh8MYwQmWh5cjqmrz8GAHjlgXZ4qU8biSsiIjJvDCNEJliz/yImrz4MAHjhPj+8GtFO4oqIiMwfwwhRDSUcuYz/++UghABG3NMKkx/qCJlMJnVZRERmj2GEqAZ+P5mDCT/th94gMDTYB+8N8mcQISKqIwwjRHewJ+0qXly6F2V6gQFdvfDR4wGQyxlEiIjqCsMI0W0cuZiPZxfvQUmZAX07NMdnUUFQMIgQEdUphhGiapzOLkD0omQUlJYjtLUrvn4mGEor/pchIqprtfrOOnfuXPj6+kKtViMsLAzJyck12m/58uWQyWQYPHhwbU5L1GAyrhZj+LfJuFqkQ4CPExaODIGNUiF1WUREFsnkMLJixQrExMQgNjYW+/btQ2BgICIjI5GdnX3b/dLS0vD666+jd+/etS6WqCHkFJRixMK/kKktQTt3e3w3OhQOamupyyIislgmh5HZs2djzJgxGD16NDp37ox58+bB1tYWixYtqnYfvV6PZ555Bu+99x78/PzuqmCi+pR/vQzRi5KRdqUYPi42WPpcGFzslFKXRURk0UwKIzqdDikpKYiIiPj7AHI5IiIikJSUVO1+06dPh7u7O5577rkanae0tBRarbbSi6i+Xdfp8fx3e3D8shZu9iosey4Mnk5qqcsiIrJ4JoWR3Nxc6PV6eHh4VNru4eGBzMzMKvf5448/sHDhQixYsKDG54mLi4OTk5PxpdFoTCmTyGRlegPG/bgPe9KuwUFthe+fDYWvm53UZRERNQn1OjSgoKAAI0aMwIIFC+Dm5lbj/SZPnoz8/HzjKyMjox6rpKbOYBB4c+UhbD2RDZWVHAtH9kBnb0epyyIiajKsTGns5uYGhUKBrKysStuzsrLg6el5S/szZ84gLS0NAwcONG4zGAwVJ7ayQmpqKtq0uXWRMZVKBZVKZUppRLUihMAHG49j9f6LsJLL8PXw7ght7Sp1WURETYpJd0aUSiWCg4ORmJho3GYwGJCYmIjw8PBb2nfs2BGHDx/GgQMHjK9BgwahX79+OHDgAB+/kOTm/X4WC/84BwD4eGgA7u/ocYc9iIiorpl0ZwQAYmJiMHLkSISEhCA0NBTx8fEoKirC6NGjAQDR0dFo0aIF4uLioFar0aVLl0r7Ozs7A8At24ka2s97M/BRwgkAwDsDOmFIdx+JKyIiappMDiNRUVHIycnB1KlTkZmZiaCgICQkJBg7taanp0Mu5yyV1LhtOZaFyasOAwBevM8Pz/fmkHMiIqnIhBBC6iLuRKvVwsnJCfn5+XB0ZMdCujsp56/i6QV/obTcgMe7+2DWEwFcgZeIqB7U9PObtzCoSTmdXYBnl+xFabkB93d0x8zHuzKIEBFJjGGEmowsbQlGLtqD/OtlCNI4Y+7T3WGt4H8BIiKp8TsxNQnakjKMXJSMi3nX4edmh0WjenDhOyKiRoJhhCxeabkeL36fghOZBWjuoMJ3z4bClevNEBE1GgwjZNEMBoHXfzmEpLNXYK+ywuJRPaBxtZW6LCIi+geGEbJoMxNO4H8HL8FaIcO84cHo0sJJ6pKIiOhfGEbIYn23Ow3zd5wFUDG76r3tar4+EhERNRyGEbJIvx3NxLT/HQUAvBHZAY914+yqRESNFcMIWZyU89cw8af9EAJ4Oqwlxva9dTFGIiJqPBhGyKKcyy3C89/tQWm5AQ90dMf0Qf6c1IyIqJFjGCGLcbVIh9GLk3GtuAwBPk748ulusOKkZkREjR6/U5NFKCnTY8z3e5F2pRg+LjZYOLIHbJUmrwNJREQSYBghs1cxl8hBpJy/Bke1FZaM7oHmDiqpyyIiohpiGCGz98mmVKw/dLliLpERwWjr7iB1SUREZAKGETJrPyWn4+vtZwAAcUMC0LMN5xIhIjI3DCNktnaeysE7a44AAF55oB2GBnMuESIic8QwQmbpVFYBxi7bB71B4LFuLfBqRDupSyIiolpiGCGzk1tYitFL9qCgtBw9fF0w8/GunEuEiMiMMYyQWbk5hPfCteto1cwW34wIgcpKIXVZRER0FxhGyGzcHMK7Pz0PjmorLBrVA652SqnLIiKiu8QwQmYjPvEU1h+6DCt5xRDeNs3tpS6JiIjqAMMImYW1By7ii8RTAIAPH+vKIbxERBaEYYQavf3p1/DGykMAgBfv88OTPTQSV0RERHWJYYQatYt51zHm+xToyg2I6OSBNx/sKHVJRERUxxhGqNEqKi3H89/tRW5hKTp6OuDzYUFQyDmEl4jI0jCMUKNkMAi8uuIAjl/Wws1eiYWjesBOxVV4iYgsEcMINUqzNqVi87EsKK3kmB8dghbONlKXRERE9YRhhBqdNfsv4qsbi999/HgAurd0kbgiIiKqTwwj1KgcyMjDm/+tGDnzct82GNythcQVERFRfWMYoUYjM78EL3y/98bIGXe88Z8OUpdEREQNgGGEGoWSMj1eWLoX2QWlaO9hj/hh3SDnyBkioiaBYYQkJ4TAmysP4dCFfLjYWuPb6B6w58gZIqImg2GEJPfNjrNYd/ASrOQyfPVMMFo2s5W6JCIiakAMIySpbSey8VHCCQBA7MDOCG/TTOKKiIioodUqjMydOxe+vr5Qq9UICwtDcnJytW1XrVqFkJAQODs7w87ODkFBQVi6dGmtCybLcSanEBOX74cQwFOhGgy/p5XUJRERkQRMDiMrVqxATEwMYmNjsW/fPgQGBiIyMhLZ2dlVtnd1dcWUKVOQlJSEQ4cOYfTo0Rg9ejR+++23uy6ezJe2pAxjvt+LgpJyhLRywXuDukAmY4dVIqKmSCaEEKbsEBYWhh49emDOnDkAAIPBAI1GgwkTJmDSpEk1Okb37t0xYMAAzJgxo0bttVotnJyckJ+fD0dHR1PKpUZIbxAY8/1ebD2RDS8nNdaNvxfNHVRSl0VERHWspp/fJt0Z0el0SElJQURExN8HkMsRERGBpKSkO+4vhEBiYiJSU1Nx3333VduutLQUWq220ossx2ebT2LriWyorOSYPyKEQYSIqIkzKYzk5uZCr9fDw8Oj0nYPDw9kZmZWu19+fj7s7e2hVCoxYMAAfPnll+jfv3+17ePi4uDk5GR8aTQaU8qkRizhyGXM2XYaAPDR4wHo6uMkcUVERCS1BhlN4+DggAMHDmDPnj344IMPEBMTg+3bt1fbfvLkycjPzze+MjIyGqJMqmcnswoQ8/NBAMBz97bmVO9ERAQAMGlmKTc3NygUCmRlZVXanpWVBU9Pz2r3k8vlaNu2LQAgKCgIx48fR1xcHPr27Vtle5VKBZWKt+4tSX5xGV74fi+KdXr0bNMMkx/qKHVJRETUSJh0Z0SpVCI4OBiJiYnGbQaDAYmJiQgPD6/xcQwGA0pLS005NZkxvUHglRX7kXalGC2cbTDn6e6wUnCKGyIiqmDynNsxMTEYOXIkQkJCEBoaivj4eBQVFWH06NEAgOjoaLRo0QJxcXEAKvp/hISEoE2bNigtLcXGjRuxdOlSfP3113X7TqjR+mzzSWxPzYHaWo5vRgTD1U4pdUlERNSImBxGoqKikJOTg6lTpyIzMxNBQUFISEgwdmpNT0+HXP73T71FRUUYO3YsLly4ABsbG3Ts2BHLli1DVFRU3b0LarR+O5pp7LA6c0gAurRgh1UiIqrM5HlGpMB5RszTmZxCPDpnFwpLyzG6ly9iB/pLXRIRETWgeplnhKimCkvL8eLSFBSWliPU1xVvP9xJ6pKIiKiRYhihOieEwBu/HMTp7EJ4OKow55lusGaHVSIiqgY/IajOfbPjLH49kglrhQxfDw+Gu4Na6pKIiKgRYxihOrXrdC4+TjgBAJg2yB/dW7pIXBERETV2DCNUZy7lXcfEn/bDIIChwT54OrSl1CUREZEZYBihOlFarsfYH/bhSpEO/t6OeH9wF8hkMqnLIiIiM8AwQnXi/fXHcSAjD45qK3z9TDDU1gqpSyIiIjPBMEJ3bfX+C1j653kAQPywILRsZitxRUREZE4YRuiuHL+sxeRVhwEAEx9oh/s7ekhcERERmRuGEao1bUkZXl6WgpIyA+5r3xyvPNBO6pKIiMgMMYxQrQgh8NbKQ0i7UgxvJzU+jwqCQs4Oq0REZDqGEaqVRbvSjBObfTU8GC5ciZeIiGqJYYRMtjftKuI2HgcAvPtIZwRpnKUtiIiIzBrDCJkkt7AU43/cj3KDwMBAb4y4p5XUJRERkZljGKEa0xsEXl1+AJnaErRpboe4IV05sRkREd01hhGqsS+3nsIfp3NhY63A18ODYa+ykrokIiKyAAwjVCN/nMrF54mnAAAfDumC9h4OEldERESWgmGE7ihLW4JXV+yHEMCwHho81s1H6pKIiMiCMIzQbZXrDZjw037kFurQ0dMB0wb5S10SERFZGIYRuq3PtpxE8rmrsFMq8NUz3bkAHhER1TmGEarWttRszN12BgAw8/EA+DW3l7giIiKyRAwjVKXL+dcRs+IAAGDEPa0wMNBb2oKIiMhiMYzQLcr1Brzy0wFcKy6Dv7cjpgzoJHVJRERkwRhG6BbxW04hOe0q7FVWmPs0+4kQEVH9YhihSnaeysHc7acBAB8O6QpfNzuJKyIiIkvHMEJG2doSvLbiAIQAngptiUHsJ0JERA2AYYQA3Fh3ZsUB43wisQM7S10SERE1EQwjBACYu+00dp+5AlulAnPYT4SIiBoQwwgh+dxVxG85CQB4f3AXtHXnfCJERNRwGEaauGtFOryyfD8MAhjSvQWGdOe6M0RE1LAYRpowIQTeWHkIl/NL4OdmhxmPdpG6JCIiaoIYRpqw73anYcvxLCgVcnz5dDfYqaykLomIiJoghpEm6sjFfHy48QQAYMqATvD3dpK4IiIiaqoYRpqgotJyTPxpP3R6A/p39kB0eCupSyIioiasVmFk7ty58PX1hVqtRlhYGJKTk6ttu2DBAvTu3RsuLi5wcXFBRETEbdtT/Zu27ijO5hbBy0mNT4YGQCaTSV0SERE1YSaHkRUrViAmJgaxsbHYt28fAgMDERkZiezs7Crbb9++HU899RS2bduGpKQkaDQa/Oc//8HFixfvungy3bqDl/BLygXIZcBnUUFwtlVKXRIRETVxMiGEMGWHsLAw9OjRA3PmzAEAGAwGaDQaTJgwAZMmTbrj/nq9Hi4uLpgzZw6io6NrdE6tVgsnJyfk5+fD0dHRlHLpHzKuFuPhz3eioLQcEx9oh5j+7aUuiYiILFhNP79NujOi0+mQkpKCiIiIvw8glyMiIgJJSUk1OkZxcTHKysrg6upabZvS0lJotdpKL7o7ZXoDJi7fj4LScoS0csHE+9tKXRIREREAE8NIbm4u9Ho9PDw8Km338PBAZmZmjY7x1ltvwdvbu1Kg+be4uDg4OTkZXxqNxpQyqQrxW05if3oeHNRWiB8WBCsF+y4TEVHj0KCfSDNnzsTy5cuxevVqqNXqattNnjwZ+fn5xldGRkYDVml5ks5cwVfbzwAAZg4JgI+LrcQVERER/c2kWa7c3NygUCiQlZVVaXtWVhY8PT1vu++sWbMwc+ZMbNmyBQEBAbdtq1KpoFKpTCmNqpFXrMNrKw5ACCAqRIMBAV5Sl0RERFSJSXdGlEolgoODkZiYaNxmMBiQmJiI8PDwavf7+OOPMWPGDCQkJCAkJKT21ZJJhBCYvOowMrUV073HDuosdUlERES3MHn+75iYGIwcORIhISEIDQ1FfHw8ioqKMHr0aABAdHQ0WrRogbi4OADARx99hKlTp+LHH3+Er6+vsW+Jvb097O25Omx9+nlvBn49kglrhQyfD+sGWyWneyciosbH5E+nqKgo5OTkYOrUqcjMzERQUBASEhKMnVrT09Mhl/99w+Xrr7+GTqfD0KFDKx0nNjYW06ZNu7vqqVpncgoxbd0xAMD//acDuvpwunciImqcTJ5nRAqcZ8Q0unIDHv96Nw5fzEfPNs2w7LkwyOWcZZWIiBpWvcwzQuZh9uaTOHwxH8621pj9ZBCDCBERNWoMIxYm6cwVfLPj5jDervB0qn4INRERUWPAMGJB8ovL8H8//z2M98EuHMZLRESNH8OIhRBC4J21R3ApvwS+zWwxdSCH8RIRkXlgGLEQaw5cxP8OXoJCLsNnUUGwU3EYLxERmQeGEQuQcbUYU9ccBQC88kA7dGvpInFFRERENccwYub0BoH/+/kgCkrLEdzKBWP7tpG6JCIiIpMwjJi5b3acQXLaVdirrBAfxdV4iYjI/PCTy4wduZiPzzafBADEDuwMjStX4yUiIvPDMGKmSsr0eG3FAZTpBR7098TQYB+pSyIiIqoVhhEz9XFCKk5lF6K5gwofDukKmYyzrBIRkXliGDFDf5zKxaJd5wAAHw8NgKudUuKKiIiIao9hxMzkF5fh9V8OAgCG39MS/Tq4S1wRERHR3WEYMTPvrj2CTG0J/Nzs8PbDnaQuh4iI6K4xjJiR/x28hHU3ZlmdHRUEWyVnWSUiIvPHMGImsrQleGfNEQDAuH5tEaRxlrYgIiKiOsIwYgaEEHhz5SHkXy9DlxaOmHB/W6lLIiIiqjMMI2bgx+R0/H4yB0orOT57MgjWnGWViIgsCD/VGrm03CK8v/44AODNyA5o5+EgcUVERER1i2GkEdMbBP7vl4O4XqbHPX6ueLZXa6lLIiIiqnMMI43Ygp1nkXL+GuxVVpj1RCDkcs6ySkRElodhpJFKzSzA7E0Vi+BNfaQzfFy4CB4REVkmhpFGqExvQMzPB6DTG/BAR3c8EcJF8IiIyHIxjDRCc7aextFLWjjbWiOOi+AREZGFYxhpZA5fyMecbacBADMe7QJ3R7XEFREREdUvhpFGpKRMj5ifD0BvEBgQ4IWBgd5Sl0RERFTvGEYakc82n8Sp7EK42asw49EuUpdDRETUIBhGGomU89cwf+dZAEDckK5wtVNKXBEREVHDYBhpBErK9Hjjl4MQAhjSvQX6d/aQuiQiIqIGwzDSCMz6LRVnc4vg4ahC7CP+UpdDRETUoBhGJLYn7SoW7joHoOLxjJOttcQVERERNSyGEQld1/39eOaJYB/c35GPZ4iIqOlhGJHQx7+dQNqVYng6qvHOI52lLoeIiEgSDCMSST53FYt3pQEAZj7eFU42fDxDRERNU63CyNy5c+Hr6wu1Wo2wsDAkJydX2/bo0aN4/PHH4evrC5lMhvj4+NrWajGu6/R4Y+VBAEBUiAZ9O7hLXBEREZF0TA4jK1asQExMDGJjY7Fv3z4EBgYiMjIS2dnZVbYvLi6Gn58fZs6cCU9Pz7su2BJ88lsqzl8phpeTGlMe6SR1OURERJIyOYzMnj0bY8aMwejRo9G5c2fMmzcPtra2WLRoUZXte/TogU8++QTDhg2DSqW664LN3d60q1i8u2L0zIdDusJRzcczRETUtJkURnQ6HVJSUhAREfH3AeRyREREICkpqc6KKi0thVarrfSyBCVleryx8pBx9Ew/Pp4hIiIyLYzk5uZCr9fDw6PyEFQPDw9kZmbWWVFxcXFwcnIyvjQaTZ0dW0qfbkrFuRuTm3H0DBERUYVGOZpm8uTJyM/PN74yMjKkLumupZy/hoV//GNyM46eISIiAgBYmdLYzc0NCoUCWVlZlbZnZWXVaedUlUplUf1LSsr0eHPlQRhurD3Dyc2IiIj+ZtKdEaVSieDgYCQmJhq3GQwGJCYmIjw8vM6LsxRfbj2FMzlFaO6gwlQ+niEiIqrEpDsjABATE4ORI0ciJCQEoaGhiI+PR1FREUaPHg0AiI6ORosWLRAXFwegotPrsWPHjL+/ePEiDhw4AHt7e7Rt27YO30rjdORiPub9fhYAMOPRLnC2VUpcERERUeNichiJiopCTk4Opk6diszMTAQFBSEhIcHYqTU9PR1y+d83XC5duoRu3boZv541axZmzZqFPn36YPv27Xf/DhqxMr0Bb6w8BL1BYECAFx7swnlWiIiI/k0mhBBSF3EnWq0WTk5OyM/Ph6Ojo9Tl1NiXiafw6eaTcLG1xuaYPnCzt5x+MERERHdS08/vRjmaxhKczCrAF1tPAQCmDfJnECEiIqoGw0g90BsE3lx5CGV6gQc6umNQoLfUJRERETVaDCP1YPGucziQkQcHlRU+eKwrZDKZ1CURERE1WgwjdSz9SjFmbUoFALw9oBM8ndQSV0RERNS4MYzUISEEJq8+hJIyA8L9mmFYD8uYxp6IiKg+MYzUoV/2XsCu01egtpYjbggfzxAREdUEw0gdydaWYMaGisndYvq3h6+bncQVERERmQeGkToyde1RFJSUI8DHCc/2ai11OURERGaDYaQO/Hr4MhKOZsJKLsNHjwfASsHLSkREVFP81LxL+cVlmLruKABgbN826ORlPjPEEhERNQYMI3cp7tfjyCkoRZvmdhh3v+Uv/EdERFTXGEbuQtKZK1i+JwMAMPPxAKisFBJXREREZH4YRmqppEyPt1cfBgA8E9YSPXxdJa6IiIjIPDGM1NIXiadwLrcIHo4qvPVQR6nLISIiMlsMI7Vw7JIW3+w4CwCY/mgXOKqtJa6IiIjIfDGMmEhvEJi06hD0BoGHungi0t9T6pKIiIjMGsOIiZbsTsOhC/lwUFvhvUH+UpdDRERk9hhGTHDhWjE+vbEi76SHOsLdkSvyEhER3S2GkRoSQmDq2qMo1unRw9cFT/VoKXVJREREFoFhpIY2Hs7E1hPZsFbIEDekK+RyrshLRERUFxhGaiC/uAyxN6Z8f7lvW7R1d5C4IiIiIsvBMFIDMxNOILewFH7N7TC2bxupyyEiIrIoDCN3sCftKn5KTgcAfPhYV6itOeU7ERFRXWIYuQ1duQGTV1VM+R4VosE9fs0kroiIiMjyMIzcxvwdZ3A6uxBu9kpMfphTvhMREdUHhpFqpOUW4YutpwEA7wzoDGdbpcQVERERWSaGkSoIIfDu2iPQlRtwb1s3PBrkLXVJREREFothpArrDl7CzlO5UFrJ8f7gLpDJOKcIERFRfWEY+Ze8Yh1mrD8GAJh4f1v4utlJXBEREZFlYxj5l48STiC3UIe27vZ44T7OKUJERFTfGEb+YW/aVfyUnAGgYk4RpRUvDxERUX3jp+0NZXoDpqw+AgB4MsQHoa1dJa6IiIioaWAYuWHRH+eQmlUAF1trTH6ok9TlEBERNRkMIwAuXCtG/JZTAIC3H+4EFzvOKUJERNRQahVG5s6dC19fX6jVaoSFhSE5Ofm27X/55Rd07NgRarUaXbt2xcaNG2tVbH2Ztu4YrpfpEdraFUODfaQuh4iIqEkxOYysWLECMTExiI2Nxb59+xAYGIjIyEhkZ2dX2X737t146qmn8Nxzz2H//v0YPHgwBg8ejCNHjtx18XVh09FMbDmeBSu5DB9wThEiIqIGJxNCCFN2CAsLQ48ePTBnzhwAgMFggEajwYQJEzBp0qRb2kdFRaGoqAjr1683brvnnnsQFBSEefPm1eicWq0WTk5OyM/Ph6Ojoynl3lZRaTn6z/4dl/JLMLZvG7z5INefISIiqis1/fw26c6ITqdDSkoKIiIi/j6AXI6IiAgkJSVVuU9SUlKl9gAQGRlZbXsAKC0thVarrfSqD18knsKl/BL4uNhgwv3t6uUcREREdHsmhZHc3Fzo9Xp4eHhU2u7h4YHMzMwq98nMzDSpPQDExcXBycnJ+NJoNKaUWSMlZXr8eqSihhmPdoGNUlHn5yAiIqI7a5SjaSZPnoz8/HzjKyMjo87PobZW4NdXemPWE4Ho19G9zo9PRERENWNlSmM3NzcoFApkZWVV2p6VlQVPT88q9/H09DSpPQCoVCqoVCpTSqsVO5UVR88QERFJzKQ7I0qlEsHBwUhMTDRuMxgMSExMRHh4eJX7hIeHV2oPAJs3b662PRERETUtJt0ZAYCYmBiMHDkSISEhCA0NRXx8PIqKijB69GgAQHR0NFq0aIG4uDgAwCuvvII+ffrg008/xYABA7B8+XLs3bsX8+fPr9t3QkRERGbJ5DASFRWFnJwcTJ06FZmZmQgKCkJCQoKxk2p6ejrk8r9vuPTs2RM//vgj3nnnHbz99tto164d1qxZgy5dutTduyAiIiKzZfI8I1Kor3lGiIiIqP7UyzwjRERERHWNYYSIiIgkxTBCREREkmIYISIiIkkxjBAREZGkGEaIiIhIUgwjREREJCmGESIiIpKUyTOwSuHmvGxarVbiSoiIiKimbn5u32l+VbMIIwUFBQAAjUYjcSVERERkqoKCAjg5OVX752YxHbzBYMClS5fg4OAAmUxWZ8fVarXQaDTIyMjgNPP1iNe54fBaNwxe54bB69ww6vM6CyFQUFAAb2/vSuvW/ZtZ3BmRy+Xw8fGpt+M7OjryH3oD4HVuOLzWDYPXuWHwOjeM+rrOt7sjchM7sBIREZGkGEaIiIhIUk06jKhUKsTGxkKlUkldikXjdW44vNYNg9e5YfA6N4zGcJ3NogMrERERWa4mfWeEiIiIpMcwQkRERJJiGCEiIiJJMYwQERGRpCw+jMydOxe+vr5Qq9UICwtDcnLybdv/8ssv6NixI9RqNbp27YqNGzc2UKXmzZTrvGDBAvTu3RsuLi5wcXFBRETEHf9e6G+m/pu+afny5ZDJZBg8eHD9FmghTL3OeXl5GDduHLy8vKBSqdC+fXt+/6gBU69zfHw8OnToABsbG2g0Grz22msoKSlpoGrN044dOzBw4EB4e3tDJpNhzZo1d9xn+/bt6N69O1QqFdq2bYslS5bUb5HCgi1fvlwolUqxaNEicfToUTFmzBjh7OwssrKyqmy/a9cuoVAoxMcffyyOHTsm3nnnHWFtbS0OHz7cwJWbF1Ov89NPPy3mzp0r9u/fL44fPy5GjRolnJycxIULFxq4cvNj6rW+6dy5c6JFixaid+/e4tFHH22YYs2Yqde5tLRUhISEiIcfflj88ccf4ty5c2L79u3iwIEDDVy5eTH1Ov/www9CpVKJH374QZw7d0789ttvwsvLS7z22msNXLl52bhxo5gyZYpYtWqVACBWr1592/Znz54Vtra2IiYmRhw7dkx8+eWXQqFQiISEhHqr0aLDSGhoqBg3bpzxa71eL7y9vUVcXFyV7Z988kkxYMCAStvCwsLEiy++WK91mjtTr/O/lZeXCwcHB/Hdd9/VV4kWozbXury8XPTs2VN8++23YuTIkQwjNWDqdf7666+Fn5+f0Ol0DVWiRTD1Oo8bN07cf//9lbbFxMSIXr161WudlqQmYeTNN98U/v7+lbZFRUWJyMjIeqvLYh/T6HQ6pKSkICIiwrhNLpcjIiICSUlJVe6TlJRUqT0AREZGVtueaned/624uBhlZWVwdXWtrzItQm2v9fTp0+Hu7o7nnnuuIco0e7W5zuvWrUN4eDjGjRsHDw8PdOnSBR9++CH0en1DlW12anOde/bsiZSUFOOjnLNnz2Ljxo14+OGHG6TmpkKKz0KzWCivNnJzc6HX6+Hh4VFpu4eHB06cOFHlPpmZmVW2z8zMrLc6zV1trvO/vfXWW/D29r7lHz9VVptr/ccff2DhwoU4cOBAA1RoGWpznc+ePYutW7fimWeewcaNG3H69GmMHTsWZWVliI2NbYiyzU5trvPTTz+N3Nxc3HvvvRBCoLy8HC+99BLefvvthii5yajus1Cr1eL69euwsbGp83Na7J0RMg8zZ87E8uXLsXr1aqjVaqnLsSgFBQUYMWIEFixYADc3N6nLsWgGgwHu7u6YP38+goODERUVhSlTpmDevHlSl2ZRtm/fjg8//BBfffUV9u3bh1WrVmHDhg2YMWOG1KXRXbLYOyNubm5QKBTIysqqtD0rKwuenp5V7uPp6WlSe6rddb5p1qxZmDlzJrZs2YKAgID6LNMimHqtz5w5g7S0NAwcONC4zWAwAACsrKyQmpqKNm3a1G/RZqg2/6a9vLxgbW0NhUJh3NapUydkZmZCp9NBqVTWa83mqDbX+d1338WIESPw/PPPAwC6du2KoqIivPDCC5gyZQrkcv58XReq+yx0dHSsl7sigAXfGVEqlQgODkZiYqJxm8FgQGJiIsLDw6vcJzw8vFJ7ANi8eXO17al21xkAPv74Y8yYMQMJCQkICQlpiFLNnqnXumPHjjh8+DAOHDhgfA0aNAj9+vXDgQMHoNFoGrJ8s1Gbf9O9evXC6dOnjWEPAE6ePAkvLy8GkWrU5joXFxffEjhuBkDBZdbqjCSfhfXWNbYRWL58uVCpVGLJkiXi2LFj4oUXXhDOzs4iMzNTCCHEiBEjxKRJk4ztd+3aJaysrMSsWbPE8ePHRWxsLIf21oCp13nmzJlCqVSKlStXisuXLxtfBQUFUr0Fs2Hqtf43jqapGVOvc3p6unBwcBDjx48XqampYv369cLd3V28//77Ur0Fs2DqdY6NjRUODg7ip59+EmfPnhWbNm0Sbdq0EU8++aRUb8EsFBQUiP3794v9+/cLAGL27Nli//794vz580IIISZNmiRGjBhhbH9zaO8bb7whjh8/LubOncuhvXfryy+/FC1bthRKpVKEhoaKP//80/hnffr0ESNHjqzU/ueffxbt27cXSqVS+Pv7iw0bNjRwxebJlOvcqlUrAeCWV2xsbMMXboZM/Tf9TwwjNWfqdd69e7cICwsTKpVK+Pn5iQ8++ECUl5c3cNXmx5TrXFZWJqZNmybatGkj1Gq10Gg0YuzYseLatWsNX7gZ2bZtW5Xfc29e25EjR4o+ffrcsk9QUJBQKpXCz89PLF68uF5rlAnBe1tEREQkHYvtM0JERETmgWGEiIiIJMUwQkRERJJiGCEiIiJJMYwQERGRpBhGiIiISFIMI0RERCQphhEiIiKSFMMIERERSYphhIiIiCTFMEJERESSYhghIiIiSf0/PJ6InJl0wxUAAAAASUVORK5CYII=","text/plain":["
"]},"metadata":{},"output_type":"display_data"}],"source":["import math\n","import matplotlib.pyplot as plt\n","from engine import Value\n","\n","g = 9.81\n","x2, y2 = 1, 0.7\n","\n","v = Value(1)# starting guess\n","alpha = Value(45*math.pi/180)# starting guess\n","\n","def plot_trajectory(v, alpha):\n"," xs = [x2*(i/100) for i in range(100)]\n"," ys = [F(v, alpha, x).data for x in xs]\n"," plt.figure()\n"," plt.title(f\"$v={v.data:.1f}$, $\\\\alpha={180*alpha.data/math.pi:.1f}$\")\n"," plt.plot(xs, ys, label=\"Trajectory\")\n"," plt.scatter(x2, y2, label=\"Target\", s=100, color=\"tab:red\")\n"," plt.gca().set_aspect(\"equal\")\n"," plt.legend()\n"," plt.show()\n","\n","def F(v, alpha, x2):\n"," \"Forward physics model\"\n"," return -(g/2)*((x2 / v * alpha.cos())**2) + (x2*(alpha.sin()/alpha.cos()))\n","\n","lr = 1e-2\n","plot_trajectory(v, alpha)\n","for i in range(1000):\n"," y = F(v, alpha, x2)\n"," loss = (y-y2)**2\n"," loss.backward()\n"," v = Value(v.data - lr*v.grad)\n"," alpha = Value(alpha.data - lr*alpha.grad)\n","plot_trajectory(v, alpha)"]}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.10"},"colab":{"provenance":[]}},"nbformat":4,"nbformat_minor":5} -------------------------------------------------------------------------------- /Autodifferentiation/Autodifferentiation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "0e209abf", 6 | "metadata": {}, 7 | "source": [ 8 | "# Autodifferentiation tutorial\n", 9 | "\n", 10 | "In this tutorial, we will be implementing autodifferentiation from scratch, without using external python libraries.\n", 11 | "\n", 12 | "## Problem overview\n", 13 | "\n", 14 | "In particular, we will be defining a custom `Value` object in python.\n", 15 | "\n", 16 | "This object defines a scalar value (analogous to a python `float`) which can have primitive operations (e.g. add/multiply/pow) applied to it. Each primitive operation returns a new `Value` object.\n", 17 | "\n", 18 | "Importantly, each `Value` object keeps track of the child `Value` objects which created it, and defines a `._backward` method which computes the vector-Jacobian product of the primative operation which created it.\n", 19 | "\n", 20 | "This extra bookmarking allows us to define a `Value.backward()` method (similar to `PyTorch`) which recursively backpropagates gradients through the entire computational graph, accumulating gradients in the leaf `Values` of the graph.\n", 21 | "\n", 22 | "\n", 23 | "## Part A: implement the `Value` class\n", 24 | "\n", 25 | "The `Value` class is defined in `engine.py`.\n", 26 | "\n", 27 | "There are a number of lines of code missing from this class that you must fill in:\n", 28 | "\n", 29 | "> **Task A.1**: Implement the primitive operations which currently raise a `NotImplementedError`\n", 30 | "\n", 31 | "> **Task A.2**: Implement the `Value.backward` method.\n", 32 | "\n", 33 | "> **Task A.3**: Test that the class is correctly implemented by running the tests defined in `engine.py`\n", 34 | "\n", 35 | "More hints are contained in `engine.py`.\n", 36 | "\n", 37 | "## Part B: use the `Value` class to solve a differentiable physics problem\n", 38 | "\n", 39 | "Once we have a working `Value` object, we can use it to solve any problem requiring gradients.\n", 40 | "\n", 41 | "In this part, we will use the `Value` object to learn how to **throw a ball** to hit a target.\n", 42 | "\n", 43 | "### Problem overview\n", 44 | "\n", 45 | "Imagine you are located at $P1 = (0,0)$ and you throw a ball with an initial velocity $v$ at an angle $\\alpha$ from the x-axis. Your goal is to hit a target at $P2 = (x_2, y_2)$. What are values of $\\alpha$ and $v$ you should use?\n", 46 | "\n", 47 | "We can treat this as a inverse problem, where the goal is to minimise the following loss function\n", 48 | "\n", 49 | "$$\n", 50 | "L(v,\\alpha) = (F(v, \\alpha, x_2) - y_2)^2 ~~~~~~~(1)\n", 51 | "$$\n", 52 | "\n", 53 | "Here $F(v, \\alpha, x_2)$ is our forward physics model, which tells us the height of the ball at $x_2$ given the initial velocity and angle of the ball.\n", 54 | "\n", 55 | "In this case, $F$ is given by\n", 56 | "\n", 57 | "$$\n", 58 | "F(v, \\alpha, x_2) = - \\frac{g}{2} \\left( \\frac{x_2}{v \\cos \\alpha} \\right)^2 + x_2 \\tan \\alpha ~~~~~~~(2)\n", 59 | "$$\n", 60 | "\n", 61 | "where $g$ is the acceleration due to gravity.\n", 62 | "\n", 63 | "> **Optional task B.1**: Derive (2) using Netwon's laws.\n", 64 | "\n", 65 | "> **Task B.2**: Write a python function that computes $F(v, \\alpha, x_2)$, where $v$ and $\\alpha$ are `Value` objects.\n", 66 | "\n", 67 | "> **Task B.3**: Use `Value.backward` to write a gradient descent algorithm which minimises the loss function (1), given the starting guess of $v$ and $\\alpha$ below.\n", 68 | "\n", 69 | "> **Task B.4**: Verify that the learned values of $\\alpha$ and $v$ hits the target by plotting the optimised trajectory.\n", 70 | "\n", 71 | "> **Task B.5**: What happens to the trajectory when you change the starting guess of $v$ and $\\alpha$? Why is this the case?" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "id": "5c638652", 78 | "metadata": { 79 | "scrolled": false 80 | }, 81 | "outputs": [], 82 | "source": [ 83 | "import math\n", 84 | "import matplotlib.pyplot as plt\n", 85 | "from engine import Value\n", 86 | "\n", 87 | "g = 9.81\n", 88 | "x2, y2 = 1, 0.7\n", 89 | "\n", 90 | "v = Value(1)# starting guess\n", 91 | "alpha = Value(45*math.pi/180)# starting guess\n", 92 | "\n", 93 | "\n", 94 | "def F(v, alpha, x2):\n", 95 | " \"Forward physics model\"\n", 96 | " \n", 97 | " # TODO: continue from here" 98 | ] 99 | } 100 | ], 101 | "metadata": { 102 | "kernelspec": { 103 | "display_name": "Python 3 (ipykernel)", 104 | "language": "python", 105 | "name": "python3" 106 | }, 107 | "language_info": { 108 | "codemirror_mode": { 109 | "name": "ipython", 110 | "version": 3 111 | }, 112 | "file_extension": ".py", 113 | "mimetype": "text/x-python", 114 | "name": "python", 115 | "nbconvert_exporter": "python", 116 | "pygments_lexer": "ipython3", 117 | "version": "3.10.10" 118 | } 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 5 122 | } 123 | -------------------------------------------------------------------------------- /Autodifferentiation/engine.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | class Value: 4 | """Defines a Value object, which stores a single scalar value and its gradient. 5 | 6 | When a primitive operation is called on this Value object (e.g. add/multiply/pow), 7 | a new Value object is returned which: 8 | 1) keeps track of its child value objects 9 | 2) has a "._backward" method defined which computes the vector-Jacobian 10 | product of the primative operation which created it 11 | 12 | After computing a series of operations with Value objects, one can call the 13 | ".backward" method. This recursively backpropagates (applies the chain rule) 14 | through the entire computational graph, accumulating gradients in the leaf 15 | Values of the graph. 16 | 17 | Your tasks: 18 | 19 | 1) Implement the primitive operations which currently raise a NotImplementedError 20 | HINT: for each primitive operation, make sure to define its vector-Jacobian 21 | product i.e. return a new value object which has a "._backward" defined 22 | Because we are only dealing with scalar primitive operations, the 23 | vector-Jacobian product just reduces to two scalar values multiplied 24 | together (no matrix operations are needed) 25 | 26 | 2) Implement the ".backward" method. 27 | HINT: In order to apply the chain rule properly, we need to call the 28 | ._backward method of each Value in the graph in topological order. 29 | So you must first sort the Values in the graph into this order and 30 | then apply the chain rule to this sorted list. 31 | """ 32 | 33 | def __init__(self, data, _children=(), _op=''): 34 | self.data = data 35 | self.grad = 0 36 | 37 | # internal variables used for autograd graph construction 38 | self._backward = lambda: None 39 | self._prev = set(_children) 40 | self._op = _op # the op that produced this node, for graphviz / debugging / etc 41 | 42 | def __add__(self, other): 43 | other = other if isinstance(other, Value) else Value(other) 44 | out = Value(self.data + other.data, (self, other), '+') 45 | 46 | def _backward(): 47 | self.grad += out.grad 48 | other.grad += out.grad 49 | out._backward = _backward 50 | 51 | return out 52 | 53 | def __mul__(self, other): 54 | raise NotImplementedError 55 | 56 | def __pow__(self, other): 57 | assert isinstance(other, (int, float)), "only supporting int/float powers for now" 58 | out = Value(self.data**other, (self,), f'**{other}') 59 | 60 | def _backward(): 61 | self.grad += (other * self.data**(other-1)) * out.grad 62 | out._backward = _backward 63 | 64 | return out 65 | 66 | def cos(self): 67 | raise NotImplementedError 68 | 69 | def sin(self): 70 | raise NotImplementedError 71 | 72 | def backward(self): 73 | 74 | # topological order all of the children in the graph 75 | topo = [] 76 | visited = set() 77 | def build_topo(v): 78 | # TODO: fill in this function. 79 | # this needs to fill topo with the nodes in a topological ordering of the graph 80 | # so we can visit them and call their _backward() in the correct order 81 | 82 | raise NotImplementedError 83 | build_topo(self) 84 | 85 | # go one variable at a time and apply the chain rule to get its gradient 86 | self.grad = 1 87 | for v in reversed(topo): 88 | v._backward() 89 | 90 | def __neg__(self): # -self 91 | return self * -1 92 | 93 | def __radd__(self, other): # other + self 94 | return self + other 95 | 96 | def __sub__(self, other): # self - other 97 | return self + (-other) 98 | 99 | def __rsub__(self, other): # other - self 100 | return (-self) + other 101 | 102 | def __rmul__(self, other): # other * self 103 | return self * other 104 | 105 | def __truediv__(self, other): # self / other 106 | return self * other**-1 107 | 108 | def __rtruediv__(self, other): # other / self 109 | return (self**-1) * other 110 | 111 | def __repr__(self): 112 | return f"Value(data={self.data}, grad={self.grad})" 113 | 114 | 115 | 116 | if __name__ == "__main__": 117 | 118 | # Run some tests on the value class 119 | 120 | x = Value(1) 121 | y = x + 2 - 3 122 | y.backward() 123 | assert x.grad == 1 124 | 125 | x = Value(1) 126 | y = 4*x/2 127 | y.backward() 128 | assert x.grad == 2 129 | 130 | x = Value(2) 131 | y = x**2 132 | y.backward() 133 | assert x.grad == 2*2 134 | 135 | x = Value(0) 136 | y = x.cos() 137 | y.backward() 138 | assert x.grad == 0 139 | 140 | x = Value(0) 141 | y = x.sin() 142 | y.backward() 143 | assert x.grad == 1 144 | 145 | x = Value(0) 146 | y = x*x.cos() + x**2 + 3 147 | y.backward() 148 | assert x.grad == 1 149 | 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /Autodifferentiation/engine_solution.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | class Value: 4 | """Defines a Value object, which stores a single scalar value and its gradient. 5 | 6 | When a primitive operation is called on this Value object (e.g. add/multiply/pow), 7 | a new Value object is returned which: 8 | 1) keeps track of its child value objects 9 | 2) has a "._backward" method defined which computes the vector-Jacobian 10 | product of the primative operation which created it 11 | 12 | After computing a series of operations with Value objects, one can call the 13 | ".backward" method. This recursively backpropagates (applies the chain rule) 14 | through the entire computational graph, accumulating gradients in the leaf 15 | Values of the graph. 16 | 17 | Your tasks: 18 | 19 | 1) Implement the primitive operations which currently raise a NotImplementedError 20 | HINT: for each primitive operation, make sure to define its vector-Jacobian 21 | product i.e. return a new value object which has a "._backward" defined 22 | Because we are only dealing with scalar primitive operations, the 23 | vector-Jacobian product just reduces to two scalar values multiplied 24 | together (no matrix operations are needed) 25 | 26 | 2) Implement the ".backward" method. 27 | HINT: In order to apply the chain rule properly, we need to call the 28 | ._backward method of each Value in the graph in topological order. 29 | So you must first sort the Values in the graph into this order and 30 | then apply the chain rule to this sorted list. 31 | """ 32 | 33 | def __init__(self, data, _children=(), _op=''): 34 | self.data = data 35 | self.grad = 0 36 | 37 | # internal variables used for autograd graph construction 38 | self._backward = lambda: None 39 | self._prev = set(_children) 40 | self._op = _op # the op that produced this node, for graphviz / debugging / etc 41 | 42 | def __add__(self, other): 43 | other = other if isinstance(other, Value) else Value(other) 44 | out = Value(self.data + other.data, (self, other), '+') 45 | 46 | def _backward(): 47 | self.grad += out.grad 48 | other.grad += out.grad 49 | out._backward = _backward 50 | 51 | return out 52 | 53 | def __mul__(self, other): 54 | other = other if isinstance(other, Value) else Value(other) 55 | out = Value(self.data * other.data, (self, other), '*') 56 | 57 | def _backward(): 58 | self.grad += other.data * out.grad 59 | other.grad += self.data * out.grad 60 | out._backward = _backward 61 | 62 | return out 63 | 64 | def __pow__(self, other): 65 | assert isinstance(other, (int, float)), "only supporting int/float powers for now" 66 | out = Value(self.data**other, (self,), f'**{other}') 67 | 68 | def _backward(): 69 | self.grad += (other * self.data**(other-1)) * out.grad 70 | out._backward = _backward 71 | 72 | return out 73 | 74 | def cos(self): 75 | out = Value(math.cos(self.data), (self,), "cos") 76 | 77 | def _backward(): 78 | self.grad += -math.sin(self.data) * out.grad 79 | out._backward = _backward 80 | 81 | return out 82 | 83 | def sin(self): 84 | out = Value(math.sin(self.data), (self,), "sin") 85 | 86 | def _backward(): 87 | self.grad += math.cos(self.data) * out.grad 88 | out._backward = _backward 89 | 90 | return out 91 | 92 | def backward(self): 93 | 94 | # topological order all of the children in the graph 95 | topo = [] 96 | visited = set() 97 | def build_topo(v): 98 | if v not in visited: 99 | visited.add(v) 100 | for child in v._prev: 101 | build_topo(child) 102 | topo.append(v) 103 | build_topo(self) 104 | 105 | # go one variable at a time and apply the chain rule to get its gradient 106 | self.grad = 1 107 | for v in reversed(topo): 108 | v._backward() 109 | 110 | def __neg__(self): # -self 111 | return self * -1 112 | 113 | def __radd__(self, other): # other + self 114 | return self + other 115 | 116 | def __sub__(self, other): # self - other 117 | return self + (-other) 118 | 119 | def __rsub__(self, other): # other - self 120 | return (-self) + other 121 | 122 | def __rmul__(self, other): # other * self 123 | return self * other 124 | 125 | def __truediv__(self, other): # self / other 126 | return self * other**-1 127 | 128 | def __rtruediv__(self, other): # other / self 129 | return (self**-1) * other 130 | 131 | def __repr__(self): 132 | return f"Value(data={self.data}, grad={self.grad})" 133 | 134 | 135 | 136 | if __name__ == "__main__": 137 | 138 | # Run some tests on the value class 139 | 140 | x = Value(1) 141 | y = x + 2 - 3 142 | y.backward() 143 | assert x.grad == 1 144 | 145 | x = Value(1) 146 | y = 4*x/2 147 | y.backward() 148 | assert x.grad == 2 149 | 150 | x = Value(2) 151 | y = x**2 152 | y.backward() 153 | assert x.grad == 2*2 154 | 155 | x = Value(0) 156 | y = x.cos() 157 | y.backward() 158 | assert x.grad == 0 159 | 160 | x = Value(0) 161 | y = x.sin() 162 | y.backward() 163 | assert x.grad == 1 164 | 165 | x = Value(0) 166 | y = x*x.cos() + x**2 + 3 167 | y.backward() 168 | assert x.grad == 1 169 | 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /Common.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | import os 4 | 5 | os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' 6 | torch.manual_seed(42) 7 | 8 | 9 | class NeuralNet(nn.Module): 10 | 11 | def __init__(self, input_dimension, output_dimension, n_hidden_layers, neurons, regularization_param, regularization_exp, retrain_seed): 12 | super(NeuralNet, self).__init__() 13 | # Number of input dimensions n 14 | self.input_dimension = input_dimension 15 | # Number of output dimensions m 16 | self.output_dimension = output_dimension 17 | # Number of neurons per layer 18 | self.neurons = neurons 19 | # Number of hidden layers 20 | self.n_hidden_layers = n_hidden_layers 21 | # Activation function 22 | self.activation = nn.Tanh() 23 | self.regularization_param = regularization_param 24 | # Regularization exponent 25 | self.regularization_exp = regularization_exp 26 | # Random seed for weight initialization 27 | 28 | self.input_layer = nn.Linear(self.input_dimension, self.neurons) 29 | self.hidden_layers = nn.ModuleList([nn.Linear(self.neurons, self.neurons) for _ in range(n_hidden_layers - 1)]) 30 | self.output_layer = nn.Linear(self.neurons, self.output_dimension) 31 | self.retrain_seed = retrain_seed 32 | # Random Seed for weight initialization 33 | self.init_xavier() 34 | 35 | def forward(self, x): 36 | # The forward function performs the set of affine and non-linear transformations defining the network 37 | # (see equation above) 38 | x = self.activation(self.input_layer(x)) 39 | for k, l in enumerate(self.hidden_layers): 40 | x = self.activation(l(x)) 41 | return self.output_layer(x) 42 | 43 | def init_xavier(self): 44 | torch.manual_seed(self.retrain_seed) 45 | 46 | def init_weights(m): 47 | if type(m) == nn.Linear and m.weight.requires_grad and m.bias.requires_grad: 48 | g = nn.init.calculate_gain('tanh') 49 | torch.nn.init.xavier_uniform_(m.weight, gain=g) 50 | # torch.nn.init.xavier_normal_(m.weight, gain=g) 51 | m.bias.data.fill_(0) 52 | 53 | self.apply(init_weights) 54 | 55 | def regularization(self): 56 | reg_loss = 0 57 | for name, param in self.named_parameters(): 58 | if 'weight' in name: 59 | reg_loss = reg_loss + torch.norm(param, self.regularization_exp) 60 | return self.regularization_param * reg_loss 61 | 62 | 63 | def fit(model, training_set, num_epochs, optimizer, p, verbose=True): 64 | history = list() 65 | 66 | # Loop over epochs 67 | for epoch in range(num_epochs): 68 | if verbose: print("################################ ", epoch, " ################################") 69 | 70 | running_loss = list([0]) 71 | 72 | # Loop over batches 73 | for j, (x_train_, u_train_) in enumerate(training_set): 74 | def closure(): 75 | # zero the parameter gradients 76 | optimizer.zero_grad() 77 | # forward + backward + optimize 78 | u_pred_ = model(x_train_) 79 | # Item 1. below 80 | loss = torch.mean((u_pred_.reshape(-1, ) - u_train_.reshape(-1, )) ** p) + model.regularization() 81 | # Item 2. below 82 | loss.backward() 83 | # Compute average training loss over batches for the current epoch 84 | running_loss[0] += loss.item() 85 | return loss 86 | 87 | # Item 3. below 88 | optimizer.step(closure=closure) 89 | 90 | if verbose: print('Loss: ', (running_loss[0] / len(training_set))) 91 | history.append(running_loss[0]) 92 | 93 | return history 94 | 95 | 96 | class Legendre(nn.Module): 97 | """ Univariate Legendre Polynomial """ 98 | 99 | def __init__(self, PolyDegree): 100 | super(Legendre, self).__init__() 101 | self.degree = PolyDegree 102 | 103 | def legendre(self,x, degree): 104 | x = x.reshape(-1, 1) 105 | list_poly = list() 106 | zeroth_pol = torch.ones(x.size(0),1) 107 | list_poly.append(zeroth_pol) 108 | # retvar[:, 0] = x * 0 + 1 109 | if degree > 0: 110 | first_pol = x 111 | list_poly.append(first_pol) 112 | ith_pol = torch.clone(first_pol) 113 | ith_m_pol = torch.clone(zeroth_pol) 114 | 115 | for ii in range(1, degree): 116 | ith_p_pol = ((2 * ii + 1) * x * ith_pol - ii * ith_m_pol) / (ii + 1) 117 | list_poly.append(ith_p_pol) 118 | ith_m_pol = torch.clone(ith_pol) 119 | ith_pol = torch.clone(ith_p_pol) 120 | list_poly = torch.cat(list_poly,1) 121 | return list_poly 122 | 123 | def forward(self, x): 124 | eval_poly = self.legendre(x, self.degree) 125 | return eval_poly 126 | 127 | 128 | 129 | 130 | class MultiVariatePoly(nn.Module): 131 | 132 | def __init__(self, dim, order): 133 | super(MultiVariatePoly, self).__init__() 134 | self.order = order 135 | self.dim = dim 136 | self.polys = Legendre(order) 137 | self.num = (order + 1) ** dim 138 | self.linear = torch.nn.Linear(self.num, 1) 139 | 140 | def forward(self, x): 141 | poly_eval = list() 142 | leg_eval = torch.cat([self.polys(x[:, i]).reshape(1, x.shape[0], self.order + 1) for i in range(self.dim) ]) 143 | for i in range(x.shape[0]): 144 | poly_eval.append(torch.torch.cartesian_prod(*leg_eval[:, i, :]).prod(dim=1).view(1, -1)) 145 | poly_eval = torch.cat(poly_eval) 146 | return self.linear(poly_eval) 147 | -------------------------------------------------------------------------------- /FNO.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/" 9 | }, 10 | "executionInfo": { 11 | "elapsed": 26305, 12 | "status": "ok", 13 | "timestamp": 1686134086861, 14 | "user": { 15 | "displayName": "Emmanuel de Bezenac", 16 | "userId": "17809083500127241366" 17 | }, 18 | "user_tz": -120 19 | }, 20 | "id": "GP7ZoFmq5TaF", 21 | "outputId": "a2e262b7-36d7-4fae-cd28-637df772a35a" 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import torch\n", 26 | "import torch.nn as nn\n", 27 | "import os\n", 28 | "import numpy as np\n", 29 | "from torch.utils.data import DataLoader, TensorDataset\n", 30 | "import torch.nn.functional as F\n", 31 | "from torch.optim import Adam\n", 32 | "import matplotlib.pyplot as plt\n" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": { 39 | "executionInfo": { 40 | "elapsed": 443, 41 | "status": "ok", 42 | "timestamp": 1686134352560, 43 | "user": { 44 | "displayName": "Emmanuel de Bezenac", 45 | "userId": "17809083500127241366" 46 | }, 47 | "user_tz": -120 48 | }, 49 | "id": "hhr6Ixsh07fD" 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "def activation(name):\n", 54 | " if name in ['tanh', 'Tanh']:\n", 55 | " return nn.Tanh()\n", 56 | " elif name in ['relu', 'ReLU']:\n", 57 | " return nn.ReLU(inplace=True)\n", 58 | " elif name in ['lrelu', 'LReLU']:\n", 59 | " return nn.LeakyReLU(inplace=True)\n", 60 | " elif name in ['sigmoid', 'Sigmoid']:\n", 61 | " return nn.Sigmoid()\n", 62 | " elif name in ['softplus', 'Softplus']:\n", 63 | " return nn.Softplus(beta=4)\n", 64 | " elif name in ['celu', 'CeLU']:\n", 65 | " return nn.CELU()\n", 66 | " elif name in ['elu']:\n", 67 | " return nn.ELU()\n", 68 | " elif name in ['mish']:\n", 69 | " return nn.Mish()\n", 70 | " else:\n", 71 | " raise ValueError('Unknown activation function')\n", 72 | "\n", 73 | "\n", 74 | "\n", 75 | "\n", 76 | "################################################################\n", 77 | "# 1d fourier layer\n", 78 | "################################################################\n", 79 | "class SpectralConv1d(nn.Module):\n", 80 | " def __init__(self, in_channels, out_channels, modes1):\n", 81 | " super(SpectralConv1d, self).__init__()\n", 82 | "\n", 83 | " \"\"\"\n", 84 | " 1D Fourier layer. It does FFT, linear transform, and Inverse FFT. \n", 85 | " \"\"\"\n", 86 | "\n", 87 | " self.in_channels = in_channels\n", 88 | " self.out_channels = out_channels\n", 89 | " self.modes1 = modes1\n", 90 | "\n", 91 | " self.scale = (1 / (in_channels * out_channels))\n", 92 | " self.weights1 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, dtype=torch.cfloat))\n", 93 | "\n", 94 | " # Complex multiplication\n", 95 | " def compl_mul1d(self, input, weights):\n", 96 | " # (batch, in_channel, x ), (in_channel, out_channel, x) -> (batch, out_channel, x)\n", 97 | " return torch.einsum(\"bix,iox->box\", input, weights)\n", 98 | "\n", 99 | "\n", 100 | "######################### TO DO ####################################\n", 101 | "\n", 102 | "\n", 103 | " def forward(self, x):\n", 104 | " batchsize = x.shape[0]\n", 105 | " # x.shape == [batch_size, in_channels, number of grid points]\n", 106 | " # hint: use torch.fft library torch.fft.rfft\n", 107 | " # use DFT to approximate the fourier transform\n", 108 | " \n", 109 | " # Compute Fourier coefficients\n", 110 | " x_ft = torch.fft.rfft(x)\n", 111 | "\n", 112 | " # Multiply relevant Fourier modes\n", 113 | " out_ft = torch.zeros(batchsize, self.out_channels, x.size(-1) // 2 + 1, device=x.device, dtype=torch.cfloat)\n", 114 | " out_ft[:, :, :self.modes1] = self.compl_mul1d(x_ft[:, :, :self.modes1], self.weights1)\n", 115 | "\n", 116 | " # Return to physical space\n", 117 | " x = torch.fft.irfft(out_ft, n=x.size(-1))\n", 118 | " return x\n", 119 | "\n", 120 | "\n", 121 | "####################################################################\n", 122 | "\n", 123 | "\n", 124 | "class FNO1d(nn.Module):\n", 125 | " def __init__(self, modes, width):\n", 126 | " super(FNO1d, self).__init__()\n", 127 | "\n", 128 | " \"\"\"\n", 129 | " The overall network. It contains 4 layers of the Fourier layer.\n", 130 | " 1. Lift the input to the desire channel dimension by self.fc0 .\n", 131 | " 2. 4 layers of the integral operators u' = (W + K)(u).\n", 132 | " W defined by self.w; K defined by self.conv .\n", 133 | " 3. Project from the channel space to the output space by self.fc1 and self.fc2 .\n", 134 | "\n", 135 | " input: the solution of the initial condition and location (a(x), x)\n", 136 | " input shape: (batchsize, x=s, c=2)\n", 137 | " output: the solution of a later timestep\n", 138 | " output shape: (batchsize, x=s, c=1)\n", 139 | " \"\"\"\n", 140 | "\n", 141 | " self.modes1 = modes\n", 142 | " self.width = width\n", 143 | " self.padding = 1 # pad the domain if input is non-periodic\n", 144 | " self.linear_p = nn.Linear(2, self.width) # input channel is 2: (u0(x), x)\n", 145 | "\n", 146 | " self.spect1 = SpectralConv1d(self.width, self.width, self.modes1)\n", 147 | " self.spect2 = SpectralConv1d(self.width, self.width, self.modes1)\n", 148 | " self.spect3 = SpectralConv1d(self.width, self.width, self.modes1)\n", 149 | " self.lin0 = nn.Conv1d(self.width, self.width, 1)\n", 150 | " self.lin1 = nn.Conv1d(self.width, self.width, 1)\n", 151 | " self.lin2 = nn.Conv1d(self.width, self.width, 1)\n", 152 | "\n", 153 | " self.linear_q = nn.Linear(self.width, 32)\n", 154 | " self.output_layer = nn.Linear(32, 1)\n", 155 | "\n", 156 | " self.activation = torch.nn.Tanh()\n", 157 | "\n", 158 | " def fourier_layer(self, x, spectral_layer, conv_layer):\n", 159 | " return self.activation(spectral_layer(x) + conv_layer(x))\n", 160 | "\n", 161 | " def linear_layer(self, x, linear_transformation):\n", 162 | " return self.activation(linear_transformation(x))\n", 163 | "\n", 164 | " def forward(self, x):\n", 165 | " # grid = self.get_grid(x.shape, x.device)\n", 166 | " # x = torch.cat((x, grid), dim=-1)\n", 167 | " x = self.linear_p(x)\n", 168 | " x = x.permute(0, 2, 1)\n", 169 | " # x = F.pad(x, [0, self.padding]) # pad the domain if input is non-periodic\n", 170 | "\n", 171 | " x = self.fourier_layer(x, self.spect1, self.lin0)\n", 172 | " x = self.fourier_layer(x, self.spect2, self.lin1)\n", 173 | " x = self.fourier_layer(x, self.spect3, self.lin2)\n", 174 | "\n", 175 | " # x = x[..., :-self.padding] # pad the domain if input is non-periodic\n", 176 | " x = x.permute(0, 2, 1)\n", 177 | "\n", 178 | " x = self.linear_layer(x, self.linear_q)\n", 179 | " x = self.output_layer(x)\n", 180 | " return x\n", 181 | "\n", 182 | "\n", 183 | "################################################################\n", 184 | "# 2d fourier layer\n", 185 | "################################################################\n", 186 | "class SpectralConv2d(nn.Module):\n", 187 | " def __init__(self, in_channels, out_channels, modes1, modes2):\n", 188 | " super(SpectralConv2d, self).__init__()\n", 189 | "\n", 190 | " \"\"\"\n", 191 | " 2D Fourier layer. It does FFT, linear transform, and Inverse FFT. \n", 192 | " \"\"\"\n", 193 | "\n", 194 | " self.in_channels = in_channels\n", 195 | " self.out_channels = out_channels\n", 196 | " self.modes1 = modes1 # Number of Fourier modes to multiply, at most floor(N/2) + 1\n", 197 | " self.modes2 = modes2\n", 198 | "\n", 199 | " self.scale = (1 / (in_channels * out_channels))\n", 200 | " self.weights1 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, dtype=torch.cfloat))\n", 201 | " self.weights2 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, dtype=torch.cfloat))\n", 202 | "\n", 203 | " # Complex multiplication\n", 204 | " def compl_mul2d(self, input, weights):\n", 205 | " # (batch, in_channel, x,y ), (in_channel, out_channel, x,y) -> (batch, out_channel, x,y)\n", 206 | " return torch.einsum(\"bixy,ioxy->boxy\", input, weights)\n", 207 | "\n", 208 | " def forward(self, x):\n", 209 | " batchsize = x.shape[0]\n", 210 | " # Compute Fourier coeffcients up to factor of e^(- something constant)\n", 211 | " x_ft = torch.fft.rfft2(x)\n", 212 | "\n", 213 | " # Multiply relevant Fourier modes\n", 214 | " out_ft = torch.zeros(batchsize, self.out_channels, x.size(-2), x.size(-1) // 2 + 1, dtype=torch.cfloat, device=x.device)\n", 215 | " out_ft[:, :, :self.modes1, :self.modes2] = \\\n", 216 | " self.compl_mul2d(x_ft[:, :, :self.modes1, :self.modes2], self.weights1)\n", 217 | " out_ft[:, :, -self.modes1:, :self.modes2] = \\\n", 218 | " self.compl_mul2d(x_ft[:, :, -self.modes1:, :self.modes2], self.weights2)\n", 219 | "\n", 220 | " # Return to physical space\n", 221 | " x = torch.fft.irfft2(out_ft, s=(x.size(-2), x.size(-1)))\n", 222 | " return x\n", 223 | "\n", 224 | "\n", 225 | "class FNO2d(nn.Module):\n", 226 | " def __init__(self, fno_architecture, device=None, padding_frac=1 / 4):\n", 227 | " super(FNO2d, self).__init__()\n", 228 | "\n", 229 | " \"\"\"\n", 230 | " The overall network. It contains 4 layers of the Fourier layer.\n", 231 | " 1. Lift the input to the desire channel dimension by self.fc0 .\n", 232 | " 2. 4 layers of the integral operators u' = (W + K)(u).\n", 233 | " W defined by self.w; K defined by self.conv .\n", 234 | " 3. Project from the channel space to the output space by self.fc1 and self.fc2 .\n", 235 | "\n", 236 | " input: the solution of the coefficient function and locations (a(x, y), x, y)\n", 237 | " input shape: (batchsize, x=s, y=s, c=3)\n", 238 | " output: the solution \n", 239 | " output shape: (batchsize, x=s, y=s, c=1)\n", 240 | " \"\"\"\n", 241 | " self.modes1 = fno_architecture[\"modes\"]\n", 242 | " self.modes2 = fno_architecture[\"modes\"]\n", 243 | " self.width = fno_architecture[\"width\"]\n", 244 | " self.n_layers = fno_architecture[\"n_layers\"]\n", 245 | " self.retrain_fno = fno_architecture[\"retrain_fno\"]\n", 246 | "\n", 247 | " torch.manual_seed(self.retrain_fno)\n", 248 | " # self.padding = 9 # pad the domain if input is non-periodic\n", 249 | " self.padding_frac = padding_frac\n", 250 | " self.fc0 = nn.Linear(3, self.width) # input channel is 3: (a(x, y), x, y)\n", 251 | " \n", 252 | " self.conv_list = nn.ModuleList(\n", 253 | " [nn.Conv2d(self.width, self.width, 1) for _ in range(self.n_layers)])\n", 254 | " self.spectral_list = nn.ModuleList(\n", 255 | " [SpectralConv2d(self.width, self.width, self.modes1, self.modes2) for _ in range(self.n_layers)])\n", 256 | "\n", 257 | " self.fc1 = nn.Linear(self.width, 128)\n", 258 | " self.fc2 = nn.Linear(128, 1)\n", 259 | "\n", 260 | " self.to(device)\n", 261 | "\n", 262 | " def forward(self, x):\n", 263 | " x = self.fc0(x)\n", 264 | " x = x.permute(0, 3, 1, 2)\n", 265 | "\n", 266 | " x1_padding = int(round(x.shape[-1] * self.padding_frac))\n", 267 | " x2_padding = int(round(x.shape[-2] * self.padding_frac))\n", 268 | " x = F.pad(x, [0, x1_padding, 0, x2_padding])\n", 269 | "\n", 270 | " for k, (s, c) in enumerate(zip(self.spectral_list, self.conv_list)):\n", 271 | "\n", 272 | " x1 = s(x)\n", 273 | " x2 = c(x)\n", 274 | " x = x1 + x2\n", 275 | " if k != self.n_layers - 1:\n", 276 | " x = F.gelu(x)\n", 277 | " x = x[..., :-x1_padding, :-x2_padding]\n", 278 | "\n", 279 | " x = x.permute(0, 2, 3, 1)\n", 280 | " x = self.fc1(x)\n", 281 | " x = F.gelu(x)\n", 282 | " x = self.fc2(x)\n", 283 | " return x" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 3, 289 | "metadata": { 290 | "executionInfo": { 291 | "elapsed": 1860, 292 | "status": "ok", 293 | "timestamp": 1686134357034, 294 | "user": { 295 | "displayName": "Emmanuel de Bezenac", 296 | "userId": "17809083500127241366" 297 | }, 298 | "user_tz": -120 299 | }, 300 | "id": "6fpdAydi5Xjw" 301 | }, 302 | "outputs": [], 303 | "source": [ 304 | "torch.manual_seed(0)\n", 305 | "np.random.seed(0)\n", 306 | "\n", 307 | "n_train =100\n", 308 | "\n", 309 | "x_data = torch.from_numpy(np.load(\"AC_data_input.npy\")).type(torch.float32)\n", 310 | "y_data = torch.from_numpy(np.load(\"AC_data_output.npy\")).type(torch.float32)\n", 311 | "\n", 312 | "\n", 313 | "temporary_tensor = torch.clone(x_data[:, :, 0])\n", 314 | "x_data[:, :, 0] = x_data[:, :, 1]\n", 315 | "x_data[:, :, 1] = temporary_tensor\n", 316 | "\n", 317 | "\n", 318 | "input_function_train = x_data[:n_train, :]\n", 319 | "output_function_train = y_data[:n_train, :]\n", 320 | "input_function_test = x_data[n_train:, :]\n", 321 | "output_function_test = y_data[n_train:, :]\n", 322 | "\n", 323 | "batch_size = 10\n", 324 | "\n", 325 | "training_set = DataLoader(TensorDataset(input_function_train, output_function_train), batch_size=batch_size, shuffle=True)\n", 326 | "testing_set = DataLoader(TensorDataset(input_function_test, output_function_test), batch_size=batch_size, shuffle=False)" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 4, 332 | "metadata": { 333 | "executionInfo": { 334 | "elapsed": 5, 335 | "status": "ok", 336 | "timestamp": 1686134357035, 337 | "user": { 338 | "displayName": "Emmanuel de Bezenac", 339 | "userId": "17809083500127241366" 340 | }, 341 | "user_tz": -120 342 | }, 343 | "id": "gz6EpTqU5-bz" 344 | }, 345 | "outputs": [], 346 | "source": [ 347 | "learning_rate = 0.001\n", 348 | "\n", 349 | "# epochs = 250\n", 350 | "epochs = 2\n", 351 | "step_size = 50\n", 352 | "gamma = 0.5\n", 353 | "\n", 354 | "modes = 16\n", 355 | "width = 64\n", 356 | "\n", 357 | "# model\n", 358 | "\n", 359 | "fno = FNO1d(modes, width)" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 5, 365 | "metadata": { 366 | "executionInfo": { 367 | "elapsed": 4, 368 | "status": "ok", 369 | "timestamp": 1686134392147, 370 | "user": { 371 | "displayName": "Emmanuel de Bezenac", 372 | "userId": "17809083500127241366" 373 | }, 374 | "user_tz": -120 375 | }, 376 | "id": "yD_urMIxcKSu" 377 | }, 378 | "outputs": [], 379 | "source": [ 380 | "x, y = TensorDataset(input_function_train, output_function_train)[0]" 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 6, 386 | "metadata": { 387 | "colab": { 388 | "base_uri": "https://localhost:8080/" 389 | }, 390 | "executionInfo": { 391 | "elapsed": 238, 392 | "status": "ok", 393 | "timestamp": 1686134399949, 394 | "user": { 395 | "displayName": "Emmanuel de Bezenac", 396 | "userId": "17809083500127241366" 397 | }, 398 | "user_tz": -120 399 | }, 400 | "id": "NZJeg5vwcQwh", 401 | "outputId": "64d26d28-8982-464b-e379-b7fede95ae9e" 402 | }, 403 | "outputs": [ 404 | { 405 | "data": { 406 | "text/plain": [ 407 | "(torch.Size([1001, 2]), torch.Size([1001]))" 408 | ] 409 | }, 410 | "execution_count": 6, 411 | "metadata": {}, 412 | "output_type": "execute_result" 413 | } 414 | ], 415 | "source": [ 416 | "x.shape, y.shape" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 7, 422 | "metadata": { 423 | "colab": { 424 | "base_uri": "https://localhost:8080/", 425 | "height": 430 426 | }, 427 | "executionInfo": { 428 | "elapsed": 484, 429 | "status": "ok", 430 | "timestamp": 1686134452434, 431 | "user": { 432 | "displayName": "Emmanuel de Bezenac", 433 | "userId": "17809083500127241366" 434 | }, 435 | "user_tz": -120 436 | }, 437 | "id": "Bxa8xZF_cSs1", 438 | "outputId": "15efdc13-e590-4d11-da0b-988f02fa91b9" 439 | }, 440 | "outputs": [ 441 | { 442 | "data": { 443 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGdCAYAAAAvwBgXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABnlUlEQVR4nO3dZ3Rc5bk+/GuKZqRRmVHvXXKV3ORewDYuGEILEEIIMYSQhMBJCDkkkPyTnCQvB06Sk0Y4QEgoCQETQu82LrgXuUqWJav3Lk1Rmb7fD3uKhbs9M3vK9VvLa2Hv8cytwZIuPft+7kcmCIIAIiIiIgnIpS6AiIiIIheDCBEREUmGQYSIiIgkwyBCREREkmEQISIiIskwiBAREZFkGESIiIhIMgwiREREJBml1AWci9PpRFdXF+Lj4yGTyaQuh4iIiC6AIAgwmUzIysqCXH7uNY+gDiJdXV3Izc2VugwiIiK6BO3t7cjJyTnnY4I6iMTHxwMQP5CEhASJqyEiIqILYTQakZub6/k+fi5BHUTct2MSEhIYRIiIiELMhbRVsFmViIiIJMMgQkRERJJhECEiIiLJBHWPCBERUbByOByw2WxSlyGZqKgoKBSKy34eBhEiIqKLNDIygo6ODgiCIHUpkpHJZMjJyUFcXNxlPQ+DCBER0UVwOBzo6OiARqNBampqRA7cFAQB/f396OjoQGlp6WWtjDCIEBERXQSbzQZBEJCamoqYmBipy5FMamoqWlpaYLPZLiuIsFmViIjoEkTiSsipfPXxM4gQERGRZBhEiIiISDIMIkRERIRt27ZBJpNBr9cH9HUZRIiIiEgy3DVDRHSKUYsdn57oRW2PCVEKOWblarGsNBVRCv7cRuQPDCJERC5vHOzA//dBDYbHJk7LzE/W4LEby7G0NEWiyiioCQJgG5PmtaM0wEXsXrFYLHj44YexYcMGGI1GzJ07F7///e8xb9680x47NjaGm2++GUajER988AF0Op0PC/diECGiiCcIAh7/qBZ/2d4EAMhL0mD55FSYbQ5sPtGH1sEx3Pn8Pvzi+un42qICaYul4GMbA/47S5rX/nEXoIq94If/8Ic/xBtvvIGXXnoJ+fn5+PWvf421a9eioaFhwuP0ej2uvfZaxMXFYdOmTdBoNL6u3INBhIgi3h8+rfeEkO9eVYrvriyB0nUrZtRix6/er8GGA+342TvHoVLI8eX5eVKWS3RJRkdH8fTTT+PFF1/EunXrAADPPfccNm3ahL/97W+eVZGenh7cdtttKC0txSuvvAKVSuXXuhhEiCiifVTVjT9urgcAPHZTGe5YkD/heqxaice/WI74aCWe29GM//d2NSZlxGNOXqIU5VIwitKIKxNSvfYFamxshM1mw5IlS7x/PSoK8+fPx4kTJzxBZPXq1Zg/fz5ee+01nxxqdz4MIkQUsQZGLPjxW1UAgG8sLTwthLjJZDL8+Jqp6DKY8cGxbvzHK4fx0YPLkBAdFchyKVjJZBd1eyTYXXvttXjjjTdQU1OD8vJyv78e28CJKGL94j2xMXVKRjx+ePWUcz5WJpPhiS+WIz9Zg079OH638WSAqiTyjeLiYqhUKuzatcvzZzabDQcOHMC0adM8f/bEE09g/fr1uOqqq1BTU+P3uhhEiCgiHWobxntHuyCXAb+5ZSZUyvN/OYyPjsJ/3yT+hPj3PS2o7jT4u0win4mNjcV9992Hhx9+GB9//DFqampw7733YmxsDPfcc8+Ex/72t7/FHXfcgZUrV6K2ttavdTGIEFHEEQQBT3wofnG9pSIH5TnaC/67S0pS8IUZmXAKwH9/eMJfJRL5xRNPPIGbb74Zd955J+bMmYOGhgZ88sknSEw8vefp97//Pb70pS9h5cqVOHnSfyuAMkEQBL89+2UyGo3QarUwGAxISEiQuhwiChM76vtx59/2Q62UY+t/LkeW7uKOcu8YHsOK326DzSHglXsXYHEx54tEErPZjObmZhQWFiI6OlrqciRzrvfhYr5/c0WEiCKOe6vu7fPzLjqEAEBOoga3u7bwsleE6PIwiBBRRDneZcCO+gHIZcA9Swsv+XnuX1EClUKOytZhHGob9mGFRJGFQYSIIsoLu1oAANeUZyI36dKnRaYnROP6WeI0zed3NvuiNKKIxCBCRBHDZLbhg2PdAIC7Fhdc9vN9fYm4ovJRdQ+69OOX/XxEkYhBhIgixntHuzFuc6A4NRYV+Zc/GXVaVgIWFSXD4RTw0p6Wyy+QQkoQ7/UICF99/AwiRBQxXqtsBwDcNi8Xsos4sfRc7l5SAAD4d2UHbA6nT56Tgpt77LnVapW4Emm5P/7LHQPPEe9EFBFqe4w42q6HUi7DF+fk+Ox5V05JQ2q8Gv0mC7bU9mHt9AyfPTcFJ6VSCY1Gg/7+fkRFRUEuj7yf6Z1OJ/r7+6HRaKBUXl6UYBAhoojw9mHxULKrpqYhJU7ts+dVKuT44uxsPLu9Ca9XdjCIRACZTIbMzEw0NzejtbVV6nIkI5fLkZeXd9mriwwiRBT2BEHAh1Vik+p1M7N8/vy3zs3Bs9ubsLWuD/0mC1LjfRd0KDipVCqUlpZG9O0ZlUrlk9UgBhEiCnvHu4xoGxpDdJQcK6ek+fz5S9LiMStXhyPterxzpBPfWFbk89eg4COXyyN6sqqvRN6NLSKKOB+4VkNWTE6DRuWfn79urhD7Tt472uWX5ycKVwwiRBTWTr0tc015pt9eZ11ZBuQy4GiHAe1DY357HaJwwyBCRGHtRLcJrYP+uy3jlhKnxqLiZADeFRgiOj8GESIKa1tqewEAS0tSEav2b1vcteViI6x7eisRnR+DCBGFtS21fQDg19UQt7XT06GQy1DVaUDbIG/PEF0IBhEiCltDo1YcbtcDAJZPTvX76yXHqbGoSLw98/FxrooQXQgGESIKW9tP9kMQgCkZ8cjSxQTkNVdPSwcAbD7RF5DXIwp1DCJEFLYCeVvGzf1ala3DMIzZAva6RKGKQYSIwpLDKWB7fT8AYEUAg0hukgaT0uPgcArYdpKrIkTnwyBCRGGppssI/ZgN8WolZufqAvraV03l7RmiC8UgQkRhaXfjAABgQVEylIrAfqm7yrUCs62uD3aHM6CvTRRqGESIKCztbhwEACx2DRkLpNl5iUjURMFotqOydTjgr08UShhEiCjsWO1OHGgZAgDPtNNAUshlWDFZXBXZfKI34K9PFEoYRIgo7Bzr0GPM6kBSrAqT0+MlqcHdILujfkCS1ycKFQwiRBR23LdlFhUlQy6XSVLDkpIUyGRAbY8JfSazJDUQhQIGESIKO+5GVSluy7glxaowPSsBALCrgasiRGfj1yDy+OOPY968eYiPj0daWhpuvPFG1NXV+fMliSjCmW0OHGrVA5CmUfVUS0vEsfK8PUN0dn4NIp999hnuv/9+7N27F5s2bYLNZsOaNWswOjrqz5clogh2qHUYVocT6QlqFKbESlrLstIUAMDO+gEIgiBpLUTByq9nYn/88ccTfv/iiy8iLS0NBw8exBVXXOHPlyaiCHWgRdwuO78wGTKZNP0hbhX5iYiOkqPPZMHJ3hFMzpCmcZYomAW0R8RgMAAAkpKSznjdYrHAaDRO+EVEdDEqW8Vtu3PzEyWuBIiOUmB+oXh7aIdr3DwRTRSwIOJ0OvHggw9iyZIlKCsrO+NjHn/8cWi1Ws+v3NzcQJVHRGHA4RRwpE0PQFyNCAbLSly3Z9iwSnRGAQsi999/P6qrq7Fhw4azPubRRx+FwWDw/Gpvbw9UeUQUBk72mmCy2BGrUmBKkNwGWerqE9nfPAQbx70TncavPSJuDzzwAN5//31s374dOTk5Z32cWq2GWq0ORElEFIYOusapz8rTBfx8mbOZnB4PnSYK+jEbqjsNmJ0XHCs1RMHCr5+pgiDggQcewFtvvYUtW7agsLDQny9HRBHOHUQq8s/chyYFuVyGeQViPfuahySuhij4+DWI3H///Xj55ZfxyiuvID4+Hj09Pejp6cH4+Lg/X5aIIlQwNaqeamGR2LC6r2lQ4kqIgo9fg8jTTz8Ng8GA5cuXIzMz0/Prtdde8+fLElEE6jOa0T40DplMvDUTTBYUiisiB1qGYWefCNEEfu0R4QAfIgoU922ZyenxSIiOkriaiaZmJiA+WgmT2Y6abiNm5OikLokoaARHNxcR0WU61ObuDwmu2zIAoJDLMN/dJ9LEPhGiUzGIEFFYONouDkyclauTtpCzWFDkblhlnwjRqRhEiCjkOZwCqruCPIi4Jqzubx6Cw8nb1kRuDCJEFPIa+kYwZnUgVqVAUWqc1OWc0fSsBMSplTCa7ajt4fEVRG4MIkQU8o626wEAZdlaKOTSHnR3NkqF3NO/wj4RIi8GESIKeUc79ACAmUF6W8bN3SdyoIVBhMiNQYSIQt6xDrE/ZGaQb4ud65r4Wtk6zPEGRC4MIkQU0sw2h6fnYkaOVuJqzm1GjhZKuQz9Jgs6hjlhmghgECGiEHei2wibQ0BSrAo5iTFSl3NO0VEKTM8Ww5J77glRpGMQIaKQ5r4tMyNHC5ksOBtVT1XhOn3XPQmWKNL5dcQ7RR6zzYG2oTEMmCwwjNtgdwpQK+WIUyuRm6RBli4maHc1UGjyNKoGeX+IW0V+Ip7f1cwgQuTCIEKXzOkUUNNtxIGWIexvHkJNtxHtQ2M416wmlVKOWbk6LClOwRdmZqI4SGc+UOioOmVFJBTMydcBEG8pjVrsiFXzyzBFNn4G0EWx2B3Y3TCIjTU9+PREH/pNltMeE69WIl0bDW1MFKIUMljtThjGbWgfGofV7sT+ZjG4/P7Tk6jIT8R3lhdj5ZS0kFhWp+BitjnQ2D8CQJwhEgoytTHI1sWgUz+Oo+16LC5JkbokIkkxiNB5CYKAqk4D/n2wA+8c6YJh3Oa5FqdWYm5BIuYVJGF2rg4l6XFIjVOfMVQ4nAJaB0exr3kIG4/3YHv9AA62DuOelyoxM1eHx24sC5lvJhQcantMcApASpwKafFqqcu5YHPyE9GpH8fB1mEGEYp4DCJ0VmNWO9481Im/72nByd4Rz5+nJ6ixZloGVk9Lx8KiZKiUF9bzrJDLUJQah6LUONw+Pw99RjP+tqsZ/9jTiqPtelz/55349pXFeGj1JCgV7KOm8zvuOl9mamZCSK2oVeTp8N7RLlSyT4SIQYRO16kfx9/3tODVfW0wmu0AALVSjrXTM3BLRQ6WlKT4pOE0LSEaj66binuWFOIX79fgg2Pd+L9tjTjUNownb5+D1BD6CZekcbxLnB8yPSu0VtLmFoiDzQ61DcPpFCBnAzdFMAYR8jjZa8KftzTgg6puz+mg+ckarF9UgJsrcqCNifLL66YlROOpr8zBurIu/Ojfx7C3aQi3PLMbL9+zALlJGr+8JoWHGk8QSZC4koszJSMeMVEKmMx2NPSPYFJ6vNQlEUmGQYRwotuIP29pwIfV3XBPnV5cnIy7lxRi5ZS0gG23/cKMLEzJSMDdL+5H6+AYbn56N165dyFK0rizhk7ncAqeiarTQiyIKBXi7rE9TYM42DrMIEIRjTfiI9jJXhO+9Y9KrPvjDnxQJYaQdWUZ+OC7S/HKvQuxelp6wGd+lKTF4d/fXozJ6fHoM1lw59/2oUvPUdh0uqb+EZhtTmhUChQmx0pdzkVzn8Rb2cI+EYpsXBGJQL1GM36/6ST+VdkOpwDIZMC15Zn4j5WlmJwh/U9m6QnRePWbC3HrM7vR2D+KO/+2D2/etwRajX9uDVFocveHTM1MCMkeC/c8kSPtDCIU2RhEIsioxY5nP2vEczuaMW5zAACunp6BH6yZhNIgWxpOilXhH/cswC1Pi2HkwdcO42/r54XkNxzyD/eOmVDrD3FzT4Jt7B+F0WxDQjSDNkUm3pqJAIIg4OPqHqz+3Wf405YGjNscmJOnwxv3LcIzd1YEXQhxy9LF4C9fmwu1Uo6tdf34w6cnpS6JgkhNt6s/JDM0g0hynBq5SeIhfe7psESRiEEkzLUNjuHrLx7At18+iC6DGTmJMXj6jjl4477FqMhPkrq88yrL1uKJm8sBAE9ubcCexkGJK6JgIAhCyG7dPZV7VeRIu17SOoikxCASphxOAc9+1ojVv/8MW+v6EaWQ4YEVJdj0/SuxrjwzpIY/3TQ7B1+elwtBAP7z9aMwmm3n/0sU1roMZujHbFDKZZiUEbq7qmbl6gAwiFBkY49IGGodHMV/vn4UB1zd+EtKkvHLG8pC+oC5//eFadjVOID2oXH86r0a/ObWmVKXRBI63ineyihJi4NaqZC4mkt3ahARBCGkfkAg8hWuiIQRQRDwz32tWPfHHTjQMoxYlQL/c3M5Xr5nQUiHEEA80+Z3X5oFmQx4/WAH9jbxFk0k8/SHhGijqtv0LC0Uchn6TRb0GM1Sl0MkCQaRMGEy2/DAK4fxk7eqMWZ1YEFhEj5+8ArcNi8vbH7KmleQhNvn5wEAfvp2Nax2p8QVkVTqekwAgKkZoR1EYlQKTHY1ix/l7RmKUAwiYaC2x4gb/rwLH1R1QymX4cfXTMGr9y4My/HoP1o7BcmxKtT3jeCFXc1Sl0MSqesVg0gwzL25XDNdt2cOM4hQhGIQCXEfVXXjxqd2oWlgFJnaaLz2rYX45hXFYTtvQ6uJwiPrpgAA/ry1AcOjVokrokAz2xxoGRgFIJ7ZEupmu4IIV0QoUjGIhChBEPDMZ42475+HYLY5saw0BR98d1lIbMm9XDfPycHUzASYzHb8eWuD1OVQgDX0jcApADpNVFic0OxeEanqMHgOmySKJAwiIcjmcOKRN6rwxEe1AIC7FhfghbvmISlWJXFlgSGXy/Coa1Xk73ta0D40JnFFFEi1rv6QyenxYdH/VJIWB41KgVGrA439I1KXQxRwDCIhxmxz4Nv/OIjXKtshlwH/dd00/Nf106FURNb/yismpWJpSQpsDgF/3sJVkUhy0tUfEg63ZQBAIZehPFscynakTS9tMUQSiKzvXiFuzGrHN16qxObaPqiVcjz3tbm4a0mh1GVJ5vurJwEA3jjUgU6e0BsxPCsiIb5j5lSz8nQAgCMdeknrIJICg0iIMJltWP/8fuxsGIBGpcCLd8/HVVPTpS5LUhX5iVhcnAy7a4osRYa6HnGGSDjsmHGb5Rr1zoZVikQMIiFg3OrA3S8cwIGWYcRHK/HyNxZgUXGy1GUFhQdWlgAANhxoRx8HQoU9/ZgVvUYLAGBSemgP6TuVu2G1tscEs+tkbKJIwSAS5Kx2J7718kFUtg4jIVqJV+9diDl5iVKXFTQWFSWjIj8RVrsTz+1okroc8jP3ILNsXQzio6MkrsZ3MrXRSI1Xw+EUUN3Jk3gpsjCIBDGHU8CDrx3G9pP9iIlS4IW756MsO3RPGvUHmUyG+1cUAxBXRUYtdokrIn+qC7NGVTeZTOY5ifdYB4MIRRYGkSD2q/dr8GFVD1QKOf7ytQpU5HMl5EyWT0pDYUosTGY73jzUIXU55EfeRtXwCiIAMCNH/CGjiisiFGEYRILU3/e04MXdLQCA3982C8tKU6UtKIjJ5TKsX5QPAHhxdwucHAoVturCOIi4t/AyiFCkYRAJQtvq+vBf7x4HAPzw6sm4dkamxBUFv5srchCnVqKxfxQ7GwakLof8QBAEnAzjIOK+7drYP4IR3mKkCMIgEmSa+kfwH68chlMAbq3IwX1XFktdUkiIj47CLRU5AMDD8MJUl8EMk8UOpVyGopTw2THjlhqvRqY2GoIA1HQZpS6HKGAYRILIuNWB7/zzEEwWO+YVJOKxm8rDYoR1oKxfXAAA2HaynwPOwpB7fkhxahxUyvD80uVeFTnGwWYUQcLzszkECYKAn7xdhdoeE1Li1HjqK3PC9outvxSmxGJRUTIEAfh3JZtWw427UXVSGN6WcZvhCiLcwkuRhN/pgsSGA+1481AnFHIZ/vyV2UhLiJa6pJB027xcAMDrB9vZtBpm3P0h4bZ191Rlrp0zxxhEKIIwiASBhr4R/OI9sTn14bWTsbCIU1Mv1dVlGYiPVqJjeBx7mgalLod8qL5PPJm2NC38+kPc3DtnmgdG2bBKEYNBRGJWuxMPvnYYZpsTy0pT8M1lRVKXFNKioxS4YVYWAOC1A+0SV0O+4nQKaOx3BZH08F0RSYlTI8vVsHqcqyIUIRhEJPbHzSdR3WmEThOF3946E3I5m1Mv121z8wAAHx/vgWHMJnE15Aud+nGYbU6oFHLkJsZIXY5flXOwGUUYBhEJVbYM4f+2iafGPvHFcqSzL8QnyrITMCUjHla7Ex9Vd0tdDvlAfZ/YH1KUGgulIry/bHGwGUWa8P6MDmJmmwM/fOMYBAG4pSIHV5dxaJmvyGQy3DArGwDwzpEuiashX2hw9YcUh3F/iFu568yZKp45QxGCQUQiT21tQFP/KFLj1fjpF6ZJXU7YuW6mGOz2Ng+i12iWuBq6XPW94d+o6uZeEWkaGIXJzFuLFP4YRCRQ22PE065bMr+6YTq0MeFznHmwyEnUoCI/EYIAvH+Mt2dCXYO7UTUtfBtV3ZJiVcjWiX0w1Z2csErhj0EkwBxOAT96owp2p4A109J5S8aP3Ltn3j3SKXEldDkEQUCDa0WkJAJWRADvqggHm1EkYBAJsL/vacHRdj3i1Ur88oYyqcsJa9eUZ0Ihl+FohwEtA6NSl0OXqNdogclih0IuQ0GKRupyAqKcg80ogvg1iGzfvh3XXXcdsrKyIJPJ8Pbbb/vz5YLe4IgFv9t0EgDww3VTkKHlLhl/SolTY3GxOBzu/WNsWg1V7kbV/CQN1EqFxNUEBldEKJL4NYiMjo5i5syZeOqpp/z5MiHjtxvrYDLbMT0rAV+Znyd1ORHh2nLx1tcnx3slroQulXvrbqTclgEmTlg1smGVwpzSn0++bt06rFu3zp8vETKqOw3Y4Jr0+fPrpkPBwWUBsWpaOuRvVaGq04CO4THkJEbG0n448Yx2T4+cIJIYq0JOYgw6hsdR3WnA4uIUqUsi8hv2iASAIAj4r3ePQxCA62ZmYX5hktQlRYyUODXmFYjvN1dFQpP71kwkrYgAwIwc3p4hPxIEoLcG2PVH4Phbkpbi1xWRi2WxWGCxWDy/NxrDY+vau0e7UNk6jJgoBR5dN0XqciLO2ukZ2Nc8hE+O9+CepYVSl0MXqaEvcrbunqosW4sPq3pwjIPNyFcsI0DzdqB+I1C/CTB2iH9eeAUw/SbJygqqIPL444/jF7/4hdRl+JTZ5sCvP64DAHxneTGydOF9TkYwWluWgV++X4MDLUPoN1mQGq+WuiS6QIMjFgyNWiGTAcWpEbYikq0DwBURugyCAAzUAw2bxPDRuhtwWL3XldFAwVJg8jXS1YggCyKPPvooHnroIc/vjUYjcnNzJazo8v1zXxs69ePISIjGvVfwZF0pZOtiMCNHi2MdBnx6ohe3s1E4ZLhXQ7J1MYhRRcaOGbey7AQAQMvgGAzjNg4+pAtjHQNadogrHvUbAX3rxOu6fKB0jfirYCmgkr5vLqiCiFqthlodPj+tmsw2PLW1AQDw4KpSREdF1hfSYLJ2egaOdRjwyfEeBpEQ4mlUjbD+EADQaVTIS9KgbWgM1Z0GLClhwyqdxWAj0PCpGDyadwAOb4sDFCogfwlQuloMH8klgCy4Nkv4NYiMjIygoaHB8/vm5mYcOXIESUlJyMsL/28Gz21vwtCoFUWpsbilIkfqciLaqqnp+M0nddjTOIhxqyPifroOVZHaqOpWnq1F29AYqhhE6FQ2M9C6E6h3hY+hxonXtbli8ChZLfZ/qIP788evQaSyshIrVqzw/N5922X9+vV48cUX/fnSkus3WfDXnc0AgIfXTA77o8uD3aT0OGTrYtCpH8eepgGsnJIudUl0ASK1UdWtLFuLD6q6UcU+ERpuFUNHw6diw6ltzHtNrgTyFrluuawGUqcE3arHufg1iCxfvhyCIPjzJYLWn7fUY8zqwMxcHa4uy5C6nIgnk8mwYkoqXt7bhi21fQwiIcKzIhJBM0ROxQmrEcxuBdp2u3o9NgEDdROvx2d6b7cUXglEJ0hTpw8EVY9IuOjUj+OV/W0AgB9dPRmyEEqm4WzF5DS8vLcNW2v7IQgC/78EOaPZhh6jGUDk3ppxN6y2smE1Mhg6xNDR8CnQtA2wjnivyRRA7gJX+FgNpJeF1KrHuTCI+MHT2xpgcwhYVJTMiYhBZHFxCtRKOTr14zjZO4LJGZG53B8q3KshafFqJERH5jdgnUaF3KQYtA+N43inAYvZJxJeHDagfZ9rrsenQN/xiddj07zBo2gFEKOTpEx/YxDxsW7DOP51QBwS871VpRJXQ6eKUSmwqDgZ2+r6saW2j0EkyDX1iycmR+pqiFt5thbtQ+OoYhAJD8Zu7w6Xpm2A5ZTBnTI5kD3X2+uRMQOQh39/IYOIjz2zrRFWhxMLCpOwsChZ6nLoc1ZOScO2un5srevDfcuLpS6HzqF5QFwRKUyJlbgSabknrLJhNUQ57EBnpWvVYyPQUzXxuiYZKFklho/ilYAm8o4AYRDxoV6jGa+6Drb73lVcDQlGKyanATiOg63DMIzZoNVE5pJ/KHCviBRF2ETVz2PDagga6feuejRuAcz6idez5niHimXNAuSRPU6AQcSHnt7WCKvdiXkFiVhUzNWQYJSbpEFJWhwa+kawvb4f183MkrokOovmAVcQifQVkSwxiLQMjsFotkVsv0xQczqArsPeVY+uwxOvR+uAkqtcqx5XAXGpkpQZrBhEfKTfZMGrrp0y37tqEndkBLHlk1LR0DeCnfUDDCJByukUvEEkNbKDSGKsCjmJMegYHkd1p4EN8MFibAho2Oyd7TE+NPF6xgzvqkd2BaDgt9uz4TvjIy/tboHF7sSsXB2WlHA1JJgtLU3BX3c2Y2fDALfxBqkuwzgsdieiFDJk86BIlGdrGUSk5nQC3Ue8t1w6KgGcMidLnQAUrxCDR8kqIJ7zoy4Ug4gPjFrs+PueFgDAt68s5je2IDe/MAkqhbiNt2VwLOKbIYORuz8kPzmWU4khNqx+VN2Dqk7j+R9MvjM+DDRudc322ASM9k+8nl7mbTTNnQ8oeNvsUjCI+MCGA+0wmu0oSonF6mmc2BnsNCol5uTrsLdpCDvr+xlEgpD7tgz/34jYsBoggiDuamlwTTNt3w8IDu91VRxQtNx7jos2W7JSwwmDyGWyOZx43nWmzDeWFUEh52pIKFhakoK9TUPYUT+AOxcVSF0OfU5Tv7h1N9L7Q9zcQaR5YJQNq75mNojzPNwTTU3dE6+nTvGueuQtApQqScoMZwwil+mDY93o1I8jJU6FL85hOg4VS0tT8duNJ7GnaRB2h5PL/0GmiTtmJkiMVXkObTzeaeSuvMshCEDfCW+TadsewGn3Xo/SiGe3lK4SVz0S86WrNUIwiFwGQRDwzGfi8ct3LylEdFRk7wUPJeXZWiREK2E023Gs04A5eYlSl0Sn4AyR05Vna9GpFxtWGUQukmUEaP7Me4CcsWPi9eQSb5Np/hIgKlqaOiMUg8hl2F4/gNoeEzQqBb66gKk5lCjkMiwuTsHHx3uwq36AQSSImG0OdBnGAbBH5FTlOVp8fJwTVi+IIAAD9a5Vj01A627AYfVeV0YDBctc22tXAUlF0tVKDCKX47ntTQCA2+fncUJnCFpaKgaRHQ0D+A9Owg0aLYOjEAQgIVqJ5Fjej3crY8PquVnHgJYdrqFimwB968TriQWuVY/VQMFSQKWRpEw6HYPIJarvNWFnwwDkMuCuxQVSl0OXYKnrALHDbcMYtdgRq+anQzBoPuW2DLfCe7kbVpsGRmEy2xDPhlVgsNG7tbZ5B+CweK8pVOJtFvcBcsklAP89BSV+5b1EL+xuAQCsmZaB3CQm61CUn6zxNAAebB3GFZM4djkYsFH1zJJObVjtMkbmoZo2M9C609vrMdQ48bo2VwwdpWvEWy9q9hiFAgaRS2AYs+HNQ2Kz011LCqQthi6ZTCbDgqIkvHmoE/uaBxlEgkQjt+6eVVl2gqdhNWKCyHCLN3g0bwfs495rcqW4pda96pE6haseIYhB5BK8VtkGs82JKRnxWFAYeUc2h5OFhcl481An9jYNnf/BFBDeYWb8afbzyrO1+OR4b3g3rNot4pba+k1iv8fAyYnX4zO9qx6FVwLRCdLUST7DIHKR7A4nXtotNkHdvaSA97BD3IIiMUge69Bj3OpAjIpbsKUkCMIpW3e5IvJ57obVsAsiho5TVj0+A6wj3msyBZC7wBs+0qdz1SPMMIhcpE9P9KFTP45ETRRumMUBZqEuL0mDTG00ug1mHGobxpISHigmpeExGwzjNgBAQTKDyOedOmF1xGJHXKg2WDtsQPs+7w6XvpqJ12PTXMFjNVC0AojRSVImBUaI/iuWzou7xXHut8/P4wCzMCCTybCgMAlvH+nCvqZBBhGJuUe7Z+tiuDp1BslxamRpo9FlMON4pwELQqlPxNjtPbm2aRtgOeUAP5kcyJknbq0tXQ1kzADknHYcKRhELsKJbiP2Ng1BIZfhqws5wCxcLChKxttHutgnEgSaeNjdeZVla9FlMKMq2IOIww50HHAdILdRPEzuVJpk7xkuxSsBDfvtIhWDyEV4cVcLAODqsgxk6WKkLYZ8xr374Ei7HmabgytdEmJ/yPmVZ2uxsSZIG1ZH+lyrHpuAxs3igXIeMiB7jmvVYw2QNQuQ83ONGEQumGHchneOdgIA1vO01rBSkKxBWrwafSYLDrfpeY6HhJoHxFszXBE5u7KcIGpYdTqAzkPeVY+uwxOvR+tcqx6rgeKrgDhukafTMYhcoLcOdcBsc2JSehzmFfBcknAizhNJxntHu7C3aZBBREI87O78JG9YHR0UVzvqN4mrH+Ofu6WZOdO76pFdASj4bYbOjf9CLoAgCHhlfxsA4I4F+dyyG4YWFCbhvaNd2Nc8KHUpEcvhFNA6OAaAU1XPJSVO7dnpFZCGVacT6D7iHaXeUQlA8F5Xa4HiFeKqR8kqID7Dv/VQ2GEQuQCVrcM42TuCmCgFbprDLbvhyN0ncrhND4vdAbWS964DrXN4HFaHEyqlnD1Y51GWrUW3PxtWx4eBxi1A/adi+Bjtn3g9vcwVPFYDufMBBc+9oUvHIHIB/rlXHGB2/cwsJPCgqbBUnBqLlDg1BkYsONpuwHxOzA24Jld/SEGyBgo5Vx3PpTxbi001vb47iVcQxF0t9RvF2y3t+wDB6b2uigOKlrtOr10FaPkDGfkOg8h5DI1a8WFVDwDgjoV5EldD/iKTyTCvIBEfVfegsnWIQUQCnv4QjnY/r3JfTFg1G8R5HvUbxZWPkZ6J11OneFc98hYBStWlvxbROTCInMe/D7bD6nCiPFuLGTk6qcshP6rIF4PIwZZhqUuJSJ4zZrh197zco96bLqZhVRCAvhPeaabtewGn3Xs9SiOe3eKeaKrjD14UGAwi5+B0Cnhln9ik+pUF/KQMd3MLxFWQg23DcDoFyHl7IKDct2bYqHp+qfHehtWaLuPZV/AsI+LZLe5VD2PHxOvJJd6Ta/MWA1HR/i+e6HMYRM5hT9MgWgbHEKdW4vqZWVKXQ342PSsB0VFy6MdsaBoYQUlavNQlRZRmDjO7KKc2rHqCiCAAA/Wu4LERaN0NOG3ev6SMBgqWucLHKiCpSJriiU7BIHIO/9wnNqneNDsbsaF6uBRdsCiFHDNydNjfPITKlmEGkQAas9rRZTADYI/IhXI3rNa19wInT3rDh75t4gMTC1zBYw1QsBSI4o4kCi787noWfUYzNh7vBcDbMpFkbn6iGERah/Hl+fz/HigtA+L8EJ0mComxbIo8r8FGXD36NmZGvYtFdSeAulNWPRQqIH+JN3wkFwOcfURBjEHkLP5V2Q67U0BFfiKmZiZIXQ4FyFzX1NxDrWxYDST2h5yHzQy07hSbTOs3AkNNmARgkmvcjTMhB/JJ7lWPZYCaq0oUOhhEzsDpFLDhQDsA4Cv8qTiizMkTg0jTwCgGRyxIjlNLXFFkcPeHFPK2jNdwiyt4bAKatwP2ce81uRLIW4Q/tRfi3bEyPH7TLZhXyKMJKDQxiJzBnqZBdAyPI16txDXlmVKXQwGk06hQmhaH+r4RHGwdxprpHFcdCE0DbFSF3QK07fGuegycnHg9PktsMC1dI26zjU7AsZcq0XCiF1WdRgYRClkMImfwr0pxNeT6WVmIUXHUd6SZW5DIIBJgTf0RemtG3+46ufZTcbiYbdR7TaYA8ha6Tq9dA6RPP63Xozxbi09P+HDCKpEEGEQ+xzBmw0fV4oTBL83NlbgaksKcvES8ur8dlewTCQhBEE5ZEQnzWzMOG9C21xU+NgF9NROvx6W7Tq5dBRStAGJ053y68hyxf+2yJqwSSYxB5HPeOdoJq92JKRnxmJGjlbockoB7sFlVhwFmmwPRUVwV86eBEStMZjtkMiA/WSN1Ob5n7PYGj6ZtgMXovSaTAznzvKPUM2YAcvkFP7V7wmpj/wjGrHZoVPySTqGH/2o/5zVXk+qX5uZCxi1vEakgWYPkWBUGR6043mVART7PnfEn92j3bF1MeIQ+hx3oOOA6QG6TeJjcqTTJrlWP1UDxSkBz6f++0uKjkZ6gRq/RgpouoydEE4USBpFTVHcacLzLCJVCjptm83TJSCWTyVCRn4iNNb2obBlmEPEzd39IYSj3h4z0iafW1m8EGreIB8p5yIDsOa6Ta1cDWbMvatXjfMqzteg19qGq08AgQiGJQeQUr7uaVFdPT+dQpQg3O08MIkfa9VKXEvbcKyLFodQf4nQAnYe8qx5dhydej9Z5m0xLrgJiU/xWSlm2Fp+e6GOfCIUsBhEXs82Bt490AWCTKgGzcnUAwCASAI2eGSJBviIyOgg0bnaFj83A+NDE65kzvaseOXMBeWBuM5W7+kS4c4ZCFYOIyyfHe2AYtyFLG42lJf776YVCw4wcLeQyoNtgRq/RjPQEnkrqL83uqarBNkPE6QS6j3jnenQeBCB4r6u1QPEKV/hYBcSnS1KmO4g09LFhlUIT/8W6vF4pHo99y9xcKHj8e8SLVSsxKT0etT0mHG7T4+oyzhPxB7vDibYh8ZyZoFgRGR8WezzqN4k9H6P9E6+nl4lNpqVrxN0uiihp6jxFWkI00uLV6DNZcKLbyJ4mCjkMIgDah8aws2EAAHBrRY7E1VCwmJ2nE4NI+zCDiJ90DI/D5hCgVsqRpZXgVFhBEHe11G8Uw0fHfkBweq+r4oCi5d5VD21wNrGXZ2uxubYPVR3c5UWhh0EEwOsHxdWQJSXJyE0KwzkGdElm5erw6v52HGnTS11K2HIfdleYEgt5oFYizQZxnkf9RnGi6UjPxOupU7yrHrkLAWXwN66XuYNIp/H8DyYKMhEfRBxOAf+u9M4OIXKb7ToAr6rTAIdT4C07P2jqD8AZM4IA9J3wrnq07wWcdu/1KI14dkupa7aHLvQOumTDKoWyiA8iuxoG0GUwQxsThbU8V4ROUZwahzi1EiMWO072mjA1M0HqksKOe7S7z/tDLCNA82fe8GHsnHg9udQbPPIWA1Gh3Yxc7poCXd9nwrjVwTOyKKREfBB5zbUacuOsrPCY6kg+o5DLMCNHi92NgzjcpmcQ8YNm94pIymXOEBEE8bRa9w6X1t2A0+a9rowGCpaJt1tKVwFJRZf3ekEmPSEaqfFq9JssqOk2oiI/UeqSiC5YRAeR4VErNh3vBQB8aR5vy9DpZuXqsLtxEEfah/GVBaG3ZB/smi5n6651FGje4TrHZSOgb5t4PbHAFTzWAAVLgSgJmmEDqDxbiy21fajuNDCIUEiJ6CDy9pFOWB1OTM9KwPQsHnBHp3P3iXCwme+NWuzoNVoAXMSKyGCj93ZLy07AYfFeU6jEwFHiajRNLgYi6LyoMlcQOdqhl7oUoosSkCDy1FNP4Te/+Q16enowc+ZMPPnkk5g/f34gXvqsBEHwHHB3G1dD6CzcE1br+0ZgMtsQHy393Ihw4R7tnhyrglZzlvfVNg607PKuegw1TbyuzfXucCm8AlAFwSwSiczKFX+YOsrQTCHG70Hktddew0MPPYRnnnkGCxYswB/+8AesXbsWdXV1SEtL8/fLn1VVpwG1PSaolHLcMDM4ZwOQ9FLj1cjWxaBTP45jHQYs4dRdnzlro+pwi6vXYxPQvB2wj3uvyaOA/EXeVY/UyRG16nEuM3N0AMSR+YZxG7QxDM0UGvweRH73u9/h3nvvxd133w0AeOaZZ/DBBx/g+eefxyOPPOLvlz8r92rIurKMs/80RgRxsFmnfhxH2vUMIj7kPnW3NDkKaNzqmma6SWw6PVV8lneHS+GVQDSbhs8kOU6NvCQN2obGcKxDj2WlqVKXRHRB/BpErFYrDh48iEcffdTzZ3K5HKtWrcKePXtOe7zFYoHF4r3nazT6ZzjPuNWBd10H3N3G2SF0HrNydXj/WDcOc7CZ7+jbkVn/Kp6L+gzLa2uAmlNWPWQKIG+hGDxKVgPp07nqcYFm5urQNjSGo+0MInR+/SYLHnnjGG6pyMHVZRmQSfR55tcgMjAwAIfDgfT0iYdBpaeno7a29rTHP/744/jFL37hz5IAAB8f74bJYkduUgwWFiX7/fUotM3O0wEQG1YFQZDskzWkOWxA217XybWfAn01uA0AFACcAOLSXbdbVgFFK4AYnaTlhqpZuTq8d7SLzdV0Qd4+3InNtX0YHLViXXmmZHUE1a6ZRx99FA899JDn90ajEbm5vl+xWFiUjO+vmoTkOFXgxkpTyJqepYVSLsPAiAWd+nHkJPIYgAti7PY2mTZ9Bli8K5yCTI4jzlJ8ap+J227/OvKmLQDkcgmLDQ/u5mqGZjofQRDwryCZKu7XIJKSkgKFQoHe3t4Jf97b24uMjNOnmKrVaqjVan+WBADI1Mbge6tK/f46FB6ioxSYnBGP411GVHcaGETOxmEHOg54t9f2Vk28rkl2rXqsxkDaEtz0+0OQy4DvTpnPEOIj07MSXKHZytBM53S0w4D6vhFER8nxhZnSrYYAfg4iKpUKFRUV2Lx5M2688UYAgNPpxObNm/HAAw/486WJfGpGjhbHu4w42mHA1WXSftIGlZE+8VZL/UagcYt4oJyHDMie4xoqthrInO0JHA2NgwCAnEQN1EpONPaV6CgFpmYmoKrTgCPtegYROiv3asi6skwkSDyWwO+3Zh566CGsX78ec+fOxfz58/GHP/wBo6Ojnl00RKGgPFuHV9GOqo4IP1TM6QA6D7lWPTYC3UcmXo9JBIqvEsNHyVVA7Jl3GblniPj1sLsINStXJwaRNj2+MCNL6nIoCJltDrx3VNywcevcHImrCUAQue2229Df34+f/exn6OnpwaxZs/Dxxx+f1sBKFMxmuA4VO9YRgffeRweBxs2uRtPNwPjQxOuZM72j1LMrAPn5VzjcW3d9ftgdYWauDv/Y28oJq3RWH1V3w2S2IycxBgsLpd+wEZBm1QceeIC3YiikTc6Ih0oph9FsR+vgGArC+Ruo0ymudLgPkOs8CEDwXldrgeIVrlWPVUD8xf9Q4V0RuczD7ug07obVqk4DbA4nohTsv6GJ3HO0vjQ3Nyg2bATVrhmiYBWlkGNaZgKOtOtxrNMQfkFkbEjs8Wj4VPw12j/xenq5uLW2dA2QMx9QXN6XDvdU1aJwex+DQFFKLOKjlTCZ7ajrMaEsm+dokVfr4Cj2Ng1BJgNuqZD+tgzAIEJ0wWbkaMUg0q7H9TND/N67IAA9x7yj1Dv2A4LTe10VDxRd6V310PruGASbw4m2oTEA7BHxB7lchpk5OuxsGMDRDj2DCE3gblK9ojQVWbrgOJGaQYToApW7vqAf6wzRhlWzQRyl3rAJqP8UGOmZeD11qnfVI3choFT5pYy2oTE4nAJiohRIj4/2y2tEulm5YhA50qbHHQvypS6HgoTd4cS/D3YAkH52yKkYRIgu0EzXvffjnQY4nAIUQXBv9ZwEAeirce1w+RRo3ws47d7rURqgaLm44lG6GtDlBaSs5n7vYXfBcH86HJ062IzIbXt9P3qNFiRqorBqmnSHzn4egwjRBSpOjUNMlAKjVgea+kdQmh4vdUmns5jEKabuUerGzonXk0tdO1xWAflLAKX/Bwh+XtOAa8cMb8v4jTs0N/SPwGS2IV7iOREUHNxNqjfNzgmq+T0MIkQXSCGXoSw7AQdahnGswxAcQUQQxNNq3dNMW3cDTpv3ujIaKLzCe45LUpF0tbq4d8wUs1HVb1Lj1cjWxaBTP46qDgMW89ToiDcwYsHmE30AgNvmBc9tGYBBhOiizMjRuYKIHjdL1XFuHQWad7hWPTYB+raJ1xMLgNK14spHwRIgKjga0twa3bdmuCLiV7NydejUj+Nwu55BhPDWoU7YnQJm5uowOSMIfog6BYMI0UXwDDYLdMPqYKN3mmnLLsBh8V5TqICCpa4dLquB5GIgiAeueWaIpHCGiD/NytXhg6puHG7TS10KSUwQBLzm2i1zWxA1qboxiBBdBPfOmZouo3+HRdnGxcDhXvUYapp4XZsnNpiWrhZvvahCY3XBZLah3ySGKK6I+Nec/EQAwKG24cibBkwTHGrTo6FvBDFRClwn8QF3Z8IgQnQRCpK9w6JO9powPcuHMxqGmr0HyDXvAOzj3mvyKCB/kXfVI3VyUK96nI17NSQlTi35QVvhriw7ASqlHEOjVjQPjHKKbQT7l6tJ9ZryzKBsXGYQIboIcrkM5dla7G4cRFWH4fKCiN0iNpe6R6kP1k+8Hp/lWvVYIw4XUwfXfd1L0dTPiaqBolYqMCNbi8rWYRxsHWYQiVCjFjvePyYecPelIDjg7kwYRIgu0owcHXY3DuJohwFfnn+Rf1nf7hootkncZmsb9V6TKYC8hd7wkTYtJFc9zqWJp+4GVEVBoieI3BqEvQHkfx8c68ao1YHClFjML0ySupwzYhAhukjuhtWqTv35H2y3ioPE3KPU+09MvB6X7tpau1ocLhaj83W5QYWn7gbW3PwkPIsmVLYOS10KScTdpHrr3Jyg7RNiECG6SO6G1boeE8w2B6KjPjcYyNjl7fVo3AZYTd5rMjmQM8+76pFeDsgj53RUnrobWHPydACAhr4R6Mes0Gn8M7afglNDnwkHW4ehkMtwy5zgvC0DMIgQXbScxBgkxaowNGpFbY8Js7LixEPj3KsevVUT/4ImxTtGvXgloAnO5VF/EwTBE0S4IhIYyXFqFKXEomlgFIfahrFySrrUJVEAbdgvroasmJyKtITgPdeJQYToIslkMizNcEDV8hl0H7wI6PeJB8p5HwFkV3i312bOjqhVj7PpMZoxZnVAIZchL0kjdTkRY05+IpoGRnGwlUEkkphtDvz7kHjA3ZfnBeYcqUvFIEJ0IZwOoPOgZ5T6n7qPAFEA3AfYxiQCxVe5ttdeBcRykuXnuXfM5CVpoFIymAXK3PxE/PtgBypb2CcSST6u7oF+zIZMbTSWT06VupxzYhAhOpvRAaBhs7jLpeFTYHziF/JjzkJUx8zHV776DXEFRB48h0gFI3ejKrfuBtbcAnGw2dEOvX+H8FFQeWWfePTDbfNyoQzy/+cMIkRuTifQfdjb69F5EIDgva7WAiUrgZLV6E1fiuv/VA2FXYab0isQwxByXu4zZrh1N7CKUuKgjYmCYdyGmi6j52ReCl/1vSbsbxmCXBZ8B9ydCYMIRbaxIaBxixg8Gj4FxgYmXk8v9/Z65MwHFOKnTJogICWuHgMjFtR0G1HhGqdNZ9fEHTOSkMtlqMhPxJbaPlS2DjOIRIBX9ourIVdNTUemNrgOvTwTBhGKLIIA9Bxz9Xp8Ku52EZze66p4oHi5d7ZHQtYZn0Ymk6E8OwFb6/pR3WlgELkAvDUjHXcQOdQ6jHuWFkpdDvmR2ebAGwfFJtWvLAjuJlU3BhEKf2YD0LjVu+ox0jPxeupU76pH7kJAeWGzFspzdNha14+qQJ/EG4LMNgc69eLZOTzsLvDcQbmydYgH4IW5D451w2i2I1sXgytKg7tJ1Y1BhMKPIAB9NZ4dLmjfBzjt3utRGnGKaelqceVDd2n3UN2Dzao6GETOp3VwDIIAxKuVSI1TS11OxJmZo4NSLkOv0YKO4XHkcvt02HLflrl9fi4U8tAInAwiFB4sJvHslvqN4qqHsXPi9eRScWtt6WogfzGgvPxvhu5R7/V9JoxbHYhRsWH1bDy3ZVJj+dO4BGJUCkzP1uJoux4HWoYYRMJUXY84SVUpl+FLIXS2EIMIhSZBAAZOulY9NgKtewCnzXtdGQ0UXuGa67EKSPL9ffH0hGikxqvRb7KgptuAivzInJh6IdioKr2FhUk42q7HvqYhfDGIx33TpXtlXysAYNXU9KCepPp5DCIUOqyjQPMO7y0XQ9vE64mFrlWPNUDBEiDK/93iM7K12Fzbh6oOBpFzaWSjquQWFCXh2e1N2Nc8KHUp5AdjVjvePCyuBIdKk6obgwgFt8FG76pHyy7AYfFeU6iAgqXe8JFcHPDyylxB5BgbVs+pqZ8rIlKbW5AEuQxoGRxDj8GMDG3o/MRM5/fW4U6YzHYUJGuwtCS0JjsziFBwsY2LgcMdPoabJ17X5nlPri1cBqik/Qnb3SdSzSByVoIgTOgRIWkkREdhWlYCqjuN2Nc8iBtmZUtdEvmIIAh4aXcLAODORQWQh0iTqhuDCElvqFlsMK3fKN56sY97r8mjgPxF3lWPlElAEDU7unfONPSNYMxqh0bFT6nPGxy1wmi2QybjqbtSW1iYjOpOI/Y2DTGIhJE9TYM42TsCjUqBW+eGXv8Pv2pS4NktQOsucaBY/UZgsH7i9YRsscG0dA1QdCWgjpemzguQlhCN9AQ1eo0W1HQZMbeAfSKf574tk6WNQXQUdxZJaUFRMv66s5l9ImHGvRryxTnZSIiOkraYS8AgQoGhbxcPj6vfJG6ztY16r8kUQN4ioNQVPtKmBdWqx/mUZ2vRa+zDsQ4Dg8gZ8LZM8JhfkASZTAyHfSYz0uLZJxLqOobHsKmmFwCwflGBtMVcIgYR8g+7FWjf6z1Arv/ExOtx6d6BYkXLgRidFFX6RHm2Dp+e6GOfyFk0u7buFrNRVXJaTRSmZiSgptuI/c1D+MKMMx9hQKHj5b1tcArAkpJklKYH7+rxuTCIkO8Yu1xj1DcBjdsAq8l7TSYXD41zr3qklwPy4D6a+kKV5yQAAHfOnAVP3Q0uC4qSUNNtxN6mQQaREGe2OfDaAXGMwddCdDUEYBChy+Gwi4fGuQ+Q662aeF2T4lr1WAUUrwQ04XnboszVsNrYP4JRix2xan5anappwD1DhCsiwWBBYTJe2NWCfU1DUpdCl+ndo10YHrMhWxeDVVPTpS7nkvErJl0cU693h0vTVvFAOQ8ZkF3hPUAuc3bYrHqcS1p8NDISotFjNON4lxHzC8MzcF0Km8OJtsExAFwRCRbuf5/1fSMYHLEgmWf/hKSJW3bzQ+ZcmTNhEKFzczqAzoPeuR7dRydej0kUVzxKVgMlVwGxoTVIx1fKc7ToqTGjqtPAIHKK9qEx2J0CYqIUyAihkdPhLClWhSkZ8ajtMWFf8xCuKc+UuiS6BPuah3C8ywi1Uo7bQuhcmTNhEKHTjQ4ADZvF4NG4GRgfnng9c5b3ALnsCkDOLZnl2VpsqulFVYde6lKCinvrbkFKbMgNWQpnC4uSUdtjwq6GAQaREPXc9iYAwK1zc5AYq5K4msvDIEKA0wl0H3btcNkIdB4CIHivq7VAyUoxfBRfBcSH7r1Ifyl3TVitYsPqBJ7+EN6WCSpLS1Lw4u4W7GwYkLoUugQNfSZsru2DTAbcs7RI6nIuG4NIpBobAhq3uHa5fAqMfe4LUnq5d5R6zjxAwX8q5+KesNo0MIoRix1xbFgF4F0RKeZE1aCysDgZCrkMrYNjaB8aQ26SRuqS6CL8dYd49MWaaelhMa2YXy0jhSAAPce8J9d2HAAEp/e6Kh4oXi4Gj5JVQAK39V2MlDg1srTR6DKYcbzTgAVFyVKXFBR42F1wilMrMTtXh8rWYexsGMDt80PrtNZI1mcy481D4im737wi9FdDAAaR8GY2AI1bvbM9RnonXk+d6l31yF0AKEP7PqPUyrK16DKIDasMIiLemgleS0tTxCBSzyASSv6xpxVWhxNz8nSoyA+PxngGkXAiCEDvce8o9ba9gODwXo+KFc9ucU801YV2p3WwmZGjxcaaXvaJuBjGbRgYsQLgYXfBaFlpCv7waT12NQ7A6RTYTBwCxqx2/GNvK4DwWQ0BGERCn8Uknt3ivuVi6pp4PbnUu8MlfzGg5MwAf3EPNqvqYBABvKPd0+LViA/Bg7jC3YwcHeLUSujHbDjeZfQ0XFPwer2yA/oxG/KTNVg9LUPqcnyGQSTUCALQX+da9dgItO4BnDbvdWUMULjM2+uRVChdrRHm1IZVk9kW8d98edhdcItSyLGwKBmfnujFjoZ+BpEgZ3M48RfXlt17lhaG9ACzz2MQCQXWUaB5u/cAOUPbxOuJha5VjzVAwRIgKkaaOiNccpwa2boYdOrHUd1pxKLiyO4TafQEETaqBqulJWIQ2dUwgO8sL5G6HDqHtw53olM/jpQ4Nb4U4gPMPo9BJBgJAjDYKK54NGwCWnYCDqv3ukItBg53+Egulq5WmqA8W+sKIoaIDyINfWIQKWEQCVpLS1MBAAdahjFudSBGxeGEwcjhFPB/WxsAAN+8ohDRUeH1/4lBJFjYxsXA4e71GG6eeF2b593hUrgMUHG5OxiV52jx8fEeNqzilCCSxiASrIpTYz3bzvc2DWLFlDSpS6IzeP9YF1oGx6DTROGOBflSl+NzDCJSGmr2bq1t3g7Yzd5r8iixudQdPlImAbLwuScYrtx9IpEeRGwOJ1pdh90xiAQvmUyG5VPS8Mq+Nmyp7WMQCUJOp4CnXKsh9ywpDMvTvcPvIwpmdgvQusvb6zFYP/F6QrZ3a23RlYA6Xpo66ZK5g0jzwCiMZhsSIrRhtXVwFHangFiVAplaHnYXzFZOFoPI1ro+CIIAGX/gCSoba3pxsncE8dFKrF9SIHU5fsEg4m/6Nm/waN4O2Ea912QKIG+Ra9VjNZA2jaseIS4xVoWcxBh0DIt9IouLI/M0YvdtmeK0OH5jC3KLS5KhUsrRMTyOhr4RlKbzB6Bg4XQK+NNm8QfWuxYXhO0PNgwivma3Au17vb0e/bUTr8ele2+3FC0HorllLtyUZ2sZRNioGjI0KiUWFiVj+8l+bKntYxAJIh9V96Cm24g4tRJfXxK+oxgYRHzB2OU9ubbpM8Bq8l6TyYGc+d5Vj4wZXPUIc+U5WnxU3YNjETzY7NQVEQp+KyenYvvJfmyt68O3ruQuvGBgdzjxv5vqAADfWFaIxNjwPYKDQeRSOOxAx37vqkdv9cTrmhRv8ChaAWjC4zwAujDuPpHqCG5YbejnjplQsnJKOv7rvRpUtgxHdG9TMHnrcCea+keRqInCPUvDdzUEYBC5cKZeoOFTMXw0bgUsp36TkQHZFd5R6pmzALlcqkpJYmVZYhBpGRyDYdwGbUxkfVF3OgU09om9UAwioSEvWYOi1Fg09Y9ix8kBXDsjU+qSIprF7sAfPhV7Q+5bXhz2U5r9FkQee+wxfPDBBzhy5AhUKhX0er2/Xso/nA6g86Br1WMj0H104vWYRHGEeukaoHglEBuZvQB0usRYFXKTYtA+NI7jnQYsLomsfxtdhnGM2xyIUsiQl6SRuhy6QCsnp6GpvxlbavsYRCS2YX87OvXjSE9Q42uLCqQux+/8FkSsVituvfVWLFq0CH/729/89TK+NToANGx2rXpsBsaHJ17PnOWdZpo9B5CH13Q78p3ybC3ah8ZxLAKDSGO/uBqSnxyLKAVXBkPFyilp+OvOZmyt64Pd4YSS/+8kMWKx48kt4tyQ/1hZGnZTVM/Eb0HkF7/4BQDgxRdf9NdLXD6nE+g67D1ArvMQAMF7PVorrna4D5CL47AfujDl2Tp8WBWZE1a5YyY0zStMgk4ThaFRK/a3DEXsji+pPb2tAQMjFhQka8LuTJmzCaoeEYvFAovF4vm90Wj0zwt1HgL2PSv2fIwNTLyWXu7dXpszD1AE1VtEIcIzYTUCd85wtHtoilLIsXpqOl4/2IFPqnsYRCTQPjSG53aIx3v8+JqpUCkjY1UqqD7Kxx9/HFqt1vMrN9dPaXBsEDi2QQwhqnhg6nXA9U8CD50A7tsJrPo5kL+IIYQumTuItA2NwTBmk7iawGpkEAlZV5dlAAA+Pt4Dp1M4z6PJ1/7n41pY7U4sKkrG6mnpUpcTMBcVRB555BHIZLJz/qqtrT3/E53Fo48+CoPB4PnV3t5+yc91TgVLgSXfA9a/D/ywCbjtZWDO14CELP+8HkUcrSbK06gZabdnuHU3dC0pSUGcWoleowVHOvRSlxNRDrYO4f1j3ZDJgJ9+YVpETSS+qB/5f/CDH+Cuu+4652OKioouuRi1Wg21Wn3Jf/+CRcUAq3/p/9ehiFaeo0Xb0BiqOg1YWhoZy9xDo1YMjVoBAEWpPCE61ERHKbBiShreO9qFj6t7MCcvUeqSIoLTKeCX79UAAG6bm4tpWQkSVxRYFxVEUlNTkZqa6q9aiMJKebYWHxzrRlWnXupSAsbdH5Kti4FGxVuboWhdWYYniDy6bkpE/WQuldcq23G0w4BYlQIPrZkkdTkB57evFG1tbRgaGkJbWxscDgeOHDkCACgpKUFcHJdsKfzNcDesRtCtGTaqhr4rJ6VCrZSjbWgMNd1GTM/ieVj+NDBiwRMfiS0ND62ZjLT4yDut2m/Nqj/72c8we/Zs/PznP8fIyAhmz56N2bNno7Ky0l8vSRRUpruCSPvQOIZdtyvCHYNI6ItVK3HlJHHl+6OqHomrCX///eEJGMZtmJaZgPWL8qUuRxJ+CyIvvvgiBEE47dfy5cv99ZJEQUUbE4WCZLFhtborMlZF2KgaHtyTVd892gVB4O4Zf9ndOIA3D3VCJgMeu6ksYofIReZHTRQgZa5VkUg5iZdbd8PD6mnp0KgUaBsaw6E2vdTlhKVxqwM/eUs8MPWOBXmYHcGNwQwiRH40IydyTuIds9rRqR8HwKmqoU6jUmLtdHGmyDtHOiWuJjz95pM6NA+MIj1BjYfXTpG6HEkxiBD5USStiNT3iqshKXEqJMaqJK6GLtcNs8S5Su8f64bN4ZS4mvCyr2kQL+wWJ6g+cfOMiDuh+/MYRIj8yB1EOvXjnvka4aqu1wQAmJQeL3El5AtLS1KQEqfC0KgVO+r7pS4nbIxZ7Xj438cgCOLMkBWTeYYZgwiRHyVER6EwRRzsFe7beOsZRMKKUiHHF2aIqyJvHe6SuJrw8av3T6BtaAxZ2mj85AtTpS4nKDCIEPmZ+9yZcO8TqXPdmpmcwSASLm6anQ0A+OR4D/Rj4b2iFwjvH+vCq/vbIJMBv7l1JhKiI/uWjBuDCJGflXv6RPTSFuJnJ3u4IhJuZuRoMSUjHla7E28dZtPq5WgbHMOjb1QBAL6zvBhLSiLj2IcLwSBC5Gflnp0zRokr8R/DmA09RjMAYFI6d8yEC5lMhtvn5wEANuxv50yRS2S1O/EfGw7DZLGjIj8R318VeWPcz4VBhMjPprsOsOrUj2NwxCJxNf5xsk9cDcnSRiOey81h5cZZ2VAr5ajrNeFIu17qckLSf713HEfb9UiIVuJPt8+O2MFlZ8N3g8jP4qOjPCfRhmvDap37tgz7Q8KOVhOFa8rFSasb9rdLXE3o+ee+VryyT+wL+eOXZyNbFyN1SUGHQYQoAMK9YfWka8fMZPaHhKUvz8sFALx3rAuGcZvE1YSO/c1D+Pk7xwEA/7lmMlZM4VbdM2EQIQqA8jAfbHaSW3fD2vzCJExKj8OY1YF/HeCqyIVoGRjFfS8fhN0p4NoZmfjO8mKpSwpaDCJEARDOKyKCIHhuzXDrbniSyWT4+pJCAMCLu1tg56TVc+o3WfC15/djcNSKsuwE/OaWGZDJZFKXFbQYRIgCYHq2FjIZ0GUwYyDMGlYHRqwYHrNBJgOKecZM2LpxdjaSYlXo1I9jY02v1OUErVGLHfe8dABtQ2PITYrB83fNg0allLqsoMYgQhQAcWolisJ0wqr7tkx+kgYxKoXE1ZC/REcpcMcCcSvv8zubJa4mOJltDnzzH5U41mFAUqwKL909H2nx0VKXFfQYRIgCZEaODgBQFWZ9InUcZBYx7lyYjyiFDJWtwzjcNix1OUHFbHPg3r9XYlfDIGJVCvxt/VwUcYXwgjCIEAWI+wC8cF0RYX9I+EtLiMYNs8Sx73/aXC9xNcHDbHPg2y8fxI76AWhUCrz49fmYnZcodVkhg0GEKEBmuCashtuKiDuIlHJFJCI8sKIECrkMW+v6cZQDzmA023DXC/uxra4f0VFyPH/XPMwrSJK6rJDCIEIUINMyEyCTAT1GM/pMZqnL8QlBEHDSfdgdg0hEKEiJxQ2zxFN5I31VpM9kxpef3Yu9TUOIUyvxwl3zsbAoWeqyQg6DCFGAxKqVKHHdMw6XbbxdBjNGLHYo5TIUuppxKfz9x8pSyGXA5tq+sFvhu1AtA6O49Zk9qOk2IiVOhQ3fXIhFxQwhl4JBhCiA3PNEqjrC4wC8uh7x4yhMiYVKyS8nkaIwJRY3unpFnvj4RMQdhrf9ZD+u//NOtA6OIS9Jg39/e7GnB4wuHr9yEAWQ+yTeqk69tIX4SE2XGESmuQ72o8jx/dWToFLIsathEJtP9EldTkAIgoC/7mjCXS/sh9Fsx+w8Hf593yIUcDXwsjCIEAVQeZjtnDnRLTaqTs1kEIk0uUka3LNMnLb62IcnYLWH97RVo9mG7244gv/vgxNwCsCX5uZgwzcXck6IDzCIEAXQtKwEyGVAr9GCPmPoN6ye6HatiDCIRKTvLC9GSpwKzQOj+PueFqnL8ZtDbcO45o878N7RLijkMvz8umn4n5tnQK3kAD9fYBAhCiCNSomSNLFhNdRXRcasdjQPjgLgikikio+Own+umQwA+P2mk+jUj0tckW/ZHU48tbUBtz6zBx3D48hNisHr316Eu5cU8uwYH2IQIQqw8mwdgNA/ibe2xwRBAFLj1UiNV0tdDknk1rm5qMhPxKjVgZ+8VRU2jasnuo344tO78ZtP6uBwCrh+ZhY++O4yzOGgMp9jECEKsPJscfUg1Lfwum/LcDUksinkMvzPzeVQKeTYVtePNw51Sl3SZbHYHfjfjXW47smdONZhQHy0Ev9760z88cuzkBAdJXV5YYlBhCjAyl1nzhwLmyDCQWaRriQtHt9bVQoA+Pk71WgeGJW4okuztbYP6/6wA09uaYDdKWDt9HRsfuhK3FyRw1sxfsQgQhRg0zLFhtV+kwW9Idyw6tm6yxURAvDtK4uxoDAJo1YHHnjlEMw2h9QlXbDG/hHc9cJ+3P3iATQNjCIlTo2n75iDZ++ci7QE7orxNwYRogCLUSk8J9WG6lkdTqeAWtepuwwiBIi3aP745dlIilXheJcRPw6BfpHBEQt++V4N1v5+O7bV9SNKIcM3ryjC1v+8EuvKM6UuL2IwiBBJYKbr9szhEA0ibUNjGLM6oFbKOdqdPDK00fjjl2dBIZfhzUOdeGprg9QlnZFh3Ib/3ViHK369Fc/vaobdKWDllDR88uAV+PE1UxHPXpCAUkpdAFEkqshPxGuV7TjUOix1KZekxtUfMjkjHkoFf54hr2WlqfjlDdPxk7eq8duNJ6HTqPDVhflSlwUAGLHY8Y89rXjms0YYxm0AxCGD/7l2Mq6clCpxdZGLQYRIAnPydQCAox162BxORIXYN3NPo2oGb8vQ6e5YkI+O4XE8va0R/+/taggA7pQwjPSbLHhpdwv+vqcFRrMdAFCaFocfrJmEtdMz2IgqMQYRIgkUpcRBGxMFw7gNtd0mzxk0ocIzUZVnzNBZ/HDtZNgdTjy3oxk/fbsaXfpxPLxmMuTywH3TbxkYxV93NuH1yg5YXCPoi1Ji8cDKEtwwKxuKANZCZ8cgQiQBuVyG2Xk6bKvrx8HWoZALIu4dM5whQmcjk8nw42umIjpKgSe3NODpbY042WPC/9wyAylx/huAZ3c48emJPvxzXyt21A94/nxWrg7fvrIYa6alBzQM0fkxiBBJpCIvEdvq+nGoTY+7lkhdzYUbGrWiyyBuO+YMEToXmUyGH6yZjMKUWDzyRhU21/Zh7e+346dfmIbrZ2b5NBDU95rw7tEu/KuyHb1Gi+v1geWTUvEt19Zi3oIJTgwiRBKZky+Oij4YYg2r7jNyilJiubuALsgX5+RgamYCvv/aEdT2mPDga0fw/K5m3HdlMVZPS7+khmf3FvIttb1472g36npNnmvJsSp8aV4uvjI/D7lJGl9+KOQHDCJEEpmZq4NcBnTqx9FrNCM9RAYnVXXoASDkbieRtKZmJuCdB5bgrzua8X9bG3Csw4D7/nkIGQnRWDs9HVdOTkV5tu6s5xaZzDbU9phQ02XEobZh7GoYwMCI1XM9SiHDlZPScMOsLKyZns6TcUMIgwiRROLUSkzOSMCJbiMOtQ6HzAAl94pIeTaDCF0ctVKB+1eU4Etzc/HS7ha8sr8NPUYzXtrTipf2tAIQPy/S4tWIVSvhFASYbQ70mSwwuXa7nEqjUmBBYRLWlWdi7bQMaDVcoQtFDCJEEqrI14lBpC2EgkgHgwhdntR4Nf5z7WQ8sLIEuxoGsKmmF5Wtw2jsH8GIxY4Ry+mhAwCytNGYmpmA6dlaLClOxuy8RKiUobX1nU7HIEIkoTl5iXh5b1vI9IkMjFjQZTBDJgOmM4jQZYqOUuCqqem4amo6AGDMakePwYw+kwVjVjvkMhlUSjnS4qORnqBmT1KYYhAhklCFq2G1utMIi90R9Pe1T21UjVPzywf5lkalRFFqHIpS46QuhQKIa1pEEspL0iA5VgWrw4nqTqPU5ZxXNW/LEJGPMYgQSUgmk2F2nrgqEgrnzhxzrYiUMYgQkY8wiBBJbF6BGET2twxJXMn5VbuCyAzX6cFERJeLQYRIYguKkgEAB1qG4HQKEldzdv0mC7rdjao8Y4aIfIRBhEhi07MSoFEpoB+z4WSf6fx/QSJH2/UAgOLUOMSyUZWIfIRBhEhiUQq5Z/fM/ubgvT1zuF3sYZmTp5O2ECIKKwwiREFgfkESAGBfEAeRQ616APA01xIR+QKDCFEQcPeJ7GsagiAEX5+IwyngqOuMmdlcESEiH2IQIQoCM3K0UCnlGBixoHlgVOpyTnOy14QxqwNxaiVK0+KlLoeIwgiDCFEQiI5SYFauDkBw9okcahP7Q2bmaqGQyySuhojCCYMIUZBYUCj2iQRjEDncpgcAzM5lfwgR+RaDCFGQWFDo6hMJyiAiroiwP4SIfM1vQaSlpQX33HMPCgsLERMTg+LiYvz85z+H1Wr110sShbQ5+Too5TJ06sfRPjQmdTke+jErGvvFvhXumCEiX/PbVKLa2lo4nU48++yzKCkpQXV1Ne69916Mjo7it7/9rb9elihkaVRKzMjR4lCbHnsaB5GbpJG6JADAEdcgs4JkDZJiVdIWQ0Rhx29B5Oqrr8bVV1/t+X1RURHq6urw9NNPM4gQncXSkhQcatNjZ8MAvjQvV+pyAHgP4+NqCBH5Q0B7RAwGA5KSks563WKxwGg0TvhFFEkWl6QAAHY3DgTNPJG9rp6V+YVn/9wlIrpUAQsiDQ0NePLJJ/Gtb33rrI95/PHHodVqPb9yc4PjJ0KiQJmdp0NMlAIDI1bU9Up/7ozZ5vDcmmEQISJ/uOgg8sgjj0Amk53zV21t7YS/09nZiauvvhq33nor7r333rM+96OPPgqDweD51d7efvEfEVEIUysVnm/4O+sHJK5GPOjOanciJU6NopRYqcshojB00T0iP/jBD3DXXXed8zFFRUWe/+7q6sKKFSuwePFi/OUvfznn31Or1VCr1RdbElFYWVqSgs9O9mNXwwC+sazo/H/Bj9xbiRcUJUEm4yAzIvK9iw4iqampSE1NvaDHdnZ2YsWKFaioqMALL7wAuZxjS4jOZ4mrT2Rf8xCsdidUSuk+b/Y1DwIAFvK2DBH5id++wnV2dmL58uXIy8vDb3/7W/T396Onpwc9PT3+ekmisDAlIx5JsSqMWb39GVKw2p046Nox4z6Uj4jI1/y2fXfTpk1oaGhAQ0MDcnJyJlwLlt0ARMFILpdhcXEy3j/Wje0n+yVrEq3qNMBscyJRE4WS1DhJaiCi8Oe3FZG77roLgiCc8RcRndvyyWkAgK11fZLV4L4tM78wCXIedEdEfsKmDaIgtHxyKmQy4HiXEb1GsyQ17G1yNaoW8rYMEfkPgwhREEqJU2NGjg4AsLU28KsiZpsD+10rIu7mWSIif2AQIQpSK123Z7ZIEEQqW4ZhtjmRnqDGpHT2hxCR/zCIEAWplVPEILKzYQAWuyOgr729vh8AsKw0lfNDiMivGESIgtT0rASkxqsxZnVgv2uwWKBsPykGkSsmXdjMICKiS8UgQhSk5HIZVkwWg0Agb8/0Gc2o7TFBJhOnvBIR+RODCFEQu2pqOgBg4/HegG193+4646Y8W4ukWFVAXpOIIheDCFEQu3JSKjQqBTr14zjaYQjIa7p36VxRytsyROR/DCJEQSw6SuFpWv2oqtvvr2exO7DNNURt9bR0v78eERGDCFGQu7Y8EwDwQVW332/P7G4cxKjVgfQENcqztX59LSIigEGEKOgtn5yGmCgFOobHUdXp39szG4/3AhBXQzjWnYgCgUGEKMjFqLy3Zz7w4+0Zp1PApyfcQSTDb69DRHQqBhGiEHCN6/bMh368PXOkQ49+kwXxaiUWFfF8GSIKDAYRohCwYkoqYlUKtA+N40DLsF9e490jXa7XSoNKyS8NRBQY/GpDFAI0KqVnVeT1ynafP7/d4cT7x8QgcuPsLJ8/PxHR2TCIEIWIW+fmAhD7REYtdp8+986GAQyMWJEUq8Iyzg8hogBiECEKEfMKElGQrMGY1YGPqnt8+tzvuG7LfGFGJqIU/LJARIHDrzhEIUImk+GWihwAwIb9bT573lGLHZ8cF4PNjbOzffa8REQXgkGEKITcOjcXUQoZKluHcaxD75PnfOdIF8asDhQkazA7V+eT5yQiulAMIkQhJD0hGl+YITaTPr+z+bKfTxAE/H1PCwDgqwvzIZNxiBkRBRaDCFGI+fqSQgDA+8e60Ws0X9ZzHWobRm2PCdFRctxakeuL8oiILgqDCFGIKc/RYn5BEuxOAc/vurxVkWc/awIAXD8zC1pNlC/KIyK6KAwiRCHom1cUAQD+vrsV/SbLJT1HXY8JG2t6IZN5n4+IKNAYRIhC0FVT0zAzV4dxmwNPb2u8pOd4amsDAODq6RkoSYv3ZXlERBeMQYQoBMlkMvxg9SQAwMt7W9HUP3JRf/9oux7vHhVnh9y/osTn9RERXSgGEaIQtaw0Bcsnp8LqcOJn7xy/4MPwBEHAL9+vAQB8cU42yrK1/iyTiOicGESIQpRMJsMvrp8OlVKOnQ0D2HDgws6g+cfeVhxsHUZMlAI/XDvFz1USEZ0bgwhRCMtPjvXcovmvd4/jRLfxnI+v7THisQ9OAAB+dPVkZGij/V4jEdG5MIgQhbh7lxXhykmpsNiduOuF/WgdHD3j49qHxnDX8wdgsTtx5aRUrF9cENhCiYjOgEGEKMTJ5TL84bZZmJwej16jBbc+swe7GwYmPOZg6xBue3YPeoxmlKbF4Q+3zeIUVSIKCjLhQjvcJGA0GqHVamEwGJCQkCB1OURBrd9kwVf/ug91vSYAwJw8HSZnxKOpfxT7mocAAEUpsXjl3oW8JUNEfnUx378ZRIjCyKjFjic+qsUr+9vgcHo/tWUy4JY5OfjpddOQEM0JqkTkXwwiRBGux2DGtro+9BjNSIlTY/nkVOQkaqQui4gixMV8/1YGqCYiCqAMbTS+PD9P6jKIiM6LzapEREQkGQYRIiIikgyDCBEREUmGQYSIiIgkwyBCREREkmEQISIiIskwiBAREZFkGESIiIhIMgwiREREJBkGESIiIpIMgwgRERFJhkGEiIiIJMMgQkRERJIJ6tN3BUEAIB4nTERERKHB/X3b/X38XII6iJhMJgBAbm6uxJUQERHRxTKZTNBqted8jEy4kLgiEafTia6uLsTHx0Mmk/n0uY1GI3Jzc9He3o6EhASfPjd58X0ODL7PgcH3OTD4PgeOv95rQRBgMpmQlZUFufzcXSBBvSIil8uRk5Pj19dISEjgP/QA4PscGHyfA4Pvc2DwfQ4cf7zX51sJcWOzKhEREUmGQYSIiIgkE7FBRK1W4+c//znUarXUpYQ1vs+Bwfc5MPg+Bwbf58AJhvc6qJtViYiIKLxF7IoIERERSY9BhIiIiCTDIEJERESSYRAhIiIiyURkEHnqqadQUFCA6OhoLFiwAPv375e6pJDy+OOPY968eYiPj0daWhpuvPFG1NXVTXiM2WzG/fffj+TkZMTFxeHmm29Gb2/vhMe0tbXh2muvhUajQVpaGh5++GHY7fZAfigh5YknnoBMJsODDz7o+TO+z77R2dmJr371q0hOTkZMTAzKy8tRWVnpuS4IAn72s58hMzMTMTExWLVqFerr6yc8x9DQEO644w4kJCRAp9PhnnvuwcjISKA/lKDlcDjw05/+FIWFhYiJiUFxcTF+9atfTTiLhO/zpdm+fTuuu+46ZGVlQSaT4e23355w3Vfv67Fjx7Bs2TJER0cjNzcXv/71r33zAQgRZsOGDYJKpRKef/554fjx48K9994r6HQ6obe3V+rSQsbatWuFF154QaiurhaOHDkiXHPNNUJeXp4wMjLiecy3v/1tITc3V9i8ebNQWVkpLFy4UFi8eLHnut1uF8rKyoRVq1YJhw8fFj788EMhJSVFePTRR6X4kILe/v37hYKCAmHGjBnC9773Pc+f832+fENDQ0J+fr5w1113Cfv27ROampqETz75RGhoaPA85oknnhC0Wq3w9ttvC0ePHhWuv/56obCwUBgfH/c85uqrrxZmzpwp7N27V9ixY4dQUlIi3H777VJ8SEHpscceE5KTk4X3339faG5uFl5//XUhLi5O+OMf/+h5DN/nS/Phhx8KP/nJT4Q333xTACC89dZbE6774n01GAxCenq6cMcddwjV1dXCq6++KsTExAjPPvvsZdcfcUFk/vz5wv333+/5vcPhELKysoTHH39cwqpCW19fnwBA+OyzzwRBEAS9Xi9ERUUJr7/+uucxJ06cEAAIe/bsEQRB/MSRy+VCT0+P5zFPP/20kJCQIFgslsB+AEHOZDIJpaWlwqZNm4Qrr7zSE0T4PvvGj370I2Hp0qVnve50OoWMjAzhN7/5jefP9Hq9oFarhVdffVUQBEGoqakRAAgHDhzwPOajjz4SZDKZ0NnZ6b/iQ8i1114rfP3rX5/wZ1/84heFO+64QxAEvs++8vkg4qv39f/+7/+ExMTECV83fvSjHwmTJ0++7Joj6taM1WrFwYMHsWrVKs+fyeVyrFq1Cnv27JGwstBmMBgAAElJSQCAgwcPwmazTXifp0yZgry8PM/7vGfPHpSXlyM9Pd3zmLVr18JoNOL48eMBrD743X///bj22msnvJ8A32dfeffddzF37lzceuutSEtLw+zZs/Hcc895rjc3N6Onp2fC+6zVarFgwYIJ77NOp8PcuXM9j1m1ahXkcjn27dsXuA8miC1evBibN2/GyZMnAQBHjx7Fzp07sW7dOgB8n/3FV+/rnj17cMUVV0ClUnkes3btWtTV1WF4ePiyagzqQ+98bWBgAA6HY8IXZQBIT09HbW2tRFWFNqfTiQcffBBLlixBWVkZAKCnpwcqlQo6nW7CY9PT09HT0+N5zJn+P7ivkWjDhg04dOgQDhw4cNo1vs++0dTUhKeffhoPPfQQfvzjH+PAgQP47ne/C5VKhfXr13vepzO9j6e+z2lpaROuK5VKJCUl8X12eeSRR2A0GjFlyhQoFAo4HA489thjuOOOOwCA77Of+Op97enpQWFh4WnP4b6WmJh4yTVGVBAh37v//vtRXV2NnTt3Sl1K2Glvb8f3vvc9bNq0CdHR0VKXE7acTifmzp2L//7v/wYAzJ49G9XV1XjmmWewfv16iasLH//617/wz3/+E6+88gqmT5+OI0eO4MEHH0RWVhbf5wgXUbdmUlJSoFAoTttV0Nvbi4yMDImqCl0PPPAA3n//fWzduhU5OTmeP8/IyIDVaoVer5/w+FPf54yMjDP+f3BfI/HWS19fH+bMmQOlUgmlUonPPvsMf/rTn6BUKpGens732QcyMzMxbdq0CX82depUtLW1AfC+T+f6upGRkYG+vr4J1+12O4aGhvg+uzz88MN45JFH8OUvfxnl5eW488478f3vfx+PP/44AL7P/uKr99WfX0siKoioVCpUVFRg8+bNnj9zOp3YvHkzFi1aJGFloUUQBDzwwAN46623sGXLltOW6yoqKhAVFTXhfa6rq0NbW5vnfV60aBGqqqom/OPftGkTEhISTvumEKmuuuoqVFVV4ciRI55fc+fOxR133OH5b77Pl2/JkiWnbT8/efIk8vPzAQCFhYXIyMiY8D4bjUbs27dvwvus1+tx8OBBz2O2bNkCp9OJBQsWBOCjCH5jY2OQyyd+y1EoFHA6nQD4PvuLr97XRYsWYfv27bDZbJ7HbNq0CZMnT76s2zIAInP7rlqtFl588UWhpqZG+OY3vynodLoJuwro3O677z5Bq9UK27ZtE7q7uz2/xsbGPI/59re/LeTl5QlbtmwRKisrhUWLFgmLFi3yXHdvK12zZo1w5MgR4eOPPxZSU1O5rfQ8Tt01Iwh8n31h//79glKpFB577DGhvr5e+Oc//yloNBrh5Zdf9jzmiSeeEHQ6nfDOO+8Ix44dE2644YYzbn+cPXu2sG/fPmHnzp1CaWlpxG8rPdX69euF7Oxsz/bdN998U0hJSRF++MMfeh7D9/nSmEwm4fDhw8Lhw4cFAMLvfvc74fDhw0Jra6sgCL55X/V6vZCeni7ceeedQnV1tbBhwwZBo9Fw++6levLJJ4W8vDxBpVIJ8+fPF/bu3St1SSEFwBl/vfDCC57HjI+PC9/5zneExMREQaPRCDfddJPQ3d094XlaWlqEdevWCTExMUJKSorwgx/8QLDZbAH+aELL54MI32ffeO+994SysjJBrVYLU6ZMEf7yl79MuO50OoWf/vSnQnp6uqBWq4WrrrpKqKurm/CYwcFB4fbbbxfi4uKEhIQE4e677xZMJlMgP4ygZjQahe9973tCXl6eEB0dLRQVFQk/+clPJmwH5ft8abZu3XrGr8nr168XBMF37+vRo0eFpUuXCmq1WsjOzhaeeOIJn9QvE4RTxtoRERERBVBE9YgQERFRcGEQISIiIskwiBAREZFkGESIiIhIMgwiREREJBkGESIiIpIMgwgRERFJhkGEiIiIJMMgQkRERJJhECEiIiLJMIgQERGRZBhEiIiISDL/P3ek0bU0xmS+AAAAAElFTkSuQmCC", 444 | "text/plain": [ 445 | "
" 446 | ] 447 | }, 448 | "metadata": {}, 449 | "output_type": "display_data" 450 | } 451 | ], 452 | "source": [ 453 | "import matplotlib.pyplot as plt\n", 454 | "plt.plot(x[:, 0])\n", 455 | "plt.plot(x[:, 1], label='ok')\n", 456 | "plt.legend()\n", 457 | "plt.show()" 458 | ] 459 | }, 460 | { 461 | "cell_type": "code", 462 | "execution_count": 8, 463 | "metadata": { 464 | "id": "Ca6C7xDQ6GDj" 465 | }, 466 | "outputs": [ 467 | { 468 | "ename": "KeyboardInterrupt", 469 | "evalue": "", 470 | "output_type": "error", 471 | "traceback": [ 472 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 473 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 474 | "Cell \u001b[0;32mIn[8], line 23\u001b[0m\n\u001b[1;32m 21\u001b[0m test_relative_l2 \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m step, (input_batch, output_batch) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(testing_set):\n\u001b[0;32m---> 23\u001b[0m output_pred_batch \u001b[38;5;241m=\u001b[39m \u001b[43mfno\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_batch\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m 24\u001b[0m loss_f \u001b[38;5;241m=\u001b[39m (torch\u001b[38;5;241m.\u001b[39mmean((output_pred_batch \u001b[38;5;241m-\u001b[39m output_batch) \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m \u001b[38;5;241m2\u001b[39m) \u001b[38;5;241m/\u001b[39m torch\u001b[38;5;241m.\u001b[39mmean(output_batch \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m \u001b[38;5;241m2\u001b[39m)) \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m \u001b[38;5;241m0.5\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m100\u001b[39m\n\u001b[1;32m 25\u001b[0m test_relative_l2 \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss_f\u001b[38;5;241m.\u001b[39mitem()\n", 475 | "File \u001b[0;32m~/opt/anaconda3/envs/dlsc/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", 476 | "Cell \u001b[0;32mIn[2], line 120\u001b[0m, in \u001b[0;36mFNO1d.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;66;03m# x = F.pad(x, [0, self.padding]) # pad the domain if input is non-periodic\u001b[39;00m\n\u001b[1;32m 119\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfourier_layer(x, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mspect1, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlin0)\n\u001b[0;32m--> 120\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfourier_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspect2\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlin1\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 121\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfourier_layer(x, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mspect3, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlin2)\n\u001b[1;32m 123\u001b[0m \u001b[38;5;66;03m# x = x[..., :-self.padding] # pad the domain if input is non-periodic\u001b[39;00m\n", 477 | "Cell \u001b[0;32mIn[2], line 107\u001b[0m, in \u001b[0;36mFNO1d.fourier_layer\u001b[0;34m(self, x, spectral_layer, conv_layer)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfourier_layer\u001b[39m(\u001b[38;5;28mself\u001b[39m, x, spectral_layer, conv_layer):\n\u001b[0;32m--> 107\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mactivation(\u001b[43mspectral_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m+\u001b[39m conv_layer(x))\n", 478 | "File \u001b[0;32m~/opt/anaconda3/envs/dlsc/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", 479 | "Cell \u001b[0;32mIn[2], line 58\u001b[0m, in \u001b[0;36mSpectralConv1d.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 52\u001b[0m batchsize \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 53\u001b[0m \u001b[38;5;66;03m# x.shape == [batch_size, in_channels, number of grid points]\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;66;03m# hint: use torch.fft library torch.fft.rfft\u001b[39;00m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;66;03m# use DFT to approximate the fourier transform\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# Compute Fourier coefficients\u001b[39;00m\n\u001b[0;32m---> 58\u001b[0m x_ft \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfft\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrfft\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;66;03m# Multiply relevant Fourier modes\u001b[39;00m\n\u001b[1;32m 61\u001b[0m out_ft \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mzeros(batchsize, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mout_channels, x\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m, device\u001b[38;5;241m=\u001b[39mx\u001b[38;5;241m.\u001b[39mdevice, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mcfloat)\n", 480 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 481 | ] 482 | } 483 | ], 484 | "source": [ 485 | "optimizer = Adam(fno.parameters(), lr=learning_rate, weight_decay=1e-5)\n", 486 | "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n", 487 | "\n", 488 | "l = torch.nn.MSELoss()\n", 489 | "freq_print = 1\n", 490 | "for epoch in range(epochs):\n", 491 | " train_mse = 0.0\n", 492 | " for step, (input_batch, output_batch) in enumerate(training_set):\n", 493 | " optimizer.zero_grad()\n", 494 | " output_pred_batch = fno(input_batch).squeeze(2)\n", 495 | " loss_f = l(output_pred_batch, output_batch)\n", 496 | " loss_f.backward()\n", 497 | " optimizer.step()\n", 498 | " train_mse += loss_f.item()\n", 499 | " train_mse /= len(training_set)\n", 500 | "\n", 501 | " scheduler.step()\n", 502 | "\n", 503 | " with torch.no_grad():\n", 504 | " fno.eval()\n", 505 | " test_relative_l2 = 0.0\n", 506 | " for step, (input_batch, output_batch) in enumerate(testing_set):\n", 507 | " output_pred_batch = fno(input_batch).squeeze(2)\n", 508 | " loss_f = (torch.mean((output_pred_batch - output_batch) ** 2) / torch.mean(output_batch ** 2)) ** 0.5 * 100\n", 509 | " test_relative_l2 += loss_f.item()\n", 510 | " test_relative_l2 /= len(testing_set)\n", 511 | "\n", 512 | " if epoch % freq_print == 0: print(\"######### Epoch:\", epoch, \" ######### Train Loss:\", train_mse, \" ######### Relative L2 Test Norm:\", test_relative_l2)\n" 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": null, 518 | "metadata": { 519 | "id": "PWJh6uI76V3U" 520 | }, 521 | "outputs": [], 522 | "source": [ 523 | "idx_data = 134\n", 524 | "input_function_test_n = input_function_test[idx_data, :].unsqueeze(0)\n", 525 | "output_function_test_n = output_function_test[idx_data, :].unsqueeze(0)\n", 526 | "print(input_function_test_n.shape)\n", 527 | "print(output_function_test_n.shape)\n", 528 | "\n", 529 | "output_function_test_pred_n = fno(input_function_test_n)\n", 530 | "print(output_function_test_pred_n.shape)\n", 531 | "print(input_function_test_n[0,:,1])\n", 532 | "plt.figure(dpi=250)\n", 533 | "plt.grid(True, which=\"both\", ls=\":\")\n", 534 | "plt.plot(input_function_test_n[0,:,1].detach(), output_function_test_n[0].detach(), label=\"True Solution\", c=\"C0\", lw=2)\n", 535 | "plt.scatter(input_function_test_n[0,:,1].detach(), output_function_test_pred_n[0].detach(), label=\"Approximate Solution\", s=8, c=\"C0\")\n", 536 | "p = 2\n", 537 | "err = (torch.mean(abs(output_function_test_n.detach().reshape(-1, ) - output_function_test_pred_n.detach().reshape(-1, )) ** p) / torch.mean(abs(output_function_test_n.detach()) ** p)) ** (1 / p) * 100\n", 538 | "print(\"Relative L2 error: \", err.item())\n", 539 | "plt.legend()" 540 | ] 541 | }, 542 | { 543 | "cell_type": "code", 544 | "execution_count": null, 545 | "metadata": { 546 | "id": "OGYUM9Mc_96p" 547 | }, 548 | "outputs": [], 549 | "source": [ 550 | "######################### TO DO ####################################\n", 551 | "# evaluate drop in performance when the skip connection is removed\n", 552 | "# aka the \" + conv_layer(x)\"\n", 553 | "####################################################################" 554 | ] 555 | } 556 | ], 557 | "metadata": { 558 | "colab": { 559 | "provenance": [] 560 | }, 561 | "kernelspec": { 562 | "display_name": "Python 3 (ipykernel)", 563 | "language": "python", 564 | "name": "python3" 565 | }, 566 | "language_info": { 567 | "codemirror_mode": { 568 | "name": "ipython", 569 | "version": 3 570 | }, 571 | "file_extension": ".py", 572 | "mimetype": "text/x-python", 573 | "name": "python", 574 | "nbconvert_exporter": "python", 575 | "pygments_lexer": "ipython3", 576 | "version": "3.8.17" 577 | } 578 | }, 579 | "nbformat": 4, 580 | "nbformat_minor": 1 581 | } 582 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 mroberto166 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CAMLab-DLSCTutorials 2 | The repository is a collection of python notebooks and scripts related to the lectures available at https://www.youtube.com/@CAMLabETHZurich/ 3 | 4 | The new 2024 course webpage can be found at https://camlab.ethz.ch/teaching/deep-learning-in-scientific-computing-2023.html 5 | 6 | The scripts are based on Python 3.8 and Jupyter Notrbook. 7 | The required packages can be installed with: 8 | 9 | 10 | python3 -m pip install -r requirements.txt 11 | 12 | 13 | The jupyter notebooks can be run with: 14 | 15 | python3 jupyter notebook name_of_the_file.ipynb 16 | -------------------------------------------------------------------------------- /antiderivative_aligned_test.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mroberto166/CAMLab-DLSCTutorials/c2ca37b6e711e3691ed95f421ffc10b6d1ad1a6f/antiderivative_aligned_test.npz -------------------------------------------------------------------------------- /antiderivative_aligned_train.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mroberto166/CAMLab-DLSCTutorials/c2ca37b6e711e3691ed95f421ffc10b6d1ad1a6f/antiderivative_aligned_train.npz -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | anyio==3.7.0 2 | appnope==0.1.3 3 | argon2-cffi==21.3.0 4 | argon2-cffi-bindings==21.2.0 5 | arrow==1.2.3 6 | asttokens==2.2.1 7 | attrs==23.1.0 8 | backcall==0.2.0 9 | beautifulsoup4==4.12.2 10 | bleach==6.0.0 11 | cachetools==5.3.1 12 | certifi==2023.5.7 13 | cffi==1.15.1 14 | charset-normalizer==3.1.0 15 | comm==0.1.3 16 | contourpy==1.1.0 17 | cycler==0.11.0 18 | debugpy==1.6.7 19 | decorator==5.1.1 20 | DeepXDE==1.9.1 21 | defusedxml==0.7.1 22 | exceptiongroup==1.1.1 23 | executing==1.2.0 24 | fastjsonschema==2.17.1 25 | filelock==3.12.2 26 | fonttools==4.40.0 27 | fqdn==1.5.1 28 | google==3.0.0 29 | google-api-core==2.11.1 30 | google-api-python-client==2.90.0 31 | google-auth==2.20.0 32 | google-auth-httplib2==0.1.0 33 | googleapis-common-protos==1.59.1 34 | httplib2==0.22.0 35 | idna==3.4 36 | importlib-metadata==6.7.0 37 | importlib-resources==5.12.0 38 | ipykernel==6.23.2 39 | ipython==8.12.2 40 | ipython-genutils==0.2.0 41 | ipywidgets==8.0.6 42 | isoduration==20.11.0 43 | jedi==0.18.2 44 | Jinja2==3.1.2 45 | joblib==1.2.0 46 | jsonpointer==2.4 47 | jsonschema==4.17.3 48 | jupyter==1.0.0 49 | jupyter-console==6.6.3 50 | jupyter-events==0.6.3 51 | jupyter_client==8.2.0 52 | jupyter_core==5.3.1 53 | jupyter_server==2.6.0 54 | jupyter_server_terminals==0.4.4 55 | jupyterlab-pygments==0.2.2 56 | jupyterlab-widgets==3.0.7 57 | kiwisolver==1.4.4 58 | MarkupSafe==2.1.3 59 | matplotlib==3.7.1 60 | matplotlib-inline==0.1.6 61 | mistune==3.0.1 62 | mpmath==1.3.0 63 | nbclassic==1.0.0 64 | nbclient==0.8.0 65 | nbconvert==7.6.0 66 | nbformat==5.9.0 67 | nest-asyncio==1.5.6 68 | networkx==3.1 69 | notebook==6.5.4 70 | notebook_shim==0.2.3 71 | numpy==1.24.3 72 | overrides==7.3.1 73 | packaging==23.1 74 | pandas==2.0.2 75 | pandocfilters==1.5.0 76 | parso==0.8.3 77 | pexpect==4.8.0 78 | pickleshare==0.7.5 79 | Pillow==9.5.0 80 | pkgutil_resolve_name==1.3.10 81 | platformdirs==3.7.0 82 | prometheus-client==0.17.0 83 | prompt-toolkit==3.0.38 84 | protobuf==4.23.3 85 | psutil==5.9.5 86 | ptyprocess==0.7.0 87 | pure-eval==0.2.2 88 | pyaml==23.5.9 89 | pyasn1==0.5.0 90 | pyasn1-modules==0.3.0 91 | pycparser==2.21 92 | Pygments==2.15.1 93 | pyparsing==3.1.0 94 | pyrsistent==0.19.3 95 | python-dateutil==2.8.2 96 | python-json-logger==2.0.7 97 | pytz==2023.3 98 | PyYAML==6.0 99 | pyzmq==25.1.0 100 | qtconsole==5.4.3 101 | QtPy==2.3.1 102 | requests==2.31.0 103 | rfc3339-validator==0.1.4 104 | rfc3986-validator==0.1.1 105 | rsa==4.9 106 | scikit-learn==1.2.2 107 | scikit-optimize==0.9.0 108 | scipy==1.10.1 109 | seaborn==0.12.2 110 | Send2Trash==1.8.2 111 | six==1.16.0 112 | sklearn==0.0.post5 113 | sniffio==1.3.0 114 | soupsieve==2.4.1 115 | stack-data==0.6.2 116 | sympy==1.12 117 | terminado==0.17.1 118 | threadpoolctl==3.1.0 119 | tinycss2==1.2.1 120 | torch==2.0.1 121 | torchaudio==2.0.2 122 | torchvision==0.15.2 123 | tornado==6.3.2 124 | traitlets==5.9.0 125 | typing_extensions==4.6.3 126 | tzdata==2023.3 127 | uri-template==1.3.0 128 | uritemplate==4.1.1 129 | urllib3==1.26.16 130 | wcwidth==0.2.6 131 | webcolors==1.13 132 | webencodings==0.5.1 133 | websocket-client==1.6.0 134 | widgetsnbextension==4.0.7 135 | zipp==3.15.0 136 | --------------------------------------------------------------------------------