├── .gitignore ├── 01-SGM-without-SDE.ipynb ├── 02-SGM-with-SDE.ipynb ├── 03-SGM-with-SDE-MNIST.ipynb ├── LICENSE ├── README.md ├── figs ├── generated-mnist.png └── sampling-swiss-sde.gif └── viscode ├── 01-sample-anim.py ├── 02-sample-right.py ├── 02-sample-sde-anim-swiss.py ├── 02-sample-sde-anim.py └── 02-sample-wrong.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /01-SGM-without-SDE.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "910f1769", 6 | "metadata": {}, 7 | "source": [ 8 | "This is the notebook to show the implementation of score-based generative model (1st part of the tutorial). In this case, we will sample the training data from the swiss roll distribution.\n", 9 | "From the training data, we will try to learn how to draw new samples from the swiss roll distribution with Score-based Generative Model (SGM)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "id": "3084c1e6", 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import torch\n", 20 | "from sklearn.datasets import make_swiss_roll\n", 21 | "\n", 22 | "# generate the swiss roll dataset\n", 23 | "xnp, _ = make_swiss_roll(1000, noise=1.0)\n", 24 | "xtns = torch.as_tensor(xnp[:, [0, 2]] / 10.0, dtype=torch.float32)\n", 25 | "dset = torch.utils.data.TensorDataset(xtns)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "id": "685e64d8", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "text/plain": [ 37 | "[]" 38 | ] 39 | }, 40 | "execution_count": 2, 41 | "metadata": {}, 42 | "output_type": "execute_result" 43 | }, 44 | { 45 | "data": { 46 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABYyUlEQVR4nO29e3Rc1Zkv+PtOleTgtLAVP7CNLIEwOETihmsJYzfcBhJIhywHEhvCa900t+OYrCFrVqbvrNW5ScfDck960j3T65Lb4Q44nqx07sLGsWWeA93BPByctmyrNCGWAL9kSy7LDyTKsoKNJdXZ88epfWqfXfu86nlUtX9rgVVVp86r9vn2t7/v9/0+YoxBQ0NDQ6M2YFT6BDQ0NDQ0ygdt9DU0NDRqCNroa2hoaNQQtNHX0NDQqCFoo6+hoaFRQ4hX+gS8MHfuXHbVVVdV+jQ0NDQ0pg0SicQIY2ye2+eRNvpXXXUVenp6Kn0aGhoaGtMGRDTo9bkO72hoaGjUELTR19DQ0KghaKOvoaGhUUPQRl9DQ0OjhqCNvoaGhkYNQRt9DQ0NjRqCNvoaGmVCYjCFp946gsRgqtKnolHDKApPn4h+AWAVgLOMsXbF57cDeBHAscxbOxhjG4pxbA2NSiIxmEL3wChWtM5BR0uj53aPbOrGxJSJ+riBZ9eu8NxeQ6NUKFZx1i8B/AzArzy2eYcxtqpIx9PQqDjCGPLugVFMTJkwGTA5ZaJ7YFQbfY2KoCjhHcbYbwF8VIx9aWhMF4iGfGLKxJM7D7mGbla0zkF93ECMgLq4gRWtc8p8thoaFsopw7CSiN4FMAzgf2WM9as2IqJ1ANYBQHNzcxlPT0MjHLgh54Z/9+ER7D/+kdLj72hpxLNrVwQKBWlolBLlSuT2AmhhjH0ewD8BeMFtQ8bYRsZYJ2Osc948V80gDY2ywC/5umZZE264chYIAEM2dKNCR0sjHr9jCQDohK5GxVAWT58xdl74+1Ui+u9ENJcxNlKO42toAMGTruL2qph9YjCFrt4ktieSmEqbiMcM1MUIaZP5hm6iktANey80qgdlMfpEtADAGcYYI6LlsFYYandIQ6MEyMfYqpKvAPDIpm5cmjTBMtul0yYeXN6MRbMv8zWi4j4vTZro6k2W3ehGZeLRqAyKEt4hoi0A9gBYSkRJIvoWEX2HiL6T2eQ+AH2ZmP5/A/AgY4y57U9Do9hwM+BeUCVf+X744CVYn61e1oTH71jiazxXtM5B3CAAVjhoeyJZsjCPW2gqn3uhUT0oiqfPGHvI5/OfwaJ0amhUBNyAT06ZgdkzbslXvp+YQbi/czFWL2sK7Cl3tDTi/s7F2Lx3CAzWKqEQ+qZbmMbLm8/nXmhUDyLdREVDo1CIRjEf9kxHS6Nj20JZOInBFBis1UE6XZjR9TLsXnUBmklU29BGX6NqoTKKnD1TCOSJwO3YslFNDKbw0MY9mEwzxAzgweXNvqsEr4Srl2H38+aDXINGdUIbfY2qRaWqYDfvHcL6F/uQNhlm1GU98K7eJCbSVjZgyrRi+oVIN3gZdi9vXjN3ahva6GtULQqJXedrGBODKax/sQ9TpmXcJyazkw1J25L0PfF4icEUntx5yHPS8gvTqLx5zdzR0EZfo2qRb+y6EMPYPTAKUyCmGQZhRescm0ETjxHSaWYzflTHW7+qDRte6bdpoYaHdEPYMI3WANLQRl+jqpFP7DqMYZQ9dFGawSDChnvbcfD0uB3uMQzCXZ+7Ao/ddo1r0vW1vlM2LdQAcMuSufjendcVxThr5o6GNvoaGhKCGkbRQ48L9E1xdQEADzyzxw73pE2GNz84i8duu8b1eHe3L8T+4x/Zr4tl8AHN3NEAKMo1Up2dnaynp6fSp6FRg/Div/P3uwdG8Y+/OYiMPQcBjsQtYGns/F//ehDiU2YA+M9/vtTBJFLF9PMJS3X1JkFAqNqBfKETwtEEESUYY51un2tPX0NDgSBJ0PWr2lAfN+zYuyi4JlInZ9RltyEA9XX+FMqwYSlOB+XsoG2JJLZ8u3RJWp0Qnr7QRl+jJpGPl9rVm7SN9+SUidSFCTy7doUtvqYqthLDKY0z65G6MFESz7h7YBST6ex6otRJWp0Qnr7QRl+j5pCPl5oYTGF7ImmHaWIZVg73yNcsawpFnSw2VrTOQV2MbE+/1ElanRCevtBGX6PmkI+X2j0wiqm0CcAK0dzfubigcEyx0dHSiC3rVoaK6RcSk9cJ4ekLbfQ1ag75eKnydzjHPh+UIgHK97kmYAI335i8fO7FEIoDoCePMkIbfY2ag8pL9TPExfJsS5EALVavALf7wN9rnFmPDa/0F3zuDqprzAAYw5TJdEK4TNBGX6MmIXqpQY1mMUI4pUiA5rNP1WpHdR8A2O8ZZHUHU7GU/CBTXcXzBdTMJ43SQBt9jZpHOZkopUiAFqtXwFNvHVE2V+HvAQwxg8CYui1kUG3/9avaEI9Z5xuPEQgI1GpSozjQRl+j5lFOJkopEqBB96mKxwdR7RTfW7+qTUk79Vot7ehN4pNJy6OfmDLRPzwGk1krBpMxfPGzV2BuwwzPfIQuBCsetNHXqHmUm4lSCqaP3z6DhLDc7kOQe+OVI9jac8LezmTA4TPjmEpzWQrgN++dQX3cwBqX5LguBCsutNHX0EDlKZelRtAQluo+BLk3bquE7oFRpNNOqZdLmTi+iIkpE8/sOoqN38xVDxDPfULH/QuGNvoaGjWAUoew+CphR2/SoTO0onUO6jKqowBQHyM8cFMz3j/VZxeScbzx/hkkBlM5LKLGmfW2vpHJgMaZ9UU991qDNvoaGjWAcoWwunqTmJgysaM3aYdhtnw7OxnwuP3SBQ3o6k3iuX1DtkFngJ08FsM5q5c1gTKfGwBSFyZKcu61Am30NTRqBEHCNIV0DHty5yGHNlFXb9Le14+/foPyXNoXzcL6F/tgMounr6J0cvVSLflQHGijr6GhAaCwKl3+Pe6NxwzC9kQSU2nvfT18czOWLmjImWjk6ufVHtpGGuGgjb6GhgaA/OsVxO8ZZHX6av7MTGzJhG789iV2EOOvVaEobeyLA230NTQyqHUueL7JXrtF5KRVtXt3+0I7Zi9X/AYp3uKrglr8DcqBohh9IvoFgFUAzjLG2hWfE4CfAvgKgAsAHmWM9Rbj2BoaxYDmgudq/4uet9/31q9qs2PzG17px7NrV+S0jXS7v4VWRNf6ZB0WxfL0fwngZwB+5fL53QCuzfx3M4D/O/OvhkZeKPaDrpuCWODXHHYCTF2YgMmY4/49fscSm3654eV+uypX1V0sXzqpW5/iWvztgqIoRp8x9lsiuspjk3sB/IpZDXm7iWg2ES1kjJ0qxvE1agul8Mplw9M4sx5PvXWkJr3HYgm4AbltHDlErn0YOqk82TsKt9IMm/cOoUugi2rkolwx/SsBnBBeJzPv5Rh9IloHYB0ANDc3l+XkNKYXSuGVy6GNYkgIT1cUS8ANyG3jCABpBmx4pR9LFzQ4krRB6KTyZM/P1atPsYYTRqVPQAZjbCNjrJMx1jlv3rxKn45GmZAYTOGpt44gMZjy3ZY/6DEqblvAjpZGPH7HEqQuTCjVJmsF3ID/1ZeWhprw+P2TRdzqYpSzbT731U2O4dm1K/DQzc0lGRPViHJ5+icBLBZeN2Xe09AIHa4pdXWp7v9avEKujpZsG8eR8Ut4++DZvGWU3eQY+Ll69Sn2Qy0lg8tl9F8C8F0ieg5WAndMx/OnD0r9QOQTriklpU/UkTk7fgk7epP2+xoWwkzUcsOaMB3LRPQNj9l/q+QY8h0TtcbcKhZlcwuA2wHMJaIkgP8NQB0AMMaeBvAqLLrmEViUzf9UjONq5KLYBrocD0SYJGo5PbJtPSfsBOS2RBJbvl3dxiAM/CZqt99JngCCjq3EYArbE0n7tWEAw+cu2gJthV4LzwlMTFZ/PqBY7J2HfD5nAB4vxrE03FEKA10OKqNbElWm4AW9viATg982cgJSJwed8AqBBf2dwoyt7oFRTKWzkswMhC378mfqyCqe/Jc2Uf0qnroit4pQCgNdyvi2qpOT2LJPpuAFaeYN+HPMgxglnoDknn6Ya9+8dwiv9Z3C3e0L8fDNzVUZL/bKqwQdh2IlLxF5GtvGmfUwyNLaNIhyagLChIr4739p0kTMIHzhs/NhEGwZiWpX8dRGv4pQCgOdb9LU7wF0M7wqCh6X6mUA4jED6bR7M+/Vy5p8DU5Xb9KhBqnahicgn951FMdGPsbVcz8d6Lo37x3CD54/AAB45/AIhkY/xi/3HK/KeLFbDD3oOFRV8oo0To7EYAobXulH2rR69K699Wr8cs9x36buXqsG/vtPmQxvfHAWcYNqpk+vNvpVhFKxWsImyDbvHcL6F/uQNhlm1GUfQHEicPMG+TU8s+sofvPeGQCWB7Z1vyXeFTcIDy5vRtuiWegeGMXwuYs5MrxeBofHhvlyPmaQ50O+69CHmJgyceTsH7Hr4Fk8cU+7skcsx2t9Tn7Cv/SfrrlK3zDjUFXJK2/PxwoDwBhDw2V1gZq6u+UYVrTOQcwgTGWoQIwx3H9TMxbNviyU/MR0hTb6VYZKC1UlBlNY/2Kf/UDxxBgAR7n87Uvnu3pXHS2N+Pzi2Xj9vTO2ceYd9tKm1VBbjPvHDAJLM8Ri/jK8YmyYANzfudjTI5wUWvtNpJlD+13lTd7dvhDvHB6xX3+5bUGOV1oLCDoOg6wKVNvI+w+bY9hwb7vjt1yd6c9bCywebfQ1PBE2Ht09MAqTZROgRsaTlsvlX3/vDOriBh5YvtjupiRiResczKgz7O9wxAwCAfb7Uyazuyql0yZe7z+NhsvqXM9XNg6rXZpxA5k4cmZiAoAYwdcrffhmq4pcjOnf1bYgL4mBfDGdcghBVgWFbiOOvUuTVqjwx1+/IUfH32+1UC3QRl/DFfmwgezk3JQls7vh3nb7O3KsPp3xuOXlNDda61e14bW+U9h9eAQMlmd++9L5AKwwz1QmycpJNmkGPP3bAbvTkup8gxiQxGAKO3qT2NZzAqbJECPgi9dfgda5n8am3ccAsBx9GXF/D9/cbBt/fswghU7F8DKnI+c8yP0pZJsVrXMQjxl2iGhbzwmbERZ0tVBN0EZfw9UzDMsGEo21HPfmxrarN4ntiSTSaVPZXQlwLrHXr2rD/uMfWfF6g/DG+2dgMiAWI5BBMMVlQAY8+fvkzkP43p3XKQ2/F5WTMzv4nhmAM+c/wduHPrSTietXteXQSPkkJzJ2GmfWO+5Fse61G7RaaC46WhpxX0cTtuwdspwNk7km72WHYDqtmoJCG/0agNfAVXmGAGyD5ZcUVVElOb9eBje2vFx++NzFnO5KABxGK3Vhwp4stu4/YXv1U+lsWIcAEAGMZV+bDNh9eAT7j38UytsVk4YcJgPeTY4Jr5lN6xONrMkYfvTCAbx98CzePngWk2kr/2AQ7AnMTcjNLyYd1PDUirfKEfTerFnWhB1SUxcV8i0em07QRr/KwZk0YvIRgCuLZkdvEl29SYen/fbBszhz/hMcPD1uf1cuorp+4eWOmP2ze4fw3P4T+MJn5+M7t13jeFj4g5UYTOV0VwKQw93uaLEkdJmQK4gREBPom+tXtaFveAwj45dw5vwn+ENyzNfjl5EYTOHkuYs2LdTIhJDktYTJgPGLk3jqrSM2f5znMdIMNutI3H5yysRrfadcvXC3sFPUdImihLBSEGHvi9eqaTqvALTRr2LkMGmmTHT1JrFDMuqiZ8gNJR/obx08i9czRuzd5AHEY4R0xt3mxnAizWxPmIT306aVsN118Cy2rFsZOL6u4m6rcgU8Edc4sx59w2N2qCgeM1AXsyh5QT1+uRnHF6+/wp48ZBCATbuPwWRWodCqf7cQr/zhlM0sUm0fM6w2gjxUpfI2VWEnleHh73slNaNoiCrd+IZ/FpSS6dUjYDqvALTRr2J0D4zazBMAIHIyX8TwCTee/cNjDirl2fOfOPY5lVaZNQsGgNZ5n8aRDz92vD+ZVsdQ3eDWhcnNU5Nj8Om0iQeXN2Poowt2EngyM+F5UTltRlCa4c0PztpGnGCxkIgAlrmffCI1GcMrfziFDfe22xNPOm3lH66a82kcG/kjTBMAEZYuaAjtbfKVBGMMRITxi5M5OQ+eM+DXEUXvsxyNb/xCWcVaNU33vIk2+lUMWUqACGhbNEvJeQaEmHwsS6U8eHoc7yYP2PsUPXkAWHD5DIx+PIGpNEM8buDm1jk4+uHHjm0MytUz4aEdVSJXDLGID7OXJ8yPR4CDisk9a1XS2I25QZlwDYM1kd1y7Vx8787rAFjVvL/uOZGlDAEwTSvG/3dfvwFrljXZ13X07B8dExGfvNyKhuT3OYOITzBpk2HT7mP2ZDQxZdororhBAJHr9VUapTCUYUM2+YRrVGNuuudNtNGvYnS0NOL+zsXYnGEtsIxx8vNe0mkTV86+zP4sbljFUXED+NNr5uK3QvHRitY5ePVApgqVMTTMiCOWWSkQZd6Gs1OSiiEj5hMuTZowMjTJx6R8gAxbk4UxxGO5PVL5taqSxvLDLYu+8YdazAd0D4zmMIbq65wTEy8AkyciVSJc1d8VyF298PtoMos9xD1/PgFY4nDMXtXs8FjVVAKlMpQqo+xmwDl1c3LKRCwWPlwj7nc650200a9yrM54nm7VjCK1UCVvfPLcRbs4SmTH8H9HP57AVMbwTKWZHevmQlY7MxRL0dC6eecMsA1dmgFvfnAWj912jeu1iXIPMYPwxD3ZOD+QNQiqpLFKvlm8L3LhDkfjzHqQcA4ifVPcRpwX7vqcevKSC9ae3TuErftP4I7Pzs9hEAFOFlDqwoRjcoplPP102jJofIUQFa+/XAlm2YDL4S8w5vwXwVYhqonh8TuWlOQaSg1t9COOMMqBIn1S/Hv1siZQ5l85hKB6QBzMnJjhiPHLyUjxNQnqhwSGeQ0zlN6dqJgoe7hb9w3ZkROTZXMBquYbP3rhQJbCaTK8ffCsKyVS5cl7eXZuHuSGV/rFyA6YyfBa3ymHUFjqwoQ9MRoAPr94tmvRkFiwxq/jjffP2Pc8FjNwX0cT2hfNUvL9VTH9k+cu4jmPVY3buCn1xBA0wVzIOTkm0kz4izsFd3x2vu2gTKYZunqT6GhpDLQK8ZsYphObRxv9CEL0vrlxEgt/VNtz4x3LFCyZzPKewZjt8cmSA/JATl2YwON3LMFTbx3JqhBOmXjoZkuMig9o2QsWWTRiWESlgyMrJj5xT/aaEoMpS/YgY1XjGQkHlZfV1ZuEnFN+44OzMM1siEMVwulocZbbX5o07YffD/x+iTAB/O6Ikx3EJSSChDJWL2vCyPgle0UEWBTP25fOx+cXzw5cWyFObjwfoGKdyL9F1FgohZ6TnJvh+ZAp00rOcwllBmB7Iok1mWfCzTFS7Tff/gFRgTb6EYNc4cljtiazxL5U0rOOAiHBEk5MmbbHqTKCbgNZbirRtmiWp6xAkLCISDUUFRNF7XKRbUTIiqGpNFHEEAsHy4SVTJN56rOvaJ2DuGEluMWHPwyFLxYzcP2CBhw4OeaqFOon9fDQz7vte9/R0oj9x1P253MbZriGD/ykoVXHVxmmKLJQinFO3IC3LZrloCwzxtB+5SybXjylqEtx02Jy+00Tgyk8ufNQQedc7lWCNvoRgzjowZhdaQpYLBHVgOLG6JNJpxdKsOLAjCHH4+vqTYIApWRC6sKEb1OJoGwHVQjJzWNyE0OzOfpCwdaK1jnYlrC8WR5GqY8beHTlVXZewU2fXU5wc2aN3wMnP/iAlXBVXYuY+D14ejznHvNaCcCanBtn1qM+RphMM9TFyPZAZWzOxP35pBwTVkNehkNlTL146GGNULEMVyEJX1VvBVlNc2XrHNvomwDOjl8KFM/n1yZOxDIhgQDfZjB+51yOVYI2+hGDaOAMI1v4Y5rMwRIRwY3Rhpf7HXIBRJm4sqQV89DGPTaNsz5u5PR+FWPu9T6skyANK9zqAlQThuqzjpbcZhvPrl2BLd/Oxui5Ue0eGLXzChOT7tW4qgR3EMiTmtu1yAaBJ2H5vZKTtHMbZmDLupWu++LXyWPUQHY1BDj1ivjEJ/YzUBlT1f1264XghWIarkISvqqJ7fE7ljhWn90Dow6HZr5L3olDVdEuM96yBt+i1crOhteEWInVljb6EYNo4NKmVfiz9tarPeWC+fceuKkZfcN9ME0GIxPmsAakUyvGq/erHHOXmSl8H2KyzGuguhkbrySYal9uBVuqbfmkqYq3i/erGGwSt/MVQzAAch7qNcuasL3nhMOzd0sei+E+HqoALC+f50zs32PSxMZ3Buz8wCVh4nObUMXfXtULwe/eFNtwBU34ynBbJcj7k1eTbv0XNu8dwt+8cMC+l/JYl/MHNn1W2M5vQqwE518b/QiCGzgGKwG1afcxbH0sV8ZABDfWpuneUg7w7/3a1Zu0w0S86Eg2yiIl0WTWykDlzXipbornHcRLDPNwrFnWhL6TY/hDcsw2hCreer7GxQ9ydy7A8iplQyR79qp76Aj3ZdhOvAZi7a1X29vJxoeDwTnxyXkCcRXxWt8px3d5LwQ/2KvTKf9et6VEkIncazUpgk+AIvXWIOf9EPc1fnHSDiuKv7PfhFgs5yMMtNGPIFa0znGIeLnF8lXtB3nSV24pB8DmpW9Zt9KO6YtshcRgCtt6Ttj7NwFH2T9nEMmUxL7hsRwKJBCsC1FQLzFoclRkMYnJaN5usRxxU16cBVjL/rs+d0UOE8eLSRPP0DTXLGvKSR7f2DQLPYMpMAb84nfHcFfbAse9ERlfADLqn2qvXRWT5v/GDGcvBC/Iq1O3XEoxEaaCVnW+QVYvjmZAmUlWrAER/31kU7dydRzEWSmV8+EGbfQjBj6Y1956teU5uMTyZQ/50ZVX5XjffDCpvOm/+/oNOft7cuchR+iHAPSfOu+QDl7/Yh823NvuoCQSkGO4oXivUA/e7+EQJxAmSS+nTXcWU7EhX5NcmOXHpJmYMrFl7xB29Cbx7NoVeHbtCluSYZ/A8JkQuObiveEx7PGLk3j6twMArIlP9sBFRwHITuJcdiLMPRJXp6W+x6VKfsp9dEWBP75yVh3Ti5FWCU/eD9roRwii5xXLJHFHP57A3e0Lc5b/sofcf+q8w/sWB54bxU+uB7gksX/qYpYy5J6jo45Vh5yM5cfglaHD5y4qNX5UKOZDIXvFYMwucOJ/lyNu6ndNXkwasbOYmIheNPsyRzyfo//kGBKDKTsJy9s0Pn7HEvzw+axmkoqFJZIGTGSTzVxnSK5Y9kI5Y9OlSH6qJhI+2TIA45emXNVO/fpOlNuT94M2+hFC98BotijKZHjh98MwyBINA+AIoaxf1ebQEXGT7ZXjy5ziJ0sYiAJjNzTNQtuVsxzc9fUv9jlWHSoWCxca27JvKKcE3mvQF+uhkI0tv6fy3+V4AL2uyS25vX5VG/7mhQNZii4sWeg9R62Vn2igOf6QHMMjm7rx6MqrbK/+ncMjGBr92BGqi8dyC4rEfIvIgALCNwgvp0dbiglGNZE0zqzH1v0nbEE7UQSwcWa9q9xDlAy8CkUx+kT0ZQA/BRADsIkx9hPp80cB/J8ATmbe+hljbFMxjj1doZJNaJxZj5jhZGjwQSg34OgbHnPoiLjJ9srxZU7xE1ka3PBTpvfr+q9mY5KJwRRSFybsWL5XgowfS67wLSdkY+v2dyXhZiD7h8cgO/NiMp/LN8uNYianTPxL/2nH9/6l/7T9+xKA+zqcuRsvo/6D5w94Fn95XVe5JtRiTzByMnr84iT+6+uHHBW9Dy5fjCszlekqKvLjdyxBYjAVaoVUCRRs9IkoBuApAHcBSALYT0QvMcbekzbdyhj7bqHHiyrCFKfI6oqiJC6P5XP6F2d9cE+eFyiNjF+ydUR4z08VhVFV8CQnqXjSTjbqYWOnlaCfTVeI+RZuJHKDN1mYJkPf8Jhd1MUbxfCQ1Y2LZ+P46AV7+xsXz8bp85/Yv4VY7OVGueXyDb/uyS3+ihqKOcHwZ1esb9i0+5jD+TKIcqq25bE+XeQYiuHpLwdwhDE2AABE9ByAewHIRr9qEfbH3iHG2CVJ3POXpvDATYvBgByRLQB2ochbB8/aFbf5xMzlLlQqTZ+wsdMoJq2iDGW1skCnFVFf50yY80YxiwTPU8S1VzTgP668SvlbuFFuZTlnvjJ0Y0pN19/ZrbezIfRR4CEdXvMiM5lUY10lFxLFe1MMo38lgBPC6ySAmxXbrSGiPwNwCMD/whg7odgGRLQOwDoAaG7ONURRhMo48vdV1ZXbRE8qZnWzmkozkNToQ/YsxAIlUxAlUxVQeRU8BTXO+XjuUUtaRRUqzZbUhQlsWbfSZurwsJuoQsp1YojIoYnE+xdzcPYWkNseUKbcpi5MOBwRwDL4M+rUWjReTk7UJwP53Ncsa3LInhhimFPKd/CEOYe8Whu/OGlXskd5pVuuRO7LALYwxi4R0WMA/hnAF1QbMsY2AtgIAJ2dnV4r3shANo5ykkemd4mx1i8snY+3D5619eh5XyovgTTx4ZQpYkCwlUdQPrP23IsPlUSDXK3ctmiWzcQRV2FufHhV7cQPnz+g1NVf0ToHdZnxGs+M15/uPGSPqbpMMxo3ETq3FeB0CG/I587gDNPICVm/axI/56snN0csKiiG0T8JYLHwugnZhC0AgDEmrj03AfiHIhw3MpCNo1dYRJ4g5jbMcHDjOVSxVM7w2Lp/CP2nzoO5UBCLSWnTnnvxIfK6DQC3LMny4uU2ifuPf5Sj1a/iw69ozco589aQ3KgBuePANDOFfKaJ/uExhyNyf+finDoOEW4rwEroyISFfO5rlllFcPlq44iMOw43RywqzlMxjP5+ANcS0dWwjP2DAB4WNyCihYyxTE893APg/SIcN1KQjaNbWERFK/z1/hM5HGwxlqrS148bhPuXNyv1v3VCNdqQfx/R4KvaSHo5DaKsA6fN9p8cw4GTY67tGrt6k8gU7GLKtJQmZUMocv7lfI/bCnA6jDu3c3czxKL4oOqaRBlyQN0aM2oroIKNPmNsioi+C+BfYVE2f8EY6yeiDQB6GGMvAfifiegeAFMAPgLwaKHHjTL8wiLiBJEYTFltBd87Y/OvuSxsYjCFp3cdxZsfnAVjzKGvnzYZFgl9bGVPQodlogu330eukOUGRG7t6PXbirF5gqWf88XPzndUBY+MX3J8h+BUCz14ehw/yBR2vZPphyz3SVCtAKfLuAu6eg0iPijKkBOAWxXVzFFbARUlps8YexXAq9J764W//wuA/1KMY00XBBlYDupm3MBt183D/IYZdvJMlEAGsk1CmCTq5OZJRPWh01CPD7mimLdJVLV29Iq1c4NPZFE9f3v4Q0ev4bkNMxzfm9sww7HPJ3cecnz+i90DOD56wZZb9ipEqqZx5yWvwOG2avPaptIrIF2RmyfyjdG5SSmk0yZuXDzbLmZ66q0jObF+Nz591DwJjfxQKA1QNC5uUr8AlLLOItoWXm57+AAwMPKxQ6rZTV++2mCHdpg3G8ev1WLUVkDa6OeBfGJ0coIuSBcpUQKZS+mq+PSl8iSilHyqFcieclhBOllt0y2v5NawBQAaLquzmUAAHFXCRMjpa1CNY8OWKs/QON1o0aJWVtuiWa73IkorIG30Ed64hfWs3RJ0KuEyMXa7Zd1KR0z/l3uO21K6IkrhSUQt+VSrCPvbisbFrV+xXJwkywZwORBeoMQrxkW1ySCT0HR2GsRnXGxCJG8jamW59bCOGmre6Odj3MJ61m4JOvFheGbXUbzx/hmbN8zP48bFs/HG+2cC6c0Xc7DpkFF0kO9vq/qerNsPxhwrz/7hMUdh2BP3tOdMHne1LfA15qrnCiiv6F0hCBLaWdE6x6GVZTJ134uooeaNfj7GLaz3pUrQ8cKXxGAKD/282256ATj1UCqVBIpa8knDH16eNf9s+NxFx3gHsjLOvOjLDuuYDFv3D6H9ylmOeHWQSUh+rnb0Ju1q4qivHIOEdgDrPsiN10VyRVQnuJo3+vkatzDelxxrFZeK3QOj9sPHQUJbtkolgaKWfNLwhp80gijwxyWCxT4DYuKXwwTwbnIM7ybH8OueE3hunXfLThHyc8UQrKlOFOAW2lEZ8odvbs5ZCUV9lVPzRr9cxo3vVx4MvCRe9PQJLOe7+ZxXod5GlJJPtYZ88kw8viy3RnSwxEyGBwSJYP652GULAJbM+zSOfPix/XoyzbAj06UryLnKzxVg1RBMh5WjyhH0mlTl50Re5XT1Jm111Ciscmre6APuxs3vwVP1OfXaXhVKevyOJdjy7RXY8HI/3k2OAbBk8gv1hHQidvoin99OrAyVWyOqpAfE/XFqKC8yMgi4uXUOjo18DJE1rBLCEs9VVmyVn6uorRzdqo79qLNiRzPVdcj3mxCtVY42+i4IIrT00MY9Ntf5iXva8cRLffbrLZmlsKrvpqqEfv1X2/DIpm6HaFshzRh0Inb6Ip/fTqwMlVsjBlnNymNz9bImtC2ahR+9cAAms0TY2hfNyhmT4rnyHsoyg0V8BsrdVMcNm/cO5VQdq+jQHPz+8M5lvzsygr3HPnLk5zhUq5yuCK1ytNF3gd+D19WbtDn0E2mGX+wecLzu6k0CyA3niLF9UfJWxbEuxEvXidjpi3x+O7/viF53kHAM357Hq93G5IrWObYOPWAlf8VnJaorztf6TuW85kbf7ZyfXbsCT+48hN8dGVE2sJcNf1RXOdrou8DvISJp+08m047XI+OXlLG9K2df5voA8f+K0YxBJ2KnL/L57YJ+J0xsmr8HIEf7n49JB4NF6KHMEdUV593tCx1Vx3e3L7T/djvnjpZGfO/O67D/+EeOBvZBGwxF4boBbfRd0dFiyRjzmJ/8g61e1oRtieyS7XOLZiF57hP787kNM5xUTaFBiiicNqEYMMXy0qM00DTCIZ/fzo2XL04EKoMGuDNLeBjTrgxHruy3isHCEdUVJ/fqVTF9r3Pmk6vY6CbIdal6YlfKGSPGVOmZaKCzs5P19PRU5NhBlqXyDynG+OWY/vC5i9iyb8hW4xPv+vKrGvHXd1/vGgfVhlsjH7hRB8Xc0fpVbXji5axcw5Zv5zZJ37x3yLHfesV2fucx3cZykHMOel2qgrjJNLO1tLxyCfmAiBKMsU63z7Wn74Igy1LZs1LpmfBtEoMpO5lDRA79/H3HrQIt8UHSXrpGoVDROB+/Y4kjDMSphIC16nxm11F8fvFs25HpPzmWs990OlyYZjqO5SDnHPS6ZFvCn/xKSTdoo++CfJalXoNATtSuf7HPYfijFO/UqA7INM7xi5M2+4azaDjhgGPne2fw+ntnEIsRDCBH6dVAbpMQDW+Ijeh5tzRelVMJ6QZt9F1QikSoPCn86IUDNg9aP0gaxYbc4GPT7mMOSWTAej9uAGnT+psboynB2BsAbrl2Lu5uX2g3CheZZxrekPsX3/m5K/DmB2dzpBvKBW30FQjKKw6ideKlcPjr7/wpunqTnlrcGhr5QtbXFyWRRS2ceMzAN25qwsj4JfzmvTP292MZihpvDgJYVbU/3Xkop9m6jCjG8UtxTjxs6/UMr2jN9i+uixt47LZr8Nht11Ts/mijLyEor9htOz4IOFNH/kz+jqoBdRQfGI3pB1Xth0oLJ502QbAYZ3UxwlTaYqQ88dVshywAOfLgPE8AONkoUeTmh3mugz57MrPp1z0ncH/nYt9iLTFvVwlooy8hKK9Y3G5iyirLvrt9ITa80u/a2LpL6F/qtu8oPjAa0UQQAyWGFEVaJZDVwonFDLu5T9wgPHxzM9oWzXJ0aPvB8wcc4xrI5gnk8So+G5cmTVfNnnIiyHMd9tnrHhh15Dwm0wyb9w5he88Jm73HEaVktjb6EoImcO2y7MxA2n14BHuOjjqUCglwCDZtTyTtzzjXOQiPOiqDRSM6yMc5UFWJdvUm0X9yDH9IjoHBEmQD4CgeXL+qzTF2OQwC9ggMIT5eV7TOQTxm2D0ktvWcqHj4Un6uVTInqmJKP+kKsbsdx0Sa4eldR/Hzb7qyJisKbfQlBE3gimXZuw+PgMHKxPPG5TGDcH/nYnuwP/XWEUylrTQZAbi/czEAtepmFItZNKKFMM6B14qAUzazzdQJZ8cvOfb9Wt8pe+wClsMCxhCLGeg/dT7HkeloacR9HU3YsnfInkgq7byoQl0qSQlVMaXbpNrRYnW36+pNYt+xj3Dk7B/tz9784CwSg6lIOmza6CsQdCkmlmWLxS5y43JALWjlprrJPTBZ6kFDgyOoc+C1InDqxmf73+469CHiBtnVpne3L1SO8ZPnLuK5fVbhFndk+L7XLGuKnJQyf67dZE7EiUEspgzSsS4xmMI3ntljr5RYhLtoaaPvgaAxU7lBiorS5raCcHtwuQfWpRBz0tAIuiL1yiPJ7B4emkynTTy4vBmLMpr7HS2NSpmFxGDKYdhXL2sKfX6VgJ/MglxMGWTS6mhpxN+6dNGKGmpWhiGIVn6YmKm4PfecZtQF+558Hk+9dQT/+JuDMJlFm/urLy2NjCStxvTB5r1D+FGmBSIA1AvyIBx8/MnsHpWUuB8FeTrJiBRTZiHId8p5P7QMgwJBDHrYhKq4PVCY+p6O62vkC9GIrxcMPuAMv3CInu2aZU1gQA7lUNaOETXkVeN3OjDQiimz4PcdsfeGQcDffu2GouvthEFRjD4RfRnATwHEAGxijP1E+nwGgF8B6AAwCuABxtjxYhw7HwQx6Cta5yBukC2MxA2v24zNDTVfSnPmTuPMevzw+QPKh8kNUV4aa0QXchcr0eDHDcLqZU3K8Ssb6fZFs1wZZV4a8hzTkYEmF1kBxVPCFHtvpBnwN88fKLvejoiCjT4RxQA8BeAuAEkA+4noJcbYe8Jm3wKQYowtIaIHAfw9gAcKPXa+COxJU6Z4mqyUqp8W+aMrr8LGdwbAmMVkeHTlVXji5X5b0Gp7zwk8cU+7MtErI0q8Xo3pAcdqM8MgMxmz2xgCuWwxlVEX49Iioyyohvx0W6mqiqwI8K06DgqZkGEiq3lUCceuGJ7+cgBHGGMDAEBEzwG4F4Bo9O8F8ETm7+0AfkZExCqUUAjiSXcPjGIqbdqJLZ6YFYtOuoSik8RgKqNtYn3fNBn6T53H5FSW6jaZZjkPlDbsGsWCbGxlJtlTbx3JUd0EgJPnLiIeM6zKXCGhKzPKgmrIT7eVqqrIimvl8MJLsR9u2Pj86mVNNhOIY2T8UsVCYMUw+lcCOCG8TgK42W0bxtgUEY0BmANgRNoORLQOwDoAaG4uXdxL9qTlH9LNW4kbVjEGA7A9kbRDNt0Do3bLOAAwDMLd7Qux99hHtqdvZChx02nZqzF9oGKSiYZJVt08fGYc//X1Q5YRjxEeXG5V4ooJXT7u+fPCqcb8fbc+ztNppSoXWdXFyPb0eeHl/uMfOfoRBO2zwe/D//61G/CjTHexuriBuQ0zKhYCi1wilzG2EcBGwGLvlOOYXj0x5W43ty+dj9ffO+NYAYiTBI+n8uYISxc04JldR3Hm/CdY2ToHv9xzfNosezWig6DeJf9MNZ5l1c2X3h22vc+pNMOi2ZfldJTySvy6aU9NFw+fo6OlEU/c046t+4dwxeWfwmO3XQMAjsJLscuYl7F2uy9ydzEAFatjKIbRPwlgsfC6KfOeapskEcUBzIKV0I0E3BJPqgEeNwh1cWspLHtCbkva3x7+EBNTJg6eGXct3tLQcIM8/sRKbxXcxrOKl2+DrJVAYjBlV6vuP/6RnXAM2nYx6qwdFcRrPnhmHI/ddg06WnILLxtn1qN/eMxRuCYba68ktrz6qVQIrBhGfz+Aa4noaljG/UEAD0vbvATgLwDsAXAfgDcrFc9XwS/xJP6QaZPhgeWLcaVQuMLB/xYLs2Txqf7hMfxYoaypoeEGR6I1I+q1LaPoqDL+XjozshSBnZxlwBMv9+P6BQ2BjLnqmZETwsUMWZRyBeHl9KmkG+IxAw8sz1XTBMIlsSsVAivY6Gdi9N8F8K+wKJu/YIz1E9EGAD2MsZcA/D8A/gcRHQHwEayJITLwSzw5NDlihv2evKx7ZtdRvPH+GZgsW5gli09t3T/k6qVNx6WxRunBx98nk1ZuiCFr/FUV227GihtsXui3dEEDntx5CL87MmIb6neTYwCs/BOfMJ7cechVLkQcrwdPj2eJDCy7cih0TJea92+HZietFVDjzHr7M26YRemGqbSJEx9dUO5rOiSxjWLshDH2KmPsOsbYNYyxH2feW58x+GCMfcIYu58xtoQxtpwzfaKEjpZGPH7HEuWPxH/IB5c3A4zhuX1DeGRTNxKDKQBZytdv3juDNIODHdHR0ojbrptn72vKzG1Rx/fxyKZu/ONvDjr2raHR0dKI9avaLKEzAXKsWf7O43csQerChNJz59t8787rUB83HLRCA8AtS+Zi/ao2bHilH7sPW5MCnwjEkKb4zPAOUXwffcNjyjGdGEzhqbeOBB7jbqGkYoHfXyNDcd3wSn/OufGJgedEdh8ecX1OvWxJFFAUo1+NkAdmR0sjFs2+zM7oi4NPpnwBsGOkADC/YYb8UQ5KPbA1pjdSFyYgRkRjBiFG3m02E4Mpm47pti13aO763BWIkWXY6+usTll8wuBHveHKWZ7NR4bPXURdzDqv+jprIpHHdD7ODTe4ftfrBb+JJnVhIodZJ4Lfp1uWzM3SOSctOqffNYSd5EqNyLF3ygW/VocP/bzbjstt+Xau9Ko4+FS62oxZmuRLFzRg9bImbO05gak0QzxGDmEqjulW0KJRXsghxtuum4f5DTOUoUIuhCY2RnlwebPntm98YK1SDQLWr2qzt+MUZQB4//S48txkmQYe7wZgt2TkYZN8qnULDZkECQ8Fef7E5O7EpAkTwO+OZOmcbpNh1JLbNenp+3kbosb4xJTV+QfIDr6/+tLSnGrcLetW4uGbm/H5plm2J8C7BgGAQRb31yCVn+++bw0NIDfE+Mb7ZzzDhJv3DmEizWzywaLZlykN/iObuvHs3iFwuXyTAW8fPAsAOHh6HJ/5dDa+LRYpinAQHdImrswcyw6bZJhCG17pR+PM+ry89kJCJkFW0W7Pn2rF/+zaFbjl2rl2qMdrZS4fe0dvsuJef016+n7ehkwrEl+7ZdxFeudDP+92dA0CkFPd67UPDQ0ZXBum/+QYJtNMKYWQGEzhyZ2HHG0NuQaUyrjuyMguyxgY+RjrftXjaJLutR8vL7lveMxR4Zu6MFH2RKfb+amKqIJw7mWPX07+uh1bbEtZSa+/Jo2+31JuzbImbO85gcm0Vam4RhGOcUNHi7Nr0JTJ0HdyzC5z16EbjbCQtWEAa4kuGzBuoFjm83jcqYgp73Nbz4kcBwcAjo9+7OgCBQAtc2biH79xoyfRQSXkpmoRWm7nRnV+QYrL/Dj361e12bIqPJSrKmbjxz557qJtF0SiR7lRtUbfK2bvFyPk4Rp5kATxThKDKcsripGd9H03OYYYAV+8/gq78ENDIyhURIEbmmZh/VezsXfRQBlksW9EvRjVPqfMXJO/ZP6fYODDP+a8v+7PvMetypBz/Sogt7NWpRGkuGz9qjZP51CV/PVawW/OGHzAksFwWx2UGlVp9IMkT9y8DdG4cz5z0GSMnNC64crLbd5zmll9M3mJt4ZGUKiIAu+fOp+zjWigVAZfHNsqKfAZdQb+8parbe0dMghtCy/HAzc156X/Lp+TisBQDqgqmtsWzfIsLgsSigpLvhBlMAyyXlcCVWn089XzdjPubl6BPBjkhFb7lbPQP3ze9qjMCPfN1IgeRCO9Zd1KbHi5P+tESM3GebjBTTNH1tpfe+vV+A/XzsObH5yFaVoyzOtXteVoxBQyVqNSqKSqaJ5Rp+5nLRtxr1CU3z2XERWGXlUa/bA3lz9cJ89d9NUs4VWKqslB5dm0LZrl2jczCq3VNKIJlQOy/qtteGRTt3Jcu2nmcIiGz2QMT//WWR/JGLM9z2LG3AvdVzGeBbd+AKkLE442pG6xf7fjb947ZHcnU91zGVGZBKvS6Ie5uXJIRiWmJO8viFaH6CWoPCdxwIheh1g2L6p1atQGvBwQlfQBh6zxJPZ6ACzDZxA55L85vJg5lUSxOO78uezqTWJ7IulJqBAnKa/jJwZTWP9in72KV91zt3OptCNXlUYfCH5z5ZDMg8ubsUghpibCayWhOq6KCiYPGL4aMIQmFiazmq5UsrWaRmng17ZQdEBiMQMnz11EYjDlOq5XtM5x7fUAWGNww73ttqNhM3xi/qqdxbzGMMg3TKsCv29rhH4Abqtrfmy3lT9gFZ2JKqWWrtYJtC+a5XDSorhqr1qjHxSqkIxXPJTP+IUs07oHnA1XiLLNVQAGIquiF7A6cOk8QHVAbFoui6DJuSPugABWrcdz+7z70na0NOL+zsU2Q0RVDyJq5bctvBwNl9U5jBzfTzGuk3vVU+n8vfRSxMBVDphcvQwiTKXdV/4yFZUjbTqdtChW4wLa6AcKBak8Dq/qQD6QGNTN0OWGK2tvvdrRXOXLbQvw0rvDYMzSMInaslsjHGTDwsMsfrkj3qWKU3+51osbFXP1siZ0eTTmkGPQQTtB8WsIEy4VC8Ty9dJLHQNXnmvaivrziVO18hepqDJEJ62YK5ViouaNPuAfCvLzOOSlIa/IBaxm6FvWrXTsXzWY72pb4PACeXN1UQdFI9rwCtmIhgWMwTAIBOaqdy8zSoJovcjjCsi2MwSQE1J8cuchXFYXs8/NzTCF8Vi5oQtSERwEKs886OTjt53yXGOWp8/j/l79Cj5RVDOLTlpU2DoyqtLoFxJHU33Xy+OQH4g1y5pymqGrHiR5MPPXP3j+gNDYglWMy6sRDn50X9Gw1Psk7kVGCWCtFvtOjuHAybFQhUDcyMcNwhc+O98RUmQA3jnsbFHNK2ZlhPFYHbIDAbp8hUE+9TJe28kSCbx6mV+zm/0QE8O/7jmBdJohFiN8o3NxTh4lCmwdGVVn9MPG0WQv3a80G3B6T0/uPGTP+BMZY12XCd0AlucQdIZ3K1vXiBZUjkFXRsdG9pjdDAv/nticw2QMf/PCAQBW/J3LL0ymGaxQM8FANr7sRfkVvfopk+GND846FDNleFXMhvFYS2nogk4+QbfzOtegLBy3xLC8XZRQdUY/6A+uSjatXtbkWZotJ3nAmKM83gTQvmgW1ixr8ozpe517VMvWax1eSVgArpO1nxGUqZQmg50M7OpN2kbaCjVbXvv6VW0A3GPxMlEAsFaN99/UjN8dGcHx0dyuTzEjK/mtEiILY8hLZeiCTj5hJ6lCztXt+1Fk7XBUndEP8oO7JZsIuRV5jklESPLwEI74aPHS6nwHQlTK1jWckKtZRdXI7oFRDJ+7aI8H1WTNxwOX6ZVDhxvubcffvHBAaDVohQRJcS485Ofl3KgkFrhT0zAjnlOYFTessJIX46SYhjxfgxh08slntVFMIx1V1g5H1Rn9MGwcOdm0elmTzZhQlWbHjGySJ5bx9Dmzwshs5+ZVBNUDimIMsNrh98CLBhawJAsYyyZhf7rzkD2WxMk6SOgQyFIp5crtg6fHEctQBoHcdoWygyIeT+yRy6UGDp4ex6bdxxzXxg0+P4dSM04KNYhBJ58wk1SxjXRUWTscVWf0gXBsHFWyyYtpA8Dx947eJD4cv4S5DTM8Qzlh4oxRGiDViKDGmENegYmaLZxSCVjOw/ULGuxjyAl+r9+fa9509SYxMn4JT+86il2HPgTLhHTW3nq1zavn35PHpdztTUwIy3F+DpksUAzGidckGkWDWMg5qa41qqwdjqo0+n4oNEYp0vF4Ozj+YLtBnmiGz13E5r1DOYJPqspA7fWHh1eSUzTGqjxO2BWYSKk8cNJqBi7vlyHXM1dheyJpkwA4CAwNl9U5VF/5uTx+xxJLd+flfvt7vNubeJ6qOL+8cghyrX7w85pLbRDF3Iv8bImfexlpkULrRwJxC4WFEWJzu4ZSPfc1afSB4njUYTwEkea1recEnt07BCDTiFpICKqSxlGMC0YZXoZnh8SyIVghjsk0U7KlVFLbIvjv+uTOQ/jdkRF7LMj5oTXLmnyZHt0Dow66L+Dkucvkg3imV+6uQx/mTBRnxy85XqsKAuWVg3hN+Y41v2eiVCFMsQCOdxYTny05XxGXVvhiOExVLR3mWv3E7/yuo9T5gJo1+sWAymvxa96yozfpZPxITCG3pHEUlsHTBW4P4+a9Q3huX7aRRSxmoG3RLGyjJABm6WEIkBO4buJ3HS3Z9nliEl7MD/HzcjN0icEUTp67iHiM7PERN4AHbmq2cwQy+WBiysTrQktDEbsOfWhr9fBzLEe+KIgnXwyHSxWicxTAATm/vzguuMRylyBt0dHS6KDQXpo08fSuo7hx8WxX5pXqWuXx19WbDHzfyxH+0ka/AKji/X6ztMySFj05IOsdEgEMBCbpfmj4Q7Vc/8HzB7B1/wmINPX7OpqQujDh2r9YfABlDr0Kq5c1gTL/isaWF0rxJK1c/yF6l3GD8KXPXZGTI+LGSM2yz8XkVK5kg3hd4utiohyTi1u+RL43cvhKZjWpHKoVrU7hutffO4M33j+jfJ7drlUO5YbRICpHPkAb/QIhei2il+A2S4v9d2OCJycm58QlvKGlGEJDtVyXvcC4ke197PaQye3sRA69+HvIMtki1VZOoE4o6j9EGmjaZPj84tk5oSTZkNy+dD7ePnjWUuE0CNcvvBx/SI7Z18iQK9lQLiphqckIsjfMAEfhWcwgfFsRvhJDrG4Syx0tTuE6IHfF4Het4vgbPncRW/YNBfbcyzFpFmT0iegzALYCuArAcQDfYIylFNulARzIvBxijN1TyHGjBtFjUxkQOeQj998VwZehU2lrUBNypRiiXPgRFcjLdZGeGxN46YA10fJiOg4el5Xl59MmcyRJc4z6ZO5qQZTgJaKc+g+ZBuoWEpFpmLcvnW9LOfQPj+H9U9kubYx5hzemc8hQ9obXZFZYtqFmzDb48qqGjwuv/AoXruPJeVXC2w/8ODwPE8ZzL/WkWain/30AbzDGfkJE38+8/mvFdhcZYzcWeKxIQbk0zyTW5jfMsL13rwy/G7yWeFEv/Iga/Oi5HDwByAXyRFkFgvU/xiwPelvPCXsfOawYcq4QVrTOcchyUGZqkX/jR1dehf5T533ZHsPnLuKnOw9hymR2825x/D24fDEunxHHpt3HYEqhwahTCYPCzRvmTDoiwvjFSc/nxOsZVE2w+TpY5cqlhEGhRv9eALdn/v5nAG9DbfSrCnKCj8vkTkyZ2PneGccSPx/vymugVIu3Vi4EeehEuYOJNMty5DOf18UINy6ejX3HrUWs2J82J07MgA2v9NshoI6WRtzX0YQtGS+UMes3FLtgjV+cxMZ3BmAyYM/AKJZmuP7iObtVkW/dP2S/l05brKFf7jluNeQxCI+uvMrh7RZCJYwC3NhU/Np4mG3T7mNK+eqgKKa3XWrPPSwKNfpXMMZOZf4+DeAKl+0+RUQ9AKYA/IQx9oLbDoloHYB1ANDcHM02gY6leebhYowpk0MOTzPTAUnFz5fhNlCqxVsrJ/weuhGJ3nhs5GOHXvrsmXXoHcpGLQ2DHJ2sOGVz9+ER+/cXGRtci0n+zfg53f/0v9kSDFNphr9/7X384eSYw0uVq8gBgAxC/3A2jh+LGWCAI2zEjZ+8KghLJYwC/Fa5qQsTMDPPoWlm5at5XYzIZvI6RqXkG8oFX6NPRDsBLFB89EPxBWOMEZEbuaCFMXaSiFoBvElEBxhjR1UbMsY2AtgIAJ2dnUHJCmWFqkKzf3jMbpJBRPYSnxsFziHmHp/MIVbBbUCtXtaEkfFLmNcwoxyXW9VIDKbw9qEPHe8dH/0YJLQvOzuezakQLIMid7ISKZsqxobXyk0qksXQRxdyVnMiz55vzxhgmtnz4gqefIIhSSfotb5T03qV6LfKVT2XfcNj2J5IYss+Jz1TBb9JRX4eRRXUuhjl9M2IKnyNPmPsTrfPiOgMES1kjJ0iooUAzrrs42Tm3wEiehvAvwegNPpRhcwLXrOsKUdFs23RLHt5KS/xebm+yAiYUNDqxOPJAxBw8pENgu9A1vAGT5qLYCbDvIYZOH3euQLgSWBV2MCPseHWaW1F6xzEY4QpgUv6tRuvdHRS40ZGXlEw5kwAr3EpNOL7ubt9oaOWYLqtEv1WuapQ3lNvHbFJEX4Tndekonoe5bBgkMboUUCh4Z2XAPwFgJ9k/n1R3oCIGgFcYIxdIqK5AG4B8A8FHreskBtWc6E1WXpBXF66KR9yRgDBMvy7D6s7IakGIADHEn+6emxRguxBG7CYGl+78UqHGmXMIDxwUzZJCgXTxo2x4VXW39HSiK3rVuKZXUdx5vwneOAmS4Pn/KUpJedfXlHcvnR+DqdfDGctXdDgMILy62KhHGEO2agDwA+fP+BwvuRQXphwqNe2queRpO/Lr6OKQo3+TwD8moi+BWAQwDcAgIg6AXyHMbYWwPUAniEiE9Yz9RPG2HsFHreskH9wwL2ww23QyB7Ya32nHDFgv6WqWLxVCJVMwwk3pgYADIx8jIEP/4jWeX+C25fOR9/wGH7xb9kkqVv9hMrb9lNX3fjNTgBqbSDVvjnXfGemcMhN90k2gqVIKpaLURa2LSkQjj3jta3b87gtkZ3cp4sUekFGnzE2CuCLivd7AKzN/P1vAG4o5DiVhpyMBWNIKypl/QaY7IG5LbX54BbVHPn3ikUl08gFD8fJRuwvb23NKfAKUj/Bwwth4uheei7ivsVajkqv9srBKMu3LSkQrs+u26To9mxv+Xa06JhBoCtyA0C1rAw7aPz26RU7LLWnVstQ3W/ZiPEEqFjgJU/UclUu/93Csq3c9Jzkc4wSi6sc56Kqws2nLWkhqxLVszcdn0dt9ANCtUwu9j4Bf69pOlLEoozugVHbg+fVtLIRExOgqgIvr6rcMOEFwD0ZKY8Jkedf6bEQ9hrzQc5qG8ATX22zKatB25Lq50sb/ZLCbwCFbcCgq3GLj8aZ9dnEeOa1yoh5JUDlqlxDkmgO6w0GTUZGycsUz6UUhpP/Jpz6/Ny+obyeAf18aaNfMNyangDeiptihSXXgnn45mZPr8mN0VPtnkkpkbowAYMsJhTvcQyES4CKDCAuwVzM36IcnnRQBHFkimE4VZz47oFRMMBuUZpvpW2+1e7VsgrQRr8A+FE5vQaQGFaYMplDvZH/JzfSbpxZD4MIQLY/ay14JqVEMeLR5TDKUfDqgxj0YiR1ZSlqWV8onukb7PZ7+Rlnt3tZK6sAbfQDwG0QeVE5Gbzb461onYOYQXYc2GRO9oE8yPjA51K6nNkznSsso4CgBjtfQ1JNCGLQC51EVVLUYiVxOm3iweXNWDT7MuVvUWiiViZscKermjSvtNH3gdcg8qJy+rXH62hpxIZ72x0ejfiAyG39RAYJb2atesA27x2yBbXcmn1oOOFnsKvJyysEQQx6oZNoV2/S0bzdIMLd7Qux99hH9nOmUknl+zt57mJBxllcZctOV1TYUoVCG30feM3wQaicXgNu6YIGPHDT4hz2QWIwhW09JxxCWpxBMjFp2to+8vEPnh7HD5632ha8c3gEL/x/Sfz13dfXpIEqJqrJyysEQQ16vpMoH/ccMQI23NtuqY7yRDljOHh63PHMyX2D3cI/YWLy8m+eujARmbxKodBG3wdB9D7yoXKqik04uE4PkBXS4l47XxnI2j4A8OTOQ45j7DuewkM/78aWb9emZ1osRIkTX2kUI4zlNonK/aM7Whrx8M3NWPerHlvjZirN7GeAiEAEpNNZTSu38E/Y1ZrqN6+WEJ42+j4Ik6QLQ9H08h7lAccnBK7t4+Zxzvl0fc4xi+WZVgtzIR9EiT1TDXCbRGVJ3cRgCj959X38Rmz+TrCVQ+W2ZrxoTg7/JAZTeHLnoVDMt2r+zbXRD4AgM3wQWdagMUK3AeflcW7eO4SX3h3OOa94plIxH6Ot6g5WqzHtavHyiol8HQG38b1mWRO27huym9czAP/Sf9rx3dZ5f4LjIx874v4EqyJX1RVNbj7D9aqCMN+q9TfXRr9I8Iv7ho0Rqgac28PCGQ+yLjsA3L50vl3QwumkXKY5KN9a7A5WyzFtjSzc6kyCwm18/+3XbnCQG77ctsChdvqXt1wNIBPmNBniLsaegz93DJba4y1L5uJ7d15X03kabfSLBL+4b7FihKrvdA84m28DWe9n16EPbQopYA3wp3cdxVsfnM3RiZH3KXcHI7hzo1Wo5ZBQtUOuM/nRi33oGx5TyiGoxoHb2Hj45uac6ufmOZ9WMtJU78n7lZ87sXdFreZpiDGFexgRdHZ2sp6enkqfRmDkI7tQrOM+sqkbE5MmDIOw9tar0XBZnaOZB5CdCNIM9iRhAPjPf77U0W9U3KfYhSiMqqemOVY3EoMpPPDMnpwwi+xEeDUDyndsuK0yvFhBqueuWp0SIkowxjrdPteefhHh57mXKkboFfbhzTxiMQP3dTSBYMX/OWSdGL99BkUtL5+rCW6GUawz4YlVVW8IcRzwTnGLPzMzkHy0fHy+v5PnLiqr2d3GnNtzV60xez9oo18lCJoD2Jzp0cux9tar8+Zbe0HTHKc//FZrPBTDc0acGy92ChN1iXinuLp4Lpfeb0UgypzEYwYMgp3wTZsMO3qTWL2sSY+5ANBGv8QoZUgnyH5lw526MAFLvccK7TRcVle0c5KPu35Vmx13rUWPajoiKK2Yg4+v1ZnqcxXTi/f2fUfoFPfvmmbhiss/hbkNMwB4tweVZU7SaRNfvP4KvJHJSzEA23pOYPWypqqlWRYT2uiXEKWKa+ezX/4wj1+ctJp7mwz1daXzhhKDKfvh33/8I7uQrFj71g+2GoXcm0KkB7jxd9P+v7t9Id45PALAMtzvJscAjIFgSY64HctN5uSx267B3IYZ2JJZuaZNZh9LjwlvaKNfQpQqrh12vzJXmQBbtE0VSy2GUS3VtQed8Co1MZT7uHLMuxAnQ/7NOBsnTJMSt7CeuMIUwT1/Nwqzn8zJDqEBvQ7nBIM2+iVEPnHtIEYj7H5FrjIAh2ibzMdfe+vV+OWe455FZkEmiFLF9INMJpViDhXjuGEmDVUT9XwmWrEIz/asDbL1bLwar8vwKiwUVWU5DMCTwiy/J/+twznhoY1+CRF2UAY1GmH3ayfTJk2YyFYlynFbkzFsfGcAjFkTw4RkOMQVg0HWefz+xDlH0RfftlQPZJDJpFLMoUKPG3bSkI9HCM89V4V0UhcmHHTfsNfiZrxFtk9MoBYHfTbcWETa2IeDNvolRphBGcZohNkvN8BdvUmMjF/CvIYZjgpGXnELwFHVazKrfaB4fp9MWgm1NLME3ThU51uKBzLIZFIp5lChxw07aaxonYO4QZhMW0Z09bImO6EadKJVVYo/fscSB923WPdQVXgFIKdZkAxd81FcaKMfIZTaWO3oTTpCAYCk629albd8CW4g2z4QcE4AIggoq3ENUg/hVrdQylCA34Tkd3z59+fUx8aZ9e6FcZlOata/4SdatzEXdKUWNh8kn1++3bj4+26xfg13aKMfIZQyRunlRYoeGKfcqSYeVTKOC10FSfR5GYSgBjlfw10ub9HN6AY5vvj7899BFAqTv9c9MIqptGnTGPMJY3mNOb8JxK27mxwq8vqtgqxuVJOhzd83CCCycw96FeCPgow+Ed0P4AkA1wNYzhhTaiYQ0ZcB/BRADMAmxthPCjluNaNUMUq/VYR4XNUSnO9jRp1VaEMAvnj9FXjstms8E73i+25GLwwjJ9/tVJWhog5LseBWVaqS9vXyhDn1kU+wKi93/OKko2dyviutfMecbLDFtoYTk6ZDOM3ttwqyupUnJsdx01YtsKoaWEONQj39PgCrATzjtgERxQA8BeAuAEkA+4noJcbYewUeWyMEwoQ83IxAkPCFm1H28uiCxrLz2Y4b+LvbF+ZUhu45OhpaHdILXlWlsrSvn4F2S76LXi7Pv8QF+m0xriEsQYAbbN7dbXLK6u7GlVkvTZrY0ZssiJQgj0mRZQQipNOathkUBRl9xtj7AECZeKILlgM4whgbyGz7HIB7AWijX2bkE08Vt+UPpizOxuFllL08uqC5jLDbiQZ+//GP7Arh3ZnKUFG3pRgGs6s3aSe6JyadVaWytK/f8eRQDw+TiPeYg9NvVSiEAuoXKlEZbDFM+MTL/fa184rZIHF+P8jHBXRMPwzKEdO/EsAJ4XUSwM1lOK6GD4J6zkEaxMg876CNYfhnfpIN/BhB4sT8WE/uPGQbeM5M+d6d12HP0Ww7Sl7JefD0OF7rO4W2hZcHphHK5yf2dzVhJb6XLmiwJyDe5DsM60q1bX3ccBTaufWCDdv8RhwPlyZNdLl450HOcemCBtzX0ZRTMRv2nroZc/m42tgHh6/RJ6KdABYoPvohY+zFYp8QEa0DsA4AmpuLs+zWUMPNc5YfNq/JwY3nHeRB5UgMpvDEy/32vvuGx9C+aJa9HyB8pWlHSyO+d+d1drhBLABae+vVdmMOBuDwmXG88Hur6xiXCvhUnb80rwgrqeqUGU5dmLAnNM5PF3sb5wNxf1MZvrsY2smn+Y04ScQNwkSm5+z2RDJwJa58bD4WZtTlx0bTNM3SwdfoM8buLPAYJwEsFl43Zd5zO95GABsBS0+/wGNreEDlfaseNq+wihvPOww4lRSwEnNc+plg6bP/h2vn2Z5tmGSd2+ri/KUpexsDwO9PnMv5rpg0DbrKqct49IDFahIlCExWvGRj3/CY3Q9BDu2Ivwdj1qTg1fxG/r1vXzofr793xpcRpJoIVWMhXzZapQrsagHlCO/sB3AtEV0Ny9g/CODhMhxXIwBk71v1sD1+xxLXh7cYtQVuMzuDFRt/84Oz9jZEwPC5i0gMpgIbftlIi2EYIuS05BNDJmFWOU98tQ39w2M5WjXiPYoZFOr8ZSQGU9ieSDrumVg/0Tiz3o73M8C36lW+vnkNM3y9czcvXDUW8mUGVarArhZQKGXz6wD+CcA8AP8vEf2eMfbnRLQIFjXzK4yxKSL6LoB/hUXZ/AVjrL/gM9coCbyKdfJl9PgV77QvmuV6PoaRrRYGLOriln1D6OpN5rXk39GbzND8LKQZ0Dzn0/i7r9/gGtMPs8r58ddvUB53zbImnB2/hF2HPsSWfUPY1nPCs7erGzg3Xzx/MWQk1lIQgP5T5z0Tx/LvHaSq120iDMrECYJi7kvDiULZO88DeF7x/jCArwivXwXwaiHH0igPwj5sfgVXXsU73GinLkzAIKcERDxG+EbnYlw+I46f7z4GSDIRfh2X3KBaVbzWdwr/41s3K6mbXvcjiDcqx9i5/vtEJowVdvLix+ThLvle2LUUGarn745YzKWwOk5e57OidQ7isazksZywL5aBLua+NLLQFbkaOQj6sPkl27yKd2RDJeqm39fRZKs6PrKpW930PZ7bcSlukK/3vGZZE7buG4Lg7KNt4eWe2i/5rnLkewBYMXav9oJ+4Mfs6k1ieyKZw0+XmUu8VqFYOk42+Oorwj22NdTQRr/KUUq9Gb9kmxzLvqwuhnjMcDVU8nnyqlQObuz5pNDR4mzaEcR77mhpxK+/86d4etdRnD3/CVa2zvGUkva7h34GU14NrF/Vhr7hMWzrOYGpNMvxlIOAH3ONSximo6XR0bREFs4rFN0DFuU1XyqmRmWhjX4Vo9S0tyDSDqJXuvP9M4gbhAeXN+d44yrjKU8aohefGEzhB88fwMj4JVtpMqj33NHSiJ9/sxMAlJ2e5MRvIffQjSG1nSeTJU85zCTtNeH0D4/Zf8vCeYVCJ1mnN7TRr2KUgvYmGyW/8Abn+U+lrfNImwyLZl9WEOUyMZjCQxv3YCITo6mLEe783BXYdejD0OX4fgYsyD0Mqyzp5im7yTiEXanJDKV4kQ2zTrJOb2ijX8Uotkfm5vWKxrjYtE6VN9s9MOpg4EylGW5cPBvfue0aX6aQav9eBszv3PNZCbjtU55gdvQm0dWbxKVJa6Uj6gR5XRufVAArJHZfRziGUBDoJOv0hTb6VYxie2RhOOul7KK1onUO6mKU9fQVnPAwxtjLgPmdez6rKbd9ypMBA2yWjqgTBHgXjMn7CdrqUKM2oI1+laOYHlmYylzZ+BWbyrdl3Up09SZBgJKtU8zQlte557uKUe1TngwA4Nf7T9heu8mYQ8TN616LYm38O9oz1wC00dcIgUI568U+lzCsmSDnkw/TyW8lEHaf/Lr499beejU27T5m69Lz6wiiQQ+E1yzSqH4QizDPtrOzk/X0KPuyaEQQpW5HWMrzKQXTKag6qXx+QUTsRM0fN4G7p946gn/8zUGYDIgR8FdfWhpaF0lj+oGIEoyxTrfPtaevUTRELbkX5nxKwXTKNwcSRMQuiCevqZUaKmijr6GB0hjIfHMgQc8lSB5FUys1ZGijr6GB0hjIfHMgQc8lyOQQtdWXRuWhY/oaGhmUOydRjONFLY+iUXnomL6GRgBUolNTMbxw7clrhIVR6RPQ0IgCVPFxDY1qhDb6GhrIxsdjlNtoXEOjmqDDOxoa0EwXjdqBNvoaGhmIPHnxtYZGNUEbfQ2NDCqRzNXQKDd0TF8jEkgMpvDUW0eQGExV7Bx0MlejFqA9fY2KIyoetpYt0KgFaKOvUXGUQvcmH+hkrkYtQBt9jYojSh62LnbSqHZoo69RcWgPW0OjfNBGXyMS0B62hkZ5UBB7h4juJ6J+IjKJyFXgh4iOE9EBIvo9EWkFNQ0NDY0KoVBPvw/AagDPBNj2DsbYSIHH09DQ0NAoAAUZfcbY+wBARMU5Gw0NDQ2NkqJcxVkMwG+IKEFE68p0TA0NDQ0NCb6ePhHtBLBA8dEPGWMvBjzOrYyxk0Q0H8DrRPQBY+y3LsdbB2AdADQ3NwfcvYaGhoZGEPgafcbYnYUehDF2MvPvWSJ6HsByAEqjzxjbCGAjYHXOKvTYGhoaGhpZlJyySUSfBmAwxsYzf38JwIYg300kEiNENFjSEwyHuQBqIRmtr7P6UCvXqq8TaPH6YkE9cono6wD+CcA8AOcA/J4x9udEtAjAJsbYV4ioFcDzma/EAWxmjP0474NWEETU49V7slqgr7P6UCvXqq/TH4Wyd55H1qCL7w8D+Erm7wEAny/kOBoaGhoaxYGWVtbQ0NCoIWijHw4bK30CZYK+zupDrVyrvk4fFBTT19DQ0NCYXtCevoaGhkYNQRt9DQ0NjRqCNvouCKEg+mUiOkhER4jo++U8x2KBiD5DRK8T0eHMv0qNYyJKZ5RSf09EL5X7PPOF329ERDOIaGvm871EdFUFTrNgBLjOR4noQ+E3XFuJ8ywURPQLIjpLRH0unxMR/bfMffgDES0r9zkWAwGu83YiGhN+z/WBdswY0/8p/gNwPYClAN4G0OmyTQzAUQCtAOoBvAvgc5U+9zyu9R8AfD/z9/cB/L3Ldn+s9LnmcW2+vxGA/wnA05m/HwSwtdLnXaLrfBTAzyp9rkW41j8DsAxAn8vnXwHwGgACsALA3kqfc4mu83YAr4Tdr/b0XcAYe58xdtBns+UAjjDGBhhjEwCeA3Bv6c+u6LgXwD9n/v5nAF+r3KkUHUF+I/H6twP4Ik0/6dhqGYu+YJZu10cem9wL4FfMQjeA2US0sDxnVzwEuM68oI1+YbgSwAnhdTLz3nTDFYyxU5m/TwO4wmW7TxFRDxF1E9HXynNqBSPIb2RvwxibAjAGoHKNevND0LG4JhPy2E5Ei8tzamVHtTyXQbCSiN4loteIqC3IF2q6XWKRFESnBbyuVXzBGGNE5MbjbWGWWmorgDeJ6ABj7Gixz1WjZHgZwBbG2CUiegzW6uYLFT4njfzRC+uZ/CMRfQXACwCu9ftSTRt9VriC6EkAorfUlHkvcvC6ViI6Q0QLGWOnMsvgsy774GqpA0T0NoB/DyuOHGUE+Y34NkkiigOYBWC0PKdXNPheJ2NMvKZNsHI51Yhp81wWAsbYeeHvV4novxPRXObToVCHdwrDfgDXEtHVRFQPKwk4bVgtAl4C8BeZv/8CQM4qh4gaiWhG5u+5AG4B8F7ZzjB/BPmNxOu/D8CbLJMpm0bwvU4prn0PgPfLeH7lxEsAvplh8awAMCaEL6sGRLSA556IaDkse+7vrFQ6Qx3V/wB8HVYs8BKAMwD+NfP+IgCvCtt9BcAhWB7vDyt93nle6xwAbwA4DGAngM9k3u+EpZYKAH8K4AAsVsgBAN+q9HmHuL6c3wiWvPc9mb8/BWAbgCMA9gForfQ5l+g6/w8A/Znf8C0An630Oed5nVsAnAIwmXlGvwXgOwC+k/mcADyVuQ8H4MK+i/p/Aa7zu8Lv2Q3gT4PsV8swaGhoaNQQdHhHQ0NDo4agjb6GhoZGDUEbfQ0NDY0agjb6GhoaGjUEbfQ1NDQ0agja6GtoaGjUELTR19DQ0Kgh/P/aEDORtD9kmwAAAABJRU5ErkJggg==\n", 47 | "text/plain": [ 48 | "
" 49 | ] 50 | }, 51 | "metadata": { 52 | "needs_background": "light" 53 | }, 54 | "output_type": "display_data" 55 | } 56 | ], 57 | "source": [ 58 | "# show the samples\n", 59 | "import matplotlib.pyplot as plt\n", 60 | "plt.plot(xtns[:, 0], xtns[:, 1], 'C0.')" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "2ff87f41", 66 | "metadata": {}, 67 | "source": [ 68 | "Now let's define the neural network that will learn the score function.\n", 69 | "This is just a simple multi-layer perceptron with LogSigmoid activation function.\n", 70 | "I used logsigmoid because of personal preference, you can also use ReLU." 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "id": "3c2b4113", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "# score_network takes input of 2 dimension and returns the output of the same size\n", 81 | "score_network = torch.nn.Sequential(\n", 82 | " torch.nn.Linear(2, 64),\n", 83 | " torch.nn.LogSigmoid(),\n", 84 | " torch.nn.Linear(64, 64),\n", 85 | " torch.nn.LogSigmoid(),\n", 86 | " torch.nn.Linear(64, 64),\n", 87 | " torch.nn.LogSigmoid(),\n", 88 | " torch.nn.Linear(64, 2),\n", 89 | ")" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "id": "4e55153d", 95 | "metadata": {}, 96 | "source": [ 97 | "Now let's implement the first and second terms of the loss function below,\n", 98 | "$$\\begin{equation}\n", 99 | "\\mathcal{L}(\\theta) = \\mathbb{E}_{\\mathbf{x}\\sim p(\\mathbf{x})}\\left[\\frac{1}{2} \\left\\lVert\\mathbf{s}(\\mathbf{x};\\theta)\\right\\rVert^2 + \\mathrm{tr}\\left(\\nabla_\\mathbf{x} \\mathbf{s}(\\mathbf{x}; \\theta)\\right) + g(\\mathbf{x})\\right].\n", 100 | "\\end{equation}$$\n", 101 | "To get the Jacobian, we will use the function from ``functorch``." 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 4, 107 | "id": "3b434448", 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "from functorch import jacrev, vmap\n", 112 | "\n", 113 | "def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:\n", 114 | " # x: (batch_size, 2) is the training data\n", 115 | " score = score_network(x) # score: (batch_size, 2)\n", 116 | " \n", 117 | " # first term: half of the squared norm\n", 118 | " term1 = torch.linalg.norm(score, dim=-1) ** 2 * 0.5\n", 119 | " \n", 120 | " # second term: trace of the Jacobian\n", 121 | " jac = vmap(jacrev(score_network))(x) # (batch_size, 2, 2)\n", 122 | " term2 = torch.einsum(\"bii->b\", jac) # compute the trace\n", 123 | " return (term1 + term2).mean()\n" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "719d42ce", 129 | "metadata": {}, 130 | "source": [ 131 | "Everything is ready, now we can start the training." 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 5, 137 | "id": "22e5221a", 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stdout", 142 | "output_type": "stream", 143 | "text": [ 144 | "0 (0.17638111114501953s): -0.05398901349306107\n", 145 | "500 (70.5479645729065s): -22.11136506652832\n", 146 | "1000 (132.07090663909912s): -46.269107513427734\n", 147 | "1500 (200.77129483222961s): -52.43773007202149\n", 148 | "2000 (261.32592582702637s): -57.97596343994141\n", 149 | "2500 (323.2231526374817s): -60.72849333190918\n", 150 | "3000 (383.2428951263428s): -64.88924407958984\n", 151 | "3500 (457.6815972328186s): -67.66869757080079\n", 152 | "4000 (520.3891460895538s): -71.52150952148438\n", 153 | "4500 (578.3121054172516s): -74.47203161621094\n" 154 | ] 155 | } 156 | ], 157 | "source": [ 158 | "# start the training loop\n", 159 | "import time\n", 160 | "opt = torch.optim.Adam(score_network.parameters(), lr=3e-4)\n", 161 | "dloader = torch.utils.data.DataLoader(dset, batch_size=32, shuffle=True)\n", 162 | "t0 = time.time()\n", 163 | "for i_epoch in range(5000):\n", 164 | " total_loss = 0\n", 165 | " for data, in dloader:\n", 166 | " opt.zero_grad()\n", 167 | "\n", 168 | " # training step\n", 169 | " loss = calc_loss(score_network, data)\n", 170 | " loss.backward()\n", 171 | " opt.step()\n", 172 | " \n", 173 | " # running stats\n", 174 | " total_loss = total_loss + loss.detach().item() * data.shape[0]\n", 175 | " \n", 176 | " # print the training stats\n", 177 | " if i_epoch % 500 == 0:\n", 178 | " print(f\"{i_epoch} ({time.time() - t0}s): {total_loss / len(dset)}\")\n" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "id": "bb8fe66c", 184 | "metadata": {}, 185 | "source": [ 186 | "Once the neural network is trained, we can generate the samples using Langevin MCMC.\n", 187 | "\n", 188 | "$$\\begin{equation}\n", 189 | " \\mathbf{x}_{i + 1} = \\mathbf{x}_i + \\varepsilon \\nabla_\\mathbf{x}\\mathrm{log}\\ p(\\mathbf{x}) + \\sqrt{2\\varepsilon} \\mathbf{z}_i\n", 190 | "\\end{equation}$$\n", 191 | "\n", 192 | "where $\\mathbf{z}_i\\sim\\mathcal{N}(\\mathbf{0}, \\mathbf{I})$ is a random number sampled from the normal distribution." 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 6, 198 | "id": "a05b11a5", 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "def generate_samples(score_net: torch.nn.Module, nsamples: int, eps: float = 0.001, nsteps: int = 1000) -> torch.Tensor:\n", 203 | " # generate samples using Langevin MCMC\n", 204 | " # x0: (sample_size, nch)\n", 205 | " x0 = torch.rand((nsamples, 2)) * 2 - 1\n", 206 | " for i in range(nsteps):\n", 207 | " z = torch.randn_like(x0)\n", 208 | " x0 = x0 + eps * score_net(x0) + (2 * eps) ** 0.5 * z\n", 209 | " return x0" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": 7, 215 | "id": "90d4e875", 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "samples = generate_samples(score_network, 1000).detach()" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": 8, 225 | "id": "b5734fc0", 226 | "metadata": {}, 227 | "outputs": [ 228 | { 229 | "data": { 230 | "text/plain": [ 231 | "[]" 232 | ] 233 | }, 234 | "execution_count": 8, 235 | "metadata": {}, 236 | "output_type": "execute_result" 237 | }, 238 | { 239 | "data": { 240 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABQOUlEQVR4nO29e5Bc5Xnn/326RwhlsYktiIbLiDFBYLEbjyTmN9x2trQOZoXghzbZJCvpx4K2ksJJll3jsMkPl6uyqVRtIdurWpOQTUz5Z4uLR2Q3VY5Y3WxjovUEMOPRZQBLAiQyeCQzQsheYdZi0HS/vz+e8/Z5z9vvufU53X26+/lUTc1M9+lzTnef8zzvcyelFARBEITeo9TuExAEQRDagygAQRCEHkUUgCAIQo8iCkAQBKFHEQUgCILQo/S1+wSiuOiii9Tg4GC7T0MQBKFj2Ldv3ztKqYuTbFtoBTA4OIjJycl2n4YgCELHQERvJt1WXECCIAg9iigAQRCEHkUUgCAIQo8iCkAQBKFHyUUBENHXiOhtInol5PnVRHSGiA56P3+cx3EFQRCExskrC2grgEcAPB6xzbhS6o6cjicIgiBkJBcLQCn1PQA/yWNfgiAkYGYCGN/CvwWhQVpZB3AjEU0B+DGA/6iU+qFrIyK6F8C9ALB06dIWnp4gdAgzE8BjdwKVD4DyecA9TwMDI+0+K6EDaVUQeD+AK5RSQwD+HMDfhm2olHpUKTWslBq++OJExWyC0FtMj7PwVxX+PT3e7jMSOpSWKACl1LtKqfe8v3cBWEBEF7Xi2ILQNhp108S9bnCUV/5U5t+Do9nPVehJWuICIqJ+ACeVUoqIRsCK53Qrji0IbaFRN02S1w2M8OPT4yz8o/Y7M5FsO6EnyUUBENE2AKsBXERExwH8JwALAEAp9VcAfgPA7xHRPICzANYrmUUpdDMuN00SAZz0dQMj8furKZM5gErA2i3A8KaG3o7QneSiAJRSG2KefwScJioIvYF20+iVvO2mCVuZx70u7vUm0+Ms/FWVf3Y9ACy5ViwBoUahu4EKQscS5aaZmQC23g5UzgHlBcCmnf7zce6dmQlgagw4MAZU56PdS4OjvPJXVf6/Wk1uiQg9gSgAQWgWYW6aqTFe4QP8e2osuF3Y67RLZ/59AMp/fZSbaO0WXvlXq0DfQgkYCwFEAQi9R9sDoxTzfwg6PqCFPyg+C2h4E7t9JBAsOBAFIPQWRSiiGtoAHPiGfw5DkSE0HzM+UCoDK+/i18adf5KAcRxtV5pCMxAFIPQWjWbn5MnACLBpR3qBmib9M0+KoDSFpiAKQOgtkmbZNJtGV+V5rObTUgSlKTQFUQBCb9GuVXSnYbp80ijNtK4icS21FVEAQu/RjlW0i7yFXx77m5kAprYBB54EqhXf5ZNEaaZ1FYlrqe2IAhCEdpC38JuZALbe4e9v0470+4tKMx19IH5/Ya6iMMUkrqW2IwpAENpB3sJvahtX/QL8e2pb+v01kmZqYruKFi0Gdny23prQ51WUeEwPIwpAENqBS1iOb4l3sYS5Yd47aW3cQKstfU7zcwARcM0a4Ob7G8tSWrQY2PNgdNGaxGPajigAQWgHLmEZ5Q6KchnNTACvP+NvW+oDhjY2dk5rNnPlsKoCR59lBZB2HwMjrMySWBOueIwEhltGqwbCCIJgMzDCvvWzp+MHvEQNgZna5reWAAGr7m5ccJ49DSjFCiDLsBlzZkFpAXDZKlYuLmFvzj7Qiu7Z/8y/ZeRlUxELQBDazeAoV/ZWqvzb5QsP85fPTLCPXa+001QWh51LHn55beE89zDw6m7gxweAk4eC3UhdVo3ZwVTHMsQaaBqiAAShEJD12yLMXz49zgFW/dqVG9MJStvdkqQbaRqB/NoetloAFuhmDMC2aqbGgPdO+d1LVRXY9xj/LWmiTUEUgCC0m+lxbu0Mxb/TDIGxV+xpfP9hcYW4bqRhsQpbOUyNGcrJY9Fi97mXytziuubK8qgpD0kTbQaiAASh3WRxu2TJpEmbijq1zc/qsbe3lcOazSzQzWwkpTjYveRa//hrNnPc4cxxb7Xvyl5KmY4qJEYUgCC0m6zpkI1WNidRPHpVv2hxMNZQ6gtubyuTw9s9q0ZDqCmOqW3AwW1BS+LkIU49RYkDx6rK25cWsFtrKKVrS0iEKABBKALtaE+RxN+vV/VEvm8eBPT/k+C2tjJZvg548wXPvdMHdm9V2NXz1hTXGqDKv/c8CMy+wkNrSiXgxn8HnD4K/OwtYOXdMse4iVCRZ7MPDw+rycnJdp+GIPQm41s4HVNVAJRYOOv5wijxhDG7HsFUJub/gDHK8py3D88qMKESP659/+XzgiMzhViIaJ9SajjJtlIHIAhFxs6TbyVmLn/fQh4veeVqsNiw6gRc2UG6zkFbNxcOsFtIVVnQf/RjcE5HU0bguHLOXYvQzs+lixAXkCAUlXZ3y3S5iJZc67t2dNwg6XnabqKbPuPNK9axAq/9xOvP+H2NygvqYxPt/ly6CFEAglBUsjaMy6Olgh2bcCmFHfe7s4OS1hiYQ+tvvp9/prbx/lzBX+kimhuiAAShqGRJD23mKtlUCjMTwXRPXcmctMYgbGh9VEO8MzMcWK4ieSM9wYkoAEEoKq3M8W+UWhEbwJXIdwWbwSU5ftIMKFOplMrAdfcA/UPxjfSEUHIJAhPR14jobSJ6JeR5IqI/I6KjRPQSEa3K47iC0PWYgdQ0mAHcZhVRmatxKgN95/t9iBo5flxg11Rq1Qpw4eXJGukJoeRlAWwF8AiAx0Oevw3AMu/negB/6f0WBKEZNLvXvms1PrQhea9/V8qoy2UUNpu4VObq4f4hGSqTgVwUgFLqe0Q0GLHJOgCPKy46+D4R/SIRXaKUeiuP4wuC4KCZxWWB1Th4NW4fK01PobB21/Z29zzt1xPse8xvO3H2tMQAGqBVdQCXAZgx/j/uPVYHEd1LRJNENHnq1KmWnJxQIDopv7uTzjVvsriYXMLetT9zu/n3WfDX6gnOea+fY+Gv3WRx30nU8z34fRYuCKyUehTAowBXArf5dIRW0kn53Z10rs0gi4vJld1kT0jTv0tloFIBoID9T/Lr5/5PsGW07jCapFtp1FS1Hvw+W6UATgAYMP6/3HtMEHw6Kb+7k861WTTqYgpTHvq3KYiX3Qoc2QnuJXQOmNyKYPsIYgsAiP9Oop7v0e+zVS6gpwHc7WUD3QDgjPj/e4ikpnUrMlfCSGv+t/Ncu4GBEd/NY37mtiC+4GLOLqq1jLCcAuYEtbjvZHDUa0xH/FvXD8xM9Oz3mYsFQETbAKwGcBERHQfwnwAsAACl1F8B2AVgLYCjAH4O4N/mcVyhA0hjWrvcAPrxpMdqxCXRiPnf7CybbifsM3cNuBnayJXBB570ewmBWPiv3ZI88whATYGoCrD7j3h/ZoC5x77PvLKAIoeQetk//y6PYwkdRlrT2uUGSCKQXQJFHz/uhm7U/G9HC+duIRDgnQP2PgSs/ly0e2hog7840Fk/QLAKOOo7qY3P9FpTw/tbf+eN1Ft0OIULAgtdRiPtDJIIZHu175ove/CpZEokr0HoYecm1KM/cz0X4I293GTObBlhYz+e1nKz6whAvgXQIy4fG1EAQjbsnu+ulVuYaR0mKOMEsmv84JnjfFNXFQ8vee9UulYEeZn/SQqaRCn4n/neh1j4q2r64Gsj1qX5Pet99PB3IgpAiCdMeAWqQc2pT33BMX5mpgXgC8Stt3O/9/KC4NCPOIFsuw92PcDzZon4t6oAr3872DAsboWXlztnejw47SqsoKlHBU6AgRF2+9jtpZPSiOXm6m7aw4gCEKKJMrMDKzA9LlBx3vbkVnbBaF+8vY+pMf4f4N/PPcwZHyC/pUASl40eVaiqwQSR6jy3J7hwoPEVXpTiC1NOixaDtQ74tw5m92CKYSKSWl9hA2fSWm5JLLE4q7aLEAUgRBMlvAI+Vc8CqJzj32ZwDXCU+VuToF7d5Rf3HPgGsGlH8myhPQ/6/eg1VGp8kPjMhDG+cL7elRNVTHR4O2qjDqnkBytLZVaSpbK0L05L1GeexnJLEjOwexyZcYIutNxEAQjRRJnZLp+qTterzPPqfNFi7vdu7+PkoeBxagPHweX9uuw/bMVm3vhLrg0et1QKpgemQQsAU6GYim9qW/jwk8fu9Nw/nvAvLzQ+L0/hKQXs/kN2lXWpUElFEqGclwWVZD+hVm13Wm6iAIRo4sxsl0+1f8ib8lTh32u31O9jasw6kDdnVnNgDOhfkbzX+4WXA7d9KbwpmEuR6JW+6XbSAqBmTVBw9OGBJ/3nzGKiM8e913kD069c7ac1jm/xeuYro3e+Cva3iTrPdtGKc4kTyq4BMI1m7CSJGZjZSbWZ9ZTtuAVGFEAv0ahv0xTyYYLUfOzsaR7xpwXergeAf7ub86xrWC6gK27gm+7Efv91h7cnSwd15f+bLpbJrV6guAqUFnCAun8Fr8R1HEK7nexUwZV3+cphfIuXP+6d/7JbfAVVKgeFlBb+QL2rTFV8hXBgzHdVFSmDqFW9caKEclzL6bQkiRkMjHBW2c4/8IfTU5kfa7dCbgKiAHqFPHybYcLWfmzRYmP1BBaaZqEPwDfy/ie4vwsAvPk88Cu/xa4hva/l6+IzROwV5HMPA6/tYWFfXsg3rjl4vDLHCqFUNoQ5+LX6HMOEhC2sLlgSdBd8fC1w2XX1Vcy24JnaBkx+HTVF99zDwLmfAwsWOWIljs+3FYKoVYHrKKGcpOV0I8eLC/6eOR50Saqq32+oyxAF0Cvk4dsM69lemeObZP4ssP0+4KfTwRsICjj2LDD9nB/cHRgBrr7Va/Tl8fJ/ZyXw83dY+IfNizVZtJhjDSjx6vrV3f7KrTLHVkS1ar1I8WNU8reFAo7t9YuRAtaKhyvmsf9xT9Ep4PVngKtudbutbMFzcJuXxVQCjuzwHy/1ASj7A0+mtrUngyjv4rgowoRyK8/BXiCV+vzFSXlBV7p/AFEAvUNYFaQWNDMT8YKlto85P8ALBIX9O6+Gv74yxwJNH+eCJfXbvPI/WKBOP8fCP27FtudBFualErDsU8CRXcFt+j/BQn3eO2ddK6ALyGYPAm+9BJw4ACBBMZLtDlNG5lHlHHDgcXeQ2DznqTHgql/ltNe3XgJO7POfX3wVcMWN/sAT0q6yUuNCsBEXUp7FcY3SynOwrY3r7kbNjI3KJitSzKYBRAH0EivWoxbwBIKTlQ5ui3cvaP/orgdY6O55kAWZE9MHZOI9NjPh/W0Ff7VAtZWFi1rAtgoo8jpHLvQFsKoC3/9L4LYvBnvHuGIYppslTMi62k8EUk+JBboZJLZ92ltv9+MO5YXADb8XVACnjwFX3OQ1Pav4QchSg37oTu9zn1eBXhzaklQlvwld1HGjUoU7CFEAvYAtBMyMFy1okroXZqf8hlrzc+xysSktAFb9G84Gmj3Igzyq82xKD230g7LVKlDuAy7/v1ho950PvPmcsaOYeUBhnSP3POgL1cocn/Md/9V/nSuT6Z6nWeGEHdMlSAdHWYjrjJGlNwA/+r7/mmW3BI/13MO+8Af477l3EVCC+rMtn2ekoipWjLMH09cPhPny41auna440qAtSeVZklrRxlXAh6UKdxCiALoN10UbJgTCfKxRF76ZBknmKp+Ai68Grrg5mKlR6/VuWB6BoOw8WxGjD9S3hxjaGP1ew1wEl3wiuKqGYqVzeDvHFsLiCtovryuYwwKSZvdI0yI6PumtIr3XvP6M71qb3Br09QP8mUMFjaWSV8DWv4LdSW+95AmmsrfaPMdxg7VbOEYSh+6BX6n6FkmSyVl7H/JjOx0q3BJT+269NtNnTyergHelCgMd5RYSBdBNmCvrvoXBHuu10XrwffemANWZKycPeemRjh49tXa6AEDANWuAo8/6N8mdj7hXSqblMT0ejBmUSv6NMzDCx0tz87hcBEMbOa1Tv4eFFwI7PsPPHXvWS8VUwRs7LuslTFmePe25raqs1C5d4aeyVj7w3ViHtwfP8RcuAjZs478PPsXCFgRcvYa/gz0P+nGLa24DLvglYN9Wv+3FrgdYkZ085Cs2l0I4eShYewBEv9dAQZsXKO/GHHhTSLu+28QV8FaqcNg9mOZ8Wqg0RAF0CzMTwZW1bkSmLybtW9eDMOwAqxbUehvAE2Bjvjlc67gJvgFuvp9/TAUCRFse2m1SmfNXsnYhmT5eo+0SbEWy96Hg8zXrI4E1ZO7TZW3Yr1t5NzD7iifQFVtMQxtYQB971t/f+2eC+9X+5Fd3BzOZFDitde0WL2vJcBVtv88Puut9m0pAXxN6X9V54zsIea9mXMUuaOsWXI0IXd9t0gp4U3lG3YNR59Mmd5sogE4iapUQtbLWvn6NvaIxBbVdoAWq7/p53d31QbIk0530ecdlduRxQ5jKzRa+pgVgWh9R5xXVksJ+3exBf3ZtteK7i45+x3cD2bURZjzGRuehr93iV1hD1WdcHd4eVADT48EUWCrFfwf2d9Ztwh+ob0Q4NQbc8eX479XEZXlG3YNRtLFZoCiAoqMFj256FiYUo1bWg6O80qlln1grmrpK1aqf2aDdNoGCnIF4/7i+QcKmO9muIju7Js8bQgvFuBhAWMZJnEKyXze0MTiMRn/WV32KV/PVCuqGoNRaEFhN7QAOqutzXXItK45jf1e/3fJ1wf8HR72sqLn6/khh79X1nXWQTzsZjkVOI9ifS5x1G0Yr6x0sRAEUGVPwmG2PXUIx7sbdtLO+74352jWbgwJSbwvEX6B2Cp1tLsel0zmza3K+IYY3BVfHaQRZnM/cpeBc34WuWdCBYvO71AFlswUBAIC4dYW579WfA/7he0Gr7uN31Lt/psd5n2Z/pCTZP/Z5N8s90S7FMrTBixEZsSn7fICg1WvOt9Dbuj6XRuoWGn1dDogCKDKm4FFeEVNUUM4Utq4L9I4vu4+jhVPlA16Rrtnsr2B1fUDUVC9XCl0j79EUhu0uQjJxKaS4PHBb8QVqFkocS7HdUM52A4ozgmxhuXaLryzK5wE3f8Z/SVRPoahW1rXB6xX/+UatsSKnmQ6McEV6lKJbscF435XgfIsoK9W14Emi6OIWSk1CFECRMStv9cATXRAERAdJ0+R/29u6mrCFDcx2pdA19B6t1X6bbggn9goNSJ8Hbr9Pe2VubhNwA5U4pmC7/5Zc66d32i4M8/ucf9/PRIq6JsLeTyPWWCtbPDdKmILW5+OqxUiTNKApeD2FKIAiowXP3oe4Tw2qfkFQXJvkWuqnl0Nu53+bZq19MSdpwmYeJ4u7Jqv52yo3gikwxrdE54GHvT5JJ0pdkGauxEHuMZNmi2lTgOo0X4Cf3/84uznCvquwvHad2eVSVlEkEe5t9Hs7cRUV1moxptInDWjarehiEAXQCrIIKe3zNQUyKOFFRcHfgYvRMmvtizmuCZt5flndNY2u9tu1uorKA48iyfvU2+jg++CoNzxHZ5dU3UN29FwCnSxgxhGq836mS1wqq14YpJnFEPX5RLkrzbhTu4Wiy8qrtfruA1ZtrM98S/J9Fk3RWYgCaDZJhVSUknBdnLpqNapNsl4h6lS3oY3hZq3t4kkjlNvlrmnX6qoVQTvzM50e92sA9JhJ8xzMDDGdLFAH1e836v1oKydJ4Nv+P8nnY8eddF1Ku7ADwGYltCvzLSltDPAmQRRAs0kipJIoCfvGjQrKaqFQq/5VwL7HeVXncjHErUqKmgbYztVVI0qv0c+xll4YEicxhbUqcTBeASy5yJ/XbHZ8dQlt85zCPlv7Wl2zOVn7a5siuUZcszIq55BLJXRR7x2PXBQAEa0B8DCAMoCvKqU2W89vAvAlACe8hx5RSn01j2MXniRCanrcX21UElYPum4wuwx92a1ev33FN5qezHXHfw26GFwKpBVpgFkp+OoqQJbPMe59hgWYFy3mRngHnuQFgHb3AckWHK5j2jORk0xtc1Ek10hAGRmzMkDARwaBmz6T7tpKWrtTADIrACIqA/gLAJ8CcBzAD4joaaXUIWvTv1ZK3Zf1eB1HEiG1aLFvtquqFcQLwSWo7TL0Cy72Wjd4j2lfcNgKzSWkWr1SS7tiKlK2UBRZP8eo9xl1jekxlnVDfBKci31M10zkNAkDSc85T5JcT3ZMx7QAfjrNQjypiypN7U4ByMMCGAFwVCn1BgAQ0VMA1gGwFUDvEiekzp6G3xK4FJ9KGSao7TJ0nclgFheZM2htXEKqlSu1IlsbWcnyOWbJIw87bvk8v9lckgUH4MWVdHDZK1BLMrUtjGYr76TXkyvGtvchrtROK7xNa16Rr1DabeWEkIcCuAzAjPH/cQDXO7b7V0T0zwC8BuCzSqkZxzYgonsB3AsAS5cuzeH0OoBFi1lgVxX/jrshQwW1owx9YITdAOYM2rCL2SUsWulmKZJfOG8a/RyzKsWw4+o21qoavsJ1tjowrg/drruoVlia68l+D3bmXVLhbVrzUMCN9wHnfzh5JXaLaVUQ+H8C2KaUmiOiTwN4DMAnXRsqpR4F8CgADA8Px0wE6QJqLQK8YG21Em9yphXUQxvis4aA8H206gYvkl+4GTTyOeahFF3HnT3oX3OuQkHTf22munZKzAXIdj0NjCRPUzWFum3Nn/9hf750AS3cPBTACQADxv+Xww/2AgCUUqZP46sAvpjDcYtDFq1eV4STYFB7WkGdZvXZztVcJwV1W0VUNk6jn9PkVg4Km83kzszw43Y6qap6NSNf99uCaIFWdLJcT7U01TkuwnzxL4Hrf69+5oIrK6rPkbEF8HnYBX1doAB+AGAZEX0MLPjXA9hobkBElyil3vL+vRPA4RyOWwyyanW73UPc8G/zxk9zIxbVTLfplPNsFS4hluWas2cEAPz3vscAesJrH+21HSHPf+1qhdApNHo9mb58ADh1xB8qZMY8bAvt7OlwpbNoMeoK+tpMZgWglJonovsAfAucBvo1pdQPiehPAUwqpZ4G8B+I6E4A8wB+AmBT1uO2HS2Iz8xkz+4wC3pmp1DX5tc8ZsFMSKEF2EIsi1toejw4I0CjvDYj5khLKvE0ste/4ze86za3XBiDo8EBPJoDj3NltrniT+pmOnu6vqCvzeQSA1BK7QKwy3rsj42/Pwfgc3kcqxDYPXXMKVmuCyDOXNc3uFkdeeAb9S0GujlIKiQni297cNSbEXDW8aTizBW96ldV4LJV3Gk0rqq329x2AyOcSLHjs/BX7QA+1A/8+GD4ih8IX6SFFfS1EakEbgRTEFcBXHcPcOHl7hvAtWrX+7C3N32Ett8VYGuj1BetbITuR1uN5syGOEwhXRtkX+HFy8D1wJvPw8/vLwebn4W5UcJaSHeLEtAprs99GfjZLI/7XHJtcA62/flEtdAoYIxLFEAj1KXDRTQCs1ftU2PBaVHmDRPwEQKA4qrL5x4Gjn7Xz8i47p5kzcfyoBtXd92CPbMh7Pup63W/ngU8FP+++BoeZG9XEkd951EtpDv1Ogkb7rN+LLhdmqpse5FWsBiXKIBGSKPJ7QvC7uQ5NRZMIavzOyoeEq5N8irY2miV8JeYQzFJ4w60t9WFSeYCZmiDb1HYKch6+I05Ta4ue624xU6JSHOtRwnxAq7yoxAF0ChJNbmryvDAk9xzhCg4UWrNZt9HCPiZGqranopCiTkUlzRxAJfFaveCmplwWxQzE8DW2/1r8sA3eJpWoH2CY2Rip5HHtd5ohl4bEQXQCkxlMTOBmt9WKaDq9RyxA0rvv8u+R96wvqKwFXR7YVYnk7a2I6xuRBMmAKfHvb44Hvq5oo3tzErSa91uG50k+FtgRAEkJQ9f+MyE12f8HGq51aUSZ17YAaXxLQitKGwVHWbO9hxp/MlR285MBBMMSn38/8yEJxgX+BaAqx11N5DkWg80eiv5qbN9C60Zwp1jLYsCSEIevvDaPrwsH5TY3eMKuM1MAGeOA+W+5D37m0U33eS9RpJFi90L/5rbgNe/HWwfvWlnfQygG4m71k0rySykm59DbYZwh1nLogCSkId/sLYPrwjkytXccCoqbbTUB1x3d2f7VoX2kHTREkg9VsC5n3NMypxNYU+L61UGR/merFSCj+vOu0MbO85aFgWQhDxa+S5aHNyHS/gD9TUGjY6iE3qbpIsWuz3BL1wUnE3x/rutOuPiMzDCwe7JrahlP5mdd/U2mrjRmQVAFEASGvWFuxpFxeVXS+BVyIOk11Eg9ZiAnxwLPv/CI8BHPhZ/3WrCgqQFEXiZGdrIrrH5OV75r91S3yAOYCWh53BQGbjp3wMvfqVwQWJRAElpxBfuahQVF8iVwKuQB7Vq4W0I7S0FeG6NBV5sSnGbA5NqxWsep9yCyxb49mxdneJcEIGXmaTBYnMIk6oAzz/stdko1nQwUQBxZDHbGl3NS+BVyAs9B0IHdM1EA+2a7P8nwIl9/Ljd/IzI7xBqFy4CVoWxmQljDEUpkMDLhSTBYvtzVMprsVGsgjlRAFE0axqTILSCsDhArY2DzkjT/YRKnPIJBVTmWfgvvR44vo9X8qVysHBxxXqrwtjIhLEtgIIIvKZiKtVSn1fj41FaAKz9L8lcaS2MFYgCiMLsCV5JMMAhrJdIki+xgAEiocMJVOuWObVYX2eVD+AHf1UwM+3kIW57/NYU8KMX/f5TAM8NCG0pYWXCAL1zTduLxatvBY7sgjdYAVh1lztWELefJrvORAFEYc73VDEDHLIO6ejAKkKh4JhdQw+MsfA+8CSw7FYW6hUFwEtLLi9k4Q9wS3KzyZvuPzU4Ghwt6mopoY9rnkMvYFtbFyxxz0/WhC34Wtx+RRRAFPZ8z6gBDtPj/k1jj3szTUOXCSg9d4Rmods5VOe966sCHNnJQmn4HqB/RfCaHN/iB4Q1pbK/ol+xHrWCMKB3VviaMMFtx/v6hwA86T1pBeEnt3rtuKtc7GnO/WhxFmD3KoA8XCp6eEaSL+P9d+F/0Ub+tO1v1autukEREV+6uIeEtJjXjL6+aqt6xQoBCAp/3RLCnjGw8i7+bVqp/UP+8KJesVqjLHU73jc9zhlUUNz6ZWrM/4x3PeB//pUPgMmvsWW2aWfL44bdqQDycqmk+TJmX3L/b/tblSMNzHUc02rotRtNyIbr+jddQa6A7prNxkD4Eu9H97nRbh7TSj28vfes1jhL3Y73lcpe1bDiz7p/BX9u1Yq9Zz/DSu+jRZ9ldyqAPF0qYV+GvSpfvg449qz//PJ1/Lu2+jItAMcq3+4YWms6RV7TqWLlDwsFxnX963YOOkh75ngwoGsKdIB7Ap37OV/H+nozrdTl64A3XyhewWIzreWklrp29S67ld1t2grQU9hC6zISTnfLke5UAM32o7lWWDrCf3g73xz6f3N1HxYDsAk0nSpxxWGnD9wQWkeS679/yC3Q5+f4+de+xYuON1/wB8TYVuqSa+uFbTvdlc1OpojyCLhcvVRGwC1cVQgKf/KrsHVQvcV0pwJoth8tzMLQM0Snx/mCMI975jj7V5M0drNv4CQtJARBE3b9J2lNolep2hIwr2/bGrb/b3c2WyuSKcI8Ai5Xb6AYjLxCMP14id1rbb63u1MBAM31o9kCetFizp5w+esBYOsdXmYFvIlKO6PPTQrIhKy4rv+pMT8QbLYmmZng6/fMcX9WMIDUVme7s9lakUETlwVUK64z8Qrslt3C6aH9Q4VZ0HWvAshCnBlru3VqwTOrbH56nLfXwzQA9gW6bgz7mNIOQsiTmQkORGrhrlM77XkA5lCYtGMe293IsNkLp6RZQCf2+75/KgGXrgRmXwZe3cOf8UoUZq6CKACNK+sm6ibQAnp8ixE8K9VP+AK8m8KzAMoL3MEjKQQTmomuBQAAEKd22tdvFTx/4sKBxgRoWgHcjHhBMxdOSbOAZiaAo9/17+dLPsFN9nQdxuTXg3OX24gogJmJYHpcYBXvfVn7nwhv+5rEX79ph9+V0aVM2m06C92PazC8/XipD3jvbYRmoyQR2Glan3TaoidNvY6pCAGvhbRRh1GQ+zwXBUBEawA8DKAM4KtKqc3W8wsBPA7gOgCnAfxrpdR0HsfORC1yb5S961W8GbGvznN719mD9QI8KuA2viXo0gmj3aaz0F2ECWqzitd2XUxtA/Y/7rku4MWqdoQHkLMK7CIsetJaIEmygMzPx2z9btdhFOQ+z6wAiKgM4C8AfArAcQA/IKKnlVKHjM1+G8BPlVJXEdF6AF8A8K+zHjszutlboOx9AXDbF4HZKT9PGuDfk1vr2+oCQdMvLBgcV0sgQV8hD+yRois3cgGSeT2a6Yb6GgSCBUqVD4C9D/mT6/IW2O1e9DSq0KKygHQA2G4FY76uYGMj87AARgAcVUq9AQBE9BSAdQBMBbAOwJ94f/8NgEeIiJRSEZMqMpBUsw+OGtOQAPaNbvRdPf1DVvFGhOmWtngrbsUgCI0QENTeoqVUrk9O0AuWugCwbmGsgGN7gennfCWSp8Bud6Zb4HOaYwV5yVDjGTr2aM2wxpEFS+7IQwFcBmDG+P84gOvDtlFKzRPRGQCLAbxj74yI7gVwLwAsXbo0/dmk0ewDI+zb142Z+hYGu/bpvP6pbdyro1oJttU19xtWvFXq42EbOz4bNL2jerUXaIUgdBiuvj+q6k5OMK9BHQAGgLdeAk4cACuMOVYifefnn7PeTmFYs0C8du8n9vlDcVz9uuIwR2uS0Tiy4Pdz4YLASqlHATwKAMPDw+kthLSmqlm8FfYlXXg5cNuXOAag2+raUXxXMHh2Ktyv6jKBOzEwJhQL06evFy1hyQmDo16/mir/1vGtutiYVTfQLaxY7ym7fcHHVRWYP8tWwZrNye7BwVFWGmZt0I7PBheOZtfPgpCHAjgBYMD4/3LvMdc2x4moD8CF4GBw/jTiWwxbidgCecUGo62upVwGRvhiMVtBjG+p96uaVZW2CWym5BUkS0DoQGr+Zkev/jrI+o1wJRJ3LxV8tVujzvW1IDi9S3NiHxdxmsFw/XrXDAS7NshMLilY+qcmDwXwAwDLiOhjYEG/HsBGa5unAdwD4AUAvwHg2ab5//P0LdoTwaDYpVPxzLwT+4Ed9/tuIx1o0/1T9ApL51+X+oI3ka142h0YE7qLOBdLrTbAaw9tL2hcSiRMyBfBek2qgMJcXyDgvZO+xQ7UL8RmJoCtt3NBZ3kBewZsy2rvQ/XJJQD/X5kLBtfbTGYF4Pn07wPwLXAa6NeUUj8koj8FMKmUehrA/wfgCSI6CuAnYCXRPFwXftSFG3bR2BPBTr3qZwVVzwFHdvDfB77B5p29eh8cReKcav2/9P0RWkWSBYd5L0UJ+XandaZRQHU1ERuD7/H17/jV+/bnMjXmP1f5gNPD9XbXfxp44RE/aYRKbF0suwV4/RmWGarKwfU3XyiEJZBLDEAptQvALuuxPzb+fh/Ab+ZxrIYIuzjiLprARDAAbz7v3n/lHAIDsfVFE6i+hL/KAuqbcknPfyGMZrlW0lrLUUK+3dZrGgUU9b4HRrhX19QY6momANQt6FQVtSmAz/9ZMKPwytX+Sn9mglf+x/aiLhurjRQuCNwUwi6OuItGTwQLFIoBfBEY/5cX1A/E1vspL6hfTchwDSEpzXKtmErFDuwmHXtouzPbmdaZVgFFuceinhvawBa/rrMg4hW//q0plYNunoER/r9gMxR6QwGEXRyBMnhHemctGDZmTFIyimtmp1DX3sG+cFb+P8B7p4ALfim4muiE4RpC+2mGa8VWKtd/mifYLV/HsaukYw/t82hnWmerFNDACAeFzTYPgR5ic+z6WbvF/fkUrOCTmhWLzYPh4WE1OTmZz86iYgB2ibbpIrK/6Kgvzt4+qvd6WAygIBeGUBCaYQGMbwGe/c9+PMvk43cAr+7m56gMfPLz3ZX6GUWj96A9CazN9zQR7VNKDSfZtjcsACB8daJdQXZ6J5CuUrcuZXR9sNJw1wPca928iW3fI+AfW5RAbxPVWMzsMdUIg6NetbrjuZ+95fe1JwqvaO02GlW0jcYXC0Kp3SdQCLQriMrhfnotmDW678/MBP+ve4GoitcThPx9gtwKxt7fY3fyyuyxO/39Cr2HfS0A/uIjj2tkYITbHrhYeTdbqyWvqnXPg71xLcbd72lf1+j+WkzvKQBbcAO+b+6Tn/c1tVYKKNWvhFzC2u4F0j/E+7ruHn69xq4F0ERdMK5zFrqXVgiVlXfXP0ZljgGcPc3WqtnLqttxLQKzvK7R/bWY3nEBAfETfcxc4OlxDo49/+cc3d/9h/5w7LAb0e4ForetxVmIc4Jdbp6wQHWHmJJCjsQlLeThntEND//XF4Cf/dh/XLuaeq0gMSpAG+XLD3tdAQO+LnpLASTJqDAFLhAcjv3cw8Blq/jGc90gZi8Q3dvnzHGvGhj8+/VneDRc0gyLdhfYCK0nSqis2ezFkzz3jF6UNILug2W2RThznJ/rAOHVMC6BnrTC2VWoGRVfLPhn11sKIMnKxhS4dtHHq7v5J+xCsIN1tfbQJeDSFcCHLvEzLFzC3HXB9OJqTAgXHi73TNJgZVjhk5nqbDY67MbsH5dFDSSrcJ4PSeboYHpHAditFhYtjnfFEAUreXXVX1hnRPOmNRu7qQr3DSqf52VfkD+UO+w89Y3aIaak0CIaWRDY/Ws27axXAq5MuG681sLct0kqnJPM+egwekMBuMw4s/WCvZrXAvfMce6Fjiq4t3/Z1/5xN56rL7tuGQHA2SMozN/fAaakkCNxLoq0CwK7f81zXwbWjwW36RVLM+x9Jqlwtif9dcFn1BsKIKr1QphZp3N5D26L9v+FYZrW+5/0LImq/3z1XP0KQvz9QlIXRVxNilmY9N6p4PNHdtcPKOoVSzPsfSatcI6bHdJh9IYCsLW+2XohyqxLc1NMbg3OAtCvB7h3iF11Q6X6FUSvrMKEcNK6KGxqCsRrY44Su33MpoaounvTh1ma3Val7nqfSa3sLrPGe0MBuAS51uSLFgO7/8hv7pRU6M5M8MAMKGDhhWxWA8CxZ/m3VgLT48EmUQAA4nM4eSjc3x8WoxA6m7gWIGldFPY+dUGiKeyr88DH1/oJCABqvenjrExJQ+5qekMBAO7WC/oGrK3OjVW6FvDmRCSzzPvra91ThADgwOO+AqgLKnvDud98nn/s+aP6HKOCdkJnEheL0tdAGheFq7Gb6WrUFelXfYrbPJjjD11WqI24JevpIouodxRAGLUVuuLfZh+gwEg34+KfGgsX/gDwoX7/77qg8teD27pcT3bQLs1sUiE/8m7Yl7QNuKso0dW2ubZPY2rd7Et+QSJKwC+vBvo/wXGumiXqJTS4OlbaiFsySF4WUUGUiCgA1wWub1QzY8dsF20H1UyoBNx8f/AxU7CX+izlQQ7Xk5UhdGI/X3RifreOpKv1NETFosJcO66AsCk43n83OLWu/xP+Pkt9wIJf4Gp2ZQj/X/7n6UYSrtiAurbnraQgwhJAPhZRgdxqvaEAGinlrs0J6PNHuukimas+6ThIiRtouVZV9hDqj9/Bj7+2x7c+TGpDJ+a8B5SY360m6Wo9DVGxKNe1aZ/D1Bhw8KmgUnrhEeMFBJz/4eBA9yM7Ebi+7EElUdiCSs++biUFEpYA8rGICuRW634FoC8g3T/lmjW8Qo+qwLVv1Olxbt+gv7ALlnCTJ7Of+mUrw9005hdeBbeTODNT73oyszE27aiPQfS6+d1KFi32mviV8h3aExaLAuIDwqCgu+fwdi+m5KGLC2uFXebiIoXbR9NKQRW2SJsaA+bP8t/zCYPWSayFRq2KPNJlC+RW634FMD3u+/IVeEX0+nfiA6v2jVrqAypV/j20gbt97vwDXwnMvuxvG3oje0ro/Xe57F7fnETuaWQDI3ysopi/vcLMBLt7qlW26tZs9vvm5P1dmDn79kSp4U1BYXPykNvdMz9Xb30Gkg9KwEevBC66it9DUlolqKJ66u9/0tiwGt0AL6m1kNWqyJoKWqCai+5XAIsWo87FUnEUYcViZQoNbwJmD3qVwo4Asnb3rLyLhfiazawwqhX2yZrnpFSwB0uUdSI0n1oMqMptO86e5sfz/i7MnH0gKNx3PeA3etPHnB6HP4/acPdE9vjZBux/AnjnVf557VvAqruDRWBxfYLyEFRRK+4wS0O3p6hhfBeu/SW1WIrgginIfd39CuDsaTiHuIetZsIuLJe7Zmhj0CdbN0im4hfcXPVJ31rQ4/ao3JX9RToe7f5RpeaufM0MHhvbLajPy1yInNgfnh0EuIVodT5YBAZEr4bzEFRxK+7I9tcL/Iy4Uh+7Tie31gfkAX6u1Mdu1qjvrUAumHbT/QpgcBToO9+LAQBYvAy44ffTmYZRF4wrQ8LuATQ/B/z4QPBYS64F/tHFbMa/+JXgvouU9dBraPePMtw/YddK1u9ocNRI2bSgUr1b8OxpBCp6j+zgRALtLgo7hnY/1jCSCgCjLcr7bDHkfc3FrbjDLI2BEXbVTo1x5t3r3wb2PQ7QE349jR0cL5V5CJNp4dhEWTY9du91vwIwTeEDTwKnj4b3UQ+7UF0XTFiGRO14Y+znr5wDUAXefSt4rNmX+SZ/84VgjyGgWFkPvUbtGqiya272YP02eWWmDIwAN97nV5FrqMzXhu0WHBwF+hYG61Oq80F3kesYOqHgvZOczVadDy5kSn1srULxPRIlPBshbsUdl6U3MMLddWuVzF7GnSIjOG4kWVx4efz5uyybomUctYDuVwBAMCsiyu8XdaHaF0zUqqYWwN0I7H0IeGOvt8qzXFHa7WO2ljbbSItLKD+SruwGR3kVWROIY/X571NjvhCenwO++Wngps+Er8KjmDsT/P+y63he777HwhciU2PA5GOoWQLVCj8WteKN8vev3Fgfy8rzmotbcScRuva9aS+azKaNjbp0ihAbaDGZFAARfRTAXwMYBDAN4LeUUj91bFcBoNNkfqSUujPLcRsiid8v7EJ13TRR+zMbw63+nNF4ruT7+6G8NhDGa2cm3H7MHjNLcyfNym5ghAP3k19HzVViCteZiWAGF6rAT94AdnyG/02tBKyiv0uGeAV+cJufNWZmvujz2P+kkQIaoqjC3p+9jSuWlTdhsYSkQte+N/VrXc9lccn1WGwgqwXwIIDvKqU2E9GD3v//r2O7s0qpFRmPlY2kF4l9oUb16Hftb3KrLwyOPQt8/HZercxOsXmtPMG/9Abg4mt8c9suFtN+TKDnzNLcSbuyG9rA35WuBjeFa11misHh7fEKwFbmtaI/7Ur0rgc9+rHqGP04PV4fN6jO178v81j6dWmKIZuJmf6aVOjq+y7qnsxCgdIzW0VWBbAOwGrv78cA7IVbARSDRi6SJK4ek8Pbg//ruoP+X/EmLnk37pvPcxaHFvJ2sZj2Y6Z1CYm1UE/alZ1tBZjC1SwQo3KwrcfyddH7ndzqz/M1GwBu2lH/nenRjzrQaX7v+v3orp+2JQlYC4o+1Nw7YYuIVqYlJpmzG0UzXTVx7rKw99Oh91xWBbBEKaWjm7MAloRsdz4RTQKYB7BZKfW3YTskonsB3AsAS5cuzXh6cPfpT0Na4bF8nd8SWlP5wOjCqOMAVnuHyFS4hMfvwSBWIhpZ2dXcMFZ2llkgtva/8LZJrq+ZCW9F71kPZitml/CNi0eZbcNdwjMgJA1XURF827YAd41XjaIVrpq4e6mugK8z77lYBUBEzwDodzz1efMfpZQiIuXYDgCuUEqdIKIrATxLRC8rpY65NlRKPQrgUQAYHh4O218ybHcMkE4J2HOEkwiPJdcCpQUh3UItf6/ZBC4qFS6p8Joe91eFScrme4m0K1zX566tMbNAbPSBZNfU9HiwbUNcK+a47z3u/ZhC0rYA2u3bzirAW+GqibIyTOVgtngvgnJNSawCUErdEvYcEZ0kokuUUm8R0SUA3g7Zxwnv9xtEtBfASgBOBZArtjsmiY9W0+hq2uWfBfiGp5J3sXiVnCutoF3YTZ1UeC1ajMAgkKiyeYFJkoKoaVRw6eB+eQGnBYc1DbQJS1VMIvjCgqZFcFPkIcCb7bKK+q5N5WCnpKYpMC0AWV1ATwO4B8Bm7/d2ewMi+giAnyul5ojoIgA3A/hixuMmw3bHxPloTRr1Mw6OGnnVGgIuXQmsvDtoLprdFfO4QM6e9jONqOSXzQtu0ir5RgSXHdwf3pSsDUNe52svMOz9RbmRmkkrYw6NEPVdR6Wkut5TgV2zWRXAZgD/nYh+G8CbAH4LAIhoGMDvKqV+B8ByAF8hIm9CBTYrpQ5lPG4y9Gq/kRhAJjPVW+GbZftvvcR2j6tyOGtQLHDOC3sqjS0TjSj5tIIrLLgP1AdqV26MTuXMM/jpmh3ct7BQwqmluBRxlEWeZiFQ4PqCTApAKXUawK86Hp8E8Dve388D+JUsx8nE8KbGgr+Nmqlm3yCT6jlgx/0AiG80c/UfuEDmvEwRlX610INpbJloRTAx6hhT2/yCskqFY1YHn0peDJXlfM2KZwCd6sMGkN16bmSFnmYhUOD6gt6oBG6URsxU88sGgjMDzN5A+kbT/mGzJ0wtU6SBG7LopnWRaKbCNIVSWHHhgScRXChEZOk0kpAQhdkjSFsABRNOicjDvRIX8M16fRR4YSYKIG/sFL3df+grAw2Rn1aoh9Xo4K1Swe1O7Ad2fDb//iwC0wyF6RJKdppjzVIEAOIFABDeKydvH3KSVNJW04iwzcO9ErZCz/NzL+jCTBRAVsICafrLXnIttxLY/4S/stc3u7544cgaAlhAHNnBf+9/Alh1V/vmsgrJSSKU0gQSm+VDLpJQalTYZnWvRFlWBfbd54UogCzMTABbb/dX+FTyKzwBfzUztJHb2GpUxX+uVPYKdezAMYJ/V8/F+4iTlv4LzSVL36lG99fpNCpsXZ9jmgreRuYUdBGiALIwNRZ07+junlPbuIpUj+q7+l9Yrh2jCEh5cQEqe/+bMQMbFV7gZacbgtjiMKeSNZpTLqQjqXBPugIvsA85N7IIW/NzTGNJxCmdHvjcRQFkghwPEfdd1379apUHypd0EVgJuHoNbzu1za8YVhX3/uqo8kxhm9DS/0pwAlQjN4qQnrzdK0Vy1zSDvIRtGksiqaXWxZ+7KIAs9A/xyl0XXgH89+vPBLczV/+oAq/uAo5+l8dEmlAJgHJXEps8/+fcZTTMXKWSlYrqyC6RthFC0UgibOOs1jSWRA+s8OMQBdAotdGBit0sV6/xJhZVeVVvuvOJgkpAVTn97oIl3sV6jtsE3PYlLlo7thccGLZjAsbrw8zVqW08TCQQWPZS/BYt5owiKGDhhUjUNiKJm0hcSUIrsN2cLtfmwAgHdA9v53Gr0+P+4y7SrPC78DoXBdAoZgaPIuCCi40VuG4QBQAEXHMbzzO14wX9Q8CmDcGLasm13gCZOctyMDCbyAFW8FfVxxEu89pQ7P4jBGfDGujRh3YgOc5NJK4koVUE3DsRrs09D/J1fuxZ1Cqcs9ZPdOl1LgogKbZgNCd3lbyPUV9kdovYm72OpDqlUzN7kKuUdebC+Bbevx4GokIGj6y6i3+Pb6k/1lV1hdnA7Cs8kMauRwhA9f3qV6yP96eaN+X8HI/AXP25bC0KumyVJeREbQ6Cnokc4tq0K5znM1TXa7o0JVQUQBLChmuUyv7qft/jQNlI0VxybVBhvP5tx47J2v8c+++vXhO++gfYfWO2o9VjJisfABf8ktcPyFjpV8/xOevKT5vyQrZG7H71oHh/au2m9OIJb+xlC6aRm6xLV1kdTTMUcqP7rLk5x3hKmz3cHnBXOJdKwXskz2KxDkcUQBLCMmyqAM793D1s3vQtjm8xqj49ygutaWBz/kX66m7fuiCyArolYPYl/3yUd4HD21a7lZ572Lc4VBV4723gti8CR7/D+4fyGpB5flRXv/qhDf5zYTervin3PsTCP8tN1qWrrI6lGQrZXuzceB9w/oeT5ezr6/COL3NtTVjuv13hbFvJZqVvUkXUpQFjUQBJiBqusXydP/Q9rIw/4C5yBK8GR4O9gFQVWHYLcNl1wJnj7JrRCqBU4uDWP3yPhX/fQuD6TwMvPOLPj73naeCyVcCRXagFeo/sAo4+y8/d/Bn3hdy30DOvAQxc7/02TGvzf5OBEXb7RH0OaT/nZk566rKbuGk0QyHbi53nvhwsoAwrcIyaAZyk9caSazlBQt9HUcot7BrpwpRQUQBJsLU/EPzb1eIZcA967x+q79M/MMIDQnb+gRfAVZxKevP9/LxZVHbjfcCLX+Gbp1Ty4w72/NjBUUOgG/7SqTHgwgFeFZlCXWdP6HN48zlg6x1sNbhG3s1M8L5AvjLLY8hHM1dZ4mJKRzMUsr3YAeKtxjhFlFRR6RGfB5/ie9b1mh67RkQBJMXW/q6LZWhjcPVg94KHCp8fuuRa4NIV3PzNHEQ++kBQKAZWUIqDu0MbfL8nEQt3My30wJNAZZ633/eYX31s94A/ezp4Y1Y+4HQ6+0YBgi0wDnyDB5uHrZDSmtrNuuHExZSOZihkvdjZ9YDv2nQNtQeCfbZKfex+tTPggGSKyv7uazEx6zU9do2IAsiCfbFMjfHqwmzwZV5kILcwNYW064YwheLJQ4aQVtwkbmgDu4Ge/zO+qfY8yApFv65/yF/ZB2LL1srLrjAOc3FNj3PtgibqRinSiqpLA3lNJU4hN+JSG97kJ0mEdSKdmWAL1HS7AoCrLiaJorK/+6GN9XEE13Zdfo2IAsiCfbHYAv7s6XrXkTZDdWHWY3cabhoAKAFXrmbB6/K72+6j6rzfbVQrBruy117Z1zAUzeRW9sdqfnEp8E8fCN6s5o1SXuBbAFE3SpFWVM1yMWWJKzQ7JtHM/buUO5DseHGKZWqbn7FWPYdaUWS14r6G4vYX9t279tOFwd4wRAFkwRUbMAW83RoacLhzPHMUQG1a2PJ14a4iu2K31MevMwV8qRQUyGZqHMDtK27698Hsiz0PBvf7v3/kWxImWqDc8Pv8+0OXcFA57EYp2ooqiYupmXN683ptEfZfZwFvC17/mY5nrfSj5iUkJal7sQuDvWGIAshKlIAPy5gxHzezi/RM2KhV89nT4NHKXquIVV5G0cFtfmrd2i318YpNO/gGfe9trlo2ewnNTPDMYhv7pq51GT3n51j3HQKu+lT9e46biFVU0grNLBZOs62jZu+/zgJW+R1vaCPHlsw2KUUYWtNliALImzSrhyhzM2zVrLN7TF9mnNmqhXH/kG9ZmHMFpseDFgSVUCsCC9zURuwBQGiVJeD7b8vnsfKx0/KKSlKhaQYoG7Vwmm0d2ftftNivNs8roFtnAT+Vz/sZGAE27eychUOHIgqg3bgURpRAj/Jl2kE0LaC00K/1KLKCv4Oj3mCaCr+WysCqf+MXqh18igU9eaMLVcW3AMxCNb3PM8d9d1Nljq2ITrmB7ZqPMzP8WYal98ZN84qi2f5mc/92MVRe7qAwC9hOM46ih/Lui4YogKISdfEnycwwW0VoAa2rhhX5KzR98y27FTiyE7V21Bde7h9DZxipKiuKVZvYmpidAvY/7ret0Cl6U2PWCUW0tWiEZgY27fTZfY/XT2GzrYSzpxu3cJol5MzPaPQB7gKrkw2aGYzX+0zqRitSllgPIgqgGzEFlPJmDAAAqsCN/8EP/gLBQrXyefX9VWYmuMpYu4gq54CfTrN1MHswONh82S2eq2lFsM310Eb/3NKM63Nt1wqBod1irhYfQHsC21kC02s2szKDpaibdQ7m9Td/li2PNZvdrwlzuUnFdksQBdCNmAKKCKgahV/nf9hfrY5vCRaqXXc3VwmbN50WhDUUzyuY/nujoAysQF5/hqeflc/zg3amKwBIVn7v2k6fy5njjQca0wiVKCGf1HXTiBBzvSZrYPrwdr/JH4iTDdIIVV31bTZgMyvCXbn0pT7fpXhiH8eEdLGg+T5dMRSxClqGKIBuJMr360wP9Z5beCELdl1JDHhpp7YLpxosBAMBlwwBPz4YdIsMjgZvZFf5/clDwIHHOQtJVd3bPfcw8NoezwXV5/dVSrP6tltdR/WdSZK95HLdpJ2l4Dq26zVps3ns77X/E15vfABQbKElRZ+TWatiFjGGxUJWbuR+/Zqodgt2DMVcmLS7dqTLyaQAiOg3AfwJgOUARpRSkyHbrQHwMIAygK8qpTZnOa6QAFNAuQq59Da6ve6p1/xCMC0shjd5g2L0ZDIdQ1DBwfPl83jgzMlDjoph40a2y+/ffxf47p8GzzuwnRdINpvaVee5p5LZz+jkoegg7MxEfatrl1CxO1Wu3ZLct28LtbBeM/ZrzO8lTNA34nJasR547xS3B587Az91uFRfTBhFXa0K/HYjAVePlQ1mVsED0e0W7BhK0WpHupisFsArAH4dwFfCNiCiMoC/APApAMcB/ICInlZKHcp4bCEpcYHGg0+xr9bk8HZWHAfGULv5iYLte08e4u2Wr6sv73eZ9/0rgBWeMhnayC2kbfRwneX/N/DK36BuuhmVeD+zB4G9XwgKp/J5nDpov9fp8fpW16H9YoxOlTv/wC+Ei3PlJO01o3Gt9sMEX5psodqK3ZvPQCWgtAAo9/kdbBtNVdWjTVXV9+sHnjN67p89zd+F3TDQ3qfrfJqdHSXUyKQAlFKHAYCIojYbAXBUKfWGt+1TANYBEAVQBGorPAs9T7VqTCVTVe5Eqn3y2rX05gvAT/+B5xT0fwL4X1+qN+9tV9TQRlYcNdcEgCtuAo5PApP2TGODf/zrvJ9A+wyPygdsxVx2Xb1Pum+hn8p69Rr/NeYqfHA0uEtV4f0dfTbelZO014z9uZurfbPxn6tbaxJBWPs+jdbiNavp8mypqmeOe80EDSEf5WoMO+ckAl5SQFtCK2IAlwGYMf4/DuD6sI2J6F4A9wLA0qVLm3tmvUBcIFILroBAJX+VbwbzoPw+Q4Bh/r9vuY+8Vb5p3rv8uoOjXJH8s1l2IZ09DfzoRYQKfwD4yTHPNRSSWvrqLh54Y/r5TVfXgTF+/uh3WTmZQmvNZt/bpfnRi75VEOWPjqrPcBG12j95yHNZVYPdWuMwZ09UvPbgurGgOXwo6rxc6M9wZiK61UmYqzFqn0JbiVUARPQMgH7HU59XSm3P+4SUUo8CeBQAhoeHc04g7zHCgoq2UjDz3uvcBI4A8KLFfLNrP72r0RyVg/upKRMvkKsb4elz066W8nm++wI6hdU4h/9zCrVRmi50ZpLp59fv971TvntGZ8fY2TL2e/n5O977CWlZbJJGqIUpDDteYTf2C8OePTG8yZ890WhQOuk5m8+LUO8oYhWAUuqWjMc4AWDA+P9y7zGh2bjcDED4dCV7/KNrlCUZQcQV6zl758Q+68AKWHwVN4wLCASjpe/swXgXyNnTvO8jO/1d/O8feXGCshdwXMD1B6ePAaeOmCdqpRRaiopK9a2uA9kyJkaH1qkxVpbmirpRP7UtMGcmOC5ifuZ2Y78wzO+6Cnb3DG/yn3dZYI2cvwj5rqIVLqAfAFhGRB8DC/71ADa24LiCy80QlVJo39x17qESu1bM1XupjLoh9ADwzqvA7j/05xLU6gm8lr7mwPlSmQfh7PgsC9bRB/xV+1W3Aq99KxiLqFZYuOmaBQD42prg8aH87KDKB/Urey1kzWlu0+Pws2XAVgzA57l8Hb8fHS/Z/4RfZR2XF5+EQPDWyLiyG/uFERdYdfUFklz7nidrGuivAfhzABcD2ElEB5VS/4KILgWne65VSs0T0X0AvgVOA/2aUuqHmc9ciCfMZE+aYjcwwn5xPb2JCLjqV4Ord11A5rIEKufCUxn7h4AVAN47Cbz2bX+A/YFvGGMo5+B29yiuWdCpg09trM8WUlU+77VbQlxVXpaP/jyGNhqN9rzjLr0euPgafm5qLBgsN3vUz7/vt7/QQlwLb3MVHoXOQNJdXn95Nc9ZTuL6SVq34GpFLrn2PU3WLKBvAvim4/EfA1hr/L8LwK4sxxIaxF7Vp02xq80bViwsjuxkgVkq+8VY/SsAEHDiAAIB3PKCYAfKNZvZz97/CWD3H3npgyW4x1CGxBY0s0b76p/NurepVrhfkX6/P/wmMPuy/7yqItAbZ/QBay7y88Dxffz+DowF9607puoZzgfG+H8dv6h6CkhbQCYuK2HRYgQmvS1fF5/uacdtXMPQTexrQXLtex6pBO5F0vhx69xAKphWWEv/06tXsHC85jZ23+gsG3OWwBt/5zeQU547qIZiBfEP34tWAMvX+X+vvDtofdSUimIBObSB38feLxg78DqZ6nx9LQDD5iKbLigQfxTk/a0/Eyhe+euaA1WtX1mHBeYDcx5iirWiqnPjqptNd1/YQkD68PQMogCEaOwUSl35qwt7xrc4VuvEufg1/3sF9bMEDC4cAM78yP9/7l1jcPh8/fYfvz3oWtF/66K02YPc+sEcIQgE96WLmkol7naqt9HZSlXd6sJTSDpYbLbUViVWbLr6dWgjWwtmywm7B39Uta855yFqRV5XnUvhr4nqqxPWzkJiAz2DKIBOpNUrtFqWkKOwKeC68CiVfWFkBnqr8/7K3+Rnb3G1qha6B54ENm0wlIDnZgEBfecDN99f/xkMb/IVwcyEezCJXbWKKu/2hUcQGGiz6i6jj43XQC+s4MnuYzMwEqyItnvwu4KxWkEkdc2Z+9CT5PpXuNtipPX1S2ygpxAF0Gm0c4XmWjEGXBca8rc3Beeu/+h261TngYuuBt55DXWrdh1/IC8Vc/Xn+PGtt6PWbtpu/xDm3nAJcdeQnMA4wr70BU96W1fqpZ3qaiuIJL2H7PcHWKmuJb+ALG1fHenD01OIAug0irZCq7VZsPzRZg94LQxDffoK+MkbLMztQjRTGOmsmB33+xk5uv3DeitI61JWLiEe2i2VrN8x+w77bKKEqZlNpTOJ4gK/puLR29YUjf58q26Fk7RCV/rw9AyiADqNoq3QzBjB/ic9N46q7wFvVgLrPPeaK8ZrMLbK0a/GXumOb+GKXpNX97BwBBprRWCv6Me3+EHd6ny8ko0aaWgL00DFbp/3GQC1TCI949l1jDDLrxaoN9JI7fYSjbR+ELoeUQCdRitWaGljDKbACOsBD6BmIZQWsJ+9f4XVIM7oGGmPNLRbHZjpo0pxSqTZpyaNayysAC6Jko1zydn7tit2L13BRXBxyiaugK9Wr+G16g6bwBWFZP/0HKIAOpFmrtCyxBhqvnPPPWP3gK/NJq5y5o/ZQtruh7P1Dv8cNu2oF5zX3OZVCHsN03Q+fx6usTRKNqlLLmwClmuOgos4pRSo11Dpev7r85Psn55DFIAQJEuMYWAkfQ94lzKb2ua3lqjM+b13zNfffD//mO4hV+ZPo+Tl5wfiJ2Al6aIZp5SyugaLFlsSWoIoACFIVkESJjhTua7sVFEV/npzP3m4xhpxf8UdNyBc57hewWzzkFTZRG2X1TVYtNiS0BJIufKyC8Lw8LCanHROmRSaSbt9wTMT0WmezTxuM9wgdR1JS+n6/LeKdn/vQi4Q0T6l1HCSbcUCEOrJM8bQiFDRrqRWC6OsbpC4bKC9DwHH9iKQplkkQSvZPz2HKAAhnDjhneT5RlfU7RBGWdwgSbKBVn8OmP57tmzMamlBaBOiAAQ3cQItiXDvtMBiFj964vcaUWAmCC2m1O4TEApK2DSxpM8D/oraHg9ZZAZGuO6g0SBq1HudHq8vMBOENiIWgOAm7YQpl8DrpbYCSd6rZNoIBUOygIRwssYAhHrkMxOaTJosIFEAgiAIXUQaBSAxAEEQhB5FFIAgCEKPIgpAEAShRxEFIPQOMxPc61/PDhCEHkfSQIXeQNodC0IdYgEIvUGSwjVB6DFEAQi9QSdWJQtCk8nkAiKi3wTwJwCWAxhRSjmT9oloGsDPAFQAzCfNURWE3OilqmRBSEjWGMArAH4dwFcSbPvPlVLvZDyeIDSOtDsWhACZFIBS6jAAEElnQ0EQhE6jVTEABeDbRLSPiO6N2pCI7iWiSSKaPHXqVItOTxAEofeItQCI6BkA/Y6nPq+U2p7wOP9UKXWCiH4JwHeI6IhS6nuuDZVSjwJ4FOBeQAn3LwiCIKQkVgEopW7JehCl1Anv99tE9E0AIwCcCkAQBEFoDU13ARHRPyKiD+m/AdwKDh4LgiAIbSSTAiCiXyOi4wBuBLCTiL7lPX4pEe3yNlsC4O+JaArABICdSqk9WY4rCIIgZKfQ8wCI6BSAN9t9Hh4XAeiVNFZ5r91LL73fXn2vVyilLk7yokIrgCJBRJO9UsAm77V76aX3K+81HmkFIQiC0KOIAhAEQehRRAEk59F2n0ALkffavfTS+5X3GoPEAARBEHoUsQAEQRB6FFEAgiAIPYoogBCI6DeJ6IdEVCWi0PQqIlpDRK8S0VEierCV55gXRPRRIvoOEb3u/f5IyHYVIjro/Tzd6vPMQtz3REQLieivvedfJKLBNpxmLiR4r5uI6JTxXf5OO84zD4joa0T0NhE5uwsQ82feZ/ESEa1q9TnmRYL3upqIzhjf6x/H7lQpJT+OH/CQm2sA7AUwHLJNGcAxAFcCOA/AFIBr233uDbzXLwJ40Pv7QQBfCNnuvXafa4PvL/Z7AvD7AP7K+3s9gL9u93k38b1uAvBIu881p/f7zwCsAvBKyPNrAewGQABuAPBiu8+5ie91NYAdafYpFkAISqnDSqlXYzYbAXBUKfWGUuoDAE8BWNf8s8uddQAe8/5+DMC/bN+pNIUk35P5GfwNgF+lzhx00S3XZCIUdxX+ScQm6wA8rpjvA/hFIrqkNWeXLwnea2pEAWTjMgAzxv/Hvcc6jSVKqbe8v2fB/ZtcnO/Navg+Ef3L1pxaLiT5nmrbKKXmAZwBsLglZ5cvSa/Jf+W5RP6GiAZac2ptoVvu0aTcSERTRLSbiP5x3MZZR0J2NDnNOugIot6r+Y9SShFRWG7wFYrnOlwJ4FkielkpdSzvcxWazv8EsE0pNUdEnwZbPp9s8zkJ2dkPvkffI6K1AP4WwLKoF/S0AlDZZx2cAGCuni73HiscUe+ViE4S0SVKqbc88/jtkH3ouQ5vENFeACvB/uaik+R70tscJ6I+ABcCON2a08uV2PeqlDLf11fBMaBupWPu0awopd41/t5FRP+NiC5SEbPYxQWUjR8AWEZEHyOi88DBw47KjvF4GsA93t/3AKizfojoI0S00Pv7IgA3AzjUsjPMRpLvyfwMfgPAs8qLrHUYse/V8oHfCeBwC8+v1TwN4G4vG+gGAGcMd2dXQUT9Om5FRCNg+R69iGl3ZLuoPwB+DewvnANwEsC3vMcvBbDL2G4tgNfAK+HPt/u8G3yviwF8F8DrAJ4B8FHv8WEAX/X+vgnAy+CskpcB/Ha7zzvle6z7ngD8KYA7vb/PB/A/ABwFz624st3n3MT3+hCAH3rf5d8B+Hi7zznDe90G4C0A57z79bcB/C6A3/WeJwB/4X0WLyMko68TfhK81/uM7/X7AG6K26e0ghAEQehRxAUkCILQo4gCEARB6FFEAQiCIPQoogAEQRB6FFEAgiAIPYooAEEQhB5FFIAgCEKP8v8D1x5GOHqUxBIAAAAASUVORK5CYII=\n", 241 | "text/plain": [ 242 | "
" 243 | ] 244 | }, 245 | "metadata": { 246 | "needs_background": "light" 247 | }, 248 | "output_type": "display_data" 249 | } 250 | ], 251 | "source": [ 252 | "plt.plot(samples[:, 0], samples[:, 1], 'C1.')" 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": null, 258 | "id": "466f2945", 259 | "metadata": {}, 260 | "outputs": [], 261 | "source": [] 262 | } 263 | ], 264 | "metadata": { 265 | "kernelspec": { 266 | "display_name": "Environment (conda_py39)", 267 | "language": "python", 268 | "name": "conda_py39" 269 | }, 270 | "language_info": { 271 | "codemirror_mode": { 272 | "name": "ipython", 273 | "version": 3 274 | }, 275 | "file_extension": ".py", 276 | "mimetype": "text/x-python", 277 | "name": "python", 278 | "nbconvert_exporter": "python", 279 | "pygments_lexer": "ipython3", 280 | "version": "3.9.7" 281 | } 282 | }, 283 | "nbformat": 4, 284 | "nbformat_minor": 5 285 | } 286 | -------------------------------------------------------------------------------- /02-SGM-with-SDE.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "910f1769", 6 | "metadata": {}, 7 | "source": [ 8 | "This is the notebook to show the implementation of score-based generative model with SDE (2nd part of the tutorial). In this case, we will sample the training data from the swiss roll distribution like the previous part.\n", 9 | "From the training data, we will try to learn how to draw new samples from the swiss roll distribution with Score-based Generative Model (SGM)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "id": "3084c1e6", 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import torch\n", 20 | "from sklearn.datasets import make_swiss_roll\n", 21 | "\n", 22 | "# generate the swiss roll dataset\n", 23 | "xnp, _ = make_swiss_roll(1000, noise=1.0)\n", 24 | "xtns = torch.as_tensor(xnp[:, [0, 2]] / 10.0, dtype=torch.float32)\n", 25 | "dset = torch.utils.data.TensorDataset(xtns)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "id": "685e64d8", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "text/plain": [ 37 | "[]" 38 | ] 39 | }, 40 | "execution_count": 2, 41 | "metadata": {}, 42 | "output_type": "execute_result" 43 | }, 44 | { 45 | "data": { 46 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABZWElEQVR4nO29aXBc15Um+J2XACiyiguKlLiBhEQttAyoqpqguJRVlmTJNUWHLJZFqbR4pkpTJVOasaNC03/aXR6xOazoGld3VLTcY/XYNNvhqg6RorlY25hRFmVtdAkQlpBMgDJICiLAJLhCSQg2YSIz350f792X9928b8t8ud8vgkEA+fKt93333HO+cw4xxqChoaGhUf8wKn0CGhoaGhrlgSZ8DQ0NjQaBJnwNDQ2NBoEmfA0NDY0GgSZ8DQ0NjQZBU6VPwAuLFi1i119/faVPQ0NDQ6Om0N/ff4kxdq3qs6ol/Ouvvx59fX2VPg0NDQ2NmgIRjXp9pl06GhoaGg0CTfgaGhoaDQJN+BoaGhoNAk34GhoaGg0CTfgaGhoaDQJN+BoaGhoNAk34GnWH/tEUnnvjJPpHU5U+FQ2NqkLV6vA1NApB/2gKX93VjZmMiZYmA88/sQFd7a2VPi0NjapALBY+Ef2QiC4Q0aDH53cR0SQRvW//2xbHcTUaF15WfPfIBGYyJkwGpDMmukcmitqfhkY9IS4L/0cAvgvgn322eYcxdl9Mx9NoYPhZ8RtWLURLk4F0xkRzk4ENqxYWtb/+0RS6RyawYdVCdLW35v3ut88w22lolBOxED5j7G0iuj6OfWloBEFlxXNS7WpvxfNPbIhEtl77kyeCbfd1YMerQ4HuIu1W0qhWlDNou5GIPiCiQ0TUodqAiLYSUR8R9V28eLGMp6ZRS+BWfIKgtOK72lvx9btvCk2yXvuTJ4JDg2dDuYsKdSuFhXY/aRSKcgVtBwC0M8Z+TURfAvAigJvljRhjOwHsBIC1a9fqZrsaShRixReyP9k9tKlzKXpPfRLoLirErRQWevWgUQzKQviMsU+Fn39KRP+NiBYxxi6V4/ga9YVS+Me72ludfe3uGcOhwbPY1Lk0byJYvWRu4LHjmJC8rlG1ehg+N+Wc72PrVxZ2AzQaAmUhfCJaAuA8Y4wR0TpYrqR417kaDQGVXz11ZSY28t/dM4a//clRAMA7Jy7h779yG75+903O5+LE4AfVdlECvmGD0lPTafznfxl2zheAJn0NT8RC+ES0B8BdABYRURLAfwDQDACMse8BeBDA/0ZEGQDTAB5hjGmXjUZkiBbuTMbEtpcGYTIWm3vj0ODZvN/jINAorpgoQelnDx8vyflq1CfiUuk8GvD5d2HJNjU0ioJo4RIRsiYDQz4xiojiAtrUudSxlPnvccCPxGUExQD497pHJtCxdF7g+WqJqAaHzrTVqCmIFm7rnBbseHXINzgqWtZNCQMPdrVhy5o2T+Lj1nHcPvEgEpdJ2S8GIK8Wnvr8Kgyd/TTvfPtHUzgwkMT+/iQyWR3k1dCEr1GDEP3jQUFU2QW0p2cMBweSvsT32PqVkYg+jAXtR+Je7p6wLp+5s5vxP/56fd45fXVXN66mTXDfadDKQqP+oQlfo6YRFETlljUnviD3T1RE8c17natM4AcGkr4TSBjZJ98nJ3uCOmdBo7GgCV+jrsEt64MDSezrO42syWIlvii+eS+IBJ4wyNcFw1cTQeokeZ8PrV2BB3xcWRqNAU34Gg2BZQtmY/v9nbFKOIF4kqxEd8/45WnseW9MOYFEXU08/8QGHBhIggBN9hoANOFr1DlKnZkad5IVABwYSConkEJWEwcHkpix3UR+qwKt5GkMaMLXqGvE4XIJQtQkK5ng5Qlp230djkpItO7PXJ5GU8JANhtuNRE2ZyHMpChmH2udf+1CE75GXWPDqoVoSti+7ER5gpZB5ZbFz7asaXNNSAcHkjhgW+W9pz4BAAyOTzp+/SaD8Mi6laFcNGFzFoImRTn7eGziN5i6mgEDfCWuGtUHTfga9Q+e1F2m5G4/ApU/Y0AuuJowMHhmEr9NmwCAq2nLKudEDQBZk2HZgtkAgOfeOBlaCuqXsxAUh5Czj7//zohzK/f3ncaerRvLSvra/VQ4NOFr1DW6RyaQsQkza7Ky6NBlAm2d0+KQs/zZljVWIhhXEf0yOenshwHImO5JiogwNZ0uSArqlbMQFIeQs4/FeTOdLc895dDVQouDJnyNukYpSxV7QWVZiwSlIldxYuIgAAmDwBgDEcBAMBnDriMfw2QsclzCL2fB7zMx+7hj6Tz88F9PYSZjrUKaE1RWbX85YjL1DE34GnWNuGvnRzluV3srnnvjZB5BqZqz8IlpJm3CBGAQXNVARbkmGAMZBGKsbHEJMfv4ix1LcHAgWREffiUm8HqCJnyNukfYksYi4vIThyWorvZWR53TsXQe5s5udh2b18Xhvn7TtCxsMIbhc1NlndAKuZ9xHrsSE3i9gKq1SvHatWtZX19fpU9DowERt584jERzajqNXUc+RtZkmNWsPibf9v3Tl3H42HkwWI0lDIMKKhGtg5/1CSLqZ4ytVX2mLXwNDQlBfuKoROllEYsTixibnUmrfdP89++8fiJXI4dQkD9fBz8bE+VsYq6hURPwa5LOifIffzaMr+7qLqqRuDixiDAM70Bo98gEMlnLnUMA7rl1sW9D9zDHjrvRum6yXr3QFr6GhgQ/P7FIlFfTVqJUoZaxE6i198dVOTs2d3q6c1rntLhiAk/eeSOevPPGyK6ZMLEFMbs2TC9ffp565VC90ISvoaGAlxuGZ+7y0sP7+k4XXJhMlm/61bkJ6uMb9fhBwU85u7Y5YWXqFtOeUaPy0ISvEQsaJQDY1d6KB7vasKdnLJZkLj/FC7+n45enXSSaujLjaqxeKPyOLWfXprOW32kmbeLZw8fx9L23FFyrn6NRxkw1QRO+RtFotGU8z4wtpRbc1ZrRoEhF0+KAnF3bnCBkswwmgF+cvITeU58on3NY2WSjjZlqgSZ8jaLRaMv4cmjBxXuaNRkeXrcCyxfMLps1LPf2Xb1kLp49fBy/OHkp8DmH0ek32pipFmjC1ygajZj9WOrkI1XNnXITotzb9+l7b0HvqU9iec6NOGaqATrxSiMWaH9s/KjGexrnOVXj9dUD/BKvNOFraGho1BH8CF8nXmloaGg0CGIhfCL6IRFdIKJBj8+JiP4rEZ0kol8S0Zo4jqtRP9DZmfFC308NFeIK2v4IwHcB/LPH55sA3Gz/Ww/g/7X/19DQEr2YEeV+aj96YyEWwmeMvU1E1/tsshnAPzMrYNBNRAuIaClj7KzPdzQaBFqiFy/C3M/+0ZTTZSsTIoO2GISdVHSj9NKjXLLM5QBOC78n7b+5CJ+ItgLYCgArV+oHXi8IeuFrQaJXS5aw3/0UiT6dzXXYminRRNs/msKjO99FOsvQnCDP/rdyKQcAkUi/lp5PJVFVOnzG2E4AOwFLpVPh09GIAWHcC9Xe1KLWXE5e95Nfx9W0CfnlMhnQOqcl9nM5MJDEDC/LkGU44FFsTi7lcGjwbKiCbeVcqdQDykX4ZwCsEH5vs/+mUecI666pZBelINSiy0l1P/l1qCwpA0Dqykzs53Fp6qrrd/LYTi7l0LF0Xt4kC8A1AagmsKtp03NS0Sgf4b8M4BtE9AKsYO2k9t/XF7yW1JV014jnBEBp8YZZVdSCyykMxOtIJAzcdcu1eHP4ArImi+W65PvZP5rCm8MXnM+bE4QH1rQpvyuXckhdmXFNsgcGkjg4kHRNAKoJjAHY35+sSGZyLSAWwieiPQDuArCIiJIA/gOAZgBgjH0PwE8BfAnASQBXAPyvcRxXo7QIS4h+Lo9C3DVhWgIGfQYgV3wsYQCMuZb84udBboBqdzmFheo64vJ9q8ZA98gEMnZ3FwLw0NoVvscQSzn0j6Zyk5NBGDozmbfKEicwkFW+GQCy2dpYhVUCcal0Hg34nAH4ehzH0igPovitg1weYd01Qf5Yv3OSP9uyps11ToBl/YndnaK4aarZ5RQF8nXEdV2qMdA6pwUGEQDmPJMo5/n8ExtwYCCJ/f1JfJCcdD4jIifesGVNGxiAzmXzsePVoZpfhZUaVRW01ageRPFbx+HyUPljxeP2j6bw7OHjnuckny8DXO4LMJbnuqgHN021QB4DrXNasOPVIWRNhoRB2HZfR0Gru+6RCWfC5siaDNtfGXKt2rasaYt1JVmv0ISvoUQUEhetMTEoF/RCiZ+r/LGJhHVceTIwFP1bVdUlt6xp8/Xh14Obplogu4u6Ryac52WaLFRAWLWC27BqIRIGOa4hILdSE3/uHpnA1+++KfRzrDXlVVzQhN9AiGLRFOK35kG1AwNJPL7xeuw68jGyJsOs5vwXSn7hHt94PQwiMJbThsMu7CdOBgaAz920KK/jktf5ytvI1xiFIAqJKzQSxPs5fG7KeY4mwkk+VavKr999E3Zs7sS2lwaRNZkz4YtxGdHFExa1qLyKA5rwawiFEkv/aMrxhWay4S2aKIQovkAzGRM73xkBN8pm0u4Xqn80hR2vDOG3adP5fNeRj2EyBiKH55G2ddtb1rS5rHev9nqFIMw9jRI7aBRL0QtiW0aDLH2/Qd6ST/H+e60qH1u/0tHkT02nMXT2U2zqXAoAzkSw7aVBZ9sw2LBqIZoMQjpruZwaxaWnCb9GUCixBPnG4zq3D05fBhHBAIMhKCYAwBBeqP7RFB79gXUdHESAyZhDDgkCsiwnsetcNt8JznnJ7Qq5P2G/I7on5MkriqVY7yuBqG0ZVfffa1XJf+bb9576BFvWtMG0V4QZk+GZF49i9ZK54e+tHVC2/o92nbX6HDXhlwFxDJBCl6Cyb5yQ7/8u5nxlAjcIWLNyAd5PTiKdMUEEPHHHDXkEKaKrvRW/PDPpWHafv/lavHbsvPUiZ0xse2kQJlMrPfi5npEafYvBXq9rCXtPp6bTnu6JsLGORlgJRG3L6OXCCasGY3AncmUZQidddY9MIJO19uEn41TlFtTyc9SEX2LENUCC6qN4kZor2cYgPLR2BR7wSUqRrbSg7WUVhcmA3lMpJAzLsjdNhh+9ewpf7FiCrvZWpa/1/dOXsf3+TqSuzDjX9faJi/aEQY71fzVt4qDwQrvONWGgySCXEifo3och6/7RFHYd+dj5neB2T4SNdTSCzzhqW8ao6i7V/i9NXcXPjp13tpFt9WISAr1yC2r5OWrCLzHiGiBB9VG8SC1q8NXli88y7O4Zw4GBpDLoyrXWzU2Gy2pnALKm9ZOoouhqb0XqygzshbSDrK3i4OqODasWOufcOqcF/+HlQZh2oa99faedCUi+t1/87GL8wYoFznU+98ZJV1zh2cPHXf7/MPeme2TC5Z5S+XvDxDpUssXn3jjpqSCqRUQda35j2sutI46L7pEJ3LX6Orx5/KJzX8VM3mITAlXvbq1nXWvCLzHiHCAqYgkzoUQJvvLz5T5rmbCBfMv6zluudSzf3lNWww0GK5XelLTvG1YtxKxmAzNpEyZyEsvWOS15L+fX774J/aMpmALhZkzmnMuGVQvRlDAcl9Wbwxfw5J03AgCee+MkWue0oMWejEwGHDlxCb2nPsl78f2W8q1zWpzzJYNw92euw/C5qcgELZPVjleHnFUUiJDJmjCIsGNzZ02XBo6ayCVvH8aAGT435QRrZzUb2P7lDsdgELctNiFQ9e4Wol6rJmjCLzFKPUDitjhETf3+/qQy6CYrcg4fO49ZzZbvnVvvBOCzS+ehc/l8l0tIJj7Rsle9nN0jExD4HgblLOyu9lY82NWG3T1jAICMreoRa65su68DhwbP4siJS8rJSwWZdLbd14Gh8Uns7R3Da8fO47Vj52EQIgfP5ZwDk8EuUWxdoMkstcnqJXOd+1yLpFIMgki6fzSFbS8NOrr8mbSJ1JUZfP3um/L2Vey74Sf1rdVnogm/DCjlACl0QvHz+/Pz5YlLfPnMP1OtAq6mTbz+qwuOq4YB+GVyEsPnp/IKZnndD1Emx10eLgubgC985jrX+c+blRvCJqzqjCJhpK7M4Ol7b0HPyERoCZ5MOqkrM7YSJLdNFBcdD2ynMyaaEoS7V1+HJoOQybqTiQArSUlVKKxWCSYqnHIMLL+gG8+29lKAyQjzbgQJFGqZ3FXQhF+DkAdp1EEZNpAsS+HEbZ9/YgO+/9ZHTsCMZ1SKCGtRA1aiDpdiMgDbXx500ua33deBwfFJ7O9P4vCH560KjLYbhARJnUHAormzlFYdE/6p7iOQs6hVGu2DA0nX+YZRO3Fw8gYsi/61Y+eRSBBg69TFfbY0GyBEq/NTL+gfTWHHq0MwGYNhEB7feL1jaABwyYsJVjxlx+ZO33vj927UuuKmEGjCrzHE0a80SiDZa9uu9lb8wYoFOPzheYe0iABiQFOTunaN3zVxnywAx/LlEwZXxXApXjprUTcDcllasFYIckmFrvZWfOsnR+3vWN/l5O1VTXPbfR0w7eNnTIbhc9YqZV9/0pGa3nvrYjx5542hrMYLUk14SwrIXJMPAbjjZiuDGLDkhVEVWdWCQs9RHGtgzEnGa7GDsa5s65vzs62jotYVN4VAE36N4eBA0pUEJCtPOFR+aO4vd0k1EwbOXJ5G/2hKOdi9/KD9oymMX552XBMmLO5NGITtX+4I1a2IQ1bCGAZZyVf2hNE6pwX/5fBxhyAtyafhTABAfvld8ZjudYf1u/yy87+nMyb29o45k47JgGdePIq/+7Pb8Idt89F7KgXGLNnok3feGEqnfd3cWa7jE+yANnKTm5xBzGMPmzqXhlZk8W0qOSEUYzWLY02U46YzprX6iTnbutYVN4VAE36MKPXL1j+awr6+064koF+czFeeAPmBVTF5iWc08lLEL7w3hoMDSdekwMmre2RC+XfROr6tbR6OnpmEyQDGLIllVGUQ99Mb9jJdnDAODCRd/u7rF/4O/uqOVRgan8S+vtPOxODVXGPLmjbst3u4NicIncvmY2h80tHtJxIGTNNE1rQmrMXzrgEw6Xw/y4BnhBUIYN3T77/1EX7+qwuu+6qyGsXVQVPCym3gCWS84FzHsvku9wVX8fSe+sTJHg0T0AzTP7aUKMZqVimZRLnlA9LKrVjUuuKmEGjCjwnl8Ad2j0y4qgYC+cFDUU4oWku88JSY0cj3ZzJrtfDMS4OOjHL7lzsc0pGvx5VRmTXRsXw+hs9PFWQpeU0qQM5Kl/3nJy/+BjteHcLzT2wIRQJd7a3Ys3VjviQyYeDhdSvQuWw+tr8yhKxpNdK4a/V1eGP4guMG4qsNEUSE1z88Dz4P8SbgXlK+PV/zLhkgj50HpFr+ogzVzyIN2z+2lIhDGcPPWbVKbARSLiU04ccEkQTljNC4IOrOOcRSwV5uHNlaEhUw/OVkLEdqM7Zbw8tSU2U8yn7zMAg7ST6wpg0/ti10Dt679O+/cpvL5RGkPBKTsbirIHVlxpVmn7oygxe2bsTBgaTTXGP7K0POfU/YaqHDUoanSqcN5HICvCBbxaL7Qna5+VmkJO330tRVJ7krjrEYZgUbp9UcZpVYzKpaB201CoacBCRmhKoQtV4N3/bBrjbs6RlTlgqWyUzUJ3NrSbRw+aTwxvAFF3kBwOJ513ha7X765CiIsvwn+58o+xR7l/rFLMQJ4czlaSQMcmXubr+/U2mVy+eyt3cM1827Bk/ZyV1vDV9wLGrDyNEt/66qjn+YEg/cfXFwIIm9vWPY0zOG/X2nHReN1z0SXUeJBOHN4xdx+MPznvcjCqKQY7mkjOL9TShcgWFiRzpoq1EQutpbXWScFTJCRQS18ZOhIrJZze7gFWBZkVPTaaelnIqk5UmBW/KD45+6ApsGAdfOneVLEnG81GGX/9z1xOV4/Fx5c2vZvz2Tzo9ZADlVjmgJ87IOflYplwvOZEy0nJ/CU7Y656G1K7Dbft6m4nkfEALsgLd2n0+gfEXB/3ZgIOlo/2eyDDteGcK2L3t3jhJdRx+cvuwUoFPFcEo5OZcDXJPP72/GZHjmpUEkCKHeK0AHbTWKxBbbKvMaQIWUKlYlAYkKDiBHZNzN3OTTUo4Pcr692CsUsKSVhkHY894YDIPQsXQeWue0lOTlDrv8d6mKjJy6RbTy/RQePBjKr9kg6x4xIbnHbwLzIrsH1rT5yif39yfzJlIvYhk+N4W9vVYA+qBdu0h20XyQnMRXd3UHWtcA8J3XT7gUTHIMJ+rzrCZyVL1HgDXpcjmt+Nz93Hw6aKtRMGSVgZidChRWqlh+0aam0/gvr1nZhj0ff4Jbl8x1kT2QU8r4neOzh4875QY4muxaMa/b2nozy/BBchIfJI8CCN9cIgq8iFZ2eYkv5sGBpGNZ89K2X7/7Jk+FB7+/4n2M4uLwIjs/wugescrvAtZzlgu7ydcqlwsQ1T1izMaPsMXmI+Kx77l1sVN9tFCyriZyFN8jQq6cfZPUu1hVnynKSrXSEtdSQBN+zOADQzXQZP37g11tgSVk5UnERQwZ07HQuavDQPAk0tXe6pQb4D5o0Qf6zomLTjcqjkODZ8tW1MvLXyy7S1QE7Kfw8CKsMOn1fo05VAR+5vK0qwGIV5IWALteUG7qJcoFgPd8zb+ukeqeic1HEgYFuufColy++SDIE7B4bUDOoi/GDRW1THitTA6a8EsAeaAdHEg6gyFKOVgO0f8uEgMHzzzc1Lk09Est+6Ah6OflsgkAHPdRORD0ooaxNlXk5EXOYctMhHmRZaJ4ZN3KPKKQn7esvmKwXDz8mF3trYEqKJdU1m4+Algurz3vjTmkVY2ISpZBz1/8vdA2ht0jQpcznzLh/PxrRe2jCb8EcPmTDcIL743BZHCaeYuV/aIMFtH/TkQgMDCWn6UZFqIPWpb/7fyLtdjdM+bECspZsle+TpWkMS5rM+5gpEy8yxbMziN7MWmNr/LEqp+mneg1OD7prACDrlclleVupaDeBpWAmC/ile/hhzCuQAAFtzFsndPicnf6xT+qLaDtB034JQC3QA4MJK1AHE/OSecPBq/BIr4QotUua7yLWUaK6hAx45a/dI+tX1mx2ux/fPO1+PmvLiBrMux4dShar9IIiDsYGbQ/OQN6T08uy7nJIMddlzWZ81nYhvMqq1fV20BccfJzirLiLBbipGcoAuyFHlM2nrasaQvVxlCF1JUZpwk74B9vq6aAdhBiIXwi+lMA3wGQALCLMfZt6fPHAfxnAGfsP32XMbYrjmNXK7hUkAkuGFUpV9Vg2d0z5hQTU+m3ZfdGHOfJM24rbaGUo+m6iLiDkV3trco6OBz8ecsknLoygx2bO13PPaqqRjU25N4GiYSBvX2nkc0yNCUIBLeMEVDHn+KEOOnBroxJCilxFMgyTV5nqVAijhJvq6aAdhCKJnwiSgB4DsAXASQB9BLRy4yxY9Kmexlj3yj2eNWCMFaQ6Jrg3YxUvlw56CQGZoFotdeLOc9qsFAKUTIViziDkaJmX6yDIx5LXFWJFUW72luxeslcfP+tjxyllJdLKyzEGMDBgSR6RiZw8uJvAFiVQ3mwXyVfLdVE6xd0LeRYomUvChc6l813ri9IHCEjKolXS0A7CHFY+OsAnGSMjQAAEb0AYDMAmfDrBmEj+H6Dxst3/9wbJ/Pqtvjpt8OeZ1BAslosFFlzH6SOqDaE8edycvCqA/T2iYtWITogVpfWgYFknvqKJwh7yVdLMdHGMd68OogZZGWfb+pc6ooN8GJ1Uc+zVsZdWMRB+MsBnBZ+TwJYr9huCxF9HsBxAP8HY+y0YpuagMsPGxAMEweNZ5u7jLtA1qxmA79NWxmhf3zzIqxftbCgFyNKMKlaBnc1TT6FIMxqSRwHcms+eYXDre8Dgt89rFpI3J7vV0RLgrD9/k4Mjk/i0tRVHBiwktjKcf+LGW+q7HO5dHJQbMzr2mpFXlkoyhW0fQXAHsbYVSJ6EsA/AfiCvBERbQWwFQBWrqzeRs68DRtj4X2tqkHalMj5CEUt+eMbr8f33h4BA/D2iUv4U4UvOAyqyVUTBdUy+RQCVWBdLGAWtOrasMrqtjUjFIojsuSVmWw4v7rqGLJyrGPpPDx8+0qsXjLXVRiO1+xR9YitFqiyz70C1uLYD7r3tSSvLBRGDPs4A0AU+LYhF5wFADDGJhhjvO3PLgBdqh0xxnYyxtYyxtZee+21MZxa/HC1YSOrkUUihMtFHqRD45O5bk2Stn7o7Keu3w8Nni3oXDn5/Ns/WV2Xgzdu9I+m8NwbJ9E/mipqP13trQ5hfnVXN/7xZ8P46q5ux3qULU/5uw+tXeGUVSBYtfIdeaXd9MbrHMXgpWzdPv/EBtxz62IwBhw9M4kdrw45pUA40lmWd07VBj55ie8dv+d8jKvGftC9D/rcD3GNnVIjDgu/F8DNRHQDLKJ/BMBj4gZEtJQxxlnrfgAfxnDcssBraWwyq0zuQ2tXYPmC2Z4p8/y7srXNCz7Jhdb6R1OY3Zxw7aeYpKdatpbLibisuyC3XZhVl5wfcd28a9B0bgqZjOnb9MYreCkeg0tdATjbNTflkr6aE9ESlCqBoNiY+Hf+TsnlwFX3vtAVcS2tDIomfMZYhoi+AeBfYMkyf8gYGyKiHQD6GGMvA/gbIrofQAbAJwAeL/a45UDQ0pgnuKgGnCqhRF7qy4XW3Ek5uWV3GC18WN9jvfsoC0UcyTNBvmVVLoVfEJ0reV7/8DyaDMJtbfOdzmKq4mAuuSOAVdf9Lv5hy+87x+gekVpJUq4HMK/UGVXNUimoDBnV+wog75n4VYAtJH7RcIlXjLGfAvip9Ldtws//HsC/j+NY5YTqQYpFuvyUN4ZHlylxe3k/YuliMhn+pGNJaLIPY2HUkiVSbsQR7wjrWw6z6pLzI7Imy+ssJhcH23Zfh1PrHwBOXvi1U6KBX6PcSlI8Jy/UipHg5ZJR9YjgVn8cCWe1FCvTmbY+8HqQosUk/56zsKz6HWIJXhnyi1/owAlrYdSSJVJuxKEOUj2/YlxqqtWkWFNHfJ5X01Zc6LNL57lKXotF7wq5xjCNZaoFXu9PmOAt4J9wFtRNrVaUZZrwfeD1IL0s5WITSrra/bM0vRB2oqglS6QSKDbeEfeL77U/cb9yl7W/+twNLsJf+DstLktWvsYgq9YlQVY0lqkmcvO6X34r6TAJZ2FWxrUSK9OEHwDVg5RfgmcPH3eKlxX6wsudsFRZmn7nGOa4tWSJ1CrifvGDCPrOW67Fa3ZV06zJMHd2M576/CrsfGcEjAEvvj8OQq5wX5AMEXDX1nHJOWOse1MqqO6/vCIPuxLgkOsfie97rUETfgSIAdmWJssXqlJNBA0ETu48SAag6PoxYYmmViwRjXyo3CtvHb/ofM5LAHNik5O35PEku/gODiRxYCCZZ8nyVWfH0nn40bunam6FKNam4hNfmJUAh1gixWTAkRNqlVQtQBN+ALxUN/wl+MXJS5Fm/v7RFB79Qbcr0eWhtSvKXj9Go/ZwUOiRm86YODR41tXZ6qG1K5yxJxKUV1McuUDY4JlJpatDrA1UzT58Fby6ickCCsDbGOIrY7FLXLWucIKgCd8D/aMpp8pgJptfxjV1ZQZP33sLek99Emnm7x6ZyEt0Eav6he2EpdFY6B9NYV/faccoSCQMdCydh3c/mgBg+dQfsFeLoutOLq8tQpZ//jI56VRn5ROESnnkpXKpJLxiEd0j7m5iqoq1YfbT1d7qvO+1tsIR0dCE7/VwlU2ShTKuvFkIgMgz/4ZVC/MSXWT1BZCvANJobHCJJmBZ83feci1+9O4pZE1LDSY3rQ/rWuTjjCcBGrCKj4kr1aglCsoNv/MR3TG8Yi0A5WQVdF31EANrWML3erhyXW3AesFami03zpvDF/D6h+ddzUKizPxd7VafUlWiCz9+Nb1MGpWDX6b2dXNnOW5AxhiGxicjWdzunBGrDLPBGFqareJjQI4Uw6hcKjlG/eTGMkkD3vLLMLLlWo+BNSzheyVpyKnpTYlciV7AqlXvdLAKkYwlg7/EfH8qLb/TS1PRIUujMaCa+GXiEssvcHWXTGJ+rg4+/nneSJO9UgDys1MBq8+uKFqoFtdGkNxYJGm/ySoO2XK1J6k1LOGrHq74EvC62uLSVq5VzxjwwenLTh/YMEtosZ8pGMt7ScVemiZQVPMLjdqFV5a3OMb4BHDm8rTTN1kuBezn6hAbpwPWSiF1ZSZPhsi19yaDI/GsluCtqomQ3/n4kXqxLptaWJ03LOF7PVwxeLri9+a4vsNT08X2dD87dh5vHr+IPV+LZlXxwK3s9xd7aRpk9dasdqtBI36EsTa5kcFlvvK2gS4KMZhJ6iYoJJQIAXLjdXB8EssXzC7V5YdCoQS7ZU2bZ92gYlw2tZDJ3rCED3j3APVq6s0/3/HKkCubMYpVJU4oYMzV4k7eRlUvpRqtBo34EcXa9NrWb9IQg8A8ULupc6mzD1Hlw2WZ3MIHgB/3nYapcCGVE2EJ1ktaHbYLlmhw8eOqnkktZLI3NOGr0NUe3NT7mFSvPqxVpQogqdLmxW1qwWrQKA2iWJteGaZyqQ5OXlPTaRhEgB2olVsCPv/EBqem/+olc53v7DrysTVR2IGsSo7JMAQbVNAw6Lx394zhGTtpq1nR9L3WVDx1S/jFuEH8BtLBgSTSQjeim6QStI4MLG0qG1CrVhWqcxY7DlW71aBRGYRp1ycmTQHIs9a5pDN1Zcblt5cNFa7OETXtlU4QDEOwUQsaiugfTTlkD8D13odV8VSbO7YuCb9Q3574cLwGEpO+c8Oi33EpbbhVxQNdcgPqIO2/fM61YDVolB9hxri8Ojw0eNZVL58BME3mlE3gfzcZ8L4gRuCQXZLVkCAoqtvE3zkKLWjI5dmiSAOwmh4B4Sa6agzi1iXhF+IGUT0cbmWLJL1lTRv2951GOsuQMIC3jl90GlRw+WbqyoyyyJSs0hFfmCBXUKUHikZ1IcwYl8luU+dSV2Y4AU4tKCtjN4fXjp3HW4IYIaoaphAUYg0XmyylOqaYeCkiYQB/t/m20Ndfje7YuiT8QoInXg9HNaD2bN2I7pEJfHD6Ml47dt7SzGcZdveM4cBAEo9vvN7xj3r592cyJvb05ILCtRDw0agehFXxyGTH/fGtc1pctaC4Dl9U5MxkTOx4ZQgP374yz79fCrIvxBouJlnK65h8n/JqPmz3OY5qfKfrkvABqy8o2f+HGTheD0ceUAcGkli+YDZa57TgzeELrkHBYDWi4KVp5ZR3fgxR1hmmk5aGhoywrj5VzIgbMoPjk2hKGMhmc+6Ovb1jLgXaB8lJDI0PKlescaJQa7gYUvU6ZuucFhhk+fp5AmZLc3hVD0c1umPrjvDlWfuBkA8pjLQtYZCV0Zi16uqYgn+PB4NMlpM3m3Yii3wMLvuUJZnadaMRBUHjJUy8qMkgPLJupcswOnrmKETXNa/XQwgX7CwEhRJ3MaSqOiYPdJvMuuYn7rgBc2c3F0zY1fZO1x3hR7EUVB3u+T6A3MPiA+qt4Qt471QKgP0SCAEcLn9758QlZ/8G5Vfm4/t8QCiWVk0DQqM+sLtnDM+8eBRZZgUa/+7PbnPcEeI7kjUZli2Y7Vj9Ftm595Ww+99y3/XwuSk8e/g4NnUujeTi8EMxxF2oMkZ1TFdfaVgNZUTFXJT9VyPqjvDDWgpiYIYP6NVL5jqWT8IgfHbpPDx8+0qsXjIXZy5Po2805drHPbcuxh+sWODyj/ae+kTZJFpGtc38GvWD/tEU/s+fHAUPOWYZ8MyLRwFYmdtiAx9ROswnAg4u29yxudMh9t09Y/jbn1j74sZNnKRf7DsRNRYgHzOIP3gzlTjaPFZi4qg7wg9rKYhFyjImw7aXBvHw7StyBaWyDB8kJ/FB8igMK/7q8tcnCHjyzhuLVgRoaMSNAwNJmNLfsgwuonp84/X4gZ1Etf2VIQDAmcvTjk/fS3Z5aPCsa79ik/RqQNTsW/ld9HuH85qpFBHPqJRks+4IHwhnKWxYtRAJg5yHZ7JcI5LfSnIseYnb5GO9R1UEaGjEDVL8zQBcWabvjkw4GvOZjIlnXjwKBih9+iI2dS51uS03dS4tyTUUCi8LfXfPmJNxLK7kvaScXklcoi7fsFdHhTSCqZRksy4JPwy62luxY3Ony+rhjUi+/9ZH+JndGFpEgoBH1q1Ex7L5TlGzsA+pGjW5GvWJjmXz8/9ItrDAFgosnncNgJwahyeRij59FVYvmYumBCGTZWhKEFYvmVuCKygcKgtddkP9yWcXF6wImtVsOC7bJ+64oWC5qp/rqJSegIYjfPFmPrZ+paNLFm/uzr9Yi909Y9jbO4ah8UmYJhyffJB14IWgkgsaGoVCJojUlRkQ3C5IkwFkMsd6B4A3hy8gnWUwDGtjxoIzSLtHJhx1GjNZVRousoUuu6HOf/rb0HE+WdQRV50rL9dRqT0BDUX4fuULZDy23kqykB+6XwMFv5m5q92/5IKGRiFQiQ8cS1QoowBYhC5a73u2bnT3bVa0SpRRjclEQZDdUFyIEVSHyIsrxO2LuRcq7im1JyAWwieiPwXwHQAJALsYY9+WPp8F4J8BdAGYAPAwY+xUHMeOgjA3M2g5Jevyxy9Po99W7zy6812ks1ZVvT1bNzrHFC2vUiewaNQ/xDGqEh/sfXKjk+9x4vwU+kdTYMxKHhJJiWeVZrKmnT/CMDg+6X1gVGcyURB4UJn78MUgs1cNHr/Me/Ha474XpZ5QiyZ8IkoAeA7AFwEkAfQS0cuMsWPCZn8NIMUYu4mIHgHwDwAeLvbYURFGcsWr47U0Gdj+5Q6lj+75JzY4ltGe96xyCn9887WYsR2hM1mG7731Ed45cdH13Vq0jjSqC7Llue2+jjzxASegAwNJZd0mERtWLUSTQZjJWgHd/f3JwIJotSgp5it2jiDXiVdSVlgPQaEo9YQah4W/DsBJxtgIABDRCwA2AxAJfzOA7fbP+wF8l4iIMSbpX0oLv5u5u2cM33rxqJMlO5Mxsbd3TDnLy5ZROmPiwqe/dR3rwqe/VbaoqzXrSKO6IFueqSszeOKOG5xyHi2Kdp3ZrInlHoHYrvZWPLR2BXb3jIHB2vbgQBIHBpJOaRJ+3GLGbLVJksNY8JVq3l7KCTUOwl8O4LTwexLAeq9tGGMZIpoEsBDAJXEjItoKYCsArFxZGm2v6mZyfa08/Vw37xq0nJ/Ks8j7R1MYvzztFJtqbjLw8O0r8eG5IWfbh29fieHzud9F+ZacuaehAbg7M3lVZJSD/1PTafzo3VPK2k1hV5MPrGlzNUTf2zsGnn/1477Tvk0/wl5XtUmSgyx41aqoHlboVRW0ZYztBLATANauXRu79e/Xa9aU2L7JAJ6680Y8deeNru/Ig+LhdSucQSEHgsTKhKWuNqhR2xDHldgsXKURF4P/u4587OjrmVC7KWg1K/qzxW3PXJ7Gnp4xZ9t0ljmKn0Kt2mqUJKvuj2jBy9VsVSqdKAld1YI4CP8MgBXC723231TbJImoCcB8WMHbsiFMr9mZjGU1feEz1+GpO28EkL+U9Vsqy6sH/nu5loIatQtxXAH+BCsG/xljIAJIIakUx6PY2vB7b48AsDTpbw5fcDLGuUGzv++0E4/ibf3kQn9RUK2Wsfy++lWz9XrHRVTjSkZGHITfC+BmIroBFrE/AuAxaZuXAfwlgHcBPAjg5+X23/tZGaqZW7W861w235V+HnbwVuuA16geiEaHyazMWK+xIm8LhTtHhFdDDwD42bHzePP4RZf7gss14/LhV5uyJ6isglc12yBU40pGRtGEb/vkvwHgX2DJMn/IGBsioh0A+hhjLwP47wD+BxGdBPAJrEmhbFD53L2qWHLIzUp228tcg4LTz2VU24DXqD6IY8TPhy9u++zh4zhy4lKeO4eDE9uZy9OO1aqCl/tChpeEMez1VcO497LCxUngP37lNieuoSpT4YVaMOxi8eEzxn4K4KfS37YJP/8WwENxHCsq/HzuXtvzl05c3nHIJWXDoloGvEb1wat5vepzcVX69L23oPfUJ57p+eK4NyhXPoEALJ43Cxd/fRVZ2+j3cyHVgqsiLGRDjpd5lmNsAHDQlrUeECZCP9SCYVdVQdtSIKw8DVBrnIfGJ7G3dwxZ03opDArXwFhDIwyCyDTIIvXqMSuP+3tuXYyf/+oCTMbQlDBwW9sCvDF8AQQr4EuAZ8kPlauC/71aic3LbSO7w46csPr5ygmR48KqKIx7JmjSrhbUPeFHWWbJsz9XMhiGgaxpojlhNSoPSkzR0AiLIDL1+lw2TGR3izzun7zzRty1+jqnPtRhuxczwMkeniU/5H21zmmpaovfbxJVucNM0+pgxzt6tc5pwXcOH3fuTyLhzxu1tAKqe8KPsszym/35wPBbIWhoRIUfmTYlDNx5y7VoShjIZHJF92TDhEs0mwzLIOGxJd6j9rp512D43BR2vDqU56IkWAFfv5If8jskHv9q2krUqqZ3Iih4qnKHiSul7pEJJ3MZAFb+3pyijldNqHvCB8L7z8PM/tqVoxEn/Mh0JmPi8LHzSBgA2aS849UhbLuvw5kkiMjR4c9kGXb3WKU+tt3Xge2vDNkdrCbxmlTumwBnxdqxbD52vDqUl4QkV4oU36GmhGUYMQD7+k6HFjCUA2FW9UGGIC83AQAnL/waj/6gG3u+prbcayFYy9EQhB8EeXD7zf5yRL9aBrlGbcArAAtAKRZggB0/ssiHl1MQFT2i5c59zocGz7raFcq497OL8ZTQsU1MGhw+N4Vtdk0pr+SvB7vasIeXY6iyMslhV/VehmBXu1Vu4nkhAc2vu1UtBGs5Gp7wvfxvfm3OasVfp1F5iATvRaReYoF9faeRyTJXu0IGoHVOi4usVi+Z6xTz4/khmzqX4l8/cndo4iAAv01nXX8TE69cbfzSaqLbsqYNB+1yDNVi1apWJf2jqYI6Uj2wps1VYgKAbw+LWlHhNTzhe/nfVFmKxTY90KhfqCx3lzTSIGRtSS/gJlJVQTSuBRfdi4AlC9728iAGxycd8QD/t2VNm3MOAPCFz1yX58oBrEnjFycvoffUJ3mqn/HL064yI4ahbuNXbVatyhADULBx1tXeiodvX+lY+QTk5TnUIhqe8IP8byrrq1b8dRrlgdeqz0XkWeYKlhoGOWPHawxy9+K7H7mDiJksw56eMezvT7oyZEWrNhf4JZj2sROG1f7w6JlJT9VPk0FWkDhrwiD/Nn7VZNV6qZmKMc7mzsrRI19Z1ToanvCDLBV5IA2OT+KBNW1O2jl/wQ4OJMEALdlsQHiRjViGI2EQQOQQ6Y7NnS7i3HZfhyMDlv3lOzZ3On0aOBjUGbLy+ZDJ8Oj6lVi2YDZa57RgaHwSH5791JVx7tLsmwwPr1uB5Qtm19SK1mvSLLStaP9oCruOfOz8zi38Wo/fNTzhA/6WitzhymkHR+Q0i370B91OgOzHvWPY++Qf1eRg0CgMvtJKoQwHoE5W6h9NOVZ0z8gEhmyjgm/Dey8fHEji4tRVpxctD9JeTVsZo0/fewu62lvzzocfm9fTMQzCF251B23F7WWjpRLNtqPCy3AL21ZUvha5gm7Cdm3VevxOE34AxIE0fnkae94bg8msJJVtLw3i4dtXuNQQGRM4UGW6ZI3Swk9aKZfhkIn+4EASg2cmHZXNTJbh+Z4xvNB7Gn+3udPp0iS6axbNnYWT56fw3imrtSaDVfmyZ2QCe7ZuVJLfc2+cdI6RNRne+NUFpyKs3yrX67NqFC+oDLcwbUVV1yLm5PAVWerKTE2sdvzQUIQfZJH4VdHjL9ve3tPOzG+a3DdKruV2lIJLGvUBmWxki18OevaPplwrQxlZk+GZF4+6LFKRmAwip049x0yWeRobrXNaQASnyU/WZK5Vgd8qV/VZLbp6EgkDZ+we1OK5qq5F1Z2ufzRV8/G7hiD8/tGUI1vLZKPVLBHB/anbXhqEaTI02bXCv3bHDfiB3YiiRVhCa9QX+DgS4zcqiFaxqvkNADx7+LiL7HlBs3OfXnX+lmVwEamrZj5jjqEhZ86qhAY7Xh1yau3zieLICbdSJwpqJdmIP4uDA0ns7TvtBLt5ElX/aMqz5Lk80VWbMqkQ1D3hi7XA+YuhskjCWizcn/q9tz7Cz391AXveGwMRYf41Tbjput/Fv9t0a00OBA1/9I+m8OjOd53sy30CaajAyUJufsP7xcq16ZsThL+55xZse3kQGfsYLQlyEalMstvu68Dg+CT29tq12xNWXIlPJvyYYhKWAWDlwjkYnbhSVBerWiK/rvZWHBxIOvd1xn4OAJSxlqDVf9zF0coZC6l7wudELhaKUqWPR7VY3vjVhZwbhzF8ciWN906l8O1DH+KbIUi/mgJeGsHoHplAOpuzpcMSpTyueJCVV6j8/bb56Fg+H53L5iN1ZQY77u/E0PikUvElrxxSV2bQuWw+9tNpmLBq4m9/xSqRIFbA7Fg6z5U5vvXzNzqrjqjqFRHFJjeVE3L6GQN8Yy0iShmvKHcspO4J39W+EMA9ty7Gk3awSr7RYS2W7hF1BiMA9J6yHqDfg6vGgJeGPzasWoiEASfzsilBSn8whzihi5JLIEc+DMDDt1srxrDjgf9d9OVzt07WBLKmSPYWkf3o3VPKMso869dPvRKEWhnLW9a04cd25myTYf0OBMdagNLGK8odC6l7wucaZz643z5xEU/ajclVgRq/my02R5nVbOl7mRAI4wh6cLUS8NJwwzAMwDRhkGVNv/BevgYeyG8+AsaQMRl6T32CLWvaYBCcRuWHBs9icHwyVzBNkliq4PLlw/Ll8962DNbP4kTAs3dFVwRXrxTj1pHPpZRjOY4VsWEYINO0nqONLWvawAB02gXk/HpelyJeUe5YSN0TPqAe3GFutDjIgPwa5Nxiem3oHF58/wwuTlkBt6AHVysBr3qBOFH7tQ70Q/fIBDJ2eyjGrICqF1nKJMjtgRnblSMWRzty4hKamww0GeTUzZHLHsjw8uXv7086+SJP3HEDfvTuKU/rtXVOCwxbtuNn3Yr3T/VZOcZyHKsI/vwYrIYw37djcCazhBaAd1ZuKeMV5Y6F1DXhy+0KxUEZdKPlQbZlTZurBvjg+CT+/iu3AbAe2je/dGtoK6SWAl61DvE5cqtaVQEyCLK8D4zl9Ufe3TOGQ4Nn0bF0nrOtYZDj+zdhWZLzZjXh+2+PAMhNGo+tX4mxT67gyIlLzhjzsvRVvnwADqExxjB3drOnUmjbfR3Y/rK14jUIeHzj9Z7WbRDZlmMsx7GKkJ/f6x+ed1o+cnevmGA57uOuixt+kti4UbeEr5Kmydad342WB9mFqauOrI3XAOeBNq+a4X4o50NuZLjdH7mSBCJphJmoZWLj++bf2d0zhr/9yVEAVhLUU59fhbmzm3Hm8jResJP1DAIGxyfx497TriAig1XjpmPZfLxz4pLzN1k2qTpP0XXUZMs0RaNGpRTa2zvmqI2yDHjXh1DDkG2px3Icqwjx+Z25PI09QuljgwgPrGlzGpfv709iz3tjTi9boPAibNWGuiV8VQVCngUJIPCByYPsurmzXJ9nssxJ2Vb1GtWWe3WAP0dRlmsyYGo67WS67us7jYyZe46AugSCSpfNcWjwrOu4745M4KVv3IH+0ZTjamlKGCDAlbLPMTg+ieULZruSqUSXEZBPOi6VSdbEI+usmjnyectjefG8awBMOp8vnncNhs9PKQm1GtyPca0iRFXRwYEkZuwyE2JdowN2yWf53tdLzK1uCd+vvknQiw2oLbp9facdy4gIrprhzx4+jk2dSz2XxlHgN2k08oRSyLXz5/js4eOO9QwAP3hnBD/811MuH3s6Y+LAQNIiA4+Vodc5bOpc6tr/sbOfon/UKn3gRPUZQ8ey+XkTEGCNrf/r/k5HDGDCWhGoCpxx0nH88HY3NlFDLp+nPJbfPH7R1e+WCxnCllaIC1HcoGEloEH79LomPjnz55IQKppWetKLC3VL+PJDlV8Y+cX2yqwV/7Zn60YcGEji0tRV/Hz4AkzBN/uLk7n+typLIOzA9vOZ1ooErhQIunb5/vKs2EtTV7Fo7ixs6lzqKjOcZYCpyM8gwBWreebFo06glWesehkNf/LZxfiZXX+ely64ms7m3Ccmc7pVyRNQJpv7TDxvUYsvks6J81N4+YNxmMxqx7ftvo7AceIay1/zdk/JKJXLJup4lktL7BBqDYXZp6pBiggxME8AHlq7wtlGjIeE9RJUI+qW8AH/+ibiix12mSb6RA9/mGssYS3TATB1/1uv5gyql8zPZ9rIck6/a1fFa7a/POgQLWBlrd73+0vx4vvjzt+IAGJAU5OBB7va0LlsPgbHJ62m3nY1Siewlzax8+2PHMvcazXALWeTwUXo/Hj8eT997y3oGZlwzlEcL+I+uV5cNGCmptP4nh30BayVpticI4rfvZJGRNTxLG7PixfK+QNe+/S6TrG0eae9+pKrjAL5+Q+1anDVNeGLUC1rD0gt2sJa4X6KDVVwOMrqws9nWg3+1ErB79rl+3to8KwrKxawCotN/MbdsYhbx9u/3OFKflK42GECTjkC1WqA90oAc9e2EcHLaXOXBF8xEnK1eeQAqywP7Gpvxf/y33vy9i1my0YZJ5U0IqKO5w2rFsIgchUvlM/Xa5+q6xw+N+XqM9CSIGy/v9NTulsPBldRhE9EvwdgL4DrAZwC8OeMsZRiuyyAo/avY4yx+4s5bqGQLX55AoiS7aiaPC5NXXW1nuOQNc9+qws/n2k5JHDVCr9rl1/yTZ1LXdYzx8LfyS8hwJhlHctqHsDyoTclDNy6ZC5+mZzMuX/IqrO+eslc7OPadzsgK3amkrFx1cK8McalvV7XoiJBOV4AuNvvRRknlTQioo7nrnZ38cKW5vzz9dqnKqa3TWoqk7bdal7ijnowuIq18L8J4HXG2LeJ6Jv27/9Osd00Y+wPizxW7BAnAD/Lyuu7ABy/3v7+pFOgan/faacuOW9uYdruHk4U8urC67w4Slm8qVbg5UtWveSrl8zFjleG8EHSUqNYkshP877LX/7B8UmQUD+YAHzupkV4+t5bAAAPf/9dh8wZs1Q5YxO/QdbWvvsFZH93VgL/8/p2zJ3dnMuozeTr7PkzVq0SRTy2fiXGJn6Dne+MgDErryDMGPK6p5U0IsKcpzj2efHCqDEHVUxPLo/SnPBvclLpexUHiiX8zQDusn/+JwBvQk34VY+os7ccQMpIlgKfMETLkWBZEFEHTiMHa0X4udzkl7yrvRXbvtyBr+7qdp6p7KtZMm8W/uaeW5xArGEQEjbntzQbLjLmliUvV3DkxCXfoOu+vtOOW+nXV7P44S8+xl997gYYZJU+MJlbZw9E8w9/80u34osdS2Ihn1IFZYuBmDSpUr4Vcr7y97giiihXYyvIbVON9yoKiiX8xYwxLkA+B2Cxx3bXEFEfgAyAbzPGXlRtRERbAWwFgJUrV6o2KRmikrA4MJw6JjafNCfcDap5EFBsXB1l4NSD77BYFDLpyc90+NyUkxwFAH9zzy2uLkZksjwtuzjJ7H1yI549fBy/OHkJsueGP1vxue4Wkntmsgy7jnwMUxgrxWq9a518vCAbU2JNoLjGvt/7XutuGz8EEj4RHQawRPHRt8RfGGOMiLwcmO2MsTNEtArAz4noKGPsI3kjxthOADsBYO3atd7O0BIh7AskNk3IZCzNNDFLt/uFz1zn6hU6fG7KsfTSWYbhc1OhB6xfaYgo36/V5aeIMJOe6nrFZ8r/55UrH1u/Mq+LkaxllyeZp++9Bb2nPnG08gTruYvJO4BVlEt08xmU8+8TrGAxY241Vz0TTRS44ym54nCy8i2uRCz5b7XutvFDIOEzxu71+oyIzhPRUsbYWSJaCuCCxz7O2P+PENGbAP4NgDzCrwW4KiEahNva5uPomUlHlvmHKxa4BomcgXlo8GyedliF3T1jrkzeIL+u33nWgxsoyOUW9nofW7/Sdf/9XnDVJCO2vvMrxtbV3oo9X9vgNB5//VfnnRRaBuCJO27A3NnNru/WM9FEgfysVclvj/4g56rza0RTCOp15QQU79J5GcBfAvi2/f9L8gZE1ArgCmPsKhEtAvA5AP+pyONWDK50dpOhY/l8z7R0IF9RwWui+6F/NIVtLw3mMnkz7vK2Yayb7pGJXGPsdO27gYIsr7Bur6BVgAg5kzWMO06V3PPcGyfx2rHzru3mzm7OC77XM9GEgXjvVHWLOLikGch1r2rk+xYFxRL+twH8mIj+GsAogD8HACJaC+ApxtgTAG4F8H0iMmF1WPs2Y+xYkcetGGTrY8uaNmxZ0+ZJRNyaFN0IQZAVBAaRaykbxpJtndOSqx0DFNzVqJrgR4gq2Z2qcXjYVU//aArbX7Ym3YQtwwwbXL+aNh03z2PrV2LDqoVobjIckpJbF2qon83X775J+XfZ11t2328NoyjCZ4xNALhH8fc+AE/YP/8rgNvkbWoVKkvTqZniAdmNEIQNqxY6CgK5uFNYSzZ1ZcZptGGQW6ddjxCfi5eyI0rw+8BA0lVRcnB8MvAcxFVVxnRngnL3DkN+68J6R9gVqerZqP7euWw+EvbYbk6Qk42sEYyGybSNE9zS7B9N4W9/chT7+5PIZAv3lavcAGGTjLwsxXpIEokK0YWiIo8o94QCfueQ+yInjJxE12Q5eW6jumvCrqq8no1q5WbltVgJcH/1uRsa8r4WCk34BUJcvovVFqP6ysX9GIIe2IsgwqoI6l1t4Acv8ohyTx5Y0+Zk0XL1jqpAm0xmTiaoHWyPUrKjHhF2VeX1bOS/8/3ZGgnsOvIxvtixpOHua6HQhK9AmBf04EDSRfaE4NaGKohugCwDfnbsPN4cvuBk6qoQ1lpsVKvSj9ij3Du5oqRfPXpRxSNmgorf86rwWK/oH01h/PJ0XmMWL/gZOeLfg+rpaHhDE76EMEvQ/tEU9vXluhY1JQgPr13h0nCHhewGANyZuhqFIY7JTtyH6Ca6mrYK4G1Z0+a5klB9z6vCYy0iyCiSG7k/vG5FLLGLrvbgejrFnHe9o+4JP8wDFrfxK68qbiMm0fz52hX4j1IRrCiNHXZs7nRV7WsuUsXR6IO6FNiwaiGaDMKMXTZ5f79F+EEuojAVHmsNfqWGVe9RNmti+YLZofNHgsbu6iVz8ee3r3BVGS3mvBsJdU34Ya11uZa6bLUFbSOrBKLK/1JXZvB3mzsxND6pVHFEIXA9qKMjbE/bh9auwO6eMcv9ls25cMJM6IVapNUIlVEEIPA9CgJPNsyazLPRfP9oCo/ufBfpLENzglw16ws577D5FPWCuib8MA9Y3oYXwBIftqz6UG3jtU9VVUSOQiakIAIvJgGpERHl/vIm11GVT2EqPNYSVEHxMO+RCmL5EFeyoUeyoCiXnckyHIiQdBVWpVXPRlNdE36YB6zaRvb/htlGtU/+AohVEcXvFDIhBbkDwlxzPQ/oqIhyf4tRPtVTAN3rPkR5RwB1kTQOsdCgiLBy2SjnLaOeixXWNeGHecBxbaPa/tnDx3HkxCXPSn+FTkjFXnM9D+ioKOT+NuK9UuWKyOqnqH1fXUXSmFUkzWTMUTOpvquSy0ZBmOdXzzksxFT93KoAa9euZX19fRU5dlzuDm7B8IHj5bJRHUvutRmlcFoc59VI0O4tf0SNSUXd1qtImt/5lPp51fKYIKJ+xtha1Wd1beEXgjjdHX7Wtl8HK14NUKy94qfLj/O8GhGNarWHRZQVYTlcZOV4XvU6JjThS4jb3aEaOEGTSvfIBNI22QPBuvxCrJF6HdAa8UPl4lCNObFPRDZr9fk9c3naqTVVTCKcRjzQhC+hHP47L1mbWJNFrK7op8vXAViNUkO2xIH8rGPxb00G4Z5bF+PN4Qt44b0x7O87DRAVVW+qlKhl901UaMKXUAp3hzygVAWh5BcobHXFuFYkjTToGw1xd4dSFacDci0asybDdDqLjGn17rU6vsXfpjAONJrBpAlfgTiXmV7deVQFoeSaLGHOgWeAprOWysFvReK1DD9oN93OmAxNBuGhAstEaFQfwmTFRs3T8FoFi+NwU+dS9J76BOmM1SScgcBC1NMpNxpNsaYJv8Tw6s4jTypFuZHsrkzW/2qoXnwAeRU/Z7IMu3vGcGAgWffWTiPAy30oZqvKgoAgq1e1Cu4fTbnG4eolc/H8ExtwYCCJ/baMkgh4fOP1VTWm6lmCqYJR6ROod4TpzsNfoH/7J6sDZWzPvXHS1XCle2QCmazddCNrZfWqGrKoXnyx1Kx8jiI5aNQuOKElKFfNlWerMuSyVUV4TRIiutpbXatQcRzyshNd7a1YvmC283fTLmcc1DConAj77tULNOGXGFvWtKElQSBY8kqv7jzyCySDW13/+LNhfHVXt/PS8Bead7c6cuKS63MO8cVPGITxy9NondPi/K2lycDt17ciYRAMFFbqWaP60NXeim33deCPblrktGkMylZVTRIiVIaH13d48TiOrF08rhCojhsHgt69eoJ26ZQYXe2t2LN1o0vhIPdaDQM/X+MDa9owdGYSv0xO5gXG5MbQfIm9570xp8hV6sqM00mIMQbDoFA9XOOCDhiXDv2jKafdY++pT7B6ydzAbNWg/BGVu8frO13trXjijhvwvbdHAFirx0L6K8tlGBqpr0Cc0IQfE/xIi78QxSgCxPo8RISp6bSrvWKTQWhusvXPtgW/u2csr7crwZoQ+MSQujKDr999k0t5QWBl64FbbSqJapl84joP2VA4OJDEsgWzsf3L/lmtoqtG/j0oyDl8bsp17nNnN8P27sNAYf2VxeNG7StQLc+0GqAJPwaEJS1x0PImGlESpbbd1+FUFPze2yPOSwRYS+WH160AAMeC5wWpOLlzNQ7/TiJhSUKfe+MkWue0oClhBa8Mg5yEmVK/IMWoJMRKi3GUnvB7juUkjTgnQTEomUgYjhrLrzorv6eqRvBeQU655adhuwmff2IDNqxaiFnNxQVGC+0rUG0GRaWhCT8GhCUtryYaXiUX5H2krsy4KgrK7RW3rGlzgmfWZpZEjjFLDscAV+OWO2+51nmpmwyCae8znWV44b0xHCyDUqdQlYQfwRR6vl7PMQ7SiDJhxJ1bwd12Zy5P44X3xjz3q6peKbsIvVw3sgBAlhgXm9vS1V5YX4FGk10GQRN+DAhLWl3t6iYaohWpkk7yF6V1TotLUUOwsnBl3bx4LmJBKsCSifLPrps7K/cyZN1aHdULUqyVq/p+oYlufgQThzUsPsdiSSPqhBGHVNBrLB30qefvql4pGQvitqKLksejHJdj2oQJawLmrkW+UiyWaAvpK9BosssgaMKPAVFIy6+JhsrfesDW8bfYFjxX4xCAO25elNdYJehc5BR5fi4Jg6z094zwwkq1UIpxd/iRXiFkoCKYYl9or3tXLGlEnTDiyPZWuQ///iu3+e5Xvk6/6pWq5ymWRx4cn3Rcizyng59XXBm/YbfXhQJz0IQfE8IORK+kFf6iiC8cA1xEweC23kWyV9UrD3Oe8gTAz+PN4Qs4/OF57O4Zw/7+JB7qaivI3cHP68zl6diL0okEE1f5aNW9K5Y0CpkwirWI/dyHXn77DasWYtt9HTg0eBabOpe6VDD9oykcGEg6fWSDssOfe+Ok41qUjZemhIEHu9p8S4bEGTOJY3VRLyiK8InoIQDbAdwKYB1jTFnAnoj+FMB3ACQA7GKMfbuY49Y6xAGo6pfr5YLZsqbN8dPL5REK9THLLwMn8WdeGgQPF8xkTFycuhrZ3SGeV1PCQJNh+YWD2suFfdHL8SKL58PLYPBjh/1enFam3/2RP/NzH4rf4Vm3iYSVg5HOMrz7kXWdj61f6WzDWwvu609i+5fdPWt58F+uF8VVZRemrrrafu7p8Y4R6UBr6VCshT8I4AEA3/fagIgSAJ4D8EUASQC9RPQyY+xYkceuC8iEyWWSHF7aZr99FKJ0kQNwYnAYABbNnRXZ3SGeVzZr4pF1K7FswWzf+i1hXvRyKWbkCQuM+Spcgq6j2HP1i/GIqhpuQXcumx+ojhF7xGaEOE7GzEkfu0cmXDGemYyJwfFJJ6/j0tRVbH95MO/ecFVZ1mR46/hFp9aOVUotvraeGuFRFOEzxj4EAPKp4QJgHYCTjLERe9sXAGwGoAkfwcv9MERRrNJFJqbWOS1IGLkeo02Gu2KnaOX6Wa/yeQUVZAvzopfT+pPPB/AnqijXUez5zKRN7HhlCB+e/RQZk7lUNdyCntUc3EVKfnNFqa/JLOnj1HQ673v7+5PoXDYfBweSrlpM4vWmrszAZMxZYTyybiUuTl3F6x+eB2PeMRcdaC0dyuHDXw7gtPB7EsB61YZEtBXAVgBYubI2suiKtTbjWO4Xq3QRA3sAsOPVIZimpdL4wmeuw1N33ujrr/ealKKeV+ucFisNX6EMUZ2zqpdA1JWNn/9f1rCDMWSyDETkmy1aKsKSA9UfJCeFT5kzSYsWtLxilCFn3f7VH12PXUc+hsksa31qOu1kyYrIZk0cGjzrUkpxebBYVkG8Dx3L5lvZ3IBvNrcOtJYOgYRPRIcBLFF89C3G2EtxngxjbCeAnYDV0zbOfZcCcVmbcSz3C1W6yIE9ALmXmDH84YoFLss+rOXq18LRa/sdrw5ZTax9yEAmEbmXgMqilSdl8blxxdOs5uCqkMPnpiwdOGPY8eqQZ6ZnqQiL7/fZw8fxi5OXnBgLAc61D41PYm/vGLImAstl833u+Zr7XL/YscT5/dnDx13bG5QjdrEEckIRiJXvgzh+grK5daC1NAgkfMbYvUUe4wyAFcLvbfbfah617mvsam/FXauvw8+OnQdgWW2cPFTWaVjLtZCJMCwZ+JHITMZ0CFnViYn/za03t6zhq+lc6Wr5eOKEZzIWanWhcn/Fga72Vjx97y2eRNs/msK+/iSypulbLtvrGuXfN3UuxTsnLjmfbf3jVZg7u9m53iBdvLxv7aqpLMrh0ukFcDMR3QCL6B8B8FgZjlty1LqvsX80hTeHLzi/JxKWn/0BQQkEuIu9hbFcC5kIo9xLLxIhRXYoAMfHfDVtOufOFSQi6e/rO+0bZwhaXci5CXHGGoJUPzwJavzytLJMcaHg0kyVVBOIZolrV03lUaws8ysA/h8A1wL4/4jofcbY/0REy2DJL7/EGMsQ0TcA/AssWeYPGWNDRZ95FaAWB7BIHN0jE06pBQBY2TobQHCxtyA5YKG680Lupfi9qem043/mx31t6JzjY2YApqbTru+8f/oyDh87bxFkQH0Wv9WFPLHFufoLeg4uNZFBVoZsliGRCL73YWJQj61fGVtlSu2qqSyKVen8BMBPFH8fB/Al4fefAvhpMceqVtTSAFZp/rkPHwBOXvwNHt35rtMBqRjVTKHkLVqrYb/Lt/nqrm5k7WAzjwHIPuihs5/mHeudExedbOPxgKJxYV0UQZNeHLV1+D7GhaQ2K6hsf5H5h8EqqXfXFSwrA51p20BQaf4fWrsCz/eMOdukszkrNwxpPXv4uJKMCp0Id/eMOdrthJGrex5EEPzaGADGcjEA2Qc9uznhInQ+OYl9AvzaO4ZJqJKLlsnnHEdtHdmqb0pYpbHJrigZZsVSqRiUTqyqHDTh1xGCSNGLwPf15/ruNidyyg4/S11VrTJqDoCKKHn5ZyCX/ANAWao3zLVxV8Te3jEcO/spDn94Hm8ev+gKdPLVjFgKIMpqJiqZRyVa1XMQ+xfw0tjLF8x2ErDCuNMqFYOqdbFDLUMTfp0gbE0bldW552sbcHAgCQbk1TfxstRFi9oAcNvy+ehYPr+oc+UqGBGmyRy9tx9B+E1Oj61fidSVGRw9M+mZ2u9lRYv7C0NUYbaJo7aOvA/xuYWtKFmOGJRqYq91sUMtQxN+ncCPaIImg0LcL66kJIPw4bkpHD0zGaqGvte58n3yVQMBaGl2672Dyk97uWHOXJ52GryoUvtl8gPy5Zx+ROVVAE91rl3tra4iZYUQrZdSR5X7EKYbWynQP5rCoz/odu7Fnq/5t0PUKD004dcJwta0KbSeu/xyii/t+OVp7PFprCHvw+tcxX3KGbBR66CLx8019bCk6XwRIatYRPITXSZBjTz8CuCp7gOPF2SyuT6zQdfk9QwA6/kOn5tSur0q6S8/OJBzFc5k3HkOtSR2qCdowq8T+FlNxdTaEcnJq449384vuOtVO12VsCQHP+W/R4E42YneIgLwYJdac98/msL45WlldU/VeXSPTDirkpm0dzkDOe4BhJuAvUg7TIeqSvrLZY1Q1afONwA04dcRvEixkCV0FHIK2r+KdMTa6X7Hj2qZ+uUE8CYv2WzO7+133KaEgYfXrXBlsapWH2InMhPwrLMjxj2A/NozXvAibXfGsLpDVSX95VvWtGF/32mkswzNCVLeb43yQhN+gyCqhRyVnPz2XwjpFGKZhskJ4PtWqY66R9yNWrJZE8sXzHZZ06IqiZchJvt3k1n/p67MBAYrE0Z+a0ovuL4ndCGT76vKlVRJf3lXeyv2bN2offVVBE34GkpEIacgOWghpBNlklAlIPnlBMguI7mWvMqVI0+AotqnOZHTwfuVXCiUfPn3Dg4ksa/vtKvBfJj9VdJfrn311QVN+BpKhCWnsK6XqC9+IccXE5BES9hrkhL937womqpRi+jTz2QZTGE/DFa+wCO2Dj6o5EKhBMhdOBmTRXKNaWiI0ISv4Ykw5FTKoGDU4/MEJALyLGG/uALskswElteoRfbpP7J+BS5OXcVrdoVRjs5l850kr+FzU1Zdf/i3c4wKrV/XKBaa8DWKQqVJSJWApLKEZcIP4/+WS0dwn/6WNW1458RFx5/PGJz6+ID1s1zTJw5o/bpGsdCEr1EUKk1CXscPk/zkd95+pSN44tTOtz/C6MSVvHLMqpo+cV6vJnqNQqEJX6NoVJqEVEHZYoOZcumIz920CE/fe4vj09/x6lDeZNA6pwWD45OuAK52u2hUEzTha9Qlip2EZJcPJ3tAPRls6lyaU/oYhEfWrQwludTQKCc04WtoKBAlc/npe2/JCx4vs/X7GhrVBE34GiVBPTS4iJq5rBU0GtUOYgFdcSqFtWvXsr6+vkqfhkYBaNQGF/UwyWnUPoionzG2VvWZtvA1YkejNriodPBaQyMIRqVPQKP+wH3ciYhdsDQ0NEoLbeFrxI5Ka/M1NDTU0ISvURJo94aGRvVBu3Q0NDQ0GgSa8DU0NDQaBJrwNTQ0NBoEmvA1NDQ0GgSa8DU0NDQaBJrwNTQ0NBoEVVtagYguAhgt82EXAbhU5mNWAvo66wuNcJ2NcI1APNfZzhi7VvVB1RJ+JUBEfV41KOoJ+jrrC41wnY1wjUDpr1O7dDQ0NDQaBJrwNTQ0NBoEmvDd2FnpEygT9HXWFxrhOhvhGoESX6f24WtoaGg0CLSFr6GhodEg0ISvoaGh0SBoaMInooeIaIiITCLylEIR0Z8S0TARnSSib5bzHOMAEf0eEb1GRCfs/5V1i4koS0Tv2/9eLvd5FoKgZ0NEs4hor/15DxFdX4HTLBohrvNxIrooPL8nKnGexYKIfkhEF4ho0ONzIqL/at+HXxLRmnKfYxwIcZ13EdGk8Dy3xXJgxljD/gNwK4DVAN4EsNZjmwSAjwCsAtAC4AMAn630uUe8zv8E4Jv2z98E8A8e2/260uca8boCnw2A/x3A9+yfHwGwt9LnXaLrfBzAdyt9rjFc6+cBrAEw6PH5lwAcAkAANgDoqfQ5l+g67wLwatzHbWgLnzH2IWNsOGCzdQBOMsZGGGMzAF4AsLn0ZxcrNgP4J/vnfwLwZ5U7lVgR5tmI174fwD1ERGU8xzhQD2MwFBhjbwP4xGeTzQD+mVnoBrCAiJaW5+ziQ4jrLAkamvBDYjmA08LvSftvtYTFjLGz9s/nACz22O4aIuojom4i+rPynFpRCPNsnG0YYxkAkwBqrclu2DG4xXZz7CeiFeU5tbKjHt7HsNhIRB8Q0SEi6ohjh3Xf4pCIDgNYovjoW4yxl8p9PqWC33WKvzDGGBF5aXHbGWNniGgVgJ8T0VHG2Edxn6tGSfAKgD2MsatE9CSsVc0XKnxOGoVjANb7+Gsi+hKAFwHcXOxO657wGWP3FrmLMwBEa6nN/ltVwe86ieg8ES1ljJ21l78XPPZxxv5/hIjeBPBvYPmOqxVhng3fJklETQDmA5goz+nFhsDrZIyJ17QLVtymHlET72OxYIx9Kvz8UyL6b0S0iDFWVGE17dIJRi+Am4noBiJqgRX4qwkFi4CXAfyl/fNfAshb2RBRKxHNsn9eBOBzAI6V7QwLQ5hnI177gwB+zuyoWA0h8DolP/b9AD4s4/mVEy8D+AtbrbMBwKTgrqwbENESHmsionWwuLp4Q6XS0eoKR8q/AssHeBXAeQD/Yv99GYCfCtt9CcBxWNbutyp93gVc50IArwM4AeAwgN+z/74WwC775z8CcBSWAuQogL+u9HmHvLa8ZwNgB4D77Z+vAbAPwEkA7wFYVelzLtF1/t8Ahuzn9waAz1T6nAu8zj0AzgJI2+/mXwN4CsBT9ucE4Dn7PhyFh7qu2v+FuM5vCM+zG8AfxXFcXVpBQ0NDo0GgXToaGhoaDQJN+BoaGhoNAk34GhoaGg0CTfgaGhoaDQJN+BoaGhoNAk34GhoaGg0CTfgaGhoaDYL/H86R3WceX6YJAAAAAElFTkSuQmCC\n", 47 | "text/plain": [ 48 | "
" 49 | ] 50 | }, 51 | "metadata": { 52 | "needs_background": "light" 53 | }, 54 | "output_type": "display_data" 55 | } 56 | ], 57 | "source": [ 58 | "# show the samples\n", 59 | "import matplotlib.pyplot as plt\n", 60 | "plt.plot(xtns[:, 0], xtns[:, 1], 'C0.')" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "2ff87f41", 66 | "metadata": {}, 67 | "source": [ 68 | "Now let's define the neural network that will learn the score function.\n", 69 | "This is just a simple multi-layer perceptron with LogSigmoid activation function.\n", 70 | "I used logsigmoid because of personal preference, you can also use ReLU." 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "id": "3c2b4113", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "# score_network takes input of 2 + 1 (time) and returns the output of the same size (2)\n", 81 | "score_network = torch.nn.Sequential(\n", 82 | " torch.nn.Linear(3, 64),\n", 83 | " torch.nn.LogSigmoid(),\n", 84 | " torch.nn.Linear(64, 64),\n", 85 | " torch.nn.LogSigmoid(),\n", 86 | " torch.nn.Linear(64, 64),\n", 87 | " torch.nn.LogSigmoid(),\n", 88 | " torch.nn.Linear(64, 2),\n", 89 | ")" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "id": "4e55153d", 95 | "metadata": {}, 96 | "source": [ 97 | "Now let's implement the denoising score matching function below,\n", 98 | "$$\\begin{equation}\n", 99 | "\\mathcal{L}(\\theta) = \\int_0^1\\lambda (t) \\mathbb{E}_{\\mathbf{x}(0)}\\mathbb{E}_{\\mathbf{x}(t)|\\mathbf{x}(0)}\\left[\\left\\lVert\\mathbf{s}(\\mathbf{x}(t), t; \\theta) - \\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0))\\right\\rVert^2\\right]\\ dt\n", 100 | "\\end{equation}$$\n", 101 | "where $\\lambda(t) = 1 - \\mathrm{exp}\\left[-\\int_0^t \\beta(s) ds\\right]$ and $\\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0))$ can be calculated analytically,\n", 102 | "$$\\begin{align}\n", 103 | "\\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0)) &= -\\frac{\\mathbf{x}(t) - \\boldsymbol{\\mu}(t)}{\\sigma^2(t)} \\\\\n", 104 | "\\boldsymbol{\\mu}(t) &= \\mathbf{x}(0)\\mathrm{exp}\\left[-\\frac{1}{2}\\int_0^t \\beta(s) ds\\right] \\\\\n", 105 | "\\sigma^2(t) &= 1 - \\mathrm{exp}\\left[-\\int_0^t \\beta(s) ds\\right]\n", 106 | "\\end{align}$$\n", 107 | "with $\\beta(t) = 0.1 + (20 - 0.1) t$." 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 4, 113 | "id": "3b434448", 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:\n", 118 | " # x: (batch_size, 2) is the training data\n", 119 | " \n", 120 | " # sample the time\n", 121 | " t = torch.rand((x.shape[0], 1), dtype=x.dtype, device=x.device) * (1 - 1e-4) + 1e-4\n", 122 | "\n", 123 | " # calculate the terms for the posterior log distribution\n", 124 | " int_beta = (0.1 + 0.5 * (20 - 0.1) * t) * t # integral of beta\n", 125 | " mu_t = x * torch.exp(-0.5 * int_beta)\n", 126 | " var_t = -torch.expm1(-int_beta)\n", 127 | " x_t = torch.randn_like(x) * var_t ** 0.5 + mu_t\n", 128 | " grad_log_p = -(x_t - mu_t) / var_t # (batch_size, 2)\n", 129 | " \n", 130 | " # calculate the score function\n", 131 | " xt = torch.cat((x_t, t), dim=-1) # (batch_size, 3)\n", 132 | " score = score_network(xt) # score: (batch_size, 2)\n", 133 | "\n", 134 | " # calculate the loss function\n", 135 | " loss = (score - grad_log_p) ** 2\n", 136 | " lmbda_t = var_t\n", 137 | " weighted_loss = lmbda_t * loss\n", 138 | " return torch.mean(weighted_loss)\n" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "id": "719d42ce", 144 | "metadata": {}, 145 | "source": [ 146 | "Everything is ready, now we can start the training." 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 5, 152 | "id": "22e5221a", 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "name": "stdout", 157 | "output_type": "stream", 158 | "text": [ 159 | "0 (0.042441368103027344s): 1.0375168743133545\n", 160 | "10000 (92.2302143573761s): 0.20365591967105864\n", 161 | "20000 (185.08470916748047s): 0.22875319838523864\n", 162 | "30000 (281.3716654777527s): 0.21506413650512696\n", 163 | "40000 (381.3604953289032s): 0.18730134618282318\n", 164 | "50000 (480.49600076675415s): 0.20452004408836363\n", 165 | "60000 (577.897787809372s): 0.21503965973854064\n", 166 | "70000 (676.1737062931061s): 0.1996930924654007\n", 167 | "80000 (776.0507819652557s): 0.21714348602294922\n", 168 | "90000 (876.1378817558289s): 0.20224706172943116\n", 169 | "100000 (974.7899713516235s): 0.18862251448631287\n", 170 | "110000 (1073.5986211299896s): 0.1907759370803833\n", 171 | "120000 (1172.9861385822296s): 0.1956720690727234\n", 172 | "130000 (1271.3041791915894s): 0.19564533567428588\n", 173 | "140000 (1370.8661642074585s): 0.2035321000814438\n" 174 | ] 175 | } 176 | ], 177 | "source": [ 178 | "# start the training loop\n", 179 | "import time\n", 180 | "opt = torch.optim.Adam(score_network.parameters(), lr=3e-4)\n", 181 | "dloader = torch.utils.data.DataLoader(dset, batch_size=256, shuffle=True)\n", 182 | "t0 = time.time()\n", 183 | "for i_epoch in range(150000):\n", 184 | " total_loss = 0\n", 185 | " for data, in dloader:\n", 186 | " opt.zero_grad()\n", 187 | "\n", 188 | " # training step\n", 189 | " loss = calc_loss(score_network, data)\n", 190 | " loss.backward()\n", 191 | " opt.step()\n", 192 | " \n", 193 | " # running stats\n", 194 | " total_loss = total_loss + loss.detach().item() * data.shape[0]\n", 195 | " \n", 196 | " # print the training stats\n", 197 | " if i_epoch % 10000 == 0:\n", 198 | " print(f\"{i_epoch} ({time.time() - t0}s): {total_loss / len(dset)}\")\n" 199 | ] 200 | }, 201 | { 202 | "cell_type": "markdown", 203 | "id": "bb8fe66c", 204 | "metadata": {}, 205 | "source": [ 206 | "Once the neural network is trained, we can generate the samples using reverse SDE\n", 207 | "\n", 208 | "$$\\begin{equation}\n", 209 | "\\mathrm{d}\\mathbf{x} = \\left[\\mathbf{f}(\\mathbf{x}, t) - g(t)^2\\mathbf{s}(\\mathbf{x}, t; \\theta)\\right]\\ \\mathrm{d}t + g(t)\\ \\mathrm{d}\\mathbf{w},\n", 210 | "\\end{equation}$$\n", 211 | "\n", 212 | "where $\\mathbf{f}(\\mathbf{x}, t) = -\\frac{1}{2}\\beta(t)\\mathbf{x}$, $g(t) = \\sqrt{\\beta(t)}$, and the integration time goes from 1 to 0.\n", 213 | "To solve the SDE, we can use the Euler-Maruyama method." 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 6, 219 | "id": "a05b11a5", 220 | "metadata": {}, 221 | "outputs": [], 222 | "source": [ 223 | "def generate_samples(score_network: torch.nn.Module, nsamples: int) -> torch.Tensor:\n", 224 | " x_t = torch.randn((nsamples, 2)) # (nsamples, 2)\n", 225 | " time_pts = torch.linspace(1, 0, 1000) # (ntime_pts,)\n", 226 | " beta = lambda t: 0.1 + (20 - 0.1) * t\n", 227 | " for i in range(len(time_pts) - 1):\n", 228 | " t = time_pts[i]\n", 229 | " dt = time_pts[i + 1] - t\n", 230 | "\n", 231 | " # calculate the drift and diffusion terms\n", 232 | " fxt = -0.5 * beta(t) * x_t\n", 233 | " gt = beta(t) ** 0.5\n", 234 | " score = score_network(torch.cat((x_t, t.expand(x_t.shape[0], 1)), dim=-1)).detach()\n", 235 | " drift = fxt - gt * gt * score\n", 236 | " diffusion = gt\n", 237 | "\n", 238 | " # euler-maruyama step\n", 239 | " x_t = x_t + drift * dt + diffusion * torch.randn_like(x_t) * torch.abs(dt) ** 0.5\n", 240 | " return x_t" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": 7, 246 | "id": "90d4e875", 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "samples = generate_samples(score_network, 1000).detach()" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 8, 256 | "id": "b5734fc0", 257 | "metadata": {}, 258 | "outputs": [ 259 | { 260 | "data": { 261 | "text/plain": [ 262 | "[]" 263 | ] 264 | }, 265 | "execution_count": 8, 266 | "metadata": {}, 267 | "output_type": "execute_result" 268 | }, 269 | { 270 | "data": { 271 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABS+UlEQVR4nO19e5Ad1Xnn77t3BChLTBxBNBhGjB0TEIkZCaYGY3ZSSmInQlAoDydBWi9SKimczZLYDussLm8lqexuoTilFCTEiSmvEbIt4aw3Dope2JgoTGRgGGk0YEviPWQkIwEKNVhlSWjuPfvH6XP79LnndJ9+3Hv7zv1+VVNzH327T3ef/p3v/ZEQAgwGg8GY/6h0egAMBoPBaA+Y8BkMBqNHwITPYDAYPQImfAaDwegRMOEzGAxGj6Cv0wOIw4UXXigGBwc7PQwGg8HoGuzbt+9NIcRFtu9KTfiDg4OYmJjo9DAYDAaja0BEr7q+Y5MOg8Fg9AiY8BkMBqNHwITPYDAYPQImfAaDwegRMOEzGAxGj4AJn8FgMHoETPiM8mJmHBjbKP8zGIzcKHUcPqOHMTMOPHgLUHsHqJ4DrNsGDIx0elQMRleDJXxGOTE9Jsle1OT/6bFOj4jB6How4TPKicFRKdlTVf4fHO30iBiMrgebdBjlxMCINONMj0myZ3MOg5EbTPiM8mJghImewSgQbNJhdC84iofBSAWW8BndCY7iYTBSoxAJn4i+RESvE9F3Hd+vIKJZIjoQ/P1xEcdl9DCyRvH4aAWsOTDmKYqS8DcBuA/A5phtxoQQNxd0PEavQ0XxKAnfJ4rHRytIoznMjHeHU7lbxsloOQohfCHE40Q0WMS+GAwvZInisWkF5u98tgEkiW66CaidBaoLgPU7ykmmbPpiaGinDf96IpoC8H0A/00I8T3bRkR0O4DbAWDJkiVtHB6j65A2isdHK1DbzJ0BiICFi+TnppQ8tUXuB5D/p7bI10kLUDukbf0YvgsYoyfQLsLfD+AyIcRJIloF4B8BXG7bUAhxP4D7AWB4eFi0aXyMXoBNKzAJeGAEWLkB2HknIOrA7rvkb3ffFZWSQdF9n3yjWHOR2j7t4mAeY+WG9KYvxrxFWwhfCPG29nonEX2eiC4UQrzZjuMzGA3oWoGLgE+dAISQhF97Bzj0cFRK3nsP8INjQKUPqNfkb8+/KFmSntoCzJ0GIJKl7aymGFOiP3WCE9gYDbQlDp+I+omIgtcjwXFPtOPYjC5Eu6JkXJE+ZlmHpavD90TA4R3A0X1AfQ648iZg/XZgaG18KYiZcWByCwBNaVXmojRjS4KtJMXACDB6Zz6y58ileYFCJHwi2gpgBYALiegIgD8BsAAAhBB/B+CjAP4LEc0BOAXgViEEm2sYzWiXk3FmHJg9AlSqQB1RkraZfhZfJd8rslc4+8NwfHGS9PSYXCAUlLlo8VX280sThWSafuLGUYSZiB2/XYuionTWJHx/H2TYJoMRj1Y5GXWiA0ICq/QB194mJXT9OKZDWL0//XaU8Puvdv9GR8MZHJh0IKRjeM/dwIrPNP/OJG5AStgmUbvI2BVZVISZiB2/XQvOtGWUC1nj6+OkVpPolq0JCawO4IKBqF0/bl/nvQvSYSvk//Pe5XdeisD33gsc3h58WAdeegx45XFg1UZgeH3zb5Rj2UXUacg4K3FnuSeMUoIJn1EupI2v95FaTaKDsBOYz74GR4G+88JtFi5yS97mOQyMAJdcAxzeCbnSBKjPyaggl3knjqjTkHFW4nbdE07o6jow4TPKhzTx9T5Sq0l0Q2vln0lWPvvSye/022H4ZvXccIGIWzgGR2WiVu1MdL/1ulvijiNqfTwLF4XOXdt+8pScNu/JfMxI7gEw4TO6Gz5Sq4vokhYGlwSsfvfAjaEjtnYmJOzpMfle1KOfNxCYgygIkhMC6Ds3/nhJEvbgqB8BF1VyOk1GMjt8SwMmfEZ3w1dq9SG6NBLw9JiUyhWoEhL2wkWS7AH5//TbodlnekzG7qvwzGtvkz6EpOMlSdjLbvW3zxchcfsujuzwLRWY8BndjyIbpfjua3BUSuVzZ4BKRTpd1e9OnYBMNakDIOCJ+6QUb8t8NaODTLjIuckvQX4E7CtxJy0KvosjO3xLBSZ8BiML4ghPLQa1d2SilqiHWbtpMl91cq5UgeUfA4bWyN80+SXWAP1DMit46ep8kTq+i0LRWhOj5WDCZzCywkV4piNVr8Oj1+xJQoSca8DEA8D+zWEIpxmnr47z6hPRiB/T1p8kcSeVgEhrEvI5X3bstgVM+AxGK6CTnMrSTUtmtmQtM4RT7W9sY1Ryn9piX3DWbUvOxNVLQFSq0UUhTuvICnbstg1M+AxGq5HVx6A0haktwL7NkswBewinLrlXqpK063PSpFSvA6iH0nqcHT1SAoIkoatQ0+kxYHamWes4sDUfSbNjt21gwmcwktBJc4NaLPqXScm+XreHcOpmpNkjwL4HgwWiIp3KgsJEsThp2uYbiEj1fXJBqdXR0DrSkLTtWrJjt21gwmcw4lC0uSHr4jG8Pt40ZNYKOrA1WhP/1Am/hii2RK7ZI9FSFNeuAyBCLaLSJyX/mXF/J7RZ94cdu20BEz6DEYcizQ15Fw+9to5ezsG235Ubwogds0aPT6La8YNhFnGlL6j9j1DqHxiRIaVTW4HJr0iNYv+X7TWBFOKuZZGhtQwnmPAZjDgUaW7wDYlMUwhOScamw/bAQ2HEDhBK+D7S9Mx4YD4KbPn1OSnVmwliKqu4PheGnsbVBGLTTcfBhM9gxCGruSGLrdq0lS9f25yYZVs0zP2CtG3OBJK6SC6frB/DzCLuXxYklBkYHJXfq8ziuJpAbLrpOJjwGYwkpDU3ZLVVN8Xdb5KSelPxNWPRMPcLhDZ8W5RO0rmYWcTX39Ec2qlL+as2xjuU81xLRqFgwmcwikbeuvN63L3N1u0qBKcfIy7xKwnmMZLOJ8mhrMNlsuLEq7aACZ/BKBou002S07YRdx84QlWDdFsIZhpSzZL4ZR7Dx9Hrk01rO/9OJF716ALDhM9g5IGr0YlNCvettz8wIiNhshBSmpaHvijK9u46/3YnXvVwZi8TPoORFXHEYSPZNFEqWUm6VeQZNx5fadl1/u2O3unhzF4mfAYjK9ISRzuiVOLMSa04bhppOc7/0M7onR4OD2XCZxSDXrSJZiGOVkep2MhTkbJeu9+VHJUWU1vjK2umGXe75k0Ph4cy4TPyo1dtomUlDpM8p8ck2aMuQyfjkqMA/8V7Zlw6lxuVNfviF72ZcWDTzeE8Wb+9c9esR8NDmfAZ+dHDNtGuII7BUSnZq2QqEZMcZVu8AXs/3T13AzW9suba+GsxtRWN5u21M/J92a/dPEMhhE9EXwJwM4DXhRA/Y/meANwLYBWAHwJYL4TYX8SxGSVAD9tEuwJ6cpSoA9WY5KimMg1bo4XY1ALw4C0BeQdtHCtVmY0bC5Hw3oJeNBW2EEVJ+JsA3Adgs+P7GwFcHvxdB+Bvg/+M+YAk0wY/tJ2Hb3JUU5kG0ay9AcFndQAVgCC1h913xZuKhtYCk18FameB6gL5Pg69aipsIQohfCHE40Q0GLPJagCbhRACwJNE9GNEdLEQ4rUijs9oA3yaWvuaCDr10Pb6wuNjfrKWaXioWXtTi4IQYR2dJHPewAhw418k9921NlvpMVNhi9AuG/4lAGa090eCz5oIn4huB3A7ACxZsqQtg2MkIA9pl8W+zwuP/xhcZRr0bddtA/beAxzeEW5HlOy0dfXd1bcxm62ossxsKsyN0jlthRD3A7gfAIaHhz2MfIyWIw9pZ7Hvz4zLEr+g/P1SFXjhyT4Gm2YwMAKcPRX97OKh+P343AN9G9Vs5YJLe1crKxjtIvyjAAa095cGnzG6AXmcsmlDF2fGgU03yWMBsrHGTX+ZP268LI7lNAtPqzSBoha/pauBlx4L3y+/zb3tzLg00eiNVGz3wNZi0WfOdFpj6hK0i/C3AbiDiB6CdNbOsv2+i5A33jxN6OL0mHTqKYiaPW487UNelph534WnCE3AdY3MMSxcFO2g5Qu1CLs6a9nOpVKVUruLyH3vkzo3sxooO3ZjUVRY5lYAKwBcSERHAPwJgAUAIIT4OwA7IUMyX4QMy/ytIo7LaCN00m6lRDU4GkiBGunX56SJRz9+FjIsQ8y8L6HllcKT6vy4yienJczh9cnal2mmueDS9CYks2evOjeisNsWO3YTUVSUzpqE7wWA/1rEsRgdRh7J03ehIGr+bHJL2P2pLPb4rPBZePKaoHyalQ+MSMm+1dfSp76PGrMrrFefc8vWhGMWFZlUBmLHrgdK57RltBBpJXPb9lnJ1nehmB6TdeAByADvwG9fnwuPVRZ7fCuR1wTle43acS3j6vsoMw9I3mPb3DDnHER0zCs3RHv2Mpxgwu8VpJXMbQW3Fl8FzB6xh8oldTKaPeK3UOgEZBKBOlZZ7PGtdhbmMUGluUbL1gAQzf1zi4Stvk9jPqj+uY4CbMrMV6vL/0Nr5V+n738Xggm/V5BWMjcLbu34FIDAXkoV4IqVwA2ftEhrWvNtICrFJUVoKOgEpMaitt/+qfC70TvzXpXsaIdpKy+SFgzzHJIyX4uEz8IegYj+T7Lz53Hw5/1dicGE3ytIq7rbCm4piBrw/COS8AFjMamFzbd1W2sdcpE4e8qdZWlWU1QSp1pUNt0cFt+a/Cqwfkc+R3KeB7rVpq12oJO+EFtGr+teNMx8Qv63jbPo9olluk8Fggm/V2B7wMxQPJMAVcEt9bDpqGsVF9ViYjbf1m2tlSrwwqNSinNlWcZVU1TkpFA7Gx4/zcOpkrpOvgG88M2wb6z5m6TFIGujkTI5nNOEiLZC0jWldNe+Fy4KHPkV9zj16zp3Opw7Wa/39Jicg6Iu/3dbYIADTPi9BF1atpXANT9TBbdUU+3aHBrVEfu0iotqMTGbb+u21tkjMokq9sEzFpWTr4evG+QULAjVBVHJ0OehNpO6FMzf+Cwgxw/Ka/OjFwM3fCL+uqoF5vyLZEXJsjicfez8nW4wDsiw0XpdapwrN9il+9mZIEQTAISch0NrsjulFy4KtVpRl+/nAZjwexE2ggTspKn+VFPt028Dx55pNsuY25kt7GbGo2V2bRLx0Fpg/1fCGPwXvim/V/tev10uKqaD0fehNpO6AFjD+VwLiBrr6bdlHRmF93/ELk02FkptgameIwuIlSWqJMnO326NpCkE89bg+tUBQfK66ds2rvFZRAQGFdU1eqc9QihJYzl1AkBFHheV6HG7GEz4nUInHUKDo9LEUqvL/4rsbJ8pqDGqh9Fllolzppnhczbp8ZqPSR+AzV7rIiczkUgtYObDPTgqNQNFwJUF8nj9y8LfAPYSAPpYTU1kcrPUhqylhY0FpnZWXgNfh3OnHYftDoFtCsEkt+nswVs0M6IBqkQ1ULXv4wf9Es0GR6UWWwZNrEAw4XcCpXAIkfHf9ZkGl2YQR0gTm6KNN/Rz1SOB5gI76dDaaDle37R/c0FS8dm7Po1G/fX1O+SfXphN/02lD5Kk56R54IobQ3ONnqBkXp/Xngk1EdNPMvlVQ8Jf4E8eZZgn7Q6BbYregT3OvuHTcZD9qo12E51vZm5ZQn8LBhN+J9AqNdlXGpwekyovhDSf7LkbWPAj4QOkJznpUM4zUQnJOI6QZsYDp2/QBs90fi1cBClGQ/5fuKhZWk+T9m9e18nNIdnW3pFEf/M90X1EMk21eHAB4PndkvCBZiLSTQhC00RMLUQtMMqGnybWvSwO3jz5AFmOtW6bvGaTW6Tfx3bv9ftBlZDEK1Xg+jvkAqEWYf06psnMbed5twlM+L4oUrVuhZqcRhrUHaCiDrz0z4hISjaTjqplLjTn2akTGiGdkQvHis9EpTAV1gkAoKjEfupE+LCSZidVD9rYxmYNIO7am9f1Ry82NjAk85nxaCJZpU8uTqImv9d7v+oL0eyR0Oyk4HLq5SGN+ZhR7PscvfVqsKg6JHFXWKdNSDCvYw9n5jLh+6Bo1TqtuujzkCQ5Gk0n6rptkqBf2oNQygZkM+qPNf+2sf8gSkc9MJGFY4+07asHauGiwGZ+Jtz9rk+HUTwrN0gzj4vQTA3g9Nvpruvxg8Bzu+Q1qZ4bmnDUNW2o+RXgPcvC8r4qFJUqUSJX12/vPVpECGTXp6T2flnQKbOCOWeKEnZ8nqPGNlq/XCL7gmoL67TVBrI5bnsUTPg+yKNaux4WX8nPd7FpxMKfCR+QpIqJKz4jCVpJ0VQJidEsrXD9Hc3Spm3hmDsT2OyDGPzLPwwc3glpJqkHZpMgTv/UCfuDqLe40+vpPHEfcOVNfrZ8ZRKAkJL7jZ+L/i6i5teAo/vlArFuW7Tht07kZlinHgbYKpNLu80K5pxZuaG48sM+z5EpWCibu21BtT1bLq1I1zr1965rME8XByZ8H2RVrX3JOm6C+S42AyPy4dSJSoW0NcIEt0SPY9rLdTVXN6fU65JsV21sVoUHRmSI5iuPB2YRzZ5aewc4fzHQd17UKaokfH0ctmtW6QtMPoGJRU/2SrrmZvSGGVbnShZThKB6terX3AzrFJDmILW4xSUuZbXjp0UasvIpjnfo4eL8CD7PUcQ2H+NgdZXzcGlFaZ7FTjvKWwgmfB9kVa19yDppgpnOwtkjoTPKxKkTUaLSQ9oqVSnxmhUJXRKkrbSCLZxwZhzY9UehCeT6O4CnvhCez9Aa+aciY/qH4u2n+jWrQ0bKPL9bjkNP9kq65jrZ2xJnGs5BI1lM7d9GTGZYZ/UcqTnYzkdv0LHr09FIHbMsRB646sT7ZBzbtjVJeelqqQUW4UfweY7inPbOXIkaMPGAjO7S57UOX8GpLI7yFoEJ3xcuYoyTqHwkmqQJZotaOLDV/jCbx1Nk653papyvbtqoOshWL4cgasCJF6VmofejnRmPhlp6OZSDbW/4hPzzXWybJPcAx6bs5zgwYk8WsxHTwIgRdfMTwFuvyEQ0FWEENIcBNso9B9DLQmSBq9uTXrso6R675p2NlBdfVZyJw8dEpW/jOnYjl0S7tnOnmwMH9O19tPQ02nwXmn6Y8PMgSTr3kWh8JpgyJ6gIEtfD7Dpeg3Qtma5xUKUVYie1EQf93C75Xy04QDqpKe4cfLHsVknIzz8SZu2qVHuXVGm7lqZPQY3n+MFouCkQ9nYdXi8XwcaCU4mapQD5uSuqJ6kpSNNiUkcjksWsEx93j+Pmnc0Z2ilCc9neB0ZkcMHEA9C858DLe6RGkuVZTLNdl5p+mPDzwIfIkh4W3wkWMe30SYemy7SjYBJVlhTzJAytDZKLzkpiU87ZudNhW8I0Zil1TbKMx3wIf+qXgMM7EFtl0WefegXPGz/XTPYKk5vlAjn5FTRIqLJA/ubJzwNvPic/00NQXeM3/R2KUPQ5hyCmXFBYYtg35DCrmbII6BpK0ljjiHVoTSDEKA3T4nfR4TuvfLbrUtMPE74LPmRYVJy0S4oxt1m3Ddh7r5SiJx6M2izVmJO6CDVteybMTDR7k/pIMcrM0WSvFtG2hL5mKRNpFiXzIYQIYuzr6bJ2dZgVPJ/6W0RKRet47Rl5jnrHruVr5SLw1ivhdhVHtm1k/FoSmGvxVFE0xw7EJym50AnJXZ93IqhT03eue8zTY+5cDH3ROv22DCwQ9faYYro0R4IJ3wZfda0oKUmVH1COSVVl0bbf53eH5gHzAWgijEDKtEXoTI+FD52oAzv+UNq5dbNHmggh9fmxqVDN1jN2fc1SOtKqzaYm8cKjYaLYdR/PGF5omKzeeF46buuBRnPRUuD4s8GmQShhxI+yNjhvYxFIMu9V+gLHdZARbS6e+r0c26hdV0sCXJkQCbsEGkTuGrMtG1uH2v7BW+IraiqYGtv67dmuUyc1pBxgwrfBl+iKkBTM8gNzQR143d6u1PTZmah0WalEJQs9Fl9PphJCVqFU0o/KPlRZroA814kHopJ3FimmoWZbfpN2f2nVZv0hbDipAxI+9kw2FXxoLbBvc9QGv3wtcMFAc2RMpU+WdH7/L0TDL48f1EpS9Mnx2ExaJolMbXUvnqYD05YAV0a7sjlWEIC63fY+My7DQlUuhssUpoQX1OX6HFfZMq7nQlp00reREUz4NvgQU1FOm+mxZhJXMeGiFk1kqvSF6f9mgSggJIw9d8sHSJeiVHilIrvROy0NTowEoixSTNxvdNOOq0CbjiwLjhp3w0kdJKL1X50tvHBgBLjpL6MamBlHr8I7938ZOLxdfqake70kBZG8j/selHb+y38RgJC5Ckqz0sevnLCxrf8Qve8qAa6sdmV9fixcJAldzVV9zHrinyJ7V6SYrXa9Uxgzi61Ziq/NYzDh2xBHWo0sUM+m3EkYHA3KC2i29MVXhWGMevJJHcC164ALLo2PY17xGWB6LyIlDQA0FYxqNDjZEo3Rj4vY8EHSb9S5JdnxzfsA+NvgB0bCRLR6XeYG2Moz+yxmerSSWX5Z/VfmKgUVegloJozgXkLIcEK1OABBbP72KNkp09S169wRRvr5qszpVtmViwpD1OfH4qvsY27kUwR2/vetcJupzNr1xw64zXd6kEF1QXv7+JYATPgu2EjLfBB9m3InHce2uLiST/QHP/YBFAjLEgSmhOUfayYOdZ7K1txqe2QWM41Jgr4alUpEUxKvnjiWdn+6rdj2G91UASBSBlm3yzdq5BuSpX4tzOSzCy6Nt0nr923lBik1u/oG+8A2r1oVhuia/6Z2F+eTGByN1q4HueeYHmSQVoiYByiE8IloJYB7AVQBfFEIscH4fj2AvwBwNPjoPiHEF4s4dlthPojX3hbacouSeGyf2WLh4x7AhpMwUIXjpKO4MbQCWaMbsoTBxR0ry/7ifjMw4u7IZWopU1uinb0AS0avxzWKq3vjalDjgiuZyxYOWrS5yDX/fc2JNk0wLuckjxDR5chN+ERUBfA3AD4C4AiAp4lomxDioLHp14QQd+Q9XkfRlMnqiLYwkUUVNn9jSvWzR+BsspxGOmo3skY3ZLXnu46VZX9Jv3EtmubnSqOa2gqcPB614avvfSR1k4QnN4cJX6qRt9rON9bdTOZS86oTYYhphBBzW+VXibPRd2ksfR4UIeGPAHhRCPEyABDRQwBWAzAJv7vhatPn87u0UoSrMBTQ3LkHaK4Vk5VU24WsfoEs5xRHwkU6pNMi7hooR2+SpD44GralpIrMA9CzTic2SUeyHp1l248rmUsn9rLPKRuUlH/gIftz26Wx9HlQBOFfAmBGe38EwHWW7X6NiH4WwPMAPiWEmLFsAyK6HcDtALBkyZIChlcA8qh+uU0HNfngHngoWv1S6FEulnC1dplo2omiz6kVDukikGrOBPNA1AN/hQ49Oism1t2WzGUTarppTkWuoVGyW09C7LZFLCfa5bT9JwBbhRBniOjjAB4E8PO2DYUQ9wO4HwCGh4fLETOVR/UbHJVSeq0u/6cxHZile83ql3ombZydt4cmdFfCvEe+kmcjMiiYI5VK8NLMBKb4+Pw0xNct80m/hi4zFdBdi1gBKILwjwIY0N5fitA5CwAQQuji5xcBfK6A47YPuVU/Yfy3wBZtseMPw4SfSl+0+qUevuZ6+GxOvR5t7VZauLRHPUrLVXLDJZnrZQYqfUD/B4Cjk4iNz/chvm5ycprX0FVmucdQBOE/DeByInovJNHfCiAS3EpEFwshXgve3gLgUAHHbR/yqH5TW8MwPFcBL1tdm4iJxkjHN6UzF3zUWkZnod8j5WjVyde3GqtZiOzKm6KCQWN+OdoFph1rNzg5kyLdehC5CV8IMUdEdwB4BDIs80tCiO8R0Z8BmBBCbAPwB0R0C4A5AP8OYH3e47YdWVS/mXGjcqLDpGPWtdkZZMGaEUFp4avWMtoLXZtrmPyCMFq9jLM+L8xILIW4hUHfVk9Cy9p/t5udnD1munGhEBu+EGIngJ3GZ3+svf4MgM8UcayOIYvtMk3RLL2uTb0e9nv1LUNgA6u15YPNLLJ8rXTMm1qgrWSAua/pMVljKUnyNpPQsiz43eTk7BZfQ5vBmbY+yBpaOTsTzcZ1SekDI1pdm3q0ld+Bh6R0t3+zvYRxElitbT3ylHCeHpPzQu8Ipu69WTJAN/OZobuVanzWd1HSeR5JOU0dfPM3aXNYivQ1uMbQhYsKE74P0tous9RCsXWXGtvYbOrJooorsFpbPPKUcFY1+l35HWbJAFemsKvGUlIDnCznmvX3kWJo9bAYmnm99GOozmKqxaavoLXnbs0UltN86bq/3eTA1sCEb0PWMDmFNLVQ1PGU6UZfGGymHra9lwtZagO5zGxpwiX1OVmphp/pZG+r+1408fkiUgwNdjI2tRZV4x9o9mHE1fuJLCo5zZeu+9ttDuwATPgmksLkfAoupVkgZsaBTTcFDwOiVRPjTD2MciBryYeGBpdAGi6SVnPS1UUsru57Vn9UHoJr5JbEkHHkGEFVUQXSej+4ntHIouJZQ0rtz2Vqct3fLnVgM+GbcEVGqD/ftn++6vP0WBC2GcB8mLwaiTM6hjymkrykoUjO7CIGAK9NGRsH5Jk1N6OIsbpCSG3HaFQWnZMJZXrvB9fi41NDylzsklouuu5vNzmwNfQu4buknKTICF9JR5+c+nsTg6OylK6S8NMU5WKUA1nvTxGkYfMJKAIDILOztbrvkbh/IzcjjvyLGGvSdbJp0mkK3yWN0SasNa6H1izINxu3C5/L3iT8OCndFhlhjZvWSiXkqR8+MCLrc9ts+Iz5j7ykYZLc9JjW4pKAn/y5qKRr5maogACT/F2aa6vnpnmMOHOWjdjjxmjT3hvXQ5PwbUJXF0bk2NCbhB8npZuREQ2JSVOB9VIJxw+mqx8+sSkse6tCLLtQUmCUCPr8OX4QYT9j0Vxe2eU01sm/G5yQPs+MSdI27d3H1NSlETk2zE/CT1qN4+yRNolJJ+5DD4fNReo1+d7Hnjg4Ksl++yfkcV56TP5PG1fPYMTh1Aktsqsi56crlHfxVfM3Mc9G0q68hqTFI43DuuSawPwj/CKcquYE0Il76epoD07zvcueCADfuTd6nEMPd4bwSz4pGTkwOIpGj2RVIXN6r72nQqXa3PYyb3vEssBG0rq/rLpALnI+7Q19HdZdoAnMP8JP41T1uRm2xcGMmnFF0ZiRPXOno/vuv1r+bycBd8GkZOSAmq977pZkj8BerfdUUDb+Wg2YeECGc67cAOz6o+CZ2Zsvwa8T8M6dCcyxoibPV5UXj3sOfB3WXRCbP/8IP0v4WBLh2hxJvo4iQIsPVo3FAzz1BeDd741PvikaXTApGTkxMCIdta8+0dxT4eQbCG38CD+f3ByN2997L3DJNd2hBfrkzgyMANs/iUjlWgSmWd+EuaTr0AWx+fOP8NOGj7VD4o2rWqn3IW0HAXfBpGTEwFcbVM/B3nuB53aF0Tfn/0Q0e1s11fnR/ujvn9sl/7pBC3QJMTpJz4zLBLVG5doqQNXkBkIKPte9C2Lz5x/hA+miXtoh8aqJMLUVOPk68MI3pYRR6QsSZBLKJ7diLCWelAwHsggnL34bjW5YKzdIU43q9Wr2S37hW1ICpor8TRkidnyI1keIUQlqAAACrrkt2kwo7vzi6unYErJK/EzNT8JPg1ZKvHqNnP6h6IN27W1ym32bg41jyicXjZJPSoYDaYWTSFIRyagUWzCBer1+R7kidtLksiQJMeZzrhzVPs+B7boDXekL6z3CN1flVkm8Zo0cqob1yOuAtOULv/LJDAaQXjjRk4r0TldmMIFOWqN3ym1sgQitDC6wFRBMs8AlkXfW53xmHJg9EpafrvTJsudTW7rSF0aiqdN9eTA8PCwmJiaK22E7I1TGNgLf/p8Ik7RIThohwjoh9Zo9NI7BANwZ3L5miOmxsL+tKr6nz/mxjcBj/zuoSFkBfnKFu9hYK58dUziqnisLCALB52dlGOX6He15RvRiakrTqfQBl38kao5Vz3DJJHwi2ieEGLZ911sSfqvs9baH0FYj58bPSbV6dkaacnzLJzN6Dy6CTZJkZ8aBvfcAz+0OhIuKu7WlWcHy5T0yssdGXq30dU1tDZ8TIBo334hqy9j1LS30664HWNQBnP2hJPi4HgQlR28RflZ7fZxU5ao+uHARsPw/yTC4838iKsHPjNs7HDEYClkI1pSUAaAuAs2S3Fnle+6WZB/npG2Vr6vR91mD2n/D0Srk/3aYTfTrjopcMAUk+fdfHU2y7EKtvLcI32XHS0Popod+9oj2YJ4JO/TYSq0mjYPBUMhCsGapbUCS/aqN0Roxusni2AFgwY8EvqSa+1i+czatnX96TOv7DOCSa6XQpH7bjhBisziifszrPh6axJ76QlSgS6qEm3UMLeSD3iJ8oFklTrJNRsrJnpbqJxBNT1eOV10FBNCkRtscxgyGDUkE62NGpIDslQP28A5pk35typinACoLZOTYUEykmI85Ka2d3yRYnezbIRjZxmzW0tKbvx87AICAPRuKs9+30bfYe4RvIkl1bpRDDrLylPrZ+I0ALlkOXDwkQy933wVrqVUuacBICxvBqmiWyS3NZQEGRqKltvuHgGNTwK5PhxmmLtTngAsG0kWvmEScxQw1MCJJ3lXDx2eRybMgmGOe2opIRry+IFWq8ro3suZRjD+jjdnvTPhJqvPAiIyPn9iEMCVbRJ1d35+UZWnXrXGXWvVpZ8dgKMT2bFXlEmCfSxcMhBEm+rZx0AUTH7ONTXjJWtZERcK8+kS6Gj5FCFERQu8D9m8OE7RUu1H1TM8eka0k9ci7IkxN5nXzLeqWAfOT8NOs+j5q49DaqJN1aK38szm7Ru+072PhImnycTVYYDAUEnu2OgjHjDARRl9YHZd9CJh5GqiflaafGz8nP/ch0LhSBmlNMHmk2yIkY33MszOBYBfAfKZnxqPJk5d/BDj/ouRj+NTq8m1snxOFED4RrQRwL4AqgC8KITYY358LYDOAawGcAPCbQojpIo7dhCyrftakDVWgKkmiUVJMvR6muAMtW8UZXY7psVB7nNP6KpvmBT1/Y2ZcCiDqdyKIMAGFJRTOvQA49ow0nZw6AfzbU+ExT53wJ9CkfhJp5nOe6J+iIof0RLTJr0ZDqfXFdHos6rRVxDz5VXcuTZps4TZYAnITPhFVAfwNgI8AOALgaSLaJoQ4qG322wDeEkK8n4huBfDnAH4z77GtaJU9zDaRfSWahmRWl+Fxxw60t0Imo7uwcBEijn89Q9YVZdboY6tFh8X1qJ0Zt5OlD4EW6UzNs6+inbq6D0SFUwPuXri2MtNxQR8+fNTi4oZFSPgjAF4UQrwMAET0EIDVAHTCXw3gT4PXXwdwHxGRaEWab7urQcZJNHr4mz4mENvzGW7oXatI68wE2KPM9twdBgpQBXjfCnfGrL4fG1n6EmiRUWZ59lV0tJvalyL4A1tlDwHzeY0syoCz2m1aPmpxZFIRhH8JgBnt/REA17m2EULMEdEsgEUA3jR3RkS3A7gdAJYsWZJ+NFkvmGlnS3rvsz9bQpa64coWyPZ8honB0aBrlYepMOLErcjfJZG9PpdV7RyFMoYL543ESfq9+b0plStfiX4/psfsZaZtQR9p+aiF96B0TlshxP0A7gdkLZ1MO0l7wWzkrJtczPfrtsnfxd1Ec9Icm5Jp2Gp8nHjFcCGNqXDuDELHrADe//Px+/7WnwDf+SsZW953XjZzYjd1aEv6ve17W2VNWylltSgn1cMq0SJaBOEfBTCgvb80+My2zREi6gNwAaTzthwwydlsTG6+n9oaldBttlIz3GvyK82JGiWZBIwSwhWDr5PO4KhWKwcABHB4J/DCo2Gde30fE5tknR2FudPpzYntzifJ65NL+r3t+8FRacbRK3cC0d91qdBWKWAfTwO4nIjeS0TnALgVwDZjm20A1gWvPwrgsZbY77NCkTNV5f+lq8P3lWqYeq6+V/Y6UZMS1s47ZdXBB2+RDwQQTohr1wH9PwPU5prraTMYvlBEq8+zgRGZSVvpQ1hcTKDRw1afj4AUXEzMHoluYzvu2MZwG1uikv590TCfzbTmz8FR+QyrarXm7839L1wkr9u+zVKoi8PAiDsMu6TILeEHNvk7ADwCGZb5JSHE94jozwBMCCG2Afg/AL5MRC8C+HfIRaE8UOSssuwWXxW83yIz657bJSfLFTfKuNv+ZUD1oWi8s6vw1IGt0egJttkzgGw1Z5RjtqaFag6vl/N1aqvUIhsZtRYn4tLVwEuPhfukikwkMqNLbOWBbeYOl+ZaJAqRpB0VN81QS2Wbn8cBFYXY8IUQOwHsND77Y+31aQC/XsSxciPuQVNmmgMPyUl2wYDMulMlFJ4PSs5WH7LH45pk3pg8KaInGPMfacwiel175SAUWqgmEJp/htaExG8rhDa8Xv6f3Ay8cxJ44wVY6z2ZyVtmYqGeqKTKfLeSHPOYP10VN+PuQd4ov3b6OFKidE7bliLuJrtseZU+oFZvnvy6RGBG4KiEKlMaevdlnTt3RnngK0Wa5Ku6pMEI1VTQiV9J6GZFx8VXyTIgLq1TH5sge3a4nqhU9jLfrrDIIrOFdZS8ZlZvEX7cg+aMlw1cDUQA9YUFq5Stz4zcsVXeU1LXvs2h9lCiScAoCL6SnZprc0brQRNTW6Ihl6pjmq3eiu3YceUZXFqnPjbUg0NXo1UsFXzIsdPSrmuMSdnCQPNi6YOSm4TmP+G7al2r3pTK+WWbGGMbw2JpQgDXrJVmHpetD2j+bPTOQK2slXYSMApAGsluYEQSqOqdsPuu5qJhM+PSf6QEjkofcONfSMn+9NvhbysLgMs/LCNz9OqZTc7VLfYkQNPEqJ4DvU6UEHaNQm1vjls9Q0A5pF2bSShuscojpbc78TMl5jfhu2pdmxK3bpLRE1GUh79Wl//7l4UT3/wuLjW95JOAkQIuiTWtZHfqhCRSl7O/YXsGAJJx3sPr5fEfuDH8rnZG1rlX0M2Ras4RBbZ2ES254GriMTDiXyfKvDb682bLUO2koGPrR+GTQ5Nm3CUP15zfhG+7cabErcIqlarctJoHnn0hgF1/FEpRKzegyfvvutklnwQMT8RJfmkX9aTtbck/QDB36837AxDJ9mxEnm0JHauAnO9KuImTYrPMWZ8M1U5BrzdEFRnOqpzYJrIIaF3S3Gh+E77rxpnSjyus0vTwA2iEux162O79d93sEk8ChifiJL80BGkLB7Rtv2wNcPI4cP5iKcXvuVv2Ve07N7D/Q0vtB3DlTcANn4iOaXoMkRLJlUpy+GFc6YU4+GaodgJ6WKuoAzs+JT+3kX7axa7kjlod85vwfSTuuLDKiM0/SN5QEv7S1elVXqDzTiyGHT73JUny81nUfcihUSNHRdKo6BzIOPobPgmc9y6ZNDWxKdzm/IuabdGzM9LOXz8bSrZx4Yd5yCvuees0BkejtW9EXWr2roYraQS0kjtqdcxvwgf8JG7V89N82M0JDISLxKkTyeVnzX1meZh4gWg90tQsz2ua8yEHtU2jGqORlH7sGeA/fyNw7H4l2FZIJ68qp6AvGkTAkg8CF10h53rcubiCEXzPuVWabN7nYGBELnY7PhWSfr1eDDl3kY9u/hO+D2yT1Izu0V/7SmjOkDhPSaCLVMWuhuu+2EgmL6H5kIMZGqlL+ACwYKGU7E+dAC7/xcBpa5gWp8eioZWvfgd49ckwLPj4QXsfWXN8tvDjds/BtM+B6vtr1sIZXg+89YpWT6juDolNgy7y0THh26BPsEofGr1sq+dIu6qvhGZuk1YS6CJVsathuy+tWmyTwgFVeQ89kkaFYk6PAa9NAYd3SZKnipyf1XNCU6PupyIylIPAT7X3njCyR5VaUBFAZSw1kGYMM+PAppsCrQeyG9WNnwvP57x3QZYQC5LOXOGmadElPjomfBsiE0xTq5Xq7CuhmduklQS6SFXsarhyMFpFdC6NctPNQQYs5P1evyO63dhG4OgkGqYeUZeCyLW3oUkLGBgBllwnJXsdRMCJl6KfHXpYmnpaVWpAnV9WCTjNczA9BllPKEDNiMJbuUE6vW25OD0AJnwbzAQtCKA2Jx+W/mXAurXRyWsmm8RFYKSRBLpIVex6mPel6MVWL0iml+FQ97Zhtw9QO9u8yDR1WQrCHvuXhYEHeib3RVcahE/Sbn3ixejYlq5uXakBde55tKU0YxgcBaoLwmtJ1TB5snYm9L1NbpbaUo9lvzPh22BOsOMHo1mRqmrg9Jj8Tj1suvlHNY4uYiw9MBE7gjip05dkfDqj6THgQtWvCUoaq+iZ6+8IFhgl4S9oXmT01oeoAD+5QiZIuch6aE3YlJsqkBnjQV/lK28Czp6SZK/MOXGlBvLMwSLMQr5jGBgJ+9KCgDMngWf/Xn4n6sDrh4FD/6SVrIC/P20eCF5M+C7oE2x6LJoVufdeWTJZ1SNRD1LE/FOTjjUzkzerhDQPJlup4CN1xpGMsrfrlSn1zmh6F6SprVGCUbZ0BVEHnrhPRpEcm5Lbmc1LgEB61Vof6mURXCbE9dvt4cc3fDI+Is123lnnYbtNk+q+qaxkHd/9evBC3QtHa0Igvky0a3EvOZjw08ZfV/pkV6GGHbUm1Uaqyu9EXUpt8kuPTF6P8XU6SqLdaMeDlCViyqwRY0qJkc5oNWDiASlhi1q4nQv1etgG03XeAyNyUTGja+LI2if82Lat7fyLjs8391/0PbdlJQuhNUTps3cGU+PRK5Uqs9DcmTBUtQufy94m/Czx17MzQbKLhiUflI6g/quBJz+PSBnbSsWdyavGEDfRyxAl0U60a4FLI3WaUVv9P2NI7BRNxmt8J6KSfByqC5KbicyMh5Lmq09Ek4Z8TB66xqq/90HeediqxSQOg6NBVrK6HyT7+No07rhG5kJ3iNfDGkRd+Fz2NuEnpZerEDldApjaKiUEVWqBKsDMU1JyePlfAvuqCEvPLl3tzuT1mei9FqnTrgcpjSNQN8nUasDRfcYGBFz38dAheOyATIKqz1m0PuN31QXS9AMR30xkZlyWVlDjUJKmi7Bsn5W1CmQR99yVM6HqCZ18Q2YiJ0nz6rqY5VeUhE+VcLHowueytwl/cDRscFLpi6aX6yFyk1+VpWl15+xlI8C/PRVI73PN+670hTZWlyrtM9GT4ra7zIaYiLI8SLr9dvIriDfJ1IHv/HVQrrgPuOZj0djvqS3SvGOiUgU++HvAmbeBk6/L39bhFgxMP4BKGrIRFtD8WR5ibWXEWN57nrSQHXgIjaJp/cuax267LnpnL1v5lS6NoOttwgcQPkDaA20LkTv0cBhlURdSNTR/1wAB/R8I37rUWd+Jbvv9fLXtt+tBcl0/laWpJHQiww6sNQtvfFQJq1HWz0qTX9950XsyuSUUIBSEWiiC31IVeM8yYPltdsHAPKZKGnKVQzA/y0usrnlcRNmDPPc8biGbHkOkaJqtfo4tu1g1l1HF42xCWxdG0PU24U+PhapavWbJiNVC5PqvDrMSRV2+V8XTKn1hE4raWQB1qfY/cGN8GdY8E71bbIhZyKAdD5KLJB+8BZg7FW4nAj9MHWg46omkCU++ARZdDrz5nLZzIaXxqS3huSxfG/h+tN+BQrIH5Ouj+2Wor05KZqkFqshoHT2r1kbk5metWEyLEDzyLhhxC9ngaLRomq1+ju4M77/aHpHTheRuQ28TflxG7PrtURv+9BjClGzI5JVltyJSr2NiU1CcKdh/fS6+Il8rxl4mlFkLsV2/Rv0ZHXXgpz8K/PDNsAOUivBQkVcf/D1g16eDxV7TGPdtliaExVdJkw1Vgt/0yRo4zz8SJXz1O3MB14laT9xKitLR7ddTW8Nti7wHehvGLEEJRcyRpCilVRvlc1ivSyeu+azozvBXHg80OkeQRZejtwnfN5xNodoXmnoOb0fD66+aU9jqcsRV5PMtlesaX9ltiO3SQrJqEbbr11R/BsB3/x9wxY2Bjb0WxtzrxKtU/qP7w2Jmogbs+ENJ9MppS9WwVeFzu8Jj/NgS4AfHm2vi6OONOzfX9yrxCgD2PQjc9JdujTMtmtowVtMHJbjmSNp7qs5/ZhzY/ilEgi2G18eHpOpjaETWxcTndzF6m/ABf4lnYERGU0w8gIgUV9OiJRYuQmjjhXxtkygUdPti7YxdOkrqSlRGoldohxaSVUJ0RXVcsTLaMhCQRHB4O4AKcOWqaJMRtS+lDb7/I8Dzu8MoLlEzzDb1cKGoVGXUDyDJXnf0FnFfp8cQqSsjasVqnNNjzW0Y0wYl2OZInnuqB1vs/zJwzW2hBu7ahzmGvImSJQYTvi9mxtEIy4qo4RQ2otj1R1EH3LW32cPA1P6O7kOkIYNZqjVpQXCNsyxSfxYtJO34s2gRJqHoD/gNnwRe+JYjfr4upfIbPhF+NLFJSvHqvlfPAX76V2VGpxDS/yNEKOHr9nRdgKjPyTGk6TCVBLOuDCDnUlGalkmUStN1fa/IXL+/tjmStXCdGWxRn5PX98DW6KLhM4Z5ilyET0Q/DuBrAAYBTAP4DSHEW5btagCeDd7+mxDiljzHbTsiWXcV+Zkqq0AVqSo3YnURfn/BQLwpZ+609qFRqnVmXJoH4haEuHGWxWaeRgvJMv4sWkRkkbBkQq/fIbc5/bYMyfzhm+FvRU3Gw6/4jHy/886oAFA7C3zvH4K8u8B8s/gqWZL4B8eiEThDayQZqXl1eIe8x0WZXAZG5LnsvQd4bjcAEXX2FrH/OKI0vwfs99ecI1k1QzPYAkBDC9fvmc8Y5inySvh3Afi2EGIDEd0VvP/vlu1OCSGW5TxW5xCx8QG4dp1MgZ89Isle1GQ0hy7924pfmfvTIzZ004/PgpA0zm50OGUZfxbprCmpxnDSKSnbjNhReHmPjNBadmtz6j5Vwv0JCu/Zi4/J/esROGrse++VJqOj+8KkriJJ/9YtrdP80vgWfCX3tPdUP7f12+X1fO0Z4O0jaNTAemlPeM+6+RnJibyEvxrAiuD1gwD2wE743Q2b6qqicoggqx+eG2RZxhS/su1PFdnqHwpDA30WBNvD0A2RO3HIOv600pkZ9bL7rrAVoNKiTPOADlUmQ90X9dsrbpQ2fDNJJ24hGxgBzv4wuv/JzcURvn7ORRJblgUkzf31Ha/NPPfCN8N7RxSURtDuWTc/IzmRl/AXCyFeC14fA7DYsd15RDQBYA7ABiHEP7p2SES3A7gdAJYsWZJzeAXBJnGoUK56XXr1V27we0htXYWA5kmrF2vTCzzFmT263RaZNP4ipFR9H7q9XC9/vfiqKDkBzX4btfAPrWkeky0iJI5klq4OczwAKZ2WuSlHVtNhK+anuZgeethwVEMKVcpk57pnPYJEwieiRwH0W776rP5GCCGIyJV/fpkQ4igRvQ/AY0T0rBDiJduGQoj7AdwPAMPDwwklBtsIU+JoSICG6h4H14NiqrqnTrgfjCSzR6ttka12CrvGX1SCj20fp04gUv7aTK0//bbWBxWylrweqaMWYpWdaZ5DEtENrwde/FYQCYRsjtV2Ouvzlmgocnym1rB0NTD9r+FCXT3HHv1ku2c9gETCF0J82PUdER0noouFEK8R0cUAXnfs42jw/2Ui2gNgOQAr4XcN4tKxXZPHFXVjU3VdD0YnzTa+pNuqUrc6yUxtSX8MF1Hpzj7drKPuwdhGRPqgXnJNc1hmntr6gFxAXvx2tvvabmd9J+agbz4KIE2kJ48D5y+ONjE391e2AIc2IK9JZxuAdQA2BP8fNjcgoncD+KEQ4gwRXQjgBgCfy3nczsNmB06aPAsX2aNuXBJgXAXATqikPpJdK0vd6n4PVevGrIMTd13iMqtXbgizMZVZR+1jcDTsg2pb3Itwlue5r+121rdqDrrun28+Spq51+0BDhmRl/A3APh7IvptAK8C+A0AIKJhAL8rhPgdAEsBfIGIAvEIG4QQB3MetxzQJUCfyXPqBCKSom4GMiXAJFt9Jyanj2TXqgdJJxk9Oqr2jozK+MFr0vYt6vFStouolFnHllKftLgXJfFmva/q+KbjuZVohRPYNd9951SaudftAQ4ZkYvwhRAnAPyC5fMJAL8TvP4OgA/kOU7p4Tt5TEkxbpKVUQLxkexa+SDpklwkfn17dLukkD+bBpU07rjFXdn7VR/VIpDGLKZrKMKiobR7PFn2Fzff0zxfaaKAujnAISM407YI+E6eNJOsrBJIkmTXigcpLjPy8I7mhiQ+18smUfqMO+6+HHhIfm5mdmY537RmMZvjuUhTS5FmuqSGI+Z1bcXzpbbvEaJXYMIHskkvNhLy+W2a7bpVAinyQXKRjfpbuChK+JfdAFx0RfJ+bRLl6J1+JGE6Ccc2ytaXplPZ1jHNB1m0u1YKCK7xZJX6Xde+iDIcLg2um56fFoIJP4v00irHZNZFZD4jifxU7oOqZf7UF4B/ezJZyk5LkLZ7Y/a6rVQD90wV2P+VsH7O5FdliQPfe5mFvFspINjG04p2ia75bjvW8YPRksfm8Xs0CicJTPhZpKmi7euq2mJSE+tehA/5qfK3e+7Wwl4T7ksagmyQR9AmTzW10edBHdGSG3pLw9rZ9LHqWci7VQKCbTxZC5y59heHppDcrcD+zWGlTrO/L5Bcp79HwYSfRZoqUn229SvViz3Np0laZN16c78P3mJ0hPK4L74EqedP6G3yzHmgqkVObQ1q56sKmTF1lfKOzRdpr32Stpn3GUhzfuaxIMLwZkBmuuvHT6rT38Ngws8iTaX9jV4v3bTnNjJ2tbo5erGn+SLp51Gxk8hBz3pGBXjfimIXy8FRe5s80+4MRE08V94MnH9Reht+0Uh77X0TyYoyISUtRja/id6YfNXG5mcqrk5/D4MJH8gmTfn+xmzKYNpzIwlFfcC7B4P+qJ7RFt3imGplmKkpARatGQ2MuNvk6fNAN3PUITNyi6xvnxX6tZ87LYUPnwXUp6qlzUGq9uErDPksRuoztd+4xcaleTGY8FuOhvQZQEVw6FEFegPlJz8fblvpi1dFy+CY8l1wWh2f3+qIpqQ2eUB5Q2kHR+VcqtUACOkrcpUcUNundWgrHwco1IZ85qTv4mKb667FtJsj3FoMJvxWo/HwaE0ZJrdEq182NVAGpCqaYArodHJWmgWn1Q9hOyKabMcwF7wyEs3AiJxLE5sgu2vV4ol1eixdmz/dxwGElUXNOWkTDnwXl7RznSPcrGDCLwJxUu7AiGzKsPsu2cFKtbNTE1afyGYD5aG18cdbuKj1EmXcuU1tTRcJMd8ewrgcgTyVGFthphtaGyaGueZKVo3R9HEAaJSQVseJu1Z5E96S0C1mzzaACT8vfB1cKzdEHXqzM/K35kQ2JStzssb1Yy16Msed28y4NA00IiESzE/zEXEJSVlNba0y0/kQa1aN0fRxVPuko1Q3G+lagKoUqz4fHI2aZ4osGlgGs2eJwISfFnE1QOZOB/VU4J6wKt5+32Ypca3bFtrwl66ONlGxTVbzoSy68bWOOAKYHtN6+HqYn+YjXFJnHlNbK810SRpWHik6ycdhVop9/TDwL3/RTMQTm9wJVVk0xE6bPUsGJvw0cNUAqVRDh9j+r8hIHFsClVog6rVoEokqBPbqE9GiV7bJGonqqQZJPptaI+XHEUBTJITD/DSf4ZI68xCnGbWlNMF2kFSSFO0TPql/rm9/6gRkYblAI/zu14OXWjQaEJC9Sqg6JU2hKzdkP/+yOtI7BBKiPE2lTAwPD4uJiYlODyPE2Ebgsf8tCZiqwM9/VkrX2z8VZFYKhNUSRXQbBTNr86d+CXhud2jDv2Q5cPFQGEpmU0dnxqUmMblFJveIIP7clmKeF3EPeS/ZRvMmLqXBxCbZ1/a1qbA1X6dNEVli+TfdFESoEfCBXwe+9w/R+Hi99aBaaB77X4YvAED1XOkHy3r+vTRPARDRPiHEsO07lvDTwCUtDK0JpfRKHxqREK4yu3qzjRceDTSEoBb70X3yb/KrcpK7JK63XtXIHmhZ3H6cGj3fnLAuZLEDZ702KmorknndQlOE73xIaxqZ2qKFIwvg2b8HPvAbAekH5hqb/6l6bvTcgfzn3yvz1ANM+GngUnnNz4H4h0hvtlGfkzVY3poGXvpnND3kZgVHs4xAAx7lBCLFvqrNjjWGHe20A9syr1sZgeW7kKU2jVj6AvzwTeC3dsVHtCk/l14rh00xhYEJPy1c0oL5eRrnmDLfmM2XbZM8UkYgQGUBcM3HklP4I8RVk2aovLXbewHttAObNvzla1tXmiFN0lPa2PyhNVHSBmRQQpK0rb4fWhM2lGGhpDAw4XcCLk1h/Y7kSa4IQVd7RR24YCD5oWj6bYZKgj1mDwWQPSSwTMdKm/Sk53r49Gu2Hee3doXtJ5ffFo1AS0IWM0wvzs2UYKdtq5G1uUpc8wzdaVs7K/uYXrESuOGT8nvzeGadk4bDd675IU5y0qq6QFQFbvrLdA9x1uvBSAdn7oZR3tm2rfpMmQ0Jmp+oAvzC/7CHAbtKfAPtud8cb98AO207hSyRDVNbkptnKOmnfxmw4w+lSn54B/D8I5KIdSIHmsdw8z1yIXE96K7xTm0NS0SImjx2mt6p3fRQduvCZLvGeka0Xt5ZzSPz/KbHtO31L+phg3RTiGgq8f2OnMt6dm8r7zfH23uBCb+VSDMJbXXxAcQ2zzh1IhrCVp8DEOQD6LHNtjG4HvRGEpmtJr+hDYp6eZKKikSahalsC4OZCLj3XuCFbyJy7+q1+H4LCxeh6V4DACoyQfCtV4An7gsKpJ0LLFtjdzSDst3vLNeU4+29wITfSuiF04hC6ciGpugMBeH+nVmYrdIXlfDVpPd9EBo2/iAC6OU9YU1+ALISYjUsjpUnqUhvlVcmwgTyVXDs9DmYlTGf2xVEhOkQ8f0WTp1w7LwOvPSY/FOonZH7szmagTBc2XeuZL2m7fSzdDGY8FsJM+Z+911uE0iDbE81f3dsyr3/9duj9n6gedKneRCW3Qq89gzw/cmwVWAjGzgof3vZh4CLrkwfPWELXy0bYQKtq+DYDgyMRCtjQoQF+SpVoP8DwNFJxOZt6OcPAKiE+zJBFTnvbCZCID0J57mmHG+fiFyET0S/DuBPASwFMCKEsHpYiWglgHsBVAF8UQixIc9xuwp6zH3cBFaLw+N/Drz9fePLGMe6bZLb3ivnnV7B0WaHVTH6lQWhpgCBSPnbmXHgA78ZmozSkr7aPmtf1FZrBb7SYqfMCEnnb1bGXLkBOHYAAAH9Q7IBeNyYB0aiEWMQwMSDiM7DYAHRu03F3YvjB/3uGZtmWoq8Ev53AfwqgC+4NiCiKoC/AfARAEcAPE1E24QQB3MeuzvgO4EbGZZnop9X+oqpU2OrsqmH2y27NTTl1AQwvE6Geqrx7v+y1uKvBuz4lHz+85RzyPJwR5LHWhin7iMtdsKMkKX9IBC91z7x9Pr5qx6xKjrrQ78PnPeu5HO29hpOmC9smmkpchG+EOIQABBZsupCjAB4UQjxcrDtQwBWA+gNwvedwJGEKlVT5+riyMxUlQ89HH1/8g2EyVx1GQGkh1w2yt8GtmFlF547Ey0LnLZsQy6VP0ges0UyZYWKlLLlQrjK9raTlJqqsya0KzR/k6bCqh6Lr+pEVarAlTelu1dqXokELVeBTTMtQzts+JcAmNHeHwFwXRuOWx74TGBT2s1TIdCGhYuk41gEJRiWrpZOO3W88y8Km1hQpdlxp8rf7rk7WgICkBUdJzalS9BRSPtw2xLPVAhglutlmrUaBb8Q1jPSY9nnzkibuB7L3k5EHPiOdoU2bS6PJkUUdGITMmrM91qbQQBqwWAzTceQSPhE9CiAfstXnxVCPFz0gIjodgC3A8CSJUuK3n150UpVVpmLRF2S1coNkryX3YqGJAskd0QaGJGhfHoJCNSlfbcSmHx8pTjbGL0dy2uA6b1Bs3cFcicRKSnVNGOYxLhsjSQ0Bf08psdC4qobseztgqoVr6KkAHu7QptEr+bWwkXxvhd1vWaPhPtAJRAWALnIbJEaoI9ZaN02LbdkDtYaO4y2IZHwhRAfznmMowAGtPeXBp+5jnc/gPsBmWmb89jdhVapsg0CCKSsYwei0riSEG0LjkmiAyOy6FqjHDQgSRByMclS7MvHLh0pCT0n7fe6Y7l/qHkfQJhhaishbRIjBFBdYK9nNDgqz6+umSfaFZWjzn3f5ijZu661zTeixqln3F5/R9QWbxbXq/QBdSG3HbgOePU7aEj5O+/0K92srrMINAS9vad+fmyzbwvaYdJ5GsDlRPReSKK/FUABXkiGN0wCcCXEmAuOq51i/xDQd14zkWZtt5gUimdLSqtDVhm94FJ5PNs+AG2hC36k77+piF3gADZt+IqQrr8jmnDUDtOEKyEPkGO0mf5ci/f0WHjPRB3Ye0/UkapfwzqAK24Ent8tF7kjT8trVJ8LpP0U2lxSzZ4yhubOU+QNy/wVAH8N4CIAO4jogBDil4joPZDhl6uEEHNEdAeARyDDMr8khPhe7pEz/GGL2vBJiImQ6JmoVHfdx4FjzwD9V/tFbMQhKVpHEZWZyanbro8fjPooIkln2sKkf+cixrhFb9XGcFEDsjcq94UzIQ/yM1eSlE1bHBxtbjauk7Z5H86/SCvjXQOuvU1GbpkF1ZIWvjhzZRlzGeYx8kbpfAPANyyffx/AKu39TgA78xyLkRMmAfiYbxpZm4EpqF4HUJfS5nfulRw0vTdfNyI1tjj/hd4PFZBRIjd8Ijpu00ehvtNt1zYbvm/0lBnh4pJMizZP6CRMFTkOdS2qC+LDfG0LmRltRZWo2adJMHgoqgGpfcX1r7XBZa7kuPu2gjNtexVJ5ptGOYVAsiQKyP8sgHoocNbO+IUGph2PPq5DDyPsh1oBLrnGISUGC5Mu9drOc2yjf9lfV3kMlwmpaPOEjYTjKqmqc3QtRqdOhFqKbRH0EQxs29nG4LMgcNx9W8GEz5BwEZged3/NWtlaUa+lAiA2E9gFH0LQy/oqsu8zbOcz4zKiRJkqKn3xUq8eauiyQ5tjs5XHsEmmecwTtpLGprNcIW3c/fSYNHntvDP0P5iLUZGaSVq7PMfdtw1M+AwJl2ptOjWBaFhmZUH6TOAkQlDkc3S/VluIgJ9cEa3wqC8IDZOPZfFphBrOhEQoglBDMzbcNjZbeYzRO+2SaRbzRFIWdFpNwbyXCxcFC1bQfap2pnmBc2kEWTQWtsuXFkz4DAmXap21M1cc4gjBSuIAIKSD2GnKCWDGpZulGCrVIIS0KklcmYFsY5s7Lc9zaK2dyG2Sd5J5wiZJJ2VBK23LVwI3xzE9FoaTAlIb0hcj1/0wP5/ayvVwuhxM+IwQNtVafWYWXssjsdkkULVvG4krHHsm+l7PHlYlKXSCmRmXmcFq8dBDOWePAPuCgmB6bPjgqFwMVHnhyYDwfROX4q6NS2I2r4eZBb1wUfrm8+Y4+s6NZgnr37kIWv+80tfczSou9p7t8qUEEz4jGVlU+zibsLKLH3pYSu1mYS+zdILC0tXR/e++S0qulQpw/R8AZ2bRkNathbvOicbWq9DUSp8098yMy+/0xDK1GKjaM3mcsi5JWr8eS1eHZSx0Cd2sH5Sm+XwSAceFp6rPZ2fCxC+uh9O1YMJnJCNi5rB1wjLgY6NXJP/K42G4p14GwKzZc+XN0do1emEuQZLsVQjhga2ybESjcFcFeN+K6JgVmak+rPs2y9+v2yYXBVuegk+CWJxU65KkGyUT6lKy19sPKkQWwQzN55MI2PW9ruElld5glB5M+IxkmEWwXt7j7pYEJBOj/j0qYYMOPR58xWeiZo0bPmEfkyt7GBTmEFQXSMnZNMU07NRz8rxU5U+XQ9YsQJeUMarOVTeDmfudGY93qKpxqpo0evP5dpIum2nmBZjwGclQD/ueuyXZJ6XUJzntbJVBbfHgacwQQFQqP/ddIYmKOrDr03b788JFiJSFVnH2tvh9W3KXLQJIOTgbJiPD7m4ufnEOVXXs6bH4zlLtAJtpuh5M+Aw/2KRul4SZ1WZs208aM4TuWNWrSirit5lCTp2ILwutYDqTDwWFYpVpqhEBJKQWcPK4n919cBSo9gW5AdWoQ3VmPDQ56YuVTy17BsMCJnyGP9Ko9VltxnnHNzAiI35MqblSDUlTX6gGR2UiUtIipmfcirpsAv7K42Hyllls7IVHg2ifoEqkWmysoY0qH6BP2u8Be9E0l1bF1SYZnmDCZ6SDL1F3koQGR4MwxNNS2v7Q78v6O64SAb7aRsOZvAfWktCRYmNzMgRUhXaqOvv7N0fLCk+PBRqIER7aVDTNUQqZq00yUoAJn1E8Ok1CKsxRRb489QVJ+C5TSFLsvL4YmGYt3f8AGMXGApt9/7KwYJkwpHXfGHhX717OamWkABM+o3iUgYRUOQRR9+/9asK1cMVV4LRpC43SDFpegSrn4BMDH6d5cFYrIwWY8BnFowwk1CjtrDJmLb1fkxCXKAXYFwNbdM/skdCh28gg1so5JMXAx4HDJRkpwITPKB5lIKGBEWkGmdgEaR+vpdM0ZsZlqGWlT9rqzYXLR4sx6/i8ZzlwdBING39Rmg+HSzI8wYTPaA3KQEJDa7Nlh5q9Xa9d16wd+Ggx+qJQB3DxkCxTzOYXRofAhM+Yv8iqaZhEfcGl0SSrONu7DnNRGFoj/9j8wugQmPAZ8xtZNA2b9O5y4CblGiT1zGUw2ggmfAbDhI2oxzZmizwqg2mLwQjAhM9g2GASdRkijxiMnGDCZzB8UIbIIwYjJ5jwGQxfsHmG0eWodHoADAaDwWgPchE+Ef06EX2PiOpENByz3TQRPUtEB4hoIs8xGQwGg5ENeU063wXwqwC+4LHtzwkh3sx5PAaDwWBkRC7CF0IcAgAiStqUwWAwGB1Gu2z4AsA3iWgfEd0etyER3U5EE0Q08cYbb7RpeAwGgzH/kSjhE9GjAPotX31WCPGw53H+oxDiKBH9BIBvEdFhIcTjtg2FEPcDuB8AhoeHhW0bBoPBYKRHIuELIT6c9yBCiKPB/9eJ6BsARgBYCV/Hvn373iSiV/MePyMuBDCffQ7z/fyA+X+OfH7djVad32WuL1oeh09E/wFARQjxg+D1LwL4M5/fCiEuaungYkBEE0IIZ+RRt2O+nx8w/8+Rz6+70YnzyxuW+StEdATA9QB2ENEjwefvIaKdwWaLAfwrEU0BGAewQwixO89xGQwGg5EeeaN0vgHgG5bPvw9gVfD6ZQBDeY7DYDAYjPzgTFs37u/0AFqM+X5+wPw/Rz6/7kbbz4+E4EAYBoPB6AWwhM9gMBg9AiZ8BoPB6BEw4QdIUQhuJRE9R0QvEtFd7RxjHhDRjxPRt4joheD/ux3b1YIidweIaFu7x5kWSfeDiM4loq8F3z9FRIMdGGZmeJzfeiJ6Q7tnv9OJcWYFEX2JiF4nou86vici+qvg/J8homvaPcY88Di/FUQ0q92/P27pgIQQ/Cf9GEsBXAFgD4BhxzZVAC8BeB+AcwBMAbiq02P3PL/PAbgreH0XgD93bHey02NNcU6J9wPA7wH4u+D1rQC+1ulxF3x+6wHc1+mx5jjHnwVwDYDvOr5fBWAXAALwQQBPdXrMBZ/fCgDb2zUelvADCCEOCSGeS9hsBMCLQoiXhRDvAHgIwOrWj64QrAbwYPD6QQC/3LmhFAaf+6Gf99cB/AJ1T7W/bp5vXhCyxMq/x2yyGsBmIfEkgB8joovbM7r88Di/toIJPx0uATCjvT8SfNYNWCyEeC14fQwyIc6G84LidU8S0S+3Z2iZ4XM/GtsIIeYAzAJY1JbR5YfvfPu1wNzxdSIaaM/Q2oZufuZ8cT0RTRHRLiL66VYeqKdaHBZUCK60iDs//Y0QQhCRKx73MiEL3b0PwGNE9KwQ4qWix8ooDP8EYKsQ4gwRfRxSm/n5Do+J4Y/9kM/cSSJaBeAfAVzeqoP1FOGL/IXgjgLQJahLg89KgbjzI6LjRHSxEOK1QCV+3bEPVejuZSLaA2A5pB25jPC5H2qbI0TUB+ACACfaM7zcSDw/IYR+Ll+E9NXMJ5T6mcsLIcTb2uudRPR5IrpQtKhZFJt00uFpAJcT0XuJ6BxIJ2DpI1kCbAOwLni9DkCTRkNE7yaic4PXFwK4AcDBto0wPXzuh37eHwXwmAi8ZV2AxPMz7Nm3ADjUxvG1A9sA3BZE63wQwKxmmux6EFG/8ikR0QgkJ7dOIOm0F7ssfwB+BdI+eAbAcQCPBJ+/B8BObbtVAJ6HlHo/2+lxpzi/RQC+DeAFAI8C+PHg82EAXwxefwjAs5DRIM8C+O1Oj9vjvJruB2Q11luC1+cB+L8AXoQs3ve+To+54PO7G8D3gnv2zwCu7PSYU57fVgCvATgbPH+/DeB3Afxu8D0B+Jvg/J+FI4KurH8e53eHdv+eBPChVo6HSyswGAxGj4BNOgwGg9EjYMJnMBiMHgETPoPBYPQImPAZDAajR8CEz2AwGD0CJnwGg8HoETDhMxgMRo/g/wNLZ/mc5yL5TgAAAABJRU5ErkJggg==\n", 272 | "text/plain": [ 273 | "
" 274 | ] 275 | }, 276 | "metadata": { 277 | "needs_background": "light" 278 | }, 279 | "output_type": "display_data" 280 | } 281 | ], 282 | "source": [ 283 | "plt.plot(samples[:, 0], samples[:, 1], 'C1.')" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "id": "466f2945", 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [] 293 | } 294 | ], 295 | "metadata": { 296 | "kernelspec": { 297 | "display_name": "Environment (conda_py39)", 298 | "language": "python", 299 | "name": "conda_py39" 300 | }, 301 | "language_info": { 302 | "codemirror_mode": { 303 | "name": "ipython", 304 | "version": 3 305 | }, 306 | "file_extension": ".py", 307 | "mimetype": "text/x-python", 308 | "name": "python", 309 | "nbconvert_exporter": "python", 310 | "pygments_lexer": "ipython3", 311 | "version": "3.9.7" 312 | } 313 | }, 314 | "nbformat": 4, 315 | "nbformat_minor": 5 316 | } 317 | -------------------------------------------------------------------------------- /03-SGM-with-SDE-MNIST.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "910f1769", 6 | "metadata": {}, 7 | "source": [ 8 | "This is the notebook to show the implementation of score-based generative model with SDE (2nd part of the tutorial). In this case, we will sample the training data from the swiss roll distribution like the previous part.\n", 9 | "From the training data, we will try to learn how to draw new samples from the swiss roll distribution with Score-based Generative Model (SGM)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "id": "3084c1e6", 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "name": "stdout", 20 | "output_type": "stream", 21 | "text": [ 22 | "torch.Size([1, 28, 28])\n" 23 | ] 24 | } 25 | ], 26 | "source": [ 27 | "import torch\n", 28 | "import torchvision\n", 29 | "\n", 30 | "# generate the MNIST dataset\n", 31 | "transforms = torchvision.transforms.Compose([\n", 32 | " torchvision.transforms.ToTensor(),\n", 33 | "])\n", 34 | "mnist_dset = torchvision.datasets.MNIST(\"mnist\", download=True, transform=transforms)\n", 35 | "print(mnist_dset[0][0].shape)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 2, 41 | "id": "685e64d8", 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "data": { 46 | "text/plain": [ 47 | "" 48 | ] 49 | }, 50 | "execution_count": 2, 51 | "metadata": {}, 52 | "output_type": "execute_result" 53 | }, 54 | { 55 | "data": { 56 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS4AAAD8CAYAAADJwUnTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWzklEQVR4nO3dfZAdVZnH8e+PEJIlBCRGYyQRIoaVABpwFrCgAAvEQFm8lIpEV0HRuEhUFF2RtYBltQpcwUWM7A4SAYt3BMm60YgsiroQCYiQgGCMQRJDYghv8pZk5tk/uoN35s49987Mnenuye9T1ZXb/XSfPmngoc/p06cVEZiZVck2RVfAzKy/nLjMrHKcuMyscpy4zKxynLjMrHKcuMyscpy4zGzISJovaZ2kpQ3ikvRNScslPSBpv1bKdeIys6F0BTArET8KmJ4vc4BLWynUicvMhkxE3AlsSOxyLHBVZO4GXiVpcrNyt21XBVuxncbEWMYN5ynNtiov8Twb42UNpox3vWNcPLmhq6V9733g5WXASzWbOiOisx+n2wV4vGZ9Vb5tTeqgQSUuSbOAi4FRwHci4vzU/mMZxwE6fDCnNLOExXH7oMtYv6GLxYumtLTv6Ml/eCkiOgZ90n4acOKSNAqYB7yTLEveI2lBRDzUrsqZWRGCrugerpOtBqbWrE/JtyUNpo9rf2B5RKyIiI3AdWTtVTOrsAC6iZaWNlgAfDh/ungg8ExEJJuJMLimYl9t0wN67yRpDtnTAsay/SBOZ2bDpZv23HFJuhY4DJgoaRVwDjAaICL+E1gIHA0sB14APtJKuUPeOZ931HUC7KgJnkPHrOSCYFObmooRMbtJPIDT+lvuYBLXgNqmZlZuAXS1pxk4ZAbTx3UPMF3SNEnbASeStVfNrOKGsY9rQAZ8xxURmyXNBRaRDYeYHxHL2lYzMytEAF0lnxl5UH1cEbGQrHPNzEaQYRsMMUDDOnLezMoviNL3cTlxmVkPEbCp3HnLicvMehNdDOp1xyHnxGVmPQTQ7TsuM6sa33GZWaVkA1CduMysQgLYFOWeY9SJy8x6CERXySdHduIyszrd4aaimVWI+7jMrIJEl/u4zKxKshlQnbjMrEIixMYYVXQ1kpy4zKxOt/u4zKxKss55NxXNrFLcOW9mFePOeTOrpC4PQDWzKgnEpih3aih37cxs2Llz3swqJ5CbimZWPe6cN7NKicDDIcysWrLOeb/yY2YV4855M6uUQJ5I0Myqx3dcZlYp2XcVnbjMrFL8JWsrmLZN/yMe9ZqJQ3r+Rz6/W8NY1/bdyWN33X1dMr79J9P/cT1x0XYNY/d1XJ88dn3X88n4ATeekYy/6XN3J+Nlln2ebAQ/VZS0EngO6AI2R0RHOyplZsWJUOmbiu2o3TsiYqaTltnI0RXbtLS0QtIsSY9IWi7pzD7ib5B0h6TfSHpA0tHNyix3WjWzYZfNx6WWlmYkjQLmAUcBM4DZkmb02u3LwA0RsS9wIvDtZuUONnEF8BNJ90qa09cOkuZIWiJpySZeHuTpzGzoqZ13XPsDyyNiRURsBK4Dju21TwA75r93Av7crNDBds4fHBGrJb0WuE3S7yLizh41iugEOgF21IQY5PnMbIhlwyFafqo4UdKSmvXO/L/5LXYBHq9ZXwUc0KuMc8lugD4FjAOOaHbSQSWuiFid/7lO0i1k2fXO9FFmVmb9fFdxfRv6t2cDV0TEhZLeDnxP0t4R0fCx84CbipLGSRq/5TdwJLB0oOWZWXl0s01LSwtWA1Nr1qfk22qdAtwAEBF3AWOB5DidwdxxTQJukbSlnGsi4seDKG/EGrXn9GQ8xoxOxv986KuS8RcPbDzmaMJO6fFIv3hrejxTkX70wvhk/IJvzUrGF+9zTcPYHze9mDz2/LXvTMZf/4uR2+uRTWvTtgGo9wDTJU0jS1gnAh/otc+fgMOBKyTtSZa4/pIqdMCJKyJWAG8d6PFmVl7tesk6IjZLmgssAkYB8yNimaTzgCURsQA4A7hM0mfJuthOjojk/xk8ct7Meshmh2jfSKmIWAgs7LXt7JrfDwEH9adMJy4z6yF75afcQzyduMysl/K/8uPEZWZ1WhkVXyQnLjProc1PFYeEE1cbdB22XzJ+0RXzkvE9RjeefmUk2xRdyfjZl5ycjG/7fHpIwttvnNswNn715uSxY9anh0tsv2RxMl51biqaWaV4znkzq5wANvuOy8yqxk1FM6uWcFPRzCpmy0SCZebEZWZ1fMdlZpXSz4kEC+HE1QZjHknPNHvvS1OT8T1Gr21nddrqjDUHJuMr/pr+vNkVu9/UMPZMd3oc1qRv/l8yPpRG7qQ1zQVic7c7582sYtzHZWbVEm4qmlnFuI/LzCrJicvMKiUQXe6cN7Oqcee8mVVKuHN+67B5zRPJ+CUXvC8Z/+qs9CfERj2wQzL+209ekoynfGX9W5Lx5Udsn4x3Pb0mGf/A2z/ZMLby08lDmcZv0zvYkAknLjOrFr9kbWYV5DsuM6uUCOjqduIys4rxU0Uzq5TATUUzqxx3zptZBUXJ5/Vx4hoGE757VzL+mv9+dTLe9eSGZHyvvT/aMLbskPnJYxd0HpqMv/bpwc2Jpbsaj8Walr4sVqCyNxWbvpAkab6kdZKW1mybIOk2Sb/P/9x5aKtpZsMle6q4TUtLUVo58xXArF7bzgRuj4jpwO35upmNEBGtLUVpmrgi4k6gd1vlWODK/PeVwHHtrZaZFSlCLS1FGWgf16SI2PKS2hPApEY7SpoDzAEYS/q9NzMrXlBsUmrFoBupEREkvi0QEZ0R0RERHaMZM9jTmdkwiBaXogw0ca2VNBkg/3Nd+6pkZoUKiG61tLRC0ixJj0haLqnP/nBJJ0h6SNIySdc0K3OgiWsBcFL++yTg1gGWY2Yl1K4+LkmjgHnAUcAMYLakGb32mQ58CTgoIvYCTm9WbtM+LknXAocBEyWtAs4BzgdukHQK8BhwQtO/gTXUtf7JQR2/6dntBnzsXh98KBn/y6Wj0gV0dw343FZebXxiuD+wPCJWAEi6juzhXu2/eB8H5kXEU9m5o2kLrmniiojZDUKHNzvWzKqnn+8qTpS0pGa9MyI6a9Z3AR6vWV8FHNCrjD0AJP0KGAWcGxE/Tp3UI+fNrKcAWk9c6yOiY5Bn3BaYTtaymwLcKWmfiHi60QHl/pSHmRWijQNQVwNTa9an5NtqrQIWRMSmiPgj8ChZImvIicvMemntiWKLTxXvAaZLmiZpO+BEsod7tX5AdreFpIlkTccVqUKduMysXpsGckXEZmAusAh4GLghIpZJOk/SMflui4AnJT0E3AF8ISKST6zcx2VmPUV7Z4eIiIXAwl7bzq75HcDn8qUlTlwjwJ5ffLRh7CP7pB/+fnfX25PxQ993WjI+/vq7k3GrKM/HZWbVU+53FZ24zKxed9EVSHPiMrOe+jeOqxBOXGZWx3POm1n1OHGZWeW4qWhmVSPfcdlQ63r6mYaxJ0/dM3nsnxa8mIyf+ZWrkvEvnXB8Mh6/2alhbOpXm3yfrOwdLSNVCFqcJLAoTlxmVq/k/89w4jKzek5cZlY5TlxmVikegGpmVeSnimZWPU5cZlY1vuOyQnX/9uFk/MR//UIyfvU5X0/G7z8wPc6LAxuH9ho3N3no9MvWJOObV6xMn9sGzn1cZlYpLU7LXCQnLjOr58RlZlUjTyRoZpXjOy4zqxKFnyqaWRX5qaKZVY7vuKzMJsxPz4k195H0dxV3PH9VMn7tGxc1jC378LeSx7556seS8b//1/SH2Lt+n/yKuyWUvamY/icPSJovaZ2kpTXbzpW0WtL9+XL00FbTzIZNZE8VW1mK0jRxAVcAs/rY/o2ImJkvC/uIm1lVRYtLQZomroi4E9gwDHUxs7KoeuJKmCvpgbwpuXOjnSTNkbRE0pJNvDyI05nZcNkyJKLZUpSBJq5Lgd2BmcAa4MJGO0ZEZ0R0RETHaMYM8HRmZn8zoMQVEWsjoisiuoHLgP3bWy0zK9RIbCpKmlyzejywtNG+ZlYxFXiq2HQcl6RrgcOAiZJWAecAh0maSZZzVwKfGLoqWpH0q/uT8Rfe+9pk/B/e/6mGscVfvDh57O/e8Z1k/IO7HZmMP3NwMmwpJR/H1TRxRcTsPjZfPgR1MbMSEOUfgOqR82ZWr+SJazDDIcxsJGpxKESrd2WSZkl6RNJySWcm9nuPpJDU0axMJy4zq9fd4tKEpFHAPOAoYAYwW9KMPvYbD3wGWNxK9Zy4zKxOG++49geWR8SKiNgIXAcc28d+/wZcALzUSqFOXGZWr/VxXBO3vBmTL3N6lbQL8HjN+qp82ysk7QdMjYj/abV67py3Qelauy4Zn/TNxvGX/nlz8tjttV0yftluP0zG33386Y3LvqWlFsnWqX+DS9dHRNM+qUYkbQNcBJzcn+OcuMysThuHQ6wGptasT8m3bTEe2Bv4mSSA1wELJB0TEUsaFerEZWb12pe47gGmS5pGlrBOBD7wymkingEmblmX9DPg86mkBe7jMrM+tOuVn4jYDMwFFgEPAzdExDJJ50k6ZqD18x2XmfXU5heo84lGF/badnaDfQ9rpUwnLjPrQflSZk5cZlav5K/8OHGZWR2/ZG2V1n3wzGT8D+8bm4zvPXNlw1izcVrNXLJh32R8+1uTD6YsxYnLzColip0ksBVOXGZWz3dcZlY17uMys+px4jKzqvEdl5lVS9DSJIFFcuIysx78sQwrnDr2TsYf/XSTOa8OujIZP2Tsxn7XqVUvx6Zk/O4N09IFdK9pY222Mk5cZlY1inJnLicuM+upzbNDDAUnLjOr4z4uM6scv/JjZtXjOy4zq5R+fKW6KE5cZlav6olL0lTgKmAS2V+nMyIuljQBuB7YDVgJnBARTw1dVbde207bNRn/w0de3zB27vuvSx77nh3WD6hO7XDW2vTn+H5+8YHJ+M5X3tXO6liuCgNQW/nKz2bgjIiYARwInCZpBnAmcHtETAduz9fNbARQd7S0FKVp4oqINRFxX/77ObJPDO0CHAtsGVZ9JXDcENXRzIZT9GMpSL/6uCTtBuwLLAYmRcSWdyqeIGtKmtkIMGKGQ0jaAfg+cHpEPJt/LhuAiAip71axpDnAHICxbD+42prZ8BgBfVxIGk2WtK6OiJvzzWslTc7jk4F1fR0bEZ0R0RERHaMZ0446m9kQU7S2FKVp4lJ2a3U58HBEXFQTWgCclP8+Cbi1/dUzs2EXQERrS0FaaSoeBHwIeFDS/fm2s4DzgRsknQI8BpwwJDUcAbbd7Q3J+DNvm5yMv/+8Hyfj//Sqm5PxoXTGmvSQhbu+3XjIw4Qrfp08duduD3coSuX7uCLilzT+Ivfh7a2OmRWtCuO4PHLezHoquBnYCicuM6vjOy4zqx4nLjOrGt9xmVm1BNBV7szlxGVmdXzHNUJsO/l1DWMb5o9LHnvqtJ8n47PHrx1Qndph7uqDk/H7Lp2ZjE+8aWkyPuE5j8WqpDY+VZQ0C7gYGAV8JyLO7xX/HPAxsplo/gJ8NCIeS5XZ0is/ZrZ1adcrP5JGAfOAo4AZwOx8WqxavwE6IuItwE3A15qV68RlZj21d1qb/YHlEbEiIjYC15FNifW300XcEREv5Kt3A1OaFeqmopn1IECtd85PlLSkZr0zIjpr1ncBHq9ZXwUckCjvFOBHzU7qxGVmdfrxJev1EZGeg7vVc0r/CHQAhzbb14nLzHpq7+ymq4GpNetT8m09SDoC+Bfg0Ih4uVmh7uMys15anNKmtbuye4DpkqZJ2g44kWxKrFdI2hf4L+CYiOhzXr/efMdlZnXaNY4rIjZLmgssIhsOMT8ilkk6D1gSEQuAfwd2AG7MZ1b+U0Qckyp3q0lcG9+VboZv/OyGZPysNy1sGDvy754fUJ3aZW3Xiw1jhyw4I3nsm7/8u2R8wtPpcVgln7bJBqqN47giYiGwsNe2s2t+H9HfMreaxGVmLYp+PVUshBOXmdUrd95y4jKzev0YDlEIJy4zq+fEZWaVEpT+qYsTl5n1IMJNRTOroO5y33JtNYlr5XHplwQe3efGITv3vKd3T8Yv/vmRybi6Gn0dLvPmr/yxYWz62sXJY7uSUdsqualoZlXkpqKZVY8Tl5lViz8Ia2ZV46/8mFkVuY/LzKrHicvMKiWA7oonLklTgauASWR/pc6IuFjSucDHyb6DBnBWPu9OKe1x6q+T8Xef+rZhqkm9PUjXrRmPxbL2Ghmd85uBMyLiPknjgXsl3ZbHvhERXx+66plZIaqeuCJiDbAm//2cpIfJPjlkZiNRAF3lHjrfr49lSNoN2BfY8h7JXEkPSJovaecGx8yRtETSkk00/XiHmRUuILpbWwrScuKStAPwfeD0iHgWuBTYHZhJdkd2YV/HRURnRHRERMdoxgy+xmY29Nr3lZ8h0dJTRUmjyZLW1RFxM0BErK2JXwb8cEhqaGbDqwJPFZvecSn7XtDlwMMRcVHN9sk1ux0PLG1/9cysECPgjusg4EPAg5Luz7edBcyWNJMsP68EPjEE9TOzIoyAp4q/BPqaEKq0Y7bMbBAioKvcowM9ct7M6lX9jsvMtkJOXGZWLVH6p4pOXGbWU0AUOLi0FU5cZlav5K/8OHGZWU8R/jyZmVWQO+fNrGrCd1xmVi0jYyJBM9uaVOAlaycuM+shgCj5Kz/9mkjQzLYC0d6JBCXNkvSIpOWSzuwjPkbS9Xl8cT5haZITl5nVie5oaWlG0ihgHnAUMINsVpkZvXY7BXgqIt4EfAO4oFm5TlxmVq99d1z7A8sjYkVEbASuA47ttc+xwJX575uAw/N5ABsa1j6u53hq/U/jpsdqNk0E1g9nHfqhrHUra73AdRuodtZt18EW8BxPLfpp3DSxxd3HSlpSs94ZEZ0167sAj9esrwIO6FXGK/tExGZJzwCvJnFNhjVxRcRratclLYmIjuGsQ6vKWrey1gtct4EqW90iYlbRdWjGTUUzG0qrgak161PybX3uI2lbYCfgyVShTlxmNpTuAaZLmiZpO+BEYEGvfRYAJ+W/3wv8b0R6BGzR47g6m+9SmLLWraz1AtdtoMpct0HJ+6zmAouAUcD8iFgm6TxgSUQsIPsYz/ckLQc2kCW3JDVJbGZmpeOmoplVjhOXmVVOIYmr2SsARZK0UtKDku7vNT6liLrMl7RO0tKabRMk3Sbp9/mfO5eobudKWp1fu/slHV1Q3aZKukPSQ5KWSfpMvr3Qa5eoVymuW5UMex9X/grAo8A7yQaj3QPMjoiHhrUiDUhaCXREROGDFSUdAvwVuCoi9s63fQ3YEBHn50l/54j4Yknqdi7w14j4+nDXp1fdJgOTI+I+SeOBe4HjgJMp8Nol6nUCJbhuVVLEHVcrrwAYEBF3kj1lqVX7esSVZP/iD7sGdSuFiFgTEfflv58DHiYbnV3otUvUy/qpiMTV1ysAZfqHF8BPJN0raU7RlenDpIhYk/9+AphUZGX6MFfSA3lTspBmbK18poF9gcWU6Nr1qheU7LqVnTvn6x0cEfuRvc1+Wt4kKqV8kF6ZxrNcCuwOzATWABcWWRlJOwDfB06PiGdrY0Veuz7qVarrVgVFJK5WXgEoTESszv9cB9xC1rQtk7V5X8mWPpN1BdfnFRGxNiK6Ivso32UUeO0kjSZLDldHxM355sKvXV/1KtN1q4oiElcrrwAUQtK4vNMUSeOAI4Gl6aOGXe3rEScBtxZYlx62JIXc8RR07fIpUS4HHo6Ii2pChV67RvUqy3WrkkJGzuePe/+Dv70C8NVhr0QfJL2R7C4LstehrimybpKuBQ4jm/ZkLXAO8APgBuANwGPACREx7J3kDep2GFlzJ4CVwCdq+pSGs24HA78AHgS2TBp1Fll/UmHXLlGv2ZTgulWJX/kxs8px57yZVY4Tl5lVjhOXmVWOE5eZVY4Tl5lVjhOXmVWOE5eZVc7/A+MVwiHJnPX7AAAAAElFTkSuQmCC\n", 57 | "text/plain": [ 58 | "
" 59 | ] 60 | }, 61 | "metadata": { 62 | "needs_background": "light" 63 | }, 64 | "output_type": "display_data" 65 | } 66 | ], 67 | "source": [ 68 | "# show a sample\n", 69 | "import matplotlib.pyplot as plt\n", 70 | "# the first index is for the dataset, the second is for the tuple, the third one is for channel\n", 71 | "plt.imshow(mnist_dset[0][0][0])\n", 72 | "plt.colorbar()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "id": "2ff87f41", 78 | "metadata": {}, 79 | "source": [ 80 | "The neural network to learn the score function is a U-Net architecture that accepts the image + the time and produce the score function with the same shape as the image." 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 3, 86 | "id": "3c2b4113", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "class ScoreNetwork0(torch.nn.Module):\n", 91 | " # takes an input image and time, returns the score function\n", 92 | " def __init__(self):\n", 93 | " super().__init__()\n", 94 | " nch = 2\n", 95 | " chs = [32, 64, 128, 256, 256]\n", 96 | " self._convs = torch.nn.ModuleList([\n", 97 | " torch.nn.Sequential(\n", 98 | " torch.nn.Conv2d(2, chs[0], kernel_size=3, padding=1), # (batch, ch, 28, 28)\n", 99 | " torch.nn.LogSigmoid(), # (batch, 8, 28, 28)\n", 100 | " ),\n", 101 | " torch.nn.Sequential(\n", 102 | " torch.nn.MaxPool2d(kernel_size=2, stride=2), # (batch, ch, 14, 14)\n", 103 | " torch.nn.Conv2d(chs[0], chs[1], kernel_size=3, padding=1), # (batch, ch, 14, 14)\n", 104 | " torch.nn.LogSigmoid(), # (batch, 16, 14, 14)\n", 105 | " ),\n", 106 | " torch.nn.Sequential(\n", 107 | " torch.nn.MaxPool2d(kernel_size=2, stride=2), # (batch, ch, 7, 7)\n", 108 | " torch.nn.Conv2d(chs[1], chs[2], kernel_size=3, padding=1), # (batch, ch, 7, 7)\n", 109 | " torch.nn.LogSigmoid(), # (batch, 32, 7, 7)\n", 110 | " ),\n", 111 | " torch.nn.Sequential(\n", 112 | " torch.nn.MaxPool2d(kernel_size=2, stride=2, padding=1), # (batch, ch, 4, 4)\n", 113 | " torch.nn.Conv2d(chs[2], chs[3], kernel_size=3, padding=1), # (batch, ch, 4, 4)\n", 114 | " torch.nn.LogSigmoid(), # (batch, 64, 4, 4)\n", 115 | " ),\n", 116 | " torch.nn.Sequential(\n", 117 | " torch.nn.MaxPool2d(kernel_size=2, stride=2), # (batch, ch, 2, 2)\n", 118 | " torch.nn.Conv2d(chs[3], chs[4], kernel_size=3, padding=1), # (batch, ch, 2, 2)\n", 119 | " torch.nn.LogSigmoid(), # (batch, 64, 2, 2)\n", 120 | " ),\n", 121 | " ])\n", 122 | " self._tconvs = torch.nn.ModuleList([\n", 123 | " torch.nn.Sequential(\n", 124 | " # input is the output of convs[4]\n", 125 | " torch.nn.ConvTranspose2d(chs[4], chs[3], kernel_size=3, stride=2, padding=1, output_padding=1), # (batch, 64, 4, 4)\n", 126 | " torch.nn.LogSigmoid(),\n", 127 | " ),\n", 128 | " torch.nn.Sequential(\n", 129 | " # input is the output from the above sequential concated with the output from convs[3]\n", 130 | " torch.nn.ConvTranspose2d(chs[3] * 2, chs[2], kernel_size=3, stride=2, padding=1, output_padding=0), # (batch, 32, 7, 7)\n", 131 | " torch.nn.LogSigmoid(),\n", 132 | " ),\n", 133 | " torch.nn.Sequential(\n", 134 | " # input is the output from the above sequential concated with the output from convs[2]\n", 135 | " torch.nn.ConvTranspose2d(chs[2] * 2, chs[1], kernel_size=3, stride=2, padding=1, output_padding=1), # (batch, chs[2], 14, 14)\n", 136 | " torch.nn.LogSigmoid(),\n", 137 | " ),\n", 138 | " torch.nn.Sequential(\n", 139 | " # input is the output from the above sequential concated with the output from convs[1]\n", 140 | " torch.nn.ConvTranspose2d(chs[1] * 2, chs[0], kernel_size=3, stride=2, padding=1, output_padding=1), # (batch, chs[1], 28, 28)\n", 141 | " torch.nn.LogSigmoid(),\n", 142 | " ),\n", 143 | " torch.nn.Sequential(\n", 144 | " # input is the output from the above sequential concated with the output from convs[0]\n", 145 | " torch.nn.Conv2d(chs[0] * 2, chs[0], kernel_size=3, padding=1), # (batch, chs[0], 28, 28)\n", 146 | " torch.nn.LogSigmoid(),\n", 147 | " torch.nn.Conv2d(chs[0], 1, kernel_size=3, padding=1), # (batch, 1, 28, 28)\n", 148 | " ),\n", 149 | " ])\n", 150 | "\n", 151 | " def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:\n", 152 | " # x: (..., ch0 * 28 * 28), t: (..., 1)\n", 153 | " x2 = torch.reshape(x, (*x.shape[:-1], 1, 28, 28)) # (..., ch0, 28, 28)\n", 154 | " tt = t[..., None, None].expand(*t.shape[:-1], 1, 28, 28) # (..., 1, 28, 28)\n", 155 | " x2t = torch.cat((x2, tt), dim=-3)\n", 156 | " signal = x2t\n", 157 | " signals = []\n", 158 | " for i, conv in enumerate(self._convs):\n", 159 | " signal = conv(signal)\n", 160 | " if i < len(self._convs) - 1:\n", 161 | " signals.append(signal)\n", 162 | "\n", 163 | " for i, tconv in enumerate(self._tconvs):\n", 164 | " if i == 0:\n", 165 | " signal = tconv(signal)\n", 166 | " else:\n", 167 | " signal = torch.cat((signal, signals[-i]), dim=-3)\n", 168 | " signal = tconv(signal)\n", 169 | " signal = torch.reshape(signal, (*signal.shape[:-3], -1)) # (..., 1 * 28 * 28)\n", 170 | " return signal\n", 171 | "\n", 172 | "score_network = ScoreNetwork0()" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "id": "4e55153d", 178 | "metadata": {}, 179 | "source": [ 180 | "Now let's implement the denoising score matching function below (same as before),\n", 181 | "$$\\begin{equation}\n", 182 | "\\mathcal{L}(\\theta) = \\int_0^1\\lambda (t) \\mathbb{E}_{\\mathbf{x}(0)}\\mathbb{E}_{\\mathbf{x}(t)|\\mathbf{x}(0)}\\left[\\left\\lVert\\mathbf{s}(\\mathbf{x}(t), t; \\theta) - \\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0))\\right\\rVert^2\\right]\\ dt\n", 183 | "\\end{equation}$$\n", 184 | "where $\\lambda(t) = 1 - \\mathrm{exp}\\left[-\\int_0^t \\beta(s) ds\\right]$ and $\\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0))$ can be calculated analytically,\n", 185 | "$$\\begin{align}\n", 186 | "\\nabla_{\\mathbf{x}(0)}\\mathrm{log}\\ p(\\mathbf{x}(t)|\\mathbf{x}(0)) &= -\\frac{\\mathbf{x}(t) - \\boldsymbol{\\mu}(t)}{\\sigma^2(t)} \\\\\n", 187 | "\\boldsymbol{\\mu}(t) &= \\mathbf{x}(0)\\mathrm{exp}\\left[-\\frac{1}{2}\\int_0^t \\beta(s) ds\\right] \\\\\n", 188 | "\\sigma^2(t) &= 1 - \\mathrm{exp}\\left[-\\int_0^t \\beta(s) ds\\right]\n", 189 | "\\end{align}$$\n", 190 | "with $\\beta(t) = 0.1 + (20 - 0.1) t$." 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 4, 196 | "id": "3b434448", 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:\n", 201 | " # x: (batch_size, nch) is the training data\n", 202 | " \n", 203 | " # sample the time\n", 204 | " t = torch.rand((x.shape[0], 1), dtype=x.dtype, device=x.device) * (1 - 1e-4) + 1e-4\n", 205 | "\n", 206 | " # calculate the terms for the posterior log distribution\n", 207 | " int_beta = (0.1 + 0.5 * (20 - 0.1) * t) * t # integral of beta\n", 208 | " mu_t = x * torch.exp(-0.5 * int_beta)\n", 209 | " var_t = -torch.expm1(-int_beta)\n", 210 | " x_t = torch.randn_like(x) * var_t ** 0.5 + mu_t\n", 211 | " grad_log_p = -(x_t - mu_t) / var_t # (batch_size, nch)\n", 212 | "\n", 213 | " # calculate the score function\n", 214 | " score = score_network(x_t, t) # score: (batch_size, nch)\n", 215 | "\n", 216 | " # calculate the loss function\n", 217 | " loss = (score - grad_log_p) ** 2\n", 218 | " lmbda_t = var_t\n", 219 | " weighted_loss = lmbda_t * loss\n", 220 | " return torch.mean(weighted_loss)\n" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "719d42ce", 226 | "metadata": {}, 227 | "source": [ 228 | "Everything is ready, now we can start the training. We use batch size 64 with about 400 epochs. Please be aware that the training could take about 2 hours with a GPU (NVIDIA T4)." 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": 5, 234 | "id": "22e5221a", 235 | "metadata": {}, 236 | "outputs": [ 237 | { 238 | "name": "stdout", 239 | "output_type": "stream", 240 | "text": [ 241 | "0 (17.730889558792114s): 0.22341656593084336\n", 242 | "20 (401.8917303085327s): 0.029081364569067954\n", 243 | "40 (790.7225224971771s): 0.023310251233975093\n", 244 | "60 (1189.1614212989807s): 0.02070757134358088\n", 245 | "80 (1576.9579727649689s): 0.019138541827599206\n", 246 | "100 (1968.4040081501007s): 0.018990544128914676\n", 247 | "120 (2350.477967977524s): 0.017998353243867556\n", 248 | "140 (2714.2453112602234s): 0.017528612997631234\n", 249 | "160 (3078.988071203232s): 0.016947320915261903\n", 250 | "180 (3441.6967012882233s): 0.016829439407090346\n", 251 | "200 (3803.7253336906433s): 0.016393583998580773\n", 252 | "220 (4166.092226266861s): 0.016545733177661896\n", 253 | "240 (4529.129833221436s): 0.01611548722734054\n", 254 | "260 (4892.061001300812s): 0.01601907907028993\n", 255 | "280 (5254.657285451889s): 0.015872681791583697\n", 256 | "300 (5616.805672168732s): 0.016094734780987104\n", 257 | "320 (5978.815507650375s): 0.015912713130315146\n", 258 | "340 (6340.980444908142s): 0.015442739987870057\n", 259 | "360 (6705.336601495743s): 0.015590764922400316\n", 260 | "380 (7069.732095956802s): 0.015302305275698503\n" 261 | ] 262 | } 263 | ], 264 | "source": [ 265 | "# start the training loop\n", 266 | "import time\n", 267 | "opt = torch.optim.Adam(score_network.parameters(), lr=3e-4)\n", 268 | "dloader = torch.utils.data.DataLoader(mnist_dset, batch_size=64, shuffle=True)\n", 269 | "device = torch.device('cuda:0') # change this if you don't have a gpu\n", 270 | "score_network = score_network.to(device)\n", 271 | "t0 = time.time()\n", 272 | "for i_epoch in range(400):\n", 273 | " total_loss = 0\n", 274 | " for data, _ in dloader: # we don't need the data class\n", 275 | " data = data.reshape(data.shape[0], -1).to(device)\n", 276 | " opt.zero_grad()\n", 277 | "\n", 278 | " # training step\n", 279 | " loss = calc_loss(score_network, data)\n", 280 | " loss.backward()\n", 281 | " opt.step()\n", 282 | "\n", 283 | " # running stats\n", 284 | " total_loss = total_loss + loss.detach().item() * data.shape[0]\n", 285 | "\n", 286 | " # print the training stats\n", 287 | " if i_epoch % 20 == 0:\n", 288 | " print(f\"{i_epoch} ({time.time() - t0}s): {total_loss / len(mnist_dset)}\")\n" 289 | ] 290 | }, 291 | { 292 | "cell_type": "markdown", 293 | "id": "bb8fe66c", 294 | "metadata": {}, 295 | "source": [ 296 | "Once the neural network is trained, we can generate the samples using reverse SDE\n", 297 | "\n", 298 | "$$\\begin{equation}\n", 299 | "\\mathrm{d}\\mathbf{x} = \\left[\\mathbf{f}(\\mathbf{x}, t) - g(t)^2\\mathbf{s}(\\mathbf{x}, t; \\theta)\\right]\\ \\mathrm{d}t + g(t)\\ \\mathrm{d}\\mathbf{w},\n", 300 | "\\end{equation}$$\n", 301 | "\n", 302 | "where $\\mathbf{f}(\\mathbf{x}, t) = -\\frac{1}{2}\\beta(t)\\mathbf{x}$, $g(t) = \\sqrt{\\beta(t)}$, and the integration time goes from 1 to 0.\n", 303 | "To solve the SDE, we can use the Euler-Maruyama method." 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": 6, 309 | "id": "a05b11a5", 310 | "metadata": {}, 311 | "outputs": [], 312 | "source": [ 313 | "def generate_samples(score_network: torch.nn.Module, nsamples: int) -> torch.Tensor:\n", 314 | " device = next(score_network.parameters()).device\n", 315 | " x_t = torch.randn((nsamples, 28 * 28), device=device) # (nsamples, nch)\n", 316 | " time_pts = torch.linspace(1, 0, 1000, device=device) # (ntime_pts,)\n", 317 | " beta = lambda t: 0.1 + (20 - 0.1) * t\n", 318 | " for i in range(len(time_pts) - 1):\n", 319 | " t = time_pts[i]\n", 320 | " dt = time_pts[i + 1] - t\n", 321 | "\n", 322 | " # calculate the drift and diffusion terms\n", 323 | " fxt = -0.5 * beta(t) * x_t\n", 324 | " gt = beta(t) ** 0.5\n", 325 | " score = score_network(x_t, t.expand(x_t.shape[0], 1)).detach()\n", 326 | " drift = fxt - gt * gt * score\n", 327 | " diffusion = gt\n", 328 | "\n", 329 | " # euler-maruyama step\n", 330 | " x_t = x_t + drift * dt + diffusion * torch.randn_like(x_t) * torch.abs(dt) ** 0.5\n", 331 | " return x_t" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "execution_count": 7, 337 | "id": "90d4e875", 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [ 341 | "samples = generate_samples(score_network, 20).detach().reshape(-1, 28, 28)" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 11, 347 | "id": "b5734fc0", 348 | "metadata": {}, 349 | "outputs": [ 350 | { 351 | "data": { 352 | "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAH6CAYAAADWXf/OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAACmaklEQVR4nO3dd7htVXn3/bE2psA5hyYdFLsoAlJsqNGYPDGxBDWWqFEkYi+xY0FRVDCKvQAPqIldEWMhEhIVRGMBDfaCFRGkHNopgBr3ev/Im+uK6/6OPL/pOnvujX4/f97XXGvOOcaYY841r71/YzKdTpskSZIkSZI0poXlPgBJkiRJkiT97vGllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdNcbsvFkMplOJpPZWtlucXGx1BYWlu/913Q6LTU6bjLvudC+5zkeQseT7pdqveNJj7Gzn7XT6XT76As2ARqrne3m2k/al5u6z3v9lujtd57vnGcMzjOuep+f51zGHqut/dd43ZRz5FK01VL0XfJ9Q66TOfu91NJxTdvRvaO1+e49nf4bdbwuLCyUsbqp2721vJ2W4Pr/jY9lKSzn8czTtvTZX/3qVytibl1p43W5DGmHdBym97F57jvzGjAHj/7cCrWy3Tz3qiGfn8dSPKNuavPcyzf1s09v3+nnxx6rrc333DrvfXue9k/fWyzFb7R0fKVjYZ5zWYpn67Rfe+N16Eup9gd/8Ae/VqOG+8UvflFqf/iHf4jfR6hB57kh/ud//udv/NmNGzeWGp1LDx33L3/5y+h4yPWuV7tstk9a4z6g/Q55wPq93/u95BBxP7/85S/Piz68iUwmk9JWm222WdkunVB72/385z8vtd///d8vtV/96lelRu1J26WTTjrOaQy1lk86tJ8tttii1DrjoNRo/NJx02dby6950rk+Rx2rrf3XOczOK/M8jPbOn/qexhzth76zN5bS45lF10RvbkzvPem9g9qBxiady+abbx4dS2v5wwZ9vnNdjDpeFxYW2urVq3+tlo4hOvfedU3zNZ0/bUd9NM8DHe1jKX4sE2qf9LpL729DrjFqW9qO2uzKK69clrl11apVv1ajNk37vTde5+mTeX6ozPNjqvfinFxzzTWlRs/CNGd2fpREtfSZaAjqKzrGDRs2LMt4/Z/Sezbdg3r3XXpupc/TeEv7g7Yj1OfpGOrV0+uEruXZuaI1Hvv0rELHQm3dGvcr/XbozTezrr766mUZq7PP/ul8ROeaPie2xnMzoe+kz1I/pfe1dL+t8Xmn70xoLND3XX311aVGx01jsDe3ps9U6XzVm1v99z1JkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjS6QZlSrdX/N0xzbNL/L26N/0+R/r+S9n3ttdeWGv3vL30f/V8nZeUM+R/3NGuEjifdD+Ve0f9k0/+e9v7fmc47zR+g/Vx55ZXRZzeVyWRS+n2eAPJeXgRtm/5vcpoVQuOc+iLNOqPx0hr3G6Hrad26daVGY4jGJY3zNIOrty2dC32esgLS/+HflKbTaRmLNDbTduldqzQ/UlvRvmk/aWYN9fs82Xqt8TxK+6E+Tue3NOOglx9F0sxEam+aH8Yer9PptLRLmg+R5va0lmfvpFmJ6byeZkbQuOrNoTQuac6kY1yzZk2pUWYESbN86Pha4/kmHYNjZW4lZs8jzcqisdDr4zSHL80nS+d/esakeZ6eD3rjiObHNLttKfK6ZtE5t8bX5DzzyNgoo2eeMdS7P89mAraW53lt6qDmNIO190xD98k065XmMvpsmtubHl8Pjd80q26loPOl8UpzT2+8ps+o6Rim46E5Jc0IS8dWTzoO6fzos+lvgjRbqyc9xyHP9f6llCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjS6QQlqk8mkBFtRUFYa0D0k/CoNj+4Fds6iUEMKFEzDo3uhnuvXr4++k0Lz0oC8NIQvDUdsjdub+pUCMTds2IDfObbZ8TVPGG8v8HieEMo0OJSukzTIjq6RXlBrGqKdhgTSNZYG8NP47QVLpmHFaQjqclhYWCjtQGMhXTCiF9ZM7Z+Gn1N/UvvReE0DPGm89fqdjpHOhY6Rwv7TYHHqA+qr3r2IAj6pv6iWLjaxlOg5IA19pbExJJSf7jcknevTZxW6H1L/9NqB7onpwgEUiE7PEJt6MY3W8tD3dLvlMJlMyv2JrsE0TLf3zJQG6afBu+kzcxosTmOw9yxA8zVJ5yMam2k7kN54XbVq1W/8eZqv0wUFNhVaRIKkc1TvN0QaVp6G96dh8unvOJpHe32bBjOni19QLf2tSeOlN1+k4djpb4flMjtu0oUw0iD81vh5jZ4F6DvTMO90/p7nmbB3PCR9l0H7oes+XThjUCh52N69xbXwO+MtJUmSJEmSpE3El1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0g4LOp9NpCQhLQ9jSEOUeCt9Kg7somCwNXkwD0XrS4Oo01HaekPQ0eLq3b/pOCsRMQ2aX0nQ6LedA7UTnOSSAMw2MTEP90hC8XiDgptxHazxW01C/NHg3DdDvXXdpHw4JVhzbdDrFtlkuaXhuGh6aXicUiLjNNtvgMaZhxelxp4tf0JxJ82AvgDUNsEzvo2ObTqflONJA6DTItTU+f+rzNFg1vSem92y6F1M4e2t5qC2N1TTQPw0/HxJumt7fhwTYj42eW9P7Az0zpIvf9ND1T+MwnVtpu3Ss9669eY4xfQZPn63T0N7W8oVr0vvb2CaTSfR8Rm1MfdZbyIWua3rmonkm/f2T3hPS8dtbRCIN0aZaeu+gdkiPu/cckI51WiwrfYZYapPJpMyH6e/odNGS1njOTZ/7qZ/mWRCNnkHoXHrzSRqUTvtOf7ulC5yk7dVa/vs/XbSjx7+UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkY3KOh8MpmUkMs0cK8X9kbSEEoKEkuDVNMwRvpsLzyQUDhYGmBL+9mwYUOpUTvQPuhYemHz1F9paPaQULOlMplMfuPjGBJ+nYZjUkgjjTcK6qRg2TS0Ox0bPWmfp6HGFBiZhlP32praLA16XbVqValReN9So2D+FLXp5ptvjtum+0j7jr5vzZo1pUbzCYWa034p6LM17ncam3St0NyQhgjTPoaEktN+aGymx7McZo9tyD1xVm+s0phJ+ze9L+22226ldv/737/Unva0p5UazRNbbbVVqfUcf/zxpfbv//7vpfa5z32u1NKQ1/R6790raR5N52a6lpfDdDotc0W6GAsF2Pbun2nIdrpYQbpgBKH9DplP0uf1W93qVqV2gxvcoNS+8Y1vlNqll15aammYem9cp6HoQwKpx7S4uFjusxTyTGOQjr93DabB8/M839P30W9DejbeddddS+3mN7857ofG4MUXX1xqH/3oR0stPb908SO6PmmuaY3PO/0duBJ+Y7XGC/TQtUn9np5ra3lQ+jy/x9I2Xb16danRuRx22GH4+cMPP7zUzj777FK7973vXWrpYhw0NtNFe3qLkqWL1Q3pV+JfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjd4eZTZlSwosT5Ntu8l5Q/ZNkHJ77TKB6XqU+o/fZZqreWrgaTJ9um50KpitNpVb8WBNC1/nhV/ltrsCgR0rLQyBm03ZLWLeVaDSlfISVcISVcE6h0joVWnaAWttL1pvqAVZ2iliNby1QAJrSi3HGiFKGqD3soYQ/YzK52jqO/oGL/2ta+V2vWvf/1oHzRmfvazn5Vaa61tt912pXbiiSeW2hve8IZSu+CCC0otnQvS1U56K5umK0xt6vvgUqJjTVcSTVcj60nvQdtvv32pnXHGGaVGq/qkK/0MWWXm6U9/elT78pe/XGpvectbSu2DH/xgdDx03L0VotL7EY11uk6Ww2QyKe2QrhBE50Bj4X/b9yxqv/QemK5EN88qq73Pb7311qX2vve9r9RudKMblRq1Nz2P0jm/9rWvLbWjjz661FrjuTVdvYz2TatdL6WFhYXynENtl656PuQaTFfnojFIv5NoTqHP/t3f/V2pPec5zyk1WtW3tfzZ/DGPeUypvfe97y21dMX0eVZRb42fma+++upSo3G5Up4D6Lk1XWmPxkfveZzaMP1tM09/pqvQ0+/tPfbYo9Ra4zl3n332KTVqx/TdA7VX+o5hyO/DtF+H8C+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNENCjqfTqclVIuCsuYJvWuNg90ozGuHHXYotXe/+92lts0225TaZZddVmo3velNS+2f/umfSm3nnXcutcsvv7zUWuOQ3j333LPU7nKXu5Ta4x73uFI75ZRTSo0CyCgwjvqqFyxH26aBgvMG124qs+NrnoD53lhNQ9HToL50O0LHOCQkkUIxqS0oqI+Oka5Z+j46xjR0sLctnUsawLgcKIw3DbClNu2NmXS+pj6mdj7qqKNKjRaMoHZOw/9vcIMblFprfI5PfOITS42CiV/ykpeU2qWXXlpqaXgsXXu9QGT6PLU39cu8YbabQhocTfd8CnLtBdhS/1I70TVB/fGABzyg1Cj8fJ4Q2d49cp4FBm5/+9uX2t57711qFPz/tre9LTrG3jlTH65fv77UaKwPWRxkKU2n09Ku6T0svS5by6+B9D6d7jtdMGVICDPNzV/96ldLjQJ+0wUAqG3omnja055Waq94xStKrbef9Fmit5DKmGis0jnR2EjDz1vLnz3TZ4Peoh6zXvrSl5baM57xjFL7/ve/X2p/9Vd/hd+57bbbltq73vWuUqNgcWofWsiH7lF0L6N2mPcZM10wZDnQs0C6uAO1c++80sVlenPzrHSs01xG459+6+y22264b/pOGofr1q0rtfT3E40Z+iwdd2+80nfSNUXfOWSBJv9SSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRrdoKDzhYWFEmyYhvFS0FkvhI+CNClQ69hjjy212972tqVGwWs3vvGNo+0e/ehHR9v1gkIpSJxCQSlcjALMttxyy1LbuHFjqaUhwtSurXFYGdUomG4lhPFOp9PSpmk44IYNG0qN+qK1PAicQv3SPqf2PPjgg0uN+vyOd7xjqd31rncttdY49PeSSy4pta985SuldvTRR5fal770pVJLw2/T0NnW8jDONIB1ucz2XxpGSnNrr53TAESaU7baaqtSe+QjH1lqdJ2dd955pfbyl7+81Og6udvd7lZqrfHYvvWtb11qj3rUo0ptn332KbV73vOepTbPvEXzZc91aREJWvAkRXNZ77vouqZt6VqnUHOao+jaoWuM2p0+O2SBgfS6TUPSDznkkFKjhVouuuii6Pta4+eXNFiV7kfLZbYN6V5AbT/kvGhs0+epRs+ENNbT0GvaB51f7576mMc8ptTSUHO6Vv74j/+41M4999xSO/PMM0vt5je/eand6la3KrXWWvvud79batSO6eIoY5tMJtG9IH3m6S1kRPfo9FmItqP7KS3e9OQnP7nUaAztu+++pda799G4pP7da6+9So3aJ12gg7ajc+n97kgXAqJxuZKeW2fRec37fJMGk9N26b23Nxcmn6Xj3m+//fDz8yxMlt4T0mcYGq+937q0b/qtnL576Fm5I1uSJEmSJEm/tXwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjco6Hw6nZZwMQpUTAPgegGnaXj0u9/97lK7853vHH12nuC1NGCtNW6LNFz17LPPLrU0RI7CxuizvWDWeYLget85poWFhRLYRuefBvD3AvioTbbeeutS+8d//MdSu8Md7lBq//AP/1BqT3jCE0otDZsdMvbputtll11KbYcddii1PfbYo9QoOPryyy8vNTpuCpbsBfCloY903nR9Usj3GGbbgfqOQi+HjNd0bqbgXepPGjMUVk7BuZ/97GdLjc75gx/8YKn1tv3Zz35WanQudO097nGPK7UTTzyx1NIAbmqH1nhupmuAFqFYCXMrBZ2nCwuk977W+PzpOynU/JWvfGWppSHiQxYOSD7bWh62nD5P0fHsueeepXbSSSeV2p/8yZ+UWm9upQDWNGy11xbLYba90sVqhgTzJ/vt1ajfqe3peGieobFF1+i97nWvUmuttec973nRd9L96KCDDiq1r371q6VGY+6YY44ptTe/+c2lRs8brbX2rW99q9RoHNIcTgt59ObwpTKdTsvx0niZJwS5953p3EzB8+95z3tKjRZlOvnkk0vt1a9+damlCyy01tpuu+1WajRn0rxH7UDPC1dccUW0j3Se7+2b5qD03rocJpNJaYd5FgzoPbem91+6LuhaT38zp+HgNNZ745WuswsvvDDaLg01p/sJHTc9Yw0JJU+fTYY8C/iXUpIkSZIkSRqdL6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLoBgWdEwoWo5CteYMwKUDu/e9/f6lRECQF11EwGYXrff/73y+1M844o9T+5V/+pdRaa22//fbD+qw0PDcNbUtDZqm9WuOwZ9o2DVwc23Q6LeOQ2mSesMHWWluzZk2pHXfccaX2x3/8x3iMsygcj9qdPktBfVTrBdlRqCcF2REKm9x1111L7aKLLoq+rxcSSGhuSUMLlyvUnMz2C7V9GnTd67f0GqDxRSG05B3veEepffGLXyy1dBGI3n0i7U+a66kdX/ayl5UancuGDRvweGb15laaS9Iw3nSxiaU0mUzK9UnXG50Tjb8hi20ceuihpUah5nRfSu+J5Pzzzy81CtX/0Y9+hJ8/77zzSo2uiSOOOKLUbn3rW5daet894IADSu2Zz3xmqR111FGl1lp+zycrKYx39rqhewYdL23XO/805DhdwCLt43TRk+OPP77U7nvf+5bakGM87LDDSu0zn/lMqdFcTzX6LM3z++67b6m11topp5xSanRPWL16daml8/pSm52T0sDjIYsT0ThKx/+VV15Zattuu22pnX766aX22Mc+ttTouTN95m2NQ6LpOymg/fGPf3ypvetd7yq1VatWxcczqxf8TeeYLkRDCwEth8XFxXJ90ZyXLkbUe26lZ6H091O6eAiNmXme1XrzCYWL0+JR9N4iXayL0Dya3gd70m17Yf/Ev5SSJEmSJEnS6HwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjcoNXUymZTwxTQcjILAKPCrNQ54TIPO3v3ud5cahZrR91EgGp0LhZff5ja3KbXW8vDAtWvXltrFF19cammIHPVLL+CaULBfGry+UsJ403C9WTT+KAS/tdYe+chHlhqFmtN3UlDu3//935cahS0/61nPKjUK/6SwvF6I+FVXXVVqN7nJTUotHUff/va3o8/SNUIhkL0+oG0pCHKe0NmlRmG8FExIx0vt1wsgpPFAc8XWW29dattss02pUZueeeaZpUZtSudC46MXGE7n+MY3vrHUjjzyyGjfP/jBD0rt8ssvLzW6ltN7VmvcrzS3pgH0y2G27dNQ5jSov7XW9t9//1J77nOfW2rUTmmoJ815J510Uql9/OMfL7VPfepT0T56derLr3/966X2ile8otQooDoNm3/+859fahQW3Bo/T1G/0jNWGog+htnxSuOQaunzW2vc/vR5Gpvpog+E9vGgBz2o1O5///uX2pBnJAo1p2uF5mt61l+/fn2pUTgwjSNaWKU1vs/Qvud9Pl5Ks+NjngD+3rPMPGOQnlv33HPPUrv00ktLje676TzR6x8aM7QY1X3uc59So4WsaFzSMzO1LfVL7/mFnmfTQGjabrkWmJo9ZxpH1Hc03/ae8em5lbZNFwJLa+kzOAX40zG3li8Ol26XLuCShrb3rsf0dxEd45C51b+UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkY3KJF6cXGxhF1RsBuFWg0Jj6bwrTQINg1Uo/A5CiZ76EMfWmoUGNkL96b2oVBR2k963GlQNAWV9cI0qb0pXC8NuB0bjdU0rHBIGC+FPn73u98ttRvf+Mal9uUvf7nUKLSRvu/QQw8tNQoYpMDqK6+8stRaa+2BD3xgqb3pTW8qNRrTNAZpHNB2NK4oqLwXypougpAG0S6X2fOjsZle/xTQ2Bq3K82Zu+66a6lRYCztmxZJSOcjukaHBJ3f9KY3LTW6LijMlEK06Z5A4zq9P/Xqadhrr1/HNnt9pQGjdA322unAAw8stZ122qnU0vsNHePTn/70UjvttNOiz5IhIaF03BQO/KQnPanUTj/99FJ79atfXWo0Z1IfPOIRjyi11lp73/veh/VZ1IcbN26MPjuG2fFKc0IaRt97PqBnwDRInNqPno9p/NNCDtSfQxZjoL5761vfWmrUPjQvr1u3rtR68/osunYuueQS3Jb6Jg2fTp9hltrs9ZneG9LfYq3xfY36Ml0w4qKLLoq2o/ak66b325DQtZwG6//whz8stfQ6ofNLF6JpLf89RdL70Rhmzy9d/GrIglg0btL5msYHLXSQLohGv6n+6q/+qtR655f+5qDzo+sxfaaiaypd8KN3PDQ3z7tghH8pJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI1uUND5ZDIpoVoU2pUGBg4JsKUAXPpOCumi8Lhtttmm1L7whS+U2rbbbltqQ4K8vvKVr5TaX//1X5faZZddVmrUDhS8lgad0Xa9IM5eAPqsIQG/Y5pMJuU4KEScxmAaMN9aax/84AdL7dOf/nSp3e9+9yu1T37yk6VG4Xh0PGkbX3XVVaXWC0t+9KMfXWo0Zqj2r//6r6WWhh/T+aXt0Bpf32kY+0oxnU5LaGAaspuG6bbG7U8Bjze84Q2jz9K+f/SjH5UaBedut912pXbssceW2v77719qrbW22267YX0WhVq+/OUvL7VTTz211NLrLF0YorW8v6ivqa8obHKpzbYLtRNdb2mQa2utrV69utTSRSjoeJ7//OeX2imnnFJqND/SuaRh2a3xedMx0r4poPTd7353qd3lLncptXQBlTvc4Q6l1lprf/M3f1Nqxx13XKmtWbOm1Gj8r4RFUFqbLyh4yDPTPPccCmv+yEc+Umq3uMUtou8bEkq7du3aUksXtUkXFKF977nnnqVG7fD973+/1HrfSagthsxNS2UymZRnn3SeoPHXG6s0B6R9lPY5HSPNmXTc9PzXe269zW1uU2p0TdA4Ov/880stDYxPnzF7gdf0nbQQDY3LdOGYpbawsFAWtkkXMxqy8BZ9J4X103a0H1qMh54TqY9o0ardd989+mxrfI60uAoF89O1R9cFjbn0PcGQhTzovUx6PD3+pZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0ukFB563VkK40EJtCyXqfTcM+KaSLvpNCLZ/3vOeVGgWrpufX86pXvarULrjgglKjADMKEUvDCNPw6F4QJ/UBhevRdish6Hw6nZa+o/6lcUWBcEPC3y699NJSO/7440stDTymoD4KbUyvm4c97GG4nwMOOKDU0gBjCrKmgMZ0EQT6bK8P6BpNP78Sxmpr/zUWZvs0DYKkcdQLvaR5mNrl9NNP7x7r/+uzr3jFK0qNFhm4053uFH0fHXNrfN69MNRZFMaehrWmway9xRHomqT90HlTO45tOp2W65jOldqJrrfeWD3ooIOi7yQ0z9C8nC7U0gutn9UbfzTH0blQO6bz45e+9KVSe/CDH1xqFDbcmwcf85jHlNoJJ5xQanQtpm221KbTaZlXqP2oP4aER9MckIaL07z3kpe8pNRufetblxr1Z7oATe/5b8stt8T6rLTNqG1oOwrWp+Mect9OjyddRGGpzR5vungT6QUMp+M6HdPpvum4ac5M7+Ot8Ry34447ltoVV1xRauk9Kl3wh66n3rmk5zhvcPRSmk6nZa5Jjy193motX/SLUH+mn6VrYvvtty81Gm+9uZWuZ1q4hNqC2iz9DZsuUNC7b6e/s+i4hyws4l9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaMblJY2mUxKqCIFWFFQ1pDAcAomozDHNKzwmmuuKTUKIUvDWqn2la98pdRaa23PPfcstb322qvUfvKTn5TaN7/5zVKjMFNqmzSArBcYmfYh7YeOJw1H3FQoOHrdunVluzT0tddONN4ohJzOn/qDxuWqVaui70vDKx/wgAeUWmv59UR9vtVWW5VaujgB9QHphXvSeVNf0/mtlDDe1mrbpNcbhcz3wnhp3NC2p556aqml4bm3ve1to8/S+aVjobU8NJ++861vfWup3f3udy+173znO6VG42hIWCvV6bjpvkWLX1D/L6XJZFLaIJ2P0vDb1voB97PScNnPfOYzpUZjIw3lHHI/TYOZr7766lJLQ43PPPPMUkv7pbePnXfeudTo/kb3iXQhj6VG45X6mO6zQwKX6XpN24r2s//++5ca9ROdyytf+cpS+8AHPlBqa9asKbXWeFEYGpvpPZWu0Xve856ldvDBB0f7fdOb3lRqreWh7+k8NPZzKy0iQdJxmXzXf6N7S7rQB81v6Wdp/D7wgQ8stac97Wml1lpre++9N9ZnbbvttqX21Kc+tdRe+9rXRt9HbUvjqncfo0VLaPymCxgth/SdAN0L5h2bdG3SNUz7prk69aQnPanUhgSxU+A+PW+nz080V9PxUDuktda4bakd573vr4ynBkmSJEmSJP1O8aWUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLpBq+9Np9OSeE8J+LSCQLraXWv56kS0HSW/0/EceeSRpfbXf/3XpUap/4RWnOrVKUGfkvo/+9nPltpjH/vYUrv44ov/3wfYeNUXOpZendqbVlqhVSXGRmOVVsLpfTaptcbnT6thpatC0DifZ5Ux6sc73OEOpdZavoISbff9738/2ne6Shntt7cSDs0jtO/0eJbD4uJiGTe0og2tRJSudtT7zp122qnUbne725VaOn9QP9FqUF/72tdK7V73ulep0apprfE1uXbt2lJ7/vOfX2p3utOdSu0f//EfS+3P/uzPSm39+vWlRuOI2qu1fLVU6sOVsurO7LHR9UrjktqE5rLWWttuu+1KjdqOxsHpp59eapdddll0jFRL5+XeKnbpiq60UlO6StEPfvCDUjvnnHNK7Y53vGO8j/S40zZbKehZgJ7B0nNtLb9n0edpBSWag2ms00p0VKPPnn/++aXWQ+eXrr5NKxseffTRpUbXN80ZO+ywAx4jrWxFzwdpX49tMpmUdh7yPErfR9LfWHS/oT6iPn/EIx5Rarvsskup0YriJ5xwQqn1zvlVr3pVqX3qU58qtdNOO63UXv7yl5fau9/97lK79NJLS43GFbVrb3Vc6hsa60NWqVsOs+ecHu+Qld/SlcXT9kvnmR133LHUbnjDG+Ixzur9tj755JOjbdP5lsYRtRe9y6DP9u7b6W8P+s4hK/L5l1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6AYFnbdWQ6zmCW3tBcGm4ZppQDLtmwIRn/zkJ5cahfUdcMABpbbVVluVWmt5QBsFht397ncvtY9//OOldsghh5TaN77xjVKjfukFIVK9F9yWfueYptNpGR/UxjSG6Dx74X3Ul2mgNn2WAlgpeI7amK5F+iyNjdZ4vBEK26NgYRr7dH4UBjskRDgNMkzbpxeovtRmx808Aae9BRpo3FDY7UUXXVRq22+/fXQ8Z5xxRqm94hWvwOOZ9bGPfazUevNOGpRJC1i8733vKzUKgD7ppJNKjcLP02DJ1lq75pprSi2dC6j/6PuW2uzxptdRGujZWh4ES7W3ve1tpZZeT+l8QufXe6aZZ5EF+izth9qLAqapvXvzyg9/+MNSo/ahe2s6Vy216XRajiUNYU+fGVrLnxuoXR73uMeVWhrQ/dKXvrTU0nsYBay3li9WRPvZd999S42C13fbbbdSo7H5ghe8oNTWrVtXaq3li35Q2/au3TEtLi6WcPF0MYb0t0Zr/CxFoeZ0D6L2pDH48Ic/vNToGqE55otf/GKp3f/+9y+11vh6pLY46qijSu2II44otf3337/UPvnJT5YaXYvU3kN+D9EYpD5I74NLjX5nUdunv9978zKNd5qj0j6hGj1H0YI4N73pTUstHYOt8SJr6WJWvd9As6i900U3eveOdKEw6oP03UFr/qWUJEmSJEmSloEvpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0ugGBZ1Pp9MSgpUGWFFoF4Vs9eoUBEZhXmlgLG1HobYf+chHSo1CzQ499NBSa621LbfcstQorPzZz352qR100EGldotb3KLUjj322FK7613visczq9cHdI5pYB+NiTRYdFOaDZqjkDg6Twp8pFC93ucpRJPOn2rp9UT9RkGH9H23utWt8DvTMMLvfOc7pfav//qv0b7pOk4XJ+iFzdMxUl9TkGHvO8c2nU7LeKBzoPDsIYHC6XjdY489Su3MM88stbPPPrvUaCzQPEiLTaTBqj00hjds2FBqn/jEJ0rtwAMPLDUKtbz66qtLjfqqFzyazkPLFbj//zKZTMr8Q9cwnT+N1V470bikz9N8/dGPfrTU5lkYhdAc3Ju/ae4Z8vlZ1A7Pfe5zS23vvfcuNTq/Xlj+a1/72lKjdqTaSplbW6vtRfNEOt5655UG6VO4+OGHH47fOestb3lLdDw0rrfddttSowUfWuPFfHqh2bPonNNnQhpHGzdujPbb2zfVKIyX7q00tyylhYWFMg7T62jIAj10Xul1/aAHPajUaJEo+uxf/MVflNq///u/lxpdi3Qfb42fGWhufetb31pqL3zhC0vtzne+c6nR88KQcGtCzwFpH6aLZCw1Cuanvuv9zpzVaz+ar+kapn3TWE8X63jwgx8cfR/t94ILLii13uep3+f5vU1tkxryWzd9LzPkuvAvpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRDQo6X1hYaFtsscWv1SiskMKvemGmhAK16PNpKO5VV11VamvWrCm1IUGBs0444QSsU3gind8TnvCEUqNA3Uc96lGlts8++5TaVlttVWrr168vtd75UahlGuy3UgJ6ZwPXKMyVxm/aZz3UbxQeR0GHFBJHn03bfeeddy61HXbYodRa4z6n/bzpTW+K9r1q1apSS8NN0/DW1vKQaQoypM8u1/idbWuaj9Kg6F5b0fhKP0+LOVDofRrqT/0xZEEMQu1D1/13v/vd6LPXv/71S41CLWm89hZ3SMc2jc3lWDBiFoWb0lil6zq9Blvj65BCouk7qUb3/HRep/6hvug951D70DHSfm5729uW2tOf/vRS+7M/+zPcd7IPWmilNV6Uhe6PFKyaBmOPYbaf00Bc6s/eeJ19Nm6ttUc84hGl9rjHPS76TpqPvv71r5cazSePfOQjS+1FL3pRqW299dal1loeskvHmC56kC74QCHT6fH1joc+vxLm1tay64bandqzFxJP29L8SM+yRxxxRLQf+l1y6aWXlhpdi6S3CASNf/pOWliF5q1TTjml1GjOSxfJ6I2r9NpZyUHnm222Wbkn0/mmc8K8z3pDnoVn0fx9hzvcodTo9xj10Zvf/GbcDz3X0PU4z/Nx+myRBr73Pp/2Ic0jPSvnqUGSJEmSJEm/M3wpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjco6HxxcbEE2lH4HIUI0nYUHtcah+alIeQUAEeBy2kgGoWp0fn1ghcp4Iu+kwJc6bN0ztQ2j3nMY0rtda97XakNCWZNw14pCI5CRJfSdDotx0bHReF21D+9YME09C4Niad2p7ZLQ/Duc5/7lFqvz6l/165dW2rnnXdeqVGwJAVMEwr5ozYcEspP19OGDRtKbUhg6lJaWFgoCzdQCCf1expe3vs81ejzX/nKV6LPpvtNr51eH9Hnqc1ofN3xjneM9kPXKN1PaL+9404Xz6DPp59dSpPJpPQn9UXvs7N6Y5XmPXpmoP4988wzS+3Od75zqdHiH+mzCh33wx72sFJrrbUHP/jBpUYLtZx99tml9nd/93ellj6X0HYnnnhiqZ100kml1hrPrem8RP0y9nPAf0vmeep36qM///M/x88/61nPKrWb3OQmpUZjifqJFuh52cteVmpPfOITS41CptPx0VoeVvulL32p1H74wx9G+77wwgtL7Za3vGWpXXbZZaVG/dIaj6/0ubUXpD0mWkQivd9QgHLvfkF1ej76+7//+1K78Y1vXGp3u9vdSu3KK68stfSeRtv1njXo8/TseeCBB5YaLRxA45faNg157o2r9Nkz/e2wHOh3VroQBun9zqK2ous6DfimGo2ZH//4x6VGi9/QWPjCF75Qaq1x+9B8RPfPNLSd2pFq6SIDreULu6QLV/X4l1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6AYl+00mkxKMRQFWFNBF4Ve98DP6fBqoRZ+lUDgK+KJ9UCgZBT5SCFhvP3e6051K7e1vf3up7bjjjqVGQWd03O9///ujY+wF1NI5UtumwXJjW1hYiMN3E73vorGVBmuuWbOm1Ciok4ISqc+pRmON+rE1DhP82Mc+VmpnnXVWqdH5pQGWaahl7xqj9qaQ2N55rxSzY4nGHI03CuHsnWv6+fTaSefHhz70oaV28sknl1o6L7fGIdU0buic73e/+0Xb0T7SkNleMGsa2knXz0qYW1urx5EGmZLeWKPrPe3fPffcs9Q+97nPldq6detKjeaOBz3oQaX2J3/yJ6X28pe/vNRa43BUOu673OUupUbPOXRPoLH6kY98pNQoLJvGb2v5IhQ0pnvz9XKYHTfUfunxPvCBD8T6HnvsUWrp4jkUGP6IRzyi1P7oj/6o1G51q1uV2g1veMNS22qrrUrtnHPOKbXWWjv99NNL7ZWvfCVuO4vGDI2vtG1ovPX6Kn3GHXKfGRMteEL3Uxq/Q4KMqT9o0R+a42iu33vvvUuN5lvq8/Q3W+/eR0Hwt7vd7UqNFnqiOZOC9Qm1IT0P9Y6bzjG9j66ExaRa+695f7b/qF3ShTmGPDOlv4vSxdhoHNKCEXSM1Pbf+c53Sq21PPSbrud0oZh0QYF0Ea3WuH2oRnMwLUxB115r/qWUJEmSJEmSloEvpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0ugGBZ1Pp9MSqkXBbvMGBqah0L2grFkU+kXhgRTGloaN9cIvTzzxxOQQMbiO9n3NNdeU2hvf+MZS+9nPfhbtl9qhtX5A4iw6xpUQxru4uFgC99KQxTTkvbU8tDgNhKMafR/1DwUMpqGDrbW2YcOGUnvVq16F285K+5zGOQXw0THSXNP7PAXrpWGLyyVpwzTUsBdWSONrnnlv48aNpbbLLruU2nHHHVdqe+21V6kdccQRpdY7FwpmpbHwsIc9LDpG+uyxxx5bahRGSp/tBVqmIZt03vTZXkj1mNLxQqGjPRSE/9jHPjbaD43pW97ylqVG1xPNrRdeeGGppWHD/1t9Fl2ftB/q82c/+9ml9q53vavUhtyf6XljSB+uBAsLC+XekS7uQGMh7cue73//+6X2iU98otTWrl1bau9973tLje6LFEpO10QvMDwNsydpYDy1I12PNDcOuW+nQem9+8zYZo+Xrjc6VhrTQ+5B9Nx72GGHldoxxxxTaocffnip7bfffqV26qmnltopp5xSaum9r7XWnv70p0e1j3/846VG4efUZnQ8FLBOY7rXB/SdVEvn/+WwuLhYnofS59EhfUznS9fwPL9HhyzGNovGcG8e7P3+SrajNqM5M/39RGOzd3+jtqC5echzEVn+tweSJEmSJEn6neNLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLrBiZWzQWTzBOcOCdykEL40PI3COimkjo7nJS95Sak9+MEPLrXVq1eXWmsclJmGNFKA2Ze//OVSe/GLXxztY0jwLAWgUftQLQ26XEqbbbZZ6eM0OD4N424tD9ml7XrfOYvak76Pwr2/8Y1vlNqDHvQg3A+F419wwQXRvinwjsZvGhJNoXy9sUph2/R5CktcKeGm0+m09HMauEntQuO6NW4X+s40cJ+ulYsvvrjULr300lJ72tOeVmo3uclNSo0CU1vj/qRrYPvtty81Ohdq79vf/vbRdjT+e4tFpKGdQ8LTxzY7x6XXURrU2RqH3n/6058utfe85z2llrZdulAL9eWQuSO959O94/Of/3ypPetZzyq1c889N9ovjdVecC7NLXTt0HGnga5LbXFxscyH1Hd0vFdeeWWpvfrVr8b97L///qVGCyrc/OY3L7XnPOc5pXbFFVeUGi0YQeOaxvCQuYOuC7p3037S6yJ9Rk0X3WmtvyDNb7rvlSBt4yHzEY11uq5pvqXfP0cffXSpHXDAAaVGi47Qc/lrXvOaUvvLv/zLUmuttb333rvU6DfRK17xilKj8ZL+LqV7NvVV77k17cOVMo+SdBGJdDGjTXE8s9KQbbr+L7vsslK7wQ1uUGpbbrllqdGzaGv8bE5tQe2YLu5D42je+0TatvP81m3Nv5SSJEmSJEnSMvCllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGt3goPPZYCwKe0tDxHuBcmnQHAUv0vFQ6NeGDRtKjULEbnGLW5TatttuW2q9Y6YgMdqWgoDvec97lhoFT6dBkENC4CisjM6FggJXQtD54uJiCYpLw8FpXA0J5aeQ0GuvvbbUKCSaAu8oKJHanfZBoXy9MNCtttqq1CigjtoxDROkuSEdL71AZDqfNPxvpYRITiaTcnxpaCYFIvbGK7VBGgBP45U+S8d48sknl9rTn/70UjvooINKrScNV6ax+alPfarUHvKQh5Raej+hcU0B/K3lgfE0N9GCH72Q6qU0e32li2gMWWyAAkE/+clPlhoFoj/zmc8sta233rrU6LjT54XDDjus1Ciov7XWHvnIR5bad77znVI76aSTSu39739/qaXBoUPmBpLe81fqc0Br/zUnzJ5HOg/SeVG/tdbam970plKj8POb3exmpfa1r32t1M4666xSSxdRoe1ozPTu21RPw4qp3+n70nsMzQO9MUzPbvR8TDV6fhrb4uJiOYc0gH1IeDu1UxoST+OfAsx32mmnUrvvfe9bai960YtK7e/+7u9KrdfnxxxzTKm99rWvjT6fBjWnIfLpIhC9fadz80p5bqW5lc6LfhNRO/eemeh6TX8r0X5ort9mm21KLR0f9FzWm1vpnkrzHtXS36a0j3R+GPJ8QOaZW1rzL6UkSZIkSZK0DHwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI1uMiTFf7PNNpvOrlBHSevpyh+0ulBrnIy/qVdESFcSOe2000rtgAMOKLVe0v4ll1xSakcffXSp/cd//Eepfe973ys1aod0VZV0xZnWOC0/XVmCjueaa6758nQ6rQ23RBYWFqaz50ArM6SrC/WuE6pTm9JYT1eII9QXNDZoFZMXv/jF+J103De+8Y1Lbf369aVG8wCdC43LtL17Y5XaNl2liPp/7LHa2n/NrbMrlKQrydG47o1XWgWFVjdKVwhJV13bb7/9Su34448vtZve9Kal1kPXz49//ONSe+hDH1pq3/72t0uNVo1KVyJJV5JqjY+b2jud13/xi1+syLk1Xe2lt0LUPKvKzLPaV7rCDY2X3kq46ep06RxF7U3Hna5mRt/XO550RSJqs40bN44+t17vetebzq4sSyutpffj3pxAbUj7oe+kcXPllVeWGq0QnY6PdMXGnvSaontP+gxONbpOenNruhJh+hx97bXXjjpe6TcWob6gMURzQmv5szw9c6UrF65ZsybajqRzY888921qs/QZi7brHUv6HEDPbHSMYz8HtPZfc+vq1at/rTbPfJSu4t0a31PneRagMUfH3XkGK7XeHJXeu9PfgnTO6SqkpHefmGdVviHj1b+UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkbH6ZYd0+m0BHpR2Nu8gVr0+TQUkT579dVXlxqFkFHQ2V/8xV+UGgUrU3hoa3mAbXo8abgb1Wgf1Dat5cF0dH4rwWQyKedAbUco0K0X8kYBbrQfaieq0filfVOwXtpnNK562+62226lRiHRhMYghXimIX+9eYXah65HmqvSMMHlkAZGDrkGKWg0Ha80Puh4aHx96UtfKrX999+/e5z/U2+O2nLLLUstXXiD0LxOoZbUDr0QfpK2I10/vWt3bLPXTS/UcxaNqyHntKkD1dP7JF03dCy9cZAu+jBkMZJZdNzpHNK7v6VtNm+I9lJaXFws94M0UJ7Oq9dWafAx9Xs6L6fPJjSXrVu3rtR6NvU1lYaf03bpM2/vO+cJPx8b/cYiaah+77pM5570dxeNt40bN5bakLknlZ4LSZ9f0qBzmkOGLDpDn0+fQZbDdDot7UXP6emiIL1g/rSt0wXM0n1Q21MtXQSotXxBnXkWjEgX1iK9Z7m0D+haod99vXnOv5SSJEmSJEnS6HwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjco6HwymZTQQQrPSoMFKTCsty0F16WhcBTcRdulgdK03144GH0nBTfSfmi7NHiTwsbS4MHed1K/rNTAyNay40jbuDdWaVymwb1pCCodYxqW+prXvKbUVq9ejcfzjW98o9R+8IMflFoa5pi2A50LfV8vqI/2Q9fjdSmov4euy3TMtMZtlQZB0mfpGkuPMZ3ThwQLp2MuDXWl7dL5rXcs6XVP+6Fw0LHH8MLCQrm+0vDO3vel0n5L58x0AQrqs3SRi94xprXePfo3/T4aQ705IA2oTYO6V4p0gR66BocsapDeK9O26j2HzNqwYUOpDXkuS5/h02tlnmfCdAGX3vFQ29KzMIXxjo0W6EmfheYNxKZtKYybDOmjWXTc6T2ytXyBAbp30CIqNF5oHkzHy5D7W9qO8y54synNtmt6vLQYUS+gnuppu6bXyqa+X/XG/zxzYTr/U9umC2cMWXgmfb64/PLL4+/0L6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0Q0KOp9OpyXYKg39pu16QWBp4HgaAEoojIv2QeeSBiu3xkF6q1atKjUKFEyDJdOweWrXXuhaGqhOn6d+GRIOuqnMHi/1W9qXaRg07beH2o76l0Jk0wBmCsZ78YtfjMeTBk+n4aZp0CVtNyRAPw0CpRr19XKM1cXFxRJOSMGEdK5pqHtreah8el2nwdNpm9K83Au/pGPsbTuL2jYNrk7Ha08axkuG7GcpJXMctefmm29ear2A1nQ+SgPv5wmYn3cxhvTZgqQB7WkYbPpc0Rofd3rdraRFJGbPOV2MIV1Epvf59B6/qReWSQNxe9LrJ72maBxtueWWpUZzRvp7oiddOGklWFxcLG01TwAz3U9bm29xk3S7eRbBou2GBESn9/J5xjmN6SGh5ul1O8+z9Rhmj5mON+3j3j2R2jpdPIf2Q+MjfSeQBob3nnnpGKlGbUbvCdJrlNqWFnzozTf03mL9+vWltm7duug7e8ftX0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo5sMCU6dTCaXttbOW7rD0W+x3afT6fZj7cyxqjmMOlZbc7xqLs6tuq5wbtV1iXOrriucW3VdguN10EspSZIkSZIkaVPw3/ckSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0V1vyMaTyWS6sPDr77Gm02nZbnab3nZU+1/2Pdfnk+8jtI8hxzLPftJ9p/tdzvaeTqdrp9Pp9r/xFww0mUzKwabjclOfe096PKn0XHpjiD6/uLgYfz7Zbimu2fQYB/T1qGP1/z+O6eyxpMc7ZBxt6nE8Tx/PeyxDxnYibW8ab5v6Wu59nvazuLg4+tw6exxpO6XbtcbnSnqfT75vOef/eebrecbqvM8vc17zK2Juned6nfc5Km2/9Pon6TWx2WabYX2e5/UxxmvPPM/MZDnm1mSsrrR70Dy/X+adb+cZl5v6++adG9LrdiU9t6Zz0hLsu9TS+WMJ2j76bM8Yv/U3dTv0Pp/qza2DXkotLCy0VatW/Vrtl7/8ZdnuD/7gD0rtV7/6VVRrjRv0937v90qN9p1O1te7Xj31eX6U9zqSjpv285//+Z+lRudHDxF0flSjffQeSugc08/T+V177bXn4Y6W0Gwf/+Ef/mHZhs5p3ps7fSfZYostSo36nMwzIf/+7/8+fiddjxs2bCg1ur7TlyV0jL15INlva61t3Lix1NJxSfPAcozVyWRSzi8dmzSOfvGLX+B+NvUPL2pn6s+036nWe+ihfdN8m960qb1pzF199dWlRnNLb1ynN33ajq7dq6++etTxurCw0DbffPNfq9F19POf/7zUqM+uvfZa3E86z9B+qO1mj7m1fLyl9+LeQxqNrXQ+Su/l9FmaB+ga6c0XtO0898drrrlmWebW2fOgsUB9TOfau66pTu1H/UTjlY6H5npq+2uuuabUaGzOPs//Nxpf6X06vSdQO9BxD/nRm75ESX+grV+/ftTxSs8BNFZpzqR7Q+95Mr2Gqd/S33c0L9MxpvNRbxz05q5ZNN5I+oKNjpv6pfcbq/PsmRwi7ns55lZ6Fhjy2VT6LETtl95T03E078t0Gku07/S403cj6WdpDu59Pn2PQrXec6v/vidJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkY3KFOqtfq/k2nGEf2PY+9/OHv/g5tsN0/eAR1j+n+rvf+rTfOF6P9P6X+36f+06f9C6X9c0yyD1rhtKc+A/pc8/b/opbSwsFDaL82Rofbs5TClGVD0ecpC6uUmzUrzSNL/X+5tS1k56WfTHCFqmzRjrTU+b/pOOu95gvo2pclkUuYuOq80u6d3XuncmvZniq4pGuvUx71+pzyUdO5Jc/ioRsc9JB+B+oDalrZLcw+W0uLiYsnVSvOf6PrvzTFpngmN/zVr1kTbpdc/9UWa+db7PF3LlFeW5gil2Tl03L12SO8paX7McphMJuU80swNOv8hOZxUS3MaafxfddVVpUbHTW1Px9LLDyHpszXNUTS/pXNe+nzQ2nzZVemz11Ki5wC6p6X5rr1zSsd/2m/U7mnOaPrbsDefpBlmac5oGhKdjqHe70/qQzrHNE9pyLW8qUyn09J/6fU67wIGdM+h7dLsaeq79B6f/t7uSbel/QzJip5FfdD7bNoWacZVj38pJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI1ucNB5EuiahnH1Ak4pAJpQoBYFd1H4GYUHpkGHtF0vZG5IAO4sCoyjWhoiRseSBqK1xuGDmzoQeVNKAu4pYI7GUC9geJ7+TYNV6XqYJ5QyDf5vLQ+rTENe5wks7AUj0jyQBkJTIOZymT2/dGzR9U9Bz63xvJeOJZr3KJiZ+oPGddpHvXk5XSggDbhPx1HaXr3rLF1wge6PNF9TcPJSmz0OOn7qc7reeuM8DUqnfqNxSd+Xjst00YEeGh80781znaTPNDRfUBu2lt/f5w3/X0qTyaQcXxrWTOffe9ZbvXp1qdF+KLg+nY/22muvUjv99NOj77viiitK7QlPeEKptdbaJz7xiVKj8ZrOt9QO6ThMf0+0xotf0Fwwz4JIS212bk0XfKA+7/02SO9B8/Rbb05JjmXIYkM03mjhKRq/1LZ0Hadz65DfPkPuhbNWwoInPWnYeBo836un12v6e4XGHPURtT2dS+/aSxe6SX9Tpb/76LNDFn9J52ZqH4POJUmSJEmStKL5UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI1uUND5dDotwVgUPtcLgpzVC4WjkDoKB6NgPgo1o1oaakY1CqrshfpRQB6FmaYBeb0QsvR4ZvUCyKhf0yDFecbEppSE8dI5pWOtJ20nGgdp8NyQkN1ZQ8K903FJ1+y6deui7SiwcN6QXDrHNGBwucy2A80d1H5DAk7TcMkh3zkrDZ6mMUzH1xvr1HfpfJuGcFON2obmh14QZ7q4An1+pQSczp5vel2nwZi9bdMxQ7U0yDoNQaW5uhfGS/e/dKEWOuf0fpTOIb3vSwNq0xDU5TLbrjQW6HjTc22N2zpF45CClO94xzuWWjqut9pqq1J7xzvegcdzoxvdqNRoLKQLXaSL7NBxp/fyIceTLjIwtul0OuhZ839Kx0FPOremi3+k4dTpPHHllVdinYLJ0/skBePTOE+fmdNng9Z4XKbts5IXkUgXI6Fz7V3XaVg5tXXad+niKOnvw965pL+Z50HjnxbOSReTaY2fG9LnC+q/3nWxMka2JEmSJEmSfqf4UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI1uULLfwsJCCZVbv359/dIwyLUnDaROP0vHQ8FiFH5IYWXpdr06BYnRd1JgGEnDIen7aL89FFZG7bgSQvim02kJYUuDHIcETdJ3pmGb1G8UgkefTYPT6bN0zbbGY+Etb3lLqR1zzDGlRuONglrT4Nw0JLO3bwr1SwMGf9Og0XlQYCQdbxr42AvjTeeuNDSV2o/CD9MgZNrHtttuW2qttfaYxzym1J761KeWGl0Xe+21V6ldcMEFpUZjmMYHza1DQsmpX1dqGG9r9TjSQFwKW+4FRKfXYTqHp4GxFH67cePGUksXS+ltSz7wgQ+U2p3vfOdS22OPPUqN5vU0BLgXNkzjn9BYHRKyvJSm02k553R+ozm411Z0z6FxmM7B9Fl6trj88stLbfvtty81Oufes9rZZ59dajRnUltQbZ75P12MoPedNN7T8POxLSwslHtqGrw95FkmfbagtqN7EM3hNI9SG6fPtzSXtdbavvvuW2p/+qd/Wmr0vLDNNtuUGs1573rXu0rtcY97XKnNu2gPfT5dOGCehRbmMTsW6RxozKS/W3v1edqa5tv0HQONYRozQxaUSrdNr1saM1Qb8q4lvW/Roi5DroHlf3sgSZIkSZKk3zm+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaPzpZQkSZIkSZJGN2gpH1rFJE3Vp0R2Wq2k93lKeU9X00pXLKJ0ejpu2kdvpZn/83/+T6m94hWvKLXdd9+91GglEVpp5eijjy61j3zkI6VGK7fQufT2na52QCsVLYfZVQToWGnVgyEr8tG50koM9J003mjlAzpu6h/ax6677lpqPbQfWs3swQ9+cKmddNJJpUar9NEYpGuM2rB3jaUrkdAKaSt5hSiat9LVN3orCaUrntA4TOfbdDVAuqboWGglqNZa22KLLbCeoP3QSkM0Dun86Pt6c0a6Cmi6gipdP0tpMpmU/pxnVaUh1zV9nto5Hau0D1rFjs7lL//yL0utt4rOzjvvXGp036bj/tnPfhZtl7YXjd/eSr9p29J1slyrQc2ilU3TFQiHrBpN55uuiEt9R23/9re/vdROPfXUUjv99NNLbZdddim13qrRO+64Y6nd5S53KbVPf/rTpZbeT9LVNdP5sjXuQ7ru09W8xjadTss50LW1YcOGUhuyolj6LE/bURvTbzm6HtJnWZqPtt5661JrrbWPfexjpUYr/81z3/7rv/7rUrvqqqtK7ZnPfGa0j9Z4XM6zWvtymE6nZYykfUzt3Luuqa3SZwkam3Q86b0y/b3Re+9AYztdsZ3QudC+acxQu/ZWl6X+SlcgHnIf9S+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNENCjonafjV6tWrS43CJlvLA34pPGuewDAKMNtyyy1LbY899ii14447Dr/zlre8ZanRcadhZTvssEOpvfGNbyw1CpmmffTCSG92s5uVWhqUTkFuY4efUxhvGnTeG5eEQigpUI72TfvZd999S43C8i+77LJSu/LKK0vtyCOPLLVekF0a9EohqE9+8pNLbb/99iu1F77whaX21a9+FY9nVhqM2loe/t0LVlwJ0gUf0tD73nfOE/qaBkYS6iOq0VhvjYPOabx+4QtfKLUrrrgi+mwaDjkk3DHtQ5r/V0J49GQyKedGxzpvcHQaWpzum6Rzwstf/vJSe9rTnlZqa9euxf1Q/dxzzy21E044odQo3JrakdqG7tlpIHrPPAsw9EJ/l9J0Oi2LAcwTAN97nqRrk8YhfZ6eBdIQWZof99lnn1L76U9/Gu2jNe5jGpuPf/zjS+2f//mf8Ttn0T2GjmdIgPc8Yd3pgkhLbfZ46dk5nfN6zwH0+XmeA9IFg9KxT0Hud77znXHfNBbo/Gg/aeB7en+n8+stJpU+i1G/0u+OIc/Hm9JvGnRO46N3DdJ8nS5MNk9/Evos9QfttzV+5qDzpns3PfPSOEoXlkgXqGmN+ytdCI7ah86vNf9SSpIkSZIkScvAl1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0g5L9JpNJCbui8CuqURhXL0SPAunS/VCNQtIojIuO56/+6q9K7fWvf32p9YJV08A9CiGjAGgK3qQw9TRErhfqTYHbn/vc56LPDwmzXSrT6TQKVaV+S4PjWuPzp3am8fakJz2p1B796EeX2s477xwdTxow2AtEpIC7NDCY2uzAAw8stU984hOlRgGW3/72t0utF8ZL503XSRrGuRwmk0kZN9T2VKPzGhIonC4EkV7XtDgEBSpSjcbRrrvuivtJQ20//OEPlxoFNFI79MIqk+16fUD1efazEqSLO1CtN67SENo0bDMNgqU5gcLyH/CAB5Taaaedht+5/fbbl9r69etLrRf+OYvOL31uos9SsHBrPI/SftLjWQ606Ekawp+GI7fGYa7p/TcN2aZjpPmEvo8WGTn66KNxP3TeNK8/4xnPKLV//dd/LbX0XkYoQL53z5qnbVdC0Pl0Oi3nQNfwH/7hH5Yanfu84dc03qhG1wQ9r6WB1XTO9OzYWmt/9Ed/VGq0OFBvrM9Kg9PJkOdtmi9o3+m8shwmk0m5J9P5ps9qvbaidqH7fhrMv/XWW5ca9THtd7fddotq//7v/477vve9711qT3/600uNfv/TAmY/+tGPSm1IgPms3linZ4R0Qam0X1rzL6UkSZIkSZK0DHwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjco2e9Xv/pVCbuiQLqNGzeWGgVm9kKG00CuNDyNvo+CHB/+8IeXGoWapyHirbX24x//uNSe+tSnlhoFpFLA3apVq0rt85//fKntsssupXbxxReX2t/+7d+WWmutfeUrXyk1CqFLw/pWQuhpGpg5JIyX6hT0RmH0L3jBC0qNxlYabkr7TcP7/rd6Ig3EpNp97nOfUjv33HNLjcI9e2geSBdBWCnmCRntBWKnYzsNAqbxReOQ0NxBY4Gu29b4nkJz1KmnnlpqadvS+KDjTsM9e3W6nucJ7V1Ki4uLpU/o2qS2G9JO1M7zBNSn7U61U045JdqO7ruttXbssceW2hFHHFFqdN+lcU5tS9cJ9Qvdi7fYYotSa42v5bQPhyy2sJQWFxfLeaRjhs6hd59Mx2Yawk9tT5+lsUCfPfnkk0vtrne9a6m11tqf//mflxq1D81H97jHPUrts5/9bKnRPSZdJKkXiEzHQ9fFSh2vtJhUGhJP7dQ7p/TemX4nHQ+NVbp2qM/TfbTW2i1ucYtSe9GLXlRqdC40Xmjf55xzTqnRYjxkyHMrtdmQthjbdDot7ZqOmfQZrLXW1qxZU2oPetCDSo3mvUMOOaTU9tprr1Kj5410UZr0XURrra1bt67U6P57s5vdrNQe+MAHlhotmPLFL36x1B73uMeVGr0TGLJoG127845N/1JKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGt2g1NTJZBKFkFNgGIVrUqhbD4UVUjhYGgRMQX8U+p0GL77qVa8qtdZaO/roo0uNgsCoXdPA1a233rrUKPDtj/7oj0rtggsuKLXWONSMahSyuRLCeCeTSTmONOi6F8BPaFyvXr261Ghs0ThKw8bTIG/SCxb+2c9+Vmrbbbfdb7wfOhcK07z//e9fav/wD/9QahdeeCHuh64dap806HI5QvnTMF5qvyHh0fSdacA3fZYWtaD5n46H9kGLTQwJCj3uuONK7fzzzy+1eYJir7766lKjNqR5oLVWFgtpje9Hafj52CiMl64ZOidq4971li5akobVEuq33rWTbPfOd74Tt6Vg1YsuuqjU0uubjjsN0B5y76BzpOuR9rMSgqNby59baU4YstgAtT+1K413aqt0rqb5lubWSy65pNQ+9alPlVprrd3rXvcqNRoL++23X6mlz7LUXunc2uuD9B6fhocvh9lzoLaj/u0FK5P089TnNKbnWTiAjiVd3Ke1fDEq+jxtRwsW/NVf/VWpXXHFFXg8qXSRDfotN+Q3ylKbHTfp+BjyTuCjH/1oqe2zzz7R8aXPemkIP0kXWOjtJ5UuIne3u92t1D73uc+V2r3vfe9S++53v4v7nmcRCrpH9fiXUpIkSZIkSRqdL6UkSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLo5k6kpjC7NDCWAu560hDPNGSNgsDueMc7lhoFmD3rWc8qtTe96U2l1ts3ncuqVatK7aSTTiq1vffeu9SovT/72c+WGgXz9QLI0nA46v80KHapzZ5DOl5IL2SRzvUe97hHqR1yyCGllobEUTBeWqPw8jPPPLPUWmvtiU98YqnR+LjhDW9YajvttFOpve997ys1CuXbc889S+32t799qX34wx8utdbykFiab1ZKuCmFR6fBpUmI7//cz6y0/dLQY9oHhT7Sfu9whztE39ca993Xv/716PPpcVPoabqYBoX2tpYHGKe1lYDuk9R2tF1v/Kbnms6Z6ThPQ1nvdKc7ldoBBxyAx/i6172u1NauXVtqtHgL7TsNk6b2HnJ/pnakfaeLtwx55tuUZtsmXdyB9OblNHA/7ZP0GYzGR9rHH/rQh7D+wAc+sNTuete7lhq143vf+95Se85znlNqb3vb26LvGxLqTO2d/kZZSc8C/xM9M6ULXvTm1nQRAhqrdF+jeSsNU0/77Na3vjUe4zbbbIP1WdQWdC5HHXVUqV166aWlRv2SLpbQWr7gRBowvRwWFhbKb4S0Dag/er+jb3vb25Zaer1eeeWVpUa/j2lhmlvd6lbRfs8666xS+7//9//i8fz4xz8utbvc5S6lRr93jjjiiFJ7ylOegvuZRc/g3/ve90ptyDNmOoYprL9nZbw9kCRJkiRJ0u8UX0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRDQo6n06nJbxuzZo1ZbuNGzeWWhrg1hoH5FHwHYVnpcHi3/nOd0qNAveuuuqqUjv++OOjfbTGodB/8id/UmrPe97zSm2XXXYptQ0bNpTaBRdcUGp/+Zd/WWpDgnPTcNWVErg3azqdlnC9NDCTxl8aDNlaa1tuuWX0+XnCjSlsj4LxX/ziF5cahZ+3xqGWNN7OPffcUjv//PNLLR0bdH477rhjqfX6j+YGClaka3RIQPVSmk6nZYzQedHYTEOPW8tDuun6pyBI2i49RgqzpwUfeoGWNB6e+cxnlhrN4eecc06p3fSmNy21s88+u9TSQONeuCP1AW27kgNOZ9H5U5h8GmrbWh6Oms7X6f2L5jw6l3e84x2l9i//8i+l1lprz3/+86N9p+G58yxOQGOtF/KdthnNA3SdLIfpdFrakI6N2p7aqncNUp/QtnT9zxN0Tt9HC5TQuKZaa6295S1vKTUKOk8Xv/nTP/3TUnv7299eanSd0djshUfP80yVhocvpel0itcSbTeLnm9696D02YIWq0j3ky5+QNcNPQeccMIJpdYa3yfpO+l46DuPOeaYUqPjpmsnPZbWeKyTdOGN5UC/s6id6VzpPcE+++yD+6E2pOv15JNPLjUKB6dnwnT+pvFP18mQBRo++tGPlhr9DnnYwx4Wf+esL37xi6VGfdU77nkWPxvCv5SSJEmSJEnS6HwpJUmSJEmSpNH5UkqSJEmSJEmj86WUJEmSJEmSRjcoiXJhYaFtscUWv1ajsDIKZqNQsl7INoWi0ba0n2uvvbbUKMDswgsvjLaj0OrZNmitHxh57LHHlhoF/NL50fG8/vWvL7XXvva1pZaG5PZC+LbaaqtSo76m4x4S8LaUZs+XAth6Y3BWL7SVwuiOOuqo6DvTsEnqNwr0e9KTnlRqdH69UP5eKOYsupbvfe97l1oa2kvn96AHPajU3vve9+Lx0DWfBk7OG8q3qUwmk3IsdGzzzq1paHIaOEv7ST9LQbd0nQ0J97788suj43nkIx9ZaocffnipXXTRRaW23377lVoaktkatwWdd3p/G9tkMilzSBocnAY69z5Pc1d6D6K5mo6H+uLBD35wqe20006lRvNyaxw8TedNx5Pet9KwcRqXtIhBa/m4pmtsJQRHt/ZfbTU7HtLxOmRxmDSEPL3Hp8+E1PZ0Lx9y7dHzcboAER3j1ltvXWrbb799qV188cWlNmQhDzoeatt5Fr1ZarPHlt6zh6B2onBxuiao7dJn2TRYmRaM2HvvvUuttXxBhR/+8IelRvd8up5oDKWLQPQCzSkcO71P0H1wOcYqoTZIg/Dpntrzyle+stTo9zHth2o0jmi79PdG7/kv/W2SLg6X/qZ65zvfWWrpc0Rv3+miHUPG5sr4RSZJkiRJkqTfKb6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJoxsUdD6dTkvwXRq8S6HHvfArCkWjADgK5EoDDClY7POf/3yp3eMe9yi1//iP/yi17bbbDveTBkFSWNkXv/jFUnvNa17zG38fBb32Ak7Xr19fammI5EoxOxaGBL2nDjvssFKjcHxC18l73vOeUvvgBz9Yap/5zGdKLQ1G7QWa0zWahi3/5Cc/KTW6vtMgxwMPPLDUdt9991JrrbVzzz231NLrbqUEnafoHCj0shdgS2jhhjSklMbwzjvvXGovfvGLS23XXXeNjq8XFErH+E//9E+l9qlPfarUbnGLW5QahaSTNWvWlNratWtLrXfc8wRS03ZjB5xOp9MyFmgcpAG0vTmY7uXUdmloNR1Pr49m0ZxJc97HP/5x/Hx6j57n+SUNfE/DW1vj9knnoJX0bDB7LOk50OIuvTFDYyQNMCdpoHS6QAnde3uf/cY3vlFq9ExIAebk+te/fqn96Ec/KjVaYGfIfTv97TAk9H1ss8dB44Cey4bMJ/Pcb3pByLPSkH+6n9785jePjq+1vN/oOYfGAc2F9Gychp9ToHlv32ltJc2ts/2cjk0K1r/VrW6F+3j2s59darSIWLrAQ/psQudCY5jmqHQfrfF1Sr8t6d6TBrmfddZZpUZtQ9dJazzm6J0CnQu9Z+g9b1+3fpFJkiRJkiTpt4IvpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0ugGBZ23VgNEKUSQgq4ogKwXUEdhXmmYYxpwSiF8//iP/1hq++yzT6ntsMMO0fe1xud45ZVXltoXvvCFUjv44IOj/dA+KJSMgjypvYZIQ2aXw+yxzRMO2Avj3XPPPUstDc9Og0xPP/30UqPrjgIaaexTwGBr3G8U0kj7pvFL443GS9peb3zjG7F+97vfvdSobem45w2635RmjzltF7r+qe1b45BS6mMaNxT6SP35wAc+sNQe9rCHRZ8lvfmEjvvDH/5wqdF18frXv77U6DrbsGFDqV1xxRWlNiQkN70/EuqDsU2n0xL2SWM1DbrthfFSv9N+aKxTja51uifQMaaLCfRC59MA8zRsmMYLhfHS9T4kODcNNabjWSnPAa3V9qf+oOOl663XVtT+1C7pvJcuakDb0Vincx7y3JqOYbpWbnCDG5QaBVzTeOvdy0i6mA3dOyhknYLul9psO9P50zhIF3dpje+J6SI06W8Q2o6C8em5jp5Re2OV+pyu5cMPPxw/P4uupzTwncZLL/Ca5pZ0QYyV8Bzw32aPLx0fpBcK/9KXvrTUKGSbQrpp3+lzf7pQRfr80xqPze23377UnvrUp5YavRuhY/za175WatRedNy9e346Duk7ad89/qWUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0g1bfm0wmJW2dUtXTVRx6qxLQyhCUlp8m7VMKPq1o8bnPfa7UaEWOISuI0fEcddRRpfaud72r1GglgnRFs3QVkyHHTUn7tF1vBYUxTSaT0i60Ek66eiSNydZa23333UstXd2I2o6Oka6TdPUU+r7eyhO0n3TFQlothcZbuhpUuhJSa7yiy7p163Db5HiWw3Q6LedMK+zQ6hs0XmnVuNa4Denz6WopNJZoddH169eXGvUb6a3iRXXqdzpGGuvf+ta3So1W06FrgmrUV63xNZmubNhb3W1M9ByQrmZDbdKbW2l1rvTZgGrUxnT909i42c1uFu2DVmBrjfuNnl+oz+lapmuWzmXeVdjSlYvTzy7H+J1MJuWcae6ga5jGTG+lTJozDz300Gg/b3/720stbb90FS96Vu/NrelzUfq8fckll5Ta6tWrS23t2rXRsfTQswS1Y7p68UpAfUR9MWR16fR3RPrsSWPwNre5TakdffTRpXbggQeWGp1L77cKHc873/nOUjvttNNKLV3Fjvog/Y3U65d0LqTzXiljdTqdDlp5+H9KV/Zujds6XYmO+i6dW+keTffZIc9q1J+PetSjon2TL37xi6VG7xjmnTPouNMV5NNnhtb8SylJkiRJkiQtA19KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0Q0OOp8NyEtDFinoqhcyR2GhFJ6bBpxSGBcF/VHYWBpq2wvy+trXvlZqxx9/fKlRW6QBj3Q8dH5U64V20rZpOCgF0FHI5lKb7Xc6JwrboxC8XpDvbrvtVmppSPf3v//9UjvyyCNLjYLjqI3TwEFqh9Y4wC8N+rv1rW9daldddVWp0bVNx03X7Pnnn19qrfHYos/3Ql1Xgul0WoIR0yDHNBC9h/ozDUOmsfmhD32o1A466KBSu9e97lVqdC5DQpipj9N+T4NH6fqh+aF3PaYhs+m9bGyTyaS0PR1/Gv7Z698h286i+Za+j/qCtjvjjDNKjRZBOfjgg/F4Tj755FI777zzSo2uWxoHvTl8VhqIPqQPhtwfV4LpdFqOL73nDDmvm9zkJqV2zDHHlBq19V3ucpdSO/XUU0vtgx/8YKnRPSHt9944etGLXlRq6WIM6SISFNZMixvQM2pvvNJcny5m0lt4aUwUyk/nSuc5JGA4XXiH0O8SGht//ud/Xmp3vvOdS42OccjxfeITnyi1ZzzjGaWWLnhCx/Obhnn/b5+lPtxyyy1LjX770n1iJSyC0lren0MWVprnGqbxSr/70ntvuphC795x29vettSe+MQnllq6sMTrX//6Uvv3f//36LN0Lr1xRL/deu8PZg0KVI+3lCRJkiRJkjYRX0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRDUr2m06nUeAYBWpR6BcFHbbG4YsUVpYGoVJY8ze/+c1SW716daml4Ze9IK9b3vKWpUbhYhR6l4an0fdR8CDtoxecS+2YBoWvBNPptIwvak9qJ6r1At1OO+20UnvMYx5TatT2N7zhDUuNwsHTgFLqM+rf3nWXButTO+6///7R96Vzw7p160rthS98IR4PnU8a0D5PgOWmNJlMyrhLwx3TxSZa475La2nwNh03hQCnAes0v7XW2rHHHhttS+G5NG+lQaj02XSs97altkjbZ2yTyaS0cxpkSe3ZOyeac9NFJNJg1TSo83Of+1ypXXTRRaX2tre9DT//spe9rNRucYtblNrPfvazUqO5jJ590ucKapveNZYuGJEGBqdz2qY2ey2m4eB0b+ktInHhhReWGi3CQfPRgx/84FK7733vW2qHH354qVFgPtX23nvvUqPn09byBUmoHS+99NJSe8pTnlJqNI5oH9Te9HzQGs+t6cJEKyWsf/bY0meUIQsYUP+m/UHXBC1kcthhh+G+E0P65xWveEWppfNRGt6ftg3Ny3S9t8ZzQ7rwUrrgx3JIQ+rpvHr3B5oDaBymC5NRjfad/ram54jePfXRj350qdG7B+rj17zmNaX2b//2b6U2z5zXW/wi/a1P3zlkkSn/UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEka3eCg8980GHjVqlWltnHjRtyW9kEhXVSj4MYzzzyz1CgQkb6PwsqGBCJ++9vfLrU0rDYNdU2lAXSt5UGqdC4rJYx3NrCNjp/aJD331jiYlgL4KNTv7LPPjo6Hguco+I+C+uhcev2Thiw+/vGPL7Wjjjoq3s8sGtO0Dwpv7X2e2iwNjl6O8PPJZFKOha7/NIy7dw7zLKhA6LrYcccdS+1GN7pRtF+ag7/1rW/hvl/wghdE30njOl3AIr0npEHfveOh46ZrfCWE8S4uLpZrKV2AgwwJv6Z2SqV9SXMHBZn+9Kc/LbWzzjoL9327292u1B71qEeVGgWZzhMSS+dHNbpPtMbPSem+V8oiErRADz2PUh/TmOmF41O7nnPOOaV2pzvdqdTSdr7ZzW5WajS3pgH1vWs0fdajPv7iF78YHSMt6kL7TRd6aW3YPJzse2yTyaT0O90v0t9IvWew9NmCvnOXXXYptec973mlli7+Qe1O21EweGut/eQnPym19PxoHNHvUtouHS9D5guyUn9jtfZf5zDbz+lCMOm9tzXuT3o+or6jtkoXm6E5mMYhBZVvueWWpdZaawcffHC077Vr15bakUceGR1jeo8f8n5jnsD9dOG11vxLKUmSJEmSJC0DX0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRDUoNpTDeNBxv3bp1pUZBZa3lQbIU+vjpT3+61CjUko6bwvXuda97ldp73vOeaB+tccgdBY5REFgaIkbSoL9eGCn1DX0nHeM8YZObyuLiYgncSwMD05Ds1lp7xzveUWoUwEyonag95wnlJ72wfPr85ptvXmpPecpTos9Sm6VhnB/+8Iejz/Y+T2HzNH6pbXuhv0tpOp2W46NxOE+gdGt5WGEapLrtttuW2nvf+95SSxdooDHzspe9DLelfk+vCzo/au80ZHrIPEhtQWOOxuZKmFun02m5vuj+R31J5967B1G/pW1HY4PuxfR98wSeXnjhhaXWGs89n/nMZ0otDSNNQ6vTINMtttii1FrjvqH2oeOhe0cv3HSpzfYfXUfUVukCO63xmDvooINK7dBDDy21o48+Ovq+9BmMpGO9J733/MVf/EWp7bXXXqV2wAEHlFo6/nvzYHrd0/NtL0h7TBTKn97nqNa73tJ7Pt2DyPbbb19qNF7SZ14al8cccwzue+utty41+r1Jc/iGDRtKLf3tQ9fTkN9x6cJTNI/S+S0HWvQkvQbTsdDblmp0H0vfUaTPMNQf9Bz8T//0T6XWQ21GvyPpekyD5WkfQ+ZWGpvpGB5y3/cvpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRDQo6by0LWqUQMQoH630XhWJRcPGrXvWqUlu9enV0PBTw9ahHParUKGyM9tELa127dm20bzrGNBQ2DSujoLNeADG1NxkSBDq2JACa2iQNN26Ng22/973vldpNb3rTUrvzne8cfd+//du/lRpdTxdffHGpnXHGGaV285vfvNRaa+3AAw8stZvc5CaltvPOO+PnZ1Gb0biksEkKzqTz66E5hAIslyPUnEwmkyiMNw0zHRJ+nobn0rXywAc+sNQo1JauKfo+GjOnnnpqqfWkodCEQi3TAP81a9aUGoV7tsZtQfMwzcErYW5dWFgo808agjkkWJnGB4URp4sVpNdO2sY77bRTqT384Q/HbaktvvnNb/7Gx5MuREDBrzTfzhs2T4sErKS5NenTdI7qzSfULtTvb3nLW0pt9913LzV6HqX+TMfrvMHCNDfPc93TmEkDkemzPekcTs9UQ+arTWE6nZZg5nTxm40bN5ZabzEpamdqJ2rn4447rtS22mqrUpsnWPmss84qtbe97W2l1lpr69evLzUalxR4nf7GSp8dhyxAlAZU03ZDnu+W2my70jyRzo2980qvw/S3G21H/UljhrZ73vOeV2q3vOUtu8c56/LLLy+197///dFnqc3SsZne81rjPqDrIg3rp+u2Nf9SSpIkSZIkScvAl1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0g4POZ6VB11SjELHWOGiLgmTvcIc7JIeI+6HwOAp6vtOd7lRqFOTVC7X95Cc/WWoUvEbtQ2FlaYBzGmrZC5un8DRqxzRkdmwLCwvl2Oi46PzTWq9+4oknltorXvGKUqO+pGDJv/zLv4z2S9/3yEc+stR6obZpYGoabkoovO+Zz3xmqdH12RtXdD3RuEzDVtNg7E1pOp2W40uD4mm7XjAk1dMgWZqXH/GIR+B+ZqVj6+Mf//hv/NnW8kBMmq9pHqWAzjRMvbc4Ah1jes8c0tdLZXFxsVyfFMCcLsrR6980zJXmChqradA5jYPHPvaxpUZzeu+Yv/3tb5fapZdeitvOoralc6axSuHHpDfn0T1/1apVpZYGzC7Hs8FkMiltQ9cMjRmq9Z5baWEC6hNqvyOPPLLUKNj5oQ99aKnd9a53LTV6bqXj7oXarlu3rtTSe8IVV1xRare73e1KjRYtoH2kQcWt8fiihYno/FYCeg5IFwShsda73mie2nXXXUvthBNOKDUab/MsIkE1upZogYbW+FzSew+1D83/9LyQ/h7q/TaksZ4+W/fG/9hogZ70eS39/dMat2va72noN6G2P+KII0qNFqXo9TsFfO+///7R8aQLc6Vh41dddVW0XWv575F08YAe/1JKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGt2goPPpdFrCydLwOArEovCy1jjEc8cdd4y2o5A6Ck+jELJDDjkk2o6Cyl7zmteUWmutvf3tb8f6LAoRo8AxCi6lwLh5A8jpOylsnvpgJZhOpyWIkMYq9S8FtfXQthQOudNOO5XaAQccUGp3uctdSi0NqKcx1AtlTaUhizQOTj/99FKjwGC6PqnWCxFOj3GlhvK3xoGRdL50rjSGe4GRdL50XdN37rDDDqV2y1veMjpGGsM0Zl71qleVWm8MU/tQECQdD90n0sDwNDB/yCIS1D70nUNC35fKZpttVsKDKbSY+iJ9XmhtviBZmpfpnrbbbruV2sc+9rFSo3FOffHOd76z1Fpr7SUveUmp0RiktqDQX/osofaicdm7xuieQtctbdcL0V4Os32VLiKQPlu1xm1I96HLLrus1Oha/+EPf1hqL3vZy0otHUfpAjs9NO/RvqnN6LNpIC4FXPfCeGl+oef1IQsvjWkymZSxkC4YQG035JxueMMblhot/kRtnPYvtfvnPve5UqPFb4Y8B9D1RJ9PF9NIryfSmwdpHk6f+ei66wXBLyV6J0CoP2h89J6ZaHxR+6cLF9FYoO+7293uVmpPfvKTSy0N1m+NF3u6/PLLS436OF0cgtBnaR4dMmekz3PpddaafyklSZIkSZKkZeBLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLpBQecUwpcGhpE0WLY1DgKj4K5tttmm1NLAVAor+8hHPlJqhx12WKn99Kc/LbXWuC2olrZFGh6YfpbOubU8ZD1tx+Uw2+9pIBy1Zy+Ml86fPn/EEUeU2pZbbllqz3/+80vtCU94QqmlIYlpGHRrrZ199tml9vWvf73UKJjyC1/4Qqn9+Mc/LjU6bjoeCiekwN/W8nBD2vdKCTpfXFws50fnlYbx9ubgNHCctqPg3cMPP7zUnvWsZ5XaBRdcUGpvfvObS+0//uM/Sq0XLJz2MUmDKen7qF/o+4b0Ae2bzm8lhEcvLi6WcHEK5aS2o3PqLSxB7UyhxWmoOfXHPvvsU2qzIe6ttXbppZeW2nOf+9xSo+eF1vJxmYbDp99Hzz5DxhDtJ23v3nU7tul0Wub59B5P13Wv/dKFVOZ5DqG2p32kY6Z3TyXUnzS+qJbeo9O5sfe8TMeY3jOH/B5ZKpPJpIwvOlYaq9R2vd8GNMfRPfq8884rtVvc4halduWVV5bae9/73lL73ve+V2onnXRSqaULZ7Q2bOGGWdSOdI2li5PQsxQtJNMaX3t0junCYSsFXW90rtR+vT5Of7umvy/IjW50o1J74QtfWGpbbLHFXPulxVDSBR5WrVpVajTW02fHdFGi3n7S35xD+JdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTRTYak+G+22WbT2VV2aGUASnkfsjJEuorJIYccUmqvfvWro/185jOfKbWnPvWppXbJJZeU2pAVbWgVCTqedDUo2jetBkCrHdAKKL3Vx9LVJlLXXHPNl6fT6QG/8RcMtNlmm01nVxBMVw+gvqDVDHpoVT36znTlD1pxKu0fWo2GxmRrvOIiHTftJ119gs45Xa2ht1IQHXd6PDSv/OIXvxh1rLbW2mQymc6uWpKuGpqugNQat1W6Ug19Nl1VJV2lk8ZHb46i/dDKL+lKWzQO09WuaGzNu1oQnQt951VXXTX63Dr7HJDeG4asUpiujEn9lt7/aMUtamNarYo+O2Q1z3Rlx1T6DJEeS2v5fS9dXe3aa68dfW693vWuN53tv3TOI0Oeg9JVNdP7VboSHR0jbdcbbzSWaP5PV4aja5zmUdouXdmzt+/0uY/mjMXFxdHn1tnn+XQ1a+qLXv/SM2BvlbhZtAo39RGNtyGrQafS1UBJ+oyVrqI55HcCXaPpfYs+u3HjxtHn1oWFhelse6WrvA1ZiTd9Vkx/C9DYpN//T3/600ttu+22KzU6F1pxvbXWTjjhhFKjcZiu/JquxJmuSNy7N6bP8On7m97vLP9SSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRodJ4x2TKfTKMiTgq6GBKpT2BmFb73jHe8otXe+853RPuYJ5qNz6YVH036uueaaaD9pSOk8YdS9faThgfP29VKaPQ46/jREs9dOaQAfBb2lbUcBc2lYchoG2Foe8J6ODTrGdMEDOpdeECddt/T5NOiPvm+pLSwslEBLCrikkNEhIdtpaGoalJuGNadtmo6j1ngcptcu9Xt6TdH50Xa9YH66ntMFF+YJaN5UptNpOTY61nkWG2gtH0dpICi1MfUFjX1abCLtx9530vlRW9B+6BkifQ6gsT9kwROSzrfLgZ5b08DYIeHx1E9pG6T7mWeRkSHPmFRPn1vTkF2av+m4ab+9+9s8bUbXY28OH1O6QNGQZ5lVq1ZF30ntRM9h6XxE55I+Q9P11Vr+u22eeyeNN/rNR4tO9Z656BhpQY10Xlous+2aLgRB139vvKah8ul30m+gD33oQ6X26Ec/utS23XbbUjv77LNL7aMf/Wip9Y6HxjY969P1SOdM44O2o1rvXUb6bJLej3r8SylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdIOCzlurgVVpoCKFzPUCbCngi4LO0gDANISTQr/WrVtXahQSSJ9tLQ8HS0Nt08Aw+uyQoEs6bto3hbZR4OJKQMdFoXxpiGRr3H7JYgC9z5I0gDkNmOuFCdK2aZulY4vGOYWJDglBTMOth/TrSkDtTH2UBsu2xu1Kn6d+SsN4qZ/SMUMBlL2AUho36b0nbQcaW+liBL0+oDFH26bh4cthtq1oXKbH2guwpfOn8UHBnOn4TQNj0+BnWjijNW4LusfOs8BAGuROevesNESVxjSNiQ0bNkTHs9SGBK/O6s1H1IbpPZCOJ30mTENyKUS5dy40H6XHmD4LzBMU3Xt2ShfjSRcPGNt0Oi3nQNc//fYZIn2+T5+P59kvnUu6mE5rPPfQuE6fc9JnmnSxhN5ckz7Dp/PFcphMJqX/qI/pNzNt1/sdnT4LpNdw+rt8l112KTU6xh/96Eeldskll+C+02eBdBGuNMCf2jDtq9byxYpoP0PmW/9SSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRrdoJTfyWRSQuUoZCsNY+wF5qUBcBSURdulAXd0PGmIZC/gNA3DIxTslwYzr1mzptTSsMnWONSMguWobam2EtBxUfBqGu7dWt5ONGbSQEUyT8h/zzzXDkkXGKBrh67tXrhneo4rJSS6Z7atKYSQ0BzVG69pECeFOaZzcBrQmAZP9/qNzpHGSBooTdL7Wxqs2hovnkFtloZej206nZaQeRqDNL/R9d+7ftOA7zToNr0X09hPg9x7c+M8i2yQNJSfFgOgY+zNrfR8QPe3dLGK5TCdTku70vmmz529Z7B0DkjDz2nfaZh6uvBCbz6hc0wXkaDt6PqhOY/GEW3XC72m/dAxroR5lEwmk9J+1Bd0/NTnvWswfdbrHeOsNHg7fdYYsihN+lxH1xidC8156aJE6cJfreVjkM57pTzLTqfT0q50vPMs7tAaX++0KEL6W4D6nZ7LbnKTm5TalVdeWWrUx713AulCEOmiAGn4ebq4T+8ZhPqL5mvq//S3TGv+pZQkSZIkSZKWgS+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6CZDAtMmk8mlrbXzlu5w9Fts9+l0uv1YO3Osag6jjtXWHK+ai3OrriucW3Vd4tyq6wrnVl2X4Hgd9FJKkiRJkiRJ2hT89z1JkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjS66w3ZeDKZTCeTya/VptNp2W6zzTYrNdquh7Yd8vnkeH71q1+V2uy59Wqkd3zpcaf7oe1oHwsL9X3jpm7X3vGQxcXFtdPpdPu5djbAwsLCdLYN0vNP2+5/qyfS8TbPPtLz6207z/il2uLiYvR9pHfcJN1P5xhHHav//3GUhp7nWu+hz6f7med4qD/SuaMn3TfN9ek1vqnbprdtqnONjjpeaawuRZ+P1R/JZ8cyz3Ev5/kNmOuXZW6dHZ/p8Q65h6Xz8Bh9nO533m2pLdJ2SD87ZB+b+hlv7GcB+o3V2a7U5m2ndFzO8wwyzxw15BmcPj/Ps0Fq3t9T6Xd22mf0uVXa1Ia+lGrXu96vf4QumFWrVpUaXfy9C5gm12uvvbbU6GUTWbNmTamtX7++1H7v936v1NIJis6vtdZ++ctflhqd3x/8wR9E+0lv2n/4h39Yav/5n/8Z1XpoIpwdD73j2bhx43nxjjaBhYWF0u90rj//+c9LbYsttii1XjtR/9K4pM/TePv93//9Uhty7cz6xS9+UWqbb745bkvnQvuh46HjprFx9dVXl1raXjSv9I6R+pXGJfXB2GP1v822Fx0b9RHNHb0HWxoPtJ95XuTQWKB+p+2GPPCm43DDhg2lRvMjfR+NTeoDasPej9d5XgrQdfGLX/xi9PE62y40p1xzzTWllo611ngs0OfTMU1tR/umz6bPGktxP6XxRp9Nt6Nz6Z1fr29mUZvRHPyrX/1q9LG6sLBQrvd0HFH70Xat5c9wNEelz4np+KI5k2q9uYjGIaE+Tu9HdE9YvXp1qdH8Tc9orXHfpM/b1LZjPwtMJpMyVtN5kNqzNybTPyagNkmfQWi8zTN+e/fTeZ6t6Xcg3cvSl2Tp74HW8mcs+u3buU8sy3OrtCn573uSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0gzKlptNp+T9Y+n9ZypJIMxla4/8Rpv81T0P46P/e0+ySNFOA/l+5tTxDgzJOOpkMpUb/A01tSP+T3esXOh/qV/o/djrnsdFYpbajcZXmOwz5PI2jTp5R9Nk0j4f+95zGVWv5+E/7l/aTfpYyCqhde/U096D3nSsBHRv1Z5qF1Foe9pnmmWzqbBY6v14OBR3jkFyNZN+U55Dm+fRyWWhbup7TDLRexs1SmUwmZRyl97Qh7UTzB43fNM8rzXCivkgz6oZk9MyT10R9nuYI0dzQy22j80kzDofkhy212XNOQ6Gp1nvWS9sqnVOols5l1PZXXXVVqQ15LqfvTMdmep2lz7dDcujmfYYf02QyKcdLfUTjl55F6Zm9Ne432pbaJM1cSsf5vOHg6fig5wi6ntL7zryLfNE1QdtSv9L5reRnWSnlX0pJkiRJkiRpdL6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJoxsUdN5aDXyjwLU0THfNmjW4Dwqko9A8CpWjz1K4XhosTtvRfnvBi1SnkMG3ve1tpbb33nuX2h577FFqFGBJ4X/UL70QYQqCpBq1D4UjUv8tpcXFxRJ6nAZB0njptVN6XvR5Gkc0NtIgyHlDJNPA4DQYO90HHSO115DgaNoPtQ9dJ2MHR7f2X8c2e8xpu6Qh2T20LdXoWqe2ojal76N+GzK3brHFFtF30jVKAadpcHV63+kF51K/pvteCWGm0+m0nG8axkvn3ptD0+eINGR9w4YNpbapx0FvvqVni3lCdnvB5Ml2Q8ZVOlbTcOvlMJ1Oy/nRmJlnQZHW+J7Te25IDJkLEzTeeuOInovmue+n45Xai9q7d4+m6z7t65Uwt04mk3IOdPw0Twy5BmnOpXamNkkXiUqDvGlc0oIlPbRvOj86HqqloeZ03Ol9sHeMtC3NQSthrEpLwb+UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkY3KIlys802a6tWrfq1GgXSUQAcBRhSWN9/7yf5PIXCzR5fDx03hd5RyCCFzPUCBelc7nvf+5baTjvtVGof/OAHS41C+GgfaYgwBT729pMGHaehlktpYWGhBC5TeCeFCNJYoxDT/95PgoIS01Db9PtorKZBjkP2PU+I5MaNG0uNAqvTkOPWuG/SQNh5gmM3pclkUs6Z5re0j2gObq0/5yafT8Oeae6hc6HP0tjqXXt0LnTt0rxF10963yEU7r5+/XrclvaThtXTdmMH808mkyjEmsYLnVNvrNK21B9DFvCYRW1Hx50unNALwaZ6ugBLulhFOi+nbThkW3ruGhJWvJQWFhbKGEsXx6Bnhp502zR8Pg3HT58F0uuxtXyxChpfdE1R26a/HehcenNGuu/0uX5si4uL0ThK+6I3D6Ztki5QQuMl/c2WzsF032yNrztqCzpGGoPpMw21V/obo3eMJA2Wl34bOLIlSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0Q0KOl9cXCyhdGk4LAXXDQkZpmDDNOiWguKoRuF/6bnc8pa3LLXWWnvHO95Rattss02p3f3udy+1iy66qNTSMOtegPmsXohq2jdDwjPHNntsFHRI50khgkPaaZ7wbNp3Oi7nGb+tceBkGqhJQbcf+9jHotrrXve6UkvDfXvbpsGxvRDK5TB7Hmlodxpe3kP9OU94bhoUnV4nvbBWOu60fdJw7DR4N/1sa3yd0efTuX45zB4HHSsFb9P11hsHNAbTYGX6TlpQYddddy21N7/5zaV2zjnnlNp97nOfUvvkJz9Zaq219ulPfzralhaCoDmY2iYdq0PC1NPFFtLtlmNhiV/96lftqquu+rXaVlttVbZLQ7J7bZXek+nzFKRM21GQMp0LzXl07fTC6NNwcTrGdDGG9HklXYCoNb7G0wUJaD8bNmzA/SyVyWRSjpd+Y6ULdcz7fJ/+vqO5nuaZNDh9yH0ivcae9axnldrZZ59dap/4xCdKjcY+jWnajtqmNT5HmjPT377SbwP/UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLrBS/n0VkL6n2hVAVp1gVZ2aI1XIKAVRu53v/uV2j/90z9Fx7P//vuX2r/8y7+UGq188LznPa/UHvnIR5Zaa7wayD3ucY9Su+SSS0qNjjtdXZDacMgKiOmKVXSMtJrGcvhNV/qhduqdE638QWispysk0nnQyh+0Ah7tl1Z4am3YiiezHv7wh5faXnvtVWq0IstJJ51UamvXri01Wnmotf4qZ7NWympQqXQeHbIiW3oNp6vBzSNduY/GTGt8jOn5UTvS9fOSl7wk+uwzn/nMUqO5vzWeS9LVfai2EqSrQVHbpfNga3mfp6s9fv7zny81Gm8HHnhgqdFxP/rRjy611lr727/921Kjlb0OP/zwUrv44otLjVbuo1We6BjTZ4jWuA/TVf7ouuvN4UtpYWGhXIvpKmA0t9J4ay1fJYvGJrVz7/l4Fs0J1O+00h6tSNz7TjoeOj+aR9OVONP7Tm/V3OT3SWv5itzLYbbv0ufpIc8G6cquNNZpHKSr2ab7pXmi95xH+9ljjz1K7clPfnKprVmzptR23333UrviiitKje4TQ8ZquoopXTsr5TeWtKn5l1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6AYFnU8mkxLSSOGAFNbWC4ckFBZHAeYHHHBAqb3oRS8qNQqW/OxnP1tq3/72t0vtDW94Q6ntt99+pfbVr3611Fpr7bjjjiu1n/70p6VGYX0UcJeGNacBm72gbgoApKBMCtRMAzqX0nQ6LeOQ2o7aiYIJe2G8abhkGp6eBqIPOcZZvT6na5nGAY3Vxz/+8aVGbUuh5nQ90JjuhbLSeVOYOwVlDpmXltrssaQLGKQh061x39EYTlHbpyHCdC507fQC1tOg0DQce/369aX2qEc9qtRoUYrvfve7pXbssceWWg+NVxrv1NfLEdY/26Y0DtIA8t74S58j0oUgdtxxx1KbJzD2oosuKrXefEL73mabbUrtmGOOKbULL7yw1I444ohS+/jHP15qFBg8JBCZtqX2ptpKCeOdTqdlPFA/pfeHXpg2tVUauJ8uVkHX/0Me8pBSO/roo0vtsssuK7WPfvSjuJ9zzjmn1E499dRSo3OhMZfeZ+m6Tduwtfz3SPrZsS0uLpbnpvSeT3rbpW1C/UEB32moebrYEl2LvWuE5hn6jbZ69epSS8cLnR8F+tMzby+gPZ1b6byHXBPSdcnK+UUmSZIkSZKk3xm+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaMbHHQ+G9pGoYbzomC/q666qtTSUEQKP3/EIx5Raqecckqp7bLLLqX2oQ99qNQOOeSQUmuNA4fT8Ezajr6Pwvp6YdazeoGHaVD6PCHJSy0J/k7PaYsttsDP0xhMw3gprJy2o/6lGvXZkJBMOpfb3va2pXbaaaeVGo3Ld7/73aX25je/udQoCJKOpRcwS21Gx0M1CpGkQP8xzI7XDRs2lG1ozFB/DgnXpPNNA8fTgN50HkxDRlvjc0yvAfpOCkKlz9I94R73uEepHX/88aXWGgek0nhP5+Cx0SISNC7p+GkMUXBua3kof7rAw89+9rNSe8lLXlJqN7jBDUqNQp6/9KUvRfttrbUb3ehGpXbkkUeW2p//+Z+X2s1vfvNSe+UrX1lq9PxCbUPXbC+ImkKEab6h/dBYpXG+1BYWFso1l4aw07n25jwar+liFfSdtNANjf/tttuu1LbccstSo3nnGc94Rqm1xufywx/+sNQ+//nPl9rJJ59cameccUappW2bBkK3lrct/W5J72VLiRaTShcTGbJICM3XhO6x1Ha0KAcdD90T0j7vzVF0//iTP/mT6Dvf8573lBqdHz3/0/U55HmS2iwNgqfP0vFI1zX+pZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0ukHJfouLiyVMjYLZKMBwSEApff7xj398qe23336l9tGPfhS/cxYF3FFw6bwBnmkwMYX40X4ooDMNmR0S2pkaEkw8pslkUto0DSUfElZI29L5U9AzBRNSGGka+Es12m9vrD7zmc8stRe/+MWlRudMY4sWGEgD/YcE6NPn6Tqh79y4cWO8n6U2G2hJYZZ0XjRP9K5Bms9uf/vbl9rjHve4UqNwZQq/pX2nIazp4g6t5UHgtB/abu+99y41Ohf67Je//OVS6wV4Ux/2QlxnLUdQNJm93tO+pO16zwFpmxAaM9Rvb33rW0stDdkdcnznnntuqT3taU8rtc985jOlduMb37jUdt5551L74z/+41L75Cc/WWrpM1trPIZp23ReWo7xO51Oy1xB55AuPNJbbCC9x6fPa+kz4Wte85pS+8QnPlFqd7zjHUvtQQ96UKm11tptbnObUrvZzW4W1Wgc0j2GznlIwDVJF9Sg5yyyEsKjqU3oOhoyt6ZjkO7b9FxC98n0Wk/n1t7iBHTffvjDH15qNDb+7d/+rdSoHai96BipvXuLJM2zcMxyLcYjLTX/UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEka3eCU69mAyDRw+Q/+4A+i7VrjsLiLLrqo1E455ZRSW716dalRWOFDHvKQUtttt91KjQIjH/OYx5RaL6AuDYUmdNwU1pcGXFMfXHPNNbjvtL8o7I/CUTds2ID7WSrT6bQEI1JIaBqy2AvbpHZKQ0vTwEi6HmhcUbAkjaE73OEOpdYah5qnQbeve93rSu2qq64qNWoHGoN0HQ8JyU2DKYdcE0uJxitdwzQWaLxttdVWuJ/nP//5pUaBy9THH/nIR0rtBz/4Qamlx01jncI/qY9626bXHl0/t7vd7XA/s+i4Kcy0FzZPx0Njm9psyAIAS2UymZT2SxcyoTm4dz+k76R+o/Gxfv36UqP5lsYQ7Te9Z/dC+al+xRVXlNqhhx5aavScQ2OIruMzzzyz1IaEElP70LnQfWIljNX/NnvM6TVIaI5pLb9PkzQkncYw7eO73/1uqX3nO98ptX/8x3/E46F+//rXv15qN73pTUtthx12KDU6PxqH6eIevb6iPqD7Oc0ZvQD7sc22C91b0pD4XtA5tR+NI2o72o6Oh+aE9NmaxkZvDv7ABz5QanTcVDv99NNLjdqM7ifpYhq9PqBrLF2AgfbdC4KXrkv8SylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdIODzmdD2yj0jgL3KLSxF4RJIccU4kZheBRmR5994QtfWGoUuEchoxRQ1wuzS8Mcr7766lKjgLs0mDUNFqZ99KR90AsFHxOF8dLxU2AghQ32+pfGNZ0/9RGFQ1J/pIHHaaDz/vvvX2q976TxRoHXdD2RNEyU+qoXdEl1agv6Tjq/lYL6Lg0Cf8pTnoLf+eQnP7nUqP1oLFCALbUzjVfaB40FGv+9YGHaNg2ppnlr5513LjUKa6U+OOmkk0qtF3JMn08DkdNQ76WWzPFpIHyvf6lN6Bqm8Zb2G83/vbDyWWlAb2scLExt8fCHP7zUaPxS7ZJLLik1eh6i4+7Ny7373izqw/Szy4Get+i5c+PGjaWWXqutzfcsnD5bp2N4yD2Vrp/3vve9pfac5zwn2jctdEPjML0X9cZrep9Jw7qXw+yx0RikuWPI75L0uae3yMgs+k1Dcx71eTrf3vKWt8T6dtttV2rU5z/72c9Kbe3atdFn6XogQ8Zq+juB5qq0X6TrmuV/eyBJkiRJkqTfOb6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJoxscdD4bkJeG61GtF9aWBlKnAd+HHHJIqW2//falRiGJxx9/fKlRwG7vXOh4KLiQ9k1hjL1gymS7NAS4NW5vOu80KLwXALtUptNpOd8hwbSz5g0rpBqFQ9LxpLX0+w499NBSa43HMB33S1/6Uvz8LBovFHhK29Ec0AuYTUPNaazSd1JA5xhmj4/6jo6X5phvf/vbuI90/qD9fPazny01CodNw+zTtu+FstJ501igz1Nw6SMe8Qjcz6xzzjmn1M4999xS641XOsd5F1wY03Q6jebNdKGOIWiOGhLgnGyXBtHTOO+1C83NJ554Yqnd5z73wc/PouOmxSbS57Pecaf3zJU0jyboeksXHqHx1lp+/6T2T+d16nc6bhpvvYWFSPrcStv12mcWncs8C8e0lv9OGNIWY5u9ZletWlW2ofBrmo969wtqp3QRJUJ9TsdD21GN7s/HHXcc7puuuyuvvLLUDj744FJL58f0vkPn3GtD+jzNN3Qtr5RQfmlT8y+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNENCjqfTqclGI6C3Sg8jgLcekGYaQgfBSVS7Q1veEO0j4c97GGllgbPUYBza3n44LbbbltqFGZIbUbHQyF61Ia9UGLadp7g5bFNJpPSxxQ4mI7LXnBuGqKZBsbSdrRvCr+k437iE59YanvssUep9fZDYbxf//rXS22rrbYqNTo/Gm80zilYshfG2wuUnkV9fc0110SfXQ5p+DWNt3/+53/G7/za175Wavvvv390PPe73/1K7ac//WmprVu3rtSo7dPrhGqt5YHNFApK30nHSGhuTIOTW8uDjslKDehN5/wh4e3p3JrOFdRvm3oc3O52t8NtP/rRj5bamjVrSi0N7//hD39YajQGqW3oPtFbqIW+87potq/S80rnqN626bVO4zVdMIaeBdJ5gp4de+hZgq6BM888s9QouJpCzdNrlL6vtXwRiXQ/K8H69etLLX3m743zNKA+bZM0jJ2+L/09tO++++K+6bg/9rGPldo3vvGNUqN2pOsu/T1F59Jb3GOLLbYoNXpmoGfUdCEP6bpmZc7CkiRJkiRJ+q3mSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjS6QUHnCwsLJRCTgut6wW6zegGnFIZMoYgUFEffSd930UUXldoZZ5xRanR+FHq35ZZbllprHDRIAacUCkhhlRQomAbvpmHDrfVDpWdRe6+EEL7pdFpCLynMlcYVBZH2AiPTMG8aM2kwahpASbWHPOQhpdY7lw996EOldsQRR5QahYxSn1NoI+2bwibnmVda47E+T5DnUptMJuVYaHykAae9BQwo5J7GMLUfBa7SAg9pqD+hOX3jxo247fbbb19qFLJOx0PXeHo/ufWtb11q6Xzbq6fz/0owmUxKu9B4ozmB5tteaHE6rtN5hvaz9dZbl9pBBx1UanTv2GabbUrtQQ96UKm1xvft9NnimGOOKbUPfOADpXb++eeXGo1fagfabw+1LQVMp+HWy4HGJp3DkIUX0nsJzYU0r9OcQPMWjX8ar/TZ3kIftOjP9a9/fdx21lOe8pRSS/ud5ka67/cWkUgXAkmvveUwew50DdNYpf7tjUlqU/pdki4YQc8GaV/SWD3yyCNLrXcul1xySan9/d//falR+9A1RvcYOsb0nHu/c9PxRue9Up8NpHmtjF9kkiRJkiRJ+p3iSylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjS6QUHnrdXQtjTANg3CbI0D4ChAjgJw99lnn1K77LLLSo2CF3fYYYdSO/TQQ0vtRz/6Uand+973LrXWWlu9enWp3f72ty81Cs2jUMBdd9211Cjg9PWvf32pUZB1L2yPwvmoRv2/EkL4JpNJCSekMUio3Xsh2xRgm4afp2GbFCybhqTvt99+pdYLoj/ttNNKja4xCn2ktu0FqifHQ23YC4ykdqTruxf+vVLMnge1y1VXXfX//FxrPC5b4/mI0Pj6yU9+UmrUJ/P0OwXsPvrRj8bPb7fddqX2yEc+stTOO++86DvTIP2tttqq1O5617uWGl1PreVtRrVewO/YZo+Njovak8Lke/egdMEMmo9o39TnRx11VLRfusbonHtzDPUlLbZCAb3/8A//UGo0P86zwAiFALfG7UhzA+27N18vh+RY5g0Hp3ZJa+liNYQ+S/dtmvt79wkKK6dncAqZvuCCC6J9p4u/0Pn1xjodY3pd0GfTZ8ZNZTKZRM8p9HwzZNEhameaz+g7afzTOErvVbSYzsEHH1xqvevh2c9+dqnRPZ/Oha5vmlupRp9Nx3RrPN+kCx2thLEqLQX/UkqSJEmSJEmj86WUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLpBq+9Np9OS8E8rItCqBLSCRm91hnSVLFpZgr5z2223LTVaEWHLLbcstQc+8IGldpvb3KbUeisf/PjHPy61V77ylaVGq2nc6la3KjVacerOd75zqR1zzDGlds973rPUaJWL1rhtqUZ9RW27HCtDzK5kQcc6ZMUSQqvc0DWRrmZI0u2e/OQnR9tdeOGFWP/whz9canQudM50zdOKVXQu6apnvX6hlVE233zzaLveqlNjo7mVxmu6iiOt0tkaX5v0eaq9853vLLWPf/zjpUZz8IEHHlhqtCITrWzX63eqf/rTny41WiGK5npqW9oHrYbzgx/8oNRodZ3ed1ItvbemK+BuSrPjiK7rrbfeutTS+bI1bhO6TxJaSfelL31ptO90dTn6LF03rbV2xRVXlNrf/u3fltpZZ51VaulKolRLVz3urRBF/UrfSatB9VZVXA6z/UfXTLoyJG3XGrdLer1SO6cr7KZjgc7lAQ94QKm1xs+4dA2cccYZpUbtk14/dM7p6pqt5eN1yCqgY5pOp6UNqE3Suad3XaefT9suXXGa+uexj31stN0Pf/jDUmuttfe///2l1lvZfVa66u3VV19dajQuh/x2oLmBVmGnZ9mVsMK5tBT8SylJkiRJkiSNzpdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdINTfnuhpL/J54YEC1KAHAXAnX/++aVGoXBvfetbS+2b3/xmqVGw+Lp160rtBS94Qam11to//MM/lBqF+FH7UJhjGkZIoeYnnHBCqR166KGl1tt2nuDZlSANE00DY1vjsGZqE2q7eUK2KVjyUY96VKnReDn11FPxOymEmNoiDcmkIEgKjEy/r7cwQrqwAlkpgZGTyaS09TyLA3z3u9/F+mGHHVZqr3rVq0qN2oXa+b73vW+p9cKef9N99PqSrin6TgrcJrSfq666qtS+/OUvR9sNCT2luYmu8eUINZ81nU5LW6ULItB2veuapAHFNM98/etfL7W99tqr1NJFG4aMVapTKD8F9KbHQ2OfAsjp/kTbtcZzUBpQvVLmVkL3Nbq26Bx64zVdNIX6jsYwjQXaNx13Giz+uMc9rtR6n6e2eNe73lVq11xzTanNs3gRtdeQxS9o33Q9pgvKLCVa8CQNuk5/L7TGYyG9t6SL9lBf/PEf/3Gp0ZxO3/fc5z4Xj4eeM2neonFA1xO1N+0jnYN7v5mpv6gt6HpaCWNVWgr+pZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0usFJy7MBa2lgHoXC9cI1KQAuDaROQ/j23XffUttqq61KjQLuXv7yl5fa8ccfX2qtcRA2nR8F7lHAKW1H7XjGGWeU2jvf+c5SO/zww0uttdbe+MY3lhr1NQXzUcjs2KbTaRTcSG1MY60XrJn2G/U51Sigka6dZz3rWaV205veNNrHaaedVmqtcV+m4ed0jVFAI0lDO4cEw9Px0HGnx7jUKDyazjcNFO2Fa1LI/Y1vfONS23bbbUuN5hkKOk/DeNeuXVtql156aanRGGytte23377UbnSjG0XHQ/M6zdUXX3xxqT3gAQ8oNWqbXhgpjXe6xunzaYj8Upsdq3Rt0XnS/NZrJxrD88wpd73rXUuNQuvXrFlTapdcckmp7bnnnqXWe6ahsP0HP/jBpXbyySfj52elocQ0Xqiveve3NFg+DVlejvDz6XQaLTqQ3td630XzdbpYBX3nhg0bSo3Cz2kspMHie+yxB9apP88666xS+9znPldq6X2L9pH+nuiNI/pOmus39cIzm8pkMinHkf4eSu8rreVB9ml/0JxAgeGvfOUrS43GBi1Y9a//+q+l1lr+vJ0uxpMuwJXuo9cH6VxI1/yQRcKk65KV8YQrSZIkSZKk3ym+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaObO9mPghwpHI/C3nqBkRR8R9+ZhkdfcMEFpXaHO9yh1Cg87thjjy214447rtQoJL01DtklFLiXhppTGOlDHvKQUnv2s59dap/97GfxeKi9qUZtRiGbYwfzTSaTMubSgHkaq70QTAorpCDINDCVtqN9H3bYYaVG7f7jH/+41CgEvzUOVLzb3e5WajRmrr322lLrBRjPSsNJh4Q8p+HpdD2l4bSbWrKIBKHxRufaGo8HGkvpvH6DG9yg1C666KJSS+cEGkdDQpjpeqax8MUvfrHU9tprr1KjEPiHPvShpfaOd7yj1Hr9R9dZuqDASgg6n0wm5fpMw5bp+HvtRAtmpM8BdF3TOKKg/ssvvzz67Nve9rZSoxD81vi4d95551JL24eunbRt6Fx6welpIDJZCWO1NX4WoPOiNqVa71kgvQemAfDpGKZ90HxyxBFHlBrNoa3xudCzMIWx03fScaeLmQwJJafvpD6kNkuf1ZfSZDIp82b6jErjpXddpyHzadvT8dzwhjcstZvd7GbRsTz/+c8vtd5viHQRLPp8Gn5O26XPielzcGu8kAe19zyLKkgr2cp4apAkSZIkSdLvFF9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0Q0KOqfASArXo+A6Ci2lz7bGoXIUFkefpzC7r33ta6V2n/vcp9S++93vltqf/umfRvvohczRMVKNwgwpPP2FL3xhqd3udrcrtX333bfU/uVf/qXUXvziF5daa9wH1K8rJcx01uLiYmnT1atXl+3SAPZesGYamJmGUFLA6JZbbhl9H332xBNPLDUKU2yNQ3op1P8mN7lJqdH1TYHOadgkjbVewC7V5wmYXQ6TyaQcc7qQAxmyiARJQ9bXrl1bahQyms4d6QILve+ka4o+T3NBGhT95S9/GY8nlQbvzhMyvZSm02lpZzouCoylOao3H9Hn6T578MEHl9pjH/vYUqNAfwq8pzG07bbbltqd7nSn6LOtcf/e/OY3L7V0gYbe9T2LrrF0vm0tD9ml8+61xXKYba90MZ557xnpMyq1Fd33ad90Tey+++6l9uQnP7nUetfeG97whlL78Ic/XGqbb755qdHzD40Zuk+kn+09d1K/0ufT++jYptNp6c/e76REr53S5/b0GZWO8TnPeU6p0Tx4xRVXlNrHP/7xUkufSVrj6ySdt+iz6SIIa9asKTVqr9b4+ThdyMBQc/22WplvFCRJkiRJkvRbzZdSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdIOCzgkFrlF4HIWW9oLrKGguDXik7SjA+fvf/36p7bzzzqX2sY99rNRe/epXl9q73vWuUmuNz/FGN7pRqT3kIQ8ptfvf//6lRoGr1LYUav7Xf/3XpUbBkkPQ+a2EMN6FhYUSJEiBg3T8FDbYC0RPg1A3btxYahRguGrVqlI777zzov2Sc889t9QojLG11v7mb/6m1CgcksYMXfN0znTcaaBxGtLdOx4KVk0Dg5faZDIp7UBtkIZ99rbbsGFDqaVjido/De0lQ0LNCR03Be/SdjvssEOpUfgtjZnb3/72pUbXWW+80phLQ9ZXwnidTCZlXqA2puufDFksg9r0Jz/5SantsccepfaBD3yg1I4++uhSo+M+5JBDSo3GUM+6detK7RnPeEb02fR+SmMjvRa32GILrKfzMB3jSno2mD0Wapc0ULp3XdNcSOdLcwrdZ+l5heYEWhCHQsnp2eIb3/hGqbXW2jHHHFNqdNxpv9PYpFo6F/TGEfVruuAKPYekC+FsSsk9MF1YpiddCIb6nNr4L/7iL0rtoQ99aLTfu9zlLqVG8yX1T+8Y098184Tg0/HQfnv9mV7f1GYrZaxKm5p/KSVJkiRJkqTR+VJKkiRJkiRJo/OllCRJkiRJkkbnSylJkiRJkiSNblDQ+XQ6jULgKIQtDTrsoRA+CnajsEo6HgoRP+OMM0rt+te/fqkdddRRpfbCF76w1FrjkLorrrii1Ch8ev369aX2hje8odRe97rXlRoFIVKth4Ik035dKWaPbZ5gwl6III03+jyNA2q79BgpmJa+7z3veU+pnX/++aXWWms77rhjqV188cW47ax5xgYFZw6ZL9KwSqoNCVleStPptPQzhQxTmCltNyQUNA39pf3QdZGOf+o3Ohbab2scak7onCn0l44xHUfULxRk2hqPY2pvOp40PHypzbZBGlqfBrq3xuODfOYznym1j3zkI6VGC31Q0Dm1exr4fdZZZ+ExHnzwwaWWjn9C+6YxRPcOurf37m9DFh5IPrscYbzT6bTsN7226LrujUvaltqarv90EQ+6Vg499NBSu9WtblVqdH5HHHEE7oeOh8YS3WfonpqeM+2DtuvNrdQH6aInacD1UppMJuXY6LjSUP4h1y/th34z0G+iE044IdrHM5/5zFK78MILS436p/cckC6okAbwp4H+VEvvHa3xPELXE31nep+QrmtWxi8ySZIkSZIk/U7xpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRrdoKDzyWRSQgwp7C0NFuyFtVH4HAVk0ndSmB2FJ37rW98qtV133RWPZ9a2225bar0wu1122aXUzj333FJLQ3Yp4JGCJak2JGSW2jbtg5UQGEmoj9Kg6yGB2DR+KdSQtqNjvP3tb19q++yzT6nd/e53L7Vb3/rWpbbvvvuWWmutnXzyyaX2gQ98oNToeqLjpjBOGhtpoHFvXKWB4NTeK2mszl6fFHpJxztkbk0/T31M/ZmG2VMf0THSWKDj6+2bzm/dunWl9oMf/KDUbnSjG5XaAQccUGoXXHBBqV1zzTXRsbTG50OBstTeVFuO8OjZsUr9S32ZBsa2xve6NOD3m9/8Zqml83I6Lo8//vhS+/u//3s8nrVr15ZaGrybnnN6fumzRmvcFjQvpYtNLIeFhYVyP6DQ79WrV5cajcE0lLy1PKSargGag29729uW2gte8IJSo7Fw0EEHldo555xTar3jSeceap+0zdKQaVqoorV8zNFxr4RFTyiUn46LFrqh8dI7p3mehe51r3uVGvXHi170olJ74xvfWGrU5+lvmtZ4jqK5Nd2O2jFdlChdEKM1HoPpc3T6zCxd1yz/LCxJkiRJkqTfOb6UkiRJkiRJ0uh8KSVJkiRJkqTR+VJKkiRJkiRJoxuUlra4uFiCHynsjQLp0pC51jjsjbZNQ7/TID0KVKQwussvvzzarrXW1q9fX2ppmCkFD26++ealloZIpu3aQ/tZ6eHR/1Pa7mnocGs8Zqid6DqhAGYalxdeeGGpXXbZZaX2yU9+MjqWK6+8stRaa22rrbYqtTREmc6Prs80yJqCo3vjvBeEPYvOZcj4X0qLi4vlnCmsfUhIcYqCPald0rB++r40wDYNTm+N57P0fvTe97631HbaaadS+8lPflJqdNxDglmpD9P26S2osdzmCQzvofFPY4u8/vWvLzVadGSPPfYoNRob73znO0stDSBvjcd1OtZpOxob6UIm84bkUh/ME2691BYXF8sxU6g5hZ9Tm/buibQtzQHpogAUHk2h5rSP73//+6VGoea9BTGo79Jxky4okz5P0na9PqCxSfMI3SfouW9sk8mkjKN0zqNrvfd8kz6b0efpOqHvO/HEE0ttnkUSeueSLqyVBvXTMdJ2dIzpYhO9On1+yBwkXdf5l1KSJEmSJEkanS+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNENWoZlYWGhrGRBq2qkKwjQCjK976TPp6vSpKslpavvpavctMarQNBqEemKb+kqPunqPL2VIegY0xVUVsqqO7PHm64UmK4K0xqvNEMrJNLqJOmKW+nqG+kKWNtss02ptcbnQjVauShd2ZGOkdqG+qC3mhntm66dLbbYotQ2bNiA3zm2yWRS2ob6Lp3LaAWZ1njlHPo89VO6Ak06rtN5vje30jmm8+MxxxyD3zkrnd/SlXha4/ah76Q2Wwkrm06n0zI20zGYjunWeAVO+k66/qmNDzvssFKjlblov/R9Q/o8XbksXfkpnfPSY+zNremqkkNWSFsOs+2Q3q9ofPRWrEtX4p3n+t9///1LjfrzlFNOwWOc1VtBkvqOnlHpntobS7PSe/yQlbupv6h96DvTa28pTafTMlbT3wHpb43W8lU5aQx+5CMfKbV//ud/jj5L+6Xt6Fx6fZH+/klX6037PO2XnnSlyfT+Jv028C+lJEmSJEmSNDpfSkmSJEmSJGl0vpSSJEmSJEnS6HwpJUmSJEmSpNENCjqfTqclQC4NsE3DBnso2DANSkwDKCm0Mf0shVu3xqGpFBSdhgin7U2fpVA/Cplujc+RPv/zn/8cP78SpeMyDRtvjfs9DeukYMk0RDkNREwDVHt1Oj86l/T8SK9tZ/XG6pBQzFm9thjbZDIp7ZD2MY0PCqVtbb5FJEg6L1N/UFAo9UdvHFFb0HFT+G0aGJ4u2pCeSw/dJ+i4V61aVWoUXr/UZq93ugbp3kDb9UJbqe3pO2mOogUMaDv6PrrPpYug9MbLPItspAtGpItfpIvB9Oq9RRRmpYHXY5g9lnQBG+qP3nxEY47mBdo3tRWN1yOPPLLUXve615XaySefjMc4qzd30DNqGh6dLlZBbUPfR3rjNV2khvpwJSwikS54QtK+aC2/J9L9hubMee7vdIz0rNcL5afPp9dy+n10faa/fXrb9c5n1kp5RpXGsHKeGiRJkiRJkvQ7w5dSkiRJkiRJGp0vpSRJkiRJkjQ6X0pJkiRJkiRpdIOCzieTSRRemYbCDQk4TUPq0qDQ9BjTcOwhgYIUakuhd/O0QxpG2kPnc10KNZ9OpyUgktqOxmAaNt7bNg39TsO4abxRLR0vvf2mQbkUqJmG2lJoI419asNe4CMddxrUnZ7fUltcXCzzAs0dJJ3zetvSNUBB6fMsLJHOo2TeANp0EQkaC2kYb3qNtjZf2PxKmYNnzyENGKaxMWTBE2o7up/SmE7DlqmN01Dy3jNNOt7SewK1Q/oMQSHCveeK9J5CfZAudLHUaBEJej6ioHLSm1vT50waI7Qd9dP73//+qEbjjfbR6yO6JudZECMd62l7DblHp/eo9NpbarNtSvNReq8acg9KF5ZI58J5xmC6oEtr+SIb8zy/zBOW37vG0gWR0sUSpN8G/qWUJEmSJEmSRudLKUmSJEmSJI3Ol1KSJEmSJEkanS+lJEmSJEmSNLrBSZSz4WwUCke1NBC6NQ6G27hxY6mtWrXq/3l8rXFIHQVzpuGhFMLXC79MQ3rTcL15pOF/rXF4YC9oOvnskDDbTWW2/agvqN/SIMch3zlPWHka7kvhrXR8vTDeNHA8DdRNAzppOxprQ8L76bjTzy9H0DmhNqV2oTYdMh9RqDl9fp4+ploapkuBv63lcz3V5gkwT6/v3nyZhvmmQdrzBsH/JmbPYZ75bUgQLI2FdN4i1HbUb9Tn897T5pln0oVIevPArN79jT5P957NN988+s7lCOhdXFwsbUPHQQtL0Dn0+o3GO+2HrmF6vqXjSZ8J02und09M52bajsYCbZc+R9N2vbmV5gKSBn2vBNSeNK6GLPiSLkJA26Wh5ukiCemc3jvmdN6jcZSGiKf3Mrpme/MFzQPp/T1dWEi6rnFkS5IkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaObDAnTnkwml7bWzlu6w9Fvsd2n0+n2Y+3Msao5jDpWW3O8ai7OrbqucG7VdYlzq64rRp9bpU1t0EspSZIkSZIkaVPw3/ckSZIkSZI0Ol9KSZIkSZIkaXS+lJIkSZIkSdLofCklSZIkSZKk0flSSpIkSZIkSaPzpZQkSZIkSZJG50spSZIkSZIkjc6XUpIkSZIkSRqdL6UkSZIkSZI0uv8PSjbqIsXGaU8AAAAASUVORK5CYII=\n", 353 | "text/plain": [ 354 | "
" 355 | ] 356 | }, 357 | "metadata": {}, 358 | "output_type": "display_data" 359 | } 360 | ], 361 | "source": [ 362 | "nrows, ncols = 3, 7\n", 363 | "plt.figure(figsize=(3 * ncols, 3 * nrows))\n", 364 | "for i in range(samples.shape[0]):\n", 365 | " plt.subplot(nrows, ncols, i + 1)\n", 366 | " plt.imshow(1 - samples[i].detach().cpu().numpy(), cmap=\"Greys\")\n", 367 | " plt.xticks([])\n", 368 | " plt.yticks([])" 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": null, 374 | "id": "466f2945", 375 | "metadata": {}, 376 | "outputs": [], 377 | "source": [] 378 | } 379 | ], 380 | "metadata": { 381 | "kernelspec": { 382 | "display_name": "Environment (conda_py39)", 383 | "language": "python", 384 | "name": "conda_py39" 385 | }, 386 | "language_info": { 387 | "codemirror_mode": { 388 | "name": "ipython", 389 | "version": 3 390 | }, 391 | "file_extension": ".py", 392 | "mimetype": "text/x-python", 393 | "name": "python", 394 | "nbconvert_exporter": "python", 395 | "pygments_lexer": "ipython3", 396 | "version": "3.9.7" 397 | } 398 | }, 399 | "nbformat": 4, 400 | "nbformat_minor": 5 401 | } 402 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Muhammad Firmansyah Kasim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Score-based generative model tutorial 2 | 3 | A short tutorial for score-based generative modelling. 4 | This repository contains the jupyter notebooks accompanying the blog posts [here](https://mfkasim1.github.io/2022/07/01/sgm-1/) and [here](https://mfkasim1.github.io/2022/07/04/sgm-2/). 5 | 6 | ![Sampling example](figs/sampling-swiss-sde.gif) 7 | 8 | ![Generated MNIST](figs/generated-mnist.png) 9 | -------------------------------------------------------------------------------- /figs/generated-mnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mfkasim1/score-based-tutorial/08e238826ce3ca63edb4303253cc7828099ab1b4/figs/generated-mnist.png -------------------------------------------------------------------------------- /figs/sampling-swiss-sde.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mfkasim1/score-based-tutorial/08e238826ce3ca63edb4303253cc7828099ab1b4/figs/sampling-swiss-sde.gif -------------------------------------------------------------------------------- /viscode/01-sample-anim.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from sklearn.datasets import make_swiss_roll 3 | 4 | 5 | # generate the swiss roll dataset 6 | xnp, _ = make_swiss_roll(1000, noise=1.0) 7 | xtns = torch.as_tensor(xnp[:, [0, 2]] / 10.0, dtype=torch.float32) 8 | dset = torch.utils.data.TensorDataset(xtns) 9 | 10 | # score_network takes input of 2 dimension and returns the output of the same size 11 | score_network = torch.nn.Sequential( 12 | torch.nn.Linear(2, 64), 13 | torch.nn.LogSigmoid(), 14 | torch.nn.Linear(64, 64), 15 | torch.nn.LogSigmoid(), 16 | torch.nn.Linear(64, 64), 17 | torch.nn.LogSigmoid(), 18 | torch.nn.Linear(64, 2), 19 | ) 20 | 21 | from functorch import jacrev, vmap 22 | 23 | def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: 24 | # x: (batch_size, 2) is the training data 25 | score = score_network(x) # score: (batch_size, 2) 26 | 27 | # first term: half of the squared norm 28 | term1 = torch.linalg.norm(score, dim=-1) ** 2 * 0.5 29 | 30 | # second term: trace of the Jacobian 31 | jac = vmap(jacrev(score_network))(x) # (batch_size, 2, 2) 32 | term2 = torch.einsum("bii->b", jac) 33 | return (term1 + term2).mean() 34 | 35 | # start the training loop 36 | from tqdm import tqdm 37 | device = torch.device("cuda:0") 38 | score_network = score_network.to(device) 39 | opt = torch.optim.Adam(score_network.parameters(), lr=3e-4) 40 | dloader = torch.utils.data.DataLoader(dset, batch_size=32, shuffle=True) 41 | for i_epoch in tqdm(range(5000)): 42 | for data, in dloader: 43 | data = data.to(device) 44 | # training step 45 | opt.zero_grad() 46 | loss = calc_loss(score_network, data) 47 | loss.backward() 48 | opt.step() 49 | 50 | def generate_samples(score_net: torch.nn.Module, nsamples: int, eps: float = 0.001, nsteps: int = 1000): 51 | # generate samples using Langevin MCMC 52 | # x0: (sample_size, nch) 53 | x0 = torch.rand((nsamples, 2)) * 2 - 1 54 | xs = [x0] 55 | for i in range(nsteps): 56 | z = torch.randn_like(x0) 57 | x0 = x0 + eps * score_net(x0) + (2 * eps) ** 0.5 * z 58 | xs.append(x0) 59 | return xs 60 | 61 | score_network = score_network.to(torch.device("cpu")) 62 | samples = generate_samples(score_network, 1000) 63 | 64 | from celluloid import Camera 65 | import matplotlib.pyplot as plt 66 | fig = plt.figure() 67 | camera = Camera(fig) 68 | for i, sample in enumerate(samples): 69 | if i < 30: # and i % 3 == 0: 70 | # if i % 10 == 0: 71 | plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 72 | camera.snap() 73 | 74 | animation = camera.animate() 75 | animation.save('01-animation-sampling.mp4') 76 | -------------------------------------------------------------------------------- /viscode/02-sample-right.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # generate the swiss roll dataset 4 | xtns = torch.randn((1000, 2)) * 0.2 + 0.3 5 | dset = torch.utils.data.TensorDataset(xtns) 6 | 7 | # score_network takes input of 2 dimension and returns the output of the same size 8 | class ScoreNet(torch.nn.Module): 9 | def forward(self, x: torch.Tensor) -> torch.Tensor: 10 | # x: (..., 2) 11 | v = -(x - 0.3) / 0.2 ** 2 12 | return v 13 | 14 | score_network = ScoreNet() 15 | 16 | from functorch import jacrev, vmap 17 | 18 | def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: 19 | # x: (batch_size, 2) is the training data 20 | score = score_network(x) # score: (batch_size, 2) 21 | # first term: half of the squared norm 22 | term1 = torch.linalg.norm(score, dim=-1) ** 2 * 0.5 23 | # second term: trace of the Jacobian 24 | jac = vmap(jacrev(score_network))(x) # (batch_size, 2, 2) 25 | term2 = torch.einsum("bii->b", jac) 26 | return (term1 + term2).mean() 27 | 28 | from tqdm import tqdm 29 | 30 | def generate_samples(score_net: torch.nn.Module, nsamples: int, eps: float = 0.001, nsteps: int = 1000): 31 | # generate samples using Langevin MCMC 32 | # x0: (sample_size, nch) 33 | x0 = torch.rand((nsamples, 2)) * 2 - 1 34 | xs = [x0] 35 | for i in range(nsteps): 36 | z = torch.randn_like(x0) 37 | x0 = x0 + eps * score_net(x0) + (2 * eps) ** 0.5 * z 38 | xs.append(x0) 39 | return xs 40 | 41 | score_network = score_network.to(torch.device("cpu")) 42 | samples = generate_samples(score_network, 1000) 43 | 44 | from celluloid import Camera 45 | import matplotlib.pyplot as plt 46 | fig = plt.figure() 47 | camera = Camera(fig) 48 | ns = 100 49 | xraw, yraw = torch.linspace(-1, 1, ns), torch.linspace(-1, 1, ns) 50 | x, y = torch.meshgrid(xraw, yraw, indexing="xy") 51 | xy = torch.stack((x, y), dim=-1).reshape(-1, 2) 52 | uv = score_network(xy).reshape((ns, ns, 2)).detach() 53 | true_logp = torch.exp(-torch.sum((xy - 0.3) ** 2, dim=-1) / (2 * 0.2 ** 2)).reshape((ns, ns)) 54 | for i, sample in tqdm(enumerate(samples)): 55 | if i < 100: 56 | _ = plt.imshow(true_logp.detach().numpy(), extent=(-1, 1, 1, -1), cmap="Oranges") 57 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 58 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 59 | plt.xlim(-1, 1) 60 | plt.ylim(-1, 1) 61 | camera.snap() 62 | 63 | animation = camera.animate() 64 | animation.save('02-animation-sampling-wrong.mp4') 65 | -------------------------------------------------------------------------------- /viscode/02-sample-sde-anim-swiss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from sklearn.datasets import make_swiss_roll 3 | 4 | # generate the swiss roll dataset 5 | xnp, _ = make_swiss_roll(1000, noise=1.0) 6 | xtns = torch.as_tensor(xnp[:, [0, 2]] / 10.0, dtype=torch.float32) 7 | dset = torch.utils.data.TensorDataset(xtns) 8 | 9 | class ScoreNet(torch.nn.Module): 10 | def __init__(self): 11 | super().__init__() 12 | self.logp_network = torch.nn.Sequential( 13 | torch.nn.Linear(3, 64), 14 | torch.nn.LogSigmoid(), 15 | torch.nn.Linear(64, 64), 16 | torch.nn.LogSigmoid(), 17 | torch.nn.Linear(64, 64), 18 | torch.nn.LogSigmoid(), 19 | torch.nn.Linear(64, 1), 20 | ) 21 | def logp(self, x, t): 22 | xt = torch.cat((x, t), dim=-1) 23 | logp = self.logp_network(xt) 24 | return logp 25 | def forward(self, x, t): 26 | x = x.requires_grad_() 27 | with torch.enable_grad(): 28 | logp = self.logp(x, t) 29 | score = torch.autograd.grad(logp, x, grad_outputs=torch.ones_like(logp), create_graph=torch.is_grad_enabled(), retain_graph=True)[0] 30 | return score 31 | 32 | score_network = ScoreNet() 33 | 34 | def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: 35 | # x: (batch_size, 2) is the training data 36 | # sample the time 37 | t = torch.rand((x.shape[0], 1), dtype=x.dtype, device=x.device) * (1 - 1e-4) + 1e-4 38 | # calculate the terms for the posterior log distribution 39 | int_beta = (0.1 + 0.5 * (20 - 0.1) * t) * t # integral of beta 40 | mu_t = x * torch.exp(-0.5 * int_beta) 41 | var_t = -torch.expm1(-int_beta) 42 | x_t = torch.randn_like(x) * var_t ** 0.5 + mu_t 43 | grad_log_p = -(x_t - mu_t) / var_t # (batch_size, 2) 44 | # calculate the score function 45 | score = score_network(x_t, t) # score: (batch_size, 2) 46 | # calculate the loss function 47 | loss = (score - grad_log_p) ** 2 48 | lmbda_t = var_t 49 | weighted_loss = lmbda_t * loss 50 | return torch.mean(weighted_loss) 51 | 52 | # start the training loop 53 | from tqdm import tqdm 54 | opt = torch.optim.Adam(score_network.parameters(), lr=3e-4) 55 | dloader = torch.utils.data.DataLoader(dset, batch_size=256, shuffle=True) 56 | for i_epoch in tqdm(range(150000)): 57 | for data, in dloader: 58 | # training step 59 | opt.zero_grad() 60 | loss = calc_loss(score_network, data) 61 | loss.backward() 62 | opt.step() 63 | 64 | from typing import List 65 | 66 | def generate_samples(score_network: torch.nn.Module, nsamples: int) -> List[torch.Tensor]: 67 | x_t = torch.randn((nsamples, 2)) # (nsamples, 2) 68 | time_pts = torch.linspace(1, 0, 1000) # (ntime_pts,) 69 | res = [x_t] 70 | beta = lambda t: 0.1 + (20 - 0.1) * t 71 | for i in range(len(time_pts) - 1): 72 | t = time_pts[i] 73 | dt = time_pts[i + 1] - t 74 | # calculate the drift and diffusion terms 75 | fxt = -0.5 * beta(t) * x_t 76 | gt = beta(t) ** 0.5 77 | score = score_network(x_t, t.expand(x_t.shape[0], 1)).detach() 78 | drift = fxt - gt * gt * score 79 | diffusion = gt 80 | # euler-maruyama step 81 | x_t = x_t + drift * dt + diffusion * torch.randn_like(x_t) * torch.abs(dt) ** 0.5 82 | res.append(x_t) 83 | return res 84 | 85 | samples = generate_samples(score_network, 1000) 86 | 87 | from celluloid import Camera 88 | import matplotlib.pyplot as plt 89 | fig = plt.figure() 90 | camera = Camera(fig) 91 | ns = 100 92 | xraw, yraw = torch.linspace(-1.6, 1.6, ns), torch.linspace(-1.6, 1.6, ns) 93 | x, y = torch.meshgrid(xraw, yraw, indexing="xy") 94 | xy = torch.stack((x, y), dim=-1).reshape(-1, 2) 95 | time_pts = torch.linspace(1, 0, 1000) # (ntime_pts,) 96 | for i, sample in tqdm(enumerate(samples)): 97 | if (i > 500 and i % 10 == 0) or (i > 800 and i % 5 == 0): 98 | t = time_pts[i].expand((xy.shape[0], 1)) 99 | uv = score_network(xy, t).reshape((ns, ns, 2)).detach() 100 | logp = score_network.logp(xy, t).reshape((ns, ns)).detach() 101 | _ = plt.imshow(torch.exp(logp).detach().numpy(), extent=(-1.6, 1.6, 1.6, -1.6), cmap="Oranges") 102 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 103 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 104 | _ = plt.xlim(-1.6, 1.6) 105 | _ = plt.ylim(-1.6, 1.6) 106 | _ = camera.snap() 107 | 108 | for j in range(10): 109 | _ = plt.imshow(torch.exp(logp).detach().numpy(), extent=(-1.6, 1.6, 1.6, -1.6), cmap="Oranges") 110 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 111 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 112 | _ = plt.xlim(-1.6, 1.6) 113 | _ = plt.ylim(-1.6, 1.6) 114 | _ = camera.snap() 115 | 116 | animation = camera.animate() 117 | animation.save('02-animation-sampling-sde.mp4') 118 | -------------------------------------------------------------------------------- /viscode/02-sample-sde-anim.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # generate the dataset 4 | xtns = torch.randn((1000, 2)) * 0.2 + 0.3 5 | dset = torch.utils.data.TensorDataset(xtns) 6 | 7 | class Sine(torch.nn.Module): 8 | def forward(self, x): 9 | return torch.sin(2 * x) 10 | 11 | class ScoreNet(torch.nn.Module): 12 | def __init__(self): 13 | super().__init__() 14 | self.logp_network = torch.nn.Sequential( 15 | torch.nn.Linear(3, 64), 16 | Sine(), 17 | torch.nn.Linear(64, 64), 18 | torch.nn.LogSigmoid(), 19 | torch.nn.Linear(64, 64), 20 | torch.nn.LogSigmoid(), 21 | torch.nn.Linear(64, 1), 22 | ) 23 | def logp(self, x, t): 24 | xt = torch.cat((x, t), dim=-1) 25 | logp = self.logp_network(xt) 26 | return logp 27 | def forward(self, x, t): 28 | x = x.requires_grad_() 29 | with torch.enable_grad(): 30 | logp = self.logp(x, t) 31 | score = torch.autograd.grad(logp, x, grad_outputs=torch.ones_like(logp), create_graph=torch.is_grad_enabled(), retain_graph=True)[0] 32 | return score 33 | 34 | score_network = ScoreNet() 35 | 36 | def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: 37 | # x: (batch_size, 2) is the training data 38 | # sample the time 39 | t = torch.rand((x.shape[0], 1), dtype=x.dtype, device=x.device) * (1 - 1e-4) + 1e-4 40 | # calculate the terms for the posterior log distribution 41 | int_beta = (0.1 + 0.5 * (20 - 0.1) * t) * t # integral of beta 42 | mu_t = x * torch.exp(-0.5 * int_beta) 43 | var_t = -torch.expm1(-int_beta) 44 | x_t = torch.randn_like(x) * var_t ** 0.5 + mu_t 45 | grad_log_p = -(x_t - mu_t) / var_t # (batch_size, 2) 46 | # calculate the score function 47 | score = score_network(x_t, t) # score: (batch_size, 2) 48 | # calculate the loss function 49 | loss = (score - grad_log_p) ** 2 50 | lmbda_t = var_t 51 | weighted_loss = lmbda_t * loss 52 | return torch.mean(weighted_loss) 53 | 54 | # start the training loop 55 | from tqdm import tqdm 56 | opt = torch.optim.Adam(score_network.parameters(), lr=3e-4) 57 | dloader = torch.utils.data.DataLoader(dset, batch_size=256, shuffle=True) 58 | for i_epoch in tqdm(range(150000)): 59 | for data, in dloader: 60 | # training step 61 | opt.zero_grad() 62 | loss = calc_loss(score_network, data) 63 | loss.backward() 64 | opt.step() 65 | 66 | from typing import List 67 | 68 | def generate_samples(score_network: torch.nn.Module, nsamples: int) -> List[torch.Tensor]: 69 | x_t = torch.randn((nsamples, 2)) # (nsamples, 2) 70 | time_pts = torch.linspace(1, 0, 1000) # (ntime_pts,) 71 | res = [x_t] 72 | beta = lambda t: 0.1 + (20 - 0.1) * t 73 | for i in range(len(time_pts) - 1): 74 | t = time_pts[i] 75 | dt = time_pts[i + 1] - t 76 | # calculate the drift and diffusion terms 77 | fxt = -0.5 * beta(t) * x_t 78 | gt = beta(t) ** 0.5 79 | score = score_network(x_t, t.expand(x_t.shape[0], 1)).detach() 80 | drift = fxt - gt * gt * score 81 | diffusion = gt 82 | # euler-maruyama step 83 | x_t = x_t + drift * dt + diffusion * torch.randn_like(x_t) * torch.abs(dt) ** 0.5 84 | res.append(x_t) 85 | return res 86 | 87 | samples = generate_samples(score_network, 1000) 88 | 89 | from celluloid import Camera 90 | import matplotlib.pyplot as plt 91 | fig = plt.figure() 92 | camera = Camera(fig) 93 | ns = 100 94 | bound = 2.0 95 | xraw, yraw = torch.linspace(-bound, bound, ns), torch.linspace(-bound, bound, ns) 96 | x, y = torch.meshgrid(xraw, yraw, indexing="xy") 97 | xy = torch.stack((x, y), dim=-1).reshape(-1, 2) 98 | time_pts = torch.linspace(1, 0, 1000) # (ntime_pts,) 99 | for i, sample in tqdm(enumerate(samples)): 100 | if (i > 500 and i % 10 == 0) or (i > 800 and i % 5 == 0): 101 | t = time_pts[i].expand((xy.shape[0], 1)) 102 | uv = score_network(xy, t).reshape((ns, ns, 2)).detach() 103 | logp = score_network.logp(xy, t).reshape((ns, ns)).detach() 104 | _ = plt.imshow(torch.exp(logp).detach().numpy(), extent=(-bound, bound, bound, -bound), cmap="Oranges") 105 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 106 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 107 | _ = plt.xlim(-bound, bound) 108 | _ = plt.ylim(-bound, bound) 109 | _ = camera.snap() 110 | 111 | for j in range(10): 112 | _ = plt.imshow(torch.exp(logp).detach().numpy(), extent=(-bound, bound, bound, -bound), cmap="Oranges") 113 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 114 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 115 | _ = plt.xlim(-bound, bound) 116 | _ = plt.ylim(-bound, bound) 117 | _ = camera.snap() 118 | 119 | animation = camera.animate() 120 | animation.save('02-animation-sampling-anim-sde.mp4') 121 | -------------------------------------------------------------------------------- /viscode/02-sample-wrong.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # generate the swiss roll dataset 4 | xtns = torch.randn((1000, 2)) * 0.2 + 0.3 5 | dset = torch.utils.data.TensorDataset(xtns) 6 | 7 | class Sine(torch.nn.Module): 8 | def forward(self, x): 9 | return torch.sin(2 * x) 10 | 11 | # score_network takes input of 2 dimension and returns the output of the same size 12 | score_network = torch.nn.Sequential( 13 | torch.nn.Linear(2, 64), 14 | Sine(), 15 | torch.nn.Linear(64, 64), 16 | Sine(), 17 | torch.nn.Linear(64, 64), 18 | torch.nn.LogSigmoid(), 19 | torch.nn.Linear(64, 2), 20 | ) 21 | 22 | from functorch import jacrev, vmap 23 | 24 | def calc_loss(score_network: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: 25 | # x: (batch_size, 2) is the training data 26 | score = score_network(x) # score: (batch_size, 2) 27 | # first term: half of the squared norm 28 | term1 = torch.linalg.norm(score, dim=-1) ** 2 * 0.5 29 | # second term: trace of the Jacobian 30 | jac = vmap(jacrev(score_network))(x) # (batch_size, 2, 2) 31 | term2 = torch.einsum("bii->b", jac) 32 | return (term1 + term2).mean() 33 | 34 | # start the training loop 35 | from tqdm import tqdm 36 | device = torch.device("cuda:0") 37 | score_network = score_network.to(device) 38 | opt = torch.optim.Adam(score_network.parameters(), lr=3e-4) 39 | dloader = torch.utils.data.DataLoader(dset, batch_size=32, shuffle=True) 40 | for i_epoch in tqdm(range(500)): 41 | for data, in dloader: 42 | data = data.to(device) 43 | # training step 44 | opt.zero_grad() 45 | loss = calc_loss(score_network, data) 46 | loss.backward() 47 | opt.step() 48 | 49 | def generate_samples(score_net: torch.nn.Module, nsamples: int, eps: float = 0.001, nsteps: int = 1000): 50 | # generate samples using Langevin MCMC 51 | # x0: (sample_size, nch) 52 | x0 = torch.rand((nsamples, 2)) * 2 - 1 53 | xs = [x0] 54 | for i in range(nsteps): 55 | z = torch.randn_like(x0) 56 | x0 = x0 + eps * score_net(x0) + (2 * eps) ** 0.5 * z 57 | xs.append(x0) 58 | return xs 59 | 60 | score_network = score_network.to(torch.device("cpu")) 61 | samples = generate_samples(score_network, 1000) 62 | 63 | from celluloid import Camera 64 | import matplotlib.pyplot as plt 65 | fig = plt.figure() 66 | camera = Camera(fig) 67 | ns = 100 68 | xraw, yraw = torch.linspace(-1, 1, ns), torch.linspace(-1, 1, ns) 69 | x, y = torch.meshgrid(xraw, yraw, indexing="xy") 70 | xy = torch.stack((x, y), dim=-1).reshape(-1, 2) 71 | uv = score_network(xy).reshape((ns, ns, 2)).detach() 72 | true_logp = torch.exp(-torch.sum((xy - 0.3) ** 2, dim=-1) / (2 * 0.2 ** 2)).reshape((ns, ns)) 73 | for i, sample in tqdm(enumerate(samples)): 74 | if i < 100: 75 | _ = plt.imshow(true_logp.detach().numpy(), extent=(-1, 1, 1, -1), cmap="Oranges") 76 | _ = plt.streamplot(xraw.numpy(), yraw.numpy(), uv[..., 0].numpy(), uv[..., 1].numpy(), color='C0') 77 | _ = plt.plot(sample[:, 0].detach().cpu().numpy(), sample[:, 1].detach().cpu().numpy(), 'C1.') 78 | plt.xlim(-1, 1) 79 | plt.ylim(-1, 1) 80 | camera.snap() 81 | 82 | animation = camera.animate() 83 | animation.save('02-animation-sampling-wrong.mp4') 84 | --------------------------------------------------------------------------------