├── .gitignore ├── README.md ├── assets ├── lorenz_trajectories.png └── pendulum_train.gif ├── setup.py ├── tests ├── __init__.py ├── ghnn_test.py └── ghnn_test │ └── pendulum_test_data.pkl ├── tutorial ├── ghnn_tutorial.ipynb └── weakform_tutorial.ipynb └── weakformghnn ├── __init__.py └── _src ├── __init__.py ├── _models.py ├── _vector_calc.py └── _weak_form.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # mac misc files 132 | *.DS_Store 133 | 134 | # vscode setup 135 | *.vscode -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Weak Form Generalized Hamiltonian Learning 2 | This library provides a PyTorch implementation for performing Weak Form Generalized Hamiltonian Learning. This code accompanies this [Neurips2020 paper](https://proceedings.neurips.cc/paper/2020/file/d93c96e6a23fff65b91b900aaa541998-Paper.pdf) [1] by Kevin L. Course, Trefor W. Evans, and Prasanth B. Nair. 3 | 4 | As everything is written in PyTorch, all algorithms provide full GPU support. 5 | 6 | --- 7 | 8 |

9 | Lorenz63 gen hamiltonian neural net 10 |

11 | 12 | 13 | Please cite our paper if you find this code useful in your research. The bibliographic information for the paper is, 14 | ```bash 15 | @inproceedings{course_wfghnn_2020, 16 | author = {Course, Kevin and Evans, Trefor and Nair, Prasanth}, 17 | booktitle = {Advances in Neural Information Processing Systems}, 18 | editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin}, 19 | pages = {18716--18726}, 20 | publisher = {Curran Associates, Inc.}, 21 | title = {Weak Form Generalized Hamiltonian Learning}, 22 | url = {https://proceedings.neurips.cc/paper/2020/file/d93c96e6a23fff65b91b900aaa541998-Paper.pdf}, 23 | volume = {33}, 24 | year = {2020} 25 | } 26 | ``` 27 | Our experiments make heavy use of Chen *et. al's* [torchdiffeq package](https://github.com/rtqichen/torchdiffeq) [2] and we use the rectified Huber unit from Kolter *et. al* [3]. 28 | 29 | --- 30 | 31 | ## Installation 32 | ```bash 33 | pip install git+https://github.com/coursekevin/weakformghnn#egg=weakformghnn 34 | ``` 35 | 36 | ## Library Overview 37 | This library provides **three** main components: 38 | 39 | * GHNN class: The class provides a convenient interface for specifying **strong priors** on the form of the generalized Hamiltonian. This module inherits from the torch nn.Module class. This allows GHNNs to be trained in the usual ways one can train a model for a continuous time ODE (ie. see [1,2]). See the ghnn_tutorial for an example of how to use this class in context. 40 | 41 | * GHNNwHPrior class: This class has a similar structure to the main GHNN class except it should only be used when the generalized Hamiltonian is known. It can be trained in the same way as a standard GHNN. 42 | 43 | * weak_form_loss: Use this loss function to perform "weak form regression" with any model for a continuous time ODE. See the weakform_tutorial for an example. 44 | 45 | 46 |

47 | Example pendulum training 48 |

49 | 50 | ## References 51 | [1] Kevin L. Course, Trefor W. Evans, Prasanth B. Nair. "Weak Form Generalized Hamiltonian Learning." Advances in Neural Information Processing Systems. 2020. 52 | 53 | [2] Ricky T. Q. Chen, Yulia Rubanova, Jesse Bettencourt, David Duvenaud. "Neural Ordinary Differential Equations." Advances in Neural Information Processing Systems. 2018. 54 | 55 | [3] J. Z. Kolter and G. Manek. “Learning Stable Deep Dynamics Models”. In: Advances in Neural Information Processing Systems 32. Ed. by H. Wallach, H. Larochelle, A. Beygelzimer, F. d. Alché-Buc, E. Fox, and R. Garnett. Curran Associates, Inc., 2019, pp. 11126–11134. -------------------------------------------------------------------------------- /assets/lorenz_trajectories.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coursekevin/weakformghnn/3c0ee28cf213e17efbf44ef15f4f339fbe52edb2/assets/lorenz_trajectories.png -------------------------------------------------------------------------------- /assets/pendulum_train.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coursekevin/weakformghnn/3c0ee28cf213e17efbf44ef15f4f339fbe52edb2/assets/pendulum_train.gif -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="weakformghnn", 8 | version="1.0.0", 9 | author="Kevin L. Course, Trefor W. Evans, Prasanth B. Nair", 10 | author_email="kevin.course@mail.utoronto.ca", 11 | description="PyTorch implementation of weak form generalized Hamiltonian learning.", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/coursekevin/weakformghnn", 15 | packages=setuptools.find_packages(), 16 | install_requires=['torch'], 17 | classifiers=[ 18 | "Programming Language :: Python :: 3", 19 | "Operating System :: OS Independent", 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coursekevin/weakformghnn/3c0ee28cf213e17efbf44ef15f4f339fbe52edb2/tests/__init__.py -------------------------------------------------------------------------------- /tests/ghnn_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit testing for generalized Hamiltonian learning 3 | """ 4 | from weakformghnn import * 5 | import torch 6 | import torch.nn as nn 7 | import pytest 8 | import numpy as np 9 | import pickle 10 | from distutils import dir_util 11 | import os 12 | from torchdiffeq import odeint 13 | import matplotlib.pyplot as plt 14 | 15 | # Constants 16 | NDIM = 3 17 | NHIDDEN = 50 18 | NLAYERS = 3 19 | 20 | 21 | @pytest.fixture 22 | def GHNN_model(): 23 | return GHNN(NDIM, NHIDDEN, NLAYERS) 24 | 25 | 26 | @pytest.fixture 27 | def convex_concave_data(): 28 | t = torch.linspace(0., 10., 100) 29 | y_conv = torch.pow(t, 2)+10 30 | y_conc = -y_conv 31 | loss = torch.nn.MSELoss() 32 | return (t.view(-1, 1), y_conv.view(-1, 1), y_conc.view(-1, 1), loss) 33 | 34 | 35 | @pytest.fixture 36 | def ivp_integration_data(datadir): 37 | y0 = torch.tensor([np.pi - np.pi/32, 0.0], requires_grad=True) 38 | t_eval = torch.linspace(0., 20., 2000) 39 | data_path = datadir.join('pendulum_test_data.pkl') 40 | with open(data_path, 'rb') as handle: 41 | y = torch.tensor(pickle.load(handle)) 42 | return (y0, t_eval, y) 43 | 44 | 45 | @pytest.fixture 46 | def vector_calc_data(): 47 | def f(x): 48 | return torch.stack([2*x[:, 0]**3 + x[:, 1], 49 | x[:, 0]*torch.sin(x[:, 1]) + x[:, 2]**2, x[:, 2]**4], dim=1) 50 | 51 | def div_f(x): 52 | return 6*x[:, 0]**2 + x[:, 0]*torch.cos(x[:, 1]) + 4*x[:, 2]**3 53 | 54 | def curl_f(x): 55 | return torch.stack([torch.sin(x[:, 1])-1., torch.zeros(x[:, 0].shape), -2*x[:, 2]], dim=1) 56 | 57 | X = torch.randn(20, 3, requires_grad=True) 58 | f_val = f(X) 59 | div_val = div_f(X) 60 | curl_val = curl_f(X) 61 | 62 | return (f_val, div_val, curl_val, X) 63 | # Tests 64 | 65 | 66 | def test_PosLinear(): 67 | pos_func = PosLinear(5, 1) 68 | x = torch.pow(torch.randn(20, 5), 2) 69 | np.testing.assert_array_less(torch.zeros( 70 | 20, 1), pos_func(x).detach().numpy()) 71 | np.testing.assert_array_less( 72 | pos_func(-x).detach().numpy(), torch.zeros(20, 1)) 73 | 74 | 75 | def test_ScalarFuncZero(): 76 | zero_func = ScalarFuncZero(NDIM, NHIDDEN, NLAYERS) 77 | x = torch.zeros(10, 20, NDIM) 78 | np.testing.assert_almost_equal(torch.zeros( 79 | 10, 20, 1), zero_func(x).detach().numpy()) 80 | 81 | 82 | def test_ScalarFuncPos(): 83 | zero_func_pos = ScalarFuncZeroPos(NDIM, NHIDDEN, NLAYERS) 84 | x = 10.*torch.randn(10, 20, NDIM)-5. 85 | np.testing.assert_array_less(torch.zeros( 86 | 10, 20, 1), zero_func_pos(x).detach().numpy()) 87 | 88 | z = torch.zeros(10, NDIM) 89 | np.testing.assert_array_almost_equal(torch.zeros( 90 | 10, 1).numpy(), zero_func_pos(z).detach().numpy()) 91 | 92 | 93 | def test_ScalarFunc(): 94 | scalar_func = ScalarFunc(0, NHIDDEN, NLAYERS) 95 | out1 = scalar_func(torch.randn(10, 0)) 96 | out2 = scalar_func(torch.randn(10, 0)) 97 | np.testing.assert_array_equal(out1.detach().numpy(), out2.detach().numpy()) 98 | 99 | 100 | def test_ZeroDivMat(): 101 | J = ZeroDivMat(NDIM, NHIDDEN, NLAYERS) 102 | x = torch.randn(10, NDIM) 103 | np.testing.assert_array_equal((10, NDIM, NDIM), J(x).shape) 104 | 105 | J = ZeroDivMat(2, NHIDDEN, NLAYERS) 106 | x = torch.randn(10, 2) 107 | J_out = J(x) 108 | np.testing.assert_array_equal( 109 | J_out[0].detach().numpy(), J_out[1].detach().numpy()) 110 | 111 | 112 | def test_GHNN(GHNN_model): 113 | # note this function just tests that number of computed outputs is correct 114 | model = GHNN_model 115 | x = torch.randn(20, NDIM, requires_grad=True) 116 | np.testing.assert_array_equal((20, NDIM), model(0., x).shape) 117 | 118 | # test additional dimensions are handled correctly 119 | x = torch.randn(3, 5, 10, NDIM) 120 | x_shape = x.shape 121 | out_loop = [] 122 | for i in range(x.shape[0]): 123 | out_tmp = [] 124 | for j in range(x.shape[1]): 125 | out_tmp.append(model(0., x[i, j, :, :])) 126 | out_loop.append(torch.stack(out_tmp)) 127 | out_loop = torch.stack(out_loop) 128 | np.testing.assert_array_almost_equal( 129 | out_loop.detach().numpy(), model(0., x).detach().numpy()) 130 | 131 | 132 | def test_hessian(GHNN_model): 133 | model = GHNN_model 134 | 135 | def gr_f(xin): 136 | x = xin[:, 0] 137 | y = xin[:, 1] 138 | z = xin[:, 2] 139 | return torch.stack([2*x*y + z, torch.pow(x, 2), x + 3*torch.pow(z, 2)]).T 140 | 141 | def h(xin): 142 | x = xin[:, 0] 143 | y = xin[:, 1] 144 | z = xin[:, 2] 145 | hess = [] 146 | for j in range(xin.shape[0]): 147 | hess.append(torch.tensor( 148 | [[2*y[j], 2*x[j], 1.], [2*x[j], 0., 0.], [1., 0., 6*z[j]]])) 149 | return torch.stack(hess) 150 | x = torch.randn(20, NDIM, requires_grad=True) 151 | 152 | np.testing.assert_array_almost_equal( 153 | model.hessian(gr_f(x), x).detach().numpy(), h(x)) 154 | 155 | 156 | def test_ConvexFunc(convex_concave_data): 157 | t, yconv, yconc, loss = convex_concave_data 158 | conv_func1 = ConvexFunc(1, 30, 3) 159 | conv_func2 = ConvexFunc(1, 30, 3) 160 | optimizer1 = torch.optim.Adam(conv_func1.parameters(), lr=1e-1) 161 | optimizer2 = torch.optim.Adam(conv_func2.parameters(), lr=1e-1) 162 | for _ in range(500): 163 | L1 = loss(conv_func1(t), yconv) 164 | L1.backward() 165 | optimizer1.step() 166 | optimizer1.zero_grad() 167 | L2 = loss(conv_func2(t), yconc) 168 | L2.backward() 169 | optimizer2.step() 170 | optimizer2.zero_grad() 171 | assert L1 < 0.5 172 | assert L2 > 1000 173 | 174 | 175 | def test_ConcaveFunc(convex_concave_data): 176 | t, yconv, yconc, loss = convex_concave_data 177 | conc_func1 = ConcaveFunc(1, 30, 3) 178 | conc_func2 = ConcaveFunc(1, 30, 3) 179 | optimizer1 = torch.optim.Adam(conc_func1.parameters(), lr=1e-1) 180 | optimizer2 = torch.optim.Adam(conc_func2.parameters(), lr=1e-1) 181 | for _ in range(500): 182 | L1 = loss(conc_func1(t), yconv) 183 | L1.backward() 184 | optimizer1.step() 185 | optimizer1.zero_grad() 186 | L2 = loss(conc_func2(t), yconc) 187 | L2.backward() 188 | optimizer2.step() 189 | optimizer2.zero_grad() 190 | assert L2 < 0.5 191 | assert L1 > 1000 192 | 193 | 194 | def test_ConcaveZero(convex_concave_data): 195 | t, _, yconc, loss = convex_concave_data 196 | conv_zero = ConcaveFuncZero(1, 30, 3) 197 | optimizer = torch.optim.Adam(conv_zero.parameters(), lr=1e-1) 198 | for _ in range(500): 199 | L = loss(conv_zero(t), yconc) 200 | L.backward() 201 | optimizer.step() 202 | optimizer.zero_grad() 203 | x_in = torch.zeros(20, 1) 204 | np.testing.assert_array_almost_equal( 205 | x_in, conv_zero(x_in).detach().numpy(), decimal=4) 206 | 207 | 208 | def test_gauss_rbf(): 209 | t = torch.tensor([1., 2., 3.]) 210 | c = torch.tensor([4., 5.]) 211 | eps = 0.3 212 | 213 | t.requires_grad = True 214 | 215 | grbf, grbf_deriv = gauss_rbf(t, c, eps) 216 | 217 | true_grbf = torch.tensor([[0.4449, 0.2369], 218 | [0.6977, 0.4449], 219 | [0.9139, 0.6977]]) 220 | 221 | true_grad_grbf = [] 222 | for i in range(grbf.shape[1]): 223 | true_grad_grbf.append(torch.autograd.grad( 224 | grbf[:, i].sum(), t, retain_graph=True)[0].reshape(-1, 1)) 225 | true_grad_grbf = torch.cat(true_grad_grbf, dim=1) 226 | 227 | # testing grbf val 228 | np.testing.assert_array_almost_equal( 229 | grbf.detach().numpy(), true_grbf.numpy(), decimal=4) 230 | # testing grbf time deriv 231 | np.testing.assert_array_almost_equal( 232 | grbf_deriv.detach().numpy(), true_grad_grbf.detach().numpy(), decimal=4) 233 | 234 | 235 | def test_poly_bf(): 236 | deg = 3 237 | t = torch.tensor([1., 2., 3.]) 238 | c = torch.tensor([4., 5.]) 239 | 240 | t.requires_grad = True 241 | poly, poly_deriv = poly_bf(t, c, deg) 242 | 243 | expected = torch.tensor([[1., 1., -3., -4., 9., 16., -27., -64.], 244 | [1., 1., -2., -3., 4., 9., -8., -27.], 245 | [1., 1., -1., -2., 1., 4., -1., -8.]]) 246 | grad_expected = [] 247 | for i in range(poly.shape[1]): 248 | grad_expected.append(torch.autograd.grad( 249 | poly[:, i].sum(), t, retain_graph=True)[0].reshape(-1, 1)) 250 | grad_expected = torch.cat(grad_expected, dim=1) 251 | 252 | # testing poly bf val 253 | np.testing.assert_almost_equal(poly.detach().numpy(), expected.numpy()) 254 | # testing poly bf time deriv. 255 | np.testing.assert_almost_equal( 256 | poly_deriv.detach().numpy(), grad_expected.detach().numpy()) 257 | 258 | 259 | def test_divergence_calc(vector_calc_data): 260 | f_val, div_val, _, X = vector_calc_data 261 | 262 | div_calc = divergence(f_val, X) 263 | 264 | np.testing.assert_array_equal( 265 | div_calc.detach().numpy(), div_val.detach().numpy()) 266 | 267 | 268 | def test_curl_calc(vector_calc_data): 269 | f_val, _, curl_val, X = vector_calc_data 270 | 271 | curl_calc = curl(f_val, X) 272 | np.testing.assert_array_equal( 273 | curl_calc.detach().numpy(), curl_val.detach().numpy()) 274 | 275 | 276 | if __name__ == "__main__": 277 | pytest.main() 278 | -------------------------------------------------------------------------------- /tests/ghnn_test/pendulum_test_data.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coursekevin/weakformghnn/3c0ee28cf213e17efbf44ef15f4f339fbe52edb2/tests/ghnn_test/pendulum_test_data.pkl -------------------------------------------------------------------------------- /tutorial/weakform_tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Weak form loss tutorial\n", 8 | "\n", 9 | "This tutorial will walk through how to learn a continuous time model for an ODE from the weak form of the governing equations. Note that the weak form loss function can be used to training any continuous time model of an ODE. In this tutorial we will use a generic fully-connected neural network. \n", 10 | "\n", 11 | "The weakformghnn package provides a squared weak form loss function to be used to efficiently learn an ODE." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "text/plain": [ 22 | "" 23 | ] 24 | }, 25 | "execution_count": 1, 26 | "metadata": {}, 27 | "output_type": "execute_result" 28 | } 29 | ], 30 | "source": [ 31 | "import torch\n", 32 | "import torch.nn as nn\n", 33 | "import numpy as np\n", 34 | "import math \n", 35 | "import matplotlib.pyplot as plt\n", 36 | "from torchdiffeq import odeint\n", 37 | "from weakformghnn import weak_form_loss, gauss_rbf\n", 38 | "torch.random.manual_seed(42)" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "## Generating data\n", 46 | "\n", 47 | "Let's generate some training data. We will simulate a duffing oscillator as it decays towards one of it's two regions of local stability." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 2, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "data": { 57 | "text/plain": [ 58 | "Text(0, 0.5, 'y')" 59 | ] 60 | }, 61 | "execution_count": 2, 62 | "metadata": {}, 63 | "output_type": "execute_result" 64 | }, 65 | { 66 | "data": { 67 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAB53ElEQVR4nO29d3hc13ng/Ttzp3eUQQdBEOxNbJIsWT22LDkqthNvHDuJpCQuydqb7K6z9m4Sb77E2c3u502y9iaPk03cEsf5YiuWLVsukm31SlKk2DtI9I7pfe73xztDDEA0kgAGJM7veYbgzNxy7p2Z9z3nrco0TTQajUaz8rBUegAajUajqQxaAWg0Gs0KRSsAjUajWaFoBaDRaDQrFK0ANBqNZoVirfQALofa2lpz9erVlR6GRqPRXFPs27dv2DTN0NTXrykFsHr1avbu3VvpYWg0Gs01hVLq/HSvaxOQRqPRrFC0AtBoNJoVilYAGo1Gs0K5pnwAGo1m6clms3R3d5NKpSo9FM0cOJ1OWlpasNls89peKwCNRjMr3d3d+Hw+Vq9ejVKq0sPRzIBpmoyMjNDd3U17e/u89tEKYCEJ90DPfkgMg7sWmndBoLnSo9JoropUKqWF/zWAUoqamhqGhobmvY/2ASwU4R448RTkEuCtk78nnpLXNZprHC38rw0u93PSK4CFomc/OP3g8EN8BEbPQGwQxrvg5o/qlYBGo1l26BXAQpEYBrtXhH/PXsinZSWQHNErAY3mKhgfH+ev//qvF+34zz77LC+//PJl77d3717+3b/7d1d0zr/8y78kkUhc0b4LiVYAC4W7FjIxmfnb3WBzixLw1MnKoGd/pUeo0VyTzKYA8vn8VR9/NgWQy+Vm3G/Pnj18/vOfv6JzXokCWIhrnYpWAAtF8y5IRcTsYzggm4BMEqo7ZGWQGJ59/3APHH0S9n5Z/i71iqHS59doZuDTn/40Z86cYceOHfze7/0ezz77LHfffTcf/OAH2bZtG52dnWzduvXi9p/73Of4oz/6IwDOnDnDfffdx+7du7n99ts5fvz4pGN3dnbyxS9+kb/4i79gx44dvPDCCzz66KP8h//wH7j77rv51Kc+xeuvv86tt97Kzp07ufXWWzlx4gQgiuOBBx4AIB6P8+u//uvceOON7Ny5k+985zuACO1PfvKTbNu2je3bt/OFL3yBz3/+8/T29nL33Xdz9913A/CNb3yDbdu2sXXrVj71qU9dHJ/X6+Uzn/kMN998M5/97Gd573vfe/G9p59+mve9731XdW+1D2ChCDTDhneLzT8+KDP/5s3gqYF0RFYIM1FyIDv9YjbKxOT5hnfP7TtYiMijqzm/ZkXx/zx5hKO9kQU95uYmP//1wS0zvv9nf/ZnHD58mAMHDgAieF9//XUOHz5Me3s7nZ2dM+77kY98hC9+8YusW7eO1157jd/+7d/mpz/96cX3V69ezcc+9jG8Xi+f/OQnAfj7v/97Tp48yTPPPINhGEQiEZ5//nmsVivPPPMM/+W//Bcef/zxSef50z/9U+655x6+9KUvMT4+zk033cQ73vEOvva1r3Hu3DnefPNNrFYro6OjVFdX8+d//uf87Gc/o7a2lt7eXj71qU+xb98+qqqquPfee3niiSd4z3veQzweZ+vWrfzxH/8xpmmyadMmhoaGCIVCfPnLX+axxx678htPBRWAUsoJPA84iuP4lmma/7VS41kQAs3i8C0JU7tXhH8qAhtum3m/cgcyTPzt2T+7AF4owT2f8+sQV80y4qabbpoz1j0Wi/Hyyy/z/ve//+Jr6XR6Xsd///vfj2EYAITDYR555BFOnTqFUopsNnvJ9j/+8Y/57ne/y+c+9zlAQmcvXLjAM888w8c+9jGsVhG11dXVl+z7xhtvcNdddxEKSbHOD33oQzz//PO85z3vwTAMfuEXfgGQCJ9f/dVf5R//8R957LHHeOWVV/ja1742r+uZiUquANLAPaZpxpRSNuBFpdQPTNN8tYJjmpNEJkc0lSOWzhFL5YincyQyefKmiWmaFEwL9sJuAj2H8OT6Md21pOtuwRLx4EpG8busBFw2XDZjImQrMSwCvBy7V8xJMxHugdf+BhIjsm91h6w2YG7FcclFzXH+cA8c+CfZLp8FwwZDx2HHB7USWGHMNlNfSjwez8X/W61WCoXCxeeljOVCoUAwGLy4crjS4//hH/4hd999N9/+9rfp7OzkrrvuumR70zR5/PHH2bBhwyWvzxWaaZrmjO85nc6Ligjgscce48EHH8TpdPL+97//omK5UiqmAEy56ljxqa34mPlOLDE940m++nInA5EUA5EUg5E0A5EU8cx8HTG1xQfA+eJjArthwe+yUeW2cb91lFr7EE5fkKDLRtBjJ2RLUx0M4p/uC1Sa+SeLwj+flsij5j3grppeccw2gy85sEszf5DnJbPVqafFue2uFsWQS8rzU0/DnkfneT80mivD5/MRjUZnfL++vp7BwUFGRkbwer1873vf47777sPv99Pe3s43v/lN3v/+92OaJm+99RY33HDDJcePRGY2a4XDYZqb5bfyla98Zdpt3vWud/GFL3yBL3zhCyilePPNN9m5cyf33nsvX/ziF7nrrrsmmYBK11RbW8vNN9/M7/zO7zA8PExVVRXf+MY3+MQnPjHteZqammhqauKzn/0sTz/99Bx3bm4q6gNQShnAPmAt8Femab42zTYfAT4CsGrVqiUbWyKd4ysvd9Lgd1Lvd7Cpyc9dG+oI+Rz4XVa8DiseuxWv04rbbmBYFBZVeogmy+QKpHN50tkC6VyBeCZHJJkjN9aNZ+QtVGKYgZyXM6kqCiNH6e0ZJ5J34CGJjyRPF3YT/vYPaQ66aKvx0BHysLbOy+7ESzS4PHg9ReFvc8ugR8+AdeOl/oa5TEXNu+Q5iIDPxCabrfoOgis4cR6bG1wFeV2jWWRqamp4+9vfztatW7n//vv5+Z//+Unv22y2i47S9vZ2Nm7cePG9r3/96/zWb/0Wn/3sZ8lms3zgAx+4RAE8+OCD/OIv/iLf+c53+MIXvnDJ+f/Tf/pPPPLII/z5n/8599xzz6T3SpOzP/zDP+R3f/d32b59O6Zpsnr1ar73ve/xm7/5m5w8eZLt27djs9n48Ic/zMc//nE+8pGPcP/999PY2MjPfvYz/vt//+/cfffdmKbJu9/9bh5++OEZ78eHPvQhhoaG2Lx582Xfy6mo2ZYfS4VSKgh8G/iEaZqHZ9puz5495lI1hCndlyvKgJxttl0ujMuFbdMuzEgvyfEBRpWfTtsGzmQCdI8l6B5Lcm44zrnhOOlcgV82fsKQGaDZnuJWZycurxef20u9NYynbQ/+Gx5CBVsmxnP0SclMLp/hpyNgdcPmB+ce85P/HmxOsE8si8nEIZuCB//i8q5fc81x7NgxNm3aVOlhLDsef/xxvvvd7/LVr351Sc/78Y9/nJ07d/Ibv/Eb074/3eellNpnmuaeqdsuiygg0zTHlVLPAvcBMyqApeSKU99nm23DzHb7SC9q84O4ATfQAkx1G+cLJr3jScb3jTE8NsaFuJVTo3Zcw+fxZE/zCgG+etRL7vljbG7qYUtTgM1Nft422k1tfevkmN+pPoZA88xCumkHXHgFlIJcBsJdcg0N2+V6y/fTEUWaFcB3v/tdfv/3f58vfelLS3re3bt34/F4+F//638tyPEqGQUUArJF4e8C3gH8j0qNZ8E49TSMnIZCTmbc1R0iDE89DYXs5dntp2BYFK3VblpvurtsFbEKMpuJhcc4GbwNd8TDkd4IR3oj/P2LZ8nmTe61DBCwdlNdXcOqajerqj2sDRSor6mdXyLI2nfI+MY6xcxkc0NwNdSsv1S4X2lEk0ZzDfHQQw/x0EMPLfl59+3bt6DHq+QKoBH4atEPYAH+xTTN71VwPFdPuAc6XwBvCJxBcZb27IOmnWIvb3ub5AfMx24/G6Wcg579IpjdtXi338auQDO7yjbL5AqcHIhy7mwd6sQPOBNJ8MrpBK/nOvGR5GXbzTS2vsqO1iA7W6vYsSpIrdcx+XpKphxvHUR6RaGVr17SkcnC/UoimqaeS5uNNJoloZJRQG8BOyt1/gWnFJYZGxT7eLANnD55b/CI/LV7RXD2FLW44ZDt/U2z5wlMx2wmmyJ2q4WtzQG2Nt8I25ugZz+F+BADOR/7cu2Yww4OdI3zxefOki+Iz6O12sWO1ireXpvm1vxrNNbXYSuZctJhaL9LFNzFk0wR7nNFFE2HNhtpNBVhWfgArnnKwzJrN8DwCRg8CqGNYFghPgTtd4pg89RA826Z+ccHwV2zNIKuqDAsyNLrgeIDIJnJc6gnzIGuMd68MM7ezlHSh17gFVKkLFFaq92sqfVwk+mm7sJb+DbdM+EjmSrc54oomg5tNtJoKoJWAAtBSYCVzDt1m2G8E0bPQu06Ef5r3zEhGN1VYvZJNS2LWa7LbnBTezU3tU9kKYZfOMPphIuzI0nODsV54dQw+/NObjcOs/eQi4ZQLZtqLHT489Tf+D6cpR3LzVNDJyA5LiGkpWJ4013rlZqNNBrNVaEVwEJQEmAl847dJUogNgg1HSL8p7Hbs+G2igv/mQjUNLI7kGD3apnd5wsFegcG6RpoZHXExuhgL//c5eRgYQ1jLxxhS3MXu1dVsbutil1tNdQ374JYv1xfaSUwk1nnSsxGmhXLH/3RH02q3TOVJ554gvXr1y9InPz1jlYAC0FJgM1l3pmH3X7ZMMWUY2RjtLpztN73AW4tXsNQNM3+C2PsPz/GvvNjfO3V8/zdi+cA+IDvIJtqrDQ32OgIGbRWe+XLNp1Z50rMRprlS4Ud+k888QQPPPCAVgDzYFkkgs2XpUwEuyxmSu5aBuadq6J7Hxz9DkT6wN8Imx+Glt0zbp7JFTjSG2b/hXFcB7/Gm6N2RpNSOMtuWFhd4+KGqgzWG3+d3VVJqsYOTQgJf5NEGekooGXHZSWCLdJv4U//9E/52te+RmtrK6FQiN27dxMIBPjbv/1bMpkMa9eu5R/+4R84cOAADzzwAIFAgEAgwOOPP85Pf/rTS7Zzu91XPJblzuUkgmkFsFBcb2GMV/tDLmYfj+ScnBmMcWYoRm//IKfG87yZX8M7Lftw+qpoDNWysVrR4S8Q2v1ejKqWuY99JdeykJ/N9fZZz8FlKYD5ZJ1fJvv27ePRRx/ltddeI5fLsWvXLj72sY/x2GOPUVMjiZR/8Ad/QH19PZ/4xCd49NFHeeCBB/jFX/xFAEZGRqbd7nrlmssEvi64lsw78+FqI3OKZp0aJ9SsDnJTkxU2WEiueRddh1/ifH8Hx8fgUG+Ul89maWaAVS/8Z1RVO6H6ZkIbb2Hzxk0EXLaru46FDjHVIauzswgO/RdeeIH3vve9F2ftpQSsw4cP8wd/8AeMj48Ti8V417veNe3+891uJaIVgGZ6rvaHPIPT2xVoZr33J6zf0cE7lQXTNBke7CN6+jwj0RxPpX0cOnwO76Gj/HtzN/66Nna3VbFzVRXbWwKsDXmxGpfRyK6kyHJZGNwrs1GLDU4/A7sfmXv/qbP9xIgOWZ2NRXLoT1ea5dFHH+WJJ57ghhtu4Ctf+QrPPvvstPvOd7uViFYAmulZiB/yTKuismMrpQhFjhGyjbKm2sKNjQmSvo1cCOd421iKb8ZdfP+tPr7xehcATpuFzY1+tjUH2NYSZFtzgI6QZ2alkBgGZUDvfunV7AxANgnnnpuIzpqJ6Wb7nS9IWG9ZwrQOWS1jERz6d9xxB48++iif/vSnyeVyPPnkk3z0ox8lGo3S2NhINpvl61//+sWSzVPLR8+0nUYrAM1MLGZkTvmxs2noPwCGU0Jn82lcgwfZ0LSTDa5hHqgZoLBtiIGclwOFNbwx6uZwT5hv7uvmq69IjwWnzcKWpoAoheYA21oCdIS8GBYlyub8SyL8S+U3lAJPaO5Z+3Srh1QELrwOm8tKEs8n03ml+AwWIdx5165d/NIv/RI7duygra2N22+/HYA/+ZM/4eabb6atrY1t27ZdFPof+MAH+PCHP8znP/95vvWtb824nUY7gTWzsZiCq3TsMz+FcDf4GsFXNDllE5AMg8WA1W+f1gmdH+tm8PjL9PV1czLm4GeRFp4fcJDMSsMel81gc5Ofm6qTvH/s/+KubqDKH8ShMpBJSn0mMw97ZumpuvfLk1cPVqcItb5DcMMvQ1Xr3M7x+TrTl7GS0OWgry20E1izMCymY7t07MSwCOPe/SL4rU4wTamouvm909vaAePUD2j0+GncvIFdmRgfSA2SX3c/ZzMBDvWEeas7zNG+CF8/lmMk08yOrpM0qHGcNoOEpxWGzxFoWI2/P8rqWjcOq3HpGKdbPTj90LAVor1gc8w9w51vv+X5OpaXsaLQXHtoBaCpLO5aCRts3iMJdKmwOGmDbTLDLqdka59BqBp9b7Ju84Osq/fxvl0STmqaJkPHfRRe+jyDuRC9aSeZ2Ajq7Mv85Ykgh597HsOiWFXtpiPkZW2dl3V18ndt6AY8R5+Q4nemKdVdM0lovXnu1UOJ+TjT5xtxpSOQNAuMVgCaylLyBzj90LJnwkRSt2lmJ/RlRCgppagzR2DLXTSMnmb76Dmohqy3g11+B3trdnBmMMapwRinB2M8d3KQbH7CLPrrzlp2OHupcioc3iBG7VqqwzlCVVV45tHwe17O9PLriY9MKELTnJjhl6rNJkek5lR5M6EliECaT3NzTeW5XJO+VgCayjKT0xBmdkL37L+8CKXEsJhxCjmo2wBWJ7Zskuax12ne8yDs2HBx02y+wIXRBKcGJHltrM/OYP9P2Ru10Tto4Dnbc7Ffc+I7P6atxk1bjYe2ajdtNW6ag24ag06aAi5cdmN+zvSSkshlJ2pJWR2iAE48BU27xESWKG8mtE/KjsyzmdDV4HQ6LyZTaSWwfDFNk5GREZxO59wbF9EKQFN5ZvI1zBZNcjkRSpcRCWQzLHSEvHSEvMVX1kJ4M/TsJxsdZLDg44xtA6uTPi6MJugcSXCkJ8yPDveTK0yefQXdNhoDLja7m9llO0eTrRNnsB77qlsI5aqoz+VxxPshOSphqekYeOpl52xSzGJWm5TjaNgyIfyvppnQdMzhV2hpaaG7u5uhoaGrO49m0XE6nbS0zD+bXisAzfJlJsVwuaGGzbtgOlt+004RevMchw1oLj7umLJJLl+gL5yieyxJXzhJXzhF77j8PTJu8kx4HeFiXSQ4D5ynnhHe4zqIcgaosrdxZ+ZnuCz9RAMbSddtwx2xEXAa1Ix1Y2+9CVXdIW1EQZoJxQeLJcWvIjS3ex/s/6r4NNwhyKWkimuZX8Fms9He3n7l59AsW7QC0FybXE6EUqAZVt8OQ8cgNS6mo+bNMru2LkxRMKthkX7N1TMfL5HJ0TueEgUxnsJ15gck440MZuwMp3K8llmDJRMmPJ7kzc4xYAwvCTaoDKcPvYThDNDs8LPWOkidCmO6aujybcPbZaF2bJSQz0HI58CdHJhfpFC4R4S/xQBXSBTj0EkIrZ/eAX210Uc6gmnZoRWAZmWw7p1QyE4Tj790JafddqtEF9UVzUupmDwyUVFKzp2Yo6fIZLOMtmwmFgmTjI1xzvp2OkYOMpQxGExXcTRp5XDKyxMD2+k/NwKMUM8IN1jO0q56WWMM02dfTdLTTL1zgJDtTQYb78EdWkXI66C2qChCJ3+EffQcWO3ifPY1iP8hOiDhuCUWIvpIRzAtS7QC0KwMlltDnnAPjHUWZ9/VMvseO4vyNeLIJmk0otBcB833sT3QDOFdZbPnteBv4vfCPcTHeoimC6TH+xky6rGOjJJP+WnKDHGYAOcibk4lM4ycfYYfF268ePp6Rvgvtn/BZ6SxWQwctjheay8JTxsBW44Ea7H1RmgKOgn07EcVcjB4XLKhHUUhfjnRRzOFup56Wvpm6FVBRdAKQLNyWE4VW3v2Q2gzDB8X4W91iuM32gd3furScZaPvTibNpx+/LXN+M+9AJZx1qxaCzYbODsgl+R2A2jdCmaBbKSfofX3MBxLMxRNYz/1fbx9q8mm47iTfcTyJkNxMCMnOVGo4fNnCgz85AUA/q3t+2x0jGJ3eXG5vVQ7hgjaenCFwvjW/zx26zyK800X6hrugeGTULcVbHYwbDB0HHZ8cGaTlTYhLShaAWg0lSAxLIluDu9E3L8jILPkuYTa1Nl0ISt9l0fPyGslhZIKy/uZGDZvHU1BF01Bl7wWV9B8q4SXFqogNQbJMPm8wfhtn2Kza+NFR/bGw0+RTdjozVgJj0fJFQoEiBI/nuBzz/2A5ioX6+p8rK/3sbFB/nbUeSZnV08X6hrrg0wCwp2iDC2GXMOpp2HPo5OvWZuQFgWtADSaSlDeRrSU0FVqnDIXUxPhHH6J3klHoHGnCNhsUhRKqYDdVF+HssDgUdkuEwe7B/zNGHWbqNlwCzXAzosbd8DYeXB4MA0niUSceMxFv6WBZPVazg3HOT0Y44VTQxeT6AyLor3Ww4Z6H5ub/NxcvZqt8Zdwhs9JCQ2AaLFntM0lkUeh9eAqQN/BS6/5avtTaKZFKwCNphJcTbXVqdnF1R1w/kUpde2uEkE6eEwEptV9qa8j3CN+kNS4bO+uhuSY2OLXvuPS89Wsk+PEB1DpMB63H0+onbpAM9s3TyTRZXIFOkfinOiPcnIgyvH+KId6wnz/UB8ADcrN73vPU+N1EKyqoc1WjctwYjHskI0Xj1KWaFYy+YycgnMvSN5GoHkiC1qX4b5qtALQaCrB1TilpyoPq60oFENyLH8LbHpo9gJ1VaukAuvoGVklOAMTAna688X6oW7jZGXVvGvSZnarhfX1YgIqZzyR4UDXOAe6xjl7fJiXhkcYGnBwm/Ky0XYBr8uN3x+g2hvFVwjDqlsmTD6FnKw+FNIz2uoUZVVKkrvaJLgVji4HrdFci1yNQ3Tvl8WEpMqct2ZBlMdMBe4WygEb7qFw/CkGs3a6Bsawdf6UbHSYzlyQsOlFuWsZX/9+7g32sL7GwDF6UrKf8zkYOgpWD9SskYS+mg7tA5gnuhy0RnM9cTURTVfS7W2hIqgCzVg2vpuGnv00OAvQ9j5MYFUkyVtjNr4/3MgP30oxXNjPmKWK9wdP09zQQHutF0ftRhg9B7m0KAAt/K8arQA0mpXGYnZ7mw9TlIkC6oF3Fh+pbJ6zL4xy7MIAXResnDnaxU8sTtZWWdhQU0eLsmJNjUl1VFdQfBQ6JPSK0ApAo1lpLLekuHLCPTh79rPZNsDm6k4KresYGbxA52iKgaEBDo1mOH5OUR8M0NbooDoVFgf1lPpFmvlRMQWglGoFvgY0AAXgb03T/N+VGo9Gs6JYTklxJcpj/UMbwOrGMnSUUG09oRqTfL2N3oyLswNh9o9keGEoSptfsSF6mo7Nu7HpkNDLppIrgBzwH03T3K+U8gH7lFJPm6Z5tIJj0mg0lWJqrH91G3iqZIa/+UGMvV+m1VtH65mfcpPhkzDT7nFeO9bJ352t5sEOKzvX3IffaZv5HDqbeBIVUwCmafYBfcX/R5VSx5BKu1oBaDQrkbk6vZU5r135NDtXVXFDvZ0LkVpi/SZfP5zg3574Kb96SxsfvcFJcOzQZEEPOpt4CsvCB6CUWo0kHr5W4aFoNJpKMVd0Usl57a2TwnTZJJZCgdWrNvFv2w2O+W/Htj/Bt597newrb7JrfRt3bmvHk0vIfhabziaewjyqOC0uSikv8Djwu6ZpRqZ5/yNKqb1Kqb26I5FGcx3TvEuikdIRyUsolbEozd5Lzmt/C1S1gd0nf4uvb9q4kb/64C4ef8hJU0M9jx+J8J//9QjPnE2Ss3ulxITdO/mcdu/8mgJdp1Q0EUwpZQO+B/zINM0/n2t7nQim0VznLISNvpjodn40yTf3dXOsL0KT38FHm8/Rsu4GMSmVl7W2uq77ktQzJYJVTAEo6S79VWDUNM3fnc8+WgFoNJo5Ofok5BLg8GOaJge7w3zvjRMQ7eNdVf1s7mjD46+RWkjjPVC3ARq2TsmJuL78AjMpgEqagN4O/Cpwj1LqQPHx7gqOR6PRXA+UmZIUJjvqLHz67iZu3bKWpyJtfP2tCCcu9FBwBsFTB/msrAaUpdiZzS+rkBVAJaOAXmRS6T+NRqNZAKZJdLNuuo17rD9m87o1/MOrXfzP0xG2JPx8tGkETz4zef8VVGV0WUQBaTQazYIyXaKbu5aGXIJP3rueZ08O8S9vdPGvw0Pc1lFLe/l2M9VFug5zCCoeBaTRaDRLQtE0pDJR7l5fy399VysFZzX/cizJt14+SjaXuzTyqEQpSzmXEMdxKbQ03FOZa1kgtALQaDQrg5JpyOqG2CAN1dX88oc/jWP3r/DjU1H+/oevMZa1Tu8A7tkvvQkGj8OZn8rfQm5uX0G4R5zSe78sf5eZwtAmII1Gs3KYYhqyA7/7C608uXY9n3r8Lf7uB1b+76+52RmYst/IqYttMXEGpe9yMRltRq6BPsZ6BaDRaFY8D97QxBP/9u247Qa//H9f5YeH+ydvkBwHiwVsblBK/los8vpMlNc2WqYRRloBaDQaDbC+3se//vatbGzw81tf38eXXjw38aYrCIUCxAZg6AR0vQ4j50QZzERieNlnHmsFoNFoNEVqvQ6+8eG3ce/mev74e0f5i6dPYpqmNJ3xNsJ4N6TD4AiIWSc5NrNdv1TbqJy5Oq8tMVoBaDQaTRkuu8Fff2g3/2ZPC//7J6f4sx8ex2zaCbFeqF4tDemrWsHuhtDmmU06c9U2WgZoJ7BGo9FMwbAo/ux927FbLfzNc2fJ5FbzmeBqVDoCqbDY85s3g7tq5qSx5dx5rYhWABqNRjMNFoviTx7eis2w8OWXOtmx3c3DW5snl6tOR2Y36SzHzmtlaAWg0Wg0M6CU4jMPbCaezvHf9o5Qo85y25Y1UwrH3VbpYV4x2geg0Wg0s6CU4r+/bzu7t23hPx5s5pWupJh0rO5lFdN/JWgFoNFoNHNgWBR/8Us7aF+zjkdermdv7cOw+cFrWviDVgAajWalcpllGhxWgy/+ym6agk4++g/76BpNLMhxK4lWABqNZuVxhcXdgm47f//ojWTzBX7zq3tJZvILctxKoRWARqNZeVxFmYaOkJcvfHAXJwejfOY7hxfsuJVAKwCNRrPyuMoyDXeuD/Hxu9fyzX3dfHNv14Idd6nRCkCj0aw8FqBMw+++Yz23rKnhD79zmNODsQU77lKiFYBGo1l5zFamYZ5OXMOi+N8f2IHTZvAfv3mQXL6wIMddSrQC0Gg0K48pzWEuxvTDZTlx6/xOPvuerRzsGudvnj+7YMddKnQmsEajWZlMV6bh6JMTTlyY+Nuzf8aY/we2N/HDw/385TMnuWdjHZsaF+a4S4FeAWg0Gk2JK3Ti/snDW+mwh/nON/6GwhtfutTEs0ydw1oBaDQaTYkrdOJW5Yb4b1u66Bse4aV+y6UmnmXqHNYKQKPRaEpcaQ3/nv3sXLeKprp6vrW/l5jpnhz/v0x7A2gFoNFoNCVmcuLOZadPDKMcPj70tjYSmTzfOdgz2cRzpcddZLQTWKPRaMq5khr+RRNPS5WfO9bV8tyJId7R4aa+qszEswx7A+gVgEaj0Vwt/ibofAmOf4/3hnpoNYb42YHTFTfxzIVWABqNRnM1hHugd7/0B3aH8GbHeKghzD921XAw4qn06GZFm4BWIuEecU4lhmXp2rxr2S1NNZprhvICcNVtAKxvHWfd4Gk+/5NT/P2jN1Z4gDOjVwArjWusXK1Gs+wZOQUDx+D0M9D1OsRHcHn8PLDWzk+OD3K0N1LpEc5IRRWAUupLSqlBpdThubfWLAjXWLlajWZZE+6BsU5Ih8EZhHwaevbB+AXetnUDXoeVv3r2dKVHOSOVNgF9Bfg/wNcqPI7lx5WYaeazT2JYZv7l2L0SmqbRaC6PnqLtf/g45JJgdUI2CYPH8Nz5EL/ytih/8/wZOofjrK69An/AIptrK7oCME3zeWC0kmNYllyJmWa++yzTjESN5pokMQxVrdC8BwwHpMLgCEDVagg08+tvX42hFF975fzlH3sJzLWVXgHMiVLqI8BHAFatWlXh0Vwhl6vFy800ML/CUdPtkxiF1/4Gqtsnztu8S75EIDP/TEwyEjfctjDXqtGsJEoTKk+NPECyfK1uQKqF3r+tkW/u6+I/3rsej+MyRO6VyIHLZNk7gU3T/FvTNPeYprknFApVejiXz5Vo8SspHDV1n/gIDB6HxMjk88KyzEjUaK5J5lHi4ZFb2oimcjxx4DJn7ktQQG7ZrwCuea5Ei5dmFaVt4yMwcBhyaakyON0KYuo+o2fAYgF33YSzt3TezQ9qga/RLASlEg89+2VC5a6V1XTZ72t3WxVbmvz846sX+NDNbfM/9tTfNCy4uXbZrwCueebS4tN1CSqfVcSG4PyLkBqHpp0zryCmzkTig1AoQHXH9OfVaDQLQ6BZJlV7Hpt2cqWU4t/saeVYX+TyQkKXoIBcpcNAvwG8AmxQSnUrpX6jkuNZFGZzus5kHoIJM03vm+AMQNtt4A3NHrZp2OH8q3DqGbDYwdcoK4FSfPJYl3b2ajQV4KEbmrAZin/d3z3/nZaggFxFTUCmaf5yJc+/aJQ7fVEQH4KqVZc6XefjuA22QmiDmHFKTA3bLCkSpx/WvQPGL8DZ56HrVfA1gLseRs9CfBS2vle21yYgjWbJqPLYuWdjHU8c6OXT92/Easxz7r3IBeS0D2ChKRfG3joR+HEg3AfjxVCwph3yNzEMyoDB18kmwmRyBfLJcXLYGHVuxhYdxj94EiIxbCqLKx/HcAXAUz/5S1GuSOIjMHQS8ilwh6CQF0VQ3QaN2yDaDwf+CTwhwNSlIDSaJeK9O1v40ZEBXj4zwh3rl0dAi1YAc7EQIZwOLwweg9VvJ246Odc3yPCxrzA8PIY/fpa+rJNw1soGdYGAinOs0MaLR04CsJE0dxpPcqzQSkR5WG09yRrLEOe8O0m/2YNj1R62Jy9QF3DjHDwOfW+BxQq5DNhcYHNDjQ3sHlFI4R7xJySGof0OUVAnntKRQBrNInPXhhBuu8GPjvRrBbBsmE3ATzebn0tYTpNpO9zXRW/vCN851c350TimCc0Mcb/9ILW2NKvdHmwOJ8H0OCiD5hove1avI18Az0AER7Qdl7MZFe3HmkoxbDYTTmTYd+w87iPHOECKDUYPylXDDcYIQWuaQH4YmzeEcleDqwZyccilIBMXs1I+c2l0kFYAGs2i4bQZ3LUhxI+PDvAnd1Vh6Xuz4gUZV7YCmEvA9+yHQk7i6dMREZYWx6XJVTChREbPQTZNytfC3mOnuXDqEGuTBxk2A1RXRdm+vYUtwRyrEkPYRxxQtRkGDkJuHHyyWqhigNZgXhJLYkmo2wyrbobD34HhXsj1cJtviF/aegPDKkj6zCtEYn4GYmmyyVEG8gVGsWBLjOGwx3E5h/GpJLbhM+IottigevXEfSj5FErKcOQUJMfBFYSaddpEpNEsEO/a0sC+Q0foeu0wbU2N859YLhIrWwHMFKN/6mlw18CBf4ZsHIKrJAInNgT9b4Fhkw/NsEPXaxKlY/dBrJ/0SCeDb/2UH8dWYc8maXSbdPgVN3jzODwHoaUGkqNgs4K3HtJjkjYOkMtCISURPCOnwWoDiwG+ehg6LbZ8hx+sLijksZ58iob17wZPHrbex7ZTP8aMNZBOxhgxfYSjCeKZcdqTZzhLNQlbiA7rIJ7ITzHW3CEhop4acRoPnYTOF+TYuTQ4fZLWbnVDrF+biDSaBeDujXX8wDjLwaEgbe2Ll+E7X1a2ApiuMFo2LYJw/b2gFJCHSLfY02N9xZWAT2bHuZSEaXrrKDgCnB4I09nTT3UuxkP2N7C3riPgtoNrm9jdC1k4/RPZ1xmAlpvh5FPyf8MOhQTYA3Lcs89L7O/6++X8538mvgQLYOalA1EhBxdegtBGsDnAV48KbcCZjtI8fp7mKi/5hI1k0oM97yYY7SSezJInQ+748yT7B6havQNfZlBWAd5itFAmIQqwkIDTT4u56PhTsPlhWPfO+X1Jdc8BzUpnmt+AP9DMzpo8BwayPFS+bYUKMq5sBTBdpt3gEYmQKTlvswlRBNFe+UANuzyGT0v1v+Qo6fEBnkutwZboI+j20NG2iarIcSiMQ2AP+Ook9DPaJ1E4NjfUbZSZf3gLjHdJOVmLFexu2b5hG9RvEuHftAsO/6uMKR0FTy3YnDJTj/bD7sfg5A9g7Kwc21svOQDNezD2fxVvdRPe+CA468ik4iRTGRzZKOdHhomMPMXxwN3cFBynyT2Odey8VDQcOCqKJjYItWtlJdT1qqxMateJcppJsHfvg/1flf3dIVFoehWhWUnMYl5ubm7l2FtniaZy+JzWYqb/IQncmCnTf5FY2QpgusJo8SFov0te8zeKsE+NSQy9MibKvRayYHMzlsqTHe/HZvGwrqmG5pogKp+GXI3U4UmNiUB3+sGwitnF6Rdhn45Aw/aJRDGbByiACdSsnVBMkV5ouREy0aJ56Iz4GnJpMeEMHJaStBYb9B2EdALWvlNMSDY35JOACYUcdpsNu9UCuNgY3MjY4AXM2ABnwz10GgZtboM6TxZX7IysdJw+UYDOKjF99b4pK4/226e3XYZ7RPhbDHCFREkOnYTQeu1o1qwcZikB07btNrwHj3DqQi+7mlxw4WV5b9WtE8mgSzRZmlMBKKU+DnzdNM2xRR/NUjNdHY/2O8WcAiKsk+My467ugPgwnH8JfI2YFjune0cZG8vTaPVzazCOo3q1NITIp8VvkE3KPrXrRRBmkhBqBX+LKJ+e/WJuaX2bCNlzL8qYatZOVBYsLQ03Pwwvf0GEsJkHV0DMVZ56yfZtu02SwJp2ikIYOAzJEfA2SiawsgA5MJxy3JoOvCqGt2U1zakwA9bNjPSdozOaJBYdp96axJPJ4mhuwJJPQ6BVVjAWQ5TfTBFEp54W5WSaMvO3OQBD/AyuatlGm4M01zszmZfPv8rm+iEsdjtn+4bZxbA0kqnfOvGbhyWbLM1nBdAAvKGU2g98CfiRaZrm4g5rCZmaaVdaugG4q2TmOnhMtHl1O4S7KRh2Tnb1cWE8Q3V1Ow1bt2I78X2pv+OplYcyYPWdMHREzB/ukAh/i3VCAE79gF3VMgOYrvhTy2649RPw/P+CdFxWFWtvhvFOUQqjZyZK0tZthnPPga9JHLnVbUUzUwqseajZICsbswANW7CcfZbGUC2N1UGSw52MDRQYTFhxJuL0dsapbV7HWrsXayosgj06IErF4Yeq9ol6RiOn4OSPZAWTjsi40jFRGGYBOlYv+QxHo6kI0xV0vPAyOIMY/noaqvvpGx2HTfPI9F9E5lQApmn+gVLqD4F7gceA/6OU+hfg703TPLPYA1xypq4K/C2w6aGLwso0TV559UVOjxdY29rI7htvQlnt4A6KA7WQl7h7bwMYBmz/BTHhTHWGTucknatWf8tuWP9OmVmUvjDJUZlpp8uKTJX8GPEBcHikWYXNJTbGdATS4+JraH2bKIwN94svgQKu5q24tj1MKBWl7/DzpIbTPH92lKNdA9zlPEtApTH8IVkZOTwSFeUIyjWlInLcsS5QgN0lPot8Vkxn4+fldYtNFMjuR5biE9Volp6pv+WBQ/L/+q2gLDTX1/GztyKkYmM4A4tb8XM25uUDME3TVEr1A/1ADqgCvqWUeto0zf+0mAOsCLPU3/jHkfWc6H2TGzdu4dYt1aiBI+I3WH077HpkemHP7skHmS3/YI7SspfMLKo7pFqoMyCz7HI/Rv9Bed1mER/B6BlxPMeHxIZfivEHOX8hJ7P7rlewKYNVd/warbEBth/4Pqnek0TiUXJGAYsZI1hQWDHleht2yHgyUaheJ6sNqx0wxIlumuKYzmdlPNmkrFDWvkOvAjTXJ1MnkrmM2PiLZp6OkJcnTRe9KQtrUsXJWwUaNM3HB/DvgEeAYeDvgN8zTTOrlLIAp4DrTwHMwDNHB/jMs2N8cOP9PLS5H3XuOZlpl/wGvfvnZ9qY6iDKZSW65vnPQcc9s9vIp84srLZiPH/oUj+Gw1+0w7vBaoXG7RJ9ZHVL2dpymnaJ87aQl2N5GyDSjWraRV3dfmhoIXbmVUYjcQrxBOOpHF6/hZAvhJEJyxI3OgCZYjekVBhcVjF5oWQ8zoCsXJSSc2insOZ6pnwiefRJMX8WWRPy4FVJjmbWsGauSd8iMp8VQC3wPtM0JzW1NE2zoJR6YHGGVUFmiF8fjKT45LcOsqXJzx9+8FbUqadg/bsmL91gfkKt3EEUH4Gzz0EmDKmozJyHjsOOD05/nOkc11O3LYVhpqKQHJMkNpsb6lpnnl1EemH12ydfTzoCR78jSsGw4zXjeL0FUmmDRCpGaixOT8SB1+0jENiL4a6aCJtNh4uJbHZRBmNdEFgt72eS4qwu702g8wY01zNTJm5uM0GbO89LyVYeWOSKn7MxHx/AZ2Z579jCDqfCzGCaMdffz6ef6MOfHeTvbnTgPPg1KbrWtBOKAUPER2QWX2rUMpsAKzfj9B6AeJ+UmHDXSpTN6BmJptnz6KXjKxeS6+6dHH5ZKuMw1ikOYFcKMEVRVHdApE+S0Eq9BMrHN3KqmKvQL2OzeySXYPiMrDR6X5SEsFQUZyGH07CQcAUZT+YpRPt48c1jdGzZRUugWUxANhfkc3IOm0sE/9BxyDTJ8QaPQGjT9Pd9rEsS0KpW61IUmuuDaSZu/Y338Mawq6LDWtl5AFPp2Q/5/OTaP5569r3yU44ch7+4oYcG9xqw18lM/cLLEn4J0LNXZr6B5rkjXcpnA2NnRfhjSnavzQ2ugsTzw6WC3eqF8AVx8NrccPPHxLRTEqCpiCiRWL8IfaVgFFEq6++HqtZL4/fDPXLsXBJiI7JPNiE+heSwnC+TksimfFpWBBYrbocVV2gNY2PDVEVH+MmBU9SHQtzpqsHlb5LVh90ljuHIQHHlU1tMquuSSKTS9ZWXsx4+LteQiuioIc31w5SZfkPPSTpPnSKZyeOyGxUZklYA5YycgrHzEt3ilJIM+cFj7D2U593Vrdy8qX3CRFK/TZyvA4clwkWpSxO4SuagqTN3f9NE967RcxKb762ROHuQXr4weWacihQzBp+DQEsxxHMUXvwLieIJNIkvofuNYhu5KJz4keQjFHJAQQRrISuRQ7FBmanf/FGJyCkUoPcgWB0y8y/kxVnsaxLlZnXINZZs+FYH2Dyohq1Uu3vwR/s557+PJw73E+AA6zxxqqpqMFCSCxHuKp47XEyMq5Ljn35GFE3JJDZ6RhSG1SXlM3S1Us11ysYGHwUTTg1G2d4SrMgYdE/gcpLjInxtbhF0NjenhmIUkuN8YIsLw+mb2NZTU8zcS4ugdgSgeffkBK7E8KVtH8M9ktCVTYgJyV0j7+Vzkix14gciuOND4hTuPSDtHPsPSQZwIQ/hblEWVpfMlM89L0kmZ5+X7ONcRpRAJibXlBybeO3MT2UW760TBfDU78Fz/xN69smxC3lRgkqJI9hdVXQiu0QROasmMpvtXskUtrmx2j08vMnHZx/ejN9tJxJPcnggRXKoU5zDqbAootS4FM7z1YGrSq6vvG1mOiIKNZecEP66l7HmOqSjTnqFd44k5thy8dArgBLhHrGRj5yWGanDR97u5nx3Al+gg/Xt7ZfWDbI5JGoHZk7gmhrxEx8QoVpK9AiukgSywWOygsgm5fzhbqBA3uoi520lV8hjj54nZ6/CYijIZrEmzmDY3GK3f+ubMtM2c7JSyKZEmcX7AQWqGrr3FmsZGRKSmYqKwFUWic3PRAALBJvF3OPwijLIpuR9V7UI7+ET8r5hF4WTz4gpqpCjNjNM7br19I3UUOg7SmpkDGWAzWLBwJCVSc/+Yp2lOIR7J8xbdZvk+MkxuRfNmyffS43mOqK1yg1A16hWAJWlNEu3FoVjNgexAcaMWjK5LNvXr0HNlaQ19b2xCzLLLi/vAOI8BuDChHPU4hDfQy5GnmKSdUH+ZJNZ8snjhHHjJ0s6M84IAVLhIWqJkjUc2JRB9dhr2MijbC4s2RSQA9M2kYRlIrNouw/Ge0VwY4KygTKlbIRpQj4CcYcIfKsd7E4xCamCrErsHkmOM2yyv90HO35FktS694kJKtJNoyVHzpEmmbEzllO4yOK1FOQLl4kVHcIxCLTJfUxF4MQPoXqtlLqo2ySKMh1Z0rhojWZBmEdUm8tuUOdzcH4kXqFBagUglGbpNpeYOjzSru3C+TGwetjS5J8+/LI8Xrdpl4RMRvqKphGvNIJ3+iVbtvM5sfWbBenKlUuLEFZWyMTI54oF2wDyxXEZYDPASoGQylDAgdNM4bYlUbk0FjMLjJM1reTyCpM8BlEsFBNuCxkUSo4XGwQs0is4PiTCPjmG5PVZJhrFmMVidO4aGaOrGjwKho/J6sKww7p3ic+i3DF7/Afij7AYYjIaP4E1G8dntWG1OYlmCkQzedyFURx2h/xAqtZIgbzkiAj9QgZGT19cTVy8z01r5b6f+rEOEdVUjvmGKl9GJ8FV1W7OaxNQhSk1Zx89JwIwNkTWYiccj2O234215CmZKV433CNJYA1bpHPXuedFyEYHi7bvjDhoU8W2jGb+YlG2QnKErAkWs4AqyDAoBgRMigsws0AWKGDNxYvPBTs58oaBiH2TQgEp/mmAiYkywcglxU9RyEjTmdIqQUYhZpx8BpS9uHNa8gesHoj0iL3e7pXaPrFeKUlRHoL62hfFJOaqlmvOJUVZ5NO4bFZsdgtjGRuxbJpxi4daFIbNKX6P8S4xVzmrwZKS0tZ3fmrCgT7dj6lp1wxZ1xrNInA57WFnqQQ6ddumoIuD3eOLP/4Z0E5gAJSEdBpW+bC8IcYSOToL9WxtC81tfy7/wJVFZq+uKukW5quTMgwo+dI4A2DzgstPHkXaVCTzhpjpDRH6ZTqgiInYhIp2oTLhX8Igj0EBA5GlqniAbN5CpGCQxE4hE5don3xmyjFKX4OCmINsLhnveJeEkzq84ty22qD97ZIwFumdfP3ZhAhwpaTCqd0rOQdWN1gMrBYL1S5F3lXNi6nVvBJrJNd7QKKDLFZRFrF+cW73HpS2m+Uhorms+DBO/xSOfhe+/0mpzKqMiVDRUg6GRrPQTP2NO/zyvJRTU05iWL7/5cwQyFDrdTAcTS/SoOfm+l8BzGfZlhoXYZfPQTYGziBj8TQBW47V3vxEvZyZmFr61eGXGXBiDHwNYsoo5CYiayJdZAw38WQSR76AxyhgVXaUmVmQSy4pDxOwGwUMUsRzVqxGBrNgwbAYU7YuFEs2FO3+zoBE+0T7ZMyOQDEJLCy5BVOrFSaGpSx1JiZRQfm0CP+xC2IKy2chn8UwoW7TbawyN3PszedZl4lQ43FidXrENJVPi7/CYpWVwXP/Q/IS/A3F6CCXtOVMjomzPBuXlVfznokfo14FaBaD6co7z1S1c7pGUzMEMoR8DuKZPIlMDrd96cXx9b0CmBqCOd1MMdwjNnpPvZhm0jEYP89wSlHt82LZOI8EpPIwRhAhGe4plnn4mZiWrG6JnR8/Sz4VJ931JrZ8FsNQ4rxdIOFfjkLEu40CfiODAnL5AtmCmIZk2WEWI4McxRr+GYkmiveJmSoVE8f14HFRkKNnJFO3/MvsrhXzVyYmkUXKJsoCBf5WKQTnb5A6RG23ceOWdbxrfYDuQi3D8Qy52BAkxmX7Qh4SozB4VFYH+ZQky0X7pSWnrZg05/BLToHdXcwd0KGimkVk6m8cZo5Oa95VzMWJiEm5FMgwzUSy1msHYDi68L//+XB9K4D5LNt69oPhhsSg2LybdpCuWie2+sYd85tRTv3AM1ERpG6/CFWF3Gl7AOIj5KM9WPMJnEYGx0U7/HSU3LkT5Kd5zEVJESgAw0Imb5IBCpgyvoIpKx8zJw5wp09i9+OD8lxZJDpK2USxdb0qyWzl1++phTV3FRvWZyfq/bTfCs075N4HWiTMNh2h3gXra5xYzTSpdIJ8QVYJpCOQGJJVQMGUVVQyLIph8ISEpaYiorRySVkxpCM6VFSzuFyGUL8YMGJ1ywrB6p4xk72mqABG4pUxA13fJqD5LNsSw8Us14mXohkLDpWlPuic33mmRghF+mDN3TB6CumGdU7aIgJZiwNl5rEaSrJkS5E/0yLvTxLy00j8fJlFZ7aEclECBZRhSs6XAQoTpRRgl9m0wzfhiTbzUrPH1ygRQqlRGIxC4w2SmNay+9Lr9zWIID77rMzazz4nl+iqkuinkZNiYoqP4Et04TQKxLIKGxnAxLA6JSy2VDvIVS1CPjUqUUnuaglDjfTIDzCbls/P2yBF8TSaxWCmKECQSp9TTczzLPDmKZp9Epn5TOUWnutbAczHFueuFTND7Ub5YLMJ0sko46aLTeF982/SXP6B7/2yKJ74gNi1E+MyYyjkKKSTUACr1Yoy88ysACyY5MXtWxYWOqOEL25TUgYzbWYBbJgUDMjmIe5qIOAqmlXsPjlQPisOX2UUo4Ny4iAPtsns3umDzhdgXVkkUPn1h3ukmJvdIzP1TEzqCblrpP3dwGG5L8qGzW7BTZps1oLFyGNYHdLDOFkMT81nJHnMNMUfUXI+Z5Pyns0JwdbZPhmNZmGYqXvgfCKDZsDjEBEcT89mCVg8rm8T0NRl2+h56HxJMk+PPikfYPMuiV0381C7DryN5DMpxpUfZ82quSNMSu0Q93554pglxVPdIeccOiYCzOrCpAAGGAouRvVMS2FC+BcFf3mE0NTHJOWQn900pEoPw8KFpINMPi9OaqsdTCXmHqtDzDAWm9jgR85CbEDeV5aJev7T0bNfErnC3VJMzmIVH0NyvFiD6MKEw9hdjT3YSM5wkc5byKfCcl8KxSuI9coxMlH5oZWa3liKmco1a2HVLZJzMdN4NJrF4HIig2bgogLIVEYBVHQFoJS6D/jfiOj6O9M0/2xBT1C+bBs6MVFuILhqsrbe9YjUz4/1QyJMv6UOi01h1K6bHMNb+lte1K13/6UzAH+LKJpCXkw/+SzkM+TzeXL5Ai4DmUnPQr70jzHZE1C+GCjHKH/fkP/kjdlXAlagoAwGMlZabTFx3GbiMvPPxmXmb3WKP8OkWBIiIvkN7XfM7HRNDMs9BskjMAvFJLviqiIxKscq5CAdR5kKnyVDMl9yTltln0JWlimqIHcgmwZffVGhK1lN+ELiBG7Zs2R9VDUa4PIig2bAXawCGk+vMBOQUsoA/gp4J9CNNJ7/rmmaRxf0RKVl29En5e90yRmbHxTbdc9+OPpdhi0eTtskG5iu10UwpqJik65aNSHs939VFEouC4N7RTBl03DhVRFIQ6fFuWwCZoFCITOrjf4SihtPEv55ZjX0G5QpgTk2NyiwwdJFPGkjb3FhWGzicAUJ31RWMY8VsrJCsnvElu8KSjSO1X3pQcM9EvV09jlZARh28FTLtk6/ONeVUQzjjMnVZeIYhTw2A5LY8eUzxQqmZrGMhZIVRGpM9skXZ0veOnE6h7sl+S6Xmb/JTqO5Wi4j3HMmDIv8uk1zNl/g4lFJE9BNwGnTNM+appkB/hl4eNHONldyRqBZFMHmh+g0VuG0GlIhM190MqbDMtPMZWW5l8uKoDv6JBx+XI7jDEgG8Ph5sadbioLLUiyhbGakj+5lcInwnocGubjJPCYVNrJ4SFLIpmT27/QXTUGFYgs7i5ivzIJcf7hHisGNXbg0AqJkE7W5xVZv2IpRPaPy3GItls62yHu5tJh2CjmwWMlZnKTzSiKCSmovly42mPfIdoWcmH8Mu6xSRs6IAkiFJepIJ4VplorLiQyagdLkrlAZ+V9RBdAMdJU97y6+Ngml1EeUUnuVUnuHhoau/GzzjeNt3oW7kKAtf64Yc47MVl3VMvsdPSPx/eeekw975IzEzQ+fgXRcHJjuatlu4KiYPkwTlAWFOf8bPovwvqxVxDxQKAoWh/QJAIkEyufEyZqNi/IyivdivFNWOeWU/CDPf07CPOPDENoINWtEWCfH5Vgjp+UepuNyD612sAfFlg9QMLFggOESp3Op/4DVUfQZBCWL2mIVE5MzCANHoKpdGvN4Q1dkh9VorojLCPecCUu0j3stb7D2wjcnfIhLSCV9AGqa1y7Rg6Zp/i3wtwB79uy5cj3pb7q06blhXFplMtDMYe/buSvxNTEpOANSlnj0jMSdpyPQd0BCHB1eiBVFempMSjpbDPDUyXbpqGTRZpOQHENdzux/lmif2Wz7MHtg6XTb5gFHISV1gjJRsPtl5WMWndTKWgwR9co12b2Tna6lSAil5DF4BOq3iGBOxyXT2tcoZrRov6ymXH4R7GZefjjRXqypGGGLjyp7sbyDxSnbGA7xozj8kqtgrYbatXLMkZOw8X5RFCUu0w6r0VwxV9PPN9yD/fQPcZEi6SgLOFnC7neVVADdQHn8XgvQO8O2V0epWFvdJklwSgzJbHfXI9PeaLvNQqTgFPNDieoO6QDmDIpj17CJCqvpKDpMC1KiYNu/gaEjYPHITDo+BJiyfeEys/2mE/ZlDt7S0xKlikGlfZlDUeSRS7AZJkYhU6yrU+wjbLHKrB2kblA2OSGs05EJIXv6mWKjmmwxPrpKBHW4SwS3zSUmIaMo1NPWYhSPVfaxOiGfIme4MYngVxkULjGh5dIi+C02aNohSsdkovHOxcS7q7PDajQVoWc/OZuXGG4Mw6hI97tKKoA3gHVKqXagB/gAsDiZPOXhWlWr5bV0pBhTvnvytuEebs68xtFcQAxkqXGpcR/aJErAWyf9ep0+CK4GlISVlvDXiePUExLbdymO3bBBdq7ErykU10hlkaCTonxgchLYxY2L20y3xCpxcRR5sBhFG79ZdK7mc8X/m5KUpVTRXOMt2u6TEsppc4spzFsPWOQ6Bw6Bu0GqidpcRY2UKybbWSQSq5CTAVgsxV7DkMoWiOW9BL1uaUxjUXK+Yv9hgqtFYUztE7D5YelyVmpMY9gl30AnhWmWO4lhMkYAALtRXMEu8eq1YgrANM2cUurjwI8QkfYl0zSPLMrJRk4VWyRGRQlUd0zuylVOz35c3ipOZxSphjackU4piRDtlf65gWaZ6Z96WpyhDr8UTov1i5nJ6hbhE2iWRKkD/wSH/xWSJoaRJU0SoxjjP5s/wADyFi5K/3IlANMI/vIdmXvmf/E/F1cJ5TkJxVwA01oUwBYxh7lsMurgajF3Ne0SRZeOQ6RbzETBdnGIG1apJZQZ4aIqivSI4Le5pGJoJgfKIJU3Gcq5SAQ20aA6AZeMJ58VYV7dDo3b5H5Ol4kJTKi0CnnTNJrLxV1Ldlj8bvZSzfklXr1WNA/ANM2ngKcW9SThHpl1Wgxx5OaSEt0TWi/x+lNJDOMPBIFxRk0fTa03FXsEDE7Upy8JplxKwiazCXFA1m2afKxAsyiDSK/MlE2TQsLAzEWwzTOaJ1+a7ZcpAZhdwM/GVOF/qRIqRixhSrhnNlm01Zsi0KtXy4w/HZVcB6cfhk+J8M/nJBonOS5CWylx9jo8xRDOYh8Cm0eOmUuTyacYz7vp9O7ijo2r4fhpMTVZbKJ4qtvFX9N3EPY8eunS+OiTYhpq2DrxWjqiK4Nqlj/Nu8h3PY6XBHYLFel+d32XggARBKHNMHx8onhYNikz2E0PXbq9u5ZGl4SG9o0naQq6Jmvlnv1in/bUwdg5EVYgyiW04dJ08EAz3PFJCWM8/hT2VJSMYSVHHgsGatZicGVKACbZ/kvvzYeptYQKRjGSctqtiwlYNvdEvH11u1TydNdO+ELcVRKDH+2RKp2mErOP4RQBb9gke7iQl9pIuUwx+U2azeTtXmJZRSqfJ+kIcdv6Bozxs4ASZWNzFYeTlfvsa5z+4hYgGUejuWrm2y2snEAzvfV3keRfCSbOQX9GJl2l4IolmMBc/wogMQxVrRLBMnpGZqiOgMxcp9b16NkPI6doDp9jlQXODdezu9E6WSuPnJKKlA4PNN0gYYilMsaldHC4NHO4Zi3UbcaID2FiYyxjUm2ksSsmbO8zMDW5q3xFMC+KG5qG+HktzKU8TGkcU4iDv1lqANm9cu8GDssmvhYYOgoX3hAhnc+J0E8My+povFsifsy8CPSyY+fzOc4kXTjzeaptJrWeHEbPa2BaijV+snJMd62c03CIz2U6FiAZR6O5Kq6iJlA/NRwsrMHlDUOobqKn+BJFA13/CqAkIDw18oBic5GyLNbufZNCRK3+Zm727Gekzwo775nc+zc5LqYJW3F/Mz8Rp17C7pXSE0PHJzsnzTzUbsBZyDEwOEYu343FsGPNz1YUTrik1APlL8xAqQx1mV9gNsfwpB3zGRHodo/MSlLRieSt2k0wdkZMW3a3NGlJjIjgtzolD8AsFpW7uMKxkC/6PrJ5E58Ro8pj4LR7RcBn0xIllS86fdNxGYfNA00dUkp6Opp3yY8FJn48uom8Zim5jBaQUxmJZbjBcha3b/0V7X+1XN/F4GDubL1wjwh/iyF1ZgoZiPeTr93Ey+EgqXVTtLArKLPjbKI4szUkZt7umdgmE5Ps1NEzoixcQfkbH4LUOJZMlKbaIDHlJpXPkb8Mx6Ux5TFbhThlmfySCH/FpWpg6teg6AOwucTPkQ7LNe38Fei4R0w+hYJ0DLM5ZXXl8Mq9S8fE1GaWMgwgj4UsJpm8QSEPhmFQ53Xh9NXKfS8UFaTDDy6f3G9PjehEq0NCbWvWTX9DFiAZR6O5Ki6jBeRUxhIZalUEl9c/+Y0lanB0/a8AZqrjXRIQPftltuoKidOyOLPf7hhi1Oyn88d/xcb29gmbXs06EYyxQQkRDbZKlUxXlcyEB4+KoA/3itmntFLI54uB+hnw1uPIJgnYbSSSWRyGCNwrcexe/j5FJ6+iWG+Hoj6wXsxYvngfrE6J5skkpQ9w/yF578KrokyVIaafTEKyd1OjxXDSPDLjNykAZt7ExMRiKKwWG0bBhOSIKAt7MU/AFZBVhs0lDmCHTZRCw2ZRNLt+beZLuppkHI3markKM+RAJEXeUYORi4Ox9GbM618BwOwCIjEM7pDMWkvCOpehKXWSvKpl75CVja1lGXrNuyTks27jhMlh7IKYR849K2GR7XfCoW+K89LuLnbY6itmv9qhbjOMncOhDArjvaTjGWyGzJYnBLoFLgaLzlY2+nIxixE2Vigk5bkqhmzmM1L+opATB68CGnaAwy2mmHPPQfON0vy9/+BEPL+iKPgt5JUUtsoWCliK5ilLsfmNQQGwAxnAKqGmzqAk0iVH5TLdNaIU0lH5XHzNl/prNJrlxFWYIbvHkjT6GiWizswXv/P18vtcAjPm9W8Cmgt3rdzwTHLCrDN6BsPmgPqt7LsQJmfzTm46PtXksOODEh2z/l3SFtEbgtoNkkE71inHTI+LgK3bDOt+Dm76Tei4G1ewCZvTK/V4CmIxF4NQARGSTsACNr+0ZFQ2EeDzQVnBYmfC5GMpCnu7fMHsHomu6bhHHOVOn5hkrMXM3WC7zOodfinvYHVJNJU7KKWdbc6LSWL5bJyYadBX8BHJO0RlGXashoHNYi8qNlXcr+hXsDnFAe6sgnQCcnGo3yAx/007YMcvQ/2mmc0/Gs1y4CrMkKmRLm5ynJdIRXdIrAelHBsdBbRIlIdsocQUEVovZSJi/WLS2PBudqTqeOGnpzh2ppNtjqGJQk3Nu6RyaDmnfjw5HLFphzhJx84We9bGJXS0acfENvEhMPM4gs1kozZyqTDk8xSMgtjtnQFRTslx2d7MSXSMxQZWr2TMSq1pJjy+Jdt7SbcXs3nN3ERXLTMv5Z5bbxHhm4lAzCmhrPmMlMZWFhHQybBcV/cbsq3NAYlRzOBqMqPdFDJhomkTR94K5FB2F1ZfDc5UL4ayiH/E5oIsxdDSor2/vCJoIS9jsnklesgYFJPawGGd1au5NrgCM2S+YFIXPYKruQqq2+QBM1cpWARWngKYGrI1fgEGj4uT1u6FlpvkYXOyrcbLKmeS4eMvw8Y6+YBnKtg0nR3QMKRqZaBFBF18WJSNu2qiq5XVBbk4tqoWjKSXdHSU8Tyk8hZSZjM1vjXUOAeKZRMSsn8+VQzTTEkdWau12LSlaDKy2Iorh7yc1ywUbfaWCbu+ssv5bQ4YHZSwVm+t5DgkR8Wpm02JAI70gTLIDh6nz76a1OAgXTGFM+NirbJiGBaUK4jHjBNw5sFpA6NWrq1UQC45VqztX4x4ysQhaxXzksMjSqJ2k5iDvCFx/uqsXs11TM9YkioiBAKrJ7+xhHksK08BlIdsxUeksJvLL7kB9ZvEdte0C3r3YwDvqo9xsivJeCJDsGntzCFaU+2AA4fExLLuXRPhp2OdotltDlEYux6BV/4a+t8CSwELCpfTgR0LA3k3g8kCe0+M0ui0sst2FrdRwOYKAA5ID0uGbHJMhKZJsYWjVQR8wRQHbanomsUuAtjqAkOJCSafg+69UNUmVU+Hj4sZLNgODg9JexUD4RQnhoOc7oOd44ewMEgOC+32LHWeKHZ7LXabRc6fKojCCXfJufytsOMDcPKHkgmcGJMVSD4vyiWXBOUXZVbVUbxPQfELtN4k90xn9WquU04ORBkx/bR6CiKLRs/I991ildpjJa4kyWyerDwFUJ45OnpGHI5WZzFBzC8F3I5+RwTkeBc73MMcUF6eGm3igyVBPp2GnhptlMvAqlsnhD/IasDqhD2PTbx2y2/DT/5EFIZhB08Iw3DS5K2javuvUfPKv5LsO8ZIPMU44EwMY7M5cBXA4m3A4amVnAOUmGwMu3xJEiOiHEAKq7mqQBVDO90h8NRKKWVlUMikiGcVqZSCcC/Jvn6OZRv5aWo9KWzElWJVdYBE2zvYk3wBX24Mw+aGtB1UFpLFxjFWu1RBVRbwN0A2KvWXbG4xY9k9co8zcVkJpKJQ0w7Va6BxB/S9CY6g/AhK6KxezXLnCgX0iYEoBwtraKQXzneKzDHs8ruNl5mcr7Lx/GysPAVQbqpJR2S2mUtOrAgGj0t0y6qbIRPDmRqnra2Kx8/luSeSosHvnDlEq9wOePTJYketMqbbr2U3bH2v7Bftl7j7XBqGT+N69o/Y5KoFzziZgpVo3sZw1kE6mSeIhb7OIf4/xy9yu3MNjbY4qzKncadj2NLd2LIx8jYvWcOHIz2IJTqIiSJpC9CXX4V96AyOTJhx00Oy5wg+kvSa1cSopsEaxe7x8mCrk+CqLbSHfNK8Oj4CB/bDcK+YorJxKXltsYs93+kDT718WWvXSimISJ8I/uDqotPYlHvtqZcCew1bJifApMblM5ntnmk0y4WryAI+ORDFCDTjCpqQGZ/oedF2mwRLlKoJXGGS2XxYeQpgkqmmaO82zYmmLxYLuOsmyjqENnNn9hA/uxDkm2+c5xO3Nc4c4jXVuRwfkkJlc4WGmQXY+G4xkZz8gUQB2N2iDCIXIDaI3RmgxuOhJp8hb/UQNQN4EhF211gZHfGwIf4q9vwQqphvWyBLNpvDyggxbFiwkMYG+STO1EGsBqRxUWXEabNFydv9NHrtOLxeHOkc2GMQfxr6T4HjdvFbnH1OfCaGTVYShnWiFlI+IxFGiREpHzFwRNpmpqOSQNZbNL2V34vND8vrIK976qXNZu36iTr/OqtXs5y5iizgI70RNjb65bvefvvkpkalApSwqLWuVp4CKDfVOP2iAEKbRcB1DYoQq+6Y2L6qFXcuwT3bfTz35jGOXjDY3OCTqB93rZSAjvQWawR1SkXQ4CoRXnHE1p1JXJqAVk5pVTJ6ZqKReiYuwtPmltlAPgkFN1hsGGaBoCUDAS8faouCZxASdeSHwpBLYVpcmDjwZBOAwqYg523EQw5LPoPKpzHctWB3Qi4HsSjkxiCRAZUuhnbairV4InDs21C1DpLDQEHMWJloUfgXs4bzGS42cQ9fkOsKrhazUO9+8atEei9NxvM1TJjNAs3Q+onpt9NoliNXWIwwnMxyejDGwzc0zZ1Itoi1rlaeAoDJpprSrD02CK4a8DVNtttnYlCzjntu+Xm+fPZf+cHBV1nTuBunxy+z4cOPQ+vbRFBaDHEq231yjKpVEhM8NWR0KqVVSWwQkhHJFi714bVYZcYN4KoVJ21yRM5RCtuM9UDdNozxrmKZT5usavJpsNgxCjls/pD0Kh47LwI6n4RkSqKFDBcUEmKqKSoNhk8CFrkGpaDntWJjdhMyo/KeUXQwKwPqtop5p1QTyFsvY6nfJsok0jv9fZg2fG7xw980mgXhCrOAD3aNA7BzVRXUzZFItoi1rnQiWKBZBNOex6Thi2FMWzfIbrXwxzdmCCfSvPjKi9KFqvMlma3HB2RG7KoSp/LoGTn2fOt5lFYl7hopqYySL5DdXSw2VyyX4KuTiBkMceI2bBGTVSoC4+dl9m1zylLSzMlqxuEV27y3Fgm/DMvrpiq2XEzLOJ3eYtG3cNEshqyQEiOyislnZaZjLUYTURATlTIkfj8dkfM4qySiyFMz0bpxieqaaDRLzly1xmbgzQvjKAU3tAZmTyRb5FpXK3MFMBNz1A3aYO3nvc0R3uhJc6pmFesyEalaqZSYgnIpEdapcTne5SzVAs2igPoPFbOH8yJsU2H5W7tBmqwPnpC+A3WbJkok+1ukI5e3UcpP2OyABbxeKeTmbxEhHusDbGLushSTxsy8KIvGncVlqylWHU9IjpPPSoKcwycrEM6J0Hd4xcZvcxbbP9bC+ndLxnAqLGa00kpKO3I11ytz1RqbgdfOjbCxwY/PaZs4zkz7LGKtK60ApjLbzU6Os72lipPRJD85PkT9Gi9+omKvr+6Anr1Szx6LOEzjQ7D6djEzzecDDDTDlvfA2Wdh+LTE6desk5m3Kwhtb5cZeWlWnYoUI4eSsjLwN0pWbyoiSqmqDTwNIqxjg2K+8RaL3qUiYLfKysLmlIxjp18ioGKDohgKpgj5bFLCNNfcKauD8fNierJ7RCl5q0XJeEOgtknDmIHD4tjSjlzN9c5lCuhkJs/ezjEeubVtEQc1P7QCuByUwhjr5OFgnOPJcS6cy7EhaMEWcsmsunYj9LwhCqGqTYrC2RyXF7e79h0iXDfcN8XmV9w/MQJdr8o5wj1I/14HNO2UFcG6e2Up6gqK8ijFJId74Ln/UWyNWQWxYXFcZ2JismnaJZVMHT6pD5QYluWs4ZCqpsFWUTxb3istNRWSPBcfEj9FyXHuqZH8h943tSNXo5mG1ztHyeQL3LYuVOmhaAUwb8I9kqBh9+KMD7E+YHJuNM+b4SA7A0lsXW9MlGKoahPnZ7kzeb5xu3MtKde9U4Ru9xsyW7e5xLnbfqc4W2dyOvfsF7PR0EkxVXlD4u9IRWSVYublfYu12MGsHTAlgctdK0oiHRFFF1ovoarOYgibv3HytdocUmBuLue3RrMCefHUEHbDwk2rqys9FK0A5k2pt/D4j8DfgNvuo9Y5zHDvOP8y0Mr7g8PYO4omD6VkllzuBL2cuN257IGlRvNKiWAu2dvLY4enkhiW8FS7r9gac1xm8IFWabZeItwDp5+B3gPyvPVtonRK9yA2KOaeTQ9NrCxOPCXKQXfk0mhmJtyD2bMf+8GX+O2GEK7kdrBXdmWsFcB8KfUW9tZJqGQ2Rn1VFRl7NUfORXn+RIpb17pxOwPFyJpiNJCnZuGdoIFmmWHnEvMPP5tPa8zSsXc/Io/pzjvda1fgBNNoVhTFiVJv0saJmIvHNrqWrO/vbGgFMF9KAtTfVCxx7IZsgtZaBw+oTp486+D5Hxznd25ppWb8LTHNhHsgewXO4PlwuU0oFrN3ru7IpdHMTjHx9I1zUVCKbR0toJIVL3So8wDmSyne11MvoZ+JEfnrrWNVrY/7bt7BaCLDZ382QKdrkxQ6Gz8PmGKftxc1fqnA09VyufHBuneuRlM5EsOYNg97O0dZV+fD77Qti/wYvQKYL+WmjlxCnKOuoNjDW26io3c/f/AOO3/5XB+f/1knv9U8TEfTWiyFnMTmV3dAbAS+/3sSPulvlFo4LXNkvc5WafByZ956pq7RVAZ3LZ19g/SFU9y7uUFeWwb5MVoBXA6zCVBfAw09+/nM3Vm+uT/NmZ4BTsVd3LGlBm8+DSd+AJEeWQnUbRAn7MtfgFs/MbMSuIpKgxqNZomZbbLWvIu3Xvo7qow0e9oCExnDFQ6W0CagBcZtN/i19XnWtK2iL5LiH1/r4q2BDIVIj/gO3CGJxXfXSEjl0e/MfLDySoOl6qSl3sQajWb5UJqs5RIyWSt1DiyafJOuBj7fs45NqxpwZ0aXjQlWrwAWgikzdXX2OdZbBmhtTHIk7OCNE1GC9iFqfE48vsaJ/ZxB6YE7E1dYafCKmW0Gs4hdiTSaa545ykJ/+80ezqQCbP25d8GampmPs8RoBbAQTG0zmQqDzYbL4WF30GDV8DCjo4quUYPRk1FuWWOn1usQM5C/TCFMFbLKsqilYCcxm7kJLs8UdTmKpFROWysWzbXMLJM10zT50kvn2NLk56b2yid/lVMRE5BS6v1KqSNKqYJSak8lxrCgJIblwwaJ/Q+0SsXNQhZVv5W6NTfQ0bGVxlAdsfFBvvvaMQ6+/jNiZ1/HdFWJUJxuCRkbhLELE5UGR89LBdKRU9JxbKEiimB2c9PlmKJmWwpPfS/cI36QSPe0y2aN5pqhFCZeTnGy9sKpYU4PxviN29pRSlVmfDNQqRXAYeB9wN9U6PwLy9Q2k97iDD3SA70HARObu4Y1Nz5C47EfMnb6NU7GXfx5ZBvuV2Pc1fd1trTW4vL4pfja4N6J5tDeBrEXDp24tOHMTLPwKzHXzGVumq8paupSOJeFkTPw/OeKdYaaJt6LD4gfJDYIVasXvN2dRrNkzJJn89f/32lCPgc/v71x9mNUgIooANM0jwHLThteMZe0mRyT8suuavCFpDmLaUKkG1frTlwdt1JjeDHPDPOTY4P8w4ERbj70GjTv5A5vFw01VVidQem5O3AYtv8bOXagecLMNHoG4oMw3iVlpMtNLPMx10xVEqiF6UpUrkjiI1Ih1eaSshXJonnM4ZVs5FJPZt0EXnOtM0NG/CvDTl49O8pnHtiMw2pUepSXoH0AC8F0bSattol6/dkkNO+R186/CuvegUNZuHtDHXe1WBg60Unh3CnG+o5zOh/gp0YrdTUhOoIWmgLVeEqC2ltXFKr7JJzUUydftgP/JPX7MaWnbvkse7pZ9XRKIj4kLSwdXqn/nxiSuv+7HpG2jfPNIi5fDY2ekaY2IDWLQPwepRIZV9oEfiEd0tq5rVkopoSJm6bJX/zzq9T5HHzw5lUVHNjMLJoCUEo9AzRM89bvm6Y5S+zjJcf5CPARgFWrludNBC5tM/n85yCXEcHXvHmiWBtMCMj4COrsc9Qlu6B5NaHxblqyKZrzfRwczfLqABw019B6/BXqG5tZV9XNuswxfKk+FHnxM7iqRKAmhqFuC5x/RYSqt07MRY07JswsJaaLWKhaBQPHofNFyCSlWmjDNunnu+Hdk2c3ygIW20Rf5CnxzheVRamZTTYt9wCkimlsUO7FlTSBX8jcCJ1nobkS5hnI8NzJIV4/N8p/fXAzTtvym/0DKNM0K3dypZ4FPmma5t75bL9nzx5z7955bVp5jj55abG2dKTYXjEjQmfgGPQflMYvdZth7Kw0eCnkyVevoSd0B8eHUhwezPLtwXoeKDzLz1neJGoN4HPYqLZlcFvBUruWgAMcCjEZKcDiKLaCbISWnZKxvPnBCeU0tZJobAgOfB3qN4tSyRWb2ddunGibCZOF5nT9Ckrb9OyXtplWO/haIDU6cf1WhygXZYHEqPRWBmjaIf0Qpgrf7n2SLxHpkz4Ijdtk2xKj5yHaC9XtlzeLn+kzmk8fZ83KZOr3f6xL+nOsunnCN5eKkF17H/d9+Qz5gsmP/v0dFTf/KKX2maZ5ScCNNgEtFjM6hYphlT37Jxq6hIrtHavWQD4PhRyGt45VoQCrrGHubbbyO2N7SfScJxOD8XyK3ryLF8JB2ukiOXySHDbc1gJVChrVCA5LnpQzhIokyCdNxm+8i+D501T3PIPD6hD/Sz49UbZ68Kj0N3ZVi3KwFU03sX6p7w8y3tf+Rmz5nrrJbR/LTUyl1VDzLjFPDR0RpWLYRID7GqFhu6wuAk3SX6B0f8oJ98Chb8Lhb4OnGtz1MHpaxjp6HtqLK4WhY9LW0nvz5c3irzTPYrHMRtoctfyZunqeIZDh+ed+zJmhBv7vr+2puPCfjYooAKXUe4EvACHg+0qpA6ZpvqsSY1k05iqTXPp7/kXJDAZRAsFmiPSL0zibEjNKYhjDFcTncoG1ippCno7GNbzdFSLZXSATG6DfqMMa7cWaSRMrOIjlsiTDMWyM8+OBRv7XkbPca3kDFymclgK7rWfJWpzYLArb4R9Ro8IklR81eo681YVCYcHEnx/hhL+W3tPPsDn6Es2JU8RtNdhUL47CWXq9W7AATfFjDO87QtJWxZBvC0lXPUopNvVCdQKsDJGz+gi7N8CQjaqzXyMa3ICyZ7Bax7EZFpx5hWX/cyQ67seTGqSq62m83S9gdVZjNU1U734wrGA3YPgYkIf4qMzi3TWQGJteIc1Eub+ixFw+iNIMMJ8X5di9F04/Lb6Sueo6zYY2R1We+SjgqZOGaQIZxnJ2Xjx4jNvWbuUdm6ZMMJYZlYoC+jbw7Uqce0mZq/ha8y4YOi42fFcBUNKEvfUmafrSsx8KWen4ZXNL8TmrTUxGiTEshh1PsB6P10+VtxbOd4PpBOUBXxNZFOlUgl8MNHHD6hYa33qCdLZABA/jbMCX7KEq1Y1RSHJetTFcCBBMjpIwsyRNO14zSoI834qH2FB4jdfNHKtNBwF68BfieM04Hs6RwM5+s5pXCmk8nMPHUZ4u7GaAGn7Z6GPIbMSkFPEVRWHybstJXi7YWK0G8JEgiptR00O7ZZCjz+yllUH6zQD3Gd2MmH4a1BguI4+FHFYKeC2jjI+YVJnjJF2N5HJWHIeep9C4G09VLYHkAI65Pp8rKZHds1+E//BxcXD76iXqa/9XxVl+pSG5c2SSahaZ+SrgqZOGKYEMpmnyrVeOM5D38ccPb1n2kY7aBFRJSt29Tj0NfQfltVW3TNjBT/1YFEIpyczXCMOnAFO+pK03icBq2gX9h+DsC0ABfHWgFLZMHFugHm/ARWPhdWishnQCEgMQPQR+F7SuAX8Tm7wNYssM3QC5dDEKyAO7HuE9Lbth75flnCN1cPx7EtNvrYfhk+Cwwqbb+Y3QWkzTpJCK8Ec2N/kN96GO5TCzCckHGD0DqQgFixWSTdxHhKyjgZxyQnwIx8hRYv513NCym6oLz5A109jjNTQXCtgzFtKmA0s+Tcz0UsilSeaAguLNpJORERMnETInX2TQDNKgxhj6wXEsnhDxmu346ttoqXLTWu2itcpNS5Ub13SrtKa18nw6BzeIII/1i/Avmclc1fLaVGFdEiqFnERW9RRXC+vvl9fKlcJSl/3QTGa+CnjqpGFKIMObp7o539PPLXf+EmtC3qW9hitAK4BKE2ie3JKxHHet2M1zSRE2Th8EWiDaJyYiq3vCrFQyP3S/IY7kfBZCG6CmQ5ynTr84Y3u/V1QoJqTGpBdwwzaobpP9S87U5t2ThV9p5pMchdpiNdP0uPgNQlsgfB5So6h0BMPuA6cfq9WAVXvEDzB6RvwAToeU0s4n5NpctbKqiY+B2423toqGlmowV8s5sqvkB5Z1gAngKobA7qYl2AL5HBvyWZI4SeQNCqPnyOQGOOfaRlehhlh0HP/Az/jW2e105aoAqGeEGyxn6XAncQXroXkXja072KxirL3wnDjTZzLvuGvldV/9xOeUS0qRv6m13Xv2i6AfOilj9jZI3saLfwFb3zs5oc9iW7qyH5pLma8CnjppCDRD1S/BhVeIn3udN0/ksAZ38MGWYZk0LXNfjlYAy5npTESFzISJaOqXat07xWQ0NULHFZTnqePicE2GpfyCxV58Pir7V7WKw3fPY9OP5cRT8qX31oHTC5kaqDYgn4KBI9C8U5bCyVF5lDqgeYrCMZ8RAde2FbpeAcMJhkP8HPkc1G+lKOXFwdz9hvhHNrwbzvxEMoqr10DHOyDWJ4qk7TYMwDt6Bm98EFwK1tzNqlLv43QELDY+UZtjeMPP0d91FvPEUfpTVZyP1TA2PkZy/3f4+ms7ucFylnVc4Db7abwOC3Z3EI/PR9WrX8LzNiTUb+QUhLtFwQZbRfhnkhBqvVRYJ4Zl5m93TawWckkwjEuzn7OpCSe47q289FyOP2hqyPeJp8jXbeLzRwLkC+f5ZNMBjIhHfk/L3JejFcByZi4T0XTbT+d47tkvX8R0RKJ3vPUSAWQiwjkVlv1nm3GWjj3eJRnInrqJ2P6D3wCbE6xOEXCmCaHNZctnE9rvkLDPEu6QJJ+1vkOed71ebFQflOeeGomOGjkpJTCqO2DN3eIPMQvgDhY7mxU7K1k3QqoJ0lGwecTcYncDCsbOYrnwKnUWRZ1pwtpV3FD2Q88nx/lYysr4BYWrp4dkGrLpBLnEMGPDitMFL6OH+xmu3UN9bQ1bXNtoHXkVRz4LVW0i/C1WUZIwYffvewtGT0HtZrAVT5aKgKvm0uznTEL3Vq4kV9oytWg6+vaRCKeH4vynLXa8gVqJDqpuK+b7jEn03OWGKS8BWgEsZ0qCBFOawM/nizOT4/nEUyKksnERxHYfYBEHpmOeDSoCzVJ2YmoegNMH3uZimQe/KIby5LPpZle+elklpCPT2lLJxCATFb9H1arpcw5K92eqsjv/kgj/fE5yK0wlq5bBY6J02u+k3ENsOP005AZpCOZhLAMuEwwveQIk42FWJce4YAmQDJ9mvH8fPzRdJM1VNLjyOEMWGpoctGy5jdX+JlS5M7Fpp/hI+t8SM5vVVrz3rulnmrpjW+WYK2pvJhLDvDpo8INDfdyxPsSGYFgmMSUFHx+58jDlJUArgOXKQoYFlr7cp56Gzhdk1t9+hyiDwWNyjnJ/wnyOVf5DWXffpUItHZlYTUw3u7JYxbYe6Z2wpbZ+YuK5u1bGWX7cqY65mQTm0SckkznaJ8IfINgm5jFPCAaPgPeuie1LAjg5JmYqZYLFjVHI4nU4IKfY5o6yraOeDK0Mj0cYHx/jWKqW/zZ0CyOdGXj5FNWe8/xm7WE21lhpawrS5lRYA61iyup8Edpuhdabxaznrbu87GedH7D4TP0+hXskWXCW+3467uRfXjnE+voQH7xpFfQOTi5vMnoGLBZw1k1U0oVlE92lFcByZaHDAkvO5nXvnBAm/hbY9NCVKZTpol1g+uXzrLOrqbHzZc/3fnkiAqrEXJExgWZYfbvMuhIjYm7xN0k9puSo2OLHz0OwfcJGO2msLXI9mVgx9NYlisHqAJsbO9AUqqbJY7LZ7uF973wH54bjvNE5yhudYyRODfBPXS4CBy5wk+0swUCAVt8u2unBlRqDxu3yGZQrutkUr84PqAzzuO+nB2N8/Fl4lzPLh2+px2ZBth09BzUbRMHHBmWyU90xcexlFN2lFcByZbHCAhfDzDCf5fOVnPdKErVgwhkOktWcTsDwCXF42z0SfTR0VBLIatZNHuvpp6VERi4pikFZxPGsLBAdFGWQCYNpgaYbUEqxJuRlTcjLL924Co5eIBwNM3T2IH0jNZwZLzA0Msg5lcHsH8M38BrcdAO7t9+P1zGPn5/OD6gMU5s8jZ6R73ax+m5Xvopf+bvXyBkh3vfLH8abOiHv+1vg1pvKFHyNNH0qJSjCsoru0gpguXKlwq9SLIZiuVLH3FSTVyoCteskdyGTFEe61XZpzZ9As5il9n9VlER1h4RuDh0FqxeGj8qszhkQh3dybCLSqYS/icDppwnEjrO2to7b6uykxuJcUK2cjDoYGujhwLe/zH/59h6a29Zyx7pa7lgfYmtTAItlmqSh6SYC2bRUlS03TYA2Ey0kk6rvFgMKvHUQH2TszSf4d6+GSOWC/PNH3kZbgx9YP+UAxZVsaSVR8nUts+iuihaDu1yuqWJwV8t8iq6tBK7W/j1T8bvS8ny6kNfpqj3u/6qEpLqqy0I/108U2SvtV0r8OveS1ExKR6SgXk07ZBPklJ0LGR8DPefYG/ZzaNzOwcIasp5GblsryuD2dbXU+51yzKkF6+IjUj7EGZDqr4NHYOy8rGyab5xi1lph35XLYa7vVem+Dx6XiDmbG7IJRlKKvzxoIVqw89hvfIIbWoNzH6/03sgpCV12BWXluYRKWheDu9a40qiE642rXVkEmiWCamrVz7lCXqee89xzIlRT49NHOsFks4HdJ4X2hooCJJuATAJrdSNrUqdZ02Byy56NxLsPMTr4FC/ltvDVU+v47sFe6hnh3dV93FiXZ22dn9WuBPaaNpkIDByWc/laoPdNcZJTkNXI8PGJZjul8ay078t8mI9fZWreSzbBwOgo/+dEkITFw6dvr6KhXPjPdrzSMWP98v/ShG4Z+HK0AljO6LDAheFKTUnl1KybvnR0uRIpN9d4aiSbOjYkvoPadaI0SlEhygW9+/E43XhaV/MBM8Yv7Ypyzt5E99G3ODRU4LunCzhOdtNmjBCoSbChwcMWFSXYcQtqvFOEfz4vmd7ZjGRl9x2AtT83f3/RSowwmo9fZUrey9GIi6+e8GHxVPGf72yiNhi8vOMtU1+OVgCa65+FWE3NR4lM9dt4aqD97RJqW7dR9rtQjAqxGBP1hEwTUuMol581/c+wZtsW7nD4SefynOiPcupCHwcG0vzJgc3ca4nTePIM9/ov0Bj00Gr2YjMRZaCUrBAad4iPYy5/Ufc+MW2ZeUnMy6Vklnq9m47K7fulbPFi+ZJJBJpJ7f5Nnv72V3jmbIK2xjo++rYGPGZ8wu9SfrxypirgZVrrSSsAzcpgIUxJcymR+eQ7lKJCRk6LIxnEp+Ao+noifdJcBHBYDba3BNne7OcXYoN8tONu9r4VInnoSU4Pp4gNnaNTFai15wl4rARUHJ/bhzFwWGpATbfCKbdHn31OhFJwlYxh6KQ8X8ys1eWw4nDXShOiUo2m6cqXAKcGonziG52M9a/mP26J8QsbnBgeHzTfOXuFULjUxLhMgzq0AtBo5stcSmQ++Q4le7HFJrkJSolDuXmzCAR/44yCoqXKTcudN8GOZvInf0zy1S/Rn/dzLFlFZDRK89gI40aQ4EA3Y7vfx42FKlrKx1duq05FJOktMVIs0ueHVAwuvCbjXbUIWasz2cqbdk3bUnHW41yNEmneJeG+FmPa8iVZbyN/98I5/vKZk3gdVj736H3cvXGWuv7zWR0uhBlyEdBRQBrNUhPugdPPiGPZE5JoHptjorR37/75RX/t/YokvBVyJC0eTmbrODaY4kB/mn+O7QCgvdbD29fWcNvaWm7PvYbHkhblcvoZUVKZhEQQhdbD4AkRqqveJgUHYWFbZB59Uq49PiDHdfilGGC0F1a/fX7RbgsVHffCn8t+maiMo7oD3FWcP9/JR49tY6y/k0faRviVbR781Y0Lo5QquPrRUUAazXIh0Ay7H5GifiWBUF6Kw9cwP39FWfVXl93LDZkYNzRE+KVfvJ/fzAR5/uQQL50e5tv7e/jHVy/wQeN53NXNbG7yc5PVSr0jiC0dk8gmswDJYZkJp6OiIBx+qGoXJbEQjJySkFWHp5hsl4KeN2UFMpdzdFKfaYdUjnVcRWmFKU79gUiKp352iFe6UtjcfXzpliG2tLfMP2JnPibGZRjUoRWARlMpZhII8xUUM5icVKCZtcDaOi+/fls72XyBg13j9L7Szdm+IX5wOMFhM8JdxmHqHWkCTht2ThDIpDAKOWk65PRLE59onyTOTcflzmiT4xIBVSqNbXNDIS2ltcuZ6hwtn/UrJY+evdC8RxztV+JMLZpkBiIpfnAixlvnugmoFHtueR+/2jaCi5ZlF7GzGGgFoNEsV+YjYOehLGyGhT2rq6HqATjxFMlMhpEzFxgaayQc6ef4mIP82Ag1RpqQA+zpPFXuOIH4GFZPtawKpkuOK5mq5lujyBWUirHZBFhdYnu3OKTPczlTnaPlIZTOgORV2N0SweOpmbz9XPcs3IPZs58z5zt55cwoh3ojOAzYs3YN977zfmqb10x0vytnGUTsLAZaAWg0y5HFKAJXXDG4XvsbWoJ2Wpq3QPVDjCkfI4efhv4jvJVuwjEyiHM4QkEZpFx+3JHjNHSP0FRfTzAYQmXjEj5at2nGWjnTjrFmHdhcsl0poW7VzbLKmK1UQnkIZXWHJNjZio2EysuYz3HPes+f5vQL3+SFrjTnooqQPcvPra9izzs/RE1z+8T5lmnEzmKgFYBGsxxZrMShQLOEeHpvvtigpwqoqrIDVaytayPFevrCKXrHE6jRXk5d6Oa75yBGnIDLxuoaD/cwhidxlqq8k8DIQZRjolbORaFbGm/5qiHWP5ETURL26+6dvTpquUAuJdgNHLq0LerRJyfdM9Puo3sswaEfPcX/Hd5MqPtpXKRoqKvj128IsaetGns+BuHDUK4AlmnEzmKgFYBGsxxZzMSh6Wa4hl2a72SSOO3QXuOm3Z2ChkbuDLRxwdLCuZEknSNxOocTPBcxqek5zvChfny2Ah6PjwZ3noDHi9NlofatpwjYweIqm4337p8I+ZyzLHgZUwWy1QY1ayevhsI9ZE89w3giR0/awVuJGg6MWIgkM4RUmGzDRj64xc2W9dsJ+V1l1z3NPV1BZVi0AtBoliOLaYaYbobrrpGHwyt9jGP9oAzY9Qi2SC8duQQddfUXD5Ea8jDeeYBoIk13xk8kEad7KMq3ewKET3ZzpzrIm2ojLm+QWq+DOr+DkD2Du/8gqbU/T02dnVqvg2qnHbdpotQ0lVBLBJrJr7ufTNdeMkM9jKsA3a6bOHcsx4XRY4T7O1nX9yRtycM4VI6UaSdk87G77m10NNaxcdVWgrtuh6PjEvlDmQKYT9/f6xitADSa5chimiGmm+Hu+KC817NfkqOad084UMMNl4zFabfRcNdHaTj6HdYlR8DTjlm1hnuVn9HuU7jOJ9hk6WM4O8qxRIhXhp2kMjlCqpNvvFx/yZDshgW71YLDKn+thiKdLZDK5kllC1Tlh7jBcpYaFWHE9HOwkGaAGuxWCx/2vMAm+xC+4Goa1DA+lwMXWajqg6Y66Hjb4t/TaxSdCKbRLFeWMnFoHtEz074/NTFrrAu6XpXkslLYZiYJzbtJYxDN2+lp+DlG4mmGYxlGYhlS2TzpXIF0Lk8mVyCdK5DLF3BYDVx2g5rCEFsiL1Fw+DCcPqptWaosSRxbHyDUtAbL9/8D2JxyzlQEov3iZFYGPPx/5ncd1zk6EUyjudZYKjPEfCKOZstZKF9NRHslssfum2ikYnPAwCEcNWtxbHg3tYHg5HPPJZCPPgm5tZdWYo0chZZSq8XiRNbpl0cmJo1zriBsdiVhqfQANBpNhSmPOCo1Lnf65fX5EGiWUhF7HpMIo+CqYrTOHin1kMvIY2oIa0nx5BKieHIJeR7umXz8xPD0vaETw/L/xhskySybkMigbEKeN95wpXdkxaBXABrNSmchI46mhmx6aibqCU2dec831HUuh/i6d0J8SK4jOTbRznPdOy9//CsMvQLQaFY6JQFbzpVGHDXvEjt8OiL1hUqJWuX180vMNbOf7zEDzeLEbrsNGrbJ3x0f1KaeeaBXABrNSmcho2MuJ4Z+vqGuMx0TxD+wwhy6C0lFFIBS6v8FHgQywBngMdM0xysxFo1mxbPQiU/zdbRejuKZeszFKJWxAqnUCuBp4D+bpplTSv0P4D8Dn6rQWDQaTSWiY65G8SzTHrvXGhVRAKZp/rjs6avAL1ZiHBqNpsJcqeJZpj12rzWWgxP414EfzPSmUuojSqm9Sqm9Q0NDSzgsjUazbFlIx/UKZtEUgFLqGaXU4WkeD5dt8/tADvj6TMcxTfNvTdPcY5rmnlAotFjD1Wg01xKXE22kmZFFMwGZpvmO2d5XSj0CPAD8nHkt1aPQaDSVZwVV7FxMKhUFdB/i9L3TNM0Fajiq0WhWFLqsw1VTKR/A/wF8wNNKqQNKqS9WaBwajUazYqlUFNDaSpxXo9FoNBMshyggjUaj0VQArQA0Go1mhaIVgEaj0axQrqmOYEqpIeD8Ep+2Fhiec6vKsdzHB3qMC4Ue48Kw3Me4GONrM03zkkSqa0oBVAKl1N7pWqktF5b7+ECPcaHQY1wYlvsYl3J82gSk0Wg0KxStADQajWaFohXA3PxtpQcwB8t9fKDHuFDoMS4My32MSzY+7QPQaDSaFYpeAWg0Gs0KRSsAjUajWaFoBVCGUur9SqkjSqmCUmrGMCylVKdS6lCxkN3eZTrG+5RSJ5RSp5VSn17iMVYrpZ5WSp0q/q2aYbslv49z3RclfL74/ltKqSUvMD+PMd6llAoX79sBpdRnlnh8X1JKDSqlDs/w/nK4h3ONsdL3sFUp9TOl1LHi7/l3ptlm8e+jaZr6UXwAm4ANwLPAnlm26wRql+sYAQM4A6wB7MBBYPMSjvF/Ap8u/v/TwP9YDvdxPvcFeDfSoU4BbwNeW+LPdz5jvAv4XiW+f8Xz3wHsAg7P8H5F7+E8x1jpe9gI7Cr+3wecrMR3Ua8AyjBN85hpmicqPY7ZmOcYbwJOm6Z51jTNDPDPwMNz7LOQPAx8tfj/rwLvWcJzz8Z87svDwNdM4VUgqJRqXGZjrCimaT4PjM6ySaXv4XzGWFFM0+wzTXN/8f9R4BgwtbnBot9HrQCuDBP4sVJqn1LqI5UezDQ0A11lz7u59Mu1mNSbptkH8kUH6mbYbqnv43zuS6Xv3XzPf4tS6qBS6gdKqS1LM7R5U+l7OF+WxT1USq0GdgKvTXlr0e9jRfoBVBKl1DNAwzRv/b5pmt+Z52Hebppmr1KqDmlqc7w441guY1TTvLag8b6zjfEyDrOo93Ea5nNfFv3ezcF8zr8fqe0SU0q9G3gCWLfYA7sMKn0P58OyuIdKKS/wOPC7pmlGpr49zS4Leh9XnAIw5+hVPM9j9Bb/Diqlvo0s2xdMcC3AGLuB1rLnLUDvVR5zErONUSk1oJRqNE2zr7hkHZzhGIt6H6dhPvdl0e/dHMx5/nJBYZrmU0qpv1ZK1ZqmuVwKnFX6Hs7JcriHSikbIvy/bprmv06zyaLfR20CukyUUh6llK/0f+BeYNpIgwryBrBOKdWulLIDHwC+u4Tn/y7wSPH/jwCXrFoqdB/nc1++C/xaMQLjbUC4ZM5aIuYco1KqQSmliv+/CfkdjyzhGOei0vdwTip9D4vn/nvgmGmafz7DZot/HyvlBV+OD+C9iNZNAwPAj4qvNwFPFf+/BonMOAgcQcwyy2qM5kQEwUkkomSpx1gD/AQ4VfxbvVzu43T3BfgY8LHi/xXwV8X3DzFLNFgFx/jx4j07CLwK3LrE4/sG0Adki9/F31iG93CuMVb6Ht6GmHPeAg4UH+9e6vuoS0FoNBrNCkWbgDQajWaFohWARqPRrFC0AtBoNJoVilYAGo1Gs0LRCkCj0WhWKFoBaDRXiVIqqJT67UqPQ6O5XLQC0GiuniCgFYDmmkMrAI3m6vkzoKNYV/7/rfRgNJr5ohPBNJqrpFjN8XumaW6t9Fg0mstBrwA0Go1mhaIVgEaj0axQtALQaK6eKNLWT6O5ptAKQKO5SkzTHAFeUkod1k5gzbWEdgJrNBrNCkWvADQajWaFohWARqPRrFC0AtBoNJoVilYAGo1Gs0LRCkCj0WhWKFoBaDQazQpFKwCNRqNZofz/D2hynVn5WFQAAAAASUVORK5CYII=\n", 68 | "text/plain": [ 69 | "
" 70 | ] 71 | }, 72 | "metadata": { 73 | "needs_background": "light" 74 | }, 75 | "output_type": "display_data" 76 | } 77 | ], 78 | "source": [ 79 | "t = torch.linspace(0., 20., 1000)\n", 80 | "class DuffingODE(nn.Module):\n", 81 | " \"\"\" Duffing oscilator with friction\n", 82 | " \"\"\"\n", 83 | " def __init__(self):\n", 84 | " super(DuffingODE, self).__init__()\n", 85 | " self.xl = torch.tensor([-2., -2.])\n", 86 | " self.xu = torch.tensor([2., 2.])\n", 87 | " self.NDIM = 2\n", 88 | " self._J = torch.tensor([[0., 1.], [-1., 0.]])\n", 89 | " self._R = 3.5*torch.Tensor([[0., 0.], [0., -0.1]])\n", 90 | " def H(self, x):\n", 91 | " x2 = x[:, 1]\n", 92 | " x1 = x[:, 0]\n", 93 | " return 0.5*x2.pow(2) - 0.5*x1.pow(2) + 0.25*x1.pow(4)\n", 94 | " def ode(self, t, x):\n", 95 | " with torch.enable_grad():\n", 96 | " if not x.requires_grad:\n", 97 | " x.requires_grad = True\n", 98 | " grad_h = torch.autograd.grad(\n", 99 | " self.H(x).sum(), x, retain_graph=False)[0]\n", 100 | " return torch.mm(grad_h, self._J.T) + torch.mm(grad_h, self._R.T)\n", 101 | " def forward(self, t, x):\n", 102 | " in_shape = x.shape\n", 103 | " x = x.reshape(-1, self.NDIM)\n", 104 | " return self.ode(t, x).reshape(in_shape)\n", 105 | "\n", 106 | "ode = DuffingODE()\n", 107 | "y = odeint(ode, torch.tensor([[-1.5,3.0]]), t).squeeze(1)\n", 108 | "y_meas = y + torch.randn(y.shape)*0.1\n", 109 | "\n", 110 | "plt.plot(y[:,0], y[:,1], label='true trajectory')\n", 111 | "plt.plot(y_meas[:,0], y_meas[:,1],'o',alpha=0.3, label='data')\n", 112 | "plt.legend()\n", 113 | "plt.xlabel('t')\n", 114 | "plt.ylabel('y')" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "## Defining an ODE model\n", 122 | "\n", 123 | "Let's now define a neural network model for an autonomous ODE." 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 3, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "class ODEModel(nn.Module):\n", 133 | " def __init__(self, num_inputs, nhidden):\n", 134 | " super(ODEModel, self).__init__()\n", 135 | " self.num_inputs = num_inputs\n", 136 | " self.mlp = nn.Sequential(\n", 137 | " nn.Linear(num_inputs, nhidden),\n", 138 | " nn.ReLU(),\n", 139 | " nn.Linear(nhidden, nhidden),\n", 140 | " nn.ReLU(),\n", 141 | " nn.Linear(nhidden, num_inputs)\n", 142 | " )\n", 143 | "\n", 144 | " def forward(self, t, x):\n", 145 | " bs = x.shape[:-1] # reshape to handle multi-dim batches\n", 146 | " return self.mlp(x.reshape(-1,self.num_inputs)).reshape(*bs, self.num_inputs)\n" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "metadata": {}, 152 | "source": [ 153 | "## Training the ODE with weak form loss\n" 154 | ] 155 | }, 156 | { 157 | "cell_type": "markdown", 158 | "metadata": {}, 159 | "source": [ 160 | "First we define a helper function for sampling data" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 4, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "def sample_data(t, y, batch_time_int, batch_size):\n", 170 | " \"\"\" Draws batch_size samples from y and t with an integration time of batch_time_int\n", 171 | " Assumes all points are evenly spaced \n", 172 | " inputs:\n", 173 | " t < tensor(num_t,) > \n", 174 | " y < tensor(num_t, num_start_times, NDIM) > \n", 175 | " batch_time_int < int > \n", 176 | " batch_size < int > \n", 177 | " outputs:\n", 178 | " t_data < tensor(batch_time_int) > \n", 179 | " y_data < tensor(batch_time_int, batch_size, NDIM) > \n", 180 | " \"\"\"\n", 181 | " num_t = len(t)\n", 182 | " t_data = t[:batch_time_int]\n", 183 | " batch_idx = np.random.choice(np.arange(num_t - batch_time_int + 1),\n", 184 | " batch_size, replace=False)\n", 185 | " run_idx = np.random.choice(np.arange(y.shape[1]), batch_size, replace=True)\n", 186 | " y_data = torch.stack(\n", 187 | " [y[batch_idx[i]:batch_idx[i] + batch_time_int, run_idx[i]]\n", 188 | " for i in range(len(batch_idx))], dim=1)\n", 189 | " return t_data, y_data" 190 | ] 191 | }, 192 | { 193 | "cell_type": "markdown", 194 | "metadata": {}, 195 | "source": [ 196 | "We will run this example on GPU if it is available" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 5, 202 | "metadata": {}, 203 | "outputs": [ 204 | { 205 | "name": "stdout", 206 | "output_type": "stream", 207 | "text": [ 208 | "Running on: cuda:0\n" 209 | ] 210 | } 211 | ], 212 | "source": [ 213 | "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n", 214 | "print(\"Running on: {}\".format(device))" 215 | ] 216 | }, 217 | { 218 | "cell_type": "markdown", 219 | "metadata": {}, 220 | "source": [ 221 | "We will use 400 radial basis functions with a shape parameter of 10 in each integration time window." 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": 6, 227 | "metadata": {}, 228 | "outputs": [ 229 | { 230 | "name": "stdout", 231 | "output_type": "stream", 232 | "text": [ 233 | "Epoch: 0050, loss: 1.53\n", 234 | "Epoch: 0100, loss: 1.16\n", 235 | "Epoch: 0150, loss: 1.11\n", 236 | "Epoch: 0200, loss: 1.07\n" 237 | ] 238 | } 239 | ], 240 | "source": [ 241 | "epoch = 200\n", 242 | "model = ODEModel(2, 200)\n", 243 | "model.to(device)\n", 244 | "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-1)\n", 245 | "rbf_ep = 10.0\n", 246 | "model.to(device)\n", 247 | "for ep in range(1,epoch+1):\n", 248 | " optimizer.zero_grad()\n", 249 | " t_sample, y_sample = sample_data(t, y_meas.unsqueeze(1), 500, 80)\n", 250 | " t_sample = t_sample.to(device)\n", 251 | " y_sample = y_sample.to(device)\n", 252 | " dydt_pred = model(t_sample, y_sample)\n", 253 | " c = torch.linspace(t_sample[0], t_sample[-1], 400, device=device)\n", 254 | " psi, psi_dot = gauss_rbf(t_sample, c, rbf_ep)\n", 255 | " loss = weak_form_loss(dydt_pred.transpose(1, 0), y_sample.transpose(1, 0), \n", 256 | " t_sample, psi, psi_dot) \n", 257 | " if ep % 50 == 0:\n", 258 | " print(\"Epoch: {:04d}, loss: {:.2f}\".format(ep,loss.item()))\n", 259 | " loss.backward()\n", 260 | " optimizer.step()\n" 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": {}, 266 | "source": [ 267 | "## Visualizing the results \n", 268 | "\n", 269 | "We see we were able to learn a model for the ODE which reproduces the data generating trajectory well." 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 7, 275 | "metadata": {}, 276 | "outputs": [ 277 | { 278 | "data": { 279 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABpXUlEQVR4nO3ddXgUVxfA4d9dy8adeEiCQwgQ3LVAKVCKtNRLS6Hu1Km7fXWBtlCFGrRQobgUd5cQEkLc3Ta78/2xIRBIILISyH2fh4ewI/dkdpmzM3PvuUJRFCRJkqTmR2XvACRJkiT7kAlAkiSpmZIJQJIkqZmSCUCSJKmZkglAkiSpmdLYO4D68PHxUcLCwuwdhiRJ0iVl586dmYqi+J77+iWVAMLCwtixY4e9w5AkSbqkCCFO1vS6vAUkSZLUTMkEIEmS1EzJBCBJktRMXVLPACRJMjMYDCQmJlJaWmrvUKQmRK/XExwcjFarrdP6MgFI0iUoMTERV1dXwsLCEELYOxypCVAUhaysLBITEwkPD6/TNvIWkCRdgkpLS/H29pYnf6mKEAJvb+96XRXKBCBJlyh58pfOVd/PhN1uAQkh9MB6wKEyjl8VRXnemm1WZGSQ/88yQMF1+HC0QUHWbE6SJKlJs+cVQBkwTFGULkBXYLQQoo+1Giva9B+xV44h7bXXSHvtdRJmzEQxmazVnCRd1nJzc/n000+ttv+1a9eyadOmem+3Y8cOHnjggQa1+f7771NcXNygbS9VdksAillh5T+1lX+sMjuN4cAmEu+ajkaTT8QPn9Fq5QqC3nsXoVKhlJdjSEuzRrOSdNm6UAIwGo2N3v+FEkBFRUWt2/Xo0YMPP/ywQW02JAFY4ne1J7s+AxBCqIUQe4B0YIWiKFtrWGeGEGKHEGJHRkZGw9rZ+QXOfuWEDC7CYf876IKD0bdrB0Da2+9w8oYbMSQnN+I3kaTm5cknnyQ2NpauXbsya9Ys1q5dy9ChQ7nhhhvo3Lkz8fHxREZGVq3/zjvv8MILLwAQGxvL6NGj6d69OwMHDuTIkSPV9h0fH8/nn3/O//73P7p27cqGDRu47bbbeOSRRxg6dChPPPEE27Zto1+/fnTr1o1+/fpx9OhRwJw4xo4dC0BRURG33347PXv2pFu3bvzxxx+A+aT92GOP0blzZ6Kiovjoo4/48MMPSU5OZujQoQwdOhSABQsW0LlzZyIjI3niiSeq4nNxceG5556jd+/evPLKK1xzzTVVy1asWMHEiRMtf8CtxK7dQBVFMQJdhRAewGIhRKSiKAfOWWcOMAegR48eDbpC0Fz3EcFDDkDSTlj+LKTshYAuALhffTV5ixeTcMd0wn7+CbWra6N+J0mytReXHuRQcr5F99kx0I3nx3Wqdfkbb7zBgQMH2LNnD2A+8W7bto0DBw4QHh5OfHx8rdvOmDGDzz//nDZt2rB161buueceVq9eXbU8LCyMu+66CxcXFx577DEAvvrqK44dO8bKlStRq9Xk5+ezfv16NBoNK1eu5Omnn+a3336r1s6rr77KsGHD+Prrr8nNzaVXr16MGDGCb7/9lri4OHbv3o1GoyE7OxsvLy/ee+891qxZg4+PD8nJyTzxxBPs3LkTT09PRo4cye+//86ECRMoKioiMjKSl156CUVR6NChAxkZGfj6+jJv3jymTZvW8ANvY02iF5CiKLnAWmC0VRrQu0HLftDtZlDrYN/PVYscIzsR/OknlJ86RfITTyLnSJakhunVq9dF+58XFhayadMmpkyZQteuXZk5cyYpKSl12v+UKVNQq9UA5OXlMWXKFCIjI3n44Yc5ePDgeesvX76cN954g65duzJkyBBKS0tJSEhg5cqV3HXXXWg05u+/Xl5e5227fft2hgwZgq+vLxqNhhtvvJH169cDoFarmTRpEmDudXPzzTfz/fffk5uby+bNm7nyyivr9Ps0BfbsBeQLGBRFyRVCOAIjgDet2qijB7QaDgd/hyteBpU5/zn36oXfrMdIe/0Ncn/9Fc8pU6wahiRZ0oW+qduSs7Nz1c8ajQbTWZ0sTvdNN5lMeHh4VF05NHT/s2fPZujQoSxevJj4+HiGDBly3vqKovDbb7/RrvJ279mvX6y75IW+COr1+qpEBDBt2jTGjRuHXq9nypQpVYnlUmDPK4AAYI0QYh+wHfMzgD+t3mqnCZCfaL4ddBbPm2/GfcIEdCGhVg9Bki51rq6uFBQU1Lrcz8+P9PR0srKyKCsr488/zf+13dzcCA8P55dffgHMJ9q9e/fWe/95eXkEVXbjnj9/fo3rjBo1io8++qjqZL57924ARo4cyeeff171MDk7O/u8Nnv37s26devIzMzEaDSyYMECBg8eXGM7gYGBBAYG8sorr3DbbbfVGnNTZM9eQPsURemmKEqUoiiRiqK8ZJOG244CoYajf1d7WahUBL7xOs59etskDEm6lHl7e9O/f38iIyOZNWvWecu1Wm3Vg9KxY8fSvn37qmU//PADX331FV26dKFTp05VD2fPNm7cOBYvXlz1EPhcjz/+OE899RT9+/c/ryfO6W/3s2fPxmAwEBUVRWRkJLNnzwZg+vTphIaGEhUVRZcuXfjxxx8B87OJK6+8kqFDhxIQEMDrr7/O0KFD6dKlC9HR0Vx99dW1Ho8bb7yRkJAQOnbsWIej13SIS+med48ePRSLTAgzfywUZ8E9m89bZCwsImvuXNzHXoVDmzaNb0uSrODw4cN06NDB3mE0Ob/99htLlizhm2++sWm79913H926deOOO+6wabs1qemzIYTYqShKj3PXbRIPgW2u3ZWQfghy4s9bpBjKyfnhB9Lf/8D2cUmS1GBLlizhmWeeYebMmTZtt3v37uzbt4+bbrrJpu1aQvNMAG0rOxsdXXbeIo2nJ1633krhqlWUxcbaODBJkhpq/PjxHDlyhH79+tm03Z07d7J+/XocHBxs2q4lNM8E4N0KfNrCsX9qXOx54w0IBweya3m4JEmSdDlongkAzFcB8RuhNO+8RRovL9yvmUDe739Q0cDRx5IkSU1d800A7caAyQDHV9W42OvWW3EZNgxTMysOJUlS89F8E0BIL3D0gmPnPwcAcAgPJ/iD99G1bGnjwCRJkmyj+SYAlRrajISY5WCsvbpg2YkTlMXF2TAwSWr6ZDnoy0PzTQBg7g5akgOJ22pcrJSXE3/dVLLmfmnjwCSpaZPloM1kOehLWathoNKeNyr4NKHT4TJ4MIWrV6Nc4EMnSc2NLActy0Ff+vRuEDbAPB5g5Cs1ruJ6xRXk//knxTt34dy7l40DlKQ6+OdJSN1v2X36d4Yr36h1sSwHfXmUg27eCQDMt4H+eRwyjoFv2/MWuwwcgNBqKVy3TiYASbqA+paDPq2srKxO+z+3HPStt95KTEwMQggMBsN56y9fvpwlS5bwzjvvADS4HDRQVQ56woQJtZaDnjZtGps3b+bbb7+t0+/TFMgE0PFq+Pdp2P1tjVcBKicnHLt2pXhbzc8JJMnuLvBN3ZZkOWhZDvrS4+oP7cfC7u/BUFLjKv4vvUjo11/ZODBJarpkOegzZDnoS13P6ebeQPt+qnGxQ3g4ajc3GwclSU2XLAddnSwHbQMWKwd9LkWBL4dDQSrcvxO0juetkjlnLlq/Frhf4EMgSbYiy0HXTJaDluWg608IuOIlyE+CzZ/UuErBsmXk/v67beOSJKnOZDno+rt0nlZYW9gA6DAO1r1pLhTnH1ltsT6qM/l//Y1iMiFUMm9KUlMzfvx4xo8fb/N2d+7cefGVmih5Jjvb2PfB0RN+vR3KCqstcuzcGVNBAeXxJ+0TmyRJkoXJBHA2Zx+YOBeyYuCPe83PBirpO3UCoPTwIXtFJ0mSZFEyAZwrYjCMeAEO/Q6bP656WRcRgcrVFWNurr0ikyRJsij5DKAm/R6ApJ2w4jkI6ALhg1DpdLTdtvWiA0gkSZIuFfIKoCZCwNWfgFcE/PlwVbloefKXJOs4u4jbkiVLeOON2kc3N7QU9QsvvFBVFuJsv//+O4cO1f/W7sXivJDXXnutQdtZmkwAtXFwNXcNzToOe34AoGDtWuKvm4qpqMjOwUnSpaEh5ZLHjx/Pk08+WetyS89FcKEEcKHS0xeL80LqmwAURalWWsNSZAK4kHZjwK8zbDfPB6AYDJTs3UvZiRN2DkyS7Cs+Pp727dtz6623EhUVxeTJk6tq6YeFhfHSSy8xYMAAfvnlF5YvX07fvn2Jjo5mypQpFBaae9gtW7aM9u3bM2DAABYtWlS17/nz53PfffcBkJaWxjXXXEOXLl3o0qULmzZtOq8UNcDbb79Nz549iYqK4vnnn6/a16uvvkq7du0YMWJEVcnos23atIklS5Ywa9YsunbtSmxsLEOGDOHpp59m8ODBfPDBByxdupTevXvTrVs3RowYQVpa2nlxZmRkMGnSJHr27EnPnj3ZuHEjYC5+N23atKrS07/99htPPvkkJSUldO3alRtvvBGA9957j8jISCIjI3n//ferjnGHDh245557iI6O5uWXX+bhhx+uin3u3Lk88sgjjXof5TOACxECut8Kfz8GyXtwCAsDoDz+JI6dO9s3Nkk6y7Rl55cgHhU2iqntp1JSUcI9K+85b/nVra9mQusJ5JTm8Mja6ieSeaPnXbTNo0eP8tVXX9G/f39uv/12Pv3006ryzXq9nv/++4/MzEwmTpzIypUrcXZ25s033+S9997j8ccf584772T16tW0bt2a6667rsY2HnjgAQYPHszixYsxGo0UFhaeV4p6+fLlxMTEsG3bNhRFYfz48axfvx5nZ2cWLlzI7t27qaioIDo6mu7du1fbf79+/Rg/fjxjx45l8uTJVa/n5uaybt06AHJyctiyZQtCCL788kveeust3n333Wr7efDBB3n44YcZMGAACQkJjBo1isOHD/Pyyy/j7u7O/v37q/Y1adIkPv7446r4d+7cybx589i6dSuKotC7d28GDx6Mp6cnR48eZd68eXz66acUFRURFRXFW2+9hVarZd68eXzxxRcXfZ8uRCaAi+k8GZY9BQd+Qzv4WRCC8pNyLIAkhYSE0L9/fwBuuukmPvzww6oEcPqEvmXLFg4dOlS1Xnl5OX379uXIkSOEh4fTpk2bqu3nzJlzXhurV6+uKq+sVqtxd3cnJyen2jrLly9n+fLldOvWDTB/646JiaGgoIBrrrkGJycngHoNEjs7ISUmJnLdddeRkpJCeXl5jSWvV65cWe02Un5+PgUFBaxcuZKFCxdWve7p6Xnetv/99x/XXHNNVbXTiRMnsmHDBsaPH0/Lli3p06cPYK6GOmzYMP788086dOiAwWCgcyO/iMoEcDGOnhDaB2JXoxr5Mhp/f8oTZAKQmpYLfWN31DhecLmn3rNO3/jPdW6niLP/ffpkpigKV1xxBQsWLKi27p49eyzWqUJRFJ566qnzSkC8//77DW7j7NLT999/P4888gjjx49n7dq1VTObnc1kMrF582YcHavXEWts6emz4wBzIbvXXnuN9u3bW2TiGfkMoC5aj4C0A5CfgnO/vmgqJ4mQpOYsISGBzZs3A+bpEwcMGHDeOn369GHjxo0cP34cgOLiYo4dO0b79u2Ji4sjNja2avuaDB8+nM8++wwwP1DOz88/r1T0qFGj+Prrr6ueLSQlJZGens6gQYNYvHgxJSUlFBQUsHTp0hrbqE/p6dqKzI0cOZKPPz4zbuj07Z1zXz999aLVaqsmsRk0aBC///47xcXFFBUVsXjxYgYOHFhjO7179+bUqVP8+OOPXH/99bXGXFcyAdRF6+Hmv0+sJfDVV/GrofwtwOGsw3y5/0s2Jm28YFaXpMtBhw4d+Oabb4iKiiI7O5u77777vHV8fX2ZP38+119/PVFRUfTp04cjR46g1+uZM2cOV111FQMGDKBly5Y1tvHBBx+wZs0aOnfuTPfu3Tl48OB5pahHjhzJDTfcQN++fencuTOTJ0+moKCA6OhorrvuOrp27cqkSZNqPalOnTqVt99+m27dulUlpLO98MILTJkyhYEDB+Lj41Nt2elv9x9++CE7duwgKiqKjh078vnnnwPw7LPPkpOTQ2RkJF26dGHNmjWAufR0VFQUN954I9HR0dx222306tWL3r17M3369KrbWTW59tpr6d+/f423k+pLloOuC5MR3mgJXa6Dq96tcZVFMYt4cfOLmBQTQS5B/DT2J9wd3G0cqNRc2LscdHx8PGPHjuXAgQN2i8He3n33XfLz83nxxRdt2u7YsWN5+OGHGT58eI3LL4ly0EKIECHEGiHEYSHEQSHEg/aK5aJUagjsCkm7KN61m5ihwyjZf2YS7picGF7e8jJ9Avqw5to1/DDmB3nyl6TL2Oeff878+fNtWgI6NzeXtm3b4ujoWOvJv77seQuoAnhUUZQOQB/gXiFE051OJygaUvejdnagIiWF8vj4qkUf7f4IR40jbw58Ex9HH7wdvckry2P+gflUmGofSCJJl6qwsLBm/e3/rrvuYv/+/VW9mGzBw8ODY8eOVU2naQl26wWkKEoKkFL5c4EQ4jAQBDTNcptB3cFkQKvJA8CQlFS16Nk+z3Iy/yQeeo+q13am7eTdne/i7ejNuFbjbB2tJEnSRTWJh8BCiDCgG7C1hmUzhBA7hBA7MjIybB5bFX9zf1tVXgxqLy8MSclVi1o4taCnf89qqw8NGUqYWxi/HvvVpmFKkiTVld0TgBDCBfgNeEhRlPxzlyuKMkdRlB6KovTwtWf3S4+WoHaAjKNoAwMxJCejKAovbn6Rv4+v49vN8WyIyajq/SOEYELrCexK38XJfDluQJKkpseuCUAIocV88v9BUZRFF1vfrlRq8GkDmcdwHTECx+huxOXF8euxX3nmz7U898dBbv5qG/9bGVO1yelbP8viltkrakmSpFrZsxeQAL4CDiuK8p694qgXn7aQcRSfu2bie++9rD21FoCK/Pb8dndfJkUH8+GqGLbHZwPmW0PRLaLJLs22X8ySdAmQ5aDtw55XAP2Bm4FhQog9lX/G2DGei/NtB7kJYChBMRr589hKjKUBPDK8F91bevHyhE74uTnwvxXHqjb5etTXPNX7KTsGLUn2I8tB16zZl4NWFOU/RVGEoihRiqJ0rfzzt73iqROftoBC4d+/ciSqCxXH96Mp68j1vUIBcNJpmD4ggk2xWew9lQuAWqUGwGiq/38ESWqqZDloWQ66+fEyVwFUqwvBaMQ705fgvr3Ra9VVq1zfO5R3Vxzll52n6BLiAcDj6x+nwlTBe0MujTtd0qXn5M23nPea65Wj8brhBkwlJZyaMfO85e7XXIPHxGuoyMkh6YHq4zBbfvftRduU5aDPkOWgmwMPc70Srdb8DcY9ri/TH6p+18rFQcPIjv78tS+F58Z2QqdR4aB2YFPyJkyKCZWwe8crSbIIWQ76DFkOujlw9AQHN9QV6Ri0OsJMBXQKdDtvtQndAlmyN5mNsZkMbdeCXv69+P3478TkxNDOq50dApcudxf6xq5ydLzgco2nZ52+8Z9LloM+Q5aDbg6EAI9QynPiSXetIESJq/HN7dfKB0etmjVH0gGqBoltS91m03AlyZpkOegzZDno5sKjJXtzTrCyGxj61FyyVa9V07+1N6uPpKMoCv7O/oS4hrA9dbuNg5Uk65HloC/9ctAoinLJ/Onevbtid/88qXz3bkslcn6kEpudUOtq322OV1o+8acSk5avKIqi/Hj4R+Xnoz/bKkrpMnfo0CG7th8XF6d06tTJrjHY2zvvvKM899xzNm/3qquuUlauXFnr8po+G8AOpYZzqnwGUF8eoRzTCXQVTgSX62q9xze0fQsA1h7NoHULV65v3/jLNUmSmobT5aDP7r5qbbm5ufTq1YsuXbpcFuWgL0nlzgEc1Om4do8jsYOHYMrLq3G9IA9Hwn2c2XIiq+q1zJJMkgqTalxfki4lshz05VEOWiaAeoopcWVkcTHhLcz3LA3p6bWu2zvci61x2RhN5sutyUsm8+key41glJo35RKazU+yjfp+JmQCqKft2Xpm5uYzPMTcF7givfYS1X0ivCkoreBwSj5CCKJ8o9ibsddWoUqXMb1eT1ZWlkwCUhVFUcjKykKv19d5G/kMoJ7WpBQwQahxdSolHai40BVAhBcAW+OyiQxyp4tvF9acWkNOaQ6eegs8wZeareDgYBITE7HrHBlSk6PX6wkODq7z+jIB1NPekl+YEBLAalUucOEEEODuSKiXE1tOZHHHgHCifKMA2Jexj8Ehg20RrnSZ0mq1NY5IlaT6kLeA6iEtv5Ry9SkijFpUJem0mPUYzn16X3CbPhFebIvLxmRS6OTdCbVQy9tAkiQ1CfIKoB72nMpGpcugFW5QkIL3PXdcdJseYV78vCORE5mFtG7hyrtD3qWdpywHIUmS/ckrgHrYdPIIQlVBB5cAyE+mIiuLspiYC24THeoBwK6EXACGhw4n2LXu9+gkSZKsRSaAetiXfgSA9h4RUJpL+huvkzDz/DK7Z4vwccFNr2F3grkGSF5ZHotiFpFSmGL1eCVJki5EJoB6OJXqQ3vNNCI8WwOgdnPEmJ1zwa54KpWgW6gnu07mApBTmsPzm55nU/ImW4QsSZJUK5kA6ig9v5TMXGdGt7wGvWsgABpnDUppKUrlTEi16RbqwbH0AvJLDbR0a4mbzo39mfttEbYkSVKtZAKoo/1JeahdDhPoXQrO5jo/ar25BlBF9oUnfY8O9URRYN+pPIQQdPbpzL7MfVaPWZIk6UJkAqijPacycAz+lmPFq8HZXBJW42Ce59eYlXWhTeka6oEQsKvyOUBn387E5sZSbLjwlYMkSZI1yQRQR7vTjiCEQiefduDsC4BDCw2Bb76BNiTkgtu66bW09nU5kwB8OmNSTBzLOWb1uCVJkmojxwHU0Ync4+AGrT1ag84JdC5otcW4j766TttHh3qy7GCqedLngN6sv269LAchSZJdySuAOig1GMk2JKBGR4hr5bd9Z18oTKd4127K4uIuuo/olh7klRg4kVmEg9pBnvwlSbI7mQDqICatEKHLwFcfhFqlNr/o7AtFGZyaMYOcWuYzPVt0qPmEv+uk+TbQmoQ1zN4422oxS5IkXYxMAHVwJDWf0tQJPN79+TMvurSAogzU3l4Ysy7cCwigla8LrnoNu0/lApBUmMTvx38nrSjNSlFLkiRdmEwAdXAsrQAdXgyLiD7zorMPFGWg8fKmIvvCvYDAPCCsa4hH1RVAZ9/OABzIbL6zKkmSZF8yAdTBvtQE/EK2kllyVuln5xZQnIXay7NOVwAA3UI9OZZWQGFZBe292qNRaeR4AEmS7EYmgDo4nnuIHP1vZJScNfmGsy8oJjRuzhcdCHZadKgHJgX2ncrFQe1Ae8/2ckSwJEl2IxPAReQUlVNgMhduC3MLO7PAxTwWwHPcEILefbdO++oWUvkguHI8QHe/7miE7IkrSZJ9yLPPRRxJLUCly8BN64WLzuXMgsrBYHp/J4joVad9uTtpaeXrzO7K0tCP9XzM0uFKkiTVmbwCuIijqfmodBmEuYdVX+Bonu+3IjmB/H/+oSInp077iw71ZPepXDmZtyRJdmfXBCCE+FoIkS6EaLJdYY6mFaB2yKKdV6vqC5zMCaDs+HGSHn6EsmMXnhjmtOiWnmQXlXMyy1wH6N5V9/LGtjcsGrMkSVJd2PsKYD4w2s4xXNCR1ALaG97kwegHqy+ovAJQ6yoAMGZl1ml/3apmCDNfMVSYKtieut0ywUqSJNWDXROAoijrgbp1obEDRVGISSuko7837g7u1RdqdKBzRaMxf5Ov6y2gNi1ccXHQVCsMdzz3uKwMKkmSzdn7CuCihBAzhBA7hBA7MjIyLr6BBaXklVKiOUyCWFjzCdrJE7UoBMCYm1unfapVgi4h7lUPgqN8ozApJg5lHbJQ1JIkSXXT5BOAoihzFEXpoShKD19fX5u2HZNeiMY5hj25f+Ogdjh/BUcvRFkuKhcXjLl5dd5vdKgnR1ILKC6vINInEkCOB5AkyeZkN9ALOJ5uLgIX7Bpypgjc2Zy8oCSb0Hlfo6lHcooO9cRoUtiXmEefCG8mtZl0psqoJEmSjcgEcAHH0wvQ6rNo7RFV8wpO3pB9AsfOneu1364hHoD5QXCfCG9e6PdC4wKVJElqAHt3A10AbAbaCSEShRB32DOecx1LywNN5vljAE5z9ILiHIq2bCHvz7/qvF9PZx0RPs7sOplb9VpuaS4lFSWNC1iSJKke7N0L6HpFUQIURdEqihKsKMpX9oznbIqiEJOVgk7lUr0ExNmcvKAsj9xffiHjgw/qtf+uoR7sOZWDoigcyjrEwJ8GsilpU+MDlyRJqqMm/xDYXjILy8kvdOa+Vt8zrtW4mlc6PRbAxbHOvYBOiw71JLOwnFPZJbTyaCUrg0qSZHMyAdQiJr0AMPfbV4laDlPlaGC1kxZTQQFKRUWd9181Q1hCDg5qB9p5tmv6PYFMRji2HLZ/CSc3gclk74gkSWoEmQBqcTy9EJ3PKv5K+bj2lU4nAEfzYTTm1b0raDt/V1wdNGyLN4+D6+zTmYOZBzGajA0P2ppyE+DzgfDjFPjrUZh3JXw5DDLrVgJDkqSmRyaAWhxPL8TB5TjJRReY8P30LSC9+Z/1uQ2kVgl6hHmyLc6cAKJ8oyiuKOZE3omGhmw9JTkw/yrIT4TJ8+DhQ3D1J5B7CuYOg7gNmEpLKd61m6JNm6iw8YA9SZIaRnYDrUVMWiFqfSbh7tG1r1R5BeDS0Z9WK5aj9fevVxu9wr1Zc/QImYVl9PLvxXN9n8PH0acxYVvH37MgPxmmLYOQnubXut0E4YNRvptM9jM3kxnjg6mgCABNYACtV61CCGHHoCVJuhiZAGpxLCMdY3BB7T2AwDwOAFBTiDqk/gO5eoWbE8iO+GxGRwYwpe2UhoRqXUk7Yf8vMOjxMyf/0zxCyDRMJHPXfFxCCvB4/lVUvqEo5WUIIVCMRkr27MGpe3f7xC5J0gXJW0A1yC0uJ8eQBEC4e3jtK2qdQO2AKSedzLlzKdlfv4e4nYPc0WtVbDlhvg2UXJjM6oTVDY7bKta+aU50/R+ocbHXjHsJfOYBggcX4Jr6Bc7du+IycCAAOQsWcvKmm8lZuNCWEUuSVEcyAdTgeHohQlQQ7NSGCPeI2lcUApy8UIqzyXj3PYq31a+ss06jonvLM88BFsUs4uG1DzedyqA5JyFmOfS4Axxcqy0q3r0bU0kJahcX3G++G3HNJ5C4DVY+X7WOx+RJuAweTOoLL5L15Ze2jl6SpIuQCaAGMemFGEsimDP8e0LcLnJrx9ELlTEfNJp6jwUA6BXmzeHUfPJKDE2vMuiub8xJLvqWai8b8/NJvPc+Ul986cyLna6BnnfClk8hdg0AKr2e4I8+xG3MGNLfeZecBQtsGb0kSRchE0ANYtIKcdSqCfJwvPjKTl6IkmzUHh4NSwDhXigK7DyZ3bQqg5pMsPsHaDMSPKonwYyPP8aYk4PXLTdX3+aKl8CnLfx+DxSbr2qEVkvgW2/iMmQIaW++JXsISVITctEEIIS4TwjhaYtgmoqY9AKcwz7ng93vX3xlR08oyUHt4d6gBNAt1AOtWrA1LhsvvRdBLkFNIwEkbofCVIicXO1lQ1o6uQsW4jF5EvqOHatvo3OCiXOgKN08VqBy3mOh0RD0/v9o+e039aqaKkmSddXlCsAf2C6E+FkIMVo0g759x9PzKNecRKEOE7c7eVUmgIZdAei1aroEe1Q9CI7yiWJfRhMoCXFkKai00HZktZez589HMZnwnjGj5u0Cu8GQp+DgInPvoUoqvR7HKHNV1fy//8aQnGy10CVJqpuLJgBFUZ4F2gBfAbcBMUKI14QQrS644SWqoNRAanEKChWEu12gB9BplVcAIR99RMiXcxvUZr9W3uxPzCW/1MD90fez4Co73ytXFDj8J4QPAr37WS8rlB07htuYMegu1O11wMMQ0hv+esw8WOwsFTk5pDz3PKfuuw9Tiax+Kkn2VKdnAIqiKEBq5Z8KwBP4VQjxlhVjs4vYjCJUDuYJ3mstA302R08wGVA7aVA51DBrWB30b+2DSYHNsVmEuIbg62Tn2ySZMZATB+2vqvayEIKQL+cS8PJLtWxYSaWGa74AxQi/312tZpDG05PAd96m7PARUp55BkWpw1WWJElWUZdnAA8IIXYCbwEbgc6KotwNdAcmWTk+m4tJK0ClMz+ovOAgsNMqy0EUrVtF2utvNOiE1i3UEyedmo3HzYlnwZEFLIldUu/9WMyJtea/Ww2reklRFIx5eQghUOn1F9+HVziMfgPiN8CWT6otch0yBN+HHiL/73/I+e57CwYuSVJ91OUKwAeYqCjKKEVRflEUxQCgKIoJGGvV6OzgeEYhaqMXo8OuxFNfh2ffjuZ1SvbvI/ubb1BKS+vdpk6jone4F//FmBPAsrhlLDxix8FTcevAI9R8Eq9UvH07MQMHUby9HmMdut0E7cfCqpcg9UC1Rd53Tsdl6FDS3nqL8sRES0UuSVI91OUZwHOKopysZdlhy4dkX8fTCmmp78Pbg+t4d+t0RVC9+dl4fSqCnq1/ax9OZBaRlFtCD/8eHMo6RGF5YYP21Sgmo/lbe/jgai/n/vorQqdD36lT3fclBIz7APQesGgGVJSdWaRSEfjG6wS98za64GALBS9JUn3IcQDniEkvJLyFru4bVF4BqB3Mt34amgAGtjHf998Yk0lP/54YFSO703c3aF+NkrIXSvMgYkjVS6biYgpWrMTtqqtQOTnVb3/OPubKoekHYfXL1Rap3d1xGz0agLITJ1CMTbQUtiRdpmQCOEupwcipvBw2Vszgh8M/1G2j0wlAZz55GXMblgDa+rng6+rAf8cz6eLbBY1Kw/a0+pWWsIiTldNShg2oeqlw3TqUkhLcxoxp2D7bjoQet8Omj888XzhLeXw8cROuIfOTT87fVpIkq5EJ4Cwns4oR2gxAwc/Jr24bnU4AmnIQAlNhQYPaFkIwoLUPG49n4qDS061FN3JLcxu0r0ZJ3Ga+/+96prR1/j/LUPv44NSjEVU9R75iHiX823Rzaemz6MLCcBs7lsxPP6Nw/fqGtyFJUr3IctBnic0oRKWr7AJalx5AABoH0Drj4Clof2A/Qq1ucPv9W/uweHcSh1PzmXvFXNSqhu+rwU5th5b9qr3k++ADGBITG/W7oXOG674zTyDz861w21+gOXOrzf+52ZQeOkTyrMcJ++03dMFBDW/rUpW0C2JWQOZRlPJiDCXOKC064jDwWvAIwZCWhtavjl9MJKkO5BXAWU5kFKLSZSAQhLqF1n1DR09EaU7jTpDAoDbmyWDWHs2wz8k/LxEKkiGkV7WXHVq1wmXw4Fo2qgffdnD1x+arjOXPVluk0usJ/uB9FJOJpAcfxFRWVstOLkMp++CrUTB3KMqa18lbvZ24Tw4Q+/Ymcr98H97vTMWcCRwfPIQTEyeS+9si+bxEsgiZAM4Sm1GEs3M2gS6B6NT1eBDsZB4NnPrKq+QtXdrg9lu46YkMcmPNkXQURWHmipl8sseG98VPbTX/HXxm4pfsb7+jcONGy7XR6Rrocy9s+wL2Vu/qqmvZksA338Cpd+9GJ9NLgqLAls9hzhDIjsXQezYJsVeRvLoCfNvg9+yzeL70AwyahUg/iF+3PMiMI+WZZ4ibPKXe809I0rlkAjjLiYxCAnU9ubXTrfXb0NETSrIp+Pdfirdta1QMw9q1YFdCDrnFBkorStmYZMGT78Wc2g4aR/DvDICpvJyM99+nYMUKy7ZzxYsQNhCW3A/x1X8/12HD8Ht8FkKjQamosGy7Tc2qF2HZE9B2FNy7jew9BkoOHcH/5ZcIX7wIr5tuRNdlEAx7BvXje/C6+xHCR6YRNKgYY+opTt58CxVZWfb+LaRLmEwAlRRFITajiO4+Q7m+/fX127haRdCG9QI6bVgHP0wKrDuWUTUeoMhQ1Kh91lniNgiKBrUWMA/+MhUX4zJkiGXbUWvNzwM8w2DhDebSE+coPXqU2CvHULL/wPnbXw42vAf//Q+6T4PrfgAnL3wffZTwn3/Cc8qU8+dT1jnDoMcQ927GbUBXIobGEHR1ABon83/hyz5ZSlYhE0CljIIyCsuL8XDLpcJUz/9MjuaKoCp39waPAzgtKsgdHxcdq4+kV40H2Jm2s1H7rBOjwTxaN7Bb1UuFa9ch9Hqc+/SxfHuOnnDjL+Zk8MNkKEyvtljTogWKsYLEBx+gIifH8u3b0+Gl5m//nadg6PoYCTNmYkhPR6XT4dC69YW39WwJN/+OevzruKq3wheDKfjlS+ImTqI8IcE28UuXDZkAKsVmFKF2PMW8hLvZnlrP/venrwDcGzYnwNlUKsHgti1YezSdzt5dcFA7sDl5c6P2WScZR8FYVpUAFEWhcM0anPv0qVvtn4bwDIPrF5pP/t9eXTWJDJiLxgV/8AHGjEySH5t1+Tz0zD5hnjAnMBrT6PdIfOghSvbswVSfLw5CQJ+7YdoyMBlRrX0OQ1ICcVOupWjTJuvFLl12ZAKoFFvZAwguMhF8TZy8wFSBxssDNI1/eDm8QwvySys4mFTCTR1uoq1n20bv86JS9pr/DugCgDEnB0wmy9/+OVdwD3MSyIqF7yZASW7VIsfOnfGb/SxFGzeS8fHH1o3DFkxGWDQThAplynxSX3uT0oMHCXzrLRzatKn//kJ6wsz1OPfpTfiQk2j1FSTcOYPsb7+9LKqsKopC6dFjZM2ZQ/JD9xA/8SrK/v0c9vxIyeIPyHz9KYrX/oPSnHqMWZgcB1DpREYROscs9Go9LZxa1G/jysFgAY/MMF+iN9KANj5oVILVR9J58sqHGr2/OknZCzoX8DJP86Dx8qLVqpVgi2/eEYPhuu/NzwN+mAw3/gqOHgB4TJlCyd69lB48iGI0Xtq9g7Z+YX7OMnEu+ZsOkPfbIrzvmonrsKEN36ezN9z4K7qAl2mpf5/kveGkvfY6ulatcOnf33Kx21jJlnUkP/Ek5Wm5AGgcjehcKhDLN4OrkZIYJzJ2usM3v6N2UHDt5InH2BE4jrjR3N348p+3yiJkAqgUm1GIk3M2Ld1aohL1vDCqTACUZFskAbjptfQM82LNkXSevLI9+eX5FJYXEugS2Oh91yplr7n3j6ryoaKimB9Eamz0EWk7EqbMg1+mwTdj4abF4OKLEAL/559HqNWX9sk/O85cFbXNKJTIyeS8fhOOXbrge999jd+3Sg0jXkAd0JVgx3soDHPBOch8AlQqKhC2eg8bSamooOLoFrSHvkK7eykaxQ2vYR64DBqEtkNf8IowfzFQafAyFOOeEkfRpvUUrN9G3r408vf/TJtDH6HybQlR15n/eF+W81ZZzKXxybCBE5mF4JdBmHu3i698rtNzAmzZSvaKuQS8+goaL69GxTO8Qwte+eswJzOLuGPNRLq16Mbbg99u1D5rZTJC6n6INk/ybszL48TYcfg9+yxuo0ZeZGML6jDOfDvop5tg3pVwyx/gHoRKZx6TYUhPJ/3Nt/B/bjZqd/eL7KyJWf6s+UQ99n8IlYrQr7/CmJ9v2ZNzpwkI33a4LrwRvh1HWedZnPp0NQGvvoJzr14X396OSnf9R/KjDyLKcgm7qgzN4Jm0fPBG8OtY6zZqv064dR2L2z1gLCigbOd/qFxSUA7+Qdr/Pscj4l30kdHQ/TaInAhaR9v9QvVkKi6mLC4OY2YmFTk5KOXlCJUK92uuQajV5n/r6jE2qY7s+gygco7ho0KI40KIJ+0VR6nBSGJOCQO8bmdqu6n130HlFYAxM5XCNWuoyMxsdEyjOplr8Sw/lEbvgN5sSdmCSTFdZKsGyooFQ1HV/f+irVupyMhA4924JNYgbUbAzYugMA2+HgXpZyqOGxITyV++nKSHH0YxGGwfW0PFbYAjf8KAhymJy8BUXIzK0dE6ZR1adIA7V0Or4bDhLURpFgm330H2d983yecCiqGcrOfuIP6m6VTkFOJ9zTB4aC+MevWCJ/9zqV1dcRpyJfS4nfKB75OX6k/cv36k/JNGxU/3wXsdzEk4+4QVf5u6MyQlkfvbb1XddzM++pj4SZM5NfMuUp58itTnnifl2dlgMPfOq1j1MRRmWDwOuyUAIYQa+AS4EugIXC+EqPs7bkHxWUUoCgwNHUoP/x7138HpgnBa8/3yevXoqEWIlxOdAt1YdjCVPgF9yC3L5XC2laZfOOcBcNHmzaicnHDs0sU67V1My35w61Jz19SvRsLxVQA4RUcT8OKLFG3aTNrrr9sntvoymeDfp8EtmIoON3LqrrtJef4F67bp6AHXL8Rh3GOEDYzFpaWatFdfJeXJpzA1YMIiazElHyXxmj6k/7wJ59buRCz6CbdZcxHO3o3ar0NEOK2XL8fz5pvIPaoQ+28YWUltUP77FD6Mhu8nw1FzDypbMhYWkfPLL8Rdex3Hh48g5ZlnKT1gHufifvV4gj74gJYLfqTVz1/R+n+30/q+1vBeG/i8P+qNL0GC5XsD2vMKoBdwXFGUE4qilAMLgavtEciJjCKENpti1RHKjeX130FVRVDzt9LGjgU4bXQnf3aezKGNq/m2lNW6g6bsAY0efNoBULxpM049eyK0Wuu0VxeBXeHOVebKpD9MgR1fA+Ax8Rq87ridnB8XkP1DHUt229O+hZC6D0a8QMbHX2AsKMB7+nTrt6tSwdCnUd/yI8H9MvHpVkHeH3+Q/c231m+7Lg4uhi+voCKnEL/briT4981oIrpabPdqDw/8n36aiKVLcOrdh5wjKpQHdsOQJyHtACy4Dt6PgjWvQ+4pi7Vbm9KjRzk+aBCps5/DVFxEi1mziPhzKfouXcBkQu9ahJt2G06b70W36Eq0W19BSxoiajJcMwfVI7vNt0gtzJ7PAIKAs498ItD73JWEEDOAGQChofUo0FYPsemFaN328equt7iq3ab61QECc1VLnQtqdQlguQQwKtKfd1ccY/sJI+0827EpeRPTO1vh5JG6D/w6gVqDISmJ8pMn8bzxBsu3U1/uwXD7Mvj1dvjzYfNYhZGv0OKRRyg/EUfuTz/jee219k1UF1JeZH7wG9SdEqUtuT8/j9ctt6BvZ4Nuvae1H4OYuQbfhTfg5JmAY4dSUBSMBQWo3dxsF8dp5UWUzLkHh5Q/UIVFE7b4C0SLBnSBrSOH8HBCPv+MipwcVJ6emHo/SMqfmXgPb4M+819Y96b5T+sR0P1WaDu6aiR8Yxhzc8lbshSh0+I5dSoOrVrhPmkSbmOuxLFrV0RFmXnmvT8/hmPLoCAFhApC+8HIV6HdlTZ5gG3PBFBTP63zblIqijIHmAPQo0cPq9zEPJFZhItzDm56b1x1rg3biaMXKlUR2uBgsFBvlTYtXIjwcebfA6k8Pe5pPBw8LLLfahTFfAuo00Tzv4XA8+abcR440PJtNYSDK0xdACtmw5ZPIWUfYsp8At9+G8VQ3nRP/gCbPoKCFJSJX5M66xU0Pj743G+BXj/15dMGpq/C+fe7YfVsjKd2Efd1Is4DBuD31JPWG+h3rtT95L50MylrS/EcNAj/l35DWOBkWxcaT/NVetmxYxRu+I/8v//Bc+p1+NzyPJr4pbD7e3PnA72HeR7rjlebZ8XT1P3LoGIyUbxtO7m//krB8uUo5eW4DB+O59SpCI0G/wduh5jl8NM7ELvG/NxN6wyth0P7q6DNyKopZm3FngkgEQg569/BQHIt61pVbEYhWucswtzDGr4TRw/UpgJar7Rc4TQhBKMi/Zm7/gQRriPwcLJ8LwByT5qngKy8/68NDMT/mact305jqDUw+nUI6m4uIPfFINTXfgOhfTCVl5P2yqt43XLzxcso2FJ+Cmz8ADpejdGjAyq9nhZPPIHaxcU+8ejd4Nrv4L/3UK18BTf/cLJ++omSXbsIeu/dhg1EqyuTCWXzp6S//Q7ZRxxx7toB3zfnW+Sbdn05dulCq2X/kPnxJ+QsXEjuosV4TJyI3+O7EQnr4cAiOLwE9nxvPjm37Avhg6Blf/MDdp1zrftOe/U1cn74AZWbGx5TJuMxegB6xxz4exac3AxpldVb3YKhy1Tzt/ywgaC1UQKugT0TwHagjRAiHEgCpgI2v++gKAonMopwcEsjzK0RDz0dPauVMrCU0Z38+WxtLKsOp+Ppe4zs0mwmt51suQbOegCsmEyU7tuHPjKyafYd7zzZ/J9w4Y0w/yoY9RrGkHEUrFlN4fr1hC34EW1AgL2jNFv9CpgqYMQLaDw9Cf2uCdx7V6nMBeUCutLi97twci0ieUcCcZMm4/vgA3jddpvlx1oUpmNcOIOkH/ZSlOKI53WT8Jv9gl0/XxpPT/xnP4vnjTeQ9dVXlMWdQOgdoe0oirLccBjwAprcfeZv63EbYMVzlVsK8AxD8QinrNCZwrgyCg+mEnBzXxz83XD3isVxYjiuwSWocj+Hv94xb6Z1Mo94H/YstL3SfLu1iQxUs9u7oChKhRDiPuBfQA18rSjKQVvHkV5QRqEhH6EU1H0WsJo4eUHqAVJmz0bt4UGLRx+1SHxRwe4EuOtZdjAVz5bL2Jq6lYltJtZ/sFptUvaCSgMtOlJ29CjxU68n8M03cL/aLs/jL86vE8xYC4tnwj+Po+20mdCP/8fJO+4m4fY7CP1mPtoW9RzJbWkpe2HPD9DvPnJW7cRlkAtaPzvHdLY2I+Debbgse5II919I2R9M4bI/8Jo2zXJtKAoc+A3+eQJTdjFlpUH4vzgLz+uus1wbjeQQEUHgq6+imMzdq415eSTMmAkGA7qwMHQREWj9x+LSfzYuwWA4vJ2kT/+mNPUISmW9SL1XORUb5uHgZ8DRyQvHQB9w9oewyhHJAV0hIMouVzt1YdeveYqi/A38bc8YYjMKweTA450/5YrwRlwGV84JUBZ7wqIDNoQQjOrkz4JtCbzWewD/xP/DwcyDdPbtbJkGUvaCbwfQ6inaZO5l5GSN6p+W5Ohhfi6w6QNY9TL65N2EvPIkCU+/Q8Jt0widP89+SUBRYNnT4OhJiftoUh+4E+8ZM2jxyMP2iac2Tl4wcQ6ayMkE//UISuZ6xI9TMHR9kLxNR/G65eaGPxvIjIG/Z1G87T8cO3dCe9vntHo+DJVj0xyIJSpHv6vc3Aj74XuKtm6lZO9eDAmnKNm5E21QMC5X3I7wH4D4/SSew9ujj4zEuU8fNL6+5p0oSpP5Vl8fTfA637biM4sBNUPDu+Pv7NTwHVVVBHXDkJpmsfjAPChs/qZ4jEXtUAkV6xLXWSYBKAok7zH3fMDc/1/XqtWlMe+sSgUDHjb3mvj1dpy23U/og3eT9NV6KtLS7ZcAjvwJJ/9DufJtUt/8Hxo/P7xnzLBPLHXRdiQiYidi2xxY/zb5f04lY487uT98Q4unnsV11Kjz5yaoTWYMbHgX066fSdvrSe4xX/z7zcSzRYdLouqkEALHqCgco6JqXK7x9KTlt9/UtrEVI7OeS+F9saqTWUU4uB1nT/bqxu3I0QsUE2oXJ4x5uRaJ7bSeYZ74uOhYd7iIrr5dWZ+43jI7LkiB4kwI6IKpvJziHTtw7tvXMvu2ldDecNcGaD0Cp7gPaXVnII6tzDWTjAUFto2losw82tS3A7nHnSk9dAi/Jx5H7VL7g8MmQeMA/e6HB/bgfffDhF5pRFWaStJDDxN3RV8Kvn235udbJpO5a+7WOfD1lSgf9aBg2Z+cWBNBbowDXnfcjvtkCz6vkiyu2V8BxGUW4eqzky/2rWRsq7EN39HpwWBOOkyNnBXsXBq1itGR/vy6M5GHJg3gr7glFBuKcdI24ooFqj0ALtm9B6W0FOd+/RofsK05ecH1C2DLp6hWPA9fDCJbcz1Zv60kdO4c6/ZwOdvWLyAnnopx35B+9+s49emD65VX2qZtS3DygsGzcO7/AOH7F5O34Guy1sRT+NMHuJ54CcXRB6PaF42LBgylkJtgnkMCwLsNaZkjyVl3AF0rf0LffQ7n3k27/pAkEwAns4pReWXQ0q2Rgy4q++/qArxwaN/e4qWLx0YF8v2WBALEaJZec6dldpq8xzz4xD8SRz8NofPn4djZQs8WbE0I6HsvhPSBX2/D6cSHZJUEE3/jTYR8+glOPRpQ4qM+chNg7Rvm22kRg3EdsR3v26fV/fZJU6JxQHSbike3qbgX52GK2wY5ByjZvYeTn2xD56tH561H6LtQUaQQ+OpL6KL64bJxI7r+J82D85piLzLpPM36FpDJpBCfVUA5aY3rAQRVVwCeI6IJ+/EHi3en6xnmha+rA//sM0+daJHCXil7wact6JxROTiYZ/9ybuK3Ky4muDvM3IC+zxW0HBiHRltKwu23k/fHH9ZrU1Hgr8fMP495G42nJ4Gvvdq0xiU0kHByR93pChjwMNrJb+D70EPoOvbGgD9leVqEVzAVBgcAXPr3x+uGG+TJ/xLSrN+ptIJSyslBh6Fxg8DgrDkBrDN/rVolGBPpz8Ltp/jpyCLm7v+UpdcsxVHTiJ4VKXsgfBDGvDyyvvoaj8mT0Fmp3IZNOXrAtd+hC/+Slo6zSdrgTvKTT6Hv1Mk6J+UDv0HMv5iGvkzKS+/jfddd6NvasNyDjWgDAvC5a6a9w5AsqFlfAcRlFqFyMJdubunWyIlcKucEKDl4mBPjxlGyf39jwzvPVVGBlFWYSMzQklac1rjicAVp5ofAAV0p2raNrDlzqMiwfLlZuxECet2J5v41hE7xJmRwJg5HPoXyYkzFxZZrJzcB/noEgnqQsbWU/L//wZiVZbn9S5IVNesrgJNZxRiLWvPDyL9p59vIboOVUxhSlk9ZzHGrnEx7tPTEz82BwydccNO5sSphFcNChzVsZ2c9AC76ZqW5/HMt3d+agiJDEc5a8+2pz/Z8xrbUbQgh8HX0paVbS6L9ounl3+v8AXItOiBmrsFl1Uuw+WOK/ltD0hot/i+90vjJbirK4bfpoCgUt32E7Lsfx+Paay+9nlRSs9WsE0B8ZhE6jZpIv2BUqkY+rFNrQeeKWm2ut260cE8gAJVKMKZzAD9sTWDCFYNYc2oNBpMBraoBowyrEkAUxZtetn/550qlBiObY7PYeTKHw6l5nCzaQ5ZmBRW6WAJy38DH2Q3FrYA8UYqLXkVS4R6WxS/Dx9GHlZNXApBXloe7w1kzhmkczBOMtB6BZv7daNUFJD34IIUTxuP39DMNq4qpKOa6RKe2UnHFxyQ98w7aoCBaPD7LQkdCkqyveSeArCK8Azex5EQZE1pPaPwOnTxRq8y3FyxVEvpcY6MCmLcxHg8lmoLyP9mRuoO+gQ34xpmyB7xbY8jMt3v5Z0VR2HEyh+82n2T5oVRKDSY0+lTcgpdicI5Fhwfh6qvw8dKRXWAk9lAXCso6ARDi5cjUKE/6tBUIISgzljFxyUTaebbjwegHaefV7kxDrYbi8NQmwjo9ROYvq8n8YwmFa9fi9+xzuI+9qj4Bm6uT7lsIQ58le3MaxuxswhYusF+xN0lqgOadADKLKffawObkcsskAEdPVKY8UKstPhjstG4hngS46zl+0oNbOt6Cv7N/w3aUshdCelMWH4/KxcVuty12J+Tw+j9H2BaXjatew5TuIXSNqOCVvc/irHXh/m6zmdB6QrU5GhRFIS6ziI2xWaw4lMbcdcl8sRZGdKjgzkHBXN/+euYfnM91f17HTR1u4p6u95wZM+HkhZj6Lb7dluHyzaOkrcvEuOZT6NcRxT0UVKoLd900lJh7/Oz5HnreCYMew6dPOc79+6PvaJcJ7SSpwZptAjCZFE7m5KL1ym58D6DTHD0Rpbm4DhuKNjDQMvs8x+nbQN9tPsn/rn0Id8cG3LYpyoK8U9BrBi79+9N2y2aLzWFQV8XlFbz+9xG+23ISHxcdL47vxOTuQTg7mH8fR5dX6RvYF0+953nbCiGI8HUhwteFm/u0JDWvlIXbE5i/KZ6VX6QxsmNX5oz8hZ9PfME3h75hzak1fHPlN/g4+pzZSbvROD4/gJZr3zIP4Pq4B3llA8neU4rXbdNxG3Nl9S6xigKxq83TO2YcwdTnETJ2O+Az0DyxirzvL12Kmm0CSCsopVyko0Uh3C3cMjt19IK8RII/+sgy+6vF2KgAvvovjn8PJhMRnEELpxaEutWj+2bKbvPflXMA2Lrf9rG0AmZ8u4OT2cXc3j+cR0e2xaAUcu+aO7m367309O/JmIgxdd6fv7ueh0a05c6BEczfFM/Hq4+z7lgG9w69kS+vuIplJ//GW1/DPLMOLohRL0Hfu2HDu6iX/oySoSNl9mzSXn4B58hgXKMjcG+vhxNrITsWk2sohS2fJf3dFRiSknDs0R23K66w3MGRJBtqtt1A4zOLUeks1AX0NCvNCXCuriEeBHk4snR/HHeuuJNfjv1Svx0k7gQEpQXOxI4da5Uuq7VZfSSNiZ9uoqjcyI/T+/DcuI6UKwVM+3ca+zP2U2QoavC+nR003Du0NasfG8wVHf14b8UxXl9czq1tHkMIQXJhMu/teA+DyVB9Q7cAuOodXN8/QMSnz9Dy9ra4hRsoOXSCnMX/wv5fwCOUuG3dOToPkt78GqHVEjpvnjz5S5e0ZnsFEJ9VhFAXoVFpLZcAnLygNJeUZ5+l/FQiLb+Zb5n9nkMIwVVRAXz9XxxDB/dm5cmVPNL9kbqXHUjcDr7tKdq5n/LjsWhsVDlz6d5kHvppDx0CXJl7Sw8C3B0pNhRz76p7SchP4NMRn9I74LxpoestwN2Rj2+IZnRkMk8v2s+YDzfwzpQu5GnXMe/gPI5kH+HdIe+eP/2n3g3RYxpOPabhBCiGUoyZGeAfDELgWvQZzmVlOEVH49yvnxzxKl3ymu8VQFYRoqAfW6/f1viiaqc5eoJiQikpwpCYaJl91uKqzgFUmBS8RXcSCxM5mnO0bhuaTOYEENKTok2bbFb+ecneZB5cuJvuoZ4snNGXAHdHDCYDj657lINZB3lr8FsWOfmfbWxUIMseGkQ7f1fu+WEXqae682Lfl9ieup1b/rmFzJLMC24vtHo0ASFVpX597r6bFg89hMugQfLkL10Wmm8CyCwi1NsJnSX/I1eWg1A5O1itG+hpUcHuhHg5cjIxHJVQseJkHecizo6F0lxMft1sVv554/FMHvlpDz3DvJh/e09cHMzHXFEUWji1YHaf2QwPHW6VtgM9HFlwZx8mRQfzwaoY1uwM46Nhn5JUmMS0ZdNIL063SruSdClovgkgq4gy77n8fcKCE5JVloNQO2owFRaiGAwX2aDhhBBc1TmQbcfL6ebbgy0pW+q24altAJTkutmk/POxtALu+m4nEb7OzLmlB04688nfpJjQqXW82O9Fy85xXAO9Vs07U6J4YnR7lu5N5ssVaj4Y8il+zn6Nq6UkSZe4ZpkAFEXhZG46uewlu9SCD21PzwmgN98ysPaEJGOjAjCaFPq63sv80fPrtlHiNnBwRxXUHrcxY3Dq1dNq8RWWVXDXdzvR69TMm9arqsvqsZxjTFk6hRN5J6zW9rmEENw9pBVvTOzMumMZvLe0nP8N+gxXnStlxjLLfg4k6RLRLBNARkEZBmGettFiD4ChKgE4+LviNn6cue+4FXUKdCPCx5m1h8rrXg7i1HYI7o5jZCRB771rtZGriqLw9KL9xGcV8dH13QjyMH/TLjOW8eSGJ8kqycJN14ASDI00tVcoH13fjV0JuUz/Zgcl5UZmrZvFncvvJK/MurftJKmpaZYJ4FTOmS6gFhsEBlWTwjhHeBD01ltovGvoe25BQgjGdQlkS1wW3+3/lenLp194noDibEg/iNG7O+UJCVaN7aftp1iyN5lHrmhLn4gzx+GjXR8RkxPDS/1fqj4wy4bGRgXy3rVd2BafzczvdzKpzbWcyDvBfavuo9hgwUqhktTENc8EkF2CyiEDjdAS6GzBEbt6D/PfJebbCRaZtOUixnUJRFFgz6kctqZs5WDWwdpXjt8AQH6citiRoyg/edIqMSXllvDyn4fo18qbe4acqb+/LWUb3xz6hmvbXsug4EFWabuuru4axBsTO7P+WAY/rtXzxoA32Ze5j1nrZ1FhqrBrbJJkK80yASRkF6OYtPTw64FaZcESCGoNOLhTnpTEka7drDsLVaXWLVzoGODGsbiWaFQa/o3/t/aV4zaA1pnCvfFoQ0PRWmHyF0VReGbxfhTgzUlR1aqs/nzsZ0JcQ3i0x6MWb7chrusZynNjO/LvwTQ27w/imd7PsD5xPR/u+tDeoUmSTTTLBHAquxiPsrHMHTXH8jt39ECtFKKUlmKyclfQ08Z1CWT/KQPRvn34N/7f2q884jdgCuhF0bbtuAwebJX5av/Yk8zaoxnMGtWOEK/q4yveGPgGc0fOtdy4Cwu4fUA4dwwIZ/6meAoyevJg9IOMazXO3mFJkk00zwSQU3zeycliHD1RKXkghNXHApw2NioAAJeK7qQUpbA3Y+/5KxWkQsYRiorDUEpLcRk82OJx5JcaePnPQ3QL9eCWvmFVrycXJpNXlodGpSHIJcji7TbWM2M6MLqTP6/8dYhgMZY2nm1QFIVT+afsHZokWVWzHM54Mu8URr85bEp+iX6BFu4H7+yDKMpE7e5ORY515gc+V4iXE9GhHhw+rmFM9Jia+7YfM98aKogpReXujnPvXhaP45PVx8kuLmf+tF6oK2/9KIrCM/89Q1ZpFovHL7bsLTcLUakE/7uuKzd8uYWHftrNr5792Jn7O5/s+YRvr/yW9l7t7R1ivSRkFbMuJpFDaRkkZ2nJLMkkWf8BiqoYRZShFlr0akdGBE3h0T634+Kg4mjOUdp6tkWjapanhGar2b3bBqOJzLJT6JU0nDRWuApwbgHph1H7RGDMtN3csOO7BPLC0kN80u5Z2ni5nr/C0X/APRS/B1/H48QJi8/+FZ9ZxNcb45gcHUzn4DOzca05tYYdaTuY3Wd2kzz5n+aoUzPn5h6M//g/Zn63k3l3jOBb3bc8uPpBFo5dWGNZ6qYkObeEn3ec5JdDq8gSG9C4HEUpjCZMuQ0fVy+K1IGoFD3GCh2F5WVkG4r4MT6P75cvp31oIaecXsFZ68KwkKGMDBtJ/6D+DZtpTrqkNLsEkJJbCjrzfL0WHQNwmosvFGXgcfUjqFxrOBFbyZioAF768xBL9yZzTW8dWpWWYNdg80JDibmccfTNqD09cere3eLtv/b3YXRqFbNGnZmBq8JUwQe7PiDcPZyJbSZavE1L83V14IubuzPl883MXpTAuxP+xx3LpzFr3Sw+v+LzJvntODajkI9WxfDXib/R+a5A5Z6Fm9qd4cETmdh+ND39Tw/061Ntu+LyCvYn5vHf8UxWHztJSdL1GFyO8nf5KpaeWIqvoy+fjfis+oxq0mWn6X2irez0GAAnjat1vtU5twBjOd43TaoaGGYLLVz19G3lzZJ9J1mY8SxXRVzF832fNy+MXQ0VJWTuEWhKFuExybIn421x2Sw/lMasUe1o4aavev2P439wIu8E7w99v0mePGsSFezBm5OieOinPSza4srsvrOZvXE27+18j8d7Pm7v8KrklRh4+98j/Lg1AQeNmi6RBVTofLi721MMCxmGVn3hb+9OOg29I7zpHeHNoyPbcSKjH4t3J7FgexwFyn4K/PazL05Law+Fw9kHCXYJxuN0N2fpsnFp/K+0oFPZxah0GbR0DbNOAy7m0spKQTomg7phE4430LioQJ5clMWVnQfxb/y/PNnrSRzUDrB3AUatL5m/rMB9rN7iCeDDVTH4uDhwe//qE+scyDpAF98uDAsZZtH2rG1CtyAOpeQzZ/0JOgZEM73zdDr7dLZ3WFWW7k3mhT93U+TyG6N7jOalUVfj6jgUnUrX4J5dEb4uPDqyHfcNa82SPZ2Yu6EXs345zNz1CZQHvEapKZ87Iu/g1k63Vpues6kzmozklOWQX55PuFs4QgjSi9MxmAz4OPiCUKNVqap1V25Oml0CSMguRikL4YqwrtZpwNkXgKx535IxbxHt9uxGpddfZCPLGB3pz+w/DqAp6UFB+b+sT1zPFb7d4egy8iuuQCnZg8fkSRZtc+fJbP47nskzYzrgqKt+j//5vs9TbCi2SndTa3tidHsOp+Qz+48DLJxxM91DzaO8y4xl5qRqB4VlFTz3xwEW7zuEV8QP6NSn6NdhGD4ulovHQaNmSo8QJkUH89f+FN5bcYyEQ1Np0XI1H+7+kD9i/+CpXk/RP6i/xdq0tP+S/mNRzCJic2NJyD9FhWIuythd+Zz0PBPJ6gUYXc2DIk0GV0xlAagMwegLx+Dr4kiwpyNBHo5E+LrQMdCN9v6uuOovz+chdkkAQogpwAtAB6CXoig7bNX2qZwSWhgmcWfUUOs0UHkFoNGb++JXZGahC7ZN10cPJx2D2viy/bAW31a+LI1dyhWpJ8BkIHdfAQ5t2qCPirJomx+uOo6Xs44b+5wZVJZXlkdeWR6hbqFNqs9/fahVgo+vj+bqT/5j5ne7WHJff7ZnLuezvZ/x/ZjvbV7G4mByHvf9uJtTxYfwbbcAldrAewM/YnCI5bvzgrln1LgugYyO9Oe3nRG8v7IlxcZ9pIb8xV0r72LeqHn08O9hlbbrw6SY2J66neXxy5nZZSZlpS78vGcvGzN2U1bsh6G0LyaDJ2rFmVTHMoLcXQlzvhKDJpIyJZtiUwbpZbGUm/YzNHQ6GQWlHC5ZzNbsMop2t8FUGgSoCPVyontLT3qGedEr3JNWvi6X5Bebc9nrCuAAMBH4wtYNJ2QXEOxlxW9wzuYEoNabADBmZYKNEgCYB4WtOpLOZO9h/Jf6JyUZ61CIovRILH7PPGPRD+3eU7msO5bBE6PbV5V5Bvhi3xf8dOQnlk9ejrejdeshWZO7k5Yvb+3BNZ9s4s5vd/DytRFklWTxyNpH+HLklza7FbL8YCoPLtyDi1sqrmFf4uscwIfDPqSVRyurt61Vq5jaK5QJ3YKYtzGMT9e2pcxhJz9t0BA0soTM8uO092p/0WcOlpZdms3vx3/nl6O/kFiYiFY4sGm/H4dPBAGhhHk/x8hWPvSJ8KJToBth3s5o1LUPe6owVVQ9p7p31Xw2JG7A2Ws5LhoPAnVd0ZRGsyHGyOLdSQB4OevoGeZJr3Bveod70SHArarrs6WdHZul2SUBKIpyGLBLBk0o+w+Ty28kFS6xzqAkJy8QKjTaUgAqsmzXFRTgio5+6LUqVHlDWd4hDMdj91Ic9QrOCXssfu//o9UxeDhpubnvmd5USYVJLDyykLERYy/pk/9prVu48uEN3bhj/nbmrnTixX4v8cSGx3lt62s83/d5q36GFUVh7oYTvP7PEaKC3Pni5mv5I76Mqe2n4u7gfvEdWJBeq+buIa2Y2jOET9aE8+3mkyzZfxyn1m8Q5BLA7L7PWnxGt9rkluYy8teRlBnLcKcdhpQbKMjrgKePB7NGBTO+S2C9B3qefYL9ZPgn5JTmsCl5ExuSNrAxaSNj2gay8KbbiM0s4KMdX1Kc14qDCSr+PWiuKuym19AzzIveEV70DvemU6DbBRNObUoqSjiafZSDWQc5lHWIQ1mHAFh89eJ676sumvwzACHEDGAGQGgja9cUl1dQpKTgiJEWTlaaB1elBicfNBrz5OYVGReedtDSnB00DG/vx38HknE9+Tr4dcZpwj2ETrRsH/wDSXmsPJzOo1e0rZrhC+Dj3R+jEiru6XqPRduzp6HtWvD0mA688tdh2vi1YXrn6Xy5/0vaerblhg43WKXN8goTz/1xgIU7jxLR4V/eH/88/u5OzOwy0yrt1ZWns45nx3bk1n5hvLv8KEtjrqciYCnTl09neOgI7u92n8WvTIoMRfx14i9O5p9kpP+dLN6dhCFjLEU5oWi1IUztEsjE6CA6B7lbLCF76j25KuIqroq4CqPJSElFCUIIjOoUVqfPA8A3zJcx3TrjYGqJMT+K/SeLWHXEPMOci4OG7i096RXuRWSQO+39XWnh6lAVX7mxnJSiFI7nHicuL47bI29HJVS8uuVV/og11xDz1nvT0bsjkT6RKIpilS8bVksAQoiVgH8Ni55RFKXOVdIURZkDzAHo0aNHo8prJuaUoNJl4O0QaN1BLi4t0Ig8fO65G33HDtZrpxbjugTS4/AbHClO459kbyalHqNloGXj+Gh1DK56Dbf2D6t67Uj2Ef468RfTIqfh71zTW3/pumNAOEdTC/hwVQwfTJ3EkJDjFFdYp3R0XrGBu77fydbEwwR2/JEcJZMTeccI9wixSnsNEeLlxPtTuzE9KYLXl/Vie8YiVpk2sDphNUuu/pswj8ZdXSuKwpHsI/wW8xtLYpdSUlGMxhDCJ7+1RafRcUWH8UwcHcSgtr5oG/BNuz7UKjUuOvO8Ge282rHm2jVsTNrIf0n/sT9zP0mFq5k7ci59pvTh96P/8v7u/0GFKwdKVGw9oIYDCmVpY3HX+eIbsIcs3e8YqD5ZlIexN54OLQjRDOeGll3x1IRTWuJKZmEZe1PKuPq/jbx8dSRdQjws+rtZLQEoijLCWvtuqFPZxagcMghxtfLgFmdfRGkWvg88YN12ajG88E+0mn9Zm9CXK9edZLvf+7R8zHKPWw6n5PPvwTQeHN4Gt7N6R+zP3I+X3ovbI2+3WFtNhRCCV66JJD6riFm/7Oer22YzsI35KtJgNFjsHnh8ZhG3z99OcvkuvNosRKd14tOh8+jaoqtF9m9pkUHu/HDHQDbEtOfVZTs4XrCTiR8d4vpeBRQ6/UEH35YMDh5MgEvARfdlMBlQFAWdWsdX+37kgz1vIBQN5XmdKc/pS3f/LkycGMyYzgFVs8vZg4+jD1e3vpqrW18NmDs9nC6/EuLhQ4+ASLJKsiipKKGwvIByI4yKDCUzx539Wb6I4igqip0xlLtjKmuBqbwFjx1OAE7P0eEIpAKpeDpp8XFxwN9djzWKyzf5W0CWFJ+Vj0qXRTsvKz88c2kB2bFU5OSglJaiDbj4h98i8lNg3Ztod87jUFoXvDYkEt/Rm4+DDnO1BU9SH68+jouD5rx+/1PaTmFsxNjLdp5dB42aubf04LovtjDzu138eGcfVPoEHlv3GG8NeqvRJ+lNsZnc/f0ucNqHLuhbIjza8+GwDy+Jq6mBbXz5p/VoNsX2ZP6meD5bexR9yzWoHZN4deurtNAHE+nTkfGtxzC85XCKDcX8Hfc3BeUFJBYkciQrliM5B4l2ns6phA4czRRoXCYS5tiba7q0ZVxU/e/r28rZz2O6+3Wnu9+FRtpHAdNQFIWcYgNFZRWUGowUlxsRAnQaFTq1CiedBi9nHTqNda9u7NUN9BrgI8AX+EsIsUdRlFHWbjchpwAlZxhXhFl5MhKXFlCYTtIDD6KYTIT98L1l9qsokLwbknZC9gkoKzD/Kc2tqvapoCIzdxhizREO+bTi5O3Xk5X1BqtOrWJ02OhGhxCTVsDfB1K4Z0gr3J3MCcWkmDiQeYAo36jL9uR/moeTju/u6MXkzzdz27xtfHBjOFqVljuX38k7g99pcLfMH7ae5Pk/DhLu48zHN97GsiQtM6JmXFLHUwhB/9Y+9G/tQ3JuJ/7e35klh3ZxJG8HSU4nSM3fwaq9GvwUNWptMSnuLwKgGPWYyn0xlnRlfZyge4COx7v2ZkSHcbTxs105FVsSQuDlrMPL2b6D6oQtZq2ylB49eig7djR8yMCd3+7gZFYRyx+2Tt/pKls+g2VPkpR3KyX7D9N65YrG7U9R4PBSWP0yZB4zv6ZzAQc30DmbS044eUNwdypCriDutodw7teXaV4jcHZxoLjFKwS5BPHVqK8a/as9uHA3Kw6l8d8Tw6o+vMviljFr/Sy+GPEF/YIsXF21iUrIKub6uVvIKzHw7tRWzIt9lsPZh7mry13M6DyjzoXvSg1GXv3rMAsO/oVv0HaWTpmPj7N15mm2l8KyCvadyuVQSj6JOcWkF5RRVmGkXMnF09EVPxd3Inxd6BDgRjs/1/MGFEqNJ4TYqSjKeQM3mtctoJxkAjxt8I3C1XzLR+vhRH5aGorJhFA18FKuogyWPgR7f4QWHWH8x9BqGLgFwlm9Aoy5uahcXdGo1YT/+gtqHx8mrjd3IXy2zx14uagb3ZMgNqOQpXuTuXNQRNXJv9xYzvu73qetZ1ubdQNsCkK9nfj17r7c8tU27v8hhufHvUSY+9d8uudTvBy8uK79dRfdx/H0Qmb+tJRksQjH4CMEebbHoBQAl1cCcHHQ0K+1D/1a22cOaKl2zWZCGEVRSFH/ziHVbOs35mbuAaFx1YDBgLGhYwEqymHB9eaT/5CnYOYGiL4Z3IOqnfzL4+OJu/Y60t99z9yury9CCK6JDkKtEmSmdmZim4mN7kb2yZrj6DQq7hwYUfXagiMLSCpM4tEejzbpcs/WEODuyM8z+xId6sEzi2MoS57Ka/3eY2Jb83iLVQmrWHlyJfnl+dW2KzUYefPfvVz92y2ku72Gq0ciD3d/mAVjf6zTw1JJspRmcwWQW2zApE7Hx8EGXenczBPNa52NABhS09D4+tZvH4oCSx+E2FUw/iOIvqXG1Yp37ybx7ntACFyvqN7xqoWrniFtfVm0K5G7hwbzR+wixrUa16AqqCezivhjTzLT+oVV1Z7JK8vji31f0D+ov+Un1rlEeDrr+GF6Hz5efZwPV8ew6rCa2AFxXNsjhO8OfcfOtJ0IBN6O3uhUjjgqISQdm0JGQSlhHZ0Z234GM7reipvOdkUDJem0ZpMAErKLUDmkE+Iaaf3GXPxAqND7qvB/4QW0/n7138feheZv/oOfqPXkX7huHYkPPIjG34/QOXPQtTx/foMbeodyxzc7+GXvfj448jaFhsIGDdL6cNVxNCrBjEFnvv0fyzmGRmh4tHvTmOTdXtQqwYMj2jCmsz9v/HOE91fG8P7KGFp630Qr76EUEUNOdjqF5cWYytzo5efKR9d3o0/EWHuHLjVzzSYBHElPRahLaOsZcfGVG0utARd/tKpcPKde/F7wefKS4O9Z0LK/OQHUwJifT9Jjs3Bo1YqQL+ei8fKqcb2h7VoQ5u3EXztNDGk/hB+P/MhtnW6rV5G2ExmFLN6dyO39w6vV++/p35MVU1bYrTpmU9PGz5WvbutJQlYxf+1PYc+pHNILHHEytaKtpxOdg90Z2dGPCN/L6x6/dOlqNgngYGYMAF3829qmQbdAyE+i9OgxhEaNQ6t6jD1Y+QIYy+HqT8ylJWqgdnMj5NNPcGjTBrWHR627UqkEt/YL48Wlh7h+0LWsPbWWhUcX1muw1oerYnDQqJk52Pw7KIrCyoSVDA8dLk/+NQj1duLuIdYv1CZJjdVsHgIXFXogsq6lR4BlyyHXqjIBJN5/P5mffFL37RK2wv6fod/94BV+3uLy+Hjy//4bAKeePS948j9tcvdgXBw0bDzgwqDgQXy570tyS3PrFM7x9AL+2JvMLf1a4utqPtn/EfsHj6x9hFUJq+r8a0mS1PQ0mwSQnutAS+1QvPQ13yqxOI9QyDmJ1t8PQ0pq3bdb8yq4+MOAh89bZMzP59Rdd5P6yqsYCwpq2LhmrnotU3uGsHRvMpPCZ9DWq+15PVNq89rfR3DRaZhR2fMntSiVt7a/RbcW3RgeOrzOMUiS1PQ0mwQQV7gfH8882zXoFQ7GMrTe7hhS65gAUvdD3Drocxc4VL9PrCgKKc8/T3liIsEffoC6nhPO3zWkFQ4aNb9vNTF/9HxC3S5eWXXdsQxWH0nn/uGt8XZxwGgy8vR/T1NhquCV/q+gEs3m4yNJl6Vm8T/YZFLIcfqedE2di5A2nqf59o3GXUdFejpKRcXFt9nyGWidoPtt5y3K+/0PCv5Zhu/99+PUo/4zMfm4OHBb/zCW7kvmcEo+2aXZ/G/n/zAYDTWuX1Zh5OU/D9HS24lb+4UB8Nnez9ieup2nej1VpwQiSVLT1iwSwKncfIQ2m2CXMNs16mW+ZaJzE2A0YkhKuvD6BWmw/xfoeqO5tMNZKjIySHv5ZZx69MB7+h0NDmnmoAg8HLU8+/sB9qbv4+sDX/PW9rdqXPfDVTEcTy/khfGdcNCYH0T3D+rPtMhpTGg9ocExSJLUdDSLBLAr+ThCKLSxRRfQ09xDQKXBuaWGkC8+R+NzkWHwO74CowH63H3eIrWPDwGvvUrAG28g1A0fbevhpOOZqzqy82QOJ0+15NaOt7Lw6EK+PvB1tfU2xWby+boTTO4ezNB2LTiafRSAbi268Uj3Ry6LuVAlSWomCeBAemUXUL82tmtUrQH3ELSmNFwGD0bl7Fz7uoZS2P4VtB0N3tW7DypGI0II3EaPtsjk8pOigxjWvgWv/HWYfl63MCpsFP/b+T9e2/oaJRUlxGYUcu8Puwj3ceax0aG8tvU1Ji+dzIqTjSxoJ0lSk9MsEkBs3gkAegTZaAzAaV4RkHWcok2bKNq2rfb19v8MxZnQt/oIXVN5OXETriH3t0UWC0kIwftTu9LS24k7v9nFEM+HuLnDzaw9tZYNMRlc98VmFJctdOm6nEl/jmXBkQXc1OEmBgdbuYKqJEk21ywGgvmLwegzPfF0tHFtcb+OsHUOaW+/g8bXB+devc5fR1Fg86fg1xnCBlZblLtwIWUxMWj8GlBK4gLc9Fp+vLMP07/ZwX0/7iXEqwdCFcmMHfuJ8HVGHbKJzan59A3sy7TIaUT62KB8hiRJNtcsEsAbE/pjMPa1fcN+ncFYhs7fk9ITJ2teJ3Y1ZByGCZ9XL+9cWEjmZ5/j1LcPLgP6Wz40Nz2L7unHol2JrDuWgcnkzp0DfJjSPRgTvdBr9LKbpyRd5ppFAgCsPnF0jfzN35wdvHUUrEvEVFKCyvGcGZ62fGouHhc5qdrL2V9/jTEnhxaPWK/Qmlat4rqeoVzX89wunU1z6j1JkixLfsWzJp+2oNbh4FEOJhNlMTHVl2ccheMroeedoDkzNZyxsJDs+d/gOno0jp3l7RdJkqyj2VwB2IVaCy06oC8xjwEoPXoUx6izahFt+RQ0euhRvTCb2sWFlt9/h6qeo30lSZLqQyYAawvti3bHN4T/ugKHtu3OvF6QBnsWQNfrwdn7vM30HTvaMEhJkpojeQvI2sIGIIwl6F0LELozt3nY+hmYDNDvgWqrZ335JclPPFm30hGSJEmNIBOAtYUPArWO0hXfk/rKqyjl5VCUZR741fHqagO/TMXFZH31NRW5OQiNvDiTJMm65FnG2vTu0HoE5RtXkbNShduYK3FK/h7Ki86b7Sv3l18w5uTgM3OmnYKVJKk5kVcAttBrBk6u6QAU//mNue5PrzuhRYeqVUzl5WR9PQ+nnj1xio62V6SSJDUjMgHYQsQQNF3H4uBuoGjVUgjuCSNeqLZK3uLfqUhLw1t++5ckyUZkArAFIWDSl7gMGkhxhp6Ksd+AtvqAMJchg/F99BGc+/ezU5CSJDU3MgHYisYBt+lPovH2oTw57bzFWj8/fO68U5ZaliTJZmQCsCGHtm1pvXZNtXv8FVlZJNw5g7LYWDtGJklScyQTgA0JIRBqNUp5OUVbtmAqLSX5iScp2rLFXBVUkiTJhmQ3UDvI+OgjsuZ+icrJCVNxMf4vv4RD69b2DkuSpGZGJgA78L3/ftTu7pTFx+M+dizOffrYOyRJkpohuyQAIcTbwDigHIgFpimKkmuPWOxB6HR4T59u7zAkSWrm7PUMYAUQqShKFHAMeMpOcUiSJDVbdkkAiqIsVxTldLWzLUCwPeKQJElqzppCL6DbgX9qWyiEmCGE2CGE2JGRkWHDsCRJki5vVnsGIIRYCfjXsOgZRVH+qFznGaAC+KG2/SiKMgeYA9CjRw/ZV1KSJMlCrJYAFEUZcaHlQohbgbHAcEWRneAlSZJszV69gEYDTwCDFUUptkcMkiRJzZ29ngF8DLgCK4QQe4QQn9spDkmSpGbLLlcAiqLIYa+SJEl2Ji6l2+9CiAzgZAM39wEyLRiOpci46kfGVT8yrvppqnFB42JrqSiK77kvXlIJoDGEEDsURelh7zjOJeOqHxlX/ci46qepxgXWia0pjAOQJEmS7EAmAEmSpGaqOSWAOfYOoBYyrvqRcdWPjKt+mmpcYIXYms0zAEmSJKm65nQFIEmSJJ1FJgBJkqRm6rJLAEKI0UKIo0KI40KIJ2tYLoQQH1Yu3yeEiK5pPxaOKUQIsUYIcVgIcVAI8WAN6wwRQuRVjozeI4R4ztpxVbYbL4TYX9nmjhqW2+N4tTvrOOwRQuQLIR46Zx2bHC8hxNdCiHQhxIGzXvMSQqwQQsRU/u1Zy7YX/CxaIa63hRBHKt+nxUIIj1q2veB7boW4XhBCJJ31Xo2pZVtbH6+fzoopXgixp5ZtrXm8ajw32OwzpijKZfMHUGOeYSwC0AF7gY7nrDMGc/lpAfQBttogrgAguvJnV8yT4Jwb1xDgTzscs3jA5wLLbX68anhPUzEPZLH58QIGAdHAgbNeewt4svLnJ4E3G/JZtEJcIwFN5c9v1hRXXd5zK8T1AvBYHd5nmx6vc5a/Czxnh+NV47nBVp+xy+0KoBdwXFGUE4qilAMLgavPWedq4FvFbAvgIYQIsGZQiqKkKIqyq/LnAuAwEGTNNi3I5sfrHMOBWEVRGjoCvFEURVkPZJ/z8tXAN5U/fwNMqGHTunwWLRqX0gQmWqrleNWFzY/XaUIIAVwLLLBUe3V1gXODTT5jl1sCCAJOnfXvRM4/0dZlHasRQoQB3YCtNSzuK4TYK4T4RwjRyUYhKcByIcROIcSMGpbb9XgBU6n9P6Y9jheAn6IoKWD+Dwy0qGEdex+3C020dLH33Bruq7w19XUttzPsebwGAmmKosTUstwmx+ucc4NNPmOXWwIQNbx2bj/XuqxjFUIIF+A34CFFUfLPWbwL822OLsBHwO+2iAnoryhKNHAlcK8QYtA5y+15vHTAeOCXGhbb63jVlT2P28UmWrrYe25pnwGtgK5ACubbLeey2/ECrufC3/6tfrwucm6odbMaXqvXMbvcEkAiEHLWv4OB5AasY3FCCC3mN/gHRVEWnbtcUZR8RVEKK3/+G9AKIXysHZeiKMmVf6cDizFfVp7NLser0pXALkVR0s5dYK/jVSnt9G2wyr/Ta1jHXp+z0xMt3ahU3ig+Vx3ec4tSFCVNURSjoigmYG4t7dnreGmAicBPta1j7eNVy7nBJp+xyy0BbAfaCCHCK789TgWWnLPOEuCWyt4tfYC805da1lJ5j/Er4LCiKO/Vso5/5XoIIXphfm+yrByXsxDC9fTPmB8iHjhnNZsfr7PU+s3MHsfrLEuAWyt/vhX4o4Z16vJZtChxZqKl8UotEy3V8T23dFxnPzO6ppb2bH68Ko0AjiiKkljTQmsfrwucG2zzGbPGk217/sHca+UY5qfjz1S+dhdwV+XPAvikcvl+oIcNYhqA+dJsH7Cn8s+Yc+K6DziI+Un+FqCfDeKKqGxvb2XbTeJ4VbbrhPmE7n7WazY/XpgTUApgwPyN6w7AG1gFxFT+7VW5biDw94U+i1aO6zjme8KnP2OfnxtXbe+5leP6rvKzsw/zCSqgKRyvytfnn/5MnbWuLY9XbecGm3zGZCkISZKkZupyuwUkSZIk1ZFMAJIkSc2UTACSJEnNlEwAkiRJzZRMAJIkSc2UTACSdAFCCA8hxD0XWO4ohFgnhFBfYJ2VtVVzlCR7kglAki7MA6g1AWCuubNIURTjBdb57iL7kCS7kAlAki7sDaBVZS34t2tYfiOVozSFEAFCiPWV6x4QQgysXGcJ5lHNktSkyIFgknQBlRUa/1QUJbKGZTogQVEU/8p/PwroFUV5tfKWkJNiLvGLECIG6KMoiq3KVUjSRWnsHYAkXcJ8gNyz/r0d+LqyuNfviqLsOWtZOuZh/DIBSE2GvAUkSQ1XAuhP/0MxTzoyCEgCvhNC3HLWuvrK9SWpyZAJQJIurADzVH3nURQlB1ALIfQAQoiWQLqiKHMxV3iMrnxdAP6YpxaUpCZDJgBJuoDKe/YbKx/q1vQQeDnmio5gnqd4jxBiNzAJ+KDy9e7AFuXMdI2S1CTIh8CS1AhCiG7AI4qi3HyBdT4AliiKssp2kUnSxckrAElqBEVRdgNrLjQQDDggT/5SUySvACRJkpopeQUgSZLUTMkEIEmS1EzJBCBJktRMyQQgSZLUTMkEIEmS1Ez9H8Uxng2jK69UAAAAAElFTkSuQmCC\n", 280 | "text/plain": [ 281 | "
" 282 | ] 283 | }, 284 | "metadata": { 285 | "needs_background": "light" 286 | }, 287 | "output_type": "display_data" 288 | } 289 | ], 290 | "source": [ 291 | "model.to('cpu')\n", 292 | "y_pred = odeint(model, y[0],t)\n", 293 | "with torch.no_grad():\n", 294 | " fig=plt.figure()\n", 295 | " ax = fig.add_subplot(111)\n", 296 | " ax.plot(t, y, label='true trajectory')\n", 297 | " ax.plot(t, y_pred,'--', label='predicted trajectory')\n", 298 | " ax.set_xlabel('t (s)')\n", 299 | " ax.set_ylabel('y')\n", 300 | " plt.legend()\n", 301 | " plt.pause(1/60) " 302 | ] 303 | } 304 | ], 305 | "metadata": { 306 | "kernelspec": { 307 | "display_name": "Python 3", 308 | "language": "python", 309 | "name": "python3" 310 | }, 311 | "language_info": { 312 | "codemirror_mode": { 313 | "name": "ipython", 314 | "version": 3 315 | }, 316 | "file_extension": ".py", 317 | "mimetype": "text/x-python", 318 | "name": "python", 319 | "nbconvert_exporter": "python", 320 | "pygments_lexer": "ipython3", 321 | "version": "3.8.5" 322 | } 323 | }, 324 | "nbformat": 4, 325 | "nbformat_minor": 4 326 | } -------------------------------------------------------------------------------- /weakformghnn/__init__.py: -------------------------------------------------------------------------------- 1 | from ._src import * 2 | -------------------------------------------------------------------------------- /weakformghnn/_src/__init__.py: -------------------------------------------------------------------------------- 1 | from ._models import * 2 | from ._weak_form import * 3 | from ._vector_calc import * 4 | -------------------------------------------------------------------------------- /weakformghnn/_src/_models.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.utils.data import Dataset 4 | from ._vector_calc import curl 5 | import math 6 | 7 | __all__ = ['PosLinear', 8 | 'ConvexFunc', 9 | 'ConvexFuncZero', 10 | 'ConcaveFunc', 11 | 'ConcaveFuncZero', 12 | 'ScalarFunc', 13 | 'ScalarFuncZero', 14 | 'ScalarFuncZeroPos', 15 | 'ScalarFuncPosUnbnd', 16 | 'ZeroDivMat', 17 | 'GHNN', 18 | 'HNN', 19 | 'ODEFCN', 20 | 'ReHU', 21 | 'GHNNwHPrior'] 22 | 23 | 24 | class PosLinear(nn.Module): 25 | """ Linear layer with positive weights only 26 | """ 27 | __constants__ = ['in_features', 'out_features'] 28 | 29 | def __init__(self, in_features, out_features): 30 | super(PosLinear, self).__init__() 31 | self.in_features = in_features 32 | self.out_features = out_features 33 | self.weight = nn.parameter.Parameter( 34 | torch.Tensor(out_features, in_features)) 35 | self.reset_parameters() 36 | 37 | def reset_parameters(self): 38 | nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) 39 | 40 | def forward(self, input): 41 | return input @ torch.pow(self.weight, 2).T 42 | 43 | def extra_repr(self): 44 | return 'in_features={}, out_features={}'.format( 45 | self.in_features, self.out_features 46 | ) 47 | 48 | 49 | class ConvexFunc(nn.Module): 50 | def __init__(self, ndim, nhidden, nlayers): 51 | super(ConvexFunc, self).__init__() 52 | self.linears = nn.ModuleList([nn.Linear(ndim, nhidden)]) 53 | self.poslinear = nn.ModuleList() 54 | hidden_lin_layers = [nn.Linear(ndim, nhidden) 55 | for j in range(nlayers - 1)] 56 | hidden_pos_layers = [PosLinear(nhidden, nhidden) 57 | for j in range(nlayers-1)] 58 | self.linears.extend(hidden_lin_layers) 59 | self.poslinear.extend(hidden_pos_layers) 60 | self.linears.extend([nn.Linear(ndim, 1)]) 61 | self.poslinear.extend([PosLinear(nhidden, 1)]) 62 | self.activation = nn.Softplus() 63 | self.ndim = ndim 64 | 65 | def forward(self, x): 66 | z = self.activation(self.linears[0](x)) 67 | for j in range(len(self.poslinear)): 68 | z = self.activation(self.poslinear[j](z) + self.linears[j+1](x)) 69 | return z 70 | 71 | 72 | class ConvexFuncZero(ConvexFunc): 73 | def __init__(self, ndim, nhidden, nlayers): 74 | super(ConvexFuncZero, self).__init__(ndim, nhidden, nlayers) 75 | self.conv_forward = super(ConvexFuncZero, self).forward 76 | self.ndim 77 | self.register_buffer('zero_input', torch.zeros(1, ndim)) 78 | 79 | def forward(self, x): 80 | return self.conv_forward(x) - self.conv_forward(self.zero_input) 81 | 82 | 83 | class ConcaveFunc(ConvexFunc): 84 | def __init__(self, ndim, nhidden, nlayers): 85 | super(ConcaveFunc, self).__init__(ndim, nhidden, nlayers) 86 | 87 | def forward(self, x): 88 | conv_out = super(ConcaveFunc, self).forward(x) 89 | return -1*conv_out 90 | 91 | 92 | class ConcaveFuncZero(ConcaveFunc): 93 | def __init__(self, ndim, nhidden, nlayers): 94 | super(ConcaveFuncZero, self).__init__(ndim, nhidden, nlayers) 95 | self.conc_forward = super(ConcaveFuncZero, self).forward 96 | self.ndim = ndim 97 | self.register_buffer('zero_input', torch.zeros(1, ndim)) 98 | 99 | def forward(self, x): 100 | return self.conc_forward(x) - self.conc_forward(self.zero_input) 101 | 102 | 103 | class ScalarFunc(nn.Module): 104 | def __init__(self, ndim, nhidden, nlayers): 105 | super(ScalarFunc, self).__init__() 106 | if ndim == 0: 107 | self.weight = nn.parameter.Parameter(torch.Tensor(1, 1)) 108 | self._res_weight() 109 | self.mlp = lambda x: torch.ones( 110 | x.shape[0], device=x.device)*self.weight 111 | else: 112 | self.linears = nn.ModuleList([nn.Linear(ndim, nhidden)]) 113 | hidden_layers = [nn.Linear(nhidden, nhidden) 114 | for j in range(nlayers - 1)] 115 | self.linears.extend(hidden_layers) 116 | self.activation = nn.Softplus() 117 | self.out = nn.Linear(nhidden, 1) 118 | self.mlp = self._mlp_forward 119 | 120 | def _mlp_forward(self, x): 121 | for _, l in enumerate(self.linears): 122 | x = self.activation(l(x)) 123 | return self.out(x) 124 | 125 | def forward(self, x): 126 | return self.mlp(x) 127 | 128 | def _res_weight(self): 129 | nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) 130 | 131 | 132 | class ScalarFuncZero(ScalarFunc): 133 | def __init__(self, ndim, nhidden, nlayers): 134 | super(ScalarFuncZero, self).__init__(ndim, nhidden, nlayers) 135 | self.ndim = ndim 136 | self.scalar = super(ScalarFuncZero, self).forward 137 | self.register_buffer('zero_input', torch.zeros(1, ndim)) 138 | 139 | def forward(self, x): 140 | return self.scalar(x) - self.scalar(self.zero_input) 141 | 142 | 143 | class ScalarFuncZeroPos(ScalarFunc): 144 | def __init__(self, ndim, nhidden, nlayers): 145 | super(ScalarFuncZeroPos, self).__init__(ndim, nhidden, nlayers) 146 | self.ndim = ndim 147 | self.scalar = super(ScalarFuncZeroPos, self).forward 148 | self.register_buffer('zero_input', torch.zeros(1, ndim)) 149 | self.make_pos = ReHU(1.) 150 | 151 | def sc_zero_pos(self, x): 152 | return self.make_pos(self.scalar(x)-self.scalar(self.zero_input)) 153 | # return torch.pow(self.scalar(x)-self.scalar(self.zero_input), 2) 154 | 155 | def forward(self, x): 156 | return self.sc_zero_pos(x) + 0.01*torch.pow(x, 2).sum(dim=-1).unsqueeze(-1) 157 | 158 | 159 | class ScalarFuncPosUnbnd(ScalarFunc): 160 | def __init__(self, ndim, nhidden, nlayers): 161 | super(ScalarFuncPosUnbnd, self).__init__(ndim, nhidden, nlayers) 162 | self.ndim = ndim 163 | self.scalar = super(ScalarFuncPosUnbnd, self).forward 164 | self.register_buffer('zero_input', torch.zeros(1, ndim)) 165 | self.make_pos = nn.Softplus() 166 | 167 | def sc_zero_pos(self, x): 168 | return self.make_pos(self.scalar(x)) 169 | 170 | def forward(self, x): 171 | return self.sc_zero_pos(x) + 0.01*torch.pow(x, 2).sum(dim=-1).unsqueeze(-1) - self.sc_zero_pos(self.zero_input) 172 | 173 | 174 | class ZeroDivMat(nn.Module): 175 | """ Class for constructing a J matrix for generalized Hamiltonian 176 | TODO: write in terms of masked NN rather than a collection of NNs 177 | """ 178 | 179 | def __init__(self, ndim, nhidden, nlayers): 180 | super(ZeroDivMat, self).__init__() 181 | self.ndim = ndim 182 | self.J = nn.ModuleDict() 183 | self.idx_list = [] 184 | self.rel_terms = [] 185 | for j in range(ndim): 186 | for i in range(ndim): 187 | if i >= j: 188 | continue 189 | else: 190 | self.rel_terms.append([ind for ind in range( 191 | self.ndim) if ind != i and ind != j]) 192 | self.idx_list.append((i, j)) 193 | self.J.update( 194 | [['{},{}'.format(i, j), ScalarFunc(ndim-2, nhidden, nlayers)]]) 195 | 196 | def forward(self, x): 197 | J_ret = torch.zeros(x.shape[0], self.ndim, self.ndim, device=x.device) 198 | for ind, (i, j) in enumerate(self.idx_list): 199 | J_ret[:, i, j] = self.J['{},{}'.format(i, j)]( 200 | x[:, self.rel_terms[ind]]).view(-1,) 201 | return J_ret - J_ret.transpose(2, 1) 202 | 203 | 204 | class GHNN(nn.Module): 205 | """ Main generalized Hamiltonian neural nets class 206 | 207 | INPUTS 208 | ndim < int > : number of input dimensions 209 | nhidden < int > : number of hidden units 210 | nlayers < int > : number of hidden layers 211 | prior < dict > : { 212 | 'H': choices = ['H0', 'H1', None], default = 'H0', 213 | 'dHdt': choices = ['decreasing', 'constant', None], default = None 214 | } 215 | Desciption of prior information dictionary: 216 | H (strong priors on the form of the Hamiltonian) 217 | 'H0': H(x) -> infty as x -> infty, H(x) + H(0) >= 0, H(0) = 0. 218 | - use if energy is bounded locally (ie. damped duffing oscilator) 219 | - makes sense in most cases 220 | 'H1': H(x) -> infty as x -> infty, H(0) = 0, H(x) > 0 forall x != 0 221 | - use if globally stable at x=0 222 | dHdt (strong priors on the energy flux rate) 223 | 'decreasing' : dHdt(x) < 0 along trajectories following dxdt = f(x) 224 | 'constant' : dHdt(x) = 0 along trajectories following dxdt = f(x) 225 | 226 | Regularization shemes: 227 | The forward function takes in an optional argument reg_schemes. This tuple 228 | should contain the names of regularization schemes you would like to use in training. 229 | 230 | The choices for regularization schemes are: 231 | - curl (use when performing curl regularization) 232 | - dhdt (use when imposing a soft energy flux rate prior) 233 | 234 | NOTE: for known gen. Hamiltonian use GHNNwHPrior class 235 | """ 236 | 237 | def __init__(self, ndim, nhidden, nlayers, prior={'H': 'H0'}): 238 | super(GHNN, self).__init__() 239 | self.H_prior = prior.get('H', None) 240 | self.dHdt_prior = prior.get('dHdt', None) 241 | 242 | if self.H_prior == 'H0': 243 | self.H = ScalarFuncPosUnbnd(ndim, nhidden, nlayers) 244 | elif self.H_prior == 'H1': 245 | self.H = ScalarFuncZeroPos(ndim, nhidden, nlayers) 246 | else: 247 | self.H = ScalarFuncZero(ndim, nhidden, nlayers) 248 | 249 | self.J = ZeroDivMat( 250 | ndim, max(1, int(nhidden/(ndim*(ndim-1)/2))), nlayers) 251 | self.ndim = ndim 252 | 253 | self.hh_strict = False 254 | self.conservative = False 255 | if self.dHdt_prior == 'decreasing': 256 | self.v = ConcaveFuncZero(ndim, nhidden, nlayers) 257 | elif self.dHdt_prior == 'constant': 258 | self.conservative = True 259 | else: 260 | self.hh_strict = True 261 | self.fd = ScalarFuncZero(ndim, nhidden, nlayers) 262 | 263 | tri_l_ind = torch.ones(ndim, ndim).tril(-1) == 1 264 | self.register_buffer('tri_l_ind', tri_l_ind) 265 | 266 | def forward(self, t, x, reg_schemes=()): 267 | in_shape = x.shape 268 | x = x.reshape(-1, self.ndim).clone() # note: clone likely unecessary 269 | with torch.enable_grad(): 270 | if not x.requires_grad: 271 | x.requires_grad = True 272 | grad_H, J_grad_H = self.conservative_forward(x) 273 | 274 | if self.conservative: 275 | grad_v, R_grad_H = torch.zeros( 276 | grad_H.shape), torch.zeros(J_grad_H.shape) 277 | else: 278 | grad_v, R_grad_H = self.nonconservative_forward(x, grad_H) 279 | dxdt = J_grad_H + R_grad_H 280 | dxdt = dxdt.reshape(in_shape) 281 | if not reg_schemes: 282 | out = dxdt 283 | else: 284 | reg_vals = [] 285 | for reg in reg_schemes: 286 | if reg == 'dhdt': 287 | reg_vals.append(torch.einsum( 288 | 'ij,ij->i', grad_H, R_grad_H).reshape(in_shape[:-1])) 289 | elif reg == 'curl': 290 | reg_vals.append(curl(R_grad_H, x)) 291 | # reg_vals.append(self._curl_R_grad_H(x, grad_H, grad_v)) 292 | else: 293 | print('Reg. scheme: {} not recognized'.format(reg)) 294 | out = (dxdt, reg_vals) 295 | return out 296 | 297 | def forward_odeint(self, t, x): 298 | """ note: only kept around for compatibility with ghnn v0.0.2 please use forward() 299 | forward method for ode integration tools 300 | """ 301 | if not x.requires_grad: 302 | x.requires_grad = True 303 | out = self.forward(t, x) 304 | return out.detach() 305 | 306 | def conservative_forward(self, x): 307 | grad_H = torch.autograd.grad(self.H(x).sum(), x, create_graph=True)[0] 308 | J_grad_H = torch.einsum('ijk,ik->ij', self.J(x), grad_H) 309 | return grad_H, J_grad_H 310 | 311 | def nonconservative_forward(self, x, grad_H): 312 | if self.hh_strict: 313 | div_f = torch.autograd.grad( 314 | self.fd(x).sum(), x, create_graph=True)[0] 315 | R_grad_H = div_f 316 | grad_v = None 317 | elif self.conservative: 318 | grad_v = torch.zeros(grad_H.shape, requires_grad=True)*x 319 | R_grad_H = torch.zeros(grad_H.shape, requires_grad=True)*x 320 | else: 321 | grad_v = torch.autograd.grad( 322 | self.v(x).sum(), x, create_graph=True)[0] 323 | R_grad_H = torch.autograd.grad( 324 | grad_v, x, create_graph=True, grad_outputs=grad_H)[0] 325 | return grad_v, R_grad_H 326 | 327 | def _curl_R_grad_H(self, x, grad_H, grad_v): 328 | """ Note: needs serious optimization 329 | """ 330 | if self.hh_strict: 331 | print('Warning, R*gradH is is stable by construction.') 332 | return torch.zeros(x.shape, device=x.device) 333 | else: 334 | hess_H = self.hessian(grad_H, x) 335 | hess_v = self.hessian(grad_v, x) 336 | VtrH = torch.einsum('ijk,ikl->ijl', hess_H, hess_v) 337 | curl_mat = VtrH - VtrH.transpose(1, 2) 338 | return curl_mat[:, self.tri_l_ind] 339 | 340 | def hessian(self, grad_f, x): 341 | hess = [] 342 | for j in range(x.shape[1]): 343 | hess.append(torch.autograd.grad( 344 | grad_f[:, j].sum(), x, create_graph=True)[0]) 345 | return torch.stack(hess, dim=2) 346 | 347 | def _set_requires_grad_true(self, x): 348 | if not x.requires_grad: 349 | x.requires_grad = True 350 | 351 | def _set_requires_grad_false(self, x): 352 | if x.requires_grad: 353 | x.requires_grad = False 354 | 355 | 356 | class GHNNwHPrior(nn.Module): 357 | """ 358 | INPUTS 359 | ndim < int > : number of input dimensions 360 | nhidden < int > : number of hidden units 361 | nlayers < int > : number of hidden layers 362 | H < function (tensor) -> (tensor) > : known gen. Hamiltonian 363 | 364 | Example: 365 | model = GHNNwHPrior(2, 200, 3, H) 366 | (training model) 367 | x = torch.randn(1,2) 368 | # get value of derivative at x 369 | dxdt = model(0., x) 370 | # get J at x 371 | J = model.J() 372 | # get R at x 373 | R = model.R() 374 | """ 375 | 376 | def __init__(self, n_dim, n_hidden, n_layers, H): 377 | super(GHNNwHPrior, self).__init__() 378 | self.n_dim = n_dim 379 | self.n_hidden = n_hidden 380 | self.n_layers = n_layers 381 | self.H = H 382 | # setting up linear layers 383 | layers = [nn.Linear(n_dim, n_hidden), ] 384 | for j in range(n_layers-1): 385 | layers.append(nn.Softplus()) 386 | layers.append(nn.Linear(n_hidden, n_hidden)) 387 | layers.append(nn.Softplus()) 388 | num_out = int(n_dim**2) 389 | layers.append(nn.Linear(n_hidden, num_out)) 390 | self.mlp = nn.Sequential(*layers) 391 | # setting up W matrix 392 | # gives full array indices lol 393 | self.ind = torch.tril_indices(n_dim, n_dim, offset=n_dim) 394 | self.register_buffer('W', torch.zeros(n_dim, n_dim)) 395 | self.W_tmp = None 396 | 397 | def forward(self, t, input): 398 | in_shape = input.shape 399 | x = input.reshape(-1, in_shape[-1]) 400 | W = self.W.repeat(x.shape[0], 1, 1) 401 | W[:, self.ind[0], self.ind[1]] = self.mlp(x) 402 | self.W_tmp = W.detach().clone() 403 | with torch.enable_grad(): 404 | if not x.requires_grad: 405 | x.requires_grad = True 406 | grad_H = torch.autograd.grad( 407 | self.H(x).sum(), x, create_graph=True)[0] 408 | out = (W @ grad_H.unsqueeze(-1)).squeeze(-1) 409 | return out.reshape(in_shape) 410 | 411 | def J(self): 412 | return 0.5*(self.W_tmp - self.W_tmp.transpose(2, 1)) 413 | 414 | def R(self): 415 | return 0.5*(self.W_tmp + self.W_tmp.transpose(2, 1)) 416 | 417 | 418 | class HNN(nn.Module): 419 | "Assumes ndim = 2n where n is the dimension of q" 420 | 421 | def __init__(self, ndim, nhidden, nlayers): 422 | super(HNN, self).__init__() 423 | self.H = ScalarFuncZero(ndim, nhidden, nlayers) 424 | self.ndim = ndim 425 | n = int(ndim/2) 426 | top = torch.cat([torch.zeros(n, n), torch.eye(n)], dim=1) 427 | bot = torch.cat([-torch.eye(n), torch.zeros(n, n)], dim=1) 428 | self.register_buffer('J', torch.cat([top, bot], dim=0)) 429 | 430 | def forward(self, t, x): 431 | # assumes x is the form [[q1, q2, ..., qn, p1, p2, ..., pn], ...] 432 | in_shape = x.shape 433 | x = x.view(-1, self.ndim).clone() 434 | with torch.enable_grad(): 435 | if not x.requires_grad: 436 | x.requires_grad = True 437 | H = self.H(x) 438 | gradH = torch.autograd.grad(H.sum(), x, create_graph=True)[0] 439 | dxdt = gradH @ self.J.T 440 | 441 | return dxdt.reshape(in_shape) 442 | 443 | 444 | class ODEFCN(nn.Module): 445 | def __init__(self, ndim, nhidden, nlayers): 446 | super(ODEFCN, self).__init__() 447 | self.linears = nn.ModuleList([nn.Linear(ndim, nhidden)]) 448 | hidden_layers = [nn.Linear(nhidden, nhidden) 449 | for j in range(nlayers - 1)] 450 | self.linears.extend(hidden_layers) 451 | self.activation = nn.Softplus() 452 | self.out = nn.Linear(nhidden, ndim) 453 | self.mlp = self._mlp_forward 454 | 455 | def _mlp_forward(self, x): 456 | for _, l in enumerate(self.linears): 457 | x = self.activation(l(x)) 458 | return self.out(x) 459 | 460 | def forward(self, t, x): 461 | return self.mlp(x) 462 | 463 | def H(self, x): 464 | return 0.*x.sum(dim=-1) 465 | 466 | 467 | class ReHU(nn.Module): 468 | """ Rectified Huber unit 469 | from: https://github.com/locuslab/stable_dynamics/blob/master/models/stabledynamics.py 470 | """ 471 | 472 | def __init__(self, d): 473 | super(ReHU, self).__init__() 474 | self.a = 1/d 475 | self.b = -d/2 476 | 477 | def forward(self, x): 478 | return torch.max(torch.clamp(torch.sign(x)*self.a/2*x**2, min=0, max=-self.b), x+self.b) 479 | -------------------------------------------------------------------------------- /weakformghnn/_src/_vector_calc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | __all__ = ['divergence', 'curl'] 4 | 5 | 6 | def divergence(f, x): 7 | """ Computes the divergence for a function f at points x 8 | INPUTS: 9 | f < tensor > : vector function values (mxn) 10 | x < tensor > : (mxn) input vector 11 | OUTPUTS 12 | divergence < tensor > : (m,) 13 | """ 14 | div = [] 15 | for j in range(x.shape[1]): 16 | grad_f = torch.autograd.grad( 17 | f[:, j].sum(), x, create_graph=True, allow_unused=True)[0] 18 | div.append(grad_f[:, j]) 19 | return torch.stack(div).sum(dim=0) 20 | 21 | 22 | def curl(f, x): 23 | """ Computes the curl for a function f at points x 24 | INPUTS: 25 | f < tensor > : vector function values (mxn) 26 | x < tensor > : mxn input vector 27 | 28 | OUTPUTS: 29 | curl < tensor > : mx(n*(n-1)/2) 30 | """ 31 | N = x.shape[1] 32 | grad_f_array = [] 33 | for i in range(N): 34 | grad_f = torch.autograd.grad( 35 | f[:, i].sum(), x, allow_unused=True, create_graph=True)[0] 36 | grad_f_array.append(grad_f) 37 | cu = [] 38 | for i in range(N): 39 | for j in range(N): 40 | if i >= j: 41 | continue 42 | else: 43 | c_ij = grad_f_array[j][:, i] - grad_f_array[i][:, j] 44 | cu.append(c_ij) 45 | return torch.stack(cu, dim=1) 46 | -------------------------------------------------------------------------------- /weakformghnn/_src/_weak_form.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | __all__ = ['gauss_rbf', 4 | 'poly_bf', 5 | 'weak_form_loss'] 6 | 7 | 8 | def gauss_rbf(t, c, eps, minimum=0.): 9 | """ Returns the gaussian rbf for a collection of points t 10 | INPUTS: 11 | t < torch.Tensor (m,) > : inputs to gaussian rbfs 12 | c < torch.Tensor (M,) > : centers of rbfs (where M is the number of GRBFS) 13 | eps < float > : shape paramter of gaussian radial basis functions 14 | minimum < float > : minimum grbf function value 15 | RETURNS: 16 | gauss_rbfs < torch.Tensor(m,M) > : grbfs evaluated at t's 17 | gauss_rbf_derivs < torch.Tensor(m,M) > grbf derivs evaluted at ts 18 | """ 19 | diff = t.view(-1, 1) - c.view(1, -1) # m x M 20 | grbf = torch.exp(-eps**2 * torch.pow(diff, 2)) + minimum 21 | grbf_deriv = torch.mul(-2*eps**2*diff, grbf) 22 | return grbf, grbf_deriv 23 | 24 | 25 | def poly_bf(t, c, deg): 26 | """ Returns polynomial basis funcs and their derivatives centered at c evaluated at points t 27 | INPUTS: 28 | t < torch.Tensor (m,) > : inputs to gaussian rbfs 29 | c < torch.Tensor (M,) > : centers of rbfs (where M is the number of GRBFS) 30 | deg < int > : polynomial bf degree (note should be < ~ 10) 31 | RETURNS: 32 | poly_bfs < torch.Tensor(m,M) > : poly basis functions evaluated at t's (M = c(deg + 1)) 33 | poly_bf_derivs < torch.Tensor(m,M) > : poly basis function derivatives evaluated at times 34 | """ 35 | diff = t.view(-1, 1) - c.view(1, -1) 36 | poly = torch.cat([torch.pow(diff, j) for j in range(deg+1)], dim=1) 37 | poly_deriv = torch.cat( 38 | [torch.ones(diff.shape)*j for j in range(deg+1)], dim=1) 39 | poly_deriv[:, 2*diff.shape[1]:] = poly_deriv[:, 2*diff.shape[1]:] * \ 40 | poly[:, diff.shape[1]:-diff.shape[1]] 41 | return poly, poly_deriv 42 | 43 | 44 | def weak_form_loss(dx_est, x, t, psi, psi_dot): 45 | """ Returns the weak-form ode model loss 46 | INPUTS: 47 | dx_est < torch.Tensor, (Bs, m, ndim) > : estimate for derivative 48 | x < torch.Tensor, (Bs, m, ndim) > : state training point 49 | t < torch.Tensor, (m) > integration times 50 | psi < torch.Tensor, (m,M) > : test functions evaluated at measurment times 51 | psi_dot < torch.Tensor, (m,M) > : test function derivatives evaluated at measurement times 52 | OUTPUTS: 53 | weak_form_loss < torch.float > : weak-form squared loss 54 | """ 55 | # boundary term 56 | RH_bound = torch.einsum('in,m->inm', x[:, -1, :], psi[-1]) 57 | LH_bound = torch.einsum('in,m->inm', x[:, 0, :], psi[0]) 58 | B = RH_bound - LH_bound 59 | 60 | x_psi_dot = torch.einsum('imn,ml->imnl', x, psi_dot) 61 | f_psi = torch.einsum('imn,ml->imnl', dx_est, psi) 62 | 63 | L = torch.trapz(f_psi + x_psi_dot, t, dim=1) - B 64 | return L.pow(2).sum(-1).mean() 65 | --------------------------------------------------------------------------------