├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── mbirl ├── README.md ├── __init__.py ├── env │ ├── __init__.py │ └── kuka_iiwa │ │ ├── meshes │ │ ├── iiwa7 │ │ │ ├── collision │ │ │ │ ├── link_0.stl │ │ │ │ ├── link_1.stl │ │ │ │ ├── link_2.stl │ │ │ │ ├── link_3.stl │ │ │ │ ├── link_4.stl │ │ │ │ ├── link_5.stl │ │ │ │ ├── link_6.stl │ │ │ │ └── link_7.stl │ │ │ └── visual │ │ │ │ ├── link_0.stl │ │ │ │ ├── link_1.stl │ │ │ │ ├── link_2.stl │ │ │ │ ├── link_3.stl │ │ │ │ ├── link_4.stl │ │ │ │ ├── link_5.stl │ │ │ │ ├── link_6.stl │ │ │ │ └── link_7.stl │ │ └── robotiq-ft300 │ │ │ ├── collision │ │ │ ├── robotiq_fts150.stl │ │ │ └── robotiq_fts300.dae │ │ │ └── visual │ │ │ ├── robotiq_fts150.stl │ │ │ └── robotiq_fts300.dae │ │ └── urdf │ │ └── iiwa7_ft_with_obj_keypts.urdf ├── experiments │ ├── __init__.py │ ├── plot_mbirl_training_and_eval.ipynb │ └── run_model_based_irl.py ├── generate_expert_demo.py ├── keypoint_mpc.py └── learnable_costs.py ├── ml3 ├── README.md ├── __init__.py ├── envs │ ├── __init__.py │ ├── bullet_sim.py │ ├── mountain_car.py │ ├── mujoco_robots │ │ ├── ground_plane.xml │ │ └── reacher.xml │ └── reacher_sim.py ├── experiments │ ├── Loss shaping visualization.ipynb │ ├── __init__.py │ ├── ml3_sine_regression_exp_viz.ipynb │ ├── run_mbrl_reacher_exp.py │ ├── run_mountain_car_exp.py │ ├── run_shaped_sine_exp.py │ └── run_sine_regression_exp.py ├── learnable_losses.py ├── mbrl_utils.py ├── ml3_test.py ├── ml3_train.py ├── optimizee.py ├── shaped_sine_utils.py ├── sine_regression_task.py └── sine_task_sampler.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.idea 2 | .DS_Store 3 | __pycache__ 4 | *.egg-info 5 | **/data/* 6 | **/model_data/* 7 | **/traj_data/* 8 | **/plots/* 9 | **/.ipynb_checkpoints/* 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. 4 | Please read the [full text](https://code.fb.com/codeofconduct/) 5 | so that you can understand what actions will and will not be tolerated. 6 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to learning-to-learn 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `master`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## License 30 | By contributing to learning-to-learn, you agree that your contributions will be licensed 31 | under the LICENSE file in the root directory of this source tree. 32 | 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Facebook, Inc. and its affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LearningToLearn 2 | This repository contains code for 3 | * ML3: Meta-Learning via Learned Losses, presented at ICPR 2020, *won best student award* ([pdf](https://arxiv.org/pdf/1906.05374.pdf)) 4 | * MBIRL: Model-Based Inverse Reinforcement Learning from Visual Demonstrations, presented at CoRL 2020 ([pdf](https://arxiv.org/pdf/2010.09034.pdf)) 5 | 6 | ## Setup 7 | In the LearningToLearn folder, run: 8 | 9 | ``` 10 | conda create -n l2l python=3.7 11 | conda activate l2l 12 | python setup.py develop 13 | ``` 14 | 15 | ## ML3 paper experiments and citation 16 | To reproduce results of the ML3 paper follow the README instructions in the `ml3` folder 17 | 18 | #### Citation 19 | ``` 20 | @inproceedings{ml3, 21 | author = {Sarah Bechtle and Artem Molchanov and Yevgen Chebotar and Edward Grefenstette and Ludovic Righetti and Gaurav Sukhatme and Franziska Meier}, 22 | title = {Meta Learning via Learned Loss}, 23 | booktitle = {International Conference on Pattern Recognition, {ICPR}, Italy, January 10-15, 2021}, 24 | year = {2021} } 25 | ``` 26 | 27 | ## MBIRL paper experiments and citation 28 | To test our MBIRL algorithm follow the README instructions in the `mbirl` folder 29 | 30 | #### Citation 31 | ``` 32 | @InProceedings{mbirl, 33 | author = {Neha Das, Sarah Bechtle, Todor Davchev, Dinesh Jayaraman, Akshara Rai and Franziska Meier}, 34 | booktitle = {Conference on Robot Learning (CoRL)}, 35 | title = {Model Based Inverse Reinforcement Learning from Visual Demonstration}, 36 | year = {2020}, 37 | video = {https://www.youtube.com/watch?v=sRrNhtLk12M&t=52s}, 38 | } 39 | ``` 40 | 41 | ## License 42 | 43 | `LearningToLearn` is released under the MIT license. See [LICENSE](LICENSE) for additional details about it. 44 | See also our [Terms of Use](https://opensource.facebook.com/legal/terms) and [Privacy Policy](https://opensource.facebook.com/legal/privacy). 45 | -------------------------------------------------------------------------------- /mbirl/README.md: -------------------------------------------------------------------------------- 1 | ## MBIRL - Model Based Inverse Reinforcement Learning 2 | 3 | ### Simulation with ground truth keypoint predictions 4 | 5 | #### Generate expert demonstrations 6 | 1. ```python mbirl/generate_expert_demo.py``` 7 | 2. Check the data and visualizations of the demonstration in 'mbirl/traj_data' 8 | 9 | #### Run Our Method 10 | 1. ```python mbirl/experiments/run_model_based_irl.py``` 11 | 2. Check the trajectories predicted during training in model_data/placing/ 12 | 13 | #### Plot the losses, evaluate our method 14 | 1. ```jupyter notebook``` 15 | 2. Access the notebook in the browser in 'mbirl/experiments/plot_mbirl_training_and_eval.ipynb' 16 | 17 | ### Simulation with learned keypoint representation and dynamics 18 | COMING SOON 19 | -------------------------------------------------------------------------------- /mbirl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/__init__.py -------------------------------------------------------------------------------- /mbirl/env/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/__init__.py -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_0.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_0.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_1.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_1.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_2.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_3.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_3.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_4.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_4.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_5.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_5.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_6.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_6.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_7.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/collision/link_7.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_0.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_0.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_1.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_1.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_2.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_2.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_3.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_3.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_4.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_4.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_5.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_5.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_6.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_6.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_7.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/iiwa7/visual/link_7.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/robotiq-ft300/collision/robotiq_fts150.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/robotiq-ft300/collision/robotiq_fts150.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/meshes/robotiq-ft300/visual/robotiq_fts150.stl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/env/kuka_iiwa/meshes/robotiq-ft300/visual/robotiq_fts150.stl -------------------------------------------------------------------------------- /mbirl/env/kuka_iiwa/urdf/iiwa7_ft_with_obj_keypts.urdf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | -------------------------------------------------------------------------------- /mbirl/experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/mbirl/experiments/__init__.py -------------------------------------------------------------------------------- /mbirl/experiments/plot_mbirl_training_and_eval.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": { 7 | "scrolled": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import os, sys\n", 12 | "import torch\n", 13 | "import numpy as np\n", 14 | "import matplotlib.pyplot as plt\n", 15 | "from os.path import dirname, abspath\n", 16 | "from differentiable_robot_model import DifferentiableRobotModel\n", 17 | "\n", 18 | "from mbirl.keypoint_mpc import KeypointMPCWrapper\n", 19 | "from mbirl.learnable_costs import *\n", 20 | "import mbirl\n", 21 | "import warnings\n", 22 | "warnings.filterwarnings('ignore')\n", 23 | "\n", 24 | "EXP_FOLDER = os.path.join(mbirl.__path__[0], \"experiments\")\n", 25 | "traj_data_dir = os.path.join(EXP_FOLDER, 'traj_data')\n", 26 | "model_data_dir = os.path.join(EXP_FOLDER, 'model_data')\n" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 4, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "experiment_type = 'placing'" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 5, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "# Get data saved during training\n", 45 | "\n", 46 | "if not os.path.exists(\n", 47 | " f\"{model_data_dir}/{experiment_type}_TimeDep\") or not os.path.exists(\n", 48 | " f\"{model_data_dir}/{experiment_type}_Weighted\") or not os.path.exists(f\"{model_data_dir}/{experiment_type}_RBF\"):\n", 49 | " assert False, \"Path does not exist\"\n", 50 | "\n", 51 | "timedep = torch.load(f\"{model_data_dir}/{experiment_type}_TimeDep\")\n", 52 | "weighted = torch.load(f\"{model_data_dir}/{experiment_type}_Weighted\")\n", 53 | "rbf = torch.load(f\"{model_data_dir}/{experiment_type}_RBF\")" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 6, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABJBUlEQVR4nO3deXxcZdnw8d81SzLZ0yVt05VSulC6pCu0RWQvIJZFZVOpgA8gKoI7+MqioviAIiiKFaqgPohQNqFCBdEChZYWQukGbaFLQrqlzb7PXO8f58xkkswk0zaTNMn15XM+c+Y+2z2Zkiv3LqqKMcYY0x5Pd2fAGGPMkc+ChTHGmA5ZsDDGGNMhCxbGGGM6ZMHCGGNMhyxYGGOM6VDSgoWIjBCRV0Rkg4isF5FvuOn9ReRfIrLZfe3npouI3CciW0RkrYhMj7rXQvf8zSKyMFl5NsYYE5ska5yFiOQD+ar6tohkAWuA84EvAftV9U4R+T7QT1W/JyLnAF8HzgGOB+5V1eNFpD+wGpgJqHufGap6ICkZN8YY00bSShaqWqKqb7v7lcBGYBhwHvCwe9rDOAEEN/0RdbwJ5LoBZz7wL1Xd7waIfwFnJSvfxhhj2vJ1xUNE5ChgGrASGKyqJe6hXcBgd38YsDPqsiI3LV56rOdcDVwNkJGRMWPChAkHn9nq7dBYBrlTW6aXrQV/NmQcdfD3NMaYHmDNmjX7VDUv1rGkBwsRyQSWADeoaoWIRI6pqopIp9WDqeoiYBHAzJkzdfXq1Qd/k5X/Ax8vhQtaXfv0KBhyKpzwx07IqTHGHHlEZHu8Y0ntDSUifpxA8VdVfdJN3u1WL4XbNfa46cXAiKjLh7tp8dKTQ0MgMX4s4oFQMGmPNcaYI1kye0MJ8BCwUVV/GXXoWSDco2kh8ExU+uVur6gTgHK3uupF4EwR6ef2nDrTTUuSEDF/LOIDbUreY40x5giWzGqoecAXgfdEpNBNuxm4E/i7iFwFbAcuco8txekJtQWoAa4AUNX9IvJj4C33vB+p6v6k5TpeycLjh1Bj0h5rjDFHsqQFC1V9DZA4h0+Lcb4CX41zr8XA4s7LXTvaCxZqwcJ0nsbGRoqKiqirq+vurJg+JhAIMHz4cPx+f8LXdElvqB5F41RDWcnCdLKioiKysrI46qijiO74YUwyqSqlpaUUFRUxevTohK+z6T7aiNfAbcHCdK66ujoGDBhggcJ0KRFhwIABB12itWDRmrVZmC5kgcJ0h0P5d2fBojUNEbOpxdosjDF9mAWLNtRKFqZPuPHGG/nVr34VeT9//ny+/OUvR95/61vf4pe//GWMKx233HILL730UrvPuO2227j77rvbpJeVlfHb3/72oPMc734AixYtYsKECUyYMIHZs2fz2muvHfT9TXwWLFqLOyjPgoXpXebNm8eKFSsACIVC7Nu3j/Xr10eOr1ixgrlz58a9/kc/+hGnn376IT37UINFPM899xy///3vee2119i0aRMPPPAAl112Gbt27Ur4Hk1NNo6qPRYs2rDeUKZvmDt3Lm+88QYA69evZ9KkSWRlZXHgwAHq6+vZuHEj06dPZ82aNXzyk59kxowZzJ8/n5ISZ2q3L33pSzzxxBMALF26lAkTJjBjxgyuv/56zj333MhzNmzYwMknn8zRRx/NfffdB8D3v/99tm7dSkFBAd/5zncAuOuuu5g1axZTpkzh1ltvjVx/xx13MG7cOE488UTef//9mJ/l5z//OXfddRcDBw4EYPr06SxcuJD7778fgKOOOop9+/YBsHr1ak4++WTAKal88YtfZN68eXzxi19k/fr1zJ49m4KCAqZMmcLmzZs75WfdG1jX2dZsnIXpDmtugAOFnXvPfgUw41dxDw8dOhSfz8eOHTtYsWIFc+bMobi4mDfeeIOcnBwmT56MiPD1r3+dZ555hry8PB577DF+8IMfsHhx87Cnuro6rrnmGpYvX87o0aO59NJLWzxn06ZNvPLKK1RWVjJ+/Hi+8pWvcOedd7Ju3ToKC53PvGzZMjZv3syqVatQVRYsWMDy5cvJyMjgb3/7G4WFhTQ1NTF9+nRmzJjR5rOsX7++TfrMmTN5+OGH25zb2oYNG3jttddIS0vj61//Ot/4xjf4/Oc/T0NDA8GgTfETZsGiNesNZfqQuXPnsmLFClasWME3v/lNiouLWbFiBTk5OcybN4/333+fdevWccYZZwAQDAbJz89vcY9NmzZx9NFHR/rsX3rppSxatChy/FOf+hSpqamkpqYyaNAgdu/e3SYfy5YtY9myZUybNg2AqqoqNm/eTGVlJRdccAHp6ekALFiwoNN/BgsWLCAtLQ2AOXPmcMcdd1BUVMSFF17I2LFjO/15PZUFi9ZsUJ7pDu2UAJIp3G7x3nvvMWnSJEaMGMEvfvELsrOzueKKK1BVjjvuuEh11aFITU2N7Hu93phtA6rKTTfdxDXXXNMiPboBvj0TJ05kzZo1nHrqqZG0NWvWcNxxxwHg8/kIhUIAbcYXZGRkRPYvu+wyjj/+eJ5//nnOOeccfv/737e4Z19mbRZtWAO36Tvmzp3Lc889R//+/fF6vfTv35+ysjLeeOMN5s6dy/jx49m7d28kWDQ2NrZoBAcYP348H374Idu2bQPgscce6/C5WVlZVFZWRt7Pnz+fxYsXU1VVBUBxcTF79uzhpJNO4umnn6a2tpbKykr+8Y9/xLzfd7/7Xb73ve9RWloKQGFhIX/605+47rrrAKfNYs2aNQAsWbIkbr4+/PBDjj76aK6//nrOO+881q5d2+Fn6SusZNGatVmYPmTy5Mns27ePyy67rEVaVVVVpLH4iSee4Prrr6e8vJympiZuuOGGyF/sAGlpafz2t7/lrLPOIiMjg1mzZnX43AEDBjBv3jwmTZrE2WefzV133cXGjRuZM2cOAJmZmfzlL39h+vTpXHzxxUydOpVBgwbFvfeCBQsoLi5m7ty5iAhZWVn85S9/iVSZ3XrrrVx11VX88Ic/jDRux/L3v/+dP//5z/j9foYMGcLNN9/c4WfpK5K2Bnd3O+TFj14+HYK1cObrLdPX3AAf/hE+V94p+TNm48aNHHvssd2djU5RVVVFZmYmqspXv/pVxo4dy4033tjd2TLtiPXvT0TWqOrMWOdbNVQbNijPmIP1hz/8gYKCAo477jjKy8vbtD2Yns+qoVqzQXnGHLQbb7zRShK9nJUs2minN5Q2QS+ttjPGmPZYsGgtbsnCLYSpDdIxxvQ9Fixaa683FFhVlDGmT0pasBCRxSKyR0TWRaU9JiKF7rYtvDa3iBwlIrVRxx6IumaGiLwnIltE5D5J9gIA7Q3KA+s+a4zpk5JZsvgTcFZ0gqperKoFqloALAGejDq8NXxMVa+NSv8d8D/AWHdrcc/OZyUL0zeUlpZSUFBAQUEBQ4YMYdiwYRQUFJCZmRkZzNaZbrvttsgzxo4dy4UXXsiGDRs65d7Tpk2LzDPV1NQUGacRNmPGDN5+++24159zzjmUlZW1+4yTTz6ZWN3xCwsLWbp06UHnOd79GhoauOGGGzjmmGMYO3Ys5513HkVFRQd9/86WtGChqsuB/bGOuaWDi4BH27uHiOQD2ar6pjoDQh4Bzu/krLZk1VCmjxgwYACFhYUUFhZy7bXXcuONN1JYWEhVVVWnTh8eLfyMzZs3c/HFF3Pqqaeyd+/ew75v9HTr7777LuPGjYu8r66uZuvWrUydOjXu9UuXLiU3N/eQnn2owSKem2++mcrKSt5//302b97M+eefz4UXXsjBjIlLxnTr3dVm8Qlgt6pGz/87WkTeEZH/isgn3LRhQHRILXLTkifeSnliwcL0Df/5z38iU4zfdtttLFy4kE984hOMGjWKJ598ku9+97tMnjyZs846i8ZG5/+HeNOYt+fiiy/mzDPP5P/+7//avcfJJ5/MN77xDQoKCpg0aRKrVq1qc6/whIjgrMNx7bXXRkoaq1atYsaMGXi9Xv7yl79EpiC/5pprIrPKRk9h/uMf/5jx48dz4okncumll7ZYbOnxxx9n9uzZjBs3jldffZWGhgZuueUWHnvsMQoKCnjssceorq7myiuvZPbs2UybNo1nnnkGgNraWi655BKOPfZYLrjgAmpra9t8jpqaGv74xz9yzz334PV6AbjiiitITU3l3//+N9u2bWPSpEmR8++++25uu+22yM/phhtuYObMmdx77708/vjjTJo0ialTp3LSSSd1+H10pLvGWVxKy1JFCTBSVUtFZAbwtIgcF/vS+ETkauBqgJEjRx5i1toZlAfWZmGS4oYXbqBwV2Gn3rNgSAG/OutXh32frVu38sorr7BhwwbmzJnDkiVL+N///V8uuOACnn/+eT71qU91OI15PNOnT2fTpk00Nja2e4+amhoKCwtZvnw5V155JevWrWtxn3nz5vH//t//A5xgceutt/Loo49SWVkZWcRp48aNPPbYY7z++uv4/X6uu+46/vrXv3L55ZdH7vPWW2+xZMkS3n33XRobG9tMid7U1MSqVatYunQpt99+Oy+99BI/+tGPWL16Nb/5zW8Ap2Rw6qmnsnjxYsrKypg9ezann346v//970lPT2fjxo2sXbuW6dOnt/l5bNmyhZEjR5Kdnd0ifebMmaxfv54xY8a0+/NsaGiIVG1NnjyZF198kWHDhnVYxZaILg8WIuIDLgQi34Cq1gP17v4aEdkKjAOKgeFRlw9302JS1UXAInCm+zikDFo1lDEtnH322fj9fiZPnkwwGOSss5xmw8mTJ7Nt27aEpjGPJ1y10tE9wmtknHTSSVRUVFBWVtai2mjUqFE0NDSwa9cuNm3axPjx45k1axYrV65kxYoVfP3rX+fll19mzZo1kfmlamtrGTRoUIv8vP7665x33nkEAgECgQCf/vSnWxy/8MILAacNJDxxYmvLli3j2WefjZRI6urq2LFjB8uXL+f6668HYMqUKUyZMiWhn9HBuPjiiyP78+bN40tf+hIXXXRRJN+HoztKFqcDm1Q1Ur0kInnAflUNisjROA3ZH6rqfhGpEJETgJXA5cCvk5u9DnpDWbAwSdAZJYBkCU8x7vF48Pv9hDskejwempqaDmsa83feeYeZM2d2eI/WnSBjdYqcO3cujz/+OPn5+YgIJ5xwAq+//jqrVq1izpw5bN68mYULF/Kzn/3soPMZFv5ZxJtqHZwAuGTJEsaPH3/Q9x8zZgw7duygsrKSrKysSPqaNWs499xzW0y1Du1Pt/7AAw+wcuVKnn/+eWbMmMGaNWsYMGDAQecpLJldZx8F3gDGi0iRiFzlHrqEtg3bJwFr3a60TwDXqmq4cfw64EFgC7AV+Gey8gy0P90HWLAwppVEpjGPZcmSJSxbtoxLL720w3uEpz1/7bXXyMnJIScnp8395s6dy69+9avIzLVz5szhkUceYciQIeTk5HDaaafxxBNPsGfPHgD279/P9u3bW9xj3rx5/OMf/6Curo6qqiqee+65Dj9HrOnWf/3rX0dKTe+88w7glIrC7TPr1q2LOf15RkYGCxcu5Jvf/GakPeWRRx6hpqaGU089lcGDB7Nnzx5KS0upr69vN39bt27l+OOP50c/+hF5eXns3Lmzw8/SnqSVLFT10jjpX4qRtgSnK22s81cDk2IdSwqrhjLmoKSkpHQ4jXnYPffcw1/+8heqq6uZNGkS//73v8nLywPanwo9EAgwbdo0Ghsb47aFzJs3jxtvvDESLPLz8wkGg8ydOxdwFkj6yU9+wplnnkkoFMLv93P//fczatSoyD1mzZrFggULmDJlCoMHD2by5MkxA1O0U045hTvvvJOCggJuuukmfvjDH3LDDTcwZcoUQqEQo0eP5rnnnuMrX/kKV1xxBcceeyzHHntszOVhAX72s5/x7W9/m3HjxuHxeJgwYQJPPfUUIoLf7+eWW25h9uzZDBs2jAkTJsTN13e+8x02b96MqnLaaae12xssETZFeWvPTYDcAjjxby3TS5bBK/PhjNcgb16n5NH0bb1pivJkOvnkk7n77ruZOTPmzNmdLjzdek1NDSeddBKLFi2K2Rjd0x3sFOU262xrVrIwpk+7+uqr2bBhA3V1dSxcuLBXBopDYcGiNWuzMOaI8p///KdLnxduVzAt2USCrcUblGclC2NMH2bBog0blGeMMa1ZsGjN2iyMMaYNCxZtxBmUZ20Wxpg+zIJFa1ayMH2I1+uNTND36U9/OjKH0LZt20hLS6OgoICpU6cyd+5c3n//fcBpcM7JyYlMb3766ae3uKeqMnDgQA4cOABASUkJIsJrr70WOScvL4/S0tK4+QqPjWhP9OR/0f7zn/9EJhU8GPHuV15ezuWXX84xxxzDmDFjuPzyyykvLz/o+/d0Fixa6yhYWJuF6UXS0tIoLCxk3bp19O/fn/vvvz9ybMyYMRQWFvLuu++ycOFCfvrTn0aOfeITn4hMb/7SSy+1uGd4qo3waOwVK1Ywbdq0yC/w999/nwEDBrQ79cSh/LIPO9RgEc9VV13F0UcfzZYtW9i6dSujR4/my1/+8kHdIzwauyezYNGGzQ1l+qY5c+ZQXBx7ns6Kigr69euX8L1aTxl+4403tgge8+Y5A1vvuusuZs2axZQpU7j11lsj12dmZgIQCoW47rrrmDBhAmeccQbnnHMOTzzxROS8X//610yfPp3JkyezadMmtm3bxgMPPMA999xDQUEBr776Knv37uUzn/kMs2bNYtasWbz++uuAs/jTmWeeyXHHHceXv/zlmOtFbNmyhTVr1vDDH/4wknbLLbewevVqtm7d2mI6d4Cvfe1r/OlPfwKcksr3vvc9pk+fzuOPP859993HxIkTmTJlCpdccknCP8sjhY2zaM3GWZhuUPNiDcHdnfvXp3ewl/T56QmdGwwGefnll7nqqqsiaVu3bqWgoIDKykpqampYuXJl5Nirr75KQUEBAJ/73Of4wQ9+0OJ+8+bN4/bbbwec9SRuv/127r33XoDIlOHLli1j8+bNrFq1ClVlwYIFLF++vMXaC08++STbtm1jw4YN7Nmzh2OPPZYrr7wycnzgwIG8/fbb/Pa3v+Xuu+/mwQcf5NprryUzM5Nvf/vbAFx22WXceOONnHjiiezYsYP58+ezceNGbr/9dk488URuueUWnn/+eR566KE2P5cNGzZQUFAQWVsCmqvu1q9f32Yq8dYGDBgQWaFv6NChfPTRR6SmpnbKlOFdzYJFa9ZmYfqQ2tpaCgoKKC4u5thjj41MEQ7N1VDgTOR39dVX88ILLwBONVR7k9jNmjWLd955h+rqahobG8nMzIxU5axYsYJvfetbPPjggyxbtoxp06YBzjQbmzdvbhEsXnvtNT73uc/h8XgYMmQIp5xySovnRE8Z/uSTTxLLSy+91GL51oqKCqqqqli+fHnkmk996lMHVXJKVPSU4VOmTOHzn/88559/Pueff36nPyvZLFi01tGgPGuzMEmQaAmgs4XbLGpqapg/fz73339/ZM2FaAsWLOCKK65I+L7p6emMHTuWxYsXR6bLOOGEE1i6dCl79uxh/PjxqCo33XQT11xzzSHnP5Epw0OhEG+++SaBQOCg7z9x4kQKCwsJhUJ4PJ7I/QoLC5k4cSK7du1KeMrw559/nuXLl/OPf/yDO+64g/feew+fr+f8CrY2izY6GJRnJQvTC6Wnp3Pffffxi1/8IuYv3ddee63DVdpaizVl+L333ssJJ5yAiDB//nwWL15MVVUVAMXFxZHpw8PmzZvHkiVLCIVC7N69O6GpP1pPGX7mmWfy6183L4MTLi1FTxn+z3/+M9J7K9oxxxzDtGnT+MlPfhJJ+8lPfsL06dM55phjGDVqFBs2bKC+vp6ysjJefvnlmHkKhULs3LmTU045hZ///OeUl5dHPndPYcGiNY03zsL9C8CChemlpk2bxpQpU3j0UWe5mXCbxdSpU7n55pt58MEHD+p+8+bN48MPP4wEi+nTp1NUVBTpFnvmmWdy2WWXMWfOHCZPnsxnP/vZFr/kAT7zmc8wfPhwJk6cyBe+8AWmT5/e4ZThn/70p3nqqaciDdz33Xcfq1evZsqUKUycOJEHHngAgFtvvZXly5dz3HHH8eSTT8Zdivmhhx7igw8+YMyYMYwZM4YPPvgg0r4xYsQILrroIiZNmsRFF10UqVJrLRgM8oUvfIHJkyczbdo0rr/++hYr/fUENkV5lJCGePWnz9AweDNnfPm7bU941A/HfgcKftr2mDEHyaYoT0x4yvDS0lJmz57N66+/zpAhQ7o7Wz2eTVF+GDziYUxoJm9Xl8U5wW9tFsZ0sXPPPZeysjIaGhr44Q9/aIGim1iwaKVJgtSREfugxw+h2I1oxpjk6Oopyk1s1mbRSsgToqGhIfZBj9/aLEyn6q3VwObIdij/7pIWLERksYjsEZF1UWm3iUixiBS62zlRx24SkS0i8r6IzI9KP8tN2yIi309WfsPUqzQ2xAkIYtVQpvMEAgFKS0stYJgupaqUlpYedFfiZFZD/Qn4DfBIq/R7VPXu6AQRmQhcAhwHDAVeEpFx7uH7gTOAIuAtEXlWVTeQJOIVtF6pbawlzZ/W8qCVLEwnGj58OEVFRezdu7e7s2L6mEAgwPDhww/qmqQFC1VdLiJHJXj6ecDfVLUe+EhEtgCz3WNbVPVDABH5m3tu0oKFx+8hhRQ+rvyYMf3HtD5owcJ0Gr/fz+jRo7s7G8YkpDvaLL4mImvdaqrw+PphwM6oc4rctHjpMYnI1SKyWkRWH+pfaz6/j1RSKaooanvQgoUxpo/q6mDxO2AMUACUAL/ozJur6iJVnamqM/Py8g7pHn6/nxRSKK6MMfumtVkYY/qoLg0WqrpbVYOqGgL+QHNVUzEwIurU4W5avPSkSU1JJYWUOCULn5UsjDF9UpcGCxHJj3p7ARDuKfUscImIpIrIaGAssAp4CxgrIqNFJAWnEfzZZObRm+IlTdIorohTsrBgYYzpg5LWwC0ijwInAwNFpAi4FThZRAoABbYB1wCo6noR+TtOw3UT8FVVDbr3+RrwIuAFFqvq+mTlGZzeUBnejNjVUNZmYYzpo5LZG+rSGMltVxdpPv8O4I4Y6UuBpZ2YtfZ5IU3S4jdwW5uFMaYPshHcrYhXCEjAShbGGBPFgkVrPkgllZLKEoKhVstcWpuFMaaPsmDRingFP36CGmR39e6WB61kYYzpoyxYtOYFrzqLs7fpEWVtFsaYPsqCRSviFbwhJ1i0aeS2koUxpo+yYNGaDyQkCNK2kdvaLIwxfZQFi9acQgUZngwrWRhjjMuCRSviFQBGZo5sW7KwNgtjTB9lwaI1d5jiqMxRsRu4rWRhjOmDEhrBLSJzgaOiz1fV1osa9QrhksWIzBG8su+VVgc7CBZV2wCFTFujwBjTu3QYLETkzzjTihcC4VFqStsV8HoHt81iWPowiiuLUVVEnADSYcnirWuhoQzmv5n0bBpjTFdKpGQxE5iofWSh4HDJYljGMGoaayirK6NfmrtGU0dtFnV7oHy9E1A8/i7IrTHGdI1E2izWAUOSnZEjhhs+8wPObOotGrk9ftCQs8XSWAmhBqjYlORMGmNM10okWAwENojIiyLybHhLdsa6S7hkMSTNiY8tGrnDpYV4VVFNFc7rgcIk5c4YY7pHItVQtyU7E0cUt80iL+Asy9pirIVEBQtvattrG91gsf8dGP3FJGbSGGO6VofBQlX/2xUZOWK4P5GBqQOBGNVQELvdItQIwTpnv6wwefkzxphuELcaSkRec18rRaQiaqsUkYquy2LXCldD+dRHXnpe4tVQjZXhGzgli77RH8AY00fEDRaqeqL7mqWq2VFblqpmd10Wu5hbDaVNyuDMweyp2dN8LBIsmtpeF66C6j8DGsugZkdSs2mMMV0p4RHcIjJIREaGtwTOXywie0RkXVTaXSKySUTWishTIpLrph8lIrUiUuhuD0RdM0NE3hORLSJyn0QGPSRHuGRBEAZnDGZPdVSwkHaqocLBYtBJzqs1chtjepEOg4WILBCRzcBHwH+BbcA/E7j3n4CzWqX9C5ikqlOAD4Cboo5tVdUCd7s2Kv13wP8AY92t9T07l9tmoUFlUMYgdldFLYDUbjWUGywGzgPEqYoyxpheIpGSxY+BE4APVHU0cBrQ4RBlVV0O7G+VtkxVw3U4bwLD27uHiOQD2ar6pjso8BHg/ATyfMjaLVm0Fyya3DaLtHzIHmeN3MaYXiWRYNGoqqWAR0Q8qvoKzqjuw3UlLUsoo0XkHRH5r4h8wk0bBkTPE17kpsUkIleLyGoRWb13795Dy1WrkkVlQyW1jbVOYiIlC3829JtmJQtjTK+SSLAoE5FMYDnwVxG5F6g+nIeKyA+AJuCvblIJMFJVpwHfBP5PRA66EV1VF6nqTFWdmZeXd2h5C5csmmBQxiCA5tJFIm0W/izoV+A0cNfvb3ueMcb0QIkEi/OAGuBG4AVgK/DpQ32giHwJOBf4fHi+KVWtd0svqOoa9xnjgGJaVlUNd9OSJ9wbKuj0hgLYXe22WxxMyQKg7N0kZtQYY7pOu8FCRLzAc6oaUtUmVX1YVe8L/2I/WCJyFvBdYIGq1kSl57nPQkSOxmnI/lBVS4AKETnB7QV1OfDMoTw7YW6wiFmySGSchS/TKVmAVUUZY3qNdoOFqgaBkIjkHOyNReRR4A1gvIgUichVwG+ALOBfrbrIngSsFZFC4AngWlUN1+FcBzwIbMEpcSTSE+uQiYjzU3EbuIHmHlEdlSx8WSAeCAyCtKHWfdYY02skMjdUFfCeiPyLqLYKVb2+vYtU9dIYyQ/FOXcJsCTOsdXApATy2Xl8zQ3cEKPNImZvqAqnvSIsd7IzXbkxxvQCiQSLJ90tWq+ey0K8AkFI86eRlZLVthoqXgO3P6pNPnUgVHyQ/MwaY0wXSCRY5KrqvdEJIvKNJOXnyOBzpvsAp90isQbuypbBwp8DjeVJzqgxxnSNRHpDLYyR9qVOzscRJVyyACdYJNbAXRE7WNiEgsaYXiBuyUJELgUuwxksF73YURatRmb3Ol6nzQJgcOZgtuzf4qS312bRWAGBwc3vU3JAgxCsAV9GkjNsjDHJ1V411AqcwXIDgV9EpVcCa5OZqe4mXnGGDAKD0gexYucK540nPLw7wZIFQEO5BQtjTI8XN1io6nZgOzCn67JzhPCBhppLFvtq9hEMBfF2NDdUrGDRWA4MTW5+jTEmyRKeorxP8dJcssgYREhDlNaWxq+GUo1fsrBGbmNML2DBIgbxSnObhTswb0/1nvgN3MFap30iepxFSlQ1lDHG9HAWLGLx0aJkAe4o7njjLKLnhQqzkoUxphfpcJyFiMwDbgNGuecLoKp6dHKz1n3EIy16Q0EHJYvIvFAWLIwxvVMig/Iewplxdg2R0Qe9nK+562ykZFG9O36bRVOMkkWKBQtjTO+RSLAoV9WkTt53pIkelJcbyMXn8bklCy8gMUoWUWtZhPkynXOtzcIY0wskEixeEZG7cOaHqg8nqurbSctVd4ua7sMjnpZrcXv8ibVZiMd5byULY0wvkEiwON59jV5KVYFTOz87R4bokgW4U37URE35Ea/Nwt9qcT+bH8oY00t0GCxU9ZSuyMgRJWq6D3C6z0ZKFhIrWMQoWYDTbmHBwhjTC3TYdVZEckTklyKy2t1+cSiLIfUk0dN9QIzJBOMFC19Wy3R/jrVZGGN6hUTGWSzGmQ/qInerAP6YzEx1Ox8QAneJcAZnDGZP9R7nfbw2C/GBN9Ay3aqhjDG9RCJtFmNU9TNR7293lz/tvcLrcAcBn1OyqG2qpaqhiqxYJYvwvFAiLdP9OVCxsStybIwxSZVIyaJWRE4Mv3EH6dUmcnMRWSwie0RkXVRafxH5l4hsdl/7uekiIveJyBYRWSsi06OuWeiev1lEYq2v0anE6/7Sj1rTAtyBefHaLFq3V4C1WRhjeo1EgsW1wP0isk1EtgG/Aa5J8P5/As5qlfZ94GVVHQu87L4HOBsY625XA78DJ7gAt+L0ypoN3BoOMEkTnom8qeUo7t3V7pQf2tTy/MZW62+HhdssbAEkY0wP12GwUNV3VXUqMAWYoqrTVDWh9SxUdTltF0o6D3jY3X8YOD8q/RF1vAnkikg+MB/4l6ruV9UDwL9oG4A6Vbsli3gN3DFLFrlOYAkmVBAzxpgjVsITCapqhapWdMIzB6tqibu/CwgvLzcM2Bl1XpGbFi+9DRG5Otxra+/evYeeQ7fNovXMs5HJBGMFC1+MYGHzQxljeolunXVWne5GnVZHo6qLVHWmqs7My8s75PtEShZubVNehnOvuG0WrRc+CoteLc8YY3qw7ggWu93qJdxXdwADxcCIqPOGu2nx0pMn3GbhlixSvCn0C/RrroaK1XU2XpsFQGNZ8vJqjDFdIJFBeZ8TkSx3//+JyJPRPZUOwbNAuEfTQuCZqPTL3V5RJ+BMYFgCvAicKSL93IbtM920pGndZgGQn5VPcWXxQbZZWMnCGNM7JFKy+KGqVrrdZ0/HmbL8d4ncXEQeBd4AxotIkYhcBdwJnCEim9373emevhT4ENgC/AG4DkBV9wM/Bt5ytx+5acnTqs0CYGTOSHaU72gbLEJBaKpuvxrK2iyMMT1cIoPywn9ffwpYpKrPi8hPErm5ql4a59BpMc5V4Ktx7rMYZyR51wj/VKJ6yI7MHsmaj9eA5LcMFk1VzqsFC2NML5ZIyaJYRH4PXAwsFZHUBK/rsWJVQ43MGcnemr3Uqqdlm0W8eaHAFkAyxvQaifzSvwinjWC+qpYB/YHvJDNT3S5GNdSo3FEA7GxsbFmyiDfjLNgCSMaYXiORYJEPPK+qm0XkZOBzwKpkZqq7te46C07JAmB7fX3iwcIWQDLG9BKJBIslQFBEjgEW4XRj/b+k5qq7teo6C83BYkd9Xas2izgLH4XZzLPGmF4gkWARUtUm4ELg16r6HZzSRq8Vq81iWNYwPOJhR11d7DaLWOMswCYTNMb0CokEi0YRuRS4HHjOTfMnL0tHgBhtFn6vn6FZQ9lRV5N4NRTYAkjGmF4hkWBxBTAHuENVPxKR0cCfk5ut7iW+tm0W4FRFba+rOvhgYSULY0wPl8issxuAbwPvicgkoEhVf570nHWnGCULcAfm1bYOFm6bRayus2DBwhjTKyQy3cfJwGbgfuC3wAciclJys9XNolfKizIyeyQ76yoJhRqaE5sqwJsGnjjjG63NwhjTCyQygvsXwJmq+j6AiIwDHgVmJDNj3UlEnDDaqhpqVO4oGkJBdjdCvqqzjGq8eaHCohdAar3sqjHG9BCJtFn4w4ECQFU/oLc3cAP4YldDAexoAtQtdiQSLGwBJGNMD5dIsFgtIg+KyMnu9gdgdbIz1t3EK22rocLBopHmdovGOGtZhNmUH8aYXiCRYPEVYANwvbttwFmXu3fzdlSyCAeLiviN22ALIBljeoUO2yxUtR74pbsBICKvA/OSmK9uJz5p02aRG8gl2x9ge2PUKO76fZBzbPwb2cyzxphe4FBnjx3Zqbk4EsUoWQCMTO/vlCzCwaK2BALtDGi3YGGM6QUONVh02rrZR6pYbRYAIzMGNAeLplpnydS0doKFtVkYY3qBuNVQInJhvENAWnKycwSJU7IYlZnHyl04bRZ1u5zE9oKFtVkYY3qB9tosPt3OsefaOdY7+GjTZgEwMjOP0hBU15eTEXK7wyYSLKxkYYzpweIGC1W9IhkPFJHxwGNRSUcDtwC5wP8Ae930m1V1qXvNTcBVOBVD16vqi8nIW4t8eiV2m0XmYAB2lO/kWL8bTdoNFlmAWLAwxvRoiYzg7lTuAL8CABHxAsXAUzgTFt6jqndHny8iE4FLgOOAocBLIjJOVWO0KHQiL9DQNrlFsMhwEwND4t9HPE7AsGooY0wP1t1raZ8GbFXV7e2ccx7wN1WtV9WPgC3A7GRnTLxtu84CjMoeCsCOimKnJ5R4IDWv/ZvZZILGmB7ukIKFiBzfSc+/BGeeqbCvichaEVksIv3ctGHAzqhzity0WPm6WkRWi8jqvXv3xjolcTGm+wDIzxyMF9he+bHbbXYweLxtr49mwcIY08Mdasni8cN9sIikAAui7vU7YAxOFVUJzgSGB0VVF6nqTFWdmZfXwV/7HeUvTtdZny/AMB9sr/i44zEWYTbzrDGmhzvUYNEZ06eeDbytqrsBVHW3qgZVNQT8geaqpmKcdb/DhrtpyRWn6yziZ5wfNpVth7qS9hu3w2y1PGNMD9edg/IuJaoKSkSif+teAKxz958FLhGRVHeVvrHAqk54frtiTfcBgMfP5FRYt38bwZqPEwsWKf2gYX+n59EYY7pKe4Py/kHsoCDAgMN5qIhkAGcA10Ql/6+IFLjP3BY+pqrrReTvOBMYNgFfTXpPKIhfsvD4mZwCdcEGtlbtZVx7PaHC0oY6VVa2poUxpodqr+vs3Yd4rEOqWk2rgKOqX2zn/DuAOw7nmQcrXpsFHj9TUp3d9+qUcYmULNKGQqjeKV2kHlacNcaYbtHeoLz/xjsmIr16xlnAGWcRBFV1Vs4LEz8TU8CDsLZB+UwiwSLd7bxVU2zBwhjTI8VtsxARr4hcKiLfFpFJbtq5IrIC+E2X5bC7hMNo69KFx0+aB45Jz+S9ehJrs0hzxmZQ+3EnZtAYY7pOe9VQD+H0QloF3CciHwMzge+r6tNdkLduJV63NBGk5U/J46woOyWQQmElCQYLt2RRm/xOXMYYkwztBYuZwBRVDYlIANgFjFHV0q7JWjdzx9lpUJHonsJusJjsb2RJI1R7ssiIcXkL4YBSYyULY0zP1F7X2QZ3zAOqWgd82GcCBW7XWWjbfVbcYOGpRIH1+7d0fDNvKqQOtJKFMabHaq9kMUFE1rr7Aoxx3wugqjol6bnrTlElixbC1VApTvp7u99j9rAEpqpKG2ptFsaYHqu9YNHOwtK9X4s2i2husBjthwyPl7W715KQtGEWLIwxPVZ7XWfbmwm29wuXLJpilyw8ApOy+vHenvcSu1/6UDjwTidm0Bhjuk57I7griT+CW1U1O2m5OgJE2ixalyyk+Uc2OWcoT+1e23YsRixpw6BuN4SawNPly4gYY8xhidvArapZqpodY8vq7YECiN9mIRIJGJMHHE1pbSm7qnZ1fL+0oYA2r9ttjDE9SHcvfnTkijcoD5obuQdNAkisKioyitvaLYwxPY8FizgiDdxxZp4FmJw/EyCxRu7IKG7rPmuM6XksWMQTrxoKIsFiQL8J5GfmJxgswqO4rWRhjOl5LFjE0W7Jwh2YR1o+xw8/nv9s+w+qHSzxEchz2jpqrGRhjOl5LFjE01HJwpsOviwWjFvAzoqdvLOrg26x4nGm/YhXstj5JPz7TKjbd3j5NsaYJLBgEUfcrrPgBIu0fBDh3HHn4hEPT296uuObpg2N32ax/k7Y9S9YvgCaag8128YYkxQWLOKJU7LQkFK5dRE1e74HQF5GHvNGzOOZ95/p+J7pcUZxV30I+9+C/Pmw701Y8XkIJX8xQGOMSZQFizjilSzq19TTVDWdpqrm9Z/On3A+a3ev5aMDH7V/07Shsdsstj/mvM5eBNN/CUVPwTvfOozcG2NM5+q2YCEi20TkPREpFJHVblp/EfmXiGx2X/u56SIi94nIFhFZKyLTk57BGNN9hKpC1L7iVBEFa/LRkHPsvPHnAXRcukgbBo3l0FTdMn37YzBwLmSMhAk3wNivwPv3QlUHwccYY7pId5csTlHVAlWd6b7/PvCyqo4FXnbfA5wNjHW3q4HfJT1nbrCILlnUvlQLTRCYF4AghA6EABjTfwyTBk1KIFi4Yy2iB+aVb4Kyd2HUxc1pE9xSxc6nDu8zGGNMJ+nuYNHaecDD7v7DwPlR6Y+o400gV0QSWKLu0ImI89Nxu842bmuk4b0GAnMD+Cc4XWeDe5sjyXnjz2P59uWU1rSz5Ed6jOVVdzwGCIz4bHNa1hjInQpFT3bOhzHGmMPUncFCgWUiskZErnbTBqtqibu/Cxjs7g8DdkZdW+SmtSAiV4vIahFZvXfv3sPPoQ+a9jRR/Xw11Y9X48n1EJgXwDvQKXZEB4vzJ5xPSEM8v/n5+PdrvbyqKmz/Gwz6ZHMgCRtxIexdAbUlGGNMd+vOYHGiqk7HqWL6qoicFH1QnVFuHYx0a0lVF6nqTFWdmZeXd9gZFL/QtKWJhvca8I3xkXlRJuIXJEXw5HpaBIsZ+TMYljWMR9c9Gv+Gaa1KFmXvQcWmllVQYSM+AygUPX3Yn8MYYw5XtwULVS12X/cATwGzgd3h6iX3dY97ejEwIury4W5aUmUsyCDjsxnkfiuXzAsz8Q72Ro5587yE9oYi70WEr83+Gi9seYHH1z8e+4b+bPBlOD2iQkFY/1MQrxsYWsmZCFnjnMF6HQkF4cBap6RijDFJ0C3BQkQyRCQrvA+cCawDngUWuqctBMItxs8Cl7u9ok4AyqOqq5LGf4yflGNTEH/btSo8Az0ES4ORHlEA3577bWYOnclXnv8Ku6t2t72hiNt9die8eYXTXjH5dmcqkFjnjrgQdr8C9fvjZ7KmCP59GvxzKrx7swUMY0xSdFfJYjDwmoi8C6wCnlfVF4A7gTNEZDNwuvseYCnwIbAF+ANwXddnuSVvntfpEbW/uXTh8/h4+PyHqWyo5CvPfyX2fFFpw5zSwrY/w5Qfw6QfxH/IiAtBg1D8j9jHi56BpVNh/2rIPxs23GkBwxiTFN2yZJuqfghMjZFeCpwWI12Br3ZB1hLmHdTcyB1u8AaYmDeRH5/yY7730vf489o/c/nUy1temD4cUJj6UzjupvYf0n8mpI9wgsvRC1se27oYVl4F/abBvL9B1jHw1nVOwEBg6h1O6cQYYzrBkdZ1tseI1SMq7FtzvsXcEXNZ+PRCPvf45/ig9IPmg8fdBCc+0XGggOaqqJIXYf/bzekHCp3AMOR0OPMNyB7nTFQ467dwzDWw4Wew5YHD/ITGGNPMgsUhEr/g6eeJGSy8Hi8vfuFFbvvkbfxz8z+ZeP9ELn7iYn698tesrKykfui5iT9owjchMBhePhX2vgEN5fDqZyF1AMz9K3hTozLlBoz8s2HNDS0DjDHGHAbpcB2GHmrmzJm6evXqpD6j6m9VBMuC5FybE/ec3VW7uePVO3h8w+ORtbp9Hh+TBk1i+pDpzBg6g+OHHc+UwVPwe/2xb1K9HV4+HepKoN902LeC0Kn/Zk/GOCrrK6lurKYp1MSonFEMTB+I1JfCC9PAkwJnvQ0p8fNnjDFhIrImakaNlscsWBy62pdrqXuzjtzv5zYvlhSHqlJcWczKopWsKVnD2yVvs6ZkDftqnPUrAr4ABUMKmDBwAuMHjOeo3KPwe/z4PD4agg0UlW5gx/r72Fa9n83ewWytKaeuqa7Nc7JTsxk/YDznDJ3MZ/c+zHGjP42c9KS1XxhjOmTBIknq19ZT80wN2ddmO72jDpKqsqN8ByuLV/Jm0Zu8XfI275e+HymBtJbhz2BkRj+OGTSNsf3HMrrfaHJSc8hMycQjHraXb2dz6WYKdxfy+o7XUZTxfrh6yDCu+uSd5Bx9mVNVZYwxMViwSJKmkiYqH6wk4zMZpExM6bT7lteVU1xZTFOoiWAoiNfjZUT2CHIDuc6cVQnYVbWLpzc+xV/euofX924mU+BLef34+pgTGJed7wwQzJkIA2ZBznGRdcWNMX2XBYsk0Ual7OdlBD4RIO2TaUl91uF4u3gV9/77W/zto9dpUOXsrFS+ngNnBurxCuBJdbrgDpgNA2Y675uqoLES6vdB3W6o3wsp/SF7PGRPgIFzYg8mNMb0WBYskqj8/nI8/TxkXZaV9Gcdrl1Vu1i0ZhG/W/07dlXtYkhGHp8dMY3P9c9lTqgYf9k7EKxpeZF4IDXP2er3OoEjnJ73CRh+AQw7FzKPtnYRY3o4CxZJVLu8lrr/1pFxcQYp4zqvKiqZGoINPLPpGR5b/xjPb36euqY6Ur2pTB08hZkDx3B0zgjys4eTnz2SnIxhpKdmke5PdzZtIK36I2TXMmewYPk656YZR8GQ05xFnHInO1VbvvRu/ZzGmINjwSKJNKhUPlhJqDZE9rXZeAI9qwG5qqGKF7a8wJtFb7L649WsKVlDVUNVh9flpObQP60/A1IzGeJVhlPNiMaPGeWp55gUOMYP/bNGIlljnFJH+ggIDHJLKQPAlwX+LPDnOu893TKZgDEmigWLJGsqaaLyoUpSpqaQ8emMLnlmsqgq5fXl7KraRUllCZUNlVQ3VFPdWE1tYy01jTVUN1ZTXlfO/rr9lNaUUlJVws7ynZTWtlz4KdeXwrhACmN9TYz11HGMH8amwBg/9PdE11oJpPSDtHxn2pLMY5xR6dkTnUb41P5d/nMwpi9qL1jYn3OdwJfvIzA3QN3rdaRMTME/puf2LBIRcgO55AZymTBwwkFdW9NYw/ay7WzZvyWyfbD/A14t/YD/278TjVqeJMufxujMPI5Kz2FUahojg15G1jQwvGotwxtfIF/qiUz2GxjkTNeeNdbZMkY5W/pIZ3S7t2dU/xnTk1nJopNok1LxhwpohOxrs5EUa+yNVtdUx4cHPmRz6WY+PPAhH5V9xEdlH7GtbBs7yndQUV/R4nxBGJyWy7BABsP8HoZ5GhmqlQzVKob6YJgPhvrcEkpggBM0wg3xgTxIHQRpg93XoZA+DAJDLLAY0w4rWXQB8QkZ52ZQ+adKav9bS/oZ1rgbLeALMDFvIhPzJsY8Xl5Xzo7yHRRXFlNUUcTO8p0UVxZTXFnMRxXFvF7+MaW1bdtSUjxehqY2MNS/i6H+3Qz1hhhKHUOllqFuQMn3Qo7HXVc9MNhpP8kYAWnDnSCSPtypAgsMcV79Odazy5hWLFh0It8IHynTU6hfWU/KpBR8+fbjTVROIIfJgclMHjw57jl1TXWUVJbwceXHfFz5McWVxc77qo8prihmXVUJy8o/pqK+ts21aV4/QwOZ5KcoQ707yPdsYSg15Es9+W5AyfdBPw+I1w+pA5tLKqkD3PettkBe8zlWYjG9nP0262Rpp6XR+EEjNc/VkHVVFuKxv1A7S8AXYHS/0YzuN7rd86oaqiJBJRxQSqqc9yVVJRRWfszSso+paqhvc22qx8eQ1DTyU5oY4t9Dvnc3QzxNDKGWfGoY4gaWwT5osYCiP8cNHOEgMtAJMin9nS21v9OI7891XlNynWusF5jpIexfaifzBDykz0+nekk19SvrCcwJdHeW+pzMlEzGDhjL2AFj2z0vOqiUVJVEgsquql2UVJWwubKEVyt2tenlFTYwNZshgUyGpAYY4vczxCcM9pQyRD5miFYzWKsYIvUM8ELcvxl8GVFBJNcJICm57vuc5ldftjNFiz/b7XKc1dz92JNq1WYm6SxYJIH/WD/+cX5qX67F099DynirojgSJRpUGoIN7KraFdlKKkvYXb07ElR2V+3mdXc/1kzAXvEyKL0/g9NyGZyayZDUDAanpDLY72Ww18Mgb4jBniCDm+oY0PgxvoqN0FAGjeXOsrodEa8TOHwZ4M8EX/SW4W7uvjfdGSzpywBvmrPvTXf2vWngS2ve9waaX8VnAamP6/JgISIjgEdw1uFWYJGq3isitwH/A+x1T71ZVZe619wEXAUEgetV9cWuzvfBEBEyzs+g8q+VVC+pRi4R/Ef33O60fV2KN4WROSMZmTOy3fNUlcqGyhYBZXeVE1R2V++OBJgN+7azu3o3DcGGNvcQhAHpAxickc+gjKkMSu/PoEA2g1KzGJSaTp4/hUEpqeT5vOR5IVdCSNCdx6upunlOr2A11O1y3jdVN2+hts9MiHjAE3ADSMApzXhTW756Utz9lDibP+rV3cTvVMVF74u7eaJfvW66N+p9rM3jvOIBj/sqnqh0cWdedgf6iCcqTVrui0SlhfeJet96/zBEeqVqy/1wd/O4aRp1TdQxf/bh56mVLu86KyL5QL6qvi0iWcAa4HzgIqBKVe9udf5E4FFgNjAUeAkYp9r+n1xd3XU2llBtiKpHqggeCJJ5SSb+oyxgGEd48OPuqt2RoLKneg97qvdEAsve6r2RtPL68pj38Xl8DEwfSF56HnkZec5rep6TltG8H94GBHJIockJHMFaaKpxtmCtMy9YsBaCde6xWgjVu2m1EKyPeh/er3MCUPh9qNFND+83uFvUvjZ18U+7u4SDSBcPTwgMhgtjL3PQkSOq66yqlgAl7n6liGwEhrVzyXnA31S1HvhIRLbgBI43kp7Zw+RJ85D5+UwqH66k6s9VePI8pBybQsqxKXgHHfz6F6b3iB78OH7g+A7Pr2+qZ1/NPvZU72FvTXMQ2Vu9l701zravZh9vl7zN3pq9lNWVxb1XVkpWywCSPoABae4W3k8fxYDsAc6ULukDyPBnJDw9fodUnYARamx+jew3RR0LttwPNblpbnpkP7yFWu2779Hm9PBf4dH7hFq9qnNtm7/aabnfohRAVHrrfWgOHK1FpUeXVsLp0no/3vFwCcfjVDEmQbe2WYjIUcA0YCUwD/iaiFwOrAa+paoHcALJm1GXFdF+cDmieDI9ZF2ZRcO6Bho3NFK3vI665XVO4JiYQsqkFLz9LXCY9qX6UhmWPYxh2Yn9028MNlJaW8re6r2U1payr2Zfi/19Nfuc4zV72bRvE6W1pW0GRkbze/z0T+sfc+sX6Ee/tH4t9sOvuYFcUlp3KxZxq5yspN2TdFuwEJFMYAlwg6pWiMjvgB/jhOQfA78ArjzIe14NXA0wcmT79ctdyZPmITArQGBWgFBViIaNbuD4bx11/63DP95PYF4A3zDrb2A6h9/rZ0jmEIZkDkn4moZgA/trnfm+SmtLKa0p5UDdgcj7A7UHIvOB7azYybu732V/7f4OJ55M96eTG8ilX8AJHuEgkpuaS04gh9xALjmp7msgJ7KfnZpNTiCHNF9a55VqzCHrluk+RMQPPAe8qKq/jHH8KOA5VZ3kNm6jqj9zj70I3Kaq7VZDHQltFh0JVYSoX1NP/ep6tE7xHeUj/ex0vAOtpGF6jsZgIwfqDnCg9gAH6g6wv3Y/B2oPUFZXRlldGQfqDkReD9QeoLy+3Hnv7oc01O79fR4fOak5keCRnZodeZ+VkkV2anaLLSs1i6yUrJivPhvX0q4jqs1CnD8RHgI2RgcKEcl32zMALgDchRJ4Fvg/EfklTgP3WGBVF2Y5aTzZHtJOSSMwL0D92/XUvVpHxaIKAicFCMwJENoXonF7I6H9ITwDPHgHefEO8uJJO3KnQdegovXORojm6lUh8tehhhQanJUGtVGhyZlbi2DL81ts4Spk91VD2lwtHKm+lRbVuc7DWmUw+p6RjjDS4n2b+3laXiMecdKir/fEOe6h1/9V7Pf6GZQxiEEZgw76WlWlqqGKsroyyuvLKa8rj+xX1FdQXlceSS+vd7bK+kqKKooi+xX1FdQH2w6wjCXgC5CVkkVmSiZZqc5ri83vvGakZDiv/ozI+wx/RovXdH96ZL8vBKHu+ITzgC8C74lIoZt2M3CpiBTg/O+9DbgGQFXXi8jfgQ1AE/DVjnpC9TSSIgROCJAyKYWaF2qoe8Vp1yD8Kf1Ao7vvgZTJKQTmBtqUQFQVrVBCFSEkS/DkeFr8otKgEtwXJLgrSHBPEG1wf0GHQDIFb38vnn4e55dxnaJ1zr1CB0IEy4JOHtR5DjT/UtRGjZxPX+nocrAivTglst86sLQIMt6W58bd90ZdH96PvjZqv8113hjP8bbdj75XdNDvDCLi/OWfmsUIRhzyfeqb6qlsqIwEj/B+67SqhiqqGqoi+5X1lZTVlVFcURw5Vt1YHXO8THtSvCmRBcIy/BnNi4X500nzpzXv+9KczU0L78d6DfgCpPndV/d9wBcg1ZeKR7r+D0abdfYI1PB+A41bG/EN8+Eb5cOT40ErleCeII1bGql/px6awDfGh3gEbVBCNc4v9Ra/rH1O6YWQ+8u/PuqvcR9IqvvLQIRQVag5OEUT8OR48OR6nJl0w39hRzqNKOITJE2QgCCpUZtHnMDSqgs4HhC/OPfz41zvc3/RQXNAir4u/Fe7CK1LAOFrIq/tdUSJdHDRtqWVcFr0faI+Z+Rcd2uRFj4n+rg2B+PI+a33gzGuCzVfp8HY57VJDzbnI+kSDDgdBa8WgSzeuTHS2wS0WME1RtA7mFJeU6gpso5LVUNVZD/Wa01jTWSdl9rGWmqaamKnu+9rm2qpbaxtMWX/wUr1ppLqS3WChze1RSAZnDGY5y577pDue0RVQ5mOpYxPaTPqW7IFT7YH/zF+Ap8IUL+ynoZNDc4v2lTB28+Lf4wf7wAvniwPocoQwdIgofJQ5BwJCN48L94hXjz9PS3mrdKQW4ooCyGe5vMlU5z/2UyPEB2gNNjOfqtA0yKtg/12j8d6RoOiIW0/P+Fgl2xC7EATo4QmXiHLk0WWNyux0p0fJNBO8IoqQapHaaKJBm2gQRuoC9VRr/XUaz11wTrqtI76UD21wVrqtI7aYC21odoWr9XB6sj7uqY66pvqqWuqI92fnBmvLVj0QJ4MD2mnppF2alqn3VM8gjfXizfXGtd7MhGJ/F8tcfv2H5kiJbvoABJdcgq2KpHFCFRxS2WxSmix7h2+Nnq/vjlIxiwRHmbpLsX977BEVV96MpNTRWXBwhhzRBCJqn7y96xAF61F6S5W8IpVrdg6UMWrogy2uib6fm6wFV9yfnYWLIwxphP15NJde47cPpjGGGOOGBYsjDHGdMiChTHGmA5ZsDDGGNMhCxbGGGM6ZMHCGGNMhyxYGGOM6ZAFC2OMMR2yYGGMMaZDFiyMMcZ0yIKFMcaYDlmwMMYY0yELFsYYYzpkwcIYY0yHLFgYY4zpUI8JFiJyloi8LyJbROT73Z0fY4zpS3pEsBARL3A/cDYwEbhURCZ2b66MMabv6BHBApgNbFHVD1W1AfgbcF4358kYY/qMnrKs6jBgZ9T7IuD41ieJyNXA1e7bKhF5/xCfNxDYd4jX9lT2mXu/vvZ5wT7zwRoV70BPCRYJUdVFwKLDvY+IrFbVmZ2QpR7DPnPv19c+L9hn7kw9pRqqGBgR9X64m2aMMaYL9JRg8RYwVkRGi0gKcAnwbDfnyRhj+oweUQ2lqk0i8jXgRcALLFbV9Ul85GFXZfVA9pl7v772ecE+c6cRVU3GfY0xxvQiPaUayhhjTDeyYGGMMaZDFiyi9IUpRURkhIi8IiIbRGS9iHzDTe8vIv8Skc3ua7/uzmtnExGviLwjIs+570eLyEr3+37M7TzRa4hIrog8ISKbRGSjiMzp7d+ziNzo/rteJyKPikigt33PIrJYRPaIyLqotJjfqzjucz/7WhGZfqjPtWDh6kNTijQB31LVicAJwFfdz/l94GVVHQu87L7vbb4BbIx6/3PgHlU9BjgAXNUtuUqee4EXVHUCMBXns/fa71lEhgHXAzNVdRJOZ5hL6H3f85+As1qlxftezwbGutvVwO8O9aEWLJr1iSlFVLVEVd929ytxfoEMw/msD7unPQyc3y0ZTBIRGQ58CnjQfS/AqcAT7im96jOLSA5wEvAQgKo2qGoZvfx7xunhmSYiPiAdKKGXfc+quhzY3yo53vd6HvCIOt4EckUk/1Cea8GiWawpRYZ1U166hIgcBUwDVgKDVbXEPbQLGNxd+UqSXwHfBULu+wFAmao2ue972/c9GtgL/NGtentQRDLoxd+zqhYDdwM7cIJEObCG3v09h8X7Xjvt95oFiz5KRDKBJcANqloRfUyd/tS9pk+1iJwL7FHVNd2dly7kA6YDv1PVaUA1raqceuH33A/nL+nRwFAgg7bVNb1esr5XCxbN+syUIiLixwkUf1XVJ93k3eHiqfu6p7vylwTzgAUisg2nevFUnPr8XLe6Anrf910EFKnqSvf9EzjBozd/z6cDH6nqXlVtBJ7E+e578/ccFu977bTfaxYsmvWJKUXcuvqHgI2q+suoQ88CC939hcAzXZ23ZFHVm1R1uKoehfO9/ltVPw+8AnzWPa23feZdwE4RGe8mnQZsoBd/zzjVTyeISLr77zz8mXvt9xwl3vf6LHC52yvqBKA8qrrqoNgI7igicg5O3XZ4SpE7ujdHnU9ETgReBd6juf7+Zpx2i78DI4HtwEWq2roRrccTkZOBb6vquSJyNE5Joz/wDvAFVa3vxux1KhEpwGnQTwE+BK7A+QOx137PInI7cDFOr793gC/j1NH3mu9ZRB4FTsaZinw3cCvwNDG+Vzdo/ganOq4GuEJVVx/Scy1YGGOM6YhVQxljjOmQBQtjjDEdsmBhjDGmQxYsjDHGdMiChTHGmA5ZsDAmBhFZ4b4eJSKXdfK9b471LGOOZNZ11ph2RI/LOIhrfFFzEcU6XqWqmZ2QPWO6jJUsjIlBRKrc3TuBT4hIobtWgldE7hKRt9z1Aa5xzz9ZRF4VkWdxRg0jIk+LyBp3fYWr3bQ7cWZFLRSRv0Y/yx1le5e7FsN7InJx1L3/E7U2xV/dwVaIyJ3irE2yVkTu7sqfkelbfB2fYkyf9n2iShbuL/1yVZ0lIqnA6yKyzD13OjBJVT9y31/pjqJNA94SkSWq+n0R+ZqqFsR41oVAAc7aEwPda5a7x6YBxwEfA68D80RkI3ABMEFVVURyO/ejG9PMShbGHJwzcebaKcSZImUAzsIyAKuiAgXA9SLyLvAmzmRuY2nficCjqhpU1d3Af4FZUfcuUtUQUAgchTMFdx3wkIhciDOdgzFJYcHCmIMjwNdVtcDdRqtquGRRHTnJaes4HZijqlNx5iQKHMZzo+cyCgLhdpHZODPKngu8cBj3N6ZdFiyMaV8lkBX1/kXgK+4074jIOHdRodZygAOqWiMiE3CWsA1rDF/fyqvAxW67SB7OSner4mXMXZMkR1WXAjfiVF8ZkxTWZmFM+9YCQbc66U8462AcBbztNjLvJfYynS8A17rtCu/jVEWFLQLWisjb7lTpYU8Bc4B3cRav+a6q7nKDTSxZwDMiEsAp8XzzkD6hMQmwrrPGGGM6ZNVQxhhjOmTBwhhjTIcsWBhjjOmQBQtjjDEdsmBhjDGmQxYsjDHGdMiChTHGmA79f1jkzX8yuiVwAAAAAElFTkSuQmCC\n", 64 | "text/plain": [ 65 | "
" 66 | ] 67 | }, 68 | "metadata": { 69 | "needs_background": "light" 70 | }, 71 | "output_type": "display_data" 72 | } 73 | ], 74 | "source": [ 75 | "# IRL Loss on train trajectories, as a function of cost function updates\n", 76 | "\n", 77 | "plt.figure()\n", 78 | "plt.plot(weighted['irl_loss_train'].detach(), color='orange', label=\"Weighted Ours\")\n", 79 | "plt.plot(timedep['irl_loss_train'].detach(), color='green', label=\"Time Dep Weighted Ours\")\n", 80 | "plt.plot(rbf['irl_loss_train'].detach(), color='violet', label=\"RBF Weighted Ours\")\n", 81 | "plt.xlabel(\"iterations\")\n", 82 | "plt.ylabel(\"IRL Loss on train\")\n", 83 | "plt.ylim([0, 2000])\n", 84 | "plt.legend()\n", 85 | "\n", 86 | "plt.savefig(f\"{model_data_dir}/{experiment_type}_IRL_loss_train.png\")" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 7, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABItUlEQVR4nO3dd5xU5b348c/3nKlbWKqAgIJIESkLAgoYLxp77ElsSSSWnxpNLMlNoubGksSboibRRGOMcqMxUWPJtZEEY8xFxQZxLYAKKCIrZSlbp895fn+cM7Ozs7MF2NmF3e/79ZrXmXnOOc88swP73aeLMQallFKqPVZPF0AppdSeT4OFUkqpDmmwUEop1SENFkoppTqkwUIppVSHfD1dgGIYPHiwGT16dE8XQyml9irLly/faowZUuhcrwwWo0ePZtmyZT1dDKWU2quIyMdtndNmKKWUUh3SYKGUUqpDGiyUUkp1qFf2WSi1N0gmk2zYsIFYLNbTRVF9TCgUYuTIkfj9/k7fo8FCqR6yYcMGysvLGT16NCLS08VRfYQxhm3btrFhwwbGjBnT6fuK1gwlIiEReV1E3hKRFSJyk5c+RkReE5E1IvKIiAS89KD3eo13fnROXtd66e+LyHHFKrNS3SkWizFo0CANFKpbiQiDBg3a6RptMfss4sBRxphpQCVwvIgcBvwU+IUx5kBgB3Chd/2FwA4v/RfedYjIJOBs4GDgeOAuEbGLWG6luo0GCtUTduXfXdGChXE1ei/93sMARwGPeen3A6d5z0/1XuOd/6y4n+hU4GFjTNwY8xGwBphdrHIrpZRqraijoUTEFpEqYAvwHLAWqDXGpLxLNgAjvOcjgE8AvPN1wKDc9AL35L7XxSKyTESW1dTU7Hqh03FI1O36/UrtBa6++mp++ctfZl8fd9xxXHTRRdnX3/rWt/j5z3/e5v3XX389//jHP9p9jxtvvJFbb721VXptbS133XXXTpe5rfwA7rnnHiZOnMjEiROZPXs2L7300k7nr9pX1GBhjEkbYyqBkbi1gYlFfK97jDEzjTEzhwwpOFu9c9IRSDV1XcGU2gPNmzePpUuXAuA4Dlu3bmXFihXZ80uXLmXu3Llt3v+DH/yAo48+epfee1eDRVueeeYZfvvb3/LSSy/x3nvvcffdd3PuueeyadOmTueRSqU6vqiP65Z5FsaYWuAFYA7QX0Qyo7BGAtXe82pgFIB3vgLYlpte4J5iFbio2SvV0+bOncsrr7wCwIoVK5g8eTLl5eXs2LGDeDzOqlWrmDFjBsuXL+c//uM/OOSQQzjuuOPYuHEjAF/96ld57DG3NXnRokVMnDiRQw45hCuuuIKTTjop+z4rV65k/vz5HHDAAdxxxx0AXHPNNaxdu5bKykq+/e1vA3DLLbcwa9Yspk6dyg033JC9/+abb2b8+PEcfvjhvP/++wU/y09/+lNuueUWBg8eDMCMGTNYsGABd955J+Au/7N161YAli1bxvz58wG3pvKVr3yFefPm8ZWvfIUVK1Ywe/ZsKisrmTp1KqtXr+6Sn3VvUbShsyIyBEgaY2pFJAwcg9tp/QLwBeBhYAHwpHfLU97rV7zz/zTGGBF5CviTiPwc2BcYB7xerHKDdjiqHrD8KthR1bV5DqiEQ35Z8NS+++6Lz+dj/fr1LF26lDlz5lBdXc0rr7xCRUUFU6ZMQUT4xje+wZNPPsmQIUN45JFH+N73vsfChQuz+cRiMS655BKWLFnCmDFjOOecc1q8z3vvvccLL7xAQ0MDEyZM4Gtf+xo/+clPePfdd6mqcj/v4sWLWb16Na+//jrGGE455RSWLFlCaWkpDz/8MFVVVaRSKWbMmMEhhxzS6rOsWLGiVfrMmTO5//77W12bb+XKlbz00kuEw2G+8Y1vcOWVV/KlL32JRCJBOp3u8P6+pJjzLIYD93sjlyzgz8aYZ0RkJfCwiPwIeBO4z7v+PuAPIrIG2I47AgpjzAoR+TOwEkgBlxtjivgtCm4/vFK929y5c1m6dClLly7lm9/8JtXV1SxdupSKigrmzZvH+++/z7vvvssxxxwDQDqdZvjw4S3yeO+99zjggAOy4/XPOecc7rnnnuz5z33ucwSDQYLBIPvssw+bN29uVY7FixezePFipk+fDkBjYyOrV6+moaGB008/nZKSEgBOOeWULv8ZnHLKKYTDYQDmzJnDzTffzIYNGzjjjDMYN25cl7/f3qxowcIY8zYwvUD6hxQYzWSMiQFfbCOvm4Gbu7qMSu0x2qgBFFOm3+Kdd95h8uTJjBo1ittuu41+/fpx/vnnY4zh4IMPzjZX7YpgMJh9btt2wb4BYwzXXnstl1xySYv03A749kyaNInly5dz1FFHZdOWL1/OwQcfDIDP58NxHIBWcwtKS0uzz88991wOPfRQnn32WU488UR++9vftsizr9O1ofKJ1ixU3zB37lyeeeYZBg4ciG3bDBw4kNraWl555RXmzp3LhAkTqKmpyQaLZDLZohMcYMKECXz44YesW7cOgEceeaTD9y0vL6ehoSH7+rjjjmPhwoU0Nroj7aurq9myZQtHHHEE//u//0s0GqWhoYGnn366YH7f+c53+O53v8u2bdsAqKqq4ve//z2XXXYZ4PZZLF++HIDHH3+8zXJ9+OGHHHDAAVxxxRWceuqpvP322x1+lr5El/tQqo+aMmUKW7du5dxzz22R1tjYmO0sfuyxx7jiiiuoq6sjlUpx1VVXZf9iBwiHw9x1110cf/zxlJaWMmvWrA7fd9CgQcybN4/JkydzwgkncMstt7Bq1SrmzJkDQFlZGQ8++CAzZszgrLPOYtq0aeyzzz5t5n3KKadQXV3N3LlzERHKy8t58MEHs01mN9xwAxdeeCHf//73s53bhfz5z3/mD3/4A36/n2HDhnHdddd1+Fn6EjG9cOTPzJkzzS5vfpRsgMQOKN2vawulVJ5Vq1Zx0EEH9XQxdltjYyNlZWUYY7j88ssZN24cV199dU8XS3Wg0L8/EVlujJlZ6Hpthiqo9wVQpYrld7/7HZWVlRx88MHU1dW16ntQvYM2Q7WiQ2eV2hlXX3211iT6AK1ZFNILm+aUUmp3aLDIp6uAKqVUKxoscqVjsP4xaPywp0uilFJ7FA0WuZIN8OpXoebFni6JUkrtUTRY5LK8/WidZM+WQ6ki27ZtG5WVlVRWVjJs2DBGjBhBZWUlZWVl2clsXenGG2/Mvse4ceM444wzWLlyZZfkPX369Ow6U6lUKjtPI+OQQw7h3//+d5v3n3jiidTW1rb7HvPnz6fQcPyqqioWLVq002VuK79EIsFVV13FgQceyLhx4zj11FPZsGHDTudfDBosclkB96jBQvVygwYNoqqqiqqqKi699FKuvvpqqqqqaGxs7NLlw3Nl3mP16tWcddZZHHXUUezW3jOe3OXW33rrLcaPH5993dTUxNq1a5k2bVqb9y9atIj+/fvv0nvvarBoy3XXXUdDQwPvv/8+q1ev5rTTTuOMM85gZ+bDFWu5dQ0WuTRYqD7uX//6V3aJ8RtvvJEFCxbwmc98hv33358nnniC73znO0yZMoXjjz+eZNL9f9LWMubtOeusszj22GP505/+1G4e8+fP58orr6SyspLJkyfz+uutF5zOLIgI7j4cl156abam8frrr3PIIYdg2zYPPvhgdgnySy65JLuqbO4S5j/84Q+ZMGEChx9+OOecc06LzZYeffRRZs+ezfjx43nxxRdJJBJcf/31PPLII1RWVvLII4/Q1NTEBRdcwOzZs5k+fTpPPukuqh2NRjn77LM56KCDOP3004lGo60+RyQS4X/+53/4xS9+gW27O0eff/75BINB/vnPf7Ju3TomT56cvf7WW2/lxhtvzP6crrrqKmbOnMntt9/Oo48+yuTJk5k2bRpHHHFEh99HZ+g8i1yZrb2dpDt8VkdGqW5y1d+uompTVZfmWTmskl8e/8vdymPt2rW88MILrFy5kjlz5vD444/zs5/9jNNPP51nn32Wz33ucx0uY96WGTNm8N5775FMJtvNIxKJUFVVxZIlS7jgggt49913W+Qzb948/uu//gtwg8UNN9zAQw89RENDQ3YTp1WrVvHII4/w8ssv4/f7ueyyy/jjH//Ieeedl83njTfe4PHHH+ett94imUy2WhI9lUrx+uuvs2jRIm666Sb+8Y9/8IMf/IBly5bx61//GnBrBkcddRQLFy6ktraW2bNnc/TRR/Pb3/6WkpISVq1axdtvv82MGTNa/TzWrFnDfvvtR79+/Vqkz5w5kxUrVjB27Nh2f56JRCLbtDVlyhT+/ve/M2LEiA6b2DpLg0UuEbd2YbRmoRTACSecgN/vZ8qUKaTTaY4//njA/WW0bt26Ti1j3pZM00pHeWT2yDjiiCOor6+ntra2RbPR/vvvTyKRYNOmTbz33ntMmDCBWbNm8dprr7F06VK+8Y1v8Pzzz7N8+fLs+lLRaJR99tmnRXlefvllTj31VEKhEKFQiJNPPrnF+TPOOANw+0AyCyfmW7x4MU899VS2RhKLxVi/fj1LlizhiiuuAGDq1KlMnTq1Uz+jnXHWWWdln8+bN4+vfvWrnHnmmdly7y4NFvksv9cMZdDZ3Kq77G4NoFgyS4xbloXf70e82rZlWaRSqd1axvzNN99k5syZHeYheTX8/NfgNkU9+uijDB8+HBHhsMMO4+WXX+b1119nzpw5rF69mgULFvDjH/94p8uZkflZtLXUOrgB8PHHH2fChAk7nf/YsWNZv349DQ0NlJeXZ9OXL1/OSSed1GKpdWh/ufW7776b1157jWeffZZDDjmE5cuXM2jQoJ0uUy7ts8inNQulOq0zy5gX8vjjj7N48WLOOeecDvPILHv+0ksvUVFRQUVFRav85s6dyy9/+cvsyrVz5szhgQceYNiwYVRUVPDZz36Wxx57jC1btgCwfft2Pv744xZ5zJs3j6effppYLEZjYyPPPPNMh5+j0HLrv/rVr7K1pjfffBNwa0WZ/pl333234PLnpaWlLFiwgG9+85vZ/pQHHniASCTCUUcdxdChQ9myZQvbtm0jHo+3W761a9dy6KGH8oMf/IAhQ4bwySefdPhZOqLBIp/4m/sslFLtCgQCPPbYY3z3u99l2rRpVFZWZjub8/3iF7/IDp198MEH+ec//8mQIUM6zCMUCjF9+nQuvfRS7rvvvoJ5z5s3jw8//DAbLIYPH046nWbu3LmAu0HSj370I4499limTp3KMccc06ojftasWZxyyilMnTqVE044gSlTphQMTLmOPPJIVq5cme3g/v73v08ymWTq1KkcfPDBfP/73wfga1/7Go2NjRx00EFcf/31BbeHBfjxj39MKBRi/PjxjBs3jkcffZS//OUviAh+v5/rr7+e2bNnc8wxxzBx4sQ2y/Xtb3+bKVOmMHnyZObOndvuaLDO0iXK8/1lJAyeA/MeBsvu2oIplaO3LFFeTPPnz+fWW29l5syCq2Z3ucxy65FIhCOOOIJ77rmnYGd0b7CzS5Rrn0U+yw9OAl2mXKm+5+KLL2blypXEYjEWLFjQawPFrtBgkc8K6DwLpfYQ//rXv7r1/TL9Cqo17bPIZwXApNCahVJKNdNgkc/yuc1QvbAvRymldpUGi3xWAJzirK2ilFJ7Kw0W+VpMylNKKQUaLFrL9lko1bvZtp1doO/kk0/OriG0bt06wuEwlZWVTJs2jblz5/L+++8DbodzRUVFdnnzo48+ukWexhgGDx7Mjh07ANi4cSMiwksvvZS9ZsiQIWzbtq3NcmXmRrQnd/G/XP/617/anOexK/nV1dVx3nnnceCBBzJ27FjOO+886urqdjr/3qBowUJERonICyKyUkRWiMiVXvqNIlItIlXe48Sce64VkTUi8r6IHJeTfryXtkZErilWmd0305qF6hvC4TBVVVW8++67DBw4kDvvvDN7buzYsVRVVfHWW2+xYMEC/vu//zt77jOf+Ux2efN//OMfLfLMLLWRmY29dOlSpk+fnv0F/v777zNo0KB2l57YlV/2GbsaLNpy4YUXcsABB7BmzRrWrl3LmDFjuOiii3Yqj8xs7L1dMWsWKeBbxphJwGHA5SIyyTv3C2NMpfdYBOCdOxs4GDgeuEtEbBGxgTuBE4BJwDk5+XQ9rVmoPmjOnDlUV1cXPFdfX8+AAQM6nVf+kuFXX311i+Axb948AG655RZmzZrF1KlTueGGG7L3l5WVAeA4DpdddhkTJ07kmGOO4cQTT+Sxxx7LXverX/2KGTNmMGXKFN577z3WrVvH3XffnZ0p/uKLL1JTU8PnP/95Zs2axaxZs3j55ZcBd/OnY489loMPPpiLLrqo4H4Ra9asYfny5dlZ2ADXX389y5YtY+3atS2Wcwf4+te/zu9//3vAral897vfZcaMGTz66KPccccdTJo0ialTp3L22Wd3+me5JynaPAtjzEZgo/e8QURWASPaueVU4GFjTBz4SETWALO9c2uMMR8CiMjD3rVds81WvsykPB0NpbpR5O8R0pu79i9Qe6hNyXElHV6XTqd5/vnnufDCC7Npa9eupbKykoaGBiKRCK+99lr23IsvvkhlZSUAX/ziF/ne977XIr958+Zx0003Ae5+EjfddBO33347QHbJ8MWLF7N69Wpef/11jDGccsopLFmypMXeC0888QTr1q1j5cqVbNmyhYMOOogLLrgge37w4MH8+9//5q677uLWW2/l3nvv5dJLL6WsrIz//M//BODcc8/l6quv5vDDD2f9+vUcd9xxrFq1iptuuonDDz+c66+/nmeffbbgMiKZZTwye0tAc9PdihUrWi0lnm/QoEHZHfr23XdfPvroI4LBYJctGd7dumVSnoiMBqYDrwHzgK+LyHnAMtzaxw7cQPJqzm0baA4un+SlH1rgPS4GLgbYb7/9dr2wOhpK9RHRaJTKykqqq6s56KCDskuEQ3MzFLgL+V188cX87W9/A9xmqPYWsZs1axZvvvkmTU1NJJNJysrKsk05S5cu5Vvf+hb33nsvixcvZvr06YC7zMbq1atbBIuXXnqJL37xi1iWxbBhwzjyyCNbvE/ukuFPPPFEwbL84x//aLF9a319PY2NjSxZsiR7z+c+97mdqjl1Vu6S4VOnTuVLX/oSp512GqeddlqXv1d3KHqwEJEy4HHgKmNMvYj8BvghbqfAD4HbgAvayaJTjDH3APeAuzbULmeUXXVWaxaq+3SmBtDVMn0WkUiE4447jjvvvDO750KuU045hfPPP7/T+ZaUlDBu3DgWLlyYXS7jsMMOY9GiRWzZsoUJEyZgjOHaa6/lkksu2eXyd2bJcMdxePXVVwmFQjud/6RJk6iqqsJxHCzLyuZXVVXFpEmT2LRpU6eXDH/22WdZsmQJTz/9NDfffDPvvPMOPt/etYBGUUdDiYgfN1D80RjzBIAxZrMxJm2McYDf0dzUVA2Myrl9pJfWVnpxZIfOKtU3lJSUcMcdd3DbbbcV/KX70ksvdbhLW75CS4bffvvtHHbYYYgIxx13HAsXLqSxsRGA6urq7PLhGfPmzePxxx/HcRw2b97cqaU/8pcMP/bYY/nVr36VfZ2pLeUuGf7Xv/41O3or14EHHsj06dP50Y9+lE370Y9+xIwZMzjwwAPZf//9WblyJfF4nNraWp5//vmCZXIch08++YQjjzySn/70p9TV1WU/996kmKOhBLgPWGWM+XlOeu42WqcDmT0SnwLOFpGgiIwBxgGvA28A40RkjIgEcDvBnypWuXW5D9UXTZ8+nalTp/LQQw8BzX0W06ZN47rrruPee+/dqfzylwyfMWMGGzZsyA6LPfbYYzn33HOZM2cOU6ZM4Qtf+EKLX/IAn//85xk5ciSTJk3iy1/+MjNmzOhwyfCTTz6Zv/zlL9kO7jvuuINly5YxdepUJk2axN133w3ADTfcwJIlSzj44IN54okn2my6vu+++/jggw8YO3YsY8eO5YMPPsj2b4waNYozzzyTyZMnc+aZZ2ab1PKl02m+/OUvM2XKFKZPn84VV1zRYqe/vUXRligXkcOBF4F3gExd7TrgHKAS97fxOuASrzMcEfkebpNUCrfZ6q9e+onALwEbWGiMubm9996tJcqXXQkfLoQzNoGvtOPrldpFukR5xzJLhm/bto3Zs2fz8ssvM2zYsJ4uVq+wxyxRbox5icL7ki5q556bgVaBwBte2+Z9XSrTwa2joZTqcSeddBK1tbUkEgm+//3va6DoQXtXD0t3sPy6rapSe4juXqJctU2X+8hnBcCk3YdSRdYbd6pUe75d+XenwSKf5XePOiJKFVkoFGLbtm0aMFS3Msawbdu2nR5OrM1Q+Wx37DbpWPvXKbWbRo4cyYYNG6ipqenpoqg+JhQKMXLkyJ26R4NFPivgHrVmoYrM7/czZsyYni6GUp2izVD5ss1QWrNQSqkMDRb5tGahlFKtaLDIJ17NIqU1C6WUytBgkS9TszCJni2HUkrtQTRY5Mv0WaTjPVsOpZTag2iwyJetWWifhVJKZWiwyJcdDaU1C6WUytBgkS9Ts0hrn4VSSmVosMiXrVlosFBKqQwNFvmy8yy0GUoppTI0WOTLBovCe/oqpVRfpMEiX6YZSudZKKVUlgaLfFqzUEqpVjRY5NMObqWUakWDRT5dSFAppVrRYJFPd8pTSqlWNFjkyy73oX0WSimV0WGwEJE/dCat19A+C6WUaqUzNYuDc1+IiA0cUpzi7AG0z0IppVppM1iIyLUi0gBMFZF679EAbAGe7ChjERklIi+IyEoRWSEiV3rpA0XkORFZ7R0HeOkiIneIyBoReVtEZuTktcC7frWILNjtT91uwb1tyTVYKKVUVpvBwhjzY2NMOXCLMaaf9yg3xgwyxlzbibxTwLeMMZOAw4DLRWQScA3wvDFmHPC89xrgBGCc97gY+A24wQW4ATgUmA3ckAkwRSHi7panwUIppbI60wz1jIiUAojIl0Xk5yKyf0c3GWM2GmP+7T1vAFYBI4BTgfu9y+4HTvOenwo8YFyvAv1FZDhwHPCcMWa7MWYH8BxwfKc/4a6wAtrBrZRSOToTLH4DRERkGvAtYC3wwM68iYiMBqYDrwFDjTEbvVObgKHe8xHAJzm3bfDS2krPf4+LRWSZiCyrqanZmeK1Znk1C2N2Lx+llOolOhMsUsYYg/uX/6+NMXcC5Z19AxEpAx4HrjLG1Oee8/Ltkt/Ixph7jDEzjTEzhwwZsnuZWX6tWSilVI7OBIsGEbkW+ArwrIhYgL8zmYuIHzdQ/NEY84SXvNlrXsI7bvHSq4FRObeP9NLaSi+eTM2ia+KYUkrt9ToTLM4C4sAFxphNuL+sb+noJhER4D5glTHm5zmnngIyI5oW0Dyy6ingPG9U1GFAnddc9XfgWBEZ4HVsH+ulFY8V0A5upZTK4evoAmPMJhF5HHeUEsBW4C+dyHsebm3kHRGp8tKuA34C/FlELgQ+Bs70zi0CTgTWABHgfO/9t4vID4E3vOt+YIzZ3on333WWH4zXZyFFfSellNordBgsROT/4Q5lHQiMxe1cvhv4bHv3GWNeou1fta3u9fovLm8jr4XAwo7K2mW0ZqGUUi10phnqctxaQj2AMWY1sE8xC9XjskNntc9CKaWgc8EibkzztnEi4qO3/xbVSXlKKdVCZ4LF/4nIdUBYRI4BHgWeLm6xepjt93bK690xUSmlOqszweIaoAZ4B7gEWGSM+V5RS9XTtGahlFItdNjBDXzDGHM78LtMgohc6aX1TnYQEtt1BrdSSnk6U7MotMrrV7u4HHsW8WnNQimlcrRZsxCRc4BzgTEi8lTOqXKguPMcepoV1NFQSimVo71mqKXARmAwcFtOegPwdjEL1eMs7bNQSqlcbQYLY8zHuDOs53RfcfYQ2Ul5WrNQSinoXJ9F32PrfhZKKZVLg0Uh4gcnoaOhlFLKo8GiECvgTcpTSikFnVtIcB5wI7C/d73grvt3QHGL1oNs7bNQSqlcnZmUdx9wNbAcSBe3OHsIXUhQKaVa6EywqDPG/LXoJdmTiLetquP0dEmUUmqP0Jlg8YKI3AI8gbtjHgDGmH8XrVQ9zQ64x+bFdpVSqk/rTLA41DvOzEkzwFFdX5w9hHhbjDvx9q9TSqk+ojPbqh7ZHQXZo1hezSKts7iVUgo6MXRWRCpE5Ocissx73CYiFd1RuB6TaYZyYj1bDqWU2kN0Zp7FQtz1oM70HvXA/xSzUD0u0wyV0mYopZSCzvVZjDXGfD7n9U0iUlWk8uwZLO3gVkqpXJ2pWURF5PDMC2+SXrR4RdoDWNrBrZRSuTpTs7gUeCCnn2IHhTdE6j20g1sppVrozGiot4BpItLPe11f9FL1NK1ZKKVUC52pWQB9JEhkZGoWjvZZKKUUFHHVWRFZKCJbROTdnLQbRaRaRKq8x4k5564VkTUi8r6IHJeTfryXtkZErilWeVvI1CzSWrNQSiko7hLlvweOL5D+C2NMpfdYBCAik4CzgYO9e+4SEVtEbOBO4ARgEnCOd21x6WgopZRqoTOT8r4oIuXe8/8SkSdEZEZH9xljlgDbO1mOU4GHjTFxY8xHwBpgtvdYY4z50BiTAB72ri0ubYZSSqkWOlOz+L4xpsEbPns07pLlv9mN9/y6iLztNVMN8NJGAJ/kXLPBS2srvRURuTgzy7ympmY3ikdOB7eOhlJKKehcsMjsYfE54B5jzLNAYBff7zfAWKAS2Ajctov5tGKMuccYM9MYM3PIkCG7l1m2ZqHBQimloHPBolpEfgucBSwSkWAn72vFGLPZGJM2xjjA73CbmQCqgVE5l4700tpKLy6tWSilVAud+aV/JvB34DhjTC0wEPj2rryZiAzPeXk6kBkp9RRwtogERWQMMA54HXgDGCciY0QkgNsJ/tSuvPdO0ZqFUkq10Jl5FsOBZ40xcRGZD0wFHujoJhF5CJgPDBaRDcANwHwRqcTdD2MdcAmAMWaFiPwZWAmkgMuNMWkvn6/jBisbWGiMWdH5j7eLMjULo8FCKaWgc8HicWCmiBwI3AM8CfwJOLG9m4wx5xRIvq+d628Gbi6QvghY1Ilydh2tWSilVAudaYZyjDEp4AzgV8aYb+PWNnov7bNQSqkWOhMskiJyDnAe8IyX5i9ekfYA2Ul5GiyUUgo6FyzOB+YANxtjPvI6oP9Q3GL1sJ1thko1Fa8sSim1B+gwWBhjVgL/CbwjIpOBDcaYnxa9ZD1JvK6czgSLVARim8GY4pZJKaV6UIcd3N4IqPtxRy8JMEpEFnjLefROIu7WqibV8bXxre6Cg8YBsYtfNqWU6gGdGQ11G3CsMeZ9ABEZDzwEHFLMgvU4y9/x2lCpqFuzEAtMGnd0r1JK9T6d6bPwZwIFgDHmA3p7Bze4/RZOBzWLeA1YQdwKl9MdpVJKqR7RmZrFMhG5F3jQe/0lYFnxirSHsHzt91lkaxU+SMW9moVSSvVOnQkWXwMuB67wXr+Iu8dE72YF2u+ziNe4geLF06B0DMz7U7cVTSmlultn9uCOAz/3HgCIyMvAvCKWq+e112fhpCAdg0//CnUr3FpFOtEXGueUUn3Uru6Ut1+XlmJPlKlZFBwSa9zgsOpW92VsS+dGTiml1F5qV4NF759UYAXa7rMwDnz8CEQ3wKDDILEdkg3dWz6llOpGbTZDicgZbZ0CwsUpzh5E/F6wMLgfOUeiDtbcBYPnwsgzYNurENsI/Q7siZIqpVTRtddncXI7555p51zvYLczdPaDX7m1iUnXQLLeTYsUf08mpZTqKW0GC2PM+d1ZkD2O+N2FBI1pVbHg44dgyOEw8BCoW+mmxTZ7s7h3tWVPKaX2XPqbrS3ZPosC3TOxGigb6z4PDPTSNutcC6VUr6XBoi2Wv3AHt5OEVD34+7uv7bAbWDRYKKV6sV0KFiJyaFcXZI+TnZSXV7NI1LrHQIU7izvYH0LDILbJbYZSSqleaFdrFo92aSn2RG3VLBLb3aN/AJCC4GAvWNRozUIp1WvtarDI7/LtfTJ9FvmT8uLb3KMvDMEhblAp2ddthupo4UGllNpL6aS8tljBwtuqZmoWwUHgr3Cfh0e4a0WlO1jSXCml9lLtTcp7msJBQYBBRStRD4un4jQlmhho+72aQn7NwgsWoX3A8vavCI+AdBSS24Gh3VlcpZTqFu1Nyrt1F8/t1RzjEE1F2x46m6lZBAY3p5WMdI+RDVBxULeUUymlulN7k/L+r61zItKrV5yNpWIY8SFOoo0+C2meXwFQMsI9Rj7ttjIqpVR3arPPQkRsETlHRP5TRCZ7aSeJyFLg1x1lLCILRWSLiLybkzZQRJ4TkdXecYCXLiJyh4isEZG3RWRGzj0LvOtXi8iC3fq0nZR20jjZPbjzg8VW8PdzO7YzSka5x9hGHT6rlOqV2uvgvg+4CLd/4g4ReRC3+elnxpjpncj798DxeWnXAM8bY8YBz3uvAU4AxnmPi4HfgBtcgBuAQ4HZwA2ZAFMMnzZ8yn/8/j9YtGYRjlhtN0P5+7s76WVkahaZJT+UUqqXaa/PYiYw1RjjiEgI2ASMNcZs60zGxpglIjI6L/lUYL73/H7gX8B3vfQHjDEGeFVE+ovIcO/a54wx2wFE5DncAPRQZ8qwswaXDObNTW8ydehU0iMH4zcpcPJ++Se2uxPyxG5O85W4I6Oim7y5Fp3ZgFAppfYe7dUsEsa4fyYbY2LAh50NFO0YaozZ6D3fRPPQoRHAJznXbfDS2kovioAdYOKgiazetppUZiqJyRsOG9/uBob8BQNDwyC+RWsWSqleqb1gMdHrP3hbRN7Jef2OiLy9u2/s1SK6bL6GiFwsIstEZFlNTc0u5zN5n8l8sP0DEpmS5W+tmtjh9lnkB4vwcG/HPJ3FrZTqfdprLynGGNDNIjLcGLPRa2ba4qVXA6NyrhvppVXT3GyVSf9XoYyNMfcA9wDMnDlzl4PQ5H0m8/CKh9majDMQIB1veUHCq1nkx9mSEe5y5RoslFK9UJs1C2PMx+09dvH9ngIyI5oWAE/mpJ/njYo6DKjzmqv+DhwrIgO8ju1jvbSimbzPZABWNnrzKXJrFk4KknUQGFCgZrEvJLa5k/OUUqqXaW8GdwNtz+A2xph+7WUsIg/h1goGi8gG3FFNPwH+LCIXAh8DZ3qXLwJOBNYAEeB83DfZLiI/BN7wrvtBprO7WLLBomEbp0HLmkVmxVl//wLBYqRbq4hugvCwYhZRKaW6XXuT8sp3J2NjzDltnPpsgWsNcHkb+SwEFu5OWXbGfhX7URGsYGX9FvDTsmaRnb3dv3WwyJ3FPbCyG0qqlFLdRzc/yuO3/UwcPJFV9ZvdhNxlyjMrzgYKTPXIBIuo7sWtlOp9NFgUcPCQg3mvbiOOAZzcZqicmkW+7MS8Ta2XCFFKqb2cBosCpg2bRiSd4KMkmHSBYBEc0vqm0FDAguhGHRGllOp1NFgUMH2Yu5rJ2wlIpyLNJ7LLkw9ufZPlg9AQiOpe3Eqp3keDRQFTh05FEN6OQyp3KGxiO+6KswWCBbjDZ+M6MU8p1ftosCigX7Afo8uHusEimVuz8FactYOFbwyP8PosNFgopXoXDRYFiAiTBo7h7QQk8puh/HmLCOYq3d/ts0jFC59XSqm9lAaLNkwdPJ61SdgRzVk7MbGt8ByLjNL93Rnc8c3dUkallOouGizaMG3IJAywqraaZNqba5HYUXjF2YzSMe6xcV13FFEppbqNBos2TBs6BYBVdZ+SSHuzuOPbCi8imFE22j027erSWUoptWfSYNGGcQMPZIAFb+7YQCTTyZ3Y4W181FbNYrR7jH6i+1oopXoVDRZtsH1h5ofh5W0f05BoACcNydr2m6GCg8AugUi1johSSvUqGizaIn6OKoH10To+2vERqfhWN73Q8uTZewRKRrnrQ2mwUEr1Ihos2mIHOCrsPn31k1dIRr0RToUWEcxVuh9EP9VgoZTqVTRYtMUKcFAAhgTCvF79CvGmT910f//278vMtXBSRS+iUkp1Fw0WbRE/IvCZAUN5ZcNrxDLBosOaxWh3N73EjqIXUSmluosGi7ZYfgjvy9GlNjXRbXywdYWbXmgRwVzZuRYfFbd8SinVjTRYtEUE9juLYx33l/4rm95x0wMFlifPVabBQinV+2iwaM/+Z3OAz2FkqJyXa1a7aaEOgkXp/u4xsr64ZVNKqW6kwSJPOmlINHkT6vpPRfqN56hSHy9tryZpl4Idaj+D8DCwAu5e3I6OiFJK9Q4aLPI4aUOswQsWYsGIkznGt4MdqSRVyTDxdLL9DMRylyrXuRZKqV5Eg0UOp9Eh9rcIqY9TOGnjdnKPPI0jS9zzz0ehLtHQcUal++tcC6VUr6LBIocEhdSKJOaTFOmkcTu5+09h331mMT0If9zRRG20nnRHzUul++le3EqpXkWDRQ7xC9ZQG2rSpOPGTfSVISNP5fIKeDca5fWNy2lKNLWfUeloiNdAsp3rnBSkmiC6BTKr2iql1B5Kg0Uee6SNbHeIN3r9FpYNo87knIoAA3wB/vTuw2yPbm8/k8zqs01tDJ+NboHGtdBU7U7ey6w7pZRSe6geCRYisk5E3hGRKhFZ5qUNFJHnRGS1dxzgpYuI3CEia0TkbRGZUcyy+Ub6EQec6jTGeLWLslH45jzAF8Ydx+IPn2Nd7TpiqVjbmZQd4B4bP2x9zklCcgfYZWASENsMyXpIt5OfUkr1sJ6sWRxpjKk0xsz0Xl8DPG+MGQc8770GOAEY5z0uBn5TzEL59nf31zabU2QHPtkhAkMO5atTz8Ng+PPKP1Mfr287k+xciwKbIDWsgXf/G144BhZNgefnQ/0qiG1rfa1SSu0h9qRmqFOB+73n9wOn5aQ/YFyvAv1FZHixCuErt6G/BZsd0omcDYwCg5g2dDJHjT6KR1Y8wqaGTc2bIuUrGQFY7lyLTO0E3A2Rll8F6x+G4BA46DsQGAgrb4FUg9YulFJ7rJ4KFgZYLCLLReRiL22oMWaj93wTMNR7PgL4JOfeDV5aCyJysYgsE5FlNTU1u1U4a4QNW9PNk/MA/KWEwiNZMG0B26Pbee7D56iuz9mfu0UG7rpSRDZAOiegbHsTNj0H4y6DeQ/BhCth4jdh2yuw5UWtXSil9lg9FSwON8bMwG1iulxEjsg9adzOAlPwzjYYY+4xxsw0xswcMqSDJTk64N/fhyQhtbF56Gus0dC4PcTJ409m0uBJ3PzizXxS/wnV9dU4hbZQLR3lDp+NVDfXGFb9FOwgHHBB83X7nQWlB8Cqn7md3alo+4VLNkJ0M6QiunWrUqrb9EiwMMZUe8ctwF+A2cDmTPOSd9ziXV4NjMq5faSXVjSB0X4AnE1p0klDMurQtC1NKuFQYpdxxwl3APD/nv5/1ERq2NSwqXUNo99BUPsO1K2EyCdQuwI2/C/sfy74S91f+qlGd8b3xG9C4xqofqrtvTCMgdhWN79EnZtnwxq3NmJ2Kq4qpdRO6/ZgISKlIlKeeQ4cC7wLPAUs8C5bADzpPX8KOM8bFXUYUJfTXFUU9kALSgQ2OyQjhoaPk1jPxZDVKdJxmDF8BneeeCcb6jdw5d+upDZWy0e1H7GlaQvxVNwdRTX1BxDaB149HxrWwapbAAfGXuT+ci/bD8oPhLL9YfjxMHAmvP9Lt3YR3dwyADgpaNoA7/4QFh8Gz82Bt2+AmqUQ3QSxLRowlFJF1RM1i6HASyLyFvA68Kwx5m/AT4BjRGQ1cLT3GmAR8CGwBvgdcFmxCygiWCNspCZNZHMKeT4Gm9LIigTxBofyYDkz953Jz475Ga9seIWLnr6Itza9RX2snnW161izfQ0fx5NsnXorjuXDefU8zPpHSY84BcffH4KD3QUJxXIfJcNh0jXupknLvwGJ7W7QMA7Ed8CON+HV8+CDO2Dfz8E+8+HTv8Jr58Oyy9y+kdgmbZZSShWNmF74F+nMmTPNsmXLdiuPpqVREs/HYKAFtQ5yoB/zQRJzfJj+04PETYz1det5YtUT3PbKbWyLbmPa0Gl8YdIXGD9wPGMGjmGgE8Vf/z4j3vkmdrqRddMXkijZD6tsFAFfiKAdJGAH8Fk+JLYZ3/pHCb1zHckRpxGZcC0pBHvLPylbczt2vIatB15F3fBTcTBYJk3/TX9l0JrbSJWNJz7jdnxl+xMsGYn4S7roJ6mU6ktEZHnOdIaW5zRYFJaoTtG00F000P5sGBnlI/WHBsyBfspOKiFQYtEQb6C6vhrbtvnLqr/w2+W/5cMdzRPxwr4ww0sHMzxUxqhAkOGDpzFqwHj2HTCWfUr3YUjpEEK+kNtBnk5CdAMDP3mYwet/T82wU6it/YC62veo9g3hk6GnUmv3pyleR8D2U+ILEfaXMNrZwZxP76UiPJCNE68nHh5NaXgA5WX7EfKX4LcDSKYGg3cU2a2fjVKqd9JgsQuMMdT+sRFrPx/2xADxiIP8XxTZ4uD7Sinlw9xO8LpYHZ82fErYH8Zv+dnYuJE129ewdvta1tWtY1PDp2yqX0914xY2NdVg8gZ5lfpLCfvDBO0gfsuiKdFIU7yeyE7uhTHEFmYHDfOGHMC08ecwZthcLEvwi4+yQCkl/jDBTC3GDoBd0twUhrgBxBh38cNMc5blc89ZPhC/u/SJUqrX0mCxixq3pklFDemUIVgqOB8lSS+OYY4KMWBOCLHcv9CbEk1sjWwllorhs3z4bT8+y9ecUSoK0WoSvoF8Gq9nQ/0GNjVuYlPjJmoiNcRSMeKpOCknRYllU2bBgHQdAwdNY0jZcAYHS6kIVtCv3xjKwoNJOkkiyQiNiUY+3r6a1Vve5INtq1le/RprInUAVFjC/PIwn62o4PCKwRxQNgTHX44THIIJ74ddth/+8FAssRDLj+WksZI7kMQOxEmAHXYfgf5I6X6I5UcsP/hKvEATAPGB2FpTUaqX0GCxixIRh7oNKUoGWpQMsklFDA2/roMRPso/X4o/3HJ8QCwVoz5eTyQRIeEkMMYgIu6MESeBZQexLBtbbGzLxpI2xhdEt0FyB0ZsHCdB2tcfx1+GEatFzUQQRAQfBjtWg5gUWxo38++V9/Li5lU8v2Mzn8TdOR4DbYs5YYuZgRQTAjDRD2P8UNGJVinHChIvHUu89EASZQeSLBtHuuwAfL4wfsuHzw7h84Ww7RCWFcC2g1iWH9v2N9dckJxajDaFKbUnai9Y+AolKpcvIJQPswmWW4gI/lJBRvswH6aI1zmtgkXIFyLkC0Gp24yVclKkTRrHOKSdNIl0gkQ6QdJJkkglSBlvPoXB/X2Ke59YAQx+LJPGFx6BzxcmaPnweQ8AxzjZfGOpGHFff5z4dsKBEv7jkG9zjK+MH1k2H9at542NVbyx6S3e2FTFou3rWwQcn1gMCoTp7w8T8gUJeJ8hIBYBgQAOJU6E0u1bKEt9QIWkqLDcmsuAUAX9S4ZQUbIPA8JDKA0NJh3oj+MrJW2HMXYplq8Eyy7B5y/B9pVg+8vx2X4ssbAsN6hYdhDLDmHZQcTK1FZsrbUotQfRYNEOyyeE+rVspw9ODRD/IEViZYJkhdUqYGSICH7bjx9/m/kbY3CMg8GQW8OzxA1ObdY82sgr5aRIJJuIRDbR0PgpSSfBUH+YU8d8hi+OPRJbbKLpGOvqP2VN3Sd82lTD9mgt22O11MUbiKXjxFJxouk4DekkiXSKhJMklooTSztEkn4iqcyEQQPUeo/VAPiBfXwwxIZ9bPc4xIbB3mOQBYNs6O/zUxEsp19wAKHgANLBQSQDg0gHBpMODsYE94HwUKzAQGw7gG2F8PmC2RqLZQcQ8WFZPiyxsSwfYtloB75SxaPBYieFxvuJlQryZoL6/hZlE/wES3et41dEsKVrOo2zwcnuT2moP0MGTCCVThBPRWlKNNCUbCKSiiF2GaMGD2T0gInYlsEWywtOdnMfRG5/i3EgHQcnCk6StJOmIRmhPh5hR7KR7bF6tsfr2RbZzrbIFrY2bWJ7dDvborW8F6tlezRCYyqeV9oksB3Yjl+E/hYMsAwDvDUcB1jQ34b+tlDhC1EeKKUiUEZ5oJR+gX6UBcspD/ajPNgf21+GY5dh/KUYXzkSGIAEBmIHB2DbIWw7gM8OY9kBLMuPWLYXZNxgI2J7gUYp1R4NFjvJsi0CJ5eQfDKCtThKY9zAlADBsj3sF44IPl8Qny9Iaag/QLYpLOWkiKfixNNxEulEtqks2xzmeEcDYLkd3VbYPZFO4vNVMDCQYDBJxEkhJoUl4j6wsSyrRdNaPJVge6yOHfF6auMN7IjXUxdvoDbeQF2i0T3G66mLbWdLrJY1iQZqo03UJWOkTBSIAoU3iCoRN8D09wJNfwsG2G4zWYU/SD9/mH6BEsoC5ZQHyikL9ac8OICy0EBKw4PxBSowvjIIDMQKDMAOVGDbYS/IBLFtf3MtxgsyYlnZYKO1GNVXaLDYBeHRfhLHh5EXYlj/jNHY5JCaHqJkgJUdIbUnsi2bsBUGoDxY3uKcMSbbHJZ/BFqkOcZp7pNx0qTSSdJOglQ6ScJxg5EbLQwYg/EbygMDKTcO+xkHIbN0QBoLBzFpLOMgGCzEnRfivWc0FacuEaEu0UB9oom6RBN1iQYaYvXUx3ZQF99BQ6yOungddfF6qhNNrExGqIvFqE/GSRMDdrT5Mwl7wabCCzgVFlTYNhU+P/18Afr5Q17AKaU8UEZZsB+lgX6Uh7yAExqMCfbH8g3ACnm1Gl8ZPjuA7QsiWNnmMrHs5mYzsb35L3vuvxelcmmw2AW2Xyg7wE+jX5AlMaxXEsS3OCQOC1I+wo8vuPf9AhARBMnWBnZXpgM+88j0z2Qemc7/tOM9TJqUk8JxEjhO2pvvkQYpodxfTnnJYHAcwJ0HIhjEGMQrcrafxx0j5o5CAyLJCLWJJhoSUeqTbi2mMbqd+tg2GmLbaYjX0eDVdOqTTdQkoqxJxqiPxalNNZA07WxyBdg0B5rmoCOU2z765QSccn+Yci/olAXKKQuUUR7sT7/QQELBAeAvxwpUIP4KrEB/7MAA8PfDCpS7gUYytRkbERvLsmk5wmzv+zen9i4aLHZRoMSi3xg/DUGgKoG8k4StDnWHpvHt5yNcYeMPyR5d0ygmy+sL2RW5tZrMAIDcgGMwpB236SyVTuIYt3bj4B6NkyblJAEH7DIqgmkqHMcNPqQxThIxxg064mSDS+ZoYblByAhxJ0ljoon6ZBN18UYaEk3Ux7a7gSa2g/p4LQ3xeuoTjdQnGmlIRnk/EaU+Fqc+laA+3cYGWTmC4tVoch79Mkdb6Ofz0c8OUOYL0s8foMwXpswfytZ2ygPllAX6YQXKEF8p4itzH/5+4CtB/OVY/n7gK8PylyOZox3IG9osrZ9rEFIeDRa7wRcQKkb6aQpbJIbaWEvjWItjOH5o3MdG9vcRnhUgWG5n/9JVHcut5djsel9QfrNZocDjOGkv2CS8YwqHNOl0EsdJAQ4lwRQlxmGocRDTXLvJznY3Brx+mxZBRyycdJqmVJSmRJT6VBMNiUbq4000JBpoiNXSmKilIVZHfaLBTUs0UZ+I8GkySn0yRn0kQUM6iTsooKntnxlQnhtkch7lea8zaeWWRZkvQJk/SLkvSLk/RLkvTMBfgtghsELgC4MdQrxJmmKH3QCUPTY/xw6711tBb1JnKOfoPRc/WDkByv3C0cC059NgsZssn1C2j00sKEQGWthbHWRTGqc6DUvjRFYmiR8epPTgIL6A/mfoTplf3Ltaw8nIDzpt1nScNGknRdrxajtOCpw0JekEIRwGmLS3lLzBOGkExws2Dm7/jtfEJgYxmV+nbv6RVIymRBONiSiNySYakxEaEhEako00JiI0JJpoSNTTmGikMdFAQ6KRrYkoHyUjNCSj1CejNKUSeZ/MAWLeo5lfhHJLKLMsL6hAuWUoF4d+YuhnQ7m46WXZ81CWk1aWkxbM+fEbLLAC7goAVv4jiNhB77nfDTqZYzY9mHN/obRA83125v6ctOx7+d3AlblG/DnvG/CWtskNZkqDRRcQEcL9bXxBoTGcxhnhgxkGsz6NtTyO80yU+qoEMt5PYLyfQJmFL9h3m6j2Nl0ddPLn1xSq8aScVHP/TjpJwKTp5wUjx6S83RlNNvg0Pwdj0m4zmxhwDJaAGINDmkgyQiTRRCQRoTHVQGM8QiQZoSkVdQNNMkJTMkJDsonGRISmZJTGZIStyQjrkhH3fDxCYzJaeIfIAvxiUWr7KbV9lNo2pbZNmWVTaluUWQ6lVpwyK0Gp1egGGjGUWYYygTJxKJMUpeJQRopy0pSSpExSFPdvr9w10TITRTPPfc3H3OfZ6zLn7ObAk11bzdf8OjNMPXOP5a14YPlzJqXm5pv/8HnX+5qfi+1ugTB0fpf/RDRYdCF/2KL/KMGk3b5YZ6iPpv1snLeTWB8kYUOMxJIY8VE+5CA/pQf5CZTsYUNuVdF0VfNaRqGaTv7zzKi1TOApy3meWQHAwcFxHG/YtMHgjljDuNUbYxzE4AYfLw3HEE/HiaaiNCXdgBNJNtGUiBBJNhJJRt0glGgikop4a5k1EUlFiXiBaHsyQiSZuT9KJBXrdAAC8IlN2BekxBck7AsStgOU2AHCPr97tH2UZIOUj7Dtp8Syc4KWj1LLosQLXiWWRaktlFhCqbhHGwecFEIKyQ68SCHZY8pNc1JgUuAkwcS812kvzW3OzF6Tbb5M5TRnpprTdlf/aXDCm11eI9Jg0cVExP1jACAg+Pf3Ex9sE53mx/k0jf1JGtYl4aMUja/EsQ/wIQmDs8OBJAQ/EyR8ULCnP4baC1hiddnoNSjc3JZfC8pNS+UFnhYByDgtg1Bm3k3OMbPsTCaAGscQd+JEk1GiqSjRZJSmZJP7OhnJpmWeR5JRol4ginrBKZKMEEnF2BaPEMlc712bNju3kjNAwA4QsoOEfCHCPvcYsoMEfQHCvhAhO+CluefD/pAXwNxlc8J2yAtmYcL+IGE7SEkgRIkdJuQLELZDhL28S+wAftuPhXGDkzEIXj+ZyTRb5veX5QSaTFOmXZz9bDRYFJlYQqjcJlBikRhkEx3p4EwPYK1PIe8ncZYnIACm3IK4IfZYhMTkJGXHl2C3sZSIUsXQVc1t+QrN3SkUgHLPdeZhjOkwIOUGpmQ6mQ04sXSMWKrlI5qMurUlL8BkJq7mXh9NRrPX1yYiRFPbiCW9c6lodqLrrrLEctdmswME7aD78AWzr0O+TFoo+7w5LUjQDjF6wGgu3/e4rvjqWtBg0U0s2w0awTKLVMyQHGwTH+/DRIEA+EIW/gDEXojhvJukbn09vvF+fKN8+PfzYZeJOzNaqb1MV8/hKaS9SaWdCUq5gSdTuyp03hhvomqBYJRZYTpt0sRTcTfAJGPE027QiSVjxNLu60QqkQ0smS0KWrz2romlY+4CpDnPGxKNxFKx7MKkmefxtLvNQeWwSi6bdXmXj8DUYNHNRAR/WPCHoWSgjZMy3lbc7hcbPL2UxgPipF6Jk1qeIL0sQRzcf5R+ICBYAyzsfWx8Q218w33Yg21ER1qpPqw7AlKutoJQ/jmg1XVAq0CUH6Byr2kR4PICljGmuRnPGDfYSXG2ndBg0cMsX8t/3ZYtlE8PEj/QT7QmjdmShu0OJAwmAcQdnHpD+tMEydwm2H6CFZLswBj8ghUWJCzYg2x8I3zY+9pYJVo7UWp3dXdwKqRQgMooxrwuDRZ7IBF3afRQPxtnf0MqYbJV3szup+m4Q2qbQ2pjGme7A7VpnCTuKt0CpCFd68BmAyuSzf+mQ4KUuEFE/DnBRdzRd9iCBMEqtbBKLfdan4Af95hZXcL20oKCBMQ9+rR2o1R3yQYs6JagpcFiD2f5hEChX8IlFgwADgQnbXDnfzU3aWWCCgbSUYf4x0nS1WmcOgfiBuIGiXltr5Z7nXiTk0kaiBn3+U4VFggI4m0t4c4qk+bJubYXcHwgfjdYSUDcBZZsLwC1WH3CO2d55zLXWd5z8Z5n7rFyrrXy8rRaXyuSd29uflbONUopDRa9gWWLO9+njT8vfEGbYH8bpoGTMnjLImEck/0Fmf87MZ1yMDFINTqYuINJgEl69zi4j6TXNJY0zc9Txh3F5+Atg9HcF4gDJmXcoeQJB1Lu9aS96zN558wzo/PD7osnL6C4s9xosaxSq7Sca7NLLrXYYVZaBimRtq/Jf1g51+c9WqRb7Zxr777cB+2cp4M8KPA+tJMPeXnRRl55z7PvWyCfQs9bNB+1U4bcNP2jQYNFn9Oyj6Tt/wC234YwBAd0/aRB4wWRTM3HZB6O19xGdjKye21m/pIXXDIBy12Y1ssrUyvK5JP2XucGt0zwMc1BybQITCbnmpbXmtzzuYHMkA2K2XLkXmNM3nvk5EWB4Gha51nwkfkhZYJpW9eo4igUYHKPHZ6TVmlS6Lq28mwrP8AeZlN+ZlkbBd91GixUt8v8Jdg6VPWNv95yt9DND46tXtPWedPu9eAFTcip6TU/shOl0zkB2snJ08mpDeYFLlMoMGXTc64l/z1zCplbY8zcl3t9/nPvteTek380OZe3ute0yqvzx5Y/64LX5V6W9920Ctw7m97qffL/cbR8ni4tzv+jvSZYiMjxwO24rdX3GmN+0sNFUmqXtGjSaP0HZmdz6aLSKNU5e8U4ShGxgTuBE4BJwDkiMqlnS6WUUn3HXhEsgNnAGmPMh8aYBPAwcGoPl0kppfqMvSVYjAA+yXm9wUvLEpGLRWSZiCyrqanp1sIppVRvt7cEiw4ZY+4xxsw0xswcMmRITxdHKaV6lb0lWFQDo3Jej/TSlFJKdYO9JVi8AYwTkTEiEgDOBp7q4TIppVSfsVcMnTXGpETk68DfcYfOLjTGrOjhYimlVJ+xVwQLAGPMImBRT5dDKaX6IjGFpn7u5USkBvh4N7IYDGztouLsLfriZ4a++bn74meGvvm5d/Yz72+MKThCqFcGi90lIsuMMTN7uhzdqS9+Zuibn7svfmbom5+7Kz/z3tLBrZRSqgdpsFBKKdUhDRaF3dPTBegBffEzQ9/83H3xM0Pf/Nxd9pm1z0IppVSHtGahlFKqQxoslFJKdUiDRQ4ROV5E3heRNSJyTU+Xp1hEZJSIvCAiK0VkhYhc6aUPFJHnRGS1dxzQ02XtaiJii8ibIvKM93qMiLzmfeePeMvJ9Coi0l9EHhOR90RklYjM6e3ftYhc7f3bfldEHhKRUG/8rkVkoYhsEZF3c9IKfrfiusP7/G+LyIydeS8NFp4+tsFSCviWMWYScBhwufdZrwGeN8aMA573Xvc2VwKrcl7/FPiFMeZAYAdwYY+UqrhuB/5mjJkITMP9/L32uxaREcAVwExjzGTcJYLOpnd+178Hjs9La+u7PQEY5z0uBn6zM2+kwaJZn9lgyRiz0Rjzb+95A+4vjxG4n/d+77L7gdN6pIBFIiIjgc8B93qvBTgKeMy7pDd+5grgCOA+AGNMwhhTSy//rnGXMgqLiA8oATbSC79rY8wSYHteclvf7anAA8b1KtBfRIZ39r00WDTrcIOl3khERgPTgdeAocaYjd6pTcDQnipXkfwS+A7geK8HAbXGmJT3ujd+52OAGuB/vOa3e0WklF78XRtjqoFbgfW4QaIOWE7v/64z2vpud+t3nAaLPkxEyoDHgauMMfW554w7prrXjKsWkZOALcaY5T1dlm7mA2YAvzHGTAeayGty6oXf9QDcv6LHAPsCpbRuqukTuvK71WDRrE9tsCQiftxA8UdjzBNe8uZMtdQ7bump8hXBPOAUEVmH28R4FG5bfn+vqQJ653e+AdhgjHnNe/0YbvDozd/10cBHxpgaY0wSeAL3++/t33VGW9/tbv2O02DRrM9ssOS11d8HrDLG/Dzn1FPAAu/5AuDJ7i5bsRhjrjXGjDTGjMb9bv9pjPkS8ALwBe+yXvWZAYwxm4BPRGSCl/RZYCW9+LvGbX46TERKvH/rmc/cq7/rHG19t08B53mjog4D6nKaqzqkM7hziMiJuO3amQ2Wbu7ZEhWHiBwOvAi8Q3P7/XW4/RZ/BvbDXeL9TGNMfufZXk9E5gP/aYw5SUQOwK1pDATeBL5sjIn3YPG6nIhU4nbqB4APgfNx/1Dstd+1iNwEnIU78u9N4CLc9vle9V2LyEPAfNylyDcDNwD/S4Hv1gucv8ZtkosA5xtjlnX6vTRYKKWU6og2QymllOqQBgullFId0mChlFKqQxoslFJKdUiDhVJKqQ5psFCqABFZ6h1Hi8i5XZz3dYXeS6k9mQ6dVaoduXMyduIeX84aRIXONxpjyrqgeEp1G61ZKFWAiDR6T38CfEZEqrw9EmwRuUVE3vD2BLjEu36+iLwoIk/hzhZGRP5XRJZ7+ypc7KX9BHc11CoR+WPue3kza2/x9mB4R0TOysn7Xzl7UvzRm2CFiPxE3H1J3haRW7vzZ6T6Fl/HlyjVp11DTs3C+6VfZ4yZJSJB4GURWexdOwOYbIz5yHt9gTdzNgy8ISKPG2OuEZGvG2MqC7zXGUAl7p4Tg717lnjnpgMHA58CLwPzRGQVcDow0RhjRKR/1350pZppzUKpnXMs7vo6VbjLowzC3UwG4PWcQAFwhYi8BbyKu4DbONp3OPCQMSZtjNkM/B8wKyfvDcYYB6gCRuMuvR0D7hORM3CXcFCqKDRYKLVzBPiGMabSe4wxxmRqFk3Zi9y+jqOBOcaYabhrEYV2431z1zBKA5l+kdm4K8meBPxtN/JXql0aLJRqXwNQnvP678DXvCXeEZHx3mZC+SqAHcaYiIhMxN2+NiOZuT/Pi8BZXr/IENwd7l5vq2DefiQVxphFwNW4zVdKFYX2WSjVvreBtNec9HvcPTBGA//2OplrKLw959+AS71+hfdxm6Iy7gHeFpF/e8ukZ/wFmAO8hbthzXeMMZu8YFNIOfCkiIRwazzf3KVPqFQn6NBZpZRSHdJmKKWUUh3SYKGUUqpDGiyUUkp1SIOFUkqpDmmwUEop1SENFkoppTqkwUIppVSH/j/ua6Br5JxuEAAAAABJRU5ErkJggg==\n", 97 | "text/plain": [ 98 | "
" 99 | ] 100 | }, 101 | "metadata": { 102 | "needs_background": "light" 103 | }, 104 | "output_type": "display_data" 105 | } 106 | ], 107 | "source": [ 108 | "# IRL Loss on test trajectories, as a function of cost function updates\n", 109 | "\n", 110 | "plt.figure()\n", 111 | "weighted_trace = weighted['irl_loss_test'].detach()\n", 112 | "w_mean = weighted_trace.mean(dim=-1)\n", 113 | "w_std = weighted_trace.std(dim=-1)\n", 114 | "timedep_trace = timedep['irl_loss_test'].detach()\n", 115 | "t_mean = timedep_trace.mean(dim=-1)\n", 116 | "t_std = timedep_trace.std(dim=-1)\n", 117 | "rbf_trace = rbf['irl_loss_test'].detach()\n", 118 | "r_mean = rbf_trace.mean(dim=-1)\n", 119 | "r_std = rbf_trace.std(dim=-1)\n", 120 | "plt.plot(w_mean, color='orange', label=\"Weighted Ours\")\n", 121 | "plt.fill_between(np.arange(len(w_mean)), w_mean - w_std, w_mean + w_std, color='orange', alpha=0.1)\n", 122 | "plt.plot(t_mean, color='green', label=\"Time Dep Weighted Ours\")\n", 123 | "plt.fill_between(np.arange(len(t_mean)), t_mean - t_std, t_mean + t_std, color='green', alpha=0.1)\n", 124 | "plt.plot(r_mean, color='violet', label=\"RBF Weighted Ours\")\n", 125 | "plt.fill_between(np.arange(len(r_mean)), r_mean - r_std, r_mean + r_std, color='blueviolet', alpha=0.1)\n", 126 | "plt.xlabel(\"iterations\")\n", 127 | "plt.ylabel(\"IRL Loss on test\")\n", 128 | "plt.legend()\n", 129 | "\n", 130 | "plt.savefig(f\"{model_data_dir}/{experiment_type}_IRL_loss_test.png\")\n", 131 | "plt.show()" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [] 140 | } 141 | ], 142 | "metadata": { 143 | "kernelspec": { 144 | "display_name": "Python 3", 145 | "language": "python", 146 | "name": "python3" 147 | }, 148 | "language_info": { 149 | "codemirror_mode": { 150 | "name": "ipython", 151 | "version": 3 152 | }, 153 | "file_extension": ".py", 154 | "mimetype": "text/x-python", 155 | "name": "python", 156 | "nbconvert_exporter": "python", 157 | "pygments_lexer": "ipython3", 158 | "version": "3.7.9" 159 | } 160 | }, 161 | "nbformat": 4, 162 | "nbformat_minor": 4 163 | } 164 | -------------------------------------------------------------------------------- /mbirl/experiments/run_model_based_irl.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import random 3 | import os 4 | import torch 5 | import numpy as np 6 | import higher 7 | import mbirl 8 | import matplotlib.pyplot as plt 9 | 10 | from differentiable_robot_model import DifferentiableRobotModel 11 | 12 | from mbirl.learnable_costs import LearnableWeightedCost, LearnableTimeDepWeightedCost, LearnableRBFWeightedCost 13 | from mbirl.keypoint_mpc import GroundTruthKeypointMPCWrapper 14 | 15 | EXP_FOLDER = os.path.join(mbirl.__path__[0], "experiments") 16 | traj_data_dir = os.path.join(EXP_FOLDER, 'traj_data') 17 | model_data_dir = os.path.join(EXP_FOLDER, 'model_data') 18 | 19 | 20 | # The IRL Loss, the learning objective for the learnable cost functions. 21 | # The IRL loss measures the distance between the demonstrated trajectory and predicted trajectory 22 | class IRLLoss(object): 23 | def __call__(self, pred_traj, target_traj): 24 | loss = ((pred_traj[:, -6:] - target_traj[:, -6:]) ** 2).sum(dim=0) 25 | return loss.mean() 26 | 27 | 28 | def evaluate_action_optimization(learned_cost, robot_model, irl_loss_fn, trajs, n_inner_iter, action_lr=0.001): 29 | # np.random.seed(cfg.random_seed) 30 | # torch.manual_seed(cfg.random_seed) 31 | 32 | eval_costs = [] 33 | for i, traj in enumerate(trajs): 34 | 35 | traj_len = len(traj['desired_keypoints']) 36 | start_pose = traj['start_joint_config'].squeeze() 37 | expert_demo = traj['desired_keypoints'].reshape(traj_len, -1) 38 | expert_demo = torch.Tensor(expert_demo) 39 | time_horizon, n_keypt_dim = expert_demo.shape 40 | 41 | keypoint_mpc_wrapper = GroundTruthKeypointMPCWrapper(robot_model, time_horizon=time_horizon - 1, n_keypt_dim=n_keypt_dim) 42 | action_optimizer = torch.optim.SGD(keypoint_mpc_wrapper.parameters(), lr=action_lr) 43 | 44 | for i in range(n_inner_iter): 45 | action_optimizer.zero_grad() 46 | 47 | pred_traj = keypoint_mpc_wrapper.roll_out(start_pose.clone()) 48 | # use the learned loss to update the action sequence 49 | learned_cost_val = learned_cost(pred_traj, expert_demo[-1]) 50 | learned_cost_val.backward(retain_graph=True) 51 | action_optimizer.step() 52 | 53 | # Actually take the next step after optimizing the action 54 | pred_state_traj_new = keypoint_mpc_wrapper.roll_out(start_pose.clone()) 55 | eval_costs.append(irl_loss_fn(pred_state_traj_new, expert_demo).mean()) 56 | 57 | return torch.stack(eval_costs).detach() 58 | 59 | 60 | # Helper function for the irl learning loop 61 | def irl_training(learnable_cost, robot_model, irl_loss_fn, train_trajs, test_trajs, n_outer_iter, n_inner_iter, 62 | data_type, cost_type, cost_lr=1e-2, action_lr=1e-3): 63 | irl_loss_on_train = [] 64 | irl_loss_on_test = [] 65 | 66 | learnable_cost_opt = torch.optim.Adam(learnable_cost.parameters(), lr=cost_lr) 67 | 68 | irl_loss_dems = [] 69 | # initial loss before training 70 | 71 | plots_dir = os.path.join(model_data_dir, data_type, cost_type) 72 | 73 | if not os.path.exists(plots_dir): 74 | os.makedirs(plots_dir) 75 | 76 | for demo_i in range(len(train_trajs)): 77 | expert_demo_dict = train_trajs[demo_i] 78 | 79 | start_pose = expert_demo_dict['start_joint_config'].squeeze() 80 | expert_demo = expert_demo_dict['desired_keypoints'].reshape(traj_len, -1) 81 | expert_demo = torch.Tensor(expert_demo) 82 | time_horizon, n_keypt_dim = expert_demo.shape 83 | 84 | keypoint_mpc_wrapper = GroundTruthKeypointMPCWrapper(robot_model, time_horizon=time_horizon - 1, n_keypt_dim=n_keypt_dim) 85 | # unroll and extract expected features 86 | pred_traj = keypoint_mpc_wrapper.roll_out(start_pose.clone()) 87 | 88 | # get initial irl loss 89 | irl_loss = irl_loss_fn(pred_traj, expert_demo).mean() 90 | irl_loss_dems.append(irl_loss.item()) 91 | 92 | irl_loss_on_train.append(torch.Tensor(irl_loss_dems).mean()) 93 | print("irl cost training iter: {} loss: {}".format(0, irl_loss_on_train[-1])) 94 | 95 | print("Cost function parameters to be optimized:") 96 | for name, param in learnable_cost.named_parameters(): 97 | print(name) 98 | print(param) 99 | 100 | # start of inverse RL loop 101 | for outer_i in range(n_outer_iter): 102 | irl_loss_dems = [] 103 | 104 | for demo_i in range(len(train_trajs)): 105 | learnable_cost_opt.zero_grad() 106 | expert_demo_dict = train_trajs[demo_i] 107 | 108 | start_pose = expert_demo_dict['start_joint_config'].squeeze() 109 | expert_demo = expert_demo_dict['desired_keypoints'].reshape(traj_len, -1) 110 | expert_demo = torch.Tensor(expert_demo) 111 | time_horizon, n_keypt_dim = expert_demo.shape 112 | 113 | keypoint_mpc_wrapper = GroundTruthKeypointMPCWrapper(robot_model, time_horizon=time_horizon - 1, 114 | n_keypt_dim=n_keypt_dim) 115 | action_optimizer = torch.optim.SGD(keypoint_mpc_wrapper.parameters(), lr=action_lr) 116 | 117 | with higher.innerloop_ctx(keypoint_mpc_wrapper, action_optimizer) as (fpolicy, diffopt): 118 | pred_traj = fpolicy.roll_out(start_pose.clone()) 119 | 120 | # use the learned loss to update the action sequence 121 | learned_cost_val = learnable_cost(pred_traj, expert_demo[-1]) 122 | diffopt.step(learned_cost_val) 123 | 124 | pred_traj = fpolicy.roll_out(start_pose) 125 | # compute task loss 126 | irl_loss = irl_loss_fn(pred_traj, expert_demo).mean() 127 | # backprop gradient of learned cost parameters wrt irl loss 128 | irl_loss.backward(retain_graph=True) 129 | irl_loss_dems.append(irl_loss.detach()) 130 | 131 | learnable_cost_opt.step() 132 | 133 | if outer_i % 25 == 0: 134 | plt.figure() 135 | plt.plot(pred_traj[:, 7].detach(), pred_traj[:, 9].detach(), 'o') 136 | plt.plot(expert_demo[:, 0], expert_demo[:, 2], 'x') 137 | plt.title("outer i: {}".format(outer_i)) 138 | plt.savefig(os.path.join(plots_dir, f'{demo_i}_{outer_i}.png')) 139 | 140 | irl_loss_on_train.append(torch.Tensor(irl_loss_dems).mean()) 141 | test_irl_losses = evaluate_action_optimization(learnable_cost.eval(), robot_model, irl_loss_fn, test_trajs, 142 | n_inner_iter) 143 | print("irl loss (on train) training iter: {} loss: {}".format(outer_i + 1, irl_loss_on_train[-1])) 144 | print("irl loss (on test) training iter: {} loss: {}".format(outer_i + 1, test_irl_losses.mean().item())) 145 | print("") 146 | irl_loss_on_test.append(test_irl_losses) 147 | learnable_cost_params = {} 148 | for name, param in learnable_cost.named_parameters(): 149 | learnable_cost_params[name] = param 150 | 151 | if len(learnable_cost_params) == 0: 152 | # For RBF Weighted Cost 153 | for name, param in learnable_cost.weights_fn.named_parameters(): 154 | learnable_cost_params[name] = param 155 | 156 | plt.figure() 157 | plt.plot(pred_traj[:, 7].detach(), pred_traj[:, 9].detach(), 'o') 158 | plt.plot(expert_demo[:, 0], expert_demo[:, 2], 'x') 159 | plt.title("final") 160 | plt.savefig(os.path.join(plots_dir, f'{demo_i}_final.png')) 161 | 162 | return torch.stack(irl_loss_on_train), torch.stack(irl_loss_on_test), learnable_cost_params, pred_traj 163 | 164 | 165 | if __name__ == '__main__': 166 | random.seed(10) 167 | np.random.seed(10) 168 | torch.manual_seed(0) 169 | 170 | rest_pose = [0.0, 0.0, 0.0, 1.57079633, 0.0, 1.03672558, 0.0] 171 | 172 | rel_urdf_path = 'env/kuka_iiwa/urdf/iiwa7_ft_with_obj_keypts.urdf' 173 | urdf_path = os.path.join(mbirl.__path__[0], rel_urdf_path) 174 | robot_model = DifferentiableRobotModel(urdf_path=urdf_path, name="kuka_w_obj_keypts") 175 | 176 | data_type = 'placing' 177 | trajs = torch.load(f'{traj_data_dir}/traj_data_{data_type}.pt') 178 | 179 | traj = trajs[0] 180 | traj_len = len(traj['desired_keypoints']) 181 | 182 | start_q = traj['start_joint_config'].squeeze() 183 | expert_demo = traj['desired_keypoints'].reshape(traj_len, -1) 184 | expert_demo = torch.Tensor(expert_demo) 185 | print(expert_demo.shape) 186 | n_keypt_dim = expert_demo.shape[1] 187 | time_horizon = expert_demo.shape[0] 188 | 189 | # type of cost 190 | #cost_type = 'Weighted' 191 | #cost_type = 'TimeDep' 192 | cost_type = 'RBF' 193 | 194 | learnable_cost = None 195 | 196 | if cost_type == 'Weighted': 197 | learnable_cost = LearnableWeightedCost(dim=n_keypt_dim) 198 | elif cost_type == 'TimeDep': 199 | learnable_cost = LearnableTimeDepWeightedCost(time_horizon=time_horizon, dim=n_keypt_dim) 200 | elif cost_type == 'RBF': 201 | learnable_cost = LearnableRBFWeightedCost(time_horizon=time_horizon, dim=n_keypt_dim) 202 | else: 203 | print('Cost not implemented') 204 | 205 | irl_loss_fn = IRLLoss() 206 | 207 | cost_lr = 1e-2 208 | action_lr = 1e-3 209 | n_outer_iter = 100 210 | n_inner_iter = 1 211 | n_test_traj = 2 212 | train_trajs = trajs[0:3] 213 | test_trajs = trajs[3:3 + n_test_traj] 214 | irl_loss_train, irl_loss_test, learnable_cost_params, pred_traj = irl_training(learnable_cost, robot_model, 215 | irl_loss_fn, 216 | train_trajs, test_trajs, 217 | n_outer_iter, n_inner_iter, 218 | cost_type=cost_type, 219 | data_type=data_type, 220 | cost_lr=cost_lr, 221 | action_lr=action_lr) 222 | 223 | if not os.path.exists(model_data_dir): 224 | os.makedirs(model_data_dir) 225 | 226 | torch.save({ 227 | 'irl_loss_train': irl_loss_train, 228 | 'irl_loss_test': irl_loss_test, 229 | 'cost_parameters': learnable_cost_params, 230 | 'fina_pred_traj': pred_traj, 231 | 'n_inner_iter': n_inner_iter, 232 | 'action_lr': action_lr 233 | }, f=f'{model_data_dir}/{data_type}_{cost_type}') 234 | -------------------------------------------------------------------------------- /mbirl/generate_expert_demo.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import os 3 | import random 4 | import torch 5 | import numpy as np 6 | import mbirl 7 | import matplotlib.pyplot as plt 8 | 9 | from differentiable_robot_model import DifferentiableRobotModel 10 | 11 | EXP_FOLDER = os.path.join(mbirl.__path__[0], "experiments") 12 | traj_data_dir = os.path.join(EXP_FOLDER, 'traj_data') 13 | 14 | 15 | class GroundTruthForwardModel(torch.nn.Module): 16 | def __init__(self, model): 17 | super(GroundTruthForwardModel, self).__init__() 18 | self.robot_model = model 19 | 20 | def forward_kin(self, x): 21 | keypoints = [] 22 | for link in [1, 2]: # , 3]: 23 | kp_pos, kp_rot = self.robot_model.compute_forward_kinematics(x, 'kp_link_' + str(link)) 24 | keypoints += 100.0*kp_pos 25 | 26 | return torch.stack(keypoints).squeeze() 27 | 28 | 29 | if __name__ == '__main__': 30 | 31 | random.seed(10) 32 | np.random.seed(10) 33 | torch.manual_seed(0) 34 | curr_dir = os.path.dirname(__file__) 35 | 36 | rel_urdf_path = 'env/kuka_iiwa/urdf/iiwa7_ft_with_obj_keypts.urdf' 37 | urdf_path = os.path.join(mbirl.__path__[0], rel_urdf_path) 38 | robot_model = DifferentiableRobotModel(urdf_path=urdf_path, name="kuka_w_obj_keypts") 39 | 40 | dmodel = GroundTruthForwardModel(robot_model) 41 | 42 | rest_pose = [0.0, 0.0, 0.0, 1.57079633, 0.0, 1.03672558, 0.0] 43 | rest_pose = torch.Tensor(rest_pose).unsqueeze(dim=0) 44 | 45 | experiment_type = 'placing' 46 | 47 | regenerate_data = True 48 | 49 | if not os.path.exists(traj_data_dir): 50 | os.makedirs(traj_data_dir) 51 | 52 | joint_limits = [2.967, 2.094, 2.967, 2.094, 2.967, 2.094, 3.054] 53 | if regenerate_data or not os.path.exists(f'{traj_data_dir}/traj_data_{experiment_type}.pt'): 54 | trajectories = [] 55 | for traj_it in range(6): 56 | print(traj_it) 57 | traj_data = {} 58 | start_pose = rest_pose.clone() 59 | start_keypts = dmodel.forward_kin(start_pose) 60 | print(f"cur keypts: {start_keypts}") 61 | goal_keypts1 = start_keypts[-3:].clone() 62 | goal_keypts1[:, 0] = goal_keypts1[:, 0] + torch.Tensor([-20.0]) + torch.randn(1)[0] 63 | goal_keypts2 = goal_keypts1.clone() 64 | goal_keypts2[:, 2] = goal_keypts2[:, 2] + torch.Tensor([-30.0]) + torch.randn(1)[0] 65 | 66 | desired_keypt_traj = torch.stack([start_keypts.clone() for i in range(5)] + [goal_keypts1.clone() for i in range(5)]) 67 | 68 | for kp_idx in range(2): 69 | desired_keypt_traj[:5, kp_idx, 0] = torch.linspace(start_keypts[kp_idx, 0], goal_keypts1[kp_idx, 0], 5) 70 | desired_keypt_traj[5:, kp_idx, 2] = torch.linspace(goal_keypts1[kp_idx, 2], goal_keypts2[kp_idx, 2], 5) 71 | 72 | traj_data['start_joint_config'] = start_pose 73 | traj_data['desired_keypoints'] = desired_keypt_traj 74 | trajectories.append(traj_data) 75 | 76 | torch.save(trajectories, f"{traj_data_dir}/traj_data_{experiment_type}.pt") 77 | 78 | # visualization - matplotlib 79 | trajs = torch.load(f"{traj_data_dir}/traj_data_{experiment_type}.pt") 80 | 81 | n_trajs = len(trajs) 82 | 83 | fig = plt.figure(figsize=(2 * 5, int(np.ceil(n_trajs/2)) * 5)) 84 | for i, traj in enumerate(trajs): 85 | ax = fig.add_subplot(2, int(np.ceil(n_trajs/2)), i + 1, projection='3d') 86 | ax.plot(trajs[i]['desired_keypoints'][:, 0, 0], trajs[i]['desired_keypoints'][:, 0, 1], trajs[i]['desired_keypoints'][:, 0, 2]) 87 | ax.scatter(trajs[i]['desired_keypoints'][:, 0, 0], trajs[i]['desired_keypoints'][:, 0, 1], trajs[i]['desired_keypoints'][:, 0, 2], 88 | color='blue') 89 | ax.scatter(start_keypts[0, 0], start_keypts[0, 1], start_keypts[0, 2], 90 | color='red') 91 | ax.scatter(trajs[i]['desired_keypoints'][-1, 0, 0], trajs[i]['desired_keypoints'][-1, 0, 1], trajs[i]['desired_keypoints'][-1, 0, 2], 92 | color='green') 93 | min_x = -100.0; max_x = -50.0 94 | min_y = 0.0; max_y = 30 95 | min_z = 50; max_z = 100 96 | ax.set_xlim([min_x, max_x]) 97 | ax.set_ylim([min_y, max_y]) 98 | ax.set_zlim([min_z, max_z]) 99 | ax.set_xlabel("x") 100 | ax.set_ylabel("y") 101 | ax.set_zlabel("z") 102 | ax.set_title(f"Trajectory {i}") 103 | 104 | plt.tight_layout() 105 | plt.savefig(f'{traj_data_dir}/traj_data_{experiment_type}.png') 106 | plt.show() 107 | 108 | -------------------------------------------------------------------------------- /mbirl/keypoint_mpc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | import numpy as np 4 | 5 | 6 | joint_limits = [2.967, 2.094, 2.967, 2.094, 2.967, 2.094, 3.054] 7 | 8 | 9 | # A wrapper class keypoint MPC with action parameters to be optimized 10 | # This implementation assumes object keypoints are known and part of the robot model 11 | # this means the keypoint dynamics model can be implemented through a forward kinematics call 12 | class GroundTruthKeypointMPCWrapper(torch.nn.Module): 13 | 14 | def __init__(self, model, time_horizon, n_keypt_dim): 15 | super().__init__() 16 | self.time_horizon = time_horizon 17 | self.n_keypt_dim = n_keypt_dim 18 | self.action_seq = torch.nn.Parameter(torch.Tensor(np.zeros([time_horizon, 7]))) 19 | self.robot_model = model 20 | 21 | def forward(self, x, u=0): 22 | xdesired = x + u 23 | tl = torch.Tensor(joint_limits) 24 | xdesired = torch.where(xdesired > tl, tl, xdesired) 25 | xdesired = torch.where(xdesired < -tl, -tl, xdesired) 26 | keypoints = [] 27 | for link in [1,2]:#,3]: 28 | kp_pos, _ = self.robot_model.compute_forward_kinematics(xdesired.reshape(1, 7), 'kp_link_'+str(link)) 29 | keypoints += 100.0*kp_pos[0] 30 | return xdesired, torch.stack(keypoints).squeeze() 31 | 32 | def roll_out(self, joint_state): 33 | qs = [] 34 | key_pos = [] 35 | joint_state, keypts = self.forward(joint_state) 36 | qs.append(joint_state) 37 | key_pos.append(keypts) 38 | for t in range(self.time_horizon): 39 | ac = self.action_seq[t] 40 | joint_state, keypts = self.forward(joint_state, ac) 41 | tl = torch.Tensor(joint_limits) 42 | joint_state = torch.where(joint_state > tl, tl, joint_state) 43 | joint_state = torch.where(joint_state < -tl, -tl, joint_state) 44 | qs.append(joint_state.clone()) 45 | key_pos.append(keypts.clone()) 46 | return torch.cat((torch.stack(qs), torch.stack(key_pos)), dim=1) 47 | 48 | def reset_actions(self): 49 | self.action_seq.data = torch.Tensor(np.zeros([self.time_horizon, 7])) 50 | -------------------------------------------------------------------------------- /mbirl/learnable_costs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | 4 | 5 | # The learned weighted cost, with fixed weights ### 6 | class LearnableWeightedCost(torch.nn.Module): 7 | def __init__(self, dim=9, weights=None): 8 | super(LearnableWeightedCost, self).__init__() 9 | if weights is None: 10 | self.weights = torch.nn.Parameter(0.1 * torch.ones([dim, 1])) 11 | else: 12 | self.weights = weights 13 | self.dim = dim 14 | self.clip = torch.nn.ReLU() 15 | self.meta_grads = [[] for _, _ in enumerate(self.parameters())] 16 | 17 | def forward(self, y_in, y_target): 18 | assert y_in.dim() == 2 19 | mse = ((y_in[:,-self.dim:] - y_target[-self.dim:]) ** 2).squeeze() 20 | 21 | # weighted mse 22 | #wmse = torch.mm(mse, self.clip(self.weights)) 23 | wmse = torch.mm(mse, self.weights) 24 | return wmse.mean() 25 | 26 | 27 | # The learned weighted cost, with time dependent weights ### 28 | class LearnableTimeDepWeightedCost(torch.nn.Module): 29 | def __init__(self, time_horizon, dim=9, weights=None): 30 | super(LearnableTimeDepWeightedCost, self).__init__() 31 | if weights is None: 32 | self.weights = torch.nn.Parameter(0.01 * torch.ones([time_horizon, dim])) 33 | else: 34 | self.weights = weights 35 | self.clip = torch.nn.ReLU() 36 | self.dim = dim 37 | self.meta_grads = [[] for _, _ in enumerate(self.parameters())] 38 | 39 | def forward(self, y_in, y_target): 40 | assert y_in.dim() == 2 41 | mse = ((y_in[:,-self.dim:] - y_target[-self.dim:]) ** 2).squeeze() 42 | # weighted mse 43 | #wmse = mse * self.clip(self.weights) 44 | wmse = mse * self.weights 45 | return wmse.mean() 46 | 47 | 48 | class RBFWeights(torch.nn.Module): 49 | 50 | def __init__(self, time_horizon, dim, width, weights=None): 51 | super(RBFWeights, self).__init__() 52 | k_list = torch.linspace(0, time_horizon-1, 5) 53 | if weights is None: 54 | self.weights = torch.nn.Parameter(0.01 * torch.ones(len(k_list), dim)) 55 | else: 56 | self.weights = weights 57 | 58 | self.dim = dim 59 | 60 | x = torch.arange(0, 10) 61 | self.K = torch.stack([torch.exp(-(int(k) - x) ** 2 / width) for k in k_list]).T 62 | print(f"\nRBFWEIGHTS: {k_list}") 63 | 64 | self.clip = torch.nn.ReLU() 65 | 66 | def forward(self): 67 | #return self.K.matmul(self.clip(self.weights)) 68 | return self.K.matmul(self.weights) 69 | 70 | 71 | class LearnableRBFWeightedCost(torch.nn.Module): 72 | def __init__(self, time_horizon, dim=9, width=2.0, weights=None): 73 | super(LearnableRBFWeightedCost, self).__init__() 74 | self.dim = dim 75 | self.weights_fn = RBFWeights(time_horizon=time_horizon, dim=dim, width=width, weights=weights) 76 | self.weights = self.weights_fn() 77 | 78 | def forward(self, y_in, y_target): 79 | assert y_in.dim() == 2 80 | mse = (y_in[:, -self.dim:] - y_target[-self.dim:]) ** 2 81 | 82 | self.weights = self.weights_fn() 83 | wmse = self.weights * mse 84 | 85 | return wmse.sum(dim=0).mean() 86 | 87 | 88 | class BaselineCost(object): 89 | def __init__(self, dim, weights): 90 | self.weights = weights 91 | self.dim = dim 92 | 93 | def __call__(self, y_in, y_target): 94 | assert y_in.dim() == 2 95 | mse = ((y_in[:, -self.dim:] - y_target[-self.dim:]) ** 2).squeeze() 96 | 97 | # weighted mse 98 | wmse = mse * self.weights 99 | return wmse.mean() 100 | 101 | 102 | class IRLLoss(object): 103 | def __init__(self, dim): 104 | self.dim = dim 105 | 106 | def __call__(self, pred_traj, target_traj): 107 | loss = ((pred_traj[:, -self.dim:] - target_traj[:, -self.dim:]) ** 2).sum(dim=0) 108 | return loss.mean() 109 | -------------------------------------------------------------------------------- /ml3/README.md: -------------------------------------------------------------------------------- 1 | # LearningToLearn 2 | 3 | ## ML3 paper experiments and citation 4 | To reproduce results of the ML3 paper follow the instructions. 5 | All loss models are stored in 'experiments/data, all plots are stored in ./plots 6 | 7 | #### Loss Learning for Regression (ML3 paper experiment section IV.A.1) 8 | For meta learning the loss run 9 | 10 | ``` 11 | python experiments/run_sine_regression_exp.py 12 | ``` 13 | 14 | For visualizing the results run `jupyter notebook` and open `ml3_sine_regression_exp_viz` 15 | 16 | #### Reward Learning for Model-based RL (MBRL) Reacher (ML3 section IV.A.2) 17 | For meta learning the reward, run 18 | 19 | ``` 20 | python experiments/run_mbrl_reacher_exp.py train 21 | ``` 22 | 23 | For testing the reward, run 24 | 25 | ``` 26 | python experiments/run_mbrl_reacher_exp.py test 27 | ``` 28 | 29 | #### Learning with extra information at meta-train time (ML3 section IV.B) 30 | The following scripts require two arguments, first one is `train\test`, the 2nd one 31 | indicates whether to use extra information by setting `True\False` (with\without extra info) 32 | 33 | ##### For meta learning the loss with extra information on sine function run: 34 | In this experiment we show how the extra info can be used to shape the loss function for easier optimization. 35 | ``` 36 | python experiments/run_shaped_sine_exp.py train True 37 | ``` 38 | To test the loss with extra information run: 39 | ``` 40 | python experiments/run_shaped_sine_exp.py test True 41 | ``` 42 | To see how these results compare to not using the extra info, run the above scripts with the 2nd argument being `False` 43 | To visualize the loss landscapes for this experiment run `jupyter notebook` and open `Loss shaping visualization.ipynb` 44 | 45 | ##### For meta learning the loss with additional goal in the mountain car experiment run: 46 | In this experiment we show how the extra info can be used to guide exploration for an RL task. 47 | ``` 48 | python experiments/run_mountain_car_exp.py train True 49 | ``` 50 | To test the loss with extra goal run: 51 | ``` 52 | python experiments/run_mountain_car_exp.py test True 53 | ``` 54 | The test script generates a gif of the final policy, and stores it in the experiment folder 55 | To see how these results compare to not using the extra info, run the above scripts with the 2nd argument being `False` 56 | -------------------------------------------------------------------------------- /ml3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/ml3/__init__.py -------------------------------------------------------------------------------- /ml3/envs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/ml3/envs/__init__.py -------------------------------------------------------------------------------- /ml3/envs/bullet_sim.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import os 3 | 4 | import numpy as np 5 | 6 | import pybullet_utils.bullet_client as bc 7 | import pybullet_data 8 | import pybullet 9 | 10 | class BulletSimulation(object): 11 | 12 | def __init__(self, gui, controlled_joints, ee_idx, torque_limits,target_pos): 13 | 14 | if gui: 15 | self.sim = bc.BulletClient(connection_mode=pybullet.GUI) 16 | else: 17 | self.sim = bc.BulletClient(connection_mode=pybullet.DIRECT) 18 | self.sim.setAdditionalSearchPath(pybullet_data.getDataPath()) 19 | 20 | self.ee_idx = ee_idx 21 | 22 | self.cur_joint_pos = None 23 | self.cur_joint_vel = None 24 | self.curr_ee = None 25 | self.babbling_torque_limits = None 26 | self.logger = None 27 | 28 | self.controlled_joints = controlled_joints 29 | self.torque_limits = torque_limits 30 | 31 | self.n_dofs = len(controlled_joints) 32 | 33 | #pybullet.setAdditionalSearchPath(pybullet_data.getDataPath()) 34 | # TODO: the following should be extracted into some world model that is loaded independently of the robot 35 | #self.planeId = pybullet.loadURDF("plane.urdf",[0,0,0]) 36 | if target_pos is not None: 37 | self.cubeId = pybullet.loadURDF("sphere_small.urdf",target_pos) 38 | 39 | def disconnect(self): 40 | self.sim.disconnect() 41 | #pybullet.disconnect() 42 | 43 | def get_random_torques(self, time_horizon, bounds_low, bounds_high): 44 | trajectory= [] 45 | for t in range(time_horizon): 46 | torque_limits_babbling = bounds_high 47 | torques = np.random.uniform(-torque_limits_babbling, torque_limits_babbling) 48 | trajectory.append(np.array(torques)) 49 | return np.array(trajectory) 50 | 51 | def get_random_torques_uinit(self, time_horizon): 52 | trajectory= [] 53 | for t in range(time_horizon): 54 | torques = np.random.uniform(-0.3*self.torque_limits, 0.3*self.torque_limits) 55 | trajectory.append(np.array(torques)) 56 | return np.array(trajectory) 57 | 58 | def get_target_joint_configuration(self, target_position): 59 | self.reset() 60 | des_joint_state = self.sim.calculateInverseKinematics(self.robot_id, 61 | self.ee_idx, 62 | np.array(target_position), jointDamping = [0.1 for i in range(self.n_dofs)]) 63 | return np.asarray(des_joint_state) 64 | 65 | def reset(self, joint_pos=None, joint_vel=None): 66 | if joint_vel is None: 67 | joint_vel = list(np.zeros(self.n_dofs)) 68 | 69 | if joint_pos is None: 70 | joint_pos = list(np.zeros(self.n_dofs)) 71 | 72 | for i in range(self.n_dofs): 73 | self.sim.resetJointState(bodyUniqueId=self.robot_id, 74 | jointIndex=self.controlled_joints[i], 75 | targetValue=joint_pos[i], 76 | targetVelocity=joint_vel[i]) 77 | 78 | self.sim.stepSimulation() 79 | self.cur_joint_pos = self.get_current_joint_pos() 80 | self.cur_joint_vel = self.get_current_joint_vel() 81 | self.curr_ee = self.get_current_ee_state() 82 | return np.hstack([self.get_current_joint_pos(),self.get_current_joint_vel()]) 83 | 84 | def move_to_joint_positions(self, joint_pos, joint_vel=None): 85 | if joint_vel is None: 86 | joint_vel = [0]*len(joint_pos) 87 | 88 | for i in range(self.n_dofs): 89 | self.sim.resetJointState(bodyUniqueId=self.robot_id, 90 | jointIndex=self.controlled_joints[i], 91 | targetValue=joint_pos[i], 92 | targetVelocity=joint_vel[i]) 93 | 94 | self.sim.stepSimulation() 95 | 96 | self.cur_joint_pos = self.get_current_joint_pos() 97 | self.cur_joint_vel = self.get_current_joint_vel() 98 | self.curr_ee = self.get_current_ee_state() 99 | return np.hstack([self.cur_joint_pos,self.cur_joint_vel]) 100 | 101 | def get_MassM(self,angles): 102 | for link_idx in self.controlled_joints: 103 | self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 104 | cur_joint_angles = list(angles) 105 | mass_m = self.sim.calculateMassMatrix(bodyUniqueId=self.robot_id, 106 | objPositions = cur_joint_angles) 107 | 108 | return np.array(mass_m) 109 | 110 | def get_F(self,angles,vel): 111 | for link_idx in self.controlled_joints: 112 | self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 113 | cur_joint_angles = list(angles) 114 | cur_joint_vel = list(vel) 115 | torques = self.sim.calculateInverseDynamics(self.robot_id, 116 | cur_joint_angles, 117 | cur_joint_vel, 118 | [0]*self.action_dim) 119 | return np.asarray(torques) 120 | 121 | 122 | 123 | def joint_angles(self): 124 | return self.cur_joint_pos 125 | 126 | def joint_velocities(self): 127 | return self.cur_joint_vel 128 | 129 | def forwad_kin(self,state): 130 | return self.endeffector_pos() 131 | 132 | def endeffector_pos(self): 133 | return self.curr_ee 134 | 135 | def get_target_ee(self, state): 136 | 137 | for i in range(self.n_dofs): 138 | self.sim.resetJointState(bodyUniqueId=self.robot_id, 139 | jointIndex=self.controlled_joints[i], 140 | targetValue=state[i], 141 | targetVelocity=0.0) 142 | self.sim.stepSimulation() 143 | 144 | ls = self.sim.getLinkState(self.robot_id, self.ee_idx)[0] 145 | return ls 146 | 147 | def reset_then_step(self, des_joint_state, torque): 148 | # for link_idx in self.controlled_joints: 149 | # self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 150 | for i in range(self.n_dofs): 151 | self.sim.resetJointState(bodyUniqueId=self.robot_id, 152 | jointIndex=self.controlled_joints[i], 153 | targetValue=des_joint_state[i], 154 | targetVelocity=des_joint_state[(i+self.n_dofs)]) 155 | 156 | return self.apply_joint_torque(torque)[0] 157 | 158 | def step_model(self,state,torque): 159 | return self.sim_step(state,torque) 160 | 161 | def sim_step(self,state,torque): 162 | for link_idx in self.controlled_joints: 163 | self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 164 | if str(state.dtype).startswith('torch'): 165 | state = state.clone().detach().numpy() 166 | if str(torque.dtype).startswith('torch'): 167 | torque = torque.clone().detach().numpy() 168 | return self.reset_then_step(state,torque) 169 | 170 | def step(self,state,torque): 171 | for link_idx in self.controlled_joints: 172 | self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 173 | if str(state.dtype).startswith('torch'): 174 | state = state.clone().detach().numpy() 175 | if str(torque.dtype).startswith('torch'): 176 | torque = torque.clone().detach().numpy() 177 | return self.reset_then_step(state,torque) 178 | 179 | def apply_joint_torque(self, torque): 180 | 181 | 182 | self.grav_comp = self.inverse_dynamics([0] * self.action_dim) 183 | torque = torque + self.grav_comp 184 | full_torque = torque.copy() 185 | 186 | 187 | #torque = torque.clip(-self.torque_limits, self.torque_limits) 188 | 189 | self.sim.setJointMotorControlArray(bodyIndex=self.robot_id, 190 | jointIndices=self.controlled_joints, 191 | controlMode=pybullet.TORQUE_CONTROL, 192 | forces=torque) 193 | self.sim.stepSimulation() 194 | 195 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 196 | cur_joint_angles = [cur_joint_states[i][0] for i in range(self.n_dofs)] 197 | cur_joint_vel = [cur_joint_states[i][1] for i in range(self.n_dofs)] 198 | 199 | next_state = cur_joint_angles + cur_joint_vel 200 | 201 | ls = list(self.sim.getLinkState(self.robot_id, self.ee_idx)[0]) 202 | self.cur_joint_pos = self.get_current_joint_pos() 203 | self.cur_joint_vel = self.get_current_joint_vel() 204 | self.curr_ee = self.get_current_ee_state() 205 | return np.hstack([self.cur_joint_pos,self.cur_joint_vel]),self.curr_ee 206 | 207 | def get_current_ee_state(self): 208 | ee_state = self.sim.getLinkState(self.robot_id, self.ee_idx) 209 | return np.array(ee_state[0]) 210 | 211 | def get_current_joint_pos(self): 212 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 213 | cur_joint_angles = [cur_joint_states[i][0] for i in range(self.n_dofs)] 214 | return np.array(cur_joint_angles) 215 | 216 | def get_current_joint_vel(self): 217 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 218 | cur_joint_vel = [cur_joint_states[i][1] for i in range(self.n_dofs)] 219 | return np.array(cur_joint_vel) 220 | 221 | def get_current_joint_state(self): 222 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 223 | cur_joint_angles = [cur_joint_states[i][0] for i in range(self.n_dofs)] 224 | cur_joint_vel = [cur_joint_states[i][1] for i in range(self.n_dofs)] 225 | return np.hstack([cur_joint_angles, cur_joint_vel]) 226 | 227 | def get_ee_jacobian(self): 228 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 229 | cur_joint_angles = [cur_joint_states[i][0] for i in range(self.n_dofs)] 230 | cur_joint_vel = [cur_joint_states[i][1] for i in range(self.n_dofs)] 231 | bullet_jac_lin, bullet_jac_ang = self.sim.calculateJacobian( 232 | bodyUniqueId=self.robot_id, 233 | linkIndex=self.ee_idx, 234 | localPosition=[0, 0, 0], 235 | objPositions=cur_joint_angles, 236 | objVelocities=cur_joint_vel, 237 | objAccelerations=[0] * self.n_dofs, 238 | ) 239 | return np.asarray(bullet_jac_lin), np.asarray(bullet_jac_ang) 240 | 241 | def inverse_dynamics(self, des_acc): 242 | for link_idx in self.controlled_joints: 243 | self.sim.changeDynamics(self.robot_id, link_idx, linearDamping=0.0, angularDamping=0.0, jointDamping=0.0) 244 | cur_joint_states = self.sim.getJointStates(self.robot_id, self.controlled_joints) 245 | cur_joint_angles = [cur_joint_states[i][0] for i in range(self.n_dofs)] 246 | cur_joint_vel = [cur_joint_states[i][1] for i in range(self.n_dofs)] 247 | torques = self.sim.calculateInverseDynamics(self.robot_id, 248 | cur_joint_angles, 249 | cur_joint_vel, 250 | des_acc) 251 | return np.asarray(torques) 252 | 253 | 254 | def detect_collision(self): 255 | return False 256 | 257 | def return_grav_comp_torques(self): 258 | return 0.0 259 | 260 | def get_pred_error(self,x,u): 261 | return np.zeros(len(u)) 262 | 263 | def sim_step_un(self,x,u): 264 | return np.zeros(len(u)),np.zeros(len(u)) 265 | 266 | def get_gravity_comp(self): 267 | return 0.0 268 | 269 | 270 | class BulletSimulationFromURDF(BulletSimulation): 271 | def __init__(self, rel_urdf_path, gui, controlled_joints, ee_idx, torque_limits, target_pos): 272 | super(BulletSimulationFromURDF, self).__init__(gui, controlled_joints, ee_idx, torque_limits, target_pos) 273 | urdf_path = os.getcwd()+'/envs/'+rel_urdf_path 274 | print("loading urdf file: {}".format(urdf_path)) 275 | 276 | self.robot_id = self.sim.loadURDF(urdf_path, basePosition=[-0.5, 0, 0.0], useFixedBase=True) 277 | self.n_dofs = len(controlled_joints) 278 | 279 | pybullet.setAdditionalSearchPath(pybullet_data.getDataPath()) 280 | #self.planeId = pybullet.loadURDF("plane.urdf") 281 | 282 | self.sim.resetBasePositionAndOrientation(self.robot_id,[-0.5,0,0.0],[0,0,0,1]) 283 | self.sim.setGravity(0, 0, -9.81) 284 | dt = 1.0/240.0 285 | self.dt = dt 286 | self.sim.setTimeStep(dt) 287 | self.sim.setRealTimeSimulation(0) 288 | self.sim.setJointMotorControlArray(self.robot_id, 289 | self.controlled_joints, 290 | pybullet.VELOCITY_CONTROL, 291 | forces=np.zeros(self.n_dofs)) 292 | 293 | 294 | class BulletSimulationFromMJCF(BulletSimulation): 295 | 296 | def __init__(self, rel_mjcf_path, gui, controlled_joints, ee_idx, torque_limits): 297 | super(BulletSimulationFromMJCF, self).__init__(gui, controlled_joints, ee_idx, torque_limits, None) 298 | print('hierhierhierhier') 299 | 300 | xml_path = os.getcwd()+'/envs/'+rel_mjcf_path 301 | if rel_mjcf_path[0] != os.sep: xml_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), rel_mjcf_path) 302 | else: xml_path = rel_mjcf_path 303 | 304 | #xml_path = '/Users/sarah/Documents/GitHub/LearningToLearn/ml3/envs/mujoco_robots/reacher.xml' 305 | print("loading this mjcf file: {}".format(xml_path)) 306 | 307 | self.world_id, self.robot_id = self.sim.loadMJCF(xml_path) 308 | 309 | pybullet.setAdditionalSearchPath(pybullet_data.getDataPath()) 310 | print(pybullet_data.getDataPath()) 311 | self.planeId = pybullet.loadURDF("plane.urdf") 312 | #self.cubeId = pybullet.loadURDF("sphere_small.urdf", [0.02534078, -0.19863741, 0.01]) #0.02534078, -0.19863741 0.10534078, 0.1663741 313 | 314 | self.n_dofs = len(controlled_joints) 315 | self.sim.setGravity(0, 0, -9.81) 316 | dt = 1.0/100.0 317 | self.dt = dt 318 | self.sim.setTimeStep(dt) 319 | self.sim.setRealTimeSimulation(0) 320 | self.sim.setJointMotorControlArray(self.robot_id, 321 | self.controlled_joints, 322 | pybullet.VELOCITY_CONTROL, 323 | forces=np.zeros(self.n_dofs)) 324 | -------------------------------------------------------------------------------- /ml3/envs/mountain_car.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import numpy as np 3 | import torch 4 | import matplotlib.pyplot as plt 5 | import matplotlib.animation as animation 6 | 7 | 8 | class MountainCar(): 9 | def __init__(self): 10 | self.m = 0.2 11 | self.g = -9.8 12 | self.k = 0.3 13 | self.max_position = 0.5 14 | self.min_position = -1.2 15 | self.max_speed = 1.5 16 | self.min_speed = -1.5 17 | self.delta_t = 0.1 18 | self.min_action = -0.25 19 | self.max_action = 0.25 20 | self.cur_pos = 0.0 21 | self.cur_vel = 0.0 22 | 23 | def sim_step_torch(self, state, action): 24 | 25 | position = state[0] 26 | velocity = state[1] 27 | 28 | action = torch.clamp(action, min=self.min_action, max=self.max_action) 29 | 30 | velocity = velocity + (self.g * self.m * torch.cos(3.0 * position) + (action / self.m) - ( 31 | self.k * velocity)) * self.delta_t 32 | position = position + (velocity * self.delta_t) 33 | 34 | if (velocity.data > self.max_speed): velocity.data = torch.Tensor([self.max_speed]) 35 | if (velocity.data < -self.max_speed): velocity.data = torch.Tensor([-self.max_speed]) 36 | 37 | if (position.data >= self.max_position): position.data = torch.Tensor([self.max_position]) 38 | if (position.data < self.min_position): position.data = torch.Tensor([self.min_position]) 39 | if (position.data == self.min_position and velocity.data < 0): velocity.data = torch.Tensor([0.0]) 40 | 41 | new_state = torch.stack([position.squeeze(), velocity.squeeze()]) 42 | return new_state 43 | 44 | def sim_step(self, state, action): 45 | position = state[0] 46 | velocity = state[1] 47 | 48 | velocity = velocity + (self.g * self.m * np.cos(3 * position) + (action / self.m) - (self.k * velocity)) * self.delta_t 49 | position = position + (velocity * self.delta_t) 50 | 51 | if (velocity > self.max_speed): velocity = self.max_speed 52 | if (velocity < -self.max_speed): velocity = -self.max_speed 53 | 54 | if (position > self.max_position): position = self.max_position 55 | if (position < self.min_position): position = self.min_position 56 | if (position==self.min_position and velocity<0): velocity = 0 57 | 58 | new_state = np.array([position, velocity]) 59 | return new_state.squeeze() 60 | 61 | def step(self, action): 62 | position = self.cur_pos 63 | velocity = self.cur_vel 64 | 65 | velocity = velocity + ( 66 | self.g * self.m * np.cos(3 * position) + (action / self.m) - (self.k * velocity)) * self.delta_t 67 | position = position + (velocity * self.delta_t) 68 | 69 | if (velocity > self.max_speed): velocity = self.max_speed 70 | if (velocity < -self.max_speed): velocity = -self.max_speed 71 | 72 | if (position > self.max_position): position = self.max_position 73 | if (position < self.min_position): position = self.min_position 74 | if (position == self.min_position and velocity < 0): velocity = 0 75 | 76 | new_state = np.array([position, velocity]) 77 | self.cur_pos = position 78 | self.cur_vel = velocity 79 | reward = 0 80 | if new_state[0] >= 0.5: 81 | reward = 100 82 | return np.array([self.cur_pos, self.cur_vel]), reward 83 | 84 | def reset(self): 85 | self.cur_pos = -0.55 86 | self.cur_vel = 0 87 | return np.array([self.cur_pos, self.cur_vel]) 88 | 89 | def reset_to(self, state): 90 | self.cur_pos = state[0] 91 | self.cur_vel = state[1] 92 | return np.array([self.cur_pos, self.cur_vel]) 93 | 94 | def render(self, position_list, file_path='./mountain_car.gif', mode='gif'): 95 | """ When the method is called it saves an animation 96 | of what happened until that point in the episode. 97 | Ideally it should be called at the end of the episode, 98 | and every k episodes. 99 | 100 | ATTENTION: It requires avconv and/or imagemagick installed. 101 | @param file_path: the name and path of the video file 102 | @param mode: the file can be saved as 'gif' or 'mp4' 103 | """ 104 | 105 | # Plot init 106 | fig = plt.figure(figsize=(4,4)) 107 | ax = fig.add_subplot(111, autoscale_on=False, xlim=(-1.3, 0.6), ylim=(-1.2, 1.5)) 108 | ax.grid(False) # disable the grid 109 | x_sin = np.linspace(start=-1.2, stop=0.5, num=100) 110 | y_sin = np.sin(3 * x_sin) 111 | ax.plot(x_sin, y_sin,c='black',linewidth=3) # plot the sine wave 112 | ax.plot(0.50, 1.16, marker="$\u2691$", markersize=25, color='green') 113 | 114 | dot, = ax.plot([], [],marker="$\u25A1$",markersize=15,color='red') 115 | time_text = ax.text(0.05, 0.9, '', transform=ax.transAxes) 116 | _position_list = position_list 117 | _delta_t = self.delta_t 118 | 119 | def _init(): 120 | dot.set_data([], []) 121 | time_text.set_text('') 122 | return dot, time_text 123 | 124 | def _animate(i): 125 | x = _position_list[i] 126 | y = np.sin(3 * x) 127 | dot.set_data(x, y) 128 | time_text.set_text("") 129 | return dot, time_text 130 | 131 | ani = animation.FuncAnimation(fig, _animate, np.arange(1, len(position_list)), 132 | blit=True, init_func=_init, repeat=False) 133 | 134 | if mode == 'gif': 135 | ani.save(file_path, writer='imagemagick', fps=int(1 / self.delta_t)) 136 | elif mode == 'mp4': 137 | ani.save(file_path, fps=int(1 / self.delta_t), writer='avconv', codec='libx264') 138 | # Clear the figure 139 | fig.clear() 140 | plt.close(fig) -------------------------------------------------------------------------------- /ml3/envs/mujoco_robots/ground_plane.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /ml3/envs/mujoco_robots/reacher.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 44 | -------------------------------------------------------------------------------- /ml3/envs/reacher_sim.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import numpy as np 3 | from ml3.envs.bullet_sim import BulletSimulationFromMJCF 4 | 5 | 6 | class ReacherSimulation(BulletSimulationFromMJCF): 7 | def __init__(self, gui, file_name = 'mujoco_robots/reacher.xml',controlled_joints=None, ee_idx=None, torque_limits=None): 8 | rel_xml_path = file_name 9 | 10 | #fingertip 11 | if ee_idx is None: 12 | self.ee_idx = 4 13 | if controlled_joints is None: 14 | controlled_joints = [0, 2] 15 | if torque_limits is None: 16 | torque_limits = np.asarray([1, 1]) 17 | 18 | self.action_dim=2 19 | self.state_dim=4 20 | self.pos_dim=2 21 | 22 | super(ReacherSimulation, self).__init__(rel_mjcf_path=rel_xml_path, 23 | gui=gui, 24 | controlled_joints=controlled_joints, 25 | ee_idx=self.ee_idx, 26 | torque_limits=torque_limits) 27 | 28 | if gui: 29 | self.sim.resetDebugVisualizerCamera(cameraDistance=0.5, cameraYaw=-50, cameraPitch=-50, 30 | cameraTargetPosition=[0, 0, 0]) 31 | 32 | n_dofs_total = self.sim.getNumJoints(self.robot_id) 33 | print("n dofs total (including fixed joints): {}".format(n_dofs_total)) 34 | 35 | for i in range(n_dofs_total): 36 | print(self.sim.getJointInfo(self.robot_id, i)) 37 | return 38 | 39 | -------------------------------------------------------------------------------- /ml3/experiments/Loss shaping visualization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import ml3\n", 11 | "EXP_FOLDER = os.path.join(ml3.__path__[0], \"experiments/data/shaped_sine\")" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "# Shaping Loss Example" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "### Before running visualization please run in Terminal:" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 5, 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "name": "stdout", 35 | "output_type": "stream", 36 | "text": [ 37 | "task loss: 0.002699082251638174\n", 38 | "task loss: 14.513303756713867\n", 39 | "task loss: 0.0004392655100673437\n", 40 | "task loss: 0.1109960600733757\n", 41 | "task loss: 0.010903225280344486\n", 42 | "task loss: 0.00028027829830534756\n", 43 | "task loss: 0.00023076884099282324\n", 44 | "task loss: 0.01268570777028799\n", 45 | "task loss: 0.005605524405837059\n", 46 | "task loss: 0.0009330477914772928\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "!python run_shaped_sine_exp.py train True" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 7, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "task loss: 1.9977433112217113e-06\n", 64 | "task loss: 0.5099665522575378\n", 65 | "task loss: 0.48222532868385315\n", 66 | "task loss: 0.0001230674679391086\n", 67 | "task loss: 0.4575786292552948\n", 68 | "task loss: 0.4234389066696167\n", 69 | "task loss: 0.5580686330795288\n", 70 | "task loss: 0.48198580741882324\n", 71 | "task loss: 0.0002105423336615786\n", 72 | "task loss: 0.4930514395236969\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "!python run_shaped_sine_exp.py train False" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "Populating the interactive namespace from numpy and matplotlib\n" 90 | ] 91 | } 92 | ], 93 | "source": [ 94 | "%pylab inline\n", 95 | "from ml3.shaped_sine_utils import render\n", 96 | "from ml3.shaped_sine_utils import plot_loss\n", 97 | "def normalize_data(data):\n", 98 | " norm_data = []\n", 99 | " for d in data:\n", 100 | " norm_data.append((d-min(d))/(max(d)-min(d)))\n", 101 | " return norm_data" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 6, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "data": { 111 | "text/plain": [ 112 | "" 113 | ] 114 | }, 115 | "execution_count": 6, 116 | "metadata": {}, 117 | "output_type": "execute_result" 118 | }, 119 | { 120 | "data": { 121 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAADQCAYAAACtIK3LAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABkyUlEQVR4nO2dd3hU1daH351OOinUAAkdAiRA6FVRQEBAxQoqooi9YL/q1evVa/2uimJBRUVR8KqICoggUgUkQCghdAKElpBAEkjP7O+PPQkBUiZ1JmG9z5NnMmfOOXvNzJnfWXvttddWWmsEQRAEQRAE++NkbwMEQRAEQRAEgzhmgiAIgiAIDoI4ZoIgCIIgCA6COGaCIAiCIAgOgjhmgiAIgiAIDoI4ZoIgCIIgCA6Ci70NqAqCgoJ0aGiovc0QHJFdu8xju3b2tUOocjZu3HhSax1sbzuqAtEwoUREw+okpelXnXDMQkNDiY6OtrcZgiMyeLB5XL7cnlYI1YBS6qC9bagqRMOEEhENq5OUpl+X3FDmP+ZtY/qfe0k4lWFvUwRBEIQ6zM9bjvL2kt0kn8m2tylCLaJORMxsJTffwv6kM3yz/hBvLt5Fr7AArunalBFdGuPr4Wpv8wRBEIQ6QuzRVB77LobcfM3HK/fx6W096N8myN5mCbUAu0TMlFJvKqV2KqW2KqXmKaX8L3i9uVLqjFLq8aps19XZiTl392HVk5fx2JVtSUrP5ukftxH18lLun72JpTtOkJdvqcomBUEQhEuMrNx8Hp0bg7+nGz/c24dAL3c+XLHX3mYJtQR7RcyWAM9orfOUUq8DzwBPFXn9v8Ci6mq8WYAnDw5pwwOXt2ZrQirzNh/hly1HWbDtGE38PLilV3Nu7NGcYB/36jKhWHJzc0lISCArK6tG263TvPCCeYyLs68dDoKHhwchISG4ukqEWBCqi7kbDrP7xBk+v6MH3VsEcH1UCO/+sYeEUxmE1Pe0t3mCg2MXx0xr/XuRp+uAcQVPlFJjgQPA2eq2QylFRDN/Ipr58+zIDizbmcjX6w7y1u+7efePPYzo3Jjb+rSgW/P6KKWq2xwSEhLw8fEhNDS0Rtq7JHCyBoVlRhNaa5KTk0lISCAsLMze5ghCneWnmCO0b+TDZe0aAHBdtxDeWbqHHzYe4eEr2tjZOsHRcYTk/0lYo2NKKW9M5OxfNW2Eq7MTw8Ib8dWdvfjjsUGM79WCZXGJXPfhWkZOW83cDYfIzsuvVhuysrIIDAwUp0yoFpRSBAYGSkRWEKqRg8ln2XzoNGO7Ni3c1izAk76tAvnfxsNYLNqO1gm1gWpzzJRSS5VS24v5G1Nkn2eBPGC2ddOLwNta6zM2nP9upVS0Uio6KSmpSm1vFezNi6PDWfePIbxyTScsWvPUD9sY8PqffLRiH2lZuVXaXlHEKROqE7m+BKF6mR9zFICrI5qct31s16YknMpk14l0e5gl1CKqbShTa31Faa8rpSYCo4AhWuuCLkQvYJxS6g3AH7AopbK01u8Xc/4ZwAyAqKioaumCeLm7ML5XC27p2ZzVe0/y0Yp9vLZoJ9OX7eWO/mHc2T8Mv3qSqyMIgiCYdIH5MUfoGRZAU/96573WIzQAgJjDp+nQ2Nce5gm1BLvkmCmlhgNPAoO01oUFxbTWA4rs8yJwpjinrKZRSjGgTTAD2gSzLSGV9//cw7Q/9vD5mgPc1b8ld/QPlXIbgiAIlzgHTp5lX9JZbu8betFroYGe+Hu6EnPoNDf3bF7zxgm1BnvlmL0P+ABLlFIxSqmP7GRHuekc4sfHt0ax4KH+9G4ZyNtLdzPg9T95f9kezmTn2ds8oRTi4uK45557GDduHB9++KG9zREEoY6xcrdJqxnctsFFrymliAjxJ+bw6Rq2Sqht2MUx01q31lo301pHWv/uKWafF7XWb9nDPlsIb+LHJ7dF8csD/YlqUZ+3ft/NgNeX8cHyvZyt5Q7aK6+8Qnh4OF26dCEyMpL169cTHx9Pp06datSOF198kbfeuvgSUEoxYcKEwud5eXkEBwczatSowm3e3t4XHdehQwc++ugjvvvuO9asWVNsm8UdV1lKeh+CINQtVu05SYtAT5oHFl8SI7KZP7sT06UTL5SKI8zKrNV0DvHjs4k9+On+fkQ08+eN33Yx4I0/mbFyH5k51TuLszpYu3Ytv/76K5s2bWLr1q0sXbqUZs2a2dus8/Dy8mL79u1kZmYCsGTJEpo2bVrGUYaff/6ZkSNHMmLEiOo0UajlKKWGK6V2KaX2KqWeLmGfG5RSO5RSsUqpb4psv10ptcf6d3vNWS3Yk5w8C2v3JzOglOr+kc390Rq2JpyuOcOEWoc4ZlVEZDN/vrijJz/c25fwJr78Z+FOBryxjE9X7Scrt/Y4aMeOHSMoKAh3d1NcNygoiCZNzOyi/Px8Jk+eTHh4OEOHDi10jMaOHUv37t0JDw9nxowZAMTHx9O+fXvGjx9Phw4dGDduHBkZ59Yn/frrr+nZsyeRkZFMmTKF/HzzGb3yyiu0bduW/v37s2vXrhLtHDFiBAsWLADg22+/5eabb7bp/Y0ePZpFixYxe/bssne2UtL769ChQ7GfR0nv4+zZs4wcOZKIiAg6derE3LlzAZg1axZdunQhIiKCW2+9tcx2S/pcS/pMhfKhlHIGpgNXAR2Bm5VSHS/Ypw2mMHY/rXU48Ih1ewDwAmYiU0/gBaVU/ZqzXrAXGw+eIiMnn4FtgkvcJzLEH0CGM4VSEcesiuneoj5f3dmL7+/pQ7tGPry8II4Bb/zJ52sO1AoHbejQoRw+fJi2bdty3333sWLFisLX9uzZw/33309sbCz+/v788MMPAMycOZONGzcSHR3NtGnTSE5OBmDXrl3cd999xMXF4evrywcffACYXK+5c+eyZs0aYmJicHZ2Zvbs2WzcuJE5c+YQExPDwoUL2bBhQ4l23nTTTcyZM4esrCy2bt1Kr169ynxvy5cv56GHHmLKlCnlipiV9P5K+jxKeh+//fYbTZo0YcuWLWzfvp3hw4cTGxvLyy+/zLJly9iyZQvvvvtume0W97mW9JkKFaInsFdrvV9rnQPMAcZcsM9kYLrW+hSA1jrRun0YsERrnWJ9bQkwvIbsFuzIqj1JODsp+rQKLHGf+l5uhAV5EXPodM0ZJtQ6LqlFzGuSqNAAZt/Vm/X7k/nvkt3865cdfLRiH/df1pobezTD3cW51OP/9UssO46mValNHZv48sLV4aXu4+3tzcaNG1m1ahV//vknN954I6+99hqDBw8mLCyMyMhIALp37058fDwA06ZNY968eQAcPnyYPXv20KhRI5o1a0a/fv0AmDBhAtOmTePxxx/njz/+YOPGjfTo0QOAzMxMGjRoQEpKCtdccw2eniY/Y/To0SXa2aVLF+Lj4/n2229tdrIGDx7M4MGDbdq3KCW9v5I+j1WrVhX7Pjp37sxjjz3GU089xahRoxgwYACzZs3i+uuvJyjIDH8EBASU2W5xn6uHh0exn6mjk5tvYduRVP4+kMLfB1L47w0R+Hu62duspsDhIs8TMBGworQFUEqtAZyBF7XWv5VwrG3j7EKtZtWek3Rr7o9PGTP0w5v4sjUhtYasEmoj4phVM71aBjJ3Sh/+2neSt5fs5p/zY/lwuXHQbohqhpuL4wUtnZ2dC52Yzp078+WXXzJ48ODC4c2CfTIzM1m+fDlLly5l7dq1eHp6Mnjw4MLK8hcWMy14rrXm9ttv59VXXz3v9Xfeeadcdo4ePZrHH3+c5cuXF0aTqprS3l9xn0dptG3blk2bNrFw4UKee+45hgwZQv36xY9ylfdzLekzdTSycvPZfOi0ccTik9l08DSZ1khyq2Avjp7OcgTHzBZcgDbAYCAEWKmU6lyeEyil7gbuBmjeXMon1GaSz2Sz/WgqU69oW+a+bRr4sGDbMbJy8/FwLb2DLlyaiGNWQ/RtFUSfloGs2ZvMf5fs4rmftvPh8n08PqwtYyObXnSzLSuyVV3s2rULJycn2rQx67nFxMTQokWLEvdPTU2lfv36eHp6snPnTtatW1f42qFDh1i7di19+vThm2++oX///gAMGTKEMWPG8OijjxZGytLT0xk4cCATJ07kmWeeIS8vj19++YUpU6aU2PakSZPw9/enc+fOLF++vGo+gHK8v5Io6X0cPXqUgIAAJkyYgL+/P59++imvvPIK11xzDVOnTiUwMJCUlBQCAgLK/bmW9JmW9t3VBOlZuWw8eKowIrYl4TS5+RqloEMjX27s0YxeYQH0CAsgyNu97BPWDEeAojNeQqzbipIArNda5wIHlFK7MY7aEYyzVvTY5cU1UhNFsoWaYfXek2gNA9qWnF9WQOsG3mgN+5LOEN7ErwasE2ob4pjVIEop+rcJol/rQFbuOclbi3fx6NwtfLX2IC+ODscRStSeOXOGBx98kNOnT+Pi4kLr1q2ZMWMGZ84Uv0rW8OHD+eijj+jQoQPt2rWjd+/eha+1a9eO6dOnM2nSJDp27Mi9994LQMeOHXn55ZcZOnQoFosFV1dXpk+fTu/evbnxxhuJiIigQYMGhcNyJRESEsJDDz1U7GsZGRmEhIRAnpmWPvXJJ5k6dWqZ77/wOCsPPvggeXl5xb6/kujWrVux72Pbtm088cQTODk54erqyocffkh4eDjPPvssgwYNwtnZma5du/LFF1+U+3P19PQs9jOtaccsN9/C5kOnWbk7iVV7kth2JBWLBhcnRecQPyb1D6NXWADdWwQ48qoZG4A2SqkwjKN1E3DLBfv8BNwMfK6UCsIMbe4H9gH/KZLwPxQzSUCow6zacxJ/T1c6Ny3b0WrT0JTk2ZsojplQPOrcaki1l6ioKB0dHW1vM8qNxaL5flMCb/y2i+Sz2cweF0KPyM64Ojve8GZ5iY+PZ9SoUWzfvt2+hhTM7GzXzr52VBFV8bnGxcXRoUOHKrPpcEoGK/cksXJ3En/tTSY9Ow9nJ0XXZv70bRVIr5aBdG3uj6db1fYDlVIbtdZRVXrSc+ceAbyDyR+bqbV+RSn1EhCttf5ZmRD3/2ES+/OBV7TWc6zHTgL+YT3VK1rrz8tqr7ZqmGBSM3q/+gdRoQFMv6Vbmfvn5Fno8M/fuHdQKx4fZoMuFeTFVtOogGAfStMviZjZEScnxQ1RzbiqUyPeW7aXjOxMdh9Pp4GvB0HebrLgtOCQWCyamITTLNlxgqU7TrAn0URTm/rXY1REEwa1DaJPqyBHjoiVidZ6IbDwgm3/LPK/BqZa/y48diYws7ptFByD3SfOcCItm4Gl1C8ripuLE6GBnuxJlMXMheIRx8wB8PFw5R8jOrB1eyye7i4cS83kVEYOTf3r4eVeO7+i0NBQ+0fL6iD2+lzzLZo1e0+yYOsx/tiZyMkz2Tg7KXqFBXBTz+YMahtMq2Av6UwIlxwrdptKKQNKqV92Ia0beLM3sfj0EEGonXf9Ooqrs+lJpWXlcex0JvuSzlDf041Gfh51YnhTqH3sPpHOD5sS+GnzEU6kZePj7sKgdsFc2bEhg9s2wM+z9kbFBKEq+HNnEu0b+dDEv57Nx7Rp4MPSuERy8iwOOTNfsC/imDkYSin86rni4+5CYno2SWeyScvMpaGvB4EyvCnUAGez85i3+QhzNhxi+5E0XJwUg9sF88LVIVzevoFM8RcEK+lZuWyIT+GuAS3LdVybht7kWzTxyWdp29CnmqwTaivimDkoTk6KRn4e1Pd05WhqFkdTM0mp5cObgmMTf/Iss9Ye5H8bD5OelUfHxr68cHVHro5o4kilLATBYViz9yR5Fs1l7WwfxgQzlAlmZqY4ZsKFyB3ewXF3dZbhTaHasFg0K/ck8eVf8SzfnYSzUozo3Jjb+4bSrbm/RGiFOs/x1CycnCDY273c1/ufO5Pw8XChW4vyLYfaMuicYyYIFyKOmYOhtS62snuxw5t+HgR6yfCmYDsF5XHSsnL5PjqBr9Yd5MDJswT7uPPQ5W0Y36s5DXw97GylIFQ/24+k8t8lu/lzVyJaQ0Nfd167rguXtbNtKTOLRbN8dyID2wSXu5Ncz82Zpv71OHDybEVMF+o4NjlmSikvIFNrbVFKtQXaA4usVa+FKsLDw4Pk5GQCAwOLdbaKDm8eOZ3J0dOZpJyV4U3BNrTWHEtM4uDpXK77zx9k5OTTtbk/794UyVWdGtfJJGTRLqE4fo89zkNzNuPt7sKDl7ehvqcr30UncOcXG3hpTCcm9C67MPOmQ6c4kZbNFR0rtiZtWJAX+8UxE4rB1rv5SmCAtZr175jK2DcC46vLsEuRkJAQEhISSEpKsmn/vJx8jmTmctCi8XJzxreeK85OEj07j+PHzaPFYl877IjWkJWXz5msPHadzOLj6NNc1akxt/dtQZcQf3ubV92IdgnnsXDbMe7/ZhNdQvz59LYogn1M/uQNUc148NvNPD9/O60beNO7ZWCp55kfcxR3Fyeu7NioQnaEBXkxP+ZIsaMkwqWNrY6Z0lpnKKXuBD7QWr+hlIqpRrsuSVxdXQkLCyvXMRk5eby/bC+frNqPh6szj13Zlgm9W+Ai+WcG6zJQl2LV7NTMXOb8fYiv1h0k4VQmjf08mNC7BYundiXw0knmF+0SCtl4MIVH5sbQrXl9vr6zF/Xczs0w9nJ34b2buzLqvdU8OjeGRQ8PwN/Trdjz5OVbWLjtGFd0aIh3BUcrwoK8SMvKI+VszqX0exRswNa7t1JK9cH0MhdYt8mceQfA082FJ4e357dHBhLZzJ8Xf9nB1e+vITo+xd6mCXbi5JlsXv9tJ/1eW8ari3bS1L8eH47vxqonL+P+y1pfajcB0S4BgIPJZ5k8ayNN/Dz45Lao85yyArzcXZh2U1dOnsnmpV92lHiuNfuSST6bw+jIJhW2p2WwF4AMZwoXYaur/whmId55WutYpVRL4M9qs0ooN62CvZk1qSe/bT/OS7/uYNxHaxnXPYSnr2ovpQ4uEY6lZjJj5X6+/fsQ2XkWRnRqzL2DW9HJhoWV6zCPINpVq9l9Ip3/RR9my+FUks5k09S/Hv3bBDGhdwubo1WnzuZwx+cb0Frz+R09CfAqPhIG0DnEj3sHtWLasr2M6x5C39YXL7X0v+jD+Hi4MLicZTKKUjAz80DSWXqEBlT4PELdw6arWmu9AlgBoJRyAk5qrR+qTsOE8qOU4qrOjRnULpj3lu3l01X7WRx7nMeHtmN8r+YyvFlHOZh8lo9W7OP7jQlYNIyNbMq9g1sV1kq6lBHtqr0cOHmWl3/dwR87E3FzdqJTU186NPbhYHIGry3ayUcr9vHy2E6M6lJ61Co7L58pX20k4VQmsyf3IizIq8y277usNfO3HOW5n7az8OEB5xVV3nMinQXbjnH3wJa4u1Q8+Nq0fj1cnZVEzISLsHVW5jfAPUA+JnnWVyn1rtb6zeo0TqgYnm4uPDW8PeO6h/DC/Fhe+DmWuRsO8++x4XRvIT2zusL+pDO8/+de5sccxdlJcWOPZkwZ2IpmAZ72Ns1hEO2qObLz8lm7L5nYo2kkpWfj6eZM0/r1GNQ2mJD6tl+TmTn5fLB8Lx+v2I+bixOPD23LLb1anBflijl8mn/9EssD32xm86HT/GNEh2InPlksmie/38rf8SlMu7mrzZEpD1dnXh7biVs/+5v/LIzjpTGdCl97e+luvNxcuGdgK5vfU3E4OylaBHpx4GTtqGWWb9EcPZ1JQ1+PSs/gzsrN58+diazee5JTGTm4uzgXTrjo2swfpwpMYtNak3Aqk9ijaZzKyMGiNQ19PGjVwJvQQM9KTbDIybNwJjsPZ6XwdHeu9hqitg5ldtRapymlxgOLgKeBjYCImwPTKtibr+7syaLtx/n3rzu47sO1XN89hKdkeLNWs+dEOu//uZdfthzFzcWJiX1DmTKwpdQfK546pV3zY46wYlcS/p5uNA+oR4fGvrRv5FvmmqVaa46mZrEtIZXYo6lsP5LK7hNnOJ2RQ77W+Hq4EhroRaemfgxqF0yvsACblt46k53H8l2JLI49wZ87EzmTnQeAj7sLmbn55FlM3byoFvW577JWXNauQYk3SK01S3ac4F+/7ODI6Uyu6dqUZ65qX+x1HdnMn7l39+E/C+P4bPUBDqVkMO2mrufljWXn5fPk91uZH3OUJ4a1Y3RE+fLBBrQJ5q7+YXy6+gA9wwIY1aUJS3acYOG24zw0pA31SxkOtZWwIC/2Jzl2xCwxPYt//bKDFbuSOJOdh4uTokuIH7f1CWVkl8blclK01vy46QivLIwj5WwOPh4uNPBxJzMnn3mbjwCmntzYrk25vnsIrRuUvSrCyTPZzNt0hO+iD7OnhIK9DXzc6d0ykN4tA+nfOojmgSV3FM5m5xF98BRr9yUTc/gUu0+cIeVsTuHrTgqaBXgSFuRFWJAXHRr70rGxL20aelcqgloUWx0zV6WUKzAWeF9rnauU0lVigVCtKGsl90Ftzx/efGJYO27p1ULKa9Qidh5P471le1m47Rj1XJ2ZPLAlkwe0FCe7dOqUdh1PzWL9gRROZeSQkZNfuL2xnwetG3jTKtiblsFeeLg4k52Xz4m0bOKOpbElIZWTZ7IBE6lp08CbHqH1CfR2x0nB6Yxc9iadYfb6g8xccwBPN2f6tw5iSIcGDGrbgIa+piq+1pr45Az+PpDM4tgTrN57kpw8C4Febozq0phh4Y3oGRaAl7sLFutakL/vOMFXaw8y6Yto2jfy4Z5BrbiyY8PC2osWi2bNvpN8tGIfa/Ym07ahN3Pu7l1muQo3FydeHB1OaKAn//p1ByOnreKpq9oTEeLPzuNpvL10D1sOn+bJ4e24d1DFoltPDm9P9MFTPPDNZj5ZuZ8tCal0bOzLXQPKN3u+JFoGe7FiVxL5Fl1lWhx7NJVv/z6EXz1XbusTSsNKdNjW7kvmgW82cTYnj2u7hdCxsS9HTmeyOPY4j8yNYdqyPTw/qqNNRXlTM3OZOjeGP3Ym0r1Ffd65MZK+rQILU2xSM3JZvjuRX7Yc49NVB/h4xX4im/lzfVQIo7o0wa/euc5HVm4+q/ec5LvowyzbmUieRdO1uT//Gh1ORDN/c72iOJaaSdyxdNbtT2bt/mR+3nIUgKb+9egS4kfzQE/867mRlZvP8dQs4o6nseNoGnkWjYuTIrypH0M7NqSpfz28PVywaJOveCD5LPEnz7J+fwqZueZ36OKkuGdQKx4f1q7Cn3cBqqASeKk7KfUQ8BSwBRgJNAe+1loPqLQFVUBUVJSOjo62txm1gr2J6bzwcyxr9iYT3sSXl8Z0ons5lxOpVQwebB5rcbmM+JNneXPxLhZsO4a3uwu3923Bnf1blprAfCmglNqotY4qYx+H1q4CyqthWmsS07PZcSyNncfS2XU8jb1JZ9iXeLbwRgGmd98y2JsuIX5ENvOnc1M/OjT2LTEalpVrhiP/2HmCZXGJHE3NAsDTzRkvdxfSs3LJyjU1AZv612NYeCOGd2pE9xb1S3UscvMtzI85ygfL97I/6SxuLk60tUYYdp9IJz0rj2Afd6YMbMntfUPLPVS0ak8SL/wce170qZGvB8+O7MDV5YyUXUh6Vi6z1h7kh00J9G4ZyD9HdbQpmmgLczcc4qkftrHqyctKTkEoh4Z9tGIfry3aiYerEzl5FpydFG+M68I1XUPKbduu4+lc9+FfNPLz4MPx3WhTZE1Pi0Xzx85EXl0Yx/6TZ7msXTDPj+pIy+Di81rjjqVx/+xNHErJ4NmRHbi9T2ipw5VJ6dn8tPkI/9t4mN0nzuCkTHQx0MudM9l57ElMJzdfE+TtxnXdQrg+quzomtaafUlnWLM3mb/jU4g9ksrR01nk5JvrOcDLjfaNfIho5k+floFEhdbH06302FW+RXMw+Sw7jhmHrmvz+lzZsWGpxxRQmn7Z5JiVcFIXrXVehQ6uYsQxKx9aaxZsO8bLv8ZxPC2Lm3o04+mr2pdYs6dWU4sds5SzOUz7Yw+z1x/ExcmJuwaEcWf/sLr5PVUAWxyzEo5zGO0qoKo0zGLRnDybTW6+xtVZEejlXuFIjNaancdNtOFwSiaZuXl4ubnQqoE3XZv7066hT7nzdvItmg3xKfwee4L45LNk5OTRMtibXmEBDO/UqFJDQbn5Fv7cmUiydYhsaMdGDr+axd8HUrjh47V8Oakng9qWMMPTRg3bm5jOiHdXM6hdMG+NiyA1M5enftjKugPJvDUuguu62+6cpZzNYfT7q8nJszD/gX409qtX7H45eRa+/CueaX/sITM3nzv6hTJ5wLm0itMZOXy19iDTlu3Br54bH4zvRs8w2/OctdZsO5LK0rhEdh5LIy0rFw9XZ9o18qF3WCD92wRVKt9La01GTj4ers41PnpUmn7ZmvzvB7wADLRuWgG8BKRWiYVCjaKUYlSXJlzWrgHT/tjDp6sPsGTHCZ4f1ZExkU2kCrWdycrN5/M18Xzw517O5uRxY4/mPHpFG8khqwAV1S6l1HDgXUzNs0+11q9d8PpETJ7aEeum97XWn1pfywe2Wbcf0lqPrvw7sQ0nJ0UDn6q5TpRSdGjsS4fGvlVyPjDDqAW5PlWNq7MTQ8MrVoXfXhTMEN2fdKZkx8wGLBbNMz9uo56bM/+5pjN+nq74eboyc2IPJs+K5skfttLYz6PY0h/F8fxP2zmRlsX39/Qt0SkDM5w8eWBLxnZtyluLd/Hp6gN8suoAbRt64+LkxN6kM+TkWbiqUyNeuaZzuaP8Sim6hPhX2wolSimHXM7QVotmAtuBG6zPbwU+B66tDqOEmsHL3YVnRnRgTGRTnpm3jUfmxvDDpgReHtuJFoFlTykXqp6Vu5N49qdtHE7JZEj7Bjx9VfvzhhCEclNu7VJKOQPTgSuBBGCDUupnrfWFFUfnaq0fKOYUmVrryMoaLtR9grzd8HF3qfRi5mv2nWRD/ClevbZz4RJTYBZL/+jW7lwzfQ33f7OJnx/oX+as7V+3HmXBtmM8MawdEc38bWo/2Med18d1YfLAliyOPc6mg6dQyjjh47qH0LFJ1Tn3lwK2OmattNbXFXn+L1nWpO7QsYkvP97bl9nrD/LGb7sY+vZKHhrShskDWjr8UEBdIflMNi8viGPe5iO0DPJi9l296Gdj71YolYpoV09gr9Z6P4BSag4wBii5FLwgVAClFC2DvSrtmM3bfAQfDxeu6dr0ote83V2YcVsUo99fzZSvNvLDvX2LXfUATG7X8z9tJyLEjykDW5bbjtYNvGndoHW5jxPOx9a7bqZSqn/BE6VUPyCzekwS7IGzk+K2PqEsnTqIy9s34M3Fu7j6vdVsPChLO1U3f+5K5Mq3V/Lr1qM8dHlrFj48QJyyqqMi2tUUOFzkeYJ124Vcp5TaqpT6XinVrMh2D6VUtFJqnVJqbEUNFy4NKlsyIyMnj9+2H2dk58YlTkoIC/Ji2s1diTuextM/bqW43HKtNf+Yt42zOfn83w0RUpDcjtj6yd8DTFdKxSul4oH3gSnVZpVgNxr5efDhhO58elsU6Vm5XPfhWp6dt62wPpFQdVgsmv/+vos7Pt9AAx93Fjw0gKlD21XZjC8BqD7t+gUI1Vp3AZYAXxZ5rYU1qfcW4B2lVLG1GpRSd1sduOikpKQqMEmojYQFeXM0NZOsIrNpy8PvsSfIyMkvNlpWlMvaNeDxoe2YH3OU1xbtvMg5m7vhMEt2nODxoW1tqh8mVB+2Lsm0BYhQSvlan6cppR4BtlajbYIduaJjQ/q0CuS/S3Yzc80Blu9K4s1xXWxOHhVKJzMnn8f+F8PCbccZ1z2El8d2EoesGqigdh0BikbAQjiX5F9w3uQiTz8F3ijy2hHr436l1HKgK7CvGNtmADPAzMq0+U0JdYqwYC+0hvjks7RvVP5crHmbj9DUv55NqxrcN7gVx1Oz+HjlfjJz83l8WDt8PVz5Lvowz8zbRr/WgdzZv/xDmELVUq7pCFrrtCJPpwLvVKk1gkPh5e7C86M6MqJzIx7/31Zu+XQ9t/ZuwdNXtXfImSy1hcS0LO6aFc22I6k8O6IDdw0Ik5mw1Uw5tWsD0EYpFYZxyG7CRL8KUUo11lofsz4dDcRZt9cHMrTW2UqpIKAfRZy28pCbm0tCQgJZWVkVOVywMx4eHoSEhODqWvqqDC2tMzMPJJXfMcvMMXXnbu3TwqZljJRSvDQmHFdnJ2auOcC8zUfwdHPmRFo2A9oEMePWKCk67gBU5u4q394lQvcWASx8aABv/b7LRM92J/LmuIhqmfJe19l+JJXJs6JJzcxlxq1RNhcjFKqUUrVLa52nlHoAWIwplzFTax2rlHoJiNZa/ww8pJQaDeQBKcBE6+EdgI+VUhZMqshrxczmtImEhAR8fHwIDQ0Vx72WobUmOTmZhIQEwsJKXyWgsGRGBSYA/B2fQk6+hQFtbB/JUErxz6s7cm23psxcfYB8reneoj43RDWTqL2DUBnHTELvlxD13Jx5flRHhoU34onvt3DTjHVM7BvKk8PblVkdWTAsjj3OI3Ni8Pd05ft7+soUcvtRpnZprRcCCy/Y9s8i/z8DPFPMcX8BnavARrKyssQpq6UopQgMDMSW3EEvdxca+rpXaGbmmr0ncXN2oldY+TvJnZr68d8bI8t9nFD9lHpHVUqlU7yIKaDkqnNCnaVnWACLHh7AG7/t4ou/4lm+K5E3r4+wKb/hUsVi0byzdDfTlu0lopk/n9zaXYrFVjN1RbvEKau9lOe7axnkzd4SFuAujVV7TtK9Rf0Sy18ItZNSZ2VqrX201r7F/PlorSVMconi6ebCi6PD+XZyb/Ismhs+Xsu/f91R4VlFdZnUzFzumhXNtGV7Gdc9hLl39xanrAYQ7RJqE+0a+bD7RDoWi+0DUUnpZoH6/uUYxhRqB3YpVKKUelMptdNaA2ieUsq/yGtdlFJrlVKxSqltSim5izkofVoFsviRgUzo1YLPVh9gxLur2HjwlL3Nchj2nEhn7PQ1rNydxL/HhPPmuC6SwyEIwkW0b+RDRk4+h09l2HzMX/tOAtBfZsrXOexVQW4J0MlaA2g31lwNpZQL8DVwj9Y6HBgM5NrJRsEGvNxd+PfYTsy+qxfZeRau/+gvXl0Yd8lHzxbHHmfs9DWkZ+XyzeTe3NpHcoWES4e4uDjuuecexo0bx4cffmhvcxye9tb1SHceT7f5mHX7k/H1cKFTU7/qMkuwE3ZxzLTWv2utCyqWrsPUCQIYCmy11h5Ca52stb607/C1hH6tg/jtkQHc2KM5H6/cz8hpq1i+K7HYCtN1mYKisVO+2kjrBt788mB/eoZJ/p1Qe1FKMWHChMLneXl5BAcHM2rUKAC8vb0vOqZDhw589NFHfPfdd6xZs6bY8xZ3XGV58cUXeeutt6r8vNVN24beKAU7j9numK0/kEKP0AApb1EHcYQ1FyYBi6z/twW0UmqxUmqTUupJO9ollBMfD1devbYzX07qSW6+ZuLnG7j1s7+JPZpqb9NqhKL5ZNd3D2HulD409qs1eeaCUCxeXl5s376dzEyzktWSJUto2rT0KvMAP//8MyNHjmTEiBHVbWKtx9PNhRYBnuw6kVb2zpj8sv1JZ+khnb46SbU5ZkqppUqp7cX8jSmyz7OYOkCzrZtcgP7AeOvjNUqpISWcX5YzcVAGtQ1mydSBPD+qI9uPpjLqvdXcPSua1XtO1tkIWuzRVMa8v7own+wNyScT6hAjRoxgwYIFAHz77bfcfPPNZR4zevRoFi1axOzZs8vct4CxY8fSvXt3wsPDmTFjBgDx8fF06NCByZMnEx4eztChQwudRIBXXnmFtm3b0r9/f3bt2gXA2bNnGTlyJBEREXTq1Im5c+cCMGvWLLp06UJERAS33nprme22b9+e8ePH06FDB8aNG0dGhskB+/rrr+nZsyeRkZFMmTKF/PzKD+y0a+Rjc8QsOt6sYSzR+LpJtTlmWusrtNadivmbD6CUmgiMAsbrc3frBGCl1vqk1joDU0eoWwnnn6G1jtJaRwUHB1fX2xAqiLuLM3f2D2PF45dx/+DWRB88xYTP1jPkvyv4Ys2BOrP2ptaar9Yd5JoP/iIzN59v75Z8MqHucdNNNzFnzhyysrLYunUrvXr1KnX/5cuX89BDDzFlypRyRcxmzpzJxo0biY6OZtq0aSQnm5Wv9uzZw/33309sbCz+/v788MMPAGzcuJE5c+YQExPDwoUL2bBhAwC//fYbTZo0YcuWLWzfvp3hw4cTGxvLyy+/zLJly9iyZQvvvvtume3u2rWL++67j7i4OHx9ffnggw+Ii4tj7ty5rFmzhpiYGJydncvlfJZE+0a+xCefJTOnbCdv/YEU6rk606mJ5JfVRewybVwpNRx4EhhkdcAKWAw8qZTyBHKAQcDbdjBRqCL8PF15fFg7Hri8NQu3HePLtQd58ZcdvPX7bq6PCuH2PqGEWitf1zbSs3J5+sdtLNh6jIFtg3n7hggCvd3tbZZQB/nXL7HsOGrbMJetdGziywtXh9u0b5cuXYiPj+fbb7+1ydEaPHgwgwcPLrdN06ZNY968eQAcPnyYPXv20KhRI8LCwoiMjASge/fuxMfHA7Bq1SquueYaPD09AROlA+jcuTOPPfYYTz31FKNGjWLAgAHMmjWL66+/nqAgM4sxICCgzHabNWtGv379AJgwYQLTpk3Dw8ODjRs30qNHDwAyMzNp0KBBud/rhbRv5INFw57EdLqE+Je6798HUuja3B83F0fIRhKqGnvV83kfcAeWWCML67TW92itTyml/otZq04DC7XWC+xko1CFeLg6c223EK7tFsLmQ6f48q94vl53kC/+imdsZFMeuaINLQJrj4O2/Ugq93+ziYRTmTw5vB33DGxl01p1glBbGT16NI8//jjLly8vjChVJcuXL2fp0qWsXbsWT09PBg8eXLhOqLv7uQ6Ps7PzeUOZxdG2bVs2bdrEwoULee655xgyZAj169cvd7sXRr6VUmituf3223n11Vcr83YvomBmZtyxtFIds9TMXOKOp/HwkDZV2r7gONjFMdNaty7lta8xJTOEOkrX5vXp2rw+/xjRgc9WH+DLtfH8suUo10c145Er2tDQwQuwfrfhMM/N306Apxtz7u4tqx4I1Y6tka3qZNKkSfj7+9O5c2eWL19e5edPTU2lfv36eHp6snPnTtatW1fmMQMHDmTixIk888wz5OXl8csvvzBlyhSOHj1KQEAAEyZMwN/fn08//ZRXXnmFa665hqlTpxIYGEhKSgoBAQGltnvo0CHWrl1Lnz59+Oabb+jfvz9DhgxhzJgxPProozRo0ICUlBTS09Np0aJFpd5/iwBP/D1d2XjwFDf2aF7ifn8fSEFrKrQMk1A7kArYgt1o4OvBMyM6cGf/MKb/uZdv/j7E/Jgj3De4FXcNaOlwyfPZefm8+PMOvv37EP1bB/HuTZEydClcMoSEhPDQQw9dtD0jI4OQkJDC51OnTmXq1Kllnu/C4x588EHy8vLo0KED7dq1o3fv3mWeo1u3btx4441ERETQoEGDwuHFbdu28cQTT+Dk5ISrqysffvgh4eHhPPvsswwaNAhnZ2e6du3KF198wfDhw/noo4+Kbbddu3ZMnz6dSZMm0bFjR+699148PT15+eWXGTp0KBaLBVdXV6ZPn15px8zJSRHVIoAN8aUX6V6z9yQerk50a+FfqfYEx0XVhVlyUVFROjo62t5mCJXkUHIGryzcweLYEzT1r8cLV3dkaHijyp20IM+lkj38Y6mZ3PP1JrYcPs19g1vx2NB2Uj/IziilNmqto+xtR1VQnIbFxcXRoUMHO1kkxMfHM2rUKLZv317hc5T3O/x4xT5eXbSTDc9eQbCPtdN3gYYNfXsFDX09+OrO0idgCI5NafolmYOCw9A80JOPb43im7t64ePhwt1fbWTqdzGkZ9l38Ye/D6Rw9Xur2Zd4ho8mdOfJ4e3FKRMEocopqEtWUA7jQhLTs9h94gx9W8kyTHUZccwEh6Nv6yB+fqA/D13emvkxR7n2g784lGz7GnJVhdaaWWvjueWTdfh6uPLT/X0Z3qmSETxBEGoFoaGhlYqWVYROTfzwcHUqcThz7T4z6aJfa8kvq8uIYyY4JG4uTkwd2o6vJvUkMT2bMdNXE3P4dI21n5Wbz5Pfb+Wf82MZ1DaYnx7oR+sGPjXWviAIlx5uLk5ENvNnQwkRs7/2mvUxw6V+WZ1GHDPBoenbOoif7u+Hj4crt3yyjlV7qn+Vh0PJGdw4Yx3/25jAQ5e35pPbovD1cK32dgVBEHqEBhB7NJXTGTnnbc/Lt7BsVyL9WgdJKkUdRxwzweEJC/Li+3v60DzAk0lfbODXrUerpR2tNbPXH2T4uyvZb80nmzq0ndQnuwRRSg1XSu1SSu1VSj1dzOsTlVJJSqkY699dRV67XSm1x/p3e81aLtR2hoU3wqLhly3n69yqvSdJSs9mTGQTO1km1BTimAm1gga+Hsyd0oeuzerz4Leb+WptfJWe/+jpTG7/fAPPzttO1+b+/PboQMknu0RRSjkD04GrgI7AzUqpjsXsOldrHWn9+9R6bADwAtAL6Am8oJQqvrKpIBRDp6Z+dGjsy3fRCedt/2FjAv6erlzWvvKrDAiOjThmQq3Br54rs+7syZD2DXh+fiz/XbIbi6Vy5V6ycvN5d+keLv+/5fx9IJmXxoTz1aReNPWvV0VWC7WQnsBerfV+rXUOMAcYY+Oxw4AlWusUrfUpYAkwvKKG1IVyRpcqlfnubogKYduRVOKOmWW48iya33ecYHREE9xdHKu+o1D1iGMm1Co8XJ35aEJ3xnUPYdofe7jpk3XsOJpWbhHMzbfw0+YjDPm/Fby9dDdD2jdkyaODuK1PqAxdCk2Bw0WeJ1i3Xch1SqmtSqnvlVLNynlsmXh4eJCcnCzOWS1Ea01ycjIeHhVbxWRMZFNcnRUzVx9AY+oo5uRZuK5bSJnHCrUfqfwv1DpcnJ14c1wXeoYF8O9fdzBi2irCgrxoFuCJj4cLvh6u+Hq44OPhwri0LJydFDE7TuDsBOlZeWw6eIrfYo9zIi2bDo19+b8bIujdUqafC+XiF+BbrXW2UmoK8CVweXlOoJS6G7gboHnzi5fgCQkJISEhgaSk6p/wIlQ9Hh4e561sUB4CvNwY36sFX/wVz+1HUzmTlceYyCZ0CZHZmJcC4pgJtRKlFDdENWNI+wYs2n6c5bsSSUrPJiElg7SsPNKzcsnOsxB18iwAk2edq6pez9WZvq0Cee3aFgxqGywRMuFCjgDNijwPsW4rRGtddBXvT4E3ihw7+IJjlxfXiNZ6BjADTOX/C193dXUlLCysfJYLdYYXru6IXz1XznyTR6C3O/93fcRFi6oLdRNxzIRaTaC3OxN6t2BC74vXqcvJs6DWvE6eRfPzA/2waOOUtQz2wtVZRvGFEtkAtFFKhWEcrZuAW4ruoJRqrLU+Zn06Goiz/r8Y+E+RhP+hwDPVb7JQ11BK8eiVbclqXh93FyeUaNYlgzhmQp3FzcUJnJ1wdYYuIf72NkeoJWit85RSD2CcLGdgptY6Vin1EhCttf4ZeEgpNRrIA1KAidZjU5RS/8Y4dwAvaa2LrxYqCDbg4SIO2aWGOGaCIAgXoLVeCCy8YNs/i/z/DCVEwrTWM4GZ1WqgIAh1FlUXZvwopZKAgzXYZBBwsgbbKwlHsENsEBsupKbsaKG1Dq6BdqqdS1TDxAbHsQEcw45LyYYS9atOOGY1jVIqWmsdJXaIDWKD49ohlIwjfEdig+PY4Ch2iA0GGbwWBEEQBEFwEMQxEwRBEARBcBDEMasYM+xtgBVHsENsMIgN53AUO4SScYTvSGwwOIIN4Bh2iA1IjpkgCIIgCILDIBEzQRAEQRAEB0Ecs0qglHpQKbVTKRWrlHqj7COqzY7HlFJaKRVkp/bftH4OW5VS85RS/jXU7nCl1C6l1F6l1NM10WYxNjRTSv2plNphvQ4etocdVluclVKblVK/2ql9f+uC3juVUnFKqT72sEOwHdEw++mXtW27apjo13ntO4x+iWNWQZRSlwFjgAitdTjwlp3saIZZ9uWQPdq3sgTopLXuAuymBpagUUo5A9OBq4COwM1KqY7V3W4x5AGPaa07Ar2B++1kB8DDnFsayB68C/ymtW4PRNjZFqEMRMMKqXH9AofRMNGvcziMfoljVnHuBV7TWmcDaK0T7WTH28CTgN2SBbXWv2ut86xP12EWbq5uegJ7tdb7tdY5wBzMTaZG0Vof01pvsv6fjvkxN61pO5RSIcBIzILaNY5Syg8YCHwGoLXO0Vqftoctgs2IhmE3/QIH0DDRr8L2HUq/xDGrOG2BAUqp9UqpFUqpHjVtgFJqDHBEa72lptsuhUnAohpopylwuMjzBOwgKEVRSoUCXYH1dmj+HczNzWKHtgHCgCTgc+twxKdKKS872SLYhmjYxdSUfoGDaZjol+Pol6yVWQpKqaVAo2Jeehbz2QVgwr89gO+UUi11FU9zLcOGf2CGAKqd0uzQWs+37vMsJjQ+uyZsciSUUt7AD8AjWuu0Gm57FJCotd6olBpck20XwQXoBjyotV6vlHoXeBp43k72CIiG2WKD6JfoFw6mX+KYlYLW+oqSXlNK3Qv8aBWxv5VSFswaW0k1YYNSqjPGy9+ilAITft+klOqptT5elTaUZkcReyYCo4AhVS3sJXAEaFbkeYh1W42jlHLFiNpsrfWPdjChHzBaKTUC8AB8lVJfa60n1KANCUCC1rqgt/09RtgEOyIaVroNRWyZSM3qFziIhol+AQ6mXzKUWXF+Ai4DUEq1BdyowcVXtdbbtNYNtNahWutQzIXVrTqcsrJQSg3HhKFHa60zaqjZDUAbpVSYUsoNuAn4uYbaLkSZO8pnQJzW+r813T6A1voZrXWI9Tq4CVhWw6KG9bo7rJRqZ900BNhRkzYI5eYnRMPspV/gABom+lVog0Ppl0TMKs5MYKZSajuQA9xegz0tR+N9wB1YYu35rtNa31OdDWqt85RSDwCLAWdgptY6tjrbLIF+wK3ANqVUjHXbP7TWC+1gi715EJhtvcnsB+6wsz1C6YiGGWpcv8BhNEz06xwOo191ovJ/UFCQDg0NtbcZgiOya5d5bNeu9P2EWsfGjRtPaq2D7W1HVSAaJpSIaFidpDT9qhMRs9DQUKKjo+1thuCIDB5sHpcvt6cVQjWglDpobxuqCtEwoUREw+okpelXnXDMhDpKfh789hR4BkLHsdDQXnUPBUEQKsDW7+DgGmh9BbQbCU6S1i2UjVwlguMSvxI2fAorXodPLoP0E/a2SBAEwTYsFvj9edj4BcydANGf2dsioZYgjpnguOyYD65eMOFHyMuCw+vsbZEgCIJtHF4PZ47D2I8goCXs+9PeFgm1hDqR/B8VFaUlP6P2kZubS0JCAllZWRe/qDWkHQUXdzOUmZoA7j5Qz798jRy3zrxvVFxtSaE24OHhQUhICK6urudtV0pt1FpH2cmsKkU0rPZRqn4BZJ6C7DPg19T8n5tl/i8vomG1morol+SYCXYjISEBHx8fQkNDsU5TP0d2OiRnQ/0w44wlWS/V4Lbla6Qgp0NmNNVKtNYkJyeTkJBAWFiYvc0RhEJK1S+t4UQsuDaCwJZwNsl0Lhu0NJ3N8iAaVmupqH7JUKZgN7KysggMDLxY1AAyT4NyMlEyADcvyM0Aba+l1AR7oJQiMDCw5KiEINiJUvUr5yxYcs9F+N28zm0XLhkqql/imAl2pVhRAxMxc/MBJ2fz3M0L0JCbWWO2CY5BideIINiZEq/NnDPm0cPXPLrUMx1NccwuOSqiX+KYCY5Hfh7kZ4Ob57ltBf9XsbAppZgw4dzqH3l5eQQHBzNq1CgATpw4wahRo4iIiKBjx46MGDECgPj4eOrVq0dkZGTh36xZsy46/+DBg6u9PlV1tLF8+fLCz0AQhHKScxZcPMDJmoKhFLh6Qq7oV020Udv1q8ZzzJRSMzGLxSZqrTsV87oC3gVGABnARK31ppq1UrArudbl6grC/wDObuavih0zLy8vtm/fTmZmJvXq1WPJkiU0bXouQfef//wnV155JQ8//DAAW7duLXytVatWxMTEVKk9gmMj+iWUidZGpzz8zt/u5gVnToAl/9xIQCUR/aqb2CNi9gUwvJTXrwLaWP/uBj6sAZsER6LA+XL1PH+7iwfkZVd5cyNGjGDBggUAfPvtt9x8882Frx07doyQkJDC5126dKl0eykpKYwdO5YuXbrQu3fvQrFcsWJFYe+1a9eupKenc+zYMQYOHEhkZCSdOnVi1apVNrURHx/PgAED6NatG926deOvv/4CTE9y8ODBjBs3jvbt2zN+/HgKZmb/9ttvtG/fnm7duvHjjz8Wnqs4uwBef/11OnfuTEREBE8//TQAn3zyCT169CAiIoLrrruOjAzjZE+cOJF77rmHqKgo2rZty6+//gpAfn4+TzzxBD169KBLly58/PHHlf58q5kvEP0SSiM/B3T++R1LANd6516vQkS/6p5+1XjETGu9UikVWsouY4BZ1sV01yml/JVSjbXWx2rGQsEuLHoajm8z/+dlmiR/1wuELT8b8nOtgmfDuH2jztCy7HVob7rpJl566SVGjRrF1q1bmTRpUqGA3H///dx44428//77XHHFFdxxxx00adIEgH379hEZGVl4nvfee48BAwaU2d4LL7xA165d+emnn1i2bBm33XYbMTExvPXWW0yfPp1+/fpx5swZPDw8mDFjBsOGDePZZ58lPz+/UCjKokGDBixZsgQPDw/27NnDzTffXDhcsHnzZmJjY2nSpAn9+vVjzZo1REVFMXnyZJYtW0br1q258cYbC89VnF2LFi1i/vz5rF+/Hk9PT1JSUgC49tprmTx5MgDPPfccn332GQ8++CBgxPbvv/9m3759XHbZZezdu5dZs2bh5+fHhg0byM7Opl+/fgwdOtRhZ2CKfgnFUlS/LLmm7qKrJ6gikTGdb0YDXOqdG+IsjUad4arXytxN9Kvu6ZcjlstoChwu8jzBuk2E7ZJAW0P9xV2ayrxexXTp0oX4+Hi+/fbbwhyMAoYNG8b+/fv57bffWLRoEV27dmX79u1AxYcCVq9ezQ8//ADA5ZdfTnJyMmlpafTr14+pU6cyfvx4rr32WkJCQujRoweTJk0iNzeXsWPHniekpZGbm8sDDzxATEwMzs7O7N69u/C1nj17FvaiIyMjiY+Px9vbm7CwMNq0aQPAhAkTmDFjBkCxdi1dupQ77rgDT08T1QwICABg+/btPPfcc5w+fZozZ84wbNiwwnZvuOEGnJycaNOmDS1btmTnzp38/vvvbN26le+//x6A1NRU9uzZ47COmQ2Ifl3qaAugTLJ/UQqSwKt4ZrnoV93TL0d0zGxCKXU3ZqiA5s2b29kaodIU9AxzsyApDvyagVfQ+ftknoZTByCo3fkTA0pj1y6bdhs9ejSPP/44y5cvJzk5+bzXAgICuOWWW7jlllsYNWoUK1eupHv37ra1Xw6efvppRo4cycKFC+nXrx+LFy9m4MCBrFy5kgULFjBx4kSmTp3KbbfdVua53n77bRo2bMiWLVuwWCx4eHgUvubufq6OkrOzM3l5eeW2qyQmTpzITz/9REREBF988QXLiyy8fOHsJKUUWmvee++98wTwUkE0rA5RNLKVtMs4ZUFtzt9Hazi+1RTM9guhKhH9Kp9dJeEo+uWIszKPAM2KPA+xbjsPrfUMrXWU1joqODi4xowTqpmCWUsX5meASf6HKs/RAJg0aRIvvPACnTt3Pm/7smXLCsPv6enp7Nu3r9I30QEDBjB79mzA5EwEBQXh6+vLvn376Ny5M0899RQ9evRg586dHDx4kIYNGzJ58mTuuusuNm2yLY88NTWVxo0b4+TkxFdffUV+fn6p+7dv3574+Hj27dsHmFyVAoqz68orr+Tzzz8v/GwKhgLS09Np3Lgxubm5he+xgP/9739YLBb27dvH/v37adeuHcOGDePDDz8kNzcXgN27d3P2bK0uKWCTfoFoWJ3EYjElfYrrOCplNEz0q0wudf1yxIjZz8ADSqk5QC8gVfIzLiFyMkxehovHxa9Vo2MWEhLCQw89dNH2jRs38sADD+Di4oLFYuGuu+6iR48ehSJQNDQ/adKkYs8xcuTIwuU4+vTpw8cff8ykSZPo0qULnp6efPnllwC88847/Pnnnzg5OREeHs5VV13FnDlzePPNN3F1dcXb27vYKe3FtfGf//yH6667jlmzZjF8+HC8vIpxdItQkA8ycuRIPD09GTBgQGGSbHF2ubu7ExMTQ1RUFG5ubowYMYL//Oc//Pvf/6ZXr14EBwfTq1evwnOAiQr17NmTtLQ0PvroIzw8PLjrrruIj4+nW7duaK0JDg7mp59+KtVWB0f061ImLxPQF+fHFuDsBnmiX2W1canrV42vlamU+hYYDAQBJ4AXAFcArfVH1unm72NmPmUAd2itSy1yIuvM1U7i4uLo0KHD+RsTd5qp5BcOA0DFhgIKhjJlORO7MnHiREaNGsW4ceMqdHxx14o91sqsDv0C0bDaSLH6dSYR0o5Aw/BzHcminD5s1s1sXI7ZkaJhdqem9cseszJvLuN1DdxfQ+YIjoQl3/Q4vRsW/3rBUEA19DgFwRZEv4RSyTkLTq7FO2UALm5mdqYlz7aZmcIliVwZguNQsNxScfllBVRTjoZQvXzxxRf2NkEQqp/cjLL1C0zZH3HMag01rV+OmPwvXKoUJP5fWFi2KC7u1gKONTsELwiCUCr5uUabbHHMJOovlII4ZoJdOS/HMeesdekl15IPcLYOBejSZ+kIdYeazoMVBFu5SL+g9I5lYcSs6lcwERyTiuiXOGaC3fDw8CA5OfnchZuTUfJspgKkx3lJobUmOTn5vDpGguAIXKRfuRmAKt0xc3IxNc4kHeOSoKL6JYPcgt0ICQkhISGBpKQkkwybdhTq1YcTmSUflJ8D6Ylw0lK6ABZw/Lh5tFRttW2h5vDw8DhvvT9BcATO0y8wMzK1BVLLKGqdngxOaeCVXvp+BYiG1Woqol/imAl2w9XV9dzSFTHfwOJ74Z410KhDyQdlpMAbA2HYq9DnvrIbufde81ikgrMgCEJlOU+/cjPhtYHQawr0fLn0A79+Hs4mwpSVtjUkGnbJIUOZgmNwYBXUC4AGHUvfr159cHaH9KM1Y5cgCEJZHF5vovmhA8ve17cxpEnNYaFkxDET7I/WEL8KQvuDUxmXpFLg20SETRAEx+HAKrNiSYs+Ze/r2xTOJplZnIJQDOKYCfbn9EFIPQxhNvQ2wThm6eKYCYLgIMSvhiZdwd2n7H19GgMa0o9Xu1lC7aTCjplSyksp5WT9v61SarRSqpQ6B4JQAgdWmcfQ/rbt79PYTBQQhEogGiZUCTln4chG2/XLt4l5lM6lUAKViZitBDyUUk2B34FbgS+qwijhEiN+FXgFQ3B72/b3tTpmUt9KqByiYULlObQOLLkQNsC2/X0am8e0I9Vnk1CrqYxjprTWGcC1wAda6+uB8KoxS7hksOTD3qXQcrDJH7MF36amQGPmqWo1TajziIYJlWfPEjMhqbkN+WVwLmImebJCCVTKMVNK9QHGAwus25wrb5JwSXFkI2QkQ9vhth9T2OOU4UyhUoiGCZVDa9i9CFoOKn0ppqLUqw8uHjKzXCiRytQxewR4BpintY5VSrUE/qwSq4RLh12LzGym1lfYfkxhj/MoNOpUtfYkbIRDa83/kbeAZ0DVnl9wJB5BNEyoDCf3wKl46PuQ7ccoVX15shkpsP0HyE6DsEEQElX1bQjVToUdM631CmAFgDWB9qTWuhxXpyAAu3+DFn2hnr/txxREzKq6x5m8Dz6/6tw6dus/guu/EHGro4iGCZVm9yLz2HZY+Y6rjpI/WsN3t5mcXTBLP13+PPR7pOwyRIJDUZlZmd8opXyVUl7AdmCHUuqJqjNNqDVknqrY2pWnDkLijvINYwL4NAJU1Qqb1vDro+DiDg/FwF3LTM/2i5Gwf3nVtSM4DKJhAmDW6M1Or9hkot2LoVFn8CvnkmG+Taq+Y7nlW+OUXfUmPLEfOo6BP/4FCx+XiVK1jMq40R211mnAWGAREIaZ1SRcKuz9A74YBa+HwsvB8OkVcPak7cfvWmgey+uYObuCd4OqFbbYeXBgBQz5JwSEQUh3mLwcAlrCtzfDwb8qfu78vPJ9LkJNIRp2KZOaAAsehzdawqshRsd2/2778WdPmhmZ5dUvsA5lHqs6hykrDX5/DkJ6Qo+7wCsQxn1uhlijP4PF/6h4W1rD2WQpiFuDVCbHzNVa82cs8L7WOlcpJW65I5O0yywdcioegtqZhFWfRuU/T2oCLHoKdv5qZkgOfsYsQv7Xe8ZRu/1n4ziVxfYfoWEnCGpdfhuqOkdjyxzwbw5Rk85t8wqE2+abqNns6+HWedCsp+3nPH0Ifrrv3HItga2h41joeTf4NKy4rZZ8U5DXww88/G2fzSpciGhYbSLzlKl5mLjDFHJt0RcaR5b/+s/PhbXTYcXrRrc63wDBbWHb/2DOLXDDLGg/ouzz7JgPOt/8psuLbxOTMpGRYnSmsuxdaiZR3fDVuWFLpeDKl8z7XfcBOLvBFS/a/nnl58Gyl2Djl5B1Gtz9oM0VRr+a966cvQXFdT0DTUdbOI/KOGYfA/HAFmClUqoFkFYVRlUruVnGsUg/ZsbgvYIgsE3Fx+Dzc+HEdji+Hc6cMBd9QEuzZlpFfnB52XBwDexfYWYsnkkEJxcjQhE3lT/f6exJ2PY9bPkGjm2xblSANjODrnodut1u249Va9j0JSx+zgjSFS9C7/vBxc28HjoAvrkRFkyFG78u/VynD0PC3yYHoiL4NjUOZlWQl22GACLHg9MFk/K8G8BtP8MXI+Dr6+CORbZNODj8N3x7kxG3XvcYATqwAlb9n3Fgu06Avg+a6JytnEmEP14yDnFBqRDvRtBuOLQbYVZOcK1n+/nADEHvWWwql2ckg6snBLWFkB6mknnBd1setIaU/XAiFvKyzDmD25nfxYWfr32pfRpmsRidST1srltXT2jQ3vYZgReitdHDIxvNY3Y6+DWFJt0qNrFGa9MB3LvUdEhO7gFtMdd5+LUQPtakC9hKfp4515ZvzESh/BwK9QuMU3T1u7bnqB7fDvPvM1rYbiRc9ZrpkIHRwa+ugR/ugodjyu5cxs4zv5WGFaiwUlhk9mjVOGb7/jCdtGa9zt+uFAx/1Xxua94BN28YZMNofXY6zL0V9v9pPuOQHpC002jP9h+geV/o/yi0udJ2R89igQ2fQPTnkBRntjm7G91qP8JEHgs+l/JwfLtxktOOmvcZEAaNupjyJRX9bDNS4Pg2c04Xd/BrBg07Vvx3Vk4qk/w/DZhWZNNBpdRllTepmvlv+4vrX7n7Qduh5mYZOrBkJ01rUxQwYQMkRJu/YzHm5nMhzm7Q4WroP7VsgUtNMLVw9vxuHLLcs+DkCo27GNHNyYDNX5mLOnI8DHvFTLkuCYvF3Gw3fWUeLXnmQh3+GrQZaoQocQcsfRF+eRiOboaRb5funJ4+DD8/aH6ooQNgzPtQP/T8fVoOMj/W5f8xjklp0aUdP5nHTteW/tmUhG9jOFSJ4cWiHFoLuRklzwz1bQy3/wKfXmmGNScvA+/gks+XesQ4Ze6+MP5/ENTGbO//iJlgsOZd831u/NzcrAY8Zn70pbFzAcx/wFQZD7/G9Fhzzpprcdv3sPELc5NueZm57tqPBA/fks+XnwvRM2Hlm2bdPlcv856y0iAzxezj4mEEuXkfswZgSE9w9y75nFlpJs/l708gec/Fr7t6mkXqG3UyeTlNupqIaXlu1FVIrdSw35810Y+iKGdo2h0ib4ZO15lIaklkp5vfe4F+HYk2jl5xNI4wHa/O40p3qHMyTMdm92KjY6mHzPb6oeb7VU5wZJOZ6LP6vzD6fWjWo/T3eXKv+X1s/Q7OJpqOTdQk83tp1BmyUiFmNix/1ThZE38tPc8rPxdWvw0r3jBO3A2zTA5WUer5w7WfwPSeJpo28v9KPl/6cdOZGfRUxSLWPkVqmTXqXP7ji6K1SStpdRk4F3NLVwpGvGX04s+XjR6Fjy39fD8/aDqSo9+Dbrede+2q18195a/34Jvrzffb/1HzvZR2/0g9Aj9ONkGHZr3gyn+bTmTyPnNd/Poo8KiJgLYfac5X1kjKofWw5J9weJ35Dfg0Mo/b/keh4x7UzgQ1WvQ1OubfrOTzWSzm/vb3J8amgnMUoiCwlXnPjToZW5t0qxrH+sKWdAXHnZVSfsALQMEChyuAl7TWqWUcNxx4F1Mv6FOt9WsXvD4ReBMoKIv8vtb609LOGRUVpaOjo20zPPpzc8PxbYxJID9iLpa4X8yP3a85dLrGfImeQcZJKipkZ6whWGd3I1whPUw+UuNIE8Wx5MHJXbD1f0Y4stOg/SjoNcX0MpxdTJTiSPQ5IUuMNef0a256IG2HGefHzfOc3dnpRlhWv2OifCPego6jz39vORnmxrjuA0jeC14NoMsNpuxDcb06i8WEqle/DVF3GiEqTmR2/256kZY8GPoSdJ9U8o8w+wy8181ER+5YVLJozbjM9KSnrCjlyyqFlW/Bsn/DP46d/zldyODB5nH58pL3+f05WPcRPBVfuuNxdDPMvAqaRJohzuIcivxcM/R5fDvcvdwMkRRH2jHzPUXPNE5h11vh8ucu7qVrbd7rny+b6+2aGcZZL0petrlJ7Fpk/tISzDXeZih0vt48unqcO9/OBUbQUvaZ66zvQ9Dq8nOifibRRDsO/mX+jm8135VyNjYUiFxIlNl+YgfE/Wx60jlnoGmUueaadDVOQtZpSIwzn8mJ7eZ8WVaZcHI112aTruav/SibhU4ptVFrXeEpsxXRsOrQLyiHhh1ca/TCr7m5sWWnGadn5wIThXDxMI552CATpdTa6FGBfiXFme8MIKCVVb+ijGMX0NJEVFIPmyjVhs/M/kFtTdS38zjzfWoNpw6c60weWGWG5Vy9TKHoNleav6KOUkFnccHjRnN73weXP3t+BEJro8Vrp5vr2MnFaGHkLdD6yuKjt4fWmTQDryCjN8WlZqQfN7MVD683jutVb5Z+jS14zNwn7v+7ZOdg/cew6EmzT3C7Mr+2i0hNgLfDYdQ7EHVH6fuWpWEnYuHDvsbh7VZKimRetkk1Ob4N7lxsfsvFsX4GLHoChrwAA6aWcK4c2P69uR+d3GXONexVCO138b4J0WaIOOcsXPWG+T6L3hcKoqy7rfp1+G9Am3tq53HGSfNrem7/5H1mUsOO+WbEoN/D0OXGc99pbpYJmBxcY34vh9eb3wmYyFeBfrXoaxz+tKPm2twyx3rfDDZ6HDYA/FuYzy1lv1W7tpnHoqM1fs2hqVW/Wl1e8ud6AaXpV2Ucsx8wM5m+tG66FYjQWpcYAlFKOQO7gSuBBGADcLPWekeRfSYCUVrrB2y1pVyOWUnkZpkw7eavTe/Pknf+6wEtzQ2nQMgadip7mCcjxYhM9GcmSufkagTkzAkjjk4u5gJpM9T8Bbcru/d1bCvMv9/c3Jp0NfkRbp7m4i8Y4mrSFfo8YHqEZY3faw1LXzBRnMjxZlig4Jj8PFj5hullNupk8hdsGXqLnml6QON/MDkJF5IYBx/0Nr2mfhWsTrBlLsy7Gx6IPheRKg5bHLMP+pjv5fZfym53+w/w/SQTXR39/sXf1+JnYe37cN1nRlTKIiPFOF5/fwwu9UxULXK8ucEkxhnxj19lhOfqaeccrJLQ+lwULfZHEw1z94Wm3cxNt6BzEdQOhv7bXHdlXXNZaWbY+eBaE11MiD5XUqQAVy9zvfW8y9zky7Lx9CEjnkc3G8fiaAxkp8J966BBhzI+NEMVOGbl0rDq0i+oAg3TGo5ugs2zzfd+4aiAh7/VAYs654iVVaPPYoG4+Wb4/fg2s827oekk5maY5wGtjPPU5kpo0a/s6GdWmrmpbvjU3BS7TjDamn4cdvxsnM56ASaBvedk23JVD/9thiC9gmHCDyaqUcCBlfDj3aYjMPo9236TZxLh3UhodxWM++zi17WGGYNMrue9a8o+X3Hk58HLDUy0aUgZ6Rxladiad01Ha2pc2UOBZxJNpxgNk/+8ONc1YSPMHAath8BN35ad4mOxGE1c+qLpEHYYbcpzNO1mnKE178KaacaxunmObb/ttKMm/3j790YfUOZ8Po2NbhzfavSm38PQ94Gyhxct+cZ5PfiXGWU5uNZEYS+kWW/ocafRsTKv41RzLz662fzujm42zlq/R+DKf5X9Hqk+xyxGax1Z1rYLXu8DvKi1HmZ9/gyA1vrVIvtMxB6OWVFyMkx4POeMGZJs2Kly4crcTNOjPb7N/DD8Qoyj03Jw6cMOJZGfa3K91s8wvRUANx+TaxQ1yTh75Qmva21C98tfNRdnz8nG5uiZ5qKLuMVE00qLTBUlLwemRZrexqRFF7++4DETDp8aV/HPNX6Nyfu6dZ7ppZREWaKWdswMb1/5kvmh28KyV4zDetlz5+dr7PgZvrvVJMeOeNO2cxVwcq+J3BXURXL1MtFaDz+44l/QfWIFkpzzIH4lbPvBXCdZqeZabjvcRA6KG/awhbxs40wd23Iu/yK0f9lOY2lYLCYKUz/U5jy0KnDMyqVh1aVfUMUaZrGYoeRTB80wYv1Q46xUdJJIgbMfv8pEDtx9zUSWloPPd4LKw6H18Nc0MzNbWwBlhrgiboSIm8ufK5kQDd/cYP7v84AZMt+1EDbNMjZe/2X5cuZ+f850qh/caBzHohzeAJ9dYTSxx13ls7Mob3cyzuy1H5e+X1ka9uXVZtbkfTamdhzbAjOHm7WJb513Lj8vIwU+HggoM5JRnuLaORmmQ7r6beO0u3mb+yeY+8ewVypWrPvkXuP4HVgJGSfNKFbLwSYyWJGJa3AuB/bgX+Y+5xlgvgffxhU7XwEZKSagY0tngtL1qzLJ/5lKqf5a69XWRvoBmWUc0xQ4XOR5AtCrmP2uU0oNxPROH9VaH75wB6XU3cDdAM2bN6+A+aXg5mlyaqoK13qmp2ZLb80WnF2NIETdaRw9S67pLVY0V0cpGPy0ucEuexl+uNNs92liplyXNw/Mxc0Mj/32lLn4W/Q991pWmgkZd7q2cs5uQa7A6YsujfJxxHozbFFMCL4kBj8Dpw+a4cXMU9D7XhMKX/ysiUQMfbn8dgS1hlvmQOJO2PmLEdrAVqb3ZuMP/SKcXYzTWprjWhFc3M3voyp/I05OFb/JV5zyaliV6Ze1verRMCcnE32vyBBbcShl8kXLMyO5LJr3guazIfO0GeJyrVe5VTZCouDOJSYP8w9rxMLZ3URArnyp/EnbfR4ww5Vr3jWjCEXZ8InpCHe5seL2gtHb1Erql8ViOkkRN9t+TOMIE9H/7jaTdjHmfeOs/HSvGc2ZtLj834WbJwx60nTqd/1mdNW3qUmVKCufsDSCWsPgp8xfVaGsuWJVrTdVuEpMZRyze4BZ1jwNgFPA7ZU3iV+Ab7XW2UqpKZhhhovuLFrrGcAMML3NKmi39qFU5couXEjX8WbmZ8IG0+NpGF7xXna320xi+YrX4dafzp1n61zTk+oxuXK2+jQxOU+VFbZjW8x5yjOzyskJxn5kolnrpps/gFZD4JqPK5fM3qD9xTlkQnVRHRpmk36BaBhgojXlWfWjNAJbmQh9ygEz5BXSw/Yo/4X4NDLDrJu/NhO46rcw28+eNLMxu080JTsqg38zM6xWGVL2Gz21Ma+pkPYjzMSkOeNhxmCzzbsh3PKdGTasKPXqm0kokeVwFIWLqMyszC1AhFLK1/o8TSn1CLC1lMOOAEWnRYRwLkm24LzJRZ5+CrxRURuFCuDkXPkaNXCuB7XoSSNu3W41uQPLXz03YaIyOLuYfIrKRsyObTWRhfIOnTg5mUTWLjeZXCl3XzM8KEuf1BoqoGGiX7WBgLDylaEpif5TzazQ+febkjlKwcInTM5SZTuWYCJmad+blIOKphUct5ZAKq9jBmYW54PRJm807ZiJulXDDEOh/FQmYgYYMSvydCrwTim7bwDaKKXCMIJ2E3BL0R2UUo211gVr7YwG4ipro2Anekw2s11/e9o4ahtmmjH9MR+UfawtVMVQwLEtJmehIihlHMzKOpmCXSmHhol+XUr4NzMlhn5+AJb+03S+Yn80q4OUNNu6vOfX+aamZmllHErj2BYzqSy4glF23yamQyk4FJV2zC6g1HEvrXWeUuoBYDFmuvlMrXWsUuolIFpr/TPwkFJqNJAHpAATq9hGoaZwcoKxH8DHg8xMRoCxH1aNqIGZRHFoXcWPTz9hZihWpLcp1FVK1DDRr0uQrhNMSZC/3jPPW14G/R6tmnMXlBNJPVwJx2yrqYFYkULQgsNS1Y5ZmXkSWuuFwMILtv2zyP/PAM9UsV2CvfBvDo9sNbkQyrli1cRLPHczM2OnokMBx60jVo27VJ1NQm2nVA0T/brEUMrM6EzZZ0rPNO1edekKftYJH6cPQ4sKHK+1iZi1H1k19ggOQ7nvZkqpdIoXLwWUM1FHuCRw96meqJSfdSjgzPHSq36XxLEY81jZyttCrUI0TCgXTk6mVmJp9RIrQtGIWUVITTCrdEjEv85RbsdMa13JqSiCUEUULZlRIcdsq6lRVJFackKtRTRMcAjcPE1droo6ZoUR/8gqM0lwDGQKmVB7KRgKqKiwHdti1hAVBEGwB/7NKj6z/NgWU0C4IouoCw6NOGZC7aVg/bTTh8p/bOYpUyRWhgEEQbAXfiGV6FhuNeuYVrRWm+CwiGMm1F7cvMx6exURtoK1/yTxXxAEe+HX3ETMKrI0okT86yzimAm1G//mZk3A8nLMmp/RSCJmgiDYCf/mkJdpZnyWhzNJkH5UIv51FHHMhNpNcHtIrEANz2NbzLJO3sFVb5MgCIItFKxnmrijfMcVVvyXiFldRBwzoXbTMNyUyzibXPa+RTm+VXqbgiDYl4bWuo4nyumYFUb8xTGri4hjJtRuCmYkJcbafkxOBpzcLY6ZIAj2xTsYvBrAiXLoF5iIf/3QqlsAXnAoxDETajeFPc5yCNuJWNAWGQYQBMH+NAyHE9vLd8zxrRItq8OIYybUbrwbmCKN5RG2gor/EjETBMHeNAyHpJ1gybdt/6xUs8Sd6FedRRwzofbTMLx8ORoJG8zwgW/T6rNJEATBFhqGQ16WcbZsISHaPDbtVn02CXZFHDOh9tOwk5mZaUuPU2uIXwMt+poFigVBEOxJQZ6srVH/g2tAOUNIz+qzSbAr4pgJtZ+G4aYWUMqBsvc9fQjSEiC0f/XbJQiCUBZB7YyjZWue7MG/oElXcPeuXrsEuyGOmVD7KehxFtT2KY2Df5nHFn2rzx5BEARbcfWAoDbnSmCURm4mHNko+lXHEcdMqP00DId6AbBzQdn7HlwNHv4Q3KHazRIEQbCJ0AFwYAVkpZW+X0I05OdAi341Y5dgF8QxE2o/zq7Q6VrYuRCy00vf9+BfprfpJJe+IAgOQpcbzQSAuF9K3+/gX4CC5r1rxCzBPsjdSagbdLnR5JnF/VryPqkJZuaT9DYFQXAkQqKgfhhsnVv6fgdWQqNOUli2jiOOmVA3COlhKmGXJmwbPgXlBO1H1phZgiAIZaIUdLnBOF5px4rfJzHOpGJ0GFOztgk1To07Zkqp4UqpXUqpvUqpp4t53V0pNdf6+nqlVGhN2yjUQpQyUbP9yyF23sWvZ6dD9EzocDUEhNW4eULdQTRMqBa63AhoWPQE5Odd/Ppf74NLPehxZ42bJtQsNeqYKaWcgenAVUBH4GalVMcLdrsTOKW1bg28DbxekzYKtZh+D5vcix/ugphvzq9rtvlrUzG770P2s0+o9YiGCdVGYCsY9qrJM5s3BTJSzr2Wn2NGA7pOAM8A+9ko1Ag1HTHrCezVWu/XWucAc4AL47JjgC+t/38PDFFKKoEKNuDmBbd8B40j4ad74f0ecCre5JX9/jw072tyOQSh4oiGCdVHn/vg8udh+/fwdjjMu8fkxh7bYtb37XOfvS0UaoCadsyaAoeLPE+wbit2H611HpAKBF54IqXU3UqpaKVUdFJSUjWZK9Q6PHzhzt9h3Ofg0wjSjkDaUYi4CW74suzjBaF0RMOE6mXg43DvWgi/BnYvNp1LJxe4bT4EtLS3dUIN4GJvAyqK1noGMAMgKipK29kcwZFwcjblMzpdC18MNEOaY963t1WCcB6iYUKJNOwIYz8AiwV+6Qcu7hA2wN5WCTVETUfMjgDNijwPsW4rdh+llAvgByTXiHVC3UM5mTpnglA1iIYJNYeTk3HKhEuKmnbMNgBtlFJhSik34Cbg5wv2+Rm43fr/OGCZ1lp6k4IgOAKiYYIgVCuqpvVCKTUCeAdwBmZqrV9RSr0ERGutf1ZKeQBfAV2BFOAmrfX+Ms6ZBBysXsvPIwg4WYPtlYQj2CE2iA0XUlN2tNBaB9dAO+chGiY21EEbwDHsuJRsKFG/atwxqwsopaK11naf3ucIdogNYoOj2iGUjCN8R2KD49jgKHaIDQap/C8IgiAIguAgiGMmCIIgCILgIIhjVjFm2NsAK45gh9hgEBvO4Sh2CCXjCN+R2GBwBBvAMewQG5AcM0EQBEEQBIdBImaCIAiCIAgOgjhmlUAp9aBSaqdSKlYp9YYd7XhMKaWVUkF2av9N6+ewVSk1TynlX0PtDldK7VJK7VVKPV0TbRZjQzOl1J9KqR3W6+Bhe9hhtcVZKbVZKfWrndr3V0p9b70W4pRSfexhh2A7omH20y9r23bVMNGv89p3GP0Sx6yCKKUuwyxWHKG1DgfespMdzYChwCF7tG9lCdBJa90F2A08U90NKqWcgenAVUBH4GalVMfqbrcY8oDHtNYdgd7A/XayA+BhIM5ObQO8C/ymtW4PRNjZFqEMRMMKqXH9AofRMNGvcziMfoljVnHuBV7TWmcDaK0T7WTH28CTgN2SBbXWv1sXawZYh1mmprrpCezVWu/XWucAczA3mRpFa31Ma73J+n865sd84aLW1Y5SKgQYCXxa021b2/cDBgKfAWitc7TWp+1hi2AzomHYTb/AATRM9KuwfYfSL3HMKk5bYIBSar1SaoVSqkdNG6CUGgMc0Vpvqem2S2ESsKgG2mkKHC7yPAE7CEpRlFKhmGrv6+3Q/DuYm5vFDm0DhAFJwOfW4YhPlVJedrJFsA3RsIupKf0CB9Mw0S/H0S8XezVcG1BKLQUaFfPSs5jPLgAT/u0BfKeUalnVa+KVYcM/MEMA1U5pdmit51v3eRYTGp9dEzY5Ekopb+AH4BGtdVoNtz0KSNRab1RKDa7JtovgAnQDHtRar1dKvQs8DTxvJ3sERMNssUH0S/QLB9MvccxKQWt9RUmvKaXuBX60itjfSikLZo2tpJqwQSnVGePlb1FKgQm/b1JK9dRaH69KG0qzo4g9E4FRwJAaWrD5CNCsyPMQ67YaRynlihG12VrrH+1gQj9gtDJrOHoAvkqpr7XWE2rQhgQgQWtd0Nv+HiNsgh0RDSvdhiK2TKRm9QscRMNEvwAH0y8Zyqw4PwGXASil2gJu1ODiq1rrbVrrBlrrUK11KObC6lYdTllZKKWGY8LQo7XWGTXU7AagjVIqTCnlBtwE/FxDbReizB3lMyBOa/3fmm4fQGv9jNY6xHod3AQsq2FRw3rdHVZKtbNuGgLsqEkbhHLzE6Jh9tIvcAANE/0qtMGh9EsiZhVnJjBTKbUdyAFur8GelqPxPuAOLLH2fNdpre+pzga11nlKqQeAxYAzMFNrHVudbZZAP+BWYJtSKsa67R9a64V2sMXePAjMtt5k9gN32NkeoXREwww1rl/gMBom+nUOh9EvqfwvCIIgCILgIMhQpiAIgiAIgoMgjpkgCIIgCIKDII6ZIAiCIAiCgyCOmSAIgiAIgoMgjpkgCIIgCIKDIOUyBIdAKRUI/GF92gjIxxS6DAWOWhfZtfVcY4HdWmupoyUIQrUj+iVUJRIxExwCrXWy1jpSax0JfAS8bf0/kvKvnzYWsFkIBUEQKoPol1CViGMm1AaclVKfKKVilVK/K6XqASilWimlflNKbVRKrVJKtVdK9QVGA28qpWKs+0xWSm1QSm1RSv2glPK079sRBOESQvRLKBfimAm1gTbAdK11OHAauM66fQZm0dnuwOPAB1rrvzDLmjxh7cHuw6wH2ENrHQHEAXfW+DsQBOFSRfRLKBeSYybUBg5orWOs/28EQpVS3kBf4H/WZVTALKtSHJ2UUi8D/oA3ZgkUQRCEmkD0SygX4pgJtYHsIv/nA/Uw0d7T1jyOsvgCGKu13qKUmggMrmL7BEEQSkL0SygXMpQp1Eq01mnAAaXU9QDKEGF9OR3wKbK7D3BMKeUKjK9ZSwVBEM5H9EsoDXHMhNrMeOBOpdQWIBYYY90+B3hCKbVZKdUKeB5YD6wBdtrFUkEQhPMR/RKKRWmt7W2DIAiCIAiCgETMBEEQBEEQHAZxzARBEARBEBwEccwEQRAEQRAcBHHMBEEQBEEQHARxzARBEARBEBwEccwEQRAEQRAcBHHMBEEQBEEQHARxzARBEARBEByE/wcHQNrQlPNnpwAAAABJRU5ErkJggg==\n", 122 | "text/plain": [ 123 | "
" 124 | ] 125 | }, 126 | "metadata": { 127 | "needs_background": "light" 128 | }, 129 | "output_type": "display_data" 130 | } 131 | ], 132 | "source": [ 133 | "figure(figsize=(10,3))\n", 134 | "\n", 135 | "freq=0.5\n", 136 | "theta_ranges, landscape_with_extra, landscape_mse = plot_loss(extra=True,exp_folder=EXP_FOLDER,freq=freq)\n", 137 | "subplot(2,2,1)\n", 138 | "plt.plot(theta_ranges,landscape_with_extra)\n", 139 | "plt.ylabel('Loss')\n", 140 | "plt.legend(['Shaped ML$^3$ Landscape'])\n", 141 | "plt.axvline(x=freq,c='red')\n", 142 | "subplot(2,2,3)\n", 143 | "plt.plot(theta_ranges,landscape_mse,c='C1')\n", 144 | "plt.xlabel('Theta')\n", 145 | "plt.ylabel('Loss')\n", 146 | "plt.legend(['MSE Loss Landscape'])\n", 147 | "plt.axvline(x=freq,c='red')\n", 148 | "\n", 149 | "\n", 150 | "theta_ranges, landscape_wo_extra, landscape_mse = plot_loss(extra=False,exp_folder=EXP_FOLDER,freq=freq)\n", 151 | "subplot(2,2,2)\n", 152 | "plt.plot(theta_ranges,landscape_wo_extra)\n", 153 | "plt.ylabel('Loss')\n", 154 | "plt.legend(['ML$^3$ Landscape'])\n", 155 | "plt.axvline(x=freq,c='red')\n", 156 | "subplot(2,2,4)\n", 157 | "plt.plot(theta_ranges,landscape_mse,c='C1')\n", 158 | "plt.xlabel('Theta')\n", 159 | "plt.ylabel('Loss')\n", 160 | "plt.legend(['MSE Loss Landscape'])\n", 161 | "plt.axvline(x=freq,c='red')" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 4, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "name": "stderr", 171 | "output_type": "stream", 172 | "text": [ 173 | "MovieWriter imagemagick unavailable; using Pillow instead.\n", 174 | "MovieWriter imagemagick unavailable; using Pillow instead.\n", 175 | "MovieWriter imagemagick unavailable; using Pillow instead.\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "theta_ranges = np.load(f'{EXP_FOLDER}/theta_ranges_True_.npy')\n", 181 | "ml3_extra_loss = normalize_data(np.load(f'{EXP_FOLDER}/landscape_with_extra_True_.npy'))\n", 182 | "ml3_mse_loss = normalize_data(np.load(f'{EXP_FOLDER}/landscape_mse_False_.npy'))\n", 183 | "ml3_not_shaped_loss = normalize_data(np.load(f'{EXP_FOLDER}/landscape_with_extra_False_.npy'))\n", 184 | "freq=0.5\n", 185 | "render(theta_ranges,ml3_extra_loss,'C0',freq=freq,file_path=f'{EXP_FOLDER}/ml3_shaped_loss_sine.gif')\n", 186 | "render(theta_ranges,ml3_not_shaped_loss,'C2',freq=freq,file_path=f'{EXP_FOLDER}/ml3_not_shaped_loss_sine.gif')\n", 187 | "render(theta_ranges,ml3_mse_loss,'C1',freq=freq,file_path=f'{EXP_FOLDER}/mse_loss_sine.gif')" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [] 196 | } 197 | ], 198 | "metadata": { 199 | "kernelspec": { 200 | "display_name": "Python 3", 201 | "language": "python", 202 | "name": "python3" 203 | }, 204 | "language_info": { 205 | "codemirror_mode": { 206 | "name": "ipython", 207 | "version": 3 208 | }, 209 | "file_extension": ".py", 210 | "mimetype": "text/x-python", 211 | "name": "python", 212 | "nbconvert_exporter": "python", 213 | "pygments_lexer": "ipython3", 214 | "version": "3.7.9" 215 | } 216 | }, 217 | "nbformat": 4, 218 | "nbformat_minor": 4 219 | } 220 | -------------------------------------------------------------------------------- /ml3/experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/LearningToLearn/fa32b98b40402fa15982b450ed09d9d3735ec924/ml3/experiments/__init__.py -------------------------------------------------------------------------------- /ml3/experiments/run_mbrl_reacher_exp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import sys 3 | import os 4 | import numpy as np 5 | import torch 6 | import pybullet 7 | import ml3 8 | from ml3.envs.reacher_sim import ReacherSimulation 9 | from ml3.mbrl_utils import Dynamics 10 | from ml3.learnable_losses import Ml3_loss_reacher as Ml3_loss 11 | from ml3.optimizee import Reacher_Policy as Policy 12 | from ml3.ml3_train import meta_train_mbrl_reacher as meta_train 13 | from ml3.ml3_test import test_ml3_loss_reacher as test_ml3_loss 14 | 15 | EXP_FOLDER = os.path.join(ml3.__path__[0], "experiments/data/mbrl_reacher") 16 | 17 | 18 | class Task_loss(object): 19 | def __call__(self, a, s, goal): 20 | loss = 10*torch.norm(s[-1,:2]-goal[:2])+torch.mean(torch.norm(s[:,:2]-goal[:2],dim=1))+0.0001*torch.mean(torch.norm(s[:,2:],dim=1)) 21 | return loss 22 | 23 | 24 | def random_babbling(env, time_horizon): 25 | # do random babbling 26 | actions = np.random.uniform(-1.0, 1.0, [time_horizon, 2]) 27 | states = [] 28 | state = env.reset() 29 | states.append(state) 30 | for u in actions: 31 | state = env.sim_step(state, u) 32 | states.append(state.copy()) 33 | 34 | return np.array(states), actions 35 | 36 | 37 | if __name__ == '__main__': 38 | 39 | if not os.path.exists(EXP_FOLDER): 40 | os.makedirs(EXP_FOLDER) 41 | 42 | np.random.seed(0) 43 | torch.manual_seed(0) 44 | 45 | # create Reacher simulation 46 | env = ReacherSimulation(gui=False) 47 | 48 | # initialize policy and save initialization for training 49 | policy = Policy(8, 2, EXP_FOLDER) 50 | policy.reset() 51 | 52 | # initialize learned loss 53 | ml3_loss = Ml3_loss(7, 1) 54 | # initialize task loss for meta training 55 | task_loss = Task_loss() 56 | 57 | # initialize learned dynamics model 58 | dmodel = Dynamics(env) 59 | 60 | # generate training task 61 | num_task = 1 62 | train_goal = np.array(env.get_target_joint_configuration(np.array([0.02534078, 0.19863741, 0.0]))) 63 | train_goal = np.hstack([train_goal, np.zeros(2)]) 64 | 65 | goals = [train_goal] 66 | time_horizon = 65 67 | 68 | if sys.argv[1] == 'train': 69 | 70 | n_outer_iter = 3000 # 3000 71 | n_inner_iter = 1 72 | 73 | for random_data in range(3): 74 | states, actions = random_babbling(env, time_horizon) 75 | dmodel.train(torch.Tensor(states), torch.Tensor(actions)) 76 | 77 | meta_train(policy, ml3_loss,dmodel,env, task_loss, goals, n_outer_iter, n_inner_iter, time_horizon, EXP_FOLDER) 78 | 79 | if sys.argv[1] == 'test': 80 | ml3_loss.load_state_dict(torch.load(f"{EXP_FOLDER}/ml3_loss_reacher.pt")) 81 | ml3_loss.eval() 82 | opt_iter = 2 83 | 84 | xy = [0.05534078, 0.150863741] 85 | 86 | test_goal = np.array(env.get_target_joint_configuration(np.array([xy[0], xy[1], 0.0]))) 87 | test_goal = np.hstack([test_goal, np.zeros(2)]) 88 | args = (torch.Tensor(test_goal),time_horizon,None,env,True) 89 | print('goal joint position:', test_goal[:2]) 90 | states = test_ml3_loss(policy, ml3_loss,opt_iter,*args) 91 | print('achieved joint position',states[-1,:2]) 92 | -------------------------------------------------------------------------------- /ml3/experiments/run_mountain_car_exp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import sys 3 | import os 4 | import numpy as np 5 | import torch 6 | import ml3 7 | from ml3.ml3_train import meta_train_mountain_car as meta_train 8 | from ml3.ml3_test import test_ml3_loss_mountain_car as test_ml3_loss 9 | from ml3.learnable_losses import Ml3_loss_mountain_car as Ml3_loss 10 | from ml3.optimizee import MC_Policy 11 | from ml3.envs.mountain_car import MountainCar 12 | 13 | EXP_FOLDER = os.path.join(ml3.__path__[0], "experiments/data/mountain_car") 14 | 15 | 16 | class Task_loss(object): 17 | def __call__(self, a, s, goal, goal_exp,shaped_loss): 18 | 19 | loss = (torch.norm(s - goal)).mean() 20 | if shaped_loss: 21 | loss = (torch.norm(s[:15] - goal_exp)).mean() + (torch.norm(s[15:] - goal)).mean() 22 | 23 | return loss 24 | 25 | 26 | if __name__ == '__main__': 27 | 28 | if not os.path.exists(EXP_FOLDER): 29 | os.makedirs(EXP_FOLDER) 30 | 31 | np.random.seed(0) 32 | torch.manual_seed(0) 33 | 34 | policy = MC_Policy(2,1) 35 | ml3_loss = Ml3_loss(4,1) 36 | 37 | task_loss = Task_loss() 38 | 39 | goal = [0.5000, 1.0375] 40 | goal_extra = [-0.9470, -0.0055] 41 | 42 | env = MountainCar() 43 | s_0 = env.reset() 44 | 45 | n_outer_iter = 300 46 | n_inner_iter = 1 47 | 48 | time_horizon = 35 49 | 50 | if sys.argv[1] == 'train': 51 | shaped_loss = sys.argv[2] == 'True' 52 | meta_train(policy, ml3_loss, task_loss, s_0, goal, goal_extra, n_outer_iter, n_inner_iter, time_horizon, shaped_loss) 53 | if shaped_loss: 54 | torch.save(ml3_loss.state_dict(), f"{EXP_FOLDER}/shaped_ml3_loss_mountain_car.pt") 55 | else: 56 | torch.save(ml3_loss.state_dict(), f"{EXP_FOLDER}/ml3_loss_mountain_car.pt") 57 | 58 | if sys.argv[1] == 'test': 59 | shaped_loss = sys.argv[2] == 'True' 60 | if shaped_loss: 61 | ml3_loss.load_state_dict(torch.load(f"{EXP_FOLDER}/shaped_ml3_loss_mountain_car.pt")) 62 | else: 63 | ml3_loss.load_state_dict(torch.load(f"{EXP_FOLDER}/ml3_loss_mountain_car.pt")) 64 | ml3_loss.eval() 65 | opt_iter = 2 66 | args = (torch.Tensor(s_0), torch.Tensor(goal), time_horizon) 67 | states = test_ml3_loss(policy, ml3_loss, opt_iter, *args) 68 | if shaped_loss: 69 | np.save(f"{EXP_FOLDER}/shaped_ml3_mc_states.npy", states) 70 | else: 71 | np.save(f"{EXP_FOLDER}/ml3_mc_states.npy", states) 72 | 73 | if shaped_loss: 74 | env.render(list(np.array(states)[:, 0]), file_path=f"{EXP_FOLDER}/shaped_ml3_mc.gif") 75 | else: 76 | env.render(list(np.array(states)[:, 0]), file_path=f"{EXP_FOLDER}/ml3_mc.gif") 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /ml3/experiments/run_shaped_sine_exp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | import numpy as np 4 | import sys 5 | import os 6 | import ml3 7 | from ml3.optimizee import ShapedSineModel 8 | from ml3.ml3_train import meta_train_shaped_sine as meta_train 9 | from ml3.learnable_losses import Ml3_loss_shaped_sine as Ml3_loss 10 | from ml3.ml3_test import test_ml3_loss_shaped_sine as test_ml3_loss 11 | 12 | EXP_FOLDER = os.path.join(ml3.__path__[0], "experiments/data/shaped_sine") 13 | 14 | 15 | class Task_loss(object): 16 | def __call__(self, input,outputs,labels,shaped,new_theta,label_thetas): 17 | if shaped: 18 | loss = (new_theta - label_thetas) ** 2 19 | else: 20 | loss = (outputs - labels) ** 2 21 | return loss 22 | 23 | 24 | def generate_sinusoid_batch(num_tasks, num_examples_task, num_steps, random_steps=False, 25 | freq_range=[-5.0, 5.0], input_range=[-5.0, 5.0]): 26 | """ Generate samples from random sine functions. """ 27 | freq = np.random.uniform(freq_range[0], freq_range[1], [num_tasks]) 28 | outputs = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 29 | thetas = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 30 | init_inputs = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 31 | 32 | for task in range(num_tasks): 33 | if random_steps: 34 | init_inputs[task] = np.random.uniform(input_range[0], input_range[1], 35 | [num_steps, num_examples_task, 1]) 36 | else: 37 | init_inputs[task] = np.repeat(np.random.uniform(input_range[0], input_range[1], 38 | [1, num_examples_task, 1]), num_steps, -3) 39 | 40 | outputs[task] = np.sin(freq[task]*init_inputs[task]) 41 | thetas[task] = np.zeros_like(outputs[task]) + freq[task] 42 | return init_inputs, outputs,thetas 43 | 44 | 45 | if __name__ == '__main__': 46 | 47 | if not os.path.exists(EXP_FOLDER): 48 | os.makedirs(EXP_FOLDER) 49 | 50 | shaped = sys.argv[2]=='True' 51 | torch.manual_seed(0) 52 | np.random.seed(0) 53 | 54 | n_outer_iter = 1500 55 | num_task = 4 56 | n_inner_iter = 10 57 | batch_size = 64 58 | 59 | ml3_loss = Ml3_loss() 60 | sine_model=ShapedSineModel() 61 | torch.save(sine_model.state_dict(), f"{EXP_FOLDER}/shaped_sine_init_policy.pt") 62 | sine_model.load_state_dict(torch.load(f"{EXP_FOLDER}/shaped_sine_init_policy.pt")) 63 | sine_model.eval() 64 | 65 | # initialize task loss for meta training 66 | task_loss_fn = Task_loss() 67 | 68 | if sys.argv[1] == 'train': 69 | meta_train(n_outer_iter, shaped, num_task, n_inner_iter, sine_model, ml3_loss,task_loss_fn, EXP_FOLDER) 70 | 71 | if sys.argv[1] == 'test': 72 | freq=0.7 73 | test_x = np.expand_dims(np.arange(-5.0,5.0,0.1),1) 74 | test_y = np.sin(freq*test_x) 75 | x = torch.Tensor(test_x) 76 | y = torch.Tensor(test_y) 77 | 78 | ml3_loss.load_state_dict(torch.load(f"{EXP_FOLDER}/ml3_loss_shaped_sine_{str(shaped)}.pt")) 79 | ml3_loss.eval() 80 | opt_iter = 1 81 | args = (torch.Tensor(test_x),torch.Tensor(test_y)) 82 | test_ml3_loss(sine_model, ml3_loss,opt_iter,*args) 83 | 84 | -------------------------------------------------------------------------------- /ml3/experiments/run_sine_regression_exp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | 3 | import os 4 | import ml3 5 | from ml3.sine_regression_task import main as meta_train 6 | 7 | EXP_FOLDER = os.path.join(ml3.__path__[0], "experiments/data/sine_exp") 8 | 9 | 10 | if __name__ == "__main__": 11 | exp_cfg = {} 12 | exp_cfg['seed'] = 0 13 | exp_cfg['num_train_tasks'] = 1 14 | exp_cfg['num_test_tasks'] = 10 15 | exp_cfg['n_outer_iter'] = 500 16 | exp_cfg['n_gradient_steps_at_test'] = 100 17 | exp_cfg['inner_lr'] = 0.001 18 | exp_cfg['outer_lr'] = 0.001 19 | 20 | exp_cfg['model'] = {} 21 | exp_cfg['model']['in_dim'] = 1 22 | exp_cfg['model']['hidden_dim'] = [100, 10] 23 | 24 | exp_cfg['metaloss'] = {} 25 | exp_cfg['metaloss']['in_dim'] = 2 26 | exp_cfg['metaloss']['hidden_dim'] = [50, 50] 27 | 28 | model_arch_str = str(exp_cfg['model']['hidden_dim']) 29 | meta_arch_str = "{}".format(exp_cfg['metaloss']['hidden_dim']) 30 | exp_cfg['log_dir'] = f"{EXP_FOLDER}" 31 | 32 | for seed in range(5): 33 | exp_cfg['seed'] = seed 34 | exp_file = "sine_regression_seed_{}.pt".format(exp_cfg['seed']) 35 | exp_cfg['exp_log_file_name'] = exp_file 36 | meta_train(exp_cfg) -------------------------------------------------------------------------------- /ml3/learnable_losses.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | def weight_init(module): 8 | if isinstance(module, nn.Linear): 9 | nn.init.xavier_uniform_(module.weight, gain=1.0) 10 | if module.bias is not None: 11 | module.bias.data.zero_() 12 | 13 | 14 | def weight_reset(m): 15 | if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): 16 | m.reset_parameters() 17 | 18 | 19 | class ML3_SineRegressionLoss(nn.Module): 20 | 21 | def __init__(self, in_dim, hidden_dim): 22 | super(ML3_SineRegressionLoss, self).__init__() 23 | w = [50, 50] 24 | self.layers = nn.Sequential( 25 | nn.Linear(in_dim, hidden_dim[0], bias=False), 26 | nn.ReLU(), 27 | nn.Linear(hidden_dim[0], hidden_dim[1], bias=False), 28 | nn.ReLU(), 29 | ) 30 | self.loss = nn.Sequential(nn.Linear(hidden_dim[1], 1, bias=False), nn.Softplus()) 31 | self.reset() 32 | 33 | def forward(self, y_in, y_target): 34 | y = torch.cat((y_in, y_target), dim=1) 35 | yp = self.layers(y) 36 | return self.loss(yp).mean() 37 | 38 | def reset(self): 39 | self.layers.apply(weight_init) 40 | self.loss.apply(weight_init) 41 | 42 | 43 | class Ml3_loss_mountain_car(nn.Module): 44 | 45 | def __init__(self, meta_in, meta_out): 46 | super(Ml3_loss_mountain_car, self).__init__() 47 | 48 | activation = torch.nn.ELU 49 | num_neurons = 400 50 | self.loss_fn = torch.nn.Sequential(torch.nn.Linear(meta_in, num_neurons), 51 | activation(), 52 | torch.nn.Linear(num_neurons, num_neurons), 53 | activation(), 54 | torch.nn.Linear(num_neurons, meta_out)) 55 | self.learning_rate = 1e-3 56 | 57 | def forward(self, x): 58 | return self.loss_fn(x) 59 | 60 | 61 | class Ml3_loss_reacher(nn.Module): 62 | 63 | def __init__(self, meta_in, meta_out): 64 | super(Ml3_loss_reacher, self).__init__() 65 | 66 | activation = torch.nn.ELU 67 | output_activation = torch.nn.Softplus 68 | num_neurons = 400 69 | self.loss_fun = torch.nn.Sequential(torch.nn.Linear(meta_in, num_neurons), 70 | activation(), 71 | torch.nn.Linear(num_neurons, num_neurons), 72 | activation(), 73 | torch.nn.Linear(num_neurons, meta_out), 74 | output_activation()) 75 | self.learning_rate = 1e-2 76 | 77 | self.norm_in = torch.Tensor(np.expand_dims(np.array([1.0, 1.0, 8.0, 8.0, 1.0, 1.0,1.0]), axis=0)) 78 | 79 | def forward(self, x): 80 | return self.loss_fun(x/self.norm_in) 81 | 82 | 83 | class Ml3_loss_shaped_sine(nn.Module): 84 | 85 | def __init__(self, meta_in=3, meta_out=1): 86 | super(Ml3_loss_shaped_sine, self).__init__() 87 | def init_weights(m): 88 | if type(m) == torch.nn.Linear: 89 | torch.nn.init.xavier_uniform_(m.weight) 90 | m.bias.data.fill_(0.01) 91 | 92 | activation = torch.nn.ELU 93 | num_neurons = 10 94 | self.loss_fn = torch.nn.Sequential(torch.nn.Linear(meta_in, num_neurons), activation(), 95 | torch.nn.Linear(num_neurons, num_neurons), activation(), 96 | torch.nn.Linear(num_neurons, num_neurons), activation(), 97 | torch.nn.Linear(num_neurons, meta_out)) 98 | 99 | self.loss_fn.apply(init_weights) 100 | 101 | self.learning_rate = 3e-3 102 | 103 | def forward(self, x): 104 | return self.loss_fn(x) 105 | -------------------------------------------------------------------------------- /ml3/mbrl_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import numpy as np 3 | from termcolor import colored 4 | import logging 5 | import torch.nn as nn 6 | import torch.utils.data 7 | 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | import torch 12 | import numpy as np 13 | import math 14 | 15 | 16 | class Dataset(torch.utils.data.Dataset): 17 | def __init__(self, x, y): 18 | self.dataset = [ 19 | (torch.FloatTensor(x[i]), torch.FloatTensor(y[i])) for i in range(len(x)) 20 | ] 21 | 22 | def __len__(self): 23 | return len(self.dataset) 24 | 25 | def __getitem__(self, idx): 26 | return self.dataset[idx] 27 | 28 | class Dynamics(nn.Module): 29 | 30 | def __init__(self,env): 31 | super(Dynamics, self).__init__() 32 | 33 | self.env=env 34 | self.dt = env.dt 35 | 36 | self.model_cfg = {} 37 | self.model_cfg['device'] = 'cpu' 38 | self.model_cfg['hidden_size'] = [100, 30] 39 | self.model_cfg['batch_size'] = 128 40 | self.model_cfg['epochs'] = 500 41 | self.model_cfg['display_epoch'] = 50 42 | self.model_cfg['learning_rate'] = 0.001 43 | self.model_cfg['ensemble_size'] = 3 44 | self.model_cfg['state_dim'] = env.state_dim 45 | self.model_cfg['action_dim'] = env.action_dim 46 | self.model_cfg['output_dim'] = env.pos_dim 47 | 48 | self.ensemble = EnsembleProbabilisticModel(self.model_cfg) 49 | 50 | self.data_X = [] 51 | self.data_Y = [] 52 | self.norm_in = torch.Tensor(np.expand_dims(np.array([1.0,1.0,8.0,8.0,1.0,1.0]),axis=0)) 53 | 54 | 55 | 56 | def train(self,states,actions): 57 | 58 | inputs = (torch.cat((states[:-1],actions),dim=1)/self.norm_in).detach().numpy() 59 | outputs = (states[1:,self.env.pos_dim:] - states[:-1,self.env.pos_dim:]).detach().numpy() 60 | 61 | self.data_X+=list(inputs) 62 | self.data_Y+=list(outputs) 63 | 64 | training_dataset = {} 65 | training_dataset['X'] = np.array(self.data_X) 66 | training_dataset['Y'] = np.array(self.data_Y) 67 | 68 | #self.ensemble = EnsembleProbabilisticModel(self.model_cfg) 69 | self.ensemble.train_model(training_dataset, training_dataset, 0.0) 70 | 71 | def step_model(self,state,action): 72 | input_x = torch.cat((state,action),dim=0)/self.norm_in 73 | pred_acc = self.ensemble.forward(input_x)[0].squeeze() 74 | 75 | #numerically integrate predicted acceleration to velocity and position 76 | 77 | pred_vel = state[self.env.pos_dim:]+pred_acc 78 | pred_pos = state[:self.env.pos_dim] + pred_vel*self.dt 79 | pred_pos = torch.clamp(pred_pos, min=-3.0, max=3.0) 80 | pred_vel = torch.clamp(pred_vel, min=-4.0, max=4.0) 81 | next_state = torch.cat((pred_pos.squeeze(),pred_vel.squeeze()),dim=0) 82 | return next_state.squeeze() 83 | 84 | 85 | 86 | # I did not make this inherit from nn.Module, because our GP implementation is not torch based 87 | class AbstractModel(object): 88 | 89 | # def forward(self, x): 90 | # raise NotImplementedError("Subclass must implement") 91 | 92 | def train_model(self, training_dataset, testing_dataset, training_params): 93 | raise NotImplementedError("Subclass must implement") 94 | 95 | # function that (if necessary) converts between numpy input x and torch, and returns a prediction in numpy 96 | def predict_np(self, x): 97 | raise NotImplementedError("Subclass must implement") 98 | 99 | def get_input_size(self): 100 | raise NotImplementedError("Subclass must implement") 101 | 102 | def get_output_size(self): 103 | raise NotImplementedError("Subclass must implement") 104 | 105 | def get_hyperparameters(self): 106 | return None 107 | 108 | 109 | 110 | class Dataset(torch.utils.data.Dataset): 111 | def __init__(self, x, y): 112 | self.dataset = [ 113 | (torch.FloatTensor(x[i]), torch.FloatTensor(y[i])) for i in range(len(x)) 114 | ] 115 | 116 | def __len__(self): 117 | return len(self.dataset) 118 | 119 | def __getitem__(self, idx): 120 | return self.dataset[idx] 121 | 122 | 123 | # creates K datasets out of X and Y 124 | # if N is the total number of data points, then this function splits it in to K subsets. and each dataset contains K-1 125 | # subsets. 126 | # so let's say K=5. We create 5 subsets. 127 | # Each datasets contains 4 out of the 5 datasets, by leaving out one of the K subsets. 128 | def split_to_subsets(X, Y, K): 129 | if K == 1: 130 | # for 1 split, do not resshuffle dataset 131 | return [Dataset(X, Y)] 132 | 133 | n_data = len(X) 134 | chunk_sz = int(math.ceil(n_data / K)) 135 | all_idx = np.random.permutation(n_data) 136 | 137 | datasets = [] 138 | # each dataset contains 139 | for i in range(K): 140 | start_idx = i * (chunk_sz) 141 | end_idx = min(start_idx + chunk_sz, n_data) 142 | dataset_idx = np.delete(all_idx, range(start_idx, end_idx), axis=0) 143 | X_subset = [X[idx] for idx in dataset_idx] 144 | Y_subset = [Y[idx] for idx in dataset_idx] 145 | datasets.append(Dataset(X_subset, Y_subset)) 146 | 147 | return datasets 148 | 149 | 150 | class NLLLoss(torch.nn.modules.loss._Loss): 151 | """ 152 | Specialized NLL loss used to predict both mean (the actual function) and the variance of the input data. 153 | """ 154 | 155 | def __init__(self, size_average=None, reduce=None, reduction="mean"): 156 | super(NLLLoss, self).__init__(size_average, reduce, reduction) 157 | 158 | def forward(self, net_output, target): 159 | assert net_output.dim() == 3 160 | assert net_output.size(0) == 2 161 | mean = net_output[0] 162 | var = net_output[1] 163 | reduction = "mean" 164 | ret = 0.5 * torch.log(var) + 0.5 * ((mean - target) ** 2) / var 165 | # ret = 0.5 * ((mean - target) ** 2) 166 | 167 | if reduction != "none": 168 | ret = torch.mean(ret) if reduction == "mean" else torch.sum(ret) 169 | return ret 170 | 171 | class EnsembleProbabilisticModel(AbstractModel): 172 | def __init__(self, model_cfg): 173 | super(EnsembleProbabilisticModel, self).__init__() 174 | 175 | self.input_dimension = model_cfg['state_dim'] + model_cfg['action_dim'] 176 | # predicting velocity only (second half of state space) 177 | assert model_cfg['state_dim'] % 2 == 0 178 | self.output_dimension = model_cfg['state_dim'] // 2 179 | if model_cfg['device'] == "gpu": 180 | self.device = model_cfg['gpu_name'] 181 | else: 182 | self.device = "cpu" 183 | self.ensemble_size = model_cfg['ensemble_size'] 184 | self.model_cfg = model_cfg 185 | 186 | self.reset() 187 | 188 | def reset(self): 189 | self.models = [PModel(self.model_cfg) for _ in range(self.ensemble_size)] 190 | 191 | def forward(self, x): 192 | x = torch.Tensor(x) 193 | means = [] 194 | variances = [] 195 | for eid in range(self.ensemble_size): 196 | mean_and_var = self.models[eid](x) 197 | means.append(mean_and_var[0]) 198 | variances.append(mean_and_var[1]) 199 | 200 | mean = sum(means) / len(means) 201 | dum = torch.zeros_like(variances[0]) 202 | for i in range(len(means)): 203 | dum_var2 = variances[i] 204 | dum_mean2 = means[i] * means[i] 205 | dum += dum_var2 + dum_mean2 206 | 207 | var = (dum / len(means)) - (mean * mean) 208 | # Clipping the variance to a minimum of 1e-3, we can interpret this as saying weexpect a minimum 209 | # level of noise 210 | # the clipping here is probably not necessary anymore because we're now clipping at the individual model level 211 | var = var.clamp_min(1e-3) 212 | return torch.stack((mean, var)) 213 | 214 | def predict_np(self, x_np): 215 | x = torch.Tensor(x_np) 216 | pred = self.forward(x).detach().cpu().numpy() 217 | return pred[0].squeeze(), pred[1].squeeze() 218 | 219 | def train_model(self, training_dataset, testing_dataset, training_params): 220 | X = training_dataset["X"] 221 | Y = training_dataset["Y"] 222 | 223 | datasets = split_to_subsets(X, Y, self.ensemble_size) 224 | 225 | for m in range(self.ensemble_size): 226 | print(colored("training model={}".format(m), "green")) 227 | self.models[m].train_model(datasets[m]) 228 | 229 | def get_gradient(self, x_np): 230 | 231 | x = torch.Tensor(x_np).requires_grad_() 232 | output_mean, _ = self.forward(x) 233 | gradients = [] 234 | # get gradients of ENN with respect to x and u 235 | for output_dim in range(self.output_dimension): 236 | grads = torch.autograd.grad( 237 | output_mean[0, output_dim], x, create_graph=True 238 | )[0].data 239 | gradients.append(grads.detach().cpu().numpy()[0, :]) 240 | 241 | return np.array(gradients).reshape( 242 | [self.output_dimension, self.input_dimension] 243 | ) 244 | 245 | def get_input_size(self): 246 | return self.input_dimension 247 | 248 | def get_output_size(self): 249 | return self.output_dimension 250 | 251 | def get_hyper_params(self): 252 | return None 253 | 254 | 255 | class PModel(nn.Module): 256 | """ 257 | Probabilistic network 258 | Output a 3d tensor: 259 | d0 : always 2, first element is mean and second element is variance 260 | d1 : batch size 261 | d2 : output size (number of dimensions in the output of the modeled function) 262 | """ 263 | 264 | def __init__(self, config): 265 | super(PModel, self).__init__() 266 | if config["device"] == "gpu": 267 | self.device = config["gpu_name"] 268 | else: 269 | self.device = "cpu" 270 | self.input_sz = config['state_dim'] + config['action_dim'] 271 | self.output_sz = config['output_dim'] 272 | 273 | self.learning_rate = config["learning_rate"] 274 | self.display_epoch = config["display_epoch"] 275 | self.epochs = config["epochs"] 276 | 277 | w = config["hidden_size"] 278 | 279 | self.layers = nn.Sequential( 280 | nn.Linear(self.input_sz, w[0]), 281 | nn.Tanh(), 282 | nn.Linear(w[0], w[1]), 283 | nn.Tanh(), 284 | ) 285 | 286 | self.mean = nn.Linear(w[1], self.output_sz) 287 | self.var = nn.Sequential(nn.Linear(w[1], self.output_sz), nn.Softplus()) 288 | self.to(self.device) 289 | 290 | def forward(self, x): 291 | x = x.to(device=self.device) 292 | assert x.dim() == 2, "Expected 2 dimensional input, got {}".format(x.dim()) 293 | assert x.size(1) == self.input_sz 294 | y = self.layers(x) 295 | mean_p = self.mean(y) 296 | var_p = self.var(y) 297 | # Clipping the variance to a minimum of 1e-3, we can interpret this as saying weexpect a minimum 298 | # level of noise 299 | var_p = var_p.clamp_min(1e-3) 300 | return torch.stack((mean_p, var_p)) 301 | 302 | def predict_np(self, x_np): 303 | x = torch.Tensor(x_np) 304 | pred = self.forward(x).detach().cpu().numpy() 305 | return pred[0].squeeze(), pred[1].squeeze() 306 | 307 | def train_model(self, training_data): 308 | train_loader = torch.utils.data.DataLoader( 309 | training_data, batch_size=64, num_workers=0 310 | ) 311 | optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) 312 | loss_fn = NLLLoss() 313 | for epoch in range(self.epochs): 314 | losses = [] 315 | for batch, (data, target) in enumerate( 316 | train_loader, 1 317 | ): # This is the training loader 318 | x = data.type(torch.FloatTensor).to(device=self.device) 319 | y = target.type(torch.FloatTensor).to(device=self.device) 320 | 321 | if x.dim() == 1: 322 | x = x.unsqueeze(0).t() 323 | if y.dim() == 1: 324 | y = y.unsqueeze(0).t() 325 | 326 | py = self.forward(x) 327 | loss = loss_fn(py, y) 328 | optimizer.zero_grad() 329 | loss.backward() 330 | optimizer.step() 331 | losses.append(loss.item()) 332 | 333 | if epoch % self.display_epoch == 0: 334 | print( 335 | colored( 336 | "epoch={}, loss={}".format(epoch, np.mean(losses)), "yellow" 337 | ) 338 | ) -------------------------------------------------------------------------------- /ml3/ml3_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | def test_ml3_loss_mountain_car(policy, ml3_loss, opt_iter, *args): 7 | 8 | opt = torch.optim.SGD(policy.parameters(), lr=policy.learning_rate) 9 | for i in range(opt_iter): 10 | s_tr, a_tr, g_tr = policy.roll_out(*args) 11 | pred_task_loss = ml3_loss(torch.cat([s_tr[:-1], a_tr, g_tr], dim=1)).mean() 12 | opt.zero_grad() 13 | pred_task_loss.backward() 14 | opt.step() 15 | s_tr, a_tr, g_tr = policy.roll_out(*args) 16 | print('last state: ', s_tr[-1]) 17 | return s_tr.detach().numpy() 18 | 19 | 20 | def test_ml3_loss_reacher(policy, ml3_loss, opt_iter, *args): 21 | opt = torch.optim.SGD(policy.parameters(), lr=policy.learning_rate) 22 | for i in range(opt_iter): 23 | s_tr, a_tr, g_tr = policy.roll_out(*args) 24 | meta_input = torch.cat([s_tr[:-1], a_tr, g_tr], dim=1) 25 | pred_task_loss = ml3_loss(meta_input).mean() 26 | opt.zero_grad() 27 | pred_task_loss.backward() 28 | opt.step() 29 | return s_tr.detach().numpy() 30 | 31 | 32 | def test_ml3_loss_shaped_sine(sine_model, ml3_loss, opt_iter, test_x, test_y): 33 | opt = torch.optim.SGD(sine_model.parameters(), lr=sine_model.learning_rate) 34 | for i in range(opt_iter): 35 | yp = sine_model(test_x) 36 | meta_input = torch.cat([test_x, yp, test_y], dim=1) 37 | pred_task_loss = ml3_loss(meta_input).mean() 38 | opt.zero_grad() 39 | pred_task_loss.backward() 40 | opt.step() 41 | yp = sine_model(test_x) 42 | print('last state: ', yp[-1]) 43 | print('label: ',test_y[-1]) 44 | plt.plot(yp.detach().numpy()) 45 | plt.plot(test_y.detach().numpy()) 46 | plt.show() -------------------------------------------------------------------------------- /ml3/ml3_train.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | import higher 4 | import numpy as np 5 | from ml3.shaped_sine_utils import plot_loss, generate_sinusoid_batch 6 | from ml3.optimizee import ShapedSineModel 7 | 8 | 9 | def meta_train_mountain_car(policy,ml3_loss,task_loss_fn,s_0,goal,goal_extra,n_outer_iter,n_inner_iter,time_horizon,shaped_loss): 10 | s_0 = torch.Tensor(s_0) 11 | goal = torch.Tensor(goal) 12 | goal_extra = torch.Tensor(goal_extra) 13 | 14 | inner_opt = torch.optim.SGD(policy.parameters(), lr=policy.learning_rate) 15 | meta_opt = torch.optim.Adam(ml3_loss.parameters(), lr=ml3_loss.learning_rate) 16 | 17 | for outer_i in range(n_outer_iter): 18 | # set gradient with respect to meta loss parameters to 0 19 | meta_opt.zero_grad() 20 | for _ in range(n_inner_iter): 21 | inner_opt.zero_grad() 22 | with higher.innerloop_ctx(policy, inner_opt, copy_initial_weights=False) as (fpolicy, diffopt): 23 | # use current meta loss to update model 24 | s_tr, a_tr, g_tr = fpolicy.roll_out(s_0, goal, time_horizon) 25 | 26 | loss_input = torch.cat([s_tr[:-1], a_tr, g_tr], dim=1) 27 | pred_task_loss = ml3_loss(loss_input).mean() 28 | diffopt.step(pred_task_loss) 29 | 30 | # compute task loss 31 | s, a, g = fpolicy.roll_out(s_0, goal, time_horizon) 32 | task_loss = task_loss_fn(a, s[:], goal, goal_extra, shaped_loss) 33 | # backprop grad wrt to task loss 34 | task_loss.backward() 35 | 36 | meta_opt.step() 37 | 38 | if outer_i % 100 == 0: 39 | print("meta iter: {} loss: {}".format(outer_i, task_loss.item())) 40 | print('last state', s[-1]) 41 | 42 | 43 | def meta_train_mbrl_reacher(policy, ml3_loss, dmodel, env, task_loss_fn, goals, n_outer_iter, n_inner_iter, time_horizon, exp_folder): 44 | goals = torch.Tensor(goals) 45 | 46 | meta_opt = torch.optim.Adam(ml3_loss.parameters(), lr=ml3_loss.learning_rate) 47 | 48 | for outer_i in range(n_outer_iter): 49 | # set gradient with respect to meta loss parameters to 0 50 | meta_opt.zero_grad() 51 | all_loss = 0 52 | for goal in goals: 53 | goal = torch.Tensor(goal) 54 | policy.reset() 55 | inner_opt = torch.optim.SGD(policy.parameters(), lr=policy.learning_rate) 56 | for _ in range(n_inner_iter): 57 | inner_opt.zero_grad() 58 | with higher.innerloop_ctx(policy, inner_opt, copy_initial_weights=False) as (fpolicy, diffopt): 59 | # use current meta loss to update model 60 | s_tr, a_tr, g_tr = fpolicy.roll_out(goal, time_horizon, dmodel, env) 61 | meta_input = torch.cat([s_tr[:-1].detach(), a_tr, g_tr.detach()], dim=1) 62 | pred_task_loss = ml3_loss(meta_input).mean() 63 | diffopt.step(pred_task_loss) 64 | # compute task loss 65 | s, a, g = fpolicy.roll_out(goal, time_horizon, dmodel, env) 66 | task_loss = task_loss_fn(a, s[:], goal).mean() 67 | 68 | # collect losses for logging 69 | all_loss += task_loss 70 | # backprop grad wrt to task loss 71 | task_loss.backward() 72 | 73 | if outer_i % 100 == 0: 74 | # roll out in real environment, to monitor training and tp collect data for dynamics model update 75 | states, actions, _ = fpolicy.roll_out(goal, time_horizon, dmodel, env, real_rollout=True) 76 | print("meta iter: {} loss: {}".format(outer_i, (torch.mean((states[-1,:2]-goal[:2])**2)))) 77 | if outer_i % 300 == 0 and outer_i < 3001: 78 | # update dynamics model under current optimal policy 79 | dmodel.train(torch.Tensor(states), torch.Tensor(actions)) 80 | 81 | # step optimizer to update meta loss network 82 | meta_opt.step() 83 | torch.save(ml3_loss.state_dict(), f'{exp_folder}/ml3_loss_reacher.pt') 84 | 85 | 86 | def meta_train_shaped_sine(n_outer_iter,shaped,num_task,n_inner_iter,sine_model,ml3_loss,task_loss_fn, exp_folder): 87 | theta_ranges = [] 88 | landscape_with_extra = [] 89 | landscape_mse = [] 90 | 91 | meta_opt = torch.optim.Adam(ml3_loss.parameters(), lr=ml3_loss.learning_rate) 92 | 93 | for outer_i in range(n_outer_iter): 94 | # set gradient with respect to meta loss parameters to 0 95 | batch_inputs, batch_labels, batch_thetas = generate_sinusoid_batch(num_task, 64, n_inner_iter) 96 | for task in range(num_task): 97 | sine_model = ShapedSineModel() 98 | inner_opt = torch.optim.SGD([sine_model.freq], lr=sine_model.learning_rate) 99 | for step in range(n_inner_iter): 100 | inputs = torch.Tensor(batch_inputs[task, step, :]) 101 | labels = torch.Tensor(batch_labels[task, step, :]) 102 | label_thetas = torch.Tensor(batch_thetas[task, step, :]) 103 | 104 | ''' Updating the frequency parameters, taking gradient of theta wrt meta loss ''' 105 | with higher.innerloop_ctx(sine_model, inner_opt) as (fmodel, diffopt): 106 | # use current meta loss to update model 107 | yp = fmodel(inputs) 108 | meta_input = torch.cat([inputs, yp, labels], dim=1) 109 | 110 | meta_out = ml3_loss(meta_input) 111 | loss = meta_out.mean() 112 | diffopt.step(loss) 113 | 114 | yp = fmodel(inputs) 115 | task_loss = task_loss_fn(inputs, yp, labels, shaped, fmodel.freq, label_thetas) 116 | 117 | sine_model.freq = torch.nn.Parameter(fmodel.freq.clone().detach()) 118 | inner_opt = torch.optim.SGD([sine_model.freq], lr=sine_model.learning_rate) 119 | 120 | ''' updating the learned loss ''' 121 | meta_opt.zero_grad() 122 | task_loss.mean().backward() 123 | meta_opt.step() 124 | 125 | if outer_i % 100 == 0: 126 | print("task loss: {}".format(task_loss.mean().item())) 127 | 128 | torch.save(ml3_loss.state_dict(), f'{exp_folder}/ml3_loss_shaped_sine_' + str(shaped) + '.pt') 129 | 130 | if outer_i%10==0: 131 | t_range, l_with_extra, l_mse = plot_loss(shaped, exp_folder) 132 | theta_ranges.append(t_range) 133 | landscape_with_extra.append(l_with_extra) 134 | landscape_mse.append(l_mse) 135 | np.save(f'{exp_folder}/theta_ranges_'+str(shaped)+'_.npy', theta_ranges) 136 | np.save(f'{exp_folder}/landscape_with_extra_'+str(shaped)+'_.npy',landscape_with_extra) 137 | np.save(f'{exp_folder}/landscape_mse_'+str(shaped)+'_.npy',landscape_mse) 138 | 139 | -------------------------------------------------------------------------------- /ml3/optimizee.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | from ml3.envs.mountain_car import MountainCar 6 | 7 | 8 | def weight_init(module): 9 | if isinstance(module, nn.Linear): 10 | nn.init.xavier_uniform_(module.weight, gain=1.0) 11 | if module.bias is not None: 12 | module.bias.data.zero_() 13 | 14 | 15 | def weight_reset(m): 16 | if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): 17 | m.reset_parameters() 18 | 19 | 20 | class SineModel(nn.Module): 21 | 22 | def __init__(self, in_dim, hidden_dim, out_dim): 23 | super(SineModel, self).__init__() 24 | net_dim = [in_dim] + hidden_dim 25 | 26 | layers = [] 27 | 28 | for i in range(1, len(net_dim)): 29 | layers.append(nn.Linear(net_dim[i-1], net_dim[i])) 30 | layers.append(nn.ReLU()) 31 | 32 | self.layers = nn.Sequential(*layers) 33 | self.mean_pred = nn.Linear(hidden_dim[-1], out_dim) 34 | 35 | def reset(self): 36 | self.layers.apply(weight_init) 37 | self.mean_pred.apply(weight_init) 38 | 39 | def forward(self, x): 40 | feat = self.layers(x) 41 | return self.mean_pred(feat) 42 | 43 | 44 | class MC_Policy(nn.Module): 45 | 46 | def __init__(self, pi_in, pi_out): 47 | super(MC_Policy, self).__init__() 48 | 49 | num_neurons = 200 50 | self.policy = nn.Sequential(nn.Linear(pi_in, num_neurons,bias=False), 51 | nn.Linear(num_neurons, pi_out,bias=False)) 52 | self.learning_rate = 1e-3 53 | self.env = MountainCar() 54 | 55 | def forward(self, x): 56 | return self.policy(x) 57 | 58 | def reset_gradients(self): 59 | for i, param in enumerate(self.policy.parameters()): 60 | param.detach() 61 | 62 | def roll_out(self, s_0, goal, time_horizon): 63 | state = torch.Tensor(self.env.reset_to(s_0)) 64 | states = [] 65 | actions = [] 66 | states.append(state) 67 | for t in range(time_horizon): 68 | 69 | u = self.forward(state) 70 | u = u.clamp(self.env.min_action, self.env.max_action) 71 | state = self.env.sim_step_torch(state.squeeze(), u.squeeze()).clone() 72 | states.append(state.clone()) 73 | actions.append(u.clone()) 74 | 75 | running_reward = torch.norm(state-goal) 76 | rewards = [torch.Tensor([running_reward])]*time_horizon 77 | return torch.stack(states), torch.stack(actions), torch.stack(rewards) 78 | 79 | 80 | class Reacher_Policy(nn.Module): 81 | 82 | def __init__(self, pi_in, pi_out,exp_folder): 83 | super(Reacher_Policy, self).__init__() 84 | 85 | num_neurons = 64 86 | self.activation = torch.nn.Tanh 87 | self.policy = torch.nn.Sequential(torch.nn.Linear(pi_in, num_neurons), 88 | self.activation(), 89 | torch.nn.Linear(num_neurons, num_neurons), 90 | self.activation(), 91 | torch.nn.Linear(num_neurons, pi_out)) 92 | self.learning_rate = 1e-4 93 | self.norm_in = torch.Tensor(np.array([1.0,1.0,8.0,8.0,1.0,1.0,1.0,1.0])) 94 | self.exp_folder = exp_folder 95 | torch.save(self.state_dict(), f"{self.exp_folder}/init_policy.pt") 96 | 97 | def forward(self, x): 98 | return self.policy(x) 99 | 100 | def reset(self): 101 | self.load_state_dict(torch.load(f"{self.exp_folder}/init_policy.pt")) 102 | self.eval() 103 | 104 | def roll_out(self, goal, time_horizon, dmodel, env, real_rollout=False): 105 | 106 | state = torch.Tensor(env.reset()) 107 | states = [] 108 | actions = [] 109 | states.append(state.clone()) 110 | for t in range(time_horizon): 111 | 112 | u = self.forward(torch.cat((state.detach(), goal[:]), dim=0) / self.norm_in) 113 | u = u.clamp(-1.0, 1.0) 114 | if not real_rollout: 115 | pred_next_state = dmodel.step_model(state.squeeze(), u.squeeze()).clone() 116 | else: 117 | pred_next_state = torch.Tensor(env.step_model(state.squeeze().detach().numpy(), u.squeeze().detach().numpy()).copy()) 118 | states.append(pred_next_state.clone()) 119 | actions.append(u.clone()) 120 | state_cost = torch.norm(pred_next_state[:]-goal[:]).detach().unsqueeze(0) 121 | state = pred_next_state.clone() 122 | 123 | # rewards to pass to meta loss 124 | rewards = [state_cost]*time_horizon 125 | return torch.stack(states), torch.stack(actions), torch.stack(rewards).detach() 126 | 127 | 128 | class ShapedSineModel(torch.nn.Module): 129 | 130 | def __init__(self,theta=None): 131 | super(ShapedSineModel, self).__init__() 132 | if theta is None: 133 | self.freq = torch.nn.Parameter(torch.Tensor([0.1])) 134 | else: 135 | self.freq = torch.nn.Parameter(torch.Tensor([theta])) 136 | self.learning_rate = 1.0 137 | 138 | def forward(self, x): 139 | return torch.sin(self.freq*x) -------------------------------------------------------------------------------- /ml3/shaped_sine_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import torch 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.animation as animation 6 | from ml3.optimizee import ShapedSineModel 7 | from ml3.learnable_losses import Ml3_loss_shaped_sine as MetaNetwork 8 | 9 | 10 | '''GENERATE DATA''' 11 | def generate_sinusoid_batch(num_tasks, num_examples_task, num_steps, random_steps=False, 12 | freq_range=[-5.0, 5.0], input_range=[-5.0, 5.0]): 13 | """ Generate samples from random sine functions. """ 14 | freq = np.random.uniform(freq_range[0], freq_range[1], [num_tasks]) 15 | outputs = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 16 | thetas = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 17 | init_inputs = np.zeros([num_tasks, num_steps, num_examples_task, 1]) 18 | 19 | for task in range(num_tasks): 20 | if random_steps: 21 | init_inputs[task] = np.random.uniform(input_range[0], input_range[1], 22 | [num_steps, num_examples_task, 1]) 23 | else: 24 | init_inputs[task] = np.repeat(np.random.uniform(input_range[0], input_range[1], 25 | [1, num_examples_task, 1]), num_steps, -3) 26 | 27 | outputs[task] = np.sin(freq[task]*init_inputs[task]) 28 | thetas[task] = np.zeros_like(outputs[task]) + freq[task] 29 | return init_inputs, outputs,thetas 30 | 31 | '''PLOTTING THE LOSS LANDSCAPES FOR ILLUSTRATION''' 32 | def plot_loss(extra, exp_folder, freq=0.5): 33 | meta = MetaNetwork() 34 | meta.load_state_dict(torch.load(f'{exp_folder}/ml3_loss_shaped_sine_'+str(extra)+'.pt')) 35 | meta.eval() 36 | 37 | loss_landscape = [] 38 | 39 | theta_ranges = np.arange(-7.0, 7.0, 0.1) 40 | test_x = np.expand_dims(np.arange(-5.0, 5.0, 0.1), 1) 41 | test_y = np.sin(freq * test_x) 42 | x = torch.Tensor(test_x) 43 | y = torch.Tensor(test_y) 44 | 45 | for theta in theta_ranges: 46 | pi = ShapedSineModel(theta) 47 | pi.learning_rate = 0.1 48 | pi_out = pi(x) 49 | loss = 0.5 * (pi_out - y) ** 2 50 | loss_landscape.append(loss.mean().detach().numpy()) 51 | 52 | meta_loss_landscape = [] 53 | for theta in theta_ranges: 54 | pi = ShapedSineModel(theta) 55 | policy_theta = torch.Tensor(np.zeros_like(test_y)) + pi.freq 56 | pi_out = pi(x) 57 | meta_input = torch.cat([x, pi_out, y], 1) 58 | loss = meta(meta_input).mean() 59 | meta_loss_landscape.append(loss.clone().mean().detach().numpy()) 60 | 61 | return theta_ranges, np.array(meta_loss_landscape), np.array(loss_landscape) 62 | 63 | 64 | def render(theta_ranges,loss,color,freq=0.5,file_path='./ml3_loss_sine.gif', mode='gif'): 65 | """ When the method is called it saves an animation 66 | of what happened until that point in the episode. 67 | Ideally it should be called at the end of the episode, 68 | and every k episodes. 69 | 70 | ATTENTION: It requires avconv and/or imagemagick installed. 71 | @param file_path: the name and path of the video file 72 | @param mode: the file can be saved as 'gif' or 'mp4' 73 | """ 74 | 75 | fig = plt.figure(figsize=(5,5)) 76 | ax = fig.add_subplot(111,autoscale_on=False, xlim=(-7.0, 7.0), ylim=(0.0, 1.0)) 77 | 78 | 79 | ax.axvline(x=freq, c='red') 80 | delta_t = 1.0/10.0 81 | dot, = ax.plot([], [],color=color) 82 | time_text = ax.text(0.25, 1.05, '', transform=ax.transAxes,fontsize=14) 83 | _theta_ranges = theta_ranges 84 | _loss = loss 85 | _delta_t = delta_t 86 | 87 | def _init(): 88 | dot.set_data([], []) 89 | time_text.set_text('') 90 | return dot, time_text 91 | 92 | def _animate(i): 93 | x = _theta_ranges[i] 94 | y = _loss[i] 95 | dot.set_data(x, y) 96 | time_text.set_text("Iteration: "+str(i)) 97 | return dot, time_text 98 | 99 | ani = animation.FuncAnimation(fig, _animate, np.arange(1, len(theta_ranges)), 100 | blit=True, init_func=_init, repeat=False) 101 | 102 | if mode == 'gif': 103 | ani.save(file_path, writer='imagemagick', fps=int(1 / delta_t)) 104 | # Clear the figure 105 | fig.clear() 106 | plt.close(fig) -------------------------------------------------------------------------------- /ml3/sine_regression_task.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | import os 3 | 4 | import numpy as np 5 | import torch.nn as nn 6 | import torch 7 | import higher 8 | 9 | from ml3.optimizee import SineModel 10 | from ml3.sine_task_sampler import SineTaskSampler 11 | from ml3.learnable_losses import ML3_SineRegressionLoss 12 | 13 | 14 | def regular_train(loss_fn, eval_loss_fn, task_model, x_tr, y_tr, exp_cfg): 15 | n_iter = exp_cfg['n_gradient_steps_at_test'] 16 | lr = exp_cfg['inner_lr'] 17 | 18 | loss_trace = [] 19 | 20 | optimizer = torch.optim.SGD(task_model.parameters(), lr=lr) 21 | for i in range(n_iter): 22 | optimizer.zero_grad() 23 | y_pred = task_model(x_tr) 24 | loss = loss_fn(y_pred, y_tr) 25 | 26 | loss.backward() 27 | optimizer.step() 28 | 29 | eval_loss = eval_loss_fn(y_pred, y_tr) 30 | loss_trace.append(eval_loss.item()) 31 | 32 | return loss_trace 33 | 34 | 35 | def meta_train(meta_loss_model, meta_optimizer, meta_objective, task_sampler_train, task_sampler_test, exp_cfg): 36 | 37 | num_tasks = exp_cfg['num_train_tasks'] 38 | n_outer_iter= exp_cfg['n_outer_iter'] 39 | inner_lr = exp_cfg['inner_lr'] 40 | 41 | results = [] 42 | 43 | task_models = [] 44 | task_opts = [] 45 | for i in range(num_tasks): 46 | task_models.append(SineModel(in_dim=exp_cfg['model']['in_dim'], 47 | hidden_dim=exp_cfg['model']['hidden_dim'], 48 | out_dim=1)) 49 | task_opts.append(torch.optim.SGD(task_models[i].parameters(), lr=inner_lr)) 50 | 51 | for outer_i in range(n_outer_iter): 52 | # Sample a batch of support and query images and labels. 53 | 54 | x_spt, y_spt, x_qry, y_qry = task_sampler_train.sample() 55 | 56 | for i in range(num_tasks): 57 | task_models[i].reset() 58 | 59 | qry_losses = [] 60 | for _ in range(1): 61 | pred_losses = [] 62 | meta_optimizer.zero_grad() 63 | 64 | for i in range(num_tasks): 65 | # zero gradients wrt to meta loss parameters 66 | with higher.innerloop_ctx(task_models[i], task_opts[i], 67 | copy_initial_weights=False) as (fmodel, diffopt): 68 | 69 | # update model parameters via meta loss 70 | yp = fmodel(x_spt[i]) 71 | pred_loss = meta_loss_model(yp, y_spt[i]) 72 | diffopt.step(pred_loss) 73 | 74 | # compute task loss with new model 75 | yp = fmodel(x_spt[i]) 76 | task_loss = meta_objective(yp, y_spt[i]) 77 | 78 | # this accumulates gradients wrt to meta parameters 79 | task_loss.backward() 80 | qry_losses.append(task_loss.item()) 81 | 82 | meta_optimizer.step() 83 | 84 | avg_qry_loss = sum(qry_losses) / num_tasks 85 | if outer_i % 10 == 0: 86 | res_train_eval_reg = eval(task_sampler=task_sampler_train, exp_cfg=exp_cfg, 87 | train_loss_fn=nn.MSELoss(), eval_loss_fn=nn.MSELoss()) 88 | 89 | res_train_eval_ml3 = eval(task_sampler=task_sampler_train, exp_cfg=exp_cfg, 90 | train_loss_fn=meta_loss_model, eval_loss_fn=nn.MSELoss()) 91 | 92 | res_test_eval_reg = eval(task_sampler=task_sampler_test, exp_cfg=exp_cfg, 93 | train_loss_fn=nn.MSELoss(), eval_loss_fn=nn.MSELoss()) 94 | 95 | res_test_eval_ml3 = eval(task_sampler=task_sampler_test, exp_cfg=exp_cfg, 96 | train_loss_fn=meta_loss_model, eval_loss_fn=nn.MSELoss()) 97 | 98 | res = {} 99 | res['train_reg'] = res_train_eval_reg 100 | res['train_ml3'] = res_train_eval_ml3 101 | res['test_reg'] = res_test_eval_reg 102 | res['test_ml3'] = res_test_eval_ml3 103 | res['task_loss'] = {} 104 | res['task_loss']['mse'] = qry_losses 105 | results.append(res) 106 | test_loss_ml3 = np.mean(res_test_eval_ml3['mse']) 107 | test_loss_reg = np.mean(res_test_eval_reg['mse']) 108 | print( 109 | f'[Epoch {outer_i:.2f}] Train Loss: {avg_qry_loss:.2f}]| Test Loss ML3: {test_loss_ml3:.2f} | TestLoss REG: {test_loss_reg:.2f}' 110 | ) 111 | 112 | return results 113 | 114 | 115 | def eval(task_sampler, exp_cfg, train_loss_fn, eval_loss_fn): 116 | seed = exp_cfg['seed'] 117 | num_tasks = task_sampler.num_tasks_total 118 | 119 | np.random.seed(seed) 120 | torch.manual_seed(seed) 121 | 122 | mse = [] 123 | nmse = [] 124 | loss_trace = [] 125 | x, y, _, _ = task_sampler.sample() 126 | for i in range(num_tasks): 127 | task_model_test = SineModel(in_dim=exp_cfg['model']['in_dim'], 128 | hidden_dim=exp_cfg['model']['hidden_dim'], 129 | out_dim=1) 130 | loss = regular_train(loss_fn=train_loss_fn, eval_loss_fn=eval_loss_fn, task_model=task_model_test, 131 | x_tr=x[i], y_tr=y[i], exp_cfg=exp_cfg) 132 | yp = task_model_test(x[i]) 133 | l = eval_loss_fn(yp, y[i]) 134 | 135 | mse.append(l.item()) 136 | nmse.append(l.item()/y[i].var()) 137 | loss_trace.append(loss) 138 | 139 | res = {'nmse': nmse, 'mse': mse, 'loss_trace': loss_trace} 140 | return res 141 | 142 | 143 | def main(exp_cfg): 144 | seed = exp_cfg['seed'] 145 | num_train_tasks = exp_cfg['num_train_tasks'] 146 | num_test_tasks = exp_cfg['num_test_tasks'] 147 | outer_lr = exp_cfg['outer_lr'] 148 | 149 | np.random.seed(seed) 150 | torch.manual_seed(seed) 151 | 152 | meta_loss_model = ML3_SineRegressionLoss(in_dim=exp_cfg['metaloss']['in_dim'], 153 | hidden_dim=exp_cfg['metaloss']['hidden_dim']) 154 | 155 | meta_optimizer = torch.optim.Adam(meta_loss_model.parameters(), lr=outer_lr) 156 | 157 | meta_objective = nn.MSELoss() 158 | 159 | task_sampler_train = SineTaskSampler(num_tasks_total=num_train_tasks, num_tasks_per_batch=num_train_tasks, num_data_points=100, 160 | amp_range=[1.0, 1.0], 161 | input_range=[-2.0, 2.0], 162 | ) 163 | 164 | task_sampler_test = SineTaskSampler(num_tasks_total=num_test_tasks, num_tasks_per_batch=num_test_tasks, num_data_points=100, 165 | input_range=[-5.0, 5.0], 166 | amp_range=[0.2, 5.0], 167 | phase_range=[-np.pi, np.pi] 168 | ) 169 | # 170 | res = meta_train(meta_loss_model=meta_loss_model, meta_optimizer=meta_optimizer, meta_objective=meta_objective, 171 | task_sampler_train=task_sampler_train, task_sampler_test=task_sampler_test, 172 | exp_cfg=exp_cfg) 173 | 174 | data_file = os.path.join(exp_cfg['log_dir'], exp_cfg['exp_log_file_name']) 175 | 176 | data_dir = os.path.dirname(data_file) 177 | if data_dir is not '' and not os.path.exists(data_dir): # Create directory if it doesn't exist. 178 | os.makedirs(data_dir) 179 | torch.save(res, data_file) 180 | 181 | 182 | if __name__ == "__main__": 183 | exp_cfg = {} 184 | exp_cfg['seed'] = 0 185 | exp_cfg['num_train_tasks'] = 1 186 | exp_cfg['num_test_tasks'] = 10 187 | exp_cfg['n_outer_iter'] = 500 188 | exp_cfg['n_gradient_steps_at_test'] = 100 189 | exp_cfg['inner_lr'] = 0.001 190 | exp_cfg['outer_lr'] = 0.001 191 | 192 | exp_cfg['model'] = {} 193 | exp_cfg['model']['in_dim'] = 1 194 | exp_cfg['model']['hidden_dim'] = [100, 10] 195 | 196 | exp_cfg['metaloss'] = {} 197 | exp_cfg['metaloss']['in_dim'] = 2 198 | exp_cfg['metaloss']['hidden_dim'] = [50, 50] 199 | 200 | model_arch_str = str(exp_cfg['model']['hidden_dim']) 201 | meta_arch_str = "{}".format(exp_cfg['metaloss']['hidden_dim']) 202 | exp_cfg['log_dir'] = "sin_cos_exp" 203 | exp_file = "sine_regression_seed_{}.pkl".format(exp_cfg['seed']) 204 | exp_cfg['exp_log_file_name'] = exp_file 205 | main(exp_cfg) 206 | -------------------------------------------------------------------------------- /ml3/sine_task_sampler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | 3 | import numpy as np 4 | import torch 5 | 6 | 7 | class SineTaskSampler(object): 8 | def __init__(self, num_tasks_total, num_tasks_per_batch, num_data_points, 9 | input_range=[-5.0, 5.0], 10 | amp_range=[1.0, 1.0], 11 | freq_range=[1.0, 1.0], 12 | phase_range=[np.pi, np.pi], 13 | fun_type="sine"): 14 | 15 | self.input_range = input_range 16 | 17 | self.amp_range = amp_range 18 | self.freq_range = freq_range 19 | self.phase_range = phase_range 20 | 21 | self.fun_type = fun_type 22 | 23 | self.observation_space = np.ones([1], dtype=np.float32) 24 | self.action_space = np.ones([1], dtype=np.float32) 25 | self.sample_space = np.ones([1], dtype=np.float32) 26 | 27 | self.num_tasks_total = num_tasks_total 28 | self.num_tasks_per_task = num_tasks_per_batch 29 | self.train_tasks = self._sample_tasks(num_tasks_total, num_data_points) 30 | self.valid_tasks = self._sample_tasks(num_tasks_total, num_data_points) 31 | 32 | def _sample_tasks(self, num_tasks, n_data_points): 33 | """ 34 | Returns a list of task parameters 35 | """ 36 | amp = np.random.uniform(self.amp_range[0], self.amp_range[1], [num_tasks]).astype(np.float32) 37 | freq = np.random.uniform(self.freq_range[0], self.freq_range[1], [num_tasks]).astype(np.float32) 38 | phase = np.random.uniform(self.phase_range[0], self.phase_range[1], [num_tasks]).astype(np.float32) 39 | inputs = np.random.uniform(self.input_range[0], self.input_range[1], [num_tasks, n_data_points, 1]).astype(np.float32) 40 | 41 | return [[amp[i], freq[i], phase[i], inputs[i]] for i in range(num_tasks)] 42 | 43 | def _sample_from_tasks(self, tasks): 44 | task_idx = np.random.permutation(self.num_tasks_total)[:self.num_tasks_per_task] 45 | inputs, targets = [], [] 46 | for i in task_idx: 47 | task_params = tasks[i] 48 | inputs_np = task_params[3] 49 | targets_np = (task_params[0] * np.sin(task_params[1] * (inputs_np - task_params[2]))).astype(np.float32) 50 | inputs.append(torch.FloatTensor(inputs_np)) 51 | targets.append(torch.FloatTensor(targets_np)) 52 | return inputs, targets 53 | 54 | def sample(self): 55 | """ 56 | Samples from a single task 57 | """ 58 | ## [traj_len=1, batch_size, obs_shape=1] 59 | train_inputs, train_targets = self._sample_from_tasks(self.train_tasks) 60 | valid_inputs, valid_targets = self._sample_from_tasks(self.valid_tasks) 61 | 62 | return train_inputs, train_targets, valid_inputs, valid_targets 63 | 64 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | ###################################################################### 3 | # \file setup.py 4 | # \author Franziska Meier 5 | ####################################################################### 6 | from setuptools import setup, find_packages 7 | 8 | install_requires = ["higher", "pybullet", "matplotlib", "termcolor", "differentiable_robot_model", "jupyter"] 9 | 10 | setup( 11 | name="l2l", 12 | author="Facebook AI Research", 13 | author_email="", 14 | version=1.0, 15 | packages=find_packages(), 16 | install_requires=install_requires, 17 | include_package_data=True, 18 | zip_safe=False, 19 | ) 20 | 21 | 22 | --------------------------------------------------------------------------------