├── .gitignore ├── README.md ├── docker ├── Dockerfile.gpu ├── Dockerfile.habana ├── Dockerfile.xlagpu ├── README.md ├── start_gpu_pytorch2 ├── start_habana_pytorch2 └── start_xla_pytorch2 ├── pytorch-compile-blogpost └── torch-compile-under-the-hood.ipynb ├── pytorch-graph-optimization ├── benchmark_torch-compile_resnet.ipynb ├── graph_optimization_torch_compile.ipynb └── inspecting_torch_compile.ipynb └── pytorch-intro-torch-compile ├── 1-toy-benchmarks.ipynb ├── 2-torch-compile-intro.ipynb ├── 3-inspecting-compiler-stack.ipynb └── 4-nn-example.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | torch_compile_debug/ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pytorch-examples 2 | A repository of PyTorch example 3 | -------------------------------------------------------------------------------- /docker/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:11.8.0-base-ubuntu22.04 2 | 3 | # Remove any third-party apt sources to avoid issues with expiring keys. 4 | RUN rm -f /etc/apt/sources.list.d/*.list 5 | 6 | # Install some basic utilities. 7 | RUN apt-get update && apt-get install -y \ 8 | build-essential \ 9 | graphviz \ 10 | autoconf \ 11 | automake \ 12 | gdb \ 13 | libffi-dev \ 14 | zlib1g-dev \ 15 | libssl-dev \ 16 | libsndfile1 \ 17 | curl \ 18 | wget \ 19 | vim \ 20 | ca-certificates \ 21 | git \ 22 | bzip2 \ 23 | libx11-6 \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | # Download and install Miniconda. 27 | RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh 28 | RUN bash /tmp/miniconda.sh -b -p /opt/conda 29 | ENV PATH=/opt/conda/bin:$PATH 30 | RUN conda init 31 | RUN pip install \ 32 | torch \ 33 | torchvision \ 34 | torchaudio \ 35 | jupyterlab \ 36 | ipykernel \ 37 | matplotlib \ 38 | ipywidgets \ 39 | pydot \ 40 | huggingface \ 41 | transformers \ 42 | datasets \ 43 | diffusers["torch"] \ 44 | accelerate \ 45 | soundfile \ 46 | librosa \ 47 | torchdiffeq 48 | RUN pip install \ 49 | xgboost \ 50 | scikit-learn \ 51 | tabulate \ 52 | bokeh 53 | 54 | RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" 55 | #ENTRYPOINT ["jupyter", "lab", "--ip='*'", "--NotebookApp.token=''", "--NotebookApp.password=''","--allow-root"] 56 | ENTRYPOINT ["jupyter", "lab", "--ip='*'", "--allow-root"] 57 | 58 | 59 | -------------------------------------------------------------------------------- /docker/Dockerfile.habana: -------------------------------------------------------------------------------- 1 | FROM vault.habana.ai/gaudi-docker/1.8.0/ubuntu20.04/habanalabs/pytorch-installer-1.13.1:latest 2 | RUN apt-get update 3 | RUN apt-get install build-essential -y 4 | RUN apt-get install autoconf automake gdb git libffi-dev zlib1g-dev libssl-dev libsndfile1 -y 5 | RUN pip install jupyterlab ipykernel matplotlib ipywidgets 6 | RUN pip install huggingface transformers datasets 7 | RUN pip install diffusers["torch"] accelerate soundfile librosa 8 | WORKDIR /pytorch-habana 9 | ENTRYPOINT ["jupyter", "lab", "--ip='*'", "--NotebookApp.token=''", "--NotebookApp.password=''","--allow-root"] 10 | -------------------------------------------------------------------------------- /docker/Dockerfile.xlagpu: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:11.8.0-base-ubuntu22.04 2 | 3 | # Remove any third-party apt sources to avoid issues with expiring keys. 4 | RUN rm -f /etc/apt/sources.list.d/*.list 5 | 6 | # Install some basic utilities. 7 | RUN apt-get update && apt-get install -y \ 8 | build-essential \ 9 | graphviz \ 10 | autoconf \ 11 | automake \ 12 | gdb \ 13 | libffi-dev \ 14 | zlib1g-dev \ 15 | libssl-dev \ 16 | libsndfile1 \ 17 | curl \ 18 | wget \ 19 | vim \ 20 | ca-certificates \ 21 | git \ 22 | bzip2 \ 23 | libx11-6 \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | # Download and install Miniconda. 27 | RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py38_23.1.0-1-Linux-x86_64.sh -O /tmp/miniconda.sh 28 | RUN bash /tmp/miniconda.sh -b -p /opt/conda 29 | ENV PATH=/opt/conda/bin:$PATH 30 | 31 | # Make RUN commands use the new environment: 32 | RUN pip install \ 33 | cloud-tpu-client==0.10 \ 34 | torch==2.0.0 \ 35 | torchvision==0.15.1 https://storage.googleapis.com/tpu-pytorch/wheels/cuda/118/torch_xla-2.0-cp38-cp38-linux_x86_64.whl 36 | RUN pip install \ 37 | jupyterlab \ 38 | ipykernel \ 39 | matplotlib \ 40 | ipywidgets \ 41 | pydot \ 42 | huggingface \ 43 | transformers \ 44 | datasets \ 45 | diffusers["torch"] \ 46 | accelerate \ 47 | soundfile \ 48 | librosa \ 49 | torchdiffeq 50 | RUN pip install \ 51 | xgboost \ 52 | scikit-learn \ 53 | bokeh 54 | 55 | RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" 56 | ENTRYPOINT ["jupyter", "lab", "--ip='*'", "--NotebookApp.token=''", "--NotebookApp.password=''","--allow-root"] 57 | 58 | 59 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | ### Dockerfiles for environments used in examples -------------------------------------------------------------------------------- /docker/start_gpu_pytorch2: -------------------------------------------------------------------------------- 1 | [ ! -z "$(docker ps -a -q)" ] && docker rm $(docker ps -a -q) 2 | callpath=$PWD 3 | cd $(dirname "$0") 4 | docker build -t pytorch2:gpu -f Dockerfile.gpu . 5 | cd $callpath 6 | [ ! -z "$(docker images -f dangling=true -q)" ] && docker rmi $(docker images -f dangling=true -q) 7 | docker run -it --rm \ 8 | --network=host \ 9 | --gpus all \ 10 | -v $PWD/:/workspace \ 11 | -v ~/.cache/:/root/.cache \ 12 | --workdir /workspace \ 13 | --name pytorch \ 14 | pytorch2:gpu 15 | -------------------------------------------------------------------------------- /docker/start_habana_pytorch2: -------------------------------------------------------------------------------- 1 | docker build -t pytorch-habana:latest -f docker/Dockerfile . 2 | docker rmi $(docker images -f dangling=true -q) 3 | docker run -it --network=host -v $PWD/:/pytorch-habana --workdir /pytorch-habana pytorch-habana:latest 4 | -------------------------------------------------------------------------------- /docker/start_xla_pytorch2: -------------------------------------------------------------------------------- 1 | [ ! -z "$(docker ps -a -q)" ] && docker rm $(docker ps -a -q) 2 | docker build -t pytorch2:xla -f docker/Dockerfile.xlagpu . 3 | [ ! -z "$(docker images -f dangling=true -q)" ] && docker rmi $(docker images -f dangling=true -q) 4 | docker run -it --rm \ 5 | --network=host \ 6 | --gpus all \ 7 | -v $PWD/:/pytorch-examples \ 8 | -v ~/.cache/:/root/.cache \ 9 | --workdir /pytorch-examples \ 10 | pytorch2:xla 11 | -------------------------------------------------------------------------------- /pytorch-compile-blogpost/torch-compile-under-the-hood.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "c0d1d55d-42e3-4445-b6c1-dfd95f7999a7", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "!rm -rf torch_compile_debug\n", 13 | "!rm *.svg" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "79b09805-e61c-4be4-aaf0-2dc148b0b736", 20 | "metadata": { 21 | "tags": [] 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "import torch\n", 26 | "import math\n", 27 | "import os\n", 28 | "import matplotlib.pyplot as plt\n", 29 | "import torch._dynamo\n", 30 | "from torchvision import models\n", 31 | "from torch.fx.passes.graph_drawer import FxGraphDrawer\n", 32 | "from IPython.display import Markdown as md\n", 33 | "\n", 34 | "device = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\"" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "id": "a51c7031-33d1-44ee-8398-3137b5f7d085", 41 | "metadata": { 42 | "tags": [] 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "def f(x):\n", 47 | " return torch.sin(x)**2 + torch.cos(x)**2\n", 48 | "\n", 49 | "md('''\n", 50 | "# $ y = f(x) = sin^2(x) + cos^2(x)$\n", 51 | "''')" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "id": "82568cbb-750b-4ed9-baf2-1cf2c6028c0d", 58 | "metadata": { 59 | "tags": [] 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "md('''\n", 64 | "## Optimization problem:\n", 65 | "### find $w^*$ that $\\displaystyle \\min_{w} f(w)$\n", 66 | "\n", 67 | "## Gradient update\n", 68 | "### $ w_{i+1} = w_{i} - g(\\\\nabla f(w))$\n", 69 | "## For SGD\n", 70 | "### $ g(\\\\nabla f(w)) = \\\\alpha*\\\\nabla f(w)$\n", 71 | "## Which makes the update for SGD:\n", 72 | "### $ w_{i+1} = w_{i} - \\\\alpha*\\\\nabla f(w)$\n", 73 | "\n", 74 | "## Loss function\n", 75 | "### $loss(w): loss(model(w,batch_{inputs}), batch_{outputs})$ \n", 76 | "''')" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "id": "4b4c1717-2fbe-43f6-86ad-1f12d397d33d", 83 | "metadata": { 84 | "tags": [] 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "md('''\n", 89 | "# **Forward graph:** $f(x) = sin^2(x)+cos^2(x)$ \\n\n", 90 | "# **Backward graph:** $\\\\frac {df(x)}{d\\\\vec{w}} = f\\'(x) = 2sin(x)cos(x) + 2cos(x)(-sin(x))$\n", 91 | "''')" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "id": "0ba8ff94-c457-41fd-b7ff-91ac36872bee", 98 | "metadata": { 99 | "tags": [] 100 | }, 101 | "outputs": [], 102 | "source": [ 103 | "torch.manual_seed(0)\n", 104 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 105 | "torch.nn.functional.mse_loss(f(x),torch.ones_like(x)) < 1e-10" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "f5f80ceb-7e1a-4140-a17c-b6827e6e7461", 112 | "metadata": { 113 | "tags": [] 114 | }, 115 | "outputs": [], 116 | "source": [ 117 | "torch.manual_seed(0)\n", 118 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 119 | "\n", 120 | "compiled_f = torch.compile(f)\n", 121 | "torch.nn.functional.mse_loss(compiled_f(x),torch.ones_like(x)) < 1e-10" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "id": "b3a9cf8b-37ef-4b00-9c1b-d21e9c0ed6af", 128 | "metadata": { 129 | "tags": [] 130 | }, 131 | "outputs": [], 132 | "source": [ 133 | "def inspect_backend(gm, sample_inputs):\n", 134 | " code = gm.print_readable()\n", 135 | " with open(\"forward.svg\", \"wb\") as file:\n", 136 | " file.write(FxGraphDrawer(gm,'f').get_dot_graph().create_svg())\n", 137 | " return gm.forward\n", 138 | "\n", 139 | "torch._dynamo.reset()\n", 140 | "compiled_f = torch.compile(f, backend=inspect_backend)\n", 141 | "\n", 142 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 143 | "out = compiled_f(x)\n", 144 | "\n", 145 | "md(f'''\n", 146 | "### Graph\n", 147 | "![]({'forward.svg'})\n", 148 | "''')" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": null, 154 | "id": "7ec2518d-1024-4aeb-a991-67de9dc51090", 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "id": "619928ed-ed3f-45a4-8ef0-783c70bafaed", 162 | "metadata": {}, 163 | "source": [ 164 | "# AOTAutograd and Aten IR" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "id": "b900c2c1-b83e-44ba-8eb2-c3d71c3bd53e", 171 | "metadata": { 172 | "tags": [] 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "import torch._dynamo\n", 177 | "from torch.fx.passes.graph_drawer import FxGraphDrawer\n", 178 | "from functorch.compile import make_boxed_func\n", 179 | "from torch._functorch.aot_autograd import aot_module_simplified\n", 180 | "\n", 181 | "def f(x):\n", 182 | " return torch.sin(x)**2 + torch.cos(x)**2\n", 183 | "\n", 184 | "def inspect_backend(gm, sample_inputs): \n", 185 | " # Forward compiler capture\n", 186 | " def fw(gm, sample_inputs):\n", 187 | " gm.print_readable()\n", 188 | " g = FxGraphDrawer(gm, 'fn')\n", 189 | " with open(\"forward_aot.svg\", \"wb\") as file:\n", 190 | " file.write(g.get_dot_graph().create_svg())\n", 191 | " return make_boxed_func(gm.forward)\n", 192 | " \n", 193 | " # Backward compiler capture\n", 194 | " def bw(gm, sample_inputs):\n", 195 | " gm.print_readable()\n", 196 | " g = FxGraphDrawer(gm, 'fn')\n", 197 | " with open(\"backward_aot.svg\", \"wb\") as file:\n", 198 | " file.write(g.get_dot_graph().create_svg())\n", 199 | " return make_boxed_func(gm.forward)\n", 200 | " \n", 201 | " # Call AOTAutograd\n", 202 | " gm_forward = aot_module_simplified(gm,sample_inputs,\n", 203 | " fw_compiler=fw,\n", 204 | " bw_compiler=bw)\n", 205 | "\n", 206 | " return gm_forward\n", 207 | "\n", 208 | "torch.manual_seed(0)\n", 209 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 210 | "y = torch.ones_like(x)\n", 211 | "\n", 212 | "torch._dynamo.reset()\n", 213 | "compiled_f = torch.compile(f, backend=inspect_backend)\n", 214 | "out = torch.nn.functional.mse_loss(compiled_f(x), y).backward()" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": null, 220 | "id": "264ad8cc-6fc2-4e58-98c7-3dcff496ab59", 221 | "metadata": { 222 | "tags": [] 223 | }, 224 | "outputs": [], 225 | "source": [ 226 | "md(f'''\n", 227 | "|![]({'forward_aot.svg'}) | < Forward graph


Backward graph >|![]({'backward_aot.svg'})|\n", 228 | "|---|---|---|\n", 229 | "''')" 230 | ] 231 | }, 232 | { 233 | "cell_type": "markdown", 234 | "id": "a5a4f76f-1461-43d8-a8d4-1d24a7f6d1a8", 235 | "metadata": {}, 236 | "source": [ 237 | "# Decomposition to Core Aten IR" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": null, 243 | "id": "bf9f0343-65cc-4d7e-8c53-55292b46c87d", 244 | "metadata": { 245 | "tags": [] 246 | }, 247 | "outputs": [], 248 | "source": [ 249 | "import torch._dynamo\n", 250 | "from torch.fx.passes.graph_drawer import FxGraphDrawer\n", 251 | "from functorch.compile import make_boxed_func\n", 252 | "from torch._functorch.aot_autograd import aot_module_simplified\n", 253 | "from torch._decomp import core_aten_decompositions\n", 254 | "\n", 255 | "def f_loss(x, y):\n", 256 | " f_x = torch.sin(x)**2 + torch.cos(x)**2\n", 257 | " return torch.nn.functional.mse_loss(f_x, y)\n", 258 | "\n", 259 | "# decompositions = core_aten_decompositions() # Use decomposition to Core Aten IR\n", 260 | "decompositions = {} # Don't use decomposition to Core Aten IR\n", 261 | "\n", 262 | "def inspect_backend(gm, sample_inputs): \n", 263 | " def fw(gm, sample_inputs):\n", 264 | " gm.print_readable()\n", 265 | " g = FxGraphDrawer(gm, 'fn')\n", 266 | " with open(\"forward_decomp.svg\", \"wb\") as file:\n", 267 | " file.write(g.get_dot_graph().create_svg())\n", 268 | " return make_boxed_func(gm.forward)\n", 269 | " \n", 270 | " def bw(gm, sample_inputs):\n", 271 | " gm.print_readable()\n", 272 | " g = FxGraphDrawer(gm, 'fn')\n", 273 | " with open(\"backward_decomp.svg\", \"wb\") as file:\n", 274 | " file.write(g.get_dot_graph().create_svg())\n", 275 | " return make_boxed_func(gm.forward)\n", 276 | "\n", 277 | " # Invoke AOTAutograd\n", 278 | " return aot_module_simplified(\n", 279 | " gm,\n", 280 | " sample_inputs,\n", 281 | " fw_compiler=fw,\n", 282 | " bw_compiler=bw,\n", 283 | " decompositions=decompositions\n", 284 | " )\n", 285 | "\n", 286 | "torch.manual_seed(0)\n", 287 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 288 | "y = torch.ones_like(x)\n", 289 | "\n", 290 | "torch._dynamo.reset()\n", 291 | "compiled_f = torch.compile(f_loss, backend=inspect_backend)\n", 292 | "out = compiled_f(x,y).backward()\n", 293 | "\n", 294 | "\n", 295 | "md('''\n", 296 | "# $MSE = (\\\\frac{1}{n})(\\\\vec{y}-\\\\vec{x})^2$\n", 297 | "\n", 298 | "''')" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "id": "2eb86a77-d050-4f80-96b0-c23ac12bd1ee", 305 | "metadata": { 306 | "tags": [] 307 | }, 308 | "outputs": [], 309 | "source": [ 310 | "md(f'''\n", 311 | "|![]({'forward_decomp.svg'}) | < Forward graph


Backward graph >|![]({'backward_decomp.svg'})|\n", 312 | "|---|---|---|\n", 313 | "''')" 314 | ] 315 | }, 316 | { 317 | "cell_type": "markdown", 318 | "id": "20a397ac-69a2-4cbe-af0d-9d3c50e00ece", 319 | "metadata": {}, 320 | "source": [ 321 | "# Decomposition to prim IR" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": null, 327 | "id": "19e71d40-6815-4167-862e-50253913e168", 328 | "metadata": { 329 | "tags": [] 330 | }, 331 | "outputs": [], 332 | "source": [ 333 | "import torch._dynamo\n", 334 | "from torch.fx.passes.graph_drawer import FxGraphDrawer\n", 335 | "from functorch.compile import make_boxed_func\n", 336 | "from torch._functorch.aot_autograd import aot_module_simplified\n", 337 | "from torch._decomp import core_aten_decompositions\n", 338 | "\n", 339 | "def f_loss(x, y):\n", 340 | " f_x = torch.sin(x)**2 + torch.cos(x)**2\n", 341 | " return torch.nn.functional.mse_loss(f_x, y)\n", 342 | "\n", 343 | "decompositions = core_aten_decompositions()\n", 344 | "decompositions.update(\n", 345 | " torch._decomp.get_decompositions([\n", 346 | " torch.ops.aten.sin,\n", 347 | " torch.ops.aten.cos,\n", 348 | " torch.ops.aten.add,\n", 349 | " torch.ops.aten.sub,\n", 350 | " torch.ops.aten.mul,\n", 351 | " torch.ops.aten.sum,\n", 352 | " torch.ops.aten.mean,\n", 353 | " torch.ops.aten.pow.Tensor_Scalar,\n", 354 | " ])\n", 355 | ")\n", 356 | "\n", 357 | "def inspect_backend(gm, sample_inputs): \n", 358 | " def fw(gm, sample_inputs):\n", 359 | " gm.print_readable()\n", 360 | " g = FxGraphDrawer(gm, 'fn')\n", 361 | " with open(\"forward_decomp_prims.svg\", \"wb\") as f:\n", 362 | " f.write(g.get_dot_graph().create_svg())\n", 363 | " return make_boxed_func(gm.forward)\n", 364 | " \n", 365 | " def bw(gm, sample_inputs):\n", 366 | " gm.print_readable()\n", 367 | " g = FxGraphDrawer(gm, 'fn')\n", 368 | " with open(\"backward_decomp_prims.svg\", \"wb\") as f:\n", 369 | " f.write(g.get_dot_graph().create_svg())\n", 370 | " return make_boxed_func(gm.forward)\n", 371 | "\n", 372 | " # Invoke AOTAutograd\n", 373 | " return aot_module_simplified(\n", 374 | " gm,\n", 375 | " sample_inputs,\n", 376 | " fw_compiler=fw,\n", 377 | " bw_compiler=bw,\n", 378 | " decompositions=decompositions\n", 379 | " )\n", 380 | "\n", 381 | "torch.manual_seed(0)\n", 382 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 383 | "y = torch.ones_like(x)\n", 384 | "\n", 385 | "torch._dynamo.reset()\n", 386 | "compiled_f = torch.compile(f_loss, backend=inspect_backend)\n", 387 | "out = compiled_f(x,y).backward()\n" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "id": "b6b79bad-edc7-413d-94be-84ba286841b5", 394 | "metadata": { 395 | "tags": [] 396 | }, 397 | "outputs": [], 398 | "source": [ 399 | "md(f'''\n", 400 | "|![]({'forward_decomp_prims.svg'}) | < Forward graph


Backward graph >|![]({'backward_decomp_prims.svg'})|\n", 401 | "|---|---|---|\n", 402 | "''')" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": null, 408 | "id": "91ff887d-46de-48d0-85a6-f1888c04e136", 409 | "metadata": { 410 | "tags": [] 411 | }, 412 | "outputs": [], 413 | "source": [ 414 | "def f(x):\n", 415 | " return torch.sin(x)**2 + torch.cos(x)**2 \n", 416 | "\n", 417 | "torch._dynamo.reset()\n", 418 | "compiled_f = torch.compile(f, backend='inductor',\n", 419 | " options={'trace.enabled':True,\n", 420 | " 'trace.graph_diagram':True})\n", 421 | "\n", 422 | "\n", 423 | "# device = 'cpu'\n", 424 | "device = 'cuda'\n", 425 | "\n", 426 | "torch.manual_seed(0)\n", 427 | "x = torch.rand(1000, requires_grad=True).to(device)\n", 428 | "y = torch.ones_like(x)\n", 429 | "\n", 430 | "out = torch.nn.functional.mse_loss(compiled_f(x),y).backward()" 431 | ] 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": null, 436 | "id": "a06362e9-7783-46bf-8498-bce83bc749ef", 437 | "metadata": { 438 | "tags": [] 439 | }, 440 | "outputs": [], 441 | "source": [ 442 | "import glob\n", 443 | "fwd = glob.glob('torch_compile_debug/run_*/aot_torchinductor/*forward*/graph_diagram.svg')[-1]\n", 444 | "bwd = glob.glob('torch_compile_debug/run_*/aot_torchinductor/*backward*/graph_diagram.svg')[-1]\n", 445 | "\n", 446 | "md(f'''\n", 447 | "|![]({fwd}) | < Forward graph


Backward graph >|![]({bwd})|\n", 448 | "|---|---|---|\n", 449 | "''')" 450 | ] 451 | }, 452 | { 453 | "cell_type": "code", 454 | "execution_count": null, 455 | "id": "fb68e773-dce1-431e-8070-e2825d255bb1", 456 | "metadata": {}, 457 | "outputs": [], 458 | "source": [] 459 | } 460 | ], 461 | "metadata": { 462 | "kernelspec": { 463 | "display_name": "Python 3 (ipykernel)", 464 | "language": "python", 465 | "name": "python3" 466 | }, 467 | "language_info": { 468 | "codemirror_mode": { 469 | "name": "ipython", 470 | "version": 3 471 | }, 472 | "file_extension": ".py", 473 | "mimetype": "text/x-python", 474 | "name": "python", 475 | "nbconvert_exporter": "python", 476 | "pygments_lexer": "ipython3", 477 | "version": "3.10.9" 478 | } 479 | }, 480 | "nbformat": 4, 481 | "nbformat_minor": 5 482 | } 483 | -------------------------------------------------------------------------------- /pytorch-graph-optimization/benchmark_torch-compile_resnet.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 19, 6 | "id": "b9fe79ed-0970-4b13-a6ac-4fc4f3c2e628", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "import torch.utils.benchmark as benchmark\n", 13 | "import torch\n", 14 | "from torchvision.models import resnet\n", 15 | "import torch._dynamo\n", 16 | "\n", 17 | "device = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\"" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 20, 23 | "id": "3bbdc444-058e-4439-9ee7-121fd962af1a", 24 | "metadata": { 25 | "tags": [] 26 | }, 27 | "outputs": [], 28 | "source": [ 29 | "def run_batch_inference(model, batch=1):\n", 30 | " x = torch.randn(batch, 3, 224, 224).to(device)\n", 31 | " model(x)\n", 32 | "\n", 33 | "def run_batch_train(model, optimizer, batch=16):\n", 34 | " x = torch.randn(batch, 3, 224, 224).to(device)\n", 35 | " optimizer.zero_grad()\n", 36 | " out = model(x)\n", 37 | " out.sum().backward()\n", 38 | " optimizer.step()\n", 39 | " \n", 40 | "model = resnet.resnet18(weights=resnet.ResNet18_Weights.IMAGENET1K_V1).to(device)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 22, 46 | "id": "7dfab6b2-57da-4bad-a8f1-af5df0c9e49f", 47 | "metadata": { 48 | "tags": [] 49 | }, 50 | "outputs": [ 51 | { 52 | "name": "stderr", 53 | "output_type": "stream", 54 | "text": [ 55 | "[2023-03-21 07:56:49,304] torch._inductor.debug: [WARNING] model__15_forward_25 debug trace: /pytorch-examples/pytorch-graph-optim/torch_compile_debug/run_2023_03_21_07_49_09_212386-pid_25659/aot_torchinductor/model__15_forward_25.10\n" 56 | ] 57 | }, 58 | { 59 | "name": "stdout", 60 | "output_type": "stream", 61 | "text": [ 62 | "Inference speedup: 1.96%\n" 63 | ] 64 | } 65 | ], 66 | "source": [ 67 | "batch = 1\n", 68 | "torch._dynamo.reset()\n", 69 | "compiled_model = torch.compile(model, options={'triton.cudagraphs': False,\n", 70 | " 'trace.enabled':True})\n", 71 | "\n", 72 | "t_model = benchmark.Timer(\n", 73 | " stmt='run_batch_inference(model, batch)',\n", 74 | " setup='from __main__ import run_batch_inference',\n", 75 | " globals={'model': model, 'batch':batch})\n", 76 | "\n", 77 | "t_compiled_model = benchmark.Timer(\n", 78 | " stmt='run_batch_inference(model, batch)',\n", 79 | " setup='from __main__ import run_batch_inference',\n", 80 | " globals={'model': compiled_model, 'batch':batch})\n", 81 | "\n", 82 | "t_model_runs = t_model.timeit(100)\n", 83 | "t_compiled_model_runs = t_compiled_model.timeit(100)\n", 84 | "\n", 85 | "print(f\"Inference speedup: {100*(t_model_runs.mean - t_compiled_model_runs.mean) / t_model_runs.mean: .2f}%\")" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": 4, 91 | "id": "bdf99896-0fe4-4b65-a1ed-13296a81c221", 92 | "metadata": { 93 | "tags": [] 94 | }, 95 | "outputs": [ 96 | { 97 | "name": "stdout", 98 | "output_type": "stream", 99 | "text": [ 100 | "\n", 101 | "run_batch_train(model, optimizer, batch)\n", 102 | "setup: from __main__ import run_batch_train\n", 103 | " 37.30 ms\n", 104 | " 1 measurement, 100 runs , 1 thread\n", 105 | "\n", 106 | "run_batch_train(model, optimizer, batch)\n", 107 | "setup: from __main__ import run_batch_train\n", 108 | " 34.57 ms\n", 109 | " 1 measurement, 100 runs , 1 thread\n", 110 | "Training speedup: 7.32%\n" 111 | ] 112 | } 113 | ], 114 | "source": [ 115 | "batch = 32\n", 116 | "torch._dynamo.reset()\n", 117 | "compiled_model = torch.compile(model)\n", 118 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n", 119 | "\n", 120 | "t_model = benchmark.Timer(\n", 121 | " stmt='run_batch_train(model, optimizer, batch)',\n", 122 | " setup='from __main__ import run_batch_train',\n", 123 | " globals={'model': model,'optimizer':optimizer, 'batch':batch})\n", 124 | "\n", 125 | "t_compiled_model = benchmark.Timer(\n", 126 | " stmt='run_batch_train(model, optimizer, batch)',\n", 127 | " setup='from __main__ import run_batch_train',\n", 128 | " globals={'model': compiled_model, 'optimizer':optimizer, 'batch':batch})\n", 129 | "\n", 130 | "t_model_runs = t_model.timeit(100)\n", 131 | "t_compiled_model_runs = t_compiled_model.timeit(100)\n", 132 | "\n", 133 | "print(t_model_runs)\n", 134 | "print(t_compiled_model_runs)\n", 135 | "\n", 136 | "print(f\"Training speedup: {100*(t_model_runs.mean - t_compiled_model_runs.mean) / t_model_runs.mean: .2f}%\")" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "id": "bf10e32d-b29f-4df1-af18-6dadcccc9a98", 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [] 146 | } 147 | ], 148 | "metadata": { 149 | "kernelspec": { 150 | "display_name": "Python 3 (ipykernel)", 151 | "language": "python", 152 | "name": "python3" 153 | }, 154 | "language_info": { 155 | "codemirror_mode": { 156 | "name": "ipython", 157 | "version": 3 158 | }, 159 | "file_extension": ".py", 160 | "mimetype": "text/x-python", 161 | "name": "python", 162 | "nbconvert_exporter": "python", 163 | "pygments_lexer": "ipython3", 164 | "version": "3.10.9" 165 | } 166 | }, 167 | "nbformat": 4, 168 | "nbformat_minor": 5 169 | } 170 | -------------------------------------------------------------------------------- /pytorch-graph-optimization/graph_optimization_torch_compile.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "afb2eda5-99df-42b0-bbd3-f711f4d086b0", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "!rm -rf torch_compile_debug" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 1, 18 | "id": "8f6b7210-882a-4c36-8060-4f324a85cccd", 19 | "metadata": { 20 | "tags": [] 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "import torch\n", 25 | "import matplotlib.pyplot as plt\n", 26 | "from matplotlib import cm\n", 27 | "from matplotlib.ticker import LinearLocator\n", 28 | "import numpy as np\n", 29 | "device = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\"" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 2, 35 | "id": "1cd9c0e4-bdc7-46d6-b0c4-d545ff5d2a29", 36 | "metadata": { 37 | "tags": [] 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "def fn(x,y):\n", 42 | " # x*exp(−(x^2+y^2))+(x^2+y^2)/20\n", 43 | " return x*torch.exp(-x**2-y**2) + (x**2+y**2)/20 " 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 7, 49 | "id": "96b96737-a9e3-4c0a-ac13-502102cad039", 50 | "metadata": { 51 | "tags": [] 52 | }, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAGFCAYAAAA8Zs7aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACRPElEQVR4nO29d3gc1fm//dmqbdKu2qrLktyBYIMBYyDYBINfIIQEAgQIGAjGJg1iQhqE8iUJEJzEBBucEBJDCIlTgOQXCL1jmg3uTbKK1ctKq7515v1jOUczs7O7s6PtPvd17aXV1tk29zznPOd5NDzP82AwGAwGQwXadG8Ag8FgMLIXJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqtGnewMYDMb04Xme/o120mq10Ol00Gg00GpDx5AajSadm87IcphEGIwMQrjDl/4vPfn9fng8HpjN5rD7CP8CU6LQ6XTQ6/X0fyITrVbLxMJQhYYXftMYDMa0iSQA6WUcxwEAOI6j/wt/jnIyEDI+Po5du3bh9NNPByDe8QslIUSn00Gn04VtDwB0dXXBarWisLCQioXIhUmFEQkWiTAYEYgWDQgvJzt/6f/Cx5CeJ0Tb8ZOoQHo7gk6nE/2N9BqCwSD8fj8CgQB4nkdBQQGMRqPocXmeR29vL5xOJxwOBwKBQNi2SKMWJhYGwCTCOApQMjQEgO5kNRqNSAzSxxDuPMnlkWQg3dmq3fFKZeD3+zE6OgqO49Da2gq/3y+6TniebKPBYIBWq4XX64XZbIbdbofdbkdBQQHsdjvdPiIvofSEYmRiYQhhEmFkDUrnCuSig0jzBcKd3v79+2Gz2VBTUxO240+GDKQ7feH/cpcLZWAwGKDRaMDzPCYmJqDX62E2m2EwGKDX6+ltyHkyD6LT6RAMBjEyMoLh4WG43W60tbXB4/FAp9PB7/cjGAzCbrcjPz8fer1e9HqZWBhSmEQYKSfWXEEkGZA5BHJ74V8C2bFKh4ak0YHw9gSy4yM7zlivQSiDSFGA3HmyfcIdPNnpGwwGWRmQ/4WT4qOjo/j0009xzDHHxPX+G41GlJSUoKSkhF7m9Xqxbds2GI1GuFwuNDc3w+fzwWaziSKW/Px8OnzGxMIAmEQYKoln8jjSfIH0caREmy8Q7oims0PieR6BQIDu6MfHx9HT0xM1IogkA+mO32w2o6CgQFYIJM12uiRqZ5yXlwej0YiysjJUV1eD53l4vV4MDw9jeHgYvb29aGxsRCAQQH5+Ph0Cs9vtsNlsYVldws9VTiwEMsTGxJK9MIkc5cSKBMh1SiaPyU51ZGQER44cwbHHHgtgepPH8bwOuR1/rPNCGZDhnsnJSdGOXygDqRASJYNMQ6PRwGQywWQyoaysDEDoPZ6cnKRDYd3d3Th48CA4jkN+fr5ofsVms4WJQfg9aWtrw9DQEBYuXEivZxFLdsIkkiMke75AboiInJf7sZMdsZrXEUsGka4j2xNpXsBiscgOEREZ7Nu3D1arFXV1dXFvdzqQi96S+XgajQYWiwUWiwXl5eX0PhMTEzRi6ezsxP79+wEABQUFoojFYrGIvitEFORxyPdSboiRyIWJJfNgEskwMnW+QM3riEcAwv/Jc8sd+ev1elit1oiiyNXIIFHEem/ife80Gg2sViusVisqKysBhD77sbExDA8PY2RkBG1tbRgdHYVWq6VS8Xq9os86UsTCcRyCwaCsWIRSYZ95+mASSQKJmC8IBoNobW1FXV2d7A8kFfMFZHvjjQh8Ph+CwSDeeeediDIwGAxUBnJCYDLIXjQaDfLz85Gfn08v4ziOimV4eBgulwterxevv/66KFqx2+3Iy8tjYskimESikMj5ArnHBSLLgOd5dHR0oKGhISE7VKkMlA4XyUUG0p2+VAaTk5NobW3FqaeeKhqyYGQ2iR4eE0KikIKCAtTU1KC5uRkjIyOoq6ujcyyNjY0YGxtDXl4eFQv5m5eXByB8iJX8lYqFSISJJfkcFRJROlcgHBaKttgMCM9CUTpfoPRLHGnCmeM4UTaRUiEQGZAUVjkhCGUgFUU8MnC73XSSmpFcEr1TTNVOludDxSAdDgccDge9PBAIYHR0lEYs3d3dGB8fh8lkEklFbtU9eVzyl+d5kVhcLhccDgfy8vKYWBJIzkiE53ls2LAB559/PioqKhRPHgNiISRaBpEgMog2JAQAu3fvFt1OKINI8wImkynqBHIqSOZRbbJgO5P0o9frUVhYiMLCQnqZ3+/HyMgIjVg6OjowOTkpu+peujgSmPou7ty5EyeffLLogEg4Yc/Eoo6ckQgAPPDAA5gzZw4KCwsVTR4n4osSSQaxhoikMpCuLiY/hpKSEuTl5YUJgR3lM7JR1GowGAwoLi5GcXExvczn81GpDA0N0VX3VqtVJBXhqnsAtBS+8GAyGAyKno+JJT5ySiJarRbBYFBVaqlQBvFkFJEvoJwMyHmTyRR1AlmOYDCI9vZ2lJWV0bA9W2A/tuwklVKSpvLGS6RV98KJe+mq+2AwiLGxMVitVtmS90rEIpUK+67nmET0ej08Hg+6urpgt9vjKkdBvjSk34JcNpEaGagl27+cR8tRcq6RyjmRRJOXlwen0wmn00mfQygWnuexd+9e7Nq1S9Gqe+F2ErEI5cLEEiKrJbJr1y6sXbsWg4ODGBwcRGdnJy655BIYDAb861//ijhJbDabI5aqEE5oZwJsZ5xa2PudOpK9s5WuuhdmDJI1LN3d3Thw4AB4nqer7slwGFl1L91WJhYxcUvk7bffxoMPPojt27eju7sbzz77LL785S9Hvc+bb76JtWvXYu/evaipqcEdd9yBa6+9dlqPCQBOpxNXXnkliouLUVRUhG984xu47bbbcNFFF8FsNsf70jKKbP7SZfO2H83kukBJRhhZdV9RUUEvHx8fp3Ms7e3tGB0dBRB51T2gTCwcx2FwcBClpaW0eGauiSVuiYyPj2PBggW4/vrrcfHFF8e8fUtLCy644AKsWbMGf/nLX/Daa6/hhhtuQEVFBVasWKHqMQnl5eW4/vrr6f9GoxEFBQVZLxBAvkJqNpGt251tZHOKb6bsRDUaDWw2G2w2G111z3EcxsfHRavuR0ZGoNPpRFIh+5tIYgkGg/jkk09w9tlnh2WE5krEErdEzjvvPJx33nmKb79p0ybU19fjV7/6FQBg/vz5ePfdd/Gb3/yGSiTex4wE6ZWQK5BFh9lGNv4QcgGy0xImhEhPPB/qbOhwOFBQUJC24dt0fK/jEZdWq4256r65uRmjo6PQ6/WijDDhqnuCTqeDVqvNyaGwpM+JvP/++1i+fLnoshUrVuCWW25J+HPp9fqckgiQvUf02brdmYJchQHpaWJiAj6fDx9++KFIEhpNqDy90WgM+6vT6eByuXD48GEEAgEqFIfDgWAwmNLPLZU7R+miYDVIV90DoSxKsjhyZGQkbNW91WoFANpNUulQGPkcs0EsSZdIT08PLSVNKCsrw8jICF0wlCh0Oh1df5ELZNIXhaGeWELw+XwIBAJhf8mORJggIq09NjY2hpkzZ4YlkUT67uj1enpETKrvut1utLS0YGxsDHv37kV3dzfsdjscDodoAR8jHJ1OF3XV/eDgIADgrbfeirjqPtvFklPfDhaJZAa5LD9hdeJ4TsBUyRmj0Sja6efl5cFms8mKIlrdNLfbjYGBAdFaCaVoNOHVd9977z2Ul5dDq9XC7XbjyJEj8Hq9NGuJ7CytVuu0P+NUz4kkIhJRinDVvdPpRF9fH84++2w6cT8yMkJX3VsslrA6YcKDgGhikVZBTpdYki6R8vJy9Pb2ii7r7e1NygS4Xq9nkUiGkA3yEy4w9fl88Hq98Pv9aG1tjSgDuUoDiRBCJkAmmIUjBx6PB263G263G52dndi3bx+0Wq1IKna7PWsWxKbj/SfRpNyqeyKVaKvuCwoKRN8dtWJJVjHUpEtkyZIleOGFF0SXvfLKK1iyZEnCnysXh7OyYWecCXAcF3d0QL4rOp0OBoMBgUAgakdDqRAygUTuFOSiA5PJhPLyctqEiuM4jI6Owu120wKJExMTsFgsVCoOh0O0eE/pcyWTdP2Oor1Oo9GI0tJSlJaW0ss8Hg+NWCL1uiflXKQ1wITPSf4K94c6nY5WQ04kcUtkbGwMTU1N9P+Wlhbs2LEDRUVFqK2txY9//GN0dnbiySefBACsWbMGGzZswA9+8ANcf/31eP311/H3v/8dzz//vOLHVEquZWcB2XFEL2W6OwdyRCWdL4gmBGHFAbmdPhGCcJJZWm1g//79MJlMqK+vn/Z7kArS8d0gUYjdbqeX+Xw+KpXe3l4cPHgQAEST9na7HSaTKeXbS0jlcJb0eeN5TrI4UrjqXigWaa97aUviSGIRtqdINHFLZNu2bTjrrLPo/2vXrgUArFy5Eps3b0Z3dzeOHDlCr6+vr8fzzz+P733ve3jooYdQXV2NP/zhDzS9V8ljKiXXJJLJQx+xEDbXijdCIJ+htOyMXP8S6SnTKg4cDRiNxrByIyQV1u1204wlk8lEpeLxeNIilXRIZDpoNBqYzWaYzeawXvfCcvlyq+7tdntC5q9iEbdEli1bFvWNkdvpL1u2DJ9++qnqx1QKm1hPHkrWIJCT1+sFx3F48803aX+WWEKQixDSIYRMeb/TQaJeu0Yz1dmwuroaQChjiUjF5XLB5XKB4zi43W4aqTgcDlEabCLJxOEstWg0GkWr7vfu3QuNRkPTkuvq6pIi7pzLzsq1OZFkoFQI0pRTIoRIUQApUMlxHBobG3HyySfT67I5qjqaSNbnpNfrRRPLe/bsAQAUFRXB7XajtbUVo6OjMBgMIqkkKsU4W4az1BJr1f3w8HDyPtukPGqaOBon1pUsSou0cpksSiM7emE0YDabZUURbQ0CYWJiAgDoQisGQwrJWKqsrKQ7vWAwiJGREZoN1t7eDo/Hk9AU43RlZ6UD4ar7ysrKpEX2OSURvV5Pj5azEakQgsEgBgYGMDw8PK1FaRaLRbUQGJlNtn5+ckfoOp0urKtholKMc2k4K9PIKYlkUiQynUVpRAjBYBAul4tKIFvWIGTKduQ6id4xZuJ8UKQUYzK/ojTFONeHs9JJTkkkWRPr0kVpSlJOE7EobevWrZg9e7boyCxbyMQdEiM2mb7DE6YYk/R/smjP7Xajt7cXhw4dAsdxonmVdFX2zqTfAZsTUYCSifXpLkqTK1uRrEVpbLEhI5VkU3tcIdJFeyRTiQyDNTU1YWxsDACwc+dOKheyEjyZsEgkw+F5Ht3d3XC5XBgYGEB7eztGRkbwk5/8BOeffz4KCwtVL0oTXkYmnFO9Sjlbv3xsu7OXVPYTSRbCTCWSYjwyMoKtW7ciPz8fg4ODaGlpgd/vpwv1yDBYolOMmUSygPr6epjNZpSUlGB0dBRlZWWYnJyE1+vNiUVp2RyJHA0/oHSTze9vKredJJHMnDkTwNSCPRKttLW1Yffu3TAYDGGT9tNJMT4afgOq96QbN26ki1cWL16Mjz76KOrt169fj7lz58JsNqOmpgbf+9734PF4ZG97//33Q6PRxOw5otFoaNja1NSESy65BGeeeSa2bNmCs88+G3V1daiqqoLT6URhYSFsNhvy8vKyRiDZ+uXL1u3OdcjaoKMR6c6cLNirrKzEMcccgyVLlmD58uU44YQTUFRUhNHRUezevRuvvvoq3n33XezZswcdHR0YGxuL+8Au138PqhS7ZcsWrF27Fps2bcLixYuxfv16rFixAgcPHqSlD4Q8/fTT+NGPfoQ//vGPOO2003Do0CFce+210Gg0+PWvfy267ccff4zf/e53OP7445W9AMFRgl6vz6kfSbbPiRwNR2HpgmT/kVXfwgWiwr/C8yT9nWQykXRasu4iW+dEEvV80VKMh4eHVaUYHw2/AVUS+fWvf41Vq1bhuuuuAxBqgfv888/jj3/8I370ox+F3X7r1q04/fTTceWVVwIA6urqcMUVV+DDDz8U3W5sbAxXXXUVHnvsMfzsZz+Le7v0ej0mJydVvKLMJZslko2k6/0mGYByO385MZAFowCwd+9e0fwdWRskTAARnielx7u6urB//3668/T7/RgbG0NhYWHSo/Vs+V6rSTEWVtllEpHB5/Nh+/bt+PGPf0wv02q1WL58Od5//33Z+5x22ml46qmn8NFHH+GUU05Bc3MzXnjhBVx99dWi233rW9/CBRdcgOXLl6uSSCatE0kE2frly9btTiTCsjLRxCBMGwemEj6kAiAlZYSXj42N4fDhw3G1VdDr9aJMJo7jqFQGBgbQ2NiIgwcP0iNtcmRuMBgS/h5lWiSihHhTjA0GA/x+f9oKThKSKe24JTIwMIBgMCjb8vbAgQOy97nyyisxMDCAM844g67KXrNmDX7yk5/Q2/ztb3/DJ598go8//jjeTaJk+4p1KbkwnJULkFpjcjKIJAZhJWKy4xcKQJj0Ibw8ngzAycnJae8YtVotPYJua2vD5z73OZhMJgwNDWFoaAgHDhzAxMQEbDYbFUoyCyUmi2RGBNFSjLu6uuD1evHmm2/SKsapTDFOBSnJznrzzTfxi1/8Ao888ggWL16MpqYm3Hzzzbj33nvx05/+FO3t7bj55pvxyiuvTMvWuVaAEcidHXEmIawmINzxj46OAgiNg0vFIC0tIxSDsFeJNC082cNCid4xarVa2jaXpMd6vV643W7afW/37t0wGo1UKIWFhXT4Rim5PMwjTDHWarXgOA4nnXQSjVaGhoZSlmIs3a5kELdESkpKoNPpZFveknFDKT/96U9x9dVX44YbbgAAfO5zn8P4+DhuvPFG3H777di+fTv6+vpw4okn0vsEg0G8/fbb2LBhA7xeryJj51op+Gz9kaV6u8kC0khzB3KXA6EdpnDHHwgEoNfrYbVa6RCO8PqjtdZYXl4eysrK6OhDMBjE8PAwHQJramoCz/Oi4a9EVd9NFOmUFil0KqxiLEwxHh4eTlqKcSqIe+uMRiMWLVqE1157DV/+8pcBhH7Er732Gr797W/L3mdiYiLsKIVIged5nH322di9e7fo+uuuuw7z5s3DD3/4Q8Uhn1arzSmJANkdiajZdmmZeiVDSHJNrKSRgtw8g/R7dfDgQej1etTV1SXi5SeddH03dDodioqKUFRURLdjbGyMDoF1dHTA6/UiPz9fNAQmHGXIxOysVD6vsCeItIoxiVhIFWObzSaSis1my6iDGVWKW7t2LVauXImTTjoJp5xyCtavX4/x8XGarXXNNdegqqoK9913HwDgwgsvxK9//WuccMIJdDjrpz/9KS688ELodDrk5+fjuOOOEz2H1WpFcXFx2OVRX0wORiLZLBFgqjJxLBnIpaLK7fjz8vKQn58vO5+QLet/MhW13zVhEyoy2Tw5OYmhoSG43W4cPnwYo6OjMJvNVCqk+nSuE4+8IqUYE6nIpRiTv7GqGCcTVRK5/PLL0d/fjzvvvBM9PT1YuHAhXnzxRRruHjlyRPSDvuOOO6DRaHDHHXegs7MTpaWluPDCC/Hzn/88Ma/iM3JtTiSTjjYIwtpjkWTg9XoBAO+//z7dWQgLUcpNMMvNJ2Ti6891EvWek5au5Cjb7/fT1eHd3d0YHBzE8PAwXC6XaAgsWQcC6YxEpgPpuU72rRzHYWxsTPReClOMiVTinaOaDqoH27797W9HHL568803xU+i1+Ouu+7CXXfdpfjxpY+hhFzrsQ4kf8iCDB1FW6SmJBWV/CUZKMPDwzj22GPpUFImlapnpB6DwSDKYNq2bRvy8/NhNBppZ8NAIICCggLREFiijrBzpZ+IVqul7W7lUoz7+vrQ2NhIU4yFUklGmjaQA7WzhJAeHLlCvF8+aSqqkiEk8n5JC05Ky9VLZRHtKCcQCODQoUNJ/eIyQmRzsUCz2Ux3hDzPY2Jigg6BHTx4EOPj43Q+gIhlOtlLmTQnkkhiVTE+fPgwvF4vli9fnpTnzymJaLXanBrOAkLDAOPj44rFIExFlQ4Rmc1mugBKen0iv+gs4mAoQVrLSppa7PP56GT9kSNHsGfPHhiNRpFUlA7bZNrEejKRq2IcCAQyJ8U3k8n0iXWO4xStXhYOIQ0ODqK1tVV2x0+O9OWyjjJhR56NE6fZuM3ZiJKdq9FoVJxaTMTicDhkU2LT+blmwm8xmfMjOSeRVEUikVJRo4lBbhVzpHpHRqMRTU1NKC4upiF/tpAJP5qjgaOhPa6QWKnFnZ2d8Hg8dF6FiIWkFh8tkUiqySmJTGdiPVoqaqRIgeM40SpmoRCEza2EYohnFXOmRBSMo4dsakoll1rs8XioVJqbm2lqsdlsRiAQwOjoaErXWWS6mBNBTklEOJwVDAZFfdGVFMATpqJKIwWr1SobPSQzFTXbBXI0/IAY6knG99tkMqGiogIVFRUAQnMBbrcbHR0dGB4exgcffEDrhZFoxW63J62GVaZEIhlVgDHTeOyxx/Dhhx+iv78f+/fvx+DgICoqKnDxxRfjkksukV3FTEoLKFnFnE6ydbFhJvxoGJlNqr7Xer0eJSUl4DgO4+PjWLJkCUZHR2m00traCr/fD7vdnrTU4lz/PaiWyMaNG/Hggw+ip6cHCxYswMMPP4xTTjkl4u3dbjduv/12PPPMMxgcHMSMGTOwfv16nH/++QBCPUba2trC7vfNb34TGzdujPi4PM+jsrISCxYsQH19Pd577z3cf//9qKurQ0lJSdavYs5GiRCybduz8ccut81ya3/ISaPRoLCwEEVFRcjLyxPdL9U7vHSUPRGWcq+rq6OpxaQwIkktJvXTpptazCQSgXg7G/p8PpxzzjlwOp345z//iaqqKrS1tcHhcNDbfPzxx6L5jD179uCcc87BpZdeGnVbbrzxRnr+2WefxWuvvRZVZtlEtkYijMQirTpM5OB2u+HxeLBr1y7RdcK1P8L5OHJ03dzcjJ07d8JqtdKJamGpjVwk0u9ImFpcVVUFILS/IlJpb2/Hnj17YDAYRFKJZ0U4k4gM8XY2/OMf/4jBwUFs3bqVLj6TFrkjC2UI999/P2bOnImlS5cq3q5ca0qVreT6jyYRkEWh0mhBGjkIuxgKqwQYjUYEg0EaWQgvj5bAQS4nazAGBwfR0tKCnTt3AgCamppQVlaGoqKipDZRStf6CSUYjUY4nU56QExSi91uN1wuF00tlg6BRUotzvXfQ0o6G/7nP//BkiVL8K1vfQv//ve/UVpaiiuvvDJihV6fz4ennnoKa9eujesDyPR1IvGS7ZFINm97vAjb28aSA0n3li4KJQIgQynk/0jzdX19fWhra0NNTY3i7SS/J+kaDL/fjzfeeAM6nQ6tra3YtWsXLBYLjVKKiopgNpsT9n6l+rsxnZ15tNRi0nhqcnJSVLWYpBZnikSSuQ0p6WzY3NyM119/HVdddRVeeOEFNDU14Zvf/Cb8fr9sPa3nnnsObrcb1157bVzblosSyVayedsB+RIy0eRAepTodDpRREDKxwgXhiarUsB0INvS0NAAm81Gh8sGBwfpanGTySQa/rJYLNN6zkx57fESK7WYRHYmk4l+H1KdWhxpu5NBSrKzOI6D0+nE73//e+h0OixatAidnZ148MEHZSXy+OOP47zzzqMVQJVCJJIp9k8ER9PRfLIRNq+SE4Lb7QbHcejt7RWtA5JGCkajkRYPlNYUy6TsvukgLZgYCATo8BeRSl5eHpUKiVQy9XeX7H1CpNTixsZGTE5O4oMPPqBDjyS9OJmpxakkJZ0NKyoqwn5g8+fPR09PD3w+nyidrq2tDa+++iqeeeaZeDct5+ZE2HBW7McXLhCNFS2Q7wapGCBtXOXxeKDT6TBjxoyUrANKBIkuwBgJvV4fJhUSqXR0dGDv3r3Iy8ujQ19FRUWwWCwRty/Xm1KR1OK+vj5otVrMmTNHlFp85MgR+Hw+UdViMoSZbaSks+Hpp5+Op59+GhzH0cm+Q4cOoaKiIuxN+9Of/gSn04kLLrgg3k1j/UQyCLXbLkxPVSIHskBUGi2QtUByl0fKqvH7/dBoNKKswaMNpZ8b2UmWlJQACH1uRCpdXV3Yt28fjEajaE7FarVm9XdaLZFSi0njrqGhITQ2NmJsbIzOh5FoJZqIM4WUdDa86aabsGHDBtx88834zne+g8bGRvziF7/Ad7/7XdHjchyHP/3pT1i5cqWqvsJ6vZ52xcsVsj0SIempsSaalaSnChtYSSecM/2Hlgwy6buh0+lEPcSFUunp6cGBAweg1+tplJLqg71Mq+IrbI8rl1pMojuSWkykUlBQEPfaN/I7TBYp6WxYU1ODl156Cd/73vdw/PHHo6qqCjfffDN++MMfih731VdfxZEjR3D99derejG5Vgo+U3eMkdJThf9zHIdPPvlENOEsJwBhORnhdZn62nOZRO5o5KQyPDxMpTIxMYHdu3ejq6uLiiWZE8+ZJhE55FKLR0ZG6FxUc3MzgsGgqGSLw+FIe8+elHQ2BIAlS5bggw8+iPqY55577rS+yLnYlCoVR5vSlrexhpGEhScjpacODw9jzpw5KCgoyLhyMozIJGtHK02Tfeedd1BdXQ2O49DX14dDhw5Bq9WKhr/y8/Oz/mBiOr9fac910myKDIFJU4uJXBKZiq2ErK+dJYS1x526j5LFbNL0VFJnTJiFRNJTpbKINeHc2NgIq9Wa8i/0dMj2HVY2QRonlZaWYubMmeA4TtQrpLGxEVqtVjRRPx2pZEMkEgthsymyNsjj8dAhsNbWVuzevZsmOJCT1WpNyPNHIqckkovrRIhEYqWnSs8L01OlAhCWqBdel+11xo5GslV80oMjIozCwkI0NDSA4ziMjIxgcHAQLpcLjY2NorpfRCpKv7O5IBE5TCYTysvLaWZsIBCgMu7t7cXBgweh0WhQU1ODY489NinbkHMSyZY5EWF6aiQhjIyMIBgMoqenh74uYcVhYe8SuUykdE44Z+vOLZvI9qZU0b4jpFw7yZTjOA6jo6MYHBzE4OAgDh8+DABUKmonnZNNOlKLhXNR5H1L5sF1Tkkk3etE5FY4RzsvTE+VCsBsNtNMs4aGhpjpqZlIJmUPMZSRymZN8TyXMEW2vr4ePM+HTTpzHCeSit1up7+XdC5ATucBlVarRUFBQVJff05JhKQFcxyXkIncSOmpkeQgTU8VyiFSFlK0aIFEKvn5+dN+LQxGLqHRaMLWXZBIhZQe4TgODocDRUVF8Hg8adnOXKqeEYmclEggEJCViLAvutLFbMBUPSSpAIgYpKUvUrWKOJPJ9R9OJtN6xZdE/9f99T9p2pLIJPp7rdFoUFBQgIKCAiqVsbExOvxFav59/PHHNFpxOBxJj+yz9fcbD1kvkUAggP7+fpom6HQ6cd9998HhcGDFihUx01OFApAWyUt3PSRW9iT1ZOM2E6TyEF5e+5fnaLQslzFHXneuNKUSFkmcMWMGDh8+jKGhITidTlr/KxAI0EiFpMgm+rfOIpEsYP369bjttttQWFiI4uJieL1ebNu2DQsXLlSdnspgZAM8z9OJU9eNV0W97ZGrvozmVbcAAC3xTiZg01GvKR2yNhqNqK2tRW1tLV1zQYa/2tvb4fP5qFRIpDJdqTCJRCGe9ribN2+mJVEIeXl5onHKZ555Bps2bcL27dsxODiITz/9FAsXLoy5Hd/61rfw3e9+F0ajEcPDw3A4HHjyySfpoqZsJpsjkVz/4SQLuVTuSCe/34+Gx9bDpfCxGx5bj/q//T/aXIl0OMzPz6e/l1QmpqS7LDpZc0GkMjExQYe/Ojo64PP5YLfbRVKJtxwTk0gE4m2PCwAFBQU4ePAg/V/6xo6Pj+OMM87AZZddhlWrVineFmFoTj7gXFkrku1fvmwVYKIhCRper1eRGABxpWG5ZlXa+76Hka7huLel5WsXYs6/XqTVeL1eL50zAID33nsPdrudRimpmDdIBbF25sI2uTU1NbRAIpHKnj174PV6aTdDMgSmRCrZ/juORUra4wKhNzJSqXgAuPrqqwEAra2tajYJgHhiPRfI5kgk1xFm7smdvF4vFYdSMQhPkXbcg7deTc8XVNrjEklBpR0A0POdy1H+8BYAoRGBiooKlJWVobOzE4sXL8bY2BhcLhfa29sRDAbpUHFxcXHCSpFkeil4YYHE6upqAKCRytDQEPbt2wePx4OCggJRoy6pVFgkIoOa9rgAMDY2hhkzZoDjOJx44on4xS9+kfAVlGT8MlciESB7j+az9YcTDAYxNjamaDgJCO9kSNb42O125OXlRW1vGw9CeQhRIhIiDyFCkQgxm81wOByorq6mGU4ul4v2Fif1rYhUsqFUeaKQSkUYqezfv5/WsRJKhUlEBjXtcefOnYs//vGPOP744zE8PIx169bhtNNOw969e+kHkgi0Wm1OVfLN9i9fJgiQVAZQMpRExNDX1xdRDNLLk5m5F0kcUiKJRE4eQoQikcvOEmY41dXV0fpWpBLv/v37kZeXR4VSVFQEk8mkaJszPRJRgtlsRlVVFS3l7vF4qFQOHDiAiYkJ6HQ6dHV10WKK2dh0KhYpyc5asmQJlixZQv8/7bTTMH/+fPzud7/Dvffem9DnyrUijJmwI840pCVjYp14no8aMZD/u7u7AQDz5s1L6+tTKg8hQpHEkoeQSBGJHML6VjNnzkQwGMTQ0BBcLhdaW1uxa9cu2Gw2GqkUFRWlvUw5IRXSMplMqKyspG29PR4PPvzwQ/A8j0OHDmF8fDwsUkmlVDJmxbqa9rhSDAYDTjjhBDQ1NcX79DHJpSKM2RyJxDufo0YMwpIx0qrD0qEkJROg/f39aW1qNvmLbwEAzMUFof9dI3HdPx55COn5zuUoXf903PfT6XSi7oZ+v58WTDx06BAmJiZQUFAgmqQnkdvRMMxjMplgMBhQU1OD8vJymsQg7GRIpEtO2RippKQ9rpRgMIjdu3fj/PPPj/fpY5Lu+lmJJNsn1oViEE40qxWD3FBSLuyIiDykmIsLFIvEXOqYerx+t+LnJsIau3cNcOoliu8nh8FgQFlZGR3q9ng8dD5l165d8Pv9cDgcKC4uBsdxKf1uZ0LtLJLEUFFRASA0v0yGv5qamkRSIRlgeXl5adnmeEhJe9z/+7//w6mnnopZs2bB7XbjwQcfRFtbG2644Qb6mGQVaVdXFwDQdGBhmeNYaDQaNpyVZKS9SiKdPB4Pdu7cedSKQQmR5CEklkiE8hBeFkskRB5ClnzwL0x+8C8Y734s5nYpwWQy0TkD4eI+l8uFYDCIjz76iEYpxcXFSe3Bnq7fUTR5GY1G0f7N5/OJCkru3LkTVqtVNPyldM4plaSkPe7Q0BBWrVqFnp4eFBYWYtGiRdi6dSuOOeYYepv//Oc/ogWJX/va1wAAd911F+6++27F25ZrkUiqiGcoSdqrRJqympeXh0OHDqGhoQFOpzOrxJDsnU3wkR/D545vmEpOJHLykF4fSSRyAhEyfPcq2BMkEoJ0cd8rr7yC4447DpOTk+jr68PBgwdhMBhEmV+JbmiW6f1EjEajKJLz+/10zqmlpQU7d+6k1QbIKROkouEz7VB3GvA8j/Lycvz1r3/FySefnO7NmTZdXV3o7e3FCSecoOr+0oiBDCXJDStFE4PcKVbpmPfffx9z587NqsoBTU1NCAaDmDt3bsIfO/jIj0X/xysSIDRHEkseYfcRiCSWPKQkWiRCXnnlFSxZsgQ2mw1AaEicrKR3uVwYHh6G2WwWZX5NZ75g37590Ol0Sflso/HOO+9g3rx5dHHndCBSIUNgIyMjsFgsokZdkeqi8TwPk8mUFJFmfe0sKbk2sS51vJKhJHIKBoMRxWCxWOIWQ7zbzgiXB8HoKIhLJKbSIphKi+BxueN6fnOpA1CZLOD57fdh+u46VfeNF7L+pKioCLNnz0YgEKBDX2S+ID8/n0pF6WrxdJPIuRiDwQCn00mrggQCASqVI0eOYM+ePTCZTKI5FYvFkpDnjkbmfwpxko3rRCKJwe12Y3x8HNu3bw8Tg7TasNFoDFvHkJeXl9ZikzkU5MZFJHFIiSUSU2l4FGcqdigWifT+nt4BRffLK5rK8kqWSGJ9N/R6vWiHSTKbXC4XXS1OJumLi4tFDagiPV+mD2fFi16vR2lpKY1yiFSGhobQ0dGBvXv30n7rM2bMSNrQV85JRK/XpzVNk8BxXNRSGMJhJRI5yZW+0Ol0qKqqCitPz470Mw/Nn38e933kRCInD9H1MUQS6f6mspKYIhEKhJAskcTzHZZmNk1MTNChryNHjsQsz5KLEpEiJxUyRJjU503qo6cYkp2VrEhETgyRTsKe6Hl5efSvXMQQSQy9vb1ob29XnJ2WSRwtkpMThz7fhsDomOLHICKJJQ8hciJRcv9IIpGThxDPb78fun+CZDLdnSspQUKKJcqVZyFzKcXFxWmNitP1W9Dr9SgpKUn6vGROSQSIf06ElN5WUhZDKAapBKTpqkQc0/kCZfuOONuGs5S+30oijnhEYiwrhbGsFIHBIUW3JwhFEpeABCKJJQ8pqZwnUUqk8iwulwvd3d3Yv38/tFotLBYLrFYriouLU7b+IlMWVSZzG3JOIjqdDl6vF0eOHEFhYWHYUJJ0WCmWGKy//gkAIO9nm2jEkMrS2Nm2IyZkwg8nkeie+Q0AgBufSNhjGsvEGTv6osK4RZI/bxb8rsG4nzteeRD0DjsCT94L/TU/VXV/QjK/18LyLLNmzUIwGMT27dvB87yoPItwkj5Z5VkyRSJABpU9yTTa2trwk5/8BL29vejp6UFTUxOuuuoqaDQaPPvss7BYLKKhJLJ4RzqUFEkMJPD33rEGXsHlJQ/9JemvLVO+fEcrRBxCtFaLYpHIRSNScYTdR6FIDKUlU+eLi+ISib7QQc/7+/qV3cchlk4iRJKq77dOp4PJZILVasXMmTNFK8UPHjwYtTzLdMnWg8B4mJZE4ulu+Mwzz+AXv/gFmpqa4Pf7MXv2bNx66620j4iUNWvW4He/+x1+85vf4JZbbom4DRaLBccddxzOPvtslJeX47bbbsPq1atxxRVXJCQboeShv2Dg5vDWo9LLkiWVbP4Sqtn296rkvz9CgpPixIkzB7fF/TyROGbvC8DeFxL2eEQkseQhuk8UkQjlIbpcgUiE8qD3c5ZGFYlUHkICT4aKp8Yrk3R8p4URgXSluFx5FpIiSzK/1AovkyKRZKFaIvF2NywqKsLtt9+OefPmwWg04r///S+uu+46OJ1OrFixQnTbZ599Fh988AGthhmN0tJSUW+Tu+66CwUFBQlNZ4skEiEDN18FU6GN/m+7+3fTft5srp0V7w9HiTwIOrNWJJK3i06SvZ0SuRj+N/U5zZlUFmHEFY2UlUFfVgZuNL7FhVKRRJKHkEgikZOH6H4RRBJNIELURiWZUgperjyLy+XC4OAgWlpaAEC0kj6e8ixMIlGIt7vhsmXLRP/ffPPNeOKJJ/Duu++KJNLZ2YnvfOc7eOmll3DBBRfEvV3JWmyoRCSeoTEqkrG7V4ddH69Ycv3LB8QnDyE6c2j4URqVCHm76CRZkQjFIURrtoBLgEj0kl47AKDNL4hbJIAyeYhuLxFJLIHQ+wlEolQeBJ0tH/wz66G5+Ja47peJCMuzzJgxAzzPY2RkBC6XS1SeRZj5Fas8S67/jlVJRG13QwLP83j99ddx8OBBPPDAA/RyjuNw9dVX47bbblPd9VCv1yctxTdekUghYjHap643fu9XUR8vWyMRIPq2q5WHlPy5oRW5Qe+UTCZaPfQ8iVLO/suqhDyfEKlI5OQhun0cItEWlcBYVAJ+OL6JdiAkEl7FWimDsxT8Z425lKKz5dPz/DPrASCmTOQaYCUbtRGBRqOB3W6H3W5HQ0MDOI6jq8Tb29uxd+/eqOVZWCQSATXdDQFgeHgYVVVV8Hq90Ol0eOSRR3DOOefQ6x944AHo9Xp897vfVbNZAJJfgLHkob9g/P9uCrtcWCAvmkgAwDc8RkXi+82tsrcxfu9XWf3lk9v2T5ecCUtxaJix5AQHBj51q3psS234UKUuT0tFYqkLv/792/8MAFjy8+gNn+KJRoDY4gh7/Bgi0RaJIw+NvTAukWg+u78GADfQp3y7HFMpwsG+HkX3EQpESCZGJYk6GCPrT4qLiyOWZyF910nJ+0z4HedMim9+fj527NiBsbExvPbaa1i7di0aGhqwbNkybN++HQ899BA++eSTab3gVJSCt975aJhIpMXtJl0jikUie/1vbkWhzYLFAPjdz4ddr1mV2I6QyYD8cFuv+BK8ox4qEELJCY6w+0QTi5w8hAhFEonGJ5/H7GuiD5PGEom2VLL4c3I86uMpQSoPIUpEopG5v7bEGVMkQnkQdM7Q64skk0jyEMI/sx6DS69GYWFhWOZjOiKRZD2fXHkWMp+yb98+AMDOnTtRWlqqqDxLNqJKImq7G2q1WsyaNQsAsHDhQuzfvx/33Xcfli1bhnfeeQd9fX2ora2ltw8Gg7j11luxfv16tLa2Ktq2VBVglBOJECKVPEfoB+cZDO+BHUsk/rEJGGzyBdT4x0ITmdoYCQT81bdHvT5ZlPzq/zAJoBWAd9QT49aC+8lEKLHkEQ8DBwcBFSIJE4cQszUukQijkWjyEBJNJHICoc8VRSRyAhGic5aHiUSJQAhF254FAGwvOYF2QExFQUA5UjWslJeXR1vk8jyPl156CWVlZRgZGUFbWxs4jqOFJuXKs2QjqiSSiO6G5D5eb2j1xdVXX43ly5eLrl+xYgWuvvpqUZ+RWKSyKVUskQCA1z2KPEc+THI1iQaHpyUSAOA8nqgiIaurtabYK3Q1xui3CVz0nZiPAYQiD0I8AiGQCGXCFf99lUQjikUSTRxS4hAJX1wOTXE5NCPxLRKUiiSaPIRIRRJLHkKISOKRBwDAOvWdPnHgU2zjFuDAgQO0yiyAlFbbTldaMQBUVlaioaEBPM9jdHSURipNTU3Q6XSizK90SXY6qB7Oire74X333YeTTjoJM2fOhNfrxQsvvIA///nPePTRRwGAvolCDAYDysvL4+oBkMyJdTmISIx2K3zD8jsRIhIpQrHoBaLwDY+KbjddkYRu440pEt7njSoS/b8fBgBoIvR1aHxyathNjTwIefkm+neo1R33/dWKhC8Jn9/QeCaVP3EMkfDFYinxBUWqRAIVC+GISOIRCL3vrPlAT4eyG1vlD4hOGtwJABg45iL09YWE9vbbb8Nut9MoZTrrMZSQ7va4Go0GBQUFKCgoQH19vag8S1dXF/bt2weTySTK/MrZ9rhA/N0Nx8fH8c1vfhMdHR0wm82YN28ennrqKVx++eXTfxUC0tHZ0Hrno/A/9H0Y7VbR5UKpRBIJITA2QUVitMvfTmsxIxihHlOqRBK6jS9MJIkWCKGwzhGXSIpnFtLzQb+8SFyNoaN5IhIAmLX2etnb8ibztEUilYfoujhEwgkeR+tWttKc3tdRCjhKgYFuxffhC6beS768GppYIokgECEln/wbji+sRHt7O04//XQMDQ1hYGAAra2t0Gg0KC4uplJJ5FqvdGRJxZr7kZZnEfYHkSvPUlRUpLqHSlLlnEudDQHg0ksvxXHHHYe1a9em/Ln9D30/5m00muiTavooEQcQEkk0eH9sgSZiaCt0G2PC5AGEC0RILJEI5SEkkkiEaLShH1gkkQBxRiSfSSSaPESPHUMinMzjxCMRziFeLa9VIBKhQIREFIkCgQAISfYzgqddSneKPM9jeHgYAwMDGBgYwPDwMC2WSCrRTqcUyccff4yKigpUV1erfox4CQQCePXVV3H22Werqs1FyrOQ1fSTk5OqyrNwHAedTjetzpDRyPraWVJSOScixXDzupgi4XkOhnzxDy4wNiE6H00k3MRkVJFoDKGPNFZUIowkuPHwIZhYEUnT314R/V9QaUf/QXUSiSYPQmGdA4BYJpHEIURn0MYUCc/xcNQWoenXf5x2ROIvqQIA6CeULyyMFo3ICQQIiSGWSKTyoJeXVEQVSSSBAKGIBBDIRKk8AJFAAEC39R/gAWjOvAIajQYOhwMOhwOzZs2C3++Hy+XCwMAA9u7dC5/Ph8LCQhql2Gy2uI6uMzESkdLzndCoTPnDWwCEl2eZnJyk8ynC8ixEKgUFBWkZsss5iaS7Pa4Skfg/G5IiMpGThtZiBjchv9Mil0eTSazhLbKoTGM0Qmu1Rrwd8sSP0fPf16A356GwfmoHFfSGHqt0rvyaif6DvbKXA8oEIqSwzgGtLvE/FPeRwZgiiQQRh5CApWBaIokkDyHRRBJJIPR6GZFEk4cUvrwamlG34ttLBSJ6rLf/Cs2ZV4guI/Oh5eXltBTJwMAAXY9BemWUlJSguLg45lF2OiUSDSIOJZjNZlRXV6O6ulpUnsXlcqG5uRkARPMp8ZRnmQ45J5F0zIlIISLR28Q/nMCY+IjfPzoWFpUQlIgiFkrmSeTmN0R4Q9HFyK598I1MQG+eik6IPGJB5CKUSbzyAID88lDa9Hj/aIxbilESjRDs1YXo//uzKL3sK2HXCaMROXFIUSMS3hDfkINUJLHkEfm5lQsEAAJWB2B1wNDTGvvGUQRC4N5/BgCgXXJx2HXCUiSkXwiZS2lubsbOnTtht9vp0JfD4ciItRjRIhE5eZAIRAnxlGcpLCxEVVXs76tack4iqYpEgsGgqJEVOU8vW3IZvF4vzjj06tS22cJ/TLGOkGINXwGxh66A8IwqYXmLaCIZ2RVaMOUbES++UyoQIUQmI13ha2aiQeRBsJbmJ0UkJBoBEFEknopZ0PnjmB9RiKdwqtho3piyXugEIpJ4BUKiEVUC+Qx/eV1kkSiQBwDwhqkDk2gyIQhXjc+dOxder5fOpezYsYOuxRCuTUn3cNbw3aGyO8LKFoR45BEJaXmWYDBIW+N2dnYqKmar+rlzbWJ99erVMJvNuPfe+Fd08zwvalwlFYTwfDAYhEajEXUylJ6nf/9yn6Ln18kUcuM8U/MMyZCJHN4j7fC6QztpqTwAdQIh6PKmnn+oJfbksFQgQuIVidJohIgEAIY7hlBzb3iF2nhEEi0aEcqDEK9EJuyhx7AMd8V1P09+aJW12XVE0e2F8pASJhIVApEjmkxkH09wRD4wMIChoSGYzWYEAgFUVVVh5syZqjOc4oWIA0iePJSQ7In1nJPIN7/5Teh0OvziF7+glwl7o4dFDJLzPM9Dp9PJSkF6Pp72t/zjdwEAtEYjuChF7uREIkSTF/uLoFGQsSEnE6E8gMREHwRdhO2OJJJo8hCSDJEIJUIoWC1eRBtvNCIViZw8hCgVCREIQalIiEAIsUQSTSBCDD2tCRMIrxPv7HWnfCnCLSNDalvt2bMHGo0GPp8PDoeDRimJnIwmfei9guoUQnmYSx2Y7HenTB6EYDAIvV7PJBKNd955B52dneju7sZTTz1Fh7RWrVqFmpoa+P1+ABB1OJSLGMj5RHU1kyNWf24imETIBIgsFN7nA+f1IfhZZlY0eQDJEQhBKhKlAiFEE4m9Nr5S6gROJlU6ESKJJQ8h0UQilQdBiUSkAiFEEolSgQCA31QAS+/hmLeLVyBS4hXK1q1bMXPmTOTn59MJepfLRYfGyAS90rUppCFXwC0empUKxFzqEF1vv3P6fYbiJaMlEk9nw7179+LOO+/E9u3b0dbWJtuxsK6uDm1tbWH3/eY3v4mNGzdG3I5TTjkFer0e5eXlOHz4MMrKynDmmWfi3HPPRVVVFfLy8lLeGz0asUQiRKtEFgqOpDjPVHPf4Lh4ESRBTh5AcgVCGGrpj1seBKlE1IpDiqWsEGMdYsGpFcmovQZmT/xl3eVEEkkghEgiiSQPIVKRxCsQug0RRBJLHkBsgcgRSyrvvfceZs+eLWqYR1aMC9em5Ofn0yjF4XBA+1mVBgAIjom/Z0KBCOUBmf1MOuRBCAaDMBgMSesjn7LOhhMTE2hoaMCll16K733ve7KP+fHHH4smxffs2YNzzjkHl156adRt+eijj+j52267DYODg7j55ptVvrLkQ4oiav78c+CzhW7g5F3OfbYDjyoTwXEA6SMhXXQYHB+H7rM5FWF0Yip2UElIS6vIHY0TxjqiV4dVKg9C2YIZmOiNfycLhCbahVljiWKidwi26tBktVQmShi114j+nzQVqhIJ3Z4Y8hDeTioSJQIBgMniWioStQIBgImymQDEMkmWQAAg+NF/oAlGy8qUaRSm1cLx0b/gADALCNXPHwPQK64KEU0eQEggprISePrD1/p8eOolOPfcc5W8hKxFdSSyePFinHzyydiwYQOAkNVramrwne98R7azoZC6ujrccsstUXunA8Att9yC//73v2hsbFQ8bvmjH/0IPT09ePjhh2PfOAPQ/OUXsW8kvU+M4Ta5xkKcTDQRKcKIJg8pcjKJRyDGArG44hWJpWwqsyhSJDUdhI9PRBIrGpHKQ0q8IskbG1AsECFEJEoFQjB44uvAKBVI2Hb0Hk6qQABEFIjG75W9XLbG2Xh4SaFoAtF8dmQvJw/7nb/D6OgoPvjgA1HPpHSQkZHIdDsbKn2Op556CmvXro1r4isT1onEA3/VT+h5pULhhSnMUdKZ5cQBJEYeBFv11A5qrKNPsUCk8iBYygoViUS4cxc+ZqJFMtE7RJ+LRCUjv9sgEknQYMaEJTFDaFIG82uA/BrYJyIv2IxEvPIAgHFzMWAOFUJ1DLXEvH0sgQDASNVxAID8vsaIt8k0gUSSh8FZisCQm17u6R+EqdgBjyt0mXTYKtvLvCtB1SRBtM6GPT3KuqLF4rnnnoPb7ca1114b1/0MBgM4Fa1BMwH+qp+IpKIInU504rw+epIS9PpkBcL5A6oEIkRr0KOgvhLWytg700gCIcgJQnhdtOtjPbYapFKzVZdi5Hcb4M6vpqd4mDTFXpcxmF8TEohK3OZyDFrj265xs7iKtruwPuJt/aYCRQIJ6KcSREadszHqnB12m0wXiMZohMFZKhKIp38Q4DiYih0AgI9PuxTNF65Be3s7JidDkenR0BoXyODFho8//jjOO++8uBfJ6HQ6mo2VrRCRRIxMIsyfADJrRfgpoWrzjPCPiH8wiZCHFGtlCca7wieE49nBSyOSaOJIB7bqUuBvv8LI10LtjX06E4zB6RWgJMjJY9hSpjgacZunSqYMWqtRNB67jLtUIPSxCuvDIhIl8gDEAhFCRJLf15hegUQZvtI5y8G5p4apaPTx2QEqkUfetx4Az/M4eWQEAwMDtKS7xWJBfn4+eJ5HIBBI2dqUdJDSzoZKaWtrw6uvvopnnnkm7vvqdLqsjUSk+C67De+88w6WLl0K/dP3K7sTH/21Gwps4CUS8g3Ft4IckJeHEBKREJmoiRDUiiPZw1pCClSKRG6SPVbkoUQkQoHQx40hkkgCoY8pEMl0BSJkqOJYAICj76CixxQSl0CUzH9YQmtbdJ/9JQKRyoOQ960Hpp5TsFp85syZ8Pv9GBwcRGdnJwKBAF577TVaz6qkpCTl3Qw1Gk3m9VhPVGfDSPzpT3+C0+nEBRdE7zwnR6qbUqUCnudFbW5FKcIxpDF1s8jRi7FwqjmWEqHEEoiQgvpKOl4cD8LeK8I0ZKVEE4lwHoegZIEmAAQ94ZIQiiQeiEimM2xFkJOHkEgiiSUQ+viF9bBOuhTdVolAArqpuTO3M9R0TqlMEiEQUo0YADQjUzIXRh8arTaqPCJhMBhQVlYGo9EIt9uNxYsX0zTiw4cPQ6/Xi9amZEPjqWikrLOhz+ejjet9Ph86OzuxY8cO2Gw22ncdCMnoT3/6E1auXKkqBExnKfhEE+noQSSUJ+XLu0STRjSiCSUeeQgn2Enor0Qmco278hz5qkViLFBWqpwPBhWJRGcyQW/Ph7dXPuU33mGteAUiF43EEkgklAoEAEY1DoxaHCifiL6QMF6BCFEiE7UC8ZfXAQD0424g3xG6z4g4EiQC0Wi18PcPQF80FXkqkYcUnueh1WphtVphtVoxY8YMWjzS5XLRxlMFBQVUKIWFhRmznk0pKets2NXVhRNOOIH+v27dOqxbtw5Lly7Fm2++SS9/9dVXceTIEVx/fXzluAkGgyFnJEKIloXNXzNV14kIRa1ApBChSKsPRyNadpYwi0VKtK6P5HqlIjGVisuWRMpSk6JUJACQVxbK1CIyiSca6dUK5vmCQIEuvpRaoUjiEYgwGolXIIQeS2j9h5xMpiMQIUQmUgq798peTgRC1qYAn6Up26cKUurH3eL7SKIPbYlTJA8iEDXyIMhNrAuLR86ZMwder5fW+dq1axcCgUBY8chMn5zPibInQjZs2IB///vfquZTMo1gMIi33noLn//85+PP8d78f9N6btLtT4p0Yp4Qz9oQoUhiyUNKJJFIxSFEqUQISkSil7QwJjIhIpGLRkTykBCvSOwTvaojkDxOebkWoUCkSEUSSyJKBBIJfVD+M9QHwl+LdJ1LJIEQefCDA2GRh+6K6GvdlOByubB3716ceeaZim7P8zxGR0dpWZbBwUGYTCYqlKKiIlVrPTiOg8FgSNrkfs6lDKS7KVUiIUcgUs+TasNer5eeSCFJen7WF+D7bNHhF468G/u5IkhDiuGz4SEik3hXpgOhiIRX+RkJI5Jo4hCizTPGJRIlEUlgeFQkEhKZkIhEOKwVTR5q6PWWoldXCifiq/YLAAP+0HtWpeuc9nb0WGZSkWSDQKg8SiqgdfdDW+IE1x+K6PRFhQgMDk0r8pASb4qvRqNBQUEBCgoK0NDQQHuuDwwM4NChQ5iYmFBVPDLZcUJOSiRbJ9ZJOiCRgeezSdympiYEAgFRxWGe56HX65GXlycqJGmz2eh5cjm0X5h6EkGEolQcYWi0MNhDWTqczERzJORKtwQjdG+MhM5qhcVqjVoJOdJzxxuRxEIqEiAkk9I3nkT/WdfEJY+RYIGiaKTXOzVE0+crgdOoXCREIADQGayKKZJoUQihxzITJb74StDHQyIFwpVUAPisP71nEtxo6PbaohJoVtyARJddne46Eb1ej9LSUpSWhj7zyclJOkHf0tICjUZDJ+hLSkoUF49MNDknkUycWCdyCIsWZM5LS9EDodeUn58fVnVYVbXha++cOh9hUl6EJvokH1mXEkkmsQpI6izmmCLRybTvjVVSP9K2JGN+RI7SN55E9xk/gEmfOHEJBRIvQoEoQYlAAGAiaMYR3UzUBiNPuKuNQhIlEN5oAv9Z9AHP1H2JPJJFoiMAs9mMmpoa1NTUiIpHtre3Y+/evbBarVQohYWFSa1GLiTnJJLq4axochD+z3EctFptWORgt9vDIgcydsnzPN544w3MmDEjKUcZcpPysaQRCaFMFFUeFkAKQwplIieOsOf8TLLxyEROJMaSKDtYBePIcrXKAOD4d3+JQ8tuUbxtkaKRaPJQEo1EEkikaCQegRCO6EKT2lKZpFogRB6TxbUwjYbquml7xJWJNV/8pqptipdkrljXarUoLCxEYWEhZs+eDb/fTyfo9+zZA5/Ph8LCQjqXUlQU30FEPOSkRBIxnBUMBhXJIRgMQqvVhomAlJQWXq7T6eIeIwWSP6YJTAklnjL1AKaqEJN/SfdFFSJXIg7ZTVAYlehJP/t45vIDgZgi0RiN0H62SC3oFqeNznlzfVwikaIk+ogmklgRiFQkagQiRBiVpEMgk8W1AADTaF+YPLjyWuhO+qKqbVJDKsueGAwGlJeXo7y8HDzPY3x8nA599ff3Y8mSJUl77pyUSLRIhOO4qMNJ5HwgEKDtb4VysFqtKCoqEl2u1+uT9mVJeV/oSIsa450/IaF0hM8iVpteaSn7sZZQaqqtXr4elFQkVBiJQIFIuIlxaC1W6BxTGT5EKHPeXA9dVS32z47d6pVEI9MZugLiG74iIpmuQAhHdDNRiXbFzy9EjUDIQki/qQCWjv2hK4WLDA1GvMRXQ+PSwPHhh3SewWazJf33lY70XI1GA5vNBpvNhrq6uqSPzOSMRPx+P3p7e3H48GEYDAasW7cODQ0NmD9/vkgOpK6WUAJ5eXkwm81wOByioaZ42t8mk3RlYYuEoqJkPQBAp1M1t6Ax6DF6qDXsciITQsH8qbUBWlOeqLdKQolDJASRUDqPYD6eUSQSNQIRRiPxzn8A049AhPg4PVoxVbyxThu7GnA05AQirDZc2ijJPvxMIHxxOe3Rrn35ZSxatAjj4+Po7++n+wkilKKiooSnwGZKAcaMLHuSaVx//fXYvHkzANA5hpdffhlf+tKXsHDhQhQUFIRNSmfCh6sEjUaTNokIUVOynt5XciQkJxXhCnDfsPLFjSP7xWPwVCrJeM9UiISgcxQC46OY3xhZJO3DU9IpscZf+6vPVwKtJv66cQMTVgwgtM0NBdGbjcXCx4W/P62cMqHIRSFCgZCFjvm8O1wchMlxKg/hL5zneZjNZhQXF6O2thbBYBCDg4MYGBjAgQMH4PF46CI/p9MJi2X61aAzRSLJJmXtcQHgH//4B37605+itbUVs2fPxgMPPIDzzz+fXs/zPO666y489thjcLvdOP300/Hoo49i9uzw8tFCyOr4srIyvPzyy1i7di22bdum9mVlFG+99RZOOukkWFXOF6QKkVhkVszLzVnEWgkfj0ykCBcx5pVPb2gIADSCnvcagxHcRPRtkxMJpagU+2vPF10kFAghXpGoFYiQaBJRGoXEA5GKnEAGjOEp0g2H/iv/QJ/JY7ykDvmzTwy7+qWXXsLnP/95WTnwPI+JiQn09/ejv78fg4ODMJvNoihFTSmSzs5OdHR0YPHixXHfN5FwHKc+m1MBKWuPu3XrVlxxxRW477778MUvfhFPP/00vvzlL+OTTz7BcceFmtb88pe/xG9/+1s88cQTqK+vx09/+lOsWLEC+/bti5qdVFtbS89nYorvdMmESCQW0j4oBw4cgNFoRENDA4BQ51H+8btEt9HbpnZgckIx2sN3xNHEEmn1u7cnvM6VqbZK/kEU/NB4vw8agwEag4KJY7MFmJTIYLAf8/EC9teeLysPNXS4p96r2kJl5WGkAgGA5hGnrEiSIRAgFKUYtQFIF2lYdKEIpP7Ia1Hvz1s++8wt+dAuuThizkS0qECj0dD6VnV1dQgEAnC5XOjv78fu3bsRCARQXFxMpaI0U5JFIjGItz3u5ZdfjvHxcfz3v1NHEqeeeioWLlyITZs2ged5VFZW4tZbb8X3v/99AMDw8DDKysqwefNmfO1rX1O0Xa+99hpuuOEG7Ny5Myc+wLfffhsnnngibLYEThSngAMHDsBgMGDmzJmxb4xwwQDRIxXDZxPngbHplXzXS/rK09IXCo/aFItEiFAqRaV42X5VxLsqiUaEAgGUSUROIEKEIkmWQACEBCLg2M4IkQYAjWCNh7+kCvqJ0MR6pOiDwPM8XnrpJSxduhRmc+zXIr3v6OgojVKGh4dhs9moUBwOR8T9THt7O3p6enDyySfH9ZyJhOd5cBxHs0OTQcra477//vtYu3at6LIVK1bgueeeAwC0tLSgp6cHy5cvp9fb7XYsXrwY77//vmKJZFt7XCVkQyQiJV6Ba75xT9hlUasE/fFuAOESIESTS6T7AEBgUL41r9ZihlZmSJH3h4ZhosqESIPIRCiVyXGci78AQFSZREIqEAA4MpSvOBqJBIlIki2QY3peov9rgvLN5Ig8/CWh6FE/MQL9xAg8hZWwzDtVcca2moNKYSmSmTNnwufz0dTZTz75BECov1JpaSlN6SccLZGIqk8/WnvcAwcOyN6np6cnajtd8ne6LXdzqXYWkDkT66lEWv5F9jT7bPj9/rA07Egnwz/WiZ6Di6PcCln7wo1HmwcJXac1maCxRtit+XxAMCATmYTue472b3glX3ywNDBuiRiNyAmEEE0ksaIQQvOIE+XW6clIyMK+5yNeF00gRB4AqDyCOmPU6EMI+f0kYoduNBpRWVmJyspK8DyP4eFh9Pf3o7W1Fbt370ZBQQGcTidKSkqYRLKVXJRItiInP47jwhZuyp04jqPlX4Qnm80m+l9pph33tR9GvX5/Z2hSeiIwdSQ56dfjzG3hEVIkyKp9fjzGjtczKbtORqPR4Nzx34cew1oQJhQh0QQSDaUCAYChMT2Gxgoxv0w+OgPCo5BoooiEnEACts/SlD8bxSXyCOTZYJl3atzPkQw0Gg0cDgccDgdmz54Nj8dDF/c1NzdDo9FAr9ejp6cHJSUlOdsiN2XtccvLy6Penvzt7e1FRUWF6DYLFy5UvG1arZYNZ6UJYfQwMTEBjuNw8ODBsJX+QPg6HbKIU3hZKn9086u0VCRC3j4pNFezBO8AADQKOknyktIxppbdQKS+30LGplZfa8ZHcW7fr4A8+UncsGNwueG0wfCLYIw9KczrJQOJEeor8trpjbEL5eEtnIo2dH5xlBjIsyGQZ8Ngfg2qqyIkREQhkZFINEwmE6qrq1FdXQ2O47B//364XC40NjZi586dKCwspHMpVqs1qw8QhaSsPe6SJUvw2muv4ZZbbqGXvfLKK3Q5fn19PcrLy/Haa69RaYyMjODDDz/ETTfdpPwFsUgk4ciVnidVhqWr/El9sGAwCL1eD51Oh8LCwrDoIRO7t1kC7YC+hkYjZkMAk/7QT+R9fB4AcBreivtxPfWfC6tJxsf4XDVc6Dts6joE+COUdFEyqa8CTcAfLhIJ0xGIJugXSYMglMeoPdTxkfShVysQIHUSEaLVamGxWODz+XDCCSfQFOKBgQE0NjYiLy9PlEKcqmKJySBl7XFvvvlmLF26FL/61a9wwQUX4G9/+xu2bduG3/8+FL5rNBrccsst+NnPfobZs2fTFN/KykoqKkUvKAclksxIhJSBiXXieR4Gg0EkAuEqf2kJmEOHDkGj0YhaH2cDwaF9sBQeIysSANiqWYoAF9oZnal5U/YxNDwXFo2E34aPKhKyk/ZUzqGX6T5bS6EfdVHJiPBI5k4URB2i54whDuF2xYPXMTWyILvdmBIIkQcA2n++uqoK01/6l3qEcyIWiwUzZszAjBkz6ELH/v5+7Nu3D16vl6YQk26GiSRjV6zH2x73tNNOw9NPP4077rgDP/nJTzB79mw899xzdI0IAPzgBz/A+Pg4brzxRrjdbpxxxhl48cUX46pgm2sSAdQNZymanP6sDIzc5HRBQUHYZfEcLWVCBKWW+VVabG+Lfbu3+WX0fCShUHguLBqJJRIgtNMmO94gKWiYH97aVucdB0zxzZNoJ0YViYNsRyyEspAjkkAmLCVhl00abJg02DDockGv001rTiEdkQh5Xrnn1Ol0NAohxRL7+/vR09OD/fv3w2Kx0Ouzoed6zrXHPXToEI477jgMDAxk9Y6M8P7772Pu3LmiUs7TnZyWnpJRBqaxsRE8z2POnDmxb5whtLW1YXR0lB7YbG+b2mkJoxEANBqRgwhFNhqJc1iL3k1mB6yLUKwQ+EwqMgTz4hNNUK9wYV2U1yG37e78UCFNaRthEn0AgNViQV9fH/r7+zE+Po7i4mI4nU6UlpbGtd7D5/Ph9ddfxznnnJPSYaPDhw9jfHwcxx9/vOL7BAIBOjlPsmCFUUq8LSF4ngfP88jLy0uajHIuXSDb14lIo4dAIICOjg50dHRk/OR0rmHR+0TZWkohEYpeEzo+I5PyciiJRoCpSEC4Qw4KSq1LhUJk4TFPrYg3TUbOspKSCHkA4u0l4iBIBUKiDwB0/qOwsBBz584NO1rPz89HaWkpnE6n4jaxqUZNiq9erxeVdB8ZGUF/fz9tPEVed2lpKex2e0a87pzbw+j1erpKM5Mmq+Qmp8nENBGDx+Oh/UmIBDiOo6mE2TA5TciF9S3CYS3p3IgSApwGei1PJ+UBAHzosQBgYeBDAMpFAoiHt4S4HKHyMlZvZFEIhUKQikWpPIDYAhm2Rh7eEgqkV1tJm3H1ekuxqCFc3MKyJGTBX19fHz766CNajdfpdKK4uDjsd5Fpw1lK0Wg0sNvtsNvtmDVrFn3d/f392L59OwDQCEW60DGV5KREgFBTqVRJRNrASunktLQEvVx/kg8//BCVlZUoLg4fB890sk0icj/4RTMComEtgl7LRx3SUsIOvbgwn0Wmla5NLz8sZcGY7OXjeSFRRJOJEKFYPMboa7/zx0OlUHiNBkM2+b4uBC0feV6SCIT0oI8lkLD7Cxb8cRyHwcFB9PX1Ye/evfD7/bQSb2lpKYxGY1olkkikr5ssdGxpacHu3btht9tplJKfn5+y15uzEgkEAtM283Qmp00mk2hy2mQyqa6kmW0741yDiGQ60YiQSb+eRiOxGAtYZUUy8dkqvFgyAWILJZY8CKNWJ/zavJi3iyUQIg8AtAGX0zigSCBhz6XV0iPx+fPnY3R0FH19fWhra8OePXvgcDiS2ho2Gqlqjztnzhx4PB5a36u5uRl6vZ5GKWSIO1nktESika6V0/GSrcNCmTBWm0haB8yoK5kUiSQR0YiQiYBRNhqJJBIgJJNIIiFEik6UyoMwHYFQeQhGmohAQvIIL/seL8I6V7NmzYLH40FfXx8tm/Tuu+/C6XTC6XRGLZyYKFJZ9sRkMqGmpgY1NTU0Ouvv78ehQ4dQVlYmyoJNNDknEZ1OB4PBgF27dkGn06Gurk5WDtkyOZ3NO+NslF8kLlk0KjuspYR4ohG1IgEiRyUEIhM3X4hihJfHj4YagXT7p6pXmGRek9LhK7WYTCbU1taipKQEb7/9NmbPno2+vj5aOJEIpbi4OCm/8XTVzhJGZ/PmzWPtcZXw0ksvYd26dejs7ERnZyf8fj++/OUvY/Hixbj33nupDLJl5bSUXNoZZzNyw1qJjkaiEU0kQOyoxM1PDXG5EN6oS04sSuQBTAlEKA6CVCDtw4UosU4kVSBSNBqNKOvJ7Xajr68PBw8ehMfjEaUPx5tGG+t5002y93HTloiaboRvv/02HnzwQWzfvh3d3d149tlnZVel79+/Hz/84Q/x1ltvIRAI4JhjjsG//vUvURMqAKiqqsLll1+OqqoqFBcXY8mSJdi1axcqK6cfIqebTPgSqiFbh+HINpNsOpJBR/6avF54bAviftxERCOAepEIBRIJqVgmA9MfR48kkED/p0BDajr+SSMCjUZD5xPmzp2LsbEx9Pf3o6urC/v27aOVeJ1OJ2w2m+rfIM/zWXGQOl2mLRE13QjHx8exYMECXH/99bj4Yvl+04cPH8YZZ5yBb3zjG7jnnntQUFCAvXv3yj7mcccdR8f8AoEAOI7Lyh1YJHLptWQCUkEQSQwODsLr9WLr1q1h2XQmk4kOdxbndaAT1QmJRpIlEiA0vKVEHrLblWCBkA6OJdYJlOf1ozOFB0exhpVsNhtsNhvq6+vh8/nQ39+Pvr4+NDc3w2AwUKHE2yY3k363GVn2BAi9SevXr8cdd9yBiy66CADw5JNPoqysDM8991zERlLnnXcezjvvvKiPffvtt+P888/HL3/5S3qZki555EPO5gWHQrL1iD5dRBKE9LycIEhyxKxZs2KWeilHAO82TT9TSy2xRAIAh8brAABOy3Bcj51IgQjb/5IhrI6OzP0+G41GVFVVoaqqita46uvrw+7duxEMBkXpwwZDjCKVrJ9IbBLVjVAKx3F4/vnn8YMf/AArVqzAp59+ivr6evz4xz9WVIwxl+pnZeuXMBnyEy7YlA4zxRIESZgg/8sJ4siRIxgeHkZhobKj9zNmefBuUygyTnU0AkQWSce4uBZV34QdgDKZJEog0t7x0jmQVH6v1e7MhTWujjnmGIyMjKCvr4+uyygsLKRRilzRRCYRBSSqG6GUvr4+jI2N4f7778fPfvYzPPDAA3jxxRdx8cUX44033sDSpUuj3j+XJAJkVlicLOQEIScL6YJNk8mkSBDJQigSJcQbjcQjEqk8pMSSSTIF0lA4Dp43pC2ynu7OXLh6fPbs2ZicnERfXx+dnLdarVQopBwJk4gMf/nLX7B69Wr6//PPx9/FTAkcF2r8c9FFF+F73/seAGDhwoXYunUrNm3aFFMi2V4/S0guDGfFEoRwTU4mCUIpb34cwLKT9UmJRpQwFrDC7VVekFBOJtMRSItrar2JzSRu2uXxa2AYO4BPWvqh1WrhdDpTPtmcjN+P2Wympd39fj8tw7Jt2zb6Oj0eT1yFIrOVuCTypS99CYsXT2VUeL2hbm3T7UYohZR9PuaYY0SXz58/H++++27U+2o0Guh0upyKRDKZSIJwuVzwer14//33qSD0ej2VgVAQwmGndApC7VHjHVcE8LO/AstOVvZzSmQ0sq/HTs9XFkaOWOQgMhkcnxrbnxGhL7sQoTSEyAnkohP9AI4XlSfp7OxEMBjErl27UFZWRjulJotkRwQGgwEVFRWoqKgAx3EYGhpCX18fhoeHMTQ0hNHRUTqPksyV4+kiLonk5+cjP3/qC8TzfEK6EUoxGo04+eSTcfDgQdHlhw4dwowZM2Len0UiiUFaNFJuHkJOEHl5eTAYDNBqtWhoaMgIQSQboUiSEY1IRSKUB6FryBi3SIQCAYC2ocir2LkonYGlAmnq1OJb/9/UtggXwFmtVnR3dyMvLw8HDhyAz+dDSUkJysrKFE1YqyFVw0parRbFxcUoLi6mkYjRaERHRwf27t2bsPRhpaRi3zGtORGl3QjPPvtsfOUrX6Gtc8fGxtDU1ESvb2lpwY4dO1BUVETXgNx22224/PLLceaZZ+Kss87Ciy++iP/3//4f3nzzzdgvKofmRJL1JYskCOl5OUFYLJaYEURrayvGx8fTVrcoHRCRnLEo9k5wOplacgIhdA2FJq+VyEQqkGgoFUhTZ2ioSigQKRqNBgaDAXPnzsWcOXMwNjaGnp4eOmFdXFyMsrIyOJ3OhBy5p3M4OC8vD/X19WhoaIDX66Xpw4cPH4bJZKLVh7Oh+VQkpr1OREk3wsOHD2NgYID+v23bNpx11ln0/7Vr1wIAVq5cic2bNwMAvvKVr2DTpk2477778N3vfhdz587Fv/71L5xxxhmxX5Ref1RHIqRwZLQMJqEghDIwm82ilf0mk0lVBJELczlqKCrKw7vbvRgf82HF0vgaQAHRo5E39uSjrETZjiZWVJIogQhRIhBAPLyk0WjoCMfs2bMxPj6Ovr4+dHR0YN++fXA4HCgrK0NZWZnq+YV0TXBLnzcvLw/V1dWorq5GMBiEy+VCX18fdu7cCZ7nafpwSUlJUqKxZJFznQ15nkdNTQ3+8Ic/4PTTT0/35kybXbt2obCwEDU1oW5vUkFEGmaSE4T0fDLrgkm7BGYD7e3tcLvd+NznPjetx/nZX0Pv6fhYaGcaTSaRohEikg8PhB+JKxUJIB+RJFIgNhNH5QHEFggQilIHBwdx4oknRr0dKaDY29uLwcFB5OfnU6HYbDZF2w8AQ0ND2LlzJ5YtW6b4Polg+/btKCkpiTkEz/M8hoeHabYXieDJsNd0JudJkpLJZEqaSHOidpaUbB3OkhPExMQEvF4vBgYG6OWRIgiHwyGSBCscGT+JOKYKDWtNvfcvvRVKwVUambz5cQBFRZGHcXoHuLgiEmBKJvEIJBZqBEJQ8v0gBRRra2vpSvLe3l4cPnwYZrOZCiVWZ8N0ptoqeV7SdM7hcGDOnDmYmJig8jxw4ABsNhsVipoujhqNJnNXrGcimZqdJexNEmmC2uPxiARBOhuaTCY4nc6MEYRScizIjYuQSIw0GgGmZCLFagsvRDg46E2YSICQTEzG+D6PSFFIY2sA+fl6COu6xyMQNQhXkpM+5L29vbSzIRFKYWFh2A4zU4azlGKxWFBXV4e6ujr4/X46j/Lxxx9Dp9OJyrBkQrJK5u+JVJDqSEROEHLnpYIwmUwxI4g9e/YgPz8/J4pJHm3IiSQeEi2S5naxFRpqIt9XKpDG1ql5mpBApohXINPdqQv7kHMcB5fLhd7eXnz66afQaDRwOp0oKyuTbZWbShIhL4PBEFcXx3SQsxJJ1MS6EkF4vV4Eg0GRIBI1xJStw0LZut3JwGqLLpLxMZ9sNAIkTiRdveEHVZGkwnFiaUQj2RFILLRaLS1Ncuyxx2JoaAi9vb10JzvdOYXpkOhIXNrFcWxsDH19fThy5Ajt4kiEEs+c0XTJSYlotVpFEpG2v400zET6tQtlIOyNTi5P1hBTtg4LZet2JxLp/EgkookkFrFEIicQOZrbOfj90WfSSRTS0jKKdTepS79N1vdCo9GgqKgIRUVFmDdvHkZGRtDb24vOzk54vV588skndC1KKo7akzmMJsxqmzlzJm2P29fXh8bGRjoEXlpaqrgWnFpyUiJkOMvlciEvLy9qFJFuQcQiW1Nlj8ZIhCRFCA9GPB4Pzp/rxQsHT0nasBYQWSRKBaKERAiEkOzvh7DWVWFhIfbv3w+73U57rxcVFdG1KIlsQiUkXe1xA4EATR/esWMHFi5cmNRoLCckMjAwgN/+9rfo6OhAe3s7mpqasGrVKni9XvzrX/9Cfn6+aJjJbrdnhCAYmUW0HzzHcWHRqvS88IBEWj34W/X92PhiadKGtYBwkcQrkGhRiFAgv1w9vQyvVE908zwPnU6HmTNnYubMmZicnERvby+6u7uxf/9+FBQUoLy8HE6nE1Zr/Gt7oj1vOg6m9Ho9TTQIBoO52dnw0UcfxaOPPorW1lYAwLHHHos777xT1GNk9erVePXVV9HV1QWbzYbTTjsNDzzwAObNmxf2eBqNBp2dnaipqcFpp52G7u5uXHnllbjqqquyfsV0tkYiQPYMZ0l7kLS3t4cJwufzQaPRiJIiTCYTbDYbPW8ymaIekNxxRQA/fiz6tiRCJMFg/O97rGGslpZQTa0L5m3FG29wKCsrQ3l5edyNmtKBdGduNptp9pPX66XptIcOHYLNZhOtRUlE9d90kuz0XiBNnQ2rq6tx//33Y/bs2eB5Hk888QQuuugifPrppzj22GMBAIsWLcJVV12F2tpaDA4O4u6778a5556LlpaWsLS24uJiPP744/T/zZs3o6SkJOsFAqT/S6iWTNruYDAYMYIQZs5ptVpoNBoMDQ3BZDIhPz+f9tw2mUy0aVUkOI4TddXkeV50HgDuWQnc9URyI9+2lqnqvDPqI5dIIcQSyMDAJB2+4vllGBoaQk9PD3bv3g2Oi18omXRwkZeXR4eBSDptb28vWlpakJeXR4e8HA5H3N9pVgpeAWo7G1544YWi/3/+85/j0UcfxQcffEAlcuONN9Lr6+rq8LOf/QwLFixAa2trzA6HuVSAEcisH12mwfM8fD6f7PASOe/3+6HVakURBBnWFFYV7u7uxuDgII4//njZ5yKCkMoh0ucjLO1Bdq4ajSZp2VrNTe6wy4hQlMhE9FiHXACAgiIL7rtBi2AwSI9qyeT1/PnzVQslE5tSCdNpg8EgXYuyfft26HQ60VoUpbJkEolBIjobBoNB/OMf/8D4+DiWLFkie5vx8XH86U9/Qn19PS3/EY1MXGyolqN9OCvSZLW0UZXRaBQJorCwUCQIg8EQM4ogQggGg7JRhBTyeCSCkf5Pzku56+s8frAp+uuOVyRyAhESKTohUQiRhpCCIgvuX6UDz/P0/SFCVCqUioqKsJ1uqr/PanbmQmmQ9Rm9vb3YtWsXOI5DaWkpysvLUVxcHHHBX7b+buMlbZ0Nd+/ejSVLlsDj8cBms+HZZ58N6x/yyCOP0AKPc+fOxSuvvKIoNS+XIpFsPZJRst3TnawWXhbryFBJFKHVauF2u9HY2CgaExdGEbEEIUQ41zI5OUlfk8fjwVdP9OCfn5wQ8z2KhFAksQQihQjFMxE5GioosuCBG0O7B/J6yfsVj1DITlcoFOFjZgPC9RnHHHMM3G43ent7sX//flEZe6fTKZoTY5GIDInsbDh37lzs2LEDw8PD+Oc//4mVK1firbfeEonkqquuwjnnnIPu7m6sW7cOl112Gd57772YKXl6vZ4WHssFsvGIhhzRj4yMRBREIiarAWVzEVLkooiKigqYzWa68zOZTKioqEB5eblsiiSRoFQQwhNZhGo2m+nrIT0lfjJnHCaTCXdulj+SjbV2ZHDQC/fgZNT3JhJKBSJEGG1JhUKESt7PaEIh4ifzUMkmkTtzjUaDwsJCFBYWYu7cuRgdHUVvby+am5vDythnikSSnk4dTxVf8oYRvF4vjjvuOHz66aeiToZLly7FwoUL8dBDDynekOXLl2PmzJn43e9+J3u9z+dDYWEh/vCHP+CKK66I+lhf/OIX8fnPfx7f/OY3FT9/ptLY2AgAUbPd0kGsyerJyUlRP3ShFIT/x5qsBqYqkcYzFyGcjyB/44kiAoEAent70dXVBbfbDbPZDIvFAp1OJ3qdRIKkhI3wdSqVIOEHm/yyl8uJ5OCOdtH/ZbXR+6sLiSWQi477KK5qudIITyoUoXiGhoZw4MABjI2N0SEjuSGvRNLZ2YmOjg5RV9ZkQMrY9/T0YGRkBBqNBlVVVWhoaEjbqnmO46DT6ZK6uDJjOhuSo7pIkC9ptNsQcq2fSKqjqkRMVo+MjCgq9y0cZiLPHU0QgHwUITyR62I9r/B1CU8ksiBRBFk7MDo6Cp/Ph/z8fFRXV6OsrAwWiyVhR3q/XGOQFYkwIpHKg9B7JNSvJ5ZMognk0R8VIBgMor+/Ab29vWhtbYXFYqGT5RaLRfZ+chGK8DMUCryoqAjFxcUoKChAZWUljVB4nqfPkwyhpCIisFqtqK+vR319PTweD9577z0MDw/j7bffVl3GPhtIS2fDH//4xzjvvPNQW1uL0dFRPP3003jzzTfx0ksvAQCam5uxZcsWnHvuuSgtLUVHRwfuv/9+mM1mnH/++bFfVJaWgo9EooezIk1WC1fyT3eymsheGkXEO8xELosnihC+xmhDTQDCooaSkhIaVZA2v0ImJibQ3d2N7u5utLW1wel00iPpROyofrkm9HwXrdofdp3VHrl1LSGaTOQE0tMWmrt89tE5AELziaS4YSAQQH9/P3p6etDc3AybzYby8vKoDaKiDXmR68kwj9yQ186dO0Pbn0ChpGNYiTRzmz9/PqxWK+0VEm8Z+2wgLZ0N+/r6cM0116C7uxt2ux3HH388XnrpJZxzzjkAQh/AO++8g/Xr12NoaAhlZWU488wzsXXrVjidzpjblGvZWfGQrslqQCyKYDCIyclJ9Pf3i3Ls5SKHeKMIEpFKIwfhKRAI0NdIhplsNhuVhNLXKMVisWDmzJloaGjAyMgIuru7sWvXLmi1WpSXl6OiokIUravl34/NDxPJ+PCoIpEA4TIhAiHSIBB5yKHX61FRUYGKigr4/X66KK+pqUm0yls6RxkIBESfyeTkpOh/n8+H+vp6BAKBqHMoiRJKuuYUyfMajUba0ZCUse/p6VFUxj4byLnOhkBoQr6+vh4//OEP070p0+bw4cPw+/2YN29e2MrqeCarpXMRSsbp1UYRZAiura2NzqGRnZGSHawwUpIbZiKRkvD1yM1H6PX6lPwoSQpod3c3+vr6YLFYqFASUZdJbVRCmByfkL08mkAiwfM8xsfH0d3djf7+foyPj8NoNMJoNFK5ywlcep5EeHKp0dI5lJ6eHlEmaLxCaW9vR29vL0466aS4X+90eP3113HiiSfC4XDIXi8sY9/b25uUMvapmBPJSYlcc801qKysxO23357uTYkLucnqgYEB+Hw+6PV6urI6XZPV0r9Kogie5+FyudDV1YX+/n7apS0/Pz+iLMh8S6SJajLUlInlNsgQEFm46HA46BDQdPpmy4kEiC4TOXnEEocwyiPSlkYV5DtIvmscx9HbkAiloqJC0Y5LeICSLKEcOXIE/f39WLRoUcztSSSvvfYaTjrpJNjtsRd6chxHU4d7e3sRCARQWlqKsrIylJSUqK7vl3ET69lCJs6JRJusJv+TnadQCnq9HjqdDvX19fQyJd3MEpHyKvyfnI8FEaF05+Pz+WA0GjEyMoKRkREAoRXCVqsVdrtdNJRmNptTFkUkGuEQkNfrpaXIDx48iJKSElRUVKCkpCRuAf77sfkAwmUSaYiLCEQqDY7jZKM74WdF5sOIxM1mM/Lz80XRhNxOzePxoLe3Fz09PWhsbERRURHKy8tRWloaUaDRJuWl30G1Q17pPE5W+h3WarWyZewbGxuxa9cu0VqUeA5GUvHaczISufHGG2Gz2XDPPfek7DnVTlZLz0snq1taWjA5OSlaP5OMKEL4Y46EVIRy8xFCEUZLe52YmEBXVxe6u7uh1WpRUVGBysrKiBlA2c74+Dh6enrQ3d0Nv98v2uklQpbnXv4xzAU2PLNpTljkIB0KJMOdQklIP6/pRnkTExP0qHp8fJyunygtLVV0VJ3ICKWtrQ0ulytmpmCiefXVV7F48eJpz5GNjY3R93J0dDSuMvbBYBAGg2FaUXAsclIiN910EwwGA37+858n5PHUTlbLiULJj1MYRbS1tWFsbEy2ejFBTg7A9KIIufkIr9cLjuNEIpQTRawSI9LXKhzustvtqKysRFlZWU6W6Od5HsPDw3SnR7KhKioqFKV+SlfAS4UhlHgkSeTl5aU0yiMC7e3txeTkJEpKSlBeXo6SkhJFUfV0hQKEsgVTLZFXXnkFS5YsSWhK78TEBE1wcLvdsNvtdGJe7gCMSUQl3/72t8HzPO6///6Yt033ZLX0vBS32439+/fT1dNkbF0oDKXF4Px+f9SUV5/PJ4oiIp2U/PDV4PP50NPTg66uLkxMTMDpdKKysjJrs1ZiQQRKJqnNZjOcTifsdnvEYcFAIAC9Xh8mcOHfeCSeasbGxqhQvF6vaNxf6fc4XqF0dnaC4zhUV1cnbR2KHC+//DJOO+20pK0LEZaxd7lcsmXsmURUcsstt2BychLr1q2LurKanE/GZDX5G09GU6QoguM49Pb2oqOjA2NjYygvL0dVVRXsdrvoNtGGmYST8tGGmZS8zlQwOjpKh7vIPENlZWXaVv4mAukCR2E0Qc6T74lOp4PFYoHdbofVahV9ZsncIaQKnudpBYyenh46kRxvSXklQmlubsbAwABsNtu0srzi5aWXXsIZZ5yR0EZXkRCWsR8YGKBl7EnNL5adpYBnn30Whw8fxpEjR/DCCy8gEAhgaGgIt912GxYsWBB1iCmeyWpA3VyEdOGc9Hw0hNHS0NAQent7MTIyAq1WC4PBAI7jaLQUbZgpmVFEsuA4Dv39/ejq6oLL5UJhYSEqKyvhdDoz7rUIs5QizUeQ1ORo8xF+v5923hsfH6c7VzUT8tkAz/MYGRmhEQrHcTTVtaioSNFBTbTSKy0tLRgbG8OCBQvA8zytyJtsobz44os488wzUz7PJyxjPzQ0hKVLlyIvb3rtjKMxbYnwfPydDe++++6wSe+5c+fiwIED9H+Px4Nbb70Vf/vb3+D1erFixQo88sgjYRWDCZdddhkAoLa2Fjt27EBBQQFWrVqF4447TnTEHolkRRFK50CiFfKTplUS+fn9foyOjmJychJOpxM1NTWqmudkC16vF93d3ejq6oLH40F5eTkqKysVfb6JQG4VvFAUQpHLyUFNajIZ/unu7kYwGKS1plL1mlMNz/OiVFcAdIhGyXebJH8Io7uBgQGYTCYcf/zxYUNeyRIKz/N46aWXsHTp0rRGz4FAAEajMbOHsx544AHcd999os6Gu3fvjtrZ8O6778Y///lPvPrqq/QyvV6PkpKpUg033XQTnn/+eWzevBl2ux3f/va3odVq8d5778Xcpp/85Cdob2/HI488Qi9LZBQRjyB4no9ZgoNkzMSai4g05zI6OorOzk50d3fDZDKhqqoKFRUVOTHsIQc5cu3q6kJPTw+MRiMqKyuntbhPGO3JSYJkngmTJ+SiiWRNWpOdK5k0JkN85eXlOVeLiSCc1+jr64NWq4XT6URJSQkMBkPEIUEyDyD8XEpLS2G326MWh0ykUIhEli1bpvo7mQjIwWcyk1SmJRGe51FZWYlbb70V3//+9wEAw8PDKCsrw+bNmyM2pbr77rvx3HPPYceOHbLXDw8Po7S0FE8//TS++tWvAgAOHDiA+fPn4/3338epp54adbtuuukmNDU14a9//WvEH3SsdFcgvigi2nwEKeQXbZgpETufYDBI505GR0dRVlaG6urqnD1qBfBZwcDQcNfg4CCKiopQWVmJ0tJS0XBXpFIpws+K7HzkPifhfES630uO42jpjP7+flitViqUZA5bpAJy0EWkIPycxsbG6JAgEConYrPZkJ+fLxK62WyWHeqMNuSVaKFwHIeXX34ZZ511Vlo/k1RIJG2dDRsbG1FZWQmTyYQlS5bgvvvuQ21tLQBg+/bt8Pv9osedN28eamtrFUkkEAjg7bffxvLly7Fq1SpccsklMJvNcUcRwFRKpdwQkzTvXigF4RdbaebWdNHpdLS959jYGDo6OvDpp58iLy8P1dXVORmdkBRZp9OJkZERurBv79699H0nn6HcIjrSQz3aIrpMgxyRO51OWtOKLPArLCxERUVFWIOkTEEqCWkUMTk5KZI5kUJJSQmqq6thNpthNBppmnR/fz8mJyfpDj5aVBZpYSPpayKcoywuLkZxcTHmz59PhRJPLS8iqnQfcKSCaUUiW7duxemnn46uri5UVFTQyy+77DJoNBps2bJF9n7/+9//MDY2hrlz56K7uxv33HMPOjs7sWfPHuTn5+Ppp5/GddddF1b2/ZRTTsFZZ52FBx54IOa2jYyM4IknnsAjjzyCvr4+rFy5EqtWrUJtba0oo4ksnos01CRNqZSbuDYajRk74Umik87OToyMjKCsrAxVVVVZN3dCsuwiDTV5PB6RzHU6HXw+H63tVFZWhpqamqzO7oqFx+Oh8ycTExMoLS1FRUVFwuowKUWYSi4VBPlNSSUhPa9UgMJJZJImTXbySrOipMPa0tR5NRFKIBDAq6++ii984QtJzYyKRcZFIonqbHjeeefR88cffzwWL16MGTNm4O9//zu+8Y1vqHpMKQUFBfjOd76Dr3zlK3jiiSewZcsWPPTQQ5gzZw60Wi1mzJiBNWvWhGXLmEwmUaXXbE+plEYnnZ2d2LFjB/Ly8ujcSTq/5ARhxCc31CRdw2I2m2G1WmNW5Q0Gg+jr60NXVxfee+89FBcX0+GuTBW/WkwmE+rq6lBXV4exsTF0d3fjwIEDCAaDtLx7IoY2hXNHcrKQSsJkMqGoqEgki0Tt1IS90IUVcltaWmC1WmndsmgZUtEiFKlQlEYoR1MkEtcn+aUvfUnUHYxECr29vaJIpLe3V9TpMBYOhwNz5sxBU1MTAKC8vBw+nw9ut1tUAbO3txfl5eXxbDJefPFF/PWvf0VtbS2OO+44ujBnZGQER44cwVe/+lUUFBTE9ZjZis1mw9y5czFr1iz09fWho6MDTU1NcDqdqK6uTlp0Il3oKCcJuUV0drsd5eXl01rDotPpaC2ryclJdHd3o7GxEfv376fZXfn5+Tn3Y7fZbJg9ezZmzZpFJ6c//fRTGAwGOn8S6UhdblW88Dz5rISRA6l9ls61LHq9nspSuG7i8OHDtCkU+T5FIlFCKS0tpY+R6yRkYv373/8+br31VgChYSSn0xl1Yl3K2NgYamtrcffdd+O73/0unVj/61//iksuuQQAcPDgQcybN0/RnEgsxsfH8Ze//AUbN25EW1sbvv71r+PGG2/EzJkzc25nEgsSnXR3d8NgMNC5k3iiEzJpHW2NBAmrI6W+pnLHQzKdurq60NvbC7PZTLO7MiEqSxZk6KerqwsDAwO0sKLBYKCSl0pCbsgp26Jzn89HV3YPDQ3RSsNlZWWKJ73jHfLq6upCZ2cnHUpN5Up5IcFgEEajMXOzs4BQiu/9998vSvHdtWuXKMVX2tnw+9//Pi688ELMmDEDXV1duOuuu7Bjxw7s27ePGvymm27CCy+8gM2bN9OhKSA0D5MoOI7DO++8gw0bNuA///kPzjrrLKxevRrLly/PuIVsyYYM+3R2dmJ4eBhOpxNVVVU0NI81HyE3LCg9n4nvaSAQoMNdbrcbJSUlqKyszOqFfcKmUHJDTqTfh16vB8dx8Pv9sFgstFKs1WrNKknEg8/no2tQyEgHGQ5TegAhXSkPhJIdhELxer144403cNJJJ4WteUmlUFIhkbR0Nuzo6MAVV1wBl8uF0tJSnHHGGfjggw+oQADgN7/5DbRaLS655BLRYsNEotVqsXTpUpx55pno6OjApk2bsGbNGuTn5+PGG2/E17/+9YgNZXIJ0oWQdFmzWCx0ZTww1V5UKASz2Swq324yTb/yazrQ6/V0zoi0vj148CD27dtHS60kolNhIhGuO5KTBVnPIowgHA4HXUdD6msRJicn6YR8R0cHbfmrtPxINmE0GlFTU4Oamhp4PB709fXRz1xYHTeaREkUIlxvFgwG6ZCXVqul69KKi4tRUlKCY445BoODg7Ll65Wuys9UcqbsSaKYnJzE3//+dzzyyCPYt28fvva1r2H16tWYP39+Vn7QwpTKSENNck2gyOrqyclJuFwuuv6nuro6ZwsiEsgiNzLcZbVa6XBXKo7QpYtTpWsmhJKIlOGkph8Lz/N0Qr6npwc8z9MV8rnQCzwak5OTonLrxcXFtBdKrKN4juMwMTEh+pwmJiYwMDCAc845J+KQFynzAiRPKFkxnJWr8DyPDz/8EBs2bMA///lPnHrqqVizZg3OP//8jMq/F5Z5iCQJstgx1nxEtC/v+Pg4Ojs70dXVBb1ej+rqalRWVub0HAIQ2qGTo3QyV1dZWTmttFkS+UUacpKThFAQqWjaRURKWv4ajUZasj5Xe74QSC+Unp4eTExMoLi4GEVFRbBYLKKSKuTk9XpF5ffJyWq1UilIq12kSigZv2L9aKGnpwe///3v8fvf/x46nQ433HADVq5cKSrTkiyEK+LlsprIpLVwEZ3cfESivkQcx9G5k6GhITp3ku0huRJIb/Guri6aVCLXB4RIItKQk7B8SqR1EpmwMp5AJuS7u7sxMDCAgoIC2pYgFw4ihNmD0tPExAQ8Hg+9rcFggM1mg8PhgNVqpZ9XpIoT0hJLqRYK2Tckcz6SSSQO/H4/nnnmGWzcuBHbtm3DpZdeitWrV2PBggWqf/CR+kYIV8QDoIvoIokiHZPWExMTNDrR6XSoqqpCZWVl1pfeiIU0OiFtBIBQ2jtZ0xJJEJne8yMaZGK6p6cHw8PDKC4uRkVFRViZmUxDGP3JnYT1tuROeXl5ogjF5/PRSsNKo9J0CIVJJEPheR6ffvopNm7ciL/+9a9YuHAhVq9ejYsuuijsyCzWfIRcIyjpMEa8lV9TDSnX3tHRgaGhIZSWlqK6ujproxPh6ni5tS1CSeTl5dES8D6fj9bucjqdGf2ZJQLhhLzH4xFNyKf6cyf9dCJJQppsID3Fs0qe9EIhO/hAIACn0xlX1lWqhMIkkgU0NTXhoYcewpYtW+DxeHDiiSfC7/fjlFNOwfLly2k6ZbT5iExpBJUIsiE6kZZQkcpCKIlIQ05ykcTY2BhtpKXRaOhwVyqaEqUTslMlE/IA6PxJohZyCgtoykmClL2RzksIT8mI/ng+1O6YTMqTXihEKEp7oZC/iRYKk0gW8MUvfhEffPABamtraTYTx3E4/vjjce211+Kkk07K2qGL6UCik87OTgwODqK0tBRVVVUoLi5O+nshHSKUykIY/UUacpqO2I+2vvFCyE6OTMibTCYqlFh1yyLNSwgXrJK1SNIowmKxpLx3vBRhuf7e3l5oNBq6g1dabiaRQgFA3zMmkQyGrO4l8DyP/fv34+GHH8ZTTz2FOXPmYPXq1fjqV7+a1r4C6WRiYoKu4NVqtTQ6Uft+CDsIykUT0mwZOVmkKvoT9o0fHx9HWVlZTveNF0LK9Hd3d8PlcqGgoABFRUWwWq1h5d7JIkhpYUapLDJ53kUIx3F0rRXphUJ28ErTpeWa4sUrFPKcTCJZitvtxubNm/HII49gaGgI1157LW644QZUV1fn/A5EDtIHo6OjA4ODg7S8tzQ6kY5vS2VByu/LleMgl2XiEKGwb7ywMGauVBYWduiUO/l8Pnpbo9GI/Px8FBYWirKccnGlPMdxdAff399PF/WShmLJjFA0Gg2OP/54JpFsJxgM4uWXX8aGDRvw6quv4rzzzsOaNWtwxhln5Pzkqxwcx8HtdqOzsxP9/f0AQPu9eL1ekSQi1W5K99DFdCAyJTWsHA4HHe7K5CNt4ZqkSAshgVAmocVikY0ojEYjLYTZ09MDr9ebMyu3lRAMBuFyuWjp+ry8PFrHS2mHSqVCIYUj2XBWDsHzPJqamrBx40Y88cQTqK6uxo033oivfe1rOTX5SiKJSHMSwnbA5Cjc4/FgYmKC7lArKiqOCsH6fD669mRycpIOd6Wr34vcMJPwJFyTFCkVVunnxn/W5ri7u5vOIZD5E6VH6NkMWX/T09ODgYEBWCwWOgQVTy8U8ldOKEwiOczY2Bj+/Oc/Y+PGjejs7MTVV1+NG2+8EfX19Rn/45EOW0hlIc2UkYsm5CKJyclJmtlFspuqqqqOirkkkuFE+sYLa3ol8vWT+aRI0YTf76cl+SNFE8nYIQmHfEhlZVKyPleG+6IRCARo6fqBgQHYbDYaoSh5/STqGBgYQHNzM5qbm9Ha2oq2tjbcd999qKmpSdq2M4mkGY7j8NZbb+Hhhx/G888/j7PPPhtr1qzBF77whbQdiQslITdkEUkSwh3OdIabyI+hs7MTLpcLxcXFdO7kaIhOSGZbV1cXXC4XCgsL6dqTWDtwYVl+uZN0PknulOySKrEgO9Tu7m4MDg7CbrfTFfK5OGcihfRC6enpweDgIAoKCtDZ2YkFCxagsrISbW1taG1tpaIQnsbGxlBeXo76+nrU19ejoaEB3/rWt+LuwxQPTCIZAs/zaGtrw6ZNm/D444+jsLCQVhJOdNOsSJIQTlwDkF3XIlzvkoodjcfjQWdnJzo7OwGAZnYdDUenQGgFPBnu8nq9dJW0wWCIGFHwPB9WcVk65JTp0S7B6/Wit7cX3d3dGB0dRWlpKcrLy1FSUpLR80dqIENSg4ODaG5uRktLC9rb2xEIBPDyyy9j27ZttA7ejBkzMHv2bNTX12PmzJloaGigf61Wa0o/36yTSGtrK+699168/vrr6OnpQWVlJb7+9a/j9ttvz4k6PkAoJfZvf/sbNm7ciMbGRlx55ZW48cYbMXfuXMWZHLHmJACIcu6lssi0VfJk7UVHRweNTqqqqrK670ckIpXoGBsbo71bgNDnl5+fD5vNFpYKm2vvCRCqXUZWyPv9flFvjmyRIqnTdeTIERpNtLS0oLW1FS0tLWhra8PIyAhKS0tpJEFE4XA4sG/fPrz88stoaGjAE088ke6XAyALJfLiiy9iy5YtuOKKKzBr1izs2bMHq1atwtVXX41169ale/MSCs/z2Lp1KzZs2IBnnnkGZ5xxBm2a1dHRAY7jYLPZwtJgyY4m1pxEtu5oSHQiLIRYVVWVNdFJpBIdwhXzcqXehRlOZEHf4OAgioqKUFVVlZN94+Ugq8R7enrQ09MDnU5H2+Kmu/cL2Z0ODQ2hpaVFdCLDUO3t7dBqtairq0NdXR0aGhroaebMmZg5c2bMlf7BYDBjIrGsk4gcDz74IB599FE0Nzene1OSwvr16/Hhhx/igw8+QHt7O/2irly5EitXrpRddZ3NklAKz/N07mRgYABFRUWorq5Oe3QiTYVNZokOj8eDrq4udHV1we/3ixppZcvR+XQgEWp3dzf6+/thsVjohHyyEjJIj56Ojo4wUZC5CbfbjaKiIhpJkGiCDDlVV1enfe4pUeSERO644w68+OKL2LZtW7o3JSncc8894DgOdXV1qKqqwr59+/CPf/wDO3bswGWXXYY1a9bguOOOy4kvpFrIzrSzsxMcx6Gqqiqp0UmkEh0kmpCW6JAKI9FzSpH6xpeXl2dU3bJkQlodkwlp0s0xVqdCOUj6sZwkyFwFANTW1qKurg4zZ86kw09EFulK0041WS+RpqYmLFq0COvWrcOqVavSvTkpg+d5bNu2DY888gi2bNmCRYsWYfXq1bjwwguPigyWSPA8D5fLRRcyqh3qkVbylSvRQRp9ZVqJjmAwiN7e3pzqGx8vXq+Xzp+Mj4+jpKQEdrudpsySlradnZ1UDNJowuVyweFw0EhCGk3U1tYelXXxpGSMRH70ox/hgQceiHqb/fv3Y968efT/zs5OLF26FMuWLcMf/vCHZG9ixtLf34/HHnsMv/vd7xAMBvGNb3wD1113HZxOZ7o3La14vV50dXXR+SMyd2KxWGjygTSKIG1OpT1B5KKJbJD15OQkHe4KBoMZ2zc+GZB2v01NTeju7sb//vc/bNmyBRUVFZicnERfXx+CwSBqampoFCGdnyguLgaAo14U0cgYifT398PlckW9TUNDA83A6urqwrJly3Dqqadi8+bNR80RVjQCgQD+/e9/Y8OGDXj//fdxySWXYPXq1Vi0aNFR9yMQdqubmJiAy+XC0NAQPB4PtFotOI4DgKjzEplYf0stkfrGl5eXZ21WI1lg193dLRtJtLS0oL+/H/n5+aivr0ddXR2Ki4vR1NSEHTt2oKKiAjt27EhZunqukjESiYfOzk6cddZZWLRoEZ566qmMyVLIFHiex549e7Bx40Y89dRTOOaYY7B69WpcfPHFOTU+Hq1EB+ktLy3RodfrMT4+DpfLJZo7yfW+4UICgQAd7kpU3/hkwfM8JiYmIg45tbW1wev1orq6WrTAThhNlJaW0nIgBJ/Ph7179+KEE05I46vLDbJOIp2dnVi2bBlmzJiBJ554QiSQZK7KzFaGhobwxz/+EZs2bcLIyAiuu+46fOMb30BlZWXGH31JS75LT7G61UUr0UEWdXV0dKC/vx+FhYWorq4+atJkCaRMP0mXJsNdSosBTheywK63txctLS2iVdhEFj09PbBarTQdVjqBXVdXx6KJNJJ1Etm8eTOuu+462euy7KWklGAwiP/973/YsGED3njjDXzxi1/EmjVrsGTJkrTtNKOV6BCWV4k0cR1PKmw0yNxJZ2cnAoEAXRWfS0UxY0Gk2tXVhb6+PthsNjrcNd25HzL/ROQgV65jYmIClZWVstHErFmzUFZWFhZNMDKDrJNIKvj5z3+O559/Hjt27IDRaITb7U73JiUMnudx6NAhbNy4EU8++SRmzJiB1atX47LLLkv4kI5wXiJSNMHzvGy3unSU6CA70s7OTvT19aGwsBBVVVVHRb90IX6/nw53kVIjZLhL7rMg0UR/f39YKiyRRFdXF/Ly8iJGE/X19bBYLEwSWQiTiAx33XUXHA4HOjo68Pjjj+eURISMjIzgySefxCOPPILe3l5cc801WLVqFWbMmKH4xxypRAcRB+lWFy2ayMQdtM/no5ldgUCAZnYdTdEJECo1Qoa7du7ciV27duH000+H1+sVSYIU/ysrK4sYTZDy/kwUuQWTSBQ2b96MW265JWclQuA4Dq+//joefvhh/O9//8OKFSuwZs0aLF26FH6/H4ODg7SZkDSikEuFlasKm62QrKaOjg709fXB4XCgqqoKZWVlGSk/tQiL/8mV6zhy5AhMJhONZO12OxYsWIAvfvGLouJ/R0MfEIaY7P11MxKGVqvF8uXL4fP5MGvWLLz88sv48pe/DKvVitHRUZxyyim45557qBRsNhtKS0vjLtGRjWg0GhQVFaGoqIhGJ83NzTh48CAqKipQXV2dNdEJGV5sb2+XXYnd1taG4eFhlJSUiIr/nXnmmVQSVVVVcLvd2LJlCxobG/H9738/3S+LkWZYJBKFoyUSIfzgBz9AV1cX6uvrUVlZiebmZrz99ts4dOgQrrzySqxatQqzZ8/OWWEohUQnZO6koKAA1dXVivp9pGLb3G63KNOJRBOkXIdWq8WMGTPCiv+RYaeCgoKj/jNmKOeoiUTUrIg/2vjlL38ZdhnHcXj33XexYcMGLF68GEuXLsWaNWtwzjnnpH2HmS6k0Ul3d7coOqmqqkpaiiwp19He3i67dqKlpQVutxuFhYWi4n+LFy/GrFmz0NDQgJqampwp/sdIP0dNJBLvinjg6ItEosHzPDo7O7Fp0yb84Q9/gNVqxapVq3DNNdfA4XCke/PSDokAyNxJQUEBnTuJV7akVa40miCiOHLkCHiej1r8L5t6bDCym6NGImpgEpHH6/Xi73//OzZu3Ig9e/bg8ssvx5o1a3DMMcewHRdAo5POzk54vV5UVlbCbrfTxbCkXAcp/idsTESE4XK5YLfbIxb/mzFjRk7PRTGyByYRGY4cOYLBwUH85z//wYMPPoh33nkHADBr1qyUreTNBniex0cffYSNGzfiH//4B0455RSsXr0aF1xwQVYUJ0wmpPjfoUOH0NfXh9tvvx2BQAAmkwkej4e2Pa2urqbRhLCLHSn+xxbYMTIdJhEZrr32WtnWk2+88QaWLVuW+g3KAnp7e/H73/8ev//976HRaHDDDTdg5cqVKC0tTfemJQUSTfT09IS1OCURBVn5TSIJn8+HnTt3YmRkBL/85S9x/fXXZ1W/cwZDDiYRRkLx+/149tlnsXHjRnz00Uf46le/ijVr1mDhwoVZt7PkeR6Tk5NUDtK6TqT4X2VlpWgVtrD4n9PpFEUTPM/j/fffR21tLaqrq9P8Chm5TmtrK+699168/vrr6OnpQWVlJb7+9a/j9ttvT1j1ZiaRFLFx40Y8+OCD6OnpwYIFC/Dwww/jlFNOSfdmJQ2e57Fjxw488sgjePrpp/G5z30Oq1evxle+8pWMKT1OFtj19fVFjCa6u7thsVhQV1cnuwq7rq4OZrM56wTJODp48cUXsWXLFlxxxRWYNWsW9uzZg1WrVuHqq6/GunXrEvIcTCIpYMuWLbjmmmuwadMmLF68GOvXr8c//vEPHDx48KhoHOVyufD4449j06ZNmJycxHXXXYcbbriBFtVLJqTIozTDSSiKiYkJlJeXh/XDJtFERUUFm5tg5AwPPvggHn30UTQ3Nyfk8ZhEUsDixYtx8sknY8OGDQBCay9qamrwne98Bz/60Y/SvHWpIxgM4r///S82bNiAt99+GxdddBHWrFmDU045RXUJERJNDAwMRGxMRIr/zZgxI2LxP6vVyiTBOCq444478OKLL2Lbtm0JeTwmkSTj8/lgsVjwz3/+E1/+8pfp5StXroTb7ca///3v9G1cmuB5Hvv378fGjRvx5z//GTNnzsTq1atx6aWXwmw2y97e5/PhyJEjsnWdWltbMTo6CqfTGVb8j4iioqICOp2OiYJxVNPU1IRFixZh3bp1WLVqVUIek0kkyXR1daGqqgpbt27FkiVL6OU/+MEP8NZbb+HDDz9M49aln+HhYWzevBmPPPIIXC4XvvSlL+HYY4+li+1IuY6Ojg7o9XrMmDFDJArhsFN+fj6TBOOoQE0Fjs7OTixduhTLli3DH/7wh4RtC5NIkmESUQbHcXjllVfw3e9+F6OjozjjjDPCJFFVVcXKdTAYiL8CR1dXF5YtW4ZTTz0VmzdvTmgF6qOmdla6KCkpgU6nQ29vr+jy3t5e1s5XgFarxYoVK3DgwAEAYKJgZBSZ1qiutLRU8Rqszs5OnHXWWVi0aBH+9Kc/JbyFQe40RMhQjEYjFi1ahNdee41exnEcXnvtNVFkwgjBsqAYmYjP58Oll16Km266Kd2bEhednZ1YtmwZamtrsW7dOvT396Onpwc9PT0Jew4WiaSAtWvXYuXKlTjppJNwyimnYP369RgfH4/YK57BYGQW99xzD4BQPb1s4pVXXkFTUxOamprCFrcmaiaDSSQFXH755ejv78edd96Jnp4eLFy4EC+++CLKysrSvWkMBiOHufbaa3Httdcm9TmYRFLEt7/9bXz7299O92YwGAxGQmFzIgwG46jkRz/6EZ2Di3QiiR6MyDCJ5Ahvv/02LrzwQlRWVkKj0eC5555L9yYxjnI2btyIuro6mEwmLF68GB999FG6N0nErbfeiv3790c9NTQ0pHszMx42nJUjjI+PY8GCBbj++utx8cUXp3tzGEc5W7Zswdq1a0X14lasWJFR9eLiSZNlRIYtNsxBNBoNnn32WVGZFQYjleRavTjWqC4yLBJhMBgJxefzYfv27fjxj39ML9NqtVi+fDnef//9NG6Zeu68805Ro7oTTjgBAGtUB7A5EQaDkWAGBgYQDAbDUtjLysoSusgtlWzevJlWjBaejnaBAEwiDAaDwZgGTCIMBiOhsHpxRxdMIgxGBpELqdqsXtzRBZNIjjA2NoYdO3Zgx44dAICWlhbs2LEDR44cSe+GMeKCpGpv3Lgx3ZsyLdauXYvHHnsMTzzxBPbv34+bbrqJ1YvLUViKb47w5ptv4qyzzgq7fOXKlVlXNI4RIttTtTds2IAHH3yQ1ov77W9/i8WLF6d7sxgJhkmEwchQsl0ijKMDNpzFYDAYDNUwiTAYDAZDNUwiDAaDwVANkwiDwWAwVMNqZzEYGcTY2Biampro/yRVu6ioCLW1tWncMgZDHpadxWBkECxVm5FtMIkwGAwGQzVsToTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZqmEQYDAaDoRomEQaDwWCohkmEwWAwGKphEmEwGAyGaphEGAwGg6EaJhEGg8FgqIZJhMFgMBiqYRJhMBgMhmqYRBgMBoOhGiYRBoPBYKiGSYTBYDAYqmESYTAYDIZq/n/4d8VeouBs5QAAAABJRU5ErkJggg==", 57 | "text/plain": [ 58 | "
" 59 | ] 60 | }, 61 | "metadata": {}, 62 | "output_type": "display_data" 63 | } 64 | ], 65 | "source": [ 66 | "# Make data.\n", 67 | "x = torch.arange(-2, 2, 0.2,requires_grad=True)\n", 68 | "y = torch.arange(-2, 2, 0.2,requires_grad=True)\n", 69 | "x, y = torch.meshgrid(x, y, indexing='ij')\n", 70 | "r = fn(x,y)\n", 71 | "z = torch.sin(r)\n", 72 | "\n", 73 | "fig, ax = plt.subplots(subplot_kw={\"projection\": \"3d\"})\n", 74 | "\n", 75 | "# Plot the surface.\n", 76 | "surf = ax.plot_surface(x.detach().numpy(), y.detach().numpy(), z.detach().numpy(), cmap=cm.coolwarm,\n", 77 | " linewidth=0, antialiased=False)\n", 78 | "# Customize the z axis.\n", 79 | "ax.set_zlim(-0.5, 1.01)\n", 80 | "ax.zaxis.set_major_locator(LinearLocator(10))\n", 81 | "# A StrMethodFormatter is used automatically\n", 82 | "ax.zaxis.set_major_formatter('{x:.02f}')\n", 83 | "ax.view_init(10, 60)\n", 84 | "plt.show()" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 7, 90 | "id": "e5d0f05c-6990-4404-8013-8b323bfa7c6c", 91 | "metadata": { 92 | "tags": [] 93 | }, 94 | "outputs": [ 95 | { 96 | "name": "stderr", 97 | "output_type": "stream", 98 | "text": [ 99 | "[2023-03-21 07:04:09,637] torch._inductor.debug: [WARNING] model__1_forward_4 debug trace: /pytorch-examples/pytorch-graph-optim/torch_compile_debug/run_2023_03_21_07_03_48_329556-pid_24339/aot_torchinductor/model__1_forward_4.2\n", 100 | "[2023-03-21 07:04:09,677] torch._inductor.debug: [WARNING] model__1_backward_5 debug trace: /pytorch-examples/pytorch-graph-optim/torch_compile_debug/run_2023_03_21_07_03_48_329556-pid_24339/aot_torchinductor/model__1_backward_5.3\n" 101 | ] 102 | }, 103 | { 104 | "name": "stdout", 105 | "output_type": "stream", 106 | "text": [ 107 | "Writing FX graph to file: /pytorch-examples/pytorch-graph-optim/torch_compile_debug/run_2023_03_21_07_03_48_329556-pid_24339/aot_torchinductor/model__1_forward_4.2/graph_diagram.svg\n", 108 | "Writing FX graph to file: /pytorch-examples/pytorch-graph-optim/torch_compile_debug/run_2023_03_21_07_03_48_329556-pid_24339/aot_torchinductor/model__1_backward_5.3/graph_diagram.svg\n" 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "fn_compiled = torch.compile(fn, backend=\"inductor\", \n", 114 | " options={'trace.graph_diagram':True,\n", 115 | " 'trace.enabled':True})\n", 116 | "\n", 117 | "out = fn_compiled(x.to(device), y.to(device)).sum().backward()" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "id": "d687a68c-afab-462e-b3cb-1d6f723603bb", 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "id": "59d094a0-84fa-430d-b8e1-94c30e310283", 132 | "metadata": { 133 | "tags": [] 134 | }, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "\n", 141 | "\n", 142 | "\n", 143 | "def forward(self, x, y):\n", 144 | " pow_1 = torch.pow(x, 2)\n", 145 | " neg = -pow_1; pow_1 = None\n", 146 | " pow_2 = torch.pow(y, 2)\n", 147 | " sub = neg - pow_2; neg = pow_2 = None\n", 148 | " exp = torch.exp(sub); sub = None\n", 149 | " mul = x * exp; exp = None\n", 150 | " pow_3 = torch.pow(x, 2); x = None\n", 151 | " pow_4 = torch.pow(y, 2); y = None\n", 152 | " add = pow_3 + pow_4; pow_3 = pow_4 = None\n", 153 | " truediv = add / 20; add = None\n", 154 | " add_1 = mul + truediv; mul = truediv = None\n", 155 | " return add_1\n", 156 | " \n" 157 | ] 158 | } 159 | ], 160 | "source": [ 161 | "from torch.fx import passes, symbolic_trace\n", 162 | "model = symbolic_trace(fn)\n", 163 | "\n", 164 | "g = passes.graph_drawer.FxGraphDrawer(model, 'fn')\n", 165 | "with open(\"unoptimized_graph1.svg\", \"wb\") as f:\n", 166 | " f.write(g.get_dot_graph().create_svg())\n", 167 | " \n", 168 | "print(model.code)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 17, 174 | "id": "090bfe7c-8737-40eb-b870-9a87bbaa09b0", 175 | "metadata": { 176 | "tags": [] 177 | }, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "\n", 184 | "\n", 185 | "\n", 186 | "def forward(self, x_1, y_1):\n", 187 | " pow_1 = torch.ops.aten.pow.Tensor_Scalar(x_1, 2)\n", 188 | " neg = torch.ops.aten.neg.default(pow_1); pow_1 = None\n", 189 | " pow_2 = torch.ops.aten.pow.Tensor_Scalar(y_1, 2)\n", 190 | " sub = torch.ops.aten.sub.Tensor(neg, pow_2); neg = pow_2 = None\n", 191 | " exp = torch.ops.aten.exp.default(sub); sub = None\n", 192 | " detach = torch.ops.aten.detach.default(exp)\n", 193 | " mul = torch.ops.aten.mul.Tensor(x_1, exp); exp = None\n", 194 | " pow_3 = torch.ops.aten.pow.Tensor_Scalar(x_1, 2); x_1 = None\n", 195 | " pow_4 = torch.ops.aten.pow.Tensor_Scalar(y_1, 2); y_1 = None\n", 196 | " add = torch.ops.aten.add.Tensor(pow_3, pow_4); pow_3 = pow_4 = None\n", 197 | " div = torch.ops.aten.div.Tensor(add, 20); add = None\n", 198 | " add_1 = torch.ops.aten.add.Tensor(mul, div); mul = div = None\n", 199 | " return add_1\n", 200 | " \n" 201 | ] 202 | } 203 | ], 204 | "source": [ 205 | "from functorch import make_fx\n", 206 | "g = make_fx(fn)(x, y)\n", 207 | "print(g.code)" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "id": "6351c611-cf45-40e0-8e78-5b3707cb8a0e", 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [] 217 | } 218 | ], 219 | "metadata": { 220 | "kernelspec": { 221 | "display_name": "Python 3 (ipykernel)", 222 | "language": "python", 223 | "name": "python3" 224 | }, 225 | "language_info": { 226 | "codemirror_mode": { 227 | "name": "ipython", 228 | "version": 3 229 | }, 230 | "file_extension": ".py", 231 | "mimetype": "text/x-python", 232 | "name": "python", 233 | "nbconvert_exporter": "python", 234 | "pygments_lexer": "ipython3", 235 | "version": "3.10.9" 236 | } 237 | }, 238 | "nbformat": 4, 239 | "nbformat_minor": 5 240 | } 241 | -------------------------------------------------------------------------------- /pytorch-graph-optimization/inspecting_torch_compile.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "2c42e096-2d57-4e1d-976b-003cade9a05c", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "import torch\n", 13 | "import torch._dynamo\n", 14 | "from torch import nn" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 3, 20 | "id": "0a16cbc2-0baa-44f4-beb1-030dcb7c1433", 21 | "metadata": { 22 | "tags": [] 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "class MLP(nn.Module):\n", 27 | " def __init__(self):\n", 28 | " super().__init__()\n", 29 | " self.fc1 = nn.Linear(32, 64)\n", 30 | "\n", 31 | " def forward(self, x):\n", 32 | " x = self.fc1(x)\n", 33 | " x = torch.nn.functional.gelu(x)\n", 34 | " return x\n", 35 | "\n", 36 | "model = MLP()\n", 37 | "\n", 38 | "batch_size = 8\n", 39 | "x = torch.randn(batch_size, 32)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "id": "43846e32-0973-4e62-ab70-563ec780bcf5", 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "from torch.fx import passes, symbolic_trace\n", 50 | "model = symbolic_trace(fn)\n", 51 | "\n", 52 | "g = passes.graph_drawer.FxGraphDrawer(model, 'fn')\n", 53 | "with open(\"unoptimized_graph.svg\", \"wb\") as f:\n", 54 | " f.write(g.get_dot_graph().create_svg())" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 21, 60 | "id": "0a7036c0-cf13-44fb-ac96-79ad76fad291", 61 | "metadata": { 62 | "tags": [] 63 | }, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "Writing FX graph to file: forward.svg\n", 70 | "Writing FX graph to file: backward.svg\n" 71 | ] 72 | }, 73 | { 74 | "name": "stderr", 75 | "output_type": "stream", 76 | "text": [ 77 | "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py:1251: UserWarning: Your compiler for AOTAutograd is returning a a function that doesn't take boxed arguments. Please wrap it with functorch.compile.make_boxed_func or handle the boxed arguments yourself. See https://github.com/pytorch/pytorch/pull/83137#issuecomment-1211320670 for rationale.\n", 78 | " warnings.warn(\n", 79 | "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py:1251: UserWarning: Your compiler for AOTAutograd is returning a a function that doesn't take boxed arguments. Please wrap it with functorch.compile.make_boxed_func or handle the boxed arguments yourself. See https://github.com/pytorch/pytorch/pull/83137#issuecomment-1211320670 for rationale.\n", 80 | " warnings.warn(\n" 81 | ] 82 | } 83 | ], 84 | "source": [ 85 | "import torch._dynamo\n", 86 | "from torch._functorch.aot_autograd import aot_module_simplified\n", 87 | "from functorch.compile import compiled_function, draw_graph\n", 88 | "\n", 89 | "\n", 90 | "def toy_backend(gm, sample_inputs): \n", 91 | " def fw(gm, sample_inputs):\n", 92 | " draw_graph(gm, \"forward.svg\")\n", 93 | " return gm.forward\n", 94 | " \n", 95 | " def bw(gm, sample_inputs):\n", 96 | " draw_graph(gm, \"backward.svg\")\n", 97 | " return gm.forward\n", 98 | "\n", 99 | " # Invoke AOTAutograd\n", 100 | " return aot_module_simplified(\n", 101 | " gm,\n", 102 | " sample_inputs,\n", 103 | " fw_compiler=fw,\n", 104 | " bw_compiler=bw\n", 105 | " )\n", 106 | "\n", 107 | "def fn(x):\n", 108 | " return x**2\n", 109 | "\n", 110 | "model = fn\n", 111 | "x = torch.tensor(5., requires_grad=True)\n", 112 | "\n", 113 | "torch._dynamo.reset()\n", 114 | "cmodel = torch.compile(model, backend=toy_backend, dynamic=True)\n", 115 | "\n", 116 | "# triggers compilation of forward graph on the first run\n", 117 | "out = cmodel(x).sum().backward()" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 5, 123 | "id": "259b9c86-5f23-4f61-b178-15e71a6e2197", 124 | "metadata": { 125 | "tags": [] 126 | }, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "\n", 133 | "\n", 134 | "\n", 135 | "def forward(self, x):\n", 136 | " param = self.param\n", 137 | " add = x + param; x = param = None\n", 138 | " linear = self.linear(add); add = None\n", 139 | " clamp = linear.clamp(min = 0.0, max = 1.0); linear = None\n", 140 | " return clamp\n", 141 | " \n" 142 | ] 143 | } 144 | ], 145 | "source": [ 146 | "print(symtraced.code)" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "id": "8125d380-aaa2-4685-95a4-a035449cacfb", 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [] 156 | } 157 | ], 158 | "metadata": { 159 | "kernelspec": { 160 | "display_name": "Python 3 (ipykernel)", 161 | "language": "python", 162 | "name": "python3" 163 | }, 164 | "language_info": { 165 | "codemirror_mode": { 166 | "name": "ipython", 167 | "version": 3 168 | }, 169 | "file_extension": ".py", 170 | "mimetype": "text/x-python", 171 | "name": "python", 172 | "nbconvert_exporter": "python", 173 | "pygments_lexer": "ipython3", 174 | "version": "3.10.9" 175 | } 176 | }, 177 | "nbformat": 4, 178 | "nbformat_minor": 5 179 | } 180 | -------------------------------------------------------------------------------- /pytorch-intro-torch-compile/1-toy-benchmarks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "b33cbc31-dc42-4987-a9e2-513e31dd92de", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "import torch\n", 13 | "import time\n", 14 | "import os\n", 15 | "\n", 16 | "from torch import nn\n", 17 | "import torchvision.models as models\n", 18 | "from triton.testing import do_bench\n", 19 | "import torch._dynamo" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "id": "9d742cd2-8f4b-40d5-a51e-36dacb9a11a0", 26 | "metadata": { 27 | "tags": [] 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "torch.set_float32_matmul_precision('high')\n", 32 | "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 3, 38 | "id": "98c588e4-db0c-4c73-8000-c51edad58d89", 39 | "metadata": { 40 | "tags": [] 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "def run_benchmark(fn):\n", 45 | " exec_time, prctl20, prctl80 = do_bench(fn,warmup=100,rep=1000)\n", 46 | " print(f\"Exec time (median): {exec_time}\")\n", 47 | " print(f\"Exec time (20th percentile): {prctl20}\")\n", 48 | " print(f\"Exec time (80th percentile): {prctl80}\\n\")\n", 49 | " return exec_time" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "id": "cbe00915-2f6e-4a45-b33e-b309dd549dae", 55 | "metadata": {}, 56 | "source": [ 57 | "## 1. ResNet50 Speedup on NVIDIA A10G" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 4, 63 | "id": "306dae3d-ad53-442a-a62f-70c18d8567fe", 64 | "metadata": { 65 | "tags": [] 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "def run_batch(model, optimizer):\n", 70 | " x = torch.randn(16, 3, 224, 224).to(device)\n", 71 | " optimizer.zero_grad()\n", 72 | " out = model(x)\n", 73 | " out.sum().backward()\n", 74 | " optimizer.step()" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 5, 80 | "id": "6e3d508d-75db-406b-92e2-6bedffd1510d", 81 | "metadata": { 82 | "tags": [] 83 | }, 84 | "outputs": [], 85 | "source": [ 86 | "model = models.resnet50().to(device)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 6, 92 | "id": "9d483ad8-2882-4ce5-a491-cb464b3e3bb5", 93 | "metadata": { 94 | "tags": [] 95 | }, 96 | "outputs": [ 97 | { 98 | "name": "stdout", 99 | "output_type": "stream", 100 | "text": [ 101 | "Resnet50 Eager mode\n", 102 | "Exec time (median): 50.456573486328125\n", 103 | "Exec time (20th percentile): 50.4313850402832\n", 104 | "Exec time (80th percentile): 50.50531768798828\n", 105 | "\n", 106 | "Resnet50 Compiled defaults\n", 107 | "Exec time (median): 46.913536071777344\n", 108 | "Exec time (20th percentile): 46.89653778076172\n", 109 | "Exec time (80th percentile): 46.9372673034668\n", 110 | "\n", 111 | "speedup: 7.55%\n" 112 | ] 113 | }, 114 | { 115 | "name": "stderr", 116 | "output_type": "stream", 117 | "text": [ 118 | "Process ForkProcess-4:\n", 119 | "Process ForkProcess-1:\n", 120 | "Process ForkProcess-3:\n", 121 | "Process ForkProcess-2:\n" 122 | ] 123 | } 124 | ], 125 | "source": [ 126 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n", 127 | "\n", 128 | "# Benchmark Eager\n", 129 | "print(\"Resnet50 Eager mode\")\n", 130 | "exec_time = run_benchmark(lambda: run_batch(model, optimizer))\n", 131 | "\n", 132 | "# Benchmark torch.compile defaults\n", 133 | "print(\"Resnet50 Compiled defaults\")\n", 134 | "opt_model = torch.compile(model)\n", 135 | "opt_exec_time = run_benchmark(lambda: run_batch(opt_model, optimizer))\n", 136 | "\n", 137 | "# Print speedups\n", 138 | "print(f\"speedup: {100*(exec_time-opt_exec_time) / opt_exec_time: .2f}%\")" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "id": "fcd3add7-7c03-4559-9109-8443700febdf", 144 | "metadata": {}, 145 | "source": [ 146 | "## 2. Custom model Speedup on NVIDIA A10G" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 7, 152 | "id": "8ca833c0-d383-45ce-9f31-06b126539376", 153 | "metadata": { 154 | "tags": [] 155 | }, 156 | "outputs": [], 157 | "source": [ 158 | "class MLP(nn.Module):\n", 159 | " def __init__(self):\n", 160 | " super().__init__()\n", 161 | " self.fc1 = nn.Linear(1024, 1024)\n", 162 | " self.fc2 = nn.Linear(1024, 1024)\n", 163 | " \n", 164 | " def forward(self, x):\n", 165 | " x = self.fc1(x).relu() ** 2\n", 166 | " return self.fc2(x).relu() ** 2" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 8, 172 | "id": "bc293b4a-6634-4e66-95f0-1675ab363114", 173 | "metadata": { 174 | "tags": [] 175 | }, 176 | "outputs": [], 177 | "source": [ 178 | "model = MLP().to(device)\n", 179 | "x = torch.randn(1024, 1024).to(device)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 9, 185 | "id": "020901c4-9054-4652-acaf-0232a31d0011", 186 | "metadata": { 187 | "tags": [] 188 | }, 189 | "outputs": [ 190 | { 191 | "name": "stdout", 192 | "output_type": "stream", 193 | "text": [ 194 | "Exec time (median): 0.7147520184516907\n", 195 | "Exec time (20th percentile): 0.7127040028572083\n", 196 | "Exec time (80th percentile): 0.7157760262489319\n", 197 | "\n", 198 | "Exec time (median): 0.6010879874229431\n", 199 | "Exec time (20th percentile): 0.6000639796257019\n", 200 | "Exec time (80th percentile): 0.6021119952201843\n", 201 | "\n", 202 | "speedup: 18.91%\n" 203 | ] 204 | } 205 | ], 206 | "source": [ 207 | "# Benchmark Eager\n", 208 | "exec_time = run_benchmark(lambda: model(x).sum().backward())\n", 209 | "\n", 210 | "torch._dynamo.reset()\n", 211 | "# Benchmark torch.compile defaults\n", 212 | "cmodel = torch.compile(model, backend='inductor')\n", 213 | "opt_exec_time = run_benchmark(lambda: cmodel(x).sum().backward())\n", 214 | "\n", 215 | "# Print speedups\n", 216 | "print(f\"speedup: {100*(exec_time-opt_exec_time) / opt_exec_time: .2f}%\")" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "id": "49a00c44-c096-4ae8-afe8-7937db196933", 222 | "metadata": {}, 223 | "source": [ 224 | "## 3. HuggingFace model Speedup on NVIDIA A10G" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "id": "d7a4eaae-8bc3-4a21-8147-a2d75e5fba3a", 231 | "metadata": { 232 | "tags": [] 233 | }, 234 | "outputs": [], 235 | "source": [ 236 | "from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC\n", 237 | "from datasets import load_dataset\n", 238 | "\n", 239 | "def run_inference(model, input_values):\n", 240 | " \n", 241 | " # retrieve logits\n", 242 | " logits = model(input_values).logits\n", 243 | " \n", 244 | " # take argmax and decode\n", 245 | " predicted_ids = torch.argmax(logits, dim=-1)\n", 246 | " transcription = processor.batch_decode(predicted_ids)" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 11, 252 | "id": "76d00a8f-4503-4fa7-90cd-263f693097da", 253 | "metadata": { 254 | "tags": [] 255 | }, 256 | "outputs": [ 257 | { 258 | "data": { 259 | "application/vnd.jupyter.widget-view+json": { 260 | "model_id": "7baf2003e8ab49cca9c52c88ba8e14c5", 261 | "version_major": 2, 262 | "version_minor": 0 263 | }, 264 | "text/plain": [ 265 | "Downloading (…)rocessor_config.json: 0%| | 0.00/158 [00:00\n", 168 | " print(\"AOTAutograd produced a fx Graph in Aten IR:\")\n", 169 | " gm.print_readable()\n", 170 | " return gm.forward\n", 171 | "\n", 172 | " # Invoke AOTAutograd\n", 173 | " return aot_module_simplified(\n", 174 | " gm,\n", 175 | " sample_inputs,\n", 176 | " fw_compiler=my_compiler\n", 177 | " )\n", 178 | "\n", 179 | "torch._dynamo.reset()\n", 180 | "cmodel = torch.compile(model, backend=toy_backend, dynamic=True)\n", 181 | "\n", 182 | "# triggers compilation of forward graph on the first run\n", 183 | "out = cmodel(input)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 6, 189 | "id": "199f5ca0-0743-4f91-859d-c495a723280c", 190 | "metadata": { 191 | "tags": [] 192 | }, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "Decomposed fx Graph in Aten IR:\n", 199 | "class GraphModule(torch.nn.Module):\n", 200 | " def forward(self, primals_1: f32[64, 32], primals_2: f32[64], primals_3: f32[s0, 32]):\n", 201 | " # File: /tmp/ipykernel_8670/1490842273.py:7, code: x = self.fc1(x)\n", 202 | " permute: f32[32, 64] = torch.ops.aten.permute.default(primals_1, [1, 0]); primals_1 = None\n", 203 | " addmm: f32[s0, 64] = torch.ops.aten.addmm.default(primals_2, primals_3, permute); primals_2 = permute = None\n", 204 | " \n", 205 | " # File: /tmp/ipykernel_8670/1490842273.py:8, code: x = torch.nn.functional.gelu(x)\n", 206 | " mul: f32[s0, 64] = torch.ops.aten.mul.Tensor(addmm, 0.5)\n", 207 | " mul_1: f32[s0, 64] = torch.ops.aten.mul.Tensor(addmm, 0.7071067811865476)\n", 208 | " erf: f32[s0, 64] = torch.ops.aten.erf.default(mul_1); mul_1 = None\n", 209 | " add: f32[s0, 64] = torch.ops.aten.add.Tensor(erf, 1); erf = None\n", 210 | " mul_2: f32[s0, 64] = torch.ops.aten.mul.Tensor(mul, add); mul = add = None\n", 211 | " return [mul_2, addmm, primals_3]\n", 212 | " \n" 213 | ] 214 | }, 215 | { 216 | "name": "stderr", 217 | "output_type": "stream", 218 | "text": [ 219 | "/opt/conda/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py:1251: UserWarning: Your compiler for AOTAutograd is returning a function that doesn't take boxed arguments. Please wrap it with functorch.compile.make_boxed_func or handle the boxed arguments yourself. See https://github.com/pytorch/pytorch/pull/83137#issuecomment-1211320670 for rationale.\n", 220 | " warnings.warn(\n" 221 | ] 222 | } 223 | ], 224 | "source": [ 225 | "from torch._inductor.decomposition import decompositions as default_decompositions\n", 226 | "\n", 227 | "decompositions = default_decompositions.copy()\n", 228 | "\n", 229 | "def toy_backend(gm, sample_inputs):\n", 230 | " def my_compiler(gm, sample_inputs):\n", 231 | " # \n", 232 | " print(\"Decomposed fx Graph in Aten IR:\")\n", 233 | " gm.print_readable()\n", 234 | " return gm\n", 235 | "\n", 236 | " # Invoke AOTAutograd\n", 237 | " return aot_module_simplified(\n", 238 | " gm,\n", 239 | " sample_inputs,\n", 240 | " decompositions=decompositions,\n", 241 | " fw_compiler=my_compiler\n", 242 | " )\n", 243 | "\n", 244 | "torch._dynamo.reset()\n", 245 | "cmodel = torch.compile(model, backend=toy_backend, dynamic=True)\n", 246 | "\n", 247 | "# triggers compilation of forward graph on the first run\n", 248 | "out = cmodel(input)" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "id": "08c92c33-0803-4ce1-906c-e00faa1a620f", 254 | "metadata": {}, 255 | "source": [ 256 | "### Prims IR (https://pytorch.org/docs/master/ir.html#prims-ir)\n", 257 | "\n", 258 | "* Explicit type promotion and broadcasting\n", 259 | "* prims.convert_element_type\n", 260 | "* prims.broadcast_in_dim\n", 261 | "* For backends with powerful compiler that can reclaim the performance by fusion, e.g. nvFuser" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 7, 267 | "id": "236f9011-fca0-45d9-a832-4f07962d8ef4", 268 | "metadata": { 269 | "tags": [] 270 | }, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | "Further decomposed fx Graph in Prims IR:\n", 277 | "class (torch.nn.Module):\n", 278 | " def forward(self, arg0_1: f32[3], arg1_1: f16[3, 3]):\n", 279 | " # File: /tmp/ipykernel_8670/2178452752.py:7, code: return a + b\n", 280 | " _to_copy: f32[3, 3] = torch.ops.aten._to_copy.default(arg1_1, dtype = torch.float32); arg1_1 = None\n", 281 | " broadcast_in_dim: f32[3, 3] = torch.ops.prims.broadcast_in_dim.default(arg0_1, [3, 3], [1]); arg0_1 = None\n", 282 | " add: f32[3, 3] = torch.ops.prims.add.default(broadcast_in_dim, _to_copy); broadcast_in_dim = _to_copy = None\n", 283 | " return (add,)\n", 284 | " \n" 285 | ] 286 | } 287 | ], 288 | "source": [ 289 | "prims_decomp = torch._decomp.get_decompositions([\n", 290 | " torch.ops.aten.add,\n", 291 | " torch.ops.aten.expand.default,\n", 292 | "])\n", 293 | "\n", 294 | "def fn(a, b):\n", 295 | " return a + b\n", 296 | "\n", 297 | "def toy_backend(gm, sample_inputs):\n", 298 | " def my_compiler(gm, sample_inputs):\n", 299 | " # \n", 300 | " print(\"Further decomposed fx Graph in Prims IR:\")\n", 301 | " gm.print_readable()\n", 302 | " return gm\n", 303 | "\n", 304 | " # Invoke AOTAutograd\n", 305 | " return aot_module_simplified(\n", 306 | " gm,\n", 307 | " sample_inputs,\n", 308 | " decompositions=prims_decomp,\n", 309 | " fw_compiler=my_compiler\n", 310 | " )\n", 311 | "\n", 312 | "torch._dynamo.reset()\n", 313 | "fn = torch.compile(backend=toy_backend)(fn)\n", 314 | "out = fn(torch.rand(3, dtype=torch.float), torch.rand(3, 3, dtype=torch.half))" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": null, 320 | "id": "0c70a43d-7a90-409e-862c-7169c6fd088f", 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [] 324 | } 325 | ], 326 | "metadata": { 327 | "kernelspec": { 328 | "display_name": "Python 3 (ipykernel)", 329 | "language": "python", 330 | "name": "python3" 331 | }, 332 | "language_info": { 333 | "codemirror_mode": { 334 | "name": "ipython", 335 | "version": 3 336 | }, 337 | "file_extension": ".py", 338 | "mimetype": "text/x-python", 339 | "name": "python", 340 | "nbconvert_exporter": "python", 341 | "pygments_lexer": "ipython3", 342 | "version": "3.10.9" 343 | } 344 | }, 345 | "nbformat": 4, 346 | "nbformat_minor": 5 347 | } 348 | -------------------------------------------------------------------------------- /pytorch-intro-torch-compile/4-nn-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "dd7ed31a-9899-4489-a7a1-bc2bb1d90ee7", 7 | "metadata": { 8 | "tags": [] 9 | }, 10 | "outputs": [], 11 | "source": [ 12 | "import argparse\n", 13 | "import json\n", 14 | "import logging\n", 15 | "import os\n", 16 | "import sys\n", 17 | "\n", 18 | "import torch\n", 19 | "import torch.nn as nn\n", 20 | "import torch.nn.functional as F\n", 21 | "import torch.optim as optim\n", 22 | "import torch.utils.data\n", 23 | "import torchvision\n", 24 | "\n", 25 | "from torchvision import datasets, transforms\n", 26 | "\n", 27 | "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n", 28 | "torch.set_float32_matmul_precision('high') #Uses TF32 when available" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "id": "37effd54-1f4c-42b2-bd08-1d524908fdc3", 35 | "metadata": { 36 | "tags": [] 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "def _get_model():\n", 41 | " class Net(nn.Module):\n", 42 | " def __init__(self):\n", 43 | " super(Net, self).__init__()\n", 44 | " self.conv1 = nn.Conv2d(3, 6, 5)\n", 45 | " self.pool = nn.MaxPool2d(2, 2)\n", 46 | " self.conv2 = nn.Conv2d(6, 16, 5)\n", 47 | " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", 48 | " self.fc2 = nn.Linear(120, 84)\n", 49 | " self.fc3 = nn.Linear(84, 10)\n", 50 | "\n", 51 | " def forward(self, x):\n", 52 | " x = self.pool(F.relu(self.conv1(x)))\n", 53 | " x = self.pool(F.relu(self.conv2(x)))\n", 54 | " x = x.view(-1, 16 * 5 * 5)\n", 55 | " x = F.relu(self.fc1(x))\n", 56 | " x = F.relu(self.fc2(x))\n", 57 | " x = self.fc3(x)\n", 58 | " return x\n", 59 | " return Net()" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 3, 65 | "id": "55a1d996-acc3-45ac-8113-3344bc7ddc28", 66 | "metadata": { 67 | "tags": [] 68 | }, 69 | "outputs": [], 70 | "source": [ 71 | "# Define data augmentation\n", 72 | "def _get_transforms():\n", 73 | " transform = transforms.Compose([\n", 74 | " transforms.RandomCrop(32, padding=4),\n", 75 | " transforms.RandomHorizontalFlip(),\n", 76 | " transforms.ToTensor(),\n", 77 | " transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n", 78 | " ])\n", 79 | " return transform" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 4, 85 | "id": "f8cc60fa-faba-40d6-a2cc-7d5f29e682c6", 86 | "metadata": { 87 | "tags": [] 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "def _get_dataloaders(batch_size):\n", 92 | " trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n", 93 | " download=True, transform=_get_transforms())\n", 94 | " testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n", 95 | " download=True, transform=_get_transforms())\n", 96 | " trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n", 97 | " shuffle=True)\n", 98 | " testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n", 99 | " shuffle=False)\n", 100 | " return trainloader, testloader" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 5, 106 | "id": "c7bda30f-df68-4a3b-925a-3153ba1f6809", 107 | "metadata": { 108 | "tags": [] 109 | }, 110 | "outputs": [], 111 | "source": [ 112 | "def test(model, test_loader, device):\n", 113 | " test_loss = 0\n", 114 | " correct = 0\n", 115 | " with torch.no_grad():\n", 116 | " for data, target in test_loader:\n", 117 | " data, target = data.to(device), target.to(device)\n", 118 | " output = model(data)\n", 119 | " test_loss += F.nll_loss(output, target, reduction='mean').item() # sum up batch loss\n", 120 | " pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n", 121 | " correct += pred.eq(target.view_as(pred)).sum().item()\n", 122 | "\n", 123 | " test_loss /= len(test_loader.dataset)\n", 124 | " print(f\"Test set: Average loss: {test_loss}, Accuracy: {correct / len(test_loader.dataset)}\\n\")" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 6, 130 | "id": "96633f85-1875-4dc9-a2ee-9784083e63c9", 131 | "metadata": { 132 | "scrolled": true, 133 | "tags": [] 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "import time\n", 138 | "def train(model, batch_size, epochs):\n", 139 | " torch.manual_seed(0)\n", 140 | " lr = 0.01\n", 141 | " momentum=0.9\n", 142 | " train_loader, test_loader = _get_dataloaders(batch_size)\n", 143 | "\n", 144 | " criterion = nn.CrossEntropyLoss()\n", 145 | " optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)\n", 146 | "\n", 147 | " for epoch in range(1, epochs + 1):\n", 148 | " start_time = time.time()\n", 149 | " for batch_idx, (data, target) in enumerate(train_loader, 1):\n", 150 | " data, target = data.to(device), target.to(device)\n", 151 | "\n", 152 | " # zero the parameter gradients\n", 153 | " optimizer.zero_grad()\n", 154 | "\n", 155 | " # forward + backward + optimize\n", 156 | " output = model(data)\n", 157 | " loss = criterion(output, target)\n", 158 | " loss.backward()\n", 159 | " optimizer.step()\n", 160 | "\n", 161 | " print(f\"Train Epoch: {epoch} Epoch time: {time.time()-start_time:0.4f} Loss: {loss.item()}\")\n", 162 | " test(model, test_loader, device)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": 7, 168 | "id": "4c4c51e7-a0ac-4a7f-88a3-9505bcd8f248", 169 | "metadata": { 170 | "scrolled": true, 171 | "tags": [] 172 | }, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "Train in eager mode on CIFAR-10\n", 179 | "Files already downloaded and verified\n", 180 | "Files already downloaded and verified\n", 181 | "Train Epoch: 1 Epoch time: 18.4492 Loss: 1.5229963064193726\n", 182 | "Test set: Average loss: -0.11479381905794143, Accuracy: 0.3574\n", 183 | "\n", 184 | "Train Epoch: 2 Epoch time: 17.1039 Loss: 1.255014419555664\n", 185 | "Test set: Average loss: -0.12258006700873375, Accuracy: 0.4207\n", 186 | "\n", 187 | "Train Epoch: 3 Epoch time: 17.0540 Loss: 1.2103943824768066\n", 188 | "Test set: Average loss: -0.14219269587993622, Accuracy: 0.4306\n", 189 | "\n", 190 | "Train Epoch: 4 Epoch time: 17.1195 Loss: 1.6141940355300903\n", 191 | "Test set: Average loss: -0.1223116991698742, Accuracy: 0.4078\n", 192 | "\n", 193 | "Train Epoch: 5 Epoch time: 17.1593 Loss: 1.7167662382125854\n", 194 | "Test set: Average loss: -0.1371232023358345, Accuracy: 0.427\n", 195 | "\n", 196 | "Train Epoch: 6 Epoch time: 17.0450 Loss: 1.6442832946777344\n", 197 | "Test set: Average loss: -0.11765576581060887, Accuracy: 0.4195\n", 198 | "\n", 199 | "Train Epoch: 7 Epoch time: 17.2778 Loss: 1.1586203575134277\n", 200 | "Test set: Average loss: -0.12454334303736686, Accuracy: 0.4175\n", 201 | "\n", 202 | "Train Epoch: 8 Epoch time: 17.1429 Loss: 2.3159003257751465\n", 203 | "Test set: Average loss: -0.14869595084786416, Accuracy: 0.4542\n", 204 | "\n", 205 | "Train Epoch: 9 Epoch time: 17.0725 Loss: 1.63433039188385\n", 206 | "Test set: Average loss: -0.13449764105677606, Accuracy: 0.4416\n", 207 | "\n", 208 | "Train Epoch: 10 Epoch time: 17.1152 Loss: 0.9652916193008423\n", 209 | "Test set: Average loss: -0.14638293540477754, Accuracy: 0.4566\n", 210 | "\n", 211 | "Train Epoch: 11 Epoch time: 17.0312 Loss: 1.4068830013275146\n", 212 | "Test set: Average loss: -0.13832889040112495, Accuracy: 0.4099\n", 213 | "\n", 214 | "Train Epoch: 12 Epoch time: 17.1650 Loss: 1.6365031003952026\n", 215 | "Test set: Average loss: -0.1226353962123394, Accuracy: 0.427\n", 216 | "\n", 217 | "Train Epoch: 13 Epoch time: 17.1685 Loss: 1.2202277183532715\n", 218 | "Test set: Average loss: -0.16288705285787583, Accuracy: 0.4464\n", 219 | "\n", 220 | "Train Epoch: 14 Epoch time: 17.1915 Loss: 1.5230098962783813\n", 221 | "Test set: Average loss: -0.13382687171697616, Accuracy: 0.4507\n", 222 | "\n", 223 | "Train Epoch: 15 Epoch time: 17.1395 Loss: 1.6405322551727295\n", 224 | "Test set: Average loss: -0.11657781254947186, Accuracy: 0.418\n", 225 | "\n", 226 | "Train Epoch: 16 Epoch time: 17.1124 Loss: 1.435512661933899\n", 227 | "Test set: Average loss: -0.16411116136312484, Accuracy: 0.4536\n", 228 | "\n", 229 | "Train Epoch: 17 Epoch time: 17.0854 Loss: 1.3877995014190674\n", 230 | "Test set: Average loss: -0.13008506304621698, Accuracy: 0.4505\n", 231 | "\n", 232 | "Train Epoch: 18 Epoch time: 17.0801 Loss: 1.3181973695755005\n", 233 | "Test set: Average loss: -0.12966735948324204, Accuracy: 0.4444\n", 234 | "\n", 235 | "Train Epoch: 19 Epoch time: 17.1122 Loss: 1.3785901069641113\n", 236 | "Test set: Average loss: -0.16746452817320823, Accuracy: 0.4708\n", 237 | "\n", 238 | "Train Epoch: 20 Epoch time: 17.0873 Loss: 2.601522922515869\n", 239 | "Test set: Average loss: -0.1661388152360916, Accuracy: 0.455\n", 240 | "\n", 241 | "Train Epoch: 21 Epoch time: 17.1412 Loss: 1.387449026107788\n", 242 | "Test set: Average loss: -0.13323016968667506, Accuracy: 0.4078\n", 243 | "\n", 244 | "Train Epoch: 22 Epoch time: 17.0955 Loss: 0.9796938300132751\n", 245 | "Test set: Average loss: -0.12620635756850243, Accuracy: 0.4348\n", 246 | "\n", 247 | "Train Epoch: 23 Epoch time: 17.1453 Loss: 1.7141715288162231\n", 248 | "Test set: Average loss: -0.14345559119582177, Accuracy: 0.4657\n", 249 | "\n", 250 | "Train Epoch: 24 Epoch time: 17.1091 Loss: 1.4967659711837769\n", 251 | "Test set: Average loss: -0.12276398456394673, Accuracy: 0.4001\n", 252 | "\n", 253 | "Train Epoch: 25 Epoch time: 17.0620 Loss: 1.9967564344406128\n", 254 | "Test set: Average loss: -0.15137818831205369, Accuracy: 0.4591\n", 255 | "\n", 256 | "Train Epoch: 26 Epoch time: 17.0726 Loss: 1.2827167510986328\n", 257 | "Test set: Average loss: -0.14104595474600792, Accuracy: 0.4339\n", 258 | "\n", 259 | "Train Epoch: 27 Epoch time: 17.1130 Loss: 1.6564006805419922\n", 260 | "Test set: Average loss: -0.15558898612856864, Accuracy: 0.457\n", 261 | "\n", 262 | "Train Epoch: 28 Epoch time: 17.0406 Loss: 1.5395925045013428\n", 263 | "Test set: Average loss: -0.15071935195326805, Accuracy: 0.4298\n", 264 | "\n", 265 | "Train Epoch: 29 Epoch time: 17.0579 Loss: 2.140176773071289\n", 266 | "Test set: Average loss: -0.15748710156083107, Accuracy: 0.4236\n", 267 | "\n", 268 | "Train Epoch: 30 Epoch time: 17.0842 Loss: 1.7239465713500977\n", 269 | "Test set: Average loss: -0.14595504192709924, Accuracy: 0.4115\n", 270 | "\n", 271 | "Train Epoch: 31 Epoch time: 17.1039 Loss: 1.7769050598144531\n", 272 | "Test set: Average loss: -0.1290455259978771, Accuracy: 0.3998\n", 273 | "\n", 274 | "Train Epoch: 32 Epoch time: 17.1451 Loss: 1.662497639656067\n", 275 | "Test set: Average loss: -0.15571664265990257, Accuracy: 0.4357\n", 276 | "\n", 277 | "Train Epoch: 33 Epoch time: 17.1315 Loss: 1.3852906227111816\n", 278 | "Test set: Average loss: -0.1613605972111225, Accuracy: 0.4172\n", 279 | "\n", 280 | "Train Epoch: 34 Epoch time: 17.0798 Loss: 1.7368806600570679\n", 281 | "Test set: Average loss: -0.12401523492336274, Accuracy: 0.3968\n", 282 | "\n", 283 | "Train Epoch: 35 Epoch time: 17.1884 Loss: 1.463071584701538\n", 284 | "Test set: Average loss: -0.13350237726569175, Accuracy: 0.4211\n", 285 | "\n", 286 | "Train Epoch: 36 Epoch time: 17.0499 Loss: 1.222204327583313\n", 287 | "Test set: Average loss: -0.16071371198892592, Accuracy: 0.4386\n", 288 | "\n", 289 | "Train Epoch: 37 Epoch time: 17.0795 Loss: 1.59268319606781\n", 290 | "Test set: Average loss: -0.1448448682665825, Accuracy: 0.4142\n", 291 | "\n", 292 | "Train Epoch: 38 Epoch time: 17.0917 Loss: 1.8278884887695312\n", 293 | "Test set: Average loss: -0.13484705570042133, Accuracy: 0.4107\n", 294 | "\n", 295 | "Train Epoch: 39 Epoch time: 17.1323 Loss: 1.2747076749801636\n", 296 | "Test set: Average loss: -0.14489886118769646, Accuracy: 0.3949\n", 297 | "\n", 298 | "Train Epoch: 40 Epoch time: 17.1330 Loss: 1.6077611446380615\n", 299 | "Test set: Average loss: -0.14588061140179634, Accuracy: 0.4289\n", 300 | "\n", 301 | "Train Epoch: 41 Epoch time: 17.2133 Loss: 1.6331448554992676\n", 302 | "Test set: Average loss: -0.09960648607611657, Accuracy: 0.3682\n", 303 | "\n", 304 | "Train Epoch: 42 Epoch time: 17.1127 Loss: 1.2302451133728027\n", 305 | "Test set: Average loss: -0.14067820476591586, Accuracy: 0.4283\n", 306 | "\n", 307 | "Train Epoch: 43 Epoch time: 17.1447 Loss: 1.6978567838668823\n", 308 | "Test set: Average loss: -0.16264084021151065, Accuracy: 0.4108\n", 309 | "\n", 310 | "Train Epoch: 44 Epoch time: 17.0487 Loss: 1.2845810651779175\n", 311 | "Test set: Average loss: -0.13672254542708398, Accuracy: 0.444\n", 312 | "\n", 313 | "Train Epoch: 45 Epoch time: 17.1562 Loss: 1.1537847518920898\n", 314 | "Test set: Average loss: -0.14150477679371834, Accuracy: 0.4273\n", 315 | "\n", 316 | "Train Epoch: 46 Epoch time: 17.1326 Loss: 1.6309105157852173\n", 317 | "Test set: Average loss: -0.13888773401975632, Accuracy: 0.3973\n", 318 | "\n", 319 | "Train Epoch: 47 Epoch time: 17.1038 Loss: 1.398644208908081\n", 320 | "Test set: Average loss: -0.15300521407723428, Accuracy: 0.4311\n", 321 | "\n", 322 | "Train Epoch: 48 Epoch time: 17.1457 Loss: 1.8404948711395264\n", 323 | "Test set: Average loss: -0.1579221700131893, Accuracy: 0.4233\n", 324 | "\n", 325 | "Train Epoch: 49 Epoch time: 17.1388 Loss: 1.3540527820587158\n", 326 | "Test set: Average loss: -0.1460476183593273, Accuracy: 0.4527\n", 327 | "\n", 328 | "Train Epoch: 50 Epoch time: 17.1742 Loss: 1.765850305557251\n", 329 | "Test set: Average loss: -0.12122882234454155, Accuracy: 0.4141\n", 330 | "\n", 331 | "CPU times: user 16min 39s, sys: 6.23 s, total: 16min 45s\n", 332 | "Wall time: 16min 47s\n" 333 | ] 334 | } 335 | ], 336 | "source": [ 337 | "%%time\n", 338 | "print(\"Train in eager mode on CIFAR-10\")\n", 339 | "model = _get_model().to(device)\n", 340 | "\n", 341 | "train(model, batch_size=16, epochs=50)" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 11, 347 | "id": "f5330b33-eb7b-4a18-939a-44564dcf4441", 348 | "metadata": { 349 | "tags": [] 350 | }, 351 | "outputs": [ 352 | { 353 | "name": "stdout", 354 | "output_type": "stream", 355 | "text": [ 356 | "Generating forward and backward graphs\n", 357 | "CPU times: user 1.01 s, sys: 12 ms, total: 1.02 s\n", 358 | "Wall time: 1.02 s\n" 359 | ] 360 | } 361 | ], 362 | "source": [ 363 | "%%time\n", 364 | "import torch._inductor.config\n", 365 | "model = _get_model().to(device)\n", 366 | "model = torch.compile(model, backend=\"inductor\",\n", 367 | " mode=\"max-autotune\")\n", 368 | "\n", 369 | "randinput = torch.randn(16,3,32,32).to(device)\n", 370 | "randoutput = torch.randn(16,10).to(device)\n", 371 | "\n", 372 | "print('Generating forward and backward graphs')\n", 373 | "out = model(randinput)\n", 374 | "nn.CrossEntropyLoss()(out, randoutput).backward()" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": 12, 380 | "id": "b5ef0dbd-2d0c-4f4c-bd93-b2d4c6765829", 381 | "metadata": { 382 | "scrolled": true, 383 | "tags": [] 384 | }, 385 | "outputs": [ 386 | { 387 | "name": "stdout", 388 | "output_type": "stream", 389 | "text": [ 390 | "Train in compiled mode on CIFAR-10\n", 391 | "Files already downloaded and verified\n", 392 | "Files already downloaded and verified\n", 393 | "Train Epoch: 1 Epoch time: 16.8979 Loss: 1.1281245946884155\n", 394 | "Test set: Average loss: -0.10141951135396958, Accuracy: 0.3753\n", 395 | "\n", 396 | "Train Epoch: 2 Epoch time: 16.8175 Loss: 1.5165073871612549\n", 397 | "Test set: Average loss: -0.11379602966904641, Accuracy: 0.4128\n", 398 | "\n", 399 | "Train Epoch: 3 Epoch time: 16.8435 Loss: 1.3197925090789795\n", 400 | "Test set: Average loss: -0.14007682649493217, Accuracy: 0.4195\n", 401 | "\n", 402 | "Train Epoch: 4 Epoch time: 17.1099 Loss: 1.7537651062011719\n", 403 | "Test set: Average loss: -0.12210166681408882, Accuracy: 0.444\n", 404 | "\n", 405 | "Train Epoch: 5 Epoch time: 16.9783 Loss: 1.5357666015625\n", 406 | "Test set: Average loss: -0.15047866759300232, Accuracy: 0.4326\n", 407 | "\n", 408 | "Train Epoch: 6 Epoch time: 16.7821 Loss: 1.6223543882369995\n", 409 | "Test set: Average loss: -0.143238749986887, Accuracy: 0.4263\n", 410 | "\n", 411 | "Train Epoch: 7 Epoch time: 16.8694 Loss: 1.7723138332366943\n", 412 | "Test set: Average loss: -0.12494028667807579, Accuracy: 0.4065\n", 413 | "\n", 414 | "Train Epoch: 8 Epoch time: 16.8882 Loss: 1.8077551126480103\n", 415 | "Test set: Average loss: -0.1637685509085655, Accuracy: 0.4561\n", 416 | "\n", 417 | "Train Epoch: 9 Epoch time: 16.8302 Loss: 1.4599609375\n", 418 | "Test set: Average loss: -0.1516111361026764, Accuracy: 0.4451\n", 419 | "\n", 420 | "Train Epoch: 10 Epoch time: 16.8638 Loss: 1.1293983459472656\n", 421 | "Test set: Average loss: -0.16525690048933028, Accuracy: 0.4697\n", 422 | "\n", 423 | "Train Epoch: 11 Epoch time: 16.8586 Loss: 0.9164922833442688\n", 424 | "Test set: Average loss: -0.15961521873474122, Accuracy: 0.4466\n", 425 | "\n", 426 | "Train Epoch: 12 Epoch time: 16.7826 Loss: 2.045623779296875\n", 427 | "Test set: Average loss: -0.10865869052410125, Accuracy: 0.4195\n", 428 | "\n", 429 | "Train Epoch: 13 Epoch time: 16.8195 Loss: 1.2999463081359863\n", 430 | "Test set: Average loss: -0.17005301181674004, Accuracy: 0.4713\n", 431 | "\n", 432 | "Train Epoch: 14 Epoch time: 16.8872 Loss: 1.5532177686691284\n", 433 | "Test set: Average loss: -0.17997907676696778, Accuracy: 0.4673\n", 434 | "\n", 435 | "Train Epoch: 15 Epoch time: 16.7638 Loss: 1.6674301624298096\n", 436 | "Test set: Average loss: -0.13775004784464837, Accuracy: 0.4458\n", 437 | "\n", 438 | "Train Epoch: 16 Epoch time: 16.8711 Loss: 1.7492315769195557\n", 439 | "Test set: Average loss: -0.17715689504146576, Accuracy: 0.4702\n", 440 | "\n", 441 | "Train Epoch: 17 Epoch time: 16.8545 Loss: 1.4167125225067139\n", 442 | "Test set: Average loss: -0.13974063433408737, Accuracy: 0.4377\n", 443 | "\n", 444 | "Train Epoch: 18 Epoch time: 16.8798 Loss: 1.9101390838623047\n", 445 | "Test set: Average loss: -0.1401785773575306, Accuracy: 0.4311\n", 446 | "\n", 447 | "Train Epoch: 19 Epoch time: 16.8081 Loss: 1.99967622756958\n", 448 | "Test set: Average loss: -0.1516777255654335, Accuracy: 0.4445\n", 449 | "\n", 450 | "Train Epoch: 20 Epoch time: 16.7739 Loss: 2.2190351486206055\n", 451 | "Test set: Average loss: -0.1643785246193409, Accuracy: 0.4545\n", 452 | "\n", 453 | "Train Epoch: 21 Epoch time: 17.1348 Loss: 1.7950257062911987\n", 454 | "Test set: Average loss: -0.17522918835878373, Accuracy: 0.4574\n", 455 | "\n", 456 | "Train Epoch: 22 Epoch time: 16.9389 Loss: 1.5520081520080566\n", 457 | "Test set: Average loss: -0.15325675513744355, Accuracy: 0.4569\n", 458 | "\n", 459 | "Train Epoch: 23 Epoch time: 16.8428 Loss: 1.663689374923706\n", 460 | "Test set: Average loss: -0.17498237799406052, Accuracy: 0.4514\n", 461 | "\n", 462 | "Train Epoch: 24 Epoch time: 16.8972 Loss: 2.0213215351104736\n", 463 | "Test set: Average loss: -0.12325149633288383, Accuracy: 0.4142\n", 464 | "\n", 465 | "Train Epoch: 25 Epoch time: 16.8106 Loss: 2.0409820079803467\n", 466 | "Test set: Average loss: -0.12809701528549194, Accuracy: 0.4128\n", 467 | "\n", 468 | "Train Epoch: 26 Epoch time: 16.7998 Loss: 1.821916103363037\n", 469 | "Test set: Average loss: -0.13387244796156883, Accuracy: 0.4204\n", 470 | "\n", 471 | "Train Epoch: 27 Epoch time: 16.8039 Loss: 1.3090702295303345\n", 472 | "Test set: Average loss: -0.15662841452360154, Accuracy: 0.4604\n", 473 | "\n", 474 | "Train Epoch: 28 Epoch time: 16.9014 Loss: 1.2767252922058105\n", 475 | "Test set: Average loss: -0.17367174760699272, Accuracy: 0.4426\n", 476 | "\n", 477 | "Train Epoch: 29 Epoch time: 16.7807 Loss: 1.9910852909088135\n", 478 | "Test set: Average loss: -0.138916358846426, Accuracy: 0.3752\n", 479 | "\n", 480 | "Train Epoch: 30 Epoch time: 16.7389 Loss: 1.2423659563064575\n", 481 | "Test set: Average loss: -0.14196027348041534, Accuracy: 0.4424\n", 482 | "\n", 483 | "Train Epoch: 31 Epoch time: 16.8019 Loss: 1.6152451038360596\n", 484 | "Test set: Average loss: -0.15072507199048996, Accuracy: 0.4381\n", 485 | "\n", 486 | "Train Epoch: 32 Epoch time: 16.8864 Loss: 1.6289972066879272\n", 487 | "Test set: Average loss: -0.11902655415534973, Accuracy: 0.4208\n", 488 | "\n", 489 | "Train Epoch: 33 Epoch time: 16.7390 Loss: 1.3837391138076782\n", 490 | "Test set: Average loss: -0.15387935359477997, Accuracy: 0.4218\n", 491 | "\n", 492 | "Train Epoch: 34 Epoch time: 16.7903 Loss: 1.516900658607483\n", 493 | "Test set: Average loss: -0.1460586778521538, Accuracy: 0.4429\n", 494 | "\n", 495 | "Train Epoch: 35 Epoch time: 16.7645 Loss: 1.8662484884262085\n", 496 | "Test set: Average loss: -0.1500231746673584, Accuracy: 0.4308\n", 497 | "\n", 498 | "Train Epoch: 36 Epoch time: 16.7263 Loss: 1.570525884628296\n", 499 | "Test set: Average loss: -0.17256923723816872, Accuracy: 0.4522\n", 500 | "\n", 501 | "Train Epoch: 37 Epoch time: 16.7440 Loss: 1.7509129047393799\n", 502 | "Test set: Average loss: -0.1331960899591446, Accuracy: 0.4025\n", 503 | "\n", 504 | "Train Epoch: 38 Epoch time: 16.6792 Loss: 1.6465048789978027\n", 505 | "Test set: Average loss: -0.17207880718111992, Accuracy: 0.463\n", 506 | "\n", 507 | "Train Epoch: 39 Epoch time: 16.7337 Loss: 1.4757962226867676\n", 508 | "Test set: Average loss: -0.1252164648413658, Accuracy: 0.435\n", 509 | "\n", 510 | "Train Epoch: 40 Epoch time: 16.7789 Loss: 2.02087140083313\n", 511 | "Test set: Average loss: -0.12817251023054124, Accuracy: 0.4488\n", 512 | "\n", 513 | "Train Epoch: 41 Epoch time: 16.7497 Loss: 1.4727895259857178\n", 514 | "Test set: Average loss: -0.1616751208484173, Accuracy: 0.4522\n", 515 | "\n", 516 | "Train Epoch: 42 Epoch time: 16.8264 Loss: 1.214515209197998\n", 517 | "Test set: Average loss: -0.09364713019430637, Accuracy: 0.3973\n", 518 | "\n", 519 | "Train Epoch: 43 Epoch time: 16.6972 Loss: 1.874068021774292\n", 520 | "Test set: Average loss: -0.17249812202453613, Accuracy: 0.4564\n", 521 | "\n", 522 | "Train Epoch: 44 Epoch time: 16.9201 Loss: 1.4941129684448242\n", 523 | "Test set: Average loss: -0.14445521401762962, Accuracy: 0.4388\n", 524 | "\n", 525 | "Train Epoch: 45 Epoch time: 16.7735 Loss: 1.3479551076889038\n", 526 | "Test set: Average loss: -0.1443574243813753, Accuracy: 0.4296\n", 527 | "\n", 528 | "Train Epoch: 46 Epoch time: 16.7243 Loss: 1.9566048383712769\n", 529 | "Test set: Average loss: -0.1257232101917267, Accuracy: 0.3808\n", 530 | "\n", 531 | "Train Epoch: 47 Epoch time: 16.7802 Loss: 1.851891279220581\n", 532 | "Test set: Average loss: -0.09098090907037258, Accuracy: 0.3395\n", 533 | "\n", 534 | "Train Epoch: 48 Epoch time: 16.7901 Loss: 1.5767955780029297\n", 535 | "Test set: Average loss: -0.1268459755897522, Accuracy: 0.4005\n", 536 | "\n", 537 | "Train Epoch: 49 Epoch time: 16.7760 Loss: 1.6413697004318237\n", 538 | "Test set: Average loss: -0.13277969150543212, Accuracy: 0.4223\n", 539 | "\n", 540 | "Train Epoch: 50 Epoch time: 16.6827 Loss: 1.7723995447158813\n", 541 | "Test set: Average loss: -0.1290963588565588, Accuracy: 0.3967\n", 542 | "\n", 543 | "CPU times: user 16min 20s, sys: 5.69 s, total: 16min 25s\n", 544 | "Wall time: 16min 27s\n" 545 | ] 546 | } 547 | ], 548 | "source": [ 549 | "%%time\n", 550 | "print(\"Train in compiled mode on CIFAR-10\")\n", 551 | "model = _get_model().to(device)\n", 552 | "train(model, batch_size=16, epochs=50)" 553 | ] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": null, 558 | "id": "63ed4c0a-e375-4bc4-b321-fa107a9341b7", 559 | "metadata": {}, 560 | "outputs": [], 561 | "source": [] 562 | } 563 | ], 564 | "metadata": { 565 | "kernelspec": { 566 | "display_name": "Python 3 (ipykernel)", 567 | "language": "python", 568 | "name": "python3" 569 | }, 570 | "language_info": { 571 | "codemirror_mode": { 572 | "name": "ipython", 573 | "version": 3 574 | }, 575 | "file_extension": ".py", 576 | "mimetype": "text/x-python", 577 | "name": "python", 578 | "nbconvert_exporter": "python", 579 | "pygments_lexer": "ipython3", 580 | "version": "3.10.9" 581 | } 582 | }, 583 | "nbformat": 4, 584 | "nbformat_minor": 5 585 | } 586 | --------------------------------------------------------------------------------