├── .gitignore
├── LICENSE
├── README.md
├── examples
    ├── Dirichlet.ipynb
    ├── MVN (Cholesky).ipynb
    ├── MVN (LRA).ipynb
    └── MVT (Cholesky).ipynb
├── pyboostlss
    ├── __init__.py
    ├── datasets
    │   ├── __init__.py
    │   ├── arcticlake.csv
    │   ├── data_loader.py
    │   ├── sim_triv_gaussian.csv
    │   └── sim_triv_student.csv
    ├── distributions
    │   ├── DIRICHLET.py
    │   ├── MVN.py
    │   ├── MVN_LRA.py
    │   ├── MVT.py
    │   ├── __init__.py
    │   └── distribution_loss_metric.py
    ├── model.py
    └── utils.py
└── setup.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Pycharm
 2 | .idea/
 3 | dist/
 4 | latex_distributions
 5 | 
 6 | # Byte-compiled / optimized / DLL files
 7 | __pycache__/
 8 | 
 9 | 
10 | # Jupyter Notebook
11 | .ipynb_checkpoints
12 | 
13 | # General
14 | Lib/
15 | Scripts/
16 | pyvenv.cfg


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Py-BoostLSS: An extension of Py-Boost to probabilistic modelling
 2 | 
 3 | We present a probabilistic extension of the recently introduced [Py-Boost](https://github.com/sb-ai-lab/Py-Boost) approach and model all moments of a parametric multivariate distribution as functions of covariates. This allows us to create probabilistic predictions from which intervals and quantiles of interest can be derived. 
 4 | 
 5 | ## Motivation
 6 | 
 7 | Existing implementations of Gradient Boosting Machines, such as [XGBoost](https://github.com/dmlc/xgboost) and [LightGBM](https://github.com/microsoft/LightGBM), are mostly designed for single-target regression tasks. While efficient for low to medium target-dimensions, the computational cost of estimating them becomes prohibitive in high-dimensional settings. 
 8 | 
 9 | As an example, consider modelling a multivariate Gaussian distribution with `D=100` target variables, where the covariance matrix is approximated using the Cholesky-Decomposition. Modelling all conditional moments (i.e., means, standard-deviations and all pairwise correlations) requires estimation of `D(D + 3)/2 = 5,150` parameters. Because most GBM implementations are based on a *one vs. all estimation strategy*, where a separate tree is grown for each parameter, estimating this many parameters for a large dataset can become computationally extremely expensive. 
10 | 
11 | The recently introduced [Py-Boost](https://github.com/sb-ai-lab/Py-Boost) approach provides a more runtime efficient GBM implementation, making it a good candidate for estimating high-dimensional target variables in a probabilistic setting. Borrowing from the original paper [SketchBoost: Fast Gradient Boosted Decision Tree for Multioutput Problems](https://openreview.net/forum?id=WSxarC8t-T), the following figure illustrates the runtime-efficiency of the Py-Boost model.
12 | 
13 | <p align="center">
14 | <img src="https://user-images.githubusercontent.com/41187941/205011855-0e06247f-609f-4c12-9c53-9e00df91b2d9.png" width="350" height="200" />
15 | </p>
16 | 
17 | Even though the original implementation of Py-Boost also supports estimation of univariate responses, Py-BoostLSS focuses on multi-target probabilistic regression settings. For univariate probabilistic GBMs, we refer to our implementations of [XGBoostLSS](https://github.com/StatMixedML/XGBoostLSS) and [LightGBMLSS](https://github.com/StatMixedML/LightGBMLSS).
18 | 
19 | ## Installation
20 | 
21 | Since Py-BoostLSS is entirely GPU-based, we first need to install the corresponding PyTorch and CuPy packages. If you are on Windows, it is preferable to install CuPy via conda. All other OS can use pip. You can check your cuda version with `nvcc --version`.
22 | 
23 | ```python
24 | # CuPy (replace with your cuda version)
25 |   # Windows only
26 |   conda install -c conda-forge cupy cudatoolkit=11.x 
27 |   # Others
28 |   pip install cupy-cuda11x
29 | 
30 | # PyTorch (replace with your cuda version)
31 | pip3 install torch --extra-index-url https://download.pytorch.org/whl/cu11x
32 | ```
33 | 
34 | Next, you can install Py-BoostLSS.
35 | 
36 | ```python
37 | pip install git+https://github.com/StatMixedML/Py-BoostLSS.git 
38 | ```
39 | 
40 | ## How to use
41 | We refer to the [examples section](https://github.com/StatMixedML/Py-BoostLSS/tree/main/examples) for example notebooks.
42 | 
43 | ## Available Distributions
44 | Py-BoostLSS currently supports the following distributions. More distribution follow soon.
45 | 
46 | | Distribution                                                 |  Usage          |Type                               | Support                   
47 | | :----------------------------------------------------------: |:--------------: |:--------------------------------: | :-----------------------: | 
48 | | Multivariate Normal <br /> (Cholesky)                        | `MVN()`         | Continous <br /> (Multivariate)   | $y \in (-\infty,\infty)$  | 
49 | | Multivariate Normal <br /> (Low-Rank Approximation)          | `MVN_LRA()`     | Continous <br /> (Multivariate)   | $y \in (-\infty,\infty)$  | 
50 | | Multivariate Student-T <br /> (Cholesky)                     | `MVT()`         | Continous <br /> (Multivariate)   | $y \in (-\infty,\infty)$  | 
51 | | Dirichlet                                                    | `DIRICHLET()`   | Continous <br /> (Multivariate)   | $y \in [0,1]$             | 
52 | 
53 | 
54 | <!---
55 | | Distribution                               |  Usage  |Type                               | Support                   | Location                   | Scale                      | Shape | Correlation          |
56 | | :----------------------------------------: |:-------:|:--------------------------------: | :-----------------------: | :------------------------: | :------------------------: | :---: | :-------------------:| 
57 | | Multivariate Normal <br /> (Cholesky)      | `MVN()` | Continous <br /> (Multivariate)   | $y \in (-\infty,\infty)$  | $\mu \in (-\infty,\infty)$ | $\sigma \in (0,\infty)$    | None  | $\rho \in [-1,1]$    |
58 | |     ...                                    |   ...   |      ...                          |    ...                    |     ...                    |       ...                  |  ...  |  ...                 |
59 | --->
60 | 
61 | 
62 | 
63 | 
64 | ## Feedback
65 | Please provide feedback on how to improve Py-BoostLSS, or if you request additional distributions to be implemented, by opening a new issue or via the discussion section.
66 | 
67 | 
68 | ## Acknowledgements
69 | 
70 | The implementation of Py-BoostLSS relies on the following resources:
71 | 
72 | - [Py-boost: a research tool for exploring GBDTs](https://github.com/sb-ai-lab/Py-Boost)
73 | - [SketchBoost: Fast Gradient Boosted Decision Tree for Multioutput Problems](https://openreview.net/forum?id=WSxarC8t-T)
74 | 
75 | We genuinely thank the original authors [Anton Vakhrushev](https://www.kaggle.com/btbpanda) and [Leonid Iosipoi](http://iosipoi.com/) for making their work publicly available. 
76 | 
77 | ## Reference Paper
78 | [![Arxiv link](https://img.shields.io/badge/arXiv-Multi%20Target%20XGBoostLSS%20Regression-color=brightgreen)](https://arxiv.org/abs/2210.06831) <br/>
79 | [![Arxiv link](https://img.shields.io/badge/arXiv-Distributional%20Gradient%20Boosting%20Machines-color=brightgreen)](https://arxiv.org/abs/2204.00778) <br/>
80 | [![Arxiv link](https://img.shields.io/badge/arXiv-XGBoostLSS%3A%20An%20extension%20of%20XGBoost%20to%20probabilistic%20forecasting-color=brightgreen)](https://arxiv.org/abs/1907.03178) <br/>
81 | 
82 | <!---
83 | März, Alexander (2022) [*Multi-Target XGBoostLSS Regression*](https://arxiv.org/abs/2210.06831). <br/>
84 | <br />
85 | März, A. and Kneib, T. (2022) [*"Distributional Gradient Boosting Machines"*](https://arxiv.org/abs/2204.00778). <br/>
86 | <br />
87 | März, Alexander (2019) [*XGBoostLSS - An extension of XGBoost to probabilistic forecasting*](https://arxiv.org/abs/1907.03178).
88 | --->
89 | 
90 | 
91 | 
92 | 


--------------------------------------------------------------------------------
/examples/Dirichlet.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "id": "22aca876-6dee-4cd3-a528-d2561a7ad301",
  6 |    "metadata": {},
  7 |    "source": [
  8 |     "<div class=\"alert alert-success\">\n",
  9 |     "    <center> <h1> <font size=\"8\"> Dirichlet Example </font> </h1> </center>\n",
 10 |     "</div>\n",
 11 |     "\n",
 12 |     "<br/>\n",
 13 |     "<br/>"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "markdown",
 18 |    "id": "c6cdf104-7c02-4888-968e-12b55187ef39",
 19 |    "metadata": {},
 20 |    "source": [
 21 |     "The Dirichlet distribution is commonly used for modelling non-negative compositional data, i.e., data that consist of sub-sets that are fractions of some total. Compositional data are typically represented as proportions or percentages summing to 100\\%, so that the Dirichlet extends the univariate beta-distribution to the multivariate case. Compositional data analysis (CoDa) is a branch of statistics that deals with multivariate observations carrying relative information and finds widespread use in ecology, economics or political science. As a result of the unit-sum constraint, models that use distributions designed for unconstrained data typically suffer from the problem of spurious correlation when applied to compositional data. "
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "markdown",
 26 |    "id": "538d04c1-08b3-488b-984d-312bc9dad21a",
 27 |    "metadata": {},
 28 |    "source": [
 29 |     "In this example, we model and predict all parameters of a Dirichlet distribution with $Y_{D}=3$ target variables using the famous Arctic-Lake dataset. The density of the Dirichlet distribution with parameters $\\mathbf{\\alpha}_{\\mathbf{x}} = (\\alpha_{\\mathbf{x},1}, \\ldots, \\alpha_{\\mathbf{x},D}) \\in \\mathbb{R}^{D}_{+}$ with $\\sum^{D}_{d=1}y_{d}=1$ for all $y_{d}\\in \\left[0,1\\right]$ is given by"
 30 |    ]
 31 |   },
 32 |   {
 33 |    "cell_type": "markdown",
 34 |    "id": "247dc33d-ac22-4e51-b500-d4b61fae21bf",
 35 |    "metadata": {},
 36 |    "source": [
 37 |     "$$\n",
 38 |     "f\\big(\\mathbf{y}|\\mathbf{\\theta}_{\\mathbf{x}}\\big) = \\frac{1}{\\mathrm{B}(\\mathbf{\\alpha}_{\\mathbf{x}})} \\prod_{d=1}^{D}y^{\\alpha_{\\mathbf{x},d-1}}_{d}\n",
 39 |     "$$"
 40 |    ]
 41 |   },
 42 |   {
 43 |    "cell_type": "markdown",
 44 |    "id": "86229dd0-bccf-41a1-800f-18892f451a21",
 45 |    "metadata": {},
 46 |    "source": [
 47 |     "To ensure positivity, we use $\\exp(\\alpha_{\\mathbf{x},d})$ for all $d=1,\\ldots, D$. The estimated parameters have the interpretation of providing the probability of an event falling into category $d$, i.e., $\\mathbb{E}(y_{d}) = \\frac{\\alpha_{d}}{\\alpha_{0}}$, with $\\alpha_{0} = \\sum^{D}_{d=1}\\alpha_{d}$. For more details, we refer to our related paper **[März, Alexander (2022), *Multi-Target XGBoostLSS Regression*](https://arxiv.org/abs/2210.06831)**.\n",
 48 |     "\n",
 49 |     "<br/>\n",
 50 |     "<br/>"
 51 |    ]
 52 |   },
 53 |   {
 54 |    "cell_type": "markdown",
 55 |    "id": "d7b4eea0-96b6-482a-afc9-7ec867bf2193",
 56 |    "metadata": {},
 57 |    "source": [
 58 |     "# Imports"
 59 |    ]
 60 |   },
 61 |   {
 62 |    "cell_type": "code",
 63 |    "execution_count": 1,
 64 |    "id": "0de768b2-ef44-434d-89e4-402170de0eb9",
 65 |    "metadata": {},
 66 |    "outputs": [],
 67 |    "source": [
 68 |     "import os\n",
 69 |     "import numpy as np\n",
 70 |     "import pandas as pd\n",
 71 |     "# Optional: set the device to run\n",
 72 |     "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
 73 |     "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
 74 |     "\n",
 75 |     "from pyboostlss.model import *\n",
 76 |     "from pyboostlss.distributions.DIRICHLET import *\n",
 77 |     "from pyboostlss.distributions.distribution_loss_metric import *\n",
 78 |     "from pyboostlss.utils import *\n",
 79 |     "from pyboostlss.datasets.data_loader import load_example_data\n",
 80 |     "\n",
 81 |     "import plotnine\n",
 82 |     "from plotnine import *\n",
 83 |     "plotnine.options.figure_size = (20, 10)"
 84 |    ]
 85 |   },
 86 |   {
 87 |    "cell_type": "markdown",
 88 |    "id": "8e90e9e4-5902-4299-9236-d31af4e29b2b",
 89 |    "metadata": {},
 90 |    "source": [
 91 |     "# Specifiy distribution and initialize model"
 92 |    ]
 93 |   },
 94 |   {
 95 |    "cell_type": "code",
 96 |    "execution_count": 2,
 97 |    "id": "75224245-9b11-4730-a4c6-d7fe6b3a272c",
 98 |    "metadata": {},
 99 |    "outputs": [],
100 |    "source": [
101 |     "distribution = DIRICHLET(D=3)     # Dirichlet distribution, where D specifies the number of target variables\n",
102 |     "pyblss = PyBoostLSS(distribution) # Initializes model with specified distribution"
103 |    ]
104 |   },
105 |   {
106 |    "cell_type": "markdown",
107 |    "id": "43c36cd5-11f4-4c03-8297-c2555924bd4a",
108 |    "metadata": {},
109 |    "source": [
110 |     "# Data"
111 |    ]
112 |   },
113 |   {
114 |    "cell_type": "code",
115 |    "execution_count": 3,
116 |    "id": "78118dfc-6b05-4634-ae15-9fbf3d30b47b",
117 |    "metadata": {},
118 |    "outputs": [],
119 |    "source": [
120 |     "data_df = load_example_data(\"arcticlake.csv\")\n",
121 |     "\n",
122 |     "# Create 80%, 10%, 10% split for train, validation and test \n",
123 |     "train, validate, test = np.split(data_df.sample(frac=1,random_state=123), [int(0.8*len(data_df)), int(0.9*len(data_df))])\n",
124 |     "\n",
125 |     "# Train\n",
126 |     "x_train = train[\"depth\"].values.reshape(-1,1)\n",
127 |     "y_train = train.drop(columns=\"depth\").values\n",
128 |     "dtrain = {\"X\": x_train, \"y\": y_train}\n",
129 |     "\n",
130 |     "# Validation\n",
131 |     "x_eval = validate[\"depth\"].values.reshape(-1,1)\n",
132 |     "y_eval = validate.drop(columns=\"depth\").values\n",
133 |     "eval_sets = [{'X': x_eval, 'y': y_eval}] # Specifies eval_sets on which the model is evaluated on\n",
134 |     "\n",
135 |     "# Test\n",
136 |     "x_test = test[\"depth\"].values.reshape(-1,1)\n",
137 |     "y_test = test.drop(columns=\"depth\").values"
138 |    ]
139 |   },
140 |   {
141 |    "cell_type": "markdown",
142 |    "id": "e742234a-ad21-4a32-aa80-08c59d318503",
143 |    "metadata": {},
144 |    "source": [
145 |     "# Hyper-Parameter Optimization via Optuna"
146 |    ]
147 |   },
148 |   {
149 |    "cell_type": "code",
150 |    "execution_count": 4,
151 |    "id": "88d8403d-ea56-4d44-ba26-1f86632b7b78",
152 |    "metadata": {},
153 |    "outputs": [
154 |     {
155 |      "name": "stderr",
156 |      "output_type": "stream",
157 |      "text": [
158 |       "\u001b[32m[I 2022-12-08 11:33:16,505]\u001b[0m A new study created in memory with name: Py-BoostLSS Hyper-Parameter Optimization\u001b[0m\n",
159 |       "C:\\Users\\Alexander\\.julia\\v0.6\\Conda\\deps\\usr\\envs\\pyboost\\lib\\site-packages\\optuna\\progress_bar.py:49: ExperimentalWarning: Progress bar is experimental (supported from v1.2.0). The interface can change in the future.\n"
160 |      ]
161 |     },
162 |     {
163 |      "data": {
164 |       "application/vnd.jupyter.widget-view+json": {
165 |        "model_id": "08991189c4d8418e86d16f8f877de7a6",
166 |        "version_major": 2,
167 |        "version_minor": 0
168 |       },
169 |       "text/plain": [
170 |        "  0%|          | 0/10 [00:00<?, ?it/s]"
171 |       ]
172 |      },
173 |      "metadata": {},
174 |      "output_type": "display_data"
175 |     },
176 |     {
177 |      "name": "stdout",
178 |      "output_type": "stream",
179 |      "text": [
180 |       "[11:33:25] Stdout logging level is INFO.\n",
181 |       "[11:33:25] GDBT train starts. Max iter 500, early stopping rounds 20\n",
182 |       "[11:33:31] Iter 0; Sample 0, NLL-score = -5.127476677854759; \n",
183 |       "[11:33:35] Early stopping at iter 190, best iter 170, best_score -10.250120833771959\n",
184 |       "\u001b[32m[I 2022-12-08 11:33:36,535]\u001b[0m Trial 0 finished with value: -10.2501292414331 and parameters: {'lr': 0.2261559634886368, 'max_depth': 1, 'sketch_outputs': 7, 'lambda_l2': 30.88877600549751, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 71.47732298654863}. Best is trial 0 with value: -10.2501292414331.\u001b[0m\n",
185 |       "[11:33:36] Stdout logging level is INFO.\n",
186 |       "[11:33:36] GDBT train starts. Max iter 500, early stopping rounds 20\n",
187 |       "[11:33:36] Iter 0; Sample 0, NLL-score = -4.7779778049753485; \n",
188 |       "[11:33:38] Early stopping at iter 62, best iter 42, best_score -9.890179422452471\n",
189 |       "\u001b[32m[I 2022-12-08 11:33:38,303]\u001b[0m Trial 1 finished with value: -9.890167654880095 and parameters: {'lr': 0.5377869141302691, 'max_depth': 2, 'sketch_outputs': 10, 'lambda_l2': 22.04734477729385, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 242.25332392584508}. Best is trial 0 with value: -10.2501292414331.\u001b[0m\n",
190 |       "[11:33:38] Stdout logging level is INFO.\n",
191 |       "[11:33:38] GDBT train starts. Max iter 500, early stopping rounds 20\n",
192 |       "[11:33:38] Iter 0; Sample 0, NLL-score = -4.7779771517360095; \n",
193 |       "[11:33:45] Early stopping at iter 297, best iter 277, best_score -9.79587912904152\n",
194 |       "\u001b[32m[I 2022-12-08 11:33:46,318]\u001b[0m Trial 2 finished with value: -9.795877693087373 and parameters: {'lr': 0.13006341497658322, 'max_depth': 4, 'sketch_outputs': 9, 'lambda_l2': 29.47977136537219, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 411.4277549678564}. Best is trial 0 with value: -10.2501292414331.\u001b[0m\n",
195 |       "[11:33:46] Stdout logging level is INFO.\n",
196 |       "[11:33:46] GDBT train starts. Max iter 500, early stopping rounds 20\n",
197 |       "[11:33:46] Iter 0; Sample 0, NLL-score = -4.777977646926431; \n",
198 |       "[11:33:51] Early stopping at iter 229, best iter 209, best_score -9.617372769127643\n",
199 |       "\u001b[32m[I 2022-12-08 11:33:51,969]\u001b[0m Trial 3 finished with value: -9.617365965817168 and parameters: {'lr': 0.3865616908849664, 'max_depth': 2, 'sketch_outputs': 2, 'lambda_l2': 14.504028702160326, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 214.03968800602453}. Best is trial 0 with value: -10.2501292414331.\u001b[0m\n",
200 |       "[11:33:52] Stdout logging level is INFO.\n",
201 |       "[11:33:52] GDBT train starts. Max iter 500, early stopping rounds 20\n",
202 |       "[11:33:52] Iter 0; Sample 0, NLL-score = -4.777978848671246; \n",
203 |       "[11:33:53] Early stopping at iter 59, best iter 39, best_score -10.45860715688298\n",
204 |       "\u001b[32m[I 2022-12-08 11:33:53,525]\u001b[0m Trial 4 finished with value: -10.458604924345618 and parameters: {'lr': 0.919274500979721, 'max_depth': 3, 'sketch_outputs': 3, 'lambda_l2': 4.634818879126481, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 375.82206166522025}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
205 |       "[11:33:53] Stdout logging level is INFO.\n",
206 |       "[11:33:53] GDBT train starts. Max iter 500, early stopping rounds 20\n",
207 |       "[11:33:53] Iter 0; Sample 0, NLL-score = -5.208636852165513; \n",
208 |       "[11:33:57] Early stopping at iter 151, best iter 131, best_score -9.509026979370176\n",
209 |       "\u001b[32m[I 2022-12-08 11:33:57,534]\u001b[0m Trial 5 finished with value: -9.50902935905508 and parameters: {'lr': 0.16105845253307519, 'max_depth': 2, 'sketch_outputs': 9, 'lambda_l2': 10.693996516830357, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 393.0733498821459}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
210 |       "[11:33:57] Stdout logging level is INFO.\n",
211 |       "[11:33:57] GDBT train starts. Max iter 500, early stopping rounds 20\n",
212 |       "[11:33:57] Iter 0; Sample 0, NLL-score = -4.777977950293839; \n",
213 |       "[11:33:58] Early stopping at iter 63, best iter 43, best_score -8.59497991615099\n",
214 |       "\u001b[32m[I 2022-12-08 11:33:59,107]\u001b[0m Trial 6 finished with value: -8.594978287635527 and parameters: {'lr': 0.505295942635048, 'max_depth': 1, 'sketch_outputs': 4, 'lambda_l2': 8.756934127722555, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 460.6811644415763}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
215 |       "[11:33:59] Stdout logging level is INFO.\n",
216 |       "[11:33:59] GDBT train starts. Max iter 500, early stopping rounds 20\n",
217 |       "[11:33:59] Iter 0; Sample 0, NLL-score = -4.777978030971868; \n",
218 |       "[11:34:01] Early stopping at iter 85, best iter 65, best_score -9.566335474922479\n",
219 |       "\u001b[32m[I 2022-12-08 11:34:01,309]\u001b[0m Trial 7 finished with value: -9.566331399467114 and parameters: {'lr': 0.6563189495925507, 'max_depth': 1, 'sketch_outputs': 3, 'lambda_l2': 19.24910924466795, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 183.3348016026964}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
220 |       "[11:34:01] Stdout logging level is INFO.\n",
221 |       "[11:34:01] GDBT train starts. Max iter 500, early stopping rounds 20\n",
222 |       "[11:34:01] Iter 0; Sample 0, NLL-score = -4.777978130290063; \n",
223 |       "[11:34:04] Early stopping at iter 145, best iter 125, best_score -8.423402068697614\n",
224 |       "\u001b[32m[I 2022-12-08 11:34:04,995]\u001b[0m Trial 8 finished with value: -8.423397062040415 and parameters: {'lr': 0.5244406900404303, 'max_depth': 3, 'sketch_outputs': 1, 'lambda_l2': 2.200791726816238, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 432.4407876376129}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
225 |       "[11:34:05] Stdout logging level is INFO.\n",
226 |       "[11:34:05] GDBT train starts. Max iter 500, early stopping rounds 20\n",
227 |       "[11:34:05] Iter 0; Sample 0, NLL-score = -4.777977557240683; \n",
228 |       "[11:34:08] Early stopping at iter 157, best iter 137, best_score -8.691995518775983\n",
229 |       "\u001b[32m[I 2022-12-08 11:34:09,090]\u001b[0m Trial 9 finished with value: -8.691999712136926 and parameters: {'lr': 0.3743890448500777, 'max_depth': 3, 'sketch_outputs': 4, 'lambda_l2': 21.375505387520626, 'colsample': 1.0, 'subsample': 1.0, 'min_gain_to_split': 363.2284392550362}. Best is trial 4 with value: -10.458604924345618.\u001b[0m\n",
230 |       "\n",
231 |       "Hyper-Parameter Optimization successfully finished.\n",
232 |       "  Number of finished trials:  10\n",
233 |       "  Best trial:\n",
234 |       "    Value: -10.458604924345618\n",
235 |       "    Params: \n",
236 |       "    lr: 0.919274500979721\n",
237 |       "    max_depth: 3\n",
238 |       "    sketch_outputs: 3\n",
239 |       "    lambda_l2: 4.634818879126481\n",
240 |       "    colsample: 1.0\n",
241 |       "    subsample: 1.0\n",
242 |       "    min_gain_to_split: 375.82206166522025\n",
243 |       "    opt_rounds: 39\n"
244 |      ]
245 |     }
246 |    ],
247 |    "source": [
248 |     "np.random.seed(123)\n",
249 |     "\n",
250 |     "# Specifies the hyper-parameters and their value range\n",
251 |     "    # The structure is as follows: \"hyper-parameter\": [lower_bound, upper_bound]\n",
252 |     "    # Currently, only the following hyper-parameters can be optimized\n",
253 |     "    \n",
254 |     "hp_dict = {\"lr\": [1e-3, 1],  \n",
255 |     "           \"max_depth\": [1, 4],\n",
256 |     "           \"sketch_outputs\": [1,10],\n",
257 |     "           \"lambda_l2\": [0, 40],     \n",
258 |     "           \"colsample\": [1.0, 1.0],                      # increased to max due to small size of dataset\n",
259 |     "           \"subsample\": [1.0, 1.0],                      # increased to max due to small size of dataset\n",
260 |     "           \"min_gain_to_split\": [0, 500]  \n",
261 |     "          }  \n",
262 |     "\n",
263 |     "opt_param = pyblss.hyper_opt(params=hp_dict,\n",
264 |     "                             dtrain=dtrain,\n",
265 |     "                             eval_sets=eval_sets,\n",
266 |     "                             use_hess=True, \n",
267 |     "                             sketch_method=\"proj\",\n",
268 |     "                             hp_seed=123,                # Seed for random number generator used in the Bayesian hyper-parameter search.\n",
269 |     "                             ntrees=500,                 # Number of boosting iterations.\n",
270 |     "                             es=20,                      # Early stopping rounds\n",
271 |     "                             n_trials=10,                # The number of trials. If this argument is set to None, there is no limitation on the number of trials.\n",
272 |     "                             max_minutes=120,            # Time budget in minutes, i.e., stop study after the given number of minutes.\n",
273 |     "                             silence=False)              # Controls the verbosity of the trail, i.e., user can silence the outputs of the trail."
274 |    ]
275 |   },
276 |   {
277 |    "cell_type": "markdown",
278 |    "id": "53213379-e7b1-4a4f-8d85-dda5a729f608",
279 |    "metadata": {},
280 |    "source": [
281 |     "# Model Training"
282 |    ]
283 |   },
284 |   {
285 |    "cell_type": "code",
286 |    "execution_count": 5,
287 |    "id": "11b9d38e-fee1-45fd-96cf-4ade74849593",
288 |    "metadata": {},
289 |    "outputs": [
290 |     {
291 |      "name": "stdout",
292 |      "output_type": "stream",
293 |      "text": [
294 |       "[11:34:09] Stdout logging level is INFO.\n",
295 |       "[11:34:09] GDBT train starts. Max iter 39, early stopping rounds 100\n",
296 |       "[11:34:09] Iter 0; \n",
297 |       "[11:34:09] Iter 38; \n"
298 |      ]
299 |     }
300 |    ],
301 |    "source": [
302 |     "opt_params = opt_param.copy()\n",
303 |     "\n",
304 |     "pyboostlss_model = pyblss.train(dtrain=dtrain,\n",
305 |     "                                lr=opt_params[\"lr\"], \n",
306 |     "                                lambda_l2=opt_params[\"lambda_l2\"],\n",
307 |     "                                max_depth=opt_params[\"max_depth\"],\n",
308 |     "                                sketch_outputs=opt_params[\"sketch_outputs\"],\n",
309 |     "                                colsample=opt_params[\"colsample\"],\n",
310 |     "                                subsample=opt_params[\"subsample\"],\n",
311 |     "                                min_gain_to_split=opt_params[\"min_gain_to_split\"],\n",
312 |     "                                ntrees=opt_params[\"opt_rounds\"],\n",
313 |     "                                use_hess=True,\n",
314 |     "                                verbose=100,                                \n",
315 |     "                                sketch_method=\"proj\",\n",
316 |     "                                seed=123)"
317 |    ]
318 |   },
319 |   {
320 |    "cell_type": "markdown",
321 |    "id": "48228365-d49a-4481-868a-35b7e97535f5",
322 |    "metadata": {},
323 |    "source": [
324 |     "# Predict"
325 |    ]
326 |   },
327 |   {
328 |    "cell_type": "code",
329 |    "execution_count": 6,
330 |    "id": "2c126862-1732-4033-a24f-c9e5287fc1ea",
331 |    "metadata": {},
332 |    "outputs": [
333 |     {
334 |      "data": {
335 |       "text/html": [
336 |        "<div>\n",
337 |        "<style scoped>\n",
338 |        "    .dataframe tbody tr th:only-of-type {\n",
339 |        "        vertical-align: middle;\n",
340 |        "    }\n",
341 |        "\n",
342 |        "    .dataframe tbody tr th {\n",
343 |        "        vertical-align: top;\n",
344 |        "    }\n",
345 |        "\n",
346 |        "    .dataframe thead th {\n",
347 |        "        text-align: right;\n",
348 |        "    }\n",
349 |        "</style>\n",
350 |        "<table border=\"1\" class=\"dataframe\">\n",
351 |        "  <thead>\n",
352 |        "    <tr style=\"text-align: right;\">\n",
353 |        "      <th></th>\n",
354 |        "      <th>alpha_1</th>\n",
355 |        "      <th>alpha_2</th>\n",
356 |        "      <th>alpha_3</th>\n",
357 |        "    </tr>\n",
358 |        "  </thead>\n",
359 |        "  <tbody>\n",
360 |        "    <tr>\n",
361 |        "      <th>0</th>\n",
362 |        "      <td>8.925608</td>\n",
363 |        "      <td>5.909983</td>\n",
364 |        "      <td>1.225174</td>\n",
365 |        "    </tr>\n",
366 |        "    <tr>\n",
367 |        "      <th>1</th>\n",
368 |        "      <td>8.925608</td>\n",
369 |        "      <td>5.909983</td>\n",
370 |        "      <td>1.225174</td>\n",
371 |        "    </tr>\n",
372 |        "    <tr>\n",
373 |        "      <th>2</th>\n",
374 |        "      <td>1.241373</td>\n",
375 |        "      <td>10.116554</td>\n",
376 |        "      <td>4.594018</td>\n",
377 |        "    </tr>\n",
378 |        "    <tr>\n",
379 |        "      <th>3</th>\n",
380 |        "      <td>1.782198</td>\n",
381 |        "      <td>11.356755</td>\n",
382 |        "      <td>9.836135</td>\n",
383 |        "    </tr>\n",
384 |        "    <tr>\n",
385 |        "      <th>4</th>\n",
386 |        "      <td>8.925608</td>\n",
387 |        "      <td>5.909983</td>\n",
388 |        "      <td>1.225174</td>\n",
389 |        "    </tr>\n",
390 |        "  </tbody>\n",
391 |        "</table>\n",
392 |        "</div>"
393 |       ],
394 |       "text/plain": [
395 |        "    alpha_1    alpha_2   alpha_3\n",
396 |        "0  8.925608   5.909983  1.225174\n",
397 |        "1  8.925608   5.909983  1.225174\n",
398 |        "2  1.241373  10.116554  4.594018\n",
399 |        "3  1.782198  11.356755  9.836135\n",
400 |        "4  8.925608   5.909983  1.225174"
401 |       ]
402 |      },
403 |      "execution_count": 6,
404 |      "metadata": {},
405 |      "output_type": "execute_result"
406 |     }
407 |    ],
408 |    "source": [
409 |     "# Predicts transformed parameters of the specified distribution. \n",
410 |     "predt_params = distribution.predict(model=pyboostlss_model,\n",
411 |     "                                    X_test=x_train,          # Here we use the train dataset to later infer the partial dependence of the parameters on x\n",
412 |     "                                    pred_type=\"parameters\")\n",
413 |     "predt_params.head()"
414 |    ]
415 |   },
416 |   {
417 |    "cell_type": "markdown",
418 |    "id": "6dc11354-6241-4f53-b860-e12747660b75",
419 |    "metadata": {},
420 |    "source": [
421 |     "Please note that the predicted parameters are not yet on the response-scale. Yet we can transform them easily as described above: the estimated parameters have the interpretation of providing the probability of an event falling into category $d$, i.e., $\\mathbb{E}(y_{d}) = \\frac{\\alpha_{d}}{\\alpha_{0}}$, with $\\alpha_{0} = \\sum^{D}_{d=1}\\alpha_{d}$."
422 |    ]
423 |   },
424 |   {
425 |    "cell_type": "code",
426 |    "execution_count": 10,
427 |    "id": "9f21270c-a600-4104-8c99-60bcfa0e802f",
428 |    "metadata": {},
429 |    "outputs": [
430 |     {
431 |      "data": {
432 |       "text/html": [
433 |        "<div>\n",
434 |        "<style scoped>\n",
435 |        "    .dataframe tbody tr th:only-of-type {\n",
436 |        "        vertical-align: middle;\n",
437 |        "    }\n",
438 |        "\n",
439 |        "    .dataframe tbody tr th {\n",
440 |        "        vertical-align: top;\n",
441 |        "    }\n",
442 |        "\n",
443 |        "    .dataframe thead th {\n",
444 |        "        text-align: right;\n",
445 |        "    }\n",
446 |        "</style>\n",
447 |        "<table border=\"1\" class=\"dataframe\">\n",
448 |        "  <thead>\n",
449 |        "    <tr style=\"text-align: right;\">\n",
450 |        "      <th></th>\n",
451 |        "      <th>sand</th>\n",
452 |        "      <th>silt</th>\n",
453 |        "      <th>clay</th>\n",
454 |        "    </tr>\n",
455 |        "  </thead>\n",
456 |        "  <tbody>\n",
457 |        "    <tr>\n",
458 |        "      <th>0</th>\n",
459 |        "      <td>0.555740</td>\n",
460 |        "      <td>0.367976</td>\n",
461 |        "      <td>0.076284</td>\n",
462 |        "    </tr>\n",
463 |        "    <tr>\n",
464 |        "      <th>1</th>\n",
465 |        "      <td>0.555740</td>\n",
466 |        "      <td>0.367976</td>\n",
467 |        "      <td>0.076284</td>\n",
468 |        "    </tr>\n",
469 |        "    <tr>\n",
470 |        "      <th>2</th>\n",
471 |        "      <td>0.077820</td>\n",
472 |        "      <td>0.634189</td>\n",
473 |        "      <td>0.287991</td>\n",
474 |        "    </tr>\n",
475 |        "    <tr>\n",
476 |        "      <th>3</th>\n",
477 |        "      <td>0.077571</td>\n",
478 |        "      <td>0.494307</td>\n",
479 |        "      <td>0.428122</td>\n",
480 |        "    </tr>\n",
481 |        "    <tr>\n",
482 |        "      <th>4</th>\n",
483 |        "      <td>0.555740</td>\n",
484 |        "      <td>0.367976</td>\n",
485 |        "      <td>0.076284</td>\n",
486 |        "    </tr>\n",
487 |        "  </tbody>\n",
488 |        "</table>\n",
489 |        "</div>"
490 |       ],
491 |       "text/plain": [
492 |        "       sand      silt      clay\n",
493 |        "0  0.555740  0.367976  0.076284\n",
494 |        "1  0.555740  0.367976  0.076284\n",
495 |        "2  0.077820  0.634189  0.287991\n",
496 |        "3  0.077571  0.494307  0.428122\n",
497 |        "4  0.555740  0.367976  0.076284"
498 |       ]
499 |      },
500 |      "execution_count": 10,
501 |      "metadata": {},
502 |      "output_type": "execute_result"
503 |     }
504 |    ],
505 |    "source": [
506 |     "# Transform to response scale\n",
507 |     "predt_params_transf = predt_params / predt_params.sum(axis=1)\n",
508 |     "predt_params_transf = predt_params.div(predt_params.sum(axis=1), axis=0)\n",
509 |     "predt_params_transf.columns = data_df.iloc[:,:3].columns\n",
510 |     "\n",
511 |     "predt_params_transf.head()"
512 |    ]
513 |   },
514 |   {
515 |    "cell_type": "code",
516 |    "execution_count": 8,
517 |    "id": "c7e1a565-b8a6-4864-861e-75a81d558528",
518 |    "metadata": {},
519 |    "outputs": [
520 |     {
521 |      "data": {
522 |       "text/plain": [
523 |        "(10000, 4, 3)"
524 |       ]
525 |      },
526 |      "execution_count": 8,
527 |      "metadata": {},
528 |      "output_type": "execute_result"
529 |     }
530 |    ],
531 |    "source": [
532 |     "# Draws random samples from the predicted distribution\n",
533 |     "torch.manual_seed(123)\n",
534 |     "n_samples = 10000\n",
535 |     "predt_samples = distribution.predict(model=pyboostlss_model,\n",
536 |     "                                     X_test=x_test,         \n",
537 |     "                                     pred_type=\"samples\",   \n",
538 |     "                                     n_samples=n_samples)\n",
539 |     "\n",
540 |     "predt_samples.shape # Output-shape is (n_samples, n_obs, n_target)"
541 |    ]
542 |   },
543 |   {
544 |    "cell_type": "markdown",
545 |    "id": "6d32e5fd-ada9-41a8-896e-b446e606a3d2",
546 |    "metadata": {},
547 |    "source": [
548 |     "# Partial Dependence Plot"
549 |    ]
550 |   },
551 |   {
552 |    "cell_type": "markdown",
553 |    "id": "178932c6-32b8-4b45-a264-db6844e62d7d",
554 |    "metadata": {},
555 |    "source": [
556 |     "Since there is only one covariate in the dataset, we can infer the effect of depth (in meters) on the sediment composition using a scatter-smooth estimate. The figure shows that with increasing depth, the relative frequency of sand decreases while the proportion of silt and clay increases."
557 |    ]
558 |   },
559 |   {
560 |    "cell_type": "code",
561 |    "execution_count": 9,
562 |    "id": "2e79d446-6617-46ce-ae8b-c5209f26abb8",
563 |    "metadata": {},
564 |    "outputs": [
565 |     {
566 |      "data": {
567 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAABlEAAAN9CAYAAAADvF+lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3gU5fr/8c+mN0IChA4hNAVBOgIhCUVBEKQpAoKEIqAC9nYsoCiIh3NUjgqICIgoKFUUEKQkIVIU6agUKdKT0Akh2ez8/uC3+2VJAgkkmWzyfl0Xl5vZZ2fu2Wdmd5x7n/uxGIZhCAAAAAAAAAAAAE7czA4AAAAAAAAAAACgICKJAgAAAAAAAAAAkAmSKAAAAAAAAAAAAJkgiQIAAAAAAAAAAJAJkigAAAAAAAAAAACZIIkCAAAAAAAAAACQCZIoAAAAAAAAAAAAmSCJAgAAAAAAAAAAkAmSKAAAAAAAAAAAAJkgiQIAQCFhsVgUHR19y6+Pjo6WxWLJvYCKiBkzZshisWjt2rVmh2K60aNHy2Kx6ODBg2aHckPz5s1TvXr15Ovrm2d9dyvHxe2cgzl971u1aqUqVarc0raA/FCUvpPWrl0ri8WiGTNm5Op6XeUzGQAAoKAjiQIAwC2w3/C49p+/v7/uvvtuvfPOO0pJScn1bR48eFCjR4/W1q1bc33dt6JKlSpO++/n56fy5curbdu2GjNmjI4cOWJ2iMgF9ptw9n9ubm4KCgpSZGSkZs+enavbOnv2rEaPHp2nCak9e/aod+/eKl68uD7++GPNmjVLtWrVyrL99fvv6empkiVLqkGDBhoyZEiRTZ5t3bpVo0ePzvHN2fw8nlxFfhz3BdWHH36Y64mD3Harx3phlt/XQK1atcqwvaCgINWtW1dvv/22zp49m6vbu103u16zWCy69957b7qe9PR0ff3114qMjFT58uXl7e2t8uXLKzw8XC+99JISExOd2p86dUqvvPKK6tatq8DAQBUrVkxVq1ZVt27dNG3atNzYNQAAiiwPswMAAMCVPfTQQ+rSpYsk6eTJk5ozZ47eeOMNxcfHa9myZbm6rYMHD+qtt95SlSpVVL9+/QzPX758We7u7rm6zZspU6aMJkyYIElKTU3VyZMntWHDBr399tsaN26c/vvf/2rYsGH5GhPyxhtvvKGaNWsqPT1dBw4c0NSpU9W3b1/9888/euWVV3JlG2fPntVbb70l6epNs7ywdu1aWa1Wffjhh2rYsGG2X2fff5vNpnPnzmn37t1atGiRpk6dqs6dO+vrr79WQECAo32/fv3Uq1cveXl5ZXsbU6dO1eTJk3O0P2bZunWr3nrrrVse0ZIfx5OryI/jvqD68MMPVaVKlUxHURaU8+F2j3Uzvf7663rllVfk7e2dJ+vPz2sgNzc3zZw50/F3UlKSFi9erFGjRun777/Xpk2b5OZWMH4jerPrtezq27ev5syZo7vvvlvDhw9XmTJldOzYMe3YsUOTJ09Wz549VapUKUnS4cOH1bRpUyUmJuqhhx7S448/Li8vL/39999at26dPvzwQw0aNCiX9hAAgKKHJAoAALehXr166tu3r+PvkSNHqkmTJlq+fLl+/fVXNWnS5La3cf78eQUGBt60nY+Pz21vK6cCAgKc9t9u79696ty5s5544gmVKVNG3bp1y/fYkLvatWunli1bOv4eMGCAatWqpXHjxumFF16Qh4drXFaeOHFCklSiRIkcve76/Zekjz76SE8//bQmT56svn37atGiRY7n3N3ds5XUNAxDly5dUkBAgDw9PeXp6ZmjuFxVfhxPFy5cULFixW57Pa7u8uXL8vT0dJlz1K4onQ95xcPDI0/7PT+ugewsFkuG6w379jZv3qzt27ffVsKioNm8ebPmzJmjJk2aKD4+PsO5cPHiRae///3vf+vkyZP68MMP9fTTT2dYn/27DwAA3JqC8VMNAAAKCU9PT0eJhn379slms2ns2LFq1aqVypUrJy8vL1WoUEH9+/fX4cOHM7zePq/J2rVr1apVKwUGBqpevXoaPXq0WrduLenqzUZ7OYtrf7Wc2Zwoc+fOVdeuXRUaGiofHx+VKFFC999/v9atW5dn74Ek1ahRQ/Pnz5fFYsn0V+VbtmzRQw89pNKlS8vLy0tVq1bVK6+8ouTkZKd29pr4SUlJGjhwoEJCQuTr66vmzZtr1apVmW57zZo16tChg4KDg+Xt7a1atWpp/PjxSk9Pd2pn/1XxiRMn1K9fP5UsWVK+vr6KjIzUb7/9lmG9aWlpGjVqlKpUqSIfHx/VqlVLkyZNyvI9uHDhgl577TXdcccd8vb2VokSJdS1a1dt377dqd21tfBnzZqlu+++Wz4+PqpQoYL+9a9/ZYhbkhISEvTcc8+pRo0a8vb2VqlSpRQREaE5c+bcUgy3IjQ0VLVr19b58+eVkJBww7ZHjx7V4MGDVaFCBXl5ealixYoaMmSIjh8/7mgzY8YMhYWFSZLeeustxzGenV9+22w2TZw40THPSWBgoNq0aaOVK1c62hw8eFAWi0WjRo2SJIWFhWV7/Vnx8vLSp59+qmbNmmnx4sVav3690/5cPyeKfdnPP/+scePGqWbNmvL29naM5spqDojs9rd0dUTYm2++qdDQUMfxn5MyWfv371d0dLTKly/v6Ksnn3zSqWxMdHS0BgwYIElq3bq1o69uZ06m64+nY8eO6YUXXlDDhg1VokQJeXt7q2bNmnrttdd0+fJlp9deew5NmTLFcQ6NGDFCkvTnn3/qqaeeUp06dVS8eHH5+vqqbt26mjBhQobzy95Hq1at0tixY1W1alX5+PioXr16jl/W7969W506dVLx4sUVFBSk6OjoDDc0peydf9k97nP6uXbo0CH16tVLpUqVkp+fn6O84uzZs9W8eXOVKFFCvr6+qly5srp3767du3dnq5+yc3xI0pkzZ/Tiiy+qRo0a8vX1VXBwsOrWratnnnlG0v+dj4cOHVJMTIxTmSZ72azMzgf7stOnT2vw4MEqXbq0AgIC1K5dO+3du1eS9P3336tJkyby8/NThQoVNG7cuAz7sWnTJg0cOFB33HGH/P395e/vryZNmmj69OkZtnezY90wDE2dOlVNmzZ1rKtFixZOSdVrTZw40XFMhIWFacyYMbJardl6/z///HNZLJYsz+l7771X/v7+On/+vKTM50TJybmVU9dfA/3222+yWCx66aWXMm0/cuRIWSwW7dy585a2Z7FYVLZsWUnKMOrv7Nmzeu655xQWFiZvb2+VKVNGvXv3dhwn18rOd4jdhg0b1LlzZ0d5rXLlyql169aO/s7O9Vp22OOMjIzMNJkYEBDgNPrR3r5t27aZrs/+PgEAgFvjWj9HAgDABezZs0eSFBISotTUVI0fP17du3fXAw88oOLFi2v79u364osvtGrVKm3fvj3DL+J/++03zZs3TwMHDlSfPn104cIF3XfffUpLS9PYsWM1ZMgQRURESLpaTutGPv74YwUHB2vw4MEqV66c/vnnH02bNk2tW7dWTEyMWrRokTdvgqS77rpLLVu2VFxcnPbu3asaNWpIkpYvX66uXbuqUqVKGjFihMqUKaNt27bpv//9r+Lj47VmzZoMv5xt3769AgMD9cYbb+j06dOaMmWK7r//fi1ZskT333+/o90XX3yhwYMHq0GDBnrllVcUFBSk+Ph4vfrqq9qyZUuGm86XLl1SRESEGjVqpDFjxujkyZP64IMP1KFDB/39999Ov2Lv16+f5s6dqzZt2ui5555TUlKSRo0apcqVK2fY9/Pnz6tly5bat2+f+vfvr3r16unMmTOaOnWqmjdvrri4uAylpKZMmeJINoSEhGjBggUaN26cAgMDnRJRhw8fVnh4uI4ePao+ffro6aefVmpqqrZs2aIffvhBvXr1uuUYcuLKlSs6fPiwPDw8FBQUlGW7o0ePqkmTJjp16pQGDx6sevXqadu2bZo6darj18plypRRZGSkPvjgAz377LPq1q2bunfvLklON4myEh0drVmzZik8PFxjx47VxYsX9fnnn6t9+/b68ssv1bdvX4WEhGjWrFlasGCBFi5cqA8++EClSpXK1vpvxGKx6PHHH9eGDRv0ww8/qHnz5jd9zYsvvqjk5GT1799fISEhqlSpUpZts9vfdv3795fFYtHIkSPl5uamTz/9VH379lW1atXUrFmzG8a1detWtWrVSn5+fho4cKBCQ0O1d+9eTZo0SatWrdKmTZtUvHhxDR06VN7e3vrss8/0r3/9yzGnTLVq1bLxjmXu+uMpJiZG8+bNU9euXTVw4EAZhqG1a9dq3Lhx2rJli5YuXZphHR999JFOnjypxx9/XBUrVnScv2vXrtWaNWvUqVMnhYWFKSUlRUuXLtWLL76ov//+W59++mmGdb366qu6cuWKnnjiCbm7u+ujjz5Sly5dNG/ePA0aNEg9e/ZU586dtX79es2cOVPe3t6aMmWK4/XZPf+yc9zn9HPt4sWLioiIUJMmTfTWW2/pwoULCggI0OzZs9W3b1+Fh4dr1KhRCggI0NGjR7V69Wr99ddfql279g37KLvHhyT17NlTa9as0ZAhQ1S/fn2lpqZq//79+vnnnyXJcT4+++yzKlWqlF577TXHdkJCQm4YhyTdf//9KlOmjEaNGqVjx47pv//9r9q1a6cxY8bo+eef17BhwzRgwADNmTNH//rXv1SlShX17t3b8fqFCxdq586deuihhxQaGqpz587p22+/1cCBA5WQkOC46Z+dY33AgAH68ssv1aVLFz366KOSpAULFqhbt26aNGmSU0nLV155RePHj1ejRo00duxYXblyRdOmTdPixYtvus/29/Xpp5/WjBkzHNuy++eff7RmzRr16dPnhqNXt2/fnuNzKyeuvQZq3LixGjZsqJkzZ+rdd991SgakpKToq6++UvPmzVWnTp1srfvaZN3p06e1ePFiLV++XG3btnU6fi9cuKDw8HDt3r1bvXv3VsuWLbV//359+umnWr58ueLj453aZ+c7xL5vbdu2VenSpfXkk0+qfPnySkxM1ObNm7V+/Xp17dpV3bt3v6XrtevZj7EffvhBzz33nMqXL5+t9tOnT9f48eNdbuQZAAAFngEAAHJszZo1hiTj1VdfNRISEoyEhARj165dxssvv2xIMsLCwoyUlBTDZrMZly5dyvD6lStXGpKM999/32m5JEOSsWzZsiy3OX369ExjkmT079/fadnFixcztDt+/LhRsmRJo2PHjk7L+/fvb+Tk0iA0NNSoVq3aDduMGDHCkGQsWbLEMAzDuHz5slG2bFmjadOmRkpKilPbefPmGZKMGTNmZIipc+fORnp6umP54cOHjYCAAKNq1aqO5cePHzd8fHyMrl27GjabzWndEyZMMCQZa9eudSyLiooyJBljx451avvNN98YkowpU6Y4lq1atcqQZHTr1s1p3X///bfh6+trSDLWrFnjWP7MM88Ynp6exoYNG5zWfebMGaNixYpGq1atHMvs/Vq2bFnj9OnTjuXp6elGrVq1jHLlyjmt44EHHjAkGfPnzzeud+17lJMYbmTUqFGGJOOHH34wEhISjBMnThgbNmwwOnXqZEgy+vTpk6HtgQMHHMv69etnSDJmz57ttN6ZM2cakoxBgwY5lh04cMCQZIwaNSpbsRnG//VNhw4dDKvV6lh+6tQpo3Tp0kZQUJBx4cKFG8aYnf2Pi4vLss3mzZsNSUaPHj0cy6ZPn57huLAvq1atmlNMdpmdg9ntb3ucHTp0yHCueHp6Gr1793Z6bVRUlBEaGuq0rH79+kZYWJiRlJTktHzjxo2Gu7u7MXr06BvuX3Zk93hKTk522g+71157zZBkbNq0ybHMfg4FBQUZx48fz/CazD4HDcMw+vTpY7i7uzu9xr5f9erVc/qM2rJliyHJsFgsxty5c53W06VLF8PT09OpT3Ny/t3ouL/Vz7WXX345w7q6detmFCtWzEhNTc30/biZ7B4fZ8+eNSQZw4YNu+k6Q0NDjaioqEyfy+x8sC8bOnSo0/IPPvjAkGQEBAQ4ndspKSlGmTJljObNmzu1z+yYSE9PNyIiIozixYs7vUc3OtYXLVpkSDL++9//Zniuc+fORmBgoHH+/HnDMAxj7969hpubW4bvv6SkJKNcuXI3/H6/Vt++fQ03Nzfj8OHDTsvHjBljSDJWrVrlWJbZ511Ozq2sZPcayDAM47PPPjMkGfPmzXNax6xZs7K9z/bjOrN/AwYMyHA98cYbbxiSjHfffddp+dq1aw1JRtu2bR3LcvId8tFHHxmSMpzXWb0/N7peuzaGrHTu3NmQZHh5eRkRERHGiy++aMybN884c+ZMhrb79+83ihcvbkgySpcubfTo0cMYP368sW7dukz7GwAA5AzlvAAAuA3jxo1TSEiIQkJCdNddd2n8+PFq3bq1VqxYIW9vb1ksFvn5+Um6Wi7i7NmzSkxMVP369VW8eHFt3Lgxwzrr1avnNLridvj7+zseX7hwQUlJSfLw8NA999yT6bZzm/3XsOfOnZMk/fzzzzpx4oSio6N14cIFJSYmOv5FRkbKz89PP/30U4b1vPrqq04TxlaqVEn9+vXT33//rS1btkiS5s2bp5SUFA0ePFhJSUlO6+7UqZMkZVi3m5ubnn32Wadl9913n6T/+zWtJM2fP98Rx7XlZcLCwjL8GtgwDMeva6tVq+YUh9VqVbt27RQXF5ehbMrAgQMVHBzsFFvbtm11/PhxR6mg06dPa+nSpYqKinL8Yv36/bmdGG6kU6dOCgkJUdmyZdWsWTOtWLFCAwcO1GeffZbla2w2mxYtWqQ77rhDffr0cXquX79+qlatmhYsWCDDMLIdx/XsffPGG284zUESEhKip556SmfPns2y9Ftuuf44v5nhw4dnawRMdvv7Ws8++2yGc+WOO+5wOp4zs3PnTm3dulW9evWSzWZzOmaqVq2q6tWrZ3pu3qqbHU++vr6O/UhLS9Pp06eVmJjoOD8z+/zq379/piVrrv0cvHLlimNd999/v9LT0zMt3/fUU085TcZdv359BQYGqly5curZs6dT26ioKKWlpTlKJuXm+Xcrn2uS9PLLL2dYFhQUpOTkZC1ZskQ2m+2m275WTo4PX19f+fj4aOPGjfr7779ztJ3sev75553+joqKkiQ9+OCDTqXQvL29dc8992Q4/q89Ji5fvqykpCSdPn1a999/v86dO6e//vorW3HMmjVLvr6+euSRR5zek8TERHXt2lXnz593lPlbuHChbDabXnjhBadjq0SJEnrqqaeyve/R0dGy2Wz68ssvnZbPnDlToaGhjlJSWbmVcysrN7sGkuQYGTN16lSn106dOlXFixfXI488kq1tubm5aeXKlY5/c+fO1VNPPaWvvvpKffv2dSptN3/+fAUGBuq5555zWkdUVJRat26t1atX68yZM462Uva+Q+yjLhctWnTbpc+yY/78+frkk0/UqFEjbdiwQf/+97/10EMPqWzZsnr55Zed9rlq1aratm2bRo4cKX9/f82fP18vv/yyWrZsqerVq2vFihV5Hi8AAIUZYzwBALgN0dHRevTRR2WxWOTr66saNWpkKEWyaNEivf/++9q8ebNSU1Odnjt9+nSGddasWTPX4tu+fbvefPNNrV69WhcuXHB6LrO5F651+fLlDDeFr6/BfTP2uuz2Ei9//PGHJOnJJ5/Uk08+melrTp48mWFZZmVm7Mv27dunRo0aOdZtv7GYnXWXL19ePj4+TstKliwpSUpKSnIs279/f5Zx3HXXXU5/22+gxcbG3rAsTWJiolMZp6pVq2Zoc20sAQEB2rdvnwzDuGkZrpzGcP2Es+7u7hle98EHH6hOnTpyc3NTYGCgatWq5XQjMjMJCQm6cOFCpqVaLBaL7rrrLn3//fc6c+ZMjid6t7PfpK1bt26G5+zL7P2XV64/zm8mu+d4dvv7WlkdR4cOHbrh6+znz7hx4zKdQyKrdWcmN46n9PR0TZgwQTNmzNCePXsy3PTPyWdncnKyxowZozlz5jjNDXGjdWW2r8HBwZmWXrMnP+2fGbf6GZCZW/lcCwkJcUrI2r322mtat26devTooeDgYIWHh6tNmzbq06fPTUsN5eT48PLy0sSJEzVixAhVq1ZNNWvWVEREhDp27KguXbo43ai+Vdf3j31/s+q3az/Ppavv/ZtvvqlFixY5zc1kl9kxkZk//vhDly9fVoUKFbJsY++fnHyP3EibNm0UGhqqmTNnOsqgrVu3Tvv27dMbb7xx0+/2Wzm3spKdayB/f3/17dtXkydP1qFDhxQaGqq//vpLsbGxeuqpp+Tr6yvpahL6+sREiRIlHHOdWCwWx3wrdj179lS5cuX0+uuva+bMmRo4cKCkq98Ld911V4bvd+nq98KaNWt04MABBQcH5+g7pFevXvrmm2/03nvv6YMPPlDTpk0VGRmpXr16ZbskWU54eno6rpeuXLmirVu3asWKFfroo4/0/vvvKygoSK+++qqjfWhoqD766CN99NFHOnXqlNavX69vv/1W33zzjbp166Zt27apevXquR4nAABFAUkUAABuQ7Vq1TL8T/21Fi9erG7duqlx48b673//q8qVKztuGNh/0Xs9+8iV23XkyBG1bNlSAQEBevXVV3XnnXfK399fbm5uGjdunFavXn3D18+dO9cxoa7dqFGjNHr06GzHsHXrVknSHXfcIUmO/X333XfVtGnTTF+T2c2/7LCv+/PPP1doaGimba6vKX6jm3m3OjrCHkdkZKTeeOONLNtdf6MpN2PJaQzlypVzWh4aGprhZnPjxo3VsmXLHMVRVFx/nN9Mbp3jmcnqOLrZMWQ/ZkaMGKEHH3ww0zb2z66byY3j6YUXXtCHH36ohx56SC+//LJKly4tLy8vHT161PFL/Otl9b4++uijWrx4sQYPHqzIyEiVKlVKHh4e2rx5s1555ZVM15XV+5id8/RWPwMycyufa1m9D9WqVdOuXbu0du1arVq1SnFxcXrhhRf0xhtvaOnSpYqMjLxpHNk9Ph5//HE9+OCDWrZsmWJjY/Xzzz9r2rRpatq0qWJiYjK9uZ0Tt9I/doZhqH379tqxY4dGjBihJk2aKDg4WO7u7lq6dKk++OCDbI/UsdlsKl68uObNm5dlm5wkSLLDYrHoscce05gxY/TLL7+oRYsWmjlzZoYJ77NyK+dWVm52DWQ3bNgwffrpp5o2bZrefvttff7555Kuzjlj9/TTT2vmzJlOr1uzZs1NJ2Tv2LGjXn/9dcdotrzk5eWlZcuW6ffff9dPP/2kdevW6YMPPtDYsWP173//O8MIqdxkH1V1zz336OGHH1bt2rU1bdo0pyTKtUqXLq0uXbqoS5cuqly5st577z3NmTNHr7/+ep7FCABAYUYSBQCAPDRz5kz5+PgoJibG6cbWpUuXHKUksutmvy693oIFC3ThwgUtWrRIbdq0cXru2kl8s9K+fXutXLnSaVl2f4kuSbt27dK6det0xx13OCaVt/9S3MfHJ1s3Xux2796dYcLu3bt3S5LjV5X2dQcHB+do3dlhn7B19+7datKkidNzu3btcvo7JCREQUFBOnPmTK7HUb16dVksFkcJs6zkNIbr+zm7N8tvJiQkRMWKFcvwHklXb2Tu2rVLwcHBjsRZTo9x6f/6ZteuXbrnnnucntu5c6dTm7xgGIajTE3nzp1zdd3Z7e/ccO0ojuwcMzfqq9w4nmbOnKmIiAh99913TsuXLVuWo/WcO3dOixcvVt++fTOUntu7d2+O48qOnJ5/N3ovc/tzzdPTU/fdd5+jdNP27dvVuHFjvfnmm1q7du1N45Cyd3xIVyfSjo6OVnR0tAzD0L/+9S/HjVz7zf5bOedv144dO/T777/rjTfe0Ntvv+303PXHrnTz/vnzzz/VoEEDx8jBrFz7PXJ9YiWzz8gb6d+/v9555x3NmDFDDRo00LfffquIiIhsfUfn1rmVE3Xr1lWLFi30xRdf6NVXX9XMmTPVrFkzp9EfL730kmMCd7t69erddN1paWmS/m9EoHT1vd63b5+uXLniVDpNuvq9YLFYFBYW5mgr5ew7pGHDho4RgmfOnFGLFi30r3/9SyNGjJCXl1eeH9d33nmngoODdfTo0Wy1b9GihSRluz0AAMiIOVEAAMhD7u7uslgsGX7ZOWbMmBzXpLeX0cpuqQ37L3Kv/wX6smXLtGnTppu+vly5crr33nud/mU3ibJ371716NFDhmHovffecyxv3769ypQpo3//+98ZSv5IktVqzXT/xo0b5/R+/fPPP5o1a5bCwsLUoEEDSVfLevj4+Gj06NGOOUSudfny5QwlzbLLPh/FuHHjnN7PAwcOaPbs2U5t3dzc1LdvX+3YsSPDr2rtMitZlh0lSpRQx44dtXbtWi1evDjD8/b3KKcxXN/P4eHhtxTf9dzc3NS1a1f9+eefGX6pPXv2bO3fv1/du3d33HDK6TEu/V/fjB071ukYSUxM1CeffKKgoCC1bdv2dnclU6mpqXrqqae0YcMGde3aVc2aNcvV9We3v3ND/fr1VbduXU2bNs1RuulahmEoISHB8feN+io3jid3d/cMn11paWlZlpLKyrXzBF3rwoUL+u9//5vjuLK7zZycfzd6L3Pzc+3a/rOzl1G7vtzV9XJyfCQnJys5OdnpeYvF4rjpfO22AgICcnS+54asvhuPHTvmGCFxrRv1z2OPPSbpagIgs9Fe1/Zz165dZbFYNGHCBF25csWx/PTp0/rkk09ytA/VqlVTRESEvv32W82ePVvnz5/PMHI0K7l1buXU0KFDdfToUQ0bNkwJCQkaMmSI0/O1a9fO8NmRnZGp9u+WRo0aOZZ1795d586d0//+9z+ntnFxcVq9erXatGnjWHdOvkMSExMzbD84OFhVq1ZVamqq41y8le+y6+3bty/LuaxiYmJ0+vRpp9Jwa9euzXDe2S1cuFBS5qXkAABA9jASBQCAPPTwww9r3rx5ioqKcvwa96efftLu3btVqlSpHK2rdu3aKlasmD799FP5+fkpKChIpUuXzjDKxK5Dhw7y9/dXv3799NRTT6lUqVL6/fffNXv2bNWtW1c7duy47f27ePGivvrqK0lXb8LYa3AvXbpUHh4emjRpkrp27epo7+fnp1mzZqlLly6qVauWBgwYoDvvvFMXLlzQ/v37tWDBAr333nsZSpIcO3ZM9957r7p166bTp09r8uTJunz5sj7++GPHTdIKFSpoypQpGjhwoO644w71799fVatW1enTp/Xnn39qwYIFWrRo0U1Lg2Smbdu2euihhzRv3jzde++96tKli06fPq1Jkyapdu3a2rx5s1P7d999V7/88ouio6O1aNEiRUREyN/fX4cPH9aqVavk6+urNWvW5DgOSfrkk0+0ZcsWde/eXX369NE999yj9PR0bdmyRVar1dEfeRlDTowdO1Y///yzevfurTVr1qhu3bratm2bpk6dqkqVKundd991tC1ZsqSqV6+uOXPmqFq1aipTpoz8/f1vOMKjTZs26tevn2bNmqXWrVurW7duunjxoj7//HOdOnVKX375ZY7m8cnKihUrdPDgQRmGofPnz2vXrl2O+RQ6d+6sWbNm3fY2MpPd/r5dFotFX331ldq0aaOGDRsqOjpadevWdUyYvmjRIvXv399Rzq9JkyZyc3PTu+++qzNnzsjf319hYWEZfsl9qx5++GFNmjRJDz30kNq1a6fTp09r9uzZOR7VUqxYMd1///2aPXu2oxzO8ePHNW3atJvOA3I7cnL+3ei4z83Ptfbt26tYsWKKjIxU5cqVlZycrDlz5ujs2bM3LfGTk+Njz549ioyMVNeuXXXXXXcpJCREf//9tyZPnqxixYo5blpLUrNmzTRt2jS98cYbqlWrltzc3NS5c+ebzrd0O+68807VqVNH77//vi5evKi77rpLBw4c0JQpU1StWrUMN75vdKz36NFDjz/+uKZOnapt27apa9euKlu2rI4dO6bNmzdr6dKljpESNWrU0PPPP68JEyYoPDxcvXv3Vmpqqj7//HOVL18+07lZbiQ6OloDBw7U888/L39/fz300EPZel1unVs51bNnTz377LP68ssvczShvJ1hGE6fd2fOnFFMTIwWLFig0NBQPf30047nXnzxRc2fP18vvviitm3bphYtWmj//v369NNPVbx4cU2cONHRNiffIe+8846WL1+uTp06KSwsTB4eHoqJidHSpUvVqVMnx2ik7FyvHTx4UO+8806m+9q3b1/t3LlT3bt3V0REhFq1aqXQ0FBdvnxZ27Zt0+zZs+Xl5eX0I5UPP/xQa9asUadOndSoUSMFBwcrMTFRP/74o2JiYlSnTp08L3cGAEChZgAAgBxbs2aNIckYM2bMTdtOmzbNqFOnjuHj42OEhIQYffr0Mf755x8jNDTUiIqKcmoryejfv3+W6/rxxx+NBg0aGN7e3oYkp9dn9tp169YZkZGRRmBgoFGsWDGjTZs2xrp164z+/fsb118GZLbsRkJDQw1Jjn8+Pj5G2bJljdatWxtvv/22cfjw4Sxf+8cffxj9+/c3KlasaHh6ehqlSpUyGjVqZLz66qtOr7PHlJiYaERHRxulSpUyvL29jXvuucf46aefMl33hg0bjIceesgoU6aM4enpaZQpU8Zo3ry5MWbMGCMpKcnRLioqyggNDc10HZm9l1euXDFef/11o1KlSoaXl5dxxx13GJ988okxffp0Q5KxZs0ap/bJycnG2LFjjXr16hm+vr6Gv7+/Ub16dePRRx91it1+LE2fPj1DHKNGjTIkGQcOHHBafvz4cWP48OFGaGio4/2LjIw0vv3221uK4UbsMcTFxWW77fXx/vPPP8agQYOMcuXKGR4eHkb58uWNxx9/3Dh27FiGdWzcuNFo0aKF4efnZ0jKso+ulZ6ebnz44YdG3bp1DW9vbyMgIMBo3bp1pvuYVYw32yf7P3d3dyMoKMioV6+eMXjwYGP16tWZvi6z4yKrY8Uuq3MwO/19o/3K7FjP6vj/559/jKeeesqoWrWq4eXlZQQFBRl169Y1nn76aWPXrl1ObWfMmGHUqlXL8PT0vOln1/Vx3ux4Sk5ONl5++WUjNDTU8PLyMqpUqWK8+uqrxh9//GFIMkaNGuVoe6NzyDAMIykpyRg6dKhRoUIFw9vb27jjjjuM999/3/j5558zvO5GfZTZZ/aNXpOT8+9mx31ufK5NnTrVaN++vVGuXDnDy8vLCAkJMSIjI425c+dm2j4z2Tk+EhMTjWeffdZo0KCBERwcbHh7exuhoaFGdHS08ccffzit7+TJk0b37t2N4OBgw2KxOB3DOfmeOnDgQIbj4kavOXTokNGrVy+jdOnSho+Pj1GvXj1j2rRpWfblzY71r7/+2mjVqpVRvHhxw8vLy6hUqZLRoUMHY9KkSU7tbDab8cEHHxjVq1c3PD09jSpVqhhvv/22sXLlyhsew5m5cOGC4e/vf8NzL7PPhZycW1nJyTXQtZ599llDkvHkk0/m6HVRUVFOn8OSDC8vL6N69erGyJEjjVOnTmV4zenTp41nnnnG6XOzV69exl9//ZWhbXa/Q9asWWM88sgjRpUqVQxfX18jMDDQuPvuu43x48cbycnJTm1vdr12o38rV640EhISjA8++MDo2LGjY3ve3t5GWFiY8dhjjxlbt2512t6GDRuMF1980WjatKlRpkwZw8PDwyhWrJjRqFEj4+233zbOnz+fo/ccAAA4sxjGLc6aCgAAkMeio6M1c+bMW57kHQAAFAyvvPKKxo8fr23btunuu+82OxwAAIBsY04UAAAAAACQZ5KTkzVt2jQ1b96cBAoAAHA5zIkCAAAAAABy3c6dO7V161Z9/fXXSkxM1IwZM8wOCQAAIMcYiQIAAAAAAHLdvHnz1K9fP23dulXvv/++HnjgAbNDAgAAyDHmRAEAAAAAAAAAAMgEI1EAAAAAAAAAAAAyQRIFAAAAAAAAAAAgEyRRAAAAAAAAAAAAMkESBQAAAAAAAAAAIBMkUQAAAAAAAAAAADJBEgUAAAAAAAAAACATJFEAAAAAAAAAAAAyQRIFAAAAAAAAAAAgEyRRAAAAAAAAAAAAMkESBQAAAAAAAAAAIBMkUQAAAAAAAAAAADJBEgUAAAAAAAAAACATJFEAAAAAAAAAAAAyQRIFAAAAAAAAAAAgEyRRAAAAAAAAAAAAMkESBQAAAAAAAAAAIBMkUQAAAAAAAAAAADJBEgUAAAAAAAAAACATJFEAAAAAAAAAAAAyQRIFAAAAAAAAAAAgEyRRAAAAAAAAAAAAMkESBQAAAAAAAAAAIBMeZgeQH86ePavk5GSzwwAAIM/4+fkpKCjI7DBwC7hOAQAUdlynuC6uUwAAhVl2r1EKfRLl7Nmz+vjjj2W1Ws0OBQCAPOPh4aHhw4dzg8LFcJ0CACgKuE5xTVynAAAKu+xeoxT6JEpycrKsVqsaNGiggIAAs8MBACDXXbx4UVu2bFFycjI3J1wM1ykAgMKO6xTXxXUKAKAwy8k1SqFPotgFBARwwQYAAAokrlMAAEBBxXUKAKCoY2J5AAAAAAAAAACATJBEAXBbTp48qQcffFCpqalmhwIAAJDrXnjhBa1atcrsMAAAQCG2du1avfbaa46/H3zwQR05csTEiABcq8iU8wIAAAAAAACAgqZVq1Zq1apVps99+OGHCg4OVv/+/fM3KAAOjEQBAAAAAAAAAADIBCNRAGRbUlKSvvjiC23fvl1Wq1V16tTR4MGDndqsXr1a8+fPV2JiogIDA9W1a1c98MADkqQRI0bokUceUcuWLR3tn3rqKfXp00fh4eH5ui8AAMD1LVy4UEuWLNGlS5cUGBiofv36qWbNmvr444914MABSVL9+vX1xBNPKCAgQJI0ePBgPfDAA4qNjdWxY8dUu3ZtPf/8847nY2NjNWvWLF28eFH33XefafsGAAAKp8yuX9LS0rRs2TJNmDDBqe3SpUsVExMji8WiH3/8UVWrVtV7771nUuRA0UUSBUC2pKen65133lHNmjU1ZcoUeXl56c8//8zQLjAwUK+99prKlSun3bt3a9SoUapZs6Zq1Kihtm3bavXq1Y4kyt69e3X27Fk1bdo0v3cHAAC4uCNHjmj27Nn68MMPVbFiRZ0+fVoXL16UJPXo0UN16tTR5cuX9d577+mrr77SsGHDHK9du3atXn/9dQUEBGjUqFFavHixHn30UR05ckQTJ07U66+/rjp16mj+/Pnat2+fWbsIAAAKmayuX/bu3Ztp+44dO2rPnj2U8wJMRjkvANmyd+9enTx5UoMHD5afn588PDxUp06dDO0aN26s8uXLy2Kx6K677lLDhg21c+dOSVdrfG7fvl1nz56VdHXUSkREhDw9PfNzVwAAQCHg7u4uSTp8+LCuXLmiEiVKqHLlyipbtqwaNGggT09PBQYG6sEHH9SuXbucXvvggw8qJCREvr6+atGihfbv3y9JWrdunRo1aqT69evLw8NDDz30kGOECgAAwO3K6voFQMHGSBQA2ZKYmKiQkJCbJjw2b96sb775RseOHZNhGLpy5YoqVKggSQoKClKDBg0UExOjBx54QHFxcXrzzTfzI3wAAFDIlCtXTs8884yWLFmijz76SHfddZcGDhwof39/ff7559q1a5cuX74swzDk6+vr9NqgoCDHY29vb6WkpEiSTp8+rZCQEMdz7u7uKlmyZL7sDwAAKPyyun4BULCRRAGQLaVKlVJCQoKsVqs8PDL/6EhLS9O4ceM0cuRItWjRQh4eHho7dqxTm7Zt22rOnDkqU6aMihcvrpo1a+ZH+AAAoBBq2bKlWrZsqStXrmjmzJn6+OOPVb58edlsNk2cOFGBgYHasGGDPv3002ytr0SJEo65VKSr5UyTkpLyKnwAAFAEZXb9cqN52CwWSz5GByAzlPMCkC01atRQSEiIpk2bpuTkZFmtVkeZLru0tDRZrVYVL15c7u7u2rJli7Zs2eLUpkmTJkpKStLXX3+tNm3a5OcuAACAQuTIkSPaunWrUlNT5eHhIR8fH7m5ueny5cvy8fGRv7+/kpKStGjRomyvMzw8XJs3b9a2bdtktVo1f/58xzwrAAAAtyur65cbCQoK0okTJ/IpQgCZYSQKgGxxd3fXG2+8oc8//1xDhgyRzWZT3bp1nYad+vn56fHHH9eECRNktVrVpEmTDJPGu7u7q1WrVlqyZIlatWqVz3sBAAAKi7S0NH311Vf6559/5ObmpqpVq+rJJ5+UzWbTBx98oN69e6tcuXJq1aqVFi5cmK11VqpUSSNGjND//vc/Xbp0Sffee6+qV6+ex3sCAACKiqyuX/76668sX3Pfffdp/Pjx6t27t6pWrap33303HyMGIEkWwzAMs4PIS8eOHdNnn32miIgIp9rHAMyzZMkS/fbbb3rrrbfMDgUoFM6ePau4uDgNGTJE5cuXNzsc5ADXKQCAwo7rFNfFdQoAoDDLyTUK5bwA5Kvk5GQtX75c999/v9mhAAAAAAAAAMANkUQBkG9+/vln9e/fX1WrVlWzZs3MDgcAAAAAAAAAbog5UQDkm3vvvVf33nuv2WEAAAAAAAAAQLYwEgUAAAAAAAAAACATRWYkysWLF80OAQCAPMF3nOujDwEAhRXfca6PPgQAFEY5+X4r9EkUq9UqSdqyZYvJkQAAkLfs33lwHVynAACKCq5TXA/XKQCAoiA71yiFPoni4XF1F1u3bq3g4OB82667u7uKFSumCxcuKD09Pd+2i5yjr1wL/eVa6K/8cebMGa1Zs8bxnQfXYcZ1Cuela6G/XAv95Vror/zBdYrr4joFN0N/uQ76yrXQX/kjJ9coReYqpkaNGipfvny+bc8wDFmtVlWqVEkWiyXftpuXbDabTpw4obJly8rNrfBMp1MY+0qiv1wN/eVaClp/HTt2TGvWrDE7DNyG/LxO4bx0LfSXa6G/XAv9lT+4TnF9XKfcvoJ2XuYW+st10Feuhf7KHzm5RjE/WgAAAAAAAAAAgAKIJAoAAAAAAAAAAEAmSKIAAAAAAAAAAABkgiQKAAAAAAAAAABAJkiiAAAAAAAAAAAAZIIkCgAAAAAAAAAAQCZIogAAAAAAAAAAAGSCJAoAAAAAAAAAAEAmSKIAAAAAAAAAAABkgiQKAAAAAAAAAABAJkiiAAAAAAAAAAAAZIIkCgAAAAAAAAAAQCZIogAAAAAAAAAAAGSCJAoAAAAAAAAAAEAmSKIAAAAAAAAAAABkgiQKAAAAAAAAAABAJjzMDiA/BAQEyMPDQ4Zh5Ns27dvKz23mNcMwHO9jYduva/9bWNBfroX+ci0Frb88PIrE1zkAAAAAAEC+KxJ3XRo0aKDg4GBZrdZ833Z6enq+bzMvBQcHy2azyWazmR1KritsfSXRX66G/nItBam/goODzQ4BAAAAAACgUCoSSZQtW7aobt26CgkJybdtGoah9PR0ubu7y2Kx5Nt285LNZlNSUpJKliwpN7fCUwmuMPaVRH+5GvrLtRS0/kpISDA7BAAAAAAAgEKpSCRRLl68KKvVasoNPIvFUmhuHFosFsf7WFj26VqFbb/oL9dCf7mWgtZfZoy0BAAAAAAAKAqKRBIFAAAAAAAAOVO2bNl8nWOWORVdC/3lOugr10J/5Y+czC9LEgUAAAAAAAAZDBo0SFL+j3xmTkXXQn+5DvrKtdBfeR9LdpFEAQAAAAAAQAbTpk1T9+7d822OWeZUdC30l+ugr1wL/ZU/cjK/LEkUAAAAAAAAZHDixAlT5pgtKHMP5paCNqdibits+1WY+6uw7VNh7iuJ/sprORllaX7KBwAAAAAAAAAAoAAiiQIAAAAAAAAAAJAJkigAAAAAAAAAAACZIIkCAAAAAAAAAACQCZIoAAAAAAAAQCGVlpamtLQ0s8MAAJdFEgUAAAAAAAAoZM6fP68uXbrI29tb3t7e6tatmy5cuGB2WADgckiiAACQjwzDUEJCgtlhAAAAACjk+vbtq+XLl8swDBmGoaVLl6pfv35mhwUALsfD7AAAACjM0tLStG3bNsXHx+uXX35RfHy8zpw5o3PnzsnDg69h5L7jx4/Ly8tLJUuWNDsUAAAAmOTKlSv64YcfZBiGY1lqaqq+//57paamysvLy8ToAMC1cPcGAIBcdPr0aW3YsMGRNNm0aZOSk5MztNu2bZsaNWpkQoQorJKTk1W/fn3t3btX//nPf/Tcc8+ZHRIAAABMYh99ktVzAIDsI4kCAMAtMgxDe/bscYww+eWXX7R79+4s24eFhalFixYKDw9XxYoV8zFSFAV+fn5yc7taqTUmJoYkCgAAQBHm4+OjDh06aNWqVUpNTZUkeXl56d5775W3t7fJ0QGAayGJAgBANqWkpOi3335TfHy8Vq9erd9//12JiYmZtvX09FTDhg0dSZMWLVqoXLly+RwxiprIyEj99ddfiouLk81mcyRVAAAAUPR8/fXX6t27t5YvXy5Jatu2rWbPnm1yVADgekiiAACQhZMnTzrNZbJ582alpaVl2rZkyZJOCZPGjRvL19c3nyNGURcVFaWpU6fqzJkz2rlzp+6++26zQwIAAIBJgoKCtGzZMl2+fFmS+P8TALhFJFEAANDV0lx//PGHYmNjHUmTv//+O8v2NWrUUEREhFq2bKkWLVqoZs2aslgs+RgxkFFkZKTjcWxsLEkUAAAAkDwBgNtEEgUAUGQdOnRIq1at0qpVq7R69WqdOHEi03a+vr5q2rSpY6RJ06ZNlZaWprJly1IuCQVKpUqVFBYWpgMHDigmJkbDhw83OyQAAAAAAFwaSRQAQJFx6tQprVmzxpE4yWqkSfny5R1lucLDw1W/fn15eno6nrfZbFkmXACzRUZG6sCBA4qNjZVhGIyQAgAAAADgNpBEAQAUWufPn1dsbKwjabJjx45M25UtW1Zt27ZVmzZt1Lp1a1WpUoUbz3BZUVFRmjlzpk6dOqW//vpLd955p9khAQAAAADgskiiAAAKjZSUFP3yyy9avXq1Vq1apV9//VXp6ekZ2gUFBalVq1aOxEmtWrVImqDQuH5eFJIoAAAAAADcOpIoAACXZbVatXnzZsecJvHx8UpJScnQztfXVy1btnQkTRo2bCh3d3cTIgbyXtWqVVWhQgUdPXpUsbGxGjJkiNkhAQAAAADgskiiAABchmEY2rVrl6M8V0xMjM6fP5+hnYeHh5o2baq2bduqbdu2atasmby9vU2IGMh/FotFkZGR+uabbxQTE8O8KAAAAAAA3AaSKACAAu3vv/92lOdavXq1Tp06lWm7+vXrq02bNmrbtq0iIiJUrFixfI4UKDiioqL0zTff6MiRIzp48KDCwsLMDgkAAAAAAJdEEgUAUKAkJiZq5cqVjtEmBw8ezLRdjRo1HEmT1q1bq1SpUvkbKFCAXTsvSkxMDEkUAAAAAABuEUkUAICpbDabNm/erKVLl2rZsmXatGmTDMPI0K58+fKOOU3atGmjypUrmxAt4BruvPNOhYSEKCEhQbGxsYqOjjY7JAAAAAAAXBJJFABAvktKStKKFSu0bNkyLV++XAkJCRnaBAcHq3Xr1o7EyR133MG8DkA22edFmT9/vmJiYswOBwAAAAAAl0USBQCQ52w2m7Zu3aply5Zp6dKl2rhxo2w2m1Mbi8Wipk2bqkOHDurQoYMaNWokd3d3kyIGXJ89ifL333/ryJEjqlixotkhAQAAAADgckiiAADyREpKilasWKEFCxZo+fLlOnnyZIY2JUuW1P33368OHTqoffv2zGsC5KKoqCjH49jYWPXp08fEaAAAAAAAcE0kUQAAuebcuXNaunSpFixYoGXLlunSpUsZ2jRp0kQdOnRQx44d1bhxY0abAHmkTp06CgoK0tmzZ0miAAAAAABwi0iiAIVIWlqa1q9fr/Pnz6t+/fpyc3MzO6RMXbp0SevXr5fValXTpk1lsVi0ceNGubu7q3nz5goICHC0PXXqlH777Tf5+vqqRo0a2rZtmzw9PRURESFfX18T9yLnru2fhg0b6vLly9q1a5dKly6tJk2a3NI6DcPQpk2bdPLkSd11112qVq1aLkd9cydPntTixYu1cOFCrVq1SmlpaU7P+/n5qXHjxurdu7e6d++u0qVL33B9f/zxh/bs2aPKlSurQYMGeRm6k7S0NP3yyy+6cOGCGjZsqPLly+fJdpKTk7V+/XqlpqaqadOmKlmyZJ5sB3B3d1dERISWLFnCvCgAAAAAANwikihAIXH27Fnde++9+v333+Xu7i53d3dNnjxZjz32mNmhOTlw4IBatWqlI0eOyGKxyM/PTxaLRZcuXZJhGKpQoYLWrFmjatWqac2aNercubNSUlKUnp4uNzc3WSwWGYah6tWra+3atSpXrpzZu5QtZ86cUdu2bbV161bHyIv09HS5u7vLarXqgQce0CeffJKjdaalpal79+764Ycf5OHhIZvNpo8//lhPPPFEXuyCkwMHDmjhwoVauHCh4uPjZRiG0/OlS5dW69attWrVKiUlJSk+Pl67du3SPffcc8MkyltvvaXRo0fLw8NDVqtVAwcO1Oeff57nE8pf3z8eHh6aN2+eHnjggVzdzuHDh9WqVSsdOnRIFotFAQEBWr58uZo1a5ar2wHsIiMjtWTJEv355586derUTZOYAAAAAADAWcH8mTqAHHvmmWe0Y8cOGYYhq9WqK1euaNiwYUpISDA7NCe9e/fWsWPHZLPZlJ6ergsXLuj8+fNKT0+XzWbT8ePH1bNnT126dEldunTRpUuXlJ6eLkmO19hsNh04cEADBgwweW+y7+mnn9auXbsc/WO1Wh2PJWnFihWaNGlSjtY5YcIE/fTTT5Ikq9Uqm82m4cOHa/v27bkev2EY2rFjh95++201aNBAVatW1fPPP69169Y5EihVqlTRs88+q7i4OB07dkx79+7V2bNnZRiG0tPTdebMGT344IMZEi52P//8s95++23H/kjSl19+qenTp+f6/lxvxIgRTv2TkpKihx9+WElJSbm6nb59++rIkSOOY/n8+fN68MEHM4zeAXLL9fOiAAAAAACAnCGJAhQSsbGxSk1NdVqWmpqqXbt2mRRRRoZh6LfffnPcIM+M1WrV1q1b9eeff+rChQtZtktLS9OGDRvyIsw8kVn/XCstLU0bN27M8Tqvv/nu5eWlTZs23VKM17PZbFq/fr1eeukl1axZU3fffbdGjRqlrVu3OtrUqVNHb775prZs2aK///5b//3vf9WyZUtJ0pYtW5z62maz6ciRI1km9jZs2CAvLy+nZenp6Vq/fn2u7M+NxMXFZeify5cva/fu3bm6nU2bNjn1mWEYSkhI0JEjR3J1O4BdgwYNHCUSSaIAAAAAAJBzlPMCCokSJUrowIEDTssMw1BQUJA5AWXCYrGoWLFiOnv27A3b+fn5qVSpUjddX2BgYC5FlvdKlCihQ4cOZfm8m5ubgoODc7TOUqVKyc3NTTabzbEsPT09x+u5VlpamtauXauFCxdq0aJFOn78eIY2zZo1U/fu3dWtWzdVr1490/W4ubnJ399fFy9edFpuPwYyExwc7LQvkuTh4XFb+5NdJUqU0OHDhzMsz+3zJzAwMNMkUvHixXN1O8g9Fy9e1CeffKLff/9dvr6+6tatm7p06ZJp29TUVM2cOdORNC1fvrzeffdd+fn55XPU/8fDw0Ph4eH66aefmBcFAAAAAIBbwEgUoJAYPXq000TyXl5eioyM1N13321iVBmNHj3aMSeIdPWm+rVxu7u7a9SoUQoNDVXPnj3l6emZ6Xrc3NwcpZ9cwfX9I8kxz4ebm5vc3d01bNiwHK3z+eefl7u7u2O9Xl5eql69ujp27Jij9SQnJ2vhwoXq16+fSpcurXbt2mnSpEmOBIqHh4fuvfdeffrppzp69KjWr1+vF198McsEin3fRo0a5dTXHh4eeuaZZ+Tr65vpa3r37q1SpUo5+tzDw0Pe3t75MsdLZufP/fffrzp16uTpdjw9PfX444+rRIkSubod5J4pU6YoLS1N06dP1+jRozVv3jxt3rw507affvqpkpKSNHHiRH3zzTcaOXJklp9h+SkyMlKStGPHDp05c8bkaAAAAIqm9PR0jR49WmXKlFGJEiUUHR2tS5cumR0WACAbGIkCFBKdOnXSkiVL9N577+ncuXNq27athg8fnuHGvdlGjhypwMBATZ48WVarVT169JC7u7u+++47ubm5aciQIRo0aJAk6auvvtJbb72lH374Qf7+/ipfvrz27dsnHx8fPf300+rVq5fJe5N9Dz74oL7//nuNHz9e586dU8uWLXXx4kVt3rxZ5cuX15gxY1SpUqUcrbN+/fqKj4/Xa6+9puPHj6tJkyaaMGFClkmKa505c0Y//PCDFixYoJ9++kmXL192et7X11ft27dX9+7d1alTp1saDfL8888rMDBQU6dOlc1mU69evfT8889n2b5EiRLatGmTnn/+ee3cuVNhYWH697//rbCwsBxvO6e6dOmixYsXa/z48Tp//rzatWunMWPG5PqE9k8++aQCAgI0adIkpaamqnv37nrllVdydRvIPSkpKYqPj9cHH3wgPz8/ValSRe3atdPKlSvVqFEjp7ZHjhzR+vXrNW3aNEf5rPw4drPDnkQxDEPr1q1Tp06dTI4IAACg6JkwYYImTZrkKO/7zTffKCkpSUuWLDE5MgDAzZBEAQqRjh07OkYh2Gw2nThxwuSIMrJYLBowYECGSeFffvnlDG09PT31zjvv6J133nEss0/87eHheh9fDzzwgB544IFMn7vV/mrSpIlWrFiRrbbHjh3TokWLtHDhQq1duzbD3DRBQUHq3LmzunXrpvbt2992CSKLxaLHH39cAwYMkIeHR7YSEhUqVNCcOXNua7u3qlOnTvlyc/mxxx7TY489lufbwe07evSoDMNQaGioY1lYWFim8/Ts3btXpUuX1ty5c7VmzRoFBgaqa9euateuXYa2iYmJSkxMlCQlJCQoPT1dkjKUs8stjRo1ko+Pj1JSUhQTE6OOHTvKZrPJZrPleqLQTPb3L6/eR7MYhkF/uRD6y7XQXwDy04wZM5zmR0xNTdUPP/ygU6dOqXTp0iZGBgC4Gde7CwkAyLa9e/dq4cKFWrhwoTZs2JDh+XLlyqlr167q1q2bWrVqVSBKDwEFRUpKSoZkor+/f4aRW9LVZMihQ4fUtGlTTZ8+XQcPHtSbb76p8uXLZygLN3/+fE2dOtXxt31UXV4mvhs0aKD169dr9erVOnnyZJ5tpyA4deqU2SEgB+gv10J/uRb6CyhYUlNTM11+5cqVfI4EAJBTJFEAoBAxDENbt251JE527tyZoU316tXVrVs3devWTffcc0+BK/kGFBQ+Pj4ZEibJycmZlszz9vaWm5ubevXqJU9PT9WoUUPh4eH69ddfMyRRevTooaioKElXky/20WRly5bNoz2R7r33Xq1fv17bt2+Xv7+/fHx8sj1CzFXYbDbHLzkL0+fatSMw6a+Cj/5yLfRX/iiIo+MBM0RFRWnNmjWOZIqHh4eqVKmiChUqmBwZAOBmSKIAgItLT0/XL7/84kicHDx4MEOb+vXrOxInderUKVQ3CoC8Yv8f2sOHD6ty5cqSpAMHDjgeX6tKlSrZXm+pUqVUqlQpSVJgYKDc3d0lKU9vdEVFRWnMmDFKT0/Xxo0b1bp1a7m5uRXKzwI3N7cCcdMwtxiG4dgn+qvgo79cC/0FID/95z//0ZAhQ/TLL79IkipXrqylS5dyngKACyCJAgAuyDAMbdiwQXPmzNF3332n48ePOz1vsVgUHh7uSJwUlAmuAVfi4+Oj8PBwzZo1S88++6xj1MjTTz+doW2dOnVUtmxZfffdd3rkkUd08OBBxcfH67XXXjMh8oyaNWsmDw8PWa1WxcTEqHXr1maHBAAAUKQEBwcrNjZWhw8fVlpamqpWreqSc30CQFHEpzUAuAjDMLRlyxbNmTNH3377rQ4dOuT0vKenp9q2batu3bqpS5cuKlOmjEmRAoXH0KFD9fHHHys6Olq+vr7q0aOHGjVqJEnq2bOnRo0apbvuukvu7u56/fXX9fHHH2vhwoUqUaKEBg0alKGUl1n8/f3VuHFjbdiwQXFxcWaHAwAAUCRZLBZ+4AYALogkCgAUcLt27dKcOXM0d+5c7d271+k5T09PtW/fXo888og6d+6s4sWLmxQlUDgFBATolVdeyfS5b7/91unvihUr6r333suPsG5JVFSUNmzYoI0bNyolJUUBAQFmhwQAAAAAQIFHEgUACqB9+/Zp7ty5mjNnTobJ4d3d3dWmTRv16tVL3bp1U3BwsElRAnAlkZGRGj9+vFJTU/Xrr79S0gsAAAAAgGwgiQIABcTRo0c1e/ZszZ07V5s3b3Z6zmKxKCIiQr169VKPHj1UunRpk6IE4KrCw8NlsVhkGIbi4uJIogAAAAAAkA0kUQDARCdOnNB3332nOXPm6Jdffsnw/D333KNevXrp4YcfVoUKFUyIEEBhUbx4cdWvX19btmzRunXrzA4HAAAAAACXQBIFAPLZ6dOntWDBAn3zzTdau3atbDab0/P169dXr1691LNnTyYdBJCrIiMjtWXLFq1fv15paWny8vIyOyQAAAAAAAo0kigAkA8Mw1BsbKwmTpyoJUuWKC0tzen5O++8U506ddLAgQNVq1Ytk6IEUNhFRUXpo48+0qVLl/T777+rWbNmZocEAAAAAECBRhIFAPLQ5cuX9fXXX2vixInavn2703NhYWHq1auXHnnkEdWpU0cnT55U2bJlTYoUQFHQsmVLx+PY2FiSKAAAAAAA3ARJFADIA4cPH9akSZP02Wef6fTp047lAQEBeuyxx/TYY4+padOmslgskpShpBcA5IWQkBDVrl1bu3fvVlxcnF566SWzQwIAAAAAoEAjiQIAucQwDMXFxel///ufFi5cqPT0dMdz1atX14gRIxQdHa3AwEATowRQ1EVERDiSKOnp6XJ3dzc7JAAAAAAACiw3swMAAFd3+fJlffHFF2rQoIGioqI0b948RwKlffv2+vHHH/XXX39p5MiRJFAAmC4yMlKSdO7cOe3cudPkaAAAAAAAKNgYiQIAt+iff/5xlOxKSkpyLPf391d0dLSGDx+uO++808QIASAjexJFkmJiYlSvXj0TowEAAAAAoGBjJAoA5IBhGFq3bp169uypsLAwjRs3zpFAqVq1qj744AMdPXpUH3/8MQkUAAVShQoVVK1aNUlXJ5cHAAAAAABZYyQKAGRDSkqK5syZo4kTJ2rLli1Oz913330aOXKkOnTowNwCAFxCy5YttX//fsXGxsowDFksFrNDAgAAAACgQGIkCgDcwNGjR/X666+rUqVKGjBggCOB4ufnpyeeeEK7du3SihUr1KlTJxIoAFxGRESEJCkhIUF//fWXydEAAAAAAFBwMRIFAK5jGIZ++eUX/e9//9P8+fNltVodz4WFhWn48OEaOHCggoKCzAsSAG5Dy5YtHY9jYmIoPwgAAAAAQBYYiQIA/19KSopmzpypxo0bq2XLlpo7d64jgdK2bVstXrxYe/fu1XPPPUcCBYBLCwsLU4UKFSRJcXFxJkcDAAAAAEDBxUgUAEXesWPHNGnSJE2ZMkUJCQmO5b6+vnrsscc0YsQI3XXXXSZGCAC5y2KxKDIyUt98841iYmKYFwUAAAAAgCyQRAFQJBmGoQ0bNmjixImaN2+eU8muKlWqOEp2BQcHmxglAOSdiIgIffPNNzpy5IgOHTqkKlWqmB0SAAAAAAAFDkkUAEXKlStX9O2332rixIn67bffnJ5r06aNRo4cySTxAIqEyMhIx+PY2FiSKAAAAAAAZIIkCoAi4fjx45o8ebImT56sU6dOOZb7+vqqX79+Gj58uOrWrWtihACQv2rVqqWSJUsqKSlJcXFxeuyxx8wOCQAAAACAAockCoBCbePGjZo4caK+/fZbp5JdlStX1vDhwzVo0CCVKFHCxAgBwBwWi0URERFatGiRYmNjzQ4HAAAAAIACiSQKgEInNTVV3333nSZOnKhNmzY5PdeqVSuNHDlSnTt3locHH4EAirbIyEgtWrRIe/bs0YkTJ1S2bFmzQwIAAAAAoEDhDiKAQuPEiROaMmWKJk+erBMnTjiW+/j4qG/fvhoxYoTuvvtuEyMEgILl2nlR4uLi9PDDD5sYDQAAAAAABQ9JFAAub9OmTY6SXWlpaY7llSpV0lNPPaXBgwerZMmSJkYIAAVTvXr1VKxYMV24cEGxsbEkUQAAAAAAuA5JFAAuKTU1VfPnz9fEiRO1ceNGp+eioqI0cuRIPfjgg5TsAoAb8PDwUIsWLfTTTz8xLwoAAAAAAJng7iIAl3Ly5ElNmjRJn332WYaSXY8++qhGjBihevXqmRghALiWyMhI/fTTT9qxY4fOnDmj4OBgs0MCAAAAAKDAIIkCwCUkJiZq3Lhx+uSTT3TlyhXH8ooVKzpKdpUqVcrECAHANdnnRTEMQ/Hx8erUqZPJEQEAAAAAUHCQRAFQoF26dEn/+c9/NGHCBF24cMGxPCIiQiNHjlTXrl0p2QUAt6FJkyby9vbWlStXFBsbSxIFAAAAAIBruJkdAABkxjAMzZ49W3fccYdGjRrlSKB07NhRGzZsUExMjB566CESKABwm7y9vdWsWTNJYl4UAAAAAACuQxIFQIHz66+/Kjw8XH379tXRo0clSeHh4YqNjdUPP/yghg0bmhwhABQu9pJemzdv1sWLF02OBgAAAACAgoMkCoAC4/jx44qOjlbTpk21fv16SVLlypU1d+5cxcXFKSIiwuQIAaBwsn++Wq1WbdiwweRoAAAAAAAoOEiiADCd1WrV+PHjVbNmTc2cOVOS5Ofnp7ffflt//vmnevbsKYvFYnKUAFB4NW/eXO7u7pKkuLg4k6MBAAAAAKDgYDIBAKY6cOCA+vbtq19++cWx7NFHH9V7772nihUrmhgZABQdAQEBatSokTZt2sS8KAAAAAAAXKNAjES5ePGixo8fr0ceeUTR0dFavHjxTV+zatUqPfjgg1q2bFk+RAggL3z99deqX7++I4HSoEEDxcfH66uvviKBAgD5zD4vyoYNG3TlyhWTowEAAAAAoGAoEEmUKVOmKC0tTdOnT9fo0aM1b948bd68Ocv258+f17x581S5cuV8jBJAbrFarRowYIAeffRRnT9/XhaLRa+88oo2bNigFi1amB0eABRJ9iRKSkqKfvvtN5OjAQAAAACgYDA9iZKSkqL4+Hj169dPfn5+qlKlitq1a6eVK1dm+Zrp06erS5cuCgwMzMdIAeQGm82mQYMGacaMGZKk8uXLa+XKlRo3bpy8vLzMDQ4AirDw8HDHY0p6AQAAAABwlelJlKNHj8owDIWGhjqWhYWF6fDhw5m237lzp/755x+1a9cuv0IEkEsMw9Dzzz+vL7/8UpLUokULbdu2TW3btjU5MgBAiRIlVLduXUkkUQAAAAAAsDN9YvmUlBT5+fk5LfP399fly5cztE1LS9PkyZP17LPPys0t6/xPYmKiEhMTJUkJCQlKTU2VdPUX8PnFMAzZbDbZbDZZLJZ8225esr9/+fk+5ofC2FdSweyvsWPH6sMPP5Qk3X333VqyZImCgoJyFCP95VroL8C1REZGaseOHYqPj1d6errc3d3NDgkAAAAAAFOZnkTx8fHJkDBJTk6Wr69vhrYLFixQnTp1VK1atRuuc/78+Zo6darj76ioKEnSiRMnciFinDp1yuwQkAMFpb9mzpypN954Q5JUpUoVzZw5UykpKZyX1yko/YXsob9Q2ERGRuqTTz7RhQsXtG3bNjVs2NDskAAAAAAAMJXpSZQKFSpIkg4fPuyYKP7AgQOZThq/bds2HTp0SL/88osk6eLFi/r777+1Z88ePf300452PXr0cCROEhIStGzZMklS2bJl83RfrmUYhqxWqzw8PArNr69tNptOnTql0qVL33AkkKspjH0lFaz+mjNnjl577TVJUrly5fTzzz8rLCzsltZFf7kW+it/kIxEbomIiHA8jo2NJYkCAAAAACjyTE+i+Pj4KDw8XLNmzdKzzz6rhIQErVixwikpYvfqq6/KarU6/h43bpzuuecetW/f3qldqVKlVKpUKUlSYGCgY7Lq/LzRZRiG3Nzc5ObmVqhuHEpy7FdhUZj7SjK/v5YvX67+/fvLMAwFBQXpp59+uuloshuhv1wL/QW4lnLlyql69erat2+fYmNj9cwzz5gdEgAAAAAApioQd36GDh0qd3d3RUdH680331SPHj3UqFEjSVLPnj21a9cuSVKxYsUUHBzs+Ofh4SE/Pz8FBASYGT6ALPzyyy/q0aOHrFar/Pz8tHTpUsekxQCAgikyMlLS1ZEohmGYHA0AAAAAAOYyfSSKJAUEBOiVV17J9Llvv/02y9eNHTs2r0ICcJt27NihBx54QMnJyfL09NSCBQvUvHlzs8MCANxEZGSkvvjiCyUlJemPP/5Q7dq1zQ4JAAAAAADTFIiRKAAKl7///lvt27fX2bNnZbFYNGvWrAxl9wAABZN9JIokxcXFmRgJAAAAAADmI4kCIFedOHFC7dq10/HjxyVJn376qR555BGTowIAZFeVKlVUsWJFSVdLegEAAAAAUJSRRAGQa86ePav27dtr//79kqR33nlHw4YNMzkqAEBOWCwWx2iUmJgY5kUBAAAAABRpJFEA5Irk5GR16tRJ27dvlyQ9++yz+te//mVyVACAWxERESFJOnr0qA4ePGhuMAAAAAAAmIgkCoDblpaWpoceekjx8fGSpP79+2vChAmyWCwmRwYAuBXXzotCSS8AAAAAQFHmYXYAAFybzWZTdHS0li1bJkl68MEH9fnnn8vNjRwtAOSnrVu3au7cubJarerUqZOioqJks9k0a9Ys/f777ypdurSGDBmiUqVK6fjx45oxY4ZOnz6tpk2bqnfv3k6J71q1aqlUqVJKTExUbGysqlSpoh9++EEWi0Vubm5KTk5WWFiYhg4dKj8/P8frli9frpUrV8rX11dhYWHavXu3fHx81LdvX9WqVcsp3pSUFH322Wfav3+/Y12+vr5KTU3V1KlTtWLFCp09e1bNmjXT4MGDVaNGDafXJycna8qUKTpw4IBq1KihwYMH5+0bDAAAAAAokkiiALhlhmFo5MiR+vrrryVJUVFRmjt3rjw8+GgBgPy0YsUKPfDAA3Jzc5NhGPrPf/6jadOmaeXKlfruu+8kSW5ubvr444+1ePFidejQQRcvXpTNZtPHH3+smJgYTZkyxbE+i8WiiIgILVy4UD/++KOmT58uDw8PWa1WGYYhNzc3eXh4aObMmVq/fr18fX01YcIEvfTSS/Lw8JDNZlN6ero8PDzk5uam//znP1q9erVatGgh6WoCJSIiQtu3b5fNZpObm5u+/PJLxcTEqGPHjlq/fr3S09MlXR0J89FHH2ndunVq3LixpKsJlGbNmumvv/5yvH727NmaM2dOPr/zAAAAAIDCjp+KA7hlo0eP1ieffCJJatiwob7//nv5+PiYHBUAFD2DBg2S1WpVamqq0tLSZBiGhg4dqjlz5shqtTqeS0pKUu/evXX+/HmlpqbKarUqPT1dn332mX777TenddpLep08eVKGYTjWK10dhZiamqo//vhDkyZNUkJCgl566SVHO3sCxL7d1NRUDRkyxLHuL774Qtu3b3fEkJqaql27dmnYsGHauHGj4/V2qampeuKJJxx/T5o0SX/99ZfT67ds2eJIGAEAAAAAkFtIogC4JRMnTtTbb78tSapZs6aWLVumwMBAk6MCgKLHMAwdO3Ysw/K0tDR5eXllWHbq1ClZrVan5V5eXhkmkLdPLn8jNptNBw8e1D///ONIsGQV4+HDhx1/HzhwIEN7m82m/fv3ZzqflmEYTvEdOHAgQ6JFktM2AAAAAADIDSRRAOTYV199paefflqSVLFiRa1YsUKlS5c2OSoAKJosFosqVqyYIfng5eWl1NRUp2Wenp4qW7asPD09nZanpqaqWrVqTsvq1aunYsWK3XDbbm5uqlatmipXrnzDubDc3NwUFhbm+Pv6bdnb1KxZUzabLcNzFovF6TXVqlWTu7u7UxvDMJy2AQAAAABAbiCJAiBHfvjhB0VHR0uSSpYsqRUrVig0NNTcoACgiJsxY4Y8PT3l7e0tb29vubm56YsvvlD//v3l4eEhLy8veXl5qUyZMvr2228VHBzsWObu7q6RI0eqQYMGTuv08PBQeHi4429vb29Hosa+znr16mno0KEqVaqUPvroI1ksFnl7ezsSHNfG9PnnnzvWNXDgQDVp0kSenp7y8vKSp6enGjRooEmTJikqKsopQWKxWOTn5+c0Z8sTTzyhunXrOvbB09NTzZs3V48ePfLk/QUAAAAAFF3M/gwg2+Li4vTwww8rPT1d/v7+Wrp0qWrVqmV2WABQ5LVu3VpbtmzRvHnzZLVa1bFjRzVr1kx9+vRRhw4dtGXLFoWEhGjAgAEKCgrSli1b9NVXX+nMmTNq2rSpunbtmul6IyMjtXz5cknSiBEj5OfnJ3d3dyUnJ6tKlSoaMGCAvL29JUnDhw/XXXfdpVWrVsnX11fVq1fXzp075e3trV69eql69eqO9Xp5eWnNmjWaMWOG/v77b4WFhWnAgAHy8vLS8uXL9eWXX2rlypU6c+aMmjdvrv79+6tKlSqO1/v4+GjdunWaPn26Dh06pOrVq6tfv35KSkrKs/cYAAAAAFA0kUQBkC1bt25Vp06dlJKSIi8vLy1evFhNmzY1OywAwP9Xu3Ztvfnmm07LLBaLevbsqZ49ezqWGYahkJAQvfjii5nOP3It++Ty9sedO3e+YfvWrVurdevWjr8feeSRLNt6eXk5TTZv5+HhoYEDB2rgwIE33JaPj4/TZPOZlQEDAAAAAOB2Uc4LwE3t3btX7du31/nz5+Xm5qZvvvlGbdu2NTssAEAea9y4sXx8fCRJsbGxJkcDAAAAAED+I4kC4IaOHj2q++67T6dOnZIkffbZZ+revbvJUQEA8oO3t7fuueceSSRRAAAAAABFE0kUAFk6ffq02rVrp0OHDkmSxo8fr0GDBpkcFQAgP9lLem3evFkXL140ORoAAAAAAPIXSRQAmbp48aI6duyo3bt3S5JeeuklvfTSSyZHBQDIb/YkSnp6utavX29yNAAAAAAA5C+SKAAyuHLlirp3766NGzdKkgYPHqz33nvP5KgAAGZo3ry5PDw8JFHSCwAAAABQ9JBEAeAkPT1d/fr108qVKyVJ3bt31+TJk2WxWEyODABgBn9/fzVq1EiSFBcXZ3I0AAAAAADkL5IoABwMw9CTTz6p7777TpLUtm1bff3113J3dzc5MgCAmSIiIiRJGzZs0JUrV0yOBgAAAACA/EMSBYDDa6+9ps8++0yS1KRJEy1cuFDe3t4mRwUAMJt9XpQrV65o06ZNJkcDAAAAAED+IYkCQJL0n//8R+PGjZMk1apVS0uXLlWxYsVMjgoAUBC0bNnSUdaRkl4AAAAAgKKEJAoATZ8+XS+88IIkqXLlylqxYoVKlSplclQAgIIiODhYderUkUQSBQAAAABQtJBEAYq4RYsWafDgwZKkkJAQrVy5UhUrVjQ5KgBAQWMv6RUfHy+r1WpyNAAAAAAA5A+SKEARtmbNGj3yyCOy2WwqVqyYli9frpo1a5odFgCgALInUS5cuKBt27aZHA0AAAAAAPmDJApQRP3222968MEHlZqaKm9vby1ZskQNGzY0OywAQAEVERHheExJLwAAAABAUUESBSiC/vzzT3Xo0EEXL16Uu7u7vv32W0VFRZkdFgCgACtXrpyqV68uSYqNjTU5GgAAAAAA8oeH2QEAyF+HDx9Wu3btlJiYKEn64osv9OCDD5ocFQDAFURGRmrfvn2KjY2VYRiyWCxmhwQAQJF18eJFffLJJ/r999/l6+urbt26qUuXLpm2TU1N1cyZMxUbG6vU1FSVL19e7777rvz8/PI5agAAXA9JFKAISUhIULt27fTPP/9Ikj744AM99thjJkcFAHAVkZGR+uKLL5SUlKQ//vhDtWvXNjskAACKrClTpigtLU3Tp0/XqVOn9MYbb6hixYpq1KhRhraffvqpUlJSNHHiRBUvXlyHDh2Sp6enCVEDAOB6KOcFFBEXLlxQhw4d9Ndff0mSXnvtNT3zzDPmBgUAcCnMiwIAQMGQkpKi+Ph49evXT35+fqpSpYratWunlStXZmh75MgRrV+/XsOHD1dwcLDc3NwUFhZGEgUAgGwiiQIUASkpKerSpYs2b94sSRo2bJjGjBljclQAAFcTFhamChUqSJJiYmJMjgYAgKLr6NGjMgxDoaGhjmVhYWE6fPhwhrZ79+5V6dKlNXfuXPXt21dPPvmkVqxYkZ/hAgDg0ijnBRRyVqtVvXv31po1ayRJjzzyiD7++GPq2AMAcsxisSgyMlLffPON4uLimBcFAACTpKSkZJjPxN/fX5cvX87QNiEhQYcOHVLTpk01ffp0HTx4UG+++abKly+vOnXqOLVNTEx0zJ+ZkJCg9PR0SZLNZsujPXFmGIZsNptsNluhusawv3/59T7mF/rLddBXroX+KnhIogCFmGEYGjp0qBYtWiRJat++vb788ku5u7ubGxgAwGVFRETom2++0ZEjR3To0CFVqVLF7JAAAChyfHx8MiRMkpOT5evrm6Gtt7e33Nzc1KtXL3l6eqpGjRoKDw/Xr7/+miGJMn/+fE2dOtXxd69evSRJJ06cyIO9KHpOnTpldgjIAfrLddBXrsUV+4skClBIGYahMWPGaMaMGZKk5s2ba/78+fLy8jI3MACAS4uMjHQ8jo2NJYkCAIAJ7OU1Dx8+rMqVK0uSDhw44Hh8rZx8V/fo0UNRUVGSro5EsZf9Klu27G1GnD2GYchqtcrDw6PQ/fr61KlTKl26tNzcCk9lffrLddBXroX+yh85+YEASRSgkHr//fc1ZcoUSVKdOnX0ww8/yN/f3+SoAACurlatWipRooROnz6tuLg4PfbYY2aHBABAkePj46Pw8HDNmjVLzz77rCPh8fTTT2doW6dOHZUtW1bfffedHnnkER08eFDx8fF67bXXMrQtVaqUSpUqJUkKDAx0VDHIr5tdhmHIzc1Nbm5uherGoZ193woL+st10Feuhf4qeFwrWgDZ8tlnn+lf//qXpKuTC65YsUIlSpQwOSoAQGHg5uamiIgISVdHogAAAHMMHTpU7u7uio6O1ptvvqkePXqoUaNGkqSePXtq165dkiR3d3e9/vrr2r59u3r16qX3339fgwYNylDKCwAAZI6RKEAhM2/ePA0bNkySFBISop9++knlypUzOSoAQGESGRmpxYsXa8+ePTpx4kS+lfgAAAD/JyAgQK+88kqmz3377bdOf1esWFHvvfdefoQFAEChw0gUoBA5ePCgHnvsMRmGoeLFi+vrr79WtWrVzA4LAFDI2EeiSNK6detMjAQAAAAAgLxFEgUoRJ5++mldvnxZFotFixYtUu3atc0OCQBQCDVo0MAxzxYlvQAAAAAAhRlJFKCQWLJkib7//ntJ0rBhwxQZGWlyRACAwsrDw0Ph4eGSSKIAAAAAAAo3kihAIZCcnKyRI0dKujoPyrvvvmtyRACAws5e0mv79u06e/asucEAAAAAAJBHSKIAhcC4ceN08OBBSdK///1vBQcHmxsQAKDQs494NAxD8fHxJkcDAAAAAEDeIIkCuLg9e/bo/ffflyS1bNlSjz32mMkRAQCKgqZNm8rLy0sSJb0AAAAAAIUXSRTAhRmGoeHDhys1NVXu7u769NNPZbFYzA4LAFAE+Pj4qGnTppKkuLg4k6MBAAAAACBvkEQBXNi8efO0cuVKSdLTTz+tunXrmhwRAKAosZf0+vXXX5WcnGxyNAAAAAAA5D6SKICLSklJ0QsvvCBJKl++vEaPHm1uQACAIsc+ubzVatXGjRtNjgYAAAAAgNxHEgVwURMnTtThw4clSePHj1exYsVMjggAUNS0aNFCbm5XLyeZFwUAAAAAUBiRRAFcUFJSksaOHStJatiwofr06WNyRACAoigwMFANGjSQRBIFAAAAAFA4kUQBXNCYMWN07tw5SdKECRMcvwIGACC/2Ut6rV+/XqmpqSZHAwAAAABA7uLOK+Bi9u3bp08++USS1KlTJ7Vu3drkiAAARZl9cvnLly/r999/NzkaAAAAAAByF0kUwMW8+uqrslqtcnNz0/jx480OBwBQxLVs2dLxmJJeAAAAAIDChiQK4ELWr1+vefPmSZIGDx6s2rVrmxwRAKCoCwkJUa1atSRJcXFxJkcDAAAAAEDuIokCuAjDMPT8889Lkvz9/fXWW2+ZHBEAAFfZS3rFxcUpPT3d5GgAAAAAAMg9JFEAF7FgwQKtX79ekvTSSy+pbNmyJkcEAMBV9iTKuXPntHPnTpOjAQAAAAAg95BEAVxAamqqXnnlFUlSuXLlHCNSAAAoCCIiIhyPKekFAAAAAChMSKIALmD27Nnat2+fJGnMmDHy9/c3OSIAAP5PpUqVVKVKFUlMLg8AAAAAKFxIogAuYNKkSZKksLAwRUdHmxsMAACZsI9GiYuLk2EYJkcDAAAAAEDuIIkCFHCbN2/Wr7/+KkkaOnSo3N3dTY4IAICM7POinDhxwjF6EgAAAAAAV0cSBSjgJk+eLEny9PTUwIEDTY4GAIDM2ZMoEiW9AAAAAACFB0kUoAA7d+6cvv76a0nSQw89pJCQEJMjAgAgczVq1FDp0qUlMbk8AAAAAKDwIIkCFGBfffWVkpOTJUnDhg0zORoAALJmsVgco1EYiQIAAAAAKCxIogAFlGEYjlJetWvXdkzYCwBAQWVPohw4cEBHjhwxORoAAAAAAG4fSRSggPrll1+0c+dOSVdHoVgsFpMjAgDgxq5N+FPSCwAAAABQGJBEAQqoSZMmSZJ8fX3Vr18/k6MBAODm6tatq+LFi0uipBcAAAAAoHAgiQIUQImJifruu+8kSb1791ZQUJC5AQEAkA3u7u5q2bKlJJIoAAAAAIDCgSQKUADNmDFDqampkphQHgDgWuwlvXbv3q3ExESTowEAAAAA4PaQRAEKGJvNpilTpkiSGjZsqMaNG5scEQAA2WefXF6S1q1bZ2IkAAAAAADcPpIoQAGzevVq7du3T5L0xBNPMKE8AMClNGrUSL6+vpIo6QUAAAAAcH0kUYACxj6hfGBgoHr16mVyNAAA5IyXl5eaNWsmSYqLizM5GgAAAAAAbg9JFKAAOXbsmBYvXixJ6tevnwICAkyOCACAnLOX9Pr999914cIFk6MBAAAAAODWkUQBCpBp06YpPT1dkjR06FCTowEA4NbYkyg2m03r1683ORoAAAAAAG4dSRSggLBarfrss88kSeHh4apbt67JEQEAcGuaNWsmDw8PScyLAgAAAABwbSRRgAJi2bJlOnLkiKSrE8oDAOCq/Pz81LhxY0kkUQAAAAAAro0kClBA2CeUL1mypHr06GFyNAAA3B57Sa9NmzYpJSXF5GgAAAAAALg1JFGAAuDAgQNavny5JGnAgAHy8fExOSIAAG5PRESEJOnKlSv69ddfTY4GAAAAAIBbQxIFKACmTp0qwzAkSUOGDDE5GgAAbl94eLgsFoskSnoBAAAAAFwXSRTAZKmpqZo2bZok6d5771WNGjVMjggAgNsXHBysu+++WxJJFAAAAACA6/IwO4D8EBAQIA8PD8cv/fODfVv5uc28ZhiG430sbPt17X/z28KFC3Xq1ClJ0rBhw3ItDvrLtdBfrqWg9ZeHR5H4OocLioyM1LZt2/TLL7/IarVyrAIAAAAAXE6R+D/ZBg0aKDg4WFarNd+3nZ6enu/bzEvBwcGy2Wyy2Wxmh5LrzOqryZMnS5LKlSunDh065OpxSn+5FvrLtRSk/goODjY7BNyGsmXL5uuPPfIzuRkREaH//e9/unjxorZs2aLGjRvn2bYKWnIzt5CMdi30l2uhv/IHCXQAAODqisTVzJYtW1S3bl2FhITk2zYNw1B6errc3d0d9cBdnc1mU1JSkkqWLCk3t8JTCc7Mvvrzzz+1du1aSdKgQYPk6+uba+umv1wL/eVaClp/JSQkmB0CbsOgQYMkKd9/7JEfyc3mzZs7Hq9du1b169fP0+0VpORmbiMZ7VroL9dCf+V9LAAAAK6sSCRRLl68KKvVasoNPIvFUmhuHFosFsf7WFj26Vpm7Ndnn30mSXJzc9OQIUNydfv0l2uhv1xLQesvM0ZaIvdMmzZN3bt3z7cfe+RncrNChQqqWbOm9uzZo/j4eL3wwgt5tq2CltzMLSSjXQv95Vror/zBjz0AAICrKxJJFKAgunz5smbMmCFJ6tSpkypVqmRuQAAAU5w4ccKUH3vkVxIwMjJSe/bsUVxcnAzDyLMbegUtuZnbCtt+0V+uhf5yLQWtv/ixBwAAcHXm/ywFKKK+/fZbnT17VtLVCeUBACiMIiMjJUmnT5/W7t27TY4GAAAAAICcIYkCmMQ+oXyVKlXUrl07k6MBACBv2JMokhQXF2diJAAAAAAA5BxJFMAEW7du1YYNGyRJQ4YMkbu7u8kRAQCQN0JDQ1W5cmVJUmxsrMnRAAAAAACQMyRRUCSlp6crKSlJhmFIujqp5OnTp5Wamnrb67avKy0tLcs29lEonp6eGjhwYLbWe+7cOaWkpDgtS09PV2Jiomw2260HDABAHouIiJB0NYli/+4FAAAAAMAVkERBkTNlyhT5+/urVKlSKlmypD799FNVr15dpUqVkp+fn15//fVbvsGzefNmVa5cWSVLlpSvr6/efPPNDOu6cOGCZs+eLUnq3r27ypQpc8N1Hjx4UPXq1VNQUJD8/Pw0YMAApaam6quvvlKxYsUUEhKi4OBgLV68+JZiBgAgr9lLeh07dkx///23ydEAAAAAAJB9HmYHAOSnH3/8UU888YQjsXHmzBkNHz5cFotF0tWRHePHj1fZsmU1fPjwHK07MTFR9913n86dO+dY19ixY1W+fHmnieNnz56tixcvSrr5hPJpaWlq166dDhw4IOnqKJevv/5aFy9e1Pz58x37cf78efXo0UO//fab6tevn6O4AQDIa9fOixIbG6tq1aqZGA0AAAAAANnHSBQUKfPnz890+bWjRaxWq2OkSE7Ex8frwoULTqW10tPT9fXXXzttx17K64477lBUVNQN17lnzx7t3btXVqvVsSw1NVVLly7NMI+Kh4eHfvjhhxzHDQBAXrvjjjsUEhIiicnlAQAAAACuhSQKihQ3NzfHqJObtbuVdd9s+YYNG7Rt2zZJV0eh3CyWrNaZ1etuJW4AAPKaxWJxjEZhcnkAAAAAgCvhjiuKlN69ezuNOrEnHa5NPri7u2d7svdrRUREKDg42GmEiLu7uwYMGOD42z4KxcfHR4899thN11mzZk3VrVtXnp6ejmWenp7q0aNHhsnkbTabunbtmuO4AQDID/Ykyv79+3X06FGTowEAAAAAIHtIoqBIadu2rb766isFBQVJkipUqKCvvvpKd955pyTJ19dX48aNu6UkSlBQkNauXauaNWtKkvz8/DR+/HhHsuT06dOaO3euJKlXr14qUaLETdfp7u6un376Sffcc48sFos8PT315JNPatq0aZo/f75KliwpSSpbtqyWLl2q2rVr5zhuAADyQ0REhOMxJb0AAAAAAK6CieVR5PTp00d9+vRRamqqvLy8ZBiGevbsKZvNJi8vr2yV+8pK7dq1tXv3bqWmpsrT09NpXTNnztSVK1ck3XxC+WuVK1dOcXFxSktLk7u7u2PUTNeuXdW1a1dduXJF3t7etxwzAAD54e6771ZgYKDOnz+v2NhY9erVy+yQAAAAAAC4KUaioMjy8vLK8PftJFButK5rJ5SvX7++mjZtmuN1enp6ZjrnCQkUAIArcHd3V8uWLSUxEgUAAAAA4DpIogD5YM2aNdqzZ4+k7E0oDwBAYWSfF2Xnzp1KSkoyORoAAAAAAG6OJAqQD+yjUIoVK6Y+ffqYHA0AAOawJ1Ekad26dSZGAgAAAABA9pBEAfLYiRMntHDhQklS3759VaxYMZMjAgDAHI0aNZKvr68kKTY21uRoAAAAAAC4OZIoQB774osvZLVaJUlDhw41ORoAAMzj5eWlZs2aSSKJAgAAAABwDSRRgDyUnp6uzz77TJLUvHlz1atXz+SIAAAwl72k1++//64LFy6YHA0AAAAAADdGEgXIQ8uXL9ehQ4ckXZ1QHgCAos6eRLHZbFq/fr3J0QAAAAAAcGMkUYA8ZJ9QvkSJEnr44YdNjgYAAPM1a9ZMHh4ekijpBQAAAAAo+EiiAHnk0KFD+vHHHyVJ0dHRjol0AQAoyvz8/NSkSRNJJFEAAAAAAAUfSRQgj3z++ecyDEOSNGTIEJOjAQCg4LCX9Nq4caNSUlJMjgYAAAAAgKyRRAHyQFpamj7//HNJUps2bXTHHXeYHBEAAAVHRESEJCk1NVWbNm0yORoAAAAAALJGEgXIA4sXL9aJEyckMaE8AADXCw8Pl8VikSTFxcWZHA0AAAAAAFkjiQLkAfuE8mXLllXXrl3NDQYAgAImKChI9erVk8S8KAAAAACAgo0kCpDL9uzZo1WrVkmSBg0aJE9PT5MjAgCg4LHPixIfHy+r1WpyNAAAAAAAZI4kCpDLZsyYIUmyWCx6/PHHzQ0GAIACyp5EuXTpkrZs2WJyNAAAAAAAZI4kCpCLDMPQvHnzJEmtW7dWaGioyREBAFAwtWzZ0vGYkl4AAAAAgIKKJAqQi3bt2qW9e/dKknr06GFyNAAAFFxlypTRHXfcIYkkCgAAAACg4CKJAuSi+fPnS7payqtbt24mRwMAQMFmL+m1bt062Ww2k6MBAAAAACAjkihALrInUZo3b65y5cqZHA0AAAWbPYly+vRp7dq1y+RoAAAAAADIiCQKkEv27t2rHTt2SKKUFwAA2WFPokiU9AIAAAAAFEwkUYBcsmDBAsfj7t27mxgJAACuoXLlygoNDZVEEgUAAAAAUDCRRAFyib2UV8OGDVWlShVzgwEAwEXYR6PExsbKMAyTowEAAAAAwBlJFCAXHD58WL/++qskSnkBAJATUVFRkqQTJ05o3759JkcDAAAAAIAzkihALli4cKHjMaW8AADIPuZFAQAAAAAUZCRRgFxgnw+ldu3auvPOO02OBgAA11G9enWVLVtWEkkUAAAAAEDBQxIFuE0nT55UXFycJEp5AQCQUxaLxWleFAAAAAAAChKSKMBtWrRokWMiXEp5AQCQc/YkysGDB3X48GGTowEAAAAA4P+QRAFuk72UV9WqVVWvXj2TowEAwPVcOy+KfXQnAAAAAAAFAUkU4DacOXNGq1evlnS1lJfFYjE5IgAAXM9dd92l4OBgSVJMTIzJ0QAAAAAA8H9IogC34fvvv5fVapVEKS8AAG6Vm5ubIiIiJDEvCgAAAACgYCGJAtwGeymvChUqqGnTpiZHAwCA67KX9Prrr7908uRJk6MBAAAAAOAqkijALbpw4YJ++uknSVdHobi5cToBAHCrmBcFAAAAAFAQcdcXuEVLly7VlStXJFHKCwCA29WgQQP5+/tLoqQXAAAAAKDgIIkCl5aQkKD169fr0KFD+b5teymvkJAQRx13AABwazw8PBQeHi6JJAoAAAAAoOAgiQKXNX36dJUvX14tWrRQlSpV9Mwzz8gwjHzZ9uXLl/Xjjz9Kkrp06SJ3d/d82S4AAIWZvaTX9u3bdebMGZOjAQAAAACAJApc1JYtWzR48GBZrVbHso8//lgzZszIl+2vXLlSly5dkiT16NEjX7YJAEBhZ0+iGIah+Ph4k6MBAAAAAIAkClxUTEyMvLy8nJalp6fr559/zpftz58/X5JUvHhxtWnTJl+2CQBAYdekSRN5e3tLoqQXAAAAAKBgIIkClxQQECCbzea0zN3dXcWKFcvzbaempur777+XJHXu3DlDMgcAANwaHx8f3XPPPZJIogAAAAAACgaSKHBJ3bp1U1BQkDw8PCRJFotFFotFQ4YMyfNtr127VmfPnpVEKS8AAHKbvaTX5s2bdfHiRZOjAQAAAAAUdSRR4JJKliyp9evXq1WrVipTpowaNGig1atXq2HDhnm+bXspLz8/P7Vv3z7PtwcAQFFiT6JYrVatX7/e5GgAAAAAAEWdh9kBALeqatWqWrlyZb5uMz09XYsWLZIkdezYUb6+vvm6fQAACrvmzZvL3d1d6enpio2N1X333Wd2SAAAAACAIoyRKEAOxMfH69SpU5Io5QUAQF4ICAhQ48aNJUkxMTEmRwMAAAAAKOpIogA5YC/l5eXlpY4dO5ocDQAAhZO9pNemTZuUkpJicjQAAAAAgKKMJAqQTTabTQsWLJAktWvXToGBgSZHBABA4WRPoly5ckW//vqrydEAAAAAAIoykihANv322286cuSIJEp5AQCQl1q2bCmLxSJJio2NNTkaAAAAAEBRRhIFyCZ7KS93d3d17tzZ5GgAACi8goKCVK9ePUkkUQAAAAAA5iKJAmSDYRiOUl6tW7dWyZIlTY4IAIDCzV7SKz4+XmlpaSZHAwAAAAAoqkiiANmwY8cO7du3TxKlvAAAyA/2JMqlS5e0ZcsWk6MBAAAAABRVJFGAbLCX8rJYLOratau5wQAAUATYkyiSFBMTY2IkAAAAAICijCQKkA32Ul7h4eEqW7asydEAAFD4hYSEqFatWpKYFwUAAAAAYB6SKMBN7NmzRzt37pREKS8AAPJTVFSUJCkuLk7p6ekmRwMAAAAAKIpIogA3YS/lJUndunUzMRIAAIoWe0mvc+fOaceOHSZHAwAAAAAoikiiADdhL+XVuHFjhYaGmhwNAABFB/OiAAAAAADMRhIFuIFDhw7pt99+kyR1797d5GgAAChaKlSooGrVqkliXhQAAAAAgDlIogA3YB+FIjEfCgAAZrDPixIbGyvDMEyOBgAAAABQ1HiYHQBQkNmTKHXq1FHNmjVNjgYAgKInMjJSX3zxhRITE/XHH3+odu3aZocEAECRUbZsWXl4eOTbDxns2ylsP5wwDMPxPhamfaO/XAd95Vror/zh4ZH91AhJFCALJ06cUHx8vCRKeQEAYBb7SBTp6rwoJFEAAMg/gwYNkiRZrdZ83W56enq+bi8/BAcHy2azyWazmR1KrqO/XAd95Vror7yPJbtIogBZWLRokSMrSikvAADMERoaqkqVKumff/5RbGysnnjiCbNDAgCgyJg2bZq6d++ukJCQfNmeYRhKT0+Xu7u7LBZLvmwzP9hsNiUlJalkyZJycys8lfXpL9dBX7kW+it/JCQkZLstSRQgC/Pnz5ckVa9eXXXr1jU5GgAAiiaLxaKoqCh99dVXiomJkWEYhep/JAAAKMhOnDghq9Wa79+9FoulUH3fWywWx/tYmPbLrrDtV2Hur8K2T4W5ryT6K6/lZJSl+SkfoABKSkrSmjVrJF0t5VUQTmwAAIqqyMhISdLx48e1f/9+k6MBAAAAABQlJFGATCxZssRRd5BSXgAAmOvaeVFiY2NNjAQAAAAAUNSQRAEyYS/lVbFiRTVu3NjkaAAAKNpq1KihMmXKSLo6uTwAAAAAAPmFJApwnfPnz2vFihWSrpbyKggTHQEAUJTZ50WRGIkCAAAAAMhf3B0GrrN06VKlpqZKopQXAAAFhX1elIMHD+rw4cMmRwMAAAAAKCo8zA5Aki5evKhPPvlEv//+u3x9fdWtWzd16dIlQ7sTJ05owoQJOnbsmAzDUKVKlRQdHa3atWubEDUKK3spr9KlSys8PNzkaAAAgPR/SRTp6miUvn37mhgNAAAAAKCoKBBJlClTpigtLU3Tp0/XqVOn9MYbb6hixYpq1KiRU7vAwEA999xzKlu2rCwWi9avX68xY8boyy+/lKenp0nRozBJTk7W0qVLJUldu3aVu7u7yREBAABJuuuuu1SiRAmdPn1aMTExJFEAAAAAAPnC9HJeKSkpio+PV79+/eTn56cqVaqoXbt2WrlyZYa2fn5+Kl++vNzc3GQYhtzc3HTp0iWdP3/ehMhRGK1YsULJycmSKOUFAEBB4ubmpoiICEnMiwIAAAAAyD+mj0Q5evSoDMNQaGioY1lYWJjWr1+f5WsGDRqk06dPKz09XW3btlXJkiWdnk9MTFRiYqIkKSEhwTG/hc1my4M9yJxhGLLZbLLZbLJYLPm23bxkf//y833MD9f21bx58yRJQUFBioyMdOl9LQr9VVjOLYn+cjWFtb+Agi4qKkqLFy/Wnj17dPz4cZUrV87skAAAAAAAhZzpSZSUlBT5+fk5LfP399fly5ezfM20adOUmpqquLi4TJ+fP3++pk6d6vg7KipK0tU5VXD7Tp06ZXYIeSI1NVVLliyRJN177706ffq0yRHljsLaX4UV/eVa6C8gf107L0pcXJx69uxpYjQAAAAAgKLA9CSKj49PhoRJcnKyfH19b/g6Ly8vtW3bVkOHDlXVqlUVFhbmeK5Hjx6OxElCQoKWLVsmSSpbtmwuR581wzBktVrl4eFRaH59bbPZdOrUKZUuXVpubqZXgss19r76+eefHaXhHn300Xw9XvJCYe+vwnRuSfSXqylo/cWPBFBU1K9fX8WKFdOFCxcUExNDEgUAAAAAkOdMT6JUqFBBknT48GFVrlxZknTgwAHH45uxWq06ceKEUxKlVKlSKlWqlKSrk9F7eXlJUr7e6LLP2eLm5laobhxKcuxXYWHvq4ULF0q6OhKqffv2hWYfC2t/FcZzS6K/XE1h6y+goHN3d1fLli21bNky5kUBAAAAAOQL0+/8+Pj4KDw8XLNmzVJycrIOHTqkFStW6L777svQdtu2bf+PvfsOj6pO////mvQCCaF3CCgdpAqEkoCsdRUpIthAwLJY2f24664F3XV1db+ua11RKYougpS1u6gQauhFCL2HnlBCQurMnN8f/M4sIQEyyWTOlOfjunI5mXLmNXljyrnnfd/atWuXHA6HCgsL9fnnnys3N1etWrWyIDkCicPh0JdffilJuuWWW664EwoAAFjD3G28ZcsW1ww8AAAAAACqiuVFFEl66KGHFBoaqjFjxuj555/XsGHD1K1bN0nSiBEjlJ6eLknKz8/XP//5T40aNUpjx47V5s2bNWnSpFKD5QF3LVu2zHUiZujQoRanAQAAl3LhXJRly5ZZmAQAAAAAEAwsb+clSdWqVdPTTz9d5m2zZ892Xe7Vq5d69erlrVgIIubcnMjISN18880WpwEAAJfSrVs3xcTEKC8vT4sXL9btt99udSQAAAAAQADziZ0ogNXMd7L26tVL1atXtzgNAAC4lIiICPXu3VuSmIsCAAAAAKhyFFEQ9M6dO6f169dLkvr162dxGgAAcCXmXJSNGzcqOzvb4jQAAAAAgEBGEQVBb9WqVbLb7ZIoogAA4A/MuShOp1PLly+3OA0AAAAAIJBRREHQW7p0qSQpJCTE1R4EAAD4rp49eyoiIkKStHjxYovTAAAAAAACGUUUBD1zHkqXLl2YhwIAgB+IiopSz549JTEXBQAAAABQtSiiIKgVFxcrLS1NktSnTx+L0wAAgPIy56KsXbtW586dszgNAAAAACBQUURBUNuwYYPy8vIkMQ8FAAB/Ys5FsdvtrjdEAAAAAADgaRRRENTMeSiS1LdvXwuTAAAAdyQlJSksLEwSc1EAAAAAAFWHIgqCmllEufrqq1WvXj2L0wAAgPKKjY1Vt27dJFFEAQAAAABUHYooCFqGYbiGyrMLBQAA/2PORVm1apXy8/MtTgMAAAAACEQUURC0tm/frpMnT0piqDwAAP7ILKIUFRVp9erVFqcBAAAAAASiMKsDwD+tWrVK3377rcLCwjR8+HC1a9fO6khuYx4KAAD+rU+fPgoJCZHT6dSSJUvUunVrqyMBAAAAAAIMO1HgtpkzZyopKUmvvfaa/vrXv6pz585auHCh1bHcZhZRGjZsqMTERIvTAAAAd8XHx6tz586SpCVLllgbBgAAAAAQkCiiwC1FRUUaP368nE6nCgsLVVRUJLvdrjFjxlgdzW1mEaVfv36y2WwWpwEAABVhtvRKS0tTUVGRxWkAAAAAAIGGIgrckpWVpYKCghLXGYahQ4cOyel0WpTKfRkZGTpw4IAkWnkBAODPzCJKfn6+Nm3aZHEaAAAAAECgoYgCt9SuXVuRkZGlrm/YsKFCQvznn9OF81D69etnYRIAAFAZF+4oTUtLszgNAAAAACDQ+M9Zb/iEiIgI/etf/5LNZlN4eLgiIiIUFhamKVOmWB3NLWYRJT4+Xu3bt7c4DQAAqKiaNWuqY8eOkqSVK1danAYAAAAAEGjCrA4A/zN69GhdddVV+vrrrxUWFqYRI0a4hrr6C7OI0qdPH4WGhsput1ucCAAAVFRycrJ++eUXrVmzRsXFxWXumgUAAAAAoCIooqBC+vXr57dtsDIzM5Weni7pf33UAQCA/0pOTtbbb7+tvLw8rV+/Xr1797Y6EgAAAAAgQNDOC0FnyZIlrsspKSnWBQEAAB7Rv39/1+ULf84DAAAAAFBZFFEQdFJTUyVJ1apVU9euXa0NAwAAKq1OnTpq166dJGnx4sUWpwEAAAAABBKKKAg65smVvn37KiyMjnYAAAQCczfK8uXL5XA4LE4DAAAAAAgUFFEQVLKysrR582ZJzEMBACCQmEWUs2fPauPGjdaGAQAAAAAEDIooCCpLly51XWYeCgAAgePCuSi09AIAAAAAeApFFAQVcx5KbGysunXrZm0YAADgMQ0aNFBiYqIkhssDAAAAADyHIgqCillE6dOnj8LDw60NAwAAPKp3796Szu88dTqdFqcBAAAAAAQCiigIGqdOnXLNQ6GVFwAAgadXr16Szv/M37Jli8VpAAAAAACBgCIKgsaSJUtkGIYkhsoDABCIzCKKxFwUAAAAAIBnUERB0DBPpsTExKh79+4WpwEAAJ7WqFEj11wUiigAAAAAAE+giIKgYc5DSUpKUkREhLVhAABAlejfv7+kkjtQAQAAAACoqAoXUX744Qf95S9/0YMPPqiDBw9KOv/H6pEjRzwWDvCU06dPa9OmTZKYhwIAQCAziyiZmZnatm2bxWkAAAAAAP7O7SJKZmam+vTpo1tuuUVTpkzRlClTlJWVJUmaOnWq/vrXv3o8JFBZS5cudb0blSIKAACB68K5Z7T0AgAAAABUlttFlCeffFKZmZnasmWLdu/eXaJNwqBBg/Tzzz97NCDgCWYrr+joaPXo0cPaMAAAoMo0b95cTZo0kUQRBQAAAABQeW4XUb799lv99a9/Vdu2bWWz2Urc1qRJEx06dMhj4QBPMU+iMA8FAIDAZrPZXLtRFi9ezFwUAAAAAECluF1Esdvtio2NLfO206dPc4IaPufMmTPasGGDpJItPgAAuJLc3Fy9+uqruvPOOzVmzBh9+eWXV3zMzz//rNtuu03ff/+9FxKiLGbrzmPHjmnnzp3WhgEAAAAA+DW3iyg9e/bU1KlTy7zt888/V58+fSodCvAk5qEAACpq8uTJKi4u1rRp0/TCCy9ozpw5Wrdu3SXvf/bsWc2ZM0dNmzb1Ykpc7MKf92ZLTwAAAAAAKsLtIspLL72kb775Rv3799e7774rm82m//znP7rjjjv01Vdf6cUXX6yKnECFma28oqKidO2111qcBgDgLwoKCrR8+XLde++9iomJUfPmzXX99dfrxx9/vORjpk2bpsGDBysuLs6LSXGxFi1aqHHjxpKkRYsWWZwGAAAAAODP3C6i9O7dW4sWLZLNZtPvfvc7GYahv/71rzp69Kh+/vlnde3atSpyAhVmvgO1d+/eioyMtDYMAMBvHD58WIZhqFmzZq7rEhMTdfDgwTLvv2XLFmVkZOj666/3VkRcgs1m04ABAySd/z2AuSgAAAAAgIoKq8iDevfurcWLFys/P1+nT59WjRo1FBMT4+lsQKVlZ2e75qHQygsA4I6CgoJSv9/ExsYqPz+/1H2Li4v1/vvva+LEiQoJufx7VLKyspSVlSVJyszMlMPhkCQ5nU4PJb88wzDkdDrldDpls9m88pzeYH79zP/2799fM2bM0PHjx7Vt2za1adPGyngVFizrFShYL//CegEAAKA83C6ijB07Vs8995wSExMVHR2t6Oho120HDhzQiy++eMmZKYC3LVu2zPXHA0PlAQDuiIqKKlUwycvLK/G7j2nevHnq0KGDWrZsecXjzp07Vx9++KHr85EjR0o6PwQdlXfixAlJUvv27V3XffXVV6pRo4ZFiXA55nrBP7Be/oX1AgAA8Ay3iyjTp0/Xww8/rMTExFK3ZWVl6eOPP6aIAp9htvKKjIxUz549rQ0DAPArjRo1kiQdPHjQNSh+3759ZQ6N37Rpkw4cOKAVK1ZIknJzc7V3717t3LlTTzzxRIn7Dhs2zFXYz8zM1IIFCyRJ9evXr7LXciHDMGS32xUWFhZw77w+ceKE6tatq5CQENWrV09NmjRRRkaGNmzYoN///vdWR6yQYFmvQMF6+RfWyzt4kwAAAPB3FWrndalfMHft2qVatWpVKhDgSeZQ+V69eikqKsriNAAAfxIVFaU+ffpoxowZmjhxoqvgcXFRRJL++Mc/ym63uz5/5ZVX1LNnT91www2l7lu7dm3Vrl1bkhQXF6fQ0FBJ8tqJLsMwFBISopCQkIA6aWgyX5t0vpXnjBkztHjxYtlsNr98vcG0XoGA9fIvrBcAAADKo1xFlH/961/617/+Jel8AeWuu+4q1cqioKBA+/fv1x133OH5lEAFnD17VuvWrZPEPBQAQMU89NBDeueddzRmzBhFR0dr2LBh6tatmyRpxIgRmjRpktq3b6/q1auXeFxYWJhiYmJUrVo1K2Lj/2cWUY4fP64dO3b47VwUAAAAAIB1ylVEadiwoeuEwZYtW9S6dWvVqVOnxH0iIiLUtm1bjRs3zvMpgQpYvnw581AAAJVSrVo1Pf3002XeNnv27Es+7uWXX66qSHDDhW+iSE1NpYgCAAAAAHBbuYoogwcP1uDBg12fP//882XORAF8iTkPJSIiQr169bI2DAAA8LrExETXXJTU1FQ9/PDDVkcCAAAAAPgZtxukTps2jQIK/IJZROnVq1ep9nMAACDw2Ww2126U1NRUGYZhbSAAAAAAgN+p0GD5M2fOaM6cOdq5c6cKCgpK3Gaz2fTmm296JBxQUTk5Oa55KLTyAgAgeDEXBQAAAABQGW4XUXbt2qWkpCQVFhbq3LlzqlOnjk6dOiW73a6EhATFx8dTRIHlli9fLofDIYmh8gAABDPmogAAAAAAKsPtdl6//e1v1bNnTx0/flyGYei7775Tfn6+Pv30U1WvXl1ffPFFVeQE3LJ48WJJUnh4OPNQAAAIYuZcFOl/rT4BAAAAACgvt4soq1ev1sMPP6zIyEhJUlFRkUJDQ3XXXXfpt7/9rR5//HGPhwTcZZ4k6dmzp2JiYqwNAwAALMNcFAAAAABAZbhdRCksLFRcXJxCQkJUs2ZNHTlyxHVbhw4dtHHjRk/mA9yWm5urNWvWSKKVFwAAkAYMGCBJOn78uLZv325xGgAAAACAP3G7iNKqVSsdOHBAktSlSxe99957ysnJUX5+viZPnqyGDRt6PCRw6tQp3X777apWrZpq166tv/zlL3I6nWXed8WKFa55KAyVBwAAF89FAQAAAACgvNwuoowcOdK12+Qvf/mL1qxZo4SEBMXFxWnu3Ll64YUXPBwRwc7pdOrmm2/W999/r3PnzunkyZP685//rFdffbXM+5snR8LDw9W7d28vJgUAAL6oefPmatq0qSSKKAAAAAAA94S5+4Df/va3rsu9evXSli1b9P3336ugoEADBw5Uhw4dPBoQ2LVrl1atWlXiOrvdrnfffVd//OMfS93fHCrfo0cPxcbGeiUjAADwXeZclE8++cQ1F8Vms1kdCwAAAADgB9wuolysSZMmevDBBz2RBShTUVFRmdcXFxeXuu7cuXNavXq1JOahAACA/zGLKCdOnND27dvVtm1bqyMBAAAAAPxAhYsomzdvVkZGhgoKCkrdNnTo0EqFAi7UqlUrNW7cWEePHnXNOomIiNCtt95a6r4rVqyQ3W6XxDwUAADwPxfPRaGIAgAAAAAoD7eLKFu2bNGIESO0Y8cOGYZR6nabzeY60Q14QmRkpBYsWKCbbrpJBw4ckCRdf/31evPNN0vd12zlFRYWpqSkJK/mBAAAvsuci3Lw4EGlpqbqN7/5jdWRAAAAAAB+wO0iyrhx4xQWFqavvvpKrVq1UkRERFXkAkpo27atdu/erYMHDyomJkb169cv837msNgePXqoWrVqXkwIAAB8GXNRAAAAAAAV4XYRJT09XXPmzNGNN95YFXmASwoLC1OLFi0ueXteXp5rHgqtvAAAwMWYiwIAAAAAcFeIuw/o3LmzTpw4URVZgEpJS0tzDZtnqDwAALjYxXNRAAAAAAC4EreLKO+8845ef/11/fjjj64B3oAvMOehhIaGMg8FAACUYs5FkSiiAAAAAADKx+0iSrt27dSrVy/deOONio6OVlxcXImP+Pj4qsgJXJF5MqR79+6qXr26tWEAAIDPMeeiSHLNRQEAAAAA4HLcnony0EMPaebMmRo6dCiD5eEz8vPztWrVKknMQwEAAJd24VyUbdu2qV27dlZHAgAAAAD4MLeLKHPnztU//vEPTZgwoSryABWycuVKFRUVSWIeCgAAuLSL56JQRAEAALgyp9Op+fPna8eOHWratKnuvPNObdq0ST/99JNiYmI0fPhwNWzYsMLHX7t2rRYtWqTo6GgNGzZMDRo0qHTmHTt26LvvvpMk3XLLLWrVqpUMw9DXX3+t9PR0NWjQQCNHjlRUVFSlnwtAYHO7iFKjRg21aNGiKrIAFWa28goNDVWfPn2sDQMAAHxWYmKimjVrpgMHDig1NZU3BgEAAFyB0+nU8OHD9fXXXyssLEwOh0Mvvvii9uzZo8jISBmGoeeff17Lly9X+/bt3T7+tGnTNH78eEVERJQ4Vtu2bSuc+aefftItt9yikJAQGYahP/7xj/r22281Y8YMffrppwoPD5fD4dCbb76pZcuWKTY2tsLPBSDwuT0T5Xe/+53efvtthsrDp5hD5bt27aq4uDiL0wAAAF/GXBQAAIDy++KLL/T111/LbreroKBAxcXF2r17twzDUEFBgQoLC5Wbm6uxY8e6fexTp07pwQcflNPpdB0rJydH48aNq3BewzB01113qaioyHXMoqIiDRs2TJ9++qkcDofrdWzdulV///vfK/xcAIKD2ztRdu/erc2bN6tly5ZKTk5WjRo1Stxus9n05ptveiofcEUFBQVauXKlJFp5AQCAK0tJSdHHH3+szMxM5qIAAABcwbZt2xQaGnrZN1Q7HA7t2LHD7WPv27ev1HHtdru2b9/u9rFMOTk5yszMLHGdYRjKzs5WVFSUHA6H6/qioiKlp6dX+LkABAe3iyjffPONQkNDJUlLly4tdTtFFHjbypUrVVhYKImh8gAA4MqYiwIAAFB+jRs3vuLuXZvNpkaNGrl97LIeY7PZ1LhxY7ePZapevbpiYmKUl5dX4vrIyMhSBZvw8HA1a9asws8FIDi43c5r3759l/3Yu3dvVeQELsls5RUSEqK+fftanAYAAPi65s2bu/5YNueqAQAAoGz33HOP2rdvr4iICElSRESEatSoIZvNJpvNptDQUIWGhuqdd95x+9j169fXs88+q9DQUNlsNoWFhSksLExvv/12hfPabDa99957stlsCgkJUUhIiGw2m95//3316tWrxOuoVauWfv/731f4uQAEB7d3ogC+xjz50aVLF8XHx1sbBgAA+AWzpZc5F8Vms1kdCQAAwCdFRUVp2bJl+uc//6kdO3aoWbNmmjhxohYsWKAffvhBsbGxGjt2rLp27Vqh4//5z39W+/bt9eOPP7qO1blz50plHj16tBo1aqQvvvhCkjRixAhdd911GjVqlN566y1t2bJFjRo10pNPPqm6detW6rkABL5yFVHmzZungQMHqkaNGpo3b94V7z906NBKBwPKg3koAACgIpiLAgAAUH4xMTH605/+VOK6ESNGaOjQoQoLC6vUG1JsNptGjhypkSNHVjZmCYMGDdKgQYNKXBcZGamnnnrKo88DIPCVq4gyfPhwrVy5Utdee62GDx9+2fvabLYSA5qAqrR69WoVFBRIoogCAADKj7koAAAAAIDyKFcRZd++fWrQoIHrMuArzFZeNpuNeSgAAKDczLkoBw4cUGpqqiZMmGB1JAAAAACADypXEcUcvCmdP1ndoEEDhYeHl7qf3W7XkSNHPJcOuAKziNK5c2fVqFHD0iwAAMC/MBcFAAAAAHAlIe4+IDExURs2bCjztk2bNikxMbHSoYDyKCgo0IoVKyRJAwYMsDgNAADwN2ZLL3MuCgAAAAAAF3O7iGIYxiVvKywsVGRkZKUCAeW1atUqFRYWSqKIAgAA3HfhXJRFixZZFwQAAAAA4LPK1c5r+/bt2rp1q+vz1NRUHTp0qMR9CgoKNHPmTLVo0cKzCYFLME92hISEqF+/fhanAQAA/ubiuSiPPPKI1ZEAAAAAAD6mXEWUWbNm6cUXX5R0fibK008/Xeb9atSooenTp3ssnKdUq1ZNYWFhl91F42nmc3nzOauaYRiur6MvvC5zHkrXrl0VFxdX4UyBuFaS762Xp7Be/oX18o6wsHL9OAdQhgEDBmj69OnMRQEAAAAAlKlcZ12efPJJjRkzRoZhqEWLFpo3b566dOlS4j4RERGqX7++T/7h2aVLFyUkJMhut3v9uR0Oh9efsyolJCTI6XTK6XRamiM/P19paWmSpP79+3tkbQNtrSTfWa+qwHr5F9ar6rMAqJiUlBRNnz5dWVlZ2rp1q9q3b291JAAAAACADylXESU+Pl7x8fGSpH379qlhw4YKDw+v0mCetGHDBnXs2FF16tTx2nMahiGHw6HQ0FCfLCxVhNPp1MmTJ1WrVi2FhLg9Tsej1qxZo6KiIknSddddV6l3YQfiWkm+tV6exHr5F9bLOzIzM62OAPit5ORk1+XU1FSKKAAAAACAEip05nn79u3q2LGjpPPD5P/f//t/2rZtmwYNGqQxY8Z4Mp9H5Obmym63W3ICz2azBcyJQ5vN5vo6Wv2aFi9eLEkKDQ1Vv379PJLHF16XJ/nSelWFQHtdrJd/8bX1smKnJRAomjdvrubNm2v//v3MRQEAAAAAlOL222cfeOABzZgxw/X5H/7wB7344ovavn27HnzwQb333nseDQiUxRwq3717d1WvXt3iNAAAwJ+lpKRIkmsuCgAAAAAAJreLKBs3blS/fv0knX/n68cff6xXX31Va9eu1QsvvKB//etfHg8JXCgvL0+rVq2SdH4YLAAAQGWYRRRzLgoAAAAAACa3iyg5OTmu+SirVq3S2bNnNXLkSElS3759tXfvXs8mBC6yYsUKFRcXS/rfSQ8AAICKunguCgAAAAAAJreLKI0bN9bKlSslSfPmzVO7du3UoEEDSdLp06cVExPj2YTARcxWXmFhYerTp4/FaQAAgL8z56JIFFEAAAAAACW5XUQZN26cnn32WfXo0UNvvvmmHnzwQddtK1euVNu2bT0aELiYeXLj2muvVbVq1awNAwAAAgJzUQAAAAAAZQlz9wFPP/20GjZsqDVr1mjChAkaM2aM67bTp09r/PjxnswHlJCbm6vVq1dLopUXAADwnJSUFE2fPt01F6V9+/ZWRwIAAEAlHDt2TH/5y1+Uk5Ojtm3bql27dmrbtq1atGihsDC3T4kCCGIV+o5x33336b777it1/fvvv1/pQMDlrFixQna7XRJD5QEAgOdcPBeFIgoAAIB/OnfunF5//XW99tprOnfuXKnbIyIi1KpVK3Xo0EEdO3Z0fTRr1kw2m82CxAB8Xbnaec2ePVunT58ucd2RI0fkcDhKXffyyy97Lh1wEXMeSnh4uJKSkixOAwAAAsWFc1HM3zcAAADgPxwOh6ZNm6ZWrVpp0qRJrgJKkyZNShRHioqKtGXLFn3++ed65plndNtttykxMVEJCQm66aab9PLLLystLU35+flWvRQAPqZcRZRRo0Zp165drs8dDoeaNGmiTZs2lbhfRkaGnnvuOc8mBC5gntTo2bOnYmJiLE4DAAACidkqdPHixXI6ndaGAQAAQLn99NNP6tatm8aOHasjR45Iknr37q3ly5fr4MGDOnfunDZs2KB///vfevbZZzV06FC1atWqRHElOztbP/zwg5577jkNHz5cCQkJSkpK0u9//3t99dVXOnnypFUvD4DFytXOq6zhmgzchLfl5ORo7dq1kmjlBQAAPG/AgAEl5qJ06NDB6kgAAAC4jPT0dD311FP6/vvvXde1aNFCf/vb3zR8+HBXkSQ6OlqdO3dW586dSzw+Ly9PW7du1ebNm7Vu3TotW7ZMv/zyiwzDUHFxsdLS0pSWlqa///3vkqR27dqpb9++6tevn/r27UsLMCBIMEUJfmPZsmWuFnIUUQAAgKddPBeFIgoAAIBvOnbsmCZNmqSPPvrItYM4ISFBzz33nCZMmKDIyMhyHScmJkbdu3dX9+7ddf/990uSTp8+re+++07p6elavny5Vq1apcLCQknS1q1btXXrVn3wwQeSpEaNGpUoqnTo0EGhoaFV8IoBWIkiCvzGzz//LEmKjIxUr169LE4DAAACTbNmzZSYmKh9+/YpNTVVjz76qNWRAAAAcIFz587pH//4h1599VXXzJPw8HA9+uijevbZZ1WzZs1KP0d8fLwGDBigUaNGKSQkRIWFha5dKuaHOTv68OHDmjVrlmbNmuV6bFJSkvr27au+ffuqR48eio6OrnQmANYqdxGlrK1pbFeDN5lFlD59+vADCAAAVImUlBTt27fPNRclJKRcIwQBAPC63Nxcvfvuu1q/fr2io6M1ZMgQDR48uMz73nbbbYqMjHSdx2nXrp1eeOEFL6YFKsfhcGjGjBl65plnXDNPJOmOO+7QK6+8opYtW1bZc0dGRiopKck1H8XpdGrbtm2ugsrSpUt14MABSefnqnz//feu9mIRERHq3r27q6jSp08fjxR6AHhXuYsod911V6kT13feeaeioqJcn+fn53suGXCBrKwsbdy4UZJ03XXXWRsGAAAErJSUFE2bNo25KAAAnzd58mQVFxdr2rRpOnHihJ577jk1btxY3bp1K/P+b7zxhho3buzllEDl/fTTT/q///s/bdq0yXVd79699f/+3/9TUlKS1/OEhISoffv2at++vR566CFJUkZGhpYvX+4qqmzevFmGYaioqEgrVqzQihUr9Nprr0mS2rdv7yqq9OvXT02bNuWN6oCPK1cRZfTo0aWuu9QP5b59+1YuEVCGRYsWuS5TRAEAAFWFuSgAAH9QUFCg5cuX64033lBMTIyaN2+u66+/Xj/++OMlz9cA/qa8Q+N9QZMmTTRy5EiNHDlSknTmzBmlpaVp6dKlWrZsmVavXu2aq5Kenq709HRNnjxZktS4ceMSc1Xat2/PXBXAx5SriDJt2rSqzgFc1k8//STpfG9JfiEEAABVhbkoAAB/cPjwYRmGoWbNmrmuS0xMVFpa2iUf8+yzz8rhcOjqq6/WmDFj1LRpU29EBdzmqaHxVqpRo4Zuuukm3XTTTZKkwsJCrV271tUCbPny5a65KocOHdLnn3+uzz//XNL/5qqYRZUePXqU6AQEwPsYLA+/YM5DSUlJUVgY/2wBAEDVYS4KAMDXFRQUKCYmpsR1sbGxl2yz/vLLL6t169YqLi7WvHnz9Pzzz+u9994rdYysrCxlZWVJkjIzM+VwOCTJdSK7qhmGIafTKafT6VO7DCrL/Pp56+voLZ5er7y8PL3xxht67bXXlJubK+n80PhHHnlEzzzzjGuWSFV/HativcLDw9W7d2/17t1bTz31lJxOp7Zu3eoqqCxbtkwHDx6UdPm5Kn369FFSUpLbc1X4f8u/sF6+h7PR8Hn79+/Xnj17JNHKCwAAVD3mogAAfF1UVFSpgkleXl6pWbYm82dZeHi47rnnHi1atEjbtm0r1elh7ty5+vDDD12fm62Jjh075sn4QevEiRNWR5B0/gTmiRMnVL16dcXGxlodR5K0d+9e3XHHHSX+rf3617/WH//4RzVv3lxFRUVe/3dY1etVu3Zt3X777br99tslnd9htmbNGq1evVqrV6/W9u3bS81VMTVp0kQdO3Z0zWZp1aqVmjRpErRv/vGV/7dQPv64XhRR4PPMXSiSNGjQIAuTAACAYMBcFACAr2vUqJEk6eDBg662XPv27St3i65LvbN52LBhrp+DmZmZWrBggSSpfv36lY1cLoZhyG63KywsLODefX3ixAnVrVvX8pPc69at0+DBg3X06FHZbDY9+uij+sc//lGhXJ5crz//+c+uIknv3r312muvWTI0XrJuverXr69u3brp4YcfliSdPn1aaWlprt0qq1evVlFRkaTzg+wzMjL03XffuR4fHR2tNm3aqG3btmrXrp3atm2rNm3aqEWLFgoPD+f/LT/C90LvcKcwSxEFPs8sojRo0EBt2rSxOA0AAAh0zEUBAPi6qKgo9enTRzNmzNDEiRNdBY8nnnii1H0PHjyo4uJiNW/eXHa7XXPnzlVRUZFat25d6r61a9dW7dq1JUlxcXGu4dbeOtllGIZCQkIUEhISUCcOTeZrs8qpU6d044036syZM5LOf73ff/99NWrUSH/4wx/cPp6n1uvkyZP65JNPJElDhgzR3LlzfWL9rV6vWrVq6de//rV+/etfSzrfxm/t2rVavXq1NmzYoPXr12v79u2u1kj5+fnasGGDNmzYUOI4oaGhSkxMVMuWLdW0adMSH02aNFHjxo39Ys7M5Vi9Vp7G90LfQxEFPs0wDC1cuFDS+VZegfiNAwAA+B7mogAAfN1DDz2kd955R2PGjFF0dLSGDRvmas81YsQITZo0Se3bt9eZM2f0r3/9S1lZWYqIiNBVV12lF198UdWqVbP4FcDbVq9erezs7BLzCIqLizV79uwKFVE85cMPP3S1p3vqqac493MJUVFR6tu3r/r27eu6Li8vT9u2bdPWrVtLfOzdu9e1zg6HQ7t379bu3bsveez69eu7iioXF1maNm2qOnXq8PswgppbRZSCggLVq1dPn376qW699daqygS4pKen6/jx45KYhwIAALznwrko6enp6tixo9WRAAAooVq1anr66afLvG327Nmuy506ddK//vUvb8WCDwsLC5NhGGVeb5Xi4mK98847kqSePXuqd+/elmXxRzExMerWrVup+Ub5+fnauXOnduzY4frYt2+fMjIydOTIETkcjhL3P3bsmI4dO6bVq1eX+TyRkZFq3LhxqeLKhZ9TmEUgc+u7ZFRUlGJiYiz95org8tNPP7kuU0QBAADekpKS4rqcmppKEQUAAPi93r17q3Hjxjpy5Ijsdruk862eHnzwQcsyffHFFzp8+LAkaeLEiZblCDTR0dG65pprdM0110gqOWPD4XDo6NGjOnjwoDIyMnTw4EHXh/n5qVOnShyvsLBQe/bs0Z49ey75nAkJCZfdzdKwYUPOKcNvuf0vd/To0froo4900003VUUeoARzHkqrVq3UpEkTi9MAAIBg0bRpU7Vo0UJ79+5VamqqHnvsMasjAQAAVEpsbKwWLVqkkSNHav369YqNjdWkSZM0duxYS/IYhqE33nhDktS4cWMNHTrUkhzBJiwsTE2aNLnsebZz585dssBiXi4sLCzxmNOnT+v06dPatGlTmccMCQlRw4YNyyywmJdr1qxJOzf4JLeLKAkJCVq5cqU6deqkG2+8UfXq1Svxj9tms1E5hkfY7XYtXrxYErtQAACA96WkpGjv3r3MRQEAAAGjRYsWWr16tU/8brNixQqtXbtWkvTYY48pPDzc0jz4n9jYWLVp00Zt2rQp83bDMJSZmXnJAsvBgwd19OjREo9xOp06dOiQDh06pBUrVpR53JiYmEvuZmnSpIkaNGigatWqUWiB17ldRPnjH/8oSTp69Ki2bNlS6naKKPCUNWvWKCcnRxJFFAAA4H0pKSmaOnWqTp48yVwUAAAQUKwuoEhy7UKJiYnRAw88YHEauMNms6lu3bqqW7euunfvXuZ9CgsLdfjw4TILLOaHed7PlJeXp+3bt2v79u2XfO6YmBjVr19f9evXV7169RQXF6cWLVqoQYMGJa6vV6+eIiMjPfq6EbzcLqI4nc6qyAGUYs5DsdlsGjBggMVpAABAsElOTnZdZi4KAACA5+zbt0/z58+XJN1///1KSEiwOBE8LTIyUi1atFCLFi0ueZ/s7OxLFlgyMjJ06NAh1/weU15envbu3au9e/deMUNCQoKrsHJhgeXCz+vWravatWuzEwqXxTQf+CxzHkrXrl1Vs2ZNi9MAAIBgw1wUAACAqvH222+73qj9+OOPW5rl6NGjeuutt3Ts2DF17txZDz/8sKV5LmX79u364IMPlJ2drf79+6tNmzb69NNPVVhYqFtuuUWDBw923XfhwoWaM2eOJOmqq67Srl275HA4NHz4cF1//fVWvYRS4uPj1bFjx0u+WcnhcGjjxo168803lZGRoZo1a6qoqEg7duyQ3W5XdHS0Tp06paysrFLFFul/c1q2bdt2xSy1atVy7a4p66NevXquy3FxcVXaUmzr1q364IMPlJubqwEDBujuu+++7PM5HA598MEHWrNmjWrXrq1HH31UTZs2rbJ8Fzpy5IjeeustHT9+XF26dNGECRMUFub5ksPOnTs1efJknTlzRn379tWYMWO82tatQq+ouLhYU6ZM0Zo1a5SRkaF3331XV199tWbNmqVOnTqpbdu2ns6JIJOXl6e0tDRJtPICAADWYS4KAACAZ509e1YfffSRJOnXv/61WrVqZVmWw4cPq3Pnzjp79qyKi4v12Wef6bvvvtOUKVMsy1SW9evXq0+fPrLb7XI4HJo+fboMw1BISIgMw9CUKVP02muv6Xe/+50++eQTjRkzRiEhIXI6nTIMQzabTTabTR999JGmTJmi+++/3+qXVC5Hjx7VjTfeqOzs7FJFktDQUIWFhWn+/PkaNGiQsrOzdezYsTI/jh8/7rqclZUlwzBKPdfJkyd18uTJchVcIiIiLltkufCjTp06brUVW716tfr37y+HwyGHw6GPP/5YGzZs0Ouvv17m/Q3D0MiRI/Xll1/KbrcrLCxMkydP1vr169WyZctyP29FZGRkqEuXLjp79qzsdrs+++wz/fe//9XXX3/t0b+bfvnlF/Xq1Ut2u112u12ffPKJVq5cqcmTJ3vsOa7E7SLK3r17NWjQIGVlZalLly5atmyZq3/dkiVL9MMPP2jatGkeD4rgsmzZMhUVFUmiiAIAAKzDXBQAAADPmjZtmutcotVzlV966SVlZ2eruLhY0vk3jv/8889auHCh7rnnHkuzXWjixIkqKipy7d4xiwAOh8N1nz/84Q964IEHNGHCBBmGUeI2wzBcj3nkkUc0evRov3hz0EsvvaQzZ86UucvEbrfL6XTq+eef1/XXX69atWqpVq1aat++/WWP+fe//13PPPOMa82l8wWZ/v37q3Hjxjpx4kSJjwvvZyoqKtKhQ4d06NChcr2O+Pj4SxZZLi7CPP744youLnattcPh0D/+8Q89+uijSkxMLHXs5cuXa+7cua71LS4ulmEYevbZZzVz5sxy5auoP//5zyUKXMXFxfrvf/+rH3/8UTfccIPHnud3v/udioqKXP+m7Xa7PvjgAz3++ONXXG9PcbuI8vjjj6tOnTpavXq1atSooYiICNdtycnJrsHzQGWY81AiIiLUt29fi9MAAIBgxVwUAAAAz3E4HHrzzTclSZ06dbJ8Bu7evXtLnSQPDw/X0aNHLUpUtgMHDlxxTrXD4dCePXt07ty5y94vPz9f2dnZfjGHZu/evWUWUExOp1OHDx9265hHjhwpdV1ISIi6d++u1157rcT1hmEoOztbx48fL1VcMT8uvO306dNlPmd2drays7O1c+dOt7Je6KWXXlK3bt1Uu3btEh979+5VRESECgsLXfe12+3at29fhZ+rvPbt21dqfcLDw5WRkeHR59m/f3+JoqB0vvCVkZHhu0WU1NRUzZw5U7Vr1y4Vvn79+j73TQb+yZyHkpSUpJiYGIvTAACAYMVcFAAAAM/56quvXCd3n3zySa/ONChLp06dlJqa6uqGIp3fZXDVVVdZmKq0Dh066MiRI2XuijBFRUWpdevWql27trKysi55v4SEBNWoUaMKUnpex44dtXjx4hLrc6GwsDC328G1adOmzOtbt25d6jqbzaYaNWqoRo0aZd5+saKiImVlZZVZYCmr+HJh4eNKpk6dqqlTp5brvjabTTk5OXrhhRdUs2ZN1apVSzVr1ixxOT4+XqGhoeV+/rJ07NhRS5cuLbE+hYWFl/waV1SHDh104MCBEv/+nU6nV1sBul1ECQsLK7NvnCQdP35c1apVq3QoBLdTp05pw4YNkmjlBQAArMdcFAAAAM/45z//KUmqW7euRo0aZW0YSc8++6y++eYb7d27VyEhISouLtb48ePVu3dvq6OV8Pbbb6tXr146c+aMQkJCVFBQoIiICNesk6KiIk2fPl0xMTGaOXOmbrnlFoWEhMjhcKi4uFjh4eGuGSn//ve/LS9elddzzz2n7777zrU+hYWFMgxDUVFRMgxD1atX1yuvvOLWMceOHavZs2dr6dKlCg0NlcPh0IABAzR69OhK542IiFDDhg3VsGHDK97XMAzl5OSUKq5s375d//rXv1yFCafTqbi4OOXn51+2iHbxsbdu3aoXX3zxkvcxC0RlFVjK+rys4sukSZP03Xffaf/+/QoJCVFRUZEmTJjg8a5Cb775plauXKlTp065/h387W9/U4sWLTz6PJfjdhElOTlZr7/+um666SbXH5A2m02GYeiDDz7gpDcqbdGiRa5CHf+eAACA1ZiLAgAAUHnr16/XkiVLJEkTJkxQVFSUxYnOz6pYt26dZs6cqePHj6tTp0666aabdPz4caujlZCYmKgtW7Zo9uzZysnJUZ8+fdSsWTPNmzdPRUVF+tWvfqUuXbpIkgYNGqRNmzbp22+/lSS1b99e6enpstvtuvXWW9WuXTsrX4pbatSoobVr12rmzJk6ceKErrnmGsXFxWn58uWqXr26hg8fXqpT0pWEh4frv//9r2bPnq19+/apRYsWGjFiRKV3ZbjLZrMpLi5OcXFxpXY+/eEPf9Dnn3+uvLw8JScnKykpyVV0ycrKKvNj48aN2r9/v+x2u8LDw3X69GmdPHnykrt4DMPQ6dOnL9mC7HK5Lyy+NGvWTAkJCbLZbGrevLmaN2+uqVOnunbwmB8JCQkV3nzRtGlTbdmyRbNmzVJ2draSkpJKtF32BreLKK+++qqSkpLUrl073XbbbbLZbHr33Xe1ZcsW7dq1S6tXr66KnAgi5jyU6tWrq0ePHhanAQAAwS4lJcV1edGiRRRRAAAAKuCNN96QdP7d+r/5zW8sTvM/MTExGjdunOvzK80esUqdOnX0yCOPlLjuySefLPO+bdq0KdFS6YYbbpDdbldYmNungi0XGxur8ePHl7iuX79+ks6v1bFjx9w+ZlhYmO666y6P5KsKdevW1YQJExQWFubaNXRh0aW8OzAMw1B+fr5OnTqlkydP6tSpU+W6XNHiy8qVK/X5559fNlP16tVdLeXKKrSUdb358fDDD1vWFcDt/3PatGmjdevW6YUXXtDMmTMVGhqqb775RoMGDdJnn32mli1bVkVOBBFzHkpKSopffnMHAACBpUmTJmrZsqX27Nmj1NRUPf7441ZHAgAA8CtHjhxxnVy9++67VbduXYsTAYHPZrMpJiZGMTExaty4cbkfZxZfylN4ufDzM2fOKD8//7LHzsnJUU5Ojg4ePFih17NgwQINGjTI7cdWVoXOUCcmJurjjz/2dBZAGRkZ2rVrlyRaeQEAAN+RkpKiPXv2MBcFAACgAt577z3Z7XZJ0sSJEy1OA+ByLiy+NGnSxK3HFhYWKjs7W2fOnCn1cfr0aR06dEjFxcWXvE9BQcElj23OwbGC20WUgQMH6r333iuxJcy0c+dOPfzww1q4cKFHwiH4mLtQJFlSVQQAAChLSkqKpkyZolOnTmnLli3q1KmT1ZEAAAD8Qn5+vt5//31J598wS2tUIHBFRkaqbt26Ze42M9uv1a9f/5JvSisoKFB2drZOnz5dZpElMTGxql9CmdwuoqSmpurs2bNl3nb27FnXgCigIsx5KPXr1/erQVcAACCwXTi4MDU1lSIKAABAOc2YMUMnT56UxC4UAJcXFRWlqKgo1atXz+ooJVSoD4E50OZiK1asoKchKswwDNdOlIEDB17y3xkAAIC3mXNRpPNFFAAAAFyZYRj65z//KUlq1aqVbrrpJmsDAUAFlGsnyiuvvKJXXnlF0vkCyoABA0ptuSksLJTdbteECRM8nxJBYdu2bTp27Jgk5qEAAADfw1wUAAAA9yxYsEDbtm2TJD3xxBP8/gTAL5WriJKUlKTf/e53MgxDf/7znzVq1Cg1bty4xH0iIiLUtm1b3XrrrVUSFIGPeSgAAMCXMRcFAADAPW+88YYkKSEhQaNHj7Y4DQBUTLmKKMnJya4+0DabTQ888IAaNmxYpcEQfMx5KFdddZWaNm1qcRoAAICSmIsCAABQflu3btV///tfSdKDDz6o2NhYixMBQMW4vYdu0qRJrgJKRkaGVqxYoXPnznk8GIKL3W539RenlRcAAPBFzEUBAAAoP3MWSmhoqB555BFrwwBAJVSoEeEHH3ygRo0aqVmzZurXr5927NghSRoyZIjefPNNjwZEcFi3bp3Onj0riSIKAADwXSkpKZLkmosCAACA0rKysjRjxgxJ0h133KEmTZpYnAgAKs7tIso///lPPfbYY7rvvvu0YMECGYbhui0lJUVffPGFRwMiOJitvGw2mwYMGGBxGgAAgLKZRRRzLgoAAABKmzx5sgoKCiRJEydOtDgNAFROuWaiXOjtt9/Wc889p2effVYOh6PEba1bt3btSgHcYQ6V79y5s2rXrm1xGgAAgLIxFwUAAODyioqK9O6770qSkpKSdO2111qcCAAqx+2dKIcPH1ZSUlKZt4WHhys3N7fSoRBc8vPztWLFCkm08gIAAL7twrkoixYtsjgNAACA75k1a5aOHj0qSXryySetDQMAHuB2EaVZs2ZavXp1mbetWrVKrVq1qnQoBJfly5ersLBQEkUUAADg+5iLAgAAUDbDMPTGG29IOn8OcciQIRYnAoDKc7uI8sADD+ill17SlClTXIPAi4uL9e233+rvf/+7HnroIY+HRGAz56GEh4erX79+FqcBAAC4PHN+2+nTp7V582aL0wAAAPiOpUuXasOGDZKkxx57TGFhbk8SAACf4/Z3sv/7v//TwYMH9eCDD7oKJn369JEkTZgwQRMmTPBsQgQ8cx5K7969FRsba3EaAACAy7t4Lso111xjYRoAAADfYe5CqVatmsaPH29xGgDwjAqVg9966y09+eST+vHHH3Xy5EnVrFlT1113na6++mpP50OAO336tNatWyeJVl4AAMA/NG7cWFdddZV2796t1NRUPfHEE1ZHAgAAsNyePXv05ZdfSpLuv/9+xcfHW5wIADyjwnvqWrRoQesuVFpqaqoMw5BEEQUAAPiPlJQU7d692zUXJSTE7S65AAAAAeWtt96SYRiy2Wy8yQRAQKlwEWXz5s3KyMhQQUFBqduGDh1aqVAIHuY8lGrVqunaa6+1OA0AAED5pKSk6KOPPnLNRaGlFwAACGbZ2dmaOnWqJOm2225Ty5YtLU4EAJ7jdhFly5YtGjFihHbs2OHaQXAhm80mh8PhkXAIfOY8lOTkZIWHh1ucBgAAoHyYiwIAAPA/U6ZMUW5uriRp4sSJFqcBAM9yu4gybtw4hYWF6auvvlKrVq0UERFRFbkQBA4fPqwdO3ZIopUXAADwL8xFAQAAOM9ut+utt96SJHXu3Fn9+/e3OBEAeJbbRZT09HTNmTNHN954Y1XkQRAxd6FIFFEAAID/YS4KAACA9J///EcHDhyQdH4Xis1mszgRAHiW23/pde7cWSdOnKiKLAgy5jyUunXrqkOHDhanAQAAcE9KSookueaiAAAABKM333xTklS/fn2NHDnS4jQA4HluF1Heeecdvf766/rxxx9lt9urIhOCgGEYrp0oAwcO5J2bAADA71w8FwUAACDYrF27VsuXL5ckPfLII7T9BxCQ3D5z3a5dO/Xq1Us33nijoqOjFRcXV+IjPj6+KnIiwOzYsUNHjhyRRCsvAADgn8y5KBJFFAAAEJzMXSiRkZF66KGHLE4DAFXD7ZkoDz30kGbOnKmhQ4cyWB4VduE8lEGDBlmYBAAAoOKYiwIAAILVoUOHNHfuXEnSvffeqzp16licCACqhttFlLlz5+of//iHJkyYUBV5ECTMeSgtWrRQ8+bNrQ0DAABQQSkpKfroo490+vRp/fLLL+rcubPVkQAAALxi1qxZrlb/Tz75pLVhAKAKuf1WuRo1aqhFixZVkQVBwuFwuFpe0MoLAAD4M3O4vERLLwAAEFx++OEHSVL79u3Vvn17i9MAQNVxu4jyu9/9Tm+//TZD5VFh69ev15kzZyRRRAEAAP6tUaNGuvrqqyVRRAEAAMEjJydHS5YskSTddNNNFqcBgKrldjuv3bt3a/PmzWrZsqWSk5NVo0aNErfbbDbXUCmgLBfOQxk4cKCFSQAAACovJSVFu3bt0pIlS5iLAgAAgsLChQtVXFwsiSIKgMDndhHlm2++UWhoqCRp6dKlpW6niIIrMeehXHPNNQwdAwAAfi8lJUUffvghc1EAAEDQ+O677yRJ1atXV58+fSxOAwBVy+0iyr59+6oiB4JEQUGBli9fLolWXgAAIDAkJye7LqemplJEAQAAAc0wDFcRZeDAgYqIiLA4EQBULXoNwKtWrFihgoICSRRRAABAYLhwLsqiRYssTgMAAFC10tPTdejQIUm08gIQHNzeiWLavXu3du7c6TohfqGhQ4dWKhQCl9nKKywsTP3797c4DQAAgGckJydr165dWrp0KXNRAABAQDN3oUjSDTfcYGESAPAOt4soZ8+e1ZAhQ5Samirp/BY+6fwsFJPD4fBMOgQcc6h8r169VK1aNYvTAAAAeEZKSoo++ugjnT59Wps3b9Y111xjdSQAAIAq8f3330uSOnXqpEaNGlmcBgCqnttvkfvDH/6gY8eOaenSpTIMQ/Pnz1dqaqrGjRunxMRErVy5sipyIgCcOXNGa9eulUQrLwAAEFgunosCAAAQiM6ePatly5ZJopUXgODhdhHlhx9+0DPPPKOePXtKkho2bKj+/fvrgw8+0ODBg/X66697PCQCw+LFi+V0OiVRRAEAAIGlcePGatmypaTzv/MAAAAEop9++kl2u12SdPPNN1ucBgC8w+0iyokTJ9SkSROFhoYqNjZWJ0+edN12880364cffvBoQAQOcx5KbGysqwgHAAAQKFJSUiSVfOMIAABAIDHnocTHx6t3794WpwEA73C7iNKkSRNlZWVJkq6++mp99dVXrtvS0tIUFRXluXQIKOY8lP79+ysiIsLiNAAAAJ5ltvQ6deqUtmzZYnEaAAAAzzIMwzUP5frrr1dYmNujlgHAL7ldRPnVr37l2lEwceJEvf/+++rWrZt69+6tSZMm6b777vN4SPi/I0eOaNu2bZJo5QUAAALThXNRaOkFAAACzS+//KIjR45IYh4KgODidsn41VdfVV5eniTp3nvvVbVq1TRnzhzl5+frnXfe0UMPPeTxkPB/CxcudF2miAIAAAJR06ZNlZiYqH379ik1NVWPPfaY1ZEAAAA8xmzlJUk33nijhUkAwLvcKqIUFRXphx9+UOfOnVW7dm1J0pAhQzRkyJBKhcjNzdW7776r9evXKzo6WkOGDNHgwYNL3W/79u2aOXOmdu/eLUlq3bq1xo8fr4YNG1bq+VH1zN1LtWvXVqdOnSxOAwAAUDVSUlK0b98+LVmyRE6nUyEhbm/8BgAA8Enz5s2TJHXt2lUNGjSQYRgWJwIA73Drr7qIiAjdddddOnjwoEdDTJ48WcXFxZo2bZpeeOEFzZkzR+vWrSt1v3PnzmnQoEH64IMPNH36dDVt2lQvvfSSR7PA8wzDcM1DGThwICcTAABAwDJbemVlZWnr1q0WpwEAAPCMHTt2aO3atZKkESNGWJwGALzL7bPZbdq08WgRpaCgQMuXL9e9996rmJgYNW/eXNdff71+/PHHUvft1q2b+vXrp9jYWIWHh+v222/XoUOHdPbsWY/lgeft2rVLhw4dkkQrLwAAENiYiwIAAALRZ5995ro8atQoC5MAgPe5PRPllVde0RNPPKF27dqpe/fulQ5w+PBhGYahZs2aua5LTExUWlraFR+7ZcsWJSQkKC4ursT1WVlZysrKkiRlZmaqqKhIkuR0Oiudt7wMw5DT6ZTT6ZTNZvPa81Yl8+vn7tfxwoLYwIEDvboO5RGIayVVfL18HevlX1gvAMGmefPmat68ufbv36/U1FQ98sgjVkcCAACoFMMwXEWU5ORkNW3a1OJEAOBdbhdRfv/73+vkyZPq2bOnatWqpXr16pU4MWaz2bRp06ZyH6+goEAxMTElrouNjVV+fv5lH3fs2DFNnjxZDz74YKnb5s6dqw8//ND1ufmOwGPHjpU7Fy7txIkTbt3fHDzWpEkTxcTEsA5e5u56wVqsl39hvQCUJTk5Wfv379fixYtlGEZAFZEBAEDwWblypfbu3StJuvvuuy1OAwDe53YRpVu3bh7ZgWKKiooqVTDJy8tTdHT0JR+TmZmp5557TsOGDVO/fv1K3T5s2DBX4SQzM1Pff/+9JKl+/foey30lhmHIbrcrLCwsYP5wdjqdOnHihOrWrVvuuSYOh8O1q+hXv/qVV9egvAJxraSKrZc/YL38C+vlHRSnAd+SkpKijz/+WJmZmdq2bZvatWtndSQAAIAKM3ehREREaPjw4RanAQDvc7uIMn36dI8GaNSokSTp4MGDru2A+/btu+TWwKysLD377LO64YYbdPvtt5d5n9q1a6t27dqSpLi4OEVEREiSV090GYahkJAQhYSEBNSJQ0mu11UeGzZs0OnTpyVJgwYN8omTjRcL5LWS3Fsvf8B6+RfWC0AwunAuSmpqKkUUAADgt4qLizVr1ixJ0i233KKEhASLEwGA91l+5icqKkp9+vTRjBkzlJeXpwMHDmjBggX61a9+Veq+J0+e1DPPPKOUlBQq337i559/dl0eOHCghUkAAAC8o3nz5q43BDFcHgAA+LMFCxa45g7fc889FqcBAGu4vRNFks6cOaM5c+Zo586dKigoKHX7W2+95dbxHnroIb3zzjsaM2aMoqOjNWzYMHXr1k2SNGLECE2aNEnt27fXggULdPToUc2fP1/z5893Pf7dd99VnTp1KvJSUMV++uknSVLHjh1Vr149i9MAAOB76tevr7CwMBmG4ZXnM5/HW8/nLYZhuL6OvvDakpOTNWPGDKWmpsrpdFZ4Nx7r5V9YL//CenlHWFiFTjsA8BGffvqpJCk+Pl4333yzxWkAwBpu/zaza9cuJSUlqbCwUOfOnVOdOnV06tQp2e12JSQkKD4+3u0iSrVq1fT000+Xedvs2bNdl0eNGqVRo0a5GxkWKSws1LJlyyRJ1113ncVpAADwTePGjZMk2e12rz6vw+Hw6vN5Q0JCgpxOp5xOp9VR1K9fP82YMUMnTpxQenq62rRpU6njsV7+hfXyL6xX1WcB4J9ycnL05ZdfSpKGDx+uqKgoixMBgDXcLqL89re/Vc+ePfXFF18oNjZW3333na655hrNmjVLf/rTn/TFF19URU74obS0NOXn50uiiAIAwKVMmTJFQ4cO9dquWsMw5HA4FBoaGlCzipxOp06ePKlatWr5xKyiAQMGuC4vX75cHTp0qNBxWC//wnr5F9bLOzIzM62OAKCC/vOf/7jO69DKC0Awc7uIsnr1ak2ZMkWRkZGSpKKiIoWGhuquu+5SVlaWHn/8cS1fvtzjQeF/zHkooaGhJQasAgCA/zl27JjsdrvXT+DZbLaAOmlos9lcX0dfeF0tW7ZUgwYNdPToUaWlpenhhx+u1PF85XV5iq+tl6cF2utivfyLr62Xt3daAvAcs5VX48aN1b9/f4vTAIB13H5bSmFhoeLi4hQSEqKaNWvqyJEjrts6dOigjRs3ejIf/Jg5D6Vnz56qXr26xWkAAAC8x2azqXfv3pLO784FAADwJ8eOHXOd17nrrrt8YmcbAFjF7e+ArVq10oEDByRJXbp00XvvvaecnBzl5+dr8uTJatiwocdDwv+cPXtWa9askUQrLwAAEJySkpIknZ8pSDsbAADgTz7//HPXXKW7777b4jQAYC23iyh33nmna7fJX/7yF61Zs0YJCQmKi4vT3Llz9cILL3g4IvzR4sWLXQMaKaIAAIBgZO5EkaSVK1damAQAAMA9n332mSSpY8eO6tSpk8VpAMBabs9E+d3vfue63KtXL23ZskU//PCD8vPzNXDgwAoPzURgMbd8xsTEqFevXhanAQAA8L6uXbsqIiJCRUVFWrFihW699VarIwEAAFzRjh07tHbtWknsQgEAyY0iytatW/X+++9r3759atSokYYPH65BgwapSZMmeuCBB6oyI/yQOVS+X79+ioyMtDgNAACA90VFRalr165auXIlc1EAAIDfMHehSOfnoQBAsCtXO69ly5apa9euevfdd7VmzRpNmTJFN9xwg95///2qzgc/dOzYMaWnp0uilRcAAAhu5lyU1atXq7i42OI0AAAAl2cYhquIkpycrCZNmlicCACsV64iyqRJk9SmTRvt379fx44d08mTJ3X77bfr2Wefrep88EMLFy50XaaIAgAAgpk5FyU/P1+//PKLxWkAAAAub+XKldq7d68kWnkBgKlcRZTNmzfr+eefd1Wf4+Li9Prrr+vUqVPKyMio0oDwP+Y8lJo1a6pz587WhgEAALCQuRNFklasWGFhEgAAgCszd6FERERo+PDhFqcBAN9QriJKVlaWGjduXOI6s6CSlZXl+VTwW4ZhuOahDBw4UCEh5fonBgAAEJAaNmyopk2bShJzUQAAgE8rLi7WrFmzJEm33HKLEhISLE4EAL6h3Ge4bTZbVeZAgNizZ48OHjwoiVZeAAAA0v92o7ATBQAA+LIFCxa43ix9zz33WJwGAHxHuYsoAwYMUFxcnOvDrEb369evxPXx8fFVFha+z9yFIkmDBg2yMAkAAIBvMOeiHDhwQEeOHLE4DQAAQNk+/fRTSVJ8fLxuvvlmi9MAgO8IK8+dJk2aVNU5ECDMeShNmzZVy5YtLU4DAABgvQvnoqSlpWnYsGEWpgEAACgtJydHX375pSTpjjvuUFRUlMWJAMB3UESBxzidTi1atEjS+VZetIADAACQrrnmGkVHRys/P58iCgAA8Enz589Xfn6+JOnuu++2OA0A+BamfsNjNm3apJMnT0piHgoAAIApPDxcPXr0kMRcFAAA4Js+++wzSVLjxo3Vv39/i9MAgG+hiAKPuXAeCkUUAACA/zHnoqxbt06FhYUWpwEAAPifY8eOudqz33XXXQoJ4XQhAFyI74rwGPMHbvv27VW/fn2L0wAAAPgOcy5KUVGR1q9fb3EaAACA//n888/ldDol0coLAMpCEQUeUVRUpKVLl0piFwoAAMDFevXq5bqclpZmYRIAAICSzFZeHTt2VKdOnSxOAwC+hyIKPGLlypXKy8uTRBEFAADgYnXr1tVVV10libkoAADAd+zYsUNr166VxC4UALgUiijwCHMeSmhoqJKTky1OAwAA4HvMll5paWkyDMPiNAAAAP/bhSKdn4cCACiNIgo8wpyH0qNHD8XHx1ucBgAAwPeYw+WPHDmigwcPWpwGAAAEO8MwXEWU5ORkNWnSxOJEAOCbKKKg0nJycrR69WpJtPICAAC4FHMnisRcFAAAYL2VK1dq7969kqR77rnH4jQA4LsooqDSlixZIrvdLokiCgAAwKW0b99e1atXl8RcFAAAYD1zF0pERISGDx9ucRoA8F0UUVBp5jyU6OhoV5sKAAAAlBQaGqqePXtKYicKAACwVnFxsWbNmiVJuuWWW1SjRg1rAwGAD6OIgkoz56H07dtXUVFRFqcBAADwXeYbTjZu3Ki8vDyL0wAAgGC1YMECZWVlSaKVFwBcCUUUVMqJEye0efNmSbTyAgAAuBJzLordbtfatWstTgMAAILVp59+KkmKj4/XzTffbHEaAPBtFFFQKQsXLnRdpogCAABweWY7L4m5KAAAwBo5OTn68ssvJUl33HEHXUUA4AoooqBSzHkoCQkJ6tKli8VpAAAAfFtCQoLatWsnibkoAADAGvPnz1d+fr4k6e6777Y4DQD4PoooqBRzHsqAAQMUGhpqcRoAAADfZ85FWbFihQzDsDgNAAAINtOnT5ckNWnSRP3797c2DAD4AYooqLC9e/dq//79kmjlBQAAUF7mXJSsrCzt3r3b4jQAACCY7N27V4sWLZIkjR49WiEhnBoEgCvhOyUqzGzlJVFEAQAAKC9zJ4pESy8AAOBdH3/8sevymDFjrAsCAH6EIgoqzGzl1bhxY7Vq1criNAAAAP6hdevWSkhIkMRweQAA4D0Oh0PTpk2TJCUnJ6tly5YWJwIA/0ARBRXidDq1cOFCSed3odhsNosTAQAA+IeQkBD16tVLEjtRAACA9yxcuFAZGRmSpLFjx1qcBgD8B0UUVMjmzZuVlZUliVZeAAAA7jLnomzevFlnz561OA0AAAgG5i6U6tWra9iwYRanAQD/QREFFXLhPJSBAwdamAQAAMD/mHNRDMPQ6tWrLU4DAAAC3enTpzVv3jxJ0p133qnY2FiLEwGA/6CIggoxiyht2rRRo0aNLE4DAADgX6699lqFhJz/VZy5KAAAoKp9/vnnKiwslCTdf//9FqcBAP9CEQVuKy4u1pIlSyTRygsAAKAiqlevrk6dOkliLgoAAKh6U6dOlSS1bt3atSMWAFA+FFHgttWrVys3N1eSNGjQIIvTAAAA+CfzBEZaWpqcTqfFaQAAQKDavHmz1q5dK+n8LhSbzWZxIgDwLxRR4DazlVdISIhSUlKsDQMAAOCnzOHy2dnZ2r59u8VpAABAoDIHyoeGhuq+++6zOA0A+B+KKHDbwoULJUndunVTjRo1rA0DAADgpy5spcFcFAAAUBWKior06aefSpJuvPFGNWjQwOJEAOB/KKLALXl5eVq5cqUk5qEAAABURosWLVS3bl1JzEUBAABV45tvvlFmZqYkBsoDQEVRRIFbVq1apeLiYkkUUQAAACrDZrO5dqOwEwUAAFSFDz/8UJJUp04d3XrrrRanAQD/RBEFblm2bJkkKTIyUn369LE4DQAAgH8z56Js375dp06dsjgNAAAIJAcOHNB///tfSdKYMWMUERFhcSIA8E8UUeAWs4iSlJSk6Ohoi9MAAAD4twvnopgtUwEAADxh6tSpMgxDkjR+/HiL0wCA/6KIgnI7efKk0tPTJdHKCwAAwBO6d++usLAwScxFAQAAnuNwODR16lRJUv/+/dWqVSuLEwGA/6KIgnJbtGiR6x0MFFEAAAAqLzo6Wl26dJHEXBQAAOA5P/zwgw4dOiRJeuCBByxOAwD+LczqAPAfCxculCTFxcWpe/fuFqcBAAAIDElJSVqzZo1Wr14tu93u2pkCAIDV6tevr7CwMNcbKqua+Tzeej5vMQzD9XX01mv76KOPJEk1atTQ0KFDq+R5WS//wVr5F9bLO9z5u4u/0FBuZhGlf//+/HEPAADgIb1799abb76p3NxcbdmyRZ07d7Y6EgAAkqRx48ZJkux2u1ef1+FwePX5vCEhIUFOp1NOp7PKn+vo0aP6+uuvJUl33323wsPDq3QNWS//wVr5F9ar6rOUF2fCUS4ZGRnatWuXJFp5AQAAeFJSUpLrclpaGkUUAIDPmDJlioYOHao6dep45fkMw5DD4VBoaKhsNptXntMbnE6nTp48qVq1aikkpOo763/66aeuk68PPvhglb0RlvXyH6yVf2G9vCMzM7Pc96WIgnL5+eefXZcHDhxoYRIAAIDA0qRJEzVq1EiHDx/WihUr9Jvf/MbqSAAASJKOHTsmu93u9ZN4NpstoE4c2mw219exql+X0+nUlClTJEk9e/ZUp06dqvT5JNbLnwTaawrktZJYr6rmzg4960s+8AtmEaVOnTpq3769xWkAAAACi7kbJS0tzeIkAADAny1atEh79+6VJI0fP97iNAAQGCii4IoMw3AVUfr06eMTlUIAAIBA0rt3b0nSnj17dOLECYvTAAAAf2UOlK9WrZpGjhxpcRoACAwUUXBF27dv19GjRyVJffv2tTgNAABA4Ll4LgoAAIC7srKyNG/ePEnSqFGjVK1aNYsTAUBgoIiCK/rpp59clymiAAAAeF6XLl0UGRkpSVqxYoXFaQAAgD+aMWOGioqKJEkPPPCAxWkAIHBQRMEVma28WrRooSZNmlicBgAAIPBERESoe/fuktiJAgAA3GcYhj788ENJ0jXXXOP6vQIAUHkUUXBZdrtdqampkqSBAwdaGwYAACCAmS291qxZ43oXKQAAQHmsWLFC27Ztk3R+oDzzbAHAcyii4LLWr1+v7OxsSRRRAAAAqpI5XL6goECbNm2yOA0AAPAn5i6UqKgo3X333RanAYDAQhEFl7Vw4ULX5QEDBliYBAAAILCZRRSJuSgAAKD8srOzNXv2bEnSHXfcoYSEBIsTAUBgoYiCyzLnoXTq1El169a1OA0AAEDgql+/vhITEyUxFwUAAJTfv//9b+Xn50tioDwAVAWKKLikwsJCLVu2TJJ03XXXWZwGAAAg8JlzUdiJAgAAysts5dWmTRv17dvX4jQAEHgoouCS0tLSVFBQIIl5KAAAAN5gtvTKyMjQoUOHLE4DAAB83bp167RhwwZJDJQHgKpCEQWXZLbyCg0NVf/+/S1OAwAAEPh69uzpumyeEAEAALgUcxdKeHi47rvvPovTAEBgooiCSzKHyvfo0UNxcXEWpwEAAAh87du3V0jI+V/Rf/nlF4vTAAAAX5abm6t///vfkqTbb79dderUsTgRAAQmiigoU05OjlavXi2JeSgAAADeEh0drauvvloSRRQAAHB5X3zxhXJyciQxUB4AqhJFFJRp6dKlstvtkpiHAgAA4E2dOnWSJG3evNniJAAAwJeZrbwSExN5AywAVCGKKCiTOQ8lKipKSUlJFqcBAAAIHmYRZceOHSooKLA4DQAA8EXp6elKS0uTJI0bN87VDhQA4Hl8h0WZzHkoffr0UVRUlMVpAAAAgkfHjh0lSU6nU1u3brU4DQAA8EXmLpSQkBCNGTPG2jAAEOAooqCUrKwsbdy4URKtvAAAALzN3Iki0dILAACUVlBQoBkzZkiSbrnlFjVq1MjiRAAQ2CiioJRFixa5LtNTEwAAwLuaNWum6tWrS2K4PAAAKG3+/Pk6deqUJAbKA4A3UERBKWYrr7i4OHXr1s3iNAAAAMElJCTE1dKLnSgAAOBiZiuvhg0b6qabbrI4DQAEPoooKMUcKp+cnKywsDCL0wAAAAQfs4jCThQAAHCh3bt3uzqIjB07lvM2AOAFFFFQQkZGhnbt2iWJeSgAAABWMeeinDhxQsePH7c4DQAA8BUfffSRJMlms2ncuHEWpwGA4EARBSWYrbwk5qEAAABY5cLh8lu2bLEwCQAA8BXFxcWaPn26JGnQoEFq3ry5pXkAIFhQREEJZiuvOnXqqEOHDhanAQAACE4X/h7GXBQAACBJX3/9tWuHKgPlAcB7KKLAxTAM106UgQMHymazWZwIAAAgONWoUUNNmzaVxE4UAABwntnKq06dOho8eLDFaQAgeFBEgcvOnTt1+PBhSbTyAgAAsJrZ0oudKAAA4ODBg/rhhx8kSaNHj1ZERITFiQAgeIRZHcAbqlWrprCwMBmG4bXnNJ/Lm89ZWWYrL0kaMGBAqeyGYbi+jv70uq7EH9eqPFgv/8J6+RdfW6+wsKD4cQ4EnU6dOumbb77R1q1bZbfbFR4ebnUkAABgkalTp7r+9hg/frzFaQAguATFWZcuXbooISFBdrvd68/tcDi8/pwV9dNPP0mSmjVrpqZNm5b59UpISJDT6ZTT6fR2vCrnT2tVXqyXf2G9/IsvrVdCQoLVEQBUgY4dO0qSCgsLtXv3brVt29biRAAAwAoOh0NTp06VJPXr10+tW7e2OBEABJegKKJs2LBBHTt2VJ06dbz2nIZhyOFwKDQ01C9mizidTi1evFjS+V0oZb3T0el06uTJk6pVq5ZCQgKnE5y/rVV5sV7+hfXyL762XpmZmVZHAFAFzHZekvTLL79QRAEAIEj997//VUZGhiQGygOAFYKiiJKbmyu73W7JCTybzeYXJw5/+eUXnTp1SpI0aNCgMjPbbDbX19EfXpO7Au11sV7+hfXyL762XlbstARQ9Vq1aqWIiAgVFRXpl19+0Z133ml1JAAAYIEPP/xQklSjRg0NHz7c4jQAEHysf/ssfMKF81AGDhxoYRIAAABI5+cdtWvXThLD5QEACFbHjh3T119/LUm65557FB0dbXEiAAg+FFEgSVq4cKEkqW3btmrQoIHFaQAAACD9r6UXRRQAAILT9OnTXTMmaeUFANagiAIVFRVpyZIlkqTrrrvO4jQAAAAwmcPl9+/fr+zsbIvTAAAAb3I6nfroo48kSddee22JeWkAAO+hiAKtWbNG586dk0QrLwAAAF9y4cmSLVu2WJgEAAB4W2pqqvbs2SNJGj9+vMVpACB4UUSBax5KSEiIUlJSrA0DAAAAF3MnikRLLwAAgo05UD42NlYjR460OA0ABC+KKHDNQ+natasSEhIsTgMAAABTvXr1VLduXUnSunXrLE4DAAC85eTJk5o3b54kadSoUapevbrFiQAgeFFECXJ5eXlKS0uTRCsvAAAAX2Oz2dS9e3dJ0qpVqyxOAwAAvGXGjBkqKiqSxEB5ALAaRZQgt2zZMtcPZYbKAwAA+J6ePXtKOj8TJScnx+I0AACgqhmG4Wrl1alTJ/Xo0cPiRAAQ3CiiBDmzlVd4eLj69OljcRoAAABc7Nprr5V0/oTK2rVrLU4DAACqWlpamrZu3Srp/EB5m81mcSIACG4UUYKcOVS+d+/eio2NtTgNAAAALta9e3fXyRNaegEAEPjMXShRUVG65557LE4DAKCIEsROnz6t9evXS2IeCgAAgK+Kj49X27ZtJUkrV660OA0AAKhK2dnZmjVrliRp+PDhSkhIsDgRAIAiShBbsmSJnE6nJOahAAAA+DKzpdeqVatkGIbFaQAAQFWZOXOm8vPzJTFQHgB8BUWUIGa28oqJiXH9YQ4AAADfYw6XP3bsmDIyMixOAwAAqorZyqt169bq16+fxWkAABJFlKBmDpXv37+/IiIiLE4DAACAS+nVq5frMi29AAAITOvXr3e1XWegPAD4DoooQer48eNKT0+XxDwUAAAAX9e+fXvFxMRIYrg8AACBytyFEh4ervvuu8/iNAAAE0WUIGXuQpEoogAAAPi6sLAw9ejRQxI7UQAACETnzp3TZ599JkkaPHiw6tata3EiAICJIkqQMosoCQkJ6ty5s7VhAAAAcEXmXJT169eruLjY4jQAAMCTZs+erZycHEkMlAcAX0MRJUiZRZSUlBSFhoZanAYAAABXYhZRCgoK9Msvv1icBgAAeNJHH30kSWrevLkGDRpkcRoAwIUoogSh/fv3a+/evZJo5QUAAOAvGC4PAEBgSk9P14oVKyRJ48aNU0gIp+sAwJfwXTkILVq0yHWZIgoAAIB/aNiwoRo3biyJ4fIAAAQScxdKSEiIxowZY20YAEApFFGCkNnKq169emrbtq3FaQAAAFBeZksvdqIAABAYCgoK9Mknn0iSbr75ZtcbJgAAvoMiSpAxDMNVRBk4cKBsNpvFiQAAAFBeZkuvXbt26dSpUxanAQAAlTV//nzXz3QGygOAb6KIEmR27typI0eOSKKVFwAAgL8xd6JI0urVqy1MAgAAPMFs5dWwYUPdfPPNFqcBAJSFIkqQMXehSBRRAAAA/E23bt0UGhoqiZZeAAD4uz179rjO09x///0KCwuzOBEAoCwUUYKM+cO5WbNmSkxMtDgNAAAA3BETE6NOnTpJYrg8AAD+ztyFIknjxo2zMAkA4HIoogQRp9OpRYsWSWIeCgAAgL8yW3qtWrVKhmFYnAYAAFREcXGxpk2bJkkaNGgQb3QFAB9GESWIbN68WSdPnpQkXXfddRanAQAAQEWYw+VPnz6tXbt2WZwGAABUxDfffKPjx49LYqA8APg6iihB5MJ5KAMGDLAwCQAAACrqwuHytPQCAMA/ffjhh5Kk2rVra/DgwRanAQBcDkWUIGIWUdq0aaOGDRtanAYAAAAV0apVK8XHx0tiuDwAAP4oIyNDP/zwgyRp9OjRioyMtDgRAOByKKIECbvdrsWLF0s6Pw8FAAAA/ikkJKTEXBQAAOBfpk6d6pprNn78eIvTAACuhCJKkFi3bp1ycnIkUUQBAADwd2YRZdOmTcrPz7c4DQAAKC+Hw6EpU6ZIkvr27as2bdpYnAgAcCUUUYLEhfNQUlJSrAsCAACASjOHy9vtdq1fv97iNAAAoLwWLFigjIwMSQyUBwB/QRElSJhFlM6dO6tWrVoWpwEAAEBlXHvtta7LtPQCAMB/mAPl4+PjNXz4cIvTAADKgyJKECgsLNSyZcsk0coLAAAgENSuXVstW7aUxHB5AAD8xfHjx/X1119Lku655x7FxMRYnAgAUB4UUYJAWlqaCgoKJFFEAQAACBRmSy92ogAA4B9mzJghu90uiYHyAOBPKKIEAbOVV2hoqPr162dxGgAAAHiCOVz+4MGDOnr0qMVpAADA5RiGoWnTpkmSunbtqs6dO1sbCABQbhRRgoBZROnRo4fi4uIsTgMAAABPMIsoErtRAADwdWvXrtX27dslSePGjbM4DQDAHRRRAlxubq7rj2paeQEAAASOzp07KzIyUhJFFAAAfN3nn38uSYqMjNSoUaMsTgMAcAdFlAC3bNkyV79NiigAAACBIyIiQl26dJHEcHkAAHxZbm6uvvrqK0nSsGHDlJCQYHEiAIA7KKIEOLOVV0REhJKSkixOAwAAAE8yh8uvXbtWDofD4jQAAKAss2fPVl5eniRp7NixFqcBALgrzOoAqFqLFi2SJCUlJSk6OtriNAAA+Jfc3Fy9++67Wr9+vaKjozVkyBANHjy41P22b9+umTNnavfu3ZKk1q1ba/z48WrYsKG3IyPImHNRcnNztXXrVnXs2NHiRAAA4GLTp0+XJCUmJmrAgAHWhgEAuI2dKAHs9OnTWr9+vSRaeQEAUBGTJ09WcXGxpk2bphdeeEFz5szRunXrSt3v3LlzGjRokD744ANNnz5dTZs21UsvvWRBYgSbC4fL09ILAADfs337di1fvlySNGbMGIWEcCoOAPwN37kD2JIlS+R0OiWJdzoAAOCmgoICLV++XPfee69iYmLUvHlzXX/99frxxx9L3bdbt27q16+fYmNjFR4erttvv12HDh3S2bNnLUiOYNK8eXPVrVtXEsPlAQDwRdOmTZMk2Ww23XfffRanAQBUBEWUAGbOQ4mJidG1115rcRoAAPzL4cOHZRiGmjVr5rouMTFRBw8evOJjt2zZooSEBMXFxVVlREA2m821G4UiCgAAvqW4uFgff/yxJCk5OVlNmza1OBEAoCKYiRLAzCJKv379FBERYXEaAAD8S0FBgWJiYkpcFxsbq/z8/Ms+7tixY5o8ebIefPDBMm/PyspSVlaWJCkzM9M1DNzcPVrVDMOQ0+mU0+mUzWbzynN6g/n189bX0VvKs17XXnutvv76a6Wnp+vMmTN+UbwL5vXyR6yXfwnU9QL80Xfffafjx49Lku68806L0wAAKooiSoA6ceKEtmzZIol5KAAAVERUVFSpgkleXp6io6Mv+ZjMzEw999xzGjZsmPr161fmfebOnasPP/zQ9fnIkSMlnS++oPJOnDhhdQSvu/rqqyWdPyG8YMEC9e3b1+JE5ReM6+XPWC//wnoB1ps6daokqWbNmrrhhhssTgMAqCiKKAEqNTXVdZl5KAAAuK9Ro0aSpIMHD7paL+zbt++SbRiysrL07LPP6oYbbtDtt99+yeMOGzZMycnJks4XXRYsWCBJql+/vgfTX5phGLLb7QoLCwu4d16fOHFCdevWDaiBreVZrxtuuEE2m02GYWjXrl0aPny4l1O6L5jXyx+xXv7F19aLNwkgWB09elTffvutJOnuu+9WZGSkxYkAABVFESVAma284uPj1aVLF4vTAADgf6KiotSnTx/NmDFDEydOdBU8nnjiiVL3PXnypJ555hmlpKRc8QR27dq1Vbt2bUlSXFycQkNDJclrJ7oMw1BISIhCQkIC6qShyXxtgaI861WjRg21a9dO6enpWr16tV+9/mBcL3/GevmXQFsvwN/MmDHD1bZ17NixFqcBAFQGv1EFKLOIkpycrLAwamUAAFTEQw89pNDQUI0ZM0bPP/+8hg0bpm7dukmSRowYofT0dEnSggULdPToUc2fP18jRoxwfWRmZloZH0HkwuHyhmFYnAYAgOBmGIamTJkiSerevbs6depkcSIAQGVwdj0AHTp0SLt27ZLEPBQAACqjWrVqevrpp8u8bfbs2a7Lo0aN0qhRo7wVCyilZ8+emjp1qo4fP64DBw6oefPmVkcCACBorVixQjt37pTELhQACATsRAlAixYtcl1mHgoAAEDg69Wrl+vyqlWrLEwCAACmTZsm6Xx7WN5oAwD+jyJKADJbedWuXVsdOnSwOA0AAACqWvv27RUbGytJWrlypcVpAAAIXufOndOsWbMkSUOHDlWNGjWsDQQAqDSKKAHGMAxXEWXAgAEMEgQAAAgCoaGh6t69uyR2ogAAYKU5c+YoNzdXknT//fdbnAYA4AmcYQ8we/fu1cGDByUxDwUAACCYmC291q9fr6KiIovTAAAQnKZOnSpJatq0KedlACBAUEQJMOYuFIkiCgAAQDDp2bOnJKmwsFCbNm2yOA0AAMFn9+7dWrJkiSRpzJgxdAcBgADBd/MAYw6Vb9iwoa6++mqL0wAAAMBbzCKKREsvAACsMH36dNflMWPGWJYDAOBZFFECyIXzUAYOHCibzWZxIgAAAHhLw4YN1aRJE0kMlwcAwNscDoc+/vhjSedn1CYmJlqcCADgKRRRAsi2bdt0/PhxSbTyAgAACEbmbhR2ogAA4F0//fSTDh06JEkaO3asxWkAAJ5EESWAMA8FAAAguJnD5Xfv3q2TJ09anAYAgOAxbdo0SVJcXJyGDh1qcRoAgCdRRAkgZhElMTFRzZo1szgNAAAAvI25KAAAeN+pU6c0f/58SdLIkSMVExNjcSIAgCdRRAkQTqdTqampktiFAgAAEKy6du2q0NBQSRRRAADwlpkzZ6qoqEiSdP/991ucBgDgaRRRAsSmTZt0+vRpSRRRAAAAglVMTIyuueYaSRRRAADwlqlTp0qS2rZtW2JXKAAgMFBECRAXzkMZMGCAhUkAAABgpQuHyzudTovTAAAQ2DZt2qT169dLOj9Q3mazWZwIAOBpFFEChFlEadu2rRo0aGBxGgAAgODmcDi0YcMGLVu2TNnZ2V59bnO4/JkzZ7Rr1y6vPjcAAMHGHCgfGhqqe+65x+I0AICqQBElABQXF2vJkiWS2IUCAABgtezsbPXt21ddu3ZVv3791LRpUy1dutRrz39hG5GVK1d67XkBAAg2RUVF+vTTTyVJt9xyi+rXr29xIgBAVaCIEgDWrVun3NxcScxDAQAAsNqjjz7qaushSTk5Obr11luVk5Pjlee/+uqrVaNGDUnMRQEAoCp9/fXXOnnypCQGygNAIKOIEgAunIeSkpJiXRAAAADo559/VlFRketzwzCUnZ2t7du3e+X5Q0JCSsxFAQAAVcNs5VW3bl3dcsstFqcBAFQViigBwCyidO7cWbVq1bI4DQAAQHCrXr26W9dXBbOIsmnTJuXl5XnteQEACBZHjhzR999/L0m65557FB4ebnEiAEBVoYji5woKCrR8+XJJzEMBAADwBX/6058UEvK/X7MjIiI0aNAgtW7d2msZzCKKw+Eo0VoMAAB4xowZM+R0OiXRygsAAh1FFD+3cuVKFRQUSGIeCgAAgC8YPXq0pk+frg4dOqh58+YaO3as/vOf/8hms3ktA8PlAQCoOoZhaOrUqZKkHj16qEOHDhYnAgBUpTCrA6ByFi1aJEkKDQ1V//79LU4DAAAASbr33nt17733Wvb8tWrV0lVXXaXdu3czFwUAAA9LS0vTzp07JUljx461OA0AoKqxE8XPmfNQunfvrri4OIvTAAAAwFf06tVLEsPlAQDwNHMXSlRUlEaOHGlxGgBAVaOI4sfOnTvnas/APBQAAABcyGzplZGRoSNHjlicBgCAwHDu3DnNmjVLkjRkyBDVqFHD2kAAgCpHEcWPLVu2THa7XRLzUAAAAFDShXNR2I0CAIBnzJ07V7m5uZIYKA8AwYIiih8z56GEh4erT58+FqcBAACAL7nmmmsUGRkpiSIKAACeMm3aNElSkyZNeEMrAAQJnyii5Obm6tVXX9Wdd96pMWPG6MsvvyzzfsXFxfrb3/6m8ePH67bbbtO6deu8nNS3mPNQevfurZiYGIvTAAAAwJdERESoa9eukuRqAQsAACpu3759Sk1NlSSNHj1aoaGh1gYCAHiFTxRRJk+erOLiYk2bNk0vvPCC5syZc8kCSdu2bTVx4kTVrl3byyl9S3Z2tutrxDwUAAAAlMVs6bV27VpXG1gAAFAxH3/8sevy6NGjLUwCAPAmy4soBQUFWr58ue69917FxMSoefPmuv766/Xjjz+Wum94eLgGDx6s9u3bKyTE8uiWWrJkiZxOpyTmoQAAAKBsvXr1knR+CG56errFaQAA8F9Op1PTp0+XJPXr109XXXWVtYEAAF5jeSXi8OHDMgxDzZo1c12XmJiogwcPWpjK95mtvKKjo0sMDQUAAABMDJcHAMAzFi9erAMHDkhioDwABJswqwMUFBSUmucRGxur/Pz8Ch8zKytLWVlZkqTMzEwVFRVJkmvnhjcYhiGn0ymn0ymbzebx45tD5ZOSkhQeHu6V12Y+hze/jt5Q1WtlFdbLv7Be/iVQ1wtA4GnWrJnq1aun48ePa9WqVXrwwQetjgQA8JDc3Fy9++67Wr9+vaKjozVkyBANHjz4so/5+eef9eabb+o3v/mNbrrpJi8lDQzmLpSYmBgNHz7c2jAAAK+yvIgSFRVVqmCSl5en6OjoCh9z7ty5+vDDD12fJycnS5KOHTtW4WP6klOnTmnTpk2SpB49enj9dZ04ccKrz4fKYb38C+vlX1gvAL7OZrOpZ8+e+uqrrxguDwAB5sL5sidOnNBzzz2nxo0bq1u3bmXe/+zZs5ozZ46aNm3q5aT+LycnR3PmzJEk3XHHHapevbrFiQAA3mR5EaVRo0aSpIMHD7p+kO/bt69SP9SHDRvmKpxkZmbq+++/lyTVr1+/kmnLzzAM2e12hYWFefzd18uWLXNdvu2227z2upxOp06cOKG6desG1EyaqlwrK7Fe/oX18i++tl6B8iYBAFXDLKJs27ZN2dnZio+PtzoSAKCSzPmyb7zxRqn5spcqokybNk2DBw/WkiVLvJzW/33xxRfKy8uTJI0ZM8baMAAAr7O8iBIVFaU+ffpoxowZmjhxojIzM7VgwQI98cQTZd6/uLhYhmHIMAw5HA4VFRUpLCysxEms2rVrq3bt2pKkuLg4RURESJJXT3QZhqGQkBCFhIR4/MTh4sWLJUnVq1dXjx49vH4Cz3xdgaIq18oXsF7+hfXyL4G2XgACkzlc3jAMrVmzRoMGDbI4EQCgsi41XzYtLa3M+2/ZskUZGRl67LHHLltEubg9usPhkOS9Nra+2g542rRpks5/jfv27ev21yNQ2wH76npVViCuF2vlX1gv32N5EUWSHnroIb3zzjsaM2aMoqOjNWzYMNc7J0aMGKFJkyapffv2kqTf/OY3rvYpL730kiTpr3/9qzp27GhNeAuYQ+X79++vsDCfWEIAAAD4qO7du8tms8kwDK1atYoiCgAEAHfmyxYXF+v999/XxIkTr/gGoIvbo48cOVJScO983rdvn6sjyNChQyvV0pd2wP6F9fIfrJV/8cf18okz8NWqVdPTTz9d5m2zZ88u8flHH33kjUg+68iRI9q+fbskacCAARanAQAAgK+Li4tTu3btlJ6erlWrVlkdBwDgAe7Ml503b546dOigli1bXvG4F7dHX7BggSTvtUf3xXbA7777ruvyb37zmwp9LXytHbCn+OJ6eUIgrhdr5V9YL+9w5w0CPlFEQfmlpqa6Ll933XXWBQEAAIDf6NWrl9LT07Vy5UoZhhFQf4wBQDByZ77spk2bdODAAa1YsUKSlJubq71792rnzp2lWqlf3B49NDRUkvfao/taO2CHw6FPPvlE0vk3spanEHU5gdYO2NfWy9MCab1YK//Cevkeiih+xmzlVbNmTXXq1MniNAAAAPAHPXv21JQpU5SZman9+/crMTHR6kgAgEpwZ77sH//4R9ntdtfnr7zyinr27KkbbrjBm5H90qJFi3To0CFJDJQHgGDmXyUfuIooKSkpflexAwAAgDXM4fKStHLlSguTAAA85aGHHlJoaKjGjBmj559/vtR82fT0dElS9erVlZCQ4PoICwtTTEyMqlWrZmV8v2AOlK9WrZqGDRtmcRoAgFXYieJH9u/fr3379kmSBg4caHEaAAAA+It27dqpWrVqys3N1apVqzRq1CirIwEAKsmd+bIXevnll6sqUkDJzs7WvHnzJJ0vSsXGxlqcCABgFbYy+JFFixa5LjNUHgAAAOUVGhqq7t27SxLD5QEAKIdZs2apoKBAknT//fdbnAYAYCWKKH7EbOVVr149tW3b1uI0AAAA8CdmS6/169ersLDQ4jQAAPi26dOnS5Kuuuoq9enTx9owAABLUUTxE4ZhuHaiDBw4UDabzeJEAAAA8Cc9e/aUJBUVFWnTpk0WpwEAwHft2LFDaWlpks4PlOccDAAEN4oofmLXrl06fPiwJFp5AQAAwH1mEUWSli1bZmESAAB828cffyxJstlsuvfeey1OAwCwGkUUP2G28pIYKg8AAAD3NWjQQG3atJEkvffee7Lb7RYnAgDA9zgcDn3yySeSpOuuu05Nmza1OBEAwGoUUfyE2cqradOmatGihcVpAAAA4I/+7//+T5K0Z88ezZkzx+I0AAD4noULF7o6gYwZM8baMAAAn0ARxQ84nU5XEWXAgAH04gQAAECF3HvvvWrUqJEk6W9/+5sMw7A4EQAAvsUcKF+9enUNGTLE2jAAAJ9AEcUPpKenKzMzUxKtvAAAAFBxERERrt0omzZt0vfff29xIgAAfEd2drbmzZsnSRoxYoRiYmIsTgQA8AUUUfyAuQtFYqg8AAAAKueBBx5QrVq1JEkvv/yyxWkAAPAds2fPVkFBgSRaeQEA/ociih8wh8pfddVVatKkicVpAAAA4M9iY2P1xBNPSJKWL1+upUuXWpwIAADf8PHHH0uSWrZsqT59+licBgDgKyii+DiHw6HU1FRJtPICAACAZzz66KOqVq2aJHajAAAgSbt27dLy5cslSaNHj2YeLQDAhSKKj9u4caOys7MlUUQBAACAZyQkJOg3v/mNJOmHH37Qhg0bLE4EAIC1zF0oknTfffdZmAQA4Gsoovg4s5WXJKWkpFgXBAAAAAFl4sSJioyMlCS98sorFqcBAMA6TqdTn3zyiaTzs2ibNWtmcSIAgC+hiOLjzCJK+/btVa9ePYvTAAAAIFA0aNDANTR3zpw52rlzp7WBAACwyKJFi5SRkSGJgfIAgNIooviw4uJi16BPWnkBAADA037/+98rJCREhmHo1VdftToOAACWmD59uiSpWrVqGjZsmLVhAAA+hyKKD1uzZo3OnTsn6fx2UgAAAMCTWrRooZEjR0qSZsyYoUOHDlmcCAAA7zp79qzmzp0rSbrjjjsUGxtrcSIAgK+hiOLDzFZeNptNycnJFqcBAABAIHr66aclnd8F/frrr1ucBgAA75ozZ47y8/MlSaNHj7Y4DQDAF1FE8WGLFi2SJHXp0kU1a9a0OA0AAAACUceOHXXrrbdKkj744ANlZWVZnAgAAO8xW3klJiaqX79+1oYBAPgkiig+qqCgQMuXL5dEKy8AAABUrT/96U+SpLy8PL311lsWpwEAwDv27NnjmkU7evRohYRwmgwAUBo/HXzUypUrVVhYKImh8gAAAKhavXr1UkpKiiTp7bff1tmzZ60NBACAF3zyySeuy/fdd5+FSQAAvowiio8y56GEhoaynRQAAABVztyNcubMGU2ePNniNAAAVC2n06mPP/5YkpScnKzExESLEwEAfBVFFB9lFlF69Oih6tWrW5wGAAAAgW7QoEHq1q2bJOkf//iHCgoKLE4EAEDVWbx4sQ4cOCBJGjNmjLVhAAA+jSKKDzp37pxWrVoliVZeAAAA8A6bzebajXLs2DHXoF0AAAKRuQslJiZGw4YNszgNAMCXUUTxQcuWLZPdbpfEUHkAAAB4z+233642bdpIkl577TXX76QAAASS3NxczZkzR5I0fPhwOoAAAC6LIooPMlt5RUREqE+fPhanAQAAQLAICQnR008/LUnat2+fZs2aZXEiAAA8b86cOTp37pwkWnkBAK6MIooPMosovXv3VnR0tMVpAAAA4A0FBQV66KGHFB8fr4SEBD3++OOaOnWqmjdvrmrVqql///7av39/hY8/bdo0NWrUSLGxsUpOTnb1gb/YXXfdpaZNm0qS/va3v8npdF7ymP/973911VVXKSYmRp07d9bGjRtVVFSkiRMnKiEhQfHx8Ro/frzy8/MrnBsAAE8zW3k1a9ZMycnJFqcBAPi6MKsDoKQzZ85o/fr1kpiHAgAAEEzGjRunOXPmqKioSJL03nvvyeFwuG5PS0tT//79lZ6e7nbbkS+++ELjx493FURWrFihlJQUbdmyRbGxsSXuGx4erqeeekqPPfaYtmzZom+++Ua33XZbqWOuWbNGt9xyiyvjli1b1L9/f91+++36/PPPVVxcLEmaMWOGcnJy2NUCAPAJ+/btU2pqqiTpvvvuU0gI7y8GAFweRRQ3GIahtLQ07du3Ty1btlSvXr0kSWfPntWiRYtUVFSkpKQkNWrUSJK0fft2bd68WbVr19aAAQMUHh5+yWM7HA4tXrxY3377reuP27CwMJ0+fVoJCQmu+xUXF2vhwoU6deqUOnbsqCNHjigrK0udO3dWu3btSh332LFjWrZsmcLCwpSSkqIaNWpIkk6cOKFFixYpPT1dTZo0Uf/+/dW6detSjz98+LBWrFihiIgI3p0BAABQRQoKCjRz5kwZhuG67sICiiTZ7XYdOXJEixcv1q9//Wu3jv/uu++W2FFit9uVkZGhJUuW6Kabbip1/7Fjx+rPf/6zMjMz9fLLL+vWW2+VzWYrcR/zXbwX5i0sLNS///3vEtmLioo0e/ZsTZs2TTExMW7lBgDA02bMmOG6fN9991mYBADgLyiilJNhGBo7dqw+/vhjRUREqKioSA888IB+//vfKzk5WcePH5fNZlN4eLi+/vpr7dy5U4888ojCw8NVXFys7t276+eff1a1atVKHbuwsFC//vWvXW28TM8//7zefPNNpaamqm3btsrNzdXAgQO1fv16hYaGqqioSDabTRERESouLtabb76pRx991PX4ZcuW6aabblJhYaEMw1CtWrW0ePFinT17VoMGDVJOTo7rD3WbzaYPP/xQ48aNcz1+0aJF+vWvf63i4mIZhqF69erpiy++UP369avoqwwAABCczN+3riQkJESFhYVuH7+goKDUdTab7ZLHiomJ0ZNPPqlnnnlGq1atUmpqqgYMGFDqmBdnttlsl2z/VVRURBEFAGApwzBcbwLo27evrrrqKosTAQD8AXsWy2nmzJn69NNPZRiGqygxZcoU3XzzzTp+/LjsdruKi4uVl5enwYMH65FHHpHT6VRhYaGcTqc2btyoZ599tsxj//3vf9fixYvldDpL/NHpcDh08uRJ3XnnnZKkZ555Rps2bZLD4XC1eTDzOJ1OPfHEE9q6davrsUOGDNG5c+dUXFwsu92urKws3XnnnRoyZIjOnj1b4o9ewzD04IMPas+ePZLOF3aGDBmivLw81+OPHz+uxx57rEq+vgAAAMGsevXquvbaa0vsXA4NDS21+yM8PFy9e/d2+/hDhgwpcWybzabIyMjLHmvChAmKi4uTJL3yyiulbr/llltKXVdcXFzqdYSHh6tr166uHdEAAFhl2bJl2rt3ryRp9OjRFqcBAPgLiijltHbt2lLXhYaGau/evbLb7SWuz83NVWhoaInrioqKlJaWVuax09LSXD2jL+ZwOLRlyxY5nU6tWLHCVTwpS3h4uDZu3ChJOnr0qLKyskq1hNi8ebMOHz5c5uNDQkL0yy+/SJIOHjyo7OzsErfb7Xalp6df8vkBAABQcfPmzSvRnrVLly4l2ozEx8frm2++UcOGDd0+9v/93//p/vvvL3Gsb7/9VvXq1bvkY2rUqKEJEyZIkn788cdSvw8PGTJEf/3rX1295CMjI/XZZ5/p66+/VteuXV33a9Omjb788ku3MwMA4GnmLpSoqCjdcccdFqcBAPgLiijlVLt27VKFEZvNdsmWBBf3sA4JCbnkH6n169cvdewLxcXFuR5/uYFnxcXFql27tiQpISGh1DsXpfN/MF/quex2u+vxtWrVKvM+8fHxl3x+AAAAVFyjRo20fv167du3TwcOHNCqVas0bdo0HTlyRNu3b9fx48dLtdQqr9DQUE2ePFmZmZnasWOHjh07Vq55d08++aSioqIklb0b5emnn9apU6e0fft2nTx5UiNHjlSdOnWUlpamAwcOaN++fdq4caMaN25codwAAHhKXl6eZs+eLUkaOnQo5zcAAOVGEaWcxo8fr/j4eFdrgvDwcCUkJOiVV14pUawICwvTI488oqSkJEVEREg6X0AJDQ3Vc889V+axn3rqKUVERJRZ9AgJCXH9wTpp0iSFhoaWWUiJiIjQtddeq4EDB0qSYmNj9dRTTyksLKzUsSZNmlTqGKGhoRowYICSkpIkSTVr1tTjjz9e4vE2m01//OMfr/zFAgAAQIWEhISoefPmatq0qev3tdq1a6tVq1aKjIys9PHdPVa9evVcM/Pmz5+vbdu2lbpPfHy8WrdurdjYWNd1NptNTZs2VfPmzS/7JiAAALxl/vz5ysnJkUQrLwCAexgsX05169bV+vXr9cwzz2jnzp1q06aNXnrpJTVs2FANGzbU+++/r4KCAg0ZMkSPPfaY8vPz9cILL2jFihWqV6+e/vSnP6l79+5lHrtNmzZau3atevbsqdzcXNWqVUvt2rVTVFSUxo4dq5EjR0qSevTooeXLl+uVV17R8ePHdfXVV+vMmTPKzMxU79699ec//7lE0eNvf/ubmjVrpi+++ELh4eEaP368RowYIcMw1KBBA73//vs6ePCgatSooWHDhun5558vsUvljTfeUIsWLTR//nxFRUXpwQcfVK9evar2Cw0AAACf8tRTT2ny5Mmy2+169dVXNX36dKsjAQDgNrOVV6NGjXTddddZnAYA4E8oorihUaNGZf7RePvtt+v2228vcV1MTIxefvllhYWFlbnD5GJxcXHKzc2VJP3pT3/Sb3/72zLv16NHD82bN69ceW02myZMmODqZX3h9ePHj9f48eMv+/iQkBA98cQTeuKJJ/6/9u47Pqoq///4e9JDIKQSWgoJCAEC0rtAaAFlLQiyLHUtq/tb3QVERRRhdcVVEVws6FpAARERvxaKiIAIBAJGeqSYhIgkJAIJLX3m9wffuV/GBARMcmeS1/Px4GHmztx7P3cOIyf3PeccSZLValVWVtZVnRsAAADVQ2RkpEaNGqX33ntPixcv1syZMxUZGWl2WQAAXLWffvpJ69atkySNHj36ilOqAwDwa4ytdxIbNmwwfuYbEQAAAHAmjz32mCwWi0pKSvTiiy+aXQ4AANdk0aJFstlskpjKCwBw7QhRnMT69eslXVzQPS4uzuRqAAAAgP8TGxtrjLx+6623lJ2dbW5BAABcJZvNZkzl1blzZ8XGxppcEQDA1RCiOAGbzWaEKH379mXxTQAAADidqVOnSpIKCgo0d+5cc4sBAOAqJSUl6eDBg5IYhQIAuD7crXcCqampysjIkCTFx8ebXA0AAABQVqdOndS/f39J0quvvqq8vDyTKwIA4LfZR6F4eXlp5MiRJlcDAHBFhChOwD4KRSJEAQAAgPN6/PHHJUlnzpzRa6+9ZnI1AABcWWFhoZYuXSpJGjp0qIKCgkyuCADgighRnIA9RGnYsKFuuOEGk6sBAAAAytenTx916dJFkjR37lzl5+ebXBEAAJf3+eef6/Tp05KYygsAcP0IUUx26Xoo8fHxslgsJlcEAAAAlM9isRijUbKzs/XOO++YXBEAAJdnn8orNDRUCQkJJlcDAHBVhCgmO3DggLKzsyUxlRcAAACc3y233KJWrVpJkp5//nkVFxebXBEAAGWdOHFCq1evliT96U9/kqenp8kVAQBcFSGKyVgPBQAAAK7Ezc1Njz32mCQpIyNDH3zwgckVAQBQ1uLFi1VaWiqJqbwAAL8PIYrJ7CFKdHS0IiMjTa4GAAAA+G0jR45UVFSUJOm5556T1Wo1tyAAAH7FPpVX27ZtdeONN5pbDADApRGimKi0tFTffPONJEahAAAAwHV4eHjokUcekSSlpKTo008/NbkiAAD+z65du7Rnzx5JjEIBAPx+hCgm2r17t06fPi2JEAUAAACuZcKECQoLC5MkPfvss7LZbCZXBADARfZRKO7u7ho1apTJ1QAAXB0hiokuXQ+lb9++JlYCAAAAXBsfHx9NmjRJkrRz506tW7fO5IoAAJCKi4u1ePFiSdLgwYONwB8AgOtFiGIie4jSsmVL1a9f3+RqAAAAgGtz//33KyAgQJJ03333KTs729yCAAA13po1a5STkyOJqbwAABWDEMUkxcXF2rRpkySm8gIAAIBr8vf317/+9S9JUnp6um6//XYVFBSYXBUAoCazT+UVGBiooUOHmlwNAKA6IEQxyY4dO3T+/HlJhCgAAABwXQ888ID++te/SpK2bt2qe++9l/VRAACmOHnypD7//HNJ0siRI+Xt7W1yRQCA6oAQxST2qbwsFot69+5tcjUAAADA9bFYLHr55Zc1YMAASdKiRYs0a9Ysk6sCorU/ugAAOvVJREFUANRES5cuVVFRkSSm8gIAVBxCFJPYQ5R27dopKCjI5GoAAACA6+fh4aFly5apRYsWkqRp06Zp+fLlJlcFAKhp7FN5NW/eXJ07dza5GgBAdUGIYoL8/Hxt3bpVElN5AQAAoHoICAjQF198oeDgYEnS2LFjtXPnTpOrAgDUFCkpKdqxY4eki6NQLBaLyRUBAKoLQhQTJCYmqrCwUBIhCgAAAKqPmJgYrVixQp6ensrPz9cf/vAHpaenm10WAKAGsI9CsVgsGjNmjMnVAACqE0IUE9in8vLw8FDPnj1NrgYAAACoODfddJPefPNNSVJmZqYGDhyoEydOmFwVAKA6Ky0t1aJFiyRJ/fv3V+PGjU2uCABQnRCimMAeonTu3Fl16tQxuRoAAACgYo0fP14zZ86UJB0+fFgJCQnKzc01tygAQLW1fv16/fzzz5JYUB4AUPEIUarY2bNnlZSUJImpvAAAAFB9Pfnkk3rooYckSbt27dLQoUN14cIFk6sCAFRH9qm86tSpo9tvv93kagAA1Q0hShX79ttvVVpaKokQBQAAANWXxWLRnDlzjHnpN2/erOHDh6u4uNjkygAA1cmZM2e0YsUKSdLw4cNVq1YtkysCAFQ3hChVzD6Vl7e3t7p162ZyNQAAAEDlcXNz09tvv60//OEPkqRVq1Zp+PDh+uWXX0yuDABQXSxfvlz5+fmSmMoLAFA5CFGqmD1E6dGjh3x8fEyuBgAAAKhcnp6e+vDDD9W7d29J0qeffqqWLVtq+fLlstlsJlcHAHB19qm8mjRpop49e5pcDQCgOiJEqUInT57Url27JDGVFwAAAGoOHx8fffbZZ/rjH/8o6WK/+O9//7sSEhKUmppqcnUAAFeVlpamTZs2SZLGjh0rNzducwEAKh7/ulShb775xvi2HSEKAAAAahJ/f38tWbJEK1euVEREhCRp3bp1at26tV588UWVlJSYXCEAwNW89957xs9jx441sRIAQHVGiFKF7FN51a5dWx07djS5GgAAAKDqDRkyRHv37tU999wjNzc35efna8qUKercubOSk5PNLg8A4CKsVqsxlVevXr0UHR1tckUAgOqKEKUK2UOUm266SZ6eniZXAwAAAJijdu3amjlzprZu3ao2bdpIkr7//nt16tRJDz/8sM6fP29yhQAAZ7d582alpaVJYkF5AEDlIkSpIpmZmUpJSZHEVF4AAACAJHXq1Ek7d+7Uc889Jx8fH1mtVs2ePVutW7fWl19+aXZ5AAAnZh+F4uvrq+HDh5tcDQCgOiNEqSIbNmwwfiZEAQAAAC7y9PTUo48+qr1796pfv36SpPT0dCUkJGj06NHKyckxuUIAwO9x+PBhde/eXf7+/mrWrJm++OILSdLbb7+tiIgIBQQEaPDgwcrKylJeXp5GjRql4OBgNWzYUC+88IKxtuylLly4oI8++kiS5O3trcaNGyssLExBQUEKCgrS+PHjHUY1rl+/XrGxsfL391fz5s0VHR0tf39/dejQQbt27XI4dmlpqaZPn24c789//rMuXLggq9WqZ599VkFBQXJ3d5enp6c6dOig/fv3O+xfUlKixx57TPXq1VNwcLDuv/9+5efnV/C7CgCoSh5mF1BT2KfyCgwMVNu2bU2uBgAAAHAuTZs21VdffaWFCxdq8uTJOnXqlBYvXqw1a9bopZde0pgxY2SxWMwuEwBwDU6ePKkePXro9OnTKikp0dmzZ3Xrrbdq6tSpmjVrlqxWqyTp66+/Vt++fRUUFKSdO3eqqKhIkvT444/Lzc1NkydPdjjuJ598orNnz0qS8vLyZLPZjMeS9MEHHygnJ0crV65UcnKyBg0apNLS0jKv2717t3r16qX9+/crIiJCkjR9+nS98MILKi4uliQtXrxYJ0+eVJcuXTR9+nSVlpZKurgmS3Jysnr06KGUlBQ1aNBAkjRlyhS9+uqrxv7vvvuucnNz9dJLL1X4+wsAqBqMRKki9hClb9++cnPjbQcAAAB+zWKxaPz48UpJSdGoUaMkXbwBN27cOA0cOFA//vijyRUCAK7FypUrlZeXp5KSEmObzWbT66+/bgQoklRcXKwffvhBW7duNQIU6eKojpdffrnMce1TedmP92tFRUVatWqVTpw4oQULFlz2daWlpSouLtby5cuNba+88ooRgNiP9dlnn2nu3LlGgHKp/Px8ffLJJ8Y55s+fX2b/Dz/8kPW+AMCFcTe/CqSlpRmLnTGVFwAAAHBl9erV0+LFi7Vq1SpFRkZKktatW6e4uDg9//zzDjfjAADOq6CgoMwXSW02W7n/H7/cF04LCgocHh87dkzr1q274j52+fn5xlRcl2OxWBym2yosLLyqOsp7zmq1OgQol7rccQEAzo8QpQqwHgoAAABw7QYPHqx9+/Zp0qRJcnNzU35+vh599FFjQXoAgHPr3bt3mcDE3d1dPXv2lKenp7HNYrGoVq1aCgsLcwhGvLy8NGTIEIf9Fy1aZIwquVw44u7urujoaIWHhyshIeGKNRYVFal///7G44EDB8rLy8vhWM2aNdPNN98sD4+ys+IXFxcb93rc3d3Vp08fh2vz8PBQq1atFBgYeMU6AADOixClCtin8qpfv75atGhhcjUAAACA66hdu7Zmz56t7du368Ybb5Qk7dq1S126dNGkSZN07tw5cwsEAFxW8+bN9eGHH8rX19fYNnHiRK1YsUJ33HGHsS0gIECrV6/W2rVrFRYWZmzv3bu3XnnlFeOxzWYzpufq3LmzXnnlFbm7u5c5b3h4uFavXi13d3fdeeedmjFjhrGulpubm/Gzh4eH3nrrLXXp0sXY991331WHDh2Mx5GRkVq1apXmz5+vnj17OpzH09NTixYtMv59kqQlS5Y4rIUbExOjzz77jHW9AMCFsbB8JbPZbEaIEh8fzz+aAAAAwHXo2LGjkpKSNGfOHD311FMqKCjQnDlztGLFCs2fP/83v2kMADDHHXfcoezsbKWlpSksLEz16tWTJC1dulSzZ89Wbm6uoqOj5evrK5vNpsOHD+vo0aPy8/NTRESEw32UpKQkHTx4UJI0btw4/fWvf9WoUaN07NgxNWrUSLm5uSoqKlJMTIzDaJAnn3xSDzzwgDIzMxUZGani4mIdP35cERERqlu3rkO9wcHB2rJli9LS0lRcXKzo6GjjWOvXr1d6erqysrLk7e2tpk2byt/f32H/evXqafv27UpNTZXValV0dLTc3NyUlZVVKe8vAKDyEaJUsoMHDyozM1MSU3kBAAAAv4enp6ceeeQRDRs2TPfff7/WrVuno0ePavDgwRo1apTmzJlj3JwDADiP2rVrKy4ursz2Ro0aqVGjRg7bvLy8FBsbW+6XUO0Lynt5eWnkyJGSpMDAQGOqrKCgoMvWEBISopCQEONxcHDwZV9rsVgUHR1d7vYmTZqoSZMml91XujjapWnTpsbjK63JAgBwfjUiRKldu7Y8PDyMOTOrgv1c9lEoktS3b98qraGi2Ww243105ev4Nfu1VKdrkmgvV0N7uRZna6/y5mYGgOosJiZGa9eu1fvvv6+JEyfq1KlTWrJkidasWaPZs2dr3LhxjAAHgGqmsLBQS5culSQNHTr0ioEJAAAVqUbcdWnXrp0CAwPLLGZWFb7++mtJUlRUlMLDw02poSIFBgbKarVWy29RlJaWml1ChaO9XAvt5Vqcqb1YpBJATWSxWDR27FgNHjxYEydO1OLFi3Xq1ClNmDBBixYt0vz58x2+BQwAcG2ff/65Tp8+LeniVF4AAFSVGhGifP/994qLi1NoaGiVndNms6m4uFjffPONpIujUFz9m8JWq1UnT55UcHCw3NzczC6nwthsNpWWlsrd3b1afWOR9nIttJdrcbb2ysnJMbsEADBNaGioFi1apDFjxuj+++9Xenq6vv76a8XFxempp57S5MmTHebFBwC4JvtUXqGhoayDBQCoUq59V/8qnTt3TiUlJVV+A2/Pnj06deqUJKlfv34ufwPRYrEY76OrX0t5qtt10V6uhfZyLc7WXq4+yhEAKsKgQYO0b98+PfXUU5ozZ44KCgo0depULV26VP/973/VqVMns0sEAFynEydOaPXq1ZKkP/3pT4TjAIAqZf7XZ6uxjRs3Gj/37dvXvEIAAACAGsDPz08vvviikpKS1K5dO0nS7t271bVrV/3jH//QuXPnTK4QAHA9Fi9ebEwRPH78eHOLAQDUOIQolWjDhg2SpBYtWqhhw4YmVwMAAADUDB06dFBSUpKef/55+fr6ymq16uWXX1arVq20atUqs8sDAFwj+1Rebdu2Vdu2bU2uBgBQ0xCiVJLi4mJ9++23ki5O5QUAAACg6nh4eGjKlCnat2+fBgwYIEnKyMjQzTffrD/+8Y86ceKEyRUCAK7Grl27tGfPHkksKA8AMAchSiXZsWOHMV1AfHy8ydUAAAAANVN0dLS+/PJLvffeewoODpYkLV26VLGxsXr33Xdls9lMrhAAcCX2USju7u4aNWqUydUAAGqiGrGwvBnWr18v6eLiw3369DG3GAAA4LTq168vDw+PKruRaz9PdbtxbLPZjPexOl0b7VVxRo8erUGDBunhhx/W+++/r9OnT+vPf/6z3n//fc2fP1/NmjX73eegvVwL7VU1PDy47YDrV1xcrMWLF0uSBg8erLCwMJMrAgDURPRmKok9RGnXrp2CgoJMrgYAADiru+++W5JUUlJSpee1L85anQQGBspqtcpqtZpdSoWjvSrunG+//bZGjhypv/3tb0pLS9OGDRvUpk0bPfHEE5o0aZI8PT1/93loL9dCe1V+LcD1WrNmjXJyciQxlRcAwDyEKJUgPz9fW7dulcRUXgAA4Mrefvtt3XHHHQoNDa2S89lsNpWWlsrd3V0Wi6VKzlkVrFarTp48qeDgYLm5VZ8Za2mvyjF48GDt2bNHM2bM0Jw5c1RYWKgnn3xSy5Yt05tvvqkuXbpc13FpL9dCe1UN+w1w4HrYp/IKDAzU0KFDTa4GAFBTEaJUgi1btqioqEgSIQoAALiyrKwslZSUVPkNPIvFUq1uGlosFuN9rE7XZVfdrssZ2qt27dp68cUXNWrUKN17771KTk7W3r171b17dz344IN6+umn5e/vf13Hpr1cS3W7Lmdrr6oeaYnq4+TJk/rss88kSSNHjpS3t7fJFQEAairzv5ZSDdmn8vLw8FCvXr1MrgYAAADA5bRv317bt2/Xiy++KF9fX9lsNv3nP/9Rw4YNdd999+m7774zu0QAqJGWLl2q4uJiSUzlBQAwFyFKJfj6668lSV26dJGfn5/J1QAAAAC4Eg8PD02ePFn79+/XwIEDJUnnz5/Xf//7X3Xs2FEdOnTQ3LlzdezYMZMrBYCawz6VV/PmzdW5c2eTqwEA1GSEKBUsLy9PO3fulCT17dvX5GoAAAAAXK0mTZpozZo1+vrrrzVy5Ehjkfnk5GRNnDhR4eHh6tGjB4EKAFSylJQU7dixQ5I0fvx4p5iaDgBQcxGiVLBvvvlGVqtVEiEKAAAA4GosFovi4+P1wQcf6Oeff9YLL7ygVq1aGc9v3bq1TKCSlpZmYsUAUP3YR6FYLBaNHj3a5GoAADUdIUoFs6+H4uvrqy5duphcDQAAAIDrFRoaqocfflj79u3TgQMHNHPmzHIDlejoaMXFxenxxx/X2rVrlZeXZ2LVAODaSktLtWjRIklS//791bhxY5MrAgDUdB5mF1Dd2NdD6dWrl7y8vEyuBgAAAFUhLy9PmZmZCg8PZ028aio2NlbTp0/X9OnTlZKSoo8++kjLli3T/v37JUn79u3Tvn37JF385nRcXJy6d++uHj16qHv37mrSpAnT0QDAVVi/fr2OHz8uiQXlAQDOgZEoFejEiRPGL07x8fEmVwMAAICqMGvWLAUGBio2NlbBwcH64IMPzC4JlcweqOzbt0+HDh3S7Nmz1adPH3l4XPyOms1m0549ezR//nyNGTNGMTExatiwoYYNG6aXXnpJ27dvV1FRkclXAQDO6b333pMk1alTR7fffrvJ1QAAwEiUCrVhwwbjZ0IUAACA6m/FihV64oknZLPZJEmFhYUaPXq0YmNjdeONN5pbHKpEs2bNNGnSJE2aNEnnzp3Ttm3btG3bNiUmJmrr1q3Kzc2VJGVlZWnFihVasWKFJMnHx0edOnUyRqt069ZNISEhJl4JAJgvLy9Pn376qSRpxIgRqlWrlskVAQBAiFKh7OuhBAQEqF27dsYv0wAAAKieVq1aVWabl5eX1q1bR4hSA/n5+al3797q16+fLBaLrFarfvjhB23ZskVbt27Vli1bdPjwYUlSQUGBvv32W3377bfG/s2bNzem/+rRo4eaN2/OFGAAapRly5apoKBAElN5AQCcByFKBbKvh9KnTx+5u7urpKTE5IoAAABQmXx8fMrc5LbZbPL29japIjgTNzc3tWzZUi1bttS9994rScrJyTECla1bt2rnzp0qLCyUJB08eFAHDx7UO++8I0kKCgpS9+7djVClY8eOfCsbQLVmn8orOjpaPXv2NLkaAAAuIkSpIOnp6UpNTZXEVF4AAAA1xbhx4zR//nzjsbu7u7y8vJjDHZcVGhqqW2+9Vbfeequki1PAJScnO4xWyc7OliSdOnVKX3zxhb744gtJkoeHh9q3b2+EKj169FCDBg1MuxYAqEiHDx/Wli1bJF3895WReAAAZ0GIUkHsU3lJUr9+/UysBAAAAFWlU6dOWrlypf7f//t/yszMVLNmzbRgwQI1btzY7NLgIry9vdWtWzd169ZN0sWRTKmpqdqyZYsRrOzfv182m00lJSVKSkpSUlKS5s6dK0mKiopymAKsdevWcnd3N/GKAOD62EehSNKYMWNMrAQAAEeEKBXEHqLUr19fsbGxJlcDAACAqjJo0CAdOXLE7DJQTVgsFsXExCgmJkZjx46VJOXm5mrbtm1GqLJ9+3adP39e0sUR8enp6Vq8eLEkqU6dOuratasRqnTt2lV16tQx7XoA4GpYrVYjROnTp4+ioqLMLQgAgEsQolQAm81mrIcSHx8vi8XCovIAAAAAKkRAQIASEhKUkJAgSSopKdGePXscpgD76aefJElnz57VV199pa+++krSxXVZ4uLiHEarREZGMk0OAKeyceNGZWRkSJJGjx5tcjUAADgiRKkAP/zwg7KysiSxHgoAAACAymVfG6V9+/Z68MEHJUk//fSTw4L1u3btUmlpqaxWq3bv3q3du3frtddekyQ1bNjQYcH6du3aydPT08xLAlDDLViwQJLk5+enO+64w9xiAAD4FUKUCmAfhSKxHgoAAACAqhceHq677rpLd911lyTp3Llz2rFjh7G2SmJiovLy8iRJx48f1/Lly7V8+XJJkq+vrzp16qQePXqoY8eOatOmjaKjo+Xm5mba9QCoOc6ePauPP/5YknTnnXeqdu3aJlcEAIAjQpQKYA9RoqOjmbcTAAAAgOlq166tvn37qm/fvpIurjdw4MABh9Eq9rV88vPztWnTJm3atMnY38/PT61bt1ZcXJxat26tRo0aqXfv3goNDTXlegBUX8uXL9eFCxckSePGjTO5GgAAyiJE+Z1KS0u1ceNGSYxCAQAAAOCc3Nzc1Lp1a7Vu3Vr33XefJOnEiRNKTEw0QpWdO3eqqKhIknT+/Hlt375d27dvdzhOgwYN1KZNG8XFxSkuLk5t2rRRbGysvL29q/yaAFQPCxculCRFRUXppptuktVqNbkiAAAcEaL8TsnJycrNzZVEiAIAAADAdYSFhem2227TbbfdJkkqKirSDz/8oN27d2vPnj3as2eP9u7dq8zMTGOfzMxMZWZm6ssvvzS2ubu764YbbjBCFXvAEhkZyZRgAK4oLS1N33zzjSRp7NixcnNzI0QBADgdQpTf6dL1UOxD5QEAAADA1Xh5ealNmzZq06aNw/bs7Gx98803On78uPbt26c9e/Zo3759xvQ7paWlSklJUUpKipYtW2bsV6dOHWP0S8uWLRUbG6uWLVuqcePGslgsVXptAJzTe++9Z/w8duxYEysBAODyCFF+J3uIEhcXp3r16plcDQAAAABUrJCQEPXo0UP169c3RpZYrValpaVp7969xoiVvXv36vDhw8a3yM+ePavExEQlJiY6HK9OnTqKjY01QhV7wBIVFSV3d/cqvz4A5rBarcZUXr169VJMTIxsNpvJVQEAUBYhyu9QUFCgzZs3S2IqLwAAAGeRnp6utLQ0RUZGKjo62uxygGrJzc1NMTExiomJMaYDky4uUp+SkuIQrBw4cEA///yz8ZqzZ88qKSlJSUlJDsf08fFRixYtHEattGzZUjExMfL09KyqSwNQRb799lulpaVJYkF5AIBzY4La3yExMVEFBQWSCFEAAACcwb/+9S81adJE/fr1U0xMjJ544gmzS4IuLmCekJCgOnXqqEGDBnrttdec8tvGBw8eVJcuXeTn56cmTZro3//+t1q3bi0/Pz+1aNFCmzZtMl67du1aNWvWTH5+fmratKkiIyNVu3ZtdejQQcnJySZexbXLysoy2qd+/foaOXKkGjRooDp16mjQoEHKysq66mP5+vqqffv2uvPOO5WTk6Pt27eroKBAEydO1ObNm/X222/r4Ycf1pAhQxQVFeWwb0FBgXbt2qUlS5boySef1LBhwxQbGys/Pz+1atVKw4cP1/Tp07V06VLt2bPH+F3sehw4cECdO3dWQECAYmJi9Pnnn1/x9VarVc8884xCQ0Pl7++v4cOHG2tjVrasrCwNGjRIderUUcOGDfX6669XynmOHDmibt26yc/PT5GRkVq+fHmlnAewW7BggSSpVq1aGjFihLnFAABwBYxE+R3sU3m5u7vrpptuMrkaAACAmm3Dhg2aMWOGJBk36GfNmqVOnTrp1ltvNbGymq2oqEj9+vXToUOHVFxcrHPnzukf//iHCgsLNXHiRLPLM/zyyy/q2bOncnNzVVJSovT0dD322GOyWCyy2Ww6dOiQBgwYoOTkZF24cEFDhgxRaWmpJOnHH380jrN//3716dNH+/btU2RkpFmXc9Xs7XP48GGjfT788EPj+Q0bNqh///5atWrVNR33rrvu0rp161RUVCRJeuWVV+Tl5aXnnnvO4XXnz5/XwYMHdeDAAaWkpOjAgQM6cOCAjhw5YkwLVlxcbGy/lJubm6Kjox2mBGvZsqVatGih2rVrX7a27Oxs9erVS3l5eSotLVVaWppuu+02bdy4Ub169Sp3n+eee04zZ85USUmJJOmzzz5Tdna2Nm7cWKnruxQWFio+Pl5Hjhwx2udvf/ubfH19NX78+Ao7z+nTp9WzZ0+dPHlSJSUlysjI0IgRI7RmzRoNHDiwws4D2J07d04fffSRJGnYsGGqU6eOyRUBAHB5hCi/w/r16yVJnTt3lr+/v8nVAAAA1Gzbtm2Th4eHcdNWuvhll02bNhGimCg5OVn79+932FZaWqp33nnHqUKU1atX68yZM8ZNcjt7IGez2WSz2bR48WKdPXv2ssexWq0qLi7W8uXLNXny5EqtuSLs3LmzTDhxqeLiYqWkpGj37t2KiIi4qmPm5OSUCV2Ki4v1+uuvlwlR/Pz81L59e7Vv395he2FhoQ4fPmyEJ/aA5eDBgyouLpZ08b0+cuSIjhw5os8++8xh/4iICCNcsQcssbGxCgwM1BdffKFz584ZIZh0sX3ffvvty4Yo8+bNc/i7UVRUpE2bNiktLa1Spw3cuXOnfvjhB4eRW1arVfPmzavQEGXt2rU6deqUwzXabDa9+eabhCg1XP369eXh4VHhowc//vhjnT9/XtLFqbwu/X/tpf+tLmw2m/E+Vqdro71cB23lWmivquHhcfXRCCHKdTpz5owxhy9TeQEAAJivdu3aZb4RbrFY+HaryQoKCozRHL/e7kzsdf6WwsJC5efnX/EXP4vFosLCwoosr9Jcrn0u5ebmdk3Xc7m2vTTg/C3e3t5q3bq1Wrdu7bC9pKREqampZcKVlJQU5efnG6/LyMhQRkaG1qxZ47C/fZqySwMU6eIv9Zfuf7W1V3Y7V9Xnp6CgQG5uZWf7vtJ7gprh7rvvlqQyAfPvZZ/KKyIiQj179ixz/F9/RquDwMBAWa1WY5RddUJ7uQ7ayrXQXpVfy9UiRLlOmzZtMv4iE6IAAACYb9iwYXr99ddVWlqqkpISubu7y8vLq0K/rY1r165dOwUGBur06dPGjWBPT08lJCSYXJmjPn36/OYvcyUlJRo0aJDOnTund99997KvKyoq0oABAyq6xErRvn17BQQEKDc3t9wgxWKxqG7duoqLi7vqYzZq1EjNmzfXjz/+aNwY9fLy0qBBg353vR4eHrrhhht0ww03OCxob7VadfToUYcpwew/nzlzxnhdZmamMjMzyz12UlKSRo4cqRtuuEHNmjUzzhMYGKhbbrlFS5cuNcIUd3d3NWzYUDExMb/7mq6kffv28vf3V15ensPnp6JH15U3PbWHh4eGDh1aoeeB63n77bd1xx13KDQ0tMKOmZ6ero0bN0qSxo4dKy8vL+M5m82m0tJSubu7V+pUeVXNarXq5MmTCg4OLjewdFW0l+ugrVwL7VU1cnJyrvq1hCjXyb4eiq+vr7p162ZyNQAAAGjYsKESExP10EMP6dChQ4qJidG8efPKLF6NqlW3bl2tXr1at9xyi/GLyh133KEpU6aYXJmjZs2a6eOPP9bIkSN14cIFSVJCQoLWrVtnhHJz5swxpjZ65plnNG3aNNlsNrm7u8tqtcpms8nT01NvvfWWOnXqZOblXLWAgACtXr1aQ4cONdonIiJCGRkZkqSQkBB9+umnqlu37lUf083NTStXrtTgwYN1+PBhSVLXrl2vGDz9Xm5ubmrSpImaNGmiIUOGGNttNpuOHz9eZs2VXbt2lZmWLT09Xenp6WWOHRwcrJiYGAUHBxsBTEhIiJYtW+Zw87cyBAYGGu3zyy+/SJKGDx+umTNnVuh5mjRpohUrVmjEiBHGFEsPPfSQ/vKXv1ToeeB6srKyVFJSUqE38d5//33j5/Hjx5d7bIvFUq1uHFosFuN9rE7XZVfdrqs6t1d1u6bq3FYS7VXZrmWUJSHKdbKHKD179pS3t7fJ1QAAAECSmjdvrq+++srsMvArnTt31rFjx5SWliZ/f3+FhYUpKyvL7LLKsAcJR48eVb169RQcHKyzZ8/q2LFjatiwoUOQMHXqVN1///3KyspSeHi4SkpKdPz4cTVo0EABAQHmXcR16NKli9E+devWVf369ZWVlaW8vDxFRUXJ09PzmtsrJiZGKSkpSk9Pl6enp8LDw035ZdlisahRo0Zq1KhRmdFB6enpWr9+vc6cOaPjx4/r0KFDOnz4sI4cOeIwfdfJkyd18uRJh31PnDihbt26KSAgQNHR0YqJiVF0dLTDz+Hh4dc01/bldO3aVceOHVN6errRPpVhyJAhysnJUXp6ukJDQxUSElIp50HNZrVajam8evbsWemjuQAAqAiEKNchOztbe/fulcRUXgAAAMDV8PLyUvPmzSXJKeZAvpxatWopNjbWeFynTh2Hx5cKDAx0mEu5bt26Fb5uQFW5tH2ki4tJ22/WX297ubu7O/UN0sjISI0ZM0YeHh4OAU9paakyMjKMUOXQoUPGn6NHjzq8H7m5uUpOTlZycnKZ47u7uysyMrLcgCU6OvqaRvd4e3s7tE9l8fX1vezfd6AibN68WWlpaZLEdJsAAJdBiHId1q9fb/wcHx9vYiUAAAAAgIrk7u5uTA3263VcCgsLlZ6ertTUVP34449KTU11+Nk+FZx0MYyxP1+eoKCgywYsjRs3lru7e6VeJ2AG+ygUX19fDR8+3NxiAAC4SoQo18E+lVdAQIDat29vcjUAAAAAgKpgHxFS3qgQm82m7OzsywYsx48fd3j9qVOndOrUKe3YsaPMsTw9PRUVFVUmYImKilJERISCgoKcYi5x4FqcO3dOy5YtkyQNGzZM/v7+JlcEAMDVIUS5DvYQpU+fPnw7CAAAAAAgi8WisLAwhYWFqVu3bmWez8/PV3p6erkBS2pqqgoKCozXFhcX6/Dhwzp8+HC556pVq5YiIiIUGRmpiIgIhz/h4eH8ngqntGLFCp0/f14SU3kBAFwLIco1SktLM+bvZD0UAAAAAMDVsK83Ut6aI1arVSdOnLhswJKVleXw+gsXLuiHH37QDz/8UO65LBaLGjRo4BCu/DpwCQwMZDQLqpR9Kq/w8HD17dvX3GIAALgGhCjX6NL1UAhRAAAAAAC/l5ubmxo0aKAGDRqoZ8+eZZ4/f/680tLSlJGRYfw5evSo8fPPP/+s0tJS4/U2m03Hjx/X8ePHtW3btnLP6efnV264Ehsbq44dO1bataJmSk9P14YNGyRJ48aNk5ubm8kVAQBw9QhRrtHPP/8sT09PhYSEqEWLFmaXAwAAAACo5vz8/NS6dWu1bt263OdLSkqUmZmpjIwMpaen68CBAzp9+rRD6JKXl+ewz/nz55WSkqKUlBSH7YMGDdKaNWsq7VpQM+3fv18BAQHKzc3VuHHjzC4HAIBrQohyjaZPn67JkycrNTWVoc8AAAAAANN5eHgoPDxc4eHh6tatm7KyslS/fn2Hb/vn5eXpp59++s3RLBERESZeCaqrm2++WZmZmdq0aZOaNm1qdjkAAFwTQpTr4Ofnp7i4OLPLAAAAAADgqtStW1d169b9zdEsfFkQlcXHx0cDBw40uwwAAK4ZIQoAAAAAADWcfTQLAAAAHLGSFwAAAAAAAAAAQDkIUQAAAAAAAAAAAMpBiAIAAAAAAAAAAFAOQhQAAAAAAAAAAIByEKIAAAAAAAAAAACUgxAFAAAAAAAAAACgHIQoAAAAAAAAAAAA5SBEAQAAAAAAAAAAKAchCgAAAAAAAAAAQDkIUQAAAAAAAAAAAMpBiAIAAAAAAAAAAFAOQhQAAAAAAAAAAIByEKIAAAAAAAAAAACUgxAFAAAAAAAAAACgHIQoAAAAAAAAAAAA5SBEAQAAAAAAAAAAKAchCgAAAAAAAAAAQDkIUQAAAAAAAAAAAMpBiAIAAAAAAAAAAFAOQhQAAAAAAAAAAIByEKIAAAAAAAAAAACUgxAFAAAAAAAAAACgHIQoAAAAAAAAAAAA5SBEAQAAAAAAAAAAKAchCgAAAAAAAAAAQDkIUQAAAAAAAAAAAMpBiAIAAAAAAAAAAFAOQhQAAAAAAAAAAIByEKIAAAAAAAAAAACUgxAFAAAAAAAAAACgHIQoAAAAAAAAAAAA5fAwu4Cq8ssvv1Tp+Tw8PBQYGKicnByVlJRU6bkrW1ZWltklVKjq3FYS7eVqaC/X4iztVdX/xqHiVWUb8rl0LbSXa6G9XAvtVTXop7g++ikVx1k+lxWF9nIdtJVrob2qxrX8+1btQ5RatWrJ09NTK1asMLsUl1dQUKCjR48qMjJSPj4+ZpeD30B7uRbay7U4Y3t5enqqVq1aZpeBa0Q/peI44+cSl0d7uRbay7U4Y3vRT3FN9FMqjjN+LnF5tJfroK1cizO219X2USw2m81WBfWYKjc3VxcuXDC7DJf3448/auLEiZozZ45iYmLMLge/gfZyLbSXa3HG9qpVq5YCAgLMLgPXgX5KxXDGzyUuj/ZyLbSXa3HG9qKf4rrop1QMZ/xc4vJoL9dBW7kWZ2yvq+2jVPuRKJIUEBBAh60CnDlzRpIUGhqqhg0bmlwNfgvt5VpoL9dCe6Ei0U+pGHwuXQvt5VpoL9dCe6Ei0U+pGHwuXQvt5TpoK9fiyu3FwvIAAAAAAAAAAADlIETBVQsJCdG9996rkJAQs0vBVaC9XAvt5VpoL8D58Ll0LbSXa6G9XAvtBTgfPpeuhfZyHbSVa3Hl9qoRa6IAAAAAAAAAAABcK0aiAAAAAAAAAAAAlIMQBQAAAAAAAAAAoByEKAAAAAAAAAAAAOXwMLsAOKfi4mLNnz9fu3fv1tmzZxUSEqIRI0aod+/ekqR77rlHubm5cnO7mMOFhobq1VdfNbPkGm3u3LnatGmTPDz+7yP96quvKjQ0VJKUk5OjefPmKSUlRXXr1tXYsWN10003mVVujTZixAiHx0VFRerYsaOeeOIJSXy2nMEXX3yh9evXKz09Xd26ddOUKVOM544ePap58+YpPT1dYWFhuu+++9S2bVvj+S1btmjhwoU6deqUWrRooYceekj16tUz4zKAaos+imuhj+Ja6Kc4N/oogPOjn+Ja6Ke4Fvopzq2691MIUVCu0tJSBQUF6ZlnnlFYWJhSUlL0z3/+U2FhYWrRooUkaerUqerQoYPJlcLu1ltv1bhx48p97sUXX1RUVJSmTZumQ4cO6ZlnnlFkZKQiIyOruEosW7bM+Lm0tFR33323evTo4fAaPlvmCgoK0ogRI7Rr1y6dPXvW2F5SUqKnn35aAwcO1KxZs7Rt2zbNmjVL8+fPV0BAgH766Se9/PLLmjp1qlq2bKn3339fzz//vF588UUTrwaofuijuB76KK6Dfopzo48COD/6Ka6HforroJ/i3Kp7P4XpvFAuHx8f/elPf1L9+vVlsVjUsmVLxcbGKiUlxezScI2OHz+uQ4cOacyYMfL29lZcXJw6d+6s9evXm11ajZecnKyCggJ1797d7FJwie7du6tr167y9/d32L53714VFhbqzjvvlKenp3r16qWIiAht2bJFkrRx40a1b99e7dq1k7e3t0aNGqW0tDRlZGSYcRlAtUUfpfqgj+Lc6Kc4H/oogPOjn1J90E9xbvRTnE9176cwEgVXpaCgQEeOHNHQoUONbXPnzpXNZlNERIRGjx6tli1bmlghvvzyS3355ZcKCQnR0KFDNWDAAEkXh8yFhoaqdu3axmubNGmiPXv2mFUq/tfXX3+tXr16ydvb22E7ny3nlJGRoaioKGNosCRFR0fr6NGjki5+1po1a2Y8V6tWLdWvX19Hjx5VREREldcL1BT0UZwffRTXRD/FddBHAZwX/RTnRz/FNdFPcR3VpZ9CiILfZLVaNXfuXDVr1kzt2rWTJE2aNEkxMTGSLv6Pa+bMmZo3b57TzVdXUwwdOlR//vOf5efnp/379+vf//63/Pz81L17dxUUFDj8oy9Jfn5+ys/PN6laSNKZM2eUlJSkWbNmOWzns+W88vPz5efn57DNz89P2dnZki7+glTe83zWgMpDH8X50UdxTfRTXAt9FMA50U9xfvRTXBP9FNdSXfopTOeFK7LZbHrttdd06tQpTZkyRRaLRZLUsmVLeXt7y9vbW0OGDFF0dLS+++47k6utuWJiYuTv7y93d3e1adNGN998szEszsfHR+fPn3d4/YULF+Tr62tGqfhfGzduVIMGDdS8eXOH7Xy2nJevr2+Zz9L58+eNz5KPj48uXLjg8DyfNaDy0EdxDfRRXBP9FNdCHwVwPvRTXAP9FNdEP8W1VJd+CiEKLstms2n+/PlKS0vTjBkzrviX183NTTabrQqrw5VYLBajPSIjI5WTk6Nz584Zz6emprIQmsm+/vpr9e/f/zdfx2fLeUREROjo0aOyWq3GtrS0NOOzFBkZqdTUVOO5/Px8ZWVl8VkDKgF9FNdFH8U10E9xLfRRAOdCP8V10U9xDfRTXEt16acQouCy3njjDR08eFAzZ85UrVq1jO05OTnav3+/iouLVVxcrC+//FKHDx82hqei6m3evFkXLlyQ1WrVgQMHtHLlSnXt2lWS1LBhQzVt2lSLFi1SYWGh9u3bp6SkJMXHx5tcdc31448/KiMjQ3369HHYzmfLOZSWlqqoqEhWq1VWq1VFRUUqKSlRXFycvLy8tGLFChUXF2vz5s06evSoevToIUnq06ePkpOTtWvXLhUVFWnJkiWKiopyqjk8geqCPorroI/ieuinOC/6KIBroJ/iOuinuB76Kc6ruvdTLDYiOZQjOztb99xzjzw9PeXu7m5sv/POO9W1a1fNnj1bmZmZ8vDwUHh4uEaPHq24uDgTK67ZHnvsMSPVtS+GlpCQYDyfk5Oj//znP0pJSVFAQIDGjBmj3r17m1hxzfbGG2/ol19+0bRp0xy2Z2Rk8NlyAkuWLNHSpUsdtsXHx+sf//iH0tPT9corryg9PV316tXTX/7yF7Vt29Z43ebNm7Vw4UKdPn1azZs319///nfmXwUqGH0U10IfxfXQT3Fe9FEA50c/xbXQT3E99FOcV3XvpxCiAAAAAAAAAAAAlIPpvAAAAAAAAAAAAMpBiAIAAAAAAAAAAFAOQhQAAAAAAAAAAIByEKIAAAAAAAAAAACUgxAFAAAAAAAAAACgHIQoAAAAAAAAAAAA5SBEAQAAAAAAAAAAKAchCgAAAAAAAAAAQDkIUQAAAAATzZgxQxaLRRaLRW5ubqpbt67i4uL0t7/9TSkpKZV67v/5n//Ra6+9Vmb7+PHj1bp16+s+7sqVK9W4cWMVFRVJkjZu3CiLxaKdO3de9zGrSm5urmbMmKEDBw5U6Xm3bNmikJAQnTlzpkrPCwAAAODKCFEAAAAAk/n6+ioxMVFbt27V8uXLNWHCBK1bt0433nijFi1aVGnnvVyI8nvYbDZNmzZNEydOlJeXlySpffv2SkxMVGxsbIWeqzLk5uZq5syZVR6i9OjRQ61atdLs2bOr9LwAAAAArowQBQAAADCZm5ubunbtqq5du2rAgAGaNGmSdu3apZ49e+ruu+9Wamqq2SVetY0bN2rfvn0aO3assc3f319du3aVn5+fiZWZw2azqbCw8Kpee/fdd+v1119XcXFxJVcFAAAA4GoRogAAAABOyMfHR/PmzVNRUZHeeusth+cWLFigNm3ayMfHR40aNdK0adNUWlrq8LzFYtG2bdsUHx+vWrVqKSoqSu+8847xmvHjx2vhwoXav3+/MZ3Y+PHjHc6zceNGtWvXTn5+furcubO+++6736x74cKF6t27t0JDQx2O8+vpvCwWi55//nnNmDFDYWFhCgkJ0YQJE3T+/PkrHn/GjBmqXbu2vv/+e3Xr1k2+vr5q3769vv/+exUUFOiBBx5QYGCgGjdurLlz55bZPzExUfHx8fLz81PdunU1atQoZWdnS5LS09PVpEkTSdLw4cON9yU9PV2SVFhYqMcff1yRkZHy9vZWbGyslixZ4nB8+1Roq1atUtu2beXt7a3PP/9cxcXFmjJliiIiIuTt7a0GDRpo6NChysvLM/a97bbblJubq1WrVv3m+wwAAACgahCiAAAAAE6qZcuWatSokRITE41tL730ku655x4NGjRIn3/+uR599FH95z//0bRp08rsP3LkSA0YMECffPKJ+vbtq7vvvltr1qyRJD355JMaMmSIoqOjlZiYqMTERD355JPGvllZWXrooYc0ZcoULVu2TAUFBbr99tt/c5TEunXr1KNHj6u6vldeeUWHDx/WwoULNX36dC1ZskRPP/30b+5XXFyscePG6b777tPHH3+s4uJi3XHHHbrnnnvk6+urZcuW6bbbbtPEiRO1detWY7/ExET16dNHdevW1Ycffqg333xTO3bs0K233ipJatCggVasWCFJevbZZ433pUGDBpKkESNG6I033tDkyZP1xRdfKCEhQaNHj9bq1asd6jt+/LgeeughTZw4UWvWrNGNN96oWbNmaf78+Xrssce0du1avfLKK2rYsKHDKBV/f3+1atVKX3311VW9fwAAAAAqn4fZBQAAAAC4vPDwcGVlZUmSzp49q6eeekqPPPKInn32WUnSgAED5OXlpUmTJmnKlCkKDg429h07dqymTp0qSRo0aJBSU1M1c+ZMJSQkKCYmRqGhoTp69Ki6du1a5rynTp3SN998o1atWkmS/Pz81LdvX23fvl09e/Yst9bMzEz9/PPPatOmzVVdW4MGDbR48WJJUkJCgpKTk7V8+XI999xzV9yvqKhI//73vzV48GBJktVq1dChQ9WlSxe99NJLkqT4+Hh99NFH+uijj9S9e3dJ0mOPPaaOHTtqxYoVslgskqS4uDhj5MiQIUPUrl07SVKzZs0c3pcNGzbos88+05dffqmBAwdKuvjeZ2Zm6qmnnjJqkaTTp09r9erV6tKli7EtKSlJAwcO1F//+ldj27Bhw8pcW9u2bbV9+/arev8AAAAAVD5GogAAAABOzGazGTf8t27dqnPnzmn48OEqKSkx/vTv31/5+fnat2+fw7633367w+Nhw4bpu+++c5j663IaNmxoBCjSxVExknTs2LHL7pOZmSlJDlN5XcmAAQMcHrds2fKKx7dzc3NTv379jMc33HCDJKl///7GNnd3d8XExOinn36SJF24cEFbtmzR8OHDVVpaarx3N9xwg8LDw7Vjx44rnnPt2rUKCgpSfHy8w3s/YMAAff/99w7vaXBwsEOAIknt27fXqlWrNGPGDO3YsUNWq7Xc84SEhBjvIwAAAADzEaIAAAAATuzYsWOqX7++JOmXX36RdPGGvKenp/GnWbNmkmQEBnb16tVzeBwWFqbi4mLjOFcSEBDg8NjLy0uSVFBQcNl97M95e3v/5vEvd46rWYTd19fXqOfS2so7nr2m06dPq7S0VBMnTnR47zw9PZWRkVHmvfu1X375RadOnSqz7z333KOSkhKH4CMsLKzM/tOmTdOjjz6qhQsXqnPnzqpfv75mzpwpm83m8Dpvb2/l5+f/5nsAAAAAoGownRcAAADgpPbv36+ff/7ZWPA9KChIkrRixQqFh4eXeb19UXS77OxsNWrUyHh84sQJeXp6KiQkpFLqtdeXm5tbKcf/PQICAmSxWPT444/rtttuK/P8b70nQUFBCg0Nveyi75cGVvaRQ5fy9vbWjBkzNGPGDB05ckTvvPOOZsyYoejoaI0ZM8Z4XW5ursOUbAAAAADMRYgCAAAAOKGCggI9+OCD8vb21j333CNJ6tatm2rVqqVjx46VmaqrPJ988omxxockffzxx+rQoYPc3d0lOY7UqAhRUVHy8vJSWlpahR2zovj5+albt25KSUnRM888c9nXXW7ETf/+/fX888/Ly8vrqtd8uZymTZvq2Wef1RtvvKGUlBSH59LT09W8efPfdXwAAAAAFYcQBQAAADCZ1WrVtm3bJEnnzp3T3r179eabbyo1NVULFixQVFSUpIujKf75z3/qkUce0bFjx9SnTx+5u7srNTVVn376qT7++GPVqlXLOO57770nX19ftW/fXkuXLtWmTZu0cuVK4/nY2Fi98847+uCDD9SsWTOFhIQY57oePj4+6tChg7777rvrPkZleuGFFxQfH6+77rpLI0eOVGBgoI4dO6avvvpKEyZMUJ8+fVS/fn0FBATogw8+UJMmTeTt7a02bdpowIABGjp0qBISEvTII4+oTZs2On/+vPbv368jR47orbfeuuK5b7vtNnXo0EHt2rWTn5+fPv/8c50+fVrx8fEOr9u5c6cmT55cmW8DAAAAgGtAiAIAAACYLD8/X926dZMk1a5dW1FRUerXr58++eQTtWjRwuG1kydPVqNGjfTSSy9p3rx58vT0VExMjG655RaHdUIk6YMPPtDUqVP1z3/+U/Xq1dObb76pIUOGGM/ffffdSkpK0oMPPqiTJ09q3LhxWrBgwe+6ljvvvFNz5syRzWYrd1orM3Xv3l2bN2/WU089pQkTJqioqEiNGzdWv3791LRpU0kXF61/99139fjjj6tfv34qLCxUWlqaoqKitHz5cj333HN67bXXdPToUdWtW1etW7fWhAkTfvPcPXr00LJlyzR79myVlJSoefPmWrx4sfr372+8Jjk5WTk5ORo2bFilvQcAAAAAro3F9uuVDAEAAAC4tAULFmjChAnKycmptPVPLicnJ0fh4eFau3atbrrppio9t6ubMmWKvvvuO61fv97sUgAAAAD8LzezCwAAAABQfYSGhuqBBx7Q3LlzzS7FpZw5c0ZvvfWWZsyYYXYpAAAAAC5BiAIAAACgQj3++OO68cYbVVRUZHYpLiMjI0NPP/00o3cAAAAAJ8N0XgAAAAAAAAAAAOVgJAoAAAAAAAAAAEA5CFEAAAAAAAAAAADKQYgCAAAAAAAAAABQDkIUAAAAAAAAAACAchCiAAAAAAAAAAAAlIMQBQAAAAAAAAAAoByEKAAAAAAAAAAAAOUgRAEAAAAAAAAAACjH/wftsmo8jqRlCQAAAABJRU5ErkJggg==\n",
568 |       "text/plain": [
569 |        "<Figure size 2000x1000 with 3 Axes>"
570 |       ]
571 |      },
572 |      "metadata": {},
573 |      "output_type": "display_data"
574 |     },
575 |     {
576 |      "name": "stdout",
577 |      "output_type": "stream",
578 |      "text": [
579 |       "\n"
580 |      ]
581 |     }
582 |    ],
583 |    "source": [
584 |     "plot_df = predt_params_transf\n",
585 |     "plot_df[\"depth\"] = x_train\n",
586 |     "\n",
587 |     "plot_df = pd.melt(plot_df,\n",
588 |     "                  id_vars=\"depth\")\n",
589 |     "\n",
590 |     "param_plot = (ggplot(plot_df,\n",
591 |     "                     aes(x=\"depth\",\n",
592 |     "                         y=\"value\")) + \n",
593 |     "              geom_point() + \n",
594 |     "              geom_smooth(span=0.7, se=False) + \n",
595 |     "              facet_wrap(\"variable\",\n",
596 |     "                         scales=\"free\") + \n",
597 |     "              theme_bw() + \n",
598 |     "              theme(subplots_adjust={\"wspace\": 0.25}) + \n",
599 |     "              labs(title = \"Partial-Dependence-Plot of Dirichlet-Parameters estimated via Py-BoostLSS\\n\",\n",
600 |     "                   y=\"Parameter Estimate\",\n",
601 |     "                   x=\"Depth (in meters)\") \n",
602 |     "             )\n",
603 |     "\n",
604 |     "print(param_plot)"
605 |    ]
606 |   }
607 |  ],
608 |  "metadata": {
609 |   "kernelspec": {
610 |    "display_name": "Python 3 (ipykernel)",
611 |    "language": "python",
612 |    "name": "python3"
613 |   },
614 |   "language_info": {
615 |    "codemirror_mode": {
616 |     "name": "ipython",
617 |     "version": 3
618 |    },
619 |    "file_extension": ".py",
620 |    "mimetype": "text/x-python",
621 |    "name": "python",
622 |    "nbconvert_exporter": "python",
623 |    "pygments_lexer": "ipython3",
624 |    "version": "3.9.15"
625 |   }
626 |  },
627 |  "nbformat": 4,
628 |  "nbformat_minor": 5
629 | }
630 | 


--------------------------------------------------------------------------------
/pyboostlss/__init__.py:
--------------------------------------------------------------------------------
1 | """Py-BoostLSS - An extension of Py-Boost to probabilistic modelling"""


--------------------------------------------------------------------------------
/pyboostlss/datasets/__init__.py:
--------------------------------------------------------------------------------
1 | """Py-BoostLSS - An extension of Py-Boost to probabilistic modelling"""


--------------------------------------------------------------------------------
/pyboostlss/datasets/arcticlake.csv:
--------------------------------------------------------------------------------
 1 | sand,silt,clay,depth
 2 | 0.775,0.195,0.03,10.4
 3 | 0.719,0.249,0.032,11.7
 4 | 0.507,0.361,0.132,12.8
 5 | 0.52357,0.41023,0.0662,13
 6 | 0.7,0.265,0.035,15.7
 7 | 0.665,0.322,0.013,16.3
 8 | 0.431,0.553,0.016,18
 9 | 0.534,0.368,0.098,18.7
10 | 0.155,0.544,0.301,20.7
11 | 0.317,0.415,0.268,22.1
12 | 0.657,0.278,0.065,22.4
13 | 0.704,0.29,0.006,24.4
14 | 0.174,0.536,0.29,25.8
15 | 0.106,0.698,0.196,32.5
16 | 0.382,0.431,0.187,33.6
17 | 0.108,0.527,0.365,36.8
18 | 0.184,0.507,0.309,37.8
19 | 0.046,0.474,0.48,36.9
20 | 0.156,0.504,0.34,42.2
21 | 0.319,0.451,0.23,47
22 | 0.095,0.535,0.37,47.1
23 | 0.171,0.48,0.349,48.4
24 | 0.105,0.554,0.341,49.4
25 | 0.04776,0.54428,0.40796,49.5
26 | 0.026,0.452,0.522,59.2
27 | 0.114,0.527,0.359,60.1
28 | 0.067,0.469,0.464,61.7
29 | 0.069,0.497,0.434,62.4
30 | 0.04,0.449,0.511,69.3
31 | 0.07407,0.51652,0.40941,73.6
32 | 0.048,0.495,0.457,74.4
33 | 0.045,0.485,0.47,78.5
34 | 0.066,0.521,0.413,82.9
35 | 0.06707,0.47347,0.45946,87.7
36 | 0.07407,0.45646,0.46947,88.1
37 | 0.06,0.489,0.451,90.4
38 | 0.063,0.538,0.399,90.6
39 | 0.025,0.48,0.495,97.7
40 | 0.02,0.478,0.502,103.7
41 | 


--------------------------------------------------------------------------------
/pyboostlss/datasets/data_loader.py:
--------------------------------------------------------------------------------
 1 | import pkg_resources
 2 | import pandas as pd
 3 | 
 4 | 
 5 | def load_example_data(dta_name: str) -> pd.DataFrame:
 6 |     """Returns dataframe of a sepecified simulated dataset example.
 7 |     """
 8 |     data_path = pkg_resources.resource_stream(__name__, dta_name)
 9 |     data_df = pd.read_csv(data_path)
10 | 
11 |     return data_df


--------------------------------------------------------------------------------
/pyboostlss/distributions/DIRICHLET.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch.distributions.dirichlet import Dirichlet
  3 | from dirichlet.dirichlet import mle as dirichlet_mle
  4 | import cupy as cp
  5 | import numpy as np
  6 | import pandas as pd
  7 | from pyboostlss.utils import * 
  8 | 
  9 | 
 10 | 
 11 | ########################################################################################################################
 12 | ###############################################      Dirichlet Distribution    #########################################
 13 | ########################################################################################################################
 14 | 
 15 | class DIRICHLET:
 16 |     """Dirichlet Distribution Class
 17 |     """
 18 |     
 19 |     def __init__(self, D:int):
 20 |         self.D = D # specifies target dimension
 21 | 
 22 | 
 23 |     def initialize(self, y_true: cp.ndarray, n_target: int) -> cp.ndarray:
 24 |         """ Function that calculates the starting values, for each distributional parameter individually.
 25 |         y_true: cp.ndarray
 26 |             Data from which starting values are calculated.
 27 |         n_target: ndarray
 28 |             Number of target variables
 29 |         """
 30 | 
 31 |         start_values = np.log(dirichlet_mle(cp.asnumpy(y_true)))
 32 | 
 33 |         return cp.array(start_values)
 34 | 
 35 | 
 36 |     def n_dist_param(self, n_targets: int) -> int:
 37 |         """Infers the number of distributional parameters from target dimension.
 38 |         """
 39 | 
 40 |         return n_targets
 41 | 
 42 | 
 43 | 
 44 |     def target_append(self, y_true: np.ndarray, n_param: int) -> np.ndarray:
 45 |         """Function that appends target to the number of specified parameters
 46 |         """
 47 | 
 48 |         return cp.array(y_true)
 49 | 
 50 | 
 51 | 
 52 | 
 53 |     def create_param_dict(self, n_target):
 54 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
 55 |         """
 56 | 
 57 |         # Alpha
 58 |         param_dict = {"alpha_" + str(i+1): exp_fn for i in range(n_target)}
 59 | 
 60 |         return param_dict    
 61 |     
 62 | 
 63 |     
 64 |     
 65 | 
 66 |     def get_params_nll(self, y_true: cp.ndarray, y_pred: cp.ndarray, requires_grad=False) -> torch.tensor:
 67 |         """ Returns estimated parameters and nll.
 68 |         
 69 |         Args:
 70 |             y_true: cp.ndarray, Input target variables
 71 |             y_pred: cp.ndarray, predictions
 72 |             requires_grad: bool(), Whether or not tensor requires gradient for automatic differentiation
 73 |             
 74 |         Returns:
 75 |         predt, nll
 76 |         """
 77 |         
 78 |         ###
 79 |         # Initialize 
 80 |         ###
 81 |         n_target = n_param = self.D
 82 |         param_dict = self.create_param_dict(n_target)
 83 | 
 84 | 
 85 |         ###
 86 |         # Target
 87 |         ###    
 88 |         target = torch.as_tensor(y_true, device="cuda").reshape(-1, n_target)
 89 | 
 90 | 
 91 | 
 92 |         ###
 93 |         # Parameters
 94 |         ###   
 95 |         predt = [
 96 |             torch.tensor(
 97 |                 y_pred[:,i].reshape(-1,1), device="cuda", requires_grad=requires_grad
 98 |             ) for i in range(n_param)
 99 |         ]
100 | 
101 |         # Alpha
102 |         predt_alpha = torch.concat(
103 |             [response_fun(predt[i]) for i, (dist_param, response_fun) in enumerate(param_dict.items())],
104 |             axis=1
105 |         )
106 | 
107 |         
108 |         
109 |         ###
110 |         # NLL
111 |         ###
112 |         dist_fit = Dirichlet(predt_alpha)
113 |         nll = -torch.nansum(dist_fit.log_prob(target)) 
114 |         
115 |         return predt, nll
116 |     
117 |     
118 |     
119 |     
120 |     def predict(self, 
121 |                 model,  
122 |                 X_test: np.array, 
123 |                 pred_type: str = "parameters",                    
124 |                 n_samples: int = 100
125 |                ):
126 |         """
127 |         Predict function.
128 | 
129 |         model: 
130 |             Instance of pyboostlss
131 |         X_test: np.array
132 |             Test data features
133 |         pred_type: str
134 |             Specifies what is to be predicted:
135 |                 "samples": draws n_samples from the predicted response distribution. Output shape is (n_samples, n_obs, n_target)
136 |                 "parameters": returns the predicted distributional parameters.
137 |         n_samples: int
138 |             If pred_type="response" specifies how many samples are drawn from the predicted response distribution.
139 |         Returns
140 |         -------
141 |         pd.DataFrame with n_samples drawn from predicted response distribution.
142 | 
143 |         """
144 |          
145 |         n_target = self.D
146 |         param_dict = self.create_param_dict(n_target)
147 |         dist_params = list(param_dict.keys())
148 |         
149 |         # Predicted parameters
150 |         params_predt = torch.tensor(model.predict(X_test), device="cuda")
151 |         params_predt = torch.cat(
152 |             [response_fun(params_predt[:, i]).reshape(-1,1) for i, (dist_param, response_fun) in enumerate(param_dict.items())],
153 |             axis=1)
154 | 
155 |         # Predicted Distribution
156 |         dirichlet_pred = Dirichlet(params_predt)
157 | 
158 |         # Output DataFrame
159 |         predt_params = pd.DataFrame(params_predt.cpu().detach().numpy(),columns=dist_params)
160 |         
161 |         if pred_type == "parameters":
162 |             return predt_params        
163 |         
164 |         elif pred_type == "samples":
165 |             torch.manual_seed(123)
166 |             y_samples = dirichlet_pred.sample((n_samples,)).cpu().detach().numpy()
167 |             return y_samples
168 | 


--------------------------------------------------------------------------------
/pyboostlss/distributions/MVN.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch.distributions.multivariate_normal import MultivariateNormal
  3 | import cupy as cp 
  4 | import numpy as np
  5 | import pandas as pd
  6 | from pyboostlss.utils import * 
  7 | 
  8 | 
  9 | 
 10 | ########################################################################################################################
 11 | ###############################################      Multivariate Normal      ##########################################
 12 | ########################################################################################################################
 13 | 
 14 | class MVN: 
 15 |     """Multivariate Normal Distribution Class, where covariance matrix \Sigma is estimated via Cholesky-decomposition
 16 |     \Sigma = LL`.
 17 |     """
 18 |     
 19 |     def __init__(self,D:int):
 20 |         self.D = D # specifies target dimension
 21 |         
 22 | 
 23 |     def initialize(self, y_true: cp.ndarray, n_target: int) -> cp.ndarray:
 24 |         """ Function that calculates the starting values, for each distributional parameter individually.
 25 |         y_true: cp.ndarray
 26 |             Data from which starting values are calculated.
 27 |         n_target: ndarray
 28 |             Number of target variables
 29 |         """
 30 |         # Indices
 31 |         tril_indices = cp.asarray(np.tril_indices(n_target))
 32 | 
 33 |         # Target
 34 |         target = y_true[:,:n_target]
 35 | 
 36 |         # Location
 37 |         loc_init = cp.mean(target,axis=0) 
 38 | 
 39 |         # Tril
 40 |         tril_init = cp.cov(target,rowvar=False)
 41 |         tril_init = cp.linalg.cholesky(tril_init)
 42 |         cp.fill_diagonal(tril_init, cp.log(cp.diagonal(tril_init))) 
 43 |         tril_init = tril_init[tril_indices[0], tril_indices[1]]
 44 |         start_values = cp.concatenate([loc_init, tril_init])
 45 | 
 46 |         return start_values
 47 | 
 48 | 
 49 | 
 50 |     def n_dist_param(self, n_targets: int) -> int:
 51 |         """Infers the number of distributional parameters from target dimension.
 52 |         """
 53 |         n_param = int((n_targets*(n_targets + 3))/2)
 54 |         
 55 |         return n_param
 56 | 
 57 | 
 58 |     def target_append(self, y_true: np.ndarray, n_param: int) -> np.ndarray:
 59 |         """Function that appends target to the number of specified parameters
 60 |         """
 61 |         n_obs = y_true.shape[0]
 62 |         n_target = y_true.shape[1]
 63 |         n_fill = n_param - n_target
 64 |         np_fill = np.ones((n_obs, n_fill))
 65 |         y_append = np.concatenate([y_true, np_fill],axis=1)
 66 | 
 67 |         return y_append
 68 | 
 69 | 
 70 | 
 71 |     def tril_dim(self, n_target: int) -> int:
 72 |         """Infers the number of lower diagonal elements from number of targets.
 73 |         """    
 74 |         n_tril = int((n_target * (n_target + 1)) / 2)
 75 | 
 76 |         return n_tril
 77 |     
 78 |     
 79 |     def rho_dim(self, n_target: int) -> int:
 80 |         """Infers the number of correlations from number of targets.
 81 |         """           
 82 |         n_rho = int((n_target * (n_target - 1)) / 2)
 83 |         return n_rho
 84 |     
 85 |     
 86 | 
 87 |     def create_param_dict(self, n_target, tril_indices):
 88 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
 89 |         """
 90 | 
 91 |         n_theta = self.n_dist_param(n_target)    
 92 |         n_tril = self.tril_dim(n_target)
 93 | 
 94 | 
 95 |         # Location
 96 |         param_dict = {"location_" + str(i+1): identity_fn for i in range(n_target)}
 97 | 
 98 |         # Tril
 99 |         tril_idx = (tril_indices.detach().numpy()) + 1
100 |         tril_indices_row = tril_idx[0]
101 |         tril_indices_col = tril_idx[1]
102 |         tril_diag = tril_idx[0] == tril_idx[1]
103 | 
104 |         tril_dict = {}
105 | 
106 |         for i in range(n_tril):
107 |             if tril_diag[i] == True:
108 |                 tril_dict.update({"scale_" + str(tril_idx[:,i][1]): exp_fn}) 
109 |             else:
110 |                 tril_dict.update({"rho_" + str(tril_idx[:,i][0]) + str(tril_idx[:,i][1]): identity_fn})
111 | 
112 |         param_dict.update(tril_dict)
113 | 
114 |         return param_dict    
115 |     
116 |     
117 | 
118 |     def create_tril_dict(self, n_target, tril_indices):
119 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
120 |         """
121 | 
122 |         n_theta = self.n_dist_param(n_target)    
123 |         n_tril = self.tril_dim(n_target)
124 | 
125 |         # Tril
126 |         tril_idx = (tril_indices.detach().numpy()) + 1
127 |         tril_indices_row = tril_idx[0]
128 |         tril_indices_col = tril_idx[1]
129 |         tril_diag = tril_idx[0] == tril_idx[1]
130 | 
131 |         tril_dict = {}
132 | 
133 |         for i in range(n_tril):
134 |             if tril_diag[i] == True:
135 |                 tril_dict.update({"scale_" + str(tril_idx[:,i][1]): exp_fn})
136 |             else:
137 |                 tril_dict.update({"rho_" + str(tril_idx[:,i][0]) + str(tril_idx[:,i][1]): identity_fn})
138 | 
139 |         return tril_dict
140 |     
141 |     
142 | 
143 |     def get_params_nll(self, y_true: cp.ndarray, y_pred: cp.ndarray, requires_grad=False) -> torch.tensor:
144 |         """ Returns estimated parameters and nll.
145 |         
146 |         Args:
147 |             y_true: cp.ndarray, Input target variables
148 |             y_pred: cp.ndarray, predictions
149 |             requires_grad: bool(), Whether or not tensor requires gradient for automatic differentiation
150 |             
151 |         Returns:
152 |         predt, nll
153 |         """
154 |         
155 |         ###
156 |         # Initialize 
157 |         ###
158 |         n_obs = y_true.shape[0]
159 |         n_param = y_true.shape[1]
160 |         n_target = self.D                                       
161 |         n_tril = self.tril_dim(n_target)
162 |         tril_indices = torch.tril_indices(row=n_target, col=n_target, offset=0)       
163 |         param_dict = self.create_param_dict(n_target,tril_indices)                       
164 |         tril_param_dict = self.create_tril_dict(n_target,tril_indices)     
165 | 
166 | 
167 |         ###
168 |         # Target
169 |         ###    
170 |         target = torch.as_tensor(y_true[:,:n_target], device="cuda").reshape(-1, n_target)   
171 | 
172 | 
173 | 
174 |         ###
175 |         # Parameters
176 |         ###   
177 |         predt = [torch.tensor(y_pred[:,i].reshape(-1,1), device="cuda", requires_grad=requires_grad) for i in range(n_param)]
178 | 
179 |         # Location
180 |         predt_location = torch.concat(predt[:n_target],axis=1)    
181 | 
182 |         # Tril
183 |         tril_predt = predt[n_target:]
184 |         tril_predt = [response_fun(tril_predt[i]) for i, (dist_param, response_fun) in enumerate(tril_param_dict.items())] 
185 |         tril_predt = torch.concat(tril_predt,axis=1)    
186 |         predt_tril = torch.zeros(n_obs, n_target, n_target, dtype=tril_predt.dtype, device="cuda")
187 |         predt_tril[:, tril_indices[0], tril_indices[1]] = tril_predt
188 |         
189 |         
190 |         ###
191 |         # NLL
192 |         ###
193 |         dist_fit = MultivariateNormal(loc=predt_location, scale_tril=predt_tril)   
194 |         nll = -torch.nansum(dist_fit.log_prob(target)) 
195 |         
196 |         return predt, nll
197 |     
198 |     
199 |     
200 |     
201 |     def predict(self, 
202 |                 model,  
203 |                 X_test: np.array, 
204 |                 pred_type: str = "parameters",                    
205 |                 n_samples: int = 100
206 |                ):
207 |         """
208 |         Predict function.
209 | 
210 |         model: 
211 |             Instance of pyboostlss
212 |         X_test: np.array
213 |             Test data features
214 |         pred_type: str
215 |             Specifies what is to be predicted:
216 |                 "samples": draws n_samples from the predicted response distribution. Output shape is (n_samples, n_obs, n_target)
217 |                 "parameters": returns the predicted distributional parameters.
218 |         n_samples: int
219 |             If pred_type="response" specifies how many samples are drawn from the predicted response distribution.
220 |         Returns
221 |         -------
222 |         pd.DataFrame with n_samples drawn from predicted response distribution.
223 | 
224 |         """
225 |          
226 |         n_target = self.D    
227 |         n_tril = self.tril_dim(n_target)
228 |         n_rho = self.rho_dim(n_target)
229 |         tril_indices = torch.tril_indices(row=n_target, col=n_target, offset=0)       
230 |         param_dict = self.create_param_dict(n_target,tril_indices)  
231 |         dist_params = list(param_dict.keys())
232 |         
233 |         # Predicted parameters
234 |         params_predt = torch.tensor(model.predict(X_test), device="cuda")
235 |         params_predt = [response_fun(params_predt[:, i]).reshape(-1,1) for i, (dist_param, response_fun) in enumerate(param_dict.items())] 
236 | 
237 | 
238 |         # Location
239 |         predt_location = torch.cat(params_predt[:n_target],axis=1)
240 |         predt_location_df = pd.DataFrame(predt_location.cpu().detach().numpy())
241 |         predt_location_df.columns = [param for param in dist_params if "location_" in param]
242 | 
243 |         # Tril
244 |         n_obs = X_test.shape[0]
245 |         tril_predt = torch.cat(params_predt[n_target:],axis=1).reshape(-1, n_tril)
246 |         predt_tril = torch.zeros(n_obs, n_target, n_target, dtype=tril_predt.dtype, device="cuda")
247 |         predt_tril[:, tril_indices[0], tril_indices[1]] = tril_predt
248 | 
249 |         # Predicted Distribution
250 |         mvn_pred = MultivariateNormal(loc=predt_location, scale_tril=predt_tril) 
251 | 
252 |         # Sigma
253 |         predt_sigma = mvn_pred.stddev.cpu().detach().numpy()
254 |         predt_sigma_df = pd.DataFrame(predt_sigma)
255 |         predt_sigma_df.columns = [param for param in dist_params if "scale_" in param]
256 | 
257 |         # Rho
258 |         cov_mat = mvn_pred.covariance_matrix
259 |         predt_rho = torch.cat([calc_corr(cov_mat[i]).reshape(-1, n_rho) for i in range(n_obs)],axis=0)
260 |         predt_rho_df = pd.DataFrame(predt_rho.cpu().detach().numpy())
261 |         predt_rho_df.columns = [param for param in dist_params if "rho_" in param]
262 | 
263 |         # Output DataFrame
264 |         predt_params = pd.concat([predt_location_df, predt_sigma_df, predt_rho_df], axis=1)  
265 |         
266 |         if pred_type == "parameters":
267 |             return predt_params        
268 |         
269 |         elif pred_type == "samples":
270 |             torch.manual_seed(123)
271 |             y_samples = mvn_pred.sample((n_samples,)).cpu().detach().numpy()
272 |             return y_samples
273 | 


--------------------------------------------------------------------------------
/pyboostlss/distributions/MVN_LRA.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch.distributions.lowrank_multivariate_normal import LowRankMultivariateNormal
  3 | import torch.optim as optim
  4 | import cupy as cp 
  5 | import numpy as np
  6 | import pandas as pd
  7 | from pyboostlss.utils import * 
  8 | 
  9 | 
 10 | 
 11 | ########################################################################################################################
 12 | ###############################################      Multivariate Normal      ##########################################
 13 | ########################################################################################################################
 14 | 
 15 | class MVN_LRA: 
 16 |     """Multivariate Normal Distribution Class, where covariance matrix \Sigma is estimated via LRA apprixmation.
 17 |     """
 18 |     
 19 |     def __init__(self, 
 20 |                  r:int,
 21 |                  D:int):
 22 |         self.r = r # specifies rank
 23 |         self.D = D # specifies target dimension
 24 |         self.dtype = torch.float32
 25 |         
 26 |         
 27 |         
 28 |     def initialize(self, y_true: cp.ndarray, n_target: list) -> cp.ndarray:    
 29 |         """ Function that calculates the starting values, for each distributional parameter individually. It uses the L-BFGS algorithm for estimating unconditional parameter estimates.
 30 |     
 31 |         y_true: cp.ndarray
 32 |                 Data from which starting values are calculated.
 33 |         n_target: list
 34 |                 List that holds number of targets and rank-parameter.
 35 |         """
 36 | 
 37 |         torch.manual_seed(123)
 38 | 
 39 |         n_param = self.n_dist_param(n_target)
 40 |         n_target = self.D
 41 |         param_init = torch.ones(1, n_param, device="cuda", dtype=self.dtype)
 42 |         param_init = torch.nn.init.xavier_uniform_(param_init)    
 43 |         param_init.requires_grad=True
 44 |         y_true_tens = torch.tensor(y_true[:,:n_target], device="cuda", dtype=self.dtype)
 45 | 
 46 | 
 47 |         def nll_init(y_true: cp.ndarray, y_pred: cp.ndarray, requires_grad=True) -> torch.tensor:        
 48 | 
 49 |             n_target = self.D  
 50 |             n_param = self.n_dist_param([self.D, self.r])
 51 |             rank = self.r
 52 | 
 53 |             ###
 54 |             # Target
 55 |             ###    
 56 |             target = y_true[:,:n_target]
 57 | 
 58 |             ###
 59 |             # Parameters
 60 |             ###
 61 |             predt = [y_pred[:, i].reshape(-1,1) for i in range(n_param)]
 62 | 
 63 |             # Location
 64 |             predt_location = torch.concat(predt[:n_target],axis=1)
 65 | 
 66 |             # Low Rank Factor
 67 |             predt_covfactor = torch.concat(predt[n_target:(n_param-n_target)], axis=1).reshape(-1, n_target, rank) # (n_obs, n_target, rank)
 68 | 
 69 |             # Low Rank Diagonal (must be positive)
 70 |             predt_covdiag = predt[-n_target:]
 71 |             predt_covdiag = [exp_fn(predt_covdiag[i]) for i in range(len(predt_covdiag))]
 72 |             predt_covdiag = torch.concat(predt_covdiag, axis=1)
 73 | 
 74 |             ###
 75 |             # NLL
 76 |             ###
 77 |             dist_fit = LowRankMultivariateNormal(loc=predt_location, cov_factor=predt_covfactor, cov_diag=predt_covdiag, validate_args=False)   
 78 |             nll = -torch.nansum(dist_fit.log_prob(target)) 
 79 | 
 80 |             return nll
 81 | 
 82 |     
 83 |         def closure():
 84 |             
 85 |             lbfgs.zero_grad()
 86 |             objective = nll_init(y_true=y_true_tens, y_pred=param_init)
 87 |             objective.backward()
 88 | 
 89 |             return objective
 90 | 
 91 | 
 92 | 
 93 |         lbfgs = optim.LBFGS(params=[param_init],
 94 |                             lr=1e-03,
 95 |                             history_size=10, 
 96 |                             max_iter=4, 
 97 |                             line_search_fn="strong_wolfe")
 98 |         
 99 | 
100 |         for i in range(20):
101 |             lbfgs.step(closure)            
102 | 
103 |         start_values = cp.array(lbfgs.param_groups[0]["params"][0].cpu().detach()).reshape(-1,)
104 | 
105 |         return start_values   
106 |     
107 |     
108 |         
109 | 
110 | #     def initialize(self, y_true: cp.ndarray, n_target: list) -> cp.ndarray:
111 | #         """ Function that initializes each distributional parameter with ones. Compared to the LBFGS, this is more runtime efficient.
112 | #         y_true: cp.ndarray
113 | #             Data from which starting values are calculated.
114 | #         n_target: list
115 | #             List that holds number of targets and rank-parameter.
116 | #         """
117 | #         n_param = self.n_dist_param(n_target)
118 | #         start_values = cp.ones((n_param,))
119 | 
120 | #         return start_values
121 | 
122 | 
123 | 
124 | 
125 |     def n_dist_param(self, n_targets: list) -> int:
126 |         """Number of distributional parameters.
127 |         """
128 |         n_param = int(n_targets[0]*(2+n_targets[1]))
129 |         
130 |         return n_param
131 | 
132 | 
133 |     def target_append(self, y_true: np.ndarray, n_param: int) -> np.ndarray:
134 |         """Function that appends target to the number of specified parameters
135 |         """
136 |         n_obs = y_true.shape[0]
137 |         n_target = y_true.shape[1]
138 |         n_fill = n_param - n_target
139 |         np_fill = np.ones((n_obs, n_fill))
140 |         y_append = np.concatenate([y_true, np_fill],axis=1)
141 | 
142 |         return y_append
143 | 
144 | 
145 | 
146 |     def tril_dim(self, n_target: int) -> int:
147 |         """Infers the number of lower diagonal elements from number of targets.
148 |         """    
149 |         n_tril = int((n_target * (n_target + 1)) / 2)
150 | 
151 |         return n_tril
152 |     
153 |     
154 |     def rho_dim(self, n_target: int) -> int:
155 |         """Infers the number of correlations from number of targets.
156 |         """           
157 |         n_rho = int((n_target * (n_target - 1)) / 2)
158 |         return n_rho
159 | 
160 | 
161 |     def create_param_dict(self, n_target):
162 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
163 |         """
164 |         n_target = self.D
165 |         rank = self.r
166 | 
167 |         # Location
168 |         param_dict = {"location_" + str(i+1): identity_fn for i in range(n_target)}
169 | 
170 |         # Low Rank Factor
171 |         lrf_dict = {"lrf_" + str(i+1): identity_fn for i in range(n_target*rank)}
172 |         param_dict.update(lrf_dict)
173 | 
174 |         # Low Rank Diagonal
175 |         lrd_dict = {"lrd_" + str(i+1): exp_fn for i in range(n_target)}
176 |         param_dict.update(lrd_dict)
177 | 
178 |         return param_dict
179 | 
180 | 
181 |     def param_names(self, n_target):
182 |         """ List that holds the name of distributional parameter.
183 |         """
184 | 
185 |         n_tril = self.tril_dim(n_target)
186 | 
187 |         # Location
188 |         param_names = ["location_" + str(i+1) for i in range(n_target)]
189 | 
190 |         # Tril
191 |         tril_indices = torch.tril_indices(row=n_target, col=n_target, offset=0)
192 |         tril_idx = (tril_indices.detach().numpy()) + 1
193 |         tril_indices_row = tril_idx[0]
194 |         tril_indices_col = tril_idx[1]
195 |         tril_diag = tril_idx[0] == tril_idx[1]
196 | 
197 |         for i in range(n_tril):
198 |             if tril_diag[i] == True:
199 |                 param_names.append("scale_" + str(tril_idx[:,i][1]))
200 |             else:
201 |                 param_names.append("rho_" + str(tril_idx[:,i][0]) + str(tril_idx[:,i][1]))
202 | 
203 |         return param_names
204 | 
205 | 
206 | 
207 | 
208 |     def get_params_nll(self, y_true: cp.ndarray, y_pred: cp.ndarray, requires_grad=False) -> torch.tensor:
209 |         """ Returns estimated parameters and nll.
210 |         
211 |         Args:
212 |             y_true: cp.ndarray, Input target variables
213 |             y_pred: cp.ndarray, predictions
214 |             requires_grad: bool(), Whether or not tensor requires gradient for automatic differentiation
215 |             
216 |         Returns:
217 |         predt, nll
218 |         """
219 |         
220 |         ###
221 |         # Initialize 
222 |         ###
223 |         n_obs = y_true.shape[0]
224 |         n_param = y_true.shape[1]
225 |         n_target = self.D                                       
226 |         rank = self.r
227 |         param_dict = self.create_param_dict(n_target)
228 | 
229 | 
230 |         ###
231 |         # Target
232 |         ###    
233 |         target = torch.as_tensor(y_true[:,:n_target], device="cuda", dtype=self.dtype).reshape(-1, n_target)
234 | 
235 | 
236 |         ###
237 |         # Parameters
238 |         ###
239 |         predt = [torch.tensor(np.nan_to_num(y_pred[:, i], nan=float(np.nanmean(y_pred[:, i]))), device="cuda", requires_grad=requires_grad, dtype=self.dtype).reshape(-1,1) for i in range(n_param)]
240 | 
241 |         # Location
242 |         predt_location = torch.concat(predt[:n_target],axis=1)
243 | 
244 |         # Low Rank Factor
245 |         predt_covfactor = torch.concat(predt[n_target:(n_param-n_target)], axis=1).reshape(-1, n_target, rank) # (n_obs, n_target, rank)
246 | 
247 |         # Low Rank Diagonal (must be positive)
248 |         predt_covdiag = predt[-n_target:]
249 |         predt_covdiag = [exp_fn(predt_covdiag[i]) for i in range(len(predt_covdiag))]
250 |         predt_covdiag = torch.concat(predt_covdiag, axis=1)
251 | 
252 | 
253 |         ###
254 |         # NLL
255 |         ###
256 |         dist_fit = LowRankMultivariateNormal(loc=predt_location, cov_factor=predt_covfactor, cov_diag=predt_covdiag, validate_args=False)   
257 |         nll = -torch.nansum(dist_fit.log_prob(target)) 
258 |         
259 |         return predt, nll
260 |     
261 |     
262 |     
263 |     
264 |     def predict(self, 
265 |                 model,  
266 |                 X_test: np.array, 
267 |                 pred_type: str = "parameters",                    
268 |                 n_samples: int = 100
269 |                ):
270 |         """
271 |         Predict function.
272 | 
273 |         model: 
274 |             Instance of pyboostlss
275 |         X_test: np.array
276 |             Test data features
277 |         pred_type: str
278 |             Specifies what is to be predicted:
279 |                 "samples": draws n_samples from the predicted response distribution. Output shape is (n_samples, n_obs, n_target)
280 |                 "parameters": returns the predicted distributional parameters.
281 |         n_samples: int
282 |             If pred_type="response" specifies how many samples are drawn from the predicted response distribution.
283 |         Returns
284 |         -------
285 |         pd.DataFrame with n_samples drawn from predicted response distribution.
286 | 
287 |         """
288 | 
289 |         n_obs = X_test.shape[0]
290 |         n_target = self.D
291 |         rank = self.r
292 |         n_param = self.n_dist_param([n_target, rank])
293 |         n_rho = self.rho_dim(n_target)
294 |         param_dict = self.create_param_dict(n_target)
295 |         dist_params = self.param_names(n_target)
296 |         
297 |         # Predicted parameters
298 |         params_predt = torch.tensor(model.predict(X_test), device="cuda")
299 |         params_predt = [response_fun(params_predt[:, i]).reshape(-1,1) for i, (dist_param, response_fun) in enumerate(param_dict.items())]
300 | 
301 | 
302 |         # Location
303 |         predt_location = torch.cat(params_predt[:n_target],axis=1)
304 |         predt_location_df = pd.DataFrame(predt_location.cpu().detach().numpy())
305 |         predt_location_df.columns = [param for param in dist_params if "location_" in param]
306 | 
307 |         # Low Rank Factor
308 |         predt_covfactor = torch.cat(params_predt[n_target:(n_param-n_target)], axis=1).reshape(-1, n_target, rank) # (n_obs, n_target, rank)
309 |         
310 |         # Low Rank Diagonal
311 |         predt_covdiag = torch.cat(params_predt[-n_target:], axis=1)
312 | 
313 |         # Predicted Distribution
314 |         mvn_lra_pred = LowRankMultivariateNormal(loc=predt_location, cov_factor=predt_covfactor, cov_diag=predt_covdiag, validate_args=False)   
315 | 
316 |         # Sigma
317 |         predt_sigma = mvn_lra_pred.stddev.cpu().detach().numpy()
318 |         predt_sigma_df = pd.DataFrame(predt_sigma)
319 |         predt_sigma_df.columns = [param for param in dist_params if "scale_" in param]
320 | 
321 |         # Rho
322 |         cov_mat = mvn_lra_pred.covariance_matrix
323 |         predt_rho = torch.cat([calc_corr(cov_mat[i]).reshape(-1, n_rho) for i in range(n_obs)],axis=0)
324 |         predt_rho_df = pd.DataFrame(predt_rho.cpu().detach().numpy())
325 |         predt_rho_df.columns = [param for param in dist_params if "rho_" in param]
326 | 
327 |         # Output DataFrame
328 |         params_df = pd.concat([predt_location_df, predt_sigma_df, predt_rho_df], axis=1)
329 |         
330 |         if pred_type == "parameters":
331 |             return params_df
332 |         
333 |         elif pred_type == "samples":
334 |             torch.manual_seed(123)
335 |             y_samples = mvn_lra_pred.sample((n_samples,)).cpu().detach().numpy()
336 |             return y_samples
337 | 


--------------------------------------------------------------------------------
/pyboostlss/distributions/MVT.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from pyro.distributions import MultivariateStudentT
  3 | from scipy.stats import t
  4 | import cupy as cp 
  5 | import numpy as np
  6 | import pandas as pd
  7 | from pyboostlss.utils import *  
  8 | 
  9 | 
 10 | 
 11 | ########################################################################################################################
 12 | ###############################################      Multivariate Student-T   ##########################################
 13 | ########################################################################################################################
 14 | 
 15 | class MVT: 
 16 |     """Multivariate Student-T Distribution Class, where covariance matrix \Sigma is estimated via Cholesky-decomposition
 17 |     \Sigma = LL`.
 18 |     """
 19 |     
 20 |     def __init__(self,D:int):
 21 |         self.D = D # specifies target dimension
 22 |         
 23 | 
 24 |     def initialize(self, y_true: cp.ndarray, n_target: int) -> cp.ndarray:
 25 |         """ Function that calculates the starting values, for each distributional parameter individually.
 26 |         y_true: cp.ndarray
 27 |             Data from which starting values are calculated.
 28 |         n_target: ndarray
 29 |             Number of target variables
 30 |         """
 31 |         # Target
 32 |         target = y_true[:,:n_target]
 33 |         
 34 |         # Fitted Student-T Parameters 
 35 |         student_param = cp.array([t.fit(cp.asnumpy(target[:,i])) for i in range(n_target)])
 36 |         
 37 |         # Df 
 38 |         df_init = (cp.log(2) + cp.log(cp.median(student_param[:,0]))).reshape(-1,)
 39 |         
 40 |         # Location
 41 |         loc_init = cp.mean(target,axis=0) 
 42 | 
 43 |         # Tril
 44 |         tril_indices = cp.asarray(np.tril_indices(n_target))
 45 |         tril_init = cp.cov(target,rowvar=False)
 46 |         tril_init = cp.linalg.cholesky(tril_init)
 47 |         cp.fill_diagonal(tril_init, cp.log(cp.diagonal(tril_init))) 
 48 |         tril_init = tril_init[tril_indices[0], tril_indices[1]]
 49 |         
 50 |         start_values = cp.concatenate([df_init, loc_init, tril_init])
 51 | 
 52 |         return start_values
 53 | 
 54 | 
 55 | 
 56 |     def n_dist_param(self, n_targets: int) -> int:
 57 |         """Infers the number of distributional parameters from target dimension.
 58 |         """
 59 |         n_param = int(1 + ((n_targets*(n_targets + 3))/2))
 60 |         
 61 |         return n_param
 62 | 
 63 | 
 64 |     def target_append(self, y_true: np.ndarray, n_param: int) -> np.ndarray:
 65 |         """Function that appends target to the number of specified parameters
 66 |         """
 67 |         n_obs = y_true.shape[0]
 68 |         n_target = y_true.shape[1]
 69 |         n_fill = n_param - n_target
 70 |         np_fill = np.ones((n_obs, n_fill))
 71 |         y_append = np.concatenate([y_true, np_fill],axis=1)
 72 | 
 73 |         return y_append
 74 | 
 75 | 
 76 |     def tril_dim(self, n_target: int) -> int:
 77 |         """Infers the number of lower diagonal elements from number of targets.
 78 |         """    
 79 |         n_tril = int((n_target * (n_target + 1)) / 2)
 80 | 
 81 |         return n_tril
 82 |     
 83 |     
 84 |     def rho_dim(self, n_target: int) -> int:
 85 |         """Infers the number of correlations from number of targets.
 86 |         """           
 87 |         n_rho = int((n_target * (n_target - 1)) / 2)
 88 |         return n_rho
 89 |     
 90 |     
 91 | 
 92 |     def create_param_dict(self, n_target, tril_indices):
 93 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
 94 |         """
 95 | 
 96 |         n_theta = self.n_dist_param(n_target)    
 97 |         n_tril = self.tril_dim(n_target)
 98 |         
 99 |         # Df
100 |         param_dict = {"df": exp_fn_df}    
101 | 
102 |         # Location
103 |         loc_dict = {"location_" + str(i+1): identity_fn for i in range(n_target)}
104 |         
105 |         param_dict.update(loc_dict)
106 | 
107 |         # Tril
108 |         tril_idx = (tril_indices.detach().numpy()) + 1
109 |         tril_indices_row = tril_idx[0]
110 |         tril_indices_col = tril_idx[1]
111 |         tril_diag = tril_idx[0] == tril_idx[1]
112 | 
113 |         tril_dict = {}
114 | 
115 |         for i in range(n_tril):
116 |             if tril_diag[i] == True:
117 |                 tril_dict.update({"scale_" + str(tril_idx[:,i][1]): exp_fn}) 
118 |             else:
119 |                 tril_dict.update({"rho_" + str(tril_idx[:,i][0]) + str(tril_idx[:,i][1]): identity_fn})
120 | 
121 |         param_dict.update(tril_dict)
122 | 
123 |         return param_dict      
124 |     
125 | 
126 |     def create_tril_dict(self, n_target, tril_indices):
127 |         """ Dictionary that holds the name of distributional parameter and their corresponding response functions.
128 |         """
129 | 
130 |         n_theta = self.n_dist_param(n_target)    
131 |         n_tril = self.tril_dim(n_target)
132 | 
133 |         # Tril
134 |         tril_idx = (tril_indices.detach().numpy()) + 1
135 |         tril_indices_row = tril_idx[0]
136 |         tril_indices_col = tril_idx[1]
137 |         tril_diag = tril_idx[0] == tril_idx[1]
138 | 
139 |         tril_dict = {}
140 | 
141 |         for i in range(n_tril):
142 |             if tril_diag[i] == True:
143 |                 tril_dict.update({"scale_" + str(tril_idx[:,i][1]): exp_fn})
144 |             else:
145 |                 tril_dict.update({"rho_" + str(tril_idx[:,i][0]) + str(tril_idx[:,i][1]): identity_fn})
146 | 
147 |         return tril_dict   
148 |     
149 | 
150 |     def get_params_nll(self, y_true: cp.ndarray, y_pred: cp.ndarray, requires_grad=False) -> torch.tensor:
151 |         """ Returns estimated parameters and nll.
152 |         
153 |         Args:
154 |             y_true: cp.ndarray, Input target variables
155 |             y_pred: cp.ndarray, predictions
156 |             requires_grad: bool(), Whether or not tensor requires gradient for automatic differentiation
157 |             
158 |         Returns:
159 |         predt, nll
160 |         """
161 |         
162 |         ###
163 |         # Initialize 
164 |         ###
165 |         n_obs = y_true.shape[0]
166 |         n_param = y_true.shape[1]
167 |         n_target = self.D                                       
168 |         n_tril = self.tril_dim(n_target)
169 |         tril_indices = torch.tril_indices(row=n_target, col=n_target, offset=0)       
170 |         param_dict = self.create_param_dict(n_target,tril_indices)                       
171 |         tril_param_dict = self.create_tril_dict(n_target,tril_indices)     
172 | 
173 | 
174 |         ###
175 |         # Target
176 |         ###    
177 |         target = torch.as_tensor(y_true[:,:n_target], device="cuda").reshape(-1, n_target)   
178 | 
179 | 
180 | 
181 |         ###
182 |         # Parameters
183 |         ###   
184 |         predt = [torch.tensor(y_pred[:,i].reshape(-1,1), device="cuda", requires_grad=requires_grad) for i in range(n_param)]
185 |         
186 |         # Df
187 |         predt_df = exp_fn_df(predt[0]).reshape(-1,) 
188 | 
189 |         # Location
190 |         predt_location = torch.concat(predt[1:(n_target+1)],axis=1)     
191 | 
192 |         # Tril
193 |         tril_predt = predt[(n_target+1):]
194 |         tril_predt = [response_fun(tril_predt[i]) for i, (dist_param, response_fun) in enumerate(tril_param_dict.items())] 
195 |         tril_predt = torch.concat(tril_predt,axis=1)    
196 |         predt_tril = torch.zeros(n_obs, n_target, n_target, dtype=tril_predt.dtype, device="cuda")
197 |         predt_tril[:, tril_indices[0], tril_indices[1]] = tril_predt
198 |         
199 |         
200 |         ###
201 |         # NLL
202 |         ###
203 |         dist_fit = MultivariateStudentT(predt_df, predt_location, predt_tril)
204 |         nll = -torch.nansum(dist_fit.log_prob(target)) 
205 |         
206 |         return predt, nll
207 |       
208 |     
209 |     def predict(self, 
210 |                 model,  
211 |                 X_test: np.array, 
212 |                 pred_type: str = "parameters",                    
213 |                 n_samples: int = 100
214 |                ):
215 |         """
216 |         Predict function.
217 | 
218 |         model: 
219 |             Instance of pyboostlss
220 |         X_test: np.array
221 |             Test data features
222 |         pred_type: str
223 |             Specifies what is to be predicted:
224 |                 "samples": draws n_samples from the predicted response distribution. Output shape is (n_samples, n_obs, n_target)
225 |                 "parameters": returns the predicted distributional parameters.
226 |         n_samples: int
227 |             If pred_type="response" specifies how many samples are drawn from the predicted response distribution.
228 |         Returns
229 |         -------
230 |         pd.DataFrame with n_samples drawn from predicted response distribution.
231 | 
232 |         """
233 |          
234 |         n_target = self.D    
235 |         n_tril = self.tril_dim(n_target)
236 |         n_rho = self.rho_dim(n_target)
237 |         tril_indices = torch.tril_indices(row=n_target, col=n_target, offset=0)       
238 |         param_dict = self.create_param_dict(n_target,tril_indices)  
239 |         dist_params = list(param_dict.keys())
240 |         
241 |         # Predicted parameters
242 |         params_predt = torch.tensor(model.predict(X_test), device="cuda")
243 |         params_predt = [response_fun(params_predt[:, i]).reshape(-1,1) for i, (dist_param, response_fun) in enumerate(param_dict.items())]
244 |         
245 |         # Df
246 |         predt_df = params_predt[0].reshape(-1,)
247 |         predt_df_pd = pd.DataFrame(predt_df.cpu().detach().numpy())
248 |         predt_df_pd.columns = ["df"]        
249 | 
250 |         # Location
251 |         predt_location = torch.cat(params_predt[1:(n_target+1)],axis=1)
252 |         predt_location_df = pd.DataFrame(predt_location.cpu().detach().numpy())
253 |         predt_location_df.columns = [param for param in dist_params if "location_" in param]
254 | 
255 |         # Tril
256 |         n_obs = X_test.shape[0]
257 |         tril_predt = torch.cat(params_predt[(n_target+1):],axis=1).reshape(-1, n_tril)
258 |         predt_tril = torch.zeros(n_obs, n_target, n_target, dtype=tril_predt.dtype, device="cuda")
259 |         predt_tril[:, tril_indices[0], tril_indices[1]] = tril_predt                
260 | 
261 |         # Predicted Distribution
262 |         mvt_pred = MultivariateStudentT(predt_df, predt_location, predt_tril)
263 | 
264 |         # Sigma
265 |         predt_sigma = mvt_pred.stddev.cpu().detach().numpy()
266 |         predt_sigma_df = pd.DataFrame(predt_sigma)
267 |         predt_sigma_df.columns = [param for param in dist_params if "scale_" in param]
268 | 
269 |         # Rho
270 |         cov_mat = mvt_pred.covariance_matrix
271 |         predt_rho = torch.cat([calc_corr(cov_mat[i]).reshape(-1, n_rho) for i in range(n_obs)],axis=0)
272 |         predt_rho_df = pd.DataFrame(predt_rho.cpu().detach().numpy())
273 |         predt_rho_df.columns = [param for param in dist_params if "rho_" in param]   
274 |         
275 | 
276 |         # Output DataFrame
277 |         predt_params = pd.concat([predt_df_pd, predt_location_df, predt_sigma_df, predt_rho_df], axis=1)  
278 |         
279 |         if pred_type == "parameters":
280 |             return predt_params        
281 |         
282 |         elif pred_type == "samples":
283 |             torch.manual_seed(123)
284 |             y_samples = mvt_pred.sample((n_samples,)).cpu().detach().numpy()
285 |             return y_samples
286 |         
287 | 


--------------------------------------------------------------------------------
/pyboostlss/distributions/__init__.py:
--------------------------------------------------------------------------------
1 | """Py-BoostLSS - An extension of Py-Boost to probabilistic modelling"""
2 | 
3 | from pyboostlss.distributions.distribution_loss_metric import *
4 | from pyboostlss.distributions.MVN import *
5 | from pyboostlss.distributions.MVN_LRA import *
6 | from pyboostlss.distributions.MVT import *
7 | from pyboostlss.distributions.DIRICHLET import *
8 | 
9 | 


--------------------------------------------------------------------------------
/pyboostlss/distributions/distribution_loss_metric.py:
--------------------------------------------------------------------------------
  1 | import cupy as cp 
  2 | from pyboostlss.utils import *
  3 | from py_boost.gpu.losses import Loss, Metric
  4 | 
  5 | 
  6 | 
  7 | class Distribution_Metric(Metric):
  8 |     
  9 |     
 10 |     def __init__(self, dist):
 11 |         self.dist = dist
 12 |             
 13 |     
 14 |     alias = "NLL-score"
 15 |     
 16 | 
 17 |     def error(self, y_true, y_pred):
 18 |         """Error metric definition. 
 19 |         Args:
 20 |             y_true: cp.array, targets
 21 |             y_pred: cp.array, predictions
 22 |             sample_weight: None or cp.ndarray, weights
 23 |         Returns:
 24 |             float, metric value
 25 |         """  
 26 | 
 27 |         _, nll = self.dist.get_params_nll(y_true, y_pred)        
 28 |         nll = cp.asarray(nll)
 29 | 
 30 |         return nll
 31 | 
 32 | 
 33 |     def compare(self, v0 ,v1):
 34 |         """
 35 |         It should return True if v0 metric value is better than v1, False othewise
 36 |         """        
 37 |         return v0 < v1
 38 |     
 39 | 
 40 |     def __call__(self, y_true, y_pred, sample_weight=None):
 41 |         """Full metric definition. 
 42 |         Args:
 43 |             y_true: cp.array, targets
 44 |             y_pred: cp.array, predictions
 45 |             sample_weight: None or cp.ndarray, weights
 46 |         Returns:
 47 |             float, metric value
 48 |         """
 49 |         
 50 |         err = self.error(y_true, y_pred)
 51 |         
 52 |         return err
 53 |     
 54 |     
 55 |     
 56 |     
 57 | class Distribution_Loss(Loss):
 58 |     
 59 |     def __init__(self, dist):
 60 |         self.dist = dist
 61 |     
 62 |     def get_grad_hess(self, y_true, y_pred):
 63 |         """
 64 |         Defines how to calculate gradients and hessians for given loss.
 65 |         Args:
 66 |             y_true: cp.array, targets
 67 |             y_pred: cp.array, predictions
 68 |             sample_weight: None or cp.ndarray, weights
 69 |         Returns:
 70 |             floats, grad, hess
 71 |         """         
 72 |         
 73 |         ###
 74 |         # Parameters and NLL
 75 |         ###
 76 |         predt, nll = self.dist.get_params_nll(y_true, y_pred, requires_grad=True)
 77 |         
 78 |         
 79 |         ###
 80 |         # Derivatives
 81 |         ###   
 82 |         grad, hess = get_derivs(nll, predt)       
 83 |         
 84 |         return grad, hess
 85 |     
 86 |     
 87 |     
 88 |     def base_score(self, y_true):
 89 |         """
 90 |         Defines how parameter estimates are initialized. 
 91 |         Args:
 92 |             y_true: cp.array, targets
 93 |         Returns:
 94 |             floats, base_margins            
 95 |         """             
 96 | 
 97 |         if hasattr(self.dist, "r"):
 98 |             n_target = [self.dist.D, self.dist.r]
 99 |         else:
100 |             n_target = self.dist.D
101 |         base_margin = self.dist.initialize(y_true, n_target)
102 |                       
103 |         return base_margin


--------------------------------------------------------------------------------
/pyboostlss/model.py:
--------------------------------------------------------------------------------
  1 | from pyboostlss.distributions.distribution_loss_metric import *
  2 | from pyboostlss.utils import *
  3 | from py_boost import SketchBoost
  4 | 
  5 | import optuna
  6 | from optuna.samplers import TPESampler
  7 | 
  8 | class PyBoostLSS:
  9 |     """
 10 |     Py-BoostLSS model class. Currently only supports SketchBoost algorithm.
 11 |     
 12 |     """
 13 | 
 14 |     def __init__(self, dist):
 15 |         self.dist = dist # pyboostlss.distributions class. Specifies distribution
 16 | 
 17 | 
 18 |     def train(self,
 19 |               dtrain=None,
 20 |               eval_sets=None, 
 21 |               ntrees=100,
 22 |               lr=0.05,
 23 |               min_gain_to_split=0,
 24 |               lambda_l2=1,
 25 |               gd_steps=1,
 26 |               max_depth=6,
 27 |               min_data_in_leaf=10,
 28 |               colsample=1.,
 29 |               subsample=1.,
 30 | 
 31 |               quantization='Quantile',
 32 |               quant_sample=2000000,
 33 |               max_bin=256,
 34 |               min_data_in_bin=3,
 35 | 
 36 |               es=100,
 37 |               seed=123,
 38 |               verbose=10,
 39 | 
 40 |               sketch_outputs=1,
 41 |               sketch_method="proj",
 42 |               use_hess=True,
 43 | 
 44 |               callbacks=None,
 45 |               sketch_params=None):
 46 |         
 47 |         """Train a pyboostlss model with given parameters.
 48 | 
 49 |         Parameters
 50 |         ----------
 51 |         dtrain: dict, Dataset used for training of the form {'X': X_train, 'y': X_train}
 52 |         eval_sets: list used to evaluate model during training, e.g., [{'X': X_train, 'y': X_train}]
 53 |         ntrees: int, maximum number of trees
 54 |         lr: float, learning rate
 55 |         min_gain_to_split: float >=0, minimal gain to split
 56 |         lambda_l2: float > 0, l2 leaf regularization
 57 |         gd_steps: int > 0, number of gradient steps
 58 |         max_depth: int > 0, maximum tree depth. Setting it to large values (>12) may cause OOM for wide datasets
 59 |         min_data_in_leaf: int, minimal leaf size. Note - for some loss fn leaf size is approximated
 60 |             with hessian values to speed up training
 61 |         colsample: float or Callable, sumsample of columns to construct trees or callable - custom sampling
 62 |         subsample: float or Callable, sumsample of rows to construct trees or callable - custom sampling
 63 |         quantization: str or Quantizer, method for quantizatrion. One of 'Quantile', 'Uniform',
 64 |             'Uniquant' or custom implementation
 65 |         quant_sample: int, subsample to quantize features
 66 |         max_bin: int in [2, 256] maximum number of bins to quantize features
 67 |         min_data_in_bin: int in [2, 256] minimal bin size. NOTE: currently ignored
 68 |         es: int, early stopping rounds. If 0, no early stopping
 69 |         seed: int, random state
 70 |         verbose: int, verbosity freq
 71 |         sketch_outputs: int, number of outputs to keep
 72 |         sketch_method: str, name of the sketching strategy. Currently the following options are available: "topk", "rand", "proj". 
 73 |         use_hess: bool, use hessians in multioutput training
 74 |         callbacks: list of Callback, callbacks to customize training are passed here
 75 |         sketch_params: dict, optional kwargs for sketching strategy
 76 |         
 77 |         """
 78 |         
 79 |         bstLSS_init = SketchBoost(loss=Distribution_Loss(self.dist),
 80 |                                   metric=Distribution_Metric(self.dist),
 81 |                                   ntrees=ntrees,
 82 |                                   lr=lr,
 83 |                                   min_gain_to_split=min_gain_to_split,
 84 |                                   lambda_l2=lambda_l2,
 85 |                                   gd_steps=gd_steps,
 86 |                                   max_depth=max_depth,
 87 |                                   min_data_in_leaf=min_data_in_leaf,
 88 |                                   colsample=colsample,
 89 |                                   subsample=subsample,
 90 | 
 91 |                                   quantization=quantization,
 92 |                                   quant_sample=quant_sample,
 93 |                                   max_bin=max_bin,
 94 |                                   min_data_in_bin=min_data_in_bin,
 95 | 
 96 |                                   es=es,
 97 |                                   seed=seed,
 98 |                                   verbose=verbose,
 99 | 
100 |                                   sketch_outputs=sketch_outputs,
101 |                                   sketch_method=sketch_method,
102 |                                   use_hess=use_hess,
103 | 
104 |                                   callbacks=callbacks,
105 |                                   sketch_params=sketch_params
106 |                                   )
107 |         
108 |         
109 |         # Append Target
110 |         if hasattr(self.dist, "r"):
111 |             n_target = [self.dist.D, self.dist.r]
112 |         else:
113 |             n_target = self.dist.D            
114 |         
115 |         y_train_append = self.dist.target_append(dtrain["y"], self.dist.n_dist_param(n_target))
116 |         
117 |         if eval_sets is not None:
118 |             y_eval_append = self.dist.target_append(eval_sets[0]["y"] , self.dist.n_dist_param(n_target))
119 |             eval_sets_append = eval_sets.copy()
120 |             eval_sets_append[0]["y"] = y_eval_append
121 |             
122 |         else:
123 |             eval_sets_append = None       
124 |             
125 |         
126 |         bstLSS_train = bstLSS_init.fit(dtrain["X"], y_train_append, eval_sets=eval_sets_append)
127 |         
128 |         return bstLSS_train
129 |             
130 |                 
131 | 
132 | 
133 | 
134 | 
135 |     def hyper_opt(self,
136 |                   params=None,
137 |                   dtrain=None,
138 |                   eval_sets=None, 
139 |                   ntrees=100,
140 |                   lr=0.05,
141 |                   min_gain_to_split=0,
142 |                   lambda_l2=1,
143 |                   gd_steps=1,
144 |                   max_depth=6,
145 |                   min_data_in_leaf=10,
146 |                   colsample=1.,
147 |                   subsample=1.,
148 | 
149 |                   quantization='Quantile',
150 |                   quant_sample=2000000,
151 |                   max_bin=256,
152 |                   min_data_in_bin=3,
153 | 
154 |                   es=100,
155 |                   seed=123,
156 |                   hp_seed=None,
157 |                   verbose=int(1e04),
158 | 
159 |                   sketch_outputs=1,
160 |                   sketch_method="proj",
161 |                   use_hess=True,
162 | 
163 |                   callbacks=None,
164 |                   sketch_params=None,
165 |                   
166 |                   max_minutes=120,           
167 |                   n_trials=None,             
168 |                   study_name=None,
169 |                   silence=False
170 |                  ):
171 |         
172 |         """Function to tune hyper-parameters using Optuna.
173 |         
174 |         Parameters
175 |         ----------
176 |         params: dict, tunable hyper-parameters and their ranges
177 |         dtrain: dict, Dataset used for training of the form {'X': X_train, 'y': X_train}
178 |         eval_sets: list used to evaluate model during training, e.g., [{'X': X_train, 'y': X_train}]
179 |         ntrees: int, maximum number of trees
180 |         lr: float, learning rate
181 |         min_gain_to_split: float >=0, minimal gain to split
182 |         lambda_l2: float > 0, l2 leaf regularization
183 |         gd_steps: int > 0, number of gradient steps
184 |         max_depth: int > 0, maximum tree depth. Setting it to large values (>12) may cause OOM for wide datasets
185 |         min_data_in_leaf: int, minimal leaf size. Note - for some loss fn leaf size is approximated
186 |             with hessian values to speed up training
187 |         colsample: float or Callable, sumsample of columns to construct trees or callable - custom sampling
188 |         subsample: float or Callable, sumsample of rows to construct trees or callable - custom sampling
189 |         quantization: str or Quantizer, method for quantizatrion. One of 'Quantile', 'Uniform',
190 |             'Uniquant' or custom implementation
191 |         quant_sample: int, subsample to quantize features
192 |         max_bin: int in [2, 256] maximum number of bins to quantize features
193 |         min_data_in_bin: int in [2, 256] minimal bin size. NOTE: currently ignored
194 |         es: int, early stopping rounds. If 0, no early stopping
195 |         seed: int, random state
196 |         hp_seed: int, Random state for random number generator used in the Bayesian hyper-parameter search
197 |         verbose: int, verbosity freq
198 |         sketch_outputs: int, number of outputs to keep
199 |         sketch_method: str, name of the sketching strategy. Currently the following options are available: "topk", "rand", "proj". 
200 |         use_hess: bool, use hessians in multioutput training
201 |         callbacks: list of Callback, callbacks to customize training are passed here
202 |         sketch_params: dict, optional kwargs for sketching strategy
203 |         max_minutes: int, Time budget in minutes, i.e., stop study after the given number of minutes.
204 |         n_trials: int, The number of trials. If this argument is set to None, there is no limitation on the number of trials.
205 |         study_name : str, Name of the hyperparameter study.
206 |         silence: bool, Controls the verbosity of the trail, i.e., user can silence the outputs of the trail.
207 | 
208 |         Returns
209 |         -------
210 |         opt_params : Dict() with optimal parameters.
211 |         """
212 | 
213 |         def objective(trial):
214 |             
215 |             hyper_params = {
216 |                 "lr": trial.suggest_float("lr", params["lr"][0], params["lr"][1]),
217 |                 "max_depth": trial.suggest_int("max_depth", params["max_depth"][0], params["max_depth"][1]),
218 |                 "sketch_outputs": trial.suggest_int("sketch_outputs", params["sketch_outputs"][0], params["sketch_outputs"][1]),
219 |                 "lambda_l2": trial.suggest_float("lambda_l2", params["lambda_l2"][0], params["lambda_l2"][1]),
220 |                 "colsample": trial.suggest_float("colsample", params["colsample"][0], params["colsample"][1]),
221 |                 "subsample": trial.suggest_float("subsample", params["subsample"][0], params["subsample"][1]),
222 |                 "min_gain_to_split": trial.suggest_float("min_gain_to_split", params["min_gain_to_split"][0], params["min_gain_to_split"][1])
223 |             }   
224 |             
225 |             bstLSS_cv = self.train(dtrain=dtrain,
226 |                                    eval_sets=eval_sets,
227 |                                    ntrees=ntrees,
228 |                                    lr=hyper_params["lr"],
229 |                                    min_gain_to_split=hyper_params["min_gain_to_split"],
230 |                                    lambda_l2=hyper_params["lambda_l2"],
231 |                                    gd_steps=gd_steps,
232 |                                    max_depth=hyper_params["max_depth"],
233 |                                    min_data_in_leaf=min_data_in_leaf,
234 |                                    colsample=hyper_params["colsample"],
235 |                                    subsample=hyper_params["subsample"],
236 | 
237 |                                    quantization=quantization,
238 |                                    quant_sample=quant_sample,
239 |                                    max_bin=max_bin,
240 |                                    min_data_in_bin=min_data_in_bin,
241 | 
242 |                                    es=es,
243 |                                    seed=seed,
244 |                                    verbose=verbose,
245 | 
246 |                                    sketch_outputs=hyper_params["sketch_outputs"],
247 |                                    sketch_method=sketch_method,
248 |                                    use_hess=use_hess,
249 | 
250 |                                    callbacks=callbacks,
251 |                                    sketch_params=sketch_params
252 |                                    )
253 | 
254 | 
255 |             # Add optimal rounds
256 |             opt_rounds = bstLSS_cv.best_round
257 |             trial.set_user_attr("opt_round", int(opt_rounds))
258 | 
259 |             # Extract the best score
260 |             y_true = eval_sets[0]["y"]
261 |             y_pred = bstLSS_cv.predict(eval_sets[0]["X"])
262 |             _, nll = self.dist.get_params_nll(y_true, y_pred)
263 |             best_score = cp.asarray(nll)
264 | 
265 |             # Replace 0 value 
266 |             best_score = cp.where(best_score == -0.0, 1e08, best_score)
267 | 
268 |             return best_score
269 |         
270 |         
271 |         if silence:
272 |             optuna.logging.set_verbosity(optuna.logging.WARNING)
273 |             
274 |         if study_name is None:
275 |             study_name = "Py-BoostLSS Hyper-Parameter Optimization"
276 |     
277 |         if hp_seed is not None:
278 |             sampler = TPESampler(seed=hp_seed) 
279 |         else:
280 |             sampler = TPESampler()            
281 |         
282 |         pruner = optuna.pruners.MedianPruner(n_startup_trials=10, n_warmup_steps=20)
283 |         study = optuna.create_study(sampler=sampler, pruner=pruner, direction="minimize", study_name=study_name)
284 |         study.optimize(objective, n_trials=n_trials, timeout=60 * max_minutes, show_progress_bar=True)
285 | 
286 | 
287 |         print("\nHyper-Parameter Optimization successfully finished.")
288 |         print("  Number of finished trials: ", len(study.trials))
289 |         print("  Best trial:")
290 |         opt_param = study.best_trial
291 | 
292 |         # Add optimal stopping round
293 |         opt_param.params["opt_rounds"] = study.trials_dataframe()["user_attrs_opt_round"][
294 |             study.trials_dataframe()["value"].idxmin()]
295 |         opt_param.params["opt_rounds"] = int(opt_param.params["opt_rounds"])
296 | 
297 |         print("    Value: {}".format(opt_param.value))
298 |         print("    Params: ")
299 |         for key, value in opt_param.params.items():
300 |             print("    {}: {}".format(key, value))
301 | 
302 |         return opt_param.params
303 |         


--------------------------------------------------------------------------------
/pyboostlss/utils.py:
--------------------------------------------------------------------------------
 1 | import torch
 2 | from torch.autograd import grad as autograd
 3 | import cupy as cp 
 4 | import numpy as np
 5 | 
 6 | ###
 7 | # Response Functions
 8 | ###
 9 | 
10 | def identity_fn(predt: torch.tensor) -> torch.tensor:
11 |     """Identity mapping of predt.
12 |     """
13 |     return predt
14 | 
15 | 
16 | def exp_fn(predt: torch.tensor) -> torch.tensor:
17 |     """Exp() function used to ensure predt is strictly positive.
18 |     """
19 |     predt_adj = torch.exp(predt) 
20 |     predt_adj = torch.nan_to_num(predt_adj, nan=float(torch.nanmean(predt_adj))) + torch.tensor(1e-6, dtype=predt_adj.dtype, device="cuda")
21 | 
22 |     return predt_adj
23 | 
24 | 
25 | def exp_fn_df(predt: torch.tensor) -> torch.tensor:
26 |     """Exp() function for StudentT df-paramter used to ensure predt is strictly positive.
27 |     """
28 |     predt_adj = torch.exp(predt) + torch.tensor(2.0, device="cuda")
29 |     predt_adj = torch.nan_to_num(predt_adj, nan=float(torch.nanmean(predt_adj))) + torch.tensor(1e-6, dtype=predt_adj.dtype, device="cuda")
30 | 
31 |     return predt_adj
32 | 
33 | 
34 | 
35 | 
36 | ###
37 | # Autograd Function 
38 | ###
39 | def get_derivs(nll: torch.tensor, predt: torch.tensor) -> cp.ndarray:
40 |     """ Calculates gradients and hessians.
41 | 
42 |     Output gradients and hessians have shape (n_samples, n_outputs).
43 | 
44 |     Args:
45 |         nll: torch.tensor, calculated NLL
46 |         predt: torch.tensor, list of predicted paramters 
47 | 
48 |     Returns:
49 |     grad, hess
50 |     """
51 | 
52 |     # Gradient and Hessian
53 |     grad_list = autograd(nll, inputs=predt, create_graph=True)
54 |     hess_list = [autograd(grad_list[i].nansum(), inputs=predt[i], retain_graph=True)[0] for i in range(len(grad_list))]         
55 | 
56 |     # Reshape
57 |     grad = cp.asarray(torch.concat(grad_list,axis=1).detach())
58 |     hess = cp.asarray(torch.concat(hess_list,axis=1).detach())      
59 | 
60 |     return grad, hess
61 | 
62 | 
63 | 
64 | 
65 | ###
66 | # Misc
67 | ###
68 | 
69 | def response_dim(y_true: int) -> int:
70 |     """Infers the number of targets from input dataset.
71 |     """    
72 |     n_obs = y_true.shape[0]
73 |     col_sums = y_true.sum(axis=0)
74 |     n_target = col_sums != n_obs
75 |     n_target = len(n_target[n_target == True])
76 | 
77 |     return n_target
78 | 
79 | 
80 | def calc_corr(cov_mat: torch.tensor) -> torch.tensor:
81 |     """Calculates the lower correlation matrix from covariance matrix.
82 |     """   
83 |     diag = torch.sqrt(torch.diag(torch.diag(cov_mat)))
84 |     diag_inv = torch.linalg.inv(diag)
85 |     cor_mat = diag_inv @ cov_mat @ diag_inv 
86 |     cor_mat = cor_mat[np.tril_indices_from(cor_mat, k=-1)]
87 | 
88 |     return cor_mat
89 | 
90 | 
91 | 
92 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from setuptools import setup, find_packages
 2 | 
 3 | 
 4 | setup(
 5 |     name="pyboostlss",
 6 |     version="0.1.0",
 7 |     description="Py-BoostLSS: An extension of Py-Boost to probabilistic modelling",
 8 |     long_description=open("README.md").read(),
 9 |     long_description_content_type="text/markdown",
10 |     author="Alexander März",
11 |     author_email="alex.maerz@gmx.net",
12 |     url="https://github.com/StatMixedML/Py-BoostLSS",
13 |     license="Apache License 2.0",
14 |     packages=find_packages(exclude=["tests"]),
15 |     include_package_data=True,
16 |     package_data={'': ['datasets/*.csv']},
17 |     zip_safe=True,
18 |     python_requires=">=3.8, <3.10",
19 |     install_requires=[
20 |         "py-boost~=0.3.0",
21 |         "optuna~=3.0.3",
22 |         "pyro-ppl~=1.8.3",
23 |         "dirichlet~=0.9",
24 |         "scikit-learn~=1.1.3",
25 |         "numpy~=1.23.5",
26 |         "pandas~=1.5.2",
27 |         "plotnine~=0.10.1",
28 |         "scipy~=1.8.1",
29 |         "tqdm~=4.64.1",
30 |         "matplotlib~=3.6.2",
31 |         "ipywidgets~=8.0.2",
32 |     ],
33 |     test_suite="tests",
34 |     tests_require=["flake8", "pytest"],
35 | )


--------------------------------------------------------------------------------