├── .gitignore ├── LICENSE.md ├── README.md ├── images └── densetree.png ├── lib ├── __init__.py ├── arch.py ├── data.py ├── nn_utils.py ├── odst.py ├── trainer.py └── utils.py ├── notebooks ├── epsilon_node_multigpu.ipynb ├── year_node_8layers.ipynb └── year_node_shallow.ipynb └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # node and NPM 2 | npm-debug.log 3 | node_modules 4 | 5 | # swap files 6 | *~ 7 | *.swp 8 | 9 | notebooks/data/* 10 | notebooks/runs/* 11 | notebooks/.ipynb_checkpoints/* 12 | 13 | env.sh 14 | # Byte-compiled / optimized / DLL files 15 | __pycache__/ 16 | *.py[cod] 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Distribution / packaging 22 | .Python 23 | env/ 24 | bin/ 25 | build/ 26 | develop-eggs/ 27 | dist/ 28 | eggs/ 29 | lib64/ 30 | parts/ 31 | sdist/ 32 | var/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg/ 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | 49 | # Translations 50 | *.mo 51 | 52 | # Mr Developer 53 | .mr.developer.cfg 54 | .project 55 | .pydevproject 56 | .idea 57 | .ipynb_checkpoints 58 | 59 | # Rope 60 | .ropeproject 61 | 62 | # Django stuff: 63 | *.log 64 | *.pot 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | docs/tmp* 69 | 70 | # OS X garbage 71 | .DS_Store 72 | 73 | # Debian things 74 | debian/reproducible-experiment-platform 75 | debian/files 76 | *.substvars 77 | *.debhelper.log 78 | 79 | .vscode -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Sergey Popov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Neural Oblivious Decision Ensembles 2 | A supplementary code for [Neural Oblivious Decision Ensembles for Deep Learning on Tabular Data](https://arxiv.org/abs/1909.06312) paper. 3 | 4 | 5 | 6 | # What does it do? 7 | It learns deep ensembles of oblivious differentiable decision trees on tabular data 8 | 9 | # What do i need to run it? 10 | * A machine with some CPU (preferably 2+ free cores) and GPU(s) 11 | * Running without GPU is possible but takes 8-10x as long even on high-end CPUs 12 | * Our implementation is memory inefficient and may require a lot of GPU memory to converge 13 | * Some popular Linux x64 distribution 14 | * Tested on Ubuntu16.04, should work fine on any popular linux64 and even MacOS; 15 | * Windows and x32 systems may require heavy wizardry to run; 16 | * When in doubt, use Docker, preferably GPU-enabled (i.e. nvidia-docker) 17 | 18 | # How do I run it? 19 | 1. Clone or download this repo. `cd` yourself to it's root directory. 20 | 2. Grab or build a working python enviromnent. [Anaconda](https://www.anaconda.com/) works fine. 21 | 3. Install packages from `requirements.txt` 22 | * It is critical that you use __torch >= 1.1__, not 1.0 or earlier 23 | * You will also need jupyter or some other way to work with .ipynb files 24 | 4. Run jupyter notebook and open a notebook in `./notebooks/` 25 | * Before you run the first cell, change `%env CUDA_VISIBLE_DEVICES=#` to an index that you plan to use. 26 | * The notebook downloads data from dropbox. You will need __1-5Gb__ of disk space depending on dataset. 27 | 28 | We showcase two typical learning scenarios for classification and regression. Please consult the original paper for training details. 29 | -------------------------------------------------------------------------------- /images/densetree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qwicen/node/3bae6a8a63f0205683270b6d566d9cfa659403e4/images/densetree.png -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | from .nn_utils import * 2 | from .odst import * 3 | from .utils import * 4 | from .data import * 5 | from .arch import * 6 | from .trainer import * 7 | -------------------------------------------------------------------------------- /lib/arch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from .odst import ODST 5 | 6 | 7 | class DenseBlock(nn.Sequential): 8 | def __init__(self, input_dim, layer_dim, num_layers, tree_dim=1, max_features=None, 9 | input_dropout=0.0, flatten_output=True, Module=ODST, **kwargs): 10 | layers = [] 11 | for i in range(num_layers): 12 | oddt = Module(input_dim, layer_dim, tree_dim=tree_dim, flatten_output=True, **kwargs) 13 | input_dim = min(input_dim + layer_dim * tree_dim, max_features or float('inf')) 14 | layers.append(oddt) 15 | 16 | super().__init__(*layers) 17 | self.num_layers, self.layer_dim, self.tree_dim = num_layers, layer_dim, tree_dim 18 | self.max_features, self.flatten_output = max_features, flatten_output 19 | self.input_dropout = input_dropout 20 | 21 | def forward(self, x): 22 | initial_features = x.shape[-1] 23 | for layer in self: 24 | layer_inp = x 25 | if self.max_features is not None: 26 | tail_features = min(self.max_features, layer_inp.shape[-1]) - initial_features 27 | if tail_features != 0: 28 | layer_inp = torch.cat([layer_inp[..., :initial_features], layer_inp[..., -tail_features:]], dim=-1) 29 | if self.training and self.input_dropout: 30 | layer_inp = F.dropout(layer_inp, self.input_dropout) 31 | h = layer(layer_inp) 32 | x = torch.cat([x, h], dim=-1) 33 | 34 | outputs = x[..., initial_features:] 35 | if not self.flatten_output: 36 | outputs = outputs.view(*outputs.shape[:-1], self.num_layers * self.layer_dim, self.tree_dim) 37 | return outputs 38 | -------------------------------------------------------------------------------- /lib/data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import bz2 3 | import numpy as np 4 | import pandas as pd 5 | import gzip 6 | import shutil 7 | import torch 8 | import random 9 | import warnings 10 | 11 | from sklearn.model_selection import train_test_split 12 | 13 | from .utils import download 14 | from sklearn.datasets import load_svmlight_file 15 | from sklearn.preprocessing import QuantileTransformer 16 | from category_encoders import LeaveOneOutEncoder 17 | 18 | 19 | class Dataset: 20 | 21 | def __init__(self, dataset, random_state, data_path='./data', normalize=False, 22 | quantile_transform=False, output_distribution='normal', quantile_noise=0, **kwargs): 23 | """ 24 | Dataset is a dataclass that contains all training and evaluation data required for an experiment 25 | :param dataset: a pre-defined dataset name (see DATSETS) or a custom dataset 26 | Your dataset should be at (or will be downloaded into) {data_path}/{dataset} 27 | :param random_state: global random seed for an experiment 28 | :param data_path: a shared data folder path where the dataset is stored (or will be downloaded into) 29 | :param normalize: standardize features by removing the mean and scaling to unit variance 30 | :param quantile_transform: transforms the features to follow a normal distribution. 31 | :param output_distribution: if quantile_transform == True, data is projected onto this distribution 32 | See the same param of sklearn QuantileTransformer 33 | :param quantile_noise: if specified, fits QuantileTransformer on data with added gaussian noise 34 | with std = :quantile_noise: * data.std ; this will cause discrete values to be more separable 35 | Please not that this transformation does NOT apply gaussian noise to the resulting data, 36 | the noise is only applied for QuantileTransformer 37 | :param kwargs: depending on the dataset, you may select train size, test size or other params 38 | If dataset is not in DATASETS, provide six keys: X_train, y_train, X_valid, y_valid, X_test and y_test 39 | """ 40 | np.random.seed(random_state) 41 | torch.manual_seed(random_state) 42 | random.seed(random_state) 43 | 44 | if dataset in DATASETS: 45 | data_dict = DATASETS[dataset](os.path.join(data_path, dataset), **kwargs) 46 | else: 47 | assert all(key in kwargs for key in ('X_train', 'y_train', 'X_valid', 'y_valid', 'X_test', 'y_test')), \ 48 | "Unknown dataset. Provide X_train, y_train, X_valid, y_valid, X_test and y_test params" 49 | data_dict = kwargs 50 | 51 | self.data_path = data_path 52 | self.dataset = dataset 53 | 54 | self.X_train = data_dict['X_train'] 55 | self.y_train = data_dict['y_train'] 56 | self.X_valid = data_dict['X_valid'] 57 | self.y_valid = data_dict['y_valid'] 58 | self.X_test = data_dict['X_test'] 59 | self.y_test = data_dict['y_test'] 60 | 61 | if all(query in data_dict.keys() for query in ('query_train', 'query_valid', 'query_test')): 62 | self.query_train = data_dict['query_train'] 63 | self.query_valid = data_dict['query_valid'] 64 | self.query_test = data_dict['query_test'] 65 | 66 | if normalize: 67 | mean = np.mean(self.X_train, axis=0) 68 | std = np.std(self.X_train, axis=0) 69 | self.X_train = (self.X_train - mean) / std 70 | self.X_valid = (self.X_valid - mean) / std 71 | self.X_test = (self.X_test - mean) / std 72 | 73 | if quantile_transform: 74 | quantile_train = np.copy(self.X_train) 75 | if quantile_noise: 76 | stds = np.std(quantile_train, axis=0, keepdims=True) 77 | noise_std = quantile_noise / np.maximum(stds, quantile_noise) 78 | quantile_train += noise_std * np.random.randn(*quantile_train.shape) 79 | 80 | qt = QuantileTransformer(random_state=random_state, output_distribution=output_distribution).fit(quantile_train) 81 | self.X_train = qt.transform(self.X_train) 82 | self.X_valid = qt.transform(self.X_valid) 83 | self.X_test = qt.transform(self.X_test) 84 | 85 | def to_csv(self, path=None): 86 | if path == None: 87 | path = os.path.join(self.data_path, self.dataset) 88 | 89 | np.savetxt(os.path.join(path, 'X_train.csv'), self.X_train, delimiter=',') 90 | np.savetxt(os.path.join(path, 'X_valid.csv'), self.X_valid, delimiter=',') 91 | np.savetxt(os.path.join(path, 'X_test.csv'), self.X_test, delimiter=',') 92 | np.savetxt(os.path.join(path, 'y_train.csv'), self.y_train, delimiter=',') 93 | np.savetxt(os.path.join(path, 'y_valid.csv'), self.y_valid, delimiter=',') 94 | np.savetxt(os.path.join(path, 'y_test.csv'), self.y_test, delimiter=',') 95 | 96 | 97 | def fetch_A9A(path, train_size=None, valid_size=None, test_size=None): 98 | train_path = os.path.join(path, 'a9a') 99 | test_path = os.path.join(path, 'a9a.t') 100 | if not all(os.path.exists(fname) for fname in (train_path, test_path)): 101 | os.makedirs(path, exist_ok=True) 102 | download("https://www.dropbox.com/s/9cqdx166iwonrj9/a9a?dl=1", train_path) 103 | download("https://www.dropbox.com/s/sa0ds895c0v4xc6/a9a.t?dl=1", test_path) 104 | 105 | X_train, y_train = load_svmlight_file(train_path, dtype=np.float32, n_features=123) 106 | X_test, y_test = load_svmlight_file(test_path, dtype=np.float32, n_features=123) 107 | X_train, X_test = X_train.toarray(), X_test.toarray() 108 | y_train[y_train == -1] = 0 109 | y_test[y_test == -1] = 0 110 | y_train, y_test = y_train.astype(np.int), y_test.astype(np.int) 111 | 112 | if all(sizes is None for sizes in (train_size, valid_size, test_size)): 113 | train_idx_path = os.path.join(path, 'stratified_train_idx.txt') 114 | valid_idx_path = os.path.join(path, 'stratified_valid_idx.txt') 115 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 116 | download("https://www.dropbox.com/s/xy4wwvutwikmtha/stratified_train_idx.txt?dl=1", train_idx_path) 117 | download("https://www.dropbox.com/s/nthpxofymrais5s/stratified_test_idx.txt?dl=1", valid_idx_path) 118 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 119 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 120 | else: 121 | assert train_size, "please provide either train_size or none of sizes" 122 | if valid_size is None: 123 | valid_size = len(X_train) - train_size 124 | assert valid_size > 0 125 | if train_size + valid_size > len(X_train): 126 | warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format( 127 | train_size + valid_size, len(X_train)), Warning) 128 | if test_size is not None: 129 | warnings.warn('Test set is fixed for this dataset.', Warning) 130 | 131 | shuffled_indices = np.random.permutation(np.arange(len(X_train))) 132 | train_idx = shuffled_indices[:train_size] 133 | valid_idx = shuffled_indices[train_size: train_size + valid_size] 134 | 135 | return dict( 136 | X_train=X_train[train_idx], y_train=y_train[train_idx], 137 | X_valid=X_train[valid_idx], y_valid=y_train[valid_idx], 138 | X_test=X_test, y_test=y_test 139 | ) 140 | 141 | 142 | def fetch_EPSILON(path, train_size=None, valid_size=None, test_size=None): 143 | train_path = os.path.join(path, 'epsilon_normalized') 144 | test_path = os.path.join(path, 'epsilon_normalized.t') 145 | if not all(os.path.exists(fname) for fname in (train_path, test_path)): 146 | os.makedirs(path, exist_ok=True) 147 | train_archive_path = os.path.join(path, 'epsilon_normalized.bz2') 148 | test_archive_path = os.path.join(path, 'epsilon_normalized.t.bz2') 149 | if not all(os.path.exists(fname) for fname in (train_archive_path, test_archive_path)): 150 | download("https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.bz2", train_archive_path) 151 | download("https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.t.bz2", test_archive_path) 152 | print("unpacking dataset") 153 | for file_name, archive_name in zip((train_path, test_path), (train_archive_path, test_archive_path)): 154 | zipfile = bz2.BZ2File(archive_name) 155 | with open(file_name, 'wb') as f: 156 | f.write(zipfile.read()) 157 | 158 | print("reading dataset (it may take a long time)") 159 | X_train, y_train = load_svmlight_file(train_path, dtype=np.float32, n_features=2000) 160 | X_test, y_test = load_svmlight_file(test_path, dtype=np.float32, n_features=2000) 161 | X_train, X_test = X_train.toarray(), X_test.toarray() 162 | y_train, y_test = y_train.astype(np.int), y_test.astype(np.int) 163 | y_train[y_train == -1] = 0 164 | y_test[y_test == -1] = 0 165 | 166 | if all(sizes is None for sizes in (train_size, valid_size, test_size)): 167 | train_idx_path = os.path.join(path, 'stratified_train_idx.txt') 168 | valid_idx_path = os.path.join(path, 'stratified_valid_idx.txt') 169 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 170 | download("https://www.dropbox.com/s/wxgm94gvm6d3xn5/stratified_train_idx.txt?dl=1", train_idx_path) 171 | download("https://www.dropbox.com/s/fm4llo5uucdglti/stratified_valid_idx.txt?dl=1", valid_idx_path) 172 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 173 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 174 | else: 175 | assert train_size, "please provide either train_size or none of sizes" 176 | if valid_size is None: 177 | valid_size = len(X_train) - train_size 178 | assert valid_size > 0 179 | if train_size + valid_size > len(X_train): 180 | warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format( 181 | train_size + valid_size, len(X_train)), Warning) 182 | if test_size is not None: 183 | warnings.warn('Test set is fixed for this dataset.', Warning) 184 | 185 | shuffled_indices = np.random.permutation(np.arange(len(X_train))) 186 | train_idx = shuffled_indices[:train_size] 187 | valid_idx = shuffled_indices[train_size: train_size + valid_size] 188 | 189 | return dict( 190 | X_train=X_train[train_idx], y_train=y_train[train_idx], 191 | X_valid=X_train[valid_idx], y_valid=y_train[valid_idx], 192 | X_test=X_test, y_test=y_test 193 | ) 194 | 195 | 196 | def fetch_PROTEIN(path, train_size=None, valid_size=None, test_size=None): 197 | """ 198 | https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#protein 199 | """ 200 | train_path = os.path.join(path, 'protein') 201 | test_path = os.path.join(path, 'protein.t') 202 | if not all(os.path.exists(fname) for fname in (train_path, test_path)): 203 | os.makedirs(path, exist_ok=True) 204 | download("https://www.dropbox.com/s/pflp4vftdj3qzbj/protein.tr?dl=1", train_path) 205 | download("https://www.dropbox.com/s/z7i5n0xdcw57weh/protein.t?dl=1", test_path) 206 | for fname in (train_path, test_path): 207 | raw = open(fname).read().replace(' .', '0.') 208 | with open(fname, 'w') as f: 209 | f.write(raw) 210 | 211 | X_train, y_train = load_svmlight_file(train_path, dtype=np.float32, n_features=357) 212 | X_test, y_test = load_svmlight_file(test_path, dtype=np.float32, n_features=357) 213 | X_train, X_test = X_train.toarray(), X_test.toarray() 214 | y_train, y_test = y_train.astype(np.int), y_test.astype(np.int) 215 | 216 | if all(sizes is None for sizes in (train_size, valid_size, test_size)): 217 | train_idx_path = os.path.join(path, 'stratified_train_idx.txt') 218 | valid_idx_path = os.path.join(path, 'stratified_valid_idx.txt') 219 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 220 | download("https://www.dropbox.com/s/wq2v9hl1wxfufs3/small_stratified_train_idx.txt?dl=1", train_idx_path) 221 | download("https://www.dropbox.com/s/7o9el8pp1bvyy22/small_stratified_valid_idx.txt?dl=1", valid_idx_path) 222 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 223 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 224 | else: 225 | assert train_size, "please provide either train_size or none of sizes" 226 | if valid_size is None: 227 | valid_size = len(X_train) - train_size 228 | assert valid_size > 0 229 | if train_size + valid_size > len(X_train): 230 | warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format( 231 | train_size + valid_size, len(X_train)), Warning) 232 | if test_size is not None: 233 | warnings.warn('Test set is fixed for this dataset.', Warning) 234 | 235 | shuffled_indices = np.random.permutation(np.arange(len(X_train))) 236 | train_idx = shuffled_indices[:train_size] 237 | valid_idx = shuffled_indices[train_size: train_size + valid_size] 238 | 239 | return dict( 240 | X_train=X_train[train_idx], y_train=y_train[train_idx], 241 | X_valid=X_train[valid_idx], y_valid=y_train[valid_idx], 242 | X_test=X_test, y_test=y_test 243 | ) 244 | 245 | 246 | def fetch_YEAR(path, train_size=None, valid_size=None, test_size=51630): 247 | data_path = os.path.join(path, 'data.csv') 248 | if not os.path.exists(data_path): 249 | os.makedirs(path, exist_ok=True) 250 | download('https://www.dropbox.com/s/l09pug0ywaqsy0e/YearPredictionMSD.txt?dl=1', data_path) 251 | n_features = 91 252 | types = {i: (np.float32 if i != 0 else np.int) for i in range(n_features)} 253 | data = pd.read_csv(data_path, header=None, dtype=types) 254 | data_train, data_test = data.iloc[:-test_size], data.iloc[-test_size:] 255 | 256 | X_train, y_train = data_train.iloc[:, 1:].values, data_train.iloc[:, 0].values 257 | X_test, y_test = data_test.iloc[:, 1:].values, data_test.iloc[:, 0].values 258 | 259 | if all(sizes is None for sizes in (train_size, valid_size)): 260 | train_idx_path = os.path.join(path, 'stratified_train_idx.txt') 261 | valid_idx_path = os.path.join(path, 'stratified_valid_idx.txt') 262 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 263 | download("https://www.dropbox.com/s/00u6cnj9mthvzj1/stratified_train_idx.txt?dl=1", train_idx_path) 264 | download("https://www.dropbox.com/s/420uhjvjab1bt7k/stratified_valid_idx.txt?dl=1", valid_idx_path) 265 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 266 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 267 | else: 268 | assert train_size, "please provide either train_size or none of sizes" 269 | if valid_size is None: 270 | valid_size = len(X_train) - train_size 271 | assert valid_size > 0 272 | if train_size + valid_size > len(X_train): 273 | warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format( 274 | train_size + valid_size, len(X_train)), Warning) 275 | 276 | shuffled_indices = np.random.permutation(np.arange(len(X_train))) 277 | train_idx = shuffled_indices[:train_size] 278 | valid_idx = shuffled_indices[train_size: train_size + valid_size] 279 | 280 | return dict( 281 | X_train=X_train[train_idx], y_train=y_train[train_idx], 282 | X_valid=X_train[valid_idx], y_valid=y_train[valid_idx], 283 | X_test=X_test, y_test=y_test, 284 | ) 285 | 286 | 287 | def fetch_HIGGS(path, train_size=None, valid_size=None, test_size=5 * 10 ** 5): 288 | data_path = os.path.join(path, 'higgs.csv') 289 | if not os.path.exists(data_path): 290 | os.makedirs(path, exist_ok=True) 291 | archive_path = os.path.join(path, 'HIGGS.csv.gz') 292 | download('https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz', archive_path) 293 | with gzip.open(archive_path, 'rb') as f_in: 294 | with open(data_path, 'wb') as f_out: 295 | shutil.copyfileobj(f_in, f_out) 296 | n_features = 29 297 | types = {i: (np.float32 if i != 0 else np.int) for i in range(n_features)} 298 | data = pd.read_csv(data_path, header=None, dtype=types) 299 | data_train, data_test = data.iloc[:-test_size], data.iloc[-test_size:] 300 | 301 | X_train, y_train = data_train.iloc[:, 1:].values, data_train.iloc[:, 0].values 302 | X_test, y_test = data_test.iloc[:, 1:].values, data_test.iloc[:, 0].values 303 | 304 | if all(sizes is None for sizes in (train_size, valid_size)): 305 | train_idx_path = os.path.join(path, 'stratified_train_idx.txt') 306 | valid_idx_path = os.path.join(path, 'stratified_valid_idx.txt') 307 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 308 | download("https://www.dropbox.com/s/i2uekmwqnp9r4ix/stratified_train_idx.txt?dl=1", train_idx_path) 309 | download("https://www.dropbox.com/s/wkbk74orytmb2su/stratified_valid_idx.txt?dl=1", valid_idx_path) 310 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 311 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 312 | else: 313 | assert train_size, "please provide either train_size or none of sizes" 314 | if valid_size is None: 315 | valid_size = len(X_train) - train_size 316 | assert valid_size > 0 317 | if train_size + valid_size > len(X_train): 318 | warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format( 319 | train_size + valid_size, len(X_train)), Warning) 320 | 321 | shuffled_indices = np.random.permutation(np.arange(len(X_train))) 322 | train_idx = shuffled_indices[:train_size] 323 | valid_idx = shuffled_indices[train_size: train_size + valid_size] 324 | 325 | return dict( 326 | X_train=X_train[train_idx], y_train=y_train[train_idx], 327 | X_valid=X_train[valid_idx], y_valid=y_train[valid_idx], 328 | X_test=X_test, y_test=y_test, 329 | ) 330 | 331 | 332 | def fetch_MICROSOFT(path): 333 | train_path = os.path.join(path, 'msrank_train.tsv') 334 | test_path = os.path.join(path, 'msrank_test.tsv') 335 | if not all(os.path.exists(fname) for fname in (train_path, test_path)): 336 | os.makedirs(path, exist_ok=True) 337 | download("https://www.dropbox.com/s/izpty5feug57kqn/msrank_train.tsv?dl=1", train_path) 338 | download("https://www.dropbox.com/s/tlsmm9a6krv0215/msrank_test.tsv?dl=1", test_path) 339 | 340 | for fname in (train_path, test_path): 341 | raw = open(fname).read().replace('\\t', '\t') 342 | with open(fname, 'w') as f: 343 | f.write(raw) 344 | 345 | data_train = pd.read_csv(train_path, header=None, skiprows=1, sep='\t') 346 | data_test = pd.read_csv(test_path, header=None, skiprows=1, sep='\t') 347 | 348 | train_idx_path = os.path.join(path, 'train_idx.txt') 349 | valid_idx_path = os.path.join(path, 'valid_idx.txt') 350 | if not all(os.path.exists(fname) for fname in (train_idx_path, valid_idx_path)): 351 | download("https://www.dropbox.com/s/pba6dyibyogep46/train_idx.txt?dl=1", train_idx_path) 352 | download("https://www.dropbox.com/s/yednqu9edgdd2l1/valid_idx.txt?dl=1", valid_idx_path) 353 | train_idx = pd.read_csv(train_idx_path, header=None)[0].values 354 | valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values 355 | 356 | X_train, y_train, query_train = data_train.iloc[train_idx, 2:].values, data_train.iloc[train_idx, 0].values, data_train.iloc[train_idx, 1].values 357 | X_valid, y_valid, query_valid = data_train.iloc[valid_idx, 2:].values, data_train.iloc[valid_idx, 0].values, data_train.iloc[valid_idx, 1].values 358 | X_test, y_test, query_test = data_test.iloc[:, 2:].values, data_test.iloc[:, 0].values, data_test.iloc[:, 1].values 359 | 360 | return dict( 361 | X_train=X_train.astype(np.float32), y_train=y_train.astype(np.int64), query_train=query_train, 362 | X_valid=X_valid.astype(np.float32), y_valid=y_valid.astype(np.int64), query_valid=query_valid, 363 | X_test=X_test.astype(np.float32), y_test=y_test.astype(np.int64), query_test=query_test, 364 | ) 365 | 366 | 367 | def fetch_YAHOO(path): 368 | train_path = os.path.join(path, 'yahoo_train.tsv') 369 | valid_path = os.path.join(path, 'yahoo_valid.tsv') 370 | test_path = os.path.join(path, 'yahoo_test.tsv') 371 | if not all(os.path.exists(fname) for fname in (train_path, valid_path, test_path)): 372 | os.makedirs(path, exist_ok=True) 373 | train_archive_path = os.path.join(path, 'yahoo_train.tsv.gz') 374 | valid_archive_path = os.path.join(path, 'yahoo_valid.tsv.gz') 375 | test_archive_path = os.path.join(path, 'yahoo_test.tsv.gz') 376 | if not all(os.path.exists(fname) for fname in (train_archive_path, valid_archive_path, test_archive_path)): 377 | download("https://www.dropbox.com/s/7rq3ki5vtxm6gzx/yahoo_set_1_train.gz?dl=1", train_archive_path) 378 | download("https://www.dropbox.com/s/3ai8rxm1v0l5sd1/yahoo_set_1_validation.gz?dl=1", valid_archive_path) 379 | download("https://www.dropbox.com/s/3d7tdfb1an0b6i4/yahoo_set_1_test.gz?dl=1", test_archive_path) 380 | 381 | for file_name, archive_name in zip((train_path, valid_path, test_path), (train_archive_path, valid_archive_path, test_archive_path)): 382 | with gzip.open(archive_name, 'rb') as f_in: 383 | with open(file_name, 'wb') as f_out: 384 | shutil.copyfileobj(f_in, f_out) 385 | 386 | for fname in (train_path, valid_path, test_path): 387 | raw = open(fname).read().replace('\\t', '\t') 388 | with open(fname, 'w') as f: 389 | f.write(raw) 390 | 391 | data_train = pd.read_csv(train_path, header=None, skiprows=1, sep='\t') 392 | data_valid = pd.read_csv(valid_path, header=None, skiprows=1, sep='\t') 393 | data_test = pd.read_csv(test_path, header=None, skiprows=1, sep='\t') 394 | 395 | X_train, y_train, query_train = data_train.iloc[:, 2:].values, data_train.iloc[:, 0].values, data_train.iloc[:, 1].values 396 | X_valid, y_valid, query_valid = data_valid.iloc[:, 2:].values, data_valid.iloc[:, 0].values, data_valid.iloc[:, 1].values 397 | X_test, y_test, query_test = data_test.iloc[:, 2:].values, data_test.iloc[:, 0].values, data_test.iloc[:, 1].values 398 | 399 | return dict( 400 | X_train=X_train.astype(np.float32), y_train=y_train, query_train=query_train, 401 | X_valid=X_valid.astype(np.float32), y_valid=y_valid, query_valid=query_valid, 402 | X_test=X_test.astype(np.float32), y_test=y_test, query_test=query_test, 403 | ) 404 | 405 | 406 | def fetch_CLICK(path, valid_size=100_000, validation_seed=None): 407 | # based on: https://www.kaggle.com/slamnz/primer-airlines-delay 408 | csv_path = os.path.join(path, 'click.csv') 409 | if not os.path.exists(csv_path): 410 | os.makedirs(path, exist_ok=True) 411 | download('https://www.dropbox.com/s/w43ylgrl331svqc/click.csv?dl=1', csv_path) 412 | 413 | data = pd.read_csv(csv_path, index_col=0) 414 | X, y = data.drop(columns=['target']), data['target'] 415 | X_train, X_test = X[:-100_000].copy(), X[-100_000:].copy() 416 | y_train, y_test = y[:-100_000].copy(), y[-100_000:].copy() 417 | 418 | y_train = (y_train.values.reshape(-1) == 1).astype('int64') 419 | y_test = (y_test.values.reshape(-1) == 1).astype('int64') 420 | 421 | cat_features = ['url_hash', 'ad_id', 'advertiser_id', 'query_id', 422 | 'keyword_id', 'title_id', 'description_id', 'user_id'] 423 | 424 | X_train, X_val, y_train, y_val = train_test_split( 425 | X_train, y_train, test_size=valid_size, random_state=validation_seed) 426 | 427 | cat_encoder = LeaveOneOutEncoder() 428 | cat_encoder.fit(X_train[cat_features], y_train) 429 | X_train[cat_features] = cat_encoder.transform(X_train[cat_features]) 430 | X_val[cat_features] = cat_encoder.transform(X_val[cat_features]) 431 | X_test[cat_features] = cat_encoder.transform(X_test[cat_features]) 432 | return dict( 433 | X_train=X_train.values.astype('float32'), y_train=y_train, 434 | X_valid=X_val.values.astype('float32'), y_valid=y_val, 435 | X_test=X_test.values.astype('float32'), y_test=y_test 436 | ) 437 | 438 | 439 | DATASETS = { 440 | 'A9A': fetch_A9A, 441 | 'EPSILON': fetch_EPSILON, 442 | 'PROTEIN': fetch_PROTEIN, 443 | 'YEAR': fetch_YEAR, 444 | 'HIGGS': fetch_HIGGS, 445 | 'MICROSOFT': fetch_MICROSOFT, 446 | 'YAHOO': fetch_YAHOO, 447 | 'CLICK': fetch_CLICK, 448 | } 449 | -------------------------------------------------------------------------------- /lib/nn_utils.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from torch.autograd import Function 8 | from collections import OrderedDict 9 | 10 | from torch.jit import script 11 | 12 | 13 | def to_one_hot(y, depth=None): 14 | r""" 15 | Takes integer with n dims and converts it to 1-hot representation with n + 1 dims. 16 | The n+1'st dimension will have zeros everywhere but at y'th index, where it will be equal to 1. 17 | Args: 18 | y: input integer (IntTensor, LongTensor or Variable) of any shape 19 | depth (int): the size of the one hot dimension 20 | """ 21 | y_flat = y.to(torch.int64).view(-1, 1) 22 | depth = depth if depth is not None else int(torch.max(y_flat)) + 1 23 | y_one_hot = torch.zeros(y_flat.size()[0], depth, device=y.device).scatter_(1, y_flat, 1) 24 | y_one_hot = y_one_hot.view(*(tuple(y.shape) + (-1,))) 25 | return y_one_hot 26 | 27 | 28 | def _make_ix_like(input, dim=0): 29 | d = input.size(dim) 30 | rho = torch.arange(1, d + 1, device=input.device, dtype=input.dtype) 31 | view = [1] * input.dim() 32 | view[0] = -1 33 | return rho.view(view).transpose(0, dim) 34 | 35 | 36 | class SparsemaxFunction(Function): 37 | """ 38 | An implementation of sparsemax (Martins & Astudillo, 2016). See 39 | :cite:`DBLP:journals/corr/MartinsA16` for detailed description. 40 | 41 | By Ben Peters and Vlad Niculae 42 | """ 43 | 44 | @staticmethod 45 | def forward(ctx, input, dim=-1): 46 | """sparsemax: normalizing sparse transform (a la softmax) 47 | 48 | Parameters: 49 | input (Tensor): any shape 50 | dim: dimension along which to apply sparsemax 51 | 52 | Returns: 53 | output (Tensor): same shape as input 54 | """ 55 | ctx.dim = dim 56 | max_val, _ = input.max(dim=dim, keepdim=True) 57 | input -= max_val # same numerical stability trick as for softmax 58 | tau, supp_size = SparsemaxFunction._threshold_and_support(input, dim=dim) 59 | output = torch.clamp(input - tau, min=0) 60 | ctx.save_for_backward(supp_size, output) 61 | return output 62 | 63 | @staticmethod 64 | def backward(ctx, grad_output): 65 | supp_size, output = ctx.saved_tensors 66 | dim = ctx.dim 67 | grad_input = grad_output.clone() 68 | grad_input[output == 0] = 0 69 | 70 | v_hat = grad_input.sum(dim=dim) / supp_size.to(output.dtype).squeeze() 71 | v_hat = v_hat.unsqueeze(dim) 72 | grad_input = torch.where(output != 0, grad_input - v_hat, grad_input) 73 | return grad_input, None 74 | 75 | 76 | @staticmethod 77 | def _threshold_and_support(input, dim=-1): 78 | """Sparsemax building block: compute the threshold 79 | 80 | Args: 81 | input: any dimension 82 | dim: dimension along which to apply the sparsemax 83 | 84 | Returns: 85 | the threshold value 86 | """ 87 | 88 | input_srt, _ = torch.sort(input, descending=True, dim=dim) 89 | input_cumsum = input_srt.cumsum(dim) - 1 90 | rhos = _make_ix_like(input, dim) 91 | support = rhos * input_srt > input_cumsum 92 | 93 | support_size = support.sum(dim=dim).unsqueeze(dim) 94 | tau = input_cumsum.gather(dim, support_size - 1) 95 | tau /= support_size.to(input.dtype) 96 | return tau, support_size 97 | 98 | 99 | sparsemax = lambda input, dim=-1: SparsemaxFunction.apply(input, dim) 100 | sparsemoid = lambda input: (0.5 * input + 0.5).clamp_(0, 1) 101 | 102 | 103 | class Entmax15Function(Function): 104 | """ 105 | An implementation of exact Entmax with alpha=1.5 (B. Peters, V. Niculae, A. Martins). See 106 | :cite:`https://arxiv.org/abs/1905.05702 for detailed description. 107 | Source: https://github.com/deep-spin/entmax 108 | """ 109 | 110 | @staticmethod 111 | def forward(ctx, input, dim=-1): 112 | ctx.dim = dim 113 | 114 | max_val, _ = input.max(dim=dim, keepdim=True) 115 | input = input - max_val # same numerical stability trick as for softmax 116 | input = input / 2 # divide by 2 to solve actual Entmax 117 | 118 | tau_star, _ = Entmax15Function._threshold_and_support(input, dim) 119 | output = torch.clamp(input - tau_star, min=0) ** 2 120 | ctx.save_for_backward(output) 121 | return output 122 | 123 | @staticmethod 124 | def backward(ctx, grad_output): 125 | Y, = ctx.saved_tensors 126 | gppr = Y.sqrt() # = 1 / g'' (Y) 127 | dX = grad_output * gppr 128 | q = dX.sum(ctx.dim) / gppr.sum(ctx.dim) 129 | q = q.unsqueeze(ctx.dim) 130 | dX -= q * gppr 131 | return dX, None 132 | 133 | @staticmethod 134 | def _threshold_and_support(input, dim=-1): 135 | Xsrt, _ = torch.sort(input, descending=True, dim=dim) 136 | 137 | rho = _make_ix_like(input, dim) 138 | mean = Xsrt.cumsum(dim) / rho 139 | mean_sq = (Xsrt ** 2).cumsum(dim) / rho 140 | ss = rho * (mean_sq - mean ** 2) 141 | delta = (1 - ss) / rho 142 | 143 | # NOTE this is not exactly the same as in reference algo 144 | # Fortunately it seems the clamped values never wrongly 145 | # get selected by tau <= sorted_z. Prove this! 146 | delta_nz = torch.clamp(delta, 0) 147 | tau = mean - torch.sqrt(delta_nz) 148 | 149 | support_size = (tau <= Xsrt).sum(dim).unsqueeze(dim) 150 | tau_star = tau.gather(dim, support_size - 1) 151 | return tau_star, support_size 152 | 153 | 154 | class Entmoid15(Function): 155 | """ A highly optimized equivalent of labda x: Entmax15([x, 0]) """ 156 | 157 | @staticmethod 158 | def forward(ctx, input): 159 | output = Entmoid15._forward(input) 160 | ctx.save_for_backward(output) 161 | return output 162 | 163 | @staticmethod 164 | @script 165 | def _forward(input): 166 | input, is_pos = abs(input), input >= 0 167 | tau = (input + torch.sqrt(F.relu(8 - input ** 2))) / 2 168 | tau.masked_fill_(tau <= input, 2.0) 169 | y_neg = 0.25 * F.relu(tau - input, inplace=True) ** 2 170 | return torch.where(is_pos, 1 - y_neg, y_neg) 171 | 172 | @staticmethod 173 | def backward(ctx, grad_output): 174 | return Entmoid15._backward(ctx.saved_tensors[0], grad_output) 175 | 176 | @staticmethod 177 | @script 178 | def _backward(output, grad_output): 179 | gppr0, gppr1 = output.sqrt(), (1 - output).sqrt() 180 | grad_input = grad_output * gppr0 181 | q = grad_input / (gppr0 + gppr1) 182 | grad_input -= q * gppr0 183 | return grad_input 184 | 185 | 186 | entmax15 = lambda input, dim=-1: Entmax15Function.apply(input, dim) 187 | entmoid15 = Entmoid15.apply 188 | 189 | 190 | class Lambda(nn.Module): 191 | def __init__(self, func): 192 | super().__init__() 193 | self.func = func 194 | 195 | def forward(self, *args, **kwargs): 196 | return self.func(*args, **kwargs) 197 | 198 | 199 | class ModuleWithInit(nn.Module): 200 | """ Base class for pytorch module with data-aware initializer on first batch """ 201 | def __init__(self): 202 | super().__init__() 203 | self._is_initialized_tensor = nn.Parameter(torch.tensor(0, dtype=torch.uint8), requires_grad=False) 204 | self._is_initialized_bool = None 205 | # Note: this module uses a separate flag self._is_initialized so as to achieve both 206 | # * persistence: is_initialized is saved alongside model in state_dict 207 | # * speed: model doesn't need to cache 208 | # please DO NOT use these flags in child modules 209 | 210 | def initialize(self, *args, **kwargs): 211 | """ initialize module tensors using first batch of data """ 212 | raise NotImplementedError("Please implement ") 213 | 214 | def __call__(self, *args, **kwargs): 215 | if self._is_initialized_bool is None: 216 | self._is_initialized_bool = bool(self._is_initialized_tensor.item()) 217 | if not self._is_initialized_bool: 218 | self.initialize(*args, **kwargs) 219 | self._is_initialized_tensor.data[...] = 1 220 | self._is_initialized_bool = True 221 | return super().__call__(*args, **kwargs) 222 | -------------------------------------------------------------------------------- /lib/odst.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | import numpy as np 5 | 6 | from .nn_utils import sparsemax, sparsemoid, ModuleWithInit 7 | from .utils import check_numpy 8 | from warnings import warn 9 | 10 | 11 | class ODST(ModuleWithInit): 12 | def __init__(self, in_features, num_trees, depth=6, tree_dim=1, flatten_output=True, 13 | choice_function=sparsemax, bin_function=sparsemoid, 14 | initialize_response_=nn.init.normal_, initialize_selection_logits_=nn.init.uniform_, 15 | threshold_init_beta=1.0, threshold_init_cutoff=1.0, 16 | ): 17 | """ 18 | Oblivious Differentiable Sparsemax Trees. http://tinyurl.com/odst-readmore 19 | One can drop (sic!) this module anywhere instead of nn.Linear 20 | :param in_features: number of features in the input tensor 21 | :param num_trees: number of trees in this layer 22 | :param tree_dim: number of response channels in the response of individual tree 23 | :param depth: number of splits in every tree 24 | :param flatten_output: if False, returns [..., num_trees, tree_dim], 25 | by default returns [..., num_trees * tree_dim] 26 | :param choice_function: f(tensor, dim) -> R_simplex computes feature weights s.t. f(tensor, dim).sum(dim) == 1 27 | :param bin_function: f(tensor) -> R[0, 1], computes tree leaf weights 28 | 29 | :param initialize_response_: in-place initializer for tree output tensor 30 | :param initialize_selection_logits_: in-place initializer for logits that select features for the tree 31 | both thresholds and scales are initialized with data-aware init (or .load_state_dict) 32 | :param threshold_init_beta: initializes threshold to a q-th quantile of data points 33 | where q ~ Beta(:threshold_init_beta:, :threshold_init_beta:) 34 | If this param is set to 1, initial thresholds will have the same distribution as data points 35 | If greater than 1 (e.g. 10), thresholds will be closer to median data value 36 | If less than 1 (e.g. 0.1), thresholds will approach min/max data values. 37 | 38 | :param threshold_init_cutoff: threshold log-temperatures initializer, \in (0, inf) 39 | By default(1.0), log-remperatures are initialized in such a way that all bin selectors 40 | end up in the linear region of sparse-sigmoid. The temperatures are then scaled by this parameter. 41 | Setting this value > 1.0 will result in some margin between data points and sparse-sigmoid cutoff value 42 | Setting this value < 1.0 will cause (1 - value) part of data points to end up in flat sparse-sigmoid region 43 | For instance, threshold_init_cutoff = 0.9 will set 10% points equal to 0.0 or 1.0 44 | Setting this value > 1.0 will result in a margin between data points and sparse-sigmoid cutoff value 45 | All points will be between (0.5 - 0.5 / threshold_init_cutoff) and (0.5 + 0.5 / threshold_init_cutoff) 46 | """ 47 | super().__init__() 48 | self.depth, self.num_trees, self.tree_dim, self.flatten_output = depth, num_trees, tree_dim, flatten_output 49 | self.choice_function, self.bin_function = choice_function, bin_function 50 | self.threshold_init_beta, self.threshold_init_cutoff = threshold_init_beta, threshold_init_cutoff 51 | 52 | self.response = nn.Parameter(torch.zeros([num_trees, tree_dim, 2 ** depth]), requires_grad=True) 53 | initialize_response_(self.response) 54 | 55 | self.feature_selection_logits = nn.Parameter( 56 | torch.zeros([in_features, num_trees, depth]), requires_grad=True 57 | ) 58 | initialize_selection_logits_(self.feature_selection_logits) 59 | 60 | self.feature_thresholds = nn.Parameter( 61 | torch.full([num_trees, depth], float('nan'), dtype=torch.float32), requires_grad=True 62 | ) # nan values will be initialized on first batch (data-aware init) 63 | 64 | self.log_temperatures = nn.Parameter( 65 | torch.full([num_trees, depth], float('nan'), dtype=torch.float32), requires_grad=True 66 | ) 67 | 68 | # binary codes for mapping between 1-hot vectors and bin indices 69 | with torch.no_grad(): 70 | indices = torch.arange(2 ** self.depth) 71 | offsets = 2 ** torch.arange(self.depth) 72 | bin_codes = (indices.view(1, -1) // offsets.view(-1, 1) % 2).to(torch.float32) 73 | bin_codes_1hot = torch.stack([bin_codes, 1.0 - bin_codes], dim=-1) 74 | self.bin_codes_1hot = nn.Parameter(bin_codes_1hot, requires_grad=False) 75 | # ^-- [depth, 2 ** depth, 2] 76 | 77 | def forward(self, input): 78 | assert len(input.shape) >= 2 79 | if len(input.shape) > 2: 80 | return self.forward(input.view(-1, input.shape[-1])).view(*input.shape[:-1], -1) 81 | # new input shape: [batch_size, in_features] 82 | 83 | feature_logits = self.feature_selection_logits 84 | feature_selectors = self.choice_function(feature_logits, dim=0) 85 | # ^--[in_features, num_trees, depth] 86 | 87 | feature_values = torch.einsum('bi,ind->bnd', input, feature_selectors) 88 | # ^--[batch_size, num_trees, depth] 89 | 90 | threshold_logits = (feature_values - self.feature_thresholds) * torch.exp(-self.log_temperatures) 91 | 92 | threshold_logits = torch.stack([-threshold_logits, threshold_logits], dim=-1) 93 | # ^--[batch_size, num_trees, depth, 2] 94 | 95 | bins = self.bin_function(threshold_logits) 96 | # ^--[batch_size, num_trees, depth, 2], approximately binary 97 | 98 | bin_matches = torch.einsum('btds,dcs->btdc', bins, self.bin_codes_1hot) 99 | # ^--[batch_size, num_trees, depth, 2 ** depth] 100 | 101 | response_weights = torch.prod(bin_matches, dim=-2) 102 | # ^-- [batch_size, num_trees, 2 ** depth] 103 | 104 | response = torch.einsum('bnd,ncd->bnc', response_weights, self.response) 105 | # ^-- [batch_size, num_trees, tree_dim] 106 | 107 | return response.flatten(1, 2) if self.flatten_output else response 108 | 109 | def initialize(self, input, eps=1e-6): 110 | # data-aware initializer 111 | assert len(input.shape) == 2 112 | if input.shape[0] < 1000: 113 | warn("Data-aware initialization is performed on less than 1000 data points. This may cause instability." 114 | "To avoid potential problems, run this model on a data batch with at least 1000 data samples." 115 | "You can do so manually before training. Use with torch.no_grad() for memory efficiency.") 116 | with torch.no_grad(): 117 | feature_selectors = self.choice_function(self.feature_selection_logits, dim=0) 118 | # ^--[in_features, num_trees, depth] 119 | 120 | feature_values = torch.einsum('bi,ind->bnd', input, feature_selectors) 121 | # ^--[batch_size, num_trees, depth] 122 | 123 | # initialize thresholds: sample random percentiles of data 124 | percentiles_q = 100 * np.random.beta(self.threshold_init_beta, self.threshold_init_beta, 125 | size=[self.num_trees, self.depth]) 126 | self.feature_thresholds.data[...] = torch.as_tensor( 127 | list(map(np.percentile, check_numpy(feature_values.flatten(1, 2).t()), percentiles_q.flatten())), 128 | dtype=feature_values.dtype, device=feature_values.device 129 | ).view(self.num_trees, self.depth) 130 | 131 | # init temperatures: make sure enough data points are in the linear region of sparse-sigmoid 132 | temperatures = np.percentile(check_numpy(abs(feature_values - self.feature_thresholds)), 133 | q=100 * min(1.0, self.threshold_init_cutoff), axis=0) 134 | 135 | # if threshold_init_cutoff > 1, scale everything down by it 136 | temperatures /= max(1.0, self.threshold_init_cutoff) 137 | self.log_temperatures.data[...] = torch.log(torch.as_tensor(temperatures) + eps) 138 | 139 | def __repr__(self): 140 | return "{}(in_features={}, num_trees={}, depth={}, tree_dim={}, flatten_output={})".format( 141 | self.__class__.__name__, self.feature_selection_logits.shape[0], 142 | self.num_trees, self.depth, self.tree_dim, self.flatten_output 143 | ) 144 | 145 | -------------------------------------------------------------------------------- /lib/trainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import glob 4 | import numpy as np 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | from .utils import get_latest_file, iterate_minibatches, check_numpy, process_in_chunks 10 | from .nn_utils import to_one_hot 11 | from collections import OrderedDict 12 | from copy import deepcopy 13 | from tensorboardX import SummaryWriter 14 | 15 | from sklearn.metrics import roc_auc_score, log_loss 16 | 17 | 18 | class Trainer(nn.Module): 19 | def __init__(self, model, loss_function, experiment_name=None, warm_start=False, 20 | Optimizer=torch.optim.Adam, optimizer_params={}, verbose=False, 21 | n_last_checkpoints=1, **kwargs): 22 | """ 23 | :type model: torch.nn.Module 24 | :param loss_function: the metric to use in trainnig 25 | :param experiment_name: a path where all logs and checkpoints are saved 26 | :param warm_start: when set to True, loads last checpoint 27 | :param Optimizer: function(parameters) -> optimizer 28 | :param verbose: when set to True, produces logging information 29 | """ 30 | super().__init__() 31 | self.model = model 32 | self.loss_function = loss_function 33 | self.verbose = verbose 34 | self.opt = Optimizer(list(self.model.parameters()), **optimizer_params) 35 | self.step = 0 36 | self.n_last_checkpoints = n_last_checkpoints 37 | 38 | if experiment_name is None: 39 | experiment_name = 'untitled_{}.{:0>2d}.{:0>2d}_{:0>2d}:{:0>2d}'.format(*time.gmtime()[:5]) 40 | if self.verbose: 41 | print('using automatic experiment name: ' + experiment_name) 42 | 43 | self.experiment_path = os.path.join('logs/', experiment_name) 44 | if not warm_start and experiment_name != 'debug': 45 | assert not os.path.exists(self.experiment_path), 'experiment {} already exists'.format(experiment_name) 46 | self.writer = SummaryWriter(self.experiment_path, comment=experiment_name) 47 | if warm_start: 48 | self.load_checkpoint() 49 | 50 | def save_checkpoint(self, tag=None, path=None, mkdir=True, **kwargs): 51 | assert tag is None or path is None, "please provide either tag or path or nothing, not both" 52 | if tag is None and path is None: 53 | tag = "temp_{}".format(self.step) 54 | if path is None: 55 | path = os.path.join(self.experiment_path, "checkpoint_{}.pth".format(tag)) 56 | if mkdir: 57 | os.makedirs(os.path.dirname(path), exist_ok=True) 58 | torch.save(OrderedDict([ 59 | ('model', self.state_dict(**kwargs)), 60 | ('opt', self.opt.state_dict()), 61 | ('step', self.step) 62 | ]), path) 63 | if self.verbose: 64 | print("Saved " + path) 65 | return path 66 | 67 | def load_checkpoint(self, tag=None, path=None, **kwargs): 68 | assert tag is None or path is None, "please provide either tag or path or nothing, not both" 69 | if tag is None and path is None: 70 | path = get_latest_file(os.path.join(self.experiment_path, 'checkpoint_temp_[0-9]*.pth')) 71 | elif tag is not None and path is None: 72 | path = os.path.join(self.experiment_path, "checkpoint_{}.pth".format(tag)) 73 | checkpoint = torch.load(path) 74 | 75 | self.load_state_dict(checkpoint['model'], **kwargs) 76 | self.opt.load_state_dict(checkpoint['opt']) 77 | self.step = int(checkpoint['step']) 78 | 79 | if self.verbose: 80 | print('Loaded ' + path) 81 | return self 82 | 83 | def average_checkpoints(self, tags=None, paths=None, out_tag='avg', out_path=None): 84 | assert tags is None or paths is None, "please provide either tags or paths or nothing, not both" 85 | assert out_tag is not None or out_path is not None, "please provide either out_tag or out_path or both, not nothing" 86 | if tags is None and paths is None: 87 | paths = self.get_latest_checkpoints( 88 | os.path.join(self.experiment_path, 'checkpoint_temp_[0-9]*.pth'), self.n_last_checkpoints) 89 | elif tags is not None and paths is None: 90 | paths = [os.path.join(self.experiment_path, 'checkpoint_{}.pth'.format(tag)) for tag in tags] 91 | 92 | checkpoints = [torch.load(path) for path in paths] 93 | averaged_ckpt = deepcopy(checkpoints[0]) 94 | for key in averaged_ckpt['model']: 95 | values = [ckpt['model'][key] for ckpt in checkpoints] 96 | averaged_ckpt['model'][key] = sum(values) / len(values) 97 | 98 | if out_path is None: 99 | out_path = os.path.join(self.experiment_path, 'checkpoint_{}.pth'.format(out_tag)) 100 | torch.save(averaged_ckpt, out_path) 101 | 102 | def get_latest_checkpoints(self, pattern, n_last=None): 103 | list_of_files = glob.glob(pattern) 104 | assert len(list_of_files) > 0, "No files found: " + pattern 105 | return sorted(list_of_files, key=os.path.getctime, reverse=True)[:n_last] 106 | 107 | def remove_old_temp_checkpoints(self, number_ckpts_to_keep=None): 108 | if number_ckpts_to_keep is None: 109 | number_ckpts_to_keep = self.n_last_checkpoints 110 | paths = self.get_latest_checkpoints(os.path.join(self.experiment_path, 'checkpoint_temp_[0-9]*.pth')) 111 | paths_to_delete = paths[number_ckpts_to_keep:] 112 | 113 | for ckpt in paths_to_delete: 114 | os.remove(ckpt) 115 | 116 | def train_on_batch(self, *batch, device): 117 | x_batch, y_batch = batch 118 | x_batch = torch.as_tensor(x_batch, device=device) 119 | y_batch = torch.as_tensor(y_batch, device=device) 120 | 121 | self.model.train() 122 | self.opt.zero_grad() 123 | loss = self.loss_function(self.model(x_batch), y_batch).mean() 124 | loss.backward() 125 | self.opt.step() 126 | self.step += 1 127 | self.writer.add_scalar('train loss', loss.item(), self.step) 128 | 129 | return {'loss': loss} 130 | 131 | def evaluate_classification_error(self, X_test, y_test, device, batch_size=4096): 132 | X_test = torch.as_tensor(X_test, device=device) 133 | y_test = check_numpy(y_test) 134 | self.model.train(False) 135 | with torch.no_grad(): 136 | logits = process_in_chunks(self.model, X_test, batch_size=batch_size) 137 | logits = check_numpy(logits) 138 | error_rate = (y_test != np.argmax(logits, axis=1)).mean() 139 | return error_rate 140 | 141 | def evaluate_mse(self, X_test, y_test, device, batch_size=4096): 142 | X_test = torch.as_tensor(X_test, device=device) 143 | y_test = check_numpy(y_test) 144 | self.model.train(False) 145 | with torch.no_grad(): 146 | prediction = process_in_chunks(self.model, X_test, batch_size=batch_size) 147 | prediction = check_numpy(prediction) 148 | error_rate = ((y_test - prediction) ** 2).mean() 149 | return error_rate 150 | 151 | def evaluate_auc(self, X_test, y_test, device, batch_size=512): 152 | X_test = torch.as_tensor(X_test, device=device) 153 | y_test = check_numpy(y_test) 154 | self.model.train(False) 155 | with torch.no_grad(): 156 | logits = F.softmax(process_in_chunks(self.model, X_test, batch_size=batch_size), dim=1) 157 | logits = check_numpy(logits) 158 | y_test = torch.tensor(y_test) 159 | auc = roc_auc_score(check_numpy(to_one_hot(y_test)), logits) 160 | return auc 161 | 162 | def evaluate_logloss(self, X_test, y_test, device, batch_size=512): 163 | X_test = torch.as_tensor(X_test, device=device) 164 | y_test = check_numpy(y_test) 165 | self.model.train(False) 166 | with torch.no_grad(): 167 | logits = F.softmax(process_in_chunks(self.model, X_test, batch_size=batch_size), dim=1) 168 | logits = check_numpy(logits) 169 | y_test = torch.tensor(y_test) 170 | logloss = log_loss(check_numpy(to_one_hot(y_test)), logits) 171 | return logloss 172 | -------------------------------------------------------------------------------- /lib/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import hashlib 4 | import gc 5 | import time 6 | import numpy as np 7 | import requests 8 | import contextlib 9 | from tqdm import tqdm 10 | import torch 11 | 12 | 13 | def download(url, filename, delete_if_interrupted=True, chunk_size=4096): 14 | """ saves file from url to filename with a fancy progressbar """ 15 | try: 16 | with open(filename, "wb") as f: 17 | print("Downloading {} > {}".format(url, filename)) 18 | response = requests.get(url, stream=True) 19 | total_length = response.headers.get('content-length') 20 | 21 | if total_length is None: # no content length header 22 | f.write(response.content) 23 | else: 24 | total_length = int(total_length) 25 | with tqdm(total=total_length) as progressbar: 26 | for data in response.iter_content(chunk_size=chunk_size): 27 | if data: # filter-out keep-alive chunks 28 | f.write(data) 29 | progressbar.update(len(data)) 30 | except Exception as e: 31 | if delete_if_interrupted: 32 | print("Removing incomplete download {}.".format(filename)) 33 | os.remove(filename) 34 | raise e 35 | return filename 36 | 37 | 38 | def iterate_minibatches(*tensors, batch_size, shuffle=True, epochs=1, 39 | allow_incomplete=True, callback=lambda x:x): 40 | indices = np.arange(len(tensors[0])) 41 | upper_bound = int((np.ceil if allow_incomplete else np.floor) (len(indices) / batch_size)) * batch_size 42 | epoch = 0 43 | while True: 44 | if shuffle: 45 | np.random.shuffle(indices) 46 | for batch_start in callback(range(0, upper_bound, batch_size)): 47 | batch_ix = indices[batch_start: batch_start + batch_size] 48 | batch = [tensor[batch_ix] for tensor in tensors] 49 | yield batch if len(tensors) > 1 else batch[0] 50 | epoch += 1 51 | if epoch >= epochs: 52 | break 53 | 54 | 55 | def process_in_chunks(function, *args, batch_size, out=None, **kwargs): 56 | """ 57 | Computes output by applying batch-parallel function to large data tensor in chunks 58 | :param function: a function(*[x[indices, ...] for x in args]) -> out[indices, ...] 59 | :param args: one or many tensors, each [num_instances, ...] 60 | :param batch_size: maximum chunk size processed in one go 61 | :param out: memory buffer for out, defaults to torch.zeros of appropriate size and type 62 | :returns: function(data), computed in a memory-efficient way 63 | """ 64 | total_size = args[0].shape[0] 65 | first_output = function(*[x[0: batch_size] for x in args]) 66 | output_shape = (total_size,) + tuple(first_output.shape[1:]) 67 | if out is None: 68 | out = torch.zeros(*output_shape, dtype=first_output.dtype, device=first_output.device, 69 | layout=first_output.layout, **kwargs) 70 | 71 | out[0: batch_size] = first_output 72 | for i in range(batch_size, total_size, batch_size): 73 | batch_ix = slice(i, min(i + batch_size, total_size)) 74 | out[batch_ix] = function(*[x[batch_ix] for x in args]) 75 | return out 76 | 77 | 78 | def check_numpy(x): 79 | """ Makes sure x is a numpy array """ 80 | if isinstance(x, torch.Tensor): 81 | x = x.detach().cpu().numpy() 82 | x = np.asarray(x) 83 | assert isinstance(x, np.ndarray) 84 | return x 85 | 86 | 87 | @contextlib.contextmanager 88 | def nop_ctx(): 89 | yield None 90 | 91 | 92 | def get_latest_file(pattern): 93 | list_of_files = glob.glob(pattern) # * means all if need specific format then *.csv 94 | assert len(list_of_files) > 0, "No files found: " + pattern 95 | return max(list_of_files, key=os.path.getctime) 96 | 97 | 98 | def md5sum(fname): 99 | """ Computes mdp checksum of a file """ 100 | hash_md5 = hashlib.md5() 101 | with open(fname, "rb") as f: 102 | for chunk in iter(lambda: f.read(4096), b""): 103 | hash_md5.update(chunk) 104 | return hash_md5.hexdigest() 105 | 106 | 107 | def free_memory(sleep_time=0.1): 108 | """ Black magic function to free torch memory and some jupyter whims """ 109 | gc.collect() 110 | torch.cuda.synchronize() 111 | gc.collect() 112 | torch.cuda.empty_cache() 113 | time.sleep(sleep_time) 114 | 115 | def to_float_str(element): 116 | try: 117 | return str(float(element)) 118 | except ValueError: 119 | return element 120 | -------------------------------------------------------------------------------- /notebooks/epsilon_node_multigpu.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "env: CUDA_VISIBLE_DEVICES=0,1\n", 13 | "reading dataset (it may take a long time)\n", 14 | "experiment: epsilon_node_2layers_2019.08.28_13:04\n" 15 | ] 16 | } 17 | ], 18 | "source": [ 19 | "%load_ext autoreload\n", 20 | "%autoreload 2\n", 21 | "%env CUDA_VISIBLE_DEVICES=0,1\n", 22 | "import os, sys\n", 23 | "import time\n", 24 | "sys.path.insert(0, '..')\n", 25 | "import numpy as np\n", 26 | "import pandas as pd\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "%matplotlib inline\n", 29 | "import lib\n", 30 | "import torch, torch.nn as nn\n", 31 | "import torch.nn.functional as F\n", 32 | "from qhoptim.pyt import QHAdam\n", 33 | "\n", 34 | "# read the data\n", 35 | "data = lib.Dataset(\"EPSILON\", random_state=1337, quantile_transform=True, quantile_noise=1e-3)\n", 36 | "\n", 37 | "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", 38 | "\n", 39 | "experiment_name = 'epsilon_node_2layers'\n", 40 | "experiment_name = '{}_{}.{:0>2d}.{:0>2d}_{:0>2d}:{:0>2d}'.format(experiment_name, *time.gmtime()[:5])\n", 41 | "print(\"experiment:\", experiment_name)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "__Note:__ make sure you're using torch version `>= 1.1.0`, the code will silently fail even on 1.0.1." 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "num_features = data.X_train.shape[1]\n", 58 | "num_classes = len(set(data.y_train))\n", 59 | "\n", 60 | "model = nn.Sequential(\n", 61 | " lib.DenseBlock(num_features, layer_dim=1024, num_layers=2, tree_dim=num_classes + 1, flatten_output=False,\n", 62 | " depth=6, choice_function=lib.entmax15, bin_function=lib.entmoid15),\n", 63 | " lib.Lambda(lambda x: x[..., :num_classes].mean(dim=-2)),\n", 64 | ").to(device)\n", 65 | "\n", 66 | "with torch.no_grad():\n", 67 | " res = model(torch.as_tensor(data.X_train[:2000], device=device))\n", 68 | " # trigger data-aware init\n", 69 | " \n", 70 | "if torch.cuda.device_count() > 1:\n", 71 | " model = nn.DataParallel(model)\n", 72 | "\n", 73 | "\n", 74 | "trainer = lib.Trainer(\n", 75 | " model=model, loss_function=F.cross_entropy,\n", 76 | " experiment_name=experiment_name,\n", 77 | " warm_start=False,\n", 78 | " Optimizer=QHAdam,\n", 79 | " optimizer_params=dict(nus=(0.7, 1.0), betas=(0.95, 0.998)),\n", 80 | " verbose=True,\n", 81 | " n_last_checkpoints=5\n", 82 | ")" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 3, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "from tqdm import tqdm\n", 92 | "from IPython.display import clear_output\n", 93 | "loss_history, err_history = [], []\n", 94 | "best_val_err = 1.0\n", 95 | "best_step = 0\n", 96 | "early_stopping_rounds = 10_000\n", 97 | "report_frequency = 100" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 4, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAFpCAYAAACfyu4TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl81NW9//HXmZlM9oRAQth3ZBEFBcENiRUV9Va6V7u3tna5tvXa1h9trW2tVavdbGutdrleu1lrN1sRVCDihgKC7EuILCFsAUL2ZWbO74+ZhCyTkEAyMyd5Px8PHs58l5kPlE7enPmcc4y1FhERERGR/s4T7wJERERERBKBgrGIiIiICArGIiIiIiKAgrGIiIiICKBgLCIiIiICKBiLiIiIiAAKxiIiIiIigIKxiIiIiAigYCwiIiIiAigYi4iIiIgA4IvXG+fm5toxY8Z0+77q6mrS09N7vqBe5mLdLtYMqjvWXKz7TGteu3ZtmbU2rwdLSnj6zHaD6o49V2vvb3V3+XPbWhuXXzNnzrSnY8WKFad1X7y5WLeLNVurumPNxbrPtGZgjY3TZ2e8fukz2w2qO/Zcrb2/1d3Vz221UoiIiIiIoB5jERERERFAwVhEREREBFAwFhEREREBFIxFRERERAAFYxERERERQMFYRERERARQMBYRERERARSMRUREREQABWMREREREUDBWEREREQEcCwYv158lD0VwXiXISIiXbD9YCVbjuozW0Tc4VQw/vo/NvJMcWO8yxARkS7431fe5tEN9fEuQ0Sky5wKxl5jCNl4VyEiIl3h8RhCVh/aIuIOt4Kxx6CPWBERN/g8GswQEbc4FYyNRoxFRJzhMYagPrNFxCFOBWOvBwVjERFH+DwGdVKIiEvcCsYaMRYRcYbXoxFjEXGLU8HYo9EHERFneNRjLCKOcSsYG0NI0+9ERJygyXci4hqngrFaKURE3OEx4ZWErL7qExFHOBWMPZp8JyLiDJ/HABDUB7eIOMKtYGzUYywi4gpPJBgHFIxFxBFOBWOv+tVERJzhjQRj7X4nIq5wKhiHJ9+JiIgLfBoxFhHHOBWMvVquTUTEGR4TGTFWMBYRRzgVjD1Gk+9ERFzh82rynYi4xbFgbNSrJiLiiKYRYwVjEXGFU8HY61GPsYiIK5qXa9OAhog4wqlgrC2hRUTc0bxcW1Af3CLiBreCsXa+ExFpZoxZYIzZbowpMsYsinL+c8aYjcaY9caYl40xU1uc+3rkvu3GmKt7oz6v0XJtIuIWp4KxV5PvREQAMMZ4gYeAa4CpwI0tg2/En6y151hrZwD3Az+O3DsVuAE4G1gA/DLyej1Kk+9ExDVOBWOPx6CPVxERAGYDRdbaYmttA/AEsLDlBdbaihZP06H5I3Qh8IS1tt5a+zZQFHm9HqXJdyLiGl+8C+gOtVKIiDQbDuxr8bwEmNP2ImPMfwO3AX7gHS3uXdXm3uE9XaAm34mIa5wKxl4FYxGRJibKsXafkNbah4CHjDEfAu4APt7Ve40xNwM3A+Tn51NYWNitArccCgDw+hurOZjV450avaqqqqrbv99EoLpjz9XaVXd0TgVjj0fBWEQkogQY2eL5CKC0k+ufAB7uzr3W2keBRwFmzZplCwoKulVgcOshWLeG886fybkjBnTr3ngrLCyku7/fRKC6Y8/V2lV3dKfsMTbG/M4Yc9gYs6mD88YY87PI7OYNxpjze77MMK8HrLqMRUQAVgMTjTFjjTF+wpPpnm55gTFmYoun1wE7I4+fBm4wxiQbY8YCE4E3errA5uXaNKIhIo7oyojxY8AvgMc7OH8N4Q/ViYT72x4mSp9bT1CPsYhImLU2YIy5BVgKeIHfWWs3G2PuAtZYa58GbjHGzAcageOE2yiIXPcksAUIAP9trQ32dI3Ny7Xpg1tEHHHKYGytXWmMGdPJJQuBx621FlhljBlgjBlqrT3QQzU2UzAWETnJWrsYWNzm2J0tHn+5k3u/D3y/96prMflOH9wi4oieWK4t2szoHp/dDOEtoTW5WUTEDR4FYxFxTE9MvuvS7GY48xnO+0saCFmrWZQx4mLNoLpjzcW6XazZRVquTURc0xPBuMszo890hvNrtVsJ7S3WLMoYcbFmUN2x5mLdLtbsIk2+ExHX9EQrxdPAxyKrU1wInOiN/mIIT+TQwIOIiBs0+U5EXHPKEWNjzJ+BAiDXGFMCfBtIArDW/orwxI9rCW8pWgN8sreK9WodYxERZ3g1YiwijunKqhQ3nuK8Bf67xyrqhDEGG35PjInW2iwiIomiKRhrxFhEXNETrRQx0/y1nD5jRUQSnibfiYhr3ArGkWpD+pAVEUl4Wq5NRFzjVDBuap/Qh6yISOLTBh8i4hqngnFTv5oGjEVEEp/HaPKdiLjFqWAcycXqVxMRcYAm34mIaxwLxk2T7/QhKyKS6DT5TkRc42Yw1uiDiEjC0+Q7EXGNU8G4+Ws5fcaKiCQ8Tb4TEdc4FYybe4z1ISsikvA0YiwirnErGHvUYywi4gqNGIuIa5wKxl5NvhMRcYaWaxMR1zgVjD3a4ENExBlark1EXONWMNYGHyIizmj6lk/LtYmIK9wKxpp8JyLiDI/HYNBntoi4w6lg7NXkOxERp3iMgrGIuMOpYGw0+U5ExCkKxiLiEqeC8clVKeJciIiIdImCsYi4xKlgrB5jERG3eIyWaxMRd7gVjLVYvIiIUzxG7W8i4g63grHRcm0iIi7xqpVCRBziVDD2RqrVmpgiIm7wGKNgLCLOcCoYe7QqhYiIUzT5TkRc4mYw1oesiIgTFIxFxCVOBeOTG3zEuRAREekSj1H7m4i4w6lgbLRcm4iIUzRiLCIucSoYe5tXpdCHrIiICxSMRcQlTgXj5nWMFYxFRJzg1aoUIuIQt4KxtoQWEXGKRoxFxCWOBePwf7UqhYiIGzT5TkRc4lQw9mpLaBERp3jQZ7aIuMOpYKwNPkRE3KJWChFxiYKxiIj0GgVjEXGJU8FYG3yIiLhFwVhEXOJUMPZogw8REad4jdHkOxFxhlvB2KNWChERl2jEWERc4lYwVo+xiIhTjIKxiDjEqWDctCV0KBTnQkREpEu8CsYi4hCngrFp6jHWiLGIiBPUSiEiLnEqGDetSmEVjEVEnKCd70TEJU4F46Ye46BaKUREnKARYxFxiVvBOFKtRh9ERNzgNUbBWESc4VQwTook48aAhoxFRFygEWMRcYlTwTglyQtAvYKxiIgTtFybiLjEqWCc7AuXW9cYjHMlIiLSFV6jtedFxB1OBWOPx+DzQF1AwVhExAUeAwGNGIuII5wKxgB+D9Q3qpVCRMQF6jEWEZd0KRgbYxYYY7YbY4qMMYuinB9ljFlhjFlnjNlgjLm250sN83uNWilERByhYCwiLjllMDbGeIGHgGuAqcCNxpipbS67A3jSWnsecAPwy54utEmSRz3GIiKu8Gi5NhFxSFdGjGcDRdbaYmttA/AEsLDNNRbIijzOBkp7rsTW/F6oUyuFiIgTvBoxFhGH+LpwzXBgX4vnJcCcNtd8B3jOGPNFIB2Y3yPVReH3GE2+ExFxhNGW0CLikK4EYxPlWNtPuRuBx6y1PzLGXAT83hgzzVrbamjXGHMzcDNAfn4+hYWF3S7YQ5ADh4+e1r3xVFVVpZpjRHXHlot1u1izq7wGrIVQyOLxRPtxIiKSOLoSjEuAkS2ej6B9q8RNwAIAa+1rxpgUIBc43PIia+2jwKMAs2bNsgUFBd0u+Cdrl9DoS6egYG63742nwsJCTuf3G08u1gyqO9ZcrNvFml3VlIWD1uKJOs4iIpI4utJjvBqYaIwZa4zxE55c93Sba/YCVwAYY6YAKcCRniy0SUaSobymoTdeWkREelhzMFafsYg44JTB2FobAG4BlgJbCa8+sdkYc5cx5vrIZV8BPmOMeQv4M/AJa3unqSzDD8drGnvjpUVEpIcpGIuIS7rSSoG1djGwuM2xO1s83gJc0rOlRZeRZKhtDFDXGCQlyRuLtxQRkdPU1D6hCXgi4gLndr7LSAp/yJZr1FhE+rkubL50mzFmS2TjpWXGmNEtzgWNMesjv9q2x/UYb9OIcVDBWEQSn3vB2B/+lD2uPmMR6ce6uPnSOmCWtfZc4Cng/hbnaq21MyK/rqeXmBaT70REEp17wThJwVhEhC5svmStXWGtrYk8XUV4VaGYahoxDqnHWEQc4F4wjowYH6tWMBaRfi3a5kvDO7n+JuDZFs9TjDFrjDGrjDHv6o0C4eTku4CCsYg4oEuT7xJJpj/8312Hq+NbiIhIfHVl86XwhcZ8BJgFzGtxeJS1ttQYMw5YbozZaK3d1ea+M96UqbGhHjC88upr5KW5Mxbj6iYwqjv2XK1ddUfnXjCOtFL8fV0JX54/Mc7ViIjETVc2X8IYMx/4JjDPWlvfdNxaWxr5b7ExphA4D2gVjHtiU6ZX9j8PNHDB7DmMyU3v9v3x4uomMKo79lytXXVH584/3yO8ke/l9hytOcWVIiJ92ik3XzLGnAc8AlxvrT3c4niOMSY58jiX8HKbW3qjSI/Rcm0i4g7nRoxFRCS8+ZIxpmnzJS/wu6bNl4A11tqngQeADOCvJhxQ90ZWoJgCPGKMCREeILkvsh59j9PkOxFxiZPBOCvFR3VDMN5liIjEVRc2X5rfwX2vAuf0bnVhRpPvRMQhzrVSABRMGszInNR4lyEiIqfg1ZbQIuIQJ4NxVX2A3UdrsOpZExFJaB4FYxFxiJPBePm28BySXUe0ZJuISCLzaOc7EXGIk8H4oxeOBsDvdbJ8EZF+o3lVCo0Yi4gDnEyWF44bBEBtoybgiYgkMrVSiIhLnAzGvshsjv3lWstYRCSRabk2EXGJk8F48cYDAHzqsTVxrkRERDrj0XJtIuIQJ4PxOcOz412CiIh0gSbfiYhLnAzG0xSMRUSc0ByMgwrGIpL4nAzG540aEO8SRESkCzRiLCIucTIYa5k2ERE3aLk2EXGJkwnTRD5oRUQksWm5NhFxiZPBuMn0Eeo1FhFJZM3LtamVQkQc4It3Aadr+sgBDEhNincZIiLSiebl2jT5TkQc4OyIcZLHUNugne9ERBKZJt+JiEucHTFes+d4vEsQEZFTUI+xiLjE2RHj/KzkeJcgIiKn0PRDRsFYRFzgbDCePyWfnDT1GIuIJDJPZMhYk+9ExAXOtlL88fW9AASCIXxa11hEJCE1fTpr8p2IuMD5RNmoD1sRkYTVNG6hEWMRcYHzwdiiD1sRkUTVPGKsHmMRcYDzwXjd3vJ4lyAiIh0wWpVCRBzifDAu3H443iWIiEgHvArGIuIQZ4PxsOwUAI5VN8a5EhER6YjWMRYRlzgbjIfnpAJQWl4b50pERKQjxhg8RpPvRMQNzgbjYQMiwfiEgrGISCLzeowm34mIE5wNxu85fwQAV03Nj3MlIiLSGa/HEFIwFhEHOBuMJw7OAGB8XkacKxERkc54jUaMRcQNzgbj1CQvALWNwThXIiIinfF4jCbfiYgT3A3G/nAw/vdbpXGuREREOuNTMBYRRzgbjJN94dLf1AYfIiIJzesxBLUqhYg4wNlgbJq2UxIRkYSmyXci4gpng7GIiLhBk+9ExBUKxiIi0qu8Xo0Yi4gbFIxFRKRXacRYRFyhYCwiIr3Ko8l3IuIIp4Px8Mi20CIikrh8mnwnIo5wOhjvL6+NdwkiInIKHrVSiIgjuhSMjTELjDHbjTFFxphFHVzzAWPMFmPMZmPMn3q2zM7tPFQZy7cTEZFu8GnynYg44pTB2BjjBR4CrgGmAjcaY6a2uWYi8HXgEmvt2cCtvVBrh378/I5Yvp2IiHSDJt+JiCu6MmI8Gyiy1hZbaxuAJ4CFba75DPCQtfY4gLX2cM+W2blnNx2M5duJiEg3eDyGkCbfiYgDuhKMhwP7WjwviRxr6SzgLGPMK8aYVcaYBT1VoIiIuM3nMQSCCsYikvh8Xbgm2t7LbT/hfMBEoAAYAbxkjJlmrS1v9ULG3AzcDJCfn09hYWF366WqqirqfafzWrHUUd2JzMWaQXXHmot1u1izyzxGy7WJiBu6EoxLgJEtno8ASqNcs8pa2wi8bYzZTjgor255kbX2UeBRgFmzZtmCgoJuF1xYWEjTfdeUrG1uozid14qllnW7wsWaQXXHmot1u1izy3xeQ31jKN5liIicUldaKVYDE40xY40xfuAG4Ok21/wTuBzAGJNLuLWiuCcLjWbhjGG9/RYiInKGtFybiLjilMHYWhsAbgGWAluBJ621m40xdxljro9cthQ4aozZAqwAvmatPdpbRTe5cuqQ3n4LERE5Qz5NvhMRR3SllQJr7WJgcZtjd7Z4bIHbIr9ixuuJ1v4sIiKJxKvJdyLiCKd3vhMRkcTnMRoxFhE3KBiLiEiv8nnVYywiblAwFhGRXuUx2hJaRNzQZ4Lx7rLqeJcgIiJR+Dxax1hE3NBngvFtT66PdwkiIhKFR5PvRMQRfSYYv7m3/NQXiYhIzGm5NhFxRZ8JxiIikpi8Hk2+ExE3OB+Mh2anxLsEERHphCbfiYgrnA/GN106Nt4liIhIJzT5TkRc4XwwPnfEgHiXICIinfB4DEFNvhMRBzgfjEcOTG1+HNRXdSIiCUcjxiLiCueD8dDsk8H4J8/viGMlIiISjUeT70TEEc4H45YefnFXvEsQEZE2vJp8JyKO6FPBWK0UIiKJx6cRYxFxRJ8IxuPy0uNdgohIzBljFhhjthtjiowxi6Kcv80Ys8UYs8EYs8wYM7rFuY8bY3ZGfn28N+v0eAyARo1FJOH1iWB85ZT8eJcgIhJTxhgv8BBwDTAVuNEYM7XNZeuAWdbac4GngPsj9w4Evg3MAWYD3zbG5PRWrb5IMNYEPBFJdH0iGLdUVlUf7xJERGJhNlBkrS221jYATwALW15grV1hra2JPF0FjIg8vhp43lp7zFp7HHgeWNBbhTaNGKvdTUQSXZ8Ixg3BUPNjBWMR6SeGA/taPC+JHOvITcCzp3nvGfEpGIuII3zxLqAnzBh5cpOPu/69hT995sI4ViMiEhMmyrGoydMY8xFgFjCvO/caY24GbgbIz8+nsLCw20VWVVXxdlkxAIUrXyI9KdpbJ56qqqrT+v3Gm+qOPVdrV93R9YlgPO+svObHr+46GsdKRERipgQY2eL5CKC07UXGmPnAN4F51tr6FvcWtLm3sO291tpHgUcBZs2aZQsKCtpeckqFhYWclTMatm3h4osvISfd3+3XiIfCwkJO5/cbb6o79lytXXVH1ydaKQakufFBKyLSg1YDE40xY40xfuAG4OmWFxhjzgMeAa631h5ucWopcJUxJicy6e6qyLFe0dRKoSXbRCTR9YkRYxGR/sZaGzDG3EI40HqB31lrNxtj7gLWWGufBh4AMoC/GmMA9lprr7fWHjPGfI9wuAa4y1p7rLdqbV6uTatSiEiC65PB+GhVPYMykuNdhohIr7LWLgYWtzl2Z4vH8zu593fA73qvupM0+U5EXNEnWinauv2pDfEuQUREIjxGwVhE3NBngvF7zx/R/HjZtsOdXCkiIrHk1YixiDiizwTj2xdMincJIiIShVeT70TEEX0mGOdnpbR6fs53em2CtYiIdINXk+9ExBF9Jhi3VVkXiHcJIiKCJt+JiDv6bDAWEZHEoMl3IuIKBWMREelVfl/4R019IBTnSkREOtengvELt82LdwkiItJGapIXgLrGYJwrERHpXJ8KxhMGZ7R6Xl7TEKdKRESkSao/HIxrGxSMRSSx9algDDAuN7358cy7X4hjJSIiApDWFIw1YiwiCa7PBeMvXjGh+bEmeoiIxF9KkkaMRcQNfS4YX3fOsFbPt5RWxKkSERGBkz3GGjEWkUTX54Jx0+znJtf+7KU4VSIiIgBpfh8ANRoxFpEE1+eCcTSaCS0iEj/JkQELjRiLSKLrF8H44cJd8S5BRKTf8ngMqUleDVKISMLrF8H4wWU7412CiEi/lur3UtMQiHcZIiKd6pPB+IH3nRvvEkREpIXUJC+1Ddr5TkQSW58Mxu+fNbLdsS/8cW0cKhEREQiPGNc2asRYRBJbnwzG0H576MUbD8apEhERCY8Yq8dYRBJbnw3GPo+JdwkiIhKRmuTVqhQikvD6bDAekp3S7pi1lvKahjhUIyLSv6X6NWIsIomvzwbjpi1IW3r4xV3MuOt59h2riUNFIiL9l0aMRcQFfTYYR9O0nvGBE3VxrkREpH9J83u1852IJLwuBWNjzAJjzHZjTJExZlEn173PGGONMbN6rsSeU1kXnhFt1H4sIhJTKX5t8CEiie+UwdgY4wUeAq4BpgI3GmOmRrkuE/gS8HpPF9nTNC9PRCS2tCqFiLigKyPGs4Eia22xtbYBeAJYGOW67wH3AwnTp7D01suiHjcaMhYRiak0v5eaxiDW2niXIiLSoa4E4+HAvhbPSyLHmhljzgNGWmv/04O1nbFJQzKjHi86XBXjSkRE+reUJC/WQn1Au9+JSOLydeGaaMOrzf/kN8Z4gJ8AnzjlCxlzM3AzQH5+PoWFhV0qsqWqqqpu3ffl85N58M36Vsduf2oDg6t2dfu9z0R3604ELtYMqjvWXKzbxZpdlxpZKai2IRh11SARkUTQlWBcArTcY3kEUNrieSYwDSiMtCgMAZ42xlxvrV3T8oWstY8CjwLMmjXLFhQUdLvgwsJCunNfAfDgm8+0O37hJXNj+uHc3boTgYs1g+qONRfrdrFm16X5I8G4MUhOnGsREelIV1opVgMTjTFjjTF+4Abg6aaT1toT1tpca+0Ya+0YYBXQLhTHU7Rd8Bb8dCUN+kpPRCQmUlsEYxGRRHXKYGytDQC3AEuBrcCT1trNxpi7jDHX93aBPeELBePbHdt9tIZp31kah2pERPqflq0UIiKJqiutFFhrFwOL2xy7s4NrC868rNjQiLGISGxoxFhEXNA/dr7rZHm2/eW1MSxERKR/0oixiLigXwTjaD3GTS65b7l2YxIR6WVNI8baFlpEElm/CMafnjuW988c0eH5yd9aws+X7YxhRSIi/UvTiLEGIkQkkfWLYJzm9/HA+6fz2Ccv6PCaHz2/g5qGQAyrEhHpP9L84SktGjEWkUTWL4Jxk4JJgzs9v6W0IkaViIj0L809xhoxFpEE1q+C8al0MkdPRETOQIo//ONGrRQiksj6XTDOTk3q8Nx7H36NL/55HQDlNQ0s3XwwVmWJiPRpfq8Hr8eoZU1EElq/C8Yvfq2g0/P/fquU2oYgn/39Wj77+7UcqayPTWEiIn2YMYbUJC+1DVo/XkQSV5c2+OhLMlM6HjFuMuXOJc2PG4P6EBcR6QkpSV5qGzViLCKJq9+NGHs9hpmjc7p8/bKth3qxGhGR/iPN79UGHyKS0PpdMAa44YKRXb72W//azOKNBzhwQjvkiYicidQkr1alEJGE1i+D8ftndT0YA3zhj29y0b3LOVRR10sViYj0fal+r9YxFpGE1i+D8em64kcvxrsEERFnpSZ5tVybiCQ0BeNuqKoPTxrZdaSKK3/8IserG+JckYiIOzRiLCKJrt8G40c+OpOFM4ZxyYRB3b73oeVF7DxcxeOv7SGgVStERLok1a8eYxFJbP1uubYmV589hKvPHsLRqnpm3v1Cl+/bWHKCv6/bD8BPXtjB8ZoGvnP92b1VpohIn5Ga5KVOI8YiksD67Yhxk0EZyd26/p2/eLnV8xd3HOnJckRE+qzUJC81GjEWkQTW74PxmbLWcriiTitWiIicgtYxFpFEp2AMDMlKOe17dx+tYfY9y5hzzzIq6hp7sCoRkb4lJclLfSBEMGTjXYqISFQKxsCiayb3yOts2n+i+fHijQcYs+gZ/rOhtMPrH3xhJx/97es98t4iIokuze8F0JJtIpKwFIyBd503nO13L2DB2UPO6HU+9OvXufHRVQD8/c0SAG7507oOr//JCzt4aWfZGb2niIgrUiPBWCtTiEiiUjCOSPZ5+dEHpp/x67xWfJRlWw+1Ovbm3uOsPRSgrjHIR3/7Ogt+urLV+er6AIfVoywifVxKUiQYq89YRBJUv12uLZr05J7547jp/9a0ev6eX74KwAsHX2XrgYp217/roVfYebiK3fdd1yPvLyKSiNI0YiwiCU4jxm28+LWCXnvtaKEYYOfhKgB+sGRbu3Mrth1mf3ltr9UkIhIrqRoxFpEEp2DcxuhB6ey659q4vPfDhbvYXVZNec3JraY/+dhqrn3wpbjUIyLSk5qCsbaFFpFEpWAchddj4vbeBT8sZP6PX2x17EStloETEfelalUKEUlw6jHuwI2zRxEIhvjr2pJeef2O2ioAyqoa2HGokv+81fFSbyIirmkKxhoxFpFEpWDcgXvfcw4Ady2cxpQ7l/T4619zivaIDzzyGuU1GikWkb4jLSn8I0eT70QkUamV4hRS/V6GD0iN+ft2FIoPVdSx92hNjKsRETlzKf7wjxwFYxFJVArGXbBwxrB4l9Bszj3LuOyBFWf8OqXltby593gPVCQi8WKMWWCM2W6MKTLGLIpy/jJjzJvGmIAx5n1tzgWNMesjv56ORb0nV6UIxOLtRES6Ta0UXWDjXQAwZtEzrZ6v3XOcSUMy+e1Lb5OTnsTHLhrTrde75AfLsRatnSziKGOMF3gIuBIoAVYbY5621m5pcdle4BPAV6O8RK21dkavF9pCmj/8I0c9xiKSqBSMu8C2SMabvns1tz/1Fos3HoxfQcB7H3611fO2wfjXK4sJhCyfLxgf9X6bCGlfRM7EbKDIWlsMYIx5AlgINAdja+3uyLlQPApsy+sxZCb7NH9CRBKWWim6YHxeOgBXn51PRrKPX354Zpwrau9EbSN/f7OEYMjynac38/3FW6NuGCIifcZwYF+L5yWRY12VYoxZY4xZZYx5V8+W1rG8zGSOVNbH6u1ERLpFI8Zd8L6ZI5iYn8mMkQPiXUqHpn/3OQBue/KtqOdDIcvH//cNbr5sHHMn5jUf/9pf3+KTl4xl6rCsmNQpIj0m2oLr3fkuaJS1ttQYMw5YbozZaK3d1eoNjLkZuBkgPz+fwsLCbhdZVVXV6r6kYC07S2pP67ViqW3drlDdsedq7ao7OgXjLjA/kzJVAAAgAElEQVTGtAvFy74yjyt+9GIHdySOf79VyjunD+NARR0v7Sxj/d5yNn736ubzf11bwl/XlqjXWMQ9JcDIFs9HAF1e/NxaWxr5b7ExphA4D9jV5ppHgUcBZs2aZQsKCrpdZGFhIS3ve6r0TTbtP8HpvFYsta3bFao79lytXXVHp1aK0zR2UHrz47zM5DhW0rkv/nkdb+0r55L7lgNQWR9g0h3PxrkqEekBq4GJxpixxhg/cAPQpdUljDE5xpjkyONc4BJa9Cb3JrVSiEgiUzA+TR6P4bPzxgFwwZic5uPbvrcgXiV1aOFDr7R6Xh+IPg9nz9FqvvP0ZkIhSzAU/ka2tiFIQyBEIJgQc3dEJMJaGwBuAZYCW4EnrbWbjTF3GWOuBzDGXGCMKQHeDzxijNkcuX0KsMYY8xawArivzWoWvWZwZgrVDUGq67Vkm4gkHrVSnIGvXzOFq6bmM2VoVvMqFSmRdTpd9IU/vsnm0gpe3HGEt8uqwweXhHf983kMRfdcG8fqRKQta+1iYHGbY3e2eLyacItF2/teBc7p9QKjaPqGrayqnvRk/QgSkcSiEeMzNHP0QNL8PpbcOpe3vn0VAM//z2Vxrur0HI58vdkcilsIhLo2p2fZ1kPamU9EOtQUjA+rnUJEEpCCcQ+ZPCSL7NQkACbmZ8a5mu4bs+iZU/b9ffJ/36A+0PnC/Df93xou/1FhD1YmIn3J4EgwVp+xiCQiBeNelpuRuBPzumvF9iNMumMJ9YEgoZClqoMewWAXR5dFpP/JUzAWkQSmYNxL/vyZC3nwhhn8+4uXxLuUHrfzUBXT73qOad9eyona6DtYjVn0DK/uKotxZSKS6HLS/Hg9hsOVdfEuRUSkHQXjXnLR+EEsnDGcodmpp7zWtTWEP/XYairrwqPF3/rnpnbbUzf5v1d3x7AqEXGB12MYlO7XiLGIJCQF4wQxcuCpA3SiaDlp5um3Slm75zg/em47+8trW123dPOhWJcmIg4YnKW1jEUkMSkYx8CogWlcMszHe84fDsDAdH/zuTuumwLA4i/NjUttPeXny4uaNxFpqbymgRXbD/P9Z2KyRKqIOCAvI5kjVQrGIpJ4tIhkDKy8/XIKCwuZe9l0vv+uc0j1exmz6BkA3jczvMSo39c3/41y9zNbeWptCQAeY/j6tVPiXJGIxFteZjKbSyviXYaISDtdSmPGmAXGmO3GmCJjzKIo528zxmwxxmwwxiwzxozu+VLd5/UYUv0nNwCZOjSLAWnh0WOfp/P/Kb74jgm9WltvaWixy94jK4uZ8q0lzL1/OXWNnS/7JiJ91+DMFI5WN2gFGxFJOKcMxsYYL/AQcA0wFbjRGDO1zWXrgFnW2nOBp4D7e7rQvmbH3dfw9C0nV6zwegy777uO7Xe331L6rPwMbrvyrFiW12OMaf28tjHIvmO13PDoqqjXbztYgbX6YSnSl+VlJhMMWY7XNMS7FBGRVroyYjwbKLLWFltrG4AngIUtL7DWrrDWNm13toooW5BKa36fB5+3/R9/sq/1ltLvmzmC5/5nHqZtwnTEv9aXRj2+fl95u2PLth5iwU9f4u9v7u/tskQkjrSWsYgkqq4E4+HAvhbPSyLHOnIT8OyZFCUnDc1OiXr8nOHZ3P/ec5k6NCvq+SFZ0e9LNEs2HWRLpNdw28FKAO59dms8SxKRXjZY20KLSILqyuS7aEOVUb/rNsZ8BJgFzOvg/M3AzQD5+fkUFhZ2rcoWqqqqTuu+eOtO3Y8tSOepHQ38p7iR3Xv2UFh4AIBzc71sKAv35g5PqmFw9S6mZTaw5UD71/j0VLg7erdCwph/37MUlYd7kM8e5GHz0fDjsqqGVn9WjSFLdYNly7EQFw+L/le2IWjxe8N/VfvD35FE4mLdLtbcl2jEWEQSVVeCcQkwssXzEUC778eNMfOBbwLzrLVRP+2stY8CjwLMmjXLFhQUdLdeCgsLOZ374q27dfuGl/Gf4td572UzKJg0GIA5Fwf53StvU98Y5HMF40nz+ziYtpcnd2xsd//Fsy+AVS/1VPm9oikUA82huMmk8+Y0b47StIIHwAfmz2HC4Mzm579YvpNnNh5k64EKnrj5Qi4cN8iJvyNbSiuYMDij1WokLtQdjYt1u1hzX6JgLCKJqiutFKuBicaYscYYP3AD8HTLC4wx5wGPANdbaw/3fJn9z6UTc1l/55XNoRgg1e/lvy+fwG1XTSLNH/43zftnjeSed5/T7v5xeencOHtUzOrtaRfduxxrbatQDDD/xytZsulg8/MfPreDrQfCrRhvvH2sw9c7cKI2YSb1lRyv4dqfvcR3/7053qWIxEWa30dGsk/bQotIwjllMLbWBoBbgKXAVuBJa+1mY8xdxpjrI5c9AGQAfzXGrDfGPN3By0k3NC3l1hmvx/ChOe0DcEqSl5y0pN4oK2bGfn1x1ON/e7OEny/byZOr97U63lHuLT5SxUX3LudXLxb3dImnpbymEYB1e9tPQBTpL/IytfudiCSeLm3wYa1dDCxuc+zOFo/n93Bd0k2jBqZRHwhy+aTBnD8qB4BgF0ZIRw5MZd+x2lNel0ie33KI57e03266IRjkRCR0ttS0VfXLRUf4fMH45uN7j9ZQXtvAuSMG9F6xIhJVXoaCsYgkHu1810esvP3ydsdyOhhxnj5yAG/tK+cfX7iYIdkpXHRv+62cXfTQil08tGIXjy1IZ+2eY6QkeTl7WDbeyFJ3odZtzFz2wAoAfv2xWcydmEtKkrftS4pILxmRk8rLRWXxLkNEpBUF4z7spkvHMjA9HI79Xg8PrShi5+Gq5l5bjzEMzU7ljW9cwex7ljXf99WrzuKHz+2IS8094V9FDfxjyWsAfHDWyXmjHY2gf+bxNXx4zii+H6VXuzc5ujS1SI84Z0Q2f1+3n4Mn6hjSwbKUIiKxpmDchyV5PXygRTBcMG0IJ2ob+dmynWwoOdE8ojy4zZrHt7xjIhdPyKVk63q+tKIG1/yj6GQ7xV/WnOxDDnWy/ewfX98b82As0p9NHxluYVq/r5wF2UPiXI2ISFhXVqWQPiIlyUt+Vgp3vnMq/77lUkYNSuvw2vNH5ZCVbLjjuikxrLB3rdlznDGLnuGf67q2s97hijp++/LbrVazsNZytEp9kSJnaurQLHwew4YSTUIVkcShEeN+KNnn5ZwR2a2OGRN9VYebLh3L0eoGHi7cxXvOH87VZw9h6eaDWAv/6GLATDS3/mU9e452PhK+52g18x4oBGDeWXlMGJwBwGOv7ua7/97C8q/MY1xexmm9f4KsGicSVylJXiYPzeQtBWMRSSAaMRYA3r73uqjHjTHMiHzlmZuRzNVnD+HHH5jBjz8wneVfibrBoRN+8kL7Huq/v1nC4YrwuqpNoRgg0GLW3ss7w5OFdh2p7t0CRfqB6SMGsKHkRKdtTiIisaRgLM1uuXxC1ONXTc3nB+89h9uuPKv5mDHmtEdME9VtT77F7HuWsaW0otXxBT89uYNg0051n3l8Dcu3tV8y7jtPb2bKt5b0bqEifcT0EQOorAvw9lH9Q1NEEoOCsTT76tWT2H1f+5FjYwwfvGDUKZcz++y8cWz4zlW88c0reOf0Yb1VZq+79mftt9K+d/FWGgIh3i47+QP8U4+taXfdY6/uprYx2KX30aoU0t81TcB7a5/aKUQkMSgYyxmZGOm9feLmC/n6NVPISklicGYKN84eeYo73fLIymIee/Vtth2sbHW8vKYh6vX7y2sJBENRz4lI2ITBGaT5vWwoORHvUkREAAVjOUNNo8hp/q5vjnHF5MG9VU6vumfxtnbHZtz1PAdPhPuS7/zXpubjl9y3nLuf2Rr1dWoaAr1ToIhjvB7DtOHZrNeIsYgkCAVjOSO3vCPclzwmN73V8Zmjc5g/ZTCfvWwcP3z/dAC+/+5pvHDbvOavT/uKC+9dxp9e38vjr+1pdXzljiNRr//go6sA2LS/Iup5kf5kxsgBbCmtoCGgb1hEJP60XJuckavPHhK1LznZ5+U3H78ACK/9O3xAKheOG4gxhg/NGcXf3iw55ZJpLvnGPza2O1ZcVk1dY7B5VL0+EOSGSCg+XaXltaQmeclJj77dt4hrzh6WRUMwRHFZFZOHZMW7HBHp5zRiLL3OGMNF4wdhIrPNcjOSefFrl7e6ZvKQzHiU1utue3J98+NlWw+zbm/nXxlvLj3Bxfcuo7S8Nur5i+9bzoX3Lot6TsRF4yOr2xRrCUQRSQAKxpIQltx6GRu/cxVvffuqVscLJuXFqaKesXjjQR5aUcTGkhN84Y9vdnjd+n3lfO3FGq772cuUnqjj1r+s7/Daen3lLH3IuLxwG9auw1VxrkRERMFY4uj/LZjc6nlmShLZqUnc955zmo9989op7Lj7Gj572TiGZKXEusQe8cDS7bzzFy9HPTdm0TO87+FXeWDpNo7UntzkoKK2MVblicRVmt/HsOwUiss0Yiwi8adgLHHz+YLxUY/fMHsUi780l+unD2Nsbjp+n4evXzuFVd+4IsYVxsaaPcc5VFHf6ljbZeFKjte0WkO5SShkOVGjEC1uG5eXQfERjRiLSPwpGEtc/e8nLoi6497UYVn87Mbz8Hlb/xX99cdmcemE3HbXP3jDjF6rMRaKonyNHAxZrLWEQpZLf7CCy39Y2O6any8vYvpdz3Gksr7dORFXjMtLp/hINdZqa2gRiS8FY4mryycP5qtXT+ry9VdOzef2BSevn5QfnrS3cMbw5s1G+orx31jMj57bwXNbDrY7V3S4ijGLnuEnL+wA4FBFXYevs6GknH3Hur8CyKK/bWDZ1vbbXjdp1AYm0kPG5aZTWR/QP/BEJO4UjMU5EyIBePSgNP7zpUvZ9r0FADz75bms+9aVfPG85HiW16N+saKIz/2h/aS9+T9+sdXz4zUNjFn0DDdGWQ7u+l+8wtz7VzQ/X7LpIGMWPdO8MUlLj73ydvPI9BOr93HT/7Xf9hrgpZ1HmPjNZ1m753h3fjsiUY2P/H96l1amEJE4UzAW56T5fey+7zpe/NrlJHk9zesE+7wectL9zMz38cJtlwHwrhnD+MqVZ8Wz3Jj46G/fAOC14qPUNQY5Vt1AZV0jG6NstfvnN/YCsPVABbvLqnmxxUYk3/n3lqi9zG29vLMMgNW7j/VE+dLPjWtasq1MfcYiEl/a4EP6pAmDM1ttPPKj53e0Or/sK/O44kcvtr2tT7jv2W089urudsf/uW4/7zpvOKFIH6fHYyiIjA7vvu86gqGT/Z11jcHO3yS8JDVqCZWeMDQrhZQkD7sOa8RYpC+obQjyxu5jnKhtZFJ+JuPy0kmKzBkKBEPsPlrDidoGvB4PgWCI/eW1HKqow1pI8npITvKQ4vPi8xoCQUvIWvIykxmRk0p9sHd/8CgYS7+w+EtzufZnLwHw71suZXxeBrvvu45j1Q2UltfywNLtrUZOXRYtFAPc+pf1vOu84c1htmX/8IRvLOacEdnNzz//h7WdvoeJJGOLkrGcOY/HMDY3QyPGIo6oqGtkw74TrN59jOKyamrqA9Q2BmkIhKgLBNlxsIqGFvNQjIEBqUlkpiRxsKLujLaAnzvcx9W9uEiVgrH0C1OHZbHk1rn89PmdTB56cpe9gel+Bqb7+d7Cadzxr00UH6niVx+ZyeQhmUz45rNxrLh3HKqoax4Zfvy1Pc3HAyHbale+FdtP/iPhlaIyLmmzEkhkE0PuX7KdGXleCgrgcEUdWalJza0t3dUYDOExBq/HnNb94rbxeelsiNL6IyLRBYIhth2s5HhNAx5jyEj2cVZ+Jqn+k5/B1trIaGw9IWuprg+w5UAFWw9UUnKgjj/sWU1lXYBDFXU0Bi2XT87jmmlD8XkMhyvr2XO0mh2HqjhwopZgyNIYDL/eseoGADwGRg1MIyPFR2qSl+QkDxkpPi6+JJdLJuSSl5HMjkOVFJdVc7y6gfLaRhZkp3BWfia5GX5C1uIxhmEDUhmanYLHGBqDIeoDIeoagzQGQyR5PRgMhyrr2H+8lsO7t/Xqn6uCsfQbk4dk8auPzox6btSgNB7/1OxWx3bfdx0LH3qFt/Z1vo2zS+bc0/3tpD/8m9f51UfOZ8G0oYRCli0HKmgZXdcfCbddzL5nGXMn5vL7m+acVm0Tv/nsGd0vbhuXl8HijQeoawye9j+uRPqq4iNV/HN9Kev2hic8NwRCbNp/guqG1m1vHgOjB6WT5vfiMYa9x2o4EWXDqBE5qXgClvryOtKTvUwbnk1jMMRTa0v4w6q9ra4dPiCVkQNTSfZ58XoM04ZnM3pQGlOGZnH+qAFkpiR1WvvUYVln+LsPGzUojQvGQOGJnT3yeh1RMBbpRHKkJ+pPn57Dsm2H+e3Lb5OZ7KOyPsBrX38HF927PM4Vxsbn/vAmny8YT7rfyw+f28GF4wa2Or8mMgnvpZ1lWGu579lt3HzZOAZlhFcIKTpcyaceW8On545l/pR8hg1Ijfo+L0Um9Un/Mz4vnZCFPUdrmDQk89Q3iCQway1HKuvJzUjG4zEEgiGWbj7EhpJyvJ7wN2MhawlZyEzxMSjdz6D0ZHIzk0nzeyk5XsPusho2l1bwVkk5RYerMAbOHpZFkteDxxjeff5wZo8dxLDsFEIWjlXXs+VAJbsOV1EfCBEMhZg2PJupw7IYkZOKz2NI9nmZlJ9JdloShYWFFBTMbVV3dX2AVcVH8fs8DM5MYUROKunJ/Ssq9q/frUg3pSV7I//1Nffmfnn+RD49d1z4uN9LTeRf7L/6yEw+F+nN3X3fdYxZ9EzsC+5FDxfuan68qrj1ahTv+9VrzY8//JvXeXXXUR5ZWUxuhp81d1zJr14sZu+xGu7812YeebGYVxa9I2Z1AxyrbmDljiO867zhMX1f6bpxuZGVKY5UKRiLs8qq6nl+yyH++PoeNu2vYGC6nwvHDWT93nJKT9SR5A1/3xYIhVsIgFYTn9vKzUhm+ohsbrhgJP917jCGZKd0+v4Lpg09o/rTk31cMSX/jF7DdQrGIp344fun89TaEqaPyCYj2ctfVu/lmnNOfvBsuWsBH3zkNV5/+xhZqT5+9ZHorRr9yau7jjY/Lqtq4I23W4fo/eW1rZ4fr26guAtLxIVClk8+tpqbLh3LZWfldaumz/5+Dat3H2fOuIEMzY4+Wi3xNWFwBqlJXl7YerjV/8dEYslaS0VtgKPV9VTWBUjze0lL9lHbEOREbQObSytYuaOM4iNVDBuQyvABqezZX8fvit+g6FAlpZH14SflZ/K1qyex63AVrxUfZcygdL67cBrvmDy43TyKmoYAR6saOFrdQFllPdUNAUbkpDJqYDq5GX6M0byLWFIwFulEbkYyn5s3HggvAbf5rgXtrmn6t77BsGDakObj7z5vOP9Yt7/d9R+cNZK/rNnXK/Umou/+ezObSytaHbt38VZunX8WqX4v533v+S69Tk1jkBd3HGH17mNsifK/Q2l5LT6vYXBm6xGV2oYgq3eH+/K0vFziSvV7+cCsEfzpjb3cvmAS+Vmdj4yJnClrLUerG3h200GWbDrA7rIajlTWt1pNIZrRg9KYMiSLAxV1LNt2GBMMMczbwMwxA/nUiGxmjRnI9BHZXQ60aX4faQN9jByY1hO/LTlDCsYiZ+jO/5rKN/+xkRkjB7Q6/pm54/jHuv3c955z+MGSbZw/KofffuICgHbBeGh2Cgei7ETXF7QNxQCPrCzmkZXFrdaabjJm0TM8/qnZTB6ayezvL+OPn57DJRNym9df9rb5YfPoyl1cdlYeC34aXo6v7Wv+de3JP2ufVrxIaJ+6dCyPr9rDY6/u5v8tmBzvcsQxoZDljd3H2HqggsOV9ZTXNGKtpSEYorS8lpLjtVTUNlIfCNEYDNGyg2HC4AzmjB3I4KwUcjP85GYkk5nio6YhSHV9gFS/lwFpfsYOSmfUoNYBNtyre2mMf7fSWxSMRc7QtOHZ/OuW9h+KU4dlNYe0G2aPinrvS7dfTm1jkLPyM3nnD5ewsewUG2v0MY+/tjvq8Y/97g1++eHzgXCI/vBvXm8+13YQ5p7F27hncdeW76k91cYlElejB6Wz4Owh/HHVHm65fEK/m/QjXRcMWbYdrGD7wUpO1DZysKKOZzYcoOR4uFXL5zEMSEvC6zH4PB7ys5KZOTqHnDQ/yT4PSV4PPq8hNcnLvEl5TMrPVMuCAArGInHxxjeuYH95bauvzr4yK4WCggJeLz7KBx9d1Xz8yqn5rCo+SmVdIB6l9qo7/7W5w3NNI8Qr22y8UtHiz8FG6Y1oCITw+zwcrapn64HKVu0TDxfu4r73nnuGVUtv+vTccTy76SBPrtnHJy8ZG+9yJA6CIUvJ8RpKjtdSVlXPseoG6hrD69ruL69ld1k1Ww9UtFqqzBi4ZHwuX7t6EpdOyCUnzY9H3xDJaVAwFomDwVkpDO6gh3LOuEH8vwWT+cGSbTxx84VcOG4Q1lrqAyGeXl/K7X/b0O6evMxkjlTW93bZMfXnN/Z2eK6zFT/OuuNZlt56GVf/dCUQbnVpcsqtriXuZo7OYfaYgfz0hZ3Mn5Kvvss+rKyqnp2HqiivaeDlPY0sfuotNu2vYNeR8HJj0QzOTGZsbjrvnTmCmaNzmDY8m4FpfjJSfM1bDoucCQVjkQT0+YLxfL5gfPNzYwwpSV6unzGMn7ywg+8tnMb8qfm8ufc4y7ce5qtXT+owLBbfcy37y2uZe/+KWJXfI14pOnrqizqw4MGVzY/v+s+W5sd7jtXwy8Ii7l9Szc9zShmcmcyccYPOqE7peQ+8/1ze+fOX+dwf1vK3z1+sDT8cVlUf4FhVA/WBIJX1AbYdqGRDSTmrdx9j15HWq9EMTD/M2cOyuGTCaCYMzmDkwDQGZ6YwKN1Pqt+L3+vRKLD0OgVjEYekJHl57esnN4k/f1QO54/KAWDxl+byhT+uZffRmlb3eDym3426dbT6xLq95c1bX3/xz+sAePvea9VbmGBGD0rnpzfM4FOPreEbf9/IA++frq3CE5y1lvKaRorLqnmlqIzC7YfZcaiKqvr2LWAD0pI4b+QA3j9rJOcMz2Zgup/tb61h4dWX6/+LEncKxiJ9xNRhWRR+7XJ2HqokaC3XPPgSP7/xvObzdy08mzv/tZnrzhnKlVPzufUv6wHYctfVTL1zabzKjrsn1+zjgxdEnxwp8fOOyfncOn8iP31hJ/uO1/DjD8zod//AS1TWWtbuOc5fVu9jc2kFFXWNlNc0NodgY+DcEQN438wRDMkOj/imJHlJTfJyVn4mIwemtgvAh7Z7FIolISgYi/QxE/PDu4a9fW/rZcs+dtEYPnbRmObnTcE4ze/rV9tbt1W4/YiCcYL68hUTGT0ojTv/uZlrHnyJRddM5kOzR+nr9F5SWddIaXkdFovXGCrrA5RV1nMissRZRV0jWw9U8ta+cvYeqyHd72X22IFMGpJJdmoSI3JSGTkwjZmjc8iNbAcv4hoFY5F+avlX5jU/Hpqdyva7F2AwfOQ3r/PG7mP85eYLOVRZz6b9J1i54wjbDlay9NbLyElLory2kTGD0jnrjmejvvblk/JYsf1I1HOJprPtWCW+jDG8+7wRzBo9kNuf2sAd/9zE394s4fPzxjNteDZDs1M0yngGrLWs21fOH1btYeWOMsqqTj2Bd/iAVKYOy+K/Lx/Pf507TEvqSZ+jv9Ei/dS4vIxWz5N94QlOf/zMHAJBS6o//Pz66cP4xrVTWl0bbUWNwq8WUPDDQgBmjMwh1e9l8caDra6547opPPjCTiqj9B3GS9GRqniXIKcwcmAaf/rMHP6xbj/ff2YrN/9+LRDeGOd9M0fwgVkj1WbRQiAYoqyqgcbIDm5V9QHKqurZe6yGdXvL2bT/BFX1AWobghytbiDd7+Wqs4cwMT+DkTlp+DyGxpAlM9lHbkYyA9KSSE7ykOb3kaEgLH2c/oaLSCtJXg/dXQTgxtmjGJObzhvfvILP/+FNPjRnFHmZya1Wylg4YxifnjuOT88dx4ETtQnTutG0IYAkNmMM7zl/BNdMG8qWAxVsKT3B8m2H+cWKIn6xoohLxufygQtGctXU/D69ikVDIMT6feVsLj3B9oOV5KT7mTYsm/pAkOXbDvPy9hpOLH22wwmoA9P9TB+RTU56eKOLacOzWThjuAKvSIT+nyAip23FVwtYunIVn3vPOQAMzkzhb5+/OOq1N7To4x2ancqvPzaLzzy+BghPAFy75zgXjhvEnqPVzP/xynb3L//KPH790tudrm98OqYMyezR15Peler3MnN0DjNH5/DRi8ZQWl7Lk2v28dc1JXzpz+tI83uZd1Yel07MZfTAdMblpTNsQGq8y24nGLLsO1ZDYzCEMXDwRD3bDlZQWl5HyFoagyH2l9dGenl9jMtLp74xxMtFZc2T3HLSkqisCxCItAPlZiQzZZCHC6aMIz8rGX9kXd80v4+8zGSGZqcwIqf9xDcROUnBWERO29jcdCYP7Hh0bvN3r+ava/bx8YvHtPthfOXUfN74xhVkpSaRkuRl7sQ8ACYMPhlUX//GFeS3aNv43sKzezwYXz55cI++nsTWsAGp3Dr/LL70jom8VnyUxRsP8NyWQzy76WQbz+yxA/nYRaOZOzGP7NSkmNZnrWVzaQXPbzlEcVk1NZG2hu2HKqlrbL+JRZrfi89j8Hk9DM1OYVJ+JlX1AdbsPg7AO6cPo2BSHueNHMDgrBTqA0G2H6zEYwxTh2axcuWLFBScFdPfo0hfomAsIr0mPdnHJzrZ1rej3f/eOX0YmSm+VqEYwNdmZ6vvv3sa10wbyto9xzl/1ABm3v1C87khWSkcrKjrtL7bF0ziCwUTTvXbEAd4PIZLJuRyyYRcvrdwGvvLa9l3vIb1+8r50+t7ueVP4XWr8zKTGZ+XzoTBGYwZlE5uRjI56X7GDkpnRE5q1BUvAsEQJ2obqa4PUtsYpD4QJJL8cBMAAAsJSURBVBCyBIKWusYgdY1BknweMpN9lFXVs6r4GBtKyjlR28ix6gaO1zTiMTBqYBrpyT4GpCXx4TmjmTQkkzS/l5CFQel+Jg3J7PZqDsk+L+eOGNAjf4YiomAsIgmo5frLbe2+7zoq6hrZcbCSWWMGAuHR56ZzNQ0BfvTcDr561STKaxtIT/aRlZJEfSDIpDuWAPDolWlcfOlc9VX2UU2b2owcmMbF43P57GXjWVV8lE37T1B0uIqiI1X8a30plXWtJ4Gm+73kZ6eQ4vNSXV2LfWMF5TUNVNR1b7JoapKXc0dkM3loFlkpSZw/agDvmDyYQVrCTCTh6aeCiDgnKyWpORS3leb38a3/mgpAqv9kb2myz8uXr5jI/Cn5HC1ap1Dcj3hbjCY3sdY2j+iWVTWw60gV2w9WUlZVT11jkMOBasYNH0B2ahI56X5y0vxkJPuatyb2eQ1ejyE1yUtKkpeGYIiqugAZKT6mDcvG7/N0UpGIJCr9ZBCRfuN/rgz3XhYWxbmQHmKMWQA8CHiB31hr72tz/jLgp8C5wA3W2qdanPs4cEfk6d3W2v+LTdWJwRjDgDQ/A9L8jMsL9yG3VFhYSEFBx99ciEjfpH/Siog4yBjjBR4CrgGmAjcaY6a2uWwv8AngT23uHQh8G5gDzAa+bYzJ6e2aRUQSnYKxiIibZgNF1tpia20D8ASwsOUF1trd1toNQNvlD64GnrfWHrPWHgeeBxbEomgRkUTWpWBsjFlgjNlujCkyxiyKcj7ZGPOXyPnXjTFjerpQERFpZTiwr8Xzksix3r5XRKTPOmWPcYuv664k/OG52hjztLV2S4vLbgKOW2snGGNu+P/t3W2IHVcdx/Hvj6xJtWqTVA1rEkwCQYhvbFwxUZHFh/SB2iJETFtotEpBEXx4oVkCYisIUZFSKqbBRqTEWo1tXUJLkLb7Nk2KmqY222yttttWkxCMRhEaPL6Yc9fp5d67M/fxHP19YNi5Z2Z2f/c/d//M3pm5C+wBPjWIwGZmBkCr/9LQ5v+ddbetpFuBWwFWrVrFzMxM5XANFy5c6Gq7UXPu4co1N+Sb3blbq3Lz3cLpOgBJjdN15QPj64FvxvmDwF2SFEK7f0ppZmY9mgfWlh6vAV6use1k07YzzSuFEPYB+wAmJibC5ORk8yqLKm5iq7/dqDn3cOWaG/LN7tytVTkwbnXK7X3t1gkhXJR0HrgcOFteye8+zIw6Ri05ZgbnHrYcc+eYuYWjwEZJ64GXgB3AjRW3PQx8u3TD3TZgqv8RzczyUuXAuMopt0qn5fzuw+SoY9SSY2Zw7mHLMXeOmZvFNyG+SHGQuwTYH0J4WtLtwLEQwrSk9wIPAiuAj0u6LYTwrhDCOUnfoji4Brg9hHBuJE/EzCwhVQ6Mq5yua6wzL2kMuAxwkzUzG6AQwsPAw01j3yjNH6Xo2a223Q/sH2hAM7PMVPlUioXTdZKWUpyum25aZxrYGee3A4/5+mIzMzMzy8mi7xhXOV0H3APcK2mO4p3iHYMMbWZmZmbWb5X+JXSF03X/Aj7Z32hmZmZmZsPj/3xnZmZmZoYPjM3MzMzMANCo7pGTdAb4UxebvoWmz0fORI65c8wMzj1sOebuNfM7Qghv7VeYHLhnZ8O5hy/X7P9vuSv17ZEdGHdL0rEQwsSoc9SVY+4cM4NzD1uOuXPMnKtca+3cw5Vrbsg3u3O35kspzMzMzMzwgbGZmZmZGZDngfG+UQfoUo65c8wMzj1sOebOMXOucq21cw9Xrrkh3+zO3UJ21xibmZmZmQ1Cju8Ym5mZmZn1XTYHxpKukjQraU7SrgTyrJX0uKRnJD0t6UtxfKWkX0s6Fb+uiOOSdGfMf1zS5tL32hnXPyVp5xCyL5H0G0mH4uP1ko7En3+/pKVxfFl8PBeXryt9j6k4PivpyiFkXi7poKSTseZbM6n1V+Lr44Sk+yRdkmK9Je2XdFrSidJY3+or6T2Snorb3ClJA8z93fg6OS7pQUnLS8ta1rFdf2m3r6yadnVNjWr289RU7ekpqdPTU1Knp484Z196eiK5a/f0noQQkp+AJcBzwAZgKfA7YNOIM40Dm+P8m4BngU3Ad4BdcXwXsCfOXwM8AgjYAhyJ4yuBP8SvK+L8igFn/yrwU+BQfPxzYEec3wt8Ps5/Adgb53cA98f5TXEfLAPWx32zZMCZfwJ8Ls4vBZanXmtgNfA88PpSnT+dYr2BDwGbgROlsb7VF3gC2Bq3eQS4eoC5twFjcX5PKXfLOtKhv7TbV54q7Zvk+naHrLX6eWoTFXt6ShM1enoqU92ePuKsPff0hHLX6uk9Zxj1zqtYqK3A4dLjKWBq1LmaMv4K+BgwC4zHsXFgNs7fDdxQWn82Lr8BuLs0/pr1BpBzDfAo8GHgUPxFOFt60S3UGjgMbI3zY3E9Nde/vN6AMr85NiM1jade69XAixQHimOx3lemWm9gXVMz6kt947KTpfHXrNfv3E3LPgEciPMt60ib/tLpd8NTpf2SfN/ukL1jP09pokZPT2Wq29NTmer29FFPvfb0VHI3LVu0p/f683O5lKLxYmyYj2NJiKe8rwCOAKtCCK8AxK9vi6u1ew7Dfm53AF8D/h0fXw78NYRwscXPX8gWl5+P6w878wbgDPDjeLrwR5IuJfFahxBeAr4HvAC8QlG/J0m/3g39qu/qON88Pgy3ULwTAvVzd/rdsMUl3bfbqdjPU1Knp6eibk9PQhc9PTV1e3qKqvT0nuRyYNzqesQkPk5D0huBXwJfDiH8rdOqLcZCh/G+k3QtcDqE8GSFXJ2WDXt/jFGcWvlhCOEK4B8Up4HaSSJ3vH7reopTPG8HLgWu7pAhidwV1M05kvySdgMXgQONoTY5ksr9PyS7+tXo50nooqenom5PT0IXPT0XObxm6vT0nuRyYDwPrC09XgO8PKIsCyS9jqKJHgghPBCH/yJpPC4fB07H8XbPYZjP7QPAdZL+CPyM4tTbHcBySWMtfv5Ctrj8MuDckDM3csyHEI7ExwcpmmrKtQb4KPB8COFMCOFV4AHg/aRf74Z+1Xc+zjePD4yKG/+uBW4K8RzbIvlajZ+l/b6yxSXZt9up2c9TUbenp6JuT09F3Z6emro9PRk1e3pPcjkwPgpsjHd+LqW4MWl6lIEkCbgHeCaE8P3SomlgZ5zfSXGtWmP85nj35xbgfDyVcRjYJmlF/Gt0WxzruxDCVAhhTQhhHUUNHwsh3AQ8Dmxvk7nxXLbH9UMc36HiUxTWAxspbq4aiBDCn4EXJb0zDn0E+D0J1zp6Adgi6Q3x9dLInXS9S/pS37js75K2xDrcXPpefSfpKuDrwHUhhH82PZ9WdWzZX2Lt2+0rW1xyfbudLvp5Erro6Unooqenom5PT03dnp6ELnp6b0Z1cXXdieKuyWcp7jrcnUCeD1K8ZX8c+G2crqG4vutR4FT8ujKuL+AHMf9TwETpe90CzMXpM0PKP8l/72DeEF9Mc8AvgGVx/JL4eC4u31Dafnd8LrP06RMGFsn7buBYrPdDFJ96kHytgduAk8AJ4F6Ku2eTqzdwH8U1c69S/BX+2X7WF5iINXgOuIumm276nHuO4rqzxu/l3sXqSJv+0m5feaq8f5Lq2x1y1urnKU5U6OkpTXV6ekpTnZ4+4px96emJ5K7d03uZ/J/vzMzMzMzI51IKMzMzM7OB8oGxmZmZmRk+MDYzMzMzA3xgbGZmZmYG+MDYzMzMzAzwgbGZmZmZGeADYzMzMzMzwAfGZmZmZmYA/AedldI+PisvigAAAABJRU5ErkJggg==\n", 108 | "text/plain": [ 109 | "" 110 | ] 111 | }, 112 | "metadata": {}, 113 | "output_type": "display_data" 114 | }, 115 | { 116 | "name": "stdout", 117 | "output_type": "stream", 118 | "text": [ 119 | "Loss 0.04272\n", 120 | "Val Error Rate: 0.11866\n", 121 | "BREAK. There is no improvment for 10000 steps\n", 122 | "Best step: 2100\n", 123 | "Best Val Error Rate: 0.10340\n" 124 | ] 125 | } 126 | ], 127 | "source": [ 128 | "for batch in lib.iterate_minibatches(data.X_train, data.y_train, batch_size=1024, \n", 129 | " shuffle=True, epochs=float('inf')):\n", 130 | " metrics = trainer.train_on_batch(*batch, device=device)\n", 131 | " \n", 132 | " loss_history.append(metrics['loss'])\n", 133 | "\n", 134 | " if trainer.step % report_frequency == 0:\n", 135 | " trainer.save_checkpoint()\n", 136 | " trainer.average_checkpoints(out_tag='avg')\n", 137 | " trainer.load_checkpoint(tag='avg')\n", 138 | " err = trainer.evaluate_classification_error(\n", 139 | " data.X_valid, data.y_valid, device=device, batch_size=1024)\n", 140 | " \n", 141 | " if err < best_val_err:\n", 142 | " best_val_err = err\n", 143 | " best_step = trainer.step\n", 144 | " trainer.save_checkpoint(tag='best')\n", 145 | " \n", 146 | " err_history.append(err)\n", 147 | " trainer.load_checkpoint() # last\n", 148 | " trainer.remove_old_temp_checkpoints()\n", 149 | " \n", 150 | " clear_output(True)\n", 151 | " plt.figure(figsize=[12, 6])\n", 152 | " plt.subplot(1, 2, 1)\n", 153 | " plt.plot(loss_history)\n", 154 | " plt.grid()\n", 155 | " plt.subplot(1,2,2)\n", 156 | " plt.plot(err_history)\n", 157 | " plt.grid()\n", 158 | " plt.show()\n", 159 | " print(\"Loss %.5f\" % (metrics['loss']))\n", 160 | " print(\"Val Error Rate: %0.5f\" % (err))\n", 161 | " \n", 162 | " if trainer.step > best_step + early_stopping_rounds:\n", 163 | " print('BREAK. There is no improvment for {} steps'.format(early_stopping_rounds))\n", 164 | " print(\"Best step: \", best_step)\n", 165 | " print(\"Best Val Error Rate: %0.5f\" % (best_val_err))\n", 166 | " break" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 6, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "Loaded logs/epsilon_node_2layers_2019.08.28_13:04/checkpoint_best.pth\n", 179 | "Best step: 2100\n", 180 | "Test Error rate: 0.10372\n", 181 | "Loaded logs/epsilon_node_2layers_2019.08.28_13:04/checkpoint_temp_12100.pth\n" 182 | ] 183 | }, 184 | { 185 | "data": { 186 | "text/plain": [ 187 | "Trainer(\n", 188 | " (model): DataParallel(\n", 189 | " (module): Sequential(\n", 190 | " (0): DenseBlock(\n", 191 | " (0): ODST(in_features=2000, num_trees=1024, depth=6, tree_dim=3, flatten_output=True)\n", 192 | " (1): ODST(in_features=5072, num_trees=1024, depth=6, tree_dim=3, flatten_output=True)\n", 193 | " )\n", 194 | " (1): Lambda()\n", 195 | " )\n", 196 | " )\n", 197 | ")" 198 | ] 199 | }, 200 | "execution_count": 6, 201 | "metadata": {}, 202 | "output_type": "execute_result" 203 | } 204 | ], 205 | "source": [ 206 | "trainer.load_checkpoint(tag='best')\n", 207 | "error_rate = trainer.evaluate_classification_error(data.X_test, data.y_test, device=device, batch_size=1024)\n", 208 | "print('Best step: ', trainer.step)\n", 209 | "print(\"Test Error rate: %0.5f\" % (error_rate))\n", 210 | "trainer.load_checkpoint()" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [] 219 | } 220 | ], 221 | "metadata": { 222 | "kernelspec": { 223 | "display_name": "Python 3", 224 | "language": "python", 225 | "name": "python3" 226 | }, 227 | "language_info": { 228 | "codemirror_mode": { 229 | "name": "ipython", 230 | "version": 3 231 | }, 232 | "file_extension": ".py", 233 | "mimetype": "text/x-python", 234 | "name": "python", 235 | "nbconvert_exporter": "python", 236 | "pygments_lexer": "ipython3", 237 | "version": "3.6.9" 238 | } 239 | }, 240 | "nbformat": 4, 241 | "nbformat_minor": 2 242 | } 243 | -------------------------------------------------------------------------------- /notebooks/year_node_8layers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "scrolled": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "env: CUDA_VISIBLE_DEVICES=0\n", 15 | "experiment: year_node_8layers_2019.08.27_17:11\n" 16 | ] 17 | } 18 | ], 19 | "source": [ 20 | "%load_ext autoreload\n", 21 | "%autoreload 2\n", 22 | "%env CUDA_VISIBLE_DEVICES=0\n", 23 | "import os, sys\n", 24 | "import time\n", 25 | "sys.path.insert(0, '..')\n", 26 | "import numpy as np\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "%matplotlib inline\n", 29 | "import lib\n", 30 | "import torch, torch.nn as nn\n", 31 | "import torch.nn.functional as F\n", 32 | "\n", 33 | "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", 34 | "\n", 35 | "experiment_name = 'year_node_8layers'\n", 36 | "experiment_name = '{}_{}.{:0>2d}.{:0>2d}_{:0>2d}:{:0>2d}'.format(experiment_name, *time.gmtime()[:5])\n", 37 | "print(\"experiment:\", experiment_name)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [ 45 | { 46 | "name": "stdout", 47 | "output_type": "stream", 48 | "text": [ 49 | "Downloading https://www.dropbox.com/s/l09pug0ywaqsy0e/YearPredictionMSD.txt?dl=1 > ./data/YEAR/data.csv\n" 50 | ] 51 | }, 52 | { 53 | "name": "stderr", 54 | "output_type": "stream", 55 | "text": [ 56 | "100%|██████████| 448576698/448576698 [00:10<00:00, 44134336.88it/s]\n" 57 | ] 58 | }, 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "Downloading https://www.dropbox.com/s/00u6cnj9mthvzj1/stratified_train_idx.txt?dl=1 > ./data/YEAR/stratified_train_idx.txt\n" 64 | ] 65 | }, 66 | { 67 | "name": "stderr", 68 | "output_type": "stream", 69 | "text": [ 70 | "100%|██████████| 2507989/2507989 [00:00<00:00, 7422252.00it/s]\n" 71 | ] 72 | }, 73 | { 74 | "name": "stdout", 75 | "output_type": "stream", 76 | "text": [ 77 | "Downloading https://www.dropbox.com/s/420uhjvjab1bt7k/stratified_valid_idx.txt?dl=1 > ./data/YEAR/stratified_valid_idx.txt\n" 78 | ] 79 | }, 80 | { 81 | "name": "stderr", 82 | "output_type": "stream", 83 | "text": [ 84 | "100%|██████████| 626904/626904 [00:00<00:00, 1869217.19it/s]\n" 85 | ] 86 | }, 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "mean = 1998.39193, std = 10.92832\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "data = lib.Dataset(\"YEAR\", random_state=1337, quantile_transform=True, quantile_noise=1e-3)\n", 97 | "in_features = data.X_train.shape[1]\n", 98 | "\n", 99 | "mu, std = data.y_train.mean(), data.y_train.std()\n", 100 | "normalize = lambda x: ((x - mu) / std).astype(np.float32)\n", 101 | "data.y_train, data.y_valid, data.y_test = map(normalize, [data.y_train, data.y_valid, data.y_test])\n", 102 | "\n", 103 | "print(\"mean = %.5f, std = %.5f\" % (mu, std))" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 3, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "model = nn.Sequential(\n", 113 | " lib.DenseBlock(in_features, 128, num_layers=8, tree_dim=3, depth=6, flatten_output=False,\n", 114 | " choice_function=lib.entmax15, bin_function=lib.entmoid15),\n", 115 | " lib.Lambda(lambda x: x[..., 0].mean(dim=-1)), # average first channels of every tree\n", 116 | " \n", 117 | ").to(device)\n", 118 | "\n", 119 | "with torch.no_grad():\n", 120 | " res = model(torch.as_tensor(data.X_train[:5000], device=device))\n", 121 | " # trigger data-aware init\n", 122 | " \n", 123 | "if torch.cuda.device_count() > 1:\n", 124 | " model = nn.DataParallel(model)" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 4, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "from qhoptim.pyt import QHAdam\n", 134 | "optimizer_params = { 'nus':(0.7, 1.0), 'betas':(0.95, 0.998) }" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": 5, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "trainer = lib.Trainer(\n", 144 | " model=model, loss_function=F.mse_loss,\n", 145 | " experiment_name=experiment_name,\n", 146 | " warm_start=False,\n", 147 | " Optimizer=QHAdam,\n", 148 | " optimizer_params=optimizer_params,\n", 149 | " verbose=True,\n", 150 | " n_last_checkpoints=5\n", 151 | ")" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 6, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "from tqdm import tqdm\n", 161 | "from IPython.display import clear_output\n", 162 | "loss_history, mse_history = [], []\n", 163 | "best_mse = float('inf')\n", 164 | "best_step_mse = 0\n", 165 | "early_stopping_rounds = 5000\n", 166 | "report_frequency = 100" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "data": { 176 | "image/png": "iVBORw0KGgoAAAANSUhEUgAABBUAAAF1CAYAAAC3TdL6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl8XHW9//H3J0n3fUlT6B5ogbJDoCBb3BAE8V65KtXrFe/V6lX8/dSr91evsogibsiVRbCgICCWRZZCS1ugTFtKW7rvW5pu6ZYmbZKmafbv74+ZpJPJTHJmMpNJZl7PxyOP5pzzPd/zmW/SzJzP+S7mnBMAAAAAAEC0MpIdAAAAAAAA6J5IKgAAAAAAgJiQVAAAAAAAADEhqQAAAAAAAGJCUgEAAAAAAMSEpAIAAAAAAIgJSQUAAAAAABATkgpACjKz3Wb2iWTHAQAA0lvgM0mtmQ0P2b/WzJyZjTez0Wb2DzMrMbNyM9tgZrcHyo0PlKsM+fpiUl4QgFaykh0AAAAAgJS2S9JUSQ9LkpmdL6lP0PFnJa2TNE5SjaTzJY0MqWOwc64+8aECiBY9FYA0YmbfMLMCMztqZrPM7PTAfjOzB82sOPCEYL2ZnRc49mkz22xmx81sv5n9MLmvAgAAdDPPSvq3oO2vSnomaPsySU8750445+qdc2ucc291aoQAYkZSAUgTZvYxSfdL+oKk0yTtkTQzcPh6SddKmiRpsKQvSioNHPuzpG865wZIOk/Sgk4MGwAAdH/LJA00s3PMLFP+zxnPhRx/1MxuM7OxSYkQQMxIKgDp48uS/uKcW+2cq5H0Y0lXmtl4SXWSBkg6W5I557Y45w4GzquTNNnMBjrnjjnnVichdgAA0L019Vb4pKStkvYHHfu8pMWS7pS0KzDfwmUh55eYWVnQ1zmdEjWAdpFUANLH6fL3TpAkOecq5e+NMMo5t0DSI5IelXTYzGaY2cBA0VslfVrSHjNbaGZXdnLcAACg+3tW0pck3a6WQx8UeGgx3Tl3rqQcSWslvWZmFlRsuHNucNDXls4KHEDbSCoA6eOA/BMgSZLMrJ+kYQo8KXDOPeScu1TSufIPg/hRYP8K59xnJY2Q9JqkFzs5bgAA0M055/bIP2HjpyW90ka5Ekm/k/9hyNDOiQ5AR5BUAFJXDzPr3fQlfzLga2Z2kZn1kvRLScudc7vN7DIzm2JmPSSdkFQtqcHMeprZl81skHOuTlKFpIakvSIAANCd/YekjznnTgTvNLNfm9l5ZpZlZgMk/aekAudcadhaAHQpJBWA1DVH0smgr2vkH6v4D0kHJZ0h6bZA2YGSnpB0TP4hEqXyPyWQpK9I2m1mFZK+JelfOyl+AACQQpxzO51zK8Mc6ivpVUllkgrl71l5S0iZMjOrDPr6QYLDBeCROeeSHQMAAAAAAOiG6KkAAAAAAABiQlIBAAAAAADEhKQCAAAAAACICUkFAAAAAAAQE5IKAAAAAAAgJlnJuvDw4cPd+PHj41rniRMn1K9fv7jWmapoq+jQXt7RVtGhvbxLp7ZatWpViXMuO9lxpAM+j3QNtFn0aLPo0F7Ro82il2pt5vXzSNKSCuPHj9fKleGWqY2dz+dTfn5+XOtMVbRVdGgv72ir6NBe3qVTW5nZnmTHkC74PNI10GbRo82iQ3tFjzaLXqq1mdfPIwx/AAAAAAAAMSGpAAAAAAAAYkJSAQAAAAAAxISkAgAAAAAAiAlJBQAAAAAAEBOSCgAAAAAAICYkFQAAAAAAQExIKgAAgC7FzG4ws21mVmBm08Mcf9DM1ga+tptZWcjxgWa238we6byoAQBIT1nJDgAAAKCJmWVKelTSJyUVSVphZrOcc5ubyjjnvh9U/ruSLg6p5ueSFnZCuAAApD16KgAAgK7kckkFzrlC51ytpJmSPttG+amS/t60YWaXSsqRND+hUQIAAEn0VAAAAF3LKEn7graLJE0JV9DMxkmaIGlBYDtD0gOSviLp45EuYGbTJE2TpJycHPl8vnjE3ayysjLudaY62ix6tFl0aK/o0WbRS9c2S5mkwomaepWebEx2GAAAoGMszD4Xoextkl52zjUEtr8taY5zbp9ZuGoClTk3Q9IMScrLy3P5+fmxRxuksdFp+a6j2r95nW6OU53pwufzKV4/h3RBm0WH9ooebRa9dG2zlEkq3PrYB9p66KRuvTHZkQAAgA4okjQmaHu0pAMRyt4m6TtB21dKusbMvi2pv6SeZlbpnGs12WMimElf/+sKXTnS9OXOuCAAAF1AyiQVth46nuwQAABAx62QNNHMJkjaL3/i4EuhhczsLElDJC1t2uec+3LQ8dsl5XVWQiFwTeVm99ehE5WddUkAAJKOiRoBAECX4Zyrl3SHpHmStkh60Tm3yczuNbNbgopOlTTTORdpaERS5Gb306EqhmMCANJHyvRUAAAAqcE5N0fSnJB9d4Vs39NOHU9LejrOobUrd3h/zVrrVF3XoN49Mjv78gAAdDp6KgAAAMRJbnY/OUm7S08kOxQAADoFSQUAAIA4mTC8nySp8AhJBQBAeiCpAAAAECe52U1JBSZrBACkB5IKAAAAcdK3Z5aG9jZ6KgAA0gZJBQAAgDga2c+0s4SkAgAgPZBUAAAAiKOR/TJUeKRSXWy1SwAAEoKkAgAAQByd1jdDx6vrVVJZm+xQAABIOJIKAAAAcTSyn0liskYAQHogqQAAABBHI/v5P14VMq8CACANkFQAAACIo2F9TL2yMuipAABICyQVAAAA4ijDTBOG92NZSQBAWmg3qWBmfzGzYjPbGOH42Wa21MxqzOyH8Q8RAACge8nN7sfwBwBAWvDSU+FpSTe0cfyopP8j6XfxCAgAAKC7yx3eX3uPVqm2vjHZoQAAkFDtJhWcc4vkTxxEOl7snFshqS6egQEAAHRXE4b3U0Oj075jVckOBQCAhGJOBQAAgDjLze4nScyrAABIeVmdeTEzmyZpmiTl5OTI5/PF/RqJqDMVVVZW0lZRoL28o62iQ3t5R1uhO8nN7i9JgRUgcpIbDAAACdSpSQXn3AxJMyQpLy/P5efnx6/yubMlSXGtM4X5fD7aKgq0l3e0VXRoL+9oK3Qng/r00PD+PempAABIeQx/AAAASIDc4f1VWFKZ7DAAAEiodnsqmNnfJeVLGm5mRZLultRDkpxzj5vZSEkrJQ2U1Ghm35M02TlXkbCoAQAAurjc7H56e/PhZIcBAEBCtZtUcM5Nbef4IUmj4xYRAABACsjN7qfSE7Uqr6rToL49kh0OAAAJwfAHAACABMgd7p+scSdDIAAAKYykAgAAQAKwrCQAIB2QVAAAAEiAMUP7KivDAstKAgCQmkgqAAAAJECPzAyNHdaXngoAgJSWckmF+oZGOeeSHQYAAIByh/djWUkAQEpLuaTCmT95S3e+vjHZYQAAACg3u792l1apoZEHHgCA1JRySQVJem7Z3mSHAAAAoNzh/VRb36j9x04mOxQAABIiJZMKAAAAXUFutn9ZSYZAAABSFUkFAACABGFZSQBAqiOpAAAAkCDD+vXUwN5Z9FQAAKQskgoAAAAJYmbKze5PTwUAQMoiqQAAAJBAudn9SCoAAFIWSQUAAIAEOiO7vw5VVOtETX2yQwEAIO5IKgAAACRQ7nD/ZI27SuitAABIPSQVAAAAEqhpWcmdR5isEQCQekgqAAAAJNC4YX1lxrKSAIDURFIBAAAggXr3yNSowX1UyPAHAEAKIqkAAACQYP5lJRn+AABIPSQVAABAl2FmN5jZNjMrMLPpYY4/aGZrA1/bzawssP8iM1tqZpvMbL2ZfbHzo48sd3g/7So5IedcskMBACCuspIdAAAAgCSZWaakRyV9UlKRpBVmNss5t7mpjHPu+0Hlvyvp4sBmlaR/c87tMLPTJa0ys3nOubLOewWRnZHdT1W1DTpUUa3TBvVJdjgAAMQNPRUAAEBXcbmkAudcoXOuVtJMSZ9to/xUSX+XJOfcdufcjsD3ByQVS8pOcLyeNa0AwWSNAIBUQ08FAADQVYyStC9ou0jSlHAFzWycpAmSFoQ5drmknpJ2Rjh3mqRpkpSTkyOfz9ehoENVVla2qvNodaMkad4Ha1RX1COu10sF4doMbaPNokN7RY82i166thlJBQAA0FVYmH2RJiG4TdLLzrmGFhWYnSbpWUlfdc41hjvROTdD0gxJysvLc/n5+TEHHI7P51Nonc45/fSDecoccrry88+N6/VSQbg2Q9tos+jQXtGjzaKXrm3G8AcAANBVFEkaE7Q9WtKBCGVvU2DoQxMzGyhptqSfOueWJSTCGJmZJgzvx/AHAEDKIakAAAC6ihWSJprZBDPrKX/iYFZoITM7S9IQSUuD9vWU9KqkZ5xzL3VSvFHJze6vwhKWlQQApBaSCgAAoEtwztVLukPSPElbJL3onNtkZvea2S1BRadKmulars/4BUnXSro9aMnJizoteA9yh/dT0bGTqq5raL8wAADdBHMqAACALsM5N0fSnJB9d4Vs3xPmvOckPZfQ4DooN7ufnJN2l57Q2SMHJjscAADigp4KAAAAneC8UYMkSSt3H0tyJAAAxE+7SQUz+4uZFZvZxgjHzcweMrMCM1tvZpfEP0wAAIDuLXd4P40e0kcLtx9JdigAAMSNl54KT0u6oY3jN0qaGPiaJumxjocFAACQWsxM103K1gcFJaqtD7vaJQAA3U67SQXn3CJJR9so8ln5Z1p2geWbBgfWiAYAAECQ6yZl60Rtg1btYQgEACA1xGNOhVGS9gVtFwX2AQAAIMhHzhyurAxjCAQAIGXEY/UHC7PPhdknM5sm/xAJ5eTkyOfzxeHy4SWy7lRQWVlJG0WB9vKOtooO7eUdbYVU0L9XlvLGD9HC7Uc0/cazkx0OAAAdFo+kQpGkMUHboyUdCFfQOTdD0gxJysvLc/n5+XG4fMDc2S0241p3CvL5fLRRFGgv72ir6NBe3tFWSBXXTRqhX8/dqsMV1coZ2DvZ4QAA0CHxGP4wS9K/BVaBuEJSuXPuYBzqBQAASDnXTcqWJC1iCAQAIAV4WVLy75KWSjrLzIrM7D/M7Ftm9q1AkTmSCiUVSHpC0rcTFi0AAEA3d85pAzRiQC/5SCoAAFJAu8MfnHNT2znuJH0nbhEBAACksKalJedvPqz6hkZlZcaj4ygAAMnBuxgAAEAnu+6sbJWfrNO6ovJkhwIAQIeQVAAAAOhkV585XBkmlpYEAHR7JBUAAAA62eC+PXXRmMEkFQAA3R5JBQAAgCS4btIIrS8q09ETtckOBQCAmKVsUmHl7qPJDgEAACCi687KlnPS4h30VgAAdF8pm1QoLDmR7BAAAAAiOn/UIA3p24MhEACAbi1lkwqW7AAAAADakJlhumZithZtL1Fjo0t2OAAAxCRlkwoAAABd3XWTslVSWaPNByuSHQoAADFJ2aSCGX0VAABA13bNpOGSWFoSANB9pWxS4RgzKQMAgC5uxIDeOvf0gSQVAADdVsomFRodYxMBAEDXd92kbK3ec0wV1XXJDgUAgKilbFIBAACgO8g/a4TqG50+KChJdigAAESNpAIAAEASXTx2sAb0ymIIBACgW0rZpAKDHwAAQHfQIzNDV505XAu3HZFj+CYAoJtJ3aQC78kAAKCbuO6sbB0or1ZBcWWyQwEAICqpm1SgrwIAAOgmrp2ULYmlJQEA3U/KJhUAAAC6i1GD+2jiiP4kFQAA3Q5JBQAAgC7gY2eP0LLCUpVW1iQ7FAAAPEvZpAJzKgAAgO7k1ktHq67B6dU1+5MdCgAAnqVsUgEAAKA7mZQzQJeMHayZK/axCgQAoNsgqQAAANBF3HbZWBUUV2rVnmPJDgUAAE9IKgAAAHQRN11wmvr1zNTMFfuSHQoAAJ6QVAAAAOgi+vXK0i0Xna7Z6w+qorou2eEAANCulE0qjBzYO9khAAAARO22y8bqZF2D3lh3INmhAADQrpRNKvzXS+uSHQIAAEDULhg9SGePHKAXGAIBAOgGUjapIElLd5YmOwQAAIComJluu2yM1heVa9OB8mSHAwBAm1I6qTD1iWXJDgEAACBq/3TxKPXMyqC3AgCgy0vppAIAAOh+zOwGM9tmZgVmNj3M8QfNbG3ga7uZlQUd+6qZ7Qh8fbVzI4+fwX176tPnjdSra/aruq4h2eEAABCRp6SChzf3cWb2rpmtNzOfmY2Of6gAACDVmVmmpEcl3ShpsqSpZjY5uIxz7vvOuYuccxdJeljSK4Fzh0q6W9IUSZdLutvMhnRm/PH0xcvG6nh1vd7aeDDZoQAAEFG7SQUvb+6SfifpGefcBZLulXR/vAMFAABp4XJJBc65QudcraSZkj7bRvmpkv4e+P5Tkt52zh11zh2T9LakGxIabQJdkTtU44f11cwPGQIBAOi6sjyUaX5zlyQza3pz3xxUZrKk7we+f0/Sa/EMEgAApI1RkoLvoovk73nQipmNkzRB0oI2zh0V5rxpkqZJUk5Ojnw+X4eDDlZZWRm3OvOG1enl7VWaOXuBRvZL3VGr8WyzdEGbRYf2ih5tFr10bTMvSQUvb+7rJN0q6Q+S/lnSADMb5pxrsfxCot/Ew0nHH6oX6foLHyvayzvaKjq0l3e0VdqwMPtchLK3SXrZOdc06YCnc51zMyTNkKS8vDyXn58fQ5iR+Xw+xavOyZdW69X7F2hXxmm6Lf+cuNTZFcWzzdIFbRYd2it6tFn00rXNvCQVvLxB/1DSI2Z2u6RFkvZLqm91UgLfxGeOLdVtM1qv9pCOP1Qv0vUXPla0l3e0VXRoL+9oq7RRJGlM0PZoSQcilL1N0ndCzs0POdcXx9g63YgBvfXxs0foH6uK9MPrz1KPzNTtrQAA6J68vDO1++bunDvgnPucc+5iST8J7OvUhZXPGTmwMy8HAAASY4WkiWY2wcx6yp84mBVayMzOkjRE0tKg3fMkXW9mQwITNF4f2Net3Xb5GJVU1urdLcXJDgUAgFa8JBXafXM3s+Fm1lTXjyX9Jb5hehCuPwUAAOhWnHP1ku6QPxmwRdKLzrlNZnavmd0SVHSqpJnOORd07lFJP5f/s8sKSfcG9nVr107M1siBvTVzxd5khwIAQCvtDn9wztWbWdObe6akvzS9uUta6ZybJX9Xw/vNzMk//OE7EStMECOpAABASnDOzZE0J2TfXSHb90Q49y9KxsONBMrKzNDn80brkfcKdKDspE4f3CfZIQEA0MzTwDzn3Bzn3CTn3BnOufsC++4KJBTknHvZOTcxUObrzrmaRAYNAACQTr6Q5x+J+sIKlpcEAHQtKTPbDx0VAABAqhoztK/yJ2XruWV7VF3X0P4JAAB0kpRJKgAAAKSyb153hkpP1OqlVUXJDgUAgGYpk1Sob4i0hDUAAED3N2XCUF00ZrCeWFSohkY+9wAAuoaUSSoM7ttDHzm93XknAQAAuiUz07euy9Xeo1V6a+PBZIcDAICkFEoqmJmmXdAr7LH752zRLY+838kRAQAAxNcnJ49U7vB++tPCQgWtpgkAQNKkTFKhLX9aVKj1ReXJDgMAAKBDMjNM37g2Vxv2l+uDnaXJDgcAgPRIKgAAAKSKf754lLIH9NLjC3cmOxQAAEgqAAAAdCe9e2Tq36+aoMU7SrRxPz0xAQDJRVIBAACgm/nSlLHq3ytLf1pUmOxQAABpLuWTCq+sZi1nAACQWgb16aEvTxmr2esPaN/RqmSHAwBIYymfVPjBi+uSHQIAAEDcfe2qCcrMMD2xmN4KAIDkSfmkAgAAQCoaOai3/vniUXpx5T6VVtYkOxwAQJoiqZAA98/ZovHTZyc7DAAAkOKmXXuGqusa9dele5IdCgAgTaVtUqGx0el4dV1C6mbSJAAA0BnOHNFfn5yco2eW7lZVbX2ywwEApKG0TSo8tGCHzr9nvo6dqE12KAAAADH71nVnqKyqTi+s2JfsUAAAaShtkwpvrDsgSSqJYQzissJSXffb93SytiHeYQEAAETl0nFDdNn4IXpy8S7VNTQmOxwAQJpJ26SCmUmSXAzn/mL2Zu0prVJBcWV8gwIAAIjBN689Q/vLTsq37UiyQwEApJn0TSokOwAAAIA4uWbScPXKytDSnaXJDgUAkGbSKqnw3tbiVvtcLF0VAAAAupBeWZm6ZOwQLd9FUgEA0LlSLqlw3z+fF/HY2n1lembpbq3ac1SB0Q9yUQ6AeGTBDm3cX9GBCAEAAOLvitxh2nywQuVViVndCgCAcFIuqfDlKeMiHnOS7np9k259bKksMAAi2p4Kv5u/vQPRAQAAJMaU3KFyTlqx+2iyQwEApJGUSyp4ZUyqAAAAUshFYwarZ1YGQyAAAJ0qvZIKYboldMacCtV1Dbrj+dU6WH4y8RcDAABpqXePTF08ZrCWFdJTAQDQedIrqRBnXns7zNt0SG+uP6hfztma2IAAAEBam5I7TJsOlKuimnkVAACdI62SCiz0AAAAUtkVE4aq0Umrdh9LdigAgDSRXkmFMFmFaFd/AAAA6KouHjtEPTMztKyQeRUAAJ0jrZIKj7xX0Py9WWyrP6S6+2Zv1rNLdyc7DAAAEIM+PTN14ZhBWraLeRUAAJ0jrZIKwVj8IbwnFu/Sna9vSnYYAAAgRlMmDNPG/eWqrKlPdigAgDTgKalgZjeY2TYzKzCz6WGOjzWz98xsjZmtN7NPxz/U5Nl0oFwNjXRpAAAAXd8VucPU0Oi0cje9FQAAidduUsHMMiU9KulGSZMlTTWzySHFfirpRefcxZJuk/THeAcab00rN7Q3/GF9UZlueuh9PRo0dCLma3a4BgAAgLZdMm6wsjJMyxkCAQDoBF56KlwuqcA5V+icq5U0U9JnQ8o4SQMD3w+SdCB+ISaG1+UgD5RVS5I27i9PYDQAAADx0bdnli4YPUjLmawRANAJsjyUGSVpX9B2kaQpIWXukTTfzL4rqZ+kT4SryMymSZomSTk5OfL5fFGG27bKykrPdR4/XilJWrlqpQ5vz9DX51dpVH/TN87vpfGDMlXf6JRp0qbiBklSSUlJq7pXrVqpkh2ZEa/RVH7zAf+YxuLiw3F/zbFqr626SpxdRTS/W+mOtooO7eUdbQV4d0XuMM1YVKiq2nr17enl4x4AALHx8i4T7pl+6KCBqZKeds49YGZXSnrWzM5zzjW2OMm5GZJmSFJeXp7Lz8+PIeTIfD6f8vPzpbmz2y07cMAAqaJcl1xyqXKz+0nz52t/pdM9S6u18qefUN4v3tElYwfrtEF9JB3U8OHDlZ+f16LuSy/N03mjBrWuPFCm6fWVr90vrV+rESNylJ9/cRxeacc1t1WokNjhF7G90AptFR3ayzvaCvBuSu4w/dG3U6v2HNM1E7OTHQ4AIIV5Gf5QJGlM0PZotR7e8B+SXpQk59xSSb0lDY9HgIkSPPwhNENyMDDkYfXeMs3ecLC5jItx/UmWrQQAAJ3p0nFDlJlhWsYQCABAgnlJKqyQNNHMJphZT/knYpwVUmavpI9LkpmdI39S4Ug8A02UcPf7tQ0NnRpDQ6PTwfKTnXpNAACQuvr3ytL5owZpeSGTNQIAEqvdpIJzrl7SHZLmSdoi/yoPm8zsXjO7JVDsvyR9w8zWSfq7pNtdrI/1O0nhkRMRj9362NJW+xK5csMD87fpyvsXaOeRygReBQAApJMpuUO1rqhMJ2s792EJACC9eOmpIOfcHOfcJOfcGc65+wL77nLOzQp8v9k5d5Vz7kLn3EXOufmJDDoeKmvqm7+PNf3hdQWJ9sov3O7v1PHxBxbqQ5Z/AgCkOTO7wcy2mVmBmU2PUOYLZrbZzDaZ2fNB+38T2LfFzB4yi/bdOnVcMWGY6hqcVu89luxQAAApzFNSAX7RJh+8dtbYduh48/er9vDGDwBIX2aWKelRSTdKmixpqplNDikzUdKPJV3lnDtX0vcC+z8i6SpJF0g6T9Jlkq7rvOi7lrzxQ5RhYmlJAEBCpWRS4RvXTPBcdsGWw+EnVuhE9Y1deqQIAACd6XJJBc65QudcraSZkj4bUuYbkh51zh2TJOdccWC/k39ep56SeknqIelwp0TdBQ3o3UPnjRqkZfSCBAAkUEomFYb17+W57EMLCuQSlFXo2rNKAADQJY2StC9ouyiwL9gkSZPMbImZLTOzG6TmFajek3Qw8DXPObelE2LusqZMGKq1e8tUXce8CgCAxMhKdgDdCTmC8PYdrVJJZY0uHjsk2aEAALq/cHMghL4FZ0maKClf/qWuF5vZefIvZ31OYJ8kvW1m1zrnFrW4gNk0SdMkKScnRz6fL27BS1JlZWXc64xV3xP1qm1o1FOzfDpnWGayw4moK7VZd0GbRYf2ih5tFr10bbOUTCpEOyNTrD0KHnx7u84cMUDTbzw7fL3N/566wMnaBmVkSL2yuu4be7Su+c17kqTdv7opyZEAAFJAkaQxQdujJR0IU2aZc65O0i4z26ZTSYZlzrlKSTKztyRdIalFUsE5N0PSDEnKy8tz+fn5cX0BPp9P8a4zVhefrNNDa+arZtAY5edPSnY4EXWlNusuaLPo0F7Ro82il65tlpLDH6JVdOxkTOe9s6VYjy/cGfF4uIkaz7lrrs766dyYrgcAQBpYIWmimU0ws56SbpM0K6TMa5I+KklmNlz+4RCFkvZKus7Mssysh/yTNKb18IdBfXpo8mkDtYzJGgEACZKSSYVoF4/aeaQyMYG0IXhJS5xS39DYYnvroQrtO1qVpGgAAJ3NOVcv6Q5J8+RPCLzonNtkZvea2S2BYvMklZrZZvnnUPiRc65U0suSdkraIGmdpHXOuTc6/UV0MVMmDNOavWWqqWdeBQBA/KVmUiHKARCNHsY/zN98OOIkR88u26MTYZIEobUGR1XDhEmtvLhyn878yVsqOnYqiXDD/y5uHl4BAEgPzrk5zrlJzrkznHP3Bfbd5ZybFfjeOed+4Jyb7Jw73zk3M7C/wTn3TefcOYFjP0jm6+gqrsgdqpr6Rq3bV57sUAAAKSglkwrRWrrTW5fAHcXhezTc+dpG3fvGZq3a0/aSTS7M90dP1Hq6dnfwxrrQIa+xnV8QoZ0BAED0Lp8wVGZiCAQAICFIKkh6aVVRh+t4YeU+3fruoMbEAAAgAElEQVTYUr214WDzPi8TQK7c3TIREW7ohnNOtz/1od7bVtz6YBcSKekCAACSZ3DfnjorZ4CW7yKpAACIP5IKUQg38WKoPW2M//eyRla4S9TUN8q37Yi+9tSKdq8fSUFxpeoaGvXqmiL95NUNMdcj+dth7saDqq1vbL9wJ6mpb2iR0AEAAKdckTtMq/Yc08lahl8CAOIrJZMK0U7U6NXOIyeiKt+0lGS4REGk/MSv527VPbM2RRtam/aXndQnfr9QE3/ylr7/wjr9bfneDtW3eEeJvvXcaj3w9rY4RehnHfjB/eqtrfrPv632PJQFAIB0cv3kHFXXNWrB1q7d6xEA0P2kZFIhUX740rqoyjvnX82gJvBE38NoCEnS0x/sbrG9vyy2JS+bHK2M77wNx6r89R0oq45rvU28tlOw/YFlQctP1sUnBuc89UwBAKA7mJI7TMP799Kb6zs2/xEAAKFIKiRQ0bGT+tKTy/XjV/zDDRqj6LEQ7OMPLIxzZNKS/fG5+Q4WuhxkW07WNrRaMaMjHUzieft/9EStJvx4TqvkDgAA3VVmhunT54/Ugq3FLGsNAIgrkgoJ9InfL9SHu9peESKSDwpK4vakPNyogjcK459UqK7znlS49Bdv69y758U9hng4EOgZ8tLKjk/gCQBAV3HzBaerpr5R7245nOxQAAAphKRCnDWE644QEJwkcM3P1sOX/9KTy/Wyx1UpCo9UavOBCs8xdgVVcZ4oqq1eDs8t26O1+8qirzNBc3MAAJAMeeOGaOTA3h1eAhoAgGBZyQ4gEToy4V9H/f7t7RGPRdvvYF8bK0kE+1hgeMTuX93kuW4vLeSc08m6BvXtGf7XJGFzDsS52p++trH5+3V3Xa9BfXu0fXmmUgAApKCMDNNNF5ymZ5buVvnJOg3q0/b7IQAAXtBTIc7a6qkQ7c1yuOJvbTioYydOTbxYeKQyukqjCOWlVUWafNc87SppuerF62v9TzgqquM7JrMjuSCvTXuo4tTkkocrqnWoPPJkk/RUAACkmpsvOE11DU7zNx1KdigAgBRBUqETzd5wsNW+uoboMg3/+bfV+uZzqyRJZVW1zb0Umnzuj0v0+tr9sQcZZOH2I5KkDfvLW+xvWo7Ka0+KaDmPKYKPPeDTR+5/t8W+9hIBwXVP+eW7uiLk/Giu39U45/TetuK2E1sAgLR20ZjBGj2kj95c3/ozCQAAsSCpkCyB+75fvbU16lOblk88HqanwOq9Zfq/M9e2W8ehE07jp89W8fHwT+rLT9YpM3CH3tFhDk8uLtRv5rb/OqPtGFB45IQOhPQ0iOfQBevQehTRm7/pkO59Y3PM57+9+bC+9tQKPbG4MI5RAQBSiZl/CMSSgpIWPR8BAIhVSiYVulOv9f2BlQbC2VNapY894OtQ/ZU19W1Oirh6T/gJDC/82fx2n9d7TTb8YvYW/dG301NZf72ei7ZreWFp1Ocka06Fac+u0l+W7Ir5/MPHayTFlqgCAKSPz1xwuuobneYyBAIAEAcpmVToDi7/5btaX9T2igSz1h1Q4ZETrfZH03PgvLvn6Qt/Whrx+Asr9mr89NmeV5oIZmYJmazxP/66MuZzQ4c/hE6cGU24zKkAAEhF554+UBOG99Ob61kFAgDQcSQVkuiWR5Z06PzQG+RYbvDf2+afN+GPvoIOx+DkVN/QqCcWFaqmPr5LRsYqNDHgpYmCizwwf1uriSqj9fW/rtT46bM7VEeoD3aWqL6hscU+ciAAAC/MTDdfcJqW7izVkUAvNwAAYpWSSYV0fcLckU4DsZzrXOspDf++Yp/um7NFj/uiH9cfvBTojEXeh0u0WWfIrXY0kzAeLK/WwwsK9NW/fNihGN7ZcrhD54dasfuovvTEcv3h3R1xrRcAkD5uvuB0NTrprY1M2AgA6JjUTCokO4AkCb1dPl5d5/ncxhiyCuGGP1TV+CePrKzxfu1wfjmn4/MCjJ8+W0tjmlPBBf71b4f2CEi2w4FlMXfGuJwoAABnjRygiSP66811JBUAAB2TkkmFdBH61D34Bt85p4eieJIday+H4NOcO9VLJB5TLQQvjbh4xxF945mVHZ7DIZbTrYt1fWl6DZ29OgUAILXcfMHpWrHnqA6Vh18JCgAAL0gqpJDg++V3thTricXeVxKIZlhA8znOtbpJj+eN7m0zTk0w+bWnVujtzYdV15C4pRn2llZp4/7yoJZI0jIQXoU0dRfLfQAAuribLzxNzkmzN9BbAQAQO5IK3dSOw8f15vqWHwKChzAcq4pu7enGGHv4R0pGHKro+FOPFbuPNX/f3g1z00vvyIRT1/72Pd388Psxn99ZuniqAwDQTZyR3V+TTxvIKhAAgA7J8lLIzG6Q9AdJmZKedM79KuT4g5I+GtjsK2mEc25wPAONRlfrrh5vB8qr9ckHF7XaH9xrIJYWqK2PPrPQYvUH55pv/t9cf1C19StbDGFoTzx+aj99baP+9YpxHaqj6TWVVPoTM15/nT79h8W6dlK2pt94doeu356mISCp/VsOAOgMN194mn4zd5v2Ha3SmKF9kx0OAKAbarengpllSnpU0o2SJkuaamaTg8s4577vnLvIOXeRpIclvZKIYOFdRpSJlUbn9K9/Xh7VOWamuRsPRTw+f/Nhvbu1uNX+guLjUV0nWCzDNFqcH9OcCt7KbT5YoccXxmfVCi+akmeFRyo1fvps7Sxue+nL5YWl2ri/XJI/MbHpQHnCYwQAdG03n3+6JIZAAABi52X4w+WSCpxzhc65WkkzJX22jfJTJf09HsHFKsU7KkTUoqdClG3gnPThrqNRX/N7L6w9VYeH8keO1+gTvz/Vy+LXc7eqpr6h3fOa5mqIlBQIfr3PLt0dcUJHJ6eGRhdlD4rE/kJd/+BC/ffL62I6t76hUTNX7JMkzVq3v82yX5yxrHl4x6x1B3TTQ+/rrS7wIXLLwQrl/eJtlVSyVjoAdLaxw/rqwtGDGAIBAIiZl+EPoyTtC9oukjQlXEEzGydpgqQFEY5PkzRNknJycuTz+aKJtV2VlZXy+Xzavqdjyxl2V2++s7D5+61bo1uSsbqm9Q1d8eHDkqR/vL9RO7Zu0Zxddbrzit7Nx6uqqlqUf2bpnjav4fP5dLCy5RCLx3w7dfzwPn1iXA+VlEaeh6HR+c9btGiRema2vMn3+XwqKTl17p2vb1LNofA9BlatWqXPP1atfj1O1RH6e7h69eoW29XVJ1VZ2dhcrrLWqbC8QRdkh//vE+n3OtL+7YdPaPvhynbLNdl8wL9sZ/Hhw/rSw/P14SF/Uqa29tQ8Gm3V4fP59PZ2f9m3P9ygPqXb2rxetJr+H3r1xPoalVTW6/HXF+nqUT3iGkt3EG17pTPaCkiMmy84XffN2aLdJSc0fni/ZIcDAOhmvCQVwj2mjfSY9zZJLzvnwj56ds7NkDRDkvLy8lx+fr6XGD3z+XzKz8/X3qW7pS2b4lp3d/CjRSebvz938jnS+rVtlG4pq0dPKSSxMCInRzp0QIuK6vX+/no1Ouna6/KleXMkSX379pWq2u5yHyw/P187j1RK7y9ssX/UuFzl55+h5/askI60Hi4hSRkZGVJjo6699lo9u3SPevfIkLSpud7n9qyUig83lz/n/Aul5a2Hc1xyyaWqWbpENUGrSDT/Hs6dHShzsbT81MoTffr0Uf/+1lzus48u0bp9Zdp8b7769gz6LxQ4Pz8/Xwu2HlbOwN469/RBLfaHFTjeKp4Iytbsl9avVU5OjmatO/VkqWfPnlIgsZCfn69lhaX67bxtmjntCvXIzGgRx4qarVLhTk0YP0H5+RPbvF60mv4fejWreK10YL/OPvsc5V86Oq6xJENxRbV69cjUoD7eEiTRtlc6o62AxLjpgtN035wtemPdAX334/F9TwAApD4vwx+KJI0J2h4tKVIfuduU5KEPEhPYSdFPVhmu63m4rpCr9x5rta+j6hvanyCy6dU4J903Z4vufL3tpFHRsao2j7elvXkXCg7754RoaHR6YlFh2Gv9+9MrddNDiVlJomleidAfcWjc//3yeq3ac0wHyk4qVLghHR/uOqqJP5mjYyf8iYndJSf0Xpg5MeIuxZazuPyX7+rK+99NdhgA4Nnpg/toyoShemHlPk/vyQAABPOSVFghaaKZTTCznvInDmaFFjKzsyQNkbQ09Bg637p9ZR2uo+UcDf6b0M8/furHu6vEey+FtiwuKNHe0ipPkyhGnqix5f7/948NYUvFsuRk6O1305WOHK/RfXO26PanVrQ4/uTiwqivEYt4J88e8xWorsFpzT5/4ij/dz597ekV7ZwVP0sKSnSytv35NbqDqhR5HQDSx79fPUFFx05q3qbD7RcGACBIu0kF51y9pDskzZO0RdKLzrlNZnavmd0SVHSqpJku0gx5nahHppdcSWr78/u74lpfvH6s5Sdbz3fx4a6juva374VdKaJJ01P5jobx6tq2JzOU2n9w3hgIommuxxM19S2O/2L2lubv/xSn1SDKq+q0Yrd/Is1k/w8rq6pVnccnWV/583Ld/fpGT2VfXbNfP3k1fDIIAJBYnzgnR+OG9dUTiwvj9p4PAEgPnu6+nXNznHOTnHNnOOfuC+y7yzk3K6jMPc656YkKNBo9s0gqxFsUiyW06XN//CCm86rr/Dex9REDif25vW9bsb7wuPcONk2ftY5V1bbYDuf+t9qeMNNrz4nbn/5Qn398qWrrG5uv53WIy3W/9ennb25u3m5sdM1P0mP5sV5079v6wYveVqtYvKNEf21nAs9gO+PU+wUAEJ3MDNN/XD1Ba/eVJWSoIwAgdXH3jW5lwdbw3TLf2eKxu2aYu+jbn1qhD3efWk6zvQc0TYf/87lVkqRDFZFXrYjkrQ0Htaf0hC677x1P5Tftrwhc+1RwoSmFtnIMwT1XfjF7i/6ypGM9Wd5YF35aleWFpdp2tANd/3k6BgBJ8y+XjtagPj30xKL49nYEAKQ2kgroVjo6f9T8zYeiPmd3aZWeWF9zqjto4J+SytrIJ7VhV8kJ/effVuu63/piOr/ptnvTgYqW+z3ej7+8al+bxxduO9Jiu7HRqdFjV5Uvzlim+z9sO8lyqLxar6wu8lQfAKDz9O2ZpS9PGat5mw9pTyk9xwAA3qRFUuHN716d7BAgqTJk7oFkqGto/+Y43FjSJQfqdTwQf2MHn6ZX1XasHZri2xZYhSLe/rp0jzYdKG/evuje+brq1wviVv9X/rxcP3hxnSqqW8+vEa3KmvqUmdwRALqCr35kvLIyTE8t2Z3sUAAA3URKJhWiXE0RneS8u+d1uI5ZEbrdx1N7KYOOpBT2l52Muod/fUOjar0suxn0e7/vqLclNSPFcrz6VOKjorpeB8urA+UjB19d5+3m/nBguEhT74eOtOd5d8/Ttb99rwM1tDR340FtPVTRfkEASFE5A3vrMxeerhdX7lN5VceTvwCA1JeSSQWkrkXbj7RfqIMi3Tdb8/HWBf7+4d52631qyS5d9asFWlZYGlU8P3nV2+oJwcMxFu3oWDtlZkSfmSvz+OGzaYLJcO28rqi89c52xLJMaCTfem61bvjfxXGrDwC6o69fnauq2gY97+G9DQAAkgpAFP76we6wK2HMWFTY7rk/e8O/AsOWg96GLewtrdI/VhVpzoaDzft2HK709GS/o/MdRsopxGMeRa89iR7z7VThkcqYrlFQXKnx02dr3qbo59AAgHQ3+fSBuurMYXr6g12qre/gZEYAgJRHUgEI4SLctpuZ7p61qdPi+Kc/LtF/vdRy6cb/fnl9x8YLhGjrtSZT+ck6/XruVk19YllM568vKpMkzd1IUgEAYvH1a3J1uKJGszckftghAKB7S8mkgrVabA+pLN7rab+1Ifob0bbmGojV0ROB4Qwx/Dp3NJrMCEmFWOsNNymjk7/dXl2zv/WxQHsyCSMAJMd1E7N15oj+emLRroS8xwEAUkdKJhWunjhcA3pnJTsMdJLP/fGDuNb37LI9ca0vVKTeAZ11vpdeCBlhyhyuqNa7Ww7HdM1bHn7/1PUD/+4qOaHrH1zU5nmx9pjg8y8AdExGhunrV0/Q5oMVWhrlXEAAgPSSkkmF4f17acM9n0p2GIijof16JjuELtH/ZfPBik65YQ53L/8vj3+gac+uiuqcJrtLW69Gce8bm7SjOLY5E7qbBVsPa/z02Z5X5QDSmZndYGbbzKzAzKZHKPMFM9tsZpvM7Pmg/WPNbL6ZbQkcH99Zcaeif7p4lIb166knF+9KdigAgC4sJZMKSD3NQwGSqLGNu/mo7vOjTAoEL+/o+XSPmYemYuuLyvTetrZXjNh39KSnOtvT1Pug6Fh86gt/jYRVHZN/rPYP8Vi7ryzJkQBdm5llSnpU0o2SJkuaamaTQ8pMlPRjSVc5586V9L2gw89I+q1z7hxJl0sq7pTAU1TvHpn6ypXjtGBrsQqKvU0yDABIPyQVAI8+//jSiMf2hHkSn0xtpRTKT7ae3+CWR5YkLpgISrtAoghAl3O5pALnXKFzrlbSTEmfDSnzDUmPOueOSZJzrliSAsmHLOfc24H9lc65rvXHuRv6yhXj1DMrQ39+f3eyQwEAdFFpMfEA46sRD1sPxecpzSthJiaMRlf5fS6trFFVbYPGDO0b9vjG/eUttl9YsVdfvGysp2Ekwa+xtr5RU59Ypv/59Nm6dNxQT7HF0kYzFu2M/iSvusjPDOgGRknaF7RdJGlKSJlJkmRmSyRlSrrHOTc3sL/MzF6RNEHSO5KmO+dazfhqZtMkTZOknJwc+Xy+uL6IysrKuNeZTFeOzNDLK/fqyv4lGtgzMV3BUq3NOgNtFh3aK3q0WfTStc3SIqkApJK2hmFE60hljcZPnx3TuZf+4h1J0u5f3SSp5ZwTu0tO6OagyRkl6f/9Y4MuHTckqmuYSYUllVq155j+55WNmvf9az2d19RGC7e3PaQj2C/nbI0qNi8Kiis1ekifuNcLpLBwd6yhf/SyJE2UlC9ptKTFZnZeYP81ki6WtFfSC5Jul/TnVhU6N0PSDEnKy8tz+fn5cQm+ic/nU7zrTKbRkyv1yQcXal1tju68fnL7J8Qg1dqsM9Bm0aG9okebRS9d24ykAtDNxHNprw1F5WH3d3ROgpdW7Qu7v7bexVx3W6teOOdarBRRFViKMplzcVRU1+kTv18Y8fjcjQe1aEeJro8uzwKkuiJJY4K2R0s6EKbMMudcnaRdZrZN/iRDkaQ1zrlCSTKz1yRdoTBJBUTnzBH99cW8MfrrB7v15SljlZvdP9khAQC6kLSYU6FXj7R4mUgTd76+qd0yyR4isbP4RNj9HVkOc/vhSn35yWXN26v2HGv+/onFhRHPe33tfo2fPlvHOjnBUF3bqsd1C996brWeX763k6IBuo0Vkiaa2QQz6ynpNkmzQsq8JumjkmRmw+Uf9lAYOHeImWUHyn1M0uZOiToN/Nf1Z6l3j8yE9OoCAHRvaXG3PSlnQLJDADrVjEWF2uth+cKa+vA3vhbFApq19Y06VF7dYt/cTYc8nx9OpNTDkoJTa6Xf+tgHzd9/sLPlGurBvSGeWrJbkrSrNHyiI5zqurYTAgASwzlXL+kOSfMkbZH0onNuk5nda2a3BIrNk1RqZpslvSfpR8650sDcCT+U9K6ZbZB/KMUTnf8qUlP2gF76zkfP1DtbDmtJQUmywwEAdCEpnVR4aOrF+tKUsckOA4hKfUNjzPMcNNlf5m25xu2HKzt0HUma/sp6XXH/u6qua/RUvqSy7R4DHxSU6ETNqWU0o0lwhNM0XKTwyAkdOV7j6Zzv/G11h64pKfzIcEl3v76xwz9fIJU55+Y45yY5585wzt0X2HeXc25W4HvnnPuBc26yc+5859zMoHPfds5dENh/e2AFCcTJ164ar9FD+ujnb25WQyMz0AIA/FI6qXDLhafrl/98frLDAKJS380+qL2y2r+aRaReD9H60pPL9X9mrpEklVXVdWjIhHSq18MPX1qna36zoMWxuoZGlVW1vud4d2vilrb/69I9CasbABKpd49M/c+nz9HWQ8f1worwc+cAANJPSicVvHj+G6ErVQHJ9bt525IdglbuOZqQer3O9bBmb1nz9x8UlLZRsv16g4+F9qb47vNrdNG9b4c97/nle7WssP1rA0A6ufG8kbp8/FA9MH+bKqrrkh0OAKALSPukwkfOGJ7sEIAWnnx/V7JD0F0eJoPsLDX13oZVBAseeRC6BOc7mw83f9/W3A//8+oG3TZjWcTjsdhd4n1eBwDoisxMd948WUeravXogoJkhwMA6ALSPqkAID4OVVS3W+Zvy6Pv+l/f0Dqp8IMX1rbYbqsDxKYDFS2252/u2CSSXoWbC+KBt7d3yrUBIJHOHz1It14yWn9Zskt7opgEFwCQmtIyqdC/V1ayQwBSzlf+/GG7Zf7+YfRjcMPdiL+yZn/U9TR5cWWRXlwZXRxvrj+gF1ZEt/zjZfe946lcVV34lMiDb2/X8sDwi/KTdWGTK/HQ2OhUGTQxphcHy082T4AJID396FNnqUdmhu5niUkASHtpmVR447tX6+VvXamHp17c6tjb3782CREB6Kiq2nrPN8dvrj8YVd13PL9G/+8fG2IJq12PrA3fw+MP7+7QF2csU2Oj04U/m68fv5KY6/989madd/c8z8tobj1UoSvvX9C8VCeA9JQzsLe+nX+G5m46pKU7mX8GANJZWiYVJClv/FB95sLTW+2fmDMgCdEA8CrcE/JF249o8l3zdN7d87TzSKWKPS4d2cTrUpNtmb3+YExLRe6vbPuJf0Pg9b7agd4ZbWmq92Stt6RC07wQTGIJ4OvX5GrUYJaYBIB0l7ZJBQDd02/aWR3j4w8s1MPtTB4WmpjwOlThycWF+qCgpMW+lbuP6sUV+/S7+clftSOSsqpajZ8+Wy+vKmp1LNpRDE3lrfWUEQDSTO8emZp+49nafLBCL69iiUkASFdpmVTgszDQfT3m25m0a/9i9hZ96cnl+vXcU2OI/+Xxpfrvf6zXri68ssOe0ipJ0jNLd3e4rqYcRLiJKAGkn5svOE2Xjhui387bzhKTAJCm0jKpAABvrj8Q87nJTGx0FfRUACD5l5i8+zOTVXqiRr+fzwo3AJCOPCUVzOwGM9tmZgVmNj1CmS+Y2WYz22Rmz8c3zO7h61dP0K2XjE52GADasXhHie54fk2ywwhry8EKlVR2fI6HYF5GOHhNErQ1XOJkbYPGT58d9UoZALq3C0YP1penjNUzS3dr04HyZIcDAOhk7SYVzCxT0qOSbpQ0WdJUM5scUmaipB9Luso5d66k7yUg1k6XmRH9ozjn6eM7ALQ2d+NB3fiHxcr7Res5Htr7y1JT36Bpz6xUQfHxiGXi0bmg6W9cuCREUzLkoXfbntMCQOr50fVna0jfnvrpaxvVyKSNAJBWvPRUuFxSgXOu0DlXK2mmpM+GlPmGpEedc8ckyTlXHN8wk+MrV4xLdggAUlzwk/9vPbe6zePBHvPt1Jq9x5q3V+8p0/zNh/U/r26MfK2Yo2wdT1eaU2HdvjLtDcwbASA5BvXtoR9/+hyt2Vuml5i0EQDSSpaHMqMkBb87FEmaElJmkiSZ2RJJmZLucc7NDa3IzKZJmiZJOTk58vl8MYQcWWVlpac6ly9frt39wudTgs8vKmo9U3pb9hXtU2VtVKcA6OIi/U2J198v5xrbrGvST98KlHMtyv16rn9iyKwM6cnr+2lLqX9JyPKyMt33t7c1dmCmxgzw/50rLPMfO15xvNW16ur8E6u9//4S9e/ZfqJg88F6SdKRI8Wt6io52ShJqq6ujvvfdyny3/jbA23x9A394n5NAN7deskovbBir3711lZdP3mkhvTrmeyQAACdwEtSIdynzNAHXlmSJkrKlzRa0mIzO885V9biJOdmSJohSXl5eS4/Pz/aeNvk8/kUqc6f996jO1/zP8G74oopGjcs6MPn3FNry+fn5zdvjxo1Stq7x/P1x4weo6NVtdKBxKwnD6DzBf9NCFY9/CzdcN5pYY9FIyMj49TfrTbqMrOWf98CZesb/TH22lkqrVimwYMH64kNRyVJu391kyRp8L4yadkSDRw4QPn5V7eoN8s3T6qv19VXX6XBfVveAIyfPlt3fPRM/fBTZzXvq1h3QFq3RtkjRig//5IW5YuOVUkL31OvXr0i/i3uiIh/4wNtkYhrAvDOzPTzfzpPNz30vn4zb6vu/9wFyQ4JANAJvAx/KJI0Jmh7tKTQadOLJL3unKtzzu2StE3+JEOX8ZUrxmns0L5RnRNLV+FvXntGDGcB6Kpmrz8Ydn+4oQqxaGviw2ANQWOU1+0ra3W8aa6D5buOtti/t7RKJcdjn/jxkfdazo/gAgFnmGnj/nIt3nGk+ZgFJlpgNDWQvs4eOVBf+8h4/f3DfVodNEQLAJC6vCQVVkiaaGYTzKynpNskzQop85qkj0qSmQ2XfzhEYTwD7Uw/u+VcSf4P++/84Frd/ZnJ7Zzh5ySdkU33WyCVfOf5yMkD5zUjEEcV1XW6541NYYIJX/7a376nrz+zso36/MMZmuZI+MeqIm3c3/7s7Sbp5off11f+/GGLfZL3RAmA1PS9T05SzsBeuvO1japvaEx2OACABGs3qeCcq5d0h6R5krZIetE5t8nM7jWzWwLF5kkqNbPNkt6T9CPnXGmigo6V15UZmmY1d3I6c8QA5Wb3T2BUALqrI3FY+tHrUo5NPvXgIq3Z27qnQrz810vrdPPD70c83lbCIPhvJ4D01b9Xlu68ebI2HajQc8u8DyMFAHRPXnoqyDk3xzk3yTl3hnPuvsC+u5xzswLfO+fcD5xzk51z5zvnZiYy6EQLfdrWI9P7p36L9g6hA66dlN1p1wLQ2sd/t7DDdUT7VP9geXX4esLWHbI3Dn+f2lpSMpoVIbYdOq6q2voOxyRlxWYAACAASURBVAOga7rp/NN0zcThemD+dhUfD/93CwCQGjwlFVLd818PWcwiZFzwZeOHKm/ckM4NyoNBfXokO4Rm554+MNkhAJ3ueE3XuCmONAzjz+/v8lzHq2vaX+2m6FiV/rTQP7ItOH1Q39Co8qq6oHjarqe6rkGf+t9F+s7f2p6XwretuMVcEgC6DzPTz245VzX1jfrl7C3JDgcAkEBpmVQIfZr2kTOHhxz3O9VTIUM/Cpr9XJLOHzUoQt0dN7C3l0U5pM9ccFocrpY4n7nw9GSHAHR58ejc5Fz4G/kNIXMjFBw+rpIIQzbueWNzxPobG50aG51uf2qFth46Lqllr6yfvrZRF947X3XtjJ0+UVOvFbuPqr4x/KSSwd7bWqzbn1qhx3wFEcvEoqK6Tp/8/UJtPlAR13oBtJab3V/fvC5Xr609oKU7u9yoWABAnKRVUsFrN+Ow3XpDdvbpmdmqTEacRj5cFZLkiOT6c0fG54JxENpm/XtlaRjrUwPtisekht/+22r965+Xt1vuRG2DPvo7nyRp9d5jmrsx/MoWoS742Xx99AGfTkTomTFrnX9BoKakQqSX9H9nrtXnH1+qo5W17V7zcIW/u/S+oyc9xejVsp2l2lFcqd+/vT2u9QII79v5Z2rM0D760cvrVBqHeWgAAF1PWiUVonfqo/ElYwfrC3mj9ZtbI6+5nGGmjAzThOHeV4DY+LNPtdrn5cnlU7df5vkanSG090f2gF5JigRIP3M3HfJc9nhgtYfP/fEDz8tiVtbUa09pVYt9wf/jq2ob/Puaho5FyCpsOejvHVBd39DuNZuqCPf38I++Av370yvarcPbFQAkUp+emXpk6iU6crxG33pulWo8/P8HAHQvJBXCaLpBDv5gnJWZod/8y4UaPbRPoEyY8wKffn/dRuLhmX+/XNNvPLt5u3dWbD+Cj549IqbzQvUL0+MiFp04PyWQUhL5fyeh8xG0Effx6joVV7SemK1p7gcvS0+2dew3c7dpwdZiVVTXtTvkAkDyXThmsB74woVasfuYfvLqxqQsx/v/27vv8Kiq9A/g3zMtyaSXSe+NkAABEgKhhIBIFVBExQqKgquuXRd118Luuvx03V13bcu6dl3sKyoLKhILTZQi0kPvoYRAAunn98eUzCQzmZJJJpn5fp5nnsy9c+69Zw4zw73vPec9RETUeXwyqGDtJP7fMwvx2ORci9et/Z/X2KRfqbESDDAOfyhKizCtG5gcZlGmJFuHW0dmmNXFsjIvXjvQbv3dyV2zVbTeC2MMRI7pzHPrz352bHiDUbWdxJPC4rntb3ldYzOKnlzu1LHNle2owJLN9uve7/Ev7CZ7tOWrbRUubUdErrmkXzzuHpOFD346hIXf7vF0dYiIyI18MqhgzUW9Y3DjsDQAZnfRrHSPbWzW3xVTWUmgoDC7QH/nlsF4eno/LDDrteCvbmluW9fyY/NinZqW7VelGfYLGTw93XYPCiKiPo8t69D2jt59tFdq1qvr8H35SQDAonUH2y37xdbjDh3TyDyQWnW+AZsPVbVTmojc6a6LsnBJvzgsWLodXzr53SUiou6LQQUr2rt576fSDxeIDvZv85p5nGFoRhSuKEyyCA98ec/IDtXrn9cXtFl3/9heVkpaZy25pNu0arRbnQh2EPmys/WOd1Uw5iToqONWhiY4y9rv5H83HHZoW2PswVrg9od2ZoRwt2teXoPJz33fZccj8nVCCPz5inz0SwjFXYs2uO03jYiIPMunggrOdjO2Vn5oRiT+cGkfPGoYKmGuvaEEmdFBSIrQmpaVpoRmjldqnBtne7goWT9tpbtmrDDfzcMTc3BlYZJ7dtxBM4tTPF0FIreZ8Ox3btnPtBdWdXgf1n46/v61Y9M/WgsmGH3wU/s9E9xpC6eVJOpy/molFt5QiBB/NW5+/UecOMcZIYiIejqfCio4ypSo0dprQuC6ISkI9FMhLz7EtD4+1B9XFCY6fIz/3j4Md4zKhEqpwAPjWvU2cNOFfoDads+EEE3LQeZPzevwsbprosYbhqZ6ugpEdu0+UY3mzkyq2MrhM65N02geOO3Id77irP4iwlpM1ZnhX0TUM8WE+OPlmYU4VVOHuW/+iNoGzghBRNSTMahgTTuJGs0VpoSbnq966CIkhmvbKW2pT0Io7jcEE24flWm3x8Bz1wwAAHxxTwn+eFkfq2U2PTbWYnl0qxkizN+P+QXBDcWpjlW6B2qvWS8f6HgQiKgzXfTMN5j/2VZPV8PCDa/80Gn7vv0dfXLFusa2Mzd01wAlEblXn4RQ/PXK/lh/4Awe+mgzZ4QgIurBGFSwwhPntKNzYto99iX94gEA2THBuHZwS5d+8/KhAWrLjRx4I50x+0N3utPY3vuLDfXrwpoQte+1Vfs8XQUL3+480WbdieqWbsrv/XjI6X0aLxnO1dqeZcJeYkYi8h4T+sbh/rHZ+HjDYXxU3uDp6hARkYsYVLDCeCHa3rhfc7eP6nhSwueuGYDvHhwFhcK5S3JjDa1dO3f00t5frUCvmGCbr//+0pYeE8agR1ca0zvGbpn22mBYZlSbddcOTu5AjYi8W72VngVERB1x+6hMXFWYhE93N2Dht7s9XR0iInIBgwpWmKaUdLAnni7I/h1ve936/NVKUyJHZ3oPGEtm6oLavJYQHmB3u/YoDVma8xND7e5Dq1Fi9vA0B/bqPtam9Wy93lpT/nZSb6x56CKrSTOemNLx/BJEZJ293529J2vcfszXVu7FrFc7bygHEXWMEAJPTuuLolglnlyyHe+sPeDpKhERkZMYVLBCmHIqdHzOdVdGF4zLs7wDHx/advpKI4VC4I2bivCfOUMs1r903UDcd7FlAkhnRytKAH0TQ/HJHcNtvu5JChufXvOpM631+7h5RDpibbSpSmn/KxHir3KsgkQ+7tWVe7Fq90nTsr3fjFF/Lmv39bP1EqnzPneqDo9/uhVlO06YEmF2n8FZRGSkVAjM6eeH0TnReOS/m/HJRsempyUiou7BJ4MK9i70TUEFNxwrxJDnoCgtwuFtLukXj7L7S/HNA6XYOn8cVjxQ2m75kmwdolr1lhjfJw4aleP/vKsfGt1mXet9AsCiVsELT1J0cka3gclhVtdntTMkhIhaPPHpVlzzr7XtlmlsanY48/v+KtczxNc2urbt+gOVePSTX7CV008SdSqVQuCFawdicFoE7n1vE77YcszTVSIiIgf5VFDhyWl9kRMbjJgQ23f+ASBMqwEAxNopZ9TepW10sD++urcET0yxPmODLalRgUiJDIRWo4KfyvbUkO4SF9p2qIS1AEJUkKZloVVPDuOiu6/1fzM+B9vmj8endwzHazcOMq0v7RXdzlb262IvaPTOLdYDKK5mqJ7UNw5pUYEubUvkra7511rk/G5ppx9HSv13t/J8vc0ytQ1NSJ33Od4zSxY57YVVeGP1fkz8+3edXkciX+evVuLlmYPQJyEUd7yzAd/vOml/IyIi8jifCiqMzNZh6d0lUNvp4l6arcPfrx6Ae8dmu+W4mdHBTvUa6IhvHxiF/901wuprzlwMZ+gCER9mLSdD5/QOMJ+es7WR2ToEaJTomxhqEUiICFTb3MYRYdr2t/dXWw/mPDSxt919zyxOabPumSvzHcq/QeRtPvzpEFLnfY6qC22zu/+w73SX1KFZSryych8e+OBnm2VOGma3+NtXO62+fvXCNZ1SNyJqEeSnwus3DkJaVCBueeNH/LS/a34jiIjIdT4VVHCUEAJT8uPt9hCYPTwd6bpATPLAzAe2JEdq0TsuxLRs3tvC/P24GhpINiSTBNyXUyEqSIMPfjXUYl1KpBaPXpKLfQsmITc+xOp2jia0jAzUWF2fFx+Kd24e7FRdfzM+B4NSI7DjD+PbLde6bmN6x+iDFBzQTT5o3kf6C/nz9a4NQaissd27wFESwIrtFe2WMX5vLzQ0WT3m6j2n8NTS7R2uC9knhBgvhNghhCgXQsyzUeZKIcRWIcQWIcQ7rV4LEUIcFkI81zU1JncK02rw5s1FiAnxw6xX12HLkSpPV4mIiNrBoEIHJEdq8fV9pdAFd9+7z2UPlGLdI2Pw8MQcjM21PwWjPeY9Lka1Gn7g6BScbelP5GNCWtpxdE40brIzm4TN6/NW1Vg5bzS2zh9ntehQK9NKWjMuVYU5Jem4cVgqALgwJEVarRuRL2hosv/BP3zmgs3XLvrLN1i37zRe+cX14IIjHbWMvymV5xsw4PdfWi3zQhmnvOtsQgglgOcBTACQC+BqIURuqzJZAB4CMExKmQfg7la7+T2Ab7qgutRJooP98dbNgxHsp8IN//6BeU2IiLoxBhW8nL9aCV2wH+aUZEBhNtWi8fy6I/kPkiK0uKowqWMVNKvD2ofHOLldS+U/v9PGDBVS3wZajeMzNvzuklykRGot1sVoFXh4Ym+bQyKseefmwbhzdKblftqZycOeCX1i8fjkXJs9L4h6smELvrb52umaelzx0mpU1jkWlattaGrbZbqdTY1Dwzo59ys5rghAuZRyj5SyHsAiAFNblbkFwPNSykoAkFKauqEIIQoAxAD4oovqS50kMVyLt24eDI1KgRkLV2P9gUpPV4mIiKxgUMFHdadzZ1frYtxuRFYU8uJDTevjwlou3F3pPTF7eBq+eWCU1WM5Y2hmFPokhFqse/Iy5xJ2mhvTOwazhrXfe4OIgEc/+QWXv7ga+0/VmNa58lvQ0NRsdb2ryVrJYQkADpotHzKsM5cNIFsIsVIIsUYIMR4AhBAKAM8AeKBLakqdLl0XhPfmFiM8UIPrXl6LleVM3khE1N04fvuWyI6W2R+cuwS3Vly0cxn/3YOjUNvQhKNVtRbHNXpr9mAUPbncqTp0lWD/jiWXJCL7th7Vd5M+e6HRtM5eHGDDgUr85kPLJI6LNx6xWrZZAsruFJn1PtZat/W/oApAFoBSAIkAvhNC9AFwHYAlUsqD7f1fJISYA2AOAMTExKCsrKzjtTZTXV3t9n16O3ttdk9fiT//2IyZr6zF7f39MCCap7D8nDmH7eU8tpnzfLXN+IvsY2YMSsKidQetvjazOAWLNx1B5fm2Gdq7WntxiSRDssjjZ/WZ2lvfgYwO8UdKpBb7T513aBx1R+vjjvLO7JP3SImse33VPswcmmr1tR/2ncb3Nu5wSgnc//4m7D5RY7H+vvc3WS3f1Czx9LIduL44BWqFQKCfCoF+/O/UjQ4BMB9blwigdYTnEIA1UsoGAHuFEDugDzIUAxghhLgNQBAAjRCiWkppkexRSrkQwEIAKCwslKWlpW59A2VlZXD3Pr2dI202ckQ9Zr36A57beBZ/ubI3pvZv3YHFt/Bz5hy2l/PYZs7z1Tbj8AcfY7rItnLR+8TUPnj/1mLX9ut6ldrtldCe1Ch9cGFcXqyVfeq56wJ8aLxrFwzGO2WOBDe2/779GSWM2PWayLrHFm+x+dpL37gvweKWI1V46ZvduP3t9Sh6cjmmPr/SbfsmAMA6AFlCiDQhhAbADACLW5X5L4BRACCEiIJ+OMQeKeW1UspkKWUqgPsBvNE6oEA9V3igBm/fMgSFKeG4+92NeGftAU9XiYiIwKCCz+voDXXrQxf0RudEt33RimkDXbvTkBiuxbb543H9kBSbZdx1Aa5xsq+zK4ESZ5JAdlT/pLBOP0Z+Yqj9QkRutu1oS4b4yc99b3ruzlhcU7N+Z8acC+UV1e7bOUFK2QjgDgDLAGwD8J6UcosQYr4QYoqh2DIAp4QQWwGsAPCAlPKUZ2pMXSnIT4XXbyrCqF7RePjjzVj4LWdkISLyNAYVyEJ8WAAA4O4x2Q6VN04xqVS0vYguydZhz5MT7e7j/rG92qxz9JI8QKO0yOHQKyZYv70T4w+W3DnC4bJdJbidrtSOXhvlxAa7pzJ22AoKWftMEHW2V1futRpAcGf/HvYV6nxSyiVSymwpZYaU8o+GdY9KKRcbnksp5b1SylwpZV8p5SIr+3hNSnlHV9edOp+/WomXrivApH5xeHLJdjy1dDuam/nNJCLyFAYVfIy9DOhajQr7FkzC5Px4h/Z3/7heuHVkBi4bYP3CUtHqwvLbVrMqWCsDuJaTYOndI/CeYfiGM8MfcuNDbL72n1uG2Aw6/OeWIU7W0HGpUYFt1kUG+Tm07b9uKMTOP0xAcUakzTLB/u4f//3bSb3dvk8it2mnq0K9jVke7O2KI5GIPEejUuDvMwbg6qJkvFC2G79etAG1DU2erhYRkU9yKKgghBgvhNghhCgXQrQZmyiEmCWEOCGE2Gh43Oz+qlJ3FOKvxrwJOVArWz5KrQMCq+aNNj1PjtR2Wl1yYkMQGmCYXcGY1LCDJ/3FGZE2gw7FGZGYMSjJ6msdTdR40/DUNutGZusAtH1Plw9MND1/8dqBuDg3BhqVwuoQjAl9YvHwxBxoNa4PtfjbVf2x6bGxSDD0ajEK02oslp2dBcTI+D6JXGXto7fnZE3blQYD5n/p1P6Z14Soe1AqBJ68rA8empCDJZuPYsbCNag4V+vpahER+Ry7QQUhhBLA8wAmAMgFcLUQItdK0XellP0Nj5fdXE9yk/5J4QCAuMCu66QS3+ri05xx+ETn6dyTf3vXza7GFi41ZLQenhnV5rWOXNC8eF0B5pRkIDWybU8IR4UGqBEaoMaSO0dg+X0jXd6PLbpgx3pkEFnz3o+HUFnTdgabc7WNVkrrXXDy7qaxlzVDC0SeJ4TA3JEZeOm6Auw4dg6XPb/KIrcKERF1Pkeu6IoAlEsp90gp6wEsAjC1c6tFneXqoiSU3V+KrHD9nWpH7yYPTLaf2M/Zi93nrhmAL+4usfraqF6OJXm0xTT8oZPP+vslWrbL3JJ0i+UBhnabPSLNqf0KIVD+xwl446YiB8paXx/UaoiDeY6D+6zksXBWqFaNDF0Q7hmTjaK0CIzNi3FouztHZ7b7OjMxUEcdPnPB6W2c6VljbxgZEXW9cXmxeP/WYjQ2N2P6i6uwYnuFp6tEROQzHBlYnQDgoNnyIQCDrZS7XAhRAmAngHuklAdbFxBCzAEwBwBiYmJQVlbmdIXbU11d7fZ9equGCzUABPqGNzvUZrf2kriQobUo23q7Q4frAADl5eUoa9hvdT/m2wSd3ol9p4F9VsrVH/oFZYfsVsumC+fPAwB+WLcOh4MsY2ftvV9br9n6bMVKiSeG+uOxVfruloE1hwEAyrNHUFamP6F5bXwg6g/+grI23wjbx7Z2LOO6xkbLO67Hjh0zPf9lyxYEnNoBAOijsLzwEZBu+X78vPlniGOWPx239QLWr7GcVq+qqsrq9kcPWf9sGJm/H6Kucr7G9vCI1q7511oA+t8FI/7fQ+R5fRJC8cntw3HzG+sw+/V1+N0luZg1NNXl4XhEROQYR4IK1n6JW9+m+RTAf6SUdUKIWwG8DmB0m42kXAhgIQAUFhbK0tJS52prR1lZGdy9T29VVlaGH39bjLAANVRK54YgLM+rRrCfCtEh/hbro7KqsPwf32PuJcPa5k5Y+jkA6P99zJ+31t5rTsjdvw5HtlWgZOgQJEVo7e/bznHb+2zVNTbhsVVLAQB3XjEGE0dWI0MXaPskxnAso79d1R+f/XwUpaWFGLxjNbYePWt5rFZ1U5UtA8wCC7GxscBhfQTm15ePspiaUrV8CRoNfbUVCoXV/doza2gqXlu1z7Tct29flObY6JVgts+QkBDgzJk2RdLS04GdO2wez/z9EHUVbWAgUOPc1JBBQUHAOX03a/7fQ9Q9xIb64725xbh70UY88elW7D5Rjccm51nkfiIiIvdy5Bf2EADzbHSJAI6YF5BSnpJS1hkW/wWgwD3Vo84UFeTndEABADJ0QW0CCoD+DsG+BZOsJmMMMeuKn59kfyhFR/3lqv741w2FLQEFO/5waR+8fbO1Djj2GRMiGkcXZEYHOXVXZHJ+PF6eWQgAeHduMTY/Pq7d8u11vDYPKADAd78ZhXfnOD5LhVqpr/fPj4/FzOIUAEBiuO2cGK4wDklJirC+X0/fULKVfJOoNSZsJOqetBoVXrquAHNHpuOtNQdw1T9X48Cp856uFhGR13Kkp8I6AFlCiDQAhwHMAHCNeQEhRJyU8qhhcQqAbW6tJfV4X947Eocq9f+hf3BrMZo6eT7pEH81Ls51bIw/AFw3JKXDx1S4eDXs9Fatmi7MOOOFFXGhAQhvNSuDLf0SQ/HmTYNR39SMEH81lArrAae8+FCHq9qeS/rFo7KmHovWWY4LERD4/jejMPz/VrjlOM5K17mexJJ6rvIK53opEFH3plAIPDShN/LiQ/HIx5sx4dlv8fiUPEwvSORwCCIiN7N7m1pK2QjgDgDLoA8WvCel3CKEmC+EmGIodqcQYosQYhOAOwHM6qwKU88UE+KPgpQIAIBaqWhzR92crTvY3ZUxaZurQQV7itIiLJYvL9BPIbl1/jj8+Yp8XDYwwaX9rn34Igw27PvXozOxaM4QhGrV7c6+sG/BJMRY6aViTe8461NxmltweT/s+MP4NusTwztv6lGjjkyrSUREPcOU/HgsvbsEfRJC8cAHP+P2d9ajsqbe09UiIvIqjvRUgJRyCYAlrdY9avb8IQAPubdq5Ivemj0Y2bFBnq6GSxwdJvDnK/Lx86EzeGO1PmGhvVjEGzcVoepCyxR5j16SiwfH94JWo8L0gkRsPeLa1FkxIf54dHIupjy3EtcMToZWY/lz0DsuGIB+uIsrfndJLkZm6zDnzZ+svu7OGToC1EqnpwX88FdDUd/YjO/LT+LpZbZzPBARUc+WEBaAd24Zgn99twfPfLEDP+2vxDNX9MfwrLbTNhMRkfOYtYa6leFZUYgOduxOeHfhp1LixWsH4j8O5i6YXpCI+VP7mJbtdcP0VystegcoFKJNAMAhVi7e8+JDsfvJiYgLbRsQmV6QiKV3j8CoHMen9zTmZBidEw1/tRLB/m2HZhintlQprL9vRzt8XD4w0fT8hmL7w1fSo1qGNUwvSETvuBDkJ4Uhg8MdyEVMqUDUcygVAreOzMDHtw1DkJ8K1/17LX7/2VbUOhmQJiKithhUIHKDCX3jHB4W0FMIIZATa38Ig7mIQH3+hiem5AEAhqRbDt149JJczCxOxY3DUjF3ZIbhOI7v/7bSDNNzjaplQ3vXdvsWTMLX95ci1vBvdO/F2abX/FS2h0E4MoQDAMY6kb+DvMeRMxc8XQUiclKfhFB89usRuKE4Bf/+fi8ufX4l1h+o9HS1iIh6NAYViDwkLtS7ghDmjFN3te6Fcd2QFARolHhsch4C/drvbWE+Y4jRjEHJpucRgRrkxOqHaDiahd+Y/8K8WiOzdXhkYm/TrA/CLHUmU3lRe5rZVYGoRwrQKDF/ah+8OmsQKs/XY9oLq/CbD37GaeZaICJyCYMKRB7y0W1D8dJ1Azv9ON0pybW1uihbrTQufnXfSIS2mtlCtuqTMM2BJJWPTc41PR+eqQMAi+EjCoXALSXpuMoQVBjZS2d3n63x0tI3NTTxX56oJxuVE43l95ViTkk6Plx/CKP+XIa31+7v9BmqiIi8DYMKRB4SFxqA8X3iPF0Nj1MpFVg1bzQeN7v4B4DoYH9M7Nt++xh7FbR3w1hplrvhyWl9UHZ/aZtgBQAMSA7HvgWTkB0T7ETtyZfVNzV7ugpE1EFBfio8PLE3ltw1AjmxwXjk418w7YWV+PnQGU9XjYiox2BQgXzGk5f1xXtziz1dDbeLDrE9BaS7PDElD3++Ir/T9h8fFgC1qu3Pkb1eFsbXHb2p5KdSIjXK/YkZu1FnEAvzp+Z5ugpERD1CdkwwFs0Zgmdn9MeRqlpMfX4lHvl4M86c55AIIiJ7GFQgn3HN4GQUpUXYL9jDRAW1H1RQGK68EyMcm/LSmplDUzG9INFuOeOMDMFW8iHYE2TIsRCm1Ti8jTF3g9osaeM9Y7ItynRk2HvrTZU2ZqyID3O9bc1dMzjZfiEncMg/EZHjhBCY2j8BX983EjcOTcOidQcx8ukyPL+iHDV1jZ6uHhFRt8WgApGXUysVeOm6Aiy6xbEpLzvi/rG98M+LtTaTMLZ3kTu5XzzmT83DXRdlmdaZX8IH+anazLAxoygJc0vScefoLNjiaBJHazTKlhosvXsEVs0bbfF6aIAa3/9mFNKs9H6YNsB+vofWguwkr7SlT4Jzs3QQEZFtwf5qPDo5F5/fORyDUsPx9LIdKHlqBV7+bg+noCQisoJBBSIv8OyM/nj5hkKbr4/vE4voLpjyUqEQ8FPaHgzQOtFi621vKE6Fv7pliscrC5NMz7+6dyT81Up8dW8JAOCSfvHwUynx0MTeFkEMdySmHNM72rCvlp3lxIa0CWoIASSGa9tsn6ELxF+u6o99CyY5ddwwrRpf3FNi8/UEsx4RNxSnmJ6Pzmk7peXckvQOBVQcdWWh/R4sREQ9UU5sCF6eOQgf3zYUufEh+MPn21Dy1Aq8sXof6hoZXCAiMnLtthgRdStT+zt/V7wnyE8KMz2PNUzBmRkd7PTFurNuG5WJr7ZV2C1nvGbXmOWDWDVvNEKsJIJ0VHuJIs2DBDqzYS8BZoGYYD8VztU1IjM6CNVd0F03LSqo049BRORJA5LD8ebswVi75xSe+WInHv1kC/75zR78enQmLi9INA3FIyLyVfwVJKIuMW1gAjQeOPHqzHv1xot8P7OgQnxYQLvDGIpSI/CMjaSX9joWNJkVMO+RcdPwVNPzdJ1+KIYuuPMTeALAxblte0kQEXmjwemReHfuELw5uwi6YD/M+2gzpr+0GgdOnfd01YiIPIpBBSLqEn+5sr/FcILO4s4jOBqQCNM63jPhsoHO9Sp57poBpue2ZrnwUykxprf+4v5XpRl446YilPaK7pSZLlrLjGZPBSLyHUIIjMjS4ePbhuIfVw/A3hPVmPj37/DJxsOerhoRkccwqEBEXs2VtALm7GX/3gAAHnZJREFUgYmnp/fDp3cMNy1bmwFiVK9om/v67aTeAICYED/sfnIiZgxKapP3YUp+vKGubStr3rvD/PW2uRz0rymEQEm2zm69iIjIdUIITM6Px5K7RiAnNhh3LdqIB97fxFkiiMgnMahARF7FeHc+2MWZFFq7ojAJfRNDTctf3lOC+VPzLMq01wPj5hHpeGp6P3xw61AoFQJCCEzsG2dRxsZMlSazhqYCAJoMXRUKUsJNOSZaa12XmBD3DoN4dkZ/t+6PiKgnSwzXYtGcIbhzdCY+WH8Ik//xPX45XOXpahERdSkGFYioU707Z0i7M1O42yX94vDBrcW4vMD1WQmMvQAutzJUIV0XhKn5+vWOdoK4sjAJSREtPQvMZ7gAWgIB1npVCCFMs2AMy4wCAIvZLorSImxuCwCf3jEc79wy2MGa2pcXz+kriYjMqZQK3Du2F96+eTBq6hsx7YVVeOX7vV0yAw8RUXfAoAIRdarB6ZEY04XJ/IQQKEyNMC27ckqnC9YPVbh+SIr1AsbOAG46X4wM1LS7u9z4EGz43cV4cFwOAGCslfYMNcw44a+2/FmPDvHH0Iwop+rz4PheNl/rirwYREQ90dCMKPzvrhKMyIrC/M+24qbX1uG7XSdQ39js6aoREXUqTilJRF7JeO3r6p0ia7kTjAI1+p4GD03s7dK+zb0yTgt1og4vf78XhSnhNsuFB2oQHqjBlifGQatRYldFNQCgl2EKysen5iEvIRTDMx0LIARqlKip18+znp8Uhk0Hz5he65sQanUbrUZp0SPivouzHToWEZGviAjU4OWZhXh91T7839IdWLHjBIL8VBiZrcNFvaMxqlc0wg2BZCIib8GgAhF5JeHWeSAsqZQK7FswyS37EgBGZOnw8+NjEeJvfxYJ49CH7JhgvDe3GPlJ+gBAiL8as4enOXzc/KQwrNp9CgDw0a+GIuPhJXa3yY1rGfqQrgvEry/Kcvh4RES+QgiBWcPScNWgZKwsP4nl24/jq20V+HzzUSgEUJgSgYt6R2NElg4Z0YHwUynt75SIqBtjUIGIqBtwJKDQmjGfgitevLYA+fO/QGyIf5teGf0Sw6xu8+9Zg3DiXK3LxyQi8iUBGiXG5MZgTG4M/tgssflwFZZvO44vt1XgT//bjj/9bzsUAkiJDESGLghZMUHINPzN0AVZ5M8hIurO+GtFRF4jJzbY01VwmrUcBU9MycNji7cAsD8zhCuGZUYiVKvG1vnjoGh1/MhAjSk/Q2uhAWpUnNUHFZhZgYjIcQqFQH5SGPKTwnDv2F44fOYCftpfifKKapRXnEN5RTW+2VmBhqaWMWaXDUjAn6b1bZPcl4iou2FQgYi8wubHx0Kt9I7cszOHpmJC31i8VLYHI7N1nXYcrabtfwFzStLb3cZ4utuZCRujgjQ4WV0PP5UCdUxwRkReKCEsAAlhARbrGpqaceD0eew6Xo11+07jlZV7sftENRZeX2hzGmEiou7AO87AichrPTwxB6N62b+wDvZXW9zNaUnU2Fk165hpVqarNBcd7I9HJ+dC1QmBkvbyTQyzk+gxwNDGKWZTZLrbY5PzALT8G9oyOT++0+pARNTV1EoFMnRBGN8nFr+7JBcLry/E7opqTHnue4tkukRErVXXNeJ0Tb3Hjs+gAhF1a3NKMvDqjUVObxcd7AcACNM6n6ugKzx1eT/8/PhYT1ejjT6GmR/G9I7BI2azW3xxTwkAIClCi39eX4C/zuhvem3BtL5t9jNraGq7x7l9VIbV9V/eU4LROdEAYDE047bStuUjuum/LRGRO1ycG4MPbxsKjUqBK/+5Gp9sPOzpKhFRN1N1vgHPfrULwxZ8jWe+2OGxejCoQEReafbwNPz1qnxcPjDR01WxSqVUuJScsau8PLMQt5gNhciOaclXMS4v1qLuM4qS22z/yCTXptsUQj915ezhaXhvbrFp/YPjc0zPX7txEJ6/ZqBL+yci6klyYkPwye3DkJ8YhrsWbcQzX+xAc3M37YJHRF3mVHUdnlq6HcP+72v89audGJQagSsLkzxWH+ZUICKvpFIqcNmA7hlQ8EZf3TsSfioFRjy1AgDs5rewNSwlNTIQQgj87pJcm9uW9tL3ZPhh7ynXKktE1INEBvnhrZsH43f//QX/+LocO4+fw1+u7G9/QyLyOhVna7Hw2z14e+0B1DY2YWLfONxemonc+BD7G3ciBhWIiKjDMqODOryPD24tdiqHhFLBznZE5Bs0KgUWXN4X2bHB+OPnWzH9pdW4KrXJ09Uioi5y+MwF/POb3Vi07iCamiWm5sfjtlEZyIzuHjOfMahARORjOnHiBodJAEVpEfhh72nTusLUCKf2cffFWWiWEq+t2ufeyhERdUNCCMwenoYMXSDufW8THl9dj3XV63Hv2Gxk6Doe2CWi7mfPiWq8WLYbH284DCGAywcm4lelGUiJDPR01Sw4FFQQQowH8CwAJYCXpZQLbJSbDuB9AIOklD+6rZZERNQjfHTbUIQGOJYr4r25xUid97nD+57dR4OhBf1MyyH+ajw+JY9BBSLyKaW9ovHNA6X47Zsr8OWOCizdcgxXFCTirjFZiAsNsL8DIur2th09i+dXlGPJ5qNQKxW4bkgKbilJbzMVbXdht++oEEIJ4HkAEwDkArhaCNFmsKsQIhjAnQDWuruSRETkOVlODG0YmBxu9Y7Z3j9NtFh2ZarPEYlqXNQ7xs7xw5zfMXUrQojxQogdQohyIcQ8G2WuFEJsFUJsEUK8Y1jXXwix2rDuZyHEVV1bc6KuE+yvxmVZGnz74CjcUJyCj9Yfxsiny/Dkkm2o9OC0ckTUMRsOVOLm19dhwrPfYcX2CswpycD3vxmNx6fkdduAAuBYT4UiAOVSyj0AIIRYBGAqgK2tyv0ewFMA7ndrDYmIeoDFdwxDQ1PPzcj92a+HY+uRs23W7/jDeIupHV0lhMCbs4vw9poDWLrlmGl9dkwQdh6v7vD+bdk6fxzW7DmFe97dhKoLDVbLBKiVuNDAscndgdmNjIsBHAKwTgixWEq51axMFoCHAAyTUlYKIaINL50HcIOUcpcQIh7AT0KIZVLKM138Noi6TFSQHx6bnIebhqXhb1/twsvf7cF/1h7A9cUpGJGlw4DkMPirlZ6uJhHZIKXEvlPnsbL8JP73y1GsLD+F0AA17hmTjVlDUxHaQ6bPdiSokADgoNnyIQCDzQsIIQYASJJSfiaEYFCBiHxOv8SefYe8T0Io+iSEtlnvp3L9ZPSNm4pwwys/mJZHZOmw+XAVlm45Bgl9AKZ/UlinBhW0GhVG58Tgy3tKUPTkcqtl/nv7MIz727edVgdyiiM3Mm4B8LyUshIApJQVhr87jQWklEeEEBUAdAAYVCCvlxShxTNX5mPuyHT8edkOvPjNbrxQthtqpUB+YhgGpUWgKC0CBSnh3Xo6YyJfUHG2Fit3n8TK8lNYVX4SR6pqAQAJYQF4aEIOrh2SgiC/npX60JHaWrtFZbodJ4RQAPgrgFl2dyTEHABzACAmJgZlZWUOVdJR1dXVbt+nt2JbOYft5Ti2lXM80V6Vpyu7/JjG4ylP63sEaKsPo6zsOPzON7QpY2t7R9oqSV2D9VaOe6au2eY269ats1gO8xM4U+dcrxN+5t3G7o0MANkAIIRYCX2up8ellEvNCwghigBoAOy2dhCej3Q/bDPn2Wqza5KBqXFa7Kpsws7KZuw4XYWF31TixbLdEACSQxTI1ykxPEGFaK3vzKLDz5jz2GbOs9ZmUkrsPtOMH4414peTTThSoz/HCFQDvSOUGJOgQW6kEjFaASEP4sfVB63suXtzJKhwCECS2XIigCNmy8EA+gAoE/ousrEAFgshprRO1iilXAhgIQAUFhbK0tJS12tuRVlZGdy9T2/FtnIO28txbCvndFl7LdUnROyXGIoF0/p13XzGhuMa32MpgKsnNpjulI2UEm8+tMSijK3tbbbV0pZkj3+dfTE+eXiJadlYvuJcLbDCek+FQYMGASv1PRX2LZiEs7UNqDhbhzF/+cax92ir7uSKdm9kGKgAZEH/cUoE8J0Qoo9xmIMQIg7AmwBmSimtRpN4PtL9sM2c50ybna9vxMYDZ7B272ms2XMKn+45jcW7G1CcHokrByVifF4cAjTePUyCnzHnsc2cZ2wzKSW2HT2HxZuO4NNNR3D4TC00KgWGpEdhZkYkhmVGITcuBApFN5iSyw0cCSqsA5AlhEgDcBjADADXGF+UUlYBiDIuCyHKANzP2R+IiLqfxXcM93QVLLreCjfka/jk9mGY+vxKBGqULv3nHBvq36Z+7B7sMfZuZBjLrJFSNgDYK4TYAX2QYZ0QIgTA5wB+K6Vc0xUVJuoJtBoVhmZGYWim/pT9yJkL+PCnQ3j/p0O4591NeNRvCyb3j8eVhUnITwx1y28zkS86VtOMZ7/ahcWbDmP3iRooFQIjsqJw78XZuDgvxmvPL+wGFaSUjUKIOwAsg76b4StSyi1CiPkAfpRSLu7sShIRUcd8cGsxjhrG7HWl928tRtV56wkS3SXDxuwUv7+0j+m5sHoDHFg1bzRCA9S4dnAyooL87B5rYHIY1h/gEP1O1O6NDIP/ArgawGtCiCjoh0PsEUJoAHwM4A0p5ftdWGeiHic+LAC/vigLt4/KxNq9p/H+jwfx0fpDeGftAWTHBOHm4em4bGAC1ErfGR5B5IzTNfUor6hueZyoRvnxczhSVQshdmJwWgRuGp6GCX3iEBGo8XR1O51DGSCklEsALGm17lEbZUs7Xi0iInKnwtQIjxx3kIeOC1j2o7d20216QSLiDdMz/fGyvo7tk3fvOpWDNzKWARgrhNgKoAnAA1LKU0KI6wCUAIgUQswy7HKWlHJj178Top5BoRAozohEcUYkHp+ah882HcVba/bjwQ9/xrPLd+FXpRm4ojCxQ0l7ibxBxdlaLN50BF9tO45dx6txymzqVn+1Ahm6IAxKi0Bg7UncedmINr0gvV3PSitJRERe57sHR+FkdV2XHzcx3Ln5nifnx0OjVOCn/ZWdVCMC7N/IkFJKAPcaHuZl3gLwVlfUkcgbhfircc3gZFxdlIQVOyrw9+Xl+O1/f8FzX5fj1pHpmFGUzOkpyaecr2/Esi3H8NH6w1hZfhLNEsiNC8GY3jHIjA4yPRLCAkzDL8vKynwuoAAwqEBERB6WFKFFUoTW6mvv3DIYSeHWX7PHXqeCgpRwp/b3j6sH4MEPNrVZnx1jffgFEVFPJITA6JwYjOoVje/LT+Ify8vx+Kdb8dyK3ZhTkoZrB6cgsIdNd0dkVNfYhJPV9VArBTRKBdRKBTQqBVQKASEEmpolVpafxMcbDmPZlmM4X9+EhLAA3FaaiUsHJCDTxpBLX8dfBCIi6raGZkTZL9RKVnQQdlVUQ2sjk3lCWACW3DkCoVrnkyXdPCId7/14yGLd1UXJTu+HiKi7E0JgRJYOI7J0WLPnFP7x9S48uWQ7nv1qFwamhKMwJQKFqeHonxTGIAN1exfqm/D22v146Zs9NntHapQKQAD1jc0I9ldhav94XDYgEYUp4V4zS0Nn4S8AERF5ld5xIdhVUW2RnDFca5kkyZWAAgBkxwRj34JJSJ3XMo0lTzOIyNsNSY/EkPRI/LT/ND7ecBg/7qvE35bvhJSAUiGQGxeCgpRwDEqNwKC0cEQH+173b+qeWgcThmVG4t6+2WiWEvWNzWho0j/qmyQamprR2NSMgcnhGJUTzeE+TmBQgYiIejStWolRvXSYPTwdACAN682HPyhduMMwPi8Wlw6Ix61vrW+3XEm2zul9ExH1RAUpEShI0SfgPVvbgPX7K/HT/kqs23cai9YdwGur9gEAMnSBKM6IxNCMKAxJj/SJ7PfUvVgLJrxw0UAUpXkugbQ3Y1CBiIh6NIVC4NUbi0zLefEh+HTTESSEOZeIsbWXri9wqFy6juMricj3hPirUdorGqW9ogEADU3N2HLkLNbuOYXVe07h4/WH8daaAwCAnNhgDEmPNM00EeLvWm8xovbUNjThwOnz+HbnCVMwYWhGJJ6/ZgAGp0d6unpejUEFIiLyKnNGpGN4ZhT6JIR6uipERD5DrVSgf1IY+ieFYe7IDDQ0NePnQ1VYs+cUVu8+ZerJoFQIFCSHY2QvHUZm65AbF8Lx6uSwmrpG7D91HvtP1WCf6W8N9p86j6NVtaZyDCZ0LQYViIjIqygUggEFIiIPUysVKEgJR0FKOG4flYm6xiZsOHAG3+06gW92nsDTy3bg6WU7EBWkQUmWDiN76TA0Iwq6YD9PV508SEqJyvMN2HeqBgdOnTcFEPaf1v89WV1vUT4qSIOUSP1wm9TIQKREapEdE4zecSEeege+iUEFIiIiF/VLZPCCiMgRfiqlKeHjA+NycOJcnSnAsGJHBT7acBgAEK5VI10XhAxdINJ1QUiP0v9NidRCrVR4+F2QO0gpcfxsnaGHQY0hcHAe+0/XYP/J8zhX12gqKwQQF+KP5EgtxvSOQXKkFikR+uBBSqQWwRxK0y0wqEBEROSixXcM93QViIh6JF2wH6YNTMS0gYloapb45XAV1u07jd0narD7RDW+3n7CYgpflUIgQxeEAclhGJgcjgHJYcjQBXHoRA9wvr4Rmw5WYcPBSqzffwYbD1Za9DhQKQSSIrRIjtCiIDkcyZGBSInQIjVKi8RwLWdh6AEYVCAiIp8ipbRfiIiIuoxSIZCfFIb8pDCL9VUXGrDnRDX2GAINW46cxZLNR7Fo3UEAQLCfCv2TwzAgORwDksKQHRuMuBB/Bho8qLquEXtOVGPX8WpsPHgG6w9UYvuxc2hq1v/fmx4ViJHZ0chPCkVaVCBSIwMRF+oPFXuh9GgMKhARkU957aYi+4WIiMjjQgPU+oBBcrhpXXOzxN5TNdhw4Aw2HKjE+gNn8NzXu2C4ZoWfSoHUyECkRmmRGhWItMhApEUF4kxdM6SUEIIBh9bqG5tRU9eIasOjpq4R5+oasf54Ixq3HodKKaBSKAx/BVRKBVQKgZPVddh9osYi8FNxrs6030CNEv2Tw/CrkRkYmBKGAUnhCOf0ol6JQQUiIvIp2THBnq4CERG5SGEYBpGhC8L0gkQA+hkBNh+uwu4T1dh3sgZ7T55HeUU1Vmw/gfqmZtO283/4CjmxwciJDUFOXDBy40KQGR3UbbvXn69vxMlz9RAC0KgUUCsVUCsF1EoFNEoFFAqBhqZmVJyrw7GqWhyrqsXRqgs4frYWR6tqcfxsLWrqmtDULNHY3Gz4K9FkeDQ0NaOmvgn1jc22K7HhR7v1DA1QI10XiBFZOmREByI9qiUnhpK9RnwCgwpEROQzgv343x4RkbcJ9FOZkkCaa2qWOHLmAvaerMGyVRvRFByDbcfO4Z0f9qO2QX8hrRBAWlQgYkP90dgkTRfejc3NpuWmZomoID+kROp7P6REapEaGYjkSC1C2kkU2NQscaGhCbUNTahrbEZ9YzPqGptQ19CMOrPnp2rqcNQUFGgJDpytbbS5b0A/bKRZSrQe1eevViAuNAAxIX5ICA+ASiGgNHvol/W9DbR+SgT7qRDop0KQ8eGvX968cT0GDixEg6EtzNukoakZYVoNMnSBiAjUsAeIj+PZFRER+YRNj46FSsmTHiIiX6E0JABMitCi+YgapaX9AOgv9g+cPo/tR89i27Fz2Hb0LE5V10GlVMBPrYDWcMGt7+ovIITAibN1+GbnCbz/0yGLY0QEahAfpg9I1DY04UJDEy7UN6G2odmil4Q9QgBRQX6IC/VHSqQWQ9IjEBsagKgg/XCBhib9hXxDk36/DY36ZYVCIC7UH7Gh/vq/If4IDVC75SL/7B4l+nKWI3IAgwpEROQTQrWuTTv14a+KsWbPaYt1L99QiNrGJndUi4iIuphSIZAWpc+1MKFvnFPbnq9vNEyBqJ8Kcd+p8zhWdQFqpQIBGiUC1Er4Gx4BaiUCNAr4qZTwV+v/+qn0gQvjc41KgYhADaKD/aFRMVkh9UwMKhAREbWjICUCBSkRFuvG5MZ4qDZERORJWo0KveNC0DsuxNNVIeo2GA4jIiIiIiIiIpcwqEBERERERERELmFQgYiIiIiIiIhcwqACEREREREREbmEQQUiIiIiIiIicgmDCkRERERERETkEgYViIiIiIiIiMglDCoQERERERERkUsYVCAiIiIiIiIilzCoQEREREREREQuYVCBiIiIiIiIiFzCoAIRERERERERuYRBBSIiIiIiIiJyiZBSeubAQpwAsN/Nu40CcNLN+/RWbCvnsL0cx7ZyDtvLcb7UVilSSp2nK+ELeD7SbbDNnMc2cw7by3lsM+d5W5s5dD7isaBCZxBC/CilLPR0PXoCtpVz2F6OY1s5h+3lOLYV9RT8rDqPbeY8tplz2F7OY5s5z1fbjMMfiIiIiIiIiMglDCoQERERERERkUu8Laiw0NMV6EHYVs5hezmObeUctpfj2FbUU/Cz6jy2mfPYZs5hezmPbeY8n2wzr8qpQERERERERERdx9t6KhARERERERFRF/GKoIIQYrwQYocQolwIMc/T9fEUIcQrQogKIcQvZusihBBfCiF2Gf6GG9YLIcTfDW32sxBioNk2Mw3ldwkhZnrivXQ2IUSSEGKFEGKbEGKLEOIuw3q2lxVCCH8hxA9CiE2G9nrCsD5NCLHW8N7fFUJoDOv9DMvlhtdTzfb1kGH9DiHEOM+8o84nhFAKITYIIT4zLLOtbBBC7BNCbBZCbBRC/GhYx+8i9Tg8H7HPmXMV0nP2nIWcP28hPUfPXaiFM+cw3qzHBxWEEEoAzwOYACAXwNVCiFzP1spjXgMwvtW6eQCWSymzACw3LAP69soyPOYAeBHQfwkAPAZgMIAiAI956RehEcB9UsreAIYAuN3wuWF7WVcHYLSUMh9AfwDjhRBDAPwfgL8a2qsSwGxD+dkAKqWUmQD+aigHQxvPAJAH/Wf1BcN32BvdBWCb2TLbqn2jpJT9zaZh4neRehSejzjsNTh+rkJ6zp6zkPPnLaTn6LkLWXL0HMZr9figAvQnj+VSyj1SynoAiwBM9XCdPEJK+S2A061WTwXwuuH56wAuNVv/htRbAyBMCBEHYByAL6WUp6WUlQC+RNv//Hs8KeVRKeV6w/Nz0P+AJoDtZZXhfVcbFtWGhwQwGsAHhvWt28vYjh8AuEgIIQzrF0kp66SUewGUQ/8d9ipCiEQAkwC8bFgWYFs5i99F6ml4PuIAJ89VCC6ds/g8F85bfJ6T5y7UPp/7bnpDUCEBwEGz5UOGdaQXI6U8Cuj/UwIQbVhvq918rj0N3c0HAFgLtpdNhi5xGwFUQH/BthvAGSllo6GI+Xs3tYvh9SoAkfCd9vobgAcBNBuWI8G2ao8E8IUQ4ichxBzDOn4XqafhZ9B1tr7v1IqD5ywEp89byLlzF2rhzDmM11J5ugJuIKys45QW9tlqN59qTyFEEIAPAdwtpTyrD8paL2plnU+1l5SyCUB/IUQYgI8B9LZWzPDXZ9tLCHEJgAop5U9CiFLjaitFfb6tzAyTUh4RQkQD+FIIsb2dsmwv6q74GaRO5cQ5C8Hp8xaf5sK5C7Vw5hzGa3lDT4VDAJLMlhMBHPFQXbqj44auwTD8rTCst9VuPtOeQgg19P85vy2l/Miwmu1lh5TyDIAy6Md1hgkhjMFJ8/duahfD66HQd3f1hfYaBmCKEGIf9N2fR0Mf/Wdb2SClPGL4WwH9iV8R+F2knoefQdfZ+r6TgZPnLGTGwfMWX+fsuQsZOHkO47W8IaiwDkCWITupBvrEZos9XKfuZDEAYxb0mQA+MVt/g9AbAqDK0D1nGYCxQohwQ5KzsYZ1XsUwTuzfALZJKf9i9hLbywohhM4Q6YcQIgDAGOjHdK4AMN1QrHV7GdtxOoCvpZTSsH6G0M94kAZ9sr0fuuZddA0p5UNSykQpZSr0v0dfSymvBdvKKiFEoBAi2Pgc+u/QL+B3kXoeno+4ztb3neDSOYvPc+G8xae5cO5CcOkcxntJKXv8A8BEADuhHyv1iKfr48F2+A+AowAaoL9jMhv68VDLAewy/I0wlBXQZ6neDWAzgEKz/dwEfVK4cgA3evp9dVJbDYe+C9fPADYaHhPZXjbbqx+ADYb2+gXAo4b16dBf6JYDeB+An2G9v2G53PB6utm+HjG04w4AEzz93jq53UoBfMa2areN0gFsMjy2GH/D+V3koyc+eD7iUBs5fK7Ch6nNnDpn4cP58xY+LNrO7rkLH6a2cuocxpsfwvDGiYiIiIiIiIic4g3DH4iIiIiIiIjIAxhUICIiIiIiIiKXMKhARERERERERC5hUIGIiIiIiIiIXMKgAhERERERERG5hEEFIiIiIiIiInIJgwpERERERERE5BIGFYiIiIiIiIjIJf8P4/aKDWqTDNwAAAAASUVORK5CYII=\n", 177 | "text/plain": [ 178 | "" 179 | ] 180 | }, 181 | "metadata": {}, 182 | "output_type": "display_data" 183 | }, 184 | { 185 | "name": "stdout", 186 | "output_type": "stream", 187 | "text": [ 188 | "Loss 0.38411\n", 189 | "Val MSE: 0.62367\n" 190 | ] 191 | } 192 | ], 193 | "source": [ 194 | "for batch in lib.iterate_minibatches(data.X_train, data.y_train, batch_size=1024, \n", 195 | " shuffle=True, epochs=float('inf')):\n", 196 | " metrics = trainer.train_on_batch(*batch, device=device)\n", 197 | " \n", 198 | " loss_history.append(metrics['loss'])\n", 199 | "\n", 200 | " if trainer.step % report_frequency == 0:\n", 201 | " trainer.save_checkpoint()\n", 202 | " trainer.average_checkpoints(out_tag='avg')\n", 203 | " trainer.load_checkpoint(tag='avg')\n", 204 | " mse = trainer.evaluate_mse(\n", 205 | " data.X_valid, data.y_valid, device=device, batch_size=16384)\n", 206 | "\n", 207 | " if mse < best_mse:\n", 208 | " best_mse = mse\n", 209 | " best_step_mse = trainer.step\n", 210 | " trainer.save_checkpoint(tag='best_mse')\n", 211 | " mse_history.append(mse)\n", 212 | " \n", 213 | " trainer.load_checkpoint() # last\n", 214 | " trainer.remove_old_temp_checkpoints()\n", 215 | "\n", 216 | " clear_output(True)\n", 217 | " plt.figure(figsize=[18, 6])\n", 218 | " plt.subplot(1, 2, 1)\n", 219 | " plt.plot(loss_history)\n", 220 | " plt.title('Loss')\n", 221 | " plt.grid()\n", 222 | " plt.subplot(1, 2, 2)\n", 223 | " plt.plot(mse_history)\n", 224 | " plt.title('MSE')\n", 225 | " plt.grid()\n", 226 | " plt.show()\n", 227 | " print(\"Loss %.5f\" % (metrics['loss']))\n", 228 | " print(\"Val MSE: %0.5f\" % (mse))\n", 229 | " if trainer.step > best_step_mse + early_stopping_rounds:\n", 230 | " print('BREAK. There is no improvment for {} steps'.format(early_stopping_rounds))\n", 231 | " print(\"Best step: \", best_step_mse)\n", 232 | " print(\"Best Val MSE: %0.5f\" % (best_mse))\n", 233 | " break" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": 8, 239 | "metadata": {}, 240 | "outputs": [ 241 | { 242 | "name": "stdout", 243 | "output_type": "stream", 244 | "text": [ 245 | "Loaded logs/year_node_8layers_2019.08.27_17:11/checkpoint_best_mse.pth\n", 246 | "Best step: 3400\n", 247 | "Test MSE: 0.63787\n" 248 | ] 249 | } 250 | ], 251 | "source": [ 252 | "trainer.load_checkpoint(tag='best_mse')\n", 253 | "mse = trainer.evaluate_mse(data.X_test, data.y_test, device=device)\n", 254 | "print('Best step: ', trainer.step)\n", 255 | "print(\"Test MSE: %0.5f\" % (mse))" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 9, 261 | "metadata": {}, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/plain": [ 266 | "76.18011616124537" 267 | ] 268 | }, 269 | "execution_count": 9, 270 | "metadata": {}, 271 | "output_type": "execute_result" 272 | } 273 | ], 274 | "source": [ 275 | "mse * std ** 2" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [] 284 | } 285 | ], 286 | "metadata": { 287 | "kernelspec": { 288 | "display_name": "Python 3", 289 | "language": "python", 290 | "name": "python3" 291 | } 292 | }, 293 | "nbformat": 4, 294 | "nbformat_minor": 2 295 | } 296 | -------------------------------------------------------------------------------- /notebooks/year_node_shallow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "scrolled": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "env: CUDA_VISIBLE_DEVICES=1,2\n", 15 | "experiment: year_node_shallow_2019.08.27_17:32\n" 16 | ] 17 | } 18 | ], 19 | "source": [ 20 | "%load_ext autoreload\n", 21 | "%autoreload 2\n", 22 | "%env CUDA_VISIBLE_DEVICES=0,1\n", 23 | "import os, sys\n", 24 | "import time\n", 25 | "sys.path.insert(0, '..')\n", 26 | "import numpy as np\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "%matplotlib inline\n", 29 | "import lib\n", 30 | "import torch, torch.nn as nn\n", 31 | "import torch.nn.functional as F\n", 32 | "\n", 33 | "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", 34 | "\n", 35 | "experiment_name = 'year_node_shallow'\n", 36 | "experiment_name = '{}_{}.{:0>2d}.{:0>2d}_{:0>2d}:{:0>2d}'.format(experiment_name, *time.gmtime()[:5])\n", 37 | "print(\"experiment:\", experiment_name)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [ 45 | { 46 | "name": "stdout", 47 | "output_type": "stream", 48 | "text": [ 49 | "mean = 1998.39193, std = 10.92832\n" 50 | ] 51 | } 52 | ], 53 | "source": [ 54 | "data = lib.Dataset(\"YEAR\", random_state=1337, quantile_transform=True, quantile_noise=1e-3)\n", 55 | "in_features = data.X_train.shape[1]\n", 56 | "\n", 57 | "mu, std = data.y_train.mean(), data.y_train.std()\n", 58 | "normalize = lambda x: ((x - mu) / std).astype(np.float32)\n", 59 | "data.y_train, data.y_valid, data.y_test = map(normalize, [data.y_train, data.y_valid, data.y_test])\n", 60 | "\n", 61 | "print(\"mean = %.5f, std = %.5f\" % (mu, std))" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 3, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "model = nn.Sequential(\n", 71 | " lib.DenseBlock(in_features, 2048, num_layers=1, tree_dim=3, depth=6, flatten_output=False,\n", 72 | " choice_function=lib.entmax15, bin_function=lib.entmoid15),\n", 73 | " lib.Lambda(lambda x: x[..., 0].mean(dim=-1)), # average first channels of every tree\n", 74 | " \n", 75 | ").to(device)\n", 76 | "\n", 77 | "with torch.no_grad():\n", 78 | " res = model(torch.as_tensor(data.X_train[:1000], device=device))\n", 79 | " # trigger data-aware init\n", 80 | " \n", 81 | "if torch.cuda.device_count() > 1:\n", 82 | " model = nn.DataParallel(model)" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 4, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "from qhoptim.pyt import QHAdam\n", 92 | "optimizer_params = { 'nus':(0.7, 1.0), 'betas':(0.95, 0.998) }" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 5, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "trainer = lib.Trainer(\n", 102 | " model=model, loss_function=F.mse_loss,\n", 103 | " experiment_name=experiment_name,\n", 104 | " warm_start=False,\n", 105 | " Optimizer=QHAdam,\n", 106 | " optimizer_params=optimizer_params,\n", 107 | " verbose=True,\n", 108 | " n_last_checkpoints=5\n", 109 | ")" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 6, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "from tqdm import tqdm\n", 119 | "from IPython.display import clear_output\n", 120 | "loss_history, mse_history = [], []\n", 121 | "best_mse = float('inf')\n", 122 | "best_step_mse = 0\n", 123 | "early_stopping_rounds = 5000\n", 124 | "report_frequency = 100" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 7, 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "image/png": "iVBORw0KGgoAAAANSUhEUgAABBUAAAF1CAYAAAC3TdL6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl83FW9//H3J5OkaWkpS6FCWVoUpOxqAZHFiIoVlerF6wUVxYv2chWuevXeX90Kt4CgiKiAYFlEEEFkrbS0LGVooaX73tImpGmb7m26pWmWSc7vj5mkk8ns+c5M8p3X8/HIg8z3e77n+8mhbeb7mXM+x5xzAgAAAAAAyFRJoQMAAAAAAAB9E0kFAAAAAACQFZIKAAAAAAAgKyQVAAAAAABAVkgqAAAAAACArJBUAAAAAAAAWSGpAAAAAAAAskJSAfAhM6s1s08VOg4AAFDcIu9JWsxsSMzxxWbmzGy4mR1nZs+a2Q4z22Nmy8zs2ki74ZF2DTFf/1aQHwhAN6WFDgAAAACAr62VdLWkeyTJzM6U1D/q/OOSlkg6UVKzpDMlvS+mj8Occ6HchwogU8xUAIqImX3HzKrNrN7MJpnZsZHjZmZ3m9m2yCcES83sjMi5y81spZntM7ONZvbjwv4UAACgj3lc0jeiXn9T0mNRr8+V9Khzbr9zLuScW+ScezmvEQLIGkkFoEiY2aWSbpf0FUnHSFon6anI6cskXSLpFEmHSfo3STsj5x6W9B/OuUGSzpA0PY9hAwCAvu8dSYea2UgzCyj8PuOvMefvM7OrzOyEgkQIIGskFYDi8TVJjzjnFjrnmiX9RNIFZjZcUqukQZJOlWTOuVXOuc2R61olnWZmhzrndjnnFhYgdgAA0Ld1zFb4tKR3JW2MOvevkmZK+oWktZF6C+fGXL/DzHZHfY3MS9QAUiKpABSPYxWenSBJcs41KDwbYZhzbrqkeyXdJ2mrmU00s0MjTa+UdLmkdWb2ppldkOe4AQBA3/e4pK9KulZdlz4o8qHFOOfc6ZKGSlos6QUzs6hmQ5xzh0V9rcpX4ACSI6kAFI9NChdAkiSZ2SGSjlTkkwLn3B+ccx+RdLrCyyD+J3J8nnNujKSjJb0g6ek8xw0AAPo459w6hQs2Xi7puSTtdkj6jcIfhhyRn+gA9ARJBcC/ysysouNL4WTAt8zsHDPrJ+mXkuY452rN7FwzO9/MyiTtl9Qkqc3Mys3sa2Y22DnXKmmvpLaC/UQAAKAvu07Spc65/dEHzexXZnaGmZWa2SBJ/ymp2jm3M24vAHoVkgqAf02RdCDq62KF1yo+K2mzpPdLuirS9lBJD0rapfASiZ0Kf0ogSddIqjWzvZKul/T1PMUPAAB8xDn3nnNufpxTAyQ9L2m3pBqFZ1ZeEdNmt5k1RH39d47DBZAmc84VOgYAAAAAANAHMVMBAAAAAABkhaQCAAAAAADICkkFAAAAAACQlZRJBTN7xMy2mdnyFO3ONbM2M/uyd+EBAAAAAIDeKp2ZCo9KGp2sgZkFJP1K0jQPYgIAAAAAAH1AaaoGzrkZZjY8RbMbFd6m7tx0bzxkyBA3fHiqbjOzf/9+HXLIIZ726ReMTXKMT2KMTXKMT2KMTXLJxmfBggU7nHNH5TmkosT7kfxibJJjfBJjbJJjfJJjfBLz4v1IyqRCKmY2TNKXJF2qDJIKw4cP1/z58bapzV4wGFRlZaWnffoFY5Mc45MYY5Mc45MYY5NcsvExs3X5jaZ48X4kvxib5BifxBib5Bif5BifxLx4P9LjpIKk30n6f865NjNL2tDMxkoaK0lDhw5VMBj04PYHNTQ0eN6nXzA2yTE+iTE2yTE+iTE2yTE+AADAD7xIKoyS9FQkoTBE0uVmFnLOvRDb0Dk3UdJESRo1apTzOltEBioxxiY5xicxxiY5xicxxiY5xgcAAPhBj5MKzrkRHd+b2aOSXoqXUAAAAAAAAP6SMqlgZk9KqpQ0xMzqJN0kqUySnHMP5DQ6AAAAAADQa6Wz+8PV6XbmnLu2R9EAAAAAAIA+o6TQAQAAAAAAgL6JpAIAAAAAAMgKSQUAAAAAAJAVkgoAAAAAACArJBUAAAAAAEBWSCoAAAAAAICs+CapsOdAq3Y3tRc6DAAAUMTeqtqhjQ28HwEAFA/fJBUu+tV0/SB4oNBhAACAInb9XxfozQ2thQ4DAIC88U1SYV9TqNAhAACAIldRFlBLW6GjAAAgf3yTVAAAACi0/uUlam53hQ4DAIC8IakAAADgkf7MVAAAFBmSCgAAAB4hqQAAKDYkFQAAADwSrqnA8gcAQPEgqQAAAOCR/uXMVAAAFBeSCgAAAB7pXxagUCMAoKiQVAAAAPAINRUAAMWGpAIAAIBHKsqpqQAAKC4kFQAAADzCTAUAQLEhqQAAAOCR/mUBNbdJzjFbAQBQHEgqAAAAeKR/eUBOUktbe6FDAQAgL0gqAAAAeKSiLCBJamohqQAAKA4kFQAAADzSP5JUONBKYQUAQHEgqQAAAOCR/uXht1YkFQAAxYKkAgAAgEc6ZyqwBQQAoEiQVAAAAPBIBcsfAABFhqQCAACARzpmKjSRVAAAFAmSCgAAAB7pX87yBwBAcSGpAAAA4BF2fwAAFBuSCgAAAB6hpgIAoNiQVAAAAL2GmY02s9VmVm1m4+KcP9HMXjezpWYWNLPjos61mdniyNek/EYe1rH8gZoKAIBiUVroAAAAACTJzAKS7pP0aUl1kuaZ2STn3MqoZr+R9Jhz7i9mdqmk2yVdEzl3wDl3Tl6DjsGWkgCAYsNMBQAA0FucJ6naOVfjnGuR9JSkMTFtTpP0euT7N+KcLyiWPwAAig0zFQAAQG8xTNKGqNd1ks6PabNE0pWSfi/pS5IGmdmRzrmdkirMbL6kkKQ7nHMvxLuJmY2VNFaShg4dqmAw6OkPUWpOa96rVbB0k6f9+kFDQ4Pn4+0njE9ijE1yjE9yjE9iXowNSQUAANBbWJxjLub1jyXda2bXSpohaaPCSQRJOsE5t8nMTpI03cyWOefe69ahcxMlTZSkUaNGucrKSo/CD+v3+mQd9b5jVVl5hqf9+kEwGJTX4+0njE9ijE1yjE9yjE9iXoyN75IKzjmZxXtPAgAAerk6ScdHvT5OUpeP+51zmyT9iySZ2UBJVzrn9kSdk3OuxsyCkj4kqVtSIdfKS4zlDwCAouG7mgpL6/YUOgQAAJCdeZJONrMRZlYu6SpJXXZxMLMhZtbx/uUnkh6JHD/czPp1tJF0oaToAo95Ux6QDrS2F+LWAADkne+SCrFzJAEAQN/gnAtJukHSNEmrJD3tnFthZhPM7IpIs0pJq81sjaShkm6LHB8pab6ZLVG4gOMdMbtG5E15wNj9AQBQNHy3/IGFDwAA9F3OuSmSpsQcGx/1/TOSnolz3SxJZ+Y8wDT0C0hNLH8AABQJ381UKKGeAgAAKKB+AbaUBAAUD98lFcgpAACAQmL5AwCgmPguqQAAAFBI5SUsfwAAFA/fJRVY/gAAAAqpPMCWkgCA4uG7pAI5BQAAUEjl1FQAABQRkgoAAAAe6kdNBQBAEfFdUoHlDwAAoJDKA1JzqF3t7a7QoQAAkHO+SyrsawoVOgQAAFDEyiPvrppCzFYAAPif75IKN09aUegQAABAESsPhGdNsgQCAFAMfJdUaGhmpgIAACic8kD4vxRrBAAUg5RJBTN7xMy2mdnyBOe/ZmZLI1+zzOxs78NMHxUVAABAIXXMVGgiqQAAKALpzFR4VNLoJOfXSvq4c+4sSbdImuhBXAAAAH1Sv46ZCi3thQ0EAIA8KE3VwDk3w8yGJzk/K+rlO5KO63lY2WPzBwAAUEjlJZGaCsxUAAAUAa9rKlwn6WWP+8wIW0oCAIBCoqYCAKCYpJypkC4z+4TCSYWLkrQZK2msJA0dOlTBYNCr23dqbNyfk377uoaGBsYlCcYnMcYmOcYnMcYmOcbHvzqTCuz+AAAoAp4kFczsLEkPSfqsc25nonbOuYmK1FwYNWqUq6ys9OL2YVMnS5IGHjJQlZWXeNevTwSDQXk63j7D+CTG2CTH+CTG2CTH+PhXv0ihxuYQSQUAgP/1ePmDmZ0g6TlJ1zjn1vQ8pJ7GU+gIAABAMWOmAgCgmKScqWBmT0qqlDTEzOok3SSpTJKccw9IGi/pSEl/tPATfcg5NypXAQMAAPRmFGoEABSTdHZ/uDrF+W9L+rZnEQEAAPRhFGoEABQTr3d/AAAAKGplJeHlmE0sfwAAFAGSCgAAAB4yM1WUBpipAAAoCr5LKhiVGgEAQIH1LyepAAAoDr5LKgAAABRa/7KADrS0FzoMAAByzndJBeYpAACAQqsoK1ETMxUAAEXAd0kFV+gAAABA0WP5AwCgWPgvqeBIKwAAgMIKL38gqQAA8D/fJRUo1AgAAAqtooyZCgCA4uC7pMKqzXsLHQIAAChy/csC1FQAABQF3yUVJGn4uMmq2d5Q6DAAAECR6l9OUgEAUBx8mVSQpHm19YUOAQAAFKn+LH8AABQJ3yYVAAAACqWCQo0AgCJBUgEAAMBj4eUP7YUOAwCAnCOpAAAAehUzG21mq82s2szGxTl/opm9bmZLzSxoZsdFnfummVVFvr6Z38gPqigNqKWtXaE2EgsAAH/zbVLBuUJHAAAAMmVmAUn3SfqspNMkXW1mp8U0+42kx5xzZ0maIOn2yLVHSLpJ0vmSzpN0k5kdnq/Yo/UvD7/FagqRVAAA+JtvkwoAAKBPOk9StXOuxjnXIukpSWNi2pwm6fXI929Enf+MpFedc/XOuV2SXpU0Og8xd9O/LCBJ1FUAAPheaaEDAAAAiDJM0oao13UKzzyItkTSlZJ+L+lLkgaZ2ZEJrh0WewMzGytprCQNHTpUwWDQq9glSQ0NDaqtq5IkBWe+raMG8BlOh4aGBs/H208Yn8QYm+QYn+QYn8S8GBuSCgAAoDexOMdiFzX+WNK9ZnatpBmSNkoKpXmtnHMTJU2UpFGjRrnKysoehNtdMBjUh044RVq+SGd/5FydMnSQp/33ZcFgUF6Pt58wPokxNskxPskxPol5MTa+TSpQUgEAgD6pTtLxUa+Pk7QpuoFzbpOkf5EkMxso6Urn3B4zq5NUGXNtMJfBJsLyBwBAsWA+HgAA6E3mSTrZzEaYWbmkqyRNim5gZkPMrOM9zE8kPRL5fpqky8zs8EiBxssix/KuM6nQSlIBAOBvJBUAAECv4ZwLSbpB4WTAKklPO+dWmNkEM7si0qxS0mozWyNpqKTbItfWS7pF4cTEPEkTIsfyrqI8nFRoIqkAAPA53y5/AAAAfZNzboqkKTHHxkd9/4ykZxJc+4gOzlwomI6ZCiQVAAB+x0wFAAAAj7H8AQBQLHybVHBUagQAAAXSP7L8oZFCjQAAn/NtUgEAAKBQygPht1gtofYCRwIAQG6RVAAAAPBYacAkSaE2pk4CAPyNpAIAAIDHyiIzFVrbmakAAPA3kgoAAAAeKy1hpgIAoDj4NqngxC9xAABQGIFIUqG1jZkKAAB/821SAQAAoFDMTOWBErUyUwEA4HMkFQAAAHKgNGAKMVMBAOBzJBUAAAByoLTEFGpnpgIAwN98m1Rw/A4HAAAFVBYooaYCAMD3fJtUAAAAKKTw8gc+5QAA+BtJBQAAgBxgpgIAoBj4NqkwbcWWQocAAACKWFmgRK3UVAAA+Jxvkwozq3YUOgQAAFDESkvY/QEA4H++TSoAAAAUUmmgRK3UVAAA+BxJBQAAgBwoC5hC7cxUAAD4G0kFAACAHAgvf2CmAgDA30gqAAAA5EBZoEQt1FQAAPgcSQUAAIAcKAuUUKgRAOB7JBUAAAByoDRgCrGlJADA50gqAAAA5EBpCbs/AAD8j6QCAABADpQFjOUPAADf83VSYcaa7YUOAQAAFKmyQIlaSSoAAHzO10mFbzwyt9AhAACAIlUaMJY/AAB8z9dJBQAAgEIpKylRqJ2ZCgAAf0uZVDCzR8xsm5ktT3DezOwPZlZtZkvN7MPehwkAANC3lAZMIWYqAAB8Lp2ZCo9KGp3k/GclnRz5Givp/p6HBQAA0LdRUwEAUAxSJhWcczMk1SdpMkbSYy7sHUmHmdkxXgUIAADQF5WWmELtzFQAAPibFzUVhknaEPW6LnIMAACgaJWVMlMBAOB/pR70YXGOxU3Lm9lYhZdIaOjQoQoGgx7cPrl83KMvaGhoYCySYHwSY2ySY3wSY2ySY3z8r6wkvPuDc05m8d4uAQDQ93mRVKiTdHzU6+MkbYrX0Dk3UdJESRo1apSrrKz04PYRUyfHPezpPfqwYDDIWCTB+CTG2CTH+CTG2CTH+CRmZqMl/V5SQNJDzrk7Ys6fIOkvkg6LtBnnnJtiZsMlrZK0OtL0Hefc9fmKO1ZpIDwhtK3dqTRAUgEA4E9eJBUmSbrBzJ6SdL6kPc65zR70CwAAioyZBSTdJ+nTCn9wMc/MJjnnVkY1+7mkp51z95vZaZKmSBoeOfeec+6cfMacSEciIdTuVBoocDAAAORIyqSCmT0pqVLSEDOrk3STpDJJcs49oPAv8sslVUtqlPStXAULAAB87zxJ1c65GkmKfGgxRlJ0UsFJOjTy/WAlmCFZaGUl4ZkKrW3tqigjqwAA8KeUSQXn3NUpzjtJ3/MsIgAAUMziFYA+P6bNzZJeMbMbJR0i6VNR50aY2SJJeyX93Dk3M4exJtUxU6G1jR0gAAD+5cXyBwAAAK+kUwD6akmPOufuMrMLJD1uZmdI2izpBOfcTjP7iKQXzOx059zeLjfIceHojiKca9e3SpJmzHxLh1V4seFW30eB0uQYn8QYm+QYn+QYn8S8GBuSCgAAoDdJpwD0dZJGS5JzbraZVUga4pzbJqk5cnyBmb0n6RRJ86MvzmnhaB0swrl13npp5TKd+9ELNOyw/p7eo6+iQGlyjE9ijE1yjE9yjE9iXoxNUafNm0NtemHRRoVXcAAAgF5gnqSTzWyEmZVLukrhotDR1kv6pCSZ2UhJFZK2m9lRkUKPMrOTJJ0sqSZvkccojdRUCLW1FyoEAAByrqhnKvz21TX605s1GtivVJ86bWihwwEAoOg550JmdoOkaQpvF/mIc26FmU2QNN85N0nSjyQ9aGY/VHhpxLXOOWdml0iaYGYhSW2SrnfO1RfoR6GmAgCgKBRdUqFme4NOOGKASgMl2ra3WZK0t6m1wFEBAIAOzrkpCu8uFX1sfNT3KyVdGOe6ZyU9m/MA01QWiMxUaGemAgDAv4pq+cOG+kZdetebunPa6kKHAgAAfK4jqdAaYqYCAMC/fJ9UuPHJRZpZtV2StL0hPDNhztqCzYQEAABFonP5AzMVAAA+5vukwj+XbNI1D88tdBgAAKDIlHUWamSmAgDAv3yfVIiHX+0AACDXOmYqsPsDAMDPiiqpYIUOAAAAFI2yzuUPfJwBAPCvokoqdHIu8h9+yQMAgNwo7Vz+wEwFAIB/FVVSwSz+XIUEhwEAALLWufsDSQUAgI8VVVKhA/MTAABArnUuf6BQIwDAx4oqqRA7ISHU3rEMIv+xAAAAfyuNzFQIsaUkAMDHiiqpEOulpZsLHQIAAPCp0hJmKgAA/K8okwrMTAAAALnWUVMhRFIBAOBjRZVU6CjI6PJcVeFXU9/VrOodeb0nAAAorIM1FVj+AADwr6JKKnTUUGhoCnU5/t9PL9HkmKUQV9z7lv705nue3Pf+4Hv66kNzPOkLAAD0DaXs/gAAKAJFlVR4cdFGSVLtzsZu5773t4WSpFdWbNHiDbu1tG6Pbn/53bzGBwAA/KNjpkLHhxoAAPhRaaEDyJcn5qxTaxq/1Mc+viAP0QAAAL8rLemoqcBMBQCAfxXNTIV7p1d321ISAAAgVw7WVGCmAgDAv4omqbB5T1OhQ4hr0pJNqtneUOgwEqrdsV8toeSfsDjntHVv7xxfAAAKxcwUKDGF2pmpAADwr6JJKkgHd39IZGbV9pR9bNvbpHm19R5FJP3Xk4t02d0zPOvPS7sbW1T5m6B+9vyypO0efmutzv/l66reti9PkQEA0DeUBYyZCgAAXyuupEKKBRDXPDw3ZR+X/+Et/esDsz2JZ2dDs6TeW8CpoTm8S8as93Ymbddxfl2cApgAABSzspISdn8AAPhaUSUVnA4+vP/PP5Zk1ceOSCLACzf8bZFnfQEAgN6nNGAKMVMBAOBjRbP7g9R1psI/FtTl5B7NoTZt2dOk/uUBtbdL7xtckbBtTxIU+5pa1d4uDR5QlnUfAAAgt0oDJdRUAAD4WnElFXKw/cOarfs0r7ZeXzv/REnST55dpucWbew8X3vH57y/qaSz/+8Vtbvc9Q8AAHqurISaCgAAfyuqpMJjs9d53mdHkcWOpMLM6h2e9Nux/rIsEH+FSkcZhv/++2I9t2ijRh5zqE593yDd/W/neHJ/AADQc2Wl1FQAAPhbUdVUyEZTa1ta7drandZs9W73gw/f8qrOuvmVlO06ZkWs2rxXz0fNkEikqbVNjS2hHscHAABSKy2hpgIAwN9IKqQw5t6302p396trdNndM7R9nzeFHPc1hXQgzYRGJs699TWdNn6a5/1KkuM9EwAAXZQFmKkAAPA3kgoprE5z9sHC9btyHEl6rv3z3M6tKuPZ1+z9LIUclKoAAMAXSgPWa7eOBgDACyQVCigXbzGCq7frL7Nqc9AzAADIVGkJMxUAAP5GUgEAACBHygLUVAAA+BtJBSCHZqzZrt++srrQYQAACoSaCgAAvyOpkIG2HK6JfGx2rYaPm6ypyzfn7B7Iv288Mld/mF5d6DAAAAVSGihRKzUVAAA+RlIhA/cmeTi0HlYrHP/iCknSTZNW9KwjL4LJEm+ZAADoqqzEFGKmAgDAx0gqZGBNkp0gNu1u8uQeW/c26/aXV3nSVzacc9qRZPeIeAqUwwAAoNcrpaYCAMDnSCpkqb3dafLSg0sV6nY1ZtxHojWWf3qzJuu4eurxd9Zp1K2vqWrrPjneA2VsQ32j7nm9So7BA4CsmdloM1ttZtVmNi7O+RPM7A0zW2RmS83s8qhzP4lct9rMPpPfyLsrpaYCAMDnSgsdQF910k+ndHndmsWnEOt2Zp6IkKTqbYlnTPRE9bZ9+r9/rpQkrd2xXyOPOTQn9/Gzb/9lvlZv3acx5wzTCUcO8KzfPY2tGjygzLP+AKC3MrOApPskfVpSnaR5ZjbJObcyqtnPJT3tnLvfzE6TNEXS8Mj3V0k6XdKxkl4zs1Occ235/SkO6hcoUQtJBQCAjzFTIQ3/+sCsrGYixHp3y96Mr7nvjWpNXb6ly7FvPjIv6TXZrkb41G9n9KgYpV8+nW9rd3poZo2aWjN/D3ogco3zsMLES0s36ewJr2jxht2e9QkAvdh5kqqdczXOuRZJT0kaE9PGSerIfA+WtCny/RhJTznnmp1zayVVR/ormLJAiVpCJBUAAP5FUiEN82p36aJfvdHjft55b2fG19w5bbWu/+sCbd3bpO37mvWFe97Sxt0Hkl6T/xoH/iqq8OLijbp18ir97rWqvNzv+scX6NLfBBOef7s6/OdmxaY9eYkHAApsmKQNUa/rIsei3Szp62ZWp/AshRszuDavykuZqQAA8DeWP+TYtr3eFHBsCbVr0pJNWrYx+wfLDfU9n20hSdv3NauxJaQTjzxEkhRqD79ZWrBuly47/X2e3KOQGlvCsw32NrVm3UcmkzamrtiSuhEAFI94merYf1WvlvSoc+4uM7tA0uNmdkaa18rMxkoaK0lDhw5VMBjsWcQxGhoaOvvctrlZTc0hz+/RV0WPDbpjfBJjbJJjfJJjfBLzYmxIKmRg8rLNqRvFuDtPn3ZHswQzBya8tDLu8Uyde9trkqTaOz4nSVofSVb8aUaN/u3c43XSUQM9uU9fxE4YANBjdZKOj3p9nA4ub+hwnaTRkuScm21mFZKGpHmtnHMTJU2UpFGjRrnKykqvYpckBYNBdfQ5p+ldTa+rkdf36KuixwbdMT6JMTbJMT7JMT6JeTE2LH9A1jpqDkQ/R//L/bNyft8N9Y060FKwmlsF4I9aFQCQpnmSTjazEWZWrnDhxUkxbdZL+qQkmdlISRWStkfaXWVm/cxshKSTJc3NW+RxlAdK1Nrm1N6DmkUAAPRmJBXyaGldz9bEL+/B0odMPfDmeynbxJvi39AUykE0XV386zf07ceSF6sstFy8dUw0AwUA/MQ5F5J0g6RpklYpvMvDCjObYGZXRJr9SNJ3zGyJpCclXevCVkh6WtJKSVMlfa+QOz9I4ZoKkqirAADwLZY/5NFzizbqX0cdr1dXbtX4L5yW0bVz1tbr5eX5W3u/cP3utKby72xo1ta9zbkPKEZH8cLepjc89re3O/3nEwv0rQtH6KMnHVnocAAgY865KQoXYIw+Nj7q+5WSLkxw7W2SbstpgBnoF5VUqCgLFDgaAAC8R1Ihz65+8B1J0shjBmV0XSZFFvO1rv/Dt7zauYVib9USatfC9buK6uF6X1NI01Zs1ez3dmrpzZ/pcX/jZjYq8M50vT3uUg+iA4Di0jlTgW0lAQA+xfKHHHty7vq4x//nmaU5vW9bu1NbzPrNTHYkiHagtU3rdzZqZ0Nzl+0se3tCQZJ+NfVdXTXxHS3r4dKTQsr2/5tXtux3KbcxBQDEVx4gqQAA8Le0kgpmNtrMVptZtZmNi3P+BDN7w8wWmdlSM7vc+1CTm/6jj+f7lr3W/HW79P6fTuncpUGSWtq6PpneksFOEPX7W3TJnW/oI7e+pgvvmJ60bS5mSTQ0h9TYkl2thqptDZKknfvjL9GYX1uvGWu2xz3n5cP89Y8v0PqdPdvSk50lAKDvYaYCAMDvUiYVzCwg6T5Jn5V0mqSrzSy2IMDPFS6k9CGFqzT/0etAUynmbQxjdTwk1+9vkSS9U7NTY19t1Ds1B+sQPPzWWknSQzNrdNndb8btZ3dja8b3jn0Qv+bhObru0Z4VVTzjpmk6Z8KrPeojkS8/MFvfeKRrYfBcPLxPXbFFN/9zhfcdx3DO6cUlG3N+HwBAesoCFGoEAPhbOjMVzpNU7Zyrcc61SHpK0piYNk7SoZHvByvOntAonDk19ZLCn/jHunXyKq3Z2hB3i8aXA0A2AAAgAElEQVQ33t3Wo/u+sGijZlbt0OtJ+qne1tA5tX5Z3R4NHzc5bv2IRJ/wvLtlr372/LKUW3VlM+lg1/6Wzm0ze4N4Myf2NLaqNeqN6qQlmzT+xXDygs3LAKDwmKkAAPC7dJIKwyRtiHpdFzkW7WZJXzezOoWrNd/oSXTo9O6WvTntf+T4qd2OZfNQGoo83K/YtEc/+PvilO0/9ds3deEd0zV83GR9928LJElvJliOEM+1j8zTE3PWa+u+prjn05104OI8sU9dsUVn/d8raceSqr9cOHvCK/ruEws7X++KzE7Jl6V1u7V5T2b1Fpxz+soDszVtRf52M/HCl/74tr7ywOxChwGgj2FLSQCA36Wz+0O857LYJ6arJT3qnLvLzC6Q9LiZneGc6/Ib1MzGShorSUOHDlUwGMwi5OI0bcXWrK4LBoOqrY3/oJlq/NeuXZvVPf8y6XXdNKvrQ346/6831IcfTtesWaNgU/d7x/YRDAbV3ByulTB79mwdURF+47a7qV0DykzlAVN9fTiOZUuXyjYn/uM+uWq/zML9r1l/cNlHS6hd015/QyapPJA6RXHgQPhnmDt3rtYf0jVnt3PnzqTjkOjcps3hn3HNmtUKHqjpdv7VlVs7r61adzD2UCjk6d+xeH1dO3W/Skx65DOHpN1PqN1pbm2jFqyr18MZXFdoi9bvl9R9HBoaGvi3LAHGJjnGpzj0o1AjAMDn0kkq1Ek6Pur1ceq+vOE6SaMlyTk328wqJA2R1GXeu3NuoqSJkjRq1ChXWVmZXdSJTJ3sbX8+UFlZqSWhKql6TdxzycZsxIgRca9LZXPZMEnvdb9XxKL1u1RRFtDIYw6Ne/+Sw45VZeXpBw9E2nT2EfW6fNZrUnOzLrjgAh0zuL8kafi4yfrkqUfr4WvP1Z9r5ko7tuvMM89S5alHdw820teW5rLO/jfNWS+tXNbZ5D9eDS/HqL3jcyl/9gHzg1Ljfp133nnhOh9RP98RRxyhysrzEsaQ6O/D1J1LpboNKjtimCorY8qZxFxb+/ZaaVW4COeBUPih/6/Xna+LTh6SMvaEksU3dbLaXeLY42lta5deeVlmltF1BZdgHILBYN/6OfKIsUmO8SkOLH8AAPhdOssf5kk62cxGmFm5woUYJ8W0WS/pk5JkZiMlVUhKfw47csY5l3Dng3z7/WtVGj5usr70x1n67O9nJmzXmsUUUYuZUNNRxyHXOya8t71Bw8dN1huru9aNWLV5nyf9f/zON/TUvPDqowdnZjdz5OG3us9uAADkB0kFAIDfpZyp4JwLmdkNkqZJCkh6xDm3wswmSJrvnJsk6UeSHjSzHyq8NOJal69F5Ujq6fkb9NjsdXHPrdu5P+m1q7d482Dc4e7X0pv1cKClTS2h9s43YsnE/il7fVV2y0SytWDdLknS5KWb9YkPHpwJ8b2/LVTN9lOy7veOl9/Vrv0tWpfhNpR94S8d/zIAKCbUVAAA+F06yx/knJuicAHG6GPjo75fKelCb0ODF15flXjnhY/fGUx67eRlm7O6Z7qzA9oS7Njw3KKNqtrWoH/eeFHKPjp66LhnbMwdoTQ0h3TNw3N06xfP0IlH5m4df/SPftermS8d6fDAm++lbtTH5XoWCQD0BuXUVAAA+Fw6yx/Qh72yMr+f3Gfi/T+dkvDcso170uqj41PvRM+nFnlyfePdbZpZtUO/nro6YV+NLd233AQAoCdY/gAA8DuSCvBcQT6AjnPTjqUJ6Xh7Y0injZ+mqq37cvYJOrP+AaD4dCQVmln+AADwKZIKyIvnF9VlfE1zqE3tMUskokt1hNrataMhXITynterNXzcZIXaDp5/p2andjd23U7TRR7tnXP63hMLo46HzatNPxER7ZkFqX++4Ortun3Kqqz67wnrZesMHOkVAEWE5Q8AAL8jqYC8+OHfl2R8zQd/PlU/e2FZwvPv1NR3fv/4O+FilNFv2u6ctloL1++Oe+3+lra4NSN++vwyBVcnrkMRK9PH9T/NSL4Tw+7GFi1YV5+0TTJ9qQhi7I4dQCE0toS0aH12yUQgHSx/AAD4HUkFeM7LD8afnLsh4bnH36nNuL90HrozWTbhta89NEdX3j87q2udc5pRlXwnV+dc1m9sN9RnthMF0Bd8/6nF+tIfZ3Wb1QR4pWOmQjbbJQMA0BeQVECf8cqKLV2SAtNWpF+EMhS1jOLlZZt1xk3TErZtbk3/jV+3HEUPEyorNu3N+tpnFtQpuDp5UmHijBqd8vOXtWt/5g9QF//6jc7v29ud5tcenFGxv5kil+ibltaFZzM1ZfD3HshEaaBEJcZMBQCAf5FUgOcem7UuJ/2OfXxByjaJ1uu/tHRT+LyTHn5rbdI+9qX5gPzyss2q3bG/8/Wf0twGcuPuA2m1y1TdrtT9PrswXPth277mrO8z5r63Neq21/TlBw7OqHhoZvIxjdaXlmgAgBfKS0vUwkwFAIBPlRY6APhPug/ludDY0hb3eHsOHmT/M6rQoyTd/vK7aV134R3TVXvH5yRJb1fv8CyedH5ELx7ol2zoXqciq+KLlFQAUCT6lQbU3Br/9xMAAH2dr2YqjDzCVz8O4vjGI3OTnp9ZlfwhvbftPPC1h+ZkfM3Nk1bo4l9P73Y8k6m1vWxDiB7btPuAqrc1FDoMeGzBul1q4kEMPlBRVsISGwCAb/nqKfyLHygvdAjIsbd6+Mn+tBVbe/xA/b0nFmr4uMk96mPBuvqsH4IfnVWrDfUHum23+UCayy/86GN3TNenfvtmWm15UO0bNtQ36sr7Z+lnzy8vdChAj1WUBdQU4t8dAIA/+SqpMGKwr34c5Mim3U09uj7eVpSZuvL+2Wk/BCfy4Mzk21N22NHQrBlrwgUce9c8jfzbtPuArrx/lsY9u7TQoeRUW7vT6i37Ch1Gj+xtapUkrdycffHSTKQzi6lmewNFSZGV/mUBHUiwPA8AgL6Op3AUnVwVSsy3hevT2/pyad0efeORudq272AyJdFkjabWtqTbnn3yrmAGEXY3p2anTv3F1KQx9MSyuj1Jd7ZoiDwQ9mSXjd7m9VVb9djs2i7H7plepc/8boZWevhzLly/S99/alG3GTJ9nWXwJ/HSu97UtX9OvgQLiKdfWUBN7P4AAPApXyUV/LZOHL1Px/ZzfdF5t70ul6JS46m/mKov3PNWwvPvbd8f93i6D2YPZrBLRCrNobZuP88X7n1L/3L/LM/u0Rdc95f5Gv/iii7HFkeKaW7dm9msnLtfXaNHEuyO8u2/zNeLizdpV2Pm25Fmo7fuEjKvNr1kHhCtorSEZVcAAN/yV1Kh0AHA96649+1Ch9AjHc9pyRJw727ZpzNvnqaRv5iqFxdvzEtcmarb1agP/nyqnpy7odu5tTviJz6Q2u9fr9KEl1YWOowu+HcdftC/PEBSAQDgW75KKgDFZO7a+swv6vz0N/mj2r6mkA60tun7Ty3O/B459C9/fFuf/u2bnYmDKVnWt6ja1hB3mcfU5Zvz8sb/V1Pf7VGxz7eqdujVlVs9jCgz7e2uV6wPb2wJadZ73m3LCuRKRSlJBQCAf5FUANLU29ZS72pszfrajpkK2/Y2aU8P+olnQ31jZ2HIdGJIx7a9TXp8dq0Wrt+tKo+2jnxy7vour+fX1uv6vy7UrZNz/0n9/cH0durY3xzSkg3dl9x8/eE5+s5j870OK22/nrZaI8dPVWNL16KFdbsataxuT4/7398c0k2Twks6ov+c/M8/lmhm1cE/W//7zFJ99cE52lDf2ON7ArnUvzygAyQVAAA+5aukAtNkkUvB1akflHu7mpilAef98nVdcMfrnt7jk3e9qW884l0CZsmG3fr8PW/pF1F1A1rSLHjW0BzS6AQFC5tb2+Wc0xNz1ulAS1vnbgMbd3Uv5FmonRS++8RCjbnv7c4Ck+nqaT2C379WlfT8swvrJEkNTV3juuhXb+gL9yauyZGuB2fWaMG67rUL/rGgTtc8fPDP1pqt4f8vjTmYNdEcast43IFEKspK1NRKoUYAgD/5K6lAVgHIWMcD2c6G5qz7iP6715Jk94jNezLfeWPMfW9r276usTWnmVSYU7NT727Zp9+8sjru+TdWb9PPnl+uX05ZFXWse/LouUV1GUQc5pzTQzNrtD0q9rZ2l7JYZrSOgouhJGOaVJb/Jt792pqCTtVui9phIl//rsf+b/nSfbN0xk3T8nNz+F4/lj8AAHzMV0mFErIKQFri/U2ZOLMm5/dNNU39raodml+bRa2IGOlue7i/Ofwmv76xpctDZUNzSLdPWaXmUOKHgPr9LfrEb4La1BD/gf/dLft06+RV+q8nF3Uee/9Pp+j/Pbs0rdjSkSy+fCj0Bg1e7BCR6NfGys3+2XYUhUehRgCAn/kqqQAgfcm2jsyHeNtQfv3hOfryA7NTXpvqYfK5RRtTtuvyMBnT7rbJq/SnGTW6bfIqJfLqyi1au2O//rw8/gyPUFu4033N4WUVHYUNn54ff9ZD/f4Wrd/ZNemSalZDY3P8h5R4V3lZWNGL9O2u/S1avrHn9Rekns1m6K1bV8JfKkoDam1z2c86AgCgFyOpAPQSsUXv4nk+wTT8TN+ompmWxT7Q5eDhavWWfdrb1KoXFm3U8HGTtbcpP2vUf/yPJarf39L5OtEz5yNvr5UkTV62WbVRD/QdhSYfm70u5b2qdqc39uOeSz5D4aJfTdcld77R+do51zle8RIw6ei4avnGPRo5fqpeznK3jJ4a/+JynTZ+apdjV94/S59PI7GV7GfP5I/s/Nr6pDM7mOiGXKooC7/dakpz6RYAAH0JSQWglzj/l6kLJv7w70viHv/D68kL68WK9/yUbp2CeNZs3adnF3RPeHzmdzNUeWdQP/h7/K0pnXM66+Zp+us7qR/eo6XzALgqzvT19piPpRetP7izQrxijp338+Cz+VTFHmOLDfak+GDsDIelkR0ZZlSlX2zUy4fsx2av6/bzxBYNzSSO2CRaqlCrtu7Tlx+YrVtfSjzzBL2HmY02s9VmVm1m4+Kcv9vMFke+1pjZ7qhzbVHnJuU38sT6lwckiSUQAABfKi10AADC9vXgU/zq7Zltsxj7oLbnQKsenVWb9f1fWrpZLy3drC+cfWy3c9EzBuLZ2xTSz19Yrq9/9MQux5fV7dGiDd13AIj2VvUO3fJS/C0g29qdlm86OBvjD69X6fV3tyXsK1f1CbyeXj983OS021oPMgPpJFIKtXSgpa1dpYH0cuIfu/1gsq5Qu3ggfWYWkHSfpE9LqpM0z8wmOec6/6I7534Y1f5GSR+K6uKAc+6cfMWbropSkgoAAP8iqQD4wJRlWzJqH/sw+P2nFsVvmKHxLy73pB9JSbcmjK4P8PBba+O2uWd6lebVhpMSTtJvX12T9H6ZLs1I9dAd+zzfowfwHs4acB6vbYmXq6ja6s0Du3NO90yvPnivFG1T2bSnKc37ptUMuXeepGrnXI0kmdlTksZIip89lK6WdFOeYstaBTMVAAA+RlIBKEKVvwl2eT13bc93XJCkmVU7POknldhlDPF0TPmXpOlxZij8c8mmLq+jH1A37u669eWBNOpdeO30nGxnmH52ItNJDp++e0aGscSXTd2NnizVoJZCrzNM0oao13WSzo/X0MxOlDRC0vSowxVmNl9SSNIdzrkXElw7VtJYSRo6dKiCwWDPI4/S0NDQpc/qreE/12/Nnqu6wQFP79XXxI4NumJ8EmNskmN8kmN8EvNibEgqAOjR+v1EYncyiCc6N7Btb3qfKEvpTetP1WRJXddClcnyFKnGZ8mG3RpQHtDJQwdJkh6cUaPbpoTX76/YtDejny0Tl/9hZtLzVVv36ZShA7sc23OgVYP7l3ly/3RmQCzecLBuxb6mVg2q6H7vUFu7qrY1aOQxh2Z4/wyRQOgL4v1fSvS/+ipJzzjnov+CnuCc22RmJ0mabmbLnHPvdevQuYmSJkrSqFGjXGVlZQ/D7ioYDCq6z5I126VFc3X62R/SucOP8PRefU3s2KArxicxxiY5xic5xicxL8aGQo0AcuI7j81Pev5Aa5veWH1wBkFwzcEiglOX93yXgqbWzApPLol6+I025t631NZ+8JlmX1Nrl9eSNOa+t7t8Uv/YO7Vdzn/7sfk9XoLw/KK6bvUUNkdN7f/2X7qP962TV+mC2w9+iPvqyi06+/9e0UtLN2lSzEyNWLFPdl1300j+dL5o/S69uyVc+PKL973deXzijJq47e+ctlqf/f1MVW+LUxskTnaoIwG0Y1/Hdp5kC3ykTtLxUa+Pk5ToD+tVkp6MPuCc2xT5b42koLrWWyiYjkKNXm7tCgBAb0FSAYBnopcNrE5jjf11UQ/C/5h/cMbz9X9dmPS6H/8j/i4YPbGvOf60+yV1e/Tcoo2dr8+8+RX98O+L1ZbBIvxUxSpTMZMenZV8h4zXVm1NeK7zIbwhHMcNf1uk/3pykVoz3Io0XV/64yyN/t1MbdiXuv+dDc1aFEno7Gho7nZ+yYbdmlfbfXlOqK394JabMTmF379Wpcdm13a7htRDnzBP0slmNsLMyhVOHHTbxcHMPijpcEmzo44dbmb9It8PkXShEtdiyKsBkaRCLmaFAQBQaCQVACBDk5Zs0pY0CwBKmc+amB/nIToT6eY7kj1ke1FnY+XO1A9QH7n1Ne090HowpjhB/esDs/Xi4oOJnRcXb+pSzDHW3a+t0fgXV2QWLHoF51xI0g2SpklaJelp59wKM5tgZldENb1a0lOua7XOkZLmm9kSSW8oXFOhVyQVBvULL/tpSJC8BACgL6OmAgB4LPahfkdDsw4fkH4dg8UJlmJkHU8W13z1oTmqveNzyft1Hf9N7w4tCWZGvJvGVo/ff2px5/c/fX5ZWveLlWpmRnu7U0kJ8xkKzTk3RdKUmGPjY17fHOe6WZLOzGlwWRpYEX671dDUmqIlAAB9DzMVAPQKfW1LvyfmJF6OEFtzoafSfcydtGST2tud3qruugtHa8jbZQ6xswmej1oeEi32/+mf3oxfU6HH8aTZbuH6xMmav8/boJN+OkWz3svPDiYoLof0Cy9/YKYCAMCPSCoA6BXmr9tV6BAy0lGfIJ7NGSyN8NJ/PblIM6q2p24YsaRutxatD497urMN4lkas5NGMht3H+hWcLJDPhJLp4+fqpaoJEtHQqJjecVXH5zTbfnJ6N/N0A+eWpT74OBb/UoDKg+UqKGZmgoAAP9h+QMAeOT1VVvVvyz+HvRV8XY2iGN+bb1unbwq6xha27o/mSd6Vr/y/nCNu9o7Pqc/v12bVv/3TK/uTJqkygHEO78gRfKoJ4sPtuxpir+DRJT9LW3ac6BVRw3ql7BNzfb9XV6/u2VfWks0gGQGVpSqoZnlDwAA//HdTIXTj81sn3MA8Mp1f5mvrz40J+32N/yt+y4XX35gdrdjFq96oYdmVm2Pu3vElGXdt/Z8dFZt5/ehtvYuBRR76q9z1mnV5swf3juWm1z+h5n6+sMHx79me+IEw4GWNjWHDn5qXLuzMeP7ApkY2K9UDU0sfwAA+I/vkgrPffdjOv6I/oUOAwBSemlp94f2eB6euVZLelC88ZaXkhfA/8+/LoxbHPK7TyTf2vOe6dX6/lOLNXX5lrjnNzZ0r+WQbJnF5KWb9ZU/dU+qpDJ1+Ra1hNq7bd156V1vJrxm5PipqrwzmLLvREs1gEwN7FdKTQUAgC/5LqnQrzSgCWPOKHQYAJCWl5ZuUu2O/Unb3P3amrT7y2ZOQ0NzSI0tidd6726MXz9i697wMojdB+Kff2tj9weojmu8tm1f+v3+8O/hnSQ272mKu0TDZbVfBpBcePkDSQUAgP/4sqZCv1Lf5UoA+NQNf1vU6//NOmfCq0nPWwapjNtffren4XRz16trdNer6SdeYnfHAPJhYL/SjJJfAAD0Fb37nWyWhh3G8gcAfUezx1s+5svKTXszvqavbR0KeIWaCgAAv/JlUmFAuS8nYABASis3Z/6gn8z2fc0Jz+2M1DDIcR3JnIoXeqrEh3NOk5duVkuoXT96eomenLs+J7HBX1j+AADwK18mFVgPC6BY/TaDZQDpOPe211K2SVaPobfbtCfz6egzqnboe39bqLteWa1nF9bpJ88ty0Fk8JtBFGoEAPiUL5MKAABkK9WD33f/ukCSVLszeYFNINqh/cvU1Nqupta+m4QDACAekgoAAES5dfKqpOf3R2ZmTFuxNR/hwCcOH1AuSdqVYDcVAAD6Kn8mFVj9AAAAepEjDgknFXY2kFQAAPiLP5MKAAAAvciRA8NJhfr9JBUAAP5CUgEAACDHOpY/kFQAAPiNL5MKAyvYUhIAAPQeR3YsfyCpAADwGV8mFQaUk1QAAAC9x+D+ZQqUmHaRVAAA+IwvkwoAAAC9SUmJ6fABZcxUAAD4TlpJBTMbbWarzazazMYlaPMVM1tpZivM7G/ehgkAANC3HT6gXPX7mwsdBgAAnkq5TsDMApLuk/RpSXWS5pnZJOfcyqg2J0v6iaQLnXO7zOzoXAUMAADQFx19aD9t3UtSAQDgL+nMVDhPUrVzrsY51yLpKUljYtp8R9J9zrldkuSc2+ZtmAAAAH3bsYP7a9PuA4UOAwAAT6WTVBgmaUPU67rIsWinSDrFzN42s3fMbLRXAQIAAPjBMYf11/aGZrWE2gsdCgAAnklnmwSLc8zF6edkSZWSjpM008zOcM7t7tKR2VhJYyVp6NChCgaDmcabVENDg+d9AgCQSja/e/idVXyGHVYh56Ste5t0/BEDCh0OAACeSCepUCfp+KjXx0naFKfNO865VklrzWy1wkmGedGNnHMTJU2UpFGjRrnKysosw44vGAyqs8+pkz3tGwCARLL5fdbldxaKwjGD+0uSNu0+QFIBAOAb6Sx/mCfpZDMbYWblkq6SNCmmzQuSPiFJZjZE4eUQNV4GCgAA0Jcde1g4qbB5T1OBIwEAwDspkwrOuZCkGyRNk7RK0tPOuRVmNsHMrog0myZpp5mtlPSGpP9xzu3MVdAAAAB9zbBIUmF9fWOBIwEAwDvpLH+Qc26KpCkxx8ZHfe8k/XfkCwAAADH6lwd07OAKrd2xv9ChAADgmXSWPwAAAMADJx01UDXbGwodBgAAniGpAAAAkCcjhhyimh37FZ7kCQBA30dSAQAAIE9OOuoQ7WsKaXtDc6FDAQDAEyQVAAAA8uSD7xskSVq1eV+BIwEAwBskFQAAAPLk9GMHS5KWb9xT4EgAAPAGSQUAANCrmNloM1ttZtVmNi7O+bvNbHHka42Z7Y46900zq4p8fTO/kac2uH+ZTjxyAEkFAIBvpLWlJAAAQD6YWUDSfZI+LalO0jwzm+ScW9nRxjn3w6j2N0r6UOT7IyTdJGmUJCdpQeTaXXn8EVI6Y9hgLdmwO3VDAAD6AGYqAACA3uQ8SdXOuRrnXIukpySNSdL+aklPRr7/jKRXnXP1kUTCq5JG5zTaLJw5bLDqdh3Qrv0thQ4FAIAeY6YCAADoTYZJ2hD1uk7S+fEamtmJkkZImp7k2mFxrhsraawkDR06VMFgsMdBR2toaEjaZ/vONknSEy/P1BlDAp7eu7dLNTbFjvFJjLFJjvFJjvFJzIuxIakAAAB6E4tzzCVoe5WkZ5xzbZlc65ybKGmiJI0aNcpVVlZmEWZiwWBQyfr8UGOrfj3vFZUMOVGVlR/w9N69XaqxKXaMT2KMTXKMT3KMT2JejA3LHwAAQG9SJ+n4qNfHSdqUoO1VOrj0IdNrC2bwgDKddNQhmru2vtChAADQY75PKpxwxIBChwAAANI3T9LJZjbCzMoVThxMim1kZh+UdLik2VGHp0m6zMwON7PDJV0WOdbrXPyBIZpTU6/mUFvqxgAA9GK+Tyoc2p8VHgAA9BXOuZCkGxROBqyS9LRzboWZTTCzK6KaXi3pKeeci7q2XtItCicm5kmaEDnW61x88lE60NqmBet61cYUAABkjCduAADQqzjnpkiaEnNsfMzrmxNc+4ikR3IWnEc++v4jVVpimlm1Qx97/5BChwMAQNZ8O1PhlKEDJUnfTVIAacw5x+YrHAAAgE4D+5XqwycerplV2wsdCgAAPeLbpMJLN16slRM+o8vPPEa1d3yu0OEAAAB08fFTjtLyjXu1ZU9ToUMBACBrvk0qlJeWaEB58tUd13/8/XmKBgAAoKvRZ7xPkjRl2eYCRwIAQPZ8m1RI5JYvntH5/eEDygsYCQAAKGbvP2qgRh5zqF5a2ut2vQQAIG1Fl1T4+vknFDoEAAAASdLnzzpGC9fv1sbdBwodCgAAWSm6pIKZFToEAAAASeGkgiS9uHhjgSMBACA7RZdUAAAA6C1OPPIQnT/iCD05d73a2l2hwwEAIGNFk1S48dIP6OFvjupyrH95oEDRAAAAhF1zwYnaUH9AM9awvSQAoO9Jvj2Cj/zosg92Oza4f1kBIgEAADjostPep6MG9dNjs2v1iVOPLnQ4AABkpGhmKmTjUyO9+cV+y5jTPekHAAD4T3lpib5+/ol6Y/V2rdq8t9DhAACQEZIKSQRKel7U8XufeL+uuWB4z4MBAAC+de3Hhmtgv1LdO7260KEAAJARkgo5dtTAfoUOAQAA9HKDB5Tpmx87UVOWb9aarfsKHQ4AAGkjqZCGB77+YZ101CFZXXvW8YfFPX5oRfrlLJ777sf01fNPyOr+AACgb/j2RSdpUL9S3fLSSjnHThAAgL6hKJMKj/37eZr6g4uTtvFi6UNpienDJxwe95xZ+v2XmOmUowf2OB4AANB7HX5IuX7wqVM0s2qHXlu1rdDhAACQlqJMKlxyylE69X2HJjz/zPUXaMb/fqLLsegUwEdPOiKt+wzMYDZCMj1PbwAAgL7gmgtO1AeOHqhbJ69Uc6it0OEAAJBSUSYVUhk1/AgNO6x/l2NfO/9ESdIh5QH9x8ff3+N7ZDBRAQAAFNMvU14AACAASURBVImyQInGf/40rdvZqIlv1hQ6HAAAUir6pMI/b7hI91z9objnopczXnLKUZKkoYMr9IkPHq1ff/msHt23LJD+0A8Z1LXY47UfG96jewMAgN7rklOO0ufOPEb3TK9W9baGQocDAEBSRZ9UOPO4wfrC2cemaOX9tIKvnX+Cxl5yUlptY2dN3HzF6Z7HAwAAeo+brzhd/csD+t9nlijU1l7ocAAASKjokwodhqTY+vHwAWWSpE988Oge3WdQpM7CF88Zpp9ePrJHfQEAAH86alA/TRhzuhau3607X1ld6HAAAEiIpELEzJjCjLGOHNhPc376yW6JgCMPKe/y+tT3DYp7/TcvCNdkOGXoINXe8TkNH9J9i8rX/vvj+lmCRMMV5wxLGp8k/ezykfrtV85O2c4r1IUAACB3xpwzTF87/wT96c0aTVm2udDhAAAQF0mFiP7lgZRthh5akXKryak/uKTz+xsvPbnz+8+nXGIhfeDogfr2xSN00xdO63buiEPKdfSg7rMpzj7+sM7vv3PJSfroSUemvM9/fDy9ZRepXPSBIZ70AwAA4vvF50/TR048XD/8+2ItWLer0OEAANANSYUkDn4S75I16zT+812TAdddNOJgX2nf0/StC0fEPTf5vy7W89/9WOfrkcccqheiXkvSsYf116PfOjfpPY6LqtFw8cnZJwb+7dzjE5678ANH6pyohEc2PnP60B5dDwBAX1dRFtCD3xilYwZX6DuPzde6nfsLHRIAAF2QVEhiwpgzdNW5x+vSUxM/3B4eWf7w7xeO0L9fFD8ZEM25+AmKaz56YsprjxrUTx864XBJ0oKff0rPf/djMjOtmjBaa279bGe7yqi6Dy/deJFu+9IZXWOI+t4SrGG4Lo2f5eIPHJXw3P8bfapOPHJAyj6S+ePXPtKj6wEA8IMjDinXn791npxz+taf56l+f0uhQwIAoBNJhSSGHlqhO648S+WliYfpnOMP0+PXnadxnz2181jwx5WaFrUMQkpdf+CWL56RvEGMIwf+//buPD6usl78+Oc7e/Y9TdqkadN9X+lKaUpZWrYKshUQBJFVxet2QRS16BX1XlRQAbkg3p8iLihWRBCByF52Cm0pLW0ppbSle5s9mef3xzmZzD6TZLJNvu/Xa8jMM8+ceeY7c+g53/MsXnxua8hGhscZs42Th+VRkBk674M3qG51lLkdtt1yKt88LXIIRri8TDfHjCiIKF85ZzhTK/KZUJ4b9XWJV9vovGtqRnXpdeErayillFL90cjiLO6+eDYfHmzgkntf4khjS183SSmllAI0qZASi8aUhJzUjyjOYlyMCRs766WvL+32NpZPLgt5nOtzB+47HZJwCMSfw4ZYJJJrr3BxxaJqTpoY2cvj9pUzGNHNXgzhXAnmulBKKaUGutkjCrnjopls+Ogwn/n1KzQ0t/V1k5RSSilNKvS25GZn6FCa6+v2e4oIP1kS+4p8jBEZAd44PTXicTgkMFwj3PIp5YH7yyaVRa0TbObwyPkZgss6G9euiNdjRSmllOoNx48fwq3nTeflbftZefeL7Dva1NdNUkopNcjpWVKQf31pMS/e0P2eAdHFvpI+pjS7h96zQ76346suCVtFwvTgKfnwwug9EjxOqz3XLR3DtUtGR60THLE/X7OQK48LXbUi0UocqXb9svGJKymllFI97IxpQ7njwlls+Ogwn7zjebbt1ckblVJK9R1NKgQZXZpNWV73ewZ0xiNfWMSfroocXvC7z87j21GWlkzW548fHTFB44/OnsrdF89m9ohCfvmpWSybVMY1NaMS9lRwOWL/TFatiJwLoqKgo1fEKVOi90IwIfcjG1AzrgRHjKTBlYuruWrxKO6+eHagLHzeiOPHl4a/rNNKc7z8/QvHBh67taeCUkqpfmLZ5DLu/+w8DjW0cNYdz/Padl1uUimlVN/Qs6QuKrJXfRiaZBKifQ6B88OWYZw4NJe8THdE/fmjivh0jKUlk/Hlk8Zx4dzQFSXOmV3JifYcBydNKuPOT82iKNubMKkwdkg2J0yIfpI+oTyXT86sYGiej3XfOZn7Lj0m5H2DV5cozvZGrAgRPIHllGF5rFoxCYDKgthzLhRkerh++XjyMz1s/t5yfnr+dFZMD5380RFnZsxk51/I9roYUdQxkWVv9ou48rhqZldFHzqilFJKAcyqKuDP1ywkx+figrtf5B9vfdTXTVJKKTUIaVKhi44fX8odF87k80vHJFW/KNvLtltO5bxjhvdwyzrP5Yx9upyf6UZEuHyRNfQgw15xItj/nDuN529YSpbXRc240pg9DF75xgn8+6tLYr5XomEYS+zeBwtGFQW13cGK6cPivi7cCRNCJ4+848KZSb0u0SCRi+al7ru94ZQJ/Onqzk2QqZRSavAZWZzFg1cvYHxZLlf/9jW+/8gGWtv8fd0spZRSg4irrxswUIlIyGSDA1n40IF2d140i8nDQpeFnFKRx0tb96f0/SXJPgDzqovYdsupyW2zE90KwueYiMUXY/jDu99dDsDPntqc/JsqpZRSKVKc7eX3V87j5ofXc9fTW3jjg4PctnIGQ1Iw2bNSSimViPZUUEytyItavmxyGRX2MITwIRI140p6pC3LJ5dTUZDBpQtH9Mj2IfmJKYuzvSE1y/Oir6DhcTlCVoa4bukYfpFk74dooi3xef/lc7u8PaWUUunP63Ly3U9M4dZzp/HmjoOccOu/+c2L7+P398b6SEoppQYzTSqoThGsK/P3XHJMj2y/JMfLs/95PNUl3VsRIxXzH/ziotDEQHAyIlqPCWNnXpwOiTmnQ2FW9F4h7TZ9bzn3XTon8PjShSO46bSJFCfZmyKe0hwv58yq6NJrp1dGLumplFI9QUSWichGEdksItfHqHOuiKwXkXUicn9QeZuIvGHfVvdeq/uPs2ZW8I/rjmPKsDy+8dDbnHvXC6zbeaivm6WUUiqNaVJBJcXntn4qxdlePC5Hp5ZzHDckh6sWjwopO8meMDJ8foOuysuInOwyx+fi+uWJl4GMdv5/7uwKirM7dyLfZl8NiheaR76wKO423M7Q2H7r9ElcdmzXJ+wMtmrFJH50zrQuvTbT44zagyKVnvpKTdLDW5RS6UlEnMDPgeXARGCliEwMqzMGuAFYaIyZBHwx6OkGY8x0+3ZGb7W7vxlZnMVvL5/Lf58zjfc+Pspptz/L1/70JnsON/Z105RSSqWhpJIKyVw1sOudLSJGRGbHqqMGpumV+Xz/rCl8/5NTOv3ax/7juIiT+8nD8th2y6lMHpbHmCHZjCzO4sZTur6Epsvp4MfndZwwi8Bb3z45JJkxpjSy98OPzp4adXvt8zwku1IEQK6d2MiLMUcF0OUlS0cUZTFnZGGXXtsuWgLn5xfMDKxk0l0r51TGTAr86tJj+N+L4/9vYWRxVtzno4m1KolSasCaA2w2xmwxxjQDDwArwup8Fvi5MeYAgDFmTy+3cUAQEc6eVUHtV5Zw+bEj+cvrH1Lz37Xc/sQmGprb+rp5Siml0kjCiRqDrhqcCOwAXhaR1caY9WH1coAvAGt6oqGqd5w2NfrkkyLCyjk9s3KFz+3kqa/UdHs7iZbG/NKJY7n6t6+FTAy5YHQxr74fe+JJX9hqF9Mr83njg4NR637m2JFkepxcMGc463ceTqrNK+cM53cvbU9Yz+Ny8Icr5zPz5sfZX9ec1LbDuZyhOcR/XLeICeW5ZPtcXHLvSyHPrf7cQs742XOBx9ctHZPERJSxEzBLxpXy/Ht7I8o/u2gks6oKYk4Wmkjwd55sLJNx4dzh/HZNarallOqUYcAHQY93AOGTyowFEJHnACfwbWPMo/ZzPhF5BWgFbjHGPBTtTUTkCuAKgCFDhlBbW5uyDwBw9OjRlG+zOxZmwZiFPn6/sZn/efxd7ntmE+eO83BMmTPuEsw9ob/Fpr/R+MSmsYlP4xOfxie2VMQmmdUfAlcNAESk/arB+rB6NwM/BL7SrRapPnPZwpHcdHrXewv0J8GJg9lVBbzy/oEub+vY0cU8u3kvxsADV8yjrqk1aj2308HF80cA1ioZ58yq4I+v7oi77f9cNi5wInzl4uqEbUl06Ld+1clMvOmxhNsBmFBureyxeGwJn14wgvue3wbA0DwfUys65lAoyRDmVhdx+5PRkwpfPXkcP3psY8L3m19dFFH3xlOT/72NLs1m856jIWXBeaTFY4vZfbiRJ9/p/kXL735icr9IKtxzyWw+8+tX+roZSvWmaP+bC08Zu4AxQA1QATwjIpONMQeB4caYnSJSDTwpIm8ZY96L2KAxvwR+CTB79mxTU1OTwo8AtbW1pHqbqXDuKbBmyz6+/bf13PHmYf65M4srF49ixfSheF2RS0b3hP4am/5C4xObxiY+jU98Gp/YUhGbZIY/RLtqMCy4gojMACqNMQ93qzVKdUOsngq/vmwOT355cUrew+d2UpTkXAs/PHsqD149n2uXjIpb7/7L5/LS15dyw/IJCbd536VzmF9dFFFekGkNvUh2ec5wxdlWTwGP08GD1ywAYMEo632unhb78w7J9XLmDOt/Bxck6MkiIly7ZHTgcaonf/S6ndz76WMCy6DeedFM1nx9Kb/77LxAnfa2xrNoTDHSy1fuYlnajTlHqoOGkyT6DSrVj+wAKoMeVwA7o9T5qzGmxRizFdiIlWTAGLPT/rsFqAVm9HSDB5q51UU8/PljuW3lDDwuJ1/701oW3vIUP/3XJg50sSecUkqpwS2ZngpxrxqIiAP4MfDphBsaZN0N+5N4sXlvWwsAO3Z8QG1t/xqa+oNFGTS1maS+1937OsaILis5FPGat3dZPQw+3vtxoKy2tpamVuvn7HJAq98q/2jXR9TWWsMiDhxoAGDt2rX4d0ZeyUnUtmOCzsnD6z733HNkuYX1OyK7/sRyVoWfF7Z0PF4w1MVZY1ys2yesef6ZmK8Lf+/gx1u2WgeSJ1U52fj6GjYCK6sMY30eSl0N1NbWBuIQbNVcJ+++sYb7lmWxb/Pr1MYYIREtRl+c1NKp/bW+vi5w//o5Ph54pxlHfcfQlbfWrkU+cnHkiNXOXe+tZ8Pe0O9rxZCD/CXo8U9qMnhocwu1Ozp6nxzYf4Da2lpKMoSPG5Jfiq3QJ+xvjF8/2w1HW5LeZNLxibZdr7/j+6psDT8n6x8KvMIXZnr5zgtdmzzujFFuVr/XiYD2kKumerv0b4/+mxXVy8AYERkJfAicD1wQVuchYCVwn4gUYw2H2CIiBUC9MabJLl+I1YNShXE6hDOmDeX0qeU8s2kv9z63lR//613uevo9zp1dyTmzK5g0NPpy00oppVS4ZJIKia4a5ACTgVr76l4ZsFpEzjDGhPTbHazdDfuDeLHZ/MwWeGcDFRWV1NQM3OEPNcCM6fuYM7Iw+uoUG/fAGy8ztmooXzljGC9v209NzRgAbvc/xUnHH8e4b1jDcoeWl1NTY03i+L+b18C+vUydOpXjxpYENndN4zvsPdpETU3iFRVO/fA19tc1U1MzDx79e6D82IXHkpcZuXJFPB/sr4enn8IhsH7VssCKEWe3VwjafrDA928/H/x7WGc2w6aNDB8+nJqajkk1T6fjt9Meh2AnL10S+UZB73/lcdVMHpZHzbShEc/H+j3+qnwPl/7q5YjyqtJCdh7dB8BVZy3lKuB7f18P27cCcOzcWcwcXkDOW8/A4cPMmjWbKRV5ke8Z1L5PLDueZS1tjP/mo4GyeROGU1MzkcfnNLP7cBMn/+TpqO0M9+TXTmD6qsfj1jlpyjD+/NqHSW1v9ecWWsNQYnyfwTweD7SEXmHMzy+A/Va85s6dC8/Uxt3GmTOG8ZfXY7dt7JBs3t19NObzXZGV6ePSFcfznRcSf8ZoamZNZPV7b6a0TV0xa/oUaiZ2vleJ/psVyRjTKiKfAx7Dmi/hXmPMOhFZBbxijFltP3eSiKwH2oCvGmP2icgC4C4R8WP1xLwlfP4nFUpEOG5sCceNLeHd3Ue4s/Y97l+znfue38bE8lzOnlXBiulDk+6hp5RSanBKJqkQ96qBMeYQEFhrTkRqga+EJxSU6g3zR0UODWi3eGwJN6+YxFkzK8jyukLq5ngkZDypx9UxMihWT/ivLUu8XGW7n184M6KsuiSLbF8yu2B0Q/MzIiaSDPaXaxZw5i+eT2pbJtEsl4AJG9Z87OjES0x+YsawwNwNyRo3JCei7PxjKlkyvpQXtuyL+pp51YXMHF6Q1Pb/32fm8Kl7Oiam9LmdTCzPZf1Hh/nW6RO5aF4VAPmZHvI7MYFkZ+omI3hei2Djy3J4Z9eRkLL+MVijdyyfXMY/3t4FdCzjmsjQPB87D/XcUnpdWblExWaMeQR4JKzspqD7BviSfQuu8zzQ+SWKFABjh+Rw63nTuen0ifztzZ388dUdrHp4Pf/1yAaOH1/K2bMqqBlXGvLvo1JKKQVJJBWSvGqgBrAc+8Q2N6PrJ7gDgYjwKXsixViqi7PYsrcuqbH3XXX98vFMrchjwajEJ+XdMWN4AU9+eTFPbNgTN9kSLJmpBH52wQymVeRT3MUrV0VZHq6u6dwY/6trRrEpylXyBaOKufuZrXz15HFJb2vRmJKIsvbPfcyIQtzOzh8wl+WGLhXqcTq48dQJfGv1uk5vK5HfXD6Xe5/dyi9qO+aeSzQHRKYnuQnY7v/sXN784BA/ePSdmHXOmhm/t0Vxtoe9R5Mbl53ttf6fM7I4i6176xLUtlw0ryqQVIiVCyvM8oSskvL015Zw/0vbuemvqf0+Ll04gi8uHdvp3kZK9Wf5mR4+NX8En5o/go27jvDgazv482sf8s/1u8n1uVg+uZwzpg9l7sjCiFWFlFJKDU5JnUUmumoQVl7T/Wap3nT2rErqm9u4YG7PLBk5kGR6rZMvlyPyQCn50fXxXbW49ybNqy7JprokO6XbzM/wUFmYmVTdsVF6Hbz6zRPjvsYVNnRlzdeXMiTXFzWpsGR8KetXnUymJzUJsSQ6bERVmBXaS+GR6xbxwYH6kLIzpg3l+mXjWTlnOOfc+UKn36N9mcvibC/XLBkdllSIrD+vuijQs2NIro9fXzYnYunQcAtGFXOgrmOOgh98cgp3P7M1ZNWNieW5/JnYSYX/OnMKf1v7EX970xoll+N1cSTGiin3fPoYILmeMu2Cq7bFeN0l80eQm+HiO3+zer67nI5O95hJ5MGr5zOrqjCl21SqvxlXlsPXT5nAV08ex7Ob9rL6zZ08vHYnv3/lAwoy3ZwwYQjLJpexcHRx3J5zSiml0pummBVOh3DpwpG9tpyU6p5cn3VV9PjxpVGfH1USvyv2DcvHJ7V8ZXfdeMqE6HNbJFAadtV/iP14kr2qw0/Pnx7yfKyEQvhwjWCnB8/xQHI9NEpzOnpmzBhuDU34/PGjQ+q0r7gQbXu3rZxBaa6P2VUFCeMfbYWP7505hW23nAqA2xn6BtHCHN62xUHzgbRvJ5Hqkmyy7N4E7efvOQmG7LicwtRhHRO8+eL0khiWn5GwDblh7xf8vfrtRs2uihz6Er7twTRERKlUczsdLBlfyo/Pm84r3ziROy+aSc24Uh5dt4vP/PoVZt38ONfe/xp/fePDkF5CSimlBof07u+uVCfFu2DaX05K8jLdvHjD0sAykOEeunYhB+tjz4h/ZS/2lEil8ryMpE6Goy2rectZU0K66d52/nR+ct70iHrxEhGjS7M52NBCc/sSIYQmGsJNGRZ95nQR4YblE5hWkc++uma++dDbEXV+fdmcmNuNuk0Eh4DfwO0rZ5Cf6cbRhYROuPFlORHRTNSpQJCQpEqs+recldzQ95duPIH9dc0suOXJiO0V2vNYZPtcnDhxCI+v3w1Yk06+s+twaLsShGPVikkpHx6hVDrK8DhZNrmcZZPLaW7188KWfTz69i4eX7+Lv6/9CBFrTpjFY0tYPLaYycPy9KKFUkqlOU0qKBVFMleu+1JZni/mczk+Nzm+1I/x7uzQgJpxkXMXJOvOi2Zx1W9e5c6LIie47Irz54QO7RERgi/2R0tERPPiDUupa2rlCw+8HvX56pJstuytI8PtpDjby7ZbTmXtjoO88F7kBJOnTCkHiJpUSDQRmlMieyo8ct0innl3b0QvjGC/uvQYhubF7x0wt7oQj8vBA1fMI8fn5uYVk1n18DqG5WewaU9yqz/UjCvhu3/fYLU1xkcJ/05i8bmdDA3qddD+M1w0ppiJQ63eKydNLOOCucMZcb21isTwokw27rYms2zv0ZOXEX+fuHj+CE0qKNVJHpfDTh6U8N1PTOatDw/x740f8+939/CzJzdx2xOb8LgcTB2Wx6yqAmZWFdDcnKrBhEoppfoLTSoolaR0PgxaMX0Yv3x6C+fMqkxYN1HCxe0UWtoMVUVdnxF/2eSypLvop8Lwokze+vBQwgkNC7M8EfMnBPvxedN49f0DISfBUyvyY67kANYcEq1JrmIQeE3YmbqIML4sl/Fl8ecNWDIucsjMjadM4HuPbAjMk1Gc7eXd7y4PPD+lIo8/XrWA//zT2kDZObMqeOiND2lpi2z3zKqCkBP4314+lxNu7ViWc+HoIp7bHH0Vj3hyfa6IxFZVURbrvnNy1O+tfZ6G9g4bo0tzuOPCmVz929c6/d4A675zMpO+9Vjg8ejSyPlClBrMnA5hemU+0yvzue6EMRysb+bFLft4bftBXn3/AL96bht3Pb0FgNvX/Zs5IwuZMiyPieV5jC/P6dIkuUoppfoHTSoopagszGTtt09O6TbjDSXob374yamsmDa02yeKOT43NVFO3OP515cW8/9efJ97nt3a5fftTs+azxw7ktGl2SwaE381kplV+fz+lQ8YVZrN+XOGU5jt4a5/b4moF94jIDym/3fZXFr9/pCy310xj/nff5JHvrCIU257Jur7t0/w+dzmvUDHihftcz7E1hGchQk+46cXjOC+57dFlP/fZXNC3uen509P2PNBqcEuP9MTGCYB0NTaxtsfHuJ3T7zCx2Tw0Os7+c2L2wHwuR3MqCxgWmU+k4bmMnlYHlWFmSkZxqWUUqrnaVJBqSDX1Izm2vtfY3hRcqsbqEjWUIKBk1AA68T0pEllcessSHJZzs4aUZzFBXOHx0wqvHjDUo7GWD2hnSNGVuGv1y4kyxu998WGVctwOwWHQ1gSY9LPYOfOrmTBqOKkV/6IxekQnI7QNgXPl3HzJyaz82ADdwStbgEErmK2/7J681TjuLGhQ3lmj9BVH5TqLK/LyayqQo5Ue6ipmYPfb9i+v563dx7i1fcP8PK2/dzz7JZAD6hcn4tplflMKM+lujiLkcVZjC/P1YSeUkr1Q5pUUCrIqVPLOXVqaLf7ycPyeGbTXkqyY0/Kp9LfNTWjI8pGl+YwtsDB986c3K1tjyrJ5i/XLODMXzwf8Vy8+TPaxbqYN60y9rCLjARDPcKJSKcSCvOqC9m660BI2cs3npDwdZ+aVwVYMYm6kkk38lXhy5VOHpbL1o/rqGtuA0J7fFw0bzi/eXE7c0dGJhCSWbVCKRWfwyGMKM5iRHEWp0215oJpbvXz7u4jrNt5iDc+OMTr2w+wZuu2kAlyq4oymTIsj0lD86guyWJUSRYVBZm6pKVSSvUhTSoolcCXTxzL8sllgUnhBqvvnzWF//7nuxwzSK/SBnfDPW5MCa9vP0hlYQZfn5vBjOGRSxp2Vme38eDVC/jkHVYSQvpwZtEvnziWKxZXM+4bj4aUP3DFfGpra0PKSuKslhHu7FkVcZ+P9pFHlWSFzGcRLtPj4tZzp/GlP7wJwB+unE9dUxv76poit2/3heiJSU+VUtF5XA4mD8tj8rA8zjvGKmvzG3YebOC9j4+ybudh3tpxiNe3H+ThtR8FXicCZbk+hhdmUlWUSVVRVsf9wizyMnU/VkqpnqRJBaUScDkdcSfaGyyqirK4feWMvm5GQlctHsW197/WrYkig335xLH8z+PvhpRdt3QMK+cMpyzPx+aUvEvnzaoq4PH/OI4Tf/w0Pzx7aq+//5kzhnHXv7dw6tTypJaLOyPOqhSdMclO7l00tyriuSe+XBO43z5p5WlTy0PqnDWzIpBUyPS4yPS4oiY7BtKcIEqlM6fD6iVVWZgZMmfNoYYWtu2tY+veOt7fV8/7++vYvq+epzZ+zMdHdoRsIy/DTVVRZkjSoarQ+lua49W5G5RSqps0qaCUSqlLFlRx9zNbcTn6ZibvaENYuuPzS8fw+aVjQsocDklqWEJPGzMkp1dXyQg2viw36ffesGpZwmUyk1Wa60vqfYcXZbLlv07p1MlCskuLKqX6Xl6Gm2mV+VGHedU3t7J9f72VbNhnJR22769n7Y5D/OPtXbQFrbjjdTkCyYbhhVnW36JMKvIzKM31ketz9WlvMKWUGgg0qaCUSqmvnzKB65dPwKlXfpSts/M3pEp3rj62TwyZaJlRpVT/k+lxxVxmt6XNz86DDXbvhnq2ByUdntu8j4aWtpD6GW4nQ3K9DMn1MSTXR1mej9IcL2V59uNcHyU5Xp3TQSk1qGlSQSmVUiKCU/MJagAKvhh51eJRFGZ6uHj+iEDZiulD+esbO3u/YUqplHE7HdbwhyhD5IwxfHy0ie376tl5qJE9hxvZdaiR3Uea2H2okTd3HOSxdY00tfojXluQ6e5IPNiJhvxMN/mZHgoy3RRleynK8lCU7SHTo4ffSqn0ov9XU0op1W3//I/j2H24sa+bkZTbV85gfFlOzOe/ceoEhuT6Ioa93HrudH7wyd6fv0Ip1TtEhNIcH6U5sYe3GWM43NDKrsON7D7caP091MjuI43sOtTEniONbPjoMHuPNuGPMTVLhttJYZaH4mxPINlQmO2hOMvLng9bkHc/DiQgCrM8Sc1bo5RSfUmTCkoppbpt7JAcxg6JfaLen5weY9LIM6YN5Z5nt1IzriTq806H4HTowb1Sc3qCMgAADwRJREFUg5mIkJfpJi/Tzbg4yUm/33CkqZVD9S3sr29mf10Te482s+9oM/uONrG/rpm9dc3sPmwlIfYdbaa5zeoBcfdbL4VsK8frojDbQ16GO3DLzXCT43OR67PuF2S6Kcj0WLcs677X5dD5IJRSvUKTCkoppRQwrTK/zya+VEqlF4dDAgmA4UWZCesbYyUhHnniGcZMnh5IQLQnI/bXNXOooYVDDS18eKCBw40tHG5oDSQiohGxekXk+Fzk+Nzktv/NCL5vJSZyfK5Auc/txOty4HY6yPA4yfa6yHA7NUGhlIpJkwpKKaWUUkr1IREh1+emLMvBrKrCpF/X2NLGoYYWDtQ3c6CuhYP1zRyob+FgQzONzW3UN7dxtKk1kIQ4WN/M9v31HG5o4XBjCy1tyS2fK2LNR+FxOijM8lCW6yPH58Ljclg3pwOf24nP7SDD7cRnJyNyfC4y3C68bgc+lxOv24HX5cDrsup6g8o8Tu1ZodRApUkFpZRSSimlBiDrRN7JkNzOL3NsjKGp1W8nGNoTDy00tvhpabNudc1t1DW1Utdk9YpoavGzr66Z3Yca+ehQI81tfppbrVtTaxuNLX4aW9swyeUqQogQSDi0JypcTsHlENxOBw31DRSufw6304HLIYiAQ4RsryswJCTD7cTtFFx2nXZ+u0E+txOfy0p6+FxWT4z2sgyP9d4dZQ4cIjS1+mn1+8lwO3E5+2a5bKX6O00qKKWUUkopNciISCApURq5+maXtScrjja1crihhYaWNpparYREY2sbTS1WAsIqs/8G3W9saaO5zdBsn8y3thla2vzsbqsn2+ui1X4OoM0Ydh1q5JDd86KxJfZwkFRwO8XqieF24nQIAnjsRIjbJdQ3tVHX3EprmyHL6yI3w0Wm2zrdMhirt4edMHHaiZFYHGIlU9xOsesKThGyfVYSxSHQ0mZo8xuMgQ8/aGaTYwt+YzCAMeB0WNtxOqzkjMNhbcNpJ2qcDrGfJ7B9h/0ah3QkblwO67ficgrGWNs2WO/rEMHtsl7bZqz2tN/8xvrM2V5rWI0ICGL/xf5PaJmIECss7fFyOIQcryvpni3Gbldji9V7p66plVZ7JlWxt+txOSjI9ERdHtYYw9GmVlwOBz53ZI8aYwytfmP9Vu3fbKudlKtvbiXH68bnduBwSOD37LZ7/TQ0t3G4sQUg8D057Vsw6zlHyHfZn2hSQSmllFJKKZUSwcmK4mxvyrZbW1tLTc3cuHX8fvvkzu8PDO0QAacIBmu4SMfNSmA02Pcb7PKmoLI2v7ETCATqNDRb9fzG4DcEemm0tBkyi5xkeVw4nUJdUytHGlupb27FOk0Wmu1kS3Orte142ox18tnSagLv1eb3c6Sx44Q4wsYNXYjswORyCB6XA2OsnijGgMNh9Ubx20mNVvsWiPU/H0243UyPtTqLCHYCzM+RxpbAai4OgaJsL2W5Pppa29hzpImD9S09+EmjE7Fi0J70cTqEc2ZX8s3TJvZ6W0CTCkop1W88ePV81mzd39fNUEoppQYkh0PwOAQP0YcpZHsH/qmPMYaGlja7J4LV4wDg8SdrOXbRIgSr9wAQ6Dng95uIXgQtbX78xtDmx/5rnZi3GTuJ4bd6PLQnahpbrMSJQzp6E4iA30Brm59WvwlcYW/vDeFwCC12IqWxpS3Qg6K9lwOAsT5Ux3Mxxs4El7a2GfbXN9PS6sfhkEAPCL+x2hk40baHz7gcDj7Yvo3Ro6rJdDvJ9LrwOB0h7Whs8XOg3poU9UBdM9DeC8URmNS0zQ91Ta18fKSJ3UcacTsdzBlZSEGmxxqW4xTcDkdgCE6W10mG28mRxlaaWq14uxxWveZWPwfqmsn0usj1Wb0u2vz+QI8Hf1Ac2hMn7QmS1rb279Oq39ZmPTetMj9Fv7LOG/h7llJKpYlZVYWdmqBLKaWUUoOLiJDpiTyF87kkLZImPaW2dic1NaP7uhlpS2cbUUoppZRSSimlVJdoUkEppZRSSimllFJdokkFpZRSSimllFJKdYkmFZRSSimllFJKKdUlmlRQSimllFJKKaVUl2hSQSmllFJKKaWUUl2iSQWllFJKKaWUUkp1iSYVlFJKKaWUUkop1SWaVFBKKaVUvyIiy0Rko4hsFpHrY9Q5V0TWi8g6Ebk/qPwSEdlk3y7pvVYrpZRSg5OrrxuglFJKKdVORJzAz4ETgR3AyyKy2hizPqjOGOAGYKEx5oCIlNrlhcC3gNmAAV61X3ugtz+HUkopNVhoTwWllFJK9SdzgM3GmC3GmGbgAWBFWJ3PAj9vTxYYY/bY5ScDjxtj9tvPPQ4s66V2K6WUUoOSJhWUUkop1Z8MAz4IerzDLgs2FhgrIs+JyIsisqwTr1VKKaVUCunwB6WUUkr1JxKlzIQ9dgFjgBqgAnhGRCYn+VpE5ArgCoAhQ4ZQW1vbjeZGOnr0aMq3mS40NvFpfGLT2MSn8YlP4xNbKmLTZ0mFV199da+IvJ/izRYDe1O8zXShsYlP4xObxiY+jU9sGpv44sWnqjcb0s/sACqDHlcAO6PUedEY0wJsFZGNWEmGHViJhuDX1oa/gTHml8AvAUTk4yVLlujxSO/R2MSn8YlNYxOfxic+jU9s3T4eEWMiEvgDloi8YoyZ3dft6I80NvFpfGLT2MSn8YlNYxOfxic6EXEB7wJLgQ+Bl4ELjDHrguosA1YaYy4RkWLgdWA69uSMwEy76mvALGPM/l78CPrdxqGxiU/jE5vGJj6NT3wan9hSERsd/qCUUkqpfsMY0yoinwMeA5zAvcaYdSKyCnjFGLPafu4kEVkPtAFfNcbsAxCRm7ESEQCrejuhoJRSSg02mlRQSimlVL9ijHkEeCSs7Kag+wb4kn0Lf+29wL093UallFJKWdJt9Ydf9nUD+jGNTXwan9g0NvFpfGLT2MSn8Ulf+t3GprGJT+MTm8YmPo1PfBqf2Lodm7SaU0EppZRSSimllFK9J916KiillFJKKaWUUqqXpEVSQUSWichGEdksItf3dXt6kojcKyJ7ROTtoLJCEXlcRDbZfwvschGR2+y4rBWRmUGvucSuv0lELgkqnyUib9mvuU1Eoq353S+JSKWIPCUiG0RknYhcZ5cP+viIiE9EXhKRN+3YfMcuHykia+zP+XsR8djlXvvxZvv5EUHbusEu3ygiJweVD/j9UEScIvK6iDxsP9b4ACKyzf7dvyEir9hlg36/aici+SLyJxF5x/7/z3yNz+A0kPfzVJEUHaekI0nhcUo6khQeq6QrScFxSrpK1bFKOkrVcUpcxpgBfcOaGfo9oBrwAG8CE/u6XT34eY/DWirr7aCyHwLX2/evB35g3z8F+AcgwDxgjV1eCGyx/xbY9wvs514C5tuv+QewvK8/cydiUw7MtO/nYC1JNlHjY7Dbm23fdwNr7M/8B+B8u/xO4Gr7/jXAnfb984Hf2/cn2vuYFxhp73vOdNkPsSZ9ux942H6s8bE+1zagOKxs0O9XQbH4NXC5fd8D5Gt8Bt9toO/nKYxDt49T0vVGio5T0vVGio5V0vlGN49T0vlGCo5V0vWWiuOUhO/R1x8yBUGaDzwW9PgG4Ia+blcPf+YRYf9YbwTK7fvlwEb7/l1Y63iH1ANWAncFld9ll5UD7wSVh9QbaDfgr8CJGp+IuGRird0+F9gLuOzywL6EtVzbfPu+y64n4ftXe7102A+BCuAJ4HjgYfvzanxMzH+odb+y2psLbMWeo0jjM3hvA30/T3EsRtCN45S+bn8vxqlLxyl93e5eik2Xj1X6uu09GJNuH6f09Wfo4fh0+1ilrz9DD8UlJccpid4nHYY/DAM+CHq8wy4bTIYYYz4CsP+W2uWxYhOvfEeU8gHH7uY1AyvLrfEh0GXuDWAP8DjWFbWDxphWu0rw5wnEwH7+EFBE52M2kPwE+Brgtx8XofFpZ4B/isirInKFXab7laUa+Bj4ld0l9X9FJAuNz2A00PfzntTZ/SHtdfM4JW2l6FglXaXiOCWdpeJYJR2l6jglrnRIKkQbW2p6vRX9U6zYdLZ8QBGRbOBB4IvGmMPxqkYpS9v4GGPajDHTsTLdc4AJ0arZfwdVbETkNGCPMebV4OIoVQdlfICFxpiZwHLgWhE5Lk7dwRYbF1ZX7zuMMTOAOqxuhLEMtvgMJvpddd6gjFkKjlPSVoqOVdJOCo9T0lkqjlXSUaqOU+JKh6TCDqAy6HEFsLOP2tJXdotIOYD9d49dHis28coropQPGCLixvqH+rfGmD/bxRqfIMaYg0At1jipfBFx2U8Ff55ADOzn84D9dD5mA8VC4AwR2QY8gNW18CdofAAwxuy0/+4B/oJ1oKf7lWUHsMMYs8Z+/Cesf7w1PoPPgN7Pe1hn94e0laLjlLTXzWOVdJSq45S0laJjlXSUquOUuNIhqfAyMMae/dSDNRnJ6j5uU29bDVxi378Ea4xee/nF9iye84BDdveWx4CTRKTAnunzJKwxWB8BR0RknogIcHHQtvo9u833ABuMMbcGPTXo4yMiJSKSb9/PAE4ANgBPAWfb1cJj0x6zs4EnjTWwajVwvj2r8EhgDNYkcgN6PzTG3GCMqTDGjMBq+5PGmAvR+CAiWSKS034fa394G92vADDG7AI+EJFxdtFSYD0an8FowO7nvaCz+0NaSuFxSlpK4bFK2knhcUpaSuGxStpJ4XFKwjca8DesWSrfxRp3dWNft6eHP+vvgI+AFqxM0mewxkg9AWyy/xbadQX4uR2Xt4DZQdu5DNhs3y4NKp+NtRO+B/yMATSpC3AsVvectcAb9u0UjY8BmAq8bsfmbeAmu7wa66R3M/BHwGuX++zHm+3nq4O2daP9+TcSNAt9uuyHQA0dsyoP+vjYMXjTvq1rb7vuVyExmg68Yu9fD2Gt3qDxGYS3gbqfpzgGKTlOSccbKTxOSccbKTxWSecb3TxOSccbKTxWScdbqo5T4t3EfrFSSimllFJKKaVUp6TD8AellFJKKaWUUkr1AU0qKKWUUkoppZRSqks0qaCUUkoppZRSSqku0aSCUkoppZRSSimlukSTCkoppZRSSimllOoSTSoopZRSSimllFKqSzSpoJRSSimllFJKqS7RpIJSSimllFJKKaW65P8Dt7C/SFFlhCYAAAAASUVORK5CYII=\n", 135 | "text/plain": [ 136 | "" 137 | ] 138 | }, 139 | "metadata": {}, 140 | "output_type": "display_data" 141 | }, 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "Loss 0.51271\n", 147 | "Val MSE: 0.61727\n", 148 | "BREAK. There is no improvment for 5000 steps\n", 149 | "Best step: 53100\n", 150 | "Best Val MSE: 0.61708\n" 151 | ] 152 | } 153 | ], 154 | "source": [ 155 | "for batch in lib.iterate_minibatches(data.X_train, data.y_train, batch_size=512, \n", 156 | " shuffle=True, epochs=float('inf')):\n", 157 | " metrics = trainer.train_on_batch(*batch, device=device)\n", 158 | " \n", 159 | " loss_history.append(metrics['loss'])\n", 160 | "\n", 161 | " if trainer.step % report_frequency == 0:\n", 162 | " trainer.save_checkpoint()\n", 163 | " trainer.average_checkpoints(out_tag='avg')\n", 164 | " trainer.load_checkpoint(tag='avg')\n", 165 | " mse = trainer.evaluate_mse(\n", 166 | " data.X_valid, data.y_valid, device=device, batch_size=1024)\n", 167 | "\n", 168 | " if mse < best_mse:\n", 169 | " best_mse = mse\n", 170 | " best_step_mse = trainer.step\n", 171 | " trainer.save_checkpoint(tag='best_mse')\n", 172 | " mse_history.append(mse)\n", 173 | " \n", 174 | " trainer.load_checkpoint() # last\n", 175 | " trainer.remove_old_temp_checkpoints()\n", 176 | "\n", 177 | " clear_output(True)\n", 178 | " plt.figure(figsize=[18, 6])\n", 179 | " plt.subplot(1, 2, 1)\n", 180 | " plt.plot(loss_history)\n", 181 | " plt.title('Loss')\n", 182 | " plt.grid()\n", 183 | " plt.subplot(1, 2, 2)\n", 184 | " plt.plot(mse_history)\n", 185 | " plt.title('MSE')\n", 186 | " plt.grid()\n", 187 | " plt.show()\n", 188 | " print(\"Loss %.5f\" % (metrics['loss']))\n", 189 | " print(\"Val MSE: %0.5f\" % (mse))\n", 190 | " if trainer.step > best_step_mse + early_stopping_rounds:\n", 191 | " print('BREAK. There is no improvment for {} steps'.format(early_stopping_rounds))\n", 192 | " print(\"Best step: \", best_step_mse)\n", 193 | " print(\"Best Val MSE: %0.5f\" % (best_mse))\n", 194 | " break" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 8, 200 | "metadata": {}, 201 | "outputs": [ 202 | { 203 | "name": "stdout", 204 | "output_type": "stream", 205 | "text": [ 206 | "Loaded logs/year_node_shallow_2019.08.27_17:32/checkpoint_best_mse.pth\n", 207 | "Best step: 53100\n", 208 | "Test MSE: 0.64902\n" 209 | ] 210 | } 211 | ], 212 | "source": [ 213 | "trainer.load_checkpoint(tag='best_mse')\n", 214 | "mse = trainer.evaluate_mse(data.X_test, data.y_test, device=device)\n", 215 | "print('Best step: ', trainer.step)\n", 216 | "print(\"Test MSE: %0.5f\" % (mse))" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 9, 222 | "metadata": {}, 223 | "outputs": [ 224 | { 225 | "data": { 226 | "text/plain": [ 227 | "77.51119264257828" 228 | ] 229 | }, 230 | "execution_count": 9, 231 | "metadata": {}, 232 | "output_type": "execute_result" 233 | } 234 | ], 235 | "source": [ 236 | "mse * std ** 2" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": null, 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [] 245 | } 246 | ], 247 | "metadata": { 248 | "kernelspec": { 249 | "display_name": "Python 3", 250 | "language": "python", 251 | "name": "python3" 252 | }, 253 | "language_info": { 254 | "codemirror_mode": { 255 | "name": "ipython", 256 | "version": 3 257 | }, 258 | "file_extension": ".py", 259 | "mimetype": "text/x-python", 260 | "name": "python", 261 | "nbconvert_exporter": "python", 262 | "pygments_lexer": "ipython3", 263 | "version": "3.6.4" 264 | } 265 | }, 266 | "nbformat": 4, 267 | "nbformat_minor": 2 268 | } 269 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.1.0 2 | numpy>=0.13 3 | scipy>=1.2.0 4 | scikit-learn>=0.17 5 | catboost==0.12.2 6 | xgboost==0.81 7 | matplotlib 8 | tqdm 9 | tensorboardX 10 | pandas 11 | prefetch_generator 12 | requests 13 | category_encoders 14 | https://github.com/facebookresearch/qhoptim/archive/master.zip 15 | --------------------------------------------------------------------------------