├── shift_happens ├── __init__.py ├── label_shift │ ├── README.md │ └── __init__.py ├── gendist │ ├── gendist │ │ ├── __init__.py │ │ ├── models.py │ │ ├── processing.py │ │ └── training.py │ ├── half-moons.mp4 │ ├── setup.py │ ├── experiments │ │ ├── multiprocess_test.py │ │ ├── mnist_zero_shot_test.py │ │ ├── mnist_rotation_meta.py │ │ └── mnist_rotation_data.py │ └── notebooks │ │ ├── dojax.py │ │ └── 013-metadata-inference.ipynb └── imagenet_flax │ ├── requirements.txt │ ├── logs │ └── lenet │ │ ├── checkpoint_1500 │ │ ├── checkpoint_3000 │ │ └── checkpoint_4500 │ ├── README.md │ ├── agents │ ├── mlp.py │ ├── lenet.py │ ├── models.py │ ├── nearest_centroid_classifier.py │ ├── nearest_centroid_classifier_old.py │ ├── models_test.py │ └── resnet.py │ ├── configs │ ├── default.py │ ├── resnet20_config.py │ ├── lenet_config.py │ └── tpu_dynamic.py │ ├── utils.py │ ├── gdumb.py │ ├── ncc_demo.py │ ├── imagenet_fake_data_benchmark.py │ ├── main.py │ ├── train_test.py │ ├── environment.py │ ├── gdumb_old.py │ └── train.py ├── README.md ├── LICENSE └── .gitignore /shift_happens/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shift_happens/label_shift/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shift_happens/label_shift/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shift-happens 2 | Research code for ML with distribution shift 3 | -------------------------------------------------------------------------------- /shift_happens/gendist/gendist/__init__.py: -------------------------------------------------------------------------------- 1 | from . import training, processing, models -------------------------------------------------------------------------------- /shift_happens/gendist/half-moons.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/probml/shift-happens/main/shift_happens/gendist/half-moons.mp4 -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/requirements.txt: -------------------------------------------------------------------------------- 1 | clu 2 | ml-collections 3 | optax 4 | tensorflow 5 | tensorflow-datasets 6 | imax -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/logs/lenet/checkpoint_1500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/probml/shift-happens/main/shift_happens/imagenet_flax/logs/lenet/checkpoint_1500 -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/logs/lenet/checkpoint_3000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/probml/shift-happens/main/shift_happens/imagenet_flax/logs/lenet/checkpoint_3000 -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/logs/lenet/checkpoint_4500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/probml/shift-happens/main/shift_happens/imagenet_flax/logs/lenet/checkpoint_4500 -------------------------------------------------------------------------------- /shift_happens/gendist/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="gendist", 5 | packages=find_packages(), 6 | install_requires=[ 7 | "jaxlib", 8 | "jax" 9 | ] 10 | ) -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/README.md: -------------------------------------------------------------------------------- 1 | # Continual Learning 2 | 3 | There are three different models 4 | - ResNet 5 | - LeNet 6 | - Nearest Centroid Classifier 7 | 8 | ### Running the Training of Deep Models 9 | You can train a single model by 10 | 11 | ```shell 12 | python main.py --workdir=./logs/lenet --config=configs/lenet_config.py 13 | ``` 14 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/mlp.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | 3 | from typing import Any, Sequence 4 | from flax import linen as nn 5 | 6 | ModuleDef = Any 7 | 8 | 9 | class MLP(nn.Module): 10 | layer_dims: Sequence[int] 11 | num_classes: int 12 | dtype: Any = jnp.float32 13 | 14 | @nn.compact 15 | def __call__(self, x, train: bool = True): 16 | x = x.reshape((x.shape[0], -1)) 17 | for layer_dim in self.layer_dims: 18 | x = nn.Dense(features=layer_dim, dtype=self.dtype)(x) 19 | x = nn.relu(x) 20 | x = nn.Dense(self.num_classes, dtype=self.dtype)(x) 21 | return x 22 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/configs/default.py: -------------------------------------------------------------------------------- 1 | import ml_collections 2 | 3 | 4 | def get_config(): 5 | """Get the default hyperparameter configuration.""" 6 | config = ml_collections.ConfigDict() 7 | 8 | # As defined in the `models` module. 9 | config.model = 'ResNet18' 10 | # `name` argument of tensorflow_datasets.builder() 11 | config.dataset = 'cifar10' 12 | config.image_size = -1 13 | 14 | config.learning_rate = 1e-7 15 | config.batch_size = -1 16 | 17 | config.num_epochs = 1 18 | config.log_every_steps = 1 19 | 20 | config.momentum_decay = 0.9 21 | config.weight_decay = 100. 22 | 23 | config.cache = False 24 | config.half_precision = False 25 | 26 | # If num_train_steps==-1 then the number of training steps is calculated from 27 | # num_epochs using the entire dataset. Similarly for steps_per_eval. 28 | config.num_train_steps = 1 29 | config.steps_per_eval = 1 30 | 31 | return config 32 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/configs/resnet20_config.py: -------------------------------------------------------------------------------- 1 | import ml_collections 2 | 3 | 4 | def get_config(): 5 | """Get the default hyperparameter configuration.""" 6 | config = ml_collections.ConfigDict() 7 | 8 | # As defined in the `models` module. 9 | config.model = 'ResNet18' 10 | # `name` argument of tensorflow_datasets.builder() 11 | config.dataset = 'cifar10' 12 | config.image_size = -1 13 | 14 | config.learning_rate = 1e-7 15 | config.batch_size = 80 16 | 17 | config.num_epochs = 300 18 | config.log_every_steps = 100 19 | 20 | config.momentum_decay = 0.9 21 | config.weight_decay = 100. 22 | 23 | config.cache = False 24 | config.half_precision = False 25 | 26 | # If num_train_steps==-1 then the number of training steps is calculated from 27 | # num_epochs using the entire dataset. Similarly for steps_per_eval. 28 | config.num_train_steps = -1 29 | config.steps_per_eval = -1 30 | return config 31 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/configs/lenet_config.py: -------------------------------------------------------------------------------- 1 | import ml_collections 2 | 3 | 4 | def get_config(seed=2): 5 | """Get the default hyperparameter configuration.""" 6 | config = ml_collections.ConfigDict() 7 | 8 | # As defined in the `models` module. 9 | config.model = 'LeNet' 10 | # `name` argument of tensorflow_datasets.builder() 11 | config.dataset = 'mnist' 12 | config.seed = seed 13 | config.image_size = -1 14 | 15 | config.learning_rate = 0.001 16 | config.batch_size = 80 17 | config.train_freq = 5 18 | 19 | config.num_epochs = 200 20 | config.log_every_steps = 100 21 | 22 | config.momentum_decay = 0.9 23 | config.weight_decay = 0.001 24 | 25 | config.cache = False 26 | config.half_precision = False 27 | 28 | # If num_train_steps==-1 then the number of training steps is calculated from 29 | # num_epochs using the entire dataset. Similarly for steps_per_eval. 30 | config.num_train_steps = -1 31 | config.steps_per_eval = -1 32 | return config 33 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/lenet.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | 3 | from typing import Any, Callable 4 | from flax import linen as nn 5 | 6 | ModuleDef = Any 7 | 8 | 9 | class LeNet5(nn.Module): 10 | num_classes: int 11 | act: Callable[[jnp.ndarray], jnp.ndarray] = nn.relu 12 | dtype: Any = jnp.float32 13 | 14 | @nn.compact 15 | def __call__(self, x, train: bool = True): 16 | """Network inspired by LeNet-5.""" 17 | x = self.act(nn.Conv(features=6, kernel_size=(5, 5), padding="SAME", dtype=self.dtype)(x)) 18 | x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2), padding="VALID") 19 | x = self.act(nn.Conv(features=16, kernel_size=(5, 5), padding="VALID", dtype=self.dtype)(x)) 20 | x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2), padding="VALID") 21 | x = x.reshape((x.shape[0], -1)) 22 | x = self.act(nn.Dense(features=120, dtype=self.dtype)(x)) 23 | x = self.act(nn.Dense(features=84, dtype=self.dtype)(x)) 24 | x = nn.Dense(features=self.num_classes, dtype=self.dtype)(x) 25 | return x 26 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/models.py: -------------------------------------------------------------------------------- 1 | import jax 2 | from functools import partial 3 | 4 | from agents.mlp import MLP 5 | from agents.lenet import LeNet5 6 | from agents.resnet import ResNet, BottleneckResNetBlock, ResNetBlock 7 | 8 | ResNet18 = partial(ResNet, stage_sizes=[2, 2, 2, 2], 9 | block_cls=ResNetBlock) 10 | ResNet34 = partial(ResNet, stage_sizes=[3, 4, 6, 3], 11 | block_cls=ResNetBlock) 12 | ResNet50 = partial(ResNet, stage_sizes=[3, 4, 6, 3], 13 | block_cls=BottleneckResNetBlock) 14 | ResNet101 = partial(ResNet, stage_sizes=[3, 4, 23, 3], 15 | block_cls=BottleneckResNetBlock) 16 | ResNet152 = partial(ResNet, stage_sizes=[3, 8, 36, 3], 17 | block_cls=BottleneckResNetBlock) 18 | ResNet200 = partial(ResNet, stage_sizes=[3, 24, 36, 3], 19 | block_cls=BottleneckResNetBlock) 20 | LeNet = partial(LeNet5, act=jax.nn.relu) 21 | 22 | mlp = partial(MLP, layer_dims=[256, 256]) 23 | 24 | # Used for testing only. 25 | _ResNet1 = partial(ResNet, stage_sizes=[1], block_cls=ResNetBlock) 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Probabilistic machine learning 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/utils.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | 3 | from jax.nn import log_softmax 4 | 5 | from collections import defaultdict 6 | 7 | 8 | def get_accuracy(y, logits): 9 | preds = jnp.argmax(logits, axis=-1) 10 | return jnp.mean(y == preds) 11 | 12 | 13 | def eval_step(env, state, predict_fn, prior=None, rotations=None): 14 | metrics = defaultdict(lambda: jnp.array([])) 15 | if prior is None: 16 | mean, *_ = state 17 | num_classes = len(mean) 18 | prior = log_softmax(jnp.ones((num_classes,))) 19 | 20 | if rotations is None: 21 | rotations = jnp.argwhere(env.rot_mat.sum(axis=0) > 0).flatten().tolist() 22 | 23 | for batch in env.test_data: 24 | input, label = jnp.array(batch['image']), jnp.array(batch['label']) 25 | if input.shape[-1] == 1: 26 | input = jnp.repeat(input, axis=-1, repeats=3) 27 | 28 | for degrees in rotations: 29 | if degrees: 30 | x = env.apply(input, degrees) 31 | else: 32 | x = input 33 | logits = predict_fn(state, x, prior) 34 | metrics[degrees] = jnp.append(metrics[degrees], get_accuracy(label, logits)) 35 | 36 | return {k: jnp.mean(v) for k, v in metrics.items()} 37 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/gdumb.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | from jax import ops, random 3 | 4 | from collections import defaultdict 5 | 6 | 7 | def init_sampler(): 8 | counts = defaultdict(int) 9 | dataset = (jnp.empty([]), jnp.empty([])) 10 | return counts, dataset 11 | 12 | 13 | def sample(state, carry, memory_size=200): 14 | counts, dataset = state 15 | key, x, y = carry 16 | n = len(counts.keys()) 17 | inputs, labels = dataset 18 | nperclass = memory_size / (n if n > 0 else 1) 19 | 20 | if n == 0: 21 | counts[y.item()] = counts[y.item()] + 1 22 | return counts, (x[None, ...], y) 23 | 24 | if y.item() not in counts or counts[y.item()] < nperclass: 25 | if len(inputs) >= memory_size: 26 | c_max = max(counts, key=counts.get) 27 | sample_key, key = random.split(key) 28 | indices = jnp.argwhere(labels == c_max) 29 | row = random.choice(sample_key, indices, shape=(1,)) 30 | inputs = ops.index_update(inputs, jnp.index_exp[row], x) 31 | labels = ops.index_update(labels, jnp.index_exp[row], y) 32 | counts[c_max] -= 1 33 | else: 34 | inputs = jnp.vstack([inputs, x[None, ...]]) 35 | labels = jnp.append(labels, y) 36 | 37 | counts[y.item()] += 1 38 | 39 | dataset = (inputs, labels) 40 | 41 | return counts, dataset 42 | -------------------------------------------------------------------------------- /shift_happens/gendist/experiments/multiprocess_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example of the multiprocessing module in gendist for 3 | transforming multiple images. 4 | 5 | In this example, we transform each image in a 6 | dataset by a different angle. The first image 7 | is rotated by 0 degrees and the last image 8 | is rotated by 360 degrees. 9 | """ 10 | 11 | import gendist 12 | import torchvision 13 | import numpy as np 14 | from augly import image 15 | 16 | def processor(X, angle): 17 | X_shift = image.aug_np_wrapper(X, image.rotate, degrees=angle) 18 | size_im = X_shift.shape[0] 19 | size_pad = (28 - size_im) // 2 20 | size_pad_mod = (28 - size_im) % 2 21 | X_shift = np.pad(X_shift, (size_pad, size_pad + size_pad_mod)) 22 | 23 | return X_shift 24 | 25 | 26 | if __name__ == "__main__": 27 | from time import time 28 | 29 | init_time = time() 30 | mnist_train = torchvision.datasets.MNIST(root="./data", train=True, download=True) 31 | images = np.array(mnist_train.data) / 255.0 32 | 33 | n_configs = len(images) 34 | degrees = np.linspace(0, 360, n_configs) 35 | configs = [{"angle": float(angle)} for angle in degrees] 36 | process = gendist.processing.Factory(processor) 37 | images_proc = process(images, configs, n_processes=90) 38 | end_time = time() 39 | 40 | print(f"Time elapsed: {end_time - init_time:.2f}s") 41 | print(images_proc.shape) 42 | -------------------------------------------------------------------------------- /shift_happens/gendist/gendist/models.py: -------------------------------------------------------------------------------- 1 | import flax.linen as nn 2 | from typing import Callable 3 | import jax.numpy as jnp 4 | 5 | class LeNet5(nn.Module): 6 | num_classes : int 7 | activation : Callable[[jnp.ndarray], jnp.ndarray] = nn.relu 8 | @nn.compact 9 | def __call__(self, x): 10 | """Aleyna's network inspired by LeNet-5.""" 11 | x = x if len(x.shape) > 1 else x[None, :] 12 | x = x.reshape((x.shape[0], 28, 28, 1)) 13 | x = self.activation(nn.Conv(features=6, kernel_size=(5,5), padding="SAME")(x)) 14 | x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2), padding="VALID") 15 | x = self.activation(nn.Conv(features=16, kernel_size=(5,5), padding="VALID")(x)) 16 | x = nn.avg_pool(x, window_shape=(2, 2), strides=(2, 2), padding="VALID") 17 | x = x.reshape((x.shape[0], -1)) 18 | x = self.activation(nn.Dense(features=120)(x)) 19 | x = self.activation(nn.Dense(features=84)(x)) 20 | x = nn.Dense(features=self.num_classes)(x) 21 | x = nn.log_softmax(x) 22 | return x 23 | 24 | 25 | class MLPDataV1(nn.Module): 26 | num_outputs: int 27 | @nn.compact 28 | def __call__(self, x): 29 | x = nn.relu(nn.Dense(800)(x)) 30 | x = nn.relu(nn.Dense(500)(x)) 31 | x = nn.Dense(self.num_outputs)(x) 32 | x = nn.log_softmax(x) 33 | return x 34 | 35 | 36 | class MLPWeightsV1(nn.Module): 37 | num_outputs: int 38 | @nn.compact 39 | def __call__(self, x): 40 | x = nn.relu(nn.Dense(200)(x)) 41 | x = nn.Dense(self.num_outputs)(x) 42 | return x 43 | 44 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/nearest_centroid_classifier.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | from jax import vmap, ops, lax 3 | from jax.scipy.stats import multivariate_normal 4 | 5 | from typing import Any, Tuple 6 | 7 | Array = Any 8 | 9 | 10 | def init(inputs: Array, labels: Array): 11 | _, *input_shape = inputs.shape 12 | num_classes = jnp.unique(labels).size 13 | 14 | input_size = jnp.product(jnp.array(input_shape)) 15 | 16 | mean = jnp.zeros((num_classes, input_size)) 17 | scale = jnp.repeat(jnp.eye(input_size)[None, ...], repeats=num_classes, axis=0) 18 | counts = jnp.zeros((num_classes,)) 19 | state = (mean, scale, counts) 20 | 21 | def init_step(state, carry): 22 | input, cls = carry 23 | state = update(state, input.reshape((1, -1)), cls) 24 | return state, None 25 | 26 | state, _ = lax.scan(init_step, state, (inputs, labels)) 27 | 28 | return state 29 | 30 | 31 | def predict(state: Tuple[Array, Array, Array], inputs: Array, prior: Array): 32 | mean, scale, _ = state 33 | 34 | def cond_prob(input): 35 | return multivariate_normal.logpdf(input.reshape((1, -1)), 36 | mean=mean, 37 | cov=scale) 38 | 39 | logits = vmap(cond_prob)(inputs) 40 | return logits + prior 41 | 42 | 43 | def update(state: Tuple[Array, Array, Array], inputs: Array, cls: int): 44 | mean, scale, counts = state 45 | n = counts[cls] + len(inputs) 46 | 47 | prev_sum = mean[cls] * counts[cls] 48 | cur_sum = inputs.reshape((len(inputs), -1)).sum(axis=0) 49 | running_avg = (prev_sum + cur_sum) / n 50 | 51 | mean = ops.index_update(mean, jnp.index_exp[cls], running_avg) 52 | state = (mean, scale, counts) 53 | return state 54 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/nearest_centroid_classifier_old.py: -------------------------------------------------------------------------------- 1 | import jax.numpy as jnp 2 | from jax import vmap, ops, lax 3 | from jax.scipy.stats import multivariate_normal 4 | 5 | from typing import Any, Tuple 6 | 7 | Array = Any 8 | 9 | 10 | def init(inputs: Array, labels: Array): 11 | _, *input_shape = inputs.shape 12 | num_classes = jnp.unique(labels).size 13 | 14 | input_size = jnp.product(jnp.array(input_shape)) 15 | 16 | mean = jnp.zeros((num_classes, input_size)) 17 | scale = jnp.repeat(jnp.eye(input_size)[None, ...], repeats=num_classes, axis=0) 18 | counts = jnp.zeros((num_classes,)) 19 | state = (mean, scale, counts) 20 | 21 | def init_step(state, carry): 22 | input, cls = carry 23 | state = update(state, input.reshape((1, -1)), cls) 24 | return state, None 25 | 26 | state, _ = lax.scan(init_step, state, (inputs, labels)) 27 | 28 | return state 29 | 30 | 31 | def predict(state: Tuple[Array, Array, Array], inputs: Array, prior: Array): 32 | mean, scale, _ = state 33 | 34 | def cond_prob(input): 35 | return multivariate_normal.logpdf(input.reshape((1, -1)), 36 | mean=mean, 37 | cov=scale) 38 | 39 | logits = vmap(cond_prob)(inputs) 40 | return logits + prior 41 | 42 | 43 | def update(state: Tuple[Array, Array, Array], inputs: Array, cls: int): 44 | mean, scale, counts = state 45 | n = counts[cls] + len(inputs) 46 | 47 | prev_sum = mean[cls] * counts[cls] 48 | cur_sum = inputs.reshape((len(inputs), -1)).sum(axis=0) 49 | running_avg = (prev_sum + cur_sum) / n 50 | 51 | mean = ops.index_update(mean, jnp.index_exp[cls], running_avg) 52 | state = (mean, scale, counts) 53 | return state 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | scripts/.DS_Store 3 | .DS_Store 4 | MNIST 5 | outputs 6 | 7 | *.mp4 8 | *.png 9 | *.gif 10 | *.pdf 11 | *.jpg 12 | *.jpeg 13 | 14 | __pycache__ 15 | *.py[cod] 16 | *$py.class 17 | 18 | # Distribution / packaging 19 | .Python build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | *.manifest 35 | *.spec 36 | 37 | # Log files 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | *.log 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | .pytest_cache/ 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # Jupyter Notebook 62 | .ipynb_checkpoints 63 | 64 | # IPython 65 | profile_default/ 66 | ipython_config.py 67 | 68 | # pyenv 69 | .python-version 70 | 71 | # pyflow 72 | __pypackages__/ 73 | 74 | # Environment 75 | .env 76 | .venv 77 | env/ 78 | venv/ 79 | ENV/ 80 | 81 | # If you are using PyCharm # 82 | .idea/* 83 | 84 | # Sublime Text 85 | *.tmlanguage.cache 86 | *.tmPreferences.cache 87 | *.stTheme.cache 88 | *.sublime-workspace 89 | *.sublime-project 90 | 91 | # sftp configuration file 92 | sftp-config.json 93 | 94 | # Package control specific files Package 95 | Control.last-run 96 | Control.ca-list 97 | Control.ca-bundle 98 | Control.system-ca-bundle 99 | GitHub.sublime-settings 100 | 101 | # Visual Studio Code # 102 | .vscode/* 103 | !.vscode/settings.json 104 | !.vscode/tasks.json 105 | !.vscode/launch.json 106 | !.vscode/extensions.json 107 | .history 108 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/ncc_demo.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import jax 3 | import jax.numpy as jnp 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | import train 8 | import agents.nearest_centroid_classifier as agent 9 | from environment import Environment 10 | 11 | if __name__ == '__main__': 12 | key = jax.random.PRNGKey(0) 13 | local_device_count = jax.local_device_count() 14 | 15 | ds_name, num_classes = 'mnist', 10 16 | batch_size, num_pulls = 256, 10 17 | T = 200 18 | 19 | trans_mat, rot_mat = train.init_trans_mat_and_rot_mat(key, num_classes=num_classes) 20 | env = Environment('mnist', class_trans_mat=trans_mat, rot_mat=rot_mat, batch_size=batch_size) 21 | inputs, labels = env.warmup(num_pulls=num_pulls) 22 | 23 | state = agent.init(inputs, labels) 24 | prior = jax.nn.log_softmax(jnp.ones((num_classes))) 25 | 26 | accuracies, results = [defaultdict(list)] * num_classes, [] 27 | for t in range(T): 28 | batch = env.get_data() 29 | logits = agent.predict(state, batch['image'], prior) 30 | loss = train.cross_entropy_loss(logits, batch['label'], num_classes) 31 | accuracy = jnp.mean(jnp.argmax(logits, -1) == batch['label']) 32 | accuracies[env.current_class][env.rot.item()].append(accuracy) 33 | results.append(accuracy) 34 | state = agent.update(state, batch['image'], env.current_class) 35 | 36 | plt.style.use('seaborn-darkgrid') 37 | plt.figure(figsize=(12, 8)) 38 | plt.plot(np.arange(len(results)), results) 39 | plt.xlabel("Iteration") 40 | plt.ylabel("Accuracy") 41 | plt.savefig('./ncc_training.png') 42 | plt.show() 43 | 44 | '''fig, axes = plt.subplots(nrows=2, ncols=5, figsize=(20, 8)) 45 | rotations = sorted(accuracies[0].keys()) 46 | for i, ax in enumerate(axes.flatten()): 47 | for rot, acc in accuracies[i].items(): 48 | ax.plot(np.arange(len(acc)), np.array(acc), 'o-', label=str(rot)) 49 | ax.legend() 50 | ax.set_title(f'Class {i}') 51 | ax.set_xlabel("Iteration") 52 | ax.set_ylabel("Accuracy") 53 | 54 | plt.tight_layout() 55 | plt.savefig('./ncc_cls_acc.png') 56 | plt.show()''' 57 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/imagenet_fake_data_benchmark.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Benchmark for the ImageNet example using fake data for quick perf results.""" 16 | 17 | import pathlib 18 | import time 19 | 20 | from absl.testing import absltest 21 | from flax.testing import Benchmark 22 | import jax 23 | 24 | import tensorflow_datasets as tfds 25 | 26 | # Local imports. 27 | from configs import fake_data_benchmark as config_lib 28 | import train 29 | 30 | # Parse absl flags test_srcdir and test_tmpdir. 31 | jax.config.parse_flags_with_absl() 32 | 33 | 34 | class ImagenetBenchmarkFakeData(Benchmark): 35 | """Runs ImageNet using fake data for quickly measuring performance.""" 36 | 37 | def test_fake_data(self): 38 | workdir = self.get_tmp_model_dir() 39 | config = config_lib.get_config() 40 | # Go two directories up to the root of the flax directory. 41 | flax_root_dir = pathlib.Path(__file__).parents[2] 42 | data_dir = str(flax_root_dir) + '/.tfds/metadata' 43 | 44 | # Warm-up first so that we are not measuring just compilation. 45 | with tfds.testing.mock_data(num_examples=1024, data_dir=data_dir): 46 | train.train_and_evaluate(config, workdir) 47 | 48 | start_time = time.time() 49 | with tfds.testing.mock_data(num_examples=1024, data_dir=data_dir): 50 | train.train_and_evaluate(config, workdir) 51 | benchmark_time = time.time() - start_time 52 | 53 | self.report_wall_time(benchmark_time) 54 | self.report_extras({ 55 | 'description': 'ImageNet ResNet50 with fake data', 56 | 'model_name': 'resnet50', 57 | 'parameters': f'hp=true,bs={config.batch_size}', 58 | }) 59 | 60 | 61 | if __name__ == '__main__': 62 | absltest.main() 63 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Main file for running the ImageNet example. 16 | 17 | This file is intentionally kept short. The majority for logic is in libraries 18 | that can be easily tested and imported in Colab. 19 | """ 20 | import train 21 | import jax 22 | import tensorflow as tf 23 | 24 | from absl import app 25 | from absl import flags 26 | from absl import logging 27 | 28 | from ml_collections import config_flags 29 | 30 | FLAGS = flags.FLAGS 31 | 32 | flags.DEFINE_string('workdir', None, 'Directory to store model data.') 33 | config_flags.DEFINE_config_file( 34 | 'config', 35 | None, 36 | 'File path to the training hyperparameter configuration.', 37 | lock_config=True) 38 | 39 | 40 | def main(argv): 41 | if len(argv) > 1: 42 | raise app.UsageError('Too many command-line arguments.') 43 | 44 | # exit() 45 | # Hide any GPUs from TensorFlow. Otherwise TF might reserve memory and make 46 | # it unavailable to JAX. 47 | tf.config.experimental.set_visible_devices([], 'GPU') 48 | 49 | logging.info('JAX process: %d / %d', jax.process_index(), jax.process_count()) 50 | logging.info('JAX local devices: %r', jax.local_devices()) 51 | 52 | # Add a note so that we can tell which task is which JAX host. 53 | # (Depending on the platform task 0 is not guaranteed to be host 0) 54 | """platform.work_unit().set_task_status(f'process_index: {jax.process_index()}', 55 | f'process_count: {jax.process_count()}') 56 | platform.work_unit().create_artifact(platform.ArtifactType.DIRECTORY, 57 | FLAGS.workdir, 'workdir')""" 58 | 59 | train.train_and_evaluate(FLAGS.config, FLAGS.workdir) 60 | 61 | 62 | if __name__ == '__main__': 63 | flags.mark_flags_as_required(['config', 'workdir']) 64 | 65 | app.run(main) 66 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/train_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for flax.examples.imagenet.train.""" 16 | import tempfile 17 | 18 | from absl.testing import absltest 19 | 20 | import jax 21 | import jax.numpy as jnp 22 | from jax import random 23 | 24 | import tensorflow as tf 25 | 26 | # Local imports. 27 | import experiments.continual_learning.agents.models as models 28 | import train 29 | from configs import default as default_lib 30 | 31 | jax.config.update('jax_disable_most_optimizations', True) 32 | 33 | 34 | class TrainTest(absltest.TestCase): 35 | 36 | def setUp(self): 37 | super().setUp() 38 | # Make sure tf does not allocate gpu memory. 39 | tf.config.experimental.set_visible_devices([], 'GPU') 40 | 41 | def test_create_model(self): 42 | """Tests creating model.""" 43 | num_classes = 10 44 | model = train.create_model(model_cls=models._ResNet1, num_classes=num_classes, 45 | half_precision=False) # pylint: disable=protected-access 46 | params, batch_stats = train.initialized(random.PRNGKey(0), 32, model) 47 | variables = {'params': params, 'batch_stats': batch_stats} 48 | x = random.normal(random.PRNGKey(1), (8, 32, 32, 3)) 49 | y = model.apply(variables, x, train=False) 50 | self.assertEqual(y.shape, (8, num_classes)) 51 | 52 | def test_train_and_evaluate(self): 53 | """Tests training and evaluation loop using mocked data.""" 54 | # Create a temporary directory where tensorboard metrics are written. 55 | workdir = tempfile.mkdtemp() 56 | 57 | # Define training configuration 58 | config = default_lib.get_config() 59 | config.model = '_ResNet1' 60 | config.batch_size = 8 61 | config.num_epochs = 1 62 | config.num_train_steps = 1 63 | config.steps_per_eval = 1 64 | train.train_and_evaluate(workdir=workdir, config=config) 65 | 66 | def test_log_likelihood(self): 67 | n, num_classes = 10, 10 68 | logits = jax.nn.log_softmax(jnp.ones((n, num_classes))) 69 | labels = jnp.arange(num_classes) 70 | loss = train.log_likelihood(logits, labels, num_classes) 71 | self.assertLess(loss, 0) 72 | self.assertEqual(loss, n * logits[0][0]) 73 | 74 | 75 | if __name__ == '__main__': 76 | absltest.main() 77 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/models_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for flax.examples.imagenet.models.""" 16 | 17 | from absl.testing import absltest 18 | 19 | import jax 20 | from jax import numpy as jnp 21 | 22 | import experiments.continual_learning.agents.models as models 23 | 24 | jax.config.update('jax_disable_most_optimizations', True) 25 | 26 | 27 | class ResNetV1Test(absltest.TestCase): 28 | """Test cases for ResNet v1 model definition.""" 29 | 30 | def test_resnet_v1_model(self): 31 | """Tests ResNet V1 model definition and output (variables).""" 32 | rng = jax.random.PRNGKey(0) 33 | model_def = models.ResNet50(num_classes=10, dtype=jnp.float32) 34 | variables = model_def.init( 35 | rng, jnp.ones((8, 224, 224, 3), jnp.float32)) 36 | 37 | self.assertLen(variables, 2) 38 | # Resnet50 model will create parameters for the following layers: 39 | # conv + batch_norm = 2 40 | # BottleneckResNetBlock in stages: [3, 4, 6, 3] = 16 41 | # Followed by a Dense layer = 1 42 | self.assertLen(variables['params'], 19) 43 | 44 | def test_lenet5_model(self): 45 | """Tests LeNeT5 model definition and output (variables).""" 46 | rng = jax.random.PRNGKey(0) 47 | num_classes = 10 48 | model_def = models.LeNet(num_classes=num_classes, dtype=jnp.float32) 49 | variables = model_def.init( 50 | rng, jnp.ones((8, 32, 32, 3), jnp.float32)) 51 | 52 | self.assertLen(variables, 1) 53 | # LeNet5 model will create parameters for the following layers: 54 | # 2 Conv + 3 Dense = 2 55 | self.assertLen(variables['params'], 5) 56 | # The output of the last layer of LeNet5 will be equal to the number 57 | # of classes. In this case, it is 10 58 | self.assertLen(variables['params']['Dense_2']['bias'], num_classes) 59 | self.assertEqual(variables['params']['Dense_2']['kernel'].shape[-1], num_classes) 60 | 61 | def test_mlp_model(self): 62 | """Tests LeNeT5 model definition and output (variables).""" 63 | rng = jax.random.PRNGKey(0) 64 | num_classes = 10 65 | layer_dims = [256, 256] 66 | model_def = models.MLP(layer_dims=layer_dims, num_classes=num_classes, dtype=jnp.float32) 67 | variables = model_def.init( 68 | rng, jnp.ones((8, 32, 32, 3), jnp.float32)) 69 | 70 | self.assertLen(variables, 1) 71 | # MLP model will create parameters for the following layers: 72 | # 2 Dense Layer + 1 Output Layer = 3 73 | self.assertLen(variables['params'], len(layer_dims) + 1) 74 | 75 | layer_dims += [num_classes] 76 | for layer_idx, layer_dim in enumerate(layer_dims): 77 | self.assertLen(variables['params'][f'Dense_{layer_idx}']['bias'], layer_dim) 78 | self.assertEqual(variables['params'][f'Dense_{layer_idx}']['kernel'].shape[-1], layer_dim) 79 | 80 | 81 | if __name__ == '__main__': 82 | absltest.main() 83 | -------------------------------------------------------------------------------- /shift_happens/gendist/gendist/processing.py: -------------------------------------------------------------------------------- 1 | """ 2 | This library contains functions to process image data used by GenDist 3 | """ 4 | import jax 5 | import numpy as np 6 | import jax.numpy as jnp 7 | from multiprocessing import Pool 8 | from augly import image 9 | 10 | # DataAugmentationFactory 11 | class Factory: 12 | """ 13 | This is a base library to process / transform the elements of a numpy 14 | array according to a given function. To be used with gendist.TrainingConfig 15 | """ 16 | def __init__(self, processor): 17 | self.processor = processor 18 | 19 | def __call__(self, img, configs, n_processes=90): 20 | return self.process_multiple_multiprocessing(img, configs, n_processes) 21 | 22 | def process_single(self, X, *args, **kwargs): 23 | """ 24 | Process a single element. 25 | 26 | Paramters 27 | --------- 28 | X: np.array 29 | A single numpy array 30 | kwargs: dict/params 31 | Processor's configuration parameters 32 | """ 33 | return self.processor(X, *args, **kwargs) 34 | 35 | def process_multiple(self, X_batch, configurations): 36 | """ 37 | Process all elements of a numpy array according to a list 38 | of configurations. 39 | Each image is processed according to a configuration. 40 | """ 41 | X_out = [] 42 | n_elements = len(X_batch) 43 | 44 | for X, configuration in zip(X_batch, configurations): 45 | X_processed = self.process_single(X, **configuration) 46 | X_out.append(X_processed) 47 | 48 | X_out = np.stack(X_out, axis=0) 49 | return X_out 50 | 51 | def process_multiple_multiprocessing(self, X_dataset, configurations, n_processes): 52 | """ 53 | Process elements in a numpy array in parallel. 54 | 55 | Parameters 56 | ---------- 57 | X_dataset: array(N, ...) 58 | N elements of arbitrary shape 59 | configurations: list 60 | List of configurations to apply to each element. Each 61 | element is a dict to pass to the processor. 62 | n_processes: int 63 | Number of cores to use 64 | """ 65 | num_elements = len(X_dataset) 66 | if type(configurations) == dict: 67 | configurations = [configurations] * num_elements 68 | 69 | dataset_proc = np.array_split(X_dataset, n_processes) 70 | config_split = np.array_split(configurations, n_processes) 71 | elements = zip(dataset_proc, config_split) 72 | 73 | with Pool(processes=n_processes) as pool: 74 | dataset_proc = pool.starmap(self.process_multiple, elements) 75 | dataset_proc = np.concatenate(dataset_proc, axis=0) 76 | pool.join() 77 | 78 | return dataset_proc.reshape(num_elements, -1) 79 | 80 | 81 | def flat_and_concat_params(params_hist): 82 | """ 83 | Flat and concat a list of parameters trained using 84 | a Flax model 85 | 86 | Parameters 87 | ---------- 88 | params_hist: list of flax FrozenDicts 89 | List of flax FrozenDicts containing trained model 90 | weights. 91 | 92 | Returns 93 | ------- 94 | jnp.array: flattened and concatenated weights 95 | function: function to unflatten (reconstruct) weights 96 | """ 97 | _, recontruct_pytree_fn = jax.flatten_util.ravel_pytree(params_hist[0]) 98 | flat_params = [jax.flatten_util.ravel_pytree(params)[0] for params in params_hist] 99 | flat_params = jnp.r_[flat_params] 100 | return flat_params, recontruct_pytree_fn -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/agents/resnet.py: -------------------------------------------------------------------------------- 1 | """ 2 | Flax implementation of ResNet V1, LeNet5 and MLP. 3 | https://github.com/google/flax/blob/main/examples/imagenet/models.py 4 | """ 5 | 6 | import jax.numpy as jnp 7 | 8 | from functools import partial 9 | from typing import Any, Callable, Sequence, Tuple 10 | from flax import linen as nn 11 | 12 | ModuleDef = Any 13 | 14 | 15 | class ResNetBlock(nn.Module): 16 | """ResNet block.""" 17 | filters: int 18 | conv: ModuleDef 19 | norm: ModuleDef 20 | act: Callable 21 | strides: Tuple[int, int] = (1, 1) 22 | 23 | @nn.compact 24 | def __call__(self, x, ): 25 | residual = x 26 | y = self.conv(self.filters, (3, 3), self.strides)(x) 27 | y = self.norm()(y) 28 | y = self.act(y) 29 | y = self.conv(self.filters, (3, 3))(y) 30 | y = self.norm(scale_init=nn.initializers.zeros)(y) 31 | 32 | if residual.shape != y.shape: 33 | residual = self.conv(self.filters, (1, 1), 34 | self.strides, name='conv_proj')(residual) 35 | residual = self.norm(name='norm_proj')(residual) 36 | 37 | return self.act(residual + y) 38 | 39 | 40 | class BottleneckResNetBlock(nn.Module): 41 | """Bottleneck ResNet block.""" 42 | filters: int 43 | conv: ModuleDef 44 | norm: ModuleDef 45 | act: Callable 46 | strides: Tuple[int, int] = (1, 1) 47 | 48 | @nn.compact 49 | def __call__(self, x): 50 | residual = x 51 | y = self.conv(self.filters, (1, 1))(x) 52 | y = self.norm()(y) 53 | y = self.act(y) 54 | y = self.conv(self.filters, (3, 3), self.strides)(y) 55 | y = self.norm()(y) 56 | y = self.act(y) 57 | y = self.conv(self.filters * 4, (1, 1))(y) 58 | y = self.norm(scale_init=nn.initializers.zeros)(y) 59 | 60 | if residual.shape != y.shape: 61 | residual = self.conv(self.filters * 4, (1, 1), 62 | self.strides, name='conv_proj')(residual) 63 | residual = self.norm(name='norm_proj')(residual) 64 | 65 | return self.act(residual + y) 66 | 67 | 68 | class ResNet(nn.Module): 69 | """ResNetV1.""" 70 | stage_sizes: Sequence[int] 71 | block_cls: ModuleDef 72 | num_classes: int 73 | num_filters: int = 64 74 | dtype: Any = jnp.float32 75 | act: Callable = nn.relu 76 | 77 | @nn.compact 78 | def __call__(self, x, train: bool = True): 79 | conv = partial(nn.Conv, use_bias=False, dtype=self.dtype) 80 | norm = partial(nn.BatchNorm, 81 | use_running_average=not train, 82 | momentum=0.9, 83 | epsilon=1e-5, 84 | dtype=self.dtype) 85 | 86 | x = conv(self.num_filters, (7, 7), (2, 2), 87 | padding=[(3, 3), (3, 3)], 88 | name='conv_init')(x) 89 | x = norm(name='bn_init')(x) 90 | x = nn.relu(x) 91 | x = nn.max_pool(x, (3, 3), strides=(2, 2), padding='SAME') 92 | for i, block_size in enumerate(self.stage_sizes): 93 | for j in range(block_size): 94 | strides = (2, 2) if i > 0 and j == 0 else (1, 1) 95 | x = self.block_cls(self.num_filters * 2 ** i, 96 | strides=strides, 97 | conv=conv, 98 | norm=norm, 99 | act=self.act)(x) 100 | x = jnp.mean(x, axis=(1, 2)) 101 | x = nn.Dense(self.num_classes, dtype=self.dtype)(x) 102 | x = jnp.asarray(x, self.dtype) 103 | return x 104 | -------------------------------------------------------------------------------- /shift_happens/gendist/experiments/mnist_zero_shot_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import gendist 4 | import torchvision 5 | import numpy as np 6 | import pandas as pd 7 | import matplotlib.pyplot as plt 8 | from datetime import datetime 9 | from tqdm import tqdm 10 | from loguru import logger 11 | from augly import image 12 | from jax.flatten_util import ravel_pytree 13 | 14 | 15 | def processor(X, angle): 16 | X_shift = image.aug_np_wrapper(X, image.rotate, degrees=angle) 17 | size_im = X_shift.shape[0] 18 | size_pad = (28 - size_im) // 2 19 | size_pad_mod = (28 - size_im) % 2 20 | X_shift = np.pad(X_shift, (size_pad, size_pad + size_pad_mod)) 21 | 22 | return X_shift 23 | 24 | 25 | def predict_shifted_dataset(ix_seed, X_batch, processor, config, wmodel, wparams, dmodel, proj, fn_reconstruct): 26 | """ 27 | Parameters 28 | ---------- 29 | ix_seed: array 30 | X_batch: array 31 | ... 32 | wmodel: model for the latent space 33 | wparams: trained weights for the latent space 34 | dmodel: model for the observed space 35 | dparams: trained model for the observed weights 36 | """ 37 | x_seed = X_batch[ix] 38 | x_shift = processor.process_single(x_seed, **config).ravel() 39 | predicted_weights = wmodel.apply(wparams, x_shift) 40 | predicted_weights = proj.inverse_transform(predicted_weights) 41 | predicted_weights = fn_reconstruct(predicted_weights) 42 | 43 | X_batch_shift = processor(X_batch, config) 44 | y_batch_hat = dmodel.apply(predicted_weights, X_batch_shift) 45 | 46 | return y_batch_hat 47 | 48 | 49 | path_experiment = "./outputs/2203221129/" 50 | path_data_model = os.path.join(path_experiment, "output", "data-model-result.pkl") 51 | path_meta_model = os.path.join(path_experiment, "output", "meta-model.pkl") 52 | path_results = os.path.join(path_experiment, "output", "accuracy.pkl") 53 | 54 | 55 | with open(path_data_model, "rb") as f: 56 | data_model_results = pickle.load(f) 57 | 58 | with open(path_meta_model, "rb") as f: 59 | meta_model_results = pickle.load(f) 60 | 61 | now_str = datetime.now().strftime("%Y%m%d%H%M") 62 | file_log = f"trench_test_{now_str}.log" 63 | path_logger = os.path.join(path_experiment, "logs", file_log) 64 | logger.remove() 65 | logger.add(path_logger, rotation="5mb") 66 | 67 | mnist_test = torchvision.datasets.MNIST(root=".", train=False, download=True) 68 | X_test = np.array(mnist_test.data) / 255 69 | y_test = np.array(mnist_test.targets) 70 | 71 | proc_class = gendist.processing.Factory(processor) 72 | pca = meta_model_results["projection_model"] 73 | 74 | meta_model = gendist.models.MLPWeightsV1(pca.n_components) 75 | data_model = gendist.models.MLPDataV1(10) 76 | 77 | _, fn_reconstruct_params = ravel_pytree(data_model_results["params"][0]) 78 | 79 | accuracy_configs_learned = [] 80 | ixs = np.arange(5) 81 | 82 | for config in tqdm(data_model_results["configs"]): 83 | acc_dict = {} 84 | for ix in ixs: 85 | y_test_hat = predict_shifted_dataset(ix, X_test, proc_class, config, 86 | meta_model, meta_model_results["params"], 87 | data_model, pca, fn_reconstruct_params) 88 | y_test_hat = y_test_hat.argmax(axis=1) 89 | accuracy_learned = (y_test_hat == y_test).mean().item() 90 | acc_dict[ix] = accuracy_learned 91 | 92 | accuracy_configs_learned.append(acc_dict) 93 | 94 | angle = config["angle"] 95 | logger_row = "|".join([format(v, "0.2%") for v in acc_dict.values()]) 96 | logger_row = f"{angle=:0.4f} | " + logger_row 97 | 98 | logger.info(logger_row) 99 | 100 | pd.DataFrame(acc_dict).to_pickle(path_results) 101 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/configs/tpu_dynamic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Copyright 2021 The Flax Authors. 16 | # 17 | # Licensed under the Apache License, Version 2.0 (the "License"); 18 | # you may not use this file except in compliance with the License. 19 | # You may obtain a copy of the License at 20 | # 21 | # http://www.apache.org/licenses/LICENSE-2.0 22 | # 23 | # Unless required by applicable law or agreed to in writing, software 24 | # distributed under the License is distributed on an "AS IS" BASIS, 25 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 26 | # See the License for the specific language governing permissions and 27 | # limitations under the License. 28 | 29 | """Hyperparameter configuration to run the example on TPUs.""" 30 | 31 | import ml_collections 32 | 33 | 34 | def get_config(model='ResNet50', 35 | dataset='imagenet2012:5.*.*', 36 | optimizer='Adam', 37 | optimizer_params=None, 38 | 39 | num_devices=8, 40 | cache=True, 41 | half_precision=True, 42 | num_classes=1000, 43 | image_size=224, 44 | crop_padding=32): 45 | """Get the hyperparameter configuration to train on TPUs.""" 46 | config = ml_collections.ConfigDict() 47 | 48 | # As defined in the `models` module. 49 | config.model = model 50 | # `name` argument of tensorflow_datasets.builder() 51 | config.dataset = dataset 52 | 53 | # Consider setting the batch size to max(tpu_chips * 256, 8 * 1024) if you 54 | # train on a larger pod slice. 55 | config.num_devices = num_devices 56 | 57 | config.cache = cache 58 | config.half_precision = half_precision 59 | # list of optimizer configs dicts 60 | config.optimizers = optimizer 61 | config.optimizers_params = [get_optimizer_config(optimizer_params, opt_num_devices=num_devices)] 62 | config.num_classes = num_classes 63 | config.image_size = image_size 64 | config.crop_padding = crop_padding 65 | return config 66 | 67 | 68 | def get_optimizer_config(config=None, learning_rate=0.1, 69 | warmup_epochs=5, 70 | momentum=0.9, 71 | num_epochs=100, 72 | log_every_steps=100, 73 | num_train_steps=-1, 74 | steps_per_eval=-1, 75 | batch_size=-1, 76 | opt_num_devices=8): 77 | if config is None: 78 | config = ml_collections.ConfigDict() 79 | config.learning_rate = learning_rate 80 | config.warmup_epochs = warmup_epochs 81 | config.momentum = momentum 82 | config.opt_num_devices = opt_num_devices 83 | config.num_epochs = num_epochs 84 | config.log_every_steps = log_every_steps 85 | 86 | # If num_train_steps==-1 then the number of training steps is calculated from 87 | # num_epochs using the entire dataset. Similarly for steps_per_eval. 88 | config.num_train_steps = num_train_steps 89 | config.steps_per_eval = steps_per_eval 90 | 91 | if batch_size == -1: 92 | config.batch_size = max(config.opt_num_devices * 256, 8 * 1024) 93 | else: 94 | config.batch_size = batch_size 95 | 96 | return config 97 | elif config is not None and type(config) is dict: 98 | return ml_collections.ConfigDict(initial_dictionary=config) 99 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/environment.py: -------------------------------------------------------------------------------- 1 | import jax 2 | import jax.numpy as jnp 3 | from jax import vmap, tree_map 4 | from jax import random 5 | from imax import transforms 6 | 7 | import tensorflow_datasets as tfds 8 | import numpy as np 9 | 10 | from functools import partial 11 | from typing import Any 12 | 13 | Array = Any 14 | 15 | 16 | class Environment: 17 | def prepare_data(self, dataset_name: str): 18 | ds_builder = tfds.builder(dataset_name) 19 | ds_builder.download_and_prepare() 20 | ds_train = ds_builder.as_dataset(split="train") 21 | self.test_data = ds_builder.as_dataset(split="test").repeat().batch(self.batch_size).as_numpy_iterator() 22 | 23 | self.num_classes = ds_builder.info.features['label'].num_classes 24 | self.current_class = np.random.choice(np.arange(self.num_classes), size=(1,)) 25 | self.sets = [ds_train.filter(lambda x: x['label'] == i) for i in range(self.num_classes)] 26 | self.counts = [5923, 6742, 5958, 6131, 5842, 5421, 5918, 6265, 5851, 5949] 27 | # [st.reduce(np.int64(0), lambda x, _ : x + 1).numpy() for st in self.sets] 28 | 29 | def __init__(self, dataset_name='mnist', class_trans_mat=None, rot_mat=None, seed=0, batch_size=64): 30 | self.class_trans_mat = class_trans_mat 31 | self.rot = 0 32 | self.rot_mat = rot_mat 33 | self.seed = seed 34 | self.batch_size = batch_size 35 | self.key = random.PRNGKey(seed) 36 | self.prepare_data(dataset_name) 37 | 38 | def get_data(self): 39 | class_key, rot_key, self.key = random.split(self.key, 3) 40 | self.current_class = random.choice(class_key, jnp.arange(self.num_classes), 41 | p=self.class_trans_mat[self.current_class].squeeze()) 42 | batch = self.get_train_batch(self.current_class) 43 | self.rot = random.choice(rot_key, jnp.arange(len(self.rot_mat)), p=self.rot_mat[self.rot].squeeze()) 44 | batch['image'] = self.apply(batch['image'], self.rot) 45 | return {'image': batch['image'], 'label': batch['label']} 46 | 47 | def apply(self, x: Array, degrees: float): 48 | rad = jnp.radians(degrees) 49 | # Creates transformation matrix 50 | transform = transforms.rotate(rad=rad) 51 | apply_transform = partial(transforms.apply_transform, transform=transform, mask_value=jnp.array([0, 0, 0])) 52 | return vmap(apply_transform)(x) 53 | 54 | def get_test_data(self, device_count=1, nchannels=3): 55 | rotations = jnp.argwhere(self.rot_mat.sum(axis=0) > 0).flatten().tolist() 56 | 57 | batch = next(self.test_data) 58 | input, label = jnp.array(batch['image']), jnp.array(batch['label']) 59 | if input.shape[-1] != nchannels: 60 | input = jnp.repeat(input, axis=-1, repeats=nchannels) 61 | 62 | label = label[None, ...] 63 | for degrees in rotations: 64 | if degrees: 65 | input = jnp.vstack([input, self.apply(input, degrees)]) 66 | label = jnp.vstack([label, label]) 67 | label = label.squeeze() 68 | if device_count > 1: 69 | input = input.reshape((device_count, -1, *input.shape[-3:])) 70 | label = label.reshape((device_count, -1)) 71 | 72 | return {'image': input, 'label': label} 73 | 74 | def get_train_batch(self, c: int, seed: int = 0): 75 | dataset = self.sets[c] 76 | nexamples = self.counts[c] 77 | batch = tree_map(jnp.array, 78 | next(dataset.shuffle(nexamples, seed=seed).batch(self.batch_size).as_numpy_iterator())) 79 | images, labels = batch['image'], batch['label'] 80 | *_, nchannels = images.shape 81 | if nchannels == 1: 82 | images = jnp.repeat(images, axis=-1, repeats=3) 83 | return {'image': images, 'label': labels} 84 | 85 | def warmup(self, num_pulls: int): 86 | warmup_classes = jnp.arange(self.num_classes) 87 | warmup_classes = jnp.repeat(warmup_classes, num_pulls).reshape(self.num_classes, -1) 88 | classes = warmup_classes.reshape(-1, order="F").astype(jnp.int32) 89 | num_warmup_classes, *_ = classes.shape 90 | seeds = jnp.arange(len(classes)) 91 | inputs, labels = [], [] 92 | 93 | for c, seed in zip(classes, seeds): 94 | batch = self.get_train_batch(c, seed) 95 | inputs.append(batch['image']) 96 | labels.append(batch['label']) 97 | return jnp.vstack(inputs), jnp.concatenate(labels) 98 | -------------------------------------------------------------------------------- /shift_happens/gendist/notebooks/dojax.py: -------------------------------------------------------------------------------- 1 | # Library for domain shift in Jax 2 | import jax 3 | import numpy as np 4 | import jax.numpy as jnp 5 | from multiprocessing import Pool 6 | from augly import image 7 | 8 | 9 | def rotation_matrix(angle): 10 | """ 11 | Create a rotation matrix that rotates the 12 | space 'angle'-radians. 13 | """ 14 | R = np.array([ 15 | [np.cos(angle), -np.sin(angle)], 16 | [np.sin(angle), np.cos(angle)] 17 | ]) 18 | return R 19 | 20 | 21 | def flat_and_concat_params(params_hist): 22 | """ 23 | Flat and concat a list of parameters trained using 24 | a Flax model 25 | 26 | 27 | Parameters 28 | ---------- 29 | params_hist: list of flax FrozenDicts 30 | List of flax FrozenDicts containing trained model 31 | weights. 32 | 33 | Returns 34 | ------- 35 | jnp.array: flattened and concatenated weights 36 | """ 37 | _, recontruct_pytree_fn = jax.flatten_util.ravel_pytree(params_hist[0]) 38 | flat_params = [jax.flatten_util.ravel_pytree(params)[0] for params in params_hist] 39 | flat_params = jnp.r_[flat_params] 40 | return flat_params, recontruct_pytree_fn 41 | 42 | 43 | def make_mse_func(model, x_batched, y_batched): 44 | def mse(params): 45 | # Define the squared loss for a single pair (x,y) 46 | def squared_error(x, y): 47 | pred = model.apply(params, x) 48 | residual = pred - y 49 | return residual @ residual / 2.0 50 | # We vectorize the previous to compute the average of the loss on all samples. 51 | return jnp.mean(jax.vmap(squared_error)(x_batched, y_batched), axis=0) 52 | return jax.jit(mse) 53 | 54 | 55 | ### Elements for shifting-mnist experiments ### 56 | 57 | class BlurRad: 58 | def __init__(self, rad): 59 | self.rad = rad 60 | 61 | def __call__(self, img): 62 | return self.blur_multiple(img) 63 | 64 | def blur(self, X): 65 | """ 66 | Blur an image using the augly library 67 | 68 | Paramters 69 | --------- 70 | X: np.array 71 | A single NxM-dimensional array 72 | radius: float 73 | The amout of blurriness 74 | """ 75 | return image.aug_np_wrapper(X, image.blur, radius=self.rad) 76 | 77 | def blur_multiple(self, X_batch): 78 | images_out = [] 79 | for X in X_batch: 80 | img_blur = self.blur(X) 81 | images_out.append(img_blur) 82 | images_out = np.stack(images_out, axis=0) 83 | return images_out 84 | 85 | def blur_multiple(radii, img_dataset): 86 | """ 87 | Blur every element of `img_dataset` given an element 88 | of `radii`. 89 | """ 90 | imgs_out = [] 91 | for radius, img in zip(radii, img_dataset): 92 | img_proc = BlurRad(radius).blur(img) 93 | imgs_out.append(img_proc) 94 | imgs_out = np.stack(imgs_out, axis=0) 95 | 96 | return imgs_out 97 | 98 | 99 | # To-do: Modify proc_dataset and proc_dataset_multiple to use 100 | # a function that modifies the image. 101 | 102 | def proc_dataset(radius, img_dataset, n_processes=90): 103 | """ 104 | Blur all images of a dataset stored in a numpy array. 105 | 106 | Parameters 107 | ---------- 108 | radius: float 109 | Intensity of bluriness 110 | img_dataset: array(N, L, K) 111 | N images of size LxK 112 | n_processes: int 113 | Number of processes to blur over 114 | """ 115 | with Pool(processes=n_processes) as pool: 116 | dataset_proc = np.array_split(img_dataset, n_processes) 117 | dataset_proc = pool.map(BlurRad(radius), dataset_proc) 118 | dataset_proc = np.concatenate(dataset_proc, axis=0) 119 | pool.terminate() 120 | pool.join() 121 | 122 | return dataset_proc 123 | 124 | 125 | def proc_dataset_multiple(radii, img_dataset, n_processes=90): 126 | """ 127 | Blur all images of a dataset stored in a numpy array with variable 128 | radius. 129 | 130 | Parameters 131 | ---------- 132 | radius: array(N,) or float 133 | Intensity of bluriness. One per image. If 134 | float, the same value is used for all images. 135 | img_dataset: array(N, L, K) 136 | N images of size LxK 137 | n_processes: int 138 | Number of processes to blur over 139 | """ 140 | 141 | if type(radii) in [float, np.float_]: 142 | radii = radii * np.ones(len(img_dataset)) 143 | 144 | with Pool(processes=n_processes) as pool: 145 | dataset_proc = np.array_split(img_dataset, n_processes) 146 | radii_split = np.array_split(radii, n_processes) 147 | 148 | elements = zip(radii_split, dataset_proc) 149 | dataset_proc = pool.starmap(blur_multiple, elements) 150 | dataset_proc = np.concatenate(dataset_proc, axis=0) 151 | 152 | return dataset_proc 153 | -------------------------------------------------------------------------------- /shift_happens/gendist/experiments/mnist_rotation_meta.py: -------------------------------------------------------------------------------- 1 | import os 2 | import jax 3 | import optax 4 | import gendist 5 | import pickle 6 | import torchvision 7 | import numpy as np 8 | from augly import image 9 | from sklearn.decomposition import PCA 10 | 11 | 12 | def rotate(X, angle): 13 | X_shift = image.aug_np_wrapper(X, image.rotate, degrees=angle) 14 | size_im = X_shift.shape[0] 15 | size_pad = (28 - size_im) // 2 16 | size_pad_mod = (28 - size_im) % 2 17 | X_shift = np.pad(X_shift, (size_pad, size_pad + size_pad_mod)) 18 | 19 | return X_shift 20 | 21 | 22 | def load_train_combo(filename): 23 | """ 24 | Load the parameters and the configurations from a file 25 | of trained models. We return the flatten weights, 26 | the reconstruct function and the configurations. 27 | """ 28 | with open(filename, "rb") as f: 29 | params = pickle.load(f) 30 | list_params = params["params"] 31 | list_configs = params["configs"] 32 | 33 | target_params, fn_reconstruct = gendist.processing.flat_and_concat_params(list_params) 34 | 35 | output = { 36 | "params": target_params, 37 | "configs": list_configs, 38 | "fn_reconstruct": fn_reconstruct 39 | } 40 | 41 | return output 42 | 43 | 44 | def configure_covariates(key, processor, X, configs, n_subset): 45 | """ 46 | Given a dataset with shape (n_train, ...), a subset size n_subset, 47 | and a list of configurations to transform the dataset, we transform the 48 | dataset in an array of shape (n_subset, n_features, ...). 49 | """ 50 | n_configs = len(configs) 51 | n_train, *elem_dims = X.shape 52 | 53 | imap = np.ones((n_configs, 1, *elem_dims)) 54 | configs_transform = np.repeat(configs, n_subset) 55 | subset_ix = jax.random.choice(key, n_train, (n_subset,), replace=False).to_py() 56 | X = X[subset_ix, ...] * imap 57 | X = processor(X.reshape(-1, *elem_dims), configs_transform) 58 | X = X.reshape((n_subset, n_configs, -1), order="F") 59 | 60 | return X 61 | 62 | 63 | def predict_shifted_dataset(ix_seed, X_batch, processor, config, meta_model, 64 | meta_params, dmodel, proj, fn_reconstruct): 65 | """ 66 | Predict weights and estimate the values 67 | 68 | Parameters 69 | ---------- 70 | ix_seed: array 71 | X_batch: array 72 | ... 73 | meta_model: model for the latent space 74 | meta_params: trained weights for the latent space 75 | dmodel: model for the observed space 76 | dparams: trained model for the observed weights 77 | """ 78 | x_seed = X_batch[ix_seed] 79 | x_shift = processor.process_single(x_seed, **config).ravel() 80 | predicted_weights = meta_model.apply(meta_params, x_shift) 81 | predicted_weights = proj.inverse_transform(predicted_weights) 82 | predicted_weights = fn_reconstruct(predicted_weights) 83 | 84 | X_batch_shift = processor(X_batch, config) 85 | y_batch_hat = dmodel.apply(predicted_weights, X_batch_shift) 86 | 87 | return y_batch_hat 88 | 89 | 90 | processing_class = gendist.processing.Factory(rotate) 91 | 92 | 93 | if __name__ == "__main__": 94 | import sys 95 | 96 | _, filename_data_model = sys.argv 97 | experiment_path, _ = os.path.split(filename_data_model) 98 | 99 | output = load_train_combo(filename_data_model) 100 | target_params = output["params"] 101 | list_configs = output["configs"] 102 | fn_reconstruct_params = output["fn_reconstruct"] 103 | 104 | processing_class = gendist.processing.Factory(rotate) 105 | key = jax.random.PRNGKey(314) 106 | key, key_subset = jax.random.split(key) 107 | 108 | mnist_train = torchvision.datasets.MNIST(root=".", train=True, download=True) 109 | X_train = np.array(mnist_train.data) / 255 110 | X_train = configure_covariates(key_subset, processing_class, X_train, list_configs, n_train_subset) 111 | 112 | n_components = 60 113 | n_classes = 10 114 | n_train_subset = 6_000 115 | n_train, *elem_dims = X_train.shape 116 | n_configs = len(list_params) 117 | 118 | pca = PCA(n_components=n_components) 119 | projected_params = pca.fit_transform(target_params)[None, ...] 120 | 121 | alpha = 0.01 122 | n_epochs = 150 123 | batch_size = 2000 124 | tx = optax.adam(learning_rate=alpha) 125 | lossfn = gendist.training.make_multi_output_loss_func 126 | weights_model = gendist.models.MLPWeightsV1(n_components) 127 | trainer = gendist.training.TrainingMeta(weights_model, lossfn, tx) 128 | 129 | meta_output = trainer.fit(key, X_train, projected_params, n_epochs, batch_size) 130 | meta_output["projection_model"] = pca 131 | 132 | filename_meta_model = "meta-model.pkl" 133 | filename_meta_model = os.path.join(experiment_path, filename_meta_model) 134 | with open(filename_meta_model, "wb") as f: 135 | pickle.dump(meta_output, f) 136 | -------------------------------------------------------------------------------- /shift_happens/gendist/experiments/mnist_rotation_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import jax 3 | import optax 4 | import pickle 5 | import gendist 6 | import torchvision 7 | import numpy as np 8 | from augly import image 9 | from datetime import datetime 10 | from tqdm import tqdm 11 | from loguru import logger 12 | 13 | def eval_acc(y, yhat): 14 | return (y.argmax(axis=1) == yhat.argmax(axis=1)).mean().item() 15 | 16 | 17 | def processor(X, angle): 18 | X_shift = image.aug_np_wrapper(X, image.rotate, degrees=angle) 19 | size_im = X_shift.shape[0] 20 | size_pad = (28 - size_im) // 2 21 | size_pad_mod = (28 - size_im) % 2 22 | X_shift = np.pad(X_shift, (size_pad, size_pad + size_pad_mod)) 23 | 24 | return X_shift 25 | 26 | 27 | def create_experiment_path(base_path, experiment_name): 28 | base_path = os.path.join(base_path, experiment_name) 29 | path_output = os.path.join(base_path, "output") 30 | path_logs = os.path.join(base_path, "logs") 31 | 32 | if not os.path.exists(path_output): 33 | os.makedirs(path_output) 34 | if not os.path.exists(path_logs): 35 | os.makedirs(path_logs) 36 | 37 | return base_path 38 | 39 | 40 | def training_loop(key, X, y, configs, trainer, n_epochs, batch_size, evalfn, logger, leave=True): 41 | """ 42 | Train a collection of models with different configurations 43 | 44 | Parameters 45 | ---------- 46 | X : ndarray 47 | Input data 48 | y : ndarray 49 | Target data 50 | configs : list 51 | List of configurations to train. 52 | Each configuration is a dictionary 53 | trainer: gendist.training 54 | Trainer object 55 | n_epochs: int 56 | Number of epochs to train 57 | batch_size: int 58 | Batch size 59 | evalfn : function 60 | Function to evaluate the model 61 | 62 | Returns 63 | ------- 64 | results : dict 65 | Dictionary with the results of the experiments 66 | """ 67 | configs_params = [] 68 | configs_losses = [] 69 | configs_metric = [] 70 | 71 | keys = jax.random.split(key, len(configs)) 72 | for key, config in tqdm(zip(keys, configs), leave=leave): # Remove zip for key 73 | train_output = trainer.fit(key, X, y, config, n_epochs, batch_size, evalfn) 74 | configs_params.append(train_output["params"]) 75 | configs_losses.append(train_output["losses"]) 76 | configs_metric.append(train_output["metric"]) 77 | 78 | name, value = config.copy().popitem() 79 | logger.info(f"{name}={value:0.3f} | {train_output['metric']:.4f}") 80 | 81 | output = { 82 | "params": configs_params, 83 | "losses": configs_losses, 84 | "metric": configs_metric 85 | } 86 | 87 | return output 88 | 89 | 90 | def main(key, base_path, trainer, X, y, configs, n_epochs, batch_size, evalfn, 91 | experiment_path=None, logname=None, filename=None, leave=True): 92 | filename = "data-model-result.pkl" if filename is None else filename 93 | logname = "log-data.log" if logname is None else logname 94 | 95 | if experiment_path is None: 96 | experiment_path = datetime.now().strftime("%y%m%d%H%M") 97 | experiment_path = create_experiment_path(base_path, experiment_path) 98 | 99 | logs_path = os.path.join(experiment_path, "logs") 100 | logs_path = os.path.join(logs_path, logname) 101 | logger.add(logs_path, rotation="5mb") 102 | 103 | experiment_results = training_loop(key, X, y, configs, trainer, n_epochs, batch_size, evalfn, logger, leave=leave) 104 | experiment_results["configs"] = configs 105 | 106 | filename = os.path.join(experiment_path, "output", filename) 107 | with open(filename, "wb") as f: 108 | pickle.dump(experiment_results, f) 109 | 110 | 111 | if __name__ == "__main__": 112 | n_configs, n_classes = 150, 10 113 | batch_size = 2000 114 | n_epochs = 50 115 | alpha = 0.001 116 | tx = optax.adam(learning_rate=alpha) 117 | # model = gendist.models.MLPDataV1(num_outputs=10) 118 | model = gendist.models.LeNet5(n_classes) 119 | processing_class = gendist.processing.Factory(processor) 120 | loss = gendist.training.make_cross_entropy_loss_func 121 | trainer = gendist.training.TrainingBase(model, processing_class, loss, tx) 122 | 123 | mnist_train = torchvision.datasets.MNIST(root="./data", train=True, download=True) 124 | X_train = np.array(mnist_train.data) / 255.0 125 | y_train = np.array(mnist_train.targets) 126 | y_train_ohe = jax.nn.one_hot(y_train, n_classes) 127 | 128 | degrees = np.linspace(0, 360, n_configs) 129 | configs = [{"angle": float(angle)} for angle in degrees] 130 | 131 | n_tests = 10 132 | base_path = "./outputs" 133 | experiment_path = "cnn-rotation-v2" 134 | key = jax.random.PRNGKey(314) 135 | keys = jax.random.split(key, n_tests) 136 | for it, key in tqdm(list(enumerate(keys))): 137 | logger.remove() # avoid output to terminal 138 | main(key, base_path, trainer, X_train, y_train_ohe, configs, n_epochs, batch_size, eval_acc, 139 | experiment_path=experiment_path, logname=f"log-cnn-{it:02}.log", filename=f"cnn-{it:02}.pkl", 140 | leave=False) 141 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/gdumb_old.py: -------------------------------------------------------------------------------- 1 | from jax._src.random import split 2 | import jax.numpy as jnp 3 | from jax import lax, ops, random 4 | 5 | from collections import defaultdict 6 | 7 | 8 | def init_sampler(x , y, num_classes): 9 | dataset = x, y 10 | counts = jnp.bincount(y, minlength=num_classes) 11 | return counts, dataset 12 | 13 | 14 | def sample(key, X, Y, counts, datasets, memory_size=200): 15 | inputs, labels = datasets 16 | n = jnp.sum(jnp.where(counts>0, 1, 0)) 17 | nperclass = memory_size // n 18 | 19 | def true_fun(key, inputs, labels, counts, x, y): 20 | c_max = jnp.argwhere(counts == jnp.max(counts)) 21 | sample_key, key = random.split(key) 22 | indices = jnp.argwhere(labels == c_max) 23 | row = random.choice(sample_key, indices, shape=(1, )) 24 | inputs = ops.index_update(inputs, jnp.index_exp[row], x) 25 | labels = ops.index_update(labels, jnp.index_exp[row], y) 26 | counts = jnp.where(jnp.arange(len(counts)) == c_max, counts-1, counts) 27 | counts = jnp.where(jnp.arange(len(counts)) == y, counts+1, counts) 28 | return inputs, labels, counts 29 | 30 | def false_fun(key, inputs, labels, counts, x, y): 31 | return inputs, labels, counts 32 | 33 | 34 | def scan_fun(state, carry): 35 | inputs, labels, counts = state 36 | key, x, y = carry 37 | 38 | inputs, labels, counts = lax.cond(counts[y] == 0 or counts[y] < nperclass, true_fun, false_fun, 39 | operands=(key, inputs, labels, x, y)) 40 | return (inputs, labels, counts), None 41 | 42 | if len(datasets) < memory_size: 43 | for x, y in zip(X, Y): 44 | if counts[y] == 0 or counts[y] < nperclass: 45 | inputs = jnp.vstack([inputs, x[None, ...]]) 46 | labels = jnp.append(labels, y) 47 | counts = jnp.where(jnp.arange(len(counts)) ==y, counts+1, counts) 48 | 49 | else: 50 | scan_key, key = split(key) 51 | keys = split(scan_key, len(X)) 52 | (inputs, labels, counts), _ = lax.scan(scan_fun, (inputs, labels, counts), (keys, X, Y)) 53 | 54 | return counts, (inputs, labels) 55 | 56 | 57 | 58 | 59 | import jax.numpy as jnp 60 | from jax import vmap 61 | from jax import random 62 | import jax.dlpack 63 | 64 | from imax import transforms 65 | 66 | import tensorflow as tf 67 | import tensorflow_datasets as tfds 68 | 69 | from functools import partial 70 | from typing import Any 71 | 72 | Array = Any 73 | 74 | def tf_to_jax(arr): 75 | return jax.dlpack.from_dlpack(tf.experimental.dlpack.to_dlpack(arr)) 76 | 77 | class Environment: 78 | def prepare_data(self, key : Any, dataset_name : str): 79 | 80 | ds_builder = tfds.builder(dataset_name) 81 | ds_builder.download_and_prepare() 82 | ds_train = ds_builder.as_dataset(split="train", as_supervised=True) 83 | self.test_data = ds_builder.as_dataset(split="test", as_supervised=True) 84 | 85 | self.num_classes = ds_builder.info.features['label'].num_classes 86 | self.current_class = random.choice(key, jnp.arange(self.num_classes), shape=(1,)) 87 | self.sets = [ds_train.filter(lambda x, y: y == i) for i in range(self.num_classes)] 88 | self.counts = [5923, 6742, 5958, 6131, 5842, 5421, 5918, 6265, 5851, 5949] 89 | # [st.reduce(np.int64(0), lambda x, _ : x + 1).numpy() for st in self.sets] 90 | 91 | 92 | def __init__(self, dataset_name='mnist', class_trans_mat=None, rot_mat=None, seed=0, batch_size=64): 93 | self.class_trans_mat = class_trans_mat 94 | self.rot = 0 95 | self.rot_mat = rot_mat 96 | self.seed= seed 97 | self.batch_size = batch_size 98 | 99 | tf.random.set_seed(seed) 100 | key = random.PRNGKey(seed) 101 | self.prepare_data(key, dataset_name) 102 | 103 | 104 | def get_data(self, key): 105 | class_key, rot_key, key = random.split(key, 3) 106 | self.current_class = random.choice(class_key, jnp.arange(self.num_classes), p=self.class_trans_mat[self.current_class].squeeze()) 107 | batch = self.get_batch(self.current_class) 108 | self.rot = random.choice(rot_key, jnp.arange(len(self.rot_mat)), p=self.rot_mat[self.rot].squeeze()) 109 | batch['image'] = self.apply(batch['image'], self.rot) 110 | return batch 111 | 112 | 113 | def apply(self, x : Array, degrees : float): 114 | rad = jnp.radians(degrees) 115 | # Creates transformation matrix 116 | transform = transforms.rotate(rad=rad) 117 | apply_transform = partial(transforms.apply_transform, transform=transform, mask_value=jnp.array([0, 0, 0])) 118 | return vmap(apply_transform)(x) 119 | 120 | def get_batch(self, c: int): 121 | dataset = self.sets[c] 122 | nexamples = self.counts[c] 123 | for images, labels in dataset.shuffle(nexamples).batch(self.batch_size): 124 | images = tf_to_jax(images) 125 | labels = tf_to_jax(labels) 126 | break 127 | 128 | *_, nchannels = images.shape 129 | if nchannels == 1: 130 | images = jnp.repeat(images, axis=-1, repeats=3) 131 | return {'image': images, 'label': labels} 132 | 133 | def warmup(self, num_pulls : int): 134 | warmup_classes = jnp.arange(self.num_classes) 135 | warmup_classes = jnp.repeat(warmup_classes, num_pulls).reshape(self.num_classes, -1) 136 | classes = warmup_classes.reshape(-1, order="F").astype(jnp.int32) 137 | num_warmup_classes, *_ = classes.shape 138 | inputs, labels = [], [] 139 | 140 | for c in classes: 141 | batch = self.get_batch(c) 142 | inputs.append(batch['image'][None, ...]) 143 | labels.append(batch['label']) 144 | 145 | images = jnp.vstack(inputs) 146 | return images.reshape((-1, *images.shape[-3:])), jnp.concatenate(labels) -------------------------------------------------------------------------------- /shift_happens/gendist/gendist/training.py: -------------------------------------------------------------------------------- 1 | import jax 2 | import optax 3 | import jax.numpy as jnp 4 | import numpy as np 5 | from functools import partial 6 | from tqdm.auto import tqdm 7 | 8 | 9 | def make_cross_entropy_loss_func(model, X, y): 10 | """ 11 | Make a loss function for a multi-output classifier, i.e., 12 | model: R^M -> (y1, y2, ..., yK) = y_hat; where y_hat is a 13 | probability distribution over K classes 14 | 15 | Make a loss function for a multi-output model, i.e., 16 | model: R^M -> R^K 17 | 18 | Parameters 19 | ---------- 20 | model: Flax model 21 | Flax model that takes X and returns y_hat 22 | X: array(N, ...) 23 | N samples of arbitrary shape 24 | y: array(N, K) 25 | N samples of K-dimensional outputs 26 | """ 27 | def loss_fn(params): 28 | y_hat = model.apply(params, X) 29 | loss = optax.softmax_cross_entropy(y_hat, y).mean() 30 | return loss 31 | return loss_fn 32 | 33 | 34 | def make_multi_output_loss_func(model, X, y): 35 | """ 36 | Make a loss function for a multi-output model, i.e., 37 | model: R^M -> R^K 38 | 39 | Parameters 40 | ---------- 41 | model: Flax model 42 | Flax model that takes X and returns y_hat 43 | X: array(N, ...) 44 | N samples of arbitrary shape 45 | y: array(N, K) 46 | N samples of K-dimensional outputs 47 | """ 48 | def loss_fn(params): 49 | y_hat = model.apply(params, X) 50 | loss = jnp.linalg.norm(y - y_hat, axis=-1) ** 2 51 | return loss.mean() 52 | return loss_fn 53 | 54 | class TrainingBase: 55 | """ 56 | Class to train a neural network model that transforms the input data 57 | given a processor function. 58 | """ 59 | def __init__(self, model, processor, loss_generator, tx): 60 | self.model = model 61 | self.processor = processor 62 | self.loss_generator = loss_generator 63 | self.tx = tx 64 | 65 | def fit(self, key, X_train, y_train, config, num_epochs, batch_size, evalfn=None): 66 | """ 67 | Train a flax.linen model by transforming the data according to 68 | process_config. 69 | 70 | Parameters 71 | ---------- 72 | key: jax.random.PRNGKey 73 | Random number generator key. 74 | model: flax.nn.Module 75 | Model to train. 76 | X_train: jnp.array(N, ...) 77 | Training data. 78 | y_train: jnp.array(N) 79 | Training target values 80 | config: dict 81 | Dictionary containing the training configuration to be passed to 82 | the processor. 83 | num_epochs: int 84 | Number of epochs to train the model. 85 | """ 86 | X_train_proc = self.processor(X_train, config) 87 | _, *input_shape = X_train_proc.shape 88 | 89 | batch = jnp.ones((batch_size, *input_shape)) 90 | params = self.model.init(key, batch) 91 | optimiser_state = self.tx.init(params) 92 | 93 | losses = [] 94 | for e in tqdm(range(num_epochs), leave=False): 95 | _, key = jax.random.split(key) 96 | params, optimiser_state, avg_loss = self.train_epoch(key, params, optimiser_state, 97 | X_train_proc, y_train, batch_size, e) 98 | losses.append(avg_loss) 99 | 100 | if evalfn is not None: 101 | yhat = self.model.apply(params, X_train_proc) 102 | metric = evalfn(y_train, yhat) 103 | else: 104 | metric = None 105 | 106 | training_output = { 107 | "losses": jnp.array(losses), 108 | "metric": metric, 109 | "params": params, 110 | } 111 | 112 | return training_output 113 | 114 | @partial(jax.jit, static_argnums=(0,)) 115 | def train_step(self, params, opt_state, X_batch, y_batch): 116 | loss_fn = self.loss_generator(self.model, X_batch, y_batch) 117 | loss_grad_fn = jax.value_and_grad(loss_fn) 118 | loss_val, grads = loss_grad_fn(params) 119 | updates, opt_state = self.tx.update(grads, opt_state) 120 | params = optax.apply_updates(params, updates) 121 | 122 | return loss_val, params, opt_state 123 | 124 | def get_batch_train_ixs(self, key, num_samples, batch_size): 125 | """ 126 | Obtain the training indices to be used in an epoch of 127 | mini-batch optimisation. 128 | """ 129 | steps_per_epoch = num_samples // batch_size 130 | batch_ixs = jax.random.permutation(key, num_samples) 131 | batch_ixs = batch_ixs[:steps_per_epoch * batch_size] 132 | batch_ixs = batch_ixs.reshape(steps_per_epoch, batch_size) 133 | 134 | return batch_ixs 135 | 136 | def train_epoch(self, key, params, opt_step, X, y, batch_size, epoch): 137 | num_samples, *_ = X.shape 138 | batch_ixs = self.get_batch_train_ixs(key, num_samples, batch_size) 139 | 140 | epoch_loss = 0.0 141 | for batch_ix in batch_ixs: 142 | X_batch = X[batch_ix, ...] 143 | y_batch = y[batch_ix, ...] 144 | loss, params, opt_step = self.train_step(params, opt_step, X_batch, y_batch) 145 | epoch_loss += loss 146 | 147 | epoch_loss = epoch_loss / len(batch_ixs) 148 | return params, opt_step, epoch_loss 149 | 150 | 151 | class TrainingSnapshot(TrainingBase): 152 | """ 153 | Extension of Training base class that saves the model parameters 154 | every snapshot_interval epochs. For this class, it is better to consider 155 | an optimiser that fluctuates the learning rate. 156 | """ 157 | def __init__(self, model, processor, loss_generator, tx, snapshot_interval): 158 | super().__init__(model, processor, loss_generator, tx) 159 | self.snapshot_interval = snapshot_interval 160 | 161 | def fit(self, key, X_train, y_train, config, num_epochs, batch_size, evalfn=None): 162 | """ 163 | Train a flax.linen model by transforming the data according to 164 | process_config. 165 | 166 | Parameters 167 | ---------- 168 | key: jax.random.PRNGKey 169 | Random number generator key. 170 | model: flax.nn.Module 171 | Model to train. 172 | X_train: jnp.array(N, ...) 173 | Training data. 174 | y_train: jnp.array(N) 175 | Training target values 176 | config: dict 177 | Dictionary containing the training configuration to be passed to 178 | the processor. 179 | num_epochs: int 180 | Number of epochs to train the model. 181 | """ 182 | X_train_proc = self.processor(X_train, config) 183 | _, *input_shape = X_train_proc.shape 184 | 185 | batch = jnp.ones((batch_size, *input_shape)) 186 | params = self.model.init(key, batch) 187 | optimiser_state = self.tx.init(params) 188 | 189 | losses = [] 190 | params_hist = [] 191 | metrics_hist = [] 192 | for e in tqdm(range(num_epochs), leave=False): 193 | _, key = jax.random.split(key) 194 | 195 | # Store the parameters and evaluate the model on 196 | # the train set 197 | if (e+1) % self.snapshot_interval == 0: 198 | params_hist.append(params) 199 | if evalfn is not None: 200 | yhat = self.model.apply(params, X_train_proc) 201 | metric = evalfn(y_train, yhat) 202 | metrics_hist.append(metric) 203 | 204 | 205 | params, optimiser_state, avg_loss = self.train_epoch(key, params, optimiser_state, 206 | X_train_proc, y_train, batch_size, e) 207 | losses.append(avg_loss) 208 | 209 | training_output = { 210 | "params": params_hist, 211 | "losses": jnp.array(losses), 212 | "metrics": metrics_hist, 213 | } 214 | 215 | return training_output 216 | 217 | 218 | class TrainingMeta(TrainingBase): 219 | """ 220 | Training class of model parameters. We consider an input of the form NxMx..., and a target 221 | variable of the form KxMxW, wher 222 | * N: number of observations 223 | * M: number of transformations per observation 224 | * ...: Dimension specification of a single instance 225 | * K: number of samples per configuration. 226 | * W: number of parameters per configuration 227 | """ 228 | def __init__(self, model, loss_generator, tx): 229 | super().__init__(model, lambda x, _: x, loss_generator, tx) 230 | 231 | def fit(self, key, X_train, y_train, num_epochs, batch_size, leave_pb=True): 232 | """ 233 | Train a flax.linen model by transforming the data according to 234 | process_config. 235 | 236 | Parameters 237 | ---------- 238 | key: jax.random.PRNGKey 239 | Random number generator key. 240 | model: flax.nn.Module 241 | Model to train. 242 | X_train: jnp.array(N, ...) 243 | Training data. 244 | y_train: jnp.array(N) 245 | Training target values 246 | num_epochs: int 247 | Number of epochs to train the model. 248 | batch_size: int 249 | Number of samples per batch. 250 | leave_pb: bool 251 | If True, the progress bar is left open. 252 | """ 253 | _, *input_shape = X_train.shape 254 | 255 | key, key_params = jax.random.split(key) 256 | batch = jnp.ones((batch_size, *input_shape)) 257 | params = self.model.init(key_params, batch) 258 | optimiser_state = self.tx.init(params) 259 | 260 | losses = [] 261 | for e in tqdm(range(num_epochs), leave=leave_pb): 262 | _, key = jax.random.split(key) 263 | params, optimiser_state, avg_loss = self.train_epoch(key, params, optimiser_state, 264 | X_train, y_train, batch_size, e) 265 | losses.append(avg_loss) 266 | 267 | training_output = { 268 | "params": params, 269 | "losses": jnp.array(losses), 270 | } 271 | 272 | return training_output 273 | 274 | 275 | def train_epoch(self, key, params, opt_step, X, y, batch_size, epoch): 276 | """ 277 | Train an model considering an input of the form NxMx..., and a target 278 | variable of the form KxMxW. 279 | """ 280 | num_samples, num_configs_X, *_ = X.shape 281 | num_cycles, num_configs_y, _ = y.shape 282 | if num_configs_X != num_configs_y: 283 | raise ValueError("The number of configurations in X and y must be the same.") 284 | num_configs = num_configs_X 285 | num_elements = num_samples * num_configs * num_cycles 286 | 287 | batch_ixs = self.get_batch_train_ixs(key, num_elements, batch_size) 288 | 289 | epoch_loss = 0.0 290 | for batch_ix in batch_ixs: 291 | X_batch = X[batch_ix % num_samples, batch_ix // (num_samples * num_cycles), ...] 292 | y_batch = y[(batch_ix // num_samples) % num_cycles, batch_ix // (num_samples * num_cycles), ...] 293 | loss, params, opt_step = self.train_step(params, opt_step, X_batch, y_batch) 294 | epoch_loss += loss 295 | 296 | epoch_loss = epoch_loss / len(batch_ixs) 297 | return params, opt_step, epoch_loss 298 | -------------------------------------------------------------------------------- /shift_happens/imagenet_flax/train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Flax Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | This script trains a ResNet-50 on the ImageNet dataset. 17 | The data is loaded using tensorflow_datasets. 18 | """ 19 | 20 | import functools 21 | import time 22 | from typing import Any 23 | 24 | from absl import logging 25 | from clu import metric_writers 26 | from clu import periodic_actions 27 | 28 | import jax 29 | import jax.numpy as jnp 30 | from jax import lax 31 | from jax import random 32 | 33 | import flax 34 | from flax import jax_utils 35 | from flax import optim 36 | from flax.training import checkpoints 37 | from flax.training import common_utils 38 | from flax.training import train_state 39 | 40 | import optax 41 | 42 | import tensorflow as tf 43 | import tensorflow_datasets as tfds 44 | 45 | import ml_collections 46 | 47 | import numpy as np 48 | 49 | # Local imports 50 | import gdumb 51 | from environment import Environment 52 | import agents.models as models 53 | 54 | 55 | def create_model(model_cls, num_classes, half_precision, **kwargs): 56 | platform = jax.local_devices()[0].platform 57 | if half_precision: 58 | if platform == 'tpu': 59 | model_dtype = jnp.bfloat16 60 | else: 61 | model_dtype = jnp.float16 62 | else: 63 | model_dtype = jnp.float32 64 | return model_cls(num_classes=num_classes, dtype=model_dtype, **kwargs) 65 | 66 | 67 | def init_trans_mat_and_rot_mat(key, num_classes: int, max_degree: int = 359, rotation_indices: list = [0, 180]): 68 | trans_mat = jax.nn.softmax(jax.random.normal(key, shape=(num_classes, num_classes))) 69 | rot_mat = np.zeros((max_degree + 1, max_degree + 1)) 70 | for row in rotation_indices: 71 | for col in rotation_indices: 72 | rot_mat[row, col] = 1 / len(rotation_indices) 73 | rot_mat = jnp.array(rot_mat) 74 | return trans_mat, rot_mat 75 | 76 | 77 | def initialized(key, image_size, model): 78 | input_shape = (1, image_size, image_size, 3) 79 | 80 | @jax.jit 81 | def init(*args): 82 | return model.init(*args) 83 | 84 | variables = init({'params': key}, jnp.ones(input_shape, model.dtype)) 85 | if "batch_stats" in variables: 86 | return variables['params'], variables['batch_stats'] 87 | return variables['params'], {"mean": jnp.array([])} 88 | 89 | 90 | def cross_entropy_loss(logits, labels, num_classes): 91 | one_hot_labels = common_utils.onehot(labels, num_classes=num_classes) 92 | xentropy = optax.softmax_cross_entropy(logits=logits, labels=one_hot_labels) 93 | return jnp.mean(xentropy) 94 | 95 | 96 | def compute_metrics(logits, labels, num_classes): 97 | print(logits.shape, labels.shape) 98 | loss = cross_entropy_loss(logits, labels, num_classes) 99 | accuracy = jnp.mean(jnp.argmax(logits, -1) == labels) 100 | metrics = { 101 | 'loss': loss, 102 | 'accuracy': accuracy, 103 | } 104 | metrics = lax.pmean(metrics, axis_name='batch') 105 | return metrics 106 | 107 | 108 | def create_learning_rate_fn( 109 | num_steps: int, 110 | init_learning_rate: float): 111 | """Create learning rate schedule.""" 112 | 113 | def schedule(step): 114 | t = step / num_steps 115 | return 0.5 * init_learning_rate * (1 + jnp.cos(t * jnp.pi)) 116 | 117 | return schedule 118 | 119 | 120 | def train_step(state, batch, learning_rate_fn, num_classes, weight_decay=0.0001): 121 | """Perform a single training step.""" 122 | 123 | def loss_fn(params): 124 | """loss function used for training.""" 125 | logits, new_model_state = state.apply_fn( 126 | {'params': params, 'batch_stats': state.batch_stats}, 127 | batch['image'], 128 | mutable=['batch_stats']) 129 | loss = cross_entropy_loss(logits, batch['label'], num_classes) 130 | weight_penalty_params = jax.tree_leaves(params) 131 | weight_l2 = sum([jnp.sum(x ** 2) 132 | for x in weight_penalty_params 133 | if x.ndim > 1]) 134 | weight_penalty = weight_decay * 0.5 * weight_l2 135 | loss = loss + weight_penalty 136 | return loss, (new_model_state, logits) 137 | 138 | step = state.step 139 | dynamic_scale = state.dynamic_scale 140 | lr = learning_rate_fn(step) 141 | 142 | if dynamic_scale: 143 | grad_fn = dynamic_scale.value_and_grad( 144 | loss_fn, has_aux=True, axis_name='batch') 145 | dynamic_scale, is_fin, aux, grads = grad_fn(state.params) 146 | # dynamic loss takes care of averaging gradients across replicas 147 | else: 148 | grad_fn = jax.value_and_grad(loss_fn, has_aux=True) 149 | aux, grads = grad_fn(state.params) 150 | # Re-use same axis_name as in the call to `pmap(...train_step...)` below. 151 | grads = lax.pmean(grads, axis_name='batch') 152 | 153 | new_model_state, logits = aux[1] 154 | metrics = compute_metrics(logits, batch['label'], num_classes) 155 | metrics['learning_rate'] = lr 156 | 157 | new_state = state.apply_gradients( 158 | grads=grads, batch_stats=new_model_state['batch_stats']) 159 | if dynamic_scale: 160 | # if is_fin == False the gradients contain Inf/NaNs and optimizer state and 161 | # params should be restored (= skip this step). 162 | new_state = new_state.replace( 163 | opt_state=jax.tree_multimap( 164 | functools.partial(jnp.where, is_fin), 165 | new_state.opt_state, 166 | state.opt_state), 167 | params=jax.tree_multimap( 168 | functools.partial(jnp.where, is_fin), 169 | new_state.params, 170 | state.params)) 171 | metrics['scale'] = dynamic_scale.scale 172 | 173 | return new_state, metrics 174 | 175 | 176 | def eval_step(state, batch, num_classes): 177 | params = state.params 178 | variables = {'params': params, 'batch_stats': state.batch_stats} 179 | 180 | logits = state.apply_fn(variables, batch['image'], train=False, mutable=False) 181 | return compute_metrics(logits, batch['label'], num_classes) 182 | 183 | 184 | class TrainState(train_state.TrainState): 185 | batch_stats: Any 186 | dynamic_scale: flax.optim.DynamicScale 187 | 188 | 189 | def restore_checkpoint(state, workdir): 190 | return checkpoints.restore_checkpoint(workdir, state) 191 | 192 | 193 | def save_checkpoint(state, workdir): 194 | if jax.process_index() == 0: 195 | # get train state from the first replica 196 | state = jax.device_get(jax.tree_map(lambda x: x[0], state)) 197 | step = int(state.step) 198 | checkpoints.save_checkpoint(workdir, state, step, keep=3) 199 | 200 | 201 | # pmean only works inside pmap because it needs an axis name. 202 | # This function will average the inputs across all devices. 203 | cross_replica_mean = jax.pmap(lambda x: lax.pmean(x, 'x'), 'x') 204 | 205 | 206 | def sync_batch_stats(state): 207 | """Sync the batch statistics across replicas.""" 208 | # Each device has its own version of the running average batch statistics and 209 | # we sync them before evaluation. 210 | return state.replace(batch_stats=cross_replica_mean(state.batch_stats)) 211 | 212 | 213 | def create_train_state(rng, config: ml_collections.ConfigDict, 214 | model, image_size, learning_rate_fn): 215 | """Create initial training state.""" 216 | dynamic_scale = None 217 | platform = jax.local_devices()[0].platform 218 | 219 | if config.half_precision and platform == 'gpu': 220 | dynamic_scale = optim.DynamicScale() 221 | else: 222 | dynamic_scale = None 223 | 224 | params, batch_stats = initialized(rng, image_size, model) 225 | 226 | tx = optax.sgd( 227 | learning_rate=learning_rate_fn, 228 | momentum=config.momentum_decay, 229 | nesterov=True, 230 | ) 231 | 232 | state = TrainState.create( 233 | apply_fn=model.apply, 234 | params=params, 235 | tx=tx, 236 | batch_stats=batch_stats, 237 | dynamic_scale=dynamic_scale) 238 | return state 239 | 240 | 241 | def get_input_dtype(half_precision, platform): 242 | if half_precision: 243 | if platform == 'tpu': 244 | input_dtype = tf.bfloat16 245 | else: 246 | input_dtype = tf.float16 247 | else: 248 | input_dtype = tf.float32 249 | return input_dtype 250 | 251 | 252 | def train_and_evaluate(config: ml_collections.ConfigDict, 253 | workdir: str) -> TrainState: 254 | """Execute model training and evaluation loop. 255 | 256 | Args: 257 | config: Hyperparameter configuration for training and evaluation. 258 | workdir: Directory where the tensorboard summaries are written to. 259 | 260 | Returns: 261 | Final TrainState. 262 | """ 263 | 264 | writer = metric_writers.create_default_writer( 265 | logdir=workdir, just_logging=jax.process_index() != 0) 266 | 267 | rng = random.PRNGKey(config.seed) 268 | 269 | if config.batch_size % jax.device_count() > 0: 270 | raise ValueError('Batch size must be divisible by the number of devices') 271 | local_batch_size = config.batch_size // jax.process_count() 272 | 273 | platform = jax.local_devices()[0].platform 274 | input_dtype = get_input_dtype(config.half_precision, platform) 275 | 276 | dataset_builder = tfds.builder(config.dataset) 277 | 278 | num_classes = dataset_builder.info.features['label'].num_classes 279 | train, test = "train", "validation" 280 | 281 | if config.dataset == "cifar10" or config.dataset == "mnist": 282 | test = "test" 283 | 284 | train_freq = config.train_freq 285 | 286 | init_key, rng = random.split(rng) 287 | trans_mat, rot_mat = init_trans_mat_and_rot_mat(init_key, num_classes) 288 | environment = Environment(config.dataset, trans_mat, rot_mat, batch_size=local_batch_size) 289 | 290 | steps_per_epoch = ( 291 | dataset_builder.info.splits[train].num_examples // config.batch_size 292 | ) 293 | 294 | if config.num_train_steps == -1: 295 | num_steps = int(steps_per_epoch * config.num_epochs) 296 | else: 297 | num_steps = config.num_train_steps 298 | 299 | if config.steps_per_eval == -1: 300 | num_validation_examples = dataset_builder.info.splits[ 301 | test].num_examples 302 | steps_per_eval = num_validation_examples // config.batch_size 303 | else: 304 | steps_per_eval = config.steps_per_eval 305 | 306 | steps_per_checkpoint = steps_per_epoch * 10 307 | 308 | model_cls = getattr(models, config.model) 309 | model = create_model( 310 | model_cls=model_cls, num_classes=num_classes, half_precision=config.half_precision) 311 | 312 | learning_rate_fn = create_learning_rate_fn( 313 | num_steps, config.learning_rate) 314 | 315 | if config.image_size == -1: 316 | image_size = dataset_builder.info.features['image'].shape[0] 317 | else: 318 | image_size = config.image_size 319 | 320 | init_key, rng = random.split(rng) 321 | state = create_train_state(init_key, config, model, image_size, learning_rate_fn) 322 | state = restore_checkpoint(state, workdir) 323 | # step_offset > 0 if restarting from checkpoint 324 | step_offset = int(state.step) 325 | state = jax_utils.replicate(state) 326 | 327 | p_train_step = jax.pmap( 328 | functools.partial(train_step, learning_rate_fn=learning_rate_fn, 329 | num_classes=num_classes, 330 | weight_decay=config.weight_decay), axis_name='batch') 331 | 332 | p_eval_step = jax.pmap(functools.partial(eval_step, num_classes=num_classes), axis_name='batch') 333 | 334 | train_metrics = [] 335 | hooks = [] 336 | if jax.process_index() == 0: 337 | hooks += [periodic_actions.Profile(num_profile_steps=5, logdir=workdir)] 338 | 339 | train_metrics_last_t = time.time() 340 | logging.info('Initial compilation, this might take some minutes...') 341 | local_device_count = jax.local_device_count() 342 | 343 | for step in range(step_offset, num_steps): 344 | batch = environment.get_data() 345 | if step == step_offset: 346 | counts, datasets = gdumb.init_sampler() 347 | 348 | if train_freq != 1: 349 | scan_key, rng = random.split(rng) 350 | keys = random.split(scan_key, local_batch_size) 351 | images, labels = batch['image'], batch['label'].flatten() 352 | images = images.reshape((local_batch_size, *images.shape[-3:])) 353 | for key, image, label in zip(keys, images, labels): 354 | counts, datasets = gdumb.sample((counts, datasets), (key, image, label)) 355 | 356 | n = (len(datasets[0]) // local_device_count) * local_device_count 357 | batch = {'image': datasets[0][:n].reshape((local_device_count, -1, *images.shape[-3:])), 358 | 'label': datasets[1][:n].reshape((local_device_count, -1))} 359 | 360 | if (step + 1) % train_freq == 0: 361 | state, metrics = p_train_step(state, batch) 362 | for h in hooks: 363 | h(step) 364 | if step == step_offset: 365 | logging.info('Initial compilation completed.') 366 | 367 | if config.get('log_every_steps'): 368 | train_metrics.append(metrics) 369 | if (step + 1) % config.log_every_steps == 0: 370 | train_metrics = common_utils.get_metrics(train_metrics) 371 | summary = { 372 | f'{train}_{k}': v 373 | for k, v in jax.tree_map(lambda x: x.mean(), train_metrics).items() 374 | } 375 | summary['steps_per_second'] = config.log_every_steps / ( 376 | time.time() - train_metrics_last_t) 377 | writer.write_scalars(step + 1, summary) 378 | train_metrics = [] 379 | train_metrics_last_t = time.time() 380 | 381 | if (step + 1) % steps_per_epoch == 0: 382 | epoch = step // steps_per_epoch 383 | eval_metrics = [] 384 | 385 | # sync batch statistics across replicas 386 | state = sync_batch_stats(state) 387 | for _ in range(steps_per_eval): 388 | eval_batch = environment.get_test_data(device_count=local_device_count) 389 | metrics = p_eval_step(state, eval_batch) 390 | eval_metrics.append(metrics) 391 | 392 | eval_metrics = common_utils.get_metrics(eval_metrics) 393 | summary = jax.tree_map(lambda x: x.mean(), eval_metrics) 394 | logging.info('eval epoch: %d, loss: %.4f, accuracy: %.2f', 395 | epoch, summary['loss'], summary['accuracy'] * 100) 396 | writer.write_scalars( 397 | step + 1, {f'eval_{key}': val for key, val in summary.items()}) 398 | writer.flush() 399 | 400 | if (step + 1) % steps_per_checkpoint == 0 or step + 1 == num_steps: 401 | state = sync_batch_stats(state) 402 | save_checkpoint(state, workdir) 403 | 404 | # Wait until computations are done before exiting 405 | random.normal(random.PRNGKey(0), ()).block_until_ready() 406 | 407 | return state 408 | -------------------------------------------------------------------------------- /shift_happens/gendist/notebooks/013-metadata-inference.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f0a5a74e-cfc6-491c-8267-60c5b1ceefa5", 6 | "metadata": {}, 7 | "source": [ 8 | "# Metadata inference" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "id": "e8326182-b4fd-4ded-ad61-7095eb4a8f69", 15 | "metadata": {}, 16 | "outputs": [ 17 | { 18 | "name": "stdout", 19 | "output_type": "stream", 20 | "text": [ 21 | "/home/gerardoduran/documents/shift-happens/gendist/experiments\n" 22 | ] 23 | } 24 | ], 25 | "source": [ 26 | "%cd ../gendist/experiments\n", 27 | "%load_ext autoreload\n", 28 | "%autoreload 2" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "id": "23c89333-052f-4bcc-9921-c28ff816419a", 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import jax\n", 39 | "import optax\n", 40 | "import gendist\n", 41 | "import torchvision\n", 42 | "import numpy as np\n", 43 | "import jax.numpy as jnp\n", 44 | "import matplotlib.pyplot as plt\n", 45 | "import mnist_rotation_meta as metaexp" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 204, 51 | "id": "48f9cbe6-cbe3-425a-933d-343c325b706c", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "%config InlineBackend.figure_format = \"retina\"" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 82, 61 | "id": "e0c25fc1-f25f-4c39-997d-f32a7ab9d284", 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "configs = np.linspace(0, 360, 150)\n", 66 | "list_configs = [{\"angle\": float(deg)} for deg in configs]" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 83, 72 | "id": "ca82ca0c-8efa-47b6-a655-725a06e65d09", 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "key = jax.random.PRNGKey(314)\n", 77 | "key_subset, key_train = jax.random.split(key)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 159, 83 | "id": "664bfc7a-7fdc-4a9d-9689-196df47b1626", 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "n_train_subset = 600\n", 88 | "mnist_train = torchvision.datasets.MNIST(root=\".\", train=True, download=True)\n", 89 | "X_train = np.array(mnist_train.data) / 255\n", 90 | "X_train = metaexp.configure_covariates(key_subset, metaexp.processing_class, X_train, list_configs, n_train_subset)\n", 91 | "X_train = X_train * 2 - 1" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 242, 97 | "id": "970348cd-f072-46ca-a3bb-6be88c781778", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "alpha = 0.00005\n", 102 | "# alpha = 0.001\n", 103 | "n_epochs = 100\n", 104 | "batch_size = 2000\n", 105 | "\n", 106 | "tx = optax.adam(learning_rate=alpha)\n", 107 | "lossfn = gendist.training.make_von_mises_loss_func\n", 108 | "meta_model = gendist.models.LeNet5Regression(1)\n", 109 | "# meta_model = gendist.models.MLPDataV1(1)\n", 110 | "\n", 111 | "configs_radians = configs / 180 * jnp.pi\n", 112 | "trainer = gendist.training.TrainingMeta(meta_model, lossfn, tx)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 243, 118 | "id": "483e53f0-a310-4f38-b69d-833afa320dd7", 119 | "metadata": {}, 120 | "outputs": [ 121 | { 122 | "data": { 123 | "application/vnd.jupyter.widget-view+json": { 124 | "model_id": "62ab7d65b64246d0aa015b59248086dc", 125 | "version_major": 2, 126 | "version_minor": 0 127 | }, 128 | "text/plain": [ 129 | " 0%| | 0/100 [00:00]" 150 | ] 151 | }, 152 | "execution_count": 244, 153 | "metadata": {}, 154 | "output_type": "execute_result" 155 | }, 156 | { 157 | "data": { 158 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxsAAAHwCAYAAADOwnKhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAABYlAAAWJQFJUiTwAACySUlEQVR4nOzdd5xcdbk/8M8zO9tLeg+kkRAIJQQIvSMiSBVUVEQQFVHItdzrvfJTinrliiBdQAQF8QpXURBEaqQn1ABpkLbpPdnN1tkyz++Pc+bM95w9Z+qZ3Znweb9eeU12Zs7M7GQXzjNPE1UFERERERFR2CID/QKIiIiIiGj3xGCDiIiIiIgKgsEGEREREREVBIMNIiIiIiIqCAYbRERERERUEAw2iIiIiIioIBhsEBERERFRQTDYICIiIiKigmCwQUREREREBcFgg4iIiIiICoLBBhERERERFQSDDSIiIiIiKojoQL8Ayo2IrALQAKBxgF8KEREREe3eJgLYpaqTsj2QwUbpaqiurh66zz77DB3oF0JEREREu68lS5ago6Mjp2MZbJSuxn322Wfo22+/PdCvg4iIiIh2YwcffDDeeeedxlyOZc8GEREREREVRGjBhoiMF5H7RGSDiMREpFFEbhaRIVk8hojIJSIyT0RaRKRdRN4VkStFpCzgmDoR+YmILBGRThFpEpHnReS0gPsfKyIPishCEdluH7NKRB4XkZN87j9RRDSDP8d4jmtMcd9Nmb4nRERERESlKpQyKhGZAuA1ACMBPAZgKYDZAOYAOFVEjlLV7Rk81O8BXAhgC4CHAbQBOBnALQCOFZHzVVWN5x0M4GUA+wFYBOBuALUAzgTwpIjMUdVbPc9xov1nPoAX7OfY0z7mDBH5qar+yLh/E4BrA17vHgAuAbAdwBs+tzcDuNnn+taAxyMiIiIi2m2E1bNxJ6xA40pVvS1xpYjcBOA7AH4G4LJUDyAiZ8MKNFYBmK2q2+zrywE8AuAzAC4C8DvjsGtgBRqPAvicqvbYx4yAdfL/SxF5SlWXGcdcr6rX+Dz/OADvAPihiNypqhsBQFWb7Ofxe80/t//6gKrGfO7S5PdcREREREQfB3mXUYnIZACnwBrBeofn5qthZQ4uFJHaNA91rn15YyLQAABV7QaQyDRcEXDMjxOBhn3MVgA3AiiHJ8hR1U6/J1fV9bCyMxEAk9O8VohIFMBX7C/vSXd/IiIiIqKPmzAyGyfal8+oaty8QVVbRORVWMHI4QCeT/E4o+3LlT63Ja6bJSKD7WxDpsf06cPwIyIjARwGIAbgwwwOOct+/pdUdWnAfSpF5EuwyrTaALxv3783k9dERERERFTKwgg29rYvPwq4fRmsYGMaUgcbiWyG37IQM9MwHcA845gx9jGLA46Z7vdkInIIgE/Deg/Gw+rZaABwhZlZSeHr9uXdKe4zGsCDnutWicjFqvpiBs8BEQmabev7fRERERERFYswplENsi+bA25PXD84zeM8YV9+V0ScRXV2uZLZoD3E55hrzGlVIjIMwHftLytFpNrn+Q6BVeZ1FaxekCiAi1X112leJ0RkIqzG9e0A/hJwt/thZVVGw2pa3x9WYDIRwFMicmC65yEiIiIiKmX9sdRP7EtNeS/gTwC+BOBTABaLyOMA2mGd1E+BlSGZCsAsQfoxrKzJ+QD2EZHnAdTAKnFqsY+v8RxjvRjVuwDcJSJVsDIjlwF4wJ6clbKZHcDXYAVqvw9oDIeqeidYLQRwmYi0AvgerKbzc9I8D1T1YL/r7YzHrHTHExERERENlDAyG4nMxaCA2xs89/Nl93ucCeD7ADbBmkx1CYB1AI6GlUUArLG4iWM2ATgUwK2wsgeXwwo0noAVpFQDaFbVrhTP26mqS1R1DqzMwzdE5Lyg+9uZlovtL3NpDL/Lvjw2h2OJiIiIiEpGGMFGopl6WsDtU+3LoJ4Oh6r2qOqNqjpTVatVtUFVT4XVjzETQAesfRrmMVtVdY6qTlbVClUdpapfhZWtEABvZvG9PGVfHp/iPmfA6hN5UVUzaST3SgRL6aZzERERERGVtDCCjbn25Ski4no8EakHcBSsIGGe98AsXAigCsAj9ijcTHzNvnwoi+cZZ1/2pLhPojE813G3R9iXfhO0iIiIiIh2G3kHG6q6AsAzsBqfv+W5+VpYn+A/oKptgLWkT0Sm21vHXUSkwee6QwFcD2vr9nWe2yIiUudzzKUALgCwAJ5gQ0SO8wZF9vVTYDWLA8CTft+riEyA1SOSqjEcIjLDbHL3HH+7/eUfgo4nIiIiItodhNUgfjmshXi3ishJAJbA2llxAqzyqauM+46zb18NK0AxPSsiHbCaqVsAzABwGqzdF+eqqjcbUANgs4g8C2C5fd0xAGYDWAHgHJ9MyGMAmkRkPoC1sN6DKQBOtf9+m6o+G/B9Xoo0jeG28wH8p4jMhbURvcV+jtNhZWj+AeCXKY4nIiIiIip5oQQbqrrC3ltxHayT9tMAbITVuH2tqu7I8KH+DODzsKZSVQPYAOBeANeraqPP/WOwplgdDeAT9nUrYI20vUlVW32OuRrJJYNnACgDsBnA3wDcq6pP+70we7TuJfaX6Uqo5sLaP3IQrLKpWgBNAF6BtXfjQVVNN52LiIiIiKikCc95S5OIvD1r1qxZb78dtPNv96SqEJH0dyQiIiKiUBx88MF455133glayZBKGA3iRP3iP//yPmb/9/P4xwcbB/qlEBEREVEGGGxQSWjc1oY/vbkWW1tiuOvFFQP9coiIiIgoAww2qCRsb0v24ze1Zzr9mIiIiIgGEoMNKgktncnVJ53dvQP4SoiIiIgoUww2qCS0xZIBRqwnPoCvhIiIiIgyxWCDSkJrLFk6xcwGERERUWlgsEElodWT2eDIZiIiIqLix2CDSkKr0bMBsJSKiIiIqBQw2KCS0NblCTa6GWwQERERFTsGG1QSWvpkNti3QURERFTsGGxQSWiLuYONTmY2iIiIiIoegw0qCd5gg5kNIiIiouLHYINKQgszG0REREQlh8EGlYQ+ZVTMbBAREREVPQYbVBJavWVUzGwQERERFT0GG1QS+jaIM7NBREREVOwYbFBJ6Dv6lpkNIiIiomLHYIOKXndvvE9wwcwGERERUfFjsEFFz1tCBbBBnIiIiKgUMNigoudtDgfYIE5ERERUChhsUNFri/XNYjCzQURERFT8GGxQ0WuNdfe5jpkNIiIiouLHYIOKXiszG0REREQlicEGFb3WTvZsEBEREZUiBhtU9PymUcWY2SAiIiIqegw2qOi1+I2+ZWaDiIiIqOgx2KCix8wGERERUWlisEFFz2/PBjMbRERERMWPwQYVPd+lfsxsEBERERU9BhtU9PymUTGzQURERFT8GGxQ0fPr2ejsZmaDiIiIqNgx2KCi519GxcwGERERUbFjsEFFz79BnJkNIiIiomLHYIOKnv/oW2Y2iIiIiIpdaMGGiIwXkftEZIOIxESkUURuFpEhWTyGiMglIjJPRFpEpF1E3hWRK0WkLOCYOhH5iYgsEZFOEWkSkedF5LSA+x8rIg+KyEIR2W4fs0pEHheRk3zuP1FENIM/xxTiPSFmNoiIiIhKVTSMBxGRKQBeAzASwGMAlgKYDWAOgFNF5ChV3Z7BQ/0ewIUAtgB4GEAbgJMB3ALgWBE5X1XVeN7BAF4GsB+ARQDuBlAL4EwAT4rIHFW91fMcJ9p/5gN4wX6OPe1jzhCRn6rqj4z7NwG4NuD17gHgEgDbAbxRoPfkY8+3Z4PTqIiIiIiKXijBBoA7YZ1UX6mqtyWuFJGbAHwHwM8AXJbqAUTkbFiBxioAs1V1m319OYBHAHwGwEUAfmccdg2sQONRAJ9T1R77mBGwTv5/KSJPqeoy45jrVfUan+cfB+AdAD8UkTtVdSMAqGqT/Tx+r/nn9l8fUNWY5+a83xMCenrjzphbESARanb1xhGPKyIRGcBXR0RERESp5F1GJSKTAZwCoBHAHZ6br4aVObhQRGrTPNS59uWNiUADAFS1G0Ai03BFwDE/TgQa9jFbAdwIoByeE3pV7fR7clVdDysTEQEwOc1rhYhEAXzF/vIez21hvScfe22xZLlUXWUUldHkjyz7NoiIiIiKWxg9Gyfal8+oquvsT1VbALwKoAbA4WkeZ7R9udLntsR1s+zSqWyO6dOH4UdERgI4DEAMwIcZHHKW/fwvqepSz21hvScfey2xbufvdZVRVJUnW3e4RZyIiIiouIVRRrW3fflRwO3LYH3KPw3A8ykeJ5HNmORzm5lpmA5gnnHMGPuYxQHHTPd7MhE5BMCnYb0H42H1bDQAuMLMrKTwdfvybp/bwnpPICJvB9zk+33tbryZjd6407LDLeJERERERS6MzMYg+7I54PbE9YPTPM4T9uV3RWRo4kq7XMls0B7ic8w15rQqERkG4Lv2l5UiUu3zfIfAKmm6ClYvSBTAxar66zSvEyIyEVbj+nYAf/G5S1jvycee2Rxe68lscCIVERERUXELq0E8lUQHr6a8F/AnAF8C8CkAi0XkcQDtsE7qp8DKBkwFYJ5h/hhWhuB8APuIyPOwypPOAtBiH1/jOcZ6Map3AbhLRKpgZUYuA/CAPSUqXeP212AFar/3aQzPRKbvCVT1YN8HsDIes3J47pJiBht1lVHXzg32bBAREREVtzAyG4lP6QcF3N7guZ8vu7fhTADfB7AJ1mSqSwCsA3A0rCwCYI3FTRyzCcChAG6FNfL2cliBxhOwgpRqAM2q2pXieTtVdYmqzoFVEvUNETkv6P52puVi+8t7Au4WyntC7oV+3p4NZjaIiIiIilsYwUaimXpawO1T7cug/gWHqvao6o2qOlNVq1W1QVVPhdWPMRNAB6x9GuYxW1V1jqpOVtUKVR2lql+Fla0QAG9m8b08ZV8en+I+Z8DqE3lRVYMayUN7Tz7uWju9ZVScRkVERERUKsIINubal6eIiOvxRKQewFGwgoR53gOzcCGAKgCP2KNwM/E1+/KhLJ5nnH3Zd4tcUqIxPCirAfTPe/KxYJZR1VdFURllZoOIiIioVOQdbKjqCgDPAJgI4Fuem6+FVd70gKq2AdaSPhGZbm/YdhGRBp/rDgVwPYBWANd5bouISJ3PMZcCuADAAniCDRE5zhsA2NdPgdUsDgBP+n2vIjIBVo9IUGM4gOzfEwrmbhAvc2U2GGwQERERFbewGsQvh7UQ71YROQnAElg7K06AVSp0lXHfcfbtq2GdjJueFZEOAAthNXjPAHAarN0X56qqd59GDYDNIvIsgOX2dccAmA1gBYBzfDIhjwFoEpH5ANbCeg+mADjV/vttqvpswPd5KTJvDM/mPaEA7p6Ncldmg2VURERERMUtlGBDVVfYeyuug3XSfhqAjbAat69V1R0ZPtSfAXwe1lSqagAbANwL4HpVbfS5fwzWFKujAXzCvm4FrJG2N6lqq88xV8PKThwOq/+iDMBmAH8DcK+qPu33wuzRupfYX6YqoQIQ6nvysdbiCjbKUMnMBhEREVHJCG30raquRXJKU6r7NSI5+tV72w0AbsjiObsBfDXT+9vH3ALglmyOsY/rRbKnI9NjMnpPKJgrs1Hl3SDOzAYRERFRMQujQZyoYMxgo7YiisooMxtEREREpYLBBhW1ls7gPRvMbBAREREVNwYbVNTautxlVGZmI8bMBhEREVFRY7BBRa3vUj9jzwYzG0RERERFjcEGFbXWWDJ7UV8ZRRUzG0REREQlg8EGFbXWWHJNSm1lFJVmZqObmQ0iIiKiYsZgg4pWT2/cCShEgJoKzwbxHmY2iIiIiIoZgw0qWm1GCVVdRRQi4t4gzswGERERUVFjsEFFq9UziQoAMxtEREREJYTBBhUt10K/SjvYYGaDiIiIqGQw2KCi5V3oBwCVzGwQERERlQwGG1S0zMyGE2xEOY2KiIiIqFQw2KCi1eoqo7KCDLNnI8bMBhEREVFRY7BBRavVldkoBwBOoyIiIiIqIQw2qGi1uno2EpkNI9hgZoOIiIioqDHYoKLl6tmo8mkQZ2aDiIiIqKgx2KCi1Zpm9G1nNzMbRERERMWMwQYVLTPYqLeDjfIygYh1XU9c0dPL7AYRERFRsWKwQUXLL7MhIu7Ffj0MNoiIiIiKFYMNKlp+ezYA7/hbBhtERERExYrBBhWt1oBgo5J9G0REREQlgcEGFS2/MirAndlgsEFERERUvBhsUNFqiyUDicToW8Cz2I9lVERERERFi8EGFa2WzvQ9G8xsEBERERUvBhtUtIIaxCvLmdkgIiIiKgUMNqgo9fTG0WFnLUSAmopkgFEZZWaDiIiIqBQw2KCi1NZl9GtURCGJTX4AqsrNaVTMbBAREREVKwYbVJSCJlEB7sxGrIeZDSIiIqJixWCDipKrX6PKHWyYmY0YMxtERERERYvBBhWlVJkN9wZxZjaIiIiIihWDDSpKrcbY2/o+ZVTs2SAiIiIqBQw2qCi1uTIbZa7buGeDiIiIqDQw2KCi1JKyQZx7NoiIiIhKAYMNKkpmZsNbRsXMBhEREVFpCC3YEJHxInKfiGwQkZiINIrIzSIyJIvHEBG5RETmiUiLiLSLyLsicqWIlAUcUyciPxGRJSLSKSJNIvK8iJwWcP9jReRBEVkoItvtY1aJyOMiclKa1zdRRH4tIivt47aLyHwR+Z7PfRtFRAP+bMr0Pfm4Mns2+jaIM7NBREREVAqi6e+SnohMAfAagJEAHgOwFMBsAHMAnCoiR6nq9gwe6vcALgSwBcDDANoAnAzgFgDHisj5qqrG8w4G8DKA/QAsAnA3gFoAZwJ4UkTmqOqtnuc40f4zH8AL9nPsaR9zhoj8VFV/5PM9fhLAo7Desyfs11cHYG8A5wC40ef7aQZws8/1rWneh4+91q7g0bfcIE5ERERUGkIJNgDcCSvQuFJVb0tcKSI3AfgOgJ8BuCzVA4jI2bACjVUAZqvqNvv6cgCPAPgMgIsA/M447BpYgcajAD6nqj32MSMAvAHglyLylKouM465XlWv8Xn+cQDeAfBDEblTVTcat00G8GcA2wGcrKofeY4tD/i2mvyei9IzMxt1KTIbncxsEBERERWtvMuo7BPxUwA0ArjDc/PVsDIHF4pIbZqHOte+vDERaACAqnYDSGQargg45seJQMM+ZiusTEM5PEGOqnb6PbmqroeVnYkAmOy5+RpYWYxvegMN4zVSiFxL/VJtEGdmg4iIiKhohZHZONG+fEZVXR8zq2qLiLwKKxg5HMDzKR5ntH250ue2xHWzRGSwqjZlcUzKPowEERkJ4DAAMQAfGteXAzgPVmnXP0RkNoCjYL13S2B9310BD1spIl+CVabVBuB9AC+pKs+Q02iNJd+iPtOomNkgIiIiKglhBBt725d9PvG3LYMVbExD6mAjkc2Y5HObmWmYDmCeccwY+5jFAcdM93syETkEwKdhvQfjYfVsNAC4wsyswCrTqgbwOoA/Afis56HWiMh5qvqmz9OMBvCg57pVInKxqr7o97p8XufbATf5fl+7i9ZYMlnUZxqVOfqWmQ0iIiKiohXGNKpB9mVzwO2J6weneZwn7MvvisjQxJUiEgVwrXG/IT7HXGNOqxKRYQC+a39ZKSLVPs93CKwyr6tg9YJEAVysqr/23G+kfXkcgNMAfBXAMAATAPwCVtbiHyIy3HPc/bCyKqNhNa3vD6uBfSKAp0TkQJ/XRLa2lJkNo0GcmQ0iIiKiohVWg3gqYl9qyntZWYMvAfgUgMUi8jiAdljTqKbAypBMBWB+lP1jWFmT8wHsIyLPA6gBcBaAFvv4Gs8x1otRvQvAXSJSBSszchmAB+zJWWafR5lx+V+qep/99Q4APxCRvWD1jnwNwM+NxzcDJABYCOAyEWkF8D1YfSDnpHlPoKoH+11vZzxmpTu+VLWmWOrHzAYRERFRaQgjs5HIXAwKuL3Bcz9fdr/HmQC+D2ATrMlUlwBYB+BoWJOgAKt3InHMJgCHArgVVvbgcliBxhOwgpRqAM0peiqgqp2qukRV58DKPHxDRM4z7rLT+PtffR4icd3sVN+f4S778tgM7/+xZAYb9d7Rt0Zmg3s2iIiIiIpXGMFGopl6WsDtU+3LoJ4Oh6r2qOqNqjpTVatVtUFVT4XVjzETQAesfRrmMVtVdY6qTlbVClUdpapfhZWtEAB+vRRBnrIvjzeu+9D4e5PPMYlgxK9Uy08iWEo3netjLdOlftyzQURERFS8wgg25tqXp4iI6/FEpB7W5KYOJJu6c3EhgCoAj2QxZvZr9uVDWTzPOPvSHKO7A8AC+8v9fI5JXNeY4XMcYV/6TdAiAL1xRYcdRIgANeXu5fFVUWY2iIiIiEpB3sGGqq4A8AysxudveW6+FtYn+A+oahtgjZIVken21nEXEWnwue5QANfD2rp9nee2iIjU+RxzKYALYAUJD3luO84bFNnXT4HVLA4AT3puTuwP+Znd45E4ZjyspYWA1XOSuH6G2eRuXD8BwO32l3/w3k4WV79GRRSRiLhur2Rmg4iIiKgkhNUgfjmshXi3ishJsPZPHAbgBFjlU1cZ9x1n374aVoBielZEOmA1U7cAmAFrAlQMwLmq6s0G1ADYLCLPAlhuX3cMrP6JFQDO8cmEPAagSUTmA1gL6z2YAuBU+++3qeqznmPuA3A6gLMBvCciT8MKos4GMBTArar6L+P+5wP4TxGZC2sjeov9HKfDytD8A8AvQb5SLfQD3JmNzu5eqCpEpM/9iIiIiGhghRJsqOoKe2/FdbBO2k8DsBFW4/a1dilSJv4M4POwplJVA9gA4F4A16tqo8/9Y7AyCkcD+IR93QpYI21vUtVWn2OuRnLJ4BmwpkxtBvA3APeq6tM+319cRM6Hlbm5GMClAOIA3gPwa1X1Zinmwto/chCssqlaWP0er8Dau/GgqqabzvWx5Z5EVdbn9mhZBGURQW9cEVegJ64oL2OwQURERFRsQht9q6prYZ2Ip7tfI5LjcL233QDghiyesxvW3ouMqeotAG7J5hj7uB77uLTH2gv7MlraR32ZwUZdVbnvfaqiEbR1WSVUnd29KC8Lo/2IiIiIiMLUH3s2aDfysycXozXWg1h3HP997v6uyVBhcZdR+T9+VXmZE2zEeuKoD/1VEBEREVG+GGxQVh55ax2aO6w2mB99et+CBBvm2Fu/ng0AqPT0bRARERFR8WHtCWWlsh/GzqbaHp7g3rXB8bdERERExYjBBmWlPxbqtaaZRgUAFa6gh5kNIiIiomLEYIOy0h+ZjXSjbwFmNoiIiIhKAYMNykpleeEzCi0ZlVExs0FERERU7BhsUFYqo8mMQn9kNuqrghrEjdfBzAYRERFRUWKwQVkxMwoF69kwplHVVqTPbHAaFREREVFxYrBBWemPjEJrLBk81GWS2ShQhoWIiIiI8sNgg7JSPA3izGwQERERFTsGG5SVYhl9a74OZjaIiIiIihODDcpKf2c2gqZRcYM4ERERUfFjsEFZqeyHZXot3LNBREREtFtgsEFZqeyH8iVXz0Zggzj3bBAREREVOwYblJWqApcv9cYV7V3Jx60xghvX62Bmg4iIiKjoMdigrBQ6s9HW5S6hikQkg9fBzAYRERFRMWKwQVlxlS8VIKPgWuhX6Z/V8L4OZjaIiIiIihODDcpKoXslMtmxAXjKqJjZICIiIipKDDYoK5UF7pXIZMcGUPgMCxERERHlj8EGZaXQmY3WDCZRAd6lfsxsEBERERUjBhuUlcpogRvEY8YkqooUwQYzG0RERERFj8EGZaWqvLCjb83HrA4Yewt4yrmY2SAiIiIqSgw2KCuFzmyYJVFmYONl3sbMBhEREVFxYrBBWak0T/ILEGyYTedmYNPndUSZ2SAiIiIqdgw2KCvuKVDhn+TnktkoRDkXEREREeWPwQZlparAG8QzzWxUFbici4iIiIjyx2CDslLozIaZpUiV2ahkZoOIiIio6DHYoKwUvkE8+ZhVqaZReV6Hqob+WoiIiIgoPww2KCuFbxBPZinMLIpXWURQXiYAAFWgq5elVERERETFhsEGZcXslShE+ZIZwFSmyGz0fS0MNoiIiIiKDYMNykp5mUCshAJ64oqekDMKmWY2AHcwEuP4WyIiIqKiw2CDsiIiriAg7PIlM0ORqmcD8DarM7NBREREVGwYbFDWXONvQz7JNzMU6TIbri3izGwQERERFR0GG5Q1MwgIe3t3LKvMBns2iIiIiIoZgw3KmmvsbAEzG+mCDW4RJyIiIipuoQUbIjJeRO4TkQ0iEhORRhG5WUSGZPEYIiKXiMg8EWkRkXYReVdErhQR3zNPEakTkZ+IyBIR6RSRJhF5XkROC7j/sSLyoIgsFJHt9jGrRORxETkpzeubKCK/FpGV9nHbRWS+iHyvUO9JMXL1SoQ8/ta9QTxdGRW3iBMREREVs1CCDRGZAuBtABcDeAPArwCsBDAHwOsiMizDh/o9gN8CmATgYQC/AVAB4BYAD4sk5iA5zzsYwOsA/h+AXgB3A/gzgP0BPCkiV/o8x4n2n48APGS/1tcAnADgORH5ScD3+EkAiwBcAuBd+7g/AmgGcI7P/cN6T4qOeZIfdkYhm8yGq5yLmQ0iIiKiohMN6XHuBDASwJWqelviShG5CcB3APwMwGWpHkBEzgZwIYBVAGar6jb7+nIAjwD4DICLAPzOOOwaAPsBeBTA51S1xz5mBKwT/F+KyFOqusw45npVvcbn+ccBeAfAD0XkTlXdaNw2GVYQsx3Ayar6kefY8kK8J8WKmQ0iIiIiykTemQ37RPwUAI0A7vDcfDWANgAXikhtmoc61768MRFoAICqdgP4kf3lFQHH/DgRaNjHbAVwI4ByeE7oVbXT78lVdT2sDEcEwGTPzdcAqAPwTW+gYbxGR4jvSVGqLOAUqE5mNoiIiIh2G2FkNk60L59RVdfHy6raIiKvwjrxPhzA8ykeZ7R9udLntsR1s0RksKo2ZXFMyj6MBBEZCeAwADEAHxrXlwM4D8AWAP8QkdkAjoL13i2B9X13eR4urPcEIvJ2wE3TM/m+CqGqkA3iOWY2OI2KiIiIqPiEEWzsbV/2+cTftgzWifU0pD6xTmQzJvncZmYapgOYZxwzxj5mccAxviflInIIgE/Deg/GAzgTQAOAK8zMCqwyrWpYvSF/AvBZz0OtEZHzVPVN47qw3pOiZGY2whx9q6pZZTaquEGciIiIqKiF0SA+yL5sDrg9cf3gNI/zhH35XREZmrhSRKIArjXuN8TnmGvMaVV28/V37S8rRaTa5/kOgVXSdBWsXpAogItV9dee+420L48DcBqArwIYBmACgF8A2BNWxmO4cUxY7wlU9WC/PwCWpju2UAo1+ra7V6Fq/b28TFAWkZT3d5dRMbNBREREVGz6Y89G4oxR09zvTwCeAjAFwGIRuUdEbgawANZJfqLJ2/wI+8cAVgM4H8ACe6zsPbCyHHEA7T7HWC9G9S5VFVhZi30B3A/gARG5y3PXMuPyv1T1PlXdoaprVPUHsJrThwP4Wprvz5Tpe1KUCtUg3unaHp46qwEAlcxsEBERERW1MIKNxKf0gwJub/Dcz5fd23AmgO8D2ARrMtUlANYBOBrWJCjA6p1IHLMJwKEAbgVQC+ByAGfBynicDCuQaPbpqTCft1NVl6jqHFijc78hIucZd9lp/P2vPg+RuG62cV0o70mxKlT5ktnkbS7sC8LMBhEREVFxCyPYSDRTTwu4fap9GdS/4FDVHlW9UVVnqmq1qjao6qmwMhUzAXTA2nVhHrNVVeeo6mRVrVDVUar6VVh9HALgTe/zpPCUfXm8cd2Hxt+bfI5JBCNmqVZo70kxKtRJvrs5PH1mo5D7PoiIiIgof2EEG3Pty1NExPV4IlIPa3JTB5JN3bm4EEAVgEe8Y2ZTSJQ1PZTF84yzL80xujtglXIBVrO4V+K6RuO6/nhPBoy7jCq8k3zzsSozyGxUlRemnIuIiIiIwpF3sKGqKwA8A2AigG95br4WVnnTA6raBlijZEVkur1h20VEGnyuOxTA9QBaAVznuS0iInU+x1wK4AJYQcJDntuO8wYA9vVTYDWLA8CTnpsTuzJ+JiJVxjHjYS3oA6yeEwDZvyelprJAy/TMLElVJj0brkZ1ZjaIiIiIik1YG8Qvh7UQ71YROQnW/onDAJwAq1ToKuO+4+zbV8M6GTc9KyIdABYCaAEwA1ZzeAzAuarq3adRA2CziDwLYLl93TGw+idWADjHJxPyGIAmEZkPYC2s92AKgFPtv9+mqs96jrkPwOkAzgbwnog8DStgOBvAUAC3quq/8nhPSkqhlukxs0FERES0ewkl2FDVFfbeiutgnbSfBmAjrMbta+1SpEz8GcDnAXwJVg/EBgD3ArheVRt97h+DlVE4GsAn7OtWwBppe5OqtvocczWSC/XOgDVlajOAvwG4V1Wf9vn+4iJyPqwsxcUALoU17eo9AL9W1T/4HBPWe1J0CpXZiOWR2WDPBhEREVHxCSuzAVVdC+tEPN39GpEc/eq97QYAN2TxnN2w9l5kTFVvAXBLNsfYx/XYx2V8bKbvSalx9WyE2CDemUdmI8zlgkREREQUjv7Ys0G7mcKNvs0us+F6HRx9S0RERFR0GGxQ1go2+jbLzIbrdTCzQURERFR0GGxQ1go1+paZDSIiIqLdC4MNyppr5GyoDeJ5bBBnZoOIiIio6DDYoKwVauRsp/FY5sSr4NdhTqNiZoOIiIio2DDYoKwVapmeOb62KprBNCou9SMiIiIqagw2KGuVBcpsxLLMbFS6Rt8ys0FERERUbBhsUNbcezYKk9mozCCzYd6nqycOVQ3ttRARERFR/hhsUNaqCrVBPMvMhoigIlqYLAsRERER5Y/BBmXNvWdj4Ho2vPcL87UQERERUf4YbFDWCjb61nisqgwyG977MbNBREREVFwYbFDWyssEEbH+3hNX9PSGc5Ify7JnA/A0iTOzQURERFRUGGxQ1kSkINkN1wbxTDMbBcqyEBEREVH+GGxQTgox/jbWw8wGERER0e6EwQblxDX+tieck/x8MxvcIk5ERERUXBhsUE5cjdkhneSbQUtuDeLMbBAREREVEwYblBPX+NsCZDYyLqNyjb5lZoOIiIiomDDYoJy4GsSZ2SAiIiIiHww2KCeVBdjczcwGERER0e6FwQblpBAZBdcG8QwzG5XlZoM4MxtERERExYTBBuUk7IyCqroyJJlmNqoKMIKXiIiIiMLBYINy4t6zkX9GwQwUKsoiiCRWlKd7HVFmNoiIiIiKFYMNyklVyA3irqxGeeY/lsxsEBERERUvBhuUE9fm7jAyG93m9vDM+jW8940xs0FERERUVBhsUE7CHn3r3h6eW2aDZVRERERExYXBBuUk7NG3Zt9Hps3hgHcqFsuoiIiIiIoJgw3KSWXIo2/dmY1syqiY2SAiIiIqVgw2KCdhj75lZoOIiIho98Ngg3LiLqNiZoOIiIiI+mKwQTmpDDmjYAYs2QQbVa4N4sxsEBERERUTBhuUkyozsxHyNKrsyqjCzbAQERERUXgYbFBOzMxGGHs2zBKo7MqomNkgIiIiKlYMNignlSFnNlwbxJnZICIiItotMNignFSFPvqWmQ0iIiKi3U1owYaIjBeR+0Rkg4jERKRRRG4WkSFZPIaIyCUiMk9EWkSkXUTeFZErRcT3DFRE6kTkJyKyREQ6RaRJRJ4XkdMC7n+siDwoIgtFZLt9zCoReVxETgo45ncioin+TPc5pjHF/Tdl+p4Uq/CX+hmZjSw2iFcys0FERERUtKJhPIiITAHwGoCRAB4DsBTAbABzAJwqIkep6vYMHur3AC4EsAXAwwDaAJwM4BYAx4rI+aqqxvMOBvAygP0ALAJwN4BaAGcCeFJE5qjqrZ7nONH+Mx/AC/Zz7Gkfc4aI/FRVfxTw+m4B0ORz/baA+zcDuNnn+taA+5cMdxlVuJkNM1uRjivDwswGERERUVEJJdgAcCesQONKVb0tcaWI3ATgOwB+BuCyVA8gImfDCjRWAZitqtvs68sBPALgMwAuAvA747BrYAUajwL4nKr22MeMAPAGgF+KyFOqusw45npVvcbn+ccBeAfAD0XkTlXd6PMyb1bVxlTfh0eT33PtDsyAIIzMRqdr9G0WmQ1zzwYzG0RERERFJe8yKhGZDOAUAI0A7vDcfDWszMGFIlKb5qHOtS9vTAQaAKCq3QASmYYrAo75cSLQsI/ZCuBGAOXwBDmq2un35Kq6HlZ2JgJgcprX+rHnbswOoYzKNfo288xGRVkEEbH+3t2r6O5ldoOIiIioWISR2TjRvnxGVV1neqraIiKvwgpGDgfwfIrHGW1frvS5LXHdLBEZrKpNWRzj24fhJSIjARwGIAbgw4C7fUpEGgD0AlgO4AVV3ZXiYStF5EuwyrTaALwP4CVVLfmP4F2jb0Moo4rlmNkQEdRURNEas2LNju5elJdx7gERERFRMQgj2Njbvvwo4PZlsIKNaUgdbCSyGZN8bjMzDdMBzDOOGWMfszjgmD7N2wAgIocA+DSs92A8rJ6NBgBXmJkVjzs9X7eIyH+pqjejkzAawIOe61aJyMWq+mLAMd7X+XbATb7fV38JvUE8x8wGYPVtJIKNzq5eNFSV5/16iIiIiCh/YXwEPMi+bA64PXH94DSP84R9+V0RGZq4UkSiAK417jfE55hrzGlVIjIMwHftLytFpNrn+Q6BVeZ1FaxekCiAi1X11z73fQnA5wBMAFANYAqA79u33S4iX/c55n5YWZXRsJrW94fVwD4RwFMicqDPMSUj9AbxHDMbAFBTkQxOOkJ4LUREREQUjrAaxFOxK+qhKe8F/AnAlwB8CsBiEXkcQDusaVRTYGVIpsIqYUr4MaysyfkA9hGR5wHUADgLQIt9fI3nGOvFqN4F4C4RqYKVGbkMwAP25Cxvn8d9nsNXArhRRD4E8HcAPxOR35rlUap6reeYhQAuE5FWAN+D1dx+Tpr3BKp6sN/1dsZjVrrjC8W9ZyPczEZVlpmNauO1tHcx2CAiIiIqFmFkNhKZi0EBtzd47ufL7vc4E1bGYBOsyVSXAFgH4GgAidG5W4xjNgE4FMCtsLIHl8MKNJ6AFaRUA2hW1a4Uz9upqktUdQ6szMM3ROS8VK/VOPYJAOsBDAewbybHALjLvjw2w/sXpWhEnMbsnriiJ8/GbDOzkc2eDQCoZmaDiIiIqCiFEWwkmqmnBdw+1b4M6ulwqGqPqt6oqjNVtVpVG1T1VFj9GDMBdMDap2Ees1VV56jqZFWtUNVRqvpVWNkKAfBmFt/LU/bl8Vkcs9W+TDdtKyERLGV6/6IkIqGOvzW3f2ezQRxwZzY6mNkgIiIiKhphBBtz7ctTRMT1eCJSD+AoWEHCPO+BWbgQQBWAR+xRuJn4mn35UBbPM86+7El5L5uIDILVqK2wRv9m4gj70m+CVkkJc/ytOY3K7AfJhCuzwWCDiIiIqGjkHWyo6goAz8BqfP6W5+ZrYX2C/4CqtgHWkj4RmW5vHXexx8p6rzsUwPWwtm5f57ktIiJ1PsdcCuACAAvgCTZE5DhvUGRfPwVWszgAPGlcP1pE9vK5fx2sBYNVAJ6zS7oSt80wm9yN6ycAuN3+8g/e20uNO7OR30l+XpkNI9hoZxkVERERUdEIq0H8clgL8W4VkZMALIG1s+IEWOVTVxn3HWffvhpWgGJ6VkQ6YDVTtwCYAeA0WLsvzlVVbzagBsBmEXkW1t4LADgGwGwAKwCc45MJeQxAk4jMB7AW1nswBcCp9t9vU9VnjftPBzBXRF63X/cW+3v4BKxJUysBXOp5jvMB/KeIzIW1Eb3Ffo7TYQUn/wDwS5Q4s7fCDBZykVdmw9z5wcwGERERUdEIJdhQ1RX23orrYJ20nwZgI6zG7WtVdUeGD/VnAJ+HNZWqGsAGAPcCuF5VG33uH4M1xepoWCf/gBVkXA3gJlVt9TnmaiSXDJ4BoAzAZgB/A3Cvqj7tuf8KAPfAakQ/E9YI33ZYvSq3A7hVVVs8x8yFtX/kIFhlU7UAmgC8AmvvxoOqmm46V9Fz79oYuMwGR98SERERFafQRt+q6loAF2dwv0Ykx+F6b7sBwA1ZPGc3gK9men/7mFsA3JLF/dcC+EaWz/EigIyW9pUy1/jbPDMb5hZyjr4lIiIi2j2E0SBOH1NmZqMzz4yC2WDO0bdEREREuwcGG5SzsEbfxuOKLjPYyKNno6Mro0FiRERERNQPGGxQztw9G7kHG13GQsCKaAQivlV2gZjZICIiIipODDYoZ66ejTwaxN39Gtn/SLozG/n1jhARERFReBhsUM7cPRu5n+SbWZFsJ1EB3swGy6iIiIiIigWDDcpZZXk4o2/NzEa2zeGAZ/Qtp1ERERERFQ0GG5QzV4N4HpkN146NLMfeAu5sCEffEhERERUPBhuUM3dmI58yqvwyG64N4mwQJyIiIioaDDYoZ2ZmI5+T/HwzGzUVyd2UzGwQERERFQ8GG5SzsEbfhpnZ4OhbIiIiouLBYINyFt7o2/wyG+Y0KpZRERERERUPBhuUs7AyG649G3mOvmUZFREREVHxYLBBOXPv2cj9JN8MVCrzXerX3QtVzfm1EBEREVF4GGxQztxlVOFkNipzyGyURQQVdpCimt9rISIiIqLwMNignLnKqELaIJ5LZgPwZDdYSkVERERUFBhsUM4qQ2sQz69nA3BvEW9nkzgRERFRUWCwQTljZoOIiIiIUmGwQTkLa/RtLITMBsffEhERERUfBhuUs8KMvs0/s8Hxt0RERETFgcEG5awwo2/zz2xwizgRERFRcWCwQTmrLMDo2zAyGx1dPTm/FiIiIiIKD4MNyllVSGVUzGwQERER7Z4YbFDOXJmNPE7ww8hsuEbfsmeDiIiIqCgw2KCcuXo2Qsps5DqNqoqjb4mIiIiKDoMNyll5WQRlEQEA9MYVPb25BRxmZiPXPRs1HH1LREREVHQYbFBewhh/29mdf2aDo2+JiIiIig+DDcpLGONvzYWAuWY2XGVUzGwQERERFQUGG5QXc3rUQGY2aiqizt/Zs0FERERUHBhsUF7M6VG5Bhvu0bc57tmoSB7HzAYRERFRcWCwQXlxZzZyLKMyG8Rz7tlIZjbYs0FERERUHBhsUF4qy82ejRzLqHpC2CDOaVRERERERYfBBuXFNY0qh5P83riiu1cBACJARVn+o2/Zs0FERERUHBhsUF7Mhu5ceja8k6hEJKfXwdG3RERERMWHwQblJd89G7Fuszk8t34NwB30sIyKiIiIqDiEFmyIyHgRuU9ENohITEQaReRmERmSxWOIiFwiIvNEpEVE2kXkXRG5UkR8z0RFpE5EfiIiS0SkU0SaROR5ETkt4P7HisiDIrJQRLbbx6wSkcdF5KSAY34nIpriz/RCvSfFzgwQcjnJD6NfA3CXUTGzQURERFQcounvkp6ITAHwGoCRAB4DsBTAbABzAJwqIkep6vYMHur3AC4EsAXAwwDaAJwM4BYAx4rI+aqqxvMOBvAygP0ALAJwN4BaAGcCeFJE5qjqrZ7nONH+Mx/AC/Zz7Gkfc4aI/FRVfxTw+m4B0ORz/TbvFSG+J0WtMs/Rt2FlNqq51I+IiIio6IQSbAC4E9ZJ9ZWqelviShG5CcB3APwMwGWpHkBEzoYVaKwCMFtVt9nXlwN4BMBnAFwE4HfGYdfACjQeBfA5Ve2xjxkB4A0AvxSRp1R1mXHM9ap6jc/zjwPwDoAfisidqrrR52XerKqNqb4PQ97vSSnId/RtWJmNajaIExERERWdvMuoRGQygFMANAK4w3Pz1bAyBxeKSG2ahzrXvrwxEWgAgKp2A0hkGq4IOObHiUDDPmYrgBsBlMNzQq+qnX5PrqrrYWUiIgAmp3mtKYX4nhQ99zSq7DMbYWwPT7yORG95V28cPb25jeElIiIiovCE0bNxon35jKq6zvBUtQXAqwBqABye5nFG25crfW5LXDfLLp3K5hjfPgwvERkJ4DAAMQAfBtztUyLyAxH5voicLSINAfcL6z0peq49GzlkNlwL/XLcHg4AIuIqperMcZs5EREREYUnjDKqve3LjwJuXwbrU/5pAJ5P8TiJbMYkn9vMTMN0APOMY8bYxywOOCaoefsQAJ+G9R6Mh9Wz0QDgCjOz4nGn5+sWEfkvVfVmL8J6TyAibwfc5Pt99bcqs4wql8xGTziZDcBqEk80h7d39aCuMqwqQSIiIiLKRRiZjUH2ZXPA7YnrB6d5nCfsy++KyNDElSISBXCtcb8hPsdcY06rEpFhAL5rf1kpItU+z3cIrJKmq2D1gkQBXKyqv/a570sAPgdgAoBqAFMAfN++7XYR+brn/mG9J0Uv/wbxcDIbgGf8bRczG0REREQDrT8++k1sadOU9wL+BOBLAD4FYLGIPA6gHdY0qimwsgFTAZi1Oj+GlSE4H8A+IvI8rPKkswC02MfXeI6xXozqXQDuEpEqWJmRywA8YE+J8vZ53Oc5fCWAG0XkQwB/B/AzEfmtqmZaR5TpewJVPdj3AayMx6wMn69g8h99a0yjCiGzkdDe3ZPinkRU7FZsbcW/PtyK0/cfg9GDqgb65RARUY7CyGwkPqUfFHB7g+d+vuzehjNhZQw2wZpMdQmAdQCOBpAYE7vFOGYTgEMB3Apr5O3lsAKNJ2AFKdUAmlW1K8XzdqrqElWdA2t07jdE5LxUr9U49gkA6wEMB7CvcVMo70kpyHepnxmgVOUx+hbwjL/lRCqiktXTG8eXf/sGfvLEYsz507sD/XKIiCgPYQQbiWbqaQG3T7Uvg/oXHKrao6o3qupMVa1W1QZVPRVWP8ZMAB2w9mmYx2xV1TmqOllVK1R1lKp+FVa2QgC8mcX38pR9eXwWx2y1L83JUqG9J8XOLF3KZfRtzJXZyO/H0TX+NuRdG6+t2IanF21CbzxtMoqI8rS9rQvrmzoAAAvWNg3siyEioryEEWzMtS9PERHX44lIPYCjYAUJ87wHZuFCAFUAHrFH4Wbia/blQ1k8zzj7MqMaHBEZBKtRW2GNuU3oj/ekKOSb2YiVQGbj7dU78YXfzMc3Hnwbf313fWiPS0T+2mLJ/wTHeuI5lWgSEVFxyDvYUNUVAJ4BMBHAtzw3XwvrE/8HVLUNsJb0ich0e8O2i98oWRE5FMD1AFoBXOe5LSIidT7HXArgAgAL4Ak2ROQ4bwBgXz8FVrM4ADxpXD9aRPbyuX8drAWDVQCes0u6AGT/npQy956N3TOz8VbjDufvb67akeKeRBSGds+HBbs6Mv2MiYiIik1YDeKXw1qId6uInARgCaydFSfAKhW6yrjvOPv21bBOxk3PikgHgIWwGrxnADgN1u6Lc1XVu0+jBsBmEXkWwHL7umMAzAawAsA5PpmQxwA0ich8AGthvQdTAJxq//02VX3WuP90AHNF5HX7dW+xv4dPwNrzsRLApXm+JyXLXUY10D0byR9n78lKPlo6k5+yNnUEtv8QUUjMzAYA7OrsxsgGNokTEZWiUIINVV1h7624DtZJ+2kANsJq3L5WVTP9OPjPAD4PaypVNYANAO4FcL2qNvrcPwZritXRsE7+ASvIuBrATara6nPM1bAmWB0O4AwAZQA2A/gbgHtV9WnP/VcAuAdWI/qZsMbVtsPqy7gdwK32oj6XEN+TopbvBvFwMxvGgsEQMxutxolPMz9hJSq4ds/vL3/viIhKV2ijb1V1LYCLM7hfI5KjX7233QDghiyesxvAVzO9v33MLQBuyeL+awF8I5vn8Byb9j0pZZV5Noi7Mxv5BRs1FYXJbOzqTJ7oNLXzpIeo0Npj3jIqjrImIipVYTSI08eYmdnozGWDuBls5Llno6pADeJmGRU/YSUqvLYud3DB3zsiotLFYIPyUkyjb82lfqGWUTHYIOpX7TEGG0REuwsGG5SXYl3qF2qDeCx5otPe1ZtTUEVEmWvjNCoiot0Ggw3Ki7uMaoBH35YXZvStWUYF8FNWokJrZxkVEdFug8EG5aWymEbfVhSmZ6PVE2zwU1aiwmqLcRoVEdHugsEG5aUq7zKq0stscCIVUWF5MxvmRDgiIiotDDYoL9GyCMoi1iTj3riipze7gMNVRpVnZsNsEPeerOSqs7sXXZ7vicEGUWF5ezaY2SAiKl0MNihvrr6NLLMbsTBH35plVDmM4fXTGusbtDTxxIeooLzTqLhng4iodDHYoLy5xt9mWb7kzmyEV0bVGVLPhreECuCnrESFxswGEdHug8EG5S2f8bdhLvVzlVF1h/NJaItPrXhze1coj01E/vr0bDDYICIqWQw2KG/5BBsFG33bFVIZlU9mg2VURIXV7plG1RLrQW9cB+jVEBFRPhhsUN7Mxu5sd20UbvRtOJmNXSyjIup3fks5/bKMRERU/BhsUN6qynPLbPT0xtFjf1oZEaC8TPJ6Hd7Rt6r5fxLqd4LDaVREhdXm82EBg3wiotLEYIPyZmY2smkQ9469Fckv2IiWRVBRZv1IxzW3vR9enEZF1L9U1TezwYlURESlicEG5a0yx8yGuzk8nB9F83GyLeny4zeNis2qRIUT64n79mcws0FEVJoYbFDeXHs28shshMHVtxFKsOFXRsVpVESF4pfVALhFnIioVDHYoLxVmns2BjizUVMRdf4edNKSDb8yquaObsQ5GYcoY109cXzrj+/gvF+/hpVbW1Pet83ndw5gZoOIqFQx2KC85Tr6trM7/MxGlWv8bf7Bht80qrgCrSFNuyL6OHhh6RY8+f5GvLV6J373WmPK+wZ9SMBgg4ioNDHYoLzlOvo21lOIzEbYZVQBn7JyIhVRxjbv6nT+vqm5M8U9/SdRAeyVIiIqVQw2KG+5jr51ZTby3B6eUB1yZqM1oE68FMfffrCuGX96Y41vaRhRIZk/c0EBfIJ3oV8CMxtERKUpmv4uRKm5Rt/25JbZMEux8mE2iIfRs2GeGA2rrcD2Nqs5vNROfLa1xnDeXa8h1hPHko27cO1Z+w30S6KPEfP3KF2wG5TZKLXfOSIisjCzQXlz9Wx055bZqCpAZiPs0bfjh1Q7f2/qKK2JVIs37HKyTu+ubRrYF0MfO22uzEbqoKHdCDYaqpKfh/n1TxERUfFjsEF5M/dsdA50ZqM83J4N81PY8UNrnL+XWhlVNmUsRGEzf/7SZjaMMqqxg5MBPjMbRESlicEG5a3KtUE888xGrBCZjRDLqHrj6joxGlfCJz7mp8npPlkmCpsZ4KbLUJi9VqMHVSWPK7HfOSIisjDYoLzlvEG8wD0b+ZZRmYFGXWUUQ2oqnK9LL9hgZiMTqoq/vL0Of5i3Gt29mf8sU2pmGVVXTzxlb5fZszGGwQYRUcljgzjlLecG8QJkNmrKzcxGfifVZrBRXxXF4Jpy5+tS2yJuBhixnji6euKoCCnA2508t2QLvvd/7wEAIiL4wmF7DvAr2j14S6daO3tQWef/O29mJEc3uLOJqgoRKcyLJCKiguDZBuUt9wbx8PdsmJmNjq78Ppk2y43qKqMYXG0GG6X1Kas3m8Hxt/7eWLXd+ftbq3cM4CvZvXh/3lJl18wsyNDacue/Lz1xDaUPi4iI+heDDcqbmZXIbvRtgTeId+d3Qm2eENVXRTHICDZKr4yqO+XXZFmzo935+/qdHQP4SnYvfTIbKYJdM7NRU1Hav3dERMRgg0LgymxktdSvwBvE82wQbzWCjbqqcgyqKd2Tnmw+Wf44W709GWysY7ARmlbPz9uuFMGumdmorSxjsEFEVOIYbFDezGAjm6bsQmQ2whx9a54QWT0byQbxdGVUqhrKno+weIMLBht9qaors7FpVyd62CSet57eeJ/fRW/wYfJmNhqMYGNXB39uiYhKDYMNypu7jKp4ejbyHX1rnpA3ZFFGpar4+oNvY98f/xN3vbgir9cQFpZRpbe9rcv1M9MbV2xuiQ3gK9o9tPn8Hqbs2ehiZoOIaHfCYIPy5hp9m02DeE9xbxD3jr6trShDNGJNwuno7g18/Mbt7Xh28WbEFbj5uY+Kohm7JYua+Y8rs4QqgX0b+fP7WUvZsxHzZDaMLeIMNoiISg+DDcpb7qNvw9+zUVORPDHJP7NhllGVQ0Rc42+D5v6bJ6id3XE8s2hTXq8jDCyjSm/tDp9go6nvdZQdv5KpVJk1V2bD0yDOXRtERKWHwQblzd2zkVtmozK0DeLJ15Jvz4Z3GhUAV/140KesG5rcn4b/bcGGvF5HGFhGlR4zG4Xhl8XwZtpMrp4NllEREZW80IINERkvIveJyAYRiYlIo4jcLCJDsngMEZFLRGSeiLSISLuIvCsiV4qI79moiNSJyE9EZImIdIpIk4g8LyKnBdz/WBF5UEQWish2+5hVIvK4iJyU4eucJiJtIqIi8oeA+zTat/v9GfiPukOU8+jbAmQ2XKNvw5xGVWkFG65dG0HBRrP7BPWVZVuxpaUzr9eSj+7eeJ8gMNXJ3sfV6h1tfa5b38RgI1++wUaGezZqvQ3iDJKJiEpOKBvERWQKgNcAjATwGIClAGYDmAPgVBE5SlW3p3iIhN8DuBDAFgAPA2gDcDKAWwAcKyLnq6oazzsYwMsA9gOwCMDdAGoBnAngSRGZo6q3ep7jRPvPfAAv2M+xp33MGSLyU1X9UYrvNQrgQQCZfITfDOBmn+tbMzi2ZOQ8+rYAPRtmGVX+06jMzIZ1wpPJRKqNTe7AIq7AE+9txCVHT8rr9eTKv4yFwYaXXxkVx9/mz+/nL2gaVW9cnf+GiFiDIzLJJhIRUfEKJdgAcCesQONKVb0tcaWI3ATgOwB+BuCyVA8gImfDCjRWAZitqtvs68sBPALgMwAuAvA747BrYAUajwL4nKr22MeMAPAGgF+KyFOqusw45npVvcbn+ccBeAfAD0XkTlXdGPBSfwhgJoB/hxUEpdLk91y7m1yDjUJkNqrDzGzEkic2iSbVTEo6vJkNAHhswfoBCzb8AotUo0c/rnzLqJjZyFubb2bD/3en3ejXqCkvg4iwZ4OIqMTlfYYnIpMBnAKgEcAdnpuvhpU5uFBEatM81Ln25Y2JQAMAVLUbQCLTcEXAMT9OBBr2MVsB3AigHJ4gR1V961lUdT2s7EwEwGS/+4jIIfZr+QmA99N8Px8b0bKIM6WpN67oznA3QawAmQ1zhG6sJ47euKa4d2otrqV+fYONpvYu3+O8PRsA8N66ZqzcOjAJrZZY3xM09my4dXT1Yos95lYkef2Gpg4YyVTKgV/JXtA0Kne/ht0nVcU9G0REpSyMj5NPtC+fUVXXWaaqtgB4FUANgMPTPM5o+3Klz22J62bZpVPZHJNpH8ZIAIcBiAH40Of2agAPAFgA4PpMHhNApYh8SUR+KCJzROSEoN6TUpdLdsO9ZyOct0VEQht/2+JbRpU6s6Gq2GCUUR0+eajz94FqFPfLbLCMym3tzmRWY8+hNc5AgM7uOLa3+QeVlJlsyvjc/RrW7zEbxImISlsYwcbe9uVHAbcnSpimpXmcRDbDr9bEzDRMz/KY6T63QUQOEZFrROSnIvI7WH0mIwF8x8ysGK63H/MiM4uSxmhY/R0/g9W78QKAZSJyXIbHQ0Te9vsT9H0NFHOaVCzDE3z3BvHwBqOFtdjP/PS13jez0ffEp7mj2+kVqakow5ePmOjc9rd31w/Ip+S+ZVRsEHcxS6j2HFqDcYOrna85kSo/5ijbhKBgw7s9HAAGpQnwiYiouIVxhjfIvmwOuD1x/eA0j/OEffldEXE+DrYbsq817jfE55hrzIyBiAwD8F37y0o7K+F1CKwyr6tg9YJEAVysqr/23tGeUnUFrHKtxWm+j4T7YWVVRsNqWt8fVgP7RABPiciBGT5OSSiWzAYQzmI/VXWVGjnTqNKc+JhZjbGDq3Hi9JGot49ds6Md765tyun15KPVt4yKwYZpzQ53sDF+iBFssG8jL/6ZNf+gwZXZqLR+j82lfpxGRURUevpjz0aiAjrdR7p/AvAUgCkAFovIPSJyM6yypdOQzJCYZ48/BrAawPkAFtijdu8BsBjWtKh2n2OsF6N6l6oKgGoA+8IKDh4QkbtcL94q27of1vSqG9N8D+bjX6uqL6jqZlVtV9WFqnoZgJvs57wmw8c52O8PrExM0ajK4QS/PzIbuU6kivXE0d1r/chWlEWc729wtTGNyjfYSJ6YjhlUharyMnxq/9HOdX97d31Orycf2ZzsfVyt2Z4ce+vNbKzbycV++QjaIO6X5fPLbNRVRmG3hKG9qzfjnjAiIioOYZzhJTIXgwJub/Dcz5fd73EmgO8D2ARrMtUlANYBOBpAYnTuFuOYTQAOBXArrOzB5QDOgpXxOBnWSX2zqgYWXatqp6ouUdU5sDIP3xCR84y73ARgOICvqGp+440siWDm2BAeq2hkm9no7k02b5dFBOVl4QUbNSGUUfkt9AM8S/18GsQ3GpOoxg6yTljPPmicc90T72/s95OloDIqNj4nrTYyGxOG1WDcEJZRhcVvGlVc/X83XdvD7cyGiLh3bbCUioiopIRxhpdopg7qyZhqXwb1dDhUtUdVb1TVmaparaoNqnoqrEzFTAAdsPZpmMdsVdU5qjpZVStUdZSqfhVWH4cAeDOL7+Up+/J447pZsIKWpeZiPgBz7du/aF+3IMPnSARL6aZzlZRsgw1XCVWIWQ0gnMV+rhIqI9hIW0bV7C6jAoDDJw3D6IYqAMCOti68ssyvJahw/IKNuAJteY4G3p24y6hqMW5wjfM1y6jyEzRm2S/j0R7rm9kA2CRORFTKwtizkTjpPkVEIuZEKhGpB3AUrCBhXh7PcSGAKgC/t0fhZuJr9uVDWTxP4iNo8/+CjwJ4y+e+Y2CVd60A8C8AazJ8jiPsS78JWiXLPCHfGTAS1uQqoQqxXwNwZzY6unPrTQjKbKTbIO4qoxpsBRiRiODMmWNxz0vWP/lf312PE6aPzOl15SKoZKq1s8fpRfk4640r1u1I/rvtOawGXUb2iYv98hO0rb6lsxuj7CA8wZXZMH6PGWwQEZWuvM80VHWFiDwDa9fGtwDcZtx8LaxP8O9W1TbAWdI3BUC3qq4wH0tEGlR1l+e6Q2FNgmoFcJ3ntgiAGlVt9Vx/KYALYPV7POS57TgAL3vH9Npb0K+yv3zS+P5cz2nc/3hYwcY8Vb3Uc9sMABtVdYfn+gkAbre//IPf45aqMYOSZSfeDdp+CpnZcC/2y61kyTWJqjJ5ouM96YnHFZFIcjGD+b2PNd6Ts2eOc4KNZxZvQmus/070g5rBWzq7MXpQle9tHyebd3U6wcWw2grUVUbd06iY2ciLWUbVUBXFLvvn0e/n0m/PhnWcUUbF4QZERCUlrLOdy2EtxLvVnty0BNbOihNglU9dZdx3nH37aliTmUzPikgHgIUAWgDMgHVCHwNwrqp6swE1ADaLyLMAltvXHQNgNqyMwzk+mZDHADSJyHwAa2G9B1MAnGr//TZVfTbbN8DjfAD/KSJzYW1Eb7Gf43RYGZp/APhlns9RVMYaJ2d+S+28CpnZMIONdp+xm5kIKqOKlkVQVxm1ex6sT23NAMTcHj52cPJEfp8x9Zg2qg4fbW5FZ3cczyzahHNnjc/ptWUraMxt0CfOHzfm2Ns9hlrlU8PrKlAZjSDWE0dLZw92dXa7Tngpc+bP39jB1di1qQWAf7Dht2cDYGaDiKiUhfKRsp2hOATA72AFGd+DdXJ9K4AjVHV78NEufwZQD+BLsEbX7g/gXgAzVPVpn/vHYE2x2gfAN+0/1bBG2s5U1UafY66G1fdxOKwg6ZsADgTwNwCnquqVGb7WVOYC+CusvpEv2N/LcQBegTVm99OpmtZL0VjjE3LzhDuImdkIcxIV4J5Glevo210BZVSA58TH2LXRG1dsMno2zGyPiOCsmclG8f5c8GcGThVGIz7H31rW7EhOopowzAo2RIS7NkJi9myYmTTfng2faVSAZzADgw0iopISWh2Hqq4FcHEG92tEchyu97YbANyQxXN2A/hqpve3j7kFwC3ZHBPwOP9C8PfxIoAX832OUpJtZqOzu38yG7mOvjVPkLyfaA+uKXdKa5o6urAnrBPUba0x9NgTtobUlLuCHgA4a+ZY3PC0NU/h1eXbEOvpRWW08AvlzaBizOAq55N8jr+1mM3hE4YmG8PHDanGym1WILJ+Zwf2GdPQ51hKLdbT65SoRSOC4XWVzm1+P39+ezYAoKHa2LXBYIOIqKT0x54N+hhwBxvpezZiPYXr2Qh79K23tyKopMMMssz3I2H8kBonA9Qb14x6W8LgCjbMT5aZ2QDgX0YFgH0bIWgzpkvVVUVdWULfno3u9NOoGGwQEZUWBhsUCrM/YVNzJ+Lx1DscYkZmI8zt4QBQFcJSP/NTV28ZlTn+tqndDDb8S6hM44ckT2bX9tOyOPN7MZvWWUZlcWU2hiUnUjPYyJ8Z0NZWRFFfmSbYMDIbNQE9G9wiTkRUWhhsUChqKqLOSXhXbxzb2mIp729mNsLu2agJYc+GaxqVp4xqUMAW8Y0BzeGm8cayuP4Yqaqqru9ljPG62CBuce/YSAaD44eyZyNf7t+jqOt3ybdBPKhno4o9G0REpYrBBoUmm/G3nQXMbJi9Erkv9TPKqFI0iO/q8M9s+JVRAcB442R2XT9kNtq7epFIMlWVRzCkJhkosWfDOnFNZKcqoxGMrE/2FJiL/dYxs5ETM9ioq4y6fpdaY31//tp9NogDnEZFRFTKGGxQaMYZn5qnaxIvZGbDDF7ac55GlWkZVXKomGuhX8D+iv7ObLiXE5anrZkvlN64pi2tGwhrPVkNc2fKuCHMbOTLDChqKzPo2QjYIN7gCvCZkSMiKiUMNig0ZmYjXY17ITMb5klKZwhlVA3eYKPav2fDXUYV1LPR38GGETRVustY+qtBfO2OdhzzPy/gyOtfcJ3cFwOzOdwsoQKAUfWVKLODj22tsZzHKH+ctXoaxM1hC36jb9uY2SAi2u0w2KDQmCfYG5vTlVEVcM9GCKNv3dOovD0bRrBhnPisz6CMag+zQbwfTrxbPDXz5slei08ZSyE8+s56bGjuxKZdnXho/pp+ec5MrTZ2bOw5zB1sRMsiGN2QzFCxSTx7ZkDrDXb9NoEHZTYYbBARlS4GGxSasVmUUW3elWwgH1JbkeKe2asOZfRtcBnVoJq+Jz6xnl5sa7W+p4hYn4r7GT2oColKnS0thf+0PFUZVX9lNjbtSgZhy7e09stzZmptwI6NBJZS5SdVGVWrp2dIVV2ZDXMalbv8qrsoS/KIiMgfgw0KjWvXRprMxsptyZPOycNrU9wze2ZmI9eT+dYUG8QHG9OoEhvENzcng6eR9VWIlvn/apWXRVzlZpksQMyHGTTVZVAzXwiJIAwAVmwtrmDDVUY1rG+wMZ7jb/PiKqNK8/MX64k7wwwqohGUG79D5WUR1NrBR1yB1i72bRARlQoGGxSabLaIr9qWLF+ZNCLkYCPPzEZvXJ0RnCLWfgCTmdlo6rAaxNc3pR97m9CffRveoCldGUshmMHGmh3truEAA8099rbvzyEzG/nx/vyl6tlwbQ+v6NvHxcV+RESlicEGhWZUfaVTIrS1JRZ4UtnVE3eVr0wcFm6wUZPnUj/zBKmuIuqaUAS4G8QTZVRmc/iYgH6NBHOxX6GDjZRlVDn2bLz40VYcd8Nc/ODP70M1fTmLGWz0xhVrthdHk3hXT9wJikXcQWACF/vlx1tGVVsRhdi/Tu1dvejpTQ6KaA/YsZHQwL4NIqKSxGCDQhMti2CU0VBrlhaZ1uxod8olxg2uDn+DeJ5L/czGaW8JFWAFM+Vl1hlTZ3ccnd29rob4sQFjbxPMk9pCbxF3lVFVRVFdXuZMWOrsjqPbONnL1K+e/Qirt7fj4bfWYvHGXWnvv62ly/V1sZRSrW/qcH4ORzdU+f4cMrORnzZPGVUkIqgzAgnz9qBJVAkMNoiIShODDQqVuV8i6JNgVwlVyP0aQN/MRiafvptSLfQDABHpMx3HXUaVLrPRf2VUZqlUQ1UUIuIuZcmylEpVXU3e6V5/W6ynT3apWJrEgzaHm5jZyI85DS3xu2QG8OY+m7aASVQJg7hrg4ioJDHYoFC5x98GBRvJk81CBBvlZRFE7U/ve+OK7t7cgw2zx8HkDTY2uhb6pQ429ujHLeKtsb6N7q7xt1kGG9tau1yPubXFP3uVvH/f21dsbfO5Z/9bs90YexsQbJg/z5t2dbrKfii9Vs+AAgCeLeLJnyUzC+mb2ahizwYRUSlisEGhGpdBk3ihMxuAu0k821Kq1jRlVAAwuCY5kaqpvdtdRlVEDeLuaVTWyVrQJ8uZWL3dHShsySnYKI7MhjmJaoLPJCrAKskbXmeNMe6Nq2uML6XnLaMC3AG8GeyaZVTV5akzGyyjIiIqHQw2KFRmGVXQ+NuVWws3iSohn8V+7oV+/sGGa7Ffe1dWZVSjG6qcvomtBd610eIzwtf8hNhvi3MqZqAIAFtbUp98b/X0awDAii2tWZe2FYJZRrVHQGYDYN9GPsyfLyezUek/pKA9bc9G7kEyERENHAYbFKpMxt+aJ6xh79hIcI+/ze6EelcGZVTmRKoNTR3OSX1FNIJhaZYURssirqCskNkN3zKqPHZtNHozG7uyz2y0dfUWRYbADDYmpJiIZmai2LeRHe+eF8C7oM/IbGTRs8HMBhFR6WCwQaFKF2y0xnqc0pvyMnGVXYUpn8xGq6ep2o+5a2PJxhbn72MGVUFE/A5xcZdSFa5vwy+zkc/420bP2NpcyqgAYMWWge3bUFV3sJEis+Fa7MfMRsasjeBmH0bqYMOV2eCeDSKi3QaDDQqVq0G8qe+n143b3E25QZu285VPz4bfp7Fe5hbxJZuS41/HpmkOT9ijn3ZtmN9LIkuTT4N4Y58yqsyDDXNdyUD3bWxr7XL2OtRXRjG4xj+DBbjLqArdY7M76eyOo9eeLVwZjaAiav2uB/ZsmJkNn987s/yPmQ0iotLBYINCNaSmHJX2SUVLrKdPbfVKs4RqRF3BXkc+i/38sgFeg4z68Q83GZmNNM3hCf212M8/s+F/speOqrqaqgErmIjHg/svzB0bM8YOcv4+0MHGmh1G0DusJmU2iuNvc2PuqzED3Ix6NvwyGzUMNoiIShGDDQqViLs0ypvdWLW18P0agKeMypPZ6OzuxWML1mPRhmbfY919DgE9G8Y0qlhPchxqpmVh/bHYr6sn7ry2sog470lQGUs63rG3ANATV+xo79sEnrC9LZnZOGzSUOfvA71rI5MdGwnj2LORE9ckKuNnLrBnoyt1ZsNVRpVlRo6IiAYOgw0KXaq+jULv2EioNhpMvZmNX/zzQ8z50wKcc8drvv0S3q3bfgYFlN2k27GR0B/jb73lYIlP790ne5l/QuxtDk9I1SS+rTUZiBw2eZjz94HObJgZmj0Dxt4meDMbqTI5lNQaMNUtaKlkeyx1ZoNlVEREpYnBBoXOPf7WG2wUfscGAFSXJ3+0zcxGV08c//f2WuvvvXG8smxbn2N3ZVRGFRBsZFhGZY5aXV+gzIbfJCrv37MZfevt10jYkmL87Tajp+OgPQejwu7R2bwrllWgEyZVxdylW5yvJ6WYRAVY2a3EoICunji2taXuUykWAx0UmWVUtZXmz59/hsKV2eA0KiKi3QaDDQpdUGZDVV09G4XasQG4y6jajZOYV5dvc5VuLN64C17uaVTpR9+aMi2jGtVQ5Ww539balXUTeyaC9oUklvt575NOUGYjqEm8s7sXLXYwU14mGFZbgYnDk0HWygHaJP7Omia8t84qoauIRnDyvqPSHjNuiBkcFncpVU9vHBf+dj4O/umzmPvhlvQHFIhZRlVfGRTsZr5no6o84gSrXT3xgu6nISKi8DDYoNCZG7Q3GD0b29u6nJPbusooRtibmQshqIzqyQ82uu63aEPfYCOosdVk9myYzKxOKmURcQVl65uyz24s2bgLy7e0BN5uNuebQVPuZVTJ1zi8Lvn9B42/NSdRDauthIhgijEUYKBKqe57dZXz97NnjnU2hKdSSk3i81ftwMvLtmFnezfunLt8wF6HGUhk1LORZs+GiLgX+zG7QURUEhhsUOiCMhveEqpM9lHkyq9BvKsnjmcWbXLdb8nGXX3KTTKZRuW3f6O+MhrYUO7H1SS+I7sT2OcWb8anbnkZJ9/0Et5evdP3Pq0B30euDeJmGdWhE5PN3kGZDbNfY3i9FZyYwcZANImvb+rAPxcmfwYuPmpSRseNL6Et4mbz+9KNLQO2rd38+Qsqowrcs+GT2QCABleTOIMNIqJSwGCDQmc2SZs9G+YkqkL2awD+o29fW7GtzxSb9q5eV3mQqrobWwOCjWhZxFUaAriDrEzks9jv3ldWOn83+w9MLQHfR30OZVSq6go2ZhuTpYJ6Nsx+jUT2YMrI5L97LpmNxxasx4W/nY8Xlm7O+lgAeOD1Rmf3w5FThmGfMQ0ZHVdKmQ0zwG+J9WBj88Bsa28NKKNyj741gw1jAaBPZgNgkzgRUSlisEGhM8uoNjV3OpmDlf3UHA4AVT7Bxj88JVQJZilVZ3ccPfbrrYhGUBn1/4QV6DuRKtPm8IRcF/ttbO7A/FU7nK+9TfgJ7oV++TWIb2vtchp46yujmD46eZIeNI3KLKNKBBt7jah3rluRZc/GttYYvv9/7+HlZdvwg798kPUn9u1dPfjf+Wucry/JMKsBeMbfFnlmY4Nn3PSHm4NL7QqpNaAc0VvGl/h3bHc1iPv/3rFJnIio9DDYoNDVVEQxxD4R7+5V56RzpfFJ9uQCNocDfcuounvjeGZx8tPwwycnP5k3m8RbXH0O/p+uJngnUmU69jZh/NDcxt/+/b0NMM+zveOFE4L2hdR5go1MTtrN7M+E4TUY2ZDsc9jamnmwYf67r97ehu7eeJ/jgjy1cBO6e63XurUlhu1twfs9/PzlnfVOZmvCsBqcOH1kxse6SgMHKFOQKe/Pw0ebBijYCCijqoxGUF5mlVB296qzC6bN+Hn169kAPLs2Orhrg4ioFDDYoIJwl1JZJ2f9NfYW8JRRdfXi9RXb0dRuBRJjB1Xhi4dNcG43MxstGSz0SxjsyWyMyzKz4d4innkZ1WMLNri+9n6SnRA0jaq8LIIqezRwb1xdnygHMf/tJg6rxYj6ZLCxZVfMN2Bx9WzYDeW1lVGnib67V139Bek88Z77+w4axesnHlfcbzSGX3zkREQimfcMuYceFHlmw5PpGrjMhv9SPxHp07fR05tcQCkC5+fTy2wQ92Y2trfGcMPTS/Hc4txK7IiIqDAYbFBBeJvEe+PqWqQ2scDBhmv0bXevq4Tq1P3GYMbYZBnQ4g3Nzsly0Am6n8HV7olUWWc2XFvEMzuBXb6lpc8ELbNUzbTLNcLX/b2YJ3uZlFKt3u4ONuoro84JYUd3r+9jmBkPMzhxTaTKsEl8865OvNG4w3XdqiyCjReXbXVG7dZXRnHeIXtkfCwADK+tdD6Nb+7odn0KX0zice3To/FREZRRefubvH0b7d3ufo2g4RFBZVSd3b244DfzcMfcFfjmQ29jbRZBLBERFRaDDSoI7yfBG5o60GWXzAyvqwzcXxGWaiOz0dLZjaeNKVSnHzAaE4fVOluKt7V2OROVgvoc/DR4y6iyzGyMrK9yTmB3tHVldAL7uCerAVjLCf0WzQWVUQHuk79Mxt82bnMHiiKCkfXJ79dv/K1fgzgA7DXSHH+bWcDw5Psb4U2eBO398HPfK8msxucO3SNtIOkViYgrmNwY0Ccz0La3daGrx12atmxzq9MU35/Mn7/aSm+w6/75a4+l79cAvGVUyZ/bX/zzQ3y02Qpcu3s1cEIbERH1PwYbVBDuzEanqzl8coGzGoA7s/HBumbstEuoRjdU4aA9hiASEdckokS2IGhcrJ++ZVTZZTbKIpLVlCNVxWNGKZH54a9fKZUZRHhPrrMdf2ue2E+yF/ON9JRSefn1bADAlBHZT6R64v2+QZYZAKXy0eYWvGxvio8IcNGREzM6zsvcoRJUujbQ/Eq8Yj3xrMrVwhJURgV4MhudPWjrCg5MTH7TqF5ettW1OwUYuGwOERH1xWCDCsJbRrXKOKksdL8G4P50tMf4VPdT+492avX3NUup7CZxdxlVmp4NT2ZjdIYL/UzZ9G0sWNvklKLVV0Zx9F7Dnds2+pxkptoXUpdFsOEdezthmPXvl65J3OzZGGYsAcx218a6ne14Z01Tn+szLaO6/9VG5++n7DsaewytCb5zCuMC9scUk6DX9eEANIm3dgaXUZmZtl2dPbllNjq7sbOtC9//v/f63C+R5SAiooHHYIMKYqxx4r2xucPdHF7gSVQAUFXuf8Jy+v5jnL+bfRuLNjQD8DaIZz6NanhdRcoxuUHcuzZSn8CajeGn7meVgiX4ZUXcWRpvGVXmuza2tsZcY2+H1VqBg7kBfssu9yf9XT1x55PniABDjI3rU1xlVK1pp2E9+X6y3+aA8YOcvzdub0t77M62Ljz6zjrn60uOznzcrZdZJlesE6mCXtdAfNKfaRlVa8yT2QiYRAW4SxebO7rxw79+gM12Vq0ymvzf2bItzGwQERWL0IINERkvIveJyAYRiYlIo4jcLCJDsngMEZFLRGSeiLSISLuIvCsiV4qI75mciNSJyE9EZImIdIpIk4g8LyKnBdz/WBF5UEQWish2+5hVIvK4iJyU4eucJiJtIqIi8ocU98v7PSlVY13lQZ39umMDcPdsJIxqqMSsPZNv/YyxyRPXRBlVNqNvzTKqbBf6Jbi3iAdnNnp643jCOOk+a+Y413P6LW5L1X/iHn+bumfDbOyfMLzGad4d2ZA8+fZuEd9u9JAMra1EmTH5aWR9pfNJd0tnT+Do3ATz+77w8AnO99Le1Ru4vTzhj2+scaYc7TeuAYdOzP1Xz5utK0bm6zLLFQdiIlVbijKqPj0bRrBRE7A9HHAH+G+v3omnjG3wN312JhI/Zmt2tKMjgylrRERUeKEEGyIyBcDbAC4G8AaAXwFYCWAOgNdFZFiGD/V7AL8FMAnAwwB+A6ACwC0AHhbPiBIRGQzgdQD/D0AvgLsB/BnA/gCeFJErfZ7jRPvPRwAesl/rawBOAPCciPwkzfcaBfAggJQLAkJ8T0rSyPpK53/821pjrjKO/ujZ8CvF+NR+Y1zjTqeOqkPU/nr19na0dHYHbt32s//4wc5J9GHGRu1sjM9wsd/rK7c7PRAj6itxxJRhacexpiqjyqZnwzv2NsE1/tZz0r+tpe/Y2wQRwWQzu7EluByqcVsbPlhvZZ0qyiI4ZcZoV7CarpTK3K1yyVGTAqccZWLsADaIr9rWltHmcvPn4Pi9k3tE+nvXRjyu7sxGRZqejVj67eGAO9hI7FwBgAtm74nTDxjjlPip5rahnoiIwhdWZuNOACMBXKmqZ6vqf6rqibBOsPcG8LN0DyAiZwO4EMAqADNU9VJVnQNgJoC/AfgMgIs8h10DYD8AjwKYqapzVPVSADMANAL4pYhM9RxzvaqOU9VzVfVKVf0vVf0igOkAtgD4oYiMQbAf2q/pqjTfUt7vSSmLlkUwuqHvtKKIAHsOy61mPhtVPiVNp+3v/metjJa5JiMt2diSsvTIa9zgavzp64fjp2fvhzknT8vpde6R4WI/s4TqjAPGoiwiKT9pj8cVrV1m/0lwzXy6YGP1dv+slKtBvMWdWdkWMPY2wWwSX57ipNBsDD922ggMqi53TiiB1BOpeuOKDzclxwQfN21E4H0z4R16kM7q7W2456UVrvcvFy8s3YwTfvkvHPuLuVjsGXvsZZZRHTst2dOzalsbYj3990m/WRZVU1HmymwBnp+/WI87s5GiZ8Nvit2k4bX40af3AQBMNX6f2SRORFQc8g42RGQygFNgndzf4bn5agBtAC4UkXQfZ59rX96oqtsSV6pqN4Af2V9eEXDMj1W1xzhmK4AbAZQDuMw8QFV9zxJUdT2sDEcEwGS/+4jIIfZr+QmA94O+kRDfk5I2xqe0aPyQmpx6G7IViYhrMdjI+kocMqFvCc2+nr6Nllhw6ZGfQycOxZcOn5D1KNWETBrEO7t78U+jXOSsmWMBpN5q3dbV44yKrakoQ7TM/avuHn2bOtgwpz6ZJ/rm6FtvOdPWgElUCZnu2jBLqM440AoWJxnB6qoUE6lWb29DZ7eVgBxRX4lhPq8jG2M8maRU/SKqiq/c/yb++x9L8YXfzEdnd+4n+ve90gjACp7+aYxw9mMGnVNH1TvBbE9cs9pLki9XCZXP74Z3QIG5WDLVNKr6qqhrCltZRPCrz810No5PG1Xv3MYmcSKi4hBGZuNE+/IZVXWVFqlqC4BXAdQAODzN44y2L1f63Ja4bpZdOpXNMZn2YYwEcBiAGIAPfW6vBvAAgAUArk/zcGG9JyXNr4+hP/o1Eszxt5/ab7Tvxmizb2Pxhl1ZLfULw4i6SlTYgcDO9m7f5XgvLN3iXD9xWI3TJD3KKFXb2hJzfXKd7vvw1syn4t78njzRT1lG5Qo23GVUgHfXhv9J4bLNLVhql/9URiM4aZ9RANwLIVNtEV+yMfnJtjnmOFcNVeVOkBbriTvjlP1saYk579v6pg48+s76nJ6zqb0Lr6/c7nydqjQo1pPsYYmI9fOxt3Hy3Z8Tqcw+IL9yxIY+PRuZTaOKRMQ1BW7OSVMxc4/BztdTR5mTzpjZICIqBmEEG3vblx8F3L7MvkxXZ5LIZviNizEzDdOzPGa6z20QkUNE5BoR+amI/A7AUlhlT98xMyuG6+3HvMjMogQI6z2BiLzt9wcB31cxGeuz5K4/gw2zVONT+/tXxrknUu3y9DkUdvEgYJ08jXNNpOr7Sf1jC5InqmfOHOf0HUTLIhhllKptMrIbrWmmatV5pgEFUdU+28MThtVWOOUxTe3drmDH3bOROrOxMmCx39+NrMaJ00c6QZMr2EhRorTUKKHaZ3R94P2y4c1uBPFmEe5+aUVOi/WeW7LFdVyqLJD57z+qoQrRsojnk/7cTr5zed2t6TIbng3ibSkmV3ldeMREAMDJ+4zC5cdPcd02dSQzG0RExSaMYCPx0XBzwO2J6weneZwn7MvviojTbWs3ZF9r3G+IzzHXmNOq7Obr79pfVtpZCa9DYJU0XQWrFyQK4GJV/bX3jvaUqitglWstTvN9AOG9JyXNbKhNmNwPY28TvnzEBJRFBCfvMwqzJ/o3cJufeC/b0oIdbcmT5HTTqMLiGn+7w30C29zRjblLtzpfn3ngWNftQX0E7klUfYOmTHs2vGNvh9YmsxSRiLiyFmYpVdBCv4QJw2qc5vz1TR19tqerqqtf4wzj+57k6dmIB5wML9loBBshZDaAzCdSeYON1dvb8dTCjQH3DmaWzyUeN+jk3/z3T7zOvUfnl9n4y9vrsN/VT+PS378V+D77aU2bWXP//JmZjeqAsdUJ3/3ENCy57lT85ssH9ykPnDyi1sn2rd3JiVRERMWgP/ZsJGpX0v2f6k8AngIwBcBiEblHRG6GVbZ0GpLZAPP/Hj8GsBrA+QAW2GNl7wGwGNa0qHafY6wXo3qXqgqAagD7ArgfwAMicpfrxVtlW/cDmA+rDyQMmb4nUNWD/f7AysQUtYEuo7r0mMlYdO0nce9Fh/iWUAHWdJtEXXt3r7o2LaebRhWWVH0bf31nHbp6k6NbzfIjwLvVOnnyuyvNJnRXGVWKzIY59nbi8No+05zMvo0tQcGGT4N4eVnENSjAe3K+ZGOLk/GoqSjDCcZkpSG1Fc5Uos7uODa3+Ddrh11GBQBjBmUWbPiVd9314oq0e0FMbbEevLRsq+u6WE888HnN6xO/e2ZmI5fxt3e/tAId3b14bslmzFu1Pf0BNlcZVbrMRqc3s5G+p6u6osx3slhVeZmTfVPNbGkkEREVVhjBRuJT+kEBtzd47ufL7m04E8D3AWyCNZnqEgDrABwNIPF/ui3GMZsAHArgVgC1AC4HcBasjMfJsAKJZlVNflzd93k7VXWJPfnqbgDfEJHzjLvcBGA4gK+oaqYfk4XynpS6MT4btfsz2ACCl/uZZozx/2fqjzIqwH+xn6rijrnLce0TyUTa2TPH9Tk2aKt1a7pgo9JdMx9klWtzeN8pYuZEquDMRt+eDQDYa0Rw34aZ1Th5n1F99qZMTDP+trmj2xkVW1EWCS2jNs4oo/LbbZKw0uc1LVy/C68s96vQ9PevD7eiq6fvhO2g6V2uYMP+3Zs8otYpdVu7o28GKZV4XNFoBJuvLMv8tacro+q7Z8Ps2cgvyDf7NjiRioho4IURbCSaqYP6DxKjZ4P6Fxyq2qOqN6rqTFWtVtUGVT0VVqZiJoAOAIs8x2y1R95OVtUKVR2lql+F1cchAN7M4nt5yr483rhuFqygZam9xE9FRAHMtW//on3dAuOY0N6TUjbOk9moiEZ8S6sGmjmRKkEEqMkgUAmDN9hojfXgm394Bzc8/aEzUWrc4Gqcf/AefY4Nmkjl6j2pzL2MqjHNMsagJvFtrcn4fkTAFChzk/gPH/0AR/z8eRx3w1x84qYX8fvXGp3bPn1A334bcyJVo89EKrNkaK+RdSgvCyeJa2Y2Uu29MN+3WXsOdv5+14srMn4uc/JU1MjMBfVtmP/+iZ+LymiZ699tWRaf9G/a1ekKdl7NIlBq7UzdIO7NrLk2iGeQ2UjF1afCJvGS1tndi7cad+Q1zY2IBl4YdSKJk+5TRCRiTl8SkXoAR8EKEubl8RwXAqgC8Ht7FG4mvmZfPpTF8yQ+OjbPvh4F8JbPfcfAKu9aAeBfANYYt/XHe1L0BteUo6o84owfnTSsNrCcaSDN8Ak26iqj/fZazTKqhRuacfYdr7rKP2ZPHIo7vjgLg2r6Bg1BZVQtaU72XA3iKYINVxnVsL7Bhiuzscs62e3pjWNnuxVsiMDV52EydyK0dfU6vSGm+soojtu7736MdE3iZr/G9DHhNIcDSLu1HbAaqs337adn748zbn8FvXHFq8u34721TTjQmKDkp7O7Fy8sSS4kPHfWODzy1joAwIqAhnq/MioA2HtUvfPz9NGmFtf0plTM7wEA3l/fjKb2Lgyu8f/3NJlDBzJpEDd/BvPPbCT/vZexSbxkqSou+8Pb+NeHW3Hg+EH427eOymspJxENnLw/7lPVFQCeATARwLc8N18Lq7zpAVVtAwARKReR6faGbRcR6XPWJyKHwpoE1QrgOs9tERGp8znmUgAXwOr3eMhz23Ei0uf7tl9PYlHfk8b3d529YND1B8AN9l3m2dddZxyT1XuyuxJxL57r7xKqTPllNvyWhxXKHp7MhhlofOXIiXjoa4f5LsYDvCe/RhlVmmlUtRVlTiNtR3cvunv7lusAnu3hw/uWUY3wWdy4o63LycgMqano08Sb8Kn9xuDA8UGVhpavHjPJdy/LpDTjb81JVPuG1K8BIO3W9sT1iT6b4XWV2Hdsgys7k0l247UV25zga8KwGtdCysDMhvF6zCA0176NtTvcwYYq8NqKzPo2zDIqv+lS0bKI0wiu6i67S7VBPBPTWEa1W3hl+Tb860OrZ+m9dc39uieGiMIVVgfs5bAW4t1qT25aAmtnxQmwSoXMbdvj7NtXwzoZNz0rIh0AFgJogbUJ/DRYuy/OVVXvPo0aAJtF5FkAy+3rjgEwG1bG4RyfTMhjAJpEZD6AtbDegykATrX/fpuqPpvtG+Ajm/dktzVucLXT6DupHydRZWN0QxWG1la4JlH1x46NhBH1laiMRhAzSlYqoxFc/5n9cc5B41Mea5aqrd9pLZoTkbQjfEUEdZVRp5G8LdbT5xNrVXVlDdJlNhLBxtYM+jUAq8n3sW8fjZbObsR64ujqiTuXXT1x1FdFXRkM08Q0W8QXG83h00eHF2yMNk7iN+/qRE9vvE8w5beX5LLjpjhb4P+5aBNWbm3F5BF9PidxmFOoTp0xOu1eElV1BRvmz8Xeo3M7+V69o+/7+sryba7AJ0hrBssx66ui6LDLYzbtSmaJavIso5o03OpT6Y0r1u20+lTSjdOl4nPb88tdX7+3rinl7wwRFa9QCpntT/IPAfA7WCfU34N1An8rgCNUNdMxJn8GUA/gS7BG1+4P4F4AM1T1aZ/7x2BNsdoHwDftP9WwRtrOVNVGn2OuhtX3cTisgOCbAA4E8DcAp6rqlRm+1pRCfE9KmrlPYXpIuw7CJiJ9Sqky2R4e5vObJ8/jBlfjL988Mm2gASRL1QCrFCkRPOwyR98GnGil69vY2hpzGne9Y28T3MGGdcJo9mv4jb31ex3D6yoxdnA1Jg2vxd6j67H/+EGBgQbgLqNavb3dNZa1N674aJM5iSq8n7vKaJnzPcUV2OxZZgi4g59EBmafMQ043i4HUwXueclvD6mlpzeO55Y4czBwyozRGDuo2vl33t7WhZ1t7pkXuzp7nExIdXkZBhsld9NyXOznLaMCMm8STzf6FnCX8iVKLYH8MxuV0TJMNHp6OJGq9MxbuR1vNO5wXffe2t16ngrRbi20MypVXQvg4gzu14jk6FfvbTcgWZ6UyXN2A/hqpve3j7kFwC3ZHBPwOP9CwPdh3Cej92R39tWjJ2HltjaMqKvM6BPRgbLvmAa8bJxI9WewAQDf+cRUXPf3xZg1YQiuO2u/wD4Hr0SpWiJ7tLG5A4Oqy9NOo/Jev8tnIpXZeO039hZwN4gnplFtazEzG+mDjVwMqi53slGxnjg27up0Ps1fvb3N+cR8RH0lhoX8GsYOrnLKfjY2dfQZhGAuKTSDom8eN8UpC3n0nfX4ziemuZYyJrzZuNPJso2sr8RBewxGJCKYPLwOi+1elJXbWnFwbXJ3jKuEanCV699qwrBaVEQj6OqJY0tLDDvbujAkg5+vNTv6BhtrdrRjzfZ219hiP+nKqIDgaW/5ZjYAK8BK9LZ8tLklbY/Mx0lzezdeWrYVR0wZVrDfz3zd9sKyPte9t66p/18IEYWiP/Zs0MfYHkNr8MAls3HjZw8MbSJQIXj7Nur6sWcDAE7dbwxe+6+TcPsXZmUcaCSM9dn9kMkmdO+uAy9XCVVAlsEMNra1dqE3rmkX+oVlomsiVfK1LnVlNcIroUoYm2Yilfm+TTbet9mThjqTqbp647jvlVW+j/+0MYXqkzNGO4MKzOldK7a4S5yCSqgAoCwirmb8TEupzGDD7Ht5eflWv7u7uMqoAoKNoKWZYUyBczWJM7PhWLa5BSf/6kVc8b/v4uL738xq70t/eXv1Dry63Er8lxlDOhZt2OU7CpqIil/xnv0R9aMZY92Nyv2d2ciH2bS83t4i3WIuVcsgs+FXRmWewE8M+CS7Mpos2emNK3a0dWG7UeIzvD67wCkbQbs2XJvDC1C6NybNrg13U33yNYoILjsuORfjgddX48WP3Cfu8bi6+zX2G+38fYrR8+Tt23Dv2Og7XnpvcxxsBsFGc0c3mtqtn6HKaATnzkrueMlkBK5rGlXAz59feVVlNBI4UCAbbBLva/GGXfjcPfOcDOQH65uxeVffMsCBdqvRq3HWzLHYc6j1356unnhWZYBEVDwYbBDBqq2vNj5RLa1gw5hIZZ90ZlZGlcx4tPose0vXHJ7g7dvojzIqwBqlnNDoCjYKm9kIWqQIWCdEicWMQN/37eR9Rjknwh3dvbjkd2/ij/OTU7PfX9/sNEsPrinH7EnJUqlUTeJ+OzZM00ZnN5FqjdGvsefQGhwzNTl++NXl29EbT/2JeFtGZVQ+U9JCauSexvG3Lu+tbcIFv5nnGoIBFF8g9t7aJicAFwG+dcJerhK4BSylIipJDDaIYKXrzX0MQaUfxSh9GVX6Bl2/LeLuno3gGv2R9e7xt+Y0qqCFfmEI2rVRqB0bCWNc77c7s7F2Z7tzIj52UFWfDfaRiOCOL8xyRtP2xhU//OsH+Pk/lvTJapy8zyhX6eEU18b14DIqM/OS4MpsbEp/8m1OopowrAbTRtU5QWVzRzcWrk/drOteKhmU2ehb3ldTkX8JFWAFeYlFiOubOnyD6d1BrKcXn737dRxwzdP49/97D++tbepzn7cad+BL985Hc0ff3/FiCzZueyGZ1fj0AWMxZUSdazy23/dHRMWPwQaR7cDxg52/+zXuFiu/LeLpNogDfbc4m7p741i5LXlSmiqz4W0Sz3YaVa4m+ZRR7ersdvooysvEdYIellS7NhoDSqhMU0fV42/fOgr7jUtmXe5+aSW+9cd38NTCjc51p84Y7Tpu0vBaJPq+1+xoR6wnmT1I1bMBAHt7MhvpavXNfo09h1rDAY7ea7hz3StpSqlasyzjS8h3ElVCRTTiev9314lUc5duxRurdmBXZw/+7+11OOuOV3HGba/gkTfXoqOrF6+v2I4v3/eG8/s9pKYc5x2cnHJXTMHG4g278JyxyPKKE/cCAFdmg8EGUWlisEFk+8ZxkzF74lCcOH1kUU/O8vKe/MZ6ep2lctGIOCNTvcxPnL09Gx+sb3bGkY4fUp1yotPIPsGGUUbVTz0ba3d0oDeuWGqUUO01sr4gQwmCFikC3h0bwQHaqIYqPPKNI3DyPiOd655auMkZN1tTUYajpw53HVNVXobx9gJI75ZyM8PiV0Y1ZlCV8+/d3NHt7EQJ4i6jsh7PfD2pRuD29Madn52IwFWeaPILNqpDymwAH4++jfd9yoo+WN+M//jL+zjsv5/DV+5/wxlfPbyuEn/6+hE4a+ZY574fFVGJ2e1zkxOoPrXfaKcUbsbYBqdRfPnWVt8sLBEVNwYbRLYxg6rxyGVH4L6vHFpSS8DMsp5NzZ2ucon6qqjvyFrrNnPPhvt/4PNXJmfcHzZpWMrnNzMbm5o7XXXhw2oLl9moq4w6mZOu3jg2NHW4NoeHuV/DNKKuEuVl1nu6s70bHV3JDEOmwQYA1FREcfeFh+Dioyb2ue2E6SP7lGABnlIq+9P63ri6luKZ28MTRMTdt5Gm0dYMZCbYWa2jjMzG26t3ur5vk7dfI/jnz69nI7xgY+pIs28j82CjLdaD7z68AOfe+WrRZ0Q+MMrZDtpzMCqiyf+l7+rscRaFjm6owsPfOBx7j653ldQt39JaFBOpPtrcgqeMEsJv21kNwPo9SQQequ7vmYhKA4MNohJXXVHmjMvtiatrz0NQCQvgPtnzjr59Y1Vy5+RhRpOyn5FGydlHm1ucnoVB1eWuk59CmGT0kqza1uaZRBV+czhg9V2YZXYbjOxGNsEGYPUKXX3GDFxzxr4wpnz2KaFKcPdtWCfCW1o6nfd8WG2Fb5ACuJum033S7yqjsieRjWqocrIFXb3xPkvXEloyGHsLBPVshBfku7/fzIIGVauH5tF31+OdNU34+T+WhPZ6wqaqrhPvmz83E/P/6yRcddo+rulx4wZX45FvHOH87Iyor8Sgauu9b431uIYLDJTbX1iORMxz8j4j+0wHnLmH2bfBYIOo1DDYINoNmKVU5olkUL8G4B49apZR9cYVbzXudL6enS7YMDIbi42T/WF1hSuhSjB7SRq3txV8ElWCewJY8mQtk54NP185ahJ+e9Gh2G9cAz4za3xgGZ9fk7hr7K1PCVXC3kZZUarMRldP3AmgROCUbgHA0Xslp1K9ssx/30Ymk6iAoJ6NwpRRZZrZ+OMba/DYgg3O16+t2O7qjSkm63Z2OOOJG6qi2HNoDYbUVuBrx07GC987Hn/46mG49swZePzbR7mWMIpIUZWYNbV34Yn3k+/5FSdO7XMfs5+OfRtEpYfBBtFuwCylMk8kU43wdZVRGQ3iSzbucr4e1VCJCWm2RZtlVGbQ0h/bic0T+pVb21zfeyEmUSWMHdS3Sbyzu9f5lLgsIthjSOr3zeuE6SPxxBXH4MbPHuhaZmby27Vh9mv4lVAlZDr+dt3OdudT5jENVaiMJgOAo6cmS+peDujbyKQ5POi2mhDLFycOr3XK3TY0d6at9V+4vhnXPr7YdV1Hd68r8C4mZlZjv3GDXOVqkYjg6KnDcdGRE337rVxLDwc42FiwtgmJScr7jxvku+3d1STO8bdEJYfBBtFuwJxAlHmw4Z/ZmL8qWR4ze9KwwJr7BDOzYSrk2NsEs1TpxY+2oqPb+hR6RH1lQYMd9wQwK9gwx++OH1JdkBIy9xZxq94+08zGdKOsbOnGlsBP7Ff7lFAlHDZpmHMCv3RTi7MgzmT+LPkt7kvw2yAeZmajvCzi+vlItUl8V2c3Ln/oHWewgsm7eLFYmMHG/uMHpbhnX9NGmlmuge1LMcuiDtpzsO99po6scwYNbGzuxJZdA1/6RUSZY7BBtBswP9E2P7U2sxde9QF7Nsx+jXQlVIB1Quk3cWh4P5dRmf0S0wuwOdw0xmexX2OW/Rq5GFZb4dTbt3X1YvOuWNqxtwlDayucLFVXbxyLNuzyvZ85iWrCUPf3UVsZxUF7DnG+fm1F3+yGWUaVKtgodM8GkNkn+KqK//i/950+lbrKKP7f6fs4t7/4YXEGG+auk/3HZRlsmO/LloHNbJgTtcxyKVO0LOL6Ht9bx74NolLCYINoN2B+op3JQj/AfSKYWHqmqnhjlTmJKn2wISIY2dA3i9AfZVRBJV77FrBfAwDGGT0yG+3SqZXbMtu4ng8R6bNJ3Gzw9VvoZ5plBArvrPYvD/JrDjcdY0yl8iulcpVRZduzEeI0KgCYNjJ9k/j9rzbin4uSk5Cu/8z++MJhe6LCHpv84eYWbCqCJmqTquL9dbkHG1M9G9bjaTbCF4qqugKHA/cI/j4O3IPL/YhKFYMNot1AUPlM6pM9c/RtD1QVy7a0YqfddDq0tgJTR2a2FM+vlGp4QHlVmGoro77PXch+DcDdI7PeJ7MxeURhgg2gb99GpmVUADDLKFN5d02T731Wu3Zs9A02jvLs2/COTnWVUaUIdmsqyuBtTQk7s5GuEfqdNTvx38bEqYuOmIBPHzAWNRVRHDopGZi9VGSlVOt2djgjrhPN4dkYXlfhTLDr6O51fob728bmTmcvT11lFJOHB//3hn0bRKWLwQbRbmBswCfaqcqoKqIRVNp9Bb1xRWd33NWvcejEIWn7NRJG+AUb/ZDZAPynPhVyEhXQdxqVqrrKuAqV2QDcE6mWb2nNuIwKAGZNMDIba4IyG8nvwy9zdMC4QU5WYtOuTqdRPSHTMioR6XN72JkN7yf4Cet2tuPB1xvxrYfeQY/9qf6B4wfhh0b51HHTkpO3XgyYvDVQvP0amf6eJoiI64OEgZpIZWYo9h83CJGAwQhA34lUA5WNIaLsMdgg2g2MrK/ynWCUqozKe3tLZ7erhGp2mmV+3uf36o+eDQCY5DmxLy+TlJ+QhqGhKuo0M3d096KpvRurtiUzAoXq2QDcwcbC9c1OJioakbQB3t6j6lFTkWy03eD5RFtVXWVU3p4NwKqfP3JK8mfj2cVbXLdnWkYF9A2Gw85sTBxW45RDbdrViZ89uRif/NVLOPp/5uJHjy1ySuAaqqK4/QuzXJO3jpuW3O7+yrJt6PFpHh8oZgnVflmWUCXksockbGYJ1QEpSqgAa+hCIhuzq7PHNZCBiIobgw2i3UBZRDC6oe8Jf/pgI3myt6uzB/NXZr7Mz1RMmY29RtYXfJmgiLiaxD/a3OKUg1SURdKWM+XDnEhlnqyNHuQfcJqiZRHXJ8Te7MbWlhg6u62T6kHV5RhU458Z+6SxdPAfH2x03dYay6yMCuj781kbcrARLYu4Stp+8/KqPmN/q8vLcMvnD8IenlKkaaPqnN+p5o7uompKNpvDDxg3OKfHyGUPSdgyaQ5PEBEcON5sEm8KvjMRFRUGG0S7Cb9SqnTBhvnJ86INzdhijzKtr4xmVYrk1zfhF4AUgrlFHAD2KfAkqgQzoHh1RTJI23NYTdqT/nzsMaTaGT/ba5SSZBrgzJow2Pn7O6ubXLe5xt6m6AM4ed9RTsbgg/XNWG18ytyaYRmV3+01IZdRAcDePj8PFdEIjps2AtedNQNzv388Tpg+ss99RATHTkv2pxRL34Z3c3i2zeEJZolZqr0rhRKPKz4wMxsZjO919W1wkzhRyWCwQbSb8DvZTNWzYd2ePNl7bkmyHOaQiUOyOmEe6cmq1FVGUeUzDrcQvJmNQvdrJJiL/V5bnpzKVMgSKsD6tN6vJ2RsioV+JtdEKk9mw9UcnmKZY0NVuetE/Ekju9HamU0ZlSfYCHHPRsJXjpyIkfWVGN1QhQtm74HffPkQLPjxJ/D7S2bjy0dMxOgU79uxZt9GDsFGrKcXc5duQVN7V06v3c/aHcnm8EHV5dhjaG5ZNLOMavmWVlfg2h9WbmtzlocOr6tI228EuIONBZxI9bHW3N6NG5/5EH95e91Av5Sc9cYV9768Ene9uALdRVSmWQjh5qyJaMD4BRvZnOz968NksJFNvwbQd4Fff/VrAH37Cgo9iSrBfL/NE59CBxuA1bfhXVKXaWbD3JOxaEMzOrt7ncBwjZGhmJBmwtFp+49xAtR/fLARlx+/FwBPGVW6zIYnGA67jAqwvt83rjo5p2OP3ms4IgLE1Srb2dnWhSG1mf1sb2npxIX3voEPN7dgUHU5Hrr0sJz7K0zerEa2zeEJQ2srMLyuAttauxDriWPtjnbfYQuFYpZQHTB+cEbfh1lqtXjDLnT1xAteMknF6YZnluIP89YAAPYYWpPRTqhi8/Cba/HTJ61peHWVUXzp8AkD/IoKh7+lRLsJv0+205dRucffJhw2Obv/cHv3bPRXvwYAVFeUObsnKsoimDE2/xO6TJiLFHuMT4X7JdgY6ZPZyDDYGFpb4bzG7l7Fog3Jk9c1GZZRAe5SqoXrdzmjf80yqtoiyGzkY3BNBWban6arAq8s77tXxM/G5g58/u55TnlSc0c3LvztfHy4Kf9yJTPYyDd4cTeJ928p1ftZllAB1s9u4ueyqzeOpZv8F1PS7i0eV/xz4Wbn62eMPTmlxOx3e90oxd0dMdgg2k3kW0aVUF1ehv2yPGEfWlOBqFF21Z/BBgD8/Nz98Yl9R+EX5x3gTKwptKCyj0KOvU0wJ1IlZFKGkuBe7tfk/H11moV+JquUKllmlCilMqdRpR1Q0Gf0bfEl27MtpVq7ox2fvft115JHANjZ3o0v3ju/z6jgbOWzOdzLvUm8fydSmdlAszwqHXffRlPg/UpVc3s3fvDn9/Fvf3oXLUZJIiUt2bTLGcgB+C8XDUM8rmgzMrVhaov1YP6qZICxcEP6HqR4XHHyTS/iaw+8hTvmLkdXT+mUXjHYINpNmIvmErKtmQesBuJsSxMinrGrw+v7r4wKAA6dOBS/+fIhOPugcf32nGMCTu4LudAvYS+fZYvptoebXE3iRt/GGqNnY0IGQdPpB/SdStXamXkZlfnzFxE4e1+Kiblv46WPtvZZYmhata0Nn7v7dazdYY0ULi8TfP+Uac77sK01hi/8Zp6roT4b3ubwTDMCQaamWXpYKF09cSzemMxKpJtEZTInUi3YzZrEO7t78bUH3sLDb63F3xZswC3PLRvol1SUvMHFh5tbsGVXZ6jP0RtXfP3BtzHj6qdx1u2v4JWQA5pXlm9Dd2/yvyWrt7c7vVhBVm5rw/ItrXh28Wb89pVVzqCQUlB8/2Unopx4P9murShL2+TtF2wclmW/RoJZStXfmY2BMManbK2mosx3MlfYJvtkNrIZt2tmNt5evROqitZYD7a3WY3MFWUR31HKXifvM8oJTBdt2IVV29pcS/3SZSrMYKS2Ippz/0EhHTB+MAbbI4C3tMSwNKAUatnmFnzu7texwd7dURGN4O4LD8a3T5yK+y8+1CkR27wrhi/8Zj7W7Wz3fZxUvM3h44fkN2J5oHZtfLS5xflU1tyfkYmZRbpJ/I1VO3DF/76LH/z5fdz07Ef44/w1eGHpZiza0IztxqfwQeJxxb//+X280ZjcdfT39zeU7PLCpvYuLNrQnDI4z5XfZLiwsxt/e3c9nltilWq9t64ZX/rtfHzhN/MCl6Fmy+yRTFi8IXVZoJkNnLlHZn1OxYLBBtFuoqE6uWgOSF9CFXSfXBvtzCbxj0OwUVVehmGek6SJw2r75X8AdZVRVzBQXxlFQwb/3gnTRtU7J/pbWmJY39ThymqMH1Kd0TSy+qpyHDs1+cn/X99djy57qkp5maTNVJg/f4UYexuGsojg6L2Sk7f8SqkWrm/G5+6Z54yOriqP4L6LDsWJ00cBsDJv9150iPN+rG/qwBd+Mx8bmzv6PFYqYTWHJ0wbmQw2VmxpzXpx4dod7Vl/D4A7SMgmqwEAM8YOcn42V2xtdf3cDgRVxQOvN+KC38zD39/bgIffWotbn1+GH/71A1zyu7dw+q2v4OCfPoezbn8FSzYGn0z+4ukP8ff3Nriu27wrhjeN4KPQVBUvL9uKlXmW+m1rjeGEX/4Lp9/6Cu6YuzykV2dp7+rBW419T/gz7afKRFdPHL967qM+17+2YjvOvfM1fO2Bt/Lqv1JVzF3a978ji9KUUi1Ym/y+D8qi9LAYMNgg2k14F82lq5cH+pa5VJRFXJ8cZuMIY6t0KU4GyYU3m9AfzeEJZpN4NiVUgHUCfaCxsfmdNU1YsyNZ2pOuX8P06QPGOH//81trnb/XVabPVJhL/woxiSos3lKqhOb2blz790U4+45XscPOCtVWlOH3F8/G0VOHux7jyCnDcc+XD3Ga6tfsaMcRP38B+1/9NI7+nxdw+q0v4wu/mYdv/uFt3P3iCt9RtO+vb3L+vn+eJVQAMKim3MnEdfXGXT07qcR6enHN44twzC/m4uj/meval5GJ91z9Gtl9H9UVZZgx1hpvrQqc++vXXJOt+lNXTxw//OtC/PixRWlHB7+3rhln3f4q7nlpRZ9sxUPzV+OuF1c4Xw8xlmn+/X13AFJIv3j6Q1z42zdw1u2vOgMfcvH39zZgZ7uVgbtj7grsbAtv9PO8ldudDzTM/3+9vGxbaFmU/31jDdbttILoITXl+Pyhe7g+fHl28WacestLuPGZD3N6/KWbWrDJp+zL7Mfy48ps7Dk4p+ceKAw2iHYj5slvuu3NQN+AZOYeg3Pej/HlIybi9i8chD9fdoSrPGN35i2lmjg885P0fJlN4rlsLHc3ie907dhIN/bWdNI+I51SqkQJEZBZs7f581esmQ3AHWy82bgDzR3deOD1Rhz3y7m4/9VGZxpZfVUUD156GA6b7F+KeNy0Ebjzi7NcwxRaYj1Yt7MDizbswmsrtuOphZvw86eW4mafT1bDbA5PcDWJZ9C3sXp7G8779ev43WuNAKza9r+8k92uA/ckqsFZHQsA//HJ6U7Qtq01hs/dPQ/PLt6c5qhwbW+N4Uu/nY//fWONc90B4wfhJ2fNwLdP2AvnHzwex0wdjmmj6pzX2tUbx3//Yym+cO88p4xu7tIt+NHfFjqPcfI+I3HHF2Y5Xz/1waasM0652NoSw29fWQXA+pn8w7zVOT/WM4uS/xYd3b24/9VVeb++hJc+SmYwPnfoHk5gtq01uMQxG22xHtz2QjIb860T9sL1nzkAz333OJxx4FjnelXgtheWY/7K7KdIvbA0WUJlTv1bmKKMqqOrF0s2Jr+/XH5vBhKDDaLdyDjjE+7MyqjcJ4T5ZCQqohF8+oCxOGTixyOrAfhlNvr2UhSKuRk7lwlYsyYkg4131+z0TKLK/PHqq8pdJ+MJ6ZrDAWDGmEFO6d/BRvBTbEY2VGG6/X539ypOuvFF/PixRWhqTzZ0HjZpKB795pGuIM7PyfuOwt0XHpw2C3b73OWuplRV98btQgQb6fo2nnh/A06/9RVXORdg9Stkqr2rx2lGF8ltfO/RU4fjD5cehkHV1n/jOrp78Y0H38Lv7QCo0JZs3IUzb3/V9X2fNXMsHvnGEbjwiIn4/if3xg3nH4gHv3oYnvnOcXjq345xNfPPW7kDn7r5Zdwxdzm+9cd3kEh0HDB+EG694CAcNnmYk3Ha3taF13M4oc3W719rdE03+ss76xDr6U1xhL+m9i5X3wkA/O61xtAma5mZxeP3HoGjjBLHl5dlv3jT6/5XVzmTrsYMqnJ2X0waXovbLjgIT155NA42/tv586eWZp1RMfs1vn7sZCQ+e1ixtRXtXf7TrxZuaHayZ1NG1Do/+6WCwQbRbsScSJVJGZU3IPm4lD+FZaynfGlSP2Y2zjloHI6ZOhwHjB+Erxw5MevjZ+1hLvfb5fpUO92ODS+zlCohk5+/QTXl+Ou3jsJNnz0QP/jU9Kyes78dt3cyoDLHbo4fUo1ff3EW/vT1wzE1w4zeSfuMwtzvH48V/30aFvz4E3jx34/H498+Cg9+dTaOsLMiqsC/PbwAW1qsbNGaHe3YZU/6GlyTf3N4wrQMJlJ1dvfiqr9+gG//8V1naWNFWcQ5SVqyaVfaSToJC9fvck6up46syygo9TN70lA8evmRzs9qXIGrH1+EnzyxuKBN1c8v2YzP/Po1rG+yymxEgB+cOh03f25mYFZ4yog6/OWbR+LKE/dy3rOWWA9uePpDtHdZJ/TjBlfj3osOQU1FFGURwenG75S3lyNsbbEePPB6o+u6ne3drgxFpuZ+uKVPSdmuzh48NH9NwBGZW7uj3RkpXVUewaETh+KYqWawkV/fRlN7F+5+aaXz9b+dPLXPv+mMsYNwy+dnOtmqBWub8NTCzPd8NLd34+3VVu+FCHDqfqOdLLUqAvt6Fqxpcv4+c4/i/WAmCIMNot3IdOPT7kxKYcwTwrKIuD7tpvQGMrNRUxHFg189DI9/++icNj8PqinHFHtMb09c8dbqZPPhhCx6NgDr5Nk7LjnTnRnTRtXj3FnjUVPEPRsA+mRvaivK8B+n7o3nvnscPrX/mJyatcsigsE1FZgwrBYHjB+MY6aOwC0XzHQGLGxrjeE7Dy9Ab1xDbw5PmOoqo+qb2Vi7ox3n3Pma62Rxz6E1+Ms3j3QWaKpapXiZ8G4Oz8eUEXV49PIjXX1mv31lFS5/6J2C7CDY2daFK/73XSdAqKuM4t4vH4JvHj8l7b9HeVkE3z1lb/z5m0f2+f1qqIri95ccipH1yQ8vzJKdfy7cVNCdCn96c60TyLqvzz5AMMvZ9jZ+tu59eRU6u4MzJT29cby7Zic6uoLvYwYTh00ahqryMhxtDKh4Y9WOlM+Rzq9fXOEst508ohafmTXe937jh9TgoiOT275/8c+l6M6w1O2lZVuNTNZgDK+rdGX3Fq4PCDZKuF8DYLBBtFs5eZ9RuPz4KTj/4PG4+KhJae8/uqHKKcc468CxOX/K+HFlZpIaqqKuxs5SYJb8mJUA2WY26iqjON5zMr67/SwdNmkYTpxu9aecf/B4zP3+8bj8+L1y7nEKMrK+Cjd/biYS566vLt+OX/9reaibw03mro2V21pdJ01bW2L44r3zXZ+2nn7AGDxx5dHYf/wgHGqUTHpLZ4K8Z5SCHRhCk/vwukr879cOxydnjHKu++eiTbjqrx+EPnb14bfWujIRj15+JE7aZ1Sao9xm7TkE/7jyGFwwe08A1gc+d194CPYa6c6KHbTHYGec+a7OnlBKhPx098bx25eTn+Z/64QpTvbl1eXbs5r21dndi399mHydN372QKevbVtrDI8YAyRMrbEefOau13HOna/hC/fOCzxxN0uoEss2xw2udj40ifXEc57etam5E797tdH5+nuf2BvRsuBT5G+dsBca7A/rGre3u3p3UplrlFCdYGdLEwMPgOAmcTPYKLVJVACDDaLdSiQi+I9Tp+OG8w/EiAz2PYgI/vzNI/D4t4/C/5x3QD+8wt3L9NH1zo6AY6eNKKm55wB8M1mjGipzOoE+3VNKlUkZVSkpiwju+8qhWHztJ3HD+QdiZAZ7SHJ19NTh+PYJezlf3/TsR3jivY3O1weEGGw0VJU7J4TdvepMIWrp7MZX7n8Da+xenopoBD89ez/cfsFBzpjl2ZOSPz+Z9m2EmdlIqK4ow51fPBiXGB+w/N/b63Dnv1akOCo7Pb1xPPh6smn6306emvMgjNrKKH5+7v5446qT8Mp/nOia5JcgIvj0gcnfqSfe39jnPmF44v0NzmCHYbUVuOLEqa4s3sNvZZ7deH3FdicYmzisBjPGNuDrx052br/7xZV9Aomunji++Ye3nQll765p8j1x7+mN49UVyczGcdOS5VPHGNmNXJfv3frCMsTs7NH+4wbhU/uNTnn/wTUV+PaJyd/RW55blrYvJR5XvGgEYyfsPRKA+8MDvybxLS2dTtleVXnE1a9XKhhsEH3MVUbLcMD4wShP8SkO+autjOLPlx2BG847AD87Z/+BfjlZ82tmzjarkeAtpSrmUbb5SPVpZ5jmnDQVs+3MQVzhnGwA4WY2AHcp1UebW9HZ3YuvP/A2FtknPmURwZ1fmIUvHT7BFVCbmY331zWlLWHZ2dblTD2rKItg+pjwTprKIoIffXofnH9wsvTlBp/dFV4L1jbhmUWb0vZ5PLdki/NvMLS2wlXmlKuR9VUYlCIbesYByed4ZtGmvEqE/Kgq7n4xmdX4ypETUVVehs/bWRcA+L+31mU8DesZo4TqE/uOgojg84fu6Xwgs76pA48tSP57xOOKH/zl/T69Fr969qM+PUAL1jY5JU5jB1W5pvGZfRsv5RBsNG5rwyNvJrMu//7JvRHJYM/Ql4+Y6GSftrd14TdGv4ef99c3O4tTh9dVOFUF+xqZjWWbW/r8O5v9GvuPG1SS/68uvVdMRFREJo+ow/mH7FFy00EAq0G33lPutOfQ3HaFeEupMpmGRsGiZRHccsHMPqV5YTaHJ0wbmTxxW7ppF77z8ALXBKSfn7M/Tt63b7nQsLpKp4Slu1fxrnFS5Od9o0RknzH1qIyGW4ImIvjZOfs7TfYA8L3/ew9vr+6bddmyqxPf/uM7OPuOV/H1B9/GDWl2JpiTri6YvUfo5XN+ZoxtcKaWtXX1+m6dzseLH211xsVWl5fhwiOsPoQTp490MuNbWmKuUa1B4nF1Nm4DwCf2tTID1RVl+OrRyYzTnf9a7jSQ/8/TS/HXd9c7tyWWXu5s78btLyxzPb5ZQnXMVHcW+bDJw5xx0ks27sLWlvQb2xNUFb985kNnfPXhk91N56lUlZfhe6dMc77+zcursMVnf0bCXON9PG7aSCegaagqx0S7j6cnrn0GNbzr2RxeikILNkRkvIjcJyIbRCQmIo0icrOIZNxxKpZLRGSeiLSISLuIvCsiV4qI72+2iNSJyE9EZImIdIpIk4g8LyKnBdz/WBF5UEQWish2+5hVIvK4iJwUcMwlIvI3EVkuIrtEpM1+vt+IyN4BxzSKiAb8yXx0ARFRgUQi0qfZMNvmcNM3jpuMaERQVR7ByfuOzPPV0ZhB1bjxswe6rguzOTxhmlGWcc9LK13Tdf7j1L3x2UP3CDx29qTkiX26evn3jZOmQu0JqIhGcNeXDnaCoK6eOL72wNtYvd0qD+uNKx58vREn3fiiqzTpnpdWBm6FXrpplxN8lUXEGYdaaCLimvT29/fCLaUysxqfn70HBtdYGYjysogrQ/Twm/69FqYF65qck/yhtRWu8bAXHjHB+VBj5dY2PL1oE+5/dZXr+S+YvYfrZ/13rzW6Fgu+aGQsjvXpDzNLQl/NcJv4h5ta8MV757t+Dv7j1OlZ/X6dPXMc9hljZSY6unvxq+eWBd7XDBZPmO7+HmakaBIv9UlUQEjBhohMAfA2gIsBvAHgVwBWApgD4HUR8d9w1NfvAfwWwCQADwP4DYAKALcAeFg8PwEiMhjA6wD+H4BeAHcD+DOA/QE8KSJX+jzHifafjwA8ZL/W1wCcAOA5EfmJzzFfArA3gPn2a7oTwCr7+31fRD4V8P00A7jW588vU78NRET9w1tKlU+wcfCEoZj/w5Pwyg9OdCYVUX5OnD4KXzsm+clwpp+6ZsPsPYgZU48uOWoSvnnclJTHZtO38Z7Rr3FgAT+hHVRTjvu/MhvD7PKdHW1duPh3b+L1Fdtx7q9fw48eW4SWmHv6Um9c8aPHFvo2lf/+tWSvxqkzRrsGQxSaWa71/NLNaIv572HI1ntrm1wBlJl9AKyFeQlzP9yCjc0dSMWcQnXS9JGujdsNVeX4sjG96ZrHF+G6JxY7X5+8zyj85Kz9cPr+Y5wgpbtX8fOnlgCwyu8SvT4RAY7eq+/vwLGuUqrUzfTN7d245vFFOO3Wl/HaimQG7/T9x6Tdk+MViQj+yxjb/fCba7B8S9+gdWtLzBmOUBYRHLOXO9jYb6zZt5HMAPbG1dXnVIqTqIDwMht3AhgJ4EpVPVtV/1NVT4R1Ir83gJ+lewARORvAhbBO4meo6qWqOgfATAB/A/AZABd5DrsGwH4AHgUwU1XnqOqlAGYAaATwSxGZ6jnmelUdp6rnquqVqvpfqvpFANMBbAHwQxHxDo0/TVX3UdUvqur3VPXfVfU0AJ+CFQzdGPBtNanqNT5/GGwQUVHwNonvkWPPRsKwukpndCuF4wenTsd1Z83Av39yb3zlyPRT5rI1dWTfkc1nzxyL/3f6Pmk/5TUzG++s2RlY36+qoU+iSmXPYTW458uHOH1EK7e24YLfzHMakQFrUdt/n7O/U4Lzxqod+NuC9a7HaW7vxl/fTW5IvyiHnTb5mDaq3hkh29kdd5Uq5eMeo7/gjAPGYPwQ9+/9hGG1ONJuXI+r1buRyrOefg2vS46ahKpy699iS0vMmX43a8/BuO2CgxAti0BE8KNP7+sc8/SizXh9xXa8umKbc/8D9xjs2+dytKdJ3C9o7I0r/jBvNY7/5Vz87rVGp5yrLCK46IgJuOH83IakHDtthPMhQFyBnz65pE/fxYtGGdjBew7p8z3sNy7Zt7HIKDdcvqUVbXbT/Yj6SowdVLjBFIWUd7AhIpMBnALr5P4Oz81XA2gDcKGIpCsEPte+vFFVnRyYqnYD+JH95RUBx/xYVXuMY7bCCgDKAVxmHqCqvgV1qroeVoYjAmByhsc8C6AJwF5+txMRFTtvDXAm+1mof0XLIvjyERPxrRP26rPPJAy1lVHsMTT5af2x00bgF+cdmFGT7LjB1U6TbHtXr9NU7rWxudMps6mtKMPkEYXfSXPwhCG48fwD+1xfURbBnJOm4qk5x+ALh+2JS4xP9X/25FJXc/LDb61BZ7cVQO0zpgGHTuz/MpawS6kat7XhqYXJx/n6sf7ZK7NR/OE31wY20a/c2orlW6wdLVXlEdd0qIRhdZXOuN+EKSNq8duLDkV1RbJKfuYeg3H2zGQ256dPLnaN0z3W57EBq7ww0Te3pSWGjzw7Y95b24QzbnsF/+9vC7GzPfnve+SUYfjHlcfg2rP2y2vXzw9OTWY3/vXhVpz4y3/hkTfXOsG3a+Tt9L4lpmYmeMmmFmdq14K1yf01M/cYXHITDxPC+K/WifblM6rq+khDVVsAvAqgBsDhaR4nMWfMr50/cd0su3Qqm2N8+zC8RGQkgMMAxACk7hRLHnM0gMEAPgi4S6WIfElEfigic0TkhKDeEyKigTCouhznHDQOgNUYOoxZiY+lfztpGuorozhl31H49RdnZRXUzJ5k7NsIKKUyG4EPGD/YVWZTSGccOBb/cWqytfLIKcPw1L8dg+98YprT5D3npKkY3ZDcB/GrZz8CYH0S/oAx7vbiIycOyMnep41Sqpc+2prxtnY/vXGrIToRNxw7bYRrGpLpkzNGYbD9Cfz6pg68EtALYWY1jpk6whU8mL5+7GRU2+/5yPpK/P6S2Rhil7qZ/uPU6U4WZNGGXXj0nWRWxduvkVAWERy1VzLLlthL0t7Vg58+sRjn3PkqFhv7YsYPqcZdX5qFhy49LJRRsvuNG4QvHJYMpjY0d+I//vI+TvnVS3j8vQ2uBndvvwZg9bkkgvaunrgTvC3YDZrDASCM2YSJ3+KPAm5fBivzMQ3A8ykeJ/FT7JcjNjMN0wHMM44ZYx+zOOCY6fAhIocA+DSs92A8gDMBNAC4wsyseI45D1bZVjWs7+c0ADsAfDvgexoN4EHPdatE5GJVfTHgGO9zvh1wk+/3RUSUrZs+eyDmnDQ157G3VPo+c/B4nHPQuIyyGV6HThzqBBNvNO7A1451FQegs7sX9xsL084zmo/7w+XH74XDJg1FXIFDJgzpEzDUVkbxo0/vi2/98R0AwAOvN+L8Q8ZjQ1Mn1u20ehWG1JTjzJn5j7vNxaThtdhvXAMWrt+Frt447py7HFeeNBW1WS7O7OzuxZw/vYunFyWDg8s8/1amymgZzj1oPO57dRUAa6O438l+uhKqhDGDqvGHSw/Dy8u24vxD9nBOrr3GDq7G14+ZjFtfWA4ATmDUUBVNWX53zNQR+McH1nCDl5dtw9RR9bjqrx84/4aAlXm5/Pi98PVjJ4c+Uey6M2dg6sg63DF3Oba1WiNu/397dx4nRX3mcfzzzAw3DMcgcsogDgIakCAjARTwwCOiaCBqojGueMRkDVGj2c2hZjeJiTGBxKznJibqJrsxG02yunhAPELUxGMTFA9AQBQEQTlHEHj2j1/1TE3TPdM9dM1MD9/361Wvoqt+1V3N0zT1dP1+v2f5u9u4/Jcv1Lbp171jvcrqcYf1L6+dXnnxW5sY0a+83gxvY4p0vAYU5s5GKvKZyx7Wbe/RyPP8IVpfYWa1P5OYWRlhUHVKzwzHXBe/YxANSL8ietjBzDJ9oo8kdPP6KmEsSBlwgbvf0sA5zoyOuRqYAawETnT3P2do+zPCXZW+QBfCoPXbgErgITPb+96uiEgLMDMqe3dp0oWmtB1NjX98kPhfVmzcq7vNAy++xbtbQxeqvuUdC1KjIl9jB/diXGWvrHcmTvlI39qBx3scvn7/Yn4WXWQDnDXuoGaZ7jabeM2N255YzvhvP8Z1v3uJZeu3NnBUnY3bdvKpO56ul2icOWZAxoKCcedU1w0Uf+Tld3g9bVrWd7fu4LlVoatPiYXB4Q0ZO7gnc44fljXRSLlk8lD6pBWmnVTVu8E6N/GB40++vp7zf/psvURj4iEVzJ9zDJcfV5VILMtKS7hg4hAe//JUrpo2bK9pxQGmHNon62cwXj/npbc3s23HrtppcM2Sm8GtOTRHnY3U32rDFXPgV8BDwFDgZTO73czmAi8S7iCk5hOLj7r5BuGCfxbwYjTV7u2Euxx7gO0Zjgkn436ruxvhLsVIQnLwCzO7NdsJuvvZ0THdgYmEwex/MrPPZmh7vbsvcPd33H27uy9290uBH0SveV0jfx+p5xmbaQFeyeV4ERGRJA09oGtt4bb3t3/I0tgF8J49Xm8w8gUTKxMZd7KvzIzrTz+MdqXhkuX5Ve/XzlRUYtTWoGgpnxg7sN7F95Ydu7hrUZjC99w7n2F+A0X/Vm7YxiduWcTzsV/JZ08awvdnjW60W1jVgd3qzRB10rwn+fr9i2vH3yxYsq528PbYwT0L1g2zS4cyrjqxfmWBbOM1Ugb16lxblySe73bv1I4bZ47inguPYnBF0+oI5aNLhzK+cGwVT14zlUsnD63tEgZhxqts4oPEF7+1ib+t3lT7Pob16UbXPO9ktSaF+BefunOR7d5WeVq7jKLxHqcBVwFrCTNT/QOwGpgEpOYnWxc7Zi0wDvgR4e7BZcDphDsexxMu6je5+84GXvcDd18SzXx1G3BJ1F2qoXPd7O6LgOmE8R23mFmu94VTycwxObYXERFptcys3sDp+LiNha+uY9n6UC+ha4cyzjnqoL2Oby2GHtCVizN0K5o2sm+jv8QnrXfXDjx25WSumz6Sgw+of8H81NJ3ueTu5xh1/cN86o6nuXnB6zy38j0+3L2HF998nzP/bRFvRDUrzODa6SP52qkjc76T9aXjh9WOsdm9x7n76ZVMuXEhcx99jd//ra4i+LSRfbM9RZPM/OhARkXdprq0L804sDpd+tTQ00f359ErJjPryEHNPt6mR+f2fOXk4Tz+5alcfdKh/PCs0UxqYOrq+PS3L6/ZzPOr6g8OL2aFSJNSg6mHZdmfmno225iOWtGMUjeRNpVs1A3qCKAGeCntmPWEeh5fTDtmKuGuyl8ae92Yh4BLgCmEeh2Nne9OM3uM0EVqfC7HUJcsJZ9ei4iINIPqIRW1XXSefWNjbeG722J3Nc6pHkR5K68s/4WpVdz/wtu1feeh+ae7zaZbx3Z8duIQzp9QyZ+WbuDnf17BY0veqf31e+euPSxatiG6I/MaXdqXsmuP19ZO6VBWwryzj+Ckw7P/up7JpKre/PayCXz7wSU8vTwkktt27mZuWgG7hsZrNEVJiXH3hUfxH8+sYlxlTw4sb3za14uPOZhFyzZQVmJ8+cRDOW5EYc+pKQ4s78hlUxqftLRPeUcO6NaB9Vt2sH3n7nqTKhRrfY2UQiQbC6P1NDMric9IZWbdCN2Naqgb1N0U5wEdgZ9HU+Hm4qJofW8erzMgWudTNSffYz4WrTPNoCUiIlJ0qivrz0iVqquRustRVmJcMLHwNUIKrVP7Uq6dPpKL7w5zswzv243xB/dq5KjmZWZMqurNpKrerH5vO/c+s4r5L61l+fpt9dql6jNAGOB+5/lHMnZw097LqIE9+OVF4/njq+u54aFXeDVt7EZVn65U9i78b6jdO7Xjc1MaLiwZN7BnZx69YnLBz6O5HN6/nIXRVL+pGamg+O9s7HM3KndfBjxMGPj8+bTd1xN+wf+Fu28DMLN2ZjY8qjpej5ntNf+amY0DbgC2At9M21diZntN1m1ms4FzCOM97k3bN9nM9nrf0fl8NXr4P7HtFWb2kfT20b5TgTOic3s8tv2w+CD32PbBwM3Rw3syPaeIiEixGdGvrk/52s1hFqc74oXjRvenfwt3RcrVCSMP5FtnHM6MI/pzy7ljW3Vtg4E9O3PNScNZcOUUnv6n4/jhWaOZNXZgvW5fB/XqzG8+N6HJiUaKmTF1eB8e/OLRfG/mqNrpggFOb6GZutqa+CDxlM7tSxmWZQarYlGo0SaXEQri/cjMjgOWEGpWTCV0n/pqrO2AaP9KQoIS94iZ1QCLgS2ESuCnEGpfnOnu6XcDOgPvmNkjwNJo29FANbAMOCPDnZAHgPfN7BngTcLfwVDgpOjPP46K9aUMAl4ws+cJXbjeIsysdQSh69SHwGx3fy92zCzgK2a2kDCIfEv0Gh8n3KF5EFAVcRERaRPKSkv46OCetfUE7ntudb3CcRcdnX2K1dbGzPj0UYP59FEtOyg8X327d+SMMQM5Y8xA3J03N9awbP1Wqof0ynua3IaUlhifPHIQ00f1579fWE3Nzt0tPoC+rYgX90sZNbB7s9WlSUpBPn3uviyqW/FNwkX7KcAawsDt6909c5Wfvd0HnA2cSxjc/TZwJ3CDu6/I0H4HYRarScAJ0bZlhOlpf+DumeaEu5ZQ92M8YYB3KfAOcD9wp7vPT2u/Evg2YUD3CUAFIcFYRRhQPs/dl6Qds5BQf2QModtUF0Kl8acIdTfudvfGZucSEREpGtWVdcnGTxYurR1LcHRV76yF4yQZZsZBFZ05qCK52jmd2pcWXULW2sVnpEo5YlDzV60vtIKluu7+JnBBDu1WUDcdbvq+G4Eb83jND4ELc20fHTMPmJdH+/eof2cml2MeJ9atSkREpK2rHlJXs2FXbO7RTDM8icjeBvToRI/O7Xh/e12nnGIfrwHNU2dDRERE2rhRA7vTPq3o2oh+5fWKrYlIdmZWbwpcKO7K4SlKNkRERGSfdWxXutevsBcfM6RVD7AWaW0Oi3Wl6te9Y05T/rZ2SjZERESkIMYNqetf3q97R04dpVmKRPIxJjZG48jK1jXtclMp2RAREZGCOG30gNqZc+YcX0W7Ul1miOTjhJEHck71QUw8pIIvHV/V+AFFoHBzoYmIiMh+7dC+3Vhw5WQ21XzIqIE9Wvp0RIpOaYnxnTMzlncrWko2REREpGAGVxS+krSIFC/d3xQRERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUQo2RARERERkUSYu7f0OUgTmNmGTp069RoxYkRLn4qIiIiItGFLliyhpqZmo7tX5Husko0iZWZvAOXAihZ4+eHR+pUWeG1pXor1/kOx3n8o1vsHxXn/0RyxrgQ2u/uQfA9UsiF5M7PnANx9bEufiyRLsd5/KNb7D8V6/6A47z9ae6w1ZkNERERERBKhZENERERERBKhZENERERERBKhZENERERERBKhZENERERERBKh2ahERERERCQRurMhIiIiIiKJULIhIiIiIiKJULIhIiIiIiKJULIhIiIiIiKJULIhIiIiIiKJULIhIiIiIiKJULIhIiIiIiKJULIhOTOzgWb2UzN728x2mNkKM5trZj1b+twkd2ZWYWazzey3ZrbUzGrMbJOZPWVmF5pZxu8FM5tgZg+a2UYz225mfzOzOWZW2tzvQfaNmZ1nZh4ts7O0UbyLlJkdbWa/MbM10Xf1GjN72MxOydBWcS5SZvbxKK6ro+/x5Wb2azP7WJb2inUrZmYzzezHZvakmW2Ovp/vaeSYvGNqZueb2bNmtjX6v/+PZnZq4d9R7DVV1E9yYWZDgUVAH+AB4BWgGpgKvApMdPcNLXeGkiszuxS4BVgDLARWAQcCZwLdgd8Aszz25WBmp0fbPwD+E9gITAcOBe5z91nN+R6k6cxsEPB3oBToClzk7nemtVG8i5SZfQ34F+Bd4A+Ef+e9gTHAQne/OtZWcS5SZvZd4GpgA3A/Id6HAKcBZcBn3P2eWHvFupUzsxeB0cBWYDUwHLjX3c/N0j7vmJrZ94Ero+e/D2gPnA30Av7R3W8u7LuKuLsWLY0uwHzAow9jfPsPou23tvQ5ask5lsdGX0gladv7EhIPBz4R214OrAN2AEfGtnckJKAOnN3S70tLTrE34FFgGXBjFLvZaW0U7yJdgFlRfB4BumXY305xLv4l+q7eDawF+qTtmxrFbrliXVxLFLuq6Ht6ShSXe7K0zTumwIRo+1KgZ2x7JSFp/QCoTOK9qRuVNMrMDgamASuAn6TtvhbYBpxnZl2a+dSkCdx9gbv/3t33pG1fC9waPZwS2zUTOAD4lbv/Ndb+A+Br0cPPJXfGUkCXE5LNCwj/bjNRvItQ1P3xu8B24FPuviW9jbt/GHuoOBevwYRu8M+4+7r4DndfCGwhxDZFsS4C7r7Q3V/3KANoRFNiemm0/pa7vxc7ZgXh2q4D4f+GglOyIbk4Nlo/nOECdQvwJ6AzML65T0wKLnUxsiu2LRX//83Q/gnCxc0EM+uQ5InJvjGzEcANwDx3f6KBpop3cZoADAEeBN6L+vNfY2ZfzNKHX3EuXq8DO4FqM+sd32FmxwDdCHcwUxTrtqcpMW3omIfS2hSUkg3JxaHR+rUs+1+P1sOa4VwkIWZWBnwmehj/Msoaf3ffBbxB6CN8cKInKE0WxfZuQje5f26kueJdnMZF63eA5wnjNW4A5gKLzOxxM4v/2q04Fyl33whcQxhr97KZ3W5m3zGz/wIeJnSjuyR2iGLd9uQV06jnyQBgq7uvyfB8iV7HlSXxpNLmdI/Wm7LsT23vkfypSIJuAA4HHnT3+bHtin/x+wZhgPAkd69ppK3iXZz6ROtLCRcaxwPPELrc3AScCPyaui6SinMRc/e5ZrYC+ClwUWzXUuCutO5VinXbk29MW/QzoDsbUggWrTW1WZEys8sJM1S8ApyX7+HRWvFvhcysmnA34yZ3/3MhnjJaK96tS2qqSwNmuvtj7r7V3V8CziDMPjM527SoGSjOrZiZXU2YTeguYCjQBRgLLAfuNbPv5fN00VqxbjuaGtNEPgNKNiQXqYy3e5b95WntpIiY2eeBecDLwNToFn2c4l+kYt2nXgO+nuNhindxSg34XO7u/xffEd3NSt2trI7WinORMrMphMkAfufuV7j7cnff7u7PExLLt4Aro8ldQLFui/KNaWPtG7vzsU+UbEguXo3W2fryVUXrbGM6pJUysznAzcBiQqKxNkOzrPGPLmaHEAaUL0/oNKXpuhLiNgL4IFbIzwkzyQHcEW2bGz1WvItTKm7vZ9mfSkY6pbVXnItPqgDbwvQd7r4deJZwfTcm2qxYtz15xdTdtxGS0K5m1i/D8yV6HadkQ3KR+kKbll5d2sy6AROBGuDp5j4xaTozuwb4IfAiIdFYl6Xpgmh9UoZ9xxBmIlvk7jsKfpKyr3YA/55leSFq81T0ONXFSvEuTk8QLi6qzKx9hv2HR+sV0VpxLl6pGYYOyLI/tX1ntFas256mxLShY05Oa1NYLV3EREtxLKioX5taCF1qHPgr0KuRtuXAelQQqk0twHVkL+qneBfhAtwTxedf07afAOwh3PXooTgX9wJ8MorPWmBA2r6To1jXABWKdXEu5FbUL6+Y0oJF/Sx6IZEGmdlQwge4D/AAsAQ4ilDx8jVggrtvaLkzlFyZ2fmEQYW7gR+TuY/mCne/K3bMDMJgxA+AXwEbgdMI0+/dB3zS9WVSVMzsOkJXqovc/c60fTNQvIuOmfUh1D06BHiS0J1mMKEfvxOK/f061n4GinPRiXoYzCfMOLYF+C0h8RhB6GJlwBx3nxc7ZgaKdasWxWhG9LAvYQa55YR/ywDvuvtVae3ziqmZ3QRcQZgw4j6gPXAWUEH4Mfnmwr8zdGdDS+4LMAj4GbCGcHt2JWFgcYO/jGtpXQt1v2g3tPwxw3ETiQqGEX41+zvwJaC0pd+Tln36HMzOsl/xLsIF6EW44/xG9D29gfAD0XjFue0sQDtgDqH78mZCF7p1hPoq0xTr4lty+L95RSFiCpwP/AXYRkhWHwdOTfK96c6GiIiIiIgkQgPERUREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEUo2REREREQkEf8PwQNkyjwqwFsAAAAASUVORK5CYII=\n", 159 | "text/plain": [ 160 | "
" 161 | ] 162 | }, 163 | "metadata": { 164 | "image/png": { 165 | "height": 248, 166 | "width": 397 167 | }, 168 | "needs_background": "light" 169 | }, 170 | "output_type": "display_data" 171 | } 172 | ], 173 | "source": [ 174 | "plt.plot(meta_output[\"losses\"])" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 241, 180 | "id": "5ac01d9a-65c5-43d7-9d27-16248e722f64", 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "data": { 185 | "text/plain": [ 186 | "[]" 187 | ] 188 | }, 189 | "execution_count": 241, 190 | "metadata": {}, 191 | "output_type": "execute_result" 192 | }, 193 | { 194 | "data": { 195 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvQAAAHwCAYAAADJpfudAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAABYlAAAWJQFJUiTwAABsaElEQVR4nO3ddXxUV/rH8c8TJ0CCuwYLUJxSpKVIS9ut67a7dXff7v7alcp2vUp1q9t2d6m7L1KsBhQoRQME96BxOb8/ZjIJIUYyyZ2ZfN+v17xuzrV5hhsmzzxz7jnmnENERERERMJTlNcBiIiIiIhIzSmhFxEREREJY0roRURERETCmBJ6EREREZEwpoReRERERCSMKaEXEREREQljSuhFRERERMKYEnoRERERkTCmhF5EREREJIwpoRcRERERCWNK6EVEREREwpgSehERERGRMBbjdQChzszWAklAusehiIiIiEhk6wbsc851P5yDlNBXLalRo0Yt+vbt28LrQEREREQkci1btozs7OzDPk4JfdXS+/bt22L+/PlexyEiIiIiEWzYsGEsWLAg/XCPUx96EREREZEwpoReRERERCSMKaEXEREREQljSuhFRERERMKYEnoRERERkTCmhF5EREREJIwpoRcRERERCWNK6EVEREREwpgSehERERGRMKaEXkREREQkjCmhFxEREREJY0roRURERETCWK0TejNraWZXmtm7ZpZmZtlmttfMZpvZFWZ2WM9hZp3M7EUz22xmuWaWbmaPmlnzSo4ZbWafmFmGmWWZ2WIzu9XMomv7+kREREREQllMEM5xLvA0sAWYDqwH2gJnAc8DJ5nZuc45V9WJzKwHMBdoA7wPLAdGALcAJ5rZGOfcrjLHnA68DeQArwMZwKnAI8AYf3wiIiIiIhEpGAn9SuA04GPnXFHxSjO7G/gOOBtfcv92Nc71FL5k/mbn3ORS53oYuA14ELi21Pok4DmgEBjnnJvnX/87YBpwjpmd75ybUqtXKCIiIiISomrd5cY5N80592HpZN6/fivwjL85rqrzmFkKMAlIB54ss/kPQCZwkZk1LrX+HKA1MKU4mfc/dw7wW3/zumq/mBCyflcW+YVFVe8oIiIiIkHjnGPngVz2ZuV7HUq1BaNCX5nif4mCauw7wb/8opwPB/vNbA6+hH8kMLXMMZ+Vc76ZQBYw2szinXO5lT25mc2vYFNqNWIPqgO5BYz9+3SiDNonN6Jzi0Z0aZFI5+aJdGmZSKfmiXRpkUirJnGYWX2HJyIiIhLWsvIK2JCRzYaMLNZnZLFhdxYbMrJ863ZnkZVXyG9P7suVx6R4HWq11FlCb2YxwMX+ZnkJd1l9/MuVFWxfhS+h701JQl/hMc65AjNbC/QHUoBl1YghJGzIyAKgyMGmPdls2pPNN2syDtmvUWw0nVs0onPzRDq38D2Gd23OwE7JSvRFRESkwdt1IJfpK3aQvjPzoMR954G8Ko9d78/HwkFdVuj/AhwBfOKc+7wa+yf7l3sr2F68vlktjymXc25Yeev9lfuhVR0fTPtzCmiXlMDWfTmV7pedX8jKbQdYue3AQetT2zXl50d25swhHWmWGFeXoYqIiIiElMIix+y0nbz+/Xq+XLqN/MIqx2U5RJP4uu7EElx1Eq2Z3QzcgW+UmouCdVr/8nCuSk2O8dyI7i345u6J5OQXsmlPtv8roCw27M5m/S7fp8v1GVnszym/J9Pyrfu578Ol/PnT5ZzYvx3nH9mZkSktiYpS1V5EREQi06Y92bw5bwNvztvIpj3Zle4bE2V0at6Izi1KujKX7uLcLDE2rHo7BD2hN7MbgMeApcBE59yhfUXKV1xNT65ge1KZ/Wp6TNhIiI2mR+sm9GjdpNzte7PyA8n9howslm3Zx+c/bSM7vxCAvIIiPli0mQ8WbaZzi0b8fHhnzhnWmXbJCfX5MkRERETqRF5BEVOXbWPK9xuYuWoH5Q2SPrRLM8b0bOXrnuy/H7FdUgLREVToDGpCb2a34hv/fQm+ZH77YRy+wr/sXcH2Xv5l6f7yK4Dh/mMOuqnV34e/O74bctccRhxhIzkxluTEZI7oWPJ5Zn9OPh8u2sLr369n0caSzzEbMrL5xxcrefjLlYzv04afH9mZ8altiI3WZMEiIiISXtK2H+CNeRt4e/5GdmUe2h++eWIsZw3txM+P7Ezvtk09iLB+BS2hN7Nf4+s3vxA43jm38zBPMd2/nGRmUWXGtG+Kb5KobOCbUsdMA34JnAj8t8z5xgKJwMyqRriJJE0TYvnFUV34xVFdWLp5H2/M28A7Czayz989p8jB1OXbmbp8O62bxnPusE5cPTZFfe1FREQkpDnn+PjHLbw8J51563Yfst0Mju7ZivOP7MJx/doQHxPtQZTesGpM4Fr1SXwTOd2Pr0o+qbJuNmYWC/QA8p1zq8ts+xzfSDYVTSz1rHOu7MRSq/F1rRlTamKpBHzJ/ijggtpMLGVm84cOHTp0/vyKRrUMfTn5hXz+01amfLeBr9fsOmR7qyZx3Htaf04e0D6s+ouJiIhIw7AhI4u73/2RWasOrRe3T07g3OGdOXdYJzq3SPQguuAZNmwYCxYsWFDRYC0VqXVCb2aXAC/jm611MuX3V093zr3s378bsBZY55zrVuZcPYC5+GaLfR/fUJNHAePxdbUZ7ZzbVeaYM4C3gBxgCpCBb+baPv7157lavMhISOhLW7crkzf8N4xs33/wFxfH9W3DA2ccQfvkRh5FJyIiIlKisMjx0py1PPTFysA9guC7qfW4vm35+YjOjO3VOmL6w3uZ0N+LbybXynzlnBvn378bFST0/u2d8VX7TwRaAluA94D7Kqr8m9kY4B58FfkEIA14EXjcOVdY3jHVFWkJfbGCwiI++2krD3y0lG37ShL7JvEx/OakVH4xootGxRERERHPLN+6j1+//SOLNuwJrDODS0d34/pxPWndNN674OqIZwl9pIvUhL7Yvpx8/vrpcv797fqD1o/o1oI/nz2gwhF2REREROpCbkEhT05L46kZqykoKslT+7Rtyl/OHsCQLs09jK5u1TSh1xAnDVxSQiwPnjmA168eSUqrxoH136VncNJjs3hyehr5hUWVnEFEREQkOOalZ/Czx2bx+LS0QDIfFx3F7cf35sObjo7oZL42lNALAEeltOSTW47hhvE9iPF3tckrKOLvn6/g1MmzWbxxj7cBioiISMQ6kFvA799fwrnPfs3qHZmB9cO7NueTW47m5om9iItR2loR/ctIQEJsNL86IZUPbjyaAaXGtl++dT9nPDmHBz9eSlZe+bPTioiIiNTEtOXbOP7hr3jl63WBiaEax0Vz/+n9eeOaUfRsE/njyNeWEno5RL8OSbx7/Wju+VlfEmJ9vyJFDp6btZYTHp3J16sPHfpSRERE5HDsycrj5v/+wOUvz2PL3pzA+gmpbfjy9mO5eFQ3DdBRTUropVwx0VFcNTaFz28dy+geLQPrN2Rkc+EL3/LG9xs8jE5ERETC2YaMLM56ei4fLNocWNeicRyPnT+YFy4ZTodmGkL7cCihl0p1bdmYf195FH87eyBJCb6JhQuLHHe9vZhH/7cSjZIkIiIih2PJpr2c9fRc1pTqK3/mkI787/ZjOX1wR01yWQNK6KVKZsZ5R3bmi9uOpW/7pMD6R/+3iv9750cKNAqOiIiIVMNXK3fw82e/Zod/csu46CgmXzCER34+mBaN4zyOLnwpoZdqa5ecwBvXjOSYXq0C66Z8v4GrXplHZq5ulhUREZGKvTlvA1e8/D2Zeb45P5MSYnj1ihGcOqiDx5GFPyX0cliaJsTywiVHctbQjoF101fs4ILnvgl82hYREREp5pxj8tRV/OqtxYGx5TskJ/DWdaM5KqVlFUdLdSihl8MWFxPFQ+cO4obxPQLrFm/cy9lPz2XNjgMeRiYiIiKhpKCwiLvfXcJDX64MrEtt15R3bxhD77YajjJYlNBLjZgZvzohlT+ecQTFI0qtz8ji7KfnsmD9bm+DExEREc9l5RVwzavz+e936wPrxvRsyZvXjqJtUoKHkUUeJfRSKxeO7MqzFw0PjFe/OyufXzz3DV8u3eZxZCIiIuKVnQdyueC5b5m6fHtg3ZlDOvLSpSNomhDrYWSRSQm91Nrx/dryn6tGBu5Oz8kv4ppX5/HqN+s8jkxERETqW/rOTM5+ei6LNuwJrLtuXA8ePm8QcTFKPeuC/lUlKIZ2ac7b142ma8tEwDez7O/eW8LfPluusepFREQaiIUb9nD203NZtysLgCiDB07vz69PTNX48nVICb0ETfdWjXn7utEM6pQcWPfUjNXc8cYi8jVWvYiISESbumwb5//za3Zl5gGQEBvFMxcO46JR3bwNrAFQQi9B1apJPP+9eiQTUtsE1r3zwybuemsxRUWq1IuIiESiWat2cO1r88nJ9xXwmifG8p+rRjKpfzuPI2sYlNBL0CXGxfDPi4ZxwYjOgXXv/rCJv36+3MOoREREpC4s2bSXa1+dT36hr3DXuUUj3r5uNEO7NPc4soZDCb3UiZjoKP505gAuGNElsO7Zr9bwwuy1HkYlIiIiwbRuVyaXvvRdYPbX9skJvHHNKFJaN/E4soZFCb3UGTPjgdP7c3y/toF1D3y0lA8WbfYwKhEREQmGnQdyueTF79h5wNdnPrlRLK9cPoL2yY08jqzhUUIvdSomOorJFwxhWNeSr93ueGMhc9N2ehiViIiI1EZmbgGXv/w96f7RbOJjonjhkuH00uyvnlBCL3UuITaaFy4ZTs82vq/f8gsdV786n5827/U4MhERETlc+YVFXPfvBSze6Ps7HmXw+AVDGN6thceRNVxK6KVeNEuM41+Xj6Cdf6rnA7kFXPrS92zIyPI4MhEREaku5xy/fmsxM1fuCKx74IwjOEGj2XhKCb3Um47NGvHy5UfSNCEGgB37c7n4xe/YdSDX48hERESkOv762Qre+WFToH3zxF788qiuHkYkoIRe6llquySeu3h4YOrntTszufxf88jKK/A4MhEREanMS3PW8sxXqwPt84/szG3H9fIwIimmhF7q3ciUljz288EUzwC9aMMebvj3As0mKyIiEqI+WryZ+z9aGmgf17cNfzzjCKz4j7l4Sgm9eOKkAe25/7T+gfb0FTu4+50fcU6zyYqIiISSuat3cvvriyj+Ez20SzMmXzCUmGilkaFCV0I8c9Gobtw4vmeg/eb8jTz0xUoPIxIREZHSlm7exzWvzCfP/y16j9aNeeGSI2kUF+1xZFKaEnrx1B2TenPe8E6B9hPT03jl63TvAhIREREANmRkcelL37E/13efW9ukeP51+QiaN47zODIpSwm9eMrM+NOZA5iQ2iaw7g8f/MSnP27xMCoREZGGbXdmHpe89B3b9/tGomuaEMO/Lh9Bp+aJHkcm5VFCL56LiY7iiV8MYXDnZgA4B7e/sYhV2/Z7G5iIiEgDVFTkuOX1hazZkQlAXHQUz108nNR2SR5HJhVRQi8hITEuhhcvPZLurRoDkJ1fyPX/XqDhLEVEROrZ01+tPmjiqEd+PpiRKS09jEiqooReQkaLxnE8feFQEmJ9v5arth/gt+8t0cg3IiIi9eTr1bt46IsVgfZ143pw8sD2HkYk1aGEXkJKarsk7j/9iED7nQWbeHPeRg8jEhERaRh27M/l5ik/UOSvo43o1oI7ju/tbVBSLUroJeScN7wz5wwrGfnmd+8vYdmWfR5GJCIiEtkKixy3TPmBHf6bYFs2juPxC4ZorPkwEZSrZGbnmNlkM5tlZvvMzJnZa4d5jkv9x1X2KCxzTLcq9p8SjNcn9e+B04+gd9smAOQWFHHDvxdwIFf96UVEROrC41NXMXf1LgDM4NHzB9MuOcHjqKS6YoJ0nt8Cg4ADwEYgtQbnWAjcV8G2Y4AJwKcVbF8EvFfO+iU1iENCQKO4aJ765VBOe2IOWXmFrNmZyf+98yOPnz9Y00yLiIgE0axVO3h82qpA+6YJvTimV2sPI5LDFayE/jZ8iXwacCww/XBP4JxbiC+pP4SZfe3/8Z8VHL7QOXfv4T6nhLaebZrypzMHcOvrCwH4cNFmjureggtHdvU2MBERkQixbV8Ot05ZSPH4E6N7tOSWib28DUoOW1C63DjnpjvnVrk6GI7EzI4ARgKbgI+DfX4JbWcM6cgFI7oE2vd/uJQlm/Z6GJGIiEhkKCgs4qb//MCuzDwAWjeN59HzBxMdpW/Cw0043OlwjX/5gnOusIJ9OpjZNWZ2t385sL6Ck7r3h1P70be9bzKLvMIirv/3Avbl5HsclYiISHh7+MuVfJeeAUCUwePnD6FNU/WbD0chndCbWSPgQqAIeL6SXY8HngEe9C8Xmdl0M+tSyTFln2t+eQ9qdj+ABFFCrK8/fZN4Xw+x9RlZ3PXmYo1PLyIiUkPTl2/nqRmrA+3bjuvNqB6aPCpchXRCD5wHNAM+dc5tKGd7FvAAMAxo7n8U9+EfB0w1s8b1EqnUqe6tGvOXswcE2p/9tJWX56Z7F5CIiEiY2rwnm9veWBhoj+3dmhvG9/QuIKm1UE/or/Yvny1vo3Nuu3Pu9865Bc65Pf7HTGAS8C3QE7iyOk/knBtW3gNYHowXIrV3ysAOXDyq5IbYP32yjIUb9ngXkIiISJjJLyzixv8sYE+Wr+tqu6QEHjlvEFHqNx/WQjahN7N+wGh8o+d8cjjHOucKKOmiMzbIoYmH7jm5LwM6JgOQX+i44d8L2JOV53FUIiIi4eFvny1nwfo9AERHGZN/MYSWTeK9DUpqLWQTeqp3M2xldviX6nITQeJjfP3pmyb4+tNv2pPNnW8uUn96ERGRKnzx01aem7U20P7VCX04slsLDyOSYAnJhN7MEoCL8N0M+0INTzPSv1wTlKAkZHRukcg/zh0UaP9v2Xaem6XLLCIiUpENGVnc+eaiQHtiahuuPibFw4gkmOo9oTezWDNLNbMelex2Lr4bXD+p4GbY4nMdZWZx5ayfgG+yK4DXahWwhKQT+rfjiqO7B9p//WyF+tOLiIiUo6CwiJv++wP7cgoA6NisEQ+p33xECcpMsWZ2BnCGv9nOvxxlZi/7f97pnLvT/3NHYBmwDuhWwSmLb4ataGbYYn8F+pvZDHx97QEGAhP8P//OOTe3yhcgYenXJ6ayYP1ufli/h8Iix6/eXMRHNx9NfEy016GJiIiEjOdnrw0UvWKjjSd+MYRmiYfUQyWMBatCPxi4xP84wb8updS6c6p7IjPrCxxN9W6GfRXfaDZHAlcB1wO9gDeAsc65P1b7FUjYiYuJ4vHzh5AY50vgV20/wONTV3kclYiISOhI236Ah79cGWjfelxvhnRp7mFEUheCktA75+51zlklj26l9k0vu67MuZb5t3eu6mZY59wLzrlTnHPdnHNNnHPxzrkuzrmfO+dmBeO1SWjr3CKR35xUMvfXM1+t4ceNez2MSEREJDQUFjl+9dYi8gqKABjQMZlrxqrffCQKyZtiRQ7HhUd15ajuvrv0y755iYiINFQvzl7LD/4hKmOjjb+fO5CYaKV+kUhXVcJeVJTxt3MGkhDr+3VevnU/T0xP8zgqERER76zZcYB/fLEi0L5pQi9S2yV5GJHUJSX0EhG6tmzMXSeUdL15anoaP21W1xsREWl4Coscd721mFz/t9X92idx3bjKBheUcKeEXiLGpaO7cWQ3340+BUWOX725mPxCdb0REZGG5V9z05m3bjcAMVG+rjax6moT0XR1JWL4ut4MIj7G92u9dMs+np6x2uOoRERE6k/6zkz+9vnyQPv68T3p3yHZw4ikPiihl4jSvVVj7pzUJ9CePG0Vy7fu8zAiERGR+lFU5Ljr7cXk5Pu+nU5t15Qbx/f0OCqpD0roJeJcfnR3hnRpBkB+oePONxep642IiES8175dx3drMwCIjjL+fs4g4mKU6jUEusoSccq+iS3ZtI9/zlzjcVQiIiJ1Z0NGFn/5tKSrzbXHpjCgk7raNBRK6CUi9WzThNuP7x1oP/a/Vazctt/DiEREROpGkX9Um6w833ycvdo04eaJvTyOSuqTEnqJWFce3Z1B/upEXmERv3pzEQXqeiMiIhHmP9+t5+s1uwCIMvjHuYOIj4n2OCqpT0roJWLFREfx93MHEecfqmvRxr08P3utx1GJiIgEz8bdWfz5k2WB9tVjezCoczPvAhJPKKGXiNa7bVNuOa7ka8eHv1xJ2vYDHkYkIiISHM45/u+dH8n0d7Xp0boxtx6nrjYNkRJ6iXhXj03hiI6+6a7zCor41VuLKCxyHkclIiJSO69/v4FZq3YCYAZ/O2cQCbHqatMQKaGXiBcbHcXfzxlEbLQB8MP6Pbw0R11vREQkfG3ek82DH5d0tbny6O4M69rcw4jES0ropUHo2z6JG8eXfA35989XsGaHut6IiEj4Ke5qsz+3APBNqnhHqUkVpeFRQi8NxvXje9Cvva/rTW5BEf/3zo84p643IiISXt5fuJmvVu4AirvaDFRXmwZOCb00GLHRUfz93IHERPm63ny7NoMPFm32OCoREZHq25+Tz4OlRrW5dHQ3juzWwsOIJBQooZcGpX+HZC4d3S3Q/tMnyzjg/8pSREQk1D32v1Xs2J8LQNukeHW1EUAJvTRAtxzXi9ZN4wHYti+Xx6eu8jgiERGRqq3ctp+X5qYH2nf/rC9N4mO8C0hChhJ6aXCaJsRyz8/6Btovzl7Lqm37PYxIRESkcs45fv/+ksCwy0d1b8Fpgzp4HJWECiX00iCdPrgDI/x9DguKHH/44CfdICsiIiHrw8Vb+GZNBgDRUcb9px+BmXkclYQKJfTSIJkZ953en2j/DbJzV+/i4x+3eByViIjIoQ7kFvDgx0sD7UtGdaNPu6YeRiShRgm9NFh92ydx0ciugfYfP1pGpm6QFRGREDN56iq27fPdCNuqSTy3Ht+riiOkoVFCLw3abcf3plWTOAC27sth8rQ0jyMSEREpkbZ9Py/MLpnd/O6fpZKUEOthRBKKlNBLg5bcKJbfnFRyg+wLs9ewWjPIiohICHDOce8HSynw3wh7ZLfmnDmko8dRSShSQi8N3llDOjKsa3MA8gsd9+oGWRERCQGfLtnK7LSdAEQZ3HeaboSV8imhlwYvKsq4//T++O+PZdaqnXz+01ZvgxIRkQYtK6+AP35UciPsxaO60a9DkocRSShTQi+CbwbZC0vdIPvAR8vIziv0MCIREWnInpiWxua9OQC0ahLHbcf39jgiCWVK6EX87ji+Dy0b+26Q3bQnmyen6wZZERGpf2t2HOC5WWsC7V+fmEpyI90IKxVTQi/il5wYy69PTA20/zlzDWt3ZnoYkYiINDTOOe79cCn5hb57uYZ2acbZQzt5HJWEOiX0IqWcM6wTgzs3AyCvsIj7PtQNsiIiUn8+/2kbM1fuAMAM7j/9CKKidCOsVE4JvUgpxTfIFg8iMGPFDr5cus3boEREpEHIzivkgVI3wv7yqC4c0THZw4gkXCihFyljYKdmXDCiS6B9/0dLycnXDbIiIlK3npqRxqY92QA0T4zlzkl9PI5IwoUSepFy/GpSH5ol+m5A2rg7m6dmrPY4IhERiWTpOzN59quDb4RtlhjnYUQSTpTQi5SjeeM47jqh5AbZZ75azbpdukFWRESCzznHfR/+RF5hEQCDOjfjvOGdPY5KwklQEnozO8fMJpvZLDPbZ2bOzF6rwXnS/ceW96hwph8zG21mn5hZhpllmdliM7vVzKJr98qkIfv5kZ0Z2MnXdzGvoIj7P1xaxREiIiKH73/LtjN9RcmNsA+c3l83wsphiQnSeX4LDAIOABuB1Mp3r9Re4NFy1h8ob2czOx14G8gBXgcygFOBR4AxwLm1iEUasOgo4/7Tj+DMp+bgHExdvp2ZK3cwtndrr0MTEZEIkVdQxIMflxSMzj+yCwM7NfMuIAlLwUrob8OXyKcBxwLTa3GuPc65e6uzo5klAc8BhcA459w8//rfAdOAc8zsfOfclFrEIw3Y4M7NOG9YZ16ftwGAP32yjDE9WxGtyomIiATBv79dR/quLACSEmK46wTdCCuHLyhdbpxz051zq1z9D9h9DtAamFKczPvjycH3rQHAdfUck0SYOyb1plGsr/fW8q37eXvBRo8jEhGRSLA3O5/Hpq4KtG+e2IvmjXUjrBy+YFXogynezC4EugCZwGJgpnOuvHEDJ/iXn5WzbSaQBYw2s3jnXG5lT2pm8yvYVJvuQxIB2iQlcPXYlMCb7kNfrODUgR1oFKdbNEREpOaempHGnqx8ADq3aMRFo7p6HJGEq1Ac5aYd8CrwIL6+9NOAVWZ2bDn7Fn8vtbLsBudcAbAW34eWlDqJVBqMq8em0LppPADb9uXy/Kw1VRwhIiJSsY27s3hpTnqgfdcJqcTHqFAkNRNqCf1LwER8SX1jYADwLNAN+NTMBpXZv3j6tL0VnK94fbOqntg5N6y8B7D88F6CRKLG8THcfnzvQPuZr1azY3+lX/qIiIhU6B+fryCvoGSYylMGtvc4IglnIZXQO+fuc85Nc85tc85lOeeWOOeuBR4GGgH3HuYpi+9crO++/RKBzh3Wid5tmwCQmVfIo/875IshERGRKi3euIf3Fm4OtO/5WV/MNNiC1FxIJfSVeMa/HFtmfXEFPpnyJZXZT6TGYqKj+L+T+gbaU77fQNr2/R5GJCIi4cY5x58+WRZoT+rXlhHdW3gYkUSCcEnot/uXjcusX+Ff9i6zHjOLAboDBYA6PEtQjOvTmjE9WwJQWOT4y6fqkSUiItU3ddl2vlmTAUBMlPGbkzT2htReuCT0o/zLson5NP/yxHKOGQskAnOrGuFGpLrMjP87qS/F34z+b9l2vl69y9ugREQkLBQUFvHnT0uq8784qgsprZt4GJFEinpP6M0s1sxSzaxHmfX9zeyQ75zMrCvwhL/5WpnNbwE7gfPNbHipYxKAP/qbTwcteBHgiI7JnDmkY6D9p0+WUVSk2zRERKRyU77fwOodmQA0iY/hlom9PI5IIkVQxqE3szOAM/zNdv7lKDN72f/zTufcnf6fOwLLgHX4Rq8pdi7wGzObjm+4yf1AD+BkIAH4BPhH6ed1zu0zs6vwJfYzzGwKkAGchm9Iy7eA14PxGkVKu3NSHz5evIXcgiJ+3LSXDxdv5vTBHas+UEREGqQDuQUHDaZw3bgetGwS72FEEkmCNbHUYOCSMutSKBn/fR1wJ5Wbji8JH4Kvi01jYA8wG9+49K+WNxOtc+49/xj19wBn40v+04Dbgcc9mL1WGoAOzRpxxdHdeWrGagD+9tkKTujfjoRYjSEsIiKHevar1ew8kAdA++QErji6u8cRSSQJSkLvnLuXag4p6ZxLp2Q4ydLrvwK+quHzzwF+VpNjRWrq2nE9mPL9BjIy89i0J5t/zU3nmmN7VH2giIg0KFv35vBcqQkJ75zURwUgCapwuSlWJOQkJcRy63El/R+fmJ7G7sw8DyMSEZFQ9NAXK8jJ900i1a990kH3YYkEgxJ6kVq4YEQXUlr5RlPdn1PA49NWeRyRiIiEkmVb9vHWgo2B9j0n9yUqSpNISXApoRephdjoKO46sWQM4Ve/XsfanZkeRiQiIqHkT58so/huPt9cJq28DUgikhJ6kVo6oX9bjuzWHICCIsffPtNkUyIiAl+t3MGsVTsBiDIOmm1cJJiU0IvUkplx989K3qQ/XbKV+esyPIxIRES8Vljk+PMnJZNInTe8M33aNfUwIolkSuhFgmBIl+acMrB9oP3gx8vQiKkiIg3X2/M3snzrfgAaxUZz+/G9PY5IIpkSepEgueuEVGKjfTc6LVi/h0+XbPU4IhER8UJWXgEPfbki0L56bAptkhI8jEginRJ6kSDp0jKRS0Z1C7T/+tly8gqKvAtIREQ88fystWzblwtA66bxXD02pYojRGpHCb1IEN04oSdJCb752tbtyuK1b9Z5HJGIiNSnnQdyefar1YH27cf3pnF8UObxFKmQEnqRIGqWGMfNE0smm3pyehqZuQUeRiQiIvXpyelpZOYVAtCrTRPOHdbJ44ikIVBCLxJkF43qSodkX1/JXZl5vDRnrccRiYhIfdi0J5t/f7M+0L7rxFRiopVqSd3Tb5lIkMXHRHPLcSVV+mdnrmFvVr6HEYmISH2YPHUVeYW+e6cGd27GcX3beByRNBRK6EXqwNlDO9G9VWMA9ucU8OzM1VUcISIi4WztzkzenL8x0L7rhD6YmYcRSUOihF6kDsRER3FbqTGHX5qTzvb9OR5GJCIidemRL1dSWOSbf2R0j5aM7tnK44ikIVFCL1JHThnQnlT/rIDZ+YU8NV1VehGRSLR08z4+WLQ50L7zhD4eRiMNkRJ6kToSFWX8qtSb+n++Xc/G3VkeRiQiInXh4VKTSB3Xty1DuzT3MBppiJTQi9ShCaltGNKlGQB5hUU8PnWVtwGJiEhQLVi/m/8t2w6AGdwxqXcVR4gEnxJ6kTpkdnCV/u0Fm1i944CHEYmISDD94/OS6vypAzvQt32Sh9FIQ6WEXqSOje7RiqP9N0cVFjke+XKlxxGJiEgwzEnbydzVuwCIjrKDBkMQqU9K6EXqQekbpD5avIWlm/d5GI2IiNSWc46/l6rOnzusZLhikfqmhF6kHgzu3Izj+7UNtB/6YkUle4uISKj737LtLNywB4C46Chuntir8gNE6pASepF6csek3hTPMTJ1+Xbmr9vtbUAiIlIjRUXuoMLMhSO70qFZIw8jkoZOCb1IPUltl8RpgzoE2n//fDnOOQ8jEhGRmvhw8WaWb90PQGJcNNeP7+FxRNLQKaEXqUe3Hdeb6Chfmf6bNRnMSdvlcUQiInI48guLDhrc4PIx3WnVJN7DiESU0IvUq26tGnPe8M6Btqr0IiLh5e35G0nf5ZskMCkhhqvGpngckYgSepF6d/PEnsTF+P7rLdq4ly+XbvM4IhERqY6c/EIeKzVB4DXH9iC5UayHEYn4KKEXqWftkxtx0ciugfZDX6yksEhVehGRUPefb9ezZW8OAK2axHHZmG7eBiTip4RexAPXjetBYlw0ACu27efDRZs9jkhERCqTmVvAk9PTAu0bxvckMS7Gw4hESiihF/FAqybxXHF090D7kf+tJL+wyMOIRESkMi/PTWdXZh4AHZIT+MVRXTyOSKSEEnoRj1x5TEqg7+W6XVm8OW+jxxGJiEh59mbl88xXqwPtW4/rTXxMtIcRiRxMCb2IR5IbxXLNsSWjIzw+dRU5+YUeRiQiIuV5duZq9ucUAJDSqjFnDe3ocUQiB1NCL+KhS0d3C4xfvHVfDq99s87jiEREpLQd+3N5aU56oH3b8b2JiVb6JKFFv5EiHkqMi+GmCT0D7admrCYzt8DDiEREpLSnZqSR7f/2tG/7JE4e0N7jiEQOpYRexGPnj+hMx2aNAMjIzOOVr1WlFxEJBdv25fDvb9cH2ndO6k2Uf7ZvkVASlITezM4xs8lmNsvM9pmZM7PXDvMcLc3sSjN718zSzCzbzPaa2Wwzu8LMDonVzLr5n6uix5RgvD6RuhQfE82Npar0/5y5mgOq0ouIeO7pGavJK/CNQDaoczMmpLbxOCKR8gVrANXfAoOAA8BGILUG5zgXeBrYAkwH1gNtgbOA54GTzOxc51x5M/AsAt4rZ/2SGsQhUu/OHtqJJ6ensXF3Nruz8nnl63SuH9ez6gNFRKRObN2bw3++K6nO33pcL8xUnZfQFKyE/jZ8iXwacCy+hPxwrQROAz52zgUG5Dazu4HvgLPxJfdvl3PsQufcvTV4TpGQEBcTxY3je/Kbd34E4J8z13DxqG40idekJSIiXnh6RlqgOj+4czPG9W7tcUQiFQtKlxvn3HTn3KoKqufVPcc059yHpZN5//qtwDP+5rhahCkS0s4e1onOLXx96fdk5fOvueneBiQi0kBt2ZvNf7/bEGirOi+hLlxuis33LyvqWNzBzK4xs7v9y4H1FZhIsMRGR3HT+F6B9nOz1rA/J7+SI0REpC48NX01ef7Zu4d0acaxqs5LiAv57/PNLAa42N/8rILdjvc/Sh83A7jEObe+3CMOfZ75FWyqyf0AIjVy5tCOPDE9jfUZWYEq/Y0TelV9oIiIBMXmPdm8/n3p6nxvVecl5IVDhf4vwBHAJ865z8tsywIeAIYBzf2P4j7844CpZta4/kIVqZ3Y6KiDRrx5btZa9qlKLyJSb56akRaozg/t0oyxvVp5HJFI1UI6oTezm4E7gOXARWW3O+e2O+d+75xb4Jzb43/MBCYB3wI9gSur81zOuWHlPfzPLVJvzhrSka4tEwHYm53Pv0rNUCgiInVnU5nq/G3Hqzov4SFkE3ozuwF4DFgKjHfOZVT3WOdcAb6hLgHG1kF4InUmJto34k2x52atUZVeRKQePDU9jfxC3/gew7o25+ieqs5LeAjJhN7MbgWewDeO/Hj/SDeHa4d/qS43EnbOHNKRbv4q/b6cAl6ane5tQCIiEW7j7izemFeqOq++8xJGQi6hN7NfA48AC/El89treKqR/uWaYMQlUp9ioqO4qdTNsC/MXsPebFXpRUTqypPTVweq80d2a86Yni09jkik+uo9oTezWDNLNbMe5Wz7Hb6bYOcDE51zO6s411FmFlfO+gn4JrsCeC0IYYvUu9MHd6B7K98XTPtyCnhpzlqPIxIRiUwbd2fx5jyNbCPhKyjDVprZGcAZ/mY7/3KUmb3s/3mnc+5O/88dgWXAOqBbqXNcAtwPFAKzgJvL+c+U7px7uVT7r0B//xCVG/3rBgIT/D//zjk3t2avSsRbvip9T25/YxEAL8xey2VjupPcKNbjyEREIsuT09MoKPJV50d0a8HoHqrOS3gJ1jj0g4FLyqxL8T/Al7zfSeW6+5fRwK0V7PMV8HKp9qvAmcCRwElALLANeAN4wjk3q8rIRULYaYM68MS0NNbszGR/TgEvzl7Lbcf39josEZGIsSEjizfnbQy0bz1es8JK+AlKlxvn3L3OOavk0a3Uvull11XzHOacG1fmmBecc6c457o555o45+Kdc12ccz9XMi+RICY6ipsmlox48+LstezNUl96EZFgOag6370Fo1JUnZfwE3I3xYrIwU4b1JGU1r6+9PtzC3hBfelFRIJiQ0YWb80vqc5rZBsJV0roRUJcdJRxy8SSEW9eUpVeRCQoJk9bFajOj0xpwSj1nZcwpYReJAycMrADPUpV6Z+frdFYRURqY/2uLN5esCnQvvU43Z8k4UsJvUgYiI4ybi5dpZ+Tzp6sPA8jEhEJb5OnraLQX50fldKSkeo7L2FMCb1ImDhlYAd6tmkCwIHcAp6fpb70IiI1kb4zk3d+KF2d71XJ3iKhTwm9SJg4tEq/lt2ZqtKLiByuydPSAtX50T1acpSq8xLmlNCLhJGTB7Snl79Kn5lXqL70IiKHKX1nJu8tVN95iSxK6EXCSNkq/ctz0slQlV5EpNoeL9V3fkzPlozo3sLjiERqTwm9SJg5eUB7erctVaWfpSq9iEh1pO/M5L1SfedvU3VeIoQSepEwE1WmSv/K1+s04o2ISDU8OT0Nf3Geo3u2Yng3VeclMiihFwlDPzuipC/9gdwCXpyT7m1AIiIhbkNG1kEj29yikW0kgiihFwlDUVHGjRN6BtovzVnL3mzNHisiUpGnZqQdNO78karOSwRRQi8Spk4Z2IGUVv7ZY3MK+NfcdG8DEhEJURt3Z/HW/I2BduluiyKRQAm9SJiKLlOlf2H2WvbnqEovIlLWM1+tJr/QV50f0a0FI1NUnZfIooReJIydNqgDXVsmArA3O59Xvl7ncUQiIqFly95s3vj+4Oq8mXkYkUjwKaEXCWMx0VHcML6kSv/8rDVk5hZ4GJGISGh59qs15BUWATC0SzPG9NSssBJ5lNCLhLkzh3SkU/NGAOzOyue1b1SlFxEB2L4vh/98tz7QVnVeIpUSepEwF1umSv/PmWvIziv0MCIRkdDw7Mw15BX4qvODOiVzbO/WHkckUjeU0ItEgLOHdqJDcgIAuzLz+Pe3qtKLSMO2Y3/uQe+Fqs5LJFNCLxIB4mKiuK5Ulf7ZmWvIyVeVXkQarudnrSEn31ed798hiQmpbTyOSKTuKKEXiRDnDe9EuyRflX7H/lymlOo3KiLSkOw6kHvQqF+qzkukU0IvEiHiY6K59tiUQPvpr1arSi8iDdILs9eS7X//S23XlOP7tvU4IpG6pYReJIKcP6ILrZvGA7BtXy5vlpoZUUSkIdiTlXfQzNk3T+xFVJSq8xLZlNCLRJCE2GiuGVuqSj89LTDCg4hIQ/Di7LVk+kf66tWmCSf2b+dxRCJ1Twm9SIT55VFdadUkDoDNe3N4e4Gq9CLSMOzNzuelOemB9k2qzksDoYReJMI0iovmqmNKqvRPTk8jv1BVehGJfC/PSWe/f7bslNaNOXlAe48jEqkfSuhFItCFI7vSPDEWgI27s3n3h00eRyQiUrf25+Tzwuw1gfZNE3oSreq8NBBK6EUiUOP4GK4sU6UvUJVeRCLYK1+vY1+OrzrfrWUipw7s4HFEIvVHCb1IhLp4VFeSG/mq9Ot2ZfHBos0eRyQiUjcO5Bbw3KyS6vwN43sSE60URxoO/baLRKimCbFccXT3QPuJaWkUFjkPIxIRqRuvfbOOPVn5AHRu0YgzhnT0OCKR+qWEXiSCXTK6G00TYgBYszOTjxarSi8ikSUrr4DnZpaqzo/rSayq89LA6DdeJIIlN4rlsjElVfrJ09IoUpVeRCLIf75dz67MPAA6NmvEWUM7eRyRSP1TQi8S4S4f040m8b4qfdr2A3y6ZKvHEYmIBEdOfiHPfFVSnb9uXA/iYpTaSMOj33qRCNcsMY5LRncNtB+fukpVehGJCP/9bj07D+QC0C4pgXOHqzovDZMSepEG4IqjU0iMiwZgxbb9fLF0m8cRiYjUjq86vzrQvvbYFOJjoj2MSMQ7QUnozewcM5tsZrPMbJ+ZOTN7rYbn6mRmL5rZZjPLNbN0M3vUzJpXcsxoM/vEzDLMLMvMFpvZrWam/9kiQIvGcVw0qqRKP3naKpxTlV5Ewteb8zeybZ+vOt+maTznj+jicUQi3glWhf63wI3AYKDGU1KaWQ9gPnAZ8B3wCLAGuAX42sxalnPM6cBMYCzwLvAkEOc/dkpNYxGJNFcdk0JCrO+//E+b9zFt+XaPIxIRqZm8giKenp4WaF89NoWEWNXwpOEKVkJ/G9AbSAKuq8V5ngLaADc7585wzv3GOTcBX3LeB3iw9M5mlgQ8BxQC45xzVzjnfoXvg8XXwDlmdn4t4hGJGK2axPPLo0r1pZ+Wpiq9iISldxZsZPPeHABaNYk76L1NpCEKSkLvnJvunFvlapEdmFkKMAlIx1dlL+0PQCZwkZk1LrX+HKA1MMU5N69UPDn4vjWA2n3AEIko14xNCYwAsWjDHmau2ulxRCIihye/sIgnZ5RU5688JoVGcarOS8MWSjfFTvAvv3DOFZXe4JzbD8wBEoGR5RzzWTnnmwlkAaPNLL6qJzez+eU9gNTDfSEioapNUgIXHNk50H58qvrSi0h4eX/hZjZkZAPQPDGWi0aqOi8SSgl9H/9yZQXbV/mXvatzjHOuAFgLxAApwQhQJBJcc2wPYqMNgPnrdvP16l0eRyQiUj2FRY4nS/Wdv+Lo7jT2z7Mh0pCFUkKf7F/urWB78fpmtTymXM65YeU9gOVVHSsSTjo0a8S5w0tV6aetqmRvEZHQ8dHizazdmQlAUkIMF4/u5m1AIiEilBL6qph/eTj9A2pyjEjEu+7YHsRE+f57fLMmg+/WZngckYhI5YqKHJOnlVTnLxvTnaSEWA8jEgkdoZTQF1fTkyvYnlRmv5oeI9LgdW6RyJlDOgbak1WlF5EQ9+mSraRtPwBAk/gYLh/T3eOIREJHKCX0K/zL3hVs7+Vflu4vX+ExZhYDdAcK8I1lLyKl3DC+J/4iPbNW7WTB+t3eBiQiUgFfdb6k8HDJ6K4kJ6o6L1IslBL66f7lJDM7KC4zawqMAbKBb0ptmuZfnljO+cbiGxVnrnMuN8ixioS9bq0ac/rgUlX6qarSi0ho+nLZNpZv3Q9AYlw0VxytsS5ESqv3hN7MYs0s1T8rbIBzbjXwBdANuKHMYfcBjYFXnHOZpda/BewEzjez4aWeIwH4o7/5dHBfgUjkuGF8T8xfpZ++YgeLN+7xNB4RkbKcczxequBw0ciutGgc52FEIqEnKGM9mdkZwBn+Zjv/cpSZvez/eadz7k7/zx2BZcA6fMl7adcDc4HHzWyif7+jgPH4utrcU3pn59w+M7sKX2I/w8ymABnAafiGtHwLeL3WL1AkQvVs04STB7Tno8VbAJg8LY3nLh5exVEiIvVn+ort/LR5HwAJsVFceYyq8yJlBatCPxi4xP84wb8updS6c6pzEn+VfjjwMr5E/g6gB/A4MMo5d8iA2c6594Bj8U0kdTZwE5AP3A6cX5vZa0Uagpsm9Ar8/OXSbSz1/+EUEfGac47HppaMbPOLEV1p3bTKuSJFGpygJPTOuXudc1bJo1upfdPLritzrg3Oucucc+2dc3HOua7OuVuccxWOq+ecm+Oc+5lzrrlzrpFzboBz7hHnXGEwXp9IJOvTrikn9m8XaD8xXX3pRSQ0zFq1k0Ub9gAQFxPFNceqOi9SnlC6KVZEPHLTxJ6Bnz/5cSsrt+33MBoRkUP7zp9/ZGfaJiV4GJFI6FJCLyL075DMcX3bBNpPlJq8RUTEC1+v2cW8db7hdGOjjWuP7VHFESINlxJ6EQEO7kv/4eLNrN5xwMNoRKShK12dP2dYZzo0a+RhNCKhTQm9iAAwqHMzju3dGgDn4MnpqtKLiDe+W5vBN2t8t85FRxnXj1N1XqQySuhFJODmiSVV+vcXbmbdrsxK9hYRqRulZ4U9a0hHOrdI9DAakdCnhF5EAoZ1bc6Yni0BKCxyPDV9tccRiUhDs2D9bmat2glAlPkmwBORyimhF5GD3FyqL/3bCzayISPLw2hEpKGZXKrv/OmDO9KtVWMPoxEJD0roReQgR6W0ZET3FgAUFDme+UpVehGpHz9u3Mv0FTsAMFXnRapNCb2IHOKWUn3p35i3gS17sz2MRkQaisdL9Z0/eUB7erZp4mE0IuFDCb2IHGJ0j5YM7dIMgPxCx9MzVKUXkbr10+a9fLl0W6B94wRV50WqSwm9iBzCzA4a8WbKdxvYujfHw4hEJNKVHnf+xP7tSG2X5GE0IuFFCb2IlOvY3q0Z1LkZAHmFRepLLyJ1ZtmWfXz+U0l1vnRBQUSqpoReRMplZtxa6o/qf75bz7Z9qtKLSPCVrs5P6teWfh1UnRc5HEroRaRC4/q0ZmCnZADyClSlF5HgW751H58u2RpoqzovcviU0ItIhczsoBFv/vPterarSi8iQTR5alrg5+P6tuWIjskeRiMSnpTQi0ilJqS2YYD/D2xuQRHPzlzjcUQiEilWbtvPJ0u2BNq3qDovUiNK6EWkUmVHvPn3t+vYsT/Xw4hEJFI8PnUVzvl+npjahgGdVJ0XqQkl9CJSpeP6tqG//ya1nPwi/jlTfelFpHZWbdvPxz+Wqs4fp+q8SE0poReRKpXtS//qN+vYeUBVehGpucnT0gLV+QmpbRjYqZmn8YiEMyX0IlItx/drS7/2JVX659SXXkRqKG37AT5cvDnQVt95kdpRQi8i1VK2L/0rX69jl6r0IlIDT0wr6Ts/rk/JJHYiUjNK6EWk2ib1a0tqu6YAZOcX8tystR5HJCLhZvWOA3ywSNV5kWBSQi8i1RYVdXBf+le+TicjM8/DiEQk3DwxLY0if3V+bO/WDOnS3NuARCKAEnoROSwn9G9Hn7a+Kn1WXiHPzVJfehGpnjU7DvD+wk2BtqrzIsGhhF5EDktUVJm+9HPT2a0qvYhUwxPTS6rzx/RqxbCuqs6LBIMSehE5bCcd0Y7ebZsAkJlXyPOzVaUXkcql78zk/YXqOy9SF5TQi8hhi4oybppQ8sf4X3PXsSdLVXoRqdgT09Mo9Jfnx/RsyfBuLTyOSCRyKKEXkRr52YD29Gzjq9IfyC3ghdka8UZEyrduVybv/lC673xvD6MRiTxK6EWkRqKjjJsm9Ay0X56Tzt6sfA8jEpFQ9WSp6vyolJaM6K7qvEgwKaEXkRo7ZWAHerRuDMD+3AJemKMqvYgcbENGFu8sKFWdP05950WCTQm9iNRYdJm+9C/NWcvebFXpRaTEk9PTKPBX54/q3oKRKS09jkgk8iihF5FaOXVQB1Ja+av0OQW8pCq9iPhtyMjirfkbA21V50XqhhJ6EamV6CjjxlJ96V+cvZZ9OarSiwg8NWN1oDo/olsLRqk6L1InlNCLSK2dNqgD3f1V+n05Bbw8J93bgETEc5v2ZPPW/A2B9q3H9cLMPIxIJHIFLaE3s05m9qKZbTazXDNLN7NHzaxa08CZ2aVm5qp4FJY5plsV+08J1usTkYrFREdx4/iSKv3zs9aoL71IA/fEtFXkF/qq80d2a86oHqrOi9SVmGCcxMx6AHOBNsD7wHJgBHALcKKZjXHO7ariNAuB+yrYdgwwAfi0gu2LgPfKWb+kiucUkSA5fXAHnpiextqdmezLKeCFWWu4fVIfr8MSEQ+s35XFm/NK+s7felxvVedF6lBQEnrgKXzJ/M3OucnFK83sYeA24EHg2spO4JxbiC+pP4SZfe3/8Z8VHL7QOXfvYUUsIkEVEx3FLRN7cevrCwF4cU46l43pTvPGcd4GJiL17rGpqw4a2Wa0qvMidarWXW7MLAWYBKQDT5bZ/AcgE7jIzBrX8PxHACOBTcDHNY9UROraqYM60KvU7LHPzlzjcUQiUt/W7DjAuz+UVOfvmNRH1XmROhaMPvQT/MsvnHNFpTc45/YDc4BEfEl5TVzjX77gnCusYJ8OZnaNmd3tXw6s4XOJSC1ERxm3Hlcypfu/5qazY3+uhxGJSH17bOoq/MV5junVSrPCitSDYHS5Ke4ku7KC7avwVfB7A1MP58Rm1gi4ECgCnq9k1+P9j9LHzgAucc6tr+Zzza9gU2p1jhcRn5OOaEdqu6Ys37qf7PxCnv1qNb89pZ/XYYlIPVi5bT8fLNocaN92fO9K9haRYAlGhT7Zv9xbwfbi9c1qcO7z/Md96pzbUM72LOABYBjQ3P84FpgOjAOm1rSrj4jUTFSUHfRH/NVv1rFtX46HEYlIfXn0fytx/ur8+D6tGdqlWgPdiUgt1cc49MUd51wNjr3av3y2vI3Oue3Oud875xY45/b4HzPxfSPwLdATuLI6T+ScG1beA9+IPSJyGCb1a8uAjr7P+rkFRTw1Pc3jiESkrv20eS+f/Lg10L79eI1yJVJfgpHQF1fgkyvYnlRmv2oxs37AaGAj8MnhHOucK6Cki87YwzlWRGrPzLi9VJX+v99tYNOebA8jEpG69uj/VgV+ntSvLQM6VZQWiEiwBSOhX+FfVtRRrpd/WVEf+4pU52bYyuzwL9XlRsQD4/q0ZkiXZgDkFRbxxDRV6UUi1eKNe/hy6bZAW33nRepXMBL66f7lJDM76Hxm1hQYA2QD31T3hGaWAFyE72bYF2oYV/GoOho3T8QDZsYdpb5yf3PeBtbvyvIwIhGpKw9/WVKzO3lge/q2T6pkbxEJtlon9M651cAXQDfghjKb78NXIX/FOZcJYGaxZpbqn122Iufiu8H1kwpuhsV/rqPM7JBZa8xsAr4JrQBeq+5rEZHgGtOzZWDIuoIix+Rpq6o4QkTCzfx1u5mxwveluBncdlyvKo4QkWAL1k2x1wPbgcfN7D0z+7OZTcOXVK8E7im1b0dgGZUPYVl8M2xFM8MW+yuwyczeNLNH/I+p/nPHA79zzs2twesRkSAo25f+nR82sXZnpocRiUiwPfzlisDPpw/qQM82TT2MRqRhCkpC76/SDwdeBo4C7gB6AI8Do5xzu6p7LjPrCxxN9W6GfRXfaDZHAlfh+2DRC3gDGOuc++NhvRARCbqRKS0Z09M37XthkeOx/x3u7TQiEqq+WbOLOWm+P/HRUcYtx6nvvIgXgjGxFAD+rjGXVWO/dEqGsixv+7LKtpfZ9wVq3sdeROrJ7cf3YU6a78uy9xdt5obxPenVVlU8kXDmnDuo7/xZQzrSvZXGoRDxQn2MQy8iDdywrs0Z16c1AM4dPLydiISnOWm7+G5tBgAxUcbNE9V3XsQrSuhFpF6U7kv/8Y9bWLp5n4fRiEhtOOd4qFTf+fOO7EznFokeRiTSsCmhF5F6MbBTM47v1zbQflR96UXC1owVO/hh/R4A4qKjuHF8T28DEmnglNCLSL25rdQNc18s3caPGw9rAmkRCQFl+87/4qgudGjWyMOIREQJvYjUm34dkvjZgHaBdunh7kQkPHy5dBs/bvJ9GI+PieL6cZVNKyMi9UEJvYjUq1uP6435x7GavmIH89ft9jYgEam2oqKDq/MXjexKm6QEDyMSEVBCLyL1rHfbppw2qEOg/ciX6ksvEi4+XbKV5Vv3A5AYF821qs6LhAQl9CJS726Z2Isof5V+dtpOvl1T7bnnRMQjhUWOR0rdzH7J6G60ahLvYUQiUkwJvYjUu5TWTThraKdA+6EvV+Kc8zAiEanKR4s3k7b9AABN4mO4+pgUjyMSkWJK6EXEEzdP6EWMv0z/3doMZq7a6XFEIlKR/MKig/rOX350d5o3jvMwIhEpTQm9iHiiS8tEzjuyc6D910+XU1SkKr1IKJry3XrW7coCILlRLFcc3d3jiESkNCX0IuKZWyb2IiHW9za0dMs+Ply82eOIRKSszNwCHpuaFmhfP64HyY1iPYxIRMpSQi8inmmblMBlY0oqfQ99sZK8giIPIxKRsl6cvZadB3IBaJ+cwCWju3kbkIgcQgm9iHjq2mNLqn3rM7KY8v16jyMSkWIZmXk8O3NNoH3rcb1IiI32MCIRKY8SehHxVHKjWG4YXzKW9eNTV5GZW+BhRCJS7MnpaRzw/3/s2aYJZ5canUpEQocSehHx3MWjutE+2Tfb5M4Debwwe63HEYnIxt1ZvPr1ukD7Vyf0ISZaaYNIKNL/TBHxXEJsNLcd1zvQfvar1ezy99kVEW88/OVK8gp997QM7dKMSf3aehyRiFRECb2IhISzhnakZ5smAGTmFfLE9LQqjhCRurJ86z7e/WFToP3rE1MxMw8jEpHKKKEXkZAQEx3Fr07oE2j/+5v1bMjI8jAikYbr75+toHjy5vF9WnNUSktvAxKRSimhF5GQMalfW4Z2aQZAXmERj5SamVJE6sf36RlMXb4dADO468RUjyMSkaoooReRkGFm/OakvoH2uws3sWzLPg8jEmlYnHP85dPlgfYZgzvSt32ShxGJSHUooReRkDKiewsmpLYBwDn422fLqzhCRILly6XbmL9uNwCx0cbtx/eu4ggRCQVK6EUk5Nx1Yh+K77+bvmIH367Z5W1AIg1AYZHj75+vCLQvHNmVzi0SPYxIRKpLCb2IhJzUdkmcObhjoP2Xz5bjiu/QE5E68faCjazafgCAJvEx3Di+p8cRiUh1KaEXkZB02/G9ifNPYvPD+j18sXSbxxGJRK6c/EIeLXUT+lXHpNCySbyHEYnI4VBCLyIhqXOLRC4c2TXQ/vvnKyjwT3IjIsH16tfr2Lw3B4BWTeK48pjuHkckIodDCb2IhKwbJ/SkSXwMAGnbD/DOgk1VHCEih2tvdj5PziiZyO2mCb1o7P9/JyLhQQm9iISsFo3juHpsSqD98Jcryckv9DAikcjz7Fer2ZOVD0CXFolcMKKLxxGJyOFSQi8iIe2Ko7vTyt+Xd+u+HP41N93bgEQiyLZ9Obw4Z22gfcek3sTFKDUQCTf6XysiIa1xfAw3TywZbeOpGavZm53vYUQikeOxqavIyffdm9KvfRKnDuzgcUQiUhNK6EUk5J1/ZBe6+MfD3pudzzNfrfY4IpHwt2bHAV7/fkOg/euTUomKMg8jEpGaUkIvIiEvLiaKO0/oE2i/NGctW/0jcohIzTz0xUoKi3zzO4xKacnYXq08jkhEakoJvYiEhVMGtKd/hyQAcvKLePjLFVUcISIVWbB+Nx//uCXQ/vVJqZipOi8SrpTQi0hYiIoyfnNSaqD95vyNLNm018OIRMJTUZHj/g+XBtonHdGOwZ2beReQiNRa0BJ6M+tkZi+a2WYzyzWzdDN71MyaH8Y50s3MVfDYWslxo83sEzPLMLMsM1tsZreaWXRwXp2IhIJjerVmQmobAJyD+z9cinPO46hEwsv7izaxcMMeAOKio/i/k/p6G5CI1FpQZo4wsx7AXKAN8D6wHBgB3AKcaGZjnHO7qnm6vcCj5aw/UMFznw68DeQArwMZwKnAI8AY4NxqvxARCXn3nNyXmSt3UFDk+C49g09+3MrJA9t7HZZIWMjKK+Cvn5Z0V7v86O50aZnoYUQiEgzBmgruKXzJ/M3OucnFK83sYeA24EHg2mqea49z7t7q7GhmScBzQCEwzjk3z7/+d8A04BwzO985N6W6L0REQluP1k24eFS3wNjZf/pkGRP7tiEhVl/IiVTlma/WsHWf74byVk3iuXFCzyqOEJFwUOsuN2aWAkwC0oEny2z+A5AJXGRmjWv7XOU4B2gNTClO5gGccznAb/3N6+rgeUXEQ7dM7EXzxFgANu3J5oXZa6s4QkQ27cnm2VJDvt51Qh+axAerriciXgpGH/oJ/uUXzrmi0hucc/uBOUAiMLKa54s3swvN7G4zu8XMxlfSF774uT8rZ9tMIAsYbWbx1XxuEQkDyYmx3H5870D7yelpbN+nYSxFKvOXT5eTW+D7M92/QxJnD+vkcUQiEizBSOiLB4deWcH2Vf5l7wq2l9UOeBVfN51H8XWdWWVmxx7OczvnCoC1+LoVpVT1pGY2v7wHkFrVsSJS/y4Y0YU+bZsCkJVXyN8+1zCWIhWZvy6DDxdtDrT/cGp/ojWJlEjECEZCn+xfVjR+XPH6ZtU410vARHxJfWNgAPAs0A341MwG1eFzi0gYiYmO4renlIzO8db8jSzeuMe7gERCVFGR475Sw1SePKA9I7q38DAiEQm2+hiHvrgEUOXYcs65+5xz05xz25xzWc65Jc65a4GHgUbAvXX43MPKe+AbsUdEQtAxvVpzXN82gbaGsRQ51Ds/bGLxRl99Ky4m6qD5HEQkMgQjoS+ugidXsD2pzH418Yx/OdaD5xaREHbPyf2IjfZ9dp+3bjcfLd5SxREiDUdmbgF/+6ykLnXVMd3p3ELDVIpEmmAk9MUdVyvqI9/Lv6yoj311bPcvy46UU+Fzm1kM0B0oANbU4rlFJIR1b9WYS0d3C7T/8ulycvILvQtIJIQ8PWM12/fnAtCmaTzXj9MwlSKRKBgJ/XT/cpKZHXQ+M2uKb3KnbOCbWjzHKP+ybGI+zb88sZxjxuIbXWeucy63Fs8tIiHupom9aNk4DvANzffcTH2GF9mQkcU/Z5X8X7jrxFQaa5hKkYhU64TeObca+ALfjas3lNl8H76q+ivOuUwAM4s1s1T/7LIBZtbfzA65S8fMugJP+Juvldn8FrATON/Mhpc6JgH4o7/5dE1el4iEj6SEWG6fVPJF3VMzVrN1r4axlIbtL58tJ88/TOXATsmcNaSjxxGJSF0J1k2x1+PrFvO4mb1nZn82s2n4ZoldCdxTat+OwDJgaplznAtsNrNPzewpM/urmb2F76bUnsAnwD9KH+Cc2wdcBUQDM8zseTP7G7AQX1X/LeD1IL1GEQlh5x/ZhdR2vmEss/ML+dvnup9dGq7v1mbwcan7SX5/Sj+iNEylSMQKSkLvr9IPB14GjgLuAHoAjwOjnHO7qnGa6cC7+Pq9/wK4HTgWmA1cApzinMsr57nf8+83EzgbuAnI9x9/vtOQFyINQnSU8ftT+wXa7yzYxMINe7wLSMQjRUWO+z/6KdA+dVAHhnfTMJUikSxonemccxuAy6qxXzolw0mWXv8V8FUNn3sO8LOaHCsikWN0j1ZM6teWL5ZuA+D+D3/i7etGY6bKpDQcby3YyJJN+wCI1zCVIg1CfYxDLyJSb+45uS9x0b63tgXr9/BBqdkxRSLdgdwC/l5q1uRrxqbQsVkjDyMSkfqghF5EIkrXlo257OhugfZfPl1Odp6GsZSG4cnpaezwD1PZNimea8f1qOIIEYkESuhFJOLcOL4nrZr4hrHcsjeHf2oYS2kANmRk8cKstYH2r09MJTFOw1SKNARK6EUk4jRNiOXOSX0C7We+Ws2WvdkeRiRS9/70yTLyCn3DVA7q3IwzBmuYSpGGQgm9iESkc4d3pl/7JMA3jOVfP9UwlhK5vlmzi0+XbA20/3CqhqkUaUiU0ItIRCo7jOV7Czczd/VODyMSqRt5BUX87r0lgfbpgzswtEtzDyMSkfqmhF5EItbIlJacPLB9oP3bd5eQk68bZCWy/HPmalZtPwBA47hoDVMp0gApoReRiPaHU/rRNN53Y+CanZk8NWO1xxGJBM/anZk8Pi0t0L5jUh/aJ2uYSpGGRgm9iES0NkkJ3FWqYvn0jDTS/NVMkXDmnOO37/1IXoHvRtgBHZO5ZHQ3b4MSEU8ooReRiPfLEV0Y0qUZAPmFjrvf/ZGiIudtUCK19O4Pm5iTtguAKIM/nzWAaN0IK9IgKaEXkYgXFWX8+awBxPiTne/WZvDW/I0eRyVSc7sz8/jjx8sC7cvGdOeIjskeRiQiXlJCLyINQmq7JK48JiXQfvCTZew8kOthRCI19+dPl5GRmQdAh+QEbj++t8cRiYiXlNCLSINxy8RedG7hu2Fwb3Y+D5aqcIqEi2/W7OKNeSXfMN1/+hE0jteMsCINmRJ6EWkwGsVF88DpRwTa7/6widmrNDa9hI/cgkLufvfHQPvE/u04rl9bDyMSkVCghF5EGpRxfdpw6qAOgfY97/2oseklbDw9YzVrdmQC0CQ+hntP6+9xRCISCpTQi0iD87tT+pKU4OuisG5XFk+UGsdbJFSt3nGAp6aXzKPwqxP60C45wcOIRCRUKKEXkQanTdMEfnNS30D7ma9Ws3Lbfg8jEqmcc4573v2RvELfmPODOjfjwpFdPY5KREKFEnoRaZDOP7Izw7s2B6CgyHH3OxqbXkLXW/M38s2aDACio4w/n6kx50WkhBJ6EWmQoqKMP5Uam37eut28Pm+Dx1GJHCojM48/fVIyItOVR3enX4ckDyMSkVCjhF5EGqzebZtyzbElY9P/+ZNlbN+f42FEIof648dL2Z2VD0DHZo245bheHkckIqFGCb2INGg3TehF15aJAOzLKeCPH2lsegkdc9N28s6CTYH2H888gsQ4jTkvIgdTQi8iDVpCbDQPnjEg0P5g0WZmrNjuYUQiPjn5hdzz3pJA++SB7Rnfp42HEYlIqFJCLyIN3tG9WnHmkI6B9u/eX0J2nsamF289NT2NtTt9Y843TYjhD6f08zgiEQlVSuhFRIB7Tu5LcqNYADZkZPPY1FUeRyQN2apt+3n6q5Ix5399YiptkjTmvIiUTwm9iAjQqkk8d/8sNdB+btYaftq818OIpKEqLHLc/e6P5Bf6hlEd2qUZvxjRxeOoRCSUKaEXEfE7b3hnRnRvAfiSqtteX0hOvrreSP16btYavk/fDUCMf3jVKI05LyKVUEIvIuJnZvz5rAEkxPreGlduO8DfPlvhcVTSkCzZtJeHvij5nbt+fE9S22nMeRGpnBJ6EZFSerRuwm9PLrn58MU5a5m5coeHEUlDkZ1XyK2vLwx0tRnUuRk3TejpcVQiEg6U0IuIlPHLo7owMbVkeMA731zE7sw8DyOShuAvny4jbfsBABrFRvPozwcTG60/0yJSNb1TiIiUYWb89ZyBtGoSB8D2/bn83zs/4pzzODKJVNNXbOdfX68LtH9/aj+6t2rsYUQiEk6U0IuIlKNVk3j+evbAQPuzn7by5vyNHkYkkWrXgVx+9ebiQPv4fm05/8jOHkYkIuFGCb2ISAUm9m3LL48qGS7wvg9+Yt2uTA8jkkjjnOM37/zIzgO5gO+D5F/OGoCZRrURkepTQi8iUonfntyPlNa+rg+ZeYXc9vpCCgqLPI5KIsXr32/gy6XbAu2/nzuQlk3iPYxIRMKREnoRkUo0ivPdnBjjHwd8wfo9PDl9dRVHiVRt7c5M7vtwaaB98aiujO/TppIjRETKp4ReRKQKAzs147bjewfaj09bxQ/rd3sYkYS7/MIibn19Idn+ict6tmnC/53U1+OoRCRcBS2hN7NOZvaimW02s1wzSzezR82seTWPb2lmV5rZu2aWZmbZZrbXzGab2RVmdkisZtbNzFwljynBen0i0rBde2wPjuzmezsrnkU2M7fA46gkXE2elsaiDXsAiI02Hv35YBrFRXsblIiErZhgnMTMegBzgTbA+8ByYARwC3CimY1xzu2q4jTnAk8DW4DpwHqgLXAW8Dxwkpmd68ofN24R8F4565cc/qsRETlUdJTx8HmDOemxWRzILSB9VxYPfLSUv5QaCUekOuav280T01YF2rcf34cjOiZ7GJGIhLugJPTAU/iS+Zudc5OLV5rZw8BtwIPAtVWcYyVwGvCxcy5wx5mZ3Q18B5yNL7l/u5xjFzrn7q3NCxARqUrnFoncf3p/bn9jEQBTvt/A+NQ2nNC/nceRSbg4kFvAba8vpMhfmjqqewuuHpvibVAiEvZq3eXGzFKASUA68GSZzX8AMoGLzKzSGTKcc9Occx+WTub967cCz/ib42obr4hIbZw5pCMnD2wfaP/m7cVs35fjYUQSTu774CfWZ2QB0DQhhod/PpjoKA1RKSK1E4w+9BP8yy/KScb3A3OARGBkLZ4j37+sqMNqBzO7xszu9i/1HbiI1Akz409nDKB9cgIAu7PyufOtxZpFVqr06Y9bDpqc7I9nHEHHZo08jEhEIkUwEvo+/uXKCrYXdxTsXcH2SplZDHCxv/lZBbsdj6+K/6B/ucjMpptZlwr2L+955pf3AFJrEreIRK7kxFgeOndQoD1z5Q7+NTfdu4Ak5G3dm8P/vftjoH3aoA6cPrijhxGJSCQJRkJffCfP3gq2F69vVsPz/wU4AvjEOfd5mW1ZwAPAMKC5/3EsvptqxwFTq+rqIyJSE6N7tuKqY7oH2n/+dDkrt+33MCIJVUVFjl+9tYg9Wb4vmzskJ/DAGUd4HJWIRJL6GIe+uHPgYX8fbWY3A3fgGzXnorLbnXPbnXO/d84tcM7t8T9m4uvT/y3QE7iyOs/lnBtW3sP/3CIih7jzhD6ktmsKQG5BETf8ewEHNJSllPHUjDRmrdoJgBk8dN5gkhvFehyViESSYCT0xRX4isbcSiqzX7WY2Q3AY8BSYLxzLqO6xzrnCvANdQkw9nCeV0SkuuJjonn8giHExfjeSldtP8CtUxZSVKT+9OLzxU9b+ccXJT1Srz4mhVE9WnoYkYhEomAk9Cv8y4r6yPfyLyvqY38IM7sVeALfOPLj/SPdHK4d/qW63IhInendtil/OnNAoP2/Zdt4+Mtqv91JBFuxdT+3vb4w0B6Z0oI7T+hT8QEiIjUUjIR+un85qexsrmbWFBgDZAPfVOdkZvZr4BFgIb5kfnsN4yoeVWdNDY8XEamWc4Z14sqjS/rTPzE9jQ8XbfYwIvHa7sw8rnplHpl5hQB0at6Ip345jNjo+ujpKiINTa3fWZxzq4EvgG7ADWU234evQv6Kcy4TwMxizSzVP7vsQczsd/hugp0PTHTO7azsuc3sKDOLK2f9BHwTWgG8dnivSETk8P3mpFTG9m4daP/qrUUs2XRYPQ0lQuQXFnH9vxcExptPjIvmuYuH06LxIX+uRESCIlgzxV4PzAUeN7OJwDLgKGA8vq4295Tat6N/+zp8HwIAMLNLgPuBQmAWcLPZIZNtpDvnXi7V/ivQ38xmAMWD+w6kZGz83znn5tbupYmIVC0mOorJFwzhzCfnsGZnJjn5RVz1yjw+uPFoWjeN9zo8qUd//GgpX6/ZFWg/fN5g+rZPquQIEZHaCUpC75xbbWbD8SXkJwI/A7YAjwP3VfOG1uLvq6OBWyvY5yvg5VLtV4EzgSOBk4BYYBvwBvCEc27WYb0QEZFaSG4Uy3OXDOeMJ+awP7eALXtzuPa1+fznqqOIj4n2OjypB//9bj3/+npdoH3bcb058Yh2HkYkIg2BaXbDypnZ/KFDhw6dP3++16GISJiYvmI7l7/8PcVvr+cN78Rfzx5IOd86SgT5bm0Gv3z+G/ILfRf+ZwPa8cQFQ4mK0nUXkeoZNmwYCxYsWOAfOr3adHeOiEiQje/Thv87qWSS6TfmbeRlzSQb0TbuzuK61+YHkvm+7ZP4x7mDlMyLSL1QQi8iUgeuOiaFs4Z0DLT/+PEyZq+q9D5/CVNZeQVc9cp8dmXmAdCycRzPXTyMxLhg3aYmIlI5JfQiInXAzPjTWQMY1LkZAIVFjhv+s4D0nZneBiZB5ZzjzjcXsWzLPgBio42nLxxGp+aJHkcmIg2JEnoRkTqSEBvNPy8aRhv/KDd7s/O58pV57M/J9zgyCZbJ09L45MeSuQ/vP/0IRnRv4WFEItIQKaEXEalDbZMS+OfFw4mL8b3dpm0/wK1TFlJYpAEJwt1nS7YeNCvwJaO6csGILh5GJCINlRJ6EZE6NrhzM/569oBAe+ry7Tz0xQoPI5LaWrZlH7e/sTDQHpXSkt+e0s+7gESkQVNCLyJSD84c0olrxqYE2k/NWM37Czd5GJHUVEZmHle9Mo+svEIAOrdoxFO/HEpstP6kiog39O4jIlJP7joxlXF9Wpe031rM3DSNfBNO9ufkc+W/vmfj7mwAGsdF8/zFR9K8cZzHkYlIQ6aEXkSknkRHGY9fMISU1o0ByC0o4vJ/fa+kPkzsz8nn0pe+Z8H6PQCYwaPnD6FPu6beBiYiDZ4SehGRepSUEMtLlx5Ju6QEAHLyldSHgwO5BVz60vfMX7c7sO4Pp/Tj+H5tPYxKRMRHCb2ISD3r2rIx/7165KFJ/Wol9aHoQG4Bl7z43cHJ/Kn9uHRMdw+jEhEpoYReRMQD3Vv5kvq2Sb4x6nPyi7j8ZSX1oaaiZP4yJfMiEkKU0IuIeKR7q8ZMuXqUkvoQVV4y//tTlMyLSOhRQi8i4qGKkvqvV+/yOLKG7UBuAZeWk8xffrSSeREJPUroRUQ8Vl5Sf9nL3ymp90hxMj9PybyIhAkl9CIiIaB7q8b896pD+9Qrqa9fB3ILuOylg5P53ymZF5EQp4ReRCREpLRuwn+vGkmbpr6kPju/UEl9PSpO5r9PPziZv0LJvIiEOCX0IiIhJKV1E6ZcfWhS/80aJfV1qbxk/rcn91UyLyJhQQm9iEiIKS+pv+wlJfV1JbOCZP7KY1I8jEpEpPqU0IuIhKCKkvoZK7Z7HFlk2XUgl0uVzItImFNCLyISolJaN+G/ZZP6l7/n4S9XUljkPI4u/M1ft5tTJs9WMi8iYU8JvYhICOvhT+rbJSUA4Bw8PnUVl770HbsO5HocXXhyzvHi7LX8/Nmv2bI3J7BeybyIhCsl9CIiIa5H6yZ8eNPRjO7RMrBu1qqdnPz4bOavy/AwsvCzPyefG/6zgPs/WkqB/1uO5EaxvHDJcCXzIhK2lNCLiISB1k3jefWKo7hpQs/Auq37cvj5s9/wwuy1OKcuOFVZvnUfpz8xh09+3BpYN7BTMh/ddDQT+7b1MDIRkdpRQi8iEiaio4w7JvXhpUuPJLlRLAAFRY4HPlrK9f9ewP6cfI8jDF1vzd/IGU/OYc3OzMC6i0Z25c1rR9G5RaKHkYmI1J4SehGRMDM+tQ0f33w0gzolB9Z9umQrpz0xh2Vb9nkYWejJyS/kN28v5s43F5GTXwRAo9hoHjt/MA+ccQTxMdEeRygiUntK6EVEwlCn5om8ce0oLh7VNbBu7c5MznhyDm/O2+BhZKFj3a5Mzn56LlO+L/n36NmmCR/cOIbTB3f0MDIRkeBSQi8iEqbiY6K5//QjeOz8wSTG+SrNuQVF/Oqtxfzm7cXk5Bd6HKF3vvhpK6dMns1Pm0u+sThtUAfev2EMvdo29TAyEZHgi/E6ABERqZ3TB3ekf4ckrnttAau2HwBgyvcbWLxxL0/9cijdWjX2OML6k19YxD8+X8GzM9cE1sVFR/G7U/tx4VFdMDMPoxMRqRuq0IuIRICebZry/o1jOGNwh8C6pVv2MemRmdz7wU9s359TydHhr7DI8fb8jUx4aMZByXzHZo1489pRXDSyq5J5EYlYqtCLiESIxLgYHvn5YIZ3a8H9Hy4lr7CIvMIiXp6bzpTv13PJ6G5cO7YHzRvHeR1q0BQVOT5dspWHv1zB6h2ZB22bkNqGh88bRLPEyHm9IiLlUUIvIhJBzIwLR3ZlUKdm/P6DJfywfg8AOflFPPvVGv79zXquOLo7VxzTnaSEWG+DrQXnHNOWb+ehL1aytMzIPs0SY7lpQi8uG92NqChV5UUk8imhFxGJQAM6JfPOdaOZvmI7//i8JOk9kFvAY1NX8a+v07lmbA8uGd2VxLjw+lMwJ20n//hiReDDSrEm8TFceUx3Lj86vD+siIgcrvB6FxcRkWozMyaktmVc7zZ89tNWHv5yJWn+m2b3ZOXz18+W88LstdwwvgcXjOhCQmxoj8k+f10G//h8JV+v2XXQ+oTYKC4d3Z1rxqZEVHciEZHqCtpNsWbWycxeNLPNZpZrZulm9qiZNa/r85jZaDP7xMwyzCzLzBab2a1mFtp/nURE6kFUlPGzAe35/NaxPHzeILqUmhl154Fc7vtwKRP+MYP/free/MIiDyMt35JNe7nspe84++mvD0rm46KjuHR0N2beNZ7fnJSqZF5EGixzztX+JGY9gLlAG+B9YDkwAhgPrADGOOd2VXyGmp/HzE4H3gZygNeBDOBUoA/wlnPu3Fq+tvlDhw4dOn/+/NqcRkQkZOQXFvHmvI1MnraKLXsPHv2mY7NGjOvTmjE9WzEqpaUnSXJ+YRGLN+5hTtouZqft5Lu1GQdtj44yzhveiRsn9KJjs0b1Hp+ISF0ZNmwYCxYsWOCcG3Y4xwUrof8cmATc7JybXGr9w8BtwLPOuWuDfR4zSwLSgGR8yf48//oEYBowCrjAOTelFq9NCb2IRKSc/EL+8+16npqRxs4DeYdsN4O+7ZIY07Mlo3u2YkS3FjSOD35PzaIix4pt+5mTtpO5q3fx7ZpdZOYdOimWGZwxuCO3TOzVoMbWF5GGw7OE3sxSgNVAOtDDOVdUaltTYAtgQBvnXGa5J6nheczscuAF4BXn3CVlzjcBmArMdM4dW4vXp4ReRCJaVl4BL89N59mv1rA3O7/C/WKijCFdmjG6RyvG9GzF4M7NiIs5/J6bzjnWZ2QxJ20Xc1bv5JvVu9iVeegHitJO7N+O2yf1prdmeRWRCFbThD4YpZYJ/uUXpZNwAOfcfjObg6/qPhJfgh3M8xQf81k555sJZAGjzSzeOZdb3RckItKQJMbFcP24nlw+pjsL1u9mrj/RXrxxL4VFJUWfgiLH9+m7+T59N49NXUWj2GiGdW1OUqPq/ykpLHIs2bSPTXuyK92vQ3ICo3u28n070KMVbZMSavz6REQiXTAS+j7+5coKtq/Cl4j3pvKEvibnqfAY51yBma0F+gMpwLJKnhszq6gEn1rZcSIikSIhNprRPVoxukcr7qQP+3Py+XZNBnNX72Lu6p0s37r/oP2z8wuZnbYzKM/dPDHW99w9WzKmRyu6tkzUzK4iItUUjIQ+2b/cW8H24vXN6uA8wXpuEREpo2lCLMf1a8tx/doCsGN/Ll+v2cVcf1/39RlZNT5347hoRnRvwZievg8Qqe2aahIoEZEaqo9x6IvfoWt7921NzlPtYyrqq+Sv3A89jOcUEYlIrZvGc9qgDpw2qAMAGzKy+GnzvoO65VRHu+R4BnZqRmx00EZOFhFp0IKR0BdXwZMr2J5UZr9gnidYzy0iIoepc4tEOpca015ERLwRjPLICv+ydwXbe/mXFfWNr815KjzGzGKA7kABsKaK5xYRERERCUvBSOin+5eTzOyg8/mHmxwDZAPf1MF5pvmXJ5ZzvrFAIjBXI9yIiIiISKSqdULvnFsNfAF0A24os/k+oDG+ceKLx46PNbNU/6ywNT6P31vATuB8MxtevNI/sdQf/c2na/ziRERERERCXLBuir0emAs8bmYT8Q0ReRQwHl8XmXtK7dvRv30dvuS9pufBObfPzK7Cl9jPMLMpQAZwGr4hLd8CXg/SaxQRERERCTlBGWLAX10fDryMLwG/A+gBPA6Mcs7tqqvzOOfeA47FN5HU2cBNQD5wO3C+q+1UuCIiIiIiISxow1Y65zYAl1Vjv3RKhpOs8XnKHDMH+NnhHCMiIiIiEgk0CLCIiIiISBhTQi8iIiIiEsaU0IuIiIiIhDEl9CIiIiIiYUwJvYiIiIhIGFNCLyIiIiISxpTQi4iIiIiEMSX0IiIiIiJhTAm9iIiIiEgYU0IvIiIiIhLGzDnndQwhzcx2NWrUqEXfvn29DkVEREREItiyZcvIzs7OcM61PJzjlNBXwczWAklAej0/dap/ubyen1e8o2vesOh6Nyy63g2LrnfDEszr3Q3Y55zrfjgHKaEPUWY2H8A5N8zrWKR+6Jo3LLreDYuud8Oi692whML1Vh96EREREZEwpoReRERERCSMKaEXEREREQljSuhFRERERMKYEnoRERERkTCmUW5ERERERMKYKvQiIiIiImFMCb2IiIiISBhTQi8iIiIiEsaU0IuIiIiIhDEl9CIiIiIiYUwJvYiIiIhIGFNCLyIiIiISxpTQhxgz62RmL5rZZjPLNbN0M3vUzJp7HZvUjJmdY2aTzWyWme0zM2dmr1VxzGgz+8TMMswsy8wWm9mtZhZdX3FLzZhZSzO70szeNbM0M8s2s71mNtvMrjCzct93dc3Dl5n91cymmtkG//XOMLMfzOwPZtaygmN0vSOEmV3kf193ZnZlBfvoeocpfx7mKnhsreCYer/emlgqhJhZD2Au0AZ4H1gOjADGAyuAMc65Xd5FKDVhZguBQcABYCOQCvzbOXdhBfufDrwN5ACvAxnAqUAf4C3n3Ln1ELbUkJldCzwNbAGmA+uBtsBZQDK+a3uuK/Xmq2se3swsD1gALAW2A42BkcBwYDMw0jm3odT+ut4Rwsw6Az8C0UAT4Crn3PNl9tH1DmNmlg40Ax4tZ/MB59w/yuzvzfV2zukRIg/gc8ABN5VZ/7B//TNex6hHja7reKAXYMA4/7V8rYJ9k/AlBLnA8FLrE/B92HPA+V6/Jj0qvd4T/G/eUWXWt8OX3DvgbF3zyHkACRWsf9B//Z7S9Y68h/89/X/AauDv/mt3ZZl9dL3D/AGkA+nV3Nez660uNyHCzFKASfh+cZ4ss/kPQCZwkZk1rufQpJacc9Odc6uc/391Fc4BWgNTnHPzSp0jB/itv3ldHYQpQeKcm+ac+9A5V1Rm/VbgGX9zXKlNuuZhzn+tyvOGf9mr1Dpd78hxM74P8Jfh+xtdHl3vhsWz662EPnRM8C+/KCcR2A/MARLxfY0rkav49+CzcrbNBLKA0WYWX38hSRDl+5cFpdbpmkeuU/3LxaXW6XpHADPrC/wFeMw5N7OSXXW9I0O8mV1oZneb2S1mNr6C/vCeXW8l9KGjj3+5soLtq/zL3vUQi3inwt8D51wBsBaIAVLqMyipPTOLAS72N0u/2euaRwgzu9PM7jWzR8xsFvAAvmT+L6V20/UOc/7/y6/i60J3dxW763pHhnb4rvmD+PrSTwNWmdmxZfbz7HrHBPuEUmPJ/uXeCrYXr29W96GIh/R7ELn+AhwBfOKc+7zUel3zyHEnvhugi30GXOqc21Fqna53+Ps9MAQ42jmXXcW+ut7h7yVgFvATsB9fMn4jcDXwqZmNcs4t8u/r2fVWhT58mH+pYYkaNv0ehCEzuxm4A9/IVRcd7uH+pa55iHPOtXPOGb5q3ln4/vD/YGZDD+M0ut4hzMxG4KvKP+Sc+zoYp/Qvdb1DlHPuPv+9Uducc1nOuSXOuWvxDVjSCLj3ME5XZ9dbCX3oKP7UllzB9qQy+0lk0u9BhDGzG4DH8A1pON45l1FmF13zCOP/w/8uvoEOWgKvlNqs6x2mSnW1WQn8rpqH6XpHruJBDsaWWufZ9VZCHzpW+JcV9ZEvHiWhoj72Ehkq/D3w/zHpju+GyjX1GZTUjJndCjwBLMGXzJc3CYmueYRyzq3D90Guv5m18q/W9Q5fTfBdt75ATukJhvCNRgfwnH/do/62rnfk2u5flh590LPrrYQ+dEz3LyeVnUnSzJoCY4Bs4Jv6Dkzq1TT/8sRyto3FN9LRXOdcbv2FJDVhZr8GHgEW4kvmt1ewq655ZOvgXxb6l7re4SsXeKGCxw/+fWb728XdcXS9I9co/7J0cu7d9fZ6wH49DpqQQBNLRfiD6k0stQNNQhLWD3xfxztgHtCiin11zcP4gW/m53blrI+iZGKpObrekf3A14+6oomldL3D9AH0L+89HOiKb/RBB9wdCtfb/E8kIcDMeuC74G2A94FlwFH4ZhpdCYx2zu3yLkKpCTM7AzjD32wHnIDvE/0s/7qdzrk7y+z/Fr5po6fgmzb6NPzTRgPnOf3HDVlmdgnwMr6K7GTK7yuZ7px7udQxZ6BrHpb83ar+jm+M6dXALnwj3RyL76bYrcBE59zSUsecga53RDGze/F1u7nKOfd8mW1noOsdlvzX9Tf4elGsxTfKTQ/gZHxJ+ifAmc65vFLHnIEH11sJfYgxs87A/fi+rmkJbAHeA+5zh95MJ2Gg1Bt9RdY557qVOWYMcA++r/QSgDTgReBx51zhIWeQkFGN6w3wlXNuXJnjdM3DkJkdgW/mxzFAJ3zD0WXiK8J8jO/6HfLeresdWSpL6P3bdb3DkH+c+WvxDVPaDl9/+T34ulK+CrxaXnLuxfVWQi8iIiIiEsZ0U6yIiIiISBhTQi8iIiIiEsaU0IuIiIiIhDEl9CIiIiIiYUwJvYiIiIhIGFNCLyIiIiISxpTQi4iIiIiEMSX0IiIiIiJhTAm9iIiIiEgYU0IvIiIiIhLGlNCLiIiIiIQxJfQiIiIiImFMCb2IiIiISBhTQi8iIiIiEsaU0IuIiIiIhDEl9CIiIiIiYUwJvYiIiIhIGPt/kaEnTlGLN8UAAAAASUVORK5CYII=\n", 196 | "text/plain": [ 197 | "
" 198 | ] 199 | }, 200 | "metadata": { 201 | "image/png": { 202 | "height": 248, 203 | "width": 378 204 | }, 205 | "needs_background": "light" 206 | }, 207 | "output_type": "display_data" 208 | } 209 | ], 210 | "source": [ 211 | "x = jnp.linspace(0, 2 * jnp.pi)\n", 212 | "plt.plot(jnp.cos(x) + 1)" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "id": "f3029221-02ff-4b70-8fed-1005878c5595", 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "jnp.sqrt(meta_output[\"losses\"])[-10:]" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 172, 228 | "id": "472ddb0c-0d3b-4aac-a523-46759d076024", 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "data": { 233 | "text/plain": [ 234 | "DeviceArray([104.64888 , 104.65269 , 104.653076, 104.651764, 104.662605,\n", 235 | " 104.65336 , 104.648094, 104.650826, 104.64987 , 104.64641 ], dtype=float32)" 236 | ] 237 | }, 238 | "execution_count": 172, 239 | "metadata": {}, 240 | "output_type": "execute_result" 241 | } 242 | ], 243 | "source": [ 244 | "jnp.sqrt(meta_output[\"losses\"])[-10:]" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 246, 250 | "id": "fd2b9ff8-8036-420a-8761-0cf817aba4b1", 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "configs_pred = meta_model.apply(meta_output[\"params\"], X_train.reshape(-1, 28**2))\n", 255 | "configs_pred = configs_pred.reshape(600, 150)" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 247, 261 | "id": "595aa4ea-0901-48ee-8114-fd1392c0084d", 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/plain": [ 267 | "DeviceArray(3.0539749, dtype=float32)" 268 | ] 269 | }, 270 | "execution_count": 247, 271 | "metadata": {}, 272 | "output_type": "execute_result" 273 | } 274 | ], 275 | "source": [ 276 | "configs_pred.min()" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 248, 282 | "id": "c64b0544-a318-48ec-a547-426fbb88bf1f", 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "data": { 287 | "text/plain": [ 288 | "DeviceArray(3.264753, dtype=float32)" 289 | ] 290 | }, 291 | "execution_count": 248, 292 | "metadata": {}, 293 | "output_type": "execute_result" 294 | } 295 | ], 296 | "source": [ 297 | "configs_pred.max()" 298 | ] 299 | } 300 | ], 301 | "metadata": { 302 | "kernelspec": { 303 | "display_name": "Python 3 (ipykernel)", 304 | "language": "python", 305 | "name": "python3" 306 | }, 307 | "language_info": { 308 | "codemirror_mode": { 309 | "name": "ipython", 310 | "version": 3 311 | }, 312 | "file_extension": ".py", 313 | "mimetype": "text/x-python", 314 | "name": "python", 315 | "nbconvert_exporter": "python", 316 | "pygments_lexer": "ipython3", 317 | "version": "3.9.5" 318 | } 319 | }, 320 | "nbformat": 4, 321 | "nbformat_minor": 5 322 | } 323 | --------------------------------------------------------------------------------