├── .gitignore └── torchdyn ├── utils.py ├── module4-model ├── m4c.multiple_shooting_layers.ipynb └── m4g_gde_node_classification.ipynb ├── experimental ├── gde_node_classification_pyg.py └── latent_sde.py ├── README.md ├── module3-tasks └── m3a_image_classification.ipynb └── module1-neuralde └── m1b_crossing_trajectories.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | *lightning_logs* 2 | *ipynb_checkpoints* 3 | -------------------------------------------------------------------------------- /torchdyn/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | def smape(yhat, y): 5 | return torch.abs(yhat - y) / (torch.abs(yhat) + torch.abs(y)) / 2 -------------------------------------------------------------------------------- /torchdyn/module4-model/m4c.multiple_shooting_layers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "7d38155d", 6 | "metadata": {}, 7 | "source": [ 8 | "### Multiple Shooting Layers" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 18, 14 | "id": "16f7165f", 15 | "metadata": {}, 16 | "outputs": [ 17 | { 18 | "name": "stdout", 19 | "output_type": "stream", 20 | "text": [ 21 | "The autoreload extension is already loaded. To reload it, use:\n", 22 | " %reload_ext autoreload\n" 23 | ] 24 | } 25 | ], 26 | "source": [ 27 | "import torchdyn\n", 28 | "import torch\n", 29 | "import torch.nn as nn\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "from torchdyn.core import MultipleShootingLayer, MultipleShootingProblem\n", 32 | "from torchdyn.numerics import Lorenz\n", 33 | "\n", 34 | "import torchdiffeq\n", 35 | "import time \n", 36 | "%load_ext autoreload\n", 37 | "%autoreload 2" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 27, 43 | "id": "c39abef8", 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "x0 = torch.randn(8, 3) + 15\n", 48 | "t_span = torch.linspace(0, 3, 3000)\n", 49 | "sys = Lorenz()" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 28, 55 | "id": "967fbf3e", 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "for sensitivity in ['autograd', 'adjoint', 'interpolated_adjoint']:\n", 60 | " mshooting = MultipleShootingProblem(sys, solver='zero', sensitivity=sensitivity)\n", 61 | " t_eval, sol = mshooting(x0, t_span)" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 29, 67 | "id": "dddc49b3", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "for sensitivity in ['autograd', 'adjoint', 'interpolated_adjoint']:\n", 72 | " mshooting = MultipleShootingLayer(sys, solver='zero', sensitivity=sensitivity)\n", 73 | " t_eval, sol = mshooting(x0, t_span)" 74 | ] 75 | } 76 | ], 77 | "metadata": { 78 | "kernelspec": { 79 | "display_name": "torchdyn", 80 | "language": "python", 81 | "name": "torchdyn" 82 | }, 83 | "language_info": { 84 | "codemirror_mode": { 85 | "name": "ipython", 86 | "version": 3 87 | }, 88 | "file_extension": ".py", 89 | "mimetype": "text/x-python", 90 | "name": "python", 91 | "nbconvert_exporter": "python", 92 | "pygments_lexer": "ipython3", 93 | "version": "3.8.8" 94 | } 95 | }, 96 | "nbformat": 4, 97 | "nbformat_minor": 5 98 | } 99 | -------------------------------------------------------------------------------- /torchdyn/experimental/gde_node_classification_pyg.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | import torch_geometric.transforms as T 6 | from torch_geometric.datasets import Planetoid 7 | from torch_geometric.nn import SplineConv 8 | from torchdyn.models import NeuralDE 9 | 10 | dataset = 'Cora' 11 | path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'datasets', dataset) 12 | dataset = Planetoid(path, dataset, transform=T.TargetIndegree()) 13 | data = dataset[0] 14 | 15 | data.train_mask = torch.zeros(data.num_nodes, dtype=torch.bool) 16 | data.train_mask[:data.num_nodes - 1000] = 1 17 | data.val_mask = None 18 | data.test_mask = torch.zeros(data.num_nodes, dtype=torch.bool) 19 | data.test_mask[data.num_nodes - 500:] = 1 20 | 21 | 22 | class GCNLayer(torch.nn.Module): 23 | def __init__(self, input_size, output_size): 24 | super(GCNLayer, self).__init__() 25 | 26 | if input_size != output_size: 27 | raise AttributeError('input size must equal output size') 28 | 29 | self.conv1 = SplineConv(input_size, output_size, dim=1, kernel_size=2).to(device) 30 | self.conv2 = SplineConv(input_size, output_size, dim=1, kernel_size=2).to(device) 31 | 32 | def forward(self, x): 33 | edge_index, edge_attr = data.edge_index, data.edge_attr 34 | x = self.conv1(x, edge_index, edge_attr) 35 | x = self.conv2(x, edge_index, edge_attr) 36 | return x 37 | 38 | 39 | class Net(torch.nn.Module): 40 | def __init__(self): 41 | super(Net, self).__init__() 42 | 43 | self.func = GCNLayer(input_size=64, output_size=64) 44 | 45 | self.conv1 = SplineConv(dataset.num_features, 64, dim=1, kernel_size=2).to(device) 46 | self.neuralDE = NeuralDE(self.func, solver='rk4', s_span=torch.linspace(0, 1, 3)).to(device) 47 | self.conv2 = SplineConv(64, dataset.num_classes, dim=1, kernel_size=2).to(device) 48 | 49 | def forward(self, x): 50 | edge_index, edge_attr = data.edge_index, data.edge_attr 51 | x = F.tanh(self.conv1(x, edge_index, edge_attr)) 52 | x = F.dropout(x, training=self.training) 53 | x = self.neuralDE(x) 54 | x = F.tanh(self.conv2(x, edge_index, edge_attr)) 55 | 56 | return F.log_softmax(x, dim=1) 57 | 58 | 59 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 60 | model, data = Net().to(device), data.to(device) 61 | optimizer = torch.optim.Adam(model.parameters(), lr=0.005, weight_decay=5e-3) 62 | 63 | 64 | def train(): 65 | model.train() 66 | optimizer.zero_grad() 67 | F.nll_loss(model(data.x)[data.train_mask], data.y[data.train_mask]).backward() 68 | optimizer.step() 69 | 70 | 71 | def test(): 72 | model.eval() 73 | logits, accs = model(data.x), [] 74 | for _, mask in data('train_mask', 'test_mask'): 75 | pred = logits[mask].max(1)[1] 76 | acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item() 77 | accs.append(acc) 78 | return accs 79 | 80 | 81 | for epoch in range(1, 201): 82 | train() 83 | log = 'Epoch: {:03d}, Train: {:.4f}, Test: {:.4f}' 84 | print(log.format(epoch, *test())) 85 | -------------------------------------------------------------------------------- /torchdyn/README.md: -------------------------------------------------------------------------------- 1 | ### Applications and tutorials 2 | The current version of `torchdyn` contains various quickstart examples / tutorials which explore different aspects of continuous / implicit learning and related numerical methods. Most tutorials are kept up-to-date in case of API changes of underlying libraries. We automatically validate via a quick dry run in `test/validate_tutorials.py`. These are indicated by ✅. Older tutorials ⬜️ might require minimal API changes to get working and are not automatically validated, though the goal is to eventually extend testing. Working on ensuring older tutorials are still runnable represents a perfect opportunity to get involved in the project, requiring minimal familiarity with the codebase. 3 | 4 | We organize the tutorials in modules. `00_quickstart.ipynb` offers a general overview of `torchdyn` features, including some comments on design philosophy and goals of the library. 5 | 6 | Each module is then focused on a specific aspect of continuous or implicit learning. For the moment, we offer the following modules and tutorials: 7 | 8 | ### Module 1: Neural Differential Equations 9 | We empirically verify several properties of Neural ODEs, and develop strategies to alleviate some of their weaknesses. Augmentation, depth-variance and more are discussed here. 10 | 11 | * ✅ `m1a_neural_ode_cookbook`: here, we explore the API and how to define Neural ODE variants within `torchdyn` 12 | * ✅ `m1b_crossing_trajectories`: a standard benchmark problem, highlighting expressivity limitations of Neural ODEs, and how they can be addressed 13 | * ✅ `m1c_augmentation_strategies`: augmentation API for Neural ODEs 14 | * ✅ `m1d_higher_order`: higher-order Neural ODE variants for classification 15 | 16 | 17 | ### Module 2: Numerics and Optimization 18 | This module is concerned with the numerics behind neural and non-neural differential equations. We provide examples of `torchdyn` numerics API, including advanced methods such as multiple shooting algorithms and hypersolvers. 19 | 20 | * ✅ `m2a_hypersolver_odeint`: solve ODEs with hybridized neural networks + ODE solvers: the hypersolver API 21 | * ✅ `m2b_multiple_shooting`: get familiar with `torchdyn`'s API dedicated to multiple shooting ODE solvers. 22 | * ✅ `m3c_hybrid_odeint`: learn how to simulate hybrid (potentially multi-mode) dynamical systems via `odeint_hybrid`. 23 | * ✅ `m3d_generalized_adjoint`: introduce integral losses in your Neural ODE training [[18](https://arxiv.org/abs/2003.08063)] to track a sinusoidal signal 24 | 25 | ### Module 3: Tasks and Benchmarks 26 | Here, we showcase how `torchdyn` models can be used in various machine learning and control tasks. The focus is on developing the problem setting rather than applying complex models. 27 | 28 | * ⬜️ `m3a_image_classification`: convolutional Neural ODEs for digit classification on MNIST 29 | * ✅ `m3b_optimal_control`: direct optimal control of dynamical systems via the Neural ODE API. 30 | * ⬜️ `m4c_pde_optimal_control`: fast optimal control of a Timoshenko beam via Multiple Shooting Layers and root tracking. 31 | * ⬜️ `m3d_continuous_normalizing_flows`: density estimation with continuous normalizing flows. 32 | 33 | ### Module 4: Models 34 | This module offers an overview of several specialized continuous or implicit models. 35 | 36 | * ⬜️ `m4a_approximate_normalizing_flows`: recover densities with FFJORD variants of continuous normalizing flows [[19](https://arxiv.org/abs/1810.01367)] 37 | 38 | * ✅ `m4b_hypersolver_optimal_control`: speed up direct optimal control of ODE with hypersolvers. 39 | * ⬜️ `m4c_multiple_shooting_layers`: apply multiple shooting layers to time series classification, speeding up Neural CDEs (WIP). 40 | * ⬜️ `m4d_hamiltonian_networks`: learn dynamics of energy preserving systems with a simple implementation of `Hamiltonian Neural Networks` in `torchdyn` [[10](https://arxiv.org/abs/1906.01563)] 41 | * ⬜️ `m4e_lagrangian_networks`: learn dynamics of energy preserving systems with a simple implementation of `Lagrangian Neural Networks` in `torchdyn` [[12](https://arxiv.org/abs/2003.04630)] 42 | * ✅ `m4f_stable_neural_odes`: learn dynamics with `Stable Neural Flows`, a generalization of HNNs [[18](https://arxiv.org/abs/2003.08063)] 43 | * ⬜️ `m4g_gde_node_classification`: first steps into the world of Neural GDEs [[9](https://arxiv.org/abs/1911.07532)], or ODEs on graphs parametrized by graph neural networks (GNN). Classification on Cora. 44 | 45 | 46 | 47 | 48 | #### Goals 49 | 50 | Our current goals are to extend model zoo with pretrained Neural *DE variants and equilibrium models. -------------------------------------------------------------------------------- /torchdyn/experimental/latent_sde.py: -------------------------------------------------------------------------------- 1 | """A partial Re-implementation of Xuechen Li's work (https://github.com/google-research/torchsde/blob/master/examples/latent_sde.py)""" 2 | 3 | 4 | import os 5 | import math 6 | import numpy as np 7 | from collections import namedtuple 8 | from matplotlib import pyplot as plt 9 | 10 | import torch 11 | from torch import nn, optim 12 | import pytorch_lightning as pl 13 | from torch.utils.data import Dataset, DataLoader 14 | from torch.distributions import Laplace 15 | 16 | from torchdyn.models import LatentNeuralSDE, LinearScheduler, EMAMetric 17 | from torchsde import BrownianPath 18 | 19 | 20 | class IrregularSineDataset(Dataset): 21 | def __init__(self, batch_size, num_batches): 22 | ts_, ts_ext_, ts_vis_, ts, ts_ext, ts_vis, ys, ys_ = self.make_irregular_sine_data() 23 | self.array = ys.view(-1).unsqueeze(0).repeat(batch_size*num_batches, 1) 24 | self.ts = ts 25 | self.ts_ext = ts_ext 26 | self.ts_vis = ts_vis 27 | self.ys = ys 28 | 29 | def __len__(self): return len(self.array) 30 | def __getitem__(self, i): return self.array[i] 31 | def s_span(self): return self.ts 32 | def s_ext_span(self): return self.ts_ext 33 | def v_span(self): return self.ts_vis 34 | def x_sample(self): return self.ys 35 | 36 | @staticmethod 37 | def make_irregular_sine_data(): 38 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 39 | Data = namedtuple('Data', ['ts_', 'ts_ext_', 'ts_vis_', 'ts', 'ts_ext', 'ts_vis', 'ys', 'ys_']) 40 | with torch.no_grad(): 41 | ts_ = np.sort(np.random.uniform(low=0.4, high=1.6, size=16)) 42 | ts_ext_ = np.array([0.] + list(ts_) + [2.0]) 43 | ts_vis_ = np.linspace(0., 2.0, 300) 44 | ys_ = np.sin(ts_ * (2. * math.pi))[:, None] * 0.8 45 | 46 | ts = torch.tensor(ts_).float().to(device) 47 | ts_ext = torch.tensor(ts_ext_).float() 48 | ts_vis = torch.tensor(ts_vis_).float() 49 | ys = torch.tensor(ys_).float().to(device) 50 | 51 | return Data(ts_, ts_ext_, ts_vis_, ts, ts_ext, ts_vis, ys, ys_) 52 | 53 | 54 | class FFunc(pl.LightningModule): 55 | """Posterior drift.""" 56 | def __init__(self): 57 | super(FFunc, self).__init__() 58 | self.net = nn.Sequential( 59 | nn.Linear(3, 200), 60 | nn.Tanh(), 61 | nn.Linear(200, 200), 62 | nn.Tanh(), 63 | nn.Linear(200, 1) 64 | ) 65 | 66 | def forward(self, t, y): 67 | if t.dim() == 0: 68 | t = float(t) * torch.ones_like(y) 69 | # Positional encoding in transformers; must use `t`, since the posterior is likely inhomogeneous. 70 | inp = torch.cat((torch.sin(t), torch.cos(t), y), dim=-1) 71 | return self.net(inp) 72 | 73 | 74 | class HFunc(pl.LightningModule): 75 | """Prior drift""" 76 | def __init__(self, theta=1.0, mu=0.0): 77 | super(HFunc, self).__init__() 78 | self.theta = nn.Parameter(torch.tensor([[theta]]), requires_grad=False) 79 | self.mu = nn.Parameter(torch.tensor([[mu]]), requires_grad=False) 80 | 81 | def forward(self, t, y): 82 | return self.theta * (self.mu - y) 83 | 84 | 85 | class GFunc(pl.LightningModule): 86 | """Diffusion""" 87 | def __init__(self, sigma=0.5): 88 | super(GFunc, self).__init__() 89 | self.sigma = nn.Parameter(torch.tensor([[sigma]]), requires_grad=False) 90 | 91 | def forward(self, t, y): 92 | return self.sigma.repeat(y.size(0), 1) 93 | 94 | 95 | class Model(pl.LightningModule): 96 | def __init__(self): 97 | super().__init__() 98 | 99 | sigma, theta, mu = 0.5, 1.0, 0.0 100 | options = {'trapezoidal_approx': False} 101 | 102 | self.f_func = FFunc() 103 | self.h_func = HFunc(theta=theta, mu=mu) 104 | self.g_func = GFunc(sigma=sigma) 105 | 106 | self.s_span = IrregularSineDataset(1, 1).s_span() 107 | 108 | self.lsde = LatentNeuralSDE(post_drift=self.f_func, diffusion=self.g_func, prior_drift=self.h_func, 109 | sigma=sigma, theta=theta, mu=mu, 110 | noise_type='diagonal', order=1, sensitivity='autograd', s_span=self.s_span, 111 | solver='srk', atol=1e-3, rtol=1e-3, intloss=None, options=options) 112 | 113 | def forward(self, eps: torch.Tensor, s_span=None): 114 | """ 115 | :param: Noise sample 116 | """ 117 | zs, log_ratio = self.lsde(eps, s_span) 118 | zs = zs.squeeze() 119 | # 120 | 121 | return zs, log_ratio 122 | 123 | 124 | class Learner(pl.LightningModule): 125 | def __init__(self, train_path): 126 | super().__init__() 127 | self.model = Model() 128 | 129 | dataset = IrregularSineDataset(1, 1) 130 | self.vis_span = dataset.v_span() 131 | self.x_sample = dataset.x_sample() 132 | self.s_span = dataset.s_span() 133 | self.s_ext_span = dataset.s_ext_span() 134 | 135 | self.train_path = train_path 136 | self.logp_metric = EMAMetric() 137 | self.log_ratio_metric = EMAMetric() 138 | self.loss_metric = EMAMetric() 139 | self.kl_scheduler = LinearScheduler(iters=10) 140 | 141 | self.scale = 0.05 142 | 143 | def forward(self, x): 144 | return self.model(x) 145 | 146 | def training_step(self, batch, batch_idx): 147 | # x, y = torch.split(batch, split_size_or_sections=1, dim=0) 148 | x = batch 149 | eps = torch.randn(batch.shape[0], 1) 150 | 151 | zs, log_ratio = self.model(eps=eps, s_span=self.s_ext_span) 152 | zs = zs[1:-1] 153 | 154 | likelihood = Laplace(loc=zs, scale=self.scale) 155 | 156 | # Bad Hack just in this case where every tensor in batch is identical 157 | logp = likelihood.log_prob(x.mean(dim=0).unsqueeze(1).to(self.device)).sum(dim=0).mean(dim=0) 158 | loss = -logp + log_ratio * self.kl_scheduler() 159 | 160 | # loss.backward() 161 | # self.optimizer.step() 162 | # self.scheduler.step() 163 | self.logp_metric.step(logp) 164 | self.log_ratio_metric.step(log_ratio) 165 | self.loss_metric.step(loss) 166 | 167 | logs = {'train_loss': loss} 168 | return {'loss': loss, 'log': logs} 169 | 170 | def on_epoch_end(self, vis_n_sim=1024): 171 | 172 | img_path = os.path.join(train_dir, f'global_step_{self.current_epoch}.png') 173 | ylims = (-1.75, 1.75) 174 | alphas = [0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55] 175 | percentiles = [0.999, 0.99, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] 176 | sample_colors = ('#8c96c6', '#8c6bb1', '#810f7c') 177 | fill_color = '#9ebcda' 178 | mean_color = '#4d004b' 179 | num_samples = len(sample_colors) 180 | vis_idx = np.random.permutation(vis_n_sim) 181 | 182 | eps = torch.randn(vis_n_sim, 1) 183 | bm = BrownianPath(t0=self.vis_span[0], w0=torch.zeros(vis_n_sim, 1)) 184 | 185 | # -- Not used -- From show_prior option in original implementation 186 | # zs = self.model.sample_p(vis_span=self.vis_span, n_sim=vis_n_sim, eps=eps, bm=bm).squeeze() 187 | # ts_vis_, zs_ = self.vis_span.cpu().numpy(), zs.cpu().numpy() 188 | # zs_ = np.sort(zs_, axis=1) 189 | 190 | zs = self.model.lsde.sample_q(vis_span=self.vis_span, n_sim=vis_n_sim, eps=eps, bm=bm).squeeze() 191 | samples = zs[:, vis_idx] 192 | s_span_vis_ = self.vis_span.cpu().detach().numpy() 193 | zs_ = zs.cpu().detach().numpy() 194 | samples_ = samples.cpu().detach().numpy() 195 | 196 | zs_ = np.sort(zs_, axis=1) 197 | 198 | with torch.no_grad(): 199 | 200 | plt.subplot(frameon=False) 201 | 202 | for alpha, percentile in zip(alphas, percentiles): 203 | idx = int((1 - percentile) / 2. * vis_n_sim) 204 | zs_bot_, zs_top_ = zs_[:, idx], zs_[:, -idx] 205 | plt.fill_between(s_span_vis_, zs_bot_, zs_top_, alpha=alpha, color=fill_color) 206 | 207 | plt.plot(s_span_vis_, zs_.mean(axis=1), color=mean_color) 208 | 209 | for j in range(num_samples): 210 | plt.plot(s_span_vis_, samples_[:, j], color=sample_colors[j], linewidth=1.0) 211 | 212 | num, ds = 12, 0.12 213 | s, x = torch.meshgrid( 214 | [torch.linspace(0.2, 1.8, num), torch.linspace(-1.5, 1.5, num)] 215 | ) 216 | 217 | s, x = s.reshape(-1, 1).to(self.device), x.reshape(-1, 1).to(self.device) 218 | 219 | ftx = self.model.lsde.defunc.f(s=s, x=x) 220 | ftx = ftx.cpu().reshape(num, num) 221 | 222 | ds = torch.zeros(num, num).fill_(ds) 223 | dx = ftx * ds 224 | ds_, dx_, = ds.cpu().detach().numpy(), dx.cpu().detach().numpy() 225 | s_, x_ = s.cpu().detach().numpy(), x.cpu().detach().numpy() 226 | 227 | plt.quiver(s_, x_, ds_, dx_, alpha=0.3, edgecolors='k', width=0.0035, scale=50) 228 | 229 | # Data. 230 | plt.scatter(self.s_span.cpu().numpy(), self.x_sample.cpu().numpy(), marker='x', zorder=3, color='k', s=35) 231 | 232 | plt.ylim(ylims) 233 | plt.xlabel('$t$') 234 | plt.ylabel('$Y_t$') 235 | plt.tight_layout() 236 | plt.savefig(img_path, dpi=400) 237 | plt.close() 238 | 239 | def configure_optimizers(self): 240 | 241 | optimizer = torch.optim.Adam(self.model.parameters(), lr=0.01) 242 | scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=.999) 243 | 244 | return [optimizer], [scheduler] 245 | 246 | def train_dataloader(self): 247 | 248 | batch_size = 512 249 | n_batches = 50 250 | 251 | return DataLoader(IrregularSineDataset(batch_size=batch_size, num_batches=n_batches), batch_size=batch_size) 252 | 253 | 254 | train_dir = os.path.join('images', 'ts_ext') 255 | trainer = pl.Trainer(gpus=0, max_epochs=200) 256 | trainer.fit(Learner(train_dir)) 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | -------------------------------------------------------------------------------- /torchdyn/module4-model/m4g_gde_node_classification.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 27, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import math\n", 10 | "import numpy as np\n", 11 | "import scipy.sparse as sp\n", 12 | "import time\n", 13 | "import torch\n", 14 | "import torch.nn as nn\n", 15 | "import torch.nn.functional as F\n", 16 | "import matplotlib.pyplot as plt\n", 17 | "from torch.nn.parameter import Parameter\n", 18 | "from torch.nn.modules.module import Module\n", 19 | "\n", 20 | "import dgl\n", 21 | "import dgl.function as fn\n", 22 | "\n", 23 | "import dgl.data\n", 24 | "import networkx as nx\n", 25 | "\n", 26 | "from torchdyn.core import NeuralODE\n", 27 | "from torchdyn.nn import DataControl, DepthCat, Augmenter\n", 28 | "from torchdyn.datasets import *\n", 29 | "from torchdyn.utils import *" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 28, 35 | "metadata": { 36 | "tags": [ 37 | "parameters" 38 | ] 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "# quick run for automated notebook validation\n", 43 | "dry_run = False" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": {}, 49 | "source": [ 50 | "# Neural Graph Differential Equations" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "## Semi-supervised node classification " 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "This notebook introduces `Neural GDEs` as a general high-performance model for graph structured data. Notebook `07_graph_differential_equations` is designed from the ground up as an introduction to Neural GDEs and therefore contains ample comments to provide insights on some of our design choices. To be accessible to practicioners/researchers without prior experience on GNNs, we discuss some features of `dgl` as well, one of the PyTorch ecosystems for geometric deep learning." 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "## Data preparation" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 29, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", 81 | "\n", 82 | "# seed for repeatability\n", 83 | "torch.backends.cudnn.deterministic = True\n", 84 | "torch.backends.cudnn.benchmark = False\n", 85 | "\n", 86 | "torch.manual_seed(0)\n", 87 | "np.random.seed(0)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 30, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | " NumNodes: 2708\n", 100 | " NumEdges: 10556\n", 101 | " NumFeats: 1433\n", 102 | " NumClasses: 7\n", 103 | " NumTrainingSamples: 140\n", 104 | " NumValidationSamples: 500\n", 105 | " NumTestSamples: 1000\n", 106 | "Done loading data from cached files.\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "# dgl offers convenient access to GNN benchmark datasets via `dgl.data`...\n", 112 | "# other standard datasets (e.g. Citeseer / Pubmed) are also accessible via the dgl.data\n", 113 | "# API. The rest of the notebook is compatible with Cora / Citeseer / Pubmed with minimal\n", 114 | "# modification required.\n", 115 | "data = dgl.data.CoraGraphDataset()[0]" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 32, 121 | "metadata": {}, 122 | "outputs": [ 123 | { 124 | "data": { 125 | "text/plain": [ 126 | "(7, 140, 500, 1000)" 127 | ] 128 | }, 129 | "execution_count": 32, 130 | "metadata": {}, 131 | "output_type": "execute_result" 132 | } 133 | ], 134 | "source": [ 135 | "# Cora is a node-classification datasets with 2708 nodes\n", 136 | "X = data.ndata['feat'].to(device)\n", 137 | "Y = data.ndata['label'].to(device)\n", 138 | "\n", 139 | "# In transductive semi-supervised node classification tasks on graphs, the model has access to all\n", 140 | "# node features but only a masked subset of the labels\n", 141 | "train_mask = data.ndata['train_mask']\n", 142 | "val_mask = data.ndata['val_mask']\n", 143 | "test_mask = data.ndata['test_mask']\n", 144 | "\n", 145 | "num_feats = X.shape[1]\n", 146 | "n_classes = 7\n", 147 | "\n", 148 | "# 140 training samples, 300 validation, 1000 test\n", 149 | "n_classes, train_mask.sum().item(), val_mask.sum().item(),test_mask.sum().item()" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "data.remove_edges()" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 35, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "ename": "TypeError", 168 | "evalue": "'bool' object is not callable", 169 | "output_type": "error", 170 | "traceback": [ 171 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 172 | "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 173 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# add self-edge for each node\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove_edges\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselfloop_edges\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_edges_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDGLGraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 174 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/networkx/classes/function.py\u001b[0m in \u001b[0;36mselfloop_edges\u001b[0;34m(G, data, keys, default)\u001b[0m\n\u001b[1;32m 1194\u001b[0m )\n\u001b[1;32m 1195\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mG\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_multigraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1197\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mkeys\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1198\u001b[0m return (\n", 175 | "\u001b[0;31mTypeError\u001b[0m: 'bool' object is not callable" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "# add self-edge for each node\n", 181 | "g = data\n", 182 | "g.remove_edges(nx.selfloop_edges(g))\n", 183 | "g.add_edges_from(zip(g.nodes(), g.nodes()))\n", 184 | "g = dgl.DGLGraph(g)\n", 185 | "edges = g.edges()\n", 186 | "n_edges = g.number_of_edges()\n", 187 | "\n", 188 | "n_edges" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 7, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "# compute diagonal of normalization matrix D according to standard formula\n", 198 | "degs = g.in_degrees().float()\n", 199 | "norm = torch.pow(degs, -0.5)\n", 200 | "norm[torch.isinf(norm)] = 0\n", 201 | "# add to dgl.Graph in order for the norm to be accessible at training time\n", 202 | "g.ndata['norm'] = norm.unsqueeze(1).to(device)" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "## Neural GCDE " 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "As Neural ODEs, GDEs require specification of an ODE function (`ODEFunc`), representing the set of layers that will be called repeatedly by the ODE solver.\n", 217 | "\n", 218 | "Here, we use the convolutional variant of Neural GDEs based on GCNs: `Neural GCDEs`. The only difference with alternative neural GDEs resides in the type of GNN layer utilized in the ODEFunc.\n", 219 | "\n", 220 | "For adaptive step GDEs (dopri5) we increase the hidden dimension to 64 to reduce the stiffness of the ODE and therefore the number of ODEFunc evaluations (`NFE`: Number Function Evaluation)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "First, we define the auxiliary GNN model as a standard GCN. Luckily, in this example the graph is static and can thus be assigned during initialization. For varying graphs, additional bookeeping is required." 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": 8, 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [ 236 | "def accuracy(y_hat:torch.Tensor, y:torch.Tensor):\n", 237 | " preds = torch.max(y_hat, 1)[1]\n", 238 | " return torch.mean((y == preds).float())" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": 9, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "class GCNLayer(nn.Module):\n", 248 | " def __init__(self, g:dgl.DGLGraph, in_feats:int, out_feats:int, activation,\n", 249 | " dropout:int, bias:bool=True):\n", 250 | " super().__init__()\n", 251 | " self.g = g\n", 252 | " self.weight = nn.Parameter(torch.Tensor(in_feats, out_feats))\n", 253 | " if bias:\n", 254 | " self.bias = nn.Parameter(torch.Tensor(out_feats))\n", 255 | " else:\n", 256 | " self.bias = None\n", 257 | " self.activation = activation\n", 258 | " if dropout:\n", 259 | " self.dropout = nn.Dropout(p=dropout)\n", 260 | " else:\n", 261 | " self.dropout = 0.\n", 262 | " self.reset_parameters()\n", 263 | "\n", 264 | " def reset_parameters(self):\n", 265 | " stdv = 1. / math.sqrt(self.weight.size(1))\n", 266 | " self.weight.data.uniform_(-stdv, stdv)\n", 267 | " if self.bias is not None:\n", 268 | " self.bias.data.uniform_(-stdv, stdv)\n", 269 | "\n", 270 | " def forward(self, h):\n", 271 | " if self.dropout:\n", 272 | " h = self.dropout(h)\n", 273 | " h = torch.mm(h, self.weight)\n", 274 | " # normalization by square root of src degree\n", 275 | " h = h * self.g.ndata['norm']\n", 276 | " self.g.ndata['h'] = h\n", 277 | " self.g.update_all(fn.copy_src(src='h', out='m'),\n", 278 | " fn.sum(msg='m', out='h'))\n", 279 | " h = self.g.ndata.pop('h')\n", 280 | " # normalization by square root of dst degree\n", 281 | " h = h * self.g.ndata['norm']\n", 282 | " # bias\n", 283 | " if self.bias is not None:\n", 284 | " h = h + self.bias\n", 285 | " if self.activation:\n", 286 | " h = self.activation(h)\n", 287 | " return h" 288 | ] 289 | }, 290 | { 291 | "cell_type": "markdown", 292 | "metadata": {}, 293 | "source": [ 294 | "Then, we construct the Neural GDE as follows:" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 34, 300 | "metadata": {}, 301 | "outputs": [], 302 | "source": [ 303 | "func = nn.Sequential(GCNLayer(g=g, in_feats=64, out_feats=64, activation=nn.Softplus(), dropout=0.9),\n", 304 | " GCNLayer(g=g, in_feats=64, out_feats=64, activation=None, dropout=0.9)\n", 305 | " ).to(device)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 35, 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "neuralDE = NeuralODE(func, solver='rk4', s_span=torch.linspace(0, 1, 3)).to(device)" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 36, 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "m = nn.Sequential(GCNLayer(g=g, in_feats=num_feats, out_feats=64, activation=None, dropout=0.4),\n", 324 | " neuralDE,\n", 325 | " GCNLayer(g=g, in_feats=64, out_feats=n_classes, activation=None, dropout=0.)\n", 326 | " ).to(device)" 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "metadata": {}, 332 | "source": [ 333 | "## Training loop" 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": 37, 339 | "metadata": {}, 340 | "outputs": [], 341 | "source": [ 342 | "class PerformanceContainer(object):\n", 343 | " \"\"\" Simple data class for metrics logging.\"\"\"\n", 344 | " def __init__(self, data:dict):\n", 345 | " self.data = data\n", 346 | " \n", 347 | " @staticmethod\n", 348 | " def deep_update(x, y):\n", 349 | " for key in y.keys():\n", 350 | " x.update({key: list(x[key] + y[key])})\n", 351 | " return x" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": 38, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "opt = torch.optim.Adam(m.parameters(), lr=1e-3, weight_decay=5e-4)\n", 361 | "criterion = torch.nn.CrossEntropyLoss()\n", 362 | "logger = PerformanceContainer(data={'train_loss':[], 'train_accuracy':[],\n", 363 | " 'test_loss':[], 'test_accuracy':[],\n", 364 | " 'forward_time':[], 'backward_time':[],\n", 365 | " })\n" 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "execution_count": 39, 371 | "metadata": {}, 372 | "outputs": [ 373 | { 374 | "name": "stdout", 375 | "output_type": "stream", 376 | "text": [ 377 | "[150], Loss: 1.457, Train Accuracy: 0.514, Test Accuracy: 0.377\n", 378 | "[300], Loss: 0.730, Train Accuracy: 0.907, Test Accuracy: 0.731\n", 379 | "[450], Loss: 0.542, Train Accuracy: 0.921, Test Accuracy: 0.766\n", 380 | "[600], Loss: 0.416, Train Accuracy: 0.950, Test Accuracy: 0.816\n", 381 | "[750], Loss: 0.557, Train Accuracy: 0.943, Test Accuracy: 0.810\n", 382 | "[900], Loss: 0.353, Train Accuracy: 0.964, Test Accuracy: 0.819\n", 383 | "[1050], Loss: 0.265, Train Accuracy: 0.971, Test Accuracy: 0.807\n", 384 | "[1200], Loss: 0.340, Train Accuracy: 0.964, Test Accuracy: 0.828\n", 385 | "[1350], Loss: 0.201, Train Accuracy: 0.971, Test Accuracy: 0.828\n", 386 | "[1500], Loss: 0.368, Train Accuracy: 0.971, Test Accuracy: 0.824\n", 387 | "[1650], Loss: 0.255, Train Accuracy: 0.979, Test Accuracy: 0.812\n", 388 | "[1800], Loss: 0.241, Train Accuracy: 0.971, Test Accuracy: 0.820\n", 389 | "[1950], Loss: 0.304, Train Accuracy: 0.979, Test Accuracy: 0.821\n", 390 | "[2100], Loss: 0.248, Train Accuracy: 0.971, Test Accuracy: 0.828\n", 391 | "[2250], Loss: 0.223, Train Accuracy: 0.979, Test Accuracy: 0.815\n", 392 | "[2400], Loss: 0.180, Train Accuracy: 0.979, Test Accuracy: 0.834\n", 393 | "[2550], Loss: 0.321, Train Accuracy: 0.986, Test Accuracy: 0.825\n", 394 | "[2700], Loss: 0.166, Train Accuracy: 0.986, Test Accuracy: 0.808\n", 395 | "[2850], Loss: 0.171, Train Accuracy: 0.986, Test Accuracy: 0.821\n", 396 | "[3000], Loss: 0.190, Train Accuracy: 0.986, Test Accuracy: 0.827\n", 397 | "[3150], Loss: 0.207, Train Accuracy: 0.993, Test Accuracy: 0.823\n", 398 | "[3300], Loss: 0.159, Train Accuracy: 0.986, Test Accuracy: 0.817\n", 399 | "[3450], Loss: 0.183, Train Accuracy: 0.993, Test Accuracy: 0.829\n", 400 | "[3600], Loss: 0.161, Train Accuracy: 0.986, Test Accuracy: 0.831\n", 401 | "[3750], Loss: 0.143, Train Accuracy: 0.986, Test Accuracy: 0.826\n", 402 | "[3900], Loss: 0.182, Train Accuracy: 0.986, Test Accuracy: 0.826\n", 403 | "[4050], Loss: 0.156, Train Accuracy: 0.993, Test Accuracy: 0.817\n", 404 | "[4200], Loss: 0.177, Train Accuracy: 0.986, Test Accuracy: 0.819\n", 405 | "[4350], Loss: 0.160, Train Accuracy: 0.993, Test Accuracy: 0.811\n", 406 | "[4500], Loss: 0.182, Train Accuracy: 0.986, Test Accuracy: 0.829\n", 407 | "[4650], Loss: 0.130, Train Accuracy: 0.986, Test Accuracy: 0.812\n", 408 | "[4800], Loss: 0.143, Train Accuracy: 0.986, Test Accuracy: 0.830\n", 409 | "[4950], Loss: 0.207, Train Accuracy: 0.993, Test Accuracy: 0.818\n" 410 | ] 411 | } 412 | ], 413 | "source": [ 414 | "steps = 5000\n", 415 | "verbose_step = 150\n", 416 | "num_grad_steps = 0\n", 417 | "\n", 418 | "for i in range(steps): # looping over epochs\n", 419 | " m.train()\n", 420 | " outputs = m(X)\n", 421 | " y_pred = outputs\n", 422 | " loss = criterion(y_pred[train_mask], Y[train_mask])\n", 423 | " opt.zero_grad()\n", 424 | " \n", 425 | " start_time = time.time()\n", 426 | " loss.backward()\n", 427 | " \n", 428 | " opt.step()\n", 429 | " num_grad_steps += 1\n", 430 | "\n", 431 | " with torch.no_grad():\n", 432 | " m.eval()\n", 433 | "\n", 434 | " # calculating outputs again with zeroed dropout\n", 435 | " y_pred = m(X)\n", 436 | "\n", 437 | " train_loss = loss.item()\n", 438 | " train_acc = accuracy(y_pred[train_mask], Y[train_mask]).item()\n", 439 | " test_acc = accuracy(y_pred[test_mask], Y[test_mask]).item()\n", 440 | " test_loss = criterion(y_pred[test_mask], Y[test_mask]).item()\n", 441 | " logger.deep_update(logger.data, dict(train_loss=[train_loss], train_accuracy=[train_acc],\n", 442 | " test_loss=[test_loss], test_accuracy=[test_acc])\n", 443 | " )\n", 444 | "\n", 445 | " if num_grad_steps % verbose_step == 0:\n", 446 | " print('[{}], Loss: {:3.3f}, Train Accuracy: {:3.3f}, Test Accuracy: {:3.3f}'.format(num_grad_steps,\n", 447 | " train_loss,\n", 448 | " train_acc,\n", 449 | " test_acc,\n", 450 | " ))" 451 | ] 452 | } 453 | ], 454 | "metadata": { 455 | "kernelspec": { 456 | "display_name": "torchdyn", 457 | "language": "python", 458 | "name": "torchdyn" 459 | }, 460 | "language_info": { 461 | "codemirror_mode": { 462 | "name": "ipython", 463 | "version": 3 464 | }, 465 | "file_extension": ".py", 466 | "mimetype": "text/x-python", 467 | "name": "python", 468 | "nbconvert_exporter": "python", 469 | "pygments_lexer": "ipython3", 470 | "version": "3.8.5" 471 | }, 472 | "latex_envs": { 473 | "LaTeX_envs_menu_present": true, 474 | "autoclose": false, 475 | "autocomplete": true, 476 | "bibliofile": "biblio.bib", 477 | "cite_by": "apalike", 478 | "current_citInitial": 1, 479 | "eqLabelWithNumbers": true, 480 | "eqNumInitial": 1, 481 | "hotkeys": { 482 | "equation": "Ctrl-E", 483 | "itemize": "Ctrl-I" 484 | }, 485 | "labels_anchors": false, 486 | "latex_user_defs": false, 487 | "report_style_numbering": false, 488 | "user_envs_cfg": false 489 | }, 490 | "varInspector": { 491 | "cols": { 492 | "lenName": 16, 493 | "lenType": 16, 494 | "lenVar": 40 495 | }, 496 | "kernels_config": { 497 | "python": { 498 | "delete_cmd_postfix": "", 499 | "delete_cmd_prefix": "del ", 500 | "library": "var_list.py", 501 | "varRefreshCmd": "print(var_dic_list())" 502 | }, 503 | "r": { 504 | "delete_cmd_postfix": ") ", 505 | "delete_cmd_prefix": "rm(", 506 | "library": "var_list.r", 507 | "varRefreshCmd": "cat(var_dic_list()) " 508 | } 509 | }, 510 | "types_to_exclude": [ 511 | "module", 512 | "function", 513 | "builtin_function_or_method", 514 | "instance", 515 | "_Feature" 516 | ], 517 | "window_display": false 518 | } 519 | }, 520 | "nbformat": 4, 521 | "nbformat_minor": 4 522 | } 523 | -------------------------------------------------------------------------------- /torchdyn/module3-tasks/m3a_image_classification.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Image Classification" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this notebook we explore standard image classification on MNIST and CIFAR10 with convolutional Neural ODE variants.\n", 15 | "* Depth-invariant neural ODE\n", 16 | "* Galerkin neural ODE (GalNODE)\n", 17 | "\n", 18 | "In the following notebooks we'll further develop intuition around `augmentation` strategies that can be easily applied to the models below with the flexible `torchdyn` API. Here, we use simple `0-augmentation`." 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "from torchdyn.core import NeuralODE\n", 28 | "from torchdyn.nn import DataControl, DepthCat, Augmenter, GalConv2d, Fourier" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "import torch\n", 38 | "import torch.nn as nn\n", 39 | "from torch.utils.data import DataLoader\n", 40 | "from torchvision import datasets, transforms\n", 41 | "\n", 42 | "import pytorch_lightning as pl\n", 43 | "from pytorch_lightning.loggers import WandbLogger\n", 44 | "from pytorch_lightning.metrics.functional import accuracy" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": { 51 | "tags": [ 52 | "parameters" 53 | ] 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "# quick run for automated notebook validation\n", 58 | "dry_run = False" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 4, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 6, 73 | "metadata": {}, 74 | "outputs": [ 75 | { 76 | "name": "stdout", 77 | "output_type": "stream", 78 | "text": [ 79 | "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", 80 | "Using downloaded and verified file: ../../data/mnist_data/MNIST/raw/train-images-idx3-ubyte.gz\n", 81 | "Extracting ../../data/mnist_data/MNIST/raw/train-images-idx3-ubyte.gz to ../../data/mnist_data/MNIST/raw\n", 82 | "\n", 83 | "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n", 84 | "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ../../data/mnist_data/MNIST/raw/train-labels-idx1-ubyte.gz\n" 85 | ] 86 | }, 87 | { 88 | "data": { 89 | "application/vnd.jupyter.widget-view+json": { 90 | "model_id": "a01fe54d2fc941268ceab26caac7e2d1", 91 | "version_major": 2, 92 | "version_minor": 0 93 | }, 94 | "text/plain": [ 95 | " 0%| | 0/28881 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m )\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 333 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, train_dataloader, val_dataloaders, datamodule)\u001b[0m\n\u001b[1;32m 456\u001b[0m )\n\u001b[1;32m 457\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 459\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 460\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 334 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m 754\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 755\u001b[0m \u001b[0;31m# dispatch `start_training` or `start_evaluating` or `start_predicting`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 757\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 758\u001b[0m \u001b[0;31m# plugin will finalized fitting (e.g. ddp_spawn will load trained model)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 335 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mdispatch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 795\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_predicting\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 796\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 797\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 798\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 799\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 336 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mstart_training\u001b[0;34m(self, trainer)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'pl.Trainer'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_evaluating\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'pl.Trainer'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 337 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py\u001b[0m in \u001b[0;36mstart_training\u001b[0;34m(self, trainer)\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'pl.Trainer'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;31m# double dispatch to initiate the training loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 144\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 146\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_evaluating\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'pl.Trainer'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 338 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 805\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredicting\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 806\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 807\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 808\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 809\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_pre_training_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 339 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_epoch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m \u001b[0;31m# run train epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 869\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 870\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_steps\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_steps\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_step\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 340 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 497\u001b[0m \u001b[0;31m# ------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 498\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_batch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 499\u001b[0;31m \u001b[0mbatch_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataloader_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 500\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 501\u001b[0m \u001b[0;31m# when returning -1 from train_step, we end epoch early\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 341 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_batch\u001b[0;34m(self, batch, batch_idx, dataloader_idx)\u001b[0m\n\u001b[1;32m 736\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 737\u001b[0m \u001b[0;31m# optimizer step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 738\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_step_and_backward_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 739\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 740\u001b[0m \u001b[0;31m# revert back to previous state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 342 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)\u001b[0m\n\u001b[1;32m 432\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 433\u001b[0m \u001b[0;31m# model hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 434\u001b[0;31m model_ref.optimizer_step(\n\u001b[0m\u001b[1;32m 435\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 436\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 343 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)\u001b[0m\n\u001b[1;32m 1401\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1402\u001b[0m \"\"\"\n\u001b[0;32m-> 1403\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1404\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1405\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_zero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 344 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure, *args, **kwargs)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0mprofiler_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"optimizer_step_and_closure_{self._optimizer_idx}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprofiler_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofiler_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 215\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_total_optimizer_step_calls\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 345 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py\u001b[0m in \u001b[0;36m__optimizer_step\u001b[0;34m(self, closure, profiler_name, **kwargs)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofiler_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_closure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mCallable\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 346 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, opt_idx, lambda_closure, **kwargs)\u001b[0m\n\u001b[1;32m 327\u001b[0m )\n\u001b[1;32m 328\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmake_optimizer_step\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 329\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 330\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost_optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost_optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 347 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mrun_optimizer_step\u001b[0;34m(self, optimizer, optimizer_idx, lambda_closure, **kwargs)\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_closure\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mCallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 335\u001b[0m ) -> None:\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_closure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlambda_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_zero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 348 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, lambda_closure, **kwargs)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlambda_closure\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mCallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlambda_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 349 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 350 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 351 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/optim/adamw.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclosure\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mgroup\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_groups\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 352 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtrain_step_and_backward_closure\u001b[0;34m()\u001b[0m\n\u001b[1;32m 730\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 731\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtrain_step_and_backward_closure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 732\u001b[0;31m result = self.training_step_and_backward(\n\u001b[0m\u001b[1;32m 733\u001b[0m \u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhiddens\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 734\u001b[0m )\n", 353 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtraining_step_and_backward\u001b[0;34m(self, split_batch, batch_idx, opt_idx, optimizer, hiddens)\u001b[0m\n\u001b[1;32m 821\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"training_step_and_backward\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 822\u001b[0m \u001b[0;31m# lightning module hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 823\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhiddens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 824\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_curr_step_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 825\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 354 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, split_batch, batch_idx, opt_idx, hiddens)\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0mmodel_ref\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"training_step\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m \u001b[0mtraining_step_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 291\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost_training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 355 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_step_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_step_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 206\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpost_training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 356 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 157\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpost_training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 357 | "\u001b[0;32m\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0my_hat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_hat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mepoch_progress\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miters\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloader_len\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 358 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 359 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 140\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 360 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 361 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 443\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conv_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 445\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 362 | "\u001b[0;32m~/.cache/pypoetry/virtualenvs/torchdyn-voYSR01p-py3.8/lib/python3.8/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36m_conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 437\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstride\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 438\u001b[0m _pair(0), self.dilation, self.groups)\n\u001b[0;32m--> 439\u001b[0;31m return F.conv2d(input, weight, bias, self.stride,\n\u001b[0m\u001b[1;32m 440\u001b[0m self.padding, self.dilation, self.groups)\n\u001b[1;32m 441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 363 | "\u001b[0;31mTypeError\u001b[0m: conv2d() received an invalid combination of arguments - got (tuple, Parameter, Parameter, tuple, tuple, tuple, int), but expected one of:\n * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)\n didn't match because some of the arguments have invalid types: (!tuple!, !Parameter!, !Parameter!, !tuple!, !tuple!, !tuple!, int)\n * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)\n didn't match because some of the arguments have invalid types: (!tuple!, !Parameter!, !Parameter!, !tuple!, !tuple!, !tuple!, int)\n" 364 | ] 365 | } 366 | ], 367 | "source": [ 368 | "learn = Learner(model)\n", 369 | "trainer = pl.Trainer(max_epochs=3,\n", 370 | " progress_bar_refresh_rate=1,\n", 371 | " gpus=1\n", 372 | " )\n", 373 | "\n", 374 | "trainer.fit(learn)" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "3 epochs are not enough. Feel free to keep training and using all kinds of scheduling and optimization tricks :)" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": {}, 387 | "source": [ 388 | "## Galerkin Data-Controlled Conv Neural ODE (IL-Augmentation)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": 12, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "func = nn.Sequential(DataControl(),\n", 398 | " DepthCat(1),\n", 399 | " GalConv2d(10+10, 12, 3, padding=1, expfunc=Fourier(5)),\n", 400 | " nn.Softplus(),\n", 401 | " DataControl(),\n", 402 | " DepthCat(1),\n", 403 | " GalConv2d(22, 10, 3, padding=1, expfunc=Fourier(5)),\n", 404 | " nn.Tanh()\n", 405 | " )\n", 406 | "\n", 407 | "neuralDE = NeuralODE(func, \n", 408 | " solver='dopri5',\n", 409 | " sensitivity='adjoint',\n", 410 | " s_span=torch.linspace(0, 1, 2)).to(device)\n", 411 | "\n", 412 | "model = nn.Sequential(Augmenter(augment_idx=1, augment_func=nn.Conv2d(1, 9, 3, padding=1)),\n", 413 | " neuralDE,\n", 414 | " nn.Conv2d(10, 1, 3, padding=1),\n", 415 | " nn.Flatten(), \n", 416 | " nn.Linear(28*28, 10)).to(device)\n" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 13, 422 | "metadata": {}, 423 | "outputs": [ 424 | { 425 | "name": "stderr", 426 | "output_type": "stream", 427 | "text": [ 428 | "GPU available: True, used: True\n", 429 | "TPU available: False, using: 0 TPU cores\n", 430 | "CUDA_VISIBLE_DEVICES: [0]\n", 431 | "\n", 432 | " | Name | Type | Params\n", 433 | "-------------------------------------\n", 434 | "0 | model | Sequential | 49 K \n" 435 | ] 436 | }, 437 | { 438 | "data": { 439 | "application/vnd.jupyter.widget-view+json": { 440 | "model_id": "0e8f6943122c4163b035e33a668edd1a", 441 | "version_major": 2, 442 | "version_minor": 0 443 | }, 444 | "text/plain": [ 445 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" 446 | ] 447 | }, 448 | "metadata": {}, 449 | "output_type": "display_data" 450 | }, 451 | { 452 | "name": "stdout", 453 | "output_type": "stream", 454 | "text": [ 455 | "\n" 456 | ] 457 | }, 458 | { 459 | "data": { 460 | "text/plain": [ 461 | "1" 462 | ] 463 | }, 464 | "execution_count": 13, 465 | "metadata": {}, 466 | "output_type": "execute_result" 467 | } 468 | ], 469 | "source": [ 470 | "learn = Learner(model)\n", 471 | "trainer = pl.Trainer(max_epochs=3,\n", 472 | " progress_bar_refresh_rate=1,\n", 473 | " )\n", 474 | "\n", 475 | "trainer.fit(learn)" 476 | ] 477 | }, 478 | { 479 | "cell_type": "markdown", 480 | "metadata": {}, 481 | "source": [ 482 | "3 epochs are not enough. Feel free to keep training and using all kinds of scheduling and optimization tricks :)" 483 | ] 484 | } 485 | ], 486 | "metadata": { 487 | "kernelspec": { 488 | "display_name": "torchdyn", 489 | "language": "python", 490 | "name": "torchdyn" 491 | }, 492 | "language_info": { 493 | "codemirror_mode": { 494 | "name": "ipython", 495 | "version": 3 496 | }, 497 | "file_extension": ".py", 498 | "mimetype": "text/x-python", 499 | "name": "python", 500 | "nbconvert_exporter": "python", 501 | "pygments_lexer": "ipython3", 502 | "version": "3.8.5" 503 | }, 504 | "latex_envs": { 505 | "LaTeX_envs_menu_present": true, 506 | "autoclose": false, 507 | "autocomplete": true, 508 | "bibliofile": "biblio.bib", 509 | "cite_by": "apalike", 510 | "current_citInitial": 1, 511 | "eqLabelWithNumbers": true, 512 | "eqNumInitial": 1, 513 | "hotkeys": { 514 | "equation": "Ctrl-E", 515 | "itemize": "Ctrl-I" 516 | }, 517 | "labels_anchors": false, 518 | "latex_user_defs": false, 519 | "report_style_numbering": false, 520 | "user_envs_cfg": false 521 | }, 522 | "varInspector": { 523 | "cols": { 524 | "lenName": 16, 525 | "lenType": 16, 526 | "lenVar": 40 527 | }, 528 | "kernels_config": { 529 | "python": { 530 | "delete_cmd_postfix": "", 531 | "delete_cmd_prefix": "del ", 532 | "library": "var_list.py", 533 | "varRefreshCmd": "print(var_dic_list())" 534 | }, 535 | "r": { 536 | "delete_cmd_postfix": ") ", 537 | "delete_cmd_prefix": "rm(", 538 | "library": "var_list.r", 539 | "varRefreshCmd": "cat(var_dic_list()) " 540 | } 541 | }, 542 | "types_to_exclude": [ 543 | "module", 544 | "function", 545 | "builtin_function_or_method", 546 | "instance", 547 | "_Feature" 548 | ], 549 | "window_display": false 550 | } 551 | }, 552 | "nbformat": 4, 553 | "nbformat_minor": 4 554 | } 555 | -------------------------------------------------------------------------------- /torchdyn/module1-neuralde/m1b_crossing_trajectories.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Approximating a Reflection Map" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this tutorial we show how to painlessly train a neural ODE for approximating the **reflection map** \n", 15 | "\n", 16 | "$$\n", 17 | " y = -x\n", 18 | "$$\n", 19 | "\n", 20 | "This tutorial also serves as a warning on limitations of *vanilla* ODE models which should always be considered when designing your task-specific architecture.\\\n", 21 | "In fact, vanilla Neural ODEs cannot approximate (in 1D) functions which requires the flows to cross, e.g. the reflection map $y=-x$ as they would break the uniqueness of solutions (and thus the determinism). As we show later, one way to overcome this issue is to employ **data-controlled** models." 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from torchdyn.core import NeuralODE\n", 31 | "from torchdyn.nn import DataControl, DepthCat, Augmenter\n", 32 | "from torchdyn.utils import *" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "**Data**" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "The dataset contains pairs of `(-1, 1)` and `(1, -1)`" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 2, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "import torch\n", 56 | "import torch.utils.data as data\n", 57 | "\n", 58 | "n_points = 100\n", 59 | "X = torch.linspace(-1,1, n_points).reshape(-1,1)\n", 60 | "y = -X\n", 61 | "\n", 62 | "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 63 | "X_train, y_train = torch.Tensor(X).to(device), torch.Tensor(y).to(device)\n", 64 | "\n", 65 | "bs = len(X)\n", 66 | "train = data.TensorDataset(X_train, y_train)\n", 67 | "trainloader = data.DataLoader(train, batch_size=bs, shuffle=False)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "**Learner**" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 37, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "import torch.nn as nn\n", 84 | "import pytorch_lightning as pl\n", 85 | "\n", 86 | "\n", 87 | "class Learner(pl.LightningModule):\n", 88 | " def __init__(self, t_span, model:nn.Module, settings:dict={}):\n", 89 | " super().__init__()\n", 90 | " self.model, self.t_span = model, t_span\n", 91 | " \n", 92 | " def forward(self, x):\n", 93 | " return self.model(x)\n", 94 | " \n", 95 | " def training_step(self, batch, batch_idx):\n", 96 | " x, y = batch \n", 97 | " t_eval, yhat = self.model(x, self.t_span)\n", 98 | " yhat = yhat[-1] # select last point of solution trajectory\n", 99 | " loss = nn.MSELoss()(yhat, y)\n", 100 | " return {'loss': loss} \n", 101 | " \n", 102 | " def configure_optimizers(self):\n", 103 | " return torch.optim.Adam(self.model.parameters(), lr=0.01)\n", 104 | "\n", 105 | " def train_dataloader(self):\n", 106 | " return trainloader" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": {}, 112 | "source": [ 113 | "## Uncontrolled Neural ODE models" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "We first consider the following neural ODE variants: `depth-invariant` and `depth-variant` (\"cat\"). As we expect, these models will **NOT** be able to approximate the reflection map." 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 38, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "# vanilla depth-invariant\n", 130 | "func = nn.Sequential(\n", 131 | " nn.Linear(1, 64),\n", 132 | " nn.Tanh(),\n", 133 | " nn.Linear(64,1)\n", 134 | " ).to(device)\n", 135 | "\n", 136 | "\n", 137 | "# vanilla depth-variant\n", 138 | "func_dv = nn.Sequential(DepthCat(1),\n", 139 | " nn.Linear(2, 64),\n", 140 | " nn.Tanh(),\n", 141 | " nn.Linear(64,1)\n", 142 | " ).to(device)\n", 143 | "\n", 144 | "funcs = [func, func_dv]\n", 145 | "\n", 146 | "t_span = torch.linspace(0,1,100)" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 39, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stderr", 156 | "output_type": "stream", 157 | "text": [ 158 | "GPU available: True, used: True\n", 159 | "TPU available: False, using: 0 TPU cores\n", 160 | "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", 161 | "\n", 162 | " | Name | Type | Params\n", 163 | "------------------------------------\n", 164 | "0 | model | NeuralODE | 193 \n", 165 | "------------------------------------\n", 166 | "193 Trainable params\n", 167 | "0 Non-trainable params\n", 168 | "193 Total params\n", 169 | "0.001 Total estimated model params size (MB)\n" 170 | ] 171 | }, 172 | { 173 | "name": "stdout", 174 | "output_type": "stream", 175 | "text": [ 176 | "Your vector field callable (nn.Module) should have both time `t` and state `x` as arguments, we've wrapped it for you.\n" 177 | ] 178 | }, 179 | { 180 | "data": { 181 | "application/vnd.jupyter.widget-view+json": { 182 | "model_id": "611741801b6e4d27a7c93c0dc57a5d9d", 183 | "version_major": 2, 184 | "version_minor": 0 185 | }, 186 | "text/plain": [ 187 | "Training: 0it [00:00, ?it/s]" 188 | ] 189 | }, 190 | "metadata": {}, 191 | "output_type": "display_data" 192 | }, 193 | { 194 | "ename": "AttributeError", 195 | "evalue": "'tuple' object has no attribute 'detach'", 196 | "output_type": "error", 197 | "traceback": [ 198 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 199 | "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", 200 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;31m# plot the learned flows\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mtraj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrajectory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt_span\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0mplot_traj_vf_1D\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt_span\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_grid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_span\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 201 | "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'detach'" 202 | ] 203 | }, 204 | { 205 | "data": { 206 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD8CAYAAADUv3dIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAANIklEQVR4nO3cYajd9X3H8fdHs6zM2TqWWyhJWi2Ls8ENdBfnKKwO3Yg+SB50lASk6wiGdrMMWgYOhyv2UVfWQSFbmzFxLVRr+6BcaEqgnSJI47yitSZiuU1dTSrz1jqfSNWw7x6c4zi9Jt5/47nfu3vyfsGF8/+f3z3n+8+5952Tc84/qSokST0uWO8BJOl8YnQlqZHRlaRGRleSGhldSWpkdCWp0arRTXJXkueTPHmW65Pk80mWkjyR5OrpjylJs2HIM927gV1vcv2NwI7x1wHgn9/6WJI0m1aNblU9CPzsTZbsAb5UI0eBS5K8a1oDStIs2TSF29gKPDuxfXK877mVC5McYPRsmIsuuuj3rrjiiincvST1evTRR39aVXPn8r3TiO5gVXUIOAQwPz9fi4uLnXcvSVOR5D/P9Xun8emFU8D2ie1t432SpBWmEd0F4MPjTzFcC7xUVW94aUGSNODlhST3ANcBW5KcBP4O+BWAqvoCcBi4CVgCXgb+fK2GlaSNbtXoVtW+Va4v4C+nNpEkzTDPSJOkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWpkdGVpEZGV5IaGV1JamR0JamR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWpkdGVpEZGV5IaGV1JamR0JamR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWpkdGVpEZGV5IaGV1JajQoukl2JXk6yVKS285w/buT3J/ksSRPJLlp+qNK0sa3anSTXAgcBG4EdgL7kuxcsexvgfuq6ipgL/BP0x5UkmbBkGe61wBLVXWiql4F7gX2rFhTwNvHl98B/GR6I0rS7BgS3a3AsxPbJ8f7Jn0KuDnJSeAw8PEz3VCSA0kWkywuLy+fw7iStLFN6420fcDdVbUNuAn4cpI33HZVHaqq+aqan5ubm9JdS9LGMSS6p4DtE9vbxvsm7QfuA6iq7wJvA7ZMY0BJmiVDovsIsCPJZUk2M3qjbGHFmh8D1wMkeR+j6Pr6gSStsGp0q+o0cCtwBHiK0acUjiW5M8nu8bJPArck+R5wD/CRqqq1GlqSNqpNQxZV1WFGb5BN7rtj4vJx4P3THU2SZo9npElSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNBkU3ya4kTydZSnLbWdZ8KMnxJMeSfGW6Y0rSbNi02oIkFwIHgT8GTgKPJFmoquMTa3YAfwO8v6peTPLOtRpYkjayIc90rwGWqupEVb0K3AvsWbHmFuBgVb0IUFXPT3dMSZoNQ6K7FXh2YvvkeN+ky4HLkzyU5GiSXWe6oSQHkiwmWVxeXj63iSVpA5vWG2mbgB3AdcA+4F+SXLJyUVUdqqr5qpqfm5ub0l1L0sYxJLqngO0T29vG+yadBBaq6rWq+hHwA0YRliRNGBLdR4AdSS5LshnYCyysWPMNRs9ySbKF0csNJ6Y3piTNhlWjW1WngVuBI8BTwH1VdSzJnUl2j5cdAV5Ichy4H/jrqnphrYaWpI0qVbUudzw/P1+Li4vrct+S9FYkebSq5s/lez0jTZIaGV1JamR0JamR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWpkdGVpEZGV5IaGV1JamR0JamR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWpkdGVpEZGV5IaGV1JamR0JamR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGhldSWpkdCWp0aDoJtmV5OkkS0lue5N1H0xSSeanN6IkzY5Vo5vkQuAgcCOwE9iXZOcZ1l0M/BXw8LSHlKRZMeSZ7jXAUlWdqKpXgXuBPWdY92ngM8DPpzifJM2UIdHdCjw7sX1yvO//JLka2F5V33yzG0pyIMliksXl5eVfelhJ2uje8htpSS4APgd8crW1VXWoquaran5ubu6t3rUkbThDonsK2D6xvW2873UXA1cCDyR5BrgWWPDNNEl6oyHRfQTYkeSyJJuBvcDC61dW1UtVtaWqLq2qS4GjwO6qWlyTiSVpA1s1ulV1GrgVOAI8BdxXVceS3Jlk91oPKEmzZNOQRVV1GDi8Yt8dZ1l73VsfS5Jmk2ekSVIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY0GRTfJriRPJ1lKctsZrv9EkuNJnkjynSTvmf6okrTxrRrdJBcCB4EbgZ3AviQ7Vyx7DJivqt8Fvg78/bQHlaRZMOSZ7jXAUlWdqKpXgXuBPZMLqur+qnp5vHkU2DbdMSVpNgyJ7lbg2Yntk+N9Z7Mf+NaZrkhyIMliksXl5eXhU0rSjJjqG2lJbgbmgc+e6fqqOlRV81U1Pzc3N827lqQNYdOANaeA7RPb28b7fkGSG4DbgQ9U1SvTGU+SZsuQZ7qPADuSXJZkM7AXWJhckOQq4IvA7qp6fvpjStJsWDW6VXUauBU4AjwF3FdVx5LcmWT3eNlngV8Hvpbk8SQLZ7k5STqvDXl5gao6DBxese+Oics3THkuSZpJnpEmSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktTI6EpSI6MrSY2MriQ1MrqS1MjoSlIjoytJjYyuJDUyupLUyOhKUiOjK0mNjK4kNTK6ktRoUHST7ErydJKlJLed4fpfTfLV8fUPJ7l06pNK0gxYNbpJLgQOAjcCO4F9SXauWLYfeLGqfgv4R+Az0x5UkmbBkGe61wBLVXWiql4F7gX2rFizB/i38eWvA9cnyfTGlKTZsGnAmq3AsxPbJ4HfP9uaqjqd5CXgN4GfTi5KcgA4MN58JcmT5zL0BraFFX8m5wGP+fxwvh3zb5/rNw6J7tRU1SHgEECSxaqa77z/9eYxnx885tmXZPFcv3fIywungO0T29vG+864Jskm4B3AC+c6lCTNqiHRfQTYkeSyJJuBvcDCijULwJ+NL/8p8O9VVdMbU5Jmw6ovL4xfo70VOAJcCNxVVceS3AksVtUC8K/Al5MsAT9jFObVHHoLc29UHvP5wWOefed8vPEJqST18Yw0SWpkdCWp0ZpH93w8hXjAMX8iyfEkTyT5TpL3rMec07TaMU+s+2CSSrKhP1405HiTfGj8OB9L8pXuGadtwM/1u5Pcn+Sx8c/2Tesx5zQluSvJ82c7pyAjnx//mTyR5OpVb7Sq1uyL0RtvPwTeC2wGvgfsXLHmL4AvjC/vBb66ljOt9dfAY/4j4NfGlz92PhzzeN3FwIPAUWB+vede48d4B/AY8Bvj7Xeu99wNx3wI+Nj48k7gmfWeewrH/YfA1cCTZ7n+JuBbQIBrgYdXu821fqZ7Pp5CvOoxV9X9VfXyePMoo88+b2RDHmeATzP6fzl+3jncGhhyvLcAB6vqRYCqer55xmkbcswFvH18+R3ATxrnWxNV9SCjT2SdzR7gSzVyFLgkybve7DbXOrpnOoV469nWVNVp4PVTiDeqIcc8aT+jvyk3slWPefzPru1V9c3OwdbIkMf4cuDyJA8lOZpkV9t0a2PIMX8KuDnJSeAw8PGe0dbVL/v73nsasH5RkpuBeeAD6z3LWkpyAfA54CPrPEqnTYxeYriO0b9kHkzyO1X13+s51BrbB9xdVf+Q5A8YfXb/yqr6n/Ue7P+TtX6mez6eQjzkmElyA3A7sLuqXmmaba2sdswXA1cCDyR5htFrXwsb+M20IY/xSWChql6rqh8BP2AU4Y1qyDHvB+4DqKrvAm9j9B/hzLJBv++T1jq65+MpxKsec5KrgC8yCu5Gf60PVjnmqnqpqrZU1aVVdSmj17F3V9U5/6ch62zIz/U3GD3LJckWRi83nGiccdqGHPOPgesBkryPUXSXW6fstwB8ePwphmuBl6rquTf9joZ3/25i9Lf8D4Hbx/vuZPRLB6MH5mvAEvAfwHvX+x3LhmP+NvBfwOPjr4X1nnmtj3nF2gfYwJ9eGPgYh9FLKseB7wN713vmhmPeCTzE6JMNjwN/st4zT+GY7wGeA15j9K+X/cBHgY9OPM4Hx38m3x/yc+1pwJLUyDPSJKmR0ZWkRkZXkhoZXUlqZHQlqZHRlaRGRleSGv0vMzgPTisUvzQAAAAASUVORK5CYII=\n", 207 | "text/plain": [ 208 | "
" 209 | ] 210 | }, 211 | "metadata": { 212 | "needs_background": "light" 213 | }, 214 | "output_type": "display_data" 215 | } 216 | ], 217 | "source": [ 218 | "import matplotlib.pyplot as plt\n", 219 | "\n", 220 | "plt.figure(figsize=(12,4))\n", 221 | "plot_settings = {'n_grid':30, 'x_span':[-1,1], 'device':device}\n", 222 | "\n", 223 | "for i, f in enumerate(funcs):\n", 224 | " # define the model\n", 225 | " model = NeuralODE(f, solver='tsit5', sensitivity='interpolated_adjoint', atol=1e-3, rtol=1e-3).to(device)\n", 226 | " # train the neural ODE\n", 227 | " learn = Learner(t_span, model)\n", 228 | " trainer = pl.Trainer(min_epochs=100, max_epochs=200, gpus=1)\n", 229 | " trainer.fit(learn)\n", 230 | " \n", 231 | " # plot the learned flows\n", 232 | " plt.subplot(1,2,1+i)\n", 233 | " traj = model.cpu().trajectory(X_train.cpu(), t_span).detach()\n", 234 | " plot_traj_vf_1D(model, t_span, traj, n_grid=30, x_span=[-1,1], device=device);" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "## Controlled Neural ODE models\n", 242 | "\n", 243 | "Following the work in [Massaroli S., Poli M., et al., 2020](https://arxiv.org/abs/2002.08071), we can easily approximate the reflection map leveraging **data-controlled Neural ODEs**. Data-control allows the Neural ODE to learn a family of vector fields instead of a single one, via conditioning the vector field `f` with the initial condition `x`." 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": 6, 249 | "metadata": {}, 250 | "outputs": [], 251 | "source": [ 252 | "# define the data-controlled model\n", 253 | "f = nn.Sequential(DataControl(),\n", 254 | " nn.Linear(2, 64),\n", 255 | " nn.Tanh(),\n", 256 | " nn.Linear(64,1)\n", 257 | ").to(device)\n", 258 | "\n", 259 | "model = NeuralODE(f, solver='dopri5').to(device)" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 7, 265 | "metadata": {}, 266 | "outputs": [ 267 | { 268 | "name": "stderr", 269 | "output_type": "stream", 270 | "text": [ 271 | "GPU available: True, used: True\n", 272 | "TPU available: False, using: 0 TPU cores\n", 273 | "CUDA_VISIBLE_DEVICES: [0]\n", 274 | "\n", 275 | " | Name | Type | Params\n", 276 | "-----------------------------------\n", 277 | "0 | model | NeuralDE | 257 \n" 278 | ] 279 | }, 280 | { 281 | "data": { 282 | "application/vnd.jupyter.widget-view+json": { 283 | "model_id": "be7d5f252708455b8784852bb74e1112", 284 | "version_major": 2, 285 | "version_minor": 0 286 | }, 287 | "text/plain": [ 288 | "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…" 289 | ] 290 | }, 291 | "metadata": {}, 292 | "output_type": "display_data" 293 | }, 294 | { 295 | "name": "stdout", 296 | "output_type": "stream", 297 | "text": [ 298 | "\n" 299 | ] 300 | }, 301 | { 302 | "data": { 303 | "text/plain": [ 304 | "1" 305 | ] 306 | }, 307 | "execution_count": 7, 308 | "metadata": {}, 309 | "output_type": "execute_result" 310 | } 311 | ], 312 | "source": [ 313 | "# train the neural ODE\n", 314 | "learn = Learner(model)\n", 315 | "trainer = pl.Trainer(min_epochs=200, max_epochs=250, gpus=1)\n", 316 | "trainer.fit(learn)" 317 | ] 318 | }, 319 | { 320 | "cell_type": "markdown", 321 | "metadata": {}, 322 | "source": [ 323 | "**Plots**" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 8, 329 | "metadata": {}, 330 | "outputs": [], 331 | "source": [ 332 | "# evaluate the trajectories of each data point\n", 333 | "s_span = torch.linspace(0,1,100)\n", 334 | "traj = model.trajectory(X_train, s_span).cpu().detach()" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 9, 340 | "metadata": {}, 341 | "outputs": [ 342 | { 343 | "data": { 344 | "text/plain": [ 345 | "Text(0.5, 1.0, 'Depth-Trajectories of Controlled Neural ODEs')" 346 | ] 347 | }, 348 | "execution_count": 9, 349 | "metadata": {}, 350 | "output_type": "execute_result" 351 | }, 352 | { 353 | "data": { 354 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAADkCAYAAACPHMP0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eZwj2VUm+l0plfu+L1WZVV1VXfva1ZttwM8242XMMO/BwIDB4PdjGs88M483zGAY4A0w9uDhDY9tGGwPi83YGHhsbrMN2MZ7b1W9d1d3rVm5Ve6LcpFS231/fDrcG6EISamUMpVV8f1+8ZMUEZJCiojz3XvO+c5RWmsECBAgQIAAxSK02wcQIECAAAH2FgLiCBAgQIAAW0JAHAECBAgQYEsIiCNAgAABAmwJAXEECBAgQIAtISCOAAECBAiwJQTEcRdDKTWqlHrbDn/nK0qpN+/kd/ocx0eVUj+zw9/5IaXUvFJqeie/txxQSr1ZKTVhvS752tmN665UuH93gOIQEMcOIXszxZRSq0qpZaXUN5RS71dKleUcKKU+oZT6UAnvG1ZKrVmLVkqtW6+/aSufp7U+qbX+0laPw3VMJf0W13G8X2v9H7fzGVuBUmo/gB8DcEJr3e+zT6tS6leUUmPZ//Z69nV3Gb5fK6UOb/dzKo3sudVKqYesdYeVUlUpKFPEv1NKXcvev2NKqY8opeqsfT6hlEpk7+1VpdTLSqlfUEq1Wfv8oFIq7brX1pRSg7vzy7aHgDh2Ft+mtW4BMALgIwA+COC3d/OAtNZjWutmWbKrz1rrvir7KqVqdukwtwSlVHgXvnYEwILWetZro1KqFsAXAJwE8A4ArQDeAGABwENe7yknquzcLQLY1sCgGGSN/nZt3K8BeAzAewG0AHgngLcA+CPXfr+Yvbd7ALwPwCMAvq6UarL2ecK+17LL1DaPb3egtQ6WHVgAjAJ4m2vdQwAyAE5lX9cB+C8AxgDMAPgogIbstjcDmADw7wHMZz/vPdltjwFIAkgAWAPwOes7/y2AFwGsAPhDAPUFjlMDOJx9/oMAvg7gl2Fu9kMAvggavHkAnwbQ7vU7wYHJTwC4kd3/jwB0Wvu+CcA3ACwDGM9+n99vOQ7gS9l9XwHwT6zP+QSA3wTwVwDWAbwtu+5D1j7vBvB89v3fAHDG2vZBAJMAVgG8DuCtPv9NG4DfAzAH4DaAn87+xrcBiGXP5RqAT3i894ey57Q5z39f6Df+BoC/zB7nUwAOZbd9JXve1rPf/93W9fJBANMA/gd4ff0KgKns8isA6uzrq8Tz+P3Z/2MBwE/B41p3/Y7/N3tM35JddxiAdv3Pvw3gTva8fAhAOLvtZwF8ytr3QPa312RffwnAh8HrNpb97PcBuJL9324C+GHr/Y7f7TrWIwDSAB5yrd8PYBPAW6zf9CHXPi3Z4/+AdS99Lc+5L+oarJYlmHHsIrTWT4M3t7iD/jOA+wGcAy/4IQD/t/WWfgDd2fU/AODjSqmjWuuPgwb8FzVHMd9mvee7wBHuQQBnwAt4K3gYvNl6wRtSAfgFAIOgodsP3sxe+NcA/imAb8nuvwQaPyilhgH8NYBfB0dp5wA87/VblFIRAJ8D8LfZ4/gRAJ9WSh21vut7s8fXAuBr9kEopS4A+B0APwygC8DHADyulKrLfsYHADyoOWJ8O2j4vPDroFG7L/ub3gvgfVrrz4Mj0ansMf+gx3vfBuBvtNZrXh9c5G/8HgA/B6ADwPXs74XW+puz22Wm+IfZ1/0AOsHZ0GOgUX8E/K/PggOXn/b5rTbynccTIGl/f3ZbF4B9BT5vA8B/kuP3wCcBpMB74DyAfwQSb7H4fvD3toCENgsOHFpBEvnl7DVRCG8FSeVpe6XWehzAkwC+1e+NWutVAH8Hc2/7YovXYFUgII7dxxSATqWUAvAvAPxfWuvF7IX3nwD8c9f+P6O13tRafxkcfX5Xgc//Na31lNZ6ETRM57Z6fFrrX9dap7TWMa31da3132WPYQ4cPX6Lz3t/GMBPaa0ntNabIMF8Z9Zt8h4An9daf0ZrndRaL2itn/f5nEcANAP4iNY6obX+IoC/AA2p4LNa669rrTNa67jr/f8CwMe01k9prdNa60+CI8ZHwBFlHYATSqmI1npUa33DfQBZ99d3A/hJrfWq1noUwC+BRqoYdIEjUD8U8xv/VGv9tNY6BZJroXOZAfAfsucqBv7nP6+1ns2eu58r8vjzncfvBPAXWuuvZLf9TPZ7C+FjAIaVUu+0Vyql+kAS/lGt9bqm6++XkXsf5MMntNavZK/ZpNb6L7XWNzTxZZCci4nddcP/nN3Jbs+HKZC4BY9k45uyyHVW1DVYTQiIY/cxBLqBegA0ArgsFxaAv8muFyxprdet17fBUV4+2Bk+G6BxglLqr60A3XvyvH/cfqGU6lVK/YFSalIpFQXwKfjfQCMA/sz6PVfAm6QPnKkUe3MMAhjXWtsG6Tb433kep8dx/Jh902a/f1BrfR3Aj4LGcDb727z+024Atdnv9TuGfFgAMJBnezG/0fNc5sGci0QHkXv8xQRn853HQVj/ffb6XCj0gVmS+Y/ZRbm+KwLgjvV9HwNnYcXCfc2+Uyn1pFJqMft570Jhow/QFet3zgay2/NB7m3Bk1rrdms5BABbuAarBgFx7CKUUg+CF9fXwIswBuCkdWG1aROwBoAOV7BtGBzVAPTzFg2t9Tu1CdB9Ot+urte/kF13RmvdCuD74LzxbYwDeKfrZqnXWk9mtx0q8junAOx3BTqHQZ+w33vcx/Fh13E0aq0/AwBa69/XWr8JNFoadBm6MQ/GXkbyHEM+fB7A213nz0Yxv3Gr8Pof3cdfTHA233m8A5IwAEAp1QjOrorB74Kuv//V9V2bALqt72rVWp/Mbl8HB1gCrwy2f/jd2eynPwFjh31a63YwFuZ3zdr4InhOHMkL2Qy6R8BkB08opZpB9+RX/fZxHHBx12DVICCOXUA2LfPdAP4ADPS9lB1p/nfQ/9qb3W9IKfV219t/TilVm02TfTeA/y+7fgb0vVcaLWAAdlkpNQTg3+XZ96MAPqyUGgEApVSPUurbs9s+DeBtSqnvUkrVKKW6lFLienH/lqdAg/HjSqmIok7k28D/rxj8dwDvV0o9nM20aVJK/WOlVItS6qhS6i1ZAxMHyTvt/gCtdRoMCn84+74RAP8GnHEVg/8BGsU/UUodU0qFsr/53yul3lWG31jM+f8MgJ/OnoduMH5WzPHnO49/DODdSqk3ZTPHfh5F2pWsy+1nwcCwrLsDupJ+KXufhJRSh5RS4g59HsA3K6aRtwH4yQJfUwu6geYApLKusX9U5PFdBX/7p5VSjyilwkqpkyARfT4b23IgGzd7AMCfg7Gg3y30PcVeg9WEgDh2Fp9TSq2CBuSnwPjA+6ztHwSDnk9m3UCfB2AHR6fBi3EKNLzv11q/lt3226CPdFkp9ecV/A0/B+ACmKX1lwD+NM++vwrgcQB/m/3dT4LBdmitx0CXwY+B0/nnwYAt4PotWusEgH8C+r7nAfw3AO+1fnteaK0vgXGO/wr+f9dhkgTqwNToefD/7QUz17zwI6BxvwnOEn8fDLoXcwyb4Aj0NTBoGgXwNOgyeWq7vxE0wJ/M/md+ca8PAbgEZtm9BOBZFJcWm+88vgLg/wD/izvg/7sVQd1nkBtHeC9o8F/Nft4fI+sy0lr/HZgd+CKAy2AcyBfZWOG/Bkl/CUyieHwLx/cBAL8FEuwa6D7+EoDvcO3349n/ZhHMvLsM4A0u1/KjKlfH8SC2dg1WBZTWVam7CeBCdgT6Ka11oYyVXYVSagzA92mtv7LbxxIgQIDKIJhxBCgblFI9YDB/dJcPJUCAABVEQBwByoLslPsagF/PuqECBAhwlyJwVQUIECBAgC0hmHEECBAgQIAtISCOAAECBAiwJVRTxcyKoLu7Wx84cGC3DyNAgAAB9hQuX748r7Xu8dp21xPHgQMHcOnSpd0+jAABAgTYU1BK3fbbFriqAgQIECDAllBVxKGU+h2l1KxS6mWf7Uop9WuKndNeVMWVRg4QIECAAGVEVREH2BDlHXm2vxNsrnIErLf/mztwTAECBAgQwEJVEUe2TMVinl2+HcDvZevqPwmgXSmVr1R1gAABAgQoM/ZacHwIzlr7E9l1+RrkBAgQIMBdD62BWAxYXzdLQwNwXwVqZu814vCqoZ8jfVdKPQa6sjA8PFzpYwoQIECAHcPmJrC2RmKwHzc2gIzVBiwcBgYq5I/Za8QxAatpDNjbOKcRjWbf6o8DwMWLF4OaKgECBNhTyGRIBGtruUsyafYLhYCmJqC5Gejv53NZ6usrd3x7jTgeB/ABpdQfgP0AVrKNXwIECBBgzyGTIRmsrprH1dXc2UN9PclhaIiPzc0kh4YGQBXTy7DMqCriUEp9BsCbAXQrpSYA/Aew/zC01h8FWz6+C2zEswFnE6QAAQIEqEpoTZdSNGrIYXWV66TOrFJAYyPQ0sLZQ0uLIYmaqrLUVUYcWuvvKbBdg93GdgTpNP2EAQIECFAsNjdJEPaytmZmEDZBDAzwUUgiVFV5rv6oKuKoJqyuAk8+CZw5A/T17fbRBAgQoNqgNQkhGgVWVgxJbG6aferrSQo9PUBrqyGIvT4gDYjDBw0NPOmXLgGPPAJ0de32EQUIEGC3kE5zMLmyAiwvG5KQWUQoRFLo7SVByFJbu3PHmEoxHXdjwyxBOu4Oo6YGePhh4OtfB55+GnjjG3khBAgQ4O5GOk1SWF4mUayskDQkFhGJAG1twMGDtAltbQxUV9rNJDoNmxjW183zRMK5fygUpOPuCmprOdv42tfotnrjG3mBBAgQ4O6A1iSF5WVgaYmPNknU1gLt7XRXt7eTJBoaKnc8koZri/jkdSzmzLSSWEljI4Pp8lyWurrKHWdAHAXQ0AA8+ihnHkIelcyPDhAgQOUQi5EghCRWVjjDADiT2AmS0NqQgQj45Hks5ty3poaD1bY2zh6amgwx7FYqLhAQR1Fobqbb6oknSB5veMPO+i4DBAiwdaTTJIalJWBxkUQRj3NbKERjPDJCkmhvL783IZn0FvC5NRqRCL+7s9Mp4Gtqql47ExBHkWhvBx58EHjqKS6PPlp9udUBAtzL2NwkQSwukixWVoyBbmxkgktHB5fW1vLFJOJxp4BPCMLOrgqFeAy2wls0GtVKDvkQmL48uHkTGB42BNHdDTzwADOtnnkGeOihvZ9WFyDAXsX6OrCwYMhifZ3rQyEO9O67zxBFOfz9os+wBXzuEiCRCLOr+voMMTQ3kzR2y61UCQTE4YPlZeDVV4E7d+imEvLo7wfOnweefRa4fBm4eHHviHYCBNirkCD24iLJYmHBjOhra+nmGRnhY1vb9u7JVIrfJSQhj3bWUm0tCWJoyGgzWloqG5AuBpkM/5dYjDOhmhqmCJcbAXH4oL0duHCBBPHUU07yGBrixfXii8Bzz3G/u2k0ESDAbkOIQkhiYcEY7oYGzv67urg0N5f+PRsbuQK+jQ2zvabGlAARAd9uEYTWhhTcSzzOR9s9BvB/CohjhzE4yEcv8hgZIXm8+irdVWfPBuQRIMB2sL4OzM+bRYiisZGuHyGKxsatf7YQkRCEPNpupqYmDhiHh0kOra07m7mUyRidhv1oL9pV67umhlmeDQ08XnkuAuZKZYAGxFEA+cjj0CGSx9WrJI/Tp3fvOAME2GvY3CRBzM3xUVJR6+s5Su7u5rLVlNhMxqnyFpKQQHk4TCM7OEi3lqi8Kx2v9BLw2SQhGV8CpQwRdHTQ02GTQkMDYyq7gYA4isDgIE/i5ctMx334YXPCjh7lBXn9Ov2qJ0/u7rEGCFCtyGTocpqb4xKNcn0kQoI4fJg1nbaSFiv1opaXzWKThKi8Dxww2oympsrNIlIpp6LbLeCzZwxKGSLo6XHqMxobSQ7V6sUIiKNIDAwwEH75MvUcjz5qyOP4ceaM37zJUcuxY7t7rAECVAvW14HZWS4LC7xPQiEGsY8do8FsayveQMbjTgHf8rIR8NXUmFIgQhKVyGZKpZzKbntxxxhEoyEzBlvZvZsCvu0iII48cJdV7+8neVy6BHzjGyxHIkGyU6c4yrl2jRfD0aO7c8wBAuwm0mm6nYQsJNDc1MTYQW8v4xTFuIVsAZ8stoCvtZWfaQv4ymWIRd1tt2aV526XUn09v7uvj4Qg4r3Gxt1zJVUaAXH4YH2d5HDiBEcKgr4+6jeeeYbbH33UBKBOnyZ5XL3KC/vIkd059gABdhKxGDAzw2V+nvdAOEz306FDJItiAtrxuBHv7ZSAL502pGCL99bXneru2lqSgbjSpANfU1N1a7kymcrIBQLi8IFcKM8+y+mnXZq4p4dxjqefNuQh086zZzlaee01vj58ePd+Q4AAlYDWdBEJWUisoqmJsQSZVRQyWGtrTgGfzE5CIZJDOQV8qZSzNauQhJ16q5QhBRHwyetqUndLWm487nx0P9/cJHk/9FD5jyEgDh9EInRFPfss8MorHFWdOGGmwl1d3P7UUyyA+OijZqp87hxP7pUr3DcgjwB7HZkMA9rT0ySLzU1e652dvC/E0PpBaxKMaDIWF026bV0dP+fgQT5uZzZh9/C2Vd42QYRCPNaODmD/fiPg24nS6PmgNf+TeNx/EULwQm0t/8u6OkO2bW2VOdaAOPIgFGKJkVdeYeB7c5OkIBdXRwcLHj7xBMnjkUd40StFdTkQkEeAvYtkknGK6Wk+plJGidzfz0c/H77WdDXZAr5UittEl9HZyQFYqcUF43HvFq2SuWQThGgzWlp2p/yHW7wngj37MR7P1WkoZcigoYGxHNFn1NWZx7q6nSW9gDgKQCkGvuvrSQLxOIsdyg3T2spS6088QbfVww/zQg3II8BeRCJBorhzx8Qr6uoY5+vvp+vDy0CJwE7Ee4uLRlzX3Mz3i4Bvq6I0rxatKyvOEiAigOvvN+K9nZxBJJOGFESbIaQgz92kEA4bPYb8L/LaJodqzLwKiKNIHD7ME/r882zs9MgjRpjU3Ay86U0kjyeeILH09OSSh9ZBwDxA9SGRIFEIWWjNUfnBgzTEMhByY2PDKeATQ97UxPR1KQuyFaIQV5PdfS8aNSm30qJVSoDIUunspWQyV7Bni/lkNiUIhYxGo6vLPK8G8V45EBDHFjA0xJP+zDPAV7/KoFN7O7c1NHDm8eSTDJpfuMCbR8hDKQbMtQbuv393f0eAADKzmJoyZNHUxAHSwIC3bzyZdAr4pBptqUpvu/ue3ctbsplElzEyYhTezc2VmUXY5T7sdqwi3LNLk8ixiRbDJgZZV1tbnTOFckFp9/zpLsPFixf1pUuXSnqvu6y6YHWV5LC5SYLo7zfbkkluW1oCzpzh+wHeJC+8AIyPc9YRiAQD7DRSKZLF5CQNv5DF4CCX1lbn/hKnmJ3l/ktLXFdTQ2PZ08Ol2CKDGxvOFq12972aGiPaa2vj83LHItLp3LasQhJuVbf0z/Bb9vJsoVgopS5rrS96bQtmHD6Qsuq3b9P1ZN8cLS10TT39NGcfJ0+adF3Jxrp0iUSxuUmikFRdpSgSzGSYjRIgQCWRydDwT06SNDIZjogPHTK1mmwkEiSJmRk+ivupvd2UBOnoKDzqT6UMScgin+XVfa9cJCEzB1u4JwThbsvqVnXbwr1qjS34IZXi/ytLMknbU19v6u2VEwFx+KC9nQRw+TLdUufPO2cWdXXMqHruOWZdra0xiB4KMej14IMkjtde4wk8eZIX4pkz3H7jBkdAp07trQs0QPVDaxrqiQm6opJJuk5GRmhEOjud+0ejRpOxvMz319aSJPr6+FhIx7C+blq0Li0ZbQdgdBEdHbyvWlq2727ya8u6vu6cOUQi/H7J3rKXap41JJMkgM1N56PfOlusaKO7OyCOHUd3N/DN32w6/h05wlIiYujDYabrvv46ZxHr6yxJEonwxjh3jgRz4wZP8vnzXC8EI+QRlGQPUA6sr5MsJibofgmHOdjZt88kawA0MvPzJIrpaVNCo72d13hfX/76UZkM3Ux2m1bRFkQi/JyjRw1RbMdAb27mtmVdXc1tyypCvf7+6hXuySzALdKTRchgc9OfCGpqjF6joYHnSV7X1vK/lteRSOXIMSCOApCg90svkRyWlhjXECWrUoxXNDdzhvHVrzIlV8SAJ05w31df5QUhqbwnTvDGvnqV5CGkEiDAVpBMMhtqfJxGHCBJHD1KIyrxuUTCqcmQOmw9PUaT4afOTqd53Ytwb2nJxCakDEdnJ5fm5tIGQclkbltWd9c9aarU22s67u12W1ZbtCek4FZ0y6MXGdg6jbo6/iZbyOd+Xi02IgiObwHj4+z6V1vLmYZ7yr+4yJmJ1tze02O2TU4ylbepiS4wSVG8cYOk0tvL2Uo1170JUB3Qmtfa2BhJI52mAd2/n7MLubZiMaPJWFzk++rrOaPIp8lIpUgO8/MkC3FfKcUAeleXIYqtlgKRdFtbtLe66iwcKAThXirVlMgP6bQhAluk5168TKgYfFugZz+XJRKpXm9DvuB4VRGHUuodAH4VQBjAb2mtP+La/mYAnwVwK7vqT7XWP5/vM8tJHAAv9EuX6Ao4doxBRvvEb2yQPFZXOauwa1zNz3NbTQ1nJZLFcvs2Camriym+7iyuAAEAGqnxcRLGxgavk6EhUyEWoLtKNBnLy1wnuof+fm8XVDptennPzzuJor3dCPc6O7d2bSYSuW1ZV1dzld3SklUet9q4qRRkMk6Bnpdgz57tCOyOe7Z6u6HBSQ7VMjPYDvYEcSilwgCuAvhWABMAngHwPVrrV6193gzg32qt313s526HOPwqS6ZSdEtNTXH0du6c05eaSjFoPj3NUeCZM+ZzolHWt0qlOMOQWcnkJN/T2soZSTX5ZgPsHrSma2lsjDEJrWnER0ZIBOEwR/B37vB6lKB0ezv1GAMDuSU9pEihiPeWlnitC1GIcK+zs/gZcCxmBHuy2LOI+nqnYK/Syu502inQ81J1uxGJ5Ir03Grue2lQt1fScR8CcF1rfRMAlFJ/AODbAbya910VQjzOeMWhQ1TQ2qO0mhq6orq6mFH15S87XVc1NSSFq1e5rK3xtdw83/RNJI+nnjJaj6Ehvu/SJVP3aidGXgGqE/E4yWJsjIauro7X4vAwDe76OnVGNll0djJ7b2Ag99pZXzfCvfl5o3SWxkfd3cXPKIQkpJGSu/yHZDGJaE8CuOWELdjzasXqLgRoK7l7epxKblkCN3HxqCbiGAIwbr2eAPCwx36PKqVeADAFzj5eqdQBtbaSGKanmfnkHrkdOMDMkcuXWafq6FHmuitlmjm1tnIm8ZWvkDw6O0kgb3yj0XpsbHDfvj5W2X36aZY1efTR4sVVAfY+tKZRHx01s4ueHpJBXx+N4dQUZ6crK3yPkMXgoDMGkEqZhkpzc6Y6bGMj9+3pIVkUMuiJhCEI6ZEhRlkpupYkC6u9vby9u6XTni3ak+fu2IIQQ2Mjj8HdhnWv6TK8kE4ziaCYJZXiY1sbbVe5UU2uqn8G4O1a6x/Kvv5+AA9prX/E2qcVQEZrvaaUeheAX9Va51R/Uko9BuAxABgeHn7g9u3bJR/X+Djw8su8SI8fJ1m4L8BUijGKyUnejOfPO2/i1VXGNmIx3uQHDnB9JsNsrbEx3sznzvGmi0ZZuiSTMUUTA9y9SCZ5nY2O0ijW1tLFOTJC98mdO7y2Fha4f3u7UXvbM4to1HTek2B4TQ2vSVF556tEK2m2ouxeWnKWI29pMYI9mU1slyTS6Vyhnl8bVumRI6RgP6/m/txupNNGpOcW7cmj13O/FF1BTQ0XScONRGg7Sq2Pt1diHI8C+Fmt9duzr38SALTWv5DnPaMALmqt5/32KUdwPB7nzGB2liO8c+e8b8CxMZJMOMx9+vrMtmSSvT1mZ2kUTp82N51kVnV0MF23ro437JNP8rsvXmTWVYC7C6urwK1b1F2k0zz/Bw4wdiFq79lZGozmZmZMDQ6aa09mFTMz3E/89q2tRriXT+Wdr+Nefb1poiREUap/X2uj5raFemtr/m1Y3UtjY3XGFzIZp/F3C/O8lnwEEA4bLYboMuzFa50s5SbOvUIcNWBw/K0AJsHg+PfariilVD+AGa21Vko9BOCPAYzoPD+inFlVExMkhnSariV3RhXAm+HyZY7+Dhwweg2AN5DEPVpbSRLSUvPOHbq0amuZWdXaygvwqaf4WWfPknAC7G1oTUN/6xaNfijE+NbBg7yuxsd5LSSTNKJDQ1ykNIhXm9aaGpJEby8Xr7RVaaRki/akBEc4zM/v6ODASHo+bBV2EyW3ots2lqLmttuvyvNqiDOIUM9LmOdWbbuLH9qwCUBSb+W1Ldazn1dTNtaeIA4AyLqffgVMx/0drfWHlVLvBwCt9UeVUh8A8C8BpADEAPwbrfU38n1mudNx43G6l6anjf/QXe8nk2EZ9Zs3eUM88ICzgNzsLGcfAN1aMjNZWWF8I5k0xRNTKcZC5uaY/huUZd+bSKVICrducbRdX2+C0rOzHJSsr9NwDgxwdtHdzYHJygqvt+lpZ5vWvj7TEMltcNJpupskzXZpyQTE6+uNDqOU/t22FsNux7qxYeIOSnFQJARhL7uRMZjJeIvyvFTcfjMCW6Fti/Ls9fayV1xnftgzxFEJlJs4BHfukEASCWo1jh7NHS3NzVH0l0jkzlA2NkgIKysMqB87xm3xOOMhy8uMqRw+zAv5hRdoXEZG6Oba6xflvYJYjGRx+zYNd0cHzyHA8zmfdbJ2d3NGKSm2CwuGLGIxnu+ODm73atNqazFEtCcGsLXVEEVn59bKnm9sGP2FPNr1oJQyhCBCPXm9E6Nnu/92vsVvZuAW5NlaDDdB3Gv3XEAcJRLHrVsc+fnVe0kmGZsYG+Po6vTp3FhEIsHA+Z07vGnPnzfuqXSaWVu3bzN98cIFXrTpNIlicpJuirNnaUyuXAGuX6fheOCB6pjWB/BGNMpzNTXF19LYKBrleU0meR3Yau+5OV4nMzO8buySIH19zpG6lAERdbeUPFeKM2BbtFdMvSIp+eFWdEtpEYCzHFuo19JSWS2GzBLytVBPRQ4AACAASURBVFvd3PRvtyoaDFuo51Zv32tksBUExFECcSwvU8cRibDx0oED/jfI4iIN/doag5cnT+b6iCU7C+B26dMBcOT54ov0VZ8/b0SB166xum5bG+MhDQ3MvHn5Zd68Dz+89ZIPASqLuTkmO8zN8XwODvIczc5ydhkKkUSGhzmDmJsjuczMcEYSiZAkBgZ4HcjgIJMxor35+VzRXlcXiamjo3AQOR53CvWiUWf2VG2tU6gnJFHugYq0W7UFevbiRQpu5bb7eX39vTk7sKE1Cd92TZaCgDhKnHFEo5xRzM1xdHjsGA2B10WZyXCEee0ajcPRo7nCwViMAfCFBc5Mzp41J3V1lUH11VVnFd6ZGcZDQiFmV3V1cd3ly7xBHn6YN3WA3YPWnClcv05DXF9PI57J8Fyl0zTAw8O8fpaWnGRRW8tZhcxKZICytmZEewsLTtGedNsrJNqLxZwd9txivaYmp1CvtbV8NaGk3apbqCev87Vb9Wq1Wl9f3aXQt4NMhv+HLKLF2MoiZGHPEnt7aSNKQUAc24xxzM2RQKJR3ljHj/unx66vc0YwO8t9T592FkPUmrOGK1d4o5w+TXcUwBP+8st0fXV20nXV0EAD8vTTvNlOniQhraww4yqddpYuCbBzyGQ4W7x+3QS8W1o4Uo5GOUIfHCRhpNOmmVIySQM4MMDtXV28FpJJQxR2em1Tk9FidHX5B5fjcaeae3nZkISI9aTDnpDEdlJcJc3W3WpVnudrtyqP9vO9OnvW2hh7twDP67WbGJLJwhoNgWg17CUc9l/X2Fi6bQiIowzBca05SnztNd4UXV2cgbgr5Aru3GH8IhajD1vKqwvW1xk4X1yka+LMGTPSm5yk60q6Bg4M8OJ67jmOUvft4/6JBAllddXZpjZAZZFOMy514waNdW0tz52Mopub6dpsbCQBTE3xXNXUGLKwM6ZEtCfFBSMRp2hPYmI2pMOereoWohGSEP2FKLpLiUVkMv7tVu0sKsCp3rbFeXul3apt6P2EeF7r3DMnL7jFefZr+9G93k0GO+mCC4ijjFlVmQxnBFevcmTZ10e3kjslF6CBuXaNBkbcV3asRGsG4K9c4UVx8qTRaqyv00W1vMwsnJMn+b5r19g4SnQgtbV0W83OMrvrxIl7279bSaRSnC3evEkjLaO6RIL/eX8/DX0sRvLf2OA56+/nrLK3l9eEtGadnXW2Zu3tNaI9L32Q3YbVrjLb1GQU3UIWW4lHaM3f4263uraW24s7EjHEIOQgj9Wi3tY6V5Xttdj7JJPe5dEFoVB+8Z17cZNENfwvW8W2iEMp5TOmdiCjtV4u5eAqjUql46bTNPrXr/OiGxhgEN3WawjW1jj7mJ3laPTUKef00Z599PRw9tDYSJJ67TUST3MzXVdtbU4dyIULfM8rr/B4enuZcVWNKtu9ilSK/+3Nm8aQhkJc6up47iMREsLyMo1EdzfJYmCARllEe1IKJBLhufJqzWprMKQdq7h9pMOereouVhdhl/eQRdJrbb94TY1TtW0L9XZDgyFE4CXA82qj6lUOXSAEkE+E51ZoV5swb6ewXeKIgwUF83FmWGtdlY6SSpRVt5FM0qDcvEkDMzhIAvEKWM/M0MCvr9NgnDhh8vG1pvvjyhU+P3bMBNdtPcixY5xZxGLUe0SjJph++zZjJM3NVJ97uTgCFA8hjBs3aGRTKY7kpcpxSwvPvyi4W1s5YxwYME2Upqd5vgFTCqS31zmrSCSMontx0bisAJ5LEep1dprOkoWO291qVQR6NtwCPSGJnWiYJGTgFuDJa1utnUh4zwaUomF3i++8xHiyBCnsxWO7xPGc1vr8dvfZLZRKHJubLJc+OEj3UqEqtckkDcytW/kJJJPhPtIy9uBB7if+31iM8Y3ZWY4mz56lwUkkmPI7Pc34yvnzvBEkmN7dzdlHNErXlVImCyvA1iCzyRs3aMgTCRojaTpUU2PamtbVMeY0MMBrZnra6DBCIf7/osMQ4Z3UiBKx3uoq14dCPOe2WC9fXMAu8WG3XbUJQpol2a1WKynQE0GelyjPTQ5epiccduosbBGee101d8+7G7Bd4qjXWnu0PdnaPruFUokjFuPo/84d3qA9PSSQvr78F2siwdmHEMjAAGcE7hjI5ibdUOPjNERurcjkJGcnbmX62BjXA8zI2rfPtLSNROimqq9n0Hx9nW4xqcYbID8yGcYwrl2j8Y/HjeBNEhtiMZ4j0Vpo7ezjLS4o6eNdU8NzLUK9+XkzA6mpMQTR1UXSyFeQULrpCVGsrTm76TU1OXUXLS3l7cft1UrV3VLVjxDcqmy/lqqBi7UwtOa1mk5zsZ+7X9fV7XJWlVJKAXgPgPu01j+vlBoG0K+1frq0w9oZbDfGsblJYz06yhujvp7B6uHh/FP6RILkcesWZyO9vSQQdxaWrRVpajJaEcCpTG9oIFH09XFE+dxzHLX29zMmsrlpWtpKEF6ysIaH+d570U9bDLQm+V65YoLaUpqjttakSra0mHiGdM/LZHhzig6jq4s3rAj15udp4AEaRVF0S6Mjt1G360CJOG9lxZna2tDgFOeVo5ueBMi9hHj5WqlGIk7xnddyr8UIxHCLpiLfc/eSb5tNDMWip4dN4UpBuYjjNwFkALxFa31cKdUB4G+11g+Wdlg7g3Km487MkEDm5njD9/WRRHp6/Ed1yaTJxEkkaJCOHMnVgczO0nBFoxx5njhh3EyLi5xRrK7SQJ06xRvyxg1mWNXUkDx6epx9Qc6dY9zj2jX6yKULYQBChHuvvMLzI2nWdXWc3dXW8r+VirPr6yae0dBgWrO2tTGILRoMabIUDhtFtxdRZDKGGGwVtxiGUMgQhOguWlpKS2u1O+Z5tVR1N0YC/Fup2uvuhphBIUFdvvW2sZfHrRh2gOc5HHYukn4bCjmfu/eTxd5mP6+pKV0fUy7ieFZrfcGOZyilXtBaV6C/VPlQiayq9XXT1jOR4E00PMzAqF8BOXfuf2srCxjaSnStKSh77TXu09vLGUhbGy/GmzcZGwFIPocO8Viee45GZ2iIpDI9bfqCnD/P737uOdPS1k97ci9hbo4xo2vXSMhtbaast51FlMmQuIUsBgdJFjU1Rqi3sGDKf3R0mO56tuspk+H32AK91VVjZCKRXHFec3PxbiaJLdgaC3tx971QiobfS4i3V/prZzL5xXVukZ2fMrtYuMV2Xs+9HvM932ltxlZQLuJ4CsAbADyTJZAecMZRlUFxQaXScQFeuNPTJJC5Oa7r6SGJ9Pd7T88zGc4Irl+nS6KhgTGM4WFzo6bTxteeTJIQjh6lIYvFOEK+c4evT52ikZJyJ5EI3VLNzaaEyaFD/IzLl53q83sRKyvMUHv5ZRKCtBqVwLEYzI0Nnof6etNqNZXKVXU3N5PgZVYh53B93XTSE6KwScIW57W1FZcBJ7MGtxBPHt0jXSEDe5F11aK58BPV5VNhy1LMyN5PVCfPpXGSlyLbTQDV8H/tJMpFHO8B8N0ALgD4JIDvBPAzWus/KteBVgKVJA4bGxv0k4+P8+aORGis9++ncXBDXF+SuROJ0O118KBxJ0mm1s2bvEn272cQvaGBxuvll2k0+vvp2kqnaRRXVjgqPnGC7x8dpXE6fdoEfu2qu/cCpE6Y9DapreXsoLfXpGpmMjwvtbX8/ySbbW7OVJ8VVbeI9RoaeJ5EvS2LxCTCYac4r709P0nYZTzcgjy3GC8cdgrxbDFeQ8POxhWkHaqfyM7rsZDoTtJtCwns3OprmwjuNWNfTpRNOa6UOgZ26FMAvqC1vlKeQ6wcSiUOqSnV01M4Fdf9vvl508lN2n7a5bPdWFqigZ+e5uvBQc5ChHA2N2nwpXX68DBdVbW1xn2lNd9z6BD3u3rVqNEjEbpl0mmSSTJJd1hLC11XW/l9ew3SD/7rX+c5CYcZm+rupu83FDKGRohAZhYyq2hrM931OjpoxG1xngS/AZKNCPTa2/kfexmvZDJXjCcEYY+k3WI8e6lUbSdbcOcW1/m1Rc03+ne3PnWL6/KRQ4DdQ7lmHP9Za/3BQuuqDaUSx8oK8JWv8HljoxFudXUVP0pPJkke4+M0MAAN1r59nCW4g5wbG8zCGhszTX/uu8+4vWIxEsL4OI3RyAjjJAAD6xMTNCbHj9NovfiiUaPffz/fOzfH3zI4SJdXJsOZh2Ry3S3QmokDX/gCSVdr/u8dHfyPxI8vqbai1k6nabC6uw3BbG46O+lJdlEkYsR5QhRuY5dIGH2FLcqzYw7SLU/6W9gxlnIlM0hvC6/F3Ro130zA3f403yL7BqP+vYmyBsdd617UWp8pwzFWDNtxVW1smAJ08/M0KiLqkgJ0XiVGvLC+ztiGtAgVLcC+fSQk262QSpkUYKm6euAAZxp1dTyua9dyCUTiH0tLHCWfOEEjJWr0I0e4/+uvmz4jExPc/+BB7n83pE2OjQF/9Vd05cXjdMtJscDOTiMiA4xYTgYHUnxwedkQhYymm5qc4jx7piaNkIQkZLFTWGtqvMV4jY2l/++2+tqrHao89+uAJ1k3Xh3v3K1RI5G74/qoFmQyxS2i2yhmm/t1SwsHn6VguwLAfwngXwG4D8ANa1MLgG9ord9T2mHtDMoV48hkaEhmZzlqF7Wv9F6QTJpiRohLSySRqSne1O6qqXYRxNlZzkLm5kwToAMHaLjcBLJ/P11Vy8ski1iMs5WDB/kZ09MkugMHSErRKIkrFKKxbWujgLCpadt/165gYQH4i78AnniCv623l/9Jb6/RZNij37Y2nre6Op4HabkqnfRaW00XPSEcW2dhd8qzZxA1NU4RnixbmT1kMk5hnZfQLh531pgSiPraT2RnL/cqEXgJ6PzW+T332+712mupBKSGWihkaqZduFD4fV7YLnG0AegA8AsAfsLatKq1XiztkHYOlQqOx+Mmw2Z+3owsW1pMk52urvw59xIPmZqiS8uvTwPAmcfoKEkimeT3HDjA0XQyyayq8XHjkjl4kMd2/Tov4pERGsKrV3ns+/bRwN2+TaMyMGDev9dcVxsbwOOP0y01P08jf/Qo/z8JEjc00KB2dBj9gWQ+CVFIJz0hi0zGaCtEb+FWawsp2GK8QgRhk4JbZGe3RHVD6mTla4m6F9Jo86EUQVwhsdx2BHReUMoYZ9FNeL12b9vuopTJ7nKvl8dyopzB8Q4ARwD8w62htf7Kto+wgtiJrCqtaVjm5mi4Fhd5gcqoVUgkX+2hTMa0EZ2eNm1E+/udMxFpCDQ6SkMWDpM8RkaMKPD2be43MMAR9+ws14XDpqnQ2Bg/f3CQ2zc2jCp9ddWUcq/mrKtUii6pz36W/1tLC4+5t5dGVIy4zDa0NtoJd8tVEfjZYjx7FiHxENFYiFrb62ZNpXJborrFdm6I2C5fW9Rq6mchzYsKCeP8RHLuR1m22uVBjGk+QVwhkZxt5L3WeZHAvYByxTh+CMD/CWAfgOcBPALgCa31W8p1oJXATqXj2shkOJKV2kR2f2hxf8jiZQwyGRrzO3cMidTUOHtR19TQrXL7NonEbk/a08PYxegoZyPd3SSghQV+pqSbSpc4MbCzs3xsbuZxt7TQdVVtrWnTaeBLXwI+9SnOkiIRJgSMjJCcGxtNJVTx+wKGxJua+P+treW2U1XKtFO1W6ra5cSluZF7EYJwl+bwa4m6G2K7dNpfGOfXrtRr/VZG7dsRx/mtsw18gMqgXMTxEoAHATyptT6XTc39Oa31d5fvUMuP3SAON9JpQyTugGtLiylyJ/WRbMhMREp0S9XVnh5TdTUcJnncvk0jKLGQgQGOom/dMmr17m7T60FG0UtLNAZtbdw/meR6EZWdOFEdgsFMBvja14BPfpIB/lCIAf/z500GkpQKEbeO9KsIh/l7lpeNG8jdKU+IIhw26bJukd36eu6Mwe585xbaSUvUchYa9BLD5RPO2QK6Ym53r1akhRb3e+5l4dzdgnIRxzNa6weVUs8DeFhrvamUel5rfa6cB1tulEoc8TjTce0y1JImud2Ko5mMydoRIpHSB/X1Jhjb0eFs+ak1Db6QiGQEdXSQQPr6+FpmIcmkKZMRDnO2sbpqCGNjg6Nuqc0kHesiEf5+EVClUnT/nDu3O32hMxngq18Ffu/3mCmVTpMwzp0z7r+2NlNwLxLh70kkTCVawJCEEEVDA3+nLbKTxZ2FJOU5vFqilkIMbrV0PpGc/bzQSN+rRWkhkZxbOR2M4gMA5SOOPwPwPgA/CuAtAJYARLTW7yrXgVYC2ymr/vrrRphlG5JQyBgRO+deRr1bNSISI7Gb+cioNhymkRMiaW83wddolCrw6WkSEUBjKGmlqRQJZH6e39HRwe2xGMkqHDbGM5UyMRSAvyGdNjGOdJr7njtnCKrSSKUY8P70p1kleHOTrriHHmJcp7mZhs5LK1Bfb4hXiNFNEu4AdEODt9CusTF/rMdWTfsJ5NxLvtvONvbFiuVkCUb3AcqFsvccV0p9C4A2AH+ttfbJEK8OlMtVlUg41b32CNUeBdqk4lUKotjRnBh3USfbVVPFKAqRtLXRyM7Okkjm5ozmpLub25NJ0zRIjjGddpaxkKwu8WPX1vJRKUMk9fU03idPVs4vL1lSf/qnVLgnEnS7PfIIg/0So7BTSpubjdZA+oDLObIvcWnI5NUW1Z7ZuTvQ5XvulRIL8D/zEsTJcXq1Lg20EtuH1s5F4lz51nnts5Xt5Vi8jt1vW7H7dnVxsFcKAuV4BWMc0sfAq/Ccu5czYFweXku+2YqkhkrhvKUlZ6e35mbjhmlpocFfWCCRiLtGfO7JJN+bSpkLLJGg8ZLZR309SQWgMZMqwMkkR99Str2clXYnJ4E//mNmSk1M8Dv7+lgy/tgxEqWQr21kpTKsQALcbqGd6DDcHencorl8rUrzCeTcz6spC2oryCcy8xOj+e3j9TzfunyPfuu8XlcLZNDlXvJtK3b/QusADhoPHSr12APl+K59v1epa3ntVerazraxy1zLYrtMEglTfVUWO/Db3GwyghIJEoGU1ZB90mm+JxLhdttllUwacZykIWYyXJdM0pAfOUKjXuooOZUCvvENzi6eeopuNVHnHz9OdXt/vzPgKnEWmWk0NZneGeKuSSRyW5d6zQxCIW9hnNdSSSLwE5cVEqGVKj7baWEa4NQbeGkQ3I/Fbivlub34rS+0rViDv1eRjzgKOhts5bhS6kVZDaAZwNfLdpR3KcToeI3MMxn/5jrz895tOGtrcxvqNDXR0MqMxW4ONDfnHI1LRzYpYicuqfV146IKhWhoMxnjigmHzb4bGyZ9Nxrl47lz3lWAvaA19SaPPw588YsM5q+vmzaXw8MkjIMHTU8KO6grBCKzvWg09zsk06mujuTZ15fbtrSQNsIWjyUSptR6KYKzfM/tlOFS4RaF+ekPJPi9VeFZoXX5RGm28Q1wd6AYL/XvA/hr7IByXCn1DgC/CiAM4Le01h9xbVfZ7e8CsAHgB7XWz5bzGHYSoZDxr3tB3Cp+7TwXF71rENXWGtFYXx+/xw7MxmLG5aS1cVtJ1o64q1Ip7i9xDiGOdNqUBk8kmOm0sMC4x/33e88+MhmSxec/D/z931PRHo3y+5uaSBZSAPLIEbqYZIZTX2+eb24aN1BDg8mqcmcLyfHaIjOp67Sw4K1M3q4YTc6pn+hMSM9PeJbvsdDzwCgH2EkUQxz3AxjXWn8PACil3gvgOwDcVkr9bLnIQykVBvAbAL4VwASAZ5RSj2utX7V2eyeoXD8C4GEAv5l9LDtSKaavVmIqXSwkiJ2vf0MqlVvLyH6+suJdvsJW/tq+YSELIRdJB5U4iRCOuIyEoNbXjfL9/Hm6sRYWgKefZknzZ59l3CIaNbEC0a80NJA0DhxgtpTtdvISgAmByCypWNgqY7f2QDKvvL4vnwDNTQSBAQ9wL6AY4vgYgLcBgFLqmwF8BMCPADgH4ONgQ6dy4CEA17XWN7Pf9QcAvh2ATRzfDuD3NAMzTyql2pVSA1rrO2U6hn9APM6mSJVAsf7XfD5ZIL//VYy6Uqa/glsb4NVHwQ4uikI6GqWhlqwyISYpq1Jby1mD6D+EfCSrScimtpZE0dRkSpC3tTGGcd99TJ21XUduQ++lS8jXxtNLpRwgQIDtoxjiCFuziu8G8HGt9Z8A+JOsGLBcGAIwbr2eQO5swmufIQAO4lBKPQbgMQAYHh4u6WAaG4G3vrW0LI9iskDyvadQ+p+4UcqRIgg402oljVdG0PX15rskaC4KbCEGyUISEhJfekODaYIkbVlFMFdTw1nJ8DAFinIM8j3y/eI28po18Vw7s0m81hXaXuiztvoev/f5fa7ftkLbS9mvmPcV+7zU/ba671Y/r9jvKfZ9+fbbymeUe30x+8h9WG4URRxKqRqtdQrs/vfYFt9fLLx+utvLXMw+0Fp/HJwN4eLFiyWFHcVNtFfg1brTFqPZswxbhCYj99paY5y15oUoLhwZ5be2MngtMREJ6K+t0S0l1WPF/SUB+EzGlBoX0WFXF2cZoieR2ZPbPeTVG9qOFwi8ctgLPfd7tGMb+dbbMzO/4/B6LHad17ZyvTfAvYGeHuqfyo1iDP9nAHxZKTUPIAbgqwCglDoMYKWMxzIBYL/1eh+AqRL2uSuQyRQWntmP+URotshMCupFIoZM5HMlWF9TYzKukkmuE/KRkU0y6SxP0dBgYg+vvcb+3lNTpraTxEh6eynmk66GqZRJr5XPlt+Syfg3IBK4e08X6lHtfrzXYxLFEEw+4il221Y+s5TvKuUYtvI5+dYXIuPtfud2tleqRFBB4tBaf1gp9QUAAwD+VhvhRwiMdZQLzwA4opQ6CGASwD8H8L2ufR4H8IFs/ONhACuViG9UCjIK9+rQ5tXO0wuhkFNk1tycK0CTrCMJckvL0tVV039CUFtrXEtCJFKGPB435chbWrg9nTaZVZLxJCm8ou+47z7g7W9ncsGTTzIwvrTEEi6pFDO9RHEvMxrAVBC2S4h4lduwZyTuon8SU5FjLQR7JlOohlOhIn97EYVcOQECeKGkkiOVglLqXQB+BUzH/Z0sab0fALTWH82m4/5XAO8A03Hfp7XOq+7bKQGgW3AmpOBe5/V32+UzvBa7pae7zIfWNJarq87WpXapDaU4a2hpMcZddAl2EUDRakgFXqXMbEZiDuLiqq835FZfb2YgYszFyIu76uZNajZGR7nPgQPUfkgjpEiExyjqbiEKmdUo5f3/2XoWt2BSanr5VYp1r/MrI14siq0Wu5Uy43uVkALsfWy3A2COYryUfXYL2yUOrc3I3dZUuFNfvRS3ojVwi87crTyLrfkkfa3dbUvtkbUQhHSkq6/n8S0uMg4hYrlwmJlNoRANssQtJLYhMRMR2okBFe2HkAPA52IgpXRJOMzPbWtjem5LC7d97nOscjs2xn3uv5+6jfZ2bwW41KIS8rPLeQjR2doW93moqXGSi1ezpEJdGu2mRPkW935+zYy2OlYrJS240Dr7tZ2MECCAYLvEEQNwLd8uANq01qWlL1UYpRLH+jrdLF6kINlGbgPkXrZzI8ZiRv0tSnCpHQXQeEonOiEJabgkDaTm5/k+gIZBKuwCJIqZGZMq295u0m+lD7oEzGXGIWQgbilJ521sNELBtjajATl82FsQmEqRQD7zGTZiamjgfiMjdL01NJhUXa0NmYiGw3bj1dUZopTMLSmpLsQu5V1isfwtWYXk7XNqk3y5Rv+ZTP5OeG4h4lbbp27HiVCMEDGfGNHrdaH1fmr1ALuL7RLHSBHfkdZaT5RycJVGqcSRTAIvvZQ7ShX/fzkh6a1221LbOErNKSGJtjYT9NKa75W2tdIkKhQiSUjb2lSK5delt3lNDWMNdXWMgczN8bPa2zkCF+2GdNOTmY1U2t3YMNqN5WVT+mRhgc/Pny9cADGRYI2qxx9n2ZLGRhJIXx+PQ+pDhULO+IbEdUQhLu1u7ZlXY6MhU1mkfMnmZu7M0b34zSD9Zo1u1+JuGj67ZpVfKRR3PSy/Glnumlj5amSVs86VrVkqpQyK1/N86/z2ybef3zpba7WXUfay6nsJu13k0A0pNChEsbxsModCIRo46UZnd6Rzf8bcHI3t/Lx5f1sb0++6u0kaKyusOHvnjnE79fdz++Ym3UXr6zSIAwP8jJkZ42JqajK9QTo6+P75eT52d3NbIsHPXF8n2ezfD5w6tbWS62trJJAvfIGf2dnJAHtbGwnAnkmIUZaUX6W4n/QakdmGxHrW1pwjcCEUqZgrz93uKjtmZcer7IQGP4KRAL9fnMqdzFCp8vQ7DVtn5FU8sdjii26tUr79/F77Pd9JFBLz5hP4eq3321e+y56p2e7d/fv9jzH/8QfEsSvfnUwakpBy6Hb1Wpk92OXQ/eo8LSyQKGZnaQwBGtPeXkMWtbWGLCYnadjCYY7gh4ZorMbGTI/yzk6K7+JxFhpMJvlZbW0kkNVVHldHB8knHifBaM3ZS0sLv3d8nL/nzBl+XqmYneXs4+mnSUI9Paxd1dFh4hru0b3EYFZWjGFobDT9Stra+B/IrETIZG3NaUgkQ83u9NjcXLiHipR88cqMK7ZvhxCivdjEYqdTy3I3jGh3A7ZwNt9jKSLffPsCxYl8tyrqLbR/Tw/wcIlFmQLi2AHi0JpGSfp5Ly0ZAw+YfhlizNra8t/8sZhpzDQ/b0RyXV2GLCSmEYuxDtTEBL8zFOL2oSHuOzvLbKbFRRqpoSEu8/PsR55Kmf7l0jFQCg/Oz3N2I6VBbt+mETxwgI9TUzym8+fLp1C9dQv4n/+TrsJEgr+lq8u46uz0V4m9SL/2mhrTBMvdRdFuflVfb8SL9uLuDKiUccO5uz02NGzNgEvGmluD49c9MF9Gl6QRu8mlUOfAu2V2E6DyKCtxKKWaAMS11kVkye8+KkUcqZSzQ5/dN7y21mmkJG6QDxKrRbCNPwAAIABJREFUmJnhItlPjY00/r29HN2LwUwmOQuYmOBsBKDx3LePo36taeRHR2lAm5po7Pv6OOu4dYtkNDjIqezUFGcOkQiznBIJptCGQny9tsbtLS3AwYOsdLuxYbKiyu3Tz2QoJPz7v+d3pdMkrtZWY8ilRHwkYsSHAA28lDkJhfhfLi05ZyV1debcyCKxK9GD2GQii23MleIx+HV73G7/DkkE8FP/+/Uoz+eSkaw5m1S22qM8mO3cG9hucDwEivHeA+BBAJsA6gDMAfgrsHZVvqyrXUW5iENSWmWRkuAAjVlHh8la8iuT7kY6zdH89DTJIpHgjd3ZSQPf10fjJ9CaM4DxcZJGJmN8mENDNFarqzT4ExPc3tNDQ9/ezvVCGEND7Aw2M8MS51pzv44O4MoVGsmhIRrrK1c4ij90iAbk9ddpcB54oLwdAL2QTAIvvsgmT5LC29Vl3FdSwkRKmog4cXnZJAl0dppZC2BiS+5ZYUODacUrj+5ECLsxl7vjo1u0KenYdpdHW29SqcZQkkptF7S0RZLuxda2FCua9BNJ+okm3boWWYLsqerFdonjywA+D+CzAF7WWmey6zsB/C+guvvPtNafKutRlwnbyaqameFofmHBCOXCYUMSnZ3FzSZsJBL83Olp0xs8EuEIua+Pj+7PW18nWYyPk8AiERp18f8D/KwbN/gYDnPbwYMcEd+4YVxSQ0OcIayskBDicc46Dh3i7GR8nIbt1CmS1M2bJKcTJ2i4Z2ZIJufO7Wxr1I0NEsjly/yNUjurs9PEOux6V93d/B1aG6IHTIVeiQvV1RkikYQFWxRZX5+brOBXwyyV8u70KCnBbqMsxOIWMNpp3jttWCVVOJ9o0ksk6X5drCPD1qjYJONV+t5Lz+L3PJgVbR/bJY6I1jpvxaBi9tktbEfH8cUv8ubu6jK9I1pbt35RxuOcIdy5QyOmNQ1Dfz+Xzs7cz0ynuf/YGIlLKRq7/ftNnadMhjGJGzc406ivpztqZITbb97ktlSK5HD//fzcl1/maLu9nc2XYjGuSyZJIP39wAsv8DMPHGBA/PnnOdo+fpwZT7uFlRXGPuQ3NDSYfuKtrcb1IgYsHCYZd3by+dISiUfiH/X1JBBJW25sNEJLIZRo1JmdJaTl1tEU0nlIGrDd6dFe3HW5RADpJ1wU3Uk1Gsl8okmvbX5iSVvDshXYBTOLXbxEkV4iSa91d+PMqVw9x78M4Nu01tFsGZB6AP9Na+1TVak6sB1X1eqqyf3fKjY2DFksLXFdSwuN8sAAR69eiEZJFhMTpsDg/v1cpIRGKsX4xc2bNICtrTTmQ0M0bqOjdD9JquzRozQwV65wRlFXRwLo6qIBnpkhiZw5Q6Mqrqhz53g8V67QSD3wQPHtYSuN+Xke+/XrNO6traaAo5Rub27mjb22ZoSL3d38/5ub+dtELCluJqncK4u4HdNpp2JfhJl+qn0hk6am4q+fZDK3QoH92q9vumSY2VUKvASMe9m4SYpuucWSbr1KqfASOlZaJJlP41KOc10u4nhBa31WKfUAWLL8LwAc0Fr/wPYPsXLYyXRcIYupKRozgAQxMGCMlRfSab7n9m2STCjE/UdGOFKWi0BqPknqbHc31dk9Pbyxxsdp9ONxrjt2jAZsdJTrMxm6r44c4UzlyhW+79gxEszzz9OQDgyQWF59lS613XBNFYvpaQbRR0dpzNvaTPxAbkQx5qEQf9/GBt/b2cnf1tvL/0HckgsLzjpc4pbs6uJnyfnQ2jS6klph0SjXyW0VCjk1I6IbaWoqbaZgk4t7EWW8X+8SqY4sROMnYLxXqwbb5FSoV3w+QWQhEaWfLqWcEF1HTw/w4IOlfkZ5iOMSgEcA/BKAr2ut/0gpdcnvg6sFlSaOeJxGf2rKzCza22l8Bwfz9/RYW6PBk9lFczPJYt8+Z1B2Y4Mup7ExXtwDA3Qpyeh/epoksLbGmIfMJhYW6NZZXaVxPHmSF9QLL3Bbdzdw9ixJ7sUX+dmnTtHIXrpEQ3TixO66poqB1vz/r17l4+oq//e2Nmc9LaUYR2pr441qZ681NZmU5I4O/udCIouLptyLNKCSRAgRRtpIp50ViWURwgKMOEsIRZ57iRG3CsnG8hMt2hqTYgSMtq7E1pfI8yDIvX0UEjtuZbE/q6mJNqUUlIs43gvgg6CL6pTWOqaUek1rfay0w9oZVII4kkkaqMlJkwrb2kpXUSGyEIM1OkoXicwuDhzIzVBaWwOuXeP3KEVCOXzYuE8WFzkrWFqiwTl+nMYvHuf6yUkey8mTNIi3bnF0rhTXDQ7S3TM+TgN44QKP6aWXaBAuXjTB970Arfmbr17leVlbM/EImYXE4zx/kQh/f1eXSVhYWOD5iUQ4UpM06Lo6Eoc7q04gWXWy+LmnhFBsUpF0X/s2rKtzEomtHSl3PEOqF+cTLspzP12JEE2+xa05uVdnNXsJZdNxKKWawbpUsWwjp5/SWr+vTMdZEZSLONJpGpfJSQrqMhne1EIWfm4ogZT4EF1FQwPJYv/+3GYr0SgJY2qKrpaREc4wJMaxtsYZxvQ01x09asoKjI6SHDIZkszhw6Z/+uIiCeTMGa579lmOgo8c4ee/8gqPsaeHJFLumlw7BSGQa9c4m9rYMC006+uNu2lxkee1oYHnsa/PkIiUXgFM+9veXhKDUqYqgOh47NIxkUiuRkTOnRcyGacY0daQuFN8RcNia0fksdLiPtHKeAkWNzedjcHkeT7zUqxo0aspV4DKY7tZVUoX2KmYfXYL2yEOSeWcmKART6VoAER57RfgtrG8zJH+1JTRVYgQzz3iWlnhaHl6mjfHwYN0E4kBTyQYq7h9m4Ry+DC3h8PG3bSywu84fZrGZHSUJBMK0Q01NMTjuXKFhCWK70uX+N4jR0hEd8NoUFxY166RjKUSsGTc9PTwP9rYMEUeW1r4Hw0MkFSkzIs0wIpE6OLr6THvl+9aXzcksrTEGYXcFfX1TiLx0oh4QcSIsgixbGzkkkpdnVMzYosSdyO1V47fLV70WyevC6XzSldLt2AxX6dHt8Yk6HNSGNslji8B+BMAn9Vaj1nrawG8CcAPAPh7rfUnynXA5USpxLG6StFZLMYLbWCArqKursI3oNYMkt+6RUNSU8MZwYED3jOTaJSEMD3Ni/u++0ga4uvOZPhZ167xphoZYWqtNGWS4HB9vXFBxWKcZczPc6R89izJ47nnaAgl4L28TG0EQBLp69vyX1X10JoziKtXTZn5mhrjB25u5v9RU8P/ZnGR+7S18b8cGKCRn5szi8Q8GhsNiXR1OckgnXZqRJaXnRqRhgYjNpS6ZflmJm4IqdiaETvV1761pXSKW4Roa0eqJa1X+r/kEy16iRdtLUkh2Ar6Yrs85mvQdTcMtNzYLnHUA/jfQeX4fQCWwDhHGMDfAvgNrfXzZT3iMqJU4shk6MqR/tjFjFCSScYLbt7kjdvYSALYv9874FmIMAAS0Kuv0hj09TFYLeRz5w5jFPE433fsGC/isTG6nQASyfAw/ffPPssR3cmTJLHr10k6zc3MvChW8b6XMTfH3y3xJREJrq+beFNvL/8nO+FBsuMGB/k/SSl66Xsixqq11ZBIV1euWyWZdJbPd5OJ6FHsMvqlpIRnMkYzYmtH5LXoWGy4G1659SO7XS6+WNjEU6i7o9c2aUe8Fe1IKOTd7TFfZ0evxlxe63aL0MsVHP8OAJ8D0AUgprVeLt8hVg47kY4bi5EsxsZ44XV1kQS83FEAjc7rr9Mw+RFGNEpSWFig++TkSRokgDf9Sy+RcFpbOZtob6dv+cUXub6rizOKhgbOVK5epcF74AEay+efJ/EMDvL995rfeHmZBHLnDm/M9nb+B0tLTv1MVxf3tUmkpcUMKCRDa2XFFISUnihS8r2ri+6tzk7v/zmVMtoQ0YesrpqMJym3b+tDWltzY2NbgZtYvBZ3xlUolNuszKuRWbXMXLYLv+6PXmJF+3WhBl1bdeqLa9UtSixGwNjcbFombBXlIo4YgD8H8H1S4FAp9T6t9e+Wdlg7g0oSRzRK4zM1xdeDgyQBP5HcxgYJY3KSJ/XgQVP/SZBIcBZw+zZHeEeP0jWllClcKPqLo0f5fUqRLF54gRfn8eP87ESCs4z5ebrZTp8muTzzDMnr+HF+/72M9XVD+pmMUY9LOq5SnIHs309jPTtbuAJAOk3ymJ/nZ0jdLCmlL7ORjg5/45/J8By5WwXbGo3a2txmVVKvqxyQlsn52iZ7pfPaehFbiOhugnWvxhmkrIufILFYEaOtJfF6DfDa3dWy6kqp5wB8DMA7APwzrXWymnuNCypBHPPzJIy5OY4gh4dpwP3KisfjHPWPZSNEBw8ysG3f4KL4fv11XjgHDzKOIaSytkZiWFzkzOPMGRo4KSEyNkajdOECjcfCAmMXqRSD4sPDPF6JZzzwgJnBBKCRHB3lsrlpVP6plOlFIjXC9u/nfy81xyTLzq/mmBCJ6EJkRgJwZiMk0tlZODsvkXAKDiWt1/brSztdu7+I9J8vN9yCRCmr4m5+5WVmamq8BYheDbDutRlxOSCajlIJulzE8azW+kJWz/E9AP43AN/QWp8v7bB2BuUiDgmwSopnXR3JYmTEX7CVTFK4d/MmT+LwMMnAfQMvLtL1FI3SpXHqlOm1kcnwM65e5QVw8qRJvV1e5oxifZ1EJNlQN25w1tLYSC1GayuP4dVX7614RimQ+l83b/J81Nby/25uNpWM3VWJ6+q4TYjErnIsRCLnU75D+rbIIqm8dkl+KaJZzI0fizm1IV6EUlPjbFJl60QqOfrX2qkJcYsQ7dfuel0CaXblJUD0en23Bqx3EuUiji9ord+aff6dAP4VgCGt9dGyHWkFsF3iyGRMSufaGo3x4cM0Gn6+3EyGI9erV3kjDA3RqLuNtS3Ua2ggKdj+yGiUsYiVFa4/fZo3h9ac8bz+Ol9fuMBRazLJrKmZGRO7CIUY9xgf5wj6/Plg9FYsFhZIIDMzfC2dFFOp3D4ooueRHuzuvioNDc6+KvY5sFN53U3AlCLpSG+Xjo6tBcvj8dxmVWtrJitMUF+fKzaUdN6ddCmJ6j2fENF+9CvVIaJEr+6Jfg2wgs6KTlSkA6BS6t0APqG17t7OwVUa28mqGh+ngd7Y4Kj98GEaB7+bVnQDr73G9/T0MI7g1nuIW0qEeocOUT8hN2gmw++9do0X9OnThlDicZLD/DyP5cwZ7rOyQi1GPM7Mq4MHTTxjaYkznfvvD0ZhpSAW4/kaG6PBktIw3d10U01McHSvFNdJHxNRqtudHFMpMxsRdXpra+55SSRMu2F5lNF4TY2z5XB7e/5qBV5Ip526ELthlVsf4m5WZT/udv2ydDq3c6LfIjqRfHWhQiFv8aGfINHd+OpuQtA6tgTiiEaBL3+ZI7wjRwrrGxYWmAK7skJDcOKEdwxhaYkzAOmpffq0cyYSjZIYolEaoFOnTCxkepozkEyG7xOX1fg4P9MuExKNsnd3IsFZRqmZFQEMZPY5OsrzGA6TvEdG+FzK0GxsONv39vUZ3cjSkhEV2j1CbFGhV6xMZiU2kUSjxgiKWt3uHVKqO9ItOrS1Iu4CilKN2GuRBlvVBkm99RIf5mt+VUgf4taG2ITiJ0T00pFUy6wnII4SXVXRKEkgH+zyHw0N1FIMDeWOIJNJ7nf7Nkdwp045jbm4n65e5cV05gxHrQCNw6uvUgTY1sbAdlMT17/8Mj+zu5vra2t5LM8+y8956KHiFO4BtoZolAQyOUmD0trKGNa+fbwmpPBlPG5IZHCQJCKj9M1NzkJmZ/ko2goJmIseJF/mlfS5lyZUdhqvzExEE9LWZtrplgp3syq7aVUs5t2syi02tJftpBTvNLQuToDofm4/FmNubU1IoQ6K7kZXfs9LQUAcFUjHTSYZYxgd5Yk5coTuIa+TNDnJ2UgiwX2OHnWOxNbWOMtYXs6dZaytMRMqGmUw/vhxXljxON1Qy8t0oR07RrK6do0usI4Ozj4qkUkTwCCV4vkdG+O5EBGhrQGRUvvxuHFniQ7ENpyrqwyyS48QGeG2tDi1IPmMrZCJLTK0+4ZImXd3I6pyGfDNzVzRoaTxenVBDIWcIkOv53u1ZpoXJN22UAfFfIt8RjGmu6cHeOSR0o41II4yEkcmwxG+pM0OD5tGSW5sbDBbanbWNEqyR/8S63j1VRLOmTMclQomJuiCCocp5hN3maTaptNcPzDA45Ig+NAQ11fLlPdegbsJV0ODacLV0EASmZ4mkYhaXPrL9/fn9pcXUeH8vCnICHA/u9lUocGBuLmERGSx1eO1tU5xoaTzljuGkUx6iw7tZlVuk+QlPHQv92LKruhB8nVOrKsznoutIiCOMhHH3BxdQ2trZPITJ7xdWVozG+f11znCPHaMJT5s95Ud5JaKtWIA0mkSzvg4DcsDD5htt25x9tLUxLTa5mbOZC5dIqEcPcogeIDdQyZDchgf5zUD8Dzu30+Sj0RouKenuUj9rOZmQyJShdf+zJUVZ48QmZE0NppmU6IFKSYJQjQhbl2IPSuQasL2UglCEWhtUnTdYkO7aZVXORDRhbg7H7qf75XSKbuNqicOpVQngD8EcADAKIDv0loveew3CmAVQBpAqpgmUuUgjvV1GuuZGRrsEyf8WdxOoe3rYxDbHeycnCQxZDJGnCdYWyMJrK46K9XaMwq7I9/6uinGeO4cZxsBqgexGGcg4+OmHlZ/P2MhPT18HYsZDYjdE0Q0ID09ue4arU3rW0njlcB1JOJsNiXlVIqB1jwe6bNuN6Kys5GkNa8tMGxq4rW+E0Y5mcxtVOXVtMqLYPwaVfk1rbrXZjKCvUAcvwhgUWv9EaXUTwDo0Fp/0GO/UQAXtdbzxX72dogjnWbM4MYN3uBHjjDO4OUCymQY2L5+nRfbqVNOtxPAi/2ll0gcHR3MdrIzXyYnqQ4Ph6nNsGtTSTzDTqtdWOB6pTj7cDeCClBdWFoyJfoTCV4ng4Mke5lhpFJGTDgzY8SE7e1GB9LW5m2c3VqQ1VWuFy2I6EC2qgUBDKHYAkPRhNiiPYmhiMhQ0nibm3cnVpFK+TeqcmtF/LKmRHzoJzb00orcDW7ivUAcrwN4s9b6jlJqAMCXvISFO0kcq6vAk0/SaO/bx6C0ny95aYmzjLU1uiNOnsydyi8uMtMpHqfhP3LE3Lh2dpTbNbW0RHJIp0k0MtOZnOR3NjYycypQgu8dZDIkh8lJzjKkmZSQiMTBJM4xM8M4mfSxr601GpCeHv/AtjSbEiLx0oLIjKS93b9kTiFsbjrFhaILsXuvA7wn3HoQeV4NSRzptL/Y0Kt5Vb70XCEbP7Gh3/Nqqt+1F4hjWWvdbr1e0lrnNC1VSt0Cy7prAB/TWn/c5/MeA/AYAAwPDz9w+/btLR9TOk1Df+iQ/0g+nWYG082bvOnOns3VbtgzkcZGGn+7HevGhmmiZGdHAUafUV9PcpCyFZI51dXFmcZui7AClI5UiuQxNUVy0JrGdHCQix1D29wk4czO8lGEeq2tRgfS1eVvfAppQWprjRZEHkslE8BU4LXFhbK4+4WEw7nNp+ylmgyqQFTuboGh12tbG5LP5Eoqrl/3Q7+GVbYepFyoCuJQSn0egFdk4KcAfLJI4hjUWk8ppXoB/B2AH9FafyXf91YqHXdxkSP+9XUGvo8fzz1pGxskn6UlzkROnXLuMzvL7QDjEzKb0JqZVjdvOvUZmQxdXWNjQebU3YhEwpDI/LwhEekD4s7Ii0ZNY6nFRV4foRAHJt3dXNrb818jmQw/x60FEbNQW+sUFba10ZBvN44hpGILDO2GVO7YhHQ39GtGVY3E4gdbgJhPcOilD8mnegeMCFEIpauLdqcUVAVx5EOxrirXe34WwJrW+r/k26/cxGHPMhobOcvo9ii6MjnJ2YJSuWm2WnMWcvUqR4sXLxpXUyrFVNvZWWo+TpzgjW+vP3KEM5MAdy8SCWZm3bljSKShgSQyMJCbdZVOkzykuZRkaoXDnDF3d9OItLUVHmyk087+IG5hYThs9B8iLmxpKe9oV/Qg7sWvV4gtNPTSg9TV3R2DLGky5RYX+jWtamkp3VbsBeL4fwAsWMHxTq31j7v2aQIQ0lqvZp//HYCf11r/Tb7PLidx2LEMv1mGO5X2/HlnHaFkkrOM2VnOQk6fNqOljQ2WCVlb4/qREa6Px7k+GiUJ2VlYAe5+JBKMc9y5Q2LIZDgTkD4g3d25I+5EwqTuzs+bQLkQiWhACs1IBNIfRLQg8mgHxhsbc5tNNTWVP8tKUna99CAbG7xf3FV2lTJpuXbzKXcjqr00c6k09gJxdAH4IwDDAMbAfh+LSqlBAL+ltX6XUuo+AH+WfUsNgN/XWn+40GeXgzjsOEV9PV1EXrOMaJSzgrU1Zyqtvf2ZZ3hhnzpliAHgaPGZZ3hTXLxoPn9tjUH6RILre3u39VMC7HGkUhx0TE+TTFIpGrueHqbu9vV5B8s3N3mNiZhQ6mRJ98OuLpO+u5WYWSzm7FoYjdLdJGZFsqzcWpByuLvyIZXyb0Ilj17B7UjE2YDKqyHVvUIwVU8clcR2icMuOjg8zIwpryn57dvMjKqt5SzDTSySahuJmEKEgokJbmtoYLcucVstLVGjEQoxOO7XWTDAvYlMhjMK0YBIqfT2dtNMyi91N5k0GpCFBRp/MQUtLYZEOju3nrGXTpuUXbvZlF3KXQjF1oHsRG8QG6mUU1TobkAlehCvuEIxTahk2askExBHCcQh6u/XXqOxP3vWu0JuKkWjPzXFUd+FC7md/STQ3dXFQLeMCLWmuvzaNRLNxYtmtCeFCuvrWWtmq2WzA9x7iEaN/kP6o9fVOVN3/bQU0qVwacmk73o1l9qqoNBGKuUUFcrzjQ3nfo2NziZT8ny3UnYTCWezKb9GVPmaUPmJC70aUlWL4DAfcVTJIVYfolEa/IEBxhW8brholKm0Gxumf7c9uksk6Lqan3cGugHeqM8/T8IZHmZMQ7aNjTGw3tbGmcZeqiAaYPcg8YUjR3jt2ULCiQkjJJTy7XaAPRw2mVgABzVra04ikYZWAA25rQFpbS0cK6mpMfvbsHuDrK6a5wsLzuyqcDi30ZQslSQVMeiFYDehise9NSHi2svXF0R6gvg1n/LSh0QiO1tGJZhx5MHKin9J8rExBsFrazmLcGs97HjGmTOmdwbgbLB04gQJRyAajd5efm61jD4C7F1ozVRb0X8sL3OdpGsKkRRySYmgUHQgS0tGSxIKkTxsHch2S7hLENwWFvqJC906EPuxoaE6M6pE1e5uOCWzF69GVPnMtVfjqfZ2p33ZCoIZR4nwIo10mrOBiQlv1xTA7JfnnuOJe+MbnSOstTXGLTY36ZqSnhxasx7WrVtUqkvb1wABtguljJvp6FEaIOkDIn3UARpYmXV0d+eO4iMRQzKCWMwQycoK74vRUW4TMhEikRlRsde1UiYo7Y4ZSgkUr2ZTc3O5OpD6ev+GU/X1u1P0UPQWxcaQtCbZ+AkM3XqQWKxyNiSYcWwBdgHCo0edZUMApz6jo4OqbtvNJLWl3MHuTIZEMzXF0cHx40H1zgA7BzG2UsJdfPVSvl00IMW4TLWmARcxoajT5TOlbpatAWltLX8dq3g8V1Qoi11OXo5JiMXdaEqWe3HmHwTHy0AcU1MMgodCzgKEglSKxn96mm6pM2ecbG/Xlnr4YRPsTqdJJnNzuW6rAAF2GqJIt/uASNrqVvuA2NjYyG0uZRvw+vpcDch2XV1+ENW63XDKfvTqCRKJ+DecklnR3UYugatqG8hk2PL15k3/rnqxGAV6q6tM173vPud2v9pSySTdVsvL1IbYcZAAAXYDSpnSIocOmfiIiAknJ5l6Dpg+IKIBsRtRuSFuIbtdciKR21xqft4EjZXK1YC0tm5fAxIKmaC6F7R2aj/cGpDlZRPbsVFT49R/iGLdrQm5G1zQAXHkweYmXVOLi7lZUYLFRe6TyXAmYc9EtGYA/fbt3NpS8TiFfevrJKNSu3QFCFBJ2PGRw4edfUAWFhgnmZjgvpK2Kw2l2tryaxikyq99z2QyvCdsQeHyMmf8AltU6NaAlMMoK2VmFX6QWYtfo6mFBe+ZC8CBo00k7kZTdm+QakVAHD6QsurJJAV9+/bl7mML9x56yDniSqeZijszk1v1dn3dqMEffthbhR4gQDXCnpHIzHptzfQBWVw0abuyr60BKaRHCoXM7MKGWwOyusqA/OSk89hsDYi9lNsIF5q1ACQNWwPi1Wxqbc1fZChlUtxk4td0aidnMgFx+KCxkcHro0dz28PmE+4BvFiefpojpdOnWddKEI2SNLQGHn00UIMH2PsQ4yw11BIJp5hwbIzZggANnDSUEk1HMSVO8mlAJF3Xbi4lNb0EkYizsZQICxsbK9eWwDb8hZBM+gsMJUV3ddWfZAD+R25RYXu70/6UCwFx+CAcZjzCjXSaQfA7d3KFewADbE8+yamr2wUlJUTCYZKGe1QVIMDdgNpaUzcL4CBpdZUkIqm7tpiwqcmQgsxmig00h8PmPTYkXdfdYGphwbjW7OO1CcXWgeyU+Fb0F/niRAJb/2GTi91oKh7nIDWdDohj17G5aWYSXhlQKyskhkyGxGCLAufnmT1VW8ttQQmRAPcKlDIZU4Jk0qTryszEdjs1NzsbSrW2bm1mIG6rxsbcwqCiVN/YMISyscF71E0qolb30n/sVpruVvUfFTmG3fvqvYXVVZJCIsGZiDuYLcQQiQBveINz5DAzwwB6UxPrTlVDm8wAAXYTkYizxAnAgZnoP1ZWTBaXoLHRkIg8ltKh0O4n4kYmk6v9EIGhl7Cwtja3sZT9/G7tzhkQRxGY+//bu/sYuaoyjuPfX7t9s1a20FKgSt+o2NpQAhUBgaBolCaGoBhFFEKMxCDGxMSAYuQP/8EYjTGAUAmQgFk0AAAKsElEQVQRNZEYBEEEwZcoSEEoCfTFipbSUijQYm2xpYHu7uMf597s3dmZnTu787LT/X2SSWfmzs6ck5nOM+fc85xnd/ri7+lJmeCVw+KdO9P0VbXAsHNn2qzwHe9Ix8bzSgmzTpo2begUF6RgUsz92LcvTRPnpkwZWkwq/3e0O9IWd+2tprLAVJ7/kW8wWXn+oaenemGp4u1u3D3XgaOOfMPBWbPSyqnKXzjbtqUlt0cemY4Xf2Hs2JFWXc2ePfyYmdU3bVqaaipON/X1DeZ95AFl+/aho4GZMweDSH5pxnLd/GR3sSxCUb6RYZ6hXkwsrJX/UUwurFVgqt2bGNbjwFFDceXU3LnpRHflfOazz6btRebNSxsSFn855AFl7tw0tdWNvyrMxqOensFckVy+1UkeUPIckFdeGVpUaubMwUCS54A0K/8DBgNLrdWSxfyPav/u25eCT6VJk4bme1QWliou2W1HgHHgqGHfvlTxr9rKqYhUtGnbtpTtvXLl0Ddr69a0YeG8eSngHA6ZombjmTS4KqqYnV4sKpVfKhMK8xPpeTApXpo9S1Am/2NgYHhBqWIOSL46rFb9j2IQOeqotKdeszlw1NDbC2efPfx8xsBA2nPqpZfSqqrly4cez7cXOfbYtKeVg4ZZ59Rarlsr/2PXrqHnKaZOHVpUqnhp1SzCpEn1M9chtbNagam8Fkg+imkFB44RVPuwrVuXPlzLlqWM8KJ86mr+/JRtPp7mJM1s0Ej5H/ky3eLl1VeHTyFNnz40oBTzP9qxTHfSpMGlwe3mwFHSoUMph2PPnjQ1lWfJ5jZvTlNb1aauzKw7FKe8KktF9/UNFpIq1gF5+eXhJ72nTRv8Uq/MA+lU/Y9mcuAo4c03Uzb4/v3pJPhxxw09vmlTOq+xYEE6H9LtHwozG66np/ooBdIPy2LOR75cd8+edD6luNlhPhVVzP8o5oB0Q2Bx4Kjj4EF47LE0X3jaacPrcGzcmPbhWbQIVqzoTBvNrLPyfJJqQaVY/6Py8vrrw6fAirvzVrvky3M7yYFjBAcOpKBx6FBK3qusK75hQ1pZtXhxqsNhZlap3kqq/v6h+R7F67W2Z88TC4s5H5W5H61MNnbgqGH/fli7Nr1hZ5459JdEsc5GtZVVZmZlTZ48crZ6xGDGerHAVH692qgFUsA65pg0vd5sDhw1TJ+eskOXLRv6hhaDxgknpONmZq2S10QfaY+7ytyPPLCMZi+vMhw4aujpGb6tejFoLF2aijOZmXVa2dyPpr1ee16m+zlomJklDhwlOGiYmQ0aF4FD0qckbZI0IGnVCI/7mKRnJW2RdE072uagYWY21LgIHMBG4BPAw7UeIGkycCNwPrAcuFhSy9czbdw4eCLcQcPMbJycHI+IzQAaOV3yNGBLRGzNHnsHcAHwj1a1K98Bd8kSr54yM8uNlxFHGfOBHYXbL2b3DSPpCknrJK3bvXv3qF5s797B5D7naZiZDWrbiEPSH4Fjqhy6NiLuKfMUVe6LKvcREWuANQCrVq2q+ph6envhrLNqF2QxM5uo2hY4IuLDY3yKF4F3FW6/E9hZ47FN4aBhZjZcN01VPQkslbRI0lTgM8C9HW6TmdmEMy4Ch6QLJb0InAH8TtKD2f3HSbofICL6gKuAB4HNwK8iYlOn2mxmNlGNl1VVdwN3V7l/J7C6cPt+4P42Ns3MzCqMixGHmZl1DwcOMzNriKKyQshhRtJuYPsYnmIO8FqTmtMNJlp/wX2eKNznxiyIiLnVDhz2gWOsJK2LiJr7Zx1uJlp/wX2eKNzn5vFUlZmZNcSBw8zMGuLAUd+aTjegzSZaf8F9nijc5ybxOQ4zM2uIRxxmZtYQBw7qVxZU8qPs+HpJp3Sinc1Uos+XZH1dL2mtpJWdaGczla0gKel9kvolXdTO9rVCmT5LOlfS01kVzr+2u43NVuKzfYSk30p6Juvz5Z1oZ7NIuk3SLkkbaxxv/vdXREzoCzAZeA5YDEwFngGWVzxmNfAAaWv304G/d7rdbejzmcDs7Pr5E6HPhcf9mbS1zUWdbncb3udeUjG047PbR3e63W3o8zeB72bX5wJ7gKmdbvsY+nwOcAqwscbxpn9/ecRRqCwYEW8BeWXBoguAn0XyONAr6dh2N7SJ6vY5ItZGxH+zm4+TtrHvZmXeZ4CvAL8GdrWzcS1Sps+fBe6KiBcAIqLb+12mzwHMUio5+nZS4OhrbzObJyIeJvWhlqZ/fzlwlKssWLr6YJdotD9fIP1i6WZ1+yxpPnAhcHMb29VKZd7ndwOzJf1F0lOSLm1b61qjTJ9vAJaR6vlsAL4aEQPtaV5HNP37a1zsjtthZSoLlq4+2CVK90fSB0mB46yWtqj1yvT5h8DVEdGffox2vTJ97gFOBc4DZgCPSXo8Iv7V6sa1SJk+fxR4GvgQsAT4g6RHIuL1VjeuQ5r+/eXAUa6yYNurD7ZYqf5IOgm4FTg/Iv7Tpra1Spk+rwLuyILGHGC1pL6I+E17mth0ZT/br0XEAeCApIeBlUC3Bo4yfb4cuD7SCYAtkp4H3gM80Z4mtl3Tv788VVWusuC9wKXZ6oTTgX0R8XK7G9pEdfss6XjgLuDzXfzrs6hunyNiUUQsjIiFwJ3AlV0cNKDcZ/se4GxJPZLeBryfVCitW5Xp8wukERaS5gEnAlvb2sr2avr314QfcUREn6S8suBk4LaI2CTpS9nxm0krbFYDW4A3SL9YulbJPn8bOAq4KfsF3hddvEFcyT4fVsr0OSI2S/o9sB4YAG6NiKrLOrtByff5O8BPJW0gTeNcHRFdu2uupF8C5wJzskqq1wFToHXfX84cNzOzhniqyszMGuLAYWZmDXHgMDOzhjhwmJlZQxw4zMysIQ4cZmbWEAcOMzNriAOH2RhJWijpoKSnC/f1F2pcPCPpa5JG9f9NUq+kKyteb1iSnqQZ2Wu+JWnO6HpjVp8Dh1lzPBcRJxduH4yIkyPivcBHSJm7143yuXuBK+s9KCIOZm3o5n3UrAs4cJiVIOmybNvx9ZIeaeRvsxoXVwBXZfsFfU7SE9no4BZJk7PXWCjpn5Juz17nzmz/qOuBJdnjv5c97WRJP8lGNA9JmtHUDpuNwIHDrA5Js4CrgTMi4iTg440+R0RsJf1/Owf4NPCBbHTQD1xSeOiJwJrsdV4njTSuIRvRRMTXs8ctBW7MRjR7gU+OqnNmo+DAYVZfP6lWxfclrYqIvaN8HpE2ozsVeDI7J3IeqcxpbkdEPJpd/wW166A8HxH5OZWngIWjbJNZwyb87rhm9UTEG5JWkEYaayTdGhE3NfIckhaTAtAe4PaI+Eatl6tzO/dm4Xoe2MzawiMOszokLY2IAxFxB3AfML3Bv59LKkd7A/An4CJJR2fHjpS0oPDw4yWdkV2/GPgb8D9g1hi7YdY0HnGY1Xdt9mV+ANgEfLHE38zIpqKmAH3Az4EfRMSApG8BD2XLcw8BXwa2Z3+3GbhM0i3Av4EfZyOeR7MluA8ANzazc2aNcj0OszGStBC4LyJWjJPn2Qas6ubiRDa+earKbOz6gSOKCYCdkCcAkkY5A51six3ePOIwM7OGeMRhZmYNceAwM7OGOHCYmVlDHDjMzKwhDhxmZtYQBw4zM2uIA4eZmTXEgcPMzBryf4aJfOOvvOqCAAAAAElFTkSuQmCC\n", 355 | "text/plain": [ 356 | "
" 357 | ] 358 | }, 359 | "metadata": { 360 | "needs_background": "light" 361 | }, 362 | "output_type": "display_data" 363 | } 364 | ], 365 | "source": [ 366 | "# plot the depth evolution of the data\n", 367 | "fig = plt.figure(figsize=(6,3))\n", 368 | "ax = fig.add_subplot(111)\n", 369 | "ax.plot(s_span, traj[:,::5,0], color='blue', alpha=.3);\n", 370 | "ax.set_xlabel(r\"$s$ [Depth]\")\n", 371 | "ax.set_ylabel(r\"$z(s)$ [State]\")\n", 372 | "ax.set_title(r\"Depth-Trajectories of Controlled Neural ODEs\")" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 10, 378 | "metadata": {}, 379 | "outputs": [ 380 | { 381 | "data": { 382 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAGyCAYAAACP2j9zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9fbxkVXnn+33W2lWnu+mGbl4VDRcnHTWAwAW911E0ZgzGIQoB7ye+5UMShzHeXK8YJ+MwzlWJmkSihviSMTGiBoIQR6+ApkVzVXyNzRAFg6LBKEpDQ/PafbrPOVV7r/XcP561dtV5o093n/ezvp/P6a7atWvXrjp16le/522JqlIoFAqFQmE6bqlPoFAoFAqF5UoRyUKhUCgUZqGIZKFQKBQKs1BEslAoFAqFWSgiWSgUCoXCLBSRLBQKhUJhFqr93F76QwqFQqGwEpH5OEhxkoVCoVAozEIRyUKhUCgUZqGIZKFQKBQKs1BEslAoFAqFWSgiWSgUCoXCLBSRLBQKhUJhFopIFgqFQqEwC0UkC4VCoVCYhSKShUKhUCjMQhHJQqFQKBRmoYhkoVAoFAqzUESyUCgUCoVZKCJZKBQKhcIsFJEsFAqFQmEWikgWCoVCoTALRSQLhUKhUJiFIpKFQqFQKMxCEclCoVAoFGahWuoTKBQWElWlaRp6vR6dTgfvPc45RAQRWerTKxQKyxxR1ce6/TFvLBSWK1kcQwjEGOn1egCThNF7j/eeqqpwzuG9L8JZKKwe5uWPuYhkYVWhqoQQaJoGVUVEUFX6/T7OuUn75Z9hRKQVzuw6s/MsFAorinn5oy3h1sKqYCZxzKI40xfBmcKteb98nGGcc61wlpBtobB2KCJZWNE8ljgeKFnwZhLP7EbzY2RKyLZQWN2UcGthRTKTOM4mTjFG6rqed/EqIdtCYVlTwq2Ftcd8OsdDpYRsC4XVTxHJwopgOYnjY1FCtoXC6qKEWwvLGlUlxkjTNMQYD8p5xRinVbcuF0rItlBYMEoLSGH1MpM4wnSHNtdj9fv9FSMw+W9yJvEsIdtCYc6UnGRh9TGbc1xLIlBCtoXC8qE4ycKyIAtAXdeH7BxnOvZKcpIHSgnZFgozUpxkYeUzkziuNed4qBxKle2w6yyveaEwnSKShSUj9y/GGIGZP+wXCuVekAdAn4Rw+KI85mIyHyHb/FMorGVKuLWw6Cy2OE4Nt0b/QfAfBTpAROr3I/r0BXv8lUAJ2RZWIaW6tbCyyAU5IQRg8ZzjsEhG+R50fgukHtphE9L/GjI/f1OrhrlW2ZaQbWGZUnKShZXBUonjVJQI1TuAesot48AorMKw66FQQraFQhHJwgKyXMQxo+5LIP8yw/fLiPqPQ/iPCH4pTm1FMdvvMbfvhBDo9/uT9i8h28JKpYRbC/NO/qDMVZVLLo6q9Pp7YORFwIMzB2F0HYSX48LrF/v0VjUlZFtYQkpOsrC8UFWaplk24phRVXrhw1D9OUgcuoHJf0a6Bde/aZHPbm0yXChUQraFBaLkJAvLg+UqjhnlAaj+OxAn3zDtFGuUhxGOXKQzW7uUkG1hpVBEsnDQLHdxzDR8CJh47O+VCtBDu78O/WsQnrA4J1eYxMEMRphpHN9yfB8WViYl3Fo4YFaKOAJE/oU+LwFp9r8zgDqIv4Jr3rWwJ1Y4ZErItrAfSri1sLisJHHM1PwZMEeBBMtZyn0Ldj6F+aOEbAuLQXGShf2yEsURIPB1al4DEuZ+JwVwEH4HF163UKdWWGT2V2WbnWf+WSnv8cJjUqpbCwvLShVHAEXpcS7Ijw7yAOuR5o+Q+Pz5PbHCsqKEbFc1JdxaWBhUtS2SyB8cK0UcMw1/B/zk4A8g46h8F6GI5GqmhGwL+6M4yULLahBHAKVHj18GeeRQDgJ6AtK8H9ET5+vUCiuYErJdcZRwa2F+WC3imOlzGZErJw8OOBDykAEVYBPSvwFhyzyeYWE1UUK2y5YSbi0cGjOJ40r/A1YeIvLxgxdIGPxpiYIGcP8E8Vfm5fwKq48Ssl3dFJFcg6xGccz0eQvQ3+9+LVNH001jHHXbIf5yGX5eOCAOdjBCCdkuL0q4dQ2x2sKqU7HBAecfeph16nVdB/GXcM2fHvpJFgozUEK2C0LJSRbmRg771HW9KsUxM8H5wA/m6U9jCtpF+p9FOG4BDl4ozMywcA5TQrZzouQkC49NFsemaYgxrqqw6lQCXwN+ePB/FvsNu0ZU7kS0iGRh8Sgh26WnOMlVyEziCNNXmF9NTPBckAdm32G/IvgYtH8F65DmMiQ+7yAPVCgsHCVkO40Sbi1MZi2KI0DDVTT88dz/JA5JMDfj+l85yDsXCovPGg7ZFpEsGPkPoK7rNSWOAEpDj2eATMzXAfezpJZD+t9A2DA/j1coLAFrZDDCvJzwqvfbq5nsHPv9Pr1erxXIFfqGPiga3g7Mk0DCHNacBO2ei3L//D1mobDI5M+IqYLonGu/cI+Pj7Nv3z727NnD7t27GR0d5brrrmPXrl1LffqLShHJFcpM4rjKQiX7RRkl8Ml5yDXOESG1lzyMVn90kA9aKCxfhoUzh2GzeMYY+Zu/+RsefvjhpT7NRaVUt64wcitHjNYLuJZc41T6vBZ4jJ7I/YVOZ+qJnAsSQO5EUWRB+k0KheVF/pwZHR1l8+bNS306i0pxkiuE4hwnE/kJys3ThW6YA3lpDmRfBeR+tPo/UeoDuGOhsLLZs2cPRxxxxFKfxqJSRHKZMyyOIYQ1L45ga0X2efX0Gx5LMOcTwdyk+w7q/24BH6hQWF70+33WrVu31KexqBSRXKYUcZydyFeBHQcWSl0IZALk1gU6eKGwvNhPJ8SqpYjkMiPnHIs4zozSp+Y/HXwodT7/zhVwXyL6983jQQuF5c1a+ywqIrlMyGXXvV6PpmmKOM5Cw9XAvoM/wHy+nDns6q9G5TvzeOBCYfkRQlgLU3qmsfae8TIji+PExEQRx/2gPELg8oUZYH5gJzKFgMptS3EmhcKiMTo6yqZNm5b6NBadIpJLRBHHA6fmXbAcqkmn/YpqqD5AdDcsxdkUCgtOv99n9+7dHH744XPa/1WvehXHHnssp5xyyoy3qyqve93r2Lp1K6eeeirf/va329tuvPFGnvKUp7B161be+c53zsv5HwplLN0io6o0TdNO71/LfY4HQuDH1JwH0ux/Zzi0+awHi44g/c8jbFnkBy4UFg5V5dxzz+Xee+9lYmKCiy66iFNPPZVTTz2VE088ccYQ7Fe/+lU2btzIhRdeyO233z7t9m3btvH+97+fbdu2sX37di6++GK2b99OCIEnP/nJ/MM//ANPfOITecYznsE111zDSSeddDCnXsbSrSSKczx4lEjDfwXmKJCwQGtK7uc6isp3F+CBC4WlQ0T4zGc+w5/92Z9x1llnccopp3Drrbfyn//zf+b++2cez/jc5z6XI488ctZjXn/99Vx44YWICM985jN59NFH2blzJzfffDNbt27l3/ybf0O32+VlL3sZ119//UI9tTlRJu4sMMU5HjqRf0T53tLnIqc+/rTz6UPnjWj954j+20U6qUJhcdi3bx8nnHACF1xwARdccMEhHeuee+7h537u59rrT3ziE7nnnntm3L59+/ZDeqxDpTjJBaJUq84PygQ1bwLCUp/K/hFAJtDOf13qMykU5p35nLYzU5pPRGbdvpQUJznPqGq7Ynhe+HQtlk3PF4EbgAeX3kUeEI8Q5Sc4fdJSn0ihMG+Mjo7Om0g+8YlP5O67726v79ixg+OPP55+vz/j9qWkfHrPEzms2uv1qGurwCzO8dBQHqLhMh5ziPlSM1uesvNyVP55sc+mUFgw9uzZM2/Dzc8991yuvPJKVJVvfetbHHHEETz+8Y/nGc94BnfeeSc/+clP6Pf7XHvttZx77rnz8pgHS3GSh0hxjgtHw8eA8eXtImfNU46j1buR+m8W+YQKhYVhz549c24BefnLX85NN93Egw8+yBOf+ET+8A//sDUPr3nNazjnnHPYtm0bW7duZcOGDXz0ox8FoKoqPvCBD/Crv/qrhBB41atexcknn7xgz2kulBaQg2QmcSyucf6I/Ct9zgdZBn2RB4tuQOqPIPqLS30mhcIh84Y3vIELL7yQ5zznOUt9KnOltIAsBSWsujjUvJ9lMTjgkBhDO7+Dyo+W+kQKhUNmLS6TBUUk50wRx8UjsB3l/1veYda5IADjqPvkUp9JoXDIFJEszEgOqxZxXBwUTePnVkDLx1zx16PyP5f6LAqFQ2J0dHTeCndWEkUkZyGLY7/fp9/vA0UcF4PAdcAPV76LzAggY2jntSg7lvpsCoWDZt++fWzcuHGpT2PRKSI5haniWIpyFg+lT8Plc5/PuqKIqPvqUp9EoXDQqOqarNxfe894FlSVGOOM4lgEcnGo9S9BH4Yoq7Cuug/VnxHd55f6RAqFA2Y/XRCrmjUvksPi2Ov1ijguEaqPoPEqmxsQHQS/uoRSsHaW6s0oo0t9NoXCAZFFci1+Jq5ZkZwqjjHGIo5LSNP8Ca7Xw/cECYIEoHYmlKtJLKlR99mlPolC4YAYHx9n/fr1S30aS8KanLgTY6Sua2K0cWdFGJcWbX6M3/0Fshp6IFYCIqhzqBMQiFW0G1FwcVDcs5J+dRKh+nNiE3DxN5f6bAqF/XLFFVfQ7XapqqqNtD0WN954IxdffDEhBC666CIuueSSSbe/613v4uqrrwagaRruuOMOHnjgAY488khOPPFENm3ahPeeqqq45ZZbFux5zZU1NXGniOPyRB98JW7M1mGMXW0XTI4jgAqxAiqHomgF6tLbshXKuPJiInoYrv/NpT6LQmG//P3f/z1f/OIX2bZtG8ceeyybNm3iaU97Gm95y1s47rjjJu17oIsmf+Yzn+Hyyy/nS1/6EgAnnngit9xyC0cfffR8nPq8fLivCScZY6RpGkKw3rsijssHGfsW/uE8CFxxExA7gICbAK0UL4L6iDpQl8RTQCuxfXGsPKHcR6z+EGn+G7I2/gwLK5Rf+7Vf49hjj0VV+fCHP8zu3bv553/+ZzZt2jRt3+FFk4F20eTZRPKaa67h5S9/+YKe/6Gyoj5WDpThnGMIoazpuAxxP3srsk+QfQ7pC9IT/JggjdW5uD5IUKSvSKNIBD8OvgY/IXRGHdWYp9rXQSY80vcQZPnnMgVwf4/6Dy31mRQK+2X37t3tcPMjjjiCs846iw0bNkzbb7bFlGdibGyMG2+8kZe85CXtNhHhBS94AWeeeSYf+tDy+NtY1V9hQwitOBZhXH7Irk8he3YhWCiV2iMdRUVxtaAdC6eKgHrMPXqF5Ci1C4iiHQvJ+p6gHUGDM310gETUp7Cs6PLKX0oP3A1o+G2E6R84hcJyYa7Tdg5k0eTPfOYzPPvZz+bII49st33jG9/g+OOPZ9euXZx99tk89alP5bnPfe7Bn/g8sKqdpHOuOMfligb8v7wXGa1gtEImHDLuYa+3yxMON+aRvl2WfnKXPZBg9S9uAlwNbkzxe9Uc5l7FTyi+p1Yhqw5p/KCtJMjy6cNUQO5DO7+Bsm+pz6ZQmJW5LpM122LKM3HttddOC7XmfY899ljOP/98br755kM46/lhVYtkYfnifvheGOtBcCZcE5WJVxCY8EhwUDtkzCO1Q8Ydss/jeg436nCj4PaB7AXXAz+RxDMobkJxNVT7lGpUqfYKfiKJbONAJVnRJX4RWnd7H/gyBL2wfNm9e/echpvPddHk3bt385WvfIXzzjuv3bZv3z5GR0fby1/4whc45ZRT5u9JHCSrOtxaHOQypR5DfnADWq9DRaEKiAATClW08OqEbUcEfESqpGi5DaQRGImgarPQO+Yi1ZGKehT1SQ9HTEiltlYSdWLhWy/EkbAMQrC15Sbj6YiettQnUyhMY3R0dE5OcrZFk//yL/8SsAWXAT796U/zghe8gMMOO6y97/3338/5558PWGvIK17xCl74whcuwLM5MFZ1C0hu+SgsL9xX/wty33bANA4UukkolXQ5tYJ0o10WLF+Jmvvy9tbUKuUbK005SEXXaWsWra8y5TRTZayKgAf1po7qFPVKXLfEgqkbkP4NCMcs4UkUCtO55JJLuOCCC3j+85+/1KdyIJQWkMIKZPQ+4r/eCmGdCV8VEKfQ76Dt5Qr10S73QDsmoNpTyEU4VQCnSO1gJKARxJkYyl4x55kdpUu3rbM8JqgNKshu01srid/XsW+FXokjzRIMK6hR/3dIeO1iPWChMCfmmpNcjaxqkSzh1mWGKvq5/8dCpQhE0KaCbsAJaOOQToM4oHGtOBIc2m2QnLPsROh7EDUxbBySHKW4NI3HAesiBBBvblH62Hafei2DXRanuJ5VzqqHMAJuvNM609hNq5Kk0144avB/Q0Rw4f9ayAcqFA6IPXv2rMm1JGGVi2RheaE7/pnm3odAbQakqxoTtcYTfES8oo0HH3CVQvCoS46y9oPc5ZCj1HQ/+gojTcphWh6Tfd7uK1h41quFWbsRN05bOBM7FpqVygYUuBpUFO0kwZyo7LiixE4zEMv5FkwB6IO/ghh+G8dh+7tHobAojI6OzqlwZzVSRLKwKGioqW94F9o4yymqEJuuOUc1tZEqmDsUT6yC6VAKrTqvaO2RbmNa0ji0G5Do0RBNVMe6qdiHlLdM/3cbJAh4yz1K35tgOqALLti4OxoTRzxoBRGhqrG+zEoJXXAxCaZTYtUktznfr1aAzm+i9RUIR+5/90JhgdmzZ08RydVICbcuH5rv3ET9YABGAJCqMeELXcRFnIfYc+AirhOhL6hTnFfoC8FHfBXRiY65RwHqCvXmLqPTFGqtoNu0xT3SiTDeaZ2nuFQd6xXtBAiCeEF6oJ1ood4uaATXNxcau0kwU9RVKyV0wAUTTPXRjgXzI5gC8FO0+m9I88F5OGChcGg0TUO3213q01gSVrVIggnlWl4wdDmg43uZ2PZxtPGIKBrFput4K9SR5PxcEr/YV1wKrQYXkbS9kWjFOtldOkAc0jHXqT5au0joWGjWWbGP+FQR2w1WwVNFiKnoRxScot1oQwc6EWrbDQfaVZyC9FIhUBeiYg5ThNhRYleQJglmJwwm/BwKEsBtR+UbiD77EA9WKBw8a/3zc9WLZGHpGbvxOuo9qQcDEG/h0xgE8RFXRQShCQ7fCaCOGBy+06BNZYU1ncZioE5NXENFTM5Pg7M8ZnCD9pB+ZdWyAuqDiWboWB9m7QfFPT6CDyaQlUI/iagDOqkHU9QeZwRLGfbtecUkoK6PCeY6iOoQsb6TONKAO4QPGAlo5w1Qvw/R//3gj1MozANrNTK36ifurNVf7HKheeA+xr7+TZq+J9SOpvbU4yP0xzvEpiL2uzRj6wh1hTYVzfgIsXFo42nGR2wOa+MJeXvtiRNdYl0Rx7uEsS6x1yGOrSNMdIgTXfu/cehEh1g7qCsY76K9Ch3rEnuVhW0bh9YOnaigV8F4ZSI5UUEtMG6j8mTC20LQEzblx40lo9cDv89+pFHceKTaZ5N+iIrrVbjxDtLzB99xLBNodTlK6fctLD67du2i1+tRVXPzUzfeeCNPecpT2Lp1K+985zun3X7TTTdxxBFHcPrpp3P66afztre9bc73XSpW9TABYNL6kYXFRVV5+M/fT+8HdwKKqrlIX6XLovhusHF0oslRGlb5OnRZABRJE3mQ5CjBwrEpJCtVtLyjaLsNF9sQbb4seUhBXpPSB3OQAJW298MrOhIGXycFWBeswEdIIVnaST8IhPWAS4MKvLWE5kkJWsXBAIQ5v5AO9KlIfSVC52B+FYXCQfHGN76Rr371q+zYsYNXvvKVnHbaaa3AjYyMTNp3LmtJ3nTTTbz73e/ms5/97AHf9yCYF4e06p1kYeno/fBHjN3xU+p+Rd2vCKGi6Y/QG+8SgyMGT3+sS4x2uckOUIWm3yHUFnZtel2a2hOCJ/QqQuPQIMR+x45T23ZtPHGiY06y1yGMdwk92xYbh/YrtNcxFzrWJYx1iGNdYu3QukLHK+ibo9SeN3dZO2RfB9lb2QD2AIx5myk77pEmOcxx+6FJ7nKv4sZtea/sLl0PpOdw4xWu5yEyWNLrsZb2kgjyI9R9epF+c4WC8ad/+qd8/OMf56yzzuI3f/M3CSHwkY98ZMblr4bXkux2u+1aknPhUO670Kz6nGQJty4N2jTs+tCnqMesAkYVnIs4H1GtiI3HVTZEoN9UiLPq1Rh8qnYNlmvMBT3Bgdj9aTzRRat8zQVAArHWthBIKssHSspzas/aNlweXeciOI9UDRI7qIs4l4p/nBXzUAUbiJ5d5boGGatSsQ/oSAPR2ddVBzoScJHWYcYRE0wktZh0B/NjY0fwdcf6Ln0alSdJKWd8y/ahuozYHIGLv7o4v8RCAWv/OOqoo3jWs57Fs571rFn3m2ktye3bt0/b7x//8R857bTTOP7443n3u9/NySefPOf7LgWrXiQLS8Oer9/K2M59xLqLk4hzSkyDVKtug0IrlCIKOEKjOB8QPCLVIFwqaXu+nEOmLqR9PFIFXAqBuiogsUoTeWJbVdsW8Xi1690GDR0kHS/4gPPOBNJHqF3qm1SkC+zrpjYThZEaGU9/Pg50XWNOE0z0RgIukEboQYy2UDTO2kd8sLF4Unu0SqPxqlR9m8O/w2JpzaFQvRntPx3hqEX4LRYKcx9JN5e1JM844wx++tOfsnHjRrZt28av//qvc+eddx7QOpSLTRHJwrwT9o5zz19/gTghqFYEBRFzh+BomgrvgznHvkck4quARk3C2dgfSDMQv9i4ti0k4vFVymUGxVcBCY6Y8poaBu6SJgmis7YSgpjb9KnlIwtup0EaR/DJoaJoFsR0m+TJPSO1DS5wVvVKp0H2ddKAAuu/lODbwesaIy73WHZNf7MbjR3AiVXK9l17OX+hYDhvCkAP7bwG6r8sQllYFOY6SGAua0kOi+0555zD7/3e7/Hggw8e0DqUi82qF8nl8m1kLbHrum8x/jBoWGdh1CSO1BVVmphThw5NHVNvpKfud/CdBueUEDwiAe/V5rm6SNWJND2XQrFKDA6XWkmaFKIVgdBoulxZq4mzHknfCcTgEBdwHqg1tXZYe4nYdACc94R2MEFuD+kildqouqpBmhHrx3SKdhpoOkkw02omTXajioZgLSeCtZRotBmyqefSNxC94msr8tEOoOB6gjoII9IWNrXD3fkR2nk9Ul+1dL/kwpphrk5yeC3JJzzhCVx77bV8/OMfn7TPfffdx3HHHYeIcPPNNxNj5KijjmLz5s37ve9SsepFsrC4TNz7CDuu/idrrVCxVg6pqNIKH/0Jj3MR7yOKEJqqrV4NoWvCl0bTNUMDBppG7RhJCJ0LxNYJpjykzyFXj68a66V0irhAbJz1ZDpPEMs/4iLON2joWp+ls1ynuFwxm/onnRJqC89K6KJJXDXnLH3q1/QBkkCKV+gEpJG2H1M12kg8RxqpB4jgneUstQMxKL4HOCGMQBXE1r+sQL1HqyY5y+8S3d8i8ZXI0i+IWVjFzNVJzmUtyU9+8pN88IMfpKoq1q9fz7XXXouIzHrf5cCqbwEpa0ouLv/yjm3s/Ny/tIU6VWXj2lSFqmpS/tHaIarkHFVpC3fyx733g6WqfDpGviypQd+nFhDEQq6IPaZ4Swb65DRJ+0hemku0LQ4SEXOQSchMbGMbrnXJUbbh2yq2giwpdykOyG0qeYmvoQIgqWIaUJDnxUYrEEJSONeGF6hPBT4OQhdboQQhJgHVSmwfTUuAdTsQXo+LL1/A32hhrfOud72Lk046iZe97GVLfSoHyrx8eyxOsjBvPHrbvdz7+X9Fo1WzNo2nris6lYU9+70uiFJVDeAIIbnKygaVh4Z0G4TGm4g5HVS8upjCrFbIE4NLRTyR0FieMjqXKl5TpayoCR1qAhlcK5Cxcaka1hOTQFrRj0DtcA5zlj7gxJsANlgIVhyuY05TJVXKdhu77uKQgCraxEnukiAIrq2IleDNnVa2j3YU32AhWa+4EaBJM2IF8EKzTpDQgHwQ3DNAti7Z772wulnLa0lCEcnCPPKDy7ezd/c6wJxgVUU0wnhTJVfZpNCpt8Idr8ToqOuKytus1aau2jymBA/kMKun0TRYIAB4qk4giu0jLhJjKgLyERqIySm2RUONx3cC4sQKgfIcWGfHd1W0ubLYYHUL1VrO1EKreZydSzlObz2MDhubF7qptURtnmwWUG+FQngdhGNdWp0kWoGOVArd5DbrJKgO4ojigq2HqR3LW8ZKbdi6CLG7D+28Cq2uBDlx8X/phVXPWl4BBNaASJbCncXhnm0/4sFbHyZGsWEATZdeD7oppNoER93vUHVqaweJnrqGThLOfnJ9VadBQ0VTV/iqwQmEpkKciWrrPn2kDlUqxLGQqktiFyRVsja551KQxlNVgSZP9/FKQNvCIecjGiMh9UvGJlrO0rm2YhaJbaFPE1wK21pPpgaruHFekU5jC0WTVyKxXKVmN+lS+DXIoEKWGlJFrPhoVbGdiGusr1I6oCGi3ob5aFq+CwXqfWj1NsKGD4BsWMJ3QWE1UkSyUDhEYoh857Jb2bPbxlR5F1IuUhgPI4gonY7lI3s9u175BucsrDoIwUKdxMyJEpqRoepYK+TxVYOQi3pMFHMoVkhVrs7aO5w3MQ2p6KcfLLxqj2sFPdJYnlODLdMlzhykuVMT7Zj6LHFqQ9ejjYZzDqRTW2Ws2HFjp0ZCt62cbdtHkkBqFRDn0Ca1oeQK2XyfKg8xSMPWk6NUDUhtbSjatTxojGk1EqeEke8izW/RHH411tRZKMwPo6OjbN68ealPY8lY9SJZnOTCc/tf3M7uu3spVAkxVvRrq2itvKIITbChAp0knk0zgh8qnmmaCueCiVp2ld7CkeYqreAm91iKJCHMogh4nybygOUcoyPUVhDUb3wrkCZglmfMAinJJebWD+dMhOskXs6Bq2pb/9LHNl/pUo5TfCrWyX2YaeqPBGc5yuQwJVRpWIAV71i/phvkL2NqKZGUv0zOU+oq9WBGJJordX3SupjgNaJyFyp/Qdj0+jQrtlA4dIqTLBQOgf7emm+/70f09nZQNdlrEwkAACAASURBVKHqpArQfl3R61tI1XtoYkW/7uBdoNMJxOjRusL7iHeBQIcazEk6pamTcFaR0Lcxw5UPxGhNDy61bYDivR2PJuczrajHOSU0qT0kC6TEJJDNoHfSmdvLFbJaJQeZx9WloiFxNqA9pgrZkNtUksA6F3CVWq6y6drQdqcmmLllxefrlp80sVRbQLr2aSTeQHCtxzJNCoqk9S5tDUy8mOucIBUWXYNrxqmPfNNSvSUKq4zx8XE2bFi7Yfw1IZJl4eWF42tv/DbjewIaxEbNqadfe5xTOsklhtBFFbod62mMsaJukjj6SIgO1Q4dbyHZJnicpJaQ6Kkb8K7Be6WXql6di0kIbUxdzlVapauJaxbFXAQkLqY+SgvbWl+m9U5ar6UJna8aq65NgwuCaLpfxFfmTp0nOciQioBIfZeC1Nr2WhItt0ntkpgCVYNLa1pKlQYWAOK8Ffw4BkMIXOq3DC61laRt3QbRvNJImgzUjbgYkN7/S+g+m7jxl5bujVFYFeTPTefW7loYa0IkCwvDnh1j3HbtvYTap6WvoOOs5SJEoW4cTpTKW7hyvGd/aJ1OwAFNGLjDyiu9YLk+XwVrkWjAu4h3kSBdQAcOVKHq1G1U0XKcaZC6jzhXpfaPYIU/Yu5UXHKdeUpPiG2biIhV5YZ6xJb08orWEe8YtI0Eay+RkHKiSUxjCs9qk/OiaYhB7RBnrSXq0/JctTfX6BSChX3VDbeo2EAEqUxEJdoXEHHRJvp4taKfdq5sCsuGFIb1yrod/5Xe499COOKFi//GKKwKho3FXNJWN954IxdffDEhBC666CIuueSSSbdfffXVXHbZZQBs3LiRD37wg5x22mkAnHjiiWzatAnvPVVVccstt8zjMzk0Vv0wAShrSi4Uf/srX+Fn33rYhClty6MCKqdUqelf1Soyq8oER9NiF1UV03Qdu29VBZs+k49RKcKgctV729eJOTy7X6TKg9BJ7jPt49v8ZBoAINGGFCApxxmTMOaB6TYz1sKvsc1hiijOWxVuPr61jaSBBm2rCFZYlIcNSHabKQeanK2rbDA7WDFPDsniB0IueWiCpHxnbitJj0XOgXbCoEI2b0/rX2q1jomf/zBxw5MX/s1QWHX8/u//Prfddhs//vGPefvb387pp5/OqaeeysaNG6ftO5f1IL/5zW/yi7/4i2zZsoXPfe5zXHrppe1KHyeeeCK33HILRx999Hw+hTJMoLB03PPtR7njm4+Qv3sIQkegk96WdQAJQkeUjoNGoe775DYtz9f006ogPuLEnKWFZU1o6iA4sbBtCBV1bSLlnIfGcofeRZp82StNAJfmvta1OUNJhUDeBevRdFbUY7nMSGgYVMqKtX1IXgRaUmtJ49JSXiZczln4NKaiH+doBxxAxPlqMB2ojohLK544JaZco/OKxFRZiyLeo5UV87i8zaVxeC45RTFhtRF5NvaP1JtJNyBhqBezqln/w4sYf8qHiBueugTvksJK5vLLL2fXrl1ccMEFVFXFVVddxXe/+13e9773ceaZZ07ad3g9SKBdD3JYJIeX2XrmM5/Jjh07FueJHCJFJAsHxZUv304vmsurMAc5rvbjgS5igwMUxiJ0gEpsJagmelDoeKtKbYK3gk+neK+M9yzs2ukEnAh143G55aOtfI04cdSkqlZnomhhVzHh9NGOLVbM04jlOkMq0qmqSAhZBAc5TRFztSFoCsXasPQYYlsF63xEmuw20/JdTtrrGmNaD9MEVPxg+EA7Mcgrzud+S0W0SZW2SkwDCGx4Qm4PSdWwVV5f0xxn6yiDQ/PoPa8QG2gaRr53CROn/AW6/glL8l4prFz6/T7HH388r371qx9zvwNdD/KKK67g3//7f99eFxFe8IIXICL87u/+7n4fbzFZEyJZ2kDml9tuuJcdPx0jRU1N4IhYl6QjAn1SFwOCB/qi9BRcgAqxIQHJOVZOqYBGBYKJZeWU0LO3pwmioLW3HKXPxT2WXwwxOUUfCbHTukZpx9IFmnowhACxPGgImta6HBKqlJeM0acQaRpr1wjBaQrhJgFL+c1Azlnm6ynHmReSdmrOMAkk0XKXiB3HpgCZ8LXnkapfY2Nj+ExEa2gcUg9CtKqkbSbk4hWNmLPse5s1Wz/E+lsvYvzMv0W7ZXmtwtyZa/vHgawH+eUvf5krrriCr3/96+22b3zjGxx//PHs2rWLs88+m6c+9ak897nPndNjLvTn+5oQycL88uHf+Z+MYy6wgxCBButCECJdTAgFocGEtFKhCwQGAupTiDYEu28lihchqNAP4EhimUa3eacEcUgjOKdULtKk4iCX1oc0gVREqpS7tGWzvAsIlrvrVE0q3FGcDHKHzlnYNqTmf59dXprQU/km9XBGXLDwaggeSZN2fEzHbcOvDTGQKmyjDS9w5h5tm+KSYErqmXS5dzL3X+bVRsQWaM45Tg2N7VenmbI+IpW1qEg/FQLlqtgQoD/OyC1voHfGu9F1xyzBu6awEpnr3Na5rgf53e9+l4suuojPfe5zHHXU4Atb3vfYY4/l/PPP5+abb55VJGPMX2wXx/ysCZEsTnL+uOE9d/DwWB8VkpNUPNDB4RFEhfF0i20XPEIP6GGus4sV+gSEvoJD6AARiEk8K7Fin34jeFEqsYpZcHinOIFaUhjWYeFSUXxeRFmSyKUKWScmeBZizZdtko9P/ZSgrXP0VUPTWCuLd4o0geAHztQmAaXcpYtIozRuMLDd8p0utY2oTcxhuDK2SflNb60wVURiaKf7tGLpZ9oWbfpOKvBxVRpM0MfaUDpWGKQxj72zSlkXdrBu+2sZf85V4MpUnsL+mc+1JH/2s59xwQUXcNVVV/HkJw+Kyfbt20eMkU2bNrFv3z6+8IUv8Ja3vGXWx3LO8f3vf5+vfe1r7Nq1i8MOO4yjjjqKE044ga1bt04K+84Ha0IkC/NDfyJw1aXfpe8iokoXh8MRFBoiCngZCGNUoUZR4iQhHbepplRo6zh7gKiJZQBqgGjXG6AvVsRTidBoGiYgiiNXvELllUacOTtng8EFxXsxZydxUpi2yb2QLiIiOAltAY5LrRzeBxqw3GGjVD6AWB9mcOCrGqFKTlSthaSxYekh5HYTbefD5rF5zuXrtjCzjcnzqTBJiTEkZ1zhqpAWmg4meGkRaXEKUYh1XjzaRt6FIAzWxEz5yujQWpHeI6z7ymuZeNZlMLJlad5IhRXDfK4l+ba3vY2HHnqI3/u932vvc8stt3D//fdz/vnnA9A0Da94xSt44Qtnbl1SVV73utexefNmer0ejzzyCOPj4+zatYsYIyeffDLnn38+z3zmM1m3bt28vAZrogUkhEDTNEt9Giued/7W1/nKp35qV5QkddBVRweHyw0bae52FkYHWEVJyl+q0jGZa4+dlk8kV2177BucpGNWgEfbY1TYYsWQQrcyJJatmySFXkkuM+ccY+qLtJAtTN7mUz9jbiHJTtPlcGmqqpXsYvOqJekcfBVat2miO3R/oRVLu24hYJ/yiTYuD3CRKhfqOKv+RWJbISs+tc/41FLi4qS2EnGK79p2JK+JmUKwXtEtJ9L75b+w4QSFwix85CMfQUR43etet9SnAthn+Sc+8QlOOumktsdymC9+8YtcddVVvPjFL+YlL3nJvIQQ14RIloWXD537frqXlz/tU/RjdoUej5j4JRSlUqHCRNO2DYVk1eHEQrJtL6T5sLRNcZgTFUj/mkDa0STdPhhN6tQcqYjt45M4SspnmrgoVdJjn3KBJqAhOciYttnggxxCze7TKlcZiGfukcy5wpSDzLfnwQV5vJ5kcUwDB1wSMyv0ySI4GG3XCqYk4fZpzU2Xj5+cY2XHct5cvPO5EKhJRUF2DBuDF9qB6qT2Ft3yv9B//mXQ3bSQb53CCua9730vJ5xwAhdeeOFSn8okVJWrrrqKV7ziFYgIO3bsYP369Rx77LHDu5U+ycLiEKPyxld8kTENkNo6JiwgihMTzE4Km9ZATWCMJgmao8KBCo3Yh7lICtWqEHKoNYdpFepWJG1bjQlmheIQaiwh6rGq2D7SiqVDUlEQ1NHaUCpRmmFnGfIkH3v7e+esEEcidRJJn9xbznEOBDKFSp2mkGzeFmlctHxp7qGMPhULxeQuFUnrWHoficEqX0n5xiqJbkw50yyAEuKkvGR2jho8QQbFPrbmpRAba3XJQ9VdHLSMuDQnNmqDPHQP1U1vp/nlt0FnfkJThdXFcltwWVUREe644w4uv/xyLrzwQu68807e+ta38rSnPY1XvepVHHfccfP6mGtCJEvhzqFxy9d38u3v3UePBocwIr51khGY0IbxtK9PolnhbF1JIoq1XXSSa3QIQSOa3R9CRx0NiooJnMdRpZymOU6lab8YWgVtTIIpSSybVjytGCiHduvUcuIFXEz7i8Ml4XRijtN7sUIfb8tueTcIvbohcXOpMCi7xSZN87HrtgZmbh/xqULVFppODtUP92BaW4q2QhhxVZppG2IKEeug8jUEW3kku8kkwBIiobbHaAVTIxK6NozdW15TOw00EVePWAj27h9Rfeb3aV78Z9BZv9hvrcIyZ7muAPLoo49ywgknAPDJT34S5xwjIyO8+c1v5kMf+lBbATsfrAmRLBw842M1//erbmQs1hZZVaVPWoqClHcUR5eKCmvfaGgsTi9JAPF0cPRV6afjimgrtA1CTyz/11HB43BEJsTCr15tn+Hwa0xhXknFP31McPOPINZCggmmSzPAXRLhPoJPLtNCruCCJoGTlNu0YhqXHaLknKWmvKdSNy6JqYVUm9rh/aB9pO2X9DH94WoaXJBWDHExrY2pbeGPhEiTcpfBp3BvCHjvbSqQGzjKKI6QBNx5RavGWmTyDNkUYvWV2DCE2icRNTHWGJBdD+A+eynxhW+C9cvvA7GwdIyOji4rkcyGZ8uWLRx11FH8wR/8AXfeeSfXX389//RP/8RNN90EzNy3ebAUkSw8Jtd/4l946EHziZL+GX77BYVAZCLJn4jlGbutm1Rq2tl1qfLVU6lgbfhYz2Byk0EESZWyFeBU8CKtRFY4GrU/FsH6L20aq7QCSaqurTApN7dq5+/UbjOXmURV0mM5O4YPlrOsXXKxHpxICtEqdVC88+3w9Ubsep6845pcvZqLfKwSNoulcyG1ruQKWuuxFIHGuXY27fBcWOcqgk8FQi7igkfqQYWsLfsF1L4dh2fhWhPvUFvlrCQ3qsERU8uI7wTk3ruQT12CvvTPoTOyEG+lwgpkOTpJVeUXfuEXeOlLX8qVV17JeeedB8Btt93Gpk3zn19fE4U7AL1eb6lPYcXxwP37+OXT/pbxMasMztWs+3tTqCqaQqdAW8wzgseppABpHmlnzrFqA6S0YuhVJm3rqJtUwOOVJJ4uhWildZFeJQmjtLfRXh5sz8LqMbF0qcjIHKeJZeW0FdNcgONzKFUE52KaCqTtQAI3VCXbqeJQqwltiFWSgNp9Ur4wDWEfDFa3ObQuV8/mkG9esNoNVeVWsV2+qx1o4HQw0af9YZDbTI9JniV7+GbkJW+FzfOb1ymsTF70ohfxqU99ar4Hj887vV6PL3zhC8QYOe+883LuslS3Hgj9fr+sKXmAXP7m7Vzz3ttBIaIElIZInX5iasmYSpLIdHk6WfA66lJoVdrjWOtHNUkgHUpHfSuQuXnEq2urYKvkHgcCKq3sVppzlJJaSbK05pF60uZGW8EcEstWSEXxzm5zqQjIKmDtf9+Ko4Vmq2oo79jmM7NgajvcoBVQZ9WonSq3iGTRTK4wr4SSq1xTmNf5vOJJWh0ljcOzUXmxFcxcQNSuTNK2kcShgiDFbdwEv/knyMbN7d9MjDYCL0ZNQxAgNJHDNpahBKuZX/qlX2L79u10Op2lPhUAbrjhBm699VZe/epX87jHPW7a7Xv37mXv3r0cffTRVFVVRPJAKCJ5YPz4jke48NnXU/djCrEOXrvJly3c2k+i2RBTAc7011qn3Xtwi1NHlfKXWdCyGFbq2m2QJvIMCaQJaxJbSC7SRHJYTF0STi8pR4mJ58BRypT/s9PNt2vbZmKFQJoum2hmsaySK3NiItTp2JJfLrk9NySYOceZ58BaK0hsi36qanDd5VmyQ2FXSbnSLIYTUUACD050qQPs7lfsaxwTEXb3PHtrx56Jin21Y2/tGE+39RqhVujHNGYwQnBCo5GAElIIPGKLUEfAOTjhhMN533//Fc58+uMP/s1WWHbkKtKzzjqLW2+9db/Fj/tbS1JVufjii9m2bRsbNmzgYx/7GGecccac7jvMnXfeyac//WkeffRRQggcd9xxHHPMMWzYsIEYIzfffDOqymte8xq2bt1aRPJAKCJ5YLz1lV9m+/X3tNezkwzYh+bAWQ7c5FT3GJJw1kk48wftsPu0eTzZfQ7IIdeOTg7FmkDK0DZtBTJvca1IOnOESSQ90oZ7PYNwbKVD7jQdJ+c0ZYpg2nGS05R8niacXnQQcpXBEmC5/zILXx5954adpAxWE7FteW1NGwawp7YVTx7tO3b3PXv6jgcnTADHgqMfZej3Mvy6RkL6/UVov7zk38Xgf3v949Dtceh3GEXb/cOU3yHAaacfy4c/9kKOP770XK4G3va2t/GlL32Jn/zkJ1x66aWcccYZnHbaaTPm/OayluS2bdt4//vfz7Zt29i+fTsXX3wx27dvn9N9p1LXNVdccQU//elPeeSRR3jggQfYvXs3T3jCE/it3/ot/t2/+3d519IneSCISBHJOfKdL+/kO9vuw6lYiC1VgPokHvlVbP9XHbiMJKJN6lkMRCqZXIod1W5vGHIpQ99U8wd8Q6SWwWLZWSA74pOoJUFri3EGTrG9roKX2AqfY8hhpn2ys2zFUwf75F7NyY4ztamkvGeftAZyKlpyUa3QJ6QKW6/UaXUTL26otQT6tfVJTij0orK3djzSOPY2wkS0HtHYvrbkIDehFa0kamLjGfL1/KplQZu65Pi032Hab/rtOqlYK07ZL3Pbrbt4xulXcuFvn8Lb//g5VJWbvlNhxfCWt7yFN77xjTzrWc9iw4YNXHPNNVxyySW84x3v4HnPe96kfeeyluT111/PhRdeiIjwzGc+k0cffZSdO3dy11137fe+w9R1TafTYXx8nKOPPpo/+ZM/WZgXYIg1I5KFuRGjcu2b/pnDelbiksfMRR18KGcXGWXgAlPTBw5PJUrOVA2PqTMBNWGsUEIWXAHVwXFy2DZ/4OcPaUXpSWSCQCS5OHV0xLVuUlCqlOsEEyUTONcKZSuOSAqhurZ/06ngZCCKHrF2kTR4Pd/XXKa204E8Q0uDpX0tJCt4VSS9ELUIPVUmojCBzaXNr2GENpTZil3r7JgkVpO+7i10G3Aej5T+06HrU7nyY7fz8au/x2Xveh4vffkvlh7lFc7mzZu56KKLHnOfuawlOdM+99xzzwGvQ5l7Hz//+c+zc+dOOp0Ov/u7v8vIyAjvec97eOELX8jJJ598QM9xf6yZr3vlj3VufPXKn3D/9/bgo1jjvakTedmPLDhdPOvUs14966nYRIfD088m7bBRO2zQivXqbbarCiPqGVHPOq3YQIeNdNlEl43a5TDtsE4rDqNiPZ4N6Zib6HCYdtiQjtNtc5fmAKMo49KwR/o8Kj32SJ3+77NXGkalZkwaxqRmjD77pGav1Oxrt0fGCeyTwD5pGHNN2t4wlrdJk/4PjElggsBY+tlHZJzIGMo4yhjKPmzFkwkV9kbhkSA8FIVd0fFQEPZExxg2bSg7vGgJ1smObx7esoL9/qYeKudv2zxu3i/9vn273T4i3JTt5PfFDDS18p9e/2X+7ZlX8bOf7j70J1FYEuY6bWcua0nOts+BrEM5fNv69eu57LLLuPvuu3nve98LmHCOjo7O+ngHS3GShZZ6IrDtrXewvj+Yu0qqbEXMzWTHZ+HV5IDU3J/iUhgwOU0kOSRtHWNMY8xVNV0fcqY4QjrWcFgvh3HbvGiuuEzbc+i2PS9RJmhat5kLfdxQyDjnJytc6xwrXBo4MAjNts5RBKeTw7YVwmEo6xA2iWMdLr1e6dyUtOrmkBhOydsy9Dwfi6kOTtIxHLa8mEu/Dy8Wcs3XXQrBOs0CLKhom4PV9lh5lLzd3h47vS4NEYcS1dYAbfLv9DFc5d13j/Lsp/8tLz7353nP+5/P+g3Lo0KyMDfm2iM5l7UkZ9un3+/PaR3KqezYsYOTTjqJ5zznOVx00UV89rOfpa5rjjzySGB+TVERyULLtj+6g/6DNZ0kbtksOAbf+NI6E+Rh5IFc9OKwQeaSjOegEKcVhrShzV+qVcFaoUlsxSULykCMB0Ia0Gnb49D2gFVfmqjm4hOlltCGi3M+M0/88VkgmVwp65PPqtIlh/JzrsNxOI6WCq+OqPZ6hGiCEtrnTBtGzs95GBneNiQ0A3GbKn4WmnVqoxY8ksLdw12n9niSpj7k7bnTVIbOYmqRlDDQOpe+0cTUNxoxR6koQiSkiIKm0HB7sBnbgeCzN/yYz3/uLt7/l7/COedunb5TYVkyV5Gcy1qS5557Lh/4wAd42ctexvbt2zniiCN4/OMfzzHHHLPf+w6Tw61vfvOb2bJlC4cddhjvfve7edOb3sQ3v/nNBRl8sGZEsoRbH5u9u3rc8hc/Zn3IAiStSNqPDLlHY9gVtbepCWScIoh5n7bQJOfc1D5oo7hW5IYjeVPFsK3elNS3l8RxcDs2FxYIMhDekGQ4iLnOiFK7wMSQXORcZoc84EA4qRrhF1zFE6SLqEMVmijEKEncJZ3nzO+vLIZT//ck0UuvW3Z6XmyPyMDh5Vdj+BhTC3Fk6P+B4xz2iFZIFdEZcoq2h0uON7tIsLwx7XkquU/VM/jC8ViBrfwwdR15zX/4Aj+/9Wau/LsX8XMnLJ+h2YWZ2bNnz5wm2MxlLclzzjmHbdu2sXXrVjZs2MBHP/rRx7zv/jj33HMBi0g94QlP4B3veAc7d+5k48aNh/CMZ2bNtICUNSUfm2teeTM/vO4eJA7CcoMQIW0YcbjNIwtj3icObc+FNiaMdo/J+2pym9KK6LBAxqH7qOTClkEbQxbFgYgObye5yCS6Mt11DgtngxLFWhue2q04vTPCk906JHpUIURHjLaIdIhCVGiCmINMrquJ6UuADs4/h6cDDL0OQ+0W5EKoIXctk69PLejJ29rnLZP306Ftw7+v7Krzvrk6VofuG9t9h67LoB0k3962hUx6/Olkxz5tu8Dr3vB03vBfnlG+vC5jbrjhBn7wgx/w9re/falPZRq5j3M/lBaQA6H8Mc7Ogz/Yw46/v5d1AuqyIzQB05xjS6G3QZ7SyKHX7DTzZRO4gRsd7J9FUlLV6+T98gc2JLfZVr6m+4q2lbZZDNvLOnCpk8K4SluJOxyuDUSO6Si/uF44eX1FN6aVS0ISRUB15veNEwg6cGdehCaFRwPJiSWBbMOYDBx0FrnMQGgme7P8JSHfMjm4Othml2cWq6koVpAlMvh9uPY2c54un6dajpah2/PkIwVciu9OFcocvs3Hm/T4Cu99zy1c/dHbufJ/vJhTTj1mDmddWGyW49zWzGJ+nq8ZkSzMzudfcwsdGrqdnENLopUSazG7o3RbUFrxtP9TtksHbR2Tw6X2QT7dadoH9iCEa4IJA/elQ4I4fD0LJri2WV6hzZOpDjkrLPwbsZ7Fx62rOfOIwDEdQdXRhOQYkemimDQrRUFxYq+DiCIqIKkoRqxXMgppibDpYZjhPO9g21R00r55YMKwOLaXxS5lWdWhQ+RFqAevbd5XUp5RJj2GDh1n8IgDkcuPY79L+2qQl0rL7S1xhhaV2YQS4JGHJ3jx8z/Jb7zyKfzxe56H92um2H5FsGfPnrYQZi1TRHKNs+Mr97P3jofoVknw1Ip0NH3gu7RclRmGQQGPfS6mD1rNoUYZiJMO5b9IoUaV9r5Ti3Ns2/Tbp7vRLKSDnOSkUK/ah7XddxA+POHwCU49eowjOtA0gzDq5Gze0H+Tr5oopiteTFCdqD1nSCKcXj9MOIbzt1PFMT+/4fzi1H2G950UTs1y2T5Xu9egkGrg6gYiO3i8YQGbfn0gmO391a7bQInBo0Vi66Sz2xx2lPn5OAZh38liaZc/cfUP+ex1/8qVn3gRZ/5vZbzdcmF0dJQnPelJS30aS86aEckSbp2Zr73+W3SrGpeq8+1DXogpztcEScIJqhFNQdc2PKeASCuwgN23daDWCtEKaus6h0KoyaEOilWYFFLNQtLeFxPP6bnTlONUqFzkcZvHOflxu9nQUXOMddWK9mzpCiHnOmymagg2ISfqoPzFwtBqrjM/t2F3zZBoMzlfO+yadco+eWTcZKc51eFNdpJTjzP7Mxt+jgMpHC47yn514EAHbSH564QyXCHsCEkoVVOvZZs/HpB/h4OvQDKtQXtsX83/8WvXcc6LnsTlf3U23a7fz7MoLDS7d+9etuHWxWTNiGRhOnd+/E7CI3vpJBepav5AnCDOPoRHSC4p2senxoiqEGL+QE9eRc09ZsHU7LAiA/epWeSkFUYT18GxYEg006d+KyzJ0eT1I7ODHThGYcumcU550i42rQ+EJuUYG5uPQ3rsYSQtpAzgfSAEWycyqhACiFM0OmKwLwExFe7k5xImCeMUV8nwbZNbW4a3D5+WMtldW9HRVFc6Nag5uDYIiU5u45kpl9lGBxgOrtLunfskLX6bvxBlkc2n3Gaq7X8VVAZh1mGXnI8829cTgG2f/Qlf3voRrv70i/lfz5y+ykNh8RgdHZ3TMIHVThHJNYqqcvs7t7NhfT99gpl7jGpFKxrzh2psp67kpoUYs7jZcTSmIFwSSEiuL2YBdIPtyX2piC27FCWFcZM4tPc3YqStMJnsQAeh33XrJ9h6wgMcfdQYGjwheBMxHXyUZyaJoouE6FpRjFFwEi2EGkHVzcUn1wAAIABJREFUEQLt65Kfc1vp2oZYk4AyqG4ddo2DStOhqleGxE8GwjjZcU4Xw2EjPFNoNt829eKwy5x6vyxccYpYDl49SV+gLNQ92ePZtkGLyNADTlHD/AVguMxoqqcFGBuvOf+Fn+YF55zIB/76BcVVLhHLuXBnMVkzIlnCrZP5/ru+hY8TqdgirYfoBZEGdOCWYvSoNvbRphZsM/cBMTo0ClHr5EzMRTAUdrXQbUDEtZ/OMYVoc1iWVmwBNxDoXOwjMgjhagp7IsrjjnuIE37uQZwIMTpCk6f52L4ipFJxE19fBULj8T4SgmXKrErVHjMGa/EI0dlPMEEMMX2BSOIYFRrF7oPdvxVLZhbHfF2n3iaD7ZP7TodbV4bvq21Ic3KYdfi+g+uTL00u9JkNC6XOXMxDmpQ0OMYgJJuXLYs67Ggn5ylje7w8eGJ6UU8Wzs9v+wlP+4Ur+LtPn8fpZ5RFoBebPXv2sHnz5qU+jSVnzYgkTC48Wcs0Y33u+R/fYcOGQIwOUv+fqiMGQVxD5QB1QGPuMiQhy51vSQxzLjJqKuHQfGPKb8b8BSVYUU0UxGWBHNgN1TwkjbbCNEaS6xtU0VRVzc///A4O29gnBiEGnwRvqNQkiaP3kaZxOJeeJ0OXxcKoIYlgSG0fdp30k8QykvYZcpBqwwRCeg0Cww4xr9gx7By1vQyDgQgDER0scaXQVooOiyxDwjlZGIfCmlPymoMQ7OTtg7zkMNLmI4elMYddZej3NUxst+bmkDR2QFMFskq71FY+cj6b4dzkTIHY8bGGc3/1U5x7/lYu/4vn0+kUV7nQPPTQQxx22GGMjo4ekJN8+OGHeelLX8pdd93FiSeeyCc+8Qm2bNkyaZ+7776bCy+8kPvuuw/nHK9+9au5+OKLAbj00kv567/+a445xlqC/viP/5hzzjln/p7YQbJmhglAWVMyc+vrr+Ohf/wZ7YdgbixXEyuNLjmwwX1iEtGBUMb0pcOEBWjDtpabshBlTmblnkvnLMyaQ3igg3BrEscsjNkFAmzctJcTnnSfuctoDjcGRwyOEJwJfLTrqtIKpzlAE/8mCKGxcGyInqax+4boCUEIwdME1wpi6yqDDQvIAmnOMYdbdZJAThXL2G4fHngwOaTaOkkZbFOGxS/fbq/FsLBO+n/K8llTW07yQIF8DNA2kgDDQjpwtoN8L5P2GZzL4Pwnn8/QuUxxy8N/gTmkmwenD0v3cPGPvQc6/N11v87TSl/lgnLttdfyV3/1V3z/+9/nN37jN3j605/OGWecwemnn86GDRtmvd8b3/hGjjzySC655BLe+c538sgjj3DZZZdN2mfnzp3s3LmTM844g9HRUc4880yuu+46TjrpJC699FI2btzIH/zBH8zXU5mX8OGaEsm6rolx6p/e2mLs7oe59cKPIGjKPVpYkdTikMPSGrP4mRMQSc6wdYd2PCXn8rR1kTEqIm07OmDhTHNv0lZlDpkVWnuawnkm3pEjj32Yo4/dbWHfaCIXoxv8NFkUzWHEKENCaU6zCZ6mHohjCI6mtv7ILIpZMJtowwRMJB1NcpENFmYNQ64xu8XhRY0jU8WRdhJQ/mMadpNZqGByq0eWrix8g5/B7RaBnl7dOj18O3s4d+Du8nHidEGcJLY6SWBnerzhUYFTXe5w6HgYC/EO3ORjffD8zn88lUv/6NklhbKAqCpnnXUWH/3oR/nOd77Dt7/9bc4//3zOPvvsWe/zlKc8hZtuuonHP/7x7Ny5k+c973n88Ic/fMzHOe+883jta1/L2WefXURyObDWRVJD5Lv/4QrGf/YQ4khukKHc4yDfBNq6NYjYuskpvBpM6LLLU/s0N42Lg4TVcHtIOqSJMULUgYMEcC62+caqW3PM4x5iw6ZxYmNtGyFUrUhmUVRNgpgLidL4uKZxxCaJY5MEMqbL+Sc4msZTN46mcdSNow6OfnQEFegK1XEjHPXUTRx/xhaOe9omtjxpA4dtqehu7OArl8LHStNXJnY3PHzfPu69cx8/+96j/Oz2R9n5433sfaih3ws5ADn0QtgXDGnFbjh4ai/g1MrQLGSITnJZMwli3j58fZKITRPImdzg5PtNFulhoR30cE6/b56IlJylDD5SZnOVMxXzTOWoo9fxPz5zPlu3bpl1n8LBo6o85znP4bbbbpvzfTZv3syjjz7aXt+yZQuPPPLIrPvfddddPPe5z+X222/n8MMP59JLL+VjH/sYhx9+OE9/+tN5z3veMy1ce4AUkTxQ1rpI7v6fd3LX2z8OyR2RnKG4VOifco9ZOMWZ2yStduFEU24Phuyf9RAKVnjTVnQoVtU6PGNRU0hVJzXnqyrOKZ1uw9HHP8DI+oamn8SxriaJYqizY3TtucQghOQE674nhur/Z++846Oo1j/8zOxmEwKhBpCACEgPCFIkcGmCkSJSvSgWQECQawGVq/xUFBuiXsSCqKiAIk1ERDEEaQETQkIo0qQoIC3UUJIg2TLn98fsTnZDAptkU/c8fiK7U3bObJnvvO95CzZ7ADab6hRJM3aHCavVRIbVjMMcQPVutWgzMZwKtW9cwLmg0DSNpJhklszcw+74FGz/6KUAVCPG1P1X7p6ioSOy/GtYnx7bZBEuBQ+Rc9/GfX/N7dU9XbRZRNDN7ZqTyGa1Kl0inbnOEyXLv/pjz+udQP9evTCpHU881RqJb3E4HNx5551s377dY/ldd93FqVOnrtn+rbfeYtiwYV6LZFpaGl26dOGll15i4MCBAJw+fZrQ0FAURWHSpEkkJycze/bs/JyGFMncYrfbcTgcN96wFOL4J4P9o6djv/QPJpPD6dl0uk8duvWnOguOO+yK0/2q39drDpewuQJqdHcq4Jba4WkZekZFOssAKJnB/yigqHpROlOAgyo1z2CxCOw2MwjFKZIuUVSc847OOVC7HmykW40KDpsZm82Mw27C5nxstQaQYbMQ3LQGEdPuJKRWCA6HA5vNZrTbKe7s3XGG919JYM+m86Dpyfuqh1xcx9Zy81xnCpKnMLqWucgqbNdaha5/Mz9k97SPa+dCr7UoXfsLJXN7Y1mW8Tu/Ju4hWR6nKNBdvnXrlmfZzwOpWq1sTu+GJJdcvHiRhx56iI0bN3q9j7fuVpvNRp8+fejRowfPPvtstq915MgR+vTpw+7du/N8DvhIJP0qutWfubQ+iQDHBQLK6sKiz/llpldoQhceVRWYzBjRpvp6xahTqgufUwyNaUXhdKtmCmum0xAQQnfvKpmXSVUFkzmDSjVTCAhAF0OhOK/omQE8qklDc7hHqOpjtNtMzj8zNquZjAwLVhFEncc7Uu+h20vFfFXTltX48qd7PZYJIVi2aB/vv5bI5bNWo/el609HMaZ4Xbcl7napuxy5C5O79ZjVSr1mj2ze3syjXFugwNVvVK/ZpLvbTc5CBhqZKSea+2EUstx0udcIyhzR4cOXaXXbXP43vRv3D2mSzTspyS2XLl3KdSGBvn378vXXXzNx4kS+/vpr+vXrd802QghGjhxJkyZNrhHI5ORkatTQyxIuW7aMZs2a5f0EfIi0JP0A+/mLHJ3wDprVhsms6XN4zghWVdXnxvTKOM48OmcOo2GFuPx0AM5ybHruoXOZ7p915vw7rVDnV0dRhNMq1ZcpKqhmOxVvOo8pQMFhU0GoTpHU/zUsRltmMI5w6HONdpsJm9WMzRqANcNC2Y4taPJyH1QvimOXNEvSW4QQLJi3h3dfTeBqqo0AYcKEagTC6CLlqqGaWS3HY6rYeO4upu7zpE7cA5nd3LOuZ57l6MQ1j11C6Vrm+mp5Y1Vm/dRcc7nutIuowfzFfSlTRt7/54fdu3fz8ccfX7cBclbOnz/P4MGDOXr0KLVr12bJkiVUrlyZkydPMmrUKKKiooiNjaVTp040b97c+B26Uj0eeeQRduzYgaIo1KlTh88//9wQzTwi3a25xV97SqbMW8KVjRvd8iCdATuaSZ+DVJ0OUYfzkupMWRQ4xdCZ0a/gqmGaaUHqFXPQA3uc5qUeFJRpS6qqBqpANdkof1MKAQGqPsfocBYA0FQ0u0u49eAco6CBM23DdtWMNSOAjIwgbho7gOq9bs/1+6BpGlartdSJZHacP3eF0Y+sZOuW05hR3CxO1bA4FZFpaboLp7uAQqa71vksi4vWc1lOIpndHKXxOJv5yWvmKrMOyrUsGwICVOZ+25uud96S/QaSGxIXF8ePP/5oNE4uoUh3q+TGWI8e4+rmOFSzHdUknHVIVRRFQ1Hszoo5+nPdNepKxsd5xdSdYS6rUf/CCFD0Oq4ugRSabjGqigKKbgvoWqRhsmiUqXSBwLIamt2EsGcKrarqEaKKyelO08AUYEezK2AzYcswkXHVQoVH7qNe7/ZF8A6WTKqEBrN05SDj+dWrdv47Zi0rov7CLgQmwKSomAxXrWpElro5vPEMH/J8nFUss3fd4nylTNerXplHweEsUaC3GBMgFGeHFYFRvcd1HGG80A2x2TQeun8F9w1uyPSP7kJVS77rvbCRJeky8SuRLA3zVLnlyi9LCbCk65ahUFACHHqgjmEyOOcXVddjYYik4lqPs/ugc17SdfF0WZ1CA8WkFyUwaqOaBKpqxxScStkqDjSHgrCZjUa/rvlJRRUoJg0cKiazvp09w4TtHzPUaUbt/xuL4geWX0ETFGTm46978LHzeVpaBk899CvrNx3FJjLdsSYyhdPdwnRv7AyZc49ZxdJFdoZfVlxttgBUkTkn6SpXl+NcpesAN+D77w4QvfIwy34aSNPw0BvvIDGQxc0z8SuR9Dcy9uxAHN2N2WLXgyAcAAqKBd1Sc7pG9VqpOF2rzg4gzpZZRikyBRCa4YJ11UXRA3L0S5ii6vmUqupADbxCmepXEZoK1gDjlVAzxVE1a2h2VbdwFYHtqkrGPxYqPvsiwbfWKfw3zI8oVy6QOcszg4IO7TvPiP5RHD53mQwwPnKT0C26rJZm5o3SteQkkNd6Sz239LAq3SxJ1+PMqFm8Fsu0VBuRdy7m6Wda8/zEdn55o5wXpCWZiRTJUorQHNjXzsdkzkBoQrfYAvTqOqpJ6P0jnYn/RpEb439OixLNyJd02RCKKlCE8HismDQUk9DFMsBGUOgVVDMIq1mPejVe2xmxarGj2Uz6XKVZ0y1HqlLxramYAwOL4N2S1GtchZh9jxjPZ72zjQ/f28IVYceqgEBPG1LR73NUp1y6qrW6C6jL6rs229P1PcoURndrMusyQaYlKdxeEdwCfdx3vY7+fTR9K0u/28fPUfdRvUY5b98Wv+Xy5cvceuutRT2MYoFfiaQ/3UXat6xBSTuBKcgODr2ajmqyYXIVEHCldqjCrQi5uyWJ86LjjGZVAVVDyfTD6tagqiFUB2qAA1PZdALKawhrgNMSdQa+Ks4mvRYHmk1FUTVUi8B+VcVRpTnl/zPJrz6bksDoF1ox+oVWABz76wKP9vyFYympWNGwKWBXMmch9UxWp3CKTMF0zVQquMqpY0ie4iF+Tty+Au5C6d6DMqtLFvDaFXvieDptW3zNm1O7MHRE8UgvKK5cvnxZulud+JVI+gvC+g8iaQGqKUNPvwi0gwNDHIVQ9GhT9xh6xS2zzZWu4bxQqWrmBVGfUxQoZg0FB2qAhmr+B3NVO2gqOFTjrh9VgAaKxYGw6iWs1QAHjgwT9rAOlH3IZzUavUbTNL+IbvUlN99aiTUHHwbAarUzYchq1sccIwMNKw5sRnpmZkF6l2g6S044lymu+y5jme7WzV7VVBSP1BDXa7i7YQHDFZs1xzI7hICXX9jI4gV7WbJ8AMFlA7x+H/yJ3HYAKc34lUj6i7XiiPsWVbuEGuxA2E0oqoYI0Oce9RqpTpFwiiWAIoTeB1DVrT5N6Nua1EwHmaKgW5OqANWOGmhDrZCBGgjYVESGM8xDFeDQhVTYVRQNlEA7mlXBUbktgUNeo7CdqnqdVQ1N07DZbPo5K4rxJ4XTOywWMx8t7QXo7+mXU7bx6fvbyEAjQ2g4ENhdhczdBBP0ps2qa7KTzIAgV7JQduQ0v+nuhs2MmM0mGjbLi7gEedfv52ha7wtmftmD3vdKt2JW5JxkJn6VJwmQkZFR1EMoWNLOIhaPRGQ4wOS8t9bDBN3mIBU95cP5L85qKJml45wWgZurVDEJXSAVB0qAgKB0TJU0sJvBpoJdRbOZwNmVQ9j0/EfQcx1t6s2YHvmi0MXIXRyzLnP96467YErhzB2/Lv2TSWM2kKY5sCkaNjQciqvbCVnEMVMksw/+yRrWkzMeAT1uZC1K4N5lxJ3ukbcw6+ueslclkJSUxN69e1m4cCFz5syhQYMGXu3nTS9JgDp16hASEoLJZMJsNpOUlJSr/XOJT6wivxPJUt9T8teXITkBEKApzuhVDHFUFc2Yi0RkzkmiOPURPd9R95Pp8476cwEmO4rFjlLRimJWwK6CVRdGbE5xtJpwZYdrDgXNGgwP/YhithTq25BVHF0WY3a4C6brsfvrmEwmaW3mkq0bTzDuvtVcstuwKhp2NKeF6SZoWUQz8/G1Una9jiDuZHXPeksZi4k5i+6hQ6daedq/tHDo0CFWrVrFtGnTCAsLIygoiBYtWjBs2DDat885T9mbXpKgi2RSUhKhoaF52j+XSJHMC6VaJFP2o64eCw4t03JURWb0qjvO5G3XJJGi6GkYxhyR4VbV9L8ADRF0FaWC/rpKhgkcKmQ4a65aVdD0Kjloulg6un6BUq1RIb8JurA5HA6jlmxe3OwuwXSJZtbvjKqqUji9ZMdvp3hq0K9ctludgunQm1Armc2ogSyBO+6PFY/VNxLMnCzL6+GyMgfe35D3Purm9wUIOnfuzJYtW8jIyGDnzp1UrlyZRo1y/i17W9w8J5HMSy9KL5AimRdKc7ssdfUwlCsHdfFyBs14XFKc7lOc7bFcZeMUMOYmdatRLyPncrESqKGVtUMwKP+oKJoCV/WC6GQ4LUcBwqGCA+yhfaDdy4V+/rmxHnOL6zVd4ptVOOX8pndsWX+C8fetJl04sCkO7AgcCByKhj0HwXS3NLMKZuY22X/O3gqmyW3/smUDmP/DvbRoVd27kyqFdOzY0aij6g3e9pKsW7culSpVQlEUxowZw+jRo3O1fy7xyY/frwJ3SjPKmQ2ojn1gARQ90lRxWpH6BgB65w9XKI7qTPUQip6/qCfAuVyrAmFyoAUKREX06nQOt1RKFdAEBDrArujWI4E4IteBqXAjBgtSHF1kN0/p7qZ1n990CanL2pTCmUnbO2sSd344AGuXHmLS6Bj+EQIbDmxOwbQrGprwbA7tClh1ZAnyyex84lmUwP1x1vSRrGT9ZNLTbfTv8QMPPxrO6+908puAPxc5GU7X6yXpLXFxcYSFhXHmzBkiIyNp3LgxnTt3zvNYCwMpkqUE0/43UILshosV4QBFM4LwEQK9N5Zwll51VdJxulUVdGHUM8XRzAJ7OSAYVIeC8o+zSbKzTqsSqIFNAQcQILDVnQo1ehT6efvCtZpXXMJnMmUGfGSd33QXTpCBQe50H1SP7oPqATD/g918+EYiNiGw4ZzDJNO69GzU7I5nYYKsy7IKpntojnvrrez4ds4eflp6kIXL+9G0mf+Utcvpt7RmzZoc96levbrR6io5OZlq1aplu11YWBgA1apVY8CAASQmJtK5c2ev9y8K/O5XWhrvCtXkeWA+B0EOCLZBkAMR5ECU0SDQgbA4EMEOCLI7l9sRZRyIsjZEsAMt2IFWzoEjREMrL7BVElhrgFZFybwTd93RB+nznUIRCIuGZqmMrf22QhdIlzja7fYiEcicUFUVk8lEQEAAgYGBBAUFYbFYCAgIMAKAHA6H0bbLZrNht9uvicD1Nx4a34zE8yPYeu5RhgxqTHkCKCcCKCsCKCNMBAkVi1AxCwXFTSU1559AF1LNaY26NhFu/2XFm2Cgy5et3HPnEl6b+JvffD5Xr14lKCgoV/u4ekkCOfaSTE9PJzU11Xj866+/Gj0jvdm/qPC7OclS11NSaJh3tgFhw8hoVFypYc4MMtdcpJIZVC+c8TaooJnQM7JNYAsErQKomoJqBdNV3XuraAIlw9gdRQNH1Y+gcrfCPd1CcK0WBjKi9sb8c8XGU3et5Pd955yVfoQz0MeVWuIUxCwfv/scJrhK6Lmvz9v3pVIFCwuW96dxeJU87V9SSE5O5oknnmD16tVe7+NNL8lDhw4xYMAAQL8OP/jgg7z00kvX3T+fyMCdvFDaekqqp15FubgUoWdSG70gFQFCzVyEM84G9OUugRQm0MwgAhQcQXYIBIQJ1aqLpKKB6YrzYEKgOAB7CKLOb2AqXG99UbpWCwMZUZszp46lMvxfKziX9g8ZbukkuRVMUxb3a25wRcA+MLQxr7/XpdRGwB44cIC3336b77//vqiHkl+kSOaF0iSSwnEJ5eS/jKgGV1aHsd4lhK4FrmurSdEF1AQOkx69ihmjXRaaHsFquuK8kGgC1YouvoGjodJThXeSlB7rMbfIiNrsiY8+yvMPxXBF2LAqwjlvKbArztQSnAUMsnxF3N8h/afhipT17rvkEQFbLoBFP/ejSSmcq0xKSmLevHnMnj27qIeSX2R0qz8jhIDUp9CCFGe6o3DWD1AyK+c4b6MFetCOw1mJWpg0vRqPq4mghy/KeRFWQQsCNUMP5HFYVEzl14Gp8FxN/iqOLmREbfa071mb384PBeCD5xJYOGcPGQgcqHqVH6d1qQmRmYep4BHZqqJHyqpZ6vrkXEfWk7Q0K/fe+T29+tXj/U+7l6pqPZcuXZLFzd3wO0vSvXZnSUZz/IE9434Uh0A4q+OAbiHidEcKxRmtirOkjsm5Xdbfc9brgkMBo74rqPbGWIIK1/VS2l2rvuR685vgHxG1Vqudke1/5o/DF/RUEldZPFdKCZ4RsuBZ5Qeyd8V6VgPK/jtoCVSZu6QPd7QP8+UpFRk//vgjf/75J6+99lpRDyW/SEvSXxHCipWnweLsvOGq0SoUo4SccCWWGV+TLPc71/v6mARoDkBB0V7FEnS/r08hR/zdeswLN0pFcd1wgKfFWZqE02IxM2+rHhRycOc5xty5kkvC5pZOInQLk0zr0j2lxD0H01U8XSdTLK/taqmTkeHgwb4/EfGvGny58B6CypTsy6osbu5Jyf915JLScMF1sA7U42AWEODQRc0kdLE0icxk/5xO9UZvgQAUBTNrCTQVjkC6p3RommZYP6Xh8yoK3FNRLBaLRyqK2axfxN1TUaxWKw6Ho1SkOTS4LZR15x9ha8oIxo65jQoigLLCTDABBAkzQcKEBT2dxFVoyuVvcUBmEJDzPy2HIgQuXKIZH3eS5rd8yXfz/yjgMyxYLl26RMWKFYt6GMUGv3O3CiGwWq1FPYw8I7hABncBV/CYTnE9zppPnVuNEQDVCWR9nkPlc4t0rRYd/hJReyXdxvA2P3H49CWsThesUbBAcRNCt6+eEefmttCbdBKBIOzmcnz/y0Cq1yhbEKdToLz99tvcfvvt/Pvf/y7qoeQXn1xISvY33w+xsxhDICHrpEkmWWPfb4RR4PLfBBFTKAJZXAsC+BOqqmI2m7FYLAQGBmKxWLBYLJhMJlRVRdO0awoflESLM7hsAN/9MYgtKSOYtagHlTDr1qUIIFiYCRQqFhRMboVeXUUKXNGz+jJxTaeR7AoVnDiWRsRt3/DWK3ElpqGC63d4+fLlXFuSKSkpREZG0qBBAyIjI7Otu7p//35atmxp/JUvX54PPvgAgMmTJ1OzZk1jXVRUlE/OyRf4nSUJJbenpMbfWOmjZ/f7EuenbOJ/BHCPb187u8PJeccSRU4RtS5KaiqKEIKXh6xnza9/k4GGVXFkzl8q1+Zduu47XdZkTmkkWWvEWiwqX87vTaeuNxfCWeWdzZs38+yzz3Lu3DkiIyPp06cPbdq0oW7dujf8fea21ZXD4aBmzZokJCRwyy23MHnyZMqVK8eECRN8eUoyTzKvlNR2WRn8H4IfffTRO3G+DWZ+xUzB/4ila7V0UNoiak8dS2XoHctJybCS4exO4nLF2tGydcW63LBZXbDgmW7iqhHbuGkl5n3fl9CqwQV7Mvlk8ODBDBw4kNOnT7N161Y6duzIc889d919ctvq6tdff+W1114jLi4OQIpkcaMkiqTGNqwMRS954yMEQBksJKIWcKCztB5LP9ezOEtSRO1Xr23niw93OK1LVyqJwymYTjLTkPXeAJlVjnO8aLrcsqPG3saLk/9VbCv29O/fn/nz53PTTTd5vU9uW12NGDGCVq1a8eSTTwK6SM6dO5fy5cvTpk0bpk2bRqVKlfJ+EjpSJPNKSRNJgcDKIwi2+s6KFAB1CeTnLL0RfIsUR/+mJNeovZJm5aEWyzh2IR2rmzvW5nTHiiyWpbdBPQ4ElgCVuQv70KlL8XPBduvWjY0bN1KmTBmP5ddrlTVs2DCvRdJqtRIWFsaePXuoXl3v2Xn69GlCQ0NRFIVJkyaRnJzsi4o/Mk8yryiKUqJE0kEUgm0+FsjuBDHDRy+Yw2Gka9XvySmH0100XcFB7vsUB+EMLmdh2V96CtSq+X8x+emNXBEO7IqGTWiZDaMRaIoe1OPKvHLPtcwqlgoKVpvGQ/f9zM23hPBj1ECqVis+UbBWqzXbLiC+aJUFsHLlSlq1amUIpGt/F4899hh9+vTJ4+h9T/G7fZN4ILBhZxoe/YHy94KojCtQgZRRq5LrURIjans8dCvx5x8l6fSjtK5blRBhoaxmJkiYKSNMmJ1NBVytu9wjYoXbv5mZWfrv4ejfl7k9fC6jR0RhsxV9d6K8Gg+5aXW1cOFChgwZ4rEsOTnZeLxs2TKjhVZxwC/drSWpXZaVz9D4MP9WpBHB+hYBDMz3uLI9hHStSnxESYioTfj1KM8MWUu60OcrbTiwOVt6CTDmLV3dQ3K6mGpkbv/Ci3dW8cfCAAAgAElEQVTw1Lg2Rfa7EULQqVMnfv/991zt502rLIArV65w8803c+jQIY+qPo888gg7duxAURTq1KnD559/To0aNfJ7OnJOMq+UFJEUXCaDrqD8k98XAsDEPAJok+9xZXsI6VqVFDDFNaLWbtd4olsUW3ef5ioaNkXD6gz0cY+KdY0oa7qIwK17jwCTCT77oge9761faOfgwmazcdddd7Ft27ZCP3YBIEUyr5QUkbzKRGB5/j5qI8XjZ8z4/kcnrUdJUVLcImq3x57iyX4rSRMObIpGBvq/rtZd2dX4yJpe4vrNhgSbWbi0L7e3ybdF5TUpKSkMHTqUDRs2FNoxCxBZcSevlISLuMYJ4GcfCKSChV98LpCy1qqkOFDcatTe3vEm4s4/yvZzI+jRoQ4VhIUQYSFYqJgFCAGa29zlNQLp9jw93U6/Xsto3Xg2Rw5fpDC4dOmSLG6eBb+Mbi0JWJmAZ0pyLhEAKgHEo+Lb3nDStSopzmSNqHUJYmFG1KqqyvSf7wZg//azjIxcwWXNQYbiwIreHNpxozrLzudnz13lzrYLqVGjLN9HDyCsZohPxpgdly9flr0ksyAtyWKIxk5gR96tSAEQgIUYTD4USBm1KimJuFyuRRVR2+j2qsSee5RtZx7l370bUolAygozZTS9E4khkiLzT3fLOn9bTh9tcnI6nVvMp2P4PE4eS833uLJDNly+Fr+ckyzujZevcico1ybteoVTIAOJR8E3uVdy3lHiDxRmRO3hfRcYdXcUp9PTsaGRoThwYHSp8wjuca8RaxJ6ZR8VhepVg5kXdS+31PWde/SXX37h999/5+233/bZaxYhck6yNGJnNZAfgQzCQqxPBdJ1Ry3nHSWlmevNb7q7brNanO43kN5St3El1h59iF3nHmPcU62pLIIoI8wECgXFZV06TRTXry3rxfrM2Sv0bPsd/Tt+z8njafk6dxey4fK1+KVIFueLvJ3/5u3+RwBYsLDOJ3OQ0rUqkXgKZ2Bg4DXCqSiK8TtxBQblRjgVReE/k9uy7fxIEg8MpU3jmwgRZixCwYSiW47C2XVEuDqOeHJw30X6tlvCiaP5d8HmZU5yyZIlhIeHo6oqSUlJOW4XHR1No0aNqF+/PlOnTjWWe9NmqyjxS5Esrlj5EshDGy9DILeikr+iwDJqVSK5PtlZnIGBgfmOqK1YpQyL4way5/xoPvn8bqoGWAhwiqUq3DuOXPtbtGZofDk9dwUA3Dly5AjDhw9n1apVnDx5ksuXL3u9b7Nmzfjhhx/o3Llzjts4HA6eeOIJVq5cyd69e1m4cCF79+4FYOrUqXTv3p2DBw/SvXt3DwEtDkiRLCZoZKAxLfdWpCGQq/PdyUO6ViWSvJHV4nQFBpnNZkwmk/HbcrlpbTbbdYWzx323kpg8kp3HRjHs4XACUTGheliTWcXy0oWreR5/7dq1efHFFzGZTOzcuZPevXvTqlUrpk2bdsN9mzRpQqNGja67TWJiIvXr16devXpYLBYeeOABli9fDsDy5csZNmwYAMOGDePHH3/M83kUBH6ZAlIcL/w2JuZ+JwFgwkIcKuXyfGwZmCOR+JbsChi4fl+u9Cn355B9YFBQsJlJH3Ri0gedOHU8jWeGrWHH72czj4OCClgCTfQcUC9f423YsCE1a9Zk+PDhdOrUCZvN5jPX54kTJ7j55syOJ7Vq1SIhIQHQO4C4StDVqFGDM2fO+OSYvsIvRRKKVycQjaMIonNnRRoCGZ1ngZTiKJEUHjkJZ3YRtdkJ5021yrFwbX80TbBp/TE+enMbR/68SFAZM2Ofb8Xd/fIuki5SU1ONwJ2AgACjm8f12mRdr5i5i+yutSXlWuO3IllcEDiw8mRud0KvpLMKlZp5O64sCCCRFDk5tRJzv4HNTjg73FmLDnfW8nmpvcuXL1OxYsVrll+vTZY31KpVi2PHjhnPjx8/TlhYGJC7NltFgd/OSRYXUdBIAA7mei4ygB/yJJAyalUiKd7kJaLWNb+Z3+IHBZUC0rZtWw4ePMjhw4exWq0sWrSIvn37Arlrs1UU+K1IFgcEV7DxXG52AAEBfImJxrk7loxalUhKLDlF1JrNZsxms/H7zm+N2itXrlCuXO6mb5YtW0atWrWIj4/nnnvuoUePHgCcPHmS3r17A2A2m5kxYwY9evSgSZMmDB48mPDwcAAmTpzI6tWradCgAatXr2bixDzEZxQgfllxB/SWMEXZxBXAzmLsTPbeihRg5lPMdM3VcaRrVSIp/WRXozbr9f1GNWo7derE9u3bS8s1wicn4bdzkkX9JRCcxc6b3m4MgImncyWQMjBHIvEf8htRW5yCGYsTfiuSRY2NuYDd63sdlZEEMNarbaU4SiQSyF1E7fnz5zl//ry8VmRBzkkWARoH0JjtnUAKULiDAC/nLmVBAIlEcj2yzm8GBgayadMmBg4cyNix3t2I+xN+a0kWpXDY+Mi7DQVAEwL5+sabSutRIpHkkoyMDF5//XV27drFypUrqVWrVlEPqdghLclCxsEmBGtvbEUKgBACmXf9zWTUqkQiyQP79++nV69e3HTTTaxatUoKZA5IS7IQEQhsvOXlPKQZC5tQrvMRyahViUTiLVeuXDFyLefOncvs2bOZNWsWbdq0KeqhFWv8ViSLAjuLgUPX38gZXBbAtzkWLJeuVYlEklvi4+N58cUXOXnyJDfddBPPPvssISEhaJrm88o9pQn5zhQSAhsOL7t8mPkWEy2ufQ3pWpVIJHlACGFU65k+fToff/wx58+f5+WXX2b+/PlFPbxijd9akoUtLDbeA27QPVzoqR5mWl+7SrpWJRJJHsjIyODNN9/k999/Jyoqyph77NatWxGPrGQgLclCQJCGxvzrW5ECFPphYYLnYllrVSKR5JEDBw7Qq1cvQkNDZXBOHpGWZCFgZTxwnRJ4AqA6Fl7PXCTnHSUSSR7RNI1vvvmGL7/8ks8//5y2bdsW9ZBKLH4rkoWF4BSCuBvMRQYRyDpn+1TpWpVIJHknJSWFp59+mooVK7Jx48ZcFyyXeOLX7tbCEJ8Mhua80ujqMQcFVbpWJRJJvti4cSO9e/fmgQce4KuvvpIC6QOkJVmA2NkFHLuuFWlmNqpogUNzSNeqRCLJE1arlTfffJNt27axYsUKateuXdRDKjX4tUgWZNV7gYadYTkLpACFezCJCOlalUgkucZqtTJw4EBuueUW4uLi6NevH6tWrSIgIKCoh1aq8Gt3a0HiYDnwT/YrBcAdmB3vSNeqRCLJE2azmV69erFx40a6du3KH3/8QZs2bRg1alRRD61U4deWZEEhSMPOi9lbkQLAgmp7Dw1NiqNEIsk1KSkpjBs3jpCQEHbs2EFISAigB/2dO3euiEdXuvBrS7KgxMnK9OsdFcW2ClWpLKvlSCSSXOMKzhk8eDBz5swxBBL0a1rVqlWLcHSlD78WyYJA4ziCBTlbkY7xmJRQKY4SiSRXWK1WXn31Vd555x1+/vln7r///nxfR0aMGEG1atVo1qxZtuuFEDz99NPUr1+f2267jW3bthnroqOjadSoEfXr12fq1Kn5Gkdxxq9FsiCEyspL2a8QgPYAAYyUAimRSHLFn3/+Se/evalQoQKrV6/mlltu8cnrDh8+nOjo6BzXr1y5koMHD3Lw4EFmzZplNGV2OBw88cQTrFy5kr1797Jw4UL27t3rkzEVN+ScpA9xsA1IvNaKFAAVMIvnC39QEomkxKJpGt9++y2ff/45n376KRERET59/c6dO3PkyJEc1y9fvpyhQ4eiKAoRERFcvHiR5ORkjhw5Qv369alXrx4ADzzwAMuXL6dp06Y+HV9xQIqkj9BTPt7MYW0gJsdyFCyFOiaJRFJyuXDhAuPGjaNs2bJs3LjRY+6xsDhx4gQ333yz8bxWrVqcOHEi2+UJCQmFPr7CQLpbfYRDRCM4lK0VadKmoRLqs2NJJJLSyW+//caxY8fYuHEjvXr1YtCgQcydO7dIBBLINo88p/zy0jqN5Nci6SuEEGjiKGDPsgIUbSSq6FoUw5JIJCWMrVu3cs8993DfffdRvXp1Dh06xNq1a7Hb7TfeuQCoVasWx44dM54fP36csLCwHJeXRvxaJPN75+NeaxWtKeBW6UKoQGPM4pl8HUMikfgHhw4d4pdffmHo0KFcuHCBL774gvr16xMdHV1glcFuRN++ffnmm28QQrB582YqVKhAjRo1aNu2LQcPHuTw4cNYrVYWLVpE3759i2SMBY1ygze/aD6ZQkIIgdVqzdN+2bWxcigz0dRZ6FO9VTA75qBQOu+uJBKJb9A0jQULFjBz5kxmzpxJhw4dCu3YQ4YMISYmhnPnzlG9enVee+01bDYbAI8//jhCCJ588kmio6MJDg5mzpw5tGnTBoCoqCjGjx+Pw+FgxIgRvPRSDpH9RYdP/L9SJHMpkjdqYyVIA1KBaiiYfDdYiURS6rh48SLjxo0jKCiIGTNmUKFChXy/ZnR0NOPGjcPhcDBq1CgmTpzosf69995j/vz5ANjtdv744w/Onj1L5cqVqVOnDiEhIZhMJsxmM0lJSfkeTxEiRdIXZGRkeLWdbIIskUh8SVxcHBMmTOD555/nwQcf9Mn1xOFw0LBhQ1avXk2tWrVo27YtCxcuzDE14+eff2b69OmsW7cOgDp16pCUlERoaKkINPTJBdrvU0Bu1AlEiqNEIvElNpuNt99+m/j4eH788Ufq1q3rs9dOTEzMVf7iwoULGTJkiM+OXxrx68CdG+FyrWqaXohc1lqVSCT54fDhw9xzzz0EBQWxdu1anwok5JzXmB1XrlwhOjqaQYMGGcsUReHuu++mdevWzJo1y6djK6n4vSWZHdJ6lEgkvkQIwYIFC/jkk0+YMWMGHTt2LLDjZCWna9fPP//Mv/71LypXrmwsi4uLIywsjDNnzhAZGUnjxo3p3LlzgYy1pOD3IunubpXiKJFIfMljjz1GlSpV2L59O9WrVycmJoaKFSsW2PFyk7+4aNGia1ytrm2rVavGgAEDSExM9HuRlO5WJ9K1KpFIfE2/fv346aefqFSpEhcvXqRr16488MAD/PNPDg3Z84m3+YuXLl1iw4YN9OvXz1iWnp5Oamqq8fjXX3/NsTuIP+H3liRgiCNI61EikeQfm83GO++8Q2xsLGvXruXWW28F9JzI/fv3ExQUVCDHNZvNzJgxgx49ehj5i+Hh4Xz22WeAnvsIsGzZMu6++27Kli1r7Hv69GkGDBgA6KkhDz74ID179iyQcZYk/DoFRAhB9+7dueWWW4iIiKB9+/bUq1cPVZUGtkQiyRtHjhxhzJgx3HnnnUyaNImAgIAb7yQpCHxi7fi1GiiKwi+//MKjjz7KhQsXePnll2nfvj0PPPAAH3zwAfHx8Vy9erWohymRSEoAQggjpWLKlCm8/vrrPhHIGzU3jomJoUKFCrRs2ZKWLVvy+uuve72v5Mb4tSWZHZqmcfDgQWJjY4mPj2f79u0EBQVxxx13EBERQUREBBkZGfz666+MGDGiqIcrkUiKAZcuXeKZZ55BVVVmzpzps+Acb4oDxMTE8L///Y8VK1bket9SjiwmUBCoqkqjRo1o1KgRI0eORAhBSkoK8fHxbNiwgf/7v//j1KlTtG/fHkVRaNeuHY0aNcJkkiXoJBJ/JD4+nmeffZbnnnuORx55xKcxDbktDuCrfSWZ+LW71RsURaFKlSo0a9aMmJgYRowYwenTp5kyZQpXr15l6tSpdOjQgYEDB/Luu++yceNG0tPTi3rYEomkgLHb7UyZMoXJkyezdOlShg4d6vOgP2+LA8THx9OiRQt69erFnj17crWv5PpIS9JLwsLCWLJkCXXq1AGgXbt2tGvXDtBdtEeOHCEuLo4ff/yRSZMmoaoqbdu2NQKCbrrpJhk1K5GUYIQQCCFQVZW///6bMWPG0KVLF9atW1dgwTneFAdo1aoVf//9N+XKlSMqKor+/ftz8OBBv2qMXJBIkfQSi8ViCGRWVFWlXr161KtXj0ceeQQhBJcvX2bz5s3ExsYyd+5czpw5Q5MmTQzRbNq0KWazfPslkpLCmTNn6Nu3L0IIzp49y3PPPcfIkSMLNHrVm+IA5cuXNx737t2b//znP5w7d86vGiMXJDJwp5Cw2+3s3LmTuLg4Nm3axN69e6latSp33HEH7du3p23btoSEhMg7PYmkmHL58mWeffZZ0tPTue+++9izZw8JCQlUrlyZpUuXFsgx7XY7DRs2ZO3atdSsWZO2bduyYMECwsPDjW1OnTpF9erVURSFxMRE7rvvPv7++28jcOd6+5ZyZKuskowQghMnThAbG8umTZvYsmULdrudVq1a0a5dOyIiIqhdu7bM2ZRIigEJCQk888wzjB8/nmHDhnnczLp6yxYU2TU3di8OMGPGDD799FPMZjNlypTh/fffNxo3l4DGyAWJFMnSRlpaGlu2bDGE8/jx4zRs2NCwNm+77TYsFktRD1Mi8Rvsdjvvvfce69evZ86cOTRo0KCohyTxHimSpR2Hw8Eff/xhiObOnTupUKGCIZp33HEHlSpV4o8//iAoKMgI9ZZIJPnn6NGjjB49mk6dOjF58mSfzT1GR0czbtw4HA4Ho0aNYuLEiR7r58+fzzvvvANAuXLl+PTTT2nRogWgN0UOCQnBZDJhNptJSkryyZhKKVIk/Q0hBGfOnGHTpk3ExsYSFxfHkSNHEEIwZswYBg0axK233ipdtBJJPhBC8P333zNt2jQ++ugjunbt6rPX9ibBf9OmTTRp0oRKlSqxcuVKJk+eTEJCAqCLZFJSEqGhoT4bUylGFhPwNxRFoXr16gwYMICKFSuybt06nn/+eVq1akVCQgKvvPIKf/31F3Xr1iUiIoJ27drRqlWrAiumLJGUNi5fvsyECROwWq3ExMR49Fr0Bd4k+LvmEwEiIiI4fvy4T8cgyR1SJEsogYGBrFixgpo1awIYd7uapvHnn38SGxvLokWLeOGFFwgKCjJyNiMiIqhataqMopVIspCYmMj48eN5+umnGT58eIF4ZLJL8HdZidnx1Vdf0atXL+O5oijcfffdKIrCmDFjGD16tM/HKPFEimQJxf1u0x1VVWnYsCENGzZkxIgRCCG4cOECmzZtIi4ujs8++4wLFy7QvHlz2rVrR/v27WVZPYlfcunSJcqVK4cQgmnTprFmzRq+++47GjZsWGDHzE2C//r16/nqq6+IjY01lsXFxREWFsaZM2eIjIykcePGft8UuaCRIlnKURSFypUr06dPH/r06QOA1Wplx44dxMbGMnXqVPbv309YWJhR6KB169YEBwdLa1NSqvn++++ZOXMmp06domnTpvz3v//1uXs1K94m+O/cuZNRo0axcuVKqlSpYix3bVutWjUGDBhAYmKiFMkCRgbuSBBCcOTIESOKNikpCVVVadOmjSGcNWrUkKIpKTUIIVi6dCn/+9//ePXVVzGbzWzevJnNmzczduxYBg8eXCDH9aY4wNGjR+nWrRvffPONh8coPT0dTdMICQkhPT2dyMhIXnnlFdkYOWdkdGt+uVEothCCcePGERUVRXBwMHPnzqVVq1ZFNNrCw1VWLyEhwWgZdvr0aRo3bmzMazZr1kyW1ZOUSFJTU5kwYQJXr17l888/L3DrMSs3Kg4watQoli5dyi233AJgpHocOnSIAQMGALrYPvjgg/5WHCC3SJHMD96EYkdFRfHxxx8TFRVFQkIC48aNu+4ke2nGbreza9cuo6ze7t27CQ0NNeY177jjDqOsnhCCS5cu+aynnkTiK7Zs2cK4ceN48sknGTFihE+Cc/Jzs32jfSX5QopkfoiPj2fy5MmsWrUKgLfffhuA//u//zO2GTNmDF27dmXIkCEANGrUiJiYGGrUqFH4Ay5mCCE4efKkka+5ZcsWbDYbt956K7t27WLw4MFMmDBB5mxKigV2u53p06ezatUq5syZQ6NGjXzyuvm52ZZNkQscn4ik317BvOm1Jvux5YyiKNSsWZP777+fjz76iI0bN9K3b19+//13OnfuTGJiIu3bt+fhhx/mo48+YsuWLVit1qIetsQPOX78OP369ePKlSvExMT4TCDBM+/RYrEYeY/uLF++3Og1GRERwcWLF0lOTvZqX0nR47eTSt6EYst+bN7z22+/oaoqO3fuNOrLOhwO9u3bR2xsLF999RU7d+6kfPnyRi/Odu3aUblyZfmeSgoEIQTLli3j3XffZfr06XTv3t3nx/Am7zGnm+3c5kxKiga/FUlvQrFlPzbv6datG926dfNYZjKZCA8PJzw8nDFjxhh9+Fxl9T744APS09Np0aKFERBUv3596aKV5JvU1FSef/550tLSWL9+vUcahS/Jz822vAkvGfitSLZt25aDBw9y+PBhatasyaJFi1iwYIHHNn379mXGjBk88MADJCQkUKFCBTkfmQ8URaFatWr079+f/v37A5CRkcGWLVuIi4tj8uTJ/Pnnn9SpU8ejrF6ZMmWKeOSS4o7NZkMIgcViISkpiXHjxvGf//yHkSNHFuhNV35utq1Wq7wJLwH4rUiazWZmzJhBjx49jFDs8PBwj1Ds3r17ExUVRf369QkODmbOnDlFPOrSR2BgIB07dqRjx45AZlm9uLg4vvvuOyZOnEhgYKBHWb1q1arJO26JB3/++SejRo3iwoULWK1WJk6cSJ8+fQrcK5Gfm+2qVavecF9J0eO30a3Fkfy00CmtuMrqxcfHGzmbFy5coFmzZkb6SePGjWVZPT/nxIkTjBkzhmbNmhEZGUlSUhLx8fFUrVqV+fPnF+ixb5T3KITgySefJDo62rjZbtOmTY77SnyGTAEpTeS3hY4/YbPZjLJ6mzZtYt++fUZZvYiICFq3bk3ZsmWltekHCCFYvnw5U6dO5f3336d79+7yc5e4kCJZmvAmb9MdlzUlU1L0C+Xff//tUVYPMMrqtW7dmvnz59OrVy9at25dxKOV+Iq0tDReeOEFLl68yBdffOHzHospKSncf//9HDlyhDp16vDdd99RqVIlj22OHTvG0KFDOXXqFKqqMnr0aMaNGwfA5MmT+eKLL6hatSoAU6ZMoXfv3j4do+S6yDzJ0kRuczKzttDxZxRFoU6dOjz88MPMnDmThIQE1qxZw4ABA4iNjaVDhw7Mnj2bDz74gE8//ZTt27djt9uLetiSfLBt2zZ69OhBREQES5cuLZAmxFOnTqV79+4cPHiQ7t27M3Xq1Gu2MZvNTJs2jT/++IPNmzfzySefsHfvXmP9M888w44dO9ixY4cUyBKK3wbuFDfy20JHkomiKJQvX56jR4+yb98+EhISqF+/Prt37yY2NpaZM2eye/duqlSpwh133GGU1Stfvrx01RVzHA4HH374IStWrGDBggU0adKkwI61fPlyYmJiABg2bBhdu3Y1YgJc1KhRw4h4DwkJoUmTJpw4ccInVXPuvPNOXnzxRSIjI3n55Ze5fPkyH330Ub5fV5I7pEgWE/LbQkdyLffeey/Dhw83gnpatmxJy5YtefLJJ42yenFxcaxbt46pU6ditVpp1aoV7dq1IyIigjp16siczWKEKzjn9ttvZ8OGDQQGBhbo8U6fPm0IYI0aNThz5sx1tz9y5Ajbt2+nXbt2xrIZM2bwzTff0KZNG6ZNm3aNu/Z6vPbaa7zyyiucOXOG7du389NPP+XtRCT5Qs5JFhPy00JH4hvS09ONnM1Nmzbx999/06BBA8PabNGiRYFfmCXXIoTgp59+YsqUKUybNo3IyEifWfx33XUXp06dumb5W2+9xbBhw7h48aKxrFKlSly4cCHb10lLS6NLly689NJLDBw4ENBFNjQ0FEVRmDRpEsnJycyePTtX4+vSpQtpaWnExMQQEhKSq30lvpmTlJZkMcGbvM3XX3+d8+fP85///MfYxxWkIsk/ZcuWpWvXrnTt2hXQczZdZfXmzJnDzp07KVeunFFSLyIiQpbVK2DS0tKYOHEiKSkprFu3zgiC8RVr1qzJcV316tVJTk6mRo0aJCcnU61atWy3s9lsDBo0iIceesgQSNf+Lh577DGj6bm37Nq1i+TkZEJDQ6VAFiHSkpRIvMS9rF5cXBzx8fGkp6dz2223GeknDRo0kC7afHL27FkGDRrEzTffzPbt23n00Ud5+eWXCz0X9r///S9VqlRh4sSJTJ06lZSUFN59912PbYQQDBs2jMqVK/PBBx94rHMJLMD06dNJSEhg0aJFXh07OTmZHj16sHjxYp5++mkmTJhAjx49fHNi/oNMAZEULN72utuyZQsREREsXryY++67r5BHWbRkZGSQlJRkuGj//PNPbrnlFqOsXuvWrWVZvVzicDh4//33Wbx4Mb179+avv/7ir7/+onHjxixevLjQLPfz588zePBgjh49Su3atVmyZAmVK1fm5MmTjBo1iqioKGJjY+nUqRPNmzc3bo5cqR6PPPIIO3bsMKKvP//8c6/KWl65coXu3bvz+uuvExkZycaNG3nhhReIj48v6FMubUiRlBQc3va6czgcREZGEhQUxIgRI/xOJLOiaRp//fWXUR1o69atWCwWj7J61atXNwpcnzlzxsMt5++cPHmSMWPG0KJFC95++21jDtgVaFWzZs0iHqGkBCHnJCUFh3uvO8DodZdVJD/++GMGDRrEli1bimKYxQ5VVWnQoAENGjTg0UcfRQjBxYsXjbJ6X3zxBSkpKdStW5cDBw7QvXt3pkyZ4vdl9YQQrFixgjfffJP//e9/3H333R4Wo6t/aV7xpjAAQJ06dQgJCcFkMnnM+Xu7v6T0ISdPJNnibVPqZcuW8fjjjxf28EoMiqJQqVIlevfuzZQpU1i3bh3PPfcce/fupWvXrpw6dYoOHTrQv39/pk6dSkxMDGlpadnmzZZW0tPTGTduHPPnz2fdunX06NHD5y5VbwoDuFi/fj07duzwCIrLzSUyExMAABsmSURBVP6S0oW0JCXZ4k1xg/Hjx/POO+/4vRWUG44fP85vv/3Gli1bqFChApBZVi8uLo4VK1YwefJkhBC0adOGdu3a0aFDB2rWrFkqo2h37NjBk08+yciRIxk7dmyBBT15UxigIPeXlFzknKQkW7ypJVu3bl1DTM+dO0dwcDCzZs0yekVK8oYQgtTUVBISEoyAoOTkZBo1amTMazZv3pyAgICiHmqecTgcfPLJJ/zwww/Mnj2bZs2aFejxKlas6FXOY926dalUqRKKojBmzBhGjx6dq/0lxQo5JykpOLzpk3f48GHj8fDhw+nTp48USB/gKqsXGRlJZGQkoIuKq6zeZ599xq5du6hcubJHWb0KFSqUCGszOTmZxx9/nPDwcDZu3EhQUJBPXvd6hQG8JS4ujrCwMM6cOUNkZCSNGzemc+fOPhmfpGQiRVKSLd4UN5AUHiaTiRYtWtCiRQueeOIJhBAkJycTFxdHTEwM7777LlevXvUoq1e3bt1ilbMphCAqKorXX3+dd999l549e/pU1H1RGMBVCrJatWoMGDCAxMREOnfu7PX+ktKHdLdKShTe5G7GxMQwfvx4bDYboaGhbNiwoQhGWvhcuXKFxMREo9DBkSNHqF+/viGaLVu2LPSyesnJySxbtozbb7+dBQsWcObMGb766qtCFxlvCgOkp6ejaRohISGkp6cTGRnJK6+8Qs+ePb3aX1LskHmSEv/Cm9zNixcv0qFDB6Kjo6lduzZnzpzx27t+V1m9uLg44uLi2LFjxzVl9apUqVKgLtqUlBQ++eQTZs2aRXBwMPXr16dDhw507ty5UCvIeFMY4NChQwwYMADQayk/+OCDvPTSS9fdX1KskSIp8S+8CSaaOXMmJ0+e5M033yySMRZnhBCcO3fOo6xeamqqR1m9hg0bcvz4cQ4cOMBdd92Vr+M5HA5mzpzJ0qVLjeCcs2fPGlbu008/7aMzk0iyRQbuSAoGTdMQQqCqarEKBMkudzMhIcFjmwMHDmCz2ejatSupqamMGzeOoUOHFvZQiyWKolC1alX69etHv379AL2s3tatW4mLi+ONN94gKSmJS5cu8e9//5vAwEBat25NcHBwro916tQpHn/8cRo3buwRnFO1alX69u3r0/OSSAqS4jOrLylyNE3Dbrejqiomk+kagfz999+JiYkxRLSw8SZ30263s3XrVn755RdWrVrFG2+8wYEDBwpriCWOwMBAOnTowJgxY4wuKBs2bKBTp0788MMP9OjRg27duvHCCy+wbNkyTp06dd3P3hWc069fP8aPH8+HH37os+hVd1JSUoiMjKRBgwZERkZmm46xf/9+o4doy5YtKV++vFGEfPLkydSsWdNYFxUV5fMxSkoH0pKUGHzzzTcsXryYtLQ0WrZsySuvvOLRmmjy5MksX76cefPm8dBDD6FpGkChRVB605i6Vq1ahIaGUrZsWcqWLUvnzp35/fffadiwYaGMsaSiaRoDBgxg0KBBANx+++0MHz7cKKu3efNmYmNj+fLLLzl//jzh4eG0a9eO9u3b07RpU0wmE1euXOHFF1/kxIkTrFmzpkBr0roq4LgCaaZOnXpNcn+jRo3YsWMHoLt+a9asacw5AjzzzDNMmDChwMYoKSUIIa73J/ETLly4IKpXry7S0tLEzp07xZQpU0Rqaqqx/urVq6Jz587i2LFj2e4fHR0t5s6dK1JSUgpsjDabTdStW1ccOnRIZGRkiNtuu03s3r3bY5u9e/eKbt26CZvNJtLT00V4eLjYtWtXgY3JH7FarWLLli1i+vTp4t///rdo1qyZ6Nixo6hbt6744IMPhMPhKPAxNGzYUJw8eVIIIcTJkydFw4YNr7v9qlWrRIcOHYznr776qnjvvfcKdIySIudG+ubVn7QkJQBcunSJ2rVr88knn/D888/TvHlzj/VHjx4lJSWFm266yVi2Z88ezp49S5cuXThw4ACHDx+md+/egH7zpWkaiqL4zNL0JnezSZMm9OzZk9tuuw1VVRk1alSBV3PxNwICAmjTpg1t2rRh/PjxCCE4ePAgBw8e5J577imUMZw+fdpoO1WjRg3OnDlz3e0XLVrEkCFDPJbNmDGDb775hjZt2jBt2jRZsFySLTK6VUJGRoaRP/f4448jhOCNN97wSJ347rvv+Oqrr1i1ahVCCOLj43nppZcwm82YTCbq1KlDeHg4Tz31FFarFYvFcs1xXO5ZRVGKVUCQpHhyvQo6w4YN87pMnNVqJSwsjD179hgu4NOnTxMaGoqiKEyaNInk5GRmz55dMCciKSp8cpGRgTsSvvvuO2bNmmU0e71y5YpxZ+66iUpISKB169YA7Nu3j++//57u3buzevVqxo4dy4YNG7j11ls5d+4cb7/9Ni1btuShhx7yaBSrqmq2EbN2u53U1NRCOlvfEx0dTaNGjahfv3623SEuXbrEvffeS4sWLQgPD2fOnDlFMMqSx5o1a9i9e/c1f/369TMq4AA3rICzcuVKWrVq5TFHWr16dUwmE6qq8thjj5GYmFjg5yMpmUiRlBAaGsqaNWvo2rUr8+bNY+zYsTRq1AjAaA68c+dOOnXqBMChQ4cIDAw0QvmFELRv3x6TycS7776L3W5nx44ddO/enXXr1gGwYMEChgwZwhdffMGqVavIyMgwBPjUqVN8+OGHxmulpKRck9pRXHE4HDzxxBOsXLmSvXv3snDhQvbu3euxzSeffELTpk2N6ODnnnsOq9VaRCMuHfTt25evv/4agK+//tpIacmOhQsXXuNqdQkswLJly6RLXpIjUiQl9OrVi++++47ExETmzZtHhw4dPDpMnD59mtOnT3PHHXcAcOuttxIfH0/t2rUBOHLkiNFEeN++fSxcuJC+ffvy7bffkpiYyNGjR4mKimL//v2oqsorr7zCrFmzUBSFAwcO8PLLLxMXFwfoorxixQqeeuopAGw2Gw6HA8h01xYn3JtTWywWozm1O4qikJqaihCCtLQ0KleujNkswwHyw8SJE1m9ejUNGjRg9erVRnnCkydPGvPioJfqW716NQMHDvTY3zXvftttt7F+/XqmT59eqOOXlBzkL1Vi5D2aTCYqVKiAEMLDJbpnzx6qVatGlSpVALj55psJCQnhhx9+oH379nzxxRcMHTqUBg0asGvXLpYuXcrhw4eJj4/HbrdTpkwZDh48yKuvvkq/fv2oXLkyUVFRnD17FoAtW7Zw+fJl+vfvz6uvvsrJkydp164dgIdYJyQkEBkZydixY3nvvfcK8R3KGW8KHDz55JP07duXsLAwUlNTWbx4cbEqPF4SqVKlCmvXrr1meVhYmEfOY3BwMOfPn79mu3nz5hXo+CSlB/lLlRjFA1xknTNMTEzkwIEDpKam4nA4KFu2LC+88AKrVq3irbfeokWLFlStWpVmzZqxdetWbDYbDRs2ZNiwYYwcOZJ//vmHc+fO0bdvXzRNo3HjxsacZ8OGDalUqRLr169nyZIlNGjQgHXr1vHXX3/Rs2dP7r77bqMlV/v27XnwwQe5fPkykH1xgcImuzFkff9WrVpFy5YtOXnypNFk2HUOEliyZAnh4eGoqkpSUlKO2+U09+tNYQGJJK9IkZTckD59+hAZGUlcXJwhph07dmTx4sV8++23fPnll/Tr149atWoxYsQIWrVqRadOnQyrcNu2bZw7d85IB9m8eTNly5alatWq7Nmzh9TUVOrVq0dAQIDRyeK1114jOjqaoKAgdu3aZYzl0KFDRo/F4hAh602Bgzlz5jBw4EAURaF+/frUrVuXffv2FfZQiy3NmjXjhx9+uG7fxuvN/boKCxw8eJDu3btnGzwlkeQV6W6V3JDmzZvz1VdfeSxzd9EGBwcb9T2feOIJRo8eTUJCAqmpqVStWpWDBw9Sr1493njjDdLS0jh69KgRSLFjxw7CwsIM9+OePXto0qQJbdu2xWq10qxZM44cOWIc99ChQ8bcaHHAm+bUtWvXZu3atXTq1InTp0+zf/9+6tWrV0QjLn40adLkhtu4z/0Cxtxv06ZNWb58OTExMQAMGzaMrl27XlN9RyLJK1IkJTfEXRBd5DSnJoQgICCAjh07GsvWr1/P+PHjqV69OlFRUTz11FN06NAB0Of0KlSoYGy7ZcsWI9Lw+PHjCCEoV64cANu2baNcuXJGwFBxwJsCB5MmTWL48OE0b94cIQTvvPMOoaGhRTzyksX15n5zW1hAIskNUiQlNyQ3QSYuF6gr+OfSpUucPn2a3r17U7VqVXr27OmxfZcuXVi/fj39+/fn/fffZ8OGDfTq1QvQL4wZGRnUqVMH0MXWJaBZg4uKkt69e3tEVIIuji7CwsL49ddfC3tYxYrrFQa4XvqGC2/mfiWSgkCKpKRAcF3AKlSowNatWwF9Xilrmbp27dqxcuVKQC8q8PDDD3vkY168eJFatWoBEBsba/Q4lBfIGzNixAhWrFhBtWrV2L179zXrhRCMGzeOqKgogoODmTt3Lq1atSqQsaxZsyZf+19v7tdVWKBGjRo3LCwgkeQWGbgjKTRcFU7c0TTNyH80m80MGTLEEMVBgwbxzDPPGPNQ+/bt41//+lfhDroEM3z4cKKjo3Ncv3LlSqPm6qxZsxg7dmwhji53uM/9Wq1WFi1aZBSzyE1hAYkkt0iRlBQprlJ1LtwLBpQrV47mzZtjt9uJj49H0zSaNm1aFMMskXTu3JnKlSvnuH758uUMHToURVGIiIjg4sWLHpVoCotly5ZRq1Yt4uPjueeee+jRowfgWRjAfe63SZMmDB48mPDwcCDnwgISiS+Q7lZJsSK7+c+kpCQGDBhAly5dsFgsxWo+siSTXTDMiRMnjCCYwmLAgAEefR5dZC0MkN3cL+RcWEAi8QVSJCXFno4dO3L27FmuXr0KyPlIXyGDYSSSGyPdrZJijysFJSgoqKiHUqrwphCCROLvSJGUFHuya68lyT99+/blm2++QQjB5s2bqVChQqG7WiWS4s6Nmi5LJJISiqIoC4GuQChwGngVCAAQQnym6HceM4CewBXgUSFEzsVTJRI/RIqkRCKRSCQ5IN2tEolEIpHkgBRJiUQikUhyQIqkRCKRSCQ5IEVSIpFIJJIckCIpkUgkEkkOSJGUSCQSiSQHpEhKJBKJRJIDUiQlEolEIskBKZISiUQikeSAFEmJRCKRSHJAiqREIpFIJDkgRVIikUgkkhyQIimRSCQSSQ5IkZRIJBKJJAekSEokEolEkgNSJCUSiUQiyQEpkhKJRCKR5IAUSYlEIpFIckCKpEQikUgkOSBFUiKRSCSSHJAiKZFIJP/f3p3H2lGWcRz//U5ZGhZFqAtCpYiKoSDQAFE2ETFBFAFDAkggGKKRBINRIEYDKlGjEBNjIhJiSCVsLiwRwxpAFitrLbQIEqCgBBQRAxRU0vbxj3nvvXNm5j3L7WnvmdvvJ2nvOe827zNz3/ucOefeGSCDJAkAQAZJEgCADJIkAAAZJEkAADJIkgAAZJAkAQDIIEkCAJBBkgQAIIMkCQBABkkSAIAMkiQAABkkSQAAMkiSAABkkCQBAMggSQIAkEGSBAAggyQJAEAGSRIAgAySJAAAGSRJAAAySJIAAGSQJAEAyCBJAgCQQZIEACCDJAkAQAZJEgCADJIkAAAZJEkAADJIkgAAZJAkAQDIIEkCAJBBkgQAIIMkCQBABkkSAIAMkiQAABkkSQAAMkiSAABkkCQBAMggSQIAkEGSBAAggyQJAEAGSRIAgIxNelbO+UAo3ugqsyTLje1d+1+1x1Nl9dLJkqiP13usIcdvLGt6NJ3tNmmKZBTj9moftcqeY/SodFfnqI/VuI2oDdI4z+pBbxrW9fruOVX7RaVhpr6xTrVY83PKtOkat199pV3P4xW1wub9MMQ2s/tQPWOcqHOf+u6i/H6v94t6UW6/Nu2TySeDzK20rZ5zGyzOpmPZPH6f/dG0/3u2HXB+TfEP80OnVB4NZcNtuyirzbxfX1e231VflC5b+ubNEXF4w0hD6ZkkI97QWzb7siRpTprBHFlzojgB7aSyTqmuWtaJertOqZ27ypT6TJVN1as2hqtl0VBW2YZUfK93KuO6vP1Su/oYTe1TWbhU36uv6n3L7V2vq7Vz9+PJdpOPo6FdSnATbVwv61hyrSy6Hhd1IXcayhra1beVvnYy41badcplHdXa1bcVXdvIjWtHwz6Jrj69+g67rfI2Jr8Ou61Owxi95lTebwO0U6fPfHvEqoZ9qIZtqTynXAydprhU218qj9sw98b2DXObaqepugH6qqNaXF39attSrUzlWCfrercr/dCp7a+uuvJ4aYyo9C3GVXffUlnUxphqF+VtTpZNtZvaVno5lW3X/UMvSj/cose4UdpHk487ayRJ22y+cp5GgLdbAQDIIEkCAJBBkgQAIIMkCQBABkkSAIAMkiQAABkkSQAAMkiSAABkkCQBAMggSQIAkEGSBAAggyQJAEAGSRIAgAySJAAAGSRJAAAySJIAAGSQJAEAyCBJAgCQQZIEACCDJAkAQIYjIl9p3yRp3oabzgYzT9JLMz2J9YC42oW42oW42mVuROy+roP0TJKzle0HI2KfmZ7HqBFXuxBXuxBXu4wqLt5uBQAggyQJAEDGxpokL57pCawnxNUuxNUuxNUuI4lro/xMEgCAQWysZ5IAAPQ165Kk7cNt/8X2k7a/3lB/ou1H0r8ltvcs1T1je7ntZbYf3LAz722AuA6x/Uqa+zLb5w7adyYNENdZpZhW2F5je9tUN5bHy/Yltl+0vSJTb9s/STE/YntRqW6cj1W/uNq6tvrF1da11S+u1q0tSbI93/Ydth+z/ajtMxrajG6NRcSs+SdpjqSnJL1X0maSHpa0W6XN/pLelh5/UtJ9pbpnJM2b6TimGdchkn43nb7jHFel/ZGSbm/B8TpY0iJJKzL1R0i6UZIlfXjie3Ccj9WAcbVubQ0YV+vW1iBxVdq2Ym2luW0vaVF6vLWkJxp+Ho5sjc22M8n9JD0ZEU9HxJuSrpJ0VLlBRCyJiH+np/dK2nEDz3E6+sa1nvqub8PO7QRJV26Qma2DiLhL0ss9mhwl6dIo3CtpG9vba7yPVd+4Wrq2BjleOa0+XhWtWFuSFBEvRMTS9Pg1SY9J2qHSbGRrbLYlyR0k/a30/DnVd17ZqSpebUwISbfYfsj2F9fD/KZr0Lg+Yvth2zfaXjhk35kw8NxsbyHpcElXl4rH9Xj1k4t7nI/VsNqytgbVtrU1sDavLdsLJO0t6b5K1cjW2CbrOskx44ayxl/ftf0xFQv5wFLxARHxvO13SLrV9uPp1dhMGySupZJ2iohVto+QdJ2k9w/Yd6YMM7cjJf0hIsqvjMf1ePWTi3ucj9XAWra2BtHGtTWMVq4t21upSOxfiYhXq9UNXaa1xmbbmeRzkuaXnu8o6flqI9sfkvRzSUdFxL8myiPi+fT1RUnXqjg1Hwd944qIVyNiVXp8g6RNbc8bpO8MGmZux6vydtAYH69+cnGP87EaSAvXVl8tXVvDaN3asr2pigR5eURc09BkdGtspj+EHeU/FWfGT0vaWVMfyi6stHmPpCcl7V8p31LS1qXHSyQdPtMxDRHXuzT1d6/7SfqrildNffuOc1yp3VtVfLayZRuOV5rTAuV/EeRT6v6lgvuH2R9jHFfr1taAcbVubQ0SV6pv49qypEsl/bhHm5GtsVn1dmtErLZ9uqSbVfwW0yUR8ajtL6X6iySdK2k7SRfalqTVUVwE952Srk1lm0i6IiJumoEwagaM61hJp9leLek/ko6P4ruise+MBFIxYFySdIykWyLi9VL3sT1etq9U8RuR82w/J+lbkjaVJmO6QcVv3z0p6Q1Jn091Y3uspIHiat3akgaKq3VrSxooLqllays5QNJJkpbbXpbKvqHiRdrI1xhX3AEAIGO2fSYJAMDIkCQBAMggSQIAkEGSBAAggyQJAEAGSRKtZfsY22H7gzM4h3fb/s2Ixjra9m6l5+fZPmwE41bvYrFsFOMCGwP+BAStZftXKu4IcFtEfHtEY24SEatHMdY0tr1Yxd0mRpJ0S+MeIunMiPh0jzZW8fNgbdPzHv3mRMSaUc4XGCecSaKV0nUbD1BxjdDjS+WH2L7L9rW2/2z7ItudVLfK9o9sL7V9m+23p/Lf2/6+7TslnWH747b/lO6nd4ntzW3vm+5LN9f2luk+drvbXuB0vz7bp9i+zvb1tlfaPt32V9NY93rqXn1fsP1AumD21ba3sL2/pM9IuiCd6e1ie7HtY1Of2pxS+TO2v5NiWj7MWXWa+2O2L1RxfdKDKs/n277Axb0Gl9s+rrSP77B9haTl63QggTFHkkRbHS3ppoh4QtLLLt1UVcWlw74maQ9Ju0j6bCrfUtLSiFgk6U4VVyCZsE1EfFTSTyUtlnRcROyh4oojp0XEA5J+K+m7ks6XdFlENN3MdndJn0tz+J6kNyJib0l/lHRyanNNROwbEXuquM3PqRGxJI1/VkTsFRFPTQxoe27TnErbfCnF9DNJZ2b210GVt1t3SeW7qril0N6Snq0830fSXpL2lHSYigS+fWkffzMidhMwi5Ek0VYnqLgXnNLXE0p190dxv7g1Ki7cPHE3irWSfpkeX6buu1RMlO8qaWVKvpL0CxU3r5Wk8yR9QkXyOD8zrzsi4rWI+KekVyRdn8qXq7iOpiTtbvtu28slnShpYX2YLr3mJEkTF3h+qLSNqrtT8t2rkoSfjeJ+e2p4fqCkKyNiTUT8Q8ULi31T3f0RsbLPvIHWm1XXbsXGwfZ2kg5VkWxCxTUYw/bZqUn1g/bcB+/l8olrVzbdSmfCtpK2UnH9y7mlPmX/Kz1eW3q+VlPrbbGkoyPiYdunqLi+Zi+95lTe5hoNv6arMZSf99puU+zArMOZJNroWBVvCe4UEQsiYr6klZo6M9zP9s7ps8jjJN2Tyjupr1S8JXqP6h6XtMD2+9Lzk1ScQUnSxZLOkXS5pB+uw/y3lvSCi9v9nFgqfy3VDTOn9ekuScfZnpM+vz1Y0v0bYLvA2CBJoo1OUHGPu7KrVSQ+qfj87weSVqhInhNtX5e00PZDKs5Ez6sOHBH/VXHHgF+nt0PXSrrI9skq7mpxRRp7X9uHTnP+56i4k/qtKhLghKsknZV+QWfiM8PsnIbcZvUzyWP7d9G1kh5RcTuh2yWdHRF/H3K7QKvxJyCYVXr9uYPtVRGx1YafFYC24kwSAIAMziQBAMjgTBIAgAySJAAAGSRJAAAySJIAAGSQJAEAyCBJAgCQ8X+XnVriEidU7QAAAABJRU5ErkJggg==\n", 383 | "text/plain": [ 384 | "
" 385 | ] 386 | }, 387 | "metadata": { 388 | "needs_background": "light" 389 | }, 390 | "output_type": "display_data" 391 | } 392 | ], 393 | "source": [ 394 | "# plot the evolution of the data in the s-x-h space\n", 395 | "from mpl_toolkits.mplot3d import Axes3D\n", 396 | "\n", 397 | "n_grid=30\n", 398 | "x_span=[-1,1]\n", 399 | "fig = plt.figure(figsize=(6,6)) ; ax =Axes3D(fig)\n", 400 | "ss = torch.linspace(s_span[0], s_span[-1], n_grid)\n", 401 | "xx = torch.linspace(x_span[0], x_span[-1], n_grid)\n", 402 | "S, X = torch.meshgrid(ss,xx) ; \n", 403 | "u_traj = traj[0,:,0].repeat(traj.shape[1],1)\n", 404 | "e = torch.abs(y.T - traj[:,:,0])\n", 405 | "color = plt.cm.plasma(e.numpy())\n", 406 | "for i in range(traj.shape[1]):\n", 407 | " tr = ax.scatter(s_span, u_traj[:,i], traj[:,i,0],\n", 408 | " c=color[:,i],alpha=1, cmap=color[:,i], zdir='z')\n", 409 | "norm = mpl.colors.Normalize(e.min(),e.max())\n", 410 | "plt.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap='plasma'),\n", 411 | " label='Approximation Error', orientation='horizontal')\n", 412 | "ax.set_xlabel(r\"$s$ [depth]\"); ax.set_ylabel(r\"$x$\"); ax.set_zlabel(r\"$z(s)$\") ; \n", 413 | "ax.xaxis._axinfo[\"grid\"]['color'] = (1,1,1,0)\n", 414 | "ax.yaxis._axinfo[\"grid\"]['color'] = (1,1,1,0)\n", 415 | "ax.zaxis._axinfo[\"grid\"]['color'] = (1,1,1,0)" 416 | ] 417 | } 418 | ], 419 | "metadata": { 420 | "kernelspec": { 421 | "display_name": "torchdyn", 422 | "language": "python", 423 | "name": "torchdyn" 424 | }, 425 | "language_info": { 426 | "codemirror_mode": { 427 | "name": "ipython", 428 | "version": 3 429 | }, 430 | "file_extension": ".py", 431 | "mimetype": "text/x-python", 432 | "name": "python", 433 | "nbconvert_exporter": "python", 434 | "pygments_lexer": "ipython3", 435 | "version": "3.8.5" 436 | }, 437 | "latex_envs": { 438 | "LaTeX_envs_menu_present": true, 439 | "autoclose": false, 440 | "autocomplete": true, 441 | "bibliofile": "biblio.bib", 442 | "cite_by": "apalike", 443 | "current_citInitial": 1, 444 | "eqLabelWithNumbers": true, 445 | "eqNumInitial": 1, 446 | "hotkeys": { 447 | "equation": "Ctrl-E", 448 | "itemize": "Ctrl-I" 449 | }, 450 | "labels_anchors": false, 451 | "latex_user_defs": false, 452 | "report_style_numbering": false, 453 | "user_envs_cfg": false 454 | }, 455 | "varInspector": { 456 | "cols": { 457 | "lenName": 16, 458 | "lenType": 16, 459 | "lenVar": 40 460 | }, 461 | "kernels_config": { 462 | "python": { 463 | "delete_cmd_postfix": "", 464 | "delete_cmd_prefix": "del ", 465 | "library": "var_list.py", 466 | "varRefreshCmd": "print(var_dic_list())" 467 | }, 468 | "r": { 469 | "delete_cmd_postfix": ") ", 470 | "delete_cmd_prefix": "rm(", 471 | "library": "var_list.r", 472 | "varRefreshCmd": "cat(var_dic_list()) " 473 | } 474 | }, 475 | "types_to_exclude": [ 476 | "module", 477 | "function", 478 | "builtin_function_or_method", 479 | "instance", 480 | "_Feature" 481 | ], 482 | "window_display": false 483 | } 484 | }, 485 | "nbformat": 4, 486 | "nbformat_minor": 4 487 | } 488 | --------------------------------------------------------------------------------