├── src ├── __init__.py ├── nn │ ├── __init__.py │ └── MLP.py ├── Learner │ ├── __init__.py │ ├── Random.py │ ├── DQN.py │ └── AWAC.py └── utils │ ├── __init__.py │ ├── memory.py │ └── train_utils.py ├── dqn_agent.pt ├── images ├── no-dip.png ├── offline-awac.png ├── online-awac.png └── 40dqn-results.png ├── LICENSE ├── .gitignore ├── README.md └── AWAC exmample.ipynb /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/nn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/Learner/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dqn_agent.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Junyoungpark/Pytorch-AWAC/HEAD/dqn_agent.pt -------------------------------------------------------------------------------- /images/no-dip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Junyoungpark/Pytorch-AWAC/HEAD/images/no-dip.png -------------------------------------------------------------------------------- /images/offline-awac.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Junyoungpark/Pytorch-AWAC/HEAD/images/offline-awac.png -------------------------------------------------------------------------------- /images/online-awac.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Junyoungpark/Pytorch-AWAC/HEAD/images/online-awac.png -------------------------------------------------------------------------------- /images/40dqn-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Junyoungpark/Pytorch-AWAC/HEAD/images/40dqn-results.png -------------------------------------------------------------------------------- /src/Learner/Random.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class DiscreteRandomAgent: 5 | 6 | def __init__(self, num_actions: int): 7 | super(DiscreteRandomAgent, self).__init__() 8 | self.num_actions = num_actions 9 | 10 | def get_action(self, state=None): 11 | return np.random.randint(low=0, high=self.num_actions) 12 | -------------------------------------------------------------------------------- /src/utils/memory.py: -------------------------------------------------------------------------------- 1 | from random import sample 2 | 3 | 4 | class ReplayMemory: 5 | def __init__(self, max_size): 6 | # deque object that we've used for 'episodic_memory' is not suitable for random sampling 7 | # here, we instead use a fix-size array to implement 'buffer' 8 | self.buffer = [None] * max_size 9 | self.max_size = max_size 10 | self.index = 0 11 | self.size = 0 12 | 13 | def push(self, obj): 14 | self.buffer[self.index] = obj 15 | self.size = min(self.size + 1, self.max_size) 16 | self.index = (self.index + 1) % self.max_size 17 | 18 | def sample(self, batch_size): 19 | indices = sample(range(self.size), batch_size) 20 | return [self.buffer[index] for index in indices] 21 | 22 | def __len__(self): 23 | return self.size 24 | 25 | def reset(self): 26 | self.buffer = [None] * self.max_size 27 | self.index = 0 28 | self.size = 0 29 | -------------------------------------------------------------------------------- /src/utils/train_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def prepare_training_inputs(sampled_exps, device='cpu'): 5 | states = [] 6 | actions = [] 7 | rewards = [] 8 | next_states = [] 9 | dones = [] 10 | for sampled_exp in sampled_exps: 11 | states.append(sampled_exp[0]) 12 | actions.append(sampled_exp[1]) 13 | rewards.append(sampled_exp[2]) 14 | next_states.append(sampled_exp[3]) 15 | dones.append(sampled_exp[4]) 16 | 17 | states = torch.cat(states, dim=0).float().to(device) 18 | actions = torch.cat(actions, dim=0).to(device) 19 | rewards = torch.cat(rewards, dim=0).float().to(device) 20 | next_states = torch.cat(next_states, dim=0).float().to(device) 21 | dones = torch.cat(dones, dim=0).float().to(device) 22 | return states, actions, rewards, next_states, dones 23 | 24 | 25 | def soft_update(net, net_target, tau): 26 | for param_target, param in zip(net_target.parameters(), net.parameters()): 27 | param_target.data.copy_(param_target.data * (1.0 - tau) + param.data * tau) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Junyoung Park 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/nn/MLP.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class MLP(nn.Module): 5 | 6 | def __init__(self, 7 | input_dim: int, 8 | output_dim: int, 9 | num_neurons: list = [64, 32], 10 | hidden_act: str = 'ReLU', 11 | out_act: str = 'Identity'): 12 | super(MLP, self).__init__() 13 | 14 | self.input_dim = input_dim 15 | self.output_dim = output_dim 16 | self.num_neurons = num_neurons 17 | self.hidden_act = getattr(nn, hidden_act)() 18 | self.out_act = getattr(nn, out_act)() 19 | 20 | input_dims = [input_dim] + num_neurons 21 | output_dims = num_neurons + [output_dim] 22 | 23 | self.layers = nn.ModuleList() 24 | for i, (in_dim, out_dim) in enumerate(zip(input_dims, output_dims)): 25 | is_last = True if i == len(input_dims) - 1 else False 26 | self.layers.append(nn.Linear(in_dim, out_dim)) 27 | if is_last: 28 | self.layers.append(self.out_act) 29 | else: 30 | self.layers.append(self.hidden_act) 31 | 32 | def forward(self, xs): 33 | for layer in self.layers: 34 | xs = layer(xs) 35 | return xs 36 | -------------------------------------------------------------------------------- /src/Learner/DQN.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | class DQN(nn.Module): 7 | 8 | def __init__(self, 9 | state_dim: int, 10 | action_dim: int, 11 | qnet: nn.Module, 12 | qnet_target: nn.Module, 13 | lr: float, 14 | gamma: float, 15 | epsilon: float): 16 | """ 17 | :param state_dim: input state dimension 18 | :param action_dim: action dimension 19 | :param qnet: main q network 20 | :param qnet_target: target q network 21 | :param lr: learning rate 22 | :param gamma: discount factor of MDP 23 | :param epsilon: E-greedy factor 24 | """ 25 | 26 | super(DQN, self).__init__() 27 | self.state_dim = state_dim 28 | self.action_dim = action_dim 29 | self.qnet = qnet 30 | self.lr = lr 31 | self.gamma = gamma 32 | self.opt = torch.optim.Adam(params=self.qnet.parameters(), lr=lr) 33 | self.register_buffer('epsilon', torch.ones(1,) * epsilon) 34 | 35 | # target network related 36 | qnet_target.load_state_dict(qnet.state_dict()) 37 | self.qnet_target = qnet_target 38 | self.criteria = nn.SmoothL1Loss() 39 | 40 | def get_action(self, state): 41 | qs = self.qnet(state) 42 | prob = np.random.uniform(0.0, 1.0, 1) 43 | if torch.from_numpy(prob).float() <= self.epsilon: # random 44 | action = np.random.choice(range(self.action_dim)) 45 | else: # greedy 46 | action = qs.argmax(dim=-1) 47 | return int(action) 48 | 49 | def update(self, state, action, reward, next_state, done): 50 | s, a, r, ns = state, action, reward, next_state 51 | 52 | # compute Q-Learning target with 'target network' 53 | with torch.no_grad(): 54 | q_max, _ = self.qnet_target(ns).max(dim=-1, keepdims=True) 55 | q_target = r + self.gamma * q_max * (1 - done) 56 | 57 | q_val = self.qnet(s).gather(1, a) 58 | loss = self.criteria(q_val, q_target) 59 | 60 | self.opt.zero_grad() 61 | loss.backward() 62 | self.opt.step() 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # pycharm 2 | .idea/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | pip-wheel-metadata/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | -------------------------------------------------------------------------------- /src/Learner/AWAC.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.distributions import Categorical 5 | 6 | from src.utils.train_utils import soft_update 7 | 8 | 9 | class AWAC(nn.Module): 10 | 11 | def __init__(self, 12 | critic: nn.Module, # Q(s,a) 13 | critic_target: nn.Module, 14 | actor: nn.Module, # pi(a|s) 15 | lam: float = 0.3, # Lagrangian parameter 16 | tau: float = 5 * 1e-3, 17 | gamma: float = 0.9, 18 | num_action_samples: int = 1, 19 | critic_lr: float = 3 * 1e-4, 20 | actor_lr: float = 3 * 1e-4, 21 | use_adv: bool = False): 22 | super(AWAC, self).__init__() 23 | 24 | self.critic = critic 25 | self.critic_target = critic_target 26 | self.critic_target.load_state_dict(critic.state_dict()) 27 | self.critic_opt = torch.optim.Adam(params=self.critic.parameters(), lr=critic_lr) 28 | 29 | self.actor = actor 30 | self.actor_opt = torch.optim.Adam(params=self.actor.parameters(), lr=actor_lr) 31 | 32 | assert lam > 0, "Lagrangian parameter 'lam' requires to be strictly larger than 0.0" 33 | self.lam = lam 34 | self.tau = tau 35 | self.gamma = gamma 36 | self.num_action_samples = num_action_samples 37 | self.use_adv = use_adv 38 | 39 | def get_action(self, state, num_samples: int = 1): 40 | logits = self.actor(state) 41 | dist = Categorical(logits=logits) 42 | return dist.sample(sample_shape=[num_samples]).T 43 | 44 | def update_critic(self, state, action, reward, next_states, dones): 45 | with torch.no_grad(): 46 | qs = self.critic_target(next_states) # [minibatch size x #.actions] 47 | sampled_as = self.get_action(next_states, self.num_action_samples) # [ minibatch size x #. action samples] 48 | mean_qsa = qs.gather(1, sampled_as).mean(dim=-1, keepdims=True) # [minibatch size x 1] 49 | q_target = reward + self.gamma * mean_qsa * (1 - dones) 50 | 51 | q_val = self.critic(state).gather(1, action) 52 | loss = F.mse_loss(q_val, q_target) 53 | 54 | self.critic_opt.zero_grad() 55 | loss.backward() 56 | self.critic_opt.step() 57 | 58 | # target network update 59 | soft_update(self.critic, self.critic_target, self.tau) 60 | 61 | return loss 62 | 63 | def update_actor(self, state, action): 64 | logits = self.actor(state) 65 | log_prob = Categorical(logits=logits).log_prob(action.squeeze()).view(-1, 1) 66 | 67 | with torch.no_grad(): 68 | if self.use_adv: 69 | qs = self.critic_target(state) # [#. samples x # actions] 70 | action_probs = F.softmax(logits, dim=-1) 71 | vs = (qs * action_probs).sum(dim=-1, keepdims=True) 72 | qas = qs.gather(1, action) 73 | adv = qas - vs 74 | else: 75 | adv = self.critic_target(state).gather(1, action) 76 | 77 | weight_term = torch.exp(1.0 / self.lam * adv) 78 | 79 | loss = (log_prob * weight_term).mean() * -1 80 | 81 | self.actor_opt.zero_grad() 82 | loss.backward() 83 | self.actor_opt.step() 84 | return loss 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A PyTorch implementation of Advantage weighted Actor-Critic (AWAC) 2 | 3 | ## Requirements 4 | 5 | - Pytorch 6 | - gym 7 | - matplotlib 8 | 9 | ## `AWAC` Class 10 | 11 | `AWAC` class only supports discrete action space (Dec 30th, 2020) 12 | 13 | (Disclaimer) I developed the codes in this repository based on [the original AWAC paper](https://arxiv.org/abs/2006.09359). Hence, the code may contain 14 | not correct implementation of what the actual paper intended. 15 | 16 | ````python 17 | class AWAC(nn.Module): 18 | def __init__(self, 19 | critic: nn.Module, # Q(s,a) 20 | critic_target: nn.Module, 21 | actor: nn.Module, # pi(a|s) 22 | lam: float = 0.3, # Lagrangian parameter 23 | tau: float = 5 * 1e-3, 24 | gamma: float = 0.9, 25 | num_action_samples: int = 1, 26 | critic_lr: float = 3 * 1e-4, 27 | actor_lr: float = 3 * 1e-4, 28 | use_adv: bool = False): 29 | ```` 30 | 31 | - `critic`: State-action value function Q(s,a) 32 | - `critic_target`: the target network of Q(s,a) 33 | - `actor`: An discrete actor. Note that the output of `actor` is logit. 34 | - `lam`: Lagrangian parameter of the AWAC actor loss. Assume to be a strictly positive value. 35 | - `tau`: the Polyak parameter for updating the target network. 36 | - `gamma`: The discount factor of target MDP 37 | - `num_action_samples`: Number of action samples for updating critic. 38 | - `critic_lr`: Learning rate of critic 39 | - `actor_lr`: Learning rate of actor 40 | - `use_adv`: If `True`, use advantage value for updating actor. Else use Q(s,a) for updating actor. 41 | 42 | ## Examples 43 | You can find the running example of `AWAC` on `gym` `cartpole-v1` environment from `AWAC-example.ipynb`. 44 | 45 | ## Experiment Results 46 | 47 | It is confirmed that `AWAC` can learn better policy than its behavior policy in offline mode. 48 | 49 | ### Offline dataset preparation 50 | - Prepare the offline dataset by using a good enough DQN (well trained but with a 40% chance act randomly) 51 | - 50 independent `cartpole-v1` trial were made. 52 | 53 | ![slightly dumb DQN](./images/40dqn-results.png) 54 | 55 | - The values on the x-axis show how long each episode was. (Longer is better) 56 | - The values on the y-axis indicate the frequency of the episode lengths. 57 | 58 | ### Offline training 59 | 60 | - After training 8000 gradient steps with 1024 sized mini-batch, `AWAC` was able to learn 61 | a policy better than the one of `good enough DQN`. 62 | 63 | ![offline AWAC](./images/offline-awac.png) 64 | 65 | - The blue distribution shows the performance distribution of the `good enough DQN`. 66 | - The orange distribution shows the performance distribution of the offline trained `AWAC`. 67 | 68 | ### Online finetuning 69 | 70 | - After `AWAC` trained on 600 episode amount of online training, 71 | `AWAC` shows even better control performance. (offline trained `AWAC` + 600 ep online training) 72 | 73 | ![offline AWAC](./images/online-awac.png) 74 | 75 | - Moreover, the `AWAC` didn't show the 'dip', a phenomenon that indicates sudden performance drop right after online training 76 | 77 | ![offline AWAC](./images/no-dip.png) 78 | 79 | - The blue curve shows the performance of `AWAC` trained on the **offline** dataset only 80 | - The orange curve shows the performance of `AWAC` trained in **online** mode. i.e., 81 | the samples from replay memory would contain some amount of distributional shift. 82 | 83 | - Also confirmed that even we clear the memory right before start to online training, 84 | the `AWAC` still can learn well. 85 | -------------------------------------------------------------------------------- /AWAC exmample.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import gym\n", 10 | "import torch\n", 11 | "\n", 12 | "from src.Learner.AWAC import AWAC\n", 13 | "from src.Learner.DQN import DQN\n", 14 | "from src.Learner.Random import DiscreteRandomAgent\n", 15 | "from src.nn.MLP import MLP\n", 16 | "from src.utils.memory import ReplayMemory\n", 17 | "from src.utils.train_utils import prepare_training_inputs\n", 18 | "\n", 19 | "import matplotlib.pyplot as plt" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "gamma = 0.9\n", 29 | "memory_size = 500000" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "qnet = MLP(4, 2, \n", 39 | " num_neurons=[128,128], \n", 40 | " out_act='ReLU')\n", 41 | "qnet_target = MLP(4, 2, \n", 42 | " num_neurons=[128,128], \n", 43 | " out_act='ReLU')\n", 44 | "pi = MLP(4, 2, num_neurons=[128,64])\n", 45 | "use_adv = True\n", 46 | "\n", 47 | "agent = AWAC(critic=qnet, \n", 48 | " critic_target=qnet_target,\n", 49 | " actor=pi, \n", 50 | " gamma=gamma, \n", 51 | " lam=1.0, \n", 52 | " num_action_samples=10,\n", 53 | " use_adv=use_adv)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 4, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "name": "stderr", 63 | "output_type": "stream", 64 | "text": [ 65 | "C:\\Users\\joon0\\Anaconda3\\envs\\gpu_torch130\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n", 66 | " warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "env = gym.make('CartPole-v1')\n", 72 | "memory = ReplayMemory(memory_size)" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "## Prepare offline data" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 6, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "use_expert = True\n", 89 | "if use_expert: \n", 90 | " qnet = MLP(4, 2, num_neurons=[128])\n", 91 | " qnet_target = MLP(4, 2, num_neurons=[128])\n", 92 | " dqn = DQN(4, 1, qnet=qnet, qnet_target=qnet_target, lr=1e-4, gamma=gamma, epsilon=1.0)\n", 93 | " state_dict = torch.load('dqn_agent.pt')\n", 94 | " \n", 95 | " # make trained agent slightly dumb\n", 96 | " # to simulate the realistic scenario where we don't have 'perfect' policy\n", 97 | " # but good enough policy.\n", 98 | " \n", 99 | " state_dict['epsilon'] = dqn.epsilon * .4 \n", 100 | " dqn.load_state_dict(state_dict) \n", 101 | " offline_agent = dqn\n", 102 | " offline_budget = 50\n", 103 | "else:\n", 104 | " offline_agent = DiscreteRandomAgent(2)\n", 105 | " offline_budget = 300\n", 106 | "\n", 107 | "online_budget = offline_budget" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 7, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "cum_rs = []\n", 117 | "for n_epi in range(offline_budget):\n", 118 | " s = env.reset()\n", 119 | " cum_r = 0\n", 120 | "\n", 121 | " while True:\n", 122 | " s = torch.tensor((s,), dtype=torch.float)\n", 123 | " a = offline_agent.get_action(s)\n", 124 | " ns, r, done, info = env.step(a)\n", 125 | "\n", 126 | " experience = (s,\n", 127 | " torch.tensor(a).view(1, 1),\n", 128 | " torch.tensor(r).view(1, 1),\n", 129 | " torch.tensor(ns).view(1, 4),\n", 130 | " torch.tensor(done).view(1, 1))\n", 131 | " memory.push(experience)\n", 132 | "\n", 133 | " s = ns\n", 134 | " cum_r += 1\n", 135 | " if done:\n", 136 | " cum_rs.append(cum_r)\n", 137 | " break" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 8, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "(array([0.01578947, 0.00614035, 0.00789474, 0.00438596, 0.00526316,\n", 149 | " 0.00087719, 0.00175439, 0. , 0.00087719, 0.00087719]),\n", 150 | " array([ 10. , 32.8, 55.6, 78.4, 101.2, 124. , 146.8, 169.6, 192.4,\n", 151 | " 215.2, 238. ]),\n", 152 | " )" 153 | ] 154 | }, 155 | "execution_count": 8, 156 | "metadata": {}, 157 | "output_type": "execute_result" 158 | }, 159 | { 160 | "data": { 161 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAATgElEQVR4nO3db4xc133e8e9TsgyQNi0daOUIJF2yztooExgqy1BEg7SxU7WkHHjTFy3INpWgCGVpk0FTNH/oGnDbd6qT1qgQlQSFELIAR4SK2MkCZkoLRhuhgGlzbciyGIXxglWtNRlzXQFqC6EmGP/6Yq6gyWh25+4fcU2e7wcY7NxzfufOuRcX8/BeztxJVSFJas+f2+gJSJI2hgEgSY0yACSpUQaAJDXKAJCkRm3e6AmsxF133VU7d+7c6GlI0m3lK1/5yneqamq0/bYKgJ07dzI3N7fR05Ck20qS/zmu3UtAktQoA0CSGmUASFKjegVAkgNJLieZT3JiTH+SPNb1v5Bkz1DfmSTXk7w4Ztwvduu9lOQTa9sUSdJKTAyAJJuAx4GDwG7gcJLdI2UHgenucQQ4OdT3JHBgzHrfD8wA76uqHwN+YxXzlyStUp8zgH3AfFVdqaobwFkGb9zDZoCnauACsDXJPQBV9Rzw6pj1fhh4tKq+29VdX+1GSJJWrk8AbANeGVpe6NpWWjPqPcBPJflSkj9I8hPjipIcSTKXZG5xcbHHdCVJffQJgIxpG72HdJ+aUZuBdwD7gV8BnknylvVU1emq2ltVe6em3vI9BknSKvUJgAVgx9DyduDqKmrGrfcz3WWjLwPfA+7qMR9J0jro803gi8B0kl3At4BDwD8aqZkFjic5C9wHvFZV1yas93eBDwD/Lcl7gC3Ad1Yy+ZXYeeJzb9eqJ3r50Q9u2GtL0lImngFU1U3gOHAeeAl4pqouJTma5GhXdg64AswDTwAfeWN8kqeBLwLvTbKQ5JGu6wzwV7uPh54FHip/nkySbple9wKqqnMM3uSH204NPS/g2BJjDy/RfgP4+d4zlSStK78JLEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY3qFQBJDiS5nGQ+yYkx/UnyWNf/QpI9Q31nklzvfvpx3Lp/OUkl8QfhJekWmhgASTYBjwMHgd3A4SS7R8oOAtPd4whwcqjvSeDAEuveAdwPfHOlE5ckrU2fM4B9wHxVXel+x/csMDNSMwM8VQMXgK1J7gGoqueAV5dY9yeBXwX8MXhJusX6BMA24JWh5YWubaU1f0aSDwHfqqqvTag7kmQuydzi4mKP6UqS+ugTABnTNvov9j41bxYnPwh8DPj4pBevqtNVtbeq9k5NTU0qlyT11CcAFoAdQ8vbgaurqBn2bmAX8LUkL3f1X03yIz3mI0laB30C4CIwnWRXki3AIWB2pGYWeLD7NNB+4LWqurbUCqvq61V1d1XtrKqdDAJkT1X9yeo2Q5K0UhMDoKpuAseB88BLwDNVdSnJ0SRHu7JzwBVgHngC+Mgb45M8DXwReG+ShSSPrPM2SJJWYXOfoqo6x+BNfrjt1NDzAo4tMfZwj/Xv7DMPSdL68ZvAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1KheAZDkQJLLSeaTnBjTnySPdf0vJNkz1HcmyfUkL46M+fUkf9TVfzbJ1rVvjiSpr4kBkGQT8DhwENgNHE6ye6TsIDDdPY4AJ4f6ngQOjFn1s8CPV9X7gD8GPrrSyUuSVq/PGcA+YL6qrlTVDeAsMDNSMwM8VQMXgK1J7gGoqueAV0dXWlWf735wHuACsH21GyFJWrk+AbANeGVoeaFrW2nNcn4B+P1xHUmOJJlLMre4uLiCVUqSltMnADKmrVZRM37lyceAm8Cnx/VX1emq2ltVe6empvqsUpLUw+YeNQvAjqHl7cDVVdS8RZKHgJ8FfqaqegWGJGl99DkDuAhMJ9mVZAtwCJgdqZkFHuw+DbQfeK2qri230iQHgF8DPlRVr69i7pKkNZgYAN1/1B4HzgMvAc9U1aUkR5Mc7crOAVeAeeAJ4CNvjE/yNPBF4L1JFpI80nX9JvBDwLNJnk9yar02SpI0WZ9LQFTVOQZv8sNtp4aeF3BsibGHl2j/0f7TlCStN78JLEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY3qFQBJDiS5nGQ+yYkx/UnyWNf/QpI9Q31nklxP8uLImB9O8mySb3R/37H2zZEk9TUxAJJsAh4HDgK7gcNJdo+UHQSmu8cR4ORQ35PAgTGrPgF8oaqmgS90y5KkW6TPGcA+YL6qrlTVDeAsMDNSMwM8VQMXgK1J7gGoqueAV8esdwb4VPf8U8DPrWYDJEmr0ycAtgGvDC0vdG0rrRn1zqq6BtD9vbvHXCRJ66RPAGRMW62iZlWSHEkyl2RucXFxPVYpSaJfACwAO4aWtwNXV1Ez6ttvXCbq/l4fV1RVp6tqb1XtnZqa6jFdSVIffQLgIjCdZFeSLcAhYHakZhZ4sPs00H7gtTcu7yxjFnioe/4Q8HsrmLckaY0mBkBV3QSOA+eBl4BnqupSkqNJjnZl54ArwDzwBPCRN8YneRr4IvDeJAtJHum6HgXuT/IN4P5uWZJ0i2zuU1RV5xi8yQ+3nRp6XsCxJcYeXqL9fwE/03umkqR15TeBJalRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1qlcAJDmQ5HKS+SQnxvQnyWNd/wtJ9kwam+TeJBeSPJ9kLsm+9dkkSVIfEwMgySbgceAgsBs4nGT3SNlBYLp7HAFO9hj7CeDfVtW9wMe7ZUnSLdLnDGAfMF9VV6rqBnAWmBmpmQGeqoELwNYk90wYW8Bf6p7/ZeDqGrdFkrQCm3vUbANeGVpeAO7rUbNtwthfAs4n+Q0GQfQ3x714kiMMzip417ve1WO6kqQ++gRAxrRVz5rlxn4Y+BdV9TtJ/iHwW8DfeUtx1WngNMDevXtHX1fL2Hnicxvyui8/+sENeV1JK9PnEtACsGNoeTtvvVyzVM1yYx8CPtM9/88MLhdJkm6RPgFwEZhOsivJFuAQMDtSMws82H0aaD/wWlVdmzD2KvC3u+cfAL6xxm2RJK3AxEtAVXUzyXHgPLAJOFNVl5Ic7fpPAeeAB4B54HXg4eXGdqv+p8B/TLIZ+H901/klSbdGn/8DoKrOMXiTH247NfS8gGN9x3bt/x34GyuZrCRp/fhNYElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWpUrwBIciDJ5STzSU6M6U+Sx7r+F5Ls6TM2yS92fZeSfGLtmyNJ6mviT0Im2QQ8DtwPLAAXk8xW1R8OlR0EprvHfcBJ4L7lxiZ5PzADvK+qvpvk7vXcsO8nO098bqOnIElv0ecMYB8wX1VXquoGcJbBG/ewGeCpGrgAbE1yz4SxHwYerarvAlTV9XXYHklST30CYBvwytDyQtfWp2a5se8BfirJl5L8QZKfGPfiSY4kmUsyt7i42GO6kqQ++gRAxrRVz5rlxm4G3gHsB34FeCbJW+qr6nRV7a2qvVNTUz2mK0nqY+L/ATD4V/uOoeXtwNWeNVuWGbsAfKaqCvhyku8BdwH+M1+SboE+ZwAXgekku5JsAQ4BsyM1s8CD3aeB9gOvVdW1CWN/F/gAQJL3MAiL76x5iyRJvUw8A6iqm0mOA+eBTcCZqrqU5GjXfwo4BzwAzAOvAw8vN7Zb9RngTJIXgRvAQ93ZgCTpFuhzCYiqOsfgTX647dTQ8wKO9R3btd8Afn4lk5UkrR+/CSxJjTIAJKlRBoAkNcoAkKRG9fpPYOl2sVH3XXr50Q9uyOtKa+EZgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGeSsIrbuNuh2DpJXxDECSGtUrAJIcSHI5yXySE2P6k+Sxrv+FJHtWMPaXk1SSu9a2KZKklZgYAEk2AY8DB4HdwOEku0fKDgLT3eMIcLLP2CQ7gPuBb655SyRJK9LnDGAfMF9VV7rf8T0LzIzUzABP1cAFYGuSe3qM/STwq4A/Bi9Jt1ifANgGvDK0vNC19alZcmySDwHfqqqvLffiSY4kmUsyt7i42GO6kqQ++gRAxrSN/ot9qZqx7Ul+EPgY8PFJL15Vp6tqb1XtnZqamjhZSVI/fQJgAdgxtLwduNqzZqn2dwO7gK8leblr/2qSH1nJ5CVJq9cnAC4C00l2JdkCHAJmR2pmgQe7TwPtB16rqmtLja2qr1fV3VW1s6p2MgiKPVX1J+u1YZKk5U38IlhV3UxyHDgPbALOVNWlJEe7/lPAOeABYB54HXh4ubFvy5ZIklak1zeBq+ocgzf54bZTQ88LONZ37JianX3mIUlaP34TWJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhrVKwCSHEhyOcl8khNj+pPksa7/hSR7Jo1N8utJ/qir/2ySreuzSZKkPiYGQJJNwOPAQWA3cDjJ7pGyg8B09zgCnOwx9lngx6vqfcAfAx9d89ZIknrrcwawD5ivqitVdQM4C8yM1MwAT9XABWBrknuWG1tVn6+qm934C8D2ddgeSVJPfQJgG/DK0PJC19anps9YgF8Afr/HXCRJ66RPAGRMW/WsmTg2yceAm8Cnx754ciTJXJK5xcXFHtOVJPXRJwAWgB1Dy9uBqz1rlh2b5CHgZ4F/XFWjoQJAVZ2uqr1VtXdqaqrHdCVJffQJgIvAdJJdSbYAh4DZkZpZ4MHu00D7gdeq6tpyY5McAH4N+FBVvb5O2yNJ6mnzpIKqupnkOHAe2AScqapLSY52/aeAc8ADwDzwOvDwcmO7Vf8m8APAs0kALlTV0fXcOKkFO098bkNe9+VHP7ghr6v1MzEAAKrqHIM3+eG2U0PPCzjWd2zX/qMrmqkkaV35TWBJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWpUr28CS1reRt2OoVUt7u+349YbngFIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGtUrAJIcSHI5yXySE2P6k+Sxrv+FJHsmjU3yw0meTfKN7u871meTJEl9TAyAJJuAx4GDwG7gcJLdI2UHgenucQQ42WPsCeALVTUNfKFbliTdIn3OAPYB81V1papuAGeBmZGaGeCpGrgAbE1yz4SxM8CnuuefAn5ujdsiSVqBPvcC2ga8MrS8ANzXo2bbhLHvrKprAFV1Lcnd4148yREGZxUA/zfJZeAu4Ds95t4C98WA++FNt2Rf5N+93a+wLu6Y42KN+/uvjGvsEwAZ01Y9a/qMXVZVnQZO/5kXS+aqau9K1nOncl8MuB/e5L54k/tieX0uAS0AO4aWtwNXe9YsN/bb3WUiur/X+09bkrRWfQLgIjCdZFeSLcAhYHakZhZ4sPs00H7gte7yznJjZ4GHuucPAb+3xm2RJK3AxEtAVXUzyXHgPLAJOFNVl5Ic7fpPAeeAB4B54HXg4eXGdqt+FHgmySPAN4F/sIJ5n55c0gz3xYD74U3uize5L5aRqhVdkpck3SH8JrAkNcoAkKRG3VYBMOmWFHe6JC8n+XqS55PMdW1N3FIjyZkk15O8ONS25LYn+Wh3nFxO8vc2ZtZvjyX2xb9J8q3u2Hg+yQNDfXfkvkiyI8l/TfJSkktJ/nnX3uRxsRq3TQD0vCVFC95fVfcOfba5lVtqPAkcGGkbu+3dcXEI+LFuzH/qjp87xZO8dV8AfLI7Nu6tqnNwx++Lm8C/rKq/BuwHjnXb2+pxsWK3TQDQ75YULWrilhpV9Rzw6kjzUts+A5ytqu9W1f9g8Om0fbdkorfAEvtiKXfsvqiqa1X11e75/wFeYnD3gSaPi9W4nQJgqdtNtKSAzyf5SneLDBi5pQYw9pYad6iltr3VY+V4dzfeM0OXPZrYF0l2An8d+BIeF73dTgGw5ttK3AF+sqr2MLgMdizJ39roCX2favFYOQm8G7gXuAb8+679jt8XSf4i8DvAL1XV/16udEzbHbUvVup2CoA+t6S4o1XV1e7vdeCzDE5fW76lxlLb3tyxUlXfrqo/rarvAU/w5qWNO3pfJPnzDN78P11Vn+maPS56up0CoM8tKe5YSf5Ckh964znwd4EXafuWGktt+yxwKMkPJNnF4HcqvrwB87tl3njD6/x9BscG3MH7IkmA3wJeqqr/MNTlcdFTn7uBfl+YcFuJFrwT+OzgmGcz8NtV9V+SXGT1t9S4bSR5Gvhp4K4kC8C/ZonbiXS3KnkG+EMGnxQ5VlV/uiETfxsssS9+Osm9DC5pvAz8M7jj98VPAv8E+HqS57u2f0Wjx8VqeCsISWrU7XQJSJK0jgwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1Kj/DxKet+OW3udaAAAAAElFTkSuQmCC\n", 162 | "text/plain": [ 163 | "
" 164 | ] 165 | }, 166 | "metadata": { 167 | "needs_background": "light" 168 | }, 169 | "output_type": "display_data" 170 | } 171 | ], 172 | "source": [ 173 | "plt.hist(cum_rs, density=True, label='expert')" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "metadata": {}, 179 | "source": [ 180 | "## offline training" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 9, 186 | "metadata": {}, 187 | "outputs": [ 188 | { 189 | "name": "stdout", 190 | "output_type": "stream", 191 | "text": [ 192 | "fitting [0] / [8000]\n", 193 | "fitting [1000] / [8000]\n", 194 | "fitting [2000] / [8000]\n", 195 | "fitting [3000] / [8000]\n", 196 | "fitting [4000] / [8000]\n", 197 | "fitting [5000] / [8000]\n", 198 | "fitting [6000] / [8000]\n", 199 | "fitting [7000] / [8000]\n" 200 | ] 201 | }, 202 | { 203 | "data": { 204 | "text/plain": [ 205 | "AWAC(\n", 206 | " (critic): MLP(\n", 207 | " (hidden_act): ReLU()\n", 208 | " (out_act): ReLU()\n", 209 | " (layers): ModuleList(\n", 210 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 211 | " (1): ReLU()\n", 212 | " (2): Linear(in_features=128, out_features=128, bias=True)\n", 213 | " (3): ReLU()\n", 214 | " (4): Linear(in_features=128, out_features=2, bias=True)\n", 215 | " (5): ReLU()\n", 216 | " )\n", 217 | " )\n", 218 | " (critic_target): MLP(\n", 219 | " (hidden_act): ReLU()\n", 220 | " (out_act): ReLU()\n", 221 | " (layers): ModuleList(\n", 222 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 223 | " (1): ReLU()\n", 224 | " (2): Linear(in_features=128, out_features=128, bias=True)\n", 225 | " (3): ReLU()\n", 226 | " (4): Linear(in_features=128, out_features=2, bias=True)\n", 227 | " (5): ReLU()\n", 228 | " )\n", 229 | " )\n", 230 | " (actor): MLP(\n", 231 | " (hidden_act): ReLU()\n", 232 | " (out_act): Identity()\n", 233 | " (layers): ModuleList(\n", 234 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 235 | " (1): ReLU()\n", 236 | " (2): Linear(in_features=128, out_features=64, bias=True)\n", 237 | " (3): ReLU()\n", 238 | " (4): Linear(in_features=64, out_features=2, bias=True)\n", 239 | " (5): Identity()\n", 240 | " )\n", 241 | " )\n", 242 | ")" 243 | ] 244 | }, 245 | "execution_count": 9, 246 | "metadata": {}, 247 | "output_type": "execute_result" 248 | } 249 | ], 250 | "source": [ 251 | "batch_size = 1024\n", 252 | "n_updates = 8000\n", 253 | "fit_device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", 254 | "\n", 255 | "agent.to(fit_device)\n", 256 | "critic_losses, actor_losses = [], []\n", 257 | "for i in range(n_updates):\n", 258 | " if i % 1000 == 0:\n", 259 | " print(\"fitting [{}] / [{}]\".format(i, n_updates))\n", 260 | " sampled_exps = memory.sample(batch_size)\n", 261 | " s, a, r, ns, done = prepare_training_inputs(sampled_exps, \n", 262 | " device=fit_device) \n", 263 | " critic_loss = agent.update_critic(s,a,r,ns,done)\n", 264 | " actor_loss = agent.update_actor(s,a)\n", 265 | " critic_losses.append(critic_loss.detach())\n", 266 | " actor_losses.append(actor_loss.detach()) \n", 267 | "agent.to('cpu')" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 10, 273 | "metadata": {}, 274 | "outputs": [ 275 | { 276 | "data": { 277 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADQCAYAAABStPXYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXiU1fXA8e/JRiCEJRD2JYBI2BKWhB0UqCyC1CpWREGsSl1wqT8tWCourRbFtoqoiFbRagWrFlEWBdkFZN/3JUAIyB5CIGS7vz9mMkySSTJJZjLzJufzPHmSeeddTgI3J/e+971HjDEopZRS/ibA1wEopZRSrmiCUkop5Zc0QSmllPJLmqCUUkr5JU1QSiml/JImKKWUUn5JE5RSPiAiH4rIKRHZUcD7IiJTReSAiGwTkU5lHaNSvhbk6wBcqV27tomKivJ1GErls3HjxjPGmEgPnGomMA34pID3BwMt7R9dgXftnwulbUf5s+K2H79MUFFRUWzYsMHXYSiVj4gc8cR5jDErRCSqkF1+DXxibE/SrxWRGiJS3xhzorDzattR/qy47UeH+JTyTw2BY06vE+3blKowNEEp5Z/ExTaX65KJyFgR2SAiG06fPu3lsJQqO5qglPJPiUBjp9eNgCRXOxpjZhhj4owxcZGRnrg9ppR/8Mt7UKpkMjIySExMJC0tzdehWF5oaCiNGjUiODjYVyHMBcaJyCxskyOSi7r/pEpP25BneKr9aIIqRxITEwkPDycqKgoRVyNEyh3GGM6ePUtiYiLNmjXzyjVE5HPgRqC2iCQCzwPB9utPB+YDNwMHgMvAfV4JROWibaj0PNl+LJWglu45xZZjF/jDTdf7OhS/lJaWpg3LA0SEWrVq4c37OcaYu4p43wCPeup651PTmbHyEEPa16ddw+qeOm25o22o9DzZfix1D2rNobPMWHHI12H4NW1YnlHefo7JVzJ4d9lB9v2S4utQ/F55+7f3BU/9DC2VoIIDhYysbF+HoZTl5Py+0PqkykoslqACyMw2ZGdrKysPli1bxurVq0t9jqFDh3ooovJL7LPWteWUD3PnzmXy5MkAzJkzh127djnemzRpEosXL3brPAkJCbRr184rMXqCpe5BBQfa8mlGdjaVAgJ9HI0qrWXLllG1alV69Ojh9jGZmZkEBVnqv61fuNaD0hRldZmZmQwbNoxhw4YBtgQ1dOhQ2rRpA8BLL73ky/A8ylI9qJCcBJWljcxf3XrrrXTu3Jm2bdsyY8YMx/aFCxfSqVMnYmNj6d+/PwkJCUyfPp1//vOfdOjQgZUrV3LkyBH69+9PTEwM/fv35+jRowCMGTOGp556ir59+zJ+/PgCr33u3DluvfVWYmJi6NatG9u2bQNg+fLldOjQgQ4dOtCxY0dSUlI4ceIEffr0oUOHDrRr146VK1d69wfjYzrEZx2ffPIJMTExxMbGMmrUKCB/G5g5cybjxo1j9erVzJ07l2eeeYYOHTpw8OBBxowZw5dffgnA+vXr6dGjB7GxsXTp0oWUlILvQaalpXHffffRvn17OnbsyNKlSwHYuXMnXbp0oUOHDsTExLB//35SU1MZMmQIsbGxtGvXjtmzZ3vlZ2GpP0WDA22tLCMzGyr5OBg/9+K3O9mVdNGj52zToBrP39K20H0+/PBDIiIiuHLlCvHx8dx+++1kZ2fz4IMPsmLFCpo1a8a5c+eIiIjgoYceomrVqjz99NMA3HLLLYwePZp7772XDz/8kMcff5w5c+YAsG/fPhYvXkxgYME95+eff56OHTsyZ84clixZwujRo9myZQuvv/46b7/9Nj179uTSpUuEhoYyY8YMBg4cyMSJE8nKyuLy5cue+0H5oZyb1kYH+dzmiza0c+dOXn75ZX766Sdq167NuXPnHO85t4GZM2cC0KNHD4YNG8bQoUMZPnx4rnOlp6dz5513Mnv2bOLj47l48SKVK1cu8Npvv/02ANu3b2fPnj0MGDCAffv2MX36dJ544gnuvvtu0tPTycrKYv78+TRo0IB58+YBkJycXNIfSaEs1YMKDsrpQelECX81depUYmNj6datG8eOHWP//v2sXbuWPn36OJ6JiIiIcHnsmjVrGDlyJACjRo1i1apVjvfuuOOOQpMTwKpVqxx/cfbr14+zZ8+SnJxMz549eeqpp5g6dSoXLlwgKCiI+Ph4PvroI1544QW2b99OeHi4J759v5Uzp0p7UP5tyZIlDB8+nNq1awO524o7bcDZ3r17qV+/PvHx8QBUq1at0OFx5/YTHR1N06ZN2bdvH927d+eVV17h1Vdf5ciRI1SuXJn27duzePFixo8fz8qVK6le3TuPLlisB2VLUOmaoIpUVE/HG5YtW8bixYtZs2YNVapU4cYbbyQtLQ1jTImmnTofExYWVuT+ru6viAgTJkxgyJAhzJ8/n27durF48WL69OnDihUrmDdvHqNGjeKZZ55h9OjRxY7RKhxDfL4Nw1J80YYKayvutAF3z1XQ/q6MHDmSrl27Mm/ePAYOHMgHH3xAv3792LhxI/Pnz+fZZ59lwIABTJo0qVjxucNSPSi9B+XfkpOTqVmzJlWqVGHPnj2sXbsWgO7du7N8+XIOHz4M4Bi2CA8PzzUm3qNHD2bNmgXAZ599Rq9evYp1/T59+vDZZ58BtmRZu3ZtqlWrxsGDB2nfvj3jx48nLi6OPXv2cOTIEerUqcODDz7I/fffz6ZNm0r9/fszxyw+bTp+rX///nzxxRecPXsWINcQX0HytqMc0dHRJCUlsX79egBSUlLIzMws8DzO7Wffvn0cPXqUVq1acejQIZo3b87jjz/OsGHD2LZtG0lJSVSpUoV77rmHp59+2mvtx5I9KB3i80+DBg1i+vTpxMTE0KpVK7p16wZAZGQkM2bM4LbbbiM7O5s6deqwaNEibrnlFoYPH84333zDW2+9xdSpU/nd737HlClTiIyM5KOPPirW9V944QXuu+8+YmJiqFKlCh9//DEAb7zxBkuXLiUwMJA2bdowePBgZs2axZQpUwgODqZq1ap88klBdQPLh2s9KM1Q/qxt27ZMnDiRG264gcDAQDp27Oi431SQESNG8OCDDzJ16lTH5AiAkJAQZs+ezWOPPcaVK1eoXLkyixcvpmrVqi7P88gjj/DQQw/Rvn17goKCmDlzJpUqVWL27Nl8+umnBAcHU69ePSZNmsT69et55plnCAgIIDg4mHfffdeTPwYH8cdpp3FxccZV0bUfdp5k7L838t1jvXS5Fhd2795N69atfR1GueHq5ykiG40xcT4KqUgFtZ1TF9Po8sqP/PXWdtzTrakPIrMGbUOe44n2Y6khvpxJEmkZWT6ORCmL0XtQyoIslaC2HL0AwBuL9/s4EqWsJUAfhFIWZKkEdTb1KgAbj5z3cST+yx+HbK2ovP0cc+Zy6SphRStv//a+4KmfoaUS1PnLGQBc0SE+l0JDQzl79qw2sFLKqWcTGhrq61A8xvGgrv7fKJS2odLzZPux1Cy+Ie3rM2/bCXQ1fNcaNWpEYmKiV+sYVRQ5FUHLC8eDuj6Nwv9pG/IMT7UfSyWo+CjbU9X6x41rwcHBXqsAq6xNb0G5R9uQf7HUEF/OWnxKqeLRchvKitxKUCIySET2isgBEZlQwD43isgWEdkpIsudtieIyHb7e/kf0CiGoEBL5VOl/IeW21AWVOQQn4gEAm8DNwGJwHoRmWuM2eW0Tw3gHWCQMeaoiNTJc5q+xpgzpQ42QHtQSpWE3rdVVuROl6QLcMAYc8gYkw7MAn6dZ5+RwNfGmKMAxphTng3TJlh7UEqVSIDoWnzKetz5jd8QOOb0OtG+zdn1QE0RWSYiG0XEeVloA/xg3z62NMEGag9KqRK59hyUZihlHe7M4nOVFfL+Lw8COgP9gcrAGhFZa4zZB/Q0xiTZh/0WicgeY8yKfBexJa+xAE2aNCnO96CUKoKW21BW5E4PKhFo7PS6EZDkYp+FxphU+72mFUAsgDEmyf75FPA/bEOG+RhjZhhj4owxcZGRkcX7LpRShdJyG8qK3ElQ64GWItJMREKAEcDcPPt8A/QWkSARqQJ0BXaLSJiIhAOISBgwANjhufCVsq6iZseKSHUR+VZEttpnx95X8mvZPmu5DWUlRQ7xGWMyRWQc8D0QCHxojNkpIg/Z359ujNktIguBbUA28IExZoeINAf+Z19mJQj4jzFmobe+GaWswp3ZscCjwC5jzC0iEgnsFZHP7JOVSkR7UMpK3FpJwhgzH5ifZ9v0PK+nAFPybDuEfajP01YfPEOPFrW9cWqlyoJjdiyAiOTMjnVOUAYIF9tfeFWBc0DBJVELodPMlRVZdt729ztO+joEpUrDndmx04DW2O75bgeeMMaUqJz0tXtQ2oVS1mHZBPXxmiO+DkGp0nBnduxAYAvQAOgATBORavlOJDJWRDaIyIaCFjnVtfiUFVk2QSllce7Mjr0P2wPwxhhzADgMROc9kTszYB0P6nogcKXKiiYopXzDndmxR7E9W4iI1AVaAYdKcjF9UFdZkaXKbShVXrgzOxb4CzBTRLZjyzHjS7qmpQ7xKSvSBKWUjxQ1O9b+kPsAT1xLdIhPWZAO8SlVkWgXSlmIpRPUlfQsX4eglGWIaA9KWYvlElR4pWujkjNXJ/guEKUsRtAOlLIWyyWoutVDHV9nZJXomUWlKiQR0bX4lKVYLkGN7d3c8XVahg7xKeWuANEelLIWyyWo0JBAx9fvLDvow0iUshZByNYEpSzEcgkqW1uYUiUjWm5DWYvlElSWJiilSkRAp/EpS7FegtJBdKVKRKeZK6uxXIKqFRbi6xCUsiRBtNyGshTLJah+0XVyvV53+JyPIlHKWkRn8SmLsVyCkjylQX/73hofRaKUtQg6xKesxXIJSilVMiKiPShlKZqglKogRKeZK4txK0GJyCAR2SsiB0RkQgH73CgiW0Rkp4gsL86xxRUSqHlVqeLStfiU1RT5m15EAoG3gcFAG+AuEWmTZ58awDvAMGNMW+AOd48tiV93aFDaUyhV4diG+DRDKetwpyvSBThgjDlkjEkHZgG/zrPPSOBrY8xRAGPMqWIcW2yju0fleq1r8ilVNH0OSlmNOwmqIXDM6XWifZuz64GaIrJMRDaKyOhiHFts7RtVz/U6+rmFurK5UkXQIT5lNe6UfBcX2/L+Nw8COgP9gcrAGhFZ6+axtouIjAXGAjRp0sSNsHLLyMomWO9NKVUgLbehrMad3+iJQGOn142AJBf7LDTGpBpjzgArgFg3jwXAGDPDGBNnjImLjIx0N34HcZkLlVI5tAelrMadBLUeaCkizUQkBBgBzM2zzzdAbxEJEpEqQFdgt5vHeoRoflKqUHoPSllNkUN8xphMERkHfA8EAh8aY3aKyEP296cbY3aLyEJgG5ANfGCM2QHg6lgvfS9KqULog7rKaty5B4UxZj4wP8+26XleTwGmuHOsUqrs2Yb4NEMp67DsrIKaVYJzvb7zvTVczdTp5koVRBeLVVZj2QT1QO/muV5vTUxm6o/7fRSNUv5P0Fl8ylosm6BGdsk/Ff3tpQd9EIlSJVOaJcRKdj3tQSlrceselD+qWUjhwtSrmYRVsuy3pioAp2XAbsL2OMZ6EZlrjNnltE/OEmKDjDFHRaSO67O5eU10Fp+yFsv2oAryj0X7aPv897wyfzdZ2docld8qzRJiJaKz+JTVlLsElXMfasaKQwz4Z6lGRJTyptIsIZaLiIwVkQ0isuH06dOFXlTvQSkrKXcJytnB06m+DkGpghRnCbEhwEDgORG5Pt9Bbq7CIjrGpyymXCcogJS0DKKfW8DyfYX/ZalUGSvNEmIlEiCi+UlZSrlPUJ+vO0paRjb/XLTP16Eo5aw0S4iViAhk600oZSHlfqrbG4tt96S0WSp/UtolxEpCF4tVVlPuE5RS/qo0S4iVhOgQn7KYcj/EdzndtvzR1mMX2P9Lio+jUcp3dC0+ZTXlPkE5u+mfK0i+nMGFy+m+DkWpsqflNpTFWDpBdWhco9jHxL70Ax1eWuSFaJTybwKaoZSlWDpB1a8eWuJjn/16m65+rioULfmurMbSCarHdbVLfOzn647R6s8LPRiNUv5NZ/Epq7F0grqna/4VzZVSrgXoWnzKYiydoERcrRajlHJFH9RVVmPpBAXw5ogOpT7HxiPnWZ9wzgPRKOXfND0pK7H8g7qBAaXrRV1Oz+T2d1cDkDB5iCdCUsovabkNZTWW70GJy0Wh3ddm0veOr//xw97ShqOU37K1FM1QyjrcSlBFlaa2l6VOtpem3iIik5zeSxCR7fbtGzwZvO38njvX1CUHWH3gDN9uTSLhjJbqUOWLlnxXVlPkEJ87pantVhpjhhZwmr72cgEeV8oRvnxGfvCz4+udLw4krFIQ2dmG9KxsQoMDPXsxpcqQ6EoSymLc6UG5U5rah7w3k+/R/2wCYOKcHUQ/p89MKWsTRNfiU5biToJypzQ1QHcR2SoiC0SkrdN2A/xgL1k9tqCLFKdsde7jbJ9jGlV3+xh3bTxyHrDVlFLK6gK0B6Usxp0E5U5p6k1AU2NMLPAWMMfpvZ7GmE7AYOBREenj6iLulq3OKz4qgkpBAUwa2sbtY9yVkpap96JU+SFCtmYoZSHuJKgiS1MbYy4aYy7Zv54PBItIbfvrJPvnU8D/sA0ZekxEWAh7/zqYuKgIT57WYcxH67xyXqXKmpbbUFbjToIqsjS1iNQT+7IOItLFft6zIhImIuH27WHAAKDEFUGL0rlpTY+fMyPrWoOOmjCPnUnJHr+GUmVBF15RVlNkgjLGZAI5pal3A1/klKbOKU8NDAd2iMhWYCowwtj+VKsLrLJvXwfMM8Z4bbbBZw90ZdrIjh495/ELV3K9zikhr5TV6GKxymrcWkmiqNLUxphpwDQXxx0CYksZo9tCgwO5qU1dr15Dix0qq9JyG8pqLL+SRF6VggJ55+5OXjv/+oTzXju3Ut6kPShlNeUuQQHc3L6+V9fV+/nQWa+dWylv0ZUklNWUywTlbXfOWMsXG47p81HKUgQd4lPWYvnVzH3lj19uA6B3y9o0qlnFx9EoVTTtQSmrKdc9qAVP9Pb6NbL0yUdlEZqglNWU6wTVun61XK+/fqSHjyJRKr+iqgQ47RcvIlkiMrxU19MhPmUx5TpB5dWpSU0iwkI8es4bpizjVEoaW49dyLX97KWrZGvvShXAqUrAYKANcJeI5Fuvy77fq9ieQyzlNbUHpaylwt2D8sbD9F1e/hGwVeT963e72HT0PJuOXuD/brqex/q39MIVVTngqBIAICI5VQLylrF5DPgKiC/tBbXchrKact+DeuU37XO9Fi+u97Js7yk+WHWYTUdtvanFe0557VrK8oqsEiAiDYHfANMphLuVALTchrKacp+gRnZtkuu1N9cjG/PR+lyv8w77KeXEnSoBbwDjjTFZhZ3I3UoA2oNSVlPuE1SOFpFhAIzre52PI1EKcKNKABAHzBKRBGzrXb4jIreW5qLagVJWUiHuQa2b2J+wENu3em+PKJ6fu9PHESl1rUoAcBxblYCRzjsYY5rlfC0iM4HvjDHOtdaKJUBEe1DKUipED6pOeChhlfLn4gbVQwHbfaroeuFeuXZ2tiE9M9sr51bW5WaVAI+yzeLTFKWso0IkqIKM62ebYTewbV0WPumy0G+p9Zi8hOv/vMAr51bWZoyZb4y53hjTwhjzsn3bdOdKAU77jjHGfFma6+liscpqKsQQX141qwRzT7emjOzaJN8kCk87eTHNq+dXyl1abkNZTYVMUJsnDfB1CEqVOe1BKaup0EN8edWrFuq1c6/VEh3Kx3QlCWU1mqCcrP1Tf6+de8SMtRw7d5n520947RpKFU5n8Slr0QSVx+bnbmLPXwZ55dy9X1vKI59tAuDwmVTSMrJYuvcUl9MzvXI9pZzpLD5lNRXyHlRhanp4MVlX0jKy6Pv6MprXDuPQmVRuiW3AW3d19Pp1VcXmxUVUlPIKt3pQRZUFEJEbRSRZRLbYPya5e6y/Cnfx3JSnfLvVtmDAoTOpts+nLzneS76cwZzNx712bVVxBYjoPShlKUX+FnYqC3ATtuVZ1ovIXGNM3lWXVxpjhpbwWL9TMyyElKveGXp7xl6NN8fOpIuOr5+cvZmle0/TvlF1WkRW9cr1VcUkAtmaoZSFuNODcpQFMMakAzllAdxRmmN9KjDANiAypH19Fj7p/cq8L8zdiTGGkxevAjDojRVev6aqWHSxWGU17iSoIssC2HUXka0iskBE2hbzWL+TM17/h5uuJ7petUL39YSZqxOYtf4Y9rxIRpbhsH0I0FnylQyW7PmlxNd5beEeOv9lUYmPV9al5TaU1biToNwpC7AJaGqMiQXeAnIWtHTnWNuObta0KStvjOhA/+g6RNWqUmbXfH/loVzDfRevZOTb55HPNvK7mRs4lVKyFSreWXaQs6npJY5RWZj2oJTFuJOgiiwLYIy5aIy5ZP96PhAsIrXdOdbpHG7VtCkrMY1q8K8x8QQFlt1M/EOnc/eYHv50I/9ek8DVzGvlgA7b98nI0l81qngENEMpS3Hnt6+jLICIhGArCzDXeQcRqSf2UrUi0sV+3rPuHKsKlpScxnPf7GTakgOsTzjHkKkrycy2/YYpzpThzKxsFu/6RYd3KjjRchvKYoqcxWeMyRSRnLIAgcCHOWUB7O9Px1ZM7WERyQSuACOM7behy2O99L141Ypn+nLs/GX2nkzhpe/KdhLilmMXeGvJgVzbilMZ+K0lB3jzx/28PzrOw5EpK7GtxacpSlmHWw/72Ift5ufZNt3p62nANHePtaImtarQpFYVel5Xu8wT1Mr9Z/JtS8vI5tTFNOpUC2XB9hMkX8kgMEC4I64x989cz5pDZ1k/8VdkZGXz5o/7AThyNv+kC1VxBOg9KGUxupKERfV9fRkA/32oOw/bl08CuCOuMT/uOQVAn9eW5poQ8fWmwh8APnbuMhlZ2czbdoLr6lRlcPv6ng9c+YyI6HNQylI0QZXA3HE9ycw23PbOahrWqMzxC1d8Fsu5PDPyoibMc3xd3Nl6vV9bmut1wuQhJQ+sABlZ2QSKEBCgC++UNS23oaxGF4stgZhGNYhtVIOBbevyzt2d+N8jPfjm0Z4+ieVRp95TUXaduFj0Tl6QkpbBmoO2ciMtJy7It5KGp2VmZTP0rZUs23vKq9exHC23oSxGE1QJBQYI742KI7ZxDTo2qUls4xo+iSNnVl9pFXR/6nxqOplZ2QW+l+XG9R//fDN3vb+Ws5dsq2R8tSmxwH3/vfYI//hhrxsRF+zMpXR2HL/I+K+8mwitRnS5WGUxmqAqqPUJ59iemOx4fcOUZfn2uZKeRce/LOKFb3NPvDxz6SrHzl2m418WMXnB7iKvtedkCgBXM/MnuquZWazcf+3B7Ofm7GBqnhmLyjO03IayGk1QFdQd09dwy7RVrDt8jo9XJ7jc59mvbT2QT9ceJTMrm/eWHyQtI4u4vy523K9asONkAedfTfNn5zFtyX5OJNtWvXD1q/Gv3+1m1L/WseN4sot3Sy8tI4vzqemculiylTfKE0Fn8Slr0UkSXvDRmHjum7ne12G45bfvrSnwvTlbri36cd3EBQCsPlh46fqoCfMY1/c61iecB+DdZQddn3vzcX46cIb/brQN9w19axUHX7m5WLEX5ZeLV4l+bqHjtTcmfViJlnxXVqM9KC/oG12Hnyb0I7peuK9D8bjl+3Kvk5h4/gr3frgu17ZpS68N0YnTE8XTnIbunpy9xZGccqQ6VRZOvpzBrHVHC7z/pYpPEIz2oZSFaILyoMVP9eGjMfEANKxRmU8f6ErXZhE+jsr78iYtZ5ecamp9vu5ooecxTrno//67hQlfb6fXq0sLPqCYnpuzgz0nfTOT0R8EBGgPSlmLJigPuq5OOH2j6zhe165aidm/7+7DiMrO4TOpuZ7BKom4l6+VATlpv2d00sW9o0W7fiHdxYSLovx77REGvbHS8fr4hSslXhXeE9yoVH23iGyzf6wWkdhSXhEPTfpUqkxogioDQ2PK/4oMOStblIbzCu07jl/r6Xz002HH16sPnuHBTzbweimmomfbf0v3nLyELi//SNtJC0uU8ErDqdr0YKANcJeItMmz22HgBmNMDPAXYEbprgk6TUJZiSaoMtC6vvcLHpZnL367y3Ev6nyqrUbWsXOXHe8fPH2Jp/+7lcysbNYeOstHqw+7PE+ONxbvy/U6NT2L85fLvEZWkdWmjTGrjTHn7S/XYitXU2K6koSyGp3FVwYGtavHlO9L9/BpRXfdxAX8vk9z5m7NXU4sLSOL/n9fDsDo7k0ZMWNtkeeauuQA2/NMa9+VdBERqBMe6rmgC+eq2nTXQva/H1jg6g0RGQuMBWjSpEmBJwgMELI0QykL0QRVBlpEVuXvd8Ty7vKDHDh1ydfhWNZ7Kw45vl6w4ySvf7+XYKeCkpuPXnD7XEv35p7YkfNYQKcmNfjq4R65Zh96SXGqTffFlqB6uXrfGDMD+/BfXFxcgRkoQMStlT+U8hc6xFdGbu/ciEV/6OPrMMqVaUsP5Ko2/Pzc0pca21SMJFdKblWbFpEY4APg18aYwh9CK0JQgDjuvyllBZqgypCI8O04l38EqxLaluj5FSjKaBTMnUrVTYCvgVHGmH0uzlEsOsSnrEaH+MpYu4bV+H2f5twR15h61UNp9/z3vg7J0lYdyF/M0QrcrFQ9CagFvGMfcsw0xpS4LHJAgA7xKWvRBFXGRIRnb27t6zBUIcrqV7gblaofAB7w1PUC9R6Ushgd4vOxsX2a07p+Ndb9qb+vQ1F25XXF78AA24O65fX7U+WPJigf+9PNrVnwRG/qVAvlu8d6cVvHhr4OqcJbuNP1Cu1WF2ivYqydKGUVbiWoopZkcdovXkSyRGS407YEEdkuIltEZIMngi6v2jWszj/u7ODrMCq8cf/Z7OsQvCInQWVm6wK8yhqKTFBuLsmSs9+r2G765tXXGNOhNDd4K6LfxtkWDggMEKJqVfFxNMrqAuzPdml+UlbhTg+qyCVZ7B4DvgJOeTC+Cq13y0jAthJF5RDbfJYBberyzt2d+M8Ku6oAABIGSURBVObRnrn2/fT+whYhUMr2HBSgU82VZbgzi6/IJVlEpCHwG6AfEJ/neAP8ICIGeM/+1LsqxFcP9+DAqRTaNawOwJD29YmuF877Kw/x11vbO4ZqnPVqWbusw1QWE5CToPQmlLIIdxKUO0uyvAGMN8ZkuVgipqcxJklE6gCLRGSPMWZFvou4uZ5YRdC5aU06N60J5K4C+7fbYnwVkioHAu1NUxOUsgp3hvjcWZIlDpglIgnAcGwPFt4KYIxJsn8+BfwP25BhPsaYGcaYOGNMXGRkZLG+CWXz4rC2vPybdgD8qnVdqlbSx9zUNYF+1IM6cOoSO457fhUQVb64k6CKXJLFGNPMGBNljIkCvgQeMcbMEZEwEQkHEJEwYACww6PfgXK4t0cUd3dtysFXbmbGqM5MGBzteO+xftcBOTWBVEUUGGBr7tlevgd17Nxlkq9k5Np2OT2TqAnzWLD9BAC/+sdyhr61KlfZFKXyKjJBGWMygZwlWXYDX+QsyZKzLEsh6gKrRGQrsA6YZ4xZWNqgVeECA4SAAOGebk3p0LgGAHWq2cpI5LxWFU/Owu/e6EGdupjGc3N28MSszfR+bSmxL/7geC/5cgYD37CN6v99Ue4lBXu/tpQ5m48DtgrH51PLvC6X8mNujQEVtSRLnu1jnL4+BJSyTLUqyE8T+tFz8pJC9/n8wW5cTMsgsmolaoeFMLBtPZr/Kdc/JdH1wtlzMiXXtrF9mjPDqbyFsr6caealTVCpVzMRsZ3v9e/38uRN1/PgJxvY6mLh3oQzqdzoVG35wKlLnE65mmufJ2dv4daODek5eQmVgwP5223t6dy0JtWrBFM1JIjNxy447smqikVvUlhYwxqVi9ynckgglUMCARjcPnfp+VZ1w9n7Swq9W9YmQIRdJ2xl1jc9dxPvLT/o+YCVTwUFlj5BZWZl09a+wPHdXZvw2c9HSTib6jI5vfTtLj78KX914/iXF+fbdt9H6wC4kpHFk7O3OLY/M7AVU77fy4O9m3E65Sp//20Hl7NYVfmkSx1VYPHNbH+VdmpSk/lP9ObpAdcDUL1ycK65m1XsCQ5g/KBovnq4BxMLWfC2e/Na3glYlYqjB1WKe1DOlaE/+/koAIt3u3700VVyKkjeApI5diXZ/mh6f+Vh5mxJYvrygwx6YwVfbLA9+ZJ8OcPRI0u6cIWLaRkuz6OsSXtQFdiLw9oxsktT2jSoBsC4fi0Z168lAOKUoRY/dQOHz6QSHxVBSJDtb5rOTWsyrEMD7nxvDQlnc9/oHtW9KWsOlaq2nvICx1p8pehBbTp63lPhuGWefVJFjpwE+ccvt3F7p0bEvmS715UweQg9Ji+hYY3KdGtei0W7TrLthYFlGqvyPO1BWdyP/3cDn/zO5cz9Aq2e0I+f/9SfwABxJKe8QgKvJaj61UPpeV1tR3LKUbdaKOMHRec91KWNf/6V2/H1us63Dx2X1yGkwFL2oD77+QjrE8o2QRWmhdO91KgJ8wDbRIuvNiVyMc02azDxvM4StDLtQVlci8iqtIisWqxjGrhx7+r3N7Qg5WomfxwYjYuHrx36XB9Jl2YRrDt8zrGtZpUQpt/TiYc+3QTAiPjG1Kpaif8+1J1Z647x1aZEl+caEd+Y3i0jGdC2Li0nLijW9+RJj9t7keWNY7HYLPcTVFpGFodOp7Lp6Hn+PMd6T4j0enWp4+tfta5DfFQE2QYGt6tHRNUQggKEKiGF/xpMunDFrTajPE8TlHIprFIQz9/S1q39vvh9d8dfsB+Niad7C9s9qGcHR/O3BXsID7X9N4uPiiA+KiJfgnr19vb0ja5DrbBKJeq93Nu9KR+vOVLs4wrSql7xEr5VXCu34V6Cys42RD9Xfp4KWbz7lON+2asL9zi2v3FnB6pXDiYiLIT0rGzumL6GH/7Qh8c/3+yY3Tp7bDfioiL4blsS2xOTmbX+GG+O6EC/6DqF/gGnSkcTlPKIF25pQ7cWtYiud23IMOfXoKsG/KvWdYkIC+aLDYkYA3XCQ0t03en3dHYspjvdQzMPG9YonyvHF3ctvuX7XE9cKG+cZw3mGPDP3KuxvbPsIIEBh1iy59qEkPs/3kDzyDAOnU5l/KBo4qJqEte0piYsD9IEpTxiTM9m+bbl/KGet7kefOVmBHj26+22/Yp5ra7NIvjZPqQ4qF09AJ4ecD2/jWtEtcrB3DhlGZeuZhbzrNfUDAsu8bH+LOcelDs9qMW7fuGBT7R8W46CkvWh06lA7h4ZwGvDY/jjl9u4sVUky/ae5s64xmw/nsyuExeJqlWFv90WQ9KFK/ySkkaD6pVZffAMieevcH+vZvRvXZfUq5kknE2lTf1qFTrhaYJSXmPsqSdvA8sZagoOsn12Z1Tv0b4teHuprYf0wrC2DH5zJa3qhjveDwoMoLn9Xtz9vZrx5o/76R9dh10nLtLrutr8d2PuYcUmEVU4WsAyO+Gh5TNBBduXkriaWXBBqOxsw8aj5zU5ldIfv9wGwDL79PnZG64VhEg4e5m73l/r8rjVB8/mu6fbsEZljl+4wpcPdScoMIBb3/4JgH/dG0f/1nW5kp7F9ztPMiy2gaOXDHAxLYOle07RsEZl4qIiXF7vmy3H6dy0Jo1q+ueogSYo5TVxTW2NokcL189FPTMwmpDAQH7TsVGh5/nusV60rl/NkaBynucxRfS92jaszr/GxDNtyf5c2796uDudm0Y47pvd3qkRD/RuxuA3VwL258DKobBKtufZDp66RI8WrmdKPjZrM/O2nXD5niobzskJbDMTAYZPX5Nr+/0f5/4jwtVQZUEe79+SzKxs3llma1NDYurzp5tbk3juMnO2JNGmQTUSz13m9s6N+PnQWbKyDTXDQhj/1Ta2vzCQI2dT2XMyhaExDUryLbpNE5Tymi7NItj10sACZ0lVrxzMpFvyFWfOZeUf+9I4wvbX3ZcPdWf3yRTHgrfuPs7TrYgHh1+/I6ZCDKNcSrMNe06au5NR3aNyvZeZlc34r7Zrcqogpv6Y+4+2edtOuPy3f8/FcmfOM2zH/WdzvvedSwSVlj4HpbyqqCm8BXmwdzMqBQU4khNAXFQEo7o1JSIsBIA+LV2XZXHkGvu9lrioCPb9dbDTQrm2HTY/dxM//t8NFSI5AXSOsq0cElm1Uq7tB09f4rqJCwqc/l+YonqbOf9WxfFAr/z3M1XFpD0o5ZcmDmnDxCGue1e1q1bipwn9qBteyeX74qLGZkhQgNOsQtvnmmEh1CzBL1CrqhRkG+I75bRY66r9Z7jnXz8XetyQ9vWZt/0EtcJCOJuazujuTenSLIKhMQ0wxvDqwr20bVCNo+cuM+X7vUSEhXAuNZ2Xf9OOu7s2Zd8vKY5ZcdteGMCOxGS6Na/Fn/63nVnrr92bmTuuJzGNbH9E3NqxIelZ2dz2zmpP/xiUhWiCUpbkzkK5eTWqWZmtxy4QVsJenaeJyCDgTSAQ+MAYMznP+2J//2bgMjDGGLPJE9f+YedJvt12gm+35q09mtu0kR0ZGtOAh48nE1U7jAuX02lYo7Kj1ykijrpjWdmGiLAQ7ujciKDAa4Mz19cN57XhMbSIrEq10GB62FcKmXx7DJNvj2FDwjn+PGcH1ztNemnXsLrj69jGNZg0tA1XM7Po3rwWSclp1A2vRGa2YWfSRR7+dCOnUq4yaWgbXvpul+O4YbEN6No8gon/u/aA8dCY+nynw5iW4R8tVSkvyHuL6tXbYxjSvj6t6oW73P+rh3uwcn/ZPPsjIoHA28BN2KpWrxeRucaYXU67DQZa2j+6Au/aP5dYTu9m7L83uny/R4taTB/VmfBKQbmGPnMSRmFVmgMDhLu6NHH53m/jGrvcDrYh2IVP9nH5nqv7GTl/nAQF2taE/P7JPpy7nE6LyKrsTLrI/O0n2P2XQY79b2pdlxHvr6VSUCDTRnbi5VszWLT7F365mMajfW2FPEf962eSLlzh/dFxNKsdxvELV3hn2UEaVA+leWRVbrZXAtiWeIH/bkjk32uPULdaJcb1vY7nvtlZ4PfmLKZRdba5WPVdFUyMl6trlkRcXJzZsEGnuaqS+XDVYV76bhcvDmvLvT2iPHpuEdlojInzwHm6Ay8YYwbaXz8LYIz5m9M+7wHLjDGf21/vBW40xhTYBSiq7WRkZbtcRuqJ/i15oHezcjvF3pOMMZxKuUrdaq4fLs/IysYY+OViGpWCAggNCaSaGz/X0ylX2ZGUTN9WdcjOtl3DXgQ534PsV9KzyDKGC5fTWbb3NPFRETStVYXE81eoVjkIQZiz+Th1q4cS17QmX2w4RrPaYQQFBLA+4RzdW9Riz4kUhsc1IvHcZUSEKxlZNKgeigHWJ5yjTf1qvLpwD/2j6yIC5y+ns/bQOeLtU9aTLlxhz8mL7PvlEgD39Yxi/KBoQoMDKUhx248mKFXuXM3M4pPVR7i7W5MST9IoiAcT1HBgkDHmAfvrUUBXY8w4p32+AyYbY1bZX/8IjDfGFNg43Gk7W49d4Ndv/0TvlrX5+x2xVA0N8vjPSSlXitt+9H+lKncqBQXyYJ/mvg6jKK6mDub9a9GdfRCRscBYgCZNXA+xOYttXMOjU4GV8hadZq6UbyQCzjdmGgF5Zyy4sw/GmBnGmDhjTFxkpOup90pZkSYopXxjPdBSRJqJSAgwApibZ5+5wGix6QYkF3b/SanyRof4lPIBY0ymiIwDvsc2zfxDY8xOEXnI/v50YD62KeYHsE0zv89X8SrlC5qglPIRY8x8bEnIedt0p68N8GhZx6WUv9AhPqWUUn5JE5RSSim/5JfPQYnIaaCgGt61gTNlGI67NK7i89fYCourqTHGb6fKFdF2wJo/c1/SuIqnqLiK1X78MkEVRkQ2eOJBSU/TuIrPX2Pz17g8wV+/N42reCpKXDrEp5RSyi9pglJKKeWXrJigZvg6gAJoXMXnr7H5a1ye4K/fm8ZVPBUiLsvdg1JKKVUxWLEHpZRSqgKwVIISkUEisldEDojIBC9fq7GILBWR3SKyU0SesG+PEJFFIrLf/rmm0zHP2mPbKyIDnbZ3FpHt9veminMluJLHFygim+0lGfwprhoi8qWI7LH/7Lr7Q2wi8gf7v+MOEflcREL9Ia6yUpZtx349bT/Fj0nbTl7GGEt8YFuv7CDQHAgBtgJtvHi9+kAn+9fhwD6gDfAaMMG+fQLwqv3rNvaYKgHN7LEG2t9bB3THVj5hATDYA/E9BfwH+M7+2l/i+hh4wP51CFDD17EBDYHDQGX76y+AMb6Oq7y2HW0/2nY8FZfPG08xflDdge+dXj8LPFuG1/8GW3nuvUB9+7b6wF5X8WBbBLS7fZ89TtvvAt4rZSyNgB+Bfk4NzB/iqmb/zyx5tvs0NnsjOwZEYFt/8jtggK/jKsP/uz5tO/ZravspPCZtOy4+rDTEl/ODypFo3+Z1IhIFdAR+Buoae8kD++c6RcTX0P513u2l8QbwRyDbaZs/xNUcOA18ZB8++UBEwnwdmzHmOPA6cBQ4ga1sxQ++jqsM+aztgLYfN2nbccFKCcqt6qIev6hIVeAr4EljzMXCdnWxzRSyvaTxDAVOGWM2untIWcRlFwR0At41xnQEUrF1/30am318/NfYhhwaAGEico+v4ypDPotb24/btO24YKUE5VZ1UU8SkWBsjeszY8zX9s2/iEh9+/v1gVNFxJdo/9pTcfcEholIAjAL6Ccin/pBXDnXSjTG/Gx//SW2Rufr2H4FHDbGnDbGZABfAz38IK6yUuZtB7T9FJO2HReslKDcqUDqMfYZJv8Cdhtj/uH01lzgXvvX92IbW8/ZPkJEKolIM6AlsM7e/U0RkW72c452OqbYjDHPGmMaGWOisP0Mlhhj7vF1XPbYTgLHRKSVfVN/YJcfxHYU6CYiVezn6w/s9oO4ykqZth3Q9lOCuLTtuFLSm2e++MBWXXQftpkhE718rV7YuqDbgC32j5uBWthusO63f45wOmaiPba9OM1QAeKAHfb3ppHnRmgpYryRazd5/SIuoAOwwf5zmwPU9IfYgBeBPfZz/hvbLCOfx1VWH2XZduzX0/ZT/Hi07eT50JUklFJK+SUrDfEppZSqQDRBKaWU8kuaoJRSSvklTVBKKaX8kiYopZRSfkkTlFJKKb+kCUoppZRf0gSllFLKL/0/k0WTvDM5Ye0AAAAASUVORK5CYII=\n", 278 | "text/plain": [ 279 | "
" 280 | ] 281 | }, 282 | "metadata": { 283 | "needs_background": "light" 284 | }, 285 | "output_type": "display_data" 286 | } 287 | ], 288 | "source": [ 289 | "fig, axes = plt.subplots(1,2, figsize=(6,3))\n", 290 | "axes[0].plot(actor_losses, label='actor loss')\n", 291 | "axes[0].legend()\n", 292 | "axes[1].plot(critic_losses, label='critic loss')\n", 293 | "axes[1].legend()\n", 294 | "fig.tight_layout()" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "## check offline perf" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 12, 307 | "metadata": {}, 308 | "outputs": [], 309 | "source": [ 310 | "awac_cum_rs = []\n", 311 | "for n_epi in range(num_runs):\n", 312 | " s = env.reset()\n", 313 | " cum_r = 0\n", 314 | "\n", 315 | " while True:\n", 316 | " s = torch.tensor((s,), dtype=torch.float)\n", 317 | " a = int(agent.get_action(s).squeeze())\n", 318 | " ns, r, done, info = env.step(a)\n", 319 | "\n", 320 | " s = ns\n", 321 | " cum_r += 1\n", 322 | " if done:\n", 323 | " awac_cum_rs.append(cum_r)\n", 324 | " break" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": 13, 330 | "metadata": {}, 331 | "outputs": [ 332 | { 333 | "data": { 334 | "text/plain": [ 335 | "" 336 | ] 337 | }, 338 | "execution_count": 13, 339 | "metadata": {}, 340 | "output_type": "execute_result" 341 | }, 342 | { 343 | "data": { 344 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ40lEQVR4nO3df5DU9Z3n8efrRpDbLLlJZNwlDLmZdUfjSBkCE+TOEDWuOCAlm5S5wMWSqAFRMCFXa4BYlcvmvFKzlueSKBzqGKkYUTHiVMRCyzMmMWIYskggiIzcqB3YZdAKF4MKhPf90V9Ip+2Z/s5MzwzM9/Wo6prv9/Pj+/18WuxXf7/97W8rIjAzs+z5d4M9ADMzGxwOADOzjHIAmJlllAPAzCyjHABmZhl10mAPoCdGjRoVdXV1gz0MM7MTyqZNm/ZFRE1x+QkVAHV1dbS1tQ32MMzMTiiSXitV7lNAZmYZ5QAwM8soB4CZWUal+gxAUjPwz0AVcE9E3FJUr6R+OnAA+FJE/CqpawFmAHsjYlxRv+uBhcBh4ImI+HrfpmNmWXLo0CFyuRzvvvvuYA/luDBixAhqa2sZNmxYqvZlA0BSFXAncBGQAzZKao2I3xQ0mwY0JI9zgOXJX4DvA98DVhVt9wJgJnB2RLwn6dRUIzYzS+RyOUaOHEldXR3596HZFRG8+eab5HI56uvrU/VJcwpoEtAeEbsi4iCwmvwLd6GZwKrI2wBUSxqdDOqnwFsltnstcEtEvJe025tqxGZmiXfffZdTTjkl8y/+AJI45ZRTenQ0lCYAxgBvFKznkrKetil2OjBF0ouSnpP0yVKNJM2T1CaprbOzM8VwzSxL/OL/Jz19LtIEQKktFt9DOk2bYicBHwImAzcAD6vE6CNiZUQ0RURTTc37vsdgZma9lOZD4BwwtmC9FtjdizaltvujyP8gwS8lHQFGAX6bb2a9UrfkiYpur+OWSyq6vUpYu3Ytp59+Oo2NjX3eVpoA2Ag0SKoHfgvMAv5rUZtWYKGk1eQ//N0fEXvKbHct8BngJ5JOB4YD+3oy+J7o7T+M4/EfgJll0+HDh1m7di0zZsyoSACUPQUUEYfJX6q5HtgOPBwR2yTNlzQ/abYO2AW0A3cD1x3tL+lB4AXgDEk5SVcnVS3A30jaSv6D5TnhnyczsxPQD37wAyZNmsT48eO55ppreO2112hoaGDfvn0cOXKEKVOm8NRTT9HR0cHHPvYx5syZw9lnn81ll13GgQMHANi0aRPnnXceEydO5OKLL2bPnvx76PPPP59vfOMbnHfeedx66620trZyww03MH78eF599dU+jTvV9wAiYh35F/nCshUFywEs6KLv7C7KDwKXpx6pmdlxaPv27Tz00EM8//zzDBs2jOuuu47nnnuOxYsXM3/+fM455xwaGxuZOnUqHR0d7Nixg3vvvZdzzz2Xq666irvuuouvfvWrXH/99Tz++OPU1NTw0EMPceONN9LS0gLA7373O5577jkAdu7cyYwZM7jsssv6PPYT6mZwZmbHm2eeeYZNmzbxyU/mL2R85513OPXUU/nWt77FI488wooVK9i8efOx9mPHjuXcc88F4PLLL2fZsmU0NzezdetWLrroIgD++Mc/Mnr06GN9vvCFL/TL2B0AZmZ9EBHMmTOHm2+++c/KDxw4QC6XA+Dtt99m5MiRwPsv1ZRERHDWWWfxwgsvlNzHBz7wgX4Yue8FZGbWJxdeeCFr1qxh7978d1nfeustXnvtNRYvXswXv/hFvv3tbzN37txj7V9//fVjL/QPPvggn/rUpzjjjDPo7Ow8Vn7o0CG2bdtWcn8jR47k97//fUXG7iMAMxsyBuOqvcbGRm666SamTp3KkSNHGDZsGLfffjsbN27k+eefp6qqikcffZT77ruPCy64gDPPPJP777+fa665hoaGBq699lqGDx/OmjVr+MpXvsL+/fs5fPgwixYt4qyzznrf/mbNmsXcuXNZtmwZa9as4bTTTuv12HUiXXjT1NQUvf1BGF8Gajb0bN++nTPPPHOwh5FaR0cHM2bMYOvWrf22j1LPiaRNEdFU3NangMzMMsoBYGY2QOrq6vr13X9POQDMzDLKAWBmllEOADOzjHIAmJlllL8HYGZDx7M3l2/TExcs7XXXZcuWsXz5ciZMmEBLSwuXXHIJ+/btY+nSpSxfvpzbbruNpqYmpk+fzg9/+EOqq6srOPB0HABmZv3grrvu4sknn6S+vp4NGzZw6NChY/cEWr58+bF269at62oT/c6ngMzM+uj2229n3LhxjBs3jjvuuIP58+eza9cuLr30Um699VYuv/xyNm/eXPIWznV1dezbt4+Ojg7OPPNM5s6dy1lnncXUqVN55513AHj11Vdpbm5m4sSJTJkyhZdffrki43YAmJn1waZNm7jvvvt48cUX2bBhA3fffTfXXHMNH/nIR3j22WdZvHgx99xzD1OmTGHz5s3d3rph586dLFiwgG3btlFdXc2jjz4KwLx58/jud7/Lpk2buO2227juuuu63EZP+BSQmVkf/PznP+ezn/3ssTt2fu5zn+NnP/tZr7ZVX1/P+PHjAZg4cSIdHR28/fbb/OIXv+Dzn//8sXbvvfde3weOA8DMrE8qeT+1k08++dhyVVUV77zzDkeOHKG6uvrPflOgUlKdApLULGmHpHZJS0rUS9KypH6LpAkFdS2S9iY//Vhq2/8gKSSN6v00zMwGx6c//WnWrl3LgQMH+MMf/sBjjz3GlClTKrb9D37wg9TX1/PII48A+cB56aWXKrLtskcAkqqAO4GLgBywUVJrRPymoNk0oCF5nAMsT/4CfB/4HrCqxLbHJtt9vfdTMDNL9OGyzd6aMGECX/rSl5g0aRIAX/7yl/nEJz5R0X088MADXHvttdx0000cOnSIWbNm8fGPf7zP2y17O2hJ/wn4VkRcnKwvBYiImwva/G/gJxHxYLK+Azg/IvYk63XAjyNiXNG21wD/A3gcaIqIfd2NxbeDNrNCJ9rtoAdCpW8HPQZ4o2A9l5T1tE3xgC4FfhsR3R7LSJonqU1SW2dnZ4rhmplZGmkCQCXKig8b0rT5U2PpL4AbgW+W23lErIyIpohoqqmpKdfczMxSShMAOWBswXotsLsXbQqdBtQDL0nqSNr/StJfpxiPmdkxJ9KvGva3nj4XaQJgI9AgqV7ScGAW0FrUphW4IrkaaDKw/+j5/y4G+euIODUi6iKijnyATIiIf+3R6M0s00aMGMGbb77pECD/4v/mm28yYsSI1H3KXgUUEYclLQTWA1VAS0RskzQ/qV8BrAOmA+3AAeDKo/0lPQicD4ySlAP+e0Tcm3qEZmZdqK2tJZfL4c8H80aMGEFtbW3q9qm+CBYR68i/yBeWrShYDmBBF31np9h+XZpxmJkVGjZsGPX19YM9jBOW7wVkZpZRDgAzs4xyAJiZZZQDwMwsoxwAZmYZ5QAwM8soB4CZWUY5AMzMMsoBYGaWUQ4AM7OMcgCYmWWUA8DMLKMcAGZmGeUAMDPLKAeAmVlGOQDMzDLKAWBmllGpAkBSs6QdktolLSlRL0nLkvotkiYU1LVI2itpa1Gff5L0ctL+MUnVfZ+OmZmlVTYAJFUBdwLTgEZgtqTGombTgIbkMQ9YXlD3faC5xKafBsZFxNnAK8DSng7ezMx6L80RwCSgPSJ2RcRBYDUws6jNTGBV5G0AqiWNBoiInwJvFW80Ip6KiMPJ6gYg/S8Zm5lZn6UJgDHAGwXruaSsp226cxXwZKkKSfMktUlq6+zs7MEmzcysO2kCQCXKohdtSm9cuhE4DDxQqj4iVkZEU0Q01dTUpNmkmZmlcFKKNjlgbMF6LbC7F23eR9IcYAZwYUSkCgwzM6uMNEcAG4EGSfWShgOzgNaiNq3AFcnVQJOB/RGxp7uNSmoGFgOXRsSBXozdzMz6oGwAJB/ULgTWA9uBhyNim6T5kuYnzdYBu4B24G7guqP9JT0IvACcISkn6eqk6nvASOBpSZslrajUpMzMrLw0p4CIiHXkX+QLy1YULAewoIu+s7so/9v0wzQzs0rzN4HNzDLKAWBmllEOADOzjHIAmJlllAPAzCyjHABmZhnlADAzyygHgJlZRjkAzMwyygFgZpZRDgAzs4xyAJiZZZQDwMwsoxwAZmYZ5QAwM8soB4CZWUY5AMzMMipVAEhqlrRDUrukJSXqJWlZUr9F0oSCuhZJeyVtLerzYUlPS9qZ/P1Q36djZmZplQ0ASVXAncA0oBGYLamxqNk0oCF5zAOWF9R9H2guseklwDMR0QA8k6ybmdkASXMEMAloj4hdEXEQWA3MLGozE1gVeRuAakmjASLip8BbJbY7E7g/Wb4f+PveTMDMzHonTQCMAd4oWM8lZT1tU+yvImIPQPL31BRjMTOzCkkTACpRFr1o0yuS5klqk9TW2dlZiU2amRnpAiAHjC1YrwV296JNsX87epoo+bu3VKOIWBkRTRHRVFNTk2K4ZmaWRpoA2Ag0SKqXNByYBbQWtWkFrkiuBpoM7D96eqcbrcCcZHkO8HgPxm1mZn1UNgAi4jCwEFgPbAcejohtkuZLmp80WwfsAtqBu4HrjvaX9CDwAnCGpJykq5OqW4CLJO0ELkrWzcxsgJyUplFErCP/Il9YtqJgOYAFXfSd3UX5m8CFqUdqZmYV5W8Cm5lllAPAzCyjHABmZhnlADAzyygHgJlZRjkAzMwyygFgZpZRDgAzs4xyAJiZZZQDwMwsoxwAZmYZlepeQHYcefbmgd3fBUsHdn9mNmB8BGBmllEOADOzjHIAmJlllAPAzCyjHABmZhnlADAzy6hUASCpWdIOSe2SlpSol6RlSf0WSRPK9ZU0XtIGSZsltUmaVJkpmZlZGmUDQFIVcCcwDWgEZktqLGo2DWhIHvOA5Sn6fgf4x4gYD3wzWTczswGS5ghgEtAeEbsi4iCwGphZ1GYmsCryNgDVkkaX6RvAB5Pl/wDs7uNczMysB9J8E3gM8EbBeg44J0WbMWX6LgLWS7qNfBD951I7lzSP/FEFH/3oR1MM18zM0kgTACpRFinbdNf3WuBrEfGopP8C3Av83fsaR6wEVgI0NTUV7/e4VbfkiV737bjlkgqOxMystDSngHLA2IL1Wt5/uqarNt31nQP8KFl+hPzpIjMzGyBpAmAj0CCpXtJwYBbQWtSmFbgiuRpoMrA/IvaU6bsbOC9Z/gyws49zMTOzHih7CigiDktaCKwHqoCWiNgmaX5SvwJYB0wH2oEDwJXd9U02PRf4Z0knAe+SnOc3M7OBkep20BGxjvyLfGHZioLlABak7ZuU/xyY2JPBmplZ5fibwGZmGeUAMDPLKAeAmVlGOQDMzDLKAWBmllEOADOzjHIAmJlllAPAzCyjHABmZhnlADAzyygHgJlZRjkAzMwyygFgZpZRDgAzs4xyAJiZZZQDwMwsoxwAZmYZlSoAJDVL2iGpXdKSEvWStCyp3yJpQpq+kq5P6rZJ+k7fp2NmZmmV/UlISVXAncBFQA7YKKk1In5T0Gwa0JA8zgGWA+d011fSBcBM4OyIeE/SqZWcWKXULXlisIdgZtYv0hwBTALaI2JXRBwEVpN/4S40E1gVeRuAakmjy/S9FrglIt4DiIi9FZiPmZmllCYAxgBvFKznkrI0bbrrezowRdKLkp6T9MlSO5c0T1KbpLbOzs4UwzUzszTSBIBKlEXKNt31PQn4EDAZuAF4WNL72kfEyohoioimmpqaFMM1M7M0yn4GQP5d+9iC9Vpgd8o2w7vpmwN+FBEB/FLSEWAU4Lf5ZmYDIM0RwEagQVK9pOHALKC1qE0rcEVyNdBkYH9E7CnTdy3wGQBJp5MPi319npGZmaVS9gggIg5LWgisB6qAlojYJml+Ur8CWAdMB9qBA8CV3fVNNt0CtEjaChwE5iRHA3Y8efbmgd3fBUsHdn9mGZbmFBARsY78i3xh2YqC5QAWpO2blB8ELu/JYM3MrHL8TWAzs4xyAJiZZZQDwMwsoxwAZmYZlepDYDsx3PHMK73qt+jC0ys8EjM7EfgIwMwsoxwAZmYZ5QAwM8soB4CZWUY5AMzMMsoBYGaWUQ4AM7OMcgCYmWWUA8DMLKMcAGZmGeVbQRyH6pY80WXdopN6d7sHM7NiPgIwM8uoVAEgqVnSDkntkpaUqJekZUn9FkkTetD3HySFpFF9m4qZmfVE2QCQVAXcCUwDGoHZkhqLmk0DGpLHPGB5mr6SxgIXAa/3eSZmZtYjaY4AJgHtEbEr+R3f1cDMojYzgVWRtwGoljQ6Rd//BXwd8I/Bm5kNsDQBMAZ4o2A9l5SladNlX0mXAr+NiJe627mkeZLaJLV1dnamGK6ZmaWRJgBUoqz4HXtXbUqWS/oL4Ebgm+V2HhErI6IpIppqamrKDtbMzNJJEwA5YGzBei2wO2WbrspPA+qBlyR1JOW/kvTXPRm8mZn1XpoA2Ag0SKqXNByYBbQWtWkFrkiuBpoM7I+IPV31jYhfR8SpEVEXEXXkg2JCRPxrpSZmZmbdK/tFsIg4LGkhsB6oAloiYpuk+Un9CmAdMB1oBw4AV3bXt19mYmZmPZLqm8ARsY78i3xh2YqC5QAWpO1bok1dmnGYmVnl+JvAZmYZ5QAwM8soB4CZWUY5AMzMMsoBYGaWUQ4AM7OMcgCYmWWUA8DMLKMcAGZmGeUAMDPLKAeAmVlGOQDMzDIq1c3grLRFJ60Z7CGYmfWajwDMzDLKAWBmllEOADOzjHIAmJllVKoAkNQsaYekdklLStRL0rKkfoukCeX6SvonSS8n7R+TVF2ZKZmZWRplA0BSFXAnMA1oBGZLaixqNg1oSB7zgOUp+j4NjIuIs4FXgKV9no2ZmaWW5ghgEtAeEbsi4iCwGphZ1GYmsCryNgDVkkZ31zcinoqIw0n/DUBtBeZjZmYppQmAMcAbBeu5pCxNmzR9Aa4CnkwxFjMzq5A0AaASZZGyTdm+km4EDgMPlNy5NE9Sm6S2zs7OFMM1M7M00gRADhhbsF4L7E7Zptu+kuYAM4AvRkRxqAAQESsjoikimmpqalIM18zM0kgTABuBBkn1koYDs4DWojatwBXJ1UCTgf0Rsae7vpKagcXApRFxoELzMTOzlMreCygiDktaCKwHqoCWiNgmaX5SvwJYB0wH2oEDwJXd9U02/T3gZOBpSQAbImJ+JSdn/euOZ17pVb9FF57edeWzN/dyNH1wgS9As2xKdTO4iFhH/kW+sGxFwXIAC9L2Tcr/tkcjNTOzivI3gc3MMsoBYGaWUQ4AM7OMcgCYmWWUfxHMzKynhsjVaj4CMDPLKAeAmVlGOQDMzDLKAWBmllH+ENh6fUuHgdZf47xj/RNd1nXcckm/7NPseOAjADOzjHIAmJlllAPAzCyjHABmZhnlADAzyyhfBWSZt+ikNV1XPrtl4AYylAz0j+wMxq0ZhgAfAZiZZZQDwMwso1IFgKRmSTsktUtaUqJekpYl9VskTSjXV9KHJT0taWfy90OVmZKZmaVRNgAkVQF3AtOARmC2pMaiZtOAhuQxD1ieou8S4JmIaACeSdbNzGyApDkCmAS0R8SuiDgIrAZmFrWZCayKvA1AtaTRZfrOBO5Plu8H/r6PczEzsx5IcxXQGOCNgvUccE6KNmPK9P2riNgDEBF7JJ1aaueS5pE/qgB4W9KOMuMdBewr06YivjYQO0lvwObdV1/7nxXdXL/Ou8JjraTj/L/3N/prw8f5vPvTN/oy9/9YqjBNAKhEWaRsk6ZvtyJiJbAybXtJbRHR1JN9DAWed7Z43tnTH3NPcwooB4wtWK8Fdqds013ff0tOE5H83Zt+2GZm1ldpAmAj0CCpXtJwYBbQWtSmFbgiuRpoMrA/Ob3TXd9WYE6yPAd4vI9zMTOzHih7CigiDktaCKwHqoCWiNgmaX5SvwJYB0wH2oEDwJXd9U02fQvwsKSrgdeBz1doTqlPFw0xnne2eN7ZU/G5K6JHp+TNzGyI8DeBzcwyygFgZpZRQyYAyt2u4kQnqUXSXklbC8q6vJ2GpKXJc7FD0sWDM+q+kTRW0rOStkvaJumrSflQn/cISb+U9FIy739Myof0vI+SVCXpXyT9OFnPyrw7JP1a0mZJbUlZ/849Ik74B/kPmF8F/gYYDrwENA72uCo8x08DE4CtBWXfAZYky0uAW5PlxuQ5OBmoT56bqsGeQy/mPBqYkCyPBF5J5jbU5y3gL5PlYcCLwOShPu+C+f834IfAj5P1rMy7AxhVVNavcx8qRwBpbldxQouInwJvFRV3dTuNmcDqiHgvIv4v+auzJg3IQCsoIvZExK+S5d8D28l/u3yozzsi4u1kdVjyCIb4vAEk1QKXAPcUFA/5eXejX+c+VAKgq1tRDHV/djsN4OjtNIbc8yGpDvgE+XfDQ37eyWmQzeS/IPl0RGRi3sAdwNeBIwVlWZg35EP+KUmbklvgQD/Pfaj8IlifbzkxxAyp50PSXwKPAosi4v9JpaaXb1qi7IScd0T8ERgvqRp4TNK4bpoPiXlLmgHsjYhNks5P06VE2Qk37wLnRsTu5L5oT0t6uZu2FZn7UDkCSHO7iqGoq9tpDJnnQ9Iw8i/+D0TEj5LiIT/voyLid8BPgGaG/rzPBS6V1EH+NO5nJP2AoT9vACJid/J3L/AY+VM6/Tr3oRIAaW5XMRR1dTuNVmCWpJMl1ZP/nYZfDsL4+kT5t/r3Atsj4vaCqqE+75rknT+S/j3wd8DLDPF5R8TSiKiNiDry/w//n4i4nCE+bwBJH5A08ugyMBXYSn/PfbA/+a7gJ+jTyV8l8ipw42CPpx/m9yCwBzhEPv2vBk4h/2M6O5O/Hy5of2PyXOwApg32+Hs550+RP6zdAmxOHtMzMO+zgX9J5r0V+GZSPqTnXfQcnM+frgIa8vMmfwXjS8lj29HXsP6eu28FYWaWUUPlFJCZmfWQA8DMLKMcAGZmGeUAMDPLKAeAmVlGOQDMzDLKAWBmllH/H9bkZ6UZ0EM8AAAAAElFTkSuQmCC\n", 345 | "text/plain": [ 346 | "
" 347 | ] 348 | }, 349 | "metadata": { 350 | "needs_background": "light" 351 | }, 352 | "output_type": "display_data" 353 | } 354 | ], 355 | "source": [ 356 | "plt.hist(cum_rs, density=True, label='expert')\n", 357 | "plt.hist(awac_cum_rs, \n", 358 | " alpha=0.5, \n", 359 | " density=True, label='offline')\n", 360 | "plt.legend()" 361 | ] 362 | }, 363 | { 364 | "cell_type": "markdown", 365 | "metadata": {}, 366 | "source": [ 367 | "## Online fine tuning" 368 | ] 369 | }, 370 | { 371 | "cell_type": "code", 372 | "execution_count": 14, 373 | "metadata": {}, 374 | "outputs": [ 375 | { 376 | "data": { 377 | "text/plain": [ 378 | "AWAC(\n", 379 | " (critic): MLP(\n", 380 | " (hidden_act): ReLU()\n", 381 | " (out_act): ReLU()\n", 382 | " (layers): ModuleList(\n", 383 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 384 | " (1): ReLU()\n", 385 | " (2): Linear(in_features=128, out_features=128, bias=True)\n", 386 | " (3): ReLU()\n", 387 | " (4): Linear(in_features=128, out_features=2, bias=True)\n", 388 | " (5): ReLU()\n", 389 | " )\n", 390 | " )\n", 391 | " (critic_target): MLP(\n", 392 | " (hidden_act): ReLU()\n", 393 | " (out_act): ReLU()\n", 394 | " (layers): ModuleList(\n", 395 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 396 | " (1): ReLU()\n", 397 | " (2): Linear(in_features=128, out_features=128, bias=True)\n", 398 | " (3): ReLU()\n", 399 | " (4): Linear(in_features=128, out_features=2, bias=True)\n", 400 | " (5): ReLU()\n", 401 | " )\n", 402 | " )\n", 403 | " (actor): MLP(\n", 404 | " (hidden_act): ReLU()\n", 405 | " (out_act): Identity()\n", 406 | " (layers): ModuleList(\n", 407 | " (0): Linear(in_features=4, out_features=128, bias=True)\n", 408 | " (1): ReLU()\n", 409 | " (2): Linear(in_features=128, out_features=64, bias=True)\n", 410 | " (3): ReLU()\n", 411 | " (4): Linear(in_features=64, out_features=2, bias=True)\n", 412 | " (5): Identity()\n", 413 | " )\n", 414 | " )\n", 415 | ")" 416 | ] 417 | }, 418 | "execution_count": 14, 419 | "metadata": {}, 420 | "output_type": "execute_result" 421 | } 422 | ], 423 | "source": [ 424 | "agent.to('cpu')" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 15, 430 | "metadata": {}, 431 | "outputs": [], 432 | "source": [ 433 | "refresh_memory = False\n", 434 | "if refresh_memory:\n", 435 | " memory.reset()" 436 | ] 437 | }, 438 | { 439 | "cell_type": "code", 440 | "execution_count": 16, 441 | "metadata": {}, 442 | "outputs": [], 443 | "source": [ 444 | "online_budget = 600\n", 445 | "online_awac_cum_rs = []\n", 446 | "for n_epi in range(online_budget):\n", 447 | " s = env.reset()\n", 448 | " cum_r = 0\n", 449 | "\n", 450 | " while True:\n", 451 | " s = torch.tensor((s,), dtype=torch.float)\n", 452 | " a = int(agent.get_action(s).squeeze())\n", 453 | " ns, r, done, info = env.step(a)\n", 454 | " \n", 455 | " experience = (s,\n", 456 | " torch.tensor(a).view(1, 1),\n", 457 | " torch.tensor(r).view(1, 1),\n", 458 | " torch.tensor(ns).view(1, 4),\n", 459 | " torch.tensor(done).view(1, 1))\n", 460 | " memory.push(experience)\n", 461 | " \n", 462 | " s = ns\n", 463 | " cum_r += 1\n", 464 | " if done:\n", 465 | " online_awac_cum_rs.append(cum_r)\n", 466 | " break\n", 467 | " \n", 468 | " if len(memory) >= batch_size:\n", 469 | " sampled_exps = memory.sample(batch_size)\n", 470 | " _s, _a, _r, _ns, _done = prepare_training_inputs(sampled_exps, device=fit_device)\n", 471 | " agent.to(fit_device)\n", 472 | " critic_loss = agent.update_critic(_s,_a,_r,_ns,_done)\n", 473 | " actor_loss = agent.update_actor(_s,_a)\n", 474 | " critic_losses.append(critic_loss.detach())\n", 475 | " actor_losses.append(actor_loss.detach())\n", 476 | " agent.to('cpu')" 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 17, 482 | "metadata": {}, 483 | "outputs": [ 484 | { 485 | "data": { 486 | "text/plain": [ 487 | "[]" 488 | ] 489 | }, 490 | "execution_count": 17, 491 | "metadata": {}, 492 | "output_type": "execute_result" 493 | }, 494 | { 495 | "data": { 496 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO19e7xdRXn28+5zkpxcyZUQEsI1IIlyMyIaBRSRm3KpolHxo4rSWrRarSjVtrT96M+2ttVetPJ5KRZFqYpSq1hA8V4xICB3IuESiCQEciPJyTlnz/fHWmvvWbPemXlnXfYt6/n9krP3WrNmZs+aedczz/vOLFJKoUaNGjVqDBYa3a5AjRo1atQoH7Vxr1GjRo0BRG3ca9SoUWMAURv3GjVq1BhA1Ma9Ro0aNQYQw92uAADMnz9fHXTQQd2uRo0aNWr0FW677banlVILuHM9YdwPOuggrFmzptvVqFGjRo2+AhE9ajtXyzI1atSoMYCojXuNGjVqDCBq416jRo0aA4jauNeoUaPGAKI27jVq1KgxgBAZdyJ6hIh+TUR3ENGa+NhcIrqRiB6K/87R0l9GRGuJ6AEiOq2qyteoUaNGDR4hzP0VSqljlFIr4+8fBnCzUmoZgJvj7yCi5QBWA1gB4HQAnyKioRLrXKNGjRo1PCgS534OgJPjz1cBuAXAh+LjX1FKjQJYR0RrARwP4OcFygrCd3+9AccfPBdPbRvFrrEJvPDAOanz655+Dk9u2YVVh82vpgJP3QOM7gCWvjj6vvYmYN4yYPZS4JefBXZsBJaeAGz+DTCyD3D0G6PP93wDOO53gYe+Bxz1RmBsJ3DHNcCs/YHlZ7fz3/Mc8IvPAIedAiw6Wiv3XmB0W5Q3ADz3NPDoT4Hl5wC//hqw7NXAyKzo+vu+DUybCyw4IqrXnV8FZh8ADE0Blrwwuv7XXwOUAl7weuDOrwArzgUmTY3OPXQTsODw6NoEj/8SeOI2gAhYdiow95DoeLMJ3PllYM5BwNQ5wMIV0XGlovbY+QxADaAxBKgmcOTZUd5Pr43abv9jABAweTqw3/OBu/4zKmPF7wC/+DQwMQYccnKU7qEbo3R7dkbtc8eXovzuvAaYNg+YPANYdFTUpr/9NTC2Czjg+PZv2PI4sOl+4LBXAXfEdR6ZBez3gnaaTQ8CD/8AGN8d1XvhiujYEWcAzXFgy2PAoa+I2mN4SlTeg98DpsyM7ukhr4za44gzgTWfj9rk8NOifnP4acCGu4DR7dH3kX2i88+ui9pr3yOB9bdG92naPOCYNwMP3gAsOgZ45CfR9SOzgLU3A5vXAs87K/rtR78Z2GcxMDEefZ9zIDBjv6idlQLWfA6YfWDUJx7/JXDwicDC5dm+PbY76qcz94vaJrnHT9wOrPthVM4dV0e/bd8js9c/8tOo3vs+L2rfydOj/jkxBtz11ej6+/8L2HgfcMxboj6Z4NGfRW2z7FRg3qFR/zzslKh9Hr8VmDQt6h8J7vpP4NBXRvUZ3QE0hoGZC6PfdvfXo7H27DrgsZ8DL34XMDwZeOAGYMujwPNfB0yP7cOuLVE/nTYPWPm26NgdXwa2rgeGJgNQUb963muADXcAy04Dbv1MdA8nTQVe/PvA0KR0OzQnojyOeXPU7yuA1LgrAP9DRArAZ5RSVwJYqJTaAABKqQ1EtG+cdjGA/9WuXR8fS4GILgZwMQAsXbrUPJ0bzzy3B+/60u04duls/OqxLQCARz52VirNKz5+C3u8NHz6pdHfy7dGf69+XTQY33Mb8J0/zqafewhw+78Dv7oaWPt94LGfAdueBJ68A3jgv6M0H3oUmDo7+vzIT4Cb/wK4/9vAO7+vlfuSbLkb7gDe9l3g6xdFg+gNXwS++yHgV/8RpZk8A7jgG8B1F7fzSa7/+kXR32lzgW/+PvDk7cCZfxcd+9LrgOGpwEd/277uc69qf/6uls9dXwG+dUk2/21P8O2x4Q7gjVcD//LC7LnLtwLfeEf0eeEK4Ht/kj73pde3v7/uc1G5etkAMGsx8P57gX97Wbo+APDpVcDoVuD1XwC+9QfZOgPAv74oWy8AuOnPI4OfpE/a4/KtwJff0E537qejOu33gugBAwD/rZXzmZfz+XOYtQi4ZnX7+4rzgPP/Hbj6d6Lvv70r6lcKwEkfBP73U8CNf5r+XTueAv77A9m89d+c4Pt/Bfz8X7JpvvkHwKb7gGceBm7/IvDoz4ELvpa9/t/PjP5+dCPwzXe18/jZPwE3/yUAAr57KbBnR/T55A+1r/3CGdHfGz4EvOvncZ8+F3jDVcDnTk3XZ8NdUT+ZuT+w/cl0HVacB9xzXfRQ+NHfR+eXviR6yF/zxijNXV9tj621N0W/G4jG0ObftOuu44bLgOYY8O41wA0fbh8/cBWw+Lh02l9+NvqdY7uAF1+MKiA17quUUk/GBvxGIrrfkZaYY5k3gsQPiCsBYOXKlaW9MWR8ogkAWP/srrKyLAcToxGr47B7a8SIAOC5Te2/29a306hm+3OSz87N7jI3PRD9HdsZ/d36RPR3+4Z2mj07gD3b3fmMbov+7ngqfXxc2Ma7nuWP29pjdIc9L/3lMrbrE+zewh/f9oT9mtGt7mtdSAy7D0l7bP+tO50Eo8a922r8tqRfqYno786ns3n42lGHrc7PbYz+ju+J67XNnU9zwrg+rtfuLcDEHn+9tjwW/d1j6StJnzcNO9DuX82J9nmzrCR/81xz3D5emmN8XuZvBdpjd9czfF4lQKS5K6WejP9uBHAdIpnlKSJaBADx3/juYj0AbS6FJQCYFt4bYXmGpTqD4DknfXtWy/hyz9seQJ63gOnX9O1bxOL7oT+wK0OJ/UlUnPQ3WcpUqm3cXXVPHibT97WnsRbdbJell2tNH9jnMmm600+9xp2IphPRzOQzgFcDuBvA9QAujJNdCOBb8efrAawmoilEdDCAZQBuLbviNvTlcG+ORRoyIBwcob+yV1slT72U5XMfogyjauZBxoO89P7kSZv3gZX8jpZhh7t9dsTGfUYO4976DXn6ksS4G23QJRIikWUWAriOok4zDODLSqkbiOiXAK4loosAPAbgfABQSt1DRNcCuBfAOIBLlFLMvKRa9CRXtd1kW4e2pW8dF/7Kon2rqs6Zi7kHGI9eZfYtA1xG/UKNLdNnQtrJl1Z6fyRjwcncY/lyui0owjE2krLFjDyUuZtt0KPGXSn1MICjmeObAZxiueYKAFcUrt3eguYEwqbqHWburTrlfWSGXlfiFLmXUQpz9/QXToLIJipej6DymDKTB57ut3DlsTv2jTQm2dNYi+bazCxL67OZenj6c48QkHqFai9gQpdlKtBIi2rbRWFKBUXKSA2cPjfuZdTfy6Q5CaJQgQXPJ8ks6caFzD20PPYagyhY2zJQvhHJMtVrC7Vx7yRsnSfxsgNClhxq3AVT88w1nTCcPtnJc03fM/cy8ihBJilFljFmnrYHejsj/rA04shbH8E1yjDaZRGrfnGo9ht6e7zbdMYxZA2usqfP/SNDWFwHjOhey9wdElxwmxjpzes7JstwBjPk8kCHaikzEqERNh8Cofl2ySgNnHFP4CUO3YCVuVtCIa19Ij4R+iNzM7SqOmdBuai3n+RtWCNamPqXLbm1HiCOPhNUplQG8uCua/njUoeqrzzX2LCFQkpkmdIcqtX33YE17n2FCUEoJLkcPD4Udah2mrkPuCzDygKtk2F5bbjTfT5ZQNOptpI6E7mVyYDcodpOJCsvdQnH+oVOfCg/qQoJhayQhdbGvaNwMfcAh2pwsUWcTtrn0jviXiLLWDXYEpj7//5r+nueOPduhEKaaEXLjOqZuQry1CPwWmXIoEX6eo+EQtbGvZNwyTIhi5g6wdw7IX8UXqHaG4tF/AjRwUv+Da0lJmVp7hUZ91DNPSSN7RqxQ7WgLFNr7uVA9TSbszlU9Q5tYaZFNPDCzL1IPoA9Oqcgc69yEFWZl8sZWLYhcEpAFUAc525BKhRSVCB/WKK5Z/xbJTlU6zj3akG9uUaVhx4tIwmFDF2hWpS5VwXvClzf9VVOf6vMKzG4ksU0BdE0mXvFK1SL1n9Ck2WqYu5sZE8Ic/dp7jYZTkcd5z5YCI2W8WdYrFxx3lUZ+pLlIlsYYB5Uytwd7LZoubnaoMzfWnAztFScewcdquJ2L9mhWiEGzrj3rOwKwNoRdeNuGxylhCYGMH5O2y7boVo0zt3H3Hu1M5S+atRVlkAmKaWdQhcxWTDeAc3dxtwlskxpoZDVY/CMe7cr4IKtY0zoK1QL5GO/IDB93ms6UYbjIVfmysBSHww2WaYTDlVGXy5SpjdapujMQ99jsETmTpqpYzV3U5bRH04FfVw1cy8HKm7InlzElICMZm+Ood2BtL9Wtt4Bh2rPRst0irl3QJbpiENVEOdeqgRVUJYR+1t86UwD4FsnImTuEtTMvRr03EycM9CmcZ8Yb6eTOBkLO1RDGVHBRrU+aXMMJlcopDR/Capk7q17XYFD1Wxr06HK3osy/Dyu3xSCUAkkB3Nv1TVHXnkWVnXJJg2cce9p2IyyztyrdIDljYqoappZmLl78utV5i5eDVlF2QXLlMoyeX9HsG+pRFmmrPKCHnDVWf6BM+79wdxN464zd1vHKCCTBE9hjfIqc1QW1Ny9098eZ+6StIWL7vD2A7mjV0xZErI6S4tLbd/B5S+dLSqE7+fO5N0B3XjwjHvPuVS5G8s1u2ncS9zy15veo/0Gy0BC5LLtDuPeq8w9k3WFzL3yUEgfcy9Lcyd/WZL6tMA5SI0+Lolzz8Pcqwh5FWDwjHtCjrtbjTY4I8ka91Yi/jObT4lvOJLWIzgfaRnC/J17y/Qocw+K4il50Gc2DuuR1+xl8jR2yqRGMUnJHBspWYaTjoSyTOmhkPXGYWL0Gm/njaRjgElkmU5Ey3Qiv6KLqypl7mUioF6l36t+WMSksh+pAVm9imjuejYO5h7qB6gXMVUD1TMDmoGTcXNswpNPboQuYuo0c3ddEhAKWaieA8LcJXHuQf3JlrbAIiZbRFkR34R5LRctkyFMOWaSbPI6FLIShDbjyX/3A6y+8ueV1AUA30FNWUZnDVWsUC0zPC0XAt+hKg6F9DDiXtl+IMPcHfWqirl3yombu80NctMYgrNeoVE5Kck98GUdZjrv9gO9MYMc7kqpFaJNjmXM4ZHNO/HI5p1V1ij72RlrXAFzz62DWsoLLT80zt0FZ5x7hQa5UFYlPnR8sO7nXhJzl4ZCBqEIcxdq7rp1z2ym5sjHWq4rSc3cK0KPyTIpY5R8EDq1rGzdod2zdfAZE+HCFtfr4fJAFJ3gOFel5l4l2+qm5l7ZIiajvOAHRsCuqHr+YubOaO5O6TFVmDCdkb8I1fWzgTPuvSe5u4ykniaAEfUbc7cXkuOSELbVI5p7kC+gYLnmvenYa/bi/Dfem/9aQJt6+xyqATPdVn7JJdxsxpBlrK+1zDGzZqXZOs49GL1n2y1TTmuaCpAZ4BJj45tJlIA8+aUcqp7fIZ5pV/xbQx6OlTF3V5nlFhkMxZAbIlk7iZk7Y6zFju6izL2WZUpB7zF3Da44d2/Fy2DuZWmrXWTurnbIq22zv7XKjtSJtk2ykzD3EjX3wghl7lLoxj0wFDIYVc10wzB4xr3rNMSEVCsPkGWCf2NBI1pVZ80VLeMKhcwkllYkrA6hCHnodCPOvZRQyCKwyDJO2162LOPIKzP2Sth+oAMYPOPec7ZdqLeFsEdnvDyXvqjmXlVnLai5l+VQrfx1dyH1KrkDN01DVtShakMBDdkqXQraSUoQuulQrZl7Oeg54251FjnS+fIJjnM300sGIlfvHnsTk5d5FTDuRftR3o7YjTj3MkMhEwT1FY0NtxZBeUIhvczdZdwtoZCp8lzhu75Zds3cK0HPyTKsnMJEy5Q9wJzMu2AZ3YyWqSIUsgrN3Rlt5LofVckyZc0WirSpKy1njAP7uQvEaO6ZeySYBUi0+SrXMQRg8Ix7UJ/q8IOgCAMOjegIMu6BW/52M87dOZUuk7kXpu72evSz5l71mElp7gWYu8hv42LuSJ9jP/vqJqhLhRg4456gd16zx2nugmZnY+EdeTqrkMPx2M04d6fxczhUc0fLVKy5B7Vf2cbdiJbpnYGhgSEijYKau5Mg2B7mEubuyZvNv8eNOxENEdGviOjb8fe5RHQjET0U/52jpb2MiNYS0QNEdFoVFbehl0hIthDHAOMMgM9Z5ByozIDJ/Xsraqg80TIuWSZvVE8V02hndFMHB3vpskwFEEmXrmsE571x7iUajj50qL4XwH3a9w8DuFkptQzAzfF3ENFyAKsBrABwOoBPEdFQOdX1I0Rz70yTcwy4A6GQRTV3Z70DWy54b5mccpN4UYqgvDJlmRDmXrYhkKxQ7alQyPivbxFTK6G0DxWJczfHXuCDpUsQGXciWgLgLACf1Q6fA+Cq+PNVAM7Vjn9FKTWqlFoHYC2A48uprh89p7lLmLvESWPN05pI+1jUoVqRLGMdS1JDVNIgq0KWyS29lN0nA/uKN2mOB3IQdCIh6AdFIq4yfUn7btt+II9DtceZ+ycAXApAr/VCpdQGAIj/7hsfXwzgcS3d+vhYR6A/+HsWnQiFDIlTZ7NzMM/K49ylsoyvXlLjXgVzd9Sjkw5VWx06UqYQGf8OBTB3QZ4AHy0jJi2hY69PNHcieg2AjUqp24R5ijQHIrqYiNYQ0ZpNmzYJs/YjhI13vMmdce7Ca83PomtLDoUsC1bNXepQ9QzOrm4/kFOWqdy/0cVFTFLjSYTizN1xLSdVuRyqmXIHh7mvAnA2ET0C4CsAXklEVwN4iogWAUD8d2Ocfj2AA7TrlwB40sxUKXWlUmqlUmrlggULCvwEI9+QtJ1oc4mzyDrV83Vc10swOONiu85XdkWyTK4wtoAZibgaFYRC5nWo9jpzN9M+txm4fB9g59P5qmXmGczcpX3Ix9wdcouCLF0mfwEqNEJe466UukwptUQpdRAiR+n3lVIXALgewIVxsgsBfCv+fD2A1UQ0hYgOBrAMwK2l19xa34C0HeHujJHNs+VvKrlgShrC9H2yRFUdsGgYW09vP+Bi7h1cxCRCgTK3PBJQjOQBE8jc7QnSX1lZRppfoNwnIh3V68ZF3sT0MQDXEtFFAB4DcD4AKKXuIaJrAdwLYBzAJUq11vt2AAGyTMeZewzXa/ZaaYRhjtbzDMPN6wzb9YygTnkgYUquk4GDLqQeZTL3kGl6ZQ/STmeUU+oKZe7i9kq9Z4+51kGwMg5VT1F5SUbJCDLuSqlbANwSf94M4BRLuisAXFGwbrnQIscdeDLKwBijjOYu0fECmXhIKKSPuf/sn/3pnQh9h2pZmns3mbsr/x5j50WMj5UvBDwwldKIg5S5e+L3RWTJNNr6NfXeMj2H7jSjEDanligUkntICDtgrq1xXYOzrFbOI8u4fleI/OFJV5htuZi767KKJbCir9kT1y+EuRtGlholvKxDOkvQ0wvr581XMlOr3lINnnGvop8WAedYk2ruIgePr0wUZ+4h5zjYHkIuJmcvXPtY0iKmquPcg5hcVZ2yrPspbdNA5p4iDglzl9RD2odCmbstr1Aixn3Xq1WdwjCAxl3eUTvuUE3ALWLKXGayb+Yh4SozRJax5VE58jB318s6ctY59OEmy7T9sWm4nCqNc89zfRX3ughzF6yE9jJ3A94tP6SEQiLLmOSqlmVKQddlGdeNbU2NGc09hA2Iol8KyjJB0QM5kYe5hzgqe0ZzDzDuZZQbPBMrknfAPRTdj5i5ex2qnvIzx20MOdlHPkBuGpRQyH6DU1q0pC23Ai42mXzmmHuZU+NA5h7KfkprOJthkDLb0OmxLVnFG4eZ+ZtM3nZdLtiMYjekIGE5ZluRQJYRERwNEl+DmGyEGncuffUBH4Nn3ENCISupgKDDcJq7rzOy0TI2LbsJnumHyCBdZO7iKXLe/AWo1KFadZy7izUXNSoBDNdaByZtijgwzN1mYAs7VC1kITX+AmbNgH9G2SEMnHEPc/xX0OhO1hn/5eLcQwdD6i9zXZmhkIURuCukU0Lo1H7uBaHXpaOaew4UikIIcZYL+ysXCmklTbY8zQOBDlXrgyWPLFMb91LQnWbUKyCQZcrS3KWsJdciJpeBDW3lUCMuLNs7iArIMmUa2Yxhcq3pK6HcjsoyAfcwSHM304cydwMih2oets+lqZl7JWhp7qljfONW0+QOWcY2NQ5+FZqAuYc4VIOZe2D6UCMulS16+R2qqXoaxrxZNXMv6/6UWwXxg4CS/wowd5FDVSq3hDL3nCSjZAyecWcaMk9QRv4KCKZkklDIYpXwMFwmveiYw6GaJ7omV7RMSChkEeZeUKrJK8uU8VBxPayKLmKqQnPnpJ6i2w+ItHGh0c4QtJq5dwWs3bEmDsx8YgzYtSWsAhzj5rb8LVOWyas9B9Unc0EX8vIwdTFzr8C4u/KqXHMPzKOQ5h6STsrcue0HijJ3D5xG2yUPcclrzb0SsF2qrCfnV98K/M2BngoIBq7UoertbI7zRR2qoRptnja2Ppxc1zj2lrH5Grz16LRx74LmnmdmJUkbIrlJyUiy/YDz2sB2sr4gR1swlXHqWhDsUHUmDkgbhsEz7kzD27tfYMM++F1BBXLEuYfqeLmZewkD0XouTyfNw7ocD63cD3Hu95coy4QsYipMRGwbbpU5sxKgUJ6c/Gd5qIgfGFwaM71ElmHq4syYqQvQkVfFDaBxj/6SoPGqkcJy3FiJjsc+JKRGkJv2muW7yhOcy+Owsx6X7grZy9EyjoeQaxFTVSirsxfKp4AsY2XuUoLgte7CWYiEiIUGSFSDwTPu8Y1IR8vY0lZRAYeBaT95itVEwvKDHKo5yshekONcoNE3z3mjZRxVSqWr2qHa4UVMzns3IA7VFnO35BvqUBX3OQERC5pRVsfgi7ysoyfBO1T5xq18EZNpZG2yTB7PvvM6owOWpbm7th8ok7nnelAUSMv+/KLGXe8HnV7E5LifrG2vgI0H5Slg7tZZmY1JS8o38xdeE8zcq2fpHPYO497JtnWxgVDm7jWKjvMhDJdlWXzW9pM9wNzFzj7zsireoepaSduFUMjSHKoB9ZDWQcLcbWNJJGnC8VJ6CXsXMvxWmt4IhRw8415RWnmmAsbHRst4M85+Lou5e8szT1XM3KXM1jbgJfn40hWOlnG0v3PjsILFWjPJcX+8aUMjSoTliEIhA2ejvhWqmYcE8em4uvjK7hJzHzzNvcWO9WO2tJXUIP2Zk1M68Zq91HfP9gNihls2+8shy1ThUK06Wqaji5iQo2MXoUQB+waJZbiAjcMKyzKWUEjXNWU4VFN1qAZ7BXO3au5VNGwmDpvroGWtUHUMliCHagnT+DyGP9fDJkBuKrRxWJWyTJVb/vry9e2xEgByvOc0xOCa40T0DlXLtdbyQ5m7rdwcabhr6lDIcHRfc9cHso25SzR3Rxo9UoCtg9lRlfGXS++63pPeWRlbetc10rzMeocwZE8627XPPhqeZ0df1uExikXLFA+mIgORi5YJZe4SuPLXT7lm41z62qFaEbINaW3aKtpcordxmjv7ILdNS33GOsSBx+STifKJkXvVXonMvZItf4WyzOgO4JNHyfLUf3NIKGQZTMTJcIrmr1/vCFcsRXN35Od1qBrHfFt+2Pq8mVcfOVQHlrlXuiuk2PhYOqRoy18BkxA7VM2OLlnaXSL7C2XuuXeFlBpRwa6c3LHx3Y56Oa7POFQrjpaRyBmpYyHMXfvskmXYa10PAu2eiDY38zB3iUM1w9wlDyrJ2AgYyxVi8Iy78BhQZDYnZakWWSbXa/aEU0iufmUxd1v+tmO2/L3X5H14hs5YHOlK3X6gB5h7JQbGtdAoaCRm8w0KhZSW74A5BK1+4hyyTJe04oGTZTrTjiHGRyJvqLC+6J1iG/kFGyrfwybQeAQzd1fRIcxd2KiVGHeX5l7xxmFsto4+U2SA2NopZCCKHKouNiwpq8jLOgKZuEhzr964DyBzjxpt664x/aAzbXghUkNmPOVbskwO5q6YL9LBEhoTXDpzD8gnOuG4pgzmLhisRRlBrzL3wqzeNMIdYu6ZS7SHVRGHqr7qWuQ/qJl715C02dM79rSPWTpV/vYNMXwMw2A1d19e3BS1LObO1ZnLO2/4VihzbwoNv2WqrucjgdShGvL79es7HefOZlsSc7fJI5l0gmttaJEfrs8bBYgdqhLmboOrHlzy6g23BAPI3JljOWb/7kLyMsvku8ncgyvgroeNudt0yjI09FCm7zsuuabTDtUQuGQZ5wrVMgxDaB55ywxl7rakJTJ3qxZvcaim0kvqLJllB8gyFT4IBs+4M41l7X5FpnOicxYjxca5ezqjy7nE1cFlBLn0mbJCjXXgw8BXH6uWGyLLCMsVP9xCfofjflUd5x4qywSNAy0tkeM+hbQfx7KFW/5miIylDC9zN79blriX7lCtjXshlL77YxBz54w7E+ceVEVPxwiVJ7iB023mLmmQ3Mzdk4/1WAgbzblxWOG+amm7soyJaQBD/CbifiBYxGT9PSHMXb/e1efMetTMvStgH5IBaYWlyCvAOlRz7C0TrJVybEY4EMVTVEce3vw99bEagioWMUmNe4BUo9elJ/aWcRmTImWGPLiF5Ui2H7Cx9byzELFDVVBGzdyrAec8Lf3hKGa1ppFMPjOSi28wsB3Y0RldHV6yiCmUiTvTm999M4mmPY25vYMzX+kUnfs9BfebcS5i6qbmHkoSHNf7dHFxORbm7rrWppW32trT5zN5Cn+HhPjUzL0a8Mydb8DKmbvNaIs0d0H50sES6hz06pYF2Z/V6aWfz8Pcq5ZlQph7t/aWQfjDt0iZIbKMlOWzL4VhCEjyl0snkWUyBCjktziQeRC50nTRuBPRCBHdSkR3EtE9RPQX8fG5RHQjET0U/52jXXMZEa0logeI6LTKas+g9FmnuBCuMAvbyGjuEpYYMPXMMHffOzu5sgp2aG/+3HHtfC5ZpsRFTEUZrlOW6cKukGU5VDNOR6HBdpZjGvdGNm8xc7f0Lcn2AxJZJo9D1Zm2u8x9FMArlVJHAzgGwOlEdAKADwO4WXyiESsAACAASURBVCm1DMDN8XcQ0XIAqwGsAHA6gE8R0VAVlecQYttzL2L68ceB5zZbMnWxAZtxF3SY9AXGX+48M1UN0RSDHaQB6SXM3TZAdEPp09gLMfeiLIGTChzl5SojIA8nU8z5u5yyTN4HBhC2K6R5fUi5juCH1MPAzD/UuLvavIvGXUXYEX+dFP9TAM4BcFV8/CoA58afzwHwFaXUqFJqHYC1AI4vtdbu+orTNvO260/+Ebj+3ZYKONhj63MOWYbLR2ysS9p+wGUgymTuquk5x3yWfLdWj/s9ZcoylocbgPL29hfmUZS5p1AWczezleznbhkDvrGRSmv4b0TXSJi7634bx7qtuRPREBHdAWAjgBuVUr8AsFAptSGqn9oAYN84+WIAj2uXr4+PmXleTERriGjNpk2bivyGFNhnpKX9mkUads8OywnBUz5j25kOo5RxKcdOHKwp9TDIs/1AQHpnXSz5W/OJ87I+uFySRtXGPS8rdCxiyvXKRW/hjmMF8xcTh7A5dBoBG4dJiQPrUBWELWfKFhAx1yzDeaxciIy7UmpCKXUMgCUAjiei5zuSS/brhFLqSqXUSqXUygULFshqKwE3Tq0O1QoaOKPPpWsCQBgK6TKWTFmp8wbzDd5+wFM+m0WJzB2w19n10Ooph6qrni7jXnBlrFm261j7ZEjm7Y8uWSakDia5oeQ/jtAw30Wz2sBombxSD8Dc7x5m7gmUUlsA3IJIS3+KiBYBQPx3Y5xsPYADtMuWAHiycE2ldWRuREj/kxckZSHMNRLN3fldMC10GhfftNHCnDsVLQPYjVxKcy/JuLOMoGAopJ7WFeeeCfkrwbi77k+nZJlOMXebXysPc6/ModqjzJ2IFhDR7PjzVACvAnA/gOsBXBgnuxDAt+LP1wNYTURTiOhgAMsA3Fp2xW1g+64lbW7N3VkBszJcx5O8Zk/wkBA7VPOwcEn5nmPWc0WYu8u4mwagV1aouox7h5h7+6TwmCBv8pXluNY8rhvfoC1/TSKT8x6xZbAX+dP1yCImya6QiwBcFUe8NABcq5T6NhH9HMC1RHQRgMcAnA8ASql7iOhaAPcCGAdwiVLeWLzSwHZdyw0vpLlbKyC4sZItf60GUftste0eIydaxFOmgXCxLtslFUfL+HwSVW4/4NTcy2DuXH2qYO6A9b6H+mVSSJi7q1gPc5fIMuZ1qQeXJVqmNOau5VcRvMZdKXUXgGOZ45sBnGK55goAVxSuXQ6E9N1KjLvLoWqTZTgZZNez/LWpzw7mzhov4UD0MXdJHs5zAtay6QFLXjpPyGvcPVN+67UlGfeUbl2ycc9z7/LKTXBsHBZEGDgJhYDRrdErCRsNeR+yjQ3fClWnQ9VRVzZ9yD2szrgP3grVgMby2vZvvx/4zqWBFXA95S3GXWnnEtzzDWDr4+DhYw6Wh4rYT+BhJ8GsLAdzv+o1/PEgzV1Yp8r3lnFcxzrXi8By71yEoIiUEeTQchjPVHoCRrcBWx4Dbvpz/lq9D3FjTPKbxP3FQwTy5NtrDtV+QKiT3omN9wEb7w2tgeWzXiD3mj1fZeLzE+PAXV818mPS+mQHtl769wBj7awLgOa45focHbusOHevLFOiEXS2f8kO1Tz+kqD8TWewlDA4M01/JQJ2b40+33NdnMTF3H3G17IjZC6tXjBWnZKqeaw27k584/b1uOnepwDYmpFvQK8s0xgKH2ySG5vrNXvx+du+4M6by0//DTs2ATvN1bWhzN1d1RSeeRj46SeZ+iFgQGnoGHPnjHveUEiHy8m7iVsgVDPHzCokf/2LKxSSOyasAzX8fiGvYXakbx0zFjE5x5Ntxaotva/s6pn7QLxm7/3X3gkAeORjZ7GNlVtzJ3JPqVkYnU4iy4iYe4yE0ST52+pgY6YfP4xJLuiMzvIc5566h69f6m8AVAkOVe/Sdtu1OZm7axfI0kMhPfeu8IxEdwYHau5i5ytZPjN5SSRIdl8ZGLMr5bi9KhqzaoIZ05589bqyx2rmLkYIZ/GGQlIjB3N3sAGbLCMaXNyDQciachmM0A5p+w0OR1aefu3cLjfnHi5S5pfXUea6rgqHalGfiDP/Iqt+XRKOxRi3llbYCIiFQJmOX1+ce4a5G29iat0nySw7QHPfcCew+Tfu/HJi8Iw7246uTuUANdzGxFcB20ArsuVvhvWzlUjnJ9Xz9e/SqeTDPxSWISlXckkJi5jM+yJxnkoYm5neVw+gglDI0HuN/L/LJcvs2Q4897Ssbpm2TeLcXddaZiIhzN05yzagb0NcJnNf/0vg+3/lzi8nBtC4M7KMJa2fuefQ3DMOVU6W4V7W4cuWYe4u+aRTDtUvnp09psMVgpZLc5c6KgXnzfo4rw14AJvXd5S5t/4zjrvaXCqXQGi4Ylx5sj+/Vh6GMZa+rMM2xiTIMHdrQoO5h+QrwOTpYemFGDzjzh2z3I+mz7rnkmUceXIGOjoBMeOSyDIZ5u6Jcxc7VAW/TQJvfVzXlrX9QCBz9+bnSNvJaBnvvQtg7j7j7tTcwYTySvuPPiMQaO4skRHMMEzN3fWQS8adKPghZIUqgMkz3PnlxOAZd7bd+ZtRieYucaiKX7NnqZN+HVuFiph7HvbnWhmYi7mXEC2T2bekCllG6lDt0CKmXPfOx9xLes2eeZwLscz0UcuXIIeqp4xU/iRIl5wP3Ftm0jR3fjkxeMadO2brUxLNvUyHqp6vURM/GyiDuVszz34vi/05VwYWZe7m4MzJ3L1GLE4T9DAStn/pxj0wFNL10PLl44xzZzN0HDeZdlMrw5WXjUDpsDlUzescvyW1aV7ZzL2WZUQI0dy9Y7VRUJbJMOjEQBeIbbZ2dkd+XWXubIGOfADMWuK41KGTunZfzOTjY+7CNrHm3yXN3Wt8pGyVSQvIZ0cmnl4LbLzfUowxTtithC31Nu9LiD8nEwppYfumLOPNW9ImunGvRpYZiDh3H6yau+8muZi7cwqnfxYyd2mHEckyTaMaTXf6SjX3HMy94eAcpW356zPuHHPPq7l3chGTh4WzRtNzza5ngSn7ZPd4UQFt8i8vdJxk6uB7iITE10vi3J0PRSUjVbYqeJl7LcuIwLZjEc09OBTSdS45mWeFanKpZKWcmV+gwcjFTEpk7q5wz0q2HxAy1Lz6spO5l/2aPd+DmWHArof+nueAf3w+cO91wKM/d2wlURAZ5p60me0dAlofYu+lnl7qUHXUrYhD1TdeKpJlBoq5P/Db7awht5MZlfpM5kBzMfcisbNFVqhW4lBlDrjYho+J6MijubuMeynM3dCKxZExOY17Jx2qzQnGAGv1mdhjnoDzd43uiF4peft/AA//IHtt4YdRUjfduDPjzur8tBAZiUPVvM75W0iYDv66m8cm1Zq7F6d94kdBdsdH3nLFuTudNIm0koO5c7KM2KHqMaZiJu7Kx5Z3jmgZJ3N3OVSlU3f4b36pskwHNfdnfgN85uX8uQ13Abd/0SjPYayUaj8Mnn6IP18KczfzkEThBDL3Ilv+oihz96Bm7jKEdDVdc2evy6O5u5w0LYeqhbk3hnnWla6U5zxTtzwO1ZCnpO+cNW1R5u75nVJdtopQyG5Fy1ihgPXcC9E8Ri0x7uO7+PNl1FcpwxZz0TKeB7leJ3G5Wt1vv8qyD1JclulgleZrrZMuy9Sauwghmnva7lrkk0IrVM3vyWeLzkpDjmyZB4Org3OygLVPcnV2dMgQZys7JS6JuTvb2lGnzLkKmPt1v+fIS0Pp+7lbkOfBrDP3sd38+TJkmSgz7TOzOIp1BMd/OQIlkWXMMp5Yk66DXlaRFao+ojQ84s8zBwbPuIdo7pbPLRQNhYTiM7bJMg2HcbfJMte8Kdrjncuv9b2Z/uusM/M9czyQ1YfkA5SouTukIp9PgmuT1OAPQCc1dytc97QAcy9LlrE6VPWy9K82gmCRZUTSmwva3jJlO1QbkwLqIcfgGfeAvpaSZVgjXMIK1dSpxEBbXtbREKhkpjF44DvAlkf5/FpfE+NuMzJlSDAOQ2pLa2XurhlMSLSMsDNInac3XS7LT5R/grK3H3DB9x7RGPOPiE8pYGIs/hwaEhyAjMHUHd66UU1dpP3Ny9yl/cNg7qU4VLXPTlKXHwNn3DmIomW4Tl7K3jLM/CDD1gTM3arX29Iyxt2r51uuF10TktbH3B1+hbLi3L0OVfPBXMDoOuPcO/yCbO64eU43ZJnoGs+1+SrHMHcbEdLKdtbB/E2+LX899QtyqLpsAHNsqGbuIvArVPmb0fSMb3ecu2Mqq2fKGhELc3cxVlukDXvMwtxtxp3rjE6Dx81IBO1hHiuqued2qBr1Us1o62J9X23zWpeR88H5DlWzWnHaYy7IXx4L1/2xGEJdc5dem6tqZh4CWcbH3M38+ILl9Usxd1/6QM29Illm8KJlAhQG/wrVPKGQAr3NytwFt0PSeTNJ4gPSB5XVWFuMsnU/E/Dt5233DqxQNR9+ra2LLdeOj9rz8iFIlon/LlkJ3HF1/jIzdbAd54yVVqdElrHlWYVDldPcXX4h089lHrPadkfdtzwW9bVkNp3Kw2vdPecNSMZ9DgwecxceA/wz8+KyjMmAbew7TieRZbj6sK+NY36QbaAGM3cDzXG+PEBuYHXkjZaRGvfMhlfcbzWZe0XG3faavZDl7lJYI5dyyjJVOVRTzN2zQtV82QU7u8zhUG2OGS/RsGn/DEIdqkPVGPe+Z+6mDMPbJJss0z5emuYucqhyzB0eWYbJ34YHbwCm75s9btXcQ2QWJn1zIidzz2HcXS/rCNHJvdEypnF3MFgfXJq71aFahnHXH2K2+8M8yJOHwK4t0fYDLpQS5976r11+iHP8/m+bmeU0wgb0t4yV7lCtZRkvMuOZ6cTVMHeJB95mJC2v2ZOEQtoMkV7uLz/LZ9GUMnfLMdvACWXuregdn3Hn9vWeiOUy5oFiPrykhkfC6iqTZSxpy2DujaF2mygFe7SMhbl/8ih3/pU5VBv528E2s+2oQzU0FLKWZVhMiJg7f613haozzt36yEgXzMoyDHNXEMoyTLnNpqM+EnBTXldDMgY3iLk3gfE9wDizMAZotw/XHs0JY6AZ59IF8fmb50TMvYhDNWBXyFablGDcM1tVWO6pNVrGh5JkGc5giiJObHmZ6YWLmFzQV8qWzdzraBkeplOUb3bL0ZQdLkmWcXVKa3hXwtwlDlUBcy8KaXhZguaE/Rz7MBoH/u8C4LYv8Ne0nFiMcVeakysziAJ28NTr9dTdzPkymXuALPOzf4oPl23cbeD6vbDs0pg7U37uNQvJamxD5uETC/PMwdynzgXO/DgwbZ4lvc7c6zh3FllbysgyeZl7ri1/DaYuZu4+4+7QTkOMGpt1IHM3T4Vq7j7nZDIYrcw9Oe5j7i5o1677kT95IYeqwxhYDU/Jxt0my3DGKoS5lxKXb44T7h2qQkPc6gO2/PRifXlqYy50+4HGEHD8O4GhyTnLLo6+N+4TxqbsAXxToLnnCIX0xtzarlGyQWWVEEqUZVzM/e5vAOt+mD5s09yVZfD7nJNJO4Qy9xDjHjq4CjlUA0IhW4cJmLU4f5mA0X6O35tblvHkK87CeMCwzF2aF8PcOVBD3gfMB2NQ37G9irB64973DtWMLBPwkExdyxr3EkIhJbJMcjy35u5gziOzgd1bXDWWM3cA+NrbmOsn0guA9Hy5evkkjpbmzhgZpbRFNqZxF67AjS4OSItiskzo7A8AQMD77wUu3yd/uSZzt257m9O4V+VQtRpEAbgFY5xDVULcOGlHIstEFwjSVIu+Z+7mveQ3DvMr8dZQyNCYb6djyCLLJGVIZJlQ5i5if0Lmbn1KjgP/9Yd8vixz9zgnXcy9Oe5wqAYY92DmXsChGhLn7jseAtfrClvg7rWw7PFdwI//IbRW9nq0iqf2A9G25a81G4ssY6IRMis3ZJlEg3/5B+yXJPUmshD3WpbxwmTu3KvzUuZVS+/aOGzj9t3YOppM8UJC2UymzjF3rtmVMM6du9TBDKfPE1zPMXc2IX/YtrzeKssIjTs3k5nY026n3A7VHMzQV+epc4Aps/hzIQ7VMiF5sUshzR3Anu2htbLUIVUB7d5aFg+NzAaGpzJ5JWG2enbclh0Bxj01W0z+89030v52R5bx3kUiOoCIfkBE9xHRPUT03vj4XCK6kYgeiv/O0a65jIjWEtEDRHRalT/ADIVsMtbdpq2n9pYxrjn+iptx5Y/j3RbzxCknufJiPnOdh7m3ZBmmLk1HtEyeBRJW56iDuc9bxl0AthNLZRnuYTe+x6G5V8jcfXU+arX9XEgoZPuEt0pe5Nn731mnqmDUQbLlrx7Dr4N1qCL7GxvDgnGt2n9Nh6qvjVLMPWTmXx4kj+hxAB9QSh0J4AQAlxDRcgAfBnCzUmoZgJvj74jPrQawAsDpAD5FlJeS+mEyd9PBClgkF6RZPL/hmEXbdcLB3F0dQzVl0+hQWSbvfjUhoZCbHwKmzOTz8DH3Ey7Jnk+6C8vcR0uKlgmE1wlsGcSAZyBXKMtkmLtNc6+g7BCY4+TB/0Hm3mYeQEP8jIh1qHKyTKg/TZtBiAxzHzB3pdQGpdTt8eftAO4DsBjAOQCuipNdBeDc+PM5AL6ilBpVSq0DsBbA8WVXvF2/9Pdxjy6Tlmj44wmayQ3KHYVhdtD4LzftVU2h5s79PodDNdcCCQvjtpXx1QuAJ2+3ZMUMIJ0FT5uTPd9iPRZZpsXcC2juwbKMz6HqkHryaO5i5u5Ip7ef00dUQJYpBYbB3LO93WY2zd2mmXNjlXWoNrJ5ZqqlohfhrL3JqIdAlmn1YfQ0c2+BiA4CcCyAXwBYqJTaAEQPAADJZiaLATyuXbY+PmbmdTERrSGiNZs2bQqveQyTqXM7PebR3AHNuOdl7pmMW9aduazp1txdsoyrfiLjztS5tCgID3Pn3vzekmVsrDZxqOaUZSZGgU8eLUubwCslEbBnB38uzzoEKXt2RViZ56Qvrei4cWfgG3O2Orau8zB3GvL38d/eBfxV7LN68ldaGUpwf/qAuScgohkAvg7gfUqpba6kzLHML1FKXamUWqmUWrlgwQJpNTIQyTIWhp7W3LPXTcBiRFxwOVQT2ByqopVq3FPIsUJUormLQyEDO6RNltENJfdyYJdDVT+fYe5VyjIVRcsU1dZdhCBlgCz3gu2jJcsyPiMqIRPSB5Bv36IEQdEyZj1CmHv3QiJFxp2IJiEy7F9SSn0jPvwUES2Kzy8CsDE+vh7AAdrlSwA8WU51szADNXjmzjN0ZbP6rUN5jLvDoWqLc0+uKxIKWUiWYeSjUqaSAuY+NDl73uVQBcpZxBQKCXO3IY9DtQzmbsoybD9mDGsyA7JF/4TCLPd3/l+2DlZYZBmbceccqpw/hIY85dqgApk7eleWISIC8DkA9yml9KDW6wFcGH++EMC3tOOriWgKER0MYBmAW8urchoS5n7Pk+2Jhm7ofdsPFJZlbCGGto45eYYjW0Nzf8VHtHMu5p7HoWpj7oGwMbLEuB95dvYcIGDuJewtEwovc3cMdl8o5Bu+GJZfKpnLuBsOVa4f73kOmXud/FbOSZ4H5sNtwRHp7+bs4UJtC9/U4iENVubOtbUlzj0PWvXwMXf9Q/WGnIOEua8C8FYArySiO+J/ZwL4GIBTieghAKfG36GUugfAtQDuBXADgEuUqm7UmaGQE0z//dh378dPHno6c9wWItnKy8ncbc4zx430dYyps+3XtvKI69KQOMtQnuY+fd+czJ25JmHBy8/hL5Myd7Pej/08sH4B8O1r7oLPoXrEWfxxCVwRVrb3BujY9kT2eLIzZVnGPTP8PVE7+z2fy8TIwsPczdBKsz1zGffEUAvGQTIjJYoirb54DnD319vnO8DcvbROKfUT2B9Tp1iuuQLAFQXqJYYZwmh7dd66zc/hZcvmO+LcOTnHw9wf/B/gy+cD77kdmHdo6yo9h6BQMxdz1/ME0p3btf2ARHO/9UqmDCO/+YeHM2NfKGRjCOCyTAaezXDZmHuVGPUs1nEZY99AZg2V0LjvdxTwyI/9+druxdYnsseSh29ZsozJ3DOEw+hvkshpL3PP0+a+MqlNfHwP36EpyUXAfddHbb97G/D818nqVwJ6wC1eDCZT52QZAOwA80fLeDT3X18b/X3iNr4c1fpPP1BMZ22FiA0Zx4po7mYZio88Cd48y+NQ9Q1i23nbVL1K2CJhWsipuQPt3zM8ElQlAMAbrwb2Xc6fS7FTxddjm8u4V8TcTanQnClyUqJUlmlyDlVLtEwwyPKZQTLu9AVZs5e2z/eC5t7ryGjulkZLjtocqk7N3TY4W4Y2bsbdWyFyqOZBS3NHukwAlcS5m4acKDxixMrcY+Nh8wckv8U2dbY5VKvEqMO4L17pYe4eWYYIOPWvgItvSR+XYOps4OATLXlrfWRijH9A7diY7TtVa+6Z+2owd71fKAU8+jPgNzd78ojxjXdErwdMzQSQ/Y2591C3MXfj+/AU/riezz5LgQ89mrMefgyccee2HwB02+hm6zq8oZD68V3PAh9bCtzyMaNQk7kT7DfcMaB/eycwtiv7QHHVD8j/fkaOpZts/rzPeDKxMXdNlrFdBwAHruJP21aoVgnbHiovfQ/wzptRyKEKAKv+ENj3yOzxItD7yPYNwA8YpXRiDzLt2NLcJTKhABnDyjD31HljVvqFM4Cf/GM6jYt5b15rHCiJuaeibmzGPIauubdgEL1pc2R+tpzof+MulGWSh4BVc2dDKB2au85KqQE8HgcE7Xomm85EHlnmZ/8MXPd7sGruRaJlTCiVfd9q4hgKyXv3Vr5eCTO0DbBEFpi5CLjg69nzFb25xokiDtU8IZpBmrBnsZcLE2OMQzVh7oGa+6v/L3/cJ8v86j+APTvb36kBHPVG/tpWGo8M5nWo5jF9DofqkMW4c7CGpZaL/jfuQoeqqWqYafXLtu0ei88nxl0gy2y4M/qcYspMFAp5mHvSqTms+7Gdue/eyl+TR5aZ2MMwd8oafJ9x3/qEx6HKdL9jL2ifnzSC8vTSgnDJMkB+WcZ2bhKz42Fo2RJppzmGTD+dyKm5z1yUPTYxBtxwWfqYOZt8+Bbg0Z+0vxMB5346iiKytY/rwSV5eU0e0uNyqA4bxpyTZZKH/BfOANbeCLsdKAd9b9x1jX3RPiP83jLgb7Vtn/dX/f0PAegOVY+TlhrAjqeiz/qKS1OWWXuTX3/9HTNyRcPEnjhPg4mMO5bT5zHuo9ssmrshy/gGyDaLcXc5VM/51/Z5bktXIF+kQ1HkcahenjxwHYbG1rcmMSt3Q8oGsveLTcMw94SFTmX2/XFWg6nHw7cAd38tfUwy82oMRWMpr3GvzKGajGkCXvz77VMZ5h6Pu6cfaB9LSEsSslvx6tW+N+6JgV48eyqaSlllGdWSZXgnqt4XNm6PjIszWkY128cbQ8DOzdHn1IBSwDff1f664Y7ob97NosZHozLN65/byKcH8mnuu7fxzlMfczc18q3rgZ98IpvPhEdzbzH3qXxb5ZpSB2Da/OyxsZ3ZYzpyD1SbcRcw94NebpRt1EHiAJ9gmPuZfxstkjv8dP/1OvT+sOI8YGQf4Dlm3ygpa3a9w9jL3PW03ArVoqGQAGbsC7zg/OhcRnOfkrk8ez9q4+5EEgo51CA0VQFZhhlkzhWqunGnBvBcvEhKdzpaPbY5p9HJFJoa6bRju+zXDOWYfo5uyxpyMJq7mfeyVwNLX9L+/siPo7xMSDX3SVPhZV3UiN8wXyKmzfWnecVHjQNGPaXMsAhz/91vp7+bD0uJcW8yzH3aPOCkS8NDM3WdeeaiKF8ujl5s3B2be7mMc3McWP/L9PdMHXIy913PxBKs0KGqg5sNV4i+N+6JgR5uEJTB3E88vL0hWWK87Q5VJu/kBj7wHeCpe9MndeMOajN3yc6Evps6Yz/7OdVEpmO5WGUIc/+je6K/u7dFMxB9ABFlf5s5SE0JyLbwJzHejSFe10003+ERvq30es1aAvyRfm9KGDBTBcbd1FhNiI1HAeZuwjR4vv1wGsORwbExSpdTkIN+/xPWvfXxbLoQ5q6awAEnMHk42nf7b4FHf9r+3pzI9qO80t6vrgYe/9/scfNhzMmhNXMPQxL6ODwUM3eNZD9vv7bhaBnvlHF3x7m3QiFvuhz49EvSJ1PMXTPuKedraKhefLPff5+djaqYuesdY2y3PcsQzX2fJdEWvLu3Rh0xNbVk4tzNQWp+t80odFnmea8BXmNIN0m0ho296gObjO95HGUmfFoz5zjkXsAsgdWhGv92bmsCW9lmmWOeCJ/GpMi4m/cp+Smh/pqhycDv/jfwez9uG2bOVyE27hSNJ66NXMZ5p7HViI+5h9Qngdlm+yzhy9T3gDLHj8n2S0b/G/fYfg41GpHmrhnshnYz2rZdp+vaR1coJAfduKsmsPMZJk2oLBP/bTTsnVc1w2QZacd9+/eivyP7AKNbo+l6iokJQiEbw8ARZ8bph9rx0iZ0hyoRsPJtxvn4Omu0jNE2ulHLtWjLQCLLJL/FxCW/aBvf1pYRZqidsN190TKv/7wgk8S4m3vJeC5rDEf3OTPzS5h7qHGfAhz0MmDRUZpxZx4wUp9Jsi2vz7jPPjB9zpwxcsY9GZsv/+Noaw0J9HYyX96irz4F2ob8MG2HltCAhILoe+M+Oh4x5ZFJDTSbaVlmuNEecN6XeDB5N13No8eqjo8yGrUtV+R3qAKaQ1WPlnFp7sliCg+TTOSbkVmxLGMYd1A2JNSctjeGo0U9H3ok2mtnj0UumtBkGfZ8PDCGLQ7V1G8xIofyLtrSkTD3KbOA132OKb8BvPB3gVP+HFj1vvhYzjhqGwFI9O5JI34JgSzG/cjXeq5rRG09tjMdmZTkl8zcXn0FcOk6d15AVpZRE8XWBySyEWvctfZOHMsJTKI1MZ5t5yS6bsgmpAAAHUlJREFUbe7B5YTWmsY9ISh6u2ZmvtWG9Pa9cX/muajB5k2fEq290W7ikGbclQJue/RZPL2j/cTVV7M6NXcOOqPwRVJkUCAuGcgyd323uRe9E3ixFqGTDDhfR0rOT5oazQQmxniPvw5zo7PGcFSvqXOiKafuTD36ze3PyQwgr0O1YfgC9LZInLwHvNhd9wR/wOinyapB68pSitr15e+PZxfI1rOoLBMUEUTZa1a+Hf5Y70Zk+MZ2pUN4k/vaaEThnC99t8zJrD/sE9btDSF1YHgkMpLcfaBG5Gu5+JZs3zYXEnLE67gLo11Jl59rH3dLDSnW9bAc2SfK60XviL63wnm1MWQa94rXawyMcV8wc3ImFHI4ZdwVXvfpn+G0T7R30EtHTWYHwjhcr73TYmlt7CR0LxmTkdrKdT105h4CLNQ2kkqYLA25GeCQlm7tjcDjv8jKMibMQaWnHx5JG3fd8IwLmbstFNLVTsnvPejlwHvvzA5QE/seCZx3JbBQ22Z2cuyraY5byhccK+pQ1ZHMDmzgmDs12nKEzTFKjcjw7Xku/bpDU+aQQi9Hl2VWnAe89brw/CZNjYhTskBQBzWAfRYD+x+blY9M5s7dx6NXR/voT5lhHxev/WT6+6HsJrhtvOGqtpTXYu5axNHEWNopWDN3N57ZuQeThxuYMWUYTZXefmBoKM3cgfR5366Qe+CY4uuOHptxZ6Ua2JmCrsHZbnwzjmKx7dDYHOfzaQylJYtj3wq87I+yZSfp9+wwjIJgVuHaY143yCGhkNyNMfceSZ3Tfvucg2Ts6Og3Aoef1v6esFhb5JMk0qKo5q7jVX8uyysVIjrUrr9NqqKhWJbZBUzWjPuMffn0PpiyDBDp35OnA/OP4K9xwRWKqce/m78vw9yZ+6iPQdv9NB8aktDQJM34aPo7ELW17oeqNXc3ntmxB/OmT0ajQWgqlTLYM0faN8fHj7jze5Sj8XXN3WbcrVvkCoy7rcON7ogut0lBasJwMGqau8msuLL1OqTSS9iqNhhMh1ODMe426eG4/xOXP8UyJXcY90SWCX2HpZ5nYuiaE+DvleCYnt+C57U/m6zYrP/+x3oqyhUdl2Vl7jbjHq86HtuZfjtSSPy1LlWk+ktcp9HtkcyTJzLEFeuvG8nDDEZtMnefQ9YkUm++NvKnmDMeyW8YiffjSfw2+jVNIzKp4pXW/W/cn9uDOdMmo0EEpZDafmDWSNtQsZq6h7mPSpm7zdDajLtt8AxJjPu26JzNWdkcT4fAtYx2I51/Ywgpg9SSZRrZY0Ca2bUrmf6qs5Rxh77oY+5n/C3wkafamrAJVxhb6wHjMVD7vQB4839q9dN+d+IE4+KjzbStY2Yd4zQf3QT8/k+BkVjHv+h/0unMfnfRTcBHHSuOOXAzjcZQm936ZJmxncD0BVEY41uYjdpcOO2v259NzR2IZoCTp/sjb2bunz02ycGU9TG37FTgTza0v3Ox9SbMB6GOZa+O/CkZ4y5g7gufH4X2nv0v2WsmxtLBD7Us48baTTuwdO40NAjYM9HEw5vahm3W1HaH4qJl9GPj5vaS8MgyTV2WsTiNQvc/FzH37QDIHsPcnIgGagvJ/jcMc09FmRiyDJBOL9khcLoWm28y94OY7Xut+7U32gObk7b0AWMaDf0lCfpfE/sfCxz+ar4uyYPMukI2gLkPT44eqklfYB+SGoaGw1luopenWCFpxl0iy0yLwhiXvSqs7BFty1q9nFmL258nT/cviPrAfdljtr2FgCy5mRyyFw/sxn3Ji7RoIVOWESzqSkJ7k7GgE6qkrVtpa+Nuxabto3h080688MA5qZj2BLNGfMa9/Xl8Int+j+sthClZxsGiAUZbKyDLbHnMz9yf95r295F9or8n/nHaaJDB3HXHawJ9QCbXpnRd43foe7KYqyNXnAd84EHgmLdoZQo0R272oxt3U29tGem4bjYDYZatt/dBL4+2FzjrH6J90E1w98ZkdWb+nIMNcLNTKZIHxrhhOFpM3tbfGtF9Gt8duFFZjA/+pi1DAOn+MveQ9uf5h7uNu62vu1bputZ2SGBuYZFAd5q6mLv3XQYMmuNpIlgzdzue3LILi2dPxXEHzgExxn2fqe0Bxu0Wqdv7MebN2qOqoEM1MUw6iwFkDlVbh392XXS9rXObkQHDI1E42wnvSocumh1Ld7wm0JlL67Njt73punFnFjDNXBjFFbcuF3Ruzhk2YwFw0ofietlkmRg245mRc3SppwGc9MGIfS17NbJg7p/5Yguzfd/wxShyxyw3j1E1wc0Ghia12866J3qjbWxE7+81MN3YYM1m3I98rduQ2R7AunFffm76XHD4sQEbcz9B3+nRYtwnTY+ibfJg17Ptz0telC8PIfrauB99wGz89MOvxHFLZ6PBjDddluF2i9QN+hjL3B3GfdsTwJO/ij5z7A5oT8Uz0+IC0TJA1BldsowN+gA2ZZkhhrnrTD/p2LpzynxI6YbK1NwTzD20/VnCXLjfMzzSZlg25p7ULbNz5cv447aH6bxDs7tEcg9n0ziaD64jXwu8/QZ3m7ngWnPA5bHqfW2jLtl8a0GOaBYT+v2cNhc48dL0qwNtsDF0nSkPT0m/TtC2+tkGc78mLlrmrdelt54w+0hrPIRuK6LhRx+P/r72n9ox8RWhr417AiLyyjJX/ujhzPktO9sGiGPuTllGx5O388cT5iR96QHH3M/422y6sd32/WRMpqu3yxTDuHOyTGrfFp3FW4yLvjuiXpapuSfQGZ0kWoDT3IcmR/Hp0+YBp/xp+lyrDRPjrhn/Ve8DDjg+Pm4YX9csIrPdAcfcjXvsWoSkD2rpBmHvvtV+Lskj+Q2n/Fl0r5MHI9eGjWHgkJPb3/c7SlYPF1IGk4BXfkQW/SMx7plN0QKN+5l/B5zzKeD99wPnX8WHbXJbBXP1CV2/8tZvAqf/TfQ52dBs0dFhUUk5MBDGHQDL3CcNuX/elp3tTs8b94JL2df9MPpr7jJou6eccefY7ehW4KyPA8dckD2XiW3WCtOn740h3qGqDyJ9VeKBL43+Ll6pFUaRfPGG/4iWqOs45s1gocsy+m876x+Ac/8tm96muY/MAi59OG2g9N+RIJFtXvtJ4NS/sDN630wJcOvGGePuIAZn/T1w9j9Hn6XMfc5B9nOJcdz/2CjaZlW8fiHpC5m3SBHwZ5uBWfu3v89cKKtHWdA3xrM+4DQjSo208zYUU2YAx74FmLUIWGFIPC3j7jHaCXPXHwLJDDL1/lsDh74COOSk9LEy5DgPBsa4m5r7zJFh1uDreDbF3LMv8xAzdxvu+6/ob2aXQYnmPpT+CwBv0d5oM3spcO6/tr/vuyL6m7C1pPPo7TJZM0CZOHdGc192avvzQauiPWP0FzgkeS8/O1qiruM1n+A7cOLgBdK/7UUXAce8KZveFy1jIuNLiB9yrZhvZoCadTGRtNV5/wZctp5P45NlTCQ+k9AoDw6thTO7gQNe1J41JL/ZnEV95LfR3+RhVcZma6F4x81tNnvipXwaXZIjAs7+pyj0sjHM7KdfAEmf8b3rdoiRZY55E/ChR6PQWhdmGrJQni2dAzEwxl2XZRbOmoJfX34a62TVwTH3tPYeMG069JXp7/qGU+a+HHq9zvx4+3Mqzj1hmJqRcK0cTHZWTAY0t+eJztx1WSbZEwZoG6XFL8xuyDR1jiE3ONqnMRQt/+fwlq9FDjJJB9cXACVwhaQlYYHJtq/JAzOJl58Vb9e7w3hDkEtGSe7B8IhdYvM5VE0kDsEyBnmSh+lkt62wTfpGB9ijFXMPjpyXl28Fjjq/fVyv0+IXtj9TI+p/L7kkmnWc9EF73hd+G/jAA+2dTn2wyTImWszdYPhTBTOKkdnAkuPb32vjLofO0nePyd4s/iyjuXPx7iKsvibNhvWOadsffP4RwPHvbH/nZBmdAU5ioiLeexfwntvbHa/F3OPOo8fa64aJhrIGXS/3MI216whZVWd7GC07NdqHQ6I5HvSy7I57LuaerJhMohKSNk0MXeJYM53gLhnFJuXomGwYfR9zT0JZyzCwyRqEfYyoLB8TTcoO1ZCrwqXrIqOcYNJItMsoENbvDn55xJSXMi/54JDcK6lxz+NQJQIu/K/29w4Y92o3N+ggdOaebAPsQ1MBc6ZNwrM7x1px7lzUjAhDk9KDRDdINlkmw/Yse8Ik4Ba3zImXsz8eO9xazD1Z2KKFjO1/TDr/pL4pJyozY0hV3RHnXhVmHxjF9ydwLfJZdBTwxi+1w8wWxS8OTxy5ibacbPmawGncmXthwryXvtcbShc1STBzIbD6y9lN0nxvBUvKluxv48J7bgd2BK6q5cDtPMltreDClH38aTJlxP3Ya9xHgIUviFav5oEelutaoFUSBsa463ZmdFzeWefNmIJnd45hT8LcGceqCI0hpJ7ouiEwHapJiFpmy1zGg58KTRRozcmAPvYtwPf+BJihOcqWnxMx8rU3IvXSYI5lmg5Xs17RF3t9ElzwDeDph/zpQuAKC2wMA0dqi7iOXg0sXBEZfQCYEzt0V5yXvk6iubseAMMjwIkfjHYFvPMaYNV77WkB4GXvi2YXK9/uTifF85g3NvmYe9L/ihr3eYdG/3wYmpzujxK0xoHAuL/v7nwPy5Ys42kvIuBdPwnPn0PVL3nHABl33aAfvUTuVZ83fTLWQpdlypyiEgCVZe7JoidXhEXLoOh7nriMWjK1jDvoCX8ArLwou4hn4fLYuA8BlBh3bq8UYwWrWQ4gY+6HnZLd2CkUZv24qJX5hwNPP5hdGUvUNuxAFGXzkaeybelk7gLjTgS8MnbyLT7Oni7B1DmRgzAEw1PdL2Yx8Yo/Ab729ujvLz8b7Xy5WQsJbhnCDskyf/IkgvxYgLZuQWAMZx+QPfaOm/0LnvZdDtx3ffiDp8cxMMY9iVl/zVGLcMV5Hs+1hvkzokGeGPc9Aazfi6lzou1HTYdLEpqWMe5MjDlJjXviOExehEH86syEzekbS3HjrcHo8ObnTiHzflImzeovAz/4a9liHK5dXExKYtw7gffeGfWnTwm15Of/TvQPAE5iIlKKROokM6AQ5InKafW3nBLgkpX+NCddGun0SbhvlVh9DfDEmurLwQAZ9627IqO26rD52GeqvBPNmxGxwERrT/T6179wCb52myXsTYrEuJud2rbk27f9gCvOuuU49EwtdSmmZTQ5+UV3uNqMe4c090w5TLnzlwHnfyF/ES7D3fJ/dCFkUMfMheXGo4duOXDgqujl6W/7bufCJ1skp8K+1hiKHPccFh3NvywkL553ZvSvAxg44z47wLAD0ev5AOCHD27CJa84rCXvTJ0UdarxSTMxPLbder0TiRxjLsVPnGkmo+fefORa7q8jWeRygGe/iiQ/avAOwpaTdRgtI5raP11fWNIh455ZIVrB7EEkywxMcFmEUH36bd+pph4uSBcYVYXf+1Fk3PPsvdNlDIxxT2LWQ1g7AMyeFqW/dd0zuG/DtlY+UydHBm3PtP0wvDWncd/3edEUTDWjdzbeflV0fOXbga3rs69P45i7tFMvXBFFLfimy0qXZRxOxIa294z+ENijtYUvGqMsJPU48+PRnj5lLJXPlOFyqDpmOP2Mbsa5S9ELD9Qk4qrP4G05Ivo8EW0koru1Y3OJ6EYieij+O0c7dxkRrSWiB4joND7X8vHeU5Zh7vTJeMESPhTKZvRnaTtHPrRxB97y2V8AAEZi5v7s4pPzV+qMv4s27V96QrT8/d23RdEjk6cDZ/yNe+FLHsYy71D/YGjp7A3tYcKUQUN8NI2+lN2MAqoKSVvscwDwqsurGfCSUEhfNEWncPSbgRe8oXg+ZYRhVo3WLK1HYvH7CJJR8u8ATjeOfRjAzUqpZQBujr+DiJYDWA1gRXzNp4gq3pE+xksPm4/b//TU1Kv1dNgUhNnT2jr2HY9taX2eFjP3dUd/ML0/eggmTwOOe2tUOBEw/zB35AgX5140TM1E8kCZPMMdt90YahszXQYZjZn7GX9bzl7kIiQ3r8IBLtlbplcW+5z3aeB1/694Po2haPZ40Y3F86oKrQdrj7R9H8Fr3JVSPwJgvJQQ5wCINQZcBeBc7fhXlFKjSql1ANYCOB49ANuE+tD5bfb8+Z+ua31ONPcmNYADXlxl1drg4tzLNu4nXwac+lfAUW+waNea5p6s1m0wxl2602UZaPkfum3cS74XvYBT/6K9U2YvouIXWgwy8s5vFyqlNgBA/DdZZ74YgP4Cw/XxsQyI6GIiWkNEazZt2sQlKRW2fWYWz+FXir3k0Hl45GNn4cTDF3QuBI6NlinZoE2aCqz6w2jQOA3aUOQQPuJM4I1XZ88X2aEvFJ2YmrvucbKB2oDFQfcFalkmN8q2WpwFZe+KUupKAFcCwMqVKyu/c2bFTjp8AX79xFYMWbaOnDKsPfc6ZtyZOPcq2WKrDFt9GsCbrkkfO+2vo60VDu+YOwWtu9eJtuBw4qXAsW/N7t1So3p0RtUdSOS1Wk8R0SKl1AYiWgQg2VhiPQB9mdgSAE8WqWBVuPL/vBBThqOOc+4x+2P77nHcfH97f4zJunH37RNSFvSHyJKVwAP/nX1FX6nl5Rg40+dFL2HoJDoiy7ji3Bu1Ye8WOnHvBxR5ZZnrAVwYf74QwLe046uJaAoRHQxgGQDHK2Q6h8xLVTQd+ROrj8VHX7M8dT4x/AA6x9z1OPdV7wPe9TPZUva86BdWRF12qNao0YfwWi0iugbAyQDmE9F6AH8O4GMAriWiiwA8BuB8AFBK3UNE1wK4F8A4gEuU6pX4sbR1N9WYudPTqz8nd0WW0aNlGlHs+u6t6TTv/L79FXu5y+NCIXspprsDskxt3HscNXMPhddqKaWY1+MAANiYPqXUFQCu4M51E5ntSYwDs0bSTdFRzf1tNwB3fdUSvWJUXN8nvih6YYGIBJ0IRez2vjE1ePQUyegv7DU92tdFiAgzR4axfXe06nJYp/Y+VmduHxuKA18S/es0OFmmF7VN6X7bRVAb995GL/bLHkefULfiSOzDhS85EKev2I9Nc8sfn6yl1427Y0uDkz4EnP/v0efzrixWSQ5Vb5gEWAZOLzGmDtSlG7td1hCgl/phf2Gv6dEUd5Kzj9kf//ZWXtqYN2MKHxrpZHVa+qPfWKCGgvxLz7pPdOZalqlRa+7B2Ot6dMPDhH/wgZOxbvNzxkUD2kz94kTsiCzTJ22xt6EOhcyNAbVaWSR9xLZoKcHSedOwdJ6xW5458GcvTb/Ts0p0QpZJsaJeHESdCIXca4ZCn6GWZfJir+nRSRfxMXcW+sC/7Ang3m8B3/qDOGMjv9/7cbSdbz9AtM1tD6CWZWrUCMZe16NzGXd9cdGUGYbhM/JbdFT6nZ2F0Qnm3uPohCxTO1RrDBj2mh6dRL/4ZBkWkk2lqkKVDNrF3HtJ4+z2xmE1egA91B/7BHuNcU+Qx7ZnGG5i+I5+U7XbA1QNLhQyMXK9JMt0ZIVqbdx7Ej3VD/sLA9+jZ44MY9rkofYb40pl7p3oeB1m7qf9NTBlFnDk2dWVG4qWP7XeW2avRS/NJPsEA2/c7/izVwMATvq7HwAowaHaaVQaLcNM3KbPB876eHVl5kEty+zFqJl7Xgx8j0409hZzzyXLmCtUB4RFuDYO6ynUDtUavd5Hew97TY9OVqjmmt3Zpuwd0QPrFaodWchSa7u9ifq+5MbeY9wT+5Dn4kGdsveLzly/aq1GfeuDsfcY9yIXm8a9k86dToRC9ryzqoNL0KfMqr6MGgHowOrkAcWAUtIskjh3lcdADGq0TL8x96qN+1u+Diw4otoyaoShlmVyY69h7gtmTAGQfr2eGEMD6lDtFydiJ1aoAsCyVwGzD/Cnq9F59Pzssvew1zD3T11wHG6896nspmASmAx35v7R33mHFK+YD5WGQvZZtEzP17NG+aiZe17sNcZ9/owpeNPxS/NdbMoyy14FvPWbwMEnFq9YN1HLMjX6BvW9D8VeY9wLgdPcD31FhwqvQyE7JsvU6D0sPSH6+4LXd7cefYjauEvges1e1ejaa/Z6CK39bvrER1CjPMw7FLh8a7dr0ZeojbsEeZyw/YB+MZYnfhAY2wWsfFu3a1KjRt+gNu69jo68ianHMTKr9/a7qVGjx9En1K1GJegXzb1GjRrBqJm7FI1h4KXv6V75+y4vP0/2Hao1atQYBNTGXYo/29y9st92QzUrJxPNfVKO2P8aNWr0NGrj3g848CXV5Ds8BTj1L4HDz6gm/xo1anQNtXHf27Hqvd2uQY0aNSpA7VCtUaNGjQFEbdxr1KhRYwBRG/caNWrUGEDUxr1GjRo1BhCVGXciOp2IHiCitUT04arKqVGjRo0aWVRi3IloCMC/AjgDwHIAbyKiClbh1KhRo0YNDlUx9+MBrFVKPayU2gPgKwDOqaisGjVq1KhhoCrjvhjA49r39fGxFojoYiJaQ0RrNm3aVFE1atSoUWPvRFWLmLitDFMbmCilrgRwJQAQ0SYierRAefMBPF3g+m6hX+sN1HXvFuq6dwe9WvcDbSeqMu7rAehvGl4C4ElbYqXUgiKFEdEapdTKInl0A/1ab6Cue7dQ17076Me6VyXL/BLAMiI6mIgmA1gN4PqKyqpRo0aNGgYqYe5KqXEiejeA7wEYAvB5pdQ9VZRVo0aNGjWyqGzjMKXUdwB8p6r8DVzZoXLKRr/WG6jr3i3Ude8O+q7upHr95cg1atSoUSMY9fYDNWrUqDGAqI17jRo1agwg+tq49/r+NUT0eSLaSER3a8fmEtGNRPRQ/HeOdu6y+Lc8QESndafWrbocQEQ/IKL7iOgeInpvfLyn609EI0R0KxHdGdf7L/qh3jqIaIiIfkVE346/90XdiegRIvo1Ed1BRGviY/1S99lE9DUiuj/u8y/pl7pboZTqy3+IonB+A+AQAJMB3AlgebfrZdTxRADHAbhbO/a3AD4cf/4wgL+JPy+Pf8MUAAfHv22oi3VfBOC4+PNMAA/Gdezp+iNaQDcj/jwJwC8AnNDr9TZ+w/sBfBnAt/uszzwCYL5xrF/qfhWAd8SfJwOY3S91t/3rZ+be8/vXKKV+BOAZ4/A5iDoS4r/nase/opQaVUqtA7AW0W/sCpRSG5RSt8eftwO4D9EWEj1dfxVhR/x1UvxPocfrnYCIlgA4C8BntcN9UXcLer7uRDQLERH7HAAopfYopbagD+ruQj8bd+/+NT2KhUqpDUBkQAHsGx/v2d9DRAcBOBYRC+75+seyxh0ANgK4USnVF/WO8QkAlwJoasf6pe4KwP8Q0W1EdHF8rB/qfgiATQC+EMthnyWi6eiPulvRz8bdu39Nn6Enfw8RzQDwdQDvU0ptcyVljnWl/kqpCaXUMYi2vTieiJ7vSN4z9Sai1wDYqJS6TXoJc6ybfWaVUuo4RFt9X0JEJzrS9lLdhxHJp59WSh0L4DlEMowNvVR3K/rZuAftX9NDeIqIFgFA/HdjfLznfg8RTUJk2L+klPpGfLhv6h9PrW8BcDr6o96rAJxNRI8gkhlfSURXoz/qDqXUk/HfjQCuQyRV9EPd1wNYH8/wAOBriIx9P9Tdin427v26f831AC6MP18I4Fva8dVENIWIDgawDMCtXagfAICICJEGeZ9S6h+0Uz1dfyJaQESz489TAbwKwP3o8XoDgFLqMqXUEqXUQYj68/eVUhegD+pORNOJaGbyGcCrAdyNPqi7Uuq3AB4noiPiQ6cAuBd9UHcnuu3RLfIPwJmIojh+A+Aj3a4PU79rAGwAMIboaX8RgHkAbgbwUPx3rpb+I/FveQDAGV2u+8sQTTXvAnBH/O/MXq8/gKMA/Cqu990A/iw+3tP1Zn7HyWhHy/R83RHp1nfG/+5JxmM/1D2uyzEA1sT95psA5vRL3W3/6u0HatSoUWMA0c+yTI0aNWrUsKA27jVq1KgxgKiNe40aNWoMIGrjXqNGjRoDiNq416hRo8YAojbuNWrUqDGAqI17jRo1agwg/j/E9t/4l3JgBwAAAABJRU5ErkJggg==\n", 497 | "text/plain": [ 498 | "
" 499 | ] 500 | }, 501 | "metadata": { 502 | "needs_background": "light" 503 | }, 504 | "output_type": "display_data" 505 | } 506 | ], 507 | "source": [ 508 | "plt.plot(range(len(awac_cum_rs)), awac_cum_rs)\n", 509 | "plt.plot(range(len(awac_cum_rs), \n", 510 | " len(awac_cum_rs + online_awac_cum_rs)), \n", 511 | " online_awac_cum_rs)" 512 | ] 513 | }, 514 | { 515 | "cell_type": "code", 516 | "execution_count": 18, 517 | "metadata": {}, 518 | "outputs": [ 519 | { 520 | "data": { 521 | "text/plain": [ 522 | "" 523 | ] 524 | }, 525 | "execution_count": 18, 526 | "metadata": {}, 527 | "output_type": "execute_result" 528 | }, 529 | { 530 | "data": { 531 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAep0lEQVR4nO3df5RVddn38fcnBKmk8MfYjUD3TN2DObKMcATMUMhAMJao6R0+usRMEIHUWpWYa7W8y7XUcvkYKfAokvpkimLarKSF5EOmJgV4jwYiOtKYE9wC/iAJFSau54+zmY7DmTl7Zs7MwOzPa62z5uzvvr57f69BzzV7n72/WxGBmZllz4e6ewBmZtY9XADMzDLKBcDMLKNcAMzMMsoFwMwsow7q7gG0xRFHHBHl5eXdPQwzswPKmjVrtkVEWfP2A6oAlJeXs3r16u4ehpnZAUXSq4XafQrIzCyjXADMzDLKBcDMLKNSfQcgaQLwE6AXsDAibmi2Xsn604GdwEUR8WyybhEwCdgSEUOb9fsGMBtoBB6NiO92LB0za8nu3btpaGjgvffe6+6hWCfp27cvgwYNonfv3qniixYASb2A24BxQAOwSlJNRLyQFzYRqExeI4H5yU+Au4BbgXuabXcsMBk4LiLel3RkqhGbWbs0NDTQr18/ysvLyf3NZj1JRPDGG2/Q0NBARUVFqj5pTgGNAOoiYmNE7ALuJ/fBnW8ycE/krAT6SxqQDOr3wJsFtnsZcENEvJ/EbUk1YjNrl/fee4/DDz/cH/49lCQOP/zwNh3hpSkAA4HX8pYbkra2xjQ3BBgt6Y+SnpB0QqEgSdMlrZa0euvWrSmGa2Yt8Yd/z9bWf980BaDQFpvPIZ0mprmDgEOBUcB3gAdUYPQRcXtEVEdEdVnZPvcxmJlZO6X5ErgBGJy3PAjY1I6YQtv9ZeQeSPAnSXuAIwD/mW/WBcrnPFrS7dXf8OWSbq8UHnnkEYYMGUJVVVV3D2W/lKYArAIqJVUAfwOmAP+rWUwNMFvS/eS+/N0eEZuLbPcR4IvA7yQNAfoA29oy+LZo73/s++N/1GZWXGNjI4888giTJk1yAWhB0VNAEdFI7lLNZcB64IGIWCdphqQZSdhSYCNQB9wBzNzbX9J9wDPA0ZIaJH09WbUI+JSkteS+WJ4afjyZWY/385//nBEjRjBs2DAuvfRSXn31VSorK9m2bRt79uxh9OjRPPbYY9TX1/OZz3yGqVOnctxxx3HOOeewc+dOANasWcMpp5zC8ccfz2mnncbmzbm/N8eMGcP3vvc9TjnlFG688UZqamr4zne+w7Bhw3jllVe6M+39Uqr7ACJiKbkP+fy2BXnvA5jVQt/zWmjfBVyQeqRmdsBbv349ixcv5umnn6Z3797MnDmTJ554gquuuooZM2YwcuRIqqqqGD9+PPX19WzYsIE777yTk046iYsvvph58+ZxxRVX8I1vfINf/epXlJWVsXjxYq655hoWLVoEwNtvv80TTzwBwMsvv8ykSZM455xzujPt/dYBNRmcmR3YHn/8cdasWcMJJ+Qu+nv33Xc58sgjufbaa3nwwQdZsGABtbW1TfGDBw/mpJNOAuCCCy5g7ty5TJgwgbVr1zJu3DgA/vnPfzJgwICmPl/96le7MKMDmwuAmXWZiGDq1Klcf/31H2jfuXMnDQ0NAOzYsYN+/foB+17WKImI4Nhjj+WZZ54puI+PfvSjnTDynslzAZlZlzn11FNZsmQJW7bk7vt88803efXVV7nqqqs4//zz+cEPfsC0adOa4v/61782fdDfd999fOELX+Doo49m69atTe27d+9m3bp1BffXr18/3nnnnU7O6sDlIwCzjOqOK9yqqqq47rrrGD9+PHv27KF3797cfPPNrFq1iqeffppevXrx0EMP8bOf/YyxY8dyzDHHcPfdd3PppZdSWVnJZZddRp8+fViyZAmXX34527dvp7GxkSuvvJJjjz12n/1NmTKFadOmMXfuXJYsWcKnP/3pLs95f6YD6cKb6urqaO8DYXwZqGXd+vXrOeaYY7p7GKnV19czadIk1q5d291DOaAU+neWtCYiqpvH+hSQmVlGuQCY2X6pvLzcf/13MhcAM7OMcgEwM8soFwAzs4xyATAzyyjfB2CWVSuuLx7TFmOvbnfXuXPnMn/+fIYPH86iRYv48pe/zLZt27j66quZP38+N910E9XV1Zx++un84he/oH///iUceHa5AJhZt5s3bx6/+c1vqKioYOXKlezevbtpTqD58+c3xS1durSlTVg7+BSQmXWpm2++maFDhzJ06FBuueUWZsyYwcaNGznjjDO48cYbueCCC6itrS04hXN5eTnbtm2jvr6eY445hmnTpnHssccyfvx43n33XQBeeeUVJkyYwPHHH8/o0aN58cUXuyPNA4ILgJl1mTVr1vCzn/2MP/7xj6xcuZI77riDSy+9lKOOOooVK1Zw1VVXsXDhQkaPHk1tbW2rUze8/PLLzJo1i3Xr1tG/f38eeughAKZPn85Pf/pT1qxZw0033cTMmTNb3EbW+RSQmXWZp556irPOOqtpxs6zzz6bJ598sl3bqqioYNiwYQAcf/zx1NfXs2PHDv7whz9w7rnnNsW9//77HR94D+UCYGZdppRzjx188MFN73v16sW7777Lnj176N+//weeKWAtS3UKSNIESRsk1UmaU2C9JM1N1j8vaXjeukWStiSPfiy07W9LCklHtD8NMzsQnHzyyTzyyCPs3LmTf/zjHzz88MOMHj26ZNv/2Mc+RkVFBQ8++CCQKzjPPfdcybbf0xQ9ApDUC7gNGAc0AKsk1UTEC3lhE4HK5DUSmJ/8BLgLuBW4p8C2Byfb/Wv7UzCzdunAZZvtNXz4cC666CJGjBgBwCWXXMLnPve5ku7j3nvv5bLLLuO6665j9+7dTJkyhc9+9rMl3UdPUXQ6aEknAtdGxGnJ8tUAEXF9Xsz/AX4XEfclyxuAMRGxOVkuB34dEUObbXsJ8EPgV0B1RGxrbSyeDtqs/Q606aCtfUo9HfRA4LW85Yakra0xzQd0BvC3iGj1+EzSdEmrJa3eunVriuGamVkaaQqACrQ1P2xIE/OvYOkjwDXA94vtPCJuj4jqiKguKysrFm5mZimlKQANwOC85UHApnbE5Ps0UAE8J6k+iX9W0r+lGI+ZmZVAmgKwCqiUVCGpDzAFqGkWUwNcmFwNNArYvvf8fyER8eeIODIiyiOinFwBGR4R/9O+NMzMrK2KFoCIaARmA8uA9cADEbFO0gxJM5KwpcBGoA64A2i69U7SfcAzwNGSGiR9vcQ5mJlZO6S6ESwilpL7kM9vW5D3PoBZLfQ9L8X2y9OMw8zMSsd3Aptl1LzaeSXd3sxhpZ9z56KLLmLSpEmcc845XHLJJXzrW9+iqqqqw9t98sknmTFjBr179+bRRx/liiuuYMmSJe3a1i233ML06dP5yEc+0q7+mzZt4vLLL2/3/jvCk8GZ2QFh4cKFJfnwh9zNYt/+9repra1l4MCBHfrwveWWW9i5c2e7+x911FHd8uEPLgBm1sWaTwfd2tTO+caMGcPeG0EPOeQQrrnmGj772c8yatQoXn/9dQC2bt3KV77yFU444QROOOEEnn766X22s3DhQh544AF+8IMfcP7551NfX8/Qobl7VO+66y7OPvtsJkyYQGVlJd/97neb+j322GOceOKJDB8+nHPPPZcdO3Ywd+5cNm3axNixYxk7dmzT2PZasmQJF110EZA7mrn88sv5/Oc/z6c+9ammD/20+7/zzjsZMmQIY8aMYdq0acyePbvd/wZ7uQCYWZcpNB30W2+91eLUzi35xz/+wahRo3juuec4+eSTueOOOwC44oor+OY3v8mqVat46KGHuOSSS/bpe8kll3DGGWfw4x//mHvvvXef9bW1tSxevJg///nPLF68mNdee41t27Zx3XXX8dvf/pZnn32W6upqbr75Zi6//PKmqaxXrFhRNP/Nmzfz1FNP8etf/5o5c/aZVq3F/W/atIkf/vCHrFy5kuXLl5fsGQf+DsDMukxL00EXmtq5NX369GHSpElN8cuXLwfgt7/9LS+88K9pyv7+97/zzjvv0K9fv9RjPPXUU/n4xz8OQFVVFa+++ipvv/02L7zwAieddBIAu3bt4sQTT0y9zb3OPPNMPvShD1FVVdV01JJm/9u2beOUU07hsMMOA+Dcc8/lpZdeavP+m3MBMLMu09LcY4Wmdm5N7969kdQU39jYCMCePXt45pln+PCHP/yB+NNOO43XX3+d6upqFi5c2Oq2m4+lsbGRiGDcuHHcd999rfYFmsYF8N5777W47bS/i7377ww+BWRmXaazp4MeP348t956a9Py3ucCLFu2jNra2qIf/i0ZNWoUTz/9NHV1dQDs3Lmz6S/wfv368c477zTFfuITn2D9+vXs2bOHhx9+uL2pfMCIESN44okneOutt2hsbCx6iiwtHwGYZVRnXLZZTKHpoA899NCSbX/u3LnMmjWL4447jsbGRk4++WQWLFhQvGMRZWVl3HXXXZx33nlNTxi77rrrGDJkCNOnT2fixIkMGDCAFStWcMMNNzBp0iQGDx7M0KFD2bFjR4f3P3DgQL73ve8xcuRIjjrqKKqqqppOE3VE0emg9yeeDtqs/Twd9IFtx44dHHLIITQ2NnLWWWdx8cUXc9ZZZ+0TV+rpoM3MrJtde+21DBs2jKFDh1JRUcGZZ57Z4W36FJCZ2QHgpptuKvk2fQRgliEH0ilfa7u2/vu6AJhlRN++fXnjjTdcBHqoiOCNN96gb9++qfv4FJBZRgwaNIiGhgb8aNWeq2/fvgwaNCh1vAuAWUb07t2bioqK7h6G7Ud8CsjMLKNcAMzMMipVAZA0QdIGSXWS9pnCLnkW8Nxk/fOShuetWyRpi6S1zfr8WNKLSfzDkvp3PB0zM0uraAGQ1Au4DZgIVAHnSWr+VIaJQGXymg7Mz1t3FzChwKaXA0Mj4jjgJeDqtg7ezMzaL80RwAigLiI2RsQu4H5gcrOYycA9kbMS6C9pAEBE/B54s/lGI+Kx5IHzACuB9F9dm5lZh6UpAAOB1/KWG5K2tsa05mLgN4VWSJouabWk1b58zcysdNIUABVoa34nSZqYwhuXrgEagX0fzQNExO0RUR0R1WVlZWk2aWZmKaS5D6ABGJy3PAjY1I6YfUiaCkwCTg3fnmhm1qXSHAGsAiolVUjqA0wBaprF1AAXJlcDjQK2R8Tm1jYqaQJwFXBGROxsx9jNzKwDihaA5Iva2cAyYD3wQESskzRD0owkbCmwEagD7gCanjQh6T7gGeBoSQ2Svp6suhXoByyXVCup409tMDOz1FJNBRERS8l9yOe3Lch7H8CsFvqe10L7f6QfppmZlZrvBDYzyygXADOzjHIBMDPLKBcAM7OMcgEwM8soFwAzs4xyATAzyygXADOzjHIBMDPLKBcAM7OMcgEwM8soFwAzs4xyATAzyygXADOzjHIBMDPLKBcAM7OMcgEwM8uoVAVA0gRJGyTVSZpTYL0kzU3WPy9peN66RZK2SFrbrM9hkpZLejn5eWjH0zEzs7SKFgBJvYDbgIlAFXCepKpmYROByuQ1HZift+4uYEKBTc8BHo+ISuDxZNnMzLpImiOAEUBdRGyMiF3A/cDkZjGTgXsiZyXQX9IAgIj4PfBmge1OBu5O3t8NnNmeBMzMrH3SFICBwGt5yw1JW1tjmvtERGwGSH4emWIsZmZWImkKgAq0RTti2kXSdEmrJa3eunVrKTZpZmakKwANwOC85UHApnbENPf63tNEyc8thYIi4vaIqI6I6rKyshTDNTOzNNIUgFVApaQKSX2AKUBNs5ga4MLkaqBRwPa9p3daUQNMTd5PBX7VhnGbmVkHFS0AEdEIzAaWAeuBByJinaQZkmYkYUuBjUAdcAcwc29/SfcBzwBHS2qQ9PVk1Q3AOEkvA+OSZTMz6yIHpQmKiKXkPuTz2xbkvQ9gVgt9z2uh/Q3g1NQjNTOzkvKdwGZmGeUCYGaWUS4AZmYZ5QJgZpZRLgBmZhnlAmBmllEuAGZmGeUCYGaWUS4AZmYZ5QJgZpZRLgBmZhmVai4g24+suL5r9zf26q7dn5l1GR8BmJlllAuAmVlGuQCYmWWUC4CZWUa5AJiZZZQLgJlZRqUqAJImSNogqU7SnALrJWlusv55ScOL9ZU0TNJKSbWSVksaUZqUzMwsjaIFQFIv4DZgIlAFnCepqlnYRKAyeU0H5qfo+yPgvyJiGPD9ZNnMzLpImiOAEUBdRGyMiF3A/cDkZjGTgXsiZyXQX9KAIn0D+Fjy/uPApg7mYmZmbZDmTuCBwGt5yw3AyBQxA4v0vRJYJukmcoXo84V2Lmk6uaMKPvnJT6YYrpmZpZGmAKhAW6SMaa3vZcA3I+IhSf8J3Al8aZ/giNuB2wGqq6ub73e/VT7n0Xb3rb/hyyUciZlZYWlOATUAg/OWB7Hv6ZqWYlrrOxX4ZfL+QXKni8zMrIukKQCrgEpJFZL6AFOAmmYxNcCFydVAo4DtEbG5SN9NwCnJ+y8CL3cwFzMza4Oip4AiolHSbGAZ0AtYFBHrJM1I1i8AlgKnA3XATuBrrfVNNj0N+Imkg4D3SM7zm5lZ10g1HXRELCX3IZ/ftiDvfQCz0vZN2p8Cjm/LYM3MrHR8J7CZWUa5AJiZZZQLgJlZRrkAmJlllAuAmVlG+aHwGTTv7efTB9fOK9l+Zw6bWbJtmVnH+QjAzCyjXADMzDLKBcDMLKNcAMzMMsoFwMwso1wAzMwyygXAzCyjXADMzDLKBcDMLKNcAMzMMsoFwMwso1IVAEkTJG2QVCdpToH1kjQ3Wf+8pOFp+kr6RrJunaQfdTwdMzNLq+hkcJJ6AbcB44AGYJWkmoh4IS9sIlCZvEYC84GRrfWVNBaYDBwXEe9LOrKUiZVK+ZxHu3sIZmadIs0RwAigLiI2RsQu4H5yH9z5JgP3RM5KoL+kAUX6XgbcEBHvA0TElhLkY2ZmKaUpAAOB1/KWG5K2NDGt9R0CjJb0R0lPSDqh0M4lTZe0WtLqrVu3phiumZmlkaYAqEBbpIxpre9BwKHAKOA7wAOS9omPiNsjojoiqsvKylIM18zM0kjzQJgGYHDe8iBgU8qYPq30bQB+GREB/EnSHuAIwH/mm5l1gTRHAKuASkkVkvoAU4CaZjE1wIXJ1UCjgO0RsblI30eALwJIGkKuWGzrcEZmZpZK0SOAiGiUNBtYBvQCFkXEOkkzkvULgKXA6UAdsBP4Wmt9k00vAhZJWgvsAqYmRwO2P/nLk6Xb1lvbi8eMvbp0+zOzVqV6JnBELCX3IZ/ftiDvfQCz0vZN2ncBF7RlsGZmVjq+E9jMLKNcAMzMMsoFwMwso1wAzMwyKtWXwHZguOXxl1LFPXvYGx9YHlVxeGcMx8z2cz4CMDPLKBcAM7OM8ikgM7M2mFc7r1v2O3PYzJJv00cAZmYZ5QJgZpZRLgBmZhnlAmBmllEuAGZmGeWrgKzLzHv7+eJBJb7CojOunDDrKXwEYGaWUT4C2A+Vz3m0xXVXHpRuugczs2J8BGBmllGpCoCkCZI2SKqTNKfAekmam6x/XtLwNvT9tqSQdETHUjEzs7YoWgAk9QJuAyYCVcB5kqqahU0EKpPXdGB+mr6SBgPjgL92OBMzM2uTNEcAI4C6iNiYPMf3fmBys5jJwD2RsxLoL2lAir7/G/gu4IfBm5l1sTQFYCDwWt5yQ9KWJqbFvpLOAP4WEc+1tnNJ0yWtlrR669atKYZrZmZppCkAKtDW/C/2lmIKtkv6CHAN8P1iO4+I2yOiOiKqy8rKig7WzMzSSVMAGoDBecuDgE0pY1pq/zRQATwnqT5pf1bSv7Vl8GZm1n5pCsAqoFJShaQ+wBSgpllMDXBhcjXQKGB7RGxuqW9E/DkijoyI8ogoJ1cohkfE/5QqMTMza13RG8EiolHSbGAZ0AtYFBHrJM1I1i8AlgKnA3XATuBrrfXtlEzMzKxNUt0JHBFLyX3I57ctyHsfwKy0fQvElKcZh5mZlY7vBDYzyygXADOzjHIBMDPLKBcAM7OMcgEwM8soFwAzs4xyATAzyygXADOzjHIBMDPLKBcAM7OMcgEwM8soFwAzs4xKNRmcFXblQUu6ewhmZu3mIwAzs4xyATAzyygXADOzjHIBMDPLqFQFQNIESRsk1UmaU2C9JM1N1j8vaXixvpJ+LOnFJP5hSf1Lk5KZmaVRtABI6gXcBkwEqoDzJFU1C5sIVCav6cD8FH2XA0Mj4jjgJeDqDmdjZmappTkCGAHURcTGiNgF3A9MbhYzGbgnclYC/SUNaK1vRDwWEY1J/5XAoBLkY2ZmKaUpAAOB1/KWG5K2NDFp+gJcDPwmxVjMzKxE0hQAFWiLlDFF+0q6BmgE7i24c2m6pNWSVm/dujXFcM3MLI00BaABGJy3PAjYlDKm1b6SpgKTgPMjonlRASAibo+I6oioLisrSzFcMzNLI00BWAVUSqqQ1AeYAtQ0i6kBLkyuBhoFbI+Iza31lTQBuAo4IyJ2ligfMzNLqehcQBHRKGk2sAzoBSyKiHWSZiTrFwBLgdOBOmAn8LXW+iabvhU4GFguCWBlRMwoZXLWuVb+5Y129RtVcXjLK//yZDtH04K3thePGesL0CybUk0GFxFLyX3I57ctyHsfwKy0fZP2/2jTSM3MrKR8J7CZWUa5AJiZZZSfB2A92ry3ny8eVDuv5PudOWxmybdpVmo+AjAzyygfAXSjZw/7W3cPwczao9RXq6XRCUeVPgIwM8soFwAzs4xyATAzyygXADOzjPKXwNbuKR26WmeNc+UrL7W47sovDemUfZrtD3wEYGaWUT4CMOsE8zrh5rI0fAOatYWPAMzMMsoFwMwso1wAzMwyyt8BWOaN+tALLa/8y+tdN5BSSPMAnK7Q1Q/ZWXF91+0rzQSDBwgfAZiZZZSPABJ9jlje5j7PfsiTuZnZgStVAUge4P4Tcs/1XRgRNzRbr2T96eSeCXxRRDzbWl9JhwGLgXKgHvjPiHir4ymZZVeq5x+U2Mz+x3X5Pq00ihYASb2A24BxQAOwSlJNROSfOJ0IVCavkcB8YGSRvnOAxyPiBklzkuWrSpeamXWFgkWnC+6D8D0PHZfmCGAEUBcRGwEk3Q9MBvILwGTgnuTh8Csl9Zc0gNxf9y31nQyMSfrfDfwOFwAzS+kDN9v1oC9mu1KaAjAQeC1vuYHcX/nFYgYW6fuJiNgMEBGbJR1ZaOeSpgPTk8UdkjYUGe8RwLYiMSXxclfsJL0uy7uj/m9pN9epeZd4rKW0n/97399ZG97P8+48s7i/I7n/e6HGNAVABdoiZUyavq2KiNuB29PGS1odEdVt2UdP4LyzxXlnT2fknuYy0AZgcN7yIGBTypjW+r6enCYi+bkl/bDNzKyj0hSAVUClpApJfYApQE2zmBrgQuWMArYnp3da61sDTE3eTwV+1cFczMysDYqeAoqIRkmzgWXkLuVcFBHrJM1I1i8AlpK7BLSO3GWgX2utb7LpG4AHJH0d+CtwbolySn26qIdx3tnivLOn5Lkrd+GOmZlljaeCMDPLKBcAM7OM6jEFQNIESRsk1SV3FvcokhZJ2iJpbV7bYZKWS3o5+Xlo3rqrk9/FBkmndc+oO0bSYEkrJK2XtE7SFUl7T8+7r6Q/SXouyfu/kvYenfdeknpJ+m9Jv06Ws5J3vaQ/S6qVtDpp69zcI+KAf5H7gvkV4FNAH+A5oKq7x1XiHE8GhgNr89p+BMxJ3s8BbkzeVyW/g4OBiuR306u7c2hHzgOA4cn7fsBLSW49PW8BhyTvewN/BEb19Lzz8v8W8Avg18lyVvKuB45o1tapufeUI4Cm6SoiYhe52xAnd/OYSioifg+82ax5MrlpNEh+npnXfn9EvB8RfyF3ddaILhloCUXE5kgmFYyId4D15O4u7+l5R0TsSBZ7J6+gh+cNIGkQ8GVgYV5zj8+7FZ2ae08pAC1NRdHTfWA6DWDvdBo97vchqRz4HLm/hnt83slpkFpyN0guj4hM5A3cAnwX2JPXloW8IVfkH5O0JpkCBzo5957yPIAOTznRw/So34ekQ4CHgCsj4u+52ccLhxZoOyDzjoh/AsMk9QceljS0lfAekbekScCWiFgjaUyaLgXaDri885wUEZuSedGWS3qxldiS5N5TjgDSTFfRE7U0nUaP+X1I6k3uw//eiPhl0tzj894rIt4mN1PuBHp+3icBZ0iqJ3ca94uSfk7PzxuAiNiU/NwCPEzulE6n5t5TCkCa6Sp6opam06gBpkg6WFIFuec0/Kkbxtchyv2pfyewPiJuzlvV0/MuS/7yR9KHgS8BL9LD846IqyNiUESUk/t/+P9FxAX08LwBJH1UUr+974HxwFo6O/fu/ua7hN+gn07uKpFXgGu6ezydkN99wGZgN7nq/3XgcOBxcjNTPw4clhd/TfK72ABM7O7xtzPnL5A7rH0eqE1ep2cg7+OA/07yXgt8P2nv0Xk3+x2M4V9XAfX4vMldwfhc8lq39zOss3P3VBBmZhnVU04BmZlZG7kAmJlllAuAmVlGuQCYmWWUC4CZWUa5AJiZZZQLgJlZRv1/wMVi0fiGpZ4AAAAASUVORK5CYII=\n", 532 | "text/plain": [ 533 | "
" 534 | ] 535 | }, 536 | "metadata": { 537 | "needs_background": "light" 538 | }, 539 | "output_type": "display_data" 540 | } 541 | ], 542 | "source": [ 543 | "plt.hist(cum_rs, density=True, label='expert')\n", 544 | "plt.hist(awac_cum_rs, alpha=0.5, density=True, label='offline')\n", 545 | "plt.hist(online_awac_cum_rs, alpha=0.5, density=True, label='online-finetuning')\n", 546 | "plt.legend()" 547 | ] 548 | } 549 | ], 550 | "metadata": { 551 | "kernelspec": { 552 | "display_name": "Python 3", 553 | "language": "python", 554 | "name": "python3" 555 | }, 556 | "language_info": { 557 | "codemirror_mode": { 558 | "name": "ipython", 559 | "version": 3 560 | }, 561 | "file_extension": ".py", 562 | "mimetype": "text/x-python", 563 | "name": "python", 564 | "nbconvert_exporter": "python", 565 | "pygments_lexer": "ipython3", 566 | "version": "3.7.4" 567 | } 568 | }, 569 | "nbformat": 4, 570 | "nbformat_minor": 4 571 | } 572 | --------------------------------------------------------------------------------