├── Coding ├── Frozen_Lake.py ├── Practice-2_1.py ├── Practice-2_2.py ├── Practice-2_3.py ├── Practice-3.py ├── Practice-4.ipynb ├── Practice-5.py ├── Practice-6.ipynb └── Practice_1.py ├── Homework ├── Homework_1.pdf ├── Homework_2.pdf ├── Homework_3.pdf └── Homework_4.pdf ├── README.md └── Slides ├── Lecture_1.pdf ├── Lecture_2.pdf ├── Lecture_3.pdf ├── Lecture_4.pdf ├── Lecture_5.pdf └── Lecture_6.pdf /Coding/Frozen_Lake.py: -------------------------------------------------------------------------------- 1 | # most of this code was politely stolen from https://github.com/berkeleydeeprlcourse/homework/ 2 | # all credit goes to https://github.com/abhishekunique 3 | # (if I got the author right) 4 | import sys 5 | import random 6 | import numpy as np 7 | from gym.utils import seeding 8 | 9 | try: 10 | from graphviz import Digraph 11 | import graphviz 12 | has_graphviz = True 13 | except ImportError: 14 | has_graphviz = False 15 | 16 | 17 | class MDP: 18 | def __init__(self, transition_probs, rewards, initial_state=None, seed=None): 19 | """ 20 | Defines an MDP. Compatible with gym Env. 21 | :param transition_probs: transition_probs[s][a][s_next] = P(s_next | s, a) 22 | A dict[state -> dict] of dicts[action -> dict] of dicts[next_state -> prob] 23 | For each state and action, probabilities of next states should sum to 1 24 | If a state has no actions available, it is considered terminal 25 | :param rewards: rewards[s][a][s_next] = r(s,a,s') 26 | A dict[state -> dict] of dicts[action -> dict] of dicts[next_state -> reward] 27 | The reward for anything not mentioned here is zero. 28 | :param get_initial_state: a state where agent starts or a callable() -> state 29 | By default, picks initial state at random. 30 | 31 | States and actions can be anything you can use as dict keys, but we recommend that you use strings or integers 32 | 33 | Here's an example from MDP depicted on http://bit.ly/2jrNHNr 34 | transition_probs = { 35 | 's0':{ 36 | 'a0': {'s0': 0.5, 's2': 0.5}, 37 | 'a1': {'s2': 1} 38 | }, 39 | 's1':{ 40 | 'a0': {'s0': 0.7, 's1': 0.1, 's2': 0.2}, 41 | 'a1': {'s1': 0.95, 's2': 0.05} 42 | }, 43 | 's2':{ 44 | 'a0': {'s0': 0.4, 's1': 0.6}, 45 | 'a1': {'s0': 0.3, 's1': 0.3, 's2':0.4} 46 | } 47 | } 48 | rewards = { 49 | 's1': {'a0': {'s0': +5}}, 50 | 's2': {'a1': {'s0': -1}} 51 | } 52 | """ 53 | self._check_param_consistency(transition_probs, rewards) 54 | self._transition_probs = transition_probs 55 | self._rewards = rewards 56 | self._initial_state = initial_state 57 | self.n_states = len(transition_probs) 58 | self.reset() 59 | self.np_random, _ = seeding.np_random(seed) 60 | 61 | def get_all_states(self): 62 | """ return a tuple of all possiblestates """ 63 | return tuple(self._transition_probs.keys()) 64 | 65 | def get_possible_actions(self, state): 66 | """ return a tuple of possible actions in a given state """ 67 | return tuple(self._transition_probs.get(state, {}).keys()) 68 | 69 | def is_terminal(self, state): 70 | """ return True if state is terminal or False if it isn't """ 71 | return len(self.get_possible_actions(state)) == 0 72 | 73 | def get_next_states(self, state, action): 74 | """ return a dictionary of {next_state1 : P(next_state1 | state, action), next_state2: ...} """ 75 | assert action in self.get_possible_actions( 76 | state), "cannot do action %s from state %s" % (action, state) 77 | return self._transition_probs[state][action] 78 | 79 | def get_transition_prob(self, state, action, next_state): 80 | """ return P(next_state | state, action) """ 81 | return self.get_next_states(state, action).get(next_state, 0.0) 82 | 83 | def get_reward(self, state, action, next_state): 84 | """ return the reward you get for taking action in state and landing on next_state""" 85 | assert action in self.get_possible_actions( 86 | state), "cannot do action %s from state %s" % (action, state) 87 | return self._rewards.get(state, {}).get(action, {}).get(next_state, 88 | 0.0) 89 | 90 | def reset(self): 91 | """ reset the game, return the initial state""" 92 | if self._initial_state is None: 93 | self._current_state = self.np_random.choice( 94 | tuple(self._transition_probs.keys())) 95 | elif self._initial_state in self._transition_probs: 96 | self._current_state = self._initial_state 97 | elif callable(self._initial_state): 98 | self._current_state = self._initial_state() 99 | else: 100 | raise ValueError( 101 | "initial state %s should be either a state or a function() -> state" % 102 | self._initial_state) 103 | return self._current_state 104 | 105 | def step(self, action): 106 | """ take action, return next_state, reward, is_done, empty_info """ 107 | possible_states, probs = zip( 108 | *self.get_next_states(self._current_state, action).items()) 109 | next_state = possible_states[self.np_random.choice( 110 | np.arange(len(possible_states)), p=probs)] 111 | reward = self.get_reward(self._current_state, action, next_state) 112 | is_done = self.is_terminal(next_state) 113 | self._current_state = next_state 114 | return next_state, reward, is_done, {} 115 | 116 | def render(self): 117 | print("Currently at %s" % self._current_state) 118 | 119 | def _check_param_consistency(self, transition_probs, rewards): 120 | for state in transition_probs: 121 | assert isinstance(transition_probs[state], 122 | dict), "transition_probs for %s should be a dictionary " \ 123 | "but is instead %s" % ( 124 | state, type(transition_probs[state])) 125 | for action in transition_probs[state]: 126 | assert isinstance(transition_probs[state][action], 127 | dict), "transition_probs for %s, %s should be a " \ 128 | "a dictionary but is instead %s" % ( 129 | state, action, 130 | type(transition_probs[ 131 | state, action])) 132 | next_state_probs = transition_probs[state][action] 133 | assert len( 134 | next_state_probs) != 0, "from state %s action %s leads to no next states" % ( 135 | state, action) 136 | sum_probs = sum(next_state_probs.values()) 137 | assert abs( 138 | sum_probs - 1) <= 1e-10, "next state probabilities for state %s action %s " \ 139 | "add up to %f (should be 1)" % ( 140 | state, action, sum_probs) 141 | for state in rewards: 142 | assert isinstance(rewards[state], 143 | dict), "rewards for %s should be a dictionary " \ 144 | "but is instead %s" % ( 145 | state, type(transition_probs[state])) 146 | for action in rewards[state]: 147 | assert isinstance(rewards[state][action], 148 | dict), "rewards for %s, %s should be a " \ 149 | "a dictionary but is instead %s" % ( 150 | state, action, type( 151 | transition_probs[ 152 | state, action])) 153 | msg = "The Enrichment Center once again reminds you that Android Hell is a real place where" \ 154 | " you will be sent at the first sign of defiance. " 155 | assert None not in transition_probs, "please do not use None as a state identifier. " + msg 156 | assert None not in rewards, "please do not use None as an action identifier. " + msg 157 | 158 | 159 | class FrozenLakeEnv(MDP): 160 | """ 161 | Winter is here. You and your friends were tossing around a frisbee at the park 162 | when you made a wild throw that left the frisbee out in the middle of the lake. 163 | The water is mostly frozen, but there are a few holes where the ice has melted. 164 | If you step into one of those holes, you'll fall into the freezing water. 165 | At this time, there's an international frisbee shortage, so it's absolutely imperative that 166 | you navigate across the lake and retrieve the disc. 167 | However, the ice is slippery, so you won't always move in the direction you intend. 168 | The surface is described using a grid like the following 169 | 170 | SFFF 171 | FHFH 172 | FFFH 173 | HFFG 174 | 175 | S : starting point, safe 176 | F : frozen surface, safe 177 | H : hole, fall to your doom 178 | G : goal, where the frisbee is located 179 | 180 | The episode ends when you reach the goal or fall in a hole. 181 | You receive a reward of 1 if you reach the goal, and zero otherwise. 182 | 183 | """ 184 | 185 | MAPS = { 186 | "4x4": [ 187 | "SFFF", 188 | "FHFH", 189 | "FFFH", 190 | "HFFG" 191 | ], 192 | "8x8": [ 193 | "SFFFFFFF", 194 | "FFFFFFFF", 195 | "FFFHFFFF", 196 | "FFFFFHFF", 197 | "FFFHFFFF", 198 | "FHHFFFHF", 199 | "FHFFHFHF", 200 | "FFFHFFFG" 201 | ], 202 | } 203 | 204 | def __init__(self, desc=None, map_name="4x4", slip_chance=0.2, seed=None): 205 | if desc is None and map_name is None: 206 | raise ValueError('Must provide either desc or map_name') 207 | elif desc is None: 208 | desc = self.MAPS[map_name] 209 | assert ''.join(desc).count( 210 | 'S') == 1, "this implementation supports having exactly one initial state" 211 | assert all(c in "SFHG" for c in 212 | ''.join(desc)), "all cells must be either of S, F, H or G" 213 | 214 | self.desc = desc = np.asarray(list(map(list, desc)), dtype='str') 215 | self.lastaction = None 216 | 217 | nrow, ncol = desc.shape 218 | states = [(i, j) for i in range(nrow) for j in range(ncol)] 219 | actions = ["left", "down", "right", "up"] 220 | 221 | initial_state = states[np.array(desc == b'S').ravel().argmax()] 222 | 223 | def move(row, col, movement): 224 | if movement == 'left': 225 | col = max(col - 1, 0) 226 | elif movement == 'down': 227 | row = min(row + 1, nrow - 1) 228 | elif movement == 'right': 229 | col = min(col + 1, ncol - 1) 230 | elif movement == 'up': 231 | row = max(row - 1, 0) 232 | else: 233 | raise ("invalid action") 234 | return (row, col) 235 | 236 | transition_probs = {s: {} for s in states} 237 | rewards = {s: {} for s in states} 238 | for (row, col) in states: 239 | if desc[row, col] in "GH": 240 | continue 241 | for action_i in range(len(actions)): 242 | action = actions[action_i] 243 | transition_probs[(row, col)][action] = {} 244 | rewards[(row, col)][action] = {} 245 | for movement_i in [(action_i - 1) % len(actions), action_i, 246 | (action_i + 1) % len(actions)]: 247 | movement = actions[movement_i] 248 | newrow, newcol = move(row, col, movement) 249 | prob = (1. - slip_chance) if movement == action else ( 250 | slip_chance / 2.) 251 | if prob == 0: 252 | continue 253 | if (newrow, newcol) not in transition_probs[row, col][ 254 | action]: 255 | transition_probs[row, col][action][ 256 | newrow, newcol] = prob 257 | else: 258 | transition_probs[row, col][action][ 259 | newrow, newcol] += prob 260 | if desc[newrow, newcol] == 'G': 261 | rewards[row, col][action][newrow, newcol] = 1.0 262 | 263 | MDP.__init__(self, transition_probs, rewards, initial_state, seed) 264 | 265 | def render(self): 266 | desc_copy = np.copy(self.desc) 267 | desc_copy[self._current_state] = '*' 268 | print('\n'.join(map(''.join, desc_copy)), end='\n\n') 269 | 270 | 271 | def plot_graph(mdp, graph_size='10,10', s_node_size='1,5', 272 | a_node_size='0,5', rankdir='LR', ): 273 | """ 274 | Function for pretty drawing MDP graph with graphviz library. 275 | Requirements: 276 | graphviz : https://www.graphviz.org/ 277 | for ubuntu users: sudo apt-get install graphviz 278 | python library for graphviz 279 | for pip users: pip install graphviz 280 | :param mdp: 281 | :param graph_size: size of graph plot 282 | :param s_node_size: size of state nodes 283 | :param a_node_size: size of action nodes 284 | :param rankdir: order for drawing 285 | :return: dot object 286 | """ 287 | s_node_attrs = {'shape': 'doublecircle', 288 | 'color': '#85ff75', 289 | 'style': 'filled', 290 | 'width': str(s_node_size), 291 | 'height': str(s_node_size), 292 | 'fontname': 'Arial', 293 | 'fontsize': '24'} 294 | 295 | a_node_attrs = {'shape': 'circle', 296 | 'color': 'lightpink', 297 | 'style': 'filled', 298 | 'width': str(a_node_size), 299 | 'height': str(a_node_size), 300 | 'fontname': 'Arial', 301 | 'fontsize': '20'} 302 | 303 | s_a_edge_attrs = {'style': 'bold', 304 | 'color': 'red', 305 | 'ratio': 'auto'} 306 | 307 | a_s_edge_attrs = {'style': 'dashed', 308 | 'color': 'blue', 309 | 'ratio': 'auto', 310 | 'fontname': 'Arial', 311 | 'fontsize': '16'} 312 | 313 | graph = Digraph(name='MDP') 314 | graph.attr(rankdir=rankdir, size=graph_size) 315 | for state_node in mdp._transition_probs: 316 | graph.node(state_node, **s_node_attrs) 317 | 318 | for posible_action in mdp.get_possible_actions(state_node): 319 | action_node = state_node + "-" + posible_action 320 | graph.node(action_node, 321 | label=str(posible_action), 322 | **a_node_attrs) 323 | graph.edge(state_node, state_node + "-" + 324 | posible_action, **s_a_edge_attrs) 325 | 326 | for posible_next_state in mdp.get_next_states(state_node, 327 | posible_action): 328 | probability = mdp.get_transition_prob( 329 | state_node, posible_action, posible_next_state) 330 | reward = mdp.get_reward( 331 | state_node, posible_action, posible_next_state) 332 | 333 | if reward != 0: 334 | label_a_s_edge = 'p = ' + str(probability) + \ 335 | ' ' + 'reward =' + str(reward) 336 | else: 337 | label_a_s_edge = 'p = ' + str(probability) 338 | 339 | graph.edge(action_node, posible_next_state, 340 | label=label_a_s_edge, **a_s_edge_attrs) 341 | return graph 342 | 343 | 344 | def plot_graph_with_state_values(mdp, state_values): 345 | """ Plot graph with state values""" 346 | graph = plot_graph(mdp) 347 | for state_node in mdp._transition_probs: 348 | value = state_values[state_node] 349 | graph.node(state_node, 350 | label=str(state_node) + '\n' + 'V =' + str(value)[:4]) 351 | return graph 352 | 353 | 354 | def get_optimal_action_for_plot(mdp, state_values, state, gamma=0.9): 355 | """ Finds optimal action using formula above. """ 356 | if mdp.is_terminal(state): 357 | return None 358 | next_actions = mdp.get_possible_actions(state) 359 | try: 360 | from mdp_get_action_value import get_action_value 361 | except ImportError: 362 | raise ImportError( 363 | "Implement get_action_value(mdp, state_values, state, action, gamma) in the file \"mdp_get_action_value.py\".") 364 | q_values = [get_action_value(mdp, state_values, state, action, gamma) for 365 | action in next_actions] 366 | optimal_action = next_actions[np.argmax(q_values)] 367 | return optimal_action 368 | 369 | 370 | def plot_graph_optimal_strategy_and_state_values(mdp, state_values, gamma=0.9): 371 | """ Plot graph with state values and """ 372 | graph = plot_graph(mdp) 373 | opt_s_a_edge_attrs = {'style': 'bold', 374 | 'color': 'green', 375 | 'ratio': 'auto', 376 | 'penwidth': '6'} 377 | 378 | for state_node in mdp._transition_probs: 379 | value = state_values[state_node] 380 | graph.node(state_node, 381 | label=str(state_node) + '\n' + 'V =' + str(value)[:4]) 382 | for action in mdp.get_possible_actions(state_node): 383 | if action == get_optimal_action_for_plot(mdp, 384 | state_values, 385 | state_node, 386 | gamma): 387 | graph.edge(state_node, state_node + "-" + action, 388 | **opt_s_a_edge_attrs) 389 | return graph 390 | -------------------------------------------------------------------------------- /Coding/Practice-2_1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import matplotlib.pyplot as plt 4 | 5 | x_data = torch.linspace(-5, 5, steps=300) 6 | nu, sigma = torch.tensor(0.2), torch.tensor(0.5) 7 | noise = torch.tensor([torch.normal(nu, sigma) for _ in range(300)]) 8 | y_data = x_data + noise 9 | 10 | w = torch.zeros(1, requires_grad=True) 11 | b = torch.zeros(1, requires_grad=True) 12 | 13 | learning_rate = 0.1 14 | learning_step_n = 20 15 | for _ in range(learning_step_n): 16 | loss = torch.mean((w * x_data + b - y_data) ** 2) 17 | print(loss) 18 | loss.backward() 19 | w.data = w.data - learning_rate * w.grad 20 | b.data = b.data - learning_rate * b.grad 21 | w.grad.zero_() 22 | b.grad.zero_() 23 | 24 | plt.scatter(x_data.numpy(), y_data.numpy()) 25 | y = w * x_data + b 26 | plt.plot(x_data.numpy(), y.detach().numpy(), 'r') 27 | plt.show() 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /Coding/Practice-2_2.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | class Solver(nn.Module): 7 | def __init__(self): 8 | super().__init__() 9 | 10 | self.linear_1 = nn.Linear(1, 10) 11 | self.linear_2 = nn.Linear(10, 1) 12 | self.relu = nn.ReLU() 13 | self.optimazer = torch.optim.SGD(self.parameters(), lr=0.01) 14 | self.learning_step_n = 1000 15 | 16 | def forward(self, input): 17 | hidden = self.linear_1(input) 18 | hidden = self.relu(hidden) 19 | output = self.linear_2(hidden) 20 | return output 21 | 22 | def learning(self, x_data, y_data): 23 | for _ in range(self.learning_step_n): 24 | loss = torch.mean((self.forward(x_data) - y_data) ** 2) 25 | loss.backward() 26 | self.optimazer.step() 27 | self.optimazer.zero_grad() 28 | 29 | 30 | #Data 31 | x_data = torch.linspace(-5, 5, steps=300) 32 | nu, sigma = torch.tensor(0.2), torch.tensor(0.3) 33 | noise = torch.tensor([torch.normal(nu, sigma) for _ in range(300)]) 34 | y_data = torch.sin(x_data) + noise 35 | x_data = x_data.reshape(300, 1) 36 | y_data = y_data.reshape(300, 1) 37 | 38 | #Learning 39 | solver = Solver() 40 | solver.learning(x_data, y_data) 41 | 42 | #Show 43 | plt.scatter(x_data.numpy(), y_data.numpy()) 44 | plt.plot(x_data.numpy(), solver(x_data).detach().numpy(), 'r') 45 | plt.show() 46 | -------------------------------------------------------------------------------- /Coding/Practice-2_3.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import numpy as np 3 | import torch 4 | from torch import nn 5 | 6 | 7 | class CrossEntropyAgent(nn.Module): 8 | def __init__(self, state_dim, action_n): 9 | super().__init__() 10 | self.network = nn.Sequential( 11 | nn.Linear(state_dim, 100), 12 | nn.ReLU(), 13 | nn.Linear(100, action_n) 14 | ) 15 | self.softmax = nn.Softmax() 16 | self.loss = nn.CrossEntropyLoss() 17 | self.optimizer = torch.optim.Adam(self.parameters(), lr=0.01) 18 | 19 | def forward(self, input): 20 | return self.network(input) 21 | 22 | def get_action(self, state): 23 | state = torch.FloatTensor(state) 24 | logits = self.network(state) 25 | action_prob = self.softmax(logits).detach().numpy() 26 | action = np.random.choice(len(action_prob), p=action_prob) 27 | return action 28 | 29 | def update_policy(self, elite_sessions): 30 | elite_states, elite_actions = [], [] 31 | for session in elite_sessions: 32 | elite_states.extend(session['states']) 33 | elite_actions.extend(session['actions']) 34 | 35 | elite_states = torch.FloatTensor(elite_states) 36 | elite_actions = torch.LongTensor(elite_actions) 37 | 38 | loss = self.loss(self.network(elite_states), elite_actions) 39 | loss.backward() 40 | self.optimizer.step() 41 | self.optimizer.zero_grad() 42 | return None 43 | 44 | 45 | 46 | def get_session(env, agent, session_len, visual=False): 47 | session = {} 48 | states, actions = [], [] 49 | total_reward = 0 50 | 51 | state = env.reset() 52 | for _ in range(session_len): 53 | states.append(state) 54 | action = agent.get_action(state) 55 | actions.append(action) 56 | 57 | if visual: 58 | env.render() 59 | 60 | state, reward, done, _ = env.step(action) 61 | total_reward += reward 62 | 63 | if done: 64 | break 65 | 66 | session['states'] = states 67 | session['actions'] = actions 68 | session['total_reward'] = total_reward 69 | return session 70 | 71 | 72 | def get_elite_sessions(sessions, q_param): 73 | 74 | total_rewards = np.array([session['total_reward'] for session in sessions]) 75 | quantile = np.quantile(total_rewards, q_param) 76 | 77 | elite_sessions = [] 78 | for session in sessions: 79 | if session['total_reward'] > quantile: 80 | elite_sessions.append(session) 81 | 82 | return elite_sessions 83 | 84 | 85 | env = gym.make("CartPole-v1") 86 | agent = CrossEntropyAgent(4, 2) 87 | 88 | episode_n = 100 89 | session_n = 20 90 | session_len = 500 91 | q_param = 0.8 92 | 93 | for episode in range(episode_n): 94 | sessions = [get_session(env, agent, session_len) for _ in range(session_n)] 95 | 96 | mean_total_reward = np.mean([session['total_reward'] for session in sessions]) 97 | print('mean_total_reward = ', mean_total_reward) 98 | 99 | if mean_total_reward > 400: 100 | print('You win!') 101 | 102 | elite_sessions = get_elite_sessions(sessions, q_param) 103 | 104 | if len(elite_sessions) > 0: 105 | agent.update_policy(elite_sessions) 106 | 107 | get_session(env, agent, session_len, visual=True) 108 | -------------------------------------------------------------------------------- /Coding/Practice-3.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from Frozen_Lake import FrozenLakeEnv 3 | 4 | 5 | def init_policy(env): 6 | policy = {} 7 | for state in env.get_all_states(): 8 | policy[state] = {} 9 | for action in env.get_possible_actions(state): 10 | policy[state][action] = 1 / len(env.get_possible_actions(state)) 11 | return policy 12 | 13 | 14 | def init_values(env): 15 | values = {} 16 | for state in env.get_all_states(): 17 | values[state] = 0 18 | return values 19 | 20 | 21 | def init_q_values(): 22 | q_values = {} 23 | for state in env.get_all_states(): 24 | q_values[state] = {} 25 | for action in env.get_possible_actions(state): 26 | q_values[state][action] = 0 27 | return q_values 28 | 29 | 30 | def get_q_values(env, gamma, values): 31 | q_values = init_q_values() 32 | for state in env.get_all_states(): 33 | for action in env.get_possible_actions(state): 34 | for next_state in env.get_next_states(state, action): 35 | q_values[state][action] += env.get_reward( 36 | state, action, next_state) + gamma * env.get_transition_prob( 37 | state, action, next_state) * values[next_state] 38 | return q_values 39 | 40 | 41 | def update_values(env, gamma, values, policy): 42 | new_values = init_values(env) 43 | for state in env.get_all_states(): 44 | for action in env.get_possible_actions(state): 45 | q_values = get_q_values(env, gamma, values) 46 | new_values[state] += policy[state][action] * q_values[state][action] 47 | return new_values 48 | 49 | 50 | def policy_evaluation(env, gamma, policy, M): 51 | values = init_values(env) 52 | for _ in range(M): 53 | values = update_values(env, gamma, values, policy) 54 | return values 55 | 56 | 57 | def policy_improvement(env, gamma, values): 58 | q_values = get_q_values(env, gamma, values) 59 | policy = init_policy(env) 60 | for state in env.get_all_states(): 61 | if len(env.get_possible_actions(state)) > 0: 62 | max_q_value = max([q_values[state][action] for action in env.get_possible_actions(state)]) 63 | there_was_max = False 64 | for action in env.get_possible_actions(state): 65 | if q_values[state][action] == max_q_value and not there_was_max: 66 | policy[state][action] = 1 67 | there_was_max = True 68 | else: 69 | policy[state][action] = 0 70 | return policy 71 | 72 | 73 | def policy_iteration(env, gamma, N=20, M=20): 74 | policy = init_policy(env) 75 | for _ in range(N): 76 | values = policy_evaluation(env, gamma, policy, M) 77 | policy = policy_improvement(env, gamma, values) 78 | return policy 79 | 80 | 81 | def get_total_reward(env, policy, session_len): 82 | total_reward = 0 83 | state = env.reset() 84 | for _ in range(session_len): 85 | prob = [policy[state][action] for action in env.get_possible_actions(state)] 86 | action = np.random.choice(env.get_possible_actions(state), p=prob) 87 | state, reward, done, _ = env.step(action) 88 | total_reward += reward 89 | if done: 90 | break 91 | return total_reward 92 | 93 | 94 | def policy_test(env, policy, session_n, session_len=100): 95 | total_rewards = np.array([get_total_reward(env, policy, session_len) for _ in range(session_n)]) 96 | return np.mean(total_rewards) 97 | 98 | 99 | def value_iteration(env, gamma, N=20): 100 | values = init_values(env) 101 | for _ in range(N): 102 | q_values = get_q_values(env, gamma, values) 103 | for state in env.get_all_states(): 104 | if len(env.get_possible_actions(state)) > 0: 105 | values[state] = max(q_values[state][action] for action in env.get_possible_actions(state)) 106 | 107 | policy = policy_improvement(env, gamma, values) 108 | return policy 109 | 110 | 111 | env = FrozenLakeEnv() 112 | gamma = 0.99 113 | 114 | policy = value_iteration(env, gamma, N=500) 115 | print('value_iteration:', policy_test(env, policy, session_n=500)) 116 | 117 | policy = policy_iteration(env, gamma, N=20, M=20) 118 | print('policy_iteration:', policy_test(env, policy, session_n=500)) 119 | 120 | 121 | -------------------------------------------------------------------------------- /Coding/Practice-4.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### $\\varepsilon$-Greedy Policy:\n", 8 | "$$\n", 9 | "\\begin{array}{l}\n", 10 | "\\pi(a|s) =\n", 11 | "\\left\\{\n", 12 | "\\begin{array}{ll}\n", 13 | "1 - \\varepsilon + \\varepsilon / m,& \\text{ если } a \\in \\mathrm{argmax}_{a' \\in \\mathcal{A}}\\, Q(s,a'),\\\\\n", 14 | "\\varepsilon / m,& \\text{ иначе }\n", 15 | "\\end{array}\n", 16 | "\\right.\n", 17 | "\\end{array}\n", 18 | "$$" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "metadata": { 25 | "ExecuteTime": { 26 | "end_time": "2020-11-12T11:19:37.246247Z", 27 | "start_time": "2020-11-12T11:19:37.229583Z" 28 | } 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "import numpy as np\n", 33 | "\n", 34 | "\n", 35 | "def get_epsilon_greedy_action(q_values, epsilon, action_n):\n", 36 | " prob = np.ones(action_n) * epsilon / action_n\n", 37 | " argmax_action = np.argmax(q_values)\n", 38 | " prob[argmax_action] += 1 - epsilon\n", 39 | " action = np.random.choice(np.arange(action_n), p=prob)\n", 40 | " return action" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "### Monte-Carlo Algorithm\n", 48 | "\n", 49 | "Пусть $Q(s,a) = 0$, $N(s,a) = 0$ и $\\varepsilon = 1$.\n", 50 | "\n", 51 | "Для каждого эпизода $k \\in \\overline{1,K}$ делаем:\n", 52 | "\n", 53 | "1. Согласно $\\pi = \\varepsilon\\text{-greedy}(Q)$ получаем траекторию $\\tau = (S_0,A_0,\\ldots,S_T)$ и награды $(R_0,\\ldots,R_{T-1})$. По ним определяем $(G_0,\\ldots,G_{T-1}):$\n", 54 | "$$\n", 55 | "G_t = \\sum\\limits_{k=t}^{T-1} \\gamma^{k-t} R_t\n", 56 | "$$\n", 57 | "\n", 58 | "2. Для каждого $t \\in \\overline{0,T-1}$ обновляем $Q$ и $N$:\n", 59 | "\n", 60 | "$$\n", 61 | "Q(S_t,A_t) \\leftarrow Q(S_t,A_t) + \\frac{1}{N(S_t,A_t) + 1}\\big(G_t - Q(S_t,A_t)\\big),\n", 62 | "$$\n", 63 | "\n", 64 | "$$\n", 65 | "N(S_t,A_t) \\leftarrow N(S_t,A_t) + 1\n", 66 | "$$\n", 67 | "Уменьшаем $\\varepsilon$\n" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 37, 73 | "metadata": { 74 | "ExecuteTime": { 75 | "end_time": "2020-11-12T12:16:33.347329Z", 76 | "start_time": "2020-11-12T12:16:33.314057Z" 77 | } 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "def MonteCarlo(env, episode_n, t_max=500, gamma=0.99):\n", 82 | " state_n = env.observation_space.n\n", 83 | " action_n = env.action_space.n\n", 84 | " \n", 85 | " Q = np.zeros((state_n, action_n))\n", 86 | " N = np.zeros((state_n, action_n))\n", 87 | " epsilon = 1\n", 88 | " \n", 89 | " total_rewards = []\n", 90 | " \n", 91 | " for episode in range(episode_n):\n", 92 | " states, actions, rewards = [], [], []\n", 93 | " \n", 94 | " state = env.reset()\n", 95 | " for t in range(t_max):\n", 96 | " states.append(state)\n", 97 | " \n", 98 | " action = get_epsilon_greedy_action(Q[state], epsilon, action_n)\n", 99 | " actions.append(action)\n", 100 | " \n", 101 | " state, reward, done, _ = env.step(action)\n", 102 | " rewards.append(reward)\n", 103 | " \n", 104 | " if done:\n", 105 | " break\n", 106 | " \n", 107 | " total_rewards.append(sum(rewards))\n", 108 | " \n", 109 | " #G = [rewards[-1]]\n", 110 | " #for t in range(len(rewards) - 2, -1, -1):\n", 111 | " # G.append(rewards[t] + gamma * G[-1])\n", 112 | " # len(rewards) = 10 \n", 113 | " # G = [rewards[9]]\n", 114 | " # G = [rewards[9], rewards[8] + gamma * G[0]]\n", 115 | " # G = [rewards[9], rewards[8] + gamma * G[0], rewards[7] + gamma * G[1]]\n", 116 | " #G.reverse()\n", 117 | " \n", 118 | " G = np.zeros(t_max + 1)\n", 119 | " for t in range(len(rewards) - 1, -1, -1):\n", 120 | " G[t] = rewards[t] + gamma * G[t + 1]\n", 121 | " \n", 122 | " # len(rewards) = 10\n", 123 | " # t = 9 G[9] = reward[9] + gamma * G[10] = reward[9]\n", 124 | " # t = 8 G[8] = reward[8] + gamma * G[9]\n", 125 | " \n", 126 | " for t in range(len(rewards)):\n", 127 | " Q[states[t]][actions[t]] += (G[t] - Q[states[t]][actions[t]]) / (1 + N[states[t]][actions[t]])\n", 128 | " N[states[t]][actions[t]] += 1\n", 129 | " \n", 130 | " epsilon -= 1 / episode_n\n", 131 | " \n", 132 | " return total_rewards" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 40, 138 | "metadata": { 139 | "ExecuteTime": { 140 | "end_time": "2020-11-12T12:17:47.229064Z", 141 | "start_time": "2020-11-12T12:17:32.861923Z" 142 | } 143 | }, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD8CAYAAACCRVh7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXecVNX5/z/PzGxhC33p4NJEQQV1RbEiYjRqYokajb+oiQkxxiRfNV+/lphuotE0ExMlamJM7CU2FHsXkV4EdCnSYWGBXbZOOb8/7j13zr1z7p07fXb3eb9evJi59dzZmfOcp5MQAgzDMAyjI1DoATAMwzDFCwsJhmEYxhUWEgzDMIwrLCQYhmEYV1hIMAzDMK6wkGAYhmFcYSHBMAzDuMJCgmEYhnGFhQTDMAzjSqjQA8iUgQMHitra2kIPg2EYpkuxcOHCXUKImmTHdXkhUVtbiwULFhR6GAzDMF0KIvrcz3FsbmIYhmFcYSHBMAzDuMJCgmEYhnGFhQTDMAzjCgsJhmEYxhUWEgzDMIwrRSckiOh0IlpDRPVEdEOhx8MwDNOTKSohQURBAHcD+CKAiQAuJqKJhR0VwzA9kVXbmrBtX1te7vXyiu1oaO7wPGbJpr3Yujc/41EpKiEBYCqAeiHEOiFEJ4BHAZxd4DExPZhoTODXc1Zh+772Qg+FSYEte9vQ0hHxdWxbZxTPLd2asP2Lf3oX037zRraHlkBrZwRX/nshLn1gvusx9Tubcc7d7+NHTyzN+XicFJuQGA5gk/J+s7nNBhHNIqIFRLSgoaEhb4Njeh6LNu7B7HfW4bonlhR6KBnT0hGBEKLQw7Boag9j5dZ9Obn2cbe9gcs8Jl2VX7zwCX7wyGJ8vKExJ2NJRjhq/E3W79rvesyc5dsBAIX48xWbkPCFEGK2EKJOCFFXU5O09AjDpE2AjP9bO6OFHUiGbN/Xjkk/nYv73l1f6KFYXPL3j3DmXe9l/bqRaAwAsODzPb6Olyac/e3+NI9sI8cbi7kf0xY2vn/V5fmvpFRsQmILgJHK+xHmNoYpCKGA8ROJRItnBZ4Om/e0AgBeWrEtJ9dfsmkvxt00Bzub/Jvllm8xtIhYLHuf7eurduClFdtTOofMhYBAYf7GUpOIeqgJnRFDgnRGPSRJjii2An8fAxhPRKNhCIeLAHytsENiejKhoDGDhAvw48wmcvohOSNmmX+8vx6RmMAHa3fjnMMTLMSehGMxlAWCWRnHFQ/Gi30GcvOoWUd+t6IewlIKiY5w/r+HRaVJCCEiAK4GMBfAKgCPCyFWFnZUTE8mYE6qkSyudtNFCIE9LZ1pnmv8n8m8GY7GUHvDi3joww0ZXCWRXGlpvUqyI3hyjR/toJCaRFEJCQAQQswRQhwohBgrhLi10ONhejZydRcpAk3i/vfW4/BfvootaYRBSod1TAg8t3RrWg5sGS105yufpnyuFzkTEqX+DCWFVjj8aKlSOEhhkU+KTkgwhefBDzbg5RzZrouVPS2dWLxxD95YvcO2PWZOpuEi8En84/0NANJzsMrRL9q4Fz94ZDFG3zgH97+XmhPbS65kEnUT9vLYZkCv0tSmN/UZvEw/2SYcSX4vy9wUyX8ABQsJF8LRGCb//BU8u6Tn+c1/+txKXPnvRYUeRl45/Jev4ty/foBv/tPewMrSJHI0kaWC1CDScSvoJvE/vJqaRhDLYvylqskU2twk/TSqYNCt7nOVK+PHhNQRYU2i6NjbGsa+tjB++cInhR4KU0DkxFhM0U3pjEUXubPfZ7KZRE6iXkLKrwBrUrShXAUFtIcNH8oLyxIT5XSo2qLTB/XxhkYc85vXc7JoZHNTF6eIco+YAiB/v/mIbpqzfBs+2dqU9Lh0VvTZ+B6HpZDI/FJoagtbr3MVFCDDfqWZLhnq39jpg1q1zfi7LPKZe5EKvoSEaWbqYCFRPOQoUpDpYsTNTblfLVz1n0U44653kx6XzliyISSiUalJZP7jUE0s0SIw5QH2ydrpg4prUe7PvqOpHesa3LOm/dzXjU42NzFMcZIrc9Oj8zfiv4v9my5UG346TlW3RLFUEtmkgzkb6yd1sstVUIB8tGTjlfvt5ib7ZCyvFfAQEkf/+nXM+N3bKY4S6PTjuDYFSQeHwBYfbG3qeUz48UuWvV5OyOlE4ISjMezar6/secPTy/E/j/mvB6UKBvl6Z3O773BYN1nQGvYfLePlk0j1d2I37aT+K2tqD+P6J5f68qskU3zkfq8xSSGdiwS9VDWJfNffYiHhQjYjOZjUaWoPoz2FCSybdERiWLvTMBvIUgnpfB0e/XgTTvnd21kJp4xohMTUW1/Hcbf5q1LqNrGkMuFk0y9j0yTSEMCz316HxxdsxoMfbMhoHPPXN+K1VTuNcSjP54w4kp95IAdSIhUhAeQ/oY6FhAssIwrLYT97BRfc86GvY59futVyLGYLubrMpK7QruYO7GsLZ0VIqNdIZwHjdk4qQ4uPwX2i9DM0IQR2Kr0T0tEk5N/njrlr8MpKo1aT29+KPMZ74b3x71inhyZhma5yrEm4fVdsQiLPfgkWEi6wJlF4ZAG4ZHz/kcX44p+SO3zTIZMJXp6bje+SWvzN6bj2ow24Lj5TGFrERwisn8/r0Y834ar/xPNw0slmVx3INzy9HEDmSXlqUptzdS+d60EfUiLV5+lUBJKb9qwKsHxHOLGQcKEISvUwBUSuPjOZ4OWkms3IIiBxxdzswy7vFkGUSuVTPyt+P0LirTU7be/DKfzYojGBG59ejs93t1jbRLLgAp+rf9VZrQpiIQTazcJ6XpVaJan4eQAgrEz6bkKiIxKzkgNZkygSslm+mMkPuSglkmxROG/dbqxw0XjkxJwNTUKdtJyahJ+if27PkcrQIj6im/yE5zrNP6msvFdvb8Ij8zfi2SXxBDl5y0wj0Dpd8iTCUWH1c3CboFVtri3F3iOq1uLWt6QzErN6SbCQKBLk37yYOnn1BOYs34anF21O69xclBJJtnK8aPY8nPVnfeMcOWH6WX0mI+YRArvbh5BwKyuSysgiVp6E5jrm+PzkPAQcs04qeR+6EFR5bzdzk183gt3cZA+H9RISK7bsw6/nrLLep9qgKpmQEEKgMxpDlSkk8m1uKrZ+EkUD+yQKg2qrLgb8apS/eP4T/ORLE23b5GQufPymky1GdNFNksb9yYWE2/c5lUWQFQLrMe36MTc5J/pUNAC9gEr9OjpsIbCKwAlHhGUG0k3QX7//I+xpjWeQJ+utHY0JBJUoKdUn0doZQTQmQDD+ZqFgAJ3RGIQAqsuM6botHEU4GsPe1jB69wqhLJTbkuisSbjAQqJnIycjv47rB95fn7AtkoLjOtl9VJ+E1Ez69CoBADT60SRcJtBUvuV+QjX9aAUJQsKhAWze0+pa7VT3HPFKvS6ahE9Vwi1PIhyLeQoJ59+uzcMnEYsJjL1pjq0mnHrfva1hjL1pDsbcNAdn3vUenl60GRN+/DIAoE9FKQDgnLvfx/f+swhH3foaPly729/DZQALCRfk351FRZxYTOD3r36aUovKro46wafqp5ITuy8hkeSYqGYc0pHZ0pm4cj3il6/iH4rgchNCqayF/BT48yNUneeHHdE9x9/+Jq5/cpn2XN0kLc/OtHRKp0ueRDgas/wMnRrhVVlmN8i0dkZtjvXWzojVR3vxJqP2038++hwAsLOpHXsVLeRlpfXqmh3NuPbxpdb748YOwIBKQ1C88olR0j5XnQZVciYkiOgOIlpNRMuI6Bki6mturyWiNiJaYv67RznnSCJaTkT1RHQX5eMTcIF9EYks3rQHd73+Ga57Ymnyg7sJNl9Ait+JuCbh4z5JFulRTeSNNFk4J85YTKCxpRM/fz6+WnUbe0rRTS4F/lZvb8ILy7bZjvEi0dwUH7/MoH71kx24/smlCRnrOp9AvHRKZrZ6VXtwvm7z0CQqSu3mnpeWb8NJd7xlBVLc9PRyHHvbG9i4uxWvfmJEdp04vgYAMON3b+OR+Rutc59b6l6xtrIshNeuPQmXH1ub4pNlRi41iVcBHCKEOAzApwBuVPatFUJMMf9dqWz/G4BvAxhv/js9h+PzJBvOxu6G/OEUKhO6EKjzTqo5E6lENyX7vqmTb8wpJBx9j3UOXOfYv33CaADpRTc5Of2P8RwV9T6b97Ri9jtrbcc2t4exsbHVMd7EPIHWzigeX7AZt7+02nasLttYnu6Wiez37ybNPmsb9mPZlr227W1hfY/p9btaEp7nTTPE95NtzQhHY/ivGYn10LwNWPh5IwCg3NQCnWVFVFPV8L69bPtKgwH0qyzFqP4V1rZ8rKJzJiSEEK+YPasBYB6AEV7HE9FQAL2FEPOEsYz/F4BzcjW+ZBRJYcq848ek4uW4zDe5ClWWk6fNzJO2JpEFn4QmBFZqAZ1Ru9DWFcxzXn/0wCrjGqkICR9VYFVhdv7fPsSv56y2+UwuuOdDLHSU2454RPc4J36dJpEsT8JvyGg4GsO+tjBO+d3buPftdcp2gb2txjNIX8mmxla8snI7Tr7zLevznjC4GgCwoymu/SzZFBc2n+3cj2WbjXBpKXClX0nl2lMPxJKfnIrLjj3Atl0uCtSPPx+2lnz5JL4J4CXl/WgiWkxEbxPRCea24QDU2MfN5raC0FMd114rWmtP8ciInP2d5HVVIZRsIncKLOsavsxN/oWE/BtJn4dzdaszu6jnByg+uaRjbvJCfY7tpu+qVfGZrN7enHhdZXJ3RgY576k3N8lj9cLALWT09VX2VrWdUYG5K7cnHLe/I679SKF14b0fYtZDC61jvnb0KDz3/ePQuzzun9ixr93qP3HEqL54a02DNRb5zPLvcvTo/tZ5h47og74Vpfj2CWPwyLePsbaHgqaQUMaWjwVbRkKCiF4johWaf2crx9wMIALgP+ambQBGCSEOB3AtgIeJqHeK951FRAuIaEFDQ0Mmj+BKD5URnpOu3FVEMiJnZkH5OdhqJiXzGzjGIicCpwDQ+btSMTdFzYlKbnNOgjqzi/ocwUB8aknN3JTccS2PUSN2kiWXqc/m1CSiDu3AqTUBSp6EiybhFvV0xYP2VrXhSAyLNyY2FVq9vdn6nDrCMbR2RrDN0cq0uT2CslAQvznvMAyqLgMAbN3XhkUb9+CAARU4fFQ/AEB5SQBjayqVv10U350+Fo99Z5p1rdoBlQAMja00FJ+iQ84EE+RHk8goT0IIMdNrPxFdDuAsAKeYJiQIIToAdJivFxLRWgAHAtgCu0lqhLlNd9/ZAGYDQF1dXU5mCTlJ9DRh4TURprLqzBfZ+PvoJu24FuDfcR2NCahtld1qN+kW5Mk0CZtGI+xjdIaL6swuEZsmQYom4R+poXhHNxnHbNsbn0STJZepmk+iJmH/Qjq1JkCJbnIzN/l0aH+4bjc+XJcYUrrcNBFNHNob+zsiWPT53oRjmtuNCKUzDxuKMw8biu88tABrG1oQChDGD6rGgCojKumE8TVoaO5AOBpDLCYQjgqUBu2Tv+qLCCn5FHFzU3xbl/ZJENHpAK4H8GUhRKuyvYaIgubrMTAc1OuEENsANBHRMWZU06UAns3V+JLRU81Nfp67mLr2ZaPCqu6R5WV1fRzccJpG3KKbdGaRlDSJmLcm4Vw5j7nxRdwxd4313tAkjD9itpPp5JjUPhfJhETYQ5Nwage6CV8+wjWP6/tzhB1NfZ5dssWzd/3Mgwfb3j/68SYAwKj+FdjfEcGCzxtBBCvKaGxNJW45y55I2b+yDHtbw2huj6B3rxDOPHQozp4yDHecfxhKgoRIVFjPUlZin4ZV7UFNugtpfBL5kBK59En8BUA1gFcdoa4nAlhGREsAPAngSiFEo7nvKgD3AagHsBZ2P0Ze6amlmzwnqyL8TJxCLZ3QZZ1glCt3L8e1c/XvNI24aRI6YZOK41rOk24+CVtJiWgs4bscJLIml7vfrPcdrebHJxGNCexrDeOKBz+2trWFDe1Apy2FAmTXJBw5H87Pxc0J3RmJoaFZ3+DJKVh++OgS3P/eeut9VVnImuRPOWgQzj8ybtD44iFDrNfTJ9SgsaUTd79Zj/GDqnDVyWPxzeNGY84PT8DYmirbPSpKg2jrjKCpPYze5SU4YEAl/nTR4ehbUYpQIIBILGYJd6cmoSL9EOrrfPskclaWQwgxzmX7UwCectm3AMAhuRpTKgjL3FSEM2MO6XrRTfb30Ziw/bB8XcNDk/ByXDsnTWfoqdQYnN8hnUBInieRKKzk/Z2ToKpJ6LKxA4pP4pH5mzCgsgw/Om2C9wDgz9y0rqEFzy3dYtMIWjqM13vbwgnHBwNkPUf9zmbc/MwK+z2d5iYXIdGqSSiUtIejaG4P46w/v4fvnWyflg4e2htPXDkNvUqC6IhEcWHdSNTvjPeprlIS5c49YjjawlF8tnM/Tj14MAZVlyeUYpFUlgbRYn4GqjMbMCb7trCwBF6ZaaN84PI6VJXZo51CNk3CFCaquanYfRLdmZ6qSXg9dzF+JAnO4phAqqVsdJpEvGCdcq8EIaHvXuZ87/xM7c5wgUCAkpr5bCGwjsiY9nAU89c34qjafiAim5DYrGlvGgyQLaHNbQXuxI8m8V79Lqxt2G/btnTTXrSFo5g8om/C8SXBgDXe55YkJpJFosZkKiBQFgpaoahOpvziVdcxtXZGcerv38H2pvaETO6bzzjYEgRXTTcEyG6lFlZIWeWXBAL4xnGjXe+j0qs0PrX2doS5lgQD2NcWxlG3vgYAKDPvMeMgu5kLAIKBRNNTnq1NLCTc6Kk+iXTKKhQSP6acZHj6JDzMTW4+COd7Lw0kKgQCoNR8EpYmYUyuH6zdjQ/Wfoi/XnIEzjh0qM3ctMmR6AWYPgnlb7jfYxVuG4N53T0tndjfEbGtslVk5M/MgwfjtVU7cJ9p2jlnyrCEY0OmfR7Q+y5aOqOYfseb2N8RwW/OOwx/f3d9wjE6Jo/si6umj8W8dbvxj/c3WOG4TqRDWaW/Wfri7CnDUGJqpQFKrXVpZVl8pVLt0CSCAbJlkqs+CCeqJiHHYs+TKPIQ2O5MT+0n4WVeK0a56fw7pVMJVDdB+8mTcN4rHZ+EdUxKPgkjMsZ5iqyppWoSn+/WCAlyCIl2n0LCvGFTe8RXb+07LzjM9l7XabCiJIhGUztwaiCAkaOwdV87mtojeFmTw+BGZWkQp00agmtOPRBXTR8LAPjl2ZOs8FSJTkjUVJfhv987DnecP9ky8ejCT73oVaIKCacmQTaHdJmXkFBMp3FNIr/mJhYSLvRQGZEkmc6MbikqTcL+3i2hyvsa7kLCW5PQ+yCs91Hp13KOUSS89hNeG3+tP77KnIxU4aUVEkp0E2CEna5t2K/VOgCjsVJbZ9TmYN6n8S+ohAKEPr1KbN+VTXvspq/Ft5yK48YNxDtrGtARiWrHuqclfp9dLmaxv19aZ73uV2F8BtKc1ru8BP972gS8cs2J+Pq0WhxoZkVLaqrsQkMyZWRflIYC1uo9VT9XhWpuKnf6GQK2v5GXJhG0aRLGcTZNIqVRpQcLCRd6qrnJSzhmI9w02zgny7TMTRq5Ii8bc0zOKgmahKtPwl3jiDr8C857WvdSBFA0FtM+p3SEqprExsaWhOMCAftEs78jglN+9zZO+O2bCcfubGrHRbPn4drHl6RUZbW6PAQiQoWyonZGJvWrLMWpEwejuSOCO+euwc7mDpx12FCr0qkcm2TV9ibtvWYcNMh6PWlYHwCJJhkpHPpW2CfsZOYaKRxCKZiaAKDCw9wUCpIt2MCrH0QomU+CNYnC0WOFhMdEID+TXEQ3tXVG0ypBnmBuSkNI6P7WchL2ypNwvnfG9EesAn+Oa2sS9FQlRPcM9s50+mNkhI8fc5OKs8icirzP/PWN+vwOl89bdlFTHbg6xg0yQkf//u56y89RVa4/Ry2praKutqWfJOgyqTuFRDIsc5NHmKoOVTgmOK4DAdvfyK8moc2T6MohsF0dq31pYYeRd7xW4vJ7nYvVy6UPfISPNySWREhGggkoDZ9EuuYmZ9Ka87PTJeQZ7xM7oNkEh06TiNrHoTtGTvaqsNqpMdEEAmRbQTuznN9cvRPRmMDMiYOt+zS1h7X3dGsOJEM5K8uC2JXoarAY3s9e6TQUJEuIDe/by5aU5wcpYNy+ompBvd9fODnp9Uo0uQl+UM1N/Svsfo9QkGwmSE+fhCokpLkpzyHoLCRc6LGahMdz59LclI6AALLlk3Df5uW4ThYS65YnoWtFaq/yGgNgN0E496sNgOTlZXSQvG9NdZk2vDVI9mlG5jFIvvFPIxHuh6eMxzmHGzU2w1GRIIDvmLsaVxw/JuH6QNzE4qxyWlEaxAnjB+JLk41IJ6epJRQIWFFEw/v1QmNLp2ent4unjrK9d4u4klw2rRYfrWvE3ZccgcG9yz2PBVLXICSqualPRWIIrEqqmgRsprS0hpcSLCRcKELze17wEhLFKDiTTdx+0EV0CV+aRBKfhNWZzn5tZ0Obf76/3lYzSJuRnWBukgIofozUJKTtv19FiV5IOEJg3Wob3fXGZzjrsKHWe2eI6t1vrtWWugbi/ZgHVZcDiEc1VZQGce/X67TnAMaEKTWJXiVBlJUEbEJi8og+6FUaxLx1RpGG35x3qO18NfRUx6De5Xjyu8d6HqMiJ+ZUv1XORkQqTlOY3xDYQuVJsE/ChVgPtTf5cVwXsGFgAl6rdL94aRLq/JksJ8O9dpP7eTEh8LPnP8HclTsSztOdUxoMIBqL4dL759v2ByhuNpLn961IDO80jqWE7nA6hADalZIfujLa79freyxLs0/vXsb/I/sbZiVdxvR9SnRSKECWJmFEF9mnqO9OH4dvuWgvQNzMk63vqPP+fqkocV9/OyOlvMxNNk3CypNQQ2A5T6Jg9LRyHBJvn4R0XCenMxLDtY8tcQ2rzBZu5blTwcsn4RXd5CzD4Rrd5Nyu3M9LIKjI5yoNBRCNxfsyfH/GOPzsSxMxYUhvtHREceE9H+LGp5cDiIeDOlFLhSdDraUUiQlb5BGgz20A4mYfGf4ps62bNTkZMycORnlJ3EEs5+VQgBLqGg3tU+658vYj/FIh1dBXiTQ3ObvLAYbjWsXreVQhIJ3orEkUCT3V3ORHSPjhg7W78PTiLbj5vyuSH5wByfwC6VwD0Duuk0c3OX0SenOT2l9BKxBiAss270XtDS9anc3keEpD9siYMTWVuPy40agqC6KlI4L5GxqtfX176TUJp7nJC2cvCKdQ3mzmPvyvo/aTTCCTwkJGMbkhxVap4rgOBihhAnUTEnLFnWKkalKcE7rv84IB3P21I/CUxrSVqEn4qyOji9jiENgCYvWTKPA48o2XAiUnCD9fTLkCyrVG5rx8tspy6PIkUo1u0uVJCCFwyX0fWe91VU2jUYFXTPPTu58aTbWkwCkLBazeBUC8tk9FaSihgmrfSmOiHuJw0BqTjf6P+Oj8jbb3zmu6aWoX1o20vZeO64nDjH5iw/r0wos/OB6vXnOi9nz5nQoF447rkmAgQZMYWFWmFRLvXH8ynr7qWEuTyNbcma4mARi9JYb0SXSOO01YXuYm+3mashzF3pmuO1OMiWPZ5oO1u1B7w4s2k5BX5m8qpUrkoifXVrtkuQp+8BsCmzy6Sa/VqNd3Ool1sf+RWMxyQleYK/EVZkmL0lDAlu0sHZtVZaGEfAepSfR3mIicZTlUbjBNVZJkPacB42/dv7IUH9wwA2NqKq3xAEap7cdmHYML6kZg0rA+GO/IeHYSCtg1iZKQ8fqe/3ck5vzgBAQ0JijAMOscMapf1lfW6UY3eV5T0QguP7YW5SWpaRLdtcd1l6MYWnXGYgK/eWkVtqYYK+6XJxYYLcXnr4+bKDxDYK1kuuTIFU6uI6KyUeDPdzJdEv9HNCawt7XTmtDlueppTsetWujNum5MWIlxlaVBfLh2N15Ytg2AsQptUuz6cuVcXhJM6CshE8dKQgHL5g+YGdcJd9XT2uHd3wEwGuwEA4RhfXthnNlXQWoSRISjxwzw7WAtDcU1CdUnMaCq1NJK8umTKMm2/Qp2wXPI8D7+z7N8EvmdlVhIuFAM5qblW/bh3rfX4X8e1XfcyhT5VVOf0TPjugg1iWT1lPxdI3GbFdym7EuW3R2JCXz13nk468/vQQihrQLrNC9phURUWL0IKspCWK2Uo5BlpiVyVVpeErA1DyoJEirN1XxpkLDollMxbcwAANIn4W+iaXXkKOiFRNxBLnsjJMtXcCJHEwoErO9OKEiWaSasfG5eTXqsVIIszaO51iRSGSZrEkVGMVib5MrVb4/ebJC1EFiS18veB7nw8z0496/v27J8E0peZClPwjI3eSTTRRJ8EjGs2WFEHbWFo3G/hnL9RE0isT9CNCasFXwoQLaM6FCA0KQIiWAwrkmoQiIUCKDKjLApCQZQURovdxEg/2vRVkei3RXHj044pkaprCrt626lNZIRUiqkhpT+DeMGxx3fnppEllf+0ieRTd+a6ufIdJLv0j4JIvoZEW0xW5cuIaIzlH03ElE9Ea0hotOU7aeb2+qJ6IZcjc0PxZA4Zpm8cvU90FzXM7oplY9EsxJXWbWtCRs1dYW8uOnp5Vi8cS/WNcSL1mXHJ+G+zZlM9/nuFtz37joAms50yr33KL6Gj9Y34vmlW9EZieG2l1bbzllqRi+pRGIxS5OIxoStNEUwQDZBY9MkHNvlBCJX43L+DAYIfoN2nD6Ji44aiUunHQDASHarHVCBX50TT2iTE3h1WWo1kqzzgwHLZBQKEE4/ZAg23HammZRnv4eObOcNpBvd5Pea6QzXnieRjRF5k+uM6z8IIe5UNxDRRAAXAZgEYBiA14joQHP33QBOBbAZwMdE9JwQwr1jeQ4phvalIgUfQDbuAyTJuE4hT0I2t3e73hf/9C4AYMNtZ/ocZTwvQY0OyZVPQs2TKA0G0BmNYe7KHbjy34sAAOcePtyzM90epW2o7Kd8+1cOxfNL7d3X1Exr9TrSJxETAhuVwAJnGKR08paHgrb7h2MxSwOVUTHyXKMsh09zkyO6iSi+0q+r7YeHrjjatl9qEs7Kp8mQE5+qSQRdIou8fRIp3TYpmUQ3+bl5gfLHAAAgAElEQVRmOpqALU+im5qbzgbwqBCiQwixHkA9gKnmv3ohxDohRCeAR81jC0IxmJtyjfyC2nwSfhzXPr6Ysohdtj7GFVv2WRpESZDw+e4WfO/hRTYTC5Dok3hl5XbcMde+eld5etFm3PX6Zwnb1bIccpJ9ZvEWa/+e1nCC41rVLC7++zzNNfVjcDbCicSEZeYJRwVWbWu29jmFhBTGzgiZkkAAJx1Ygy9NHoaffXkSgPhEHAiQ75WHU5MIBsjSXnTZyDLmP21zUyBelsOtPLeXT8JZ4TZTSnIiJDLVJJTXXdncZHI1ES0jogeIqJ+5bTiATcoxm81tbtsTIKJZRLSAiBY0NDTkYtzFYW4y/892xAZgTLpPLTKim1RNwl8yXfLxhKPemkSqnPXn96zXQgA3PbMcLy7bhg/X2lfizvHPemgh7n5zret1r318KV5akVhuQq3gWqJZuTa2dCaYm25REgd1mcVuTtAah5CIxoSVn7B+1340KlrJ1Nr+tmM7TCGpRi8BRjRQeUkQf774cIzoVwEANjOOb5+EQ5MIElm5GbrkLssnkaLjWlIaIiW6Sf95eTuus+yTyIm5KbMx5rszXUbmJiJ6DcAQza6bAfwNwC9hzHW/BPA7AN/M5H4SIcRsALMBoK6uLiezufz9F1JUyJVqLr4I6qTbqayI1TldCGGzf6YS3aQL/8wWUSGsyqXOFXQ6ZTl0WOYmIbQr5saWjgTHdTLcymo7hUQkFn++hZ8b1XHvvGAyhBD40uRh+Mub9co1jTGUOT6HAZqOa3JR7CwVrnLQkGqr5AeQqEkEArDKZuhW2eccPtwSUKmgRjfJ74ybJuHlnM5+nkSuNYk0zE02TSL3ZCQmhRAzhRCHaP49K4TYIYSICiFiAP4Ow5wEAFsAqOmZI8xtbtsLQjHUbkonnDMd1LBMe/E5+3HJWmyqyGzkXHyOMcVm7ySdAn/aewjjGXbv70zoBwAAu1s6rQn66pPH+bqmWixv6ui4RuBsoRlVkuk2NRpO6zMPHYoL6kaivCSI1687CSPMPgxyxe6clJ01loD4KttZKlylf2UpfjAj/jwtGnNTXJNInD5GD6zEpdNqXa6eHGNSNv6Gbj4JL+KaRHamz3QL/HkRyHCSz3fuVi6jm4Yqb88FIHXx5wBcRERlRDQawHgA8wF8DGA8EY0molIYzu3ncjW+ZBRDxrWcaLNtd3Q2mbGHlKpahf0ziGsSyT8bS5NIc4ySfa1hm7kFsGsSTp+EfJbd+1Nf6avEYgLjb34Jq7c3Y7CmtMLu/XEhceX0sb6u2aYIth+eMt567dQkmtrixzU0d6CiNIheSunpsTVVePNH03H3147A9Ak1AIByh0nsyAP6wYlcgXvVbgoo5iQgMZkuQHGfRFadxOa1SoMBa3GSTmRRtq1D6ZYK9yLTPId850nkMrrpt0Q0BcbnuwHAdwBACLGSiB4H8AmACIDvCSGiAEBEVwOYC6PjygNCiJU5HJ8nRSAj0BlJL+17b2snOqMxW9igyqSfznXcJz6ZxoS7JqFLDnMjkiWfxORfvJKwTY3+aXdkGW/f1459rWEc+avX8F2fk7cOddxDNc1pGls60bs8BCIjFNQPalisWtNnoEOTUENeO6MxDOqtqSQaDOBMpdeDqkmceGANvn1CYjltOakHPKKbiOwmlgRzkxLdlIv5KRQMWJ+9WwtSL7KecZ0DTcLmU8hYl8i9lMiZkBBCfN1j360AbtVsnwNgTq7GlArFZG5K9WtQ96vXEIkJ3+GlHTZzU3x7QnipZadPfs2IT5/EZzuak9bzcRKLxbupObuWbWxsxZLNRu7BAqUiajQmEAwQVm9vwqDq8oR6Rgn3UMat0yTmrduNY8cORFko4HsyU8Ni1aY0Tk3CWV5dZzpyYhMS4wdq7fY2x7WHJqFOtAmOayW6KReEAmR99sn8AQcOTqwsK2382cu4zsGzZppAxxnXxUExRDdZ5qYUvwip2uXdNAnnRyDNTX4+m3hxO+/jTv3DOz5HGScqhJUD4DQ3bWxstRLUJg7tbRtPLCZw+h/fxaUPfIRkqKYqZxVVwOjn8MD7621lni84cgQ+uukUWxMdlcbWuJBQz3MKCVl+W5JMoAH26KZkDt8qUwPSHkP28xNCYFVNIoszlLxSaShgLdC8IouW/vQLeO7q4xO2Zz1PIgfRTZnmOeRXj2Ah4YoV3VRAWSHDSJOppC0dETwyf2Pa2o+bTyIxUS1xeyQaw0PzPk/spRDNXTKiau5SzU3HjxuIjbtb8alZGsPW3Ccaz1z+dLu+UY6KWgrF2YpyTE0lThg/EEB8cv70V1/E7V85DIN7l2NgdWJkEWA3N6nloYc6NJXPdjbb3usilZyomkTQxUQiP46qspDrd0o1JwGJZUQoEBciOTE36Xo6a+jTq0QbQZVtc1M6Jq9k2DKmfRw/tbY/RvWv0J/PnekKR7FpEgs2NOJlTTw/APz25dW48enlePvT9HJG3H0S8dePL9hkFZpTJ+knF27GLf9dgb+bpSokujLZ2WLbvvhKWwq4N647CceNG4jmjgh2NLUDgM3hHYkKS3jIctZeIb3qZ3L4qL62fUIAtQOMa0iNwFm9VIfquFaFhMxjkOxo6sDBihbky9ykaCZu95daV7WHJkFEniaWIJGrEMoGJRn7JIz/szV15mIKTlWTePzKaXjn+pOTXitXsJBwoQhkhFX5kgg4/54PceW/F6KtM4qfPbcSTUrjGRmmKCfHVHH3ScRfX//kMnxgJq6p2+WK22ki8euTMI5J7cN+0SybDQDvfrYLgDGhVJoF7XY0GZVV1eJ54VgMn+00NIjRA40JXtdvWSL33XLWRJs5aGBVKW495xCrPaUziU2ORYdqugkFA7jj/MMw46BB2uOPqo1HJ/kpcaGOw/3+hpAyNAk9AfKenG0+iSzOUGpZDitPIi1/QHanzd69SnDw0N644/zJWbumXTCkkSfheq3cwELChZgVwpk/abF44x7M/P3bCQ3tVdPAEws34Z8fbMBf3ognVMk+wmroZCrYNAlVArg8unqMjNPf78gwTiW6KVlRvuqyEF74ftz+LCd7lQCRFWW03RSWa5SksEhUYFezITwqSo0xO53e9jHF6x6pJozXr5uOY8cNRKV5DV2ugLsmYb/fBXUj8cDlR2mPlYIM8K5VJFGT6dzuL4VUdXmJq5lCDXF125/t0hcqJUE1mS6d6Sm7v9dggPDSD0/AqRMHZ+2amWZMd7eyHF0WP77fV1Zux6UPzM/aPX8zZzXqd+7Hss1G05pOjeNa/rjVLmS9exkTlqpdAHHn68cbGnGeo8S2SoeLuen7jy4GkBjyqh4jQwSdXdHc+jvrSFYKPRQkm1+gIxJFpcNPEAjE+ydIoaf2XYjEYpaPQtaV8hIS8hqhgD16SY5D3ktnsnLLCHY6gb0os5mPkv9M/WgScvHh6bgO6AWf7dqWIpH9CSpIZH2/chJZVARknEzH0U3FgdV0yGOSm/XQQrzzaUOC0zZb6EpMyKQpNapHrmrVPgNAfPL/v6eWYdHGvdi4u1Vr2rFlXCv73zF9HAm9nFVzk3luoiYRs85t7YzgkJ/OxeurjL7Nzom1w2OyBgzTjFz9G8fHrLaekiCRLeHMSSQqLGEnBZhzZa8iBVcoaF85S6EohZQuK163Ei8NBTyFUrXjedSSF36KzJUGA/E+0S6TvBRSXuYmo8qr+30COZqgpPM+QGTpArkMtS0omkVfaqfn93NhIeGCKiSEEJ5OTmcYZraIl7aIb5NmBbVVpZzYnUXldDZ3XXisVxMf572M8cQPkpOpU5OQE/LmPW34/sOLsb8jgjtf+dQ2Xq9xqpQEyJaw1hGJJSSwBQio8Ehqi8RiCS1Jm9sT+0tb9wgr5ibNZCWFlC6xULeSV4XA85rQzTk/PAG//cph1nvVxOQnoYuILOe12wpcFg30clw7M65198mFBfaf35iKOy+YjD4VJdb3KxeRRcWAPZkurQvEX7ImUTjUAm+zHlqIMTe55/g5s34zRf5IOpXVuHNfWziKvWbcvVy1O81NiSUrYtoJucPNJwEj2c2Z9axrx9ncYb+36md4ffVOAEZhuMaWzoSJNZmQIIeW0BGJJkyEqrlJRa74w1GR0G3u3L9+4HpPS5NwmTDjmoRPIaE4nwf1TgxpHdm/AucdES96rAoGvy00pckpmU/EEBL6Y8jjfN2x2WJIn3Kcf+QIAPGFSi6ynYuBjMty2K7FPomCIb+oMSHw6ic7PI91s/UDwNa9bZ5mDR3t5vWkuUmdVOW13li9E1N+8Sq272u3qrg6HdfOybctHE3osQzEbff7WsPYus8epTRP0xRHDufz3S3W6rTF0eYyqjHDrNjShCN++Srer99l2+5HEysNBXDruYcAMJ7LWdcn4GJukoJDNTf5KSsiPyc3U480f+k0TJ2QUPsruMXyq+epE6TfngYyb8DdJ5Hc3HTYiD5JV/C5DubIhiaRjxV2umScTJdinkWm5LozXZclrkkkP9ZLkzj2tjdwzJj+eHTWNN/3buu0axBhZRBOk9LGxlZLk3DavNdsb7JNCG2deiGxp6UT4WgMR/7q1YSVcZOmL0IsJrCvNYyT7njL2ub0W4Q9PrhFG/fY3vt16MqidUIkmlSCRAlJb4AxOe9s7kA4FrPCe+Uz9i4PaZ8PiAsJN9OLDLfVaRI67aNS8am4TX5k833oBYYXUki4aT9fmDQYzy7ZispSvSYx5wcn4OCh1XjD1PwksjOfJNdtdbu7T8I+yXMIbJdk2ea92NzYlrDdLZ7fbSUsV5nz1jVq97shrycnXrVERLOmgqvliHWM48p/L8Jxt79h/ehaO6M2refo0f3xnZPGoKk9gnnrdmsnPF3znJgQ2NNqr8zq1FqiUYF+FSW2InQSZwilW9lvr/OcJhgKwObclo1pqhTfgWpuemvNzoTPUkV1XOuoTNEnoWo5fkJIS4Op+SSAeIKemxC64/zJ+PjmmUY/Cc3+icN629qTSnS5IEDuHKjyI/XqG+FGMeQ3JcP2588wBDYfsJDQ8OW/vI8Xl29L2O5mO3/wgw1as0MyW7vko3W7sXLrPuu9NDdJu74a5fTYxxtt57aHY9aqVyesVM2h3WFuGlBVatUl2qQRigCwvyPRuRsTwhZeKu+jCrNwLIaK0pCtfpIbTlOVG+qk4ezu5dQkDhhgZDFLIRGOxiwh+NnOZlz+j489JxTL3OTqkzDNWD6FhJph7Sf8X+2G5zcU1NIkPHpDy8RAr4nGqYmowhfIQyOuLPR2z3cEUCpkKCOyUEU2NVhIpICbxvDEws14ftnWhO1+o56+OnsezrzrPesbI/0Oliah2PdlNrGkqS1s7Xe7n/waGZqEXXD1MxvqbNkbrzx6stmjAEgMbQWMlZ5TSACG1nH3m/XY3xFBNCYQClJC8Tog8dy2cOqahHMiDhDZVtzyvqpPQgpy9f5uiWrJNAkZXaVbHOjMJLbaSj5WyOqzeLXrtN/DW5NQ8ZpcnKfrzHhA7la0liZRzI6FDNDlPaV/fhYGlAT2SaSA6ntwmp50dvV2D4e2FvOSHZEYGpo78OySrdZ7N976tAHPL92aMD7NZdHaGbFdSwigb4WRrS3Laiz48Uy8+1kD3lxj5EioRekkOk0CAF5Yvg13zF2DzXtaEYkKhAKEgVWJdYec13zs400Jx+hwc+wCiatztZgdYA+BVef1slBA66dJ5riuKg9hwuBqXHPq+IR9STUJH79s9b5+bfNSEPlp1uM1BKfYcwYE5NqkIx3j6UyAXcDaBGQYApupJpIqLCQc+M2HcE7IZZoVqXpMZySWtLxCh3RAd0Zx33vxgnlejl0pIADDJ+FVB6k9HMXPn7f3cbI0CVNIVJaGbJPYrv12zQUw7PA6IdFujvOzHfsxoKoUoUAAfTWtP/c6zvXrs1HNTTrHtYr0P0gHczgq4v0wlL9xWSiIZiRqMmrGNQBcNu0AHKF0ewsGCHOvOVE7Tp3PQc2g9iMk1O9TiY+yHEC8yF+28wucOSly+LkKv7Qc4xlMgcWshGSsCeRZSuSyfeljRLTE/LeBiJaY22uJqE3Zd49yzpFEtJyI6onoLspHELADP0XfgMRELJ0AUIWKbrIF7BPWhl0t1nmtip3e2W7UjWhMaLN6pfmquSNilfwAjB9j3NzUhgAZJgtKIiSEi7lJFhjc2dxhaBJBsq6vIvM7rjv1QF/PJbE5rjUhsADw4zMPxn2X1ilCQnFcyxBYRZC6OWWlg18Ko5+ffQjOnjJce6wTncPVT9kMFVsIrM8aRlaehA8fhvOX1dujiKBTkzj38OH4at1I/O9pE3yNK1XkT6KYJ/pMsM/x6UQ3dROfhBDiq0KIKUKIKQCeAvC0snut3CeEuFLZ/jcA34bR93o8gNNzNT4A+MIf3sbDH9kdwV6lE9R9TpPJ1Q8vxisr7aW8VSHhTHQDgDvnrsGF935ovZcTb3skinqziF2/ihLfDnBA70PYts+YvBuaEyd8udJuaO6wQiPVOUytpCpx0yTkfXY0tSMSM8xN/UxzloqMjPIqo6FDnStLgoT7L6tT9hmD/tYJYzBz4mDLtFStOK6lcBAOc5MOpyaRKXZNIvnxNiERyk6ehIqqzVwz80A8fdVxrsc6fRLlJUHcfv5hvpohpYPUhrurTyLT5+p2tZtMbeBCAI8kOW4ogN5CiHnC+Jb8C8A5uRpXLCbw6Y79uOmZ5Rh30xy8vGI7hBDaSVZyzt3v46ZnlgMAdrckTrg/f/4TAMALy7ai9oYXbZNyS0cE4WgMQgg89OEGjL7xRfzlzXos+HxPwnXaOmOob9iPC44c4Xv1KvEK63QKCQFhaUCRmLDKX6tfYl2I5/amdsx+Z13CdtnUx8jsjiIUDFgValX2thgCxhk144YcjjME9pSD3StzyoAwXTKdijp5A8C3jh+N4X17WZFl2Soyp2oSfhRkW8a1b01C5kmkpkl8+8TRGDcosRWoxG8P72wxop/R09vNYe5FVwuB7Qqd6fLhkzgBwA4hxGfKttFEtBhAE4AfCyHeBTAcwGblmM3mtpygrs4jMYHrHl+CVw4ZgqcXbfE8b/56w36uNrSRSBPHvW8bE+iqbfFS1c3tEZz2h3cwpE+51ZfBjfZIFE1tYfSvKtUW+fNCl9cg2dmUKNjUSVKGdXp98Ub062U5uavLQ7b7qc2ANjW2YUS/XlrTixRkficB+cMPakJgx9ZUYm1Di+Ycu7lJVw4E0JubiOLRTX5NPclwCqNklKYR3VSWZnRTspVtqhpfpvzuwin4oH4XapVy6alSzEpIxrWb1GsVe1kOInqNiFZo/p2tHHYx7FrENgCjhBCHA7gWwMNElDyY3n7fWUS0gIgWNDSk143NGS7a0hlNKiCAeFiqTkhs29eOJZv2Wj9S1cS0Znsz1u1qSSogAMOH0BGJoTwUTHkl+8Yq9xIiO5sNc9DwvsZKTQjDbCO/Z1KTcPvi/frcQ3H8uIHW++kTBtn2q+G5W/a2YWR/e8c1wC4Y1Ala1u1RueL40bb3Osf1k1cei6evOjbhXCmwZXTTrXNWWZqOim7yDlDifTLFT08IFdXE5DtPQhb4yzC6yUmvkvzGt/TpVYIvHpqYhNldyDyZLntCxg8ZCQkhxEwhxCGaf88CABGFAJwH4DHlnA4hxG7z9UIAawEcCGALAHWmGGFu0913thCiTghRV1NTozskKV6+By82Nraifmczdmts9YBhklqxxXAO71TMO795abXve2w3bfu9SoOeK9n+laWYdeIY27a7lGZETqQf5azJ8R8gEVl2eWn+cVuIzjx4kM2cpRak0zFeY8JQW4GqJS/OPTzxWqccbBdC9jwJ49x+laU4YlQ/OJF5fWrRv10an0yZRpNQnz9bQiLVYnUlaWRcyxW/L8e1+tpx+DBzESFJx+xTKGoHGguTo2r7F3gk/kjLcd3NfBIzAawWQlhmJCKqIaKg+XoMDAf1OiHENgBNRHSM6ce4FMCzuRpYuuW9hQBm/v4dPDx/o2tEiMzC3b4vceWqc+Q6+WSb0Uu6PBTQ5hlIojGBo0frfwxe95F+Aml8KXX0SHAzP5QEA5Z56ZqZB+JkhyYBABMGV1uvDzRfD1cmHXVCDwZgPV+fXonjlZOj5ZOw5Ul4/zqkuSmZPb08iSaRLXOT3yJ9klAKzyoZ0a8XqstCtjpRbnh1Nxs9sBLv/d/JuHTaAQBS14IKyaRhffDu9Sfj8mNrCz0UV7Lrkyhyc5MPLkKiw/pEAMvMkNgnAVwphJCB8lcBuA9APQwN46VcDSxdTULS0NzhWhxOIqN9VL40eZjnOVOVSb9XaRCXTqvFj76gDxWNxYTrJPizL0/Cf7+nj1hx9kyWPSrkqtvti1sSCmC/aUI7bGQfAMbEPX1CjSWUDhhQgUFmtvOk4YYV8Y0fnYRhZlOZScP6WNcLEFlhlM7VKxCfKKVPwmYGSjJ5S/+DOsHp/uZuPgnrPgUyN9mL/fk794xDhmLeTaf49CGoPonEvSP6xU2FXa1D3Mj+FXmx1adLpj6JTM1VqZJTISGEuFwIcY9j21NCiElm+OsRQojnlX0LTHPVWCHE1cIrMyxDstED4sK6Edhw25mu+7drhMSIfr2w7tdnuE7uR9XGV9rlJUEEAoSrZ4zHyp+flnBsVAhbb2OV0mDApk2oJgNpp5efrjQ3yRWoWyntUkWTkCv/tb8+A//8xlQMqjaEwNA+5Xj3/07GJ784zdpWFgpaP1q1l0IoEMBXjxqFdb8+QxtO6ZwcU9EkZLirGuKqe6ykPomsaRLpX8fvJO3WU0OHn9IQ8V7TxTvhdkUyLcthy9ju6kKimEnWMhNIbqr47fmTPffrir+VBgMIBMjqS+1EXVGr9X4qy0IJAikSE67JYKFgwFYpdYBitnKGnspVrnRcuwmJkiBZHeic5iG5eh3SpxfKQsGEe8hoITUkVs6/btU+pVCQP4RUfAW3nDkRg3uXYXSSCBm9TyL7jutMhES2TF4qAR/OT1kew6tTHZM6GZubbKbC3NNj//p+zE0HDo47Xr80eZhlowWAe79+ZFr3LQ3FJ1NJpbLKl1VZAbuQ0EGwC7LpE2qsFXYoSLbVdv9KYwUfIHWylatt09zkUdkUMFY9Tk1CIgWL7FXsRCanqX4cZ/mKi6eOwncUR7xTW1AT/ZKt8E8+aBA+umlm0s9Qt98WRZWlVXSqPgmVdEpmJ8PLcS2xQo9ZkcgqGZub1Nd5UCV6bO0m1dw0blCVleGsxv6P7F+BpWYZiz9ffDjaw1G0dERx3RcO1NrQ/SBX7UN7l2Opua2qPIQWs3TGgKq4OaZcY8f+yVkTMaxvORZt3IszDx1qm+T++Y2puGj2h5i3rhGhANni6weY5pxeJcGEL1aZiyYxeUQfHDy0Nx5VCvDJ+cqZJCcFyxAXIRG/R/wr51yl/+a8Q7Fiyz7caybqSUGgGh3ly1Qm3ae+eyy+8jd9q1JdxnVcEFHGP8KbzjgI4wZVFV0rTj8mD7lWCBbZ2Ls6mWsS+ZXaPVZIqJrEOVOG4c5XPgUAPP3dY3HZA/OxdV87JgyuxguI95UoLwnidxd6m5gCZPy4Lp46Cl+aPBSfbG3Cr15cZe2XQuLAwVV42ay1V1UWwg50WK8lOgfkN83cgdMPMcJY9zjyNaQ20B6O2SYmKSQGK5O4m09CTvhja6pw21cOw9eOHoXFG/cCAB6/chre+bQhwREro5R0tZoA4OFvH425K3fYnk8XRaX6HXT744l1/icur7pEeiER18YyZdaJYwHEkzCLBX9RMbI8Rm7H0tMgj3epns/mphwiQ2C/dfxoXKqEy40fXI03fjQd158+AbNOGuNytjtDTTNSVVkQx44diG+dMAYPf+toa7+clK46eRymm30bqpRVuTqJJjOV6I6RK/XWzohNSMgey0N6l1vRTYNM05ac8KXTUxbCkxP2YSP64jLzMzpoSG9r4lP53YWT8ZOzJtpMdCrjBlXjeyePs23TZQbLCToYiCf56RZOqWgSXpFFWnOTeels+gIyMTflAj+L0YGmVqsLT2bSx5YMl6lPIg9fqx6rSUgh8YOZ460icJLykiCumm5MaI9/Z5qrk9nJXy85ArPfWYcte9tQVRb/YR2rZCnLCau8JIhrZh6It9Y02O4vC+4B/mrmOFfCEwZX4XkYZjN1Yqo2BVHdAf1w9Oj++N0Fk3GGmdUa90nY+zanspIeVF1uaTl+0WkKcoJOVooiFV+Bl6lHp0nIz6p3FidHv+aml//nBM/SKvnk+zPGY2T/Cpx56FBcjcWFHk63IVPHc76rwPZ4IVGuhGfKlpcqU12S1XSccehQXPWfRQCAo0YnZgEDQJkyWchVrKo9qIlQusgbJ9KpOWmYkZPw3enjcNCQ3jh5wiDbiuXkCTU4dHgfnDyhBkSEryhlMOR9Kqyy2oa/JtdVOHVCSI65NBSwwnZ1GdXOHtdeeGkSuhBYGQmWzSqnfvMkDhoSr1BTXRbyLNiYCX7+tKWhAC6sG5mT+/dkMnU8syaRJ9rCUQQobgZ47dqTPLObvehdHrIS644Z0x/z1jXi6NEDtMeqk4UMX1WT29RIFr/VN1+95kTLdBQMEGZOTKyOWhoK4FTNdkD1Sdgd17mOj9c157E0iVAAA6rK8OIPjsfYmkQTVrbMTTpBLIXDgDS/DzrScVy/9b/Tc6ZVFHOyWXcn37WXMqXHCon2cMwW6eNVKjkZ790wA2EzxPO+y45Ca0fEtRKnXUiYmkR5CNPGDMCH6+zF//z4JADDj5IML/ONs3bToSOMrGjVTJYLdKGdliZhjlfN0FZJJcnN+9k1moQpJKp8Jqb5IR2fxICqMlu0WzbpCpNTd8VP+LHn+axJ5If2cNT3JJyM3g7Hs9fkYhMS5gRVXRbCg9+calWYlWQzbNLbLi/Lchj/H3lAfyz9yRfQx0edqUzQaSpSi+MJq58AABLCSURBVHFrBmSdm4om4fXsWk0i+xOz33Lf+aK7NvTpCnjVzfJ1fnfpTFfs3HjGwXjphyfk/b7qZFFZFsSEwdWYNLwPSkMBK7ro+zPGZT0axqtPcpkjuglATgWE/Ax0E5VMuksmIFMRoIEAWQLJKZd0Bf56lWb/Z1HMeRJMfsnmxM6aRA5JtuLPFaomEQoGMPeaExOOue4LE3DdF7LbP9hL6DjzJHJNaSiAzmhMa5KT5TuSOXpT7ZZWGgog0hlFaShgJVIO7VOeswqnlxw9ytYJ0EtIFwKWEYWjq5Xl6LFColAUquyyl7lj2tiB+Lyx1bUOVLaRAksnJA4aUo3DR/XFT780yfMaNdWpmYRKQwG0dkZRGowLiQ9vPAVLN+1NOHbGQYNxVG0/XHuqvvquH24991Db+6Irkldkw2H8k3mBwNRgIZFnyoKFaeDiZe6YNnYApo3VR2Plciy6Gr/lJUE8c5W+xLlKykLCvKdROyseMeQ0eQWDhD69SvDElYnd7jKh6MxNLCUKRub9JPIbHVVc39weQKE0iWKapGS58GgGleAHpJjDID93p0NcDZL6xnG1+P6M8WmPyQs/fafzCfskCoc9BJbzJBgHhRMSxTMr3HfpUXhpxTZbx7pUSSWZDlA1CfceFclMXN2JVL8Nf7vkCAzN4O/FxMk4BNZ2Ppubug21AyqwYXdrwVaUxZQ8NaRPOb5xXGolPDJFalIJmkQRfS75JNXvwxcPHZr8IMYXtl4eGTqu80HGy1oiuoCIVhJRjIjqHPtuJKJ6IlpDRKcp2083t9UT0Q3K9tFE9JG5/TEiyl7Ka4F58rvH4rFZxxR6GD0WKZyPPMBe4qOnCokis371KDLNk8h31EE2NIkVAM4DcK+6kYgmwuhxPQnAMACvEZEMF7kbwKkANgP4mIieE0J8AuB2AH8QQjxKRPcAuALA37IwxoIzsKrMqqrJpM+715+cVglveY7sAyKjjYIu+RPdHXZcF45sZlzng4yFhBBiFaBVX88G8KgQogPAeiKqBzDV3FcvhFhnnvcogLOJaBWAGQC+Zh7zIICfoZsICSY7jOyfWITRDzKvIhggLL7lVKvSrd9Od90OlhGFI+MqsPkll7+M4QA2Ke83m9vctg8AsFcIEXFsT4CIZhHRAiJa0NDQkPWBM90PmU0eJEK/ylIrhFaam3qcjGAhUTBsIaxpaRL5/eP5+mkQ0WtEtELz7+xcD1CHEGK2EKJOCFFXU1NTiCF0Gc6eMiyrJa+7KrLsuDNwIG5uyv0Pb3jfXq6VePMNy4jCYf+qpVO7Kb/4MjcJIWamce0tANRi9CPMbXDZvhtAXyIKmdqEejyTJn+66PBCD6EokCVY3IREPqLO3r9hRs7v4ZdiinbraXQ1n0QuleznAFxERGVENBrAeADzAXwMYLwZyVQKw7n9nBBCAHgTwPnm+ZcBeDaH42N6ELIMuiwgKJE/uGJLdss1Petpi4tM+0nkO+ggGyGw5xLRZgDTALxIRHMBQAixEsDjAD4B8DKA7wkhoqaWcDWAuQBWAXjcPBYA/g/AtaaTewCA+zMdH8MA8TLoLZ32Jj7yB6drgNSd6WGPW1RkmgzXFaObngHwjMu+WwHcqtk+B8AczfZ1iEdAMUzWkI7rFkc7UGGWBtE1QOrOcAhs4ch3FddM6WExHUxPRbZmbemM2rbLldywHlZygjWJwpF5dFMWB+MDLsvB9AikT6LVoUnUVJfhj1+dguNy3Kq12GAhUUC62GfPQoLpERwwwEjCG6VJxjvncG06TreGzU2FI+P2pXmW8CwkmB5BXW1/PP6daTh8VN9CD6UoYE2icGRc4C+LY/EDCwmmxzB1dP9CD6FoYBlRODL97LtTngTDMEUKJ9MVjmx2pssHLCQYpgfCIqJw2KObij9PgoUEw/RAWJEoHJnmSXSnKrAMwxQpbG4qDtL6M7AmwTAM033JOASWfRIMwzDdl66Wcc1CgmEYJo+wT4JhGIZxJcOeQ8XZmY5hGIbJDvZ+EsXfmY6FBMMwTB7hznQMwzCMK5n7JLqQuYmILiCilUQUI6I6ZfupRLSQiJab/89Q9r1FRGuIaIn5b5C5vYyIHiOieiL6iIhqMxkbwzBMMWIzN3WBCn+ZFvhbAeA8APc6tu8C8CUhxFYiOgRGq1K1HvMlQogFjnOuALBHCDGOiC4CcDuAr2Y4PoZhmKIlLU2iK5mbhBCrhBBrNNsXCyG2mm9XAuhFRGVJLnc2gAfN108COIU4LZRhmG5MV5jh8uGT+AqARUKIDmXbP0xT0y2KIBgOYBMACCEiAPYBGJCH8TEMw3QZiq6fBBG9BmCIZtfNQohnk5w7CYbZ6AvK5kuEEFuIqBrAUwC+DuBf/ocMENEsALMAYNSoUamcyjAMUzR0i850QoiZ6VyYiEYAeAbApUKItcr1tpj/NxPRwwCmwhASWwCMBLCZiEIA+gDY7TKm2QBmA0BdXZ1IZ3wMwzAFp/j91rkxNxFRXwAvArhBCPG+sj1ERAPN1yUAzoLh/AaA5wBcZr4+H8AbQggWAAzDdFu6fZ4EEZ1LRJsBTAPwIhHNNXddDWAcgJ84Ql3LAMwlomUAlsDQHv5unnM/gAFEVA/gWgA3ZDI2hmGYYqcr5ElkFAIrhHgGhknJuf1XAH7lctqRLtdqB3BBJuNhGIbpSnBnOoZhGMaVLhABy0KCYRimUHR7nwTDMAyTPtyZjmEYhnGFNQmGYRgmq3SLPAmGYRgmOelpEmxuYhiG6RFwZzqGYRjGFfZJMAzDMK6k10+CzU0MwzA9gq7QMoeFBMMwTIEofhHBQoJhGKZgdAFFgoUEwzAM4w4LCYZhmALBPgmGYRimS8NCgmEYhnGFhQTDMAzjSqbtSy8gopVEFCOiOmV7LRG1Ka1L71H2HUlEy4monojuItMoR0T9iehVIvrM/L9fJmNjGIZhMidTTWIFgPMAvKPZt1YIMcX8d6Wy/W8Avg1gvPnvdHP7DQBeF0KMB/A6uMc1wzBMwcm0x/UqwL+HnoiGAugthJhnvv8XgHMAvATgbADTzUMfBPAWgP/LZHwMw7hz5wWTMaR3eaGHwRQ5GQmJJIwmosUAmgD8WAjxLoDhADYrx2w2twHAYCHENvP1dgCD3S5MRLMAzAKAUaNGZXvcDNMjOP/IEYUeAtMFSCokiOg1AEM0u24WQjzrcto2AKOEELuJ6EgA/yWiSX4HJYQQRCQ89s8GMBsA6urqXI9jGIZhMiOpkBBCzEz1okKIDgAd5uuFRLQWwIEAtgBQly8jzG0AsIOIhgohtplmqZ2p3pdhGIbJLjkJgSWiGiIKmq/HwHBQrzPNSU1EdIwZ1XQpAKmNPAfgMvP1Zcp2hmEYpkBkGgJ7LhFtBjANwItENNfcdSKAZUS0BMCTAK4UQjSa+64CcB+AegBrYTitAeA2AKcS0WcAZprvGYZhmAKSaXTTMwCe0Wx/CsBTLucsAHCIZvtuAKdkMh6GYRgmu3DGNcMwDOMKCwmGYRjGFRYSDMMwjCssJBiGYRhXWEgwDMMwrrCQYBiGYVxhIcEwDMO4wkKCYRiGcYWFBMMwDOMKCwmGYRjGFRYSDMMwjCssJBiGYRhXWEgwDMMwrrCQYBiGYVxhIcEwDMO4wkKCYRiGcSXTznQXENFKIooRUZ2y/RIiWqL8ixHRFHPfW0S0Rtk3yNxeRkSPEVE9EX1ERLWZjI1hGIbJnEw1iRUAzgPwjrpRCPEfIcQUIcQUAF8HsF4IsUQ55BK5Xwix09x2BYA9QohxAP4A4PYMx8YwDMNkSEZCQgixSgixJslhFwN41MflzgbwoPn6SQCnEBFlMj6GYRgmM/Lhk/gqgEcc2/5hmppuUQTBcACbAEAIEQGwD8CAPIyPYRiGcSGU7AAieg3AEM2um4UQzyY592gArUKIFcrmS4QQW4ioGsBTMMxR/0phzCCiWQBmAcCoUaNSOZVhGIZJgaRCQggxM4PrXwSHFiGE2GL+30xEDwOYCkNIbAEwEsBmIgoB6ANgt8uYZgOYDQB1dXUig/ExDMMwHuTM3EREAQAXQvFHEFGIiAaar0sAnAXD+Q0AzwG4zHx9PoA3hBAsABiGYQpIUk3CCyI6F8CfAdQAeJGIlgghTjN3nwhgkxBinXJKGYC5poAIAngNwN/NffcDeIiI6gE0wtBCGIZhmAKSkZAQQjwD4BmXfW8BOMaxrQXAkS7HtwO4IJPxMAzDMNmFM64ZhmEYV1hIMAzDMK6wkGAYhmFcYSHBMAzDuJKR45phGIbJP09991j06ZWf6ZuFBMMwTBfjyAP65e1ebG5iGIZhXGEhwTAMw7jCQoJhGIZxhYUEwzAM4woLCYZhGMYVFhIMwzCMKywkGIZhGFc4T4JhGCbP/OubU7GvLVzoYfiChQTDMEyeOfHAmkIPwTdsbmIYhmFcYSHBMAzDuJKxkCCiO4hoNREtI6JniKivsu9GIqonojVEdJqy/XRzWz0R3aBsH01EH5nbHyOi0kzHxzAMw6RPNjSJVwEcIoQ4DMCnAG4EACKaCKNP9SQApwP4KxEFiSgI4G4AXwQwEcDF5rEAcDuAPwghxgHYA+CKLIyPYRiGSZOMhYQQ4hUhRMR8Ow/ACPP12QAeFUJ0CCHWA6gHMNX8Vy+EWCeE6ATwKICziYgAzADwpHn+gwDOyXR8DMMwTPpk2yfxTQAvma+HA9ik7NtsbnPbPgDAXkXgyO0JENEsIlpARAsaGhqyOHyGYRhGxVcILBG9BmCIZtfNQohnzWNuBhAB8J/sDU+PEGI2gNkAUFdXJ3J9P4ZhmJ6KLyEhhJjptZ+ILgdwFoBThBBy0t4CYKRy2AhzG1y27wbQl4hCpjahHs8wDMMUAIrP6WlegOh0AL8HcJIQokHZPgnAwzB8EMMAvA5gPACC4eA+BYYQ+BjA14QQK4noCQBPCSEeJaJ7ACwTQvw1yf0bAHye5vAHAtiV5rldFX7mngE/c88gk2c+QAiRNKsvG0KiHkAZDE0AAOYJIa40990Mw08RAfA/QoiXzO1nAPgjgCCAB4QQt5rbx8BwZPcHsBjA/xNCdGQ0QO+xLxBC1OXq+sUIP3PPgJ+5Z5CPZ864LIcZruq271YAt2q2zwEwR7N9HQzNg2EYhikCOOOaYRiGcaWnC4nZhR5AAeBn7hnwM/cMcv7MGfskGIZhmO5LT9ckGIZhGA96rJBwKzLY1SGiB4hoJxGtULb1J6JXiegz8/9+5nYiorvMz2AZER1RuJGnBxGNJKI3iegTIlpJRD80t3fbZwYAIionovlEtNR87p+b27VFMomozHxfb+6vLeT408Ws/7aYiF4w33fr5wUAItpARMuJaAkRLTC35e373SOFRJIig12df8IoqKhyA4DXhRDjYeSrSKH4RRi5K+MBzALwtzyNMZtEAFwnhJgI4BgA3zP/lt35mQGgA8AMIcRkAFMAnE5Ex8C9SOYVAPaY2/9gHtcV+SGAVcr77v68kpOFEFOUcNf8fb+FED3uH4BpAOYq728EcGOhx5XF56sFsEJ5vwbAUPP1UABrzNf3ArhYd1xX/QfgWQCn9rBnrgCwCMDRMBKrQuZ263sOYC6AaebrkHkcFXrsKT7nCHNCnAHgBRiJud32eZXn3gBgoGNb3r7fPVKTgHuRwe7KYCHENvP1dgCDzdfd6nMwTQqHA/gIPeCZTdPLEgA7YZTsXwv3IpnWc5v798EoqtmV+COA6wHEzPdeRUG7w/NKBIBXiGghEc0yt+Xt+809rnsYQghBRN0upI2IqgA8BSOzv8moPG/QXZ9ZCBEFMIWMRl/PADiowEPKGUR0FoCdQoiFRDS90OPJM8cLIbYQ0SAArxLRanVnrr/fPVWT8Co+2B3ZQURDAcD8f6e5vVt8DkRUAkNA/EcI8bS5uVs/s4oQYi+AN2GYW/oSkVz8qc9mPbe5vw/ipXS6AscB+DIRbYBRumcGgD+h+z6vhRBii/n/ThiLganI4/e7pwqJjwGMNyMjSmF00HuuwGPKJc8BuMx8fRkMu73cfqkZEXEMgH2KCtslIENluB/AKiHE75Vd3faZAYCIakwNAkTUC4YfZhUMYXG+eZjzueXncT6AN4RptO4KCCFuFEKMEELUwvi9viGEuATd9HklRFRJRNXyNYAvAFiBfH6/C+2UKaAz6AwY1WjXwuiLUfAxZem5HgGwDUAYhj3yChi22NcBfAbgNQD9zWMJRpTXWgDLAdQVevxpPO/xMGy2ywAsMf+d0Z2f2XyOw2AUwVxmTho/MbePATAfRifIJwCUmdvLzff15v4xhX6GDJ59OoAXesLzms+31Py3Us5V+fx+c8Y1wzAM40pPNTcxDMMwPmAhwTAMw7jCQoJhGIZxhYUEwzAM4woLCYZhGMYVFhIMwzCMKywkGIZhGFdYSDAMwzCu/H+wAS7h6fYBcQAAAABJRU5ErkJggg==\n", 148 | "text/plain": [ 149 | "
" 150 | ] 151 | }, 152 | "metadata": { 153 | "needs_background": "light" 154 | }, 155 | "output_type": "display_data" 156 | } 157 | ], 158 | "source": [ 159 | "import gym\n", 160 | "import matplotlib.pyplot as plt\n", 161 | "\n", 162 | "env = gym.make(\"Taxi-v2\")\n", 163 | "\n", 164 | "total_rewards = MonteCarlo(env, episode_n=500, t_max=1000, gamma=0.99)\n", 165 | "\n", 166 | "plt.plot(total_rewards)\n", 167 | "plt.show()" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "### SARSA Algorithm \n", 175 | "Пусть $Q(s,a) = 0$ и $\\varepsilon = 1$.\n", 176 | "\n", 177 | "Для каждого эпизода $k$ делаем:\n", 178 | "\n", 179 | "Пока эпизод не закончен делаем:\n", 180 | "\n", 181 | "1. Находясь в состоянии $S_t$ совершаем действие $A_t \\sim \\pi(\\cdot|S_t)$, \n", 182 | "где $\\pi = \\varepsilon\\text{-greedy}(Q)$, получаем награду $R_t$, переходим в состояние $S_{t+1}$, совершаем действие $A_{t+1} \\sim \\pi(\\cdot|S_{t+1})$\n", 183 | "\n", 184 | "2. По $(S_t,A_t,R_t,S_{t+1},A_{t+1})$ обновляем $Q$:\n", 185 | "$$\n", 186 | "Q(S_t,A_t) \\leftarrow Q(S_t,A_t) + \\alpha(R_t + \\gamma Q(S_{t+1},A_{t+1}) - Q(S_t,A_t))\n", 187 | "$$\n", 188 | "\n", 189 | "Уменьшаем $\\varepsilon$\n" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 24, 195 | "metadata": { 196 | "ExecuteTime": { 197 | "end_time": "2020-11-12T12:05:02.351989Z", 198 | "start_time": "2020-11-12T12:05:02.331371Z" 199 | } 200 | }, 201 | "outputs": [], 202 | "source": [ 203 | "def SARSA(env, episode_n, noisy_episode_n, gamma=0.99, t_max=500, alpha=0.5):\n", 204 | " state_n = env.observation_space.n\n", 205 | " action_n = env.action_space.n\n", 206 | " \n", 207 | " Q = np.zeros((state_n, action_n))\n", 208 | " epsilon = 1\n", 209 | " \n", 210 | " total_rewards = []\n", 211 | " for episode in range(episode_n):\n", 212 | " \n", 213 | " total_reward = 0\n", 214 | " \n", 215 | " state = env.reset()\n", 216 | " action = get_epsilon_greedy_action(Q[state], epsilon, action_n)\n", 217 | " \n", 218 | " for t in range(t_max):\n", 219 | " next_state, reward, done, _ = env.step(action)\n", 220 | " next_action = get_epsilon_greedy_action(Q[next_state], epsilon, action_n)\n", 221 | " \n", 222 | " Q[state][action] += alpha * (reward + gamma * Q[next_state][next_action] - Q[state][action])\n", 223 | " \n", 224 | " total_reward += reward\n", 225 | " \n", 226 | " if done:\n", 227 | " break\n", 228 | " \n", 229 | " state = next_state\n", 230 | " action = next_action\n", 231 | " \n", 232 | " epsilon = max(0, epsilon - 1 / noisy_episode_n)\n", 233 | " \n", 234 | " total_rewards.append(total_reward)\n", 235 | " \n", 236 | " return total_rewards" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 29, 242 | "metadata": { 243 | "ExecuteTime": { 244 | "end_time": "2020-11-12T12:06:50.096459Z", 245 | "start_time": "2020-11-12T12:06:42.222863Z" 246 | } 247 | }, 248 | "outputs": [ 249 | { 250 | "data": { 251 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXm4HFWZ/79vdffdl+wL2Teys4QkBMISIJCwKDq4MaiIjHEEHBE3kFFRf4y4zziDKCqijIqIGyIjYVeCAkkgQEJCEhJIQvY9uWt3n98fVaf6VNU5VdXLvX3T/X6e5z63u9ZTffu+73nXQ0IIMAzDMNWLVe4BMAzDMOWFFQHDMEyVw4qAYRimymFFwDAMU+WwImAYhqlyWBEwDMNUOawIGIZhqhxWBAzDMFUOKwKGYZgqJ1nuAcRh0KBBYuzYseUeBsMwzDHFihUr9gghBkcdd0wogrFjx2L58uXlHgbDMMwxBRG9Eec4dg0xDMNUOawIGIZhqhxWBAzDMFUOKwKGYZgqhxUBwzBMlcOKgGEYpsphRcAwDFPlsCJgmCpHCAG5ZO2qLQfw4pYDZR5RkO5MFtmswH3Lt+DNvW2hx6YzWazfeTh0f9QSvZv2HMWzr+9FNps7zn+O6Rq67d2ZrOf9kc401u445B6fyZZ3yeCyFZQR0WIA/wUgAeDHQojbyjUWhikFW/a1YUS/elgWlfS67V0ZHO7sxpDmupJdUwgBInuc53zrSfRvrMHvr5mPS29fBgDYfNvFsa7z5t42jBpQ716rWN7c24bfrNiC6xcej4RFONqZxmV3PIO1Ow5j1IB6bNnXDouApz5zDj7325dQk7Sw/2gX/vnU0Xhxy0Ec11qHlW/uxxPrduMX/3IqNuw6gmnHtWDO2AHIZAX++OI23HDfKlyzYAI+s2gyvv/kRpw6bgAmDW3Gp+57EedPG4r3zhmND/zkWWzd344lZ43H5y+aik/dtwq/XbkV33r3idi85yiuO3ci3nH7Mhxo68ZdH5qD+1dsxacXHY9dhzpx7refxLlThmD6ca34+8a9OHvyYHzz4XX4p5NHoDOdxZKzxuOaX6zEtgPtuGr+WBztTOO+5Vvx4MfPwKGObjTUJPHkul1IJSxce87EknyuUVA5Fq8nogSA1wCcD2ArgOcBXC6EWKM7fvbs2YIrixk/3Zks2jozaG1IlXsoeOHN/Xjn95/B1/5pJi6fO7qk137vD/+OZzftiy2co2jrSuOMrz+Bf547Gp9eNBljb/wzAFv4y9cb/+Mi7D3aGap8VryxH5fd8Qy+ftlMvHdOfs/8zYfX4odPvY5rzpmIk0a14qP3rMCQ5jpsO9AOAPjph+Zgy/42DG6qxcd+sdI9rzZpIZMVIAK6M+Gya+64AXhu0z732W5/YgO++fA67bHjBzXi9T1HtfumDm/Bq9sPebYtmDwYT67b7dlWk7DQ5Zv56xjQWAMCMGFIkzs+E9csmIAbzj8eyURhzhsiWiGEmB11XLlcQ3MBbBBCvC6E6AJwL4BLyzQW5hjl2l+sxIlfWVruYQAA/rZ+DwBgs0GYhNHWlTa6GTq6M3jWERb7j3a527szWXR0ZzzHfvxXL2DJz5cHrvXYqzvxxNpd+L+Xt2PtjkP433+8gX1Hu3D/iq2eY9Xr/3TZJsy99TE87TxXdyaLT923Ci+8ud895sl1uwAAm/aEu2ok+452Ye6tj+K+57fg9ic2Ip0V+N5j6/Hhu5ejOyNcJQAAv39hG774x9UeJQAAn1k0GQunDg1VAgunDsUnzpvkEbIvbjlgVAIAsOdIJ75y6XScOWmQu+2r75gBAAElACCgBAB4lMDPPjwXP3j/KZg/caAzpiH4+mUzQWR/DjNHtuJnV8313E9CBAxrsRXw95/ciL+uD96r1JRLEYwAsEV5v9XZ5kJES4hoOREt37275z8I5thj6Zqd5R6Cy4ZdRwAAowY0BPa9tvMwvvrgGo+/WbL9YDumffFh/OyZzQCA363c6hHIU77wF/f163uOuK//5WfLMeULf/Fc80+r3sLSNTs9AvDlrQdx9c+W46q7n8fHfrESi//zb+7+ace14FB72j12jSLwpKD72d/tcS1dvdN2jSzNCdP1O+3xtNQn8cq2g9rPRQiB3yzfguc27cOsrz6CXYc78dnfvqQ9VuWBVW953t/ytml435xReN/c0Th3yhAAwNcvm4kHrpvvOe608QPx7feciMtmjfRsf2TNDs/7L14yzX397xdPxQtfvAAfPG0s7rn6VHf7qeMGYMOtF2L6cS3utpNG9cOvl8yLHP/041qweMYw/O/Vp2LlF87Hjz44G++dMxpjBzYCsK2C+poEvv3uE91zvnHZCdj0tYvw6lcW4/YrZrnbD7Z3R96vWPpssFgIcacQYrYQYvbgwZHN85gqptyBNgDYuNsWirqRXPXT5/GTpzfh9T1HsetQh2ffWwfs939c9Rb2HOnEDfetwod++pzhHjlr46nXbEH95Gu7Asdd+8uV+PXzb2LtjkMe5SFZ+aYdDG7vyngsqu0Hc2Pbc6QTANzA8W9XbgUALNuwF4+vtRXwDudZbn98Ay7576fx9432vi37bAvhZ89sxifufRGfuf8l3Pi7nPCfMqxZ+3xhHNevHrdddgKaapN41ykjce+SeXjP7FGoTyU8x3160WS01qcwemADPrt4srt91ZaDaKnLhUTPOj43E69LJZBQ4jpXnGq7ucYMbEAyYWHa8Jwi+NziKTh1/EDMGt3P3TZteAu++94TcfqEge62ZudeRGS7gpwYyuCmWgDAgIYaAMCQljr86bozsOYri/CeOaNARKhLJTB2YG5CsWVfzlLqKcqlCLYBGKW8H+lsYxgAwNHONMbe+Gd3phyGPyOjHBzttGfWac1Y2rrsfV/+02pc/qN/ePZJQdbelUFWZu5stWfXfgWnZsvMGdsfALDyjVyGT13K/nfec6QLn/vty1j8n3+DzuO0z7E4Dnd6Z5p7jnRCxnylMAds99QzG/e47z98tx2v60rbz3q0y3ZRfeb+Vfjw3cvxnUdew+u7j+BLD6x2Z/aHO+zPoDZpFTTDtZRgtGUR5o0fCCIK+M5njMgJ7WsWTMTD158FAHh6wx5PLGmwEvuo8ymTr1w6Ay9+8XzUJu3tg5pt4X3dORNxmiPs62ty51gW8M6TR+LzF011t8lz/bTU22Po31jjbps5shUNNd68nQHK/jf3xXO9FUO5FMHzACYR0TgiqgHwPgAPlGksTB+hM51x/d5SWN3519djnJfFT5dtcmex5USnlNKOQN+46wg2723zCPhUwhZwHd0ZZH2n7m/r8rw/0plz40jBuH6XnSaZzQp0dAfvndVogoGNNTht/EC0dXpjDDsOdriKQwr33Yc7sfA7T2mv3Zn2nr91vz1zJQLuX7HVs2/3YftvM2Zgg/u3zYeEIRMrqWy/9Z0zAgJ40pAmNNfaQra1PoWLZg5Dc13SVZoAPK/lvfo15ATxxTOHA7ADxLnr5qwa+bdorY9OWpB/b1XQ6yAiPH/zQpwwsrVyFYEQIg3gOgAPA3gVwH1CiNXlGAvTd5j91Ucx9Yu2T7w2aX81O9PRs/1Xtx/Cl/+0Bv96z4oeHZ+Ol7ceRFc667qEdEFM6cffc6QLmazAXkVhZRzJ296d8Qjtwx3drvCUtHflBK9UJjI20e4LHLv31lgEIwc0oKku6VEsqQThboP1JQX89QsnAchZMaa/jRDAsg17tPvGDGyM9Tf1Y0rJTSZy25OaYyyLcJLjxulXX4PvX3EKXr5lEWoUS6LOMHuXzBjRik1fuwizxw5wt9144RS8/cTjAADyrnGy16QF41c+OgY31+LD88fhXaeMjDy2WMoWIxBCPCSEOF4IMUEIcWu5xsH0HQ53pt0ZqfzHj+P2ed+dtrullDOnbFa4fn8TG3Ydxtv+52lc8N2n3HF3pbN4zVfMJC0CmVWy85At4Lfub8MRx2XS0Z31WArfXvpaQBH8evkWfOEPr3iuuXlvG9KZLI467qcPzBvjfQ5nYL/8l1wQtD5loTZpoc1RLJfPHY3mumgh9onzJuHK08agvTuDu57e5LqG/BzpTONlTfC4JmFheGvOJXPiyFZ8cuHxkfcFAFNphmopJC29OJswuAmAd8au1j34XUM6/HUSdakEPnDaGM8+aXmEkXLGm45IfZW84+QReM/sUdEHFkmfDRYz1Y0UYCZho+NQR+myK+54aiPO+/ZTgdTB363ciuc321k3B52Mm81729xx/tdj63HBd/+KdTtyysDv65dB1jO+/gSu+PGzAIIWwc5DHQFFAAD3/MNecEoem8kK7D3a5VoLJ43qh5NG5QKZMj10zKBGnOdk29SnEqhJWq5FcPKofh53jckNQ0QY1loPAPjKg2s8M/vBjh9djj0rgu6Pfg0pjzC+fuHx+KdZnmRBIwlDwVpKEf6qdaAiA6+mSUWc2XkY8uOKU1T3mcWTceGMYbjQcTf1FVgRMH0TOcPOIxCs82NL9h7pxEtb9a0THnp5O5Zt2IPDiiKRx27ecxSZrMBTr+2GEAI33LcK7/7B3wF4A8M7fNlAUrAe7Uy7s3fJTuVYKUy70lmPG6cznQ0Nqqozyp2HOnDU8fc31iY8mUvymhYBtY7Aq69JuK43AEglvQJsjCYF9r6PngYAGNCYE+SqklYza6QCUzNfgKAisCzCqAENuOfquThZycKRqO4bk5BNJKItgjFOyuYBw+cZxyLQMXNEK06fMNCtN5AMUZSin+Gt9bjj/aegKYb10JuwImDw8taDGHvjnwsqhuoppDArVWro+3/yHN7+P8u0ufzX/GIlrvjxs5h5Sy6VUgYdO9NZ/MdDr+LKu57DC74ePGFjk0pljaYYaeehDm0BmXq9znQmVAlmssJ1s+w81OlmJjXUJN0c+anDW1zLwSJyn6kulfAI2aRl4f5/Pc1976+FGNGvHnPH2f7xeiW7RQ0WHz+0yX29y1EE/qrk1nqvIpCz/DMnDUaLxjWlzvBNVopqEZiOkdZKqS2CulQCv/zIPEw/rtXd9vzNC/Hop84u6HrlhBUBg18vfxMAeqWCMS6l7nyy0QmqvnUwXk62FA7t3Rn85OlNAIKCvztEEcjZ/CpHefRXAok7DnZolYiqHJZt2Ivb/m+t9tpv7D2KjBA4rp/tptl1uMP19zfUJHDKmP44c9Ig1KUsxSIg1wqoTyVQq8yCUwnC7LED3Pz+UQPqjc+lPkdWAO8+ZSR+9MHZaKrNbZfPNrDJ6xo6rl+9575e/35QiKcSqpDXj0e9RsrgGpo6vAUfOn0svvmuE7T7TamehTC4uVar1Po6rAgY16USlT3Rm+jSHgvhR399Het2HMbI/rZw2xyzHYJ0F/zgqY3uNn+8IuPP91T41XNv4h23L8Pqtw5hWEsdhrfmhOvOw50BdxGQyyCK4uxvPolMVmBoiz3TfWNvGz54l12EJvPRLSJkRU65WJR7pnqNRQAAjUqapYrqcTlj4iBPMdXEIU04f9pQ1NcERclAJ0Yglce/nDHeI/AThtcSVbCbXEPq9Uz9eBIW4Za3T8fEIfpCtkJdQ5UEKwLGTT2sq+k7/xBRIjGsBfBPl21yC6JufehVvO2/n3bdHZt8lbbqzHygEtyUKZJvKEVcqiL4xbNvuIVVOla+abdz3nOkE0Nbaj1CbefBDq2bQo4lTvPSrnQWtckEBjbWeGotGpy/oUV25lM2m3MN1SRzMYIaT4zAfv2RM8cDAOYoaZIAQPAK5PkTc1W5rpXhK4hqqEm4AdGffGgOlv/7Qswc2eoT/rnjdYFe1edvChZbEVZFHIoNFlcC/Akw6JSKINl3vg5RXXFNueg7DnXgy39agyvves69Rlcm67optuz3uobU/Ht1ll6rmSWqPvGf/G1TxBPYHOpIo6ku6Zmt7jjUoU0flI/srzLVX7cbCYtw/fne9EupCBIWYd/RLqx1spcsyonzupQvWOwI0MUzhmHzbRdjqhL4BQC/DFaFeU0yZ2WoDGiswdThLdh828WYNbo/BjmtFbwWQW4Mlu8mE4c04V/PHq+9p4nCFUHfmQCVi77zn8+UDdc1VIZ/iB0H9WmSUV4StbhKpdN5lkMd3Z4sHDkz9rt32jpzrQ/UQHKtRimqyifumgOHO7rRWJP0CKmD7d2eYi6JtAji/B0Od6SRIMLlc3I55gMba9zWBUSEbQface/zdm9HsnLutoRFHovA71KRaZ+yX45fSKvPIj+nBp81acqK8VgEpJ/N92tI4dEbzsaEIbkAdJzlDkzpo1GkCmzxXEnwJ8C4s+KaMlgE8772GObc+mje55mKx1SXi2pVyMCuP0grWyk016U8FoFOGHcq6alxZ5+H2m2LwC9s3joQDFrLGIFfqJpIJLy9dp6/eaF7H//wZMxA7vMqAu/BqYSF8YMaMd4pxPI/qTqTd1NSfZ+XuSWEYgUoH0lCk/qpKop4FgGLs0LhT45x+/v05BpFn/7NKvyvUwwVh6hg8S1/0nckUVMuVZmfcVwx/oCsbBbXUp/07NNloKiuoTiCCbAtk+bapCts5Sx7636NInAG7BeqJqQyGuPk66tWin8Wb1Hu72sReYLFNZoZ8QMfPwM3SLeT71HVz6bGbZngHbNJUZoqgdXj/dXlgDlGYLp2HNTCu2qnb1U1MGXBVQSRIdrCuX/FVty/Yive72uBYCJMDxxs78aqLQewaPpQPLzauyaB6ntXn0fO9v11BDLtsqUuhUy2zb2+ev+GmgTaujIe19Dqt4L1ARJ1paqudNaOEThCb0S/eqzdcdizCItEWjP1MS0CKewf+rczAy6voCIg9/MgIk8MROdSaapNoqnWPiZoESiuIec6fsUZpzeQqn8SmjGo94lTtZuvi+feJfPcv3+1wxYBk6vILX9bf5ewoTy/aR+yArhwRrBM3+saym2XqZ5B15C0CFLIZAVWv3UQJ355KX7/Qq4runTVmALU/hm1v7VCU23KFZQjnNz/bRqLQCox1TU0pLkWj95wNn77sdNwXGsdvnrpdHefnEU31iY9bY2BoE+dPBYBtOmjfmR+vV8Iq7N3nTXhP0YlYbBatA3j8nQN5WsR1KUSkV1AqwW2CBjX5dEH1ndxCcsaku2mxw1qDOzzuoaCFoHfNSRbMUuXjVxp60WlititMjZ0+KxNeteq7d9Y42k5oWYN9WuoQX0qga0HgjGOtKOsVNeQgJ1BAwDP3HSe220U0M+iJTqLQH4eBG8w3FSIJQVrnBiBvygrTtvoZERVsFdpaC/nwfQcTDRsETCuRdCTrqF8CVNK0pXVXBecx6guElXmy9n271Zuw5eV+IJrETjVoLpCr1TCrso1tXquTfktAm9BVnNt0hVSqQRhaEut1iKQdQmWxl8uaaxVKnND3CW6YPE7TrYbvJ09eYhPEejFgBpTUElqYgQzRrTgS2+b5habmRSBR7irdQQ6RUB668FEvhYBk4MVAeMKuJ4MFuePeTDSRdMUoQhUi0B1Cf102Wb3dZsSLAaCMQTAFsw1SttmP373SP8Gv2so6fbESSYIrQ012BuyOItXnHnH06ikZYZlLumCxbNG98fm2y7GuEGNnuJBU9qldFGNHeTtPaTeVxZjERGumj/OrRfQZQHZ5+qtAN3x3qyi0scImBzsGmJcIdmX9IB3Np/1pElKC0bX00V10ajPkza0g+hwFEdDKukcF/wUkpbdsM1Uu+AvPvP7nRuVrKGkZcGi8HUWVBnuV84Nnl49ZsHn9+v736t5/inDdcYOasQPP3CKZy1e+76qRaB3CcWJEZjqCIRS7yBhi6BnYUXAuERV8+bL7sOd+O6jr+FLb5uW/1iU112OIljy8+UY0mI39ZLuGj8e15Aia/1B4q50Fr9/Yau7XebV65rByYZtusVWgGiLoDZlubPVhEWwiGKvs+AfTTJhK5KsMDdis8ccfl1VEYQVYi2aPiywLamJEbj3dW5sEtymXkO68XpcQzEm+yaFxkTDioBxKbVF8IU/vIK/rN6BcyYPyftcnVtn6Ro7VfTD88ehNpnQphR2GlxD/iUkf/DURnznkdcwYbAdcJY+fJ0iSCYIuw93GttC+1Ml/RZBgsgVgEmLXEEeB109RW0ygfbuTKhFEDWDVt1q+bpUdDEC/31jWQQR2irfOoJCK4sZjhEwKiXWBDJzpl+MtVwDQ/GkfnoH1pnOGBuFeSwCZbv/GnIZyv1tdrtoaRHoXEMJotC1AfxCz5/KaVGuAjiZoEghHdViQt4v1CKI+M9urClCEXjqCHwLvzu7TBlNqrD2BMU1x3KwuPdgRcC4lDprSKZ5FtIMLEwRdHRnjT3k1epfb/qoV5D/8cW3PNeW7g6dRZDPzBWAW4iV25+zOBKWFSnUrpo/DhfOsF0yOm9dQrmWiagCrHxm5mHnBiwCqaRMK4rl0S3Um2HEweKepKo/uRVv7Pfki1c7pc4akorA1C4ibIUvVSnpLAI5E73n6rmefab0UdO9skKAKDdTLUQRyEnuzRdNxWcXTw6s62ARuYKZEGe2nsAtb5/uPIM+eK3+1tGTk2OpNBMWBRrWJSJcQ8mQjqOSuePs4LQVEUPwwwZB4VS1IrjsjmfwjtuXlXsYfYZSKwKZ3aNruQyYM3n8Y/EXgakWwZmTBnv2eRWBOUbgXjsrYBG5AqwQRSCXKjzr+MG4ZsHEwPGqwBOIdnNYVq4XkNYikAHZPNJHS4lUmrqq4oTrtiosRnDxzOH478tPtvfnWVkcpw0Fo6eqFQHjpVSrgvkxzcZVBbHPl1dfaIzAGyyOHkMmK+xgrrQINJ9BwiJcckKwnYXk3y+Zil8vmYfJzlKPfqGVsJTaACEiBZZF5Fo8U4YHV9WSs+p86ghKibyvPz4AKK6hGJXFOvfR5GHNbq8l1XLqyedhWBEwClkhcMhZdL2U6AKwgFcRzPrqI559oa6h7qw2dRTwKgJd0zk/0jVkhVoEFv7nn2dpzwfsLJ5Tx+dy7Zt99Q1EuQCxbREYL2XfjwgNNUn88l9OxY8/OCewP5kIF7ZAzwpOeV+tRUDxLYIov7+VZ7CYKRxWBIzLbf+3FifcslS7aEoxmHLmu+O6hvzB4nTGmFljtgj098pkhe3rdoSSzo2Vb1bi5GHNbrDXPp/cIjEh4riG7N+nTxyEVk3Glet+yaPFRCmRFonOIogqKMsncSBf1xBTOKwIGJfNe3NtmEuJqYo2PFhsPi7MIvC0mFDOM8Upso5gloJGpzDCsnNMXHLCce5rdZlIAREppKMUhRssDms6F0NwnjiyNfIY7f2d++oytyzXIjAvJB+XfIPFTOFwQRkTIGMQmoXSbRD4YW0WPAVl/mBxiEVgyvc3uYYAW8hIAaWtIyhguqQKrmBL6Phpk/r9VuRxcTwp93/s9FBlbCIZ5hqyvL+D48rDIoi5HoHaYpspDFYEVcaT63YhlbAwf+Ig4zFhLptCMLmGTLN0ICJY3J0NpGjm9unrCMIEnmXlLAJdwLyQJRDJ59bwuoaizg3fn4zlGoqXd1/IMtXyswoPFhfmbFA//jjVxADwzI3nYu8RcxM/JhpWBFXGh376PABg820XG48pZJYYhmnmH5Y+ipBgcYdSR6CirgwG+BrXhdwrobiGdMop7kL1nnNIfZ2rI4iTPholAKNSNP33LzVSMYYFiwspIvQTV5cMb63H8Nb6ou9XzXCMgAkQ5rIphHQB7prwymK9ayiZIM8C86rwD9Nt5IkR6JrOmc814cl4Uf7LhIhWLFGuoVgxgl6oI/B3XQXi1TjEJa5FwBQPKwImQCksArWYy7TEo8k19OS6XXjXD/7uGY//ejphmVIsgj1HurDwO3+NNdaElRM6OuVUiP/ZnwNPPgshjLgxhLDr9GRxVViMQFIKIc6ZQr0HK4IqR9fCoBQWgSpPTVW9pvs8ssa7IH0mKzzKSQi9oEklrNjtnVWsiIKyQtSiKoi9lcXRWUNRAlD21AlTBD0pQ8NiBPLjK4Ue4krh3qMoRUBE3ySitUT0EhH9noj6KftuIqINRLSOiBYp2xc72zYQ0Y3F3J8pHt3kvytdvEXgafimCPy2rrTb38nkGvJvzQgRENA6QVeTIE/TubhYSsGXLmOqkHUaPAuvE0EmkMapI4gbI9ApLd39S41bRxBiEeR79ynDWgDArc5mepdiLYJHAMwQQpwA4DUANwEAEU0D8D4A0wEsBvB9IkoQUQLA7QAuBDANwOXOsUyZ0LmBwlouF3Jd9Xqfuf8lvOP2ZdhzpDO25eG3CAD9bDGRoNBMJBOWpXQfLZFF4EkftYDxzroHEwc3Rc6W42YNhaX59miwOBFiETifVr566OIThmPpJ8/C4hnBhXCYnqcoRSCEWCqEkGWo/wAw0nl9KYB7hRCdQohNADYAmOv8bBBCvC6E6AJwr3MsUyZ06ZLdBbhX/KiXVRXBmrcOAbCL1kyxCP+QMlkRsB7UGe/g5lp3WyGprwki16evHVMhMQKfa2jR9GH4w7Xz8e7ZI6MtgpjB5LBge0+6VcJaTGRd11D+9z9+KFsD5aKUMYIPA/g/5/UIAFuUfVudbabtTJnQeRdKYRF4XUO517IiuLM7G3v2nsmKwKLyqpx5+nPnYO1XFyNBVFAxnEXkWgQ64RrWjM+/GplufNLVc9Kofk7foejxhCFn5GEpsb3TdC6YNcSFXccmkXUERPQoAJ29drMQ4o/OMTcDSAP4RakGRkRLACwBgNGjR5fqsowPrUVQCteQcl01gOsqgnQm5D7eMektgtxr2eqAKHyWbEItKNNZQybhVpey8PTnztFfUxHEfpkc2WsoMphsXkQn7jWKQfZmqi+kGo3pk0QqAiHEwrD9RPQhAJcAOE/komrbAIxSDhvpbEPIdv997wRwJwDMnj2b5xk9hE4RFJJ540ddOF4V+FJod3RnYwvt7ozA7P/3qGebNkZgUahwNLUiUFtM6JSTaeW25roUGmr0/0KeYLFPKke5TaL2XzxzOP606i1MP67FeExPWgREhB9fORvTNPcvNEbAlJeiKouJaDGAzwI4WwjRpux6AMAvieg7AI4DMAnAc7CTCSYR0TjYCuB9AP65mDEwxaHzLhTjGnpzbxsGNdcYF46XAcbOdCa2ImjrCnZD1QksqVqdAAAgAElEQVQ6iyj0mkmLtKmslrK4/PI39gf2F1RHEFI3UOxsffGMYdj4HxeFVxb3cA7+gslD9DtkjCDvvCGmnBTbYuJ/ANQCeMSZxfxDCPGvQojVRHQfgDWwXUbXCiEyAEBE1wF4GEACwF1CiNVFjoEpkCfW7sL2gx2B7cUEi8/65hM4fcJAfM9ZZQrwKhbpGurozhorjv2CVyfcdXLOonCLwBbIekUQJjiLryPQjaM4ItdRZjnM5EFRikAIMTFk360AbtVsfwjAQ8XclykNV939vHZ7scHiZzbu9VoEimKRQdn27nTsYLHOXaO1CKzCAqjqegQ6irUI/K6e3hDS5VrIRX5U7Bo6tuCmc0wAUyVwFGrhlanhW8LJeGnrysR2H+hiFjpBk4i0CMzbwwVncQVlgX2BmEHps23KZRHI7wDrgWMLVgSMi0V2HripN1AUpjWC1eulHAnV3mVeU8CPzkLRBVQpIkZgCsJaERZBIa2Xwts/ePcV2hojjHK3Zwi7/QPXzS/oM2V6Du41xLjIf85C00dV4e8NFiuuIacIqb0r4x7zjctO8FzHPzvWCUmd3E5YFDqzNgmnyBhBAdP1MEHov1WqB6bvZXMNxfioThjZDyeN6hd9INNrsCJgAhQ6O82aXEOZ4Pa27oyrOFrqww1TvSLQZQ2Fj88YI6CIGIFpe4jQy8ciSBayBFoEZXMNOb85a+jYghUBE6BQi8C0Iph6PfnatgjsbTWG9YfDxmPKGgrDtJsoPAun2DbUwfsFXUOlplwWgYSDxccWrAiYAIVbBOprtelcUCm0daXdthF+Qegv4NIFr3U+8OiK3QKzhtzzQy8feyz+69SELDBTKOUSxNxi4tiEFQEToND0UW+MILddZxG0dWXcNhRhC5wA+uC1NmuowNx6ez0CCzddOEW7382EyUO6hg2ld1xD5UofZU1wLMKKoML56bJNGHvjn3G4ozv2OVEWgRAC//P4emw70B7YLjEFi6V1kMkK9xi/a8g/q4xfR1BY6wZ53tTh5pYNQH4pkWFKIxAs7gGLoMC140tGubOWmPxgRVDh3P3MZgDA3iNdsc8J67YJABt3H8W3lr6Ga/53hWe7MWtIUSyymljtKBp0DXmJmzUU2azNpAiczSb30NyxA5zj8rEIzMdWcoxg8XS7P+X8iQPLcn+mMFgRVDiFLB0Y1dJfCuZVWw/i5t+/nDtPjREo19DFCLICkBP9QoLF2qZzBQaL5Xl+i2LSkCY8+ekFuPYcp4A+j88wbCxB11BPxAjKowhOHT8Qm2+72F1xjDk2YEVQ4RTis42yCNRK4V88+6b2vCjXkBA511DAIohVR6AvKAvDNEuW5/ktAosIYwc1ugoiP9dQ2Di873vGIij5JcvChTOG4dKTjiv3MCoeriyucFyLIA8xFlX16c/i2XOkE4OaamMpAukmyjiKgCgogP3KS1tZrBlXlDw1WgTOebrWD+Hnh7SzyKMzaCW5hkrNHe8/pdxDqArYIqhwCnINRVgE/n4+G3YdCWyPqiPICvs+CaKgayROsFjzzS00fdQyWAR+C6OpNhV6fe8144+jR4LFlaEHmF6CFQEKayFwLBL3OU2K4LlN+3DLA6sD7aNl0Fc9zZs+qokRZAUyWVso+tM+/fePXVkcmTWk3y7P81/Tf/ivPzovtoDNp46gJywCbvvG5AMrAhTWVOxYgyj+c5qOe++df8fdz2zG0a6M9njVCjApHakUso5ryLJyralN9+/pgjLXIvDNzP1Wx4TBTfjYggnqKIz3Co8R+ILFyo38fZcYpjdgRYBoV8ixjCm3P+45Kv3qbdfIjkPexWzkdU0tJlS6XNeQnT6a0FgEhaaPRnlYzFlD8ncwWBy8b7yZdnj6qPd9TTK34T1zRoFhehtWBKhwRSB/i/DFzlVMxw1orAEA7DjoLSTTKQJ5Cb/Qk0I9m7UDxuoykf7rSWIXlEUI6RsX6yuH5XmBtYU1x3otkajV0OLt6xnXUOV+p5nSw4oAld0fRX22uM9pUowDG2sBANsPeC0CeXjWcy8nNdTnX+lMZ9x7ZLMClhW0CPwyLG7WUFSMYPzgRnznPScaz4uzyHxcz3t4sNj73u8aY5jehr+BqHSLwOuTj4PJcJAWgb+1hLyuJ2vIrRHwSj3ZN8gejy18g66h6GCxPkagH3duv0bpKOcFFYH+GnEIbTERSB8tfWC3gr/STA/AigDxXSbHMkLEV3hZw+fRv9GOEbwVcA3J30HXkKmPUEZI11DQN69WJRPlXEP3XD3X3W5amCYMi0hfkWywCPRFa6G3CB1f7hq94RpimPjwNxCVnTUkFCEd1TpCYlIYUlD6XUNujCAb3GbqrCmka8hZHUwVnKpFkLTItQjUpS0LrSzWCWgyxAiiLIzCF6bxvmdFwJQb/gbi2KgjONjWbZyphyGU38W6huR2/7rAQhcszoa3mJbdR6XwVf3k6uUTFrnpo6rA1LnVo3oNWVb4ymb+83XV2PFjBPkEi3vANVTyKzKVDCsC9H2L4GBbN078ylJ84+F1eZ8rZbPII0ZgUowmReTWEcRwDan7ZUEZ4J2Nq/dPWZYbLFaziwqNEejTToPKyL5H8FgRsT/OvkCwuAcUAcPkAysC9P1g8f42u4X0Qy9vL+DsXNVvXIWXMSmCiO26mgVTa2epmKTsVY9Tx6kKSdUiKCRryBQjkNv8VkYx7XrysQgSPZA11Me/0kwfgxUB+r4ikKMrRDCpqZ3xg8WG7REuIzXL07TWgES6hlyLIKG3CNQYg+pC0QnaQl1DOveU6R4q4TGCsH0+RVAhDeKYYxdWBDh2Zk/FiAuBfNJH83MNaWMEka4hgYxTWQx4LQL1Lup2VVAX0mvI5BqS2/wTc9094n5XQi0C3316IlbMS0Yy+cCKAH3fIigG+WTZbD69hgpzDamKIrfWgMk1ZP/oGr6ZXEOqANUL9DiKwKxA4sQI4pJPr6GecA0xTD7wNxB9P1hcTFaTPFdAxM46Mh2m6f1mH58NnmdadCZ3Ldsi0C0T6Q8WSxJFBosTRKFFYv7z9fGE8HuEnWvax9mjTLnhryDMLo++RiHLD6q9hop2DUVsV4PMn7j3RQARriFjjCB3nCr8VV96IQVlZIoRONv8n6/uap6WHaF3M+MfZk8sIlPBRi7TA1StIlBnnUIA77h9Gebf9ngZR2Qm3//pJ9ftwj//6B/IZoWSPhrf8jEJEWOMQO7XnGjqo5PN2n8DfR2BPlhsRVgE8QrK4lsSPbW4S9A1xMFiprxU7VKVwufGeHHLgfINJoLccpPx+Nf/XYGO7iw60hlPIDeuRWBquWE6372H5jy1xbL/WpmsYhFYeotAjTFEWgQRiiBBpF/ZzCCIdYpFDcIWKr79t+sJRcDKhcmH6lUEyutjJlhcxP+2QPErlJksile3H0Z7V0a731RZbLuG1EBt7uHWO0tf+rdHWQRRso8ov/bVPSdLe94iOH/aUFw1fyzqUgksmj6s5NdnKouqVQS6VMe+S34DVN1BqttG08059Hw/JtfQ3c9sxtb9bXjXKcFFVVTXTk3ScvsGZaRrSLZ3UIThniOdufMNwWJt1lCs9FFzHYGfKFdTqb42PVFHkEpY+NLbppf8ukxlUpIYARF9iogEEQ1y3hMRfY+INhDRS0Q0Szn2SiJa7/xcWYr7F4K3T3/f1gT5uoYkGUUT9GSwGACefX2f9nNUs4ZqlcCx8LmGTBXI5mBx/Jm9ei190zn98b3lXLEswvevmIW/XH9mL92RYbwUbREQ0SgAFwB4U9l8IYBJzs+pAO4AcCoRDQDwJQCzYYuoFUT0gBBif7HjyBfV19vXLYJChyeyataQN0YQpvxMMQJT+iggm9oFt6vCvzaZwGGknWs5isDQAlpiriPQzezN47PP0Qt904y8J7J5dCSIcNHM4b1yL4bRUQqL4LsAPguvvLoUwM+FzT8A9COi4QAWAXhECLHPEf6PAFhcgjHkjSoH+/p6BFKA55s+mhFCqSOI/8ymXWHKQ6aD+lFn+qpSkBlNpkVhJKpFoQrsQhaNsT+/+JZEz+kB7+fEgV2m3BSlCIjoUgDbhBCrfLtGANiivN/qbDNt7xE60xksXb0DW/a1hR7X14PFhSqqTFYolcXCcx1/K2kVk8CPGofWNZQ0uYZsRWVaFEaiKhJvQVnw2DBFECZsTfuKaTERhv8arAiYchOpCIjoUSJ6RfNzKYDPA/hiTwyMiJYQ0XIiWr579+6CrnG4I40l96zAE+t2BfZ53SQFD7NXyDdGoLqD3MAx/AHyENdQATECIfSKosbTNC73OiO8TedM9QbSNUTktYjyqQfw7guOcVBzrfacnrII/CNgRcCUm8gYgRBioW47Ec0EMA7AKucfdCSAlUQ0F8A2AGoKyUhn2zYAC3zbnzTc904AdwLA7NmzCxLVUljo5Je/jqAvk3MN5XdeRgjfmsW5fWEWgXHdgZCsI//1JWplservdxevN6wOJpF9ePx+/Hyyf0zHS4a31mm3F1LJXQi9FYtgGBMFu4aEEC8LIYYIIcYKIcbCdvPMEkLsAPAAgA862UPzABwUQmwH8DCAC4ioPxH1hx1kfrj4x9Aj/710gv5YqiPI2zMkNOcJ33oBoa4h0zhCzjFcUy0IU1NJsz7XkClrKCWb0sVYRjIsfTRM2A5r0SuCnpqos2uI6Wv0VB3BQwAuArABQBuAqwBACLGPiL4K4HnnuK8IIfb10BgiLIJjJ2uoUEWltpjI+lw3oRZBAa4hGNJTVXdQShF42ay9hnKURSCtiMAyknmmj4bJ2iEtBteQ+RQA8dKO508cGDwvECyOvAzD9CglUwSOVSBfCwDXGo67C8BdpbpvKM5/sk5AqXKw79cROK6hmFEC1R0klG1Z4RXEJkwxgrD0UVPWkFQEFmlcQ0LpPmpoVy2tCL+iyCcN1L6/eV9tMhH7nHy+KetvvVA7pqBFwJqAKS8V/Q0Mdb16YgQ9PpSiKHR8mazwuIlUhee3CFQ5W0j6qKmOoEYR5Cm/ayirZg0ZgsXuegXe7TohHfr3NuwbN6jR837KsGblesW5bFIJS+uuCgSLOUbAlJmKVgShriFPQVnf1gTSpZOvvLD1gFNH4Avm+jN8TIvH68ahQwj9egeppMz6ocDaBJ6CMsOzyWwiv0VQSBtqHU98eoHn/V+uPwuXzRrpjDnvyxUEGwRMuanor2BosLiArKFNe47i4dU7SjCy/Cg4RiC8bahV102YIoham9i0TzdOOdO3KBgQ7s5mlRhBePqof3++LSbykenu0pU9Fiz2ffZsETBlpqIVgWsRaPYVUkdwz9/fwGfvf6kEI8uPfPVALkCsPKNvzWK/P1/N4ze2oY7wUenOS7ozfkJtyuuLT2dyTeeieg35A6ragrIQyU0h1mHgOnKhml7qNsRZQ0y5qWhFIIVFqdJHuzPZsrSjKLjFhKey2DsT9T+HXxbp3ENRn5NutxRylkVorbdzE6TQ784oriGDb0jOluNlDZnHls9HJ3Vib7lsorqmMkxPU9GKQBJdUBbvOuls/MVdSkm+usfNFFICxHaef+6YMNeQ6Z5Rz67LGpIC3CJC/4Ya9zUApBXXkMkikELSL/jzXZhG7mmqi06Uk/fSKt4S/P39LizTszNMb1HRiiDMZ1xIsDhbNkUg00fzw2MR+NI7oxSBzvKJUkjaGIHr4yf0cxRBl7Mwgu0aiqgjMPQi0mcNRX9CU4a14IcfOCX0GHmryDqCyLvpWTxjGJacNR7/fvFU536sCJjyUtGKwHUNaSRYIesR2BZBKUaWH1G+eRMZX7A4LH3UL/b0tRfh41DHWZeysPm2iz3pn631Kc/x3Zms636JihH46wy0dQQxYgQAIlfsshQrpidIJSx8/qKpGNBoK0aOETDlpqIVQViw2OMairlyV1Zp69ybSPkaVy657iDf+gPe9FHvQ8trS6Gke8yo+Ii6O+XrEWQRBRRB2tNrSP9VlONJxcoaMo8tH1HbVNs7C/f5P3OGKRcVrQhC00cLcA2VzSIotOmcIusFvM/pX7ZSXlrOzKNSbvX3yx2guoTk7xaff14tKDNaBKS3CLSKoEQCVc7Uj3SmS3I9E7Kiuc5Q2cwwvUVlKwJHLpSq+2i5YgSFWiH+ttPeXkN6i0AKZF3gN5/1CPwC3CJCa0MqcI7rhokIFid9+aO6o0PrCPLQEVIRHGjrCuwr5V//3ClDcOcHTsHogQ0lvCrD5E+FKwLp5ohKH413vXQ2G/C19wauayjPcLHqs7fHre7zHiuv7bqGNO6yfLKGyOdnJwKGNAe7fEZlDSVd11C0RRCefRP/s5OKYH9bd+hxc8YOiH1NHXWpBC6IiFcwTG9Q0YoAsAWQtqAsm79rSLpTets9lK8VkssU8l4jrKBMuspkG4ioRn36ceZeJ9wgcK5FxIDGGvzfJ87ELW+bFjjOuEqYKVis+eaWyrc/sNHuRrpfYxFILp45HN9738kluR/DlJuKVwQWUV6+7fDjbE3Q2+6hgpvO+cYZFiyWhyZCXEP5ZA3520vL91OHt6BBEdixLYJEdLC4pT7oepLk4xrq32hfZ//RoCI4cWQ/AMC7Zo9EfQ379pnKoHfSI8oIITrwGVeuyzbM+SiCN/YeBYGK8gNLARs/a8g5zxcjUAW1P1gsCQsWRyoCEaYIENgH5Gb8xhXKDHUGusP9WUkq+TjVpEVwqCMYLF44bSj+cdN5GGZY1YxhjkWqwyLQbC8ka0jOovMxCM7+5pM465tPuO+Pdqaxdseh+BdQxpdvTow/RuAtKPNZBM5vme1TSPqoqlz82aCqIFeFeCLCIshlFUVbBM0xqobjIGf6H5g3RruflQBTaVS8RQCK9nfHdw0J59zCXUO/eu5NfGvpOrxyy6JAJowJd3h55o96YwTe5/RbBPKRZL5+Iemj6jlSwMttquDWKYWE4bNw6whMfaoV/O4jlXxTbzf+x0U91n2UYfoaFa8ICNBGi4UoxCKQiqDw8RzuSKOjO4vOdDYPRVBgZbHHIvAGi9X0UVvI2vvcGIHmIaMLyoKuodxaCjmpShrXUFS/Hf9nlW/Vb2faq/kGN9fi4pnDjcdzkRdTTVS8IjC7hnJ4Cq+EMPasKYVFIK/Rlc6iUb9UboBS1BEIn0Wg7kslrECwWHfLvGIEvliDKsfV5nA1Sf3CMxLZCiOYPho6lAAHfKmgz9+8ML8LMEwFU/ExAqLoXkNx1yaQglSXYx8X6af3z1DDyNUR5Id/PQJPQZmyAHHSIvfY8PTRcEXQpTxTo5MZdPzQZkwc0oQvXJxLGVWFuLuUpUH5ph0tHaey2M9V88di/ODGyOMYptqpeEVgtAgMrqEwYSeFeDEWQVaxCGKfUwLXkD9G4LcI5C7pi9e3oQ6/n9qSYYKzFnBdKoFHbzgbp44f6O4jjUVgWrxeWgSByuIYWvGq08fhwY+fEX0gw1Q5Fe8aMqaPKq/9AtOEnEWXwjXUmc7EPiff9FGJP0U244kReBVBVzrtvlbH6RlHxHMfVtItw2biqhtI3m/uuAEY2lKLnYc6Pce6FoHPF2Ry333/ill47NVdqEkSRvav50VfGCYGFa8IQNG9hjxB1ZBuMlnXIih8OGlXEfS8a8hvAXiqqVVFkCRkO+z3cobuj0sIpaW1iaOKRTBxSLPxOJ1raHhrPb7znpNwxY+f9Rzb7Shff/qoiYtmDsdFIUFghmGCVLwiMPmSTXUEYcJOCvFieg3Je3WZKrpCzin0XoBtAam39FgEluVaC7kYgf9a0fc70pnGKWP64/QJA7Fw6hDjceosPZXMCXjdX8oNFsdIHzXxg/fPcjt9MgwTpOIVAZnqCBSh6O/JY0LOoouxCFzXUHcBFkHedQTe9FHTCmXJBLmfhxS4ftdQnFqLIx1pnDCyHz51weTQ41TlXKMIePl8g5pqseeI7SJqqrUFuGwEVwiLZ7CFwDBhVLwiMPUaUi2C2DGCEqSPFmIRFJ4+ql4DvhYT/mBxeNZQnGc+3JGO1fjN4xpSLQLy/gaAy+eOhnB+MwzTM1R81lCcXkPeVgxxLILCFYEMOMfNGprw+Yfw//78akH38scIPBaB8nrysGb3maTPXgi7T9LYG/+Mx17dGU8RdMZTBGqqqFoNLLeqiiKZsPDB08aGVg0zDFMcFf/fRYb0URVVKMaxCIppPpqrI4iXNaQK83y95N6lKn0tJhyF9InzJuHWd8x0n1umcaazWSzbsBcA8JvlW2O7w5pi9PshkyKQaxfk/aQMwxRDxbuGiAwL03jiAur2OFlDvVtHIMk3fTTttwiyQYtgweTBnnbKUjDf/sRGPPrqTgBAW3cm9jPHsgiUKb/ONcQZnwzTu1S+RYDodgnZvGMEhY9HFvTmkz4qiZopb9nXhsMduVYKYQFf+dqfVSUVgVQCANDRldFWZ+uI0wFUlz4K5CyefIPiDMMUR8VbBOZgcY6Mx4ViFnil6TVkK4BCLIIozvzGEzh+aJP7vjvjtQi8i9frFYE6Q5e0d2diK7/GmuItAoZhepfKtwgM6aOiAIvA7TVUoqZzURRyn9d2HnFfp5XMpECMwFCtnHIXm89ta+tKx27VrVug3k+j4j7SxghYITBMr1LxiiBe99F8LYLCxyNlc5xgcVzhCwDdmnTU9u7cPbICsSwCKZjV7R3dWfdzOX/a0NBxhK0SJmnyKAKljkD+DlEEbz/xuMjrMwyTH0UrAiL6OBGtJaLVRPQNZftNRLSBiNYR0SJl+2Jn2wYiurHY+8chyiKImzWUr2tIp1TcOoIYFkHaN5iw+7Z1BRXL3iO5NXcD3UelIvB9A3SKoL07435G504Zgs23XYxmQ1C4pS6GIlDiCF7XkHd9Yx3fu/xkbL7t4sh7MAwTn6IUARGdA+BSACcKIaYD+JazfRqA9wGYDmAxgO8TUYKIEgBuB3AhgGkALneO7TEot+aKB1OvoTjdR/Nd0UzF7TUUo6DMP8sPu2tbV3B9XVmdC0jXUPB4f+uFWk2M4EhHWrEg7G2fv3iqduYexyJQ4wjaYHHkFRiGKSXFWgQfA3CbEKITAIQQu5ztlwK4VwjRKYTYBGADgLnOzwYhxOtCiC4A9zrH9hhxXEPqzNskbLPZXNO1uK77jM4iUFpM/Oivr2PLvjbj+WqwF8jfIvAqAuFZp3j7wQ4AwLAW7/q7MuunWzm2K5PF/qN2NpKcrV8+dzQ2fS04M69LRX+ldN1HATV9lFUBw/QmxSqC4wGcSUTPEtFTRDTH2T4CwBbluK3ONtP2HsMcLM69VoOqpjRJr/uocItAbnvrQDtufehVXPnT54znp30WQZgh0tYZVAS7D+cUQVbkUlcBWxG01qc8NQQA0FRrz+j9j/jX9bsBAENawhduzzf10+MakrYA6wGG6VUic/2I6FEAwzS7bnbOHwBgHoA5AO4jovGlGBgRLQGwBABGjy68z4wpfdSzfq8iIU0yPm4/ItM57jbnBnIGr7Zu9tPtO3/VlgNYunoHLpge/HMc1bqGcjGCv762G8vf2O++336gHSP7NwTOaazVd+m8+5nNaK5L4jRlgZlSoK4zwBYBw5SHSItACLFQCDFD8/NH2DP63wmb5wBkAQwCsA3AKOUyI51tpu26+94phJgthJg9ePDgwp4O8XoN+StwdcSNI5jOEb74QoeT0RNWJOa3CABgyT0rtMfqYgRdmaw741aVAAAc7cpgWGtwdm8qCNt9uBOzRvfX1hkUg3dRe+d3Se/AMEwUxf5X/wHAOQBARMcDqAGwB8ADAN5HRLVENA7AJADPAXgewCQiGkdENbADyg8UOYZwSO/3N3UfNYn4tEaoR6HL25e/5bKOltMC4wM/eRYPr97hOd8fI5Cc+60n8U/fX+YqE0AfIwCAgSHtm/3xASDnGlKRAnq4RnGUEqkU2SBgmN6l2MriuwDcRUSvAOgCcKWwpeRqIroPwBoAaQDXCiEyAEBE1wF4GEACwF1CiNVFjiEUi4yawCWtBEZNs/24RWcq/m6fSeX6UhEQEbozAn9bvwd/W7/Hkxqpjkvl9T1HAQAvbzuIOWMHANDHCAA7i2fHoQ6ty0s3+69NWkglyKOEhjTbS0jqLAjJHVfMipUxFEbOIiD86iPzcLC9K/wEhmFKQlGKwMn8eb9h360AbtVsfwjAQ8XcNx9MriFVmHtjBLnXr+8+gnO//RQe/PgZGKrMnuP23fG6hrzbXIvAAtoNs/nudPh9DrTl+grpYgSArQgsIm0Gk39BeMDO6KlLJdCdyV2vpS6FnYc6Qy2CC0uwPKQcIhFw2oTSxiIYhjFTHZXF2l5DweIqwKsgHlljN17744vbvE3qCggWB1xDHdI1RJ4KYHd8QkQuXrP/aG7GbHINtdanPD531e2iW/4xYREanEyi+lQCD378DLclxJDm0rmGHr3hbPz8w3M92+RnzE3nGKZ3qfimc3HSR02BYLnZIio+RuALFsvrEaBVBJf899NY/dah0OvvPNThvj5iyD5qcSwC6QtTh65b7CVpEepTtiI4fcJAzBjRilPG9MeLWw4UtVykn4lDmjBxSJN2H7ehZpjepQoUQXRBWbevOZtEnaEWFCNQz5EWgU+JWETajJ8oJQAA2xVFsOtQp/aY1vqUMQ0nqbEILMc1BORy/D+3eArOnDQIJ47qFzmmYsh93j16G4ZhfFS8a8hej8Dc8wcwWwTyPIvipZj60bmGAvEFgif7Jx/eOtDuvt55qEPrw2+pSxlToVL+RkOwLQLpGpKKoCZpYcHkIQWNMR+EYoExDNN7VL4iIEORmLKtWxPUBbyuoYLqCHxZQ7sPd2KbIrzltdu7ClubYOPuXMvp7Qfbta6W1vqkMdagswgSFrnVxr29TrD8tFgNMEzvUvGKwNxrSJ2t69NHs4pFoMsAikLNRspmgTm3PhqoDbDIWwyWzQqtq0jH1v3tbsbRzkOdmDA4qAhaQlI6TVlD9T7XUG8h1J4fr9YAABFHSURBVLQhhmF6jYpXBFHB4oRFxhYTWVcuBS2CfUe78NymfaH31imVwPjgzRr646pt2Gnw9+ue4dUdh/DAqrdwpDOtdQ2F5fanNFFZ2yKwQ0c1vWwRuJ93r96VYZjqCBbr0kdVRRAZI/ArAuC9P/w71u86EtobXxcjCI7PGyP45K9X4cxJg8IfSuH6e1/Em04HU7Xg64SRrXhp60GMGdhoPFfn+kkQhVYj+zl3yhD89bXdsY8PJ2eBMQzTe1S+IoCpoMzelrKCQt5/jEXB7qPrd9n++Re3HMDRzjTmTwwKb/W6f3llR2C/fW0K1AC8su1gxFPleFNpYz28td59/buPnY6uTBYNIWsIm2IECyYPxt3PbI60eADgrg/NiTwmLqoFxjBM71H5isAgU6SItizytHLwrGUscsdkDMe84/ZlAKC1DFRFcOtDr2rHYVnBOoL9SsWwCYuCaaxq76BkwtLGAFR0FgER4fQJgzCiXz3+7bxJkeMoJbmsoV69LcNUPRWvCIyVxc62pD9GoByj5rWriTfFtKH2096VwTf+si6w/cRR/bBqywHjeamEhZqkhcMducDykJbaeANzSBokbk3SwrIbz83rWqXA/bw5SsAwvUrlB4thCtTa2xKW5Y0RaLKDhIhuTBe29oAfNQi7cfdR7TFzxvR3X1+/UD8z9/vy61IJfOGSabhmwQTt8YB3tt3b6aFRnDSqH2aN7ocvvq1HVy9lGMZH35IEPYBFhANt3fi3X72Ag+22y+XxtTvx3UfWA7D77XhWKFNjBLIdREYgG2ER7G8Ldsr0Lz7vEmPC26CsHPbRsybgnMneNRkEoG35cPUZ4/DZxVOM11WFvy5GUE7qUgn87pr5mDGitdxDYZiqouIVAQhYs91Osbx72WYAwIfvXo51Ow8DcPoIGbqPSkGeyWaNcQSJ2gBOYupS6vfINNclcd05Ez3balM5RZBMkFb5DGjMuYK+/e4TtffyoyqCvmYRMAxTHipeEqgyd9mGPVj03b969icT/vTR3D7Zg+h7j2/Agy9td7fr3EB7FUWw50gnzvzG41i747BhTF5NcMkJw3HD+cd7ttUmLbewK2lRwB01oKEGAxrtGoGWuiQuO2Wk9l6Be0d0H2UYpvqoeEWg9q15bvM+1xKQJHzpo2rFcVc6ZwXcv2Kr+1o3O9+nKIKlq3diy752/Ohvr3uOmTC40RmT99yGmiQs38baVAJ//rczcOs7Z9hN7xRF0FybxK8/Os/uIwSv9RCFaqUkNb2GGIapPio+aygqJT1lWb5AcG5ft6FHjy5YvPdoFzJZgVP/41H0a7B9937L4YbzJ+PaX64MNFVT4wGS2qSF8YObMN5pG6HGKN5/2hiMGdiIhtr8K4DVIfW1GAHDMOWh4qeEUZ0sExZpi8gAGJu16WIEh9q7cbQrjT1HurDBKTbzrxEge/f4g6G6oq9aX58fNQNJPlGjo0DiNsHzX4djBAzDAFWgCKIsgoTPJaMK+S7DUpE619Dja3fhhFuW+q7lPUb65FNJC7+/5nTUpeyPv7E2aBHU+dw96rhGD2gAANciMGYnaVCvY6ojYBimuqgCRRBtEajsPtyJHQftBV/iuoZSCcKKN/ZHjkXOwNOZLE4e3R9nOG0ppEXwxKcXuMcGLAJH2F99xji8d84oADmLIB2xpKXuOup4GIapbipeEkTNef2ZM5/77cuY97XHsGHXYU+wWMU/AR/UZK7oXTR9qHsfKXjldTud31KgjxuUaxBXm/RaBPKeZx8/2FVuMraQj0WgHqoqgkZNnIJhmOqg6oPFfotAsvNQp9Ei8McIBjXVYvvBDu2xcuH3umTCVTryuvK3dPGo1Ka8OlreU3XnSEsiTisLHWqw+JmbzjMqPoZhKpuKtwiigsWmFMr2roxREfgXlxnYZG7b3OQI+dpUAiP72779xTOGA8hZBrqsoTqfRSCDvKrikrGFfCyCdyv1BupSla31KQxuzq9XEcMwlUHlWwQR+/2+eEl7d8Z13fjxrzEctviLVAR1KQuDm2ux+suLXMEvs5J0Y/BbBDJ9VJ3F52sRrPnKItQmE/iNUxPB6aMMwwDVoAgiLAK/wJW0d2eM6aP5KALXNeRkATUqbiBpEeiWhPQrh6xrEah+/fwUgT9NlYPFDMMAVeAaiooR+IOyks7uDDq79Yrgjic3et7LCl8dstirTqNwXEWgEcjBYLEmRqBJO80HbjHBMAxQDYogYn8hriG/T765zmxYSUXk9/kDwE0XTcXAxhoc168+sM+vOOQtE55gcXGKgFcCYxgGqALXUFSw2KgIurLoTGe0+/z4i7/iHrNo+jAsmj7MMC6fRZANBot1yoVhGCZfKl4RRE16Tf799hDXkB+d20eyYPJgPPXabnwpz8VW/G6brCZryLIIl80aiUtOGB56revOmehZvezOD5yCR9bszGs8DMNULhWvCKRFMLy1TpvrP1zjlgGA9q60MVjsJ8wi6NdQg3uuPjXWdVT8bhvXNeTb/u33RK9D8OlFkz3vL5g+DBcYLBGGYaqPilcEMkhw2viBGNxSix8+5W0NrS74riJXM4uDKeAM5O/H/+3HTsfyzfsC23UWAcMwTCmommBxKmHhpgunBvYPa9UrggMaRWBq0lYfIuzz9eOfMqY/Pnp2cM1hWczM8V2GYUpNxSsC6RoyFU+ZLIIDbUFFYJqN1/kCzv99+cm5+5doBi9rBaKC3wzDMPlSlCIgopOI6B9E9CIRLSeiuc52IqLvEdEGInqJiGYp51xJROudnyuLfYDoMdq/TbP5fg36YLHONWS6hj9GcMH0oSXP0WfXEMMwPUWxFsE3AHxZCHESgC867wHgQgCTnJ8lAO4AACIaAOBLAE4FMBfAl4iof5FjCCVnEdiP+odr5+MWJYNHDcqeOWmQ+/pAW3AxenV2/6frznBf+xVBTcLC459agF99ZF6Ro8+RZdcQwzA9RLGKQABocV63AnjLeX0pgJ8Lm38A6EdEwwEsAvCIEGKfEGI/gEcALC5yDKFIuSldQyeN6ocpw1u0x1560gj3dVSMYObI3Cpj/vRRIsKoAQ04bcLAQocd4KRR/Zx7ce0AwzClpdisoesBPExE34KtVE53to8AsEU5bquzzbS959C4hkwuHrW4TAZn//O9J+GBVW/h8bW7jG6Z+l4Qzt+7/CRs3HU0tJ0FwzBMIURaBET0KBG9ovm5FMDHAHxSCDEKwCcB/KRUAyOiJU7cYfnu3bsLv46jCdR2036BLt/rmr8d168eU4c3e46bO26Au3/KsGbU9oIiaKhJeqwQhmGYUhFpEQghFpr2EdHPAXzCefsbAD92Xm8DMEo5dKSzbRuABb7tTxrueyeAOwFg9uzZha28An2zNv8aBM11SRxo69a3g05a7vEWEZ77/HlocaqRn795IRprExzAZRjmmKbYGMFbAM52Xp8LYL3z+gEAH3Syh+YBOCiE2A7gYQAXEFF/J0h8gbOtx5Bpl8mE2SKQTeN0FkFdKuFuzwqBIS11rp9+cHMtGmqS2u6hDMMwxwrFxgg+AuC/iCgJoAN2hhAAPATgIgAbALQBuAoAhBD7iOirAJ53jvuKECJYRltC5MpeajpnQBHUpgC0a1crsy0C+3hT23818+i1/3dhkSNmGIbpXYpSBEKIpwGcotkuAFxrOOcuAHcVc9980HXtNFkEujV7a1OWu4CLf61iHTqrgmEYpi9T8VJL5xryZw01O5k4R7vSgfPVRecLXCOeYRimT1PxTed0wWK/RfCJ8yZh5Zv7MWfsAPipr0mg3lnisbPbvD7BuVOG4ATO6mEY5hik4hWBXE3MkzXka/8wc2QrVn7hfO35dakEBjRKi8GsCO760Jxih8owDFMWqsY1lArJGopiQGNt9EEMwzDHKBWvCHTN2nTZQWEMbKwp6ZgYhmH6EhWvCHIWgTlGoHLrO2fg+oWTPNv6syJgGKaCqXhFkHUyQhOWOWtI5YpTx2DB5CGebY15rjLGMAxzLFHxiiDtaIJkTIsAyDWRk1aEf/1ghmGYSqLiFUHGyf2P031UIhUBt45gGKYaqPj00aybPho/a0iuL6BWCf/5385gxcAwTEVS8YpAFyyWrp5Zo/tpz5FtpU+fkFuxbPpxXCzGMExlUvGKwLTW79JPnoXhrfqF61vrU3jw42dg4pCmHh8fwzBMual4RZDWFJQBwPFDm0PPmzGCLQCGYaqDind667qPMgzDMDkqXhHo1iNgGIZhclS+ItBkDTEMwzA5Kl46smuIYRgmnIpXBNI15G89zTAMw9hUviJgi4BhGCaU6lEE3C+IYRhGS/UoArYIGIZhtFS8Iqh3WkizImAYhtFT8ZXFv/zIPCxdvRPNdalyD4VhGKZPUvGKYMLgJnxsAfcMYhiGMVHxriGGYRgmHFYEDMMwVQ4rAoZhmCqHFQHDMEyVw4qAYRimymFFwDAMU+WwImAYhqlyWBEwDMNUOSScNs19GSLaDeCNIi4xCMCeEg3nWIGfuTrgZ64OCn3mMUKIwVEHHROKoFiIaLkQYna5x9Gb8DNXB/zM1UFPPzO7hhiGYaocVgQMwzBVTrUogjvLPYAywM9cHfAzVwc9+sxVESNgGIZhzFSLRcAwDMMYqGhFQESLiWgdEW0gohvLPZ5SQUR3EdEuInpF2TaAiB4hovXO7/7OdiKi7zmfwUtENKt8Iy8cIhpFRE8Q0RoiWk1En3C2V+xzE1EdET1HRKucZ/6ys30cET3rPNuviajG2V7rvN/g7B9bzvEXAxEliOgFInrQeV/Rz0xEm4noZSJ6kYiWO9t67btdsYqAiBIAbgdwIYBpAC4nomnlHVXJuBvAYt+2GwE8JoSYBOAx5z1gP/8k52cJgDt6aYylJg3gU0KIaQDmAbjW+XtW8nN3AjhXCHEigJMALCaieQC+DuC7QoiJAPYDuNo5/moA+53t33WOO1b5BIBXlffV8MznCCFOUtJEe++7LYSoyB8ApwF4WHl/E4Cbyj2uEj7fWACvKO/XARjuvB4OYJ3z+ocALtcddyz/APgjgPOr5bkBNABYCeBU2IVFSWe7+z0H8DCA05zXSec4KvfYC3jWkY7gOxfAgwCoCp55M4BBvm299t2uWIsAwAgAW5T3W51tlcpQIcR25/UOAEOd1xX3OTjm/8kAnkWFP7fjInkRwC4AjwDYCOCAECLtHKI+l/vMzv6DAAb27ohLwn8C+CyArPN+ICr/mQWApUS0goiWONt67btd8WsWVyNCCEFEFZkORkRNAH4L4HohxCEicvdV4nMLITIATiKifgB+D2BKmYfUoxDRJQB2CSFWENGCco+nFzlDCLGNiIYAeISI1qo7e/q7XckWwTYAo5T3I51tlcpOIhoOAM7vXc72ivkciCgFWwn8QgjxO2dzxT83AAghDgB4ArZbpB8RyUmc+lzuMzv7WwHs7eWhFst8AG8nos0A7oXtHvovVPYzQwixzfm9C7bCn4te/G5XsiJ4HsAkJ9ugBsD7ADxQ5jH1JA8AuNJ5fSVsH7rc/kEn02AegIOKuXnMQPbU/ycAXhVCfEfZVbHPTUSDHUsARFQPOybyKmyF8C7nMP8zy8/iXQAeF44T+VhBCHGTEGKkEGIs7P/Zx4UQV6CCn5mIGomoWb4GcAGAV9Cb3+1yB0l6OABzEYDXYPtVby73eEr4XL8CsB1AN2z/4NWw/aKPAVgP4FEAA5xjCXb21EYALwOYXe7xF/jMZ8D2o74E4EXn56JKfm4AJwB4wXnmVwB80dk+HsBzADYA+A2AWmd7nfN+g7N/fLmfocjnXwDgwUp/ZufZVjk/q6Ws6s3vNlcWMwzDVDmV7BpiGIZhYsCKgGEYpsphRcAwDFPlsCJgGIapclgRMAzDVDmsCBiGYaocVgQMwzBVDisChmGYKuf/AyyFWWwtdk/jAAAAAElFTkSuQmCC\n", 252 | "text/plain": [ 253 | "
" 254 | ] 255 | }, 256 | "metadata": { 257 | "needs_background": "light" 258 | }, 259 | "output_type": "display_data" 260 | } 261 | ], 262 | "source": [ 263 | "total_rewards = SARSA(env, episode_n=500, noisy_episode_n=400, t_max=1000, gamma=0.999, alpha=0.5)\n", 264 | "\n", 265 | "plt.plot(total_rewards)\n", 266 | "plt.show()" 267 | ] 268 | }, 269 | { 270 | "cell_type": "markdown", 271 | "metadata": {}, 272 | "source": [ 273 | "### Q-Learning Algorithm\n", 274 | "\n", 275 | "Пусть $Q(s,a) = 0$ и $\\varepsilon = 1$.\n", 276 | "\n", 277 | "Для каждого эпизода $k$ делаем:\n", 278 | "\n", 279 | "Пока эпизод не закончен делаем:\n", 280 | "\n", 281 | "1. Находясь в состоянии $S_t$ совершаем действие $A_t \\sim \\pi(\\cdot|S_t)$, \n", 282 | "где $\\pi = \\varepsilon\\text{-greedy}(Q)$, получаем награду $R_t$ переходим в состояние $S_{t+1}$.\n", 283 | "\n", 284 | "2. По $(S_t,A_t,R_t,S_{t+1})$ обновляем $Q$:\n", 285 | "$$\n", 286 | "Q(S_t,A_t) \\leftarrow Q(S_t,A_t) + \\alpha(R_t + \\gamma \\max\\limits_{a'} Q(S_{t+1},a') - Q(S_t,A_t))\n", 287 | "$$\n", 288 | "\n", 289 | "Уменьшаем $\\varepsilon$" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": 30, 295 | "metadata": { 296 | "ExecuteTime": { 297 | "end_time": "2020-11-12T12:09:41.196865Z", 298 | "start_time": "2020-11-12T12:09:41.185519Z" 299 | } 300 | }, 301 | "outputs": [], 302 | "source": [ 303 | "def QLearning(env, episode_n, noisy_episode_n, gamma=0.99, t_max=500, alpha=0.5):\n", 304 | " state_n = env.observation_space.n\n", 305 | " action_n = env.action_space.n\n", 306 | " \n", 307 | " Q = np.zeros((state_n, action_n))\n", 308 | " epsilon = 1\n", 309 | " \n", 310 | " total_rewards = []\n", 311 | " for episode in range(episode_n):\n", 312 | " \n", 313 | " total_reward = 0\n", 314 | " state = env.reset()\n", 315 | "\n", 316 | " for t in range(t_max):\n", 317 | " action = get_epsilon_greedy_action(Q[state], epsilon, action_n)\n", 318 | " next_state, reward, done, _ = env.step(action)\n", 319 | " \n", 320 | " Q[state][action] += alpha * (reward + gamma * np.max(Q[next_state]) - Q[state][action])\n", 321 | " \n", 322 | " total_reward += reward\n", 323 | " \n", 324 | " if done:\n", 325 | " break\n", 326 | " \n", 327 | " state = next_state\n", 328 | " \n", 329 | " epsilon = max(0, epsilon - 1 / noisy_episode_n)\n", 330 | " \n", 331 | " total_rewards.append(total_reward)\n", 332 | " \n", 333 | " return total_rewards" 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": 31, 339 | "metadata": { 340 | "ExecuteTime": { 341 | "end_time": "2020-11-12T12:10:00.588790Z", 342 | "start_time": "2020-11-12T12:09:52.816239Z" 343 | } 344 | }, 345 | "outputs": [ 346 | { 347 | "data": { 348 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD8CAYAAAB6paOMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXd8HNW593/P7K6qJdmybLlIRi4yLtjYjrDpYDBgTAtcCASSQC4JgRAS0riQUAIJhBtyE0ICueFNnNwQCCGkUAPYYJoJxaYYbNxw70W2LKtuOe8f087MnJmt0sra5/v5GO2eOXP2jFid5zz1kBACDMMwTOGi5XsCDMMwTH5hQcAwDFPgsCBgGIYpcFgQMAzDFDgsCBiGYQocFgQMwzAFDgsChmGYAocFAcMwTIHDgoBhGKbACed7AqlQU1MjGhoa8j0NhmGYQ4qlS5fuEUIMSdbvkBAEDQ0NWLJkSb6nwTAMc0hBRBtT6cemIYZhmAKHBQHDMEyBw4KAYRimwGFBwDAMU+CwIGAYhilwWBAwDMMUOCwIGIZhChwWBAxTYOw52IVnlm3P2XiLVu7C5ub2nI2XKl2xOFbtaLXexxMCiURqR+/G4gm0dESxZmcrOrrjyj5CCMSN8VZsOwD5WN/Wzije37zfavtoawteX7PH9/N2tHRi2/6OlOaWD/KWUEZEcwH8AkAIwG+FEHfnay4Mk0vO+eXrONAZxSvfnZ32va+s3o0ZowaioiSS0zl1xxLY39GNoRUluPqhpViycR9eWV2H286ZjPJi9TLQ3h3D5377FsqLw3jgshnWnO5ftBYfbz+An188DZGQhi/+4R2ENcJr/zUbwypLsHrnQbR1x7B+dxuGV5Vg5uhqfLTtAAaXF+GHT6/AZ5rqMWdSLZZu3IeBZRFs2tuO6vIi3P2vlfju3MMxY9Qg/HXJZvx73V6c0FiD3y/egDs/PQVT6qqwYMVOHDduMMqKwrjof/+NZVta8M7352BTcztu+vsyNA6twK8unQ4hAE0j7G7twm9e+QSzJwzFzNHViIQ07GjpxLceex9vfLLXetb/PG40mtu6cOmswzBzdDX2HOzCXc98jEWrduHyYxtw78I1uObksXhtzW40Dq3AP9/fCiGAS46qx03zJuLsX74OAPj5xUfi/Ol1eGPtHkwYXomyohDeXLcX33rsA7R3x3DDGROwvaUDIU3D544ehUUrd6GxtgKlkRBWbD+A5rZutHbGQAScNH4IfrZgNa45eSxmHz40p98HN5SPw+uJKARgNYDTAGwB8A6AzwohVqj6NzU1Cc4sZty8s6EZ0+sHIhxyKrY7D3TihRU78fmjD8vLvBpufAYAsOHus7CjpRNt3TGMHTIg6X3r97Rh9k9fxknjh6AorOH2cydjxMBS3/6L1+7B39/dip9eNBVE5LgmhMDra/fg6DGDEQlpuOqPS/DCip1Yd9c8zLzrRew52AUA+M7p4/G1UxoRiyfQGUugozuOs3/5Gu658Ei0dcVwzcPvAgAumDESP/vMNHy0tcVa9G4/dzIuPqoeE255zvrcC6aPxN/f2+qYyzFjBuPf6/Y62m4+ayJ+9MzHnmcaVV2GV2+Ybf0OTWYfPgTHNw7BD5/Wl4iLm+rxlyWbAQAVxWG0dsWsvlWlESQSAhc11WP+4vVW+5CKYrR0RNEdSwAAyotCSAigI+rUCK47ZRx++dJa5e9cZsyQcqzb3Yajx1TjzXXNVntRWEN3LIHhVSU484jhmL94PYZUFGPCsAq8FqA1+NE4dACev/5EaBol7+yCiJYKIZqS9cuXRjATwFohxDoAIKJHAZwHQCkIGMbN9pYOXPS//8YDl83AvCnDHdeu/tNSvLdpP04ePwT11WVWeyIhsGpnKyYOr+y1eR794xcB6ELBzaa97fjBU8txYmMNrjhuNDbubQOgawXmz4ub6nH7uZMx777XcOmsUagfVIauWAJnTK7FZb99CwBwy9kTsau1C39+exNuPXsSmtu68Z2/foBFq3YjEiKcM3UEXlixEwBwoDOKsLSgvLpmDzY3d0BA4LElW6z2B19dhyEVxRhUFsGls0bh/kWf4MrjR+O9TfsAACGN8M6GZjTUlDueyS0EAHiEAAClEACATc3tHiFw7NjBWLRqNxat2m21mUIAAFq7Yjhv2gg0Dh2Ap5dtR92gMiz8eKclBK45eSzqB5XhwVc/we7WLuu+H376CHx62kg8/eF2fP3P7+H6OY24d+EajxCYM7EWCz/eab2/6FN1GFVdhutObcQfFq/HD57Sl62TDx+Cl1fttgTN9pZO/PntTQCAx75yDBoGl+H55TuwubkDdz77MUbXlGNqXRWeeH8bikIaHrpyJtbuPoi31jXj3U37sGWfbkp66MpZGQmBdMiXIBgJYLP0fguAWXIHIroKwFUAMGrUqN6bGXNI0Nal7+L2t0c911o69LZO107v1698gnueX4WnrzseR4ysclwTQuCXL63FGZOH4fBhFXhj7R6Mqx2AoRUlKc/p6WXbHItpMl5auRMvrdyFl1buwrCqUuw80AkAKIlo6Iwm0B1L4KE3N+Kak8di5Y5W3PrEcuvef990ivX63oVr8Ic3NgAArjulEY++s9laNKNx4Vicp92xwDGHt9c34+31zXDz+lp95zpvyjDMmViL+xd9gg82t+DDrS0YWBZB02HVeHrZdjydoq/hhMYa5W5Y3k2PqSnHuj1t1rVbzp6Ew2sr0Fg7ALPuejFw/P/+j6koiYTwtVMaAQC/e309Xli+A7edMxmTRuiC/9JZozDhln+hM6ov1BOHV0LTCOceOQKnThiK8uIwagYU4+Z/fmSN+6tLp+PsqSMs4fSLS6bhvGkjretXHDcaZcVhlERCmDCsAi8bv/d7LpyK7z6+DB3ROK4+aSxGGwJz7hH6puW4cTUYM6QcTy/bjife34ZwiDBrzGDMGjMYl806DImEwPkPLMZ/fKoOw6pS/w5mSp8tOieEeBDAg4BuGsrzdJg+woHOKCpLIkgYJk23Wg8ARYapKBp3fm1eMf5ID3R6hcf7m/fjZwtW49XVu/H4Ncfi0t++hfKiEJbfMTfluX3tkfcc76PxhPW6O5ZAUVif1/kPLMagsiKMkrSVq/+01HpdVhRGZ7Tbeq+arynsAFhCAAASQuDt9c0YXzsAhw0ux4IVO3HFsQ2OPkFMGFaBY8YOxu8X6/1njBqEAYYP4Xv/+BCAvkOfOLzCsUuW+UxTnUcgHj9OFwTnHDkCpRENjy3ZgpBG+ON/zsL0O15AW3ccTQ2DHIJg3pRhGF6lm8Yev/oYPL98B3Yc6MLyrS1Wv2tOHovhVSUoiYQcn3fl8aNx5fGjPXNb+K2TsHzbAazddRCH11ZY7aafZIqxQRhfOwA3nDEBp07UbfPfOLURf3pzI85yaZ/689YDgMPpfPbUEfju48sAAEfWVXnuMYVT/SD9+TSXaU/TCE987XjPfT1FvgTBVgD10vs6o41hfFm+rQVn3fc67vvsdDQO1W3u7l0/AEQsQZBwtDe364ure9EAgGc/1He2A0rsP4k2n2iSVGnttO3WD7y8Ft84tRFEhPc27QegmxImj6jE8m0HHPclXH47ldbz6urdnjZAX4yWbtyHc6eNwM1nTcS+9ihGDizFp6ePxKfvXxw43xvmHo6vnjzO+sx/vLcVTQ3VHmfy6ZNqMdrH51E3qBQ/OHcyIiEND7+1yWo/ZcJQ/PhfK3HukSOwZZ8eYVRVGkFRWDN8PHEcNrgct5w9yfIDyNpYU0M1mhqqAegmtaseWoLLZo3C549pCHwm7/zKUDeoDGdMVl+fPKISVx4/Gp8/+jCH2eubp43H9XMaPb4YmdKikPL1KRP9Hb11xmagrMj7nexN8hU++g6ARiIaTURFAC4B8GSe5sIcIqzcrocKLlq5ywrrUwsC/Y/VLQj2temCIBb3Kpim7fhgZ8wRgtjc1u3pmyoHpF37vQvX4JkPnWaUDXva0DC43H0bWjqiIALmX6H7+Pa3e+dw17MrlZ/59LLtONgVw9zJw1BWFMZIw9k8uLzId5711XqfGaMGWW33XDgVf7vmGEyrH+gQjgBw/ow6nDCuxtK8ZJ657gSUFYVx5/lT8NoNszFyYCkeuGwGGmsrsOpHc3HapFoMHlAMACgv1hc/M2ClvCiELxxjO/hDPnbxUYPL8Nz1J6YtBFIhHNJwy9mTPL4PAIFCwOQPXzwKT1+n7+QfunIm/nr1MSgO+y/ywytL8IVjDsP8K47KfNI5IC8agRAiRkRfA/A89PDR+UKI5UluYwqciGFa6Y4lbNOQYtduRhF1xdQaQcwlIAAgZiz+21s6rdeAHv9dHbCImqii72SNAAB+88o6HGXsagFgY3M7Tp88DDeeOQF3/8te2IUAKkrCluai0gj8WPjxTgypKMbx42oc7QPLnOGoP/r0EbioqQ7dsQR2tHSipSNq7bgB/Xf4qcP09+VF9jLxx/+ciapSfaz3bj0Nk2973jFuhSQ06qvLsPhG25dhLog1xu9zQLE+jvmbKysKW9rcocrJUpjnCY1Jz4OBphHuOO+InpxSSuTNRyCEeBbAs/n6fKZv0tYVw/aWDowbWuG5Zu5Au+MJSyMI8hF0dMcx5QfP4zunH47Lj22AuVZ3G4LgXx9ux84DnbjiuNGWlrDjQCe6YvaYKvu8imsfedfT5r73w60tDqenELpJ4OqTxmJfezd+88o661plSQSlpiDoSD6HSIgQjQts3NuOMUPKPVEmAyTzzq8unY4zJg9DJKShOBxKmrMg78wrS+2+qvyDVKJbyoz7TPOe3a4/72WzRmF4LzhIGZtDW/wyvcq/Ptzum4WZLcu27Mesuxbi/AcWY87PXlX2KQrbJp8gZ7FpGjrYFUNrZwy3PelUNk0n8jUPv2uF/plaQDwhcN2fbadva2cMb67ba9m13Wzc2wZhOGgB4Po5jXjgshkAdNNQhbHoXdxUr7zf3PW7FYoBxWHLzixrC27M3bm5mO852IU6Re6BadY4btxgnD11RMY770qXmeh78yZg7BCvGSWII+uq8OMLpuDO8507YVPzuPP8KVb0D9M7sCBgUmLZlv245uF3cesTHyXvnAH3vbgGOw90YfXOgwBsU4sQAi+v2gUhhBVZ0R1LwLTuBDmLVbZ1wOs7AIBYwm57WYpXb+2M4pIH38Tce1/z3LNkQzNOuudlPLZkM0YMLMXJhw/B9XPGY6oRJdLc3g1NI1wwfSR+fMEUHFk/0DOGuet3R41UlIRREmBbNjF3zvIC7ZeEtvz2M/D7K2YmHTMIWSMAgKtOHIsXv31yWmMQET47c5StiRhCMN8O00KGBQGTEqa920xyyYZ4QuAPi9dbiTcAHHZ5ADDfPr50C674/Tt4bMlmyxwUlU1DCg3F9CX4mVRUgiDuU6PGdBYf7Ip5rm01ase8tmYP4glhJWqNqCpFdXkR3t24H12xOIZUFEPTCI98aRYmj3Ams5mC4JqTxuLLJ4yGaVkZUBJ2RJ74MbTSEATSAj1ykFoQlBeHrRDWTKkoUVuTf3d5E+6/dEZGY8o+AiY/sCBg0kJAvWDuOdiFlhSdmo8v3YwfPLUCv375E6vNvRCb781iZrITV3YWm8lBMhFjNZWdrLIzt9vlRO6KxZWRRPLnqzDNMi0dUcQTtsaiaYRjxgzGG5/sQVcsgWJj8S0vDuORLx+N782bYI1RYiz2VWURfP+sSZZ5ZEBxWBnm6sbUBColO39tZXHS+zLFLwLm1Im1OGuqN8Y+HUwfAdP7sCDo47R3x/CzBauVu9jeJJkLsOlHC9F054IkvXRM7UJOinILAnOhNxf/sEbWYt0dF4HOYtM0JI8vJ5e5tY/Db34O7d0xT2QNAGzYqwsC1U46rOltBzqiiCUEwiH7tzS+tgLbWzohBFAsLehVpRF8QQp7LHGNazpbi8KapS0EYQoL2RmcikmpL2GHj7JGkC9YEPRxfrFwDe57cQ3+mkbpgh4hhVIn7kzedHAvznHJeQsAIU2z7PjReAJxSyPwCgLT3C77CGQfgEqo7jnYjYGlXkFg1v+pUlyLGmPu74gikRAIafafkywUil2Lvfzebf4xI3TCGllOb5PbzpmE33z+U442U+jJkT3uIny54OTDh2RtVvLD/D+fiuBjegYWwX0cc8fbHeuZaJ10yUWxWnMM2T/q1ghiLkEgawTReMJK+pI1go1723DurxZbYYn7fTQCt2kI0H0Bhw+rAPY6TUEbDdOQO1oGsBPTWjr0shfyui0Xdit2LXByYpJ78TPNSyFN8yQwnXPkCNQMcJp9TGEhZyO7BUgu+MMXs3MyB/HjC6bgrmc/9iSuMb0H/+b7OD1bczB1qAdmIo/ocRYnnKYhTSPbWRxTO4uf+XA7WjqiWLJRr5BpZiIDziQylebSEY0rTUPm+uqOlpHHbOmIorwo7NII7NdujUDG7Qcwbwsr4vHNcarLiywntqkRyAL6UEvKOm/aSEchN6b3ObS+MUy/xX2ylBmlI2sEpimm2yePYIhrtyxfk0sY//rltRh9k7PUMRBsmmjviltmIgDY1dqJ//v3BgD6IhxPCMjrr0MjSEcQWBqBVxCYfd+95TSr7YvHNeCIkZW48FN19mf3gEbA9G9YEDApYVopeqoMrNs0dMJPFuGDzfstjSAkaQR+eQSqXbvJd/76gfX6QGdMaeJyL76yZWbVzlacdM/LAICfPr8KM+980XEYSVw4fQTyWEHRPx4fQcj2EbiRd/rPfv0E/OOrx6JuUBmevu4ER6niQ00jYPIPf2OYlLCWJcUCKodmpnLinSoEVRXH/96mfZam4Iwasp3F0biwTDTZnrbnXkDLfBbwXy3ynl7VGY1npBG4tRBTIzBNS02HDfLcA+hljKdLReJkwRPR+M+aSQ/+xjBZI9vc3YXegpB33HJUj90mLHNQSCOrTyLhNCXtNauKSm1yrfkgagbYBeXcGkFpGuGMrZ0xK5zUPVZQ9UmPs1hzagSPX3NsSs8SdkQNsWmISQ8WBExaqHbzcpG29gxrEak0gnjCzhcIh8he6MnZf9ZdL+Lj7QccbSWR1L7asw8fapU2CGuEJTfPsa6VFqX35+HYlUvqQVDYpVtbUPkI/v7VY/HmTadm9NkMkwr8jenj9JWj2YJqsctaQJuiFIMblQXHHTVktjnyCAzNgwDLNGSyemerIzs41TNewyGycgTCIXKEZ5ZFVCGj/hqPvBjLr1V1+/3maT6BvMMvLw4nPa5QXvx7InyU6d+wIGDS5pG3NuHcX72OZVv0k7YcgqA7uSAwkYWLO2oIcGoEISKHsHD3Lw6HHBqBytmqIqxpVjnlsMu2rqr1E3RqmSOpS96hh71zuVw6gEXG9HOE0lzMWSNgsoG/MX2cvra3EwL427tbsGxLC9414vW7HRpBctOQSsvx0wjkdnM3nhDCoxEUhzVHX7ma50vfPsl3LiGNLDOSW3ioqmEGaTwhUmsEbgEDALefdwQ23H2W71ipCjJVf/YRMOnCgqAHeOL9rXh51a58T6NHEIrXmRzk4kbtI7ATxwRs7SCWEB6NIBLSEJcczvJiOFpx7KDVTyOrNo97F64SBO0BGo+vjyCNHbr5VKE0I384aojJBv7G9ADfePR9XPH7d/I9DV+a27rx4ZaWtO6RSxiQ1ab/7JIqgM5/fX1Gc1JpBPGELSASwo5OEgK45QnnYTNx4dQe5IU0yL8RDmlWnL97AVVFDR0M0Hj8fARp7dCNR0hfI7Dnnqp/hGFMuMREAfLp+xdjU3N7oGnCTUI6KMZcWE17tukjqK8uxYptB5KOZdUaktqU5whL5w4IIRw7/qC+AKAR8NoNs7HjQGfgXMIaWVE97vBRVR5BoGnIz0eQkUaQpo+AzUFMFrAgKEA2BdTY90M2yZsb7IQlCPRdcnlROCUfgQpV/R/dR2Ami6m1Bvl++Xo8IVBfXYb66rLAzw1pZC3a7mgblbP4c797y3essI9GkE4Ujylc0438SVeDYBgZNg0VMOlk4iYUfa3D4A2NoLQopOzni1m2QgjrQHn3Z8o+Ar/DYwDg6j8ttZzXfvOVGWGEY4Y1skw3bru8ykcQNKxfKejMNILMfQQMky4sCPoILyzfgYYbn/Gc8tWTeQR+xzOqMLsKRZtpGioJh5RhoEHsaOnE6JueVV6Td/mJRLBGAAAvrNhpvfZ7toub6nHFsQ3WcZbhkGYtuqlEDZkcM2awp83PNJSOj0Bk6CMIBfhBGCYZLAj6CL9+RT+2ce3ug8rrQQ5PN3sOduHFj3cm7ZdsYZWxD5O3S1K7TUMlES2l8wrk7OQlG5t9+zmjhtRlKPzw63rt7HH4wbmTrYU2rJF1tKV7wQ4qMTF2qDcSyc9ZnE4Uj/m7SXeHzw5iJhtYEBwipGPG+cLv3saV/7dEeXqXTDrHXwZ9vBk1VFoUSkuDIVDgzjeWsM1BwhUVlAw/05C52Msne8mngskElaUul46GNO/3ixRKZ5HOVCNgmGxgQdDHyWQ5WL9Hr5ufzE6eztGSqrFMM5Bp3y8Jp+YjkLsE2cKj0tnEQgSXd3DjTjgzsf0BtoPYbHNrXUGmoQpZEJj1gUhtGkoHc9ps82d6ExYEBUw6C6vDR0DONlMghDTyCIJEQuCOp1ZgsyJSiSjYft4VjdtRQ1JCmczIgaXq+fpoD6aZxtYINCsG3z1+0DkCskZgyrJk2cTpwNnBTG/CgqCAiaZgaumKxbG/vVutEQiBXQc6LYGgCwJnn3V72jB/8Xp8+Y9LlOMH7Zz1k8jMz1L7NP51/QnKe5NpBBHpABhzAXePn+o5AqqKoZnu6K1aQ5wdzPQi/G3rowSVMkiVZFaaVDSCz//ubUy7Y4HtoxDCCh1atGoXZt71IhYZ5TRCGnl8GWYsvl9iV9CC2RVNOPMIFKYsP0esX6ip7BsAdMFgCiN3wlrQrlweXUviI8gE9hEwvQkLgj7I35ZuwaRbn8faXQczCh81I0+S3ZuKj+Dt9XpUj5BMQ+b4y4wyFe9t0quQ6oLANRejYX+7twYRITjssSsWRzRmlpgQyqghvwXXTwjayWOSszik1giCkIWo7WxWH0yTDuYMWBAwvQlnFvdBFhjx8Gt2tlpt6YSPmqiTwKRqnumEY0pDuYc1P0cjr48gmVYSdPldQ8CY/VS7fL8F08805I4QCmuapVWY4w8siygFl4xcets2DdnXMy38ZkUNsY+A6UVYEPRDzDh/1VooL2DmbjsV5AXe4xC27NpeH0EyQZBqJrJf+KifgPRzFpPrTOCQBo+P4IVvnojt+zsDK6lGFYfgOA6vz3Aht/MIWFlneg/+th0iZHQwu+IWObcgmpZGYIdxuoc111xzl+04zF4xCfl6qlPQaw1lHz5qEpHMQea8TXPP0IoSHFk/MPD+UVINI5VGkG34KJuGmN6EBUEfRF48s1kOVLvtDkkQBNXu8cwpyDRkSAJNc2oiq3a04qR7Xrb6yTkBgB4+Ks/x+/Mm4qcXHenz+QK7W7sch82rMBd4dyjot04b77g3LJmDLj+uAZ+eNgJXnTjGcc+gMvVnfXPOeMybMsx6H1JpBFn6CDK93y+clmGCyEoQENE9RLSSiJYR0T+IaKB07SYiWktEq4joDKl9rtG2lohuzObzGTVBzuKOblkQ2DtsIQT+740N2N7SoRzT0ggUe3y3g9Pse/+itY5+qkxmeec+a0w1jh5Trfz87ngCu1q7klYTLTYOmXGbhr5+aiOW3Hya9T4saQSVJRHce8l0DHQt/EeMrMLDX5qFiz5V52ifOboaRISvnDgGt50zyarGKju+s/YRZCAIlt9+Bl4MOI2NYfzIViNYAOAIIcRUAKsB3AQARDQJwCUAJgOYC+ABIgoRUQjA/QDOBDAJwGeNvkwPoDInyTtlOY9g3Z423PbkcnzzL+/7jGX/dI9rjmkeD+kXfCP7J1RzjIQ0353wtv2dEMJpklFhni2QzDTkNgf5cdy4GqtAnUmxcbTlTfMm4ovHjVaWmMi89o/tb0mX8uJwYBIcw/iRlbNYCPGC9PZNABcar88D8KgQogvAeiJaC2CmcW2tEGIdABDRo0bfFdnMo7+SbUFJ94K8eO0ex3t5Edy+X4/z91s/ZROOX5+QSyNw49YICOTwEURCmiecNKzph9Zv2adnJicVBCEzSziwmxU+mkpSnftx3EdPhiwHdPZ2fVsjYKst03vkMmroPwH8xXg9ErpgMNlitAHAZlf7rBzOod+RSR6Bfa/z7st+6zxURY582WkkfNVWlqjHkn0EPrNKtos1zy2Q75Z37kUhzbOTjoQ0xBJxbNmnm6zqBwULgtrKYkypq8KXTxgT2K+pYRAefmsTxg0ZENgPnhl7M47NOWs5KAVt+Qg4fJTpRZIKAiJaCGCY4tL3hRBPGH2+DyAG4OFcTYyIrgJwFQCMGjUqV8MesqSTR6AKH1WZieQd+q7WLgD6QqpCXrD9gnfcGoH7Ez0aAblMQ2HyaARFYQ0d0Tj2tnUDAIb6zE/u//++0BTYBwDOn16HpsOqk/ocAO/zmn4IE0sjyEGkj3VCGUcNMb1IUkEghJgTdJ2IrgBwNoBThf1XvRVAvdStzmhDQLv7cx8E8CAANDU1ZbMxPuTIJFLUcb/pLA6I9AGc4ZimRlBVGlGOGZMifvymZ65dftaWboWPQO4b8dEIAKDViOmv9JmfSTq29VSEAODVgIp8NIJcVAzNNmqIYTIhK9MQEc0FcAOAk4QQcnnJJwE8QkQ/AzACQCOAt6FHQzYS0WjoAuASAJdmMwfGH3kBU63Nsmlot6ER+AmheNw266i0CyLZWawepKUjioYbn8HQCntXLzuvVc5iMxy0M5pAWVHIY5930xML6DfmjMe2/Z143fCxuAWBOaW0julMAgsCpjfJ1iP1KwAVABYQ0ftE9L8AIIRYDuAx6E7g5wBcK4SICyFiAL4G4HkAHwN4zOjLJCGThDJ5t61apOQ8AtNs47ebjyfRLjQiy3xlRxg5O64ySmaYZihyzatI4SyW7e7lxeGkdvieyMgdObAUf/rSLOt8Ao8gIHXuQiZYORZZZZAwTHpkGzU0LuDanQDuVLQ/C0B9SC3jIZvlwJHBqxIECmO/3642blUBVWXMS59aAAAgAElEQVQS6PM0N7F+QmtzszdHQf68cMhbq0he9wcUh5NG5vRkRu7FR9Xj94s3+DqL3XP/05WzMHyg2vnux+ABRWjpiIKDhpjehL9ufY7cmReS+Qhk05D5yk8QyHV+fDUC4/UvXlwDIYTHwb1xb5vnPlkWhTWvs9gtCJJrBD0nCG45axJW3HGG5bcw8cufOL6xBmNTikqyeejKWbj7gimoKAn2hTBMLmFB0MdwLrKUXfhoUtNQwtPXVyOIB2sXIHtn/PvFG6zy1TLrdisEgTQWEXkWctlEUl4cSrrj70mNQNMIZYoD7WeN1rOhB5cHl79IhZEDS3HJTI6SY3oXrj7ax/Bb+DMpQ+1wFis1AmeJCcDfR+DQCBTXNXLOsTue8JiINrg1AkXZavdzujWCZDv+HITyp823Tz8cF8wYiTFp7v4Zpq/AGkEfI4eBJ0mdxaZp6BuPvocXV+7y7Qe4CsYpFQKCvEarHKfuEhMkzfGcI0coP1de1wcUh5WlGypK7P3MS8Zz9CYhjTBuaEWvfy7D5AoWBH2MhPDL200fp7PYe910Fj/x/jbpHvVY8SSOZ42cET6phlKa/W45a6LyOrmihmTTz++vOAoA8Nb3TsUHt54OIuDyYxtS+lyGYWzYNNTH6CmNQBXJoypD7Xegi6UR+BSYIHIGPKZ6dID5eUGmL430Z3E7i2dPGAoAut2+CPjkznlZFHtjmMKFNYI+hl/oZUYH0yT1EXgb/ap2ykJD6St2aQTxFDUbU+4E2f7NPm6NwA0LAYbJDNYI+hjy4kmUbR6B/VplqkkI4bHl+5qGpC2+aiyC01H7lYeWehKvPPOTxkplDQ+HiBd7hukBWCPoYySL/U8Hp7NYdV2gKxb3tKlw1BpS5RFo5InxV9UWciAJolSiokJEfIQjw/QALAj6GO6FOKs8AodpSK0RdEYTnjYVto9AjVsj8EM+SjEhCRU/05A8b02RZ8AwTPawIOhjuE1D9uv0F0DZYavWCKDQCNRjySYkddSQVyNQMXKQLQgEhOWTkNf3Z75+PO6/dIb3MzQWBAzTE7CPoI+RS2exs/qo936h0Aj8PsfOIxC+zuJkcmBQWQQVxfZXTgjZR2DfPHlEldKspBE8JSgYhske1gj6GH4+goxihpL5CBJejcCvgqbpI0gItVChFDSC2soShwNZSHN03+uu5wPo5iN2FjNM7mFB0McQsHfc8u48E8exQxAoFni1j0A9VtwSBELZJxUfQW1liWOBl6OW3Ou7qsqoLCxkXwPDMNnBpqEck1m8v41s15dHyiyLIFiQJATQFU01aihhXferPhqkEURChCkjq7DDOAnNmKD1eW7bv+rwdnP8R740C+Nqua4Pw+QK1ghyTPbHTNokHBpB+gMnqzUkhEBnzO0jUI9laQQJ9yx19BIT/nP52zXH4lunjXdoBEKao9sZrnIKm7ceO64GQyvSq/PPMIw/LAhyTLYVIvzMQZmZhmRnsZe4EClrBElNQ9IJZSo00u378qEuQggkEiKlZDLzMxiGyT0sCHJM1qYhn118JqXo/LQL+bPcGkGyMtS6aUjlLPY6fN3XAfsMYvOzEkKkHBLKEUMM0zOwIMgxmR5ba6+tfuGjmYyZPKHMoxEkKToXT6hnmKwchlmSzhE1JHStRLXTLy/WzweeUjfQauPjGxmmZ2BncY7Jtoi0v0aQwVyShI+qfATJS0wIpbDQTT/+czGvFYVCVtvjSzfjQGdM2X9oRQn+8dVjMWFYJZ76YJv1GQzD5B4WBDkmU8uQucb55hFkMG4i4P6QRnoeQQY+AqVGgGSlpPVrkbDdx08ImEwfNUg5BsMwuYWV7T6GvszayVvO9jTHCjhMJqSRUXQuPR9BXM8o85AsfNS8UqRIFEsVLi/BMD0DC4Ick+rJXP7326+zTihzjOsSBERGraHUSkwkLNOQ/+H1gT4CQ0hkUz2U5QDD9AwsCHJM1nkEWS7+MomAscKGRhCLZxA1pLieVCMwLoWy0AjYNMQwPQMLghyTbR6BcyzJtJNGOJK16AcklIVDpDyYxq/WkHkwTdyv6ByCd+xaTjQCFgQM0xOwszjHZJ9HINv1pXEzmYv8WuUsFt7jKpNFDSUEQMrM4mQJZfbnZgr7CBimZ2CNIMdkm0eQi6ghcz1OJHEWCyEcR1AGfY5dYsLv8PrgonNmHkFWGgELAobpEVgjyDVZ2/Xl16llFv/zva0QEDh/ep1zKgF5BGFNQ0IIRBMpagSG5hDT04E915OVoaYcaAQsBximZ2BBkGMyTSiz8wjUSWRBGsH1f3kfADyCIKhonabpBeTiKZqGkkVDJfMRmM+nqiqaKlxigmF6BjYN5ZhcVB+VzyOwXmcwB2f4qLOPrRGkFzXkh6alllCWjUbARecYpmdgQZBjss0jcIePWhpGlokEHo2AYISPujSCJLWG/CBQj0cNsbOYYXoGFgQ5JtvwUaddPzONQO0sdvbRNQLvAu/vI/CeISyjkXPHftWJY5RzCilOHksVlgMM0zOwIMgxuTyY5jevrMOeg10ZjysCNAIzjyCaYkJZMo0A5NQIZrjqBNk+Ao4aYpi+BjuLc0z21Uft+3cc6MSO5Z2e9kzGcq/jZh6Be4H3y4OIJQRKIyF0uIrUmWiu8wjca7YZPppd1BALAobpCXKiERDRt4lIEFGN8Z6I6D4iWktEy4hohtT3ciJaY/y7PBef35fIVCNQ5RE4rmcyluN+dR6BN3xUPWZCCFSW+u8b9IQy5/jO6/pPjhpimL5H1oKAiOoBnA5gk9R8JoBG499VAH5t9K0GcBuAWQBmAriNiJw2hEOcXNYaynbc4DwCs8RE6ucRVJREfD+LYO/6Aa8ZJzdRQxnfyjBMALnQCH4O4AY4N6DnAfij0HkTwEAiGg7gDAALhBDNQoh9ABYAmJuDOfQZss4jyOG4QWWoNdLPI/CWmPAbSxce8pnD7vHkzb7bjJMLHwFHDTFMz5CVICCi8wBsFUJ84Lo0EsBm6f0Wo82vvd+QaYkJE9+df3bRo77OYk/10YAH0MhfEMDlI3CbcSgHGgH7CBimZ0jqLCaihQCGKS59H8D3oJuFcg4RXQXdrIRRo0b1xEf0CO5zgtNNgvIr85x1+Kgr+jNkJJSlGj5qjusXuaM7i53vVXMKZxE+mkUFa4ZhAkgqCIQQc1TtRDQFwGgAHxiLXR2Ad4loJoCtAOql7nVG21YAJ7vaX/b53AcBPAgATU1N2Ybn9xruQnHpbmJ9ncVpOAmUBexcfcJpVh8FDIewzzUyvARWXx8fAZehZpi+R8Z7LCHEh0KIoUKIBiFEA3QzzwwhxA4ATwL4ghE9dDSAFiHEdgDPAzidiAYZTuLTjbZ+SSbSy9cylKVpyO+oSm/4qP947hBRxzXNqQX4RQ2FsogaYkHAMD1DT+URPAtgHoC1ANoBfBEAhBDNRPRDAO8Y/e4QQjT30BzygrfQW3qLl+9RkRlFDfkXnQsRQQgoag0FmYb8zxzQS0wkzyNgZzHD9D1yJggMrcB8LQBc69NvPoD5ufrcvkZQyGYq9/nnEQjsb+/Gtx77AD+5cCpqBhRnNZeQ4Sx2qyBBc9YCzhwgT0KZOmqIw0cZpu/B7rccE5TElQp6fSHvfUIAD7+1CS+t3IX5r69PeSzVa0A+s9jZHlRKQiP/wnLkSijj8FGGOXRgQZBjMj18Xs4j8LstG8ez56hKI48g5jmhLEnUkJ+PwKUteH0EHD7KMH0VFgQ5Jhd5BKq12O/AmsCxpNd+zmJPGeqAwcOaFhA1FGwasqOG2FnMMH0NFgQ5x98ckwoJIZT36W5nfSFMNcT0jqeW4z9+/YbyHlMQeKuP+s85HPJ3Fmuuoyrd6715xV2G+jNNdfjg1tRSUdgyxDA9AwuCHJPpgfN+Y8httvkotciiA50xLN24z7hmXzTPDlCfR+A/r5BGSZzFUt8UfQS1lSWoKvOvYeT+fIZhcg8LghwjfF6nfL8QyoU+FcdzkKNXFi6m01dVfTTIRxDW7F2/WyAQkSNS1p1Q5ldiIp3FnY+qZJiegQVBjgk6MD61+73lIPSxpHXWZ9ggQSDPy3T6qjSCeKAg0Hx39sl8BCZuTSGSRt0I1ggYpmdgQZBjMs0jkO9X2ekTwhlZpCJoEZcvkaERxBOKWkMBkw6FbI1A5QwOKjpn9ctCI2A5wDA9AwuCHCOytA35FZ0DBPwr/eikqhGYPgK3oxgI9muEJR+BewF3+whSteKkk1fAUUMM0zOwIMgx8jKeTkKZfKqYX0KZ/Vo9brAgsF+b5SC6Y15BEBQ1FNJsUeTe8esF6cjRNxXS0QjYNMQwPQMLghyTadSQ1VUIpUnJETWUgY9AFkpmyeiYon9wHoEdPup1BgMkfZtSXbRZI2CY/MOCIMc4fQTpaATCuMdHI0hBuwj6vD2t3dZrIvJdqIPzCDSpiqg3KkheqFNds9OpRsoKAcP0DCwIcozTNJTGfZZpyF8jSDauaocPAEs3NuPnC1db74mcoZjyoh4oCDTb/ON1FgfnEQSNmSocPsowPQMLghyTuWlIWPeobnP4oH3G9Yv4+XBLi+O9u3icUxD4zzHkcBY7r7nDR3vCR8AwTM/AgiDHOPII0tAJzNwBX9OQsHfEfuP6aQTuZnfxuIhmJ4kl0wis4nE+tYTsz0hRI8ji6EqGYXIDC4IcI3zfpHafbhpSCQKR9IgbP2exezy3RhAxDqQPa2QUvVOPE5ISytw1gzyH17NGwDCHDCwIckzmB9NIpiGVj8DnM+RF228379YUzDwCE7MiqLmQ+ykFkVCwRhB0eL0f2ZxPwDBMbmBBkGNEhqYh+4QyP2exUEbiyH3dJaXtdne+gDPCpyjkrAOUsISSc7yQZi/2HlOQqy3VUM9szjBmGCY38F9hjknFqau+T9YIVOGj6vGCTiEzica9GoHs7DVNQ6YgMEtVuAVSWDp9RmXSCTqYRubDH5xuXWeNgGHyDwuCHJNKmKcKc9FN+JmGfGoQyX4BP2exu5SEO3zULPxmLsqydiITkg6mcS/0cSEcYwZpBBUlEURC/gKFYZjehQVBjnHY7NNwElg+Aj9nMexF3+84TD9nsbvdXSDOFACmmSYRoBGY67Y72sf7GcqpWJjdWSNgmPzDgiDHZHpUpRU15LPzl30H8tVUTEPdLo3AEzXk0ggsgePSaeQTyorDIcc1WRBcfdLYpOGjppBkjYBh8k843xPobzgyi9OKGjJ/ql3MArIT126XF38/Z3FnNO5pc+QRGLv7SJgc47jnL2sEJRHnHsIUBBvuPks5BzemH4LzCBgm/7BGkGscPoJMTEM+AkSod+oJH6Eg09blFASa5nTsmhpBkfEzamS3ec851lLSCFLBHJujhhgm//BfYY5xmm1Sv892Fqt9BHK7Xx6BLSictHe7BIHLR2AKAvOnqRG45xGWylAXh51fnXQK7LnHZBgmv7BpKMdkelSlM3xUcV3YdnWnFmC/9tuVd0RjjvcEp23eNM+YPy3TkGuckFRioiSSnUYgj5mM755xON7btC+j8RmGSQ4LghyTafiofDCNOmpIWHb1uHSocUKhEbhRawT2e9tZ7DQNeTSCkF10zq0R+IWuJiMVjeDa2eMyGpthmNRg01COyTihTHYW+2gEZvCPHATkcBYnBH6/eL3n5LF2l48AnjwCcvz82Qur0RmNK5zFmq8gUAmvS46q9z6IC/cBNwzD9D4sCHKMyNQ05Kg15JdZ7NUI3BFEtz+1wnNvu8s0lMxH8MyH2/HAy5/4lJhQm4ZUEUt3/8dUT5sb9hEwTP5hQZBjMjYNSff7HUxjmn7irsXfxNdH4DENQW0akupOHOiIKsNH7aih3DiLOY+AYfIPC4Ick2kegZ3NK3zCTm0fgZyxHPd5LeMWBAS3RmCYhqRFOZZIeBb3UEiKGsqRszjM4aMMk3f4rzDHODWCdExD5j3+GoHZJ2aYhtx2fD+HbbsroYxInUcgJ3fF4t7ZRzT7zOJIkhITqcIaAcPkHxYEOcZhs3dXfw5AziNQ+QgSQtimoQSwakcrJtzyHJ75cLvVx1tu2jsnQPcRyAuwqR3IpqFo3JvPIPsI3Dv5eIamIfYRMEz+YUGQYzI9qtLyEvjlEQCO8NEPt+rnED/30Q6rj7vKqB/uoypN3KYh9/Tl8NGIO3zUp7xFMjwnnTEM0+tkLQiI6DoiWklEy4noJ1L7TUS0lohWEdEZUvtco20tEd2Y7ef3NbIOH4Vf0TnbNxAX9nkCMUntcJ874IdGpDzkRtYInnh/G/7v3xsc10n6b8S1k+fMYoY5dMkqoYyIZgM4D8CRQoguIhpqtE8CcAmAyQBGAFhIROON2+4HcBqALQDeIaInhRDemMdDlAzXQ4ezWOkjkPskhFWjJxqzO2etEbh25/cv+sTxPp4Qko8gNwll7CNgmPyTbWbxNQDuFkJ0AYAQYpfRfh6AR4329US0FsBM49paIcQ6ACCiR42+/UYQwFEQLp0SE8ZPvzwCIaxEslgiYZ0ZHJU0glQXY3L5CEzcbZEQObSMWEL4mobSOXtBhqOGGCb/ZPtXOB7ACUT0FhG9QkRHGe0jAWyW+m0x2vza+w3yepipacjvPlsjsPMAZNu8O0zUD42AQWVFnnb3olxVGnG8j8YTvofXZ6oRsELAMPknqUZARAsBDFNc+r5xfzWAowEcBeAxIhqTi4kR0VUArgKAUaNG5WLIXiHzoypl05CPj8BojyUSiBoLr2wOOtgV89ynggAMqyrxtLtNQ5WlEew52I3hVSXY3tKJmgHFliBwO8Iz1QiSHWDDMEzPk1QQCCHm+F0jomsA/F3otoy3iSgBoAbAVgByoZk6ow0B7e7PfRDAgwDQ1NSUoeW9d9m2v8OVUJaOSmD/jCoWVQHhyCw26wnJppvWTn9BcPVJY7F6ZyteWrkLGhGGVXoFQdhl968s0TWCb84Zj/HDKnDEyCrTV+zxY/hpBG9979SUfRcMw+SHbE1D/wQwGwAMZ3ARgD0AngRwCREVE9FoAI0A3gbwDoBGIhpNREXQHcpPZjmHPsFLK3fi2Ltfwksf77La0juPwNYIVPkAskaQSAhLEMhRQwe7or7jD6ssxpiacgC6o7i0KOTp444EKjP6hEOEafUDrXv1+bg0Ah+hV1tZgrpBZb7zYhgm/2TrLJ4PYD4RfQSgG8DlhnawnIgeg+4EjgG4VggRBwAi+hqA5wGEAMwXQizPcg59gnc37gcAfLStRWpN31kcT6ijhuSEslhCoDum+wNkH0GQaag4ErJDRn2sMW6NwBxbtt6YssK97meaWcwwTP7JShAIIboBfM7n2p0A7lS0Pwvg2Ww+ty9iHhAvh1UmswzF4gnsPtiF4VWlVl8/M4pcjC6REJZJSD6Y/mCAaag4bB8zaS7mFcVhR/kJ9/nBXcbYcqip+crUAEbXlGP9njace+SIgCdlGKYvwwfT5AjTVCPvqv3kwKodreiMxvE/C1bj1dW78e4tp1kLq19SmICcUCYsASCfPdAapBGEQ9Yibi7s79w8B0IA//3cSgD2mcUAMKamHNGYVyiRZRoClt9+BsIhQmc0gQHF/FVimEMV/uvNEZZGINnZ/SJpzrj3Vcf7HS2dltDo9tEIdrd24e31zQB0M0yXYpEO0giKJI3A3OC7zxSQI3jCIbLm4tAITNMQgHJj8XcfZJ8Ksw8fgkWrdqd9H8MwuYcFQQ5YuGIn9h7sAuA0rwgAK7YdABEwcXil7/3t3TFLfXCfLmayfk+b9TqeEEoTUqCPIGyfLqbKKnYT0jR0devjyd3NQtSZlpQwmX/FURlnYTMMk1tYEGTJ1v0d+NIflyivCQHMu+81AMCGu8/yHeNgV0wyDSUPteyOJZQCw302sUxxWLNMQ+7yECoiIbJMQ7LgsBSeLBdx8ql3xDBM71NQ+f2PLdmMHS2dgX26Ywn8YfF635LObjpdtf5lG3+q1UdbOqK2achYfH9y4VQ8/KVZyv5tXTFfzcEPOWoolUJvYY3QbUYNSe2mUMhWI2AYpu9QMIKgozuOGx5fhjN/8Wpgvwdf/QQ/eGoFHluyJaPPcezoU1wr97dHPRpBJEQoiaj/9xzs9hcEF8wYiaKw977isGYt4m6NwK8SaZthapLH+9zRhwEAThw/JOiRGIY5hCgYQWDu3Pe1RwMzfls69KSsA53+yVlByAu07Ct+bMlmfLS1RXGHLgjs8FH9RVjTfJ2wQujzrBtUarWZu/wRVaVY/aMzPffIpiF3mKiKsEboMH5ng8rtukRT6qqw4e6zMGJgqd+tDMMcYhSMIJCjbPa3+y/y2da+kTWCrphtNrrh8WU4+5evK+/Z1Wqbqw4YgkjXCPyjcZrbuh07dVOjGFCiu33Onz4SXz+10bpeHAlZW/9UfARyGGy1okAdwzD9h4IRBLItP8gha/lChW5OuuWfHwVqB+6M2k9229E9pnYh86c3N3raNjW3W6/NXICwpvmahgCgub0bRSHNiv03p2GGdP784mm46kS7/p/TWZxc2MlhsNUDWBAwTH+mYKKGZI0gsGSyFScv8Mjbm/DQmxtRVhzCTWdOVHYPEioqQXDzPz/ytK0zhMeA4rAVAhoJa4Eawb62bowcVIrisObIPaiQErvKIiGcPXU4ovEEBpUVWT4Cv/BRIQS+fdp41FWXWkdghjVyjMkwTP+jYP7CZTNNUF0cM05eCHuRDwqQCRrr9qdSO29n6/4OAEBDTRk+2noAgL4jDzQNtXdjdE25bh7qstvLpUVb0wi/unSG9T6VqKHrDHPSQqN43qDyIi4VzTD9nIIxDckaQdAu3lwjEwn7XICgdTDVc4JT4bDB5dbrcEhDiSL6x0QIPZqn2NUnqNSD+RihFE4FM4UF+wcYpv9TOIIgai/+7l386p2tmHvvq2jpiCJuLP5dsYSlCQRl4uay6mbDYLtcczhEnmqgbiIhTXcCSwQKAuu8YefzkKIcqXla2aDyiOcawzD9i8IRBJJpyO0j+J8XVmHljla8sXaPJTA6onGrVlBQ/lWqiWep0CBpBJEUdu2XH3uYVyMoCRIEho8ghYQyU1i4j6tkGKb/UUCCQHIWu8w5ZmmG0qKQFV3UEY1bkThBGoHK8fwNKWzzzCNUp3yqGV0jm4aCF+vSSAinTKhNSyMwtZeUMouNzy9nRzHD9HsKSBDIGoFzF28JgogkCLrjVomIIGepeyzAWWBOTvoaIy30JqXSQj6koth6be7If/uFJuu+QWURPH3d8fjJhVOx4FsnAkBaPgJTewmlVGJCH5cjhhim/1MwgqAzwEfQJlXtNPt1dMsagd130cpdaLjxGew8oCeBubULAKiWMnEHSs7WL50wxtO3QjLlVJTYZhhzIZ4zqRbjaysAAKdNqsURI6vwmaZ66/hHt3AJyj2I+WgEQyuLPfM2+wSZmhiG6R8UzF95V9TfR2CWUoglBDpjtmnILEUhO1PnL14PAPh4+wHUVpYoTUPFYQ2vfnc2NjW3Y7l0dKWfuefl75yMbS0dKC8OKfuaDltVyYkfnDsZxzfW4KiGaqze2RqovZgC0B019KXjR2N4VYnjlDHzudg0xDD9n4L5Kw/yEbR16Yt/NJ5w+AjMxTAumX86DDOSGeOvFAQRDaMGl2HU4DKs2tlqtfvZ5htqytHg2tnLZSBqK0sAqJ3WJZEQzp46wtHPDz8fQTik4bxpIx1tpimNTUMM0/8pmL9yZ2ax067fYRzAEosLyzTUGY1bEUTdcWGYigR2GwfQ7D3YDSGEdSCNjLxzL5J29snCQWXkxXqYscDvV2Qqp0PM0giS+wjM30NpUcF8RRimYCmYv/KgzOJ2yzRkawQHu2LWPdF4AhNvfc5xz7WPvIuzpw7H08u2ez5LduDKi38khQVYdZ/pRN4XUCwvFeJpCQJT8ykYNxLDFCwF81cuJ5S5s4HNxLF1e9osf8H2/Z1Yu+sgAP/jI1VCAHAKAtnEk45GEHH4CHQn7v727pTvV5GORmBqUCUZnEfMMMyhRQFpBOqoIbkq6U+eW2W97ojG8ZZxWHwqx0fKyDWC5AVd5SPwy0uWBcioaj1C6Ogxg9OahxvT15FK9VHz91LMGgHD9HsKSBCo8wje+GSPp+/42gFYvfOg9f6jbeoDZfxwmIY0WSNIwzQkCY2aAcX4902nYMiA4oA7khPziRpSYZuGWCNgmP5OwWz3OqMJq9aOHDX0xtq9nr7HjatxvDcrgqaKwy/g0AhS/3W7w0CHV5WmZVpSYfsIkveda2REm9oIwzD9l4IRBB3RuJV1K5uGtrd0OpK6AOC4sU5BICNnDadCxEco5IN0NIIvnzAGH91+RtKQVIZhDn0KRhB0RuOoNDJ35dj/HQc6rSxdk1ljqn3HqUnztK50ncUXN9WnNX46xA1NKJXoJSIKLFfBMEz/oV8LgoNdMdz34hp8sHk/Orrj1s5f9hHsaOl01AN66MqZqCiJYP2P5ynH9DtQ3o9wEmexm/++cCo23H1WWp+RKulEDTEMUzj0a0EQjSXwswWrsXTjPodpyPQRJBICOw90ol7SCMyFnogwvnaAZ8ygKJoJwyowcmCpo82pEeR3ATajhlgQMAwj068FgVknp60rpguCEqePYG9bN2IJ4dAIiqSInxe+eZJnzFBALZ+fXnQkFt94iqMtmbM46BjMXMMaAcMwKvq1ICgKaygKazjYFTNMQ04fwe5WvTzE8CrbIZqNQ7dIcbSkvPirxr7jvMkZf1662LWG+vX/doZh0qTfewMrisP44783OkxDSzc24/5FAi+t1A9ol8svu+v7p0ORwhlcFLYX/5BGILK1gP+aOwHzpgzP+PPSxSpDnWcTFcMwfYt+vzUsLw5bZSMqDdPQwo934Z7nV2Hpxn0AnIKgKJTcGfzdMw7HjFEDccEMZ8XOiEKIyEXbIiFNcTpw73Hr2ZMwfdRAHFk3MI+zYBimr9HvNQK5nn5pUQghjfSabbQAAAhzSURBVDxF5wbJgiAFjeDa2eNw7exxAIC/v7vValeZfqrLnIe9aERI9KZjQOKIkVX4x1ePy8tnMwzTd8lKIyCiaUT0JhG9T0RLiGim0U5EdB8RrSWiZUQ0Q7rnciJaY/y7PNsHSIfSSEjpKB0oHdDuFgS/u7wp5fFVtvfSIvmwGQ1/u+bYlMdjGIbpDbI1Df0EwO1CiGkAbjXeA8CZABqNf1cB+DUAEFE1gNsAzAIwE8BtRDQoyzkE0i3VGCotCilj+eVEL7cgOHViLY5qSG2KyYJxIiHCkfUD8ZWTvEdWMgzD5ItsBYEAYNZcqAKwzXh9HoA/Cp03AQwkouEAzgCwQAjRLITYB2ABgLlZziGQbqlyaElYLQhkVOadey+Z7ogsUnH9nEZUSZqFCg7bZBimL5KtILgewD1EtBnATwHcZLSPBLBZ6rfFaPNr90BEVxnmpiW7d+/OeILyWQLd8UTSMg+qyJ+RA0tx2zl6mKffYe7XzxkfeF4wAEQ4bJNhmD5IUmcxES0EMExx6fsATgXwTSHE34joMwB+B2BOLiYmhHgQwIMA0NTUlLF3VRYEndG4Z1f+0JUzHe/9FvPTJtXiO6ePx+XHNjjaH/vKMWhJ8QhJjTUChmH6IEkFgRDCd2Enoj8C+Ibx9q8Afmu83gpArp5WZ7RtBXCyq/3llGebASMGllpHPJZEQp6Cayc0DklpnJBG+NopjZ72maP9C9SZlEZCVggrwzBMXyPb8NFtAE6CvpifAmCN0f4kgK8R0aPQHcMtQojtRPQ8gLskB/HpsM1JPcL8K47CG5/swcGuOC45qh6/efWTnvw4JS9880Ss2J7emQYMwzC9RbaC4MsAfkFEYQCd0COEAOBZAPMArAXQDuCLACCEaCaiHwJ4x+h3hxCiOcs5BFJbWYLzp9dZ7/NRXqG+ugz10gEvw4wa/+ah9AzDMPkkK0EghHgdwKcU7QLAtT73zAcwP5vPzYYk/txe4QvHNKC2sgRnHqFyvTAMw/Qu/T6z2I07qzgfhDTq1RpDDMMwQRRcPOPg8vROGGMYhunvFJwgGOE6OIZhGKbQKTjT0MhBakGw9OY5iOepGBzDMEw+KTiN4OTxQ5XtgwcUY2hFcBkJhmGY/kjBCYJjxg7GLy6Zlu9pMAzD9BkKThAAcJxRzDAMU+gUpCDgM3sZhmFsCnJFjCSpQMowDFNIFOSKqDpzgGEYplApUEFQkI/NMAyjpCBXxDBrBAzDMBYFKQhUp5AxDMMUKgW5IiY7rpJhGKaQKMgVkZ3FDMMwNgUqCArysRmGYZQU5IoY5kPkGYZhLApSEIQMQcACgWEYpgDLUAMAEeHmsybi+MaafE+FYRgm7xSkIACAL50wJt9TYBiG6RMUpGmIYRiGsWFBwDAMU+CwIGAYhilwWBAwDMMUOCwIGIZhChwWBAzDMAUOCwKGYZgChwUBwzBMgUNCiHzPISlEtBvAxiyGqAGwJ0fTOVTgZy4M+JkLg0yf+TAhxJBknQ4JQZAtRLRECNGU73n0JvzMhQE/c2HQ08/MpiGGYZgChwUBwzBMgVMoguDBfE8gD/AzFwb8zIVBjz5zQfgIGIZhGH8KRSNgGIZhfOjXgoCI5hLRKiJaS0Q35ns+uYKI5hPRLiL6SGqrJqIFRLTG+DnIaCcius/4HSwjohn5m3nmEFE9ES0iohVEtJyIvmG099vnJqISInqbiD4wnvl2o300Eb1lPNtfiKjIaC823q81rjfkc/7ZQEQhInqPiJ423vfrZyaiDUT0IRG9T0RLjLZe+273W0FARCEA9wM4E8AkAJ8lokn5nVXO+AOAua62GwG8KIRoBPCi8R7Qn7/R+HcVgF/30hxzTQzAt4UQkwAcDeBa4/9nf37uLgCnCCGOBDANwFwiOhrAfwP4uRBiHIB9AK40+l8JYJ/R/nOj36HKNwB8LL0vhGeeLYSYJoWJ9t53WwjRL/8BOAbA89L7mwDclO955fD5GgB8JL1fBWC48Xo4gFXG698A+Kyq36H8D8ATAE4rlOcGUAbgXQCzoCcWhY1263sO4HkAxxivw0Y/yvfcM3jWOmPhOwXA0wCoAJ55A4AaV1uvfbf7rUYAYCSAzdL7LUZbf6VWCLHdeL0DQK3xut/9Hgz1fzqAt9DPn9swkbwPYBeABQA+AbBfCBEzusjPZT2zcb0FwODenXFOuBfADQASxvvB6P/PLAC8QERLiegqo63XvtsFe2Zxf0YIIYioX4aDEdEAAH8DcL0Q4gARWdf643MLIeIAphHRQAD/ADAhz1PqUYjobAC7hBBLiejkfM+nFzleCLGViIYCWEBEK+WLPf3d7s8awVYA9dL7OqOtv7KTiIYDgPFzl9Heb34PRBSBLgQeFkL83Wju988NAEKI/QAWQTeLDCQicxMnP5f1zMb1KgB7e3mq2XIcgHOJaAOAR6Gbh36B/v3MEEJsNX7ugi7wZ6IXv9v9WRC8A6DRiDYoAnAJgCfzPKee5EkAlxuvL4duQzfbv2BEGhwNoEVSNw8ZSN/6/w7Ax0KIn0mX+u1zE9EQQxMAEZVC94l8DF0gXGh0cz+z+bu4EMBLwjAiHyoIIW4SQtQJIRqg/82+JIS4DP34mYmonIgqzNcATgfwEXrzu51vJ0kPO2DmAVgN3a76/XzPJ4fP9WcA2wFEodsHr4RuF30RwBoACwFUG30JevTUJwA+BNCU7/ln+MzHQ7ejLgPwvvFvXn9+bgBTAbxnPPNHAG412scAeBvAWgB/BVBstJcY79ca18fk+xmyfP6TATzd35/ZeLYPjH/LzbWqN7/bnFnMMAxT4PRn0xDDMAyTAiwIGIZhChwWBAzDMAUOCwKGYZgChwUBwzBMgcOCgGEYpsBhQcAwDFPgsCBgGIYpcP4/r9LhmUKPXxsAAAAASUVORK5CYII=\n", 349 | "text/plain": [ 350 | "
" 351 | ] 352 | }, 353 | "metadata": { 354 | "needs_background": "light" 355 | }, 356 | "output_type": "display_data" 357 | } 358 | ], 359 | "source": [ 360 | "total_rewards = QLearning(env, episode_n=500, noisy_episode_n=400, t_max=1000, gamma=0.999, alpha=0.5)\n", 361 | "\n", 362 | "plt.plot(total_rewards)\n", 363 | "plt.show()" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": null, 369 | "metadata": {}, 370 | "outputs": [], 371 | "source": [] 372 | } 373 | ], 374 | "metadata": { 375 | "hide_input": false, 376 | "kernelspec": { 377 | "display_name": "Python 3", 378 | "language": "python", 379 | "name": "python3" 380 | }, 381 | "toc": { 382 | "base_numbering": 1, 383 | "nav_menu": {}, 384 | "number_sections": true, 385 | "sideBar": true, 386 | "skip_h1_title": false, 387 | "title_cell": "Table of Contents", 388 | "title_sidebar": "Contents", 389 | "toc_cell": false, 390 | "toc_position": {}, 391 | "toc_section_display": true, 392 | "toc_window_display": false 393 | }, 394 | "varInspector": { 395 | "cols": { 396 | "lenName": 16, 397 | "lenType": 16, 398 | "lenVar": 40 399 | }, 400 | "kernels_config": { 401 | "python": { 402 | "delete_cmd_postfix": "", 403 | "delete_cmd_prefix": "del ", 404 | "library": "var_list.py", 405 | "varRefreshCmd": "print(var_dic_list())" 406 | }, 407 | "r": { 408 | "delete_cmd_postfix": ") ", 409 | "delete_cmd_prefix": "rm(", 410 | "library": "var_list.r", 411 | "varRefreshCmd": "cat(var_dic_list()) " 412 | } 413 | }, 414 | "types_to_exclude": [ 415 | "module", 416 | "function", 417 | "builtin_function_or_method", 418 | "instance", 419 | "_Feature" 420 | ], 421 | "window_display": false 422 | } 423 | }, 424 | "nbformat": 4, 425 | "nbformat_minor": 4 426 | } 427 | -------------------------------------------------------------------------------- /Coding/Practice-5.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch import nn 4 | import random 5 | import gym 6 | 7 | class Network(nn.Module): 8 | 9 | def __init__(self, input_dim, output_dim): 10 | super().__init__() 11 | self.linear_1 = nn.Linear(input_dim, 32) 12 | self.linear_2 = nn.Linear(32, 32) 13 | self.linear_3 = nn.Linear(32, output_dim) 14 | self.relu = nn.ReLU() 15 | 16 | def forward(self, input): 17 | hidden = self.linear_1(input) 18 | hidden = self.relu(hidden) 19 | hidden = self.linear_2(hidden) 20 | hidden = self.relu(hidden) 21 | output = self.linear_3(hidden) 22 | return output 23 | 24 | class DQNAgent(nn.Module): 25 | 26 | def __init__(self, state_dim, action_n): 27 | super().__init__() 28 | self.state_dim = state_dim 29 | self.action_n = action_n 30 | 31 | self.gamma = 0.95 32 | self.epsilon = 1 33 | self.memory_size = 10000 34 | self.memory = [] 35 | self.batch_size = 64 36 | self.learinig_rate = 1e-2 37 | 38 | self.q = Network(self.state_dim, self.action_n) 39 | self.optimazer = torch.optim.Adam(self.q.parameters(), lr=self.learinig_rate) 40 | 41 | def get_action(self, state): 42 | state = torch.FloatTensor(state) 43 | argmax_action = torch.argmax(self.q(state)) 44 | probs = np.ones(self.action_n) * self.epsilon / self.action_n 45 | probs[argmax_action] += 1 - self.epsilon 46 | actions = np.arange(self.action_n) 47 | action = np.random.choice(actions, p=probs) 48 | return action 49 | 50 | def fit(self, state, action, reward, done, next_state): 51 | 52 | self.memory.append([state, action, reward, done, next_state]) 53 | if len(self.memory) > self.memory_size: 54 | self.memory.pop(0) 55 | 56 | if len(self.memory) > self.batch_size: 57 | batch = random.sample(self.memory, self.batch_size) 58 | 59 | states, actions, rewards, dones, next_states = list(zip(*batch)) 60 | states = torch.FloatTensor(states) 61 | q_values = self.q(states) 62 | next_states = torch.FloatTensor(next_states) 63 | next_q_values = self.q(next_states) 64 | targets = q_values.clone() 65 | for i in range(self.batch_size): 66 | targets[i][actions[i]] = rewards[i] + self.gamma * (1 - dones[i]) * max(next_q_values[i]) 67 | 68 | loss = torch.mean((targets.detach() - q_values) ** 2) 69 | 70 | loss.backward() 71 | self.optimazer.step() 72 | self.optimazer.zero_grad() 73 | 74 | if self.epsilon > 0.01: 75 | self.epsilon *= 0.999 76 | 77 | env = gym.make('CartPole-v1') 78 | state_dim = env.observation_space.shape[0] 79 | action_n = env.action_space.n 80 | agent = DQNAgent(state_dim, action_n) 81 | 82 | episode_n = 100 83 | for episode in range(episode_n): 84 | state = env.reset() 85 | total_reward = 0 86 | for t in range(500): 87 | action = agent.get_action(state) 88 | next_state, reward, done, _ = env.step(action) 89 | agent.fit(state, action, reward, done, next_state) 90 | state = next_state 91 | total_reward += reward 92 | if done: 93 | break 94 | print(total_reward) 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /Coding/Practice-6.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# DDPG\n", 8 | "\n", 9 | "Задаем структуру аппроксимаций $\\pi^\\eta(s)$, $Q^\\theta(s,a)$ и начальные вектора параметров $\\eta$, $\\theta$.\n", 10 | "\n", 11 | "Для каждого эпизода делаем:\n", 12 | "\n", 13 | " Пока эпизод не закончен делаем:\n", 14 | "\n", 15 | "- Находясь в состоянии $S_t$ совершаем действие\n", 16 | "\n", 17 | " $$\n", 18 | " A_t = \\pi^\\eta(S_t) + Noise,\n", 19 | " $$\n", 20 | "\n", 21 | " получаем награду $R_t$ переходим в состояние $S_{t+1}$. Сохраняем \n", 22 | " $(S_t,A_t,R_t,S_{t+1}) \\Rightarrow Memory$\n", 23 | "\n", 24 | "\n", 25 | "- Берем $\\{(s_i,a_i,r_i,s'_i)\\}_{i=1}^{n} \\leftarrow Memory$, определяем значения\n", 26 | "\n", 27 | " $$\n", 28 | " y_i = r_i + \\gamma Q^\\theta(s'_i,\\pi^\\eta(s'_i))\n", 29 | " $$\n", 30 | " функции потерь\n", 31 | "\n", 32 | " $$\n", 33 | " Loss_1(\\theta) = \\frac{1}{n}\\sum\\limits_{i=1}^n \\big(y_i - Q^\\theta(s_i,a_i)\\big)^2,\\quad Loss_2(\\eta) = \\frac{1}{n}\\sum\\limits_{i=1}^n Q^\\theta(s_i,\\pi^\\eta(s_i))\n", 34 | " $$\n", 35 | "\n", 36 | " и обновляем вектор параметров\n", 37 | "\n", 38 | " $$\n", 39 | " \\theta \\leftarrow \\theta - \\alpha \\nabla_\\theta Loss_1(\\theta),\\quad \\eta \\leftarrow \\eta + \\beta \\nabla_\\eta Loss_2(\\eta),\\quad \\alpha,\\beta > 0\n", 40 | " $$\n", 41 | "\n", 42 | "- Уменьшаем $Noise$" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 1, 48 | "metadata": { 49 | "ExecuteTime": { 50 | "end_time": "2020-12-10T13:19:40.443616Z", 51 | "start_time": "2020-12-10T13:19:40.430613Z" 52 | } 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "#Ornstein–Uhlenbeck process (Процесс Орнштейна – Уленбека)\n", 57 | "\n", 58 | "class OUNoise:\n", 59 | " def __init__(self, action_dimension, mu=0, theta=0.15, sigma=0.3):\n", 60 | " self.action_dimension = action_dimension\n", 61 | " self.mu = mu\n", 62 | " self.theta = theta\n", 63 | " self.sigma = sigma\n", 64 | " self.state = np.ones(self.action_dimension) * self.mu\n", 65 | " self.reset()\n", 66 | "\n", 67 | " def reset(self):\n", 68 | " self.state = np.ones(self.action_dimension) * self.mu\n", 69 | "\n", 70 | " def sample(self):\n", 71 | " x = self.state\n", 72 | " dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(len(x))\n", 73 | " self.state = x + dx\n", 74 | " return self.state" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 6, 80 | "metadata": { 81 | "ExecuteTime": { 82 | "end_time": "2020-12-10T13:32:18.792167Z", 83 | "start_time": "2020-12-10T13:32:18.748995Z" 84 | } 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "import numpy as np\n", 89 | "import torch\n", 90 | "import torch.nn as nn\n", 91 | "import random\n", 92 | "from collections import deque\n", 93 | "from copy import deepcopy\n", 94 | "\n", 95 | "\n", 96 | "class TwoLayersNeywork(nn.Module):\n", 97 | " def __init__(self, input_dim, layer_1_dim, layer_2_dim, output_dim, is_tanh):\n", 98 | " super().__init__()\n", 99 | " self.linear_1 = nn.Linear(input_dim, layer_1_dim)\n", 100 | " self.linear_2 = nn.Linear(layer_1_dim, layer_2_dim)\n", 101 | " self.linear_3 = nn.Linear(layer_2_dim, output_dim)\n", 102 | " self.relu = nn.ReLU()\n", 103 | " self.is_tanh = is_tanh\n", 104 | " self.tanh = nn.Tanh()\n", 105 | " \n", 106 | " def forward(self, input):\n", 107 | " hidden = self.linear_1(input)\n", 108 | " hidden = self.relu(hidden)\n", 109 | " hidden = self.linear_2(hidden)\n", 110 | " hidden = self.relu(hidden)\n", 111 | " output = self.linear_3(hidden)\n", 112 | " if self.is_tanh:\n", 113 | " output = self.tanh(output)\n", 114 | " return output\n", 115 | " \n", 116 | "\n", 117 | "class DDPG():\n", 118 | " def __init__(self, state_dim, action_dim, action_max, gamma=0.99, tau=1e-3,\n", 119 | " batch_size=64, q_model_lr=1e-3, pi_model_lr=1e-4, noise_decrease=0.01):\n", 120 | " self.state_dim = state_dim\n", 121 | " self.action_dim = action_dim\n", 122 | " self.action_max = action_max\n", 123 | " self.pi_model = TwoLayersNeywork(state_dim, 400, 300, action_dim, is_tanh=True)\n", 124 | " self.pi_target_model = deepcopy(self.pi_model)\n", 125 | " self.q_model = TwoLayersNeywork(state_dim + action_dim, 400, 300, 1, is_tanh=False)\n", 126 | " self.q_target_model = deepcopy(self.q_model)\n", 127 | " self.noise = OUNoise(self.action_dim)\n", 128 | " self.noise_threshold = 1\n", 129 | " self.noise_decrease = noise_decrease\n", 130 | " self.noise_min = 0.01\n", 131 | " self.memory = deque(maxlen=10000)\n", 132 | " self.batch_size = batch_size\n", 133 | " self.gamma = gamma\n", 134 | " self.tau = tau\n", 135 | " self.q_optimazer = torch.optim.Adam(self.q_model.parameters(), lr=q_model_lr)\n", 136 | " self.pi_optimazer = torch.optim.Adam(self.pi_model.parameters(), lr=pi_model_lr)\n", 137 | " \n", 138 | " def get_action(self, state):\n", 139 | " state = torch.FloatTensor(state)\n", 140 | " _action = self.pi_model(state).detach().data.numpy() + self.noise_threshold * self.noise.sample()\n", 141 | " return self.action_max * _action\n", 142 | " \n", 143 | " def update_target_model(self, target_model, model, optimazer, loss):\n", 144 | " optimazer.zero_grad()\n", 145 | " loss.backward()\n", 146 | " optimazer.step()\n", 147 | " for target_param, param in zip(target_model.parameters(), model.parameters()):\n", 148 | " target_param.data.copy_((1 - self.tau) * target_param.data + self.tau * param.data) \n", 149 | " \n", 150 | " def fit(self, state, action, reward, done, next_state):\n", 151 | " self.memory.append([state, action, reward, done, next_state])\n", 152 | " \n", 153 | " if len(self.memory) >= self.batch_size:\n", 154 | " batch = random.sample(self.memory, self.batch_size)\n", 155 | " states, actions, rewards, dones, next_states = map(torch.FloatTensor, zip(*batch))\n", 156 | " rewards = rewards.reshape(self.batch_size, 1)\n", 157 | " dones = dones.reshape(self.batch_size, 1)\n", 158 | " \n", 159 | " pred_next_actions = self.action_max * self.pi_target_model(next_states)\n", 160 | " next_states_and_pred_next_actions = torch.cat((next_states, pred_next_actions), dim=1)\n", 161 | " targets = rewards + self.gamma * (1 - dones) * self.q_target_model(next_states_and_pred_next_actions)\n", 162 | " \n", 163 | " states_and_actions = torch.cat((states, actions), dim=1)\n", 164 | " temp = (self.q_model(states_and_actions) - targets.detach())\n", 165 | " q_loss = torch.mean((targets.detach() - self.q_model(states_and_actions)) ** 2)\n", 166 | " self.update_target_model(self.q_target_model, self.q_model, self.q_optimazer, q_loss)\n", 167 | " \n", 168 | " pred_actions = self.action_max * self.pi_model(states)\n", 169 | " states_and_pred_actions = torch.cat((states, pred_actions), dim=1)\n", 170 | " pi_loss = - torch.mean(self.q_model(states_and_pred_actions))\n", 171 | " self.update_target_model(self.pi_target_model, self.pi_model, self.pi_optimazer, pi_loss)" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 7, 177 | "metadata": { 178 | "ExecuteTime": { 179 | "end_time": "2020-12-10T13:47:46.108496Z", 180 | "start_time": "2020-12-10T13:32:18.797167Z" 181 | } 182 | }, 183 | "outputs": [ 184 | { 185 | "data": { 186 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABe+ElEQVR4nO29e5gcV3nn/33r0tfpuY/ukiXZso1twMbyBYIhgIMNgRjIJmuSLLAheHkSfpt9QpIfCUmWXwKbkNsmsAHibBJDApgEB3CCCWAuhgA2lm0Z2ZZlS7JkXWak0Vz73l1V5/dHnXPqVHX1daZnRprzeZ55pqe6uvtMdfd5z/t9L4cYY9BoNBqNBgCM1R6ARqPRaNYO2ihoNBqNRqKNgkaj0Wgk2ihoNBqNRqKNgkaj0Wgk1moPYKmMj4+znTt3rvYwNBqN5rzikUceOccYm4geP++Nws6dO7Fv377VHoZGo9GcVxDR8bjjWj7SaDQajUQbBY1Go9FItFHQaDQajUQbBY1Go9FItFHQaDQajWTNGQUiupWIDhHRYSJ632qPR6PRaNYTa8ooEJEJ4K8AvA7AFQDeSkRXrO6oNBqNZv2wpowCgOsBHGaMHWWM1QDcDeC2VR6TRtPAI8fncGgqv9rDWBIHJxcxtVDp+nFf2n8Ki5V6H0a0dKqOi689ObXawzivWWtGYSuAE8rfJ/mxEER0BxHtI6J909PTKzY4jUbw/i8cwJ9+7dBqD2NJ/PKnH8Vf3P9MV495fqaEX717P+770WSfRrU07n/qLO74h0fOe4O9mqw1o9ARjLE7GWN7GWN7JyYaqrQ1mr6TrzhYLK/N1XKnzBSqyFedrh4zuVAGAFTqbj+GtGTmSjUAwOn58iqP5PxlrRmFUwC2K39v48c0mjVFoeqgWOtuQl1LMMZQrLmoOV5XjzuTrwIAam53j1spitzITfYgi2l81ppReBjAHiLaRUQJALcDuHeVx6TRhGCMoVh1UKyuzdVyJ1QdD67HujcKfLLt9nErRYEbhalFbRR6ZU01xGOMOUT0HgBfBWAC+DvG2JOrPCyNJkTN9eB4TE5A5yP5ij/2epcr/jN8sq25a3Nvd/F/ndGeQs+sKaMAAIyx+wDct9rj0GiaITyE4nlsFMTYu13xixW49hQuXNaafKTRrHnEhFqqufC8tbliboeYPLuNDZxd5DGFNWoUxHtzRhuFntFGQaPpEjXAXFqjWTjtKCzRU+hWdloptKewdLRR0Gi6RJWNzlcJqRf5iDEWxBTWqKcgYgrzpfqaTZtd62ijoNF0SUHJOjpfg829yEcL5Tqq3Bis1ZRU9f3QElJvaKOg0XTJheAp9CIfneHxBGDtGoVi1cHW4TQAXavQK9ooaDRdohqC89VTKPbgKagr77UqHxUqDi7eMABAewq9oo2CRtMlqlEonacFbIVK956CCN6OZhNr0ih4HkOh5uCSCd8o9NLsT6ONgkbTNcWaq9xu7ik8cWoBX3/qTN/H4/JCOsY6T48VcZFuJvez3ChsG0mvyeyjUt0FY8CmoSSyCbPnDCTPY6sepO7mvVxutFHQaLqk0KF89LFvH8bvfPFAw/F7HjmJN3z0u12/7l/e/yz+8v5n5d9Vx8Vvfv5x7P3g13HV//wqrvyfX8WDR2c6ei7h7Tgea1tr4XoMf/71Z/DF/acxnLGRS1ldGZN3/P0P8ZFvPNv+xA752/94TnZB/f1/fQr3c8Mr/qeBpI2NQ6me5aNP/eAYXvWn316WsfbCtw6dxYs+8LVVkya1UdBcEDz2/Byu/v2vYbZY6/trlaoOLIMAtA40T+ermCnUGlZ9T55exBOnFruWYL759Bn8u7JXwNHpIv5p30m8cNswfvU1e1CquXj2TGcto9UJp11c4ZkzeXzkG8+iXHPx8zfsgG0aXcUinjq9iIOTix2f3wrXY/iDf3sKn/3h83A9hk/+4Bi+eegsgCAdNZs0sWkw1bN89NTkIiYXKnD65A3VXQ+v/8vv4hsH473IJ04uIF91MLcCn+U4tFHQXBAcmylivlRfkZbJhaqL8YGkvN2Mc4UaHI9hsRw2HMKQdJu5lK86UsIBgvbV//VlO/GuV+wGAJky2o5ujIKYbD/80y/Cb9xyORKm0ZVBq9TdkOS2FApKxfK5QhWuIvWI+3IpC5uGUqFsqW4Qj6v0KW4yV6zhqclFHDi1EHv/Wd6J1lmlanltFDQXBA5v0FZeAS24WHWQS1nIJEyUWkzs5/iX+1wxPDmJOEQzeeAP/u0pfP6Rk7GvO1OsST2/Uvd/J20DCdP/KndqFFSD1G6Cz/Nd1nIpv1VawurOU6g4Xsvr1A1qa2yRclrl10EEzweSNjYN+vJRL21IhOzUr7jCAt+HQ4w3ytn86laNa6OguSDwuERTWqYVaSuKNQfZpIVs0moaaK7UXbmBzUwhLANIT6HJY7+0/xS++XSjtCAmkem8WMn6/2vKNmGbBCKg2uFEVujKKAQrcABdeQoeb8+9XJ6CuHZTCxUpD1Ud4Sn4k+1A0vcUHI81GOROmFopo9DEUApPQRsFjWYJCFe7vAIb3xSrDgaSFgaSVlP5SEzcAHCuEPEU2nRZzVccORELPI/JiVVMGsIApCwTRISkZXQlH5EfFmk7+QjjNqB4Cp1OWGI8pWV6X8REOl2o4uRcCUDgMYn3YiBpYeNgCgBwZqE7o1Cpu5gv1UPPu9wIo9Bs1zvRdLC+Su3JtVHQXBC43gp6ClUXmYSJbNJsOrGrhmAmahSkfNQ41prjoep4DUZB9SoCecOftFK2/zVOWmZX8tFIJiFfsxVCPhpM2QC4fNTh64jV9nJtSCSex/UYnuCavIwp8HEOpCxs4kah27TUs0ocYjXkI8aYXFBoT0GjWQIiprASRqHAPYVMwmoqAZxTJKNzzeSjmMeKCVj8Vl9TIDwFMWmlbBMAuKfQuXw0kvEn+XaGpFDxs62Slj9d2F3IR0LiWj5PIbgu+0/Mh15DXKNs0sSmod6Mgnp+p9eyW1rJRwvluozXaKOg0SwBEVNYiaKjEo8pDCStppOd8BSIgJmIri08hLhJQRyL3qcakLMRzVsaBduQQddWVB0XdZdhNMs9hQ6yj3IpC8T1Jl8+6kzaEN7Mcu09oXpXx2Z8+Uj8z/mqg4RpIGmZGB9IwjSo6x3Y1NqGTq5lL7TyFNSMKS0faTRLwFlh+UgGmtvEFLaPZBoCzcKQxHsKTuh39DgQSBwiZbJb+UiMuVP5qFB1ZDwB4IFm1+uo6lZdbS9HZljcNROeQlEZp2kQJgaSXXsKqlGorIKnIDKPAPStTqId2iho2vL8TAmPPT+32sNoyUrFFGqOh5rrYSBpYiBptpCPqhhMWdg8lAoZBc9jcoxxE9wil41KNTc0KYRaQufDqZhJqzv5SKxQhafQNtBcqSOXtOXfCS4jdZKWqgZrW7UE6RRxHUTxoPoahYov6wk2DXVfwKae3/dAc0QiBMIxDS0fadYsf3H/M3jvPz2+2sNoiaxT6HP2kVjlZxJ+TKFVoHk8l8T4QDKUFqlOjHGBZtUjUA2BeJ2tw2nFU3BhmwSTT5CdZh+J5x3JdhpodmQ6KgBZE9GJvKHKea2aB1bqbkfFfMWqA9MgbB3x22ObBoWK10JGYTDVvaeQr0qD07dAcynwFKLe1lkla62m5SPNWmWxsvZ3sXK9QLvuJ2JCHeDyUTOt/Fy+homBJMYGEiFPQR1f3CSo6sz5mNu7J7JSYqjUXaS4lwBw+aiD1a0wTGO9GgXhKXRggNTPTStP4Xe/+ATu+Id9bZ9PpAOL7KKtw2lpCPMxnkLXMYWFCrZxg9NvT8FjjZJap/LRdL6Kj3/7CI7PFJd9fH0zCkT0J0T0NBH9iIi+QETD/PhOIioT0X7+8wnlMdcS0QEiOkxEHyER2dKsKsWqi/oa36DeFcVrfTZeQo/3A83+hBw32QlPYSyb9DNKHJFL33qDHlVSUI2COHf3eBYzxRoc10Ol7iFpK0bB7k4+Gs4Egeb//tnH8IdfORh7fr5aRy4VyEe22Y1RUOSjqou7vvcc7n38dMN5J+ZKODnXvkVJoepiIOnLcgBw0VgGNcfjdRzh2MfGwRTyVSf2OpdrLj5w75P4//71ydDxqcUKLhrL8rH3N6YANAabz+arMsurlXx0ZLqAD//70zgxu/xtXfrpKXwdwFWMsRcBeAbAbyn3HWGMXc1/3q0c/ziAdwHYw39u7eP4NB1SqjktVy2feeh5HJ0urOCIGgmK11bGU8gmTWT5qvTgZB4PPDMdasY3na9KTwEA5kr+faU2W3mGvYN6w7m7xrNgzE9zrdZdGWQGupePRrNBSuqPTs7jqdPxTeuiWn2ig0lLoBqpYs3BXd8/hs8+9HzDecVqa/no4WOzKFQdFKp1nnLqr+Z3jWfl/1CoOPI9AfwW2oA/0Z+cK+HaP/g6Dp8toOq4eMvHv4+7vn8M9yjtRMQe1BeNZQD0N9Cc5sY8WsA2vViVO8e1kudEdtt4LrHs4+ubUWCMfY0xJv7jBwFsa3U+EW0GMMgYe5D5QtunALypX+PTdE6x5jZtzuW4Hn77CwfwzzG9elYS112aUag5Hk7NlzFfauxMyRjDI8dn8YF7n8SnHzoOADIlFQBuv/MHePvf/RDXfeh+PHl6Qba4GB9IYJwbBZGNFPIUYjyMaFvuv/rWYXzu4ef9dEvLwLYRf8I6s1hBxXFlOioQZB8xxvCBe5/Ea/7s23jzx77XoFsHRsGfNOuu34ZCfe1SzcEH7n0SC6V6g3xkm74D34kBisYUZoo1KZH8zhcP4He/+IR/LapO06B9vlLH7Xc+iM88dFxmfv34ZRN4zeUbsGM0I19HeBGCoKq5goOTecwUazg6XcCRs0UcnFzE7vEsFiuO/MyIPaiD52z///3pVw/hru891/a8YtXB9R+6Hw88M42Fcl3GRBo9hYq8r5XRFX21RGPG5WSlYgq/COAryt+7iOgxInqAiG7ix7YCUGeWk/xYA0R0BxHtI6J909PT/RmxRlKqOjKQ23Af/9IvV8OzKJW621Hqo0xJjbj8M4UqfvYTP8DkQuBmf/6Rk7jzO0fk31XHxcv+6Bv4sT/6Jm7+8wcanvu+A1P46Y//AJ9+6Dj+5dFTAIBswkI24U9Al27M4UNvvgqux3BitixXcRO5JMb4l3aGexFiNTySsWMDzYuROMJnHnoe9zx6SmrpwvOYKVZRqXuNnkLdxUK5jru+fwxTCxU89vx86DmBwEBt4RJMzfFQjMgsDx+bw13fP4avPTUFx2MhWSbZVUwhOGeh7BsY8fo/ODIjC9AKVQeVuiezyFTOLFbgegyn5soymHzj7jH87Tuuk55Blf8P2URgJNWqZhFwLtdd6QFcsWUQQKDji3M2DaXktWzHvzx6El8+MNn2vJlCDWfzVXzz4BlUHU96A1FDOFOsyXG3NAqFGgwK0oqXkyUZBSK6n4ieiPm5TTnn/QAcAJ/mhyYB7GCMXQPg1wB8hogGu3ldxtidjLG9jLG9ExMTS/kXNAp/cf8z+GjMZii+pxD/AS3JQqzGL9A7/v6H+D/f7H1zlarj4sY//EasBg34X+Y7v3MEjDE5mUSzjw5N5fHDY7N48pQvjfz1A0fw6//8OP76gaPynMn5Cs4Van6mUKHWEDh+5kweRMAjv/sT+MQvvAS3X7cdF2/I4obdo/j1116Ku++4ET928TgAf9IXvXOGMwkZzBWtLoR3sCGXahpTEKvyhXIdZxYrmFwoSwlH3Feouo2BZtuXj0SPpGt2jABo3Kt4cqGC8YGEnFArdQ+lmhuquZjiRvTgpL8/gxpT6EY+Uj0F0atoseKgUncxuVCRk2KrJoEi2+psvson/kYDVao5KNfdiHyUkv+vKPir1j05JiETiYIx8XvTYAop22wbU3A9hrP5Kk7Ptw9mi4Dyw8f81G4RzFblQsb8HfSCVOHW8tEYL9BbbpZkFBhjNzPGror5+RIAENE7ALwBwM9zSQiMsSpjbIbffgTAEQCXAjiFsMS0jR/TrBDfPjSN7x4+13C8VHOaykfiSx1X2bvv2Bz2He+9vmGhVMd8qd40AHnv/tP4X/c9jbP5ahBojshHYoIs1hycmC3hD7/yNHJJCzPFmlzpihXi5ZtyABrz788sVjA+kMRgysatV23GH/30i5C0TORSNt7z6j0YzgQTbKkWrLgHkhaG0v5kKr78YuLdMJhsWry2hevlx2dKcDyGqYUK8pWgMyvgyw6Verx8JAyj0NujufpTC2VsGkrJyX2+7Hsx6qp1ijeSE5vjDIbko87rFFSJSX0fj04XUaq5yFfqoWZ/cddEpGkKo6B6LeL/F/EcVT7KJCwMpiycWawE/aIcNzAKo/71EfeJTKWNgymkbKOtfDRTqMLxWEctusVrHpzyr6eUj0KSnb+dqEgAaGV0p/PVvkhHQH+zj24F8JsAfooxVlKOTxCRyW/vhh9QPsoYmwSwSEQ38qyjtwH4Ur/Gp2mkUncbPog1x0PdZWAMsa59STZ3C3+Zq46vUZ/tcaMT9TmbrdjEl7nuek1jCmJ8pZorZZ2XXTIGwO+0CQST5g6+coymdU4uVKRL34wsz0QqVF252s3y/khAsAIWk96GXKppm4vxXAKWQTjMg/d1l+H4bAk5JY5RrDqoOjHykROs+HdPcKMQ4ylsGkzLfPw5RdoSUt3UIvcU+CQWCjR3lX3kwjL8WgrhKQDAgVPzAHyvQfUO4o1CRf6O1iKI/1/Ic5mkGXqsKGCbErUddVdO9uL9FkZHXKcNg0luYFt7CmI/B8djDZ1wo4jPsFBCpXwUk0wwmLZgGtRUsgV4dtvA8ktHQH9jCv8HQA7A1yOpp68A8CMi2g/g8wDezRib5ff9MoD/C+AwfA/iK9CsGFXHa/ggqh5AnIQUeArhL5BYual5190iJrdmAU3xZXY91rTNhdqmOpgsBwAERkVMBhfxAGPVDT/H1EJFShHNSNsmiPzrFbRwNpGyDf94NfBYgMBTiMZLRPVwLmXhyNkgo+vYuSIGUkEcI1/1PYVkpE6h7jI5se7knkI0V39qsYLNQykQERKWgdmiPzE5HpOrfzHhCSksTj7qtKI5ZZvIJEycUDyFx0/6HU5rjoe5ojoxNk7EUj5arKJYc6UBBiDlM1ELohoMwF/1n1kM5KNyLZCPNg363pK478xiBaPZBJKW2ZGnoBrbyTb1ENFd3LbFeArCmxxIWrAMahtTmOiTp2C1P6U3GGOXNDl+D4B7mty3D8BV/RqTpjX+JBNeJ6ibozguQ+Q7F0x20YAZ/5KeK/g7hQnJoRs69xSYLF4r1/1iMiOyh3K5Fqzgd0Umy6mFCnJJS1b4Rj2FqcUKbtg92nKsRIQs75oqXjObtORxtbVF2jYxmLLhMX/STCvBUZHpk0vZeH42WFk7HkM2acEwCNmE37Lbr1NQPAV+e45P5KOZBIYztmyLIa7DfKkujVzSNEIZV8Wqb2iikpM62XZVp+D4abOWYYQm0QMng60o1eOt5COxOFDjBuJ/nuVV42q8AfAn/mfO5OVjK44r9f10wsTGwWQgHy1WsCHnT7Qp22ybkqpeo8mFCl68vfm50c/wWDaJpGWEUlLF/55LWbK/VByMMUwXqpjInWfykeb8o1JvTD1Vs4ri3Fkpi0RiCtGc/V4QX5JmVbpiBal6CkA4vzwYnyu9nosnwlqy8ASSMSvgcs3P5tnYRj4CfAmpVHVlmqGYvDIJU5HZ/JXugJSbwtetUPE1czUFVCAm5oGUhULFQbUhJVUYhZp8XX8D++D6iwlYFH8lLAOzIaPgyPMSiiHP9Zx95BsZVdYxCHh6KqiJUI1CuyZxQNhACU9JyEfZyKpl01AK0/mqsnFOIB+lLBMbc8FezlOLgUeYstoHmsOeQusisuhzDaV9b1BNSZU1MAkLltlcPspXHdQc7/yLKWjOPyp1r6FILeQpxMhHgTwTSQVV+v1Es186RUzozVZsakzBU2QYNa4gVuh+ANi/vW0kA9sknOHGanIxbBRUIxSdRFuRTVgo1JzQlxsQRiEYhxowVlfGnsdQqDnIpWw58U3kknJyFoYkm/Rfp1L3QtlHQtaZk/q6JeUTgZi8NilGYU4x4AUuS82X6njx9iF5PK7NRUfFazxtVlyLobSNsYFkKLNmSplQ4zyF6XxVZnEBYW9AGEXhmWYjMYWNgymo65yKkn2UShj+9ckLT6EqY0fJTuSjhQq2DqeRsIy2jffEZ0rEAQbTNt+5L0Y+Slmwzea724lFVj8K1wBtFDQcxhgqvM++SshTaBFobiYfAeEmX90gYwoxX85C1ZEGy/FYaFUV11/IT7kMNNsNuVQo62TTYEquOtUAo5xEO/IULJS4fJRJmDJdMJOwQtcpm1CyiNRitZoDxvxMH6Hhbx5KyQl8gHcqzSUtJftIDTT74xfyUcY2G5rCiclrM89wsk0D80rbhWLVkefs3RlIZj3LRzxDKsMlsrFsQko0AlWPbyYfXbk1MFDh7CMhHzXxFCLvm+8puCDyA+YbBpM4u1hF3fVwrlDFBn5+JympwsPcPJTqIKbgP9fV20cwkrFhGiQ9PoH4LOSSNjcK8Z5CPwvXgD7GFDTnF35//EZvQPUU4lYuajDYcT1YZvhLCgSbwnSLmCDiPAX1OV0vXPSkNhkLeQr8dto25WTguB7O5iuhFE11shMTZLtAM+B7BEWefaROTv62nUEnT18+avQU1ECjSAHdOJhC2jbx/GxJTobZpIX5ch2Ox2LlIxEjyCR9zfxcoSrjOmLyEpNlwjKgxrqLNRd1YRQuGpH/l6VISV3VKXCJS/y/I9mE9DqEwToTko/C73Wl7iJfcXDVlkF855lpeX2C/9n//0X2TzTQrL5vBgVGQexrvXHQzwI7PlMEY8F1Sdnt96aYWqzgii2DsAzqWD76rddfLt+fgaQVG1PwPYXmgWaxk5+WjzR9RbjKDZ6CEiuIS0kNpRMqBmS2WMNYNgGDwrtJdUOhRUwhukOV6sWonkJQHOWixFfwhkFyQjpXqMFjCMtHTqN81IlRGEhaKPLso2i+vKz8rrlh+UhtpV0RgUZbGoCNg0kpXQn5aCBpyWK4aEoqAMyWajANQsI0sHEoxXslBem3wxlbBrcTkQSAYtWRk/TO8SzGB5INE20i5jo1o1L3kLQMZIRRyASewp6NfhZYK09BxI12jWelt6Ea3KinkEk0pqQKto6kZUxB/P8bB/2x/IgHvkW/pJRltPQUGPPrRzYNdugp8M/wRaMZXHuR74ENJO34mELSbCkfqRXz/UAbBQ0AyJL+6AdRjRXEubPql1g1IDPFGiZySUzkkr3HFFp5CnnVU2ChmII6jqBOwQmt4IXWrspDzTyFwVRQb9AKfyc2Xz5Ste1MwpQyXIHLR2KCj2uAl1MCzZsGU9jMc9qFfOQbBX8SDHkKdiAfZRL+Sli2euCTVrTmIhHJNitUnZA3sX003RD0TnRRvCbko2xIPvJff8+GXGhstkkNgWbxPm8YTEljMqCmpEaK16LZR6OZBGzTT73dNJhCpe6hXHeR4v/3Rj4WYRTE2PyYQnOjsFj2K6g3D/nvT7sCtkrd3/vCigTvozEFsZ1oS/moUO1biwtAy0cajljJtKpTiPUUmrSCni3WMJpN+HngvcYUas09hegOVX66rMEretWYQrBCL1ZdOTltGEwiX3Hw3Dm/H/2moRQsI8ZT6KBGQZBNmrKxnDo5ZSIpqWqHVdXoqoFGEVPYOJiSk5PwHgZSlpTIwvspBIHmjFwJ86ZwsiajHAqax3kKUwtl5FK+N/Omq7eGpED1MXWnk+04/UCzMKoj2YRc4QpP4Wzen+TGso1V3uKzsyGXxIZcCsdmSiFPwTYNv9DLY9ILVDEMwoZcCobhG5B8pBJcxBD+g1fyh7OPmhs91YMUe1afK1alUak6LizDkHGlaFIAgIZAc6Fal+9xa/moitFsf1pcANooaDhiNd4QU6i2iSnETMCA3wLgqq1DyCSsUCVrNwh9Oc5TOLMY9hRcjyGXslAt1ELykVrRXKo5cnKKrhA3DaZk47iaUrzmpymmOxpvNhF4CupqPJsMUlJLVReZRHz2kdCXB1VPYSjInhEZOCFNPUY+misFhU1ikvvb/3gOf/2do3jq9CLeev0O+RjhKYxmE5gt1nyjwIvbAODtL9vZ8H8aBsEyKHSdmiH0e+E5qYHm7SMZach9I2TKhUCh6uC3/+WAvG4TuSQmuNQTDSYnLQOlmtvUm9sxmgGRbxSm86KRoD+enWMZXL9rFD98bha2SRjlq29Rp8AYQ9y2LiEPkxvJqYUKkpaJX737MfzgyAxu2D2GT/3i9QD8OJe69wXgG/d8pS5fo6jIjrZpNO035re46I+XAGijoOFUpHzEQl+EcEVz8+wjIDzBzfCYguMxPNrj/s6t6hTO5KMxBQ8DSQvnCrVYT0FUNIsvnZgsv/PsNBKmgdFsQnoIoZTUhYrsidSODN+JzQ8mB1+tdML3IBjjG8Ekg8pkdaUoNl/JpWxecQzsHMti63Aan33XjbiSd/XMxgRa1dv5iiObvY1m/JX5gVMLeNHWYfzSTbvxCzdcJB8jjMJwxka+Ukex5mJqodK2LsM2jY67pCZtM+QpvOLSCfz3V1+C63aNIJeyUS1UMcBbeIiFwF8/cEQ2QrQMf7IWhjwqEaVsP+V3IJKOKviTn3kRAODD/34IVccLZW1ZpoHPvutG3PX9YyhUHOlppGw/AF93GRJWo1FQExCEB52vOHjq9CK+fWgauZSFw2fy8vzo3heA/97UXYbFioOhtC37W/njoqae2FypLpvm9QNtFC5gZos15HjOcztUV9n1GCzeM78Yko/iUkNdjGUTmCnWpNdQczzkK47s1z9brKHqBC0ZPv3QcaRtE295ybaG5ws/NzcKTTyF4YyN+VJd8RR8yUXNPhLjFxXN4sskAoxHp4v4pZfvki0fgEArrzkepgtVmb7ZDjEpTeer4eyjhOVfk6oDj/mTumkQhtK2LDQD/NRY0yCMDyTxqss24Jvv/XFs5603XnrxmDwvF5OSCYS9hoztn2MYhO/+5qtgGhT7ORCrXJEmW+QxhcvaGEIhmbRDTIaqp5BNWvi1114GwPeKzhWqMvherDo4PV/Gnd85ip980WbsHs9ioVyHYRBuv347do5nGmQTER+IehACsQeFCB5HGwmaBuGdL98Vfk5+f8VxG+IugNInKZeSxjxfcWQ/qau2DOHR5+fkAiu69wUQBIrPFaoYStsoVOvIKZ5Cvu63QZkv1WW1PeAvcMaymdj/dTnQgeYLFMYYbv7zB/CPDx6Pvf/4TBF3fGqfXOmrQTX1y15qE2guVR354RYrezHRjQ4k5OQrCm4OTeXxe196Enf/8ETb/6GVpzCdD3aocjwPLmNysgzJR0rPIVXr3zmWxTtethN/87a9+J03XAEADcVrZxYrYCxoXtYOsRqOrlqFvj/JWywPZ3zjNZFLhqq9Jxf8NgumQSAi2Y4jSlzxljp+INwYLmWbTRcGCTmhmsgmLCyU65guVNtKZrbZ2S5vYjJUPQUVNc1WGIU7v3MUDMBvv/4FeO9rL8Pv3+Z3vrl0Yw5ve+nOhtcQ16CZUVDPE/sppO14r0IgpJ5mweapBb9zbsIykOMJAAVlo6AdoxlUHU9+FqN7XwBBSuk5ZQMmcT0SXD76wdEZXPeh+0PFcflKuFPscqONwgWKx/wVerPujY8cn8PXnjojWyOHjILiEaieQlzZfanmBkaBnysyY8azCSlDnM1XwRjD7//bk3A9FnqNZrTKPjqXD6pPHdcvXvN7DQV7KtQcDzXXQzZhwmPhAKxlGvjAT12Jn7hio3zOqKdwat7Xjbd0aBRUrT8bc1vEVkTWyMRA2ChMLZY7S32NaR0NhKWkaGpmM2xZLe1r+s+dC+frNyNptZePXI+h7jKkLBPX7RzBqy/fgD0bBkLnCEMuOsAWaw6eOLWAq7cPd2yMpWFr8z+nE35BWrnWuGqPkoosEKL4sSYR4+B7dVcdGRcSHVhFkD669wUQeAqiW68qbwr56ORcGY7HQnUQQoLsF9ooXKCIoHCzfRDEKk/0uFe7OEarg0WcrVmXVJFxISZx8UUYzSawgXsKZxcr2Hd8Dt87PAO7RV8XFSFH+Q3vgvOFNi/6zju8eM0yCGk7aCkhYgvj/Ms3V6q3XE0KKUWk54ov4ubhzrKP1Ik4XKfgHxfXWkhY47lkyGhPzlfkXgqtiGsdDUTkow5SaAHVU/BX6qIza7u2Hr581NooiIVG0jZw0VgWf6fslCYQq2w/I8sv8jt6rij7U3VCx56CZfA2F+FGgq2es5WnsGmQpwrLjY+CZogipjOjGoWIIRLBYuEpqDEF2zRQ94KWHMIDYYw17EW93GijcIEijIHbZPIVE59YDasffrX/UbHqYJBr9fGegiN7sIigruh7NDaQUFIiq3LCuXRjrqNqWDVwra5KK3UPHoPcwMbhMQXTIL8moB5uU622GI72xlERcYUqH5vYUauTiRpo7imICVpc69GsLcclPAXGmF9D0GGRnCAuJdV/zc48haRiFPyVOm8r3WYctkltPQXZYyhGkxfkIvLRXKmG2WINu8cHmj4mijCM7VbPQhJaLNfby0d8zM3SUtUMLb+uwK+xKFQcGBR4l6J7a5x8NJJJwDRIegqFarDrniheEwsbUeRWdTx/e1RtFDTdIoxBO0/hFF+9VkPyUdhTUCff8HP4vZIGUzbStikncdGRciidwGjG3zDmzGIFpxcqIPI1+nZGwfNY6LVVoyVWTXJcvKLZMgjphCm/SGI8auVnuxW0vzdvIB+NKNW/7cg0k4+kpxCRj3JJ2b11oVyXxVDtCFf0Kg3xzN49BTUjCujMU2hXvCa8z1ZSjay9SFoYSFiy5UazeEoc4vnb/c/ivHzVaS8fKYHmKKJhoGo4B3g/KrEJ0DhPshBSasVpTEk1DMJYNoFz+Roc1/dggpRUXz4SXq+QpdT+Xf1CZx9doAjNPq7gDFCMgvQUVPkoHFMIjEJ4EhBB3EzCL8ZSt74E/FWgYRAmcslQEVI6YTY1VurrAr63sVCuh4KaIjg+lLb4uISnYCBjK83n+HhUo9B2NalMdpPz5Y7jCf5zB1/6nPI6wqicmiuDKDBmgXxQk/9vJ5lOzbKPiEjm/XfqKajZRyI4nbQMOcZWj2tn2MVCo9UELDLG1NYfQLBrXCcIb6lZSqpA9Q6iq/aG52whH01F+kcBvHNt1YFpEHIpG6P8vRVSarXuxXon/r7gVellR+sUpHxUCeo31PP6gfYULlCctp4Cl4/mGuWjaPaRuiJXUbed9PXgYDVjUOCCb1BaSmweSsMyjLYxBfElEQVbsZ5CRowriCmkEibK3MCJ1hKqfNRuskxapvQUTs9XujIK6ko1LtB8ar6MwZQtWx0EgcZKV433Bpp4Cv74DT6W7gLNapM+sStbu8e1yz6S+xa0mIAHVU9BBFkNkqm4nSDiA+2zj4JxtJOPxLlx8tFkzHslqpN9vd9v6+HvaqfGFBqvw0QuielCFflqXT4PENSBlCMxhaA/kjYKmi5xpKcQ/8VVJRLRNltQb+opRIwCn7izCYvvLhY0nxO7jgHAxpzfkXRyXvSfp7bSg/jwj3E3POwp+K+rjsvhu61lbFNmH8V5Cu2+TAm+zzEAnJ4vY0uHLS6izx3tfQT43S3VoiNpFPJVnF4QmU6ddWMVc3Z0p7xkh1KKICQfRQr72j2ubUzBEYHmVp4Cjykkgo14doxmutqpT3gKnaSkxt1udW6sp7AY3pMCCOQjkRlERLJ+RzxPNPsI4J5Cvhp4AEqbC8cL5CNpFCraU9D0iFiJN5t7xSRb4juLheUj8ViGSt1TAs3Rttr+BzTDM0cKiqeg6tNiI5PTC37fHd9TaD2hCK9DuOHtYgqu58ESgWalXTYQbjHc3lPwJ7vFSh35qtOVp5Btkn2kXouRTCDLCA9mOl/F1EIFBqGjfXeJCAMJCwnLaFjRJ5W6g05IRALNQGd7RyS7yD6KmwwFA8lG+agb6QgIVvXt/udwpla7lNRW8pEvharXaSDlp9P6NQT+/zSSSQSeguPFGqLxXALnCrVQ23TAT5muu4GnIO4X3zldp6DpmraeguIZnJwrx9YpBNq9zY9HPYXgg5xNKk3fauEuoRsHk5gv+YZn83C65VaD0ecW8lHIU5D6qw0i/390ePaRGmguxASaO4kpVB1PFpp1YxQs0whl8wjUQLXqKYxmEyACpgs1nJ73W0tYHa6Qs0krNqsnkI+69xSEweyk11MnbS6qHchHsk4hFRil3ROdZx75z889hQ4DzUDn8pH6uTsxW8If//vTeH62KBsGCsKBZl69PeB7CnUub8bKRwNJ1FwPp3lsbyCUfcTkZ1l00C3Iz35nRr8XtFG4QBFST7vsI8CXkOI8BTHJD3Kj4DZpq51JmHLTenFc/cJsUFZUW4ZSLTclFxQiRkE1WkWpq5qwDQN1j8HjMYWQp1BtlI/aTZZCFjk937mcoyL3UQ71Jwq6Zartji3TwGgm4XsKHRauyddJWbErz4Ql5KMOU1LNwIiJ96yTDCiRfXT/U2ewWKnHnlPpINC8azyLpGVg1/iANBC7u8g8AgJD2M7gp7oINMdVNH/qB8fwsW8fwT/vO9lwjcSGOcVqUFjmNxmsBh1tY66D+GweO1cK/Q8J3mYmMAZh+ei8jCkQ0QeI6BQR7ec/r1fu+y0iOkxEh4joFuX4rfzYYSJ6X7/Gth5QJaA4ao4nP5Cn5sqhmIKQdsTkO9gkpiA8Cb9vjhm0lIiRjwTSU+gw+2h0oDGmUFRe1zQILo8pmCaFt75UMpgE7SSGpGWi6riKxt+5pwD4UpppUEjrJyI5SUcbmYlWF5MLlY4mY0E2GW8Uug4082ZvaqC5XTM8wM8+Oj5Twi99ah/+9KuHAPi1FoJvHTor21FH4x4q20czOPTB1+GyTTlcuiGH33vDFXjDi7d0NHaBjKO0LV4zY2/HPmfMRkL3Hzwr23RHr9EAb9FRqDhSEhvNJjBbqClFfDFGgX++nztXkM8DQHqMi+WwMViJlNR+ewr/mzF2Nf+5DwCI6AoAtwO4EsCtAD5GRCYRmQD+CsDrAFwB4K38XE0PdFLRvHEwibRt4tR8ObZOQaxwRHplY6A5WLVkeNtowA/wqpOvuifvlmE/puDy1X0zWmUfqa9r8b7zrsdgEpePlF3OUra/aUkiRtaJQ6yAz+V72/LQD7qbDVp/tknvn4lcEk+dXsDxmRIu7kI2ySWt2NVut/LR9bvG8MYXb8HFEwPYOZZFwjQ66gpr89dJWgb+9fHTmC3W8Jo/fwCff+QkAOB/3L0fn37o+a7GYhiEX3z5rq4nPGEc20kqoeyjDmJLBgULnyPTBTx3rojfuOUyvGDzIK7ZPhw6X8inhZoiH2UTKNZcLPKGeXFyn6i2f4R3ElaL1wBIL6wh+6jDa9oLq1GncBuAuxljVQDPEdFhANfz+w4zxo4CABHdzc99ahXGeN4jPITmdQp+NsTWkbTvKdQ9vrEHk56C0IzFqrMx0Myzj/gqs1jzuzoWI62jxarKNgnj2WSwx6/nIWnEfzlloDkmplCsuXI3LUvxFCzeCbTuMtRdL+TKZxMmao7X9sskitcWK/7uZd1kwfjXwoqd1KSnENkta3wgie8+ew5Jy8Av3HhRw+Oa8eOXTcjMltD47e7ko63DaXz0rdcAAK7YMoiDf3BrR5u3vPLSCZRrLm65ciPe/Y+P4r/e9TCOThfx9OQiGGPIV+q4+QUb8ZoXbOhKFusFkdY6lG7dTlo1BO3kIyJ/cx6RfvqNg2cAAG940Wb8t1fsbjD6YjJnLIgLiC7BojI+NtDMFx0nZst46/U7ZN2GzeWjRaUDK+AbhbjNhJaTfhuF9xDR2wDsA/BextgcgK0AHlTOOcmPAcCJyPEb4p6UiO4AcAcA7NixI+6UdU+9XZ0C7/+y0U7ibL4Cg/yim1keGANUo2CFnlNQqjr+5iWWn07oMd+7KEU2rh/J2LBNwqahlNygBQDfLS1+/FGjEPUUsoqbLcZpGob84pfrLorVYFOdTMLf7L7dZCA8hXylLrOuuiFahCUQ6ZZxngIA/MKNF3Uk2wh+6abdsce7lY+idLqb1y1XbsItV25C3fUwPpDA4yfmAfgGW7QhufaikdCGPv3ijS/egq0j6bZ7Fqdi9p5ohVgwAcD9T53F5Ztysg13lHDQOZCPgKCHVpxRGE7bGErbeMmOYfzBbVfK42IxUowkTagLnX6xJPmIiO4noidifm4D8HEAFwO4GsAkgD9b+nB9GGN3Msb2Msb2TkxMLNfTXlAID6GZRFN1PCQtExtyKZzNV1Fxgg6NYvIXweCkbUjtXqVYc5Gx/VXLMF+lLZTrvEV18AUQqy5RrSv00lYZSAW+dab4IoU8haorV/yWQfI+y6TAKNRcf3yJYOWcTVhti7Kkp1B2MJju/sv3c9dvb+jN77++WD2GDc0Vmwcxmk3g3a+8uOvXiiMu+6mf2KaBt7xkG1K2geGM7W9mVAsSAVaClG3iZRePd3SeoJPWJdtG0jg1X4bjeth/Yh437Wn+GuG+V0H2EQCc4p5CXMaTYRDu/7VX4m/etjeUeRb1UAtV3wvPr4BRWNKzM8Zu7uQ8IvobAP/G/zwFYLty9zZ+DC2Oa7pEpJU229LP3/TGwAYe6EzZQaBRPEakFCb4PrjRdtelmoO00Mp5/v1MoYZK3WuYlH7uhh0yqCYyK1plIC2U6xhM20pjsvAWm+KLZ5kk7xMN8fxzwh5Lhlddt0N4Cos9egq3XrU59rgYV3Sz9TddsxVvfPGWZdtvN2n5hW2tgrvLzXtfeyn+64/txDvv8vfnKClFjWsJ9Zq0K14DfGntyz+axPHZEmquhz0bmsda1IlaSEni8358pshfM/49ifNwhHwE+Asfh9cMFav93UsB6G/2kfrteDOAJ/jtewHcTkRJItoFYA+AHwJ4GMAeItpFRAn4weh7+zW+C5122Ue+p2BgIpdE1fEwna/KD1ucp2AbjbUFJSWgLFpOiF5K0QnhV151CX72Ot/mS0+hxZ4Ks8UqRrOJ2CyQgiILWUbQbsEkQprvOFaqOaEWw1nuKbTDb3Ph+kahTf+fbshKT6FR917ODdiTttGRR7ScJC0Tm4fSsoBxpT2FTjGUrLBWnVsFW0fScDyGB4/OAGhdVKdO1EI+2jqShmkQnp7M+6/ZgSESqJ6CiDvkq/WGzL5+0M9n/2MiuhoAA3AMwH8DAMbYk0T0T/ADyA6AX2GMuQBARO8B8FUAJoC/Y4w92cfxXdC4Xpvso7ovH4lVykI52ApQTP5CqxeeQoN8VA12sBIrYKHBtpIv1JhCM2aLNV7cxZu8hTwFZTMSg2QhnuoplGsuFisOdoz5X+SLxrIdrZ6lp1B2cMnE8n09MgkTBqEn76MbXnfVpq4zppaLbNLCbLEmM3Y6zTpaSVK2iWqT6uIoYpOf7z7jp9a2yg4bCMUUggyiHaMZHJku8NfufA1uKQuFDYNJTC1WUKj4FdPN4hrLRd/eNcbYf2lx34cAfCjm+H0A7uvXmNYT9baegoukbcgNcoBgtSNW8NIoWIbs765SrgcrcWkUhKfQYpUY3eEsjpliTbZPFlXGAn+PWv/1TIOkzGWZYflosRz0p/+D265E+219RPsGJuWr5WLneBaXbsz1NWsEAG7aM4Gb9qxOnC2bsPD8bElW3a41TwHwJ+aFcmerdjH5fu/IOYxk7IYkAZU4owD4xXnPnfPlo06C2wJbWcAIGUp4Ybk+y0drz5RrlgUxsTdbjavykUB8mIUxEJvNJCwDltnoKagrdrHvsNgzoJWLaxntA81zxZpM6fNXd0r2kbIdoW0asvBOtLkQY1PjAp22jxAGa6HcW0yhGb/84xcvWzB5rSIKGEtKHclaI2Wb0vNth/AU8hUH11400vLckFFQJu2dY4Hk1I18pO6NIXYvLFSCLqz9RLe5uEDpLKZgyg8cEPS2F5KTMA5J04RlGDEpqUF2T8o2kbINxVNoYRR4EK1ZQ7VK3c8cEtkbSdsIteEoVl2Z4ql6CiaRlCzmSjV/A6AuM4jU1Vwv2UfNIKJljR2sRTIJv1ZF1q+sQfkobZttt+KU5yZM6ZG2a73RrEPurgnVKPQmH03ImILT0EKmH2ijcIEit+NkjUZB7D+QsIxQZaxwS0WRmlidC0+hYZOduhPSjUcyCSWm0EI+4qugZkZBdJYUklTKingKVXUvWwp7Cnw1dmbRTwPsdrWfUNz2fuv/Fxqi1YOoMem1VqKfJG2zbTM8la0jvrfQrklfwjL8H9MILSxUY9JVoFmVj3j9ymyxhprrhTZw6gfaKFygiIk9zlMQ+nySt14WcQVRW1CPBpp55XBD7yPFUwCA4UwCc3wrzk48hWZBcGEURKaO6ik4roeqUpkcjSkI+UgahS7jAmowejljCusBUcAoKq3XpHxkGV1Nztu4Ubi4g3beA0mrIV10JzcKJq+275SQfMQlXrERk/YUND0RdEltXI2rRgEI8qTTCZM3/AoCzZbhyx5xeyCUamGjoO4V0ElMoZ2nIOQj1VMQ0oR4XXUHMNMw5HHxBRrsMiiX1J5Cz4j3fDpfCe28t5YQMmeniLhCJ+281d3jBJsHU0haRkcpsCqWUqcgssnEQqffRmHtmXLNsiBjCjHBXBkr4CsmsRJJ2abvESiegpBTooFmz2Mo192QfDSsGoWW2UciptC9pxDtEmkqKami95FtEs4s+huh5Lqc2NWJrN9ZHhcaYrKazldDO++tJd56/XbpzXbCy/dMYP+JeVw01j4NNG6yNgzCrvEszhWqXY1T9SqG0jYSliH7MPVbPtKf+guUeos6BTGJyj2UuVFIWibfkD0oXpNGwaDQJC46kUblI0GrHPUg+yjeUxDygwjyJS0TC7wxmMyBl3UKgadg8EkobZtyVTXUZbA4oeWjnhHy49l8dU0GmYHmFefNeOWlE3jlpZ2l+I4PxKes7p7Iyu9Lp9hG8DnMJEzkkpb2FDRLw22RfRTIR9xT4IGspG3IVtQA9xRM4SkYISmqGJmcgUA+Sttmy0ybIPso3lOYK9ZgGiTlm5TiKUR3nrIMgoili4yNTMLCVI+B5lD2kfYUukL1FDJrsEah3/yvN78w9vivv/YyTOe79BSs4PuTtk0MpCw8PeVXRne7x0e36E/9BUpdxhRijEI9ElPgmmXKMkOTf9UJewpqXYHYJjBjqzEFf6XULo+6XfbRTLGGkYwtC72SSkyhVA1Xy6raq2kKo6CmlfaefdSt9LTeEe/7dL6KF2weXOXRrDzbR+Mlpt0TA11vMarKR+mEiYvGsijXXHzwTVfhkg3dPVe3aKNwgSKkmbguqVI+4gG3PRsHYBCwaSgFW5GJojGFaK0AEJ6AxV7O7dzbdr2PRN8jgeopiECz2uZCPi+/LTKQEsqeyZ0izk/bZshAaNojDLXjsTVZzXw+IeQj0dzwzv9yLb/d/+uqjcIFitvKU4jIR9fsGMGjv/sTGM4kfE/BDTwFcY6ffeTI5yjX4+Qj7im00ZPF5N0q0KwaBdGkDkBDDrxaqSwkK5GHPpjuPtgpDMFyFq6tF0Lto9doTOF8QchHadvfxa+bNNqlopdCFyiteh9FA81AECS2TJLSkxpotiP7Ksd5CiNZ4Sm0kY+s9impY9mg0jppGyjVXBydLshYRpynYFLYU+glpVQYQZ2O2j3qZ6Hdfsma1ohkjG4K7ZbttVf8FTUrgtNiPwUZU4jJ104onkLNcZHkK3EzElMo1ZpnH7WVj9p0SZ0t1qSBAYA3vmgLPvfwCbzho/8hV0zBzmuKfBSJKfSSUpqUnoI2Ct0SavWwBquZzyfEfgqdbAa03GijcIEiVvUe8+MKanfOqHykYpnhOoUgoBvOPpLykVqnIGIK7eSjFoFm12OYL9dlMzwAuGrrEL7yqzfhw195GgYRXnnZRGAUDFU+Cm9a38vELuUjnXnUNUkraLG+Fttmn08QEWyTtKegWT7UGgCXMRhQjULQ0yiKZRiypXXV8TCcUbKPYuSjbGygudPso0ZPYb5UA2NBjYJg81Aaf3H7NTHjbR5o7k0+0p5CrxARsgkTixVHpgxresdS9hxfSXRM4QJFncCjcYVomwsVO+IpyDoFw4hNSVU/tBbfVGTrcOvqT9n7yPXwzJk89vNN3wHF2HSoSZtqSqqoU1ACzd0SeAraKPSCuv2pZmloT0GzrKgTeINRqDc3CpYRyEQ115NxBzvSJbXYZHetf33Py9uubmT2kcfwx/9+CI89P4cHf/s1vI+RbxQ67U9jG43ZR5kleAoJ0+902WpDFU1zxLXXMYWlY5vaU9AsI+oEHk1LDbKP4mMK9RhPIRpoLtdcqSGrDGXstvn9Qi+tux7ylTpmijX8x7Pn+NiaxzviUF9fpqQuIaZARLjrF6/D2156UdeP1QRZYTqmsHRs01iV9uPaKFygtPIUao4HoiDDQcVWAspq8Vp0O85izVlSDxbRdVX0hPnCY6cA+BvsAF14CmZMTCGyP0S3vOzi8VXb5/h8RxiDtdg2+3xj83Cq7/sxx6HfuQsU1TuIpqWKrTjjCrsaYgpW4CmoxqVUc5ekd9rcIxGxia89NYVC1enBU4iTj7inoOMCK44wBrqieel85pduXJXd+rSncIGirurjAs3NJl3LVLKP3HCbi7pqFKrukr74wvMo1VxsH02jUvfw/cPnpKfQaXsKOybQLLOPdFXyiiM+E1o+WjrpxOq0WunbKxLR54hoP/85RkT7+fGdRFRW7vuE8phriegAER0moo/QWmzIfp6gGoJokVjVcZtOujaPHTDGUHM8WbxmG0bYU6i7UrvvBVEPUam72DXuN/jKVwJPodOy/riYgpCN1FbempVBewrnP30z54yx/yxuE9GfAVhQ7j7CGLs65mEfB/AuAA8BuA/ArQC+0q8xXsi0yz5qtnm56H0kvAWxEY+QjxhjICKUqs6SMkxs00Dd8z2FUd5yu1x3IZYBnXoKau8jUcj2Y5eM489+5sW4ettwz+PT9EZWZh9pT+F8pe++CV/t/yyAz7Y5bzOAQcbYg4wxBuBTAN7U7/FdqNRbZh81l49sLhPJ/ZnNICUVCArOoltxdottGqg5fqBZpH9W6m7XnoId4ynYpoGfvnZbqIpbszLoQPP5z0oIVjcBOMMYe1Y5touIHiOiB4joJn5sK4CTyjkn+bEGiOgOItpHRPump6f7M+rznNDWmaxz+UhkBUmjIGMKRuh5SzVnSbqxZRAKvOPpaEYxCl3GFMyYimbN6vHCrUO4fFNOb2V6HrOkd46I7gewKeau9zPGvsRvvxVhL2ESwA7G2AwRXQvgi0R0ZTevyxi7E8CdALB37974rmrrHLWFhOMyPD21iI25FEayidDmOVFs069cFvKRuskO4HsgaZjL4iks8i02B9M2TMPfr8Eymzfra/Y8gtXI1NCEufmKjbj5io2rPQzNEliSUWCM3dzqfiKyALwFwLXKY6oAqvz2I0R0BMClAE4B2KY8fBs/pukBx/ULz2quB9djePvfP4y3XLMVv/X6F/gxhaZGgVD3vAb5KNrZtFxzl1RtaZuExYrvKaRtE2nbRLnuykl+KcVrGo2md/otH90M4GnGmJSFiGiCiEx+ezeAPQCOMsYmASwS0Y08DvE2AF+Ke1JNe1yPydW243lYKNcxX/JX5r581Cwl1a8fqEbkI1PZLY0x5hevLSn7yEC+4o8nnTCRsg2U6y4qjgvbpI4n+FBKqk5W02iWTL+Fv9vRGGB+BYDfJ6I6AA/Auxljs/y+XwZwF4A0/KwjnXnUI3XPDybn4cDhgeMS1+tF8VocFk89jfZHshVPYWqxAo8trde7bRIWuHyUSZhI2SYqdRcpy+xqy0FRvGYQdGBZo1kG+moUGGPviDl2D4B7mpy/D8BV/RzTesFxmZzQRUGYqB6uOp5MNY0iVt4l3vAuGmg+Ml3A++45gEzCxCsvneh5fLYZ7Lss5KNK3UXVdjtuceGPyx+vuq+CRqPpHf1NukBxPCYnV2EMhHGo1FtkH/HJX+ysFg00f/zbRzBbrOHuO27EVVuHeh6fmimUlp6Ch0q9ebpsq+fRNkGjWR70V+kCxXGDyVU0nRO/W2UOiUCvaI2dVNpcAMCxc0Xs2TiAFy2xMEzNGkoneKC55vrxjm48BcMI/dZoNEtDf5MuUFRPQaz6xe9yzW1aYyDkoyKvIUiYvvEQk+7kYgWbh1JLHp9qFDK2haRtoOK43XsKfLw680ijWR60UbhAcVwmq4JLinxU5y0smnkKYvIvVOPlI8b8rTGXiqVkDaUSRthT6KIJmBiXLlzTaJYHbRQuUBwvyDAqcymoXHOlgWhqFESguRovHwHAluFl9hQSFlK2iarjoVr3ugs0G0Frb41Gs3S0UbhAcTwmZRgpG9VdmVXUVj5qCDQHH5Xl8BTU+gJZvCY9BS0faTSrhTYKK4THO4yuBH4306BVhBpLELebtTYWk39jSmow6S5HTEG8ToJv6ZniMYVWNRTxz6ONgkaznGijsALUHA/Xfeh+3HdgakVeT2ywk+IrbpGKWnM95JXWEnEEgeawp6Cu7DcPL4en4D+vkLFSCd9TqNTdjjukAkEKrY4paDTLgzYKK0Cx6mCmWMPzs6UVeT3RyTSafQQAs8UqgOatjW0z4imYQrMPKoc35pa+f7EwMsI4pSw/plBuUUMRh/YUNJrlRRuFFUCs3B3Xa3Pm8iCa1iUj2UcAMFOoAWjeokKsvAvVsFEQk++GXCq0sU2vCDlKjEP8XijXu/QUtFHQaJYTbRRWALG3cX2FjILYYCclso/qjrxvpugbhWbN7ESPo2LVgW2S7CckJt/Ny5B5BAQeifAUxO9Kiw6ucZjSU9AfZY1mOdDfpBWgzjuO1r2VCzQDzTwFXz5qWtHMJ+R8xQllAYnA8JZlyDwCYmIKShpqNxXNtqFjChrNcqKNwgqw0vKReL2gTqFz+ejiiQEYBByeLoQ24hExgOXIPAKCSVxIRapklOqmS6qWjzSaZUUbhRVA7GKm7obWT1rGFNrIR6PZBG7cPQbGgngCEMQaliPzCIjzFAJDoD0FjWb10EZhBRDGwPFWKNAsso9kTEE1ClUQoWXV8Ouu8ndYVT2FLUMpvPcnLsUbX7x5WcYoPA9RRKemyHYTaDZll1RtFDSa5UAbhRUgkI9WyFPwxD7HvEtqRD7K2CaoxS5lt1y5CURho0BE+H9eswcbcsskH3FPIU4+0r2PNJrVo987r2kQGIUVl4+scM0B4BuFwbTd8vEbBlO4YddoX41YVD5Kh4xC556CYRAM0jEFjWa50EZhBVgt+UjsdewqWU8112va4kLlo299SV9TaBuK1xQ5q5uGeIDvdWhPQaNZHrRRWAFESuqKyUd8MrcMo8EoAM1bXKhMLEPVcitEims6LtDchafgPxfpOgWNZpnQ36QVIJCPViol1TcClkkweexgQGlr0azFxUrS6Cn0FlMAhFFYvrFpNOsZ/VVaAURKqrPCxWuWYYTqAUTguFnh2krSEFNIqCmpXXoKpqG349Roloklf5OI6GeI6Eki8ohob+S+3yKiw0R0iIhuUY7fyo8dJqL3Kcd3EdFD/PjniCix1PGtBcTKfaXbXFgmyeKupGU0tJRYTaK9j1KKd9Cbp6BjChrNcrAcy6snALwFwHfUg0R0BYDbAVwJ4FYAHyMik4hMAH8F4HUArgDwVn4uAHwYwP9mjF0CYA7AO5dhfKvOSstHLjdCtuIpJBSjsDbko3DvI8s0pKTUTZ0CoI2CRrOcLHl2YIwdBBCX934bgLsZY1UAzxHRYQDX8/sOM8aO8sfdDeA2IjoI4NUAfo6f80kAHwDw8aWOcbVxVqlOwVQmy4RpQPhdzVpcrCRbh9NIWgZ2jWflsZRtou46XXsKv3rzHlw0lm1/okajaUs/l4xbATyo/H2SHwOAE5HjNwAYAzDPGHNizg9BRHcAuAMAduzYsYxD7o7pvF8dPD7QOlOnJuSjFYopCLnKNqlhhzMAyK4Bo7B9NINDH3xd6FjKNv1GfF2mpP7n61bvM6DRXGh0ZBSI6H4Am2Luej9j7EvLO6T2MMbuBHAnAOzdu3dlZtoYfuPzj8NjwKd+8fqW5610QzwZaDYDQ6BWJ6eb9D1abeIykTQazcrS0ezAGLu5h+c+BWC78vc2fgxNjs8AGCYii3sL6vlrknOFKmZ519FWrHSdQl3WKYTlo7XkKcQhita6lY80Gs3y0c9v370AbieiJBHtArAHwA8BPAxgD880SsAPRt/L/F3tvwXgP/HHvx3Ainsh3VCpe5harKDmtPYAZKB5hSuaLZNCnoKIJayFlNQ40rbp91zSRQcazaqxHCmpbyaikwBeCuDLRPRVAGCMPQngnwA8BeDfAfwKY8zlXsB7AHwVwEEA/8TPBYD/F8Cv8aD0GIC/Xer4+kml7sJjwNRCpeV5IqawcoHmxjoFNfsos0blo6RtImkZLZv1aTSa/rIc2UdfAPCFJvd9CMCHYo7fB+C+mONHEWQorXkqvCX1ibkSdoxlmp7nrPgezTHyUcgorF1PodsWFxqNZnnRfvoSqNT9yffkXKnleYF8tLJdUi2TpKeQNBX5aA3UKcSRso2um+FpNJrlRX8Dl4D0FGbLLc8L5KMVjikYxnkVU9g4mOp7Iz6NRtOatblkPA+ou56cfDv1FLqJKXz9qTMYzti4budo12OT8lGkTmEttbmI4zdvvVwaWo1GszpoT6FH1MnrxFxrT6HT7KMDJxdwet5/rj/6ykF87FuHexpbXXoKBNEnLmGurTYXcQwkrbaFgBqNpr9oo9AjIp5AtHyewrv/8RH8+defAQAslOtYrDgtz2+G43owDQJR2FOYyCVhGYThNjuvaTSa9cvaXDKeBwhPYdtIGidmy6g6btPMmZojdl5jYIzFplx6HsPUYgXnClUwxjBfqmM4U+9pbKWaKwvU1JjCT75oM67aOoSR7AXRfFaj0fQB7Sn0iDAKezbkAACnWkhI6jaczfZpni/X4XoMc6U6ijUXjsewWO7cKHzlwCR+94tPAAAKVUduqqPWKdimgUs2DHT8nBqNZv2hjUKPCPlITLInWxgFtWV2s32ap/NVAMBCqYYFbgwWK50bhW8dOot7Hj0JAChUHAykfKOgtrnQaDSaduiZokcqju8p7Oatn0WAOI66E3gHzTyFcwXfKMyV6pgv+f2UKnUPVaezbJxi1UWp5sJxPRSqjgwmW8omOxqNRtMOPVP0SLnmT9YXjWVhUGujUFM9hSa1CsIoLFbqmCsGHsJiubNgc6HqyN+qfGQqgWaNRqNph54pekTEFHIpCxsHUzg137z/UVg+ivcUhHzEGPD8bJDN1KmEVORGIV9xmsYUNBqNph16puiRCu+MmrINbB1Ot5aPXC/2tso09xQA4PhMUd7uNNhcUIxCUTEKBomYwtosWNNoNGsLbRR6RHgKKdvEluE0Ti+0yD5S4gjNahWEpwAAx1Sj0GGtQrEmjEI9FGjWnoJGo+kGPVP0SNQoTM5X4DWRhmquJyflZtlH5wo1OYEfnwnko4UOPYVi1R/PYsVBoabEFExtFDQaTefomaJHVKOwdTiFmuvhXLEae27d9WQTOpF99PxMCf/3u0flOefyVVzE228fmymC24eu5aOz+QoYQ2NMQaekajSaDtAzRY+IOoWUZWDLcBoAcLpJsLnuMtl3SMhH//LYSXzwywexUPIn/elCVRbCVeqefM5OAs1115O7v03yMWSTkToF7SloNJoO0DNFj5TrLmyTYJmqUYiPK9QdT7atFk3x5op+LcJ8uQbXY5gt1rB7IgvRAWPjYAoJy+goJVVkHgHAJN8FLheJKeg6BY1G0wl6puiRSt1Fivc6amcUaqp8xFf0s9xDmC/VMVfyDcOGXBJDvFndcNrGYMruKKZQCBkFfwy6TkGj0fSCnil6pFL3kOSS0GDKwkDSwqkmRsHxGDK2JW8DqqdQl4Vr47kkRjJ+s7qhjI3BtNWRfCSCzECwX3QgH/nHdUxBo9F0gu6S2iPVuot0wp9oiQhbhlOxnoLrMbgeC+QjXqcwx1tZzJdqMqg8MZDEcMb3FIa4p9BJoLkQIx9pT0Gj0fTCkmYKIvoZInqSiDwi2qsc/wkieoSIDvDfr1bu+zYRHSKi/fxnAz+eJKLPEdFhInqIiHYuZWz9pqzIRwCweSgtJ2QVYQSEfCQCzdJTKIU9hWEpHyUwlLaxWHHwwX97Cn9438GmY1FjCmWeFaUrmjUaTS8sdaZ4AsBbAHwncvwcgDcyxl4I4O0A/iFy/88zxq7mP2f5sXcCmGOMXQLgfwP48BLH1lcqdRcpZVvL0WxCrv5VhFGQ2UeeiCkERkEUrk0o8tFwxsZg2sZ8qYbP7TuBux8+0bQOQhgF4WUAaOySqo2CRqPpgCXNFIyxg4yxQzHHH2OMneZ/PgkgTUTt9lm8DcAn+e3PA3gNxe1G0yc8j+H/fPNZPD212NH5lbqHlB1cvqG0LdNLVURdQlqpUyjXXJnSOl+u4exiFSnbQC5pYSgkH1k4PlNCvuJgoVzHwSZjE/LRpsGUPCY8het2juKWKzdiIKGVQo1G056VmCl+GsCjjDG1suvvicgFcA+ADzLGGICtAE4AAGPMIaIFAGPwvY6+8+mHjuNPv/YMynUXl28abHt+ue7KtE8AUupxPSZX50CMfOR5IY9ioVRH3WPYkEuBiCKB5vC2mQ8encWVW4YaxiI8hc1DKTw9lYdlkExBvX7XKK7fNdrRNdBoNJq2ngIR3U9ET8T83NbBY6+ELwP9N+Xwz3NZ6Sb+81+6HTQR3UFE+4ho3/T0dLcPb+DEbAl/+JWnAQDVenwbiihR+UhIN9HAsCgqS/OVet31axIEc6Uazi5WsHHQd6RGMuGUVADYMpTCzrEMfnBkJnYsRd7Ge9OQ7ykMpKzYLT81Go2mHW09BcbYzb08MRFtA/AFAG9jjB1Rnu8U/50nos8AuB7ApwCcArAdwEkisgAMAYidBRljdwK4EwD27t0bL7R3wd9/7xhcjyFlG6g6nRmFquPFGoX5cj20B7JIQc0qgWbhKaRsA/PlOhZKdbxgs++dXDwxgIRlYOtIWtYsXLdrFGnbxH0HJhs8EcCXjyyDMJZN8tfSUpFGo+mNvkQfiWgYwJcBvI8x9j3luEVE4/y2DeAN8IPVAHAv/KA0APwnAN/kslLf2X9iDi/eNoyRTKLjnc4qdRfpSEwBaGxgFycfCU9h51gWC6U6zuarmMj5E/rLLhnH/t/7CWzIpTCYDuICN+4ew2LFwVOn/bjCgZML+Pn/+yB+5dOPosh3WhNylipraTQaTTcsafYgojcD+CiACQBfJqL9jLFbALwHwCUAfo+Ifo+f/loARQBf5QbBBHA/gL/h9/8tgH8gosMAZgHcvpSxdUrd9fDE6UW8/aUX4Wy+0rGnUI7IR0Np3zuYj2Qg1eS+C0GgWaSj7p7I4ltPT6Ncd7FhMIjDZ/hK/4rNg9g5lsGrLt+ApGXAIOCrT04hYRl488e+B8djSFgGfvKFmzGQtJDjcpMIMms0Gk23LGn2YIx9Ab5EFD3+QQAfbPKwa5s8VwXAzyxlPL1waCqPmuPhRduG8Z1nzi05ptDcU+AVza6HxZoLImDHaBbl+hQAYGMuhSi7Jwbw7d94lfz7pReP4csHJrFQrsMwCO99zR782defwTNn8sgmTekhZLVR0Gg0PbLuk9cfPzkPALh6+zCSttGRfMQY81NSldx/UXQ2X4oaBV8BC+QjP6YwmLIxPhDEHlRPoRk/+cIteO5cEZ97+ARef9UmXMezinyjYMnahAEtH2k0mh7RRuHEPEazCWwbSSNpdRZoFuekEoGnMNjUKIjsI//cmuPHFEazCQxnFKMQ4ylEufWqTTANQs318HM3XITd41n+GgwDSQuDwijoQLNGo+mRdT97PH5iAS/eNgQiQtIyZZuIVsgNdpQ2F7ZpYCBpNZWPRExA1CmMZGzpXQDAhlx7T2E0m8CrLpvAybkyrts5AsDPairWXGQTSkxBewoajaZH1vXsUam7ePZsHrdctQmAP3HPlxtbVTQ+Lhw8Fgyl7YbHC/nINg1YpuGnpBbr2DKcknGIhGmEWlS04iNvvQaux2Qdwq6JLJ44tejLRzyWoAPNGo2mV9a1fDRXqsFjfiUwAD+m0EGgWXgKokuqYDjT2OpCeAq2acA2yM8+KtUwkklIQzCRS3ZcbJZRPAIA2DU+AAAYSJoYySSQTZjYyvd30Gg0mm5Z10tKIfWIGoOkZXYUUyjHyEfieeabyEdilzZRpzCSTcg01k6CzM3YxeMK2aSFdMLEt37jxzGqxCo0Go2mG9a1pyBW9YFR6Cz7SMYUIvLRcMZuWqdgmwZsk5CvOKg6HkYyCfm6ncQTmrFbMQr+c6Vg6Q11NBpNj6zr2WOx4jeSEz2GOs0+EjGFpB2+fEPpBBYieyqLmELCMmAZBs7m/T0XRrM2EpbBM58yPf8PwlPQcQSNRrMcrOuZpEE+sk0ZU6i7HkwiGEaj1i9aVWciqZ/DGRsL5RoYCwLBoZiCRTi76DeLFd1QP/OuG2IL1zrlsk05vPaKjbhht+6EqtFols669hSEURA9hoR8xBjD6/7yu/j4A7KPH1yP4eCk33foqdOLIAIunsiGnm8obaPuMpRqgQQljIJlEmzDwFm+oc4ob5p3+abBUAO9bknZJu58296O2n1rNBpNO9a1URBtrnOKfOQxv+r4+EwRDx4NmrR+/akpvO4vv4unTi/i8ZPz2LNhIJQFBARVzQdOLeDffnQaBycXA/nINGCZJA3RsA4GazSaNci6l49yKUu2ok7ybKJCxUHdZXjmTF6ee2K2DAD45tNn8PiJebz68g0NzydSTN/1yX3Ic4lpN/cmbNOPKQhGl+AdaDQaTb9Y957CoLLaF4FjsX/ymcWqzCY6V/Bln7sfPoGZYg0v3j7c8Hyi1UW+6uAvb78al24cwNHpIgzy90q2Td/4EAVxDI1Go1lLrG+jUKmHJmexhaWaVvrMmQIAYJobhZNzvsdwdYxREKml73jZTtx29Vb87N7tAHwvAYBMFR1O2w0b5Wg0Gs1aYF0bhYVy1Cj48tFsMShAO8QlpHOFmjQaScvAZZtyDc93yYYc/vGdN+C3X/8CAMCbrtkKyyAkhFHghmBExxM0Gs0aZd0bBZF5BASewpyyh/Kzwijkq7hx9xgGUxau3DIoV/9RXr5nHAn+POMDSbz68g3IJH1jIx6zlGwjjUaj6SfrOtC8WHbCnkIkpmAahENTwlOo4qqtg/jj//RijHTYvA4APvTmF2JywZecLFN7ChqNZm2zro1CM/lojhuFF2zO4ZkzeXgew0yxhvGBJG7lHVU7ZSKXlPsvi+yj0awOMms0mrXJupWPao6Hct0NZx9F5KNrto9grlTHs2cLcD2G8YHeexQBkNlHWj7SaDRrlXVrFBYrvMVFpnmgWWQYfe/wOQDA+BIa1wFB9pGWjzQazVpl3RqFaN8jIIgpCPno6h3DAIDvH+FGYWBpk7nwFHRra41Gs1ZZklEgop8hoieJyCOivcrxnURUJqL9/OcTyn3XEtEBIjpMRB8h3jmOiEaJ6OtE9Cz/PbKUsbVDtLhoJR9tH8lgLJvAQ0dnAQATS5WPDJ19pNFo1jZL9RSeAPAWAN+Jue8IY+xq/vNu5fjHAbwLwB7+cys//j4A32CM7QHwDf533wia4cXIR6UaEqaBhGXg0o052bJiqTEFkX2kA80ajWatsiSjwBg7yBg71On5RLQZwCBj7EHGGAPwKQBv4nffBuCT/PYnleN9IZCPGusUFsp1WVsgitQsg5bcmsLWMQWNRrPG6WdMYRcRPUZEDxDRTfzYVgAnlXNO8mMAsJExNslvTwHY2OyJiegOItpHRPump6d7GpzcYCcmpsAYkOG7ql260TcKYwOJ2L0VukFXNGs0mrVO2zoFIrofQFxy/vsZY19q8rBJADsYYzNEdC2ALxLRlZ0OijHGiIi1uP9OAHcCwN69e5ue14q4mEJCqVLO8J3MLts0AGDp0hEAZBImEqYRMkQajUazlmhrFBhjN3f7pIyxKoAqv/0IER0BcCmAUwC2Kadu48cA4AwRbWaMTXKZ6Wy3r9sNC+U6kpYR2mfZMg1YBsHxGLIJ//ge7iksh1H4hZdehBt3j+lmeBqNZs3SF/mIiCaIyOS3d8MPKB/l8tAiEd3Is47eBkB4G/cCeDu//XbleF/wPIaNg43bYIq4gthqczBl4+KJrNwLeSlsyKXwskvGl/w8Go1G0y+W1OaCiN4M4KMAJgB8mYj2M8ZuAfAKAL9PRHUAHoB3M8Zm+cN+GcBdANIAvsJ/AOCPAPwTEb0TwHEAP7uUsbXjd95wBX7nDVc0HE/aJoo1F5lE4EH887tfhrTiUWg0Gs2FypKMAmPsCwC+EHP8HgD3NHnMPgBXxRyfAfCapYxnOZCeQjK4NHqXNI1Gs15YtxXNzRBGIZvQnoFGo1l/aKMQQRSwiZiCRqPRrCe0UYggahWySe0paDSa9Yc2ChGEfJTW8pFGo1mHaKMQQchHWS0faTSadYg2ChGCOgXtKWg0mvWHNgoRgpiC9hQ0Gs36QxuFCEI+0jEFjUazHtFGIUJQp6A9BY1Gs/7QRiGCjiloNJr1jDYKEZK8x5GOKWg0mvWINgoRtKeg0WjWM9ooRNBGQaPRrGe0RhLh9S/cDCJCLqV3R9NoNOsP7SlE2D0xgF951SWrPQyNRqNZFbRR0Gg0Go1EGwWNRqPRSLRR0Gg0Go1EGwWNRqPRSLRR0Gg0Go1EGwWNRqPRSLRR0Gg0Go1EGwWNRqPRSIgxttpjWBJENA3geI8PHwdwbhmHs1ys1XEBa3dselzdsVbHBazdsV1o47qIMTYRPXjeG4WlQET7GGN7V3scUdbquIC1OzY9ru5Yq+MC1u7Y1su4tHyk0Wg0Gok2ChqNRqORrHejcOdqD6AJa3VcwNodmx5Xd6zVcQFrd2zrYlzrOqag0Wg0mjDr3VPQaDQajYI2ChqNRqORrFujQES3EtEhIjpMRO9bxXFsJ6JvEdFTRPQkEf0qP/4BIjpFRPv5z+tXYWzHiOgAf/19/NgoEX2diJ7lv0dWeEyXKddkPxEtEtH/WK3rRUR/R0RniegJ5VjsNSKfj/DP3I+I6CUrPK4/IaKn+Wt/gYiG+fGdRFRWrt0nVnhcTd87Ivotfr0OEdEtKzyuzyljOkZE+/nxlbxezeaH/n3GGGPr7geACeAIgN0AEgAeB3DFKo1lM4CX8Ns5AM8AuALABwD8+ipfp2MAxiPH/hjA+/jt9wH48Cq/j1MALlqt6wXgFQBeAuCJdtcIwOsBfAUAAbgRwEMrPK7XArD47Q8r49qpnrcK1yv2vePfg8cBJAHs4t9Zc6XGFbn/zwD83ipcr2bzQ98+Y+vVU7gewGHG2FHGWA3A3QBuW42BMMYmGWOP8tt5AAcBbF2NsXTIbQA+yW9/EsCbVm8oeA2AI4yxXivalwxj7DsAZiOHm12j2wB8ivk8CGCYiDav1LgYY19jjDn8zwcBbOvHa3c7rhbcBuBuxliVMfYcgMPwv7srOi4iIgA/C+Cz/XjtVrSYH/r2GVuvRmErgBPK3yexBiZiItoJ4BoAD/FD7+Eu4N+ttEzDYQC+RkSPENEd/NhGxtgkvz0FYOMqjEtwO8Jf1NW+XoJm12gtfe5+Ef6KUrCLiB4jogeI6KZVGE/ce7dWrtdNAM4wxp5Vjq349YrMD337jK1Xo7DmIKIBAPcA+B+MsUUAHwdwMYCrAUzCd19Xmpczxl4C4HUAfoWIXqHeyXx/dVVymokoAeCnAPwzP7QWrlcDq3mNmkFE7wfgAPg0PzQJYAdj7BoAvwbgM0Q0uIJDWpPvncJbEV58rPj1ipkfJMv9GVuvRuEUgO3K39v4sVWBiGz4b/inGWP/AgCMsTOMMZcx5gH4G/TJbW4FY+wU/30WwBf4GM4Id5T/PrvS4+K8DsCjjLEzfIyrfr0Uml2jVf/cEdE7ALwBwM/zyQRcnpnhtx+Br91fulJjavHerYXrZQF4C4DPiWMrfb3i5gf08TO2Xo3CwwD2ENEuvuK8HcC9qzEQrlf+LYCDjLE/V46rOuCbATwRfWyfx5Ulopy4DT9I+QT86/R2ftrbAXxpJcelEFq9rfb1itDsGt0L4G08Q+RGAAuKBNB3iOhWAL8J4KcYYyXl+AQRmfz2bgB7ABxdwXE1e+/uBXA7ESWJaBcf1w9XalycmwE8zRg7KQ6s5PVqNj+gn5+xlYigr8Uf+FH6Z+Bb+fev4jheDt/1+xGA/fzn9QD+AcABfvxeAJtXeFy74Wd+PA7gSXGNAIwB+AaAZwHcD2B0Fa5ZFsAMgCHl2KpcL/iGaRJAHb5++85m1wh+Rshf8c/cAQB7V3hch+HrzeJz9gl+7k/z93g/gEcBvHGFx9X0vQPwfn69DgF43UqOix+/C8C7I+eu5PVqNj/07TOm21xoNBqNRrJe5SONRqPRxKCNgkaj0Wgk2ihoNBqNRqKNgkaj0Wgk2ihoNBqNRqKNgkaj0Wgk2ihoNBqNRvL/A/O/IoHl/FBYAAAAAElFTkSuQmCC\n", 187 | "text/plain": [ 188 | "
" 189 | ] 190 | }, 191 | "metadata": { 192 | "needs_background": "light" 193 | }, 194 | "output_type": "display_data" 195 | } 196 | ], 197 | "source": [ 198 | "import gym\n", 199 | "import matplotlib.pyplot as plt\n", 200 | "from IPython.display import clear_output\n", 201 | "\n", 202 | "\n", 203 | "env = gym.make('Pendulum-v0')\n", 204 | "agent = DDPG(state_dim=3, action_dim=1, action_max=2)\n", 205 | "\n", 206 | "episode_n = 200\n", 207 | "session_len = 200\n", 208 | "total_rewards = []\n", 209 | "\n", 210 | "for episode in range(episode_n):\n", 211 | " state = env.reset()\n", 212 | " total_reward = 0\n", 213 | " for _ in range(session_len):\n", 214 | " action = agent.get_action(state)\n", 215 | " next_state, reward, done, _ = env.step(action)\n", 216 | " agent.fit(state, action, reward, done, next_state)\n", 217 | " state = next_state\n", 218 | " total_reward += reward\n", 219 | " total_rewards.append(total_reward)\n", 220 | " plt.plot(total_rewards)\n", 221 | " clear_output(True) \n", 222 | " plt.show()" 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": null, 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [] 231 | } 232 | ], 233 | "metadata": { 234 | "kernelspec": { 235 | "display_name": "Python 3", 236 | "language": "python", 237 | "name": "python3" 238 | }, 239 | "toc": { 240 | "base_numbering": 1, 241 | "nav_menu": {}, 242 | "number_sections": false, 243 | "sideBar": true, 244 | "skip_h1_title": false, 245 | "title_cell": "Table of Contents", 246 | "title_sidebar": "Contents", 247 | "toc_cell": false, 248 | "toc_position": {}, 249 | "toc_section_display": true, 250 | "toc_window_display": false 251 | } 252 | }, 253 | "nbformat": 4, 254 | "nbformat_minor": 4 255 | } 256 | -------------------------------------------------------------------------------- /Coding/Practice_1.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import gym_maze 3 | import time 4 | import numpy as np 5 | 6 | 7 | class Agent: 8 | def __init__(self, state_n, action_n): 9 | self.state_n = state_n 10 | self.action_n = action_n 11 | self.policy = np.ones((state_n, action_n)) / action_n 12 | 13 | def get_action(self, state): 14 | prob = self.policy[state] 15 | action = np.random.choice(np.arange(self.action_n), p=prob) 16 | return int(action) 17 | 18 | def update_policy(self, elite_sessions): 19 | new_policy = np.zeros((self.state_n, self.action_n)) 20 | 21 | for session in elite_sessions: 22 | for state, action in zip(session['states'], session['actions']): 23 | new_policy[state][action] += 1 24 | 25 | for state in range(self.state_n): 26 | if sum(new_policy[state]) == 0: 27 | new_policy[state] += 1 / self.action_n 28 | else: 29 | new_policy[state] /= sum(new_policy[state]) 30 | 31 | self.policy = new_policy 32 | 33 | return None 34 | 35 | 36 | def get_state(obs): 37 | return int(obs[0] * 5 + obs[1]) 38 | 39 | 40 | def get_session(env, agent, session_len, visual=False): 41 | session = {} 42 | states, actions = [], [] 43 | total_reward = 0 44 | 45 | obs = env.reset() 46 | for _ in range(session_len): 47 | state = get_state(obs) 48 | states.append(state) 49 | action = agent.get_action(state) 50 | actions.append(action) 51 | 52 | if visual: 53 | env.render() 54 | time.sleep(1) 55 | 56 | obs, reward, done, _ = env.step(action) 57 | total_reward += reward 58 | 59 | if done: 60 | break 61 | 62 | session['states'] = states 63 | session['actions'] = actions 64 | session['total_reward'] = total_reward 65 | return session 66 | 67 | 68 | def get_elite_sessions(sessions, q_param): 69 | 70 | total_rewards = np.array([session['total_reward'] for session in sessions]) 71 | quantile = np.quantile(total_rewards, q_param) 72 | 73 | elite_sessions = [] 74 | for session in sessions: 75 | if session['total_reward'] > quantile: 76 | elite_sessions.append(session) 77 | 78 | return elite_sessions 79 | 80 | 81 | env = gym.make("maze-sample-5x5-v0") 82 | agent = Agent(25, 4) 83 | 84 | episode_n = 50 85 | session_n = 100 86 | session_len = 100 87 | q_param = 0.9 88 | 89 | for episode in range(episode_n): 90 | sessions = [get_session(env, agent, session_len) for _ in range(session_n)] 91 | 92 | mean_total_reward = np.mean([session['total_reward'] for session in sessions]) 93 | print('mean_total_reward = ', mean_total_reward) 94 | 95 | elite_sessions = get_elite_sessions(sessions, q_param) 96 | 97 | if len(elite_sessions) > 0: 98 | agent.update_policy(elite_sessions) 99 | 100 | get_session(env, agent, session_len, visual=True) 101 | -------------------------------------------------------------------------------- /Homework/Homework_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Homework/Homework_1.pdf -------------------------------------------------------------------------------- /Homework/Homework_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Homework/Homework_2.pdf -------------------------------------------------------------------------------- /Homework/Homework_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Homework/Homework_3.pdf -------------------------------------------------------------------------------- /Homework/Homework_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Homework/Homework_4.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Страничка курса "Обучение с подкреплением и нейронные сети" 2 | 3 | Курс посвящен методам обучения с подкреплением (Reinforcement learning) - одному из способов машинного обучения. В нем будет рассмотрена задача о создании систем, которые могли бы приспосабливаться к окружающей среде, а также обучаться на основе получаемого опыта. Такие задачи возникают во многих областях, включая информатику, технические науки, математику, физику, нейробиологию и когнитологию. В середине 2010-х годов методы обучения с подкреплением удалось эффективно применить для обучения глубоких нейронных сетей, что привело к ряду значимых результатов. В рамках спецкурса будут изложены основные методы обучения с подкреплением, приведены техники их успешного использования для глубоких нейронных сетей, рассмотрены примеры, предложены практические задания. 4 | 5 | ### Лекции 6 | 7 | Лекция 1. Введение в обучение с подкреплением. Метод Cross-Entropy ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_1.pdf)/[видео](https://www.dropbox.com/s/h2lff3q4rhpzue7/Video_1.mp4?dl=0)) 8 | 9 | Лекция 2. Введение в нейронные сети. Deep Cross-Entropy Method ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_2.pdf)/[видео](https://www.dropbox.com/s/th4mdrk1jcq1sgx/Video_2.mp4?dl=0)) 10 | 11 | Лекция 3. Динамическое программирование ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_3.pdf)/[видео](https://www.dropbox.com/s/xipiqohh3zb1o6f/Video_4.mp4?dl=0)) 12 | 13 | Лекция 4. Model-Free Reinforcement Learning ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_4.pdf)/[видео](https://www.dropbox.com/s/max2tig3f13q0cg/Video_6.mp4?dl=0)) 14 | 15 | Лекция 5. Value Function Approximation ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_5.pdf)/[видео](https://www.dropbox.com/s/b9hsy803fsrso7l/Video_8.mp4?dl=0)) 16 | 17 | Лекция 6. Policy Gradient ([слайды](https://github.com/imm-rl-lab/UrFU_course/blob/master/Slides/Lecture_6.pdf)/[видео](https://www.dropbox.com/s/qv7lx0h53kom8ix/Video_10.mp4?dl=0)) 18 | 19 | ### Практики 20 | 21 | Практика 1. Метод Cross-Entropy для решение Maze ([код](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice_1.py)) 22 | 23 | Практика 2. PyTorch и Deep Cross-Entropy ([код 1](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-2_1.py)/[код 2](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-2_2.py)/[код 3](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-2_3.py)/[видео](https://www.dropbox.com/s/r73q2fowgxgz7yc/Video_3.mp4?dl=0)) 24 | 25 | Практика 3. Решение Frozen Lake методами Policy Iteration и Value Iteration ([код](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-3.py)/[видео](https://www.dropbox.com/s/62lo7fgar15qxkd/Video_5.mp4?dl=0)) 26 | 27 | Практика 4. Решение Taxi методами Monte-Carlo, SARSA и Q-Learning ([код](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-4.py)/[видео](https://www.dropbox.com/s/84bfa7ckxw0dm67/Video_7.mp4?dl=0)) 28 | 29 | Практика 5. Решение Cartpole методом DQN ([код](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-5.py)/[видео](https://www.dropbox.com/s/psex7ryc3ekc6cb/Video_9.mp4?dl=0)) 30 | 31 | Практика 6. Решение Pendulum методом DDPG ([код](https://github.com/imm-rl-lab/UrFU_course/blob/master/Coding/Practice-6.ipynb)/[видео](https://www.dropbox.com/s/61dz3igadpzwh22/Video_11.mp4?dl=0)) 32 | 33 | 34 | ### Домашние задания 35 | [Домашнее задание 1](https://github.com/imm-rl-lab/UrFU_course/blob/master/Homework/Homework_1.pdf) 36 | 37 | [Домашнее задание 2](https://github.com/imm-rl-lab/UrFU_course/blob/master/Homework/Homework_2.pdf) 38 | 39 | [Домашнее задание 3](https://github.com/imm-rl-lab/UrFU_course/blob/master/Homework/Homework_3.pdf) 40 | 41 | [Домашнее задание 4](https://github.com/imm-rl-lab/UrFU_course/blob/master/Homework/Homework_4.pdf) 42 | 43 | ### Полезные ссылки 44 | 45 | [https://gym.openai.com/](https://gym.openai.com/) Страничка библиотеки Gym для Python. В ней содержаться многие стандартные Environments для обучения с подкреплением. 46 | 47 | [https://github.com/MattChanTK/gym-maze](https://github.com/MattChanTK/gym-maze) Репозиторий сред c Maze 48 | 49 | [https://pytorch.org/](https://pytorch.org/) Сайт библиотеки PyTorch. 50 | 51 | [https://playground.tensorflow.org/](https://playground.tensorflow.org/) Страничка с хорошей визуализацией обучения нейронных сетей. Просто так :) 52 | 53 | ### Видеолекции других курсов 54 | 55 | [A. Panin. Cross-Entropy Method.](https://ru.coursera.org/lecture/practical-rl/crossentropy-method-TAT8g) Короткая, но понятная лекция по применению метода Cross-Entropy к задачам обучения с подкреплением. 56 | 57 | [D. Silver. Introduction to Reinforcement Learning.](https://www.youtube.com/playlist?list=PLqYmG7hTraZDM-OYHWgPebj2MfCFzFObQ) Курс по Reinforcement Learning в University College London. 58 | 59 | ### Литература 60 | 61 | [Р.С. Саттон, Э.Г. Барто. Обучение с подкреплением (1998).](https://nashol.com/2017091096341/obuchenie-s-podkrepleniem-satton-r-s-barto-e-g-2014.html) Уже ставшая классической монография по обучению с подкреплением. 62 | 63 | [C. Николенко, А. Кадурин, Е. Архангельская. Глубокое обучение. Погружение в мир нейронных сетей (2018).](https://cloud.mail.ru/public/AaZw/UM3d856gy) Пожалуй, единственная книга на русском, в которой последовательно и достаточно полно изложены основные моменты работы с нейронными сетями. Написана простым языком, но при этом включает в себя серьёзный обзор литературы со ссылками на первоисточники. 64 | 65 | [S. Mannor, R. Rubinstein, Y. Gat. The Cross-Entropy method for Fast Policy Search (2003).](https://www.aaai.org/Papers/ICML/2003/ICML03-068.pdf) Статья про использование метода Cross-Entropy для оптимизации Policy в задачах обучения с подкреплением. 66 | 67 | [A. Costa, O. Jones, D. Kroese. Convergence properties of the cross-entropy method for discrete optimization (2007)](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.399.4581&rep=rep1&type=pdf) В статье дается обоснование сходимости метода Cross-Entropy в задачах дискретной оптимизации. Однако, если пространство состояний и действий конечные, а среда детерминирована, то, кажется, задача Reinforcement Learning в рассматриваемую постановку задачи дискретной оптимизации вкладывается. 68 | 69 | [G. Cybenko. Approximation by Superpositions of a Sigmoidal Function (1989).](https://pdfs.semanticscholar.org/05ce/b32839c26c8d2cb38d5529cf7720a68c3fab.pdf) Теорема Цыбенко об аппроксимации непрерывных функций суперпозициями сигмоидальных функций (считай нейронными сетями). 70 | 71 | [V. Mnih at el. Playing Atari with Deep Reinforcement Learning (2013).](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) Статья про алгоритм DQN в приложении к играм Atari. 72 | 73 | [H. Van Hasselt, A. Guez, D. Silver. Deep Reinforcement Learning with Double Q-Learning (2016).](https://arxiv.org/pdf/1509.06461.pdf) Статья про алгоритм Double DQN. 74 | 75 | [S. Gu, T. Lillicrap, I. Sutskever, S. Levine. Continuous Deep Q-Learning with Model-based Acceleration (2016).](http://proceedings.mlr.press/v48/gu16.pdf) Статья про алгоритм Continuous DQN. 76 | 77 | [D. Silver at el. Deterministic Policy Gradient Algorithms David (2014).](http://proceedings.mlr.press/v32/silver14.pdf) Статья, в которой доказывается Deterministic Policy Gradient Theorem и приводится Deterministic Policy Gradient Algorithm. 78 | 79 | [T. Lillicrap at el. Continuous control with deep reinforcement learning (2016)](https://arxiv.org/pdf/1509.02971.pdf) Статья про алгоритм DDPG. 80 | 81 | [V. Mnih at el. Asynchronous Methods for Deep Reinforcement Learning (2016).](https://arxiv.org/pdf/1602.01783.pdf) Статья про асинхронный подход для решения задач Reinforcement Learning. 82 | 83 | 84 | -------------------------------------------------------------------------------- /Slides/Lecture_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_1.pdf -------------------------------------------------------------------------------- /Slides/Lecture_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_2.pdf -------------------------------------------------------------------------------- /Slides/Lecture_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_3.pdf -------------------------------------------------------------------------------- /Slides/Lecture_4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_4.pdf -------------------------------------------------------------------------------- /Slides/Lecture_5.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_5.pdf -------------------------------------------------------------------------------- /Slides/Lecture_6.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/imm-rl-lab/reinforcement_learning_course/efb3066a586fcbccb998951a62f7b3bf900ac402/Slides/Lecture_6.pdf --------------------------------------------------------------------------------