├── README.md ├── agent.py ├── deep_qlearning.ipynb ├── environment.py └── maze_generator ├── maze.npy └── maze_generator.py /README.md: -------------------------------------------------------------------------------- 1 | # Deep Q-learning for maze solving 2 | 3 | A simple implementation of DQN that uses PyTorch and a fully connected neural network to estimate the q-values of each state-action pair. 4 | 5 | The environment is a maze that is randomly generated using a deep-first search algorithm to estimate the Q-values. Four moves are possible for the agent (up, down, left and right), whose objective is to reach a predetermined cell. The agent implements either an epsilon-greedy policy or a softmax behaviour policy with temperature equal to epsilon. After each episode, the starting position is sampled in such a way that at the beginning of the training the agent explores the area surrounding the goal, and as the training goes on it will explore further and further areas of the maze. 6 | 7 | A convolutional neural network is also implemented for completeness. 8 | -------------------------------------------------------------------------------- /agent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.special as sp 3 | import matplotlib.pyplot as plt 4 | import copy 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | import collections 9 | 10 | Transition = collections.namedtuple('Experience', 11 | field_names=['state', 'action', 12 | 'next_state', 'reward', 13 | 'is_game_on']) 14 | 15 | 16 | class Agent: 17 | def __init__(self, maze, memory_buffer, use_softmax = True): 18 | self.env = maze 19 | self.buffer = memory_buffer # this is actually a reference 20 | self.num_act = 4 21 | self.use_softmax = use_softmax 22 | self.total_reward = 0 23 | self.min_reward = -self.env.maze.size 24 | self.isgameon = True 25 | 26 | 27 | def make_a_move(self, net, epsilon, device = 'cuda'): 28 | action = self.select_action(net, epsilon, device) 29 | current_state = self.env.state() 30 | next_state, reward, self.isgameon = self.env.state_update(action) 31 | self.total_reward += reward 32 | 33 | if self.total_reward < self.min_reward: 34 | self.isgameon = False 35 | if not self.isgameon: 36 | self.total_reward = 0 37 | 38 | transition = Transition(current_state, action, 39 | next_state, reward, 40 | self.isgameon) 41 | 42 | self.buffer.push(transition) 43 | 44 | 45 | def select_action(self, net, epsilon, device = 'cuda'): 46 | state = torch.Tensor(self.env.state()).to(device).view(1,-1) 47 | qvalues = net(state).cpu().detach().numpy().squeeze() 48 | 49 | # softmax sampling of the qvalues 50 | if self.use_softmax: 51 | p = sp.softmax(qvalues/epsilon).squeeze() 52 | p /= np.sum(p) 53 | action = np.random.choice(self.num_act, p = p) 54 | 55 | # else choose the best action with probability 1-epsilon 56 | # and with probability epsilon choose at random 57 | else: 58 | if np.random.random() < epsilon: 59 | action = np.random.randint(self.num_act, size=1)[0] 60 | else: 61 | action = np.argmax(qvalues, axis=0) 62 | action = int(action) 63 | 64 | return action 65 | 66 | 67 | def plot_policy_map(self, net, filename, offset): 68 | net.eval() 69 | with torch.no_grad(): 70 | fig, ax = plt.subplots() 71 | ax.imshow(self.env.maze, 'Greys') 72 | 73 | for free_cell in self.env.allowed_states: 74 | self.env.current_position = np.asarray(free_cell) 75 | qvalues = net(torch.Tensor(self.env.state()).view(1,-1).to('cuda')) 76 | action = int(torch.argmax(qvalues).detach().cpu().numpy()) 77 | policy = self.env.directions[action] 78 | 79 | ax.text(free_cell[1]-offset[0], free_cell[0]-offset[1], policy) 80 | ax = plt.gca(); 81 | 82 | plt.xticks([], []) 83 | plt.yticks([], []) 84 | 85 | ax.plot(self.env.goal[1], self.env.goal[0], 86 | 'bs', markersize = 4) 87 | plt.savefig(filename, dpi = 300, bbox_inches = 'tight') 88 | plt.show() 89 | -------------------------------------------------------------------------------- /deep_qlearning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import scipy.special as sp\n", 11 | "\n", 12 | "from IPython.display import display, clear_output\n", 13 | "import matplotlib.pyplot as plt\n", 14 | "import copy\n", 15 | "import time\n", 16 | "import random\n", 17 | "\n", 18 | "import torch\n", 19 | "import torch.nn as nn\n", 20 | "import torch.optim as optim\n", 21 | "import collections" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": {}, 27 | "source": [ 28 | "**Introduce experience replay.**" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "Transition = collections.namedtuple('Experience',\n", 38 | " field_names=['state', 'action',\n", 39 | " 'next_state', 'reward',\n", 40 | " 'is_game_on'])\n", 41 | "\n", 42 | "class ExperienceReplay:\n", 43 | " def __init__(self, capacity):\n", 44 | " self.capacity = capacity\n", 45 | " self.memory = collections.deque(maxlen=capacity)\n", 46 | "\n", 47 | " def __len__(self):\n", 48 | " return len(self.memory)\n", 49 | "\n", 50 | " def push(self, transition):\n", 51 | " self.memory.append(transition)\n", 52 | "\n", 53 | " def sample(self, batch_size, device = 'cuda'):\n", 54 | " indices = np.random.choice(len(self.memory), batch_size, replace = False)\n", 55 | " \n", 56 | " states, actions, next_states, rewards, isgameon = zip(*[self.memory[idx] \n", 57 | " for idx in indices])\n", 58 | " \n", 59 | " return torch.Tensor(states).type(torch.float).to(device), \\\n", 60 | " torch.Tensor(actions).type(torch.long).to(device), \\\n", 61 | " torch.Tensor(next_states).to(device), \\\n", 62 | " torch.Tensor(rewards).to(device), torch.tensor(isgameon).to(device)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "**Networks definition.**" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 3, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "class fc_nn(nn.Module):\n", 79 | " def __init__(self, Ni, Nh1, Nh2, No = 4):\n", 80 | " super().__init__()\n", 81 | " \n", 82 | " self.fc1 = nn.Linear(Ni, Nh1)\n", 83 | " self.fc2 = nn.Linear(Nh1, Nh2)\n", 84 | " self.fc3 = nn.Linear(Nh2, No)\n", 85 | " \n", 86 | " self.act = nn.ReLU()\n", 87 | " \n", 88 | " def forward(self, x, classification = False, additional_out=False):\n", 89 | " x = self.act(self.fc1(x))\n", 90 | " x = self.act(self.fc2(x))\n", 91 | " out = self.fc3(x)\n", 92 | " \n", 93 | " return out" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 4, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "class conv_nn(nn.Module):\n", 103 | " \n", 104 | " channels = [16, 32, 64]\n", 105 | " kernels = [3, 3, 3]\n", 106 | " strides = [1, 1, 1]\n", 107 | " in_channels = 1\n", 108 | " \n", 109 | " def __init__(self, rows, cols, n_act):\n", 110 | " super().__init__()\n", 111 | " self.rows = rows\n", 112 | " self.cols = cols\n", 113 | "\n", 114 | " self.conv = nn.Sequential(nn.Conv2d(in_channels = self.in_channels,\n", 115 | " out_channels = self.channels[0],\n", 116 | " kernel_size = self.kernels[0],\n", 117 | " stride = self.strides[0]),\n", 118 | " nn.ReLU(),\n", 119 | " nn.Conv2d(in_channels = self.channels[0],\n", 120 | " out_channels = self.channels[1],\n", 121 | " kernel_size = self.kernels[1],\n", 122 | " stride = self.strides[1]),\n", 123 | " nn.ReLU()\n", 124 | " )\n", 125 | " \n", 126 | " size_out_conv = self.get_conv_size(rows, cols)\n", 127 | " \n", 128 | " self.linear = nn.Sequential(nn.Linear(size_out_conv, rows*cols*2),\n", 129 | " nn.ReLU(),\n", 130 | " nn.Linear(rows*cols*2, int(rows*cols/2)),\n", 131 | " nn.ReLU(),\n", 132 | " nn.Linear(int(rows*cols/2), n_act),\n", 133 | " )\n", 134 | "\n", 135 | " def forward(self, x):\n", 136 | " x = x.view(len(x), self.in_channels, self.rows, self.cols)\n", 137 | " out_conv = self.conv(x).view(len(x),-1)\n", 138 | " out_lin = self.linear(out_conv)\n", 139 | " return out_lin\n", 140 | " \n", 141 | " def get_conv_size(self, x, y):\n", 142 | " out_conv = self.conv(torch.zeros(1,self.in_channels, x, y))\n", 143 | " return int(np.prod(out_conv.size()))" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 5, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "def Qloss(batch, net, gamma=0.99, device=\"cuda\"):\n", 153 | " states, actions, next_states, rewards, _ = batch\n", 154 | " lbatch = len(states)\n", 155 | " state_action_values = net(states.view(lbatch,-1))\n", 156 | " state_action_values = state_action_values.gather(1, actions.unsqueeze(-1))\n", 157 | " state_action_values = state_action_values.squeeze(-1)\n", 158 | " \n", 159 | " next_state_values = net(next_states.view(lbatch, -1))\n", 160 | " next_state_values = next_state_values.max(1)[0]\n", 161 | " \n", 162 | " next_state_values = next_state_values.detach()\n", 163 | " expected_state_action_values = next_state_values * gamma + rewards\n", 164 | " \n", 165 | " return nn.MSELoss()(state_action_values, expected_state_action_values)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "**Import the maze and define the environment.**" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 6, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "from environment import MazeEnvironment\n", 182 | "\n", 183 | "maze = np.load('maze_generator/maze.npy')\n", 184 | "\n", 185 | "initial_position = [0,0]\n", 186 | "goal = [len(maze)-1, len(maze)-1]\n", 187 | "\n", 188 | "maze_env = MazeEnvironment(maze, initial_position, goal)" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 7, 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "data": { 198 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAADrCAYAAACICmHVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAFBUlEQVR4nO3dQU4jRxiA0XaUIxDhXeYO+P4nMHcgO1DmDs5ylJHNuBmXuz54T2KBMO1yw6dC4ld5dzqdFmB+f2y9AOA6YoUIsUKEWCFCrBAhVoj4c82DHx4eTt++fRu0lK/t+fn56sc+PT0Nue4aW69h1PNvfd2Xl5fl+/fvu3Nf2635P+vhcDgdj8erH8/1druzP5+z1vzM1lx3ja3XMOr5t77u4XBYjsfj2Qv7MxgiPh7rfr8su92Pj/3+hssCfvbxWN/e3v8cuCl/BkOEWCHi47E+Pr7/OXBTq/7P+j+vrzdcBvAr/gyGiI/vrL9Q+mf81v/gX7uGUUatYevX9llel50VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRw8YNZxjhG2HUiNmoc4JmWMOI686w1jVu8TOzs0KEWCFCrBAhVogQK0SIFSLEChFihQixQoRYISJ3uuHWaq+rNpo44ppbr/VW7KwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAhVohYNW74/Pw85A2KR9n6pLxRZj6B75ytTzcc5d6/N3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIGHZg2hozTKOMUJuMGmXE4Waf+X5dYmeFCLFChFghQqwQIVaIECtEiBUixAoRYoUIsULEqnHDp6en5Xg83nwRW4+O1cYdt75fNbUD5i6xs0KEWCFCrBAhVogQK0SIFSLEChFihQixQoRYIWKK92cddQrg1iflle7BWjPcsy2vuSzenxW4QKwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAxxemGo1w74jXDqF/tujOsYWtONwTOEitEiBUixAoRYoUIsUKEWCFCrBAhVogQK0RMcbrhKFuPrs1wD9aYYb0zrOFaTjcEzhIrRIgVIsQKEWKFCLFChFghQqwQIVaIECtEON1wpdqbE289crksrdMNZx53tLNChFghQqwQIVaIECtEiBUixAoRYoUIsUKEWCHC6YbLurXOMDpXurejzHAP7s3OChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiFXjhqPMMMI34vlLr2tZPu8Ji59l7NPOChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiCneTHnUCN8Io56/Nuq39c9hlFEjore4X3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIECtEDHsz5RmMWOsMJxbOMOpX+j2ojYheYmeFCLFChFghQqwQIVaIECtEiBUixAoRYoWIKQ5MG2XEhElpcmdZ5j4A7F4+y3vq2lkhQqwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAx7MC0Gca2rl1DbcyuNvI44v7W7sG1r+twOFz8mp0VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRU5xuOGqEb+uRtBnGGGcY+xxhhrHPe7OzQoRYIUKsECFWiBArRIgVIsQKEWKFCLFChFghIne64VccM/vZDKOJo5RGHu/NzgoRYoUIsUKEWCFCrBAhVogQK0SIFSLEChFihYgpTjccZevxuRlGLrd+o+q1j+UyOytEiBUixAoRYoU72++XZbf78bHfX/d9YoU7e3t7//NLxAoRYoUIscKdPT6+//klq4YigN/3+vqx77OzQsRu5ejav8uy/DNuOfDl/X06nf4694VVsQLb8WcwRIgVIsQKEWKFCLFChFghQqwQIVaIECtE/AfYYY6NVPlPoAAAAABJRU5ErkJggg==\n", 199 | "text/plain": [ 200 | "
" 201 | ] 202 | }, 203 | "metadata": {}, 204 | "output_type": "display_data" 205 | } 206 | ], 207 | "source": [ 208 | "maze_env.draw('maze_20.pdf')" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "**Define the agent and the buffer for experience replay.**" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 9, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "buffer_capacity = 10000\n", 225 | "buffer_start_size = 1000\n", 226 | "memory_buffer = ExperienceReplay(buffer_capacity)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": 10, 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "from agent import Agent\n", 236 | "agent = Agent(maze = maze_env,\n", 237 | " memory_buffer = memory_buffer,\n", 238 | " use_softmax = True\n", 239 | " )" 240 | ] 241 | }, 242 | { 243 | "cell_type": "markdown", 244 | "metadata": {}, 245 | "source": [ 246 | "** Define the network.**" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 82, 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [ 255 | "net = fc_nn(maze.size, maze.size, maze.size, 4)\n", 256 | "optimizer = optim.Adam(net.parameters(), lr=1e-4)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 83, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/plain": [ 267 | "fc_nn(\n", 268 | " (fc1): Linear(in_features=400, out_features=400, bias=True)\n", 269 | " (fc2): Linear(in_features=400, out_features=400, bias=True)\n", 270 | " (fc3): Linear(in_features=400, out_features=4, bias=True)\n", 271 | " (act): ReLU()\n", 272 | ")" 273 | ] 274 | }, 275 | "execution_count": 83, 276 | "metadata": {}, 277 | "output_type": "execute_result" 278 | } 279 | ], 280 | "source": [ 281 | "device = 'cuda'\n", 282 | "batch_size = 24\n", 283 | "gamma = 0.9\n", 284 | "\n", 285 | "net.to(device)" 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "**Define the epsilon profile and plot the resetting probability.**" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": 84, 298 | "metadata": { 299 | "scrolled": false 300 | }, 301 | "outputs": [ 302 | { 303 | "data": { 304 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU5dnH8e9NIAgIyBIQWQQRF9wAU9S64dbiBi5VodrWpeLyUtdaaW21pXZxqRvyqthatyoub7WoKG7YWhUlIIKgaGQpoGLABQUEQu73j+ekGWKWSTInJ5P5fa5rrjnnOSczv5wkc+dsz2PujoiI5K4WSQcQEZFkqRCIiOQ4FQIRkRynQiAikuNUCEREclzLpAPUVdeuXb1v375JxxARySqzZs1a5e4FVS3LukLQt29fioqKko4hIpJVzGxpdct0aEhEJMepEIiI5DgVAhGRHKdCICKS41QIRERynAqBiEiOUyEQEclxWXcfQYM8fz98WAwtWoC1CM8dusLRY8Lylx6Cr9fCd04Py0REckBuFYKXHoQ3pm7Z1nuXikIwZSK8/TL02AH2Gtbo8UREkpBbheDqp8A9epRBWVmYLverR2BUD5j3LxUCEckZsR7/MLPhZrbQzIrNbFwVy/uY2XQze9PM5prZUXHmid40HPbJawmt8iG/dcWyTt2h356hEIiI5IjYCoGZ5QETgSOBgcBoMxtYabVfAg+7+2BgFPC/ceVJ2x4HwYLXoHRT0klERBpFnIeGhgLF7r4IwMwmAyOBBSnrONAhmu4IfBhjnvTsfzxs1RY2rIeWrZJOIyISuzgLQU9gWcr8cmCfSuv8GnjWzH4CtAMOr+qFzGwMMAagT58+GQ+6hUGHhIeISI5I+hrJ0cDd7t4LOAq4z8y+kcndJ7l7obsXFhRU2Z12Zm38GpbMj/99RESagDgLwQqgd8p8r6gt1VnAwwDu/hqwFdA1xkzpuXUsXHpQuKpIRKSZi7MQzAQGmFk/M8snnAyeUmmd/wCHAZjZroRCUBJjpvTsfgB8+Sn8552kk4iIxC62QuDupcBYYBrwDuHqoPlmNt7MRkSrXQqcbWZvAQ8Cp7unXtifkN0PDM9vv5xsDhGRRhDrDWXuPhWYWqntypTpBcD+cWaolx47QOfoxrJjzk06jYhIrJI+Wdw0mcEeB8K8l7e881hEpBnKrS4m6uLky8PVQyIizZwKQXUGDEk6gYhIo9ChoZrMeg6mT046hYhIrFQIavLkbXDXz5NOISISKxWCmgw6FFYugY8WJ51ERCQ2KgQ1GXRoeH5rerI5RERipEJQkz67hjEK5ryYdBIRkdioENTEDPY6BJbMSzqJiEhsdPlobS68A9q2TzqFiEhsVAhq065D7euIiGQxHRpKx19/CbddlHQKEZFYqBCkY9VyeOF+jU8gIs2SCkE6Bh0Ka1bDYp00FpHmR4UgHXtFYxjrfgIRaYZiLQRmNtzMFppZsZmNq2L5jWY2J3q8Z2afx5mn3rr1hu12hDdfSDqJiEjGxXbVkJnlAROBI4DlwEwzmxINRgOAu1+csv5PgMFx5WmwQ0+FtU2zTomINEScl48OBYrdfRGAmU0GRgILqll/NHBVjHka5oe/TjqBiEgs4jw01BNYljK/PGr7BjPbHugHVNmXg5mNMbMiMysqKUlwbHt3+DzB9xcRiUFTOVk8CnjU3TdXtdDdJ7l7obsXFhQUNHK0FL8+Hn4xPLn3FxGJQZyFYAXQO2W+V9RWlVHAgzFmyYydvwXFs7VXICLNSpyFYCYwwMz6mVk+4cN+SuWVzGwXoBPwWoxZMmPv74Tn2c8lm0NEJINiKwTuXgqMBaYB7wAPu/t8MxtvZiNSVh0FTHZ3jytLxuw4BDp0gaJpSScREcmYWDudc/epwNRKbVdWmv91nBkyKi8PhhwBs58NJ47Nkk4kItJg6n20ro67AA4ZHfodystLOo2ISIOpENTVwP2STiAiklFN5fLR7LJoLjx/X9IpREQyQoWgPp67B248G9avTTqJiEiDqRDUx9CjYNMGmKNO6EQk+6kQ1MfuB4ZxjGc8mXQSEZEGUyGoj1b5sPd34Y2nwmWkIiJZTIWgvvY5Br78FD5enHQSEZEGUSGor4NPhkdXQ48dkk4iItIguo+gvlq3STqBiEhGaI+gIeb+C8Z+Cz79OOkkIiL1pkLQEO06wntFMPPppJOIiNSbCkFD7LAndO0Fr+syUhHJXioEDWEG+x4Ds56FjRuSTiMiUi8qBA21zzGw/iuY+1LSSURE6iXWQmBmw81soZkVm9m4atY52cwWmNl8M3sgzjyxGHQo7H88tNk66SQiIvUS2+WjZpYHTASOAJYDM81sirsvSFlnAPBzYH93/8zMusWVJzat28BVf086hYhIvcW5RzAUKHb3Re6+EZgMjKy0ztnARHf/DMDdP4kxT7xWrQgPEZEsE2ch6AksS5lfHrWl2gnYycxeMbMZZja8qhcyszFmVmRmRSUlJTHFbYD1a+FH/eHvNyadRESkzpI+WdwSGAAMA0YDd5rZNpVXcvdJ7l7o7oUFBQWNHDENbdrBXofAK4+pEzoRyTpxFoIVQO+U+V5RW6rlwBR33+Tui4H3CIUh+xxwAny0CBbPSzqJiEidxFkIZgIDzKyfmeUDo4ApldZ5nLA3gJl1JRwqWhRjpvjsNyLcV/BvnTgWkewSWyFw91JgLDANeAd42N3nm9l4MxsRrTYNWG1mC4DpwGXuvjquTLHq1B12OwBefTzpJCIidWKeZce0CwsLvaioKOkYVVs0Fzp2hS7bJZ1ERGQLZjbL3QurWqZuqDNphz2TTiAiUmdJXzXU/Mx4EiZekHQKEZG0qRBk2rJ34R8T4MMPkk4iIpIWFYJMO/jk8PzPh5LNISKSJhWCTOvWBwZ+G16anHQSEZG0qBDEYdiocGPZ0gW1rysikjAVgjgcdBIM2Bu+/DTpJCIitdLlo3HovC1MbKL3OoiIVKI9gjit/wrWZOeN0iKSO1QI4vL1OhjdEx65LukkIiI1UiGIy1ZtK64eKitLOo2ISLVUCOJ02Gmwcim8/e+kk4iIVEuFIE7fPi4Mav/cPUknERGplgpBnNq0gwO/By8/AhvWJ51GRKRKKgRxO2Uc3PBvaN0m6SQiIlWKtRCY2XAzW2hmxWY2rorlp5tZiZnNiR4/jjNPInrvrO6pRaRJi60QmFkeMBE4EhgIjDazgVWs+pC7D4oef44rT6KWvwfX/ABWf5h0EhGRb4hzj2AoUOzui9x9IzAZGBnj+zVtL9wPLz6QdAoRkW+IsxD0BJalzC+P2io70czmmtmjZta7qhcyszFmVmRmRSUlJXFkjVevnWDXfcPVQ1k2NKiINH9Jnyx+Aujr7nsCzwFVXmfp7pPcvdDdCwsKCho1YMYc/kNY8ja8PzvpJCIiW4izEKwAUv/D7xW1/Ze7r3b3DdHsn4G9Y8yTrGGjIH8reKZ5ngYRkewVZyGYCQwws35mlg+MAqakrmBmPVJmRwDvxJgnWe07wbHnQ+ftkk4iIrKF2LqhdvdSMxsLTAPygLvcfb6ZjQeK3H0KcIGZjQBKgU+B0+PK0ySc86ekE4iIfIN5lp28LCws9KKiLO7rf/NmeHcG7LZ/0klEJIeY2Sx3L6xqWdIni3PP1Elw8QFhKEsRkSZAhaCxHXQStMqHqXcmnUREBEizEJjZCWb2vpl9YWZrzOxLM1sTd7hmqWNX2P8EeOE+dUQnIk1CunsE1wIj3L2ju3dw9/bu3iHOYM3aUWPgq8/h5UeTTiIiknYhWOnuzffSzsa217Bwt/GrjyedREQk7ctHi8zsIeBxoPwGMNz977Gkau7M4HfPQLcqe9QQEWlU6RaCDsA64DspbQ6oENRXj37huawMWuicvYgkJ61C4O5nxB0kJ70xFSacD7e8AZ26JZ1GRHJUulcN9TKzx8zsk+jxf2bWK+5wzd62O4TB7Z/WpaQikpx0j0n8ldBP0HbR44moTRqizy4w5Ah48jYo3ZR0GhHJUekWggJ3/6u7l0aPu4Es7Q+6iRn5E1i1Al55LOkkIpKj0i0Eq83sNDPLix6nAavjDJYzhh4F2/aDf0xIOomI5Kh0rxo6E5gA3Ei4WuhVQCeQMyEvD867Cdp1TDqJiOSodK8aWkoYL0DisJ82rYgkp8ZCYGYTCHsAVXL3CzKeKFd9thIeuBqOuwB6Dkg6jYjkkNr2CBrU8b+ZDQduJgxM82d3/2M1650IPAp8y92zeLCBBnAPXVRvLoULbks6jYjkkBoLgbtXOZh8OswsD5gIHAEsB2aa2RR3X1BpvfbAhcDr9X2vZqHztmGA+2fvhh/8RjeYiUijqfGqITO7KXp+wsymVH7U8tpDgWJ3X+TuG4HJwMgq1vstcA3wdT3yNy/f+yls2gBTbk06iYjkkNoODd0XPV9fj9fuCSxLmV8O7JO6gpkNAXq7+1Nmdll1L2RmY4AxAH369KlHlCzRe2fYdwRMmQgnXw5t2iWdSERyQG2HhmZFz/8sbzOzToQP77kNeWMzawHcQBoD1rv7JGAShDGLG/K+Td4pl0P7TrBhnQqBiDSKtC4fNbOXCJePtgRmAZ+Y2SvufkkNX7YCSO1nuVfUVq49sDvwkpkBbAtMMbMROXvCGGDgfuEhItJI0r2zuKO7rwFOAO51932Aw2v5mpnAADPrZ2b5wChCf0UAuPsX7t7V3fu6e19gBmEUtNwtAqneK4IZTyadQkRyQLqFoKWZ9QBOBtL6dHL3UmAsMA14B3jY3eeb2Xgz0x1UtZn0U7j5HNioc+giEq90C8F4wgf6B+4+08x2AN6v7Yvcfaq77+Tu/d39d1Hble7+jSuO3H2Y9gZSnHolrP4QnvlL0klEpJkz9+w691pYWOhFRTlQL9zh0oPg4yVwdzHkt046kYhkMTOb5e6FVS1Ld2CaHaJ7CUqigWn+Ee0VSFzMwl7BquXwrIZ+EJH4pHto6AHgYaAHYWCaR4AH4wolkSGHQ+F3NWiNiMQq3W6o27r7fSnz99d0A5hkiBn87unwLCISk3T3CJ42s3Fm1tfMtjeznwFTzayzmXWOM2DOMwvnC155HL78LOk0ItIMpbtHcHL0fE6l9lGEbqp1viBOy96F3xwPo34OZ/4+6TQi0sykOzBNv7iDSA367AqHfB8euymMcdylR9KJRKQZqa330Z+lTJ9UaZn+NW1MPxofThr/7bdJJxGRZqa2cwSjUqZ/XmnZ8AxnkZps1x+OGgNP3wnLFiadRkSakdoKgVUzXdW8xO20q6DXzuGOYxGRDKntHIFXM13VvMStUzeYNE+Xk4pIRtW2R7CXma0xsy+BPaPp8vk9GiGfVGYGGzfAk3foRjMRyYjaBqbJa6wgUgdzXoBbzoXSjXDcT5JOIyJZLt0byqQp+daRMPgwuO8qWPNp0mlEJMupEGQjMzj3Rlj7Bdz9y6TTiEiWUyHIVv32CDeXPXU7vPtG0mlEJIvFWgjMbLiZLTSzYjMbV8Xyc81snpnNMbN/m9nAOPM0Oz8cD0OPhlb5SScRkSwW28A0ZpYHvAccASwnjGE82t0XpKzTIRoLmWj4yvPdvcYb1XJmYBoRkQxq8MA09TQUKHb3Re6+EZgMjExdobwIRNqhexPqZ82nMOF/4JNlSScRkSwUZyHoCaR+Mi2P2rZgZv9jZh8A1wIXVPVCZjbGzIrMrKikpCSWsFlt7RdhFLPbLkw6iYhkocRPFrv7RHfvD1wOVHkJjLtPcvdCdy8sKCho3IDZoEe/MKzlK4/Bvx5NOo2IZJk4C8EKoHfKfK+orTqTgeNizNO8nfRTGLA3TDgfPtdek4ikL85CMBMYYGb9zCyf0JPplNQVzGxAyuzRwPsx5mne8lrCZffAui/gTo0iKiLpS3eEsjpz91IzGwtMA/KAu9x9vpmNB4rcfQow1swOBzYBnwE/iitPTui7G1x+P+yyb9JJRCSLxHb5aFx0+Wia3OHrddCmXdJJRKQJqOny0dj2CCRB7nBVdKXub/6hbqtFpEaJXzUkMTCDQYfCjCfgif9NOo2INHEqBM3V8ReGXkrvuBQWz0s6jYg0YSoEzZUZ/PRu2Hob+P1o2LA+6UQi0kSpEDRnnbrBZffCujWwcknSaUSkidLJ4uau8Dvw1/cgf6ukk4hIE6U9glyQvxVsLoW7fgFLF9S+vojkFBWCXLFmNUy7C35zPKxdU/v6IpIzVAhyRafucMXD8OEHcP3p4V4DERFUCHLLngfB2deFXkofuibpNCLSRKgQ5JoTLoJho+CBq+GzlUmnEZEmQFcN5RozuPQuWL4wHC4SkZynPYJc1LoN9B8Upp+/H1bVNEyEiDR3KgS57NOPYcJ5cOUIWL826TQikhAVglzWeVv4xWRYNAeuPglKNyWdSEQSEGshMLPhZrbQzIrNbFwVyy8xswVmNtfMXjCz7ePMI1XY52i44HaY+TRcdzqUlSWdSEQaWWyFwMzygInAkcBAYLSZDay02ptAobvvCTwKXBtXHqnBUWfDGb+H6Q/AnBeTTiMijSzOPYKhQLG7L3L3jYTB6UemruDu0919XTQ7gzDAvSRh1Di4+TUYcnjSSUSkkcVZCHoCy1Lml0dt1TkLeLqqBWY2xsyKzKyopKQkgxHlv8xg12is43kvw+MTks0jIo2mSdxHYGanAYXAwVUtd/dJwCQIYxY3YrTcNO0uePbu0FHdiRcnnUZEYhZnIVgB9E6Z7xW1bcHMDgeuAA529w0x5pF0XTQJ1n0Jd1wS9hROuCjpRCISozgPDc0EBphZPzPLB0YBU1JXMLPBwB3ACHf/JMYsUhctW8EvHoQDToTbL4a/35R0IhGJUWyFwN1LgbHANOAd4GF3n29m481sRLTadcDWwCNmNsfMplTzctLY/lsMToCFM9VbqUgzZp5lf+CFhYVeVFSUdIzcsbk0FIGWreDzEujQBVroPkSRbGNms9y9sKpl+ouWmuW1DEVg/VdwyQFwzWm6A1mkmVEhkPRs1Q6OOB2mPwhXHqtRzkSaERUCSY8ZjP45XHwnzH4eLt4fPl6SdCoRyQAVAqmbI38Mv38GSpbBxJ8knUZEMqBJ3FAmWWbI4XDzDNh6mzBfuimcRxCRrKQ9AqmfPruEbqw3bw7nDG67CDZtTDqViNSDCoE0jJdB713hsZvhskM02plIFlIhkIZp2QrOuzEa4OYtOH8IvKmurEWyiQqBZMawU2DCG9C+M9xwpg4TiWQRnSyWzNl+INw6E1YuhVb5oRh8shR6Dkg6mYjUQHsEkllttoa+u4XpB38P5+4FT9ymvopEmjAVAonP0WNg9wNhwvnws0NhRXHSiUSkCioEEp8u24Wbzy6aBO/PhnP2gBcfSDqViFSiQiDxMoOjzoa/vAP7HAN9dw/tZWXJ5hKR/9LJYmkcXbaDXz1SMX/96ZDXCs78PXTqnlgsEYl5j8DMhpvZQjMrNrNxVSw/yMxmm1mpmX0vzizShLhD5x7wwn1wxk5hBDR1bS2SmNgKgZnlAROBI4GBwGgzG1hptf8ApwM6cJxLzODH18Ad82Dgt8NwmGP2gPdmJZ1MJCfFuUcwFCh290XuvhGYDIxMXcHdl7j7XEAHjHNR753hd1Nh/BPQriMU9ArtGutApFHFWQh6AstS5pdHbXVmZmPMrMjMikpKSjISTpoIM9j3GJjwejhX4A5XHAk/Owzmv5J0OpGckBVXDbn7JHcvdPfCgoKCpONInMrK4OBTYPE8uPgAuPRgKJqmG9JEYhRnIVgB9E6Z7xW1iVQvLw+OvwDuXQzn3QQffQC/GA4v3J90MpFmK85CMBMYYGb9zCwfGAVMifH9pDlp0w6OvxDu/gAuvQsOODG0T58Mj94AX32ebD6RZiS2QuDupcBYYBrwDvCwu883s/FmNgLAzL5lZsuBk4A7zGx+XHkkS+W3hu+eAVu1DfOzpsGkS2F0T7jlfPjPO8nmE2kGzLPs2GthYaEXFRUlHUOSVPwm/GNC6K5i0wY44WI494akU4k0aWY2y90Lq1qWFSeLRbaw4+BwuOhvy+DMP8CgQ0P7ZyvhxrNh/qs6uSxSB+piQrLXNgUwKuWG9fdnw/QH4ek/w3b9w9VHw0aF/o3Mkssp0sRpj0Caj6FHwuSP4JK/QPd+8NAfw3gIn60My79el2w+kSZKewTSvLRtD8PPDI/PPoH5/4bO24Zl40+EjxeFXlD3PRZ22z+MuSyS47RHIM1Xp25wwAkV898+Drr3hSm3wmWHwMnd4L7fJBZPpKnQHoHkjmPOCY91X8Ls5+D1J6Fj17Bs/Vdw6UGwx8Ew+DDY8+CwdyGSA1QIJPe0bR/2FFL3Fj4vgfZd4Knb4bGboEUe7DIUzr4edvt2uApJJ5ylmVIhEAHo0Q+ueQ42fh0uP53zAsx5EVq3Ccv/+TDce2XoNrv80XuX0CWGSJZTIRBJlb8VDD40PFJ16AK9dw2Hk569O7S1bgv3LQmXsS5+G8pKoc9AaJXf2KlFGkSFQCQdQw4PD3f4sBjemQFLF1ScY5j8B5j+QCgC2+8O/faA/oPghIuSzS2SBnUxIZIJHy2GhW9A8exwY9vS+aFI3DE3LP/VsfDZx2GPoc+u0HMAbB9NizSCmrqY0B6BSCb06Bcew06paNuwvmJ6p8Iw0M6bz8Pz94a2bx0ZRmgD+O1J4VDTdv1hux2h+/bhuVP3xvseJGepEIjEpfxEM8APrqqYXrsmjLNQfhWSe+hW+93XK4oEwNHnwIW3w+bNcNkw6NITCnpDtz7hecfBoWCINJAKgUhja9chfIiXMwtXLEHYi/h4MaxcWnFH9PovIa8VvD8LXn089LgKcPrV8P0roGR56EqjU3fotG3F87BTYNd9Yf3acKiqQ5fwaNsBWuheUqmgQiDSlLRuE84dbD+wom3rbeC6F8O0e7jnoeQ/sE230NYiDw4ZDZ9+HM5DvFcUnvsPCoVgyTy4cL+K12uRB+07wSV3wX7HhmFBH7k+FIn2ncOyth1gyBGhGK39Irxn2w6hiOVv1XjbQxpFrIXAzIYDNwN5wJ/d/Y+VlrcG7gX2BlYDp7j7kjgziWQ1s9B1RqduFW1desDYW7+5bllZeO61M4x/Atashi8/jZ5XQ7doJNnPP4G5L4Vl67+q+PrrXwqF4PWn4I+nVrS3yg9F4Y/PQ/+9wvInbw+DB23VLpzraN0WTv5ZOGH+wVvwwZuhLXX5gCGhr6f1a6F0YygwrVprbyUBsRUCM8sDJgJHAMuBmWY2xd0XpKx2FvCZu+9oZqOAa4BTvvlqIlJn5R+o7TvBvsdUv97gw+D+pWF644awB7BuDXTZLrTtuh/87N7QtnZNeF63Jtw/AfD1Wli9IhzW+notbFgXHiPHhuUznoB7fvXN932kJBSKB38XLr8tl9cyFIRHV4cR6u4fDy9NDm3ljzZbw9VPhfWn3hku523VOhSpvFbQriOc+suw/N+PhXMyeS1D4clrFbbJQSeF5fNeDkUwr2VYltcyfP1Oe4flK96HTRtDe4sWYY+qdduKQ3dfrAp7auXLrEXIUb7nVFYWCngTvjM9zj2CoUCxuy8CMLPJwEggtRCMBH4dTT8K3Gpm5tl2TatIc5HfGvIr7XGUXxFVnYNPDo/qHH8hHHZaRZEof956m7B8n6OhY0E495H6KL8xr2sv2H63LZelXpG1dH4YwnTj11C6CTZvCt2FlBeC5+6G1yoNl75d/4pCcO9V8Nb0LZf3HwS3vRmm//D9cLgt1e4Hwg3/CtOXHAjL3t1yeeoVYaf1gVUrQiFokRcKxkEnw+X3heU/6Bv2xMqXtciDQ74PZ18blp+1ayg0+xwN5/yp+u3cAHEWgp7AspT55cA+1a3j7qVm9gXQBViVupKZjQHGAPTp0yeuvCISh7bta+7Ab7f9w6M65d2KV+e8m8KjOr98JBx6Kt1UUShSXXxn2MMp3QSbS8Py/JQrvs66BtasgrLN4Qquss0Ve0MAp10Z9ig2bwYvC8u7961YfuKlYS+rLGV53z0qlh94UiiMXlbx+qnniHYcHPYqusX32RfbDWVm9j1guLv/OJr/AbCPu49NWeftaJ3l0fwH0TqrqnpN0A1lIiL1kdSYxSuA3inzvaK2Ktcxs5ZAR8JJYxERaSRxFoKZwAAz62dm+cAooNKBOqYAP4qmvwe8qPMDIiKNK7ZzBNEx/7HANMLlo3e5+3wzGw8UufsU4C/AfWZWDHxKKBYiItKIYr2PwN2nAlMrtV2ZMv01cFKcGUREpGa6c0NEJMepEIiI5DgVAhGRHKdCICKS47JuhDIzKwGW1vPLu1LpruUmQrnqRrnqrqlmU666aUiu7d29oKoFWVcIGsLMiqq7sy5JylU3ylV3TTWbctVNXLl0aEhEJMepEIiI5LhcKwSTkg5QDeWqG+Wqu6aaTbnqJpZcOXWOQEREvinX9ghERKQSFQIRkRyXM4XAzIab2UIzKzazcTG/V28zm25mC8xsvpldGLX/2sxWmNmc6HFUytf8PMq20My+G2duM1tiZvOiDEVRW2cze87M3o+eO0XtZma3RO8/18yGpLzOj6L13zezH1X3fmlm2jllu8wxszVmdlES28zM7jKzT6KBk8rbMrZ9zGzvaPsXR1+b1mC21eS6zszejd77MTPbJmrva2brU7bb7bW9f3XfYz1zZeznZqEr+9ej9ocsdGtf31wPpWRaYmZzEthe1X0+JPc75u7N/kHoBvsDYAcgH3gLGBjj+/UAhkTT7YH3gIGE8Zl/WsX6A6NMrYF+Uda8uHIDS4CuldquBcZF0+OAa6Lpo4CnAQP2BV6P2jsDi6LnTtF0pwz+vD4Gtk9imwEHAUOAt+PYPsAb0boWfe2RDcj1HaBlNH1NSq6+qetVep0q37+677GeuTL2cwMeBkZF07cD59U3V6XlfwKuTGB7Vff5kNjvWK7sEQwFit19kbtvBCYDI+N6M3f/yN1nR9NfAu8QxmeuzkhgsrtvcPfFQHGUuTFzjwTuiabvAY5Lab/XgxnANmbWA/gu8Jy7f+runwHPAcMzlOUw4AN3r+kO8ti2mbv/izA+RuX3a/D2iZZ1cPcZHv5i7015rTrncvdn3b00mp1BGAmwWrW8f3XfY5rWTEkAAATnSURBVJ1z1aBOP7foP9lDgUczmSt63ZOBB2t6jZi2V3WfD4n9juVKIegJLEuZX07NH8wZY2Z9gcHA61HT2Gj37q6UXcnq8sWV24FnzWyWmY2J2rq7+0fR9MdA94SyQRigKPUPtClss0xtn57RdKbzAZxJ+O+vXD8ze9PM/mlmB6bkre79q/se6ysTP7cuwOcpxS5T2+tAYKW7v5/S1ujbq9LnQ2K/Y7lSCBJhZlsD/wdc5O5rgNuA/sAg4CPCrmkSDnD3IcCRwP+Y2UGpC6P/IhK5rjg6/jsCeCRqairb7L+S3D7VMbMrgFLgb1HTR0Afdx8MXAI8YGYd0n29DHyPTe7nVslotvxno9G3VxWfDw16vYbIlUKwAuidMt8raouNmbUi/JD/5u5/B3D3le6+2d3LgDsJu8M15Yslt7uviJ4/AR6LcqyMdinLd4c/SSIboTjNdveVUcYmsc3I3PZZwZaHbxqcz8xOB44BTo0+QIgOvayOpmcRjr/vVMv7V/c91lkGf26rCYdCWlZqr7fotU4AHkrJ26jbq6rPhxpeL/7fsXRObmT7gzAk5yLCyanyE1G7xfh+Rjgud1Ol9h4p0xcTjpUC7MaWJ9AWEU6eZTw30A5onzL9KuHY/nVseaLq2mj6aLY8UfWGV5yoWkw4SdUpmu6cgW03GTgj6W1GpZOHmdw+fPNE3lENyDUcWAAUVFqvAMiLpncgfBDU+P7VfY/1zJWxnxth7zD1ZPH59c2Vss3+mdT2ovrPh8R+x2L5IGyKD8KZ9/cIlf6KmN/rAMJu3VxgTvQ4CrgPmBe1T6n0x3JFlG0hKWf4M507+iV/K3rML39NwrHYF4D3gedTfqEMmBi9/zygMOW1ziSc7Csm5cO7AdnaEf4D7JjS1ujbjHDI4CNgE+H46lmZ3D5AIfB29DW3Et3hX89cxYTjxOW/Z7dH654Y/XznALOBY2t7/+q+x3rmytjPLfqdfSP6Xh8BWtc3V9R+N3BupXUbc3tV9/mQ2O+YupgQEclxuXKOQEREqqFCICKS41QIRERynAqBiEiOUyEQEclxKgQiETPbbFv2gJqxXmqj3i3frn1NkcbXsvZVRHLGencflHQIkcamPQKRWkT91l8b9e/+hpntGLX3NbMXo47VXjCzPlF7dwtjA7wVPb4dvVSemd0Z9UH/rJm1ida/IOqbfq6ZTU7o25QcpkIgUqFNpUNDp6Qs+8Ld9yDcpXlT1DYBuMfd9yR09nZL1H4LoQuDvQj94c+P2gcAE919N+Bzwt2sELoTGBy9zrlxfXMi1dGdxSIRM/vK3beuon0JcKi7L4o6C/vY3buY2SpC1wmbovaP3L2rmZUAvdx9Q8pr9CX0HT8gmr8caOXuV5vZM8BXwOPA4+7+VczfqsgWtEcgkh6vZrouNqRMb6biHN3RhL5khgAzU3raFGkUKgQi6Tkl5fm1aPpVwiA6AKcCL0fTLwDnAZhZnpl1rO5FzawF0NvdpwOXAx2Bb+yViMRJ/3mIVGhj0WDmkWfcvfwS0k5mNpfwX/3oqO0nwF/N7DKgBDgjar8QmGRmZxH+8z+P0AtmVfKA+6NiYcAt7v55xr4jkTToHIFILaJzBIXuvirpLCJx0KEhEZEcpz0CEZEcpz0CEZEcp0IgIpLjVAhERHKcCoGISI5TIRARyXH/D4n8Iw0tmc4LAAAAAElFTkSuQmCC\n", 305 | "text/plain": [ 306 | "
" 307 | ] 308 | }, 309 | "metadata": {}, 310 | "output_type": "display_data" 311 | }, 312 | { 313 | "data": { 314 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU1f34/9d7JvseIOw7IoiBRAhLVUSKIlbrUm2rxQWrUm2t3X5W+6kfa639dLHf1rp83LW2olL8VKUVtSKgUlEJiAuo7EJYA7IGQpLJ+/fHmUwmeyaZLcn7+XjMI3ebe99zk9z3nHPuOVdUFWOMMSaYJ9YBGGOMiT+WHIwxxjRgycEYY0wDlhyMMcY0YMnBGGNMAwmxDiAcevTooYMHD451GMYY06GsWLFij6rmNbauUySHwYMHU1xcHOswjDGmQxGRz5taZ9VKxhhjGrDkYIwxpgFLDsYYYxroFG0OxoSqsrKSkpISysvLYx2KMRGXkpJC//79SUxMbPV7LDmYLqmkpITMzEwGDx6MiMQ6HGMiRlXZu3cvJSUlDBkypNXvs2ol0yWVl5fTvXt3Swym0xMRunfvHnIp2ZKD6bIsMZiuoi1/61attOVT2LcTRABxP70JMOpLbn3JWjhyEIYVuuXGGNMFWMlh21r4YAmsWgyrFsH7r7v5Ghs/gHf+BVs+iVWEppPyer0UFhYGXr/97W9D3kdxcTE33ngjAH/5y1+44YYbwh1mm3z66acUFhZy0kknsWHDBk4++WQANm/eTH5+fkj7Onr0KFOmTMHn84X0voyMjEaX33bbbSxcuBCAu+++myNHjoS032i65pprWLNmDQD/8z//0+L2l1xyCevWrQvPwVW1w7/GjRun7VJd7V4+n2pVlWpVZe26imOqT/+P6rJ/tu8YJq6sWbMm1iFoenp6WPf3xBNP6Pe+972w7rM5VVVVTa77zW9+o7/61a8aLN+0aZOeeOKJIR3nvvvu07vvvrvV21dXV6vP52vV+R00aJCWlpY2uq65zxessrKy5Y3CoDWfZ8mSJXrNNdc0uq6xv3mgWJu4rlrJAVxVkgh4POD11q0+SkyCngNhd5O9zI0Jq8GDB/PTn/6U0aNHM2HCBNavXw/AvHnzyM/Pp6CggNNOOw2AJUuWcO655zbYx+bNm/nyl7/MmDFjmDZtGlu2bAFg1qxZ3HjjjZx88skMHTqU5557rtH3jhw5kpkzZ3LCCSdw8cUXB75dDx48mJtvvpmxY8cyb948Vq1axaRJkxgzZgwXXngh+/btY8GCBdx999088MADTJ06FWj8W7zP5+Omm25i/PjxjBkzhoceeqjR8zFnzhzOP/98AA4fPsy0adMYO3Yso0eP5sUXXwzEPGLECK644gry8/PZunUrAD/60Y848cQTmTZtGqWlpYFz8Nxzz3HPPfewfft2pk6dWifOn/zkJxQUFLBs2TLuuOMOxo8fT35+PrNnz0b9T848/fTT+eEPf0hRURG//vWvGTJkCJWVlQAcPHiwznyNmuPWqDknS5Ys4fTTT+fiiy8OnPfg4xQXF3PLLbdw9OhRCgsLmTlzJmVlZZxzzjkUFBSQn5/P3LlzAZg8eTILFy6kqqqq0XMZkqayRkd6tbvk0JIP31R98heqR8siexwTNQ2+Rb36RMPXp++5dZUVja9f/75bf7Ss4bpW8Hg8WlBQEHg9++yzquq+zd55552qqvrkk0/qOeeco6qq+fn5WlJSoqqq+/btU1XVxYsXB9YHlxzOPfdc/ctf/qKqqo899pief/75qqp65ZVX6sUXX6w+n09Xr16tw4YNaxDXpk2bFNClS5eqqupVV12ld911VyC23/3ud4FtR48erUuWLFFV1f/+7//WH/zgB6qq+otf/CLwHtXab73BJYeHHnooULooLy/XcePG6caNG+vEcuzYMe3Vq1dgvrKyUg8cOKCqqqWlpTps2DCtrq7WTZs2qYjosmXLAtsC+tRTT6mq6i9/+cvAubnyyit13rx5gc8TXHIAdO7cuYH5vXv3BqYvu+wynT9/vqqqTpkyRa+//vrAulmzZunzzz8f+Fw//vGPG5zX4OMGn5PFixdrVlaWbt26VX0+n06aNEnfeuutwHGWL19eZ3tV1eeee65OCWH//v2B6TPOOEOLi4sbHN9KDpHQb7hroK4Orc7TmOakpqayatWqwOub3/xmYN2ll14a+Lls2TIATjnlFGbNmsUjjzzSYv37smXL+Na3vgXA5ZdfztKlSwPrLrjgAjweD6NGjWLXrl2Nvn/AgAGccsopAFx22WV13l8T54EDB9i/fz9TpkwB4Morr+TNN99s9ef/97//zV//+lcKCwuZOHEie/fubVBfvmfPHnJycgLzqsp//dd/MWbMGM444wy2bdsW+AyDBg1i0qRJgW09Hk8g1vqfoSler5eLLrooML948WImTpzI6NGjWbRoEatXr25wHsC1DTzxxBMAPPHEE1x11VWtPg8AEyZMoH///ng8HgoLC9m8eXOz248ePZrXXnuNm2++mbfeeovs7OzAup49e7J9+/aQjt8Yu/2mNbr1di/TeU2f1fS6hMTm16ekNb++DYJvPayZfvDBB3n33Xd56aWXGDduHCtWrGjTvpOTkwPT6q++aO749efT09PbdNz6VJV7772Xs846q8ltUlNT69yfP2fOHEpLS1mxYgWJiYkMHjw4sL6luFpzO2dKSgperxdwfWG++93vUlxczIABA7j99tvrxBJ8vFNOOYXNmzezZMkSfD5fo43uCQkJVFdXA1BdXU1FRUVgXfDvxOv1tlgtdPzxx7Ny5UoWLFjArbfeyrRp07jtttsCcaemprb4WVsS9ZKDiMwQkc9EZL2I3NLENt8QkTUislpEno52jI3y+WBf49+yjAm3mjrkuXPn8qUvuduqN2zYwMSJE7njjjvIy8sL1Ks35uSTT+bZZ58F3AV18uTJIR1/y5YtgRLL008/zamnntpgm+zsbHJzc3nrrbcA+Nvf/hYoRbTGWWedxQMPPBCom1+7di1lZWV1tsnNzcXn8wUuygcOHKBnz54kJiayePFiPv+86bbA6urqQB1/U58hMzOTQ4cONfr+mmP26NGDw4cPN9o+E+yKK67gW9/6VpOlhsGDBwcS+vz58xu0SbQkMTEx8J7t27eTlpbGZZddxk033cTKlSsD261duzbkO8IaE9WSg4h4gfuBM4ESYLmIzFfVNUHbDAd+BpyiqvtEpGc0Y2zSR2/Cx2/BN29xjdTGtFNNA2ONGTNmBG5n3bdvH2PGjCE5OZlnnnkGgJtuuol169ahqkybNo2CggLeeOONRvd97733ctVVV3HXXXeRl5cXqPJorREjRnD//ffz7W9/m1GjRnH99dc3ut2TTz7Jddddx5EjRxg6dGhIx7nmmmvYvHkzY8eORVXJy8vjhRdeaLDd9OnTWbp0KWeccQYzZ87kq1/9KqNHj6aoqIiRI0c2uf/09HTee+897rzzTnr27BlIuMFmz57NjBkz6Nu3L4sXL66zLicnh2uvvZb8/Hx69+7N+PHjm/08M2fO5NZbbw1UCdZ37bXXcv7551NQUMCMGTNCLoHNnj2bMWPGMHbsWK644gpuuukmPB4PiYmJPPDAAwDs2rWL1NRUevcOQ01HU40RkXgBXwJeDZr/GfCzetv8HrgmlP1GvEFaVXXbetcovW195I9lIi4ebmVtSnO3V0ZDW243jaQVK1boZZddFuswWjRv3ryYx/nHP/5RH3300UbXhdogHe02h35AcFm4BJhYb5vjAUTkP4AXuF1VX4lOeM3o0d/d7rp7C/QdFutojOkyxo4dy9SpU/H5fIH2gHjz/e9/n5dffpkFCxbENI6cnBwuv/zysOwrHhukE4DhwOlAf+BNERmtqvuDNxKR2cBsgIEDB0Y+qqRk6NbH+juYiGvpTpVIGzx4MB9//HFMY6jv29/+dqxDaNa9994b6xAAQr5LqjnRbpDeBgwImu/vXxasBJivqpWquglYi0sWdajqw6papKpFeXmNPh87/HoNgtIS8IWhg4kxxsSxaCeH5cBwERkiIknAJcD8etu8gCs1ICI9cNVMG6MZZJOOGwtfngli3UOMMZ1bVKuVVLVKRG4AXsW1JzyuqqtF5A5cw8h8/7rpIrIG8AE3qereaMbZpJw89zLGmE4u6m0OqroAWFBv2W1B0wr82P+KP7u3woFSGD421pEYY0zEWP1IqDZ/BMtfdp3ijGmHmiG78/Pz+frXvx7y0NFNDUndlPoDv9VoatjvBx98kL/+9a+B5e0ZkiF4gMD58+cH+nOUlpYyceJETjrpJN566y3mzZvHCSecEBgIz8SOJYdQ9R4CVZWwt347ujGhqRlb6eOPPyYpKYkHH3ywznpVDQy3EElFRUXcc889DZZfd911XHHFFUD7k0Ow8847j1tucYMjvP7664wePZr333+fyZMn89hjj/HII4806JDWlLCMPmoaZckhVL0Gu/4OOzfFOhLTiUyePJn169c3OvT0M888w+jRo8nPz+fmm2+u877GhqR+5JFHGD9+PAUFBVx00UV1SiQLFy6kqKiI448/nn/9619A08N+33777fzhD3/gueeeo7i4mJkzZ1JYWMhLL73EBRdcENjutdde48ILL2zw/ldeeYWRI0cyduxY/vGPfwSW15ROVq1axU9/+lNefPFFCgsL+eUvf8nSpUu5+uqruemmm5oc0nvJkiVMnjyZ8847j1GjRgHw1FNPMWHCBAoLC/nOd74TGJgwIyODn//85xQUFDBp0qTAIH27du3iwgsvpKCggIKCAt5+++1m99MVxWM/h/iWnAq5vV1yGNP6cWRM/Hp2aRlb94T3IjCgh5dLTm3d8AhVVVW8/PLLzJgxA4B169bx5JNPMmnSJLZv387NN9/MihUryM3NZfr06bzwwgtccMEFlJWVUVRUxJ/+9CfuuOMOfvnLX3Lffffxta99jWuvvRaAW2+9lccee4zvf//7gOtD8d5777FhwwamTp0aeFZEcy6++GLuu+8+/vCHP1BUVISq8pOf/ITS0tLA0Bz1+yGUl5dz7bXXsmjRIo477rg6I5jWKCws5I477qC4uJj77rsPcKOg1hzn4YcfJjs7m+XLl3Ps2DFOOeUUpk+fDsDKlSv5+OOPGTJkCJ988glz587lP//5D4mJiXz3u99lzpw5XHHFFZSVlTFp0iR+/etf89Of/pRHHnmEW2+9lRtvvJEpU6bw/PPP4/P5OHz4cLP76Yqs5NAWvYfAgT0QhSK/6bxqxlYqKipi4MCBXH311UDdoaeXL1/O6aefTl5eHgkJCcycOTMwLHZTQ1J//PHHTJ48mdGjRzNnzpw6w0x/4xvfwOPxMHz4cIYOHcqnn34actwiwuWXX85TTz3F/v37WbZsGWeffXadbT799FOGDBnC8OHDEREuu+yykI/T3JDeEyZMYMiQIYCrmlqxYgXjx4+nsLCQ119/nY0b3d3vSUlJgVLRuHHjAh0MFy1aFBgvyuv1kp2d3ex+uiIrObTFmCkw9gz35DjT4bX2G3641bQ51NfWIbFrhqSeNWsWL7zwAgUFBfzlL39hyZIlDbZpar61rrrqKr761a+SkpLC17/+dRISwn8p0SaG9F6yZEmdc6SqXHnllfzmN79psI/ExMTAZ2xpKOzm9tMV2dWtLZKSLTGYqJgwYQJvvPEGe/bswefz8cwzzwSGxW5qSOpDhw7Rp08fKisrmTNnTp39zZs3j+rqajZs2MDGjRsZMWJEq+KoP7R137596du3L3feeWejQzaMHDmSzZs3s2HDBoDAyLKhaM2Q3gDTpk3jueeeY/fu3QB88cUXzQ7lXfOempFMfT4fBw4caNN+OjO7wrXVmmXw1v/FOgrTyfXp04ff/va3TJ06lYKCAsaNGxd4nnLNkNT5+fksWrQo8LCXX/3qV0ycOJFTTjmlwZDWAwcOZMKECZx99tk8+OCDpKSktCqOWbNmcd1111FYWMjRo0cBN0T1gAEDOOGEExpsn5KSwsMPP8w555zD2LFj6dkz9JH3r7nmGkaNGsXYsWPJz8/nO9/5TqPf/EeNGsWdd97J9OnTGTNmDGeeeSY7duxodt9//vOfWbx4MaNHj2bcuHGsWbOmTfvpzESbeBJUR1JUVKTFxcXRPej7i2D1UvjGza4kYTqUTz75pNGLmmm9G264gZNOOinQVmLiW2N/8yKyQlWLGtveSg5t1XuIa5Au3RLrSIyJunHjxvHhhx+2qaHZdAzWIN1Wef3B44Udm6Bfg0FjjenU2vr8atNxWMmhrRISoecA2Nl1b3Xr6DpDlaoxrdGWv3VLDu0x6ETI6WX9HTqglJQU9u7dawnCdHqqyt69e1t980ENq1ZqjxHj3ct0OP3796ekpCQw5IQxnVlKSgr9+/cP6T2WHNpLFcrLIDW0ETJNbCUmJgZ62BpjGrJqpfZa/jL860GXJIwxppOw5NBe3fvC0cOwb1esIzHGmLCx5NBefYa5nzs2xDYOY4wJI0sO7ZWWCbm9YFvLQx8bY0xHYckhHPoMg91boLIi1pEYY0xY2N1K4TCsEHr0s5FajTGdhiWHcMjt6V7GGNNJ2FfdcDm4F9ZGeWRYY4yJkKgnBxGZISKfich6EbmlkfWzRKRURFb5X9dEO8Y22bYO3vkXHN4f60iMMabdopocRMQL3A+cDYwCLhWRUY1sOldVC/2vR6MZY5vVjMxasja2cRhjTBhEu81hArBeVTcCiMizwPnAmijHAcDqLZW8tOIoIiACnsBPQSCwPHidIHg80DfXy+n5yaSn+PNrVnf3KlkLIyfE4uMYY0zYRDs59AO2Bs2XABMb2e4iETkNWAv8SFW31t9ARGYDs8E9+rAtRNwNRqrgq4YqdSMYqirVCopbp4qbVw1s+966CpZ+eoxbvpZFdpo/QfQ/Hj59z93SmpjUppiMMSYexOPdSv8EnlHVYyLyHeBJ4Mv1N1LVh4GHwT0mtC0HGjUgkVEDEtsU5Iadlfxx/iGefrOM62dkuoX9/Mlh307o2baEZYwx8SDaDdLbgAFB8/39ywJUda+qHvPPPgqMi1JsIRnWO5GzClNYubGSbV/4H3reaxB846eWGIwxHV60k8NyYLiIDBGRJOASYH7wBiLSJ2j2POCTKMYXktPzU/B6YNln/p7RHg8kJcc2KGOMCYOoJgdVrQJuAF7FXfT/rqqrReQOETnPv9mNIrJaRD4AbgRmRTPGUGSleRjVP5GVG4KGzdi3G155HPbuiF1gxhjTTlFvc1DVBcCCestuC5r+GfCzaMfVVvmDEvloSyWlB3zkZXshJR1Kt0LJZ9C9T8s7MMaYOGQ9pNvphP6uQfuTkkq3IDUdevS3/g7GmA7NkkM79c7xkJEibNxVVbuw//GwdzscORS7wIwxph0sObSTiDC4ZwKfl/pqF/Y/3v3cti42QRljTDtZcgiDQXletn/h41ilv7tFTk8YOgZSM2IbmDHGtFE8doLrcAb3TKBaoWSvj2G9E1zX61O/FuuwjDGmzazkEAZ9u3kB2LHPV3dF+RE4tC8GERljTPtYcgiDHpkeErz1koMq/OtBeH9h7AIzxpg2suQQBh6P0CvbWzc5iLiG6W3roKoydsEZY0wbWHIIkz65XnbWr1YaeIIboXXHxtgEZYwxbWTJIUz65HrYc6iayqqgAWJ7DYakFNgSt8NDGWNMoyw5hEmvXC+qsPtgUOnB63VVSyWfQXV17IIzxpgQ2a2sYdIj092xtOdgNf26Ba0YMwUKproRW40xpoOw5BAmeVnu4r/nYL0SQlb3GERjjDHtY19nwyQzVUhKgD2HfA1X7t4CS5+3qiVjTIdhySFMRITumd6GJQeAo4dh4wew6/PoB2aMMW3QbHIQkV7RCqQz6JHlaTw59BsOiUmw+ePoB2WMMW3QUsnhiqhE0UnkZXnYc9CHqtZdkZAIA0bCljXga6TayRhj4kxLyeFCEblBREZEJZoOrkeWl/JKKDumDVcOOhGOHYWd1iHOGBP/WkwOwDrgAhF5JArxdGjdM93p3NtY1VLfYZDX30oOxpgOodlbWVV1F/Cq/2Va0C3DJYd9ZdUMqr/SmwBnXxP1mIwxpi1afbeSiJwtIu+KyGci8ncR+VIkA+uIcmuSw+Fmbln1VdnjQ40xcS+UW1n/F/gxMAl4GLhLRC6NSFQdVGaq4PU0kxxU4Z8PQLEVxIwx8S2U5LBbVf+jqvtUdSFwFvDzUA8oIjP8pY/1InJLM9tdJCIqIkWhHiNWPCLkpHvYV9ZEchCBPkPdWEsVx6IbnDHGhCCU5LBJRO4UkST/fCVQFcrBRMQL3A+cDYwCLhWRUY1slwn8AHg3lP3Hg9wMT/PVSkPGuOc7bLWRWo0x8SuU5FCNu3tpq4gsBdYDS0RkeAj7mACsV9WNqloBPAuc38h2vwJ+B5SHsO+4kNtcyQHcHUtZ3WHDB9ELyhhjQtTq5KCq31LVE4GBuG/1twMCPCIiW1q5m37A1qD5Ev+yABEZCwxQ1Zea25GIzBaRYhEpLi0tbeXhI6+m5NCgI1wNERhaADs3QdmB6AZnjDGtFPKorKp6DFjhf4WViHiAPwKzWhHHw7iGcYqKipq4EkdfbrqHSp/rCJeRIo1vdNxJru0hLSu6wRljTCtFe+C9bcCAoPn+/mU1MoF8XHXVZtydUfM7UqN0q25nTct01UvSRPIwxpgYi3ZyWA4MF5Eh/obtS4D5NStV9YCq9lDVwao6GHgHOE9Vi6McZ5t1a01yADdS67J/wp5tzW9njDEx0OpqJRFJBi4CBge/T1XvaO0+VLVKRG7A9bj2Ao+r6moRuQMoVtX5ze8h/tWUHL5oKTl4E2HTh+4JcT36Nb+tMcZEWShtDi8CB3BtDW2+SV9VFwAL6i27rYltT2/rcWIlK1XwCM3fsQSQlOxGat38MRSd5YbXMMaYOBHKFam/qs6IWCSdhMcjZKUJB1pKDgDDCmHTR7D1Mxh8YuSDM8aYVgqlzeFtERkdsUg6kZx0DweOtOIGqt5DID0b1q+MfFDGGBOCUEoOpwKzRGQTrlpJAFXVMRGJrAPLTvOw91ArSg4eD5wwCQ7vd+Mu2d1Lxpg4EUpyODtiUXQy2WkeNu5q5cgio2xwW2NM/Gl1clDVzyMZSGeSk+7h0FGlyqckeFtRGlCF0q3QvR94vZEP0BhjWtBim4N/HCVE5JCIHAx6HRKRg5EPsePJTncJ4eCRVlQtAWzfAK88DtvWRTAqY4xpvRaTg6qe6v+ZqapZQa9MVbXxHxqRneZOa6sapcE/lEYmrAv7iCTGGNMm0e4h3SXkpLvTur81t7OCa5g+bixsX2+D8Rlj4kIojwktEpHnRWSliHwoIh+JyIeRDK6jqi05tDI5gBuMD2Cd3dZqjIm9UO5WmgPcBHyEe7aDaUJWqiASQskBICMH+h7nnhJXcLrd1mqMialQkkNpZxj7KBo8HiErVVrf5lBj0lchJc0SgzEm5kJJDr8QkUeB1wkaW0lV/xH2qDqB7HRP64bQCJbub9+3DnHGmBgLJTlcBYwEEqmtVlLAkkMjctJaeFxoU/bugDfmwpRvQPe+4Q/MGGNaIZTkMF5VR0Qskk4mO93D5t2t7CUdLLMbHDsCn7wLp14Y/sCMMaYVQh14b1TEIulkctKEQ0cVX3WI7Q5JyW601s0fw9GyyARnjDEtCCU5TAJWichnditry7LTPShwMNRGaYARE6DaZ53ijDExE0q1kj3LIQTBfR1qng7X+jf3gL7DYO1yOPEUG2/JGBN1NvBehITcS7q+gqmu7cFjndiNMdFnz6aMkDb1kg6W1z+M0RhjTGja9LVURHqHO5DOJitNENpRcgCorID3X4cdG8MWlzHGtEZb6ywWhDWKTsjrETLb0ks6mMcLGz6Aj94MX2DGGNMKbU0O1n23FbLTPe0rOXi97jGiOzfDnm1hi8sYY1rS1uTwSFij6KRy0jxtb3OoMXwcJKXAmrfDE5QxxrRCm5KDqv5vWw8oIjP8fSXWi8gtjay/zt+HYpWILO3IHe/aNL5SfUnJcHwRfL4GDn4RnsCMMaYFUb1PUkS8wP3A2cAo4NJGLv5Pq+poVS0Efg/8MZoxhlN2mnDwqFIdai/p+kZOhEEdNkcaYzqgaN9EPwFYr6obVbUCeBY4P3gDVQ1+LnU6bnC/Dikn3YMqHDzazo+QlgmnfR2yuoUnMGOMaUGzyUFEeoX5eP2ArUHzJf5l9Y/7PRHZgCs53NhEbLNFpFhEiktLS8McZnjUdIRrd9VSjQN7YMOq8OzLGGOa0VLJ4YqoRFGPqt6vqsOAm4Fbm9jmYVUtUtWivLy86AbYSjUd4do0dHdjPn0Xlv3TnjNtjIm4lpLDhSJyg4iEa6jubcCAoPn+/mVNeRa4IEzHjrqaMZXadTtrsPxT3c+Pl4Znf8YY04QWkwOwDrhARMJx++pyYLiIDBGRJOASoM6jR0VkeNDsOf7jd0iZbXmWdHPSs91w3utWQtnBlrc3xpg2anZsJVXdBbzqf7WbqlaJyA3+/XmBx1V1tYjcART7n1F9g4icAVQC+4Arw3HsWPB6hOw0CV9yABg9GTa8D6uXwoSvhG+/xhgTpNUD74nI2cDtQA7wAfAnVV0W6gFVdQH1ht9Q1duCpn8Q6j7jWU56Gx8X2pSMHNfvwZsYvn0aY0w9oYzK+r/AZcAaYBxwl4jcr6rPRCSyTiIn3UPpgTAmB4DxZ4PYCCbGmMgJpZ/DblX9j6ruU9WFwFnAzyMUV6cR9pID1CaGHZtgf3zexmuM6dhCSQ6bROROf0MyuDaBqgjE1KnkpHs4ckypqApzX76KY/DGXHh/YXj3a4wxhJYcqnF3L20VkaXAemBJvbuLTD257X0iXFOSkt0jRLd+Bru3hHffxpgur9XJQVW/paonAgOBH+AapwV4RETs6tSEdj8utDkjJ0JqBqxcCNphRxkxxsShkB8TqqrHgBX+l2lBTXIIe7sDQGISFJwO7/zLlSAGjgz/MYwxXZI9vT7CctJd4/H+wxFIDgDHjYVeg8BXGZn9G2O6pJBLDiY0qUlCckKEqpUAPB6YPstubTXGhFWrSw6NPXNtnbYAABscSURBVHRHRE4PazSdkIj4HxcawTYBEaiuhvXvw9HDkTuOMabLCKVa6e8icrM4qSJyL/CbSAXWmeS291nSrVF2wLU9rFoU2eMYY7qEUJLDRNyIqm/jBtDbDpwSiaA6m4h0hKsvMxdGTnClh707InssY0ynF0pyqASOAqlACrBJVSN8xesccjLcs6Q10rebjpkCyWnw3gK7tdUY0y6hJIfluOQwHpiMe/7zvIhE1cnkpHmoqobD5RG+YCelwLgzoXSrG9bbGGPaKJS7la5W1WL/9A7gfBG5PAIxdTrBD/3JTI3w3cNDC2DnZlfNZIwxbdTq5KCqxSKSCwzHVSsBfB6RqDqZQF+HsmoG9IjwwUTglA778DxjTJwI5XkO1+CGzegPrAImAcuAL0cmtM4jor2km1JVCR+9Cb2HQp8h0TuuMaZTCKWO4we49obPVXUqcBKwPyJRdTLZaR5EYF+kekk3ZfNqePdf4LPBc40xoQklOZSrajmAiCSr6qfAiMiE1bkkeIWcNA97D0UxOSQkwsRz4OBeWLU4esc1xnQKoSSHEhHJAV4AXhORF7E2h1brnhnl5ADQdxgMHwtr3obSkuge2xjToYUyZPeFqrpfVW8H/ht4FDg/UoF1Nt1ikRwAxp0FaVnwzj+t74MxptVCaZAuwj0WdJD/fQL8GhgTmdA6l+6ZHlZsqKa6WvF4ojhIXlIyTL4YEpNtcD5jTKuF0s9hDnAT8BHuqXAmBN0zPfiqYf8RpVtGlC/SPQfUTleUu85yxhjTjFDaHEpVdb6qblLVz2teoR5QRGaIyGcisl5Ebmlk/Y9FZI2IfCgir4vIoFCPEY+6Z7pT/cUhX+yCWPEavPSwe/60McY0I5Tk8AsReVRELhWRr9W8QjmYiHiB+4GzgVG4ITjqDwX+PlCkqmOA54Dfh3KMeNU9wwsQm3aHGgNGwOF9buwlY4xpRijVSlcBI4FEaquVFPhHCPuYAKxX1Y0AIvIsrlF7Tc0Gqhp83+U7wGUh7D9udfOXHGKaHHoOdIPzfbDE3ck01JqLjDGNCyU5jFfV9vZr6AdsDZovwQ0F3pSrgZcbWyEis4HZAAMHDmxnWJGXnChkpAhfRLsjXH2jT4MdG13nuB79IatbbOMxxsSlUKqV3m7saXCRIiKXAUXAXY2tV9WHVbVIVYvy8vKiFVa7xOx21mAeD0y+CFLSXRWTMcY0IpSSwyRglYhsAo7hbmVVf9tAa23DPTCoRn//sjpE5AzcbbNTVLXTtJ52z/Cwc38c3OiVng3n3QBeb6wjMcbEqVCSw4wwHG85MFxEhuCSwiXAt4I3EJGTgIeAGaq6OwzHjBvdMz2s2VqJqiKx7nPg9bpOcZ++60oRQ0bHNh5jTFwJZcjudg+VoapVInID8CrgBR5X1dUicgdQrKrzcdVIGcA8/wV0i6qe195jx4PumV6OVbmH/mSmxkGHNFXY8gns3Q45PSG3V6wjMsbEiVBKDmGhqguABfWW3RY0fUa0Y4qWvGzXxFN6MAoP/WkNjwdO+zq89BAsmQvnzLYOcsYYILQGadNOPbNdHf/uAzHsCFdfaoZLEGX7Yek/oDoO2kSMMTFnySGKemR5EGD3gTi7APccCEUzYNs62L0l1tEYY+JA1KuVurJEr9At00NpPJUcaoycAL0GWbuDMQawkkPU9cz2sCsekwPUJobtG1xHOWNMl2XJIcrysryUxlu1UrDqalj5Grzxd9hfGutojDExYskhynpmezhcrpSVx2mC8Hjg9EvA44XXn4KyA7GOyBgTA5YcoqzmjqXSg3GaHAAycmDaZe7ZD68/BeVHYh2RMSbKLDlEWU9/X4e4up21Md37wNRL4dA+2PB+rKMxxkSZ3a0UZT2yavo6xHHJoUbvwXDOdyC7R6wjMcZEmZUcoiw5UeiW4WHHvjgvOdTIyXPPnj64F5bNB19VrCMyxkSBJYcY6NvNy/YvOkhyqFG6FdatdHcxWYIwptOz5BADfbt52bnfh69aYx1K6w0rhInnQMlaeHOeJQhjOjlLDjHQt5uXKl+c37HUmBHjXYLY+pmVIIzp5KxBOgb65rpG6e1f+Oid08EeuDNivGuD2Pih6zDXwcI3xrSOlRxioE+32uTQIR1fBNNnQWISVByDo2WxjsgYE2aWHGIgJVHonunpuMkBXE9qgKX/B68+Dof3xzYeY0xYWXKIkb7dvB3ndtbm5J8K5WXwymPuiXLGmE7BkkOM9Pcnh0pfB7pjqTE9B8JZV4F44NUn3GNHjTEdniWHGBnUMwFfNWzb2wlKD7m94CvXup/LX4GqylhHZIxpJ7tbKUYG5rlG6c9LqxjcsxP8GlIz4Mwr4chBSEgEnw+02k0bYzocKznESI9MD2nJwuelnaDkUCMhEbK6u+mVr7l2iEP7YhuTMaZNLDnEiIgwKM/L57s7aUeyPkPdHUwvPeR6VRtjOpSoJwcRmSEin4nIehG5pZH1p4nIShGpEpGLox1fNA3OS2DbF52gUbox/Y93I7pm5MKip+H9111VkzGmQ4hqchARL3A/cDYwCrhUREbV22wLMAt4OpqxxUJNo3TJnk560czMhRnfhuFjYc0yKLO+EMZ0FNEuOUwA1qvqRlWtAJ4Fzg/eQFU3q+qHQAcbeCh0Q3q5hugNOztp1RK4dogvnQfnfa+2PWLHJtBOWFoyphOJdnLoB2wNmi/xL+uSumV46JHlYe32LnDrZ2au+7ltPbz2JCyZC0cPxzYmY0yTOmyDtIjMFpFiESkuLS2NdThtNrxPAut2VKFd5Zt032EwbjpsWwfz74cNq6wUYUwcinZy2AYMCJrv718WMlV9WFWLVLUoLy8vLMHFwvF9Ezlcruzc1+lr0RwROPFkOPc6yM6D/7wA/3k+1lEZY+qJdu+r5cBwERmCSwqXAN+Kcgxx5fi+7lewdkdlYLTWLiEnzw27sbbYdaAD/91MCt5O0CnQmA4uqiUHVa0CbgBeBT4B/q6qq0XkDhE5D0BExotICfB14CERWR3NGKMtL8tDTrrwSUkXaHeoT8Q9H2LgCW7+03fhnw9Yvwhj4kDUv6Kp6gJgQb1ltwVNL8dVN3UJIkL+wCRWbKigyqckeCXWIcVOt94uYSx6GvoNh6KzILtHrKMypkvqsA3SncnoQYkcrdDOfUtra/QZCude75LC7i3wz/+FT96JdVTGdElWuRsHRvVPxOuBjz6vZES/Lj5QndcLo74EQ0bDR29Ctz5u+bGj7mdyauxiM6YLsZJDHEhJEo7vm8AHmyu6zi2tLUnNgAlfgV6D3PwHi+H5P8OHb9QmCmNMxFhyiBNjhyaxc381JZ3h+Q6RMHycSxSrFsM//gQrXrNOdMZEkCWHODFuWBJeD7y7tiLWocSn3F4w9VL46vXQfwSseRtWLox1VMZ0WpYc4kRmqocTByTy3roKqq1qqWm5vWDyRXD+96HgdLds73Z3h9P2Ddbb2pgwseQQRyYdn8S+smpWb+mCfR5CldUNMnLcdNlB2LMNFv7NDcnx6XtQcSy28RnTwVlyiCMnDU0iO01Y9JFd2EIycCRc9CM49WuQmAzvLXBJorqLDEliTATYraxxJMErTDkxhfnLj7Jzv4/eOV1oOI328ibA0DHutWebezypx+OqmV6fAz0HwrACSM+OdaTGdAhWcogzU05MJsELr6y02zXbrEc/GJLvpo8dhWofrFoE//cnePkx17HO7nQyplmWHOJMVpqHqfkpvP1ZBTu+sNta2y0lDaZfCRf+AE6aBlUVsPwVKPU/VqT8iGuzMMbUYdVKcejssSm8uaac/3vnCDd8JTPW4XQOmbkwerJ7HdhT25i9drnrO9Gtj3vu9YARblq68BhXxmAlh7iUmerh3KJUPthcyYoN1u8h7LJ71A4LPjgfxp7hHmf60Zvw0sPwwj21jdk+K72ZrslKDnHqzIIUlq+v4Ok3yxjeJ4GsNMvjEZHVHfJPda/yI+4JdUcOusZsgFcfd20WvYe4gQF7DoLEpNjGbEwU2BUnTnk9wlVT0ymvVB7692GqfNa5K+JS0twdTaMnu3lV96yJpFTXd+L1OTD3t/D+67XvsYZt00lZySGO9e+RwBWnp/PowjL+uqSMWV9Ox2N14dEjUluqqKp0jdg7N9WOFHvkEDz3/1z7Rd4A9+re1/XiTujio+uaDs+SQ5ybeHwypQerefG9o4jA5VPSu/YDgWIlIdFVK/UZWrvM44XxM1zS2LUZNn3klp/6Ndff4uAXsG2tSya5vSEpOSahG9MWlhw6gHOLUqlW+Ofyo+w5WM13pmdYG0Q8SEmDEya5l6prq9i7w/WzAJc0lr9Su31GDmTnuaHIM3OhohwQSxomLlly6CDOG59KzywPTy4p4xfPHuCSU9OYMDwJsWqm+CDiel8H98AeVgB9h8EXO1zSOFAK+3e7IT7APTN71WL3nuw81ziemQvHj3cPPVK1W2pNzFhy6EAmjUhmQJ6XJxeX8ejCMhZ+WM5541PJH5hoSSJepWa452H3G95wXd/jQDwuYRzY40oa1T4YOdGtf+dfsH09ZHZzr4wcyMit7f1tycNEkCWHDqZftwRuuTCL/3xawUsrjnLPS4fpme3h1BOSKRqWRF62jcfUYfToV1sFBe5iX1Fee8HvOQB8lXDoC9j6ibvVNrNbbXJ4fQ7s21lbYknLgm69YVihW39oHyQkueovSyImRNIZHktZVFSkxcXFsQ4j6qp8SvH6Ct765Bhrt1cB0Lebl9EDEzmuTwJDe1n/iE6lsgIqjtZWXX223FVZlR1wryMH3d1S02e59S/e50okHg+kpENKBvQ7zg0jArB2hVuXlOoSSFKqK+nYc7q7DBFZoapFja2zkkMHluAVJo1IZtKIZHYf8PHB5ko+2FTBwg/LeXWV26ZHloe+uV765Hrp081Lr2wP3TI8ZKd78Hrs22SHkphUtwPeiPF116uCr6p2fuyZcHg/lB92/THKy4Cg3/mKf0NlveHhhxXCKRe4fb14nyt5JKe6V2Kyqx4beILrQb7xA7cs+JWWCUkpYf/oJvqinhxEZAbwZ8ALPKqqv623Phn4KzAO2At8U1U3RzvOjqZntpczC7ycWZBCRZWypbSKDTur2Lzbx459PtZsraQq6PEGHoHsNA/dMj1kpwkZKR7SU4SMFDedkSKkJQspSUJKopCc6H4meLH2jXglUrd/xYARzW9/0Y9dSeTYETd67bGjkJ7l1lVXQ4/+teuPHHSJJC3LJYeKcnj7xYb7LPwyjDnNDWY4/34XT0KiSzIJiTDqZBg0yq3/YLFb5k2s3a7vcMjt6WLZvcUNc+Lx1v7MzHXJx1flSlKBdR6rOguzqCYHEfEC9wNnAiXAchGZr6prgja7GtinqseJyCXA74BvRjPOji4pQTiuTyLH9am9UPiqldKD1ZQe8LHvcDVfBL127q/m8NEqDpcr1S3UMno9kJwoJCcIyYnuWAleV4pJ9AZPE5gPXu7xuH14xPUC9/inPSJuuQe8Ah6PuOWB7d16ETcPbr2I+y4sDaalmXU109LMOndM/PMeIfClu/4lqOY9wStr9h/XkpLdq2YQwmBeL5x6YTPvTYGv/dAljJpXxTHI6enWe7xw3Emu86Cv0v2sqnTLwSWXHRvdKLlVlbUlntRMlxz27YLFzzQ87umXuIc77dzk2lxqiLh9T7sMeg+GkrXuNuKapOLxuJ+Tvgo5ebBtvRt0UTy168TjqtzSMmHnZij5rOH6kRPdOduzzT2e1uOt/cNB3FhdXi98sdO1FYkn6A/L4+5eE4GDe11pruZ9Ho9b393fwbLMn4yD3+vx1FYpVpS7cb+83oiV1KLa5iAiXwJuV9Wz/PM/A1DV3wRt86p/m2UikgDsBPK0mUC7aptDuKkqRyuUw+VKWblSdqyaY5VQXqmUVyjHKpXySvezZrqyyrV9VFVDpa/ufFW9eV8XfzBbcAKpma6fPyRow8a2aU0Samz7OssaOWaDZW3Ia9IgbYayv+B/b3Hz1dVIzfKaf/+aC311tUs6qP+t/vUJyYjX45JNxbG6+1VFUtLcPqoq3AU2+L2oKxmJByrLXekFXAw1m2TkuOPXlLSod1nK7OY+bPkRpKKRZ7JkdXe//6OH/f1c6p2krO5u+uihBo+6FY/HPR4X/MmjAhISmTKxF9PGtC1BxFObQz9ga9B8CTCxqW1UtUpEDgDdgT3BG4nIbGA2wMCBAyMVb5ci4qqS0pKBCDwwrVrV/U9X+6vHq11JxVcN1dWKz10PqNa6877quu+tVg1cK6qVBtOqoGjQNI1MaxPLg+c1MF0zSGvgUqD15qmNQ4Nm6n+jqdlfo/sIWtfYsvrba9DKlvZRG1e9fQRfOxtMNNTUqrZ+yWzuXS3vsuEAiLXvSQCau2AmAGnNHCsDyKgbX/Ctw9XpoKm1fzCB3SaACFqVCr4kak+4/2ey/5JbmQpVNZdfrd0k1b/sWDJUeequE4F0//rURKgCvAlkpUamhNphG6RV9WHgYXAlhxiHY1rBI4LHCwmBu23jvNrFmIjJiPD69ov2fY7bgAFB8/39yxrdxl+tlI1rmDbGGBMl0U4Oy4HhIjJERJKAS4D59baZD1zpn74YWNRce4Mxxpjwi2q1kr8N4QbgVdytrI+r6moRuQMoVtX5wGPA30RkPfAFLoEYY4yJoqi3OajqAmBBvWW3BU2XA1+PdlzGGGNq2dgKxhhjGrDkYIwxpgFLDsYYYxqw5GCMMaaBTjFkt4iUAp+38e09qNf7Ok5YXKGL19gsrtBYXKFpT1yDVDWvsRWdIjm0h4gUNzW2SCxZXKGL19gsrtBYXKGJVFxWrWSMMaYBSw7GGGMasOTgH7wvDllcoYvX2Cyu0FhcoYlIXF2+zcEYY0xDVnIwxhjTgCUHY4wxDXTp5CAiM0TkMxFZLyK3ROF4A0RksYisEZHVIvID//LbRWSbiKzyv74S9J6f+eP7TETOilTsIrJZRD7yH7/Yv6ybiLwmIuv8P3P9y0VE7vEf+0MRGRu0nyv9268TkSubOl4rYxoRdE5WichBEflhLM6XiDwuIrtF5OOgZWE7PyIyzn/+1/vf26onITUR110i8qn/2M+LSI5/+WARORp03h5s6fhNfcY2xhW235u4Yf/f9S+fK+4RAG2Na25QTJtFZFUMzldT14bY/Y25RyF2vRduyPANwFDc8wY/AEZF+Jh9gLH+6UxgLTAKuB34/xrZfpQ/rmRgiD9ebyRiBzYDPeot+z1wi3/6FuB3/umvAC/jHuU2CXjXv7wbsNH/M9c/nRvG39dOYFAszhdwGjAW+DgS5wd4z7+t+N97djvimg4k+Kd/FxTX4ODt6u2n0eM39RnbGFfYfm/A34FL/NMPAte3Na566/8fcFsMzldT14aY/Y115ZLDBGC9qm5U1QrgWeD8SB5QVXeo6kr/9CHgE9wzs5tyPvCsqh5T1U3Aen/c0Yr9fOBJ//STwAVBy/+qzjtAjoj0Ac4CXlPVL1R1H/AaMCNMsUwDNqhqcz3hI3a+VPVN3PNF6h+v3efHvy5LVd9R91/816B9hRyXqv5bVav8s+/gnrjYpBaO39RnDDmuZoT0e/N/4/0y8Fw44/Lv9xvAM83tI0Lnq6lrQ8z+xrpycugHbA2aL6H5C3VYichg4CTgXf+iG/zFw8eDiqJNxRiJ2BX4t4isEJHZ/mW9VHWHf3on0CsGcdW4hLr/tLE+XxC+89PPPx3u+AC+jfuWWGOIiLwvIm+IyOSgeJs6flOfsa3C8XvrDuwPSoDhOl+TgV2qui5oWdTPV71rQ8z+xrpycogZEckA/g/4oaoeBB4AhgGFwA5c0TbaTlXVscDZwPdE5LTglf5vGzG579lfn3weMM+/KB7OVx2xPD9NEZGfA1XAHP+iHcBAVT0J+DHwtIhktXZ/YfiMcfd7q+dS6n4Bifr5auTa0K79tUdXTg7bgAFB8/39yyJKRBJxv/w5qvoPAFXdpao+Va0GHsEVp5uLMeyxq+o2/8/dwPP+GHb5i6M1Rend0Y7L72xgparu8scY8/PlF67zs426VT/tjk9EZgHnAjP9FxX81TZ7/dMrcPX5x7dw/KY+Y8jC+Hvbi6tGSai3vM38+/oaMDco3qier8auDc3sL/J/Y61pLOmML9wjUjfiGsBqGrtOjPAxBVfXd3e95X2Cpn+Eq38FOJG6DXUbcY10YY0dSAcyg6bfxrUV3EXdxrDf+6fPoW5j2Hta2xi2CdcQluuf7haG8/YscFWszxf1GijDeX5o2Fj4lXbENQNYA+TV2y4P8Pqnh+IuDs0ev6nP2Ma4wvZ7w5Uigxukv9vWuILO2RuxOl80fW2I2d9YxC6EHeGFa/Ffi/tG8PMoHO9UXLHwQ2CV//UV4G/AR/7l8+v9E/3cH99nBN1dEM7Y/X/4H/hfq2v2h6vbfR1YBywM+iMT4H7/sT8CioL29W1cg+J6gi7o7YgtHfdNMTtoWdTPF666YQdQiauvvTqc5wcoAj72v+c+/KMXtDGu9bh655q/sQf9217k//2uAlYCX23p+E19xjbGFbbfm/9v9j3/Z50HJLc1Lv/yvwDX1ds2muerqWtDzP7GbPgMY4wxDXTlNgdjjDFNsORgjDGmAUsOxhhjGrDkYIwxpgFLDsYYYxqw5GBMM0TEJ3VHhg3b6L3+UT8/bnlLY6IvoeVNjOnSjqpqYayDMCbarORgTBv4x/3/vX98/PdE5Dj/8sEissg/uNzrIjLQv7yXuGcrfOB/nezflVdEHvGP4f9vEUn1b3+jf2z/D0Xk2Rh9TNOFWXIwpnmp9aqVvhm07oCqjsb1Nr3bv+xe4ElVHYMb8O4e//J7cMMzFOCeJ7Dav3w4cL+qngjsx/XKBTdUwkn+/VwXqQ9nTFOsh7QxzRCRw6qa0cjyzcCXVXWjf8C0naraXUT24IaFqPQv36GqPUSkFOivqseC9jEYN/b+cP/8zUCiqt4pIq8Ah4EXgBdU9XCEP6oxdVjJwZi20yamQ3EsaNpHbTvgObixc8YCy4NGIDUmKiw5GNN23wz6ucw//TbuwUQAM4G3/NOvA9cDiIhXRLKb2qmIeIABqroYuBnIBhqUXoyJJPs2YkzzUsX/wHm/V1S15nbWXBH5EPft/1L/su8DT4jITUApcJV/+Q+Ah0XkalwJ4Xrc6KCN8QJP+ROIAPeo6v6wfSJjWsHaHIxpA3+bQ5Gq7ol1LMZEglUrGWOMacBKDsYYYxqwkoMxxpgGLDkYY4xpwJKDMcaYBiw5GGOMacCSgzHGmAb+f2u0ErEwfpEQAAAAAElFTkSuQmCC\n", 315 | "text/plain": [ 316 | "
" 317 | ] 318 | }, 319 | "metadata": {}, 320 | "output_type": "display_data" 321 | } 322 | ], 323 | "source": [ 324 | "num_epochs = 20000\n", 325 | "\n", 326 | "cutoff = 3000\n", 327 | "epsilon = np.exp(-np.arange(num_epochs)/(cutoff))\n", 328 | "epsilon[epsilon > epsilon[100*int(num_epochs/cutoff)]] = epsilon[100*int(num_epochs/cutoff)]\n", 329 | "plt.plot(epsilon, color = 'orangered', ls = '--')\n", 330 | "plt.xlabel('Epochs')\n", 331 | "plt.ylabel('Epsilon')\n", 332 | "plt.savefig('epsilon_profile.pdf', dpi = 300, bbox_inches = 'tight')\n", 333 | "plt.show()\n", 334 | "\n", 335 | "mp = []\n", 336 | "mpm = []\n", 337 | "reg = 200\n", 338 | "for e in epsilon:\n", 339 | " a = agent.env.reset_policy(e)\n", 340 | " mp.append(np.min(a))\n", 341 | " mpm.append(np.max(a))\n", 342 | "\n", 343 | "plt.plot(epsilon/1.3, color = 'orangered', ls = '--', alpha = 0.5,\n", 344 | " label= 'Epsilon profile (arbitrary units)')\n", 345 | "\n", 346 | "plt.plot(np.array(mpm)-np.array(mp), label = 'Probability difference', color = 'cornflowerblue')\n", 347 | "plt.xlabel('Epochs')\n", 348 | "plt.ylabel(r'max $p^r$ - min $p^r$')\n", 349 | "plt.legend()\n", 350 | "plt.savefig('reset_policy.pdf', dpi = 300, bbox_inches = 'tight')\n", 351 | "plt.show()" 352 | ] 353 | }, 354 | { 355 | "cell_type": "markdown", 356 | "metadata": {}, 357 | "source": [ 358 | "**Training the network.**" 359 | ] 360 | }, 361 | { 362 | "cell_type": "code", 363 | "execution_count": 85, 364 | "metadata": {}, 365 | "outputs": [ 366 | { 367 | "name": "stdout", 368 | "output_type": "stream", 369 | "text": [ 370 | "Epoch 19999 (number of moves 43)\n", 371 | "Game won\n", 372 | "[####################################################################################################]\n", 373 | "\t Average loss: 0.00128\n", 374 | "\t Best average loss of the last 50 epochs: 0.00133, achieved at epoch 17204\n" 375 | ] 376 | } 377 | ], 378 | "source": [ 379 | "loss_log = []\n", 380 | "best_loss = 1e5\n", 381 | "\n", 382 | "running_loss = 0\n", 383 | "\n", 384 | "for epoch in range(num_epochs):\n", 385 | " loss = 0\n", 386 | " counter = 0\n", 387 | " eps = epsilon[epoch]\n", 388 | " \n", 389 | " agent.isgameon = True\n", 390 | " _ = agent.env.reset(eps)\n", 391 | " \n", 392 | " while agent.isgameon:\n", 393 | " agent.make_a_move(net, eps)\n", 394 | " counter += 1\n", 395 | " \n", 396 | " if len(agent.buffer) < buffer_start_size:\n", 397 | " continue\n", 398 | " \n", 399 | " optimizer.zero_grad()\n", 400 | " batch = agent.buffer.sample(batch_size, device = device)\n", 401 | " loss_t = Qloss(batch, net, gamma = gamma, device = device)\n", 402 | " loss_t.backward()\n", 403 | " optimizer.step()\n", 404 | " \n", 405 | " loss += loss_t.item()\n", 406 | " \n", 407 | " if (agent.env.current_position == agent.env.goal).all():\n", 408 | " result = 'won'\n", 409 | " else:\n", 410 | " result = 'lost'\n", 411 | " \n", 412 | " if epoch%1000 == 0:\n", 413 | " agent.plot_policy_map(net, 'sol_epoch_'+str(epoch)+'.pdf', [0.35,-0.3])\n", 414 | " \n", 415 | " loss_log.append(loss)\n", 416 | " \n", 417 | " if (epoch > 2000):\n", 418 | " running_loss = np.mean(loss_log[-50:])\n", 419 | " if running_loss < best_loss:\n", 420 | " best_loss = running_loss\n", 421 | " torch.save(net.state_dict(), \"best.torch\")\n", 422 | " estop = epoch\n", 423 | " \n", 424 | " print('Epoch', epoch, '(number of moves ' + str(counter) + ')')\n", 425 | " print('Game', result)\n", 426 | " print('[' + '#'*(100-int(100*(1 - epoch/num_epochs))) +\n", 427 | " ' '*int(100*(1 - epoch/num_epochs)) + ']')\n", 428 | " print('\\t Average loss: ' + f'{loss:.5f}')\n", 429 | " if (epoch > 2000):\n", 430 | " print('\\t Best average loss of the last 50 epochs: ' + f'{best_loss:.5f}' + ', achieved at epoch', estop)\n", 431 | " clear_output(wait = True)" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": 86, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "torch.save(net.state_dict(), \"net.torch\")" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": 87, 446 | "metadata": {}, 447 | "outputs": [ 448 | { 449 | "data": { 450 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXwU5f3A8c+zR7IJOQgQIBAgAZErJAEC4VbAG8ULT1TwomortbZWalur/NR6tYrWSqWioNQDapHiUeXyQJTLcAa5RAhnCEdIyL3P74+ZLJuQzbm7k5Dv+/XaV2ZnZme+O7uZ7z7zPPM8SmuNEEIIAWCzOgAhhBCNhyQFIYQQHpIUhBBCeEhSEEII4SFJQQghhIfD6gAaok2bNjohIcHqMIQQoklZu3btEa11bFXLmnRSSEhIYM2aNVaHIYQQTYpS6idfy+TykRBCCA9JCkIIITwkKQghhPBo0nUKomkqKSkhKyuLwsJCq0MR4qzmcrmIj4/H6XTW+jWSFETQZWVlERkZSUJCAkopq8MR4qyktSYnJ4esrCwSExNr/Tq5fCSCrrCwkNatW0tCECKAlFK0bt26ziVySQrCEpIQhAi8+vyfNdukcOBoGdv2l1gdhhBCNCrNNik8+u4Jnltw0uowhEXsdjupqamex9NPP13nbaxZs4YpU6YA8Oabb/KLX/zC32HWy9atW0lNTaVfv37s3LmToUOHArB7926SkpLqtK2CggLOO+88ysrK6vS6iIiIKuc/+uijLF68GIAXX3yRU6dO1Wm7wXTXXXexZcsWAJ566inP/OLiYkaOHElpaalVoQWUVDSLZiksLIyMjIwGbSMtLY20tDQ/RVQ3ZWVl2O32KpctWLCA8ePH84c//AGAb775pt77mTVrFtdcc43PfVWmtaa6gbumTZvmmX7xxRe55ZZbCA8PP2O96t6ft9LSUhyOwJzG/vnPf3qmn3rqKR555BEAQkJCGDNmDO+99x4TJkwIyL6t1GxLCkJUJSEhgd/+9rf07duXQYMGsWPHDgDmzZtHUlISKSkpjBw5EoDly5dz+eWXn7GN3bt3M3r0aJKTkxkzZgx79uwBYNKkSUyZMoWhQ4fStWtX5s+fX+Vre/bsyYQJE+jVqxfjx4/3/JpOSEjg4Ycfpn///sybN4+MjAwGDx5McnIyV199NceOHePjjz/mxRdf5NVXX2XUqFFA1b/ay8rKeOihhxg4cCDJycn84x//qPJ4zJ07lyuvvBKAvLw8xowZQ//+/enbty8ffvihJ+YePXpw2223kZSUxN69ewH41a9+RZ8+fRgzZgzZ2dmeYzB//nxeeukl9u/fz6hRoyrE+etf/5qUlBRWrlzJtGnTGDhwIElJSUyePNmTbM4//3weeOAB0tLSePLJJ0lMTKSkxLgUnJubW+F5ufL9lis/JsuXL+f8889n/PjxnuPuvZ81a9YwdepUCgoKSE1N9SSBq666irlz51Z5zJo6KSkI683/y5nzug+AlPOhpBg+fPnM5b2GQJ+hUJAHH1U6oY3/dY27LP8nL/e73/2OG264AYDo6Gg2btzInDlzeOCBB1i0aBHTpk3jf//7Hx07duT48ePVbvv+++9n4sSJTJw4kVmzZjFlyhQWLFgAwIEDB/j666/ZunUr48aNY/z48We8/ocffuD1119n2LBh3HHHHfz973/nN7/5DQCtW7dm3bp1ACQnJ/Pyyy9z3nnn8eijj/L444/z4osvcs899xAREeF5TVVef/11oqOjWb16NUVFRQwbNoyLLrqoQtPF4uJidu3aRXmnky6Xi//85z9ERUVx5MgRBg8ezLhx4wDYvn07s2fPZvDgwQDk5+eTlpbGCy+8wLRp03j88cf529/+5tn2lClT+Otf/8qyZcto06aN5zXp6en85S/G96F37948+uijANx6660sWrSIK664whNbeb9nu3fv5qOPPuKqq67i3Xff5ZprrqlTu/zvv/+ezZs306FDB4YNG8aKFSsYPny4Z/nTTz/N3/72twoly6SkJFavXl3rfTQlUlIQzVL55aPyR3lCALjppps8f1euXAnAsGHDmDRpEjNnzqzx+vrKlSu5+eabAeNk9vXXX3uWXXXVVdhsNnr37s2hQ4eqfH2nTp0YNmwYALfcckuF15fHeeLECY4fP855550HwMSJE/nyyy9r/f4/++wz5syZQ2pqKunp6eTk5LB9+/YK6xw5coSWLVt6nmuteeSRR0hOTuaCCy5g3759nvfQpUsXT0IAsNlsnlgrvwdf7HY71157ref5smXLSE9Pp2/fvixdupTNmzefcRzAuPb/xhtvAPDGG29w++231/o4AAwaNIj4+HhsNhupqans3r27VrGGhIRw8uTZVy8pJQVhvep+2TtDql8eFlGrkkFdeDfjK5+eMWMG3333HR999BEDBgxg7dq19dp2aGioZ9rXtffKzQi9n7do0aJe+61Ma83LL7/MxRdf7HOdsLCwCm3c586dS3Z2NmvXrsXpdJKQkOBZXlNctWka6XK5PPUIhYWF3HfffaxZs4ZOnTrx2GOPVYjFe3/Dhg1j9+7dLF++nLKysior0x0OB263GwC3201xcbFnmfdnYrfba12BXFRUhMvlqtW6TYmUFISo5L333vP8HTJkCAA7d+4kPT2dadOmERsb67luXpWhQ4fy7rvvAsaJdMSIEXXa/549ezwllH/9618VLmWUi46OJiYmhq+++gqAt956y1NqqI2LL76YV1991XPtfdu2beTn51dYJyYmhrKyMs/J+MSJE7Rt2xan08myZcv46SefvS/jdrs91/B9vYfIyEifv7TL99mmTRvy8vKqrH/xdtttt3HzzTf7LCUkJCR4EvnChQvPqHOoidPprPCanJwc2rRpU6fLVE2FJAXRLJXXKZQ/pk6d6ll27NgxkpOTmT59Oi+88AIADz30EH379iUpKYmhQ4eSkpLic9svv/wyb7zxBsnJybz11ltMnz69TrH16NGDV155hV69enHs2DHuvffeKtebPXs2Dz30EMnJyWRkZHiuv9fGXXfdRe/evenfvz9JSUn87Gc/q/IX8kUXXeS59DNhwgTWrFlD3759mTNnDj179vS5/RYtWrBq1SqSkpJYunRplbFNnjyZSy65xFPR7K1ly5bcfffdJCUlcfHFFzNw4MBq38+ECRM4duyY59JfZXfffTdffPGFpxK7riWuyZMnk5yc7KloXrZsGWPHjq3TNpqM8iZk/n4As4DDwCavea2Az4Ht5t8Yc74CXgJ2ABuA/rXZx4ABA3R93fVKjr7rlZx6v17U35YtW6wOwacuXbro7Oxsy/b/448/6j59+li2/8rWrl2rb7nlFqvDqNG8efOCGufVV1+tf/jhh6DtryGq+n8D1mgf59VAlhTeBC6pNG8qsERr3R1YYj4HuBTobj4mA68GMC4hRC3179+fUaNG1fnmtWC6//77mTp1Kn/84x+Dsr/i4mKuuuoqzj333KDsL9gCVtGstf5SKZVQafaVwPnm9GxgOfCwOX+OmcG+VUq1VErFaa0PBCo+IapSm5YngZSQkMCmTZssjaGyO+64w+oQqvXyy1U0WQ6gkJAQbrvttqDuM5iCXafQzutEfxBoZ053BLxr7rLMeWdQSk1WSq1RSq0pvyFGCCGEf1hW0WyWCnzfD+/7da9prdO01mmxsbEBiEwIIZqvYCeFQ0qpOADz72Fz/j6gk9d68eY8IYQQQRTspLAQmGhOTwQ+9Jp/mzIMBk5IfYIQQgRfwJKCUuodYCXQQymVpZS6E3gauFAptR24wHwO8DGwC6NJ6kzgvkDFJQSc7jo7KSmJ6667rs5dOPvqGtqXyh2ylfPV/faMGTOYM2eOZ/7+/fvrtD9v3h33LVy40NNNeHZ2Nunp6fTr14+vvvqKefPm0atXryrvG2hMxo8fz65du+r0mvLO7SrzPh4LFizwdJXdGFX3nbjxxhvP6KakvgLZ+qjqu0hgTBXrauDngYpFiMq8u86eMGECM2bM4MEHH/Qs97TZtgW2MO2r++177rnHM/3mm2+SlJREhw4dGry/cePGeTqxW7JkCX379vV0EX3JJZcwc+bMKu8+rkogu632ZfPmzZSVldG1a9dav6a65rTex2PBggVcfvnl9O7d+4z1avteA3lMqvtO3HvvvTz77LPMnDmzwfuRO5pFszdixAh27NhRZRfQ77zzjudO5ocffrjC66rqGnrmzJkMHDiQlJQUrr322golkMWLF5OWlsa5557LokWLAN/dbz/22GM8//zzzJ8/nzVr1jBhwgRSU1M9vYGW+/zzz7n66qvPeP2nn35Kz5496d+/Px988IFnfnlpJCMjg9/+9rd8+OGHpKam8vjjj/P1119z55138tBDD/nsWnv58uWMGDGCcePGeU6eb7/9NoMGDSI1NZWf/exnnpNwREQEv//970lJSWHw4MGezvMOHTrE1VdfTUpKCikpKZ7xHnxtx5t3V95gnAzT0tLo06cPf/rTnzzzK3czDkZXIOWlw1WrVlU4Ht988w0LFy7koYceIjU1lZ07d1boonv69On897//9ZSsLrjgAs/7eeyxx7j11lsZNmwYt956KyNHjqzQo+rw4cNZv359hfdReVCmyy+/nOXLl1d73Hx9JwoKChgxYgSLFy/2y8A/khSE5RYvXnzGY9u2bYDxy6uq5Tt37gSMPnIqL6uL0tJSPvnkE/r27QsYXUDfd999bN68GafTycMPP8zSpUvJyMhg9erVni6wy7uG3rx5M+eddx6PP/44ANdccw2rV69m/fr19OrVi9dff92zr927d7Nq1So++ugj7rnnnloNqD5+/HjS0tKYO3cuGRkZXHbZZWzdutWThN54440z7iMoLCzk7rvv5r///S9r167l4MGDZ2w3NTWVadOmccMNN5CRkcGf/vQnz36ee+65Cl1rr169mpkzZ/Ljjz8CsG7dOqZPn862bdvIzMzkvffeY8WKFWRkZGC32z3jDOTn5zN48GDWr1/PyJEjPb9ip0yZwnnnncf69etZt24dffr0qXY73lasWMGAAQM8z5988knWrFnDhg0b+OKLL9iwYYNnWXk34zfeeCMAp06dIiMjg7///e9nHLOhQ4cybtw4nnvuOTIyMujWrRtwuovuX//61wwfPpxvv/2W77//nhtvvJFnn33W8/otW7awePFi3nnnHe68807efPNNwOhTqrCwsNpuUSrzddzKVf5OhIWFYbPZOOecc85IPvUhSUE0S+V9H6WlpdG5c2fuvPNOoGIX0KtXr+b8888nNjYWh8PBhAkTPN1T++oaetOmTYwYMYK+ffsyd+7cCt09X3/99dhsNrp3707Xrl3ZunVrneNWSnHrrbfy9ttvc/z4cVauXMmll15aYZ2tW7eSmJhI9+7dUUpxyy231Hk/1XWtPWjQIM+4C0uWLGHt2rUMHDiQ1NRUlixZ4rneHxIS4ikFDRgwwHNj4NKlSz39OdntdqKjo6vdjrcDBw7g3RT9/fffp3///vTr14/NmzdXqBPw7l4bTneJPnLkSHJzc2scF6PyNrKysrj44ovp27cvzz33XIXPdty4cYSFhQFw3XXXsWjRIkpKSpg1axaTJk2qcT/efB23mrRt27ZBdU/lpOtsYbkLLrjA5zKHw1HtcpfLVe1yX3wNx1nfrqnLu4aeNGkSCxYsICUlhTfffNNzScB7HV/Pa+v222/niiuuwOVycd111wXkGrb20bX28uXLKxwjrTUTJ07kz3/+8xnbcDqdnvdYU5fU1W3Hm3d33j/++CPPP/88q1evJiYmhkmTJvnsXhvqd/y9t3H//ffz4IMPMm7cOJYvX85jjz1W5Xrh4eFceOGFfPjhh7z//vtVdrPu3ZU3UCHuuhw3b4WFhZ7E1BBSUhDCh0GDBvHFF19w5MgRysrKeOeddzzdU/vqGvrkyZPExcVRUlJyxuWPefPm4Xa72blzJ7t27aJHjx61iqNyF9MdOnSgQ4cOPPHEE1V2Fd2zZ092797tucT2zjvv1Pm916ZrbYAxY8Ywf/58Dh82bjk6evRotV1ql7/m1VeN7s3Kyso4ceJErbfTq1cvzxCpubm5tGjRgujoaA4dOsQnn3xS7X7Lu0T/+uuviY6OJjo6usLy6rryBqPr8I4djY4WZs+eXe2+7rrrLqZMmcLAgQOJiYk5Y3lCQgIZGRm43W727t3rqeOorapi3bZtW5VjSdSVlBSE8CEuLo6nn36aUaNGobVm7NixnkrO8q6hn3jiCdq2bes54fzf//0f6enpxMbGkp6eXuEft3PnzgwaNIjc3FxmzJhR6wFaJk2axD333ENYWBgrV64kLCyMCRMmkJ2dTa9evc5Y3+Vy8dprrzF27FjCw8MZMWJEnUcIu+uuu9i9ezf9+/dHa01sbKynPsVb7969eeKJJ7joootwu904nU5eeeUVunTp4nPb06dPZ/Lkybz++uvY7XZeffVVhgwZUqvtjB07luXLl3PBBReQkpJCv3796NmzZ4XR6nxxuVz069fPc1mnshtvvJG7776bl156qcrmw4899hjXXXcdMTExjB492lPHUpUBAwYQFRXlc3yHYcOGkZiYSO/evenVqxf9+/evNvbKKn8ncnNzCQsLo3379nXaTlWU9jH6U1OQlpamq2p7XBt3//0oADPva+XPkEQtZGZmVnkyE7X3i1/8gn79+nnqQpqLgoICRo0axYoVKzyjtDVG+/fv5/zzz2fr1q0Bb9YM8MILLxAVFVXl96Gq/zel1Fqt9ZltoZHLR0I0OQMGDGDDhg31qkBu6sLCwnj88cfZt6/x9oIzZ84c0tPTefLJJ4OSEMAYlGjixIk1r1gLcvlIiCamvuNDny2qG1e6MbjtttuC3rW2r8tU9SElBSGEEB6SFIQQQnhIUhBCCOEhSUEIIYSHJAXRLNW162shmgtJCkIIITwkKQhh2r17N6NHjyY5OZkxY8awZ88ewOieIikpiZSUFEaOHAkY/fqXd/OcnJzstwFOhLCa3KcgLPXu1/nsPeJ7EJT66NTGzo3D696x3f3338/EiROZOHEis2bNYsqUKSxYsIBp06bxv//9j44dO3p61pwxYwa//OUvmTBhAsXFxdUO5CJEUyIlBSFMK1eu5Oabbwbg1ltv9XSHPWzYMCZNmsTMmTM9J/8hQ4bw1FNP8cwzz/DTTz/5pXdKIRoDKSkIS9XnF32wzZgxg++++46PPvqIAQMGsHbtWm6++WbS09P56KOPuOyyy/jHP/7B6NGjrQ5ViAaTkoIQpqFDh/Luu+8CxrCPI0aMAGDnzp2kp6czbdo0YmNj2bt3L7t27aJr165MmTKFK6+8ssKIX0I0ZVJSEM3SqVOniI+P9zx/8MEHefnll7n99tt57rnniI2N5Y033gDgoYceYvv27WitGTNmDCkpKTzzzDO89dZbOJ1O2rdvzyOPPGLVWxHCryQpiGbJe9Qrb0uXLj1jnvfA9+WmTp3K1KlT/R6XEFaTy0dCCCE8JCkIIYTwkKQgLNGUR/wToqmoz/+ZJAURdC6Xi5ycHEkMQgSQ1pqcnJxajwVeTiqaRdDFx8eTlZVFdna21aEIcVZzuVwVWtnVhiQFEXROp5PExESrwxBCVMGSy0dKqV8ppTYrpTYppd5RSrmUUolKqe+UUjuUUu8ppUKsiE0IIZqzoCcFpVRHYAqQprVOAuzAjcAzwAta63OAY8CdwY5NCCGaO6sqmh1AmFLKAYQDB4DRwHxz+WzgKotiE0KIZivoSUFrvQ94HtiDkQxOAGuB41rrUnO1LKBjsGMTQojmzorLRzHAlUAi0AFoAVxSh9dPVkqtUUqtkdYrQgjhX1ZcProA+FFrna21LgE+AIYBLc3LSQDxwL6qXqy1fk1rnaa1TouNjQ1OxEII0UxYkRT2AIOVUuFKKQWMAbYAy4Dx5joTgQ8tiE0IIZo1K+oUvsOoUF4HbDRjeA14GHhQKbUDaA28HuzYhBCiubPk5jWt9Z+AP1WavQsYZEE4QgghTNL3kRBCCA9JCkIIITwkKQghhPCQpCCEEMJDkoIQQggP6Tp78zegFCgbhIZB12Rj/r7tUFYGnXoYy4UQohmQpPD57NPTMe1OJ4UVC2D/DrjmV9C5pzWxCSFEkElSuOMp0BrcbrB5XU278DZ45yn4YZUkBSFEsyF1ClGtIboNxLQ1/paLaQfdUmHH91BW6vv1QghxFpGkUJ1zB0LRKfhpi9WRCCFEUEhSqE7nntBrMIRHWR2JEEIEhdQpVMfugItvtzoKIYQIGikp1Maxw5Cz3+oohBAi4CQp1ERrmP88rFxodSRCCBFwkhRqohR0HwA/boTiQqujEUKIgJKkUBs9BhrNUndmWB2JEEIElCSF2mifaNzDkPmt1ZEIIURASVKoDaWg52Cj24vCU1ZHI4QQASNNUmsrdbTxcIVbHYkQQgSMJIXaCmthdQRCCBFwcvmoLo4dgvefhYO7rY5ECCECQpJCXYRHweE9kLnS6kiEECIgJCnURWiY0XPqD6ul51QhxFlJkkJd9RoChfnGzWxCCHGWkaRQV517QYto2PKN1ZEIIYTfSeujurLZIH0s2OxWRyKEEH4nSaE+ks+zOgIhhAgIuXxUX0UFsHmF0YuqEEKcJSQp1NdPm+HzOTJUpxDirGJJUlBKtVRKzVdKbVVKZSqlhiilWimlPldKbTf/xlgRW611S4WwCNj0ldWRCCGE31hVUpgOfKq17gmkAJnAVGCJ1ro7sMR83njZHdB7KOxaD/m5VkcjhBB+EfSKZqVUNDASmASgtS4GipVSVwLnm6vNBpYDDwc6nkWLFqGUQilFREQEI0eOBGDVqlUcP368PGYAoqOjGTRo0OkXJw2HtZ8ZzVMHXhLoUIUQIuCsaH2UCGQDbyilUoC1wC+BdlrrA+Y6B4F2Vb1YKTUZmAzQuXPnBgfTsmVL3G43AGFhYZ75DocDh8M4PFprtFeFstaavLw8ImPaQfy5Mn6zEOKsYUVScAD9gfu11t8ppaZT6VKR1lorpaps1qO1fg14DSAtLa3BTX+GDx9e5fz+/fv7fE1mZiZbtmzhiiuuIPTK+8EZ0tAwhBCiUbCiTiELyNJaf2c+n4+RJA4ppeIAzL+HLYitVjp06EBxcTGZmZmnE0JRgbVBCSGEHwQ9KWitDwJ7lVI9zFljgC3AQmCiOW8i8GGwY6utli1bEh8fz86dOykrK4Md38Nrv4FjjTaPCSFErVjV+uh+YK5SagOQCjwFPA1cqJTaDlxgPm+0zjnnHIqKijh06BDEdTNuYtuw3OqwhBCiQapNCkqpW7ymh1Va9ov67lRrnaG1TtNaJ2utr9JaH9Na52itx2itu2utL9BaH63v9oOhXbt2OBwO9u7dCy2i4Nw04w7n4kKrQxNCiHqrqaTwoNf0y5WW3eHnWJoUu93OqFGj6NevnzEjZZSREDK/tTYwIYRogJqSgvIxXdXzZic2NpaQELOiOS4R2iXA+uXSH5IQosmqqUmq9jFd1fNmp6ysjK1bt9KqVSvi4uJg9M3gagGq2edLIUQTVVNS6GlWBiugmzmN+bxrQCNrAmw2G5mZmcTHxxtJoV0Xq0MSQogGqSkp9ApKFE2UUorY2FhycnJOz8zNgWXvwJAroW0n64ITQoh6qLZOQWv9k/cDyMO40ayN+bzZi4mJ4cSJE5SWlhozQsJg33ZY+z9rAxNCiHqoqUnqIqVUkjkdB2zCaHX0llLqgSDE1+jFxBg9fJd3nocrHPqOhO1r4cQRCyMTQoi6q6n1UaLWepM5fTvwudb6CiCdZt4ktVxMTAw2m438/PzTM1NHAwrWLbYsLiGEqI+akkKJ1/QY4GMArfVJwB2ooJqSFi1acP3119Oli1clc2QM9EyHzV9DQZ51wQkhRB3VVNG8Vyl1P0Yndv2BTwGUUmGAM8CxNQnlYzGcYcBF0DLWGIxHCCGaiJpKCncCfTAGxLlBa21eOGcw8EYA42pSdu/ezbJlyyqMuUDrOBh0GYS4rAtMCCHqqNqfsVrrw8A9VcxfBiwLVFBNTVFREQcOHKCgoIDw8PDTC7SG7eugrAR6DbYuQCGEqKVqk4JSamF1y7XW4/wbTtMUHR0NwIkTJyomBaWMTvKy90C3fhASalGEQghROzVd8B4C7AXeAb5D+juqUnlSyM3NNe5s9jb4cnjvGdjwBaRdZEF0QghRezXVKbQHHgGSgOnAhcARrfUXWusvAh1cU+FyuXA6neTm5p65MK4rdO4F6z6DkuLgByeEEHVQ0x3NZVrrT7XWEzEql3cAyxsylsLZSClF+/btcTh8FLwGXwGnTsLGL4MbmBBC1FGN7SWVUqHAWOAmIAF4CfhPYMNqekaMGOF7YYdu0GcYRMQELyAhhKiHmiqa52BcOvoYeNzr7mZRVxfeZnUEQghRo5rqFG4BugO/BL5RSuWaj5NKqSouoDdf2dnZLFy4kKNHqxlFtLQEvl8ChaeCF5gQQtRBTfcp1JQ0hMnhcJCXl8fJkydp1apV1SsdOwRfzoP8EzD8muAGKIQQtSAnfT+JjIwE4OTJk75Xio2HHoMgYynkHfe9nhBCWESSgp84HA7CwsKqTwoAQ8aB2w3fLgpOYEIIUQeSFPwoMjKy5qQQ3cYYb2HLCuNykhBCNCLShacfdezYkcLCwppXTB8Lxw4aFc9CCNGISFLwo169ajmkdXgkXCMD1wkhGh+5fORnWmvc7lqOP3TqpFG3UNv1hRAiwCQp+FFubi7vv/8+e/furd0L9m2Hb/8Lm74ObGBCCFFLkhT8KCwsjLKyMvLyajkE5zn9IP5c+GaB3NAmhGgULEsKSim7Uup7pdQi83miUuo7pdQOpdR7SqkQq2KrL6fTicvlqrkFUjmlYOT1UHQKvpMmqkII61lZUvglkOn1/BngBa31OcAxjKFAm5xaNUv11rYTJI2A9csgZ3/gAhNCiFqwJCkopeIxel79p/lcAaOB+eYqs4GrrIitoeqcFACGXgk90yE0vOZ1hRAigKxqkvoi8Fsg0nzeGjiutS41n2cBHat6oVJqMjAZoHPnzgEOs+7i4+Np0aIFWmuMXFcLYRFw0aSAxiWEELUR9KERTH4AABrWSURBVJKCUupy4LDWem19Xq+1fk1rnaa1TouNjfVzdA0XHx9P3759a58QvB3PhkUzjKaqQghhAStKCsOAcUqpywAXEIUx1GdLpZTDLC3EA/ssiK3BtNYUFRWhlCI0NLRuLy4rhV0bwBkKF98emACFEKIaQS8paK1/p7WO11onADcCS7XWE4BlwHhztYnAh8GOzR9KS0v54IMP2LFjR91f3DoO0i6GzG9hz1b/ByeEEDVoTPcpPAw8qJTagVHH8LrF8dRLnZulVjboMoiOhSVvQXGRf4MTQogaWJoUtNbLtdaXm9O7tNaDtNbnaK2v01o32TNivVoglXM44aKJkJsDaz71b2BCCFED6RAvACIjI9m/vwH3HHTsDpdNhi59/BeUEELUQmO6fHTWiIyMpLCwkJKSBnSN3b0/hIQa3WvLZSQhRJBISSEAOnbsiMvlaviGSkvgnaegwzkwZkLDtyeEEDWQkkIAtGzZkm7duuF0Ohu2IYcTEpJg45fw40b/BCeEENWQpBAgR48e5cSJEw3f0JBx0CYePnsT8v2wPSGEqIYkhQBZvnw5mZmZNa9YE4cTLr0LSovh01mgdcO3KYQQPkhSCJAGNUutrHUcnH8jFBfKuAtCiICSiuYAaXCz1Mp6D4VeQ8AmeVwIEThyhmmAkwVudh0qrXKZX5qlelPKSAiFp2Dx21BQy9HdhBCiDiQpNMCz/8nlz//OrXJZZKTRK3ith+asrRPZkLkSPvknuN3+3bYQotmTpNAAB4/7Pim3bduWUaNGERER4d+dtusCo26CPZmwcqF/ty2EaPakTiFAXC4XcXFxgdl40nA4+COs/gTaJ0K3lMDsRwjR7EhJIYAOHDjg38pmb+ffaJQavv63XEYSQviNlBQCaNOmTQB06NDB/xt3OOHye43KZ2mRJITwEzmbBFBkZKT/K5or7CAGWkQbJYWMZcbIbUII0QCSFAIoIiKCgoIC/zVL9WX/Dlj+Lnw+R+54FkI0iCSFAIqKigLw353NvsSfC0OvhK3fwaqPA7svIcRZTZJCAEVHRwP4p2O8mgy8FHoNNpqpbl4R+P0JIc5KUtEcQJGRkYwdO9ZzI1tAKQUX3AqnTsLy9yAxGcKDsF8hxFlFkkIA2Ww2T2khKOwOGPszOHZQEoIQol7k8lGAHThwgA0bNgRvhyGhxv0LAJu/gYO7g7dvIUST1+yTgg5wa53s7Gw2bdpEWVlZQPdzhpJiWPUR/Gc6HN4T3H0LIZqsZp8UAi2olc3enCFw7YMQGgb/fkESgxCiViQpBFh5UsjNrbo31YCKag3jfy2JQQhRa5IUAiwyMhKlFMePH7cmAO/EsG+HNTEIIZoMaX0UYHa73TPgjmWiWsMtj0KIy3heXGRUSAshRCWSFILg0ksvxW63WxtEeUI4vBf+8yKMuQXO6WdtTEKIRkcuHwWB5QnBW2QriGkHH/0DNn5ldTRCiEYm6ElBKdVJKbVMKbVFKbVZKfVLc34rpdTnSqnt5t+YYMcWKHl5eXz55ZccOXLE6lAgrAVc/QB06QNL3oaV/5VO9IQQHlaUFEqBX2utewODgZ8rpXoDU4ElWuvuwBLz+VnB4XCQlZXVOJICGM1Vr7gPeg+F7xbBlm+sjkgI0UgEvU5Ba30AOGBOn1RKZQIdgSuB883VZgPLgYeDHV99aK1RSvlc7nK5CAsL4+jRo0GMqgZ2O1x4G3TpDef0tzoaIUQjYWmdglIqAegHfAe0MxMGwEGgnUVhBUSrVq0aV1IAoxO9HgONBHHqJMx73qiIFkI0W5YlBaVUBPBv4AGtdYU7u7TR90SVF7qVUpOVUmuUUmuys7ODEGnNanNFPiYmhtzcXEpLG+noaAUnIfcIvPe0dL0tRDNmSVJQSjkxEsJcrfUH5uxDSqk4c3kccLiq12qtX9Nap2mt02JjY4MTcAN8taWIh+ccp02bNrRp08ba+xWq07oD3PR76NjdGMHts9lQGuAR44QQjY4VrY8U8DqQqbX+q9eihcBEc3oi8GGwYwuEOcvzOZrnpsjelnk70jhwshHfNBYeCVdNgfSxRuXzV/+2OiIhRJBZcfPaMOBWYKNSKsOc9wjwNPC+UupO4Cfgegtiqx8N+K5nBmDLXuNX98afSuge5wx8TPVls8GQcRDXDdp2NuYVF4Iz1KiDEEKc1axoffQ1vk+hY4IZSzAtWFUAwPZt29DpKdW2VmoUEvoYf91uWPAyuFoYI7vJ4D1CnNWa/R3N/rhtqy7bKC4pIT8/3w97DRKljO4wftoMb0+DXUEcMEgIEXTNPikEn6KxtJqqFaWg/wVw0yNGKWHhK/DJP6HwlNWRCSECQJJCkNlsNg4frrJhVePWpqORGIaMg+wsYzxoIcRZR5KCH9Sl66AWLcI5dOhQ4IIJJLvDaJk04Y9GVxmlJUbz1aMHrY5MCOEn8nMvyMIjY+jWuRtutxubrYnm5PJeX7P3wo51kLkSUkfDoLHgCrc2NiFEgzTRs1LTUbmN0eofnbTr1LPpJgRvcV1h4v9B72Hw/RKY86jRHbf0uipEk3UWnJmanv05RY2nx9SGCo+EC24x6htatjVKDUKIJkuSggV27NjB0qVLKSktxX22/Kpu2xmuewiu+LnRYik/F/79Auz9werIhBB1IEnBD6o9rVdxj1qbNm0oLS3lD3OPM+WfxwIVVvApZQziA3AiG44dhH//FT54EfZttzY2IUStSFIIsKruW27VqjUOh4Oj+TaKztY+5zp0g0lPwIjxcCTL6JZ73vNQVmZ1ZEKIakjrIwvYbDY6deoEZ0m1gk8OJwy4EJLPM7rjzj1yuuVS1jajf6XGNH61EEKSwrtfn2L9jyU8c1vL+m+kmutHSlW9fNb3Peq/v6bGGQKpo04/zzkA8/8CES2h70hIGgEtoqyLTwjh0ewvHy3bWMTRPLfVYTQvrdrDuJ8bYzisXAizfgefzoKTZ1H9ihBNVLMvKfjDWdJ+KHiUgq7JxuPoQVi/DLavhfNuMJZnZ0GLaOmRVQgLSFIIsNp0kL1lyxZ69+4d8FgapVbtYdRNMPL60/ULS96Gwz9BYjL0Hmp04y19LQkRFPKfZoHKtyZkZmZy7rnn4nA044/Du8L5wttg8zew9VvYmQGh4TDoMqPSWggRUM2+TsEfGnr/WVFRETt27PBPMGeD1h1g5Hi48xmj7iGxrzHID0BBHix+G/ZkQlmptXEKcRZqxj9Ng6SK60e6Ui1E27Zt2bJlC926dcPpbMRDdQab3X667qFcdhb8sAo2fQUhLujc21h+Tj/juRCiQaSk0AikpqZSWFjI5s2brQ6l8evcEyY/b5Qgzh0IB3bCZ29CoTma3aGfYP9OuUlOiHqSkkKAVVXR/NWWogrP27RpQ3p6Oh06dAhOUE2dM+R0CUJrOHoAoloby1Z/anTn7QyFjt2hcy/o1BNi462NWYgmQpJCgKkqssLB42feF9GtWzcA3G43Wmvscqdv7Shl1EGUGzMBegw06hz2boXdmyC2E0z4g7F86yqIjIG2XYzkIoSoQJKCH9S1nln7qJl2u90sWbKEiIgIBg8ejKoqo4jqhUVA9/7GA4wb4k7lGtNut9HctaQIbDYjWcR1M+oj4s+1LmYhGhFJChYo9tFoxmaz0b59ezZu3IjL5SI1NVUSQ0NFxhgPMBLBHU/BgV3mY6dRYR0abiSFogL48GWI7Wx0Bd62M7SKq1f/TPmFbr7dVszovqHyGYomRZKCBXJO+u5WIykpicLCQjIzMyktLWXAgAFnxyhtjUVYRMUWTWVlUGZ2VVuQZ/zd8o1xlzWAzQ6X3AHnphljROzbDq3jjAGFqrmh7q0v8lm7s4QusXbOiZMWZaLpkKTgD+bVoBf+m4tNKe69JIIQh/Hr0FepwFtJqQYFTrtCKUVaWhoOh4PMzEwABg4cGKjIhd1+uiTQMhau/61xmelENhzeY4xDXV5nsW8bfDzTmLbZoGU7447sYddATFs4ddK4NBXZivxC40tRm89fiMZEkkIDVO4Adcte4wzw5PxcHr8xutbbue+1Y0S4FC/cYVzmUErRr18/oqKiaN3aaFVTXFyM3W6XCuhgsNkgpp3x6OGVkLumwM1/MFo7eR4HTyeVzG/hq/lgs6FCJgNd0BnLoO0ICA2DvOPgLoMWLaXLcNFoSVLwg8rVxvuP1r2NfF7hmZXP5S2SANatW8eBAwfo3r073bp1IywsrM77EA3kcELbTsajKl2TwRUOx7NRP7igEPTebWA3uw1f+xl8v8RoMRUeBRExENkKxk425h3cDcUFEBZpLA+LMBKUEEEkSaEhzKLCvBWnOFl4Zj2Br1ZGvny1pZDUxBAiw848ESQkJHDq1Ck2bNjAxo0badu2LV27diUxMbG+0Qt/Ky9dALZjJ2FPCXrsZCOZAPQaYlyKOnkMTh6FvGOQf/x0u+U1/zPusSinFLTuCLf80Xj+/RKjtOFqcfoR0RLiuhrLy0ql40DRYPINMt3996O8cEdLIlx1/2X2VWbRGfP2HillxdYz51dnzvJTzFl+ivTuIcRG2xjR20WrCCOe9u3b0759e3Jzc9m1axdZWVnk5OSQmJiI2+1mxYoVREVFER0dTXR0NFFRUXKpyULl7Y3c3r8LqitlAIy8DvqNNiq0T5kPm9dnuPcH2LMFSr3GcG3XBW56xJh+7xnjkparhdGiKiwCOpwDQ680lq9bbCSOEBeEhBmXtCJbnb6xryDPuOnP7qj6BhvBrCV5FBZr7rv07O3WvVElBaXUJcB0wA78U2v9dDD3/+Cs4/z9ZzE47A3/h5j2fm69X/vd9mIAFq0pBGBEr1BuGB5OqFMRFRVFamoqqamplJldORQWFnLixAmysrIqlE4GDBhAjx49KCgoYNOmTbhcLkJCQggNDSU0NJSYmBhcLpfnhjmbzSbNJ/2k/DD+7eM8Zt7XqnYvimplPHwZd5/xt7TEOIGXd+1RLmmEUUFemH/6UVRwevn3i88cyOjcNLjsbmP6zT8Y69vsxo19jhDoNRiGX2Ms/+BFo9TjMJc5Q4y+p7qlGK24Nn1lLnMaicXuoCSyHaVRsYTZ3UbCsjsocDsICbFjd4ZASGiTKt2s/KHY6hACrtF8GkopO/AKcCGQBaxWSi3UWm8JVgwauPcfp/9pusTauX5YOG2j7RzLc+MKUbRvGfwT51eZRVWWRioaesacZUuAJUfNZ6frJyLsJ8grCwVOcXmam7XbT3HghI0Iey6hthJiQk9SSiglzvbY7Q4GJpSweddRnHaNxkbrFsVEhZYR064LPTpFkHviBNt2HyGnIISYcDdtIty4sdEhLo4wVwhHj5/EXnqMMreixG0nLAQ0Cpsrli5tQ8jPzyMvP5/jp2xEuhQohVsr4tu1BJuNw0dP4VTFlLkh1Kk4lg8twyE6OgqbUhQUFlJQWIJSCqcdSspAKRtRkeHkFWqOnywipoXGYYPiMoXLaew/zBWKW0NJSQk2r4+0qFRx5KQmrpWTsjJNfmEZkWE2Qp2KMrcmr1AT4VIcz4c2UTbcbu0pGhw85sZugw0/nf41P2tJHlekhdEmykZpGSzdWEirSBuJbR20bGEjK6eMzrF2Coo0Px4uJalzCKVlRnLfdaiUjq3slLnBFWK8P21z8NOpSIpLI4iNtvPh0jw6t3Gw7+gArhkchtOusNvAYVdorSkp1bg1hNz+FDZ3GbnHTxFpK0KVFOC2h1JY5CbUqbAPu4aTecWcOOUm3nncaEkV0x6tNaUlboqLFargFOGlh6C0GF1cjAoNpzQhGUdJEUXL5qNROChjvepJsv6BP0c/zN6CY7x2C5TNfQqFZkrIkwwuW0cIxdg7JHLztT2NCvt/PYHb5kDZnSiHg5O2SCKHX2LciJizHz6bTYnNicOmUHY7GhuHeo+l1TldUUcPYF/7KW6bHYdNGYnNZjfGB28dBzkHOLklgwinG2W348ZGmbLj7DkAIlpyeM8hYo/9gLLbWLq/FbFhpfRtU4BOSEKFtYDj2XAkC+hifKg/bsSNjTd3dCKtu4ueUXnY8nPIK7LRMhxQNjQK1baT0aggPxd3QZ7RvFwp3G7FqRJYlxNFVLiNrIMFfLqxjCeuhJYtbOavCuUZpvbg4QL25ZQyINFu1DPZ7KcvS/qZqut170BRSg0BHtNaX2w+/x2A1vrPvl6Tlpam16xZU+d9fZ1ZxOxl+TWvKIQ4g8KNbuR9aUbofCLVKU44WlJSpinRp7s0CSvNocBhtOqLCS3kWJGrymUNFaqLKFKhDdqGy1GKdpdR5K64ndaOfK4d1ZaB3eu3faXUWq11WlXLGtMn2xHY6/U8y5xXgVJqslJqjVJqTXZ2dr12FOFSxMUYb/3yNBd/uzuGP1wnA8cLURstQvzbA21ESEnNK2GcIMslxFZfWu/aJZIO3TrSsbWdskoXRDrEOuhsnvfdthCiQ41LQnEtThLX6sxTYkyYUUpvHZqPovbjuWuHE7s680d3uM3YXvvwmi9FxYS7CQsBp6p4w0uX1orw0MCcvhvN5aPa0lq/BrwGRkmhPttITQwhNbFiZ2hdYh21v/YrhGgiIqqY5+v/vKb//8Z0fghcLI2ppLAP8G6aEW/OE0IIESSNKSmsBrorpRKVUiHAjcBCi2MSQohmpdFcPtJalyqlfgH8D6NJ6iyttQxFJoQQQdRokgKA1vpj4GOr4xBCiOaqMV0+EkIIYTFJCkIIITwkKQghhPCQpCCEEMKj0XRzUR9KqWzgp3q+vA1wxI/h+IvEVTcSV9011tgkrrppSFxdtNaxVS1o0kmhIZRSa3z1/WEliatuJK66a6yxSVx1E6i45PKREEIID0kKQgghPJpzUnjN6gB8kLjqRuKqu8Yam8RVNwGJq9nWKQghhDhTcy4pCCGEqESSghBCCI9mmRSUUpcopX5QSu1QSk0N8L46KaWWKaW2KKU2K6V+ac5/TCm1TymVYT4u83rN78zYflBKXRzIuJVSu5VSG80Y1pjzWimlPldKbTf/xpjzlVLqJXP/G5RS/b22M9Fcf7tSamIDY+rhdVwylFK5SqkHrDhmSqlZSqnDSqlNXvP8dnyUUgPM47/DfG2tBgD3EddzSqmt5r7/o5Rqac5PUEoVeB23GTXt39d7rGdcfvvclNG1/nfm/PeU0c1+feN6zyum3UqpDAuOl6/zg3XfMa11s3pgdMu9E+gKhADrgd4B3F8c0N+cjgS2Ab2Bx4DfVLF+bzOmUCDRjNUeqLiB3UCbSvOeBaaa01OBZ8zpy4BPMIapHwx8Z85vBewy/8aY0zF+/LwOYoyYHvRjBowE+gObAnF8gFXmusp87aUNiOsiwGFOP+MVV4L3epW2U+X+fb3Hesblt88NeB+40ZyeAdxb37gqLf8L8KgFx8vX+cGy71hzLCkMAnZorXdprYuBd4ErA7UzrfUBrfU6c/okkEkVY097uRJ4V2tdpLX+EdhhxhzMuK8EZpvTs4GrvObP0YZvgZZKqTjgYuBzrfVRrfUx4HPgEj/FMgbYqbWu7s71gB0zrfWXwNEq9tfg42Mui9Jaf6uN/945Xtuqc1xa68+01uWD+X6LMXqhTzXs39d7rHNc1ajT52b+wh0NzPdnXOZ2rwfeqW4bATpevs4Pln3HmmNS6Ajs9XqeRfUnab9RSiUA/YDvzFm/MIuAs7yKm77iC1TcGvhMKbVWKTXZnNdOa33AnD4ItLMoNjBG4PP+Z20Mx8xfx6ejOe3v+ADuwPhVWC5RKfW9UuoLpdQIr3h97d/Xe6wvf3xurYHjXonPX8drBHBIa73da17Qj1el84Nl37HmmBQsoZSKAP4NPKC1zgVeBboBqcABjOKrFYZrrfsDlwI/V0qN9F5o/rqwpN2yeb14HDDPnNVYjpmHlcfHF6XU74FSYK456wDQWWvdD3gQ+JdSKqq22/PDe2x0n1slN1Hxh0fQj1cV54cGba8hmmNS2Ad08noeb84LGKWUE+MDn6u1/gBAa31Ia12mtXYDMzGKzNXFF5C4tdb7zL+Hgf+YcRwyi53lRebDVsSGkajWaa0PmTE2imOG/47PPipe4mlwfEqpScDlwATzZIJ5eSbHnF6Lcb3+3Br27+s91pkfP7ccjMsljkrz683c1jXAe17xBvV4VXV+qGZ7gf+O1aYy5Gx6YAxBugujYqu8EqtPAPenMK7jvVhpfpzX9K8wrq0C9KFi5dsujIo3v8cNtAAivaa/wagLeI6KlVzPmtNjqVjJtUqfruT6EaOCK8acbuWHY/cucLvVx4xKFY/+PD6cWQl4WQPiugTYAsRWWi8WsJvTXTFOCtXu39d7rGdcfvvcMEqN3hXN99U3Lq9j9oVVxwvf5wfLvmMBORE29gdGDf42jF8Avw/wvoZjFP02ABnm4zLgLWCjOX9hpX+c35ux/YBXSwF/x21+4debj83l28S4drsE2A4s9vpyKeAVc/8bgTSvbd2BUVG4A68TeQNia4HxyzDaa17QjxnGZYUDQAnG9dg7/Xl8gDRgk/mav2H2MlDPuHZgXFcu/57NMNe91vx8M4B1wBU17d/Xe6xnXH773Mzv7Crzvc4DQusblzn/TeCeSusG83j5Oj9Y9h2Tbi6EEEJ4NMc6BSGEED5IUhBCCOEhSUEIIYSHJAUhhBAekhSEEEJ4SFIQohpKqTJVscdWv/Wqa/bGuanmNYUIHkfNqwjRrBVorVOtDkKIYJGSghD1YPa//6zZT/0qpdQ55vwEpdRSs/O3JUqpzub8dsoY42C9+RhqbsqulJpp9qX/mVIqzLI3JQSSFISoSVily0c3eC07obXui3GX6IvmvJeB2VrrZIwO6V4y57+E0Z1CCka//pvN+d2BV7TWfYDjGHfTCmEZuaNZiGoopfK01hFVzN8NjNZa7zI7NDuotW6tlDqC0Y1DiTn/gNa6jVIqG4jXWhd5bSMBow/87ubzhwGn1vqJwL8zIaomJQUh6k/7mK6LIq/pMqSeT1hMkoIQ9XeD19+V5vQ3GAMDAUwAvjKnlwD3Aiil7Eqp6GAFKURdyK8SIaoXpswB3U2faq3Lm6XGKKU2YPzav8mcdz/whlLqISAbuN2c/0vgNaXUnRglgnsxeu0UolGROgUh6sGsU0jTWh+xOhYh/EkuHwkhhPCQkoIQQggPKSkIIYTwkKQghBDCQ5KCEEIID0kKQgghPCQpCCGE8Ph/DMWs5nf+aCoAAAAASUVORK5CYII=\n", 451 | "text/plain": [ 452 | "
" 453 | ] 454 | }, 455 | "metadata": {}, 456 | "output_type": "display_data" 457 | } 458 | ], 459 | "source": [ 460 | "plt.plot(epsilon*90, alpha = 0.6, ls = '--', label = 'Epsilon profile (arbitrary unit)', color = 'orangered')\n", 461 | "plt.plot((np.array(mpm)-np.array(mp))*120, alpha = 0.6, ls = '--',\n", 462 | " label = 'Probability difference (arbitrary unit)', color = 'dimgray')\n", 463 | "plt.plot(loss_log, label = 'Loss', color = 'cornflowerblue')\n", 464 | "plt.xlabel('Epoch')\n", 465 | "plt.ylabel('MSE')\n", 466 | "plt.legend()\n", 467 | "plt.savefig('loss.pdf', dpi = 300, bbox_inches='tight')\n", 468 | "plt.show()" 469 | ] 470 | }, 471 | { 472 | "cell_type": "markdown", 473 | "metadata": {}, 474 | "source": [ 475 | "**Show the maze solution and the policy learnt.**" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": 88, 481 | "metadata": { 482 | "scrolled": true 483 | }, 484 | "outputs": [ 485 | { 486 | "data": { 487 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAADrCAYAAACICmHVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAE6UlEQVR4nO3dQW4bNxiA0VHSI6RpdvUdrPufQL6Du2tdX6FVdwVaSIEGMUV+9ntAFoaVETXyBxrwD+pwPp83YH2fZi8AuI1YIUKsECFWiBArRIgVIn7a8+AvX76cHx4eBi3lY3t6err5sY+Pj0Ouu8fsNYx6/tnXfX5+3l5fXw+XvnfY83fW4/F4Pp1ONz+e2x0OF9+fi/a8Z3uuu8fsNYx6/tnXPR6P2+l0unhhvwZDhFghQqwQIVaIECtEiBUixAoRYoWIXRNMe5T+GD/7D/x71zDKqDXMfm3v5XXZWSFCrBAhVogQK0SIFSLEChFihQixQoRYIUKsEDFs3HCFEb4RRo2YjTonaIU1jLjuCmvd4y3eMzsrRIgVIsQKEWKFCLFChFghQqwQIVaIECtEiBUicqcbzlZ7XbXRxBHXnL3Wt2JnhQixQoRYIUKsECFWiBArRIgVIsQKEWKFCLFCxK5xw6enpyEfUDzK7JPyRln5BL5LZp9uOMq9f27srBAhVogQK0SIFSLEChFihQixQoRYIUKsEDHswLQ9VphGGaE2GTXKiMPN3vP9usbOChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiF3jho+Pj9vpdHrzRcweHauNO86+XzW1A+ausbNChFghQqwQIVaIECtEiBUixAoRYoUIsUKEWCFiic9nHXUK4OyT8kr3YK8V7tnMa26bz2cFrhArRIgVIsQKEWKFCLFChFghQqwQIVaIECtELHG64Si3jnitMOpXu+4Ka5jN6YbARWKFCLFChFghQqwQIVaIECtEiBUixAoRYoWIJU43HGX26NoK92CPFda7whpu5XRD4CKxQoRYIUKsECFWiBArRIgVIsQKEWKFCLFChNMNd6p9OPHskctta51uuPK4o50VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRTjfc9q11hdG50r0dZYV7cG92VogQK0SIFSLEChFihQixQoRYIUKsECFWiBArROwaNxxlhRG+Ec9fel3b9n5PWHwvY592VogQK0SIFSLEChFihQixQoRYIUKsECFWiBArRCzxYcqjRvhGGPX8tVG/2e/DKKNGRN/iftlZIUKsECFWiBArRIgVIsQKEWKFCLFChFghQqwQMezDlFcwYq0rnFi4wqhf6eegNiJ6jZ0VIsQKEWKFCLFChFghQqwQIVaIECtEiBUiljgwbZQREyalyZ1tW/sAsHt5L5+pa2eFCLFChFghQqwQIVaIECtEiBUixAoRYoUIsULEsAPTVhjbunUNtTG72sjjiPtbuwe3vq7j8Xj1e3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIECtELHG64agRvtkjaSuMMa4w9jnCCmOf92ZnhQixQoRYIUKsECFWiBArRIgVIsQKEWKFCLFCRO50w484ZvZ/K4wmjlIaebw3OytEiBUixAoRYoUIsUKEWCFCrBAhVogQK0SIFSKWON1wlNnjcyuMXM7+oOq9j+U6OytEiBUixAoRYoU7e/n8bdsOh3//vXz+dtP/Eyvc2de///ju19eIFSLEChFihTt7+fTLd7++ZtdQBPDjvv71+3+/vvH/2Vkh4rBzdO3Pbdt+G7cc+PB+PZ/PP1/6xq5YgXn8GgwRYoUIsUKEWCFCrBAhVogQK0SIFSLEChH/AFuQjEBDA4n2AAAAAElFTkSuQmCC\n", 488 | "text/plain": [ 489 | "
" 490 | ] 491 | }, 492 | "metadata": {}, 493 | "output_type": "display_data" 494 | } 495 | ], 496 | "source": [ 497 | "net.eval()\n", 498 | "agent.isgameon = True\n", 499 | "agent.use_softmax = False\n", 500 | "_ = agent.env.reset(0)\n", 501 | "while agent.isgameon:\n", 502 | " agent.make_a_move(net, 0)\n", 503 | " agent.env.draw('')\n", 504 | " clear_output(wait = True)" 505 | ] 506 | }, 507 | { 508 | "cell_type": "code", 509 | "execution_count": 89, 510 | "metadata": {}, 511 | "outputs": [ 512 | { 513 | "data": { 514 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO0AAADrCAYAAACFFBGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWbklEQVR4nO2dP2hdx/LHR7/8QAlOZfKiNMkvhd3YAhWyESgYQ0CkCypDEL4IkdaVQFV6gYs0dnBpoVQubJQgBblSIUso0Q12iIsQEmKDiR5JIWPLlgprX+Gf/K6k82fn7M7OzJ75wIH33t03Z3Z29175nu/9Tp9zDgzD0MP/cCdgGAYOO7SGoQw7tIahDDu0hqEMO7SGoQw7tIahjP/FDH7nnXfchx9+SJRKu+l2u95jh4eHSeJi4M6B6v7ccf/880/4559/+ioHOee8r+HhYSeN7e1td+/ePe40ggEA7+soVTXAxNWUA9X9ueP+/xmrPIfq/zz+7bff4KuvvuJOgxUJNeDOger+EuNGO7Q///wz/PHHH6xjy9jf34fvvvtOzVjDqCLaod3d3YXx8XGvA0Y1toj9/X2YnJyE1dVVFWMNow7UF1EHfPPNNzA7O3vsf//rr7/gs88+gx9++IF8rC/Xr1+H+fl5OHPmDCwuLh567fTp03D79m1RYw2jlrp/9DrPL6IePnzohoaG3OrqaumY2GPX1tbcxsaG63Q6bnNz0+3u7h4b8/TpU3fx4kV348aN2ntxjoWGX2rU1QATV1MOVPfnjuvzRVS0Q3vnzh139+7d0tcpxs7MzLiRkRE3ODjohoaG3NbWVuG4Z8+euStXrnjdj2ts0w1QV4MUh5YjB6r7c8dNemi5mJ6edqdOnXKPHz/mTiWIphvAueoapDi0HDlQ3Z87rs+h7XOI39OeO3fObW5ueo9PxcuXL+GNN97gTiOIvr7q5+m9FK1ZWQ0wcTFw50B1f+64586dg83NzcrAjb6I8iHlZik7sL45YN64Us4LQ4w3LaocQuNy3z913DrUiysMo20EHdonT57ATz/9FCuXxjncv3+fJC733HyhqoFBR8iaNT60T548gU8++QRGR0fh+++/bxomGAqZmZS5HVCnCqOWEEpQu+U2lkXG+MUXX8Do6Ch8/PHH8OWXX8KjR4+ahhKHtLmFqsJS3t/G4sdiafxF1NzcHDx48ACuXr0Kt27dgjfffDNmXqxwzo1CFUZ1fxuLHxuFumdCvdfR57Q//vij63Q6x55BYZ9hYa6jxMihiLK4qebVS52CLNY6NL2/jcWPLVuz7H+at76+Dvv7+wDw6gfJe3t7zBnR8Ouvv8LXX38NH3300bHXUtSg6v42Fj82eM3qTrUT/EkbS2aGeSdMMS8MVFI7g45QGaPqT9rZ2Vm4cOEC7O7uwtLSEgwMDHCnlByrgT5C14xMxqhJPoepgVRFFJXUzqBDnIwx5UFIqTum2tihhytGDagOOEVcCbliiLlmqv88Now2ov7QmoxRRq5UOVDElVKvxvu27puq3qvq97T37993v//+e+FrL1++dN9++23p/xcCvuGkeE67vb3tRkZGXH9/v1taWirNu25emLFNa1CXKyauhBx840rIlWLfJv32uEy2pdHUzEfGKMXYTYLkkioHirgS6hVM3al2BZ+08/Pz7uzZs8eukydPuvPnzx9657h27Zrr6+srHD8+Ph70rln1joWJe5Tnz5+/jvvixYvC2D7zSlGDulwxcSXk4BtXQq4U+9bnk7bRt8cTExMwMTFx6H979OgRfPrpp8d+uXDp0iW4efMmTE5OQqfTaXK75Lz11luv/3OZ7hgzL8oa+ORKDVUOFHEl1CuUaH8el8m23n77bVhcXIS///471q1ewyljxMyLsgaGPkL3bbTntGNjY6WvnThxAqanp2Pd6jULCwuwsrICOzs7MDU1BcvLy0kVQZh5UdXA0EfovlX9yMckfIZGxMoYUUkwu+VhakBFDgqfGDn4xs2hXuJkjFSTLyKljDHlvGJA9YbErWnmftOIgckYDaMlmBtjRVzuuflCKSHkdnnUtA4YzI0xczfGKkJyle7y6Du3VL1/zY1RMJrmFpKrdJfH2HLSUKS4MTb+wYB0ORgmh6PEks+lqIGP5NI5nPQ0Zm0P8m1y/1hy0ia1xeQbs7bkXfO4PaJi5YApKveBxdagCg0uj3Vzo+z92yRf37EifuXDQVvcGKnQ5PJYRkqJqBQ3RrLntCngljFqp0p6mqK2VffHkEoiismXsraqP2lNxkiH1ZYOsTJGbjkYJgdMDSQoojD5hpKDy6PUXJPIGLvdrkg9b2jTX6mLWoYGx8BeuLXHVHA5aCb781iCuoYKirlJqJcENRJ3DhLVZskOLbW6hhOKuXHXS4IqjDsHyvuzKKKo8JWkYaRrqWRuGDCSOA4kqMK4c+C+fxmiDq2vJE2KE2II3BLCOubm5uDzzz+Hd999F1ZXV+GDDz5oXQ7c9y9D1HPa69evw/z8PJw5cwYWFxcPvXb69Gm4ffs2ahx2LBXcjaKbIMEAjTsH7vuXkeTQrq+vv/6mrNvtwuDgIPT39x8b5+taKMUJEcBvbhj3St+Yhl5C1zfJn8cLCwtw+fJl6Ha7MDU1Bdvb24XjfCVpkpwQfed2lCqZW9OYhg6C17dOnNx7QYDwenp62p06dco9fvy4VmydmpB5OUczt6qYmHxD5ib5ByHc98fmcJSy9fX5wQBKEdXX1+c9uChumbqGmxgP/ynmFkONhAG5F1hz4L4/Ngff88Bq7FZE2aaWoG4JhULqJ/ENrgkUBwETUyriFVGGYcTBZIwRyLmHqjQJX2pE1qDuH729Fwj4Bz33dZRce6hS9ZyNtRdSwFGD7J0rJJBrD1UJOTQllmui1BrYoQ2EQuomQT4nIYemxHJNlFoDUTJGjeTaQ1VCDj5gJKJYOanUGoiSMRr5E3svYCSiWDkpFaE1SHJozYDNOCDFXjiQiI6OjkYdGwsVxm5mEmYckGIvjI2NeR9CzNhYJDV2o5IxalJEIevFHhdDyrlZT+FEMsbh4WEIaSodasBGhaY3DQD+esUgh57CoetgMkbDaAnqD60ESRy3jFFbj14q90pu2ScGFW6MVOToWoiNqa1Hb+x8uV0bmyDCjTFmw92mY8ugdGPULGPUKOErQmKulG6b0Q4tVcPdUNdCajdGzTJGjRK+IiTmSum22UhcQSUdo3AtpHZj1CJjzEXCVwR3rsndNut+BtR7HW0q3UvMhru+Y9fW1tzGxobrdDpuc3PT7e7uHhuToulwDB8j35hNalAGR1Ppunwp6oXJNXQv+NS2qgZJf5oXq+EuZqyPq13KpsMchDj7xWjqjKUtTpOkbpt1p9p5ftJyEcMJERR/0jpH53SZ2okQE5cq19C94EsyN0ZMf9qUhDohSpAbYuIWkdrpUpOMUaoiSnV/WirXQm4dq6YaYAmtWUpJK9X6Zt+f1jCMOCR1Y9QiiWuSQ46NjzWhrQbi3Rhzdvarm5svEmobsg6cxFoDLBT7Vowbo0SZmS91cjTfuVFJKaXVlkPOiqmB9mbkAIn+PJYoM/OlTo7mMzdKKaW02nLIWX1rkEMzcoBEHlHcMjNfmsjRfOZGKaXkrK0UOatvDbQ1Iy+l7u9nF+HftFV/w4f824BKEteLj+SyTghBLaWUJCzgkLPW1eAAn3XAjKXYt2L+TUtFCklcDKlf7lLKXjjkrL5IaUaupqm0JkkcFozksAwJtQ3JgZsYa4CFYt+KaypdEdd7bFFcc/aLExeDhJpx0wo3RipCJXESDoy2uBJy4Cb0jctkjIbREkzGGCkHCjdGKofFHOulDfFujJqc/bBQzY1qXrnWSxvi3Ri5pXba3Bhzpi31Uu/GyCm10+jGmDNtqZdqN0YAXqmdRjfGnMmxXqndGBsdWk3NeS9dugQ3b96EyclJ6HQ6yXILgaoJtzX3pgF7HkLXgcWNMRa5ujFSyTPb4oQogSzcGDlkjL6E3L9qblTzkhA3JAcO2WWKeWFQ78ZIJWNMdX+quDGacFPFxUBVM24w8ypCtRtjKKmdCH0JrQGVJarU5t5ScvDF3BgNw/DCZIyRcuBsKq2NnOfmi8kYFcryqhQzqaR+vqqdmGZtvnOjMmCTYuwmQsZYRc7StaZzq1LMpKqXr2onplmbz9yoDNhyMXZL4lzx/Pnz11/zv3jxIuhr/qOkfnxwlLq5Oefc/Py8O3v27LHr5MmT7vz58+iYTfL1zQGTK3ZevvW6du2a6+vrK4w9Pj4uZizFvvV55KPebob70NblUAZHb1hsDthxvmNjGOFJGMt1aFV/e7y+vg77+/sA8Opx1N7eHnNG/nAoyJrmkIsBmxRjt+B9W3eqneBP2pmZGTcyMuIGBwfd0NCQ29raahy36bzq5taUFJ+0XHCYsFFAsW+z/6SdnZ2FCxcuwO7uLiwtLcHAwAB3SoZRS+i+NTdGJJh5UZGyWbVRjrkxBpBSlidBn4uBW8oJQPcGTkHKNzmTMRpGS1AvY6SCWx6prV4YNM1Novw2ibGbRlkepzwyZ8dCTXOTKr9NYuymWZbHgck+ZSA217pnQr3XgXNFrrK8XmK4UTSZV+x6Sbg0zY0qV9/95fOcNomxG4UDn28OUkzoMOToWHiAprlJzVW1sVvTHOrGaZZHGvIJ3V+NPmmLGBsbixWKPIe6cQsLC7CysgI7OzswNTUFy8vLprYyohG6v+w5bQEmjzQoSSpjlOrGSEWRzEyCIopbNYRF09yoci2K22o3xjJCcy2SmUnYgJpqS4WEGoQiXsZIqSzRoq7BkOu8AHTNTWSudc+Eei9o+Fxqe3vbjYyMuP7+fre0tBTteVesuEVsb2+7e/fuFb7mC9W8UlFVA4lrFhuqfXsQu6i2Yn5PS6UsoVSs+MjMqNz6QuYV0zmRQsqZQmUUqwbc+6uMJIeWqidprn1vQ+YV0zmRghRrFqsGUnvpRntOWwWVsiTXvre+88L0RU3dQ7WM2GtGWQOpiqgkhzZHJPS9xUg0Nco5fWhjDUxcUYCPzExq39tYzomapZyp3SOxiJEx5oSvzOzEiRMwPT3NkGE5GDlp1VjNUs5YNaAiuLZ1Xy+7CI986uCOW4SEZtXcUDW2rkJTvUJqkKyptFQ3xtC4ZTGlNqtOCVVj6zJyVNwBFOebRMZIBdUBD0XiZqWkKN8Yja255yZVTtpaGSMVFMZu3GZxWDStmaZcAYQYu1WhyczrAAo1EHcvXQya1kxTrgeIV0Rh5GCUjXwNf8SamhUgMVeMlBKLKBmj6Ea+LUOqhK8IiblSSkRFyRgppYEGDqkSviK4c00tERXx7fEBEqSBAK8UKwff7HW7XRgcHIT+/n5xMQ0ZYOWRoXtBlIxRijRwYWEBLl++DN1uF6ampmB7e1tkTEMuVfLI0L0g6tACyJAGUhi7mVlcuxgbG4PR0dHC17LoT4shZc/ZUGM3XxUMNi4VmsQgEnIN3eNNFVHiPmklEUMNlCKmoZOme0FEU2lNGl2q+2v6ywSAfx2okKCZr0O9jFGbNNCgQdv+arWMUZM00KBB4/4SIWPkcrXzASONzHlsU2I6PFKM5d5fqUnSVFqLa2LOY0OgcnjM3TWRikaKKE2udhhpZM5jfaFyN2yjayIZddYWvddBJ/giHj586IaGhtzq6mrh62Wdr51rbtuxtrbmNjY2XKfTcZubm253d/dY7KdPn7qLFy+6GzdulOae89imte2lbm0ljOXYXxRxk3YY4HC185GDYaSROY8NgcrdULprIpX0NDhu3al2np+0dVC8EzoXx4AtZ0JqqwmO/UURN7qxm9T+tKEGbDmjSbhChQTpqW/c1sgY7cAalFDtryQyRkxTaQmk/HFBqvtjc6BC0z6gqhfXOqj/pJUgY+R2ArSG3XRI2F9HUX9ouWWM3E6AVPfnnpcUuPdXEeoPLTfcEjqNDbslEcs10VdOGkN2aoc2EG4JXY4Nu1MSwzXRV04aS3YqythNI9wSuhwbdlNB5ZroKyeNJTtVfWjN4dDAQOWa6OsiGsttVPWfx+ZwaIQSwzXRV04aS3aq+tCaw6ERSizXRF8X0Rhuo2RujFRQycxC7l+GBHFFzq05fQkVQaSUyfrIGEVojzUhYbNSKXFy1Slre+NyzuWvPTaMNpHUjVGaHCwXJMgNc11fifNKdmglysFyQIrcMNf1lTivJG6MGCjkYNLcDWOiqWG3rVkckrgx+kIhB5PobhgTLQ27bc3ikcSN0RcKORi3uyE1Whp225pFpM6Ppvdq6sbo45ronL8ToRZ3Q6rrKFX+SFRz60W6KybVvAjXl9+NkVMOJtHdkAPKuWlxxcQiViZbd6qd5ydtHbm4JoLST9rYczuK5PWlmhfh+spxY8zBNdEUUeU5SF1fCW6MGFyNIkpEJ3huja42+Z62Nw6K+nLvGWwOvnFbY6FqGG2iVTJGCXI/XyTUCwNl82Vup8tWNpUG4JeDSZH7+cJdLywU+UpYs6ybSksnhbsgVUPlttIWR0gsrTm0KdwFqRoqt5W2OEJiUW3shiG2uyBlk2TjFTk6QsYgySft+vo67O/vA8ArV7u9vb0UtyVlYmICfvnll0PX0tISvP/++8f+rYIZC6CvXtry9YVqXqFxk3zSLiwswMrKCuzs7MDU1BQsLy9nacJ2IOUsMwrzHautXtry9YVqXsFx6yRTvRfok4MdokzuFxKTCo56SVjfo8RYM4nzStZUOlQRxSAH8xonVRGVul4YqNaXYs0wpJxX06bSqD+Ph4eHIUR7XKZLpToI3Js7dFFj6HiRb8pB90qpO0755imtqXRrHvkYRi6oP7QS5H4UUjuJ8jmuuFqkpxhEyBip1EB1Y7nlflRSO4nyudRxfWur0TBOhIyRSg0kXTlkUjs6fGrbRsM4cmO33JVDc3Nz8ODBA7h69SrcunXLlDsR8altKw3j6p4J9V5Njd0ox1ZZrUCC53hVOYTEjTWvnOM6p9MwrmpePs9pkxi7UY01+VzecX3QaBgXXK+6U+08P2k5mJmZcSMjI25wcNANDQ25ra2tQ6+D0k/amPPSHLeqthKgqFfST1oOcm0qTTUvbXFzJbReWfSnzaGpdFHclPI5yXGlokLG2O12vRNNubkl2nbGgEr2mTouBm7paUpMxmgYLSGpGyOFHC1XGaM2JKyDJkTIGKugdNXLVcbYC5dEFDOOex20IULGWEXOUr8Uc+OWiEqXkraNJHYzOUv9Ys+NWyKqUUraOuoe5PZeoEzmhonb9P5VOYTGPYBLIooZl3odtF2+9cpeXJGrjPEoHBJRzLi2rEMsVLgxUpGrC+BRxsbGWMfWjWvLOsQitF6qP2lNPicDWwccSWWMEvrTFsXNVcaoDcnukdz47ltxboxUpHZ59EXCAU9pNSp1HaQiXsaYs2qIe27c9zfSol4RxQ333Ljvb6QniRtjKkVUTFme71jfuVE5+0lTm3FLLjWOxZLEjTFVn1EKWV7dWJ+5UTr7Sevhyi251DD2vfcA+vr+e733Xm24Q5C7MQLQ9BmlkOU1kfD5zI3S2Y+zhyu35FLjWACAf/8bKv97LXWSqd4rxI2RSsaIyQE7zndsDMfAkBpwSAhD6tX2sQDHrwN8ZIzRDu2dO3fc3bt3S19PcWjrcsCO8x3rYz727Nkzd+XKldLXczm0sWub41gxh7aOFIeWixiOgbkcWqOegYHDB3Zg4L+vRe9PS2Xs1jblUBE5iCuMcHwUUaq1x4bRRrDa478B4CFdOobRev7POfevqgGoQ2sYBj/257FhKMMOrWEoww6tYSjDDq1hKMMOrWEoww6tYSjDDq1hKMMOrWEoww6tYSjjP/B1t4mP6K/+AAAAAElFTkSuQmCC\n", 515 | "text/plain": [ 516 | "
" 517 | ] 518 | }, 519 | "metadata": {}, 520 | "output_type": "display_data" 521 | } 522 | ], 523 | "source": [ 524 | "agent.plot_policy_map(net, 'solution.pdf', [0.35,-0.3])" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": 93, 530 | "metadata": {}, 531 | "outputs": [ 532 | { 533 | "data": { 534 | "text/plain": [ 535 | "" 536 | ] 537 | }, 538 | "execution_count": 93, 539 | "metadata": {}, 540 | "output_type": "execute_result" 541 | } 542 | ], 543 | "source": [ 544 | "best_net = copy.deepcopy(net)\n", 545 | "best_net.load_state_dict(torch.load('best.torch'))" 546 | ] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "execution_count": 147, 551 | "metadata": {}, 552 | "outputs": [ 553 | { 554 | "data": { 555 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO0AAADrCAYAAACFFBGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWF0lEQVR4nO2dT2hdxR7Hf3k+qFJX4jNu6nPRbmogi1QCKUEQgjvpskjIJQS3XQWych/Iwk2VLg3tqouUKIkkqyzyh9RcUWkWIogWSvPQRYqJ3i6aeYu8+G6Se86ZOTO/f3N+Hzjwnhnm/GbmzL1tzqff6XPOgWEYevgHdwGGYYRhm9YwlGGb1jCUYZvWMJRhm9YwlGGb1jCU8c+Qxq+//rp7++23kUppNu1227vt0NAQSr8hcNeAdX/ufn/55Rf4/fff+0obOee8r6GhISeN/f19991333GXEQ0AeF9nKZuDkH411YB1f+5+/7fHSveh+j8e//TTT/Dpp59yl8GKhDngrgHr/hL7TbZpf/jhB/j5559Z2xZxdHQEX331lZq2hlFGsk3b6XTgxo0bXhsMq20vjo6OYHJyEtbX11W0NYwqgn4RdcK9e/dgdnb23H9/+vQp3Lx5Ex4+fIje1pc7d+7A3bt34erVq7C0tHTqZ1euXIEHDx6IamsYlVT9pdd5/iLq119/dYODg259fb2wTeq2m5ubbnt727VaLbezs+M6nc65Nn/88Yd777333BdffFF5L862UPOXGlVzENKvphqw7s/dr88vopJt2tXVVbexsVH4c4y2MzMzbnh42A0MDLjBwUG3t7fXs93BwYGbm5vzuh9X27oPQNUcUGxajhqw7s/dL+mm5WJ6etpdvnzZPXnyhLuUKOo+AM6VzwHFpuWoAev+3P36bNo+F/Dvaa9du+Z2dna821Px4sULeOmll7jLiKKvr/x9eje91qxoDkL6DYG7Bqz7c/d77do12NnZKe241i+ifKB8WIo2rG8NIR9clOMKIcWHFlYNsf1y35+63yrUyxWG0TSiNu2zZ8/g22+/TVVL7Rq+//57lH65x+aLplqNY2Ke29qb9tmzZ/DBBx/AyMgIfP3113W7iQZDM5MythPKrDCKWiXYbrm1ZdEYP/74YxgZGYH3338fPvnkE3j8+HHdrsQhbWxlVhhFrRJst5zbhlL7F1Hz8/Owu7sLt2/fhoWFBXj55ZdT1sUK59hCrbDUtUqw3XJum4Sqd0Ld19n3tN98841rtVrn3kGFvsMKuc6SooZeFPVLNa5uqgyyVOtQ9/7WNrxt0Zpl/0/ztra24OjoCACO/0Hy8+fPmSvC4ccff4TPP/8crl+/Lv7+1ra6bexzi/aeloLFxUVYW1uDw8NDmJqagpWVFejv7+cuKzljY2Nq7m9tq9vGPreqv2lnZ2dhdHQUOp0OLC8vZ7lhjfyIfW7RNEZN+lzIHEg1ooqIVe0MPMRpjJQbgdI7znlzYdWA0a+EWkPoVYNpjIbREExjLOmXe2y+SKgVqwaMfqXMV+3ntuqdUPfV/Z52f3/fDQ8PuwsXLrjl5eXSd1YvXrxwX375ZeHPQdh7Wt+xVY0rpG3dOaiqNaRfCTX49iuhVoznFvU9ra8+pzHUzGdsUoLdJCiXWDVg9CthvqKp2tWu4Jv2zz///PvT4q+//jr3iXHCZ5995vr6+tw777xz7rpx40bUp2bZJ1ZIv2fxGZvPuCjmoKrWkH4l1ODbr4RaMZ5bn2/a2r89fuWVV/7+32W+68TEBNy/fx8mJyeh1WrVvR0pPmMLGRfmHPiuAyZYNWD0K2G+YkH/7fGrr74KS0tL8NtvvyXvm1NjDBkX5hwY+lChMV68eBGmp6eT98utMYaMC2sODH2Yxmgao6EMsRpjUBHMaXkhc4BFDoZPihp8+81hvsRpjFiD7wWlxkg5rhRgfSBxa5fcHxopMI3RMBqCaYwl/XKPzch3HSyNMfM0xhBSJAZSnaVbVavvOkipN6StpTEmRvPYYhMDKbXTqlpT66SxSEljRNMYgVkHC6nhLKn0Oew5uHv3bk818rXXXnPvvvvuqbp92/rqmaH1htTqsw4h9UqZ225iNEZLY+zRZ1m/3Bu2qN4TYhMDfc/dTVGvT61l6+tbr5S59RmXpTE2kNjEQErlMkXKpNR6LY2xAG6NUSIpEgOplMtUKZMS67U0xgJMYzQ0IlZj5NbBQmoImQMJRlRIvVhItYx6IbVWEo2x3W6L9HljD/2VuqhFSPCJQ+B2j7GIHZd4jRHLXJJArmOTMC5uIwozsI7ciAoFw1ySQq5j4x4Xt5mGeX8WIwoLXyUtRF2j0txS1SChXglwm2nc9y9C1Kb1VdKkJCH6oq1eKczPz8NHH30Eb7zxBqyvr8Nbb73VqPsXIeo97Z07d+Du3btw9epVWFpaOvWzK1euwIMHD4LahbbFQlu9UuAOYeO+fyFVylT3BTV1sM3NTbe9ve1arZbb2dlxnU6np9rlq9D5tvNtW3dcvmPjrBdrzShqcI7/cG+MA7vL5laMxri4uAi3bt2CdrsNU1NTsL+/37Odr5ImKQnRZ2yS6vXFd82McKLntmpXuwTftM45Nz097S5fvuyePHnS81OLk5hxOUc/tpB6sdaMqoYcv2nL5tbnmzbIiOrr6/Nu3KvfInOJmxQv/ynHpsk2S1FDL7jvH1qD79yyBrv1ouihlmC3xBL7YIc8LJSk+CDC2AhS5ysE8UaUYRhpMI0xARhj49b3TmrACs3T8iyYxpih6geQfmzc+t4JWGum5VkwjdHwRqo+p4VUqYlS18E2rUCk6nNaSJWaKHUdRGmMxjFi9TmB3Lt3D2ZnZ8/996dPn8LNmzfh4cOHtdoCyF0Hkm/anAPYch0b1rhS9zs+Pg6PHj06dS0vL8OlS5fO/Z0xpC0mKoLdcg5gy3VsWOOimK+TJMSRkZGkbVMRPQdVylT3BcqUOIyrF0Vji+lXQqZ0qjXD6JcK0xiJlTgMiuar19hyPUM1tAaMfkPmCwsVGuPQ0BDEHCodG8CGRYpNQOlUU84X1rhyOFM4dh1MYzSMhqB+00pQ4iTUoIlctc8QVGiMWEhQ4iTUoIlctc8QRGiMKQ/crdu2CG3phljJjRhJlxKQqBumeG6LSLZpsQ7cjT2cV1u6IVZyI0bSpRQk6oaYh0rXkiuw1LFQzcwHbemGWMmNGEmXUuDWDTGe21KqXuR2X2cPle4m5YG7vm1TJSFCxEvyshrq9Js6uTG0Lfah0qlSHs8iQUbppu5zi34SfDerq6tuY2OjdCCp287MzLjh4WE3MDDgBgcH3d7eXs92BwcHbm5urvAeMQtVVkPdfqvqxW7r067u2KrWLJdNW/e5Jd20XKRIQoxdKAyNUToxY8PQGKVt2irINMaQ82kpiU1CxEpjzCHYrQhNGqNUI0r1+bRY53xSeqy9aqCcgxT9hhA7Z5RKK9YHYvbn0xqGkQbSNEZpqXa51KBN4cNA2xyI1xilptrlUINGhS81GudAhMZYhkTNzJdUOhqWGihtbjl01pA5yOFwb5JNK1Ez8yWFjoapBkqbWw6d1XcOcjncmyQjilsz8wVLR8NUAznnVorO6jsH2RzuXfUit/uCiBfP1IfzhvRbRpVyKUGllCQWcOisVXNwgpTDyFUcKo0FxcHHJ2l9169fr12DlIOiKaiaL4q2RUg53FvNodLUqXYh/cbCrVJK+qblwuebNjUYz624NMaSfr3b9upXQrKfBJUytt8QUsyZdhqRxohFrBKXYsNQHiotoV8JNXAT+8FlGqNhNATTGIXWkPOBztqUQwxMY8xQY8z1QGeNyiEGIjRGyYfzStXRmgj3s0CF+jRGTtVOso7WRKRpl1ioTmME4FXtROtoDUSL0hoCdRpjrU07Pj4O4+Pjp/7b48eP4cMPPyQ/nPfk1+btdhsGBgbgwoULp9pMTEzA/fv3YXJyElqtFksNEvrE7LfphO6H2HVI9sfjFJpZKBIUQgyVEkvPpNA+jWPK9gOpxhiTxsihMfoSc/+yGrDGJaHfmBo4tEuKcYWgPo0RS2Okun9RDdx6Jma/IYQ8Y00yolSnMcYSm8aIVUPsHFAf6Ey5ZkVIqMEXS2M0DMML0xiFkuu4AExjBDCNkV1jPCFlqBnFuHzrTTku32eB+4ze0LahiNAYy2iKukZ57m4KfGtIOS6fZ4H7jN7QttSQBLvNz8/D7u4u3L59GxYWFrIwYbjP3Q3Ftwbscfk8C9xn9Ia2JafqnVD3BQLezWH1W/f+3VAFlVHXmzqsrSoahvuMXt+2GM9t9sFuW1tbcHR0BADHr6OeP3/OWk+qoDKqcfnWm0sAm5Rgt+j1rdrVTvA3baoDimPGhUHKcUmEI4QNA4znNvtv2tnZWRgdHYVOpwPLy8vQ39/PXVISch2XcUzs+loaYyAh44oFS2M00mBpjBFQanmUfm4KjVGCy4v1AY4B5YecaYyG0RDUa4xYcGuE3PfHRNOzIFG/JQl2o0rgS6nlceuR3PfHQlMao1T9liTYjUpjxNDyjLRoUlql1koS7IahMWJoeQY+mpRWqbWSBLthJPD51iAlhM44RlMao9RaVQe71a2hqh23Hsl9fwOX2PVN9q98xsbGUnWFXkNVu8XFRVhbW4PDw0OYmpqClZUVUiuJ+/4GLrHra+9pe8CtEXLf38CFVGOUmsaIRWzCYgjceiYmuRpRKjRGqWmMRcTWGpuwGPsASk5N1LS5pCJeY8Q0S7TYNcYxmtZMYq2qg90wjZWcNUJfsA7LNiNKiMZYBpZZgmms+GhmVOfepkxDDGmLoVJSWEap5oD7+SqCZNNinUnalHNvsVIeOXROijVLNQdSz9IlSWPEMktyPPcWKw1Ris6Zes0w50CqEaU6IypVv71IkRgYM65uUqch+rbVtma+4wppSz0H2WdEYeGjmWGfe9sNVhqihERIDKjTI0MRozHmhK9mdvHiRZienkavJ0QRTdVWs0qJNV+piJ7bqq9il+iPx2Vw99sLCYdVc4N1sHUZmuYrZg7IDpWWmsYY229RnxIOq+aGOhEyR+MOoHe9JBojFlgbPBaJDysmverNIRFSwgn3vWisxogFlg2kybLSVG+Tni/VGiMmGDaQtrA2LfU27fkSpzFSqYFGPkgMYAtRKUMRpTFKPsjXkItE3RBTERWlMYo+yNcQC7duSK2Iivjt8QkTExNw//59mJychFarxVbH1tbW37/Za7fbMDAwABcuXBDXJyba6uUkNPEzdm5FaYyUamAZi4uLcOvWLWi32zA1NQX7+/si+8REW73SKNMjo+e2yr7ovgBJEg8hpIa6tTpXbKzE9JvKMMK6NNV7llRif+xz44t6IyoEyjNnY4PdevUrOaxNU72Bzy17Db2oa0SJ+uOxNFLYQBR9YqKtXk3UnVsRh0prcnSx7q/pTyYA/OuABdY3eMr5Uq8xalLtDDy0PV+N1hi1qHYGHhqfLxEaI1eqnQ8hamTObevClQbp25b7+aKG5FBpLamJObeNQUIapNTniwOSQ6W1pCbm3NYXCWmQmp4vFqpe5HZf/3vx2xOOVLvNzU23vb3tWq2W29nZcZ1O51zfPqmJObetO7fdcKVBhrTleL4w+iVNY+RItfPRwULUyJzbxsCRBhnTNhVYKiepxlj2TVsFVn5sigC2nImZW01wPF8Y/SbXGKWeTxsbwJYzmsQVLCSop779NkZjtA1rYIL1fJFojCGHSkuA8h8XUN0/tAYsND0HWPPFtQ7qv2klaIzcSYB2YDceEp6vs6jftNwaI3cSoMYDuzXB/Xz1Qv2m5YZbodN4YLckUqUm+uqkKbRT27SRcCt0OR7YTUmK1ERfnTSVdioq2E0j3Apdjgd2Y4GVmuirk6bSTlVvWksMNELASk30TRFNlTaq+o/HlhhoxJIiNdFXJ02lnaretLOzszA6OgqdTgeWl5fVHHpsyGFsbAxGRkZ6/izk+fI9YDzFQeRoaYxYYGlmMfcvQoJckfPRnL7EShCUmqyPxijCPdaEhIcVy8TJ1VPW9sHlnMvfPTaMJkGaxihNB8sFCbphrusrcVxkm1aiDpYDUnTDXNdX4rhI0hhDwNDBpKUbpkTTgd22ZmkgSWP0BUMHk5humBItB3bbmqWDJI3RFwwdjDvdEBstB3bbmiWkKo+m+6qbxuiTmuicfxKhlnRDrOssPseIYiY3Sk/FxBoX4vrypzFy6mAS0w05wBybllTMUMRqslW72nl+01aRS2oiKP2mTT22s0heX6xxIa6vnDTGHFITzYgqrkHq+kpIYwzBVRhRIk6C53Z0tel72j44MOaX+5kJrcG338ZEqBpGk2iUxihB9/NFwnyFgHn4MnfSZSMPlQbg18Gk6H6+cM9XKBj1SlizrA+Vlg5FuiDWgcpNpSmJkKE0ZtNSpAtiHajcVJqSCBmK6mC3EFKnC2Iekmwck2MiZApIvmm3trbg6OgIAI5T7Z4/f05xW1TGx8fh0aNHp67l5WW4dOnSub+rhLQF0Ddf2ur1BWtcsf2SfNMuLi7C2toaHB4ewtTUFKysrGQZwnaichYFhfm21TZf2ur1BWtc0f1WKVPdF+jTwU5RpPvF9IkFx3xJWN+zpFgzieMiO1Q61ohi0MG82kk1oqjnKwSs9cVYsxAox1X3UOmgPx4PDQ1BjHtc5KVibQTuhzt2UVN4vIEfylH3ovSOKT88pR0q3ZhXPoaRC+o3rQTdD0O1k6jPcfWrRT0NQYTGiGUDVbXl1v2wVDuJ+hx1v75zqzEwToTGiGUDSTeHTLXDw2dumxgYhx7slrs5ND8/D7u7u3D79m1YWFgwcychPnPbyMC4qndC3VfdYDfMtmVRK0DwHq+shph+U40r536d0xkYVzYun/e0JMFuWG1Nn8u7Xx80BsZFz1fVrnae37QczMzMuOHhYTcwMOAGBwfd3t7eqZ+D0m/alOPS3G/Z3EoAY75Iv2k5yPVQaaxxaes3V2LnK4vzaXM4VLpXv5T6nOR+paJCY2y3296FUj7cEmM7U4ClfVL3GwK3ekqJaYyG0RBI0xgxdLRcNUZtSFgHTYjQGMvATNXLVWPshksRDWnHvQ7aEKExlpGz6kcxNm5FVLpK2jRI4mZyVv1Sj41bEdWokjaOqhe53Rco09xC+q17/7IaYvs9gUsRDWlHvQ7aLt/5yl6uyFVjPAuHIhrSrinrkAoVaYxY5JoCeJaxsTHWtlXtmrIOqYidL9XftKbPycDWIQxSjVHC+bS9+s1VY9SG5PRIbnyfW3FpjFhQpzz6ImGDU0aNSl0HqYjXGHO2hrjHxn1/gxb1RhQ33GPjvr9BD0kaI5URlVLL823rOzasZD9pthm3cqmxbSgkaYxU54xiaHlVbX3GhpnsJ+0MV27lUkPbN98E6Ov7//Xmm5XdnQI9jREA55xRDC2vjsLnMzbMZD/OM1y5lUuNbQEA/vMfKP3/lVQpU91XTBojlsYYUkNoO9+2KRIDY+aAQyGMma+mtwU4f53gozEm27Srq6tuY2Oj8OcUm7aqhtB2vm19wscODg7c3Nxc4c9z2bSp5zbHtmI2bRUUm5aLFImBuWxao5r+/tMbtr///z9Lfj4tVrBb08yhXuQgVxjx+BhRqt1jw2gioe7xbwDwK145htF4/u2c+1dZg6BNaxgGP/bHY8NQhm1aw1CGbVrDUIZtWsNQhm1aw1CGbVrDUIZtWsNQhm1aw1CGbVrDUMZ/AV8KrKQ+sGjtAAAAAElFTkSuQmCC\n", 556 | "text/plain": [ 557 | "
" 558 | ] 559 | }, 560 | "metadata": {}, 561 | "output_type": "display_data" 562 | } 563 | ], 564 | "source": [ 565 | "agent.plot_policy_map(best_net, 'solution_best.pdf', [0.35,-0.3])" 566 | ] 567 | } 568 | ], 569 | "metadata": { 570 | "kernelspec": { 571 | "display_name": "Python 3", 572 | "language": "python", 573 | "name": "python3" 574 | }, 575 | "language_info": { 576 | "codemirror_mode": { 577 | "name": "ipython", 578 | "version": 3 579 | }, 580 | "file_extension": ".py", 581 | "mimetype": "text/x-python", 582 | "name": "python", 583 | "nbconvert_exporter": "python", 584 | "pygments_lexer": "ipython3", 585 | "version": "3.6.9" 586 | } 587 | }, 588 | "nbformat": 4, 589 | "nbformat_minor": 2 590 | } 591 | -------------------------------------------------------------------------------- /environment.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.special as sp 3 | import matplotlib.pyplot as plt 4 | import copy 5 | 6 | class MazeEnvironment: 7 | def __init__(self, maze, init_position, goal): 8 | x = len(maze) 9 | y = len(maze) 10 | 11 | self.boundary = np.asarray([x, y]) 12 | self.init_position = init_position 13 | self.current_position = np.asarray(init_position) 14 | self.goal = goal 15 | self.maze = maze 16 | 17 | self.visited = set() 18 | self.visited.add(tuple(self.current_position)) 19 | 20 | # initialize the empty cells and the euclidean distance from 21 | # the goal (removing the goal cell itself) 22 | self.allowed_states = np.asarray(np.where(self.maze == 0)).T.tolist() 23 | self.distances = np.sqrt(np.sum((np.array(self.allowed_states) - 24 | np.asarray(self.goal))**2, 25 | axis = 1)) 26 | 27 | del(self.allowed_states[np.where(self.distances == 0)[0][0]]) 28 | self.distances = np.delete(self.distances, np.where(self.distances == 0)[0][0]) 29 | 30 | self.action_map = {0: [0, 1], 31 | 1: [0, -1], 32 | 2: [1, 0], 33 | 3: [-1, 0]} 34 | 35 | self.directions = {0: '→', 36 | 1: '←', 37 | 2: '↓ ', 38 | 3: '↑'} 39 | 40 | # the agent makes an action from the following: 41 | # 1 -> right, 2 -> left 42 | # 3 -> down, 4 -> up 43 | 44 | # introduce a reset policy, so that for high epsilon the initial 45 | # position is nearer to the goal (useful for large mazes) 46 | def reset_policy(self, eps, reg = 7): 47 | return sp.softmax(-self.distances/(reg*(1-eps**(2/reg)))**(reg/2)).squeeze() 48 | 49 | # reset the environment when the game is completed 50 | # with probability prand the reset is random, otherwise 51 | # the reset policy at the given epsilon is used 52 | def reset(self, epsilon, prand = 0): 53 | if np.random.rand() < prand: 54 | idx = np.random.choice(len(self.allowed_states)) 55 | else: 56 | p = self.reset_policy(epsilon) 57 | idx = np.random.choice(len(self.allowed_states), p = p) 58 | 59 | self.current_position = np.asarray(self.allowed_states[idx]) 60 | 61 | self.visited = set() 62 | self.visited.add(tuple(self.current_position)) 63 | 64 | return self.state() 65 | 66 | 67 | def state_update(self, action): 68 | isgameon = True 69 | 70 | # each move costs -0.05 71 | reward = -0.05 72 | 73 | move = self.action_map[action] 74 | next_position = self.current_position + np.asarray(move) 75 | 76 | # if the goals has been reached, the reward is 1 77 | if (self.current_position == self.goal).all(): 78 | reward = 1 79 | isgameon = False 80 | return [self.state(), reward, isgameon] 81 | 82 | # if the cell has been visited before, the reward is -0.2 83 | else: 84 | if tuple(self.current_position) in self.visited: 85 | reward = -0.2 86 | 87 | # if the moves goes out of the maze or to a wall, the 88 | # reward is -1 89 | if self.is_state_valid(next_position): 90 | self.current_position = next_position 91 | else: 92 | reward = -1 93 | 94 | self.visited.add(tuple(self.current_position)) 95 | return [self.state(), reward, isgameon] 96 | 97 | # return the state to be feeded to the network 98 | def state(self): 99 | state = copy.deepcopy(self.maze) 100 | state[tuple(self.current_position)] = 2 101 | return state 102 | 103 | 104 | def check_boundaries(self, position): 105 | out = len([num for num in position if num < 0]) 106 | out += len([num for num in (self.boundary - np.asarray(position)) if num <= 0]) 107 | return out > 0 108 | 109 | 110 | def check_walls(self, position): 111 | return self.maze[tuple(position)] == 1 112 | 113 | 114 | def is_state_valid(self, next_position): 115 | if self.check_boundaries(next_position): 116 | return False 117 | elif self.check_walls(next_position): 118 | return False 119 | return True 120 | 121 | 122 | def draw(self, filename): 123 | plt.figure() 124 | im = plt.imshow(self.maze, interpolation='none', aspect='equal', cmap='Greys'); 125 | ax = plt.gca(); 126 | 127 | plt.xticks([], []) 128 | plt.yticks([], []) 129 | 130 | ax.plot(self.goal[1], self.goal[0], 131 | 'bs', markersize = 4) 132 | ax.plot(self.current_position[1], self.current_position[0], 133 | 'rs', markersize = 4) 134 | plt.savefig(filename, dpi = 300, bbox_inches = 'tight') 135 | plt.show() 136 | -------------------------------------------------------------------------------- /maze_generator/maze.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/giorgionicoletti/deep_Q_learning_maze/5806405207d9f49d043047eae328c885e08dbfc6/maze_generator/maze.npy -------------------------------------------------------------------------------- /maze_generator/maze_generator.py: -------------------------------------------------------------------------------- 1 | # Adapted from http://code.activestate.com/recipes/578356-random-maze-generator/ 2 | # Random Maze Generator using Depth-first Search 3 | # http://en.wikipedia.org/wiki/Maze_generation_algorithm 4 | # FB36 - 20130106 5 | 6 | import random 7 | import matplotlib.pyplot as plt 8 | import numpy as np 9 | 10 | 11 | mx = 20; my = 20 # width and height of the maze 12 | 13 | maze = [[0 for x in range(mx)] for y in range(my)] 14 | dx = [0, 1, 0, -1]; dy = [-1, 0, 1, 0] # 4 directions to move in the maze 15 | color = [(0, 0, 0), (255, 255, 255)] # RGB colors of the maze 16 | 17 | # start the maze from a random cell 18 | cx = random.randint(0, mx - 1) 19 | cy = random.randint(0, my - 1) 20 | maze[cy][cx] = 1 21 | stack = [(cx, cy, 0)] # stack element: (x, y, direction) 22 | 23 | while len(stack) > 0: 24 | (cx, cy, cd) = stack[-1] 25 | # to prevent zigzags: 26 | # if changed direction in the last move then cannot change again 27 | if len(stack) > 2: 28 | if cd != stack[-2][2]: dirRange = [cd] 29 | else: dirRange = range(4) 30 | else: dirRange = range(4) 31 | 32 | # find a new cell to add 33 | nlst = [] # list of available neighbors 34 | for i in dirRange: 35 | nx = cx + dx[i] 36 | ny = cy + dy[i] 37 | if nx >= 0 and nx < mx and ny >= 0 and ny < my: 38 | if maze[ny][nx] == 0: 39 | ctr = 0 # of occupied neighbors must be 1 40 | for j in range(4): 41 | ex = nx + dx[j]; ey = ny + dy[j] 42 | if ex >= 0 and ex < mx and ey >= 0 and ey < my: 43 | if maze[ey][ex] == 1: ctr += 1 44 | if ctr == 1: nlst.append(i) 45 | 46 | # if 1 or more neighbors available then randomly select one and move 47 | if len(nlst) > 0: 48 | ir = nlst[random.randint(0, len(nlst) - 1)] 49 | cx += dx[ir]; cy += dy[ir]; maze[cy][cx] = 1 50 | stack.append((cx, cy, ir)) 51 | else: stack.pop() 52 | 53 | maze = np.array(maze) 54 | maze -= 1 55 | maze = abs(maze) 56 | 57 | maze[0][0] = 0 58 | maze[mx-1][my-1] = 0 59 | 60 | np.save('maze', np.array(maze)) 61 | --------------------------------------------------------------------------------