├── README.md
├── agent.py
├── deep_qlearning.ipynb
├── environment.py
└── maze_generator
    ├── maze.npy
    └── maze_generator.py


/README.md:
--------------------------------------------------------------------------------
1 | # Deep Q-learning for maze solving
2 | 
3 | A simple implementation of DQN that uses PyTorch and a fully connected neural network to estimate the q-values of each state-action pair.
4 | 
5 | The environment is a maze that is randomly generated using a deep-first search algorithm to estimate the Q-values. Four moves are possible for the agent (up, down, left and right), whose objective is to reach a predetermined cell. The agent implements either an epsilon-greedy policy or a softmax behaviour policy with temperature equal to epsilon. After each episode, the starting position is sampled in such a way that at the beginning of the training the agent explores the area surrounding the goal, and as the training goes on it will explore further and further areas of the maze.
6 | 
7 | A convolutional neural network is also implemented for completeness.
8 | 


--------------------------------------------------------------------------------
/agent.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import scipy.special as sp
 3 | import matplotlib.pyplot as plt
 4 | import copy
 5 | import torch
 6 | import torch.nn as nn
 7 | import torch.optim as optim
 8 | import collections
 9 | 
10 | Transition = collections.namedtuple('Experience',
11 |                                     field_names=['state', 'action',
12 |                                                  'next_state', 'reward',
13 |                                                  'is_game_on'])
14 | 
15 | 
16 | class Agent:
17 |     def __init__(self, maze, memory_buffer, use_softmax = True):
18 |         self.env = maze
19 |         self.buffer = memory_buffer # this is actually a reference
20 |         self.num_act = 4
21 |         self.use_softmax = use_softmax
22 |         self.total_reward = 0
23 |         self.min_reward = -self.env.maze.size
24 |         self.isgameon = True
25 | 
26 |         
27 |     def make_a_move(self, net, epsilon, device = 'cuda'):
28 |         action = self.select_action(net, epsilon, device)
29 |         current_state = self.env.state()
30 |         next_state, reward, self.isgameon = self.env.state_update(action)
31 |         self.total_reward += reward
32 |         
33 |         if self.total_reward < self.min_reward:
34 |             self.isgameon = False
35 |         if not self.isgameon:
36 |             self.total_reward = 0
37 |         
38 |         transition = Transition(current_state, action,
39 |                                 next_state, reward,
40 |                                 self.isgameon)
41 |         
42 |         self.buffer.push(transition)
43 |             
44 |         
45 |     def select_action(self, net, epsilon, device = 'cuda'):
46 |         state = torch.Tensor(self.env.state()).to(device).view(1,-1)
47 |         qvalues = net(state).cpu().detach().numpy().squeeze()
48 | 
49 |         # softmax sampling of the qvalues
50 |         if self.use_softmax:
51 |             p = sp.softmax(qvalues/epsilon).squeeze()
52 |             p /= np.sum(p)
53 |             action = np.random.choice(self.num_act, p = p)
54 |             
55 |         # else choose the best action with probability 1-epsilon
56 |         # and with probability epsilon choose at random
57 |         else:
58 |             if np.random.random() < epsilon:
59 |                 action = np.random.randint(self.num_act, size=1)[0]
60 |             else:                
61 |                 action = np.argmax(qvalues, axis=0)
62 |                 action = int(action)
63 |         
64 |         return action
65 |     
66 |     
67 |     def plot_policy_map(self, net, filename, offset):
68 |         net.eval()
69 |         with torch.no_grad():
70 |             fig, ax = plt.subplots()
71 |             ax.imshow(self.env.maze, 'Greys')
72 | 
73 |             for free_cell in self.env.allowed_states:
74 |                 self.env.current_position = np.asarray(free_cell)
75 |                 qvalues = net(torch.Tensor(self.env.state()).view(1,-1).to('cuda'))
76 |                 action = int(torch.argmax(qvalues).detach().cpu().numpy())
77 |                 policy = self.env.directions[action]
78 | 
79 |                 ax.text(free_cell[1]-offset[0], free_cell[0]-offset[1], policy)
80 |             ax = plt.gca();
81 | 
82 |             plt.xticks([], [])
83 |             plt.yticks([], [])
84 | 
85 |             ax.plot(self.env.goal[1], self.env.goal[0],
86 |                     'bs', markersize = 4)
87 |             plt.savefig(filename, dpi = 300, bbox_inches = 'tight')
88 |             plt.show()
89 | 


--------------------------------------------------------------------------------
/deep_qlearning.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import numpy as np\n",
 10 |     "import scipy.special as sp\n",
 11 |     "\n",
 12 |     "from IPython.display import display, clear_output\n",
 13 |     "import matplotlib.pyplot as plt\n",
 14 |     "import copy\n",
 15 |     "import time\n",
 16 |     "import random\n",
 17 |     "\n",
 18 |     "import torch\n",
 19 |     "import torch.nn as nn\n",
 20 |     "import torch.optim as optim\n",
 21 |     "import collections"
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "markdown",
 26 |    "metadata": {},
 27 |    "source": [
 28 |     "**Introduce experience replay.**"
 29 |    ]
 30 |   },
 31 |   {
 32 |    "cell_type": "code",
 33 |    "execution_count": 2,
 34 |    "metadata": {},
 35 |    "outputs": [],
 36 |    "source": [
 37 |     "Transition = collections.namedtuple('Experience',\n",
 38 |     "                                    field_names=['state', 'action',\n",
 39 |     "                                                 'next_state', 'reward',\n",
 40 |     "                                                 'is_game_on'])\n",
 41 |     "\n",
 42 |     "class ExperienceReplay:\n",
 43 |     "    def __init__(self, capacity):\n",
 44 |     "        self.capacity = capacity\n",
 45 |     "        self.memory = collections.deque(maxlen=capacity)\n",
 46 |     "\n",
 47 |     "    def __len__(self):\n",
 48 |     "        return len(self.memory)\n",
 49 |     "\n",
 50 |     "    def push(self, transition):\n",
 51 |     "        self.memory.append(transition)\n",
 52 |     "\n",
 53 |     "    def sample(self, batch_size, device = 'cuda'):\n",
 54 |     "        indices = np.random.choice(len(self.memory), batch_size, replace = False)\n",
 55 |     "        \n",
 56 |     "        states, actions, next_states, rewards, isgameon = zip(*[self.memory[idx] \n",
 57 |     "                                                                for idx in indices])\n",
 58 |     "        \n",
 59 |     "        return torch.Tensor(states).type(torch.float).to(device), \\\n",
 60 |     "               torch.Tensor(actions).type(torch.long).to(device), \\\n",
 61 |     "               torch.Tensor(next_states).to(device), \\\n",
 62 |     "               torch.Tensor(rewards).to(device), torch.tensor(isgameon).to(device)"
 63 |    ]
 64 |   },
 65 |   {
 66 |    "cell_type": "markdown",
 67 |    "metadata": {},
 68 |    "source": [
 69 |     "**Networks definition.**"
 70 |    ]
 71 |   },
 72 |   {
 73 |    "cell_type": "code",
 74 |    "execution_count": 3,
 75 |    "metadata": {},
 76 |    "outputs": [],
 77 |    "source": [
 78 |     "class fc_nn(nn.Module):\n",
 79 |     "    def __init__(self, Ni, Nh1, Nh2, No = 4):\n",
 80 |     "        super().__init__()\n",
 81 |     "        \n",
 82 |     "        self.fc1 = nn.Linear(Ni, Nh1)\n",
 83 |     "        self.fc2 = nn.Linear(Nh1, Nh2)\n",
 84 |     "        self.fc3 = nn.Linear(Nh2, No)\n",
 85 |     "        \n",
 86 |     "        self.act = nn.ReLU()\n",
 87 |     "        \n",
 88 |     "    def forward(self, x, classification = False, additional_out=False):\n",
 89 |     "        x = self.act(self.fc1(x))\n",
 90 |     "        x = self.act(self.fc2(x))\n",
 91 |     "        out = self.fc3(x)\n",
 92 |     "        \n",
 93 |     "        return out"
 94 |    ]
 95 |   },
 96 |   {
 97 |    "cell_type": "code",
 98 |    "execution_count": 4,
 99 |    "metadata": {},
100 |    "outputs": [],
101 |    "source": [
102 |     "class conv_nn(nn.Module):\n",
103 |     "    \n",
104 |     "    channels = [16, 32, 64]\n",
105 |     "    kernels = [3, 3, 3]\n",
106 |     "    strides = [1, 1, 1]\n",
107 |     "    in_channels = 1\n",
108 |     "    \n",
109 |     "    def __init__(self, rows, cols, n_act):\n",
110 |     "        super().__init__()\n",
111 |     "        self.rows = rows\n",
112 |     "        self.cols = cols\n",
113 |     "\n",
114 |     "        self.conv = nn.Sequential(nn.Conv2d(in_channels = self.in_channels,\n",
115 |     "                                            out_channels = self.channels[0],\n",
116 |     "                                            kernel_size = self.kernels[0],\n",
117 |     "                                            stride = self.strides[0]),\n",
118 |     "                                  nn.ReLU(),\n",
119 |     "                                  nn.Conv2d(in_channels = self.channels[0],\n",
120 |     "                                            out_channels = self.channels[1],\n",
121 |     "                                            kernel_size = self.kernels[1],\n",
122 |     "                                            stride = self.strides[1]),\n",
123 |     "                                  nn.ReLU()\n",
124 |     "                                 )\n",
125 |     "        \n",
126 |     "        size_out_conv = self.get_conv_size(rows, cols)\n",
127 |     "        \n",
128 |     "        self.linear = nn.Sequential(nn.Linear(size_out_conv, rows*cols*2),\n",
129 |     "                                    nn.ReLU(),\n",
130 |     "                                    nn.Linear(rows*cols*2, int(rows*cols/2)),\n",
131 |     "                                    nn.ReLU(),\n",
132 |     "                                    nn.Linear(int(rows*cols/2), n_act),\n",
133 |     "                                   )\n",
134 |     "\n",
135 |     "    def forward(self, x):\n",
136 |     "        x = x.view(len(x), self.in_channels, self.rows, self.cols)\n",
137 |     "        out_conv = self.conv(x).view(len(x),-1)\n",
138 |     "        out_lin = self.linear(out_conv)\n",
139 |     "        return out_lin\n",
140 |     "    \n",
141 |     "    def get_conv_size(self, x, y):\n",
142 |     "        out_conv = self.conv(torch.zeros(1,self.in_channels, x, y))\n",
143 |     "        return int(np.prod(out_conv.size()))"
144 |    ]
145 |   },
146 |   {
147 |    "cell_type": "code",
148 |    "execution_count": 5,
149 |    "metadata": {},
150 |    "outputs": [],
151 |    "source": [
152 |     "def Qloss(batch, net, gamma=0.99, device=\"cuda\"):\n",
153 |     "    states, actions, next_states, rewards, _ = batch\n",
154 |     "    lbatch = len(states)\n",
155 |     "    state_action_values = net(states.view(lbatch,-1))\n",
156 |     "    state_action_values = state_action_values.gather(1, actions.unsqueeze(-1))\n",
157 |     "    state_action_values = state_action_values.squeeze(-1)\n",
158 |     "    \n",
159 |     "    next_state_values = net(next_states.view(lbatch, -1))\n",
160 |     "    next_state_values = next_state_values.max(1)[0]\n",
161 |     "    \n",
162 |     "    next_state_values = next_state_values.detach()\n",
163 |     "    expected_state_action_values = next_state_values * gamma + rewards\n",
164 |     "    \n",
165 |     "    return nn.MSELoss()(state_action_values, expected_state_action_values)"
166 |    ]
167 |   },
168 |   {
169 |    "cell_type": "markdown",
170 |    "metadata": {},
171 |    "source": [
172 |     "**Import the maze and define the environment.**"
173 |    ]
174 |   },
175 |   {
176 |    "cell_type": "code",
177 |    "execution_count": 6,
178 |    "metadata": {},
179 |    "outputs": [],
180 |    "source": [
181 |     "from environment import MazeEnvironment\n",
182 |     "\n",
183 |     "maze = np.load('maze_generator/maze.npy')\n",
184 |     "\n",
185 |     "initial_position = [0,0]\n",
186 |     "goal = [len(maze)-1, len(maze)-1]\n",
187 |     "\n",
188 |     "maze_env = MazeEnvironment(maze, initial_position, goal)"
189 |    ]
190 |   },
191 |   {
192 |    "cell_type": "code",
193 |    "execution_count": 7,
194 |    "metadata": {},
195 |    "outputs": [
196 |     {
197 |      "data": {
198 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAADrCAYAAACICmHVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAFBUlEQVR4nO3dQU4jRxiA0XaUIxDhXeYO+P4nMHcgO1DmDs5ylJHNuBmXuz54T2KBMO1yw6dC4ld5dzqdFmB+f2y9AOA6YoUIsUKEWCFCrBAhVoj4c82DHx4eTt++fRu0lK/t+fn56sc+PT0Nue4aW69h1PNvfd2Xl5fl+/fvu3Nf2635P+vhcDgdj8erH8/1druzP5+z1vzM1lx3ja3XMOr5t77u4XBYjsfj2Qv7MxgiPh7rfr8su92Pj/3+hssCfvbxWN/e3v8cuCl/BkOEWCHi47E+Pr7/OXBTq/7P+j+vrzdcBvAr/gyGiI/vrL9Q+mf81v/gX7uGUUatYevX9llel50VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRw8YNZxjhG2HUiNmoc4JmWMOI686w1jVu8TOzs0KEWCFCrBAhVogQK0SIFSLEChFihQixQoRYISJ3uuHWaq+rNpo44ppbr/VW7KwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAhVohYNW74/Pw85A2KR9n6pLxRZj6B75ytTzcc5d6/N3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIGHZg2hozTKOMUJuMGmXE4Waf+X5dYmeFCLFChFghQqwQIVaIECtEiBUixAoRYoUIsULEqnHDp6en5Xg83nwRW4+O1cYdt75fNbUD5i6xs0KEWCFCrBAhVogQK0SIFSLEChFihQixQoRYIWKK92cddQrg1iflle7BWjPcsy2vuSzenxW4QKwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAxxemGo1w74jXDqF/tujOsYWtONwTOEitEiBUixAoRYoUIsUKEWCFCrBAhVogQK0RMcbrhKFuPrs1wD9aYYb0zrOFaTjcEzhIrRIgVIsQKEWKFCLFChFghQqwQIVaIECtEON1wpdqbE289crksrdMNZx53tLNChFghQqwQIVaIECtEiBUixAoRYoUIsUKEWCHC6YbLurXOMDpXurejzHAP7s3OChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiFXjhqPMMMI34vlLr2tZPu8Ji59l7NPOChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiCneTHnUCN8Io56/Nuq39c9hlFEjore4X3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIECtEDHsz5RmMWOsMJxbOMOpX+j2ojYheYmeFCLFChFghQqwQIVaIECtEiBUixAoRYoWIKQ5MG2XEhElpcmdZ5j4A7F4+y3vq2lkhQqwQIVaIECtEiBUixAoRYoUIsUKEWCFCrBAx7MC0Gca2rl1DbcyuNvI44v7W7sG1r+twOFz8mp0VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRU5xuOGqEb+uRtBnGGGcY+xxhhrHPe7OzQoRYIUKsECFWiBArRIgVIsQKEWKFCLFChFghIne64VccM/vZDKOJo5RGHu/NzgoRYoUIsUKEWCFCrBAhVogQK0SIFSLEChFihYgpTjccZevxuRlGLrd+o+q1j+UyOytEiBUixAoRYoU72++XZbf78bHfX/d9YoU7e3t7//NLxAoRYoUIscKdPT6+//klq4YigN/3+vqx77OzQsRu5ejav8uy/DNuOfDl/X06nf4694VVsQLb8WcwRIgVIsQKEWKFCLFChFghQqwQIVaIECtE/AfYYY6NVPlPoAAAAABJRU5ErkJggg==\n",
199 |       "text/plain": [
200 |        "<Figure size 432x288 with 1 Axes>"
201 |       ]
202 |      },
203 |      "metadata": {},
204 |      "output_type": "display_data"
205 |     }
206 |    ],
207 |    "source": [
208 |     "maze_env.draw('maze_20.pdf')"
209 |    ]
210 |   },
211 |   {
212 |    "cell_type": "markdown",
213 |    "metadata": {},
214 |    "source": [
215 |     "**Define the agent and the buffer for experience replay.**"
216 |    ]
217 |   },
218 |   {
219 |    "cell_type": "code",
220 |    "execution_count": 9,
221 |    "metadata": {},
222 |    "outputs": [],
223 |    "source": [
224 |     "buffer_capacity = 10000\n",
225 |     "buffer_start_size = 1000\n",
226 |     "memory_buffer = ExperienceReplay(buffer_capacity)"
227 |    ]
228 |   },
229 |   {
230 |    "cell_type": "code",
231 |    "execution_count": 10,
232 |    "metadata": {},
233 |    "outputs": [],
234 |    "source": [
235 |     "from agent import Agent\n",
236 |     "agent = Agent(maze = maze_env,\n",
237 |     "              memory_buffer = memory_buffer,\n",
238 |     "              use_softmax = True\n",
239 |     "             )"
240 |    ]
241 |   },
242 |   {
243 |    "cell_type": "markdown",
244 |    "metadata": {},
245 |    "source": [
246 |     "** Define the network.**"
247 |    ]
248 |   },
249 |   {
250 |    "cell_type": "code",
251 |    "execution_count": 82,
252 |    "metadata": {},
253 |    "outputs": [],
254 |    "source": [
255 |     "net = fc_nn(maze.size, maze.size, maze.size, 4)\n",
256 |     "optimizer = optim.Adam(net.parameters(), lr=1e-4)"
257 |    ]
258 |   },
259 |   {
260 |    "cell_type": "code",
261 |    "execution_count": 83,
262 |    "metadata": {},
263 |    "outputs": [
264 |     {
265 |      "data": {
266 |       "text/plain": [
267 |        "fc_nn(\n",
268 |        "  (fc1): Linear(in_features=400, out_features=400, bias=True)\n",
269 |        "  (fc2): Linear(in_features=400, out_features=400, bias=True)\n",
270 |        "  (fc3): Linear(in_features=400, out_features=4, bias=True)\n",
271 |        "  (act): ReLU()\n",
272 |        ")"
273 |       ]
274 |      },
275 |      "execution_count": 83,
276 |      "metadata": {},
277 |      "output_type": "execute_result"
278 |     }
279 |    ],
280 |    "source": [
281 |     "device = 'cuda'\n",
282 |     "batch_size = 24\n",
283 |     "gamma = 0.9\n",
284 |     "\n",
285 |     "net.to(device)"
286 |    ]
287 |   },
288 |   {
289 |    "cell_type": "markdown",
290 |    "metadata": {},
291 |    "source": [
292 |     "**Define the epsilon profile and plot the resetting probability.**"
293 |    ]
294 |   },
295 |   {
296 |    "cell_type": "code",
297 |    "execution_count": 84,
298 |    "metadata": {
299 |     "scrolled": false
300 |    },
301 |    "outputs": [
302 |     {
303 |      "data": {
304 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU5dnH8e9NIAgIyBIQWQQRF9wAU9S64dbiBi5VodrWpeLyUtdaaW21pXZxqRvyqthatyoub7WoKG7YWhUlIIKgaGQpoGLABQUEQu73j+ekGWKWSTInJ5P5fa5rrjnnOSczv5wkc+dsz2PujoiI5K4WSQcQEZFkqRCIiOQ4FQIRkRynQiAikuNUCEREclzLpAPUVdeuXb1v375JxxARySqzZs1a5e4FVS3LukLQt29fioqKko4hIpJVzGxpdct0aEhEJMepEIiI5DgVAhGRHKdCICKS41QIRERynAqBiEiOUyEQEclxWXcfQYM8fz98WAwtWoC1CM8dusLRY8Lylx6Cr9fCd04Py0REckBuFYKXHoQ3pm7Z1nuXikIwZSK8/TL02AH2Gtbo8UREkpBbheDqp8A9epRBWVmYLverR2BUD5j3LxUCEckZsR7/MLPhZrbQzIrNbFwVy/uY2XQze9PM5prZUXHmid40HPbJawmt8iG/dcWyTt2h356hEIiI5IjYCoGZ5QETgSOBgcBoMxtYabVfAg+7+2BgFPC/ceVJ2x4HwYLXoHRT0klERBpFnIeGhgLF7r4IwMwmAyOBBSnrONAhmu4IfBhjnvTsfzxs1RY2rIeWrZJOIyISuzgLQU9gWcr8cmCfSuv8GnjWzH4CtAMOr+qFzGwMMAagT58+GQ+6hUGHhIeISI5I+hrJ0cDd7t4LOAq4z8y+kcndJ7l7obsXFhRU2Z12Zm38GpbMj/99RESagDgLwQqgd8p8r6gt1VnAwwDu/hqwFdA1xkzpuXUsXHpQuKpIRKSZi7MQzAQGmFk/M8snnAyeUmmd/wCHAZjZroRCUBJjpvTsfgB8+Sn8552kk4iIxC62QuDupcBYYBrwDuHqoPlmNt7MRkSrXQqcbWZvAQ8Cp7unXtifkN0PDM9vv5xsDhGRRhDrDWXuPhWYWqntypTpBcD+cWaolx47QOfoxrJjzk06jYhIrJI+Wdw0mcEeB8K8l7e881hEpBnKrS4m6uLky8PVQyIizZwKQXUGDEk6gYhIo9ChoZrMeg6mT046hYhIrFQIavLkbXDXz5NOISISKxWCmgw6FFYugY8WJ51ERCQ2KgQ1GXRoeH5rerI5RERipEJQkz67hjEK5ryYdBIRkdioENTEDPY6BJbMSzqJiEhsdPlobS68A9q2TzqFiEhsVAhq065D7euIiGQxHRpKx19/CbddlHQKEZFYqBCkY9VyeOF+jU8gIs2SCkE6Bh0Ka1bDYp00FpHmR4UgHXtFYxjrfgIRaYZiLQRmNtzMFppZsZmNq2L5jWY2J3q8Z2afx5mn3rr1hu12hDdfSDqJiEjGxXbVkJnlAROBI4DlwEwzmxINRgOAu1+csv5PgMFx5WmwQ0+FtU2zTomINEScl48OBYrdfRGAmU0GRgILqll/NHBVjHka5oe/TjqBiEgs4jw01BNYljK/PGr7BjPbHugHVNmXg5mNMbMiMysqKUlwbHt3+DzB9xcRiUFTOVk8CnjU3TdXtdDdJ7l7obsXFhQUNHK0FL8+Hn4xPLn3FxGJQZyFYAXQO2W+V9RWlVHAgzFmyYydvwXFs7VXICLNSpyFYCYwwMz6mVk+4cN+SuWVzGwXoBPwWoxZMmPv74Tn2c8lm0NEJINiKwTuXgqMBaYB7wAPu/t8MxtvZiNSVh0FTHZ3jytLxuw4BDp0gaJpSScREcmYWDudc/epwNRKbVdWmv91nBkyKi8PhhwBs58NJ47Nkk4kItJg6n20ro67AA4ZHfodystLOo2ISIOpENTVwP2STiAiklFN5fLR7LJoLjx/X9IpREQyQoWgPp67B248G9avTTqJiEiDqRDUx9CjYNMGmKNO6EQk+6kQ1MfuB4ZxjGc8mXQSEZEGUyGoj1b5sPd34Y2nwmWkIiJZTIWgvvY5Br78FD5enHQSEZEGUSGor4NPhkdXQ48dkk4iItIguo+gvlq3STqBiEhGaI+gIeb+C8Z+Cz79OOkkIiL1pkLQEO06wntFMPPppJOIiNSbCkFD7LAndO0Fr+syUhHJXioEDWEG+x4Ds56FjRuSTiMiUi8qBA21zzGw/iuY+1LSSURE6iXWQmBmw81soZkVm9m4atY52cwWmNl8M3sgzjyxGHQo7H88tNk66SQiIvUS2+WjZpYHTASOAJYDM81sirsvSFlnAPBzYH93/8zMusWVJzat28BVf086hYhIvcW5RzAUKHb3Re6+EZgMjKy0ztnARHf/DMDdP4kxT7xWrQgPEZEsE2ch6AksS5lfHrWl2gnYycxeMbMZZja8qhcyszFmVmRmRSUlJTHFbYD1a+FH/eHvNyadRESkzpI+WdwSGAAMA0YDd5rZNpVXcvdJ7l7o7oUFBQWNHDENbdrBXofAK4+pEzoRyTpxFoIVQO+U+V5RW6rlwBR33+Tui4H3CIUh+xxwAny0CBbPSzqJiEidxFkIZgIDzKyfmeUDo4ApldZ5nLA3gJl1JRwqWhRjpvjsNyLcV/BvnTgWkewSWyFw91JgLDANeAd42N3nm9l4MxsRrTYNWG1mC4DpwGXuvjquTLHq1B12OwBefTzpJCIidWKeZce0CwsLvaioKOkYVVs0Fzp2hS7bJZ1ERGQLZjbL3QurWqZuqDNphz2TTiAiUmdJXzXU/Mx4EiZekHQKEZG0qRBk2rJ34R8T4MMPkk4iIpIWFYJMO/jk8PzPh5LNISKSJhWCTOvWBwZ+G16anHQSEZG0qBDEYdiocGPZ0gW1rysikjAVgjgcdBIM2Bu+/DTpJCIitdLlo3HovC1MbKL3OoiIVKI9gjit/wrWZOeN0iKSO1QI4vL1OhjdEx65LukkIiI1UiGIy1ZtK64eKitLOo2ISLVUCOJ02Gmwcim8/e+kk4iIVEuFIE7fPi4Mav/cPUknERGplgpBnNq0gwO/By8/AhvWJ51GRKRKKgRxO2Uc3PBvaN0m6SQiIlWKtRCY2XAzW2hmxWY2rorlp5tZiZnNiR4/jjNPInrvrO6pRaRJi60QmFkeMBE4EhgIjDazgVWs+pC7D4oef44rT6KWvwfX/ABWf5h0EhGRb4hzj2AoUOzui9x9IzAZGBnj+zVtL9wPLz6QdAoRkW+IsxD0BJalzC+P2io70czmmtmjZta7qhcyszFmVmRmRSUlJXFkjVevnWDXfcPVQ1k2NKiINH9Jnyx+Aujr7nsCzwFVXmfp7pPcvdDdCwsKCho1YMYc/kNY8ja8PzvpJCIiW4izEKwAUv/D7xW1/Ze7r3b3DdHsn4G9Y8yTrGGjIH8reKZ5ngYRkewVZyGYCQwws35mlg+MAqakrmBmPVJmRwDvxJgnWe07wbHnQ+ftkk4iIrKF2LqhdvdSMxsLTAPygLvcfb6ZjQeK3H0KcIGZjQBKgU+B0+PK0ySc86ekE4iIfIN5lp28LCws9KKiLO7rf/NmeHcG7LZ/0klEJIeY2Sx3L6xqWdIni3PP1Elw8QFhKEsRkSZAhaCxHXQStMqHqXcmnUREBEizEJjZCWb2vpl9YWZrzOxLM1sTd7hmqWNX2P8EeOE+dUQnIk1CunsE1wIj3L2ju3dw9/bu3iHOYM3aUWPgq8/h5UeTTiIiknYhWOnuzffSzsa217Bwt/GrjyedREQk7ctHi8zsIeBxoPwGMNz977Gkau7M4HfPQLcqe9QQEWlU6RaCDsA64DspbQ6oENRXj37huawMWuicvYgkJ61C4O5nxB0kJ70xFSacD7e8AZ26JZ1GRHJUulcN9TKzx8zsk+jxf2bWK+5wzd62O4TB7Z/WpaQikpx0j0n8ldBP0HbR44moTRqizy4w5Ah48jYo3ZR0GhHJUekWggJ3/6u7l0aPu4Es7Q+6iRn5E1i1Al55LOkkIpKj0i0Eq83sNDPLix6nAavjDJYzhh4F2/aDf0xIOomI5Kh0rxo6E5gA3Ei4WuhVQCeQMyEvD867Cdp1TDqJiOSodK8aWkoYL0DisJ82rYgkp8ZCYGYTCHsAVXL3CzKeKFd9thIeuBqOuwB6Dkg6jYjkkNr2CBrU8b+ZDQduJgxM82d3/2M1650IPAp8y92zeLCBBnAPXVRvLoULbks6jYjkkBoLgbtXOZh8OswsD5gIHAEsB2aa2RR3X1BpvfbAhcDr9X2vZqHztmGA+2fvhh/8RjeYiUijqfGqITO7KXp+wsymVH7U8tpDgWJ3X+TuG4HJwMgq1vstcA3wdT3yNy/f+yls2gBTbk06iYjkkNoODd0XPV9fj9fuCSxLmV8O7JO6gpkNAXq7+1Nmdll1L2RmY4AxAH369KlHlCzRe2fYdwRMmQgnXw5t2iWdSERyQG2HhmZFz/8sbzOzToQP77kNeWMzawHcQBoD1rv7JGAShDGLG/K+Td4pl0P7TrBhnQqBiDSKtC4fNbOXCJePtgRmAZ+Y2SvufkkNX7YCSO1nuVfUVq49sDvwkpkBbAtMMbMROXvCGGDgfuEhItJI0r2zuKO7rwFOAO51932Aw2v5mpnAADPrZ2b5wChCf0UAuPsX7t7V3fu6e19gBmEUtNwtAqneK4IZTyadQkRyQLqFoKWZ9QBOBtL6dHL3UmAsMA14B3jY3eeb2Xgz0x1UtZn0U7j5HNioc+giEq90C8F4wgf6B+4+08x2AN6v7Yvcfaq77+Tu/d39d1Hble7+jSuO3H2Y9gZSnHolrP4QnvlL0klEpJkz9+w691pYWOhFRTlQL9zh0oPg4yVwdzHkt046kYhkMTOb5e6FVS1Ld2CaHaJ7CUqigWn+Ee0VSFzMwl7BquXwrIZ+EJH4pHto6AHgYaAHYWCaR4AH4wolkSGHQ+F3NWiNiMQq3W6o27r7fSnz99d0A5hkiBn87unwLCISk3T3CJ42s3Fm1tfMtjeznwFTzayzmXWOM2DOMwvnC155HL78LOk0ItIMpbtHcHL0fE6l9lGEbqp1viBOy96F3xwPo34OZ/4+6TQi0sykOzBNv7iDSA367AqHfB8euymMcdylR9KJRKQZqa330Z+lTJ9UaZn+NW1MPxofThr/7bdJJxGRZqa2cwSjUqZ/XmnZ8AxnkZps1x+OGgNP3wnLFiadRkSakdoKgVUzXdW8xO20q6DXzuGOYxGRDKntHIFXM13VvMStUzeYNE+Xk4pIRtW2R7CXma0xsy+BPaPp8vk9GiGfVGYGGzfAk3foRjMRyYjaBqbJa6wgUgdzXoBbzoXSjXDcT5JOIyJZLt0byqQp+daRMPgwuO8qWPNp0mlEJMupEGQjMzj3Rlj7Bdz9y6TTiEiWUyHIVv32CDeXPXU7vPtG0mlEJIvFWgjMbLiZLTSzYjMbV8Xyc81snpnNMbN/m9nAOPM0Oz8cD0OPhlb5SScRkSwW28A0ZpYHvAccASwnjGE82t0XpKzTIRoLmWj4yvPdvcYb1XJmYBoRkQxq8MA09TQUKHb3Re6+EZgMjExdobwIRNqhexPqZ82nMOF/4JNlSScRkSwUZyHoCaR+Mi2P2rZgZv9jZh8A1wIXVPVCZjbGzIrMrKikpCSWsFlt7RdhFLPbLkw6iYhkocRPFrv7RHfvD1wOVHkJjLtPcvdCdy8sKCho3IDZoEe/MKzlK4/Bvx5NOo2IZJk4C8EKoHfKfK+orTqTgeNizNO8nfRTGLA3TDgfPtdek4ikL85CMBMYYGb9zCyf0JPplNQVzGxAyuzRwPsx5mne8lrCZffAui/gTo0iKiLpS3eEsjpz91IzGwtMA/KAu9x9vpmNB4rcfQow1swOBzYBnwE/iitPTui7G1x+P+yyb9JJRCSLxHb5aFx0+Wia3OHrddCmXdJJRKQJqOny0dj2CCRB7nBVdKXub/6hbqtFpEaJXzUkMTCDQYfCjCfgif9NOo2INHEqBM3V8ReGXkrvuBQWz0s6jYg0YSoEzZUZ/PRu2Hob+P1o2LA+6UQi0kSpEDRnnbrBZffCujWwcknSaUSkidLJ4uau8Dvw1/cgf6ukk4hIE6U9glyQvxVsLoW7fgFLF9S+vojkFBWCXLFmNUy7C35zPKxdU/v6IpIzVAhyRafucMXD8OEHcP3p4V4DERFUCHLLngfB2deFXkofuibpNCLSRKgQ5JoTLoJho+CBq+GzlUmnEZEmQFcN5RozuPQuWL4wHC4SkZynPYJc1LoN9B8Upp+/H1bVNEyEiDR3KgS57NOPYcJ5cOUIWL826TQikhAVglzWeVv4xWRYNAeuPglKNyWdSEQSEGshMLPhZrbQzIrNbFwVyy8xswVmNtfMXjCz7ePMI1XY52i44HaY+TRcdzqUlSWdSEQaWWyFwMzygInAkcBAYLSZDay02ptAobvvCTwKXBtXHqnBUWfDGb+H6Q/AnBeTTiMijSzOPYKhQLG7L3L3jYTB6UemruDu0919XTQ7gzDAvSRh1Di4+TUYcnjSSUSkkcVZCHoCy1Lml0dt1TkLeLqqBWY2xsyKzKyopKQkgxHlv8xg12is43kvw+MTks0jIo2mSdxHYGanAYXAwVUtd/dJwCQIYxY3YrTcNO0uePbu0FHdiRcnnUZEYhZnIVgB9E6Z7xW1bcHMDgeuAA529w0x5pF0XTQJ1n0Jd1wS9hROuCjpRCISozgPDc0EBphZPzPLB0YBU1JXMLPBwB3ACHf/JMYsUhctW8EvHoQDToTbL4a/35R0IhGJUWyFwN1LgbHANOAd4GF3n29m481sRLTadcDWwCNmNsfMplTzctLY/lsMToCFM9VbqUgzZp5lf+CFhYVeVFSUdIzcsbk0FIGWreDzEujQBVroPkSRbGNms9y9sKpl+ouWmuW1DEVg/VdwyQFwzWm6A1mkmVEhkPRs1Q6OOB2mPwhXHqtRzkSaERUCSY8ZjP45XHwnzH4eLt4fPl6SdCoRyQAVAqmbI38Mv38GSpbBxJ8knUZEMqBJ3FAmWWbI4XDzDNh6mzBfuimcRxCRrKQ9AqmfPruEbqw3bw7nDG67CDZtTDqViNSDCoE0jJdB713hsZvhskM02plIFlIhkIZp2QrOuzEa4OYtOH8IvKmurEWyiQqBZMawU2DCG9C+M9xwpg4TiWQRnSyWzNl+INw6E1YuhVb5oRh8shR6Dkg6mYjUQHsEkllttoa+u4XpB38P5+4FT9ymvopEmjAVAonP0WNg9wNhwvnws0NhRXHSiUSkCioEEp8u24Wbzy6aBO/PhnP2gBcfSDqViFSiQiDxMoOjzoa/vAP7HAN9dw/tZWXJ5hKR/9LJYmkcXbaDXz1SMX/96ZDXCs78PXTqnlgsEYl5j8DMhpvZQjMrNrNxVSw/yMxmm1mpmX0vzizShLhD5x7wwn1wxk5hBDR1bS2SmNgKgZnlAROBI4GBwGgzG1hptf8ApwM6cJxLzODH18Ad82Dgt8NwmGP2gPdmJZ1MJCfFuUcwFCh290XuvhGYDIxMXcHdl7j7XEAHjHNR753hd1Nh/BPQriMU9ArtGutApFHFWQh6AstS5pdHbXVmZmPMrMjMikpKSjISTpoIM9j3GJjwejhX4A5XHAk/Owzmv5J0OpGckBVXDbn7JHcvdPfCgoKCpONInMrK4OBTYPE8uPgAuPRgKJqmG9JEYhRnIVgB9E6Z7xW1iVQvLw+OvwDuXQzn3QQffQC/GA4v3J90MpFmK85CMBMYYGb9zCwfGAVMifH9pDlp0w6OvxDu/gAuvQsOODG0T58Mj94AX32ebD6RZiS2QuDupcBYYBrwDvCwu883s/FmNgLAzL5lZsuBk4A7zGx+XHkkS+W3hu+eAVu1DfOzpsGkS2F0T7jlfPjPO8nmE2kGzLPs2GthYaEXFRUlHUOSVPwm/GNC6K5i0wY44WI494akU4k0aWY2y90Lq1qWFSeLRbaw4+BwuOhvy+DMP8CgQ0P7ZyvhxrNh/qs6uSxSB+piQrLXNgUwKuWG9fdnw/QH4ek/w3b9w9VHw0aF/o3Mkssp0sRpj0Caj6FHwuSP4JK/QPd+8NAfw3gIn60My79el2w+kSZKewTSvLRtD8PPDI/PPoH5/4bO24Zl40+EjxeFXlD3PRZ22z+MuSyS47RHIM1Xp25wwAkV898+Drr3hSm3wmWHwMnd4L7fJBZPpKnQHoHkjmPOCY91X8Ls5+D1J6Fj17Bs/Vdw6UGwx8Ew+DDY8+CwdyGSA1QIJPe0bR/2FFL3Fj4vgfZd4Knb4bGboEUe7DIUzr4edvt2uApJJ5ylmVIhEAHo0Q+ueQ42fh0uP53zAsx5EVq3Ccv/+TDce2XoNrv80XuX0CWGSJZTIRBJlb8VDD40PFJ16AK9dw2Hk569O7S1bgv3LQmXsS5+G8pKoc9AaJXf2KlFGkSFQCQdQw4PD3f4sBjemQFLF1ScY5j8B5j+QCgC2+8O/faA/oPghIuSzS2SBnUxIZIJHy2GhW9A8exwY9vS+aFI3DE3LP/VsfDZx2GPoc+u0HMAbB9NizSCmrqY0B6BSCb06Bcew06paNuwvmJ6p8Iw0M6bz8Pz94a2bx0ZRmgD+O1J4VDTdv1hux2h+/bhuVP3xvseJGepEIjEpfxEM8APrqqYXrsmjLNQfhWSe+hW+93XK4oEwNHnwIW3w+bNcNkw6NITCnpDtz7hecfBoWCINJAKgUhja9chfIiXMwtXLEHYi/h4MaxcWnFH9PovIa8VvD8LXn089LgKcPrV8P0roGR56EqjU3fotG3F87BTYNd9Yf3acKiqQ5fwaNsBWuheUqmgQiDSlLRuE84dbD+wom3rbeC6F8O0e7jnoeQ/sE230NYiDw4ZDZ9+HM5DvFcUnvsPCoVgyTy4cL+K12uRB+07wSV3wX7HhmFBH7k+FIn2ncOyth1gyBGhGK39Irxn2w6hiOVv1XjbQxpFrIXAzIYDNwN5wJ/d/Y+VlrcG7gX2BlYDp7j7kjgziWQ1s9B1RqduFW1desDYW7+5bllZeO61M4x/Atashi8/jZ5XQ7doJNnPP4G5L4Vl67+q+PrrXwqF4PWn4I+nVrS3yg9F4Y/PQ/+9wvInbw+DB23VLpzraN0WTv5ZOGH+wVvwwZuhLXX5gCGhr6f1a6F0YygwrVprbyUBsRUCM8sDJgJHAMuBmWY2xd0XpKx2FvCZu+9oZqOAa4BTvvlqIlJn5R+o7TvBvsdUv97gw+D+pWF644awB7BuDXTZLrTtuh/87N7QtnZNeF63Jtw/AfD1Wli9IhzW+notbFgXHiPHhuUznoB7fvXN932kJBSKB38XLr8tl9cyFIRHV4cR6u4fDy9NDm3ljzZbw9VPhfWn3hku523VOhSpvFbQriOc+suw/N+PhXMyeS1D4clrFbbJQSeF5fNeDkUwr2VYltcyfP1Oe4flK96HTRtDe4sWYY+qdduKQ3dfrAp7auXLrEXIUb7nVFYWCngTvjM9zj2CoUCxuy8CMLPJwEggtRCMBH4dTT8K3Gpm5tl2TatIc5HfGvIr7XGUXxFVnYNPDo/qHH8hHHZaRZEof956m7B8n6OhY0E495H6KL8xr2sv2H63LZelXpG1dH4YwnTj11C6CTZvCt2FlBeC5+6G1yoNl75d/4pCcO9V8Nb0LZf3HwS3vRmm//D9cLgt1e4Hwg3/CtOXHAjL3t1yeeoVYaf1gVUrQiFokRcKxkEnw+X3heU/6Bv2xMqXtciDQ74PZ18blp+1ayg0+xwN5/yp+u3cAHEWgp7AspT55cA+1a3j7qVm9gXQBViVupKZjQHGAPTp0yeuvCISh7bta+7Ab7f9w6M65d2KV+e8m8KjOr98JBx6Kt1UUShSXXxn2MMp3QSbS8Py/JQrvs66BtasgrLN4Qquss0Ve0MAp10Z9ig2bwYvC8u7961YfuKlYS+rLGV53z0qlh94UiiMXlbx+qnniHYcHPYqusX32RfbDWVm9j1guLv/OJr/AbCPu49NWeftaJ3l0fwH0TqrqnpN0A1lIiL1kdSYxSuA3inzvaK2Ktcxs5ZAR8JJYxERaSRxFoKZwAAz62dm+cAooNKBOqYAP4qmvwe8qPMDIiKNK7ZzBNEx/7HANMLlo3e5+3wzGw8UufsU4C/AfWZWDHxKKBYiItKIYr2PwN2nAlMrtV2ZMv01cFKcGUREpGa6c0NEJMepEIiI5DgVAhGRHKdCICKS47JuhDIzKwGW1vPLu1LpruUmQrnqRrnqrqlmU666aUiu7d29oKoFWVcIGsLMiqq7sy5JylU3ylV3TTWbctVNXLl0aEhEJMepEIiI5LhcKwSTkg5QDeWqG+Wqu6aaTbnqJpZcOXWOQEREvinX9ghERKQSFQIRkRyXM4XAzIab2UIzKzazcTG/V28zm25mC8xsvpldGLX/2sxWmNmc6HFUytf8PMq20My+G2duM1tiZvOiDEVRW2cze87M3o+eO0XtZma3RO8/18yGpLzOj6L13zezH1X3fmlm2jllu8wxszVmdlES28zM7jKzT6KBk8rbMrZ9zGzvaPsXR1+b1mC21eS6zszejd77MTPbJmrva2brU7bb7bW9f3XfYz1zZeznZqEr+9ej9ocsdGtf31wPpWRaYmZzEthe1X0+JPc75u7N/kHoBvsDYAcgH3gLGBjj+/UAhkTT7YH3gIGE8Zl/WsX6A6NMrYF+Uda8uHIDS4CuldquBcZF0+OAa6Lpo4CnAQP2BV6P2jsDi6LnTtF0pwz+vD4Gtk9imwEHAUOAt+PYPsAb0boWfe2RDcj1HaBlNH1NSq6+qetVep0q37+677GeuTL2cwMeBkZF07cD59U3V6XlfwKuTGB7Vff5kNjvWK7sEQwFit19kbtvBCYDI+N6M3f/yN1nR9NfAu8QxmeuzkhgsrtvcPfFQHGUuTFzjwTuiabvAY5Lab/XgxnANmbWA/gu8Jy7f+runwHPAcMzlOUw4AN3r+kO8ti2mbv/izA+RuX3a/D2iZZ1cPcZHv5i7015rTrncvdn3b00mp1BGAmwWrW8f3XfY5rWTEkAAATnSURBVJ1z1aBOP7foP9lDgUczmSt63ZOBB2t6jZi2V3WfD4n9juVKIegJLEuZX07NH8wZY2Z9gcHA61HT2Gj37q6UXcnq8sWV24FnzWyWmY2J2rq7+0fR9MdA94SyQRigKPUPtClss0xtn57RdKbzAZxJ+O+vXD8ze9PM/mlmB6bkre79q/se6ysTP7cuwOcpxS5T2+tAYKW7v5/S1ujbq9LnQ2K/Y7lSCBJhZlsD/wdc5O5rgNuA/sAg4CPCrmkSDnD3IcCRwP+Y2UGpC6P/IhK5rjg6/jsCeCRqairb7L+S3D7VMbMrgFLgb1HTR0Afdx8MXAI8YGYd0n29DHyPTe7nVslotvxno9G3VxWfDw16vYbIlUKwAuidMt8raouNmbUi/JD/5u5/B3D3le6+2d3LgDsJu8M15Yslt7uviJ4/AR6LcqyMdinLd4c/SSIboTjNdveVUcYmsc3I3PZZwZaHbxqcz8xOB44BTo0+QIgOvayOpmcRjr/vVMv7V/c91lkGf26rCYdCWlZqr7fotU4AHkrJ26jbq6rPhxpeL/7fsXRObmT7gzAk5yLCyanyE1G7xfh+Rjgud1Ol9h4p0xcTjpUC7MaWJ9AWEU6eZTw30A5onzL9KuHY/nVseaLq2mj6aLY8UfWGV5yoWkw4SdUpmu6cgW03GTgj6W1GpZOHmdw+fPNE3lENyDUcWAAUVFqvAMiLpncgfBDU+P7VfY/1zJWxnxth7zD1ZPH59c2Vss3+mdT2ovrPh8R+x2L5IGyKD8KZ9/cIlf6KmN/rAMJu3VxgTvQ4CrgPmBe1T6n0x3JFlG0hKWf4M507+iV/K3rML39NwrHYF4D3gedTfqEMmBi9/zygMOW1ziSc7Csm5cO7AdnaEf4D7JjS1ujbjHDI4CNgE+H46lmZ3D5AIfB29DW3Et3hX89cxYTjxOW/Z7dH654Y/XznALOBY2t7/+q+x3rmytjPLfqdfSP6Xh8BWtc3V9R+N3BupXUbc3tV9/mQ2O+YupgQEclxuXKOQEREqqFCICKS41QIRERynAqBiEiOUyEQEclxKgQiETPbbFv2gJqxXmqj3i3frn1NkcbXsvZVRHLGencflHQIkcamPQKRWkT91l8b9e/+hpntGLX3NbMXo47VXjCzPlF7dwtjA7wVPb4dvVSemd0Z9UH/rJm1ida/IOqbfq6ZTU7o25QcpkIgUqFNpUNDp6Qs+8Ld9yDcpXlT1DYBuMfd9yR09nZL1H4LoQuDvQj94c+P2gcAE919N+Bzwt2sELoTGBy9zrlxfXMi1dGdxSIRM/vK3beuon0JcKi7L4o6C/vY3buY2SpC1wmbovaP3L2rmZUAvdx9Q8pr9CX0HT8gmr8caOXuV5vZM8BXwOPA4+7+VczfqsgWtEcgkh6vZrouNqRMb6biHN3RhL5khgAzU3raFGkUKgQi6Tkl5fm1aPpVwiA6AKcCL0fTLwDnAZhZnpl1rO5FzawF0NvdpwOXAx2Bb+yViMRJ/3mIVGhj0WDmkWfcvfwS0k5mNpfwX/3oqO0nwF/N7DKgBDgjar8QmGRmZxH+8z+P0AtmVfKA+6NiYcAt7v55xr4jkTToHIFILaJzBIXuvirpLCJx0KEhEZEcpz0CEZEcpz0CEZEcp0IgIpLjVAhERHKcCoGISI5TIRARyXH/D4n8Iw0tmc4LAAAAAElFTkSuQmCC\n",
305 |       "text/plain": [
306 |        "<Figure size 432x288 with 1 Axes>"
307 |       ]
308 |      },
309 |      "metadata": {},
310 |      "output_type": "display_data"
311 |     },
312 |     {
313 |      "data": {
314 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEGCAYAAACO8lkDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU1f34/9d7JvseIOw7IoiBRAhLVUSKIlbrUm2rxQWrUm2t3X5W+6kfa639dLHf1rp83LW2olL8VKUVtSKgUlEJiAuo7EJYA7IGQpLJ+/fHmUwmeyaZLcn7+XjMI3ebe99zk9z3nHPuOVdUFWOMMSaYJ9YBGGOMiT+WHIwxxjRgycEYY0wDlhyMMcY0YMnBGGNMAwmxDiAcevTooYMHD451GMYY06GsWLFij6rmNbauUySHwYMHU1xcHOswjDGmQxGRz5taZ9VKxhhjGrDkYIwxpgFLDsYYYxroFG0OxoSqsrKSkpISysvLYx2KMRGXkpJC//79SUxMbPV7LDmYLqmkpITMzEwGDx6MiMQ6HGMiRlXZu3cvJSUlDBkypNXvs2ol0yWVl5fTvXt3Swym0xMRunfvHnIp2ZKD6bIsMZiuoi1/61attOVT2LcTRABxP70JMOpLbn3JWjhyEIYVuuXGGNMFWMlh21r4YAmsWgyrFsH7r7v5Ghs/gHf+BVs+iVWEppPyer0UFhYGXr/97W9D3kdxcTE33ngjAH/5y1+44YYbwh1mm3z66acUFhZy0kknsWHDBk4++WQANm/eTH5+fkj7Onr0KFOmTMHn84X0voyMjEaX33bbbSxcuBCAu+++myNHjoS032i65pprWLNmDQD/8z//0+L2l1xyCevWrQvPwVW1w7/GjRun7VJd7V4+n2pVlWpVZe26imOqT/+P6rJ/tu8YJq6sWbMm1iFoenp6WPf3xBNP6Pe+972w7rM5VVVVTa77zW9+o7/61a8aLN+0aZOeeOKJIR3nvvvu07vvvrvV21dXV6vP52vV+R00aJCWlpY2uq65zxessrKy5Y3CoDWfZ8mSJXrNNdc0uq6xv3mgWJu4rlrJAVxVkgh4POD11q0+SkyCngNhd5O9zI0Jq8GDB/PTn/6U0aNHM2HCBNavXw/AvHnzyM/Pp6CggNNOOw2AJUuWcO655zbYx+bNm/nyl7/MmDFjmDZtGlu2bAFg1qxZ3HjjjZx88skMHTqU5557rtH3jhw5kpkzZ3LCCSdw8cUXB75dDx48mJtvvpmxY8cyb948Vq1axaRJkxgzZgwXXngh+/btY8GCBdx999088MADTJ06FWj8W7zP5+Omm25i/PjxjBkzhoceeqjR8zFnzhzOP/98AA4fPsy0adMYO3Yso0eP5sUXXwzEPGLECK644gry8/PZunUrAD/60Y848cQTmTZtGqWlpYFz8Nxzz3HPPfewfft2pk6dWifOn/zkJxQUFLBs2TLuuOMOxo8fT35+PrNnz0b9T848/fTT+eEPf0hRURG//vWvGTJkCJWVlQAcPHiwznyNmuPWqDknS5Ys4fTTT+fiiy8OnPfg4xQXF3PLLbdw9OhRCgsLmTlzJmVlZZxzzjkUFBSQn5/P3LlzAZg8eTILFy6kqqqq0XMZkqayRkd6tbvk0JIP31R98heqR8siexwTNQ2+Rb36RMPXp++5dZUVja9f/75bf7Ss4bpW8Hg8WlBQEHg9++yzquq+zd55552qqvrkk0/qOeeco6qq+fn5WlJSoqqq+/btU1XVxYsXB9YHlxzOPfdc/ctf/qKqqo899pief/75qqp65ZVX6sUXX6w+n09Xr16tw4YNaxDXpk2bFNClS5eqqupVV12ld911VyC23/3ud4FtR48erUuWLFFV1f/+7//WH/zgB6qq+otf/CLwHtXab73BJYeHHnooULooLy/XcePG6caNG+vEcuzYMe3Vq1dgvrKyUg8cOKCqqqWlpTps2DCtrq7WTZs2qYjosmXLAtsC+tRTT6mq6i9/+cvAubnyyit13rx5gc8TXHIAdO7cuYH5vXv3BqYvu+wynT9/vqqqTpkyRa+//vrAulmzZunzzz8f+Fw//vGPG5zX4OMGn5PFixdrVlaWbt26VX0+n06aNEnfeuutwHGWL19eZ3tV1eeee65OCWH//v2B6TPOOEOLi4sbHN9KDpHQb7hroK4Orc7TmOakpqayatWqwOub3/xmYN2ll14a+Lls2TIATjnlFGbNmsUjjzzSYv37smXL+Na3vgXA5ZdfztKlSwPrLrjgAjweD6NGjWLXrl2Nvn/AgAGccsopAFx22WV13l8T54EDB9i/fz9TpkwB4Morr+TNN99s9ef/97//zV//+lcKCwuZOHEie/fubVBfvmfPHnJycgLzqsp//dd/MWbMGM444wy2bdsW+AyDBg1i0qRJgW09Hk8g1vqfoSler5eLLrooML948WImTpzI6NGjWbRoEatXr25wHsC1DTzxxBMAPPHEE1x11VWtPg8AEyZMoH///ng8HgoLC9m8eXOz248ePZrXXnuNm2++mbfeeovs7OzAup49e7J9+/aQjt8Yu/2mNbr1di/TeU2f1fS6hMTm16ekNb++DYJvPayZfvDBB3n33Xd56aWXGDduHCtWrGjTvpOTkwPT6q++aO749efT09PbdNz6VJV7772Xs846q8ltUlNT69yfP2fOHEpLS1mxYgWJiYkMHjw4sL6luFpzO2dKSgperxdwfWG++93vUlxczIABA7j99tvrxBJ8vFNOOYXNmzezZMkSfD5fo43uCQkJVFdXA1BdXU1FRUVgXfDvxOv1tlgtdPzxx7Ny5UoWLFjArbfeyrRp07jtttsCcaemprb4WVsS9ZKDiMwQkc9EZL2I3NLENt8QkTUislpEno52jI3y+WBf49+yjAm3mjrkuXPn8qUvuduqN2zYwMSJE7njjjvIy8sL1Ks35uSTT+bZZ58F3AV18uTJIR1/y5YtgRLL008/zamnntpgm+zsbHJzc3nrrbcA+Nvf/hYoRbTGWWedxQMPPBCom1+7di1lZWV1tsnNzcXn8wUuygcOHKBnz54kJiayePFiPv+86bbA6urqQB1/U58hMzOTQ4cONfr+mmP26NGDw4cPN9o+E+yKK67gW9/6VpOlhsGDBwcS+vz58xu0SbQkMTEx8J7t27eTlpbGZZddxk033cTKlSsD261duzbkO8IaE9WSg4h4gfuBM4ESYLmIzFfVNUHbDAd+BpyiqvtEpGc0Y2zSR2/Cx2/BN29xjdTGtFNNA2ONGTNmBG5n3bdvH2PGjCE5OZlnnnkGgJtuuol169ahqkybNo2CggLeeOONRvd97733ctVVV3HXXXeRl5cXqPJorREjRnD//ffz7W9/m1GjRnH99dc3ut2TTz7Jddddx5EjRxg6dGhIx7nmmmvYvHkzY8eORVXJy8vjhRdeaLDd9OnTWbp0KWeccQYzZ87kq1/9KqNHj6aoqIiRI0c2uf/09HTee+897rzzTnr27BlIuMFmz57NjBkz6Nu3L4sXL66zLicnh2uvvZb8/Hx69+7N+PHjm/08M2fO5NZbbw1UCdZ37bXXcv7551NQUMCMGTNCLoHNnj2bMWPGMHbsWK644gpuuukmPB4PiYmJPPDAAwDs2rWL1NRUevcOQ01HU40RkXgBXwJeDZr/GfCzetv8HrgmlP1GvEFaVXXbetcovW195I9lIi4ebmVtSnO3V0ZDW243jaQVK1boZZddFuswWjRv3ryYx/nHP/5RH3300UbXhdogHe02h35AcFm4BJhYb5vjAUTkP4AXuF1VX4lOeM3o0d/d7rp7C/QdFutojOkyxo4dy9SpU/H5fIH2gHjz/e9/n5dffpkFCxbENI6cnBwuv/zysOwrHhukE4DhwOlAf+BNERmtqvuDNxKR2cBsgIEDB0Y+qqRk6NbH+juYiGvpTpVIGzx4MB9//HFMY6jv29/+dqxDaNa9994b6xAAQr5LqjnRbpDeBgwImu/vXxasBJivqpWquglYi0sWdajqw6papKpFeXmNPh87/HoNgtIS8IWhg4kxxsSxaCeH5cBwERkiIknAJcD8etu8gCs1ICI9cNVMG6MZZJOOGwtfngli3UOMMZ1bVKuVVLVKRG4AXsW1JzyuqqtF5A5cw8h8/7rpIrIG8AE3qereaMbZpJw89zLGmE4u6m0OqroAWFBv2W1B0wr82P+KP7u3woFSGD421pEYY0zEWP1IqDZ/BMtfdp3ijGmHmiG78/Pz+frXvx7y0NFNDUndlPoDv9VoatjvBx98kL/+9a+B5e0ZkiF4gMD58+cH+nOUlpYyceJETjrpJN566y3mzZvHCSecEBgIz8SOJYdQ9R4CVZWwt347ujGhqRlb6eOPPyYpKYkHH3ywznpVDQy3EElFRUXcc889DZZfd911XHHFFUD7k0Ow8847j1tucYMjvP7664wePZr333+fyZMn89hjj/HII4806JDWlLCMPmoaZckhVL0Gu/4OOzfFOhLTiUyePJn169c3OvT0M888w+jRo8nPz+fmm2+u877GhqR+5JFHGD9+PAUFBVx00UV1SiQLFy6kqKiI448/nn/9619A08N+33777fzhD3/gueeeo7i4mJkzZ1JYWMhLL73EBRdcENjutdde48ILL2zw/ldeeYWRI0cyduxY/vGPfwSW15ROVq1axU9/+lNefPFFCgsL+eUvf8nSpUu5+uqruemmm5oc0nvJkiVMnjyZ8847j1GjRgHw1FNPMWHCBAoLC/nOd74TGJgwIyODn//85xQUFDBp0qTAIH27du3iwgsvpKCggIKCAt5+++1m99MVxWM/h/iWnAq5vV1yGNP6cWRM/Hp2aRlb94T3IjCgh5dLTm3d8AhVVVW8/PLLzJgxA4B169bx5JNPMmnSJLZv387NN9/MihUryM3NZfr06bzwwgtccMEFlJWVUVRUxJ/+9CfuuOMOfvnLX3Lffffxta99jWuvvRaAW2+9lccee4zvf//7gOtD8d5777FhwwamTp0aeFZEcy6++GLuu+8+/vCHP1BUVISq8pOf/ITS0tLA0Bz1+yGUl5dz7bXXsmjRIo477rg6I5jWKCws5I477qC4uJj77rsPcKOg1hzn4YcfJjs7m+XLl3Ps2DFOOeUUpk+fDsDKlSv5+OOPGTJkCJ988glz587lP//5D4mJiXz3u99lzpw5XHHFFZSVlTFp0iR+/etf89Of/pRHHnmEW2+9lRtvvJEpU6bw/PPP4/P5OHz4cLP76Yqs5NAWvYfAgT0QhSK/6bxqxlYqKipi4MCBXH311UDdoaeXL1/O6aefTl5eHgkJCcycOTMwLHZTQ1J//PHHTJ48mdGjRzNnzpw6w0x/4xvfwOPxMHz4cIYOHcqnn34actwiwuWXX85TTz3F/v37WbZsGWeffXadbT799FOGDBnC8OHDEREuu+yykI/T3JDeEyZMYMiQIYCrmlqxYgXjx4+nsLCQ119/nY0b3d3vSUlJgVLRuHHjAh0MFy1aFBgvyuv1kp2d3ex+uiIrObTFmCkw9gz35DjT4bX2G3641bQ51NfWIbFrhqSeNWsWL7zwAgUFBfzlL39hyZIlDbZpar61rrrqKr761a+SkpLC17/+dRISwn8p0SaG9F6yZEmdc6SqXHnllfzmN79psI/ExMTAZ2xpKOzm9tMV2dWtLZKSLTGYqJgwYQJvvPEGe/bswefz8cwzzwSGxW5qSOpDhw7Rp08fKisrmTNnTp39zZs3j+rqajZs2MDGjRsZMWJEq+KoP7R137596du3L3feeWejQzaMHDmSzZs3s2HDBoDAyLKhaM2Q3gDTpk3jueeeY/fu3QB88cUXzQ7lXfOempFMfT4fBw4caNN+OjO7wrXVmmXw1v/FOgrTyfXp04ff/va3TJ06lYKCAsaNGxd4nnLNkNT5+fksWrQo8LCXX/3qV0ycOJFTTjmlwZDWAwcOZMKECZx99tk8+OCDpKSktCqOWbNmcd1111FYWMjRo0cBN0T1gAEDOOGEExpsn5KSwsMPP8w555zD2LFj6dkz9JH3r7nmGkaNGsXYsWPJz8/nO9/5TqPf/EeNGsWdd97J9OnTGTNmDGeeeSY7duxodt9//vOfWbx4MaNHj2bcuHGsWbOmTfvpzESbeBJUR1JUVKTFxcXRPej7i2D1UvjGza4kYTqUTz75pNGLmmm9G264gZNOOinQVmLiW2N/8yKyQlWLGtveSg5t1XuIa5Au3RLrSIyJunHjxvHhhx+2qaHZdAzWIN1Wef3B44Udm6Bfg0FjjenU2vr8atNxWMmhrRISoecA2Nl1b3Xr6DpDlaoxrdGWv3VLDu0x6ETI6WX9HTqglJQU9u7dawnCdHqqyt69e1t980ENq1ZqjxHj3ct0OP3796ekpCQw5IQxnVlKSgr9+/cP6T2WHNpLFcrLIDW0ETJNbCUmJgZ62BpjGrJqpfZa/jL860GXJIwxppOw5NBe3fvC0cOwb1esIzHGmLCx5NBefYa5nzs2xDYOY4wJI0sO7ZWWCbm9YFvLQx8bY0xHYckhHPoMg91boLIi1pEYY0xY2N1K4TCsEHr0s5FajTGdhiWHcMjt6V7GGNNJ2FfdcDm4F9ZGeWRYY4yJkKgnBxGZISKfich6EbmlkfWzRKRURFb5X9dEO8Y22bYO3vkXHN4f60iMMabdopocRMQL3A+cDYwCLhWRUY1sOldVC/2vR6MZY5vVjMxasja2cRhjTBhEu81hArBeVTcCiMizwPnAmijHAcDqLZW8tOIoIiACnsBPQSCwPHidIHg80DfXy+n5yaSn+PNrVnf3KlkLIyfE4uMYY0zYRDs59AO2Bs2XABMb2e4iETkNWAv8SFW31t9ARGYDs8E9+rAtRNwNRqrgq4YqdSMYqirVCopbp4qbVw1s+966CpZ+eoxbvpZFdpo/QfQ/Hj59z93SmpjUppiMMSYexOPdSv8EnlHVYyLyHeBJ4Mv1N1LVh4GHwT0mtC0HGjUgkVEDEtsU5Iadlfxx/iGefrOM62dkuoX9/Mlh307o2baEZYwx8SDaDdLbgAFB8/39ywJUda+qHvPPPgqMi1JsIRnWO5GzClNYubGSbV/4H3reaxB846eWGIwxHV60k8NyYLiIDBGRJOASYH7wBiLSJ2j2POCTKMYXktPzU/B6YNln/p7RHg8kJcc2KGOMCYOoJgdVrQJuAF7FXfT/rqqrReQOETnPv9mNIrJaRD4AbgRmRTPGUGSleRjVP5GVG4KGzdi3G155HPbuiF1gxhjTTlFvc1DVBcCCestuC5r+GfCzaMfVVvmDEvloSyWlB3zkZXshJR1Kt0LJZ9C9T8s7MMaYOGQ9pNvphP6uQfuTkkq3IDUdevS3/g7GmA7NkkM79c7xkJEibNxVVbuw//GwdzscORS7wIwxph0sObSTiDC4ZwKfl/pqF/Y/3v3cti42QRljTDtZcgiDQXletn/h41ilv7tFTk8YOgZSM2IbmDHGtFE8doLrcAb3TKBaoWSvj2G9E1zX61O/FuuwjDGmzazkEAZ9u3kB2LHPV3dF+RE4tC8GERljTPtYcgiDHpkeErz1koMq/OtBeH9h7AIzxpg2suQQBh6P0CvbWzc5iLiG6W3roKoydsEZY0wbWHIIkz65XnbWr1YaeIIboXXHxtgEZYwxbWTJIUz65HrYc6iayqqgAWJ7DYakFNgSt8NDGWNMoyw5hEmvXC+qsPtgUOnB63VVSyWfQXV17IIzxpgQ2a2sYdIj092xtOdgNf26Ba0YMwUKproRW40xpoOw5BAmeVnu4r/nYL0SQlb3GERjjDHtY19nwyQzVUhKgD2HfA1X7t4CS5+3qiVjTIdhySFMRITumd6GJQeAo4dh4wew6/PoB2aMMW3QbHIQkV7RCqQz6JHlaTw59BsOiUmw+ePoB2WMMW3QUsnhiqhE0UnkZXnYc9CHqtZdkZAIA0bCljXga6TayRhj4kxLyeFCEblBREZEJZoOrkeWl/JKKDumDVcOOhGOHYWd1iHOGBP/WkwOwDrgAhF5JArxdGjdM93p3NtY1VLfYZDX30oOxpgOodlbWVV1F/Cq/2Va0C3DJYd9ZdUMqr/SmwBnXxP1mIwxpi1afbeSiJwtIu+KyGci8ncR+VIkA+uIcmuSw+Fmbln1VdnjQ40xcS+UW1n/F/gxMAl4GLhLRC6NSFQdVGaq4PU0kxxU4Z8PQLEVxIwx8S2U5LBbVf+jqvtUdSFwFvDzUA8oIjP8pY/1InJLM9tdJCIqIkWhHiNWPCLkpHvYV9ZEchCBPkPdWEsVx6IbnDHGhCCU5LBJRO4UkST/fCVQFcrBRMQL3A+cDYwCLhWRUY1slwn8AHg3lP3Hg9wMT/PVSkPGuOc7bLWRWo0x8SuU5FCNu3tpq4gsBdYDS0RkeAj7mACsV9WNqloBPAuc38h2vwJ+B5SHsO+4kNtcyQHcHUtZ3WHDB9ELyhhjQtTq5KCq31LVE4GBuG/1twMCPCIiW1q5m37A1qD5Ev+yABEZCwxQ1Zea25GIzBaRYhEpLi0tbeXhI6+m5NCgI1wNERhaADs3QdmB6AZnjDGtFPKorKp6DFjhf4WViHiAPwKzWhHHw7iGcYqKipq4EkdfbrqHSp/rCJeRIo1vdNxJru0hLSu6wRljTCtFe+C9bcCAoPn+/mU1MoF8XHXVZtydUfM7UqN0q25nTct01UvSRPIwxpgYi3ZyWA4MF5Eh/obtS4D5NStV9YCq9lDVwao6GHgHOE9Vi6McZ5t1a01yADdS67J/wp5tzW9njDEx0OpqJRFJBi4CBge/T1XvaO0+VLVKRG7A9bj2Ao+r6moRuQMoVtX5ze8h/tWUHL5oKTl4E2HTh+4JcT36Nb+tMcZEWShtDi8CB3BtDW2+SV9VFwAL6i27rYltT2/rcWIlK1XwCM3fsQSQlOxGat38MRSd5YbXMMaYOBHKFam/qs6IWCSdhMcjZKUJB1pKDgDDCmHTR7D1Mxh8YuSDM8aYVgqlzeFtERkdsUg6kZx0DweOtOIGqt5DID0b1q+MfFDGGBOCUEoOpwKzRGQTrlpJAFXVMRGJrAPLTvOw91ArSg4eD5wwCQ7vd+Mu2d1Lxpg4EUpyODtiUXQy2WkeNu5q5cgio2xwW2NM/Gl1clDVzyMZSGeSk+7h0FGlyqckeFtRGlCF0q3QvR94vZEP0BhjWtBim4N/HCVE5JCIHAx6HRKRg5EPsePJTncJ4eCRVlQtAWzfAK88DtvWRTAqY4xpvRaTg6qe6v+ZqapZQa9MVbXxHxqRneZOa6sapcE/lEYmrAv7iCTGGNMm0e4h3SXkpLvTur81t7OCa5g+bixsX2+D8Rlj4kIojwktEpHnRWSliHwoIh+JyIeRDK6jqi05tDI5gBuMD2Cd3dZqjIm9UO5WmgPcBHyEe7aDaUJWqiASQskBICMH+h7nnhJXcLrd1mqMialQkkNpZxj7KBo8HiErVVrf5lBj0lchJc0SgzEm5kJJDr8QkUeB1wkaW0lV/xH2qDqB7HRP64bQCJbub9+3DnHGmBgLJTlcBYwEEqmtVlLAkkMjctJaeFxoU/bugDfmwpRvQPe+4Q/MGGNaIZTkMF5VR0Qskk4mO93D5t2t7CUdLLMbHDsCn7wLp14Y/sCMMaYVQh14b1TEIulkctKEQ0cVX3WI7Q5JyW601s0fw9GyyARnjDEtCCU5TAJWichnditry7LTPShwMNRGaYARE6DaZ53ijDExE0q1kj3LIQTBfR1qng7X+jf3gL7DYO1yOPEUG2/JGBN1NvBehITcS7q+gqmu7cFjndiNMdFnz6aMkDb1kg6W1z+M0RhjTGja9LVURHqHO5DOJitNENpRcgCorID3X4cdG8MWlzHGtEZb6ywWhDWKTsjrETLb0ks6mMcLGz6Aj94MX2DGGNMKbU0O1n23FbLTPe0rOXi97jGiOzfDnm1hi8sYY1rS1uTwSFij6KRy0jxtb3OoMXwcJKXAmrfDE5QxxrRCm5KDqv5vWw8oIjP8fSXWi8gtjay/zt+HYpWILO3IHe/aNL5SfUnJcHwRfL4GDn4RnsCMMaYFUb1PUkS8wP3A2cAo4NJGLv5Pq+poVS0Efg/8MZoxhlN2mnDwqFIdai/p+kZOhEEdNkcaYzqgaN9EPwFYr6obVbUCeBY4P3gDVQ1+LnU6bnC/Dikn3YMqHDzazo+QlgmnfR2yuoUnMGOMaUGzyUFEeoX5eP2ArUHzJf5l9Y/7PRHZgCs53NhEbLNFpFhEiktLS8McZnjUdIRrd9VSjQN7YMOq8OzLGGOa0VLJ4YqoRFGPqt6vqsOAm4Fbm9jmYVUtUtWivLy86AbYSjUd4do0dHdjPn0Xlv3TnjNtjIm4lpLDhSJyg4iEa6jubcCAoPn+/mVNeRa4IEzHjrqaMZXadTtrsPxT3c+Pl4Znf8YY04QWkwOwDrhARMJx++pyYLiIDBGRJOASoM6jR0VkeNDsOf7jd0iZbXmWdHPSs91w3utWQtnBlrc3xpg2anZsJVXdBbzqf7WbqlaJyA3+/XmBx1V1tYjcART7n1F9g4icAVQC+4Arw3HsWPB6hOw0CV9yABg9GTa8D6uXwoSvhG+/xhgTpNUD74nI2cDtQA7wAfAnVV0W6gFVdQH1ht9Q1duCpn8Q6j7jWU56Gx8X2pSMHNfvwZsYvn0aY0w9oYzK+r/AZcAaYBxwl4jcr6rPRCSyTiIn3UPpgTAmB4DxZ4PYCCbGmMgJpZ/DblX9j6ruU9WFwFnAzyMUV6cR9pID1CaGHZtgf3zexmuM6dhCSQ6bROROf0MyuDaBqgjE1KnkpHs4ckypqApzX76KY/DGXHh/YXj3a4wxhJYcqnF3L20VkaXAemBJvbuLTD257X0iXFOSkt0jRLd+Bru3hHffxpgur9XJQVW/paonAgOBH+AapwV4RETs6tSEdj8utDkjJ0JqBqxcCNphRxkxxsShkB8TqqrHgBX+l2lBTXIIe7sDQGISFJwO7/zLlSAGjgz/MYwxXZI9vT7CctJd4/H+wxFIDgDHjYVeg8BXGZn9G2O6pJBLDiY0qUlCckKEqpUAPB6YPstubTXGhFWrSw6NPXNtnbYAABscSURBVHRHRE4PazSdkIj4HxcawTYBEaiuhvXvw9HDkTuOMabLCKVa6e8icrM4qSJyL/CbSAXWmeS291nSrVF2wLU9rFoU2eMYY7qEUJLDRNyIqm/jBtDbDpwSiaA6m4h0hKsvMxdGTnClh707InssY0ynF0pyqASOAqlACrBJVSN8xesccjLcs6Q10rebjpkCyWnw3gK7tdUY0y6hJIfluOQwHpiMe/7zvIhE1cnkpHmoqobD5RG+YCelwLgzoXSrG9bbGGPaKJS7la5W1WL/9A7gfBG5PAIxdTrBD/3JTI3w3cNDC2DnZlfNZIwxbdTq5KCqxSKSCwzHVSsBfB6RqDqZQF+HsmoG9IjwwUTglA778DxjTJwI5XkO1+CGzegPrAImAcuAL0cmtM4jor2km1JVCR+9Cb2HQp8h0TuuMaZTCKWO4we49obPVXUqcBKwPyJRdTLZaR5EYF+kekk3ZfNqePdf4LPBc40xoQklOZSrajmAiCSr6qfAiMiE1bkkeIWcNA97D0UxOSQkwsRz4OBeWLU4esc1xnQKoSSHEhHJAV4AXhORF7E2h1brnhnl5ADQdxgMHwtr3obSkuge2xjToYUyZPeFqrpfVW8H/ht4FDg/UoF1Nt1ikRwAxp0FaVnwzj+t74MxptVCaZAuwj0WdJD/fQL8GhgTmdA6l+6ZHlZsqKa6WvF4ojhIXlIyTL4YEpNtcD5jTKuF0s9hDnAT8BHuqXAmBN0zPfiqYf8RpVtGlC/SPQfUTleUu85yxhjTjFDaHEpVdb6qblLVz2teoR5QRGaIyGcisl5Ebmlk/Y9FZI2IfCgir4vIoFCPEY+6Z7pT/cUhX+yCWPEavPSwe/60McY0I5Tk8AsReVRELhWRr9W8QjmYiHiB+4GzgVG4ITjqDwX+PlCkqmOA54Dfh3KMeNU9wwsQm3aHGgNGwOF9buwlY4xpRijVSlcBI4FEaquVFPhHCPuYAKxX1Y0AIvIsrlF7Tc0Gqhp83+U7wGUh7D9udfOXHGKaHHoOdIPzfbDE3ck01JqLjDGNCyU5jFfV9vZr6AdsDZovwQ0F3pSrgZcbWyEis4HZAAMHDmxnWJGXnChkpAhfRLsjXH2jT4MdG13nuB79IatbbOMxxsSlUKqV3m7saXCRIiKXAUXAXY2tV9WHVbVIVYvy8vKiFVa7xOx21mAeD0y+CFLSXRWTMcY0IpSSwyRglYhsAo7hbmVVf9tAa23DPTCoRn//sjpE5AzcbbNTVLXTtJ52z/Cwc38c3OiVng3n3QBeb6wjMcbEqVCSw4wwHG85MFxEhuCSwiXAt4I3EJGTgIeAGaq6OwzHjBvdMz2s2VqJqiKx7nPg9bpOcZ++60oRQ0bHNh5jTFwJZcjudg+VoapVInID8CrgBR5X1dUicgdQrKrzcdVIGcA8/wV0i6qe195jx4PumV6OVbmH/mSmxkGHNFXY8gns3Q45PSG3V6wjMsbEiVBKDmGhqguABfWW3RY0fUa0Y4qWvGzXxFN6MAoP/WkNjwdO+zq89BAsmQvnzLYOcsYYILQGadNOPbNdHf/uAzHsCFdfaoZLEGX7Yek/oDoO2kSMMTFnySGKemR5EGD3gTi7APccCEUzYNs62L0l1tEYY+JA1KuVurJEr9At00NpPJUcaoycAL0GWbuDMQawkkPU9cz2sCsekwPUJobtG1xHOWNMl2XJIcrysryUxlu1UrDqalj5Grzxd9hfGutojDExYskhynpmezhcrpSVx2mC8Hjg9EvA44XXn4KyA7GOyBgTA5YcoqzmjqXSg3GaHAAycmDaZe7ZD68/BeVHYh2RMSbKLDlEWU9/X4e4up21Md37wNRL4dA+2PB+rKMxxkSZ3a0UZT2yavo6xHHJoUbvwXDOdyC7R6wjMcZEmZUcoiw5UeiW4WHHvjgvOdTIyXPPnj64F5bNB19VrCMyxkSBJYcY6NvNy/YvOkhyqFG6FdatdHcxWYIwptOz5BADfbt52bnfh69aYx1K6w0rhInnQMlaeHOeJQhjOjlLDjHQt5uXKl+c37HUmBHjXYLY+pmVIIzp5KxBOgb65rpG6e1f+Oid08EeuDNivGuD2Pih6zDXwcI3xrSOlRxioE+32uTQIR1fBNNnQWISVByDo2WxjsgYE2aWHGIgJVHonunpuMkBXE9qgKX/B68+Dof3xzYeY0xYWXKIkb7dvB3ndtbm5J8K5WXwymPuiXLGmE7BkkOM9Pcnh0pfB7pjqTE9B8JZV4F44NUn3GNHjTEdniWHGBnUMwFfNWzb2wlKD7m94CvXup/LX4GqylhHZIxpJ7tbKUYG5rlG6c9LqxjcsxP8GlIz4Mwr4chBSEgEnw+02k0bYzocKznESI9MD2nJwuelnaDkUCMhEbK6u+mVr7l2iEP7YhuTMaZNLDnEiIgwKM/L57s7aUeyPkPdHUwvPeR6VRtjOpSoJwcRmSEin4nIehG5pZH1p4nIShGpEpGLox1fNA3OS2DbF52gUbox/Y93I7pm5MKip+H9111VkzGmQ4hqchARL3A/cDYwCrhUREbV22wLMAt4OpqxxUJNo3TJnk560czMhRnfhuFjYc0yKLO+EMZ0FNEuOUwA1qvqRlWtAJ4Fzg/eQFU3q+qHQAcbeCh0Q3q5hugNOztp1RK4dogvnQfnfa+2PWLHJtBOWFoyphOJdnLoB2wNmi/xL+uSumV46JHlYe32LnDrZ2au+7ltPbz2JCyZC0cPxzYmY0yTOmyDtIjMFpFiESkuLS2NdThtNrxPAut2VKFd5Zt032EwbjpsWwfz74cNq6wUYUwcinZy2AYMCJrv718WMlV9WFWLVLUoLy8vLMHFwvF9Ezlcruzc1+lr0RwROPFkOPc6yM6D/7wA/3k+1lEZY+qJdu+r5cBwERmCSwqXAN+Kcgxx5fi+7lewdkdlYLTWLiEnzw27sbbYdaAD/91MCt5O0CnQmA4uqiUHVa0CbgBeBT4B/q6qq0XkDhE5D0BExotICfB14CERWR3NGKMtL8tDTrrwSUkXaHeoT8Q9H2LgCW7+03fhnw9Yvwhj4kDUv6Kp6gJgQb1ltwVNL8dVN3UJIkL+wCRWbKigyqckeCXWIcVOt94uYSx6GvoNh6KzILtHrKMypkvqsA3SncnoQYkcrdDOfUtra/QZCude75LC7i3wz/+FT96JdVTGdElWuRsHRvVPxOuBjz6vZES/Lj5QndcLo74EQ0bDR29Ctz5u+bGj7mdyauxiM6YLsZJDHEhJEo7vm8AHmyu6zi2tLUnNgAlfgV6D3PwHi+H5P8OHb9QmCmNMxFhyiBNjhyaxc381JZ3h+Q6RMHycSxSrFsM//gQrXrNOdMZEkCWHODFuWBJeD7y7tiLWocSn3F4w9VL46vXQfwSseRtWLox1VMZ0WpYc4kRmqocTByTy3roKqq1qqWm5vWDyRXD+96HgdLds73Z3h9P2Ddbb2pgwseQQRyYdn8S+smpWb+mCfR5CldUNMnLcdNlB2LMNFv7NDcnx6XtQcSy28RnTwVlyiCMnDU0iO01Y9JFd2EIycCRc9CM49WuQmAzvLXBJorqLDEliTATYraxxJMErTDkxhfnLj7Jzv4/eOV1oOI328ibA0DHutWebezypx+OqmV6fAz0HwrACSM+OdaTGdAhWcogzU05MJsELr6y02zXbrEc/GJLvpo8dhWofrFoE//cnePkx17HO7nQyplmWHOJMVpqHqfkpvP1ZBTu+sNta2y0lDaZfCRf+AE6aBlUVsPwVKPU/VqT8iGuzMMbUYdVKcejssSm8uaac/3vnCDd8JTPW4XQOmbkwerJ7HdhT25i9drnrO9Gtj3vu9YARblq68BhXxmAlh7iUmerh3KJUPthcyYoN1u8h7LJ71A4LPjgfxp7hHmf60Zvw0sPwwj21jdk+K72ZrslKDnHqzIIUlq+v4Ok3yxjeJ4GsNMvjEZHVHfJPda/yI+4JdUcOusZsgFcfd20WvYe4gQF7DoLEpNjGbEwU2BUnTnk9wlVT0ymvVB7692GqfNa5K+JS0twdTaMnu3lV96yJpFTXd+L1OTD3t/D+67XvsYZt00lZySGO9e+RwBWnp/PowjL+uqSMWV9Ox2N14dEjUluqqKp0jdg7N9WOFHvkEDz3/1z7Rd4A9+re1/XiTujio+uaDs+SQ5ybeHwypQerefG9o4jA5VPSu/YDgWIlIdFVK/UZWrvM44XxM1zS2LUZNn3klp/6Ndff4uAXsG2tSya5vSEpOSahG9MWlhw6gHOLUqlW+Ofyo+w5WM13pmdYG0Q8SEmDEya5l6prq9i7w/WzAJc0lr9Su31GDmTnuaHIM3OhohwQSxomLlly6CDOG59KzywPTy4p4xfPHuCSU9OYMDwJsWqm+CDiel8H98AeVgB9h8EXO1zSOFAK+3e7IT7APTN71WL3nuw81ziemQvHj3cPPVK1W2pNzFhy6EAmjUhmQJ6XJxeX8ejCMhZ+WM5541PJH5hoSSJepWa452H3G95wXd/jQDwuYRzY40oa1T4YOdGtf+dfsH09ZHZzr4wcyMit7f1tycNEkCWHDqZftwRuuTCL/3xawUsrjnLPS4fpme3h1BOSKRqWRF62jcfUYfToV1sFBe5iX1Fee8HvOQB8lXDoC9j6ibvVNrNbbXJ4fQ7s21lbYknLgm69YVihW39oHyQkueovSyImRNIZHktZVFSkxcXFsQ4j6qp8SvH6Ct765Bhrt1cB0Lebl9EDEzmuTwJDe1n/iE6lsgIqjtZWXX223FVZlR1wryMH3d1S02e59S/e50okHg+kpENKBvQ7zg0jArB2hVuXlOoSSFKqK+nYc7q7DBFZoapFja2zkkMHluAVJo1IZtKIZHYf8PHB5ko+2FTBwg/LeXWV26ZHloe+uV765Hrp081Lr2wP3TI8ZKd78Hrs22SHkphUtwPeiPF116uCr6p2fuyZcHg/lB92/THKy4Cg3/mKf0NlveHhhxXCKRe4fb14nyt5JKe6V2Kyqx4beILrQb7xA7cs+JWWCUkpYf/oJvqinhxEZAbwZ8ALPKqqv623Phn4KzAO2At8U1U3RzvOjqZntpczC7ycWZBCRZWypbSKDTur2Lzbx459PtZsraQq6PEGHoHsNA/dMj1kpwkZKR7SU4SMFDedkSKkJQspSUJKopCc6H4meLH2jXglUrd/xYARzW9/0Y9dSeTYETd67bGjkJ7l1lVXQ4/+teuPHHSJJC3LJYeKcnj7xYb7LPwyjDnNDWY4/34XT0KiSzIJiTDqZBg0yq3/YLFb5k2s3a7vcMjt6WLZvcUNc+Lx1v7MzHXJx1flSlKBdR6rOguzqCYHEfEC9wNnAiXAchGZr6prgja7GtinqseJyCXA74BvRjPOji4pQTiuTyLH9am9UPiqldKD1ZQe8LHvcDVfBL127q/m8NEqDpcr1S3UMno9kJwoJCcIyYnuWAleV4pJ9AZPE5gPXu7xuH14xPUC9/inPSJuuQe8Ah6PuOWB7d16ETcPbr2I+y4sDaalmXU109LMOndM/PMeIfClu/4lqOY9wStr9h/XkpLdq2YQwmBeL5x6YTPvTYGv/dAljJpXxTHI6enWe7xw3Emu86Cv0v2sqnTLwSWXHRvdKLlVlbUlntRMlxz27YLFzzQ87umXuIc77dzk2lxqiLh9T7sMeg+GkrXuNuKapOLxuJ+Tvgo5ebBtvRt0UTy168TjqtzSMmHnZij5rOH6kRPdOduzzT2e1uOt/cNB3FhdXi98sdO1FYkn6A/L4+5eE4GDe11pruZ9Ho9b393fwbLMn4yD3+vx1FYpVpS7cb+83oiV1KLa5iAiXwJuV9Wz/PM/A1DV3wRt86p/m2UikgDsBPK0mUC7aptDuKkqRyuUw+VKWblSdqyaY5VQXqmUVyjHKpXySvezZrqyyrV9VFVDpa/ufFW9eV8XfzBbcAKpma6fPyRow8a2aU0Samz7OssaOWaDZW3Ia9IgbYayv+B/b3Hz1dVIzfKaf/+aC311tUs6qP+t/vUJyYjX45JNxbG6+1VFUtLcPqoq3AU2+L2oKxmJByrLXekFXAw1m2TkuOPXlLSod1nK7OY+bPkRpKKRZ7JkdXe//6OH/f1c6p2krO5u+uihBo+6FY/HPR4X/MmjAhISmTKxF9PGtC1BxFObQz9ga9B8CTCxqW1UtUpEDgDdgT3BG4nIbGA2wMCBAyMVb5ci4qqS0pKBCDwwrVrV/U9X+6vHq11JxVcN1dWKz10PqNa6877quu+tVg1cK6qVBtOqoGjQNI1MaxPLg+c1MF0zSGvgUqD15qmNQ4Nm6n+jqdlfo/sIWtfYsvrba9DKlvZRG1e9fQRfOxtMNNTUqrZ+yWzuXS3vsuEAiLXvSQCau2AmAGnNHCsDyKgbX/Ctw9XpoKm1fzCB3SaACFqVCr4kak+4/2ey/5JbmQpVNZdfrd0k1b/sWDJUeequE4F0//rURKgCvAlkpUamhNphG6RV9WHgYXAlhxiHY1rBI4LHCwmBu23jvNrFmIjJiPD69ov2fY7bgAFB8/39yxrdxl+tlI1rmDbGGBMl0U4Oy4HhIjJERJKAS4D59baZD1zpn74YWNRce4Mxxpjwi2q1kr8N4QbgVdytrI+r6moRuQMoVtX5wGPA30RkPfAFLoEYY4yJoqi3OajqAmBBvWW3BU2XA1+PdlzGGGNq2dgKxhhjGrDkYIwxpgFLDsYYYxqw5GCMMaaBTjFkt4iUAp+38e09qNf7Ok5YXKGL19gsrtBYXKFpT1yDVDWvsRWdIjm0h4gUNzW2SCxZXKGL19gsrtBYXKGJVFxWrWSMMaYBSw7GGGMasOTgH7wvDllcoYvX2Cyu0FhcoYlIXF2+zcEYY0xDVnIwxhjTgCUHY4wxDXTp5CAiM0TkMxFZLyK3ROF4A0RksYisEZHVIvID//LbRWSbiKzyv74S9J6f+eP7TETOilTsIrJZRD7yH7/Yv6ybiLwmIuv8P3P9y0VE7vEf+0MRGRu0nyv9268TkSubOl4rYxoRdE5WichBEflhLM6XiDwuIrtF5OOgZWE7PyIyzn/+1/vf26onITUR110i8qn/2M+LSI5/+WARORp03h5s6fhNfcY2xhW235u4Yf/f9S+fK+4RAG2Na25QTJtFZFUMzldT14bY/Y25RyF2vRduyPANwFDc8wY/AEZF+Jh9gLH+6UxgLTAKuB34/xrZfpQ/rmRgiD9ebyRiBzYDPeot+z1wi3/6FuB3/umvAC/jHuU2CXjXv7wbsNH/M9c/nRvG39dOYFAszhdwGjAW+DgS5wd4z7+t+N97djvimg4k+Kd/FxTX4ODt6u2n0eM39RnbGFfYfm/A34FL/NMPAte3Na566/8fcFsMzldT14aY/Y115ZLDBGC9qm5U1QrgWeD8SB5QVXeo6kr/9CHgE9wzs5tyPvCsqh5T1U3Aen/c0Yr9fOBJ//STwAVBy/+qzjtAjoj0Ac4CXlPVL1R1H/AaMCNMsUwDNqhqcz3hI3a+VPVN3PNF6h+v3efHvy5LVd9R91/816B9hRyXqv5bVav8s+/gnrjYpBaO39RnDDmuZoT0e/N/4/0y8Fw44/Lv9xvAM83tI0Lnq6lrQ8z+xrpycugHbA2aL6H5C3VYichg4CTgXf+iG/zFw8eDiqJNxRiJ2BX4t4isEJHZ/mW9VHWHf3on0CsGcdW4hLr/tLE+XxC+89PPPx3u+AC+jfuWWGOIiLwvIm+IyOSgeJs6flOfsa3C8XvrDuwPSoDhOl+TgV2qui5oWdTPV71rQ8z+xrpycogZEckA/g/4oaoeBB4AhgGFwA5c0TbaTlXVscDZwPdE5LTglf5vGzG579lfn3weMM+/KB7OVx2xPD9NEZGfA1XAHP+iHcBAVT0J+DHwtIhktXZ/YfiMcfd7q+dS6n4Bifr5auTa0K79tUdXTg7bgAFB8/39yyJKRBJxv/w5qvoPAFXdpao+Va0GHsEVp5uLMeyxq+o2/8/dwPP+GHb5i6M1Rend0Y7L72xgparu8scY8/PlF67zs426VT/tjk9EZgHnAjP9FxX81TZ7/dMrcPX5x7dw/KY+Y8jC+Hvbi6tGSai3vM38+/oaMDco3qier8auDc3sL/J/Y61pLOmML9wjUjfiGsBqGrtOjPAxBVfXd3e95X2Cpn+Eq38FOJG6DXUbcY10YY0dSAcyg6bfxrUV3EXdxrDf+6fPoW5j2Hta2xi2CdcQluuf7haG8/YscFWszxf1GijDeX5o2Fj4lXbENQNYA+TV2y4P8Pqnh+IuDs0ev6nP2Ma4wvZ7w5Uigxukv9vWuILO2RuxOl80fW2I2d9YxC6EHeGFa/Ffi/tG8PMoHO9UXLHwQ2CV//UV4G/AR/7l8+v9E/3cH99nBN1dEM7Y/X/4H/hfq2v2h6vbfR1YBywM+iMT4H7/sT8CioL29W1cg+J6gi7o7YgtHfdNMTtoWdTPF666YQdQiauvvTqc5wcoAj72v+c+/KMXtDGu9bh655q/sQf9217k//2uAlYCX23p+E19xjbGFbbfm/9v9j3/Z50HJLc1Lv/yvwDX1ds2muerqWtDzP7GbPgMY4wxDXTlNgdjjDFNsORgjDGmAUsOxhhjGrDkYIwxpgFLDsYYYxqw5GBMM0TEJ3VHhg3b6L3+UT8/bnlLY6IvoeVNjOnSjqpqYayDMCbarORgTBv4x/3/vX98/PdE5Dj/8sEissg/uNzrIjLQv7yXuGcrfOB/nezflVdEHvGP4f9vEUn1b3+jf2z/D0Xk2Rh9TNOFWXIwpnmp9aqVvhm07oCqjsb1Nr3bv+xe4ElVHYMb8O4e//J7cMMzFOCeJ7Dav3w4cL+qngjsx/XKBTdUwkn+/VwXqQ9nTFOsh7QxzRCRw6qa0cjyzcCXVXWjf8C0naraXUT24IaFqPQv36GqPUSkFOivqseC9jEYN/b+cP/8zUCiqt4pIq8Ah4EXgBdU9XCEP6oxdVjJwZi20yamQ3EsaNpHbTvgObixc8YCy4NGIDUmKiw5GNN23wz6ucw//TbuwUQAM4G3/NOvA9cDiIhXRLKb2qmIeIABqroYuBnIBhqUXoyJJPs2YkzzUsX/wHm/V1S15nbWXBH5EPft/1L/su8DT4jITUApcJV/+Q+Ah0XkalwJ4Xrc6KCN8QJP+ROIAPeo6v6wfSJjWsHaHIxpA3+bQ5Gq7ol1LMZEglUrGWOMacBKDsYYYxqwkoMxxpgGLDkYY4xpwJKDMcaYBiw5GGOMacCSgzHGmAb+f2u0ErEwfpEQAAAAAElFTkSuQmCC\n",
315 |       "text/plain": [
316 |        "<Figure size 432x288 with 1 Axes>"
317 |       ]
318 |      },
319 |      "metadata": {},
320 |      "output_type": "display_data"
321 |     }
322 |    ],
323 |    "source": [
324 |     "num_epochs = 20000\n",
325 |     "\n",
326 |     "cutoff = 3000\n",
327 |     "epsilon = np.exp(-np.arange(num_epochs)/(cutoff))\n",
328 |     "epsilon[epsilon > epsilon[100*int(num_epochs/cutoff)]] = epsilon[100*int(num_epochs/cutoff)]\n",
329 |     "plt.plot(epsilon, color = 'orangered', ls = '--')\n",
330 |     "plt.xlabel('Epochs')\n",
331 |     "plt.ylabel('Epsilon')\n",
332 |     "plt.savefig('epsilon_profile.pdf', dpi = 300, bbox_inches = 'tight')\n",
333 |     "plt.show()\n",
334 |     "\n",
335 |     "mp = []\n",
336 |     "mpm = []\n",
337 |     "reg = 200\n",
338 |     "for e in epsilon:\n",
339 |     "    a = agent.env.reset_policy(e)\n",
340 |     "    mp.append(np.min(a))\n",
341 |     "    mpm.append(np.max(a))\n",
342 |     "\n",
343 |     "plt.plot(epsilon/1.3, color = 'orangered', ls = '--', alpha = 0.5,\n",
344 |     "         label= 'Epsilon profile (arbitrary units)')\n",
345 |     "\n",
346 |     "plt.plot(np.array(mpm)-np.array(mp), label = 'Probability difference', color = 'cornflowerblue')\n",
347 |     "plt.xlabel('Epochs')\n",
348 |     "plt.ylabel(r'max $p^r$ - min $p^r$')\n",
349 |     "plt.legend()\n",
350 |     "plt.savefig('reset_policy.pdf', dpi = 300, bbox_inches = 'tight')\n",
351 |     "plt.show()"
352 |    ]
353 |   },
354 |   {
355 |    "cell_type": "markdown",
356 |    "metadata": {},
357 |    "source": [
358 |     "**Training the network.**"
359 |    ]
360 |   },
361 |   {
362 |    "cell_type": "code",
363 |    "execution_count": 85,
364 |    "metadata": {},
365 |    "outputs": [
366 |     {
367 |      "name": "stdout",
368 |      "output_type": "stream",
369 |      "text": [
370 |       "Epoch 19999 (number of moves 43)\n",
371 |       "Game won\n",
372 |       "[####################################################################################################]\n",
373 |       "\t Average loss: 0.00128\n",
374 |       "\t Best average loss of the last 50 epochs: 0.00133, achieved at epoch 17204\n"
375 |      ]
376 |     }
377 |    ],
378 |    "source": [
379 |     "loss_log = []\n",
380 |     "best_loss = 1e5\n",
381 |     "\n",
382 |     "running_loss = 0\n",
383 |     "\n",
384 |     "for epoch in range(num_epochs):\n",
385 |     "    loss = 0\n",
386 |     "    counter = 0\n",
387 |     "    eps = epsilon[epoch]\n",
388 |     "    \n",
389 |     "    agent.isgameon = True\n",
390 |     "    _ = agent.env.reset(eps)\n",
391 |     "    \n",
392 |     "    while agent.isgameon:\n",
393 |     "        agent.make_a_move(net, eps)\n",
394 |     "        counter += 1\n",
395 |     "        \n",
396 |     "        if len(agent.buffer) < buffer_start_size:\n",
397 |     "            continue\n",
398 |     "            \n",
399 |     "        optimizer.zero_grad()\n",
400 |     "        batch = agent.buffer.sample(batch_size, device = device)\n",
401 |     "        loss_t = Qloss(batch, net, gamma = gamma, device = device)\n",
402 |     "        loss_t.backward()\n",
403 |     "        optimizer.step()\n",
404 |     "        \n",
405 |     "        loss += loss_t.item()\n",
406 |     "    \n",
407 |     "    if (agent.env.current_position == agent.env.goal).all():\n",
408 |     "        result = 'won'\n",
409 |     "    else:\n",
410 |     "        result = 'lost'\n",
411 |     "    \n",
412 |     "    if epoch%1000 == 0:\n",
413 |     "        agent.plot_policy_map(net, 'sol_epoch_'+str(epoch)+'.pdf', [0.35,-0.3])\n",
414 |     "    \n",
415 |     "    loss_log.append(loss)\n",
416 |     "    \n",
417 |     "    if (epoch > 2000):\n",
418 |     "        running_loss = np.mean(loss_log[-50:])\n",
419 |     "        if running_loss < best_loss:\n",
420 |     "            best_loss = running_loss\n",
421 |     "            torch.save(net.state_dict(), \"best.torch\")\n",
422 |     "            estop = epoch\n",
423 |     "    \n",
424 |     "    print('Epoch', epoch, '(number of moves ' + str(counter) + ')')\n",
425 |     "    print('Game', result)\n",
426 |     "    print('[' + '#'*(100-int(100*(1 - epoch/num_epochs))) +\n",
427 |     "          ' '*int(100*(1 - epoch/num_epochs)) + ']')\n",
428 |     "    print('\\t Average loss: ' + f'{loss:.5f}')\n",
429 |     "    if (epoch > 2000):\n",
430 |     "        print('\\t Best average loss of the last 50 epochs: ' + f'{best_loss:.5f}' + ', achieved at epoch', estop)\n",
431 |     "    clear_output(wait = True)"
432 |    ]
433 |   },
434 |   {
435 |    "cell_type": "code",
436 |    "execution_count": 86,
437 |    "metadata": {},
438 |    "outputs": [],
439 |    "source": [
440 |     "torch.save(net.state_dict(), \"net.torch\")"
441 |    ]
442 |   },
443 |   {
444 |    "cell_type": "code",
445 |    "execution_count": 87,
446 |    "metadata": {},
447 |    "outputs": [
448 |     {
449 |      "data": {
450 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXwU5f3A8c+zR7IJOQgQIBAgAZErJAEC4VbAG8ULT1TwomortbZWalur/NR6tYrWSqWioNQDapHiUeXyQJTLcAa5RAhnCEdIyL3P74+ZLJuQzbm7k5Dv+/XaV2ZnZme+O7uZ7z7zPPM8SmuNEEIIAWCzOgAhhBCNhyQFIYQQHpIUhBBCeEhSEEII4SFJQQghhIfD6gAaok2bNjohIcHqMIQQoklZu3btEa11bFXLmnRSSEhIYM2aNVaHIYQQTYpS6idfy+TykRBCCA9JCkIIITwkKQghhPBo0nUKomkqKSkhKyuLwsJCq0MR4qzmcrmIj4/H6XTW+jWSFETQZWVlERkZSUJCAkopq8MR4qyktSYnJ4esrCwSExNr/Tq5fCSCrrCwkNatW0tCECKAlFK0bt26ziVySQrCEpIQhAi8+vyfNdukcOBoGdv2l1gdhhBCNCrNNik8+u4Jnltw0uowhEXsdjupqamex9NPP13nbaxZs4YpU6YA8Oabb/KLX/zC32HWy9atW0lNTaVfv37s3LmToUOHArB7926SkpLqtK2CggLOO+88ysrK6vS6iIiIKuc/+uijLF68GIAXX3yRU6dO1Wm7wXTXXXexZcsWAJ566inP/OLiYkaOHElpaalVoQWUVDSLZiksLIyMjIwGbSMtLY20tDQ/RVQ3ZWVl2O32KpctWLCA8ePH84c//AGAb775pt77mTVrFtdcc43PfVWmtaa6gbumTZvmmX7xxRe55ZZbCA8PP2O96t6ft9LSUhyOwJzG/vnPf3qmn3rqKR555BEAQkJCGDNmDO+99x4TJkwIyL6t1GxLCkJUJSEhgd/+9rf07duXQYMGsWPHDgDmzZtHUlISKSkpjBw5EoDly5dz+eWXn7GN3bt3M3r0aJKTkxkzZgx79uwBYNKkSUyZMoWhQ4fStWtX5s+fX+Vre/bsyYQJE+jVqxfjx4/3/JpOSEjg4Ycfpn///sybN4+MjAwGDx5McnIyV199NceOHePjjz/mxRdf5NVXX2XUqFFA1b/ay8rKeOihhxg4cCDJycn84x//qPJ4zJ07lyuvvBKAvLw8xowZQ//+/enbty8ffvihJ+YePXpw2223kZSUxN69ewH41a9+RZ8+fRgzZgzZ2dmeYzB//nxeeukl9u/fz6hRoyrE+etf/5qUlBRWrlzJtGnTGDhwIElJSUyePNmTbM4//3weeOAB0tLSePLJJ0lMTKSkxLgUnJubW+F5ufL9lis/JsuXL+f8889n/PjxnuPuvZ81a9YwdepUCgoKSE1N9SSBq666irlz51Z5zJo6KSkI683/y5nzug+AlPOhpBg+fPnM5b2GQJ+hUJAHH1U6oY3/dY27LP8nL/e73/2OG264AYDo6Gg2btzInDlzeOCBB1i0aBHTpk3jf//7Hx07duT48ePVbvv+++9n4sSJTJw4kVmzZjFlyhQWLFgAwIEDB/j666/ZunUr48aNY/z48We8/ocffuD1119n2LBh3HHHHfz973/nN7/5DQCtW7dm3bp1ACQnJ/Pyyy9z3nnn8eijj/L444/z4osvcs899xAREeF5TVVef/11oqOjWb16NUVFRQwbNoyLLrqoQtPF4uJidu3aRXmnky6Xi//85z9ERUVx5MgRBg8ezLhx4wDYvn07s2fPZvDgwQDk5+eTlpbGCy+8wLRp03j88cf529/+5tn2lClT+Otf/8qyZcto06aN5zXp6en85S/G96F37948+uijANx6660sWrSIK664whNbeb9nu3fv5qOPPuKqq67i3Xff5ZprrqlTu/zvv/+ezZs306FDB4YNG8aKFSsYPny4Z/nTTz/N3/72twoly6SkJFavXl3rfTQlUlIQzVL55aPyR3lCALjppps8f1euXAnAsGHDmDRpEjNnzqzx+vrKlSu5+eabAeNk9vXXX3uWXXXVVdhsNnr37s2hQ4eqfH2nTp0YNmwYALfcckuF15fHeeLECY4fP855550HwMSJE/nyyy9r/f4/++wz5syZQ2pqKunp6eTk5LB9+/YK6xw5coSWLVt6nmuteeSRR0hOTuaCCy5g3759nvfQpUsXT0IAsNlsnlgrvwdf7HY71157ref5smXLSE9Pp2/fvixdupTNmzefcRzAuPb/xhtvAPDGG29w++231/o4AAwaNIj4+HhsNhupqans3r27VrGGhIRw8uTZVy8pJQVhvep+2TtDql8eFlGrkkFdeDfjK5+eMWMG3333HR999BEDBgxg7dq19dp2aGioZ9rXtffKzQi9n7do0aJe+61Ma83LL7/MxRdf7HOdsLCwCm3c586dS3Z2NmvXrsXpdJKQkOBZXlNctWka6XK5PPUIhYWF3HfffaxZs4ZOnTrx2GOPVYjFe3/Dhg1j9+7dLF++nLKysior0x0OB263GwC3201xcbFnmfdnYrfba12BXFRUhMvlqtW6TYmUFISo5L333vP8HTJkCAA7d+4kPT2dadOmERsb67luXpWhQ4fy7rvvAsaJdMSIEXXa/549ezwllH/9618VLmWUi46OJiYmhq+++gqAt956y1NqqI2LL76YV1991XPtfdu2beTn51dYJyYmhrKyMs/J+MSJE7Rt2xan08myZcv46SefvS/jdrs91/B9vYfIyEifv7TL99mmTRvy8vKqrH/xdtttt3HzzTf7LCUkJCR4EvnChQvPqHOoidPprPCanJwc2rRpU6fLVE2FJAXRLJXXKZQ/pk6d6ll27NgxkpOTmT59Oi+88AIADz30EH379iUpKYmhQ4eSkpLic9svv/wyb7zxBsnJybz11ltMnz69TrH16NGDV155hV69enHs2DHuvffeKtebPXs2Dz30EMnJyWRkZHiuv9fGXXfdRe/evenfvz9JSUn87Gc/q/IX8kUXXeS59DNhwgTWrFlD3759mTNnDj179vS5/RYtWrBq1SqSkpJYunRplbFNnjyZSy65xFPR7K1ly5bcfffdJCUlcfHFFzNw4MBq38+ECRM4duyY59JfZXfffTdffPGFpxK7riWuyZMnk5yc7KloXrZsGWPHjq3TNpqM8iZk/n4As4DDwCavea2Az4Ht5t8Yc74CXgJ2ABuA/rXZx4ABA3R93fVKjr7rlZx6v17U35YtW6wOwacuXbro7Oxsy/b/448/6j59+li2/8rWrl2rb7nlFqvDqNG8efOCGufVV1+tf/jhh6DtryGq+n8D1mgf59VAlhTeBC6pNG8qsERr3R1YYj4HuBTobj4mA68GMC4hRC3179+fUaNG1fnmtWC6//77mTp1Kn/84x+Dsr/i4mKuuuoqzj333KDsL9gCVtGstf5SKZVQafaVwPnm9GxgOfCwOX+OmcG+VUq1VErFaa0PBCo+IapSm5YngZSQkMCmTZssjaGyO+64w+oQqvXyy1U0WQ6gkJAQbrvttqDuM5iCXafQzutEfxBoZ053BLxr7rLMeWdQSk1WSq1RSq0pvyFGCCGEf1hW0WyWCnzfD+/7da9prdO01mmxsbEBiEwIIZqvYCeFQ0qpOADz72Fz/j6gk9d68eY8IYQQQRTspLAQmGhOTwQ+9Jp/mzIMBk5IfYIQQgRfwJKCUuodYCXQQymVpZS6E3gauFAptR24wHwO8DGwC6NJ6kzgvkDFJQSc7jo7KSmJ6667rs5dOPvqGtqXyh2ylfPV/faMGTOYM2eOZ/7+/fvrtD9v3h33LVy40NNNeHZ2Nunp6fTr14+vvvqKefPm0atXryrvG2hMxo8fz65du+r0mvLO7SrzPh4LFizwdJXdGFX3nbjxxhvP6KakvgLZ+qjqu0hgTBXrauDngYpFiMq8u86eMGECM2bM4MEHH/Qs97TZtgW2MO2r++177rnHM/3mm2+SlJREhw4dGry/cePGeTqxW7JkCX379vV0EX3JJZcwc+bMKu8+rkogu632ZfPmzZSVldG1a9dav6a65rTex2PBggVcfvnl9O7d+4z1avteA3lMqvtO3HvvvTz77LPMnDmzwfuRO5pFszdixAh27NhRZRfQ77zzjudO5ocffrjC66rqGnrmzJkMHDiQlJQUrr322golkMWLF5OWlsa5557LokWLAN/dbz/22GM8//zzzJ8/nzVr1jBhwgRSU1M9vYGW+/zzz7n66qvPeP2nn35Kz5496d+/Px988IFnfnlpJCMjg9/+9rd8+OGHpKam8vjjj/P1119z55138tBDD/nsWnv58uWMGDGCcePGeU6eb7/9NoMGDSI1NZWf/exnnpNwREQEv//970lJSWHw4MGezvMOHTrE1VdfTUpKCikpKZ7xHnxtx5t3V95gnAzT0tLo06cPf/rTnzzzK3czDkZXIOWlw1WrVlU4Ht988w0LFy7koYceIjU1lZ07d1boonv69On897//9ZSsLrjgAs/7eeyxx7j11lsZNmwYt956KyNHjqzQo+rw4cNZv359hfdReVCmyy+/nOXLl1d73Hx9JwoKChgxYgSLFy/2y8A/khSE5RYvXnzGY9u2bYDxy6uq5Tt37gSMPnIqL6uL0tJSPvnkE/r27QsYXUDfd999bN68GafTycMPP8zSpUvJyMhg9erVni6wy7uG3rx5M+eddx6PP/44ANdccw2rV69m/fr19OrVi9dff92zr927d7Nq1So++ugj7rnnnloNqD5+/HjS0tKYO3cuGRkZXHbZZWzdutWThN54440z7iMoLCzk7rvv5r///S9r167l4MGDZ2w3NTWVadOmccMNN5CRkcGf/vQnz36ee+65Cl1rr169mpkzZ/Ljjz8CsG7dOqZPn862bdvIzMzkvffeY8WKFWRkZGC32z3jDOTn5zN48GDWr1/PyJEjPb9ip0yZwnnnncf69etZt24dffr0qXY73lasWMGAAQM8z5988knWrFnDhg0b+OKLL9iwYYNnWXk34zfeeCMAp06dIiMjg7///e9nHLOhQ4cybtw4nnvuOTIyMujWrRtwuovuX//61wwfPpxvv/2W77//nhtvvJFnn33W8/otW7awePFi3nnnHe68807efPNNwOhTqrCwsNpuUSrzddzKVf5OhIWFYbPZOOecc85IPvUhSUE0S+V9H6WlpdG5c2fuvPNOoGIX0KtXr+b8888nNjYWh8PBhAkTPN1T++oaetOmTYwYMYK+ffsyd+7cCt09X3/99dhsNrp3707Xrl3ZunVrneNWSnHrrbfy9ttvc/z4cVauXMmll15aYZ2tW7eSmJhI9+7dUUpxyy231Hk/1XWtPWjQIM+4C0uWLGHt2rUMHDiQ1NRUlixZ4rneHxIS4ikFDRgwwHNj4NKlSz39OdntdqKjo6vdjrcDBw7g3RT9/fffp3///vTr14/NmzdXqBPw7l4bTneJPnLkSHJzc2scF6PyNrKysrj44ovp27cvzz33XIXPdty4cYSFhQFw3XXXsWjRIkpKSpg1axaTJk2qcT/efB23mrRt27ZBdU/lpOtsYbkLLrjA5zKHw1HtcpfLVe1yX3wNx1nfrqnLu4aeNGkSCxYsICUlhTfffNNzScB7HV/Pa+v222/niiuuwOVycd111wXkGrb20bX28uXLKxwjrTUTJ07kz3/+8xnbcDqdnvdYU5fU1W3Hm3d33j/++CPPP/88q1evJiYmhkmTJvnsXhvqd/y9t3H//ffz4IMPMm7cOJYvX85jjz1W5Xrh4eFceOGFfPjhh7z//vtVdrPu3ZU3UCHuuhw3b4WFhZ7E1BBSUhDCh0GDBvHFF19w5MgRysrKeOeddzzdU/vqGvrkyZPExcVRUlJyxuWPefPm4Xa72blzJ7t27aJHjx61iqNyF9MdOnSgQ4cOPPHEE1V2Fd2zZ092797tucT2zjvv1Pm916ZrbYAxY8Ywf/58Dh82bjk6evRotV1ql7/m1VeN7s3Kyso4ceJErbfTq1cvzxCpubm5tGjRgujoaA4dOsQnn3xS7X7Lu0T/+uuviY6OJjo6usLy6rryBqPr8I4djY4WZs+eXe2+7rrrLqZMmcLAgQOJiYk5Y3lCQgIZGRm43W727t3rqeOorapi3bZtW5VjSdSVlBSE8CEuLo6nn36aUaNGobVm7NixnkrO8q6hn3jiCdq2bes54fzf//0f6enpxMbGkp6eXuEft3PnzgwaNIjc3FxmzJhR6wFaJk2axD333ENYWBgrV64kLCyMCRMmkJ2dTa9evc5Y3+Vy8dprrzF27FjCw8MZMWJEnUcIu+uuu9i9ezf9+/dHa01sbKynPsVb7969eeKJJ7joootwu904nU5eeeUVunTp4nPb06dPZ/Lkybz++uvY7XZeffVVhgwZUqvtjB07luXLl3PBBReQkpJCv3796NmzZ4XR6nxxuVz069fPc1mnshtvvJG7776bl156qcrmw4899hjXXXcdMTExjB492lPHUpUBAwYQFRXlc3yHYcOGkZiYSO/evenVqxf9+/evNvbKKn8ncnNzCQsLo3379nXaTlWU9jH6U1OQlpamq2p7XBt3//0oADPva+XPkEQtZGZmVnkyE7X3i1/8gn79+nnqQpqLgoICRo0axYoVKzyjtDVG+/fv5/zzz2fr1q0Bb9YM8MILLxAVFVXl96Gq/zel1Fqt9ZltoZHLR0I0OQMGDGDDhg31qkBu6sLCwnj88cfZt6/x9oIzZ84c0tPTefLJJ4OSEMAYlGjixIk1r1gLcvlIiCamvuNDny2qG1e6MbjtttuC3rW2r8tU9SElBSGEEB6SFIQQQnhIUhBCCOEhSUEIIYSHJAXRLNW162shmgtJCkIIITwkKQhh2r17N6NHjyY5OZkxY8awZ88ewOieIikpiZSUFEaOHAkY/fqXd/OcnJzstwFOhLCa3KcgLPXu1/nsPeJ7EJT66NTGzo3D696x3f3338/EiROZOHEis2bNYsqUKSxYsIBp06bxv//9j44dO3p61pwxYwa//OUvmTBhAsXFxdUO5CJEUyIlBSFMK1eu5Oabbwbg1ltv9XSHPWzYMCZNmsTMmTM9J/8hQ4bw1FNP8cwzz/DTTz/5pXdKIRoDKSkIS9XnF32wzZgxg++++46PPvqIAQMGsHbtWm6++WbS09P56KOPuOyyy/jHP/7B6NGjrQ5ViAaTkoIQpqFDh/Luu+8CxrCPI0aMAGDnzp2kp6czbdo0YmNj2bt3L7t27aJr165MmTKFK6+8ssKIX0I0ZVJSEM3SqVOniI+P9zx/8MEHefnll7n99tt57rnniI2N5Y033gDgoYceYvv27WitGTNmDCkpKTzzzDO89dZbOJ1O2rdvzyOPPGLVWxHCryQpiGbJe9Qrb0uXLj1jnvfA9+WmTp3K1KlT/R6XEFaTy0dCCCE8JCkIIYTwkKQgLNGUR/wToqmoz/+ZJAURdC6Xi5ycHEkMQgSQ1pqcnJxajwVeTiqaRdDFx8eTlZVFdna21aEIcVZzuVwVWtnVhiQFEXROp5PExESrwxBCVMGSy0dKqV8ppTYrpTYppd5RSrmUUolKqe+UUjuUUu8ppUKsiE0IIZqzoCcFpVRHYAqQprVOAuzAjcAzwAta63OAY8CdwY5NCCGaO6sqmh1AmFLKAYQDB4DRwHxz+WzgKotiE0KIZivoSUFrvQ94HtiDkQxOAGuB41rrUnO1LKBjsGMTQojmzorLRzHAlUAi0AFoAVxSh9dPVkqtUUqtkdYrQgjhX1ZcProA+FFrna21LgE+AIYBLc3LSQDxwL6qXqy1fk1rnaa1TouNjQ1OxEII0UxYkRT2AIOVUuFKKQWMAbYAy4Dx5joTgQ8tiE0IIZo1K+oUvsOoUF4HbDRjeA14GHhQKbUDaA28HuzYhBCiubPk5jWt9Z+AP1WavQsYZEE4QgghTNL3kRBCCA9JCkIIITwkKQghhPCQpCCEEMJDkoIQQggP6Tp78zegFCgbhIZB12Rj/r7tUFYGnXoYy4UQohmQpPD57NPTMe1OJ4UVC2D/DrjmV9C5pzWxCSFEkElSuOMp0BrcbrB5XU278DZ45yn4YZUkBSFEsyF1ClGtIboNxLQ1/paLaQfdUmHH91BW6vv1QghxFpGkUJ1zB0LRKfhpi9WRCCFEUEhSqE7nntBrMIRHWR2JEEIEhdQpVMfugItvtzoKIYQIGikp1Maxw5Cz3+oohBAi4CQp1ERrmP88rFxodSRCCBFwkhRqohR0HwA/boTiQqujEUKIgJKkUBs9BhrNUndmWB2JEEIElCSF2mifaNzDkPmt1ZEIIURASVKoDaWg52Cj24vCU1ZHI4QQASNNUmsrdbTxcIVbHYkQQgSMJIXaCmthdQRCCBFwcvmoLo4dgvefhYO7rY5ECCECQpJCXYRHweE9kLnS6kiEECIgJCnURWiY0XPqD6ul51QhxFlJkkJd9RoChfnGzWxCCHGWkaRQV517QYto2PKN1ZEIIYTfSeujurLZIH0s2OxWRyKEEH4nSaE+ks+zOgIhhAgIuXxUX0UFsHmF0YuqEEKcJSQp1NdPm+HzOTJUpxDirGJJUlBKtVRKzVdKbVVKZSqlhiilWimlPldKbTf/xlgRW611S4WwCNj0ldWRCCGE31hVUpgOfKq17gmkAJnAVGCJ1ro7sMR83njZHdB7KOxaD/m5VkcjhBB+EfSKZqVUNDASmASgtS4GipVSVwLnm6vNBpYDDwc6nkWLFqGUQilFREQEI0eOBGDVqlUcP368PGYAoqOjGTRo0OkXJw2HtZ8ZzVMHXhLoUIUQIuCsaH2UCGQDbyilUoC1wC+BdlrrA+Y6B4F2Vb1YKTUZmAzQuXPnBgfTsmVL3G43AGFhYZ75DocDh8M4PFprtFeFstaavLw8ImPaQfy5Mn6zEOKsYUVScAD9gfu11t8ppaZT6VKR1lorpaps1qO1fg14DSAtLa3BTX+GDx9e5fz+/fv7fE1mZiZbtmzhiiuuIPTK+8EZ0tAwhBCiUbCiTiELyNJaf2c+n4+RJA4ppeIAzL+HLYitVjp06EBxcTGZmZmnE0JRgbVBCSGEHwQ9KWitDwJ7lVI9zFljgC3AQmCiOW8i8GGwY6utli1bEh8fz86dOykrK4Md38Nrv4FjjTaPCSFErVjV+uh+YK5SagOQCjwFPA1cqJTaDlxgPm+0zjnnHIqKijh06BDEdTNuYtuw3OqwhBCiQapNCkqpW7ymh1Va9ov67lRrnaG1TtNaJ2utr9JaH9Na52itx2itu2utL9BaH63v9oOhXbt2OBwO9u7dCy2i4Nw04w7n4kKrQxNCiHqrqaTwoNf0y5WW3eHnWJoUu93OqFGj6NevnzEjZZSREDK/tTYwIYRogJqSgvIxXdXzZic2NpaQELOiOS4R2iXA+uXSH5IQosmqqUmq9jFd1fNmp6ysjK1bt9KqVSvi4uJg9M3gagGq2edLIUQTVVNS6GlWBiugmzmN+bxrQCNrAmw2G5mZmcTHxxtJoV0Xq0MSQogGqSkp9ApKFE2UUorY2FhycnJOz8zNgWXvwJAroW0n64ITQoh6qLZOQWv9k/cDyMO40ayN+bzZi4mJ4cSJE5SWlhozQsJg33ZY+z9rAxNCiHqoqUnqIqVUkjkdB2zCaHX0llLqgSDE1+jFxBg9fJd3nocrHPqOhO1r4cQRCyMTQoi6q6n1UaLWepM5fTvwudb6CiCdZt4ktVxMTAw2m438/PzTM1NHAwrWLbYsLiGEqI+akkKJ1/QY4GMArfVJwB2ooJqSFi1acP3119Oli1clc2QM9EyHzV9DQZ51wQkhRB3VVNG8Vyl1P0Yndv2BTwGUUmGAM8CxNQnlYzGcYcBF0DLWGIxHCCGaiJpKCncCfTAGxLlBa21eOGcw8EYA42pSdu/ezbJlyyqMuUDrOBh0GYS4rAtMCCHqqNqfsVrrw8A9VcxfBiwLVFBNTVFREQcOHKCgoIDw8PDTC7SG7eugrAR6DbYuQCGEqKVqk4JSamF1y7XW4/wbTtMUHR0NwIkTJyomBaWMTvKy90C3fhASalGEQghROzVd8B4C7AXeAb5D+juqUnlSyM3NNe5s9jb4cnjvGdjwBaRdZEF0QghRezXVKbQHHgGSgOnAhcARrfUXWusvAh1cU+FyuXA6neTm5p65MK4rdO4F6z6DkuLgByeEEHVQ0x3NZVrrT7XWEzEql3cAyxsylsLZSClF+/btcTh8FLwGXwGnTsLGL4MbmBBC1FGN7SWVUqHAWOAmIAF4CfhPYMNqekaMGOF7YYdu0GcYRMQELyAhhKiHmiqa52BcOvoYeNzr7mZRVxfeZnUEQghRo5rqFG4BugO/BL5RSuWaj5NKqSouoDdf2dnZLFy4kKNHqxlFtLQEvl8ChaeCF5gQQtRBTfcp1JQ0hMnhcJCXl8fJkydp1apV1SsdOwRfzoP8EzD8muAGKIQQtSAnfT+JjIwE4OTJk75Xio2HHoMgYynkHfe9nhBCWESSgp84HA7CwsKqTwoAQ8aB2w3fLgpOYEIIUQeSFPwoMjKy5qQQ3cYYb2HLCuNykhBCNCLShacfdezYkcLCwppXTB8Lxw4aFc9CCNGISFLwo169ajmkdXgkXCMD1wkhGh+5fORnWmvc7lqOP3TqpFG3UNv1hRAiwCQp+FFubi7vv/8+e/furd0L9m2Hb/8Lm74ObGBCCFFLkhT8KCwsjLKyMvLyajkE5zn9IP5c+GaB3NAmhGgULEsKSim7Uup7pdQi83miUuo7pdQOpdR7SqkQq2KrL6fTicvlqrkFUjmlYOT1UHQKvpMmqkII61lZUvglkOn1/BngBa31OcAxjKFAm5xaNUv11rYTJI2A9csgZ3/gAhNCiFqwJCkopeIxel79p/lcAaOB+eYqs4GrrIitoeqcFACGXgk90yE0vOZ1hRAigKxqkvoi8Fsg0nzeGjiutS41n2cBHat6oVJqMjAZoHPnzgEOs+7i4+Np0aIFWmuMXFcLYRFw0aSAxiWEELUR9KERTH4AABrWSURBVJKCUupy4LDWem19Xq+1fk1rnaa1TouNjfVzdA0XHx9P3759a58QvB3PhkUzjKaqQghhAStKCsOAcUqpywAXEIUx1GdLpZTDLC3EA/ssiK3BtNYUFRWhlCI0NLRuLy4rhV0bwBkKF98emACFEKIaQS8paK1/p7WO11onADcCS7XWE4BlwHhztYnAh8GOzR9KS0v54IMP2LFjR91f3DoO0i6GzG9hz1b/ByeEEDVoTPcpPAw8qJTagVHH8LrF8dRLnZulVjboMoiOhSVvQXGRf4MTQogaWJoUtNbLtdaXm9O7tNaDtNbnaK2v01o32TNivVoglXM44aKJkJsDaz71b2BCCFED6RAvACIjI9m/vwH3HHTsDpdNhi59/BeUEELUQmO6fHTWiIyMpLCwkJKSBnSN3b0/hIQa3WvLZSQhRJBISSEAOnbsiMvlaviGSkvgnaegwzkwZkLDtyeEEDWQkkIAtGzZkm7duuF0Ohu2IYcTEpJg45fw40b/BCeEENWQpBAgR48e5cSJEw3f0JBx0CYePnsT8v2wPSGEqIYkhQBZvnw5mZmZNa9YE4cTLr0LSovh01mgdcO3KYQQPkhSCJAGNUutrHUcnH8jFBfKuAtCiICSiuYAaXCz1Mp6D4VeQ8AmeVwIEThyhmmAkwVudh0qrXKZX5qlelPKSAiFp2Dx21BQy9HdhBCiDiQpNMCz/8nlz//OrXJZZKTRK3ith+asrRPZkLkSPvknuN3+3bYQotmTpNAAB4/7Pim3bduWUaNGERER4d+dtusCo26CPZmwcqF/ty2EaPakTiFAXC4XcXFxgdl40nA4+COs/gTaJ0K3lMDsRwjR7EhJIYAOHDjg38pmb+ffaJQavv63XEYSQviNlBQCaNOmTQB06NDB/xt3OOHye43KZ2mRJITwEzmbBFBkZKT/K5or7CAGWkQbJYWMZcbIbUII0QCSFAIoIiKCgoIC/zVL9WX/Dlj+Lnw+R+54FkI0iCSFAIqKigLw353NvsSfC0OvhK3fwaqPA7svIcRZTZJCAEVHRwP4p2O8mgy8FHoNNpqpbl4R+P0JIc5KUtEcQJGRkYwdO9ZzI1tAKQUX3AqnTsLy9yAxGcKDsF8hxFlFkkIA2Ww2T2khKOwOGPszOHZQEoIQol7k8lGAHThwgA0bNgRvhyGhxv0LAJu/gYO7g7dvIUST1+yTgg5wa53s7Gw2bdpEWVlZQPdzhpJiWPUR/Gc6HN4T3H0LIZqsZp8UAi2olc3enCFw7YMQGgb/fkESgxCiViQpBFh5UsjNrbo31YCKag3jfy2JQQhRa5IUAiwyMhKlFMePH7cmAO/EsG+HNTEIIZoMaX0UYHa73TPgjmWiWsMtj0KIy3heXGRUSAshRCWSFILg0ksvxW63WxtEeUI4vBf+8yKMuQXO6WdtTEKIRkcuHwWB5QnBW2QriGkHH/0DNn5ldTRCiEYm6ElBKdVJKbVMKbVFKbVZKfVLc34rpdTnSqnt5t+YYMcWKHl5eXz55ZccOXLE6lAgrAVc/QB06QNL3oaV/5VO9IQQHlaUFEqBX2utewODgZ8rpXoDU4ElWuvuwBLz+VnB4XCQlZXVOJICGM1Vr7gPeg+F7xbBlm+sjkgI0UgEvU5Ba30AOGBOn1RKZQIdgSuB883VZgPLgYeDHV99aK1RSvlc7nK5CAsL4+jRo0GMqgZ2O1x4G3TpDef0tzoaIUQjYWmdglIqAegHfAe0MxMGwEGgnUVhBUSrVq0aV1IAoxO9HgONBHHqJMx73qiIFkI0W5YlBaVUBPBv4AGtdYU7u7TR90SVF7qVUpOVUmuUUmuys7ODEGnNanNFPiYmhtzcXEpLG+noaAUnIfcIvPe0dL0tRDNmSVJQSjkxEsJcrfUH5uxDSqk4c3kccLiq12qtX9Nap2mt02JjY4MTcAN8taWIh+ccp02bNrRp08ba+xWq07oD3PR76NjdGMHts9lQGuAR44QQjY4VrY8U8DqQqbX+q9eihcBEc3oi8GGwYwuEOcvzOZrnpsjelnk70jhwshHfNBYeCVdNgfSxRuXzV/+2OiIhRJBZcfPaMOBWYKNSKsOc9wjwNPC+UupO4Cfgegtiqx8N+K5nBmDLXuNX98afSuge5wx8TPVls8GQcRDXDdp2NuYVF4Iz1KiDEEKc1axoffQ1vk+hY4IZSzAtWFUAwPZt29DpKdW2VmoUEvoYf91uWPAyuFoYI7vJ4D1CnNWa/R3N/rhtqy7bKC4pIT8/3w97DRKljO4wftoMb0+DXUEcMEgIEXTNPikEn6KxtJqqFaWg/wVw0yNGKWHhK/DJP6HwlNWRCSECQJJCkNlsNg4frrJhVePWpqORGIaMg+wsYzxoIcRZR5KCH9Sl66AWLcI5dOhQ4IIJJLvDaJk04Y9GVxmlJUbz1aMHrY5MCOEn8nMvyMIjY+jWuRtutxubrYnm5PJeX7P3wo51kLkSUkfDoLHgCrc2NiFEgzTRs1LTUbmN0eofnbTr1LPpJgRvcV1h4v9B72Hw/RKY86jRHbf0uipEk3UWnJmanv05RY2nx9SGCo+EC24x6htatjVKDUKIJkuSggV27NjB0qVLKSktxX22/Kpu2xmuewiu+LnRYik/F/79Auz9werIhBB1IEnBD6o9rVdxj1qbNm0oLS3lD3OPM+WfxwIVVvApZQziA3AiG44dhH//FT54EfZttzY2IUStSFIIsKruW27VqjUOh4Oj+TaKztY+5zp0g0lPwIjxcCTL6JZ73vNQVmZ1ZEKIakjrIwvYbDY6deoEZ0m1gk8OJwy4EJLPM7rjzj1yuuVS1jajf6XGNH61EEKSwrtfn2L9jyU8c1vL+m+kmutHSlW9fNb3Peq/v6bGGQKpo04/zzkA8/8CES2h70hIGgEtoqyLTwjh0ewvHy3bWMTRPLfVYTQvrdrDuJ8bYzisXAizfgefzoKTZ1H9ihBNVLMvKfjDWdJ+KHiUgq7JxuPoQVi/DLavhfNuMJZnZ0GLaOmRVQgLSFIIsNp0kL1lyxZ69+4d8FgapVbtYdRNMPL60/ULS96Gwz9BYjL0Hmp04y19LQkRFPKfZoHKtyZkZmZy7rnn4nA044/Du8L5wttg8zew9VvYmQGh4TDoMqPSWggRUM2+TsEfGnr/WVFRETt27PBPMGeD1h1g5Hi48xmj7iGxrzHID0BBHix+G/ZkQlmptXEKcRZqxj9Ng6SK60e6Ui1E27Zt2bJlC926dcPpbMRDdQab3X667qFcdhb8sAo2fQUhLujc21h+Tj/juRCiQaSk0AikpqZSWFjI5s2brQ6l8evcEyY/b5Qgzh0IB3bCZ29CoTma3aGfYP9OuUlOiHqSkkKAVVXR/NWWogrP27RpQ3p6Oh06dAhOUE2dM+R0CUJrOHoAoloby1Z/anTn7QyFjt2hcy/o1BNi462NWYgmQpJCgKkqssLB42feF9GtWzcA3G43Wmvscqdv7Shl1EGUGzMBegw06hz2boXdmyC2E0z4g7F86yqIjIG2XYzkIoSoQJKCH9S1nln7qJl2u90sWbKEiIgIBg8ejKoqo4jqhUVA9/7GA4wb4k7lGtNut9HctaQIbDYjWcR1M+oj4s+1LmYhGhFJChYo9tFoxmaz0b59ezZu3IjL5SI1NVUSQ0NFxhgPMBLBHU/BgV3mY6dRYR0abiSFogL48GWI7Wx0Bd62M7SKq1f/TPmFbr7dVszovqHyGYomRZKCBXJO+u5WIykpicLCQjIzMyktLWXAgAFnxyhtjUVYRMUWTWVlUGZ2VVuQZ/zd8o1xlzWAzQ6X3AHnphljROzbDq3jjAGFqrmh7q0v8lm7s4QusXbOiZMWZaLpkKTgD+bVoBf+m4tNKe69JIIQh/Hr0FepwFtJqQYFTrtCKUVaWhoOh4PMzEwABg4cGKjIhd1+uiTQMhau/61xmelENhzeY4xDXV5nsW8bfDzTmLbZoGU7447sYddATFs4ddK4NBXZivxC40tRm89fiMZEkkIDVO4Adcte4wzw5PxcHr8xutbbue+1Y0S4FC/cYVzmUErRr18/oqKiaN3aaFVTXFyM3W6XCuhgsNkgpp3x6OGVkLumwM1/MFo7eR4HTyeVzG/hq/lgs6FCJgNd0BnLoO0ICA2DvOPgLoMWLaXLcNFoSVLwg8rVxvuP1r2NfF7hmZXP5S2SANatW8eBAwfo3r073bp1IywsrM77EA3kcELbTsajKl2TwRUOx7NRP7igEPTebWA3uw1f+xl8v8RoMRUeBRExENkKxk425h3cDcUFEBZpLA+LMBKUEEEkSaEhzKLCvBWnOFl4Zj2Br1ZGvny1pZDUxBAiw848ESQkJHDq1Ck2bNjAxo0badu2LV27diUxMbG+0Qt/Ky9dALZjJ2FPCXrsZCOZAPQaYlyKOnkMTh6FvGOQf/x0u+U1/zPusSinFLTuCLf80Xj+/RKjtOFqcfoR0RLiuhrLy0ql40DRYPINMt3996O8cEdLIlx1/2X2VWbRGfP2HillxdYz51dnzvJTzFl+ivTuIcRG2xjR20WrCCOe9u3b0759e3Jzc9m1axdZWVnk5OSQmJiI2+1mxYoVREVFER0dTXR0NFFRUXKpyULl7Y3c3r8LqitlAIy8DvqNNiq0T5kPm9dnuPcH2LMFSr3GcG3XBW56xJh+7xnjkparhdGiKiwCOpwDQ680lq9bbCSOEBeEhBmXtCJbnb6xryDPuOnP7qj6BhvBrCV5FBZr7rv07O3WvVElBaXUJcB0wA78U2v9dDD3/+Cs4/z9ZzE47A3/h5j2fm69X/vd9mIAFq0pBGBEr1BuGB5OqFMRFRVFamoqqamplJldORQWFnLixAmysrIqlE4GDBhAjx49KCgoYNOmTbhcLkJCQggNDSU0NJSYmBhcLpfnhjmbzSbNJ/2k/DD+7eM8Zt7XqnYvimplPHwZd5/xt7TEOIGXd+1RLmmEUUFemH/6UVRwevn3i88cyOjcNLjsbmP6zT8Y69vsxo19jhDoNRiGX2Ms/+BFo9TjMJc5Q4y+p7qlGK24Nn1lLnMaicXuoCSyHaVRsYTZ3UbCsjsocDsICbFjd4ZASGiTKt2s/KHY6hACrtF8GkopO/AKcCGQBaxWSi3UWm8JVgwauPcfp/9pusTauX5YOG2j7RzLc+MKUbRvGfwT51eZRVWWRioaesacZUuAJUfNZ6frJyLsJ8grCwVOcXmam7XbT3HghI0Iey6hthJiQk9SSiglzvbY7Q4GJpSweddRnHaNxkbrFsVEhZYR064LPTpFkHviBNt2HyGnIISYcDdtIty4sdEhLo4wVwhHj5/EXnqMMreixG0nLAQ0Cpsrli5tQ8jPzyMvP5/jp2xEuhQohVsr4tu1BJuNw0dP4VTFlLkh1Kk4lg8twyE6OgqbUhQUFlJQWIJSCqcdSspAKRtRkeHkFWqOnywipoXGYYPiMoXLaew/zBWKW0NJSQk2r4+0qFRx5KQmrpWTsjJNfmEZkWE2Qp2KMrcmr1AT4VIcz4c2UTbcbu0pGhw85sZugw0/nf41P2tJHlekhdEmykZpGSzdWEirSBuJbR20bGEjK6eMzrF2Coo0Px4uJalzCKVlRnLfdaiUjq3slLnBFWK8P21z8NOpSIpLI4iNtvPh0jw6t3Gw7+gArhkchtOusNvAYVdorSkp1bg1hNz+FDZ3GbnHTxFpK0KVFOC2h1JY5CbUqbAPu4aTecWcOOUm3nncaEkV0x6tNaUlboqLFargFOGlh6C0GF1cjAoNpzQhGUdJEUXL5qNROChjvepJsv6BP0c/zN6CY7x2C5TNfQqFZkrIkwwuW0cIxdg7JHLztT2NCvt/PYHb5kDZnSiHg5O2SCKHX2LciJizHz6bTYnNicOmUHY7GhuHeo+l1TldUUcPYF/7KW6bHYdNGYnNZjfGB28dBzkHOLklgwinG2W348ZGmbLj7DkAIlpyeM8hYo/9gLLbWLq/FbFhpfRtU4BOSEKFtYDj2XAkC+hifKg/bsSNjTd3dCKtu4ueUXnY8nPIK7LRMhxQNjQK1baT0aggPxd3QZ7RvFwp3G7FqRJYlxNFVLiNrIMFfLqxjCeuhJYtbOavCuUZpvbg4QL25ZQyINFu1DPZ7KcvS/qZqut170BRSg0BHtNaX2w+/x2A1vrPvl6Tlpam16xZU+d9fZ1ZxOxl+TWvKIQ4g8KNbuR9aUbofCLVKU44WlJSpinRp7s0CSvNocBhtOqLCS3kWJGrymUNFaqLKFKhDdqGy1GKdpdR5K64ndaOfK4d1ZaB3eu3faXUWq11WlXLGtMn2xHY6/U8y5xXgVJqslJqjVJqTXZ2dr12FOFSxMUYb/3yNBd/uzuGP1wnA8cLURstQvzbA21ESEnNK2GcIMslxFZfWu/aJZIO3TrSsbWdskoXRDrEOuhsnvfdthCiQ41LQnEtThLX6sxTYkyYUUpvHZqPovbjuWuHE7s680d3uM3YXvvwmi9FxYS7CQsBp6p4w0uX1orw0MCcvhvN5aPa0lq/BrwGRkmhPttITQwhNbFiZ2hdYh21v/YrhGgiIqqY5+v/vKb//8Z0fghcLI2ppLAP8G6aEW/OE0IIESSNKSmsBrorpRKVUiHAjcBCi2MSQohmpdFcPtJalyqlfgH8D6NJ6iyttQxFJoQQQdRokgKA1vpj4GOr4xBCiOaqMV0+EkIIYTFJCkIIITwkKQghhPCQpCCEEMKj0XRzUR9KqWzgp3q+vA1wxI/h+IvEVTcSV9011tgkrrppSFxdtNaxVS1o0kmhIZRSa3z1/WEliatuJK66a6yxSVx1E6i45PKREEIID0kKQgghPJpzUnjN6gB8kLjqRuKqu8Yam8RVNwGJq9nWKQghhDhTcy4pCCGEqESSghBCCI9mmRSUUpcopX5QSu1QSk0N8L46KaWWKaW2KKU2K6V+ac5/TCm1TymVYT4u83rN78zYflBKXRzIuJVSu5VSG80Y1pjzWimlPldKbTf/xpjzlVLqJXP/G5RS/b22M9Fcf7tSamIDY+rhdVwylFK5SqkHrDhmSqlZSqnDSqlNXvP8dnyUUgPM47/DfG2tBgD3EddzSqmt5r7/o5Rqac5PUEoVeB23GTXt39d7rGdcfvvclNG1/nfm/PeU0c1+feN6zyum3UqpDAuOl6/zg3XfMa11s3pgdMu9E+gKhADrgd4B3F8c0N+cjgS2Ab2Bx4DfVLF+bzOmUCDRjNUeqLiB3UCbSvOeBaaa01OBZ8zpy4BPMIapHwx8Z85vBewy/8aY0zF+/LwOYoyYHvRjBowE+gObAnF8gFXmusp87aUNiOsiwGFOP+MVV4L3epW2U+X+fb3Hesblt88NeB+40ZyeAdxb37gqLf8L8KgFx8vX+cGy71hzLCkMAnZorXdprYuBd4ErA7UzrfUBrfU6c/okkEkVY097uRJ4V2tdpLX+EdhhxhzMuK8EZpvTs4GrvObP0YZvgZZKqTjgYuBzrfVRrfUx4HPgEj/FMgbYqbWu7s71gB0zrfWXwNEq9tfg42Mui9Jaf6uN/945Xtuqc1xa68+01uWD+X6LMXqhTzXs39d7rHNc1ajT52b+wh0NzPdnXOZ2rwfeqW4bATpevs4Pln3HmmNS6Ajs9XqeRfUnab9RSiUA/YDvzFm/MIuAs7yKm77iC1TcGvhMKbVWKTXZnNdOa33AnD4ItLMoNjBG4PP+Z20Mx8xfx6ejOe3v+ADuwPhVWC5RKfW9UuoLpdQIr3h97d/Xe6wvf3xurYHjXonPX8drBHBIa73da17Qj1el84Nl37HmmBQsoZSKAP4NPKC1zgVeBboBqcABjOKrFYZrrfsDlwI/V0qN9F5o/rqwpN2yeb14HDDPnNVYjpmHlcfHF6XU74FSYK456wDQWWvdD3gQ+JdSKqq22/PDe2x0n1slN1Hxh0fQj1cV54cGba8hmmNS2Ad08noeb84LGKWUE+MDn6u1/gBAa31Ia12mtXYDMzGKzNXFF5C4tdb7zL+Hgf+YcRwyi53lRebDVsSGkajWaa0PmTE2imOG/47PPipe4mlwfEqpScDlwATzZIJ5eSbHnF6Lcb3+3Br27+s91pkfP7ccjMsljkrz683c1jXAe17xBvV4VXV+qGZ7gf+O1aYy5Gx6YAxBugujYqu8EqtPAPenMK7jvVhpfpzX9K8wrq0C9KFi5dsujIo3v8cNtAAivaa/wagLeI6KlVzPmtNjqVjJtUqfruT6EaOCK8acbuWHY/cucLvVx4xKFY/+PD6cWQl4WQPiugTYAsRWWi8WsJvTXTFOCtXu39d7rGdcfvvcMEqN3hXN99U3Lq9j9oVVxwvf5wfLvmMBORE29gdGDf42jF8Avw/wvoZjFP02ABnm4zLgLWCjOX9hpX+c35ux/YBXSwF/x21+4debj83l28S4drsE2A4s9vpyKeAVc/8bgTSvbd2BUVG4A68TeQNia4HxyzDaa17QjxnGZYUDQAnG9dg7/Xl8gDRgk/mav2H2MlDPuHZgXFcu/57NMNe91vx8M4B1wBU17d/Xe6xnXH773Mzv7Crzvc4DQusblzn/TeCeSusG83j5Oj9Y9h2Tbi6EEEJ4NMc6BSGEED5IUhBCCOEhSUEIIYSHJAUhhBAekhSEEEJ4SFIQohpKqTJVscdWv/Wqa/bGuanmNYUIHkfNqwjRrBVorVOtDkKIYJGSghD1YPa//6zZT/0qpdQ55vwEpdRSs/O3JUqpzub8dsoY42C9+RhqbsqulJpp9qX/mVIqzLI3JQSSFISoSVily0c3eC07obXui3GX6IvmvJeB2VrrZIwO6V4y57+E0Z1CCka//pvN+d2BV7TWfYDjGHfTCmEZuaNZiGoopfK01hFVzN8NjNZa7zI7NDuotW6tlDqC0Y1DiTn/gNa6jVIqG4jXWhd5bSMBow/87ubzhwGn1vqJwL8zIaomJQUh6k/7mK6LIq/pMqSeT1hMkoIQ9XeD19+V5vQ3GAMDAUwAvjKnlwD3Aiil7Eqp6GAFKURdyK8SIaoXpswB3U2faq3Lm6XGKKU2YPzav8mcdz/whlLqISAbuN2c/0vgNaXUnRglgnsxeu0UolGROgUh6sGsU0jTWh+xOhYh/EkuHwkhhPCQkoIQQggPKSkIIYTwkKQghBDCQ5KCEEIID0kKQgghPCQpCCGE8Ph/DMWs5nf+aCoAAAAASUVORK5CYII=\n",
451 |       "text/plain": [
452 |        "<Figure size 432x288 with 1 Axes>"
453 |       ]
454 |      },
455 |      "metadata": {},
456 |      "output_type": "display_data"
457 |     }
458 |    ],
459 |    "source": [
460 |     "plt.plot(epsilon*90, alpha = 0.6, ls = '--', label = 'Epsilon profile (arbitrary unit)', color = 'orangered')\n",
461 |     "plt.plot((np.array(mpm)-np.array(mp))*120, alpha = 0.6, ls = '--',\n",
462 |     "         label = 'Probability difference (arbitrary unit)', color = 'dimgray')\n",
463 |     "plt.plot(loss_log, label = 'Loss', color = 'cornflowerblue')\n",
464 |     "plt.xlabel('Epoch')\n",
465 |     "plt.ylabel('MSE')\n",
466 |     "plt.legend()\n",
467 |     "plt.savefig('loss.pdf', dpi = 300, bbox_inches='tight')\n",
468 |     "plt.show()"
469 |    ]
470 |   },
471 |   {
472 |    "cell_type": "markdown",
473 |    "metadata": {},
474 |    "source": [
475 |     "**Show the maze solution and the policy learnt.**"
476 |    ]
477 |   },
478 |   {
479 |    "cell_type": "code",
480 |    "execution_count": 88,
481 |    "metadata": {
482 |     "scrolled": true
483 |    },
484 |    "outputs": [
485 |     {
486 |      "data": {
487 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOsAAADrCAYAAACICmHVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAE6UlEQVR4nO3dQW4bNxiA0VHSI6RpdvUdrPufQL6Du2tdX6FVdwVaSIEGMUV+9ntAFoaVETXyBxrwD+pwPp83YH2fZi8AuI1YIUKsECFWiBArRIgVIn7a8+AvX76cHx4eBi3lY3t6err5sY+Pj0Ouu8fsNYx6/tnXfX5+3l5fXw+XvnfY83fW4/F4Pp1ONz+e2x0OF9+fi/a8Z3uuu8fsNYx6/tnXPR6P2+l0unhhvwZDhFghQqwQIVaIECtEiBUixAoRYoWIXRNMe5T+GD/7D/x71zDKqDXMfm3v5XXZWSFCrBAhVogQK0SIFSLEChFihQixQoRYIUKsEDFs3HCFEb4RRo2YjTonaIU1jLjuCmvd4y3eMzsrRIgVIsQKEWKFCLFChFghQqwQIVaIECtEiBUicqcbzlZ7XbXRxBHXnL3Wt2JnhQixQoRYIUKsECFWiBArRIgVIsQKEWKFCLFCxK5xw6enpyEfUDzK7JPyRln5BL5LZp9uOMq9f27srBAhVogQK0SIFSLEChFihQixQoRYIUKsEDHswLQ9VphGGaE2GTXKiMPN3vP9usbOChFihQixQoRYIUKsECFWiBArRIgVIsQKEWKFiF3jho+Pj9vpdHrzRcweHauNO86+XzW1A+ausbNChFghQqwQIVaIECtEiBUixAoRYoUIsUKEWCFiic9nHXUK4OyT8kr3YK8V7tnMa26bz2cFrhArRIgVIsQKEWKFCLFChFghQqwQIVaIECtELHG64Si3jnitMOpXu+4Ka5jN6YbARWKFCLFChFghQqwQIVaIECtEiBUixAoRYoWIJU43HGX26NoK92CPFda7whpu5XRD4CKxQoRYIUKsECFWiBArRIgVIsQKEWKFCLFChNMNd6p9OPHskctta51uuPK4o50VIsQKEWKFCLFChFghQqwQIVaIECtEiBUixAoRTjfc9q11hdG50r0dZYV7cG92VogQK0SIFSLEChFihQixQoRYIUKsECFWiBArROwaNxxlhRG+Ec9fel3b9n5PWHwvY592VogQK0SIFSLEChFihQixQoRYIUKsECFWiBArRCzxYcqjRvhGGPX8tVG/2e/DKKNGRN/iftlZIUKsECFWiBArRIgVIsQKEWKFCLFChFghQqwQMezDlFcwYq0rnFi4wqhf6eegNiJ6jZ0VIsQKEWKFCLFChFghQqwQIVaIECtEiBUiljgwbZQREyalyZ1tW/sAsHt5L5+pa2eFCLFChFghQqwQIVaIECtEiBUixAoRYoUIsULEsAPTVhjbunUNtTG72sjjiPtbuwe3vq7j8Xj1e3ZWiBArRIgVIsQKEWKFCLFChFghQqwQIVaIECtELHG64agRvtkjaSuMMa4w9jnCCmOf92ZnhQixQoRYIUKsECFWiBArRIgVIsQKEWKFCLFCRO50w484ZvZ/K4wmjlIaebw3OytEiBUixAoRYoUIsUKEWCFCrBAhVogQK0SIFSKWON1wlNnjcyuMXM7+oOq9j+U6OytEiBUixAoRYoU7e/n8bdsOh3//vXz+dtP/Eyvc2de///ju19eIFSLEChFihTt7+fTLd7++ZtdQBPDjvv71+3+/vvH/2Vkh4rBzdO3Pbdt+G7cc+PB+PZ/PP1/6xq5YgXn8GgwRYoUIsUKEWCFCrBAhVogQK0SIFSLEChH/AFuQjEBDA4n2AAAAAElFTkSuQmCC\n",
488 |       "text/plain": [
489 |        "<Figure size 432x288 with 1 Axes>"
490 |       ]
491 |      },
492 |      "metadata": {},
493 |      "output_type": "display_data"
494 |     }
495 |    ],
496 |    "source": [
497 |     "net.eval()\n",
498 |     "agent.isgameon = True\n",
499 |     "agent.use_softmax = False\n",
500 |     "_ = agent.env.reset(0)\n",
501 |     "while agent.isgameon:\n",
502 |     "    agent.make_a_move(net, 0)\n",
503 |     "    agent.env.draw('')\n",
504 |     "    clear_output(wait = True)"
505 |    ]
506 |   },
507 |   {
508 |    "cell_type": "code",
509 |    "execution_count": 89,
510 |    "metadata": {},
511 |    "outputs": [
512 |     {
513 |      "data": {
514 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO0AAADrCAYAAACFFBGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWbklEQVR4nO2dP2hdx/LHR7/8QAlOZfKiNMkvhd3YAhWyESgYQ0CkCypDEL4IkdaVQFV6gYs0dnBpoVQubJQgBblSIUso0Q12iIsQEmKDiR5JIWPLlgprX+Gf/K6k82fn7M7OzJ75wIH33t03Z3Z29175nu/9Tp9zDgzD0MP/cCdgGAYOO7SGoQw7tIahDDu0hqEMO7SGoQw7tIahjP/FDH7nnXfchx9+SJRKu+l2u95jh4eHSeJi4M6B6v7ccf/880/4559/+ioHOee8r+HhYSeN7e1td+/ePe40ggEA7+soVTXAxNWUA9X9ueP+/xmrPIfq/zz+7bff4KuvvuJOgxUJNeDOger+EuNGO7Q///wz/PHHH6xjy9jf34fvvvtOzVjDqCLaod3d3YXx8XGvA0Y1toj9/X2YnJyE1dVVFWMNow7UF1EHfPPNNzA7O3vsf//rr7/gs88+gx9++IF8rC/Xr1+H+fl5OHPmDCwuLh567fTp03D79m1RYw2jlrp/9DrPL6IePnzohoaG3OrqaumY2GPX1tbcxsaG63Q6bnNz0+3u7h4b8/TpU3fx4kV348aN2ntxjoWGX2rU1QATV1MOVPfnjuvzRVS0Q3vnzh139+7d0tcpxs7MzLiRkRE3ODjohoaG3NbWVuG4Z8+euStXrnjdj2ts0w1QV4MUh5YjB6r7c8dNemi5mJ6edqdOnXKPHz/mTiWIphvAueoapDi0HDlQ3Z87rs+h7XOI39OeO3fObW5ueo9PxcuXL+GNN97gTiOIvr7q5+m9FK1ZWQ0wcTFw50B1f+64586dg83NzcrAjb6I8iHlZik7sL45YN64Us4LQ4w3LaocQuNy3z913DrUiysMo20EHdonT57ATz/9FCuXxjncv3+fJC733HyhqoFBR8iaNT60T548gU8++QRGR0fh+++/bxomGAqZmZS5HVCnCqOWEEpQu+U2lkXG+MUXX8Do6Ch8/PHH8OWXX8KjR4+ahhKHtLmFqsJS3t/G4sdiafxF1NzcHDx48ACuXr0Kt27dgjfffDNmXqxwzo1CFUZ1fxuLHxuFumdCvdfR57Q//vij63Q6x55BYZ9hYa6jxMihiLK4qebVS52CLNY6NL2/jcWPLVuz7H+at76+Dvv7+wDw6gfJe3t7zBnR8Ouvv8LXX38NH3300bHXUtSg6v42Fj82eM3qTrUT/EkbS2aGeSdMMS8MVFI7g45QGaPqT9rZ2Vm4cOEC7O7uwtLSEgwMDHCnlByrgT5C14xMxqhJPoepgVRFFJXUzqBDnIwx5UFIqTum2tihhytGDagOOEVcCbliiLlmqv88Now2ov7QmoxRRq5UOVDElVKvxvu27puq3qvq97T37993v//+e+FrL1++dN9++23p/xcCvuGkeE67vb3tRkZGXH9/v1taWirNu25emLFNa1CXKyauhBx840rIlWLfJv32uEy2pdHUzEfGKMXYTYLkkioHirgS6hVM3al2BZ+08/Pz7uzZs8eukydPuvPnzx9657h27Zrr6+srHD8+Ph70rln1joWJe5Tnz5+/jvvixYvC2D7zSlGDulwxcSXk4BtXQq4U+9bnk7bRt8cTExMwMTFx6H979OgRfPrpp8d+uXDp0iW4efMmTE5OQqfTaXK75Lz11luv/3OZ7hgzL8oa+ORKDVUOFHEl1CuUaH8el8m23n77bVhcXIS///471q1ewyljxMyLsgaGPkL3bbTntGNjY6WvnThxAqanp2Pd6jULCwuwsrICOzs7MDU1BcvLy0kVQZh5UdXA0EfovlX9yMckfIZGxMoYUUkwu+VhakBFDgqfGDn4xs2hXuJkjFSTLyKljDHlvGJA9YbErWnmftOIgckYDaMlmBtjRVzuuflCKSHkdnnUtA4YzI0xczfGKkJyle7y6Du3VL1/zY1RMJrmFpKrdJfH2HLSUKS4MTb+wYB0ORgmh6PEks+lqIGP5NI5nPQ0Zm0P8m1y/1hy0ia1xeQbs7bkXfO4PaJi5YApKveBxdagCg0uj3Vzo+z92yRf37EifuXDQVvcGKnQ5PJYRkqJqBQ3RrLntCngljFqp0p6mqK2VffHkEoiismXsraqP2lNxkiH1ZYOsTJGbjkYJgdMDSQoojD5hpKDy6PUXJPIGLvdrkg9b2jTX6mLWoYGx8BeuLXHVHA5aCb781iCuoYKirlJqJcENRJ3DhLVZskOLbW6hhOKuXHXS4IqjDsHyvuzKKKo8JWkYaRrqWRuGDCSOA4kqMK4c+C+fxmiDq2vJE2KE2II3BLCOubm5uDzzz+Hd999F1ZXV+GDDz5oXQ7c9y9D1HPa69evw/z8PJw5cwYWFxcPvXb69Gm4ffs2ahx2LBXcjaKbIMEAjTsH7vuXkeTQrq+vv/6mrNvtwuDgIPT39x8b5+taKMUJEcBvbhj3St+Yhl5C1zfJn8cLCwtw+fJl6Ha7MDU1Bdvb24XjfCVpkpwQfed2lCqZW9OYhg6C17dOnNx7QYDwenp62p06dco9fvy4VmydmpB5OUczt6qYmHxD5ib5ByHc98fmcJSy9fX5wQBKEdXX1+c9uChumbqGmxgP/ynmFkONhAG5F1hz4L4/Ngff88Bq7FZE2aaWoG4JhULqJ/ENrgkUBwETUyriFVGGYcTBZIwRyLmHqjQJX2pE1qDuH729Fwj4Bz33dZRce6hS9ZyNtRdSwFGD7J0rJJBrD1UJOTQllmui1BrYoQ2EQuomQT4nIYemxHJNlFoDUTJGjeTaQ1VCDj5gJKJYOanUGoiSMRr5E3svYCSiWDkpFaE1SHJozYDNOCDFXjiQiI6OjkYdGwsVxm5mEmYckGIvjI2NeR9CzNhYJDV2o5IxalJEIevFHhdDyrlZT+FEMsbh4WEIaSodasBGhaY3DQD+esUgh57CoetgMkbDaAnqD60ESRy3jFFbj14q90pu2ScGFW6MVOToWoiNqa1Hb+x8uV0bmyDCjTFmw92mY8ugdGPULGPUKOErQmKulG6b0Q4tVcPdUNdCajdGzTJGjRK+IiTmSum22UhcQSUdo3AtpHZj1CJjzEXCVwR3rsndNut+BtR7HW0q3UvMhru+Y9fW1tzGxobrdDpuc3PT7e7uHhuToulwDB8j35hNalAGR1Ppunwp6oXJNXQv+NS2qgZJf5oXq+EuZqyPq13KpsMchDj7xWjqjKUtTpOkbpt1p9p5ftJyEcMJERR/0jpH53SZ2okQE5cq19C94EsyN0ZMf9qUhDohSpAbYuIWkdrpUpOMUaoiSnV/WirXQm4dq6YaYAmtWUpJK9X6Zt+f1jCMOCR1Y9QiiWuSQ46NjzWhrQbi3Rhzdvarm5svEmobsg6cxFoDLBT7Vowbo0SZmS91cjTfuVFJKaXVlkPOiqmB9mbkAIn+PJYoM/OlTo7mMzdKKaW02nLIWX1rkEMzcoBEHlHcMjNfmsjRfOZGKaXkrK0UOatvDbQ1Iy+l7u9nF+HftFV/w4f824BKEteLj+SyTghBLaWUJCzgkLPW1eAAn3XAjKXYt2L+TUtFCklcDKlf7lLKXjjkrL5IaUaupqm0JkkcFozksAwJtQ3JgZsYa4CFYt+KaypdEdd7bFFcc/aLExeDhJpx0wo3RipCJXESDoy2uBJy4Cb0jctkjIbREkzGGCkHCjdGKofFHOulDfFujJqc/bBQzY1qXrnWSxvi3Ri5pXba3Bhzpi31Uu/GyCm10+jGmDNtqZdqN0YAXqmdRjfGnMmxXqndGBsdWk3NeS9dugQ3b96EyclJ6HQ6yXILgaoJtzX3pgF7HkLXgcWNMRa5ujFSyTPb4oQogSzcGDlkjL6E3L9qblTzkhA3JAcO2WWKeWFQ78ZIJWNMdX+quDGacFPFxUBVM24w8ypCtRtjKKmdCH0JrQGVJarU5t5ScvDF3BgNw/DCZIyRcuBsKq2NnOfmi8kYFcryqhQzqaR+vqqdmGZtvnOjMmCTYuwmQsZYRc7StaZzq1LMpKqXr2onplmbz9yoDNhyMXZL4lzx/Pnz11/zv3jxIuhr/qOkfnxwlLq5Oefc/Py8O3v27LHr5MmT7vz58+iYTfL1zQGTK3ZevvW6du2a6+vrK4w9Pj4uZizFvvV55KPebob70NblUAZHb1hsDthxvmNjGOFJGMt1aFV/e7y+vg77+/sA8Opx1N7eHnNG/nAoyJrmkIsBmxRjt+B9W3eqneBP2pmZGTcyMuIGBwfd0NCQ29raahy36bzq5taUFJ+0XHCYsFFAsW+z/6SdnZ2FCxcuwO7uLiwtLcHAwAB3SoZRS+i+NTdGJJh5UZGyWbVRjrkxBpBSlidBn4uBW8oJQPcGTkHKNzmTMRpGS1AvY6SCWx6prV4YNM1Novw2ibGbRlkepzwyZ8dCTXOTKr9NYuymWZbHgck+ZSA217pnQr3XgXNFrrK8XmK4UTSZV+x6Sbg0zY0qV9/95fOcNomxG4UDn28OUkzoMOToWHiAprlJzVW1sVvTHOrGaZZHGvIJ3V+NPmmLGBsbixWKPIe6cQsLC7CysgI7OzswNTUFy8vLprYyohG6v+w5bQEmjzQoSSpjlOrGSEWRzEyCIopbNYRF09yoci2K22o3xjJCcy2SmUnYgJpqS4WEGoQiXsZIqSzRoq7BkOu8AHTNTWSudc+Eei9o+Fxqe3vbjYyMuP7+fre0tBTteVesuEVsb2+7e/fuFb7mC9W8UlFVA4lrFhuqfXsQu6i2Yn5PS6UsoVSs+MjMqNz6QuYV0zmRQsqZQmUUqwbc+6uMJIeWqidprn1vQ+YV0zmRghRrFqsGUnvpRntOWwWVsiTXvre+88L0RU3dQ7WM2GtGWQOpiqgkhzZHJPS9xUg0Nco5fWhjDUxcUYCPzExq39tYzomapZyp3SOxiJEx5oSvzOzEiRMwPT3NkGE5GDlp1VjNUs5YNaAiuLZ1Xy+7CI986uCOW4SEZtXcUDW2rkJTvUJqkKyptFQ3xtC4ZTGlNqtOCVVj6zJyVNwBFOebRMZIBdUBD0XiZqWkKN8Yja255yZVTtpaGSMVFMZu3GZxWDStmaZcAYQYu1WhyczrAAo1EHcvXQya1kxTrgeIV0Rh5GCUjXwNf8SamhUgMVeMlBKLKBmj6Ea+LUOqhK8IiblSSkRFyRgppYEGDqkSviK4c00tERXx7fEBEqSBAK8UKwff7HW7XRgcHIT+/n5xMQ0ZYOWRoXtBlIxRijRwYWEBLl++DN1uF6ampmB7e1tkTEMuVfLI0L0g6tACyJAGUhi7mVlcuxgbG4PR0dHC17LoT4shZc/ZUGM3XxUMNi4VmsQgEnIN3eNNFVHiPmklEUMNlCKmoZOme0FEU2lNGl2q+2v6ywSAfx2okKCZr0O9jFGbNNCgQdv+arWMUZM00KBB4/4SIWPkcrXzASONzHlsU2I6PFKM5d5fqUnSVFqLa2LOY0OgcnjM3TWRikaKKE2udhhpZM5jfaFyN2yjayIZddYWvddBJ/giHj586IaGhtzq6mrh62Wdr51rbtuxtrbmNjY2XKfTcZubm253d/dY7KdPn7qLFy+6GzdulOae89imte2lbm0ljOXYXxRxk3YY4HC185GDYaSROY8NgcrdULprIpX0NDhu3al2np+0dVC8EzoXx4AtZ0JqqwmO/UURN7qxm9T+tKEGbDmjSbhChQTpqW/c1sgY7cAalFDtryQyRkxTaQmk/HFBqvtjc6BC0z6gqhfXOqj/pJUgY+R2ArSG3XRI2F9HUX9ouWWM3E6AVPfnnpcUuPdXEeoPLTfcEjqNDbslEcs10VdOGkN2aoc2EG4JXY4Nu1MSwzXRV04aS3YqythNI9wSuhwbdlNB5ZroKyeNJTtVfWjN4dDAQOWa6OsiGsttVPWfx+ZwaIQSwzXRV04aS3aq+tCaw6ERSizXRF8X0Rhuo2RujFRQycxC7l+GBHFFzq05fQkVQaSUyfrIGEVojzUhYbNSKXFy1Slre+NyzuWvPTaMNpHUjVGaHCwXJMgNc11fifNKdmglysFyQIrcMNf1lTivJG6MGCjkYNLcDWOiqWG3rVkckrgx+kIhB5PobhgTLQ27bc3ikcSN0RcKORi3uyE1Whp225pFpM6Ppvdq6sbo45ronL8ToRZ3Q6rrKFX+SFRz60W6KybVvAjXl9+NkVMOJtHdkAPKuWlxxcQiViZbd6qd5ydtHbm4JoLST9rYczuK5PWlmhfh+spxY8zBNdEUUeU5SF1fCW6MGFyNIkpEJ3huja42+Z62Nw6K+nLvGWwOvnFbY6FqGG2iVTJGCXI/XyTUCwNl82Vup8tWNpUG4JeDSZH7+cJdLywU+UpYs6ybSksnhbsgVUPlttIWR0gsrTm0KdwFqRoqt5W2OEJiUW3shiG2uyBlk2TjFTk6QsYgySft+vo67O/vA8ArV7u9vb0UtyVlYmICfvnll0PX0tISvP/++8f+rYIZC6CvXtry9YVqXqFxk3zSLiwswMrKCuzs7MDU1BQsLy9nacJ2IOUsMwrzHautXtry9YVqXsFx6yRTvRfok4MdokzuFxKTCo56SVjfo8RYM4nzStZUOlQRxSAH8xonVRGVul4YqNaXYs0wpJxX06bSqD+Ph4eHIUR7XKZLpToI3Js7dFFj6HiRb8pB90qpO0755imtqXRrHvkYRi6oP7QS5H4UUjuJ8jmuuFqkpxhEyBip1EB1Y7nlflRSO4nyudRxfWur0TBOhIyRSg0kXTlkUjs6fGrbRsM4cmO33JVDc3Nz8ODBA7h69SrcunXLlDsR8altKw3j6p4J9V5Njd0ox1ZZrUCC53hVOYTEjTWvnOM6p9MwrmpePs9pkxi7UY01+VzecX3QaBgXXK+6U+08P2k5mJmZcSMjI25wcNANDQ25ra2tQ6+D0k/amPPSHLeqthKgqFfST1oOcm0qTTUvbXFzJbReWfSnzaGpdFHclPI5yXGlokLG2O12vRNNubkl2nbGgEr2mTouBm7paUpMxmgYLSGpGyOFHC1XGaM2JKyDJkTIGKugdNXLVcbYC5dEFDOOex20IULGWEXOUr8Uc+OWiEqXkraNJHYzOUv9Ys+NWyKqUUraOuoe5PZeoEzmhonb9P5VOYTGPYBLIooZl3odtF2+9cpeXJGrjPEoHBJRzLi2rEMsVLgxUpGrC+BRxsbGWMfWjWvLOsQitF6qP2lNPicDWwccSWWMEvrTFsXNVcaoDcnukdz47ltxboxUpHZ59EXCAU9pNSp1HaQiXsaYs2qIe27c9zfSol4RxQ333Ljvb6QniRtjKkVUTFme71jfuVE5+0lTm3FLLjWOxZLEjTFVn1EKWV7dWJ+5UTr7Sevhyi251DD2vfcA+vr+e733Xm24Q5C7MQLQ9BmlkOU1kfD5zI3S2Y+zhyu35FLjWACAf/8bKv97LXWSqd4rxI2RSsaIyQE7zndsDMfAkBpwSAhD6tX2sQDHrwN8ZIzRDu2dO3fc3bt3S19PcWjrcsCO8x3rYz727Nkzd+XKldLXczm0sWub41gxh7aOFIeWixiOgbkcWqOegYHDB3Zg4L+vRe9PS2Xs1jblUBE5iCuMcHwUUaq1x4bRRrDa478B4CFdOobRev7POfevqgGoQ2sYBj/257FhKMMOrWEoww6tYSjDDq1hKMMOrWEoww6tYSjDDq1hKMMOrWEoww6tYSjjP/B1t4mP6K/+AAAAAElFTkSuQmCC\n",
515 |       "text/plain": [
516 |        "<Figure size 432x288 with 1 Axes>"
517 |       ]
518 |      },
519 |      "metadata": {},
520 |      "output_type": "display_data"
521 |     }
522 |    ],
523 |    "source": [
524 |     "agent.plot_policy_map(net, 'solution.pdf', [0.35,-0.3])"
525 |    ]
526 |   },
527 |   {
528 |    "cell_type": "code",
529 |    "execution_count": 93,
530 |    "metadata": {},
531 |    "outputs": [
532 |     {
533 |      "data": {
534 |       "text/plain": [
535 |        "<All keys matched successfully>"
536 |       ]
537 |      },
538 |      "execution_count": 93,
539 |      "metadata": {},
540 |      "output_type": "execute_result"
541 |     }
542 |    ],
543 |    "source": [
544 |     "best_net = copy.deepcopy(net)\n",
545 |     "best_net.load_state_dict(torch.load('best.torch'))"
546 |    ]
547 |   },
548 |   {
549 |    "cell_type": "code",
550 |    "execution_count": 147,
551 |    "metadata": {},
552 |    "outputs": [
553 |     {
554 |      "data": {
555 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO0AAADrCAYAAACFFBGSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWF0lEQVR4nO2dT2hdxR7Hf3k+qFJX4jNu6nPRbmogi1QCKUEQgjvpskjIJQS3XQWych/Iwk2VLg3tqouUKIkkqyzyh9RcUWkWIogWSvPQRYqJ3i6aeYu8+G6Se86ZOTO/f3N+Hzjwnhnm/GbmzL1tzqff6XPOgWEYevgHdwGGYYRhm9YwlGGb1jCUYZvWMJRhm9YwlGGb1jCU8c+Qxq+//rp7++23kUppNu1227vt0NAQSr8hcNeAdX/ufn/55Rf4/fff+0obOee8r6GhISeN/f19991333GXEQ0AeF9nKZuDkH411YB1f+5+/7fHSveh+j8e//TTT/Dpp59yl8GKhDngrgHr/hL7TbZpf/jhB/j5559Z2xZxdHQEX331lZq2hlFGsk3b6XTgxo0bXhsMq20vjo6OYHJyEtbX11W0NYwqgn4RdcK9e/dgdnb23H9/+vQp3Lx5Ex4+fIje1pc7d+7A3bt34erVq7C0tHTqZ1euXIEHDx6IamsYlVT9pdd5/iLq119/dYODg259fb2wTeq2m5ubbnt727VaLbezs+M6nc65Nn/88Yd777333BdffFF5L862UPOXGlVzENKvphqw7s/dr88vopJt2tXVVbexsVH4c4y2MzMzbnh42A0MDLjBwUG3t7fXs93BwYGbm5vzuh9X27oPQNUcUGxajhqw7s/dL+mm5WJ6etpdvnzZPXnyhLuUKOo+AM6VzwHFpuWoAev+3P36bNo+F/Dvaa9du+Z2dna821Px4sULeOmll7jLiKKvr/x9eje91qxoDkL6DYG7Bqz7c/d77do12NnZKe241i+ifKB8WIo2rG8NIR9clOMKIcWHFlYNsf1y35+63yrUyxWG0TSiNu2zZ8/g22+/TVVL7Rq+//57lH65x+aLplqNY2Ke29qb9tmzZ/DBBx/AyMgIfP3113W7iQZDM5MythPKrDCKWiXYbrm1ZdEYP/74YxgZGYH3338fPvnkE3j8+HHdrsQhbWxlVhhFrRJst5zbhlL7F1Hz8/Owu7sLt2/fhoWFBXj55ZdT1sUK59hCrbDUtUqw3XJum4Sqd0Ld19n3tN98841rtVrn3kGFvsMKuc6SooZeFPVLNa5uqgyyVOtQ9/7WNrxt0Zpl/0/ztra24OjoCACO/0Hy8+fPmSvC4ccff4TPP/8crl+/Lv7+1ra6bexzi/aeloLFxUVYW1uDw8NDmJqagpWVFejv7+cuKzljY2Nq7m9tq9vGPreqv2lnZ2dhdHQUOp0OLC8vZ7lhjfyIfW7RNEZN+lzIHEg1ooqIVe0MPMRpjJQbgdI7znlzYdWA0a+EWkPoVYNpjIbREExjLOmXe2y+SKgVqwaMfqXMV+3ntuqdUPfV/Z52f3/fDQ8PuwsXLrjl5eXSd1YvXrxwX375ZeHPQdh7Wt+xVY0rpG3dOaiqNaRfCTX49iuhVoznFvU9ra8+pzHUzGdsUoLdJCiXWDVg9CthvqKp2tWu4Jv2zz///PvT4q+//jr3iXHCZ5995vr6+tw777xz7rpx40bUp2bZJ1ZIv2fxGZvPuCjmoKrWkH4l1ODbr4RaMZ5bn2/a2r89fuWVV/7+32W+68TEBNy/fx8mJyeh1WrVvR0pPmMLGRfmHPiuAyZYNWD0K2G+YkH/7fGrr74KS0tL8NtvvyXvm1NjDBkX5hwY+lChMV68eBGmp6eT98utMYaMC2sODH2Yxmgao6EMsRpjUBHMaXkhc4BFDoZPihp8+81hvsRpjFiD7wWlxkg5rhRgfSBxa5fcHxopMI3RMBqCaYwl/XKPzch3HSyNMfM0xhBSJAZSnaVbVavvOkipN6StpTEmRvPYYhMDKbXTqlpT66SxSEljRNMYgVkHC6nhLKn0Oew5uHv3bk818rXXXnPvvvvuqbp92/rqmaH1htTqsw4h9UqZ225iNEZLY+zRZ1m/3Bu2qN4TYhMDfc/dTVGvT61l6+tbr5S59RmXpTE2kNjEQErlMkXKpNR6LY2xAG6NUSIpEgOplMtUKZMS67U0xgJMYzQ0IlZj5NbBQmoImQMJRlRIvVhItYx6IbVWEo2x3W6L9HljD/2VuqhFSPCJQ+B2j7GIHZd4jRHLXJJArmOTMC5uIwozsI7ciAoFw1ySQq5j4x4Xt5mGeX8WIwoLXyUtRF2j0txS1SChXglwm2nc9y9C1Kb1VdKkJCH6oq1eKczPz8NHH30Eb7zxBqyvr8Nbb73VqPsXIeo97Z07d+Du3btw9epVWFpaOvWzK1euwIMHD4LahbbFQlu9UuAOYeO+fyFVylT3BTV1sM3NTbe9ve1arZbb2dlxnU6np9rlq9D5tvNtW3dcvmPjrBdrzShqcI7/cG+MA7vL5laMxri4uAi3bt2CdrsNU1NTsL+/37Odr5ImKQnRZ2yS6vXFd82McKLntmpXuwTftM45Nz097S5fvuyePHnS81OLk5hxOUc/tpB6sdaMqoYcv2nL5tbnmzbIiOrr6/Nu3KvfInOJmxQv/ynHpsk2S1FDL7jvH1qD79yyBrv1ouihlmC3xBL7YIc8LJSk+CDC2AhS5ysE8UaUYRhpMI0xARhj49b3TmrACs3T8iyYxpih6geQfmzc+t4JWGum5VkwjdHwRqo+p4VUqYlS18E2rUCk6nNaSJWaKHUdRGmMxjFi9TmB3Lt3D2ZnZ8/996dPn8LNmzfh4cOHtdoCyF0Hkm/anAPYch0b1rhS9zs+Pg6PHj06dS0vL8OlS5fO/Z0xpC0mKoLdcg5gy3VsWOOimK+TJMSRkZGkbVMRPQdVylT3BcqUOIyrF0Vji+lXQqZ0qjXD6JcK0xiJlTgMiuar19hyPUM1tAaMfkPmCwsVGuPQ0BDEHCodG8CGRYpNQOlUU84X1rhyOFM4dh1MYzSMhqB+00pQ4iTUoIlctc8QVGiMWEhQ4iTUoIlctc8QRGiMKQ/crdu2CG3phljJjRhJlxKQqBumeG6LSLZpsQ7cjT2cV1u6IVZyI0bSpRQk6oaYh0rXkiuw1LFQzcwHbemGWMmNGEmXUuDWDTGe21KqXuR2X2cPle4m5YG7vm1TJSFCxEvyshrq9Js6uTG0Lfah0qlSHs8iQUbppu5zi34SfDerq6tuY2OjdCCp287MzLjh4WE3MDDgBgcH3d7eXs92BwcHbm5urvAeMQtVVkPdfqvqxW7r067u2KrWLJdNW/e5Jd20XKRIQoxdKAyNUToxY8PQGKVt2irINMaQ82kpiU1CxEpjzCHYrQhNGqNUI0r1+bRY53xSeqy9aqCcgxT9hhA7Z5RKK9YHYvbn0xqGkQbSNEZpqXa51KBN4cNA2xyI1xilptrlUINGhS81GudAhMZYhkTNzJdUOhqWGihtbjl01pA5yOFwb5JNK1Ez8yWFjoapBkqbWw6d1XcOcjncmyQjilsz8wVLR8NUAznnVorO6jsH2RzuXfUit/uCiBfP1IfzhvRbRpVyKUGllCQWcOisVXNwgpTDyFUcKo0FxcHHJ2l9169fr12DlIOiKaiaL4q2RUg53FvNodLUqXYh/cbCrVJK+qblwuebNjUYz624NMaSfr3b9upXQrKfBJUytt8QUsyZdhqRxohFrBKXYsNQHiotoV8JNXAT+8FlGqNhNATTGIXWkPOBztqUQwxMY8xQY8z1QGeNyiEGIjRGyYfzStXRmgj3s0CF+jRGTtVOso7WRKRpl1ioTmME4FXtROtoDUSL0hoCdRpjrU07Pj4O4+Pjp/7b48eP4cMPPyQ/nPfk1+btdhsGBgbgwoULp9pMTEzA/fv3YXJyElqtFksNEvrE7LfphO6H2HVI9sfjFJpZKBIUQgyVEkvPpNA+jWPK9gOpxhiTxsihMfoSc/+yGrDGJaHfmBo4tEuKcYWgPo0RS2Okun9RDdx6Jma/IYQ8Y00yolSnMcYSm8aIVUPsHFAf6Ey5ZkVIqMEXS2M0DMML0xiFkuu4AExjBDCNkV1jPCFlqBnFuHzrTTku32eB+4ze0LahiNAYy2iKukZ57m4KfGtIOS6fZ4H7jN7QttSQBLvNz8/D7u4u3L59GxYWFrIwYbjP3Q3Ftwbscfk8C9xn9Ia2JafqnVD3BQLezWH1W/f+3VAFlVHXmzqsrSoahvuMXt+2GM9t9sFuW1tbcHR0BADHr6OeP3/OWk+qoDKqcfnWm0sAm5Rgt+j1rdrVTvA3baoDimPGhUHKcUmEI4QNA4znNvtv2tnZWRgdHYVOpwPLy8vQ39/PXVISch2XcUzs+loaYyAh44oFS2M00mBpjBFQanmUfm4KjVGCy4v1AY4B5YecaYyG0RDUa4xYcGuE3PfHRNOzIFG/JQl2o0rgS6nlceuR3PfHQlMao1T9liTYjUpjxNDyjLRoUlql1koS7IahMWJoeQY+mpRWqbWSBLthJPD51iAlhM44RlMao9RaVQe71a2hqh23Hsl9fwOX2PVN9q98xsbGUnWFXkNVu8XFRVhbW4PDw0OYmpqClZUVUiuJ+/4GLrHra+9pe8CtEXLf38CFVGOUmsaIRWzCYgjceiYmuRpRKjRGqWmMRcTWGpuwGPsASk5N1LS5pCJeY8Q0S7TYNcYxmtZMYq2qg90wjZWcNUJfsA7LNiNKiMZYBpZZgmms+GhmVOfepkxDDGmLoVJSWEap5oD7+SqCZNNinUnalHNvsVIeOXROijVLNQdSz9IlSWPEMktyPPcWKw1Ris6Zes0w50CqEaU6IypVv71IkRgYM65uUqch+rbVtma+4wppSz0H2WdEYeGjmWGfe9sNVhqihERIDKjTI0MRozHmhK9mdvHiRZienkavJ0QRTdVWs0qJNV+piJ7bqq9il+iPx2Vw99sLCYdVc4N1sHUZmuYrZg7IDpWWmsYY229RnxIOq+aGOhEyR+MOoHe9JBojFlgbPBaJDysmverNIRFSwgn3vWisxogFlg2kybLSVG+Tni/VGiMmGDaQtrA2LfU27fkSpzFSqYFGPkgMYAtRKUMRpTFKPsjXkItE3RBTERWlMYo+yNcQC7duSK2Iivjt8QkTExNw//59mJychFarxVbH1tbW37/Za7fbMDAwABcuXBDXJyba6uUkNPEzdm5FaYyUamAZi4uLcOvWLWi32zA1NQX7+/si+8REW73SKNMjo+e2yr7ovgBJEg8hpIa6tTpXbKzE9JvKMMK6NNV7llRif+xz44t6IyoEyjNnY4PdevUrOaxNU72Bzy17Db2oa0SJ+uOxNFLYQBR9YqKtXk3UnVsRh0prcnSx7q/pTyYA/OuABdY3eMr5Uq8xalLtDDy0PV+N1hi1qHYGHhqfLxEaI1eqnQ8hamTObevClQbp25b7+aKG5FBpLamJObeNQUIapNTniwOSQ6W1pCbm3NYXCWmQmp4vFqpe5HZf/3vx2xOOVLvNzU23vb3tWq2W29nZcZ1O51zfPqmJObetO7fdcKVBhrTleL4w+iVNY+RItfPRwULUyJzbxsCRBhnTNhVYKiepxlj2TVsFVn5sigC2nImZW01wPF8Y/SbXGKWeTxsbwJYzmsQVLCSop779NkZjtA1rYIL1fJFojCGHSkuA8h8XUN0/tAYsND0HWPPFtQ7qv2klaIzcSYB2YDceEp6vs6jftNwaI3cSoMYDuzXB/Xz1Qv2m5YZbodN4YLckUqUm+uqkKbRT27SRcCt0OR7YTUmK1ERfnTSVdioq2E0j3Apdjgd2Y4GVmuirk6bSTlVvWksMNELASk30TRFNlTaq+o/HlhhoxJIiNdFXJ02lnaretLOzszA6OgqdTgeWl5fVHHpsyGFsbAxGRkZ6/izk+fI9YDzFQeRoaYxYYGlmMfcvQoJckfPRnL7EShCUmqyPxijCPdaEhIcVy8TJ1VPW9sHlnMvfPTaMJkGaxihNB8sFCbphrusrcVxkm1aiDpYDUnTDXNdX4rhI0hhDwNDBpKUbpkTTgd22ZmkgSWP0BUMHk5humBItB3bbmqWDJI3RFwwdjDvdEBstB3bbmiWkKo+m+6qbxuiTmuicfxKhlnRDrOssPseIYiY3Sk/FxBoX4vrypzFy6mAS0w05wBybllTMUMRqslW72nl+01aRS2oiKP2mTT22s0heX6xxIa6vnDTGHFITzYgqrkHq+kpIYwzBVRhRIk6C53Z0tel72j44MOaX+5kJrcG338ZEqBpGk2iUxihB9/NFwnyFgHn4MnfSZSMPlQbg18Gk6H6+cM9XKBj1SlizrA+Vlg5FuiDWgcpNpSmJkKE0ZtNSpAtiHajcVJqSCBmK6mC3EFKnC2Iekmwck2MiZApIvmm3trbg6OgIAI5T7Z4/f05xW1TGx8fh0aNHp67l5WW4dOnSub+rhLQF0Ddf2ur1BWtcsf2SfNMuLi7C2toaHB4ewtTUFKysrGQZwnaichYFhfm21TZf2ur1BWtc0f1WKVPdF+jTwU5RpPvF9IkFx3xJWN+zpFgzieMiO1Q61ohi0MG82kk1oqjnKwSs9cVYsxAox1X3UOmgPx4PDQ1BjHtc5KVibQTuhzt2UVN4vIEfylH3ovSOKT88pR0q3ZhXPoaRC+o3rQTdD0O1k6jPcfWrRT0NQYTGiGUDVbXl1v2wVDuJ+hx1v75zqzEwToTGiGUDSTeHTLXDw2dumxgYhx7slrs5ND8/D7u7u3D79m1YWFgwcychPnPbyMC4qndC3VfdYDfMtmVRK0DwHq+shph+U40r536d0xkYVzYun/e0JMFuWG1Nn8u7Xx80BsZFz1fVrnae37QczMzMuOHhYTcwMOAGBwfd3t7eqZ+D0m/alOPS3G/Z3EoAY75Iv2k5yPVQaaxxaes3V2LnK4vzaXM4VLpXv5T6nOR+paJCY2y3296FUj7cEmM7U4ClfVL3GwK3ekqJaYyG0RBI0xgxdLRcNUZtSFgHTYjQGMvATNXLVWPshksRDWnHvQ7aEKExlpGz6kcxNm5FVLpK2jRI4mZyVv1Sj41bEdWokjaOqhe53Rco09xC+q17/7IaYvs9gUsRDWlHvQ7aLt/5yl6uyFVjPAuHIhrSrinrkAoVaYxY5JoCeJaxsTHWtlXtmrIOqYidL9XftKbPycDWIQxSjVHC+bS9+s1VY9SG5PRIbnyfW3FpjFhQpzz6ImGDU0aNSl0HqYjXGHO2hrjHxn1/gxb1RhQ33GPjvr9BD0kaI5URlVLL823rOzasZD9pthm3cqmxbSgkaYxU54xiaHlVbX3GhpnsJ+0MV27lUkPbN98E6Ov7//Xmm5XdnQI9jREA55xRDC2vjsLnMzbMZD/OM1y5lUuNbQEA/vMfKP3/lVQpU91XTBojlsYYUkNoO9+2KRIDY+aAQyGMma+mtwU4f53gozEm27Srq6tuY2Oj8OcUm7aqhtB2vm19wscODg7c3Nxc4c9z2bSp5zbHtmI2bRUUm5aLFImBuWxao5r+/tMbtr///z9Lfj4tVrBb08yhXuQgVxjx+BhRqt1jw2gioe7xbwDwK145htF4/u2c+1dZg6BNaxgGP/bHY8NQhm1aw1CGbVrDUIZtWsNQhm1aw1CGbVrDUIZtWsNQhm1aw1CGbVrDUMZ/AV8KrKQ+sGjtAAAAAElFTkSuQmCC\n",
556 |       "text/plain": [
557 |        "<Figure size 432x288 with 1 Axes>"
558 |       ]
559 |      },
560 |      "metadata": {},
561 |      "output_type": "display_data"
562 |     }
563 |    ],
564 |    "source": [
565 |     "agent.plot_policy_map(best_net, 'solution_best.pdf', [0.35,-0.3])"
566 |    ]
567 |   }
568 |  ],
569 |  "metadata": {
570 |   "kernelspec": {
571 |    "display_name": "Python 3",
572 |    "language": "python",
573 |    "name": "python3"
574 |   },
575 |   "language_info": {
576 |    "codemirror_mode": {
577 |     "name": "ipython",
578 |     "version": 3
579 |    },
580 |    "file_extension": ".py",
581 |    "mimetype": "text/x-python",
582 |    "name": "python",
583 |    "nbconvert_exporter": "python",
584 |    "pygments_lexer": "ipython3",
585 |    "version": "3.6.9"
586 |   }
587 |  },
588 |  "nbformat": 4,
589 |  "nbformat_minor": 2
590 | }
591 | 


--------------------------------------------------------------------------------
/environment.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import scipy.special as sp
  3 | import matplotlib.pyplot as plt
  4 | import copy
  5 | 
  6 | class MazeEnvironment:    
  7 |     def __init__(self, maze, init_position, goal):
  8 |         x = len(maze)
  9 |         y = len(maze)
 10 |         
 11 |         self.boundary = np.asarray([x, y])
 12 |         self.init_position = init_position
 13 |         self.current_position = np.asarray(init_position)
 14 |         self.goal = goal
 15 |         self.maze = maze
 16 |         
 17 |         self.visited = set()
 18 |         self.visited.add(tuple(self.current_position))
 19 |                 
 20 |         # initialize the empty cells and the euclidean distance from
 21 |         # the goal (removing the goal cell itself)
 22 |         self.allowed_states = np.asarray(np.where(self.maze == 0)).T.tolist()
 23 |         self.distances = np.sqrt(np.sum((np.array(self.allowed_states) -
 24 |                                          np.asarray(self.goal))**2,
 25 |                                          axis = 1))
 26 |         
 27 |         del(self.allowed_states[np.where(self.distances == 0)[0][0]])
 28 |         self.distances = np.delete(self.distances, np.where(self.distances == 0)[0][0])
 29 |                 
 30 |         self.action_map = {0: [0, 1],
 31 |                            1: [0, -1],
 32 |                            2: [1, 0],
 33 |                            3: [-1, 0]}
 34 |         
 35 |         self.directions = {0: '→',
 36 |                            1: '←',
 37 |                            2: '↓ ',
 38 |                            3: '↑'}
 39 |         
 40 |         # the agent makes an action from the following:
 41 |         # 1 -> right, 2 -> left
 42 |         # 3 -> down, 4 -> up
 43 |         
 44 |     # introduce a reset policy, so that for high epsilon the initial
 45 |     # position is nearer to the goal (useful for large mazes)
 46 |     def reset_policy(self, eps, reg = 7):
 47 |         return sp.softmax(-self.distances/(reg*(1-eps**(2/reg)))**(reg/2)).squeeze()
 48 |     
 49 |     # reset the environment when the game is completed
 50 |     # with probability prand the reset is random, otherwise
 51 |     # the reset policy at the given epsilon is used
 52 |     def reset(self, epsilon, prand = 0):
 53 |         if np.random.rand() < prand:
 54 |             idx = np.random.choice(len(self.allowed_states))
 55 |         else:
 56 |             p = self.reset_policy(epsilon)
 57 |             idx = np.random.choice(len(self.allowed_states), p = p)
 58 | 
 59 |         self.current_position = np.asarray(self.allowed_states[idx])
 60 |         
 61 |         self.visited = set()
 62 |         self.visited.add(tuple(self.current_position))
 63 | 
 64 |         return self.state()
 65 |     
 66 |     
 67 |     def state_update(self, action):
 68 |         isgameon = True
 69 |         
 70 |         # each move costs -0.05
 71 |         reward = -0.05
 72 |         
 73 |         move = self.action_map[action]
 74 |         next_position = self.current_position + np.asarray(move)
 75 |         
 76 |         # if the goals has been reached, the reward is 1
 77 |         if (self.current_position == self.goal).all():
 78 |                 reward = 1
 79 |                 isgameon = False
 80 |                 return [self.state(), reward, isgameon]
 81 |             
 82 |         # if the cell has been visited before, the reward is -0.2
 83 |         else:
 84 |             if tuple(self.current_position) in self.visited:
 85 |                 reward = -0.2
 86 |         
 87 |         # if the moves goes out of the maze or to a wall, the
 88 |         # reward is -1
 89 |         if self.is_state_valid(next_position):
 90 |             self.current_position = next_position
 91 |         else:
 92 |             reward = -1
 93 |         
 94 |         self.visited.add(tuple(self.current_position))
 95 |         return [self.state(), reward, isgameon]
 96 | 
 97 |     # return the state to be feeded to the network
 98 |     def state(self):
 99 |         state = copy.deepcopy(self.maze)
100 |         state[tuple(self.current_position)] = 2
101 |         return state
102 |         
103 |     
104 |     def check_boundaries(self, position):
105 |         out = len([num for num in position if num < 0])
106 |         out += len([num for num in (self.boundary - np.asarray(position)) if num <= 0])
107 |         return out > 0
108 |     
109 |     
110 |     def check_walls(self, position):
111 |         return self.maze[tuple(position)] == 1
112 |     
113 |     
114 |     def is_state_valid(self, next_position):
115 |         if self.check_boundaries(next_position):
116 |             return False
117 |         elif self.check_walls(next_position):
118 |             return False
119 |         return True
120 |     
121 |     
122 |     def draw(self, filename):
123 |         plt.figure()
124 |         im = plt.imshow(self.maze, interpolation='none', aspect='equal', cmap='Greys');
125 |         ax = plt.gca();
126 | 
127 |         plt.xticks([], [])
128 |         plt.yticks([], [])
129 | 
130 |         ax.plot(self.goal[1], self.goal[0],
131 |                 'bs', markersize = 4)
132 |         ax.plot(self.current_position[1], self.current_position[0],
133 |                 'rs', markersize = 4)
134 |         plt.savefig(filename, dpi = 300, bbox_inches = 'tight')
135 |         plt.show()
136 | 


--------------------------------------------------------------------------------
/maze_generator/maze.npy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/giorgionicoletti/deep_Q_learning_maze/5806405207d9f49d043047eae328c885e08dbfc6/maze_generator/maze.npy


--------------------------------------------------------------------------------
/maze_generator/maze_generator.py:
--------------------------------------------------------------------------------
 1 | # Adapted from http://code.activestate.com/recipes/578356-random-maze-generator/
 2 | # Random Maze Generator using Depth-first Search
 3 | # http://en.wikipedia.org/wiki/Maze_generation_algorithm
 4 | # FB36 - 20130106
 5 | 
 6 | import random
 7 | import matplotlib.pyplot as plt
 8 | import numpy as np
 9 | 
10 | 
11 | mx = 20; my = 20 # width and height of the maze
12 | 
13 | maze = [[0 for x in range(mx)] for y in range(my)]
14 | dx = [0, 1, 0, -1]; dy = [-1, 0, 1, 0] # 4 directions to move in the maze
15 | color = [(0, 0, 0), (255, 255, 255)] # RGB colors of the maze
16 | 
17 | # start the maze from a random cell
18 | cx = random.randint(0, mx - 1)
19 | cy = random.randint(0, my - 1)
20 | maze[cy][cx] = 1
21 | stack = [(cx, cy, 0)] # stack element: (x, y, direction)
22 | 
23 | while len(stack) > 0:
24 |     (cx, cy, cd) = stack[-1]
25 |     # to prevent zigzags:
26 |     # if changed direction in the last move then cannot change again
27 |     if len(stack) > 2:
28 |         if cd != stack[-2][2]: dirRange = [cd]
29 |         else: dirRange = range(4)
30 |     else: dirRange = range(4)
31 | 
32 |     # find a new cell to add
33 |     nlst = [] # list of available neighbors
34 |     for i in dirRange:
35 |         nx = cx + dx[i]
36 |         ny = cy + dy[i]
37 |         if nx >= 0 and nx < mx and ny >= 0 and ny < my:
38 |             if maze[ny][nx] == 0:
39 |                 ctr = 0 # of occupied neighbors must be 1
40 |                 for j in range(4):
41 |                     ex = nx + dx[j]; ey = ny + dy[j]
42 |                     if ex >= 0 and ex < mx and ey >= 0 and ey < my:
43 |                         if maze[ey][ex] == 1: ctr += 1
44 |                 if ctr == 1: nlst.append(i)
45 | 
46 |     # if 1 or more neighbors available then randomly select one and move
47 |     if len(nlst) > 0:
48 |         ir = nlst[random.randint(0, len(nlst) - 1)]
49 |         cx += dx[ir]; cy += dy[ir]; maze[cy][cx] = 1
50 |         stack.append((cx, cy, ir))
51 |     else: stack.pop()
52 | 
53 | maze = np.array(maze)
54 | maze -= 1
55 | maze = abs(maze)
56 |         
57 | maze[0][0] = 0
58 | maze[mx-1][my-1] = 0
59 | 
60 | np.save('maze', np.array(maze))
61 | 


--------------------------------------------------------------------------------