├── IntroToOpenAIGym.ipynb
├── QLearningIntro.ipynb
├── HillClimbingInOpenAIGym.ipynb
├── QLearningWithNeuralNetworks.ipynb
├── FasterQLearningTrainingWithExperienceReplay.ipynb
├── DeepQNetworksInOpenAIGym.ipynb
├── DiscreteVsContinuousEnvironments.ipynb
├── DiscretizedDQNsForContinuousControl.ipynb
└── PrioritizedExperienceReplayInOpenAIGym.ipynb


/IntroToOpenAIGym.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 7,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import gym\n",
 10 |     "import random"
 11 |    ]
 12 |   },
 13 |   {
 14 |    "cell_type": "code",
 15 |    "execution_count": 6,
 16 |    "metadata": {},
 17 |    "outputs": [
 18 |     {
 19 |      "name": "stdout",
 20 |      "output_type": "stream",
 21 |      "text": [
 22 |       "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
 23 |       "Observation space: Box(4,)\n",
 24 |       "Action space: Discrete(2)\n"
 25 |      ]
 26 |     }
 27 |    ],
 28 |    "source": [
 29 |     "env_name = \"CartPole-v1\"\n",
 30 |     "env = gym.make(env_name)\n",
 31 |     "print(\"Observation space:\", env.observation_space)\n",
 32 |     "print(\"Action space:\", env.action_space)"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "code",
 37 |    "execution_count": 14,
 38 |    "metadata": {},
 39 |    "outputs": [],
 40 |    "source": [
 41 |     "class Agent():\n",
 42 |     "    def __init__(self, env):\n",
 43 |     "        self.action_size = env.action_space.n\n",
 44 |     "        print(\"Action size:\", self.action_size)\n",
 45 |     "        \n",
 46 |     "    def get_action(self, state):\n",
 47 |     "#         action = random.choice(range(self.action_size))\n",
 48 |     "        pole_angle = state[2]\n",
 49 |     "        action = 0 if pole_angle < 0 else 1\n",
 50 |     "        return action"
 51 |    ]
 52 |   },
 53 |   {
 54 |    "cell_type": "code",
 55 |    "execution_count": 16,
 56 |    "metadata": {},
 57 |    "outputs": [
 58 |     {
 59 |      "name": "stdout",
 60 |      "output_type": "stream",
 61 |      "text": [
 62 |       "Action size: 2\n",
 63 |       "\u001b[33mWARN: You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior.\u001b[0m\n"
 64 |      ]
 65 |     }
 66 |    ],
 67 |    "source": [
 68 |     "agent = Agent(env)\n",
 69 |     "state = env.reset()\n",
 70 |     "\n",
 71 |     "for _ in range(200):\n",
 72 |     "#     action = env.action_space.sample()\n",
 73 |     "    action = agent.get_action(state)\n",
 74 |     "    state, reward, done, info = env.step(action)\n",
 75 |     "    env.render()"
 76 |    ]
 77 |   },
 78 |   {
 79 |    "cell_type": "code",
 80 |    "execution_count": null,
 81 |    "metadata": {},
 82 |    "outputs": [],
 83 |    "source": []
 84 |   }
 85 |  ],
 86 |  "metadata": {
 87 |   "kernelspec": {
 88 |    "display_name": "Python 3",
 89 |    "language": "python",
 90 |    "name": "python3"
 91 |   },
 92 |   "language_info": {
 93 |    "codemirror_mode": {
 94 |     "name": "ipython",
 95 |     "version": 3
 96 |    },
 97 |    "file_extension": ".py",
 98 |    "mimetype": "text/x-python",
 99 |    "name": "python",
100 |    "nbconvert_exporter": "python",
101 |    "pygments_lexer": "ipython3",
102 |    "version": "3.6.5"
103 |   }
104 |  },
105 |  "nbformat": 4,
106 |  "nbformat_minor": 2
107 | }
108 | 


--------------------------------------------------------------------------------
/QLearningIntro.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 11,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import gym\n",
 10 |     "import random\n",
 11 |     "import numpy as np\n",
 12 |     "import time\n",
 13 |     "from gym.envs.registration import register\n",
 14 |     "from IPython.display import clear_output"
 15 |    ]
 16 |   },
 17 |   {
 18 |    "cell_type": "code",
 19 |    "execution_count": 6,
 20 |    "metadata": {},
 21 |    "outputs": [
 22 |     {
 23 |      "name": "stdout",
 24 |      "output_type": "stream",
 25 |      "text": [
 26 |       "Observation space: Discrete(16)\n",
 27 |       "Action space: Discrete(4)\n"
 28 |      ]
 29 |     },
 30 |     {
 31 |      "data": {
 32 |       "text/plain": [
 33 |        "gym.spaces.discrete.Discrete"
 34 |       ]
 35 |      },
 36 |      "execution_count": 6,
 37 |      "metadata": {},
 38 |      "output_type": "execute_result"
 39 |     }
 40 |    ],
 41 |    "source": [
 42 |     "# https://github.com/openai/gym/blob/master/gym/envs/toy_text/frozen_lake.py\n",
 43 |     "# https://github.com/openai/gym/blob/master/gym/envs/__init__.py\n",
 44 |     "try:\n",
 45 |     "    register(\n",
 46 |     "        id='FrozenLakeNoSlip-v0',\n",
 47 |     "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
 48 |     "        kwargs={'map_name' : '4x4', 'is_slippery':False},\n",
 49 |     "        max_episode_steps=100,\n",
 50 |     "        reward_threshold=0.78, # optimum = .8196\n",
 51 |     "    )\n",
 52 |     "except:\n",
 53 |     "    pass\n",
 54 |     "\n",
 55 |     "# env_name = \"CartPole-v1\"\n",
 56 |     "# env_name = \"MountainCar-v0\"\n",
 57 |     "# env_name = \"MountainCarContinuous-v0\"\n",
 58 |     "# env_name = \"Acrobot-v1\"\n",
 59 |     "# env_name = \"Pendulum-v0\"\n",
 60 |     "env_name = \"FrozenLake-v0\"\n",
 61 |     "env_name = \"FrozenLakeNoSlip-v0\"\n",
 62 |     "env = gym.make(env_name)\n",
 63 |     "print(\"Observation space:\", env.observation_space)\n",
 64 |     "print(\"Action space:\", env.action_space)\n",
 65 |     "type(env.action_space)"
 66 |    ]
 67 |   },
 68 |   {
 69 |    "cell_type": "code",
 70 |    "execution_count": 7,
 71 |    "metadata": {},
 72 |    "outputs": [],
 73 |    "source": [
 74 |     "class Agent():\n",
 75 |     "    def __init__(self, env):\n",
 76 |     "        self.is_discrete = \\\n",
 77 |     "            type(env.action_space) == gym.spaces.discrete.Discrete\n",
 78 |     "        \n",
 79 |     "        if self.is_discrete:\n",
 80 |     "            self.action_size = env.action_space.n\n",
 81 |     "            print(\"Action size:\", self.action_size)\n",
 82 |     "        else:\n",
 83 |     "            self.action_low = env.action_space.low\n",
 84 |     "            self.action_high = env.action_space.high\n",
 85 |     "            self.action_shape = env.action_space.shape\n",
 86 |     "            print(\"Action range:\", self.action_low, self.action_high)\n",
 87 |     "        \n",
 88 |     "    def get_action(self, state):\n",
 89 |     "        if self.is_discrete:\n",
 90 |     "            action = random.choice(range(self.action_size))\n",
 91 |     "        else:\n",
 92 |     "            action = np.random.uniform(self.action_low,\n",
 93 |     "                                       self.action_high,\n",
 94 |     "                                       self.action_shape)\n",
 95 |     "        return action"
 96 |    ]
 97 |   },
 98 |   {
 99 |    "cell_type": "code",
100 |    "execution_count": 21,
101 |    "metadata": {},
102 |    "outputs": [
103 |     {
104 |      "name": "stdout",
105 |      "output_type": "stream",
106 |      "text": [
107 |       "Action size: 4\n",
108 |       "State size: 16\n"
109 |      ]
110 |     }
111 |    ],
112 |    "source": [
113 |     "class QAgent(Agent):\n",
114 |     "    def __init__(self, env, discount_rate=0.97, learning_rate=0.01):\n",
115 |     "        super().__init__(env)\n",
116 |     "        self.state_size = env.observation_space.n\n",
117 |     "        print(\"State size:\", self.state_size)\n",
118 |     "        \n",
119 |     "        self.eps = 1.0\n",
120 |     "        self.discount_rate = discount_rate\n",
121 |     "        self.learning_rate = learning_rate\n",
122 |     "        self.build_model()\n",
123 |     "        \n",
124 |     "    def build_model(self):\n",
125 |     "        self.q_table = 1e-4*np.random.random([self.state_size, self.action_size])\n",
126 |     "        \n",
127 |     "    def get_action(self, state):\n",
128 |     "        q_state = self.q_table[state]\n",
129 |     "        action_greedy = np.argmax(q_state)\n",
130 |     "        action_random = super().get_action(state)\n",
131 |     "        return action_random if random.random() < self.eps else action_greedy\n",
132 |     "    \n",
133 |     "    def train(self, experience):\n",
134 |     "        state, action, next_state, reward, done = experience\n",
135 |     "        \n",
136 |     "        q_next = self.q_table[next_state]\n",
137 |     "        q_next = np.zeros([self.action_size]) if done else q_next\n",
138 |     "        q_target = reward + self.discount_rate * np.max(q_next)\n",
139 |     "        \n",
140 |     "        q_update = q_target - self.q_table[state,action]\n",
141 |     "        self.q_table[state,action] += self.learning_rate * q_update\n",
142 |     "        \n",
143 |     "        if done:\n",
144 |     "            self.eps = self.eps * 0.99\n",
145 |     "        \n",
146 |     "agent = QAgent(env)"
147 |    ]
148 |   },
149 |   {
150 |    "cell_type": "code",
151 |    "execution_count": 26,
152 |    "metadata": {},
153 |    "outputs": [
154 |     {
155 |      "name": "stdout",
156 |      "output_type": "stream",
157 |      "text": [
158 |       "s: 15 a: 2\n",
159 |       "Episode: 99, Total reward: 99.0, eps: 0.006570483042414605\n",
160 |       "  (Right)\n",
161 |       "SFFF\n",
162 |       "FHFH\n",
163 |       "FFFH\n",
164 |       "HFF\u001b[41mG\u001b[0m\n",
165 |       "[[2.26657246e-04 5.78706808e-05 1.31003002e-01 4.79979472e-04]\n",
166 |       " [1.04844339e-04 3.54687230e-05 2.55427402e-01 7.48193170e-04]\n",
167 |       " [1.65884052e-04 4.36189268e-01 7.00960632e-05 8.88959016e-04]\n",
168 |       " [1.17590145e-03 5.49928561e-05 6.38101134e-05 3.16729025e-05]\n",
169 |       " [6.15095796e-05 4.13004395e-05 3.01916296e-05 5.90834920e-05]\n",
170 |       " [4.25841839e-05 7.56731911e-06 8.26995681e-05 6.60735347e-05]\n",
171 |       " [6.37688773e-05 6.48470611e-01 7.98525028e-05 3.01763264e-03]\n",
172 |       " [5.30045086e-05 5.07371856e-05 1.99983098e-05 9.32633643e-05]\n",
173 |       " [4.18707042e-05 5.71000704e-05 2.32037264e-04 3.18999734e-05]\n",
174 |       " [4.15603587e-05 6.91116263e-03 2.75917720e-04 2.88792860e-05]\n",
175 |       " [3.17844770e-04 8.42711089e-01 2.42033377e-05 2.93321581e-03]\n",
176 |       " [9.02368275e-05 6.37119718e-05 6.00035959e-05 7.00083118e-05]\n",
177 |       " [7.71263802e-05 1.73696049e-05 4.11883145e-05 3.22260325e-05]\n",
178 |       " [5.22960464e-05 8.34910792e-05 1.07448006e-01 1.17173895e-04]\n",
179 |       " [3.04657414e-03 6.80853272e-02 9.72068773e-01 1.34302488e-02]\n",
180 |       " [1.35322054e-06 5.97967404e-05 5.91493712e-05 5.44460043e-05]]\n"
181 |      ]
182 |     }
183 |    ],
184 |    "source": [
185 |     "total_reward = 0\n",
186 |     "for ep in range(100):\n",
187 |     "    state = env.reset()\n",
188 |     "    done = False\n",
189 |     "    while not done:\n",
190 |     "        action = agent.get_action(state)\n",
191 |     "        next_state, reward, done, info = env.step(action)\n",
192 |     "        agent.train((state,action,next_state,reward,done))\n",
193 |     "        state = next_state\n",
194 |     "        total_reward += reward\n",
195 |     "        \n",
196 |     "        print(\"s:\", state, \"a:\", action)\n",
197 |     "        print(\"Episode: {}, Total reward: {}, eps: {}\".format(ep,total_reward,agent.eps))\n",
198 |     "        env.render()\n",
199 |     "        print(agent.q_table)\n",
200 |     "        time.sleep(0.05)\n",
201 |     "        clear_output(wait=True)"
202 |    ]
203 |   },
204 |   {
205 |    "cell_type": "code",
206 |    "execution_count": null,
207 |    "metadata": {},
208 |    "outputs": [],
209 |    "source": []
210 |   }
211 |  ],
212 |  "metadata": {
213 |   "kernelspec": {
214 |    "display_name": "Python 3",
215 |    "language": "python",
216 |    "name": "python3"
217 |   },
218 |   "language_info": {
219 |    "codemirror_mode": {
220 |     "name": "ipython",
221 |     "version": 3
222 |    },
223 |    "file_extension": ".py",
224 |    "mimetype": "text/x-python",
225 |    "name": "python",
226 |    "nbconvert_exporter": "python",
227 |    "pygments_lexer": "ipython3",
228 |    "version": "3.6.5"
229 |   }
230 |  },
231 |  "nbformat": 4,
232 |  "nbformat_minor": 2
233 | }
234 | 


--------------------------------------------------------------------------------
/HillClimbingInOpenAIGym.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import gym\n",
 10 |     "import random\n",
 11 |     "import numpy as np"
 12 |    ]
 13 |   },
 14 |   {
 15 |    "cell_type": "code",
 16 |    "execution_count": 2,
 17 |    "metadata": {},
 18 |    "outputs": [
 19 |     {
 20 |      "name": "stdout",
 21 |      "output_type": "stream",
 22 |      "text": [
 23 |       "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
 24 |       "Observation space: Box(4,)\n",
 25 |       "Action space: Discrete(2)\n"
 26 |      ]
 27 |     }
 28 |    ],
 29 |    "source": [
 30 |     "env_name = \"CartPole-v0\"\n",
 31 |     "env = gym.make(env_name)\n",
 32 |     "print(\"Observation space:\", env.observation_space)\n",
 33 |     "print(\"Action space:\", env.action_space)"
 34 |    ]
 35 |   },
 36 |   {
 37 |    "cell_type": "code",
 38 |    "execution_count": 3,
 39 |    "metadata": {},
 40 |    "outputs": [],
 41 |    "source": [
 42 |     "class HillClimbingAgent():\n",
 43 |     "    def __init__(self, env):\n",
 44 |     "        self.state_dim = env.observation_space.shape\n",
 45 |     "        self.action_size = env.action_space.n\n",
 46 |     "        self.build_model()\n",
 47 |     "        \n",
 48 |     "    def build_model(self):\n",
 49 |     "        self.weights = 1e-4*np.random.rand(*self.state_dim, self.action_size)\n",
 50 |     "        self.best_reward = -np.Inf\n",
 51 |     "        self.best_weights = np.copy(self.weights)\n",
 52 |     "        self.noise_scale = 1e-2\n",
 53 |     "        \n",
 54 |     "    def get_action(self, state):\n",
 55 |     "        p = np.dot(state, self.weights)\n",
 56 |     "        action = np.argmax(p)\n",
 57 |     "        return action\n",
 58 |     "    \n",
 59 |     "    def update_model(self, reward):\n",
 60 |     "        if reward >= self.best_reward:\n",
 61 |     "            self.best_reward = reward\n",
 62 |     "            self.best_weights = np.copy(self.weights)\n",
 63 |     "            self.noise_scale = max(self.noise_scale/2, 1e-3)\n",
 64 |     "        else:\n",
 65 |     "            self.noise_scale = min(self.noise_scale*2, 2)\n",
 66 |     "            \n",
 67 |     "        self.weights = self.best_weights + self.noise_scale * np.random.rand(*self.state_dim, self.action_size)"
 68 |    ]
 69 |   },
 70 |   {
 71 |    "cell_type": "code",
 72 |    "execution_count": 5,
 73 |    "metadata": {},
 74 |    "outputs": [
 75 |     {
 76 |      "name": "stdout",
 77 |      "output_type": "stream",
 78 |      "text": [
 79 |       "Episode: 0, total_reward: 50.00\n",
 80 |       "Episode: 1, total_reward: 9.00\n",
 81 |       "Episode: 2, total_reward: 74.00\n",
 82 |       "Episode: 3, total_reward: 79.00\n",
 83 |       "Episode: 4, total_reward: 156.00\n",
 84 |       "Episode: 5, total_reward: 44.00\n",
 85 |       "Episode: 6, total_reward: 68.00\n",
 86 |       "Episode: 7, total_reward: 61.00\n",
 87 |       "Episode: 8, total_reward: 12.00\n",
 88 |       "Episode: 9, total_reward: 41.00\n",
 89 |       "Episode: 10, total_reward: 10.00\n",
 90 |       "Episode: 11, total_reward: 45.00\n",
 91 |       "Episode: 12, total_reward: 57.00\n",
 92 |       "Episode: 13, total_reward: 200.00\n",
 93 |       "Episode: 14, total_reward: 112.00\n",
 94 |       "Episode: 15, total_reward: 73.00\n",
 95 |       "Episode: 16, total_reward: 132.00\n",
 96 |       "Episode: 17, total_reward: 29.00\n",
 97 |       "Episode: 18, total_reward: 10.00\n",
 98 |       "Episode: 19, total_reward: 52.00\n",
 99 |       "Episode: 20, total_reward: 75.00\n",
100 |       "Episode: 21, total_reward: 9.00\n",
101 |       "Episode: 22, total_reward: 10.00\n",
102 |       "Episode: 23, total_reward: 48.00\n",
103 |       "Episode: 24, total_reward: 200.00\n",
104 |       "Episode: 25, total_reward: 200.00\n",
105 |       "Episode: 26, total_reward: 200.00\n",
106 |       "Episode: 27, total_reward: 200.00\n",
107 |       "Episode: 28, total_reward: 200.00\n",
108 |       "Episode: 29, total_reward: 200.00\n",
109 |       "Episode: 30, total_reward: 200.00\n",
110 |       "Episode: 31, total_reward: 200.00\n",
111 |       "Episode: 32, total_reward: 200.00\n",
112 |       "Episode: 33, total_reward: 200.00\n",
113 |       "Episode: 34, total_reward: 200.00\n",
114 |       "Episode: 35, total_reward: 200.00\n",
115 |       "Episode: 36, total_reward: 200.00\n",
116 |       "Episode: 37, total_reward: 200.00\n",
117 |       "Episode: 38, total_reward: 200.00\n",
118 |       "Episode: 39, total_reward: 200.00\n",
119 |       "Episode: 40, total_reward: 200.00\n",
120 |       "Episode: 41, total_reward: 200.00\n",
121 |       "Episode: 42, total_reward: 200.00\n",
122 |       "Episode: 43, total_reward: 200.00\n",
123 |       "Episode: 44, total_reward: 200.00\n",
124 |       "Episode: 45, total_reward: 200.00\n",
125 |       "Episode: 46, total_reward: 200.00\n",
126 |       "Episode: 47, total_reward: 200.00\n",
127 |       "Episode: 48, total_reward: 200.00\n",
128 |       "Episode: 49, total_reward: 200.00\n",
129 |       "Episode: 50, total_reward: 200.00\n",
130 |       "Episode: 51, total_reward: 200.00\n",
131 |       "Episode: 52, total_reward: 200.00\n",
132 |       "Episode: 53, total_reward: 200.00\n",
133 |       "Episode: 54, total_reward: 200.00\n",
134 |       "Episode: 55, total_reward: 200.00\n",
135 |       "Episode: 56, total_reward: 200.00\n",
136 |       "Episode: 57, total_reward: 200.00\n",
137 |       "Episode: 58, total_reward: 200.00\n",
138 |       "Episode: 59, total_reward: 200.00\n",
139 |       "Episode: 60, total_reward: 200.00\n",
140 |       "Episode: 61, total_reward: 200.00\n",
141 |       "Episode: 62, total_reward: 200.00\n",
142 |       "Episode: 63, total_reward: 200.00\n",
143 |       "Episode: 64, total_reward: 200.00\n",
144 |       "Episode: 65, total_reward: 200.00\n",
145 |       "Episode: 66, total_reward: 200.00\n",
146 |       "Episode: 67, total_reward: 200.00\n",
147 |       "Episode: 68, total_reward: 200.00\n",
148 |       "Episode: 69, total_reward: 200.00\n",
149 |       "Episode: 70, total_reward: 200.00\n",
150 |       "Episode: 71, total_reward: 200.00\n",
151 |       "Episode: 72, total_reward: 200.00\n",
152 |       "Episode: 73, total_reward: 200.00\n",
153 |       "Episode: 74, total_reward: 200.00\n",
154 |       "Episode: 75, total_reward: 200.00\n",
155 |       "Episode: 76, total_reward: 200.00\n",
156 |       "Episode: 77, total_reward: 200.00\n",
157 |       "Episode: 78, total_reward: 200.00\n",
158 |       "Episode: 79, total_reward: 200.00\n",
159 |       "Episode: 80, total_reward: 200.00\n",
160 |       "Episode: 81, total_reward: 200.00\n",
161 |       "Episode: 82, total_reward: 200.00\n",
162 |       "Episode: 83, total_reward: 200.00\n",
163 |       "Episode: 84, total_reward: 200.00\n",
164 |       "Episode: 85, total_reward: 200.00\n",
165 |       "Episode: 86, total_reward: 200.00\n",
166 |       "Episode: 87, total_reward: 200.00\n",
167 |       "Episode: 88, total_reward: 200.00\n",
168 |       "Episode: 89, total_reward: 200.00\n",
169 |       "Episode: 90, total_reward: 200.00\n",
170 |       "Episode: 91, total_reward: 200.00\n",
171 |       "Episode: 92, total_reward: 200.00\n",
172 |       "Episode: 93, total_reward: 200.00\n",
173 |       "Episode: 94, total_reward: 200.00\n",
174 |       "Episode: 95, total_reward: 200.00\n",
175 |       "Episode: 96, total_reward: 200.00\n",
176 |       "Episode: 97, total_reward: 200.00\n",
177 |       "Episode: 98, total_reward: 200.00\n",
178 |       "Episode: 99, total_reward: 200.00\n"
179 |      ]
180 |     }
181 |    ],
182 |    "source": [
183 |     "agent = HillClimbingAgent(env)\n",
184 |     "num_episodes = 100\n",
185 |     "\n",
186 |     "for ep in range(num_episodes):\n",
187 |     "    state = env.reset()\n",
188 |     "    total_reward = 0\n",
189 |     "    done = False\n",
190 |     "    while not done:\n",
191 |     "        action = agent.get_action(state)\n",
192 |     "        state, reward, done, info = env.step(action)\n",
193 |     "        env.render()\n",
194 |     "        total_reward += reward\n",
195 |     "        \n",
196 |     "    agent.update_model(total_reward)\n",
197 |     "    print(\"Episode: {}, total_reward: {:.2f}\".format(ep, total_reward))"
198 |    ]
199 |   },
200 |   {
201 |    "cell_type": "code",
202 |    "execution_count": null,
203 |    "metadata": {},
204 |    "outputs": [],
205 |    "source": []
206 |   }
207 |  ],
208 |  "metadata": {
209 |   "kernelspec": {
210 |    "display_name": "Python 3",
211 |    "language": "python",
212 |    "name": "python3"
213 |   },
214 |   "language_info": {
215 |    "codemirror_mode": {
216 |     "name": "ipython",
217 |     "version": 3
218 |    },
219 |    "file_extension": ".py",
220 |    "mimetype": "text/x-python",
221 |    "name": "python",
222 |    "nbconvert_exporter": "python",
223 |    "pygments_lexer": "ipython3",
224 |    "version": "3.6.5"
225 |   }
226 |  },
227 |  "nbformat": 4,
228 |  "nbformat_minor": 2
229 | }
230 | 


--------------------------------------------------------------------------------
/QLearningWithNeuralNetworks.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 5,
  6 |    "metadata": {},
  7 |    "outputs": [
  8 |     {
  9 |      "name": "stderr",
 10 |      "output_type": "stream",
 11 |      "text": [
 12 |       "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
 13 |       "  from ._conv import register_converters as _register_converters\n"
 14 |      ]
 15 |     }
 16 |    ],
 17 |    "source": [
 18 |     "import gym\n",
 19 |     "import random\n",
 20 |     "import numpy as np\n",
 21 |     "import tensorflow as tf\n",
 22 |     "import time\n",
 23 |     "from gym.envs.registration import register\n",
 24 |     "from IPython.display import clear_output"
 25 |    ]
 26 |   },
 27 |   {
 28 |    "cell_type": "code",
 29 |    "execution_count": 2,
 30 |    "metadata": {},
 31 |    "outputs": [
 32 |     {
 33 |      "name": "stdout",
 34 |      "output_type": "stream",
 35 |      "text": [
 36 |       "Observation space: Discrete(16)\n",
 37 |       "Action space: Discrete(4)\n"
 38 |      ]
 39 |     },
 40 |     {
 41 |      "data": {
 42 |       "text/plain": [
 43 |        "gym.spaces.discrete.Discrete"
 44 |       ]
 45 |      },
 46 |      "execution_count": 2,
 47 |      "metadata": {},
 48 |      "output_type": "execute_result"
 49 |     }
 50 |    ],
 51 |    "source": [
 52 |     "# https://github.com/openai/gym/blob/master/gym/envs/toy_text/frozen_lake.py\n",
 53 |     "# https://github.com/openai/gym/blob/master/gym/envs/__init__.py\n",
 54 |     "try:\n",
 55 |     "    register(\n",
 56 |     "        id='FrozenLakeNoSlip-v0',\n",
 57 |     "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
 58 |     "        kwargs={'map_name' : '4x4', 'is_slippery':False},\n",
 59 |     "        max_episode_steps=100,\n",
 60 |     "        reward_threshold=0.78, # optimum = .8196\n",
 61 |     "    )\n",
 62 |     "except:\n",
 63 |     "    pass\n",
 64 |     "\n",
 65 |     "# env_name = \"CartPole-v1\"\n",
 66 |     "# env_name = \"MountainCar-v0\"\n",
 67 |     "# env_name = \"MountainCarContinuous-v0\"\n",
 68 |     "# env_name = \"Acrobot-v1\"\n",
 69 |     "# env_name = \"Pendulum-v0\"\n",
 70 |     "env_name = \"FrozenLake-v0\"\n",
 71 |     "env_name = \"FrozenLakeNoSlip-v0\"\n",
 72 |     "env = gym.make(env_name)\n",
 73 |     "print(\"Observation space:\", env.observation_space)\n",
 74 |     "print(\"Action space:\", env.action_space)\n",
 75 |     "type(env.action_space)"
 76 |    ]
 77 |   },
 78 |   {
 79 |    "cell_type": "code",
 80 |    "execution_count": 3,
 81 |    "metadata": {},
 82 |    "outputs": [],
 83 |    "source": [
 84 |     "class Agent():\n",
 85 |     "    def __init__(self, env):\n",
 86 |     "        self.is_discrete = \\\n",
 87 |     "            type(env.action_space) == gym.spaces.discrete.Discrete\n",
 88 |     "        \n",
 89 |     "        if self.is_discrete:\n",
 90 |     "            self.action_size = env.action_space.n\n",
 91 |     "            print(\"Action size:\", self.action_size)\n",
 92 |     "        else:\n",
 93 |     "            self.action_low = env.action_space.low\n",
 94 |     "            self.action_high = env.action_space.high\n",
 95 |     "            self.action_shape = env.action_space.shape\n",
 96 |     "            print(\"Action range:\", self.action_low, self.action_high)\n",
 97 |     "        \n",
 98 |     "    def get_action(self, state):\n",
 99 |     "        if self.is_discrete:\n",
100 |     "            action = random.choice(range(self.action_size))\n",
101 |     "        else:\n",
102 |     "            action = np.random.uniform(self.action_low,\n",
103 |     "                                       self.action_high,\n",
104 |     "                                       self.action_shape)\n",
105 |     "        return action"
106 |    ]
107 |   },
108 |   {
109 |    "cell_type": "code",
110 |    "execution_count": 21,
111 |    "metadata": {},
112 |    "outputs": [
113 |     {
114 |      "name": "stdout",
115 |      "output_type": "stream",
116 |      "text": [
117 |       "Action size: 4\n",
118 |       "State size: 16\n"
119 |      ]
120 |     }
121 |    ],
122 |    "source": [
123 |     "class QNAgent(Agent):\n",
124 |     "    def __init__(self, env, discount_rate=0.97, learning_rate=0.01):\n",
125 |     "        super().__init__(env)\n",
126 |     "        self.state_size = env.observation_space.n\n",
127 |     "        print(\"State size:\", self.state_size)\n",
128 |     "        \n",
129 |     "        self.eps = 1.0\n",
130 |     "        self.discount_rate = discount_rate\n",
131 |     "        self.learning_rate = learning_rate\n",
132 |     "        self.build_model()\n",
133 |     "        \n",
134 |     "        self.sess = tf.Session()\n",
135 |     "        self.sess.run(tf.global_variables_initializer())\n",
136 |     "        \n",
137 |     "    def build_model(self):\n",
138 |     "        tf.reset_default_graph()\n",
139 |     "        self.state_in = tf.placeholder(tf.int32, shape=[1])\n",
140 |     "        self.action_in = tf.placeholder(tf.int32, shape=[1])\n",
141 |     "        self.target_in = tf.placeholder(tf.float32, shape=[1])\n",
142 |     "        \n",
143 |     "        self.state = tf.one_hot(self.state_in, depth=self.state_size)\n",
144 |     "        self.action = tf.one_hot(self.action_in, depth=self.action_size)\n",
145 |     "        \n",
146 |     "        self.q_state = tf.layers.dense(self.state, units=self.action_size, name=\"q_table\")\n",
147 |     "        self.q_action = tf.reduce_sum(tf.multiply(self.q_state, self.action), axis=1)\n",
148 |     "        \n",
149 |     "        self.loss = tf.reduce_sum(tf.square(self.target_in - self.q_action))\n",
150 |     "        self.optimizer = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n",
151 |     "        \n",
152 |     "    def get_action(self, state):\n",
153 |     "        q_state = self.sess.run(self.q_state, feed_dict={self.state_in: [state]})\n",
154 |     "        action_greedy = np.argmax(q_state)\n",
155 |     "        action_random = super().get_action(state)\n",
156 |     "        return action_random if random.random() < self.eps else action_greedy\n",
157 |     "    \n",
158 |     "    def train(self, experience):\n",
159 |     "        state, action, next_state, reward, done = ([exp] for exp in experience)\n",
160 |     "        \n",
161 |     "        q_next = self.sess.run(self.q_state, feed_dict={self.state_in: next_state})\n",
162 |     "        q_next[done] = np.zeros([self.action_size])\n",
163 |     "        q_target = reward + self.discount_rate * np.max(q_next)\n",
164 |     "        \n",
165 |     "        feed = {self.state_in: state, self.action_in: action, self.target_in: q_target}\n",
166 |     "        self.sess.run(self.optimizer, feed_dict=feed)\n",
167 |     "        \n",
168 |     "        if experience[4]:\n",
169 |     "            self.eps = self.eps * 0.99\n",
170 |     "            \n",
171 |     "    def __del__(self):\n",
172 |     "        self.sess.close()\n",
173 |     "        \n",
174 |     "agent = QNAgent(env)"
175 |    ]
176 |   },
177 |   {
178 |    "cell_type": "code",
179 |    "execution_count": 27,
180 |    "metadata": {},
181 |    "outputs": [
182 |     {
183 |      "name": "stdout",
184 |      "output_type": "stream",
185 |      "text": [
186 |       "s: 15 a: 2\n",
187 |       "Episode: 99, Total reward: 71.0, eps: 0.13397967485796175\n",
188 |       "  (Right)\n",
189 |       "SFFF\n",
190 |       "FHFH\n",
191 |       "FFFH\n",
192 |       "HFF\u001b[41mG\u001b[0m\n",
193 |       "[[ 0.26639792  0.23388803  0.2838942   0.17449419]\n",
194 |       " [ 0.27005327 -0.4024756   0.1345939  -0.03563018]\n",
195 |       " [ 0.00683864  0.11768234  0.1508714  -0.0175943 ]\n",
196 |       " [ 0.1645656  -0.4792729   0.1205796  -0.1818824 ]\n",
197 |       " [ 0.24408905  0.2570015  -0.39948407  0.18414727]\n",
198 |       " [-0.51439786 -0.14780176  0.41053963 -0.07503477]\n",
199 |       " [-0.4445371   0.1159418  -0.51110405 -0.03309001]\n",
200 |       " [-0.20044279 -0.25773728  0.26571983 -0.49883124]\n",
201 |       " [ 0.31056625 -0.52676606  0.41068304  0.20587952]\n",
202 |       " [ 0.33791235  0.3192833   0.41197717 -0.54344785]\n",
203 |       " [ 0.3656563   0.13440336 -0.32820502 -0.02268323]\n",
204 |       " [ 0.19982225 -0.49265525  0.4245258   0.05591565]\n",
205 |       " [ 0.14388543  0.22361499 -0.33384812  0.517786  ]\n",
206 |       " [-0.30385983  0.2902795   0.47450408  0.2592849 ]\n",
207 |       " [ 0.13970053  0.21021995  0.5043464   0.2025718 ]\n",
208 |       " [ 0.15857393  0.19134241  0.45319474 -0.47139642]]\n"
209 |      ]
210 |     }
211 |    ],
212 |    "source": [
213 |     "total_reward = 0\n",
214 |     "for ep in range(100):\n",
215 |     "    state = env.reset()\n",
216 |     "    done = False\n",
217 |     "    while not done:\n",
218 |     "        action = agent.get_action(state)\n",
219 |     "        next_state, reward, done, info = env.step(action)\n",
220 |     "        agent.train((state,action,next_state,reward,done))\n",
221 |     "        state = next_state\n",
222 |     "        total_reward += reward\n",
223 |     "        \n",
224 |     "        print(\"s:\", state, \"a:\", action)\n",
225 |     "        print(\"Episode: {}, Total reward: {}, eps: {}\".format(ep,total_reward,agent.eps))\n",
226 |     "        env.render()\n",
227 |     "        with tf.variable_scope(\"q_table\", reuse=True):\n",
228 |     "            weights = agent.sess.run(tf.get_variable(\"kernel\"))\n",
229 |     "            print(weights)\n",
230 |     "        time.sleep(0.05)\n",
231 |     "        clear_output(wait=True)"
232 |    ]
233 |   },
234 |   {
235 |    "cell_type": "code",
236 |    "execution_count": null,
237 |    "metadata": {},
238 |    "outputs": [],
239 |    "source": []
240 |   }
241 |  ],
242 |  "metadata": {
243 |   "kernelspec": {
244 |    "display_name": "Python 3",
245 |    "language": "python",
246 |    "name": "python3"
247 |   },
248 |   "language_info": {
249 |    "codemirror_mode": {
250 |     "name": "ipython",
251 |     "version": 3
252 |    },
253 |    "file_extension": ".py",
254 |    "mimetype": "text/x-python",
255 |    "name": "python",
256 |    "nbconvert_exporter": "python",
257 |    "pygments_lexer": "ipython3",
258 |    "version": "3.6.5"
259 |   }
260 |  },
261 |  "nbformat": 4,
262 |  "nbformat_minor": 2
263 | }
264 | 


--------------------------------------------------------------------------------
/FasterQLearningTrainingWithExperienceReplay.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 6,
  6 |    "metadata": {},
  7 |    "outputs": [
  8 |     {
  9 |      "name": "stdout",
 10 |      "output_type": "stream",
 11 |      "text": [
 12 |       "Using OpenAI Gym: 0.10.5\n",
 13 |       "Using Tensorflow: 1.9.0\n"
 14 |      ]
 15 |     }
 16 |    ],
 17 |    "source": [
 18 |     "import gym\n",
 19 |     "import random\n",
 20 |     "import numpy as np\n",
 21 |     "import tensorflow as tf\n",
 22 |     "import time\n",
 23 |     "from gym.envs.registration import register\n",
 24 |     "from IPython.display import clear_output\n",
 25 |     "from collections import deque\n",
 26 |     "print(\"Using OpenAI Gym:\", gym.__version__)\n",
 27 |     "print(\"Using Tensorflow:\", tf.__version__)"
 28 |    ]
 29 |   },
 30 |   {
 31 |    "cell_type": "code",
 32 |    "execution_count": 2,
 33 |    "metadata": {},
 34 |    "outputs": [
 35 |     {
 36 |      "name": "stdout",
 37 |      "output_type": "stream",
 38 |      "text": [
 39 |       "Observation space: Discrete(16)\n",
 40 |       "Action space: Discrete(4)\n"
 41 |      ]
 42 |     },
 43 |     {
 44 |      "data": {
 45 |       "text/plain": [
 46 |        "gym.spaces.discrete.Discrete"
 47 |       ]
 48 |      },
 49 |      "execution_count": 2,
 50 |      "metadata": {},
 51 |      "output_type": "execute_result"
 52 |     }
 53 |    ],
 54 |    "source": [
 55 |     "# https://github.com/openai/gym/blob/master/gym/envs/toy_text/frozen_lake.py\n",
 56 |     "# https://github.com/openai/gym/blob/master/gym/envs/__init__.py\n",
 57 |     "try:\n",
 58 |     "    register(\n",
 59 |     "        id='FrozenLakeNoSlip-v0',\n",
 60 |     "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
 61 |     "        kwargs={'map_name' : '4x4', 'is_slippery':False},\n",
 62 |     "        max_episode_steps=100,\n",
 63 |     "        reward_threshold=0.78, # optimum = .8196\n",
 64 |     "    )\n",
 65 |     "except:\n",
 66 |     "    pass\n",
 67 |     "\n",
 68 |     "# env_name = \"CartPole-v1\"\n",
 69 |     "# env_name = \"MountainCar-v0\"\n",
 70 |     "# env_name = \"MountainCarContinuous-v0\"\n",
 71 |     "# env_name = \"Acrobot-v1\"\n",
 72 |     "# env_name = \"Pendulum-v0\"\n",
 73 |     "env_name = \"FrozenLake-v0\"\n",
 74 |     "env_name = \"FrozenLakeNoSlip-v0\"\n",
 75 |     "env = gym.make(env_name)\n",
 76 |     "print(\"Observation space:\", env.observation_space)\n",
 77 |     "print(\"Action space:\", env.action_space)\n",
 78 |     "type(env.action_space)"
 79 |    ]
 80 |   },
 81 |   {
 82 |    "cell_type": "code",
 83 |    "execution_count": 3,
 84 |    "metadata": {},
 85 |    "outputs": [],
 86 |    "source": [
 87 |     "class Agent():\n",
 88 |     "    def __init__(self, env):\n",
 89 |     "        self.is_discrete = \\\n",
 90 |     "            type(env.action_space) == gym.spaces.discrete.Discrete\n",
 91 |     "        \n",
 92 |     "        if self.is_discrete:\n",
 93 |     "            self.action_size = env.action_space.n\n",
 94 |     "            print(\"Action size:\", self.action_size)\n",
 95 |     "        else:\n",
 96 |     "            self.action_low = env.action_space.low\n",
 97 |     "            self.action_high = env.action_space.high\n",
 98 |     "            self.action_shape = env.action_space.shape\n",
 99 |     "            print(\"Action range:\", self.action_low, self.action_high)\n",
100 |     "        \n",
101 |     "    def get_action(self, state):\n",
102 |     "        if self.is_discrete:\n",
103 |     "            action = random.choice(range(self.action_size))\n",
104 |     "        else:\n",
105 |     "            action = np.random.uniform(self.action_low,\n",
106 |     "                                       self.action_high,\n",
107 |     "                                       self.action_shape)\n",
108 |     "        return action"
109 |    ]
110 |   },
111 |   {
112 |    "cell_type": "code",
113 |    "execution_count": 13,
114 |    "metadata": {},
115 |    "outputs": [
116 |     {
117 |      "name": "stdout",
118 |      "output_type": "stream",
119 |      "text": [
120 |       "Action size: 4\n",
121 |       "State size: 16\n"
122 |      ]
123 |     }
124 |    ],
125 |    "source": [
126 |     "class QNAgent(Agent):\n",
127 |     "    def __init__(self, env, discount_rate=0.97, learning_rate=0.001):\n",
128 |     "        super().__init__(env)\n",
129 |     "        self.state_size = env.observation_space.n\n",
130 |     "        print(\"State size:\", self.state_size)\n",
131 |     "        \n",
132 |     "        self.eps = 1.0\n",
133 |     "        self.discount_rate = discount_rate\n",
134 |     "        self.learning_rate = learning_rate\n",
135 |     "        self.build_model()\n",
136 |     "        \n",
137 |     "        self.sess = tf.Session()\n",
138 |     "        self.sess.run(tf.global_variables_initializer())\n",
139 |     "        self.replay_buffer = deque(maxlen=1000)\n",
140 |     "        \n",
141 |     "    def build_model(self):\n",
142 |     "        tf.reset_default_graph()\n",
143 |     "        self.state_in = tf.placeholder(tf.int32, shape=[None])\n",
144 |     "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
145 |     "        self.target_in = tf.placeholder(tf.float32, shape=[None])\n",
146 |     "        \n",
147 |     "        self.state = tf.one_hot(self.state_in, depth=self.state_size)\n",
148 |     "        self.action = tf.one_hot(self.action_in, depth=self.action_size)\n",
149 |     "        \n",
150 |     "        self.q_state = tf.layers.dense(self.state, units=self.action_size, name=\"q_table\")\n",
151 |     "        self.q_action = tf.reduce_sum(tf.multiply(self.q_state, self.action), axis=1)\n",
152 |     "        \n",
153 |     "        self.loss = tf.reduce_sum(tf.square(self.target_in - self.q_action))\n",
154 |     "        self.optimizer = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n",
155 |     "        \n",
156 |     "    def get_action(self, state):\n",
157 |     "        q_state = self.sess.run(self.q_state, feed_dict={self.state_in: [state]})\n",
158 |     "        action_greedy = np.argmax(q_state)\n",
159 |     "        action_random = super().get_action(state)\n",
160 |     "        return action_random if random.random() < self.eps else action_greedy\n",
161 |     "    \n",
162 |     "    def train(self, experience, batch_size=50):\n",
163 |     "        self.replay_buffer.append(experience)\n",
164 |     "        samples = random.choices(self.replay_buffer, k=batch_size)\n",
165 |     "        state, action, next_state, reward, done = (list(col) for col in zip(experience, *samples))\n",
166 |     "#         state, action, next_state, reward, done = ([exp] for exp in experience)\n",
167 |     "        \n",
168 |     "        q_next = self.sess.run(self.q_state, feed_dict={self.state_in: next_state})\n",
169 |     "        q_next[done] = np.zeros([self.action_size])\n",
170 |     "        q_target = reward + self.discount_rate * np.max(q_next, axis=1)\n",
171 |     "        \n",
172 |     "        feed = {self.state_in: state, self.action_in: action, self.target_in: q_target}\n",
173 |     "        self.sess.run(self.optimizer, feed_dict=feed)\n",
174 |     "        \n",
175 |     "        if experience[4]:\n",
176 |     "            self.eps = self.eps * 0.99\n",
177 |     "            \n",
178 |     "    def __del__(self):\n",
179 |     "        self.sess.close()\n",
180 |     "        \n",
181 |     "agent = QNAgent(env)"
182 |    ]
183 |   },
184 |   {
185 |    "cell_type": "code",
186 |    "execution_count": 15,
187 |    "metadata": {},
188 |    "outputs": [
189 |     {
190 |      "name": "stdout",
191 |      "output_type": "stream",
192 |      "text": [
193 |       "s: 15 a: 2\n",
194 |       "Episode: 99, Total reward: 73.0, eps: 0.13397967485796175\n",
195 |       "  (Right)\n",
196 |       "SFFF\n",
197 |       "FHFH\n",
198 |       "FFFH\n",
199 |       "HFF\u001b[41mG\u001b[0m\n",
200 |       "[[ 0.2915219   0.4591379   0.2689546   0.31623498]\n",
201 |       " [ 0.31743488 -0.3748523   0.04215977  0.00409675]\n",
202 |       " [ 0.1477144   0.48421127  0.04322524  0.31961307]\n",
203 |       " [ 0.00712917  0.08716363  0.0486892   0.02223596]\n",
204 |       " [ 0.31576723  0.48447582 -0.51515913  0.311779  ]\n",
205 |       " [ 0.0684554   0.04598576  0.05356741 -0.01328123]\n",
206 |       " [-0.42640948  0.5349467  -0.41976473  0.30681062]\n",
207 |       " [ 0.25691265  0.30407184 -0.5459194   0.25798124]\n",
208 |       " [ 0.34918264 -0.38702574  0.41161922  0.34177908]\n",
209 |       " [ 0.35890004  0.27314752  0.437125   -0.5167427 ]\n",
210 |       " [ 0.2928187   0.56286657 -0.51294297  0.39509773]\n",
211 |       " [-0.36676133  0.50335765 -0.16498888 -0.16828948]\n",
212 |       " [ 0.01138669  0.49019372 -0.12844104  0.17367804]\n",
213 |       " [-0.53203475  0.01639959  0.24097243 -0.0812238 ]\n",
214 |       " [ 0.08020027  0.47967038  0.49306536  0.0293826 ]\n",
215 |       " [ 0.00829726  0.12029958 -0.30489913 -0.42049965]]\n"
216 |      ]
217 |     }
218 |    ],
219 |    "source": [
220 |     "total_reward = 0\n",
221 |     "for ep in range(100):\n",
222 |     "    state = env.reset()\n",
223 |     "    done = False\n",
224 |     "    while not done:\n",
225 |     "        action = agent.get_action(state)\n",
226 |     "        next_state, reward, done, info = env.step(action)\n",
227 |     "        agent.train((state,action,next_state,reward,done))\n",
228 |     "        state = next_state\n",
229 |     "        total_reward += reward\n",
230 |     "        \n",
231 |     "        print(\"s:\", state, \"a:\", action)\n",
232 |     "        print(\"Episode: {}, Total reward: {}, eps: {}\".format(ep,total_reward,agent.eps))\n",
233 |     "        env.render()\n",
234 |     "        with tf.variable_scope(\"q_table\", reuse=True):\n",
235 |     "            weights = agent.sess.run(tf.get_variable(\"kernel\"))\n",
236 |     "            print(weights)\n",
237 |     "        time.sleep(0.05)\n",
238 |     "        clear_output(wait=True)"
239 |    ]
240 |   },
241 |   {
242 |    "cell_type": "code",
243 |    "execution_count": null,
244 |    "metadata": {},
245 |    "outputs": [],
246 |    "source": []
247 |   }
248 |  ],
249 |  "metadata": {
250 |   "kernelspec": {
251 |    "display_name": "Python 3",
252 |    "language": "python",
253 |    "name": "python3"
254 |   },
255 |   "language_info": {
256 |    "codemirror_mode": {
257 |     "name": "ipython",
258 |     "version": 3
259 |    },
260 |    "file_extension": ".py",
261 |    "mimetype": "text/x-python",
262 |    "name": "python",
263 |    "nbconvert_exporter": "python",
264 |    "pygments_lexer": "ipython3",
265 |    "version": "3.6.5"
266 |   }
267 |  },
268 |  "nbformat": 4,
269 |  "nbformat_minor": 2
270 | }
271 | 


--------------------------------------------------------------------------------
/DeepQNetworksInOpenAIGym.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 11,
  6 |    "metadata": {},
  7 |    "outputs": [
  8 |     {
  9 |      "name": "stdout",
 10 |      "output_type": "stream",
 11 |      "text": [
 12 |       "Gym: 0.10.5\n",
 13 |       "Tensorflow: 1.9.0\n"
 14 |      ]
 15 |     }
 16 |    ],
 17 |    "source": [
 18 |     "import gym\n",
 19 |     "import random\n",
 20 |     "import numpy as np\n",
 21 |     "import tensorflow as tf\n",
 22 |     "from collections import deque\n",
 23 |     "print(\"Gym:\", gym.__version__)\n",
 24 |     "print(\"Tensorflow:\", tf.__version__)"
 25 |    ]
 26 |   },
 27 |   {
 28 |    "cell_type": "code",
 29 |    "execution_count": 3,
 30 |    "metadata": {},
 31 |    "outputs": [
 32 |     {
 33 |      "name": "stdout",
 34 |      "output_type": "stream",
 35 |      "text": [
 36 |       "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
 37 |       "Observation space: Box(4,)\n",
 38 |       "Action space: Discrete(2)\n"
 39 |      ]
 40 |     }
 41 |    ],
 42 |    "source": [
 43 |     "env_name = \"CartPole-v0\"\n",
 44 |     "env = gym.make(env_name)\n",
 45 |     "print(\"Observation space:\", env.observation_space)\n",
 46 |     "print(\"Action space:\", env.action_space)"
 47 |    ]
 48 |   },
 49 |   {
 50 |    "cell_type": "code",
 51 |    "execution_count": 6,
 52 |    "metadata": {},
 53 |    "outputs": [],
 54 |    "source": [
 55 |     "class QNetwork():\n",
 56 |     "    def __init__(self, state_dim, action_size):\n",
 57 |     "        self.state_in = tf.placeholder(tf.float32, shape=[None, *state_dim])\n",
 58 |     "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
 59 |     "        self.q_target_in = tf.placeholder(tf.float32, shape=[None])\n",
 60 |     "        action_one_hot = tf.one_hot(self.action_in, depth=action_size)\n",
 61 |     "        \n",
 62 |     "        self.hidden1 = tf.layers.dense(self.state_in, 100, activation=tf.nn.relu)\n",
 63 |     "        self.q_state = tf.layers.dense(self.hidden1, action_size, activation=None)\n",
 64 |     "        self.q_state_action = tf.reduce_sum(tf.multiply(self.q_state, action_one_hot), axis=1)\n",
 65 |     "        \n",
 66 |     "        self.loss = tf.reduce_mean(tf.square(self.q_state_action - self.q_target_in))\n",
 67 |     "        self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(self.loss)\n",
 68 |     "        \n",
 69 |     "    def update_model(self, session, state, action, q_target):\n",
 70 |     "        feed = {self.state_in: state, self.action_in: action, self.q_target_in: q_target}\n",
 71 |     "        session.run(self.optimizer, feed_dict=feed)\n",
 72 |     "        \n",
 73 |     "    def get_q_state(self, session, state):\n",
 74 |     "        q_state = session.run(self.q_state, feed_dict={self.state_in: state})\n",
 75 |     "        return q_state"
 76 |    ]
 77 |   },
 78 |   {
 79 |    "cell_type": "code",
 80 |    "execution_count": 12,
 81 |    "metadata": {},
 82 |    "outputs": [],
 83 |    "source": [
 84 |     "class ReplayBuffer():\n",
 85 |     "    def __init__(self, maxlen):\n",
 86 |     "        self.buffer = deque(maxlen=maxlen)\n",
 87 |     "        \n",
 88 |     "    def add(self, experience):\n",
 89 |     "        self.buffer.append(experience)\n",
 90 |     "        \n",
 91 |     "    def sample(self, batch_size):\n",
 92 |     "        sample_size = min(len(self.buffer), batch_size)\n",
 93 |     "        samples = random.choices(self.buffer, k=sample_size)\n",
 94 |     "        return map(list, zip(*samples))"
 95 |    ]
 96 |   },
 97 |   {
 98 |    "cell_type": "code",
 99 |    "execution_count": 13,
100 |    "metadata": {},
101 |    "outputs": [],
102 |    "source": [
103 |     "class DQNAgent():\n",
104 |     "    def __init__(self, env):\n",
105 |     "        self.state_dim = env.observation_space.shape\n",
106 |     "        self.action_size = env.action_space.n\n",
107 |     "        self.q_network = QNetwork(self.state_dim, self.action_size)\n",
108 |     "        self.replay_buffer = ReplayBuffer(maxlen=10000)\n",
109 |     "        self.gamma = 0.97\n",
110 |     "        self.eps = 1.0\n",
111 |     "        \n",
112 |     "        self.sess = tf.Session()\n",
113 |     "        self.sess.run(tf.global_variables_initializer())\n",
114 |     "        \n",
115 |     "    def get_action(self, state):\n",
116 |     "        q_state = self.q_network.get_q_state(self.sess, [state])\n",
117 |     "        action_greedy = np.argmax(q_state)\n",
118 |     "        action_random = np.random.randint(self.action_size)\n",
119 |     "        action = action_random if random.random() < self.eps else action_greedy\n",
120 |     "        return action\n",
121 |     "    \n",
122 |     "    def train(self, state, action, next_state, reward, done):\n",
123 |     "        self.replay_buffer.add((state, action, next_state, reward, done))\n",
124 |     "        states, actions, next_states, rewards, dones = self.replay_buffer.sample(50)\n",
125 |     "        q_next_states = self.q_network.get_q_state(self.sess, next_states)\n",
126 |     "        q_next_states[dones] = np.zeros([self.action_size])\n",
127 |     "        q_targets = rewards + self.gamma * np.max(q_next_states, axis=1)\n",
128 |     "        self.q_network.update_model(self.sess, states, actions, q_targets)\n",
129 |     "        \n",
130 |     "        if done: self.eps = max(0.1, 0.99*self.eps)\n",
131 |     "    \n",
132 |     "    def __del__(self):\n",
133 |     "        self.sess.close()"
134 |    ]
135 |   },
136 |   {
137 |    "cell_type": "code",
138 |    "execution_count": 14,
139 |    "metadata": {},
140 |    "outputs": [
141 |     {
142 |      "name": "stdout",
143 |      "output_type": "stream",
144 |      "text": [
145 |       "Episode: 0, total_reward: 17.00\n",
146 |       "Episode: 1, total_reward: 11.00\n",
147 |       "Episode: 2, total_reward: 14.00\n",
148 |       "Episode: 3, total_reward: 22.00\n",
149 |       "Episode: 4, total_reward: 19.00\n",
150 |       "Episode: 5, total_reward: 37.00\n",
151 |       "Episode: 6, total_reward: 13.00\n",
152 |       "Episode: 7, total_reward: 13.00\n",
153 |       "Episode: 8, total_reward: 18.00\n",
154 |       "Episode: 9, total_reward: 21.00\n",
155 |       "Episode: 10, total_reward: 20.00\n",
156 |       "Episode: 11, total_reward: 41.00\n",
157 |       "Episode: 12, total_reward: 21.00\n",
158 |       "Episode: 13, total_reward: 25.00\n",
159 |       "Episode: 14, total_reward: 20.00\n",
160 |       "Episode: 15, total_reward: 11.00\n",
161 |       "Episode: 16, total_reward: 13.00\n",
162 |       "Episode: 17, total_reward: 19.00\n",
163 |       "Episode: 18, total_reward: 14.00\n",
164 |       "Episode: 19, total_reward: 39.00\n",
165 |       "Episode: 20, total_reward: 12.00\n",
166 |       "Episode: 21, total_reward: 37.00\n",
167 |       "Episode: 22, total_reward: 23.00\n",
168 |       "Episode: 23, total_reward: 15.00\n",
169 |       "Episode: 24, total_reward: 40.00\n",
170 |       "Episode: 25, total_reward: 14.00\n",
171 |       "Episode: 26, total_reward: 11.00\n",
172 |       "Episode: 27, total_reward: 30.00\n",
173 |       "Episode: 28, total_reward: 13.00\n",
174 |       "Episode: 29, total_reward: 13.00\n",
175 |       "Episode: 30, total_reward: 12.00\n",
176 |       "Episode: 31, total_reward: 14.00\n",
177 |       "Episode: 32, total_reward: 17.00\n",
178 |       "Episode: 33, total_reward: 15.00\n",
179 |       "Episode: 34, total_reward: 10.00\n",
180 |       "Episode: 35, total_reward: 10.00\n",
181 |       "Episode: 36, total_reward: 13.00\n",
182 |       "Episode: 37, total_reward: 12.00\n",
183 |       "Episode: 38, total_reward: 11.00\n",
184 |       "Episode: 39, total_reward: 9.00\n",
185 |       "Episode: 40, total_reward: 15.00\n",
186 |       "Episode: 41, total_reward: 17.00\n",
187 |       "Episode: 42, total_reward: 15.00\n",
188 |       "Episode: 43, total_reward: 31.00\n",
189 |       "Episode: 44, total_reward: 10.00\n",
190 |       "Episode: 45, total_reward: 19.00\n",
191 |       "Episode: 46, total_reward: 8.00\n",
192 |       "Episode: 47, total_reward: 22.00\n",
193 |       "Episode: 48, total_reward: 11.00\n",
194 |       "Episode: 49, total_reward: 10.00\n",
195 |       "Episode: 50, total_reward: 13.00\n",
196 |       "Episode: 51, total_reward: 33.00\n",
197 |       "Episode: 52, total_reward: 10.00\n",
198 |       "Episode: 53, total_reward: 11.00\n",
199 |       "Episode: 54, total_reward: 12.00\n",
200 |       "Episode: 55, total_reward: 12.00\n",
201 |       "Episode: 56, total_reward: 19.00\n",
202 |       "Episode: 57, total_reward: 65.00\n",
203 |       "Episode: 58, total_reward: 15.00\n",
204 |       "Episode: 59, total_reward: 34.00\n",
205 |       "Episode: 60, total_reward: 43.00\n",
206 |       "Episode: 61, total_reward: 72.00\n",
207 |       "Episode: 62, total_reward: 45.00\n",
208 |       "Episode: 63, total_reward: 27.00\n",
209 |       "Episode: 64, total_reward: 43.00\n",
210 |       "Episode: 65, total_reward: 62.00\n",
211 |       "Episode: 66, total_reward: 45.00\n",
212 |       "Episode: 67, total_reward: 60.00\n",
213 |       "Episode: 68, total_reward: 60.00\n",
214 |       "Episode: 69, total_reward: 57.00\n",
215 |       "Episode: 70, total_reward: 19.00\n",
216 |       "Episode: 71, total_reward: 67.00\n",
217 |       "Episode: 72, total_reward: 99.00\n",
218 |       "Episode: 73, total_reward: 131.00\n",
219 |       "Episode: 74, total_reward: 23.00\n",
220 |       "Episode: 75, total_reward: 39.00\n",
221 |       "Episode: 76, total_reward: 90.00\n",
222 |       "Episode: 77, total_reward: 58.00\n",
223 |       "Episode: 78, total_reward: 62.00\n",
224 |       "Episode: 79, total_reward: 91.00\n",
225 |       "Episode: 80, total_reward: 48.00\n",
226 |       "Episode: 81, total_reward: 47.00\n",
227 |       "Episode: 82, total_reward: 68.00\n",
228 |       "Episode: 83, total_reward: 51.00\n",
229 |       "Episode: 84, total_reward: 92.00\n",
230 |       "Episode: 85, total_reward: 75.00\n",
231 |       "Episode: 86, total_reward: 67.00\n",
232 |       "Episode: 87, total_reward: 100.00\n",
233 |       "Episode: 88, total_reward: 52.00\n",
234 |       "Episode: 89, total_reward: 50.00\n",
235 |       "Episode: 90, total_reward: 122.00\n",
236 |       "Episode: 91, total_reward: 65.00\n",
237 |       "Episode: 92, total_reward: 62.00\n",
238 |       "Episode: 93, total_reward: 61.00\n",
239 |       "Episode: 94, total_reward: 76.00\n",
240 |       "Episode: 95, total_reward: 89.00\n",
241 |       "Episode: 96, total_reward: 11.00\n",
242 |       "Episode: 97, total_reward: 99.00\n",
243 |       "Episode: 98, total_reward: 147.00\n",
244 |       "Episode: 99, total_reward: 156.00\n",
245 |       "Episode: 100, total_reward: 100.00\n",
246 |       "Episode: 101, total_reward: 88.00\n",
247 |       "Episode: 102, total_reward: 167.00\n",
248 |       "Episode: 103, total_reward: 124.00\n",
249 |       "Episode: 104, total_reward: 123.00\n",
250 |       "Episode: 105, total_reward: 55.00\n",
251 |       "Episode: 106, total_reward: 124.00\n",
252 |       "Episode: 107, total_reward: 81.00\n",
253 |       "Episode: 108, total_reward: 129.00\n",
254 |       "Episode: 109, total_reward: 165.00\n",
255 |       "Episode: 110, total_reward: 123.00\n",
256 |       "Episode: 111, total_reward: 131.00\n",
257 |       "Episode: 112, total_reward: 63.00\n",
258 |       "Episode: 113, total_reward: 169.00\n",
259 |       "Episode: 114, total_reward: 198.00\n",
260 |       "Episode: 115, total_reward: 141.00\n",
261 |       "Episode: 116, total_reward: 174.00\n",
262 |       "Episode: 117, total_reward: 135.00\n",
263 |       "Episode: 118, total_reward: 196.00\n",
264 |       "Episode: 119, total_reward: 200.00\n",
265 |       "Episode: 120, total_reward: 200.00\n",
266 |       "Episode: 121, total_reward: 200.00\n",
267 |       "Episode: 122, total_reward: 200.00\n",
268 |       "Episode: 123, total_reward: 200.00\n",
269 |       "Episode: 124, total_reward: 131.00\n",
270 |       "Episode: 125, total_reward: 200.00\n",
271 |       "Episode: 126, total_reward: 200.00\n",
272 |       "Episode: 127, total_reward: 200.00\n",
273 |       "Episode: 128, total_reward: 125.00\n",
274 |       "Episode: 129, total_reward: 200.00\n",
275 |       "Episode: 130, total_reward: 200.00\n",
276 |       "Episode: 131, total_reward: 200.00\n",
277 |       "Episode: 132, total_reward: 191.00\n",
278 |       "Episode: 133, total_reward: 200.00\n",
279 |       "Episode: 134, total_reward: 200.00\n",
280 |       "Episode: 135, total_reward: 200.00\n",
281 |       "Episode: 136, total_reward: 200.00\n",
282 |       "Episode: 137, total_reward: 200.00\n",
283 |       "Episode: 138, total_reward: 200.00\n",
284 |       "Episode: 139, total_reward: 200.00\n",
285 |       "Episode: 140, total_reward: 200.00\n",
286 |       "Episode: 141, total_reward: 200.00\n",
287 |       "Episode: 142, total_reward: 200.00\n",
288 |       "Episode: 143, total_reward: 200.00\n",
289 |       "Episode: 144, total_reward: 200.00\n",
290 |       "Episode: 145, total_reward: 200.00\n",
291 |       "Episode: 146, total_reward: 200.00\n",
292 |       "Episode: 147, total_reward: 200.00\n",
293 |       "Episode: 148, total_reward: 200.00\n",
294 |       "Episode: 149, total_reward: 200.00\n",
295 |       "Episode: 150, total_reward: 200.00\n",
296 |       "Episode: 151, total_reward: 200.00\n",
297 |       "Episode: 152, total_reward: 200.00\n",
298 |       "Episode: 153, total_reward: 200.00\n",
299 |       "Episode: 154, total_reward: 200.00\n",
300 |       "Episode: 155, total_reward: 200.00\n",
301 |       "Episode: 156, total_reward: 200.00\n",
302 |       "Episode: 157, total_reward: 200.00\n",
303 |       "Episode: 158, total_reward: 200.00\n",
304 |       "Episode: 159, total_reward: 200.00\n",
305 |       "Episode: 160, total_reward: 200.00\n",
306 |       "Episode: 161, total_reward: 200.00\n",
307 |       "Episode: 162, total_reward: 200.00\n",
308 |       "Episode: 163, total_reward: 200.00\n",
309 |       "Episode: 164, total_reward: 200.00\n",
310 |       "Episode: 165, total_reward: 200.00\n",
311 |       "Episode: 166, total_reward: 200.00\n",
312 |       "Episode: 167, total_reward: 200.00\n",
313 |       "Episode: 168, total_reward: 200.00\n",
314 |       "Episode: 169, total_reward: 200.00\n",
315 |       "Episode: 170, total_reward: 200.00\n",
316 |       "Episode: 171, total_reward: 200.00\n",
317 |       "Episode: 172, total_reward: 200.00\n",
318 |       "Episode: 173, total_reward: 200.00\n",
319 |       "Episode: 174, total_reward: 200.00\n",
320 |       "Episode: 175, total_reward: 200.00\n",
321 |       "Episode: 176, total_reward: 200.00\n",
322 |       "Episode: 177, total_reward: 200.00\n",
323 |       "Episode: 178, total_reward: 200.00\n",
324 |       "Episode: 179, total_reward: 200.00\n",
325 |       "Episode: 180, total_reward: 200.00\n",
326 |       "Episode: 181, total_reward: 200.00\n",
327 |       "Episode: 182, total_reward: 200.00\n",
328 |       "Episode: 183, total_reward: 174.00\n",
329 |       "Episode: 184, total_reward: 200.00\n",
330 |       "Episode: 185, total_reward: 200.00\n",
331 |       "Episode: 186, total_reward: 200.00\n",
332 |       "Episode: 187, total_reward: 200.00\n",
333 |       "Episode: 188, total_reward: 200.00\n",
334 |       "Episode: 189, total_reward: 200.00\n",
335 |       "Episode: 190, total_reward: 200.00\n",
336 |       "Episode: 191, total_reward: 200.00\n",
337 |       "Episode: 192, total_reward: 196.00\n",
338 |       "Episode: 193, total_reward: 200.00\n",
339 |       "Episode: 194, total_reward: 200.00\n",
340 |       "Episode: 195, total_reward: 200.00\n",
341 |       "Episode: 196, total_reward: 200.00\n",
342 |       "Episode: 197, total_reward: 200.00\n",
343 |       "Episode: 198, total_reward: 200.00\n",
344 |       "Episode: 199, total_reward: 200.00\n",
345 |       "Episode: 200, total_reward: 200.00\n",
346 |       "Episode: 201, total_reward: 200.00\n",
347 |       "Episode: 202, total_reward: 158.00\n",
348 |       "Episode: 203, total_reward: 200.00\n",
349 |       "Episode: 204, total_reward: 200.00\n",
350 |       "Episode: 205, total_reward: 198.00\n",
351 |       "Episode: 206, total_reward: 200.00\n",
352 |       "Episode: 207, total_reward: 200.00\n",
353 |       "Episode: 208, total_reward: 199.00\n",
354 |       "Episode: 209, total_reward: 200.00\n",
355 |       "Episode: 210, total_reward: 200.00\n",
356 |       "Episode: 211, total_reward: 200.00\n",
357 |       "Episode: 212, total_reward: 180.00\n",
358 |       "Episode: 213, total_reward: 164.00\n",
359 |       "Episode: 214, total_reward: 138.00\n",
360 |       "Episode: 215, total_reward: 177.00\n",
361 |       "Episode: 216, total_reward: 200.00\n",
362 |       "Episode: 217, total_reward: 200.00\n",
363 |       "Episode: 218, total_reward: 200.00\n",
364 |       "Episode: 219, total_reward: 200.00\n",
365 |       "Episode: 220, total_reward: 185.00\n",
366 |       "Episode: 221, total_reward: 196.00\n",
367 |       "Episode: 222, total_reward: 200.00\n",
368 |       "Episode: 223, total_reward: 200.00\n",
369 |       "Episode: 224, total_reward: 200.00\n",
370 |       "Episode: 225, total_reward: 200.00\n",
371 |       "Episode: 226, total_reward: 200.00\n",
372 |       "Episode: 227, total_reward: 200.00\n",
373 |       "Episode: 228, total_reward: 200.00\n",
374 |       "Episode: 229, total_reward: 200.00\n",
375 |       "Episode: 230, total_reward: 122.00\n",
376 |       "Episode: 231, total_reward: 193.00\n",
377 |       "Episode: 232, total_reward: 180.00\n",
378 |       "Episode: 233, total_reward: 200.00\n",
379 |       "Episode: 234, total_reward: 200.00\n",
380 |       "Episode: 235, total_reward: 200.00\n",
381 |       "Episode: 236, total_reward: 194.00\n",
382 |       "Episode: 237, total_reward: 200.00\n",
383 |       "Episode: 238, total_reward: 200.00\n",
384 |       "Episode: 239, total_reward: 200.00\n",
385 |       "Episode: 240, total_reward: 200.00\n"
386 |      ]
387 |     },
388 |     {
389 |      "name": "stdout",
390 |      "output_type": "stream",
391 |      "text": [
392 |       "Episode: 241, total_reward: 199.00\n",
393 |       "Episode: 242, total_reward: 200.00\n",
394 |       "Episode: 243, total_reward: 200.00\n",
395 |       "Episode: 244, total_reward: 200.00\n",
396 |       "Episode: 245, total_reward: 200.00\n",
397 |       "Episode: 246, total_reward: 200.00\n",
398 |       "Episode: 247, total_reward: 200.00\n",
399 |       "Episode: 248, total_reward: 200.00\n",
400 |       "Episode: 249, total_reward: 200.00\n",
401 |       "Episode: 250, total_reward: 200.00\n",
402 |       "Episode: 251, total_reward: 200.00\n",
403 |       "Episode: 252, total_reward: 200.00\n",
404 |       "Episode: 253, total_reward: 200.00\n",
405 |       "Episode: 254, total_reward: 200.00\n",
406 |       "Episode: 255, total_reward: 200.00\n",
407 |       "Episode: 256, total_reward: 200.00\n",
408 |       "Episode: 257, total_reward: 200.00\n",
409 |       "Episode: 258, total_reward: 200.00\n",
410 |       "Episode: 259, total_reward: 200.00\n",
411 |       "Episode: 260, total_reward: 200.00\n",
412 |       "Episode: 261, total_reward: 200.00\n",
413 |       "Episode: 262, total_reward: 200.00\n",
414 |       "Episode: 263, total_reward: 200.00\n",
415 |       "Episode: 264, total_reward: 200.00\n",
416 |       "Episode: 265, total_reward: 200.00\n",
417 |       "Episode: 266, total_reward: 170.00\n",
418 |       "Episode: 267, total_reward: 200.00\n",
419 |       "Episode: 268, total_reward: 181.00\n",
420 |       "Episode: 269, total_reward: 200.00\n",
421 |       "Episode: 270, total_reward: 200.00\n",
422 |       "Episode: 271, total_reward: 200.00\n",
423 |       "Episode: 272, total_reward: 200.00\n",
424 |       "Episode: 273, total_reward: 200.00\n",
425 |       "Episode: 274, total_reward: 200.00\n",
426 |       "Episode: 275, total_reward: 200.00\n",
427 |       "Episode: 276, total_reward: 200.00\n",
428 |       "Episode: 277, total_reward: 200.00\n",
429 |       "Episode: 278, total_reward: 200.00\n",
430 |       "Episode: 279, total_reward: 158.00\n",
431 |       "Episode: 280, total_reward: 200.00\n",
432 |       "Episode: 281, total_reward: 200.00\n",
433 |       "Episode: 282, total_reward: 200.00\n",
434 |       "Episode: 283, total_reward: 200.00\n",
435 |       "Episode: 284, total_reward: 200.00\n",
436 |       "Episode: 285, total_reward: 200.00\n",
437 |       "Episode: 286, total_reward: 198.00\n",
438 |       "Episode: 287, total_reward: 200.00\n",
439 |       "Episode: 288, total_reward: 200.00\n",
440 |       "Episode: 289, total_reward: 200.00\n",
441 |       "Episode: 290, total_reward: 200.00\n",
442 |       "Episode: 291, total_reward: 200.00\n",
443 |       "Episode: 292, total_reward: 200.00\n",
444 |       "Episode: 293, total_reward: 200.00\n",
445 |       "Episode: 294, total_reward: 200.00\n",
446 |       "Episode: 295, total_reward: 200.00\n",
447 |       "Episode: 296, total_reward: 200.00\n",
448 |       "Episode: 297, total_reward: 200.00\n",
449 |       "Episode: 298, total_reward: 200.00\n",
450 |       "Episode: 299, total_reward: 200.00\n",
451 |       "Episode: 300, total_reward: 158.00\n",
452 |       "Episode: 301, total_reward: 200.00\n",
453 |       "Episode: 302, total_reward: 200.00\n",
454 |       "Episode: 303, total_reward: 148.00\n",
455 |       "Episode: 304, total_reward: 200.00\n",
456 |       "Episode: 305, total_reward: 200.00\n",
457 |       "Episode: 306, total_reward: 200.00\n",
458 |       "Episode: 307, total_reward: 200.00\n",
459 |       "Episode: 308, total_reward: 200.00\n",
460 |       "Episode: 309, total_reward: 172.00\n",
461 |       "Episode: 310, total_reward: 200.00\n",
462 |       "Episode: 311, total_reward: 195.00\n",
463 |       "Episode: 312, total_reward: 200.00\n",
464 |       "Episode: 313, total_reward: 200.00\n",
465 |       "Episode: 314, total_reward: 200.00\n",
466 |       "Episode: 315, total_reward: 200.00\n",
467 |       "Episode: 316, total_reward: 200.00\n",
468 |       "Episode: 317, total_reward: 199.00\n",
469 |       "Episode: 318, total_reward: 200.00\n",
470 |       "Episode: 319, total_reward: 200.00\n",
471 |       "Episode: 320, total_reward: 200.00\n",
472 |       "Episode: 321, total_reward: 200.00\n",
473 |       "Episode: 322, total_reward: 187.00\n",
474 |       "Episode: 323, total_reward: 200.00\n",
475 |       "Episode: 324, total_reward: 200.00\n",
476 |       "Episode: 325, total_reward: 200.00\n",
477 |       "Episode: 326, total_reward: 200.00\n",
478 |       "Episode: 327, total_reward: 200.00\n",
479 |       "Episode: 328, total_reward: 187.00\n",
480 |       "Episode: 329, total_reward: 156.00\n",
481 |       "Episode: 330, total_reward: 200.00\n",
482 |       "Episode: 331, total_reward: 200.00\n",
483 |       "Episode: 332, total_reward: 200.00\n",
484 |       "Episode: 333, total_reward: 200.00\n",
485 |       "Episode: 334, total_reward: 200.00\n",
486 |       "Episode: 335, total_reward: 166.00\n",
487 |       "Episode: 336, total_reward: 200.00\n",
488 |       "Episode: 337, total_reward: 200.00\n",
489 |       "Episode: 338, total_reward: 145.00\n",
490 |       "Episode: 339, total_reward: 149.00\n",
491 |       "Episode: 340, total_reward: 200.00\n",
492 |       "Episode: 341, total_reward: 200.00\n",
493 |       "Episode: 342, total_reward: 164.00\n",
494 |       "Episode: 343, total_reward: 159.00\n",
495 |       "Episode: 344, total_reward: 178.00\n",
496 |       "Episode: 345, total_reward: 200.00\n",
497 |       "Episode: 346, total_reward: 200.00\n",
498 |       "Episode: 347, total_reward: 200.00\n",
499 |       "Episode: 348, total_reward: 200.00\n",
500 |       "Episode: 349, total_reward: 154.00\n",
501 |       "Episode: 350, total_reward: 188.00\n",
502 |       "Episode: 351, total_reward: 176.00\n",
503 |       "Episode: 352, total_reward: 200.00\n",
504 |       "Episode: 353, total_reward: 199.00\n",
505 |       "Episode: 354, total_reward: 200.00\n",
506 |       "Episode: 355, total_reward: 193.00\n",
507 |       "Episode: 356, total_reward: 200.00\n",
508 |       "Episode: 357, total_reward: 163.00\n",
509 |       "Episode: 358, total_reward: 200.00\n",
510 |       "Episode: 359, total_reward: 200.00\n",
511 |       "Episode: 360, total_reward: 200.00\n",
512 |       "Episode: 361, total_reward: 200.00\n",
513 |       "Episode: 362, total_reward: 191.00\n",
514 |       "Episode: 363, total_reward: 179.00\n",
515 |       "Episode: 364, total_reward: 175.00\n",
516 |       "Episode: 365, total_reward: 200.00\n",
517 |       "Episode: 366, total_reward: 146.00\n",
518 |       "Episode: 367, total_reward: 200.00\n",
519 |       "Episode: 368, total_reward: 200.00\n",
520 |       "Episode: 369, total_reward: 200.00\n",
521 |       "Episode: 370, total_reward: 200.00\n",
522 |       "Episode: 371, total_reward: 200.00\n",
523 |       "Episode: 372, total_reward: 159.00\n",
524 |       "Episode: 373, total_reward: 200.00\n",
525 |       "Episode: 374, total_reward: 200.00\n",
526 |       "Episode: 375, total_reward: 196.00\n",
527 |       "Episode: 376, total_reward: 200.00\n",
528 |       "Episode: 377, total_reward: 116.00\n",
529 |       "Episode: 378, total_reward: 156.00\n",
530 |       "Episode: 379, total_reward: 200.00\n",
531 |       "Episode: 380, total_reward: 200.00\n",
532 |       "Episode: 381, total_reward: 200.00\n",
533 |       "Episode: 382, total_reward: 200.00\n",
534 |       "Episode: 383, total_reward: 200.00\n",
535 |       "Episode: 384, total_reward: 200.00\n",
536 |       "Episode: 385, total_reward: 74.00\n",
537 |       "Episode: 386, total_reward: 200.00\n",
538 |       "Episode: 387, total_reward: 200.00\n",
539 |       "Episode: 388, total_reward: 200.00\n",
540 |       "Episode: 389, total_reward: 200.00\n",
541 |       "Episode: 390, total_reward: 200.00\n",
542 |       "Episode: 391, total_reward: 200.00\n",
543 |       "Episode: 392, total_reward: 200.00\n",
544 |       "Episode: 393, total_reward: 200.00\n",
545 |       "Episode: 394, total_reward: 200.00\n",
546 |       "Episode: 395, total_reward: 200.00\n",
547 |       "Episode: 396, total_reward: 200.00\n",
548 |       "Episode: 397, total_reward: 200.00\n",
549 |       "Episode: 398, total_reward: 200.00\n",
550 |       "Episode: 399, total_reward: 200.00\n"
551 |      ]
552 |     }
553 |    ],
554 |    "source": [
555 |     "agent = DQNAgent(env)\n",
556 |     "num_episodes = 400\n",
557 |     "\n",
558 |     "for ep in range(num_episodes):\n",
559 |     "    state = env.reset()\n",
560 |     "    total_reward = 0\n",
561 |     "    done = False\n",
562 |     "    while not done:\n",
563 |     "        action = agent.get_action(state)\n",
564 |     "        next_state, reward, done, info = env.step(action)\n",
565 |     "        agent.train(state, action, next_state, reward, done)\n",
566 |     "        env.render()\n",
567 |     "        total_reward += reward\n",
568 |     "        state = next_state\n",
569 |     "        \n",
570 |     "    print(\"Episode: {}, total_reward: {:.2f}\".format(ep, total_reward))"
571 |    ]
572 |   },
573 |   {
574 |    "cell_type": "code",
575 |    "execution_count": 15,
576 |    "metadata": {},
577 |    "outputs": [],
578 |    "source": [
579 |     "env.close()"
580 |    ]
581 |   }
582 |  ],
583 |  "metadata": {
584 |   "kernelspec": {
585 |    "display_name": "Python 3",
586 |    "language": "python",
587 |    "name": "python3"
588 |   },
589 |   "language_info": {
590 |    "codemirror_mode": {
591 |     "name": "ipython",
592 |     "version": 3
593 |    },
594 |    "file_extension": ".py",
595 |    "mimetype": "text/x-python",
596 |    "name": "python",
597 |    "nbconvert_exporter": "python",
598 |    "pygments_lexer": "ipython3",
599 |    "version": "3.6.5"
600 |   }
601 |  },
602 |  "nbformat": 4,
603 |  "nbformat_minor": 2
604 | }
605 | 


--------------------------------------------------------------------------------
/DiscreteVsContinuousEnvironments.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "code",
   5 |    "execution_count": 10,
   6 |    "metadata": {},
   7 |    "outputs": [],
   8 |    "source": [
   9 |     "import gym\n",
  10 |     "import random\n",
  11 |     "import numpy as np"
  12 |    ]
  13 |   },
  14 |   {
  15 |    "cell_type": "code",
  16 |    "execution_count": 22,
  17 |    "metadata": {},
  18 |    "outputs": [
  19 |     {
  20 |      "name": "stdout",
  21 |      "output_type": "stream",
  22 |      "text": [
  23 |       "Observation space: Discrete(16)\n",
  24 |       "Action space: Discrete(4)\n"
  25 |      ]
  26 |     },
  27 |     {
  28 |      "data": {
  29 |       "text/plain": [
  30 |        "gym.spaces.discrete.Discrete"
  31 |       ]
  32 |      },
  33 |      "execution_count": 22,
  34 |      "metadata": {},
  35 |      "output_type": "execute_result"
  36 |     }
  37 |    ],
  38 |    "source": [
  39 |     "env_name = \"CartPole-v1\"\n",
  40 |     "env_name = \"MountainCar-v0\"\n",
  41 |     "env_name = \"MountainCarContinuous-v0\"\n",
  42 |     "env_name = \"Acrobot-v1\"\n",
  43 |     "env_name = \"Pendulum-v0\"\n",
  44 |     "env_name = \"FrozenLake-v0\"\n",
  45 |     "env = gym.make(env_name)\n",
  46 |     "print(\"Observation space:\", env.observation_space)\n",
  47 |     "print(\"Action space:\", env.action_space)\n",
  48 |     "type(env.action_space)"
  49 |    ]
  50 |   },
  51 |   {
  52 |    "cell_type": "code",
  53 |    "execution_count": 23,
  54 |    "metadata": {},
  55 |    "outputs": [],
  56 |    "source": [
  57 |     "class Agent():\n",
  58 |     "    def __init__(self, env):\n",
  59 |     "        self.is_discrete = \\\n",
  60 |     "            type(env.action_space) == gym.spaces.discrete.Discrete\n",
  61 |     "        \n",
  62 |     "        if self.is_discrete:\n",
  63 |     "            self.action_size = env.action_space.n\n",
  64 |     "            print(\"Action size:\", self.action_size)\n",
  65 |     "        else:\n",
  66 |     "            self.action_low = env.action_space.low\n",
  67 |     "            self.action_high = env.action_space.high\n",
  68 |     "            self.action_shape = env.action_space.shape\n",
  69 |     "            print(\"Action range:\", self.action_low, self.action_high)\n",
  70 |     "        \n",
  71 |     "    def get_action(self, state):\n",
  72 |     "        if self.is_discrete:\n",
  73 |     "            action = random.choice(range(self.action_size))\n",
  74 |     "        else:\n",
  75 |     "            action = np.random.uniform(self.action_low,\n",
  76 |     "                                       self.action_high,\n",
  77 |     "                                       self.action_shape)\n",
  78 |     "        return action"
  79 |    ]
  80 |   },
  81 |   {
  82 |    "cell_type": "code",
  83 |    "execution_count": 24,
  84 |    "metadata": {},
  85 |    "outputs": [
  86 |     {
  87 |      "name": "stdout",
  88 |      "output_type": "stream",
  89 |      "text": [
  90 |       "Action size: 4\n",
  91 |       "  (Up)\n",
  92 |       "\u001b[41mS\u001b[0mFFF\n",
  93 |       "FHFH\n",
  94 |       "FFFH\n",
  95 |       "HFFG\n",
  96 |       "  (Up)\n",
  97 |       "\u001b[41mS\u001b[0mFFF\n",
  98 |       "FHFH\n",
  99 |       "FFFH\n",
 100 |       "HFFG\n",
 101 |       "  (Right)\n",
 102 |       "S\u001b[41mF\u001b[0mFF\n",
 103 |       "FHFH\n",
 104 |       "FFFH\n",
 105 |       "HFFG\n",
 106 |       "  (Down)\n",
 107 |       "SF\u001b[41mF\u001b[0mF\n",
 108 |       "FHFH\n",
 109 |       "FFFH\n",
 110 |       "HFFG\n",
 111 |       "  (Up)\n",
 112 |       "SF\u001b[41mF\u001b[0mF\n",
 113 |       "FHFH\n",
 114 |       "FFFH\n",
 115 |       "HFFG\n",
 116 |       "  (Left)\n",
 117 |       "SFFF\n",
 118 |       "FH\u001b[41mF\u001b[0mH\n",
 119 |       "FFFH\n",
 120 |       "HFFG\n",
 121 |       "  (Down)\n",
 122 |       "SFFF\n",
 123 |       "FHFH\n",
 124 |       "FF\u001b[41mF\u001b[0mH\n",
 125 |       "HFFG\n",
 126 |       "  (Right)\n",
 127 |       "SFFF\n",
 128 |       "FHFH\n",
 129 |       "FFF\u001b[41mH\u001b[0m\n",
 130 |       "HFFG\n",
 131 |       "  (Down)\n",
 132 |       "SFFF\n",
 133 |       "FHFH\n",
 134 |       "FFF\u001b[41mH\u001b[0m\n",
 135 |       "HFFG\n",
 136 |       "  (Down)\n",
 137 |       "SFFF\n",
 138 |       "FHFH\n",
 139 |       "FFF\u001b[41mH\u001b[0m\n",
 140 |       "HFFG\n",
 141 |       "  (Left)\n",
 142 |       "SFFF\n",
 143 |       "FHFH\n",
 144 |       "FFF\u001b[41mH\u001b[0m\n",
 145 |       "HFFG\n",
 146 |       "  (Up)\n",
 147 |       "SFFF\n",
 148 |       "FHFH\n",
 149 |       "FFF\u001b[41mH\u001b[0m\n",
 150 |       "HFFG\n",
 151 |       "  (Down)\n",
 152 |       "SFFF\n",
 153 |       "FHFH\n",
 154 |       "FFF\u001b[41mH\u001b[0m\n",
 155 |       "HFFG\n",
 156 |       "  (Left)\n",
 157 |       "SFFF\n",
 158 |       "FHFH\n",
 159 |       "FFF\u001b[41mH\u001b[0m\n",
 160 |       "HFFG\n",
 161 |       "  (Down)\n",
 162 |       "SFFF\n",
 163 |       "FHFH\n",
 164 |       "FFF\u001b[41mH\u001b[0m\n",
 165 |       "HFFG\n",
 166 |       "  (Left)\n",
 167 |       "SFFF\n",
 168 |       "FHFH\n",
 169 |       "FFF\u001b[41mH\u001b[0m\n",
 170 |       "HFFG\n",
 171 |       "  (Down)\n",
 172 |       "SFFF\n",
 173 |       "FHFH\n",
 174 |       "FFF\u001b[41mH\u001b[0m\n",
 175 |       "HFFG\n",
 176 |       "  (Left)\n",
 177 |       "SFFF\n",
 178 |       "FHFH\n",
 179 |       "FFF\u001b[41mH\u001b[0m\n",
 180 |       "HFFG\n",
 181 |       "  (Up)\n",
 182 |       "SFFF\n",
 183 |       "FHFH\n",
 184 |       "FFF\u001b[41mH\u001b[0m\n",
 185 |       "HFFG\n",
 186 |       "  (Right)\n",
 187 |       "SFFF\n",
 188 |       "FHFH\n",
 189 |       "FFF\u001b[41mH\u001b[0m\n",
 190 |       "HFFG\n",
 191 |       "  (Left)\n",
 192 |       "SFFF\n",
 193 |       "FHFH\n",
 194 |       "FFF\u001b[41mH\u001b[0m\n",
 195 |       "HFFG\n",
 196 |       "  (Down)\n",
 197 |       "SFFF\n",
 198 |       "FHFH\n",
 199 |       "FFF\u001b[41mH\u001b[0m\n",
 200 |       "HFFG\n",
 201 |       "  (Up)\n",
 202 |       "SFFF\n",
 203 |       "FHFH\n",
 204 |       "FFF\u001b[41mH\u001b[0m\n",
 205 |       "HFFG\n",
 206 |       "  (Left)\n",
 207 |       "SFFF\n",
 208 |       "FHFH\n",
 209 |       "FFF\u001b[41mH\u001b[0m\n",
 210 |       "HFFG\n",
 211 |       "  (Left)\n",
 212 |       "SFFF\n",
 213 |       "FHFH\n",
 214 |       "FFF\u001b[41mH\u001b[0m\n",
 215 |       "HFFG\n",
 216 |       "  (Down)\n",
 217 |       "SFFF\n",
 218 |       "FHFH\n",
 219 |       "FFF\u001b[41mH\u001b[0m\n",
 220 |       "HFFG\n",
 221 |       "  (Down)\n",
 222 |       "SFFF\n",
 223 |       "FHFH\n",
 224 |       "FFF\u001b[41mH\u001b[0m\n",
 225 |       "HFFG\n",
 226 |       "  (Up)\n",
 227 |       "SFFF\n",
 228 |       "FHFH\n",
 229 |       "FFF\u001b[41mH\u001b[0m\n",
 230 |       "HFFG\n",
 231 |       "  (Down)\n",
 232 |       "SFFF\n",
 233 |       "FHFH\n",
 234 |       "FFF\u001b[41mH\u001b[0m\n",
 235 |       "HFFG\n",
 236 |       "  (Down)\n",
 237 |       "SFFF\n",
 238 |       "FHFH\n",
 239 |       "FFF\u001b[41mH\u001b[0m\n",
 240 |       "HFFG\n",
 241 |       "  (Left)\n",
 242 |       "SFFF\n",
 243 |       "FHFH\n",
 244 |       "FFF\u001b[41mH\u001b[0m\n",
 245 |       "HFFG\n",
 246 |       "  (Up)\n",
 247 |       "SFFF\n",
 248 |       "FHFH\n",
 249 |       "FFF\u001b[41mH\u001b[0m\n",
 250 |       "HFFG\n",
 251 |       "  (Down)\n",
 252 |       "SFFF\n",
 253 |       "FHFH\n",
 254 |       "FFF\u001b[41mH\u001b[0m\n",
 255 |       "HFFG\n",
 256 |       "  (Right)\n",
 257 |       "SFFF\n",
 258 |       "FHFH\n",
 259 |       "FFF\u001b[41mH\u001b[0m\n",
 260 |       "HFFG\n",
 261 |       "  (Down)\n",
 262 |       "SFFF\n",
 263 |       "FHFH\n",
 264 |       "FFF\u001b[41mH\u001b[0m\n",
 265 |       "HFFG\n",
 266 |       "  (Right)\n",
 267 |       "SFFF\n",
 268 |       "FHFH\n",
 269 |       "FFF\u001b[41mH\u001b[0m\n",
 270 |       "HFFG\n",
 271 |       "  (Right)\n",
 272 |       "SFFF\n",
 273 |       "FHFH\n",
 274 |       "FFF\u001b[41mH\u001b[0m\n",
 275 |       "HFFG\n",
 276 |       "  (Right)\n",
 277 |       "SFFF\n",
 278 |       "FHFH\n",
 279 |       "FFF\u001b[41mH\u001b[0m\n",
 280 |       "HFFG\n",
 281 |       "  (Up)\n",
 282 |       "SFFF\n",
 283 |       "FHFH\n",
 284 |       "FFF\u001b[41mH\u001b[0m\n",
 285 |       "HFFG\n",
 286 |       "  (Left)\n",
 287 |       "SFFF\n",
 288 |       "FHFH\n",
 289 |       "FFF\u001b[41mH\u001b[0m\n",
 290 |       "HFFG\n",
 291 |       "  (Right)\n",
 292 |       "SFFF\n",
 293 |       "FHFH\n",
 294 |       "FFF\u001b[41mH\u001b[0m\n",
 295 |       "HFFG\n",
 296 |       "  (Left)\n",
 297 |       "SFFF\n",
 298 |       "FHFH\n",
 299 |       "FFF\u001b[41mH\u001b[0m\n",
 300 |       "HFFG\n",
 301 |       "  (Up)\n",
 302 |       "SFFF\n",
 303 |       "FHFH\n",
 304 |       "FFF\u001b[41mH\u001b[0m\n",
 305 |       "HFFG\n",
 306 |       "  (Left)\n",
 307 |       "SFFF\n",
 308 |       "FHFH\n",
 309 |       "FFF\u001b[41mH\u001b[0m\n",
 310 |       "HFFG\n",
 311 |       "  (Left)\n",
 312 |       "SFFF\n",
 313 |       "FHFH\n",
 314 |       "FFF\u001b[41mH\u001b[0m\n",
 315 |       "HFFG\n",
 316 |       "  (Right)\n",
 317 |       "SFFF\n",
 318 |       "FHFH\n",
 319 |       "FFF\u001b[41mH\u001b[0m\n",
 320 |       "HFFG\n",
 321 |       "  (Down)\n",
 322 |       "SFFF\n",
 323 |       "FHFH\n",
 324 |       "FFF\u001b[41mH\u001b[0m\n",
 325 |       "HFFG\n",
 326 |       "  (Left)\n",
 327 |       "SFFF\n",
 328 |       "FHFH\n",
 329 |       "FFF\u001b[41mH\u001b[0m\n",
 330 |       "HFFG\n",
 331 |       "  (Down)\n",
 332 |       "SFFF\n",
 333 |       "FHFH\n",
 334 |       "FFF\u001b[41mH\u001b[0m\n",
 335 |       "HFFG\n",
 336 |       "  (Left)\n",
 337 |       "SFFF\n",
 338 |       "FHFH\n",
 339 |       "FFF\u001b[41mH\u001b[0m\n",
 340 |       "HFFG\n",
 341 |       "  (Right)\n",
 342 |       "SFFF\n",
 343 |       "FHFH\n",
 344 |       "FFF\u001b[41mH\u001b[0m\n",
 345 |       "HFFG\n",
 346 |       "  (Left)\n",
 347 |       "SFFF\n",
 348 |       "FHFH\n",
 349 |       "FFF\u001b[41mH\u001b[0m\n",
 350 |       "HFFG\n",
 351 |       "  (Down)\n",
 352 |       "SFFF\n",
 353 |       "FHFH\n",
 354 |       "FFF\u001b[41mH\u001b[0m\n",
 355 |       "HFFG\n",
 356 |       "  (Left)\n",
 357 |       "SFFF\n",
 358 |       "FHFH\n",
 359 |       "FFF\u001b[41mH\u001b[0m\n",
 360 |       "HFFG\n",
 361 |       "  (Down)\n",
 362 |       "SFFF\n",
 363 |       "FHFH\n",
 364 |       "FFF\u001b[41mH\u001b[0m\n",
 365 |       "HFFG\n",
 366 |       "  (Left)\n",
 367 |       "SFFF\n",
 368 |       "FHFH\n",
 369 |       "FFF\u001b[41mH\u001b[0m\n",
 370 |       "HFFG\n",
 371 |       "  (Left)\n",
 372 |       "SFFF\n",
 373 |       "FHFH\n",
 374 |       "FFF\u001b[41mH\u001b[0m\n",
 375 |       "HFFG\n",
 376 |       "  (Down)\n",
 377 |       "SFFF\n",
 378 |       "FHFH\n",
 379 |       "FFF\u001b[41mH\u001b[0m\n",
 380 |       "HFFG\n",
 381 |       "  (Up)\n",
 382 |       "SFFF\n",
 383 |       "FHFH\n",
 384 |       "FFF\u001b[41mH\u001b[0m\n",
 385 |       "HFFG\n",
 386 |       "  (Left)\n",
 387 |       "SFFF\n",
 388 |       "FHFH\n",
 389 |       "FFF\u001b[41mH\u001b[0m\n",
 390 |       "HFFG\n",
 391 |       "  (Right)\n",
 392 |       "SFFF\n",
 393 |       "FHFH\n",
 394 |       "FFF\u001b[41mH\u001b[0m\n",
 395 |       "HFFG\n",
 396 |       "  (Right)\n",
 397 |       "SFFF\n",
 398 |       "FHFH\n",
 399 |       "FFF\u001b[41mH\u001b[0m\n",
 400 |       "HFFG\n",
 401 |       "  (Up)\n",
 402 |       "SFFF\n",
 403 |       "FHFH\n",
 404 |       "FFF\u001b[41mH\u001b[0m\n",
 405 |       "HFFG\n",
 406 |       "  (Left)\n",
 407 |       "SFFF\n",
 408 |       "FHFH\n",
 409 |       "FFF\u001b[41mH\u001b[0m\n",
 410 |       "HFFG\n",
 411 |       "  (Right)\n",
 412 |       "SFFF\n",
 413 |       "FHFH\n",
 414 |       "FFF\u001b[41mH\u001b[0m\n",
 415 |       "HFFG\n",
 416 |       "  (Up)\n",
 417 |       "SFFF\n",
 418 |       "FHFH\n",
 419 |       "FFF\u001b[41mH\u001b[0m\n",
 420 |       "HFFG\n",
 421 |       "  (Down)\n",
 422 |       "SFFF\n",
 423 |       "FHFH\n",
 424 |       "FFF\u001b[41mH\u001b[0m\n",
 425 |       "HFFG\n",
 426 |       "  (Right)\n",
 427 |       "SFFF\n",
 428 |       "FHFH\n",
 429 |       "FFF\u001b[41mH\u001b[0m\n",
 430 |       "HFFG\n",
 431 |       "  (Up)\n",
 432 |       "SFFF\n",
 433 |       "FHFH\n",
 434 |       "FFF\u001b[41mH\u001b[0m\n",
 435 |       "HFFG\n",
 436 |       "  (Down)\n",
 437 |       "SFFF\n",
 438 |       "FHFH\n",
 439 |       "FFF\u001b[41mH\u001b[0m\n",
 440 |       "HFFG\n",
 441 |       "  (Up)\n",
 442 |       "SFFF\n",
 443 |       "FHFH\n",
 444 |       "FFF\u001b[41mH\u001b[0m\n",
 445 |       "HFFG\n",
 446 |       "  (Right)\n",
 447 |       "SFFF\n",
 448 |       "FHFH\n",
 449 |       "FFF\u001b[41mH\u001b[0m\n",
 450 |       "HFFG\n",
 451 |       "  (Left)\n",
 452 |       "SFFF\n",
 453 |       "FHFH\n",
 454 |       "FFF\u001b[41mH\u001b[0m\n",
 455 |       "HFFG\n",
 456 |       "  (Up)\n",
 457 |       "SFFF\n",
 458 |       "FHFH\n",
 459 |       "FFF\u001b[41mH\u001b[0m\n",
 460 |       "HFFG\n",
 461 |       "  (Down)\n",
 462 |       "SFFF\n",
 463 |       "FHFH\n",
 464 |       "FFF\u001b[41mH\u001b[0m\n",
 465 |       "HFFG\n",
 466 |       "  (Down)\n",
 467 |       "SFFF\n",
 468 |       "FHFH\n",
 469 |       "FFF\u001b[41mH\u001b[0m\n",
 470 |       "HFFG\n",
 471 |       "  (Left)\n",
 472 |       "SFFF\n",
 473 |       "FHFH\n",
 474 |       "FFF\u001b[41mH\u001b[0m\n",
 475 |       "HFFG\n",
 476 |       "  (Down)\n",
 477 |       "SFFF\n",
 478 |       "FHFH\n",
 479 |       "FFF\u001b[41mH\u001b[0m\n",
 480 |       "HFFG\n",
 481 |       "  (Up)\n",
 482 |       "SFFF\n",
 483 |       "FHFH\n",
 484 |       "FFF\u001b[41mH\u001b[0m\n",
 485 |       "HFFG\n",
 486 |       "  (Right)\n",
 487 |       "SFFF\n",
 488 |       "FHFH\n",
 489 |       "FFF\u001b[41mH\u001b[0m\n",
 490 |       "HFFG\n",
 491 |       "  (Up)\n",
 492 |       "SFFF\n",
 493 |       "FHFH\n",
 494 |       "FFF\u001b[41mH\u001b[0m\n",
 495 |       "HFFG\n",
 496 |       "  (Up)\n",
 497 |       "SFFF\n",
 498 |       "FHFH\n",
 499 |       "FFF\u001b[41mH\u001b[0m\n",
 500 |       "HFFG\n",
 501 |       "  (Right)\n",
 502 |       "SFFF\n",
 503 |       "FHFH\n",
 504 |       "FFF\u001b[41mH\u001b[0m\n",
 505 |       "HFFG\n",
 506 |       "  (Left)\n",
 507 |       "SFFF\n",
 508 |       "FHFH\n",
 509 |       "FFF\u001b[41mH\u001b[0m\n",
 510 |       "HFFG\n",
 511 |       "  (Up)\n",
 512 |       "SFFF\n",
 513 |       "FHFH\n",
 514 |       "FFF\u001b[41mH\u001b[0m\n",
 515 |       "HFFG\n",
 516 |       "  (Left)\n",
 517 |       "SFFF\n",
 518 |       "FHFH\n",
 519 |       "FFF\u001b[41mH\u001b[0m\n",
 520 |       "HFFG\n",
 521 |       "  (Left)\n",
 522 |       "SFFF\n",
 523 |       "FHFH\n",
 524 |       "FFF\u001b[41mH\u001b[0m\n",
 525 |       "HFFG\n",
 526 |       "  (Up)\n",
 527 |       "SFFF\n",
 528 |       "FHFH\n",
 529 |       "FFF\u001b[41mH\u001b[0m\n",
 530 |       "HFFG\n",
 531 |       "  (Left)\n",
 532 |       "SFFF\n",
 533 |       "FHFH\n",
 534 |       "FFF\u001b[41mH\u001b[0m\n",
 535 |       "HFFG\n",
 536 |       "  (Down)\n",
 537 |       "SFFF\n",
 538 |       "FHFH\n",
 539 |       "FFF\u001b[41mH\u001b[0m\n",
 540 |       "HFFG\n",
 541 |       "  (Down)\n",
 542 |       "SFFF\n",
 543 |       "FHFH\n",
 544 |       "FFF\u001b[41mH\u001b[0m\n",
 545 |       "HFFG\n",
 546 |       "  (Left)\n",
 547 |       "SFFF\n",
 548 |       "FHFH\n",
 549 |       "FFF\u001b[41mH\u001b[0m\n",
 550 |       "HFFG\n",
 551 |       "  (Left)\n",
 552 |       "SFFF\n",
 553 |       "FHFH\n",
 554 |       "FFF\u001b[41mH\u001b[0m\n",
 555 |       "HFFG\n",
 556 |       "  (Right)\n",
 557 |       "SFFF\n",
 558 |       "FHFH\n",
 559 |       "FFF\u001b[41mH\u001b[0m\n",
 560 |       "HFFG\n",
 561 |       "  (Down)\n",
 562 |       "SFFF\n",
 563 |       "FHFH\n",
 564 |       "FFF\u001b[41mH\u001b[0m\n",
 565 |       "HFFG\n",
 566 |       "  (Right)\n",
 567 |       "SFFF\n",
 568 |       "FHFH\n",
 569 |       "FFF\u001b[41mH\u001b[0m\n",
 570 |       "HFFG\n",
 571 |       "  (Left)\n",
 572 |       "SFFF\n",
 573 |       "FHFH\n",
 574 |       "FFF\u001b[41mH\u001b[0m\n",
 575 |       "HFFG\n",
 576 |       "  (Right)\n",
 577 |       "SFFF\n",
 578 |       "FHFH\n",
 579 |       "FFF\u001b[41mH\u001b[0m\n",
 580 |       "HFFG\n",
 581 |       "  (Left)\n",
 582 |       "SFFF\n",
 583 |       "FHFH\n",
 584 |       "FFF\u001b[41mH\u001b[0m\n",
 585 |       "HFFG\n",
 586 |       "  (Down)\n",
 587 |       "SFFF\n",
 588 |       "FHFH\n",
 589 |       "FFF\u001b[41mH\u001b[0m\n",
 590 |       "HFFG\n",
 591 |       "  (Right)\n",
 592 |       "SFFF\n",
 593 |       "FHFH\n",
 594 |       "FFF\u001b[41mH\u001b[0m\n",
 595 |       "HFFG\n",
 596 |       "  (Up)\n",
 597 |       "SFFF\n",
 598 |       "FHFH\n",
 599 |       "FFF\u001b[41mH\u001b[0m\n",
 600 |       "HFFG\n",
 601 |       "  (Left)\n",
 602 |       "SFFF\n",
 603 |       "FHFH\n",
 604 |       "FFF\u001b[41mH\u001b[0m\n",
 605 |       "HFFG\n",
 606 |       "  (Left)\n",
 607 |       "SFFF\n",
 608 |       "FHFH\n",
 609 |       "FFF\u001b[41mH\u001b[0m\n",
 610 |       "HFFG\n",
 611 |       "  (Left)\n",
 612 |       "SFFF\n",
 613 |       "FHFH\n",
 614 |       "FFF\u001b[41mH\u001b[0m\n",
 615 |       "HFFG\n",
 616 |       "  (Left)\n",
 617 |       "SFFF\n",
 618 |       "FHFH\n",
 619 |       "FFF\u001b[41mH\u001b[0m\n",
 620 |       "HFFG\n",
 621 |       "  (Left)\n",
 622 |       "SFFF\n",
 623 |       "FHFH\n",
 624 |       "FFF\u001b[41mH\u001b[0m\n",
 625 |       "HFFG\n",
 626 |       "  (Left)\n",
 627 |       "SFFF\n",
 628 |       "FHFH\n",
 629 |       "FFF\u001b[41mH\u001b[0m\n",
 630 |       "HFFG\n",
 631 |       "  (Left)\n",
 632 |       "SFFF\n",
 633 |       "FHFH\n",
 634 |       "FFF\u001b[41mH\u001b[0m\n",
 635 |       "HFFG\n",
 636 |       "  (Left)\n",
 637 |       "SFFF\n",
 638 |       "FHFH\n",
 639 |       "FFF\u001b[41mH\u001b[0m\n",
 640 |       "HFFG\n",
 641 |       "  (Right)\n",
 642 |       "SFFF\n",
 643 |       "FHFH\n",
 644 |       "FFF\u001b[41mH\u001b[0m\n",
 645 |       "HFFG\n",
 646 |       "  (Down)\n",
 647 |       "SFFF\n",
 648 |       "FHFH\n",
 649 |       "FFF\u001b[41mH\u001b[0m\n",
 650 |       "HFFG\n",
 651 |       "  (Right)\n",
 652 |       "SFFF\n",
 653 |       "FHFH\n",
 654 |       "FFF\u001b[41mH\u001b[0m\n",
 655 |       "HFFG\n",
 656 |       "  (Down)\n",
 657 |       "SFFF\n",
 658 |       "FHFH\n",
 659 |       "FFF\u001b[41mH\u001b[0m\n",
 660 |       "HFFG\n",
 661 |       "  (Down)\n",
 662 |       "SFFF\n",
 663 |       "FHFH\n",
 664 |       "FFF\u001b[41mH\u001b[0m\n",
 665 |       "HFFG\n",
 666 |       "  (Up)\n",
 667 |       "SFFF\n",
 668 |       "FHFH\n",
 669 |       "FFF\u001b[41mH\u001b[0m\n",
 670 |       "HFFG\n",
 671 |       "  (Right)\n",
 672 |       "SFFF\n",
 673 |       "FHFH\n",
 674 |       "FFF\u001b[41mH\u001b[0m\n",
 675 |       "HFFG\n",
 676 |       "  (Up)\n",
 677 |       "SFFF\n",
 678 |       "FHFH\n",
 679 |       "FFF\u001b[41mH\u001b[0m\n",
 680 |       "HFFG\n",
 681 |       "  (Left)\n",
 682 |       "SFFF\n",
 683 |       "FHFH\n",
 684 |       "FFF\u001b[41mH\u001b[0m\n",
 685 |       "HFFG\n",
 686 |       "  (Up)\n",
 687 |       "SFFF\n",
 688 |       "FHFH\n",
 689 |       "FFF\u001b[41mH\u001b[0m\n",
 690 |       "HFFG\n",
 691 |       "  (Down)\n",
 692 |       "SFFF\n",
 693 |       "FHFH\n",
 694 |       "FFF\u001b[41mH\u001b[0m\n",
 695 |       "HFFG\n",
 696 |       "  (Right)\n",
 697 |       "SFFF\n",
 698 |       "FHFH\n",
 699 |       "FFF\u001b[41mH\u001b[0m\n",
 700 |       "HFFG\n",
 701 |       "  (Down)\n",
 702 |       "SFFF\n",
 703 |       "FHFH\n",
 704 |       "FFF\u001b[41mH\u001b[0m\n",
 705 |       "HFFG\n",
 706 |       "  (Up)\n",
 707 |       "SFFF\n",
 708 |       "FHFH\n",
 709 |       "FFF\u001b[41mH\u001b[0m\n",
 710 |       "HFFG\n",
 711 |       "  (Down)\n",
 712 |       "SFFF\n",
 713 |       "FHFH\n",
 714 |       "FFF\u001b[41mH\u001b[0m\n",
 715 |       "HFFG\n",
 716 |       "  (Right)\n",
 717 |       "SFFF\n",
 718 |       "FHFH\n",
 719 |       "FFF\u001b[41mH\u001b[0m\n",
 720 |       "HFFG\n",
 721 |       "  (Left)\n",
 722 |       "SFFF\n",
 723 |       "FHFH\n",
 724 |       "FFF\u001b[41mH\u001b[0m\n",
 725 |       "HFFG\n",
 726 |       "  (Down)\n",
 727 |       "SFFF\n",
 728 |       "FHFH\n",
 729 |       "FFF\u001b[41mH\u001b[0m\n",
 730 |       "HFFG\n",
 731 |       "  (Down)\n",
 732 |       "SFFF\n",
 733 |       "FHFH\n",
 734 |       "FFF\u001b[41mH\u001b[0m\n",
 735 |       "HFFG\n",
 736 |       "  (Left)\n",
 737 |       "SFFF\n",
 738 |       "FHFH\n",
 739 |       "FFF\u001b[41mH\u001b[0m\n",
 740 |       "HFFG\n",
 741 |       "  (Down)\n",
 742 |       "SFFF\n",
 743 |       "FHFH\n",
 744 |       "FFF\u001b[41mH\u001b[0m\n",
 745 |       "HFFG\n",
 746 |       "  (Right)\n",
 747 |       "SFFF\n",
 748 |       "FHFH\n",
 749 |       "FFF\u001b[41mH\u001b[0m\n",
 750 |       "HFFG\n",
 751 |       "  (Down)\n",
 752 |       "SFFF\n",
 753 |       "FHFH\n",
 754 |       "FFF\u001b[41mH\u001b[0m\n",
 755 |       "HFFG\n",
 756 |       "  (Right)\n",
 757 |       "SFFF\n",
 758 |       "FHFH\n",
 759 |       "FFF\u001b[41mH\u001b[0m\n",
 760 |       "HFFG\n",
 761 |       "  (Left)\n",
 762 |       "SFFF\n",
 763 |       "FHFH\n",
 764 |       "FFF\u001b[41mH\u001b[0m\n",
 765 |       "HFFG\n",
 766 |       "  (Up)\n",
 767 |       "SFFF\n",
 768 |       "FHFH\n",
 769 |       "FFF\u001b[41mH\u001b[0m\n",
 770 |       "HFFG\n",
 771 |       "  (Left)\n",
 772 |       "SFFF\n",
 773 |       "FHFH\n",
 774 |       "FFF\u001b[41mH\u001b[0m\n",
 775 |       "HFFG\n",
 776 |       "  (Left)\n",
 777 |       "SFFF\n",
 778 |       "FHFH\n",
 779 |       "FFF\u001b[41mH\u001b[0m\n",
 780 |       "HFFG\n",
 781 |       "  (Up)\n",
 782 |       "SFFF\n",
 783 |       "FHFH\n",
 784 |       "FFF\u001b[41mH\u001b[0m\n",
 785 |       "HFFG\n",
 786 |       "  (Up)\n",
 787 |       "SFFF\n",
 788 |       "FHFH\n",
 789 |       "FFF\u001b[41mH\u001b[0m\n",
 790 |       "HFFG\n",
 791 |       "  (Right)\n",
 792 |       "SFFF\n",
 793 |       "FHFH\n",
 794 |       "FFF\u001b[41mH\u001b[0m\n",
 795 |       "HFFG\n",
 796 |       "  (Left)\n",
 797 |       "SFFF\n",
 798 |       "FHFH\n",
 799 |       "FFF\u001b[41mH\u001b[0m\n",
 800 |       "HFFG\n",
 801 |       "  (Right)\n",
 802 |       "SFFF\n",
 803 |       "FHFH\n",
 804 |       "FFF\u001b[41mH\u001b[0m\n",
 805 |       "HFFG\n",
 806 |       "  (Up)\n",
 807 |       "SFFF\n",
 808 |       "FHFH\n",
 809 |       "FFF\u001b[41mH\u001b[0m\n",
 810 |       "HFFG\n",
 811 |       "  (Down)\n",
 812 |       "SFFF\n",
 813 |       "FHFH\n",
 814 |       "FFF\u001b[41mH\u001b[0m\n",
 815 |       "HFFG\n",
 816 |       "  (Up)\n",
 817 |       "SFFF\n",
 818 |       "FHFH\n",
 819 |       "FFF\u001b[41mH\u001b[0m\n",
 820 |       "HFFG\n",
 821 |       "  (Right)\n",
 822 |       "SFFF\n",
 823 |       "FHFH\n",
 824 |       "FFF\u001b[41mH\u001b[0m\n",
 825 |       "HFFG\n",
 826 |       "  (Right)\n",
 827 |       "SFFF\n",
 828 |       "FHFH\n",
 829 |       "FFF\u001b[41mH\u001b[0m\n",
 830 |       "HFFG\n",
 831 |       "  (Up)\n",
 832 |       "SFFF\n",
 833 |       "FHFH\n",
 834 |       "FFF\u001b[41mH\u001b[0m\n",
 835 |       "HFFG\n",
 836 |       "  (Right)\n",
 837 |       "SFFF\n",
 838 |       "FHFH\n",
 839 |       "FFF\u001b[41mH\u001b[0m\n",
 840 |       "HFFG\n",
 841 |       "  (Down)\n",
 842 |       "SFFF\n",
 843 |       "FHFH\n",
 844 |       "FFF\u001b[41mH\u001b[0m\n",
 845 |       "HFFG\n",
 846 |       "  (Up)\n",
 847 |       "SFFF\n",
 848 |       "FHFH\n",
 849 |       "FFF\u001b[41mH\u001b[0m\n",
 850 |       "HFFG\n",
 851 |       "  (Left)\n",
 852 |       "SFFF\n",
 853 |       "FHFH\n",
 854 |       "FFF\u001b[41mH\u001b[0m\n",
 855 |       "HFFG\n",
 856 |       "  (Left)\n",
 857 |       "SFFF\n",
 858 |       "FHFH\n",
 859 |       "FFF\u001b[41mH\u001b[0m\n",
 860 |       "HFFG\n",
 861 |       "  (Right)\n",
 862 |       "SFFF\n",
 863 |       "FHFH\n",
 864 |       "FFF\u001b[41mH\u001b[0m\n",
 865 |       "HFFG\n",
 866 |       "  (Down)\n",
 867 |       "SFFF\n",
 868 |       "FHFH\n",
 869 |       "FFF\u001b[41mH\u001b[0m\n",
 870 |       "HFFG\n",
 871 |       "  (Up)\n",
 872 |       "SFFF\n",
 873 |       "FHFH\n",
 874 |       "FFF\u001b[41mH\u001b[0m\n",
 875 |       "HFFG\n",
 876 |       "  (Right)\n",
 877 |       "SFFF\n",
 878 |       "FHFH\n",
 879 |       "FFF\u001b[41mH\u001b[0m\n",
 880 |       "HFFG\n",
 881 |       "  (Left)\n",
 882 |       "SFFF\n",
 883 |       "FHFH\n",
 884 |       "FFF\u001b[41mH\u001b[0m\n",
 885 |       "HFFG\n",
 886 |       "  (Left)\n",
 887 |       "SFFF\n",
 888 |       "FHFH\n",
 889 |       "FFF\u001b[41mH\u001b[0m\n",
 890 |       "HFFG\n",
 891 |       "  (Left)\n",
 892 |       "SFFF\n",
 893 |       "FHFH\n",
 894 |       "FFF\u001b[41mH\u001b[0m\n",
 895 |       "HFFG\n",
 896 |       "  (Down)\n",
 897 |       "SFFF\n",
 898 |       "FHFH\n",
 899 |       "FFF\u001b[41mH\u001b[0m\n",
 900 |       "HFFG\n",
 901 |       "  (Down)\n",
 902 |       "SFFF\n",
 903 |       "FHFH\n",
 904 |       "FFF\u001b[41mH\u001b[0m\n",
 905 |       "HFFG\n",
 906 |       "  (Up)\n",
 907 |       "SFFF\n",
 908 |       "FHFH\n",
 909 |       "FFF\u001b[41mH\u001b[0m\n",
 910 |       "HFFG\n",
 911 |       "  (Up)\n",
 912 |       "SFFF\n",
 913 |       "FHFH\n",
 914 |       "FFF\u001b[41mH\u001b[0m\n",
 915 |       "HFFG\n",
 916 |       "  (Right)\n",
 917 |       "SFFF\n",
 918 |       "FHFH\n",
 919 |       "FFF\u001b[41mH\u001b[0m\n",
 920 |       "HFFG\n",
 921 |       "  (Down)\n",
 922 |       "SFFF\n",
 923 |       "FHFH\n",
 924 |       "FFF\u001b[41mH\u001b[0m\n",
 925 |       "HFFG\n",
 926 |       "  (Up)\n",
 927 |       "SFFF\n",
 928 |       "FHFH\n",
 929 |       "FFF\u001b[41mH\u001b[0m\n",
 930 |       "HFFG\n",
 931 |       "  (Right)\n",
 932 |       "SFFF\n",
 933 |       "FHFH\n",
 934 |       "FFF\u001b[41mH\u001b[0m\n",
 935 |       "HFFG\n",
 936 |       "  (Down)\n",
 937 |       "SFFF\n",
 938 |       "FHFH\n",
 939 |       "FFF\u001b[41mH\u001b[0m\n",
 940 |       "HFFG\n",
 941 |       "  (Down)\n",
 942 |       "SFFF\n",
 943 |       "FHFH\n",
 944 |       "FFF\u001b[41mH\u001b[0m\n",
 945 |       "HFFG\n",
 946 |       "  (Left)\n",
 947 |       "SFFF\n",
 948 |       "FHFH\n",
 949 |       "FFF\u001b[41mH\u001b[0m\n",
 950 |       "HFFG\n",
 951 |       "  (Down)\n",
 952 |       "SFFF\n",
 953 |       "FHFH\n",
 954 |       "FFF\u001b[41mH\u001b[0m\n",
 955 |       "HFFG\n",
 956 |       "  (Left)\n",
 957 |       "SFFF\n",
 958 |       "FHFH\n",
 959 |       "FFF\u001b[41mH\u001b[0m\n",
 960 |       "HFFG\n",
 961 |       "  (Right)\n",
 962 |       "SFFF\n",
 963 |       "FHFH\n",
 964 |       "FFF\u001b[41mH\u001b[0m\n",
 965 |       "HFFG\n",
 966 |       "  (Up)\n",
 967 |       "SFFF\n",
 968 |       "FHFH\n",
 969 |       "FFF\u001b[41mH\u001b[0m\n",
 970 |       "HFFG\n",
 971 |       "  (Right)\n",
 972 |       "SFFF\n",
 973 |       "FHFH\n",
 974 |       "FFF\u001b[41mH\u001b[0m\n",
 975 |       "HFFG\n",
 976 |       "  (Down)\n",
 977 |       "SFFF\n",
 978 |       "FHFH\n",
 979 |       "FFF\u001b[41mH\u001b[0m\n",
 980 |       "HFFG\n",
 981 |       "  (Up)\n",
 982 |       "SFFF\n",
 983 |       "FHFH\n",
 984 |       "FFF\u001b[41mH\u001b[0m\n",
 985 |       "HFFG\n",
 986 |       "  (Right)\n",
 987 |       "SFFF\n",
 988 |       "FHFH\n",
 989 |       "FFF\u001b[41mH\u001b[0m\n",
 990 |       "HFFG\n",
 991 |       "  (Up)\n",
 992 |       "SFFF\n",
 993 |       "FHFH\n",
 994 |       "FFF\u001b[41mH\u001b[0m\n",
 995 |       "HFFG\n",
 996 |       "  (Right)\n",
 997 |       "SFFF\n",
 998 |       "FHFH\n",
 999 |       "FFF\u001b[41mH\u001b[0m\n",
1000 |       "HFFG\n",
1001 |       "  (Left)\n",
1002 |       "SFFF\n",
1003 |       "FHFH\n",
1004 |       "FFF\u001b[41mH\u001b[0m\n",
1005 |       "HFFG\n",
1006 |       "  (Right)\n",
1007 |       "SFFF\n",
1008 |       "FHFH\n",
1009 |       "FFF\u001b[41mH\u001b[0m\n",
1010 |       "HFFG\n",
1011 |       "  (Left)\n",
1012 |       "SFFF\n",
1013 |       "FHFH\n",
1014 |       "FFF\u001b[41mH\u001b[0m\n",
1015 |       "HFFG\n",
1016 |       "  (Right)\n",
1017 |       "SFFF\n",
1018 |       "FHFH\n",
1019 |       "FFF\u001b[41mH\u001b[0m\n",
1020 |       "HFFG\n",
1021 |       "  (Right)\n",
1022 |       "SFFF\n",
1023 |       "FHFH\n",
1024 |       "FFF\u001b[41mH\u001b[0m\n",
1025 |       "HFFG\n",
1026 |       "  (Down)\n",
1027 |       "SFFF\n",
1028 |       "FHFH\n",
1029 |       "FFF\u001b[41mH\u001b[0m\n",
1030 |       "HFFG\n",
1031 |       "  (Down)\n",
1032 |       "SFFF\n",
1033 |       "FHFH\n",
1034 |       "FFF\u001b[41mH\u001b[0m\n",
1035 |       "HFFG\n",
1036 |       "  (Down)\n",
1037 |       "SFFF\n",
1038 |       "FHFH\n",
1039 |       "FFF\u001b[41mH\u001b[0m\n",
1040 |       "HFFG\n",
1041 |       "  (Right)\n",
1042 |       "SFFF\n",
1043 |       "FHFH\n",
1044 |       "FFF\u001b[41mH\u001b[0m\n",
1045 |       "HFFG\n",
1046 |       "  (Left)\n",
1047 |       "SFFF\n",
1048 |       "FHFH\n",
1049 |       "FFF\u001b[41mH\u001b[0m\n",
1050 |       "HFFG\n",
1051 |       "  (Right)\n",
1052 |       "SFFF\n",
1053 |       "FHFH\n",
1054 |       "FFF\u001b[41mH\u001b[0m\n",
1055 |       "HFFG\n",
1056 |       "  (Right)\n",
1057 |       "SFFF\n",
1058 |       "FHFH\n",
1059 |       "FFF\u001b[41mH\u001b[0m\n",
1060 |       "HFFG\n",
1061 |       "  (Left)\n",
1062 |       "SFFF\n",
1063 |       "FHFH\n",
1064 |       "FFF\u001b[41mH\u001b[0m\n",
1065 |       "HFFG\n",
1066 |       "  (Down)\n",
1067 |       "SFFF\n",
1068 |       "FHFH\n",
1069 |       "FFF\u001b[41mH\u001b[0m\n",
1070 |       "HFFG\n",
1071 |       "  (Down)\n",
1072 |       "SFFF\n",
1073 |       "FHFH\n",
1074 |       "FFF\u001b[41mH\u001b[0m\n",
1075 |       "HFFG\n",
1076 |       "  (Up)\n",
1077 |       "SFFF\n",
1078 |       "FHFH\n",
1079 |       "FFF\u001b[41mH\u001b[0m\n",
1080 |       "HFFG\n",
1081 |       "  (Up)\n",
1082 |       "SFFF\n",
1083 |       "FHFH\n",
1084 |       "FFF\u001b[41mH\u001b[0m\n",
1085 |       "HFFG\n",
1086 |       "  (Right)\n",
1087 |       "SFFF\n",
1088 |       "FHFH\n",
1089 |       "FFF\u001b[41mH\u001b[0m\n",
1090 |       "HFFG\n"
1091 |      ]
1092 |     }
1093 |    ],
1094 |    "source": [
1095 |     "agent = Agent(env)\n",
1096 |     "state = env.reset()\n",
1097 |     "\n",
1098 |     "for _ in range(200):\n",
1099 |     "#     action = env.action_space.sample()\n",
1100 |     "    action = agent.get_action(state)\n",
1101 |     "    state, reward, done, info = env.step(action)\n",
1102 |     "    env.render()"
1103 |    ]
1104 |   },
1105 |   {
1106 |    "cell_type": "code",
1107 |    "execution_count": null,
1108 |    "metadata": {},
1109 |    "outputs": [],
1110 |    "source": []
1111 |   }
1112 |  ],
1113 |  "metadata": {
1114 |   "kernelspec": {
1115 |    "display_name": "Python 3",
1116 |    "language": "python",
1117 |    "name": "python3"
1118 |   },
1119 |   "language_info": {
1120 |    "codemirror_mode": {
1121 |     "name": "ipython",
1122 |     "version": 3
1123 |    },
1124 |    "file_extension": ".py",
1125 |    "mimetype": "text/x-python",
1126 |    "name": "python",
1127 |    "nbconvert_exporter": "python",
1128 |    "pygments_lexer": "ipython3",
1129 |    "version": "3.6.5"
1130 |   }
1131 |  },
1132 |  "nbformat": 4,
1133 |  "nbformat_minor": 2
1134 | }
1135 | 


--------------------------------------------------------------------------------
/DiscretizedDQNsForContinuousControl.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 8,
  6 |    "metadata": {
  7 |     "colab": {},
  8 |     "colab_type": "code",
  9 |     "id": "-578TXDLZF47"
 10 |    },
 11 |    "outputs": [],
 12 |    "source": [
 13 |     "# Run this to install tensorflow version 1.9\n",
 14 |     "# !pip install tensorflow==1.9"
 15 |    ]
 16 |   },
 17 |   {
 18 |    "cell_type": "code",
 19 |    "execution_count": 9,
 20 |    "metadata": {
 21 |     "colab": {
 22 |      "base_uri": "https://localhost:8080/",
 23 |      "height": 51
 24 |     },
 25 |     "colab_type": "code",
 26 |     "id": "HnA-Zf8JqsL4",
 27 |     "outputId": "70062eee-0733-4f72-b717-2aeec3d927b9"
 28 |    },
 29 |    "outputs": [
 30 |     {
 31 |      "name": "stdout",
 32 |      "output_type": "stream",
 33 |      "text": [
 34 |       "Gym: 0.10.11\n",
 35 |       "Tensorflow: 1.9.0\n"
 36 |      ]
 37 |     }
 38 |    ],
 39 |    "source": [
 40 |     "import gym\n",
 41 |     "import random\n",
 42 |     "import numpy as np\n",
 43 |     "import tensorflow as tf\n",
 44 |     "from collections import deque\n",
 45 |     "print(\"Gym:\", gym.__version__)\n",
 46 |     "print(\"Tensorflow:\", tf.__version__)"
 47 |    ]
 48 |   },
 49 |   {
 50 |    "cell_type": "code",
 51 |    "execution_count": 11,
 52 |    "metadata": {
 53 |     "colab": {},
 54 |     "colab_type": "code",
 55 |     "id": "79GCjcYYqsMG"
 56 |    },
 57 |    "outputs": [],
 58 |    "source": [
 59 |     "class QNetwork():\n",
 60 |     "    def __init__(self, state_dim, action_size, tau=0.01):\n",
 61 |     "        tf.reset_default_graph()\n",
 62 |     "        self.state_in = tf.placeholder(tf.float32, shape=[None, *state_dim])\n",
 63 |     "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
 64 |     "        self.q_target_in = tf.placeholder(tf.float32, shape=[None])\n",
 65 |     "        self.importance_in = tf.placeholder(tf.float32, shape=[None])\n",
 66 |     "        action_one_hot = tf.one_hot(self.action_in, depth=action_size)\n",
 67 |     "        \n",
 68 |     "        self.q_state_local = self.build_model(action_size, \"local\")\n",
 69 |     "        self.q_state_target = self.build_model(action_size, \"target\")\n",
 70 |     "        \n",
 71 |     "        self.q_state_action = tf.reduce_sum(tf.multiply(self.q_state_local, action_one_hot), axis=1)\n",
 72 |     "        self.error = self.q_state_action - self.q_target_in\n",
 73 |     "        self.loss = tf.reduce_mean(tf.multiply(tf.square(self.error), self.importance_in))\n",
 74 |     "        self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(self.loss)\n",
 75 |     "        \n",
 76 |     "        self.local_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=\"local\")\n",
 77 |     "        self.target_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=\"target\")\n",
 78 |     "        self.updater = tf.group([tf.assign(t, t + tau*(l-t)) for t,l in zip(self.target_vars, self.local_vars)])\n",
 79 |     "        \n",
 80 |     "    def build_model(self, action_size, scope):\n",
 81 |     "        with tf.variable_scope(scope):\n",
 82 |     "            hidden1 = tf.layers.dense(self.state_in, 100, activation=tf.nn.relu)\n",
 83 |     "            q_state = tf.layers.dense(hidden1, action_size, activation=None)\n",
 84 |     "            return q_state\n",
 85 |     "        \n",
 86 |     "    def update_model(self, session, state, action, q_target, importance):\n",
 87 |     "        feed = {self.state_in: state, self.action_in: action, self.q_target_in: q_target, self.importance_in: importance}\n",
 88 |     "        error, _, _ = session.run([self.error, self.optimizer, self.updater], feed_dict=feed)\n",
 89 |     "        return error\n",
 90 |     "        \n",
 91 |     "    def get_q_state(self, session, state, use_target=False):\n",
 92 |     "        q_state_op = self.q_state_target if use_target else self.q_state_local\n",
 93 |     "        q_state = session.run(q_state_op, feed_dict={self.state_in: state})\n",
 94 |     "        return q_state"
 95 |    ]
 96 |   },
 97 |   {
 98 |    "cell_type": "code",
 99 |    "execution_count": 12,
100 |    "metadata": {
101 |     "colab": {},
102 |     "colab_type": "code",
103 |     "id": "EYKcRk3ZqsMJ"
104 |    },
105 |    "outputs": [],
106 |    "source": [
107 |     "class PrioritizedReplayBuffer():\n",
108 |     "    def __init__(self, maxlen):\n",
109 |     "        self.buffer = deque(maxlen=maxlen)\n",
110 |     "        self.priorities = deque(maxlen=maxlen)\n",
111 |     "        \n",
112 |     "    def add(self, experience):\n",
113 |     "        self.buffer.append(experience)\n",
114 |     "        self.priorities.append(max(self.priorities, default=1))\n",
115 |     "        \n",
116 |     "    def get_probabilities(self, priority_scale):\n",
117 |     "        scaled_priorities = np.array(self.priorities) ** priority_scale\n",
118 |     "        sample_probabilities = scaled_priorities / sum(scaled_priorities)\n",
119 |     "        return sample_probabilities\n",
120 |     "    \n",
121 |     "    def get_importance(self, probabilities):\n",
122 |     "        importance = 1/len(self.buffer) * 1/probabilities\n",
123 |     "        importance_normalized = importance / max(importance)\n",
124 |     "        return importance_normalized\n",
125 |     "        \n",
126 |     "    def sample(self, batch_size, priority_scale=1.0):\n",
127 |     "        sample_size = min(len(self.buffer), batch_size)\n",
128 |     "        sample_probs = self.get_probabilities(priority_scale)\n",
129 |     "        sample_indices = random.choices(range(len(self.buffer)), k=sample_size, weights=sample_probs)\n",
130 |     "        samples = np.array(self.buffer)[sample_indices]\n",
131 |     "        importance = self.get_importance(sample_probs[sample_indices])\n",
132 |     "        return map(list, zip(*samples)), importance, sample_indices\n",
133 |     "    \n",
134 |     "    def set_priorities(self, indices, errors, offset=0.1):\n",
135 |     "        for i,e in zip(indices, errors):\n",
136 |     "            self.priorities[i] = abs(e) + offset"
137 |    ]
138 |   },
139 |   {
140 |    "cell_type": "code",
141 |    "execution_count": 14,
142 |    "metadata": {
143 |     "colab": {},
144 |     "colab_type": "code",
145 |     "id": "XKIXlcQSqsMN"
146 |    },
147 |    "outputs": [],
148 |    "source": [
149 |     "class DoubleDQNAgent():\n",
150 |     "    def __init__(self, env):\n",
151 |     "        self.state_dim = env.observation_space.shape\n",
152 |     "        self.action_size = env.action_space.n\n",
153 |     "        self.q_network = QNetwork(self.state_dim, self.action_size)\n",
154 |     "        self.replay_buffer = PrioritizedReplayBuffer(maxlen=100000)\n",
155 |     "        self.gamma = 0.97\n",
156 |     "        self.eps = 1.0\n",
157 |     "        \n",
158 |     "        self.sess = tf.Session()\n",
159 |     "        self.sess.run(tf.global_variables_initializer())\n",
160 |     "        \n",
161 |     "    def get_action(self, state):\n",
162 |     "        q_state = self.q_network.get_q_state(self.sess, [state])\n",
163 |     "        action_greedy = np.argmax(q_state)\n",
164 |     "        action_random = np.random.randint(self.action_size)\n",
165 |     "        action = action_random if random.random() < self.eps else action_greedy\n",
166 |     "        return action\n",
167 |     "    \n",
168 |     "    def get_env_action(self, action):\n",
169 |     "        return action\n",
170 |     "    \n",
171 |     "    def train(self, state, action, next_state, reward, done, use_DDQN=True, a=0.0):\n",
172 |     "        self.replay_buffer.add((state, action, next_state, reward, done))\n",
173 |     "        (states, actions, next_states, rewards, dones), importance, indices = self.replay_buffer.sample(50, priority_scale=a)\n",
174 |     "        \n",
175 |     "        next_actions = np.argmax(self.q_network.get_q_state(self.sess, next_states, use_target=False), axis=1)\n",
176 |     "        q_next_states = self.q_network.get_q_state(self.sess, next_states, use_target=use_DDQN)\n",
177 |     "        q_next_states[dones] = np.zeros([self.action_size])\n",
178 |     "        q_next_states_next_actions = q_next_states[np.arange(next_actions.shape[0]), next_actions]\n",
179 |     "        q_targets = rewards + self.gamma * q_next_states_next_actions\n",
180 |     "        errors = self.q_network.update_model(self.sess, states, actions, q_targets, importance**(1-self.eps))\n",
181 |     "        \n",
182 |     "        self.replay_buffer.set_priorities(indices, errors)\n",
183 |     "        \n",
184 |     "        if done: self.eps = max(0.1, 0.98*self.eps)\n",
185 |     "    \n",
186 |     "    def __del__(self):\n",
187 |     "        self.sess.close()"
188 |    ]
189 |   },
190 |   {
191 |    "cell_type": "code",
192 |    "execution_count": 22,
193 |    "metadata": {},
194 |    "outputs": [],
195 |    "source": [
196 |     "class DiscretizedDQNAgent(DoubleDQNAgent):\n",
197 |     "    def __init__(self, env, n_actions=10):\n",
198 |     "        self.is_discrete = type(env.action_space) == gym.spaces.discrete.Discrete\n",
199 |     "        if not self.is_discrete:\n",
200 |     "            env.action_space.n = n_actions\n",
201 |     "            self.actions = np.linspace(env.action_space.low, env.action_space.high, n_actions)\n",
202 |     "        super().__init__(env)\n",
203 |     "        \n",
204 |     "    def get_env_action(self, action):\n",
205 |     "        if not self.is_discrete:\n",
206 |     "            action = [self.actions[action]]\n",
207 |     "        return action"
208 |    ]
209 |   },
210 |   {
211 |    "cell_type": "code",
212 |    "execution_count": 30,
213 |    "metadata": {
214 |     "colab": {
215 |      "base_uri": "https://localhost:8080/",
216 |      "height": 51
217 |     },
218 |     "colab_type": "code",
219 |     "id": "YHkYC1tmqsMB",
220 |     "outputId": "c3abd168-27fc-4f2c-be25-18194375bd0d"
221 |    },
222 |    "outputs": [
223 |     {
224 |      "name": "stdout",
225 |      "output_type": "stream",
226 |      "text": [
227 |       "Observation space: Box(6,)\n",
228 |       "Action space: Discrete(3)\n"
229 |      ]
230 |     }
231 |    ],
232 |    "source": [
233 |     "env_names = [\"CartPole-v0\",\n",
234 |     "             \"MountainCar-v0\",\n",
235 |     "             \"MountainCarContinuous-v0\",\n",
236 |     "             \"Pendulum-v0\",\n",
237 |     "             \"Acrobot-v1\"]\n",
238 |     "env = gym.make(env_names[4])\n",
239 |     "print(\"Observation space:\", env.observation_space)\n",
240 |     "print(\"Action space:\", env.action_space)"
241 |    ]
242 |   },
243 |   {
244 |    "cell_type": "code",
245 |    "execution_count": 33,
246 |    "metadata": {
247 |     "colab": {
248 |      "base_uri": "https://localhost:8080/",
249 |      "height": 187
250 |     },
251 |     "colab_type": "code",
252 |     "id": "zv9Amjj4qsMQ",
253 |     "outputId": "3e61344f-d87a-41e6-d28c-4ca3511ef4df"
254 |    },
255 |    "outputs": [
256 |     {
257 |      "name": "stdout",
258 |      "output_type": "stream",
259 |      "text": [
260 |       "Run 0\n",
261 |       "Episode: 0, total_reward: -500.00\n",
262 |       "Episode: 1, total_reward: -500.00\n",
263 |       "Episode: 2, total_reward: -500.00\n",
264 |       "Episode: 3, total_reward: -500.00\n",
265 |       "Episode: 4, total_reward: -500.00\n",
266 |       "Episode: 5, total_reward: -500.00\n",
267 |       "Episode: 6, total_reward: -472.00\n",
268 |       "Episode: 7, total_reward: -500.00\n",
269 |       "Episode: 8, total_reward: -500.00\n",
270 |       "Episode: 9, total_reward: -500.00\n",
271 |       "Episode: 10, total_reward: -500.00\n",
272 |       "Episode: 11, total_reward: -500.00\n",
273 |       "Episode: 12, total_reward: -446.00\n",
274 |       "Episode: 13, total_reward: -500.00\n",
275 |       "Episode: 14, total_reward: -500.00\n",
276 |       "Episode: 15, total_reward: -367.00\n",
277 |       "Episode: 16, total_reward: -500.00\n",
278 |       "Episode: 17, total_reward: -469.00\n",
279 |       "Episode: 18, total_reward: -434.00\n",
280 |       "Episode: 19, total_reward: -500.00\n",
281 |       "Episode: 20, total_reward: -500.00\n",
282 |       "Episode: 21, total_reward: -500.00\n",
283 |       "Episode: 22, total_reward: -322.00\n",
284 |       "Episode: 23, total_reward: -171.00\n",
285 |       "Episode: 24, total_reward: -294.00\n",
286 |       "Episode: 25, total_reward: -308.00\n",
287 |       "Episode: 26, total_reward: -322.00\n",
288 |       "Episode: 27, total_reward: -384.00\n",
289 |       "Episode: 28, total_reward: -375.00\n",
290 |       "Episode: 29, total_reward: -422.00\n",
291 |       "Episode: 30, total_reward: -373.00\n",
292 |       "Episode: 31, total_reward: -500.00\n",
293 |       "Episode: 32, total_reward: -256.00\n",
294 |       "Episode: 33, total_reward: -293.00\n",
295 |       "Episode: 34, total_reward: -243.00\n",
296 |       "Episode: 35, total_reward: -219.00\n",
297 |       "Episode: 36, total_reward: -210.00\n",
298 |       "Episode: 37, total_reward: -156.00\n",
299 |       "Episode: 38, total_reward: -186.00\n",
300 |       "Episode: 39, total_reward: -500.00\n",
301 |       "Episode: 40, total_reward: -261.00\n",
302 |       "Episode: 41, total_reward: -188.00\n",
303 |       "Episode: 42, total_reward: -242.00\n",
304 |       "Episode: 43, total_reward: -234.00\n",
305 |       "Episode: 44, total_reward: -198.00\n",
306 |       "Episode: 45, total_reward: -207.00\n",
307 |       "Episode: 46, total_reward: -137.00\n",
308 |       "Episode: 47, total_reward: -140.00\n",
309 |       "Episode: 48, total_reward: -130.00\n",
310 |       "Episode: 49, total_reward: -135.00\n",
311 |       "Episode: 50, total_reward: -199.00\n",
312 |       "Episode: 51, total_reward: -132.00\n",
313 |       "Episode: 52, total_reward: -456.00\n",
314 |       "Episode: 53, total_reward: -244.00\n",
315 |       "Episode: 54, total_reward: -123.00\n",
316 |       "Episode: 55, total_reward: -165.00\n",
317 |       "Episode: 56, total_reward: -257.00\n",
318 |       "Episode: 57, total_reward: -156.00\n",
319 |       "Episode: 58, total_reward: -148.00\n",
320 |       "Episode: 59, total_reward: -132.00\n",
321 |       "Episode: 60, total_reward: -155.00\n",
322 |       "Episode: 61, total_reward: -160.00\n",
323 |       "Episode: 62, total_reward: -94.00\n",
324 |       "Episode: 63, total_reward: -147.00\n",
325 |       "Episode: 64, total_reward: -100.00\n",
326 |       "Episode: 65, total_reward: -109.00\n",
327 |       "Episode: 66, total_reward: -161.00\n",
328 |       "Episode: 67, total_reward: -149.00\n",
329 |       "Episode: 68, total_reward: -105.00\n",
330 |       "Episode: 69, total_reward: -101.00\n",
331 |       "Episode: 70, total_reward: -118.00\n",
332 |       "Episode: 71, total_reward: -124.00\n",
333 |       "Episode: 72, total_reward: -157.00\n",
334 |       "Episode: 73, total_reward: -127.00\n",
335 |       "Episode: 74, total_reward: -178.00\n",
336 |       "Episode: 75, total_reward: -119.00\n",
337 |       "Episode: 76, total_reward: -144.00\n",
338 |       "Episode: 77, total_reward: -145.00\n",
339 |       "Episode: 78, total_reward: -133.00\n",
340 |       "Episode: 79, total_reward: -138.00\n",
341 |       "Episode: 80, total_reward: -151.00\n",
342 |       "Episode: 81, total_reward: -98.00\n",
343 |       "Episode: 82, total_reward: -108.00\n",
344 |       "Episode: 83, total_reward: -110.00\n",
345 |       "Episode: 84, total_reward: -128.00\n",
346 |       "Episode: 85, total_reward: -156.00\n",
347 |       "Episode: 86, total_reward: -105.00\n",
348 |       "Episode: 87, total_reward: -152.00\n",
349 |       "Episode: 88, total_reward: -88.00\n",
350 |       "Episode: 89, total_reward: -122.00\n",
351 |       "Episode: 90, total_reward: -180.00\n",
352 |       "Episode: 91, total_reward: -102.00\n",
353 |       "Episode: 92, total_reward: -130.00\n",
354 |       "Episode: 93, total_reward: -101.00\n",
355 |       "Episode: 94, total_reward: -83.00\n",
356 |       "Episode: 95, total_reward: -115.00\n",
357 |       "Episode: 96, total_reward: -103.00\n",
358 |       "Episode: 97, total_reward: -124.00\n",
359 |       "Episode: 98, total_reward: -146.00\n",
360 |       "Episode: 99, total_reward: -100.00\n"
361 |      ]
362 |     }
363 |    ],
364 |    "source": [
365 |     "num_runs = 1\n",
366 |     "run_rewards = []\n",
367 |     "\n",
368 |     "for n in range(num_runs):\n",
369 |     "    print(\"Run {}\".format(n))\n",
370 |     "    ep_rewards = []\n",
371 |     "    agent = None\n",
372 |     "    agent = DiscretizedDQNAgent(env)\n",
373 |     "    num_episodes = 100\n",
374 |     "\n",
375 |     "    for ep in range(num_episodes):\n",
376 |     "        state = env.reset()\n",
377 |     "        total_reward = 0\n",
378 |     "        done = False\n",
379 |     "        while not done:\n",
380 |     "            action = agent.get_action(state)\n",
381 |     "            next_state, reward, done, info = env.step(agent.get_env_action(action))\n",
382 |     "            agent.train(state, action, next_state, reward, done, a=0.7)\n",
383 |     "            env.render()\n",
384 |     "            total_reward += reward\n",
385 |     "            state = next_state\n",
386 |     "\n",
387 |     "        ep_rewards.append(total_reward)\n",
388 |     "        print(\"Episode: {}, total_reward: {:.2f}\".format(ep, total_reward))\n",
389 |     "        \n",
390 |     "    run_rewards.append(ep_rewards)"
391 |    ]
392 |   },
393 |   {
394 |    "cell_type": "code",
395 |    "execution_count": 34,
396 |    "metadata": {
397 |     "colab": {
398 |      "base_uri": "https://localhost:8080/",
399 |      "height": 311
400 |     },
401 |     "colab_type": "code",
402 |     "id": "JU1OmrO0uHpO",
403 |     "outputId": "d8cb4c76-6112-4327-f334-6e9669e0b990"
404 |    },
405 |    "outputs": [
406 |     {
407 |      "data": {
408 |       "text/plain": [
409 |        "<matplotlib.legend.Legend at 0xb249ce860>"
410 |       ]
411 |      },
412 |      "execution_count": 34,
413 |      "metadata": {},
414 |      "output_type": "execute_result"
415 |     },
416 |     {
417 |      "data": {
418 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XecXHW5+PHPM312Z1u2pGzKhhAgBDDAQqiCgFdABEEgwStdEAULqFdUkKjAT+zee5WmdBAQRLoULyDS0wyhB5KQTU+27+z05/fHObOZ3WyZJDu72eR5v17nNWe+pz0zyX6f+Z7v95wjqooxxhgzmDzDHYAxxpgdjyUXY4wxg86SizHGmEFnycUYY8ygs+RijDFm0FlyMcYYM+gsuZgdgogcLiLvDXccI5GIhEXkURFpEZG/DHc8ZsdgycVsMxF5XkRiItLuTn1W8iIyR0SSOeu2i0jztsagqi+q6u7bup/hJiJHikjDEB/2VGA0UKmqpw3xsc0OypKLGSyXqGrEnQaq5O/LWTeiquWFDExEfIXc/0gmIl5gEvC+qqa2Ynv7bk2vLLmY7YqIqIhcJCIfiEiTiPxeHEERaRaRvXLWrRaRThGp6fmLX0SWicj3RGQR0CEiPhGZ5raymkXkLRE5MWf929xjPS4ibSLymohM6RHX19y42kTkpyIyRUReEZFWEblfRAI5658gIgvdY70sIvv0iO07IrLIPRV1n4iERKQYeBIYl9OqG9fLd3SbiNwgIs+4sbwgIpNylu/hLmsUkfdE5PQe214vIk+ISAfwT+BHwCz3eOeLiEdErhCR5SKyTkTuEJEyd/s697s4X0Q+Bv4vp+xcEVnh/rtdJCIHuJ+xWUT+NyeGKSLyfyKyUUQ2iMjdIlKes7zX7ydn+Unud9sqIh+KyLFueZmI/ElEVovIShG52k2eZjioqk02bdMEPA+sBzYALwFH9rPuHOCufpYr8BhQDkx093usu+wW4JqcdS8G/u7OHwk05CxbBiwEJgBhwA8sAX4ABICjgDZgd3f924BG4EDAB9wN3NsjrkeAUmA6EAf+AewClAFvA2e76+4HrANmAl7gbDeeYE5srwPjgFHAO8BFvX2OPr6j29zYPwkEgd8B/3KXFQMrgHPdz7Gf++8yPWfbFuBQnB+XoZ7/JsB57ne1CxAB/grc6S6rc7+LO9xjhXPKbnD39x9ADPgbUAPUut/HEe4+dgU+7cZejZPgftvj366v7+dAN/5Pu/HXAnu4y/4G3OjGVePu4yvD/fexs07DHoBNI39yK9ESt7I42634pvSx7hwgATTnTM/lLFfgsJz39wOXu/PHAB/lLHsJOMud71YpuxXUeTnvDwfWAJ6csj8Dc9z524A/5iw7Hni3R1yH5ryfB3wv5/2vshUkcD3w0x6f+72cynUZ8KWcZT8Hbujtc/TxHd5G98QXAdI4iXQW8GKP9W8ErsrZ9o5e/k1yk8s/gK/lvN8dSOIkqzr3u9glZ3m2rDanbCMwK+f9g8C3+vg8nwcW9Pi36+v7uRH4TS/7GI2T8MM5ZWfk/t+yaWgnO19qtpmqvpbz9nYROQOncv6fPja5X1W/1M8u1+TMR3EqT4D/A8IiMtNdZwbwUD/7WZEzPw5YoaqZnLLlOL98Bzpu1tqc+c5e3o9x5ycBZ4vI13OWB9wY+jrWZqe/BtD12VS1XUQa3X1MAmZK90ESPuDO3rbtwzic7yZrubuP0QPsY6DvJwIgIjXAf+Mk/BKcFkhTj3319f1MAJ7o5diTcFqnq0UkW+bpI04zBCy5mEJQQAZca0t3qpoRkftxfpGuBR5T1bYB4shaBUwQEU9OgpkIvD/YceJUaNeo6jVbsW2+tymfkJ0RkQjO6aNV7rFfUNVPb8MxVuFU1lkTgRTOdz5+C+Pszf9zt99HVTeKyOeB/x1gm6wVwJQ+yuNAlW7FwAQz+KxD32wTESkXkc+4HdI+EflPnL6Apwp0yHtwTv38pzufr9eADuC/RMQvIkcCnwPuHfQI4WbgIhGZKY5iEfmsiJTkse1aoDLbgd6P40XkMHcQwU+B11R1BU5/1W4icqb7Of1ux/q0LYj/z8ClIjLZTVzX4ozwG6xKuwRoB5pFpBb47hZs+yfgXBE52h14UCsie6jqauBp4FciUuoumyIiRwxSzGYLWXIx28oPXM2mDv2vA59X1f4uaMyOTMqdavI5mHsKrgPnNMmT+QapqgngROA4N84/4PTXvJvvPrbgWHOBC3B+jTfhdI6fk+e27+JU7h+5o6z6Ol12D3AVziCE/XGSLW5L7j+A2TgtkDXAdTj9Yfm6Bec02j+BpTid81/vd4st82OcgQYtwOM4Awbyoqqv4wxW+I27/QtsamWdhXP68W2c7/0BYOygRW22iLgdX8aYEUJEbsPp9L9iuGMxpi/WcjHGGDPoLLkYY4wZdHZazBhjzKCzlosxxphBt9Ne51JVVaV1dXXDHYYxxowo8+bN26Cq1QOtt9Mml7q6OubOnTvcYRhjzIgiIssHXstOixljjCkASy7GGGMGnSUXY4wxg86SizHGmEFnycUYY8ygs+RijDFm0FlyMcYYM+h22utcjDFmR5DOKNFEis5EmmgiTWfSfU2knfKkM58tjyXTnHfoZCqKAwWNy5KLMcYMkWQ6Q0c8RXs8RUc8TXs8RTSRosN9H02k6Eikicbd18Sm8mgi3bUsmthUFk9lBj5wDo/ASTPGWXIxxpjhlMkoHYkUbTFnao8naY2laI85SaI9lqItnn2fpD3urJdNGB3Z5JFIk8gzEYhA2O+lOOijOOClKOCjOOilLOxnXFmIcMBLUcBLccDXNR8O+CjyZ+edbbLz4ZzygNeDyKA/hXwzllyMMTu0dEZpiyVp6XSm1s4UrbHsfJK2mPO+LZaitTPZNZ99355IMdDN40WgOOAjEnSSQCTkpyToo7ok6CYIp6IvDvooDvqIBDfNF7uJIxJ0EkUk6CPk8+LxFD4BFJIlF2PMiBBPpZ0EEU3SFE3SHE3QHE3S3Jl9dZY1dyZo6UzSHHUSSFss1e9+PQKlYT8lIR+lIT+lIT8TRxVREsqW+brmI+58JOijJORMETdBjPRkMNgsuRhjCkpV6UikaYslu04hxRJpUhkllckQS2a6JYPWmNOiaOlM0hRN0NThvEYT6T6P4fMIZWE/5UV+ysJ+qiNBptaUUBb2Uxr2U+6+luVMJSEfZWE/RQHvkJwm2tlYcjHGbLHORJr1bXHWtcVY3xZnfXucDW1x1rcn2NgeZ2NHgsaOBM3RBK2xFOlMfg8l9HudJFEa8lMS9lNTEmL30aWUF/mpKPJTVhSgLOzMV2TniwMUW4LY7lhyMcYATgujNZZiTUuMta0x1rTGWNcaY0N7YlPycKe2+OanmkRgVFGAqkiQUcUB9hxX6iSErpaCv+s0Usjvxe8VfB4PQb+na52w35LEjsKSyzboTKRZ1xZj4qgi+4Mw24VYMk1zNEmj23LY2BGnpdMZwRR1Ry7Fcq57yI52yiaVzuTmp55KQj6qI0EqIwH2GFvCJ3erprokSE1JkOqcaVRRAJ/Xrss2DksuW+mtVS1ceMc8VjZ3UlkcoL6ugr1ry6goDlAeDlAU8Dp/xMk0GYXqkiCjS4NUFgcJeD14vYJXBL9X8HrEktMOSlWJpzK0xVKkMhnSGe2aMqqkMkomAxl13ifTzrJUOkM8lSGWTBNLpdnYnmBdW5y1rTEaO5xTTW2xJLFEmqS7v2giRSzZ91BXj0CRO3Q17HemSMhHeVGACaOKOGqPGsaUhhhdFmJsWYjRJSFqSoOE/N4h/MbMjmKHSS4icizwO8AL/FFVf1aoYz28cCXfe3AR5eEAV56wJ2+tauGNZY089dbard5nwOvB6xF8XsHncRKO1+OcNnBenWVej8c9nSD4vB43OXkIeAW/10PA56wvCCLgFcHjbu8RuiUxjwhej/NKtjjn1LjiVI65wzA9HkH6WCbi7MvjHssrTtL0iBOLuOtky7weuuIEcipbJaN0xSsCqt2Pt2kbt2LOKIm0kkxnSKYzzvpsWt+J19lHRpV0jzpY0W6f3TmIs2FaNyWEZFq7kkTKTQSJtJME4qkMiVSmK1Gk0kpbLEWi58G2UtDnocb9gVIa8jG+Ikw45/RSOOClvMhPeThAeZGfUcUBKosDlBcF3FNRQ3N9gzGwgyQXEfECvwc+DTQAb4jII6r69mAeR1W59ol3uPnFpRxYN4rf/+d+VJcEu5bHkumuUS/RRIqQ++tQBDa0x1nbGmdje7zr12kykyHtVoiJtJLOZJwRNGkl7VaYyXT2F22GVFq7Rtik3EounsyQzKRJpjLufpxl2XjTbkWacSvIrIxbUWd/QXf/PnPm2ZQYshV0RrUrkeTmpOwydSvkTI/kU2gBn4eA14PP6yQvJ/7cz+MmPc+m5bmcxOfM58bt9TitTG+3hC74PR48Hijx+6iKBAn5PQR93q6E7fGIM7Q17PQz+N0fENl9ebrm3fUl++PC+QwBn4ew30vI7+3qu7DkYEaKHSK5AAcCS1T1IwARuRc4CRjU5CIihAM+zjmkjh9+dhr+HueXQ25FMLo0tNm2kyqLBzOUEcNpKbitDuhqUeSeCsoS2dRqyyazbILKTWbate9sosAqXWO2MztKcqkFVuS8bwBm9lxJRC4ELgSYOHHiVh3o0mOmWkW2BUQEr8Cm827GmJ3BjpJcequ5Njsho6o3ATcB1NfXb9UJG0ssxpgt5bTgM04/IOq25LWrHNj0ngzqtui7ttHNt1M2nf7ObtPful2vKHWldQS8duPKfDQAE3LejwdWDVMsZgeVrQgyZJzXHlP2j7zXskzv2ylKWtOb9t1jWW/7zlYSPeczmiGt6W7b9lzecz+bHSdnu9wKa7Pj5lSMPd/33F+3ijV3v71UitnvF2VTnNmKMc+Yet1vH8fsbR8Z3XT83irm/pZn99Fz+fbm4c8/zC5luxT0GAMmFxE5Dfi7qraJyBXAfsDVqjq/oJFtmTeAqSIyGVgJzAa+OLwhDb3ciimVSXWrbLKvPSuhdCbdbR1VJZlJktY0qUyKtKZJZ7rvI6UpMplM1zo9j9Nz/d6W5VbSva3vDEbY9NrX8s0qebci71qWU7H3uU2P7yS3ksmt+LfHSmKoOAM7BI94cMci4vV4N5XjcUf2CV7pvVzI2d5dD9hsPQ8ePOL0Z3rE022bzeZz9t21/x7HQujaZ29x9DWf+5qNJzufG1/Xej3i6Pps4sGDByeUvo/Ra1nOa/Y7yu63r226HZtNnzl7fEGoCdcU/P9MPi2XK1X1LyJyGPAZ4JfA9fTSpzFcVDUlIpcAT+EMRb5FVd8qxLFeWvkSa6NryY6TEpGuyietaZKZJIl0gkQ6QWeqk1g6RiwV61aBZddJZpLdKtZsZZ7KpEhmks6UTm6q5N2pr4p3e5f9o/SJr9sfncfjwSveruU957MVUe6ynuU+j29TRZRT7sGD1+Pt+oPubdve9p8tz1ZKfU5s2j77R5ydzz1W7vq523fF5HEq2s22zamIu1WMPSrJ3H1mt/N4NiWCbp8pJ+6ecfRV0RqzpfJJLtla67PA9ar6sIjMKVxIW0dVnwCeKPRx7nrnLv618l95revz+Ah7wwR9wW4Vl9/jJ+AN4Pf48Xq8XeVBXxCfx4df/Pi9fmfe47z2Vvn1VfFmK6veKuy+1skt93l8Xcf0irfX9XweHz7xbZYYepb1TAbGmJ1DPsllpYjcCBwDXCciQWCnvcfDtYddSzwd73aON7eSzyYNv9eP3+Mf7nCNMWZY5JNcTgeOBX6pqs0iMhb4bmHD2n5VhCqGOwRjjNnu9ZlcRGRUztvnc8riwNzChmWMMWYk66/lMg/nWhEBJgJN7nw58DEwueDRGWOMGZH67DtR1cmqugvOCKzPqWqVqlYCJwB/HaoAjTHGjDz5dMwf4I7EAkBVnwSOKFxIxhhjRrp8OvQ3uBdP3oVzmuxLwMaCRmWMMWZEy6flcgZQDTzkTtVumTHGGNOrflsu7nNSvq+q3xyieIwxxuwA+m25qGoa2H+IYjHGGLODyKfPZYGIPAL8BejIFqqqjRgzxhjTq3ySyyicDvyjcsoUG45sjDGmDwMmF1U9dygCMcYYs+PI53kuIeB8YDrQ9XB4VT2vgHEZY4wZwfIZinwnMAbnWS4v4Dzlsa2QQRljjBnZ8kkuu6rqlUCHqt6O81yXvQsbljHGmJEsn+SSdF+bRWQvoAyoK1RAIvILEXlXRBaJyEMiUp6z7PsiskRE3hORz+SUH+uWLRGRywsVmzHGmPzkk1xuEpEK4ErgEeBt4LoCxvQMsJeq7gO8D3wfQET2BGbj9P0cC/xBRLzuhZ6/B44D9gTOcNc1xhgzTPIZLfZHd/YFYJfChgOq+nTO21eBU935k4B7VTUOLBWRJcCB7rIlqvoRgIjc6677dkECfPJyWPNmQXZtjDEFN2ZvOO5nBT/MgC0XEflQRO4WkYuGoUVwHvCkO18LrMhZ1uCW9VW+GRG5UETmisjc9evXFyBcY4wxkN9FlHsCM4HDgV+KyB7Av1X15K09qIg8izMCracfqurD7jo/BFLA3dnNellf6T1Bam/HVdWbgJsA6uvre11nQEOQ8Y0xZqTLJ7mkcTr100AGWAus25aDquox/S0XkbNxHkp2tKpmk0ADMCFntfHAKne+r3JjjDHDIJ/k0gq8CfwauFlVC/osFxE5FvgecISqRnMWPQLcIyK/BsYBU4HXcVo0U0VkMrASp9P/i4WM0RhjTP/ySS5nAIcBXwO+LCIvA/9U1X8UKKb/BYLAMyIC8KqqXqSqb4nI/Tgd9SngYveuzYjIJTiPY/YCt6jqWwWKzRhjTB5k01mnAVZ0+lqOA74F1KhquJCBFVp9fb3OnTt3uMMwxphtpqqkkhkS0RSJWIpELE0yliKZyJCMp0jG0iQ608Q7kyQ60xxwQh3hSGCrjiUi81S1fqD18rm32IPADGAJ8CJwFvDaVkVljDGmV5pR4p0pEp2pTa/RFPFokng0RawjSawjRaw9QTy6+XqZ9MANBREIhH3sfWTtVieXfOVzWuxnwPzsKShjjDGbyyaHWEfSbSmkut53tiWItSeJRVMkoininUmS8QypRJpUIk3cTRC9j3N1iEAo4idU7CdY5CMUCVBWHSYQ9hEscsoCYR+BsJdA0Ic/5MUf9OILOK/BIh/+oBe3u6Hg8kkubwHfF5GJqnqhiEwFdlfVxwocmzHGDDlVJRlP09mWoLMtSWd7TnJoTxKPOkkiHnVOMSViTush1pFCM31nB5/fQ9BNDMGwj1CRD195EK/fQ6jIt2lZkd9JECGfk0Tc8kDIh3iGJjEMhnySy63APOAQ930DzlMpLbkYY7Zr2UQRa3eSRKw96Zxecl/jHdlE4ZxuiroJJZ3M9Lo/r89DqHhTIigqC1A+uohAyEso4iccCRAsdhJBIOzbVF4SwB/wDvGnH175JJcpqjpLRM4AUNVOGap2lTHG5Egm0nS2Jrr1RXS2JehoTdDZmtiURDo2TZlUH60JgWDY19VaCEX8VIwpJlwaoKgkQLjE35UYwhFnfihPK410+SSXhIiEcc8GisgUIF7QqIwxO410OkO0JUFHc9yZWuJ0tCSItmZPRTmtiWhbgmSsj65fgVDxpkRQXlNEqNhHKOInWOx3k0PAWcfttwgU+fCMoNNMI00+yeUq4O/ABBG5GzgUOKeQQRljdgyZdIbO9iTRlgTtTTHaGuO0NcZob4zR5k7R1sRmHdkejxAudVsPxX5qJoUoKg0SLnUSSKjY77Q6in1dCcXjzecm72ao9Jtc3NNf7wKnAAfhXA3/TVXdMASxGWO2Y6pKtCVBy4ZOWjd00rYxRuuGTqfV0ZIg2hqnsz25WeLw+j1EKoKUjAoxaXolkYogxeU5U1mQcMQ/ojqvzeb6TS6qqiLyN1XdH3h8iGIyxmwnErEULes6aV4bpXldlLbGGB1Ncdqa4rRt6CTVo+O7uCxAcUWIksoQoyeXUlQaoLgsQFGZkzhKRoUIl/it32InkM9psVdF5ABVfaPg0RhjhoyqEm1N0LrebXk0xmlrck5ZtTfFaW+Kk+hMddumqDRApCJIxegiJk4fRVlVmNLqMGVVYSKjgvj8O9eIKNO3fJLLp4CviMhyoAPn1Ji6T4o0xmyn0ukMic4U0dYEbRtitGzo3PS6sZOWDTFS8e4d5OESP5GKEGXVYWp3qyBSEaSsJkx5TRFl1WF8O9lwWrP18kkuxxU8CmPMFlNVOprjNK7qoHF1B83rnBZI6/pOOloTmyUOAF/AQ2lVmNKqMLW7V1BW7cyXVYeJjArtdNdimMLJ5zHHy4ciEGN2dolYyukUdzvGm1Z3sHGlkzhUtWsIbSaj7pXjiW7XcASLfJRVh6meWEJdebDr+o1wxE9JVYjSyrD1d5ghk0/LxRgzyDpa4qz/uI31H7exoaGdDQ3ttK7v7LZOsMhHZW2E3Q4cjccrXbcf8XiFqgklTtIYFWLU2GJGjSsmXFLYGxEasyUsuRhTYO1NcdYtb+1KJus/bnOu7QAQKKsKUz0hwh4HjaF8dBElldbKMCNfXslFRCYBU1X1WfdqfZ+qthUyMBH5DvALoFpVN7jX3PwOOB6IAueo6nx33bOBK9xNr1bV2wsZmzG9SXSmaFzTQfPaKE1rojSu6mDd8laiLU4iEYGKscVMmDaK6oklVE8qoWp8hEDIfuOZHU8+z3O5ALgQGAVMwXlG/Q3A0YUKSkQmAJ8GPs4pPg7n0cZTgZnA9cBMERmFcxeBepzLteaJyCOq2lSo+MzOK5NR2ptitKzrpGVdlOa1nTStjdK4up32xk13RfJ4hLKaMOP3qKBmUimj60qpHB+xDnOz08jnJ9PFwIG4DwhT1Q9EpKagUcFvgP8CHs4pOwm4Q51HZ74qIuUiMhY4EnhGVRsBROQZ4FjgzwWO0ewgVHWz00+qSqw9ycaV7axb3sa65W00rmqndUOMdGrThYM+v4eymiLGTiln1OHFjBpbTMWYIkqrw3jtdiRmJ5ZPcomraiL7xyciPvp9pM22EZETgZWq+u8ef/C1wIqc9w1uWV/lve37QpxWGBMnThzEqM1Ikk5mWP1RC6uXNLPmwxbWLG0lk1GKSwMUlQVIJTK0rO/sdgFhaVWIytoIdXtXUVYTpqymiPKaMMVlQbtNiTG9yCe5vCAiPwDCIvJp4GvAo9tyUBF5FhjTy6IfAj8A/qO3zXop037KNy9UvQm4CaC+vr5gCdJsX9KpDBtWtLP6w2ZWvNPEqg+aSCUyIDBqbDG77l+DP+ClozVOtCVBKOJnzORSymqKqBhTRM2kUkIR/3B/DGNGlHySy+XA+cCbwFeAJ4A/bstBVfWY3spFZG9gMpBttYwH5ovIgTgtkgk5q48HVrnlR/Yof35b4jMjW6wjyZqPWli9pIXVHzazbnlb18OfykcXMe3QcUyYNopxu5YRLLKkYUwh5HMRZQa42Z0KSlXfBLr6c0RkGVDvjhZ7BLhERO7F6dBvUdXVIvIUcK2IVLib/Qfw/ULHarYPsfYkjavb2biyg7VLW1m7rJXmtVHA6VSvmhBhr8NrGTOljLFTyiguDw5zxMbsHPIZLfYmm59magHm4gz73ViIwHrxBM4w5CU4Q5HPBVDVRhH5KZC9seZPsp37ZuTIZJRkLEU6paRTGRKxFJ2tia4HRGWfFAiwblkrqz9sYd2y1k3XiwDh0gBjJpeyx8FjGLNLGTV1pTY6y5hhks9psSeBNHCP+362+9oK3AZ8bvDDcqhqXc684oxc6229W4BbChWH2XbpZIbG1R2sX9FG6/pOom3OkwajLQmiLXGibUk0k383WPnoIibsOYrKcRFGjXOuUI9UBO2iQ2O2E/kkl0NV9dCc92+KyEuqeqiIfKlQgZmRraMlzqoPnNFYqz9sYePKdjJpJ3mIRwiX+CkqDVBUGqByfITi0gChiB+vz4PX78Ef9DrPMS8N4A96iUdTzvPQ0xmqJ5YQjtitTozZnuWTXCIiMlNVXwNwO9cj7rJU35uZnYmqsuqDZpYu2sCKtxtpXNUBOHfhHT25lBnHTKR6YglVEyKUVYW3ePhuyahCRG2MKZR8ksuXgVtEJIIz7LcV+LKIFAP/r5DBmZGh4d1GXn90Kas/bMHr8zB21zJ2P2gMtbtVUDUhYhcTGrMTyme02BvA3iJSBoiqNucsvr9gkZkhl+hM0dmexOsTvH4PXp8Hn9+Dp0dyUFWa10ZZvngjHy1Yz+oPWyguD3LEF3dn94PGWCe6MSbvG1d+FpgOhLIdpqr6kwLGZQZJKpmm4d0mPlq4nuY1UTxuwhCBZCJDKpEm0ZmivTlOMrb5w6XAueFiNtl4fR5UneeJAFSMKeKw06cy/fBx9ohbY0yXfIYi3wAU4Tzu+I/AqcDrBY5ru9XREneu7t5KqWS66665bRuc53d4vB48PsHjETxeQTzO5PEIIoJ4nE5wEacMcSr8juY4TWucfSVjKcQrzr5y9tO6vpNkPI0/5KVmYgmZVIZoZwpVxR/wEgh5iZQHmTBtFJGKEOESP5m0kkqmSSUypFPulMx0DRPWjFJTV8rEPUdRWhUerK/WGLMDyaflcoiq7iMii1T1xyLyK+CvhQ5se/Xcne+yfPHgXNoTLg0gApm0kkllyChoWsmkM2geo3I9XqGspohR44oJFfnIpJV0WtGMM2UyyujJpezyiWrG716B1299H8aYoZFPcom5r1ERGQdsxLlFy07pE8dMYNf6rb8ptMcrlNcUUT66qN/neKgqqnQliq75bLkqgbDPOsuNMdulfJLLoyJSjvPgrvk4V+sX/FYw26sJewzNmFgRQQSwO+4aY0agfpOLiHiAf7gjxB4UkceAkKq2DEl0xhhjRqR+z6m4N638Vc77uCUWY4wxA8nnhP3TIvIFsZs2GWOMyVM+fS6XAcVAWkQ6ca7SV1UtLWhkxhhjRqx8rtAvGYpAjDHG7DgGPC0mji+JyJXu+wnuzSuNMcaYXuXT5/IH4GDgi+77duD3BYvIGGPMiJdPcpmpqhfjXkypqk1AQR+mISJfF5H3ROQtEfl5Tvn3RWSJu+wzOeXHumVLROT3LPIfAAAgAElEQVTyQsZmjDFmYPl06CdFxIv7qGMRqQa2/uZaAxCRTwEnAfuoalxEatzyPXGegjkdGAc8KyK7uZv9Hvg00AC8ISKPqOrbhYrRGGNM//Jpufw38BBQIyLXAP8Cri1gTF8FfqaqcQBVXeeWnwTc615rsxRYAhzoTktU9SNVTQD3uusaY4wZJvmMFrtbROYBR+MMQ/68qr5TwJh2Aw53E1kM+I77TJla4NWc9RrcMoAVPcpn9rZjEbkQuBBg4sSJgxy2McaYrHxuuf874D5VHbROfBF5FhjTy6IfujFVAAcBBwD3i8guOImtJ6X31lev9xRW1ZuAmwDq6+vzuO+wMcaYrZFPn8t84Aq3f+MhnEQzd1sOqqrH9LVMRL4K/FVVFXhdRDJAFU6LZELOquOBVe58X+XGGGOGwYB9Lqp6u6oej9O38T5wnYh8UMCY/gYcBeAmtACwAXgEmC0iQRGZDEzFeWjZG8BUEZksIgGcTv9HChifMcaYAeT1mGPXrsAeQB1QyJFYtwC3iMhiIAGc7bZi3hKR+91jp4CLVTUNICKXAE8BXuAWVX2rgPEZY4wZgOgAjzwUkeuAU4APgftxTlk1D0FsBVVfX69z527T2T1jjNnpiMg8Va0faL18Wi5LgYNVdcO2h2WMMWZnkM9Q5BtEpMK9n1gop/yfBY3MGGPMiJXPUOQvA9/EGYW1EGeI8Cu4ne7GGGNMT/lcof9NnOtNlqvqp4B9gfUFjcoYY8yIlk9yialqDEBEgqr6LrB7YcMyxhgzkuXTod8gIuU41588IyJN2EWKxhhj+pFPh/7J7uwcEXkOKAP+XtCojDHGjGhbchElqvpCoQIxxhiz48inz8UYY4zZIpZcjDHGDLq8kouITBKRY9z5sIiUFDYsY4wxI9mAyUVELgAeAG50i8bjjBwzxhhjepVPy+Vi4FCgFUBVPwBqChmUMcaYkS2f5BJ3n00PgIj46ONJj8YYYwzkl1xeEJEfAGER+TTwF+DRwoZljDFmJMsnuVyOcy+xN4GvAE8AVxQqIBGZISKvishCEZnr3o0Zcfy3iCwRkUUisl/ONmeLyAfudHahYjPGGJOffK7QzwA3u9NQ+DnwY1V9UkSOd98fCRyH82jjqcBM4HpgpoiMAq4C6nFO180TkUdUtWmI4jXGmBFBVUksXUpwl10Kfqw+k4uIvEk/fSuquk9BInKOWerOl7HpPmYnAXe4jzx+VUTKRWQsTuJ5RlUb3bifAY4F/lyg+IwxZsTQdJrORYtoe/ZZ2p55luTHH7PLk08QnDy5oMftr+Vygvt6sft6p/v6n0C0YBHBt4CnROSXOKftDnHLa4EVOes1uGV9lW9GRC4ELgSYOHHi4EZtjDHDRNNpovPmEX/3PadAhHRTE50LF9C58N9kolHw+Sg+6CAqzzsPX2VlwWPqM7mo6nInRjlUVQ/NWXS5iLwE/GRrDyoizwJjeln0Q+Bo4FJVfVBETgf+BBwDSG9h9lO+eaHqTcBNAPX19TbizRgzYmkiQXT+fNqeeZbWp58ivb7Hk+g9HoK77UbZ508ivN/+RD55ON7S0t53VgD53LiyWEQOU9V/AYjIIUDxthxUVY/pa5mI3IHzgDJwRqb90Z1vACbkrDoe55RZA86psdzy57clPmOM2d6oKoklS4jOnUv7Sy8RffkVMtEoEgwSOeIISo/9DEUzZ4LHGaflCYXwhMPDFm8+yeV84BYRKXPfNwPnFS4kVgFH4CSIo4AP3PJHgEtE5F6cDv0WVV0tIk8B14pIhbvefwDfL2B8xhgzJDSZpP2ll2h97HE6XnqJdJMzTsk3biylJ36OyCc/SfHMmXiKt+n3fkHkM1psHvAJESkFRFVbChzTBcDv3Is1Y7h9JDhDoI8HluD0+ZzrxtcoIj8F3nDX+0m2c98YY0aaTCJB9LXXaHv2H7Q9/TTppia8ZWVEjjySogMPoOiAA/BPmIBIbz0C2w9xBl/1s4LTYrkK+KRb9AJOBV7oJFNQ9fX1Onfu3OEOwxhjSDQ00PHyy3S8/AodL75IpqMDKSqi5MgjKD3hc0QOOxQJBIY7TABEZJ6q1g+0Xj6nxW4BFgOnu+/PBG4FTtn68IwxZueWXLmS5ocfpvXhR0gsXw6Ab/RoSo8/jsjRR1N88MF4gsFhjnLr5ZNcpqjqF3Le/1hEFhYqIGOM2VFl4nHann6G5r8+SPTV10CVopkzqTjzTIoPOZjA5Mnb/emufOWTXDp7jBY7FOgsbFjGGLNjSK1fT3T+AjpeeZnWJ54k09qKv7aWqosvpuzznycwvtfL8ka8fJLLV4Hb3b4XARqBcwoZlDHGjESayZD48EOi8xfQOX8+0QULSH78MQASClFy9NGUn/oFimbORDw79oOA8xkttpBNo8VQ1daCR2WMMSNA9tqT9pdeIjp3Lp1z55FubgbAO2oU4f32pWL2bIr225fQnntuN53yQ2HA5CIi38TpwG8DbnbvRny5qj5d6OCMMWZ7o4kEHa+9Tvtzz9H+wgskV64EwD9xIpGjjqJo//0p2n8//JMm7TD9J1sjn9Ni56nq70TkMzhPoDwXJ9lYcjHG7BQ0kaDthRdofeIJOv7pDhUOhSg++GAqL7iAyBGfxD927HCHuV3JJ7lkU+/xwK2q+m/ZmdOxMWanoJkMsTffpOXRx2h97DHSzc14KyudocJHHeUMFQ6FhjvM7VY+yWWeiDwNTAa+LyIlQKawYRljzNDKJBIkP/6Y+NKlRF97nbZnniG1di3i9xM55mjKTz6Z4kMOQXz5VJsmnyv0PcAM4CNVbRaRSqBWVRcNRYCFYlfom+GQTCZpaGggFosNdyg7PVVFEwk0FkPjcTSVylkqSCiIJxRCQqEdfmRXb0KhEOPHj8fv93cr3+Yr9EVkD1V9FyexAOxiZ8OM2TYNDQ2UlJRQV1e3U3f2DodMIkG6qYlMZ6eTTJJJ8PmgpATP6NF4ioqQYBBPIIgEA4jXO9whDxtVZePGjTQ0NDB5Kx8q1l/77jKcm0b+qrdj49yx2BizBWKxmCWWIaTpNJmODtJNTaTb2gD3VvTZRBIOO/M7cSLpjYhQWVnJ+vXrt3of/T0s7EL39VNbvXdjzGYssRRWJpEgvXEjmY4OMu7pR/H58FVX462owLMTXWuyLbb1/2k+17mEgK8Bh+G0WF4EblBVO2lsjNluZBIJUuvXd13E6CkuxldTg6eoyGmd7IT9JsMpn2/7DmA68D/A/wJ7AncWMihjTOGsWbOG2bNnM2XKFPbcc0+OP/543n///by3v/baa7u993q9zJgxg7322ovTTjuNaDTa7/aRSGSr4s6lqnS2tXH6Kaew6+TJHPiJT/D+c8+Rbm7GV1FBcOpUgnV1+Gtq8EYilliGQT7f+O6qer6qPudOFwK7FTowY8zgU1VOPvlkjjzySD788EPefvttrr32WtauXZvXtplMZrPkEg6HWbhwIYsXLyYQCHDDDTcUJvZMhlRTE/Fly4i/+y43XncdpT4fbz72OF//8pe58g9/ILjbbvjHjbNTX9uBfAZsLxCRg1T1VQARmQm8tC0HFZHTgDnANOBAVZ2bs+z7OI9WTgPfUNWn3PJjgd8BXuCPqvozt3wycC8wCpgPnKmqiW2Jz5ih8ONH3+LtVYN7q749x5Vy1eem97n8ueeew+/3c9FFF3WVzZgxg/b2do4++miamppIJpNcffXVnHTSSSxbtozjjjuOT33qU7zyyivMmDGDzs5OZsyYwfTp07n77ru77f/www9n0SLnKoVf//rX3HLLLQB8+ctf5lvf+tZm8fziF7/g/vvvJx6Pc/LJJ/PjH/+42/J0Os2UXXbhjb8/RYkqmkmz12c/y/MPP8wTL7/MVVdeSWjaHpyx+25c+uMf2zUo25F8Wi4zgZdFZJmILANeAY4QkTdFZGuvdVmM87Cxf+YWisiewGyc03DHAn8QEa+IeIHfA8fhnJY7w10X4DrgN6o6FWjCSUzGmF4sXryY/ffff7PyUCjEQw89xPz583nuuef49re/TfYauPfee4+zzjqLBQsWcOutt3a1VHomllQqxZNPPsnee+/NvHnzuPXWW3nttdd49dVXufnmm1mwYEG39Z9++mk++OADXnvtNea//ApzX36ZfzzwAIlVq0iuXUty9WqSS5bw2cMP568PPoCnJMKCDRuYPHUqE2bMYNW6dUyaOhXxePD5fJSVlbFx48bCfXlmi+ST5o8d7IOq6jvQ62iEk4B7VTUOLBWRJcCB7rIlqvqRu929wEki8g7OkOgvuuvcjtMiun6wYzZmsPXXwhhqqsoPfvAD/vnPf+LxeFi5cmXXqbJJkyZx0EEH9blttiUDTsvl/PPP5/rrr+fkk0+muLgYgFNOOYUXX3yRfffdt+t4f3/8cZ5+8kn2nT4dVaUjGuWD997j0OnT0XQaRPBGIsz60pe49je/4StXXMFffv1rZs2a1bWPnmwk3vYjn1vuLxeRw4CpqnqriFQBJaq6tADx1AKv5rxvcMsAVvQonwlUAs2qmupl/c2IyIU41+4wceLEQQrZmJFj+vTpPPDAA5uV33333axfv5558+bh9/upq6vruotANkH0JduSydVbxa+qpDs6QJX4+x+Qbm7m2+edz1e+fD6ekhKn493n4/e//z0333wzAE888QSHf+YzLLnkEtavX8/f/vY3rrjiCgDGjx/PihUrGD9+PKlUipaWFkaNGrVV34sZfAOeFhORq4DvAd93iwLAXXls96yILO5lOqm/zXop060o75Wq3qSq9apaX11d3f8HMGYHdNRRRxGPx7sqb4A33niD5cuXU1NTg9/v57nnnmO5+0z33vj9fpLJJOB0sgOkNmwktXEjqcZGkuvWcfDuu/PQ/ffTtGgRjYsW8df77mPm+PEkli4FVSQY4NgTT+SuJ58gUVGBr7ycVWvXsm7dOi6++GIWLlzIwoULGTduHCLCySefzGWXXca0adOorKwE4MQTT+T2228H4IEHHuCoo46ylst2JJ/TYicD++J0lqOqq9ybV/ZLVY/ZingagAk578cDq9z53so3AOUi4nNbL7nrG2N6EBEeeughvvWtb/Gzn/2MUChEXV0dc+bM4Rvf+Ab19fXMmDGDPfbYo899XHjhheyzzz7MmD6dW6+5BjIZkmtWd1vnE7vswplf+AKHnXYaAOfOns1+Bx2ENxIBj4dgXR3H1dXxfkMDBx98MOAMUb7rrruoqanZ7JizZs3igAMO4LbbbusqO//88znzzDPZddddGTVqFPfee+8gfENmsORz48rXVfVAEZmvqvuJSDHwiqrus80HF3ke+E52tJiITAfuwelnGQf8A5iK00J5HzgaWAm8AXxRVd8Skb8AD6rqvSJyA7BIVf8w0LHtxpVmOLzzzjtMmzZtuMPYIqpKpr2ddFMTmkyiqbRzk0fN4AkX4aupxhMOg6pzOsHrtetKdhC9/X/d5htX5rhfRG7EaSFcAJwH3DzANv0SkZNxLsqsBh4XkYWq+hk3WdwPvA2kgItVNe1ucwnwFM5Q5FtU9S13d98D7hWRq4EFwJ+2JTZjjENTKdItLaQ2bkQTCcTnQ0Ih58aOPi+ekhI8xcXdTkXZSSmTlU+H/i9F5NNAK7A78CNVfWZbDqqqDwEP9bHsGuCaXsqfAJ7opfwjNo0oM8ZsA81kyLS1kW5uId3eBqp4wmF848fjLS21FonJW15XHLnJZJsSijFm+6GqkEo5r6pOK6W5hUxrC5pOOzd6HDUKb3m5c8rLmC1kl7Mas4NSVTKtraTb2yGTcZJIOo0mkmgqCT37W8WDt7QUb3kZnkjERl6ZbWLJxZgdjGYyTl/J+g1oIu48q8TrRcQDXg+eojASKHNulSICIojX6/Sf2HNNzCDJK7mIyChAVbWpwPEYY7aSplKkmppINzaiySSeUAj/hAl4SkutFWKGXJ+9cyIyUUTuFZH1wGvAGyKyzi2rG6oAjTGbU1Uy0U7nosU1a0gsX07svfdJrV2LBAIEJk0iMGUK3rKyzRKLiHDmmWd2vU+lUlRXV3PCCScMepy33XYbq1YNfOnZj370I5599tkB13vyySepr69n2rRp7LHHHnznO98ZjDA3s3TpUmbOnMnUqVOZNWsWiYTdC3dL9Tf04z6cEV1jVHWqqu4KjAX+hnMXYrOdyMTjqP3n3yGpqjOCKx4n3d5OqrGRxIoG4u++S/yjD0muWtU1VNhbXkZw110JTp6Mt6Skz9ZKcXExixcvprOzE4BnnnmG2to+75q0TfJNLj/5yU845pj+r7tevHgxl1xyCXfddRfvvPMOixcvZpdddsk7llQqNfBKru9973tceumlfPDBB1RUVPCnP9kVDluqz4soReQD907DW7RspNhRLqJs+8c/WH3FlQR3352Jt/zJhopu57pdlPbk5bDmTQA0k0aTScgooF0XJPZ2MyMRwOtz+1I8Tp9J9gqTMXvDcT/rN4ZIJMI3vvEN9ttvP0499VTOOusspk+fzosvvshjjz1GY2Mj5513Hh999BFFRUXcdNNN7LPPPsyZM4dIJNLVWthrr7147LHHADjuuOM47LDDePnll6mtreXhhx/m8ccf55xzzqG2tpZwOMwrr7zCL37xCx599FE6Ozs55JBDuPHGGxERzjnnHE444QROPfVU6urqOPvss3n00UdJJpP85S9/YY899uCss87iyCOP5LzzztvsMz366KNcffXVJBIJKisrufvuuxk9ejRz5sxh1apVLFu2jKqqKu65555u/xZnn302r7/+OgDLli3jxBNP5N///jfV1dWsWbMGn8/HK6+8wpw5c3jqqafy/nfeUWzLRZT91UTzROQPIjJTRMa500wR+QPOxYpmGGU6Olh1xRU0XHwJEgwSffVVmu+7b7jDMltI1WmVZDpjkM6AR8DjBZ8P8fmRgDN5ggE8oZDTGV9UhCcYdC5qFM+mxLIFZs+ezb333kssFmPRokXMnDmza9lVV13Fvvvuy6JFi7j22ms566yzBtzfBx98wMUXX8xbb71FeXk5Dz74IKeeeir19fXcfffdLFy4kHA4zCWXXMIbb7zR1XLKJqeeqqqqmD9/Pl/96lf55S9/CfT9uACAww47jFdffZUFCxYwe/Zsfv7zn3ctmzdvHg8//HC3xAIwbdo0EokEH330EQD33Xcfp59+Ohs3bqS8vByf+2yY8ePHs3LlygG/A9Ndfx36Z+E8G+XHOHcaFpx7fz2CXQU/rKJvvMGqH/yQZEMDlRdcQNXXL6Hhoq+y7he/JHLEEfjHjctrP4lly1j3q1/RuehNJt11J4EJEwbeyGwzTaVIt7WRmf410hPbAcVXWYmvunrIRmvts88+LFu2jD//+c8cf/zx3Zb961//4sEHHwScG11u3LiRlpaWfvc3efLkrtvu77///ixbtqzX9Z577jl+/vOfE41GaWxsZPr06Xzuc5/bbL1TTjmla19//etfB/w8DQ0NzJo1i9WrV5NIJJg8eXLXshNPPJFwH9fqnH766dx///1cfvnl3Hfffdx33312K/9B0mdycZ/meD32bJTtRiYaZd1vfkvTnXfinzCBSXfeQVG90zod85Of8NGJJ7L6qjlMuOnGfv8YEg0rabz9dpr+/GckEEA8HlZ+61Im/fmeYXs8bGLFCmKLF+OvrcU/YQLe8vIh/4PORKM0P/AAsbfeJtXUSLqpGW9FOZEjjiByxJEExteSSSRINzWT6eiATBpNO3cFFr/f+S79PufUpDuJ34/4/WTa22l75llSteOIpdNd23gryvFVVuIJBof0s4JT6X7nO9/h+eef7/aQrb4qV5/PR8a9CzLQdUt+gGBO/F6vt6s/J1csFuNrX/sac+fOZcKECcyZM6fbPnJl9+f1erv6SqZPn868efP4xCc+sdn6X//617nssss48cQTef7555kzZ07XstxHBpx77rksWLCAcePG8cQTTzBr1ixOO+00TjnlFESEqVOnoqo0NzeTSqXw+Xw0NDQwLs8fbGaTPpOLiPhwWi6fx2m5KM4dhx8G/qSqySGJ0AAQnTePVT/4AcnlH1PxpS9Rc9mleIqKupYHxtdSc+mlrL3mGlb/8Ao0mST58cdoKuWMHKqrQxNx2l94gfgHS8DjofzUU6n++iV0LlpEw8WXsO66nzPmyiuG9HOpKi0PPsiaa65FcyokTyTiJJpx4/CNGY03UoInEsFbVoZ//HgCE8bjKS6m49XXaP/nC3Qu/DfekhJ8NTX4qqqQQMCp3D3iVJYZhUzG2UdFBd7ycnxVlfiqqvCWldH65JNs/NMtpBsb8Y0bi29UJd7ycpLLP2btT69m7U+vRoqK0Gh02z7vDTfgq652bqUSCg3rL+LzzjuPsrIy9t57b55//vmu8k9+8pPcfffdXHnllTz//PNUVVVRWlpKXV1d12ms+fPns3TpwI90Kikpoa2tDdiUjKqqqmhvb+eBBx7g1FNPzTve7373u5xyyikcdthh7LbbbmQyGX77299y2WWX0dLS0jUoIXsb/t7ceuut3d5PmTIFr9fLT3/6066HkIkIn/rUp3jggQeYPXs2t99+Oyed1N+TQkxv+jstdifQjHNarMEtGw+cjfM8l1mFDc0AZDo7Wf/b39J4x534a2uZePvtFM/s/VZqFf/5RdqeeYaWhx7CP3Ys/kkT8Xh9dL75Jq1//zt4PBTV11NzyhcoOfooAu4D00qOPppRZ59N4+23U3TAAZQe+5ktjjO1fj0drzkdo76qKnxVlUgw6FwFruoMmW1YSXJlAyBO4hg7hqZ77qH1iScpmjmTmku/RWrjRhIfryDZ0EBy1SqSK1fSuWCBc5V5H6N9vBUVFNXXk4nHSK5eTeebb0Iy6SSVdLqrFQGQaW93ynpRfOihVF18MUX77dutPL50Ke3Pv0BqzWq85eV4Kyo2XXDocU5jaTKJJhJOpzzOlfCkM2gqhaZSiEcoPuwwlqriHz16i7/fQhg/fjzf/OY3NyufM2cO5557Lvvssw9FRUVdlfUXvvAF7rjjDmbMmMEBBxzAbrvtNuAxzjnnHC666KKuDv0LLriAvffem7q6Og444IAtinefffbht7/9LWeccQbRaBQR4bOf/WxXzKeddhq1tbUcdNBBeSW+rFmzZvHd73632zbXXXcds2fP5oorrmDffffl/PPt6elbqr/RYu+p6u59LHtfVQf+n7Ud295Hi6Wammh56G803XMPyYYGKr74RWq+fRmeAZ4KqOk0mk5vdnork0hAKtWttdNtu0SCZV86k/g77+CfONG5r1RVJYFJkwhOnoy/tpbUhg0kVqwgtXo1mskgPj9kMkQXLCD+zjtb90G9Xqq/8Q0qv3x+v/0NqorG46Sbmkg2NJBY0UC6qYmi+v0J7bVX3n0VXbePb2wktbGR1Ib1pBsbCU2bRriX0y2DbSTect/svAp1y/0mETkN51kpGXenHuA0wK7UL5BMPM6aq+bQ+vjjaDJJeMYMxl79U4r7eYZ5LvF6e61oPYEA9NOfIoEA4//nv9n4xz+RWruWVONGYm+9TdvTz2z2S99TVoZ4vc4zPdJpQtOmUX3ppRQfeiieUNB5KuGGDc6veHFOM3jKygiMH4+/thZUSa5eTbKhAX9tLcFddx34c4k4t3sfOxb/2LEUbeGv3tz9eEtK8JaUEJg0aav2YYwZWH/JZTZwHfAHEckmk3LgOXeZKYAN//M/tPztb1R88YuUz55FKI9TD4PFP3o0Y374g25lmkiQWLGC5KpV+KqqnM72SKTf/eSTLIJTphCcMmWb4jXGbL/6Gy22DLdfRUQqcU6hbRiMg7otojnANODAnCdRfhr4GRAAEsB3VfX/3GX7A7cBYZznunxTVdW979l9QB2wDDh9pN4DLbpgARtvuZXy005jzI+uHO5wAKdFY4nAGLOl8rqcW1U35iYWNwlsi8XAKcA/e5RvAD6nqnvjDBy4M2fZ9cCFOI89ngoc65ZfDvzDvWPAP9z3I04mFmP193+Ab8xoar73X8MdjjHGbJOtvVfINl1EqarvqOp7vZQvUNXsjYjeAkIiEhSRsUCpqr6izgiEO3CGSAOcBGTHHt6eUz6irP/t70gsW8a4q68e8LSTMcZs7/q7zuWRvhYBlYUJp5svAAtUNS4itWwaDo07n73T3mhVXQ2gqqtFpKavHYrIhTitHya6w3ALTVVJrV+Pv6bPsOh8800ab7+d8jNmU3zIIUMSlzHGFFJ/LZfDgRuBX/UytQ+0YxF5VkQW9zINeDWSiEzHGUzwlWxRL6v1Poa6H6p6k6rWq2p9dXX1lm6+VVr++hBLPnkEjT3ua5QTE2uvuRZvZSU13/72kMRkdm47+y33X3/99f/f3r2HVVnlCxz//gAFxNIBFD2ipaPhlfCIl7ylOZaOl8kJb3m8ZOOZsUibmTKdzmmyp2nMzI5OVpqJZg2iTGppns5MXnNMI2crXshboibgpTQVUpB1/nhfdhvc4AY2bIHf53l4YK/3tpYL38V617t+i969exMVFUXr1q351a9+RXYZJsdeuHCBN954w/n59OnTpZoU6kvGGKZMmULLli2Jjo5m9+7dFXMRd1/ABqBvMdu2Fndcab6AzUBskbRI4BDQwyWtMZDm8nk0sND++Sugsct+X3ly7U6dOpnKcOp3vzcHolqbA1GtzbnF79yw/cK6deZAVGvz3apVlZIf5VsHDhzwdRZMSEiIiYmJMdnZ2cYYYz7++GNz9913m0GDBnn9Wvfee6/54osvvHKu1NRU06JFC3Pw4EFjjDG5ublmwYIFHh+fm5trMjMzTbNmzcw///lPY4wx+fn5ZtWqVSYzM7PU+fn6669Nu3btSn3crWD9+vVmwIABJj8/3+zYscN06dLF7X7ufl+BFOPBPbakt8UGlrCtd3katOKISH1gPTDDGLPd5XoZInJJRLphLVw2DviLvflDrJEExKAAABeZSURBVMH/Wfb3tRWRt7LKcTioe999+AUFcuaVV8i/coXw+McRPz/yf/iBM3NeJbBNG+oNG+brrKpK9vKul0n7Ns2r52wd2ppnujxz0/0GDhzI+vXriYuLIzExkdGjR7Nt2zYAr4bcT0lJYcyYMV4JuT979myeffZZWrduDUBAQACPPfYY4HnI/ZYtWzJ+/HjuuecewOrFFfQ2Sir3iRMnOHbsGCdOnODJJ59kypQpTJ8+naNHjxITE0P//v15/PHHGTx4MPv27WPp0qV8+OGHZGdnc/ToUYYNG+aM1Fy3bl0uX7Ye/iQnJ7Nu3TqWLl1Keno6EydO5OzZszRo0ICEhASaNWtW6N/G9fiMjAxGjhzJ999/T15eHm+++Sa9evUqVM9du3ZlyZIltGvXDoA+ffrw6quvsnbtWsaNG4eI0K1bNy5cuEBGRgaNGzcu42/ejXyy+IeIDBORU8A9wHoRKVgoIR5oCfy3iDjsr4LBisnAYuAIcBSrZwVWo9JfRA4DBa8yV5gzc18j84UXPNo37+xZcr/5hjqxsfzbK69Qb9gwzr3xBl8P+yWXNm7i/JIl5GVkEDFjuq5dripVTQ25X9I5Sip3Wloan3zyCbt27WLmzJnk5uYya9YsfvrTn+JwOHjllVduOJ/D4SApKYnU1FSSkpI4efJkif+G8fHxjBs3jr179zJmzBimTJlS4v5//etfeeCBB3A4HOzZs8cZldrVqFGjWLlyJQAZGRmcPn2aTp068c0339DUJQp6RSwrUNIkygpjjFmNtcpl0fQXgReLOSYFaO8m/TzQz9t5LM6148e5euSIR/vm7NkDQHBMDOLvT+M/vUhIjx6c+8tfOGX/xXXb/fcT0sV9rDBVvXnSw6goGnL/RiWVe9CgQQQGBhIYGEjDhg3Jysq66fn69etHvXr1AGjbti3p6emFbuhF7dixw1nWsWPHMm1ayVMSOnfuzMSJE8nNzeXBBx9027iMGDGC/v37M3PmTFauXMnw4cOB4iNfe5MuW1hKAeFh5LmEJy9JjsMBtWoR1K4tAOLnR73Bg2ixfp0V0qVXLyJ0TovykYKQ+6NHjy6UXtyNpzQh990tKVwQcj85OZnU1FQmTZpUppD77jzxxBPEx8eTmprKwoULC53XNeR+Seco6YbrSfmKK0PRY1xv4sWV33U/1393YwzX7CXNe/fuzdatW2nSpAljx47l3XffZfXq1cTExBATE0NKSgpNmjQhLCyMvXv3kpSUxKhRVnCVyMjIQj2pilhW4KaNi4jcEDbVXVpN4R8eTv7Fi1YgyJvIdjgIatvmhrU6JCCA+nFxNHt7kRVrSykfmDhxIs899xwdOnQolF4Qch+4IeR+wVtF3gq5XxpPP/00L730EocOHQIgPz+fuXPnAngccj8+Pp5ly5axc+dOZ9p7771HZmZmseX2pGylERERwcGDB8nPz2f16h8f4HTv3p0VK1YA8P7779OzZ08A7rzzTmeDuHbtWnJzrdVO0tPTadiwIZMmTeLRRx9l9+7dDBs2DIfDgcPhINZe66ngMeHFixeddT106FDeffddjDF8/vnn1KtXz6vjLeBZz2W8m7QJXs1FFRIQFg7A9Zv0XkxuLj/s218pkXaVKouSQu6npKQQHR3N9OnTC4Xc//bbb4mJieHNN98sVcj9mJgYAgMDnSH3H3zwwXKF3G/Tpg3t27cnIyPDmefhw4fTq1cvwsPDiz1HREQEK1as4KmnniIqKoo2bdqwbds2br/99mLLXZywsDB69OhB+/btefrppz0ux6xZsxg8eDD33XdfoRv6/PnzSUhIIDo6muXLlzNv3jwAJk2axJYtW+jSpQs7d+509sQ2b95MTEwMHTt25G9/+5vbugSIi4tjxYoVjBgxwpn285//nBYtWtCyZUsmTZpU6JVqbykp5P5o4GGgJ7DNZdPtQJ4x5mdez00lKmvI/Uuffsqpx+O5c9UqgjvcMATklJO6j+PDh9Nk7qvcXuSZtqq5NOS+qkoqKuT+P4EMIBxr4mSBS8DeMuSzWggIs4IT5J0vHMMzZ/9+clJS+In9el+OwwFYg/lKKVXTlDTPJR1IF5GfATnGmHwRuQtoDaRWVgZvNf7h7h+LXVixggurkvELCaF+XBw5e/ZYS+56+TmmUkpVBZ6MuWzFCiDZBCvq8CNYoe9rJGfP5VzhxiU3IxOAzD+9xNVjx8hxOKxXkH24RrpSSvmKJ42LGGOysULk/8UYMwxoW7HZunX5BQfjFxJC3rnCj8XysjIJvvtu/AIDORX/BLmnTulgvlKqxvKocRGRe4AxWKFZwEeTL28V/uFhXC8y5pKbmUVQhw40fuklrh07BkBwRx1vUUrVTJ40LlOBGcBqY8x+EWmBtdRxjRUQFl7osdj1y1fIv3SJWo0iuO2+voSOH4dfvXoEta2xHTylVA1308bFGLPVGDPUGPOy/fmYMabkoDfVXEB4eKFZ+nlZ1nhLQEQjABpOn06rTRvxCwrySf6UKom/vz8xMTG0b9+e4cOHlyncvDt1PVjkrk+fPpRlCkBZmcoILa/c8mSGfgMReUVEPhaRjQVflZG5W1VAeFihMZfcTKtxqdXYalxEBL86dXySN6VuJjg4GIfDwb59+6hduzZvvfWWr7NUYTZs2MDhw4c5fPgwixYtYvLkyb7OUo3hydjJ+0ASMBj4DdaM/bMVmalbnX9YGPkXL2KuXUNq1ybPblwCGjXycc5UVZL50ktcPejdkPuBbVrT6A9/8Hj/Xr16sXevNW3tvffeY/78+Vy7do2uXbvyxhtv4O/vT926dZk6dSrr1q0jODiYtWvXEhERwddff83DDz9MXl4eAwYMcJ5z8+bNzJkzxxnxOD4+ntjYWCZMmFDo2sWFnp8wYQLBwcGkpaWRnp5OQkICy5YtY8eOHXTt2pWlS5feUA5fhpZX7nky5hJmjHkHyDXGbDHGTAS6VXC+bmkFIWDyvv0W+LHnElDCUsZK3Wry8vLYsGEDHTp04ODBgyQlJbF9+3YcDgf+/v7OOFtXrlyhW7du7Nmzh969e/P2228DMHXqVCZPnswXX3xBIy//YfXdd9+xceNGXnvtNYYMGcJvf/tb9u/fT2pqKg57grIrX4aWV+550nPJtb9niMgg4DTWapE1VkADu3E5d55ajRqRl5mFf3g4frVr+zhnqiopTQ/Dm3Jycpzh2Xv16sWjjz7KokWL+PLLL53xvnJycmho/7FUu3Zt5xLInTp14u9//zsA27dvd4aoHzt2LM88470lBIYMGYKI0KFDByIiIpwBF9u1a8fx48dvCC/vy9Dyyj1PGpcXRaQe8Hus1R9vB56s0Fzd4gomUha8jpybmUmtiAhfZkkpjxWMubgyxjB+/Hj+/Oc/37B/rVq1nDfkouHm3d2oSwrN76qk0PMF4er9/PwKha738/MjLy+P1atXM3PmTAAWL15MbGxsodDyCxcuBContLxyz5O3xdYZYy4aY/YZY/oaYzoBPy3PRUVkuIjsF5F8EbkhAJqINBORyyLylEvaABH5SkSOiMh0l/TmIrJTRA6LSJKIVHj3oSAETMGgfl5mpo63qCqtX79+JCcnc+bMGcBa8jc9Pb3EY3r06FEoRHyBO+64gwMHDnD16lUuXrzIp59+6vb44kLPe+JWCi2v3CvrYmG/K+d192HN+N9azPbX+HEZY0TEH1gADMSKDjBaRAomkbwMvGaMaQV8BzxazrzdVNEQMLlZWdTSxkVVYW3btuXFF1/k/vvvJzo6mv79+zvD2Rdn3rx5LFiwgM6dOxdaqbJp06aMGDGC6OhoxowZQ8eOHd0eX1zo+bLyVWh55V6xIfdLPEjkpDGm+PU6PT/PZuApewnjgrQHgR7AFeCyMWaOHSHgeWPMA/Y+M+zdZ2G9udbIGJNXdL+SlDXkfoGvOsVS76Ff0nDqVL7qFEuD3/+O8EmTynw+VTNoyH1VlZQn5H5Zey6lb5E8ICIhwDPAzCKbmgAnXT6fstPCgAvGmLwi6cWd/z9FJEVEUs6eLd/b1P7hYVw/d55cey1t7bkopdSPih3QF5FLuG9EBAi+2YlF5B+Auzvus8aYtcUcNhPrEdflIgOF7l7vMCWku2WMWQQsAqvnUtx+nggIs2bpO+e46IC+Uko5lbSey23lOXEZV6rsCsSJyGygPpAvIj8AXwKuj+EisV6JPgfUF5EAu/dSkF7hAsLCuHrsGLmZds9FBwmVUsrplopubIzpVfCziDyPNebyuogEAK1EpDnwDTAKeNgYY0RkExAHrMCKHlBcr8irAhqEk71rF7mZ1qCnTqBUSqkflXXMpVxEZJiInALuAdaLyCcl7W/3SuKBT4CDwEpjzH578zPA70TkCNYYzDsVl/Mf+YeFcf3iRXJPfYN/aCh+Lu/iK6VUTeeTnosxZjVQ4ovtxpjni3z+GPjYzX7HgC7ezJ8nCkLA/HDggA7mK6VUET7puVQHAeHWXJerhw/rBEpV5axevRoRIS3NO4EzJ0yYQHJyssf7OxwOPv74hr8Vb+rZZ5+ladOmHoX3V76ljUsZ+dsTKbl+nVqN9E0xVbUkJibSs2dP5wx7T7iGfSmvsjYuQ4YMYdeuXV7Lh6o4t9SAflUS0KDBjz830jfFVOltW3mIcycve/Wc4U3r0mvEXSXuc/nyZbZv386mTZsYOnQozz//PACzZ89m+fLl+Pn5MXDgQGbNmkWfPn3o3r0727dvZ+jQocTFxTFx4kTOnj1LgwYNSEhIoFmzZgD84x//YN68eWRlZTF37lwGDx7MDz/8wOTJk0lJSSEgIIC5c+fSo0cPnnvuOXJycvjss8+YMWMGI0eOdOZvw4YNJCQkOKMcb968mVdffZWPPvqIbt1qdED2KkUblzIqCAEDaM9FVSlr1qxhwIAB3HXXXYSGhrJ7926ysrJYs2YNO3fupE6dOnxrLycBcOHCBbZs2QJYPYdx48Yxfvx4lixZwpQpU1izZg0Ax48fZ8uWLRw9epS+ffty5MgRFixYAEBqaippaWncf//9HDp0iBdeeIGUlBRef/31G/LXv39/fv3rX3PlyhVCQkJISkoq1PioqkEblzLyCw7Gr04d8rOzncsbK1UaN+thVJTExESefNIKbD5q1CgSExPJz8/nkUceoY69gmpoaKhzf9cb+44dO/jggw8AK8z+tGnTnNtGjBiBn58frVq1okWLFqSlpfHZZ5/xxBNPANC6dWvuuOMODh06VGL+AgICGDBgAB999BFxcXGsX7+e2bNne6fwqtJo41IO/uHh5J844VzeWKlb3fnz59m4cSP79u1DRLh+/ToiwkMPPVTsOichISHFns/1mKLHi4jb9VTceeCBB8jKyiI2NpbFixczcuRIFixYQGhoKJ07d+a228o1p1v5gA7ol0OAHXpfQ7+oqiI5OZlx48aRnp7O8ePHOXnyJM2bNyc0NJQlS5aQnZ0NUOixmKvu3bsXCrPfs2dP57ZVq1aRn5/P0aNHOXbsGFFRUfTu3dsZjv/QoUOcOHGCqKgobrvtNi5duuQ89pNPPsHhcLB48WLAWqZ49+7dvP322/pIrIrSxqUcAsLC8P/JT3QCpaoyEhMTGTZsWKG0hx56iNOnTzN06FBiY2OJiYlhzpw5bo+fP38+CQkJREdHs3z5cubNm+fcFhUVxb333svAgQN56623CAoK4rHHHuP69et06NCBkSNHsnTpUgIDA+nbty8HDhwgJiaGpKSkG67j7+/P4MGD2bBhg3MVTIBp06YRGRlJdnY2kZGRzpcR1K2nTCH3q4PyhtwHuLJzF7knT1A/Ls5LuVLVnYbcV1VJeULu65hLOYR07QJdKz04gFJK3fL0sZhSSimv08ZFqUpWUx9Fq6qlvL+n2rgoVYmCgoI4f/68NjDqlmaM4fz58wQFBZX5HDrmolQlioyM5NSpU5R3mW2lKlpQUBCRkZFlPl4bF6UqUa1atWjevLmvs6FUhdPHYkoppbxOGxellFJep42LUkopr6uxM/RF5CyQXsbDw4FzXsxOVVETy10Tyww1s9xaZs/cYYxpcLOdamzjUh4ikuJJ+IPqpiaWuyaWGWpmubXM3qWPxZRSSnmdNi5KKaW8ThuXslnk6wz4SE0sd00sM9TMcmuZvUjHXJRSSnmd9lyUUkp5nTYuSimlvE4bl1ISkQEi8pWIHBGR6b7OT0UQkaYisklEDorIfhGZaqeHisjfReSw/f0nvs6rt4mIv4j8S0TW2Z+bi8hOu8xJIlLb13n0NhGpLyLJIpJm1/k91b2uReS39u/2PhFJFJGg6ljXIrJERM6IyD6XNLd1K5b59r1tr4j8e3murY1LKYiIP7AAGAi0BUaLSFvf5qpC5AG/N8a0AboBj9vlnA58aoxpBXxqf65upgIHXT6/DLxml/k74FGf5KpizQP+1xjTGrgbq/zVtq5FpAkwBYg1xrQH/IFRVM+6XgoMKJJWXN0OBFrZX/8JvFmeC2vjUjpdgCPGmGPGmGvACuAXPs6T1xljMowxu+2fL2HdbJpglXWZvdsy4EHf5LBiiEgkMAhYbH8W4D4g2d6lOpb5dqA38A6AMeaaMeYC1byusSLCB4tIAFAHyKAa1rUxZivwbZHk4ur2F8C7xvI5UF9EGpf12tq4lE4T4KTL51N2WrUlIncCHYGdQIQxJgOsBgho6LucVYj/AaYB+fbnMOCCMSbP/lwd67sFcBZIsB8HLhaREKpxXRtjvgHmACewGpWLwJdU/7ouUFzdevX+po1L6YibtGr7LreI1AX+BjxpjPne1/mpSCIyGDhjjPnSNdnNrtWtvgOAfwfeNMZ0BK5QjR6BuWOPMfwCaA78GxCC9UioqOpW1zfj1d93bVxK5xTQ1OVzJHDaR3mpUCJSC6thed8Y84GdnFXQTba/n/FV/ipAD2CoiBzHetx5H1ZPpr796ASqZ32fAk4ZY3ban5OxGpvqXNc/A742xpw1xuQCHwDdqf51XaC4uvXq/U0bl9L5Amhlv1VSG2sQ8EMf58nr7LGGd4CDxpi5Lps+BMbbP48H1lZ23iqKMWaGMSbSGHMnVr1uNMaMATYBcfZu1arMAMaYTOCkiETZSf2AA1TjusZ6HNZNROrYv+sFZa7Wde2iuLr9EBhnvzXWDbhY8PisLHSGfimJyM+x/qL1B5YYY/7k4yx5nYj0BLYBqfw4/vAHrHGXlUAzrP+gw40xRQcLqzwR6QM8ZYwZLCItsHoyocC/gP8wxlz1Zf68TURisF5iqA0cAx7B+sOz2ta1iMwERmK9Gfkv4FdY4wvVqq5FJBHogxVaPwv4I7AGN3VrN7SvY71dlg08YoxJKfO1tXFRSinlbfpYTCmllNdp46KUUsrrtHFRSinlddq4KKWU8jptXJRSSnmdNi5KeYmIXBcRh8tXiTPdReQ3IjLOC9c9LiLh5T2PUt6kryIr5SUictkYU9cH1z2OFeH3XGVfW6niaM9FqQpm9yxeFpFd9ldLO/15EXnK/nmKiByw19FYYaeFisgaO+1zEYm208NE5P/sQJMLcYkJJSL/YV/DISIL7WUilKp02rgo5T3BRR6LjXTZ9r0xpgvWDOj/cXPsdKCjMSYa+I2dNhP4l532B+BdO/2PwGd2oMkPsWZaIyJtsGad9zDGxADXgTHeLaJSngm4+S5KKQ/l2Dd1dxJdvr/mZvte4H0RWYMVngOgJ/AQgDFmo91jqYe1/sov7fT1IvKdvX8/oBPwhRXJg2CqV8BJVYVo46JU5TDF/FxgEFajMRT4bxFpR8kh0N2dQ4BlxpgZ5cmoUt6gj8WUqhwjXb7vcN0gIn5AU2PMJqzFyuoDdYGt2I+17GCa5+x1dVzTBwIF69t/CsSJSEN7W6iI3FGBZVKqWNpzUcp7gkXE4fL5f40xBa8jB4rITqw/6EYXOc4feM9+5CVY67hfEJHnsVaI3IsVpbYgTPpMIFFEdgNbsCLbYow5ICL/Bfyf3WDlAo8D6d4uqFI3o68iK1XB9FVhVRPpYzGllFJepz0XpZRSXqc9F6WUUl6njYtSSimv08ZFKaWU12njopRSyuu0cVFKKeV1/w+iQXuBLthKfAAAAABJRU5ErkJggg==\n",
419 |       "text/plain": [
420 |        "<Figure size 432x288 with 1 Axes>"
421 |       ]
422 |      },
423 |      "metadata": {},
424 |      "output_type": "display_data"
425 |     }
426 |    ],
427 |    "source": [
428 |     "import matplotlib.pyplot as plt\n",
429 |     "%matplotlib inline\n",
430 |     "\n",
431 |     "for n, ep_rewards in enumerate(run_rewards):\n",
432 |     "    x = range(len(ep_rewards))\n",
433 |     "    cumsum = np.cumsum(ep_rewards)\n",
434 |     "    avgs = [cumsum[ep]/(ep+1) if ep<100 else (cumsum[ep]-cumsum[ep-100])/100 for ep in x]\n",
435 |     "    plt.plot(x, avgs, label=env_names[n])\n",
436 |     "    \n",
437 |     "plt.title(\"5 Environment performance\")\n",
438 |     "plt.xlabel(\"Episode\")\n",
439 |     "plt.ylabel(\"Last 100 episode average rewards\")\n",
440 |     "plt.legend()"
441 |    ]
442 |   },
443 |   {
444 |    "cell_type": "code",
445 |    "execution_count": 32,
446 |    "metadata": {
447 |     "colab": {},
448 |     "colab_type": "code",
449 |     "id": "_UBHQMHzqsMU"
450 |    },
451 |    "outputs": [],
452 |    "source": [
453 |     "env.close()"
454 |    ]
455 |   }
456 |  ],
457 |  "metadata": {
458 |   "colab": {
459 |    "name": "PrioritizedExperienceReplayInOpenAIGym.ipynb",
460 |    "provenance": [],
461 |    "version": "0.3.2"
462 |   },
463 |   "kernelspec": {
464 |    "display_name": "Python 3",
465 |    "language": "python",
466 |    "name": "python3"
467 |   },
468 |   "language_info": {
469 |    "codemirror_mode": {
470 |     "name": "ipython",
471 |     "version": 3
472 |    },
473 |    "file_extension": ".py",
474 |    "mimetype": "text/x-python",
475 |    "name": "python",
476 |    "nbconvert_exporter": "python",
477 |    "pygments_lexer": "ipython3",
478 |    "version": "3.6.5"
479 |   }
480 |  },
481 |  "nbformat": 4,
482 |  "nbformat_minor": 1
483 | }
484 | 


--------------------------------------------------------------------------------
/PrioritizedExperienceReplayInOpenAIGym.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |   "nbformat": 4,
  3 |   "nbformat_minor": 0,
  4 |   "metadata": {
  5 |     "colab": {
  6 |       "name": "PrioritizedExperienceReplayInOpenAIGym.ipynb",
  7 |       "version": "0.3.2",
  8 |       "provenance": []
  9 |     },
 10 |     "kernelspec": {
 11 |       "display_name": "Python 3",
 12 |       "language": "python",
 13 |       "name": "python3"
 14 |     }
 15 |   },
 16 |   "cells": [
 17 |     {
 18 |       "metadata": {
 19 |         "id": "-578TXDLZF47",
 20 |         "colab_type": "code",
 21 |         "colab": {}
 22 |       },
 23 |       "cell_type": "code",
 24 |       "source": [
 25 |         "# Run this to install tensorflow version 1.9\n",
 26 |         "!pip install tensorflow==1.9"
 27 |       ],
 28 |       "execution_count": 0,
 29 |       "outputs": []
 30 |     },
 31 |     {
 32 |       "metadata": {
 33 |         "id": "HnA-Zf8JqsL4",
 34 |         "colab_type": "code",
 35 |         "outputId": "70062eee-0733-4f72-b717-2aeec3d927b9",
 36 |         "colab": {
 37 |           "base_uri": "https://localhost:8080/",
 38 |           "height": 51
 39 |         }
 40 |       },
 41 |       "cell_type": "code",
 42 |       "source": [
 43 |         "import gym\n",
 44 |         "import random\n",
 45 |         "import numpy as np\n",
 46 |         "import tensorflow as tf\n",
 47 |         "from collections import deque\n",
 48 |         "print(\"Gym:\", gym.__version__)\n",
 49 |         "print(\"Tensorflow:\", tf.__version__)"
 50 |       ],
 51 |       "execution_count": 2,
 52 |       "outputs": [
 53 |         {
 54 |           "output_type": "stream",
 55 |           "text": [
 56 |             "Gym: 0.10.11\n",
 57 |             "Tensorflow: 1.9.0\n"
 58 |           ],
 59 |           "name": "stdout"
 60 |         }
 61 |       ]
 62 |     },
 63 |     {
 64 |       "metadata": {
 65 |         "id": "YHkYC1tmqsMB",
 66 |         "colab_type": "code",
 67 |         "outputId": "c3abd168-27fc-4f2c-be25-18194375bd0d",
 68 |         "colab": {
 69 |           "base_uri": "https://localhost:8080/",
 70 |           "height": 51
 71 |         }
 72 |       },
 73 |       "cell_type": "code",
 74 |       "source": [
 75 |         "env_name = \"CartPole-v0\"\n",
 76 |         "env = gym.make(env_name)\n",
 77 |         "print(\"Observation space:\", env.observation_space)\n",
 78 |         "print(\"Action space:\", env.action_space)"
 79 |       ],
 80 |       "execution_count": 3,
 81 |       "outputs": [
 82 |         {
 83 |           "output_type": "stream",
 84 |           "text": [
 85 |             "Observation space: Box(4,)\n",
 86 |             "Action space: Discrete(2)\n"
 87 |           ],
 88 |           "name": "stdout"
 89 |         }
 90 |       ]
 91 |     },
 92 |     {
 93 |       "metadata": {
 94 |         "id": "79GCjcYYqsMG",
 95 |         "colab_type": "code",
 96 |         "colab": {}
 97 |       },
 98 |       "cell_type": "code",
 99 |       "source": [
100 |         "class QNetwork():\n",
101 |         "    def __init__(self, state_dim, action_size, tau=0.01):\n",
102 |         "        tf.reset_default_graph()\n",
103 |         "        self.state_in = tf.placeholder(tf.float32, shape=[None, *state_dim])\n",
104 |         "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
105 |         "        self.q_target_in = tf.placeholder(tf.float32, shape=[None])\n",
106 |         "        self.importance_in = tf.placeholder(tf.float32, shape=[None])\n",
107 |         "        action_one_hot = tf.one_hot(self.action_in, depth=action_size)\n",
108 |         "        \n",
109 |         "        self.q_state_local = self.build_model(action_size, \"local\")\n",
110 |         "        self.q_state_target = self.build_model(action_size, \"target\")\n",
111 |         "        \n",
112 |         "        self.q_state_action = tf.reduce_sum(tf.multiply(self.q_state_local, action_one_hot), axis=1)\n",
113 |         "        self.error = self.q_state_action - self.q_target_in\n",
114 |         "        self.loss = tf.reduce_mean(tf.multiply(tf.square(self.error), self.importance_in))\n",
115 |         "        self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(self.loss)\n",
116 |         "        \n",
117 |         "        self.local_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=\"local\")\n",
118 |         "        self.target_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=\"target\")\n",
119 |         "        self.updater = tf.group([tf.assign(t, t + tau*(l-t)) for t,l in zip(self.target_vars, self.local_vars)])\n",
120 |         "        \n",
121 |         "    def build_model(self, action_size, scope):\n",
122 |         "        with tf.variable_scope(scope):\n",
123 |         "            hidden1 = tf.layers.dense(self.state_in, 100, activation=tf.nn.relu)\n",
124 |         "            q_state = tf.layers.dense(hidden1, action_size, activation=None)\n",
125 |         "            return q_state\n",
126 |         "        \n",
127 |         "    def update_model(self, session, state, action, q_target, importance):\n",
128 |         "        feed = {self.state_in: state, self.action_in: action, self.q_target_in: q_target, self.importance_in: importance}\n",
129 |         "        error, _, _ = session.run([self.error, self.optimizer, self.updater], feed_dict=feed)\n",
130 |         "        return error\n",
131 |         "        \n",
132 |         "    def get_q_state(self, session, state, use_target=False):\n",
133 |         "        q_state_op = self.q_state_target if use_target else self.q_state_local\n",
134 |         "        q_state = session.run(q_state_op, feed_dict={self.state_in: state})\n",
135 |         "        return q_state"
136 |       ],
137 |       "execution_count": 0,
138 |       "outputs": []
139 |     },
140 |     {
141 |       "metadata": {
142 |         "id": "EYKcRk3ZqsMJ",
143 |         "colab_type": "code",
144 |         "colab": {}
145 |       },
146 |       "cell_type": "code",
147 |       "source": [
148 |         "class PrioritizedReplayBuffer():\n",
149 |         "    def __init__(self, maxlen):\n",
150 |         "        self.buffer = deque(maxlen=maxlen)\n",
151 |         "        self.priorities = deque(maxlen=maxlen)\n",
152 |         "        \n",
153 |         "    def add(self, experience):\n",
154 |         "        self.buffer.append(experience)\n",
155 |         "        self.priorities.append(max(self.priorities, default=1))\n",
156 |         "        \n",
157 |         "    def get_probabilities(self, priority_scale):\n",
158 |         "        scaled_priorities = np.array(self.priorities) ** priority_scale\n",
159 |         "        sample_probabilities = scaled_priorities / sum(scaled_priorities)\n",
160 |         "        return sample_probabilities\n",
161 |         "    \n",
162 |         "    def get_importance(self, probabilities):\n",
163 |         "        importance = 1/len(self.buffer) * 1/probabilities\n",
164 |         "        importance_normalized = importance / max(importance)\n",
165 |         "        return importance_normalized\n",
166 |         "        \n",
167 |         "    def sample(self, batch_size, priority_scale=1.0):\n",
168 |         "        sample_size = min(len(self.buffer), batch_size)\n",
169 |         "        sample_probs = self.get_probabilities(priority_scale)\n",
170 |         "        sample_indices = random.choices(range(len(self.buffer)), k=sample_size, weights=sample_probs)\n",
171 |         "        samples = np.array(self.buffer)[sample_indices]\n",
172 |         "        importance = self.get_importance(sample_probs[sample_indices])\n",
173 |         "        return map(list, zip(*samples)), importance, sample_indices\n",
174 |         "    \n",
175 |         "    def set_priorities(self, indices, errors, offset=0.1):\n",
176 |         "        for i,e in zip(indices, errors):\n",
177 |         "            self.priorities[i] = abs(e) + offset"
178 |       ],
179 |       "execution_count": 0,
180 |       "outputs": []
181 |     },
182 |     {
183 |       "metadata": {
184 |         "id": "XKIXlcQSqsMN",
185 |         "colab_type": "code",
186 |         "colab": {}
187 |       },
188 |       "cell_type": "code",
189 |       "source": [
190 |         "class DoubleDQNAgent():\n",
191 |         "    def __init__(self, env):\n",
192 |         "        self.state_dim = env.observation_space.shape\n",
193 |         "        self.action_size = env.action_space.n\n",
194 |         "        self.q_network = QNetwork(self.state_dim, self.action_size)\n",
195 |         "        self.replay_buffer = PrioritizedReplayBuffer(maxlen=100000)\n",
196 |         "        self.gamma = 0.97\n",
197 |         "        self.eps = 1.0\n",
198 |         "        \n",
199 |         "        self.sess = tf.Session()\n",
200 |         "        self.sess.run(tf.global_variables_initializer())\n",
201 |         "        \n",
202 |         "    def get_action(self, state):\n",
203 |         "        q_state = self.q_network.get_q_state(self.sess, [state])\n",
204 |         "        action_greedy = np.argmax(q_state)\n",
205 |         "        action_random = np.random.randint(self.action_size)\n",
206 |         "        action = action_random if random.random() < self.eps else action_greedy\n",
207 |         "        return action\n",
208 |         "    \n",
209 |         "    def train(self, state, action, next_state, reward, done, use_DDQN=True, a=0.0):\n",
210 |         "        self.replay_buffer.add((state, action, next_state, reward, done))\n",
211 |         "        (states, actions, next_states, rewards, dones), importance, indices = self.replay_buffer.sample(50, priority_scale=a)\n",
212 |         "        \n",
213 |         "        next_actions = np.argmax(self.q_network.get_q_state(self.sess, next_states, use_target=False), axis=1)\n",
214 |         "        q_next_states = self.q_network.get_q_state(self.sess, next_states, use_target=use_DDQN)\n",
215 |         "        q_next_states[dones] = np.zeros([self.action_size])\n",
216 |         "        q_next_states_next_actions = q_next_states[np.arange(next_actions.shape[0]), next_actions]\n",
217 |         "        q_targets = rewards + self.gamma * q_next_states_next_actions\n",
218 |         "        errors = self.q_network.update_model(self.sess, states, actions, q_targets, importance**(1-self.eps))\n",
219 |         "        \n",
220 |         "        self.replay_buffer.set_priorities(indices, errors)\n",
221 |         "        \n",
222 |         "        if done: self.eps = max(0.1, 0.99*self.eps)\n",
223 |         "    \n",
224 |         "    def __del__(self):\n",
225 |         "        self.sess.close()"
226 |       ],
227 |       "execution_count": 0,
228 |       "outputs": []
229 |     },
230 |     {
231 |       "metadata": {
232 |         "id": "zv9Amjj4qsMQ",
233 |         "colab_type": "code",
234 |         "outputId": "3e61344f-d87a-41e6-d28c-4ca3511ef4df",
235 |         "colab": {
236 |           "base_uri": "https://localhost:8080/",
237 |           "height": 187
238 |         }
239 |       },
240 |       "cell_type": "code",
241 |       "source": [
242 |         "num_runs = 10\n",
243 |         "run_rewards = []\n",
244 |         "\n",
245 |         "for n in range(num_runs):\n",
246 |         "    print(\"Run {}\".format(n))\n",
247 |         "    ep_rewards = []\n",
248 |         "    agent = None\n",
249 |         "    agent = DoubleDQNAgent(env)\n",
250 |         "    num_episodes = 200\n",
251 |         "\n",
252 |         "    for ep in range(num_episodes):\n",
253 |         "        state = env.reset()\n",
254 |         "        total_reward = 0\n",
255 |         "        done = False\n",
256 |         "        while not done:\n",
257 |         "            action = agent.get_action(state)\n",
258 |         "            next_state, reward, done, info = env.step(action)\n",
259 |         "            agent.train(state, action, next_state, reward, done, a=(n%2==0)*0.7)\n",
260 |         "            #env.render()\n",
261 |         "            total_reward += reward\n",
262 |         "            state = next_state\n",
263 |         "\n",
264 |         "        ep_rewards.append(total_reward)\n",
265 |         "        #print(\"Episode: {}, total_reward: {:.2f}\".format(ep, total_reward))\n",
266 |         "        \n",
267 |         "    run_rewards.append(ep_rewards)"
268 |       ],
269 |       "execution_count": 8,
270 |       "outputs": [
271 |         {
272 |           "output_type": "stream",
273 |           "text": [
274 |             "Run 0\n",
275 |             "Run 1\n",
276 |             "Run 2\n",
277 |             "Run 3\n",
278 |             "Run 4\n",
279 |             "Run 5\n",
280 |             "Run 6\n",
281 |             "Run 7\n",
282 |             "Run 8\n",
283 |             "Run 9\n"
284 |           ],
285 |           "name": "stdout"
286 |         }
287 |       ]
288 |     },
289 |     {
290 |       "metadata": {
291 |         "id": "JU1OmrO0uHpO",
292 |         "colab_type": "code",
293 |         "outputId": "d8cb4c76-6112-4327-f334-6e9669e0b990",
294 |         "colab": {
295 |           "base_uri": "https://localhost:8080/",
296 |           "height": 311
297 |         }
298 |       },
299 |       "cell_type": "code",
300 |       "source": [
301 |         "import matplotlib.pyplot as plt\n",
302 |         "%matplotlib inline\n",
303 |         "\n",
304 |         "for n, ep_rewards in enumerate(run_rewards):\n",
305 |         "    x = range(len(ep_rewards))\n",
306 |         "    cumsum = np.cumsum(ep_rewards)\n",
307 |         "    avgs = [cumsum[ep]/(ep+1) if ep<100 else (cumsum[ep]-cumsum[ep-100])/100 for ep in x]\n",
308 |         "    col = \"r\" if (n%2==0) else \"b\"\n",
309 |         "    plt.plot(x, avgs, color=col, label=n)\n",
310 |         "    \n",
311 |         "plt.title(\"Prioritized Replay performance\")\n",
312 |         "plt.xlabel(\"Episode\")\n",
313 |         "plt.ylabel(\"Last 100 episode average rewards\")\n",
314 |         "plt.legend()"
315 |       ],
316 |       "execution_count": 9,
317 |       "outputs": [
318 |         {
319 |           "output_type": "execute_result",
320 |           "data": {
321 |             "text/plain": [
322 |               "<matplotlib.legend.Legend at 0x7f52d2b3e5c0>"
323 |             ]
324 |           },
325 |           "metadata": {
326 |             "tags": []
327 |           },
328 |           "execution_count": 9
329 |         },
330 |         {
331 |           "output_type": "display_data",
332 |           "data": {
333 |             "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEVCAYAAAACW4lMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xm8DtUfwPHPPPvdufa1Eg6V9Esh\nIW2ULNn3vRAqW6hsyZpEyL6vRVRoIZUUsqTIdihkTe51uduzP/P7Yx66uNd9LNd63q+XV88zM+fM\nd+b2mu8zZ86co+m6jqIoinJnM93oABRFUZQbTyUDRVEURSUDRVEURSUDRVEUBZUMFEVRFFQyUBRF\nUQDLjQ5AuXkIIXTgL8CH8UPhDNBHSvldOtsWAFZKKR+4zH3MARZLKZcLIRoBX0spE9MuvwbH0Rco\nKqVsfcHyKsAqYH9wkRb87zQp5cir2N8s4E8p5eArreNaEkLEAD8CEUB5KWX8DQ5JuQWoZKBcqIqU\n8giAEOJxYLkQQkgpT6bdSEp5FLisRBAs1zLN13eAdUDiBcuz0iEpZYmzX4QQeYBfhBC/Sym/vU4x\nZLUHgRxSykI3OhDl1qGSgZIhKeU6IcSfwGNCiO3AeuAT4GGgFcavYYsQwgS8C9QLFv0F6CylTBFC\nrMG44NcF2gFDgWnAU4AA1gghWgODg8tdwc9n3Q28IaUcF0xOY4DsQBzQVEq5XwgRBswCygMHgT2X\ncYwnhBC/AA8B3wohCgITg7EBvC6l/Dp4VzEW+BaoAdiAJlLKX9LWJ4R4DBiP8as8ALwmpVwthNgM\njJBSfhrcrgYwWEr50AXlDwITgEZAYWCSlLJfcF3t4LmJAP4MHn+cEGIgUAAoDawAXgLyCCH2ABUx\nkvYHQDjG3V5nKeWW4HmvBcQAvwJfAcOAjcHlp4DOwHCgJDBZSjkgGEs/oDnGNWQ30FxKeToYS840\n8cQBtaWUx4UQRTD+TvmBBKCDlHJrRuc847+akhXUMwMlM1bAHfycE/hdSvnEBds0BJ4HygD3A9mA\nbmnWlwHul1KuP7tAStk2+LGKlPLnNMs/lVKWCP56bw8cA+YJIaKA5cBbUsqiwIfAomCxNkBe4F6M\npFM11IMTQpTASExnY5sdPMbiQPXgvnME190HbJJSCmAIxgXsQlOAkcH4hwOTgssXAk3TbFcH+DiD\nsB4DymKcy85CiNLBC+lcjARUBPghTd0EY60upRwEtOS/OyAXsBh4Nfj9PWBBMIGDca46Sil7Bb8/\nDHyOcS4DGIntBeAZ4C0hhEMIUQboAjwKFAPswe9nNQC6Buv4Fzj7t54CLAz+/YYEjwcufc6V60Ql\nAyVDQojnMS6y64KLrMBn6Wz6AjBbSpkipfQDMzn/gvyVlDJwmfvOjnGRaCGlTAAqAUfONuVIKRcC\nRYUQhYHKwFIppS/YPr7iElUXFkLsCf47DnyJ8Ut0nRAiAngSGB3cx5/AT8HjA0jmvwS0BHhICBF+\nQf0PpdnmJ6BI8PMnwHNCiBghhBmomWa7C82RUvqllP8G66gAPAeskVLuCG4zCagVrAtgo5QyLp26\nymGct3XBY1qCkdTvDq7fK6Xcl2b701LKNVJKHdgJ/CilTA1+NgO5pJS/AoWklInBv+v6NMcJsFZK\n+Xewjt8wzrkD49wuDG7zBVAuhHOuXCeqmUi50BohxNkHyAeB56WUyUKInIBfSpmYTplcGLf9ZyUA\nudN8P3UFcUwHZp69iGHcbdwbbPo4yx3cdyxG80fa/UdlUO+5ZwZCiGrAR8DS4LoYjIfK64U422JB\nJPA9cAhICF7gAE6niSutZsBrwTsZc7A+pJRHhRCbMO5c/gIOSin3k7605ysBo1kMoPIFx38GyJFO\nmbQu/Nucjf3s3+fCcklpPvsxEiBSSl0IEQDMwQQ4Oth0Bsb5//KCuNLWYQ5uc7ZTAsHzmCyEyE/G\n51y5jlQyUC507gHyZTjBfxclgp9PXGkAQohXgnWkfXZwDNgtpXwkne0TMC7kZ+UKZT9SypVCiCMY\n7eKjMZo0/MAjUsrkC/ZRhfOP8ewF+lSabQoAU4FyUsrfhRDFgL1pyizEaEL5E+NOISM503yODe7D\nBayWUta/cOM0F9H0nPe3EUJowTpPACUyKpSJrhjNQ2WCPxSGYDwjuJR4QA/GEheM417gbzI458r1\npZqJlGthBdBcCBEuhLBgPCj+MpMyYHRhPe+XtRDiAeBtoNkFTUsbgXxCiHLB7YoIIeYGLyobCDaZ\nBO9gql9G7G8BfYUQ2aWUvmDcHYP7CBdCzBBCnO2VEy6EeDH4uT6wRUrpSlNXLiAF2BM8D+2D9UQG\n1y/GeKBbn4ybiAAaCSFMwZ5OFTGaTVYClYLPDhBClBVCfBjC8W0C8gYfbAM0Bo5g3PVdqdzAnmAi\nuAvjfEdeqoCU0o3Rrbd1cFE1jOZDL5c+58p1opKBci18itET5VdgB3AYo+dNZhZhNA80TLOsG8aF\nZXWatv1RUkonxkV0nBBiN8azi8XB5oapGM0P+zGafNJ7rpGu4EPt9UDf4KJXgCeCzTFbgf1SysPB\ndQeBikKIvRhJpNMF1W3DOA97MRLUcoyeVT8G93UKWAscSFNnenZiXMR3AmOllDullMeBl4HPgsc/\nnkvfXZw9vhSMB/zjg8fUCWicprnrSkzCOEcSGAV0B54WQnTNpNxLQE0hxH6Mu76zD9Qvdc6V60RT\n8xkoSuaCzUTTgj1hrqaeCcAOKeWEDNYfxOim+XN66xUlq6g7A0W5ToLPEKoD8290LIpyIZUMFOU6\nEEIMwmgz7yKlPJPZ9opyvalmIkVRFEXdGSiKoii36HsGJ08mXfHtTPbs4SQkpF7LcK6ZmzU2Fdfl\nUXFdvps1ttstrly5orSM1t1xdwYWiznzjW6QmzU2FdflUXFdvps1tjsprjsuGSiKoigXU8lAURRF\nUclAURRFyeIHyEKI9zCGHrZgTJqxGWMMczNwHGN4YrcQohnG4FcBYIqUcnpWxqUoiqKcL8vuDIQQ\nTwIPSCkfwxiLfQwwCPhISlkJY+TGtsHxzPtjTJ5RBegmhIjNqrgURVGUi2VlM9FajOF6wRg/PQLj\nYr8suGw5RgIoB2yWUp4JDka2Dng8C+NSFEVRLpBlzUTBGa9Sgl/bYYzmWC04lC0YY8fnw5hJK+1k\n62eXZyh79vCr6lqVK1dG857ceDdrbCquy6Piunw3a2x3SlxZ/tJZcBLvdhjTIKadXi+jlx8yfCni\nrKt5CSRXrihOnkzKfMMb4GaNTcV1eVRcl+9mje1miuvvvzVmz7bStKmXxx6LvKK4LpVAsvoBcjWM\niUqek1KeEUIkCyHCgs1BBTBmrzqGcXdwVgGMMeAVRVHuGLoO48fbWLv24laPQAA2bzbjcmkULRrg\nscfSqeAqZVkyEELEACOBZ4KTegCsBuoB84L//QZjBqtpQohsGDNfPY7Rs0hRFOW24fWC253x+gkT\nbLz/vj3D9XnyBBg61EWTJr4siC5r7wwaYczluijNHK2tMC78HTDmPp0tpfQKIfpgTOunA++oIX4V\nRbnVJSXBxo1mVqyw8PPPFo4c0QgELt0KbrPpeDzpb3PihInu3cOwWp106XLt483KB8hTgCnprHo2\nnW0/xZg6UVEU5ZYWF6fRv7+dpUst5y7+OXIEKFvWT1Q6TfZOJyQlaRw7pnHypIly5XzkDk9CS07G\nlJCAFncSLTUVzePBZvbxSEI4UPOax31LjlqqKIpyM/rlFzNt2zqIizNRsqSfZ5/18cwzfsqW9WNK\npyP/zp0mXnghnNRUI2k8V83D4rxdiJw99dw2usWC/6670WNzoNvtpJQdmCWxq2SgKIpylbxemDHD\nyqBBdnQdBgxw0bGjF7MZTpzQmDzZitdrXPBNyYnYF3+M6cQJJvrak0oEfbTh3Kvvo+nKBYTjQtc0\nMJtB08BkwnMkjm8PlWaTVo6G25PIXu3aH4NKBoqiKJdB12HVKjMTJthIStLQdTh1SuP4cRPZsunM\nmOGkYkU/phP/4H9vHA0+e4M9yYXS1JALePXct0H2wfRz90M3mcBqJWAKB7sdNNDDI/jHXpgBh1/m\nkDc/HmzkPpCd0llwXCoZKIqiXCglBS3gB08i5n2HjL6dwOHDGh99ZGP/5tOU1v4kwuYFjB/xD5T1\n81zpo2TvvRwt6Qz6yVM09s9nD4V4ianUZelFu8lJHI+4t3CkcDkmFxtJ7A+fkxwIZ5uzNC7sdEqY\nSHW+Zhrrz5VJyjsUsiAdqGSgKIoC4PMR+XYvbF9/ifmf4+cWpx0oLZY0vWJ0wA0+zJwgD95NVlI3\n6TjRASu9mM0S6lOZtYwt9B6UeRA9Ry70iAgC+fJjXbsG54Y4/oh6jrrHJpFwKJJPeIdK/ASAiQBW\nfGyhDN9ThWiSCGCm9LrULBmvRyUDRVEUIGLIO4TNnEYgV248VZ5Ct9lwmmL4eXcsB49Y8fk1oqJ0\nypfzk7tkNg5EP0j/5RVYvSM//kD6w7w9HrWNpTlew37qJHs+381xTgOg8ztfUJv8PIDjtIvOjKEB\niynIUfZThBPkBuA3/kdP3seNHTN+HLiYdPdGlQwURVGuObeb8LEfEP7Rh/iKFuP0N9/zrzsbs2db\nGTPGjscDRYv6adTIhzcmQK1Jdvav/u/iX9r0ByW0Xdh1F57YPPweeBCX30ZYSjyHk2IombwaFw6S\niDxXJi/H+YlKFOWv80IZw+t05wMqPB7g1Vc91Knsp4HFD5wdgkcDymfJaVDJQFGUO5bpn+PE1K+F\nZa/EnzsPibMWsHhVDrp2deDxaOTJAyNHplIjx3r6D45iyIZyWE1+Spl34fPrlGYbJQO78GPhOPn4\n6lQZ4siFCT8v8iOvMJF8+nEs+LDixYIPCz6iSSSSFH6wPMuegk8SKFyY1BL/Iz77vcy6z81zz/nQ\nMh2l7dpSyUBRlNtXcjLW334FTcNbsfJ/y3Ud815J9EstseyVOFu0IWXAIDbvzU7Xrg4cDqN7aKdO\nDkwL5rOw1a+M1ycB4A2Y+YP7ANjN/Rft0oSfRTSkXvCBsdcRgeZyYULHjwk/ZnTAawun9OrBPFCi\nZJrSnqw6E5lSyUBRlNuSZfNGYhrVxZRsjO6ZOGEq7udrEDZvFmETx2M+dhSA1PavkDhwOHPm2hg6\n1I7PBzMG7qPQhAEs62tnlf4M85kYrFXnPXrRkEVoBBhFD5bm6sBD5cwUDIvnPs/v1Dg4kXu2LSfx\nrvuwVipL2LxZBHLmxF/4LvyF7sJX6kGw2nA+UxV/seI36OxcTNN1/UbHcNlOnky64qBvpiFpL3Sz\nxqbiujwqrst3rWPTTpwg+zOVMJ38F2e79jg+WYjmTEW3OzAlJ6GHR+B+7nk8VZ4mqU5TOnQM48sv\nrVg1LyVtf5HgjuAwhdAIoGPCjI+i7GW8rQfPeL65rFj8+fJzevlKAoXvumbHd6XnK1euqAwbn9Sd\ngaIotxevl5iXWmI+8Q/J/d/F2eV1PM+9QEzT+ugxMaS80gVn2/b4s+fghx/M9H/Kzr59ZkDHq1vZ\n7i5xrqownFRmLaPpRglkuq04OoDNjh4dRSBHLvx33433kbLokZGgmfA8V51A/gLX6+ivmEoGiqLc\n8kzHjqIlJgIQNmMK1o0bcNWqg7PzawB4Kz1B/M4/0SMiwWxG16HTKw6WLrUGa9DR0CnOHhbTgLyc\nwI8Jk8VMTt8/mNCNi77ViqdCJdz1G+J94kkCkVFgtRpvDN/iMk0GQogyQD4p5QohxBCMfk0DpZQ/\nZXl0iqIomXDMnEZknx5oaZq8fSVKkjTmI9J2ydGjY859njnTei4R3MufDKIv92iHeEzfgA8zf9uK\nUdj7F1afF91kIqVzV1K794KIiOt3YNdZKHcGY4HWQohKwKMYg2qMB57KysAURVEyY9m0kci3e6HH\nxuKqVQcAPSwc50sdIDLyvG3dbvD/sZvvR+3mze9aAPA/trKJsljwgw5Htfzk049xr2ePUSh7dhI+\nXYa/VFaMBnRzCSUZuKSU+4QQ7YEpUspdQohAVgemKIpyKdqJE0S3awGBAIlTZuGt9ESG265bZ6Zl\ncxtJKWWBsgDkII61VOYPSrE26nk6J42ggH4M3RFG0sDBeCs9QWyZB/An3rjuntdTKMkgQgjRAKgD\nvCuEiAWyZ21YiqIol+D1Ev1yq3MPiTNKBLoO8+ebeaNnGP40P2GjtGS26v/Dq9kpqB/m9aRhAHgq\nVOTM/MX/NQfZ7dzIvv/XUyjJ4E3gdeAtKWWiEGIg8EEolQshHgC+AEZLKccLIRZjjN8KxphPvwBD\ngT+AX4PLT0opG4R+CIqi3FFSU4nu0AbbL+tx13zx3ENiAHw+LDv/wFdMsPKnKLp2dRAf/9/QERo6\n4SYnmwMPU4Bj6LqOBZ1A9liS3xmMu1Ezrvurv5dw4oTG3LlWkpM1Tp7UOHjQxFtvuXnxxWu/r0yT\ngZTyB+CHNN8HhlKxECICGAd8l6ZsgzTrZwDT/lslq4QUsaIodywtKZGYRnWxbtmE9777cT/5NI4Z\nUzAfPYqWmoJt9So8h07QOuwTFjlrEez4SW7+IV7Lhab7+V57BsE+/GiYTRqB7LEkrPqRQKHCN/bg\ngC1bTJw4YaJo0QBff21h3DhjzoSzIiJ03O6s2XeGySD4XCCjl7t8UsrM+lK5gepA73TqFkA2KeUm\nIcTdIcaqKModTEs4RUyLxli3bEK327Hu2om1+6vnbXPYfi9Ph+9jX2rBs6XIZzrBQ4GttNen8Czf\nEuE3Bn0zmU2gaSROm3PDE8HRoxpvv23nq6+s5y2PidEZNszFww/7iY7WuftuHbM5a2K41J2BFWOI\nvLeB7cD3gBljQvtM36GWUvoAn3Hdv8jrGHcNZ+UVQnwK5Ac+klLODyl6RVFuf4EAYeNGE/7hKEzJ\nycYyk5nUV7sZcwNHReHPX5DdcTmp17sU//z732Xt6fw76HysH3X4HADdbscfkwdTYiJ4PSROnoH3\n8Uo34qg4elRj0SIryckwe7aNxESNsmV9VKniZ+9eExUq+KlTx0tMTOZ1XRO6rl/yX/HixX9IZ9nX\nmZVLs+3A4sWLd0nz3Va8ePHtab5HFS9evE3x4sWtxYsXz1m8ePH9xYsXz3epOr1en64oyh3A59P1\nNm10HXTdZDL+W7u2rh87dm6TQ4d0vUsXXdc0Y/XZfw34RB9Kb10HPdUUruslSuh6dLSxMjpa1xcv\nviGH5Hbr+mef6XrOnP/FGhmp61On6nogkOW7z/C6Gmpvog7Az0AAqADBmReuzBPAprNfpJRJwMzg\n1zghxBagBHA8nbIAJCSkZrQqU3fS+CzXiorr8qi4Ll+6sek6kd26ELZgLrrJhBYIkNz3HZyvdgVN\nI+VgEq1ahbF27YWXsQC1+RwfFt5kBCnZ8uGIssKePfiKC3zPvUDKm/0IFCgImZyPa3XO9u/XmD7d\nxtatZnbvNpGaqmG16gwa5KZUqQDFiwfIlUsnLi60+q5ibKIM14WSDJoDA4DOGM1GO4GWlx3Ffx4F\ntp39IoR4EqgppewefOj8ELD3KupXFOU2ED6oP2EL5gKgx8SQ9O5w3A2bAHDypEb79g7WrTMuYRaL\njs+n8T/HLlq4pjCVl9mklcdvC8MebcN86G9SX+tOytsDrqq3kN8P335r5vhxEyVLBtizx8T69Wa2\nbjWTL1+Ahx4KUKqUn5MnNbZvN7Njh4lTpzROndLQdSMBFC0aoGJFP40beylV6uZ5ZSuUZHCvlLLZ\n5VYcHMZiFHA34BVC1AfqAvngvOl9fgJaCSE2YDyTGCalPHq5+1MU5Ran69gXf4xl2+/YV32N+e+D\nALgaNSV56Hvsj4thyps2du40sXmzGb/fuKjniPUTf8pMectm3ne9TgfTNL4PPE2know/W14sh/4m\ntWOXq0oEgQB88YWFESPs7N9/8RSXsbEBNm82s3Hj+ZfU6GidvHkD3HdfgFatvDz/vA+r9aLiN4VQ\nkkF3IcS3wQfCIZNS/gpUSWfVqxds5wNaX07diqLcfqLat8bxxWfnvusOh3E30Kotq1aZ6dQpjMRE\nDU3TyZdP59gxDbtdJ/6Umaqs5MPwvkz0tGWbqxRmjF/c5hP/kNqhEynvDLmiRJCQAEOG2PjySwt/\n/mnGatVp3txDmTLGXUGxYgEef9xHkSI6qanwxx9mdu40kSuXzoMP+rnrLv1mem3hkkJJBqeBXUKI\nraR5FU9KeTVNRYqiKGinEwh/fzh8tRzHkSPGDGAVKuJ9tCypPd8kYLUzcoSNUaPsOBw6Y8Y4iY/X\nePddBxaLjtut8TJTeKvaL8zb3oIxiZ3QAG/p/5HS523895cikDffFcUWF6fRpAls22YnLEynfn0v\nvXu7ueuu9HvcR0RA+fJ+ypf3X/kJuYFCSQYrgv/SuvVmxFEU5aZi2fYb0a2bYT56BADdZuPM9Dl4\nq1UHjKaZV191sHixlcKFA0yf7mTpUisTJ9qw2XQ8Ho2XmEqvx9cw8bdnGf5vWzTAVb0mSTPnXfad\ngNMJEybY+PNPEykp8MsvFk6fhlatPLzzjpvw8Gt9Bm4uobyBPDvtdyGEDZgPzMmqoBRFuX1ppxOw\nL1lEZL830XzB1ufq1Yn/cBJ6TDbA6HD5xht2Fi+2UqaMn2HDnPTv72DDBgvR0QESE03U5nO63fsF\nY3Y1ZmxCSzTA/XRVkmbMvexE8McfJjp1ciDlf290FSgQoHt3jQ4d3LdMU8/VCGU+gxYYYxHFBhcF\nSDPEhKIoSqbcbsKmTcKxYB6WffLcYh1I7f02EUMHocenGMt06NvXzty5NkqV8tOvn4v69SNITNQo\nWMDPkaNmyrCFNwvMYvuRWMa5W6ABnnKPkTj3YzBd/IA3PcePa3z1lYW//zYxfboVr1ejXTsPr7zi\nweGA3Ln1YBfOLDgfN6FQmoleA0oBHwMvAM2AM1kZlKIotwftzGm0hASiO7TB+tvW89Z5KlQk5e0B\n+B4tR0TwAq7rxgPbqVNtCOFn2DAXrVqFkZQEDz/kZevvVsqwhRmF+1Po8C+U0xPQAWfLNiQPHQmW\nS1/SdB0OHNDYts1Mnz4OEhKMn/y5cwf48EMnTz99a7b3XwuhJIMzUsp/hBBmKWUKMEUIsRL4JItj\nUxTlFhY2YRwR7/ZH8/93gXVVr4G7YVO8Zcuj58x5UZnRo22MHWunSJEA48e7aN06jPh4Ew8+6GPr\n71bq8CkzI18j5tBxdCCARtLsBXiefyHTeAIBaN/ewbJlRt9Ou12nf38X998foEwZP9HR1+zQb0mh\nJAO/EKIGcDg4fPVO4K4sjUpRlFtPaqrxy9xqJXzUCCLeG4o/V240rxfT6QRSur1B6pv9Miz++ecW\nhg+3U6hQgFmzUunQIYyjR03cd5+f7dstLKU2L7IMLTg8kY5GygdjQ0oEAEOH2li2zErp0n6eespH\n7do+7rvv5nnp60YLJRm0wHhRrCswGPgfF7wroCjKnc3090Gy1XoOAgG8FSvjWLIIf/4C6GYz5pP/\n4mzWktQ+fTMsv2MHdOvmICJCZ/ZsJ717O9i920yxYn4O70piF+UpiUQH1mqVOd66J890EyF3G50+\n3crYsXbuuSfAokWpZFfTc10klGTQF1gJ7JNSts/ieBRFucVocXHENKqD+fgxdIvFSASFCkNqKpb4\nOFJf70HKW/3T7eHj959tGgKXS2PyZCcffGBjwwYLVcPWMnFfKwpzGAt+PFh56655NFpcnafu1gnl\nN31CAkybZmPkSDu5cgVYuFAlgoyEkgzWATWB4UKIkxiJYaWU8vcsjUxRlJuf00lMy8ZY9v9F6mvd\ncTVpRtiHH2Bf9jmaM5Wkoe/heqljukV1HXr1MnoN5c8PAwY42fS9kxUrYilr3coXzqr4MGPBz7/k\nousjaxiyqBCRkZm/5uR2w6RJNkaPtpGaqhEbG2DxYidFiqhXpDISynsGnxB8WCyEKItxpzAklLKK\notzGAgGiXu2IdcsmXHUbkPL2AOxfLMWxZBFoGklTZuKuXTfdom43vPXWf91Hf5ywh3GtNzL9r7aU\nYjsrvU8RTw4KcIyD3MXLRb9jyqLcREZeOqTjxzV697azdq2F1FSNnDkD9Orlplmz6zgvwC0q1PcM\nnsAYVvoo8C1GQlAU5Q4WMXQQjmWf4bu3KKYjh4lu1RTbyq/QIyJJnLMQb8XK6ZY7cwYaNw7n11/N\n3HefnyX9NvJ51fkMcH5EAe0og/R+mPFTgGNsoDwv51zK3EVRmd4RGPWGnXvW8NxzPl57zaOSQIhC\n+XU/EtgKfAT8IKX8J2tDUhTlZudYMJfwsR8QiIrC8tef8NefAPhz5+HMwiX4Sz2YbrlAALp0CePX\nX83UretlTKcd/F7rQ152zieG03yjV+UBduE3W3lHG8Roay8+XeijYMGMnxAkJMB779n56isLx4+b\naNfOw9Chd8Zbw9dSKM1EeYUQDwBPAh8JIfIB26WU6TcEKopyW7Ns/IXInq+jm0yYkpLwPvgQSZOn\ng6bhz5ufjAbxcTphyBA7K1daqFzZx4Th/3L46YE0Tl2AFS+f8yL3cJAtHcfTbFED/jydizlTnZQu\nnXEiSE2FZs3C2bLFTEyMTocOHgYOVIngSoTa7n8Q+BNjjuJcwP1ZFZCiKDcv09EjxDStBz4fGuBs\n157kgUPAbr9kuT/+MNG2bRh//22icOEAM5p8ianGCBoenkcqEcyjGY+zjh+6LabLiprsO2Vm+HAX\nVaum/0aw0wnz51v5+GMr27cbdxnjx7syewFZuYRQnhn8CoQD3wOrgOFSSjUchaLcaU7+S/anHseU\nlETA7iBxwhQ8NV/MtNjq1WZeeikMpxNeafov7xxtT45XllGT5RygCN34gGYs4Lu2M3lnYw327TPT\nqZOHtm296daXlATNm4exYYNx+apVy8vYsSoRXK1QTl89IBm4W0q5RQgR2ihQiqLcNsy7dpKtRlVM\nyUkEYnOQ8PV3BO4pkmm55cstdOjgwGqB+bXm03jJS2huN/0Kz+SrQy9QlW8YRh+2d5zAxJPNWL/e\nQs2aXvr3d6dbX3y8RuPGYWzbZqZWLS9DhrjJk0d1F70WQrmwlwc2ALOC38cJIdplWUSKotxUrN+v\nJvuzlTElJ+HPk5f4TdtCSgQpjWVtAAAgAElEQVTffGOmfXsHdluALwu/TJMvWhDIHsviLqsYfKg1\nd3OAYbzJgpdXsSjHKyxZYuWRR/yMH++6aOBRXYdffzVRu7aRCJo39zB5skslgmsolDuDHkBp4Mvg\n957AGmB6ZgWDD56/AEZLKccLIWYBZYD44CYjpZRfCiGaYQx3EQCmSCkzrVtRlKxn++Yrots0Q/P7\nCWSPJeG7nwllRLctW0x06BCG3arztaMOlfYux9WgMX+0HU77F7LhwMkw+jCzxiKKlCzEkO5wzz0B\n5s51EhZ2fl1Hjmi0bx/Gli3GXAMdOxqTzaiHxNdWqKOWpgohAJBSOoUQnkzKIISIAMZx8dwHb0op\nV1ywXX+gLMa0mpuFEJ9JKU+FeAyKomQB884dRL/UEvx+ApGRnF66Aj137kzLrVplpmPHMDweWJqv\nE5WOLCe53yCONuxMq//FcToQQx+GMuuBkbRskpOWLe3kyAELF6aSI8f5v/Q3bTLRunUYcXEmqlXz\n0a6dhyee8KtEkAVCSQZxQohWQJgQ4mGgERDKdA9uoDrQO5PtygGbzz6UFkKsAx4HloewD0VRsoDp\nn+PENKiF5vGgh4Vx5rMv8d//QKblFi+20KWLA4cd5hYdQC05mdT2r3Cy5au0KHWI3b4HqcsSPivQ\nhfeGmGnWLAyLBb74gouGivjkEws9ejjw+2HoUBft2nlVEshCoSSDjhijlUYB04CfgJcyKySl9AG+\ns3cUaXQRQnQH/gW6AHk5P7n8izFKaoayZw/HYjFfapNLypUr6orLZrWbNTYV1+W5peM6fhyefwri\n4sBmQ/vuO7I/9limxdavh27dIDpa57sK/Snz9WCoVw/PkLE0vVey2fUgz/MlP4dXZeqEKF5+GZKT\nYdEiePxxMC4xhvfeg969IVs2Y/2zzzoAxxUf99W4pf+WlyGUZPCYlLLLNdrfXCBeSvm7EKIPMBBY\nf8E2meb+hITUKw7AmMYu6YrLZ6WbNTYV1+W5lePS/v2X7NWqYD56FN1i4cyiz/EWfQAyKXf4sEbt\n2uH4/Rqz6y2hzIzBeMs8yrEhE2go9rMp8X5q8zk/Upk+/a20axcgLs7EiBEuqlTxAv/FNnasjcGD\n7eTPbwwuV6xY4IZNPXkr/y0zKpeRUHoTdRdCXJMevFLK79KMdroMYzrNYxh3B2cVCC5TFOU60k6e\nJFvNZzEfPYJuNnNm4RK8FSpmWu7MGWjRwmjXH9ZhH7XmNiWQMycJU+fSpWYcm04Vpx6fsplHqNUy\nkjlzrMa2w1y0aXP+uwTLllkYPNhOwYIBli9PpVgxNfnM9RLKRf40sEsIsRXjAS8AUsqWl7szIcQS\n4A0p5X6gCrAD2AhME0JkA3wYzwu6Xm7diqJcOS0+nmwvVsdy4AAASROm4n3iyUuW0XWYNs3KqFE2\nTp0y0aZJIt2XPQM+H4kTp/Nuby8r9j1AFX5gFyV58Lk8JCbCrl1mWrXy0K7d+YlgzRozXboYE9zM\nm+ekUCHVbfR6CiUZrAj+uyxCiDLAKOBuwCuEqI/Ru+gTIUQqxotsbYK9k/pgzJOgA++oN5wV5Tpy\nu4lpWg/LPglASs8+uOvUv2QRp9OYmWzpUivR0Tpv9XHy9ob6mI8cJqX328xZX5zx35agJLsw46Nw\nNUHZsj4GDXLwyCN+hgw5/6WyVauMuwuA6dOdajrKEIwdO4qdO3egaRqvv96DkiWvbpSgUAaqm30l\nFUspf8X49X+hJels+ynw6ZXsR1GUq6DrRPbrg/W3rQC4q1UntWefSxYJBKBjRwdff23l0Uf9zJyR\nSpFhnQn78VvcFSux/meNnuuKEEs8j7CZsPZNqFrVQ6NGYeTOHWDGDCc223/17d1ron59YyK02bOd\nPPlk+uMRKf/ZtGkTR44cZvLkmRw8eIBhwwYxefLMq6pTjeahKHco+5JFhH/wHpZ9ewHwFbmXpI8m\nc9HrvxcYMcLG119bqVjRx8KFTrKPG07Ygrl4HyzNgfhsNNn9Ojoaz7KKXN0a06y5h6pVw9E0mD7d\nRd68/zX/nB3ALikJJk1y3XKJIGJgX+zLP7+mdbprvkjKwMGX3GbDhg1UqlQFgLvvvoekpERSUpKJ\niMhk9p9LUOMMKcodyPbtN0R1ehnzwQPoZjN6eDiJsxeiR196JpilSy2MHm3n7rsDTJvmJOrbZUS8\nNxR/4buIL/EYdXcPJ56c+LAQ0+5FurzqoU2bMOLjTQwZ4qZcOeNi7/XCu+/aePbZcP7+20T//lC3\nru96HPptIS4ujmzZsp37ni1bduLj4y9RInMh3RkEh5UoKqX8XAiRTUp5+qr2qijKDWPes5uoDu3A\nZiOQIyfmY0c5M24yflHikuV++81E164OIiN15s51kmfXWqI7v4weHsGZanVpNbUau7kPCx4qlPfT\n/x0fHTo42L7dGEuodWvjgfGxYxovvxzG5s1m7r47wMiRTurXD79h3UevRsrAwZn+ir8edP3qH7aH\nMoR1N6AJYAc+B/oJIRKklDf+DCiKclm0U/HEtGiEKTkJ74OlsW7fRuqr3fDUrH3JcsePa7RsGYbb\nDTNmOHkg4WdimjUAv5/4Fh1pNrUaq6hGGKnkymtiyiwfI0bY+PJLozmpYkU/JUtGULx4gL17TZw6\nZaJOHS+jRrkynddYuVju3LnPuxOIi4sjZ86cV1VnKM1ETTBGLj07VtAbQI2r2quiKNef10t0u5aY\n/z6I57EKWLdvw1P5SVLe6n/JYqdPQ6tWYZw4YaJ/fzfV7tlDdMvG4PXyb/teNJhek1VUI4YENJuN\nOR/7WbnSwrhxdu69N0CVKj46dgwjJUVj40YzyckaI0a4mDRJJYIr9fjjj7NmjTHsm5R7yJkzJ+Hh\nEVdVZyjNRElSykCageoCQgjV70tRbjWvvYZt3U/4CxTEtmE9/kKFSZwyA8wZD+1y4IBG06bh/PWX\niSZNvHRu9A/RLzTAdPo0cS91o8FHVfmJSuTnCMcoyIxJTk6f1ujZ00G2bDqtW3vo189BnjwBFixw\nki+fjsmkExt7HY/7NvTwww8jREk6dmyLpml0757ZEHCZCyUZ/CWEGABkF0LUxRiobtdV71lRlOvG\nMWMqTJqEbjJhPnoEX7HiJE6eiR6bI8MyJ05o1K8fzuHDJrp0cdP3jSSyNWyK5cB+nDXr8NqMsvyk\nV0KwB0kJevZ0U6xYgFq1jDmQ69XzMniwnchInU8+Ue8OXGuvvPLqNa0vlGaizkAKcBRojvHGcOdr\nGoWiKFnGunYNkW+9AWYzWiBA0vsfkvDTJvwPlMqwTEoKNGsWxuHDJnr1ctO/n5uYHl2wbtyA59Fy\njPryARYGGlOQw0hK8PzzXqpU8VGrVjgJCRoFCuhMn27DboeZM1UiuBWEcmfgBz4I/gOMpqIsi0hR\nlGvGvP9Potu1MMaOCARI7fQarpZtLllG16F7d6MXULNmHnp0cxE+YiiOTz/BV0zw7a856R8YiB0X\nRyhEyRJe2rf30LBhOG43VKjgY/16C9Wre3n/fTc5c6phJW4FoSSDFMCWdoEQQgf2AR2klGuzIjBF\nUa6OlniG6BaNMZ0Jju5StSop/d65ZBm/H4YPt/HZZ1bKlvUx+tllxD7WC/PBA/jz5mP/ISvNA3MB\naJb3W5qMKwPZYmjYMByXC555xseqVVZKlfIzcaLrolnLlJtXKMlgAMZgdZ9iTEtZF4gG1gIfAZkP\ndK4oyvWVmkp080b/vV18TxEsH38MvowfFp8+bYwPtHGjhYIFA8xp+Bk52zUBiwXni3Vxr91KPfcC\nTpOdZ0r8zbA1T7Bjp5mGDcNISNC45x4/q1ZZKVLEGHJCJYJbSyjPDJ6XUk6RUp6SUp6WUs4Aqksp\ntwLezAorinKdud3EtG6K7Zf16GYzgcgoEud8DNmzZ1jE64V27YxEUKOGl596fEzx3k3Qw8I5PecT\n2LmHVqfGsINSiNgTzP8xlgMHTTRoEMapUxolS/o5cMBMrVpeVq9O4a67VNPQrSaUO4MoIcQLwI8Y\ndwYVgAJCiFLcqKmHFEVJn9dLdPs22NZ8j+5woLlcJE6Yesm3ixMSoEcPBz/9ZOG557zMqzyR6B7d\n0CMiOTPvE8KHDabzvm4sozax1iRWbraTlAQtW4Zx6pSJ++/3s3OnmSef9DFxogur9fodrnLthHJn\n0B7oAxzHmJ5yOMZ0lTmAblkXmqIolyvyrV7Yv15BICYGzeUipU9fPM9Vz3D7vXtNVKoUwYoVVh4t\n42GeuwExvV5Hj4omceY8wseNZvwv5ZjOS9hNXr7doGMPN9O+fRh795rJkyfAzp1mKlb0MX26UyWC\n62j//j9p2LA2S5Z8ck3qC2UI661ApbTLhBD1pJQXDUWtKMqN45g/h7DZ0wnExmI6dQp3zRdJ7fZG\nhtsnJUHr1g7+/ddEn94u3t7ZnMgVXxhvJfd6k6guHfn+4L304j1MBFiwyE2hwhr9+tn5/nsLsbEB\nTpww8eKLXsaPd503LLWStVJTUxk9eiRlypS9ZnWGMjZRYYw7gbMDX9iBp0hnXgJFUW4My9YtRPbu\nTsDhwHTqFL6S95P44QRjkoB0+HzQubODP/8088orHt62jiByxRI85SuQNPQ9stWvxcF/wqjDZwQw\n8VaPRCpVNjF/vpXJk21ERemcOmWialUfH3105zYNDRxoZ/nyazsTQM2aPgYOdF9yG5vNxvvvf8i8\neVc03Uy6QmkmmosxLtFjwK9ALqDFNYtAUZSrop08SXSb5uDxYHK58OfNx5k5C8lo4J9AAF57zcE3\n31ipVMnHkALjiRw8EH/efCQNG0W2xnVx/pPIE/xIKhE885SHrr1NbNhgplcvO3a7TlKSRrlyPqZO\nVU1DN4LFYsFuv7aPbENJaT4p5XAhxHNSyo+EENOBhcDqaxqJoiiXz+sl+uVWmI8fA8BXtBhnPvmM\nQKHCGRYZNMjOp59aeeQRPx/XnU/2bj0I5MxFateeRL/WAdORw9Tke45SkIIF/cyd7+HgQY02bRz4\n/eD1atx3n59581T30YED3Zn+ir9VhJIMwoQQBYGAEKII8DfGvMaZCs6D8AUwWko5XghRCJgJWDG6\npTaXUv4jhPAC69IUfVpKeWtNeaQoN0DEoP7Y1v+MbjKhx8Rkmghmz7YyYYKNYsX8LOr5E/laticQ\nGUUgdx6i+vQAoD8DWcOT2O0633yTSmrqfz2HzGadHDkCzJvnJObS8+Aot5hQksF7wNPASOB3jOEp\nFmRWSAgRAYwDvkuzeDAwRUq5SAjRGegO9ALOSCmrXF7oinJnsy9ZRPjkj9AtFvD7SZw045KJYN48\n6NXLTmxsgAUj/+Kul4z5CLzlK2BfvRJPucdYuuku3tX7o6GzYIGTHDmMRLBnjxm7XcfrhcmTXRQs\nqN4juN2Ekgw2SimPAwghYoEoKWVCCOXcQHUg7diqnQBX8PNJ4OHLiFVRlCDzjj+I6tYF3WxG8/lI\n6f023iefznD7L7+00K4dREfDopnxPNSnDqa4OLwPPoR99Up8oiR795lors8FNN7o5aZSJT8DB9r5\n9lsLNpuO260xdqyTypXVTfuNtmPHDt59dwj//HMci8XCDz98x9ChI4nOZNrSS9Eymy5NCPG9lPKp\nK92BEGIgECelHJ9mmRn4HhgkpfxOCJEMLAPuApZIKT9It7Ign8+vWywZv1avKLe1U6fgoYfg8GHj\n+wsvwLJlGU5kv2kTVKlirP5htZ9Hh7wIK1ZAVJTRv7R8eVy6jXwbP+M0sTzzDHz7LcyaBW3agNVq\nvKE8ejR07XrdjlLJGul3LyO0O4O9Qog5wHrAc3ZhcFiKyxZMBHOB76WUZ5uQegLzAB1YK4RYK6Xc\nklEdCQmpV7JrAHLliuLkyaQrLp+VbtbYVFyXJ0vj8vuJaVAHWzAROJu2IHn4KIhPSXfzQ4c0Xngh\nHLdb44svNO6f3RVWrCAQGYkpKYmUbj3R/TqVxzblNLHkyRNgzpwUli83Xiwzm40Hxj16uGnWzJNl\n8xTfkX/Lq3ClceXKFZXhulCSgR3jOUG5NMt04IqSAcYD5H1SynPDJ0opJ539LIT4DigFZJgMFOVO\nFT5sELafjYGCUzt2IeWdIRm+S3DmjDEnQVyciWHDXNQ4NAMmGTfopuRknC3a4PvfI7RoGcZvPIzV\nqrNyZQrHjhk9h3w+0HWN5s099OrlSXcfyu0jlDeQ2wghTEBuKeU/V7MzIUQzwCOlHJBmmcAYGbUZ\nYAYexxghVVGUNGxfrSBi7GgA3E89Q8qAdzNMBGcHnpPSTPv2HjoWXQUNuwDgeeJJnB064StSlAGV\nNrKcRphMOp99lkpEBNSsGUZ8vNHkVKGCj+HD3RntRrmNhPIG8lPAdIwHwiWEEKOB1VLKLzMpVwYY\nhdEN1SuEqA/kBlxCiDXBzXZJKTsJIQ4DmzAGwlsmpdx0hcejKLcl8769RLU3JqXxFbmXpGmzM5y7\nWNeNXkNr11qoVs3H0KqriWlcD3Qdb6nSnFm4BC01hQWPz+Ijb19AZ8oUJw89FKBxY6PnEEDhwgGm\nT1fDTNwpQmkmGgqUBz4Ofh8CrAAumQyklL8CVUIJQkp59bM5K8ptSktKJKZBLUweN4GoaM4s/gI9\nMuO233HjbMyfb+PBB/1MeWMH2WvUQ/P5IG9eziz+HAIBfq/7Pl1PGP00Xn3VQ82afjp3dvDzzxbM\nZh2bDWbPdpIjh+pCeqcIZTiKZCnlibNfpJRxpHmQrChKFgoEiG7THPOxY+gmE2cWLrnkuwTLllkY\nPNhO/vwB5k2OI3+7OphcLnS7HVavRo/NQXzPD6i1fQg6Jio85qVvXw/Dh9v49FMrdruO368xbpyL\n++9Xs9veSUK5M3AKIZ4ANCFEdqAx/70roChKFgofOQzb2jUAJI0ej69suQy33bzZROfODiIidObP\nS6XYoLZY/j4IQOKUWcTcfz/e0bN49uMOOIkgZw4/8+a7mDfPyujRdsLCdJxOja5d3dSq5bsOR6dc\njQkTPmTbtt/x+/20aNGaJ5644jcAgNCSQSdgIvAo8CfwM8YcB4qiZCHrd6sIHzUCAGfz1ribNM9w\n24MHNVq1CsPnM5p3HlkzGsfXXwGQ8sabeB8pS6BOPWp8/jZHKIzdGmDJUhdbthiDzzkcRiJ49lkf\nffqoG/+b3S+//ML+/X8xefJMzpw5TZs2za5LMrgPqCmlVI2HinKdmA4eILpNczTA+8CDJI8Yle52\nTif89ZeJDh0cxMWZGDHCRTXHj0S8a3TYc1d9ntRXXiVbnRdoua0bv/EwFnOA5V86CQvTad8+HE0D\nl0ujaFE/Eyc6M3p3TUlHxMC+2Jd/fk3rdNd8kZSBgy+5zaOPPkr+/EUAiIyMwuVy4ff7MWfQqSAU\noSSDnsA0IcRiYI6U8vcr3puiKJlLSSFb/dqYXC4CMTGcWbiE9MaJ3rHDRP36xgByAG8338dLZeOI\nrtEQTdfx3XMviROmENWhLQO21Wc+LQCdWbNdFC0a4IUXwjl9WsNi0YmK0pkzx0l09HU+VuWKmM1m\nwoJDxq5Y8QWPPVbhqhIBhPaewbNCiNxAPWBM8LnBAinliKvas6Io59N1rD98R8TwwZgPHTQeGC/4\nFD1Pnos2/ecfjebNjUTQpImXWrFraTS1OszzoAGBiAjOLP6c8MGDeOnbZszHaGLq3t3Ds8/6adfO\nwe7dZsxmHU2DKVOcFC2qbv4vV8rAwZn+is9KP/20hhUrvmD06I+uuq6QpuiRUv4LTBRCbAHaAW8B\nKhkoyjUUNm4MkYPPvY9J8tD38D168QPjY8c0GjQI49gxE337uun64j6yV2sEXi8aoJstJM5fjH3l\nV4yZlfNcImjUCHr39jBmjI0VK6yYTDoWC8ya5eTpp9Xgc7eajRs3MGfODEaNGkdkBhMZXY5QXjor\nDzQAagH7gflAxhOrKopy2azfryZi8AACYeGYnKk4m7bA1ebli7Y7cUKjVq1wDh0y8corHl5rE09M\njUaY4uMB4/nC6eUrsfyxnXX9fqBf8HWg++/3M2uWmSVLzAwfbsNsNrqQTp6sEsGtKCkpiQkTPmTM\nmAlXNVJpWqHcGYzFGESuYtr3DRRFuTZMBw8Q3bEtWCyYnKl4H/qfMfjcBWNAeDzQrp2DQ4dM9Ojh\npldPF9GtX8Kye5cxuU10NIkz52GKj+NUm7dpGFiNjkZ0dIC5c50cPBhJx45GO7Pfr/H2226qV1dd\nSG9FX331FadPn6Zfvz7nlvXtO4i8efNecZ2hPDMoK4SIBGKFEIUxBq6bL6Use8V7VRTFkJJCTOtm\nmE6fRjeZCOTIQeKMeeC4eH7bAQPsbNpkoXZtL716eYgYORz7yq/RbXbwuEmcNAM9Mgp79RrUiZ9P\nIjFoms7MmS6io3WqV4fkZCPBtG3r4bXXVBfSW1WjRo146qnq17TOUJqJ3gDexkgCyUAYRlORoihX\nw+8nutPLWHbtQA8LA7ebxKmzCRQsdNGmn3xiYfp0GyVL+hk92oV99TdEvD8c3W5Hc7tJeas/3oqV\nia5Xiw4HevIHpQHo189NuXJ+GjUKY+9eo666db0MHaoGn1POF0qP4gYYA8z9IqXMBTQFdmRpVIpy\nB4h4px/2r1cQiI5GczpJ6f8u3oqVL9pu61YTb7zhIDpaZ+ZMJ9En9xPVqb0xy5nbjfv5GqS+2o3I\nN3syY+ODzKY1ADVqeOnUycvrrztYt8743ffUUz7GjXOpdwmUi4Tyv0SSlNID2ACklMuA2lkalaLc\n5hxzZxE+aTy6yYQpMRHXi3VxvtLlou327ze6kHo8MHGik3tznSGmZRNMZ06j+f347i1K0vhJOGZO\nZd3cw3TBmK+gaFE/Y8e6mDLFypIlxjsKJUvC9OnO9F5ZUJSQHiAnBOch2CGEmAnsAvJnbViKcvuy\nblhH5BvG/JF6tuyktm5L6ms9LnpgnJQELVoYk9OMHOni2cfOEFP7BSxyNwCBmGwkzlqA9Zf1HOs7\ng7raJgK6ifBwnXnznPzxh5mBA+2AjsMBS5dqRERc76NVbhWhJIOWGM1EnwFdgYJAk6wMSlFuV6a/\nDxLdvCEEAgRy5CBhzQYCeS7uARIIQOfODvbtM9Oxo4dWDc4Q07g+1j+2AZDSsw/exythW/k1nqFj\neSGwkSSiAJ1p05yEh0ObNg4CASPHTJ7sokSJsCybtlK59YXSmygVOBj8OjRLo1GU21lyMjGN6mBK\nSkK3Wjmz6PN0EwHA6NE2vvnGSqVKPvr3SSKmdVNsv6wHIKXz61h27iDi/eH4MFPHupq/AvcC8Oab\nHsqX91O3bvi5YSrGjHHy/POqC6lyaSG9gawoylXSdaJe7Yhl/18AJH04AV+p0uluumqVmffes1Go\nUIApU1xke6cPtjXfA+CpXAXN78f+9Qq8j5TlDcdYVv9svKVcvbqX1q091KsXzu+/G+PUdO7spkkT\nlQhuN06nk379+pCQcAq3203r1i/x+OOVrqrOLE0GQogHgC+A0f9n78zjbCrfAP495+5zZ2OMse/m\nZE8KLUqUJAol+74L2eWXNNZkzxbZI7IUkSwRElG2EE6yrzNmhtnufs/5/XHGVoxpZjDqfD8fn7n3\n3Pc87+Mu5znv+2yyLE+VJKkgsBCt1/EloKUsy+5Un0QvtLaXn8myPOd+6qWj86CxTZ+Cde1qABzt\nO+N+q/Edx+3caaBTJxsWC8yb5yTfpi+wzZuNCij5C+CtWAn7J+PxlSjJvGbf8UmfMEBzGH/yiYt2\n7Wzs368Zguef9/H++3ouwb+RLVu28NhjpWjevDWXL1+iV693HowxSL2ol5BleZUkSaGyLF9Lxzl2\nYAqw+ZbDw4BpsiwvlyRpFNBOkqTPgSFAZbQOar9KkrRSluX4f/qf0dHJjpi2b8M+7AMAvE88Scqw\nO++27tploFkzG14vzJ3rpFLCFoJ6d0cFsFpxv1oX+yfj8RcsxPaotfRsnxNQsdth0SIn06eb2b5d\n+0lXq+ZjwQInRn3tf1+JirKwZk3Wvsn16vmIinKnOaZOnTo89VQSANHR0eTOnTvT86Yn6aw3msPY\nAqwCPpAk6aosy/cq1ecG6gC39jeuDnRJfbwGrTy2DPwqy3JC6nw7gGdTX9fReaQx7fyJ4NZNQVVR\ncoaRsGDJHctR795toEkTLYR0zhwXdQofJrhOU/BpWzwp3XthH/8x/vwFOPzp9zRpVxy3W+B6M/s/\n/xSZONECwFNP+fjiC+edkph1/mV06dKOmJhoxoyZlHlhqqqm+S8yMvKXyMhIMTIyckvqczEyMnLX\nvc675fyoyMjI7qmPY245XjwyMnJnZGRks8jIyIm3HB8eGRnZKS2ZXq9P1dHJ9ixerKoGg6qC9nfn\nzjsO+/13VQ0JUVWjUVW//lpV1bg4VS1aVDsPVHXwYFXNmVNVzWY1YctetXTpmy9FRanq4cOqajJp\nzwsV0k7X+e9w5MgRtW7duqqiKOkZftfranrWN0myLCuSJAGQ+jgrOmXfLRn+nknyV686MjxpeHgQ\nV64kZfj8+0l21U3X658RHh7E1c3bCW3TBtC+0Ekjx+AqURb+ou+VKwKvvhpAQoLItGlOnqvqwtPg\nLcynTgFau0vjt2sxxceTOPYTWk4sx5Ej2rm1avlo1cpJhQqBeL0CRYsqrF7twO9X7xhCml3fL8i+\numVXvaKjzwAWIiLykCtXATweL8ePnyVHjpxpnhceHnTX19KTgXxCkqQPgRySJDWUJGkpWuJZRkiW\nJMmW+jg/cDH1363xddeP6+g8msTFEdy2BXg8CH4/rreb4mrb4W/DnE5o1crG2bMi/fq5adTIh33Y\nEMw/bgHAU606iAKmA/txNW7GFFcnVq/WtpgKFVIYMMBFnToBJCUJ5M2rsH17ChEReoOa/wJ79uzh\nyy+1EnHx8XE4HA5CQkIzJTM9xuAdIAW4ALQAdqceywib0Dqmkfp3faq8pyRJCk2tjvossD2D8nV0\nHi5+PzRtiuHcWQTAV0NjwbMAACAASURBVKYcSWMm/i27WFGgZ08re/caePNNL/37e7AuXkjADK2c\nhK9YMVz1Xsf2+Tx8pcuyrOY0hnyoZRODytmzIi+9FMixYwZMJpUNG1Iwmx/4/1bnIdGkSROuXo2n\nW7cO9O/fiz59BiJmsuCUoKr3505CkqRKwHigCOBFMybNgfmAFTgDtJVl2StJ0ltoDXNUYIosy2lW\nRb1yJSnDSmfXZR9kX910vdJPwKhh2CeN0/oLBAVxdeM2lKLF/jZu1CgzkyZZqFLFx4oVTgJ3biak\n6ZugKKhBQSR+OoeQjq1RjSY2TviV+t1L4vGAqmpGpVw5P7//LmIywYoVDqpUuffObXZ8v66TXXX7\nt+kVHh501234u/oMUv0Cd7vo+mRZtqQ1qSzLe9Gih/7Ky3cYuwJYkZY8HZ3sjnndWuyTxoHRiODz\nkTh91h0NwbJlRiZNslCkiML8+S4CLvxJcLuW2nLBYCBp+mwChwxCcDrZO3o5zQaUwO2G6+605s09\nrF9vxGCAxYud6TIEOjr3Ii0Hsgnt2/c+cBD4IXX8S0Dk/VdNR+fRwbxxHUHdOqKKIoLPR0rfgXhe\nrv23cQcPivTrp5WjXrzYQZg1meD6zRBTkgFIGj0O65JFGE+e4GKnQbw981WuXr25/H/6aR+//WYg\nLk4rXletmt6yUidruKsxkGXZDyBJUnVZlofe8tJSSZLW3XfNdHQeBTwe7COitHLUBgOCokDt2jj6\nvfe3oXFxAm3a2HC7tVLSJYp4CW7dBqN8DABHp64ISclYvluD65nnaXsyilOnbhqC0qX95M6t8vPP\nRlq08NCqlfdB/S91/gOkJ7TULklSZ+AntHIRz6BVMdXR+U8jnjxBcJd2mA7sRwnNgXjtKv5ChTF8\n8QX4DbeN9fmgUycr58+LDBjg5uWXfAR16oDl+w0AeKrXwFPrVULero8/Ig9DK33NxilmtJ1agZIl\n/bz+upfRo61UquTno4/0TmU6WUt6jEEL4EO0CCIBLay01f1USkcnW+P3Y12yCPsHgxBTkvFHRGCI\njsZfuAgJC5eSM2fOv+UTjBhhYft2I7Vre+nTx0Ng/15Yv/kaAPfLtUkZOpLQN14FUeSbd77l4w+1\nUhMgIEl+PvjATevWNsLDFebOdWJJ02Ono/PPSU8J6z+A5pIkhQGKLMtX779aOjrZE8PRI1rf4t8P\noVhtKKE5MERH465dh6QpM1DvEOu9apWR6dPNFC+uMHWqi8BRH2L9fB4qkDJkOM5OXQl9sx7ilRiO\n9ZtGh4kV0YL8BKpU8TFtmovatQMQBK1URd68ei6BTtZzz8BUSZKelSTpBHAU+EOSpGOSJD15/1XT\n0cleWL75mhyv1sT4+yG8TzyJ4PMiJCaQPHgoifMX39EQHDki0quXFbtdZcECJ7m2rSJg8kQAkiZM\nxtn9XewjojDt/pnEem/T/IdOqQ5jgUKFFJYtc9K5s9btbOhQN1Wr6g5jnZu43S7efvsNvvsu86Xc\n0pOl8BHwhizLuWVZDkcrWjch0zPr6Dwq+HzYh35AcMc2qID7hRcx7duDGhpKwvJvcPbszZ06zF+7\nBq1b23A4BKZMcfEYRwnu1BYBcHbpjrtFG8xrVhEwYyrRxSrztmM++/Zpi3WrVUskGzLEciMxrUMH\n3WGsczvz588hODgkS2Slx2fgl2X58PUnsizvlyRJ75ah859ASE4iqFNbLJs24itSFDUkBMu2LXjL\nlifx8yUoBQre8Ty/H7p2tXHmjMi777qpW/0aOR5/CcHvx/P0M6QMG4Xhz+ME9ezGaetjVIjeQeJJ\n7edoMGiGYMMGIwsWmClTxs/48S7dYZwNsUcNxrJmVZbKdNerT0rUvYpCw5kzpzl9+hRPP/1slsyb\nHmOgSJLUEK2UBEBtQF+r6vzrEc+fI6T52xiP/o73qcqIFy9i/O0ArtcbkPTJdNLqLj92rJnNm428\n+KKP9wa6Ca35EmJiAv6IPCQsX41w7SrBbZqRkgLPhe4h8ZqR6zmeCxc6cTgEBg60EhKiMm+e1tNY\nR+dWpk6dSO/eA1i37tsskZceY9AFrUnNHLRv68/c7Emgo/Pvw+XCsnIF9pFDMcRE436pFuadOxAc\nKaQMfB9HnwF/qzV0K4sWwYQJFgoXVpgxw0nIe70wHT2CajZz9fttoCgEt2qK8Mdxns15hgvxdgRB\nRVUFhg1zUayYwmuvBeD1woIFTooU0R3G2ZWUqBHpuovPalatWkWZMuXIly9/lslMTzTRcbTVAACS\nJImyLOv57zr/SgwnjhPS5E0MZ06jGo24Xq+P5dvVYLGQMHcRnrqv3zbe5QKzWfMPLFli4tw5kc8/\nh+Bglc8/dxKx7esbkUMJCxajRuQhqEcXzLt20rTAVg6dL3DDELz5ppfmzb3Urh1AXJzIuHEuatTQ\nF+E6f2fr1q2cOnWGnTt/4sqVGEwmE+HhuXnqqSoZlpmeTmdtgABgJrANKChJ0mhZlj/N8Kw6OtkQ\n4/69hDR7CzEuDme7jqCo2ObPRsmVi4RFy/A9cTOITlXhs89MDBtmwWbTfAQpKdpqwWaDzz93UsZ2\nguBuHRAAR/deeGvWwjpvNtZlSxifbwxfnn8B0AxBqVJ+xo1z0a+flePHDXTurGcY69ydSZMm3ShU\nN2fOTPLmzZcpQwDp2ybqjFZwrgFwGHgerU6Rbgx0/jWYfviekHatwOkgadRYTHt2Y/16Bb7iJUhY\n8hVKkaI3xrpc0L+/laVLTeTKpRAUpGUYDxzo5tln/VSoYEdxOsjxeE0Enw9vpadI+WAotpnTsH/4\nPpsCX6f/pX4AGAwQEKD5BebMMfPVVyYqVdKSzHR0HiTpMQZOWZbdkiTVARaldjrTNzF1/hUIsbEE\nDhmEdcVSrcicqhI4eCCCouB9qgoJC79EzRkGgMMBmzYZmTbNzP79BipW9DN/vvNvSWBhYeB5tgFi\nXCxKaA6urVxLwMSx2EeP4EzY47ye9DWqKmCxqLjdApMnO1m/3siIERby51eYNcup9ybQSTft23fO\nEjnpMQZIkjQNrelMR0mSnkbrR6Cj80hiOHYU84bv8FWuSmCfHhhP/IkSGIiYnIyvZCRqgB1fhYok\nD/8IbDZUFU6dEmjSJIDTp7V8gkaNvIwb58Jmu8ME/fph3vkTqsHA1TUbMP+0jYCPR3Ix3xNUF3fi\njDNgNmuGoEsXD5s2GfjiCzMREQpffeWgQAH9XkvnwZMeY9AcaAxMlmXZL0lSEfRoIp1HFOPBA4S8\n9TritWs3jqmAmJyM58WaJMxZCIGBAERHC3R428qhQ1rROYdDoH17D82aeSlX7s4xFNa5n8H48ahA\n4ifTwWwmqGtHDhqfoGbKTuISzIiiiscj8MwzPqKjBVauNFG+vJ9585wULKgbAp2HQ1rNbSrKsrwf\nKIXWzyC/JEn5gWgg7AHpp6OTZRgP7COkUX2ExAQc77yLZfmXGGKi8RcrTsqwUXherk1SssCArlZO\nnRKJjha4cEGkSBEFnw9GjnTTvPldnLpeL4F9e2JL7UvrbN8JT903yPHay8QmmHglaBtxCSYEQUVR\nIE8ehSee8DN1qoVKlfx89ZVDzyXQeaiktTJoCewHPrjDayqaE1lH55HAuH+vZgiSk0iaOBXL2tUY\nYqLxPFuNxHmLUENzcOWKQJMmNg4dMiCK2h16v35u+vf3pJn9KyRcI7jJW5j3/qIdaNyYlJFjCOrR\nBd/vf1An7AjRcVqCms2m4vVC27YePvrISoECCgsW6EllOg+ftJrb9En9+2JWTSZJUns0I3OdJ4E9\ngB1IST3WN7Vlpo5OlmD4/TAhjRtohmDSNCyrV2LZtBFPjZdImL8YrFaiowUaNrRx/LiBli09jBrl\nxuOBoKC0ZRt/209wyyYYLl8CIKXPAOzjRmMdM5Fvl3toL8aSGKdtO0VEKERHi/Tq5WbSJAt2u8qi\nRU5y59a3hnQePunJM3gBrTBdKbQVwUGgtyzLu/7pZLIsz0HLZL4u922gDND21vpHOjpZheHP44Q2\negPx2jUSx07CuuorzD9swl3zZRLnfQFWK5cuCTRsGMCJEyJdungYOlRrHJNmz4DkZOwfDcM2dxaC\n348qCCSPn4yrRWvsu3Zx5P0VNGE7fsWAzaZSvLjC4cNawbklS0y4XFrZidKl9fxNnexBehzIk4C+\nwA605jbV0HIMKmZy7iFozukvMylHR+eOiGfPaM7i2CskD/gfAXM/w3j0CO6XapE4dxFYrVy4INCg\ngRYl1KOHm8GD094SAjDu/InQxg0QtC71qBYLiXMX4nm5Nt8vS2Z6HwN7/D/gx0Dhwn5atfIyfLiV\nxx/3c+SISHS0yNChLmrV0rOLdTLG7t276dmzJ0WKFAOgePES9O49IHNCVVVN819kZOSmOxz79l7n\n3UPmU5GRkfNTH2+NjIz8OjIy8sfIyMiZkZGRtnud7/X6VB2dNLlwQVWLF1dVUNX//U9VCxfWHnfv\nrqoej6qqqnrqlKoWLaodHjxYVRUlDXkej6p266aqERHaCaCqBQqoaqtWqrpnj6qqqrr3u8uqVXCq\nIj4VFNVmU9WFC1XVYFDV3LlVtXRp7bQuXe4xl47OPdi1a5fao0ePjJx61+tqelYGuyVJ6g1sQOt/\nUAM4IklSMQBZlk9mwAZ1AOanPv4EOCjL8glJkj5Fa685Lq2Tr151ZGBKjfDwoBtp3NmN7Krbo6SX\nacd2LF8vx/zjVgxnTpPStTuWFV9hPHOGlPcG4+gzgMnDFKZMUUlJAZ9PYMAANz17eoiN/csEqopt\n5jSMhw5iOHIY0++HtcQ0wF2nHonzFt0oWHdp+X4aNs2FS7ViFH2YzTBunJOePa0IAthsCkeOGGjd\n2kNUlPvvcz0AsuvnCNlXt3vpFRVlYc2adKVrpZt69XxERd07A93t9v7j9yw8/O5OsPT8L5ql/u35\nl+ON0HwIxf6RNhrVgR4AsiyvvOX4GrScBh2df4xp50+ao9jjAcDRsSvmn3di/EPG0aU7jt79mTrV\nxIgRFsLCFMqXV2nUyEv79ncIF1UUAt/ri23+nNsOC4pC8ocjcL5z8+dw/It9NOpTkotqPk0Pi4HP\nPnPy0UcWrl4VyJ1b4cwZAx06eBg5Um9kr5M1nD59ioEDe5OYmEi7dh156qmqmZKXnqqlRe815p8g\nSVI+IFmWZY8kSQLwPfCWLMvX0IyE7kjW+ccYjh0luHUzUFUS5i/GW6kSwd06YTqwD1eT5qQMHcm8\n+WaGDbOSL5/C6tUOChW6SxSPohDYvze2hfNQ7IGIKcl4y1XA3agxvrLl8T73/I2he8dsp+m4Z7iG\n1vKyVCk/S5caGDzYxJEjBoKDVWJibndM6/x7iIpyp+suPqspUqQIbdt2pEaNl7l48QI9enRm6dJV\nmEymDMtMTzRRYWA8ECbL8ouSJHUAtqWWts4IeYEYAFmWVUmSPgM2S5KUAlwAojIoV+c/iHj2DOLV\neILbNEdMuEbi9Fl4Xn6F4I5tMG/fhvvVuiRNmMLyFSYGDrSSK5fCihVpGAKvl9C6tTDt34sSEICY\nkoz75VdInLMQrLdXYdnaZz0tF9XDgxZ2VKuWj88+c/LFF0GsXm3CYlFJTBTo2dPN++/f2zGto5Ne\nIiIiqFmzFgD58xcgLCyMK1diMtXfID3bRLOAqWgRRQB/AJ8BGco/SM0hePWW58uAZRmRpfMfZ+xY\nwgbcjKBIfv9D3HXfILh9Kyzr1+Kp9gKJM+fyw48W3n1X6xq2fLmTEiXuYgh8PnJUq4Lx5J8AiA4H\n7rpvkDhjDrdVjlNV1rX5hvbrmuDHgIpAtWo+IiIUevWy8s03WutKt1ugb183AwbohkAna1m9ejWn\nTp2nWbOWxMXFEh8fT3h47kzJTI8xMMmyvDrViYwsyz9KkpSpSXV0MoyqYtz7K6Y9v8CH7+PPmw9P\n9Rr4yj+Oq1Vbgls31RLKqlUnYcFiDhwLoF07G0ajFtdfpsxd4vr9fnK8UBXjyT9RbTYSFi4FoxFv\n5apgvOVn4vPxVcOv6barraYOIhERCtu3G9m+XRtiMIDfrxmCgQM99/kN0fkvUqNGDXr06MVPP23D\n6/XSr997mdoigvRXLQ0ltUGrJEllgDvVatTRue/Yh35AwPTJ2pOAABIXLcVXrgKoKoHv9b0ts/jk\nRRvNmtlwuWDuXBdVq94lrl9RyFGzGsbjf6BabcRv/wWlUGEtmmjGNMSLF3DUfBVCQ/igdQwzLrVH\n+zkIhIdrWcWvv+6lUycP06eb+e47E2+95WXAAN0Q6NwfAgMDGTNmYpbKTI8xGArsAvJKknQQyAW0\nyFItdHT+gmnbFuxjRoHix1+sBJ5qL2C4eIGA6ZPxFS+Bs0MXgt6ogy9XAQBssz7FNm82vtJlSZy9\ngJhEG02aBBAbKzJmjIs6dXx3nkhRCK1dA+ORw6gWC/FbfsK0bw/CxnUYTp0kYNYMZtGBbjNrYcCP\nO7V6e5FCPpxuA9HRIp07a87hqVM1Q1ClCkyY4NK3hnQeKdITTbRVkqSKQFnADfwhy7Lrvmum859F\nPHOa4PatEJISwWjEtHcP1uVaoroSFEziwqX4S5QkKDwIriRh3rAO+weD8OeOIOGLZSSqQTRvbuP0\naZE+fdy0aXOXSqOqSkj9OpgO7EM1mYjfsA3Lpo0EfjCIRTSnL+N5zNqGH11VMAgKbtWE3exhwCAf\ns+daiY4W6d3bTYcOXnr2tLJsmYl8+RRWrRIxGB7gG6ajkwWka5tIlmUn8Ot91uX+4/HA8eMQmudh\na6JzNzwegju3RUxMIGnSNFxNmmP8/RDGvXtAVfE+/wL+4iVvDDce+o3gzu3AaiVx0VKuBhagcaMA\nfvvNQPPmnrvu2YtnzxDUvTPmXTtRjUaufvs91nVrsH88ksUhnWmdOB2DoPCjKwKjUcXnE2nUyMu7\n73po3DiACxdEBg5006iRl1dfDeDsWZHy5f1Mm+YiTx47V648qDdMRydryNrUuWxOwMSxMHEs4r7f\nUTIRgqWTdQjxcZj2/oqvVBmU/AWwj4jCtG8vrrca42raAgQBX7kKml/gr1y4QHCLxuB0kDh3EYmR\nT9DkzQD27TPQuLGXceNuj+s37ttDUJ+eiJcvIcbHAaAaDFxbtgr7xLFY1q/lm5ytaZXwKfZAWLjQ\nzZAhAgcPGnjiCT+VK/t5880AoqM1w3DihEjt2tpWVJ8+WqlrfUWg86jynzIGgssFioJ48YJuDLIB\n4pnThDZ4DcP5cwD4c0dgiInGV6IkSWMmkuame3IyvFkPw6WLjH1lPb9+V4PLcwX27tUqg37yiQtR\nTJ3n/DksK1dgHz0CfD4EVUUF/KXL4OjYlYB5s7GsX8umcj1p8sdEzGaYO9fBqFEWDh40EBamsG+f\ngX37tCt98eJ+li/XIjcCAlSGDnXRtetdtqJ0dB4R0jQGkiQ1AeoA+dDCJ84Dq/9SQuKRQU0tTi8k\nZb8aKP81xIsXCK1fB8OF87iaNEdISsK45xeU4BASZy240XryjigKwd07w/79zH1mBgM2vHLjpeee\ndvPJ6AREUQt4sy5aQGCfHtxqVnwlI3F06ErQoL4E9+4OwK5y7WhwaiJ+RWDubCeTJ1v45RcjuXIp\nxMaK1KvnTW1NaeLECQM1a/oYPNjNY48p+mpA519BWm0vpwIFgOXAZbTy1fmATpIkPSPLcv8Ho2LW\noQQHAyAk68bgoZKSojWEuXCelP8NwdGrn3b8ej3Q67f0d8E2ZSKW79aw+4mudNvbieBglXl99+Id\n9ymv/bwU+2MufKXKoAYHa43pBQFVEHDVfxNvzZfx58lDaIvGYLHgaNOBY44C1F3dl5QUgRkzXCxe\nbGL79puGoHBhBZsNJk2ykJIi8M47HoYM0UtL6DxcNm5cxxdffI7BYKBDhy4888xzmZKX1srgcVmW\n/yZdkqRFwPZMzfqQ+Pn3UGoBrugk0r7c6Nw3FIXgbh0xHfoNZ8s2ON7te/M1QUh7awgwf78e+0fD\nORtRiQYXpuD1wsKxp6k/tDaCMwFvjer4EhMxHjyA4PGgGo0IPh9JE6fiatYS8fQpvnxxIcOdJ0kw\n5aLEdoHYWIG4qyLjxrnYtMnIunUmrFaV2FjtW3LmjMiZMyL58ysMHuymXTuvbgh0HipXr15l7txZ\nzJ27EIfDyZw5M++rMTBJkhQky/Jfb6NDgMyluj0kTsdpxcTiTyeT6yHr8l/FPnIolnXf4qn2Asmj\nx6d58Reio8FswvrlYsw/fI+vbHnE2fOYaejGx57hXLpqYGzu0bzV630ERSFpzERcbdprJ/t8hDRv\nhHnLZhxde+Bq1hIhPo4f6s2kR8oUgq0eIiMFjh4V8XoFhgxxsXWrgW+/1b7aLpem19NPa+WEjUYo\nVUq5LRlZR8ceNRjLmlVZKtNdrz4pUSPSHPPzzz/z5JOVCQiwExBgZ+DA9zM9b1pf7VnAIUmSfkDb\nJgLIDzwPZH7mh4Axp7YP7buqbxM9cHw+7MOGEDBjKr5ixUmcvQDulj6vKAT27o5tyaLbDm/dZqQ7\nezhOScxX3QxiFH2SRuF7qgquNxrcNASAfdgQzFs24675MilDhoHTyZ8Nh9I6ehoBJi9fr/VTrpyX\nhATYvNnIJ5+YOXrUwPXM4hIl/IwY4eb55/26AdDJdpw/fx6328XAgb1JSkqiXbtOPPlk5UzJvOvX\nXJbl2ZIkrQNe4qYD+XdggCzL0Zma9SFhCtMcyEp84kPW5D+G201wh1ZYNqzDVzKShC+Wo+bIeeex\nfj+B7w/AtmQRvkgJf8FCuEtVYHBCf6YszI1BVOhum8v/HIPJu/ozrjx5nr96cK2LF2pGp2QkSTPn\nIiQnkdCiHw2OTMKFlc/nOClXTqtRNHKkmfnztaqjoqiiKAJFiyqsWqU3qte5NylRI+55F3+/SEhI\nYNSosURHX6ZHj8589dW3CJnYv7zXPU9BoASaMVAAC3AEeCSNgSW35kBWE/WVwQPD4yG4fUssG9fj\neeFFEucuRA0KvuNQITqa4G4dMG/fhq9ESRKWfIVSsBCDBlmYs9BMicCLLE2uyxMp+0l+/0OoWxeu\nd3ry+zGv/w7DmdPYR0ahhIaSsHApgsOB2LApDf+cy2XyMnxIMq/U1i7yY8eabhiCEiX8/PmngYgI\nhWXLHLoh0MnWhIWFUa5ceYxGI/nzFyAgwM61a1fJcbebrHRwVz+qJEkfAJMBF7AT2I22Oph3vYLp\no4Y1Qtsm0kNLHxz2sR9phqB6DRIWLr2rITCcOE6OV6pj3r6NjUItdv8ZTo5K5fix2gjmzDFTxiiz\nN/kxylWAhHlf4OzZ5/Z5Rg4lpG1zAqPeB0Uhcc5C1OAQ7G81pOWfQzlIBdq2cdPpHRWfD95/38LY\nsVqdoRdf9HHqlEhIiMrSpU4KF9YNgU725rnnnmPv3l9RFIWEhGs4nQ5CQkIzJTOtlUEd4FlZlm/L\nppEkaQywBcjaknkPgMC82jaRIUXfJnoQGHf9jG3KRPyFi5A49+/NYa4jXjhPaN1aiHFxbKAWtdUN\nAIQaErkmB2PBxWL/2xj6dOVa/0F/2xYyr1tLwNRJ+IoVx9HvPXylyuAvUZKQN+rQ5Y8+rKUuL77o\nY+QoD/v2iQwcaOXgQU3GY4/52brVQGAgLFnioHTpu5S41tHJRkRERFC9ek06d24DQO/e/RHvEZJ9\nL+61TXSnX4ZCGiuK7ExQbis+DJgc+srgvuN2E9SrGwCJUz9DDfxLI25VxbxpA+KZ09hmzUCMi+Ma\nIbSzLcboVald28fBg4FUKuygQ4NL5Ku7Bkdojr9NI546SVCPLqg2G4lzF+EvXQZUlatdoui5rzdf\n0pTHH/czZ46Tn3828PbbNvx+bV81Z06FY8cM5MypsGSJk4oVdUOg8+hQv/6b1K//ZpbJS8sYfAf8\nIknSN9yMJsoH1AcWZmQySZKqoyWx/Z566BAwJlWeAbgEtJRl+b40Fc2RExIJxuzWVwb3CyE6GsHt\nwrLqK4wnT+Do2AVflVsadScng92Ode5nBA26PW+xR8gCLiaEMXCgm759by0wl5s7btw4nQS3b4WY\nmEDi5E9vGIKD3eZTd+VIXNgoX9bLkiVuYmMFOnS42YbDalWJjxepWFEzFAUK6FtDOv9t0oomGi5J\n0ga0FpWVUg+fA9rKsrw/E3Nuk2X5retPJEmaB0yTZXm5JEmjgHbAp5mQf1eCgjRjYPXoK4MsRVUR\nz57B+tUyAiaMuZHspeTKhWPA/24MM/66m9AGr6HkyAkxV5hpfoeinj+okXM/yxssYNGcOjz+uJ+e\nPdPRFMblglatMB0+iLNFa9xNmoOikNxzKC2/6ocXE5M+vETjLoE4nVCjRgBXr16PtFBxuQReeUXr\nWWzTWzXp6KS9TSTL8i/AL389LklSc1mWv8giHaoDXVIfrwH6cZ+MgShCsiGEHL6z6D2osgbxzGlC\nWjXFeFRb7PlzR+AvXgLTrp0kDx2Fet2p5fMROLAvgseDGBPNp2oX3vFMBSDI7cP1uQGLRWXKFNdd\n0w9u4PUS3L4lfL+B8y+8zW9vTkLYpRI0aQxdf2jHJfIR1TeWZu8E4nLByy8HcOnSTT9DSIhKt24e\nevTw6DkEOjqpZPSn0B7IqDEoLUnSaiAnWhc1+y3bQjFA3gzKTRdOUzCBrkTiVfWepQ900sa0bQvB\n3ToiXonBXas23qrP4mrRCrw+AsZ9BIKA8bf9GP48jmn7NvyHZcZbhpDHfYZBxnGECsm89mwcv14s\ngMGg0LOnB0m6x769qhLYvxeW7zfwaanJdN/eHWXb9c9xJAAdWyXRdYCFH3800KGDlWvXREClTh0f\nrVp5efppv74a0NH5C2kVqvv8Li8JQKkMzncczQAsA4qhRSXdqkO6rs45cgRgNP7zUpFer2YMDC6F\n8AAx7cqYD4nw8KB7D3oI3KaX1wudOsH8+dpya+pULO+8gwUIVFVo2BBWrYJ5s2+T0Y1pfOrWnMr4\nYOZM6NTp1s8gT6+3ngAAIABJREFUjSu0osDYsTBjBpw+zedFhtDtaA9y54bWj+3G9+MOzoSUp+1n\nz1CzXhCdO8PCWzxbQ4cKDBli4kFVUnkkPsdsRnbV7b+iV1org3LAWrQL+F+pdIdj90SW5QvA0tSn\nJyRJugw8JUmSLbWbWn7g4r3kXL3qyMj0TJhgpnySFud+4ehFzEXu6yLkHxMeHsSVK9nPn3GbXj4f\nQV3aY129Em+FiiSP/wRf+ccRjpzENmcmxt0/Y9mh1TH0547AX7gIgsvF0kNl+ZRulC7uoE4DA4oC\nb7zhSV9HMJeLoO6dsa5eiRIYxL5aA+i4JYrQUJXVdaZQZf67+PPm49q6zfjz+mnf3suSJSaul5bo\n3dtN167pnCsLeCQ+x2xGdtXt36ZXWgYkLWPwNjAPGPHXnseSJLX5x1po5zUH8sqyPE6SpDxAROoc\nbwKLUv+uz4js9BAaqpKIZgxSLiVlO2PwKGAfNgTr6pV4qjxNwuIVEBSEGH2ZkIZ1MR7/A9Auwb7S\nZTD+IUPMFd5jNOPoT0CAyqzPVUqWvLPHxrj3V0w7tuOrXBXvk5XBaARVJahHF23Oqs8QM3MxrZoU\nwOMV+Mrcnirz5+IvVIRry1fhypWfaZPMqYYAQKBxYw+DBukeIp1/F8uXL2fFiq9vPJflo3z/feaK\nSacVTXRckqRacEdf64AMzrcaWCxJ0huAGegK7Ac+lySpM3AGWJBB2ffkySf97LthDJL5e9S6TlqY\ntmzWav7kL4DhwnnCniyL77HSGA8dRExOwp96PCVqJM5uPRDi4/j4nRjGba5M8QJOZs5XKFnyDj4B\nj4fAD97DOn8OgqqFeCqhoXiefxEMItZvvsZb5WkSln/DqFHBHD1qoDMzqOtYhqtJc/Y0iGL+Z4VY\nv97IhQuafwAEnnzSx6RJ9yVKWUfnodKoUSOqV68NwP79e/nhh02ZlnmvaKI77sfIsvxrRiZLLYdd\n7w4vvZwRef+U0qUVfhRDQAFXTPZb+mVnTD/9SPA7HVENBgyXLwGg5MmL+ecdKKmZxYYL53G/8irO\nrt05dEhkz548rNisstTehtdKXMY4wYwSnhvDcRnBkYKnZi28zzyHbfZMLOu+xfdYKRxde2Datxfz\npg1YV2sN9fz58pMwZyE/7TQx41MjJfmDMbnHwq6DrN0XQYsWNlwuAVG8mStQp46XKVNcehcynftK\nVJSFNWuyNiStXj2tbHp6mT9/NkOGDM/0vP+pwDqjEQw5giEOEs/rxiA9CLGxMHQQIdOngyCghuVC\nuHaVhGWr8FZ9huC2zbGs/w7vE5XwPv0cjnf7MGGihdGjLTzNTnZTn9wpV2DrTZmqIIDRiOnAfhj/\nMQCeatVJWPglBATgbtpCy104dxbDyROgKJxoOZquB0YgYmdurv68k/9rvqlQFKdTRVG0JLLrPQj6\n9PHw3nv61pDOv5+jR38nd+4IwsIy36HlP2UMAILya8Yg5rhuDO6IqmLeuB78fox/HMM2eSIkJ+Ev\nVhxXoyYEfjwSZ8u2eJ+thnX2DCzrv8PzbDUSlq0Ck4m1a42MHm2hYe4fWXL1VYx+N0kjx+Fu3BSc\nLsToyyiFC4MgYNq6BdP+vagmI46efSEg4KYegoBSqDBqgJ0D1QZSP24uDgIYUf5Lmp1byLn9wUgS\nJCaqXLok3DAEDRt6GThQNwQ6D4aoKPc/uovPatasWcWrr9bNEln3NAaSJI2WZfm9vxybLctyhyzR\n4AETXjwYDkLCueSHrUq2xLxuLSFtmt14roSFQZvuKL8dImDOTFSjEce7fcDhwD5xHIo9kMTP5oPJ\nRHS0wMge14gyjeP95LEYVA+J8xbhefU1TVhgEP7w8BuyPXVfx1P39bsr43TyZ7OPaBA3B6/BytAP\nPYwZ35TkZBg82MWJE1aWLNHKZL30ko9GjbzUrevT00d0/jPs37+X3r0z6sK9nbTyDBoADYGXJEnK\nd8tLZqBalsz+ECj2eDCsBE+svjL4G14v9uFDUA0GHH0HotoDcTVqTK7XXsJ86hQAjs7voIaGYpv1\nKeKVGFJ690MNDwevl/1NZrA/eRSBpKCYA0mcOe+mIUgHLhd8t9LP9gm/YRBU1KQUVsRpDWleftHH\n6NEWHA6BgQNdTJ9uJj5eO69ZMw8TJ+oN6nX+W8TGXsFmC8B0z5T99JHWymA9Wkbwk8DmW44rwIdZ\nMvtDIHfJEAAMKbox+CvWRQswnvgTZ+v2OPppi0Hr3Flw6hSOjl1IGT4a25yZ5CpREAAlKBhXi9a4\nzsTgatiBlue2Em8MJ3HYUDxNmv69UmkaJCdD0yZWdv9iAl64cTyf4TLRfhMbN5kxGlUsFpVPPrHg\nTl2ZP/+8jwkTdEOg898jNjY2U81s/kpaoaVOYIckSRUBkyzLSZIkRQCR3DkR7ZEgKL8WWmr1JpGc\nnC2TkB8KQkwM9o+GodgDSek/CADD0SPYx3/MMVtFlgSNoMYBHznHrua4pTGRQZco6jpGWKVyhKXK\nWGd+ndCV0yjxVMg/mtvrhZYtbez+xUhDvmLQk+vxd+nIl1MTmXSgBna7gNOp4vdD4cIKly6JqKpA\naChMnuwik2XcdXQeSR57rBTjx0/OMnnpcSCPBQ5IkrQSrePZHqAF0DnLtHiAhBbSjEEwifzxh8gT\nT/y3a9ibftyK4Y9jmH/cinjtGkkfjUUND8c6dxaBUe9zzFWEFwJ/4sqEAIZNgBysZjWvU9a9gyvk\nYjuvYBQUnFWeo+yidwkM/udX5unTzezYYaQ+q5hfaDA9iu1i7yQ7hw9rX0+3W6V0aYUaNXwsWmTC\n4RB4/HE/8+cbyJdPLz2to5MVpMcYVJRluYckSV2A+amlrTff86xsiiVPDhQE8nGRA8f+u8ZAPH2K\noIF9MG+5+VE6n6hC7Outydu6OSvWhzDb+D37rFVISjbxTqdk4j7/ngGu4VRkP3LFt1hSazaxyTZa\ntvRQoVjGLsonDrsZ97GNPIYr1PBvIu/l30hZZr7xepEiCgsXOoiKsjJ5soWAAJXhw1106OAlT56g\nB1ZiQkfn3056jMH13di6wODUx5b7o84DwGrlvK0k5ZyHmP2rSLNm9z7l34QQG4t16WICxo1GTEnG\n8/yLuN56G9+pCzTYMZAd5QIoo4zgIBUQ/CqF8quMG6TSamM7rK7lAHieeY6cX8/mHVEEMh5WFxer\n0vmVGNy+XLzIRnoyFTwQGqpQq5afa9cEevVy06pVAKdOibzwgo9x41x6j2IdnftAeozBH5IkHQGu\nyLJ8QJKkVkD8fdbrvnK1QFkKHf+ay3svo1XS/vcjxMRgHz8a66IFCF4vSlAwidM+w/1WY3x+gR49\nrPz4i4kIIZqDVOCpIpf5Il9fiiQdxjTkMsTEaMliFitJE6aQ2Y36HTsMfNj0HAe9pSksnGGJ2hyT\nSaVcOT+zZrkoWFAlJkagTp0Azp4V6dXLzaBBHt1RrKNzn0iPMeiAVsH0aOrz3295/GhSvgwc/5qQ\nM4eB5x+2Nvcd495fCWnZGDE2Fl/RYrjad8LV8G38OXPx03YDQ4daOHTIQOU8p9l6uRTXmrQjYvlM\nxNP+GzJUQcAvlSKl/yCUYsUzpc+gQWY+nwNeJGw4OKMWJjLSz9KlTvLn1zKKo6MF2rWzcfasSJ8+\nbj2jWEfnPpPeDOR8QHlJkkDbInofKHKfdLrvBD1bGr6CEs5DJCY+T3Dww9bo/mH6YZOWROZ242jb\ngcQPP2LyzCB+aGvg3DmRixe1O/wm9a4xbUNVLEFm8iybgaAoqAYDrqbNsb3bg9iCJTO1GoiPh+XL\nTUyaZCYuTuD67qOTAF56yceUKS7CwlQOHxZp00YzAgANGugZxTo6D4L0GINFQA6gAvATUJVHOM8A\nIPi5MgCU5yAHDxp47jn/Pc54RFBVcDgImD4Zy4qlCE4n3stXUfBgQCVg3mxSFq3lLW8eXiQXZ60l\nKVTcR/nwC4Rv/RHRk3SjRq3nmedI+mQ6SuEi2MKDIBM13TdtEmnT0oLHr33dBBSq8wPVnnLw/KgX\nKV9B8wHs3GmgRQsbyckCtWt7KVtWoXt3fWtIR+evpKSk8L//9ScpKRGv10vbth2pUuXpTMlMjzEo\nIMtyNUmStsqy3EiSpMLAe8DcTM38EFELFyZZDKKC8hvV2lt55RU/kyZlz3h18eQJlDx5b9bt8Xgw\n7f4ZyzcrETxuVJsNITkZ4+GDGI4dvVECWrVaUV0eLlOQcxTigLUKAa546ni/o5ThD2z+/eD6Hk4A\nJ7Siz6rBgOD343qlDkmfL8l0W9BVqwwMG2bh/HmR6yuBamxjQoHxSP1q42raAgRN3717RZo1s+H1\nwqxZTt54w5epuXV0/s2sXLmSQoUK06VLd2Jjr9CzZxcWL/4qUzL/SaE6oyRJVlmWz0iSVCZTsz5s\nBIFLucohxezGcdXDl19aKVhQoX//7LUdYdq2hdBGb6DYA/GVK4/hwnnEC+cRlL+Hw6oBAfgqPYVq\nsaAGBGD+fgNdTHOZ5W3Le++56dPHQ1ycwKn4yRQvrpCSkoh4+jSBw4dg3voDamAgYnIy3rLlSZox\n5x8bArcbNmwwsny5kd8PQvwVBYfPjGYEVF7ie3q/dICq7z+Hv/QiUhSB7dsMWCxw7JjIiBFaVvHs\n2S5ee003BDqPBvaowVjWrMpSme569UmJGpHmmBw5cnDgwGEAEhMTCQkJzfS86TEGP0iSNABYBeyV\nJOk0kA3vof8ZOV4ojXH5TspxiPN5KjF2rIWDB0Wef95Px47eh60eKAr2YUMAUHPkwLxrJ/48efFW\nror/sVK4G7yFP28+SHGwZksIx50FCctjJMJxigLDerDM8hmz3G2RJD/vdHMjXrpELrOFsJJavrAa\nFIyaMyembVvwR+TBEH0Zf6HCJC5ZAXZ7utVMToZFsxU+mR5A3LW/10gJIpFvqEd86WfZ8NgHbFgp\nIqyCTZuM/P77zWYDQUEqn36qGwIdnfTw2muv8eWXy2ncuD5JSUmMGTMp0zLvaQxkWf5QkiSDLMt+\nSZJ2orWq3JDpmR8i5nVrMZQrDsvhNdaysWhFkpIENmwwsXGjkQoV/FSunPXJaMZ9ewgY+xFqaA6U\nfPnxFyyE+40GqH+tL+JwYPtiAaZDv+Fq2IikT2eDxwMWC6qqlW8wm7Ue8f37W1i40HzLyaUAretR\nWdMxvjv1GvnKxSMmXEMVRbxVnsb79LOo9kAsq1YgqCqG6Ms4wvJzevIqwiPy/E3vhATYs0fE6xXw\neuHyZYEffjBy7nASf54wcNUfgoD2fpXmdwYxip+piv3FyohBVvruW8r+I3ngyE2ZgqDy5pte8uZV\nUFWBLl08RETo+QM6jxYpUSPueRd/P/jmm2+IiMjDhAlTOH78D0aPHs6cOQszJVNQ1X/+A7xTWesH\nyZUrSRm+aoSLHtTwcHwVK2E4fIjDnkgq8Bt582p18QEKF1bp0sXDk0/6qVAh/UZBPHmCkBZv4y9S\nFEfv/vierHxju0WIiSFHzecwRF++7RzFHoi3xkuoFgvW2Gj8f55AvHhBi+YxmYjfsQelSNEb4wcO\ntDB/volChVSuXRNISBB4gr0M5wNiycVlIkgkhAAcdLPOxf5YPoSUFPwlJcQrMZh+3f03vY8KpXlV\nXcsZipA7t4LVqgUOXe8SduqUyB12phBQMOLFhwkVkXZ51vBidQ/fOl7iyKlADh26eedfr56Xtm29\nmM0qiiIQEaFQtGjmLv7/tmbl95vsqhdkX92yq15Tp46jbNmKVK9eE4A33qjN11+vxXCP1n7h4UF3\n3f/NaHObyhk8D0mSxqCVwDYCHwGvA5WAuNQhY2VZXptR+ffk+HEEVcW4fy8XStWg/JHNhHOFS5dy\nU7Omj82bjZw5IzBokBWTSeXLL51Uq3bvaCMhNpaQpm9iPHUS45/HsWzaiK94CdTAIMSYaASfD/FK\nDMmDo3DXfxPx4kVMe3/FNmPq7XuO+fLjrfI0J3M9ya5CjSiYWJyyioIowrZtBubNM5Mrl0JKMuQ2\nx9OQbxhvGEAOfxy+1Ph/NTgYT+3XOFrtZ866IihU6OYF3pIcS9iezfhnLiDs4HYW0ozeORdqr19S\niYkRMRjUG850RdGMgqqqqOrt3yMVES9mgoKgZUs3zz1XkzZtbHg82rgaNXw0b+6lalU/4eH6Xb+O\nTlZRuHBhjhw5TPXqNbl8+RI2W8A9DcG9yKgxyFCYiSRJLwJlZVl+WpKkMGA/8AMwSJblbzOoyz/j\nxAkABFXllFKIAkABznGF3FjiL2IyFcTrFciZUyExUaBFCxsLFjipXv3vBkFITkK4cgUlIg8hLRtj\nPHUSx7t98VSvgXX+HCwb14GioOSOQEhIwNWwEc4evW908fJVfRpn524IcXEIbhe/RZdgxbci+/eL\n7LreV3Ua5MunUKWKnx07DBgMKqteGMMzqwYj+FN18oOvVBmurtt8I+rI7YZXKtiJj7/VvaNQHB9F\nKISPYVwmDycNJfHGaWPKlvWTK5fKb78ZSEwEv1/7mK9Pkye3lzaOGYQlnyEg1ExM83eo2TiEAgVU\n5s4106aNDYMBZsxw8vzzmiwdHZ2sp3HjxvTtO4Du3Tvh9/von1ppODNk1Bhk9Ff+I/BL6uNrgB14\nsC3LU40BgP9SLFcJ4QhlqMAB5u5/hbdYwU9Uo3K85hZZ76vN22/bKFfaw5oOK7C/+ixqWBimnT8R\n1K0jhosX8BcshOHcWVxvNSblf0NAEPA+W40kr1e7rb5DzGpMjMDy5UaCg03YbPnZscPAF1/c3Puv\nUcNHvXo+duww8N13RlauNAEqIwNG8OxXmmNZCQzCW/EJPHXq4X6z0W1tI7dvNxAfL1K0qJ/z57X9\nfhA5QQlOUAIAq0WlRDGFSpU8dO7sJTZWYPNmA8HB6o0m382be+je3UzclRT2/m8tlw4bOZmnLnGl\nnuHsRgMfzRJvrARy5FCZNUszBDo6OvcPu93O8OGjs1TmXX0GkiSd484XfQHIJcuyLTMTS5LUCW27\nyA/kQeugFgN0l2U5Nq1zfT6/ajRmzIaorVrTd2EFjhjKkdMfw6XcFdkaU5pnTL+w1fssHoONOWUn\n0e23Thjxs53naMjXxBJOX8YxyjYcQ/kyGHb/rMXllysHBw6w6fG+7Ko/mq7djezZAzt2QOXK2h36\nvn3www+QkgK5c0N4OGzcyI1OXdcpVw4mTYJSpSBPHvjycw99eni4nKQ1XRDxU5bDlBJkgp6UKF6v\nNEUjTVy4AN9+C/v3gyRBRIQ25/nzmtzgYJXBrx+kzaKX8OcryJ4xW9i6P4QtWyBHDggLg9On4Zdf\nbuoSHq71ejh3Dnx3CfAJDYWSJbW/VatCnz7aYx0dnWzLXXd10jIGhdOSKMvymYxqI0nSG8D/gFpo\nndTiUovgvYeW5NY9rfMz40C+8kpXyuxfdOO5iIKCyOrVDl64tJTgzu0AUASRgepoaph/opxnLy/z\nPccoRVkOMZCPyU00Q/mQeCEXflXgOBIAZrN64075dlTu9DkIgkq+fCqRkQoFCxo59rsH+Q8RBZGk\nJBEbDhqxnGKcYJPhVfb5y+PgzqGfRQu4OXfJhM+vrUQM+KiV7xAfp3TnUEJhVlGflTTAR/ra5NkN\nTsrkvEh+9ynCE0/wQqETFJrei6ACwQQHq9jtmc5LyxTZ1bmn6/XPya66/dv0ypADOTMX+7SQJOkV\ntNpGtWVZTuD2lpqrgU/vx7zX2X5Ca+dcv3osytYdrOYNQoN9PPXYNdR4K863m2JbtgTH+x/yxP5w\nnlq7g1zEMYke1GE9hylLSxYhoFCZ3ThUG2cpTCmOUIsNzPJ0wocFBQOgYsSPBRcpaHf3gqCSI4eK\nxawSHS2gqCIXLghcuHB9K8mEgIqKiIDC82xjAn0II54P/cM5F/QYnSrvZddeK49d+5ln2UlJ/uRF\ntlDi/J8cpgw+jBTjJCIKky++y7OsIwmtAJPNppI/t598+VTq1PGRlATXLjixJ11GOHmaa4cvkiIE\n8rxlN21dn2K74gKzGdfrr5E8bhJqaBAZ3yXU0dHJrmTUZ5AhJEkKQeuc9pIsy/Gpx74C+suyfBKo\nDhy+X/OfO+Zge2J5AI5czEEVEniNb3kmcScR0mgEVUW12UicPgvjoYM0XfshPksAl4tX52V5Gw6/\nnSEMZSJ98GJmN1otECM+CnOakhz/f3t3Hh9FmSZw/NfdOUhihBASMKjhkDwguuMCElGUcz1Rx4sd\n11uU9cBjHBFdVg0OsuPFDB+dkfXC0VlW12FV1B3cIe6goKwI4sHxwCgImwQ5hNxHH7V/VCXpxEYU\n7G60n+/nwyfV1ZWqJy9v11PvW9Xvy1jKeZ1zmMl07uRXBLzn77dyOFUcRoGzg/Ldf8diZyyKsIk+\npBOkgO2Mp5xxlHOsbw3ZTgPd2UUGQZr9XZibcQvhpiBP107iw/Is8tnFxxzHck4C3Mc8M2mmia/3\n3mVkOJw3+DOuG/4+w0f48DfU4a+qIm2Nkv7uMgJbNrdtGy7qTc2/ziNUOoP63bfQtHULeUOOoTaU\n0KpijEmw/fqewf7y7hOUARuiVs8DpgANQB1wlapu/6b97G830fybVvHIi32pCeRRHc7ltFODLBhy\nH+nLlkIoSHjgILo8N6/tKR0nKxtfYwNOIEC4X38Cmz7HFwpR0eNY5u6ayJvOeCroTRoRttDeq/YT\nVvMUV9OL7eyhG5UUcQLLyaSZBrLJY883xrmbbtSSSy25fOHvw/2Ru3iXkR22yWcnh1HFFxS3XfUD\n+H0Rig6pITetkcJQJSfkfsLUwmfJX70k5rEih3YlNOx4QiUDaRk9luDJoyC9YzfSj62pHG8W13d3\nsMb2Y4srHt8z2C+q+gTwRIy3fp+I44/YvpCtPMKA8AaqyeXCi0Kc9Yd7kUERZs50Z+yK5Pcg+zcP\n42Rl46+tITTwaBwg7bONkJFJ7X2zyLjmOnKeTmfFXV145GfLuH5bGe91O511a3zkbfyAs3mNXOoI\nFxZS1LiVwXVrCTl+FjOetzmFQawjgp8g6RxOBWmEqKA3ixnPm5zGTgrag464N47P5RUuYAFDWcmR\nbKFx+EgqZzwGh/nZtmwNO6f/jqY9TfzUeYX82vY7006dD1+lQ/Opp9NyxgR81dU4OTlECgoJ9+tP\neEBJ+7fLjDEpK6Xa/h/lnAhAJk3k+78iVAtLlmSxZAmUloY5+4wmMt94DV8ohBMK4mRmkrbeHUMh\nVCLU33k3LaefCcCiRW7RTchdQuabKxm7ezFjgeCw4bScNJmafgNwcrKJ9CoiNHAQaWvXMOrPixi3\nchFkpONk5+Ckp9Gku6iK9MQZeBxn5RcysGotFVsiVGyJ4A8FGXHEFs4bvI4j/RX4t1USKRhIuN8E\ncoJBji67nMCaT/lJnXuFUH/bHYQGz6GmqdF9vPXEkUR6FOCrrcXJz098gRtj4iISifDAA/ezadNn\npKWlMXXqP1Fc3OeA9plSyeCv27sCcGReHdN3X8fs2+/A58sjMxNuvrkL04gQqF/MMYd+wbU1synN\n+pi1464kvKee8lX5vH51KWMCL3PywG28u34aQ9NWU/LkdCLdu9Mw+XpCxxxL7q1TSP/g/Q7Hdfx+\nwiVC+KgSIj0LyXhnCYTDOF2y6FJVSTdg0Ia9fOduo/cvBsfvJzyghODhR9B08aW0nHNe7O0sERjz\no1JeXk59fR1z5z5DRcX/MWfOwwc8WF1KJYNJc0ooXfgBl80ZwZ8ZRjCSwfksYPTg3dyyahI9nS+J\n4Ke8ppRyXoRGOjzrlO4P8VT4KJ5a476+KvAMddPvpfGa6yA7m67nT8AXidBw821ECgpwAgECmzeR\n/tFq0j75mLT17myh4Z69cHIPJVBVSeOkyTSddxH+HdvpWvcVDRs34dv9FS1jxhM8aSSBDRtIW7cG\n//Yv3XsYTY0QCBAcMozQ0GE4uT/iadqMOciVlWW2fUHz+3L22SHKypq/cZvNmzczaJA7k0Dv3oez\nbVsV4XD4gIakSKlkkNc/j6bBR9DQ4COLEA4+pvEAw1eu4Ab+se3Jn7/Sn9/4buXLrD4MalhFFo30\n43Mm5CzhP4tvZrNTzNiaVxme/xnO27mkf7Qax+8nY9k7NJ96OvX/XPb1g0ci+HbuxF9XQ7hvf/cB\nfcfp+KB+QS71nW4KhUpPIFR6QhxLxRjzQ1NSUsKTTz7NxIkXU1GxlcrKCqqr99C9+/73AqRUMgD4\n07NfAr14h5M5gq0UZNbiOBn4W9w5DJysbIomjmLGjacQ6dMX/5Y+dHnpBdJWVRP4PJfLPp3evrOt\nHfftZGRQf88vYx/Y78cpLCRcWNi+zuZzNOYHrayseZ9X8fEwatQoli5dzpQp19K//wCKi/tyoE+G\nplQyiERg0dvZHEYlQ1hF4zXXsevnU3EK3Kd3/Js3uZO+RPWxR44spuEX09pe+ysr8O/cQfjIYreL\nJhLBV12Nf89unOxsIkW9E/53GWNSz+TJN7QtT5x4Lnmd50X5jn7wM5Z9Fx+/tZudLYdyBn8i3K8/\n9bMebEsEAJE+ffd5szVS1JvQ3xyH0y3PfSQzPR2nRw/CRw2wRGCMSYj169cza9YMAJYvf5eSkoH4\nD3AS95RqGbw3cynwD5zpX0TDXXcnOxxjjNkvJSUlOI7DtddeTkZGJvfsrXv6O0ipZDB0WISJmxZS\nOn8KzScNSXY4xhizX/x+P9Onl32v+0ypZDD84XM56/cH59fLjTEmmVLqnoExxpjYLBkYY4yxZGCM\nMcaSgTHGGCwZGGOMwZKBMcYYLBkYY4zBkoExxhgSPAeyMcaYg5O1DIwxxlgyMMYYY8nAGGMMlgyM\nMcZgycAYYwyWDIwxxmDJwBhjDCk2uY2I/Bo4AXCAW1R1RRJjeRA4Gff/4F+Ac4ChwC5vk4dU9Y0k\nxDUaeAlY4636BHgQeB4IAFXAZaranOC4JgGXRa0aBnwA5AD13rpfqOrKBMZ0DPAq8GtVfUxEjiBG\nOYnIJcB3ineXAAAGhElEQVStQAR4QlWfTkJc84B0IAhcqqrbRCQILIv61XGqGk5gXM8So84fBOX1\nEtA6OXp3YDkwC/ez0Fq/dqjqRXGOq/M5YgVxrF8pkwxEZBQwQFVHiMgg4BlgRJJiGQMc48WSD3wI\nvAXcpaqvJyOmTpao6oWtL0RkHvBbVX1JRGYBVwOPJzIgr4I/7cUzCpgIDAauUtVPExmLF0MO8ChQ\nHrX6PjqVk4g8B9wDDAdagBUi8rKqfpXAuGbiniT+Q0RuBG4D7gCqVXV0POL4lnFBpzrvbZfU8oo+\nyYvIM8BT7W8lrLxinSPKiWP9SqVuonHAKwCqug7IE5FDkxTL20BrhduDe3UbSFIs38ZoYKG3/Bow\nPnmhAG7lP/AZwA9MM3AmUBm1bjRfL6dSYIWqVqtqI+6V+EkJjusGYIG3vAPIj+Px9yZWXLEcDOUF\ngIgI0E1V34/j8fcm1jliNHGsXynTMgB60d7EA/dD0QuoSXQgXlO8tWtjEvBfQBiYIiK3AduBKaq6\nM9GxeY4WkYW4TeQZQE5Ut9B24LAkxYWIHA9s9bo5AO4TkR7AOuBW7wMRd6oaAkJeDK1ilVMv3LpG\np/UJi0tV6wFEJADciNuCAegiIvOBYmCBqs5OZFyeDnWeg6C8otyC22po1UtE/ggU4V6h/1sc44p1\njjgtnvUrlVoGnfmSHYCInIv7Hz0Fty/wTlUdC6wGypIU1kbcBHAucAVu10z0RUOyy+0a4FlveQ4w\nVVVPwe0vvTFZQcWwt3JKSvl5ieB54C1Vbe0SuR2YDJwKXCIiwxIc1rep88kqrwxgpKr+j7dqF3A3\ncDHu/b1fikjcL4o6nSOife/1K5VaBpW4WbRVEe5NmKQQkdOA6cDpqlpNx77UhSS4T76VqlYAL3ov\nPxORbcDxIpLlXXX3Zt9N/XgaDdwEoKovR61/Dfj7ZAQUpS5GOXWud71xb0gm2jxgo6rOaF2hqnNb\nl0WkHDgW96Z8QkQlJWiv83/k4CivUUBb95Cq1uKWIcBOEfkAGEgczyGdzxEiEtf6lUotg/8GLgQQ\nkSFApfcfnHAi0hV4CJjQeqNHRBaISD9vk9FAwm+KenFcIiK3e8u9gJ64H4ILvE0uABYlKbYioE5V\nW0TEJyKLRaSb9/ZoklRmURbz9XL6X9xk2k1EDsHtz30nkUF5T5u0qOq9UetEROZ75ZjmxbVmrzuJ\nT1yx6nzSy8tzPPBR6wsRGSMis73lHOA4YEO8Dh7rHEGc61dKDWEtIr8C2roUVPWjffxKvOKYjNsk\njq5M83Cbgg1AHe5TMtuTEFsuMB/oBmTgdhl9CDwHdAG+8GILJiG2ocBMVT3Dez0RmIbbt1oBTFLV\nhgTG8gjQB/dxzQrgEtwurA7lJCIXAlNxH2l+NJ59zXuJqxBoov3+2FpVvUFEHgDG4n4eFqrq/QmO\n61HgTjrV+YOgvM7HrfdLVfVFb7s03KeKBPdhj8dVdV6sfX5PccU6R1zhxRCX+pVSycAYY0xsqdRN\nZIwxZi8sGRhjjLFkYIwxxpKBMcYYLBkYY4whtb50ZkxMItIHUOC9Tm+9oaoP7eV3XsAdJbXiAI57\nFLBYVfvs7z6M+b5YMjDGteO7jEipqj+LYyzGJJwlA2O+gYiEcEdIHQMcAlypqp+KyGbcUSO7AE/g\njn6ZDdznjclfivtlpiDul4GmqOpaETkRmIs7uNjKqOPkeesLgK7AI6o6PyF/pDHYPQNj9iUAfOq1\nGh6nfcTPVtcCr6rqGOBs2oeHfg74ubd+NvBbb/3DwDRVHQdsi9rPTGCRN2jbKbijsRZgTIJYy8AY\nV4GI/KXTuju8n296P5fhfu0/2gLgWREpBl4HnvfGS+oZNZPeX4AXvOVjgaXe8lvAzd7yGNwxZq7w\nXgeBvnQcntiYuLFkYIwr5j0Db5z71ha0D7fLp42qvu1NmzgOuBK4FLi+026if8+HOxYQdJzQqBm4\nQVUTNmqoMdGsm8iYfRvr/RwJfBz9hojcBByuqq/hjjtf6g1JXuXdNwD33kLrsMJraZ9uNXrGuKW4\nU3kiIlki8jtvcDRjEsIqmzGuWN1Em7yffysi1wN5wOWdtlkP/LuI1OBe6d/prb8cmC0iYdxZ7Fpb\nC3cAj4nIFtzRYFuVAU+JyFIgE3fO4tAB/1XGfEs2aqkx30BEHCDdTszmx866iYwxxljLwBhjjLUM\njDHGYMnAGGMMlgyMMcZgycAYYwyWDIwxxgD/D0K6ttk1mbNeAAAAAElFTkSuQmCC\n",
334 |             "text/plain": [
335 |               "<Figure size 432x288 with 1 Axes>"
336 |             ]
337 |           },
338 |           "metadata": {
339 |             "tags": []
340 |           }
341 |         }
342 |       ]
343 |     },
344 |     {
345 |       "metadata": {
346 |         "id": "_UBHQMHzqsMU",
347 |         "colab_type": "code",
348 |         "colab": {}
349 |       },
350 |       "cell_type": "code",
351 |       "source": [
352 |         "env.close()"
353 |       ],
354 |       "execution_count": 0,
355 |       "outputs": []
356 |     }
357 |   ]
358 | }


--------------------------------------------------------------------------------