104 |
105 |
106 |
107 |
108 |
109 |
1
110 |
111 | 0
112 |
113 | ←
114 |
115 |
116 |
117 | 0
118 |
119 | →
120 |
121 |
122 |
123 | 0
124 |
125 | ↑
126 |
127 |
128 |
129 | 0
130 |
131 | ↓
132 |
133 |
134 |
135 |
136 |
2
137 |
138 | 0
139 |
140 | ←
141 |
142 |
143 |
144 | 0
145 |
146 | →
147 |
148 |
149 |
150 | 0
151 |
152 | ↑
153 |
154 |
155 |
156 | 0
157 |
158 | ↓
159 |
160 |
161 |
162 |
3
163 |
164 | 0
165 |
166 | ←
167 |
168 |
169 |
170 | 0
171 |
172 | →
173 |
174 |
175 |
176 | 0
177 |
178 | ↑
179 |
180 |
181 |
182 | 0
183 |
184 | ↓
185 |
186 |
187 |
188 |
4
189 |
190 | 0
191 |
192 | ←
193 |
194 |
195 |
196 | 0
197 |
198 | →
199 |
200 |
201 |
202 | 0
203 |
204 | ↑
205 |
206 |
207 |
208 | 0
209 |
210 | ↓
211 |
212 |
213 |
214 |
5
215 |
216 | 0
217 |
218 | ←
219 |
220 |
221 |
222 | 0
223 |
224 | →
225 |
226 |
227 |
228 | 0
229 |
230 | ↑
231 |
232 |
233 |
234 | 0
235 |
236 | ↓
237 |
238 |
239 |
240 |
6
241 |
242 | 0
243 |
244 | ←
245 |
246 |
247 |
248 | 0
249 |
250 | →
251 |
252 |
253 |
254 | 0
255 |
256 | ↑
257 |
258 |
259 |
260 | 0
261 |
262 | ↓
263 |
264 |
265 |
266 |
7
267 |
268 | 0
269 |
270 | ←
271 |
272 |
273 |
274 | 0
275 |
276 | →
277 |
278 |
279 |
280 | 0
281 |
282 | ↑
283 |
284 |
285 |
286 | 0
287 |
288 | ↓
289 |
290 |
291 |
292 |
8
293 |
294 | 0
295 |
296 | ←
297 |
298 |
299 |
300 | 0
301 |
302 | →
303 |
304 |
305 |
306 | 0
307 |
308 | ↑
309 |
310 |
311 |
312 | 0
313 |
314 | ↓
315 |
316 |
317 |
318 |
9
319 |
320 | 0
321 |
322 | ←
323 |
324 |
325 |
326 | 0
327 |
328 | →
329 |
330 |
331 |
332 | 0
333 |
334 | ↑
335 |
336 |
337 |
338 | 0
339 |
340 | ↓
341 |
342 |
343 |
344 |
345 |
438 |
439 |
440 |
--------------------------------------------------------------------------------
/rl/policy_gradient.py:
--------------------------------------------------------------------------------
1 | from random import randint
2 | import tensorflow as tf
3 | import numpy as np
4 | import scipy.signal
5 |
6 | """
7 | Exemple of the Policy Gradient Algorithm
8 | """
9 |
10 | class Buffer:
11 |
12 | def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):
13 | self.obs_buf = np.zeros(Buffer.combined_shape(size, obs_dim), dtype=np.float32)
14 | # Actions buffer
15 | self.act_buf = np.zeros(size, dtype=np.float32)
16 | # Advantages buffer
17 | self.adv_buf = np.zeros(size, dtype=np.float32)
18 | # Rewards buffer
19 | self.rew_buf = np.zeros(size, dtype=np.float32)
20 | # Log probability of action a with the policy
21 | self.logp_buf = np.zeros(size, dtype=np.float32)
22 | # Gamma and lam to compute the advantage
23 | self.gamma, self.lam = gamma, lam
24 | # ptr: Position to insert the next tuple
25 | # path_start_idx Posittion of the current trajectory
26 | # max_size Max size of the buffer
27 | self.ptr, self.path_start_idx, self.max_size = 0, 0, size
28 |
29 | @staticmethod
30 | def discount_cumsum(x, discount):
31 | """
32 | x = [x0, x1, x2]
33 | output: [x0 + discount * x1 + discount^2 * x2, x1 + discount * x2, x2]
34 | """
35 | return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]
36 |
37 | @staticmethod
38 | def combined_shape(length, shape=None):
39 | if shape is None:
40 | return (length,)
41 | return (length, shape) if np.isscalar(shape) else (length, *shape)
42 |
43 | def store(self, obs, act, rew, logp):
44 | """
45 | Append one timestep of agent-environment interaction to the buffer.
46 | """
47 | assert self.ptr < self.max_size
48 | self.obs_buf[self.ptr] = obs
49 | self.act_buf[self.ptr] = act
50 | self.rew_buf[self.ptr] = rew
51 | self.logp_buf[self.ptr] = logp
52 | self.ptr += 1
53 |
54 | def finish_path(self, last_val=0):
55 | # Select the path
56 | path_slice = slice(self.path_start_idx, self.ptr)
57 | # Append the last_val to the trajectory
58 | rews = np.append(self.rew_buf[path_slice], last_val)
59 | # Advantage
60 | self.adv_buf[path_slice] = Buffer.discount_cumsum(rews, self.gamma)[:-1]
61 | self.path_start_idx = self.ptr
62 |
63 | def get(self):
64 | assert self.ptr == self.max_size # buffer has to be full before you can get
65 | self.ptr, self.path_start_idx = 0, 0
66 | # the next two lines implement the advantage normalization trick
67 | # Normalize the Advantage
68 | self.adv_buf = (self.adv_buf - np.mean(self.adv_buf)) / np.std(self.adv_buf)
69 | return self.obs_buf, self.act_buf, self.adv_buf, self.logp_buf
70 |
71 |
72 | class PolicyGradient(object):
73 | """
74 | Implementation of Policy gradient algorithm
75 | """
76 | def __init__(self, input_space, action_space, pi_lr, buffer_size, seed):
77 | super(PolicyGradient, self).__init__()
78 |
79 | # Stored the spaces
80 | self.input_space = input_space
81 | self.action_space = action_space
82 | self.seed = seed
83 | # NET Buffer defined above
84 | self.buffer = Buffer(
85 | obs_dim=input_space,
86 | act_dim=action_space,
87 | size=buffer_size
88 | )
89 | # Learning rate of the policy network
90 | self.pi_lr = pi_lr
91 | # The tensorflow session (set later)
92 | self.sess = None
93 | # Apply a random seed on tensorflow and numpy
94 | tf.set_random_seed(42)
95 | np.random.seed(42)
96 |
97 | def compile(self):
98 | """
99 | Compile the model
100 | """
101 | # tf_map: Input: Input state
102 | # tf_adv: Input: Advantage
103 | self.tf_map, self.tf_a, self.tf_adv = PolicyGradient.inputs(
104 | map_space=self.input_space,
105 | action_space=self.action_space
106 | )
107 | # mu_op: Used to get the exploited prediction of the model
108 | # pi_op: Used to get the prediction of the model
109 | # logp_a_op: Used to get the log likelihood of taking action a with the current policy
110 | # logp_pi_op: Used to get the log likelihood of the predicted action @pi_op
111 | # log_std: Used to get the currently used log_std
112 | self.mu_op, self.pi_op, self.logp_a_op, self.logp_pi_op = PolicyGradient.mlp(
113 | tf_map=self.tf_map,
114 | tf_a=self.tf_a,
115 | action_space=self.action_space,
116 | seed=self.seed
117 | )
118 | # Error
119 | self.pi_loss = PolicyGradient.net_objectives(
120 | tf_adv=self.tf_adv,
121 | logp_a_op=self.logp_a_op
122 | )
123 | # Optimization
124 | self.train_pi = tf.train.AdamOptimizer(learning_rate=self.pi_lr).minimize(self.pi_loss)
125 | # Entropy
126 | self.approx_ent = tf.reduce_mean(-self.logp_a_op)
127 |
128 |
129 | def set_sess(self, sess):
130 | # Set the tensorflow used to run this model
131 | self.sess = sess
132 |
133 | def step(self, states):
134 | # Take actions given the states
135 | # Return mu (policy without exploration), pi (policy with the current exploration) and
136 | # the log probability of the action chossen by pi
137 | mu, pi, logp_pi = self.sess.run([self.mu_op, self.pi_op, self.logp_pi_op], feed_dict={
138 | self.tf_map: states
139 | })
140 | return mu, pi, logp_pi
141 |
142 | def store(self, obs, act, rew, logp):
143 | # Store the observation, action, reward and the log probability of the action
144 | # into the buffer
145 | self.buffer.store(obs, act, rew, logp)
146 |
147 | def finish_path(self, last_val=0):
148 | self.buffer.finish_path(last_val=last_val)
149 |
150 | def train(self, additional_infos={}):
151 | # Get buffer
152 | obs_buf, act_buf, adv_buf, logp_last_buf = self.buffer.get()
153 | # Train the model
154 | pi_loss_list = []
155 | entropy_list = []
156 |
157 | for step in range(5):
158 | _, entropy, pi_loss = self.sess.run([self.train_pi, self.approx_ent, self.pi_loss], feed_dict= {
159 | self.tf_map: obs_buf,
160 | self.tf_a:act_buf,
161 | self.tf_adv: adv_buf
162 | })
163 |
164 | pi_loss_list.append(pi_loss)
165 | entropy_list.append(entropy)
166 |
167 | print("Entropy : %s, Loss: %s" % (np.mean(entropy_list), np.mean(pi_loss_list)), end="\r")
168 |
169 |
170 | @staticmethod
171 | def gaussian_likelihood(x, mu, log_std):
172 | # Compute the gaussian likelihood of x with a normal gaussian distribution of mean @mu
173 | # and a std @log_std
174 | pre_sum = -0.5 * (((x-mu)/(tf.exp(log_std)+1e-8))**2 + 2*log_std + np.log(2*np.pi))
175 | return tf.reduce_sum(pre_sum, axis=1)
176 |
177 | @staticmethod
178 | def inputs(map_space, action_space):
179 | """
180 | @map_space Tuple of the space. Ex (size,)
181 | @action_space Tuple describing the action space. Ex (size,)
182 | """
183 | # Map of the game
184 | tf_map = tf.placeholder(tf.float32, shape=(None, *map_space), name="tf_map")
185 | # Possible actions (Should be two: x,y for the beacon game)
186 | tf_a = tf.placeholder(tf.int32, shape=(None,), name="tf_a")
187 | # Advantage
188 | tf_adv = tf.placeholder(tf.float32, shape=(None,), name="tf_adv")
189 | return tf_map, tf_a, tf_adv
190 |
191 | @staticmethod
192 | def mlp(tf_map, tf_a, action_space, seed=None):
193 | if seed is not None:
194 | tf.random.set_random_seed(seed)
195 |
196 | # Expand the dimension of the input
197 | tf_map_expand = tf.expand_dims(tf_map, axis=3)
198 |
199 | flatten = tf.layers.flatten(tf_map_expand)
200 | hidden = tf.layers.dense(flatten, units=256, activation=tf.nn.relu)
201 | spacial_action_logits = tf.layers.dense(hidden, units=action_space, activation=None)
202 |
203 | # Add take the log of the softmax
204 | logp_all = tf.nn.log_softmax(spacial_action_logits)
205 | # Take random actions according to the logits (Exploration)
206 | pi_op = tf.squeeze(tf.multinomial(spacial_action_logits,1), axis=1)
207 | mu = tf.argmax(spacial_action_logits, axis=1)
208 |
209 | # Gives log probability, according to the policy, of taking actions @a in states @x
210 | logp_a_op = tf.reduce_sum(tf.one_hot(tf_a, depth=action_space) * logp_all, axis=1)
211 | # Gives log probability, according to the policy, of the action sampled by pi.
212 | logp_pi_op = tf.reduce_sum(tf.one_hot(pi_op, depth=action_space) * logp_all, axis=1)
213 |
214 | return mu, pi_op, logp_a_op, logp_pi_op
215 |
216 | @staticmethod
217 | def net_objectives(logp_a_op, tf_adv, clip_ratio=0.2):
218 | """
219 | @v_op: Predicted value function
220 | @tf_tv: Expected advantage
221 | @logp_a_op: Log likelihood of taking action under the current policy
222 | @tf_logp_old_pi: Log likelihood of the last policy
223 | @tf_adv: Advantage input
224 | """
225 | pi_loss = -tf.reduce_mean(logp_a_op*tf_adv)
226 | return pi_loss
227 |
228 | class GridWorld(object):
229 | """
230 | docstring for GridWorld.
231 | """
232 | def __init__(self):
233 | super(GridWorld, self).__init__()
234 |
235 | self.rewards = [
236 | [0, 0, 0, 0, -1, 0, 0],
237 | [0, -1, -1, 0, -1, 0, 0],
238 | [0, -1, -1, 1, -1, 0, 0],
239 | [0, -1, -1, 0, -1, 0, 0],
240 | [0, 0, 0, 0, 0, 0, 0],
241 | [0, 0, 0, 0, 0, 0, 0],
242 | [0, 0, 0, 0, 0, 0, 0],
243 | ]
244 | self.position = [6, 6] # y, x
245 |
246 | def gen_state(self):
247 | # Generate a state given the current position of the agent
248 | state = np.zeros((7, 7))
249 | state[self.position[0]][self.position[1]] = 1
250 | return state
251 |
252 | def step(self, action):
253 | if action == 0: # Top
254 | self.position = [(self.position[0] - 1) % 7, self.position[1]]
255 | elif action == 1: # Left
256 | self.position = [self.position[0], (self.position[1] - 1) % 7]
257 | elif action == 2: # Right
258 | self.position = [self.position[0], (self.position[1] + 1) % 7]
259 | elif action == 3: # Down
260 | self.position = [(self.position[0] + 1) % 7, self.position[1]]
261 |
262 | reward = self.rewards[self.position[0]][self.position[1]]
263 | done = False if reward == 0 else True
264 | state = self.gen_state()
265 | if done: # The agent is dead, reset the game
266 | self.position = [6, 6]
267 | return state, reward, done
268 |
269 | def display(self):
270 | y = 0
271 | print("="*14)
272 | for line in self.rewards:
273 | x = 0
274 | for case in line:
275 | if case == -1:
276 | c = "0"
277 | elif (y == self.position[0] and x == self.position[1]):
278 | c = "A"
279 | elif case == 1:
280 | c = "T"
281 | else:
282 | c = "-"
283 | print(c, end=" ")
284 | x += 1
285 | y += 1
286 | print()
287 |
288 | def main():
289 | grid = GridWorld()
290 | buffer_size = 1000
291 |
292 | # Create the NET class
293 | agent = PolicyGradient(
294 | input_space=(7, 7),
295 | action_space=4,
296 | pi_lr=0.001,
297 | buffer_size=buffer_size,
298 | seed=42
299 | )
300 | agent.compile()
301 | # Init Session
302 | sess = tf.Session()
303 | # Init variables
304 | sess.run(tf.global_variables_initializer())
305 | # Set the session
306 | agent.set_sess(sess)
307 |
308 | rewards = []
309 |
310 | b = 0
311 |
312 | for epoch in range(10000):
313 |
314 | done = False
315 | state = grid.gen_state()
316 |
317 | while not done:
318 | _, pi, logpi = agent.step([state])
319 | n_state, reward, done = grid.step(pi[0])
320 | agent.store(state, pi[0], reward, logpi)
321 | b += 1
322 |
323 | state = n_state
324 |
325 | if done:
326 | agent.finish_path(reward)
327 | rewards.append(reward)
328 | if len(rewards) > 1000:
329 | rewards.pop(0)
330 | if b == buffer_size:
331 | if not done:
332 | agent.finish_path(0)
333 | agent.train()
334 | b = 0
335 |
336 | if epoch % 1000 == 0:
337 | print("Rewards mean:%s" % np.mean(rewards))
338 |
339 | for epoch in range(10):
340 | import time
341 | print("=========================TEST=================================")
342 | done = False
343 | state = grid.gen_state()
344 |
345 | while not done:
346 | time.sleep(1)
347 | _, pi, logpi = agent.step([state])
348 | n_state, _, done = grid.step(pi[0])
349 | grid.display()
350 | state = n_state
351 | print("reward=>", reward)
352 |
353 | if __name__ == '__main__':
354 | main()
355 |
--------------------------------------------------------------------------------
/rl/policy_gradient_continuous_actions.py:
--------------------------------------------------------------------------------
1 | from random import randint
2 | import tensorflow as tf
3 | import numpy as np
4 | import scipy.signal
5 |
6 | """
7 | Exemple of the Policy Gradient Algorithm
8 | """
9 |
10 | class Buffer:
11 |
12 | def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):
13 | self.obs_buf = np.zeros(Buffer.combined_shape(size, obs_dim), dtype=np.float32)
14 | # Actions buffer
15 | self.act_buf = np.zeros((size, act_dim), dtype=np.float32)
16 | # Advantages buffer
17 | self.adv_buf = np.zeros(size, dtype=np.float32)
18 | # Rewards buffer
19 | self.rew_buf = np.zeros(size, dtype=np.float32)
20 | # Log probability of action a with the policy
21 | self.logp_buf = np.zeros(size, dtype=np.float32)
22 | # Gamma and lam to compute the advantage
23 | self.gamma, self.lam = gamma, lam
24 | # ptr: Position to insert the next tuple
25 | # path_start_idx Posittion of the current trajectory
26 | # max_size Max size of the buffer
27 | self.ptr, self.path_start_idx, self.max_size = 0, 0, size
28 |
29 | @staticmethod
30 | def discount_cumsum(x, discount):
31 | """
32 | x = [x0, x1, x2]
33 | output: [x0 + discount * x1 + discount^2 * x2, x1 + discount * x2, x2]
34 | """
35 | return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]
36 |
37 | @staticmethod
38 | def combined_shape(length, shape=None):
39 | if shape is None:
40 | return (length,)
41 | return (length, shape) if np.isscalar(shape) else (length, *shape)
42 |
43 | def store(self, obs, act, rew, logp):
44 | """
45 | Append one timestep of agent-environment interaction to the buffer.
46 | """
47 | assert self.ptr < self.max_size
48 | self.obs_buf[self.ptr] = obs
49 | self.act_buf[self.ptr] = act
50 | self.rew_buf[self.ptr] = rew
51 | self.logp_buf[self.ptr] = logp
52 | self.ptr += 1
53 |
54 | def finish_path(self, last_val=0):
55 | # Select the path
56 | path_slice = slice(self.path_start_idx, self.ptr)
57 | # Append the last_val to the trajectory
58 | rews = np.append(self.rew_buf[path_slice], last_val)
59 | # Advantage
60 | self.adv_buf[path_slice] = Buffer.discount_cumsum(rews, self.gamma)[:-1]
61 | self.path_start_idx = self.ptr
62 |
63 | def get(self):
64 | assert self.ptr == self.max_size # buffer has to be full before you can get
65 | self.ptr, self.path_start_idx = 0, 0
66 | # the next two lines implement the advantage normalization trick
67 | # Normalize the Advantage
68 | if np.std(self.adv_buf) != 0:
69 | self.adv_buf = (self.adv_buf - np.mean(self.adv_buf)) / np.std(self.adv_buf)
70 | return self.obs_buf, self.act_buf, self.adv_buf, self.logp_buf
71 |
72 |
73 | class PolicyGradient(object):
74 | """
75 | Implementation of Policy gradient algorithm
76 | """
77 | def __init__(self, input_space, action_space, pi_lr, buffer_size, seed):
78 | super(PolicyGradient, self).__init__()
79 |
80 | # Stored the spaces
81 | self.input_space = input_space
82 | self.action_space = action_space
83 | self.seed = seed
84 | # NET Buffer defined above
85 | self.buffer = Buffer(
86 | obs_dim=input_space,
87 | act_dim=action_space,
88 | size=buffer_size
89 | )
90 | # Learning rate of the policy network
91 | self.pi_lr = pi_lr
92 | # The tensorflow session (set later)
93 | self.sess = None
94 | # Apply a random seed on tensorflow and numpy
95 | tf.set_random_seed(42)
96 | np.random.seed(42)
97 |
98 | def compile(self):
99 | """
100 | Compile the model
101 | """
102 | # tf_map: Input: Input state
103 | # tf_adv: Input: Advantage
104 | self.tf_map, self.tf_a, self.tf_adv = PolicyGradient.inputs(
105 | map_space=self.input_space,
106 | action_space=self.action_space
107 | )
108 | # mu_op: Used to get the exploited prediction of the model
109 | # pi_op: Used to get the prediction of the model
110 | # logp_a_op: Used to get the log likelihood of taking action a with the current policy
111 | # logp_pi_op: Used to get the log likelihood of the predicted action @pi_op
112 | # log_std: Used to get the currently used log_std
113 | self.mu_op, self.pi_op, self.logp_a_op, self.logp_pi_op, self.std = PolicyGradient.mlp(
114 | tf_map=self.tf_map,
115 | tf_a=self.tf_a,
116 | action_space=self.action_space,
117 | seed=self.seed
118 | )
119 | # Error
120 | self.pi_loss = PolicyGradient.net_objectives(
121 | tf_adv=self.tf_adv,
122 | logp_a_op=self.logp_a_op
123 | )
124 | # Optimization
125 | self.train_pi = tf.train.AdamOptimizer(learning_rate=self.pi_lr).minimize(self.pi_loss)
126 | # Entropy
127 | self.approx_ent = tf.reduce_mean(-self.logp_a_op)
128 |
129 |
130 | def set_sess(self, sess):
131 | # Set the tensorflow used to run this model
132 | self.sess = sess
133 |
134 | def step(self, states):
135 | # Take actions given the states
136 | # Return mu (policy without exploration), pi (policy with the current exploration) and
137 | # the log probability of the action chossen by pi
138 | mu, pi, logp_pi = self.sess.run([self.mu_op, self.pi_op, self.logp_pi_op], feed_dict={
139 | self.tf_map: states
140 | })
141 | return mu, pi, logp_pi
142 |
143 | def store(self, obs, act, rew, logp):
144 | # Store the observation, action, reward and the log probability of the action
145 | # into the buffer
146 | self.buffer.store(obs, act, rew, logp)
147 |
148 | def finish_path(self, last_val=0):
149 | self.buffer.finish_path(last_val=last_val)
150 |
151 | def train(self, additional_infos={}):
152 | # Get buffer
153 | obs_buf, act_buf, adv_buf, logp_last_buf = self.buffer.get()
154 | # Train the model
155 | pi_loss_list = []
156 | entropy_list = []
157 |
158 | import time
159 | t = time.time()
160 |
161 | for step in range(5):
162 | _, entropy, pi_loss, std = self.sess.run([self.train_pi, self.approx_ent, self.pi_loss, self.std], feed_dict= {
163 | self.tf_map: obs_buf,
164 | self.tf_a:act_buf,
165 | self.tf_adv: adv_buf
166 | })
167 |
168 | pi_loss_list.append(pi_loss)
169 | entropy_list.append(entropy)
170 |
171 | print("Std: %.2f, Entropy : %.2f" % (std[0], np.mean(entropy_list)), end="\r")
172 |
173 |
174 | @staticmethod
175 | def gaussian_likelihood(x, mu, log_std):
176 | # Compute the gaussian likelihood of x with a normal gaussian distribution of mean @mu
177 | # and a std @log_std
178 | pre_sum = -0.5 * (((x-mu)/(tf.exp(log_std)+1e-8))**2 + 2*log_std + np.log(2*np.pi))
179 | return tf.reduce_sum(pre_sum, axis=1)
180 |
181 | @staticmethod
182 | def inputs(map_space, action_space):
183 | """
184 | @map_space Tuple of the space. Ex (size,)
185 | @action_space Tuple describing the action space. Ex (size,)
186 | """
187 | # Map of the game
188 | tf_map = tf.placeholder(tf.float32, shape=(None, *map_space), name="tf_map")
189 | # Possible actions (Should be two: x,y for the beacon game)
190 | tf_a = tf.placeholder(tf.float32, shape=(None, action_space), name="tf_a")
191 | # Advantage
192 | tf_adv = tf.placeholder(tf.float32, shape=(None,), name="tf_adv")
193 | return tf_map, tf_a, tf_adv
194 |
195 | @staticmethod
196 | def gaussian_likelihood(x, mu, log_std):
197 | # Compute the gaussian likelihood of x with a normal gaussian distribution of mean @mu
198 | # and a std @log_std
199 | pre_sum = -0.5 * (((x-mu)/(tf.exp(log_std)+1e-8))**2 + 2*log_std + np.log(2*np.pi))
200 | return tf.reduce_sum(pre_sum, axis=1)
201 | #pre_sum = (1.*tf.exp((-(x-mu)**2)/(2*std**2)))/tf.sqrt(2*3.14*std**2)
202 | #retun
203 |
204 | @staticmethod
205 | def mlp(tf_map, tf_a, action_space, seed=None):
206 | if seed is not None:
207 | tf.random.set_random_seed(seed)
208 |
209 | # Expand the dimension of the input
210 | tf_map_expand = tf.expand_dims(tf_map, axis=3)
211 |
212 | flatten = tf.layers.flatten(tf_map_expand)
213 | hidden = tf.layers.dense(flatten, units=256, activation=tf.nn.relu)
214 | mu = tf.layers.dense(hidden, units=action_space, activation=tf.nn.relu)
215 |
216 | log_std = tf.get_variable(name='log_std', initializer=-0.5*np.ones(action_space, dtype=np.float32))
217 | std = tf.exp(log_std)
218 | pi_op = mu + tf.random_normal(tf.shape(mu)) * std
219 |
220 | # Gives log probability, according to the policy, of taking actions @a in states @x
221 | logp_a_op = PolicyGradient.gaussian_likelihood(tf_a, mu, log_std)
222 | # Gives log probability, according to the policy, of the action sampled by pi.
223 | logp_pi_op = PolicyGradient.gaussian_likelihood(pi_op, mu, log_std)
224 |
225 | return mu, pi_op, logp_a_op, logp_pi_op, std
226 |
227 | @staticmethod
228 | def net_objectives(logp_a_op, tf_adv, clip_ratio=0.2):
229 | """
230 | @v_op: Predicted value function
231 | @tf_tv: Expected advantage
232 | @logp_a_op: Log likelihood of taking action under the current policy
233 | @tf_logp_old_pi: Log likelihood of the last policy
234 | @tf_adv: Advantage input
235 | """
236 | pi_loss = -tf.reduce_mean(logp_a_op*tf_adv)
237 | return pi_loss
238 |
239 | class GridWorld(object):
240 | """
241 | docstring for GridWorld.
242 | """
243 | def __init__(self):
244 | super(GridWorld, self).__init__()
245 |
246 | self.rewards = [
247 | [0, 0, 0, 0, -1, 0, 0],
248 | [0, -1, -1, 0, -1, 0, 0],
249 | [0, -1, -1, 1, -1, 0, 0],
250 | [0, -1, -1, 0, -1, 0, 0],
251 | [0, 0, 0, 0, 0, 0, 0],
252 | [0, 0, 0, 0, 0, 0, 0],
253 | [0, 0, 0, 0, 0, 0, 0],
254 | ]
255 | self.position = [6, 6] # y, x
256 |
257 | def gen_state(self):
258 | # Generate a state given the current position of the agent
259 | state = np.zeros((7, 7))
260 | state[self.position[0]][self.position[1]] = 1
261 | return state
262 |
263 | def step(self, y, x):
264 | # y and x coordinates
265 |
266 | # Move in the direction of the "click"
267 | self.position = [
268 | self.position[0] + min(1, max(-1, (y - self.position[0]))),
269 | self.position[1] + min(1, max(-1, (x - self.position[1])))
270 | ]
271 | reward = self.rewards[self.position[0]][self.position[1]]
272 | done = False if reward == 0 else True
273 | state = self.gen_state()
274 | if done: # The agent is dead, reset the game
275 | self.position = [6, 6]
276 | return state, reward, done
277 |
278 | def display(self):
279 | y = 0
280 | print("="*14)
281 | for line in self.rewards:
282 | x = 0
283 | for case in line:
284 | if case == -1:
285 | c = "0"
286 | elif (y == self.position[0] and x == self.position[1]):
287 | c = "A"
288 | elif case == 1:
289 | c = "T"
290 | else:
291 | c = "-"
292 | print(c, end=" ")
293 | x += 1
294 | y += 1
295 | print()
296 |
297 | def main():
298 | grid = GridWorld()
299 | buffer_size = 1000
300 |
301 | # Create the NET class
302 | agent = PolicyGradient(
303 | input_space=(7, 7),
304 | action_space=2,
305 | pi_lr=0.001,
306 | buffer_size=buffer_size,
307 | seed=42
308 | )
309 | agent.compile()
310 | # Init Session
311 | sess = tf.Session()
312 | # Init variables
313 | sess.run(tf.global_variables_initializer())
314 | # Set the session
315 | agent.set_sess(sess)
316 |
317 | rewards = []
318 |
319 | b = 0
320 | display = False
321 |
322 | for epoch in range(100000):
323 |
324 | done = False
325 | state = grid.gen_state()
326 |
327 | s = 0
328 | while not done and s < 20:
329 | s += 1
330 | _, pi, logpi = agent.step([state])
331 |
332 | y = max(0, min(6, int(round((pi[0][0]+1.) / 2*6))))
333 | x = max(0, min(6, int(round((pi[0][1]+1.) / 2*6))))
334 |
335 | if display:
336 | import time
337 | time.sleep(0.1)
338 | grid.display()
339 |
340 | n_state, reward, done = grid.step(y, x)
341 | agent.store(state, pi[0], -0.1 if reward == 0 else reward, logpi)
342 | b += 1
343 |
344 | state = n_state
345 |
346 | if done:
347 | agent.finish_path(reward)
348 | rewards.append(reward)
349 | if len(rewards) > 1000:
350 | rewards.pop(0)
351 | if b == buffer_size:
352 | if not done:
353 | agent.finish_path(0)
354 | agent.train()
355 | b = 0
356 |
357 | if epoch % 1000 == 0:
358 | print("\nEpoch: %s Rewards mean:%s" % (epoch, np.mean(rewards)))
359 |
360 | for epoch in range(10):
361 | import time
362 | print("=========================TEST=================================")
363 | done = False
364 | state = grid.gen_state()
365 |
366 | while not done:
367 | time.sleep(1)
368 | mu, pi, logpi = agent.step([state])
369 |
370 | y = max(0, min(6, int(round((pi[0][0]+1.) / 2*6))))
371 | x = max(0, min(6, int(round((pi[0][1]+1.) / 2*6))))
372 |
373 | n_state, _, done = grid.step(y, x)
374 | grid.display()
375 | state = n_state
376 | print("reward=>", reward)
377 |
378 | if __name__ == '__main__':
379 | main()
380 |
--------------------------------------------------------------------------------
/genetic/zombies/src/index.ts:
--------------------------------------------------------------------------------
1 |
2 | function mod(x: number, n: number): number {
3 | return (x % n + n) % n;
4 | }
5 |
6 | export interface Zombie {
7 | sprite: PIXI.Sprite;
8 | vector: number[];
9 | follow: boolean;
10 | speed: number;
11 | perception: number;
12 | }
13 |
14 | export interface Human {
15 | sprite: PIXI.Sprite;
16 | vector: number[];
17 | followdist: number;
18 | speed: number;
19 | perception: number;
20 | ammunition: number;
21 | shot_accuracy: number;
22 | genome: number[];
23 | lifeduration: number;
24 | }
25 |
26 | export interface Bullet {
27 | sprite: PIXI.Sprite;
28 | }
29 |
30 | export interface WorldConf {
31 | [key:string]: any,
32 | nbzombie: number;
33 | nbhuman: number;
34 | }
35 |
36 | export interface Phenotype {
37 | speed: number,
38 | perception: number,
39 | shot_accuracy: number,
40 | ammunition: number
41 | }
42 |
43 | export interface Position {
44 | y: number;
45 | x: number;
46 | }
47 |
48 | class GeneticZombie {
49 |
50 | private canvasId: string;
51 | protected app: PIXI.Application = null; // The pixi app.
52 | protected sprite: PIXI.Sprite = null;
53 | protected zombies: Zombie[] = [];
54 | protected bullets: Bullet[] = [];
55 | protected humans: Human[] = [];
56 | protected humansToConvert: string[] = [];
57 | protected zombieToKill: string[] = [];
58 | protected sounds: Position[] = [];
59 | protected isplaying: boolean = true;
60 | protected population: Human[] = [];
61 | protected last_population: Human[] = [];
62 | protected stepnb: number = 0;
63 | protected callback: any = null;
64 |
65 | protected width: number = 1200;
66 | protected height: number = 800;
67 |
68 | protected config: WorldConf = {
69 | nbzombie: 50,
70 | nbhuman: 50,
71 | nbbullets: 10,
72 | };
73 |
74 | constructor(canvasId: string, config: WorldConf) {
75 | if (!canvasId){
76 | console.error("You must specify the canvasId");
77 | }
78 |
79 | for (let key in this.config){
80 | if (config && key in config){
81 | this.config[key] = config[key];
82 | }
83 | }
84 |
85 | this.canvasId = canvasId;//Create a Pixi Application
86 | this.app = new PIXI.Application({
87 | width: this.width,
88 | height: this.height,
89 | backgroundColor: 0x002628
90 | });
91 | //Add the canvas that Pixi automatically created for you to the HTML document
92 | document.getElementById(this.canvasId).appendChild(this.app.view);
93 |
94 | this.app.ticker.add(delta => {
95 | if (this.callback){
96 | this.callback(this, delta);
97 | }
98 | });
99 | }
100 |
101 | protected vectorrand(vector: number []){
102 | let a = Math.random();
103 | let b = 1 - a;
104 | vector[0] = vector[0] * a;
105 | vector[1] = vector[1] * b;
106 | return vector;
107 | }
108 |
109 | public run(){
110 | this.isplaying = true;
111 | }
112 |
113 | public stop(){
114 | this.isplaying = false;
115 | }
116 |
117 | /*
118 | Step in the environment
119 | */
120 | public step(delta: number){
121 | this.stepnb += 1;
122 |
123 | // Track the human that are going to be catch by zombies
124 | this.humansToConvert = [];
125 | // Step all the zombies
126 | let n_zombies: Zombie[] = [];
127 | for (let i in this.zombies) {
128 | this.stepZombie(this.zombies[i], i);
129 | if (this.zombieToKill.indexOf(i) == -1){
130 | n_zombies.push(this.zombies[i]);
131 | }
132 | }
133 | this.zombies = n_zombies;
134 | if (this.zombies.length == 0){
135 | let zombie = this.createZombie();
136 | }
137 | // Track the zombie that are going to be kill
138 | this.zombieToKill = [];
139 | // Step all the humans
140 | this.sounds = [];
141 | let n_humans: Human[] = [];
142 | for (let i in this.humans) {
143 | this.stepHuman(this.humans[i], i);
144 | if (this.humansToConvert.indexOf(i) == -1){
145 | this.humans[i].lifeduration = this.stepnb;
146 | n_humans.push(this.humans[i])
147 | }
148 | else {
149 | let zombie = this.createZombie();
150 | zombie.sprite.position.x = this.humans[i].sprite.position.x;
151 | zombie.sprite.position.y = this.humans[i].sprite.position.y;
152 | this.humans[i].sprite.visible = false;
153 | }
154 | }
155 | this.humans = n_humans;
156 | }
157 |
158 | /*
159 | Move a particular Zombie
160 | */
161 | public stepZombie(z: Zombie, hid: string){
162 | for (let s in this.sounds){
163 | let sound = this.sounds[s];
164 | let dist = Math.sqrt((z.sprite.position.x-sound.x)**2 + (z.sprite.position.y-sound.y)**2);
165 | if (dist < 200){
166 | z.vector[0] = sound.x <= z.sprite.position.x ? -1 : 1;
167 | z.vector[1] = sound.y <= z.sprite.position.y ? -1 : 1;
168 | }
169 | }
170 | for (let i in this.humans) {
171 | let h = this.humans[i];
172 |
173 | let dist = Math.sqrt((h.sprite.position.x-z.sprite.position.x)**2 + (h.sprite.position.y-z.sprite.position.y)**2);
174 |
175 | if (dist < 10){
176 | if (this.humansToConvert.indexOf(i) == -1){
177 | this.humansToConvert.push(i);
178 | this.app.stage.removeChild(h.sprite);
179 | }
180 | }
181 | // The speed of the human increase the perception of the zombie
182 | // because of the sound
183 | let factornoise = h.speed > 1.0 ? (h.speed-1.0)*50 : 0.0;
184 | if (dist < z.perception * factornoise){
185 | z.vector[0] = h.sprite.position.x <= z.sprite.position.x ? -1 : 1;
186 | z.vector[1] = h.sprite.position.y <= z.sprite.position.y ? -1 : 1;
187 | //z.vector = this.vectorrand(z.vector);
188 | }
189 | }
190 | z.sprite.position.x = mod(z.sprite.position.x + (z.speed*z.vector[0]) , this.width);
191 | z.sprite.position.y = mod(z.sprite.position.y + (z.speed*z.vector[1]), this.height);
192 | }
193 |
194 | /*
195 | Move a particular human
196 | */
197 | public stepHuman(h: Human, hid: string): void {
198 |
199 | h.followdist = 10000;
200 |
201 | let n_bullets: Bullet[] = [];
202 | let bullet_to_add = 0;
203 | for (let b in this.bullets) {
204 | let bullet = this.bullets[b];
205 | let dist = Math.sqrt((h.sprite.position.x-bullet.sprite.position.x)**2 + (h.sprite.position.y-bullet.sprite.position.y)**2);
206 | if (dist < h.perception){
207 | h.vector[0] = h.sprite.position.x <= bullet.sprite.position.x ? 1 : -1;
208 | h.vector[1] = h.sprite.position.y <= bullet.sprite.position.y ? 1 : -1;
209 | h.vector = this.vectorrand(h.vector);
210 | }
211 | // Take the bullets
212 | if (dist < 10){
213 | bullet.sprite.visible = false;
214 | this.app.stage.removeChild(bullet.sprite);
215 | h.ammunition += 5;
216 | bullet_to_add += 1;
217 | } else{
218 | n_bullets.push(bullet);
219 | }
220 | }
221 | this.bullets = n_bullets;
222 | for (let b=0; b < bullet_to_add; b++){
223 | this.createBullet();
224 | }
225 |
226 | for (let i in this.zombies) {
227 | let z = this.zombies[i];
228 |
229 | let dist = Math.sqrt((h.sprite.position.x-z.sprite.position.x)**2 + (h.sprite.position.y-z.sprite.position.y)**2);
230 |
231 | if (dist < h.perception && h.followdist+10 > dist){
232 | if (h.ammunition > 0){ // Try to shot the Zombie
233 | h.ammunition = h.ammunition - 1;
234 | this.sounds.push({
235 | y: h.sprite.position.y,
236 | x: h.sprite.position.x
237 | });
238 | if (Math.random() < h.shot_accuracy){
239 | this.zombieToKill.push(i);
240 | this.app.stage.removeChild(z.sprite);
241 | }
242 | }
243 | h.vector[0] = h.sprite.position.x <= z.sprite.position.x ? -1 : 1;
244 | h.vector[1] = h.sprite.position.y <= z.sprite.position.y ? -1 : 1;
245 | h.vector = this.vectorrand(h.vector);
246 | h.followdist = dist;
247 | }
248 | }
249 |
250 | h.sprite.position.x = mod(h.sprite.position.x + (h.speed*h.vector[0]), this.width);
251 | h.sprite.position.y = mod(h.sprite.position.y + (h.speed*h.vector[1]), this.height);
252 | }
253 |
254 | public emptyHumans(){
255 | for (let h in this.humans){
256 | this.humans[h].sprite.visible = false;
257 | this.app.stage.removeChild(this.humans[h].sprite);
258 | }
259 | this.humans = [];
260 | this.last_population = this.population;
261 | this.population = [];
262 | }
263 |
264 | public emptyZomvies(){
265 | for (let z in this.zombies){
266 | this.zombies[z].sprite.visible = false;
267 | this.app.stage.removeChild(this.zombies[z].sprite);
268 | }
269 | this.zombies = [];
270 | }
271 |
272 | public createZombies(){
273 | console.log("create zombies", this.config.nbzombie - this.zombies.length);
274 | for (let i = 0; i < this.config.nbzombie - this.zombies.length; i++) {
275 | this.createZombie();
276 | }
277 | }
278 |
279 | public reset() {
280 | this.stepnb = 0;
281 |
282 | for (let b in this.bullets){
283 | this.bullets[b].sprite.visible = false;
284 | this.app.stage.removeChild(this.bullets[b].sprite);
285 | }
286 | this.bullets = [];
287 | for (let z in this.zombies){
288 | this.zombies[z].sprite.visible = false;
289 | this.app.stage.removeChild(this.zombies[z].sprite);
290 | }
291 | this.zombies = [];
292 | for (let h in this.humans){
293 | this.humans[h].sprite.visible = false;
294 | this.app.stage.removeChild(this.humans[h].sprite);
295 | }
296 | this.humans = [];
297 | this.population = [];
298 | for (let i = 0; i < this.config.nbbullets; i++) {
299 | this.createBullet();
300 | }
301 | for (let i = 0; i < this.config.nbzombie; i++) {
302 | this.createZombie();
303 | }
304 | }
305 |
306 | public init(oninit: any, callback: any): void {
307 | PIXI.loader
308 | .add("/public/images/boy.png")
309 | .add("/public/images/girl.png")
310 | .add("/public/images/zombie.png")
311 | .add("/public/images/bullet.png")
312 | .load(() => {
313 |
314 | for (let i = 0; i < this.config.nbbullets; i++) {
315 | this.createBullet();
316 | }
317 |
318 | for (let i = 0; i < this.config.nbzombie; i++) {
319 | this.createZombie();
320 | }
321 | //for (let i = 0; i < this.config.nbhuman; i++) {
322 | // this.createHuman();
323 | //}
324 |
325 | if (oninit){
326 | oninit(this);
327 | }
328 |
329 | this.callback = callback;
330 |
331 | });
332 | }
333 |
334 | /**
335 | * Create a new Zombie
336 | */
337 | public createZombie(): Zombie {
338 | let sprite = new PIXI.Sprite(PIXI.loader.resources["/public/images/zombie.png"].texture);
339 | sprite.scale.x = 0.05;
340 | sprite.scale.y = 0.05;
341 | // Position of the Zombie
342 | sprite.position.x = Math.floor(Math.random() * this.width);
343 | sprite.position.y = Math.floor(Math.random() * this.height);
344 | this.app.stage.addChild(sprite);
345 |
346 | let vector: number[] = [
347 | Math.random() > 0.75 ? -1 : 1,
348 | Math.random() > 0.25 ? -1 : 1
349 | ]
350 | vector = this.vectorrand(vector);
351 |
352 | let zombie: Zombie = {
353 | sprite: sprite,
354 | vector: vector,
355 | follow: false,
356 | speed: 0.8,
357 | perception: 30.0
358 | }
359 | this.zombies.push(zombie);
360 | return zombie;
361 | }
362 |
363 |
364 | /**
365 | * Create a Bullet
366 | */
367 | public createBullet(): Bullet {
368 | let sprite = new PIXI.Sprite(PIXI.loader.resources["/public/images/bullet.png"].texture);
369 | sprite.scale.x = 0.03;
370 | sprite.scale.y = 0.03;
371 | // Position of the Zombie
372 | sprite.position.x = Math.floor(Math.random() * this.width);
373 | sprite.position.y = Math.floor(Math.random() * this.height);
374 | this.app.stage.addChild(sprite);
375 |
376 | let bullet: Bullet = {
377 | sprite: sprite
378 | }
379 |
380 | this.bullets.push(bullet);
381 | return bullet;
382 | }
383 |
384 | /*
385 | Genome to phenotype
386 | */
387 | protected genomeToPhenotype(genome: number[]): Phenotype {
388 | let phenotype: Phenotype = {
389 | speed: 1.0 + (0.4*genome[0]-0.2),
390 | perception: 50 + (300*genome[1]-150),
391 | shot_accuracy: 0.3 + (0.6*genome[2]-0.3),
392 | ammunition: 4 + (20*genome[3]-10),
393 | };
394 | return phenotype;
395 | }
396 |
397 | /**
398 | * Create a new Human
399 | */
400 | public createHuman(genome: number[]): void {
401 | let phenotype = this.genomeToPhenotype(genome);
402 | let sprite = new PIXI.Sprite(PIXI.loader.resources[
403 | Math.random() > 0.5 ? "/public/images/boy.png" : "/public/images/girl.png"
404 | ].texture);
405 | sprite.scale.x = 0.05;
406 | sprite.scale.y = 0.05;
407 | // Position of the Zombie
408 | sprite.position.x = Math.floor(Math.random() * this.width);
409 | sprite.position.y = Math.floor(Math.random() * this.height);
410 | this.app.stage.addChild(sprite);
411 |
412 | let vector: number[] = [
413 | Math.random() > 0.5 ? -1 : 1,
414 | Math.random() > 0.5 ? -1 : 1
415 | ]
416 | vector = this.vectorrand(vector);
417 |
418 | let n_human = {
419 | sprite: sprite,
420 | vector: vector,
421 | followdist: 10000,
422 | speed: phenotype.speed,
423 | perception: phenotype.perception,
424 | ammunition: phenotype.ammunition,
425 | shot_accuracy: phenotype.shot_accuracy,
426 | genome: genome,
427 | lifeduration: 0
428 | };
429 | this.humans.push(n_human);
430 | this.population.push(n_human);
431 | }
432 |
433 | }
434 |
435 | const geneticzombie = {
436 | env: GeneticZombie
437 | }
438 |
439 | export default geneticzombie;
440 |
--------------------------------------------------------------------------------
/rl/actor_critic.py:
--------------------------------------------------------------------------------
1 | from random import randint
2 | import tensorflow as tf
3 | import numpy as np
4 | import scipy.signal
5 |
6 | """
7 | Exemple of the Policy Gradient Algorithm
8 | """
9 |
10 | class Buffer:
11 |
12 | def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):
13 | self.obs_buf = np.zeros(Buffer.combined_shape(size, obs_dim), dtype=np.float32)
14 | # Actions buffer
15 | self.act_buf = np.zeros(size, dtype=np.float32)
16 | # Advantages buffer
17 | self.adv_buf = np.zeros(size, dtype=np.float32)
18 | # Value function buffer
19 | self.val_buf = np.zeros(size, dtype=np.float32)
20 | # Rewards buffer
21 | self.rew_buf = np.zeros(size, dtype=np.float32)
22 | # Log probability of action a with the policy
23 | self.logp_buf = np.zeros(size, dtype=np.float32)
24 | # Gamma and lam to compute the advantage
25 | self.gamma, self.lam = gamma, lam
26 | # Rreturn buffer (used to train the value fucntion)
27 | self.ret_buf = np.zeros(size, dtype=np.float32)
28 | # ptr: Position to insert the next tuple
29 | # path_start_idx Posittion of the current trajectory
30 | # max_size Max size of the buffer
31 | self.ptr, self.path_start_idx, self.max_size = 0, 0, size
32 |
33 | @staticmethod
34 | def discount_cumsum(x, discount):
35 | """
36 | x = [x0, x1, x2]
37 | output: [x0 + discount * x1 + discount^2 * x2, x1 + discount * x2, x2]
38 | """
39 | return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]
40 |
41 | @staticmethod
42 | def combined_shape(length, shape=None):
43 | if shape is None:
44 | return (length,)
45 | return (length, shape) if np.isscalar(shape) else (length, *shape)
46 |
47 | def store(self, obs, act, rew, logp, val):
48 | # Append one timestep of agent-environment interaction to the buffer.
49 | assert self.ptr < self.max_size
50 | self.obs_buf[self.ptr] = obs
51 | self.act_buf[self.ptr] = act
52 | self.rew_buf[self.ptr] = rew
53 | self.logp_buf[self.ptr] = logp
54 | self.val_buf[self.ptr] = val
55 | self.ptr += 1
56 |
57 | def finish_path(self, last_val=0):
58 | # Select the path
59 | path_slice = slice(self.path_start_idx, self.ptr)
60 | # Append the last_val to the trajectory
61 | rews = np.append(self.rew_buf[path_slice], last_val)
62 | # Append the last value to the value function
63 | vals = np.append(self.val_buf[path_slice], last_val)
64 | # Deltas = r + y*v(s') - v(s)
65 | deltas = rews[:-1] + self.gamma * vals[1:] - vals[:-1]
66 | self.adv_buf[path_slice] = Buffer.discount_cumsum(deltas, self.gamma)
67 | # Advantage
68 | self.ret_buf[path_slice] = Buffer.discount_cumsum(rews, self.gamma)[:-1]
69 | self.path_start_idx = self.ptr
70 |
71 | def get(self):
72 | assert self.ptr == self.max_size # buffer has to be full before you can get
73 | self.ptr, self.path_start_idx = 0, 0
74 | # the next two lines implement the advantage normalization trick
75 | # Normalize the Advantage
76 | if np.std(self.adv_buf) != 0:
77 | self.adv_buf = (self.adv_buf - np.mean(self.adv_buf)) / np.std(self.adv_buf)
78 | return self.obs_buf, self.act_buf, self.adv_buf, self.logp_buf, self.ret_buf
79 |
80 |
81 | class ActorCritic(object):
82 | """
83 | Implementation of Policy gradient algorithm
84 | """
85 | def __init__(self, input_space, action_space, pi_lr, v_lr, buffer_size, seed):
86 | super(ActorCritic, self).__init__()
87 |
88 | # Stored the spaces
89 | self.input_space = input_space
90 | self.action_space = action_space
91 | self.seed = seed
92 | # NET Buffer defined above
93 | self.buffer = Buffer(
94 | obs_dim=input_space,
95 | act_dim=action_space,
96 | size=buffer_size
97 | )
98 | # Learning rate of the policy network and the value network
99 | self.pi_lr = pi_lr
100 | self.v_lr = v_lr
101 | # The tensorflow session (set later)
102 | self.sess = None
103 | # Apply a random seed on tensorflow and numpy
104 | tf.set_random_seed(42)
105 | np.random.seed(42)
106 |
107 | def compile(self):
108 | """
109 | Compile the model
110 | """
111 | # tf_a : Input: Chosen action
112 | # tf_map: Input: Input state
113 | # tf_tv: Input: Target value function
114 | # tf_adv: Input: Advantage
115 | self.tf_map, self.tf_a, self.tf_adv, self.tf_tv = ActorCritic.inputs(
116 | map_space=self.input_space,
117 | action_space=self.action_space
118 | )
119 | # mu_op: Used to get the exploited prediction of the model
120 | # pi_op: Used to get the prediction of the model
121 | # logp_a_op: Used to get the log likelihood of taking action a with the current policy
122 | # logp_pi_op: Used to get the log likelihood of the predicted action @pi_op
123 | # v_op: Used to get the value function of the given state
124 | self.mu_op, self.pi_op, self.logp_a_op, self.logp_pi_op, self.v_op = ActorCritic.mlp(
125 | tf_map=self.tf_map,
126 | tf_a=self.tf_a,
127 | action_space=self.action_space,
128 | seed=self.seed
129 | )
130 | # Error
131 | self.pi_loss, self.v_loss = ActorCritic.net_objectives(
132 | tf_adv=self.tf_adv,
133 | logp_a_op=self.logp_a_op,
134 | v_op=self.v_op,
135 | tf_tv=self.tf_tv
136 | )
137 | # Optimization
138 | self.train_pi = tf.train.AdamOptimizer(learning_rate=self.pi_lr).minimize(self.pi_loss)
139 | self.train_v = tf.train.AdamOptimizer(learning_rate=self.v_lr).minimize(self.v_loss)
140 | # Entropy
141 | self.approx_ent = tf.reduce_mean(-self.logp_a_op)
142 |
143 |
144 | def set_sess(self, sess):
145 | # Set the tensorflow used to run this model
146 | self.sess = sess
147 |
148 | def step(self, states):
149 | # Take actions given the states
150 | # Return mu (policy without exploration), pi (policy with the current exploration) and
151 | # the log probability of the action chossen by pi
152 | mu, pi, logp_pi, v = self.sess.run([self.mu_op, self.pi_op, self.logp_pi_op, self.v_op], feed_dict={
153 | self.tf_map: states
154 | })
155 | return mu, pi, logp_pi, v
156 |
157 | def store(self, obs, act, rew, logp, val):
158 | # Store the observation, action, reward, log probability of the action and state value
159 | # into the buffer
160 | self.buffer.store(obs, act, rew, logp, val)
161 |
162 | def finish_path(self, last_val=0):
163 | self.buffer.finish_path(last_val=last_val)
164 |
165 | def train(self, additional_infos={}):
166 | # Get buffer
167 | obs_buf, act_buf, adv_buf, logp_last_buf, ret_buf = self.buffer.get()
168 | # Train the model
169 | pi_loss_list = []
170 | entropy_list = []
171 | v_loss_list = []
172 |
173 | for step in range(5):
174 | _, entropy, pi_loss = self.sess.run([self.train_pi, self.approx_ent, self.pi_loss], feed_dict= {
175 | self.tf_map: obs_buf,
176 | self.tf_a:act_buf,
177 | self.tf_adv: adv_buf
178 | })
179 | entropy_list.append(entropy)
180 | pi_loss_list.append(pi_loss)
181 |
182 |
183 | for step in range(5):
184 | _, v_loss = self.sess.run([self.train_v, self.v_loss], feed_dict= {
185 | self.tf_map: obs_buf,
186 | self.tf_tv: ret_buf,
187 | })
188 |
189 | v_loss_list.append(v_loss)
190 |
191 | print("Entropy : %s" % (np.mean(entropy_list)), end="\r")
192 |
193 |
194 | @staticmethod
195 | def gaussian_likelihood(x, mu, log_std):
196 | # Compute the gaussian likelihood of x with a normal gaussian distribution of mean @mu
197 | # and a std @log_std
198 | pre_sum = -0.5 * (((x-mu)/(tf.exp(log_std)+1e-8))**2 + 2*log_std + np.log(2*np.pi))
199 | return tf.reduce_sum(pre_sum, axis=1)
200 |
201 | @staticmethod
202 | def inputs(map_space, action_space):
203 | """
204 | @map_space Tuple of the space. Ex (size,)
205 | @action_space Tuple describing the action space. Ex (size,)
206 | """
207 | # Map of the game
208 | tf_map = tf.placeholder(tf.float32, shape=(None, *map_space), name="tf_map")
209 | # Possible actions (Should be two: x,y for the beacon game)
210 | tf_a = tf.placeholder(tf.int32, shape=(None,), name="tf_a")
211 | # Advantage
212 | tf_adv = tf.placeholder(tf.float32, shape=(None,), name="tf_adv")
213 | # Target value
214 | tf_tv = tf.placeholder(tf.float32, shape=(None,), name="tf_tv")
215 | return tf_map, tf_a, tf_adv, tf_tv
216 |
217 | @staticmethod
218 | def mlp(tf_map, tf_a, action_space, seed=None):
219 | if seed is not None:
220 | tf.random.set_random_seed(seed)
221 |
222 | # Expand the dimension of the input
223 | tf_map_expand = tf.expand_dims(tf_map, axis=3)
224 |
225 | flatten = tf.layers.flatten(tf_map_expand)
226 | hidden = tf.layers.dense(flatten, units=256, activation=tf.nn.relu)
227 |
228 | # Logits policy
229 | spacial_action_logits = tf.layers.dense(hidden, units=action_space, activation=None)
230 | # Logits value function
231 | v_op = tf.layers.dense(hidden, units=1, activation=None)
232 |
233 | # Add take the log of the softmax
234 | logp_all = tf.nn.log_softmax(spacial_action_logits)
235 | # Take random actions according to the logits (Exploration)
236 | pi_op = tf.squeeze(tf.multinomial(spacial_action_logits,1), axis=1)
237 | mu = tf.argmax(spacial_action_logits, axis=1)
238 |
239 | # Gives log probability, according to the policy, of taking actions @a in states @x
240 | logp_a_op = tf.reduce_sum(tf.one_hot(tf_a, depth=action_space) * logp_all, axis=1)
241 | # Gives log probability, according to the policy, of the action sampled by pi.
242 | logp_pi_op = tf.reduce_sum(tf.one_hot(pi_op, depth=action_space) * logp_all, axis=1)
243 |
244 | return mu, pi_op, logp_a_op, logp_pi_op, v_op
245 |
246 | @staticmethod
247 | def net_objectives(logp_a_op, tf_adv, v_op, tf_tv, clip_ratio=0.2):
248 | """
249 | @v_op: Predicted value function
250 | @tf_tv: Expected value function
251 | @logp_a_op: Log likelihood of taking action under the current policy
252 | @tf_logp_old_pi: Log likelihood of the last policy
253 | @tf_adv: Advantage input
254 | """
255 | pi_loss = -tf.reduce_mean(logp_a_op*tf_adv)
256 | v_loss = tf.reduce_mean((tf_tv - v_op)**2)
257 | return pi_loss, v_loss
258 |
259 | class GridWorld(object):
260 | """
261 | docstring for GridWorld.
262 | """
263 | def __init__(self):
264 | super(GridWorld, self).__init__()
265 |
266 | self.rewards = [
267 | [0, 0, 0, 0, -1, 0, 0],
268 | [0, -1, -1, 0, -1, 0, 0],
269 | [0, -1, -1, 1, -1, 0, 0],
270 | [0, -1, -1, 0, -1, 0, 0],
271 | [0, 0, 0, 0, 0, 0, 0],
272 | [0, 0, 0, 0, 0, 0, 0],
273 | [0, 0, 0, 0, 0, 0, 0],
274 | ]
275 | self.position = [6, 6] # y, x
276 |
277 | def gen_state(self):
278 | # Generate a state given the current position of the agent
279 | state = np.zeros((7, 7))
280 | state[self.position[0]][self.position[1]] = 1
281 | return state
282 |
283 | def step(self, action):
284 | if action == 0: # Top
285 | self.position = [(self.position[0] - 1) % 7, self.position[1]]
286 | elif action == 1: # Left
287 | self.position = [self.position[0], (self.position[1] - 1) % 7]
288 | elif action == 2: # Right
289 | self.position = [self.position[0], (self.position[1] + 1) % 7]
290 | elif action == 3: # Down
291 | self.position = [(self.position[0] + 1) % 7, self.position[1]]
292 |
293 | reward = self.rewards[self.position[0]][self.position[1]]
294 | done = False if reward == 0 else True
295 | state = self.gen_state()
296 | if done: # The agent is dead, reset the game
297 | self.position = [6, 6]
298 | return state, reward, done
299 |
300 | def display(self):
301 | y = 0
302 | print("="*14)
303 | for line in self.rewards:
304 | x = 0
305 | for case in line:
306 | if case == -1:
307 | c = "0"
308 | elif (y == self.position[0] and x == self.position[1]):
309 | c = "A"
310 | elif case == 1:
311 | c = "T"
312 | else:
313 | c = "-"
314 | print(c, end=" ")
315 | x += 1
316 | y += 1
317 | print()
318 |
319 | def main():
320 | grid = GridWorld()
321 | buffer_size = 1000
322 |
323 | # Create the NET class
324 | agent = ActorCritic(
325 | input_space=(7, 7),
326 | action_space=4,
327 | pi_lr=0.001,
328 | v_lr=0.001,
329 | buffer_size=buffer_size,
330 | seed=42
331 | )
332 | agent.compile()
333 | # Init Session
334 | sess = tf.Session()
335 | # Init variables
336 | sess.run(tf.global_variables_initializer())
337 | # Set the session
338 | agent.set_sess(sess)
339 |
340 | rewards = []
341 |
342 | b = 0
343 |
344 | for epoch in range(10000):
345 |
346 | done = False
347 | state = grid.gen_state()
348 |
349 | while not done:
350 | _, pi, logpi, v = agent.step([state])
351 | n_state, reward, done = grid.step(pi[0])
352 | agent.store(state, pi[0], reward, logpi, v)
353 | b += 1
354 |
355 | state = n_state
356 |
357 | if done:
358 | agent.finish_path(reward)
359 | rewards.append(reward)
360 | if len(rewards) > 1000:
361 | rewards.pop(0)
362 | if b == buffer_size:
363 | if not done:
364 | # Bootstrap the last value
365 | agent.finish_path(v)
366 | agent.train()
367 | b = 0
368 |
369 | if epoch % 1000 == 0:
370 | print("Rewards mean:%s" % np.mean(rewards))
371 |
372 | for epoch in range(10):
373 | import time
374 | print("=========================TEST=================================")
375 | done = False
376 | state = grid.gen_state()
377 |
378 | while not done:
379 | time.sleep(1)
380 | _, pi, logpi, v = agent.step([state])
381 | n_state, _, done = grid.step(pi[0])
382 | print("v", v)
383 | grid.display()
384 | state = n_state
385 | print("reward=>", reward)
386 |
387 | if __name__ == '__main__':
388 | main()
389 |
--------------------------------------------------------------------------------
/meta-learning/Meta Learning.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "## Classification Demo"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": 1,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "import numpy as np\n",
17 | "import matplotlib.pyplot as plt\n",
18 | "from scipy.spatial.distance import cdist\n",
19 | "import copy\n",
20 | "import sys\n",
21 | "import os\n",
22 | "\n",
23 | "DATASET = \"/home/thibault/work/datasets/omniglot/python/\""
24 | ]
25 | },
26 | {
27 | "cell_type": "markdown",
28 | "metadata": {},
29 | "source": [
30 | "## One-shot classification demo with Modified Hausdorff Distance"
31 | ]
32 | },
33 | {
34 | "cell_type": "markdown",
35 | "metadata": {},
36 | "source": [
37 | "Running this demo should lead to a result of 38.8 percent errors. M.-P. Dubuisson, A. K. Jain (1994). A modified hausdorff distance for object matching. International Conference on Pattern Recognition, pp. 566-568. ** Models should be trained on images in 'images_background' directory to avoid using images and alphabets used in the one-shot evaluation **"
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "execution_count": 2,
43 | "metadata": {},
44 | "outputs": [],
45 | "source": [
46 | "def mod_hausdorff_distance(itemA, itemB):\n",
47 | " \"\"\"\n",
48 | " Modified Hausdorff Distance\n",
49 | "\n",
50 | " Input:\n",
51 | " itemA : [n x 2] coordinates of \"inked\" pixels\n",
52 | " itemB : [m x 2] coordinates of \"inked\" pixels\n",
53 | " M.-P. Dubuisson, A. K. Jain (1994). A modified hausdorff distance for object matching.\n",
54 | " International Conference on Pattern Recognition, pp. 566-568.\n",
55 | " \"\"\"\n",
56 | " D = cdist(itemA,itemB)\n",
57 | " mindist_A = D.min(axis=1)\n",
58 | " mindist_B = D.min(axis=0)\n",
59 | " mean_A = np.mean(mindist_A)\n",
60 | " mean_B = np.mean(mindist_B)\n",
61 | " return max(mean_A,mean_B)"
62 | ]
63 | },
64 | {
65 | "cell_type": "code",
66 | "execution_count": 23,
67 | "metadata": {},
68 | "outputs": [
69 | {
70 | "data": {
71 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD8CAYAAAB+fLH0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADnhJREFUeJzt3V2oZXd5x/HvrzNGa6Tm7RDiTOxMMShBsJGDjaQUMUrTVEwuRBLEDjIwN7bGF9CkvZDeKYiagkgHo06LRG0MTQiipGOk9KJTz2gwL2PMNDZmhsQcqdFiLzT49GKvwfM/OafnzF77Ze053w8czl5rr733M/+Z/Naz/mvtlVQVknTG78y7AEnDYihIahgKkhqGgqSGoSCpYShIahgKkhpTCYUk1yV5LMnJJLdO4zMkTUcmffFSkl3AD4G3AqeA7wA3V9WjE/0gSVOxewrv+QbgZFU9AZDky8ANwKahcMkll9S+ffumUIqkM44fP/7TqlraartphMIe4Kk1y6eAP1q/UZJDwCGAV77ylaysrEyhFElnJHlyO9vNbaKxqg5X1XJVLS8tbRlekmZkGqFwGrh8zfLebp2kBTCNUPgOcEWS/UnOA24C7p3C50iagonPKVTV80n+EvgmsAv4fFU9MunPkTQd05hopKq+Dnx9Gu8tabq8olFSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUmNsUMhyeVJHkjyaJJHktzSrb8oyf1JHu9+Xzi5ciVNW59O4XngQ1V1JXA18N4kVwK3Aker6grgaLcsaUGMHQpV9XRVfbd7/D/ACWAPcANwpNvsCHBj3yIlzc5E5hSS7AOuAo4Bl1bV091TzwCXTuIzJM1G71BI8jLga8D7q+oXa5+rqgJqk9cdSrKSZGV1dbVvGZImpFcoJHkRo0D4UlXd3a3+SZLLuucvA57d6LVVdbiqlqtqeWlpqU8Zkiaoz9mHAHcAJ6rqk2ueuhc40D0+ANwzfnmSZm13j9deA7wbeCjJg926vwY+Bnw1yUHgSeCd/UqUNEtjh0JV/RuQTZ6+dtz3lTRfXtEoqWEoSGoYCpIahoKkhqEgqWEoSGoYCpIahoKkhqEgqWEoSGoYCpIahoKkRp9vSUoAjL5FPx2j+/RoluwUJDXsFPQC09zzn631tdg5TJ+dgqSGncION6SuYDvsHKbPTkFSw05hh1i0jmC7zvy57Bgmx05BUsNQkNQwFHROSHLOHiLNmqEgqeFEoyZuEpN+4+71nXjsz05BUsNOQVuax153/Weebeewdnu7hrNjpyCpYaegFxjinvVMTZ5hmD47BUkNO4UdYoh7/3GM0zF4RuLs2ClIahgKWkhV5Z5/SgwFSY3ecwpJdgErwOmqeluS/cCXgYuB48C7q+pXfT9nJ1iUmfVF3UM7t7A9k+gUbgFOrFn+OPCpqnoV8DPg4AQ+Q9KM9AqFJHuBPwc+1y0HeDNwV7fJEeDGPp9xLjjzDb6tfhbFotZ9xqLWPSt9O4VPAx8GftMtXww8V1XPd8ungD0bvTDJoSQrSVZWV1d7liFpUsYOhSRvA56tquPjvL6qDlfVclUtLy0tjVvGIJwrHcC45vnn7HMWYqf8/ZytPhON1wBvT3I98BLg94DbgQuS7O66hb3A6f5lSpqVsTuFqrqtqvZW1T7gJuBbVfUu4AHgHd1mB4B7elc5MDutE9iueY6LHcPkTOM6hY8AH0xyktEcwx1T+AxJUzKR7z5U1beBb3ePnwDeMIn3nQf3GJMzj/9xi9+m7M8rGiU1/JZkZ2h7liFeddd3jGZ5N6S177/dur3iccROQVLDTmGGFn0PNMnjdffKw2WnIKlhKEhqePgwRedqa9z39usbvXaaY3W2hz07/dDGTkFSw06hh526J1lvEhOQO33vPCR2CpIadgpjcG+2sUnMNUyzY/AS6O2xU5DUMBQ0NYt+G/ad+pVqQ0FSw1AYwyLsQYZU4zgdw5Dq32kMBUkNzz70MIRz61vtTTd7fh41j/N15mnW4RWOG7NTkNSwU+gs2jnsRamzr522lx4COwVJDUNhHWfKZ2MI1zAMoYYhMhQkNQwFaZt2SkdoKEhqGAqbcG5BO5WhIKnhdQoL6myvq3CWfXOLdo3KtNkpSGrYKWxhKNfrb8YOQJNmpyCpYShMgWchtMgMBUmNXqGQ5IIkdyX5QZITSd6Y5KIk9yd5vPt94aSKnTevlddO0LdTuB34RlW9BngdcAK4FThaVVcAR7tlSQti7FBI8nLgT4A7AKrqV1X1HHADcKTb7AhwY98iJc1On05hP7AKfCHJ95J8Lsn5wKVV9XS3zTPApX2LXFROOGoR9QmF3cDrgc9W1VXAL1l3qFCjA/AND8KTHEqykmRldXW1RxmSJqlPKJwCTlXVsW75LkYh8ZMklwF0v5/d6MVVdbiqlqtqeWlpqUcZkiZp7FCoqmeAp5K8ult1LfAocC9woFt3ALinV4WSZqrvZc5/BXwpyXnAE8B7GAXNV5McBJ4E3tnzMwbHW4SPx/mVxdArFKrqQWB5g6eu7fO+kubHL0TN0E7tGOwQFouXOUtq2Cn0MO7NOXZKxzCJDuFcH6MhslOQ1LBTmAA7hslzTObHTkFSw05Bg2KHMH92CpIadgoT5K3CW9sZDzuD4bFTkNSwU5iCcb8bsdF7nAvOpT/LTmCnIKlhKExRnxu9etcmzYuhIKnhnMLAbdYteJw+OXZkLTsFSQ07hRmYxvULZ/tedhbaLjsFSQ07hRlav7ee5bGscxO/Ne6475SxslOQ1DAUJDU8fJijIXyB6lz5spKnFSfHTkFSw05hAIbQMWxkaPXMyyJ1TJNgpyCpYacwIP/fHsm9tmbFTkFSw05hQZztca2dxfh22hzCenYKkhp2CueorfZ2dhK/tdM7g/XsFCQ17BR2qHP9TId7//H16hSSfCDJI0keTnJnkpck2Z/kWJKTSb6S5LxJFStp+sYOhSR7gPcBy1X1WmAXcBPwceBTVfUq4GfAwUkUqtk5c8PZRf7R+PrOKewGfjfJbuClwNPAm4G7uuePADf2/AxJMzR2KFTVaeATwI8ZhcHPgePAc1X1fLfZKWBP3yIlzU6fw4cLgRuA/cArgPOB687i9YeSrCRZWV1dHbcMSRPW5/DhLcCPqmq1qn4N3A1cA1zQHU4A7AVOb/TiqjpcVctVtby0tNSjDEmT1CcUfgxcneSlGZ3DuhZ4FHgAeEe3zQHgnn4lSpqlPnMKxxhNKH4XeKh7r8PAR4APJjkJXAzcMYE6Jc1Ir4uXquqjwEfXrX4CeEOf95U0P17mLKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKmxZSgk+XySZ5M8vGbdRUnuT/J49/vCbn2S/F2Sk0m+n+T10yxe0uRtp1P4InDdunW3Aker6grgaLcM8GfAFd3PIeCzkylT0qxsGQpV9a/Af69bfQNwpHt8BLhxzfp/qJF/By5IctmkipU0fePOKVxaVU93j58BLu0e7wGeWrPdqW7dCyQ5lGQlycrq6uqYZUiatN4TjVVVQI3xusNVtVxVy0tLS33LkDQh44bCT84cFnS/n+3WnwYuX7Pd3m6dpAUxbijcCxzoHh8A7lmz/i+6sxBXAz9fc5ghaQHs3mqDJHcCbwIuSXIK+CjwMeCrSQ4CTwLv7Db/OnA9cBL4X+A9U6hZ0hRtGQpVdfMmT127wbYFvLdvUZLmxysaJTUMBUkNQ0FSw1CQ1MhobnDORSSrwC+Bn867lm24hOHXaY2Tswh1brfG36+qLa8UHEQoACRZqarledexlUWo0xonZxHqnHSNHj5IahgKkhpDCoXD8y5gmxahTmucnEWoc6I1DmZOQdIwDKlTkDQAgwiFJNcleay7t+OtW79i+pJcnuSBJI8meSTJLd36De9POedadyX5XpL7uuX9SY514/mVJOcNoMYLktyV5AdJTiR549DGMskHur/rh5PcmeQlQxjLWd8nde6hkGQX8BlG93e8Erg5yZXzrQqA54EPVdWVwNXAe7u6Nrs/5TzdApxYs/xx4FNV9SrgZ8DBuVTVuh34RlW9Bngdo3oHM5ZJ9gDvA5ar6rXALuAmhjGWX2SW90mtqrn+AG8Evrlm+TbgtnnXtUGd9wBvBR4DLuvWXQY8Nue69nb/KN4M3AeE0YUsuzca3znV+HLgR3RzWGvWD2Ys+e2tBC9i9O3h+4A/HcpYAvuAh7caO+DvgZs32m67P3PvFDiL+zrOS5J9wFXAMTa/P+W8fBr4MPCbbvli4Lmqer5bHsJ47gdWgS90hzmfS3I+AxrLqjoNfAL4MfA08HPgOMMbyzN63yd1M0MIhUFL8jLga8D7q+oXa5+rURTP7fRNkrcBz1bV8XnVsE27gdcDn62qqxhd0t4cKgxgLC9kdDfy/cArgPN5Ycs+SJMeuyGEwmDv65jkRYwC4UtVdXe3erP7U87DNcDbk/wX8GVGhxC3M7q1/pkb6AxhPE8Bp6rqWLd8F6OQGNJYvgX4UVWtVtWvgbsZje/QxvKMqd0ndQih8B3gim6W9zxGkzv3zrkmkgS4AzhRVZ9c89Rm96ecuaq6rar2VtU+RuP2rap6F/AA8I5us7nWCFBVzwBPJXl1t+pa4FEGNJaMDhuuTvLS7u/+TI2DGss1pnef1HlN7KybRLke+CHwn8DfzLuerqY/ZtSSfR94sPu5ntEx+1HgceBfgIvmXWtX75uA+7rHfwD8B6N7Zf4T8OIB1PeHwEo3nv8MXDi0sQT+FvgB8DDwj8CLhzCWwJ2M5jl+zajrOrjZ2DGaaP5M99/SQ4zOppzV53lFo6TGEA4fJA2IoSCpYShIahgKkhqGgqSGoSCpYShIahgKkhr/ByMg6OKiVzxNAAAAAElFTkSuQmCC\n",
72 | "text/plain": [
73 | "
19 | 