├── .gitattributes
├── .idea
├── ReinforceLearning.iml
├── inspectionProfiles
│ └── profiles_settings.xml
├── misc.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── .ipynb_checkpoints
└── QTable-checkpoint.ipynb
├── Actor_Critic
├── AC_Pole.py
└── AC_continue_Pendulum.py
├── DDPG
└── DDPG.py
├── DQLearing
├── Brains
│ ├── DQLearning.py
│ ├── DQN.py
│ ├── DQNTF2.py
│ ├── DuelingDQN.py
│ ├── Priority_DQN.py
│ ├── RL_brain.py
│ └── __pycache__
│ │ ├── DQN.cpython-36.pyc
│ │ ├── DQN.cpython-37.pyc
│ │ ├── DQNTF2.cpython-36.pyc
│ │ ├── DQNTF2.cpython-37.pyc
│ │ └── DuelingDQN.cpython-36.pyc
├── gym
│ ├── CartPole.py
│ ├── Mountain_Car.py
│ ├── Pendulum.py
│ ├── checkpoint
│ ├── logs
│ │ └── events.out.tfevents.1579495704.DESKTOP-JBV63R4
│ ├── tmp_model.data-00000-of-00001
│ └── tmp_model.index
├── logs
│ ├── events.out.tfevents.1578211690.DESKTOP-JBV63R4
│ ├── events.out.tfevents.1578225153.DESKTOP-JBV63R4
│ ├── events.out.tfevents.1578226418.DESKTOP-JBV63R4
│ ├── events.out.tfevents.1578226429.DESKTOP-JBV63R4
│ └── events.out.tfevents.1578228234.DESKTOP-JBV63R4
├── run_dqn.py
└── tkinter
│ ├── chessboard.py
│ └── maze_env.py
├── PolicyGradient
├── Brain
│ ├── PG.py
│ └── PolicyGradient.py
└── gym
│ └── PGCartPole.py
└── QTable
├── 五子棋
├── chessboard.py
├── q_table1.csv
├── q_table2.csv
└── train.py
└── 走迷宫
├── RL_brain.py
├── RunQTable.py
├── RunSarsaTable.py
├── SarsaLambda.py
├── SarsaLearning.py
└── maze_env.py
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
--------------------------------------------------------------------------------
/.idea/ReinforceLearning.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
169 |
170 |
171 |
172 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 | 1577806526676
190 |
191 |
192 | 1577806526676
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
--------------------------------------------------------------------------------
/.ipynb_checkpoints/QTable-checkpoint.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "import tkinter"
10 | ]
11 | }
12 | ],
13 | "metadata": {
14 | "kernelspec": {
15 | "display_name": "Python 3",
16 | "language": "python",
17 | "name": "python3"
18 | },
19 | "language_info": {
20 | "codemirror_mode": {
21 | "name": "ipython",
22 | "version": 3
23 | },
24 | "file_extension": ".py",
25 | "mimetype": "text/x-python",
26 | "name": "python",
27 | "nbconvert_exporter": "python",
28 | "pygments_lexer": "ipython3",
29 | "version": "3.7.1"
30 | }
31 | },
32 | "nbformat": 4,
33 | "nbformat_minor": 2
34 | }
35 |
--------------------------------------------------------------------------------
/Actor_Critic/AC_Pole.py:
--------------------------------------------------------------------------------
1 | """
2 | Actor-Critic using TD-error as the Advantage, Reinforcement Learning.
3 | The cart pole example. Policy is oscillated.
4 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
5 | Using:
6 | tensorflow 1.0
7 | gym 0.8.0
8 | """
9 |
10 | import numpy as np
11 | import tensorflow as tf
12 | import gym
13 |
14 | np.random.seed(2)
15 | tf.set_random_seed(2) # reproducible
16 |
17 | # Superparameters
18 | OUTPUT_GRAPH = False
19 | MAX_EPISODE = 3000
20 | DISPLAY_REWARD_THRESHOLD = 200 # renders environment if total episode reward is greater then this threshold
21 | MAX_EP_STEPS = 1000 # maximum time step in one episode
22 | RENDER = False # rendering wastes time
23 | GAMMA = 0.9 # reward discount in TD error
24 | LR_A = 0.001 # learning rate for actor
25 | LR_C = 0.01 # learning rate for critic
26 |
27 | env = gym.make('CartPole-v0')
28 | env.seed(1) # reproducible
29 | env = env.unwrapped
30 |
31 | N_F = env.observation_space.shape[0]
32 | N_A = env.action_space.n
33 |
34 |
35 | class Actor(object):
36 | def __init__(self, sess, n_features, n_actions, lr=0.001):
37 | self.sess = sess
38 |
39 | self.s = tf.placeholder(tf.float32, [1, n_features], "state")
40 | self.a = tf.placeholder(tf.int32, None, "act")
41 | self.td_error = tf.placeholder(tf.float32, None, "td_error") # TD_error
42 |
43 | with tf.variable_scope('Actor'):
44 | l1 = tf.layers.dense(
45 | inputs=self.s,
46 | units=20, # number of hidden units
47 | activation=tf.nn.relu,
48 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
49 | bias_initializer=tf.constant_initializer(0.1), # biases
50 | name='l1'
51 | )
52 |
53 | self.acts_prob = tf.layers.dense(
54 | inputs=l1,
55 | units=n_actions, # output units
56 | activation=tf.nn.softmax, # get action probabilities
57 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
58 | bias_initializer=tf.constant_initializer(0.1), # biases
59 | name='acts_prob'
60 | )
61 |
62 | with tf.variable_scope('exp_v'):
63 | log_prob = tf.log(self.acts_prob[0, self.a])
64 | self.exp_v = tf.reduce_mean(log_prob * self.td_error) # advantage (TD_error) guided loss
65 |
66 | with tf.variable_scope('train'):
67 | self.train_op = tf.train.AdamOptimizer(lr).minimize(-self.exp_v) # minimize(-exp_v) = maximize(exp_v)
68 |
69 | def learn(self, s, a, td):
70 | s = s[np.newaxis, :]
71 | feed_dict = {self.s: s, self.a: a, self.td_error: td}
72 | _, exp_v = self.sess.run([self.train_op, self.exp_v], feed_dict)
73 | return exp_v
74 |
75 | def choose_action(self, s):
76 | s = s[np.newaxis, :]
77 | probs = self.sess.run(self.acts_prob, {self.s: s}) # get probabilities for all actions
78 | return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel()) # return a int
79 |
80 |
81 | class Critic(object):
82 | def __init__(self, sess, n_features, lr=0.01):
83 | self.sess = sess
84 |
85 | self.s = tf.placeholder(tf.float32, [1, n_features], "state")
86 | self.v_ = tf.placeholder(tf.float32, [1, 1], "v_next")
87 | self.r = tf.placeholder(tf.float32, None, 'r')
88 |
89 | with tf.variable_scope('Critic'):
90 | l1 = tf.layers.dense(
91 | inputs=self.s,
92 | units=20, # number of hidden units
93 | activation=tf.nn.relu, # None
94 | # have to be linear to make sure the convergence of actor.
95 | # But linear approximator seems hardly learns the correct Q.
96 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
97 | bias_initializer=tf.constant_initializer(0.1), # biases
98 | name='l1'
99 | )
100 |
101 | self.v = tf.layers.dense(
102 | inputs=l1,
103 | units=1, # output units
104 | activation=None,
105 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
106 | bias_initializer=tf.constant_initializer(0.1), # biases
107 | name='V'
108 | )
109 |
110 | with tf.variable_scope('squared_TD_error'):
111 | self.td_error = self.r + GAMMA * self.v_ - self.v
112 | self.loss = tf.square(self.td_error) # TD_error = (r+gamma*V_next) - V_eval
113 | with tf.variable_scope('train'):
114 | self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
115 |
116 | def learn(self, s, r, s_):
117 | s, s_ = s[np.newaxis, :], s_[np.newaxis, :]
118 |
119 | v_ = self.sess.run(self.v, {self.s: s_})
120 | td_error, _ = self.sess.run([self.td_error, self.train_op],
121 | {self.s: s, self.v_: v_, self.r: r})
122 | return td_error
123 |
124 |
125 | sess = tf.Session()
126 |
127 | actor = Actor(sess, n_features=N_F, n_actions=N_A, lr=LR_A)
128 | critic = Critic(sess, n_features=N_F, lr=LR_C) # we need a good teacher, so the teacher should learn faster than the actor
129 |
130 | sess.run(tf.global_variables_initializer())
131 |
132 | if OUTPUT_GRAPH:
133 | tf.summary.FileWriter("logs/", sess.graph)
134 |
135 | for i_episode in range(MAX_EPISODE):
136 | s = env.reset()
137 | t = 0
138 | track_r = []
139 | while True:
140 | if RENDER: env.render()
141 |
142 | a = actor.choose_action(s)
143 |
144 | s_, r, done, info = env.step(a)
145 |
146 | if done: r = -20
147 |
148 | track_r.append(r)
149 |
150 | td_error = critic.learn(s, r, s_) # gradient = grad[r + gamma * V(s_) - V(s)]
151 | actor.learn(s, a, td_error) # true_gradient = grad[logPi(s,a) * td_error]
152 |
153 | s = s_
154 | t += 1
155 |
156 | if done or t >= MAX_EP_STEPS:
157 | ep_rs_sum = sum(track_r)
158 |
159 | if 'running_reward' not in globals():
160 | running_reward = ep_rs_sum
161 | else:
162 | running_reward = running_reward * 0.95 + ep_rs_sum * 0.05
163 | if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True # rendering
164 | print("episode:", i_episode, " reward:", int(running_reward))
165 | break
--------------------------------------------------------------------------------
/Actor_Critic/AC_continue_Pendulum.py:
--------------------------------------------------------------------------------
1 | """
2 | Actor-Critic with continuous action using TD-error as the Advantage, Reinforcement Learning.
3 | The Pendulum example (based on https://github.com/dennybritz/reinforcement-learning/blob/master/PolicyGradient/Continuous%20MountainCar%20Actor%20Critic%20Solution.ipynb)
4 | Cannot converge!!! oscillate!!!
5 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
6 | Using:
7 | tensorflow r1.3
8 | gym 0.8.0
9 | """
10 |
11 | import tensorflow as tf
12 | import numpy as np
13 | import gym
14 |
15 | np.random.seed(2)
16 | tf.set_random_seed(2) # reproducible
17 |
18 |
19 | class Actor(object):
20 | def __init__(self, sess, n_features, action_bound, lr=0.0001):
21 | self.sess = sess
22 |
23 | self.s = tf.placeholder(tf.float32, [1, n_features], "state")
24 | self.a = tf.placeholder(tf.float32, None, name="act")
25 | self.td_error = tf.placeholder(tf.float32, None, name="td_error") # TD_error
26 |
27 | l1 = tf.layers.dense(
28 | inputs=self.s,
29 | units=30, # number of hidden units
30 | activation=tf.nn.relu,
31 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
32 | bias_initializer=tf.constant_initializer(0.1), # biases
33 | name='l1'
34 | )
35 |
36 | mu = tf.layers.dense(
37 | inputs=l1,
38 | units=1, # number of hidden units
39 | activation=tf.nn.tanh,
40 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
41 | bias_initializer=tf.constant_initializer(0.1), # biases
42 | name='mu'
43 | )
44 |
45 | sigma = tf.layers.dense(
46 | inputs=l1,
47 | units=1, # output units
48 | activation=tf.nn.softplus, # get action probabilities
49 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
50 | bias_initializer=tf.constant_initializer(1.), # biases
51 | name='sigma'
52 | )
53 | global_step = tf.Variable(0, trainable=False)
54 | # self.e = epsilon = tf.train.exponential_decay(2., global_step, 1000, 0.9)
55 | self.mu, self.sigma = tf.squeeze(mu*2), tf.squeeze(sigma+0.1)
56 | self.normal_dist = tf.distributions.Normal(self.mu, self.sigma)
57 |
58 | self.action = tf.clip_by_value(self.normal_dist.sample(1), action_bound[0], action_bound[1])
59 |
60 | with tf.name_scope('exp_v'):
61 | log_prob = self.normal_dist.log_prob(self.a) # loss without advantage
62 | self.exp_v = log_prob * self.td_error # advantage (TD_error) guided loss
63 | # Add cross entropy cost to encourage exploration
64 | self.exp_v += 0.01*self.normal_dist.entropy()
65 |
66 | with tf.name_scope('train'):
67 | self.train_op = tf.train.AdamOptimizer(lr).minimize(-self.exp_v, global_step) # min(v) = max(-v)
68 |
69 | def learn(self, s, a, td):
70 | s = s[np.newaxis, :]
71 | feed_dict = {self.s: s, self.a: a, self.td_error: td}
72 | _, exp_v = self.sess.run([self.train_op, self.exp_v], feed_dict)
73 | return exp_v
74 |
75 | def choose_action(self, s):
76 | s = s[np.newaxis, :]
77 | return self.sess.run(self.action, {self.s: s}) # get probabilities for all actions
78 |
79 |
80 | class Critic(object):
81 | def __init__(self, sess, n_features, lr=0.01):
82 | self.sess = sess
83 | with tf.name_scope('inputs'):
84 | self.s = tf.placeholder(tf.float32, [1, n_features], "state")
85 | self.v_ = tf.placeholder(tf.float32, [1, 1], name="v_next")
86 | self.r = tf.placeholder(tf.float32, name='r')
87 |
88 | with tf.variable_scope('Critic'):
89 | l1 = tf.layers.dense(
90 | inputs=self.s,
91 | units=30, # number of hidden units
92 | activation=tf.nn.relu,
93 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
94 | bias_initializer=tf.constant_initializer(0.1), # biases
95 | name='l1'
96 | )
97 |
98 | self.v = tf.layers.dense(
99 | inputs=l1,
100 | units=1, # output units
101 | activation=None,
102 | kernel_initializer=tf.random_normal_initializer(0., .1), # weights
103 | bias_initializer=tf.constant_initializer(0.1), # biases
104 | name='V'
105 | )
106 |
107 | with tf.variable_scope('squared_TD_error'):
108 | self.td_error = tf.reduce_mean(self.r + GAMMA * self.v_ - self.v)
109 | self.loss = tf.square(self.td_error) # TD_error = (r+gamma*V_next) - V_eval
110 | with tf.variable_scope('train'):
111 | self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
112 |
113 | def learn(self, s, r, s_):
114 | s, s_ = s[np.newaxis, :], s_[np.newaxis, :]
115 |
116 | v_ = self.sess.run(self.v, {self.s: s_})
117 | td_error, _ = self.sess.run([self.td_error, self.train_op],
118 | {self.s: s, self.v_: v_, self.r: r})
119 | return td_error
120 |
121 |
122 | OUTPUT_GRAPH = False
123 | MAX_EPISODE = 1000
124 | MAX_EP_STEPS = 200
125 | DISPLAY_REWARD_THRESHOLD = -100 # renders environment if total episode reward is greater then this threshold
126 | RENDER = False # rendering wastes time
127 | GAMMA = 0.9
128 | LR_A = 0.001 # learning rate for actor
129 | LR_C = 0.01 # learning rate for critic
130 |
131 | env = gym.make('Pendulum-v0')
132 | env.seed(1) # reproducible
133 | env = env.unwrapped
134 |
135 | N_S = env.observation_space.shape[0]
136 | A_BOUND = env.action_space.high
137 |
138 | sess = tf.Session()
139 |
140 | actor = Actor(sess, n_features=N_S, lr=LR_A, action_bound=[-A_BOUND, A_BOUND])
141 | critic = Critic(sess, n_features=N_S, lr=LR_C)
142 |
143 | sess.run(tf.global_variables_initializer())
144 |
145 | if OUTPUT_GRAPH:
146 | tf.summary.FileWriter("logs/", sess.graph)
147 |
148 | for i_episode in range(MAX_EPISODE):
149 | s = env.reset()
150 | t = 0
151 | ep_rs = []
152 | while True:
153 | # if RENDER:
154 | env.render()
155 | a = actor.choose_action(s)
156 |
157 | s_, r, done, info = env.step(a)
158 | r /= 10
159 |
160 | td_error = critic.learn(s, r, s_) # gradient = grad[r + gamma * V(s_) - V(s)]
161 | actor.learn(s, a, td_error) # true_gradient = grad[logPi(s,a) * td_error]
162 |
163 | s = s_
164 | t += 1
165 | ep_rs.append(r)
166 | if t > MAX_EP_STEPS:
167 | ep_rs_sum = sum(ep_rs)
168 | if 'running_reward' not in globals():
169 | running_reward = ep_rs_sum
170 | else:
171 | running_reward = running_reward * 0.9 + ep_rs_sum * 0.1
172 | if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True # rendering
173 | print("episode:", i_episode, " reward:", int(running_reward))
174 | break
--------------------------------------------------------------------------------
/DDPG/DDPG.py:
--------------------------------------------------------------------------------
1 | """
2 | Deep Deterministic Policy Gradient (DDPG), Reinforcement Learning.
3 | DDPG is Actor Critic based algorithm.
4 | Pendulum example.
5 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
6 | Using:
7 | tensorflow 1.0
8 | gym 0.8.0
9 | """
10 |
11 | import tensorflow as tf
12 | import numpy as np
13 | import gym
14 | import time
15 |
16 |
17 | np.random.seed(1)
18 | tf.set_random_seed(1)
19 |
20 | ##################### hyper parameters ####################
21 |
22 | MAX_EPISODES = 200
23 | MAX_EP_STEPS = 200
24 | LR_A = 0.001 # learning rate for actor
25 | LR_C = 0.001 # learning rate for critic
26 | GAMMA = 0.9 # reward discount
27 | REPLACEMENT = [
28 | dict(name='soft', tau=0.01),
29 | dict(name='hard', rep_iter_a=600, rep_iter_c=500)
30 | ][0] # you can try different target replacement strategies
31 | MEMORY_CAPACITY = 10000
32 | BATCH_SIZE = 32
33 |
34 | RENDER = False
35 | OUTPUT_GRAPH = True
36 | ENV_NAME = 'Pendulum-v0'
37 |
38 | ############################### Actor ####################################
39 |
40 |
41 | class Actor(object):
42 | def __init__(self, sess, action_dim, action_bound, learning_rate, replacement):
43 | self.sess = sess
44 | self.a_dim = action_dim
45 | self.action_bound = action_bound
46 | self.lr = learning_rate
47 | self.replacement = replacement
48 | self.t_replace_counter = 0
49 |
50 | with tf.variable_scope('Actor'):
51 | # input s, output a
52 | self.a = self._build_net(S, scope='eval_net', trainable=True)
53 |
54 | # input s_, output a, get a_ for critic
55 | self.a_ = self._build_net(S_, scope='target_net', trainable=False)
56 |
57 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval_net')
58 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target_net')
59 |
60 | if self.replacement['name'] == 'hard':
61 | self.t_replace_counter = 0
62 | self.hard_replace = [tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)]
63 | else:
64 | self.soft_replace = [tf.assign(t, (1 - self.replacement['tau']) * t + self.replacement['tau'] * e)
65 | for t, e in zip(self.t_params, self.e_params)]
66 |
67 | def _build_net(self, s, scope, trainable):
68 | with tf.variable_scope(scope):
69 | init_w = tf.random_normal_initializer(0., 0.3)
70 | init_b = tf.constant_initializer(0.1)
71 | net = tf.layers.dense(s, 30, activation=tf.nn.relu,
72 | kernel_initializer=init_w, bias_initializer=init_b, name='l1',
73 | trainable=trainable)
74 | with tf.variable_scope('a'):
75 | actions = tf.layers.dense(net, self.a_dim, activation=tf.nn.tanh, kernel_initializer=init_w,
76 | bias_initializer=init_b, name='a', trainable=trainable)
77 | scaled_a = tf.multiply(actions, self.action_bound, name='scaled_a') # Scale output to -action_bound to action_bound
78 | return scaled_a
79 |
80 | def learn(self, s): # batch update
81 | self.sess.run(self.train_op, feed_dict={S: s})
82 |
83 | if self.replacement['name'] == 'soft':
84 | self.sess.run(self.soft_replace)
85 | else:
86 | if self.t_replace_counter % self.replacement['rep_iter_a'] == 0:
87 | self.sess.run(self.hard_replace)
88 | self.t_replace_counter += 1
89 |
90 | def choose_action(self, s):
91 | s = s[np.newaxis, :] # single state
92 | return self.sess.run(self.a, feed_dict={S: s})[0] # single action
93 |
94 | def add_grad_to_graph(self, a_grads):
95 | with tf.variable_scope('policy_grads'):
96 | # ys = policy;
97 | # xs = policy's parameters;
98 | # a_grads = the gradients of the policy to get more Q
99 | # tf.gradients will calculate dys/dxs with a initial gradients for ys, so this is dq/da * da/dparams
100 | self.policy_grads = tf.gradients(ys=self.a, xs=self.e_params, grad_ys=a_grads)
101 |
102 | with tf.variable_scope('A_train'):
103 | opt = tf.train.AdamOptimizer(-self.lr) # (- learning rate) for ascent policy
104 | self.train_op = opt.apply_gradients(zip(self.policy_grads, self.e_params))
105 |
106 |
107 | ############################### Critic ####################################
108 |
109 | class Critic(object):
110 | def __init__(self, sess, state_dim, action_dim, learning_rate, gamma, replacement, a, a_):
111 | self.sess = sess
112 | self.s_dim = state_dim
113 | self.a_dim = action_dim
114 | self.lr = learning_rate
115 | self.gamma = gamma
116 | self.replacement = replacement
117 |
118 | with tf.variable_scope('Critic'):
119 | # Input (s, a), output q
120 | self.a = tf.stop_gradient(a) # stop critic update flows to actor
121 | self.q = self._build_net(S, self.a, 'eval_net', trainable=True)
122 |
123 | # Input (s_, a_), output q_ for q_target
124 | self.q_ = self._build_net(S_, a_, 'target_net', trainable=False) # target_q is based on a_ from Actor's target_net
125 |
126 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval_net')
127 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target_net')
128 |
129 | with tf.variable_scope('target_q'):
130 | self.target_q = R + self.gamma * self.q_
131 |
132 | with tf.variable_scope('TD_error'):
133 | self.loss = tf.reduce_mean(tf.squared_difference(self.target_q, self.q))
134 |
135 | with tf.variable_scope('C_train'):
136 | self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss)
137 |
138 | with tf.variable_scope('a_grad'):
139 | self.a_grads = tf.gradients(self.q, a)[0] # tensor of gradients of each sample (None, a_dim)
140 |
141 | if self.replacement['name'] == 'hard':
142 | self.t_replace_counter = 0
143 | self.hard_replacement = [tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)]
144 | else:
145 | self.soft_replacement = [tf.assign(t, (1 - self.replacement['tau']) * t + self.replacement['tau'] * e)
146 | for t, e in zip(self.t_params, self.e_params)]
147 |
148 | def _build_net(self, s, a, scope, trainable):
149 | with tf.variable_scope(scope):
150 | init_w = tf.random_normal_initializer(0., 0.1)
151 | init_b = tf.constant_initializer(0.1)
152 |
153 | with tf.variable_scope('l1'):
154 | n_l1 = 30
155 | w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], initializer=init_w, trainable=trainable)
156 | w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], initializer=init_w, trainable=trainable)
157 | b1 = tf.get_variable('b1', [1, n_l1], initializer=init_b, trainable=trainable)
158 | net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
159 |
160 | with tf.variable_scope('q'):
161 | q = tf.layers.dense(net, 1, kernel_initializer=init_w, bias_initializer=init_b, trainable=trainable) # Q(s,a)
162 | return q
163 |
164 | def learn(self, s, a, r, s_):
165 | self.sess.run(self.train_op, feed_dict={S: s, self.a: a, R: r, S_: s_})
166 | if self.replacement['name'] == 'soft':
167 | self.sess.run(self.soft_replacement)
168 | else:
169 | if self.t_replace_counter % self.replacement['rep_iter_c'] == 0:
170 | self.sess.run(self.hard_replacement)
171 | self.t_replace_counter += 1
172 |
173 |
174 | ##################### Memory ####################
175 |
176 | class Memory(object):
177 | def __init__(self, capacity, dims):
178 | self.capacity = capacity
179 | self.data = np.zeros((capacity, dims))
180 | self.pointer = 0
181 |
182 | def store_transition(self, s, a, r, s_):
183 | transition = np.hstack((s, a, [r], s_))
184 | index = self.pointer % self.capacity # replace the old memory with new memory
185 | self.data[index, :] = transition
186 | self.pointer += 1
187 |
188 | def sample(self, n):
189 | assert self.pointer >= self.capacity, 'Memory has not been fulfilled'
190 | indices = np.random.choice(self.capacity, size=n)
191 | return self.data[indices, :]
192 |
193 |
194 | env = gym.make(ENV_NAME)
195 | env = env.unwrapped
196 | env.seed(1)
197 |
198 | state_dim = env.observation_space.shape[0]
199 | action_dim = env.action_space.shape[0]
200 | action_bound = env.action_space.high
201 |
202 | # all placeholder for tf
203 | with tf.name_scope('S'):
204 | S = tf.placeholder(tf.float32, shape=[None, state_dim], name='s')
205 | with tf.name_scope('R'):
206 | R = tf.placeholder(tf.float32, [None, 1], name='r')
207 | with tf.name_scope('S_'):
208 | S_ = tf.placeholder(tf.float32, shape=[None, state_dim], name='s_')
209 |
210 |
211 | sess = tf.Session()
212 |
213 | # Create actor and critic.
214 | # They are actually connected to each other, details can be seen in tensorboard or in this picture:
215 | actor = Actor(sess, action_dim, action_bound, LR_A, REPLACEMENT)
216 | critic = Critic(sess, state_dim, action_dim, LR_C, GAMMA, REPLACEMENT, actor.a, actor.a_)
217 | actor.add_grad_to_graph(critic.a_grads)
218 |
219 | sess.run(tf.global_variables_initializer())
220 |
221 | M = Memory(MEMORY_CAPACITY, dims=2 * state_dim + action_dim + 1)
222 |
223 | if OUTPUT_GRAPH:
224 | tf.summary.FileWriter("logs/", sess.graph)
225 |
226 | var = 3 # control exploration
227 |
228 | t1 = time.time()
229 | for i in range(MAX_EPISODES):
230 | s = env.reset()
231 | ep_reward = 0
232 |
233 | for j in range(MAX_EP_STEPS):
234 |
235 | if RENDER:
236 | env.render()
237 |
238 | # Add exploration noise
239 | a = actor.choose_action(s)
240 | a = np.clip(np.random.normal(a, var), -2, 2) # add randomness to action selection for exploration
241 | s_, r, done, info = env.step(a)
242 |
243 | M.store_transition(s, a, r / 10, s_)
244 |
245 | if M.pointer > MEMORY_CAPACITY:
246 | var *= .9995 # decay the action randomness
247 | b_M = M.sample(BATCH_SIZE)
248 | b_s = b_M[:, :state_dim]
249 | b_a = b_M[:, state_dim: state_dim + action_dim]
250 | b_r = b_M[:, -state_dim - 1: -state_dim]
251 | b_s_ = b_M[:, -state_dim:]
252 |
253 | critic.learn(b_s, b_a, b_r, b_s_)
254 | actor.learn(b_s)
255 |
256 | s = s_
257 | ep_reward += r
258 |
259 | if j == MAX_EP_STEPS-1:
260 | print('Episode:', i, ' Reward: %i' % int(ep_reward), 'Explore: %.2f' % var, )
261 | if ep_reward > -300:
262 | RENDER = True
263 | break
264 |
265 | print('Running time: ', time.time()-t1)
--------------------------------------------------------------------------------
/DQLearing/Brains/DQLearning.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import tensorflow.compat.v1 as tf
4 |
5 | np.random.seed(1)
6 | tf.set_random_seed(1)
7 |
8 |
9 | # Deep Q Network off-policy
10 | class DeepQNetwork:
11 | def __init__(
12 | self,
13 | n_actions,
14 | n_features,
15 | learning_rate=0.01,
16 | reward_decay=0.9,
17 | e_greedy=0.9,
18 | replace_target_iter=300,
19 | memory_size=500,
20 | batch_size=32,
21 | e_greedy_increment=None,
22 | output_graph=False,
23 | ):
24 | self.n_actions = n_actions
25 | self.n_features = n_features
26 | self.lr = learning_rate
27 | self.gamma = reward_decay
28 | self.epsilon_max = e_greedy
29 | self.replace_target_iter = replace_target_iter
30 | self.memory_size = memory_size
31 | self.batch_size = batch_size
32 | self.epsilon_increment = e_greedy_increment
33 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
34 |
35 | # total learning step
36 | self.learn_step_counter = 0
37 |
38 | # initialize zero memory [s, a, r, s_]
39 | self.memory = np.zeros((self.memory_size, n_features * 2 + 2))
40 |
41 | # consist of [target_net, evaluate_net]
42 | self._build_net()
43 | t_params = tf.get_collection('target_net_params')
44 | e_params = tf.get_collection('eval_net_params')
45 | self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]
46 |
47 | self.sess = tf.Session()
48 |
49 | if output_graph:
50 | # $ tensorboard --logdir=logs
51 | # tf.train.SummaryWriter soon be deprecated, use following
52 | tf.summary.FileWriter("logs/", self.sess.graph)
53 |
54 | self.sess.run(tf.global_variables_initializer())
55 | self.cost_his = []
56 |
57 | def _build_net(self):
58 | # ------------------ build evaluate_net ------------------
59 | self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s') # input
60 | self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculating loss
61 | with tf.variable_scope('eval_net'):
62 | # c_names(collections_names) are the collections to store variables
63 | c_names, n_l1, w_initializer, b_initializer = \
64 | ['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 10, \
65 | tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) # config of layers
66 |
67 | # first layer. collections is used later when assign to target net
68 | with tf.variable_scope('l1'):
69 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
70 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
71 | l1 = tf.nn.relu(tf.matmul(self.s, w1) + b1)
72 |
73 | # second layer. collections is used later when assign to target net
74 | with tf.variable_scope('l2'):
75 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
76 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
77 | self.q_eval = tf.matmul(l1, w2) + b2
78 |
79 | with tf.variable_scope('loss'):
80 | self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
81 | with tf.variable_scope('train'):
82 | self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
83 |
84 | # ------------------ build target_net ------------------
85 | self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_') # input
86 | with tf.variable_scope('target_net'):
87 | # c_names(collections_names) are the collections to store variables
88 | c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
89 |
90 | # first layer. collections is used later when assign to target net
91 | with tf.variable_scope('l1'):
92 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
93 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
94 | l1 = tf.nn.relu(tf.matmul(self.s_, w1) + b1)
95 |
96 | # second layer. collections is used later when assign to target net
97 | with tf.variable_scope('l2'):
98 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
99 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
100 | self.q_next = tf.matmul(l1, w2) + b2
101 |
102 | def store_transition(self, s, a, r, s_):
103 | if not hasattr(self, 'memory_counter'):
104 | self.memory_counter = 0
105 |
106 | transition = np.hstack((s, [a, r], s_))
107 |
108 | # replace the old memory with new memory
109 | index = self.memory_counter % self.memory_size
110 | self.memory[index, :] = transition
111 |
112 | self.memory_counter += 1
113 |
114 | def choose_action(self, observation):
115 | # to have batch dimension when feed into tf placeholder
116 | observation = observation[np.newaxis, :]
117 |
118 | if np.random.uniform() < self.epsilon:
119 | # forward feed the observation and get q value for every actions
120 | actions_value = self.sess.run(self.q_eval, feed_dict={self.s: observation})
121 | action = np.argmax(actions_value)
122 | else:
123 | action = np.random.randint(0, self.n_actions)
124 | return action
125 |
126 | def learn(self):
127 | # check to replace target parameters
128 | if self.learn_step_counter % self.replace_target_iter == 0:
129 | self.sess.run(self.replace_target_op)
130 | print('\ntarget_params_replaced\n')
131 |
132 | # sample batch memory from all memory
133 | if self.memory_counter > self.memory_size:
134 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
135 | else:
136 | sample_index = np.random.choice(self.memory_counter, size=self.batch_size)
137 | batch_memory = self.memory[sample_index, :]
138 |
139 | q_next, q_eval = self.sess.run(
140 | [self.q_next, self.q_eval],
141 | feed_dict={
142 | self.s_: batch_memory[:, -self.n_features:], # fixed params
143 | self.s: batch_memory[:, :self.n_features], # newest params
144 | })
145 |
146 | # change q_target w.r.t q_eval's action
147 | q_target = q_eval.copy()
148 |
149 | batch_index = np.arange(self.batch_size, dtype=np.int32)
150 | eval_act_index = batch_memory[:, self.n_features].astype(int)
151 | reward = batch_memory[:, self.n_features + 1]
152 |
153 | q_target[batch_index, eval_act_index] = reward + self.gamma * np.max(q_next, axis=1)
154 |
155 | """
156 | For example in this batch I have 2 samples and 3 actions:
157 | q_eval =
158 | [[1, 2, 3],
159 | [4, 5, 6]]
160 | q_target = q_eval =
161 | [[1, 2, 3],
162 | [4, 5, 6]]
163 | Then change q_target with the real q_target value w.r.t the q_eval's action.
164 | For example in:
165 | sample 0, I took action 0, and the max q_target value is -1;
166 | sample 1, I took action 2, and the max q_target value is -2:
167 | q_target =
168 | [[-1, 2, 3],
169 | [4, 5, -2]]
170 | So the (q_target - q_eval) becomes:
171 | [[(-1)-(1), 0, 0],
172 | [0, 0, (-2)-(6)]]
173 | We then backpropagate this error w.r.t the corresponding action to network,
174 | leave other action as error=0 cause we didn't choose it.
175 | """
176 |
177 | # train eval network
178 | _, self.cost = self.sess.run([self._train_op, self.loss],
179 | feed_dict={self.s: batch_memory[:, :self.n_features],
180 | self.q_target: q_target})
181 | self.cost_his.append(self.cost)
182 |
183 | # increasing epsilon
184 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
185 | self.learn_step_counter += 1
186 |
187 | def plot_cost(self):
188 | import matplotlib.pyplot as plt
189 | plt.plot(np.arange(len(self.cost_his)), self.cost_his)
190 | plt.ylabel('Cost')
191 | plt.xlabel('training steps')
192 | plt.show()
--------------------------------------------------------------------------------
/DQLearing/Brains/DQN.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 | import pandas as pd
5 | from torch.autograd import Variable
6 |
7 |
8 | class DeepQ(nn.Module):
9 | def __init__(
10 | self,
11 | n_actions,
12 | n_features,
13 | hidden=10
14 | ):
15 | super(DeepQ, self).__init__()
16 | self.layer = nn.Sequential(
17 | nn.Linear(n_features, hidden),
18 | nn.ReLU(),
19 | nn.Linear(hidden, n_actions)
20 | )
21 |
22 | def forward(self, x):
23 | x = self.layer(x)
24 |
25 | return x
26 |
27 |
28 | class DeepQNet():
29 | def __init__(
30 | self,
31 | n_actions,
32 | n_features,
33 | learning_rate=0.001,
34 | reward_decay=0.9,
35 | e_greedy=0.9,
36 | replace_target_iter=300,
37 | memory_size=500,
38 | batch_size=32,
39 | e_greedy_increment=None,
40 | output_graph=False,
41 | ):
42 | self.memory_counter = 0
43 | self.n_actions = n_actions
44 | self.n_features = n_features
45 | self.lr = learning_rate
46 | self.gamma = reward_decay
47 | self.epsilon_max = e_greedy
48 | self.replace_target_iter = replace_target_iter
49 | self.memory_size = memory_size
50 | self.batch_size = batch_size
51 | self.epsilon_increment = e_greedy_increment
52 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
53 | self.learn_step_counter = 0
54 | self.memory = np.zeros((self.memory_size, n_features * 2 + 2))
55 | self.cost_his = []
56 | hidden = 10
57 | self.eval_net, self.target_net = DeepQ(n_actions, n_features, hidden), DeepQ(n_actions, n_features, hidden)
58 | self.loss_fn = nn.MSELoss()
59 | self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=learning_rate)
60 |
61 | def store_transition(self, s, a, r, s_):
62 | transition = np.hstack((s, [a, r], s_))
63 | index = self.memory_counter % self.memory_size
64 |
65 | self.memory[index, :] = transition
66 | self.memory_counter += 1
67 |
68 | def choose_action(self, observation):
69 | observation = torch.Tensor(observation[np.newaxis, :])
70 |
71 | if np.random.uniform() < self.epsilon:
72 | actions_value = self.eval_net(observation)
73 | action = np.argmax(actions_value.detach().numpy())
74 | else:
75 | action = np.random.randint(0, self.n_actions)
76 | return action
77 |
78 | def learn(self):
79 | if self.learn_step_counter % self.replace_target_iter == 0:
80 | self.target_net.load_state_dict(self.eval_net.state_dict())
81 | print('\ntarget_params_replaced\n')
82 | # 随机选取读取记忆的坐标
83 | if self.memory_counter > self.memory_size:
84 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
85 | else:
86 | sample_index = np.random.choice(self.memory_counter, size=self.batch_size)
87 | batch_memory = self.memory[sample_index, :]
88 | # q_next 用旧网络、动作后的环境预测，q_eval，q_target 用新网络、动作前的环境；同时预测记忆中的情形
89 | q_next, q_eval = self.target_net(torch.Tensor(batch_memory[:, -self.n_features:])).detach().numpy(), \
90 | self.eval_net(torch.Tensor(batch_memory[:, :self.n_features])).detach().numpy()
91 | q_target = q_eval#np.zeros((len(sample_index), self.n_actions))
92 | # ；动作在 n_features 处；reward 在动作后
93 | batch_index = np.arange(self.batch_size, dtype=np.int32)
94 | eval_act_index = batch_memory[:, self.n_features].astype(int)
95 | reward = batch_memory[:, self.n_features + 1]
96 | # 每次学习都用下一个状态的动作结合反馈作为当前动作值（这样，将未来状态的动作作为目标，有一定前瞻性）
97 | q_target[batch_index, eval_act_index] = reward + self.gamma * np.max(q_next, axis=1)
98 | loss = self.loss_fn(self.eval_net(torch.Tensor(batch_memory[:, :self.n_features])), torch.Tensor(q_target))
99 | self.optimizer.zero_grad()
100 | loss.backward()
101 | self.optimizer.step()
102 | cost = loss.item()
103 |
104 | self.cost_his.append(cost)
105 |
106 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
107 | self.learn_step_counter += 1
108 |
109 | def plot_cost(self):
110 | import matplotlib.pyplot as plt
111 | plt.plot(np.arange(len(self.cost_his)), self.cost_his)
112 | plt.ylabel('Cost')
113 | plt.xlabel('training steps')
114 | plt.show()
115 |
--------------------------------------------------------------------------------
/DQLearing/Brains/DQNTF2.py:
--------------------------------------------------------------------------------
1 | import tensorflow as tf
2 | from tensorflow.keras import layers
3 | import numpy as np
4 |
5 | class DQN(tf.keras.Model):
6 | def __init__(
7 | self,
8 | n_actions,
9 | n_features,
10 | n_hidden
11 | ):
12 | super(DQN, self).__init__(name='DQN')
13 | self.net = tf.keras.Sequential(
14 | [layers.Dense(n_features, activation=tf.keras.activations.relu),
15 | layers.Dense(n_hidden, activation=tf.keras.activations.relu),
16 | layers.Dense(units=n_actions, activation=tf.keras.activations.softmax)]
17 | )
18 |
19 | def call(self, inputs, training=None, mask=None):
20 | return self.net(inputs)
21 |
22 | class DeepQN():
23 | def __init__(
24 | self,
25 | n_actions,
26 | n_features,
27 | learning_rate=0.001,
28 | reward_decay=0.9,
29 | e_greedy=0.9,
30 | replace_target_iter=300,
31 | memory_size=500,
32 | batch_size=32,
33 | e_greedy_increment=None,
34 | ):
35 | self.memory_counter = 0
36 | self.n_actions = n_actions
37 | self.n_features = n_features
38 | self.lr = learning_rate
39 | self.reward_decay = reward_decay
40 | self.epsilon_max = e_greedy
41 | self.replace_target_iter = replace_target_iter
42 | self.memory_size = memory_size
43 | self.batch_size = batch_size
44 | self.epsilon_increment = e_greedy_increment
45 | self.memory = np.zeros((memory_size, n_features*2 + 2))
46 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
47 | self.learn_step_counter = 0
48 | self.cost_his = []
49 | print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
50 |
51 | self.eval_net = DQN(n_actions, n_features, n_hidden=10)
52 | self.eval_net.compile(optimizer='rmsprop',loss=tf.keras.losses.mean_squared_error)
53 | self.target_net = DQN(n_actions, n_features, n_hidden=10)
54 | self.target_net.compile(optimizer=tf.keras.optimizers.RMSprop(),
55 | loss=tf.keras.losses.mean_squared_error)
56 |
57 | def store_transition(self, s, a, r, s_):
58 | transition = np.hstack((s, [a, r], s_))
59 |
60 | index = self.memory_counter % self.memory_size
61 | self.memory[index, :] = transition
62 |
63 | self.memory_counter += 1
64 |
65 | def choose_action(self, observation):
66 | observation = observation[np.newaxis, :]
67 |
68 | if np.random.uniform() < self.epsilon:
69 | print('predict')
70 | action_value = self.eval_net.predict(observation)
71 | action = np.argmax(action_value)
72 | else:
73 | action = np.random.randint(0, self.n_actions)
74 |
75 | return action
76 |
77 | def learn(self):
78 | if self.learn_step_counter % self.replace_target_iter == 0:
79 | print("replace")
80 | self.eval_net.save_weights('./tmp_model')
81 | self.target_net.load_weights('./tmp_model')
82 |
83 | if self.memory_counter > self.memory_size:
84 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
85 | else:
86 | sample_index = np.random.choice(self.memory_counter, size=self.batch_size)
87 |
88 | batch_memory = self.memory[sample_index,:]
89 | q_next = self.target_net.predict(batch_memory[:, -self.n_features])
90 | q_target = self.eval_net.predict(batch_memory[:, self.n_features])
91 |
92 | batch_index = np.arange(self.batch_size, dtype=np.int32)
93 | eval_act_index = batch_memory[:, self.n_features].astype(int)
94 | reward = batch_memory[:, self.n_features + 1]
95 |
96 | q_target[batch_index, eval_act_index] = reward + self.reward_decay * np.max(q_next, axis=1)
97 | self.eval_net.fit(batch_memory[:, self.n_features], q_target)
98 |
99 | cost = self.eval_net.total_loss
100 |
101 | self.cost_his.append(cost)
102 |
103 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
104 | self.learn_step_counter += 1
105 |
--------------------------------------------------------------------------------
/DQLearing/Brains/DuelingDQN.py:
--------------------------------------------------------------------------------
1 | """
2 | The Dueling DQN based on this paper: https://arxiv.org/abs/1511.06581
3 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
4 | Using:
5 | Tensorflow: 1.0
6 | gym: 0.8.0
7 | """
8 |
9 | import numpy as np
10 | import tensorflow as tf
11 |
12 | np.random.seed(1)
13 | tf.set_random_seed(1)
14 |
15 |
16 | class DuelingDQN:
17 | def __init__(
18 | self,
19 | n_actions,
20 | n_features,
21 | learning_rate=0.001,
22 | reward_decay=0.9,
23 | e_greedy=0.9,
24 | replace_target_iter=200,
25 | memory_size=500,
26 | batch_size=32,
27 | e_greedy_increment=None,
28 | output_graph=False,
29 | dueling=True,
30 | sess=None,
31 | ):
32 | self.n_actions = n_actions
33 | self.n_features = n_features
34 | self.lr = learning_rate
35 | self.gamma = reward_decay
36 | self.epsilon_max = e_greedy
37 | self.replace_target_iter = replace_target_iter
38 | self.memory_size = memory_size
39 | self.batch_size = batch_size
40 | self.epsilon_increment = e_greedy_increment
41 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
42 |
43 | self.dueling = dueling # decide to use dueling DQN or not
44 |
45 | self.learn_step_counter = 0
46 | self.memory = np.zeros((self.memory_size, n_features*2+2))
47 | self._build_net()
48 | t_params = tf.get_collection('target_net_params')
49 | e_params = tf.get_collection('eval_net_params')
50 | self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]
51 |
52 | if sess is None:
53 | self.sess = tf.Session()
54 | self.sess.run(tf.global_variables_initializer())
55 | else:
56 | self.sess = sess
57 | if output_graph:
58 | tf.summary.FileWriter("logs/", self.sess.graph)
59 | self.cost_his = []
60 |
61 | def _build_net(self):
62 | def build_layers(s, c_names, n_l1, w_initializer, b_initializer):
63 | with tf.variable_scope('l1'):
64 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
65 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
66 | l1 = tf.nn.relu(tf.matmul(s, w1) + b1)
67 |
68 | if self.dueling:
69 | # Dueling DQN
70 | with tf.variable_scope('Value'):
71 | w2 = tf.get_variable('w2', [n_l1, 1], initializer=w_initializer, collections=c_names)
72 | b2 = tf.get_variable('b2', [1, 1], initializer=b_initializer, collections=c_names)
73 | self.V = tf.matmul(l1, w2) + b2
74 |
75 | with tf.variable_scope('Advantage'):
76 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
77 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
78 | self.A = tf.matmul(l1, w2) + b2
79 |
80 | with tf.variable_scope('Q'):
81 | out = self.V + (self.A - tf.reduce_mean(self.A, axis=1, keep_dims=True)) # Q = V(s) + A(s,a)
82 | else:
83 | with tf.variable_scope('Q'):
84 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
85 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
86 | out = tf.matmul(l1, w2) + b2
87 |
88 | return out
89 |
90 | # ------------------ build evaluate_net ------------------
91 | self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s') # input
92 | self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculating loss
93 | with tf.variable_scope('eval_net'):
94 | c_names, n_l1, w_initializer, b_initializer = \
95 | ['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 20, \
96 | tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) # config of layers
97 |
98 | self.q_eval = build_layers(self.s, c_names, n_l1, w_initializer, b_initializer)
99 |
100 | with tf.variable_scope('loss'):
101 | self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
102 | with tf.variable_scope('train'):
103 | self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
104 |
105 | # ------------------ build target_net ------------------
106 | self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_') # input
107 | with tf.variable_scope('target_net'):
108 | c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
109 |
110 | self.q_next = build_layers(self.s_, c_names, n_l1, w_initializer, b_initializer)
111 |
112 | def store_transition(self, s, a, r, s_):
113 | if not hasattr(self, 'memory_counter'):
114 | self.memory_counter = 0
115 | transition = np.hstack((s, [a, r], s_))
116 | index = self.memory_counter % self.memory_size
117 | self.memory[index, :] = transition
118 | self.memory_counter += 1
119 |
120 | def choose_action(self, observation):
121 | observation = observation[np.newaxis, :]
122 | if np.random.uniform() < self.epsilon: # choosing action
123 | actions_value = self.sess.run(self.q_eval, feed_dict={self.s: observation})
124 | action = np.argmax(actions_value)
125 | else:
126 | action = np.random.randint(0, self.n_actions)
127 | return action
128 |
129 | def learn(self):
130 | if self.learn_step_counter % self.replace_target_iter == 0:
131 | self.sess.run(self.replace_target_op)
132 | print('\ntarget_params_replaced\n')
133 |
134 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
135 | batch_memory = self.memory[sample_index, :]
136 |
137 | q_next = self.sess.run(self.q_next, feed_dict={self.s_: batch_memory[:, -self.n_features:]}) # next observation
138 | q_eval = self.sess.run(self.q_eval, {self.s: batch_memory[:, :self.n_features]})
139 |
140 | q_target = q_eval.copy()
141 |
142 | batch_index = np.arange(self.batch_size, dtype=np.int32)
143 | eval_act_index = batch_memory[:, self.n_features].astype(int)
144 | reward = batch_memory[:, self.n_features + 1]
145 |
146 | q_target[batch_index, eval_act_index] = reward + self.gamma * np.max(q_next, axis=1)
147 |
148 | _, self.cost = self.sess.run([self._train_op, self.loss],
149 | feed_dict={self.s: batch_memory[:, :self.n_features],
150 | self.q_target: q_target})
151 | self.cost_his.append(self.cost)
152 |
153 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
154 | self.learn_step_counter += 1
--------------------------------------------------------------------------------
/DQLearing/Brains/Priority_DQN.py:
--------------------------------------------------------------------------------
1 | """
2 | The DQN improvement: Prioritized Experience Replay (based on https://arxiv.org/abs/1511.05952)
3 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
4 | Using:
5 | Tensorflow: 1.0
6 | gym: 0.8.0
7 | """
8 |
9 | import numpy as np
10 | import tensorflow as tf
11 |
12 | np.random.seed(1)
13 | tf.set_random_seed(1)
14 |
15 |
16 | class SumTree(object):
17 | """
18 | This SumTree code is a modified version and the original code is from:
19 | https://github.com/jaara/AI-blog/blob/master/SumTree.py
20 | Story data with its priority in the tree.
21 | """
22 | data_pointer = 0
23 |
24 | def __init__(self, capacity):
25 | self.capacity = capacity # for all priority values
26 | self.tree = np.zeros(2 * capacity - 1)
27 | # [--------------Parent nodes-------------][-------leaves to recode priority-------]
28 | # size: capacity - 1 size: capacity
29 | self.data = np.zeros(capacity, dtype=object) # for all transitions
30 | # [--------------data frame-------------]
31 | # size: capacity
32 |
33 | def add(self, p, data):
34 | tree_idx = self.data_pointer + self.capacity - 1
35 | self.data[self.data_pointer] = data # update data_frame
36 | self.update(tree_idx, p) # update tree_frame
37 |
38 | self.data_pointer += 1
39 | if self.data_pointer >= self.capacity: # replace when exceed the capacity
40 | self.data_pointer = 0
41 |
42 | def update(self, tree_idx, p):
43 | change = p - self.tree[tree_idx]
44 | self.tree[tree_idx] = p
45 | # then propagate the change through tree
46 | while tree_idx != 0: # this method is faster than the recursive loop in the reference code
47 | tree_idx = (tree_idx - 1) // 2
48 | self.tree[tree_idx] += change
49 |
50 | def get_leaf(self, v):
51 | """
52 | Tree structure and array storage:
53 | Tree index:
54 | 0 -> storing priority sum
55 | / \
56 | 1 2
57 | / \ / \
58 | 3 4 5 6 -> storing priority for transitions
59 | Array type for storing:
60 | [0,1,2,3,4,5,6]
61 | """
62 | parent_idx = 0
63 | while True: # the while loop is faster than the method in the reference code
64 | cl_idx = 2 * parent_idx + 1 # this leaf's left and right kids
65 | cr_idx = cl_idx + 1
66 | if cl_idx >= len(self.tree): # reach bottom, end search
67 | leaf_idx = parent_idx
68 | break
69 | else: # downward search, always search for a higher priority node
70 | if v <= self.tree[cl_idx]:
71 | parent_idx = cl_idx
72 | else:
73 | v -= self.tree[cl_idx]
74 | parent_idx = cr_idx
75 |
76 | data_idx = leaf_idx - self.capacity + 1
77 | return leaf_idx, self.tree[leaf_idx], self.data[data_idx]
78 |
79 | @property
80 | def total_p(self):
81 | return self.tree[0] # the root
82 |
83 |
84 | class Memory(object): # stored as ( s, a, r, s_ ) in SumTree
85 | """
86 | This Memory class is modified based on the original code from:
87 | https://github.com/jaara/AI-blog/blob/master/Seaquest-DDQN-PER.py
88 | """
89 | epsilon = 0.01 # small amount to avoid zero priority
90 | alpha = 0.6 # [0~1] convert the importance of TD error to priority
91 | beta = 0.4 # importance-sampling, from initial value increasing to 1
92 | beta_increment_per_sampling = 0.001
93 | abs_err_upper = 1. # clipped abs error
94 |
95 | def __init__(self, capacity):
96 | self.tree = SumTree(capacity)
97 |
98 | def store(self, transition):
99 | max_p = np.max(self.tree.tree[-self.tree.capacity:])
100 | if max_p == 0:
101 | max_p = self.abs_err_upper
102 | self.tree.add(max_p, transition) # set the max p for new p
103 |
104 | def sample(self, n):
105 | b_idx, b_memory, ISWeights = np.empty((n,), dtype=np.int32), np.empty((n, self.tree.data[0].size)), np.empty((n, 1))
106 | pri_seg = self.tree.total_p / n # priority segment
107 | self.beta = np.min([1., self.beta + self.beta_increment_per_sampling]) # max = 1
108 |
109 | min_prob = np.min(self.tree.tree[-self.tree.capacity:]) / self.tree.total_p # for later calculate ISweight
110 | for i in range(n):
111 | a, b = pri_seg * i, pri_seg * (i + 1)
112 | v = np.random.uniform(a, b)
113 | idx, p, data = self.tree.get_leaf(v)
114 | prob = p / self.tree.total_p
115 | ISWeights[i, 0] = np.power(prob/min_prob, -self.beta)
116 | b_idx[i], b_memory[i, :] = idx, data
117 | return b_idx, b_memory, ISWeights
118 |
119 | def batch_update(self, tree_idx, abs_errors):
120 | abs_errors += self.epsilon # convert to abs and avoid 0
121 | clipped_errors = np.minimum(abs_errors, self.abs_err_upper)
122 | ps = np.power(clipped_errors, self.alpha)
123 | for ti, p in zip(tree_idx, ps):
124 | self.tree.update(ti, p)
125 |
126 |
127 | class DQNPrioritizedReplay:
128 | def __init__(
129 | self,
130 | n_actions,
131 | n_features,
132 | learning_rate=0.005,
133 | reward_decay=0.9,
134 | e_greedy=0.9,
135 | replace_target_iter=500,
136 | memory_size=10000,
137 | batch_size=32,
138 | e_greedy_increment=None,
139 | output_graph=False,
140 | prioritized=True,
141 | sess=None,
142 | ):
143 | self.n_actions = n_actions
144 | self.n_features = n_features
145 | self.lr = learning_rate
146 | self.gamma = reward_decay
147 | self.epsilon_max = e_greedy
148 | self.replace_target_iter = replace_target_iter
149 | self.memory_size = memory_size
150 | self.batch_size = batch_size
151 | self.epsilon_increment = e_greedy_increment
152 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
153 |
154 | self.prioritized = prioritized # decide to use double q or not
155 |
156 | self.learn_step_counter = 0
157 |
158 | self._build_net()
159 | t_params = tf.get_collection('target_net_params')
160 | e_params = tf.get_collection('eval_net_params')
161 | self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]
162 |
163 | if self.prioritized:
164 | self.memory = Memory(capacity=memory_size)
165 | else:
166 | self.memory = np.zeros((self.memory_size, n_features*2+2))
167 |
168 | if sess is None:
169 | self.sess = tf.Session()
170 | self.sess.run(tf.global_variables_initializer())
171 | else:
172 | self.sess = sess
173 |
174 | if output_graph:
175 | tf.summary.FileWriter("logs/", self.sess.graph)
176 |
177 | self.cost_his = []
178 |
179 | def _build_net(self):
180 | def build_layers(s, c_names, n_l1, w_initializer, b_initializer, trainable):
181 | with tf.variable_scope('l1'):
182 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names, trainable=trainable)
183 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names, trainable=trainable)
184 | l1 = tf.nn.relu(tf.matmul(s, w1) + b1)
185 |
186 | with tf.variable_scope('l2'):
187 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names, trainable=trainable)
188 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names, trainable=trainable)
189 | out = tf.matmul(l1, w2) + b2
190 | return out
191 |
192 | # ------------------ build evaluate_net ------------------
193 | self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s') # input
194 | self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculating loss
195 | if self.prioritized:
196 | self.ISWeights = tf.placeholder(tf.float32, [None, 1], name='IS_weights')
197 | with tf.variable_scope('eval_net'):
198 | c_names, n_l1, w_initializer, b_initializer = \
199 | ['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 20, \
200 | tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) # config of layers
201 |
202 | self.q_eval = build_layers(self.s, c_names, n_l1, w_initializer, b_initializer, True)
203 |
204 | with tf.variable_scope('loss'):
205 | if self.prioritized:
206 | self.abs_errors = tf.reduce_sum(tf.abs(self.q_target - self.q_eval), axis=1) # for updating Sumtree
207 | self.loss = tf.reduce_mean(self.ISWeights * tf.squared_difference(self.q_target, self.q_eval))
208 | else:
209 | self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
210 | with tf.variable_scope('train'):
211 | self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
212 |
213 | # ------------------ build target_net ------------------
214 | self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_') # input
215 | with tf.variable_scope('target_net'):
216 | c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
217 | self.q_next = build_layers(self.s_, c_names, n_l1, w_initializer, b_initializer, False)
218 |
219 | def store_transition(self, s, a, r, s_):
220 | if self.prioritized: # prioritized replay
221 | transition = np.hstack((s, [a, r], s_))
222 | self.memory.store(transition) # have high priority for newly arrived transition
223 | else: # random replay
224 | if not hasattr(self, 'memory_counter'):
225 | self.memory_counter = 0
226 | transition = np.hstack((s, [a, r], s_))
227 | index = self.memory_counter % self.memory_size
228 | self.memory[index, :] = transition
229 | self.memory_counter += 1
230 |
231 | def choose_action(self, observation):
232 | observation = observation[np.newaxis, :]
233 | if np.random.uniform() < self.epsilon:
234 | actions_value = self.sess.run(self.q_eval, feed_dict={self.s: observation})
235 | action = np.argmax(actions_value)
236 | else:
237 | action = np.random.randint(0, self.n_actions)
238 | return action
239 |
240 | def learn(self):
241 | if self.learn_step_counter % self.replace_target_iter == 0:
242 | self.sess.run(self.replace_target_op)
243 | print('\ntarget_params_replaced\n')
244 |
245 | if self.prioritized:
246 | tree_idx, batch_memory, ISWeights = self.memory.sample(self.batch_size)
247 | else:
248 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
249 | batch_memory = self.memory[sample_index, :]
250 |
251 | q_next, q_eval = self.sess.run(
252 | [self.q_next, self.q_eval],
253 | feed_dict={self.s_: batch_memory[:, -self.n_features:],
254 | self.s: batch_memory[:, :self.n_features]})
255 |
256 | q_target = q_eval.copy()
257 | batch_index = np.arange(self.batch_size, dtype=np.int32)
258 | eval_act_index = batch_memory[:, self.n_features].astype(int)
259 | reward = batch_memory[:, self.n_features + 1]
260 |
261 | q_target[batch_index, eval_act_index] = reward + self.gamma * np.max(q_next, axis=1)
262 |
263 | if self.prioritized:
264 | _, abs_errors, self.cost = self.sess.run([self._train_op, self.abs_errors, self.loss],
265 | feed_dict={self.s: batch_memory[:, :self.n_features],
266 | self.q_target: q_target,
267 | self.ISWeights: ISWeights})
268 | self.memory.batch_update(tree_idx, abs_errors) # update priority
269 | else:
270 | _, self.cost = self.sess.run([self._train_op, self.loss],
271 | feed_dict={self.s: batch_memory[:, :self.n_features],
272 | self.q_target: q_target})
273 |
274 | self.cost_his.append(self.cost)
275 |
276 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
277 | self.learn_step_counter += 1
--------------------------------------------------------------------------------
/DQLearing/Brains/RL_brain.py:
--------------------------------------------------------------------------------
1 | """
2 | This part of code is the DQN brain, which is a brain of the agent.
3 | All decisions are made in here.
4 | Using Tensorflow to build the neural network.
5 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
6 | Using:
7 | Tensorflow: 1.0
8 | gym: 0.7.3
9 | """
10 |
11 | import numpy as np
12 | import pandas as pd
13 | import tensorflow as tf
14 |
15 | np.random.seed(1)
16 | tf.set_random_seed(1)
17 |
18 |
19 | # Deep Q Network off-policy
20 | class DeepQNetwork:
21 | def __init__(
22 | self,
23 | n_actions,
24 | n_features,
25 | learning_rate=0.01,
26 | reward_decay=0.9,
27 | e_greedy=0.9,
28 | replace_target_iter=300,
29 | memory_size=500,
30 | batch_size=32,
31 | e_greedy_increment=None,
32 | output_graph=True,
33 | ):
34 | self.n_actions = n_actions
35 | self.n_features = n_features
36 | self.lr = learning_rate
37 | self.gamma = reward_decay
38 | self.epsilon_max = e_greedy
39 | self.replace_target_iter = replace_target_iter
40 | self.memory_size = memory_size
41 | self.batch_size = batch_size
42 | self.epsilon_increment = e_greedy_increment
43 | self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max
44 |
45 | # total learning step
46 | self.learn_step_counter = 0
47 |
48 | # initialize zero memory [s, a, r, s_]
49 | self.memory = np.zeros((self.memory_size, n_features * 2 + 2))
50 |
51 | # consist of [target_net, evaluate_net]
52 | self._build_net()
53 | t_params = tf.get_collection('target_net_params')
54 | e_params = tf.get_collection('eval_net_params')
55 | self.replace_target_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]
56 |
57 | self.sess = tf.Session()
58 |
59 | if output_graph:
60 | # $ tensorboard --logdir=logs
61 | # tf.train.SummaryWriter soon be deprecated, use following
62 | tf.summary.FileWriter("logs/", self.sess.graph)
63 |
64 | self.sess.run(tf.global_variables_initializer())
65 | self.cost_his = []
66 |
67 | def _build_net(self):
68 | # ------------------ build evaluate_net ------------------
69 | self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s') # input
70 | self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculating loss
71 | with tf.variable_scope('eval_net'):
72 | # c_names(collections_names) are the collections to store variables
73 | c_names, n_l1, w_initializer, b_initializer = \
74 | ['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 10, \
75 | tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) # config of layers
76 |
77 | # first layer. collections is used later when assign to target net
78 | with tf.variable_scope('l1'):
79 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
80 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
81 | l1 = tf.nn.relu(tf.matmul(self.s, w1) + b1)
82 |
83 | # second layer. collections is used later when assign to target net
84 | with tf.variable_scope('l2'):
85 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
86 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
87 | self.q_eval = tf.matmul(l1, w2) + b2
88 |
89 | with tf.variable_scope('loss'):
90 | self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
91 | with tf.variable_scope('train'):
92 | self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
93 |
94 | # ------------------ build target_net ------------------
95 | self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_') # input
96 | with tf.variable_scope('target_net'):
97 | # c_names(collections_names) are the collections to store variables
98 | c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
99 |
100 | # first layer. collections is used later when assign to target net
101 | with tf.variable_scope('l1'):
102 | w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
103 | b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
104 | l1 = tf.nn.relu(tf.matmul(self.s_, w1) + b1)
105 |
106 | # second layer. collections is used later when assign to target net
107 | with tf.variable_scope('l2'):
108 | w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
109 | b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
110 | self.q_next = tf.matmul(l1, w2) + b2
111 |
112 | def store_transition(self, s, a, r, s_):
113 | if not hasattr(self, 'memory_counter'):
114 | self.memory_counter = 0
115 |
116 | transition = np.hstack((s, [a, r], s_))
117 |
118 | # replace the old memory with new memory
119 | index = self.memory_counter % self.memory_size
120 | self.memory[index, :] = transition
121 |
122 | self.memory_counter += 1
123 |
124 | def choose_action(self, observation):
125 | # to have batch dimension when feed into tf placeholder
126 | observation = observation[np.newaxis, :]
127 |
128 | if np.random.uniform() < self.epsilon:
129 | # forward feed the observation and get q value for every actions
130 | actions_value = self.sess.run(self.q_eval, feed_dict={self.s: observation})
131 | action = np.argmax(actions_value)
132 | else:
133 | action = np.random.randint(0, self.n_actions)
134 | return action
135 |
136 | def learn(self):
137 | # check to replace target parameters
138 | if self.learn_step_counter % self.replace_target_iter == 0:
139 | self.sess.run(self.replace_target_op)
140 | print('\ntarget_params_replaced\n')
141 |
142 | # sample batch memory from all memory
143 | if self.memory_counter > self.memory_size:
144 | sample_index = np.random.choice(self.memory_size, size=self.batch_size)
145 | else:
146 | sample_index = np.random.choice(self.memory_counter, size=self.batch_size)
147 | batch_memory = self.memory[sample_index, :]
148 |
149 | q_next, q_eval = self.sess.run(
150 | [self.q_next, self.q_eval],
151 | feed_dict={
152 | self.s_: batch_memory[:, -self.n_features:], # fixed params
153 | self.s: batch_memory[:, :self.n_features], # newest params
154 | })
155 |
156 | # change q_target w.r.t q_eval's action
157 | q_target = q_eval.copy()
158 |
159 | batch_index = np.arange(self.batch_size, dtype=np.int32)
160 | eval_act_index = batch_memory[:, self.n_features].astype(int)
161 | reward = batch_memory[:, self.n_features + 1]
162 |
163 | q_target[batch_index, eval_act_index] = reward + self.gamma * np.max(q_next, axis=1)
164 |
165 | """
166 | For example in this batch I have 2 samples and 3 actions:
167 | q_eval =
168 | [[1, 2, 3],
169 | [4, 5, 6]]
170 | q_target = q_eval =
171 | [[1, 2, 3],
172 | [4, 5, 6]]
173 | Then change q_target with the real q_target value w.r.t the q_eval's action.
174 | For example in:
175 | sample 0, I took action 0, and the max q_target value is -1;
176 | sample 1, I took action 2, and the max q_target value is -2:
177 | q_target =
178 | [[-1, 2, 3],
179 | [4, 5, -2]]
180 | So the (q_target - q_eval) becomes:
181 | [[(-1)-(1), 0, 0],
182 | [0, 0, (-2)-(6)]]
183 | We then backpropagate this error w.r.t the corresponding action to network,
184 | leave other action as error=0 cause we didn't choose it.
185 | """
186 |
187 | # train eval network
188 |
189 | _, self.cost = self.sess.run([self._train_op, self.loss],
190 | feed_dict={self.s: batch_memory[:, :self.n_features],
191 | self.q_target: q_target})
192 | self.cost_his.append(self.cost)
193 |
194 | # increasing epsilon
195 | self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
196 | self.learn_step_counter += 1
197 |
198 | def plot_cost(self):
199 | import matplotlib.pyplot as plt
200 | plt.plot(np.arange(len(self.cost_his)), self.cost_his)
201 | plt.ylabel('Cost')
202 | plt.xlabel('training steps')
203 | plt.show()
--------------------------------------------------------------------------------
/DQLearing/Brains/__pycache__/DQN.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/Brains/__pycache__/DQN.cpython-36.pyc
--------------------------------------------------------------------------------
/DQLearing/Brains/__pycache__/DQN.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/Brains/__pycache__/DQN.cpython-37.pyc
--------------------------------------------------------------------------------
/DQLearing/Brains/__pycache__/DQNTF2.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/Brains/__pycache__/DQNTF2.cpython-36.pyc
--------------------------------------------------------------------------------
/DQLearing/Brains/__pycache__/DQNTF2.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/Brains/__pycache__/DQNTF2.cpython-37.pyc
--------------------------------------------------------------------------------
/DQLearing/Brains/__pycache__/DuelingDQN.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/Brains/__pycache__/DuelingDQN.cpython-36.pyc
--------------------------------------------------------------------------------
/DQLearing/gym/CartPole.py:
--------------------------------------------------------------------------------
1 | """
2 | Deep Q network,
3 | Using:
4 | Tensorflow: 1.0
5 | gym: 0.7.3
6 | """
7 |
8 |
9 | import gym
10 | from DQLearing.Brains.DQN import DeepQNet
11 | from DQLearing.Brains.DQNTF2 import DeepQN
12 | from DQLearing.Brains.DuelingDQN import DuelingDQN
13 |
14 | env = gym.make('CartPole-v0')
15 | env = env.unwrapped
16 |
17 | print(env.action_space)
18 | print(env.observation_space)
19 | print(env.observation_space.high)
20 | print(env.observation_space.low)
21 |
22 |
23 | RL = DuelingDQN(n_actions=env.action_space.n,
24 | n_features=env.observation_space.shape[0],
25 | learning_rate=0.01,
26 | e_greedy=0.9,
27 | replace_target_iter=100,
28 | memory_size=2000,
29 | e_greedy_increment=0.001,)
30 |
31 | total_steps = 0
32 |
33 |
34 | for i_episode in range(80):
35 |
36 | observation = env.reset()
37 | ep_r = 0
38 | while True:
39 | env.render()
40 |
41 | action = RL.choose_action(observation)
42 |
43 | observation_, reward, done, info = env.step(action)
44 |
45 | # the smaller theta and closer to center the better
46 | x, x_dot, theta, theta_dot = observation_
47 | r1 = (env.x_threshold - abs(x))/env.x_threshold - 0.8
48 | r2 = (env.theta_threshold_radians - abs(theta))/env.theta_threshold_radians - 0.5
49 | reward = r1 + r2
50 |
51 | RL.store_transition(observation, action, reward, observation_)
52 |
53 | ep_r += reward
54 | if total_steps > 1000:
55 | RL.learn()
56 |
57 | if done:
58 | print('episode: ', i_episode,
59 | 'ep_r: ', round(ep_r, 2),
60 | ' epsilon: ', round(RL.epsilon, 2))
61 | break
62 |
63 | observation = observation_
64 | total_steps += 1
65 |
66 | RL.plot_cost()
--------------------------------------------------------------------------------
/DQLearing/gym/Mountain_Car.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/gym/Mountain_Car.py
--------------------------------------------------------------------------------
/DQLearing/gym/Pendulum.py:
--------------------------------------------------------------------------------
1 | """
2 | Dueling DQN & Natural DQN comparison
3 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
4 | Using:
5 | Tensorflow: 1.0
6 | gym: 0.8.0
7 | """
8 |
9 |
10 | import gym
11 | from DQLearing.Brains.DuelingDQN import DuelingDQN
12 | import numpy as np
13 | import matplotlib.pyplot as plt
14 | import tensorflow as tf
15 |
16 |
17 | env = gym.make('Pendulum-v0')
18 | env = env.unwrapped
19 | env.seed(1)
20 | MEMORY_SIZE = 3000
21 | ACTION_SPACE = 25
22 |
23 | sess = tf.Session()
24 | with tf.variable_scope('natural'):
25 | natural_DQN = DuelingDQN(
26 | n_actions=ACTION_SPACE, n_features=3, memory_size=MEMORY_SIZE,
27 | e_greedy_increment=0.001, sess=sess, dueling=False)
28 |
29 | with tf.variable_scope('dueling'):
30 | dueling_DQN = DuelingDQN(
31 | n_actions=ACTION_SPACE, n_features=3, memory_size=MEMORY_SIZE,
32 | e_greedy_increment=0.001, sess=sess, dueling=True, output_graph=True)
33 |
34 | sess.run(tf.global_variables_initializer())
35 |
36 |
37 | def train(RL):
38 | acc_r = [0]
39 | total_steps = 0
40 | observation = env.reset()
41 | while True:
42 | # if total_steps-MEMORY_SIZE > 9000: env.render()
43 |
44 | action = RL.choose_action(observation)
45 |
46 | f_action = (action-(ACTION_SPACE-1)/2)/((ACTION_SPACE-1)/4) # [-2 ~ 2] float actions
47 | observation_, reward, done, info = env.step(np.array([f_action]))
48 |
49 | reward /= 10 # normalize to a range of (-1, 0)
50 | acc_r.append(reward + acc_r[-1]) # accumulated reward
51 |
52 | RL.store_transition(observation, action, reward, observation_)
53 |
54 | if total_steps > MEMORY_SIZE:
55 | RL.learn()
56 |
57 | if total_steps-MEMORY_SIZE > 15000:
58 | break
59 |
60 | observation = observation_
61 | total_steps += 1
62 | return RL.cost_his, acc_r
63 |
64 | c_natural, r_natural = train(natural_DQN)
65 | c_dueling, r_dueling = train(dueling_DQN)
66 |
67 | plt.figure(1)
68 | plt.plot(np.array(c_natural), c='r', label='natural')
69 | plt.plot(np.array(c_dueling), c='b', label='dueling')
70 | plt.legend(loc='best')
71 | plt.ylabel('cost')
72 | plt.xlabel('training steps')
73 | plt.grid()
74 |
75 | plt.figure(2)
76 | plt.plot(np.array(r_natural), c='r', label='natural')
77 | plt.plot(np.array(r_dueling), c='b', label='dueling')
78 | plt.legend(loc='best')
79 | plt.ylabel('accumulated reward')
80 | plt.xlabel('training steps')
81 | plt.grid()
82 |
83 | plt.show()
84 | """
85 | Dueling DQN & Natural DQN comparison
86 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
87 | Using:
88 | Tensorflow: 1.0
89 | gym: 0.8.0
90 | """
91 |
92 |
93 | import gym
94 | from DQLearing.Brains.DuelingDQN import DuelingDQN
95 | import numpy as np
96 | import matplotlib.pyplot as plt
97 | import tensorflow as tf
98 |
99 |
100 | env = gym.make('Pendulum-v0')
101 | env = env.unwrapped
102 | env.seed(1)
103 | MEMORY_SIZE = 3000
104 | ACTION_SPACE = 25
105 |
106 | sess = tf.Session()
107 | with tf.variable_scope('natural'):
108 | natural_DQN = DuelingDQN(
109 | n_actions=ACTION_SPACE, n_features=3, memory_size=MEMORY_SIZE,
110 | e_greedy_increment=0.001, sess=sess, dueling=False)
111 |
112 | with tf.variable_scope('dueling'):
113 | dueling_DQN = DuelingDQN(
114 | n_actions=ACTION_SPACE, n_features=3, memory_size=MEMORY_SIZE,
115 | e_greedy_increment=0.001, sess=sess, dueling=True, output_graph=True)
116 |
117 | sess.run(tf.global_variables_initializer())
118 |
119 |
120 | def train(RL):
121 | acc_r = [0]
122 | total_steps = 0
123 | observation = env.reset()
124 | while True:
125 | # if total_steps-MEMORY_SIZE > 9000: env.render()
126 |
127 | action = RL.choose_action(observation)
128 |
129 | f_action = (action-(ACTION_SPACE-1)/2)/((ACTION_SPACE-1)/4) # [-2 ~ 2] float actions
130 | observation_, reward, done, info = env.step(np.array([f_action]))
131 |
132 | reward /= 10 # normalize to a range of (-1, 0)
133 | acc_r.append(reward + acc_r[-1]) # accumulated reward
134 |
135 | RL.store_transition(observation, action, reward, observation_)
136 |
137 | if total_steps > MEMORY_SIZE:
138 | RL.learn()
139 |
140 | if total_steps-MEMORY_SIZE > 15000:
141 | break
142 |
143 | observation = observation_
144 | total_steps += 1
145 | return RL.cost_his, acc_r
146 |
147 | c_natural, r_natural = train(natural_DQN)
148 | c_dueling, r_dueling = train(dueling_DQN)
149 |
150 | plt.figure(1)
151 | plt.plot(np.array(c_natural), c='r', label='natural')
152 | plt.plot(np.array(c_dueling), c='b', label='dueling')
153 | plt.legend(loc='best')
154 | plt.ylabel('cost')
155 | plt.xlabel('training steps')
156 | plt.grid()
157 |
158 | plt.figure(2)
159 | plt.plot(np.array(r_natural), c='r', label='natural')
160 | plt.plot(np.array(r_dueling), c='b', label='dueling')
161 | plt.legend(loc='best')
162 | plt.ylabel('accumulated reward')
163 | plt.xlabel('training steps')
164 | plt.grid()
165 |
166 | plt.show()
--------------------------------------------------------------------------------
/DQLearing/gym/checkpoint:
--------------------------------------------------------------------------------
1 | model_checkpoint_path: "tmp_model"
2 | all_model_checkpoint_paths: "tmp_model"
3 |
--------------------------------------------------------------------------------
/DQLearing/gym/logs/events.out.tfevents.1579495704.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/gym/logs/events.out.tfevents.1579495704.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/gym/tmp_model.data-00000-of-00001:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/gym/tmp_model.data-00000-of-00001
--------------------------------------------------------------------------------
/DQLearing/gym/tmp_model.index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/gym/tmp_model.index
--------------------------------------------------------------------------------
/DQLearing/logs/events.out.tfevents.1578211690.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/logs/events.out.tfevents.1578211690.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/logs/events.out.tfevents.1578225153.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/logs/events.out.tfevents.1578225153.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/logs/events.out.tfevents.1578226418.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/logs/events.out.tfevents.1578226418.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/logs/events.out.tfevents.1578226429.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/logs/events.out.tfevents.1578226429.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/logs/events.out.tfevents.1578228234.DESKTOP-JBV63R4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/belowthetree/ReinforceLearning/f7912fff6fa75bda1649bab0c9743db39e0f44f6/DQLearing/logs/events.out.tfevents.1578228234.DESKTOP-JBV63R4
--------------------------------------------------------------------------------
/DQLearing/run_dqn.py:
--------------------------------------------------------------------------------
1 | from DQLearing.tkinter.maze_env import Maze
2 | from DQLearing.Brains.DQN import DeepQNet
3 |
4 |
5 | def run_maze():
6 | step = 0
7 | for episode in range(300):
8 | # initial observation
9 | observation = env.reset()
10 |
11 | while True:
12 | # fresh env
13 | env.render()
14 |
15 | # RL choose action based on observation
16 | action = RL.choose_action(observation)
17 |
18 | # RL take action and get next observation and reward
19 | observation_, reward, done = env.step(action)
20 |
21 | RL.store_transition(observation, action, reward, observation_)
22 |
23 | if (step > 200) and (step % 5 == 0):
24 | RL.learn()
25 |
26 | # swap observation
27 | observation = observation_
28 |
29 | # break while loop when end of this episode
30 | if done:
31 | break
32 | step += 1
33 |
34 | # end of game
35 | print('game over')
36 | env.destroy()
37 |
38 |
39 | if __name__ == "__main__":
40 | # maze game
41 | env = Maze()
42 | # RL = DeepQNetwork(env.n_actions, env.n_features,
43 | # learning_rate=0.01,
44 | # reward_decay=0.9,
45 | # e_greedy=0.9,
46 | # replace_target_iter=200,
47 | # memory_size=2000,
48 | # # output_graph=True
49 | # )
50 | RL = DeepQNet(env.n_actions, env.n_features,
51 | learning_rate=0.001,
52 | reward_decay=0.9,
53 | e_greedy=0.9,
54 | replace_target_iter=200,
55 | memory_size=2000
56 | )
57 |
58 | env.after(100, run_maze)
59 | env.mainloop()
60 | RL.plot_cost()
--------------------------------------------------------------------------------
/DQLearing/tkinter/chessboard.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import time
4 | import tkinter as tk
5 |
6 |
7 | def key(event):
8 | print("pressed", repr(event.char))
9 |
10 |
11 | class ChessBoard(tk.Tk, object):
12 | def __init__(self, size, scale=35, train=True, win_cnt=5):
13 | super(ChessBoard, self).__init__()
14 | self.scale = scale
15 | if not train:
16 | self.next = False
17 | self.train = train
18 | self.win_cnt = win_cnt
19 | self.full = size * size
20 | self.cnt = 0
21 | self.winner = 0
22 | self.size = size * self.scale
23 | self.map = np.zeros((size + 1, size + 1), dtype=np.int16)
24 | self.objects = []
25 | self.canvas = tk.Canvas(self, bg='grey', height=self.size + self.scale,
26 | width=self.size + self.scale)
27 | self.create_window()
28 |
29 | def create_window(self):
30 | self.title('五子棋')
31 | self.geometry(str(self.size + self.scale) + 'x' + str(self.size + self.scale))
32 | self.canvas.pack()
33 | for i in range(int(self.size / self.scale + 1)):
34 | self.canvas.create_line(self.scale, i * self.scale, self.size, i * self.scale)
35 | self.canvas.create_line(i * self.scale, self.scale, i * self.scale, self.size)
36 | if not self.train:
37 | self.canvas.bind("", key)
38 | self.canvas.bind("", self.callback)
39 | self.canvas.pack()
40 |
41 | def callback(self, event):
42 | x = int((event.x - self.scale / 2) / self.scale)
43 | y = int((event.y - self.scale / 2) / self.scale)
44 | print("clicked at", x, y)
45 | if x < 0 or x > self.size / self.scale or y < 0 or y > self.size / self.scale:
46 | return
47 | self.map[x][y] = 2
48 | self.render()
49 | self.game_check()
50 | print(self.winner)
51 | self.next = True
52 |
53 | def render(self):
54 | half = self.scale / 2
55 | for i in range(len(self.map)):
56 | for j in range(len(self.map)):
57 | if self.map[i][j] == 1:
58 | self.objects.append(self.canvas.create_oval(self.scale + i * self.scale - half,
59 | self.scale + j * self.scale - half,
60 | self.scale + i * self.scale + half,
61 | self.scale + j * self.scale + half, fill='black'))
62 | elif self.map[i][j] == 2:
63 | self.objects.append(self.canvas.create_oval(self.scale + i * self.scale - half,
64 | self.scale + j * self.scale - half,
65 | self.scale + i * self.scale + half,
66 | self.scale + j * self.scale + half, fill='white'))
67 | self.update()
68 | # time.sleep(0.5)
69 |
70 | def step(self, role, index):
71 | reward = -1
72 | state = self.convert()
73 | if index.max() > self.size or index.min() < 0 or self.map[index[0], index[1]] != 0:
74 | # print('index error, x: {}, y: {}'.format(index[0], index[1]))
75 | return state, reward, False
76 | self.map[index[0]][index[1]] = role
77 | self.game_check()
78 | if self.winner != 0:
79 | print(self.winner)
80 | if self.winner == 0:
81 | reward = 0
82 | elif self.winner != role:
83 | reward = -1
84 | else:
85 | reward = 1
86 | self.cnt += 1
87 | if self.cnt >= self.full:
88 | reward = -1
89 | return state, reward, True
90 |
91 | def reset(self):
92 | size = len(self.map)
93 | for i in range(size):
94 | for j in range(size):
95 | self.map[i][j] = 0
96 | for i in range(len(self.objects)):
97 | self.canvas.delete(self.objects[i])
98 | self.update()
99 | self.winner = 0
100 | self.cnt = 0
101 |
102 | return self.convert()
103 |
104 | def game_check(self):
105 | if self.count(1):
106 | self.winner = 1
107 | elif self.count(2):
108 | self.winner = 2
109 |
110 | def count(self, role):
111 | size = int(self.size / self.scale)
112 | for i in range(size):
113 | col_cnt = row_cnt = 0
114 | for j in range(size):
115 | if self.map[i][j] == role:
116 | col_cnt += 1
117 | else:
118 | col_cnt = 0
119 | if self.map[j][i] == role:
120 | row_cnt += 1
121 | else:
122 | row_cnt = 0
123 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
124 | return True
125 |
126 | for i in range(size):
127 | col_cnt = row_cnt = 0
128 | for j in range(size - i):
129 | if self.map[i + j][j] == role:
130 | row_cnt += 1
131 | else:
132 | row_cnt = 0
133 | if self.map[size - i - 1 - j][size - j - 1] == role:
134 | col_cnt += 1
135 | else:
136 | col_cnt = 0
137 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
138 | return True
139 | col_cnt = row_cnt = 0
140 | for j in range(i, size):
141 | if self.map[j - i][j] == role:
142 | row_cnt += 1
143 | else:
144 | row_cnt = 0
145 | if self.map[size - 1 - j + i][j] == role:
146 | col_cnt += 1
147 | else:
148 | col_cnt = 0
149 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
150 | return True
151 | return False
152 |
153 | def convert(self):
154 | res = ''
155 | size = len(self.map)
156 | for i in range(size):
157 | for j in range(size):
158 | res += str(self.map[i][j])
159 |
160 | return res
161 |
162 | def get_map(self):
163 | return self.map
164 |
--------------------------------------------------------------------------------
/DQLearing/tkinter/maze_env.py:
--------------------------------------------------------------------------------
1 | """
2 | Reinforcement learning maze example.
3 | Red rectangle: explorer.
4 | Black rectangles: hells [reward = -1].
5 | Yellow bin circle: paradise [reward = +1].
6 | All other states: ground [reward = 0].
7 | This script is the environment part of this example.
8 | The RL is in RL_brain.py.
9 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
10 | """
11 | import numpy as np
12 | import time
13 | import sys
14 | if sys.version_info.major == 2:
15 | import Tkinter as tk
16 | else:
17 | import tkinter as tk
18 |
19 | UNIT = 40 # pixels
20 | MAZE_H = 4 # grid height
21 | MAZE_W = 4 # grid width
22 |
23 |
24 | class Maze(tk.Tk, object):
25 | def __init__(self):
26 | super(Maze, self).__init__()
27 | self.action_space = ['u', 'd', 'l', 'r']
28 | self.n_actions = len(self.action_space)
29 | self.n_features = 2
30 | self.title('maze')
31 | self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))
32 | self._build_maze()
33 |
34 | def _build_maze(self):
35 | self.canvas = tk.Canvas(self, bg='white',
36 | height=MAZE_H * UNIT,
37 | width=MAZE_W * UNIT)
38 |
39 | # create grids
40 | for c in range(0, MAZE_W * UNIT, UNIT):
41 | x0, y0, x1, y1 = c, 0, c, MAZE_H * UNIT
42 | self.canvas.create_line(x0, y0, x1, y1)
43 | for r in range(0, MAZE_H * UNIT, UNIT):
44 | x0, y0, x1, y1 = 0, r, MAZE_W * UNIT, r
45 | self.canvas.create_line(x0, y0, x1, y1)
46 |
47 | # create origin
48 | origin = np.array([20, 20])
49 |
50 | # hell
51 | hell1_center = origin + np.array([UNIT * 2, UNIT])
52 | self.hell1 = self.canvas.create_rectangle(
53 | hell1_center[0] - 15, hell1_center[1] - 15,
54 | hell1_center[0] + 15, hell1_center[1] + 15,
55 | fill='black')
56 | # hell
57 | # hell2_center = origin + np.array([UNIT, UNIT * 2])
58 | # self.hell2 = self.canvas.create_rectangle(
59 | # hell2_center[0] - 15, hell2_center[1] - 15,
60 | # hell2_center[0] + 15, hell2_center[1] + 15,
61 | # fill='black')
62 |
63 | # create oval
64 | oval_center = origin + UNIT * 2
65 | self.oval = self.canvas.create_oval(
66 | oval_center[0] - 15, oval_center[1] - 15,
67 | oval_center[0] + 15, oval_center[1] + 15,
68 | fill='yellow')
69 |
70 | # create red rect
71 | self.rect = self.canvas.create_rectangle(
72 | origin[0] - 15, origin[1] - 15,
73 | origin[0] + 15, origin[1] + 15,
74 | fill='red')
75 |
76 | # pack all
77 | self.canvas.pack()
78 |
79 | def reset(self):
80 | self.update()
81 | time.sleep(0.1)
82 | self.canvas.delete(self.rect)
83 | origin = np.array([20, 20])
84 | self.rect = self.canvas.create_rectangle(
85 | origin[0] - 15, origin[1] - 15,
86 | origin[0] + 15, origin[1] + 15,
87 | fill='red')
88 | # return observation
89 | return (np.array(self.canvas.coords(self.rect)[:2]) - np.array(self.canvas.coords(self.oval)[:2]))/(MAZE_H*UNIT)
90 |
91 | def step(self, action):
92 | s = self.canvas.coords(self.rect)
93 | base_action = np.array([0, 0])
94 | if action == 0: # up
95 | if s[1] > UNIT:
96 | base_action[1] -= UNIT
97 | elif action == 1: # down
98 | if s[1] < (MAZE_H - 1) * UNIT:
99 | base_action[1] += UNIT
100 | elif action == 2: # right
101 | if s[0] < (MAZE_W - 1) * UNIT:
102 | base_action[0] += UNIT
103 | elif action == 3: # left
104 | if s[0] > UNIT:
105 | base_action[0] -= UNIT
106 |
107 | self.canvas.move(self.rect, base_action[0], base_action[1]) # move agent
108 |
109 | next_coords = self.canvas.coords(self.rect) # next state
110 |
111 | # reward function
112 | if next_coords == self.canvas.coords(self.oval):
113 | reward = 1
114 | done = True
115 | elif next_coords in [self.canvas.coords(self.hell1)]:
116 | reward = -1
117 | done = True
118 | else:
119 | reward = 0
120 | done = False
121 | s_ = (np.array(next_coords[:2]) - np.array(self.canvas.coords(self.oval)[:2]))/(MAZE_H*UNIT)
122 | return s_, reward, done
123 |
124 | def render(self):
125 | # time.sleep(0.01)
126 | self.update()
--------------------------------------------------------------------------------
/PolicyGradient/Brain/PG.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 |
5 |
6 | class PolicyGradientNet(nn.Module):
7 | def __init__(self, n_actions, n_features, n_hiddens):
8 | super(PolicyGradientNet, self).__init__()
9 | self.fc1 = nn.Sequential(
10 | nn.Linear(n_features, n_hiddens),
11 | nn.Tanh()
12 | )
13 | self.fc2 = nn.Sequential(
14 | nn.Linear(n_hiddens, n_actions),
15 | nn.Softmax()
16 | )
17 |
18 | def forward(self, x):
19 | x = self.fc1(x)
20 | x = self.fc2(x)
21 |
22 | return x
23 |
24 |
25 | class PolicyGradient():
26 | def __init__(
27 | self,
28 | n_actions,
29 | n_features,
30 | learning_rate=0.01,
31 | reward_decay=0.95,
32 | output_graph=False,
33 | ):
34 | self.n_actions = n_actions
35 | self.n_features = n_features
36 | self.lr = learning_rate
37 | self.gamma = reward_decay
38 |
39 | self.obs = []
40 | self.acs = []
41 | self.rws = []
42 |
43 | self.net = PolicyGradientNet(n_actions, n_features, 10)
44 | self.loss = nn.CrossEntropyLoss()
45 | self.optimizer = torch.optim.Adam(self.net.parameters(), lr=learning_rate)
46 |
47 | def choose_action(self, observation):
48 | self.net.eval()
49 | actions = self.net(torch.Tensor(observation[np.newaxis, :]))
50 | action = np.random.choice(range(actions.shape[1]), p=actions.view(-1).detach().numpy())
51 | return action
52 |
53 | def store_transition(self, s, a, r):
54 | self.obs.append(s)
55 | self.acs.append(a)
56 | self.rws.append(r)
57 |
58 | def learn(self):
59 | self.net.train()
60 | discount = self._discount_and_norm_rewards()
61 | output = self.net(torch.Tensor(self.obs))
62 | one_hot = torch.zeros(len(self.acs), self.n_actions).\
63 | scatter_(1, torch.LongTensor(self.acs).view(-1,1), 1)
64 | neg = torch.sum(-torch.log(output) * one_hot, 1)
65 | loss = neg * torch.Tensor(discount)
66 | loss = loss.mean()
67 | self.optimizer.zero_grad()
68 | loss.backward()
69 | self.optimizer.step()
70 |
71 | self.acs = []
72 | self.obs = []
73 | self.rws = []
74 | return discount
75 |
76 | def _discount_and_norm_rewards(self):
77 | discount = np.zeros_like(self.rws)
78 | tmp = 0
79 | for i in reversed(range(len(self.rws))):
80 | tmp = tmp * self.gamma + self.rws[i]
81 | discount[i] = tmp
82 |
83 | discount -= np.mean(discount)
84 | discount /= np.std(discount)
85 |
86 | return discount
87 |
--------------------------------------------------------------------------------
/PolicyGradient/Brain/PolicyGradient.py:
--------------------------------------------------------------------------------
1 | """
2 | This part of code is the reinforcement learning brain, which is a brain of the agent.
3 | All decisions are made in here.
4 | Policy Gradient, Reinforcement Learning.
5 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
6 | Using:
7 | Tensorflow: 1.0
8 | gym: 0.8.0
9 | """
10 |
11 | import numpy as np
12 | import tensorflow as tf
13 |
14 | # reproducible
15 | np.random.seed(1)
16 | tf.set_random_seed(1)
17 |
18 |
19 | class PolicyGradient:
20 | def __init__(
21 | self,
22 | n_actions,
23 | n_features,
24 | learning_rate=0.01,
25 | reward_decay=0.95,
26 | output_graph=False,
27 | ):
28 | self.n_actions = n_actions
29 | self.n_features = n_features
30 | self.lr = learning_rate
31 | self.gamma = reward_decay
32 |
33 | self.ep_obs, self.ep_as, self.ep_rs = [], [], []
34 |
35 | self._build_net()
36 |
37 | self.sess = tf.Session()
38 |
39 | if output_graph:
40 | tf.summary.FileWriter("logs/", self.sess.graph)
41 |
42 | self.sess.run(tf.global_variables_initializer())
43 |
44 | def _build_net(self):
45 | with tf.name_scope('inputs'):
46 | self.tf_obs = tf.placeholder(tf.float32, [None, self.n_features], name="observations")
47 | self.tf_acts = tf.placeholder(tf.int32, [None, ], name="actions_num")
48 | self.tf_vt = tf.placeholder(tf.float32, [None, ], name="actions_value")
49 | # fc1
50 | layer = tf.layers.dense(
51 | inputs=self.tf_obs,
52 | units=10,
53 | activation=tf.nn.tanh, # tanh activation
54 | name='fc1'
55 | )
56 | # fc2
57 | all_act = tf.layers.dense(
58 | inputs=layer,
59 | units=self.n_actions,
60 | activation=None,
61 | name='fc2'
62 | )
63 |
64 | self.all_act_prob = tf.nn.softmax(all_act, name='act_prob') # use softmax to convert to probability
65 |
66 | with tf.name_scope('loss'):
67 | # to maximize total reward (log_p * R) is to minimize -(log_p * R), and the tf only have minimize(loss)
68 | neg_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=all_act, labels=self.tf_acts) # this is negative log of chosen action
69 | # or in this way:
70 | # neg_log_prob = tf.reduce_sum(-tf.log(self.all_act_prob)*tf.one_hot(self.tf_acts, self.n_actions), axis=1)
71 | loss = tf.reduce_mean(neg_log_prob * self.tf_vt) # reward guided loss
72 |
73 | with tf.name_scope('train'):
74 | self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)
75 |
76 | def choose_action(self, observation):
77 | prob_weights = self.sess.run(self.all_act_prob, feed_dict={self.tf_obs: observation[np.newaxis, :]})
78 | action = np.random.choice(range(prob_weights.shape[1]), p=prob_weights.ravel()) # select action w.r.t the actions prob
79 | return action
80 |
81 | def store_transition(self, s, a, r):
82 | self.ep_obs.append(s)
83 | self.ep_as.append(a)
84 | self.ep_rs.append(r)
85 |
86 | def learn(self):
87 | # discount and normalize episode reward
88 | discounted_ep_rs_norm = self._discount_and_norm_rewards()
89 |
90 | # train on episode
91 | self.sess.run(self.train_op, feed_dict={
92 | self.tf_obs: np.vstack(self.ep_obs), # shape=[None, n_obs]
93 | self.tf_acts: np.array(self.ep_as), # shape=[None, ]
94 | self.tf_vt: discounted_ep_rs_norm, # shape=[None, ]
95 | })
96 |
97 | self.ep_obs, self.ep_as, self.ep_rs = [], [], [] # empty episode data
98 | return discounted_ep_rs_norm
99 |
100 | def _discount_and_norm_rewards(self):
101 | # discount episode rewards
102 | discounted_ep_rs = np.zeros_like(self.ep_rs)
103 | running_add = 0
104 | for t in reversed(range(0, len(self.ep_rs))):
105 | running_add = running_add * self.gamma + self.ep_rs[t]
106 | discounted_ep_rs[t] = running_add
107 |
108 | # normalize episode rewards
109 | discounted_ep_rs -= np.mean(discounted_ep_rs)
110 | discounted_ep_rs /= np.std(discounted_ep_rs)
111 | return discounted_ep_rs
--------------------------------------------------------------------------------
/PolicyGradient/gym/PGCartPole.py:
--------------------------------------------------------------------------------
1 | """
2 | Policy Gradient, Reinforcement Learning.
3 | The cart pole example
4 | View more on my tutorial page: https://morvanzhou.github.io/tutorials/
5 | Using:
6 | Tensorflow: 1.0
7 | gym: 0.8.0
8 | """
9 |
10 | import gym
11 | from PolicyGradient.Brain.PG import PolicyGradient
12 | # from DQLearing.Brains.PolicyGradient import PolicyGradient
13 | import matplotlib.pyplot as plt
14 |
15 | DISPLAY_REWARD_THRESHOLD = 400 # renders environment if total episode reward is greater then this threshold
16 | RENDER = False # rendering wastes time
17 |
18 | env = gym.make('CartPole-v0')
19 | env.seed(1) # reproducible, general Policy gradient has high variance
20 | env = env.unwrapped
21 |
22 | print(env.action_space)
23 | print(env.observation_space)
24 | print(env.observation_space.high)
25 | print(env.observation_space.low)
26 |
27 | RL = PolicyGradient(
28 | n_actions=env.action_space.n,
29 | n_features=env.observation_space.shape[0],
30 | learning_rate=0.02,
31 | reward_decay=0.99,
32 | # output_graph=True,
33 | )
34 |
35 | for i_episode in range(3000):
36 |
37 | observation = env.reset()
38 |
39 | while True:
40 | if RENDER: env.render()
41 |
42 | action = RL.choose_action(observation)
43 |
44 | observation_, reward, done, info = env.step(action)
45 |
46 | RL.store_transition(observation, action, reward)
47 |
48 | if done:
49 | ep_rs_sum = sum(RL.rws)
50 |
51 | if 'running_reward' not in globals():
52 | running_reward = ep_rs_sum
53 | else:
54 | running_reward = running_reward * 0.99 + ep_rs_sum * 0.01
55 | if running_reward > DISPLAY_REWARD_THRESHOLD: RENDER = True # rendering
56 | print("episode:", i_episode, " reward:", int(running_reward))
57 |
58 | vt = RL.learn()
59 |
60 | if i_episode == 0:
61 | plt.plot(vt) # plot the episode vt
62 | plt.xlabel('episode steps')
63 | plt.ylabel('normalized state-action value')
64 | plt.show()
65 | break
66 |
67 | observation = observation_
--------------------------------------------------------------------------------
/QTable/五子棋/chessboard.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import time
4 | import tkinter as tk
5 |
6 |
7 | def key(event):
8 | print("pressed", repr(event.char))
9 |
10 |
11 | class ChessBoard(tk.Tk, object):
12 | def __init__(self, size, scale=35, train=True, win_cnt=5):
13 | super(ChessBoard, self).__init__()
14 | self.scale = scale
15 | if not train:
16 | self.next = False
17 | self.train = train
18 | self.win_cnt = win_cnt
19 | self.full = size * size
20 | self.cnt = 0
21 | self.winner = 0
22 | self.size = size * self.scale
23 | self.map = np.zeros((size + 1, size + 1), dtype=np.int16)
24 | self.objects = []
25 | self.canvas = tk.Canvas(self, bg='grey', height=self.size + self.scale,
26 | width=self.size + self.scale)
27 | self.create_window()
28 |
29 | def create_window(self):
30 | self.title('五子棋')
31 | self.geometry(str(self.size + self.scale) + 'x' + str(self.size + self.scale))
32 | self.canvas.pack()
33 | for i in range(int(self.size / self.scale + 1)):
34 | self.canvas.create_line(self.scale, i * self.scale, self.size, i * self.scale)
35 | self.canvas.create_line(i * self.scale, self.scale, i * self.scale, self.size)
36 | if not self.train:
37 | self.canvas.bind("", key)
38 | self.canvas.bind("", self.callback)
39 | self.canvas.pack()
40 |
41 | def callback(self, event):
42 | x = int((event.x - self.scale / 2) / self.scale)
43 | y = int((event.y - self.scale / 2) / self.scale)
44 | print("clicked at", x, y)
45 | if x < 0 or x > self.size / self.scale or y < 0 or y > self.size / self.scale:
46 | return
47 | self.map[x][y] = 2
48 | self.render()
49 | self.game_check()
50 | print(self.winner)
51 | self.next = True
52 |
53 | def render(self):
54 | half = self.scale / 2
55 | for i in range(len(self.map)):
56 | for j in range(len(self.map)):
57 | if self.map[i][j] == 1:
58 | self.objects.append(self.canvas.create_oval(self.scale + i * self.scale - half,
59 | self.scale + j * self.scale - half,
60 | self.scale + i * self.scale + half,
61 | self.scale + j * self.scale + half, fill='black'))
62 | elif self.map[i][j] == 2:
63 | self.objects.append(self.canvas.create_oval(self.scale + i * self.scale - half,
64 | self.scale + j * self.scale - half,
65 | self.scale + i * self.scale + half,
66 | self.scale + j * self.scale + half, fill='white'))
67 | self.update()
68 | # time.sleep(0.5)
69 |
70 | def step(self, role, index):
71 | reward = -1
72 | state = self.convert()
73 | if index.max() > self.size or index.min() < 0 or self.map[index[0], index[1]] != 0:
74 | # print('index error, x: {}, y: {}'.format(index[0], index[1]))
75 | return state, reward, False
76 | self.map[index[0]][index[1]] = role
77 | self.game_check()
78 | if self.winner != 0:
79 | print(self.winner)
80 | if self.winner == 0:
81 | reward = 0
82 | elif self.winner != role:
83 | reward = -1
84 | else:
85 | reward = 1
86 | self.cnt += 1
87 | if self.cnt >= self.full:
88 | reward = -1
89 | return state, reward, True
90 |
91 | def reset(self):
92 | size = len(self.map)
93 | for i in range(size):
94 | for j in range(size):
95 | self.map[i][j] = 0
96 | for i in range(len(self.objects)):
97 | self.canvas.delete(self.objects[i])
98 | self.update()
99 | self.winner = 0
100 | self.cnt = 0
101 |
102 | return self.convert()
103 |
104 | def game_check(self):
105 | if self.count(1):
106 | self.winner = 1
107 | elif self.count(2):
108 | self.winner = 2
109 |
110 | def count(self, role):
111 | size = int(self.size / self.scale)
112 | for i in range(size):
113 | col_cnt = row_cnt = 0
114 | for j in range(size):
115 | if self.map[i][j] == role:
116 | col_cnt += 1
117 | else:
118 | col_cnt = 0
119 | if self.map[j][i] == role:
120 | row_cnt += 1
121 | else:
122 | row_cnt = 0
123 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
124 | return True
125 |
126 | for i in range(size):
127 | col_cnt = row_cnt = 0
128 | for j in range(size - i):
129 | if self.map[i + j][j] == role:
130 | row_cnt += 1
131 | else:
132 | row_cnt = 0
133 | if self.map[size - i - 1 - j][size - j - 1] == role:
134 | col_cnt += 1
135 | else:
136 | col_cnt = 0
137 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
138 | return True
139 | col_cnt = row_cnt = 0
140 | for j in range(i, size):
141 | if self.map[j - i][j] == role:
142 | row_cnt += 1
143 | else:
144 | row_cnt = 0
145 | if self.map[size - 1 - j + i][j] == role:
146 | col_cnt += 1
147 | else:
148 | col_cnt = 0
149 | if col_cnt == self.win_cnt or row_cnt == self.win_cnt:
150 | return True
151 | return False
152 |
153 | def convert(self):
154 | res = ''
155 | size = len(self.map)
156 | for i in range(size):
157 | for j in range(size):
158 | res += str(self.map[i][j])
159 |
160 | return res
161 |
162 | def get_map(self):
163 | return self.map
164 |
--------------------------------------------------------------------------------
/QTable/五子棋/q_table2.csv:
--------------------------------------------------------------------------------
1 | ,"[0, 0]","[0, 1]","[0, 2]","[1, 0]","[1, 1]","[1, 2]","[2, 0]","[2, 1]","[2, 2]"
2 | 0000000000000000,-0.28247276097355545,-0.2697050167115671,-0.27446107216768767,-0.3448166349967724,-0.2786164286217885,-0.26611825592888555,-0.2808256722816426,-0.2694661375850095,-0.2700019147534119
3 | 0100000000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
4 | 0100000020000000,-0.006727290133047975,-0.01,-0.022484584121978887,-0.01818831324057749,-0.01,-0.027951808254563646,-0.01,-0.006672068212513574,-0.0018890521651192726
5 | 1100000020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
6 | 1100002020000000,0.0,0.0,0.0,0.0,-0.01,-0.01,0.0,-0.030257290435244083,-0.0004702008865678372
7 | 1100012020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
8 | 1100012022000000,-0.01,-0.01,0.0,-0.01,0.0,-0.01,-0.01,0.0,-0.008817859368655702
9 | 1100112022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
10 | 0000100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
11 | 0000100000200000,-0.017562984778473078,-0.029740997561148962,-0.027899015945162205,-0.01,-0.014464773932411185,-0.028681389069738904,-0.01,-0.012598578735750685,-0.01
12 | 0100100000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
13 | 0100102000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.03540912230831964,0.0
14 | 0100102010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
15 | 0100102012200000,0.0,-0.01,-0.01,-0.01,-0.01365490355818151,-0.01,-0.01,-0.01,-0.01
16 | 0110102012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
17 | 1000000000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
18 | 1020000000000000,0.0,-0.006506634379379596,-0.01,0.0,-0.023619423277822783,0.0,-0.013348431073593325,0.0,0.0
19 | 1020100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
20 | 1020100020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.016479544535300376
21 | 1120100020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
22 | 1120100020200000,-0.01,0.0,-0.01,0.0,0.0,0.008121095831433018,0.0,0.0,-0.01
23 | 1120110020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
24 | 0010000000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
25 | 0010020000000000,-0.007017877664665998,-0.0118273000119164,-0.01,-0.007770118815905523,0.0,-0.01,-0.014408377122885918,-0.026613998931996397,-0.00286374663704767
26 | 0010020001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
27 | 0010020001200000,0.0,0.0,0.0,0.0,0.0,0.0,-0.003535489675367494,0.0,0.0
28 | 0110020001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
29 | 0110020021200000,0.0,0.0,0.0,-0.004364802068354932,0.0,0.0,0.0,0.0,0.0
30 | 0110021021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
31 | 0000000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
32 | 0000200000100000,0.0062405715300314565,-0.0031276951093877595,-0.0066526475664608,-0.01,-0.006242529186873214,0.0,-0.0058177080344341495,-0.01,0.0
33 | 1000200000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
34 | 1020200000100000,0.0,0.0,0.0,0.0,-0.008213145143778768,0.0,0.0,0.0,0.0
35 | 1020201000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
36 | 1020221000100000,-0.01,0.0,0.0,0.0,-0.01,0.0,-0.00017245106504918122,-0.016922013234115046,-0.01
37 | 1120221000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
38 | 0000000001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
39 | 2000000001000000,-0.01,-0.027914803397533987,-0.016101226588715378,0.0,0.0,-0.015504421681221126,0.0,0.0,0.0
40 | 2000000001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
41 | 2000002001100000,0.0,0.0,-0.019141261334840873,0.0,0.0,0.0,0.0,0.0,0.0
42 | 2100002001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
43 | 2100000000000000,0.0,0.0,-0.013321096343334437,0.0,0.0,-0.027924968282725515,-0.029174304732267776,-0.030232496980668506,-0.02966019028342771
44 | 2100001000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
45 | 2100001020000000,-0.01,-0.01,0.0,0.0,-0.027309343336829054,0.0,0.0,0.0,0.0
46 | 2100001020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
47 | 2100021020100000,-0.01,-0.01,-0.008687836380019002,0.0,-0.01,-0.01,0.0,0.0,0.0
48 | 2100021021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
49 | 0000010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
50 | 0000012000000000,-0.026785543587508703,-0.0228187821037876,-0.0004766683463283749,-0.01686016159507648,-0.01,-0.0199,-0.012473857888841762,-0.01,-0.007763501591364569
51 | 0000012001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
52 | 0200012001000000,0.0,-0.01,-0.015063116532855952,-0.01,0.0,-0.01,-0.022433748062471583,-0.01,0.0
53 | 0210012001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
54 | 0210012021000000,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0,-0.02184689538556861
55 | 0210112021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
56 | 0000001000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
57 | 0000021000000000,-0.02022100108111663,-0.01052241508674002,-0.018800968902074562,-0.0008832369036724884,-0.01,-0.01,-0.01,0.0,-0.028763108923216794
58 | 1000021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
59 | 1000021000200000,0.0,0.0,0.0,-0.03551001105419529,0.0,0.0,0.0,0.0,0.0
60 | 1100021000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
61 | 1100221000200000,-0.01,-0.01,-0.014199359287712397,-0.01,-0.01,-0.01,-0.01,0.0,-0.01
62 | 1100221010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
63 | 0000000010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
64 | 0200000010000000,0.0,0.0,0.0,0.0,-0.005398667386796772,-0.009030060931076243,-0.01,-0.01,0.0
65 | 0200000011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
66 | 1000000002000000,0.0,-0.008228421561952096,0.0,0.0,0.0,0.0,0.0,0.0,0.0
67 | 1010000002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
68 | 1210000002000000,0.0,0.0,0.0,-0.010158545138212469,0.0,0.0,0.0,0.0,0.0
69 | 1210000012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
70 | 0000000002100000,-0.012235234334084629,0.0,-0.015483226852937754,0.0,0.0,0.0,0.0,0.0,0.0
71 | 0000000012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
72 | 0020000012100000,0.0,0.0,-0.01,-0.011253203555765515,0.0,0.0,0.0,0.0,0.0
73 | 0120000012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
74 | 0120200012100000,-0.01389284389600682,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
75 | 0120210012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
76 | 0000200010000000,-0.00259357121535256,0.0,-0.01,-0.01,-0.016632294640337626,-0.0010171585340076855,0.0,-0.018963748828699965,-0.019962090374802535
77 | 1000200010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
78 | 1000200010200000,-0.01,-0.01,0.0,0.0,-0.002815414711592307,0.0,-0.01,0.0,0.0
79 | 1100200010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
80 | 1100220010200000,0.0,0.0,0.0,0.0,0.0,-0.0034758206315954423,0.0,0.0,0.0
81 | 1100220011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
82 | 0100000002000000,-0.022062195614715112,-0.01,-0.028353117959514958,-0.01,-0.0199,-0.03656618863322528,-0.018539449577421818,-0.01,-0.013111454952846024
83 | 0100100002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
84 | 0100100002200000,-0.016186981423266668,-0.01,-0.00685654433127676,0.0,0.0,0.0,0.0,0.0,0.0
85 | 0100100012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
86 | 2100100012200000,-0.01,0.0,-0.012325836645734918,-0.01,-0.011317730973005119,0.0,-0.01,-0.01,-0.01
87 | 2100101012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
88 | 0010200010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
89 | 0010200000000000,-0.01,-0.029079202865214697,0.0,-0.01,0.0,-0.005936734989087762,-0.021858506125466966,-0.013982360599715898,-0.02518818173204846
90 | 0010201000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
91 | 0210201000000000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,-0.03197561784059552
92 | 0210201010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
93 | 0210201010200000,-0.01,0.0,-0.01,-0.01,-0.006051792345938109,-0.01,-0.01,0.0,-0.01
94 | 1210201010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
95 | 0200100000000000,0.0,0.0,0.0,0.0,0.0,-0.01650543700931233,0.0,0.042570583838105455,0.0
96 | 0210100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
97 | 0210102000000000,0.0,0.0,0.0,0.0,-0.011610971765328802,0.0,0.0,0.0,0.0
98 | 1210102000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
99 | 0200010000000000,-0.023504958744431863,-0.01,-0.018540640924357633,-0.02024561823488945,-0.0199,0.0,-0.009723635030659667,-0.02747494941333063,-0.022118113580067995
100 | 0200110000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
101 | 0200110020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012004487692172413,0.0
102 | 1200110020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
103 | 0010002000000000,-0.01,0.0,-0.01,-0.005894250789495155,-0.010917711727044208,-0.0199,-0.007806028922988482,-0.0017809008723064282,0.0
104 | 0010012000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
105 | 0010012020000000,-0.02639081580601917,0.0,0.0,0.0,-0.01,0.0,-0.01,-0.009637072744430228,0.0
106 | 1010012020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
107 | 0000001002000000,-0.017116305919683942,0.002985813720080513,-0.005887765194832635,-0.01928435428801213,-0.015142743034392408,-0.01,-0.013524372103240425,-0.01,-0.01
108 | 0000011002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
109 | 2000001000000000,0.0,-0.015316112137350716,0.0,0.0,0.0,-0.01,-0.005584474414484827,-0.01256245351968558,0.0
110 | 2000011000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
111 | 2000011002000000,0.0,0.0,0.0,-0.015510190129014357,0.0,0.0,0.0,0.0,0.0
112 | 2010011002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
113 | 0000001000200000,-0.00039939137716310183,-0.0066938081325444414,-0.012580813209150764,-0.019953376813916634,-0.01,-0.01,0.0,-0.005663289060649834,-0.01
114 | 0000011000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
115 | 0020011000200000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01553186815944539,0.0,0.0
116 | 0020011001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
117 | 0200001000000000,0.0,-0.01,-0.005592286746170033,-0.023170224770236433,-0.02388386145882329,0.0,-0.01,-0.009780966776179316,-0.008061783775908054
118 | 0200011000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
119 | 0200211000000000,0.0,-0.01,-0.00935762749461292,0.0,-0.0199,-0.01,-0.01,-0.010508444464934829,0.0
120 | 0200211010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
121 | 0220211010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01155262653655915,0.0
122 | 0220211010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
123 | 0000001020000000,-0.016791638704356827,0.0,-0.015048289833526877,-0.017906192082752764,0.0,0.0,0.0,0.0,0.0
124 | 0000011020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
125 | 0000211020000000,0.0,-0.026169408728315666,-0.0061064014308054426,0.0,0.0,-0.01,-0.01,0.0,0.0
126 | 0100211020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
127 | 0200011002000000,0.0,0.0,-0.012075267624912732,0.0,0.0,0.0,0.0,0.0,0.0
128 | 0200011002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
129 | 0000121000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
130 | 0020121000000000,-0.02311152179625751,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
131 | 0020121000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
132 | 2020121000100000,-0.01,0.0,0.0,0.0,0.0,0.0,-0.015901147627408345,0.0,-0.01
133 | 2020121001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
134 | 0020000001000000,-0.009044034964832038,-0.009384168455829234,0.0,-0.024566009021850927,0.0,-0.01,0.0,-0.01,0.0
135 | 0020001001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
136 | 0020201001000000,0.0,-0.010664224626658446,0.0,-0.01,0.0,0.0,-0.01,-0.01,0.0
137 | 0020201011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
138 | 2000011020000000,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0038340582007547162,0.0
139 | 2010011020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
140 | 0000021000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
141 | 1000020000000000,0.0,0.0,0.0,0.0,-0.01,0.0,-0.00016730880417350873,-0.020025016958497913,0.0
142 | 1000020001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
143 | 1000020021000000,-0.01,0.0,0.01917487783598565,0.0,0.0,0.0,0.0,0.0,-0.03131944183777884
144 | 1100020021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
145 | 1000020021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
146 | 0000010002000000,-0.02096698024061887,0.0,0.0,-0.0243385546444072,-0.01,0.0,0.0,0.0,0.0
147 | 0000010012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
148 | 2000010012000000,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0,-0.01961130961837961
149 | 2100010012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
150 | 0100020000000000,-0.011297996079426578,-0.01,-0.0008761455036942439,-0.041900486066591046,-0.01,-0.01,-0.01,-0.020353070367047126,-0.030077350668263905
151 | 0100020010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
152 | 0100020010200000,0.0,0.0,0.0,0.0,0.0,-0.033496952702734215,0.0,0.0,-0.01
153 | 0110020010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
154 | 0110022010200000,0.0,-0.01,-0.01,0.0,-0.01,-0.01,-0.01,-0.012701915457094747,-0.01
155 | 0110122010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
156 | 0000000020100000,0.0,0.0007852246862732848,0.0,-0.019801518223081314,0.0,0.0,0.0,0.0,0.0
157 | 0000001020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
158 | 0000010020000000,-0.01,-0.02545434068063916,-0.0233087663894431,0.0,0.0,0.0,0.0,0.0,-0.02549362728502479
159 | 1000010020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
160 | 1200010020000000,-0.01,0.0,0.0,0.0,0.0,-0.026450755495563374,0.0,0.0,0.0
161 | 1210010020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
162 | 0000000010200000,-0.020716384598960247,-0.030416158048350232,-0.01862078487110863,-0.0199,-0.021190958274475057,-0.02465508807004524,-0.0199,-0.01773685630318699,-0.0199
163 | 0000100010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
164 | 0200100010200000,-0.029646109442723136,-0.01,0.0,0.0,0.0,0.0,-0.01,0.0,0.0
165 | 0200110010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
166 | 2200110010200000,0.0,-0.01,0.0,-0.01,-0.01,-0.022465589057812655,0.0,0.0,0.0
167 | 2200110011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
168 | 1000002000000000,-0.01,-0.009775583369554324,0.0,-0.02528462661586702,-0.008100000000000001,-0.01,0.0,0.0,-0.002847981842326535
169 | 1010002000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
170 | 1010202000000000,-0.01,0.0,0.0,-0.01,-0.007329302455663904,0.0,-0.012333614483358736,0.0,-0.01
171 | 1010202000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
172 | 1010202020100000,0.0,0.0,0.0,0.0,-0.006452696972045044,-0.01,0.0,0.0,0.0
173 | 1010202021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
174 | 0000201000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.029700832114658134,-0.025531917394438483
175 | 0000201001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
176 | 0000201001200000,0.0,0.0,0.0,-0.01,0.0,0.0,-0.026568994656970706,0.0,0.0
177 | 1000201001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
178 | 1000201021200000,0.0,-0.015317275053239718,0.0,-0.01,0.0,-0.01,-0.01,0.0,0.0
179 | 1010201021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
180 | 0000120000000000,0.0,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,-0.01705235355899845
181 | 1000120000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
182 | 0200000001000000,0.0,0.0,0.0,0.0,-0.030794206930618,-0.026459587747803378,0.0,0.0,0.0
183 | 0200010001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
184 | 0200112001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
185 | 0220112001000000,0.02090958324824781,-0.01,0.0,-0.01,0.0,-0.01,0.0,-0.01,-0.0007566531992374341
186 | 1220112001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
187 | 2010000000000000,0.0,0.0,-0.01,-0.008575274486472339,-0.00625039193765506,0.0,0.0,0.0,0.0
188 | 2010000010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
189 | 2010200010000000,0.0,0.0,0.0,0.0,-0.010586758625274481,0.0,0.0,0.0,0.0
190 | 2110200010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
191 | 2110220010000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,-0.003790212810965517,0.0
192 | 2110221010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
193 | 0010021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
194 | 2000000000100000,0.0,0.0,0.0,-0.010276764894149292,0.0,0.0,-0.016854355277878288,-0.027977363129512208,0.0
195 | 0000020000100000,0.0,-0.025776316272605397,0.0,-0.007679180107850857,-0.01,0.0,-0.02029058268139323,0.0,-0.01
196 | 0000020001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
197 | 0000220001100000,0.0,-0.00948046939320904,0.0,0.0,0.0,0.0,-0.0077124466569601586,0.0,0.0
198 | 0000221001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
199 | 1000001020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
200 | 1020001020000000,0.0,0.0,0.0,-0.01857820254056704,0.0,0.0,0.0,0.0,0.0
201 | 1120001020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
202 | 1120201020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015970435208911792,-0.01
203 | 1120201020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
204 | 0000002000100000,0.0,-0.025101527325609885,-0.02740345903303719,-0.001636185472819833,-0.0033255677827501297,-0.01,-0.016178567591925375,-0.00068719271927553,0.0
205 | 0010002000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
206 | 0210002000100000,0.0,-0.01,0.0,-0.030989539908160578,-0.004740886813928259,0.0,0.0,0.0,-0.01
207 | 1210002000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
208 | 1210202000100000,-0.01,-0.01,-0.01,0.0,0.0,0.0,-0.01,-0.011410539589145858,-0.01
209 | 1210202010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
210 | 0000100002000000,-0.017994875424633062,0.0,-0.0151321996635262,0.0,0.0,0.0,0.0,0.0,0.0
211 | 0000100012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
212 | 2000100012000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,-0.015081352575267607
213 | 2010100012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
214 | 1020000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
215 | 1020020000100000,0.0,0.0,0.0,-0.02365405163515134,-0.01,0.0,0.0,0.0,0.0
216 | 1020021000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
217 | 0000020001000000,0.0,0.0,0.0,-0.016891498661624187,0.0,-0.012716013575111793,-0.01,-0.01,0.0
218 | 0000020011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
219 | 0000002010000000,-0.01797005222523328,-0.010475853735320869,-0.006901133853815808,-0.01368823143598649,-0.01481586943952886,0.0,-0.01,-0.01,-0.011885201166453295
220 | 0000012010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
221 | 0200012010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012933152759655415
222 | 0200012011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
223 | 0000200001000000,-0.016962197778948845,0.0,-0.020375080841198077,-0.01,0.0,0.0,0.0,0.0,0.0
224 | 0000200011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
225 | 0020200011000000,0.0,0.0,0.0,-0.01,0.0,0.0,-0.01,0.0,-0.010752051198906489
226 | 0120200011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
227 | 0120200011200000,0.0,0.0,-0.01,0.0,0.0,-0.0040421447933340765,0.0,0.0,0.0
228 | 1120200011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
229 | 0110000002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
230 | 2110000002000000,-0.01,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,-0.013961706342165191
231 | 2110001002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
232 | 2110001002200000,0.0,0.0,0.0,-0.017236674496500234,0.0,0.0,0.0,0.0,0.0
233 | 2110001012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
234 | 2000000010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026963193089682043,0.0
235 | 2000000010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
236 | 2000000012100000,0.0,0.0,0.0,-0.028750484818902647,0.0,0.0,0.0,0.0,-0.01
237 | 2000001012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
238 | 2000201012100000,0.0,0.0,-0.012820989785375188,-0.01,-0.01,0.0,0.0,-0.01,-0.01
239 | 2000211012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
240 | 0210211000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
241 | 0000101020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
242 | 2000101020000000,-0.01,0.0,-0.013247429819166453,0.0,-0.013766214234938462,0.0,-0.01,0.0,0.0
243 | 2100101020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
244 | 0020100000000000,0.0,0.0,0.0,0.0,-0.020995095631406828,0.0,0.0,0.0,0.0
245 | 0020100001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
246 | 0020120001000000,0.0,-0.025919871149884942,0.0,0.0,0.0,0.0,0.0,0.0,0.0
247 | 0020120001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
248 | 0220120001100000,-0.01,-0.01,0.0,0.0,-0.01,-0.013809323943361335,0.0,0.0,0.0
249 | 1220120001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
250 | 0110200000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
251 | 0110200020000000,-0.01354367429959881,0.0,0.0,-0.01,-0.01,0.0,0.0,0.0,0.0
252 | 0110210020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
253 | 0100010002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
254 | 0120010002000000,0.0,-0.01,0.0,-0.011302263611283638,-0.01,0.0,-0.01,-0.01,-0.030868949793499415
255 | 0120010012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
256 | 0120010012200000,0.0,-0.0199,-0.01,0.0,0.0,-0.01117173306946835,0.0,-0.01,-0.01
257 | 1120010012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
258 | 0000120000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
259 | 0000120020100000,0.0,-0.01858037293307916,0.0,-0.01,0.0,0.0,0.0,0.0,0.0
260 | 0010120020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
261 | 0210120020100000,0.0,0.0,0.0,0.0,-0.0199,-0.007498448431865717,0.0,0.0,0.0
262 | 1210120020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
263 | 1200000000000000,-0.01,-0.01,-0.009228003064798918,-0.012272790714854636,-0.01,0.0,-0.008807786074408087,0.0,0.0
264 | 1200001000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
265 | 1200201000000000,0.0,0.0,0.0,0.0,-0.015151593475129186,0.0,0.0,0.0,0.0
266 | 1200201010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
267 | 1200221010000000,0.0,0.0,-0.018705670956949586,0.0,0.0,0.0,0.0,0.0,0.0
268 | 1200221011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
269 | 0210001000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
270 | 0210021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.029486248714596645
271 | 0210121000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
272 | 0210121000200000,-0.006087827197607268,-0.01,-0.01,-0.01,-0.01,0.0,0.0,0.0,-0.01
273 | 0210121001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
274 | 0000000021000000,0.0,-0.003558827970263112,0.0,0.0,-0.025368747888600857,0.0,0.0,0.0,0.0
275 | 1000000021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
276 | 1010020021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
277 | 1010020021200000,-0.01,0.0,-0.01,-0.0033069449185516944,-0.0199,0.0,-0.01,-0.01,0.0
278 | 1010021021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
279 | 1010200000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
280 | 0000210000000000,-0.014093526513946141,-0.0025638512453877733,-0.01,0.0,0.0,0.0,-0.04224237227602683,-0.024560846736990587,0.0
281 | 0010210000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
282 | 0010210020000000,0.0,0.0,-0.01,0.0,-0.01,0.0,0.0,0.0,-0.01241359999757201
283 | 0010211020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
284 | 0010211020200000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,-0.006574607525639401,0.0
285 | 1010211020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
286 | 2000010000000000,-0.01,0.0,-0.01,0.0,0.0,-0.02962702747997952,0.0,0.0,-0.019984546059458593
287 | 2010010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
288 | 2010010000200000,0.0,0.0,0.0,-0.02467227908575135,-0.01,0.0,-0.0033573512655444395,0.0,0.0
289 | 2010010001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
290 | 2010210001200000,0.0,0.0,0.0,-0.0199,0.0,-0.01,-0.0012482667601315253,0.0,-0.01
291 | 2010211001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
292 | 0000010010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
293 | 2000010010200000,0.0,0.0,-0.012315013446802888,0.0,0.0,0.0,0.0,0.0,0.0
294 | 2000011010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
295 | 2020011010200000,-0.01,0.0,0.0,-0.0064243460551789075,0.0,0.0,0.0,0.0,0.0
296 | 2120011010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
297 | 0020000001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
298 | 0120000001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
299 | 2120000001000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01116547526522474,0.0,0.0
300 | 2120100001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
301 | 2120100021000000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,-0.004672268351203681
302 | 2120101021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
303 | 0100002000000000,-0.015319897585680437,-0.03940399,-0.020672218322159354,-0.010944101301155758,-0.029701,-0.0199,-0.01350385948152081,-0.016095877499175362,-0.036140187387708765
304 | 0100002000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
305 | 0120002000100000,0.0,0.0,0.0,-0.009986264550147119,0.0,0.0,0.0,0.0,0.0
306 | 0120002010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
307 | 0100002020100000,-0.019973540236944917,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
308 | 0100012020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
309 | 2100012020100000,-0.01,0.0,4.343566737794057e-06,0.0,-0.01,0.0,0.0,0.0,-0.01
310 | 2100112020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
311 | 0000210000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
312 | 0200210000100000,0.0,0.0,-0.0031652359818404915,0.0,0.0,0.0,-0.003861351986898467,0.0,0.0
313 | 0200211000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
314 | 0100200000000000,-0.024609012356584806,-0.029701,-0.03818113478444556,-0.029701,-0.0315461037871369,-0.022996992007928777,-0.028511636902308173,-0.02066393758809919,-0.025440935316656692
315 | 1100200000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
316 | 0000110020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
317 | 0000110020200000,0.0,-0.01,-0.020383499363099154,0.0,0.0,-0.013632702377455914,-0.01,0.0,-0.01
318 | 0100110020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
319 | 0120110020200000,-0.01,-0.01,-0.01,0.0,0.0,0.0,-0.01,0.034108300897354574,0.0
320 | 0000200001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
321 | 0100220001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
322 | 0100220021100000,0.0,0.0,0.0,0.0,0.0,-0.009521539082666858,0.0,0.0,0.0
323 | 1100220021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
324 | 0000100020000000,-0.01713201315954313,0.0,0.0,0.0,-0.018996971552342163,-0.025343013183959897,-0.01,0.0,0.0
325 | 2000101021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
326 | 2000121021000000,0.0,0.0,0.014424317627030255,-0.01,0.0,-0.01,0.0,0.0,-0.01
327 | 2000121021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
328 | 0100200001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
329 | 0100220001000000,0.0,0.0,0.0,0.0,0.0,0.009695886836007172,0.0,0.0,0.0
330 | 1100220001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
331 | 1100002000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
332 | 1100102020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
333 | 1100220000000000,0.0,0.0,-0.005016529527483013,-0.01,-0.01,0.0,-0.0018286993134812946,0.0,0.0
334 | 1100220010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
335 | 1120220010000000,0.0,0.0,0.0,0.0,0.0,0.004699695888609946,0.0,-0.01,0.0
336 | 1120220011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
337 | 0100202001000000,0.0,0.0,0.0,-0.01,-0.006657366460914168,0.0,-0.020457382425198293,0.0,0.0
338 | 1100202001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
339 | 1100202000000000,-0.01,0.0,0.0,-0.01,0.0,-0.01,-0.0059381587799366845,0.0,-0.01
340 | 1100202000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
341 | 1100020000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
342 | 1120020000000000,0.0,-0.01,0.0,0.0,0.0,0.0,0.007287729762353143,0.0,-0.01242247549450116
343 | 1120020001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
344 | 1120020001200000,-0.01,-0.01,-0.01,0.0,0.0,0.01834495710467428,-0.006680432660706176,0.0,-0.01
345 | 1120120001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
346 | 0100020000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
347 | 0100020002100000,-0.018675709440555355,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,0.0
348 | 0100021002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
349 | 0100000000200000,-0.048545027942357204,-0.0490099501,-0.049189819689348005,-0.055272534672242205,-0.10363124856674964,-0.04525264050772587,-0.05529173187122769,-0.05399125487922845,-0.07725530557207991
350 | 1100000000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
351 | 1100000002200000,0.0,0.0,0.0,0.0,0.0,-0.014239998499830596,-0.01,0.0,0.0
352 | 1100000012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
353 | 1100002012200000,0.0,0.0,0.0,-0.017580245061519242,0.0,0.0,0.0,0.0,0.0
354 | 1100012012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
355 | 0120000000000000,-0.01,-0.01,-0.01,-0.018794190712073843,-0.013285335297890897,-0.013163787718481774,-0.0031044011980557364,-0.024840663883433794,-0.015059333635161033
356 | 1120000000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
357 | 1120021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
358 | 1100000002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
359 | 0000100002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
360 | 0020100002100000,0.0,-0.018681727979661982,0.0,0.0,0.0,0.0,0.0,0.0,0.0
361 | 0020110002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
362 | 0220110002100000,-0.007565709043265322,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,-0.01
363 | 0220110012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
364 | 2100000001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
365 | 2100002001000000,-0.01,-0.01,0.0,0.0,0.0,0.0,-0.024958496395132267,0.0,0.0
366 | 2110002001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
367 | 2110002021000000,-0.01,-0.01,0.0,-0.01,-0.0019456731965925434,0.0,-0.01,0.0,0.0
368 | 2110102021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
369 | 1100002000200000,0.0,0.0,0.0,0.0,-0.01,0.0,0.0049463385402417626,-0.005117971446323476,-0.01
370 | 1100102000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
371 | 0000002001000000,0.0,-0.01,-0.0032060041746770564,-0.01657047976441062,0.0,-0.01,-0.008968839881829588,-0.01,-0.027067919369638437
372 | 0000102001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
373 | 0000102021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01107264182941925
374 | 0100102021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
375 | 0100102021200000,0.0,0.0,0.0,0.0,-0.0045307755363804955,0.0,0.0,-0.01,0.0
376 | 0110102021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
377 | 0100001002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
378 | 0100001022000000,0.0,0.0,0.0,-0.008267599608657859,0.0,-0.01,0.0,0.0,0.0
379 | 0110001022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
380 | 0110201022000000,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,-0.00025544826803513225
381 | 0110211022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
382 | 0100021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
383 | 0100221000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.03221929934284205,0.0
384 | 1100221000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
385 | 1100221002000000,-0.01,-0.01,-0.013839107190316403,-0.01,0.0,-0.01,0.0,-0.01,-0.01
386 | 1100221002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
387 | 0100012000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
388 | 0100212000000000,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,-0.01082745857867527,0.0
389 | 0100212010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
390 | 0100212012000000,-0.00415707754713501,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.0
391 | 0100212012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
392 | 0000000012000000,0.0,-0.002103791020176899,-0.01,0.0,-0.0234419287104836,-0.018333382514737873,-0.01,0.0,0.0
393 | 0010000012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
394 | 0010020012000000,0.0,0.005459548284040202,0.0,0.0,0.0,0.0,0.0,0.0,0.0
395 | 1010020012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
396 | 0020000010000000,-0.008192375255796785,0.0,-0.01,-0.0199,0.0,-0.027984534863650032,-0.01,-0.017142036826040586,0.018172167913326407
397 | 0020001010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
398 | 2020001010000000,-0.01,0.0,0.0,0.0,-0.0001407944761420256,0.0,0.0,0.0,0.0
399 | 2020001011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
400 | 2020021011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00017382034091608067
401 | 2120021011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
402 | 0010000002000000,0.0,0.0,-0.01,-0.02471925028733593,0.0,0.0,-0.0083809231736651,0.0,0.0
403 | 0010000002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
404 | 0010000022100000,-0.010346818732919877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
405 | 0010010022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
406 | 2010010022100000,0.0,0.0,-0.01,0.0,0.0,-0.003424506527846164,0.0,0.0,0.0
407 | 2010110022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
408 | 1020000010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
409 | 1220000010000000,0.0,0.0,0.0,-0.008032881949851361,0.0,0.0,0.0,0.0,0.0
410 | 1220010010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
411 | 1220210010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.009917138209693033
412 | 1220211010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
413 | 0020000000100000,0.0,-0.0163895485896724,0.0,-0.027331934306776346,0.0,-0.01,0.0,-0.02787748071015098,-0.01
414 | 0020010000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
415 | 0020001000000000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,-0.020051613673716103,0.0
416 | 0020001012000000,0.0,-0.024755078609526058,0.0,0.0,0.0,0.0,0.0,0.0,0.0
417 | 0020011012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
418 | 0220011012000000,-0.0013622766850448582,-0.01,0.0,0.0,0.0,-0.01,-0.01,-0.01,0.0
419 | 0220011012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
420 | 0100000010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
421 | 0120000010200000,0.0,0.0,0.0,-0.022988623297664965,-0.012408615370524526,-0.01,-0.01,0.0,0.0
422 | 0120010010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
423 | 0120210010200000,0.0,-0.01,0.0,-0.01,0.0,0.0,0.0,-0.015669892420165116,0.0
424 | 0120211010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
425 | 0010201000200000,0.0,0.0,0.0,0.0,-0.02604508723067897,-0.01,0.0,0.0,0.0
426 | 1010201000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
427 | 1010221000200000,-0.01,0.0,0.0,-0.01,0.0,0.0,-0.005061985278392183,-0.01,-0.01
428 | 1010221001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
429 | 0200002011000000,0.0,0.0,0.0,-0.01114822337169906,0.0,0.0,0.0,0.0,0.0
430 | 0210002011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
431 | 0100001000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
432 | 0100001020200000,0.0,0.0,0.004934413630511633,-0.010804135033843195,0.0,0.0,0.0,0.0,0.0
433 | 0100011020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
434 | 0100211020200000,-0.002815201844616354,-0.01,-0.007704149786943127,0.0,0.0,-0.01,0.0,0.01292083840524324,0.0
435 | 0110211020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
436 | 0200001000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
437 | 0220001000100000,0.0,0.0,0.0,0.0,-0.006904057711321022,0.0,0.0,0.0,0.0
438 | 1220001000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
439 | 1220021000100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.00852352803866793,0.0,0.0
440 | 1220121000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
441 | 1200010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
442 | 1220010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.011392596376294972
443 | 1220110000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
444 | 2000100000000000,0.0,0.0,-0.01,0.0,0.0,-0.0296610323073767,-0.018780293597586976,0.0,0.0
445 | 2000110000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
446 | 2000110020000000,-0.01,0.0,-0.016278453890545618,0.0,0.0,0.0,0.0,0.0,0.0
447 | 2000110021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
448 | 2020110021000000,0.0,0.0,0.0,0.0,0.0,-0.012465255129622944,0.0,0.0,-0.01
449 | 2020110021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
450 | 0100012000200000,-0.01,-0.01,-2.2234147314302566e-05,0.0049092994924008135,-0.01,-0.01,-0.0011798302719330054,-0.0032852144853568787,-0.01
451 | 0110012000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
452 | 0110012020200000,0.003206101695197897,0.0,0.0,0.0,0.0,0.0,-0.01,-0.005383970945777365,-0.01
453 | 0110112020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
454 | 0000021001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
455 | 0000221001000000,-0.013399632162207275,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.0
456 | 0000221011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
457 | 2000221011000000,0.0,0.0,0.0,-0.01,-0.01,0.0,0.0,0.0,0.0023734861213055188
458 | 2100221011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
459 | 1000000000200000,-0.01,0.0,-0.010308775852065485,-0.02731053244687457,-0.009363982685322148,-0.013094239071719097,0.0,0.0037986934887137376,-0.01
460 | 1000000001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
461 | 1000002001200000,0.0,0.0,-0.01616572724903592,0.0,0.0,0.0,0.0,0.0,0.0
462 | 1000012001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
463 | 0020200001100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.023842598344456605,-0.01,-0.01
464 | 0020201001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
465 | 0020201021100000,0.0,-0.01727679713167896,0.0,0.0,0.0,-0.01,0.0,-0.01,0.0
466 | 1020201021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
467 | 1000000012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
468 | 1000020012000000,0.0,0.0,0.0,-0.010152214768722306,-0.01,-0.01,-0.01,-0.01,-0.02472224315863938
469 | 1000021012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
470 | 1000221012000000,0.0,-0.0031278993579062797,-0.01,0.0,0.0,0.0,0.0,0.0,0.0
471 | 1010221012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
472 | 1120200000000000,-0.01,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,-0.0013226985173169948
473 | 1120200010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
474 | 1120200010200000,0.0,0.0,0.0,0.0,0.0,0.0009589734812452314,0.0,0.0,0.0
475 | 1120210010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
476 | 2000101000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
477 | 2200101000000000,0.0,-0.01,-0.0012327090633780704,0.0,0.0,0.0,-0.01,0.0,0.0
478 | 2200101010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
479 | 0000102000000000,0.0,0.0,-0.018662795345008237,0.0,0.0,0.0,0.0,0.0,0.0033671067195942196
480 | 1000102000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
481 | 1020102000000000,-0.01,-0.01609936800031738,0.0,0.0,0.0,0.0,0.0,0.0,0.0
482 | 1020102000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
483 | 0220110000000000,0.0,-0.01,0.0,0.0,0.0,-0.007300228857686268,-0.01,0.0,0.0
484 | 0220110010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
485 | 0220112010000000,0.00121897750695584,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0
486 | 0220112011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
487 | 0010000020000000,-0.01,0.0,0.0,-0.011568614843385179,-0.030701564322389047,-0.028686161542766625,0.0,-0.014952893546355574,-0.024836869931001183
488 | 1010000020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
489 | 1010000022000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,-0.01,-0.0005492492042495142
490 | 1010001022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
491 | 0010000000200000,-0.017671572896065565,-0.017871753208464843,-0.0199,-0.02299419780605585,-0.017103478846947327,-0.01687283024411705,-0.015239584175216632,-0.014166279045198609,-0.0199
492 | 0010000001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
493 | 2010000001200000,-0.01,0.0,0.0,0.0,0.0,-0.0056649240387278395,0.0,-0.01,0.0
494 | 2010000011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
495 | 1000002000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
496 | 1000002002100000,0.0,0.0,0.0,0.0,-0.012644676627480478,0.0,0.0,0.0,0.0
497 | 1010002002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
498 | 1010022002100000,0.0,-0.006926804797257251,-0.01,0.0,0.0,0.0,0.0,0.0,0.0
499 | 1010022012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
500 | 0000002001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
501 | 0020002001100000,-0.017255783624116634,-0.01,-0.01,0.0,0.0,0.0,0.0,0.0,-0.01
502 | 0120002001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
503 | 2010000001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
504 | 2210000001000000,0.0,0.0,0.0,0.0,0.0,-0.03446272024386917,0.0,0.0,0.0
505 | 2210100001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
506 | 2210102001000000,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,0.0,-0.01,-0.008547554646689623
507 | 2210112001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
508 | 0000002011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
509 | 0000022011000000,0.0,0.0,-0.010236045480153713,0.0,-0.01,0.0,0.0,0.0,0.0
510 | 1000022011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
511 | 0000020010000000,-0.0025464497945026357,-0.01,0.0,0.0,0.0,0.0,-0.01,-0.027370321493267746,-0.007718012536411374
512 | 0000020011200000,0.0,0.0,0.0,0.0,0.0,-0.009528410538779473,0.0,0.0,0.0
513 | 0000120011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
514 | 0000122011200000,0.0,0.00959859685944461,-0.0021771232110645833,-0.01,0.0,0.0,0.0,-0.01,-0.01
515 | 0100122011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
516 | 2100021000000000,-0.0199,-0.01,-0.01,0.0,0.0,0.0,0.0,0.0,0.010641805507197422
517 | 2110021000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
518 | 1000200000000000,0.0,0.0,0.0,-0.01,-0.022104308572033063,0.0,0.0,-0.007058428999558343,0.0
519 | 1200100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
520 | 1200100002000000,0.0,-0.01,0.0,0.0,0.0969346362774374,0.0,0.0,0.0,0.0
521 | 1210100002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
522 | 1200101002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
523 | 0200100001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
524 | 0200102001000000,0.0,0.0,-0.003548790168967943,0.0,0.0,0.0,0.0,0.0,0.0
525 | 0200000000100000,-0.01,-0.01,-0.023754219310442356,-0.017701421968516016,-0.01153023550715105,-0.02031720871338274,-0.0072780465765534385,-0.008840565945118563,-0.0199
526 | 1200000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
527 | 1200100020000000,0.0,-0.01,0.0,0.0,-0.0010787777387716588,0.0,0.0,0.0,0.0
528 | 1210100020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
529 | 1210120020000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,0.01070145139647665,0.0
530 | 1210121020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
531 | 0200000001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
532 | 0220000001100000,0.0,0.0,0.0,0.0,0.0017276313030948564,0.0,0.0,0.0,0.0
533 | 1220000001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
534 | 1000000020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
535 | 2000001000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
536 | 2000001002100000,0.0,0.0,-0.03453995448087922,0.0,0.0,0.0,0.0,0.0,0.0
537 | 2000011002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
538 | 2020011002100000,-0.01,-0.01,-0.01,-0.008964356565388515,-0.01,-0.01,0.0,-0.01,-0.01
539 | 2120011002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
540 | 0200010000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
541 | 0200010020100000,0.0,0.0,-0.018065956536473127,0.0,0.0,0.0,0.0,0.0,0.0
542 | 0200011020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
543 | 0220011020100000,-0.0064070264365623545,0.0,0.0,0.0,-0.01,-0.01,0.0,0.0,0.0
544 | 0220011021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
545 | 1200020000100000,0.0,0.0,0.0,-0.024452420326771106,0.0,0.0,0.0,0.0,0.0
546 | 1200021000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
547 | 1200221000100000,0.0,0.0,-0.01842428477812419,-0.01,0.0,-0.01,0.0,0.0,0.0
548 | 1200221010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
549 | 0200100000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
550 | 0200100002100000,0.0,-0.01,0.0,0.0,-0.0048313816030192665,0.0,0.0,0.0,-0.01
551 | 1200100002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
552 | 0220100000100000,0.0,-0.01,-0.01,-0.01,0.0,0.0,0.0,-0.009580421984147845,0.0
553 | 1220100000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
554 | 0200000010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
555 | 0200200010100000,0.0,0.0,0.0,0.0,0.00045508006619948807,-0.01,0.0,0.0,0.0
556 | 0200201010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
557 | 1010000000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
558 | 0110000000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
559 | 0110020000200000,0.0,0.0,-0.01,-0.022195200496874595,-0.01,0.0,0.0035403881572741726,0.0,0.0
560 | 0110220010200000,-0.0051567005460755435,0.0,-0.01,-0.01,0.0,0.0,0.0,0.0,-0.01
561 | 0110220011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
562 | 1010000020200000,0.0,0.0,-0.01,0.0,0.0,-0.019149170133560607,0.0,0.0,-0.01
563 | 1010000021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
564 | 1010200002000000,0.0,-0.01102021286902653,-0.01,-0.01,0.0,0.0,0.0,-0.01,0.0
565 | 1010200002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
566 | 0200010002100000,0.0,0.0,0.0,-0.01,0.0,0.0,-0.02953048226400373,0.0,0.0
567 | 0200110002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
568 | 0200110022100000,0.0,-0.01,-0.01,-0.01,0.0,-0.006244445198536673,-0.01,0.0,-0.01
569 | 0210110022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
570 | 1010002002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0021986431298904245
571 | 1010102002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
572 | 1010020000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
573 | 1010200020000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,-0.005286716725171785
574 | 1010210020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
575 | 1010210020200000,0.0,-0.006526810771817018,0.0,0.0,0.0,0.0,0.0,0.0,0.0
576 | 1210020000000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,-0.005680993769115073,0.0
577 | 1210020000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
578 | 0200120000100000,0.0,-0.01,-0.004291157698698722,-0.01,0.0,0.0,-0.01,0.0,-0.01
579 | 0200120010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
580 | 0010100000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
581 | 0010100002200000,0.0,-0.010408964529104187,0.0,0.0,0.0,0.0,0.0,0.0,0.0
582 | 0010100012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
583 | 0010000010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
584 | 0010000012200000,0.0,0.0034761006099531793,0.0,-0.01,-0.005787766286731729,0.0,-0.01,0.0,-0.01
585 | 1010000012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
586 | 0210000012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,-0.00406716071779179
587 | 0210100012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
588 | 0210100012200000,0.0,0.0,0.0,0.0,-0.00502118607134789,0.0,0.0,0.0,0.0
589 | 0210101012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
590 | 0010002012000000,0.0,-0.022633805573750453,0.0,0.0,0.0,0.0,0.0,0.0,0.0
591 | 0010102012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
592 | 0020010010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
593 | 0020012010000000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,-0.030307170955113608,0.007717075380029796
594 | 0020012010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
595 | 0020012012100000,0.0012413930460553437,0.0,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,0.0
596 | 0020112012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
597 | 0200101000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
598 | 0200101002000000,0.0,0.0,0.0,0.0,0.00040000103221589524,0.0,0.0,0.0,0.0
599 | 0200101012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
600 | 0200001010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
601 | 0200001010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.009952819476429692,0.0
602 | 0200101010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
603 | 0210000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
604 | 0210002010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
605 | 0210022010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00585294668386205,0.0
606 | 0210122010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
607 | 0000211000200000,0.0,-0.024633798535699556,0.0,0.0,0.0,0.0,-0.014468904698322736,0.0,-0.01
608 | 0010211000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
609 | 0210211000200000,-0.01,0.0,-0.01,-0.01,0.0,0.0,-0.001014448830269286,0.0,-0.01
610 | 1210211000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
611 | 0000001021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
612 | 0200001021000000,0.0,0.0,0.006921463236910363,0.0,-0.01,0.0,0.0,0.0,0.0
613 | 0200011021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
614 | 0220011021000000,0.008545016341864638,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
615 | 0000112000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
616 | 0000112000200000,0.0,0.0,-0.005267411635751987,0.0,0.0,0.0,0.0,0.0,0.0
617 | 1000112000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
618 | 0110002000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
619 | 0110022000000000,-0.004294637596470367,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,0.0
620 | 0110022000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
621 | 2110022000100000,0.0,0.0,0.0,-0.005302021724037494,0.0,0.0,0.0,0.0,0.0
622 | 2110022010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
623 | 1000100020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
624 | 1000102020000000,-0.01,0.0,0.0,0.0,0.0,0.0,-0.01,-0.02010020779325389,0.0
625 | 1000102020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
626 | 1000102022100000,-0.01,0.0,-0.018290211542834726,0.0,0.0,0.0,0.0,0.0,0.0
627 | 1100102022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
628 | 0200010010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
629 | 0200010010200000,0.0,0.0,0.0,-0.02730631306181233,0.0,0.0,0.0,0.0,0.0
630 | 1200010010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
631 | 1200210010200000,-0.01,0.0,0.0,0.0,-0.01,-0.00867914563614661,0.0,-0.01,-0.01
632 | 1200210011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
633 | 0100002010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
634 | 2100002010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01891345380948202
635 | 2100002011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
636 | 0000021010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
637 | 0200021010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012990635909555575,0.0
638 | 0200121010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
639 | 1000200001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
640 | 1000220001000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.027289269842016118,0.0,0.0
641 | 1000220001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
642 | 1000220021100000,-0.01,-0.016990515588901915,0.0,0.0,-0.01,0.0,0.0,-0.01,0.0
643 | 1000221021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
644 | 0000001002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
645 | 0000021002100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.018694744486904884,0.0,0.0
646 | 0100021022100000,-0.01614806353742552,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0
647 | 0100121022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
648 | 2000012001000000,-0.01,0.0,0.0,0.0,0.0,-0.01,-0.01582033062984914,-0.01,0.0
649 | 2000112001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
650 | 1000020010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
651 | 1000021012200000,0.0,0.0,-0.005237356779612318,0.0,0.0,0.0,-0.01,-0.01,-0.01
652 | 1100021012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
653 | 0100100020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
654 | 0100120020000000,-0.016608705307639342,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,0.0
655 | 0100121020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
656 | 2100121020000000,-0.01,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,-0.0036649511566464485
657 | 2100121021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
658 | 2020000001100000,0.0,0.0,0.0,0.0,0.0,-0.012195132736953752,0.0,-0.01,0.0
659 | 2020010001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
660 | 0000000001200000,-0.02352575921771798,-0.012540147810138251,-0.018734925381557146,-0.01330126328284109,-0.023632361592667094,-0.01848961143884313,-0.013589169971637977,-0.01,-0.01
661 | 0000010001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
662 | 0120100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
663 | 0120100000200000,0.0,0.0,0.0,0.0,0.0,-0.010607123357880109,0.0,0.0,-0.01
664 | 0120100010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
665 | 0010010020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
666 | 0010112020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
667 | 2010112020000000,-0.01,0.0,0.0,0.0,-0.01,0.0,0.0,-0.022071717850183113,0.0
668 | 2110112020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
669 | 0100020012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.03379052036205893
670 | 0100021012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
671 | 0100021012200000,0.0,-0.01,-0.0139851707906504,-0.01,-0.01,-0.01,-0.01,-0.01,0.0
672 | 0100121012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
673 | 1000020001200000,0.0,0.0,-0.01,0.0,-0.01,0.0,-0.00849530813895697,0.0,-0.01
674 | 1010020001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
675 | 1000002001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
676 | 1020002001000000,0.0,0.0,0.0,-0.003958029845280313,0.0,0.0,0.0,0.0,0.0
677 | 1120002001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
678 | 1120202001000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.004886456599111502,0.0,0.0
679 | 1120202001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
680 | 1200002000100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,-0.0025538536344373174,0.0
681 | 1200002010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
682 | 2010000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
683 | 2010020000100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.007716533256364273,0.0,0.0
684 | 2010020001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
685 | 2010020021100000,0.0,0.0,0.0,0.0,0.0,-0.009526584267116382,0.0,0.0,0.0
686 | 2110020021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
687 | 2000010000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
688 | 2000010020100000,0.0,0.0,0.0,0.0,0.0,-0.020807846022071978,0.0,0.0,0.0
689 | 2100010020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
690 | 2010100000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
691 | 2010102000000000,-0.01,0.0,0.0,-0.01,0.0,-0.01,-0.022500255727670157,0.0,0.0
692 | 2010102000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
693 | 2010102020100000,0.0,-0.01,0.0,0.0,0.0,0.0,-0.01,-0.005865361330704287,-0.01
694 | 2110102020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
695 | 0100202000100000,0.0,0.0,0.0,0.0,0.009851873993568305,0.0,0.0,0.0,-0.01
696 | 2010012000000000,-0.01,0.0,-0.01,0.0,-0.01,0.0,-0.02242126056472341,0.0,0.0
697 | 2110012000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
698 | 2110012020000000,0.0,-0.01,0.0,0.0,-0.01,0.0,0.0,-0.014602299342209937,0.0
699 | 0000210002100000,0.0,-0.018628308197881967,0.0,-0.01,-0.01,0.0,0.0,0.0,0.0
700 | 0000211002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
701 | 0200020011000000,-0.034589554840143304,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,0.0
702 | 0200120011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
703 | 2200120011000000,-0.01,-0.01,0.0003181683714192732,-0.0199,-0.0199,0.0,0.0,-0.0199,0.0
704 | 2200121011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
705 | 0020100010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
706 | 0020100012000000,0.0,0.0,-0.01,0.0,0.0,-0.013806940995045388,0.0,0.0,0.0
707 | 0020110012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
708 | 0000201010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
709 | 0000221010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.020533697086836576,0.0
710 | 0000221010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
711 | 0000221012100000,-0.011050515648654493,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,-0.01
712 | 0100221012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
713 | 0100201002000000,0.0,0.0,-0.017046721975327098,0.0,0.0,0.0,0.0,-0.01,0.0
714 | 0100201002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
715 | 0120201002100000,0.0,0.0,-0.01,0.0,-0.01,0.0,0.006803510131870186,0.0,-0.01
716 | 0120211002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
717 | 0020010000000000,0.0,0.0,0.0,-0.006823275463074825,0.0,0.0,0.0,0.0,0.0
718 | 1020010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
719 | 1020210000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.009766112191345823,0.0
720 | 1020211000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
721 | 0120000021000000,0.0,0.0,0.0,0.0,0.018813412862001146,0.0,0.0,0.0,0.0
722 | 0120001021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
723 | 0020100010200000,0.0,0.0,0.0,0.0,0.0,0.022434775201637537,0.0,0.0,0.0
724 | 0020100011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
725 | 0120000000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
726 | 0120000020100000,0.0,0.0,0.0,0.0,0.034644063948161365,0.0,0.0,0.0,0.0
727 | 0120001020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
728 | 0120010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
729 | 0100210000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
730 | 0100210020000000,0.0,-0.01,0.0,-0.01,-0.01,0.0,-0.01,-0.014755575173564654,-0.006637262161890902
731 | 0100211022000000,0.004296503008533947,-0.01,-0.01,0.0,0.0,0.0,-0.01,0.0,0.0
732 | 0100010020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
733 | 1100211020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
734 | 0020011000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
735 | 0020211000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0012650550259023187
736 | 0020211001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
737 | 0120010020000000,-0.01297823831949029,-0.01,-0.01,0.0,0.0,-0.030606335451507805,-0.01,0.0003657639126240259,-0.01
738 | 1120010020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
739 | 0010000021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
740 | 0010020021000000,-0.01,0.0,-0.01,0.0,-0.01,-0.010390902658251018,-0.01,-0.01,0.0
741 | 1010022021000000,0.0,0.0,0.0,-0.0034916973910242605,0.0,0.0,0.0,0.0,-0.01
742 | 1010022021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
743 | 0100012002000000,0.0,0.0,-0.03486003278421671,0.0,-0.01,0.0,0.0,0.0,-0.012186979879858804
744 | 0100112002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
745 | 0100112002200000,0.006910772341768815,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,-0.01
746 | 0100112012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
747 | 0100010000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
748 | 0100210000200000,-0.013898304201813804,-0.01,-0.01,-0.01,-0.01,-0.0042085043082951005,-0.003698039460330701,-0.0052016270236305735,-0.01
749 | 0110210000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
750 | 0100210010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
751 | 0100210012200000,0.0020471969862658635,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,0.0
752 | 0100211012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
753 | 0000020010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
754 | 2000020010100000,0.0,0.0,0.0,0.0,0.0,-0.0031437651783983176,0.0,0.0,0.0
755 | 2100020010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
756 | 0100211000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
757 | 2100211000200000,0.0,-0.01,0.0,0.0,-0.01,0.0,0.0,-0.007337353832028341,0.0
758 | 2100211010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
759 | 1100012000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
760 | 2100100000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.013135853912653749,0.0,0.0
761 | 2110100000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
762 | 2110100020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01644154627486365,0.0
763 | 2110110020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
764 | 0100010022000000,0.0,-0.01,-0.002289483297972808,0.0,-0.01,0.0,-0.01,0.0,0.015151292115644595
765 | 0100011022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
766 | 0120011022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.009615309517430087
767 | 1120011022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
768 | 2110101020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
769 | 2100010000200000,-0.01,-0.0199,-0.011397413575618343,-0.015553456367627314,-0.01,-0.01,-0.0022769600601170695,-0.018639602186251915,0.0
770 | 2100110000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
771 | 0010001000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
772 | 0210001000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00826396065746227,0.0
773 | 0210101000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
774 | 2100000010200000,-0.01,0.0,-0.01,0.004431335757011249,0.0,-0.00800816358247631,-0.01,-0.025415064616242992,-0.01
775 | 2110000010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
776 | 0100210002000000,-0.025511034069644875,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
777 | 0100210012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
778 | 2100210012000000,0.0,0.0,-0.020416253299321747,-0.01,0.0,0.0,-0.01,0.0,0.0
779 | 2100210012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
780 | 0010010000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
781 | 0010210000200000,0.0,-0.028387901828328094,0.0,0.0,0.0,0.0,0.0,0.0,0.0
782 | 0010210001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
783 | 0210210001200000,0.0,-0.01,-0.01,0.0,0.0,0.0,-0.019542700658496358,-0.01,0.0
784 | 1210210001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
785 | 1000010000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
786 | 1000210000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.03371670672453923,0.0
787 | 1000210010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
788 | 1000210012200000,-0.01,-0.01,-0.013662513505259436,-0.01,-0.01,0.0,-0.01,0.0,-0.01
789 | 1100210012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
790 | 0120000010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
791 | 0120200010000000,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,-0.023104007845966477,0.0
792 | 1120200012000000,-0.01,-0.01,-0.01,0.0,0.003372948019660279,0.0,0.0,-0.01,0.0
793 | 1120201012000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
794 | 2100011000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
795 | 2100001000200000,-0.01,-0.01,0.0,0.0,0.03768827921155604,0.0,0.0,0.0,0.0
796 | 2100001001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
797 | 2100101000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
798 | 2100101020200000,0.0,0.0,0.0,0.0,0.01234678119708688,0.0,0.0,0.0,-0.01
799 | 0010120000200000,0.0,-0.006057586202163866,0.0,0.0,-0.01,-0.00047466070391127493,0.0,0.0,0.0
800 | 0110120000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
801 | 0110122000200000,0.011622221149350284,-0.01,0.0,0.0,0.0,0.0,0.0,0.0,0.0
802 | 0110122001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
803 | 0120210000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0007925751729098638,0.0
804 | 1120210000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
805 | 2100010020000000,0.0,0.0,0.0,0.0,0.0,0.0020922265919098093,-0.01,0.0,0.0
806 | 2100110020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
807 | 1100220000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
808 | 1100020000200000,0.0,0.0,-0.012078507497581189,-0.0019265091987840399,0.0,0.0,0.0,0.0,-0.01
809 | 0100012002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
810 | 0120012002100000,0.0,-0.01,-0.01,-0.01,-0.01,-0.01,-0.0010063535996130406,-0.0199,-0.01
811 | 0120112002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
812 | 1210000000200000,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,0.0007178152391526157,0.0
813 | 1210100000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
814 | 0100201000200000,0.0,-0.01,-0.023294735536431757,0.0,-0.020471686913818232,0.0,0.0,0.0,-0.01
815 | 0100201010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
816 | 0120201010200000,0.0,0.0,-0.01,-0.01,0.0028259463455373095,0.0,-0.01,0.0,-0.01
817 | 0120201011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
818 | 0100010002200000,-0.00487183853463431,-0.01,-0.01,-0.010010175819153798,-0.01,-0.0024519801025655284,-0.01,-0.01,-0.01
819 | 0100010012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
820 | 0100012012200000,-0.005335962566031877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
821 | 2100010000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
822 | 2100010002000000,-0.01,-0.01,0.0,0.0,-0.01,0.0,-0.02382780091582007,0.0,0.0
823 | 2100011002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
824 | 2100011022000000,0.0,-0.01,-0.008949324887765729,0.0,0.0,-0.01,0.0,-0.01,0.0
825 | 2100011022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
826 | 0000010000200000,-0.0205343578547743,-0.01765103803358354,-0.024598866454076572,-0.017592789952478496,-0.0199,-0.02314286882696033,-0.020132574907490645,-0.017470494099020024,-0.0199
827 | 0110220000000000,0.0,0.0,0.0,-0.01,0.0,0.0,-0.023741417951402805,0.0,0.0
828 | 0110220001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
829 | 0110220021000000,0.0,-0.01,-0.01,0.0,0.0,0.014252930482247917,-0.01,-0.01,-0.01
830 | 0110220021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
831 | 2100001010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
832 | 2100021010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
833 | 0000012001200000,0.0,0.0,0.0,0.0,0.0,0.0,-0.022826680788695226,0.0,0.0
834 | 0000112001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
835 | 0000112021200000,0.0,0.004044857462079664,0.0,0.0,0.0,-0.0199,-0.01,0.0,-0.01
836 | 1000112021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
837 | 2100210000000000,0.0,0.0,-0.01815927451430222,0.0,0.0,0.0,0.0,0.0,0.0
838 | 2100210000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
839 | 2120210000100000,0.0,-0.01,-0.01,-0.01,0.0,0.0,0.004218986820617021,0.0,0.0
840 | 2120211000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
841 | 1120000000200000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,-0.0033665231703482457,0.0
842 | 1120001000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
843 | 1120001002200000,0.0,-0.01,0.0,0.0,0.007214566117439047,0.0,0.0,0.0,0.0
844 | 1120101002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
845 | 0100000001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
846 | 2100000001200000,0.0,0.0,0.0,0.0,0.03396937505899468,0.0,0.0,0.0,0.0
847 | 2110000001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
848 | 0000101000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
849 | 0000121000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005558284031683549,0.0
850 | 1000121000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
851 | 1000121002200000,0.0,0.01712836602429331,0.0,0.0,-0.01,0.0,0.0,-0.01,0.0
852 | 1010121002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
853 | 2100100001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
854 | 0000100001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
855 | 0020100001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0022661062466253767,0.0,0.0
856 | 0120100001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
857 | 0210000010200000,0.0,0.0,0.0,0.0,0.0,0.0019982602564887165,0.0,0.0,0.0
858 | 1210000010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
859 | 1210002010200000,0.0,0.0,0.0,0.002466987970973725,0.0,0.0,0.0,0.0,0.0
860 | 1210002011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
861 | 0010001002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
862 | 2120110000200000,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,-0.00502577895994259,0.0
863 | 2120110010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
864 | 2100100010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
865 | 0020001000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
866 | 0020001002100000,0.0,0.0,-0.01,-0.010317238453628646,0.0,-0.01,0.0,-0.01,-0.01
867 | 1020001002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
868 | 1020201002100000,0.0,0.0,0.0,0.0,-0.0033794215113986206,0.0,0.0,0.0,-0.01
869 | 1020201012100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
870 | 2100000011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
871 | 2100002011200000,0.0,0.0,0.0,0.0,0.00013997320152977453,0.0,-0.01,0.0,0.0
872 | 2110002011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
873 | 1100020001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
874 | 1120020011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
875 | 0100002010200000,0.0,0.0,0.0,-0.01011762745568813,0.0,0.0,0.0,0.0,0.0
876 | 0110002010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
877 | 0000110000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
878 | 0020110000200000,-0.01,0.0,-0.01,-0.01,-0.01,0.0,0.0,-0.015590652646145514,0.0
879 | 1020110000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
880 | 0010120000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
881 | 0010120002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.03285678880493379
882 | 0010121002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
883 | 0010121002200000,-0.01,0.0,0.0,-0.01,-0.01,-0.01,0.0121520182617582,-0.0199,-0.0199
884 | 0210000001200000,0.0,0.0,0.0,0.0,0.0,-0.006767486374239051,-0.01,0.0,0.0
885 | 0210000011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
886 | 0100011002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
887 | 0100211002200000,0.0,0.0,-1.2689636057798892e-05,0.0,0.0,0.0,0.0,0.0,0.0
888 | 0100002001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
889 | 0100002001200000,0.0,0.0,0.0,0.0,-0.006185617315905748,0.0,0.0,0.0,0.0
890 | 1100002001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
891 | 0020002011000000,0.0,0.0,0.0,0.0,0.0018271605849953722,-0.01,0.0,0.0,0.0
892 | 0020102011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
893 | 0020122011000000,0.0,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.015130560257575625
894 | 0120122011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
895 | 1000002010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
896 | 0100112000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
897 | 0110002000200000,-0.03036139755809393,0.0,0.0,0.0,-0.0028699713357282857,-0.01,0.0,0.0,0.0
898 | 0110002001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
899 | 0110022001200000,0.007971389520304392,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01
900 | 0110022011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
901 | 0000202011000000,0.0,0.0,-0.00995971559288116,-0.01,0.0,0.0,0.0,-0.01,0.0
902 | 1000202011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
903 | 0010002010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
904 | 2010002010000000,-0.01,0.004658561042934544,-0.01,0.0,0.0,-0.01,0.0,0.0,0.0
905 | 2010002011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
906 | 0210100000200000,0.0,0.0,0.0,0.0,0.0,-0.0025635151475459695,0.0,0.0,0.0
907 | 2120010000000000,0.0,-0.01,0.0,-0.01,0.0,0.0025212651200520173,0.0,0.0,0.0
908 | 2120110000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
909 | 2120112000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0031126729877185393,0.0,0.0
910 | 2120112000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
911 | 0000120010200000,0.0,0.0,0.0,0.0,0.0,-0.012287268529291219,-0.01,-0.01,0.0
912 | 0010122011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
913 | 0110010002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
914 | 0120012000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.022735217212027082,0.0
915 | 0120112000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
916 | 0120112002000000,0.0044306292282650424,-0.01,-0.01,-0.01,-0.01,0.0,0.0,0.0,0.0
917 | 0000100021200000,0.0,0.002016837294002595,0.0,0.0,0.0,0.0,-0.01,0.0,-0.01
918 | 0010100021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
919 | 0210100021200000,0.0024899225851883885,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
920 | 0210101021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
921 | 0100010020200000,0.0,0.0,0.01987728175911318,0.0,0.0,-0.012957146109002329,-0.01,0.0,0.0
922 | 1100010020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
923 | 0100220010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.018329661634096356
924 | 0100220011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
925 | 0100220011200000,0.0,-0.01,0.0038231677760163303,0.0,-0.01,0.0,0.0,0.0,-0.01
926 | 2110001000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
927 | 1120200001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
928 | 1120200001200000,0.0,0.0,-0.01,0.0,0.0,0.009157598914909993,0.0,0.0,0.0
929 | 0010021000200000,0.0,0.0,0.0,0.0037875015252355334,0.0,0.0,0.0,0.0,0.0
930 | 1010021000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
931 | 1100210000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
932 | 1100000020200000,0.0,0.0,0.0,0.0,0.0,0.010584504545239907,0.0,0.0,0.0
933 | 0120001010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
934 | 0120021010200000,0.004238259053169538,0.0,0.0,0.0,-0.01,0.0,0.0,0.0,-0.01
935 | 0120021011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
936 | 2100110002200000,0.0,0.0,-0.01,0.0,-0.01,0.0009220172312356675,0.0,-0.01,-0.01
937 | 2110110002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
938 | 0000000011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
939 | 2000000011200000,0.0,0.0,0.0,-0.01,0.0,-0.023511293022311266,0.0,0.0,0.0
940 | 2000100011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
941 | 2000102011200000,0.0,0.0,0.015067735236590711,-0.01,0.0,-0.01,-0.01,-0.01,-0.01
942 | 2000112011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
943 | 1000000010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
944 | 1000002010200000,0.0,0.0,0.0,0.0,-0.020421929152257948,0.0,0.0,0.0,0.0
945 | 1010002010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
946 | 1010022010200000,0.0,0.0,0.0,-0.0010372895818428287,-0.01,-0.01,0.0,-0.01,0.0
947 | 1010022011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
948 | 0110201000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
949 | 0110221000200000,0.010922370723838103,-0.01,-0.01,-0.01,0.0,0.0,0.0,-0.01,0.0
950 | 0110221001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
951 | 0020000011200000,0.0,0.0,0.0,-0.01,-0.016209305565549666,0.0,0.0,0.0,0.0
952 | 0020120011200000,0.008748876446246261,0.0,0.0,-0.01,-0.01,0.0,0.0,0.0,-0.01
953 | 0020121011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
954 | 0110010020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
955 | 0110010022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
956 | 0100012010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
957 | 0000102010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
958 | 0000102010200000,0.0,0.0,0.0,0.0,0.004156921876042255,0.0,0.0,0.0,0.0
959 | 0000102011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
960 | 0100012020000000,0.0,0.0,0.0,0.0,-0.01,0.0,-0.01,-0.023709610934279177,-0.01
961 | 0100012022100000,-0.008674762786877899,0.0,0.0,0.0,-0.01,0.0,-0.01,-0.01,0.0
962 | 0110012022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
963 | 0110002020000000,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,0.0006359039075772157,0.0
964 | 0110102020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
965 | 0110102022000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.013314896978474653
966 | 0110112022000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
967 | 0010002001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
968 | 0010202001000000,0.0,0.0,0.0,0.0,0.0,0.0,-0.007276852826537234,0.0,0.0
969 | 0010202001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
970 | 0010202021100000,0.013894575641440483,-0.01,0.0,0.0,0.0,-0.01,0.0,0.0,0.0
971 | 0110202021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
972 | 0120002010000000,0.004636700005188554,0.0,-0.01,0.0,0.0,0.0,-0.01,0.0,0.0
973 | 0120002011000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
974 | 0110102000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
975 | 2110102000200000,-0.01,0.0,-0.01,-0.01,-0.01,-0.01,0.0010043554321194485,0.0,-0.01
976 | 2110112000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
977 | 0000001010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
978 | 2000001010200000,-0.01,0.0,0.004034697048600542,0.0,0.0,0.0,0.0,0.0,0.0
979 | 2000001011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
980 | 2020001011200000,0.0,0.0,0.0,0.004981107467408085,0.0,0.0,0.0,0.0,0.0
981 | 2120001011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
982 | 1000000012200000,0.0,0.0,0.0,0.0,0.004689745047794735,0.0,0.0,0.0,0.0
983 | 1000001012200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
984 | 0100100020200000,0.0,0.0,0.027884324844275363,0.0,0.0,0.0,-0.01,0.0,-0.01
985 | 1100100002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
986 | 1120100002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.014685460227425506,-0.01,-0.01
987 | 1010120000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
988 | 1210120000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0031129611306754098,-0.01,0.0
989 | 1210120001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
990 | 0100200000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
991 | 0120200000100000,0.0,0.0,0.0,0.0,0.0,-0.027214035394645526,0.0,0.0,0.0
992 | 1120200000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
993 | 1120202000100000,-0.01,0.0,-0.01,-0.01,0.014762237659904648,-0.01,-0.01,0.0,-0.01
994 | 1120202010100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
995 | 0100100021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
996 | 0120100021200000,0.0,0.0,0.0,0.0,0.013817236099915425,0.0,0.0,0.0,0.0
997 | 0120101021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
998 | 0100000021200000,0.0,-0.01,0.0,0.0,-0.03320015920828655,0.0,0.0,-0.01,-0.01
999 | 0100120021200000,0.010081452191736943,-0.01,0.0,-0.01,-0.0199,-0.01,-0.01,-0.01,-0.01
1000 | 0100121021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1001 | 0110021000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1002 | 0200101000200000,0.0,0.0,0.0,0.0,0.0,0.0,-0.009588854696337388,0.0,0.0
1003 | 0200101001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1004 | 0200101021200000,-0.0022692496514820854,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.0
1005 | 0100001002200000,0.0,0.0,0.0,0.003713932234949195,0.0,0.0,0.0,-0.01,0.0
1006 | 1100001002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1007 | 0100101020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1008 | 0120101020200000,0.003408943263326005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1009 | 0110000020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.019103447416158784,-0.01
1010 | 0110100020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1011 | 0100120000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1012 | 2100120000000000,-0.01,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.007649999057350948
1013 | 2100120010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1014 | 2010011000200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1015 | 2010011020200000,0.0,0.0,0.0,0.007228545548629117,0.0,0.0,0.0,0.0,-0.01
1016 | 2110011020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1017 | 0200010001200000,0.0,0.0,0.0,-0.007664726490841305,0.0,0.0,0.0,0.0,0.0
1018 | 0210010001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1019 | 0000200021100000,0.0,0.0,0.003478573335719931,0.0,0.0,0.0,0.0,0.0,-0.01
1020 | 0000210021100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1021 | 0000011020200000,0.0,0.0,-0.0011683973500637755,0.0,0.0,0.0,0.0,0.0,0.0
1022 | 0010010020200000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.00017461000395835915,0.0
1023 | 0010110020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1024 | 0100112020200000,-0.01,0.0,0.0,0.0,-0.01,0.0,-0.01,0.014734473323866145,0.0
1025 | 1100112020200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1026 | 2010000010200000,0.0,0.0010586727212863918,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1027 | 2010100010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1028 | 2210100010200000,0.0,0.0,0.0,0.0,0.013959263406929425,-0.01,0.0,0.0,0.0
1029 | 2210101010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1030 | 0100001021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1031 | 2000100010200000,0.0,0.0,-0.01426180476747794,0.0,0.0,0.0,0.0,0.0,-0.01
1032 | 2000101010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1033 | 0120011002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1034 | 0120211002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.01395341186578227
1035 | 1120211002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1036 | 0000211002000000,0.0,0.0,-0.010135835071868105,0.0,-0.01,0.0,0.0,-0.01,-0.01
1037 | 0000211001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1038 | 0000211021200000,0.0,-0.009707216427865773,0.0,0.0,0.0,0.0,-0.01,0.0,0.0
1039 | 0010211021200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1040 | 2100010010200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1041 | 0000010002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1042 | 2000010002100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.015105227572943986,0.0,0.0
1043 | 2100010002100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1044 | 2100010022100000,-0.01,0.0,0.0,0.0,0.0,-0.0008358925505257986,0.0,0.0,-0.01
1045 | 2100110022100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1046 | 0010201002000000,0.0,0.0,-0.01,0.0,0.0,-0.01,0.0,-0.01,-0.022592712961013076
1047 | 0010211002000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1048 | 0010211002200000,-0.01,-0.006080144782718924,0.0,-0.01,0.0,0.0,0.0,0.0,-0.01
1049 | 1010211002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1050 | 0120010020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1051 | 2120010020100000,0.0,0.0,0.0,0.0031664129800112704,-0.01,-0.01,0.0,0.0,0.0
1052 | 2120011020100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1053 | 0020011020000000,0.0,-0.01,-0.01,-0.016272298128001246,0.0,0.0,0.0,0.0,0.0
1054 | 0120011020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1055 | 0120211020000000,0.0,0.0,0.0,0.0,-0.01,-0.01,0.0,0.0,-0.0030319416981400573
1056 | 1120211020000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1057 | 0010012000200000,0.0,0.0,0.0,0.0038676953990537293,0.0,0.0,0.0,-0.01,0.0
1058 | 0010012001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1059 | 0000211000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1060 | 0210211020000000,0.0,0.0,0.0,-0.0199,-0.01,-0.01,0.0,0.0,-0.005541992641591087
1061 | 0210211021000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1062 | 2000200001100000,0.0,0.007704409296335132,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1063 | 2000210001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1064 | 2200210001100000,0.0,0.0,0.009511616415228557,0.0,0.0,0.0,0.0,0.0,0.0
1065 | 2200211001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1066 | 0200110000200000,0.0,0.0,0.0,-0.01,0.0,0.0,-0.00482332900607239,-0.01,-0.01
1067 | 0200110001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1068 | 0020012001000000,-0.003400157847154919,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,0.0
1069 | 0020112001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1070 | 0000012000100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1071 | 0000212000100000,0.0,0.0,0.0,0.0,0.0,0.0,-0.010139398406497422,0.0,0.0
1072 | 0000212001100000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1073 | 0000212001000000,0.0,0.0,-0.01575724685568875,0.0,0.0,0.0,0.0,0.0,-0.01
1074 | 0000002011200000,0.0,0.0,0.0,0.0,-0.016476305755410335,0.0,-0.01,-0.01,-0.01
1075 | 0100002011200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1076 | 0100022011200000,-0.0034158180407803038,0.0,0.0,0.0,0.0,-0.01,0.0,-0.01,0.0
1077 | 2200010001000000,0.0,0.0,-0.01,0.0,0.0,0.0,0.0,0.0,-0.02347958961199796
1078 | 2210010001000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1079 | 2210010001200000,0.0,-0.01,0.0,-0.01,0.0,0.0,-0.008140410072112017,-0.01,0.0
1080 | 2210110001200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1081 | 1020012010000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1082 | 0100110002200000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1083 | 1000122000000000,0.0,0.0,0.0,0.0,0.0,-0.01,0.0,0.0,0.0
1084 | 1100122000000000,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
1085 |
--------------------------------------------------------------------------------
/QTable/五子棋/train.py:
--------------------------------------------------------------------------------
1 | import QTable.五子棋.chessboard as cb
2 | from QTable.走迷宫.SarsaLambda import SarsaLambdaTable
3 | import numpy as np
4 |
5 | board = cb.ChessBoard(3, train=False, win_cnt=3)
6 | board.render()
7 |
8 | epochs = 1000
9 |
10 | actions = []
11 | func = {}
12 | size = len(board.map) - 1
13 | print('size:', size)
14 | for i in range(size):
15 | for j in range(size):
16 | actions.append(str([i, j]))
17 | func[str([i, j])] = [i, j]
18 |
19 | # ai1 = AI.QLearning(actions=actions, map=board.get_map())
20 | # ai1.load('q_table1.csv')
21 | # observation = board.reset()
22 | #
23 | # while True:
24 | # board.reset()
25 | # while True:
26 | # print(str(board.convert()))
27 | # while True:
28 | # action = ai1.choose_action(str(board.convert()), test=True)
29 | # print(action)
30 | # observation_, reward, error = board.step(role=1, index=np.array(func[action]))
31 | # if error:
32 | # break
33 | # if board.winner != 0:
34 | # break
35 | # while not board.next:
36 | # board.render()
37 | # board.next = False
38 | # if board.winner != 0:
39 | # break
40 | #
41 | ai1 = SarsaLambdaTable(actions=actions)
42 | ai2 = SarsaLambdaTable(actions=actions)
43 | # ai1.load('q_table1.csv')
44 | # ai2.load('q_table2.csv')
45 |
46 | for epoch in range(epochs):
47 | observation = board.reset()
48 | print('epoch: ', epoch)
49 | while True:
50 | board.render()
51 | action = ai1.choose_action(str(observation))
52 | while True:
53 | observation_, reward, error = board.step(role=1, index=np.array(func[action]))
54 | if error:
55 | break
56 | if reward == -1 and error:
57 | break
58 | # print(reward)
59 | action_ = ai1.choose_action(str(observation))
60 | ai1.learn(str(observation), action, reward, str(observation_), action_)
61 | action = ai1.choose_action(str(observation))
62 | action_ = ai1.choose_action(str(observation))
63 | ai1.learn(str(observation), action, reward, str(observation_), action_)
64 |
65 | observation = observation_
66 | if reward == -1 and error:
67 | print('winner: ai2')
68 | break
69 | elif reward == 1:
70 | print('winner: ai1')
71 | break
72 |
73 | action = ai2.choose_action(str(observation))
74 | while True:
75 | observation_, reward, error = board.step(role=2, index=np.array(func[action]))
76 | if error:
77 | break
78 | if reward == -1 and error:
79 | break
80 | # print(reward)
81 | action_ = ai2.choose_action(str(observation))
82 | ai2.learn(str(observation), action, reward, str(observation_), action_)
83 | action = ai2.choose_action(str(observation))
84 | action_ = ai2.choose_action(str(observation))
85 | ai2.learn(str(observation), action, reward, str(observation_), action_)
86 |
87 | observation = observation_
88 | if reward == -1 and error:
89 | print('winner: ai1')
90 | break
91 | elif reward == 1:
92 | print('winner: ai2')
93 | break
94 |
95 | ai1.save('q_table1.csv')
96 | ai2.save('q_table2.csv')
97 |
98 |
99 | while True:
100 | board.reset()
101 | while True:
102 | print(str(board.convert()))
103 | while True:
104 | action = ai1.choose_action(str(board.convert()))
105 | print(action)
106 | observation_, reward, error = board.step(role=1, index=np.array(func[action]))
107 | if error:
108 | break
109 | if board.winner != 0 or board.cnt >= board.full:
110 | break
111 | while not board.next:
112 | board.render()
113 | board.next = False
114 | if board.winner != 0 or board.cnt >= board.full:
115 | break
116 |
--------------------------------------------------------------------------------
/QTable/走迷宫/RL_brain.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 |
5 | class QLearningTable:
6 | """
7 | q_table 的 index 由动作组成
8 | """
9 | def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
10 | # 初始动作；学习率；反馈比率；非随机比率；QTable初始化
11 | self.actions = actions # a list
12 | self.lr = learning_rate
13 | self.gamma = reward_decay
14 | self.epsilon = e_greedy
15 | self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)
16 |
17 | def choose_action(self, observation):
18 | self.check_state_exist(observation)
19 | # (1 - epsilon) 几率随机走，epsilon 几率通过 QLearning 选择行为
20 | if np.random.uniform() < self.epsilon:
21 | # 行为位置，列为动作；选择当前位置所有动作；值相等，随机选择
22 | state_action = self.q_table.loc[observation, :]
23 | action = np.random.choice(state_action[state_action == np.max(state_action)].index)
24 | else:
25 | action = np.random.choice(self.actions)
26 |
27 | return action
28 |
29 | def learn(self, s, action, reward, s_):
30 | # 检查是否存在这个位置的动作，不存在则添加
31 | self.check_state_exist(s_)
32 | q_predict = self.q_table.loc[s, action]
33 | if s_ != 'terminal':
34 | q_target = reward + self.gamma * self.q_table.loc[s_, :].max()
35 | else:
36 | q_target = reward
37 | # q_target 是回馈加上下一步的得分 * gamma；如果 q_target 高于当前，说明下一步值得走
38 | self.q_table.loc[s, action] += self.lr * (q_target - q_predict)
39 |
40 | def check_state_exist(self, state):
41 | # 如果状态不存在，添加该行
42 | print(self.q_table.index)
43 |
44 | if state not in self.q_table.index:
45 | self.q_table = self.q_table.append(
46 | pd.Series(
47 | [0]*len(self.actions),
48 | index=self.q_table.columns,
49 | name=state,
50 | )
51 | )
--------------------------------------------------------------------------------
/QTable/走迷宫/RunQTable.py:
--------------------------------------------------------------------------------
1 | from QTable.走迷宫.maze_env import Maze
2 | import 五子棋.AI_Brain as AI
3 |
4 |
5 | def update():
6 | for episode in range(100):
7 | # initial observation
8 | observation = env.reset()
9 |
10 | while True:
11 | # fresh env
12 | env.render()
13 |
14 | # RL choose action based on observation
15 | action = RL.choose_action(str(observation))
16 |
17 | # RL take action and get next observation and reward
18 | observation_, reward, done = env.step(action)
19 |
20 | # RL learn from this transition
21 | RL.learn(str(observation), action, reward, str(observation_))
22 |
23 | # swap observation
24 | observation = observation_
25 |
26 | # break while loop when end of this episode
27 | if done:
28 | break
29 |
30 | # end of game
31 | print('game over')
32 | env.destroy()
33 |
34 |
35 | if __name__ == "__main__":
36 | env = Maze()
37 | # RL = QLearningTable(actions=list(range(env.n_actions)))
38 | RL = AI.QLearning(actions=list(range(env.n_actions)))
39 |
40 | env.after(100, update)
41 | env.mainloop()
42 |
--------------------------------------------------------------------------------
/QTable/走迷宫/RunSarsaTable.py:
--------------------------------------------------------------------------------
1 | """
2 | Sarsa is a online updating method for Reinforcement learning.
3 | Unlike Q learning which is a offline updating method, Sarsa is updating while in the current trajectory.
4 | You will see the sarsa is more coward when punishment is close because it cares about all behaviours,
5 | while q learning is more brave because it only cares about maximum behaviour.
6 | """
7 |
8 | from QTable.走迷宫.maze_env import Maze
9 | from QTable.走迷宫.SarsaLearning import SarsaTable
10 |
11 |
12 | def update():
13 | for episode in range(100):
14 | # initial observation
15 | observation = env.reset()
16 |
17 | # RL choose action based on observation
18 | action = RL.choose_action(str(observation))
19 |
20 | while True:
21 | # fresh env
22 | env.render()
23 |
24 | # RL take action and get next observation and reward
25 | observation_, reward, done = env.step(action)
26 |
27 | # RL choose action based on next observation
28 | action_ = RL.choose_action(str(observation_))
29 |
30 | # RL learn from this transition (s, a, r, s, a) ==> Sarsa
31 | RL.learn(str(observation), action, reward, str(observation_), action_)
32 |
33 | # swap observation and action
34 | observation = observation_
35 | action = action_
36 |
37 | # break while loop when end of this episode
38 | if done:
39 | break
40 |
41 | # end of game
42 | print('game over')
43 | env.destroy()
44 |
45 |
46 | if __name__ == "__main__":
47 | env = Maze()
48 | RL = SarsaTable(actions=list(range(env.n_actions)))
49 |
50 | env.after(100, update)
51 | env.mainloop()
--------------------------------------------------------------------------------
/QTable/走迷宫/SarsaLambda.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 |
5 | class RL(object):
6 | def __init__(self, action_space, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
7 | self.actions = action_space # a list
8 | self.lr = learning_rate
9 | self.gamma = reward_decay
10 | self.epsilon = e_greedy
11 |
12 | self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)
13 |
14 | def check_state_exist(self, state):
15 | if state not in self.q_table.index:
16 | # append new state to q table
17 | self.q_table = self.q_table.append(
18 | pd.Series(
19 | [0]*len(self.actions),
20 | index=self.q_table.columns,
21 | name=state,
22 | )
23 | )
24 |
25 | def choose_action(self, observation):
26 | self.check_state_exist(observation)
27 | # action selection
28 | if np.random.rand() < self.epsilon:
29 | # choose best action
30 | state_action = self.q_table.loc[observation, :]
31 | # some actions may have the same value, randomly choose on in these actions
32 | action = np.random.choice(state_action[state_action == np.max(state_action)].index)
33 | else:
34 | # choose random action
35 | action = np.random.choice(self.actions)
36 | return action
37 |
38 | def learn(self, *args):
39 | pass
40 |
41 | def save(self, filename):
42 | self.q_table.to_csv(filename, index=True)
43 |
44 | def load(self, filename):
45 | self.q_table = pd.read_csv(filename, index_col=0)
46 |
47 |
48 | # backward eligibility traces
49 | class SarsaLambdaTable(RL):
50 | def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9, trace_decay=0.9):
51 | super(SarsaLambdaTable, self).__init__(actions, learning_rate, reward_decay, e_greedy)
52 |
53 | # backward view, eligibility trace.
54 | self.lambda_ = trace_decay
55 | self.eligibility_trace = self.q_table.copy()
56 |
57 | def check_state_exist(self, state):
58 | if state not in self.q_table.index:
59 | # append new state to q table
60 | to_be_append = pd.Series(
61 | [0] * len(self.actions),
62 | index=self.q_table.columns,
63 | name=state,
64 | )
65 | self.q_table = self.q_table.append(to_be_append)
66 |
67 | # also update eligibility trace
68 | self.eligibility_trace = self.eligibility_trace.append(to_be_append)
69 |
70 | def learn(self, s, a, r, s_, a_):
71 | self.check_state_exist(s_)
72 | q_predict = self.q_table.loc[s, a]
73 | if s_ != 'terminal':
74 | q_target = r + self.gamma * self.q_table.loc[s_, a_] # next state is not terminal
75 | else:
76 | q_target = r # next state is terminal
77 | error = q_target - q_predict
78 |
79 | # increase trace amount for visited state-action pair
80 |
81 | # Method 1:
82 | # self.eligibility_trace.loc[s, a] += 1
83 |
84 | # Method 2:
85 | self.eligibility_trace.loc[s, :] *= 0
86 | self.eligibility_trace.loc[s, a] = 1
87 |
88 | # Q update
89 | self.q_table += self.lr * error * self.eligibility_trace
90 |
91 | # decay eligibility trace after update
92 | self.eligibility_trace *= self.gamma*self.lambda_
--------------------------------------------------------------------------------
/QTable/走迷宫/SarsaLearning.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 |
5 | class RL(object):
6 | def __init__(self, action_space, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
7 | self.actions = action_space # a list
8 | self.lr = learning_rate
9 | self.gamma = reward_decay
10 | self.epsilon = e_greedy
11 |
12 | self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)
13 |
14 | def check_state_exist(self, state):
15 | if state not in self.q_table.index:
16 | # append new state to q table
17 | self.q_table = self.q_table.append(
18 | pd.Series(
19 | [0]*len(self.actions),
20 | index=self.q_table.columns,
21 | name=state,
22 | )
23 | )
24 |
25 | def choose_action(self, observation):
26 | self.check_state_exist(observation)
27 | # action selection
28 | if np.random.rand() < self.epsilon:
29 | # choose best action
30 | state_action = self.q_table.loc[observation, :]
31 | # some actions may have the same value, randomly choose on in these actions
32 | action = np.random.choice(state_action[state_action == np.max(state_action)].index)
33 | else:
34 | # choose random action
35 | action = np.random.choice(self.actions)
36 | return action
37 |
38 | def learn(self, *args):
39 | pass
40 |
41 |
42 | # off-policy
43 | class QLearningTable(RL):
44 | def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
45 | super(QLearningTable, self).__init__(actions, learning_rate, reward_decay, e_greedy)
46 |
47 | def learn(self, s, a, r, s_):
48 | self.check_state_exist(s_)
49 | q_predict = self.q_table.loc[s, a]
50 | if s_ != 'terminal':
51 | q_target = r + self.gamma * self.q_table.loc[s_, :].max() # next state is not terminal
52 | else:
53 | q_target = r # next state is terminal
54 | self.q_table.loc[s, a] += self.lr * (q_target - q_predict) # update
55 |
56 |
57 | # on-policy
58 | class SarsaTable(RL):
59 |
60 | def __init__(self, actions, learning_rate=0.01, reward_decay=0.9, e_greedy=0.9):
61 | super(SarsaTable, self).__init__(actions, learning_rate, reward_decay, e_greedy)
62 |
63 | def learn(self, state, a, r, next_state, a_):
64 | self.check_state_exist(next_state)
65 | q_predict = self.q_table.loc[state, a]
66 | if next_state != 'terminal':
67 | q_target = r + self.gamma * self.q_table.loc[next_state, a_] # next state is not terminal
68 | else:
69 | q_target = r # next state is terminal
70 | self.q_table.loc[state, a] += self.lr * (q_target - q_predict) # update
--------------------------------------------------------------------------------
/QTable/走迷宫/maze_env.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import time
3 | import sys
4 | if sys.version_info.major == 2:
5 | import Tkinter as tk
6 | else:
7 | import tkinter as tk
8 |
9 |
10 | UNIT = 40 # pixels
11 | MAZE_H = 4 # grid height
12 | MAZE_W = 4 # grid width
13 |
14 |
15 | class Maze(tk.Tk, object):
16 | def __init__(self):
17 | super(Maze, self).__init__()
18 | self.action_space = ['u', 'd', 'l', 'r']
19 | self.n_actions = len(self.action_space)
20 | self.title('maze')
21 | self.geometry('{0}x{1}'.format(MAZE_H * UNIT, MAZE_H * UNIT))
22 | self._build_maze()
23 |
24 | def _build_maze(self):
25 | self.canvas = tk.Canvas(self, bg='white',
26 | height=MAZE_H * UNIT,
27 | width=MAZE_W * UNIT)
28 |
29 | # create grids
30 | for c in range(0, MAZE_W * UNIT, UNIT):
31 | x0, y0, x1, y1 = c, 0, c, MAZE_H * UNIT
32 | self.canvas.create_line(x0, y0, x1, y1)
33 | for r in range(0, MAZE_H * UNIT, UNIT):
34 | x0, y0, x1, y1 = 0, r, MAZE_W * UNIT, r
35 | self.canvas.create_line(x0, y0, x1, y1)
36 |
37 | # create origin
38 | origin = np.array([20, 20])
39 |
40 | # hell
41 | hell1_center = origin + np.array([UNIT * 2, UNIT])
42 | self.hell1 = self.canvas.create_rectangle(
43 | hell1_center[0] - 15, hell1_center[1] - 15,
44 | hell1_center[0] + 15, hell1_center[1] + 15,
45 | fill='black')
46 | # hell
47 | hell2_center = origin + np.array([UNIT, UNIT * 2])
48 | self.hell2 = self.canvas.create_rectangle(
49 | hell2_center[0] - 15, hell2_center[1] - 15,
50 | hell2_center[0] + 15, hell2_center[1] + 15,
51 | fill='black')
52 |
53 | # create oval
54 | oval_center = origin + UNIT * 2
55 | self.oval = self.canvas.create_oval(
56 | oval_center[0] - 15, oval_center[1] - 15,
57 | oval_center[0] + 15, oval_center[1] + 15,
58 | fill='yellow')
59 |
60 | # create red rect
61 | self.rect = self.canvas.create_rectangle(
62 | origin[0] - 15, origin[1] - 15,
63 | origin[0] + 15, origin[1] + 15,
64 | fill='red')
65 |
66 | # pack all
67 | self.canvas.pack()
68 |
69 | def reset(self):
70 | self.update()
71 | time.sleep(0.1)
72 | self.canvas.delete(self.rect)
73 | origin = np.array([20, 20])
74 | self.rect = self.canvas.create_rectangle(
75 | origin[0] - 15, origin[1] - 15,
76 | origin[0] + 15, origin[1] + 15,
77 | fill='red')
78 | # return observation
79 | return self.canvas.coords(self.rect)
80 |
81 | def step(self, action):
82 | s = self.canvas.coords(self.rect)
83 | base_action = np.array([0, 0])
84 | if action == 0: # up
85 | if s[1] > UNIT:
86 | base_action[1] -= UNIT
87 | elif action == 1: # down
88 | if s[1] < (MAZE_H - 1) * UNIT:
89 | base_action[1] += UNIT
90 | elif action == 2: # right
91 | if s[0] < (MAZE_W - 1) * UNIT:
92 | base_action[0] += UNIT
93 | elif action == 3: # left
94 | if s[0] > UNIT:
95 | base_action[0] -= UNIT
96 |
97 | self.canvas.move(self.rect, base_action[0], base_action[1]) # move agent
98 |
99 | s_ = self.canvas.coords(self.rect) # next state
100 |
101 | # reward function
102 | if s_ == self.canvas.coords(self.oval):
103 | reward = 1
104 | done = True
105 | s_ = 'terminal'
106 | elif s_ in [self.canvas.coords(self.hell1), self.canvas.coords(self.hell2)]:
107 | reward = -1
108 | done = True
109 | s_ = 'terminal'
110 | else:
111 | reward = 0
112 | done = False
113 |
114 | return s_, reward, done
115 |
116 | def render(self):
117 | time.sleep(0.1)
118 | self.update()
119 |
120 |
121 | def update():
122 | for t in range(10):
123 | s = env.reset()
124 | while True:
125 | env.render()
126 | a = 1
127 | s, r, done = env.step(a)
128 | if done:
129 | break
130 |
131 | if __name__ == '__main__':
132 | env = Maze()
133 | env.after(100, update)
134 | env.mainloop()
--------------------------------------------------------------------------------