├── README.md
└── algorithm
    ├── policy gradient
        ├── A2C.py
        ├── A3C
        │   ├── main.py
        │   ├── model.py
        │   └── utils.py
        ├── Actor_Critic.py
        ├── DDPG
        │   ├── experience_replay.py
        │   ├── main.py
        │   └── model.py
        ├── DDPG_discrete
        │   ├── experience_replay.py
        │   ├── gumbel_softmax.py
        │   ├── main.py
        │   └── model.py
        ├── REINFORCE.py
        ├── SAC
        │   ├── experience_replay.py
        │   ├── main.py
        │   └── model.py
        ├── TD3
        │   ├── experience_replay.py
        │   ├── main.py
        │   └── model.py
        ├── TRPO
        │   ├── main.py
        │   └── model.py
        └── baseline_REINFORCE.py
    └── value-based
        ├── DQN.py
        ├── DoubleDQN.py
        ├── DuelingDQN.py
        └── Sarsa.py


/README.md:
--------------------------------------------------------------------------------
 1 | # Deep-RL-with-pytorch
 2 | Practice for deep reinforcement learning algorithms by a starter.  
 3 | Test environment is Gym-CartPolev0 for discrete action space and Gym-PendulmV0 for continuous action space.  
 4 | Under active development.  
 5 | Including:DQN, REINFORCE, baseline-REINFORCE, Actor-Critic, Double DQN, Dueling DQN, Sarsa, DDPG, DDPG for discrete action space, A2C, A3C, TD3, SAC, TRPO
 6 | 
 7 | ## 2020-9-19 implement
 8 | ### algorithm:  
 9 |   1.DQN  
10 |   2.REINFORCE 
11 | ### components:  
12 | 1.experience replay
13 | ## 2020-9-20 implement
14 | ### algorithm:
15 |   1.baseline-REINFORCE  
16 |   2.Actor-Critic  
17 |   
18 |   Add CUDA support
19 | ## 2021-1-15 implement
20 | ### algorithm:  
21 |   1.Double DQN  
22 |   2.Dueling DQN 
23 | ## 2021-1-19 implement
24 | ### algorithm:  
25 |   1.Sarsa
26 | ## 2021-1-23 implement
27 | ### algorithm:  
28 |   1.DDPG  
29 |   2.DDPG for discrete action space using gumbel softmax
30 | ## 2021-1-26 implement
31 | ### algorithm:  
32 |   1.A2C
33 | ## 2021-1-27 implement
34 | ### algorithm:  
35 |   1.A3C
36 | ## 2021-2-4 implement
37 | ### algorithm:  
38 |   1.TD3  
39 |   2.SAC
40 | ## 2021-2-25 implement
41 | ### algorithm:  
42 |   1.TRPO(Natural Policy gradient).  
43 |   Unknown bug exists: Hessian matrix may not be positive definite at the beginning of training(But the training will usually converge)
44 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/A2C.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | import torch.nn as nn
  5 | import torch.nn.functional as F
  6 | from torch.distributions.categorical import Categorical
  7 | lr=0.001
  8 | gamma=0.9
  9 | hidden=32
 10 | env=gym.make('CartPole-v0')
 11 | device="cuda"
 12 | env=env.unwrapped
 13 | n_action=env.action_space.n
 14 | n_state=env.observation_space.shape[0]
 15 | 
 16 | class actor(nn.Module):      #policy net
 17 |     def __init__(self):
 18 |         super(actor,self).__init__()
 19 |         self.fc1=nn.Linear(n_state,hidden)
 20 |         self.fc2=nn.Linear(hidden,n_action)
 21 |         self.softmax=nn.Softmax()
 22 |     def forward(self,x):
 23 |         x=self.fc1(x)
 24 |         x=F.relu(x)
 25 |         x=self.fc2(x)
 26 |         prob=self.softmax(x)
 27 |         return prob
 28 | 
 29 | 
 30 | class Q(nn.Module):     
 31 |     def __init__(self):
 32 |         super(Q,self).__init__()
 33 |         self.q=nn.Sequential(nn.Linear(n_state,hidden),
 34 |                                        nn.ReLU(),
 35 |                              nn.Linear(hidden,n_action))
 36 |     def forward(self,x):
 37 |         q=self.q(x)
 38 |         return q
 39 | 
 40 | class V(nn.Module):     
 41 |     def __init__(self):
 42 |         super(V,self).__init__()
 43 |         self.v=nn.Sequential(nn.Linear(n_state,hidden),
 44 |                              nn.ReLU(),
 45 |                              nn.Linear(hidden,1))
 46 |     def forward(self,x):
 47 |         v=self.v(x)
 48 |         return v
 49 | 
 50 | class critic(nn.Module):     
 51 |     def __init__(self):
 52 |         super(critic,self).__init__()
 53 |         self.v=V()
 54 |         self.q=Q()
 55 | 
 56 |     def forward(self,x):
 57 |         v=self.v(x)
 58 |         q=self.q(x)
 59 |         advantage=q-v.repeat(2)
 60 |         return advantage
 61 | 
 62 | 
 63 | class AC():
 64 |     def __init__(self):
 65 |         self.actor=actor().to(device)
 66 |         self.critic=critic().to(device)
 67 | 
 68 |         self.Aoptimizer=th.optim.Adam(self.actor.parameters(),lr=lr)
 69 |         self.Qoptimizer=th.optim.Adam(self.critic.q.parameters(),lr=lr)
 70 |         self.Voptimizer=th.optim.Adam(self.critic.v.parameters(),lr=lr)
 71 | 
 72 |     def choose_action(self,s):
 73 |         s=th.FloatTensor(s).to(device)
 74 |         a_prob=self.actor(s)
 75 |         dist=Categorical(a_prob)
 76 |         action=dist.sample().tolist()
 77 |         return action
 78 | 
 79 |     def actor_learn(self,s,a,A):
 80 |         s=th.FloatTensor(s).to(device)
 81 |         a_prob=self.actor(s)[a]
 82 |         loss=-(th.log(a_prob)*A.detach())
 83 | 
 84 |         self.Aoptimizer.zero_grad()
 85 |         loss.backward()
 86 |         self.Aoptimizer.step()
 87 | 
 88 |     def critic_learn(self,transition):    #transition=[s,[r],[a],s_,[done]]
 89 |         s=th.FloatTensor(transition[0]).to(device)
 90 |         r=transition[1][0]
 91 |         s_=th.FloatTensor(transition[3]).to(device)
 92 |         done=transition[4][0]
 93 | 
 94 |         a=transition[2][0]
 95 |         q=self.critic.q(s)[a]
 96 |         v=self.critic.v(s)
 97 |         A=q-v
 98 |         v_=self.critic.v(s_)*gamma+r
 99 |         if not done:
100 |             q_target=th.max(self.critic.q(s_))*gamma+r   
101 |             loss_q=(q-q_target.detach())**2
102 |         else:
103 |             q_target=r
104 |             loss_q=(q-q_target)**2
105 |         loss_v=(v-v_.detach())**2
106 |         #print(loss)
107 |         self.Qoptimizer.zero_grad()
108 |         loss_q.backward()
109 |         self.Qoptimizer.step()
110 |         self.Voptimizer.zero_grad()
111 |         loss_v.backward()
112 |         self.Voptimizer.step()
113 |         return A
114 |     
115 | 
116 | ac=AC()
117 |     
118 | for episode in range(10000):
119 |     t=0
120 |     s=env.reset()
121 |     total_reward=0
122 |     while(t<300):
123 |         a=ac.choose_action(s)
124 |         s_,r,done,_=env.step(a)
125 |         total_reward+=r
126 |         transition=[s,[r],[a],s_,[done]]
127 | 
128 |         A=ac.critic_learn(transition)
129 |         ac.actor_learn(s,a,A)
130 |         if done:
131 |             break
132 |         s=s_
133 |     if(episode%10==0):
134 |         print("Episode:"+format(episode)+",score:"+format(total_reward))
135 | 
136 | 
137 |         
138 | 
139 | 
140 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/A3C/main.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | from utils import SharedAdam
 3 | from model import Worker,Net
 4 | import gym
 5 | import torch.multiprocessing as mp
 6 | 
 7 | env=gym.make('CartPole-v0')
 8 | n_action=env.action_space.n
 9 | n_state=env.observation_space.shape[0]
10 | 
11 | global_net=Net(n_state,n_action)
12 | global_net.share_memory()
13 | optA=SharedAdam(global_net.policy.parameters(), lr=1e-4, betas=(0.92, 0.999))
14 | optC=SharedAdam(global_net.v.parameters(), lr=1e-4, betas=(0.92, 0.999))
15 | workers=[Worker(global_net,optA,optC,str(i)) for i in range(8)]
16 | [w.start() for w in workers]
17 | 
18 | [w.join() for w in workers]
19 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/A3C/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th 
 2 | import torch.nn as nn
 3 | import torch.nn.functional as F
 4 | import multiprocessing as mp
 5 | import gym
 6 | from utils import update_global
 7 | 
 8 | max_lstep=200
 9 | update_interval=10
10 | 
11 | class Net(nn.Module):
12 |     def __init__(self,n_state,n_action):
13 |         super(Net,self).__init__()
14 |         self.n_state=n_state
15 |         self.n_action=n_action
16 |         self.v=nn.Sequential(
17 |                               nn.Linear(n_state,256),
18 |                               nn.Linear(256,1))
19 |         self.policy=nn.Sequential(
20 |                               nn.Linear(n_state,256),
21 |                               nn.Linear(256,n_action))
22 |     def forward(self,x):
23 |         value=self.v(x)
24 |         prob=F.softmax(self.policy(x))
25 |         #print(prob)
26 |         return value,prob
27 | 
28 | class Worker(mp.Process):
29 |     def __init__(self,gnet,optA,optC,name):
30 |         super(Worker,self).__init__()
31 |         self.name='Worker '+name
32 |         self.optA=optA
33 |         self.optC=optC
34 |         self.env=gym.make('CartPole-v0')
35 |         self.lnet=Net(self.env.observation_space.shape[0],self.env.action_space.n)
36 |         self.gnet=gnet
37 |         self.queue=[]
38 |         self.max_episode=1000
39 |         self.cur_episode=0
40 |     def choose_action(self,s):
41 |         _,prob=self.lnet(th.Tensor(s))
42 |         dist=th.distributions.categorical.Categorical(prob)
43 |         a=dist.sample().tolist()
44 |         return a
45 | 
46 |     def run(self):
47 |         buffer_a,buffer_s,buffer_r,buffer_s_,buffer_d=[],[],[],[],[]
48 |         while self.cur_episode<self.max_episode:
49 |             s=self.env.reset()
50 |             total_reward=0
51 |             total_step=0
52 |             for _ in range(max_lstep):
53 |                 a=self.choose_action(s)
54 |                 s_,r,done,_=self.env.step(a)
55 |                 total_reward+=r
56 |                 buffer_a.append([a])
57 |                 buffer_s.append(s)
58 |                 buffer_r.append([r])
59 |                 buffer_s_.append(s_)
60 |                 buffer_d.append(done)
61 |                 s=s_
62 |                 total_step+=1
63 |                 if total_step%update_interval==0 or done:
64 |                     #opt,gnet,lnet,bs,ba,br,bs_
65 |                     update_global(self.optA,self.optC,self.gnet,self.lnet,buffer_s,buffer_a,buffer_r,buffer_s_,buffer_d)
66 |                     buffer_a,buffer_s,buffer_r,buffer_s_,buffer_d=[],[],[],[],[]
67 |                     if done:
68 |                         #print('done')
69 |                         self.queue.append(total_reward)
70 |                         if len(self.queue)==11:
71 |                             del self.queue[0]
72 |                         break
73 | 
74 |             self.cur_episode+=1
75 |             if self.name=='Worker 0':
76 |                 if self.cur_episode%10==0:
77 |                     print('Episode:',self.cur_episode,', last 10 episode mean reward of worker0:',th.mean(th.Tensor(self.queue)))
78 | 
79 | 
80 | 
81 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/A3C/utils.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | 
 3 | 
 4 | class SharedAdam(th.optim.Adam):
 5 |     #update parameters in parallel
 6 |     #Code from https://github.com/MorvanZhou/pytorch-A3C/blob/master/shared_adam.py
 7 |     def __init__(self, params, lr=1e-3, betas=(0.9, 0.99), eps=1e-8,
 8 |                  weight_decay=0):
 9 |         super(SharedAdam, self).__init__(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
10 |         # State initialization
11 |         for group in self.param_groups:
12 |             for p in group['params']:
13 |                 state = self.state[p]
14 |                 state['step'] =0
15 |                 state['exp_avg'] = th.zeros_like(p.data)
16 |                 state['exp_avg_sq'] = th.zeros_like(p.data)
17 | 
18 |                 # share in memory
19 |                 state['exp_avg'].share_memory_()
20 |                 state['exp_avg_sq'].share_memory_()
21 | 
22 | def update_global(optA,optC,gnet,lnet,bs,ba,br,bs_,bd):
23 |     gamma=0.9
24 |     bs=th.Tensor(bs)
25 |     bs_=th.Tensor(bs_)
26 |     ba=th.LongTensor(ba)
27 |     br=th.Tensor(br)
28 |     #critic
29 |     v_target,_=lnet(bs_)
30 |     v_target=v_target*gamma+br
31 |     v_eval,prob=lnet(bs)
32 |     for i in range(br.shape[0]):
33 |         if bd[i]:
34 |             v_target=br[i]
35 |     td_error=v_target.detach()-v_eval
36 |     loss_c=(td_error**2).mean()
37 |     optC.zero_grad()
38 |     loss_c.backward()
39 |     for lp, gp in zip(lnet.parameters(), gnet.parameters()):
40 |         gp._grad = lp.grad
41 |     optC.step()
42 |     #actor
43 |     prob=th.gather(prob,1,ba)
44 |     log_prob=th.log(prob)
45 |     loss_a=(-td_error.detach()*log_prob).mean()
46 | 
47 |     optA.zero_grad()
48 |     loss_a.backward()
49 |     for lp, gp in zip(lnet.parameters(), gnet.parameters()):
50 |         gp._grad = lp.grad
51 |     optA.step()
52 | 
53 |     lnet.load_state_dict(gnet.state_dict())
54 | 
55 | 
56 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/Actor_Critic.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | import torch.nn as nn
  5 | import torch.nn.functional as F
  6 | 
  7 | lr=0.001
  8 | gamma=0.9
  9 | hidden=32
 10 | env=gym.make('CartPole-v0')
 11 | device="cuda"
 12 | env=env.unwrapped
 13 | n_action=env.action_space.n
 14 | n_state=env.observation_space.shape[0]
 15 | 
 16 | class actor(nn.Module):      #policy net
 17 |     def __init__(self):
 18 |         super(actor,self).__init__()
 19 |         self.fc1=nn.Linear(n_state,hidden)
 20 |         self.fc2=nn.Linear(hidden,n_action)
 21 |         self.softmax=nn.Softmax()
 22 |     def forward(self,x):
 23 |         x=self.fc1(x)
 24 |         x=F.relu(x)
 25 |         x=self.fc2(x)
 26 |         prob=self.softmax(x)
 27 |         return prob
 28 | 
 29 | 
 30 | class critic(nn.Module):     #Q net
 31 |     def __init__(self):
 32 |         super(critic,self).__init__()
 33 |         self.fc1=nn.Linear(n_state,hidden)
 34 |         self.fc2=nn.Linear(hidden,1)
 35 |     def forward(self,x):
 36 |         x=self.fc1(x)
 37 |         x=F.relu(x)
 38 |         x=self.fc2(x)
 39 |         return x
 40 | 
 41 | class AC():
 42 |     def __init__(self):
 43 |         self.actor=actor().to(device)
 44 |         self.critic=critic().to(device)
 45 | 
 46 |         self.Aoptimizer=th.optim.Adam(self.actor.parameters(),lr=lr)
 47 |         self.Coptimizer=th.optim.Adam(self.critic.parameters(),lr=lr)
 48 | 
 49 |     def choose_action(self,s):
 50 |         s=th.FloatTensor(s).to(device)
 51 |         a_prob=self.actor(s)
 52 |         rand=np.random.uniform()
 53 |         accumulation=0
 54 |         action=0
 55 |         for i in range(n_action):
 56 |             accumulation+=a_prob[i]
 57 |             if accumulation>=rand:
 58 |                 action=i
 59 |                 break
 60 |         return action
 61 | 
 62 |     def actor_learn(self,s,a,td_error):
 63 |         s=th.FloatTensor(s).to(device)
 64 |         a_prob=self.actor(s)[a]
 65 |         loss=-(th.log(a_prob)*td_error.detach())
 66 | 
 67 |         self.Aoptimizer.zero_grad()
 68 |         loss.backward()
 69 |         self.Aoptimizer.step()
 70 | 
 71 |     def critic_learn(self,transition):    #transition=[s,[r],[a],s_,[done]]
 72 |         s=th.FloatTensor(transition[0]).to(device)
 73 |         r=transition[1][0]
 74 |         s_=th.FloatTensor(transition[3]).to(device)
 75 |         done=transition[4][0]
 76 | 
 77 |         v_eval=self.critic(s)
 78 |         v_target=self.critic(s_)*gamma+r
 79 | 
 80 |         td_error=v_eval-v_target.detach()
 81 |         loss=td_error**2
 82 | 
 83 |         self.Coptimizer.zero_grad()
 84 |         loss.backward()
 85 |         self.Coptimizer.step()
 86 |         return td_error
 87 |     
 88 | 
 89 | ac=AC()
 90 |     
 91 | for episode in range(10000):
 92 |     t=0
 93 |     s=env.reset()
 94 |     total_reward=0
 95 |     while(t<300):
 96 |         a=ac.choose_action(s)
 97 |         s_,r,done,_=env.step(a)
 98 |         total_reward+=r
 99 |         transition=[s,[r],[a],s_,[done]]
100 | 
101 |         td_error=ac.critic_learn(transition)
102 |         ac.actor_learn(s,a,td_error)
103 |         if done:
104 |             break
105 |         s=s_
106 |     if(episode%10==0):
107 |         print("Episode:"+format(episode)+",score:"+format(total_reward))
108 | 
109 | 
110 |         
111 | 
112 | 
113 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG/experience_replay.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | class replay_memory():
 3 |     def __init__(self,replay_memory_size):
 4 |         self.memory_size=replay_memory_size
 5 |         self.memory=np.array([])
 6 |         self.cur=0
 7 |         self.new=0
 8 |     def size(self):
 9 |         return self.memory.shape[0]
10 | #[s,a,r,s_,done] make sure all info are lists, i.e. [[[1,2],[3]],[1],[0],[[4,5],[6]],[True]]
11 |     def store(self,trans):
12 |         if(self.memory.shape[0]<self.memory_size):
13 |             if self.new==0:
14 |                 self.memory=np.array(trans)
15 |                 self.new=1
16 |             elif self.memory.shape[0]>0:
17 |                 self.memory=np.vstack((self.memory,trans))
18 |         else:
19 |             self.memory[self.cur,:]=trans
20 |             self.cur=(self.cur+1)%self.memory_size
21 |     
22 |     def sample(self,batch_size):
23 |         if self.memory.shape[0]<batch_size:
24 |             return -1
25 |         sam=np.random.choice(self.memory.shape[0],batch_size)
26 |         return self.memory[sam]
27 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG/main.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np 
  4 | from model import Critic,Actor
  5 | from experience_replay import replay_memory
  6 | 
  7 | 
  8 | lr=0.001
  9 | tau=0.05
 10 | max_t=200
 11 | gamma=0.9
 12 | memory_size=2000
 13 | warmup=500
 14 | batchsize=32
 15 | env=gym.make('Pendulum-v0')
 16 | device="cuda"
 17 | env=env.unwrapped
 18 | n_action=1
 19 | n_state=env.observation_space.shape[0]
 20 | max_action = float(env.action_space.high[0])
 21 | #print(max_action)
 22 | class DDPG():
 23 |     def __init__(self):
 24 |         self.actor=Actor(n_state,n_action,max_action).to(device)
 25 |         self.target_actor=Actor(n_state,n_action,max_action).to(device)
 26 |         self.critic=Critic(n_state,n_action).to(device)
 27 |         self.target_critic=Critic(n_state,n_action).to(device)
 28 |         self.memory=replay_memory(memory_size)
 29 |         self.Aoptimizer=th.optim.Adam(self.actor.parameters(),lr=lr)
 30 |         self.Coptimizer=th.optim.Adam(self.critic.parameters(),lr=lr)
 31 | 
 32 |     def actor_learn(self,batch):
 33 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 34 |         action=self.actor(b_s)
 35 |         #print(action)
 36 |         loss=-(self.critic(b_s,action).mean())
 37 |         self.Aoptimizer.zero_grad()
 38 |         loss.backward()
 39 |         self.Aoptimizer.step()
 40 |     
 41 |     def critic_learn(self,batch):
 42 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 43 |         b_r=th.FloatTensor(batch[:,1].tolist()).to(device)
 44 |         b_a=th.FloatTensor(batch[:,2].tolist()).to(device)
 45 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 46 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 47 | 
 48 |         next_action=self.target_actor(b_s_)
 49 |         #print(next_action)
 50 |         target_q=self.target_critic(b_s_,next_action)
 51 |         for i in range(b_d.shape[0]):
 52 |             if b_d[i]:
 53 |                 target_q[i]=b_r[i]
 54 |             else:
 55 |                 target_q[i]=b_r[i]+gamma*target_q[i]
 56 |         eval_q=self.critic(b_s,b_a)
 57 | 
 58 |         td_error=eval_q-target_q.detach()
 59 |         loss=(td_error**2).mean()
 60 |         self.Coptimizer.zero_grad()
 61 |         loss.backward()
 62 |         self.Coptimizer.step()
 63 | 
 64 |     def soft_update(self):
 65 |         for param,target_param in zip(self.actor.parameters(),self.target_actor.parameters()):
 66 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 67 |         for param,target_param in zip(self.critic.parameters(),self.target_critic.parameters()):
 68 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 69 | ddpg=DDPG()
 70 | 
 71 | def main():
 72 |     var=3
 73 |     for episode in range(2000):
 74 |         s=env.reset()
 75 |         total_reward=0
 76 |         Normal=th.distributions.normal.Normal(th.FloatTensor([0]),th.FloatTensor([var]))
 77 |         t=0
 78 |         while t<max_t:
 79 |             noise=th.clamp(Normal.sample(),env.action_space.low[0], env.action_space.high[0]).to(device)
 80 |             a=ddpg.actor(th.FloatTensor(s).to(device))+noise
 81 |             a=th.clamp(a,env.action_space.low[0], env.action_space.high[0]).to(device)
 82 |             
 83 |             s_,r,done,_=env.step(a.tolist())
 84 |             total_reward+=r
 85 |             transition=[s,[r],[a],s_,[done]]
 86 |             ddpg.memory.store(transition)
 87 |             #print(done)
 88 |             if done:
 89 |                 break
 90 |             s=s_
 91 |             if(ddpg.memory.size()<warmup):
 92 |                 #print(ddpg.memory.size())
 93 |                 continue
 94 |             var*=0.9995
 95 |             batch=ddpg.memory.sample(batchsize)
 96 |             ddpg.critic_learn(batch)
 97 |             ddpg.actor_learn(batch)
 98 | 
 99 |             ddpg.soft_update()
100 |             t+=1
101 |         print("episode:"+format(episode)+",test score:"+format(total_reward)+',variance:',var)
102 | 
103 | if __name__=='__main__':
104 |     main()
105 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import torch.nn as nn
 3 | import torch.nn.functional as F
 4 | import numpy as np
 5 | 
 6 | class Critic(nn.Module):
 7 |     def __init__(self,state_dim,act_dim):
 8 |         super(Critic,self).__init__()
 9 |         self.fc1=nn.Linear(state_dim,64)
10 |         self.fc2=nn.Linear(64+act_dim,64)
11 |         self.fc3=nn.Linear(64,1)
12 |         
13 |     def forward(self,state,action):
14 |         x=self.fc1(state)
15 |         x=F.relu(x)
16 |         x=self.fc2(th.cat((x,action),1))
17 |         x=F.relu(x)
18 |         x=self.fc3(x)
19 |         return x
20 | 
21 | class Actor(nn.Module):
22 |     def __init__(self,state_dim,act_dim,max_a):
23 |         super(Actor,self).__init__()
24 |         self.fc1=nn.Linear(state_dim,128)
25 |         self.fc1.weight.data.normal_(0, 0.1)
26 |         self.fc2=nn.Linear(128,64)
27 |         self.fc2.weight.data.normal_(0, 0.1)
28 |         self.fc3=nn.Linear(64,act_dim)
29 |         self.fc3.weight.data.normal_(0, 0.1)
30 |         self.max_a=max_a
31 |     def forward(self,x):
32 |         x=F.relu(self.fc1(x))
33 |         x=F.relu(self.fc2(x))
34 |         x=self.fc3(x)
35 |         return th.tanh(x)*self.max_a


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG_discrete/experience_replay.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | class replay_memory():
 3 |     def __init__(self,replay_memory_size):
 4 |         self.memory_size=replay_memory_size
 5 |         self.memory=np.array([])
 6 |         self.cur=0
 7 |         self.new=0
 8 |     def size(self):
 9 |         return self.memory.shape[0]
10 | #[s,a,r,s_,done] make sure all info are lists, i.e. [[[1,2],[3]],[1],[0],[[4,5],[6]],[True]]
11 |     def store(self,trans):
12 |         if(self.memory.shape[0]<self.memory_size):
13 |             if self.new==0:
14 |                 self.memory=np.array(trans)
15 |                 self.new=1
16 |             elif self.memory.shape[0]>0:
17 |                 self.memory=np.vstack((self.memory,trans))
18 |         else:
19 |             self.memory[self.cur,:]=trans
20 |             self.cur=(self.cur+1)%self.memory_size
21 |     
22 |     def sample(self,batch_size):
23 |         if self.memory.shape[0]<batch_size:
24 |             return -1
25 |         sam=np.random.choice(self.memory.shape[0],batch_size)
26 |         return self.memory[sam]
27 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG_discrete/gumbel_softmax.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import numpy as np
 3 | device='cpu'
 4 | def gumbel_sample(shape,eps=1e-10):
 5 |     seed=th.FloatTensor(shape).uniform_().to(device)
 6 |     return -th.log(-th.log(seed+eps)+eps)
 7 | 
 8 | def gumbel_softmax_sample(logits,temperature=1.0):
 9 |     #print(logits)
10 |     logits=logits+gumbel_sample(logits.shape,1e-10)
11 |     #print(logits)
12 |     return (th.nn.functional.softmax(logits/temperature,dim=1))
13 | 
14 | def gumbel_softmax(prob,temperature=1.0,hard=False):
15 |     #print(prob)
16 |     logits=th.log(prob)
17 |     y=gumbel_softmax_sample(prob,temperature)
18 |     if hard==True:   #one hot but differenttiable
19 |         y_onehot=onehot_action(y)
20 |         y=(y_onehot-y).detach()+y
21 |     return y
22 | 
23 | def onehot_action(prob):
24 |     y=th.zeros_like(prob).to(device)
25 |     index=th.argmax(prob,dim=1).unsqueeze(1)
26 |     y=y.scatter(1,index,1)
27 |     return y.to(th.long)
28 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG_discrete/main.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | from gumbel_softmax import gumbel_softmax
  5 | from model import Critic,Actor
  6 | from experience_replay import replay_memory
  7 | 
  8 | 
  9 | lr=0.001
 10 | tau=0.05
 11 | max_t=200
 12 | gamma=0.9
 13 | memory_size=2000
 14 | batchsize=32
 15 | warmup=batchsize
 16 | env=gym.make('CartPole-v0')
 17 | device="cpu"
 18 | env=env.unwrapped
 19 | n_action=env.action_space.n
 20 | n_state=env.observation_space.shape[0]
 21 | 
 22 | class DDPG():
 23 |     def __init__(self):
 24 |         self.actor=Actor(n_state,n_action).to(device)
 25 |         self.target_actor=Actor(n_state,n_action).to(device)
 26 |         self.critic=Critic(n_state,n_action).to(device)
 27 |         self.target_critic=Critic(n_state,n_action).to(device)
 28 |         self.memory=replay_memory(memory_size)
 29 |         self.Aoptimizer=th.optim.Adam(self.actor.parameters(),lr=lr)
 30 |         self.Coptimizer=th.optim.Adam(self.critic.parameters(),lr=lr)
 31 |     
 32 |     def choose_action(self,state,eps):
 33 |         prob=self.actor(th.FloatTensor(state).to(device))
 34 |         prob=th.nn.functional.softmax(prob,0)
 35 |         #print(prob)
 36 |         if np.random.uniform()>eps:
 37 |             action=th.argmax(prob,dim=0).tolist()
 38 |         else:
 39 |             action=np.random.randint(0,n_action)
 40 |         return action
 41 |     
 42 |     def actor_learn(self,batch):
 43 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 44 |         b_r=th.FloatTensor(batch[:,1].tolist()).to(device)
 45 |         b_a=th.FloatTensor(batch[:,2].tolist()).to(device)
 46 |         
 47 |         differentiable_a=th.nn.functional.gumbel_softmax(th.log(th.nn.functional.softmax(self.actor(b_s),dim=1)),hard=True)
 48 |         #print(differentiable_a)
 49 |         #differentiable_a2=th.nn.functional.softmax(th.nn.functional.softmax(self.actor(b_s),dim=1),dim=1)
 50 |         #index=th.argmax(differentiable_a2,dim=1).unsqueeze(1)
 51 |         #oh=th.zeros_like(differentiable_a2).scatter_(1,index,1)
 52 |         #differentiable_a2=(oh-differentiable_a2).detach()+differentiable_a2
 53 | 
 54 |         loss=-self.critic(b_s,differentiable_a).mean()
 55 |         self.Aoptimizer.zero_grad()
 56 |         loss.backward()
 57 |         self.Aoptimizer.step()
 58 | 
 59 |     def critic_learn(self,batch):
 60 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 61 |         b_r=th.FloatTensor(batch[:,1].tolist()).to(device)
 62 |         b_a=th.zeros(batchsize,n_action).scatter_(1,th.LongTensor(batch[:,2].tolist()),1).to(device)
 63 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 64 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 65 | 
 66 |         eval_q=self.critic(b_s,b_a)
 67 | 
 68 |         next_action=th.nn.functional.softmax(self.target_actor(b_s_),dim=1)
 69 | 
 70 |         index=th.argmax(next_action,dim=1).unsqueeze(1)
 71 |         next_action=th.zeros_like(next_action).scatter_(1,index,1).to(device)
 72 |         print(next_action)
 73 |         target_q=th.zeros_like(eval_q).to(device)
 74 | 
 75 |         for i in range(b_d.shape[0]):
 76 |             target_q[i]=(1-b_d[i,0])*gamma*self.target_critic(b_s_,next_action)[i].detach()+b_r[i]
 77 |         td_error=eval_q-target_q
 78 |         loss=(td_error**2).mean()
 79 |         self.Coptimizer.zero_grad()
 80 |         loss.backward()
 81 |         self.Coptimizer.step()
 82 | 
 83 | 
 84 |     def soft_update(self):
 85 |         for param,target_param in zip(self.actor.parameters(),self.target_actor.parameters()):
 86 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 87 |         for param,target_param in zip(self.critic.parameters(),self.target_critic.parameters()):
 88 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 89 | for j in range(10):
 90 |     ddpg=DDPG()
 91 |     highest=0
 92 |     for episode in range(300):
 93 |         s=env.reset()
 94 |         t=0
 95 |         total_reward=0
 96 |         while(t<max_t):
 97 |             a=ddpg.choose_action(s,0.1)
 98 |             s_,r,done,_=env.step(a)
 99 |             total_reward+=r
100 |             transition=[s,[r],[a],s_,[done]]
101 |             ddpg.memory.store(transition)
102 |             if done:
103 |                 break
104 |             s=s_
105 |             if(ddpg.memory.size()<warmup):
106 |                 continue
107 |             batch=ddpg.memory.sample(batchsize)
108 |             ddpg.critic_learn(batch)
109 |             ddpg.actor_learn(batch)
110 |             ddpg.soft_update()
111 | 
112 |             t+=1
113 |         if episode%10==0:
114 |             total_reward=0.0
115 |             for i in range(1):
116 |                 t_s=env.reset()
117 |                 t_r=0.0
118 |                 tr=0.0
119 |                 time=0
120 |                 while(time<300):
121 |                     time+=1
122 |                     t_a=ddpg.choose_action(t_s,0)
123 |                     ts_,tr,tdone,_=env.step(t_a)
124 |                     t_r+=tr
125 |                     if tdone:
126 |                         break
127 |                     t_s=ts_
128 |                 total_reward+=t_r
129 |                 if total_reward>highest:
130 |                     highest=total_reward
131 |                 print("episode:"+format(episode)+",test score:"+format(total_reward))
132 |     if(highest>20):
133 |         print(format(j+1)+"th round did it")
134 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/DDPG_discrete/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import torch.nn as nn
 3 | import torch.nn.functional as F
 4 | import numpy as np
 5 | 
 6 | class Critic(nn.Module):
 7 |     def __init__(self,state_dim,act_dim):
 8 |         super(Critic,self).__init__()
 9 |         self.fc1=nn.Linear(state_dim,64)
10 |         self.fc2=nn.Linear(64+act_dim,32)
11 |         self.fc3=nn.Linear(32,1)
12 |         
13 |     def forward(self,state,action):
14 |         x=self.fc1(state)
15 |         x=F.relu(x)
16 |         x=self.fc2(th.cat((x,action),1))
17 |         x=F.relu(x)
18 |         x=self.fc3(x)
19 |         return x
20 | 
21 | class Actor(nn.Module):
22 |     def __init__(self,state_dim,act_dim):
23 |         super(Actor,self).__init__()
24 |         self.fc1=nn.Linear(state_dim,64)
25 |         self.fc2=nn.Linear(64,32)
26 |         self.fc3=nn.Linear(32,act_dim)
27 | 
28 |     def forward(self,x):
29 |         x=F.relu(self.fc1(x))
30 |         x=F.relu(self.fc2(x))
31 |         x=self.fc3(x)
32 |         return th.tanh(x)
33 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/REINFORCE.py:
--------------------------------------------------------------------------------
 1 | import gym
 2 | import torch as th
 3 | import numpy as np
 4 | import torch.nn as nn
 5 | import torch.nn.functional as F
 6 | 
 7 | alpha=0.001
 8 | max_t=200
 9 | gamma=0.9
10 | hidden=32
11 | env=gym.make('CartPole-v0')
12 | device="cuda"
13 | env=env.unwrapped
14 | n_action=env.action_space.n
15 | n_state=env.observation_space.shape[0]
16 | 
17 | class policy(nn.Module):
18 |     def __init__(self):
19 |         super(policy,self).__init__()
20 |         self.fc1=nn.Linear(n_state,hidden)
21 |         self.fc2=nn.Linear(hidden,n_action)
22 |         self.softmax=nn.Softmax()
23 |     def forward(self,x):
24 |         x=self.fc1(x)
25 |         x=F.relu(x)
26 |         x=self.fc2(x)
27 |         prob=self.softmax(x)
28 |         return prob
29 | 
30 | class REINFORCE():
31 |     def __init__(self):
32 |         self.policy_net=policy().to(device)
33 |         self.g=0
34 |         self.optimizer=th.optim.Adam(self.policy_net.parameters(),lr=alpha)
35 |     def choose_action(self,s):
36 |         s=th.FloatTensor(s).to(device)
37 |         a_prob=self.policy_net(s)
38 |         rand=np.random.uniform()
39 |         accumulation=0
40 |         action=0
41 |         for i in range(n_action):
42 |             accumulation+=a_prob[i]
43 |             if accumulation>=rand:
44 |                 action=i
45 |                 break
46 |         return action
47 | 
48 |     def learn(self,transition):
49 |         timestep=len(transition)
50 |         loss=0
51 |         returns=th.zeros(timestep,1)
52 |         log_prob=th.zeros(timestep,1)
53 |         self.g=0
54 |         for i in reversed(range(timestep)):
55 |             s=th.FloatTensor(transition[i,0]).to(device)
56 |             a=transition[i,1][0]
57 |             r=transition[i,2][0]
58 |             log_prob[i]=th.log(self.policy_net(s))[a].unsqueeze(0)
59 |             self.g=gamma*self.g+r
60 |             returns[i]=self.g
61 | 
62 | 
63 |         loss=-(log_prob*returns.detach()).sum()
64 |         self.optimizer.zero_grad()
65 |         loss.backward()
66 |         self.optimizer.step()
67 | 
68 | reinforce=REINFORCE()
69 | 
70 | for episode in range(10000):
71 |     t=0
72 |     s=env.reset()
73 |     transition=np.array([])
74 |     total_reward=0
75 |     while(t<300):
76 |         a=reinforce.choose_action(s)
77 |         s_,r,done,_=env.step(a)
78 |         total_reward+=r
79 |         trans=[s,[a],[r]]
80 |         if t==0:
81 |             transition=trans
82 |         else:
83 |             transition=np.vstack((transition,trans))
84 |         if done:
85 |             reinforce.learn(transition)
86 |             break
87 |         s=s_
88 |         t+=1
89 |     if episode%100==0:
90 |         print("Episode:"+format(episode)+",total score:"+format(total_reward))
91 | 
92 | 
93 |     
94 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/SAC/experience_replay.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | class replay_memory():
 3 |     def __init__(self,replay_memory_size):
 4 |         self.memory_size=replay_memory_size
 5 |         self.memory=np.array([])
 6 |         self.cur=0
 7 |         self.new=0
 8 |     def size(self):
 9 |         return self.memory.shape[0]
10 | #[s,a,r,s_,done] make sure all info are lists, i.e. [[[1,2],[3]],[1],[0],[[4,5],[6]],[True]]
11 |     def store(self,trans):
12 |         if(self.memory.shape[0]<self.memory_size):
13 |             if self.new==0:
14 |                 self.memory=np.array(trans)
15 |                 self.new=1
16 |             elif self.memory.shape[0]>0:
17 |                 self.memory=np.vstack((self.memory,trans))
18 |         else:
19 |             self.memory[self.cur,:]=trans
20 |             self.cur=(self.cur+1)%self.memory_size
21 |     
22 |     def sample(self,batch_size):
23 |         if self.memory.shape[0]<batch_size:
24 |             return -1
25 |         sam=np.random.choice(self.memory.shape[0],batch_size)
26 |         return self.memory[sam]


--------------------------------------------------------------------------------
/algorithm/policy gradient/SAC/main.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | from model import Q,V,Actor
  5 | from experience_replay import replay_memory
  6 | from torch.distributions.normal import Normal
  7 | 
  8 | lr=0.001
  9 | tau=0.005
 10 | max_t=200
 11 | gamma=0.9
 12 | memory_size=2000
 13 | warmup=500
 14 | batchsize=32
 15 | env=gym.make('Pendulum-v0')
 16 | device="cuda"
 17 | alpha=0.05
 18 | n_action=1
 19 | n_state=env.observation_space.shape[0]
 20 | max_action = float(env.action_space.high[0])
 21 | 
 22 | class SAC():
 23 |     def __init__(self):
 24 |         self.V=V(n_state).to(device)
 25 |         self.target_V=V(n_state).to(device)
 26 |         self.policy=Actor(n_state,max_action).to(device)
 27 |         self.Q=Q(n_state,n_action).to(device)
 28 |         
 29 |         self.optimV=th.optim.Adam(self.V.parameters(),lr=lr)
 30 |         self.optimQ=th.optim.Adam(self.Q.parameters(),lr=lr)
 31 |         self.optimP=th.optim.Adam(self.policy.parameters(),lr=lr)
 32 | 
 33 |         self.memory=replay_memory(memory_size)
 34 | 
 35 |     def choose_action(self,s):
 36 |         mu,log_std=self.policy(s)
 37 | 
 38 |         dist=Normal(mu,th.exp(log_std))
 39 |         action=dist.sample()
 40 |         action = th.tanh(action)
 41 |         
 42 |         return action
 43 | 
 44 | 
 45 |     def V_learn(self,batch):
 46 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 47 |         b_a=th.FloatTensor(batch[:,2].tolist()).to(device)
 48 | 
 49 |         mu,log_std=self.policy(b_s)
 50 |         dist=Normal(mu,th.exp(log_std))
 51 | 
 52 |         z=dist.sample()
 53 |         b_a=th.tanh(z)
 54 |         prob=dist.log_prob(z)
 55 |         qs=self.Q(b_s,b_a)
 56 | 
 57 |         v=self.V(b_s)
 58 |         target_v=qs-prob
 59 | 
 60 |         loss=(v-target_v.detach())**2
 61 |         loss=loss.mean()
 62 | 
 63 |         self.optimV.zero_grad()
 64 |         loss.backward()
 65 |         self.optimV.step()
 66 | 
 67 | 
 68 | 
 69 |     def Q_learn(self,batch):
 70 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 71 |         b_r=th.FloatTensor(batch[:,1].tolist()).to(device)
 72 |         b_a=th.FloatTensor(batch[:,2].tolist()).to(device)
 73 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 74 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 75 | 
 76 |         target_q=b_r+(1-b_d)*gamma*self.target_V(b_s_)
 77 | 
 78 |         eval_q=self.Q(b_s,b_a)
 79 |         loss=(eval_q-target_q.detach())**2
 80 |         loss=loss.mean()
 81 |         self.optimQ.zero_grad()
 82 |         loss.backward()
 83 |         self.optimQ.step()
 84 |         
 85 | 
 86 |     def P_learn(self,batch):
 87 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 88 | 
 89 |         norm=Normal(th.zeros((batchsize,1)),th.ones((batchsize,1)))
 90 |         #norm=Normal(0,1)
 91 |         mu,log_std=self.policy(b_s)
 92 | 
 93 |         z=norm.sample()
 94 |         b_a=th.tanh(mu+th.exp(log_std)*z.to(device))
 95 | 
 96 |         dist=Normal(mu,th.exp(log_std))
 97 |         log_prob=dist.log_prob(mu+th.exp(log_std)*z.to(device))- th.log(1 - b_a.pow(2) + 1e-7)
 98 |         qs=self.Q(b_s,b_a)
 99 | 
100 |         loss=alpha*log_prob-qs
101 |         loss=loss.mean()
102 | 
103 |         self.optimP.zero_grad()
104 |         loss.backward()
105 |         self.optimP.step()
106 | 
107 | 
108 |     def soft_update(self):
109 |         for param,target_param in zip(self.V.parameters(),self.target_V.parameters()):
110 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
111 | 
112 | 
113 | sac=SAC()
114 | def main():
115 |     for episode in range(2000):
116 |         s=env.reset()
117 |         total_reward=0
118 |         t=0
119 |         while t<max_t:
120 |             a=sac.choose_action(th.FloatTensor(s).to(device))
121 |             s_,r,done,_=env.step([a.tolist()[0]*max_action])
122 |             total_reward+=r
123 |             transition=[s,[r],[a],s_,[done]]
124 |             sac.memory.store(transition)
125 |             if done:
126 |                 break
127 |             s=s_
128 |             if(sac.memory.size()<warmup):
129 |                 continue
130 |             batch=sac.memory.sample(batchsize)
131 |             sac.V_learn(batch)
132 |             sac.Q_learn(batch)
133 |             sac.P_learn(batch)
134 |             sac.soft_update()
135 |             t+=1
136 |         print("episode:"+format(episode)+",test score:"+format(total_reward))
137 | 
138 | if __name__=='__main__':
139 |     main()
140 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/SAC/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import torch.nn as nn
 3 | import torch.nn.functional as F
 4 | import numpy as np
 5 | 
 6 | class Q(nn.Module):
 7 |     def __init__(self,state_dim,act_dim):
 8 |         super(Q,self).__init__()
 9 |         self.fc1=nn.Linear(state_dim,64)
10 |         self.fc2=nn.Linear(64+act_dim,256)
11 |         self.fc3=nn.Linear(256,1)
12 |         
13 |     def forward(self,state,action):
14 |         x=self.fc1(state)
15 |         x=F.relu(x)
16 |         x=self.fc2(th.cat((x,action),1))
17 |         x=F.relu(x)
18 |         x=self.fc3(x)
19 |         return x
20 | 
21 | class V(nn.Module):
22 |     def __init__(self,state_dim):
23 |         super(V,self).__init__()
24 |         self.fc1=nn.Linear(state_dim,256)
25 |         self.fc2=nn.Linear(256,1)
26 |         
27 |     def forward(self,state):
28 |         x=self.fc1(state)
29 |         x=F.relu(x)
30 |         x=self.fc2(x)
31 |         return x
32 | 
33 | class Actor(nn.Module):
34 |     def __init__(self,state_dim,max_a):
35 |         super(Actor,self).__init__()
36 |         self.fc1=nn.Linear(state_dim,256)
37 |         self.fc1.weight.data.normal_(0, 0.1)
38 |         self.fc2=nn.Linear(256,64)
39 |         self.fc2.weight.data.normal_(0, 0.1)
40 |         self.std_head=nn.Linear(64,1)
41 |         self.std_head.weight.data.normal_(0, 0.1)
42 |         self.mu_head=nn.Linear(64,1)
43 |         self.mu_head.weight.data.normal_(0, 0.1)
44 |         self.max_a=max_a
45 | 
46 |         self.max_std=2
47 |         self.min_std=-2
48 | 
49 |     def forward(self,x):
50 |         x=self.fc2(F.relu(self.fc1(x)))
51 |         std=self.std_head(x)
52 |         mu=self.mu_head(x)
53 |         std= th.clamp(std, self.min_std, self.max_std)
54 |         return mu,std


--------------------------------------------------------------------------------
/algorithm/policy gradient/TD3/experience_replay.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | class replay_memory():
 3 |     def __init__(self,replay_memory_size):
 4 |         self.memory_size=replay_memory_size
 5 |         self.memory=np.array([])
 6 |         self.cur=0
 7 |         self.new=0
 8 |     def size(self):
 9 |         return self.memory.shape[0]
10 | #[s,a,r,s_,done] make sure all info are lists, i.e. [[[1,2],[3]],[1],[0],[[4,5],[6]],[True]] 
11 |     def store(self,trans):
12 |         if(self.memory.shape[0]<self.memory_size):
13 |             if self.new==0:
14 |                 self.memory=np.array(trans)
15 |                 self.new=1
16 |             elif self.memory.shape[0]>0:
17 |                 self.memory=np.vstack((self.memory,trans))
18 |         else:
19 |             self.memory[self.cur,:]=trans
20 |             self.cur=(self.cur+1)%self.memory_size
21 |     
22 |     def sample(self,batch_size):
23 |         if self.memory.shape[0]<batch_size:
24 |             return -1
25 |         sam=np.random.choice(self.memory.shape[0],batch_size)
26 |         return self.memory[sam]
27 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/TD3/main.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | from model import Critic,Actor
  5 | from experience_replay import replay_memory
  6 | 
  7 | 
  8 | lr=0.001
  9 | tau=0.005
 10 | max_t=200
 11 | gamma=0.99
 12 | c=0.5
 13 | d=2
 14 | memory_size=2000
 15 | warmup=500
 16 | batchsize=32
 17 | env=gym.make('Pendulum-v0')
 18 | device="cuda"
 19 | env=env.unwrapped
 20 | n_action=1
 21 | n_state=env.observation_space.shape[0]
 22 | max_action = float(env.action_space.high[0])
 23 | #print(max_action)
 24 | class TD3():
 25 |     def __init__(self):
 26 |         self.actor=Actor(n_state,n_action,max_action).to(device)
 27 |         self.target_actor=Actor(n_state,n_action,max_action).to(device)
 28 |         self.target_actor.load_state_dict(self.actor.state_dict())
 29 | 
 30 |         self.critic1=Critic(n_state,n_action).to(device)
 31 |         self.target_critic1=Critic(n_state,n_action).to(device)
 32 |         self.target_critic1.load_state_dict(self.critic1.state_dict())
 33 | 
 34 |         self.critic2=Critic(n_state,n_action).to(device)
 35 |         self.target_critic2=Critic(n_state,n_action).to(device)
 36 |         self.target_critic2.load_state_dict(self.critic2.state_dict())
 37 | 
 38 |         self.memory=replay_memory(memory_size)
 39 |         self.Aoptimizer=th.optim.Adam(self.actor.parameters(),lr=lr)
 40 |         self.C1optimizer=th.optim.Adam(self.critic1.parameters(),lr=lr)
 41 |         self.C2optimizer=th.optim.Adam(self.critic2.parameters(),lr=lr)
 42 | 
 43 |     def actor_learn(self,batch):
 44 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 45 |         action=self.actor(b_s)
 46 |         loss=-(self.critic1(b_s,action).mean())
 47 |         self.Aoptimizer.zero_grad()
 48 |         loss.backward()
 49 |         self.Aoptimizer.step()
 50 |     
 51 |     def critic_learn(self,batch,policy_noise):
 52 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 53 |         b_r=th.FloatTensor(batch[:,1].tolist()).to(device)
 54 |         b_a=th.FloatTensor(batch[:,2].tolist()).to(device)
 55 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 56 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 57 | 
 58 |         dist=th.distributions.normal.Normal(th.FloatTensor([0]),th.Tensor([policy_noise]))
 59 |         next_action=self.target_actor(b_s_)+th.clamp(dist.sample(),-c,c).to(device)
 60 |         next_action=th.clamp(next_action,env.action_space.low[0],env.action_space.high[0])
 61 |         #print(th.cat((self.target_critic1(b_s_,next_action),self.target_critic2(b_s_,next_action)),dim=1))
 62 |         target_q=th.min(self.target_critic1(b_s_,next_action),self.target_critic2(b_s_,next_action))
 63 |         #print(target_q)
 64 |         for i in range(b_d.shape[0]):
 65 |             if b_d[i]:
 66 |                 target_q[i]=b_r[i]
 67 |             else:
 68 |                 target_q[i]=b_r[i]+gamma*target_q[i]
 69 |         #target_q=b_r+gamma*target_q
 70 |         eval_q1=self.critic1(b_s,b_a)
 71 |         td_error=eval_q1-target_q.detach()
 72 |         loss=(td_error**2).mean()
 73 |         self.C1optimizer.zero_grad()
 74 |         loss.backward()
 75 |         self.C1optimizer.step()
 76 | 
 77 |         eval_q2=self.critic2(b_s,b_a)
 78 |         td_error=eval_q2-target_q.detach()
 79 |         loss=(td_error**2).mean()
 80 |         self.C2optimizer.zero_grad()
 81 |         loss.backward()
 82 |         self.C2optimizer.step()
 83 | 
 84 |     def soft_update(self):
 85 |         for param,target_param in zip(self.actor.parameters(),self.target_actor.parameters()):
 86 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 87 |         for param,target_param in zip(self.critic1.parameters(),self.target_critic1.parameters()):
 88 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 89 |         for param,target_param in zip(self.critic2.parameters(),self.target_critic2.parameters()):
 90 |             target_param.data.copy_(tau*param.data+(1-tau)*target_param.data)
 91 | td3=TD3()
 92 | 
 93 | def main():
 94 |     exploration_noise=3
 95 |     policy_noise=0.2
 96 |     for episode in range(2000):
 97 |         s=env.reset()
 98 |         total_reward=0
 99 |         t=0
100 |         while t<max_t:
101 |             explore_N=th.distributions.normal.Normal(th.FloatTensor([0]),th.Tensor([exploration_noise]))
102 |             #noise=th.clamp(explore_N.sample(),env.action_space.low[0], env.action_space.high[0]).to(device)
103 |             noise=explore_N.sample()
104 |             a=td3.actor(th.FloatTensor(s).to(device))+noise.to(device)
105 |             #a=th.clamp(a,env.action_space.low[0], env.action_space.high[0]).to(device)
106 |             
107 |             s_,r,done,_=env.step(a.tolist())
108 |             total_reward+=r
109 |             transition=[s,[r],[a],s_,[done]]
110 |             td3.memory.store(transition)
111 |             if done:
112 |                 break
113 |             s=s_
114 |             if(td3.memory.size()<warmup):
115 |                 continue
116 |             batch=td3.memory.sample(batchsize)
117 |             td3.critic_learn(batch,policy_noise)
118 |             exploration_noise*=0.995
119 |             if not t%d:
120 |                 td3.actor_learn(batch)
121 |                 td3.soft_update()
122 |                 #policy_noise*=0.9995
123 |             t+=1
124 |         print("episode:"+format(episode)+",test score:"+format(total_reward))
125 | 
126 | if __name__=='__main__':
127 |     main()
128 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/TD3/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import torch.nn as nn
 3 | import torch.nn.functional as F
 4 | import numpy as np
 5 | 
 6 | class Critic(nn.Module):
 7 |     def __init__(self,state_dim,act_dim):
 8 |         super(Critic,self).__init__()
 9 |         self.fc1=nn.Linear(state_dim,64)
10 |         self.fc2=nn.Linear(64+act_dim,256)
11 |         self.fc3=nn.Linear(256,1)
12 |         
13 |     def forward(self,state,action):
14 |         x=self.fc1(state)
15 |         x=F.relu(x)
16 |         x=self.fc2(th.cat((x,action),1))
17 |         x=F.relu(x)
18 |         x=self.fc3(x)
19 |         return x
20 | 
21 | class Actor(nn.Module):
22 |     def __init__(self,state_dim,act_dim,max_a):
23 |         super(Actor,self).__init__()
24 |         self.fc1=nn.Linear(state_dim,128)
25 |         self.fc1.weight.data.normal_(0, 0.1)
26 |         self.fc2=nn.Linear(128,256)
27 |         self.fc2.weight.data.normal_(0, 0.1)
28 |         self.fc3=nn.Linear(256,act_dim)
29 |         self.fc3.weight.data.normal_(0, 0.1)
30 |         self.max_a=max_a
31 |     def forward(self,x):
32 |         x=F.relu(self.fc1(x))
33 |         x=F.relu(self.fc2(x))
34 |         x=self.fc3(x)
35 |         return th.tanh(x)*self.max_a


--------------------------------------------------------------------------------
/algorithm/policy gradient/TRPO/main.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | import torch.nn as nn
  5 | import torch.nn.functional as F
  6 | import time
  7 | from model import net,v
  8 | 
  9 | delta=0.01
 10 | 
 11 | alpha=0.001
 12 | max_t=200
 13 | gamma=0.9
 14 | update_interval=3
 15 | v_update_iter=10
 16 | env=gym.make('CartPole-v0')
 17 | device="cuda"
 18 | env=env.unwrapped
 19 | n_action=env.action_space.n
 20 | n_state=env.observation_space.shape[0]
 21 | 
 22 | class TRPO():
 23 |     def __init__(self):
 24 |         self.policy=net(n_state,n_action).to(device)
 25 |         self.old_policy=net(n_state,n_action).to(device)
 26 | 
 27 |         self.v=v(n_state).to(device)
 28 |         self.optim=th.optim.Adam(self.v.parameters(),lr=0.001)
 29 | 
 30 |     def choose_action(self,s):
 31 |         dist=th.distributions.Categorical(self.old_policy(th.Tensor(s).to(device)))
 32 |         return dist.sample().tolist()
 33 | 
 34 |     def flat_grad(self, grads, hessian=False):
 35 |       grad_flatten = []
 36 |       if hessian == False:
 37 |          for grad in grads:
 38 |             grad_flatten.append(grad.view(-1))
 39 |          grad_flatten = th.cat(grad_flatten)
 40 |          return grad_flatten
 41 |       elif hessian == True:
 42 |          for grad in grads:
 43 |             grad_flatten.append(grad.contiguous().view(-1))
 44 |          grad_flatten = th.cat(grad_flatten).data
 45 |          return grad_flatten
 46 | 
 47 |     def hessian_vector_product(self, obs, p, damping_coeff=0.1):
 48 |         #p=p.detach()
 49 |         self.policy.zero_grad()
 50 |         old_pi=self.old_policy(obs)
 51 |         pi=self.policy(obs)+1e-8
 52 |         kl=(old_pi*th.log(old_pi/pi)).sum(1)
 53 | 
 54 |         kl=kl.mean()
 55 | 
 56 |         kl_grad = th.autograd.grad(kl, self.policy.parameters(), create_graph=True)
 57 |         kl_grad = self.flat_grad(kl_grad)
 58 | 
 59 |         kl_grad_p = (kl_grad * p).sum() 
 60 |         kl_hessian = th.autograd.grad(kl_grad_p, self.policy.parameters())
 61 |         kl_hessian = self.flat_grad(kl_hessian, hessian=True)
 62 |         #print(p)
 63 |         return kl_hessian + p.detach() * damping_coeff
 64 | 
 65 |     def cg(self, obs, b, cg_iters=10, EPS=1e-5, residual_tol=1e-10):
 66 |         # Conjugate gradient algorithm
 67 |         # (https://en.wikipedia.org/wiki/Conjugate_gradient_method)
 68 |         x = th.zeros(b.size()).to(device)
 69 |         r = b.clone()
 70 |         p = r.clone()
 71 |         rdotr = th.dot(r,r).to(device)
 72 | 
 73 |         for _ in range(cg_iters):
 74 |             Ap = self.hessian_vector_product(obs, p)
 75 |             alpha = rdotr / (th.dot(p, Ap).to(device) + EPS)
 76 | 
 77 |             x += alpha * p
 78 |             r -= alpha * Ap
 79 |             
 80 |             new_rdotr = th.dot(r, r)
 81 |             p = r + (new_rdotr / rdotr) * p
 82 |             rdotr = new_rdotr
 83 | 
 84 |             if rdotr < residual_tol:
 85 |                 break
 86 |         return x
 87 | 
 88 |     def update_params(self,policy,flat_params):
 89 |         index=0
 90 |         for p in policy.parameters():
 91 |             length=len(p.view(-1))
 92 |             params=flat_params[index:index+length]
 93 |             params=params.view(p.size())
 94 |             p.data.copy_(params)
 95 |             index+=length
 96 | 
 97 | 
 98 | 
 99 | 
100 |     def learn(self,batch):
101 | 
102 |         for trajectory in batch:
103 |             s=th.Tensor(trajectory[0]).to(device)
104 |             a=th.LongTensor(trajectory[1]).to(device)
105 |             r=th.Tensor(trajectory[2]).to(device)
106 |             g=trajectory[3]
107 |             
108 |             for _ in range(v_update_iter):
109 |                 v=self.v(s)
110 |                 loss=(v-g.detach())**2
111 |                 loss=loss.sum()
112 |                 self.optim.zero_grad()
113 |                 loss.backward()
114 |                 self.optim.step()
115 |             adv=g-v
116 |             adv = (adv - adv.mean()) /(adv.std() + 1e-8)
117 |             log_prob=th.log(th.gather(self.policy(s),1,a))
118 |             log_prob_old=th.log(th.gather(self.old_policy(s),1,a))+1e-8
119 |             prob=th.exp(log_prob-log_prob_old.detach())
120 |             policy_loss=(prob*adv.detach()).mean()
121 |             
122 |             gra=th.autograd.grad(policy_loss,self.policy.parameters(),create_graph=True)
123 |             gra=self.flat_grad(gra)
124 |             x=self.cg(s,gra.data)
125 | 
126 |             H=self.hessian_vector_product(s,x)
127 |             xHx=(H*x).sum(0)
128 |             if xHx<0:    #unknown bug, H may not be positive definite at the beginning of training
129 |                 xHx=th.Tensor([1000]).to(device)
130 |             if xHx>0:
131 |                 lr=th.sqrt(2*delta/xHx)
132 |                 
133 |                 self.old_policy.load_state_dict(self.policy.state_dict())
134 |                 #natural policy gradient
135 |                 params=self.policy.parameters()
136 |                 p=[]
137 |                 for i in params:
138 |                     p.append(i.view(-1))
139 |                 params=th.cat(p)
140 |                 params+=lr*x
141 |                 self.update_params(self.policy,params)
142 |             
143 | trpo=TRPO()
144 | start_time=time.time()
145 | transition_s=[]
146 | transition_a=[]
147 | transition_r=[]
148 | trajectory=[]
149 | for episode in range(10000):
150 |     t=0
151 |     s=env.reset()
152 |     total_reward=0
153 | 
154 |     while(t<300):
155 |         a=trpo.choose_action(s)
156 |         s_,r,done,_=env.step(a)
157 |         total_reward+=r
158 | 
159 |         transition_s.append(s.tolist())
160 |         transition_a.append([a])
161 |         transition_r.append([r])
162 | 
163 |         if done:
164 |             g=th.zeros((np.array(transition_r).shape[0]),1).to(device)
165 |             for i in reversed(range(0,np.array(transition_r).shape[0]-1)):
166 |                 g[i]=g[i+1]*gamma+transition_r[i][0]
167 | 
168 |             trajectory.append([transition_s,transition_a,transition_r,g])
169 |             transition_s=[]
170 |             transition_a=[]
171 |             transition_r=[]
172 |             if len(trajectory)==update_interval:
173 |                 trpo.learn(np.array(trajectory))
174 |                 trajectory=[]
175 |             break
176 |         s=s_
177 |         t+=1
178 |     if episode%100==0:
179 |         print("Episode:"+format(episode)+",total score:"+format(total_reward))
180 | end_time=time.time()
181 | print(end_time-start_time)
182 |         
183 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/TRPO/model.py:
--------------------------------------------------------------------------------
 1 | import torch as th
 2 | import numpy as np
 3 | import torch.nn as nn
 4 | import torch.nn.functional as F
 5 | 
 6 | class net(nn.Module):
 7 |     def __init__(self,n_state,n_action):
 8 |         super(net,self).__init__()
 9 |         self.fc1=nn.Linear(n_state,256)
10 |         self.fc2=nn.Linear(256,n_action)
11 |         self.softmax=nn.Softmax()
12 |     def forward(self,s):
13 |         s=th.relu(self.fc1(s))
14 |         return self.softmax(self.fc2(s))
15 | 
16 | class v(nn.Module):
17 |     def __init__(self,n_state):
18 |         super(v,self).__init__()
19 |         self.fc1=nn.Linear(n_state,256)
20 |         self.fc2=nn.Linear(256,1)
21 |     def forward(self, s):
22 |         return self.fc2(F.relu(self.fc1(s)))
23 | 


--------------------------------------------------------------------------------
/algorithm/policy gradient/baseline_REINFORCE.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np
  4 | import torch.nn as nn
  5 | import torch.nn.functional as F
  6 | import time
  7 | 
  8 | alpha=0.001
  9 | max_t=200
 10 | gamma=0.9
 11 | hidden=32
 12 | env=gym.make('CartPole-v0')
 13 | device="cuda"
 14 | env=env.unwrapped
 15 | n_action=env.action_space.n
 16 | n_state=env.observation_space.shape[0]
 17 | 
 18 | 
 19 | class policy(nn.Module):
 20 |     def __init__(self):
 21 |         super(policy,self).__init__()
 22 |         self.fc1=nn.Linear(n_state,hidden)
 23 |         self.fc2=nn.Linear(hidden,n_action)
 24 |         self.softmax=nn.Softmax()
 25 |     def forward(self,x):
 26 |         x=self.fc1(x)
 27 |         x=F.relu(x)
 28 |         x=self.fc2(x)
 29 |         prob=self.softmax(x)
 30 |         return prob
 31 | 
 32 | class Vnet(nn.Module):
 33 |     def __init__(self):
 34 |         super(Vnet,self).__init__()
 35 |         self.fc1=nn.Linear(n_state,hidden)
 36 |         self.fc2=nn.Linear(hidden,1)
 37 |     def forward(self,x):
 38 |         x=self.fc1(x)
 39 |         x=F.relu(x)
 40 |         out=self.fc2(x)
 41 |         return out
 42 | 
 43 | 
 44 | class baseline_REINFORCE():
 45 |     def __init__(self):
 46 |         self.policy_net=policy().to(device)
 47 |         self.V=Vnet().to(device)               #baseline
 48 |         self.g=0
 49 |         self.optimizerP=th.optim.Adam(self.policy_net.parameters(),lr=alpha)
 50 |         self.optimizerV=th.optim.Adam(self.V.parameters(),lr=alpha)
 51 |     def choose_action(self,s):
 52 |         s=th.FloatTensor(s).to(device)
 53 |         a_prob=self.policy_net(s)
 54 |         rand=np.random.uniform()
 55 |         accumulation=0
 56 |         action=0
 57 |         for i in range(n_action):
 58 |             accumulation+=a_prob[i]
 59 |             if accumulation>=rand:
 60 |                 action=i
 61 |                 break
 62 |         return action
 63 | 
 64 |     def learn(self,transition):
 65 |         timestep=len(transition)
 66 |         loss=0
 67 |         loss_v=0
 68 |         returns=th.zeros(timestep,1)
 69 |         log_prob=th.zeros(timestep,1)
 70 |         v=th.zeros(timestep,1)
 71 |         self.g=0
 72 |         for i in reversed(range(timestep)):
 73 |             s=th.FloatTensor(transition[i,0]).to(device)
 74 |             a=transition[i,1][0]
 75 |             r=transition[i,2][0]
 76 |             v[i]=self.V(s)            #calculate b(s)
 77 |             log_prob[i]=th.log(self.policy_net(s))[a].unsqueeze(0)
 78 |             self.g=gamma*self.g+r
 79 |             returns[i]=self.g
 80 | 
 81 |         loss=-(log_prob*(returns-v).detach()).sum()
 82 |         loss_v=(0.5*(returns-v)**2).sum()
 83 | 
 84 |         #update separately
 85 |         self.optimizerP.zero_grad()
 86 |         loss.backward()
 87 |         self.optimizerP.step()
 88 | 
 89 |         self.optimizerV.zero_grad()
 90 |         loss_v.backward()
 91 |         self.optimizerV.step()
 92 | 
 93 | reinforce=baseline_REINFORCE()
 94 | start_time=time.time()
 95 | for episode in range(1000):
 96 |     t=0
 97 |     s=env.reset()
 98 |     total_reward=0
 99 |     while(t<300):
100 |         a=reinforce.choose_action(s)
101 |         s_,r,done,_=env.step(a)
102 |         total_reward+=r
103 |         trans=[s,[a],[r]]
104 |         if t==0:
105 |             transition=trans
106 |         else:
107 |             transition=np.vstack((transition,trans))
108 |         if done:
109 |             reinforce.learn(transition)
110 |             break
111 |         s=s_
112 |         t+=1
113 |     if episode%100==0:
114 |         print("Episode:"+format(episode)+",total score:"+format(total_reward))
115 | end_time=time.time()
116 | print(end_time-start_time)
117 | 
118 | 
119 |     
120 | 


--------------------------------------------------------------------------------
/algorithm/value-based/DQN.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np 
  4 | 
  5 | batch_size=50
  6 | lr=0.001
  7 | episilon=0.9
  8 | replay_memory_size=10000
  9 | gamma=0.9
 10 | target_update_iter=100
 11 | env=gym.make('CartPole-v0')
 12 | device="cuda"
 13 | env=env.unwrapped
 14 | n_action=env.action_space.n
 15 | n_state=env.observation_space.shape[0]
 16 | hidden=32
 17 | 
 18 | class net(th.nn.Module):
 19 |     def __init__(self):
 20 |         super(net,self).__init__()
 21 |         self.fc1=th.nn.Linear(n_state,hidden)
 22 |         self.fc1.weight.data.normal_(0, 0.1)
 23 |         self.out=th.nn.Linear(hidden,n_action)
 24 |         self.out.weight.data.normal_(0, 0.1)
 25 |     
 26 | 
 27 |     def forward(self,x):
 28 |         x=self.fc1(x)
 29 |         x=th.nn.functional.relu(x)
 30 |         out=self.out(x)
 31 |         return out
 32 | 
 33 | class replay_memory():
 34 |     def __init__(self):
 35 |         self.memory_size=replay_memory_size
 36 |         self.memory=np.array([])
 37 |         self.cur=0
 38 |         self.new=0
 39 |     def size(self):
 40 |         return self.memory.shape[0]
 41 | #[s,a,r,s_,done]
 42 |     def store_transition(self,trans):
 43 |         if(self.memory.shape[0]<self.memory_size):
 44 |             if self.new==0:
 45 |                 self.memory=np.array(trans)
 46 |                 self.new=1
 47 |             elif self.memory.shape[0]>0:
 48 |                 self.memory=np.vstack((self.memory,trans))
 49 | 
 50 |         else:
 51 |             self.memory[self.cur,:]=trans
 52 |             self.cur=(self.cur+1)%self.memory_size
 53 |     
 54 |     def sample(self):
 55 |         if self.memory.shape[0]<batch_size:
 56 |             return -1
 57 |         sam=np.random.choice(self.memory.shape[0],batch_size)
 58 |         return self.memory[sam]
 59 |     
 60 | class DQN(object):
 61 |     def __init__(self):
 62 |         self.eval_q_net,self.target_q_net=net().to(device),net().to(device)
 63 |         self.replay_mem=replay_memory()
 64 |         self.iter_num=0
 65 |         self.optimizer=th.optim.Adam(self.eval_q_net.parameters(),lr=lr)
 66 |         self.loss=th.nn.MSELoss().to(device)
 67 |     def choose_action(self,qs):
 68 |         if np.random.uniform()<episilon:
 69 |             return th.argmax(qs).tolist()
 70 |         else:
 71 |             return np.random.randint(0,n_action)
 72 |     def greedy_action(self,qs):
 73 |         return th.argmax(qs)
 74 |     def learn(self):
 75 |         if(self.iter_num%target_update_iter==0):
 76 |             self.target_q_net.load_state_dict(self.eval_q_net.state_dict())
 77 |         self.iter_num+=1
 78 | 
 79 |         batch=self.replay_mem.sample()
 80 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 81 |         b_a=th.LongTensor(batch[:,1].astype(int).tolist()).to(device)
 82 |         b_r=th.FloatTensor(batch[:,2].tolist()).to(device)
 83 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 84 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 85 |         q_target=th.zeros((batch_size,1)).to(device)
 86 |         q_eval=self.eval_q_net(b_s)
 87 |         q_eval=th.gather(q_eval,dim=1,index=th.unsqueeze(b_a,1))
 88 |         q_next=self.target_q_net(b_s_).detach()
 89 |         for i in range(b_d.shape[0]):
 90 |             if(int(b_d[i].tolist()[0])==0):
 91 |                 q_target[i]=b_r[i]+gamma*th.unsqueeze(th.max(q_next[i],0)[0],0)
 92 |             else:
 93 |                 q_target[i]=b_r[i]
 94 |         td_error=self.loss(q_eval,q_target)
 95 | 
 96 |         self.optimizer.zero_grad()
 97 |         td_error.backward()
 98 |         self.optimizer.step()
 99 | 
100 | dqn=DQN()
101 | 
102 | for episode in range(10000):
103 |     s=env.reset()
104 |     t=0
105 |     r=0.0
106 |     while(t<300):
107 |         t+=1
108 |         qs=dqn.eval_q_net(th.FloatTensor(s).to(device))
109 |         a=dqn.choose_action(qs)
110 |         s_,r,done,_=env.step(a)
111 |         transition=[s.tolist(),a,[r],s_.tolist(),[done]]
112 |         dqn.replay_mem.store_transition(transition)
113 |         s=s_
114 |         if dqn.replay_mem.size()>batch_size:
115 |             dqn.learn()
116 |         if done:
117 |             break
118 |     if episode%100==0: #test
119 |         total_reward=0.0
120 |         for i in range(10):
121 |             t_s=env.reset()
122 |             t_r=0.0
123 |             tr=0.0
124 |             time=0
125 |             while(time<300):
126 |                 time+=1
127 |                 t_qs=dqn.eval_q_net(th.FloatTensor(t_s).to(device))
128 |                 t_a=dqn.greedy_action(t_qs).item()
129 |                 ts_,tr,tdone,_=env.step(t_a)
130 |                 t_r+=tr
131 |                 if tdone:
132 |                     break
133 |                 t_s=ts_
134 |             total_reward+=t_r
135 |         print("episode:"+format(episode)+",test score:"+format(total_reward/10))
136 | 
137 | 
138 | 
139 | 
140 | 


--------------------------------------------------------------------------------
/algorithm/value-based/DoubleDQN.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np 
  4 | 
  5 | batch_size=50
  6 | lr=0.001
  7 | episilon=0.9
  8 | replay_memory_size=10000
  9 | gamma=0.9
 10 | target_update_iter=100
 11 | log_internval=10
 12 | env=gym.make('CartPole-v0')
 13 | device="cuda"
 14 | env=env.unwrapped
 15 | n_action=env.action_space.n
 16 | n_state=env.observation_space.shape[0]
 17 | hidden=256
 18 | 
 19 | class net(th.nn.Module):
 20 |     def __init__(self):
 21 |         super(net,self).__init__()
 22 |         self.fc1=th.nn.Linear(n_state,hidden)
 23 |         self.fc1.weight.data.normal_(0, 0.1)
 24 |         self.out=th.nn.Linear(hidden,n_action)
 25 |         self.out.weight.data.normal_(0, 0.1)
 26 |     
 27 | 
 28 |     def forward(self,x):
 29 |         x=self.fc1(x)
 30 |         x=th.nn.functional.relu(x)
 31 |         out=self.out(x)
 32 |         return out
 33 | 
 34 | class replay_memory():
 35 |     def __init__(self):
 36 |         self.memory_size=replay_memory_size
 37 |         self.memory=np.array([])
 38 |         self.cur=0
 39 |         self.new=0
 40 |     def size(self):
 41 |         return self.memory.shape[0]
 42 | #[s,a,r,s_,done]
 43 |     def store_transition(self,trans):
 44 |         if(self.memory.shape[0]<self.memory_size):
 45 |             if self.new==0:
 46 |                 self.memory=np.array(trans)
 47 |                 self.new=1
 48 |             elif self.memory.shape[0]>0:
 49 |                 self.memory=np.vstack((self.memory,trans))
 50 | 
 51 |         else:
 52 |             self.memory[self.cur,:]=trans
 53 |             self.cur=(self.cur+1)%self.memory_size
 54 |     
 55 |     def sample(self):
 56 |         if self.memory.shape[0]<batch_size:
 57 |             return -1
 58 |         sam=np.random.choice(self.memory.shape[0],batch_size)
 59 |         return self.memory[sam]
 60 |     
 61 | class DQN(object):
 62 |     def __init__(self):
 63 |         self.eval_q_net,self.target_q_net=net().to(device),net().to(device)
 64 |         self.replay_mem=replay_memory()
 65 |         self.iter_num=0
 66 |         self.optimizer=th.optim.Adam(self.eval_q_net.parameters(),lr=lr)
 67 |         self.loss=th.nn.MSELoss().to(device)
 68 |     def choose_action(self,qs):
 69 |         if np.random.uniform()<episilon:
 70 |             return th.argmax(qs).tolist()
 71 |         else:
 72 |             return np.random.randint(0,n_action)
 73 |     def greedy_action(self,qs):
 74 |         return th.argmax(qs)
 75 |     def learn(self):
 76 |         if(self.iter_num%target_update_iter==0):
 77 |             self.target_q_net.load_state_dict(self.eval_q_net.state_dict())
 78 |         self.iter_num+=1
 79 | 
 80 |         batch=self.replay_mem.sample()
 81 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 82 |         b_a=th.LongTensor(batch[:,1].astype(int).tolist()).to(device)
 83 |         b_r=th.FloatTensor(batch[:,2].tolist()).to(device)
 84 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
 85 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
 86 |         q_target=th.zeros((batch_size,1)).to(device)
 87 |         q_eval=self.eval_q_net(b_s)
 88 |         q=q_eval
 89 |         q_eval=th.gather(q_eval,dim=1,index=th.unsqueeze(b_a,1))
 90 |         q_next=self.target_q_net(b_s_).detach()
 91 |         for i in range(b_d.shape[0]):
 92 |             if(int(b_d[i].tolist()[0])==0):
 93 |                 #decouple action selection and action evaluation
 94 |                 action=th.argmax(q[i],0).detach()
 95 |                 #q_target[i]=b_r[i]+gamma*th.unsqueeze(th.max(q_next[i],0)[0],0) for DQN
 96 |                 q_target[i]=b_r[i]+gamma*q_next[i,action]
 97 |             else:
 98 |                 q_target[i]=b_r[i]
 99 |         td_error=self.loss(q_eval,q_target)
100 | 
101 |         self.optimizer.zero_grad()
102 |         td_error.backward()
103 |         self.optimizer.step()
104 | 
105 | dqn=DQN()
106 | 
107 | for episode in range(10000):
108 |     s=env.reset()
109 |     t=0
110 |     r=0.0
111 |     while(t<300):
112 |         t+=1
113 |         qs=dqn.eval_q_net(th.FloatTensor(s).to(device))
114 |         a=dqn.choose_action(qs)
115 |         s_,r,done,_=env.step(a)
116 |         transition=[s.tolist(),a,[r],s_.tolist(),[done]]
117 |         dqn.replay_mem.store_transition(transition)
118 |         s=s_
119 |         if dqn.replay_mem.size()>batch_size:
120 |             dqn.learn()
121 |         if done:
122 |             break
123 |     if episode%log_internval==0: #test
124 |         total_reward=0.0
125 |         for i in range(10):
126 |             t_s=env.reset()
127 |             t_r=0.0
128 |             tr=0.0
129 |             time=0
130 |             while(time<300):
131 |                 time+=1
132 |                 t_qs=dqn.eval_q_net(th.FloatTensor(t_s).to(device))
133 |                 t_a=dqn.greedy_action(t_qs).item()
134 |                 ts_,tr,tdone,_=env.step(t_a)
135 |                 t_r+=tr
136 |                 if tdone:
137 |                     break
138 |                 t_s=ts_
139 |             total_reward+=t_r
140 |         print("episode:"+format(episode)+",test score:"+format(total_reward/10))
141 | 
142 | 
143 | 
144 | 
145 | 


--------------------------------------------------------------------------------
/algorithm/value-based/DuelingDQN.py:
--------------------------------------------------------------------------------
  1 | import gym
  2 | import torch as th
  3 | import numpy as np 
  4 | 
  5 | batch_size=10
  6 | lr=0.001
  7 | episilon=0.9
  8 | replay_memory_size=10000
  9 | gamma=0.9
 10 | target_update_iter=100
 11 | log_internval=10
 12 | env=gym.make('CartPole-v0')
 13 | device="cuda"
 14 | env=env.unwrapped
 15 | n_action=env.action_space.n
 16 | n_state=env.observation_space.shape[0]
 17 | hidden1=256
 18 | hidden2=256
 19 | 
 20 | class net(th.nn.Module):
 21 |     def __init__(self):
 22 |         super(net,self).__init__()
 23 |         self.fc1=th.nn.Linear(n_state,hidden1)
 24 |         self.fc1.weight.data.normal_(0, 0.1)
 25 |         self.fc2=th.nn.Linear(hidden1,hidden2)
 26 |         self.fc2.weight.data.normal_(0, 0.1)
 27 |         self.V=th.nn.Linear(hidden2,1)
 28 |         self.V.weight.data.normal_(0,0.1)
 29 |         self.A=th.nn.Linear(hidden2,n_action)
 30 |         self.A.weight.data.normal_(0,0.1)
 31 |     
 32 | 
 33 |     def forward(self,x):
 34 |         x=self.fc1(x)
 35 |         x=th.nn.functional.relu(x)
 36 |         x=self.fc2(x)
 37 |         x=th.nn.functional.relu(x)
 38 |         v=self.V(x)
 39 |         a=self.A(x)
 40 |         if len(a.shape)==1:
 41 |             a-=th.mean(a,dim=0)
 42 |         else:   #for batch
 43 |             a-=th.mean(a,dim=-1).unsqueeze(1).repeat(1,2)
 44 | 
 45 |         if len(a.shape)==1:
 46 |             out=a+v.squeeze()
 47 |         else:   #for batch
 48 |             out=a+v.repeat(1,2)
 49 |         return out
 50 | 
 51 | class replay_memory():
 52 |     def __init__(self):
 53 |         self.memory_size=replay_memory_size
 54 |         self.memory=np.array([])
 55 |         self.cur=0
 56 |         self.new=0
 57 |     def size(self):
 58 |         return self.memory.shape[0]
 59 |     #[s,a,r,s_,done]
 60 |     def store_transition(self,trans):
 61 |         if(self.memory.shape[0]<self.memory_size):
 62 |             if self.new==0:
 63 |                 self.memory=np.array(trans)
 64 |                 self.new=1
 65 |             elif self.memory.shape[0]>0:
 66 |                 self.memory=np.vstack((self.memory,trans))
 67 | 
 68 |         else:
 69 |             self.memory[self.cur,:]=trans
 70 |             self.cur=(self.cur+1)%self.memory_size
 71 |     
 72 |     def sample(self):
 73 |         if self.memory.shape[0]<batch_size:
 74 |             return -1
 75 |         sam=np.random.choice(self.memory.shape[0],batch_size)
 76 |         return self.memory[sam]
 77 |     
 78 | class DQN(object):
 79 |     def __init__(self):
 80 |         self.eval_q_net,self.target_q_net=net().to(device),net().to(device)
 81 |         self.replay_mem=replay_memory()
 82 |         self.iter_num=0
 83 |         self.optimizer=th.optim.Adam(self.eval_q_net.parameters(),lr=lr)
 84 |         self.loss=th.nn.MSELoss().to(device)
 85 |     def choose_action(self,qs):
 86 |         if np.random.uniform()<episilon:
 87 |             return th.argmax(qs).tolist()
 88 |         else:
 89 |             return np.random.randint(0,n_action)
 90 |     def greedy_action(self,qs):
 91 |         return th.argmax(qs)
 92 |     def learn(self):
 93 |         if(self.iter_num%target_update_iter==0):
 94 |             self.target_q_net.load_state_dict(self.eval_q_net.state_dict())
 95 |         self.iter_num+=1
 96 | 
 97 |         batch=self.replay_mem.sample()
 98 |         b_s=th.FloatTensor(batch[:,0].tolist()).to(device)
 99 |         b_a=th.LongTensor(batch[:,1].astype(int).tolist()).to(device)
100 |         b_r=th.FloatTensor(batch[:,2].tolist()).to(device)
101 |         b_s_=th.FloatTensor(batch[:,3].tolist()).to(device)
102 |         b_d=th.FloatTensor(batch[:,4].tolist()).to(device)
103 |         q_target=th.zeros((batch_size,1)).to(device)
104 |         q_eval=self.eval_q_net(b_s)
105 |         q_eval=th.gather(q_eval,dim=1,index=th.unsqueeze(b_a,1))
106 |         q_next=self.target_q_net(b_s_).detach()
107 |         for i in range(b_d.shape[0]):
108 |             if(int(b_d[i].tolist()[0])==0):
109 |                 q_target[i]=b_r[i]+gamma*th.unsqueeze(th.max(q_next[i],0)[0],0)
110 |             else:
111 |                 q_target[i]=b_r[i]
112 |         td_error=self.loss(q_eval,q_target)
113 | 
114 |         self.optimizer.zero_grad()
115 |         td_error.backward()
116 |         self.optimizer.step()
117 | 
118 | dqn=DQN()
119 | 
120 | for episode in range(10000):
121 |     s=env.reset()
122 |     t=0
123 |     r=0.0
124 |     while(t<300):
125 |         t+=1
126 |         qs=dqn.eval_q_net(th.FloatTensor(s).to(device))
127 |         a=dqn.choose_action(qs)
128 |         s_,r,done,_=env.step(a)
129 |         transition=[s.tolist(),a,[r],s_.tolist(),[done]]
130 |         dqn.replay_mem.store_transition(transition)
131 |         s=s_
132 |         if dqn.replay_mem.size()>batch_size:
133 |             dqn.learn()
134 |         if done:
135 |             break
136 |     if episode%log_internval==0: #test
137 |         total_reward=0.0
138 |         for i in range(10):
139 |             t_s=env.reset()
140 |             t_r=0.0
141 |             tr=0.0
142 |             time=0
143 |             while(time<300):
144 |                 time+=1
145 |                 t_qs=dqn.eval_q_net(th.FloatTensor(t_s).to(device))
146 |                 t_a=dqn.greedy_action(t_qs).item()
147 |                 ts_,tr,tdone,_=env.step(t_a)
148 |                 t_r+=tr
149 |                 if tdone:
150 |                     break
151 |                 t_s=ts_
152 |             total_reward+=t_r
153 |         print("episode:"+format(episode)+",test score:"+format(total_reward/10))
154 | 
155 | 
156 | 
157 | 
158 | 


--------------------------------------------------------------------------------
/algorithm/value-based/Sarsa.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import gym
  3 | import torch as th
  4 | 
  5 | env=gym.make('CartPole-v0')
  6 | gamma=0.9
  7 | episilon=0.9
  8 | lr=0.001
  9 | target_update_iter=100
 10 | log_internval=10
 11 | env=gym.make('CartPole-v0')
 12 | device="cuda"
 13 | env=env.unwrapped
 14 | n_action=env.action_space.n
 15 | n_state=env.observation_space.shape[0]
 16 | hidden=256
 17 | 
 18 | class net(th.nn.Module):
 19 |     def __init__(self):
 20 |         super(net,self).__init__()
 21 |         self.fc1=th.nn.Linear(n_state,hidden)
 22 |         self.out=th.nn.Linear(hidden,n_action)
 23 |     
 24 | 
 25 |     def forward(self,x):
 26 |         x=self.fc1(x)
 27 |         x=th.nn.functional.relu(x)
 28 |         out=self.out(x)
 29 |         return out
 30 | 
 31 | class Sarsa():
 32 |     def __init__(self):
 33 |         self.net,self.target_net=net(),net()
 34 |         self.iter_num=0
 35 |         self.optimizer=th.optim.Adam(self.net.parameters(),lr=lr)
 36 | 
 37 |     def learn(self,s,a,s_,r,done):
 38 |         eval_q=self.net(th.Tensor(s))[a]
 39 |         target_q=self.target_net(th.FloatTensor(s_))
 40 |         target_a=self.choose_action(target_q)
 41 |         target_q=target_q[target_a]
 42 |         if not done:
 43 |             y=gamma*target_q+r
 44 |         else:
 45 |             y=r
 46 |         loss=(y-eval_q)**2
 47 |         self.optimizer.zero_grad()
 48 |         loss.backward()
 49 |         self.optimizer.step()
 50 |         self.iter_num+=1
 51 |         if self.iter_num%10==0:
 52 |             self.target_net.load_state_dict(self.net.state_dict())
 53 |         return target_a
 54 | 
 55 |     def greedy_action(self,qs):
 56 |         return th.argmax(qs)
 57 | 
 58 |     def random_action(self):
 59 |         return np.random.randint(0,n_action)
 60 | 
 61 |     def choose_action(self,qs):
 62 |         if np.random.rand()>episilon:
 63 |             return self.random_action()
 64 |         else:
 65 |             return self.greedy_action(qs).tolist()
 66 | 
 67 | sarsa=Sarsa()
 68 | for episode in range(10000):
 69 |     s=env.reset()
 70 |     t=0
 71 |     r=0.0
 72 |     qs=sarsa.net(th.Tensor(s))
 73 |     a=sarsa.choose_action(qs)
 74 |     while(t<300):
 75 |         t+=1
 76 |         #print(a)
 77 |         s_,r,done,_=env.step(a)
 78 |         a=sarsa.learn(s,a,s_,r,done)
 79 |         s=s_
 80 |         if done:
 81 |             break
 82 |     if episode%log_internval==0: #test
 83 |         total_reward=0.0
 84 |         for i in range(10):
 85 |             t_s=env.reset()
 86 |             t_r=0.0
 87 |             tr=0.0
 88 |             time=0
 89 |             while(time<300):
 90 |                 time+=1
 91 |                 qs=sarsa.net(th.Tensor(t_s))
 92 |                 a=sarsa.greedy_action(qs)
 93 |                 ts_,tr,tdone,_=env.step(a.tolist())
 94 |                 t_r+=tr
 95 |                 if tdone:
 96 |                     break
 97 |                 t_s=ts_
 98 |             total_reward+=t_r
 99 |         print("episode:"+format(episode)+",test score:"+format(total_reward/10))
100 | 
101 | 
102 | 
103 |         
104 | 
105 | 
106 | 


--------------------------------------------------------------------------------