├── .gitattributes
├── Cartpole.png
├── model
    ├── actor.pkl
    └── critic.pkl
├── reinforcement_learning.png
├── .idea
    ├── misc.xml
    ├── modules.xml
    ├── Actor-Critic-pytorch.iml
    └── workspace.xml
├── LICENSE
├── .gitignore
├── README.md
└── Actor-Critic.py


/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 | 


--------------------------------------------------------------------------------
/Cartpole.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yc930401/Actor-Critic-pytorch/HEAD/Cartpole.png


--------------------------------------------------------------------------------
/model/actor.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yc930401/Actor-Critic-pytorch/HEAD/model/actor.pkl


--------------------------------------------------------------------------------
/model/critic.pkl:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yc930401/Actor-Critic-pytorch/HEAD/model/critic.pkl


--------------------------------------------------------------------------------
/reinforcement_learning.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yc930401/Actor-Critic-pytorch/HEAD/reinforcement_learning.png


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/Actor-Critic-pytorch.iml" filepath="$PROJECT_DIR$/.idea/Actor-Critic-pytorch.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/Actor-Critic-pytorch.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="projectConfiguration" value="Twisted Trial" />
10 |     <option name="PROJECT_TEST_RUNNER" value="Twisted Trial" />
11 |   </component>
12 | </module>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Yang Cheng
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # DQN to play Cartpole game with pytorch
 2 | 
 3 | DQN to play Cartpole game with pytorch
 4 | 
 5 | ## Introduction
 6 | 
 7 | Humans excel at solving a wide variety of challenging problems, from low-level motor control through to high-level cognitive tasks. 
 8 | Like a human, our agents learn for themselves to achieve successful strategies that lead to the greatest long-term rewards. This paradigm of 
 9 | learning by trial-and-error, solely from rewards or punishments, is known as reinforcement learning (RL). Also like a human, our agents 
10 | construct and learn their own knowledge directly from raw inputs, such as vision, without any hand-engineered features or domain heuristics. 
11 | This is achieved by deep learning of neural networks.
12 | The agents must continually make value judgements so as to select good actions over bad. This knowledge is represented by a Q-network that 
13 | estimates the total reward that an agent can expect to receive after taking a particular action. The key idea was to use deep neural networks 
14 | to represent the Q-network, and to train this Q-network to predict total reward. Previous attempts to combine RL with neural networks had 
15 | largely failed due to unstable learning. To address these instabilities, our Deep Q-Networks (DQN) algorithm stores all of the agent's experiences
16 |  and then randomly samples and replays these experiences to provide diverse and decorrelated training data. </br>
17 | Reinforcement learning: </br>
18 | ![reinforcement learning](reinforcement_learning.png) </br>
19 | In this post, I implement a DQN to Cartpole game: </br>
20 | ![Cartpole](Cartpole.png) </br>
21 | 
22 | 
23 | ## Methodology
24 | 
25 | 1. Define a Actor network and a Critic Network
26 | 2. Get data (state, next_state, reward, done signals) from gym
27 | 3. Play Cartpole game and calculate rewards for each step at the end of one game, train the two networks
28 | 4. Save the model
29 | 
30 | 
31 | 
32 | ## References:
33 | https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/5-2-policy-gradient-softmax2/ </br>
34 | https://github.com/higgsfield/RL-Adventure-2/blob/master/1.actor-critic.ipynb </br>
35 | https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/5-2-policy-gradient-softmax2/ </br>
36 | https://arxiv.org/pdf/1509.02971.pdf </br>
37 | 


--------------------------------------------------------------------------------
/Actor-Critic.py:
--------------------------------------------------------------------------------
  1 | import gym, os
  2 | from itertools import count
  3 | import torch
  4 | import torch.nn as nn
  5 | import torch.optim as optim
  6 | import torch.nn.functional as F
  7 | from torch.distributions import Categorical
  8 | 
  9 | 
 10 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 11 | env = gym.make("CartPole-v0").unwrapped
 12 | 
 13 | state_size = env.observation_space.shape[0]
 14 | action_size = env.action_space.n
 15 | lr = 0.0001
 16 | 
 17 | class Actor(nn.Module):
 18 |     def __init__(self, state_size, action_size):
 19 |         super(Actor, self).__init__()
 20 |         self.state_size = state_size
 21 |         self.action_size = action_size
 22 |         self.linear1 = nn.Linear(self.state_size, 128)
 23 |         self.linear2 = nn.Linear(128, 256)
 24 |         self.linear3 = nn.Linear(256, self.action_size)
 25 | 
 26 |     def forward(self, state):
 27 |         output = F.relu(self.linear1(state))
 28 |         output = F.relu(self.linear2(output))
 29 |         output = self.linear3(output)
 30 |         distribution = Categorical(F.softmax(output, dim=-1))
 31 |         return distribution
 32 | 
 33 | 
 34 | class Critic(nn.Module):
 35 |     def __init__(self, state_size, action_size):
 36 |         super(Critic, self).__init__()
 37 |         self.state_size = state_size
 38 |         self.action_size = action_size
 39 |         self.linear1 = nn.Linear(self.state_size, 128)
 40 |         self.linear2 = nn.Linear(128, 256)
 41 |         self.linear3 = nn.Linear(256, 1)
 42 | 
 43 |     def forward(self, state):
 44 |         output = F.relu(self.linear1(state))
 45 |         output = F.relu(self.linear2(output))
 46 |         value = self.linear3(output)
 47 |         return value
 48 | 
 49 | 
 50 | def compute_returns(next_value, rewards, masks, gamma=0.99):
 51 |     R = next_value
 52 |     returns = []
 53 |     for step in reversed(range(len(rewards))):
 54 |         R = rewards[step] + gamma * R * masks[step]
 55 |         returns.insert(0, R)
 56 |     return returns
 57 | 
 58 | 
 59 | def trainIters(actor, critic, n_iters):
 60 |     optimizerA = optim.Adam(actor.parameters())
 61 |     optimizerC = optim.Adam(critic.parameters())
 62 |     for iter in range(n_iters):
 63 |         state = env.reset()
 64 |         log_probs = []
 65 |         values = []
 66 |         rewards = []
 67 |         masks = []
 68 |         entropy = 0
 69 |         env.reset()
 70 | 
 71 |         for i in count():
 72 |             env.render()
 73 |             state = torch.FloatTensor(state).to(device)
 74 |             dist, value = actor(state), critic(state)
 75 | 
 76 |             action = dist.sample()
 77 |             next_state, reward, done, _ = env.step(action.cpu().numpy())
 78 | 
 79 |             log_prob = dist.log_prob(action).unsqueeze(0)
 80 |             entropy += dist.entropy().mean()
 81 | 
 82 |             log_probs.append(log_prob)
 83 |             values.append(value)
 84 |             rewards.append(torch.tensor([reward], dtype=torch.float, device=device))
 85 |             masks.append(torch.tensor([1-done], dtype=torch.float, device=device))
 86 | 
 87 |             state = next_state
 88 | 
 89 |             if done:
 90 |                 print('Iteration: {}, Score: {}'.format(iter, i))
 91 |                 break
 92 | 
 93 | 
 94 |         next_state = torch.FloatTensor(next_state).to(device)
 95 |         next_value = critic(next_state)
 96 |         returns = compute_returns(next_value, rewards, masks)
 97 | 
 98 |         log_probs = torch.cat(log_probs)
 99 |         returns = torch.cat(returns).detach()
100 |         values = torch.cat(values)
101 | 
102 |         advantage = returns - values
103 | 
104 |         actor_loss = -(log_probs * advantage.detach()).mean()
105 |         critic_loss = advantage.pow(2).mean()
106 | 
107 |         optimizerA.zero_grad()
108 |         optimizerC.zero_grad()
109 |         actor_loss.backward()
110 |         critic_loss.backward()
111 |         optimizerA.step()
112 |         optimizerC.step()
113 |     torch.save(actor, 'model/actor.pkl')
114 |     torch.save(critic, 'model/critic.pkl')
115 |     env.close()
116 | 
117 | 
118 | if __name__ == '__main__':
119 |     if os.path.exists('model/actor.pkl'):
120 |         actor = torch.load('model/actor.pkl')
121 |         print('Actor Model loaded')
122 |     else:
123 |         actor = Actor(state_size, action_size).to(device)
124 |     if os.path.exists('model/critic.pkl'):
125 |         critic = torch.load('model/critic.pkl')
126 |         print('Critic Model loaded')
127 |     else:
128 |         critic = Critic(state_size, action_size).to(device)
129 |     trainIters(actor, critic, n_iters=100)


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="b1fc59c3-33d8-4195-82ed-c4ce3d0ef42e" name="Default" comment="" />
  5 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  6 |     <option name="TRACKING_ENABLED" value="true" />
  7 |     <option name="SHOW_DIALOG" value="false" />
  8 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
  9 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 10 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 11 |   </component>
 12 |   <component name="FileEditorManager">
 13 |     <leaf>
 14 |       <file leaf-file-name="Actor-Critic.py" pinned="false" current-in-tab="false">
 15 |         <entry file="file://$PROJECT_DIR$/Actor-Critic.py">
 16 |           <provider selected="true" editor-type-id="text-editor">
 17 |             <state relative-caret-position="119">
 18 |               <caret line="76" column="71" selection-start-line="76" selection-start-column="51" selection-end-line="76" selection-end-column="71" />
 19 |               <folding>
 20 |                 <element signature="e#0#14#0" expanded="true" />
 21 |               </folding>
 22 |             </state>
 23 |           </provider>
 24 |         </entry>
 25 |       </file>
 26 |       <file leaf-file-name="DDPG.py" pinned="false" current-in-tab="true">
 27 |         <entry file="file://$PROJECT_DIR$/DDPG.py">
 28 |           <provider selected="true" editor-type-id="text-editor">
 29 |             <state relative-caret-position="249">
 30 |               <caret line="108" column="17" selection-start-line="108" selection-start-column="17" selection-end-line="108" selection-end-column="17" />
 31 |               <folding>
 32 |                 <element signature="e#0#9#0" expanded="true" />
 33 |               </folding>
 34 |             </state>
 35 |           </provider>
 36 |         </entry>
 37 |       </file>
 38 |     </leaf>
 39 |   </component>
 40 |   <component name="FileTemplateManagerImpl">
 41 |     <option name="RECENT_TEMPLATES">
 42 |       <list>
 43 |         <option value="Python Script" />
 44 |       </list>
 45 |     </option>
 46 |   </component>
 47 |   <component name="FindInProjectRecents">
 48 |     <findStrings>
 49 |       <find>curr</find>
 50 |       <find>finish_episode</find>
 51 |       <find>floa</find>
 52 |       <find>print</find>
 53 |       <find>action</find>
 54 |     </findStrings>
 55 |   </component>
 56 |   <component name="IdeDocumentHistory">
 57 |     <option name="CHANGED_PATHS">
 58 |       <list>
 59 |         <option value="$PROJECT_DIR$/Actor-Critic.py" />
 60 |         <option value="$PROJECT_DIR$/test2.py" />
 61 |         <option value="$PROJECT_DIR$/test.py" />
 62 |         <option value="$PROJECT_DIR$/DDPG.py" />
 63 |       </list>
 64 |     </option>
 65 |   </component>
 66 |   <component name="ProjectFrameBounds" extendedState="7">
 67 |     <option name="x" value="450" />
 68 |     <option name="y" value="57" />
 69 |     <option name="width" value="1401" />
 70 |     <option name="height" value="981" />
 71 |   </component>
 72 |   <component name="ProjectView">
 73 |     <navigator proportions="" version="1">
 74 |       <foldersAlwaysOnTop value="true" />
 75 |     </navigator>
 76 |     <panes>
 77 |       <pane id="ProjectPane">
 78 |         <subPane>
 79 |           <expand>
 80 |             <path>
 81 |               <item name="Actor-Critic-pytorch" type="b2602c69:ProjectViewProjectNode" />
 82 |               <item name="Actor-Critic-pytorch" type="462c0819:PsiDirectoryNode" />
 83 |             </path>
 84 |           </expand>
 85 |           <select />
 86 |         </subPane>
 87 |       </pane>
 88 |       <pane id="Scope" />
 89 |     </panes>
 90 |   </component>
 91 |   <component name="PropertiesComponent">
 92 |     <property name="settings.editor.selected.configurable" value="configurable.group.appearance" />
 93 |   </component>
 94 |   <component name="RecentsManager">
 95 |     <key name="MoveFile.RECENT_KEYS">
 96 |       <recent name="E:\Actor-Critic-pytorch" />
 97 |       <recent name="E:\DDPG" />
 98 |     </key>
 99 |   </component>
100 |   <component name="RunDashboard">
101 |     <option name="ruleStates">
102 |       <list>
103 |         <RuleState>
104 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
105 |         </RuleState>
106 |         <RuleState>
107 |           <option name="name" value="StatusDashboardGroupingRule" />
108 |         </RuleState>
109 |       </list>
110 |     </option>
111 |   </component>
112 |   <component name="RunManager" selected="Python.DDPG">
113 |     <configuration name="Actor-Critic" type="PythonConfigurationType" factoryName="Python" temporary="true">
114 |       <module name="Actor-Critic-pytorch" />
115 |       <option name="INTERPRETER_OPTIONS" value="" />
116 |       <option name="PARENT_ENVS" value="true" />
117 |       <envs>
118 |         <env name="PYTHONUNBUFFERED" value="1" />
119 |       </envs>
120 |       <option name="SDK_HOME" value="" />
121 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
122 |       <option name="IS_MODULE_SDK" value="true" />
123 |       <option name="ADD_CONTENT_ROOTS" value="true" />
124 |       <option name="ADD_SOURCE_ROOTS" value="true" />
125 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/Actor-Critic.py" />
126 |       <option name="PARAMETERS" value="" />
127 |       <option name="SHOW_COMMAND_LINE" value="false" />
128 |       <option name="EMULATE_TERMINAL" value="false" />
129 |       <option name="MODULE_MODE" value="false" />
130 |     </configuration>
131 |     <configuration name="DDPG" type="PythonConfigurationType" factoryName="Python" temporary="true">
132 |       <module name="Actor-Critic-pytorch" />
133 |       <option name="INTERPRETER_OPTIONS" value="" />
134 |       <option name="PARENT_ENVS" value="true" />
135 |       <envs>
136 |         <env name="PYTHONUNBUFFERED" value="1" />
137 |       </envs>
138 |       <option name="SDK_HOME" value="" />
139 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
140 |       <option name="IS_MODULE_SDK" value="true" />
141 |       <option name="ADD_CONTENT_ROOTS" value="true" />
142 |       <option name="ADD_SOURCE_ROOTS" value="true" />
143 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/DDPG.py" />
144 |       <option name="PARAMETERS" value="" />
145 |       <option name="SHOW_COMMAND_LINE" value="false" />
146 |       <option name="EMULATE_TERMINAL" value="false" />
147 |       <option name="MODULE_MODE" value="false" />
148 |     </configuration>
149 |     <configuration name="test" type="PythonConfigurationType" factoryName="Python" temporary="true">
150 |       <module name="Actor-Critic-pytorch" />
151 |       <option name="INTERPRETER_OPTIONS" value="" />
152 |       <option name="PARENT_ENVS" value="true" />
153 |       <envs>
154 |         <env name="PYTHONUNBUFFERED" value="1" />
155 |       </envs>
156 |       <option name="SDK_HOME" value="" />
157 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
158 |       <option name="IS_MODULE_SDK" value="true" />
159 |       <option name="ADD_CONTENT_ROOTS" value="true" />
160 |       <option name="ADD_SOURCE_ROOTS" value="true" />
161 |       <option name="SCRIPT_NAME" value="E:\Actor-Critic-pytorch\Actor-Critic.py" />
162 |       <option name="PARAMETERS" value="" />
163 |       <option name="SHOW_COMMAND_LINE" value="false" />
164 |       <option name="EMULATE_TERMINAL" value="false" />
165 |       <option name="MODULE_MODE" value="false" />
166 |     </configuration>
167 |     <configuration name="test2" type="PythonConfigurationType" factoryName="Python" temporary="true">
168 |       <module name="Actor-Critic-pytorch" />
169 |       <option name="INTERPRETER_OPTIONS" value="" />
170 |       <option name="PARENT_ENVS" value="true" />
171 |       <envs>
172 |         <env name="PYTHONUNBUFFERED" value="1" />
173 |       </envs>
174 |       <option name="SDK_HOME" value="" />
175 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
176 |       <option name="IS_MODULE_SDK" value="true" />
177 |       <option name="ADD_CONTENT_ROOTS" value="true" />
178 |       <option name="ADD_SOURCE_ROOTS" value="true" />
179 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/test2.py" />
180 |       <option name="PARAMETERS" value="" />
181 |       <option name="SHOW_COMMAND_LINE" value="false" />
182 |       <option name="EMULATE_TERMINAL" value="false" />
183 |       <option name="MODULE_MODE" value="false" />
184 |     </configuration>
185 |     <list>
186 |       <item itemvalue="Python.Actor-Critic" />
187 |       <item itemvalue="Python.test" />
188 |       <item itemvalue="Python.test2" />
189 |       <item itemvalue="Python.DDPG" />
190 |     </list>
191 |     <recent_temporary>
192 |       <list>
193 |         <item itemvalue="Python.DDPG" />
194 |         <item itemvalue="Python.test" />
195 |         <item itemvalue="Python.test2" />
196 |         <item itemvalue="Python.Actor-Critic" />
197 |       </list>
198 |     </recent_temporary>
199 |   </component>
200 |   <component name="SvnConfiguration">
201 |     <configuration />
202 |   </component>
203 |   <component name="TaskManager">
204 |     <task active="true" id="Default" summary="Default task">
205 |       <changelist id="b1fc59c3-33d8-4195-82ed-c4ce3d0ef42e" name="Default" comment="" />
206 |       <created>1533277484576</created>
207 |       <option name="number" value="Default" />
208 |       <option name="presentableId" value="Default" />
209 |       <updated>1533277484576</updated>
210 |     </task>
211 |     <servers />
212 |   </component>
213 |   <component name="ToolWindowManager">
214 |     <frame x="-7" y="-7" width="1295" height="695" extended-state="6" />
215 |     <editor active="true" />
216 |     <layout>
217 |       <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.11553474" />
218 |       <window_info anchor="bottom" id="TODO" order="6" />
219 |       <window_info anchor="bottom" id="Event Log" side_tool="true" />
220 |       <window_info active="true" anchor="bottom" id="Run" order="2" visible="true" weight="0.33843538" />
221 |       <window_info anchor="bottom" id="Version Control" show_stripe_button="false" />
222 |       <window_info anchor="bottom" id="Python Console" />
223 |       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
224 |       <window_info anchor="bottom" id="Terminal" />
225 |       <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
226 |       <window_info id="Favorites" side_tool="true" />
227 |       <window_info anchor="bottom" id="Find" order="1" />
228 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
229 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
230 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
231 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
232 |       <window_info anchor="bottom" id="Message" order="0" />
233 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
234 |     </layout>
235 |   </component>
236 |   <component name="VcsContentAnnotationSettings">
237 |     <option name="myLimit" value="2678400000" />
238 |   </component>
239 |   <component name="editorHistoryManager">
240 |     <entry file="file://D:/Python3.6.3/Lib/site-packages/torch/nn/functional.py">
241 |       <provider selected="true" editor-type-id="text-editor">
242 |         <state relative-caret-position="102">
243 |           <caret line="861" selection-start-line="861" selection-end-line="861" />
244 |         </state>
245 |       </provider>
246 |     </entry>
247 |     <entry file="file://D:/Python3.6.3/Lib/site-packages/torch/nn/modules/activation.py">
248 |       <provider selected="true" editor-type-id="text-editor">
249 |         <state relative-caret-position="274">
250 |           <caret line="720" selection-start-line="720" selection-end-line="720" />
251 |         </state>
252 |       </provider>
253 |     </entry>
254 |     <entry file="file://$PROJECT_DIR$/test2.py">
255 |       <provider selected="true" editor-type-id="text-editor">
256 |         <state relative-caret-position="145">
257 |           <caret line="76" column="33" selection-start-line="76" selection-start-column="33" selection-end-line="76" selection-end-column="33" />
258 |           <folding>
259 |             <element signature="e#0#15#0" expanded="true" />
260 |           </folding>
261 |         </state>
262 |       </provider>
263 |     </entry>
264 |     <entry file="file://$PROJECT_DIR$/Actor-Critic.py">
265 |       <provider selected="true" editor-type-id="text-editor">
266 |         <state relative-caret-position="216">
267 |           <caret line="155" column="15" lean-forward="true" selection-start-line="155" selection-start-column="4" selection-end-line="155" selection-end-column="15" />
268 |           <folding>
269 |             <element signature="e#0#9#0" expanded="true" />
270 |           </folding>
271 |         </state>
272 |       </provider>
273 |     </entry>
274 |     <entry file="file://$PROJECT_DIR$/Actor-Critic.py">
275 |       <provider selected="true" editor-type-id="text-editor">
276 |         <state relative-caret-position="119">
277 |           <caret line="76" column="71" selection-start-line="76" selection-start-column="51" selection-end-line="76" selection-end-column="71" />
278 |           <folding>
279 |             <element signature="e#0#14#0" expanded="true" />
280 |           </folding>
281 |         </state>
282 |       </provider>
283 |     </entry>
284 |     <entry file="file://$PROJECT_DIR$/DDPG.py">
285 |       <provider selected="true" editor-type-id="text-editor">
286 |         <state relative-caret-position="249">
287 |           <caret line="108" column="17" selection-start-line="108" selection-start-column="17" selection-end-line="108" selection-end-column="17" />
288 |           <folding>
289 |             <element signature="e#0#9#0" expanded="true" />
290 |           </folding>
291 |         </state>
292 |       </provider>
293 |     </entry>
294 |   </component>
295 | </project>


--------------------------------------------------------------------------------