├── xvfb.pid
├── stop-docker
├── Workshop
├── Images
│ ├── AI.jpg
│ ├── Agent.png
│ ├── CartPole-v1.gif
│ ├── Discussion.jpg
│ ├── Taxi_matrix.png
│ └── Taxi_matrix_initial.png
├── freeze_weights
│ ├── CartPole-v0
│ │ ├── local_network.h5
│ │ └── target_network.h5
│ └── CartPole-v1
│ │ ├── local_network.h5
│ │ └── target_network.h5
├── Workshop_Q_table.ipynb
├── Workshop_Q_table_solution.ipynb
└── Workshop_DQN.ipynb
├── .idea
├── libraries
│ └── R_User_Library.xml
├── vcs.xml
├── misc.xml
├── modules.xml
├── Reinforcement-Learning-Workshop.iml
└── workspace.xml
├── start-docker
├── xvfb
├── AMI-user-data
├── README.md
├── .gitignore
├── Workshop_facilitation
└── generate-groups.py
└── LICENSE
/xvfb.pid:
--------------------------------------------------------------------------------
1 | 60
2 |
--------------------------------------------------------------------------------
/stop-docker:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | docker stop $(docker ps -q)
3 |
4 |
--------------------------------------------------------------------------------
/Workshop/Images/AI.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/AI.jpg
--------------------------------------------------------------------------------
/Workshop/Images/Agent.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/Agent.png
--------------------------------------------------------------------------------
/Workshop/Images/CartPole-v1.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/CartPole-v1.gif
--------------------------------------------------------------------------------
/Workshop/Images/Discussion.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/Discussion.jpg
--------------------------------------------------------------------------------
/Workshop/Images/Taxi_matrix.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/Taxi_matrix.png
--------------------------------------------------------------------------------
/Workshop/Images/Taxi_matrix_initial.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/Images/Taxi_matrix_initial.png
--------------------------------------------------------------------------------
/.idea/libraries/R_User_Library.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/Workshop/freeze_weights/CartPole-v0/local_network.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/freeze_weights/CartPole-v0/local_network.h5
--------------------------------------------------------------------------------
/Workshop/freeze_weights/CartPole-v0/target_network.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/freeze_weights/CartPole-v0/target_network.h5
--------------------------------------------------------------------------------
/Workshop/freeze_weights/CartPole-v1/local_network.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/freeze_weights/CartPole-v1/local_network.h5
--------------------------------------------------------------------------------
/Workshop/freeze_weights/CartPole-v1/target_network.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fabiansd/Reinforcement-Learning-Workshop/HEAD/Workshop/freeze_weights/CartPole-v1/target_network.h5
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/start-docker:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | TMP=$(sudo docker run --rm -d -v /home/ubuntu/RL_folder:/notebooks -p 8888:8888 justheuristic/practical_rl /run_jupyter.sh)
3 | IP=$(curl http://checkip.amazonaws.com)
4 | # The script sleeps to ensure the docker image is finished with the startup such that the docker log is generated
5 | sleep 120
6 | docker logs $TMP 2>&1 | grep -P " .*\K:8888\/\?token=[a-zA-Z0-9]*$" | sudo sed "s/^\(.*\)token=\(.*\)$/http:\/\/$IP:8888\/\?token=\2/" > /home/ubuntu/RL_folder/LINK/"$IP".txt
7 | sleep 5
8 | aws s3 sync /home/ubuntu/RL_folder/LINK s3://rl-ws-eirik-fabian/RL-WS-links
9 |
--------------------------------------------------------------------------------
/.idea/Reinforcement-Learning-Workshop.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/xvfb:
--------------------------------------------------------------------------------
1 | #taken from https://gist.github.com/jterrace/2911875
2 | XVFB=/usr/bin/Xvfb
3 | XVFBARGS=":1 -screen 0 1024x768x24 -ac +extension GLX +render -noreset"
4 | PIDFILE=./xvfb.pid
5 | case "$1" in
6 | start)
7 | echo -n "Starting virtual X frame buffer: Xvfb"
8 | start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --background --exec $XVFB -- $XVFBARGS
9 | echo "."
10 | ;;
11 | stop)
12 | echo -n "Stopping virtual X frame buffer: Xvfb"
13 | start-stop-daemon --stop --quiet --pidfile $PIDFILE
14 | echo "."
15 | ;;
16 | restart)
17 | $0 stop
18 | $0 start
19 | ;;
20 | *)
21 | echo "Usage: /etc/init.d/xvfb {start|stop|restart}"
22 | exit 1
23 | esac
24 |
25 | exit 0
26 |
--------------------------------------------------------------------------------
/AMI-user-data:
--------------------------------------------------------------------------------
1 | Content-Type: multipart/mixed; boundary="//"
2 | MIME-Version: 1.0
3 |
4 | --//
5 | Content-Type: text/cloud-config; charset="us-ascii"
6 | MIME-Version: 1.0
7 | Content-Transfer-Encoding: 7bit
8 | Content-Disposition: attachment; filename="cloud-config.txt"
9 |
10 | #cloud-config
11 | cloud_final_modules:
12 | - [scripts-user, always]
13 |
14 | --//
15 | Content-Type: text/x-shellscript; charset="us-ascii"
16 | MIME-Version: 1.0
17 | Content-Transfer-Encoding: 7bit
18 | Content-Disposition: attachment; filename="userdata.txt"
19 |
20 | #!/bin/bash
21 | cd /home/ubuntu/RL_folder/
22 | git clone https://github.com/fabiansd/Reinforcement-Learning-Workshop.git
23 | cd /home/ubuntu/RL_folder/Reinforcement-Learning-Workshop
24 | sudo bash /home/ubuntu/RL_folder/Reinforcement-Learning-Workshop/start-docker
25 | --//
26 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [![CC BY-NC-SA 4.0][cc-by-nc-sa-shield]][cc-by-nc-sa]
2 |
3 | [cc-by-nc-sa]: https://creativecommons.org/licenses/by-nc-sa/4.0/
4 | [cc-by-nc-sa-shield]: https://mirrors.creativecommons.org/presskit/buttons/80x15/svg/by-nc-sa.svg
5 |
6 | # Reinforcement-Learning-Workshop
7 | Necessary files and notebooks to host a RL workshop on AWS
8 |
9 | ## AWS
10 |
11 | ### Setup
12 |
13 | An AWS Image is used to create a docker container with the reinforcement learning environment, and automatically launch a Jupyter Notebook on port 8888. To access this notebook, a link is generated with the corresponding IP, port and token.
14 |
15 | ### Steps to launch instances for the WS
16 |
17 | 1. Select number of instances to start
18 | 2. Paste in the user data
19 | 3. Remember to attach the correct network security group (which opens up port 8888) and IAM role (to give write access to s3)
20 |
21 | ### Visualization
22 | Visualizations in the DQN-notebooks are only supported for Linux and OSX, in addition to headless Linux servers, e.g. an AWS EC2 Linux instance.
23 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
--------------------------------------------------------------------------------
/Workshop_facilitation/generate-groups.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import boto3
3 | from botocore.exceptions import ClientError
4 | from pathlib import Path
5 | import os
6 | import json
7 | import argparse
8 | import shutil
9 | import pandas as pd
10 |
11 | TMP_FOLDER = 'LINK_TEMP'
12 | IP_DICT_JSON_NAME = 'active_ips.json'
13 | GROUP_DICT_NAME = 'groups.json'
14 | GROUP_EXCEL_NAME = 'Groups.xlsx'
15 | AMI_ID = 'ami-060e29600c7769bc0'
16 |
17 |
18 | def bucket_exists(bucket_name):
19 | """Determine whether bucket_name exists and the user has permission to access it
20 |
21 | :param bucket_name: string
22 | :return: True if the referenced bucket_name exists, otherwise False
23 | """
24 |
25 | s3 = boto3.client('s3')
26 | try:
27 | response = s3.head_bucket(Bucket=bucket_name)
28 | except ClientError as e:
29 | logging.debug(e)
30 | return False
31 | return True
32 |
33 |
34 | def write_json(d, dest):
35 | with open(dest, 'w') as cred:
36 | json.dump(d, cred)
37 |
38 |
39 | def load_json(dest):
40 | with open(dest, 'r') as cred:
41 | return json.load(cred)
42 |
43 |
44 | def list_ec2():
45 | ec2 = boto3.client('ec2')
46 | response = ec2.describe_instances()
47 |
48 | running_ws_ips = {}
49 |
50 | for reservation in response["Reservations"]:
51 | for instance in reservation["Instances"]:
52 |
53 | if instance['ImageId'] == AMI_ID \
54 | and instance['State']['Name'] == 'running':
55 |
56 | running_ws_ips[instance['InstanceId']] = \
57 | instance['NetworkInterfaces'][0]['Association']['PublicIp']
58 |
59 | print(f'\nFound the running ec2 instances:')
60 | print(f'{running_ws_ips}\n')
61 |
62 | return running_ws_ips
63 |
64 |
65 | def download_files(bucket_name, LINK_FOLDER):
66 |
67 | ip_dict = list_ec2()
68 |
69 | if Path(TMP_FOLDER).exists():
70 | shutil.rmtree(TMP_FOLDER)
71 | os.mkdir(TMP_FOLDER)
72 |
73 |
74 | s3 = boto3.resource('s3')
75 | rl_bucket = s3.Bucket(bucket_name)
76 |
77 | # Downloads only files with an ip as name in the bucket
78 | for element in rl_bucket.objects.all():
79 |
80 | element_list = element.key.split('/')
81 |
82 | if element_list[0] == LINK_FOLDER and os.path.splitext(
83 | element_list[-1])[-1] == '.txt':
84 | p_ip, _ = os.path.splitext(element_list[-1])
85 |
86 | if p_ip in ip_dict.values():
87 |
88 | s3.meta.client.download_file(bucket_name,
89 | element.key,
90 | str(Path(TMP_FOLDER).joinpath(
91 | element_list[-1])))
92 |
93 | # Save the ip dict to track last active ec2 instances
94 | write_json(ip_dict, IP_DICT_JSON_NAME)
95 |
96 |
97 | def allocate_new_groups():
98 |
99 | ip_active_dict = load_json(IP_DICT_JSON_NAME)
100 |
101 | if Path(GROUP_DICT_NAME).exists():
102 | os.remove(Path(GROUP_DICT_NAME))
103 |
104 | group_dict = {}
105 |
106 | group_number = 1
107 | while True:
108 |
109 | popped_active_ip = ip_active_dict.pop(
110 | next(iter(ip_active_dict)))
111 | link_temp = ''
112 | try:
113 | with open(Path(TMP_FOLDER).joinpath(
114 | popped_active_ip + '.txt'), 'r') as link_file:
115 | link_temp = link_file.read().replace('\n', '')
116 | except IOError:
117 | link_temp = 'NO LINK FOUND'
118 |
119 | group_dict[group_number] = {
120 | 'ip': popped_active_ip,
121 | 'link': link_temp
122 | }
123 |
124 | if len(ip_active_dict) == 0:
125 | break
126 |
127 | group_number += 1
128 |
129 | print('\nNew groups allocated to the running instances')
130 | print(group_dict)
131 | print('\n')
132 |
133 | write_json(group_dict, GROUP_DICT_NAME)
134 |
135 |
136 | def update_and_allocate_instances(ip_active_dict):
137 |
138 | # ip_active_dict = load_json(IP_DICT_JSON_NAME)
139 |
140 | try:
141 | group_dict = load_json(Path(GROUP_DICT_NAME))
142 | except IOError:
143 | allocate_new_groups()
144 | return
145 |
146 | # If any groups no longer has a running EC2 among the active list,
147 | # remove the ip and link
148 | inactive_list = []
149 | for group_n, info_d in group_dict.items():
150 | if str(info_d['ip']) not in ip_active_dict.values():
151 | inactive_list.append(group_n)
152 |
153 | for n in inactive_list:
154 | group_dict[n] = {'ip': None, 'link': None}
155 |
156 | # This is untested
157 |
158 | # Here we round up the used ips og the active ips to find out what
159 | # active instances are vacant
160 | non_vacant_instances = []
161 | for group_n, info_d in group_dict.items():
162 | if not group_dict[group_n] == {'ip': None, 'link': None}:
163 | non_vacant_instances.append(group_dict[group_n]['ip'])
164 |
165 |
166 | def test_connection(BUCKET_NAME):
167 |
168 | # Set up logging
169 | logging.basicConfig(level=logging.DEBUG,
170 | format='%(levelname)s: %(asctime)s: %(message)s')
171 |
172 | # Check if the bucket exists
173 | if bucket_exists(BUCKET_NAME):
174 | logging.info(f'{BUCKET_NAME} exists and you have permission to access it.')
175 | else:
176 | logging.info(f'{BUCKET_NAME} does not exist or '
177 | f'you do not have permission to access it.')
178 |
179 |
180 | def generate_excel():
181 |
182 | group_dict = load_json(GROUP_DICT_NAME)
183 |
184 | group_df = pd.DataFrame.from_dict(group_dict, orient='index')
185 |
186 | group_df.to_excel('Groups.xlsx')
187 |
188 | print('Excel generated and saved')
189 |
190 |
191 | def clean_up():
192 | if Path(TMP_FOLDER).exists():
193 | shutil.rmtree(TMP_FOLDER)
194 | if Path(IP_DICT_JSON_NAME).exists():
195 | os.remove(IP_DICT_JSON_NAME)
196 | if Path(GROUP_DICT_NAME).exists():
197 | os.remove(GROUP_DICT_NAME)
198 | if Path(GROUP_EXCEL_NAME).exists():
199 | os.remove(GROUP_EXCEL_NAME)
200 |
201 |
202 | if __name__ == '__main__':
203 |
204 | # Assign this value before running the program
205 | BUCKET_NAME = 'rl-workshop-bucket'
206 | LINK_FOLDER = 'RL-WS-links'
207 |
208 | # input arguments
209 | parser = argparse.ArgumentParser(description= 'Allocate running EC2 instances of RL-WS-image into groups')
210 | parser.add_argument('-i', '--init', type=bool,
211 | help='Set this to yes or no wheter you are creating the groups for the first time or updating')
212 | parser.add_argument('-c', '--clean', type=bool,
213 | help='Enable this if you want to clean the entire group setting')
214 | parser.add_argument('-t', '--test', type=bool,
215 | help='Test the connection to the s3 bucket')
216 | args = vars(parser.parse_args())
217 |
218 | if args['clean']:
219 | clean_up()
220 | print('CLEANED UP')
221 |
222 | if args['test']:
223 | test_connection(BUCKET_NAME)
224 |
225 | if args['init']:
226 | download_files(BUCKET_NAME, LINK_FOLDER)
227 | allocate_new_groups()
228 | generate_excel()
229 |
230 | # ip_org_dict = {'i-0345dc555876e6985': '54.174.112.245', 'i-0116da57317bfa93c': '3.87.192.207', 'i-0a2eb4c7cc444b410': '52.23.168.91'}
231 | # ip_new_dict = {'i-0345dc555876e6985': '54.174.112.245', 'i-0a2eb4c7cc444b410': '52.23.168.91'}
232 | # ip_replaced_dict = {'i-0116da57317bfa93c': '3.87.192.207', 'i-0116daas317bfa93c': '9.87.192.207', 'i-0a2eb4c7cc444b410': '52.23.168.91', 'i-0116da57317bfa93c': '3.87.192.207'}
233 | # elif args['update']:
234 | # download_files(BUCKET_NAME, LINK_FOLDER)
235 | # update_and_allocate_instances()
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
72 |
73 |
74 |
75 | save_ip_dict
76 | list_ec2
77 | shutil
78 |
79 |
80 |
81 |
82 |
83 |
84 |
92 |
93 |
94 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 | 1566239158481
225 |
226 |
227 | 1566239158481
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
2 | Public License
3 |
4 | By exercising the Licensed Rights (defined below), You accept and agree
5 | to be bound by the terms and conditions of this Creative Commons
6 | Attribution-NonCommercial-ShareAlike 4.0 International Public License
7 | ("Public License"). To the extent this Public License may be
8 | interpreted as a contract, You are granted the Licensed Rights in
9 | consideration of Your acceptance of these terms and conditions, and the
10 | Licensor grants You such rights in consideration of benefits the
11 | Licensor receives from making the Licensed Material available under
12 | these terms and conditions.
13 |
14 |
15 | Section 1 -- Definitions.
16 |
17 | a. Adapted Material means material subject to Copyright and Similar
18 | Rights that is derived from or based upon the Licensed Material
19 | and in which the Licensed Material is translated, altered,
20 | arranged, transformed, or otherwise modified in a manner requiring
21 | permission under the Copyright and Similar Rights held by the
22 | Licensor. For purposes of this Public License, where the Licensed
23 | Material is a musical work, performance, or sound recording,
24 | Adapted Material is always produced where the Licensed Material is
25 | synched in timed relation with a moving image.
26 |
27 | b. Adapter's License means the license You apply to Your Copyright
28 | and Similar Rights in Your contributions to Adapted Material in
29 | accordance with the terms and conditions of this Public License.
30 |
31 | c. BY-NC-SA Compatible License means a license listed at
32 | creativecommons.org/compatiblelicenses, approved by Creative
33 | Commons as essentially the equivalent of this Public License.
34 |
35 | d. Copyright and Similar Rights means copyright and/or similar rights
36 | closely related to copyright including, without limitation,
37 | performance, broadcast, sound recording, and Sui Generis Database
38 | Rights, without regard to how the rights are labeled or
39 | categorized. For purposes of this Public License, the rights
40 | specified in Section 2(b)(1)-(2) are not Copyright and Similar
41 | Rights.
42 |
43 | e. Effective Technological Measures means those measures that, in the
44 | absence of proper authority, may not be circumvented under laws
45 | fulfilling obligations under Article 11 of the WIPO Copyright
46 | Treaty adopted on December 20, 1996, and/or similar international
47 | agreements.
48 |
49 | f. Exceptions and Limitations means fair use, fair dealing, and/or
50 | any other exception or limitation to Copyright and Similar Rights
51 | that applies to Your use of the Licensed Material.
52 |
53 | g. License Elements means the license attributes listed in the name
54 | of a Creative Commons Public License. The License Elements of this
55 | Public License are Attribution, NonCommercial, and ShareAlike.
56 |
57 | h. Licensed Material means the artistic or literary work, database,
58 | or other material to which the Licensor applied this Public
59 | License.
60 |
61 | i. Licensed Rights means the rights granted to You subject to the
62 | terms and conditions of this Public License, which are limited to
63 | all Copyright and Similar Rights that apply to Your use of the
64 | Licensed Material and that the Licensor has authority to license.
65 |
66 | j. Licensor means the individual(s) or entity(ies) granting rights
67 | under this Public License.
68 |
69 | k. NonCommercial means not primarily intended for or directed towards
70 | commercial advantage or monetary compensation. For purposes of
71 | this Public License, the exchange of the Licensed Material for
72 | other material subject to Copyright and Similar Rights by digital
73 | file-sharing or similar means is NonCommercial provided there is
74 | no payment of monetary compensation in connection with the
75 | exchange.
76 |
77 | l. Share means to provide material to the public by any means or
78 | process that requires permission under the Licensed Rights, such
79 | as reproduction, public display, public performance, distribution,
80 | dissemination, communication, or importation, and to make material
81 | available to the public including in ways that members of the
82 | public may access the material from a place and at a time
83 | individually chosen by them.
84 |
85 | m. Sui Generis Database Rights means rights other than copyright
86 | resulting from Directive 96/9/EC of the European Parliament and of
87 | the Council of 11 March 1996 on the legal protection of databases,
88 | as amended and/or succeeded, as well as other essentially
89 | equivalent rights anywhere in the world.
90 |
91 | n. You means the individual or entity exercising the Licensed Rights
92 | under this Public License. Your has a corresponding meaning.
93 |
94 |
95 | Section 2 -- Scope.
96 |
97 | a. License grant.
98 |
99 | 1. Subject to the terms and conditions of this Public License,
100 | the Licensor hereby grants You a worldwide, royalty-free,
101 | non-sublicensable, non-exclusive, irrevocable license to
102 | exercise the Licensed Rights in the Licensed Material to:
103 |
104 | a. reproduce and Share the Licensed Material, in whole or
105 | in part, for NonCommercial purposes only; and
106 |
107 | b. produce, reproduce, and Share Adapted Material for
108 | NonCommercial purposes only.
109 |
110 | 2. Exceptions and Limitations. For the avoidance of doubt, where
111 | Exceptions and Limitations apply to Your use, this Public
112 | License does not apply, and You do not need to comply with
113 | its terms and conditions.
114 |
115 | 3. Term. The term of this Public License is specified in Section
116 | 6(a).
117 |
118 | 4. Media and formats; technical modifications allowed. The
119 | Licensor authorizes You to exercise the Licensed Rights in
120 | all media and formats whether now known or hereafter created,
121 | and to make technical modifications necessary to do so. The
122 | Licensor waives and/or agrees not to assert any right or
123 | authority to forbid You from making technical modifications
124 | necessary to exercise the Licensed Rights, including
125 | technical modifications necessary to circumvent Effective
126 | Technological Measures. For purposes of this Public License,
127 | simply making modifications authorized by this Section 2(a)
128 | (4) never produces Adapted Material.
129 |
130 | 5. Downstream recipients.
131 |
132 | a. Offer from the Licensor -- Licensed Material. Every
133 | recipient of the Licensed Material automatically
134 | receives an offer from the Licensor to exercise the
135 | Licensed Rights under the terms and conditions of this
136 | Public License.
137 |
138 | b. Additional offer from the Licensor -- Adapted Material.
139 | Every recipient of Adapted Material from You
140 | automatically receives an offer from the Licensor to
141 | exercise the Licensed Rights in the Adapted Material
142 | under the conditions of the Adapter's License You apply.
143 |
144 | c. No downstream restrictions. You may not offer or impose
145 | any additional or different terms or conditions on, or
146 | apply any Effective Technological Measures to, the
147 | Licensed Material if doing so restricts exercise of the
148 | Licensed Rights by any recipient of the Licensed
149 | Material.
150 |
151 | 6. No endorsement. Nothing in this Public License constitutes or
152 | may be construed as permission to assert or imply that You
153 | are, or that Your use of the Licensed Material is, connected
154 | with, or sponsored, endorsed, or granted official status by,
155 | the Licensor or others designated to receive attribution as
156 | provided in Section 3(a)(1)(A)(i).
157 |
158 | b. Other rights.
159 |
160 | 1. Moral rights, such as the right of integrity, are not
161 | licensed under this Public License, nor are publicity,
162 | privacy, and/or other similar personality rights; however, to
163 | the extent possible, the Licensor waives and/or agrees not to
164 | assert any such rights held by the Licensor to the limited
165 | extent necessary to allow You to exercise the Licensed
166 | Rights, but not otherwise.
167 |
168 | 2. Patent and trademark rights are not licensed under this
169 | Public License.
170 |
171 | 3. To the extent possible, the Licensor waives any right to
172 | collect royalties from You for the exercise of the Licensed
173 | Rights, whether directly or through a collecting society
174 | under any voluntary or waivable statutory or compulsory
175 | licensing scheme. In all other cases the Licensor expressly
176 | reserves any right to collect such royalties, including when
177 | the Licensed Material is used other than for NonCommercial
178 | purposes.
179 |
180 |
181 | Section 3 -- License Conditions.
182 |
183 | Your exercise of the Licensed Rights is expressly made subject to the
184 | following conditions.
185 |
186 | a. Attribution.
187 |
188 | 1. If You Share the Licensed Material (including in modified
189 | form), You must:
190 |
191 | a. retain the following if it is supplied by the Licensor
192 | with the Licensed Material:
193 |
194 | i. identification of the creator(s) of the Licensed
195 | Material and any others designated to receive
196 | attribution, in any reasonable manner requested by
197 | the Licensor (including by pseudonym if
198 | designated);
199 |
200 | ii. a copyright notice;
201 |
202 | iii. a notice that refers to this Public License;
203 |
204 | iv. a notice that refers to the disclaimer of
205 | warranties;
206 |
207 | v. a URI or hyperlink to the Licensed Material to the
208 | extent reasonably practicable;
209 |
210 | b. indicate if You modified the Licensed Material and
211 | retain an indication of any previous modifications; and
212 |
213 | c. indicate the Licensed Material is licensed under this
214 | Public License, and include the text of, or the URI or
215 | hyperlink to, this Public License.
216 |
217 | 2. You may satisfy the conditions in Section 3(a)(1) in any
218 | reasonable manner based on the medium, means, and context in
219 | which You Share the Licensed Material. For example, it may be
220 | reasonable to satisfy the conditions by providing a URI or
221 | hyperlink to a resource that includes the required
222 | information.
223 | 3. If requested by the Licensor, You must remove any of the
224 | information required by Section 3(a)(1)(A) to the extent
225 | reasonably practicable.
226 |
227 | b. ShareAlike.
228 |
229 | In addition to the conditions in Section 3(a), if You Share
230 | Adapted Material You produce, the following conditions also apply.
231 |
232 | 1. The Adapter's License You apply must be a Creative Commons
233 | license with the same License Elements, this version or
234 | later, or a BY-NC-SA Compatible License.
235 |
236 | 2. You must include the text of, or the URI or hyperlink to, the
237 | Adapter's License You apply. You may satisfy this condition
238 | in any reasonable manner based on the medium, means, and
239 | context in which You Share Adapted Material.
240 |
241 | 3. You may not offer or impose any additional or different terms
242 | or conditions on, or apply any Effective Technological
243 | Measures to, Adapted Material that restrict exercise of the
244 | rights granted under the Adapter's License You apply.
245 |
246 |
247 | Section 4 -- Sui Generis Database Rights.
248 |
249 | Where the Licensed Rights include Sui Generis Database Rights that
250 | apply to Your use of the Licensed Material:
251 |
252 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right
253 | to extract, reuse, reproduce, and Share all or a substantial
254 | portion of the contents of the database for NonCommercial purposes
255 | only;
256 |
257 | b. if You include all or a substantial portion of the database
258 | contents in a database in which You have Sui Generis Database
259 | Rights, then the database in which You have Sui Generis Database
260 | Rights (but not its individual contents) is Adapted Material,
261 | including for purposes of Section 3(b); and
262 |
263 | c. You must comply with the conditions in Section 3(a) if You Share
264 | all or a substantial portion of the contents of the database.
265 |
266 | For the avoidance of doubt, this Section 4 supplements and does not
267 | replace Your obligations under this Public License where the Licensed
268 | Rights include other Copyright and Similar Rights.
269 |
270 |
271 | Section 5 -- Disclaimer of Warranties and Limitation of Liability.
272 |
273 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
274 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
275 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
276 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
277 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
278 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
279 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
280 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
281 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
282 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
283 |
284 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
285 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
286 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
287 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
288 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
289 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
290 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
291 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
292 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
293 |
294 | c. The disclaimer of warranties and limitation of liability provided
295 | above shall be interpreted in a manner that, to the extent
296 | possible, most closely approximates an absolute disclaimer and
297 | waiver of all liability.
298 |
299 |
300 | Section 6 -- Term and Termination.
301 |
302 | a. This Public License applies for the term of the Copyright and
303 | Similar Rights licensed here. However, if You fail to comply with
304 | this Public License, then Your rights under this Public License
305 | terminate automatically.
306 |
307 | b. Where Your right to use the Licensed Material has terminated under
308 | Section 6(a), it reinstates:
309 |
310 | 1. automatically as of the date the violation is cured, provided
311 | it is cured within 30 days of Your discovery of the
312 | violation; or
313 |
314 | 2. upon express reinstatement by the Licensor.
315 |
316 | For the avoidance of doubt, this Section 6(b) does not affect any
317 | right the Licensor may have to seek remedies for Your violations
318 | of this Public License.
319 |
320 | c. For the avoidance of doubt, the Licensor may also offer the
321 | Licensed Material under separate terms or conditions or stop
322 | distributing the Licensed Material at any time; however, doing so
323 | will not terminate this Public License.
324 |
325 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
326 | License.
327 |
328 |
329 | Section 7 -- Other Terms and Conditions.
330 |
331 | a. The Licensor shall not be bound by any additional or different
332 | terms or conditions communicated by You unless expressly agreed.
333 |
334 | b. Any arrangements, understandings, or agreements regarding the
335 | Licensed Material not stated herein are separate from and
336 | independent of the terms and conditions of this Public License.
337 |
338 |
339 | Section 8 -- Interpretation.
340 |
341 | a. For the avoidance of doubt, this Public License does not, and
342 | shall not be interpreted to, reduce, limit, restrict, or impose
343 | conditions on any use of the Licensed Material that could lawfully
344 | be made without permission under this Public License.
345 |
346 | b. To the extent possible, if any provision of this Public License is
347 | deemed unenforceable, it shall be automatically reformed to the
348 | minimum extent necessary to make it enforceable. If the provision
349 | cannot be reformed, it shall be severed from this Public License
350 | without affecting the enforceability of the remaining terms and
351 | conditions.
352 |
353 | c. No term or condition of this Public License will be waived and no
354 | failure to comply consented to unless expressly agreed to by the
355 | Licensor.
356 |
357 | d. Nothing in this Public License constitutes or may be interpreted
358 | as a limitation upon, or waiver of, any privileges and immunities
359 | that apply to the Licensor or You, including from the legal
360 | processes of any jurisdiction or authority.
361 |
362 | =======================================================================
363 |
364 | Creative Commons is not a party to its public
365 | licenses. Notwithstanding, Creative Commons may elect to apply one of
366 | its public licenses to material it publishes and in those instances
367 | will be considered the “Licensor.” The text of the Creative Commons
368 | public licenses is dedicated to the public domain under the CC0 Public
369 | Domain Dedication. Except for the limited purpose of indicating that
370 | material is shared under a Creative Commons public license or as
371 | otherwise permitted by the Creative Commons policies published at
372 | creativecommons.org/policies, Creative Commons does not authorize the
373 | use of the trademark "Creative Commons" or any other trademark or logo
374 | of Creative Commons without its prior written consent including,
375 | without limitation, in connection with any unauthorized modifications
376 | to any of its public licenses or any other arrangements,
377 | understandings, or agreements concerning use of licensed material. For
378 | the avoidance of doubt, this paragraph does not form part of the
379 | public licenses.
380 |
381 | Creative Commons may be contacted at creativecommons.org.
382 |
--------------------------------------------------------------------------------
/Workshop/Workshop_Q_table.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Q-learning using tables\n",
8 | "##### Authors: Eirik Fagtun Kjærnli and Fabian Dietrichson [Accenture]"
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "## Welcome \n",
16 | "Welcome to this workshop!\n",
17 | "\n",
18 | "The workshop is structured such that for each cell, you will write your own code and the code will then be asserted in the next cell. The assertion cell is marked \"# Do not edit - Assertion cell #\". \n",
19 | "The code should be written between \"Write code below\" and \"Write code above\". If your code does not pass the assertion, you will have to rewrite it before continuing.\n",
20 | "\n",
21 | "### Task:\n",
22 | "Implement the method multiply_input_by_2\n",
23 | "- The code should take the input variabel \"input_variabel\", and multiply it by 2\n",
24 | "- The answer should be stored in the variabel \"result\"\n",
25 | "- When the method is implemented, run the cell.\n",
26 | "\n",
27 | "Hotkey to run a cell: CTRL + ENTER"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": null,
33 | "metadata": {},
34 | "outputs": [],
35 | "source": [
36 | "def multiply_input_by_2(input_variabel):\n",
37 | "\n",
38 | " \"Write code below\" \n",
39 | " \n",
40 | " \"Write code above\" \n",
41 | " \n",
42 | " return result"
43 | ]
44 | },
45 | {
46 | "cell_type": "code",
47 | "execution_count": null,
48 | "metadata": {},
49 | "outputs": [],
50 | "source": [
51 | "# Do not edit - Assertion cell #\n",
52 | "assert(multiply_input_by_2(10) == 20), \"Your method did not multiple the input by 2\"\n",
53 | "print(\"Great, you correctly implemented the method!\")"
54 | ]
55 | },
56 | {
57 | "cell_type": "markdown",
58 | "metadata": {},
59 | "source": [
60 | "## Import necessary packages\n",
61 | "To implement the native Q-learning algorithm and use it in an environment, we just need two additional packages.\n",
62 | "\n",
63 | "_Numpy_\n",
64 | "- Numpy adds support for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays. We could do this workshop using native Python arrays and built-in methods, however using Numpy simplifies the process. \n",
65 | "- To call the methods in the package, we use the \"as np\" command. Such that, to create a numpy table with 2 rows and 2 columns where all the cells are zero, you can simply write np.seros(2,2).\n",
66 | "\n",
67 | "_OpenAI Gym_\n",
68 | "- The Gym library is a collection of demo problems an associated environment, that you can use to work out your reinforcement learning algorithms. These range from simple text problems, to complex physical problems, to Atari video games. OpenAI Gym is therefore by many the preferred framework to learn and test Reinforcement learning algorithms.\"\n",
69 | "\n",
70 | "\n",
71 | "### Task:\n",
72 | "To import the necessary packages, simply run the cell below, and then run the assertion cell to verify that they have been imported correctly."
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": null,
78 | "metadata": {},
79 | "outputs": [],
80 | "source": [
81 | "import numpy as np\n",
82 | "import gym"
83 | ]
84 | },
85 | {
86 | "cell_type": "code",
87 | "execution_count": null,
88 | "metadata": {},
89 | "outputs": [],
90 | "source": [
91 | "# Do not edit - Assertion cell #\n",
92 | "try:\n",
93 | " assert(gym)\n",
94 | " print(\"Great, the packages were imported correctly!\")\n",
95 | "except:\n",
96 | " print(\"You did not run the cell above, do this before you continue!\")"
97 | ]
98 | },
99 | {
100 | "cell_type": "markdown",
101 | "metadata": {},
102 | "source": [
103 | "## Problem Introduction\n",
104 | "\n",
105 | "The environment we are going to use is a simple grid world, where the agent is controlling a taxi. The environment was introduced in [Dietterich2000] to illustrate some issues in hierarchical reinforcement learning. There are 4 locations (labeled by different letters) and your job is to pick up the passenger at one location and drop the passenger off in another. You can read more about the environment [here](https://gym.openai.com/envs/Taxi-v2/).\n",
106 | "\n",
107 | "### Task:\n",
108 | "Create the environment variable containing all necessary methods to run the Taxi-v3 game. \n",
109 | "\n",
110 | "_Tip: Just run the cell below_"
111 | ]
112 | },
113 | {
114 | "cell_type": "code",
115 | "execution_count": null,
116 | "metadata": {},
117 | "outputs": [],
118 | "source": [
119 | "environment_id = \"Taxi-v3\"\n",
120 | "env = gym.make(environment_id)"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": null,
126 | "metadata": {},
127 | "outputs": [],
128 | "source": [
129 | "# Do not edit - Assertion cell #\n",
130 | "try:\n",
131 | " assert(env)\n",
132 | " print(\"You successfully created the environement {}\".format(env.spec.id))\n",
133 | "except:\n",
134 | " print(\"You create the environment incorrectly!\")"
135 | ]
136 | },
137 | {
138 | "cell_type": "markdown",
139 | "metadata": {},
140 | "source": [
141 | "## Build support methods for workshop\n",
142 | "Before you can go on, the cell below must be run. These are methods used to verify your work, in addition to the support function that will be used throughout the workshop. \n",
143 | "\n",
144 | "### Task\n",
145 | "Do as you have done before, simply mark the cell below and press CTRL + Enter!"
146 | ]
147 | },
148 | {
149 | "cell_type": "code",
150 | "execution_count": null,
151 | "metadata": {
152 | "scrolled": true
153 | },
154 | "outputs": [],
155 | "source": [
156 | "from IPython.display import clear_output\n",
157 | "import gym\n",
158 | "import os\n",
159 | "import time\n",
160 | "from copy import deepcopy\n",
161 | "import seaborn as sb\n",
162 | "import matplotlib.pyplot as plt\n",
163 | "\n",
164 | "class TestEnvironment():\n",
165 | " \n",
166 | " def __init__(self):\n",
167 | " self.env = gym.make(environment_id)\n",
168 | " self.env.reset()\n",
169 | " \n",
170 | " def play(self, action):\n",
171 | " if not (0 <= int(action) <= 5):\n",
172 | " print(\"Action value must either be 0,1,2,3,4 or 5\")\n",
173 | " return\n",
174 | "\n",
175 | " clear_output()\n",
176 | " _, reward, done, _ = self.env.step(action)\n",
177 | " self.env.render()\n",
178 | " print(\"Reward: \", reward)\n",
179 | " \n",
180 | " if(done):\n",
181 | " print(\"Game completed, resetting environment!\")\n",
182 | " self.reset_env()\n",
183 | " \n",
184 | " def reset_env(self):\n",
185 | " self.env.reset()\n",
186 | " print(\"Environment has been reset\")\n",
187 | " time.sleep(3)\n",
188 | " clear_output()\n",
189 | "\n",
190 | "class MockData():\n",
191 | " def __init__(self):\n",
192 | " self.env = gym.make(environment_id)\n",
193 | " self.env.seed(10)\n",
194 | " self.should_assert = True\n",
195 | " \n",
196 | " def get_Q(self):\n",
197 | " Q = np.zeros([self.env.observation_space.n, self.env.action_space.n])\n",
198 | " Q[0:2,0] = 10\n",
199 | " return Q\n",
200 | " \n",
201 | " def get_env(self):\n",
202 | " return self.env\n",
203 | " \n",
204 | " def turn_off_assertion(self):\n",
205 | " self.should_assert = False\n",
206 | " \n",
207 | " def do_assertion(self):\n",
208 | " return self.should_assert\n",
209 | "\n",
210 | "def visualize_q_table():\n",
211 | " fig=plt.figure(figsize=(10, 10))\n",
212 | " heat_map = sb.heatmap(Q_trained)\n",
213 | " plt.show()\n",
214 | "\n",
215 | "def plot_visualization(data):\n",
216 | " \n",
217 | " reward_list = data[0]\n",
218 | " iteration_list = data[1]\n",
219 | " epsilon_list = data[2]\n",
220 | " \n",
221 | " episodes = range(len(reward_list))\n",
222 | " \n",
223 | " plt.figure(figsize=(10,10))\n",
224 | " ax = plt.subplot(311)\n",
225 | " ax.set_title(\"Reward\")\n",
226 | " ax = plt.plot(episodes, reward_list)\n",
227 | "\n",
228 | " ax = plt.subplot(312)\n",
229 | " ax.set_title(\"Iterations per episode\")\n",
230 | " ax.plot(episodes, iteration_list)\n",
231 | " \n",
232 | " ax = plt.subplot(313)\n",
233 | " ax.set_title(\"Epsilon\")\n",
234 | " ax.set_xlabel(\"Episodes\")\n",
235 | " ax.plot(episodes, epsilon_list)\n",
236 | " \n",
237 | " plt.show()\n",
238 | "\n",
239 | "def reset_env_and_update_params(env):\n",
240 | " done = False\n",
241 | " state = env.reset()\n",
242 | " iterations = 0\n",
243 | " total_reward = 0\n",
244 | " \n",
245 | " return state, done, iterations, total_reward\n",
246 | "\n",
247 | "def render_performance(env, Q):\n",
248 | " iterations = 0\n",
249 | " total_reward = 0\n",
250 | " t_sleep = 1.2\n",
251 | " \n",
252 | " state = env.reset()\n",
253 | " env.render()\n",
254 | " time.sleep(t_sleep)\n",
255 | " clear_output()\n",
256 | " done = False\n",
257 | "\n",
258 | " while not done:\n",
259 | " action = np.argmax(Q[state,:]) \n",
260 | "\n",
261 | " new_state, reward, done, _ = env.step(action)\n",
262 | " state = new_state\n",
263 | "\n",
264 | " env.render()\n",
265 | " time.sleep(t_sleep)\n",
266 | " clear_output()\n",
267 | " \n",
268 | " iterations += 1\n",
269 | " total_reward += reward\n",
270 | " \n",
271 | " if (iterations >= 20): \n",
272 | " done = True\n",
273 | " print(\"Agent did not complete the episode within 20 iterations, train your agent better!\")\n",
274 | " return\n",
275 | " \n",
276 | " print(\"Your agent completed the task using {} iterations, \\\n",
277 | " and got a total reward of {}\".format(iterations, total_reward))\n",
278 | " \n",
279 | "test_env = TestEnvironment()\n",
280 | "mock = MockData()"
281 | ]
282 | },
283 | {
284 | "cell_type": "code",
285 | "execution_count": null,
286 | "metadata": {},
287 | "outputs": [],
288 | "source": [
289 | "# Do not edit - Assertion cell #\n",
290 | "try:\n",
291 | " assert(mock)\n",
292 | " print(\"Great, the support methods were created!\")\n",
293 | "except:\n",
294 | " print(\"You did not run the cell above, do this before you continue!\")"
295 | ]
296 | },
297 | {
298 | "cell_type": "markdown",
299 | "metadata": {},
300 | "source": [
301 | "## Experiment with the environment\n",
302 | "Before we start creating the Q-learning agent, we will explore the environment we are going to use. This is done using the support methods we have created for you.\n",
303 | "\n",
304 | "**How to play**\n",
305 | "1. Simply input an action, e.g. the integer 1, where it is indicated in the cell below, and click CTRL + Enter. \n",
306 | "2. The game will reset when you have picked up and then dropped off the passenger at the indicated location.\n",
307 | "3. If you would like to reset the test environment during the execution, run the cell which indicates this below. \n",
308 | "\n",
309 | "*Tip*: The bold blue colored letter shows where the passenger should be picked up, and the pink letter indicates the drop-off spot.\n",
310 | "\n",
311 | "**Possible actions**\n",
312 | "0. Move down\n",
313 | "1. Move up\n",
314 | "2. Move right\n",
315 | "3. Move left\n",
316 | "4. Pick up passenger\n",
317 | "5. Drop off passenger\n",
318 | "\n",
319 | "### Task:\n",
320 | "Play around with the environment to understand the rewards and how the game dynamics work. When you feel confident with the environment, move on to the next cell."
321 | ]
322 | },
323 | {
324 | "cell_type": "code",
325 | "execution_count": null,
326 | "metadata": {},
327 | "outputs": [],
328 | "source": [
329 | "# Run this to reset the environment\n",
330 | "test_env.reset_env()"
331 | ]
332 | },
333 | {
334 | "cell_type": "code",
335 | "execution_count": null,
336 | "metadata": {},
337 | "outputs": [],
338 | "source": [
339 | "# Run this cell to play the game\n",
340 | "\n",
341 | "# Input your action below\n",
342 | "action = 0\n",
343 | "# Input your action above\n",
344 | "\n",
345 | "test_env.play(action)"
346 | ]
347 | },
348 | {
349 | "cell_type": "markdown",
350 | "metadata": {},
351 | "source": [
352 | "## Discussion point\n",
353 | "The game above is a simple environment, which is easy for humans to solve. However, if you were to use traditional programming, how would you solve it? \n",
354 | "\n",
355 | "### Task\n",
356 | "Use 5 minutes in the group to come up with a strategy, and create a quick draft!\n",
357 | "\n",
358 | ""
359 | ]
360 | },
361 | {
362 | "cell_type": "markdown",
363 | "metadata": {},
364 | "source": [
365 | "# Q-learning\n",
366 | "\n",
367 | "In Reinforcement learning an agent moves from a state $S_{t}$, to a new state $S_{t+1}$ by taking an action $A_{t}$, and receiving the reward $R_{t}$ as illustrated below. The agent will repeat this process for a defined amount of iterations, and this process as a whole is known as an episode. The goal of the Reinforcement learning agent is to maximize the total reward it can collect during an episode. To improve the performance, it learns from each state transition, i.e. move from $S_{t}$ to $S_{t+1}$. How it learns, is what separates the different Reinforcement learning algorithms.\n",
368 | "\n",
369 | "\n",
370 | "\n",
371 | "In this notebook, we will work with one of the most iconic Reinforcement learning algorithms in its simplest form, namely Q-learning using Q-tables. The Q-tables holds the values of the strategy that gives us the highest reward, according to the agent's knowledge. The table can be thought of as the brain of the agent. Each row in the table represents a state, and the columns represent the possible actions in that state. This will be explained further in the next section.\n",
372 | "\n",
373 | "A Q-value is a measure of the total expected reward the agent will receive if it chooses that action and always picks the action with the highest Q-value in the succeeding states, based on its current knowledge of the environment, i.e. the Q-table. Simply said, the higher the Q-value, the better the agent believes the action is.\n",
374 | "\n",
375 | "The Q-learning algorithm is used to update the Q-values for a given action $A_{t}$ in the state $S_{t}$, and is updated after each iteration. The equation is shown below.\n",
376 | "\n",
377 | "$\n",
378 | "\\begin{equation}\n",
379 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]\n",
380 | "\\end{equation}\n",
381 | "$\n",
382 | "\n",
383 | "A key point to note in this equation is that we are updating the Q(s, a) by using the highest Q-value in the next state, $S_{t+1}$. This value is unknown, and it is therefore just the agent's estimate of that state. This makes it an optimization problem were we gradually adjust each Q-value as we receive a reward moving between the states.\n",
384 | "\n",
385 | "\n",
386 | "This might seem daunting at first, however, we will through this notebook break it into simple pieces, which will hopefully, make it easier to grasp."
387 | ]
388 | },
389 | {
390 | "cell_type": "markdown",
391 | "metadata": {},
392 | "source": [
393 | "## Create the Q-table\n",
394 | "The Q-table is the brain of the agent and stores the agent's current knowledge of the Q-values of each state. Each row represents a state, and the columns represent all possible actions in that state. As a result, our table will have the dimensions, (rows=number of states, columns=number of actions). An illustration of the table is shown below, although without the labels on the columns. \n",
395 | "\n",
396 | "\n",
397 | "\n",
398 | "### Task:\n",
399 | "Create and return a Q-table using numpy, where each cell is initialized to zeros:\n",
400 | "- Create the Q-table by using the numpy command: np.zeros(shape=[rows, columns]). Read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.zeros.html)\n",
401 | "- Use the following command to get the number of states in the environment:\n",
402 | " - env.observation_space.n\n",
403 | "- Use the following command to get the number of possible actions. PS. all states have the same amount of actions.\n",
404 | " - env.action_space.n"
405 | ]
406 | },
407 | {
408 | "cell_type": "code",
409 | "execution_count": null,
410 | "metadata": {},
411 | "outputs": [],
412 | "source": [
413 | "def create_q_table(env):\n",
414 | " \n",
415 | " \"Write code below\" \n",
416 | " \n",
417 | " \"Write code above\" \n",
418 | " \n",
419 | " return Q_table"
420 | ]
421 | },
422 | {
423 | "cell_type": "code",
424 | "execution_count": null,
425 | "metadata": {},
426 | "outputs": [],
427 | "source": [
428 | "# Do not edit - Assertion cell #\n",
429 | "if mock.do_assertion():\n",
430 | " assert(np.count_nonzero(mock.get_Q() == 0)), \"All values in Q-table should be zero\"\n",
431 | " assert(create_q_table(mock.get_env()).shape == (mock.get_env().observation_space.n, mock.get_env().action_space.n)), \\\n",
432 | " \"The dimensions are wrong\"\n",
433 | " print(\"The Q-table was correctly built! \" +\n",
434 | " \"It has {} rows, each representing a unique state, and {} columns, each representing an action for that state.\"\\\n",
435 | " .format(mock.get_env().observation_space.n, mock.get_env().action_space.n))"
436 | ]
437 | },
438 | {
439 | "cell_type": "markdown",
440 | "metadata": {},
441 | "source": [
442 | "## Select the best action\n",
443 | "Before we implement the Q-learning algorithm, we will implement a method that, given a state, chooses the action with the highest Q-value.\n",
444 | "\n",
445 | "### Task\n",
446 | "Return the column index of the highest Q-value in the Q-table, given a state.\n",
447 | "- Tip: Use the argmax function from the numpy library. You can read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.argmax.html)"
448 | ]
449 | },
450 | {
451 | "cell_type": "code",
452 | "execution_count": null,
453 | "metadata": {},
454 | "outputs": [],
455 | "source": [
456 | "def get_best_action(Q_table, state):\n",
457 | " \n",
458 | " \"Write code below\" \n",
459 | " \n",
460 | " \"Write code above\"\n",
461 | " \n",
462 | " return best_action"
463 | ]
464 | },
465 | {
466 | "cell_type": "code",
467 | "execution_count": null,
468 | "metadata": {},
469 | "outputs": [],
470 | "source": [
471 | "# Do not edit - Assertion cell #\n",
472 | "if mock.do_assertion():\n",
473 | " assert(get_best_action(mock.get_Q(), 1) == 0), \"The method did not pick the action with the highest Q_value\"\n",
474 | " print(\"The best action for the test state was chosen, excellent!\")"
475 | ]
476 | },
477 | {
478 | "cell_type": "markdown",
479 | "metadata": {},
480 | "source": [
481 | "## Select an action\n",
482 | "\n",
483 | "Initially, all the Q-values in the Q-table are set to zero. As we begin to move around the environment we can end up in a loop, such that we only explore a small part of the environment. This problem is so common its gotten a name; The Exploration vs Exploitation dilemma!\n",
484 | "\n",
485 | "One simple and very effective solution is to use a strategy called Epsilon-Greedy. The strategy makes the agent choose mostly random actions in the initial episodes, which forces the agent to explore the environment. The strategy then gradually shifts the agent behavior towards making greedy choices, i.e. choosing the action with the highest Q-value. The parameter deciding the degree of random actions taken is called epsilon.\n",
486 | "\n",
487 | "### Task:\n",
488 | "Create a method that randomly picks an action IF a random value in the range [0,1] is less than epsilon, or ELSE picks the best action using the method we created earlier. \n",
489 | "- To pick a random action, use env.action_space.sample()\n",
490 | "- To generate a random number with uniform probability in the range [0,1], use np.random.uniform()"
491 | ]
492 | },
493 | {
494 | "cell_type": "code",
495 | "execution_count": null,
496 | "metadata": {},
497 | "outputs": [],
498 | "source": [
499 | "def select_action(Q_table, state, epsilon):\n",
500 | " \n",
501 | " \"Write code below\"\n",
502 | " \n",
503 | " \"Write code above\" \n",
504 | " \n",
505 | " return action"
506 | ]
507 | },
508 | {
509 | "cell_type": "code",
510 | "execution_count": null,
511 | "metadata": {
512 | "scrolled": true
513 | },
514 | "outputs": [],
515 | "source": [
516 | "# Do not edit - Assertion cell #\n",
517 | "if mock.do_assertion():\n",
518 | " assert(select_action(mock.get_Q(), 1, 0) == 0), \\\n",
519 | " \"Method should always return the same value for this state, since epsilon is 0\"\n",
520 | " assert not (len(set([select_action(mock.get_Q(), 1, 1) for x in range(20)])) <= 2), \\\n",
521 | " \"Method should not return identical values when a random action should be chosen\"\n",
522 | " print(\"The correct actions were picked, great job!\")"
523 | ]
524 | },
525 | {
526 | "cell_type": "markdown",
527 | "metadata": {},
528 | "source": [
529 | "## Gradually shift towards exploitation and reduce exploration of state-space\n",
530 | "\n",
531 | "To make sure we are gradually moving from exploration to exploitation, we need to reduce the possibility of choosing a random action. In other words, we need to reduce epsilon. This is done by multiplying it with a constant called epsilon decay.\n",
532 | "\n",
533 | "### Task:\n",
534 | "Create a method which reduces epsilon by multiplying itself with the defined constant epsilon decay."
535 | ]
536 | },
537 | {
538 | "cell_type": "code",
539 | "execution_count": null,
540 | "metadata": {},
541 | "outputs": [],
542 | "source": [
543 | "def update_epsilon(epsilon):\n",
544 | " epsilon_decay = 0.95\n",
545 | " \n",
546 | " \"Write code below\" \n",
547 | "\n",
548 | " \"Write code above\"\n",
549 | " \n",
550 | " return epsilon"
551 | ]
552 | },
553 | {
554 | "cell_type": "code",
555 | "execution_count": null,
556 | "metadata": {},
557 | "outputs": [],
558 | "source": [
559 | "# Do not edit - Assertion cell #\n",
560 | "if mock.do_assertion():\n",
561 | " assert(update_epsilon(5) == 4.75), \"Given an input of 5, the output should have been 4.75\"\n",
562 | " print(\"Epsilon was correctly updated!\")"
563 | ]
564 | },
565 | {
566 | "cell_type": "markdown",
567 | "metadata": {},
568 | "source": [
569 | " ## Find the highest Q_value for a given state\n",
570 | "\n",
571 | "Now it is finally time to begin building our Q-learning algorithm. The method we are going to create first is the part marked red in the equation below. This method returns the highest Q-value of the new state.\n",
572 | "\n",
573 | "$\n",
574 | "\\begin{equation}\n",
575 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ {\\color{red}{\\underset{a}{max} \\ Q(s_{t+1},a)}} - Q(s_{t},a_{t})\\big]\n",
576 | "\\end{equation}\n",
577 | "$ \n",
578 | "\n",
579 | "\n",
580 | "### Task\n",
581 | "Return the highest Q-value for a given state, which is here called new_state.\n",
582 | "- Tip: Use the numpy.max function to retrieve the column (action) with the highest Q-value for the given row(state)"
583 | ]
584 | },
585 | {
586 | "cell_type": "code",
587 | "execution_count": null,
588 | "metadata": {},
589 | "outputs": [],
590 | "source": [
591 | "def find_highest_Q_value_in_state(Q_table, new_state):\n",
592 | " \"Write code below\"\n",
593 | "\n",
594 | " \"Write code above\"\n",
595 | " \n",
596 | " return best_Q_value"
597 | ]
598 | },
599 | {
600 | "cell_type": "code",
601 | "execution_count": null,
602 | "metadata": {},
603 | "outputs": [],
604 | "source": [
605 | "# Do not edit - Assertion cell #\n",
606 | "if mock.do_assertion():\n",
607 | " assert(find_highest_Q_value_in_state(mock.get_Q(), 1) == 10), \\\n",
608 | " \"The method did not pick the action with the highest Q_value\"\n",
609 | " print(\"Perfect, the method returned the highest Q-value for that state!\")"
610 | ]
611 | },
612 | {
613 | "cell_type": "markdown",
614 | "metadata": {},
615 | "source": [
616 | "## Calculate error in Q-value\n",
617 | "The next method we are going to implement calculates how much we need to adjust our Q-value based on the reward we receive from moving from state $S_{t}$ to state $S_{t+1}$. In the intro to the Q-learning section, we defined the Q-value as the assumed expected total reward the agent will receive if it moves from state $S_{t}$ taking action $A_{t}$, and then always picks the action with the highest Q-value throughout the episode using the current Q-table. As a result, the the correct Q-value $Q(S_{t}, A_{t})$ can be defined as\n",
618 | "\n",
619 | "$\n",
620 | "Q(s_{t}, a_{t}) = r_{t} + \\underset{a}{max} \\ Q(s_{t+1},a)\n",
621 | "$\n",
622 | "\n",
623 | "Assuming $\\gamma$ is equal to one, we see that the statement marked red would become equal to 0, if our Q-value was perfect, thereby making no changes to the Q-value.\n",
624 | "\n",
625 | "$\n",
626 | "\\begin{equation}\n",
627 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[{\\color{red}{r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})}}\\big]\n",
628 | "\\end{equation}\n",
629 | "$\n",
630 | "\n",
631 | "The parameter $\\gamma$ is connected to how the agent values immediate rewards compared to later rewards, however explanation of this is outside the scope of this notebook. \n",
632 | "\n",
633 | "### Task:\n",
634 | "Implement the part of the equation mark red.\n",
635 | "- Use the method we created in the previous section to find the highest_q_value_in_state\n",
636 | "- Use brackets notation to select the correct cell i.e. Q_table[Desired_state, desired_action]"
637 | ]
638 | },
639 | {
640 | "cell_type": "code",
641 | "execution_count": null,
642 | "metadata": {},
643 | "outputs": [],
644 | "source": [
645 | "def calculate_error_in_Q_value(Q_table, state, action, reward, new_state):\n",
646 | " \n",
647 | " discount_factor = 0.95\n",
648 | " \n",
649 | " \"Write code below\"\n",
650 | " \n",
651 | " \"Write code above\"\n",
652 | " \n",
653 | " return error_in_Q_value"
654 | ]
655 | },
656 | {
657 | "cell_type": "code",
658 | "execution_count": null,
659 | "metadata": {},
660 | "outputs": [],
661 | "source": [
662 | "# Do not edit - Assertion cell #\n",
663 | "if mock.do_assertion():\n",
664 | " assert(calculate_error_in_Q_value(mock.get_Q(), 0, 0, 10, 1) == 9.5), \\\n",
665 | " \"The method did not calculate the correct error value for the test sample\"\n",
666 | " print(\"The error in Q-value have been calculated correctly!\")"
667 | ]
668 | },
669 | {
670 | "cell_type": "markdown",
671 | "metadata": {},
672 | "source": [
673 | "## Update Q-table\n",
674 | "\n",
675 | "We are almost ready to complete our Q-learning algorithm. \n",
676 | "\n",
677 | "The only element not yet explained is the parameter $\\alpha$. $\\alpha$ is known as the learning rate and controls how quick we update the Q-table. For example, a learning rate of 0.1 indicates that we will only use 10% of the actual error calculated when updating the Q-table. This is because we calculate the Q-value by using the highest Q-value of the next state, which is just our best estimate so far and might be erroneous. If that state has not been visited yet, it would simply have the initialized value of the table, i.e. 0. By using the learning rate, we account for this uncertainty and get a less oscillating system which is proven to give better results. \n",
678 | "\n",
679 | "$\n",
680 | "\\begin{equation}\n",
681 | "Q(s_{t},a_{t})^{new} \\leftarrow {\\color{red}{Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]}}\n",
682 | "\\end{equation}\n",
683 | "$\n",
684 | "\n",
685 | "To implement the algorithm there is one factor which needs to be accounted for. If the episode has ended due to a terminal state, the last reward received will be the Q-value of that state, since there is no possible next state. As a result, if the done parameter is true (an input to the method below), we set the Q-value of that state-action pair equal to the reward.\n",
686 | "\n",
687 | "### Task\n",
688 | "Create a method which either returns\n",
689 | " - The Q-table with the Q-value of that state-action pair equal to the reward, IF it was the final episode. (Done is true)\n",
690 | " - ELSE the Q-table updated with the Q-learning algorithm, using the methods we have created."
691 | ]
692 | },
693 | {
694 | "cell_type": "code",
695 | "execution_count": null,
696 | "metadata": {},
697 | "outputs": [],
698 | "source": [
699 | "def update_q_table(Q_table, state, action, done, reward, new_state):\n",
700 | " \n",
701 | " learning_rate = 0.8\n",
702 | " \n",
703 | " \"Write code below\"\n",
704 | "\n",
705 | " \"Write code above\"\n",
706 | " \n",
707 | " return Q_table"
708 | ]
709 | },
710 | {
711 | "cell_type": "code",
712 | "execution_count": null,
713 | "metadata": {},
714 | "outputs": [],
715 | "source": [
716 | "# Do not edit - Assertion cell #\n",
717 | "if mock.do_assertion():\n",
718 | " assert(update_q_table(mock.get_Q(), 2, 1, 1, 10, 3)[2,1] == 10), \\\n",
719 | " \"The updated Q-value when simulation was done was incorrect\"\n",
720 | " assert(update_q_table(mock.get_Q(), 0, 0, 0, 10, 1)[0,0] == 17.6), \\\n",
721 | " \"The updated Q-value when the episode was not done, was incorrect\"\n",
722 | " print(\"Q-tabel have been correctly updated!\")"
723 | ]
724 | },
725 | {
726 | "cell_type": "markdown",
727 | "metadata": {},
728 | "source": [
729 | "## Take one step\n",
730 | "\n",
731 | "Now we are ready to create a method which makes our agent make a move in the environment.\n",
732 | "\n",
733 | "### Task\n",
734 | "Create a method which selects the best action, passes this to our environment, and returns the action, reward, done new_state variables.\n",
735 | "- To choose an action use the select_action method we created earlier\n",
736 | "- To make a step in the environment, use this method env.step(action)\n",
737 | "- env.step(action) return the following values accordingly: new_state, reward, done, state_transition_probability\n",
738 | "- Ignore the state_transition_probability, as this is not used in this environment\n",
739 | " "
740 | ]
741 | },
742 | {
743 | "cell_type": "code",
744 | "execution_count": null,
745 | "metadata": {},
746 | "outputs": [],
747 | "source": [
748 | "def step(Q_table, env, state, epsilon):\n",
749 | " \n",
750 | " \"Write code below\"\n",
751 | " \n",
752 | " \"Write code above\"\n",
753 | " \n",
754 | " return action, reward, done, new_state"
755 | ]
756 | },
757 | {
758 | "cell_type": "code",
759 | "execution_count": null,
760 | "metadata": {},
761 | "outputs": [],
762 | "source": [
763 | "# Do not edit - Assertion cell #\n",
764 | "if mock.do_assertion():\n",
765 | " mock_env1 = deepcopy(mock.get_env())\n",
766 | " mock_env1.reset()\n",
767 | " mock_env2 = deepcopy(mock_env1)\n",
768 | "\n",
769 | " new_state, _, _, _ = mock_env1.step(0)\n",
770 | " assert(step(mock.get_Q(), mock_env2, 0, 0) == (0, -1, False, new_state)), \\\n",
771 | " \"Your environment did not return the correct values\"\n",
772 | " print(\"Your step method seems to function properly, good job!\")"
773 | ]
774 | },
775 | {
776 | "cell_type": "markdown",
777 | "metadata": {},
778 | "source": [
779 | "## Create train method\n",
780 | "\n",
781 | "Finally, we can create the method which will train our agent, i.e. update the Q-table.\n",
782 | "\n",
783 | "### Task\n",
784 | "In this method, you will enter your code at multiple places, but always between \"Write code below\" and \"Write code above\". \n",
785 | "- You will only input methods you have already created. \n",
786 | "- Remember to store the value in the right variable, i.e. the Q-table is stored in the variable Q_table\n",
787 | "- To see which variables the methods returns, look at your implementation in the previous cells.\n",
788 | "- PS. The following cells do not have assertion cells."
789 | ]
790 | },
791 | {
792 | "cell_type": "code",
793 | "execution_count": null,
794 | "metadata": {},
795 | "outputs": [],
796 | "source": [
797 | "def train(env, num_episodes, max_iterations):\n",
798 | " done = False\n",
799 | " epsilon = 1\n",
800 | " reward_list, iterations_list, epsilon_list = [], [], []\n",
801 | " \n",
802 | " \"Write code below\"\n",
803 | " # Create Q-table\n",
804 | "\n",
805 | " \"Write code above\"\n",
806 | "\n",
807 | " for episode in range(num_episodes):\n",
808 | "\n",
809 | " if (done and episode % 10 == 0): \n",
810 | " print(\"Episode: {} | Iterations: {} | Total Reward: {} | Epsilon: {}\"\\\n",
811 | " .format(episode, iterations, total_reward, epsilon))\n",
812 | " \n",
813 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
814 | " \n",
815 | " \"Write code below\"\n",
816 | " # Update epsilon\n",
817 | " \n",
818 | " \"Write code above\"\n",
819 | " \n",
820 | " while not done:\n",
821 | " \n",
822 | " \"Write code below\"\n",
823 | " \n",
824 | " # Take step\n",
825 | " \n",
826 | " # Update Q-table\n",
827 | " \n",
828 | " \"Write code above\"\n",
829 | " \n",
830 | " # Set the new state as the current state\n",
831 | " state = new_state\n",
832 | " \n",
833 | " # Update episode information\n",
834 | " iterations += 1\n",
835 | " total_reward += reward\n",
836 | " \n",
837 | " # End episode if it has lasted for too many iterations\n",
838 | " if (iterations >= max_iterations): done = True\n",
839 | "\n",
840 | " reward_list.append(total_reward)\n",
841 | " iterations_list.append(iterations)\n",
842 | " epsilon_list.append(epsilon)\n",
843 | " \n",
844 | " return Q_table, (reward_list, iterations_list, epsilon_list)"
845 | ]
846 | },
847 | {
848 | "cell_type": "markdown",
849 | "metadata": {},
850 | "source": [
851 | "## Create play method\n",
852 | "\n",
853 | "This method can be used if you want to test the performance of your agent, by only running one episode."
854 | ]
855 | },
856 | {
857 | "cell_type": "code",
858 | "execution_count": null,
859 | "metadata": {},
860 | "outputs": [],
861 | "source": [
862 | "def play(env, Q_table):\n",
863 | " \n",
864 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
865 | "\n",
866 | " while not done:\n",
867 | "\n",
868 | " # Take step\n",
869 | " action, reward, done, new_state = step(Q_table, env, state, 0)\n",
870 | "\n",
871 | " # Set the new state as the current state\n",
872 | " state = new_state\n",
873 | "\n",
874 | " # Update episode information\n",
875 | " iterations += 1\n",
876 | " total_reward += reward\n",
877 | "\n",
878 | " if (iterations >= 100): done = True\n",
879 | " \n",
880 | " print(\"Iterations: {} | Total Reward: {}\".format(iterations, total_reward))"
881 | ]
882 | },
883 | {
884 | "cell_type": "markdown",
885 | "metadata": {
886 | "collapsed": true
887 | },
888 | "source": [
889 | "## Time to put our methods to use!\n",
890 | "\n",
891 | "Congratulation, you have created every method need to train your Q-learning agent!\n",
892 | "\n",
893 | "### Task\n",
894 | "To train your agent, simply set a value for the maximum_episodes and maximum_iterations\n",
895 | "- Tip 1: Maximum_episodes should be less than 2000\n",
896 | "- Tip 2: Maximum iterations should be less than 200"
897 | ]
898 | },
899 | {
900 | "cell_type": "code",
901 | "execution_count": null,
902 | "metadata": {},
903 | "outputs": [],
904 | "source": [
905 | "maximum_episodes = 0\n",
906 | "maximum_iterations = 0\n",
907 | "\n",
908 | "Q_table_trained, data = train(env, maximum_episodes, maximum_iterations)"
909 | ]
910 | },
911 | {
912 | "cell_type": "markdown",
913 | "metadata": {},
914 | "source": [
915 | "## Plot key variables \n",
916 | "\n",
917 | "This method plots different parameters during training. As a result, the training cell above must be run before this one.\n",
918 | "\n",
919 | "### Task\n",
920 | "- Simply run the cell below to create the plots"
921 | ]
922 | },
923 | {
924 | "cell_type": "code",
925 | "execution_count": null,
926 | "metadata": {},
927 | "outputs": [],
928 | "source": [
929 | "try: \n",
930 | " assert(data)\n",
931 | " plot_visualization(data)\n",
932 | "except:\n",
933 | " print(\"You need to run the **train** cell before running this cell, as it uses the data from the training!\")"
934 | ]
935 | },
936 | {
937 | "cell_type": "markdown",
938 | "metadata": {},
939 | "source": [
940 | "## Play around for one round\n",
941 | "\n",
942 | "This method runs one episode, using your trained Q-table. This is a quick way to test the performance.\n",
943 | "\n",
944 | "### Task\n",
945 | "- Run the cell below to quickly see the performance of your agent. This method does not alter your Q_table, so you can run it as many times as you like!"
946 | ]
947 | },
948 | {
949 | "cell_type": "code",
950 | "execution_count": null,
951 | "metadata": {},
952 | "outputs": [],
953 | "source": [
954 | "play(env, Q_table_trained)"
955 | ]
956 | },
957 | {
958 | "cell_type": "markdown",
959 | "metadata": {},
960 | "source": [
961 | "## Visualize an episode\n",
962 | "\n",
963 | "### Task\n",
964 | "This method works similarly to the one above, but visualizes the environment.\n",
965 | "- If you would like to end the simulation, click on the Kernal tab in the toolbar, and click \"Interrupt\"."
966 | ]
967 | },
968 | {
969 | "cell_type": "code",
970 | "execution_count": null,
971 | "metadata": {},
972 | "outputs": [],
973 | "source": [
974 | "render_performance(env, Q_table_trained)"
975 | ]
976 | },
977 | {
978 | "cell_type": "markdown",
979 | "metadata": {},
980 | "source": [
981 | "## And we are done, almost!\n",
982 | "Now you have everything you need to play around with the environment and alter different parameters to see how it impacts our agent. Can you improve the parameters already chosen? Suggestions:\n",
983 | "- Increase the number of episodes and reduce iterations\n",
984 | "- Change epsilon decay to change the Epsilon-Greedy Strategy\n",
985 | "- Change the learning rate to make smaller or larger corrections to the Q-table per update\n",
986 | "- Change the discount factor $\\gamma$\n",
987 | "\n",
988 | "If you would like to create your own cells, do the following:\n",
989 | "1. Click on an existing cell\n",
990 | "2. Click the Esc - button\n",
991 | "3. Press \"b\" to add a cell below, press \"a\" the add a cell above, or double tap \"d\" to delete a cell\n",
992 | "\n",
993 | "### Task\n",
994 | "Run the cell below to turn off the assertion cells. \n",
995 | "Have fun!"
996 | ]
997 | },
998 | {
999 | "cell_type": "code",
1000 | "execution_count": null,
1001 | "metadata": {},
1002 | "outputs": [],
1003 | "source": [
1004 | "mock.turn_off_assertion()"
1005 | ]
1006 | },
1007 | {
1008 | "cell_type": "markdown",
1009 | "metadata": {},
1010 | "source": [
1011 | "## Want to try a different environment?\n",
1012 | "If you want to try a different environment suitable for Q-learning with tables, you can try this one:\n",
1013 | "- FrozenLake-v0\n",
1014 | "- You can read more about it [here](https://gym.openai.com/envs/FrozenLake-v0/)\n",
1015 | "- Tip: This environment may require the agent to do a more thorough exploration.\n",
1016 | "\n",
1017 | "To use it, simply replace it with \"Taxi-v2\" in the cell where we created the env-variabel. \n",
1018 | "Tip: Remember to rerun the cells."
1019 | ]
1020 | }
1021 | ],
1022 | "metadata": {
1023 | "kernelspec": {
1024 | "display_name": "Python 3",
1025 | "language": "python",
1026 | "name": "python3"
1027 | },
1028 | "language_info": {
1029 | "codemirror_mode": {
1030 | "name": "ipython",
1031 | "version": 3
1032 | },
1033 | "file_extension": ".py",
1034 | "mimetype": "text/x-python",
1035 | "name": "python",
1036 | "nbconvert_exporter": "python",
1037 | "pygments_lexer": "ipython3",
1038 | "version": "3.7.4"
1039 | }
1040 | },
1041 | "nbformat": 4,
1042 | "nbformat_minor": 2
1043 | }
1044 |
--------------------------------------------------------------------------------
/Workshop/Workshop_Q_table_solution.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Q-learning using tables\n",
8 | "##### Authors: Eirik Fagtun Kjærnli and Fabian Dietrichson [Accenture]"
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "## Welcome \n",
16 | "Welcome to this workshop!\n",
17 | "\n",
18 | "The workshop is structured such that for each cell, you will write your own code and the code will then be asserted in the next cell. The assertion cell is marked \"# Do not edit - Assertion cell #\". \n",
19 | "The code should be written between \"Write code below\" and \"Write code above\". If your code does not pass the assertion, you will have to rewrite it before continuing.\n",
20 | "\n",
21 | "### Task:\n",
22 | "Implement the method multiply_input_by_2\n",
23 | "- The code should take the input variabel \"input_variabel\", and multiply it by 2\n",
24 | "- The answer should be stored in the variabel \"result\"\n",
25 | "- When the method is implemented, run the cell.\n",
26 | "\n",
27 | "Hotkey to run a cell: CTRL + ENTER"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": null,
33 | "metadata": {},
34 | "outputs": [],
35 | "source": [
36 | "def multiply_input_by_2(input_variabel):\n",
37 | "\n",
38 | " \"Write code below\" \n",
39 | " result = input_variabel * 2\n",
40 | " \n",
41 | " \"Write code above\" \n",
42 | " \n",
43 | " return result"
44 | ]
45 | },
46 | {
47 | "cell_type": "code",
48 | "execution_count": null,
49 | "metadata": {},
50 | "outputs": [],
51 | "source": [
52 | "# Do not edit - Assertion cell #\n",
53 | "assert(multiply_input_by_2(10) == 20), \"Your method did not multiple the input by 2\"\n",
54 | "print(\"Great, you correctly implemented the method!\")"
55 | ]
56 | },
57 | {
58 | "cell_type": "markdown",
59 | "metadata": {},
60 | "source": [
61 | "## Import necessary packages\n",
62 | "To implement the native Q-learning algorithm and use it in an environment, we just need two additional packages.\n",
63 | "\n",
64 | "_Numpy_\n",
65 | "- Numpy adds support for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays. We could do this workshop using native Python arrays and built-in methods, however using Numpy simplifies the process. \n",
66 | "- To call the methods in the package, we use the \"as np\" command. Such that, to create a numpy table with 2 rows and 2 columns where all the cells are zero, you can simply write np.seros(2,2).\n",
67 | "\n",
68 | "_OpenAI Gym_\n",
69 | "- The Gym library is a collection of demo problems an associated environment, that you can use to work out your reinforcement learning algorithms. These range from simple text problems, to complex physical problems, to Atari video games. OpenAI Gym is therefore by many the preferred framework to learn and test Reinforcement learning algorithms.\"\n",
70 | "\n",
71 | "\n",
72 | "### Task:\n",
73 | "To import the necessary packages, simply run the cell below, and then run the assertion cell to verify that they have been imported correctly."
74 | ]
75 | },
76 | {
77 | "cell_type": "code",
78 | "execution_count": null,
79 | "metadata": {},
80 | "outputs": [],
81 | "source": [
82 | "import numpy as np\n",
83 | "import gym"
84 | ]
85 | },
86 | {
87 | "cell_type": "code",
88 | "execution_count": null,
89 | "metadata": {},
90 | "outputs": [],
91 | "source": [
92 | "# Do not edit - Assertion cell #\n",
93 | "try:\n",
94 | " assert(gym)\n",
95 | " print(\"Great, the packages were imported correctly!\")\n",
96 | "except:\n",
97 | " print(\"You did not run the cell above, do this before you continue!\")"
98 | ]
99 | },
100 | {
101 | "cell_type": "markdown",
102 | "metadata": {},
103 | "source": [
104 | "## Problem Introduction\n",
105 | "\n",
106 | "The environment we are going to use is a simple grid world, where the agent is controlling a taxi. The environment was introduced in [Dietterich2000] to illustrate some issues in hierarchical reinforcement learning. There are 4 locations (labeled by different letters) and your job is to pick up the passenger at one location and drop the passenger off in another. You can read more about the environment [here](https://gym.openai.com/envs/Taxi-v2/).\n",
107 | "\n",
108 | "### Task:\n",
109 | "Create the environment variable containing all necessary methods to run the Taxi-v3 game. \n",
110 | "\n",
111 | "_Tip: Just run the cell below_"
112 | ]
113 | },
114 | {
115 | "cell_type": "code",
116 | "execution_count": null,
117 | "metadata": {},
118 | "outputs": [],
119 | "source": [
120 | "environment_id = \"Taxi-v3\"\n",
121 | "env = gym.make(environment_id)"
122 | ]
123 | },
124 | {
125 | "cell_type": "code",
126 | "execution_count": null,
127 | "metadata": {},
128 | "outputs": [],
129 | "source": [
130 | "# Do not edit - Assertion cell #\n",
131 | "try:\n",
132 | " assert(env)\n",
133 | " print(\"You successfully created the environement {}\".format(env.spec.id))\n",
134 | "except:\n",
135 | " print(\"You create the environment incorrectly!\")"
136 | ]
137 | },
138 | {
139 | "cell_type": "markdown",
140 | "metadata": {},
141 | "source": [
142 | "## Build support methods for workshop\n",
143 | "Before you can go on, the cell below must be run. These are methods used to verify your work, in addition to the support function that will be used throughout the workshop. \n",
144 | "\n",
145 | "### Task\n",
146 | "Do as you have done before, simply mark the cell below and press CTRL + Enter!"
147 | ]
148 | },
149 | {
150 | "cell_type": "code",
151 | "execution_count": null,
152 | "metadata": {
153 | "scrolled": true
154 | },
155 | "outputs": [],
156 | "source": [
157 | "from IPython.display import clear_output\n",
158 | "import gym\n",
159 | "import os\n",
160 | "import time\n",
161 | "from copy import deepcopy\n",
162 | "import seaborn as sb\n",
163 | "import matplotlib.pyplot as plt\n",
164 | "\n",
165 | "class TestEnvironment():\n",
166 | " \n",
167 | " def __init__(self):\n",
168 | " self.env = gym.make(environment_id)\n",
169 | " self.env.reset()\n",
170 | " \n",
171 | " def play(self, action):\n",
172 | " if not (0 <= int(action) <= 5):\n",
173 | " print(\"Action value must either be 0,1,2,3,4 or 5\")\n",
174 | " return\n",
175 | "\n",
176 | " clear_output()\n",
177 | " _, reward, done, _ = self.env.step(action)\n",
178 | " self.env.render()\n",
179 | " print(\"Reward: \", reward)\n",
180 | " \n",
181 | " if(done):\n",
182 | " print(\"Game completed, resetting environment!\")\n",
183 | " self.reset_env()\n",
184 | " \n",
185 | " def reset_env(self):\n",
186 | " self.env.reset()\n",
187 | " print(\"Environment has been reset\")\n",
188 | " time.sleep(3)\n",
189 | " clear_output()\n",
190 | "\n",
191 | "class MockData():\n",
192 | " def __init__(self):\n",
193 | " self.env = gym.make(environment_id)\n",
194 | " self.env.seed(10)\n",
195 | " self.should_assert = True\n",
196 | " \n",
197 | " def get_Q(self):\n",
198 | " Q = np.zeros([self.env.observation_space.n, self.env.action_space.n])\n",
199 | " Q[0:2,0] = 10\n",
200 | " return Q\n",
201 | " \n",
202 | " def get_env(self):\n",
203 | " return self.env\n",
204 | " \n",
205 | " def turn_off_assertion(self):\n",
206 | " self.should_assert = False\n",
207 | " \n",
208 | " def do_assertion(self):\n",
209 | " return self.should_assert\n",
210 | "\n",
211 | "def visualize_q_table():\n",
212 | " fig=plt.figure(figsize=(10, 10))\n",
213 | " heat_map = sb.heatmap(Q_trained)\n",
214 | " plt.show()\n",
215 | "\n",
216 | "def plot_visualization(data):\n",
217 | " \n",
218 | " reward_list = data[0]\n",
219 | " iteration_list = data[1]\n",
220 | " epsilon_list = data[2]\n",
221 | " \n",
222 | " episodes = range(len(reward_list))\n",
223 | " \n",
224 | " plt.figure(figsize=(10,10))\n",
225 | " ax = plt.subplot(311)\n",
226 | " ax.set_title(\"Reward\")\n",
227 | " ax = plt.plot(episodes, reward_list)\n",
228 | "\n",
229 | " ax = plt.subplot(312)\n",
230 | " ax.set_title(\"Iterations per episode\")\n",
231 | " ax.plot(episodes, iteration_list)\n",
232 | " \n",
233 | " ax = plt.subplot(313)\n",
234 | " ax.set_title(\"Epsilon\")\n",
235 | " ax.set_xlabel(\"Episodes\")\n",
236 | " ax.plot(episodes, epsilon_list)\n",
237 | " \n",
238 | " plt.show()\n",
239 | "\n",
240 | "def reset_env_and_update_params(env):\n",
241 | " done = False\n",
242 | " state = env.reset()\n",
243 | " iterations = 0\n",
244 | " total_reward = 0\n",
245 | " \n",
246 | " return state, done, iterations, total_reward\n",
247 | "\n",
248 | "def render_performance(env, Q):\n",
249 | " iterations = 0\n",
250 | " total_reward = 0\n",
251 | " t_sleep = 1.2\n",
252 | " \n",
253 | " state = env.reset()\n",
254 | " env.render()\n",
255 | " time.sleep(t_sleep)\n",
256 | " clear_output()\n",
257 | " done = False\n",
258 | "\n",
259 | " while not done:\n",
260 | " action = np.argmax(Q[state,:]) \n",
261 | "\n",
262 | " new_state, reward, done, _ = env.step(action)\n",
263 | " state = new_state\n",
264 | "\n",
265 | " env.render()\n",
266 | " time.sleep(t_sleep)\n",
267 | " clear_output()\n",
268 | " \n",
269 | " iterations += 1\n",
270 | " total_reward += reward\n",
271 | " \n",
272 | " if (iterations >= 20): \n",
273 | " done = True\n",
274 | " print(\"Agent did not complete the episode within 20 iterations, train your agent better!\")\n",
275 | " return\n",
276 | " \n",
277 | " print(\"Your agent completed the task using {} iterations, \\\n",
278 | " and got a total reward of {}\".format(iterations, total_reward))\n",
279 | " \n",
280 | "test_env = TestEnvironment()\n",
281 | "mock = MockData()"
282 | ]
283 | },
284 | {
285 | "cell_type": "code",
286 | "execution_count": null,
287 | "metadata": {},
288 | "outputs": [],
289 | "source": [
290 | "# Do not edit - Assertion cell #\n",
291 | "try:\n",
292 | " assert(mock)\n",
293 | " print(\"Great, the support methods were created!\")\n",
294 | "except:\n",
295 | " print(\"You did not run the cell above, do this before you continue!\")"
296 | ]
297 | },
298 | {
299 | "cell_type": "markdown",
300 | "metadata": {},
301 | "source": [
302 | "## Experiment with the environment\n",
303 | "Before we start creating the Q-learning agent, we will explore the environment we are going to use. This is done using the support methods we have created for you.\n",
304 | "\n",
305 | "**How to play**\n",
306 | "1. Simply input an action, e.g. the integer 1, where it is indicated in the cell below, and click CTRL + Enter. \n",
307 | "2. The game will reset when you have picked up and then dropped off the passenger at the indicated location.\n",
308 | "3. If you would like to reset the test environment during the execution, run the cell which indicates this below. \n",
309 | "\n",
310 | "*Tip*: The bold blue colored letter shows where the passenger should be picked up, and the pink letter indicates the drop-off spot.\n",
311 | "\n",
312 | "**Possible actions**\n",
313 | "0. Move down\n",
314 | "1. Move up\n",
315 | "2. Move right\n",
316 | "3. Move left\n",
317 | "4. Pick up passenger\n",
318 | "5. Drop off passenger\n",
319 | "\n",
320 | "### Task:\n",
321 | "Play around with the environment to understand the rewards and how the game dynamics work. When you feel confident with the environment, move on to the next cell."
322 | ]
323 | },
324 | {
325 | "cell_type": "code",
326 | "execution_count": null,
327 | "metadata": {},
328 | "outputs": [],
329 | "source": [
330 | "# Run this to reset the environment\n",
331 | "test_env.reset_env()"
332 | ]
333 | },
334 | {
335 | "cell_type": "code",
336 | "execution_count": null,
337 | "metadata": {},
338 | "outputs": [],
339 | "source": [
340 | "# Run this cell to play the game\n",
341 | "\n",
342 | "# Input your action below\n",
343 | "action = 0\n",
344 | "# Input your action above\n",
345 | "\n",
346 | "test_env.play(action)"
347 | ]
348 | },
349 | {
350 | "cell_type": "markdown",
351 | "metadata": {},
352 | "source": [
353 | "## Discussion point\n",
354 | "The game above is a simple environment, which is easy for humans to solve. However, if you were to use traditional programming, how would you solve it? \n",
355 | "\n",
356 | "### Task\n",
357 | "Use 5 minutes in the group to come up with a strategy, and create a quick draft!\n",
358 | "\n",
359 | ""
360 | ]
361 | },
362 | {
363 | "cell_type": "markdown",
364 | "metadata": {},
365 | "source": [
366 | "# Q-learning\n",
367 | "\n",
368 | "In Reinforcement learning an agent moves from a state $S_{t}$, to a new state $S_{t+1}$ by taking an action $A_{t}$, and receiving the reward $R_{t}$ as illustrated below. The agent will repeat this process for a defined amount of iterations, and this process as a whole is known as an episode. The goal of the Reinforcement learning agent is to maximize the total reward it can collect during an episode. To improve the performance, it learns from each state transition, i.e. move from $S_{t}$ to $S_{t+1}$. How it learns, is what separates the different Reinforcement learning algorithms.\n",
369 | "\n",
370 | "\n",
371 | "\n",
372 | "In this notebook, we will work with one of the most iconic Reinforcement learning algorithms in its simplest form, namely Q-learning using Q-tables. The Q-tables holds the values of the strategy that gives us the highest reward, according to the agent's knowledge. The table can be thought of as the brain of the agent. Each row in the table represents a state, and the columns represent the possible actions in that state. This will be explained further in the next section.\n",
373 | "\n",
374 | "A Q-value is a measure of the total expected reward the agent will receive if it chooses that action and always picks the action with the highest Q-value in the succeeding states, based on its current knowledge of the environment, i.e. the Q-table. Simply said, the higher the Q-value, the better the agent believes the action is.\n",
375 | "\n",
376 | "The Q-learning algorithm is used to update the Q-values for a given action $A_{t}$ in the state $S_{t}$, and is updated after each iteration. The equation is shown below.\n",
377 | "\n",
378 | "$\n",
379 | "\\begin{equation}\n",
380 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]\n",
381 | "\\end{equation}\n",
382 | "$\n",
383 | "\n",
384 | "A key point to note in this equation is that we are updating the Q(s, a) by using the highest Q-value in the next state, $S_{t+1}$. This value is unknown, and it is therefore just the agent's estimate of that state. This makes it an optimization problem were we gradually adjust each Q-value as we receive a reward moving between the states.\n",
385 | "\n",
386 | "\n",
387 | "This might seem daunting at first, however, we will through this notebook break it into simple pieces, which will hopefully, make it easier to grasp."
388 | ]
389 | },
390 | {
391 | "cell_type": "markdown",
392 | "metadata": {},
393 | "source": [
394 | "## Create the Q-table\n",
395 | "The Q-table is the brain of the agent and stores the agent's current knowledge of the Q-values of each state. Each row represents a state, and the columns represent all possible actions in that state. As a result, our table will have the dimensions, (rows=number of states, columns=number of actions). An illustration of the table is shown below, although without the labels on the columns. \n",
396 | "\n",
397 | "\n",
398 | "\n",
399 | "### Task:\n",
400 | "Create and return a Q-table using numpy, where each cell is initialized to zeros:\n",
401 | "- Create the Q-table by using the numpy command: np.zeros(shape=[rows, columns]). Read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.zeros.html)\n",
402 | "- Use the following command to get the number of states in the environment:\n",
403 | " - env.observation_space.n\n",
404 | "- Use the following command to get the number of possible actions. PS. all states have the same amount of actions.\n",
405 | " - env.action_space.n"
406 | ]
407 | },
408 | {
409 | "cell_type": "code",
410 | "execution_count": null,
411 | "metadata": {},
412 | "outputs": [],
413 | "source": [
414 | "def create_q_table(env):\n",
415 | " \n",
416 | " \"Write code below\" \n",
417 | " action_size = env.action_space.n\n",
418 | " observation_size = env.observation_space.n\n",
419 | " \n",
420 | " Q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
421 | " \n",
422 | " \"Write code above\" \n",
423 | " \n",
424 | " return Q_table"
425 | ]
426 | },
427 | {
428 | "cell_type": "code",
429 | "execution_count": null,
430 | "metadata": {},
431 | "outputs": [],
432 | "source": [
433 | "# Do not edit - Assertion cell #\n",
434 | "if mock.do_assertion():\n",
435 | " assert(np.count_nonzero(mock.get_Q() == 0)), \"All values in Q-table should be zero\"\n",
436 | " assert(create_q_table(mock.get_env()).shape == (mock.get_env().observation_space.n, mock.get_env().action_space.n)), \\\n",
437 | " \"The dimensions are wrong\"\n",
438 | " print(\"The Q-table was correctly built! \" +\n",
439 | " \"It has {} rows, each representing a unique state, and {} columns, each representing an action for that state.\"\\\n",
440 | " .format(mock.get_env().observation_space.n, mock.get_env().action_space.n))"
441 | ]
442 | },
443 | {
444 | "cell_type": "markdown",
445 | "metadata": {},
446 | "source": [
447 | "## Select the best action\n",
448 | "Before we implement the Q-learning algorithm, we will implement a method that, given a state, chooses the action with the highest Q-value.\n",
449 | "\n",
450 | "### Task\n",
451 | "Return the column index of the highest Q-value in the Q-table, given a state.\n",
452 | "- Tip: Use the argmax function from the numpy library. You can read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.argmax.html)"
453 | ]
454 | },
455 | {
456 | "cell_type": "code",
457 | "execution_count": null,
458 | "metadata": {},
459 | "outputs": [],
460 | "source": [
461 | "def get_best_action(Q_table, state):\n",
462 | " \n",
463 | " \"Write code below\" \n",
464 | " best_action = np.argmax(Q_table[state,:]) \n",
465 | " \n",
466 | " \"Write code above\"\n",
467 | " \n",
468 | " return best_action"
469 | ]
470 | },
471 | {
472 | "cell_type": "code",
473 | "execution_count": null,
474 | "metadata": {},
475 | "outputs": [],
476 | "source": [
477 | "# Do not edit - Assertion cell #\n",
478 | "if mock.do_assertion():\n",
479 | " assert(get_best_action(mock.get_Q(), 1) == 0), \"The method did not pick the action with the highest Q_value\"\n",
480 | " print(\"The best action for the test state was chosen, excellent!\")"
481 | ]
482 | },
483 | {
484 | "cell_type": "markdown",
485 | "metadata": {},
486 | "source": [
487 | "## Select an action\n",
488 | "\n",
489 | "Initially, all the Q-values in the Q-table are set to zero. As we begin to move around the environment we can end up in a loop, such that we only explore a small part of the environment. This problem is so common its gotten a name; The Exploration vs Exploitation dilemma!\n",
490 | "\n",
491 | "One simple and very effective solution is to use a strategy called Epsilon-Greedy. The strategy makes the agent choose mostly random actions in the initial episodes, which forces the agent to explore the environment. The strategy then gradually shifts the agent behavior towards making greedy choices, i.e. choosing the action with the highest Q-value. The parameter deciding the degree of random actions taken is called epsilon.\n",
492 | "\n",
493 | "### Task:\n",
494 | "Create a method that randomly picks an action IF a random value in the range [0,1] is less than epsilon, or ELSE picks the best action using the method we created earlier. \n",
495 | "- To pick a random action, use env.action_space.sample()\n",
496 | "- To generate a random number with uniform probability in the range [0,1], use np.random.uniform()"
497 | ]
498 | },
499 | {
500 | "cell_type": "code",
501 | "execution_count": null,
502 | "metadata": {},
503 | "outputs": [],
504 | "source": [
505 | "def select_action(Q_table, state, epsilon):\n",
506 | " \n",
507 | " \"Write code below\"\n",
508 | " if epsilon > np.random.uniform():\n",
509 | " action = env.action_space.sample()\n",
510 | " else:\n",
511 | " action = get_best_action(Q_table, state)\n",
512 | " \n",
513 | " \"Write code above\" \n",
514 | " \n",
515 | " return action"
516 | ]
517 | },
518 | {
519 | "cell_type": "code",
520 | "execution_count": null,
521 | "metadata": {
522 | "scrolled": true
523 | },
524 | "outputs": [],
525 | "source": [
526 | "# Do not edit - Assertion cell #\n",
527 | "if mock.do_assertion():\n",
528 | " assert(select_action(mock.get_Q(), 1, 0) == 0), \\\n",
529 | " \"Method should always return the same value for this state, since epsilon is 0\"\n",
530 | " assert not (len(set([select_action(mock.get_Q(), 1, 1) for x in range(20)])) <= 2), \\\n",
531 | " \"Method should not return identical values when a random action should be chosen\"\n",
532 | " print(\"The correct actions were picked, great job!\")"
533 | ]
534 | },
535 | {
536 | "cell_type": "markdown",
537 | "metadata": {},
538 | "source": [
539 | "## Gradually shift towards exploitation and reduce exploration of state-space\n",
540 | "\n",
541 | "To make sure we are gradually moving from exploration to exploitation, we need to reduce the possibility of choosing a random action. In other words, we need to reduce epsilon. This is done by multiplying it with a constant called epsilon decay.\n",
542 | "\n",
543 | "### Task:\n",
544 | "Create a method which reduces epsilon by multiplying itself with the defined constant epsilon decay."
545 | ]
546 | },
547 | {
548 | "cell_type": "code",
549 | "execution_count": null,
550 | "metadata": {},
551 | "outputs": [],
552 | "source": [
553 | "def update_epsilon(epsilon):\n",
554 | " epsilon_decay = 0.95\n",
555 | " \n",
556 | " \"Write code below\" \n",
557 | " epsilon *= epsilon_decay\n",
558 | " \"Write code above\"\n",
559 | " \n",
560 | " return epsilon"
561 | ]
562 | },
563 | {
564 | "cell_type": "code",
565 | "execution_count": null,
566 | "metadata": {},
567 | "outputs": [],
568 | "source": [
569 | "# Do not edit - Assertion cell #\n",
570 | "if mock.do_assertion():\n",
571 | " assert(update_epsilon(5) == 4.75), \"Given an input of 5, the output should have been 4.75\"\n",
572 | " print(\"Epsilon was correctly updated!\")"
573 | ]
574 | },
575 | {
576 | "cell_type": "markdown",
577 | "metadata": {},
578 | "source": [
579 | " ## Find the highest Q_value for a given state\n",
580 | "\n",
581 | "Now it is finally time to begin building our Q-learning algorithm. The method we are going to create first is the part marked red in the equation below. This method returns the highest Q-value of the new state.\n",
582 | "\n",
583 | "$\n",
584 | "\\begin{equation}\n",
585 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ {\\color{red}{\\underset{a}{max} \\ Q(s_{t+1},a)}} - Q(s_{t},a_{t})\\big]\n",
586 | "\\end{equation}\n",
587 | "$ \n",
588 | "\n",
589 | "\n",
590 | "### Task\n",
591 | "Return the highest Q-value for a given state, which is here called new_state.\n",
592 | "- Tip: Use the numpy.max function to retrieve the column (action) with the highest Q-value for the given row(state)"
593 | ]
594 | },
595 | {
596 | "cell_type": "code",
597 | "execution_count": null,
598 | "metadata": {},
599 | "outputs": [],
600 | "source": [
601 | "def find_highest_Q_value_in_state(Q_table, new_state):\n",
602 | " \"Write code below\"\n",
603 | " best_Q_value = np.max(Q_table[new_state,:])\n",
604 | " \"Write code above\"\n",
605 | " \n",
606 | " return best_Q_value"
607 | ]
608 | },
609 | {
610 | "cell_type": "code",
611 | "execution_count": null,
612 | "metadata": {},
613 | "outputs": [],
614 | "source": [
615 | "# Do not edit - Assertion cell #\n",
616 | "if mock.do_assertion():\n",
617 | " assert(find_highest_Q_value_in_state(mock.get_Q(), 1) == 10), \\\n",
618 | " \"The method did not pick the action with the highest Q_value\"\n",
619 | " print(\"Perfect, the method returned the highest Q-value for that state!\")"
620 | ]
621 | },
622 | {
623 | "cell_type": "markdown",
624 | "metadata": {},
625 | "source": [
626 | "## Calculate error in Q-value\n",
627 | "The next method we are going to implement calculates how much we need to adjust our Q-value based on the reward we receive from moving from state $S_{t}$ to state $S_{t+1}$. In the intro to the Q-learning section, we defined the Q-value as the assumed expected total reward the agent will receive if it moves from state $S_{t}$ taking action $A_{t}$, and then always picks the action with the highest Q-value throughout the episode using the current Q-table. As a result, the the correct Q-value $Q(S_{t}, A_{t})$ can be defined as\n",
628 | "\n",
629 | "$\n",
630 | "Q(s_{t}, a_{t}) = r_{t} + \\underset{a}{max} \\ Q(s_{t+1},a)\n",
631 | "$\n",
632 | "\n",
633 | "Assuming $\\gamma$ is equal to one, we see that the statement marked red would become equal to 0, if our Q-value was perfect, thereby making no changes to the Q-value.\n",
634 | "\n",
635 | "$\n",
636 | "\\begin{equation}\n",
637 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[{\\color{red}{r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})}}\\big]\n",
638 | "\\end{equation}\n",
639 | "$\n",
640 | "\n",
641 | "The parameter $\\gamma$ is connected to how the agent values immediate rewards compared to later rewards, however explanation of this is outside the scope of this notebook. \n",
642 | "\n",
643 | "### Task:\n",
644 | "Implement the part of the equation mark red.\n",
645 | "- Use the method we created in the previous section to find the highest_q_value_in_state\n",
646 | "- Use brackets notation to select the correct cell i.e. Q_table[Desired_state, desired_action]"
647 | ]
648 | },
649 | {
650 | "cell_type": "code",
651 | "execution_count": null,
652 | "metadata": {},
653 | "outputs": [],
654 | "source": [
655 | "def calculate_error_in_Q_value(Q_table, state, action, reward, new_state):\n",
656 | " \n",
657 | " discount_factor = 0.95\n",
658 | " \n",
659 | " \"Write code below\"\n",
660 | " highest_q_value_in_state = find_highest_Q_value_in_state(Q_table, new_state)\n",
661 | " \n",
662 | " error_in_Q_value = reward + discount_factor * highest_q_value_in_state - Q_table[state,action]\n",
663 | " \n",
664 | " \"Write code above\"\n",
665 | " \n",
666 | " return error_in_Q_value"
667 | ]
668 | },
669 | {
670 | "cell_type": "code",
671 | "execution_count": null,
672 | "metadata": {},
673 | "outputs": [],
674 | "source": [
675 | "# Do not edit - Assertion cell #\n",
676 | "if mock.do_assertion():\n",
677 | " assert(calculate_error_in_Q_value(mock.get_Q(), 0, 0, 10, 1) == 9.5), \\\n",
678 | " \"The method did not calculate the correct error value for the test sample\"\n",
679 | " print(\"The error in Q-value have been calculated correctly!\")"
680 | ]
681 | },
682 | {
683 | "cell_type": "markdown",
684 | "metadata": {},
685 | "source": [
686 | "## Update Q-table\n",
687 | "\n",
688 | "We are almost ready to complete our Q-learning algorithm. \n",
689 | "\n",
690 | "The only element not yet explained is the parameter $\\alpha$. $\\alpha$ is known as the learning rate and controls how quick we update the Q-table. For example, a learning rate of 0.1 indicates that we will only use 10% of the actual error calculated when updating the Q-table. This is because we calculate the Q-value by using the highest Q-value of the next state, which is just our best estimate so far and might be erroneous. If that state has not been visited yet, it would simply have the initialized value of the table, i.e. 0. By using the learning rate, we account for this uncertainty and get a less oscillating system which is proven to give better results. \n",
691 | "\n",
692 | "$\n",
693 | "\\begin{equation}\n",
694 | "Q(s_{t},a_{t})^{new} \\leftarrow {\\color{red}{Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]}}\n",
695 | "\\end{equation}\n",
696 | "$\n",
697 | "\n",
698 | "To implement the algorithm there is one factor which needs to be accounted for. If the episode has ended due to a terminal state, the last reward received will be the Q-value of that state, since there is no possible next state. As a result, if the done parameter is true (an input to the method below), we set the Q-value of that state-action pair equal to the reward.\n",
699 | "\n",
700 | "### Task\n",
701 | "Create a method which either returns\n",
702 | " - The Q-table with the Q-value of that state-action pair equal to the reward, IF it was the final episode. (Done is true)\n",
703 | " - ELSE the Q-table updated with the Q-learning algorithm, using the methods we have created."
704 | ]
705 | },
706 | {
707 | "cell_type": "code",
708 | "execution_count": null,
709 | "metadata": {},
710 | "outputs": [],
711 | "source": [
712 | "def update_q_table(Q_table, state, action, done, reward, new_state):\n",
713 | " \n",
714 | " learning_rate = 0.8\n",
715 | " \n",
716 | " \"Write code below\"\n",
717 | " if(done):\n",
718 | " Q_table[state, action] = reward\n",
719 | " else:\n",
720 | " Q_table[state, action] = Q_table[state, action] + \\\n",
721 | " learning_rate * calculate_error_in_Q_value(Q_table, state, action, reward, new_state)\n",
722 | " \"Write code above\"\n",
723 | " \n",
724 | " return Q_table"
725 | ]
726 | },
727 | {
728 | "cell_type": "code",
729 | "execution_count": null,
730 | "metadata": {},
731 | "outputs": [],
732 | "source": [
733 | "# Do not edit - Assertion cell #\n",
734 | "if mock.do_assertion():\n",
735 | " assert(update_q_table(mock.get_Q(), 2, 1, 1, 10, 3)[2,1] == 10), \\\n",
736 | " \"The updated Q-value when simulation was done was incorrect\"\n",
737 | " assert(update_q_table(mock.get_Q(), 0, 0, 0, 10, 1)[0,0] == 17.6), \\\n",
738 | " \"The updated Q-value when the episode was not done, was incorrect\"\n",
739 | " print(\"Q-tabel have been correctly updated!\")"
740 | ]
741 | },
742 | {
743 | "cell_type": "markdown",
744 | "metadata": {},
745 | "source": [
746 | "## Take one step\n",
747 | "\n",
748 | "Now we are ready to create a method which makes our agent make a move in the environment.\n",
749 | "\n",
750 | "### Task\n",
751 | "Create a method which selects the best action, passes this to our environment, and returns the action, reward, done new_state variables.\n",
752 | "- To choose an action use the select_action method we created earlier\n",
753 | "- To make a step in the environment, use this method env.step(action)\n",
754 | "- env.step(action) return the following values accordingly: new_state, reward, done, state_transition_probability\n",
755 | "- Ignore the state_transition_probability, as this is not used in this environment\n",
756 | " "
757 | ]
758 | },
759 | {
760 | "cell_type": "code",
761 | "execution_count": null,
762 | "metadata": {},
763 | "outputs": [],
764 | "source": [
765 | "def step(Q_table, env, state, epsilon):\n",
766 | " \n",
767 | " \"Write code below\"\n",
768 | " action = select_action(Q_table, state, epsilon)\n",
769 | " new_state, reward, done, _ = env.step(action)\n",
770 | " \n",
771 | " \"Write code above\"\n",
772 | " \n",
773 | " return action, reward, done, new_state"
774 | ]
775 | },
776 | {
777 | "cell_type": "code",
778 | "execution_count": null,
779 | "metadata": {},
780 | "outputs": [],
781 | "source": [
782 | "# Do not edit - Assertion cell #\n",
783 | "if mock.do_assertion():\n",
784 | " mock_env1 = deepcopy(mock.get_env())\n",
785 | " mock_env1.reset()\n",
786 | " mock_env2 = deepcopy(mock_env1)\n",
787 | "\n",
788 | " new_state, _, _, _ = mock_env1.step(0)\n",
789 | " assert(step(mock.get_Q(), mock_env2, 0, 0) == (0, -1, False, new_state)), \\\n",
790 | " \"Your environment did not return the correct values\"\n",
791 | " print(\"Your step method seems to function properly, good job!\")"
792 | ]
793 | },
794 | {
795 | "cell_type": "markdown",
796 | "metadata": {},
797 | "source": [
798 | "## Create train method\n",
799 | "\n",
800 | "Finally, we can create the method which will train our agent, i.e. update the Q-table.\n",
801 | "\n",
802 | "### Task\n",
803 | "In this method, you will enter your code at multiple places, but always between \"Write code below\" and \"Write code above\". \n",
804 | "- You will only input methods you have already created. \n",
805 | "- Remember to store the value in the right variable, i.e. the Q-table is stored in the variable Q_table\n",
806 | "- To see which variables the methods returns, look at your implementation in the previous cells.\n",
807 | "- PS. The following cells do not have assertion cells."
808 | ]
809 | },
810 | {
811 | "cell_type": "code",
812 | "execution_count": null,
813 | "metadata": {},
814 | "outputs": [],
815 | "source": [
816 | "def train(env, num_episodes, max_iterations):\n",
817 | " done = False\n",
818 | " epsilon = 1\n",
819 | " reward_list, iterations_list, epsilon_list = [], [], []\n",
820 | " \n",
821 | " \"Write code below\"\n",
822 | " # Create Q-table\n",
823 | " Q_table = create_q_table(env)\n",
824 | " \"Write code above\"\n",
825 | "\n",
826 | " for episode in range(num_episodes):\n",
827 | "\n",
828 | " if (done and episode % 10 == 0): \n",
829 | " print(\"Episode: {} | Iterations: {} | Total Reward: {} | Epsilon: {}\"\\\n",
830 | " .format(episode, iterations, total_reward, epsilon))\n",
831 | " \n",
832 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
833 | " \n",
834 | " \"Write code below\"\n",
835 | " # Update epsilon\n",
836 | " epsilon = update_epsilon(epsilon)\n",
837 | " \n",
838 | " \"Write code above\"\n",
839 | " \n",
840 | " while not done:\n",
841 | " \n",
842 | " \"Write code below\"\n",
843 | " \n",
844 | " # Take step\n",
845 | " action, reward, done, new_state = step(Q_table, env, state, epsilon)\n",
846 | " \n",
847 | " # Update Q-table\n",
848 | " Q_table = update_q_table(Q_table, state, action, done, reward, new_state)\n",
849 | " \n",
850 | " \"Write code above\"\n",
851 | " \n",
852 | " # Set the new state as the current state\n",
853 | " state = new_state\n",
854 | " \n",
855 | " # Update episode information\n",
856 | " iterations += 1\n",
857 | " total_reward += reward\n",
858 | " \n",
859 | " # End episode if it has lasted for too many iterations\n",
860 | " if (iterations >= max_iterations): done = True\n",
861 | "\n",
862 | " reward_list.append(total_reward)\n",
863 | " iterations_list.append(iterations)\n",
864 | " epsilon_list.append(epsilon)\n",
865 | " \n",
866 | " return Q_table, (reward_list, iterations_list, epsilon_list)"
867 | ]
868 | },
869 | {
870 | "cell_type": "markdown",
871 | "metadata": {},
872 | "source": [
873 | "## Create play method\n",
874 | "\n",
875 | "This method can be used if you want to test the performance of your agent, by only running one episode."
876 | ]
877 | },
878 | {
879 | "cell_type": "code",
880 | "execution_count": null,
881 | "metadata": {},
882 | "outputs": [],
883 | "source": [
884 | "def play(env, Q_table):\n",
885 | " \n",
886 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
887 | "\n",
888 | " while not done:\n",
889 | "\n",
890 | " # Take step\n",
891 | " action, reward, done, new_state = step(Q_table, env, state, 0)\n",
892 | "\n",
893 | " # Set the new state as the current state\n",
894 | " state = new_state\n",
895 | "\n",
896 | " # Update episode information\n",
897 | " iterations += 1\n",
898 | " total_reward += reward\n",
899 | "\n",
900 | " if (iterations >= 100): done = True\n",
901 | " \n",
902 | " print(\"Iterations: {} | Total Reward: {}\".format(iterations, total_reward))"
903 | ]
904 | },
905 | {
906 | "cell_type": "markdown",
907 | "metadata": {
908 | "collapsed": true
909 | },
910 | "source": [
911 | "## Time to put our methods to use!\n",
912 | "\n",
913 | "Congratulation, you have created every method need to train your Q-learning agent!\n",
914 | "\n",
915 | "### Task\n",
916 | "To train your agent, simply set a value for the maximum_episodes and maximum_iterations\n",
917 | "- Tip 1: Maximum_episodes should be less than 2000\n",
918 | "- Tip 2: Maximum iterations should be less than 200"
919 | ]
920 | },
921 | {
922 | "cell_type": "code",
923 | "execution_count": null,
924 | "metadata": {},
925 | "outputs": [],
926 | "source": [
927 | "maximum_episodes = 0\n",
928 | "maximum_iterations = 0\n",
929 | "\n",
930 | "Q_table_trained, data = train(env, maximum_episodes, maximum_iterations)"
931 | ]
932 | },
933 | {
934 | "cell_type": "markdown",
935 | "metadata": {},
936 | "source": [
937 | "## Plot key variables \n",
938 | "\n",
939 | "This method plots different parameters during training. As a result, the training cell above must be run before this one.\n",
940 | "\n",
941 | "### Task\n",
942 | "- Simply run the cell below to create the plots"
943 | ]
944 | },
945 | {
946 | "cell_type": "code",
947 | "execution_count": null,
948 | "metadata": {},
949 | "outputs": [],
950 | "source": [
951 | "try: \n",
952 | " assert(data)\n",
953 | " plot_visualization(data)\n",
954 | "except:\n",
955 | " print(\"You need to run the **train** cell before running this cell, as it uses the data from the training!\")"
956 | ]
957 | },
958 | {
959 | "cell_type": "markdown",
960 | "metadata": {},
961 | "source": [
962 | "## Play around for one round\n",
963 | "\n",
964 | "This method runs one episode, using your trained Q-table. This is a quick way to test the performance.\n",
965 | "\n",
966 | "### Task\n",
967 | "- Run the cell below to quickly see the performance of your agent. This method does not alter your Q_table, so you can run it as many times as you like!"
968 | ]
969 | },
970 | {
971 | "cell_type": "code",
972 | "execution_count": null,
973 | "metadata": {},
974 | "outputs": [],
975 | "source": [
976 | "play(env, Q_table_trained)"
977 | ]
978 | },
979 | {
980 | "cell_type": "markdown",
981 | "metadata": {},
982 | "source": [
983 | "## Visualize an episode\n",
984 | "\n",
985 | "### Task\n",
986 | "This method works similarly to the one above, but visualizes the environment.\n",
987 | "- If you would like to end the simulation, click on the Kernal tab in the toolbar, and click \"Interrupt\"."
988 | ]
989 | },
990 | {
991 | "cell_type": "code",
992 | "execution_count": null,
993 | "metadata": {},
994 | "outputs": [],
995 | "source": [
996 | "render_performance(env, Q_table_trained)"
997 | ]
998 | },
999 | {
1000 | "cell_type": "markdown",
1001 | "metadata": {},
1002 | "source": [
1003 | "## And we are done, almost!\n",
1004 | "Now you have everything you need to play around with the environment and alter different parameters to see how it impacts our agent. Can you improve the parameters already chosen? Suggestions:\n",
1005 | "- Increase the number of episodes and reduce iterations\n",
1006 | "- Change epsilon decay to change the Epsilon-Greedy Strategy\n",
1007 | "- Change the learning rate to make smaller or larger corrections to the Q-table per update\n",
1008 | "- Change the discount factor $\\gamma$\n",
1009 | "\n",
1010 | "If you would like to create your own cells, do the following:\n",
1011 | "1. Click on an existing cell\n",
1012 | "2. Click the Esc - button\n",
1013 | "3. Press \"b\" to add a cell below, press \"a\" the add a cell above, or double tap \"d\" to delete a cell\n",
1014 | "\n",
1015 | "### Task\n",
1016 | "Run the cell below to turn off the assertion cells. \n",
1017 | "Have fun!"
1018 | ]
1019 | },
1020 | {
1021 | "cell_type": "code",
1022 | "execution_count": null,
1023 | "metadata": {},
1024 | "outputs": [],
1025 | "source": [
1026 | "mock.turn_off_assertion()"
1027 | ]
1028 | },
1029 | {
1030 | "cell_type": "markdown",
1031 | "metadata": {},
1032 | "source": [
1033 | "## Want to try a different environment?\n",
1034 | "If you want to try a different environment suitable for Q-learning with tables, you can try this one:\n",
1035 | "- FrozenLake-v0\n",
1036 | "- You can read more about it [here](https://gym.openai.com/envs/FrozenLake-v0/)\n",
1037 | "- Tip: This environment may require the agent to do a more thorough exploration.\n",
1038 | "\n",
1039 | "To use it, simply replace it with \"Taxi-v2\" in the cell where we created the env-variabel. \n",
1040 | "Tip: Remember to rerun the cells."
1041 | ]
1042 | }
1043 | ],
1044 | "metadata": {
1045 | "kernelspec": {
1046 | "display_name": "Python 3",
1047 | "language": "python",
1048 | "name": "python3"
1049 | },
1050 | "language_info": {
1051 | "codemirror_mode": {
1052 | "name": "ipython",
1053 | "version": 3
1054 | },
1055 | "file_extension": ".py",
1056 | "mimetype": "text/x-python",
1057 | "name": "python",
1058 | "nbconvert_exporter": "python",
1059 | "pygments_lexer": "ipython3",
1060 | "version": "3.7.4"
1061 | }
1062 | },
1063 | "nbformat": 4,
1064 | "nbformat_minor": 2
1065 | }
1066 |
--------------------------------------------------------------------------------
/Workshop/Workshop_DQN.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Deep Q-learning\n",
8 | "##### Authors: Eirik Fagtun Kjærnli and Fabian Dietrichson [Accenture]"
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "Deep Reinforcement learning has resulted in superman performance in a range of scenarios, such as Chess, Atari 2600, and Starcraft, but also in robotics. The main reason for this success compared to traditional Reinforcement Learning is the introduction of deep neural networks. The first successful implementation of deep neural networks in combination with a Reinforcement learning algorithm was Mnih et. al. which was able to reach superhuman performance in the classic Atari games. Their approach was coined Deep Q-Networks(DQN), and uses the Q-learning algorithm we used in the previous notebook, in combination with deep neural networks.\n",
16 | "\n",
17 | "In the DQN approach, a neural network replaces the Q-table which we used in the previous notebook. The main reason is the neural networks ability to generalize around similar states. In a continuous environment, a Q-table would quickly grow to tens of millions of unique states, although many of these are so similar that the same action should be applied. This is both an issue due to the huge table we would need, but also because we would need a tremendous amount of training data to update all the states. \n",
18 | "\n",
19 | "Neural networks address this issue, but come with their own challenges. Neural network struggles with instabilities during training, in addition to critical forgetting and divergence. Mnih's success was due to several ingenious tricks which stabilized the network during training, and we will go through these during the workshop.\n",
20 | "\n",
21 | "In the previous notebook, we created a lot of methods, which resulted in many parameters being passed from method to method. This can quickly become convoluted, and in this notebook we instead use classes. An example can be seen below. When the class is created the init method is run, and the different parameters are set and stored in this instance of the class. The parameters can be called within the class using \"self.\" syntax, e.g. self.learning_rate."
22 | ]
23 | },
24 | {
25 | "cell_type": "markdown",
26 | "metadata": {},
27 | "source": [
28 | "## Import packages and create support methods for workshop\n",
29 | "Before you can go on, the cell below must be run. These are methods used to verify your work, in addition to the support function that will be used throughout the notebook. \n",
30 | "\n",
31 | "### Task 1\n",
32 | "Import the packages needed for this workshop, simply mark the cell below and press CTRL + Enter\n",
33 | "\n",
34 | "### Task 2\n",
35 | "Create the necessary support methods by running the second cell below. Mark it and press CTRL + Enter"
36 | ]
37 | },
38 | {
39 | "cell_type": "code",
40 | "execution_count": null,
41 | "metadata": {},
42 | "outputs": [],
43 | "source": [
44 | "import warnings\n",
45 | "import logging\n",
46 | "import os\n",
47 | "import datetime\n",
48 | "import gym\n",
49 | "import numpy as np\n",
50 | "import random\n",
51 | "from copy import deepcopy\n",
52 | "from statistics import mean\n",
53 | "from IPython.display import HTML\n",
54 | "from collections import deque, namedtuple\n",
55 | "\n",
56 | "from tensorflow.python.keras import Sequential\n",
57 | "from tensorflow.python.keras.layers import Dense\n",
58 | "from tensorflow.python.keras.metrics import MSE\n",
59 | "from tensorflow.python.keras.optimizers import Adam\n",
60 | "from tensorflow.python.keras.initializers import glorot_uniform\n",
61 | "\n",
62 | "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\")) == 0:\n",
63 | " !bash ../xvfb start\n",
64 | " %env DISPLAY = : 1\n",
65 | " \n",
66 | "if not os.path.exists(\"videos\"):\n",
67 | " os.makedirs(\"videos\")\n",
68 | "\n",
69 | "warnings.filterwarnings(\"ignore\")\n",
70 | "logging.getLogger('tensorflow').disabled = True\n",
71 | "%precision 3\n",
72 | "should_assert = True"
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": null,
78 | "metadata": {},
79 | "outputs": [],
80 | "source": [
81 | "def generate_blank_weights(layer_dims):\n",
82 | " \n",
83 | " # Xavier/Glorot Initialization\n",
84 | " new_weights = np.random.randn(layer_dims[0], layer_dims[1])*np.sqrt(1/layer_dims[0])\n",
85 | " new_bias = np.zeros((layer_dims[1]))\n",
86 | " \n",
87 | " return [new_weights, new_bias]\n",
88 | "\n",
89 | "def create_dummybuffer():\n",
90 | " mock_env = gym.make(\"CartPole-v1\")\n",
91 | " parameters = {\"buffer_size\": 6,\n",
92 | " \"batch_size\": 3}\n",
93 | " DummyBuffer = ExperienceReplay(mock_env, parameters)\n",
94 | " \n",
95 | " return DummyBuffer\n",
96 | "\n",
97 | "def get_dummy_parameters_and_env():\n",
98 | " \n",
99 | " parameters = {\n",
100 | " \"tau\" : 0.4,\n",
101 | " \"gamma\" : 1,\n",
102 | " \"epsilon_init\" : 1,\n",
103 | " \"epsilon_decay\" : 1,\n",
104 | " \"epsilon_minimum\": 0.01,\n",
105 | " \"buffer_size\" : 2000,\n",
106 | " \"batch_size\" : 64,\n",
107 | " \"epochs\": 1,\n",
108 | " \"loss_metric\" : \"mse\",\n",
109 | " \"learning_rate\" : 0.01,\n",
110 | " \"learning_rate_decay\": 0.01,\n",
111 | " \"hidden_layer_1\": 5,\n",
112 | " \"hidden_layer_2\": 5}\n",
113 | " \n",
114 | " dummy_env = gym.make(\"CartPole-v0\")\n",
115 | " return parameters, dummy_env \n",
116 | " \n",
117 | "def clear_video_folder():\n",
118 | " video_path = \"videos\"\n",
119 | " for item in os.listdir(video_path):\n",
120 | " os.remove(os.path.join(video_path, item))\n",
121 | "\n",
122 | "def play(agent):\n",
123 | " \n",
124 | " old_epsilon = agent.epsilon\n",
125 | " \n",
126 | " done = False\n",
127 | " agent.epsilon = 0\n",
128 | " total_reward = 0\n",
129 | " state = agent.env.reset()\n",
130 | " \n",
131 | " while not done:\n",
132 | " action, reward, done, new_state = agent.step(state)\n",
133 | " state = new_state\n",
134 | " \n",
135 | " total_reward += reward\n",
136 | " print(\"Total Reward: {}\".format(total_reward)) \n",
137 | " \n",
138 | " agent.epsilon = old_epsilon\n",
139 | " \n",
140 | "def generate_video(agent):\n",
141 | " \n",
142 | " if os.listdir(\"./videos/\"):\n",
143 | " clear_video_folder()\n",
144 | "\n",
145 | " monitor = gym.wrappers.Monitor(agent.env, directory=\"videos\", force=True)\n",
146 | " original_env = agent.env\n",
147 | " agent.env = monitor\n",
148 | " \n",
149 | " old_epsilon = agent.epsilon\n",
150 | " \n",
151 | " for _ in range(3):\n",
152 | " total_reward = 0\n",
153 | " done = False\n",
154 | " agent.epsilon = 0\n",
155 | " total_reward = 0\n",
156 | " state = agent.env.reset()\n",
157 | " while not done:\n",
158 | " action, reward, done, new_state = agent.step(state)\n",
159 | " state = new_state\n",
160 | "\n",
161 | " total_reward += reward\n",
162 | " \n",
163 | " print(\"Total Reward: {}\".format(total_reward)) \n",
164 | " agent.epsilon = old_epsilon\n",
165 | " agent.env = original_env\n",
166 | " monitor.close()\n",
167 | " "
168 | ]
169 | },
170 | {
171 | "cell_type": "markdown",
172 | "metadata": {},
173 | "source": [
174 | "## Create Q-Networks\n",
175 | "There are several frameworks which can be used to create neural networks such as Tensorflow from Google, PyTorch from Facebook or MXNet from Apache. In this notebook, we will be using Tensorflow, with a high-level API called Keras on top. Keras greatly simplifies the effort needed to create and train a neural network, and it is often referred to as the best Deep Learning framework for those who are just starting with neural networks.\n",
176 | "\n",
177 | "Neural networks are partially inspired by the structure of the human brain and consist of a network of interconnected neurons. The neural network consists of an input layer, a set of hidden layers, and an output layer, as shown in the figure below.\n",
178 | "\n",
179 | "\n",
180 | "\n",
181 | "\n",
182 | "\n",
183 | "### Task\n",
184 | "Now we will experience how easy it is to create a neural network using Keras. We are going to create a neural network with an input layer, two hidden layers, and one output layer. We have completed steps 1,2, 5, and your task is to complete task 3 and 4 according to the design criterions below: \n",
185 | "\n",
186 | "1. The model should be fully connected \n",
187 | " \n",
188 | " \n",
189 | "2. The input layer and first hidden layer is created together, and should:\n",
190 | " - Be of type Dense\n",
191 | " - Use the ReLU activation function\n",
192 | " - Have input size equal to the observations size of the environment\n",
193 | " - Have hidden layers size equal to parameter hidden_layer_1 \n",
194 | " \n",
195 | " \n",
196 | "3. The second hidden layer should:\n",
197 | " - Be of type Dense\n",
198 | " - Use the ReLU activation function \n",
199 | " - The hidden layer should have size equal to parameter hidden_layer_2\n",
200 | " - Tip1: This layer is similar to previous, but without the input_dim parameter\n",
201 | " - Tip2: Remember to add the dense layer using model.add()\n",
202 | " \n",
203 | " \n",
204 | "4. The output layer should:\n",
205 | " - Be of type Dense\n",
206 | " - The activation function should be linear\n",
207 | " - Should have size equal to action vector, i.e. action_size\n",
208 | " \n",
209 | " \n",
210 | "5. The final part is to compile the network. Before we do this we must define som parameters\n",
211 | " - Loss metric should be Mean-squared-error (MSE)\n",
212 | " - The optimizer should be Adaptive Moment Estimation (Adam)\n",
213 | " - Learning rate should equal to learning_rate\n",
214 | " - Learning rate decay should be equal to learning_rate_decay\n",
215 | " \n",
216 | "\n",
217 | "**Ps!** \n",
218 | "At the end of the workshop, you can experiment with different structures and parameters, however, in the first walkthrough, you will follow the defined steps."
219 | ]
220 | },
221 | {
222 | "cell_type": "code",
223 | "execution_count": null,
224 | "metadata": {},
225 | "outputs": [],
226 | "source": [
227 | "class QNetwork:\n",
228 | "\n",
229 | " def __init__(self, env, parameters):\n",
230 | " self.observations_size = env.observation_space.shape[0]\n",
231 | " self.action_size = env.action_space.n\n",
232 | " self.learning_rate = parameters[\"learning_rate\"]\n",
233 | " self.learning_rate_decay = parameters[\"learning_rate_decay\"]\n",
234 | " self.loss_metric = parameters[\"loss_metric\"]\n",
235 | " self.hidden_layer_1 = parameters[\"hidden_layer_1\"]\n",
236 | " self.hidden_layer_2 = parameters[\"hidden_layer_2\"] \n",
237 | " \n",
238 | " def build_q_network(self):\n",
239 | " \n",
240 | " model = Sequential()\n",
241 | " model.add(Dense(self.hidden_layer_1, input_dim=self.observations_size, activation='relu'))\n",
242 | " \n",
243 | " \"Input code below\"\n",
244 | " \n",
245 | " \"Input code above\"\n",
246 | " \n",
247 | " model.compile(loss=self.loss_metric, optimizer=Adam(lr=self.learning_rate, decay=self.learning_rate_decay))\n",
248 | " \n",
249 | " return model\n",
250 | " "
251 | ]
252 | },
253 | {
254 | "cell_type": "code",
255 | "execution_count": null,
256 | "metadata": {},
257 | "outputs": [],
258 | "source": [
259 | "# Do not edit - Assertion cell #\n",
260 | "if(should_assert):\n",
261 | " mock_parameters = {\"loss_metric\" : \"mse\",\n",
262 | " \"learning_rate\" : 0.01,\n",
263 | " \"learning_rate_decay\": 0.01,\n",
264 | " \"hidden_layer_1\": 24,\n",
265 | " \"hidden_layer_2\": 24}\n",
266 | "\n",
267 | " mock_env = gym.make(\"CartPole-v0\")\n",
268 | " mock_q_network = QNetwork(mock_env, mock_parameters)\n",
269 | " mock_model = mock_q_network.build_q_network()\n",
270 | " config_network = mock_model.get_config()\n",
271 | " assert(config_network.get(\"name\")[0:10] == \"sequential\")\n",
272 | "\n",
273 | " # Layers\n",
274 | " config_layers = config_network.get(\"layers\")\n",
275 | " assert(len(config_layers) == 3), \"You should only have 2 dense layers\"\n",
276 | "\n",
277 | " # First layer\n",
278 | " assert(config_layers[0].get(\"class_name\") == \"Dense\"),\"Incorrect layertype in first layer\"\n",
279 | " assert(config_layers[0].get(\"config\").get(\"batch_input_shape\") == (None, mock_q_network.observations_size))\n",
280 | " assert(config_layers[0].get(\"config\").get(\"units\") == 24), \"Incorrect number of neurons in first layer\"\n",
281 | " assert(config_layers[0].get(\"config\").get(\"activation\") == \"relu\"), \\\n",
282 | " \"Activation function for first layer should be relu\"\n",
283 | "\n",
284 | " # Second layer\n",
285 | " assert(config_layers[1].get(\"class_name\") == \"Dense\"),\"Incorrect layertype in first layer\"\n",
286 | " assert(config_layers[1].get(\"config\").get(\"units\") == 24), \"Incorrect number of neurons in first layer\"\n",
287 | " assert(config_layers[1].get(\"config\").get(\"activation\") == \"relu\"), \\\n",
288 | " \"Activation function for second layer should be relu\"\n",
289 | "\n",
290 | " # Thrid layer\n",
291 | " assert(config_layers[2].get(\"class_name\") == \"Dense\"),\"Incorrect layertype in first layer\"\n",
292 | " assert(config_layers[2].get(\"config\").get(\"units\") == mock_q_network.action_size), \\\n",
293 | " \"Incorrect number of neurons in first layer\"\n",
294 | " assert(config_layers[2].get(\"config\").get(\"activation\") == \"linear\"),\\\n",
295 | " \"Activation function for third layer should be linear\"\n",
296 | "\n",
297 | " config_optimizer = mock_model.optimizer.get_config()\n",
298 | " assert(0.0099 < config_optimizer.get(\"lr\") <= 0.01),\"Learning rate should be 0.01\"\n",
299 | " assert(0.0099 < config_optimizer.get(\"decay\") <= 0.01),\"Learning rate decay should be 0.01\"\n",
300 | " assert(mock_model.loss == \"mse\"), \"Loss metric should me mse\"\n",
301 | " \n",
302 | " print(\"Superb, you implemented the layers correct! Information about your model is shown below.\\n\")\n",
303 | " print(\"PS. The input layer is not shown, altough you should be able to calculate it by looking at \")\n",
304 | " print(\" of the first hidden layer.\\n\")\n",
305 | " mock_model.summary()\n",
306 | " "
307 | ]
308 | },
309 | {
310 | "cell_type": "markdown",
311 | "metadata": {},
312 | "source": [
313 | "## Create an Experience Buffer\n",
314 | "\n",
315 | "For a neural network to perform optimally we want the data to be I.I.D (Independent and Identically Distributed). In supervised learning, where for instance the network is fed an image and it will predict either cat or dog, this is achieved by randomly sampling the training data from the full data set. As a result:\n",
316 | "- Batches have close-to similar data distribution.\n",
317 | "- Samples in each batch are independent of each other.\n",
318 | "\n",
319 | "In Reinforcement learning where our agent samples data by moving from state to state, the recently sampled data will be highly correlated to each other. As a result, we will feed our network data which closely resembles each other, which is a recipe for disaster when working with a neural network. Furthermore, the data distribution of the initial states will be different from that of the later stage, i.e. this does not satisfy the I.I.D criterion. This is bad news!\n",
320 | "\n",
321 | "Luckily Mnih et. al. has a solution. Instead of training on the just the previously collected samples, we store all states in an **Experience Buffer** and sample randomly from this buffer.\n"
322 | ]
323 | },
324 | {
325 | "cell_type": "markdown",
326 | "metadata": {},
327 | "source": [
328 | "## Tasks\n",
329 | "\n",
330 | "This section is divided into multiple tasks. You will first complete task 1, and then run the first assertion cell for task 1. Then you will continue to task 2 and run assertion cell 2 etc.\n",
331 | "\n",
332 | "### Task 1 - Add experience\n",
333 | "\n",
334 | "The first method we are going to create is a method which adds an experience, i.e. a state-transition. This is stored as a tuple which contains the following fields; state, action, reward, done, new_state.\n",
335 | "\n",
336 | "To store a state-transition we are going to use a specific type of tuple, namely a namedtuple. You can read more about it [here](https://docs.python.org/2/library/collections.html#collections.namedtuple). We have created a namedtuple which you are going to use called experience_tuple. The experience_tuple have the following field where you can store information; state, action, reward, done, new_state. To create a state-transition do the following:\n",
337 | "- self.experience_tuple(state=your_state, action=your_action, reward=your_reward, done=your_done, new_state=your_new_state)\n",
338 | "\n",
339 | "To add a named_tuple to the buffer use the append method of the experience_buffer:\n",
340 | "- self.experience_buffer.append(your_experience_tuple)\n",
341 | "\n",
342 | "\n",
343 | "### Task 2 - Get batch\n",
344 | "\n",
345 | "The next method we are going to create is a method which returns a batch of experience_tuples, which is used during training. The methods should set the variable *experiences* equal to either\n",
346 | "- All the samples in the experience_buffer IF it is less than the batch size\n",
347 | "- or ELSE a *batch_size* of randomly sample experience_tuples from the experience_buffer. To randomly sample use the random.sample() method. You can read more about the method [here](https://docs.python.org/2/library/random.html).\n",
348 | "\n",
349 | "The last part of the method is created by us, and it creates vectors of all elements of each field. This is done to vectorize the training and gives a significant reduction in training time.\n",
350 | "\n",
351 | "### Task 3 - Warm-up\n",
352 | "\n",
353 | "The Experience Buffer is a collection of all the experiences the agent has collected during its interaction with the environment. As a result, the experience buffer is empty at the begin of the first episode. This is not an issue, as you in the previous task implemented the necessary logic to handle the case when there are fewer experience_touples in the buffer than the batch size. However, this does slow down training as we train with few and highly correlated samples. \n",
354 | "\n",
355 | "To counteract this issue, we introduce a method called warm_up. The warm_up method performs completes a certain amount of episodes taking random actions, thereby creating reducing the correlation between samples in the initial training phase.\n",
356 | "\n",
357 | "We have created the high-level structure of this method, but you will create the code which completes one episode. The method should do the following\n",
358 | "1. Use a While-loop which runs as long as the Done parameter is \"not true\"\n",
359 | "2. Chooses a random action - Use env.action_space.sample()\n",
360 | "3. Use the random action to make a step in the environment - Use env.step(random_action).\n",
361 | " - env.step() returns the following variables accordingly; new_state, reward, done, state_transition_probability\n",
362 | " - We will not use the state_transition_probability variable\n",
363 | "4. Add the experience to the replay_buffer - Use the add_experience method we created earlier\n",
364 | "5. Remember to update the current state, to the new_state, e.g. state = new_state\n",
365 | "\n",
366 | "Remember to test your method by running the assertion cell for task 3."
367 | ]
368 | },
369 | {
370 | "cell_type": "code",
371 | "execution_count": null,
372 | "metadata": {},
373 | "outputs": [],
374 | "source": [
375 | "class ExperienceReplay:\n",
376 | " \n",
377 | " def __init__(self, env, parameters):\n",
378 | " self.env = env\n",
379 | " self.buffer_size = parameters[\"buffer_size\"]\n",
380 | " self.batch_size = parameters[\"batch_size\"]\n",
381 | " self.experience_buffer = deque(maxlen=self.buffer_size)\n",
382 | " \n",
383 | " self.experience_tuple = namedtuple(\"Experience\", \n",
384 | " field_names=[\"state\", \"action\", \"reward\", \"done\", \"new_state\"])\n",
385 | " \n",
386 | " def add_experience(self, state, action, reward, done, new_state):\n",
387 | " \n",
388 | " \"Input code below\"\n",
389 | " \n",
390 | " \"Input code above\"\n",
391 | " \n",
392 | " def get_batch(self):\n",
393 | " \n",
394 | " \"Input code below\"\n",
395 | " \n",
396 | " \"Input code above\"\n",
397 | " \n",
398 | " states = np.vstack([e.state for e in experiences if e is not None])\n",
399 | " actions = np.vstack([e.action for e in experiences if e is not None])\n",
400 | " rewards = np.vstack([e.reward for e in experiences if e is not None])\n",
401 | " new_states = np.vstack([e.new_state for e in experiences if e is not None])\n",
402 | " dones = np.vstack([e.done for e in experiences if e is not None])\n",
403 | " \n",
404 | " return (states, actions, rewards, dones, new_states)\n",
405 | " \n",
406 | " def warm_up(self):\n",
407 | " for _ in range(10):\n",
408 | " state = self.env.reset()\n",
409 | " done = False\n",
410 | " \n",
411 | " \"Input code below\"\n",
412 | " \n",
413 | " \"Input code above\""
414 | ]
415 | },
416 | {
417 | "cell_type": "markdown",
418 | "metadata": {},
419 | "source": [
420 | "### Assertion - Task 1"
421 | ]
422 | },
423 | {
424 | "cell_type": "code",
425 | "execution_count": null,
426 | "metadata": {
427 | "scrolled": true
428 | },
429 | "outputs": [],
430 | "source": [
431 | "# Do not edit - Assertion cell #\n",
432 | "if should_assert:\n",
433 | " DummyBuffer = create_dummybuffer()\n",
434 | "\n",
435 | " for i in range(6):\n",
436 | " DummyBuffer.add_experience(i, i%2, i, i%2, i+1)\n",
437 | "\n",
438 | " dummy_experience_buffer = DummyBuffer.experience_buffer.copy()\n",
439 | " assert(len(dummy_experience_buffer) == 6),\\\n",
440 | " \"Length of your experience buffer is wrong, should be 6, was {}\".format(len(dummy_experience_buffer))\n",
441 | "\n",
442 | " for e in range(6):\n",
443 | " dummy_experience = DummyBuffer.experience_tuple(state=e, action=e%2, reward=e, done=e%2, new_state=e+1)\n",
444 | " stored_experience = dummy_experience_buffer.popleft()\n",
445 | " assert(dummy_experience == stored_experience), \\\n",
446 | " \"The values were incorrectly stored, was {}, should have been {}\".format(stored_experience, dummy_experience)\n",
447 | "\n",
448 | " print(\"You implemented the add_experience method correctly, great job!\")"
449 | ]
450 | },
451 | {
452 | "cell_type": "markdown",
453 | "metadata": {},
454 | "source": [
455 | "### Assertion - Task 2"
456 | ]
457 | },
458 | {
459 | "cell_type": "code",
460 | "execution_count": null,
461 | "metadata": {},
462 | "outputs": [],
463 | "source": [
464 | "# Do not edit - Assertion cell #\n",
465 | "if should_assert:\n",
466 | " DummyBuffer = create_dummybuffer()\n",
467 | " for i in range(2):\n",
468 | " DummyBuffer.add_experience(i, i%2, i, i%2, i+1)\n",
469 | "\n",
470 | " assert(len(DummyBuffer.get_batch()[0]) == 2),\\\n",
471 | " \"Should return 2 samples, when there are less experience_tuples than the batch_size. Your method returned {} samples\"\\\n",
472 | " .format(len(DummyBuffer.get_batch()[0]))\n",
473 | "\n",
474 | " DummyBuffer.add_experience(3, 3%2, 3, 3%2, 3+1)\n",
475 | " dummy_batch = DummyBuffer.get_batch()\n",
476 | " assert(len(dummy_batch[0]) == 3),\\\n",
477 | " \"Should return 3 samples, when there are 3 experience_tuples in the experience_buffer. Your method returned {} samples\"\\\n",
478 | " .format(len(DummyBuffer.get_batch()[0]))\n",
479 | "\n",
480 | "\n",
481 | " DummyBuffer.add_experience(4, 4%2, 4, 4%2, 4+1)\n",
482 | " dummy_batch = DummyBuffer.get_batch()\n",
483 | " assert(len(dummy_batch[0] == 3)),\\\n",
484 | " \"Should return 3 samples, when there are more experience_tuples than the batch_size. Your method returned {} samples\"\\\n",
485 | " .format(len(DummyBuffer.get_batch()[0]))\n",
486 | "\n",
487 | " assert(len(np.unique(dummy_batch[0][0:3])) == 3), \"The method did not return the values correctly, states should be all unique values. You returned {}\".format(dummy_batch[0])\n",
488 | " assert(len(np.unique(dummy_batch[-1][0:3])) == 3), \"The method did not return the values correctly, new_states should be all unique values. You returned {}\".format(dummy_batch[-1])\n",
489 | "\n",
490 | " print(\"Great, you implemented the get_batch() method correctly\")"
491 | ]
492 | },
493 | {
494 | "cell_type": "markdown",
495 | "metadata": {},
496 | "source": [
497 | "### Assertion Task 3"
498 | ]
499 | },
500 | {
501 | "cell_type": "code",
502 | "execution_count": null,
503 | "metadata": {},
504 | "outputs": [],
505 | "source": [
506 | "# Do not edit - Assertion cell #\n",
507 | "if should_assert:\n",
508 | " parameters = {\"buffer_size\": 1000,\n",
509 | " \"batch_size\": 1}\n",
510 | " mock_env1 = gym.make(\"CartPole-v1\")\n",
511 | " mock_experience_replay = ExperienceReplay(mock_env1, parameters)\n",
512 | " mock_env1.reset()\n",
513 | " mock_env2 = deepcopy(mock_env1)\n",
514 | "\n",
515 | " mock_experience_replay.warm_up()\n",
516 | " mock_state = mock_env2.reset()\n",
517 | " while 0 < len(mock_experience_replay.experience_buffer):\n",
518 | " mock_action = mock_env2.action_space.sample()\n",
519 | " mock_new_state, mock_reward, mock_done, _ = mock_env2.step(mock_action)\n",
520 | " \n",
521 | " mock_experience_tuple = mock_experience_replay.experience_buffer.popleft()\n",
522 | " np.testing.assert_array_equal(mock_experience_tuple[0], mock_state, \"State was incorrect\")\n",
523 | " assert(mock_experience_tuple[1] == mock_action), \"Action was incorrect\"\n",
524 | " assert(mock_experience_tuple[2] == mock_reward), \"Reward was incorrect\"\n",
525 | " np.testing.assert_array_equal(mock_experience_tuple[4], mock_new_state, \"New_state was incorrect\")\n",
526 | " \n",
527 | " if mock_done:\n",
528 | " mock_state = mock_env2.reset()\n",
529 | " else:\n",
530 | " mock_state = mock_new_state\n",
531 | " \n",
532 | " print(\"Your warm-up method seems to function properly, good job!\")"
533 | ]
534 | },
535 | {
536 | "cell_type": "markdown",
537 | "metadata": {},
538 | "source": [
539 | "## Create Agent\n",
540 | "\n",
541 | "### Target network\n",
542 | "\n",
543 | "In the supervised learning example, where we predict cat or dog, the labels are stationary, e.g. an image of a cat will always have the label cat. This is not the case in Reinforcement learning.\n",
544 | "\n",
545 | "As we experienced in the previous notebook, the Q-value of the next states is calculated based on the assumption that the Q-table is correct. However, since we begin each training session with a blank table and update the values as we go along, the Q-values of the next state is just an estimate. As a result, we are chasing a moving target, which degrades the training of the neural network and may lead to divergence.\n",
546 | "\n",
547 | "Mnih et. al. solved this by using a target network to choose action and to predict the Q-values of the next state $S_{t+1}$, and used a local network to predict the Q-values of state $S_{t}$. The two networks have an identical structure. To update the target network, the equation below is used, where $\\theta$ represents the weights of each network.\n",
548 | "\n",
549 | "$\n",
550 | "\\theta_{target} = \\tau * \\theta_{local} + (1-\\tau) * \\theta_{target}\n",
551 | "$\n",
552 | "\n",
553 | "### Task 1 - Target network\n",
554 | "Implement the target network algorithm in the \"update_target_network\" method below. We have already created the shell, where we loop through each layer of the network. Your task\n",
555 | "- Update the weights of each layer by using the algorithm above. Remember to use the self.tau parameter.\n",
556 | "- Remember to run the assertion cell\n",
557 | "\n",
558 | "\n",
559 | "### Task 2 - Q-learning algorithm\n",
560 | "Go through the Q-learning algorithm, and compare it to the Q-learning algorithm we used in the previous notebook. Make sure you understand the implementation before moving on.\n"
561 | ]
562 | },
563 | {
564 | "cell_type": "code",
565 | "execution_count": null,
566 | "metadata": {},
567 | "outputs": [],
568 | "source": [
569 | "class Agent: \n",
570 | "\n",
571 | " def __init__(self, env, parameters):\n",
572 | " self.env = env\n",
573 | " self.q_network = QNetwork(env, parameters)\n",
574 | " self.local_network = self.q_network.build_q_network()\n",
575 | " self.target_network = self.q_network.build_q_network()\n",
576 | " self.experience_replay = ExperienceReplay(env, parameters)\n",
577 | " \n",
578 | " self.epsilon = parameters[\"epsilon_init\"]\n",
579 | " self.epsilon_decay = parameters[\"epsilon_decay\"]\n",
580 | " self.epsilon_minimum = parameters[\"epsilon_minimum\"]\n",
581 | " self.tau = parameters[\"tau\"]\n",
582 | " self.gamma = parameters[\"gamma\"]\n",
583 | " self.epochs = parameters[\"epochs\"]\n",
584 | "\n",
585 | " def update_local_network(self):\n",
586 | " states, actions, rewards, dones, next_states = self.experience_replay.get_batch()\n",
587 | " \n",
588 | " # Get Q-values for the next state, Q(next_state), using the target network\n",
589 | " Q_target = self.target_network.predict(next_states)\n",
590 | "\n",
591 | " # Apply Q-learning algorithm and Q-value for next state to calculate the actual Q-value the Q(state)\n",
592 | " Q_calc = rewards + (self.gamma * np.amax(Q_target, axis=1).reshape(-1, 1) * (1 - dones))\n",
593 | " \n",
594 | " # Calculate Q-value Q(state) we predicted earlier using the local network\n",
595 | " Q_local = self.local_network.predict(states)\n",
596 | " \n",
597 | " # Update Q_values with \"correct\" Q-values calculated using the Q-learning algorithm \n",
598 | " for row, col_id in enumerate(actions):\n",
599 | " Q_local[row, np.asscalar(col_id)] = Q_calc[row]\n",
600 | " \n",
601 | " # Train network by minimizing the difference between Q_local and modified Q_local\n",
602 | " self.local_network.fit(states, Q_local, epochs=self.epochs, verbose=0)\n",
603 | " \n",
604 | " def update_target_network(self):\n",
605 | " local_weights = self.local_network.get_weights()\n",
606 | " target_weights = self.target_network.get_weights()\n",
607 | "\n",
608 | " for layer in range(len(local_weights)):\n",
609 | " \n",
610 | " \"Input code below\"\n",
611 | " \n",
612 | " \"Input code above\"\n",
613 | " \n",
614 | " self.target_network.set_weights(target_weights)\n",
615 | " \n",
616 | " def update_epsilon(self):\n",
617 | " if self.epsilon >= self.epsilon_minimum:\n",
618 | " self.epsilon *= self.epsilon_decay\n",
619 | " \n",
620 | " def select_action(self, state):\n",
621 | " \n",
622 | " if self.epsilon > np.random.uniform():\n",
623 | " action = self.env.action_space.sample()\n",
624 | " else:\n",
625 | " action = np.argmax(self.local_network.predict(np.array([state])))\n",
626 | "\n",
627 | " return action\n",
628 | " \n",
629 | " def step(self, state):\n",
630 | " \n",
631 | " action = self.select_action(state)\n",
632 | " new_state, reward, done, _ = self.env.step(action)\n",
633 | "\n",
634 | " return action, reward, done, new_state\n",
635 | " \n",
636 | " def save(self):\n",
637 | " save_dir = os.path.join(os.getcwd(), \n",
638 | " self.env.spec.id +\"_\"+ datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))\n",
639 | " os.makedirs(save_dir)\n",
640 | " self.target_network.save(\"target_network.h5\")\n",
641 | " self.local_network.save(\"local_network.h5\")\n",
642 | " print(\"Weights saved successfully\")\n",
643 | " \n",
644 | " def load(self):\n",
645 | " uri = \"freeze_weights/\" + self.env.spec.id + \"/\"\n",
646 | " self.target_network.load_weights(uri + \"target_network.h5\")\n",
647 | " self.local_network.load_weights(uri + \"local_network.h5\")\n",
648 | " print(\"Weights loaded successfully for environment: {}\".format(self.env.spec.id))\n",
649 | " \n",
650 | " def freeze_network(self, freeze_layers):\n",
651 | " # Freezes first freeze_layers of the network and resets the other if freeze_layers is not zero.\n",
652 | " network_size = len(self.local_network.get_config().get(\"layers\"))\n",
653 | " assert(freeze_layers <= network_size),\\\n",
654 | " \"Tried to freeze more layers than there are layers in local network!\"\n",
655 | " \n",
656 | " self.load()\n",
657 | " \n",
658 | " if freeze_layers == 0:\n",
659 | " print(\"No layers frozen, using loaded weights\")\n",
660 | " elif freeze_layers == network_size:\n",
661 | " for layer in range(network_size):\n",
662 | " self.local_network.layers[layer].trainable = False \n",
663 | " print(\"Frozen all layers, and using loaded weights\")\n",
664 | " else: \n",
665 | " for layer in range(network_size):\n",
666 | " if layer < freeze_layers:\n",
667 | " self.local_network.layers[layer].trainable = False \n",
668 | " else:\n",
669 | " new_weights = generate_blank_weights(self.local_network.layers[layer].get_weights()[0].shape)\n",
670 | " self.local_network.layers[layer].set_weights(new_weights)\n",
671 | " self.target_network.layers[layer].set_weights(new_weights) \n",
672 | " \n",
673 | " self.local_network.compile(loss=self.q_network.loss_metric, \n",
674 | " optimizer=Adam(lr=self.q_network.learning_rate, \n",
675 | " decay=self.q_network.learning_rate_decay))\n",
676 | " print(\"Networks first {} layers are succesfully frozen\".format(freeze_layers))\n",
677 | " "
678 | ]
679 | },
680 | {
681 | "cell_type": "code",
682 | "execution_count": null,
683 | "metadata": {},
684 | "outputs": [],
685 | "source": [
686 | "# Do not edit - Assertion cell #\n",
687 | "if should_assert:\n",
688 | " dummy_parameters, dummy_env = get_dummy_parameters_and_env()\n",
689 | " dummy_dqn = Agent(dummy_env, dummy_parameters)\n",
690 | "\n",
691 | " dummy_local_weights = deepcopy(dummy_dqn.local_network.get_weights())\n",
692 | " dummy_target_weights = deepcopy(dummy_dqn.target_network.get_weights())\n",
693 | "\n",
694 | " dummy_dqn.update_target_network()\n",
695 | " for i in range(len(dummy_target_weights)):\n",
696 | " np.testing.assert_array_equal(\n",
697 | " dummy_dqn.target_network.get_weights()[i], \n",
698 | " dummy_parameters.get(\"tau\") * dummy_local_weights[i] + (1 - dummy_parameters.get(\"tau\")) * dummy_target_weights[i],\n",
699 | " err_msg=\"\\nThe target network was not implemented correctly. Layer {} was incorrect\\n\".format(i))\n",
700 | "\n",
701 | " print(\"Great job, you implemented the update_target_network-method correctly!\")\n"
702 | ]
703 | },
704 | {
705 | "cell_type": "markdown",
706 | "metadata": {},
707 | "source": [
708 | "## Train\n",
709 | "\n",
710 | "The training method is more or less identical to the method we used in the previous notebook, with some modifications due to the introduction of a neural network to represent the Q-table.\n",
711 | "\n",
712 | "\n",
713 | "### Task\n",
714 | "Make sure you understand the method before moving on!\n",
715 | "- Do you understand why the episodic reward is a better metric of the neural network's performance than the loss-metric when doing Reinforcement learning?\n"
716 | ]
717 | },
718 | {
719 | "cell_type": "code",
720 | "execution_count": null,
721 | "metadata": {},
722 | "outputs": [],
723 | "source": [
724 | "def train(agent, iterations, episodes):\n",
725 | " \n",
726 | " total_reward = 0\n",
727 | " total_reward_list, iterations_list = [], []\n",
728 | " agent.experience_replay.warm_up()\n",
729 | " \n",
730 | " for episode in range(episodes):\n",
731 | " \n",
732 | " state = env.reset()\n",
733 | " total_reward=0\n",
734 | " \n",
735 | " if (episode != 0): \n",
736 | " agent.update_epsilon()\n",
737 | " \n",
738 | " for iteration in range(iterations):\n",
739 | " \n",
740 | " action, reward, done, new_state = agent.step(state)\n",
741 | " agent.experience_replay.add_experience(state, action, reward, done, new_state)\n",
742 | " \n",
743 | " state = new_state\n",
744 | " \n",
745 | " agent.update_local_network()\n",
746 | " agent.update_target_network()\n",
747 | " total_reward += reward\n",
748 | " \n",
749 | " if done: \n",
750 | " break\n",
751 | " \n",
752 | " total_reward_list.append(total_reward)\n",
753 | " iterations_list.append(iteration+1)\n",
754 | " \n",
755 | " if episode % 5 == 0 and episode != 0:\n",
756 | " print(\"Episode: {0:d}-{1:d} | Avg. iterations: {2:0.2f} | Max total reward: {3:0.2f} | Avg. total reward: {4:0.2f} | Epsilon: {5:0.4f}\" \\\n",
757 | " .format(episode-10, episode, mean(iterations_list), max(total_reward_list), mean(total_reward_list), agent.epsilon))\n",
758 | " total_reward_list.clear()\n",
759 | " iterations_list.clear()"
760 | ]
761 | },
762 | {
763 | "cell_type": "markdown",
764 | "metadata": {},
765 | "source": [
766 | "## Hyperparameters\n",
767 | "\n",
768 | "The hyperparameters are the customizable parameters which are not optimized by the agent, but are set by us.\n",
769 | "\n",
770 | "The number of parameters passed to the agent significantly increases when we begin using neural networks. To simplify the process, we define them in a dictionary and simply pass the dictionary to the agent. We have chosen the hyperparameters for some environments, and they should not be changed the first time you go through the notebook.\n",
771 | "\n",
772 | "PS. You may change them as you please when you have passed through the notebook once."
773 | ]
774 | },
775 | {
776 | "cell_type": "code",
777 | "execution_count": null,
778 | "metadata": {},
779 | "outputs": [],
780 | "source": [
781 | "def get_hyperparameters(env):\n",
782 | " \n",
783 | " env_id = env.spec.id\n",
784 | " \n",
785 | " if env_id == \"CartPole-v1\":\n",
786 | " print(\"Hyperparameters for {} chosen!\".format(\"CartPole-v1\"))\n",
787 | " parameters = {\n",
788 | " \"tau\" : 0.05,\n",
789 | " \"gamma\" : 0.99,\n",
790 | " \"epsilon_init\" : 1,\n",
791 | " \"epsilon_decay\" : 0.95,\n",
792 | " \"epsilon_minimum\": 0.01,\n",
793 | " \"buffer_size\" : 2000,\n",
794 | " \"batch_size\" : 64,\n",
795 | " \"epochs\": 1,\n",
796 | " \"loss_metric\" : \"mse\",\n",
797 | " \"learning_rate\" : 0.01,\n",
798 | " \"learning_rate_decay\": 0.01,\n",
799 | " \"hidden_layer_1\": 24,\n",
800 | " \"hidden_layer_2\": 24}\n",
801 | " \n",
802 | " else:\n",
803 | " print(\"Standard hyperparameters chosen!\")\n",
804 | " parameters = {\n",
805 | " \"tau\" : 0.05,\n",
806 | " \"gamma\" : 0.95,\n",
807 | " \"epsilon_init\" : 1,\n",
808 | " \"epsilon_decay\" : 0.97,\n",
809 | " \"epsilon_minimum\": 0.01,\n",
810 | " \"buffer_size\" : 10000,\n",
811 | " \"batch_size\" : 32,\n",
812 | " \"epochs\": 1,\n",
813 | " \"loss_metric\" : \"mse\",\n",
814 | " \"learning_rate\" : 0.01,\n",
815 | " \"learning_rate_decay\": 0.01,\n",
816 | " \"hidden_layer_1\": 24,\n",
817 | " \"hidden_layer_2\": 24}\n",
818 | " \n",
819 | " return parameters"
820 | ]
821 | },
822 | {
823 | "cell_type": "markdown",
824 | "metadata": {},
825 | "source": [
826 | "# Environment\n",
827 | "For this walkthorugh we are going to use an environment called CartPole-v1.\n",
828 | "\n",
829 | "\n",
830 | "\n",
831 | "The pendulum starts upright, and the goal is to prevent it from falling over by applying a horizontal force to the cart, of either +1 or -1. A reward of +1 is provided for every timestep that the pole remains upright. The episode ends when the pole is more than 15 degrees from vertical, the cart moves more than 2.4 units from the center, or the agent manages to stay \"alive\" for 500 iterations.\n",
832 | "\n",
833 | "### Task\n",
834 | "- Create an agent by running the cell below"
835 | ]
836 | },
837 | {
838 | "cell_type": "code",
839 | "execution_count": null,
840 | "metadata": {},
841 | "outputs": [],
842 | "source": [
843 | "environment = \"CartPole-v1\"\n",
844 | "env = gym.make(environment)\n",
845 | "parameters = get_hyperparameters(env)\n",
846 | "\n",
847 | "dqn_agent = Agent(env, parameters)"
848 | ]
849 | },
850 | {
851 | "cell_type": "markdown",
852 | "metadata": {},
853 | "source": [
854 | "## Play\n",
855 | "To run a single episode using our agent we use a method we created for you called play(), which takes an Agent-object as input. The logic in the play method is identical to the method we used in the previous notebook.\n",
856 | "\n",
857 | "### Task\n",
858 | "- Test the performance of our agent before any training"
859 | ]
860 | },
861 | {
862 | "cell_type": "code",
863 | "execution_count": null,
864 | "metadata": {},
865 | "outputs": [],
866 | "source": [
867 | "play(dqn_agent)"
868 | ]
869 | },
870 | {
871 | "cell_type": "markdown",
872 | "metadata": {},
873 | "source": [
874 | "## Visualize performance\n",
875 | "We can also visualize the performance of the agent by using another method we created for you, called generate_video(), which takes an Agent-object as input. \n",
876 | "\n",
877 | "Since the workshop is hosted on a headless servers, i.e. an EC2 instance on AWS, we need to make some hacks to visualize the performance. The generate_video() method will therefore first generate an .mp4-file, which we will play using the HTML snippet below.\n",
878 | "\n",
879 | "### Task\n",
880 | "- Visualize the performance of the agent by running the cell below"
881 | ]
882 | },
883 | {
884 | "cell_type": "code",
885 | "execution_count": null,
886 | "metadata": {},
887 | "outputs": [],
888 | "source": [
889 | "generate_video(dqn_agent)\n",
890 | "HTML(\"\"\"\n",
891 | "\n",
894 | "\"\"\".format(\"./videos/\"+list(filter(lambda s: s.endswith(\".mp4\"), os.listdir(\"./videos/\")))[-1]))"
895 | ]
896 | },
897 | {
898 | "cell_type": "markdown",
899 | "metadata": {},
900 | "source": [
901 | "## Train Agent\n",
902 | "\n",
903 | "We will begin training our agent for 300 episodes, with a maximum iteration of 500. This will take some time, so be patient."
904 | ]
905 | },
906 | {
907 | "cell_type": "code",
908 | "execution_count": null,
909 | "metadata": {},
910 | "outputs": [],
911 | "source": [
912 | "episodes = 150\n",
913 | "iterations = 500\n",
914 | "\n",
915 | "train(dqn_agent, iterations, episodes)"
916 | ]
917 | },
918 | {
919 | "cell_type": "markdown",
920 | "metadata": {},
921 | "source": [
922 | "## Visualize performance\n",
923 | "Now lets visualize the performance of our trained agent!"
924 | ]
925 | },
926 | {
927 | "cell_type": "code",
928 | "execution_count": null,
929 | "metadata": {},
930 | "outputs": [],
931 | "source": [
932 | "generate_video(dqn_agent)\n",
933 | "HTML(\"\"\"\n",
934 | "\n",
937 | "\"\"\".format(\"./videos/\"+list(filter(lambda s: s.endswith(\".mp4\"), os.listdir(\"./videos/\")))[-1]))"
938 | ]
939 | },
940 | {
941 | "cell_type": "markdown",
942 | "metadata": {},
943 | "source": [
944 | "## And we are through!\n",
945 | "\n",
946 | "You have implemented the DQN-algorithm, congratulations!\n",
947 | "\n",
948 | "Try optimizing the hyperparameters to improve the training results. Suggestions:\n",
949 | "- Number of layers\n",
950 | "- Number of neurons in each layer\n",
951 | "- Learning rate\n",
952 | "- Epsilon strategy\n",
953 | "\n",
954 | "If you like to try a different environment, try the \"Acrobat-v1\" environment.\n",
955 | "\n",
956 | "Remember to turn of the assertion by running the cell below!"
957 | ]
958 | },
959 | {
960 | "cell_type": "code",
961 | "execution_count": null,
962 | "metadata": {},
963 | "outputs": [],
964 | "source": [
965 | "should_assert = False"
966 | ]
967 | }
968 | ],
969 | "metadata": {
970 | "kernelspec": {
971 | "display_name": "Python 3",
972 | "language": "python",
973 | "name": "python3"
974 | },
975 | "language_info": {
976 | "codemirror_mode": {
977 | "name": "ipython",
978 | "version": 3
979 | },
980 | "file_extension": ".py",
981 | "mimetype": "text/x-python",
982 | "name": "python",
983 | "nbconvert_exporter": "python",
984 | "pygments_lexer": "ipython3",
985 | "version": "3.7.3"
986 | }
987 | },
988 | "nbformat": 4,
989 | "nbformat_minor": 2
990 | }
991 |
--------------------------------------------------------------------------------
![\"drawing\"](\"Images/Discussion.jpg\")
"
359 | ]
360 | },
361 | {
362 | "cell_type": "markdown",
363 | "metadata": {},
364 | "source": [
365 | "# Q-learning\n",
366 | "\n",
367 | "In Reinforcement learning an agent moves from a state $S_{t}$, to a new state $S_{t+1}$ by taking an action $A_{t}$, and receiving the reward $R_{t}$ as illustrated below. The agent will repeat this process for a defined amount of iterations, and this process as a whole is known as an episode. The goal of the Reinforcement learning agent is to maximize the total reward it can collect during an episode. To improve the performance, it learns from each state transition, i.e. move from $S_{t}$ to $S_{t+1}$. How it learns, is what separates the different Reinforcement learning algorithms.\n",
368 | "\n",
369 | "![\"drawing\"](\"Images/Agent.png\")
\n",
370 | "\n",
371 | "In this notebook, we will work with one of the most iconic Reinforcement learning algorithms in its simplest form, namely Q-learning using Q-tables. The Q-tables holds the values of the strategy that gives us the highest reward, according to the agent's knowledge. The table can be thought of as the brain of the agent. Each row in the table represents a state, and the columns represent the possible actions in that state. This will be explained further in the next section.\n",
372 | "\n",
373 | "A Q-value is a measure of the total expected reward the agent will receive if it chooses that action and always picks the action with the highest Q-value in the succeeding states, based on its current knowledge of the environment, i.e. the Q-table. Simply said, the higher the Q-value, the better the agent believes the action is.\n",
374 | "\n",
375 | "The Q-learning algorithm is used to update the Q-values for a given action $A_{t}$ in the state $S_{t}$, and is updated after each iteration. The equation is shown below.\n",
376 | "\n",
377 | "$\n",
378 | "\\begin{equation}\n",
379 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]\n",
380 | "\\end{equation}\n",
381 | "$\n",
382 | "\n",
383 | "A key point to note in this equation is that we are updating the Q(s, a) by using the highest Q-value in the next state, $S_{t+1}$. This value is unknown, and it is therefore just the agent's estimate of that state. This makes it an optimization problem were we gradually adjust each Q-value as we receive a reward moving between the states.\n",
384 | "\n",
385 | "\n",
386 | "This might seem daunting at first, however, we will through this notebook break it into simple pieces, which will hopefully, make it easier to grasp."
387 | ]
388 | },
389 | {
390 | "cell_type": "markdown",
391 | "metadata": {},
392 | "source": [
393 | "## Create the Q-table\n",
394 | "The Q-table is the brain of the agent and stores the agent's current knowledge of the Q-values of each state. Each row represents a state, and the columns represent all possible actions in that state. As a result, our table will have the dimensions, (rows=number of states, columns=number of actions). An illustration of the table is shown below, although without the labels on the columns. \n",
395 | "\n",
396 | "![\"drawing\"](\"Images/Taxi_matrix_initial.png\")
\n",
397 | "\n",
398 | "### Task:\n",
399 | "Create and return a Q-table using numpy, where each cell is initialized to zeros:\n",
400 | "- Create the Q-table by using the numpy command: np.zeros(shape=[rows, columns]). Read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.zeros.html)\n",
401 | "- Use the following command to get the number of states in the environment:\n",
402 | " - env.observation_space.n\n",
403 | "- Use the following command to get the number of possible actions. PS. all states have the same amount of actions.\n",
404 | " - env.action_space.n"
405 | ]
406 | },
407 | {
408 | "cell_type": "code",
409 | "execution_count": null,
410 | "metadata": {},
411 | "outputs": [],
412 | "source": [
413 | "def create_q_table(env):\n",
414 | " \n",
415 | " \"Write code below\" \n",
416 | " \n",
417 | " \"Write code above\" \n",
418 | " \n",
419 | " return Q_table"
420 | ]
421 | },
422 | {
423 | "cell_type": "code",
424 | "execution_count": null,
425 | "metadata": {},
426 | "outputs": [],
427 | "source": [
428 | "# Do not edit - Assertion cell #\n",
429 | "if mock.do_assertion():\n",
430 | " assert(np.count_nonzero(mock.get_Q() == 0)), \"All values in Q-table should be zero\"\n",
431 | " assert(create_q_table(mock.get_env()).shape == (mock.get_env().observation_space.n, mock.get_env().action_space.n)), \\\n",
432 | " \"The dimensions are wrong\"\n",
433 | " print(\"The Q-table was correctly built! \" +\n",
434 | " \"It has {} rows, each representing a unique state, and {} columns, each representing an action for that state.\"\\\n",
435 | " .format(mock.get_env().observation_space.n, mock.get_env().action_space.n))"
436 | ]
437 | },
438 | {
439 | "cell_type": "markdown",
440 | "metadata": {},
441 | "source": [
442 | "## Select the best action\n",
443 | "Before we implement the Q-learning algorithm, we will implement a method that, given a state, chooses the action with the highest Q-value.\n",
444 | "\n",
445 | "### Task\n",
446 | "Return the column index of the highest Q-value in the Q-table, given a state.\n",
447 | "- Tip: Use the argmax function from the numpy library. You can read more about it [here](https://docs.scipy.org/doc/numpy/reference/generated/numpy.argmax.html)"
448 | ]
449 | },
450 | {
451 | "cell_type": "code",
452 | "execution_count": null,
453 | "metadata": {},
454 | "outputs": [],
455 | "source": [
456 | "def get_best_action(Q_table, state):\n",
457 | " \n",
458 | " \"Write code below\" \n",
459 | " \n",
460 | " \"Write code above\"\n",
461 | " \n",
462 | " return best_action"
463 | ]
464 | },
465 | {
466 | "cell_type": "code",
467 | "execution_count": null,
468 | "metadata": {},
469 | "outputs": [],
470 | "source": [
471 | "# Do not edit - Assertion cell #\n",
472 | "if mock.do_assertion():\n",
473 | " assert(get_best_action(mock.get_Q(), 1) == 0), \"The method did not pick the action with the highest Q_value\"\n",
474 | " print(\"The best action for the test state was chosen, excellent!\")"
475 | ]
476 | },
477 | {
478 | "cell_type": "markdown",
479 | "metadata": {},
480 | "source": [
481 | "## Select an action\n",
482 | "\n",
483 | "Initially, all the Q-values in the Q-table are set to zero. As we begin to move around the environment we can end up in a loop, such that we only explore a small part of the environment. This problem is so common its gotten a name; The Exploration vs Exploitation dilemma!\n",
484 | "\n",
485 | "One simple and very effective solution is to use a strategy called Epsilon-Greedy. The strategy makes the agent choose mostly random actions in the initial episodes, which forces the agent to explore the environment. The strategy then gradually shifts the agent behavior towards making greedy choices, i.e. choosing the action with the highest Q-value. The parameter deciding the degree of random actions taken is called epsilon.\n",
486 | "\n",
487 | "### Task:\n",
488 | "Create a method that randomly picks an action IF a random value in the range [0,1] is less than epsilon, or ELSE picks the best action using the method we created earlier. \n",
489 | "- To pick a random action, use env.action_space.sample()\n",
490 | "- To generate a random number with uniform probability in the range [0,1], use np.random.uniform()"
491 | ]
492 | },
493 | {
494 | "cell_type": "code",
495 | "execution_count": null,
496 | "metadata": {},
497 | "outputs": [],
498 | "source": [
499 | "def select_action(Q_table, state, epsilon):\n",
500 | " \n",
501 | " \"Write code below\"\n",
502 | " \n",
503 | " \"Write code above\" \n",
504 | " \n",
505 | " return action"
506 | ]
507 | },
508 | {
509 | "cell_type": "code",
510 | "execution_count": null,
511 | "metadata": {
512 | "scrolled": true
513 | },
514 | "outputs": [],
515 | "source": [
516 | "# Do not edit - Assertion cell #\n",
517 | "if mock.do_assertion():\n",
518 | " assert(select_action(mock.get_Q(), 1, 0) == 0), \\\n",
519 | " \"Method should always return the same value for this state, since epsilon is 0\"\n",
520 | " assert not (len(set([select_action(mock.get_Q(), 1, 1) for x in range(20)])) <= 2), \\\n",
521 | " \"Method should not return identical values when a random action should be chosen\"\n",
522 | " print(\"The correct actions were picked, great job!\")"
523 | ]
524 | },
525 | {
526 | "cell_type": "markdown",
527 | "metadata": {},
528 | "source": [
529 | "## Gradually shift towards exploitation and reduce exploration of state-space\n",
530 | "\n",
531 | "To make sure we are gradually moving from exploration to exploitation, we need to reduce the possibility of choosing a random action. In other words, we need to reduce epsilon. This is done by multiplying it with a constant called epsilon decay.\n",
532 | "\n",
533 | "### Task:\n",
534 | "Create a method which reduces epsilon by multiplying itself with the defined constant epsilon decay."
535 | ]
536 | },
537 | {
538 | "cell_type": "code",
539 | "execution_count": null,
540 | "metadata": {},
541 | "outputs": [],
542 | "source": [
543 | "def update_epsilon(epsilon):\n",
544 | " epsilon_decay = 0.95\n",
545 | " \n",
546 | " \"Write code below\" \n",
547 | "\n",
548 | " \"Write code above\"\n",
549 | " \n",
550 | " return epsilon"
551 | ]
552 | },
553 | {
554 | "cell_type": "code",
555 | "execution_count": null,
556 | "metadata": {},
557 | "outputs": [],
558 | "source": [
559 | "# Do not edit - Assertion cell #\n",
560 | "if mock.do_assertion():\n",
561 | " assert(update_epsilon(5) == 4.75), \"Given an input of 5, the output should have been 4.75\"\n",
562 | " print(\"Epsilon was correctly updated!\")"
563 | ]
564 | },
565 | {
566 | "cell_type": "markdown",
567 | "metadata": {},
568 | "source": [
569 | " ## Find the highest Q_value for a given state\n",
570 | "\n",
571 | "Now it is finally time to begin building our Q-learning algorithm. The method we are going to create first is the part marked red in the equation below. This method returns the highest Q-value of the new state.\n",
572 | "\n",
573 | "$\n",
574 | "\\begin{equation}\n",
575 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ {\\color{red}{\\underset{a}{max} \\ Q(s_{t+1},a)}} - Q(s_{t},a_{t})\\big]\n",
576 | "\\end{equation}\n",
577 | "$ \n",
578 | "\n",
579 | "\n",
580 | "### Task\n",
581 | "Return the highest Q-value for a given state, which is here called new_state.\n",
582 | "- Tip: Use the numpy.max function to retrieve the column (action) with the highest Q-value for the given row(state)"
583 | ]
584 | },
585 | {
586 | "cell_type": "code",
587 | "execution_count": null,
588 | "metadata": {},
589 | "outputs": [],
590 | "source": [
591 | "def find_highest_Q_value_in_state(Q_table, new_state):\n",
592 | " \"Write code below\"\n",
593 | "\n",
594 | " \"Write code above\"\n",
595 | " \n",
596 | " return best_Q_value"
597 | ]
598 | },
599 | {
600 | "cell_type": "code",
601 | "execution_count": null,
602 | "metadata": {},
603 | "outputs": [],
604 | "source": [
605 | "# Do not edit - Assertion cell #\n",
606 | "if mock.do_assertion():\n",
607 | " assert(find_highest_Q_value_in_state(mock.get_Q(), 1) == 10), \\\n",
608 | " \"The method did not pick the action with the highest Q_value\"\n",
609 | " print(\"Perfect, the method returned the highest Q-value for that state!\")"
610 | ]
611 | },
612 | {
613 | "cell_type": "markdown",
614 | "metadata": {},
615 | "source": [
616 | "## Calculate error in Q-value\n",
617 | "The next method we are going to implement calculates how much we need to adjust our Q-value based on the reward we receive from moving from state $S_{t}$ to state $S_{t+1}$. In the intro to the Q-learning section, we defined the Q-value as the assumed expected total reward the agent will receive if it moves from state $S_{t}$ taking action $A_{t}$, and then always picks the action with the highest Q-value throughout the episode using the current Q-table. As a result, the the correct Q-value $Q(S_{t}, A_{t})$ can be defined as\n",
618 | "\n",
619 | "$\n",
620 | "Q(s_{t}, a_{t}) = r_{t} + \\underset{a}{max} \\ Q(s_{t+1},a)\n",
621 | "$\n",
622 | "\n",
623 | "Assuming $\\gamma$ is equal to one, we see that the statement marked red would become equal to 0, if our Q-value was perfect, thereby making no changes to the Q-value.\n",
624 | "\n",
625 | "$\n",
626 | "\\begin{equation}\n",
627 | "Q(s_{t},a_{t})^{new} \\leftarrow Q(s_{t}, a_{t}) + \\alpha \\ \\big[{\\color{red}{r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})}}\\big]\n",
628 | "\\end{equation}\n",
629 | "$\n",
630 | "\n",
631 | "The parameter $\\gamma$ is connected to how the agent values immediate rewards compared to later rewards, however explanation of this is outside the scope of this notebook. \n",
632 | "\n",
633 | "### Task:\n",
634 | "Implement the part of the equation mark red.\n",
635 | "- Use the method we created in the previous section to find the highest_q_value_in_state\n",
636 | "- Use brackets notation to select the correct cell i.e. Q_table[Desired_state, desired_action]"
637 | ]
638 | },
639 | {
640 | "cell_type": "code",
641 | "execution_count": null,
642 | "metadata": {},
643 | "outputs": [],
644 | "source": [
645 | "def calculate_error_in_Q_value(Q_table, state, action, reward, new_state):\n",
646 | " \n",
647 | " discount_factor = 0.95\n",
648 | " \n",
649 | " \"Write code below\"\n",
650 | " \n",
651 | " \"Write code above\"\n",
652 | " \n",
653 | " return error_in_Q_value"
654 | ]
655 | },
656 | {
657 | "cell_type": "code",
658 | "execution_count": null,
659 | "metadata": {},
660 | "outputs": [],
661 | "source": [
662 | "# Do not edit - Assertion cell #\n",
663 | "if mock.do_assertion():\n",
664 | " assert(calculate_error_in_Q_value(mock.get_Q(), 0, 0, 10, 1) == 9.5), \\\n",
665 | " \"The method did not calculate the correct error value for the test sample\"\n",
666 | " print(\"The error in Q-value have been calculated correctly!\")"
667 | ]
668 | },
669 | {
670 | "cell_type": "markdown",
671 | "metadata": {},
672 | "source": [
673 | "## Update Q-table\n",
674 | "\n",
675 | "We are almost ready to complete our Q-learning algorithm. \n",
676 | "\n",
677 | "The only element not yet explained is the parameter $\\alpha$. $\\alpha$ is known as the learning rate and controls how quick we update the Q-table. For example, a learning rate of 0.1 indicates that we will only use 10% of the actual error calculated when updating the Q-table. This is because we calculate the Q-value by using the highest Q-value of the next state, which is just our best estimate so far and might be erroneous. If that state has not been visited yet, it would simply have the initialized value of the table, i.e. 0. By using the learning rate, we account for this uncertainty and get a less oscillating system which is proven to give better results. \n",
678 | "\n",
679 | "$\n",
680 | "\\begin{equation}\n",
681 | "Q(s_{t},a_{t})^{new} \\leftarrow {\\color{red}{Q(s_{t}, a_{t}) + \\alpha \\ \\big[r_{t} + \\gamma \\ \\underset{a}{max} \\ Q(s_{t+1},a) - Q(s_{t},a_{t})\\big]}}\n",
682 | "\\end{equation}\n",
683 | "$\n",
684 | "\n",
685 | "To implement the algorithm there is one factor which needs to be accounted for. If the episode has ended due to a terminal state, the last reward received will be the Q-value of that state, since there is no possible next state. As a result, if the done parameter is true (an input to the method below), we set the Q-value of that state-action pair equal to the reward.\n",
686 | "\n",
687 | "### Task\n",
688 | "Create a method which either returns\n",
689 | " - The Q-table with the Q-value of that state-action pair equal to the reward, IF it was the final episode. (Done is true)\n",
690 | " - ELSE the Q-table updated with the Q-learning algorithm, using the methods we have created."
691 | ]
692 | },
693 | {
694 | "cell_type": "code",
695 | "execution_count": null,
696 | "metadata": {},
697 | "outputs": [],
698 | "source": [
699 | "def update_q_table(Q_table, state, action, done, reward, new_state):\n",
700 | " \n",
701 | " learning_rate = 0.8\n",
702 | " \n",
703 | " \"Write code below\"\n",
704 | "\n",
705 | " \"Write code above\"\n",
706 | " \n",
707 | " return Q_table"
708 | ]
709 | },
710 | {
711 | "cell_type": "code",
712 | "execution_count": null,
713 | "metadata": {},
714 | "outputs": [],
715 | "source": [
716 | "# Do not edit - Assertion cell #\n",
717 | "if mock.do_assertion():\n",
718 | " assert(update_q_table(mock.get_Q(), 2, 1, 1, 10, 3)[2,1] == 10), \\\n",
719 | " \"The updated Q-value when simulation was done was incorrect\"\n",
720 | " assert(update_q_table(mock.get_Q(), 0, 0, 0, 10, 1)[0,0] == 17.6), \\\n",
721 | " \"The updated Q-value when the episode was not done, was incorrect\"\n",
722 | " print(\"Q-tabel have been correctly updated!\")"
723 | ]
724 | },
725 | {
726 | "cell_type": "markdown",
727 | "metadata": {},
728 | "source": [
729 | "## Take one step\n",
730 | "\n",
731 | "Now we are ready to create a method which makes our agent make a move in the environment.\n",
732 | "\n",
733 | "### Task\n",
734 | "Create a method which selects the best action, passes this to our environment, and returns the action, reward, done new_state variables.\n",
735 | "- To choose an action use the select_action method we created earlier\n",
736 | "- To make a step in the environment, use this method env.step(action)\n",
737 | "- env.step(action) return the following values accordingly: new_state, reward, done, state_transition_probability\n",
738 | "- Ignore the state_transition_probability, as this is not used in this environment\n",
739 | " "
740 | ]
741 | },
742 | {
743 | "cell_type": "code",
744 | "execution_count": null,
745 | "metadata": {},
746 | "outputs": [],
747 | "source": [
748 | "def step(Q_table, env, state, epsilon):\n",
749 | " \n",
750 | " \"Write code below\"\n",
751 | " \n",
752 | " \"Write code above\"\n",
753 | " \n",
754 | " return action, reward, done, new_state"
755 | ]
756 | },
757 | {
758 | "cell_type": "code",
759 | "execution_count": null,
760 | "metadata": {},
761 | "outputs": [],
762 | "source": [
763 | "# Do not edit - Assertion cell #\n",
764 | "if mock.do_assertion():\n",
765 | " mock_env1 = deepcopy(mock.get_env())\n",
766 | " mock_env1.reset()\n",
767 | " mock_env2 = deepcopy(mock_env1)\n",
768 | "\n",
769 | " new_state, _, _, _ = mock_env1.step(0)\n",
770 | " assert(step(mock.get_Q(), mock_env2, 0, 0) == (0, -1, False, new_state)), \\\n",
771 | " \"Your environment did not return the correct values\"\n",
772 | " print(\"Your step method seems to function properly, good job!\")"
773 | ]
774 | },
775 | {
776 | "cell_type": "markdown",
777 | "metadata": {},
778 | "source": [
779 | "## Create train method\n",
780 | "\n",
781 | "Finally, we can create the method which will train our agent, i.e. update the Q-table.\n",
782 | "\n",
783 | "### Task\n",
784 | "In this method, you will enter your code at multiple places, but always between \"Write code below\" and \"Write code above\". \n",
785 | "- You will only input methods you have already created. \n",
786 | "- Remember to store the value in the right variable, i.e. the Q-table is stored in the variable Q_table\n",
787 | "- To see which variables the methods returns, look at your implementation in the previous cells.\n",
788 | "- PS. The following cells do not have assertion cells."
789 | ]
790 | },
791 | {
792 | "cell_type": "code",
793 | "execution_count": null,
794 | "metadata": {},
795 | "outputs": [],
796 | "source": [
797 | "def train(env, num_episodes, max_iterations):\n",
798 | " done = False\n",
799 | " epsilon = 1\n",
800 | " reward_list, iterations_list, epsilon_list = [], [], []\n",
801 | " \n",
802 | " \"Write code below\"\n",
803 | " # Create Q-table\n",
804 | "\n",
805 | " \"Write code above\"\n",
806 | "\n",
807 | " for episode in range(num_episodes):\n",
808 | "\n",
809 | " if (done and episode % 10 == 0): \n",
810 | " print(\"Episode: {} | Iterations: {} | Total Reward: {} | Epsilon: {}\"\\\n",
811 | " .format(episode, iterations, total_reward, epsilon))\n",
812 | " \n",
813 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
814 | " \n",
815 | " \"Write code below\"\n",
816 | " # Update epsilon\n",
817 | " \n",
818 | " \"Write code above\"\n",
819 | " \n",
820 | " while not done:\n",
821 | " \n",
822 | " \"Write code below\"\n",
823 | " \n",
824 | " # Take step\n",
825 | " \n",
826 | " # Update Q-table\n",
827 | " \n",
828 | " \"Write code above\"\n",
829 | " \n",
830 | " # Set the new state as the current state\n",
831 | " state = new_state\n",
832 | " \n",
833 | " # Update episode information\n",
834 | " iterations += 1\n",
835 | " total_reward += reward\n",
836 | " \n",
837 | " # End episode if it has lasted for too many iterations\n",
838 | " if (iterations >= max_iterations): done = True\n",
839 | "\n",
840 | " reward_list.append(total_reward)\n",
841 | " iterations_list.append(iterations)\n",
842 | " epsilon_list.append(epsilon)\n",
843 | " \n",
844 | " return Q_table, (reward_list, iterations_list, epsilon_list)"
845 | ]
846 | },
847 | {
848 | "cell_type": "markdown",
849 | "metadata": {},
850 | "source": [
851 | "## Create play method\n",
852 | "\n",
853 | "This method can be used if you want to test the performance of your agent, by only running one episode."
854 | ]
855 | },
856 | {
857 | "cell_type": "code",
858 | "execution_count": null,
859 | "metadata": {},
860 | "outputs": [],
861 | "source": [
862 | "def play(env, Q_table):\n",
863 | " \n",
864 | " state, done, iterations, total_reward = reset_env_and_update_params(env)\n",
865 | "\n",
866 | " while not done:\n",
867 | "\n",
868 | " # Take step\n",
869 | " action, reward, done, new_state = step(Q_table, env, state, 0)\n",
870 | "\n",
871 | " # Set the new state as the current state\n",
872 | " state = new_state\n",
873 | "\n",
874 | " # Update episode information\n",
875 | " iterations += 1\n",
876 | " total_reward += reward\n",
877 | "\n",
878 | " if (iterations >= 100): done = True\n",
879 | " \n",
880 | " print(\"Iterations: {} | Total Reward: {}\".format(iterations, total_reward))"
881 | ]
882 | },
883 | {
884 | "cell_type": "markdown",
885 | "metadata": {
886 | "collapsed": true
887 | },
888 | "source": [
889 | "## Time to put our methods to use!\n",
890 | "\n",
891 | "Congratulation, you have created every method need to train your Q-learning agent!\n",
892 | "\n",
893 | "### Task\n",
894 | "To train your agent, simply set a value for the maximum_episodes and maximum_iterations\n",
895 | "- Tip 1: Maximum_episodes should be less than 2000\n",
896 | "- Tip 2: Maximum iterations should be less than 200"
897 | ]
898 | },
899 | {
900 | "cell_type": "code",
901 | "execution_count": null,
902 | "metadata": {},
903 | "outputs": [],
904 | "source": [
905 | "maximum_episodes = 0\n",
906 | "maximum_iterations = 0\n",
907 | "\n",
908 | "Q_table_trained, data = train(env, maximum_episodes, maximum_iterations)"
909 | ]
910 | },
911 | {
912 | "cell_type": "markdown",
913 | "metadata": {},
914 | "source": [
915 | "## Plot key variables \n",
916 | "\n",
917 | "This method plots different parameters during training. As a result, the training cell above must be run before this one.\n",
918 | "\n",
919 | "### Task\n",
920 | "- Simply run the cell below to create the plots"
921 | ]
922 | },
923 | {
924 | "cell_type": "code",
925 | "execution_count": null,
926 | "metadata": {},
927 | "outputs": [],
928 | "source": [
929 | "try: \n",
930 | " assert(data)\n",
931 | " plot_visualization(data)\n",
932 | "except:\n",
933 | " print(\"You need to run the **train** cell before running this cell, as it uses the data from the training!\")"
934 | ]
935 | },
936 | {
937 | "cell_type": "markdown",
938 | "metadata": {},
939 | "source": [
940 | "## Play around for one round\n",
941 | "\n",
942 | "This method runs one episode, using your trained Q-table. This is a quick way to test the performance.\n",
943 | "\n",
944 | "### Task\n",
945 | "- Run the cell below to quickly see the performance of your agent. This method does not alter your Q_table, so you can run it as many times as you like!"
946 | ]
947 | },
948 | {
949 | "cell_type": "code",
950 | "execution_count": null,
951 | "metadata": {},
952 | "outputs": [],
953 | "source": [
954 | "play(env, Q_table_trained)"
955 | ]
956 | },
957 | {
958 | "cell_type": "markdown",
959 | "metadata": {},
960 | "source": [
961 | "## Visualize an episode\n",
962 | "\n",
963 | "### Task\n",
964 | "This method works similarly to the one above, but visualizes the environment.\n",
965 | "- If you would like to end the simulation, click on the Kernal tab in the toolbar, and click \"Interrupt\"."
966 | ]
967 | },
968 | {
969 | "cell_type": "code",
970 | "execution_count": null,
971 | "metadata": {},
972 | "outputs": [],
973 | "source": [
974 | "render_performance(env, Q_table_trained)"
975 | ]
976 | },
977 | {
978 | "cell_type": "markdown",
979 | "metadata": {},
980 | "source": [
981 | "## And we are done, almost!\n",
982 | "Now you have everything you need to play around with the environment and alter different parameters to see how it impacts our agent. Can you improve the parameters already chosen? Suggestions:\n",
983 | "- Increase the number of episodes and reduce iterations\n",
984 | "- Change epsilon decay to change the Epsilon-Greedy Strategy\n",
985 | "- Change the learning rate to make smaller or larger corrections to the Q-table per update\n",
986 | "- Change the discount factor $\\gamma$\n",
987 | "\n",
988 | "If you would like to create your own cells, do the following:\n",
989 | "1. Click on an existing cell\n",
990 | "2. Click the Esc - button\n",
991 | "3. Press \"b\" to add a cell below, press \"a\" the add a cell above, or double tap \"d\" to delete a cell\n",
992 | "\n",
993 | "### Task\n",
994 | "Run the cell below to turn off the assertion cells. ![\"drawing\"](\"Images/Neural_network__achitecture.svg\")
\n",
180 | "\n",
181 | "\n",
182 | "\n",
183 | "### Task\n",
184 | "Now we will experience how easy it is to create a neural network using Keras. We are going to create a neural network with an input layer, two hidden layers, and one output layer. We have completed steps 1,2, 5, and your task is to complete task 3 and 4 according to the design criterions below: ![\"drawing\"](\"Images/CartPole-v1.gif\")
\n",
830 | "\n",
831 | "The pendulum starts upright, and the goal is to prevent it from falling over by applying a horizontal force to the cart, of either +1 or -1. A reward of +1 is provided for every timestep that the pole remains upright. The episode ends when the pole is more than 15 degrees from vertical, the cart moves more than 2.4 units from the center, or the agent manages to stay \"alive\" for 500 iterations.\n",
832 | "\n",
833 | "### Task\n",
834 | "- Create an agent by running the cell below"
835 | ]
836 | },
837 | {
838 | "cell_type": "code",
839 | "execution_count": null,
840 | "metadata": {},
841 | "outputs": [],
842 | "source": [
843 | "environment = \"CartPole-v1\"\n",
844 | "env = gym.make(environment)\n",
845 | "parameters = get_hyperparameters(env)\n",
846 | "\n",
847 | "dqn_agent = Agent(env, parameters)"
848 | ]
849 | },
850 | {
851 | "cell_type": "markdown",
852 | "metadata": {},
853 | "source": [
854 | "## Play\n",
855 | "To run a single episode using our agent we use a method we created for you called play(), which takes an Agent-object as input. The logic in the play method is identical to the method we used in the previous notebook.\n",
856 | "\n",
857 | "### Task\n",
858 | "- Test the performance of our agent before any training"
859 | ]
860 | },
861 | {
862 | "cell_type": "code",
863 | "execution_count": null,
864 | "metadata": {},
865 | "outputs": [],
866 | "source": [
867 | "play(dqn_agent)"
868 | ]
869 | },
870 | {
871 | "cell_type": "markdown",
872 | "metadata": {},
873 | "source": [
874 | "## Visualize performance\n",
875 | "We can also visualize the performance of the agent by using another method we created for you, called generate_video(), which takes an Agent-object as input. \n",
876 | "\n",
877 | "Since the workshop is hosted on a headless servers, i.e. an EC2 instance on AWS, we need to make some hacks to visualize the performance. The generate_video() method will therefore first generate an .mp4-file, which we will play using the HTML snippet below.\n",
878 | "\n",
879 | "### Task\n",
880 | "- Visualize the performance of the agent by running the cell below"
881 | ]
882 | },
883 | {
884 | "cell_type": "code",
885 | "execution_count": null,
886 | "metadata": {},
887 | "outputs": [],
888 | "source": [
889 | "generate_video(dqn_agent)\n",
890 | "HTML(\"\"\"\n",
891 | "\n",
894 | "\"\"\".format(\"./videos/\"+list(filter(lambda s: s.endswith(\".mp4\"), os.listdir(\"./videos/\")))[-1]))"
895 | ]
896 | },
897 | {
898 | "cell_type": "markdown",
899 | "metadata": {},
900 | "source": [
901 | "## Train Agent\n",
902 | "\n",
903 | "We will begin training our agent for 300 episodes, with a maximum iteration of 500. This will take some time, so be patient."
904 | ]
905 | },
906 | {
907 | "cell_type": "code",
908 | "execution_count": null,
909 | "metadata": {},
910 | "outputs": [],
911 | "source": [
912 | "episodes = 150\n",
913 | "iterations = 500\n",
914 | "\n",
915 | "train(dqn_agent, iterations, episodes)"
916 | ]
917 | },
918 | {
919 | "cell_type": "markdown",
920 | "metadata": {},
921 | "source": [
922 | "## Visualize performance\n",
923 | "Now lets visualize the performance of our trained agent!"
924 | ]
925 | },
926 | {
927 | "cell_type": "code",
928 | "execution_count": null,
929 | "metadata": {},
930 | "outputs": [],
931 | "source": [
932 | "generate_video(dqn_agent)\n",
933 | "HTML(\"\"\"\n",
934 | "\n",
937 | "\"\"\".format(\"./videos/\"+list(filter(lambda s: s.endswith(\".mp4\"), os.listdir(\"./videos/\")))[-1]))"
938 | ]
939 | },
940 | {
941 | "cell_type": "markdown",
942 | "metadata": {},
943 | "source": [
944 | "## And we are through!\n",
945 | "\n",
946 | "You have implemented the DQN-algorithm, congratulations!\n",
947 | "\n",
948 | "Try optimizing the hyperparameters to improve the training results. Suggestions:\n",
949 | "- Number of layers\n",
950 | "- Number of neurons in each layer\n",
951 | "- Learning rate\n",
952 | "- Epsilon strategy\n",
953 | "\n",
954 | "If you like to try a different environment, try the \"Acrobat-v1\" environment.\n",
955 | "\n",
956 | "Remember to turn of the assertion by running the cell below!"
957 | ]
958 | },
959 | {
960 | "cell_type": "code",
961 | "execution_count": null,
962 | "metadata": {},
963 | "outputs": [],
964 | "source": [
965 | "should_assert = False"
966 | ]
967 | }
968 | ],
969 | "metadata": {
970 | "kernelspec": {
971 | "display_name": "Python 3",
972 | "language": "python",
973 | "name": "python3"
974 | },
975 | "language_info": {
976 | "codemirror_mode": {
977 | "name": "ipython",
978 | "version": 3
979 | },
980 | "file_extension": ".py",
981 | "mimetype": "text/x-python",
982 | "name": "python",
983 | "nbconvert_exporter": "python",
984 | "pygments_lexer": "ipython3",
985 | "version": "3.7.3"
986 | }
987 | },
988 | "nbformat": 4,
989 | "nbformat_minor": 2
990 | }
991 |
--------------------------------------------------------------------------------