├── code ├── a3c │ ├── env8-a3c-rewards.npy │ ├── env8-a3c-durations.npy │ ├── shared_adam.py │ ├── a3cutils.py │ ├── A3C.py │ └── trainingA3C.py ├── experiments │ ├── ANIMATIONS │ │ ├── 78.gif │ │ ├── 457.gif │ │ ├── env4-solution.gif │ │ ├── env5-solution.gif │ │ ├── env7-solution.gif │ │ └── env8-solution.gif │ ├── env1-a3c-rewards.npy │ ├── env1-sql-rewards.npy │ ├── env1-dqn0-rewards.npy │ ├── env1-sql-durations.npy │ ├── env1-sql0-rewards.npy │ ├── env1-dqn0-durations.npy │ ├── env1-sql0-durations.npy │ ├── TEST_A3C │ │ ├── env1-a3c-rewards.npy │ │ ├── env4-a3c-rewards.npy │ │ ├── env5-a3c-rewards.npy │ │ ├── env7-a3c-rewards.npy │ │ ├── env8-a3c-rewards.npy │ │ ├── env1-a3c-durations.npy │ │ ├── env4-a3c-durations.npy │ │ ├── env5-a3c-durations.npy │ │ ├── env7-a3c-durations.npy │ │ └── env8-a3c-durations.npy │ ├── TEST_DQN │ │ ├── env1-dqn-rewards.npy │ │ ├── env4-dqn-rewards.npy │ │ ├── env5-dqn-rewards.npy │ │ ├── env7-dqn-rewards.npy │ │ ├── env8-dqn-rewards.npy │ │ ├── env1-dqn-durations.npy │ │ ├── env4-dqn-durations.npy │ │ ├── env5-dqn-durations.npy │ │ ├── env7-dqn-durations.npy │ │ └── env8-dqn-durations.npy │ ├── TEST_DQN0 │ │ ├── env1-dqn0-rewards.npy │ │ ├── env4-dqn0-rewards.npy │ │ ├── env5-dqn0-rewards.npy │ │ ├── env7-dqn0-rewards.npy │ │ ├── env8-dqn0-rewards.npy │ │ ├── env1-dqn0-durations.npy │ │ ├── env4-dqn0-durations.npy │ │ ├── env5-dqn0-durations.npy │ │ ├── env7-dqn0-durations.npy │ │ └── env8-dqn0-durations.npy │ ├── TEST_DISTRAL │ │ ├── Distral_2col-78-rewards.npy │ │ ├── Distral_2col-457-rewards.npy │ │ ├── Distral_2col-78-durations.npy │ │ ├── Distral_2col0-457-rewards.npy │ │ ├── Distral_2col-457-durations.npy │ │ ├── Distral_2col0-457-durations.npy │ │ ├── Benchmark78-distral-duration.eps │ │ └── Benchmark457-distral-duration.eps │ ├── testDistral.py │ ├── testDistral0.py │ ├── testA3C.py │ ├── testSQL.py │ ├── testDQN0.py │ ├── testSQL0.py │ ├── testDQN.py │ └── plot.py ├── dqn0 │ ├── __pycache__ │ │ ├── network.cpython-36.pyc │ │ ├── trainingDQN0.cpython-36.pyc │ │ └── memory_replay.cpython-36.pyc │ ├── memory_replay.py │ ├── network.py │ └── trainingDQN0.py ├── sql0 │ ├── __pycache__ │ │ ├── network.cpython-36.pyc │ │ ├── trainingSQL0.cpython-36.pyc │ │ └── memory_replay.cpython-36.pyc │ ├── memory_replay.py │ ├── network.py │ └── trainingSQL0.py ├── distral_1col0 │ ├── Distral_1col-distral0-rewards.npy │ ├── Distral_1col-distral0-duration.npy │ ├── distral.py │ └── distral1col.py ├── distral_2col0 │ ├── __pycache__ │ │ ├── network.cpython-36.pyc │ │ ├── memory_replay.cpython-36.pyc │ │ └── trainingDistral2col0.cpython-36.pyc │ ├── memory_replay.py │ ├── trainingDistral2col0.py │ └── network.py ├── envs │ ├── plan1.txt │ ├── plan3.txt │ ├── plan4.txt │ ├── plan5.txt │ ├── plan6.txt │ ├── plan7.txt │ ├── plan8.txt │ ├── plan2.txt │ ├── plotting.py │ └── gridworld_env.py ├── dqn │ ├── memory_replay.py │ ├── trainingDQN.py │ └── network.py ├── sql │ ├── memory_replay.py │ ├── trainingSQL.py │ └── network.py ├── distral_2col │ ├── memory_replay.py │ ├── trainingDistral2col.py │ └── network.py └── utils.py ├── .gitignore ├── LICENSE └── README.md /code/a3c/env8-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/a3c/env8-a3c-rewards.npy -------------------------------------------------------------------------------- /code/a3c/env8-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/a3c/env8-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/78.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/78.gif -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/457.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/457.gif -------------------------------------------------------------------------------- /code/experiments/env1-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/env1-sql-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-sql-rewards.npy -------------------------------------------------------------------------------- /code/experiments/env1-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/env1-sql-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-sql-durations.npy -------------------------------------------------------------------------------- /code/experiments/env1-sql0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-sql0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/env1-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-dqn0-durations.npy -------------------------------------------------------------------------------- /code/experiments/env1-sql0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/env1-sql0-durations.npy -------------------------------------------------------------------------------- /code/dqn0/__pycache__/network.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/dqn0/__pycache__/network.cpython-36.pyc -------------------------------------------------------------------------------- /code/sql0/__pycache__/network.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/sql0/__pycache__/network.cpython-36.pyc -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/env4-solution.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/env4-solution.gif -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/env5-solution.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/env5-solution.gif -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/env7-solution.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/env7-solution.gif -------------------------------------------------------------------------------- /code/experiments/ANIMATIONS/env8-solution.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/ANIMATIONS/env8-solution.gif -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env1-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env1-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env4-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env4-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env5-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env5-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env7-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env7-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env8-a3c-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env8-a3c-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env1-dqn-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env1-dqn-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env4-dqn-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env4-dqn-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env5-dqn-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env5-dqn-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env7-dqn-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env7-dqn-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env8-dqn-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env8-dqn-rewards.npy -------------------------------------------------------------------------------- /code/dqn0/__pycache__/trainingDQN0.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/dqn0/__pycache__/trainingDQN0.cpython-36.pyc -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env1-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env1-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env4-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env4-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env5-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env5-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env7-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env7-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_A3C/env8-a3c-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_A3C/env8-a3c-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env1-dqn-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env1-dqn-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env4-dqn-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env4-dqn-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env5-dqn-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env5-dqn-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env7-dqn-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env7-dqn-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN/env8-dqn-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN/env8-dqn-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env1-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env1-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env4-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env4-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env5-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env5-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env7-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env7-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env8-dqn0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env8-dqn0-rewards.npy -------------------------------------------------------------------------------- /code/sql0/__pycache__/trainingSQL0.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/sql0/__pycache__/trainingSQL0.cpython-36.pyc -------------------------------------------------------------------------------- /code/distral_1col0/Distral_1col-distral0-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/distral_1col0/Distral_1col-distral0-rewards.npy -------------------------------------------------------------------------------- /code/dqn0/__pycache__/memory_replay.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/dqn0/__pycache__/memory_replay.cpython-36.pyc -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env1-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env1-dqn0-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env4-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env4-dqn0-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env5-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env5-dqn0-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env7-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env7-dqn0-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DQN0/env8-dqn0-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DQN0/env8-dqn0-durations.npy -------------------------------------------------------------------------------- /code/sql0/__pycache__/memory_replay.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/sql0/__pycache__/memory_replay.cpython-36.pyc -------------------------------------------------------------------------------- /code/distral_1col0/Distral_1col-distral0-duration.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/distral_1col0/Distral_1col-distral0-duration.npy -------------------------------------------------------------------------------- /code/distral_2col0/__pycache__/network.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/distral_2col0/__pycache__/network.cpython-36.pyc -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col-78-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col-78-rewards.npy -------------------------------------------------------------------------------- /code/distral_2col0/__pycache__/memory_replay.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/distral_2col0/__pycache__/memory_replay.cpython-36.pyc -------------------------------------------------------------------------------- /code/envs/plan1.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 2 | 1 0 0 0 3 0 0 1 3 | 1 0 1 0 0 0 1 1 4 | 1 0 0 0 0 0 0 1 5 | 1 0 0 4 0 0 1 1 6 | 1 0 0 0 0 0 0 1 7 | 1 0 1 0 0 0 0 1 8 | 1 1 1 1 1 1 1 1 9 | -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col-457-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col-457-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col-78-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col-78-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col0-457-rewards.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col0-457-rewards.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col-457-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col-457-durations.npy -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Distral_2col0-457-durations.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/experiments/TEST_DISTRAL/Distral_2col0-457-durations.npy -------------------------------------------------------------------------------- /code/distral_2col0/__pycache__/trainingDistral2col0.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alfo5123/Robust-Multitask-RL/HEAD/code/distral_2col0/__pycache__/trainingDistral2col0.cpython-36.pyc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | code/__pycache__/* 2 | 3 | code/envs/__pycache__/* 4 | 5 | code/envs/.ipynb_checkpoints/* 6 | 7 | code/a3c/__pycache__/* 8 | 9 | code/dqn/__pycache__/* 10 | 11 | code/sql/__pycache__/* 12 | -------------------------------------------------------------------------------- /code/envs/plan3.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 2 | 1 0 0 0 1 0 0 3 1 3 | 1 0 1 0 0 0 1 1 1 4 | 1 0 0 0 1 0 0 0 1 5 | 1 0 0 0 0 0 0 1 1 6 | 1 0 0 1 0 0 0 0 1 7 | 1 0 0 0 0 1 0 0 1 8 | 1 0 0 4 0 0 0 0 1 9 | 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/envs/plan4.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 2 | 1 4 0 1 1 1 1 0 0 1 3 | 1 0 0 1 1 1 1 0 0 1 4 | 1 0 0 1 1 1 1 0 0 1 5 | 1 0 0 0 0 0 0 0 0 1 6 | 1 0 0 1 1 1 1 0 0 1 7 | 1 0 0 1 1 1 1 0 0 1 8 | 1 0 0 1 1 1 1 0 3 1 9 | 1 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/envs/plan5.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 2 | 1 4 0 1 1 1 1 0 0 1 3 | 1 0 0 1 1 1 1 0 0 1 4 | 1 0 0 1 1 1 1 0 0 1 5 | 1 0 0 0 0 0 0 3 0 1 6 | 1 0 0 1 1 1 1 0 0 1 7 | 1 0 0 1 1 1 1 0 0 1 8 | 1 0 0 1 1 1 1 0 0 1 9 | 1 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/envs/plan6.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 2 | 1 4 0 1 1 1 1 0 0 1 3 | 1 0 0 1 1 1 1 0 0 1 4 | 1 0 0 1 1 1 1 0 0 1 5 | 1 0 0 3 0 0 0 0 0 1 6 | 1 0 0 1 1 1 1 0 0 1 7 | 1 0 0 1 1 1 1 0 0 1 8 | 1 0 0 1 1 1 1 0 0 1 9 | 1 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/envs/plan7.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 2 | 1 4 0 1 1 1 1 0 3 1 3 | 1 0 0 1 1 1 1 0 0 1 4 | 1 0 0 1 1 1 1 0 0 1 5 | 1 0 0 0 0 0 0 0 0 1 6 | 1 0 0 1 1 1 1 0 0 1 7 | 1 0 0 1 1 1 1 0 0 1 8 | 1 0 0 1 1 1 1 0 0 1 9 | 1 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/envs/plan8.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 2 | 1 0 0 1 1 1 1 0 3 1 3 | 1 0 0 1 1 1 1 0 0 1 4 | 1 0 0 1 1 1 1 0 0 1 5 | 1 0 0 0 0 0 0 0 0 1 6 | 1 0 0 1 1 1 1 0 0 1 7 | 1 0 0 1 1 1 1 0 0 1 8 | 1 0 4 1 1 1 1 0 0 1 9 | 1 1 1 1 1 1 1 1 1 1 10 | -------------------------------------------------------------------------------- /code/experiments/testDistral.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | from utils import play_game 5 | sys.path.append('../distral_2col') 6 | import trainingDistral2col 7 | 8 | models, policy, episode_rewards, episode_durations = trainingDistral2col.trainD() -------------------------------------------------------------------------------- /code/experiments/testDistral0.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | from utils import play_game 5 | sys.path.append('../distral_2col0') 6 | import trainingDistral2col0 7 | 8 | models, policy, episode_rewards, episode_durations = trainingDistral2col0.trainD() 9 | -------------------------------------------------------------------------------- /code/experiments/testA3C.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | sys.path.append('../a3c') 5 | import trainingA3C 6 | 7 | import gym 8 | 9 | trainingA3C.trainA3C(file_name="env3", 10 | env= GridworldEnv(3), 11 | update_global_iter=10, 12 | gamma=0.95, 13 | is_plot=False, 14 | num_episodes=500, 15 | max_num_steps_per_episode=1000, 16 | learning_rate=0.001 ) 17 | -------------------------------------------------------------------------------- /code/envs/plan2.txt: -------------------------------------------------------------------------------- 1 | 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 | 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 3 | 1 1 0 4 0 1 1 1 1 1 1 0 0 0 1 1 4 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 5 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 6 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 7 | 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 8 | 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 9 | 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 1 10 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 11 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 12 | 1 1 0 0 0 1 1 1 1 1 1 0 0 0 1 1 13 | 1 1 0 0 0 1 1 1 1 1 1 0 3 0 1 1 14 | 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 15 | 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 -------------------------------------------------------------------------------- /code/experiments/testSQL.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | sys.path.append('../sql') 5 | import trainingSQL 6 | 7 | 8 | trainingSQL.trainSQL(file_name="env1", 9 | env=GridworldEnv(1), 10 | batch_size=128, 11 | gamma=0.95, 12 | beta=5, 13 | eps_start=0.9, 14 | eps_end=0.05, 15 | eps_decay=300, 16 | is_plot=False, 17 | num_episodes=500, 18 | max_num_steps_per_episode=1000, 19 | learning_rate=0.001, 20 | memory_replay_size=10000,) 21 | -------------------------------------------------------------------------------- /code/experiments/testDQN0.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | from utils import play_game 5 | sys.path.append('../dqn0') 6 | import trainingDQN0 7 | 8 | import gym 9 | agent, _, _ = trainingDQN0.trainDQN0(file_name="env8", 10 | env=GridworldEnv(8), 11 | batch_size=128, 12 | gamma=0.9, 13 | eps_start=0.9, 14 | eps_end=0.05, 15 | eps_decay=10000, 16 | is_plot=False, 17 | num_episodes=500, 18 | max_num_steps_per_episode=10000, 19 | learning_rate=0.001, 20 | memory_replay_size=10000, 21 | ) 22 | 23 | #play_game(GridworldEnv(1), agent) 24 | -------------------------------------------------------------------------------- /code/dqn/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | def push(self, *args): 15 | """Saves a transition.""" 16 | if len(self.memory) < self.capacity: 17 | self.memory.append(None) 18 | self.memory[self.position] = Transition(*args) 19 | self.position = (self.position + 1) % self.capacity 20 | 21 | def sample(self, batch_size): 22 | return random.sample(self.memory, batch_size) 23 | 24 | def __len__(self): 25 | return len(self.memory) 26 | 27 | -------------------------------------------------------------------------------- /code/dqn0/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | def push(self, *args): 15 | """Saves a transition.""" 16 | if len(self.memory) < self.capacity: 17 | self.memory.append(None) 18 | self.memory[self.position] = Transition(*args) 19 | self.position = (self.position + 1) % self.capacity 20 | 21 | def sample(self, batch_size): 22 | return random.sample(self.memory, batch_size) 23 | 24 | def __len__(self): 25 | return len(self.memory) 26 | 27 | -------------------------------------------------------------------------------- /code/sql/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | def push(self, *args): 15 | """Saves a transition.""" 16 | if len(self.memory) < self.capacity: 17 | self.memory.append(None) 18 | self.memory[self.position] = Transition(*args) 19 | self.position = (self.position + 1) % self.capacity 20 | 21 | def sample(self, batch_size): 22 | return random.sample(self.memory, batch_size) 23 | 24 | def __len__(self): 25 | return len(self.memory) 26 | 27 | -------------------------------------------------------------------------------- /code/sql0/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | def push(self, *args): 15 | """Saves a transition.""" 16 | if len(self.memory) < self.capacity: 17 | self.memory.append(None) 18 | self.memory[self.position] = Transition(*args) 19 | self.position = (self.position + 1) % self.capacity 20 | 21 | def sample(self, batch_size): 22 | return random.sample(self.memory, batch_size) 23 | 24 | def __len__(self): 25 | return len(self.memory) 26 | 27 | -------------------------------------------------------------------------------- /code/experiments/testSQL0.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | from utils import play_game 5 | sys.path.append('../sql0') 6 | import trainingSQL0 7 | 8 | import gym 9 | agent, _, _ = trainingSQL0.trainSQL0(file_name="env1", 10 | env=GridworldEnv(1), 11 | batch_size=128, 12 | gamma=0.90, ## To Tune 13 | beta=5.0, ## To tune 14 | eps_start=0.9, 15 | eps_end=0.05, 16 | eps_decay=10000, 17 | is_plot=False, 18 | num_episodes=500, 19 | max_num_steps_per_episode=10000, 20 | learning_rate=0.001, 21 | memory_replay_size=10000, 22 | ) 23 | 24 | #play_game(GridworldEnv(1), agent) 25 | -------------------------------------------------------------------------------- /code/a3c/shared_adam.py: -------------------------------------------------------------------------------- 1 | """ 2 | Shared optimizer, the parameters in the optimizer will shared in the multiprocessors. 3 | """ 4 | 5 | import torch 6 | 7 | 8 | class SharedAdam(torch.optim.Adam): 9 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.9), eps=1e-8, 10 | weight_decay=0): 11 | super(SharedAdam, self).__init__(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 12 | # State initialization 13 | for group in self.param_groups: 14 | for p in group['params']: 15 | state = self.state[p] 16 | state['step'] = 0 17 | state['exp_avg'] = torch.zeros_like(p.data) 18 | state['exp_avg_sq'] = torch.zeros_like(p.data) 19 | 20 | # share in memory 21 | state['exp_avg'].share_memory_() 22 | state['exp_avg_sq'].share_memory_() 23 | 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Alfredo de la Fuente Briceño 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /code/experiments/testDQN.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('../') 3 | from envs.gridworld_env import GridworldEnv 4 | from utils import play_game 5 | sys.path.append('../dqn') 6 | import trainingDQN 7 | 8 | 9 | 10 | # trainingDQN.trainDQN(file_name="env1", 11 | # env=GridworldEnv(1), 12 | # batch_size=128, 13 | # gamma=0.999, 14 | # eps_start=0.9, 15 | # eps_end=0.05, 16 | # eps_decay=1000, 17 | # is_plot=True, 18 | # num_episodes=500, 19 | # max_num_steps_per_episode=1000, 20 | # learning_rate=0.0001, 21 | # memory_replay_size=10000, 22 | # ) 23 | 24 | agent, _, _ = trainingDQN.trainDQN(file_name="env1", 25 | env=GridworldEnv(1), 26 | batch_size=128, 27 | gamma=0.999, 28 | eps_start=0.9, 29 | eps_end=0.05, 30 | eps_decay=1000, 31 | is_plot=False, 32 | num_episodes=500, 33 | max_num_steps_per_episode=1000, 34 | learning_rate=0.0001, 35 | memory_replay_size=10000, 36 | ) 37 | 38 | play_game(GridworldEnv(1), agent) 39 | -------------------------------------------------------------------------------- /code/distral_2col/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward', 'time')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity, policy_capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | self.policy_capacity = policy_capacity 15 | self.policy_memory = [] 16 | self.policy_position = 0 17 | 18 | def push(self, *args): 19 | """Saves a transition.""" 20 | if len(self.memory) < self.capacity: 21 | self.memory.append(None) 22 | self.memory[self.position] = Transition(*args) 23 | self.position = (self.position + 1) % self.capacity 24 | 25 | if len(self.policy_memory) < self.policy_capacity: 26 | self.policy_memory.append(None) 27 | self.policy_memory[self.policy_position] = Transition(*args) 28 | self.policy_position = (self.policy_position + 1) % self.policy_capacity 29 | 30 | def sample(self, batch_size): 31 | return random.sample(self.memory, batch_size) 32 | 33 | def policy_sample(self, batch_size): 34 | return random.sample(self.policy_memory, batch_size) 35 | 36 | def __len__(self): 37 | return len(self.memory) 38 | -------------------------------------------------------------------------------- /code/distral_2col0/memory_replay.py: -------------------------------------------------------------------------------- 1 | import random 2 | from collections import namedtuple 3 | 4 | Transition = namedtuple('Transition', 5 | ('state', 'action', 'next_state', 'reward', 'time')) 6 | 7 | class ReplayMemory(object): 8 | 9 | def __init__(self, capacity, policy_capacity): 10 | self.capacity = capacity 11 | self.memory = [] 12 | self.position = 0 13 | 14 | self.policy_capacity = policy_capacity 15 | self.policy_memory = [] 16 | self.policy_position = 0 17 | 18 | def push(self, *args): 19 | """Saves a transition.""" 20 | if len(self.memory) < self.capacity: 21 | self.memory.append(None) 22 | self.memory[self.position] = Transition(*args) 23 | self.position = (self.position + 1) % self.capacity 24 | 25 | if len(self.policy_memory) < self.policy_capacity: 26 | self.policy_memory.append(None) 27 | self.policy_memory[self.policy_position] = Transition(*args) 28 | self.policy_position = (self.policy_position + 1) % self.policy_capacity 29 | 30 | def sample(self, batch_size): 31 | return random.sample(self.memory, batch_size) 32 | 33 | def policy_sample(self, batch_size): 34 | return random.sample(self.policy_memory, batch_size) 35 | 36 | def __len__(self): 37 | return len(self.memory) 38 | -------------------------------------------------------------------------------- /code/experiments/plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | 5 | smoothing_window = 20 6 | 7 | dqn7_rewards = np.load('TEST_DQN/env7-dqn-rewards.npy')[:200] 8 | dqn8_rewards = np.load('TEST_DQN/env8-dqn-rewards.npy')[:200] 9 | 10 | distral7_rewards = np.load('TEST_DISTRAL/Distral_2col-78-rewards.npy')[0][:200] 11 | distral8_rewards = np.load('TEST_DISTRAL/Distral_2col-78-rewards.npy')[1][:200] 12 | 13 | dqn7_smooth = pd.Series(dqn7_rewards).rolling(smoothing_window,min_periods=smoothing_window).mean() 14 | dqn8_smooth = pd.Series(dqn8_rewards).rolling(smoothing_window,min_periods=smoothing_window).mean() 15 | 16 | dist7_smooth = pd.Series(distral7_rewards).rolling(smoothing_window,min_periods=smoothing_window).mean() 17 | dist8_smooth = pd.Series(distral8_rewards).rolling(smoothing_window,min_periods=smoothing_window).mean() 18 | 19 | plt.figure(figsize=(20,10)) 20 | plt.title('Benchmark Training Results DQN vs Distral', fontsize='20') 21 | plt.xlabel('Episodes ', fontsize='16') 22 | plt.ylabel('Reward', fontsize='16') 23 | 24 | 25 | plt.plot(dqn7_smooth, label="Env 7 - DQN") 26 | plt.plot(dqn8_smooth, label="Env 8 - DQN") 27 | 28 | plt.plot(dist7_smooth, label="Env 7 - DISTRAL") 29 | plt.plot(dist8_smooth, label="Env 8 - DISTRAL") 30 | 31 | plt.legend(loc='best', fontsize='20') 32 | plt.savefig('Benchmark-dqn-vs-distral78-reward.eps', format='eps', dpi=1000) 33 | # plt.show() 34 | # plt.close() 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Robust-Multitask-RL 2 | Machine Learning Course Project Skoltech 2018 3 | 4 | We implemented the Distral algorithm and replicated the results obtained in the recent Google DeepMind's paper: *"Distral: Robust Multitask Reinforcement Learning"* 5 | 6 | ![alt text](https://github.com/Alfo5123/Robust-Multitask-RL/blob/master/code/experiments/ANIMATIONS/457.gif "Distral1") 7 | 8 | ## Authors 9 | 10 | * **Anastasia Koloskova** 11 | * **Alfredo de la Fuente** 12 | 13 | ## Resources & References 14 | 15 | Papers: 16 | 17 | - [Distral: Robust Multitask Reinforcement Learning](https://arxiv.org/pdf/1707.04175.pdf) 18 | - [Actor-Mimic: Deep Multitask and Transfer Reinforcement Learning](https://arxiv.org/abs/1511.06342) 19 | - [Reinforcement Learning with Deep Energy-Based Policies](https://arxiv.org/pdf/1702.08165.pdf) 20 | - [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/pdf/1602.01783.pdf) 21 | - [Equivalence Between Policy Gradients and Soft Q-Learning](https://arxiv.org/pdf/1704.06440.pdf) 22 | - [Policy Gradient Methods for Reinforcement Learning with Function Approximation](https://papers.nips.cc/paper/1713-policy-gradient-methods-for-reinforcement-learning-with-function-approximation.pdf) 23 | 24 | Textbooks: 25 | 26 | - [Reinforcement Learning: An Introduction (2nd Edition)](http://incompleteideas.net/book/bookdraft2018jan1.pdf) 27 | 28 | Classes: 29 | 30 | - [David Silver's Reinforcement Learning Course (UCL, 2015)](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching.html) 31 | - [CS294 - Deep Reinforcement Learning (Berkeley, Fall 2015)](http://rll.berkeley.edu/deeprlcourse/) 32 | 33 | Repositories: 34 | 35 | - [Denny Britz's reinforcement learning compilation of algorithms and material](https://github.com/dennybritz/reinforcement-learning) 36 | - [OpenAI Gym](https://github.com/openai/gym) 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /code/a3c/a3cutils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions that use multiple times 3 | """ 4 | 5 | from torch import nn 6 | import torch 7 | import numpy as np 8 | from torch.autograd import Variable 9 | 10 | 11 | def v_wrap(np_array, dtype=np.float32): 12 | if np_array.dtype != dtype: 13 | np_array = np_array.astype(dtype) 14 | return Variable(torch.from_numpy(np_array)) 15 | 16 | 17 | def set_init(layers): 18 | for layer in layers: 19 | nn.init.normal(layer.weight, mean=0., std=0.1) 20 | nn.init.constant(layer.bias, 0.1) 21 | 22 | 23 | def push_and_pull(opt, lnet, gnet, done, s_, bs, ba, br, gamma): 24 | if done: 25 | v_s_ = 0. # terminal 26 | else: 27 | v_s_ = lnet.forward(v_wrap(s_[None, :]))[-1].data.numpy()[0, 0] 28 | 29 | buffer_v_target = [] 30 | for r in br[::-1]: # reverse buffer r 31 | v_s_ = r + gamma * v_s_ 32 | buffer_v_target.append(v_s_) 33 | buffer_v_target.reverse() 34 | 35 | loss = lnet.loss_func( 36 | v_wrap(np.vstack(bs)), 37 | v_wrap(np.array(ba), dtype=np.int64) if ba[0].dtype == np.int64 else v_wrap(np.vstack(ba)), 38 | v_wrap(np.array(buffer_v_target)[:, None])) 39 | 40 | # calculate local gradients and push local parameters to global 41 | opt.zero_grad() 42 | loss.backward() 43 | for lp, gp in zip(lnet.parameters(), gnet.parameters()): 44 | gp._grad = lp.grad 45 | opt.step() 46 | 47 | # pull global parameters 48 | lnet.load_state_dict(gnet.state_dict()) 49 | 50 | 51 | def record(global_ep, global_ep_r, ep_r, res_queue, name): 52 | with global_ep.get_lock(): 53 | global_ep.value += 1 54 | with global_ep_r.get_lock(): 55 | if global_ep_r.value == 0.: 56 | global_ep_r.value = ep_r 57 | else: 58 | global_ep_r.value = global_ep_r.value * 0.99 + ep_r * 0.01 59 | res_queue.put(global_ep_r.value) 60 | print( 61 | name, 62 | "Ep:", global_ep.value, 63 | "| Ep_r: %.2f" % global_ep_r.value, 64 | ) -------------------------------------------------------------------------------- /code/utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from PIL import Image 4 | import imageio 5 | import torch 6 | from torch.autograd import Variable 7 | use_cuda = torch.cuda.is_available() 8 | 9 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 10 | Tensor = FloatTensor 11 | 12 | def get_screen(env): 13 | screen = env.current_grid_map 14 | screen = np.ascontiguousarray(screen, dtype=np.float32) 15 | screen = torch.from_numpy(screen) 16 | return screen.unsqueeze(0).unsqueeze(0).type(Tensor) 17 | 18 | def play_game(env, agent, max_steps_num=100): #### To fix for dqn0, sql0 19 | 20 | keyFrames = [] 21 | env.reset() 22 | plt.ion() 23 | for steps_done in range(0, max_steps_num): 24 | plt.figure(1) 25 | state = get_screen(env) 26 | plt.imshow(state.cpu().squeeze(0).squeeze(0).numpy(), 27 | interpolation='none') 28 | 29 | filename = "env4_0" + str(steps_done) + ".png" 30 | plt.savefig(filename) 31 | keyFrames.append(filename) 32 | 33 | plt.draw() 34 | plt.pause(0.1) 35 | action = agent(Variable(state, volatile=True).type(FloatTensor)).data.max(1)[1].view(1, 1) 36 | _, reward, done, _ = env.step(action[0, 0]) 37 | 38 | if done: 39 | state = get_screen(env) 40 | plt.imshow(state.cpu().squeeze(0).squeeze(0).numpy(),interpolation='none') 41 | 42 | filename = "env4_0" + str(steps_done+1) + ".png" 43 | plt.savefig(filename) 44 | keyFrames.append(filename) 45 | break 46 | plt.ioff() 47 | 48 | images = [imageio.imread(fn) for fn in keyFrames] 49 | gifFilename = "env4-solution.gif" 50 | imageio.mimsave(gifFilename, images, duration=0.3) 51 | 52 | 53 | 54 | def plot_durations(episode_durations, mean_durations): 55 | plt.figure(2) 56 | plt.clf() 57 | # durations_t = torch.FloatTensor(episode_durations) 58 | plt.title('Durations, training...') 59 | plt.xlabel('Episode') 60 | plt.ylabel('Duration') 61 | plt.plot(episode_durations, label="durations") 62 | # Take 100 episode averages and plot them too 63 | # if len(episode_durations) >= 100: 64 | # means = durations_t.unfold(0, 100, 1).mean(1).view(-1) 65 | # means = torch.cat((torch.zeros(99), means)) 66 | mean_durations.append(np.mean(np.array(episode_durations)[::-1][:100])) 67 | plt.plot(mean_durations, label="means") 68 | plt.legend() 69 | 70 | plt.pause(0.001) # pause a bit so that plots are updated 71 | # if is_ipython: 72 | # display.clear_output(wait=True) 73 | # display.display(plt.gcf()) 74 | 75 | def plot_rewards(episode_rewards, mean_rewards): 76 | plt.figure(3) 77 | plt.clf() 78 | plt.title('Rewards, training') 79 | plt.xlabel('Episode') 80 | plt.ylabel('Reward') 81 | plt.plot(episode_rewards, label="Reward per Episode") 82 | mean_rewards.append(np.mean(np.array(episode_rewards)[::-1][:100])) 83 | plt.plot(mean_rewards, label="Mean reward") 84 | plt.legend() 85 | 86 | plt.pause(0.00001) # pause a bit so that plots are updated 87 | # if is_ipython: 88 | # display.clear_output(wait=True) 89 | # display.display(plt.gcf()) 90 | 91 | def plot_state(state): 92 | if state is not None: 93 | plt.figure(1) 94 | plt.clf() 95 | plt.imshow(state.cpu().squeeze(0).squeeze(0).numpy(), 96 | interpolation='none') 97 | plt.draw() 98 | plt.pause(0.000001) 99 | -------------------------------------------------------------------------------- /code/envs/plotting.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import numpy as np 3 | import pandas as pd 4 | from collections import namedtuple 5 | from matplotlib import pyplot as plt 6 | from mpl_toolkits.mplot3d import Axes3D 7 | 8 | EpisodeStats = namedtuple("Stats",["episode_lengths", "episode_rewards"]) 9 | 10 | def plot_cost_to_go_mountain_car(env, estimator, num_tiles=20): 11 | x = np.linspace(env.observation_space.low[0], env.observation_space.high[0], num=num_tiles) 12 | y = np.linspace(env.observation_space.low[1], env.observation_space.high[1], num=num_tiles) 13 | X, Y = np.meshgrid(x, y) 14 | Z = np.apply_along_axis(lambda _: -np.max(estimator.predict(_)), 2, np.dstack([X, Y])) 15 | 16 | fig = plt.figure(figsize=(10, 5)) 17 | ax = fig.add_subplot(111, projection='3d') 18 | surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, 19 | cmap=matplotlib.cm.coolwarm, vmin=-1.0, vmax=1.0) 20 | ax.set_xlabel('Position') 21 | ax.set_ylabel('Velocity') 22 | ax.set_zlabel('Value') 23 | ax.set_title("Mountain \"Cost To Go\" Function") 24 | fig.colorbar(surf) 25 | plt.show() 26 | 27 | 28 | def plot_value_function(V, title="Value Function"): 29 | """ 30 | Plots the value function as a surface plot. 31 | """ 32 | min_x = min(k[0] for k in V.keys()) 33 | max_x = max(k[0] for k in V.keys()) 34 | min_y = min(k[1] for k in V.keys()) 35 | max_y = max(k[1] for k in V.keys()) 36 | 37 | x_range = np.arange(min_x, max_x + 1) 38 | y_range = np.arange(min_y, max_y + 1) 39 | X, Y = np.meshgrid(x_range, y_range) 40 | 41 | # Find value for all (x, y) coordinates 42 | Z_noace = np.apply_along_axis(lambda _: V[(_[0], _[1], False)], 2, np.dstack([X, Y])) 43 | Z_ace = np.apply_along_axis(lambda _: V[(_[0], _[1], True)], 2, np.dstack([X, Y])) 44 | 45 | def plot_surface(X, Y, Z, title): 46 | fig = plt.figure(figsize=(20, 10)) 47 | ax = fig.add_subplot(111, projection='3d') 48 | surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, 49 | cmap=matplotlib.cm.coolwarm, vmin=-1.0, vmax=1.0) 50 | ax.set_xlabel('Player Sum') 51 | ax.set_ylabel('Dealer Showing') 52 | ax.set_zlabel('Value') 53 | ax.set_title(title) 54 | ax.view_init(ax.elev, -120) 55 | fig.colorbar(surf) 56 | plt.show() 57 | 58 | plot_surface(X, Y, Z_noace, "{} (No Usable Ace)".format(title)) 59 | plot_surface(X, Y, Z_ace, "{} (Usable Ace)".format(title)) 60 | 61 | 62 | 63 | def plot_episode_stats(stats, smoothing_window=10, noshow=False): 64 | # Plot the episode length over time 65 | fig1 = plt.figure(figsize=(10,5)) 66 | plt.plot(stats.episode_lengths) 67 | plt.xlabel("Episode") 68 | plt.ylabel("Episode Length") 69 | plt.title("Episode Length over Time") 70 | if noshow: 71 | plt.close(fig1) 72 | else: 73 | plt.show(fig1) 74 | 75 | # Plot the episode reward over time 76 | fig2 = plt.figure(figsize=(10,5)) 77 | rewards_smoothed = pd.Series(stats.episode_rewards).rolling(smoothing_window, min_periods=smoothing_window).mean() 78 | plt.plot(rewards_smoothed) 79 | plt.xlabel("Episode") 80 | plt.ylabel("Episode Reward (Smoothed)") 81 | plt.title("Episode Reward over Time (Smoothed over window size {})".format(smoothing_window)) 82 | if noshow: 83 | plt.close(fig2) 84 | else: 85 | plt.show(fig2) 86 | 87 | # Plot time steps and episode number 88 | fig3 = plt.figure(figsize=(10,5)) 89 | plt.plot(np.cumsum(stats.episode_lengths), np.arange(len(stats.episode_lengths))) 90 | plt.xlabel("Time Steps") 91 | plt.ylabel("Episode") 92 | plt.title("Episode per time step") 93 | if noshow: 94 | plt.close(fig3) 95 | else: 96 | plt.show(fig3) 97 | 98 | return fig1, fig2, fig3 99 | -------------------------------------------------------------------------------- /code/dqn0/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from memory_replay import Transition 9 | 10 | use_cuda = torch.cuda.is_available() 11 | 12 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 13 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 14 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 15 | Tensor = FloatTensor 16 | 17 | class DQN(nn.Module): 18 | """ 19 | Deep neural network with represents an agent. 20 | """ 21 | def __init__(self, input_size, num_actions): 22 | super(DQN, self).__init__() 23 | self.linear1 = nn.Linear(input_size, 50) 24 | self.linear2 = nn.Linear(50, 50) 25 | self.head = nn.Linear(50, num_actions) 26 | 27 | def forward(self, x): 28 | x = F.leaky_relu(self.linear1(x)) 29 | x = F.leaky_relu(self.linear2(x)) 30 | return self.head(x) 31 | 32 | 33 | def select_action(state, model, num_actions, 34 | EPS_START, EPS_END, EPS_DECAY, steps_done): 35 | """ 36 | Selects whether the next action is choosen by our model or randomly 37 | """ 38 | sample = random.random() 39 | eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 40 | math.exp(-1. * steps_done / EPS_DECAY) 41 | if sample > eps_threshold: 42 | return model( 43 | Variable(state, volatile=True).type(FloatTensor)).data.max(1)[1].view(1, 1) 44 | else: 45 | return LongTensor([[random.randrange(num_actions)]]) 46 | 47 | 48 | def optimize_model(model, optimizer, memory, BATCH_SIZE, GAMMA): 49 | global last_sync 50 | if len(memory) < BATCH_SIZE: 51 | return 52 | transitions = memory.sample(BATCH_SIZE) 53 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 54 | # detailed explanation). 55 | batch = Transition(*zip(*transitions)) 56 | 57 | # Compute a mask of non-final states and concatenate the batch elements 58 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 59 | batch.next_state))) 60 | 61 | # We don't want to backprop through the expected action values and volatile 62 | # will save us on temporarily changing the model parameters' 63 | # requires_grad to False! 64 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 65 | if s is not None]), 66 | volatile=True) 67 | state_batch = Variable(torch.cat(batch.state)) 68 | action_batch = Variable(torch.cat(batch.action)) 69 | reward_batch = Variable(torch.cat(batch.reward)) 70 | 71 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 72 | # columns of actions taken 73 | state_action_values = model(state_batch).gather(1, action_batch) 74 | 75 | # Compute V(s_{t+1}) for all next states. 76 | next_state_values = Variable(torch.zeros(BATCH_SIZE).type(Tensor)) 77 | next_state_values[non_final_mask] = model(non_final_next_states).max(1)[0] 78 | # Now, we don't want to mess up the loss with a volatile flag, so let's 79 | # clear it. After this, we'll just end up with a Variable that has 80 | # requires_grad=False 81 | next_state_values.volatile = False 82 | # Compute the expected Q values 83 | expected_state_action_values = (next_state_values * GAMMA) + reward_batch 84 | 85 | # Compute Huber loss 86 | loss = F.smooth_l1_loss(state_action_values, expected_state_action_values) 87 | 88 | # Optimize the model 89 | optimizer.zero_grad() 90 | loss.backward() 91 | for param in model.parameters(): 92 | param.grad.data.clamp_(-500, 500) 93 | optimizer.step() 94 | -------------------------------------------------------------------------------- /code/sql0/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from memory_replay import Transition 9 | 10 | use_cuda = torch.cuda.is_available() 11 | 12 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 13 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 14 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 15 | Tensor = FloatTensor 16 | 17 | class DQN(nn.Module): 18 | """ 19 | Deep neural network with represents an agent. 20 | """ 21 | def __init__(self, input_size, num_actions): 22 | super(DQN, self).__init__() 23 | self.linear1 = nn.Linear(input_size, 50) 24 | #self.linear2 = nn.Linear(50, 50) 25 | self.head = nn.Linear(50, num_actions) 26 | 27 | def forward(self, x): 28 | x = F.leaky_relu(self.linear1(x)) 29 | #x = F.leaky_relu(self.linear2(x)) 30 | return self.head(x) 31 | 32 | 33 | def select_action(state, model, num_actions, 34 | EPS_START, EPS_END, EPS_DECAY, steps_done): 35 | """ 36 | Selects whether the next action is choosen by our model or randomly 37 | """ 38 | sample = random.random() 39 | eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 40 | math.exp(-1. * steps_done / EPS_DECAY) 41 | if sample > eps_threshold: 42 | return model( 43 | Variable(state, volatile=True).type(FloatTensor)).data.max(1)[1].view(1, 1) 44 | else: 45 | return LongTensor([[random.randrange(num_actions)]]) 46 | 47 | 48 | def optimize_model(model, optimizer, memory, BATCH_SIZE, GAMMA, BETA): 49 | global last_sync 50 | if len(memory) < BATCH_SIZE: 51 | return 52 | transitions = memory.sample(BATCH_SIZE) 53 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 54 | # detailed explanation). 55 | batch = Transition(*zip(*transitions)) 56 | 57 | # Compute a mask of non-final states and concatenate the batch elements 58 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 59 | batch.next_state))) 60 | # We don't want to backprop through the expected action values and volatile 61 | # will save us on temporarily changing the model parameters' 62 | # requires_grad to False! 63 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 64 | if s is not None]), 65 | volatile=True) 66 | state_batch = Variable(torch.cat(batch.state)) 67 | action_batch = Variable(torch.cat(batch.action)) 68 | reward_batch = Variable(torch.cat(batch.reward)) 69 | 70 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 71 | # columns of actions taken 72 | state_action_values = model(state_batch).gather(1, action_batch) 73 | 74 | # Compute V(s_{t+1}) for all next states. 75 | next_state_values = Variable(torch.zeros(BATCH_SIZE).type(Tensor)) 76 | next_state_values[non_final_mask] = torch.log( torch.exp( 77 | BETA * model(non_final_next_states)).sum(1)) / BETA 78 | # Now, we don't want to mess up the loss with a volatile flag, so let's 79 | # clear it. After this, we'll just end up with a Variable that has 80 | # requires_grad=False 81 | next_state_values.volatile = False 82 | # Compute the expected Q values 83 | expected_state_action_values = (next_state_values * GAMMA) + reward_batch 84 | 85 | # Compute Huber loss 86 | loss = F.mse_loss(state_action_values, expected_state_action_values) 87 | 88 | # Optimize the model 89 | optimizer.zero_grad() 90 | loss.backward() 91 | for param in model.parameters(): 92 | param.grad.data.clamp_(-500, 500) 93 | optimizer.step() 94 | -------------------------------------------------------------------------------- /code/sql/trainingSQL.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor 10 | import sys 11 | sys.path.append('../') 12 | from envs.gridworld_env import GridworldEnv 13 | from utils import plot_rewards, plot_durations, plot_state, get_screen 14 | 15 | def trainSQL(file_name="SQL", env=GridworldEnv(1), batch_size=128, 16 | gamma=0.999, beta=5, eps_start=0.9, eps_end=0.05, eps_decay=1000, 17 | is_plot=False, num_episodes=500, max_num_steps_per_episode=1000, 18 | learning_rate=0.001, memory_replay_size=10000): 19 | """ 20 | Soft Q-learning training routine. Retuns rewards and durations logs. 21 | Plot environment screen 22 | """ 23 | if is_plot: 24 | env.reset() 25 | plt.ion() 26 | plt.figure() 27 | plt.imshow(get_screen(env).cpu().squeeze(0).squeeze(0).numpy(), 28 | interpolation='none') 29 | plt.draw() 30 | plt.pause(0.00001) 31 | 32 | num_actions = env.action_space.n 33 | model = DQN(num_actions) 34 | optimizer = optim.Adam(model.parameters(), lr=learning_rate) 35 | # optimizer = optim.RMSprop(model.parameters(), ) 36 | 37 | use_cuda = torch.cuda.is_available() 38 | if use_cuda: 39 | model.cuda() 40 | 41 | memory = ReplayMemory(memory_replay_size) 42 | 43 | episode_durations = [] 44 | mean_durations = [] 45 | episode_rewards = [] 46 | mean_rewards = [] 47 | 48 | steps_done, t = 0, 0 49 | # plt.ion() 50 | for i_episode in range(num_episodes): 51 | print("Cur episode:", i_episode, "steps done:", t, 52 | "exploration factor:", eps_end + (eps_start - eps_end) * \ 53 | math.exp(-1. * steps_done / eps_decay)) 54 | # Initialize the environment and state 55 | env.reset() 56 | # last_screen = env.current_grid_map 57 | current_screen = get_screen(env) 58 | state = current_screen # - last_screen 59 | for t in count(): 60 | # Select and perform an action 61 | action = select_action(state, model, num_actions, 62 | eps_start, eps_end, eps_decay, steps_done) 63 | _, reward, done, _ = env.step(action[0, 0]) 64 | reward = Tensor([reward]) 65 | 66 | # Observe new state 67 | last_screen = current_screen 68 | current_screen = get_screen(env) 69 | if not done: 70 | next_state = current_screen # - last_screen 71 | else: 72 | next_state = None 73 | 74 | # Store the transition in memory 75 | memory.push(state, action, next_state, reward) 76 | 77 | # Move to the next state 78 | state = next_state 79 | # plot_state(state) 80 | # env.render() 81 | 82 | # Perform one step of the optimization (on the target network) 83 | optimize_model(model, optimizer, memory, batch_size, gamma, beta) 84 | if done or t + 1 >= max_num_steps_per_episode: 85 | episode_durations.append(t + 1) 86 | episode_rewards.append(env.episode_total_reward) 87 | if is_plot: 88 | plot_durations(episode_durations, mean_durations) 89 | plot_rewards(episode_rewards, mean_rewards) 90 | steps_done += 1 91 | break 92 | 93 | print('Complete') 94 | env.render(close=True) 95 | env.close() 96 | if is_plot: 97 | plt.ioff() 98 | plt.show() 99 | 100 | ## Store Results 101 | 102 | np.save(file_name + '-sql-rewards', episode_rewards) 103 | np.save(file_name + '-sql-durations', episode_durations) 104 | 105 | return model, episode_rewards, episode_durations 106 | -------------------------------------------------------------------------------- /code/dqn/trainingDQN.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor 10 | import sys 11 | sys.path.append('../') 12 | from envs.gridworld_env import GridworldEnv 13 | from utils import plot_rewards, plot_durations, plot_state, get_screen 14 | from IPython.display import clear_output 15 | 16 | # env = GridworldEnv(1) 17 | # plt.ion() 18 | 19 | # optimizer = optim.RMSprop(model.parameters(), ) 20 | 21 | def trainDQN(file_name="DQN", env=GridworldEnv(1), batch_size=128, 22 | gamma=0.999, eps_start=0.9, eps_end=0.05, eps_decay=1000, 23 | is_plot=False, num_episodes=500, max_num_steps_per_episode=1000, 24 | learning_rate=0.0001, memory_replay_size=10000): 25 | """ 26 | DQN training routine. Retuns rewards and durations logs. 27 | Plot environment screen 28 | """ 29 | if is_plot: 30 | env.reset() 31 | plt.ion() 32 | plt.figure() 33 | plt.imshow(get_screen(env).cpu().squeeze(0).squeeze(0).numpy(), 34 | interpolation='none') 35 | plt.title("") 36 | plt.draw() 37 | plt.pause(0.00001) 38 | 39 | num_actions = env.action_space.n 40 | model = DQN(num_actions) 41 | optimizer = optim.Adam(model.parameters(), lr=learning_rate) 42 | 43 | use_cuda = torch.cuda.is_available() 44 | if use_cuda: 45 | model.cuda() 46 | 47 | memory = ReplayMemory(memory_replay_size) 48 | 49 | episode_durations = [] 50 | mean_durations = [] 51 | episode_rewards = [] 52 | mean_rewards = [] 53 | steps_done = 0 54 | for i_episode in range(num_episodes): 55 | if i_episode % 20 == 0: 56 | clear_output() 57 | print("Cur episode:", i_episode, "steps done:", steps_done, 58 | "exploration factor:", eps_end + (eps_start - eps_end) * \ 59 | math.exp(-1. * steps_done / eps_decay)) 60 | # Initialize the environment and state 61 | env.reset() 62 | # last_screen = env.current_grid_map 63 | current_screen = get_screen(env) 64 | state = current_screen # - last_screen 65 | for t in count(): 66 | # Select and perform an action 67 | action = select_action(state, model, num_actions, 68 | eps_start, eps_end, eps_decay, steps_done) 69 | steps_done += 1 70 | _, reward, done, _ = env.step(action[0, 0]) 71 | reward = Tensor([reward]) 72 | 73 | # Observe new state 74 | last_screen = current_screen 75 | current_screen = get_screen(env) 76 | if not done: 77 | next_state = current_screen # - last_screen 78 | else: 79 | next_state = None 80 | 81 | # Store the transition in memory 82 | memory.push(state, action, next_state, reward) 83 | 84 | # Move to the next state 85 | state = next_state 86 | # plot_state(state) 87 | # env.render() 88 | 89 | # Perform one step of the optimization (on the target network) 90 | optimize_model(model, optimizer, memory, batch_size, gamma) 91 | if done or t + 1 >= max_num_steps_per_episode: 92 | episode_durations.append(t + 1) 93 | episode_rewards.append(env.episode_total_reward) 94 | if is_plot: 95 | plot_durations(episode_durations, mean_durations) 96 | plot_rewards(episode_rewards, mean_rewards) 97 | break 98 | 99 | print('Complete') 100 | env.render(close=True) 101 | env.close() 102 | if is_plot: 103 | plt.ioff() 104 | plt.show() 105 | 106 | ## Store Results 107 | 108 | np.save(file_name + '-dqn-rewards', episode_rewards) 109 | np.save(file_name + '-dqn-durations', episode_durations) 110 | 111 | return model, episode_rewards, episode_durations 112 | -------------------------------------------------------------------------------- /code/sql0/trainingSQL0.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor 10 | import sys 11 | from IPython.display import clear_output 12 | sys.path.append('../') 13 | from envs.gridworld_env import GridworldEnv 14 | from utils import plot_rewards, plot_durations, plot_state, get_screen 15 | 16 | def trainSQL0(file_name="SQL0", env=GridworldEnv(1), batch_size=128, 17 | gamma=0.999, beta=5, eps_start=0.9, eps_end=0.05, eps_decay=1000, 18 | is_plot=False, num_episodes=500, max_num_steps_per_episode=1000, 19 | learning_rate=0.001, memory_replay_size=10000): 20 | """ 21 | Soft Q-learning training routine when observation vector is input 22 | Retuns rewards and durations logs. 23 | Plot environment screen 24 | """ 25 | if is_plot: 26 | env.reset() 27 | plt.ion() 28 | plt.figure() 29 | plt.imshow(get_screen(env).cpu().squeeze(0).squeeze(0).numpy(), 30 | interpolation='none') 31 | plt.draw() 32 | plt.pause(0.00001) 33 | 34 | num_actions = env.action_space.n 35 | input_size = env.observation_space.shape[0] 36 | model = DQN(input_size, num_actions) 37 | optimizer = optim.Adam(model.parameters(), lr=learning_rate) 38 | # optimizer = optim.RMSprop(model.parameters(), ) 39 | 40 | use_cuda = torch.cuda.is_available() 41 | if use_cuda: 42 | model.cuda() 43 | 44 | memory = ReplayMemory(memory_replay_size) 45 | 46 | episode_durations = [] 47 | mean_durations = [] 48 | episode_rewards = [] 49 | mean_rewards = [] 50 | 51 | steps_done, t = 0, 0 52 | # plt.ion() 53 | for i_episode in range(num_episodes): 54 | if i_episode % 20 == 0: 55 | clear_output() 56 | if i_episode != 0: 57 | print("Cur episode:", i_episode, "steps done:", episode_durations[-1], 58 | "exploration factor:", eps_end + (eps_start - eps_end) * \ 59 | math.exp(-1. * steps_done / eps_decay), "reward:", env.episode_total_reward) 60 | # Initialize the environment and state 61 | state = torch.from_numpy( env.reset() ).type(torch.FloatTensor).view(-1,input_size) 62 | 63 | for t in count(): 64 | # Select and perform an action 65 | action = select_action(state, model, num_actions, 66 | eps_start, eps_end, eps_decay, steps_done) 67 | next_state_tmp, reward, done, _ = env.step(action[0, 0]) 68 | reward = Tensor([reward]) 69 | 70 | # Observe new state 71 | next_state = torch.from_numpy( next_state_tmp ).type(torch.FloatTensor).view(-1,input_size) 72 | 73 | if done: 74 | next_state = None 75 | 76 | # Store the transition in memory 77 | memory.push(state, action, next_state, reward) 78 | 79 | # Move to the next state 80 | state = next_state 81 | # plot_state(state) 82 | # env.render() 83 | 84 | # Perform one step of the optimization (on the target network) 85 | optimize_model(model, optimizer, memory, batch_size, gamma, beta) #### Difference w.r.t DQN 86 | if done or t + 1 >= max_num_steps_per_episode: 87 | episode_durations.append(t + 1) 88 | episode_rewards.append(env.episode_total_reward) ##### Modify for OpenAI envs such as CartPole 89 | if is_plot: 90 | plot_durations(episode_durations, mean_durations) 91 | plot_rewards(episode_rewards, mean_rewards) 92 | steps_done += 1 93 | break 94 | 95 | print('Complete') 96 | env.render(close=True) 97 | env.close() 98 | if is_plot: 99 | plt.ioff() 100 | plt.show() 101 | 102 | ## Store Results 103 | 104 | np.save(file_name + '-sql0-rewards', episode_rewards) 105 | np.save(file_name + '-sql0-durations', episode_durations) 106 | 107 | return model, episode_rewards, episode_durations 108 | -------------------------------------------------------------------------------- /code/dqn0/trainingDQN0.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor 10 | import sys 11 | sys.path.append('../') 12 | from envs.gridworld_env import GridworldEnv 13 | from utils import plot_rewards, plot_durations, plot_state, get_screen 14 | from IPython.display import clear_output 15 | 16 | # env = GridworldEnv(1) 17 | # plt.ion() 18 | 19 | # optimizer = optim.RMSprop(model.parameters(), ) 20 | 21 | def trainDQN0(file_name="DQN0", env=GridworldEnv(1), batch_size=128, 22 | gamma=0.999, eps_start=0.9, eps_end=0.05, eps_decay=1000, 23 | is_plot=False, num_episodes=500, max_num_steps_per_episode=1000, 24 | learning_rate=0.0001, memory_replay_size=10000): 25 | """ 26 | DQN training routine when we recieve an observation vector as input 27 | Retuns rewards and durations logs. 28 | Plot environment screen 29 | """ 30 | 31 | if is_plot: 32 | env.reset() 33 | plt.ion() 34 | plt.figure() 35 | plt.imshow(get_screen(env).cpu().squeeze(0).squeeze(0).numpy(), 36 | interpolation='none') 37 | plt.title("") 38 | plt.draw() 39 | plt.pause(0.00001) 40 | 41 | num_actions = env.action_space.n 42 | input_size = env.observation_space.shape[0] 43 | model = DQN(input_size, num_actions) 44 | optimizer = optim.Adam(model.parameters(), lr=learning_rate) 45 | 46 | use_cuda = torch.cuda.is_available() 47 | if use_cuda: 48 | model.cuda() 49 | 50 | memory = ReplayMemory(memory_replay_size) 51 | 52 | episode_durations = [] 53 | mean_durations = [] 54 | episode_rewards = [] 55 | mean_rewards = [] 56 | steps_done = 0 57 | 58 | for i_episode in range(num_episodes): 59 | if i_episode % 20 == 0: 60 | clear_output() 61 | if i_episode != 0: 62 | print("Cur episode:", i_episode, "steps done:", episode_durations[-1], 63 | "exploration factor:", eps_end + (eps_start - eps_end) * \ 64 | math.exp(-1. * steps_done / eps_decay), "reward:", env.episode_total_reward) 65 | # Initialize the environment and state 66 | state = torch.from_numpy( env.reset() ).type(torch.FloatTensor).view(-1,input_size) 67 | 68 | for t in count(): 69 | # Select and perform an action 70 | action = select_action(state, model, num_actions, 71 | eps_start, eps_end, eps_decay, steps_done) 72 | steps_done += 1 73 | next_state_tmp, reward, done, _ = env.step(action[0, 0]) 74 | reward = Tensor([reward]) 75 | 76 | # Observe new state 77 | next_state = torch.from_numpy( next_state_tmp ).type(torch.FloatTensor).view(-1,input_size) 78 | if done: 79 | next_state = None 80 | 81 | # Store the transition in memory 82 | memory.push(state, action, next_state, reward) 83 | 84 | # Move to the next state 85 | state = next_state 86 | # plot_state(state) 87 | # env.render() 88 | 89 | # Perform one step of the optimization (on the target network) 90 | optimize_model(model, optimizer, memory, batch_size, gamma) 91 | if done or t + 1 >= max_num_steps_per_episode: 92 | episode_durations.append(t + 1) 93 | episode_rewards.append(env.episode_total_reward) ##### Modify for OpenAI envs such as CartPole 94 | if is_plot: 95 | plot_durations(episode_durations, mean_durations) 96 | plot_rewards(episode_rewards, mean_rewards) 97 | break 98 | 99 | print('Complete') 100 | env.render(close=True) 101 | env.close() 102 | if is_plot: 103 | plt.ioff() 104 | plt.show() 105 | 106 | ## Store Results 107 | 108 | np.save(file_name + '-dqn0-rewards', episode_rewards) 109 | np.save(file_name + '-dqn0-durations', episode_durations) 110 | 111 | return model, episode_rewards, episode_durations 112 | 113 | -------------------------------------------------------------------------------- /code/a3c/A3C.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import gym 4 | import numpy as np 5 | from itertools import count 6 | from collections import namedtuple 7 | 8 | import torch 9 | import torch.nn as nn 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch.autograd import Variable 13 | from torch.distributions import Categorical 14 | 15 | import sys 16 | sys.path.append('../') 17 | from envs.gridworld_env import GridworldEnv 18 | 19 | 20 | parser = argparse.ArgumentParser(description='PyTorch actor-critic example') 21 | parser.add_argument('--gamma', type=float, default=0.999, metavar='G', 22 | help='discount factor (default: 0.99)') 23 | parser.add_argument('--seed', type=int, default=543, metavar='N', 24 | help='random seed (default: 1)') 25 | parser.add_argument('--render', action='store_true', 26 | help='render the environment') 27 | parser.add_argument('--log-interval', type=int, default=10, metavar='N', 28 | help='interval between training status logs (default: 10)') 29 | args = parser.parse_args() 30 | 31 | 32 | #env = gym.make('CartPole-v0') 33 | 34 | env = GridworldEnv(8) 35 | env.seed(args.seed) 36 | #torch.manual_seed(args.seed) 37 | 38 | 39 | SavedAction = namedtuple('SavedAction', ['log_prob', 'value']) 40 | 41 | 42 | class Policy(nn.Module): 43 | def __init__(self): 44 | super(Policy, self).__init__() 45 | self.affine1 = nn.Linear(3, 128) 46 | self.action_head = nn.Linear(128, 5) 47 | self.value_head = nn.Linear(128, 1) 48 | 49 | self.saved_actions = [] 50 | self.rewards = [] 51 | 52 | def forward(self, x): 53 | x = F.relu(self.affine1(x)) 54 | action_scores = self.action_head(x) 55 | state_values = self.value_head(x) 56 | return F.softmax(action_scores, dim=-1), state_values 57 | 58 | 59 | model = Policy() 60 | optimizer = optim.Adam(model.parameters(), lr=0.009) 61 | 62 | 63 | def select_action(state): 64 | state = torch.from_numpy(state).float() 65 | probs, state_value = model(Variable(state)) 66 | m = Categorical(probs) 67 | action = m.sample() 68 | model.saved_actions.append(SavedAction(m.log_prob(action), state_value)) 69 | return action.data[0] 70 | 71 | 72 | def finish_episode(): 73 | R = 0 74 | saved_actions = model.saved_actions 75 | policy_losses = [] 76 | value_losses = [] 77 | rewards = [] 78 | for r in model.rewards[::-1]: 79 | R = r + args.gamma * R 80 | rewards.insert(0, R) 81 | rewards = torch.Tensor(rewards) 82 | rewards = (rewards - rewards.mean()) / (rewards.std() + np.finfo(np.float32).eps) 83 | for (log_prob, value), r in zip(saved_actions, rewards): 84 | reward = r - value.data[0] 85 | policy_losses.append(-log_prob * reward) 86 | value_losses.append(F.smooth_l1_loss(value, Variable(torch.Tensor([r])))) 87 | optimizer.zero_grad() 88 | loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum() 89 | loss.backward() 90 | for param in model.parameters(): 91 | param.grad.data.clamp_(-100, 100) 92 | optimizer.step() 93 | del model.rewards[:] 94 | del model.saved_actions[:] 95 | 96 | 97 | def main(): 98 | 99 | episode_rewards = [] 100 | episode_durations = [] 101 | for i_episode in range(500): 102 | 103 | total_reward = 0 104 | episode_duration = 0 105 | state = env.reset() 106 | for t in range(1000): # Don't infinite loop while learning 107 | action = select_action(state) 108 | state, reward, done, _ = env.step(action) 109 | total_reward += reward 110 | episode_duration += 1 111 | if args.render: 112 | env.render() 113 | model.rewards.append(reward) 114 | if done: 115 | break 116 | 117 | episode_rewards.append(total_reward) 118 | episode_durations.append(episode_duration) 119 | 120 | finish_episode() 121 | if i_episode % args.log_interval == 0: 122 | print('Episode {}\tLength: {:5d}\tTotal Reward: {:.2f}'.format( 123 | i_episode, episode_durations[-1] , episode_rewards[-1])) 124 | 125 | np.save('env8' + '-a3c-rewards', episode_rewards) 126 | np.save('env8' + '-a3c-durations', episode_durations) 127 | 128 | 129 | if __name__ == '__main__': 130 | main() -------------------------------------------------------------------------------- /code/dqn/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from memory_replay import Transition 9 | 10 | use_cuda = torch.cuda.is_available() 11 | 12 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 13 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 14 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 15 | Tensor = FloatTensor 16 | 17 | class DQN(nn.Module): 18 | """ 19 | Deep neural network with represents an agent. 20 | """ 21 | def __init__(self, num_actions): 22 | super(DQN, self).__init__() 23 | self.conv1 = nn.Conv2d(1, 5, kernel_size=2) 24 | self.bn1 = nn.BatchNorm2d(5) 25 | self.conv2 = nn.Conv2d(5, 10, kernel_size=3) 26 | self.bn2 = nn.BatchNorm2d(10) 27 | self.conv3 = nn.Conv2d(10, 10, kernel_size=3) 28 | self.bn3 = nn.BatchNorm2d(10) 29 | self.head = nn.Linear(200, num_actions) 30 | 31 | def forward(self, x): 32 | x = F.leaky_relu(self.bn1(self.conv1(x))) 33 | x = F.leaky_relu(self.bn2(self.conv2(x))) 34 | x = F.leaky_relu(self.bn3(self.conv3(x))) 35 | return self.head(x.view(x.size(0), -1)) 36 | 37 | # class DQN(nn.Module): 38 | # """ 39 | # Deep neural network with represents an agent. 40 | # """ 41 | # def __init__(self, num_actions): 42 | # super(DQN, self).__init__() 43 | # self.conv1 = nn.Conv2d(1, 5, kernel_size=2) 44 | # self.max_pool = nn.MaxPool2d((2,2)) 45 | # self.bn1 = nn.BatchNorm2d(5) 46 | # self.conv2 = nn.Conv2d(5, 20, kernel_size=3) 47 | # self.bn2 = nn.BatchNorm2d(20) 48 | # self.linear = nn.Linear(80, 20) 49 | # # self.bn3 = nn.BatchNorm1d(50) 50 | # self.head = nn.Linear(20, num_actions) 51 | 52 | # def forward(self, x): 53 | # x = F.leaky_relu(self.max_pool(self.bn1(self.conv1(x)))) 54 | # x = F.leaky_relu((self.bn2(self.conv2(x)))) 55 | # x = F.leaky_relu(self.linear(x.view(x.size(0), -1))) 56 | # return self.head(x) 57 | 58 | def select_action(state, model, num_actions, 59 | EPS_START, EPS_END, EPS_DECAY, steps_done): 60 | """ 61 | Selects whether the next action is choosen by our model or randomly 62 | """ 63 | sample = random.random() 64 | eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 65 | math.exp(-1. * steps_done / EPS_DECAY) 66 | if sample > eps_threshold: 67 | return model( 68 | Variable(state, volatile=True).type(FloatTensor)).data.max(1)[1].view(1, 1) 69 | else: 70 | return LongTensor([[random.randrange(num_actions)]]) 71 | 72 | 73 | def optimize_model(model, optimizer, memory, BATCH_SIZE, GAMMA): 74 | global last_sync 75 | if len(memory) < BATCH_SIZE: 76 | return 77 | transitions = memory.sample(BATCH_SIZE) 78 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 79 | # detailed explanation). 80 | batch = Transition(*zip(*transitions)) 81 | 82 | # Compute a mask of non-final states and concatenate the batch elements 83 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 84 | batch.next_state))) 85 | 86 | # We don't want to backprop through the expected action values and volatile 87 | # will save us on temporarily changing the model parameters' 88 | # requires_grad to False! 89 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 90 | if s is not None]), 91 | volatile=True) 92 | state_batch = Variable(torch.cat(batch.state)) 93 | action_batch = Variable(torch.cat(batch.action)) 94 | reward_batch = Variable(torch.cat(batch.reward)) 95 | 96 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 97 | # columns of actions taken 98 | state_action_values = model(state_batch).gather(1, action_batch) 99 | 100 | # Compute V(s_{t+1}) for all next states. 101 | next_state_values = Variable(torch.zeros(BATCH_SIZE).type(Tensor)) 102 | next_state_values[non_final_mask] = model(non_final_next_states).max(1)[0] 103 | # Now, we don't want to mess up the loss with a volatile flag, so let's 104 | # clear it. After this, we'll just end up with a Variable that has 105 | # requires_grad=False 106 | next_state_values.volatile = False 107 | # Compute the expected Q values 108 | expected_state_action_values = (next_state_values * GAMMA) + reward_batch 109 | 110 | # Compute Huber loss 111 | loss = F.smooth_l1_loss(state_action_values, expected_state_action_values) 112 | 113 | # Optimize the model 114 | optimizer.zero_grad() 115 | loss.backward() 116 | for param in model.parameters(): 117 | param.grad.data.clamp_(-1, 1) 118 | optimizer.step() 119 | -------------------------------------------------------------------------------- /code/sql/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | import torch 4 | import torch.nn as nn 5 | import torch.optim as optim 6 | import torch.nn.functional as F 7 | from torch.autograd import Variable 8 | from memory_replay import Transition 9 | 10 | use_cuda = torch.cuda.is_available() 11 | 12 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 13 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 14 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 15 | Tensor = FloatTensor 16 | 17 | 18 | class DQN(nn.Module): 19 | """ 20 | Deep neural network with represents an agent. 21 | """ 22 | def __init__(self, num_actions): 23 | super(DQN, self).__init__() 24 | self.conv1 = nn.Conv2d(1, 5, kernel_size=2) 25 | self.bn1 = nn.BatchNorm2d(5) 26 | self.conv2 = nn.Conv2d(5, 10, kernel_size=3) 27 | self.bn2 = nn.BatchNorm2d(10) 28 | self.conv3 = nn.Conv2d(10, 10, kernel_size=3) 29 | self.bn3 = nn.BatchNorm2d(10) 30 | self.head = nn.Linear(200, num_actions) 31 | 32 | def forward(self, x): 33 | x = F.leaky_relu(self.bn1(self.conv1(x))) 34 | x = F.leaky_relu(self.bn2(self.conv2(x))) 35 | x = F.leaky_relu(self.bn3(self.conv3(x))) 36 | return self.head(x.view(x.size(0), -1)) 37 | 38 | # class DQN(nn.Module): 39 | # """ 40 | # Deep neural network with represents an agent. 41 | # """ 42 | # def __init__(self, num_actions): 43 | # super(DQN, self).__init__() 44 | # self.conv1 = nn.Conv2d(1, 10, kernel_size=2) 45 | # self.max_pool = nn.MaxPool2d((2,2)) 46 | # self.bn1 = nn.BatchNorm2d(10) 47 | # self.conv2 = nn.Conv2d(10, 20, kernel_size=3) 48 | # self.bn2 = nn.BatchNorm2d(20) 49 | # self.linear = nn.Linear(80, 20) 50 | # # self.bn3 = nn.BatchNorm1d(50) 51 | # self.head = nn.Linear(20, num_actions) 52 | 53 | # def forward(self, x): 54 | # x = F.leaky_relu(self.max_pool(self.bn1(self.conv1(x)))) 55 | # x = F.leaky_relu(self.bn2(self.conv2(x))) 56 | # x = F.leaky_relu(self.linear(x.view(x.size(0), -1))) 57 | # return self.head(x) 58 | 59 | def select_action(state, model, num_actions, 60 | EPS_START, EPS_END, EPS_DECAY, steps_done): 61 | """ 62 | Selects whether the next action is choosen by our model or randomly 63 | """ 64 | sample = random.random() 65 | eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 66 | math.exp(-1. * steps_done / EPS_DECAY) 67 | if sample > eps_threshold: 68 | return model( 69 | Variable(state, volatile=True).type(FloatTensor)).data.max(1)[1].view(1, 1) 70 | else: 71 | return LongTensor([[random.randrange(num_actions)]]) 72 | 73 | 74 | def optimize_model(model, optimizer, memory, BATCH_SIZE, GAMMA, BETA): 75 | global last_sync 76 | if len(memory) < BATCH_SIZE: 77 | return 78 | transitions = memory.sample(BATCH_SIZE) 79 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 80 | # detailed explanation). 81 | batch = Transition(*zip(*transitions)) 82 | 83 | # Compute a mask of non-final states and concatenate the batch elements 84 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 85 | batch.next_state))) 86 | # We don't want to backprop through the expected action values and volatile 87 | # will save us on temporarily changing the model parameters' 88 | # requires_grad to False! 89 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 90 | if s is not None]), 91 | volatile=True) 92 | state_batch = Variable(torch.cat(batch.state)) 93 | action_batch = Variable(torch.cat(batch.action)) 94 | reward_batch = Variable(torch.cat(batch.reward)) 95 | 96 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 97 | # columns of actions taken 98 | state_action_values = model(state_batch).gather(1, action_batch) 99 | 100 | # Compute V(s_{t+1}) for all next states. 101 | next_state_values = Variable(torch.zeros(BATCH_SIZE).type(Tensor)) 102 | next_state_values[non_final_mask] = torch.log( torch.exp( 103 | BETA * model(non_final_next_states)).sum(1)) / BETA 104 | # Now, we don't want to mess up the loss with a volatile flag, so let's 105 | # clear it. After this, we'll just end up with a Variable that has 106 | # requires_grad=False 107 | next_state_values.volatile = False 108 | # Compute the expected Q values 109 | expected_state_action_values = (next_state_values * GAMMA) + reward_batch 110 | 111 | # Compute Huber loss 112 | loss = F.mse_loss(state_action_values, expected_state_action_values) 113 | 114 | # Optimize the model 115 | optimizer.zero_grad() 116 | loss.backward() 117 | for param in model.parameters(): 118 | param.grad.data.clamp_(-1, 1) 119 | optimizer.step() 120 | -------------------------------------------------------------------------------- /code/distral_2col/trainingDistral2col.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor, optimize_policy, PolicyNetwork 10 | import sys 11 | sys.path.append('../') 12 | from envs.gridworld_env import GridworldEnv 13 | from utils import plot_rewards, plot_durations, plot_state, get_screen 14 | 15 | def trainD(file_name="Distral_1col", list_of_envs=[GridworldEnv(4), 16 | GridworldEnv(5)], batch_size=128, gamma=0.999, alpha=0.9, 17 | beta=5, eps_start=0.9, eps_end=0.05, eps_decay=5, 18 | is_plot=False, num_episodes=200, 19 | max_num_steps_per_episode=1000, learning_rate=0.001, 20 | memory_replay_size=10000, memory_policy_size=1000): 21 | """ 22 | Soft Q-learning training routine. Retuns rewards and durations logs. 23 | Plot environment screen 24 | """ 25 | num_actions = list_of_envs[0].action_space.n 26 | num_envs = len(list_of_envs) 27 | policy = PolicyNetwork(num_actions) 28 | models = [DQN(num_actions) for _ in range(0, num_envs)] ### Add torch.nn.ModuleList (?) 29 | memories = [ReplayMemory(memory_replay_size, memory_policy_size) for _ in range(0, num_envs)] 30 | 31 | use_cuda = torch.cuda.is_available() 32 | if use_cuda: 33 | policy.cuda() 34 | for model in models: 35 | model.cuda() 36 | 37 | optimizers = [optim.Adam(model.parameters(), lr=learning_rate) 38 | for model in models] 39 | policy_optimizer = optim.Adam(policy.parameters(), lr=learning_rate) 40 | # optimizer = optim.RMSprop(model.parameters(), ) 41 | 42 | episode_durations = [[] for _ in range(num_envs)] 43 | episode_rewards = [[] for _ in range(num_envs)] 44 | 45 | steps_done = np.zeros(num_envs) 46 | episodes_done = np.zeros(num_envs) 47 | current_time = np.zeros(num_envs) 48 | 49 | # Initialize environments 50 | for env in list_of_envs: 51 | env.reset() 52 | 53 | while np.min(episodes_done) < num_episodes: 54 | # TODO: add max_num_steps_per_episode 55 | 56 | # Optimization is given by alterating minimization scheme: 57 | # 1. do the step for each env 58 | # 2. do one optimization step for each env using "soft-q-learning". 59 | # 3. do one optimization step for the policy 60 | 61 | for i_env, env in enumerate(list_of_envs): 62 | # print("Cur episode:", i_episode, "steps done:", steps_done, 63 | # "exploration factor:", eps_end + (eps_start - eps_end) * \ 64 | # math.exp(-1. * steps_done / eps_decay)) 65 | 66 | # last_screen = env.current_grid_map 67 | current_screen = get_screen(env) 68 | state = current_screen # - last_screen 69 | # Select and perform an action 70 | action = select_action(state, policy, models[i_env], num_actions, 71 | eps_start, eps_end, eps_decay, 72 | episodes_done[i_env], alpha, beta) 73 | steps_done[i_env] += 1 74 | current_time[i_env] += 1 75 | _, reward, done, _ = env.step(action[0, 0]) 76 | reward = Tensor([reward]) 77 | 78 | # Observe new state 79 | last_screen = current_screen 80 | current_screen = get_screen(env) 81 | if not done: 82 | next_state = current_screen # - last_screen 83 | else: 84 | next_state = None 85 | 86 | # Store the transition in memory 87 | time = Tensor([current_time[i_env]]) 88 | memories[i_env].push(state, action, next_state, reward, time) 89 | 90 | # Perform one step of the optimization (on the target network) 91 | optimize_model(policy, models[i_env], optimizers[i_env], 92 | memories[i_env], batch_size, alpha, beta, gamma) 93 | if done: 94 | print("ENV:", i_env, "iter:", episodes_done[i_env], 95 | "\treward:", env.episode_total_reward, 96 | "\tit:", current_time[i_env], "\texp_factor:", eps_end + 97 | (eps_start - eps_end) * math.exp(-1. * episodes_done[i_env] / eps_decay)) 98 | env.reset() 99 | episodes_done[i_env] += 1 100 | episode_durations[i_env].append(current_time[i_env]) 101 | current_time[i_env] = 0 102 | episode_rewards[i_env].append(env.episode_total_reward) 103 | if is_plot: 104 | plot_rewards(episode_rewards, i_env) 105 | 106 | 107 | optimize_policy(policy, policy_optimizer, memories, batch_size, 108 | num_envs, gamma) 109 | 110 | print('Complete') 111 | env.render(close=True) 112 | env.close() 113 | if is_plot: 114 | plt.ioff() 115 | plt.show() 116 | 117 | ## Store Results 118 | 119 | np.save(file_name + '-distral-2col-rewards', episode_rewards) 120 | np.save(file_name + '-distral-2col-durations', episode_durations) 121 | 122 | return models, policy, episode_rewards, episode_durations 123 | -------------------------------------------------------------------------------- /code/distral_1col0/distral.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gym 3 | import numpy as np 4 | from itertools import count 5 | from collections import namedtuple 6 | 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch.autograd import Variable 12 | from torch.distributions import Categorical 13 | 14 | 15 | parser = argparse.ArgumentParser(description='PyTorch distral example') 16 | parser.add_argument('--gamma', type=float, default=0.99, metavar='G', 17 | help='discount factor (default: 0.99)') 18 | parser.add_argument('--seed', type=int, default=543, metavar='N', 19 | help='random seed (default: 1)') 20 | parser.add_argument('--log-interval', type=int, default=10, metavar='N', 21 | help='interval between training status logs (default: 10)') 22 | args = parser.parse_args() 23 | 24 | import sys 25 | sys.path.append('../') 26 | from envs.gridworld_env import GridworldEnv 27 | 28 | env = gym.make('CartPole-v0') 29 | env.seed(args.seed) 30 | torch.manual_seed(args.seed) 31 | 32 | 33 | SavedAction = namedtuple('SavedAction', ['log_prob', 'value']) 34 | 35 | 36 | class Distral(nn.Module): 37 | 38 | def __init__(self, tasks = 2 ): 39 | 40 | super(Distral, self).__init__() 41 | self.affines = torch.nn.ModuleList ( [ nn.Linear(4, 128) for i in range(tasks+1) ] ) 42 | self.action_head = torch.nn.ModuleList ( [ nn.Linear(128, 2) for i in range(tasks+1) ] ) 43 | self.value_head = nn.Linear(128, 1) 44 | 45 | self.saved_actions = [[] for i in range(tasks+1)] 46 | self.rewards = [[] for i in range(tasks+1)] 47 | self.tasks = tasks 48 | 49 | def forward(self, x): 50 | 51 | action_scores = torch.cat( [ F.softmax(self.action_head[i](F.relu(self.affines[i](x))), dim=-1) for i in range(self.tasks+1) ]) 52 | state_values = self.value_head(F.relu(self.affines[0](x))) 53 | return action_scores.view(self.tasks+1,-1) , state_values 54 | 55 | 56 | model = Distral( ) 57 | optimizer = optim.Adam(model.parameters(), lr=3e-2) 58 | 59 | 60 | def select_action(state, tasks): 61 | state = torch.from_numpy(state).float() 62 | probs, state_value = model(Variable(state)) 63 | 64 | # Obtain the most probable action for each one of the policies 65 | actions = [] 66 | for i in range(tasks+1): 67 | m = Categorical(probs[i]) 68 | actions.append( m.sample() ) 69 | model.saved_actions[i].append(SavedAction(m.log_prob(actions[i]), state_value)) 70 | 71 | return torch.cat( actions ) 72 | 73 | 74 | def finish_episode( tasks , alpha , beta , gamma ): 75 | 76 | ### Calculate loss function according to Equation 1 77 | R = 0 78 | saved_actions = model.saved_actions[1] 79 | policy_losses = [] 80 | value_losses = [] 81 | 82 | ## Obtain the discounted rewards backwards 83 | rewards = [] 84 | for r in model.rewards[1][::-1]: 85 | R = r + gamma * R 86 | rewards.insert(0, R) 87 | 88 | ## Standardize the rewards to be unit normal (to control the gradient estimator variance) 89 | rewards = torch.Tensor(rewards) 90 | rewards = (rewards - rewards.mean()) / (rewards.std() + np.finfo(np.float32).eps) 91 | 92 | for (log_prob, value), r in zip(saved_actions, rewards): 93 | reward = r - value.data[0] 94 | policy_losses.append(-log_prob * reward) 95 | value_losses.append(F.smooth_l1_loss(value, Variable(torch.Tensor([r])))) 96 | 97 | 98 | optimizer.zero_grad() 99 | loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum() 100 | loss.backward() 101 | optimizer.step() 102 | 103 | #Clean memory 104 | for i in range(tasks+1): 105 | del model.rewards[i][:] 106 | del model.saved_actions[i][:] 107 | 108 | 109 | def trainDistral( file_name="Distral", envs=[gym.make('CartPole-v0').unwrapped], 110 | batch_size=128, alpha = 0.5 , beta = 0.5, gamma=0.999, is_plot=False, 111 | num_episodes=500, max_num_steps_per_episode=10000, learning_rate=0.001 ): 112 | 113 | #Run each one of the policies for the different environments 114 | #update the policies 115 | 116 | tasks = len(envs) 117 | 118 | for env in envs: ### From task 1 to N 119 | for i_episode in count(1): 120 | total_reward = 0 121 | state = env.reset() 122 | for t in range(max_num_steps_per_episode): # Don't infinite loop while learning 123 | action = select_action(state, tasks ) 124 | state, reward, done, _ = env.step(action.data[1]) 125 | if is_plot: 126 | env.render() 127 | model.rewards[1].append(reward) 128 | total_reward += reward 129 | if done: 130 | break 131 | 132 | finish_episode( tasks , alpha , beta, gamma ) 133 | if i_episode % args.log_interval == 0: 134 | print('Episode {}\tLast length: {:5d}\tTotal Reward: {:.2f}'.format( 135 | i_episode, t, total_reward)) 136 | if total_reward > 200: 137 | print("Solved! Total reward is now {} and " 138 | "the last episode runs to {} time steps!".format(total_reward, t)) 139 | break 140 | 141 | 142 | if __name__ == '__main__': 143 | trainDistral() -------------------------------------------------------------------------------- /code/a3c/trainingA3C.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from a3cutils import v_wrap, set_init, push_and_pull, record 4 | import torch.nn.functional as F 5 | import torch.multiprocessing as mp 6 | from shared_adam import SharedAdam 7 | import gym 8 | import sys 9 | sys.path.append('../') 10 | from envs.gridworld_env import GridworldEnv 11 | import numpy as np 12 | 13 | 14 | class Net(nn.Module): 15 | def __init__(self, s_dim, a_dim): 16 | super(Net, self).__init__() 17 | self.s_dim = s_dim 18 | self.a_dim = a_dim 19 | self.pi1 = nn.Linear(s_dim, 50) 20 | self.pi2 = nn.Linear(50,50) 21 | self.pi3 = nn.Linear(50, a_dim) 22 | self.v1 = nn.Linear(s_dim, 50) 23 | self.v2 = nn.Linear(50,50) 24 | self.v3 = nn.Linear(50, 1) 25 | set_init([self.pi1, self.pi2, self.pi3, self.v1, self.v2, self.v3]) 26 | self.distribution = torch.distributions.Categorical 27 | 28 | def forward(self, x): 29 | pi1 = F.relu(self.pi1(x)) 30 | logits = self.pi3(F.relu(self.pi2(pi1))) 31 | v1 = F.relu(self.v1(x)) 32 | values = self.v3(F.relu(self.v2(v1))) 33 | return logits, values 34 | 35 | def choose_action(self, s): 36 | self.eval() 37 | logits, _ = self.forward(s) 38 | prob = F.softmax(logits, dim=1).data 39 | m = self.distribution(prob) 40 | return m.sample().numpy()[0] 41 | 42 | def loss_func(self, s, a, v_t): 43 | self.train() 44 | logits, values = self.forward(s) 45 | td = v_t - values 46 | c_loss = td.pow(2) 47 | 48 | probs = F.softmax(logits, dim=1) 49 | m = self.distribution(probs) 50 | exp_v = m.log_prob(a) * td.detach() 51 | a_loss = -exp_v 52 | total_loss = (c_loss + a_loss).mean() 53 | return total_loss 54 | 55 | 56 | class Worker(mp.Process): 57 | 58 | def __init__(self, gnet, opt, global_ep, global_ep_r, res_queue, name, \ 59 | update_global_iter , num_episodes , max_num_steps_per_episode, \ 60 | gamma, env, ns, na ): 61 | 62 | super(Worker, self).__init__() 63 | self.name = 'w%i' % name 64 | self.g_ep, self.g_ep_r, self.res_queue = global_ep, global_ep_r, res_queue 65 | self.gnet, self.opt = gnet, opt 66 | 67 | self.ns = ns 68 | self.na = na 69 | 70 | self.lnet = Net(ns, na) # local network 71 | self.env = env 72 | 73 | self.update_global_iter = update_global_iter 74 | self.num_episodes = num_episodes 75 | self.max_num_steps_per_episode = max_num_steps_per_episode 76 | self.gamma = gamma 77 | 78 | 79 | def run(self): 80 | 81 | total_step = 1 82 | while self.g_ep.value < self.num_episodes: 83 | #s = np.reshape( self.env.reset() , ( self.ns, 1 ) ).flatten() ## Line to fix for arbitrary 84 | s = self.env.reset() 85 | 86 | buffer_s, buffer_a, buffer_r = [], [], [] 87 | ep_r = 0.0 88 | 89 | for _ in range(self.max_num_steps_per_episode): 90 | 91 | a = self.lnet.choose_action(v_wrap(s[None,:])) 92 | s_, r, done, _ = self.env.step(a) 93 | #s_ = np.reshape( s_ , ( self.ns, 1 ) ).flatten() ## Line to fix for arbitrary environment 94 | 95 | ep_r += r 96 | buffer_a.append(a) 97 | buffer_s.append(s) 98 | buffer_r.append(r) 99 | 100 | if total_step % self.update_global_iter == 0 or done: # update global and assign to local net 101 | # sync 102 | push_and_pull(self.opt, self.lnet, self.gnet, done, s_, buffer_s, buffer_a, buffer_r, self.gamma) 103 | buffer_s, buffer_a, buffer_r = [], [], [] 104 | 105 | if done: # done and print information 106 | record(self.g_ep, self.g_ep_r, ep_r, self.res_queue, self.name) 107 | break 108 | s = s_ 109 | total_step += 1 110 | 111 | self.res_queue.put(None) 112 | 113 | 114 | def trainA3C(file_name="A3C", env=GridworldEnv(1), update_global_iter=10, 115 | gamma=0.999, is_plot=False, num_episodes=500, 116 | max_num_steps_per_episode=1000, learning_rate=0.0001 ): 117 | 118 | """ 119 | A3C training routine. Retuns rewards and durations logs. 120 | Plot environment screen 121 | """ 122 | ns = env.observation_space.shape[0] ## Line to fix for arbitrary environment 123 | na = env.action_space.n 124 | 125 | gnet = Net(ns, na) # global network 126 | gnet.share_memory() # share the global parameters in multiprocessing 127 | opt = SharedAdam(gnet.parameters(), lr = learning_rate ) # global optimizer 128 | global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue() 129 | 130 | # parallel training 131 | workers = [Worker(gnet, opt, global_ep, global_ep_r, res_queue, i, update_global_iter, num_episodes , max_num_steps_per_episode, gamma, env, ns, na ) for i in range(mp.cpu_count())] 132 | 133 | [w.start() for w in workers] 134 | episode_rewards = [] # record episode reward to plot 135 | while True: 136 | r = res_queue.get() 137 | if r is not None: 138 | episode_rewards.append(r) 139 | else: 140 | break 141 | [w.join() for w in workers] 142 | 143 | #Store results 144 | np.save(file_name + '-a3c-rewards', episode_rewards) 145 | 146 | 147 | return episode_rewards 148 | -------------------------------------------------------------------------------- /code/distral_2col0/trainingDistral2col0.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | from itertools import count 4 | import torch.optim as optim 5 | import torch 6 | import math 7 | import numpy as np 8 | from memory_replay import ReplayMemory, Transition 9 | from network import DQN, select_action, optimize_model, Tensor, optimize_policy, PolicyNetwork 10 | import sys 11 | sys.path.append('../') 12 | from envs.gridworld_env import GridworldEnv 13 | from utils import plot_rewards, plot_durations, plot_state, get_screen 14 | 15 | def trainD(file_name="Distral_1col", list_of_envs=[GridworldEnv(5), 16 | GridworldEnv(4), GridworldEnv(6)], batch_size=128, gamma=0.999, alpha=0.8, 17 | beta=5, eps_start=0.9, eps_end=0.05, eps_decay=5, 18 | is_plot=False, num_episodes=200, 19 | max_num_steps_per_episode=1000, learning_rate=0.001, 20 | memory_replay_size=10000, memory_policy_size=1000): 21 | """ 22 | Soft Q-learning training routine. Retuns rewards and durations logs. 23 | Plot environment screen 24 | """ 25 | num_actions = list_of_envs[0].action_space.n 26 | input_size = list_of_envs[0].observation_space.shape[0] 27 | num_envs = len(list_of_envs) 28 | policy = PolicyNetwork(input_size, num_actions) 29 | models = [DQN(input_size,num_actions) for _ in range(0, num_envs)] ### Add torch.nn.ModuleList (?) 30 | memories = [ReplayMemory(memory_replay_size, memory_policy_size) for _ in range(0, num_envs)] 31 | 32 | use_cuda = torch.cuda.is_available() 33 | if use_cuda: 34 | policy.cuda() 35 | for model in models: 36 | model.cuda() 37 | 38 | optimizers = [optim.Adam(model.parameters(), lr=learning_rate) 39 | for model in models] 40 | policy_optimizer = optim.Adam(policy.parameters(), lr=learning_rate) 41 | # optimizer = optim.RMSprop(model.parameters(), ) 42 | 43 | episode_durations = [[] for _ in range(num_envs)] 44 | episode_rewards = [[] for _ in range(num_envs)] 45 | 46 | steps_done = np.zeros(num_envs) 47 | episodes_done = np.zeros(num_envs) 48 | current_time = np.zeros(num_envs) 49 | 50 | # Initialize environments 51 | states = [] 52 | for env in list_of_envs: 53 | states.append(torch.from_numpy( env.reset() ).type(torch.FloatTensor).view(-1,input_size)) 54 | 55 | while np.min(episodes_done) < num_episodes: 56 | # TODO: add max_num_steps_per_episode 57 | 58 | # Optimization is given by alterating minimization scheme: 59 | # 1. do the step for each env 60 | # 2. do one optimization step for each env using "soft-q-learning". 61 | # 3. do one optimization step for the policy 62 | 63 | for i_env, env in enumerate(list_of_envs): 64 | # print("Cur episode:", i_episode, "steps done:", steps_done, 65 | # "exploration factor:", eps_end + (eps_start - eps_end) * \ 66 | # math.exp(-1. * steps_done / eps_decay)) 67 | 68 | # Select and perform an action 69 | #print(states[i_env]) 70 | 71 | action = select_action(states[i_env], policy, models[i_env], num_actions, 72 | eps_start, eps_end, eps_decay, 73 | episodes_done[i_env], alpha, beta) 74 | 75 | steps_done[i_env] += 1 76 | current_time[i_env] += 1 77 | next_state_tmp, reward, done, _ = env.step(action[0,0]) 78 | reward = Tensor([reward]) 79 | 80 | # Observe new state 81 | next_state = torch.from_numpy( next_state_tmp ).type(torch.FloatTensor).view(-1,input_size) 82 | 83 | if done: 84 | next_state = None 85 | 86 | # Store the transition in memory 87 | time = Tensor([current_time[i_env]]) 88 | memories[i_env].push(states[i_env], action, next_state, reward, time) 89 | 90 | # Perform one step of the optimization (on the target network) 91 | optimize_model(policy, models[i_env], optimizers[i_env], 92 | memories[i_env], batch_size, alpha, beta, gamma) 93 | 94 | # Update state 95 | states[i_env] = next_state 96 | 97 | # Check if agent reached target 98 | if done: 99 | print("ENV:", i_env, "iter:", episodes_done[i_env], 100 | "\treward:", env.episode_total_reward, 101 | "\tit:", current_time[i_env], "\texp_factor:", eps_end + 102 | (eps_start - eps_end) * math.exp(-1. * episodes_done[i_env] / eps_decay)) 103 | states[i_env] = torch.from_numpy( env.reset() ).type(torch.FloatTensor).view(-1,input_size) 104 | episodes_done[i_env] += 1 105 | episode_durations[i_env].append(current_time[i_env]) 106 | current_time[i_env] = 0 107 | episode_rewards[i_env].append(env.episode_total_reward) 108 | if is_plot: 109 | plot_rewards(episode_rewards, i_env) 110 | 111 | 112 | optimize_policy(policy, policy_optimizer, memories, batch_size, 113 | num_envs, gamma) 114 | 115 | print('Complete') 116 | env.render(close=True) 117 | env.close() 118 | if is_plot: 119 | plt.ioff() 120 | plt.show() 121 | 122 | ## Store Results 123 | 124 | np.save(file_name + '-distral-1col-rewards', episode_rewards) 125 | np.save(file_name + '-distral-1col-durations', episode_durations) 126 | 127 | return models, policy, episode_rewards, episode_durations 128 | -------------------------------------------------------------------------------- /code/distral_2col0/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import random 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torch.autograd import Variable 9 | from memory_replay import Transition 10 | from itertools import count 11 | from torch.distributions import Categorical 12 | 13 | 14 | use_cuda = torch.cuda.is_available() 15 | 16 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 17 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 18 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 19 | Tensor = FloatTensor 20 | 21 | class DQN(nn.Module): 22 | """ 23 | Deep neural network with represents an agent. 24 | """ 25 | def __init__(self, input_size, num_actions): 26 | super(DQN, self).__init__() 27 | self.linear1 = nn.Linear(input_size, 50) 28 | self.linear2 = nn.Linear(50, 50) 29 | self.head = nn.Linear(50, num_actions) 30 | 31 | def forward(self, x): 32 | x = F.leaky_relu(self.linear1(x)) 33 | x = F.leaky_relu(self.linear2(x)) 34 | return self.head(x) 35 | 36 | class PolicyNetwork(nn.Module): 37 | """ 38 | Deep neural network which represents policy network. 39 | """ 40 | def __init__(self, input_size, num_actions): 41 | super(PolicyNetwork, self).__init__() 42 | self.linear1 = nn.Linear(input_size, 50) 43 | self.linear2 = nn.Linear(50, 50) 44 | self.head = nn.Linear(50, num_actions) 45 | 46 | def forward(self, x): 47 | x = F.leaky_relu(self.linear1(x)) 48 | x = F.leaky_relu(self.linear2(x)) 49 | return F.softmax(self.head(x)) 50 | 51 | def select_action(state, policy, model, num_actions, 52 | EPS_START, EPS_END, EPS_DECAY, steps_done, alpha, beta): 53 | """ 54 | Selects whether the next action is choosen by our model or randomly 55 | """ 56 | # sample = random.random() 57 | # eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 58 | # math.exp(-1. * steps_done / EPS_DECAY) 59 | # .data.max(1)[1].view(1, 1) 60 | # if sample <= eps_threshold: 61 | # return LongTensor([[random.randrange(num_actions)]]) 62 | 63 | 64 | 65 | Q = model(Variable(state, volatile=True).type(FloatTensor)) 66 | pi0 = policy(Variable(state, volatile=True).type(FloatTensor)) 67 | # print(pi0.data.numpy()) 68 | V = torch.log((torch.pow(pi0, alpha) * torch.exp(beta * Q)).sum(1) ) / beta 69 | 70 | #### FOUND ERROR: ( Q ) returns a tensor of nan at some point 71 | if np.isnan( Q.sum(1).data[0]) : 72 | print("Q = ", Q) 73 | print("state = ", state) 74 | 75 | pi_i = torch.pow(pi0, alpha) * torch.exp(beta * (Q - V)) 76 | m = Categorical(pi_i) 77 | action = m.sample().data.view(1, 1) 78 | return action 79 | # numpy.random.choice(numpy.arange(0, num_actions), p=probabilities) 80 | 81 | 82 | def optimize_policy(policy, optimizer, memories, batch_size, 83 | num_envs, gamma): 84 | loss = 0 85 | for i_env in range(num_envs): 86 | size_to_sample = np.minimum(batch_size, len(memories[i_env])) 87 | transitions = memories[i_env].policy_sample(size_to_sample) 88 | batch = Transition(*zip(*transitions)) 89 | 90 | state_batch = Variable(torch.cat(batch.state)) 91 | # print(batch.action) 92 | time_batch = Variable(torch.cat(batch.time)) 93 | actions = np.array([action.numpy()[0][0] for action in batch.action]) 94 | 95 | cur_loss = (torch.pow(Variable(Tensor([gamma])), time_batch) * 96 | torch.log(policy(state_batch)[:, actions])).sum() 97 | loss -= cur_loss 98 | # loss = cur_loss if i_env == 0 else loss + cur_loss 99 | 100 | optimizer.zero_grad() 101 | loss.backward() 102 | 103 | # for param in policy.parameters(): 104 | # param.grad.data.clamp_(-1, 1) 105 | optimizer.step() 106 | 107 | def optimize_model(policy, model, optimizer, memory, batch_size, 108 | alpha, beta, gamma): 109 | if len(memory) < batch_size: 110 | return 111 | transitions = memory.sample(batch_size) 112 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 113 | # detailed explanation). 114 | batch = Transition(*zip(*transitions)) 115 | 116 | # Compute a mask of non-final states and concatenate the batch elements 117 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 118 | batch.next_state))) 119 | # We don't want to backprop through the expected action values and volatile 120 | # will save us on temporarily changing the model parameters' 121 | # requires_grad to False! 122 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 123 | if s is not None]), 124 | volatile=True) 125 | 126 | state_batch = Variable(torch.cat(batch.state)) 127 | action_batch = Variable(torch.cat(batch.action)) 128 | reward_batch = Variable(torch.cat(batch.reward)) 129 | 130 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 131 | # columns of actions taken 132 | state_action_values = model(state_batch).gather(1, action_batch) 133 | 134 | # Compute V(s_{t+1}) for all next states. 135 | next_state_values = Variable(torch.zeros(batch_size).type(Tensor)) 136 | next_state_values[non_final_mask] = torch.log( 137 | (torch.pow(policy(non_final_next_states), alpha) 138 | * (torch.exp(beta * model(non_final_next_states)) + 1e-16)).sum(1)) / beta 139 | try: 140 | np.isnan(next_state_values.sum().data[0]) 141 | except Exception: 142 | print("next_state_values:", next_state_values) 143 | print(policy(non_final_next_states)) 144 | print(torch.exp(beta * model(non_final_next_states))) 145 | print(model(non_final_next_states)) 146 | 147 | # Now, we don't want to mess up the loss with a volatile flag, so let's 148 | # clear it. After this, we'll just end up with a Variable that has 149 | # requires_grad=False 150 | next_state_values.volatile = False 151 | # Compute the expected Q values 152 | expected_state_action_values = (next_state_values * gamma) + reward_batch 153 | 154 | # Compute Huber loss 155 | loss = F.mse_loss(state_action_values, expected_state_action_values) 156 | 157 | # Optimize the model 158 | optimizer.zero_grad() 159 | loss.backward() 160 | # for param in model.parameters(): 161 | # param.grad.data.clamp_(-1, 1) 162 | optimizer.step() 163 | -------------------------------------------------------------------------------- /code/distral_1col0/distral1col.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gym 3 | import numpy as np 4 | from itertools import count 5 | from collections import namedtuple 6 | 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch.autograd import Variable 12 | from torch.distributions import Categorical 13 | 14 | 15 | parser = argparse.ArgumentParser(description='PyTorch distral example') 16 | parser.add_argument('--gamma', type=float, default=0.99, metavar='G', 17 | help='discount factor (default: 0.99)') 18 | parser.add_argument('--seed', type=int, default=512, metavar='N', 19 | help='random seed (default: 1)') 20 | parser.add_argument('--log-interval', type=int, default=5, metavar='N', 21 | help='interval between training status logs (default: 5)') 22 | args = parser.parse_args() 23 | 24 | import sys 25 | sys.path.append('../') 26 | from envs.gridworld_env import GridworldEnv 27 | 28 | #env = gym.make('CartPole-v0') 29 | #env.seed(args.seed) 30 | 31 | 32 | #torch.manual_seed(args.seed) 33 | 34 | 35 | class Policy(nn.Module): 36 | 37 | def __init__(self, input_size, num_actions ): 38 | 39 | super(Policy, self).__init__() 40 | self.affines = nn.Linear(input_size, 100) 41 | self.action_head = nn.Linear(100, num_actions) 42 | 43 | self.saved_actions = [] 44 | self.rewards = [] 45 | 46 | def forward(self, x): 47 | action_scores = F.softmax(self.action_head(F.relu(self.affines(x))), dim=-1) 48 | return action_scores 49 | 50 | 51 | 52 | def select_action(state, policy , distilled ): 53 | 54 | # Format the state 55 | state = torch.from_numpy(state).float() 56 | 57 | # Run the policy 58 | probs = policy(Variable(state)) 59 | 60 | # Obtain the most probable action for the policy 61 | m = Categorical(probs) 62 | action = m.sample() 63 | policy.saved_actions.append( m.log_prob(action)) 64 | 65 | 66 | # Run distilled policy 67 | probs0 = distilled(Variable(state)) 68 | 69 | # Obtain the most probably action for the distilled policy 70 | m = Categorical(probs0) 71 | action_tmp = m.sample() 72 | distilled.saved_actions.append( m.log_prob(action_tmp) ) 73 | 74 | # Return the most probable action for the policy 75 | return action 76 | 77 | 78 | def finish_episode( policy, distilled, opt_policy , opt_distilled, alpha, beta , gamma): 79 | 80 | ### Calculate loss function according to Equation 1 81 | 82 | ## Store three type of losses 83 | reward_losses = [] 84 | distill_losses = [] 85 | entropy_losses = [] 86 | 87 | # Give format 88 | alpha = Variable(torch.Tensor([alpha])) 89 | beta = Variable(torch.Tensor([beta])) 90 | 91 | # Retrive distilled policy actions 92 | distill_actions = distilled.saved_actions 93 | 94 | # Retrieve policy actions and rewards 95 | policy_actions = policy.saved_actions 96 | rewards = policy.rewards 97 | 98 | # Obtain discounts 99 | R = 1. 100 | discounts = [] 101 | for r in policy.rewards[::-1]: 102 | R *= gamma 103 | discounts.insert(0,R) 104 | 105 | discounts = torch.Tensor(discounts) 106 | #print(discounts) 107 | 108 | for log_prob_i, log_prob_0, d , r in zip(policy_actions, distill_actions, discounts, rewards ): 109 | reward_losses.append( -d * Variable(torch.Tensor([r]) ) ) 110 | distill_losses.append( -( (d*alpha)/beta ) * log_prob_0 ) 111 | entropy_losses.append( (d/beta) * log_prob_i ) 112 | 113 | 114 | 115 | #print('Reward Loss: ',torch.stack(reward_losses).sum().data[0]) 116 | #print('Entropy Loss: ',torch.stack(entropy_losses).sum().data[0]) 117 | #print('Distill Loss: ',torch.stack(distill_losses).sum().data[0]) 118 | 119 | # Perform optimization step 120 | opt_policy.zero_grad() 121 | opt_distilled.zero_grad() 122 | 123 | loss = torch.stack(reward_losses).sum() + torch.stack(entropy_losses).sum() + torch.stack(distill_losses).sum() 124 | 125 | loss.backward(retain_graph=True) 126 | 127 | 128 | #for param in policy.parameters(): 129 | # param.grad.data.clamp_(-1, 1) 130 | 131 | opt_policy.step() 132 | opt_distilled.step() 133 | 134 | #Clean memory 135 | del policy.rewards[:] 136 | del policy.saved_actions[:] 137 | del policy.saved_actions[:] 138 | 139 | 140 | def trainDistral( file_name="Distral_1col", list_of_envs=[GridworldEnv(5), GridworldEnv(4)], batch_size=128, gamma=0.80, alpha=0.5, 141 | beta=0.005, is_plot=False, num_episodes=1000, 142 | max_num_steps_per_episode=10, learning_rate=0.001, 143 | memory_replay_size=10000, memory_policy_size=1000 ): 144 | 145 | 146 | # Specify Environment conditions 147 | input_size = list_of_envs[0].observation_space.shape[0] 148 | num_actions = list_of_envs[0].action_space.n 149 | tasks = len(list_of_envs) 150 | 151 | # Define our set of policies, including distilled one 152 | models = torch.nn.ModuleList( [Policy(input_size, num_actions) for _ in range(tasks+1)] ) 153 | optimizers = [optim.Adam(model.parameters(), lr=learning_rate) for model in models] 154 | 155 | # Store the total rewards 156 | episode_rewards = [ [] for i in range(num_episodes) ] 157 | episode_duration = [ [] for i in range(num_episodes) ] 158 | 159 | for i_episode in range(num_episodes): 160 | 161 | # For each one of the envs 162 | for i_env, env in enumerate(list_of_envs): 163 | 164 | #Initialize state of envs 165 | state = env.reset() 166 | 167 | #Store total reward per environment per episode 168 | total_reward = 0 169 | 170 | # Store duration of each episode per env 171 | duration = 0 172 | 173 | for t in range(max_num_steps_per_episode): 174 | 175 | # Run our policy 176 | action = select_action(state, models[i_env + 1], models[0] ) 177 | 178 | next_state, reward, done, _ = env.step(action.data[0]) 179 | models[i_env+1].rewards.append(reward) 180 | total_reward += reward 181 | duration += 1 182 | 183 | #if is_plot: 184 | # env.render() 185 | 186 | if done: 187 | break 188 | 189 | #Update state 190 | state = next_state 191 | 192 | episode_rewards[i_episode].append(total_reward) 193 | episode_duration[i_episode].append(duration) 194 | 195 | # Distill for each environment 196 | finish_episode(models[i_env + 1], models[0], optimizers[i_env + 1], optimizers[0] , alpha , beta, gamma ) 197 | 198 | 199 | if i_episode % args.log_interval == 0: 200 | for i in range(tasks): 201 | print('Episode: {}\tEnv: {}\tDuration: {}\tTotal Reward: {:.2f}'.format( 202 | i_episode, i, episode_duration[i_episode][i], episode_rewards[i_episode][i])) 203 | 204 | np.save(file_name + '-distral0-rewards' , episode_rewards) 205 | np.save(file_name + '-distral0-duration' , episode_duration) 206 | 207 | print('Completed') 208 | 209 | if __name__ == '__main__': 210 | trainDistral() -------------------------------------------------------------------------------- /code/distral_2col/network.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import random 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | import torch.nn.functional as F 8 | from torch.autograd import Variable 9 | from memory_replay import Transition 10 | from itertools import count 11 | from torch.distributions import Categorical 12 | 13 | 14 | use_cuda = torch.cuda.is_available() 15 | 16 | FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor 17 | LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor 18 | ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor 19 | Tensor = FloatTensor 20 | 21 | class DQN(nn.Module): 22 | """ 23 | Deep neural network with represents an agent. 24 | """ 25 | def __init__(self, num_actions): 26 | super(DQN, self).__init__() 27 | self.conv1 = nn.Conv2d(1, 5, kernel_size=2) 28 | self.bn1 = nn.BatchNorm2d(5) 29 | self.conv2 = nn.Conv2d(5, 10, kernel_size=3) 30 | self.bn2 = nn.BatchNorm2d(10) 31 | self.conv3 = nn.Conv2d(10, 10, kernel_size=3) 32 | self.bn3 = nn.BatchNorm2d(10) 33 | self.head = nn.Linear(200, num_actions) 34 | 35 | def forward(self, x): 36 | x = F.leaky_relu(self.bn1(self.conv1(x))) 37 | x = F.leaky_relu(self.bn2(self.conv2(x))) 38 | x = F.leaky_relu(self.bn3(self.conv3(x))) 39 | # x = F.leaky_relu(self.conv1(x)) 40 | # x = F.leaky_relu(self.conv2(x)) 41 | # x = F.leaky_relu(self.conv3(x)) 42 | return self.head(x.view(x.size(0), -1)) 43 | 44 | class PolicyNetwork(nn.Module): 45 | """ 46 | Deep neural network which represents policy network. 47 | """ 48 | def __init__(self, num_actions): 49 | super(PolicyNetwork, self).__init__() 50 | self.conv1 = nn.Conv2d(1, 5, kernel_size=2) 51 | self.bn1 = nn.BatchNorm2d(5) 52 | self.conv2 = nn.Conv2d(5, 10, kernel_size=3) 53 | self.bn2 = nn.BatchNorm2d(10) 54 | self.conv3 = nn.Conv2d(10, 10, kernel_size=3) 55 | self.bn3 = nn.BatchNorm2d(10) 56 | self.head = nn.Linear(200, num_actions) 57 | self.softmax = nn.Softmax() 58 | 59 | def forward(self, x): 60 | x = F.leaky_relu(self.bn1(self.conv1(x))) 61 | x = F.leaky_relu(self.bn2(self.conv2(x))) 62 | x = F.leaky_relu(self.bn3(self.conv3(x))) 63 | x = F.leaky_relu(self.head(x.view(x.size(0), -1))) 64 | return self.softmax(x) 65 | 66 | def select_action(state, policy, model, num_actions, 67 | EPS_START, EPS_END, EPS_DECAY, steps_done, alpha, beta): 68 | """ 69 | Selects whether the next action is choosen by our model or randomly 70 | """ 71 | # sample = random.random() 72 | # eps_threshold = EPS_END + (EPS_START - EPS_END) * \ 73 | # math.exp(-1. * steps_done / EPS_DECAY) 74 | # .data.max(1)[1].view(1, 1) 75 | # if sample <= eps_threshold: 76 | # return LongTensor([[random.randrange(num_actions)]]) 77 | 78 | # print("state = ", state) 79 | # print("forward = ", model(Variable(state, volatile=True))) 80 | Q = model(Variable(state, volatile=True).type(FloatTensor)) 81 | pi0 = policy(Variable(state, volatile=True).type(FloatTensor)) 82 | V = torch.log((torch.pow(pi0, alpha) * torch.exp(beta * Q)).sum(1)) / beta 83 | # print("pi0 = ", pi0) 84 | # print(torch.pow(pi0, alpha) * torch.exp(beta * Q)) 85 | # print("V = ", V) 86 | pi_i = torch.pow(pi0, alpha) * torch.exp(beta * (Q - V)) 87 | if sum(pi_i.data.numpy()[0] < 0) > 0: 88 | print("Warning!!!: pi_i has negative values: pi_i", pi_i.data.numpy()[0]) 89 | pi_i = torch.max(torch.zeros_like(pi_i) + 1e-15, pi_i) 90 | # probabilities = pi_i.data.numpy()[0] 91 | # print("pi_i = ", pi_i) 92 | m = Categorical(pi_i) 93 | action = m.sample().data.view(1, 1) 94 | return action 95 | # numpy.random.choice(numpy.arange(0, num_actions), p=probabilities) 96 | 97 | 98 | 99 | 100 | 101 | 102 | def optimize_policy(policy, optimizer, memories, batch_size, 103 | num_envs, gamma): 104 | loss = 0 105 | for i_env in range(num_envs): 106 | size_to_sample = np.minimum(batch_size, len(memories[i_env])) 107 | transitions = memories[i_env].policy_sample(size_to_sample) 108 | batch = Transition(*zip(*transitions)) 109 | 110 | state_batch = Variable(torch.cat(batch.state)) 111 | # print(batch.action) 112 | time_batch = Variable(torch.cat(batch.time)) 113 | actions = np.array([action.numpy()[0][0] for action in batch.action]) 114 | 115 | cur_loss = (torch.pow(Variable(Tensor([gamma])), time_batch) * 116 | torch.log(policy(state_batch)[:, actions])).sum() 117 | loss -= cur_loss 118 | # loss = cur_loss if i_env == 0 else loss + cur_loss 119 | 120 | optimizer.zero_grad() 121 | loss.backward() 122 | 123 | for param in policy.parameters(): 124 | param.grad.data.clamp_(-500, 500) 125 | # print("policy:", param.grad.data) 126 | optimizer.step() 127 | 128 | def optimize_model(policy, model, optimizer, memory, batch_size, 129 | alpha, beta, gamma): 130 | if len(memory) < batch_size: 131 | return 132 | transitions = memory.sample(batch_size) 133 | # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for 134 | # detailed explanation). 135 | batch = Transition(*zip(*transitions)) 136 | 137 | # Compute a mask of non-final states and concatenate the batch elements 138 | non_final_mask = ByteTensor(tuple(map(lambda s: s is not None, 139 | batch.next_state))) 140 | # We don't want to backprop through the expected action values and volatile 141 | # will save us on temporarily changing the model parameters' 142 | # requires_grad to False! 143 | non_final_next_states = Variable(torch.cat([s for s in batch.next_state 144 | if s is not None]), 145 | volatile=True) 146 | 147 | state_batch = Variable(torch.cat(batch.state)) 148 | action_batch = Variable(torch.cat(batch.action)) 149 | reward_batch = Variable(torch.cat(batch.reward)) 150 | 151 | # Compute Q(s_t, a) - the model computes Q(s_t), then we select the 152 | # columns of actions taken 153 | state_action_values = model(state_batch).gather(1, action_batch) 154 | 155 | # Compute V(s_{t+1}) for all next states. 156 | next_state_values = Variable(torch.zeros(batch_size).type(Tensor)) 157 | next_state_values[non_final_mask] = torch.log( 158 | (torch.pow(policy(non_final_next_states), alpha) 159 | * torch.exp(beta * model(non_final_next_states))).sum(1)) / beta 160 | # Now, we don't want to mess up the loss with a volatile flag, so let's 161 | # clear it. After this, we'll just end up with a Variable that has 162 | # requires_grad=False 163 | next_state_values.volatile = False 164 | # Compute the expected Q values 165 | expected_state_action_values = (next_state_values * gamma) + reward_batch 166 | 167 | # Compute Huber loss 168 | loss = F.mse_loss(state_action_values + 1e-16, expected_state_action_values) 169 | # print("loss:", loss) 170 | # Optimize the model 171 | optimizer.zero_grad() 172 | loss.backward() 173 | for param in model.parameters(): 174 | param.grad.data.clamp_(-500, 500) 175 | # print("model:", param.grad.data) 176 | optimizer.step() 177 | -------------------------------------------------------------------------------- /code/envs/gridworld_env.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import sys 3 | import os 4 | import copy 5 | from gym import spaces 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | from gym.utils import seeding 9 | 10 | EMPTY = BLACK = 0 11 | WALL = GRAY = 1 12 | TARGET = GREEN = 3 13 | AGENT = RED = 4 14 | SUCCESS = PINK = 6 15 | COLORS = {BLACK: [0.0, 0.0, 0.0], GRAY: [0.5, 0.5, 0.5], GREEN: [0.0, 1.0, 0.0], 16 | RED: [1.0, 0.0, 0.0], PINK: [1.0, 0.0, 1.0]} 17 | 18 | NOOP = 0 19 | DOWN = 1 20 | UP = 2 21 | LEFT = 3 22 | RIGHT = 4 23 | 24 | class GridworldEnv(): 25 | 26 | metadata = {'render.modes': ['human','rgb_array']} 27 | num_env = 0 28 | 29 | def __init__(self, plan): 30 | 31 | self.actions = [NOOP, UP, DOWN, LEFT, RIGHT] 32 | self.inv_actions = [0, 2, 1, 4, 3] 33 | self.action_space = spaces.Discrete(5) 34 | self.action_pos_dict = {NOOP: [0, 0], UP: [-1, 0], DOWN: [1, 0], LEFT: [0, -1], RIGHT: [0, 1]} 35 | 36 | self.img_shape = [256, 256, 3] # visualize state 37 | 38 | # initialize system state 39 | this_file_path = os.path.dirname(os.path.realpath(__file__)) 40 | self.grid_map_path = os.path.join(this_file_path, 'plan{}.txt'.format(plan)) 41 | self.start_grid_map = self._read_grid_map(self.grid_map_path) # initial grid map 42 | self.current_grid_map = copy.deepcopy(self.start_grid_map) # current grid map 43 | self.grid_map_shape = self.start_grid_map.shape 44 | self.observation_space = spaces.Box(low=np.array([-1.0, -1.0, -1.0 ]) , \ 45 | high = np.array([ 1.0, 1.0, 1.0]) ) 46 | 47 | # agent state: start, target, current state 48 | self.agent_start_state, self.agent_target_state = self._get_agent_start_target_state() 49 | self.agent_state = copy.deepcopy(self.agent_start_state) 50 | 51 | # set other parameters 52 | self.restart_once_done = False # restart or not once done 53 | 54 | # set seed 55 | self.seed() 56 | 57 | # consider total episode reward 58 | self.episode_total_reward = 0.0 59 | 60 | # consider viewer for compatibility with gym 61 | self.viewer = None 62 | 63 | def seed( self, seed = None): 64 | 65 | # Fix seed for reproducibility 66 | 67 | self.np_random, seed = seeding.np_random(seed) 68 | return [seed] 69 | 70 | def get_state ( self , coordinates, action, reward ) : 71 | 72 | # Return a triple with: current location of the agent in the map 73 | # given coordinates, the previous action and the previous reward 74 | 75 | ## Normalized for better perform of the NN 76 | 77 | return np.asarray([ 2.*(self.grid_map_shape[0]*coordinates[0]+coordinates[1]) / ( self.grid_map_shape[0] * self.grid_map_shape[1] ) - 1., \ 78 | (action-2.5)/5. , reward ] ) 79 | 80 | def step(self, action): 81 | 82 | # Return next observation, reward, finished, success 83 | 84 | action = int(action) 85 | info = {'success': False} 86 | done = False 87 | 88 | #Penalties 89 | penalty_step = 0.1 90 | penalty_wall = 0.5 91 | 92 | reward = -penalty_step 93 | nxt_agent_state = (self.agent_state[0] + self.action_pos_dict[action][0], 94 | self.agent_state[1] + self.action_pos_dict[action][1]) 95 | 96 | if action == NOOP: 97 | info['success'] = True 98 | self.episode_total_reward += reward #Update total reward 99 | return self.get_state(self.agent_state, action, reward), reward, False, info 100 | 101 | #Make a step 102 | next_state_out_of_map = (nxt_agent_state[0] < 0 or nxt_agent_state[0] >= self.grid_map_shape[0]) or \ 103 | (nxt_agent_state[1] < 0 or nxt_agent_state[1] >= self.grid_map_shape[1]) 104 | 105 | if next_state_out_of_map: 106 | info['success'] = False 107 | self.episode_total_reward += reward #Update total reward 108 | return self.get_state(self.agent_state, action, reward), reward, False, info 109 | 110 | # successful behavior 111 | target_position = self.current_grid_map[nxt_agent_state[0], nxt_agent_state[1]] 112 | 113 | if target_position == EMPTY: 114 | 115 | self.current_grid_map[nxt_agent_state[0], nxt_agent_state[1]] = AGENT 116 | 117 | elif target_position == WALL: 118 | 119 | info['success'] = False 120 | self.episode_total_reward += (reward-penalty_wall) #Update total reward 121 | return self.get_state(self.agent_state, action, reward-penalty_wall), (reward-penalty_wall), False, info 122 | 123 | elif target_position == TARGET: 124 | 125 | self.current_grid_map[nxt_agent_state[0], nxt_agent_state[1]] = SUCCESS 126 | 127 | self.current_grid_map[self.agent_state[0], self.agent_state[1]] = EMPTY 128 | self.agent_state = copy.deepcopy(nxt_agent_state) 129 | info['success'] = True 130 | 131 | if nxt_agent_state[0] == self.agent_target_state[0] and nxt_agent_state[1] == self.agent_target_state[1]: 132 | done = True 133 | reward += 1.0 134 | if self.restart_once_done: 135 | self.reset() 136 | 137 | self.episode_total_reward += reward #Update total reward 138 | return self.get_state(self.agent_state, action, reward), reward, done, info 139 | 140 | def reset(self): 141 | 142 | # Return the initial state of the environment 143 | 144 | self.agent_state = copy.deepcopy(self.agent_start_state) 145 | self.current_grid_map = copy.deepcopy(self.start_grid_map) 146 | self.episode_total_reward = 0.0 147 | return self.get_state(self.agent_state, 0.0, 0.0) 148 | 149 | def close(self): 150 | if self.viewer: self.viewer.close() 151 | 152 | def _read_grid_map(self, grid_map_path): 153 | 154 | # Return the gridmap imported from a txt plan 155 | 156 | grid_map = open(grid_map_path, 'r').readlines() 157 | grid_map_array = [] 158 | for k1 in grid_map: 159 | k1s = k1.split(' ') 160 | tmp_arr = [] 161 | for k2 in k1s: 162 | try: 163 | tmp_arr.append(int(k2)) 164 | except: 165 | pass 166 | grid_map_array.append(tmp_arr) 167 | grid_map_array = np.array(grid_map_array, dtype=int) 168 | return grid_map_array 169 | 170 | def _get_agent_start_target_state(self): 171 | start_state = np.where(self.start_grid_map == AGENT) 172 | target_state = np.where(self.start_grid_map == TARGET) 173 | 174 | start_or_target_not_found = not(start_state[0] and target_state[0]) 175 | if start_or_target_not_found: 176 | sys.exit('Start or target state not specified') 177 | start_state = (start_state[0][0], start_state[1][0]) 178 | target_state = (target_state[0][0], target_state[1][0]) 179 | 180 | return start_state, target_state 181 | 182 | def _gridmap_to_image(self, img_shape=None): 183 | 184 | # Return image from the gridmap 185 | 186 | if img_shape is None: 187 | img_shape = self.img_shape 188 | observation = np.random.randn(*img_shape) * 0.0 189 | gs0 = int(observation.shape[0] / self.current_grid_map.shape[0]) 190 | gs1 = int(observation.shape[1] / self.current_grid_map.shape[1]) 191 | for i in range(self.current_grid_map.shape[0]): 192 | for j in range(self.current_grid_map.shape[1]): 193 | for k in range(3): 194 | this_value = COLORS[self.current_grid_map[i, j]][k] 195 | observation[i * gs0:(i + 1) * gs0, j * gs1:(j + 1) * gs1, k] = this_value 196 | return (255*observation).astype(np.uint8) 197 | 198 | def render(self, mode='human',close = False): 199 | 200 | # Returns a visualization of the environment according to specification 201 | 202 | if close: 203 | plt.close(1) #Final plot 204 | return 205 | 206 | img = self._gridmap_to_image() 207 | if mode == 'rgb_array': 208 | return img 209 | elif mode == 'human': 210 | plt.figure() 211 | plt.imshow(img) 212 | return 213 | -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Benchmark78-distral-duration.eps: -------------------------------------------------------------------------------- 1 | %!PS-Adobe-3.0 EPSF-3.0 2 | %%Title: Benchmark78-distral-duration.eps 3 | %%Creator: matplotlib version 2.1.0, http://matplotlib.org/ 4 | %%CreationDate: Sun Mar 18 23:25:31 2018 5 | %%Orientation: portrait 6 | %%BoundingBox: -414 36 1026 756 7 | %%EndComments 8 | %%BeginProlog 9 | /mpldict 8 dict def 10 | mpldict begin 11 | /m { moveto } bind def 12 | /l { lineto } bind def 13 | /r { rlineto } bind def 14 | /c { curveto } bind def 15 | /cl { closepath } bind def 16 | /box { 17 | m 18 | 1 index 0 r 19 | 0 exch r 20 | neg 0 r 21 | cl 22 | } bind def 23 | /clipbox { 24 | box 25 | clip 26 | newpath 27 | } bind def 28 | %!PS-Adobe-3.0 Resource-Font 29 | %%Title: DejaVu Sans 30 | %%Copyright: Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. DejaVu changes are in public domain 31 | %%Creator: Converted from TrueType to type 3 by PPR 32 | 25 dict begin 33 | /_d{bind def}bind def 34 | /_m{moveto}_d 35 | /_l{lineto}_d 36 | /_cl{closepath eofill}_d 37 | /_c{curveto}_d 38 | /_sc{7 -1 roll{setcachedevice}{pop pop pop pop pop pop}ifelse}_d 39 | /_e{exec}_d 40 | /FontName /DejaVuSans def 41 | /PaintType 0 def 42 | /FontMatrix[.001 0 0 .001 0 0]def 43 | /FontBBox[-1021 -463 1793 1232]def 44 | /FontType 3 def 45 | /Encoding [ /space /parenleft /parenright /zero /one /two /three /four /five /six /seven /eight /A /B /D /E /I /L /R /S /T /a /c /d /e /g /h /i /k /l /m /n /o /p /r /s /t /u /v ] def 46 | /FontInfo 10 dict dup begin 47 | /FamilyName (DejaVu Sans) def 48 | /FullName (DejaVu Sans) def 49 | /Notice (Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. DejaVu changes are in public domain ) def 50 | /Weight (Book) def 51 | /Version (Version 2.35) def 52 | /ItalicAngle 0.0 def 53 | /isFixedPitch false def 54 | /UnderlinePosition -130 def 55 | /UnderlineThickness 90 def 56 | end readonly def 57 | /CharStrings 40 dict dup begin 58 | /.notdef 0 def 59 | /space{318 0 0 0 0 0 _sc 60 | }_d 61 | /parenleft{390 0 86 -131 310 759 _sc 62 | 310 759 _m 63 | 266 683 234 609 213 536 _c 64 | 191 463 181 389 181 314 _c 65 | 181 238 191 164 213 91 _c 66 | 234 17 266 -56 310 -131 _c 67 | 232 -131 _l 68 | 183 -54 146 20 122 94 _c 69 | 98 168 86 241 86 314 _c 70 | 86 386 98 459 122 533 _c 71 | 146 607 182 682 232 759 _c 72 | 310 759 _l 73 | _cl}_d 74 | /parenright{390 0 80 -131 304 759 _sc 75 | 80 759 _m 76 | 158 759 _l 77 | 206 682 243 607 267 533 _c 78 | 291 459 304 386 304 314 _c 79 | 304 241 291 168 267 94 _c 80 | 243 20 206 -54 158 -131 _c 81 | 80 -131 _l 82 | 123 -56 155 17 177 91 _c 83 | 198 164 209 238 209 314 _c 84 | 209 389 198 463 177 536 _c 85 | 155 609 123 683 80 759 _c 86 | _cl}_d 87 | /zero{636 0 66 -13 570 742 _sc 88 | 318 664 _m 89 | 267 664 229 639 203 589 _c 90 | 177 539 165 464 165 364 _c 91 | 165 264 177 189 203 139 _c 92 | 229 89 267 64 318 64 _c 93 | 369 64 407 89 433 139 _c 94 | 458 189 471 264 471 364 _c 95 | 471 464 458 539 433 589 _c 96 | 407 639 369 664 318 664 _c 97 | 318 742 _m 98 | 399 742 461 709 505 645 _c 99 | 548 580 570 486 570 364 _c 100 | 570 241 548 147 505 83 _c 101 | 461 19 399 -13 318 -13 _c 102 | 236 -13 173 19 130 83 _c 103 | 87 147 66 241 66 364 _c 104 | 66 486 87 580 130 645 _c 105 | 173 709 236 742 318 742 _c 106 | _cl}_d 107 | /one{636 0 110 0 544 729 _sc 108 | 124 83 _m 109 | 285 83 _l 110 | 285 639 _l 111 | 110 604 _l 112 | 110 694 _l 113 | 284 729 _l 114 | 383 729 _l 115 | 383 83 _l 116 | 544 83 _l 117 | 544 0 _l 118 | 124 0 _l 119 | 124 83 _l 120 | _cl}_d 121 | /two{{636 0 73 0 536 742 _sc 122 | 192 83 _m 123 | 536 83 _l 124 | 536 0 _l 125 | 73 0 _l 126 | 73 83 _l 127 | 110 121 161 173 226 239 _c 128 | 290 304 331 346 348 365 _c 129 | 380 400 402 430 414 455 _c 130 | 426 479 433 504 433 528 _c 131 | 433 566 419 598 392 622 _c 132 | 365 646 330 659 286 659 _c 133 | 255 659 222 653 188 643 _c 134 | 154 632 117 616 78 594 _c 135 | 78 694 _l 136 | 118 710 155 722 189 730 _c 137 | 223 738 255 742 284 742 _c 138 | }_e{359 742 419 723 464 685 _c 139 | 509 647 532 597 532 534 _c 140 | 532 504 526 475 515 449 _c 141 | 504 422 484 390 454 354 _c 142 | 446 344 420 317 376 272 _c 143 | 332 227 271 164 192 83 _c 144 | _cl}_e}_d 145 | /three{{636 0 76 -13 556 742 _sc 146 | 406 393 _m 147 | 453 383 490 362 516 330 _c 148 | 542 298 556 258 556 212 _c 149 | 556 140 531 84 482 45 _c 150 | 432 6 362 -13 271 -13 _c 151 | 240 -13 208 -10 176 -4 _c 152 | 144 1 110 10 76 22 _c 153 | 76 117 _l 154 | 103 101 133 89 166 81 _c 155 | 198 73 232 69 268 69 _c 156 | 330 69 377 81 409 105 _c 157 | 441 129 458 165 458 212 _c 158 | 458 254 443 288 413 312 _c 159 | 383 336 341 349 287 349 _c 160 | }_e{202 349 _l 161 | 202 430 _l 162 | 291 430 _l 163 | 339 430 376 439 402 459 _c 164 | 428 478 441 506 441 543 _c 165 | 441 580 427 609 401 629 _c 166 | 374 649 336 659 287 659 _c 167 | 260 659 231 656 200 650 _c 168 | 169 644 135 635 98 623 _c 169 | 98 711 _l 170 | 135 721 170 729 203 734 _c 171 | 235 739 266 742 296 742 _c 172 | 370 742 429 725 473 691 _c 173 | 517 657 539 611 539 553 _c 174 | 539 513 527 479 504 451 _c 175 | 481 423 448 403 406 393 _c 176 | _cl}_e}_d 177 | /four{636 0 49 0 580 729 _sc 178 | 378 643 _m 179 | 129 254 _l 180 | 378 254 _l 181 | 378 643 _l 182 | 352 729 _m 183 | 476 729 _l 184 | 476 254 _l 185 | 580 254 _l 186 | 580 172 _l 187 | 476 172 _l 188 | 476 0 _l 189 | 378 0 _l 190 | 378 172 _l 191 | 49 172 _l 192 | 49 267 _l 193 | 352 729 _l 194 | _cl}_d 195 | /five{{636 0 77 -13 549 729 _sc 196 | 108 729 _m 197 | 495 729 _l 198 | 495 646 _l 199 | 198 646 _l 200 | 198 467 _l 201 | 212 472 227 476 241 478 _c 202 | 255 480 270 482 284 482 _c 203 | 365 482 429 459 477 415 _c 204 | 525 370 549 310 549 234 _c 205 | 549 155 524 94 475 51 _c 206 | 426 8 357 -13 269 -13 _c 207 | 238 -13 207 -10 175 -6 _c 208 | 143 -1 111 6 77 17 _c 209 | 77 116 _l 210 | 106 100 136 88 168 80 _c 211 | 199 72 232 69 267 69 _c 212 | }_e{323 69 368 83 401 113 _c 213 | 433 143 450 183 450 234 _c 214 | 450 284 433 324 401 354 _c 215 | 368 384 323 399 267 399 _c 216 | 241 399 214 396 188 390 _c 217 | 162 384 135 375 108 363 _c 218 | 108 729 _l 219 | _cl}_e}_d 220 | /six{{636 0 70 -13 573 742 _sc 221 | 330 404 _m 222 | 286 404 251 388 225 358 _c 223 | 199 328 186 286 186 234 _c 224 | 186 181 199 139 225 109 _c 225 | 251 79 286 64 330 64 _c 226 | 374 64 409 79 435 109 _c 227 | 461 139 474 181 474 234 _c 228 | 474 286 461 328 435 358 _c 229 | 409 388 374 404 330 404 _c 230 | 526 713 _m 231 | 526 623 _l 232 | 501 635 476 644 451 650 _c 233 | 425 656 400 659 376 659 _c 234 | 310 659 260 637 226 593 _c 235 | }_e{192 549 172 482 168 394 _c 236 | 187 422 211 444 240 459 _c 237 | 269 474 301 482 336 482 _c 238 | 409 482 467 459 509 415 _c 239 | 551 371 573 310 573 234 _c 240 | 573 159 550 99 506 54 _c 241 | 462 9 403 -13 330 -13 _c 242 | 246 -13 181 19 137 83 _c 243 | 92 147 70 241 70 364 _c 244 | 70 479 97 571 152 639 _c 245 | 206 707 280 742 372 742 _c 246 | 396 742 421 739 447 735 _c 247 | 472 730 498 723 526 713 _c 248 | _cl}_e}_d 249 | /seven{636 0 82 0 551 729 _sc 250 | 82 729 _m 251 | 551 729 _l 252 | 551 687 _l 253 | 286 0 _l 254 | 183 0 _l 255 | 432 646 _l 256 | 82 646 _l 257 | 82 729 _l 258 | _cl}_d 259 | /eight{{636 0 68 -13 568 742 _sc 260 | 318 346 _m 261 | 271 346 234 333 207 308 _c 262 | 180 283 167 249 167 205 _c 263 | 167 161 180 126 207 101 _c 264 | 234 76 271 64 318 64 _c 265 | 364 64 401 76 428 102 _c 266 | 455 127 469 161 469 205 _c 267 | 469 249 455 283 429 308 _c 268 | 402 333 365 346 318 346 _c 269 | 219 388 _m 270 | 177 398 144 418 120 447 _c 271 | 96 476 85 511 85 553 _c 272 | 85 611 105 657 147 691 _c 273 | 188 725 245 742 318 742 _c 274 | }_e{390 742 447 725 489 691 _c 275 | 530 657 551 611 551 553 _c 276 | 551 511 539 476 515 447 _c 277 | 491 418 459 398 417 388 _c 278 | 464 377 501 355 528 323 _c 279 | 554 291 568 251 568 205 _c 280 | 568 134 546 80 503 43 _c 281 | 459 5 398 -13 318 -13 _c 282 | 237 -13 175 5 132 43 _c 283 | 89 80 68 134 68 205 _c 284 | 68 251 81 291 108 323 _c 285 | 134 355 171 377 219 388 _c 286 | 183 544 _m 287 | 183 506 194 476 218 455 _c 288 | }_e{242 434 275 424 318 424 _c 289 | 360 424 393 434 417 455 _c 290 | 441 476 453 506 453 544 _c 291 | 453 582 441 611 417 632 _c 292 | 393 653 360 664 318 664 _c 293 | 275 664 242 653 218 632 _c 294 | 194 611 183 582 183 544 _c 295 | _cl}_e}_d 296 | /A{684 0 8 0 676 729 _sc 297 | 342 632 _m 298 | 208 269 _l 299 | 476 269 _l 300 | 342 632 _l 301 | 286 729 _m 302 | 398 729 _l 303 | 676 0 _l 304 | 573 0 _l 305 | 507 187 _l 306 | 178 187 _l 307 | 112 0 _l 308 | 8 0 _l 309 | 286 729 _l 310 | _cl}_d 311 | /B{{686 0 98 0 615 729 _sc 312 | 197 348 _m 313 | 197 81 _l 314 | 355 81 _l 315 | 408 81 447 92 473 114 _c 316 | 498 136 511 169 511 215 _c 317 | 511 260 498 293 473 315 _c 318 | 447 337 408 348 355 348 _c 319 | 197 348 _l 320 | 197 648 _m 321 | 197 428 _l 322 | 343 428 _l 323 | 391 428 426 437 450 455 _c 324 | 474 473 486 500 486 538 _c 325 | 486 574 474 602 450 620 _c 326 | 426 638 391 648 343 648 _c 327 | 197 648 _l 328 | 98 729 _m 329 | 350 729 _l 330 | }_e{425 729 483 713 524 682 _c 331 | 564 650 585 606 585 549 _c 332 | 585 504 574 468 553 442 _c 333 | 532 416 502 399 462 393 _c 334 | 510 382 548 360 575 327 _c 335 | 601 294 615 253 615 204 _c 336 | 615 138 592 88 548 53 _c 337 | 504 17 441 0 360 0 _c 338 | 98 0 _l 339 | 98 729 _l 340 | _cl}_e}_d 341 | /D{770 0 98 0 711 729 _sc 342 | 197 648 _m 343 | 197 81 _l 344 | 316 81 _l 345 | 416 81 490 103 537 149 _c 346 | 583 195 607 267 607 365 _c 347 | 607 463 583 534 537 580 _c 348 | 490 625 416 648 316 648 _c 349 | 197 648 _l 350 | 98 729 _m 351 | 301 729 _l 352 | 442 729 546 699 612 641 _c 353 | 678 582 711 490 711 365 _c 354 | 711 239 677 147 611 88 _c 355 | 545 29 441 0 301 0 _c 356 | 98 0 _l 357 | 98 729 _l 358 | _cl}_d 359 | /E{632 0 98 0 568 729 _sc 360 | 98 729 _m 361 | 559 729 _l 362 | 559 646 _l 363 | 197 646 _l 364 | 197 430 _l 365 | 544 430 _l 366 | 544 347 _l 367 | 197 347 _l 368 | 197 83 _l 369 | 568 83 _l 370 | 568 0 _l 371 | 98 0 _l 372 | 98 729 _l 373 | _cl}_d 374 | /I{295 0 98 0 197 729 _sc 375 | 98 729 _m 376 | 197 729 _l 377 | 197 0 _l 378 | 98 0 _l 379 | 98 729 _l 380 | _cl}_d 381 | /L{557 0 98 0 552 729 _sc 382 | 98 729 _m 383 | 197 729 _l 384 | 197 83 _l 385 | 552 83 _l 386 | 552 0 _l 387 | 98 0 _l 388 | 98 729 _l 389 | _cl}_d 390 | /R{{695 0 98 0 666 729 _sc 391 | 444 342 _m 392 | 465 334 486 319 506 296 _c 393 | 526 272 546 240 566 199 _c 394 | 666 0 _l 395 | 560 0 _l 396 | 467 187 _l 397 | 443 235 419 268 397 284 _c 398 | 374 300 343 308 304 308 _c 399 | 197 308 _l 400 | 197 0 _l 401 | 98 0 _l 402 | 98 729 _l 403 | 321 729 _l 404 | 404 729 466 711 507 677 _c 405 | 548 642 569 589 569 519 _c 406 | 569 473 558 434 537 404 _c 407 | 515 374 484 353 444 342 _c 408 | 197 648 _m 409 | 197 389 _l 410 | 321 389 _l 411 | }_e{368 389 404 400 428 422 _c 412 | 452 444 465 476 465 519 _c 413 | 465 561 452 593 428 615 _c 414 | 404 637 368 648 321 648 _c 415 | 197 648 _l 416 | _cl}_e}_d 417 | /S{{635 0 66 -13 579 742 _sc 418 | 535 705 _m 419 | 535 609 _l 420 | 497 627 462 640 429 649 _c 421 | 395 657 363 662 333 662 _c 422 | 279 662 237 651 208 631 _c 423 | 179 610 165 580 165 542 _c 424 | 165 510 174 485 194 469 _c 425 | 213 452 250 439 304 429 _c 426 | 364 417 _l 427 | 437 403 491 378 526 343 _c 428 | 561 307 579 260 579 201 _c 429 | 579 130 555 77 508 41 _c 430 | 460 5 391 -13 300 -13 _c 431 | 265 -13 228 -9 189 -2 _c 432 | }_e{150 5 110 16 69 32 _c 433 | 69 134 _l 434 | 109 111 148 94 186 83 _c 435 | 224 71 262 66 300 66 _c 436 | 356 66 399 77 430 99 _c 437 | 460 121 476 152 476 194 _c 438 | 476 230 465 258 443 278 _c 439 | 421 298 385 313 335 323 _c 440 | 275 335 _l 441 | 201 349 148 372 115 404 _c 442 | 82 435 66 478 66 534 _c 443 | 66 598 88 649 134 686 _c 444 | 179 723 242 742 322 742 _c 445 | 356 742 390 739 426 733 _c 446 | 461 727 497 717 535 705 _c 447 | }_e{_cl}_e}_d 448 | /T{611 0 -2 0 614 729 _sc 449 | -2 729 _m 450 | 614 729 _l 451 | 614 646 _l 452 | 355 646 _l 453 | 355 0 _l 454 | 256 0 _l 455 | 256 646 _l 456 | -2 646 _l 457 | -2 729 _l 458 | _cl}_d 459 | /a{{613 0 60 -13 522 560 _sc 460 | 343 275 _m 461 | 270 275 220 266 192 250 _c 462 | 164 233 150 205 150 165 _c 463 | 150 133 160 107 181 89 _c 464 | 202 70 231 61 267 61 _c 465 | 317 61 357 78 387 114 _c 466 | 417 149 432 196 432 255 _c 467 | 432 275 _l 468 | 343 275 _l 469 | 522 312 _m 470 | 522 0 _l 471 | 432 0 _l 472 | 432 83 _l 473 | 411 49 385 25 355 10 _c 474 | 325 -5 287 -13 243 -13 _c 475 | 187 -13 142 2 109 33 _c 476 | 76 64 60 106 60 159 _c 477 | }_e{60 220 80 266 122 298 _c 478 | 163 329 224 345 306 345 _c 479 | 432 345 _l 480 | 432 354 _l 481 | 432 395 418 427 391 450 _c 482 | 364 472 326 484 277 484 _c 483 | 245 484 215 480 185 472 _c 484 | 155 464 127 453 100 439 _c 485 | 100 522 _l 486 | 132 534 164 544 195 550 _c 487 | 226 556 256 560 286 560 _c 488 | 365 560 424 539 463 498 _c 489 | 502 457 522 395 522 312 _c 490 | _cl}_e}_d 491 | /c{{550 0 55 -13 488 560 _sc 492 | 488 526 _m 493 | 488 442 _l 494 | 462 456 437 466 411 473 _c 495 | 385 480 360 484 334 484 _c 496 | 276 484 230 465 198 428 _c 497 | 166 391 150 339 150 273 _c 498 | 150 206 166 154 198 117 _c 499 | 230 80 276 62 334 62 _c 500 | 360 62 385 65 411 72 _c 501 | 437 79 462 90 488 104 _c 502 | 488 21 _l 503 | 462 9 436 0 410 -5 _c 504 | 383 -10 354 -13 324 -13 _c 505 | 242 -13 176 12 128 64 _c 506 | }_e{79 115 55 185 55 273 _c 507 | 55 362 79 432 128 483 _c 508 | 177 534 244 560 330 560 _c 509 | 358 560 385 557 411 551 _c 510 | 437 545 463 537 488 526 _c 511 | _cl}_e}_d 512 | /d{{635 0 55 -13 544 760 _sc 513 | 454 464 _m 514 | 454 760 _l 515 | 544 760 _l 516 | 544 0 _l 517 | 454 0 _l 518 | 454 82 _l 519 | 435 49 411 25 382 10 _c 520 | 353 -5 319 -13 279 -13 _c 521 | 213 -13 159 13 117 65 _c 522 | 75 117 55 187 55 273 _c 523 | 55 359 75 428 117 481 _c 524 | 159 533 213 560 279 560 _c 525 | 319 560 353 552 382 536 _c 526 | 411 520 435 496 454 464 _c 527 | 148 273 _m 528 | 148 207 161 155 188 117 _c 529 | 215 79 253 61 301 61 _c 530 | }_e{348 61 385 79 413 117 _c 531 | 440 155 454 207 454 273 _c 532 | 454 339 440 390 413 428 _c 533 | 385 466 348 485 301 485 _c 534 | 253 485 215 466 188 428 _c 535 | 161 390 148 339 148 273 _c 536 | _cl}_e}_d 537 | /e{{615 0 55 -13 562 560 _sc 538 | 562 296 _m 539 | 562 252 _l 540 | 149 252 _l 541 | 153 190 171 142 205 110 _c 542 | 238 78 284 62 344 62 _c 543 | 378 62 412 66 444 74 _c 544 | 476 82 509 95 541 113 _c 545 | 541 28 _l 546 | 509 14 476 3 442 -3 _c 547 | 408 -9 373 -13 339 -13 _c 548 | 251 -13 182 12 131 62 _c 549 | 80 112 55 181 55 268 _c 550 | 55 357 79 428 127 481 _c 551 | 175 533 241 560 323 560 _c 552 | 397 560 455 536 498 489 _c 553 | }_e{540 441 562 377 562 296 _c 554 | 472 322 _m 555 | 471 371 457 410 431 440 _c 556 | 404 469 368 484 324 484 _c 557 | 274 484 234 469 204 441 _c 558 | 174 413 156 373 152 322 _c 559 | 472 322 _l 560 | _cl}_e}_d 561 | /g{{635 0 55 -207 544 560 _sc 562 | 454 280 _m 563 | 454 344 440 395 414 431 _c 564 | 387 467 349 485 301 485 _c 565 | 253 485 215 467 188 431 _c 566 | 161 395 148 344 148 280 _c 567 | 148 215 161 165 188 129 _c 568 | 215 93 253 75 301 75 _c 569 | 349 75 387 93 414 129 _c 570 | 440 165 454 215 454 280 _c 571 | 544 68 _m 572 | 544 -24 523 -93 482 -139 _c 573 | 440 -184 377 -207 292 -207 _c 574 | 260 -207 231 -204 203 -200 _c 575 | 175 -195 147 -188 121 -178 _c 576 | }_e{121 -91 _l 577 | 147 -105 173 -115 199 -122 _c 578 | 225 -129 251 -133 278 -133 _c 579 | 336 -133 380 -117 410 -87 _c 580 | 439 -56 454 -10 454 52 _c 581 | 454 96 _l 582 | 435 64 411 40 382 24 _c 583 | 353 8 319 0 279 0 _c 584 | 211 0 157 25 116 76 _c 585 | 75 127 55 195 55 280 _c 586 | 55 364 75 432 116 483 _c 587 | 157 534 211 560 279 560 _c 588 | 319 560 353 552 382 536 _c 589 | 411 520 435 496 454 464 _c 590 | 454 547 _l 591 | 544 547 _l 592 | }_e{544 68 _l 593 | _cl}_e}_d 594 | /h{634 0 91 0 549 760 _sc 595 | 549 330 _m 596 | 549 0 _l 597 | 459 0 _l 598 | 459 327 _l 599 | 459 379 448 417 428 443 _c 600 | 408 469 378 482 338 482 _c 601 | 289 482 251 466 223 435 _c 602 | 195 404 181 362 181 309 _c 603 | 181 0 _l 604 | 91 0 _l 605 | 91 760 _l 606 | 181 760 _l 607 | 181 462 _l 608 | 202 494 227 519 257 535 _c 609 | 286 551 320 560 358 560 _c 610 | 420 560 468 540 500 501 _c 611 | 532 462 549 405 549 330 _c 612 | _cl}_d 613 | /i{278 0 94 0 184 760 _sc 614 | 94 547 _m 615 | 184 547 _l 616 | 184 0 _l 617 | 94 0 _l 618 | 94 547 _l 619 | 94 760 _m 620 | 184 760 _l 621 | 184 646 _l 622 | 94 646 _l 623 | 94 760 _l 624 | _cl}_d 625 | /k{579 0 91 0 576 760 _sc 626 | 91 760 _m 627 | 181 760 _l 628 | 181 311 _l 629 | 449 547 _l 630 | 564 547 _l 631 | 274 291 _l 632 | 576 0 _l 633 | 459 0 _l 634 | 181 267 _l 635 | 181 0 _l 636 | 91 0 _l 637 | 91 760 _l 638 | _cl}_d 639 | /l{278 0 94 0 184 760 _sc 640 | 94 760 _m 641 | 184 760 _l 642 | 184 0 _l 643 | 94 0 _l 644 | 94 760 _l 645 | _cl}_d 646 | /m{{974 0 91 0 889 560 _sc 647 | 520 442 _m 648 | 542 482 569 511 600 531 _c 649 | 631 550 668 560 711 560 _c 650 | 767 560 811 540 842 500 _c 651 | 873 460 889 403 889 330 _c 652 | 889 0 _l 653 | 799 0 _l 654 | 799 327 _l 655 | 799 379 789 418 771 444 _c 656 | 752 469 724 482 686 482 _c 657 | 639 482 602 466 575 435 _c 658 | 548 404 535 362 535 309 _c 659 | 535 0 _l 660 | 445 0 _l 661 | 445 327 _l 662 | 445 379 435 418 417 444 _c 663 | 398 469 369 482 331 482 _c 664 | }_e{285 482 248 466 221 435 _c 665 | 194 404 181 362 181 309 _c 666 | 181 0 _l 667 | 91 0 _l 668 | 91 547 _l 669 | 181 547 _l 670 | 181 462 _l 671 | 201 495 226 520 255 536 _c 672 | 283 552 317 560 357 560 _c 673 | 397 560 430 550 458 530 _c 674 | 486 510 506 480 520 442 _c 675 | _cl}_e}_d 676 | /n{634 0 91 0 549 560 _sc 677 | 549 330 _m 678 | 549 0 _l 679 | 459 0 _l 680 | 459 327 _l 681 | 459 379 448 417 428 443 _c 682 | 408 469 378 482 338 482 _c 683 | 289 482 251 466 223 435 _c 684 | 195 404 181 362 181 309 _c 685 | 181 0 _l 686 | 91 0 _l 687 | 91 547 _l 688 | 181 547 _l 689 | 181 462 _l 690 | 202 494 227 519 257 535 _c 691 | 286 551 320 560 358 560 _c 692 | 420 560 468 540 500 501 _c 693 | 532 462 549 405 549 330 _c 694 | _cl}_d 695 | /o{612 0 55 -13 557 560 _sc 696 | 306 484 _m 697 | 258 484 220 465 192 427 _c 698 | 164 389 150 338 150 273 _c 699 | 150 207 163 156 191 118 _c 700 | 219 80 257 62 306 62 _c 701 | 354 62 392 80 420 118 _c 702 | 448 156 462 207 462 273 _c 703 | 462 337 448 389 420 427 _c 704 | 392 465 354 484 306 484 _c 705 | 306 560 _m 706 | 384 560 445 534 490 484 _c 707 | 534 433 557 363 557 273 _c 708 | 557 183 534 113 490 63 _c 709 | 445 12 384 -13 306 -13 _c 710 | 227 -13 165 12 121 63 _c 711 | 77 113 55 183 55 273 _c 712 | 55 363 77 433 121 484 _c 713 | 165 534 227 560 306 560 _c 714 | _cl}_d 715 | /p{{635 0 91 -207 580 560 _sc 716 | 181 82 _m 717 | 181 -207 _l 718 | 91 -207 _l 719 | 91 547 _l 720 | 181 547 _l 721 | 181 464 _l 722 | 199 496 223 520 252 536 _c 723 | 281 552 316 560 356 560 _c 724 | 422 560 476 533 518 481 _c 725 | 559 428 580 359 580 273 _c 726 | 580 187 559 117 518 65 _c 727 | 476 13 422 -13 356 -13 _c 728 | 316 -13 281 -5 252 10 _c 729 | 223 25 199 49 181 82 _c 730 | 487 273 _m 731 | 487 339 473 390 446 428 _c 732 | 418 466 381 485 334 485 _c 733 | }_e{286 485 249 466 222 428 _c 734 | 194 390 181 339 181 273 _c 735 | 181 207 194 155 222 117 _c 736 | 249 79 286 61 334 61 _c 737 | 381 61 418 79 446 117 _c 738 | 473 155 487 207 487 273 _c 739 | _cl}_e}_d 740 | /r{411 0 91 0 411 560 _sc 741 | 411 463 _m 742 | 401 469 390 473 378 476 _c 743 | 366 478 353 480 339 480 _c 744 | 288 480 249 463 222 430 _c 745 | 194 397 181 350 181 288 _c 746 | 181 0 _l 747 | 91 0 _l 748 | 91 547 _l 749 | 181 547 _l 750 | 181 462 _l 751 | 199 495 224 520 254 536 _c 752 | 284 552 321 560 365 560 _c 753 | 371 560 378 559 386 559 _c 754 | 393 558 401 557 411 555 _c 755 | 411 463 _l 756 | _cl}_d 757 | /s{{521 0 54 -13 472 560 _sc 758 | 443 531 _m 759 | 443 446 _l 760 | 417 458 391 468 364 475 _c 761 | 336 481 308 485 279 485 _c 762 | 234 485 200 478 178 464 _c 763 | 156 450 145 430 145 403 _c 764 | 145 382 153 366 169 354 _c 765 | 185 342 217 330 265 320 _c 766 | 296 313 _l 767 | 360 299 405 279 432 255 _c 768 | 458 230 472 195 472 151 _c 769 | 472 100 452 60 412 31 _c 770 | 372 1 316 -13 246 -13 _c 771 | 216 -13 186 -10 154 -5 _c 772 | }_e{122 0 89 8 54 20 _c 773 | 54 113 _l 774 | 87 95 120 82 152 74 _c 775 | 184 65 216 61 248 61 _c 776 | 290 61 323 68 346 82 _c 777 | 368 96 380 117 380 144 _c 778 | 380 168 371 187 355 200 _c 779 | 339 213 303 226 247 238 _c 780 | 216 245 _l 781 | 160 257 119 275 95 299 _c 782 | 70 323 58 356 58 399 _c 783 | 58 450 76 490 112 518 _c 784 | 148 546 200 560 268 560 _c 785 | 301 560 332 557 362 552 _c 786 | 391 547 418 540 443 531 _c 787 | }_e{_cl}_e}_d 788 | /t{392 0 27 0 368 702 _sc 789 | 183 702 _m 790 | 183 547 _l 791 | 368 547 _l 792 | 368 477 _l 793 | 183 477 _l 794 | 183 180 _l 795 | 183 135 189 106 201 94 _c 796 | 213 81 238 75 276 75 _c 797 | 368 75 _l 798 | 368 0 _l 799 | 276 0 _l 800 | 206 0 158 13 132 39 _c 801 | 106 65 93 112 93 180 _c 802 | 93 477 _l 803 | 27 477 _l 804 | 27 547 _l 805 | 93 547 _l 806 | 93 702 _l 807 | 183 702 _l 808 | _cl}_d 809 | /u{634 0 85 -13 543 560 _sc 810 | 85 216 _m 811 | 85 547 _l 812 | 175 547 _l 813 | 175 219 _l 814 | 175 167 185 129 205 103 _c 815 | 225 77 255 64 296 64 _c 816 | 344 64 383 79 411 110 _c 817 | 439 141 453 183 453 237 _c 818 | 453 547 _l 819 | 543 547 _l 820 | 543 0 _l 821 | 453 0 _l 822 | 453 84 _l 823 | 431 50 405 26 377 10 _c 824 | 348 -5 315 -13 277 -13 _c 825 | 214 -13 166 6 134 45 _c 826 | 101 83 85 140 85 216 _c 827 | 311 560 _m 828 | 311 560 _l 829 | _cl}_d 830 | /v{592 0 30 0 562 547 _sc 831 | 30 547 _m 832 | 125 547 _l 833 | 296 88 _l 834 | 467 547 _l 835 | 562 547 _l 836 | 357 0 _l 837 | 235 0 _l 838 | 30 547 _l 839 | _cl}_d 840 | end readonly def 841 | 842 | /BuildGlyph 843 | {exch begin 844 | CharStrings exch 845 | 2 copy known not{pop /.notdef}if 846 | true 3 1 roll get exec 847 | end}_d 848 | 849 | /BuildChar { 850 | 1 index /Encoding get exch get 851 | 1 index /BuildGlyph get exec 852 | }_d 853 | 854 | FontName currentdict end definefont pop 855 | end 856 | %%EndProlog 857 | mpldict begin 858 | -414 36 translate 859 | 1440 720 0 0 clipbox 860 | gsave 861 | 0 0 m 862 | 1440 0 l 863 | 1440 720 l 864 | 0 720 l 865 | cl 866 | 1.000 setgray 867 | fill 868 | grestore 869 | gsave 870 | 180 79.2 m 871 | 1296 79.2 l 872 | 1296 633.6 l 873 | 180 633.6 l 874 | cl 875 | 1.000 setgray 876 | fill 877 | grestore 878 | 0.800 setlinewidth 879 | 1 setlinejoin 880 | 0 setlinecap 881 | [] 0 setdash 882 | 0.000 setgray 883 | gsave 884 | /o { 885 | gsave 886 | newpath 887 | translate 888 | 0.8 setlinewidth 889 | 1 setlinejoin 890 | 0 setlinecap 891 | 0 0 m 892 | 0 -3.5 l 893 | 894 | gsave 895 | 0.000 setgray 896 | fill 897 | grestore 898 | stroke 899 | grestore 900 | } bind def 901 | 305.127 79.2 o 902 | grestore 903 | /DejaVuSans findfont 904 | 10.000 scalefont 905 | setfont 906 | gsave 907 | 298.767898 64.606250 translate 908 | 0.000000 rotate 909 | 0.000000 0.000000 m /six glyphshow 910 | 6.362305 0.000000 m /zero glyphshow 911 | grestore 912 | gsave 913 | /o { 914 | gsave 915 | newpath 916 | translate 917 | 0.8 setlinewidth 918 | 1 setlinejoin 919 | 0 setlinecap 920 | 0 0 m 921 | 0 -3.5 l 922 | 923 | gsave 924 | 0.000 setgray 925 | fill 926 | grestore 927 | stroke 928 | grestore 929 | } bind def 930 | 440.4 79.2 o 931 | grestore 932 | gsave 933 | 434.040625 64.606250 translate 934 | 0.000000 rotate 935 | 0.000000 0.000000 m /eight glyphshow 936 | 6.362305 0.000000 m /zero glyphshow 937 | grestore 938 | gsave 939 | /o { 940 | gsave 941 | newpath 942 | translate 943 | 0.8 setlinewidth 944 | 1 setlinejoin 945 | 0 setlinecap 946 | 0 0 m 947 | 0 -3.5 l 948 | 949 | gsave 950 | 0.000 setgray 951 | fill 952 | grestore 953 | stroke 954 | grestore 955 | } bind def 956 | 575.673 79.2 o 957 | grestore 958 | gsave 959 | 566.133665 64.606250 translate 960 | 0.000000 rotate 961 | 0.000000 0.000000 m /one glyphshow 962 | 6.362305 0.000000 m /zero glyphshow 963 | 12.724609 0.000000 m /zero glyphshow 964 | grestore 965 | gsave 966 | /o { 967 | gsave 968 | newpath 969 | translate 970 | 0.8 setlinewidth 971 | 1 setlinejoin 972 | 0 setlinecap 973 | 0 0 m 974 | 0 -3.5 l 975 | 976 | gsave 977 | 0.000 setgray 978 | fill 979 | grestore 980 | stroke 981 | grestore 982 | } bind def 983 | 710.945 79.2 o 984 | grestore 985 | gsave 986 | 701.406392 64.606250 translate 987 | 0.000000 rotate 988 | 0.000000 0.000000 m /one glyphshow 989 | 6.362305 0.000000 m /two glyphshow 990 | 12.724609 0.000000 m /zero glyphshow 991 | grestore 992 | gsave 993 | /o { 994 | gsave 995 | newpath 996 | translate 997 | 0.8 setlinewidth 998 | 1 setlinejoin 999 | 0 setlinecap 1000 | 0 0 m 1001 | 0 -3.5 l 1002 | 1003 | gsave 1004 | 0.000 setgray 1005 | fill 1006 | grestore 1007 | stroke 1008 | grestore 1009 | } bind def 1010 | 846.218 79.2 o 1011 | grestore 1012 | gsave 1013 | 836.679119 64.606250 translate 1014 | 0.000000 rotate 1015 | 0.000000 0.000000 m /one glyphshow 1016 | 6.362305 0.000000 m /four glyphshow 1017 | 12.724609 0.000000 m /zero glyphshow 1018 | grestore 1019 | gsave 1020 | /o { 1021 | gsave 1022 | newpath 1023 | translate 1024 | 0.8 setlinewidth 1025 | 1 setlinejoin 1026 | 0 setlinecap 1027 | 0 0 m 1028 | 0 -3.5 l 1029 | 1030 | gsave 1031 | 0.000 setgray 1032 | fill 1033 | grestore 1034 | stroke 1035 | grestore 1036 | } bind def 1037 | 981.491 79.2 o 1038 | grestore 1039 | gsave 1040 | 971.951847 64.606250 translate 1041 | 0.000000 rotate 1042 | 0.000000 0.000000 m /one glyphshow 1043 | 6.362305 0.000000 m /six glyphshow 1044 | 12.724609 0.000000 m /zero glyphshow 1045 | grestore 1046 | gsave 1047 | /o { 1048 | gsave 1049 | newpath 1050 | translate 1051 | 0.8 setlinewidth 1052 | 1 setlinejoin 1053 | 0 setlinecap 1054 | 0 0 m 1055 | 0 -3.5 l 1056 | 1057 | gsave 1058 | 0.000 setgray 1059 | fill 1060 | grestore 1061 | stroke 1062 | grestore 1063 | } bind def 1064 | 1116.76 79.2 o 1065 | grestore 1066 | gsave 1067 | 1107.224574 64.606250 translate 1068 | 0.000000 rotate 1069 | 0.000000 0.000000 m /one glyphshow 1070 | 6.362305 0.000000 m /eight glyphshow 1071 | 12.724609 0.000000 m /zero glyphshow 1072 | grestore 1073 | gsave 1074 | /o { 1075 | gsave 1076 | newpath 1077 | translate 1078 | 0.8 setlinewidth 1079 | 1 setlinejoin 1080 | 0 setlinecap 1081 | 0 0 m 1082 | 0 -3.5 l 1083 | 1084 | gsave 1085 | 0.000 setgray 1086 | fill 1087 | grestore 1088 | stroke 1089 | grestore 1090 | } bind def 1091 | 1252.04 79.2 o 1092 | grestore 1093 | gsave 1094 | 1242.497301 64.606250 translate 1095 | 0.000000 rotate 1096 | 0.000000 0.000000 m /two glyphshow 1097 | 6.362305 0.000000 m /zero glyphshow 1098 | 12.724609 0.000000 m /zero glyphshow 1099 | grestore 1100 | /DejaVuSans findfont 1101 | 14.000 scalefont 1102 | setfont 1103 | gsave 1104 | 634.429688 47.887500 translate 1105 | 0.000000 rotate 1106 | 0.000000 0.000000 m /E glyphshow 1107 | 8.845703 0.000000 m /p glyphshow 1108 | 17.732422 0.000000 m /i glyphshow 1109 | 21.622070 0.000000 m /s glyphshow 1110 | 28.916016 0.000000 m /o glyphshow 1111 | 37.481445 0.000000 m /d glyphshow 1112 | 46.368164 0.000000 m /e glyphshow 1113 | 54.981445 0.000000 m /space glyphshow 1114 | 59.431641 0.000000 m /D glyphshow 1115 | 70.211914 0.000000 m /u glyphshow 1116 | 79.084961 0.000000 m /r glyphshow 1117 | 84.840820 0.000000 m /a glyphshow 1118 | 93.419922 0.000000 m /t glyphshow 1119 | 98.909180 0.000000 m /i glyphshow 1120 | 102.798828 0.000000 m /o glyphshow 1121 | 111.364258 0.000000 m /n glyphshow 1122 | 120.237305 0.000000 m /space glyphshow 1123 | 124.687500 0.000000 m /parenleft glyphshow 1124 | 130.149414 0.000000 m /S glyphshow 1125 | 139.036133 0.000000 m /m glyphshow 1126 | 152.673828 0.000000 m /o glyphshow 1127 | 161.239258 0.000000 m /o glyphshow 1128 | 169.804688 0.000000 m /t glyphshow 1129 | 175.293945 0.000000 m /h glyphshow 1130 | 184.166992 0.000000 m /e glyphshow 1131 | 192.780273 0.000000 m /d glyphshow 1132 | 201.666992 0.000000 m /parenright glyphshow 1133 | grestore 1134 | gsave 1135 | /o { 1136 | gsave 1137 | newpath 1138 | translate 1139 | 0.8 setlinewidth 1140 | 1 setlinejoin 1141 | 0 setlinecap 1142 | 0 0 m 1143 | -3.5 0 l 1144 | 1145 | gsave 1146 | 0.000 setgray 1147 | fill 1148 | grestore 1149 | stroke 1150 | grestore 1151 | } bind def 1152 | 180 125.631 o 1153 | grestore 1154 | /DejaVuSans findfont 1155 | 10.000 scalefont 1156 | setfont 1157 | gsave 1158 | 160.281250 121.833894 translate 1159 | 0.000000 rotate 1160 | 0.000000 0.000000 m /one glyphshow 1161 | 6.362305 0.000000 m /five glyphshow 1162 | grestore 1163 | gsave 1164 | /o { 1165 | gsave 1166 | newpath 1167 | translate 1168 | 0.8 setlinewidth 1169 | 1 setlinejoin 1170 | 0 setlinecap 1171 | 0 0 m 1172 | -3.5 0 l 1173 | 1174 | gsave 1175 | 0.000 setgray 1176 | fill 1177 | grestore 1178 | stroke 1179 | grestore 1180 | } bind def 1181 | 180 202.554 o 1182 | grestore 1183 | gsave 1184 | 160.281250 198.756971 translate 1185 | 0.000000 rotate 1186 | 0.000000 0.000000 m /two glyphshow 1187 | 6.362305 0.000000 m /zero glyphshow 1188 | grestore 1189 | gsave 1190 | /o { 1191 | gsave 1192 | newpath 1193 | translate 1194 | 0.8 setlinewidth 1195 | 1 setlinejoin 1196 | 0 setlinecap 1197 | 0 0 m 1198 | -3.5 0 l 1199 | 1200 | gsave 1201 | 0.000 setgray 1202 | fill 1203 | grestore 1204 | stroke 1205 | grestore 1206 | } bind def 1207 | 180 279.477 o 1208 | grestore 1209 | gsave 1210 | 160.281250 275.680048 translate 1211 | 0.000000 rotate 1212 | 0.000000 0.000000 m /two glyphshow 1213 | 6.362305 0.000000 m /five glyphshow 1214 | grestore 1215 | gsave 1216 | /o { 1217 | gsave 1218 | newpath 1219 | translate 1220 | 0.8 setlinewidth 1221 | 1 setlinejoin 1222 | 0 setlinecap 1223 | 0 0 m 1224 | -3.5 0 l 1225 | 1226 | gsave 1227 | 0.000 setgray 1228 | fill 1229 | grestore 1230 | stroke 1231 | grestore 1232 | } bind def 1233 | 180 356.4 o 1234 | grestore 1235 | gsave 1236 | 160.281250 352.603125 translate 1237 | 0.000000 rotate 1238 | 0.000000 0.000000 m /three glyphshow 1239 | 6.362305 0.000000 m /zero glyphshow 1240 | grestore 1241 | gsave 1242 | /o { 1243 | gsave 1244 | newpath 1245 | translate 1246 | 0.8 setlinewidth 1247 | 1 setlinejoin 1248 | 0 setlinecap 1249 | 0 0 m 1250 | -3.5 0 l 1251 | 1252 | gsave 1253 | 0.000 setgray 1254 | fill 1255 | grestore 1256 | stroke 1257 | grestore 1258 | } bind def 1259 | 180 433.323 o 1260 | grestore 1261 | gsave 1262 | 160.281250 429.526202 translate 1263 | 0.000000 rotate 1264 | 0.000000 0.000000 m /three glyphshow 1265 | 6.362305 0.000000 m /five glyphshow 1266 | grestore 1267 | gsave 1268 | /o { 1269 | gsave 1270 | newpath 1271 | translate 1272 | 0.8 setlinewidth 1273 | 1 setlinejoin 1274 | 0 setlinecap 1275 | 0 0 m 1276 | -3.5 0 l 1277 | 1278 | gsave 1279 | 0.000 setgray 1280 | fill 1281 | grestore 1282 | stroke 1283 | grestore 1284 | } bind def 1285 | 180 510.246 o 1286 | grestore 1287 | gsave 1288 | 160.281250 506.449279 translate 1289 | 0.000000 rotate 1290 | 0.000000 0.000000 m /four glyphshow 1291 | 6.362305 0.000000 m /zero glyphshow 1292 | grestore 1293 | gsave 1294 | /o { 1295 | gsave 1296 | newpath 1297 | translate 1298 | 0.8 setlinewidth 1299 | 1 setlinejoin 1300 | 0 setlinecap 1301 | 0 0 m 1302 | -3.5 0 l 1303 | 1304 | gsave 1305 | 0.000 setgray 1306 | fill 1307 | grestore 1308 | stroke 1309 | grestore 1310 | } bind def 1311 | 180 587.169 o 1312 | grestore 1313 | gsave 1314 | 160.281250 583.372356 translate 1315 | 0.000000 rotate 1316 | 0.000000 0.000000 m /four glyphshow 1317 | 6.362305 0.000000 m /five glyphshow 1318 | grestore 1319 | /DejaVuSans findfont 1320 | 14.000 scalefont 1321 | setfont 1322 | gsave 1323 | 153.375000 326.001563 translate 1324 | 90.000000 rotate 1325 | 0.000000 0.000000 m /D glyphshow 1326 | 10.780273 0.000000 m /u glyphshow 1327 | 19.653320 0.000000 m /r glyphshow 1328 | 25.409180 0.000000 m /a glyphshow 1329 | 33.988281 0.000000 m /t glyphshow 1330 | 39.477539 0.000000 m /i glyphshow 1331 | 43.367188 0.000000 m /o glyphshow 1332 | 51.932617 0.000000 m /n glyphshow 1333 | grestore 1334 | 1.500 setlinewidth 1335 | 2 setlinecap 1336 | 0.122 0.467 0.706 setrgbcolor 1337 | gsave 1338 | 1116 554.4 180 79.2 clipbox 1339 | 230.727273 464.4 m 1340 | 237.490909 409.015385 l 1341 | 244.254545 378.553846 l 1342 | 251.018182 351.169231 l 1343 | 257.781818 336.092308 l 1344 | 264.545455 327.784615 l 1345 | 271.309091 300.707692 l 1346 | 278.072727 300.707692 l 1347 | 291.6 299.476923 l 1348 | 298.363636 277.015385 l 1349 | 305.127273 276.092308 l 1350 | 311.890909 258.553846 l 1351 | 318.654545 256.707692 l 1352 | 325.418182 258.553846 l 1353 | 332.181818 233.015385 l 1354 | 338.945455 214.553846 l 1355 | 345.709091 214.861538 l 1356 | 352.472727 213.630769 l 1357 | 359.236364 212.092308 l 1358 | 366 209.323077 l 1359 | 372.763636 187.476923 l 1360 | 379.527273 184.092308 l 1361 | 386.290909 182.246154 l 1362 | 393.054545 183.476923 l 1363 | 399.818182 183.169231 l 1364 | 406.581818 180.092308 l 1365 | 413.345455 171.476923 l 1366 | 420.109091 166.553846 l 1367 | 426.872727 164.092308 l 1368 | 433.636364 162.553846 l 1369 | 440.4 161.630769 l 1370 | 447.163636 160.4 l 1371 | 453.927273 160.092308 l 1372 | 460.690909 157.015385 l 1373 | 467.454545 155.476923 l 1374 | 474.218182 151.784615 l 1375 | 480.981818 154.246154 l 1376 | 487.745455 156.092308 l 1377 | 494.509091 155.784615 l 1378 | 501.272727 156.4 l 1379 | 508.036364 154.553846 l 1380 | 514.8 154.246154 l 1381 | 521.563636 154.553846 l 1382 | 528.327273 155.169231 l 1383 | 535.090909 152.092308 l 1384 | 541.854545 149.630769 l 1385 | 548.618182 148.092308 l 1386 | 555.381818 147.476923 l 1387 | 562.145455 145.015385 l 1388 | 568.909091 143.169231 l 1389 | 575.672727 143.169231 l 1390 | 582.436364 140.092308 l 1391 | 589.2 138.861538 l 1392 | 602.727273 138.246154 l 1393 | 609.490909 138.553846 l 1394 | 616.254545 137.630769 l 1395 | 623.018182 136.4 l 1396 | 629.781818 134.553846 l 1397 | 636.545455 133.938462 l 1398 | 643.309091 132.4 l 1399 | 650.072727 133.323077 l 1400 | 656.836364 133.938462 l 1401 | 663.6 131.476923 l 1402 | 670.363636 129.938462 l 1403 | 677.127273 130.246154 l 1404 | 683.890909 128.707692 l 1405 | 697.418182 128.707692 l 1406 | 704.181818 129.015385 l 1407 | 710.945455 128.707692 l 1408 | 717.709091 130.246154 l 1409 | 724.472727 130.861538 l 1410 | 731.236364 128.707692 l 1411 | 738 126.861538 l 1412 | 744.763636 129.015385 l 1413 | 751.527273 129.323077 l 1414 | 758.290909 128.4 l 1415 | 765.054545 128.4 l 1416 | 771.818182 127.169231 l 1417 | 778.581818 127.169231 l 1418 | 785.345455 126.553846 l 1419 | 792.109091 125.323077 l 1420 | 798.872727 123.476923 l 1421 | 805.636364 124.092308 l 1422 | 812.4 125.015385 l 1423 | 819.163636 122.246154 l 1424 | 825.927273 121.015385 l 1425 | 832.690909 120.707692 l 1426 | 839.454545 119.476923 l 1427 | 846.218182 118.861538 l 1428 | 852.981818 118.553846 l 1429 | 859.745455 117.630769 l 1430 | 873.272727 116.4 l 1431 | 886.8 117.630769 l 1432 | 893.563636 117.323077 l 1433 | 900.327273 116.707692 l 1434 | 907.090909 116.4 l 1435 | 913.854545 117.015385 l 1436 | 920.618182 117.015385 l 1437 | 927.381818 117.323077 l 1438 | 934.145455 116.707692 l 1439 | 940.909091 116.707692 l 1440 | 947.672727 115.784615 l 1441 | 954.436364 116.092308 l 1442 | 961.2 116.092308 l 1443 | 967.963636 115.476923 l 1444 | 981.490909 116.092308 l 1445 | 988.254545 114.861538 l 1446 | 995.018182 114.553846 l 1447 | 1001.781818 114.553846 l 1448 | 1008.545455 114.246154 l 1449 | 1015.309091 113.630769 l 1450 | 1028.836364 113.630769 l 1451 | 1035.6 114.246154 l 1452 | 1042.363636 113.938462 l 1453 | 1055.890909 113.938462 l 1454 | 1062.654545 114.246154 l 1455 | 1076.181818 114.246154 l 1456 | 1082.945455 112.092308 l 1457 | 1096.472727 112.707692 l 1458 | 1103.236364 112.092308 l 1459 | 1110 112.092308 l 1460 | 1116.763636 113.323077 l 1461 | 1123.527273 112.707692 l 1462 | 1137.054545 110.861538 l 1463 | 1143.818182 110.861538 l 1464 | 1150.581818 109.938462 l 1465 | 1157.345455 109.938462 l 1466 | 1164.109091 110.553846 l 1467 | 1170.872727 110.246154 l 1468 | 1177.636364 110.861538 l 1469 | 1184.4 110.861538 l 1470 | 1191.163636 110.246154 l 1471 | 1197.927273 111.476923 l 1472 | 1211.454545 111.476923 l 1473 | 1218.218182 110.861538 l 1474 | 1224.981818 111.476923 l 1475 | 1231.745455 111.476923 l 1476 | 1238.509091 112.092308 l 1477 | 1245.272727 112.092308 l 1478 | 1245.272727 112.092308 l 1479 | stroke 1480 | grestore 1481 | 1.000 0.498 0.055 setrgbcolor 1482 | gsave 1483 | 1116 554.4 180 79.2 clipbox 1484 | 230.727273 608.4 m 1485 | 237.490909 538.246154 l 1486 | 244.254545 457.323077 l 1487 | 251.018182 433.015385 l 1488 | 257.781818 412.4 l 1489 | 264.545455 375.169231 l 1490 | 271.309091 373.015385 l 1491 | 278.072727 359.476923 l 1492 | 284.836364 356.4 l 1493 | 291.6 345.323077 l 1494 | 298.363636 307.784615 l 1495 | 305.127273 305.938462 l 1496 | 311.890909 301.938462 l 1497 | 318.654545 289.938462 l 1498 | 325.418182 280.4 l 1499 | 332.181818 277.630769 l 1500 | 338.945455 278.861538 l 1501 | 345.709091 279.169231 l 1502 | 352.472727 281.323077 l 1503 | 359.236364 236.4 l 1504 | 366 236.4 l 1505 | 372.763636 234.246154 l 1506 | 379.527273 224.4 l 1507 | 386.290909 225.323077 l 1508 | 393.054545 223.476923 l 1509 | 399.818182 223.784615 l 1510 | 406.581818 225.630769 l 1511 | 413.345455 225.630769 l 1512 | 420.109091 224.4 l 1513 | 426.872727 225.323077 l 1514 | 433.636364 224.092308 l 1515 | 440.4 223.476923 l 1516 | 447.163636 224.707692 l 1517 | 453.927273 190.246154 l 1518 | 460.690909 189.630769 l 1519 | 467.454545 185.323077 l 1520 | 474.218182 177.015385 l 1521 | 480.981818 178.246154 l 1522 | 487.745455 177.938462 l 1523 | 494.509091 167.784615 l 1524 | 501.272727 165.938462 l 1525 | 508.036364 165.938462 l 1526 | 514.8 165.015385 l 1527 | 521.563636 165.323077 l 1528 | 528.327273 161.630769 l 1529 | 535.090909 160.092308 l 1530 | 541.854545 157.630769 l 1531 | 548.618182 156.707692 l 1532 | 555.381818 158.246154 l 1533 | 562.145455 158.861538 l 1534 | 568.909091 157.938462 l 1535 | 575.672727 155.476923 l 1536 | 582.436364 154.246154 l 1537 | 589.2 154.861538 l 1538 | 595.963636 155.784615 l 1539 | 602.727273 155.476923 l 1540 | 609.490909 153.630769 l 1541 | 616.254545 153.323077 l 1542 | 623.018182 150.246154 l 1543 | 629.781818 145.630769 l 1544 | 636.545455 146.861538 l 1545 | 643.309091 144.4 l 1546 | 656.836364 143.784615 l 1547 | 663.6 144.4 l 1548 | 670.363636 145.630769 l 1549 | 677.127273 142.553846 l 1550 | 683.890909 140.707692 l 1551 | 690.654545 139.476923 l 1552 | 697.418182 139.784615 l 1553 | 704.181818 138.553846 l 1554 | 710.945455 136.092308 l 1555 | 717.709091 135.784615 l 1556 | 724.472727 134.246154 l 1557 | 731.236364 133.015385 l 1558 | 738 130.861538 l 1559 | 744.763636 128.4 l 1560 | 751.527273 128.092308 l 1561 | 758.290909 128.4 l 1562 | 765.054545 128.092308 l 1563 | 771.818182 128.092308 l 1564 | 778.581818 127.476923 l 1565 | 785.345455 126.246154 l 1566 | 792.109091 125.630769 l 1567 | 798.872727 124.707692 l 1568 | 805.636364 124.4 l 1569 | 812.4 124.707692 l 1570 | 819.163636 124.4 l 1571 | 825.927273 122.861538 l 1572 | 832.690909 122.553846 l 1573 | 839.454545 121.630769 l 1574 | 846.218182 123.476923 l 1575 | 852.981818 122.861538 l 1576 | 859.745455 123.784615 l 1577 | 866.509091 123.476923 l 1578 | 880.036364 121.630769 l 1579 | 886.8 121.323077 l 1580 | 893.563636 119.784615 l 1581 | 900.327273 117.630769 l 1582 | 907.090909 117.015385 l 1583 | 913.854545 117.323077 l 1584 | 920.618182 117.323077 l 1585 | 927.381818 115.476923 l 1586 | 934.145455 115.476923 l 1587 | 940.909091 115.169231 l 1588 | 954.436364 115.169231 l 1589 | 967.963636 116.4 l 1590 | 974.727273 114.861538 l 1591 | 981.490909 115.476923 l 1592 | 988.254545 115.784615 l 1593 | 995.018182 115.169231 l 1594 | 1008.545455 113.323077 l 1595 | 1015.309091 113.938462 l 1596 | 1022.072727 113.323077 l 1597 | 1028.836364 112.092308 l 1598 | 1035.6 111.169231 l 1599 | 1049.127273 111.169231 l 1600 | 1062.654545 109.323077 l 1601 | 1069.418182 109.938462 l 1602 | 1076.181818 110.861538 l 1603 | 1082.945455 109.938462 l 1604 | 1089.709091 110.553846 l 1605 | 1096.472727 110.553846 l 1606 | 1103.236364 108.707692 l 1607 | 1110 109.323077 l 1608 | 1116.763636 108.707692 l 1609 | 1123.527273 109.630769 l 1610 | 1130.290909 110.246154 l 1611 | 1137.054545 110.246154 l 1612 | 1143.818182 111.169231 l 1613 | 1157.345455 109.323077 l 1614 | 1164.109091 109.323077 l 1615 | 1170.872727 108.707692 l 1616 | 1177.636364 108.4 l 1617 | 1184.4 105.015385 l 1618 | 1191.163636 105.015385 l 1619 | 1197.927273 104.4 l 1620 | 1211.454545 105.630769 l 1621 | 1224.981818 105.630769 l 1622 | 1231.745455 106.553846 l 1623 | 1238.509091 108.092308 l 1624 | 1245.272727 108.4 l 1625 | 1245.272727 108.4 l 1626 | stroke 1627 | grestore 1628 | 0.800 setlinewidth 1629 | 0 setlinejoin 1630 | [] 0 setdash 1631 | 0.000 setgray 1632 | gsave 1633 | 180 79.2 m 1634 | 180 633.6 l 1635 | stroke 1636 | grestore 1637 | gsave 1638 | 1296 79.2 m 1639 | 1296 633.6 l 1640 | stroke 1641 | grestore 1642 | gsave 1643 | 180 79.2 m 1644 | 1296 79.2 l 1645 | stroke 1646 | grestore 1647 | gsave 1648 | 180 633.6 m 1649 | 1296 633.6 l 1650 | stroke 1651 | grestore 1652 | /DejaVuSans findfont 1653 | 18.000 scalefont 1654 | setfont 1655 | gsave 1656 | 570.210938 639.600000 translate 1657 | 0.000000 rotate 1658 | 0.000000 0.000000 m /B glyphshow 1659 | 12.348633 0.000000 m /e glyphshow 1660 | 23.422852 0.000000 m /n glyphshow 1661 | 34.831055 0.000000 m /c glyphshow 1662 | 44.727539 0.000000 m /h glyphshow 1663 | 56.135742 0.000000 m /m glyphshow 1664 | 73.669922 0.000000 m /a glyphshow 1665 | 84.700195 0.000000 m /r glyphshow 1666 | 92.100586 0.000000 m /k glyphshow 1667 | 102.524414 0.000000 m /space glyphshow 1668 | 108.246094 0.000000 m /T glyphshow 1669 | 119.209961 0.000000 m /r glyphshow 1670 | 126.610352 0.000000 m /a glyphshow 1671 | 137.640625 0.000000 m /i glyphshow 1672 | 142.641602 0.000000 m /n glyphshow 1673 | 154.049805 0.000000 m /i glyphshow 1674 | 159.050781 0.000000 m /n glyphshow 1675 | 170.458984 0.000000 m /g glyphshow 1676 | 181.884766 0.000000 m /space glyphshow 1677 | 187.606445 0.000000 m /R glyphshow 1678 | 200.113281 0.000000 m /e glyphshow 1679 | 211.187500 0.000000 m /s glyphshow 1680 | 220.565430 0.000000 m /u glyphshow 1681 | 231.973633 0.000000 m /l glyphshow 1682 | 236.974609 0.000000 m /t glyphshow 1683 | 244.032227 0.000000 m /s glyphshow 1684 | 253.410156 0.000000 m /space glyphshow 1685 | 259.131836 0.000000 m /D glyphshow 1686 | 272.992188 0.000000 m /I glyphshow 1687 | 278.300781 0.000000 m /S glyphshow 1688 | 289.726562 0.000000 m /T glyphshow 1689 | 300.721680 0.000000 m /R glyphshow 1690 | 313.228516 0.000000 m /A glyphshow 1691 | 325.541992 0.000000 m /L glyphshow 1692 | grestore 1693 | 1.000 setlinewidth 1694 | 0 setlinecap 1695 | 0.800 setgray 1696 | gsave 1697 | 1161.765625 558.88125 m 1698 | 1282 558.88125 l 1699 | 1284.666667 558.88125 1286 560.214583 1286 562.88125 c 1700 | 1286 619.6 l 1701 | 1286 622.266667 1284.666667 623.6 1282 623.6 c 1702 | 1161.765625 623.6 l 1703 | 1159.098958 623.6 1157.765625 622.266667 1157.765625 619.6 c 1704 | 1157.765625 562.88125 l 1705 | 1157.765625 560.214583 1159.098958 558.88125 1161.765625 558.88125 c 1706 | cl 1707 | gsave 1708 | 1.000 setgray 1709 | fill 1710 | grestore 1711 | stroke 1712 | grestore 1713 | 1.500 setlinewidth 1714 | 1 setlinejoin 1715 | 2 setlinecap 1716 | [] 0 setdash 1717 | 0.122 0.467 0.706 setrgbcolor 1718 | gsave 1719 | 1165.765625 607.396875 m 1720 | 1205.765625 607.396875 l 1721 | stroke 1722 | grestore 1723 | 0.000 setgray 1724 | /DejaVuSans findfont 1725 | 20.000 scalefont 1726 | setfont 1727 | gsave 1728 | 1221.765625 600.396875 translate 1729 | 0.000000 rotate 1730 | 0.000000 0.000000 m /E glyphshow 1731 | 12.636719 0.000000 m /n glyphshow 1732 | 25.312500 0.000000 m /v glyphshow 1733 | 37.148438 0.000000 m /space glyphshow 1734 | 43.505859 0.000000 m /seven glyphshow 1735 | grestore 1736 | 1.000 0.498 0.055 setrgbcolor 1737 | gsave 1738 | 1165.765625 578.0375 m 1739 | 1205.765625 578.0375 l 1740 | stroke 1741 | grestore 1742 | 0.000 setgray 1743 | gsave 1744 | 1221.765625 571.037500 translate 1745 | 0.000000 rotate 1746 | 0.000000 0.000000 m /E glyphshow 1747 | 12.636719 0.000000 m /n glyphshow 1748 | 25.312500 0.000000 m /v glyphshow 1749 | 37.148438 0.000000 m /space glyphshow 1750 | 43.505859 0.000000 m /eight glyphshow 1751 | grestore 1752 | 1753 | end 1754 | showpage 1755 | -------------------------------------------------------------------------------- /code/experiments/TEST_DISTRAL/Benchmark457-distral-duration.eps: -------------------------------------------------------------------------------- 1 | %!PS-Adobe-3.0 EPSF-3.0 2 | %%Title: Benchmark-distral-duration.eps 3 | %%Creator: matplotlib version 2.1.0, http://matplotlib.org/ 4 | %%CreationDate: Sun Mar 18 23:02:36 2018 5 | %%Orientation: portrait 6 | %%BoundingBox: -414 36 1026 756 7 | %%EndComments 8 | %%BeginProlog 9 | /mpldict 8 dict def 10 | mpldict begin 11 | /m { moveto } bind def 12 | /l { lineto } bind def 13 | /r { rlineto } bind def 14 | /c { curveto } bind def 15 | /cl { closepath } bind def 16 | /box { 17 | m 18 | 1 index 0 r 19 | 0 exch r 20 | neg 0 r 21 | cl 22 | } bind def 23 | /clipbox { 24 | box 25 | clip 26 | newpath 27 | } bind def 28 | %!PS-Adobe-3.0 Resource-Font 29 | %%Title: DejaVu Sans 30 | %%Copyright: Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. DejaVu changes are in public domain 31 | %%Creator: Converted from TrueType to type 3 by PPR 32 | 25 dict begin 33 | /_d{bind def}bind def 34 | /_m{moveto}_d 35 | /_l{lineto}_d 36 | /_cl{closepath eofill}_d 37 | /_c{curveto}_d 38 | /_sc{7 -1 roll{setcachedevice}{pop pop pop pop pop pop}ifelse}_d 39 | /_e{exec}_d 40 | /FontName /DejaVuSans def 41 | /PaintType 0 def 42 | /FontMatrix[.001 0 0 .001 0 0]def 43 | /FontBBox[-1021 -463 1793 1232]def 44 | /FontType 3 def 45 | /Encoding [ /space /parenleft /parenright /zero /one /two /four /five /six /seven /eight /A /B /D /E /I /L /R /S /T /a /c /d /e /g /h /i /k /l /m /n /o /p /r /s /t /u /v ] def 46 | /FontInfo 10 dict dup begin 47 | /FamilyName (DejaVu Sans) def 48 | /FullName (DejaVu Sans) def 49 | /Notice (Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved. Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved. DejaVu changes are in public domain ) def 50 | /Weight (Book) def 51 | /Version (Version 2.35) def 52 | /ItalicAngle 0.0 def 53 | /isFixedPitch false def 54 | /UnderlinePosition -130 def 55 | /UnderlineThickness 90 def 56 | end readonly def 57 | /CharStrings 39 dict dup begin 58 | /.notdef 0 def 59 | /space{318 0 0 0 0 0 _sc 60 | }_d 61 | /parenleft{390 0 86 -131 310 759 _sc 62 | 310 759 _m 63 | 266 683 234 609 213 536 _c 64 | 191 463 181 389 181 314 _c 65 | 181 238 191 164 213 91 _c 66 | 234 17 266 -56 310 -131 _c 67 | 232 -131 _l 68 | 183 -54 146 20 122 94 _c 69 | 98 168 86 241 86 314 _c 70 | 86 386 98 459 122 533 _c 71 | 146 607 182 682 232 759 _c 72 | 310 759 _l 73 | _cl}_d 74 | /parenright{390 0 80 -131 304 759 _sc 75 | 80 759 _m 76 | 158 759 _l 77 | 206 682 243 607 267 533 _c 78 | 291 459 304 386 304 314 _c 79 | 304 241 291 168 267 94 _c 80 | 243 20 206 -54 158 -131 _c 81 | 80 -131 _l 82 | 123 -56 155 17 177 91 _c 83 | 198 164 209 238 209 314 _c 84 | 209 389 198 463 177 536 _c 85 | 155 609 123 683 80 759 _c 86 | _cl}_d 87 | /zero{636 0 66 -13 570 742 _sc 88 | 318 664 _m 89 | 267 664 229 639 203 589 _c 90 | 177 539 165 464 165 364 _c 91 | 165 264 177 189 203 139 _c 92 | 229 89 267 64 318 64 _c 93 | 369 64 407 89 433 139 _c 94 | 458 189 471 264 471 364 _c 95 | 471 464 458 539 433 589 _c 96 | 407 639 369 664 318 664 _c 97 | 318 742 _m 98 | 399 742 461 709 505 645 _c 99 | 548 580 570 486 570 364 _c 100 | 570 241 548 147 505 83 _c 101 | 461 19 399 -13 318 -13 _c 102 | 236 -13 173 19 130 83 _c 103 | 87 147 66 241 66 364 _c 104 | 66 486 87 580 130 645 _c 105 | 173 709 236 742 318 742 _c 106 | _cl}_d 107 | /one{636 0 110 0 544 729 _sc 108 | 124 83 _m 109 | 285 83 _l 110 | 285 639 _l 111 | 110 604 _l 112 | 110 694 _l 113 | 284 729 _l 114 | 383 729 _l 115 | 383 83 _l 116 | 544 83 _l 117 | 544 0 _l 118 | 124 0 _l 119 | 124 83 _l 120 | _cl}_d 121 | /two{{636 0 73 0 536 742 _sc 122 | 192 83 _m 123 | 536 83 _l 124 | 536 0 _l 125 | 73 0 _l 126 | 73 83 _l 127 | 110 121 161 173 226 239 _c 128 | 290 304 331 346 348 365 _c 129 | 380 400 402 430 414 455 _c 130 | 426 479 433 504 433 528 _c 131 | 433 566 419 598 392 622 _c 132 | 365 646 330 659 286 659 _c 133 | 255 659 222 653 188 643 _c 134 | 154 632 117 616 78 594 _c 135 | 78 694 _l 136 | 118 710 155 722 189 730 _c 137 | 223 738 255 742 284 742 _c 138 | }_e{359 742 419 723 464 685 _c 139 | 509 647 532 597 532 534 _c 140 | 532 504 526 475 515 449 _c 141 | 504 422 484 390 454 354 _c 142 | 446 344 420 317 376 272 _c 143 | 332 227 271 164 192 83 _c 144 | _cl}_e}_d 145 | /four{636 0 49 0 580 729 _sc 146 | 378 643 _m 147 | 129 254 _l 148 | 378 254 _l 149 | 378 643 _l 150 | 352 729 _m 151 | 476 729 _l 152 | 476 254 _l 153 | 580 254 _l 154 | 580 172 _l 155 | 476 172 _l 156 | 476 0 _l 157 | 378 0 _l 158 | 378 172 _l 159 | 49 172 _l 160 | 49 267 _l 161 | 352 729 _l 162 | _cl}_d 163 | /five{{636 0 77 -13 549 729 _sc 164 | 108 729 _m 165 | 495 729 _l 166 | 495 646 _l 167 | 198 646 _l 168 | 198 467 _l 169 | 212 472 227 476 241 478 _c 170 | 255 480 270 482 284 482 _c 171 | 365 482 429 459 477 415 _c 172 | 525 370 549 310 549 234 _c 173 | 549 155 524 94 475 51 _c 174 | 426 8 357 -13 269 -13 _c 175 | 238 -13 207 -10 175 -6 _c 176 | 143 -1 111 6 77 17 _c 177 | 77 116 _l 178 | 106 100 136 88 168 80 _c 179 | 199 72 232 69 267 69 _c 180 | }_e{323 69 368 83 401 113 _c 181 | 433 143 450 183 450 234 _c 182 | 450 284 433 324 401 354 _c 183 | 368 384 323 399 267 399 _c 184 | 241 399 214 396 188 390 _c 185 | 162 384 135 375 108 363 _c 186 | 108 729 _l 187 | _cl}_e}_d 188 | /six{{636 0 70 -13 573 742 _sc 189 | 330 404 _m 190 | 286 404 251 388 225 358 _c 191 | 199 328 186 286 186 234 _c 192 | 186 181 199 139 225 109 _c 193 | 251 79 286 64 330 64 _c 194 | 374 64 409 79 435 109 _c 195 | 461 139 474 181 474 234 _c 196 | 474 286 461 328 435 358 _c 197 | 409 388 374 404 330 404 _c 198 | 526 713 _m 199 | 526 623 _l 200 | 501 635 476 644 451 650 _c 201 | 425 656 400 659 376 659 _c 202 | 310 659 260 637 226 593 _c 203 | }_e{192 549 172 482 168 394 _c 204 | 187 422 211 444 240 459 _c 205 | 269 474 301 482 336 482 _c 206 | 409 482 467 459 509 415 _c 207 | 551 371 573 310 573 234 _c 208 | 573 159 550 99 506 54 _c 209 | 462 9 403 -13 330 -13 _c 210 | 246 -13 181 19 137 83 _c 211 | 92 147 70 241 70 364 _c 212 | 70 479 97 571 152 639 _c 213 | 206 707 280 742 372 742 _c 214 | 396 742 421 739 447 735 _c 215 | 472 730 498 723 526 713 _c 216 | _cl}_e}_d 217 | /seven{636 0 82 0 551 729 _sc 218 | 82 729 _m 219 | 551 729 _l 220 | 551 687 _l 221 | 286 0 _l 222 | 183 0 _l 223 | 432 646 _l 224 | 82 646 _l 225 | 82 729 _l 226 | _cl}_d 227 | /eight{{636 0 68 -13 568 742 _sc 228 | 318 346 _m 229 | 271 346 234 333 207 308 _c 230 | 180 283 167 249 167 205 _c 231 | 167 161 180 126 207 101 _c 232 | 234 76 271 64 318 64 _c 233 | 364 64 401 76 428 102 _c 234 | 455 127 469 161 469 205 _c 235 | 469 249 455 283 429 308 _c 236 | 402 333 365 346 318 346 _c 237 | 219 388 _m 238 | 177 398 144 418 120 447 _c 239 | 96 476 85 511 85 553 _c 240 | 85 611 105 657 147 691 _c 241 | 188 725 245 742 318 742 _c 242 | }_e{390 742 447 725 489 691 _c 243 | 530 657 551 611 551 553 _c 244 | 551 511 539 476 515 447 _c 245 | 491 418 459 398 417 388 _c 246 | 464 377 501 355 528 323 _c 247 | 554 291 568 251 568 205 _c 248 | 568 134 546 80 503 43 _c 249 | 459 5 398 -13 318 -13 _c 250 | 237 -13 175 5 132 43 _c 251 | 89 80 68 134 68 205 _c 252 | 68 251 81 291 108 323 _c 253 | 134 355 171 377 219 388 _c 254 | 183 544 _m 255 | 183 506 194 476 218 455 _c 256 | }_e{242 434 275 424 318 424 _c 257 | 360 424 393 434 417 455 _c 258 | 441 476 453 506 453 544 _c 259 | 453 582 441 611 417 632 _c 260 | 393 653 360 664 318 664 _c 261 | 275 664 242 653 218 632 _c 262 | 194 611 183 582 183 544 _c 263 | _cl}_e}_d 264 | /A{684 0 8 0 676 729 _sc 265 | 342 632 _m 266 | 208 269 _l 267 | 476 269 _l 268 | 342 632 _l 269 | 286 729 _m 270 | 398 729 _l 271 | 676 0 _l 272 | 573 0 _l 273 | 507 187 _l 274 | 178 187 _l 275 | 112 0 _l 276 | 8 0 _l 277 | 286 729 _l 278 | _cl}_d 279 | /B{{686 0 98 0 615 729 _sc 280 | 197 348 _m 281 | 197 81 _l 282 | 355 81 _l 283 | 408 81 447 92 473 114 _c 284 | 498 136 511 169 511 215 _c 285 | 511 260 498 293 473 315 _c 286 | 447 337 408 348 355 348 _c 287 | 197 348 _l 288 | 197 648 _m 289 | 197 428 _l 290 | 343 428 _l 291 | 391 428 426 437 450 455 _c 292 | 474 473 486 500 486 538 _c 293 | 486 574 474 602 450 620 _c 294 | 426 638 391 648 343 648 _c 295 | 197 648 _l 296 | 98 729 _m 297 | 350 729 _l 298 | }_e{425 729 483 713 524 682 _c 299 | 564 650 585 606 585 549 _c 300 | 585 504 574 468 553 442 _c 301 | 532 416 502 399 462 393 _c 302 | 510 382 548 360 575 327 _c 303 | 601 294 615 253 615 204 _c 304 | 615 138 592 88 548 53 _c 305 | 504 17 441 0 360 0 _c 306 | 98 0 _l 307 | 98 729 _l 308 | _cl}_e}_d 309 | /D{770 0 98 0 711 729 _sc 310 | 197 648 _m 311 | 197 81 _l 312 | 316 81 _l 313 | 416 81 490 103 537 149 _c 314 | 583 195 607 267 607 365 _c 315 | 607 463 583 534 537 580 _c 316 | 490 625 416 648 316 648 _c 317 | 197 648 _l 318 | 98 729 _m 319 | 301 729 _l 320 | 442 729 546 699 612 641 _c 321 | 678 582 711 490 711 365 _c 322 | 711 239 677 147 611 88 _c 323 | 545 29 441 0 301 0 _c 324 | 98 0 _l 325 | 98 729 _l 326 | _cl}_d 327 | /E{632 0 98 0 568 729 _sc 328 | 98 729 _m 329 | 559 729 _l 330 | 559 646 _l 331 | 197 646 _l 332 | 197 430 _l 333 | 544 430 _l 334 | 544 347 _l 335 | 197 347 _l 336 | 197 83 _l 337 | 568 83 _l 338 | 568 0 _l 339 | 98 0 _l 340 | 98 729 _l 341 | _cl}_d 342 | /I{295 0 98 0 197 729 _sc 343 | 98 729 _m 344 | 197 729 _l 345 | 197 0 _l 346 | 98 0 _l 347 | 98 729 _l 348 | _cl}_d 349 | /L{557 0 98 0 552 729 _sc 350 | 98 729 _m 351 | 197 729 _l 352 | 197 83 _l 353 | 552 83 _l 354 | 552 0 _l 355 | 98 0 _l 356 | 98 729 _l 357 | _cl}_d 358 | /R{{695 0 98 0 666 729 _sc 359 | 444 342 _m 360 | 465 334 486 319 506 296 _c 361 | 526 272 546 240 566 199 _c 362 | 666 0 _l 363 | 560 0 _l 364 | 467 187 _l 365 | 443 235 419 268 397 284 _c 366 | 374 300 343 308 304 308 _c 367 | 197 308 _l 368 | 197 0 _l 369 | 98 0 _l 370 | 98 729 _l 371 | 321 729 _l 372 | 404 729 466 711 507 677 _c 373 | 548 642 569 589 569 519 _c 374 | 569 473 558 434 537 404 _c 375 | 515 374 484 353 444 342 _c 376 | 197 648 _m 377 | 197 389 _l 378 | 321 389 _l 379 | }_e{368 389 404 400 428 422 _c 380 | 452 444 465 476 465 519 _c 381 | 465 561 452 593 428 615 _c 382 | 404 637 368 648 321 648 _c 383 | 197 648 _l 384 | _cl}_e}_d 385 | /S{{635 0 66 -13 579 742 _sc 386 | 535 705 _m 387 | 535 609 _l 388 | 497 627 462 640 429 649 _c 389 | 395 657 363 662 333 662 _c 390 | 279 662 237 651 208 631 _c 391 | 179 610 165 580 165 542 _c 392 | 165 510 174 485 194 469 _c 393 | 213 452 250 439 304 429 _c 394 | 364 417 _l 395 | 437 403 491 378 526 343 _c 396 | 561 307 579 260 579 201 _c 397 | 579 130 555 77 508 41 _c 398 | 460 5 391 -13 300 -13 _c 399 | 265 -13 228 -9 189 -2 _c 400 | }_e{150 5 110 16 69 32 _c 401 | 69 134 _l 402 | 109 111 148 94 186 83 _c 403 | 224 71 262 66 300 66 _c 404 | 356 66 399 77 430 99 _c 405 | 460 121 476 152 476 194 _c 406 | 476 230 465 258 443 278 _c 407 | 421 298 385 313 335 323 _c 408 | 275 335 _l 409 | 201 349 148 372 115 404 _c 410 | 82 435 66 478 66 534 _c 411 | 66 598 88 649 134 686 _c 412 | 179 723 242 742 322 742 _c 413 | 356 742 390 739 426 733 _c 414 | 461 727 497 717 535 705 _c 415 | }_e{_cl}_e}_d 416 | /T{611 0 -2 0 614 729 _sc 417 | -2 729 _m 418 | 614 729 _l 419 | 614 646 _l 420 | 355 646 _l 421 | 355 0 _l 422 | 256 0 _l 423 | 256 646 _l 424 | -2 646 _l 425 | -2 729 _l 426 | _cl}_d 427 | /a{{613 0 60 -13 522 560 _sc 428 | 343 275 _m 429 | 270 275 220 266 192 250 _c 430 | 164 233 150 205 150 165 _c 431 | 150 133 160 107 181 89 _c 432 | 202 70 231 61 267 61 _c 433 | 317 61 357 78 387 114 _c 434 | 417 149 432 196 432 255 _c 435 | 432 275 _l 436 | 343 275 _l 437 | 522 312 _m 438 | 522 0 _l 439 | 432 0 _l 440 | 432 83 _l 441 | 411 49 385 25 355 10 _c 442 | 325 -5 287 -13 243 -13 _c 443 | 187 -13 142 2 109 33 _c 444 | 76 64 60 106 60 159 _c 445 | }_e{60 220 80 266 122 298 _c 446 | 163 329 224 345 306 345 _c 447 | 432 345 _l 448 | 432 354 _l 449 | 432 395 418 427 391 450 _c 450 | 364 472 326 484 277 484 _c 451 | 245 484 215 480 185 472 _c 452 | 155 464 127 453 100 439 _c 453 | 100 522 _l 454 | 132 534 164 544 195 550 _c 455 | 226 556 256 560 286 560 _c 456 | 365 560 424 539 463 498 _c 457 | 502 457 522 395 522 312 _c 458 | _cl}_e}_d 459 | /c{{550 0 55 -13 488 560 _sc 460 | 488 526 _m 461 | 488 442 _l 462 | 462 456 437 466 411 473 _c 463 | 385 480 360 484 334 484 _c 464 | 276 484 230 465 198 428 _c 465 | 166 391 150 339 150 273 _c 466 | 150 206 166 154 198 117 _c 467 | 230 80 276 62 334 62 _c 468 | 360 62 385 65 411 72 _c 469 | 437 79 462 90 488 104 _c 470 | 488 21 _l 471 | 462 9 436 0 410 -5 _c 472 | 383 -10 354 -13 324 -13 _c 473 | 242 -13 176 12 128 64 _c 474 | }_e{79 115 55 185 55 273 _c 475 | 55 362 79 432 128 483 _c 476 | 177 534 244 560 330 560 _c 477 | 358 560 385 557 411 551 _c 478 | 437 545 463 537 488 526 _c 479 | _cl}_e}_d 480 | /d{{635 0 55 -13 544 760 _sc 481 | 454 464 _m 482 | 454 760 _l 483 | 544 760 _l 484 | 544 0 _l 485 | 454 0 _l 486 | 454 82 _l 487 | 435 49 411 25 382 10 _c 488 | 353 -5 319 -13 279 -13 _c 489 | 213 -13 159 13 117 65 _c 490 | 75 117 55 187 55 273 _c 491 | 55 359 75 428 117 481 _c 492 | 159 533 213 560 279 560 _c 493 | 319 560 353 552 382 536 _c 494 | 411 520 435 496 454 464 _c 495 | 148 273 _m 496 | 148 207 161 155 188 117 _c 497 | 215 79 253 61 301 61 _c 498 | }_e{348 61 385 79 413 117 _c 499 | 440 155 454 207 454 273 _c 500 | 454 339 440 390 413 428 _c 501 | 385 466 348 485 301 485 _c 502 | 253 485 215 466 188 428 _c 503 | 161 390 148 339 148 273 _c 504 | _cl}_e}_d 505 | /e{{615 0 55 -13 562 560 _sc 506 | 562 296 _m 507 | 562 252 _l 508 | 149 252 _l 509 | 153 190 171 142 205 110 _c 510 | 238 78 284 62 344 62 _c 511 | 378 62 412 66 444 74 _c 512 | 476 82 509 95 541 113 _c 513 | 541 28 _l 514 | 509 14 476 3 442 -3 _c 515 | 408 -9 373 -13 339 -13 _c 516 | 251 -13 182 12 131 62 _c 517 | 80 112 55 181 55 268 _c 518 | 55 357 79 428 127 481 _c 519 | 175 533 241 560 323 560 _c 520 | 397 560 455 536 498 489 _c 521 | }_e{540 441 562 377 562 296 _c 522 | 472 322 _m 523 | 471 371 457 410 431 440 _c 524 | 404 469 368 484 324 484 _c 525 | 274 484 234 469 204 441 _c 526 | 174 413 156 373 152 322 _c 527 | 472 322 _l 528 | _cl}_e}_d 529 | /g{{635 0 55 -207 544 560 _sc 530 | 454 280 _m 531 | 454 344 440 395 414 431 _c 532 | 387 467 349 485 301 485 _c 533 | 253 485 215 467 188 431 _c 534 | 161 395 148 344 148 280 _c 535 | 148 215 161 165 188 129 _c 536 | 215 93 253 75 301 75 _c 537 | 349 75 387 93 414 129 _c 538 | 440 165 454 215 454 280 _c 539 | 544 68 _m 540 | 544 -24 523 -93 482 -139 _c 541 | 440 -184 377 -207 292 -207 _c 542 | 260 -207 231 -204 203 -200 _c 543 | 175 -195 147 -188 121 -178 _c 544 | }_e{121 -91 _l 545 | 147 -105 173 -115 199 -122 _c 546 | 225 -129 251 -133 278 -133 _c 547 | 336 -133 380 -117 410 -87 _c 548 | 439 -56 454 -10 454 52 _c 549 | 454 96 _l 550 | 435 64 411 40 382 24 _c 551 | 353 8 319 0 279 0 _c 552 | 211 0 157 25 116 76 _c 553 | 75 127 55 195 55 280 _c 554 | 55 364 75 432 116 483 _c 555 | 157 534 211 560 279 560 _c 556 | 319 560 353 552 382 536 _c 557 | 411 520 435 496 454 464 _c 558 | 454 547 _l 559 | 544 547 _l 560 | }_e{544 68 _l 561 | _cl}_e}_d 562 | /h{634 0 91 0 549 760 _sc 563 | 549 330 _m 564 | 549 0 _l 565 | 459 0 _l 566 | 459 327 _l 567 | 459 379 448 417 428 443 _c 568 | 408 469 378 482 338 482 _c 569 | 289 482 251 466 223 435 _c 570 | 195 404 181 362 181 309 _c 571 | 181 0 _l 572 | 91 0 _l 573 | 91 760 _l 574 | 181 760 _l 575 | 181 462 _l 576 | 202 494 227 519 257 535 _c 577 | 286 551 320 560 358 560 _c 578 | 420 560 468 540 500 501 _c 579 | 532 462 549 405 549 330 _c 580 | _cl}_d 581 | /i{278 0 94 0 184 760 _sc 582 | 94 547 _m 583 | 184 547 _l 584 | 184 0 _l 585 | 94 0 _l 586 | 94 547 _l 587 | 94 760 _m 588 | 184 760 _l 589 | 184 646 _l 590 | 94 646 _l 591 | 94 760 _l 592 | _cl}_d 593 | /k{579 0 91 0 576 760 _sc 594 | 91 760 _m 595 | 181 760 _l 596 | 181 311 _l 597 | 449 547 _l 598 | 564 547 _l 599 | 274 291 _l 600 | 576 0 _l 601 | 459 0 _l 602 | 181 267 _l 603 | 181 0 _l 604 | 91 0 _l 605 | 91 760 _l 606 | _cl}_d 607 | /l{278 0 94 0 184 760 _sc 608 | 94 760 _m 609 | 184 760 _l 610 | 184 0 _l 611 | 94 0 _l 612 | 94 760 _l 613 | _cl}_d 614 | /m{{974 0 91 0 889 560 _sc 615 | 520 442 _m 616 | 542 482 569 511 600 531 _c 617 | 631 550 668 560 711 560 _c 618 | 767 560 811 540 842 500 _c 619 | 873 460 889 403 889 330 _c 620 | 889 0 _l 621 | 799 0 _l 622 | 799 327 _l 623 | 799 379 789 418 771 444 _c 624 | 752 469 724 482 686 482 _c 625 | 639 482 602 466 575 435 _c 626 | 548 404 535 362 535 309 _c 627 | 535 0 _l 628 | 445 0 _l 629 | 445 327 _l 630 | 445 379 435 418 417 444 _c 631 | 398 469 369 482 331 482 _c 632 | }_e{285 482 248 466 221 435 _c 633 | 194 404 181 362 181 309 _c 634 | 181 0 _l 635 | 91 0 _l 636 | 91 547 _l 637 | 181 547 _l 638 | 181 462 _l 639 | 201 495 226 520 255 536 _c 640 | 283 552 317 560 357 560 _c 641 | 397 560 430 550 458 530 _c 642 | 486 510 506 480 520 442 _c 643 | _cl}_e}_d 644 | /n{634 0 91 0 549 560 _sc 645 | 549 330 _m 646 | 549 0 _l 647 | 459 0 _l 648 | 459 327 _l 649 | 459 379 448 417 428 443 _c 650 | 408 469 378 482 338 482 _c 651 | 289 482 251 466 223 435 _c 652 | 195 404 181 362 181 309 _c 653 | 181 0 _l 654 | 91 0 _l 655 | 91 547 _l 656 | 181 547 _l 657 | 181 462 _l 658 | 202 494 227 519 257 535 _c 659 | 286 551 320 560 358 560 _c 660 | 420 560 468 540 500 501 _c 661 | 532 462 549 405 549 330 _c 662 | _cl}_d 663 | /o{612 0 55 -13 557 560 _sc 664 | 306 484 _m 665 | 258 484 220 465 192 427 _c 666 | 164 389 150 338 150 273 _c 667 | 150 207 163 156 191 118 _c 668 | 219 80 257 62 306 62 _c 669 | 354 62 392 80 420 118 _c 670 | 448 156 462 207 462 273 _c 671 | 462 337 448 389 420 427 _c 672 | 392 465 354 484 306 484 _c 673 | 306 560 _m 674 | 384 560 445 534 490 484 _c 675 | 534 433 557 363 557 273 _c 676 | 557 183 534 113 490 63 _c 677 | 445 12 384 -13 306 -13 _c 678 | 227 -13 165 12 121 63 _c 679 | 77 113 55 183 55 273 _c 680 | 55 363 77 433 121 484 _c 681 | 165 534 227 560 306 560 _c 682 | _cl}_d 683 | /p{{635 0 91 -207 580 560 _sc 684 | 181 82 _m 685 | 181 -207 _l 686 | 91 -207 _l 687 | 91 547 _l 688 | 181 547 _l 689 | 181 464 _l 690 | 199 496 223 520 252 536 _c 691 | 281 552 316 560 356 560 _c 692 | 422 560 476 533 518 481 _c 693 | 559 428 580 359 580 273 _c 694 | 580 187 559 117 518 65 _c 695 | 476 13 422 -13 356 -13 _c 696 | 316 -13 281 -5 252 10 _c 697 | 223 25 199 49 181 82 _c 698 | 487 273 _m 699 | 487 339 473 390 446 428 _c 700 | 418 466 381 485 334 485 _c 701 | }_e{286 485 249 466 222 428 _c 702 | 194 390 181 339 181 273 _c 703 | 181 207 194 155 222 117 _c 704 | 249 79 286 61 334 61 _c 705 | 381 61 418 79 446 117 _c 706 | 473 155 487 207 487 273 _c 707 | _cl}_e}_d 708 | /r{411 0 91 0 411 560 _sc 709 | 411 463 _m 710 | 401 469 390 473 378 476 _c 711 | 366 478 353 480 339 480 _c 712 | 288 480 249 463 222 430 _c 713 | 194 397 181 350 181 288 _c 714 | 181 0 _l 715 | 91 0 _l 716 | 91 547 _l 717 | 181 547 _l 718 | 181 462 _l 719 | 199 495 224 520 254 536 _c 720 | 284 552 321 560 365 560 _c 721 | 371 560 378 559 386 559 _c 722 | 393 558 401 557 411 555 _c 723 | 411 463 _l 724 | _cl}_d 725 | /s{{521 0 54 -13 472 560 _sc 726 | 443 531 _m 727 | 443 446 _l 728 | 417 458 391 468 364 475 _c 729 | 336 481 308 485 279 485 _c 730 | 234 485 200 478 178 464 _c 731 | 156 450 145 430 145 403 _c 732 | 145 382 153 366 169 354 _c 733 | 185 342 217 330 265 320 _c 734 | 296 313 _l 735 | 360 299 405 279 432 255 _c 736 | 458 230 472 195 472 151 _c 737 | 472 100 452 60 412 31 _c 738 | 372 1 316 -13 246 -13 _c 739 | 216 -13 186 -10 154 -5 _c 740 | }_e{122 0 89 8 54 20 _c 741 | 54 113 _l 742 | 87 95 120 82 152 74 _c 743 | 184 65 216 61 248 61 _c 744 | 290 61 323 68 346 82 _c 745 | 368 96 380 117 380 144 _c 746 | 380 168 371 187 355 200 _c 747 | 339 213 303 226 247 238 _c 748 | 216 245 _l 749 | 160 257 119 275 95 299 _c 750 | 70 323 58 356 58 399 _c 751 | 58 450 76 490 112 518 _c 752 | 148 546 200 560 268 560 _c 753 | 301 560 332 557 362 552 _c 754 | 391 547 418 540 443 531 _c 755 | }_e{_cl}_e}_d 756 | /t{392 0 27 0 368 702 _sc 757 | 183 702 _m 758 | 183 547 _l 759 | 368 547 _l 760 | 368 477 _l 761 | 183 477 _l 762 | 183 180 _l 763 | 183 135 189 106 201 94 _c 764 | 213 81 238 75 276 75 _c 765 | 368 75 _l 766 | 368 0 _l 767 | 276 0 _l 768 | 206 0 158 13 132 39 _c 769 | 106 65 93 112 93 180 _c 770 | 93 477 _l 771 | 27 477 _l 772 | 27 547 _l 773 | 93 547 _l 774 | 93 702 _l 775 | 183 702 _l 776 | _cl}_d 777 | /u{634 0 85 -13 543 560 _sc 778 | 85 216 _m 779 | 85 547 _l 780 | 175 547 _l 781 | 175 219 _l 782 | 175 167 185 129 205 103 _c 783 | 225 77 255 64 296 64 _c 784 | 344 64 383 79 411 110 _c 785 | 439 141 453 183 453 237 _c 786 | 453 547 _l 787 | 543 547 _l 788 | 543 0 _l 789 | 453 0 _l 790 | 453 84 _l 791 | 431 50 405 26 377 10 _c 792 | 348 -5 315 -13 277 -13 _c 793 | 214 -13 166 6 134 45 _c 794 | 101 83 85 140 85 216 _c 795 | 311 560 _m 796 | 311 560 _l 797 | _cl}_d 798 | /v{592 0 30 0 562 547 _sc 799 | 30 547 _m 800 | 125 547 _l 801 | 296 88 _l 802 | 467 547 _l 803 | 562 547 _l 804 | 357 0 _l 805 | 235 0 _l 806 | 30 547 _l 807 | _cl}_d 808 | end readonly def 809 | 810 | /BuildGlyph 811 | {exch begin 812 | CharStrings exch 813 | 2 copy known not{pop /.notdef}if 814 | true 3 1 roll get exec 815 | end}_d 816 | 817 | /BuildChar { 818 | 1 index /Encoding get exch get 819 | 1 index /BuildGlyph get exec 820 | }_d 821 | 822 | FontName currentdict end definefont pop 823 | end 824 | %%EndProlog 825 | mpldict begin 826 | -414 36 translate 827 | 1440 720 0 0 clipbox 828 | gsave 829 | 0 0 m 830 | 1440 0 l 831 | 1440 720 l 832 | 0 720 l 833 | cl 834 | 1.000 setgray 835 | fill 836 | grestore 837 | gsave 838 | 180 79.2 m 839 | 1296 79.2 l 840 | 1296 633.6 l 841 | 180 633.6 l 842 | cl 843 | 1.000 setgray 844 | fill 845 | grestore 846 | 0.800 setlinewidth 847 | 1 setlinejoin 848 | 0 setlinecap 849 | [] 0 setdash 850 | 0.000 setgray 851 | gsave 852 | /o { 853 | gsave 854 | newpath 855 | translate 856 | 0.8 setlinewidth 857 | 1 setlinejoin 858 | 0 setlinecap 859 | 0 0 m 860 | 0 -3.5 l 861 | 862 | gsave 863 | 0.000 setgray 864 | fill 865 | grestore 866 | stroke 867 | grestore 868 | } bind def 869 | 305.127 79.2 o 870 | grestore 871 | /DejaVuSans findfont 872 | 10.000 scalefont 873 | setfont 874 | gsave 875 | 298.767898 64.606250 translate 876 | 0.000000 rotate 877 | 0.000000 0.000000 m /six glyphshow 878 | 6.362305 0.000000 m /zero glyphshow 879 | grestore 880 | gsave 881 | /o { 882 | gsave 883 | newpath 884 | translate 885 | 0.8 setlinewidth 886 | 1 setlinejoin 887 | 0 setlinecap 888 | 0 0 m 889 | 0 -3.5 l 890 | 891 | gsave 892 | 0.000 setgray 893 | fill 894 | grestore 895 | stroke 896 | grestore 897 | } bind def 898 | 440.4 79.2 o 899 | grestore 900 | gsave 901 | 434.040625 64.606250 translate 902 | 0.000000 rotate 903 | 0.000000 0.000000 m /eight glyphshow 904 | 6.362305 0.000000 m /zero glyphshow 905 | grestore 906 | gsave 907 | /o { 908 | gsave 909 | newpath 910 | translate 911 | 0.8 setlinewidth 912 | 1 setlinejoin 913 | 0 setlinecap 914 | 0 0 m 915 | 0 -3.5 l 916 | 917 | gsave 918 | 0.000 setgray 919 | fill 920 | grestore 921 | stroke 922 | grestore 923 | } bind def 924 | 575.673 79.2 o 925 | grestore 926 | gsave 927 | 566.133665 64.606250 translate 928 | 0.000000 rotate 929 | 0.000000 0.000000 m /one glyphshow 930 | 6.362305 0.000000 m /zero glyphshow 931 | 12.724609 0.000000 m /zero glyphshow 932 | grestore 933 | gsave 934 | /o { 935 | gsave 936 | newpath 937 | translate 938 | 0.8 setlinewidth 939 | 1 setlinejoin 940 | 0 setlinecap 941 | 0 0 m 942 | 0 -3.5 l 943 | 944 | gsave 945 | 0.000 setgray 946 | fill 947 | grestore 948 | stroke 949 | grestore 950 | } bind def 951 | 710.945 79.2 o 952 | grestore 953 | gsave 954 | 701.406392 64.606250 translate 955 | 0.000000 rotate 956 | 0.000000 0.000000 m /one glyphshow 957 | 6.362305 0.000000 m /two glyphshow 958 | 12.724609 0.000000 m /zero glyphshow 959 | grestore 960 | gsave 961 | /o { 962 | gsave 963 | newpath 964 | translate 965 | 0.8 setlinewidth 966 | 1 setlinejoin 967 | 0 setlinecap 968 | 0 0 m 969 | 0 -3.5 l 970 | 971 | gsave 972 | 0.000 setgray 973 | fill 974 | grestore 975 | stroke 976 | grestore 977 | } bind def 978 | 846.218 79.2 o 979 | grestore 980 | gsave 981 | 836.679119 64.606250 translate 982 | 0.000000 rotate 983 | 0.000000 0.000000 m /one glyphshow 984 | 6.362305 0.000000 m /four glyphshow 985 | 12.724609 0.000000 m /zero glyphshow 986 | grestore 987 | gsave 988 | /o { 989 | gsave 990 | newpath 991 | translate 992 | 0.8 setlinewidth 993 | 1 setlinejoin 994 | 0 setlinecap 995 | 0 0 m 996 | 0 -3.5 l 997 | 998 | gsave 999 | 0.000 setgray 1000 | fill 1001 | grestore 1002 | stroke 1003 | grestore 1004 | } bind def 1005 | 981.491 79.2 o 1006 | grestore 1007 | gsave 1008 | 971.951847 64.606250 translate 1009 | 0.000000 rotate 1010 | 0.000000 0.000000 m /one glyphshow 1011 | 6.362305 0.000000 m /six glyphshow 1012 | 12.724609 0.000000 m /zero glyphshow 1013 | grestore 1014 | gsave 1015 | /o { 1016 | gsave 1017 | newpath 1018 | translate 1019 | 0.8 setlinewidth 1020 | 1 setlinejoin 1021 | 0 setlinecap 1022 | 0 0 m 1023 | 0 -3.5 l 1024 | 1025 | gsave 1026 | 0.000 setgray 1027 | fill 1028 | grestore 1029 | stroke 1030 | grestore 1031 | } bind def 1032 | 1116.76 79.2 o 1033 | grestore 1034 | gsave 1035 | 1107.224574 64.606250 translate 1036 | 0.000000 rotate 1037 | 0.000000 0.000000 m /one glyphshow 1038 | 6.362305 0.000000 m /eight glyphshow 1039 | 12.724609 0.000000 m /zero glyphshow 1040 | grestore 1041 | gsave 1042 | /o { 1043 | gsave 1044 | newpath 1045 | translate 1046 | 0.8 setlinewidth 1047 | 1 setlinejoin 1048 | 0 setlinecap 1049 | 0 0 m 1050 | 0 -3.5 l 1051 | 1052 | gsave 1053 | 0.000 setgray 1054 | fill 1055 | grestore 1056 | stroke 1057 | grestore 1058 | } bind def 1059 | 1252.04 79.2 o 1060 | grestore 1061 | gsave 1062 | 1242.497301 64.606250 translate 1063 | 0.000000 rotate 1064 | 0.000000 0.000000 m /two glyphshow 1065 | 6.362305 0.000000 m /zero glyphshow 1066 | 12.724609 0.000000 m /zero glyphshow 1067 | grestore 1068 | /DejaVuSans findfont 1069 | 14.000 scalefont 1070 | setfont 1071 | gsave 1072 | 634.429688 47.887500 translate 1073 | 0.000000 rotate 1074 | 0.000000 0.000000 m /E glyphshow 1075 | 8.845703 0.000000 m /p glyphshow 1076 | 17.732422 0.000000 m /i glyphshow 1077 | 21.622070 0.000000 m /s glyphshow 1078 | 28.916016 0.000000 m /o glyphshow 1079 | 37.481445 0.000000 m /d glyphshow 1080 | 46.368164 0.000000 m /e glyphshow 1081 | 54.981445 0.000000 m /space glyphshow 1082 | 59.431641 0.000000 m /D glyphshow 1083 | 70.211914 0.000000 m /u glyphshow 1084 | 79.084961 0.000000 m /r glyphshow 1085 | 84.840820 0.000000 m /a glyphshow 1086 | 93.419922 0.000000 m /t glyphshow 1087 | 98.909180 0.000000 m /i glyphshow 1088 | 102.798828 0.000000 m /o glyphshow 1089 | 111.364258 0.000000 m /n glyphshow 1090 | 120.237305 0.000000 m /space glyphshow 1091 | 124.687500 0.000000 m /parenleft glyphshow 1092 | 130.149414 0.000000 m /S glyphshow 1093 | 139.036133 0.000000 m /m glyphshow 1094 | 152.673828 0.000000 m /o glyphshow 1095 | 161.239258 0.000000 m /o glyphshow 1096 | 169.804688 0.000000 m /t glyphshow 1097 | 175.293945 0.000000 m /h glyphshow 1098 | 184.166992 0.000000 m /e glyphshow 1099 | 192.780273 0.000000 m /d glyphshow 1100 | 201.666992 0.000000 m /parenright glyphshow 1101 | grestore 1102 | gsave 1103 | /o { 1104 | gsave 1105 | newpath 1106 | translate 1107 | 0.8 setlinewidth 1108 | 1 setlinejoin 1109 | 0 setlinecap 1110 | 0 0 m 1111 | -3.5 0 l 1112 | 1113 | gsave 1114 | 0.000 setgray 1115 | fill 1116 | grestore 1117 | stroke 1118 | grestore 1119 | } bind def 1120 | 180 155.64 o 1121 | grestore 1122 | /DejaVuSans findfont 1123 | 10.000 scalefont 1124 | setfont 1125 | gsave 1126 | 160.281250 151.843298 translate 1127 | 0.000000 rotate 1128 | 0.000000 0.000000 m /two glyphshow 1129 | 6.362305 0.000000 m /zero glyphshow 1130 | grestore 1131 | gsave 1132 | /o { 1133 | gsave 1134 | newpath 1135 | translate 1136 | 0.8 setlinewidth 1137 | 1 setlinejoin 1138 | 0 setlinecap 1139 | 0 0 m 1140 | -3.5 0 l 1141 | 1142 | gsave 1143 | 0.000 setgray 1144 | fill 1145 | grestore 1146 | stroke 1147 | grestore 1148 | } bind def 1149 | 180 259.365 o 1150 | grestore 1151 | gsave 1152 | 160.281250 255.568344 translate 1153 | 0.000000 rotate 1154 | 0.000000 0.000000 m /four glyphshow 1155 | 6.362305 0.000000 m /zero glyphshow 1156 | grestore 1157 | gsave 1158 | /o { 1159 | gsave 1160 | newpath 1161 | translate 1162 | 0.8 setlinewidth 1163 | 1 setlinejoin 1164 | 0 setlinecap 1165 | 0 0 m 1166 | -3.5 0 l 1167 | 1168 | gsave 1169 | 0.000 setgray 1170 | fill 1171 | grestore 1172 | stroke 1173 | grestore 1174 | } bind def 1175 | 180 363.09 o 1176 | grestore 1177 | gsave 1178 | 160.281250 359.293390 translate 1179 | 0.000000 rotate 1180 | 0.000000 0.000000 m /six glyphshow 1181 | 6.362305 0.000000 m /zero glyphshow 1182 | grestore 1183 | gsave 1184 | /o { 1185 | gsave 1186 | newpath 1187 | translate 1188 | 0.8 setlinewidth 1189 | 1 setlinejoin 1190 | 0 setlinecap 1191 | 0 0 m 1192 | -3.5 0 l 1193 | 1194 | gsave 1195 | 0.000 setgray 1196 | fill 1197 | grestore 1198 | stroke 1199 | grestore 1200 | } bind def 1201 | 180 466.815 o 1202 | grestore 1203 | gsave 1204 | 160.281250 463.018437 translate 1205 | 0.000000 rotate 1206 | 0.000000 0.000000 m /eight glyphshow 1207 | 6.362305 0.000000 m /zero glyphshow 1208 | grestore 1209 | gsave 1210 | /o { 1211 | gsave 1212 | newpath 1213 | translate 1214 | 0.8 setlinewidth 1215 | 1 setlinejoin 1216 | 0 setlinecap 1217 | 0 0 m 1218 | -3.5 0 l 1219 | 1220 | gsave 1221 | 0.000 setgray 1222 | fill 1223 | grestore 1224 | stroke 1225 | grestore 1226 | } bind def 1227 | 180 570.54 o 1228 | grestore 1229 | gsave 1230 | 153.921875 566.743483 translate 1231 | 0.000000 rotate 1232 | 0.000000 0.000000 m /one glyphshow 1233 | 6.362305 0.000000 m /zero glyphshow 1234 | 12.724609 0.000000 m /zero glyphshow 1235 | grestore 1236 | /DejaVuSans findfont 1237 | 14.000 scalefont 1238 | setfont 1239 | gsave 1240 | 147.015625 326.001563 translate 1241 | 90.000000 rotate 1242 | 0.000000 0.000000 m /D glyphshow 1243 | 10.780273 0.000000 m /u glyphshow 1244 | 19.653320 0.000000 m /r glyphshow 1245 | 25.409180 0.000000 m /a glyphshow 1246 | 33.988281 0.000000 m /t glyphshow 1247 | 39.477539 0.000000 m /i glyphshow 1248 | 43.367188 0.000000 m /o glyphshow 1249 | 51.932617 0.000000 m /n glyphshow 1250 | grestore 1251 | 1.500 setlinewidth 1252 | 2 setlinecap 1253 | 0.122 0.467 0.706 setrgbcolor 1254 | gsave 1255 | 1116 554.4 180 79.2 clipbox 1256 | 230.727273 608.4 m 1257 | 237.490909 499.073801 l 1258 | 244.254545 483.307594 l 1259 | 251.018182 478.743692 l 1260 | 257.781818 476.876641 l 1261 | 264.545455 443.788352 l 1262 | 271.309091 439.535625 l 1263 | 278.072727 434.867998 l 1264 | 284.836364 403.128133 l 1265 | 291.6 358.941264 l 1266 | 298.363636 353.340111 l 1267 | 305.127273 349.398559 l 1268 | 311.890909 340.37448 l 1269 | 318.654545 331.661576 l 1270 | 325.418182 282.703355 l 1271 | 332.181818 273.471825 l 1272 | 338.945455 271.604775 l 1273 | 345.709091 271.812225 l 1274 | 352.472727 263.721671 l 1275 | 359.236364 260.921095 l 1276 | 366 246.814489 l 1277 | 372.763636 245.258613 l 1278 | 379.527273 242.354312 l 1279 | 386.290909 234.471208 l 1280 | 393.054545 227.93653 l 1281 | 399.818182 212.688948 l 1282 | 406.581818 208.332496 l 1283 | 413.345455 206.050545 l 1284 | 420.109091 195.574316 l 1285 | 426.872727 190.699239 l 1286 | 440.4 188.832188 l 1287 | 447.163636 186.239062 l 1288 | 453.927273 177.214983 l 1289 | 460.690909 172.65108 l 1290 | 467.454545 172.02873 l 1291 | 474.218182 170.887755 l 1292 | 480.981818 165.494052 l 1293 | 487.745455 164.041902 l 1294 | 494.509091 162.900926 l 1295 | 501.272727 162.174851 l 1296 | 508.036364 159.996625 l 1297 | 521.563636 158.855649 l 1298 | 528.327273 159.374275 l 1299 | 535.090909 159.581725 l 1300 | 541.854545 159.166824 l 1301 | 548.618182 156.988598 l 1302 | 555.381818 155.225273 l 1303 | 568.909091 154.188022 l 1304 | 575.672727 154.291747 l 1305 | 589.2 153.773122 l 1306 | 602.727273 153.047047 l 1307 | 609.490909 150.661371 l 1308 | 616.254545 150.03902 l 1309 | 629.781818 149.520395 l 1310 | 636.545455 148.898045 l 1311 | 643.309091 147.549619 l 1312 | 650.072727 146.823544 l 1313 | 656.836364 145.682568 l 1314 | 663.6 144.230418 l 1315 | 670.363636 144.022968 l 1316 | 677.127273 144.541593 l 1317 | 683.890909 143.608068 l 1318 | 690.654545 145.475118 l 1319 | 697.418182 144.956493 l 1320 | 710.945455 145.060218 l 1321 | 717.709091 145.993744 l 1322 | 724.472727 145.993744 l 1323 | 731.236364 146.201194 l 1324 | 738 145.578843 l 1325 | 744.763636 145.890019 l 1326 | 751.527273 145.786293 l 1327 | 758.290909 145.475118 l 1328 | 765.054545 144.749043 l 1329 | 771.818182 144.230418 l 1330 | 785.345455 144.230418 l 1331 | 798.872727 143.919243 l 1332 | 825.927273 143.400617 l 1333 | 832.690909 143.608068 l 1334 | 839.454545 143.504342 l 1335 | 846.218182 143.608068 l 1336 | 852.981818 143.504342 l 1337 | 859.745455 142.985717 l 1338 | 866.509091 141.637292 l 1339 | 880.036364 140.288866 l 1340 | 893.563636 139.459066 l 1341 | 913.854545 138.94044 l 1342 | 934.145455 139.147891 l 1343 | 940.909091 139.044165 l 1344 | 947.672727 139.666516 l 1345 | 954.436364 139.770241 l 1346 | 961.2 140.081416 l 1347 | 974.727273 139.666516 l 1348 | 981.490909 139.977691 l 1349 | 988.254545 140.081416 l 1350 | 995.018182 139.977691 l 1351 | 1001.781818 139.666516 l 1352 | 1008.545455 138.421815 l 1353 | 1015.309091 137.384565 l 1354 | 1022.072727 137.28084 l 1355 | 1028.836364 134.480263 l 1356 | 1035.6 134.065363 l 1357 | 1042.363636 133.857913 l 1358 | 1049.127273 134.169088 l 1359 | 1055.890909 133.650463 l 1360 | 1069.418182 133.339288 l 1361 | 1076.181818 133.650463 l 1362 | 1082.945455 133.546738 l 1363 | 1089.709091 133.235563 l 1364 | 1096.472727 133.443013 l 1365 | 1110 133.443013 l 1366 | 1116.763636 134.065363 l 1367 | 1123.527273 134.065363 l 1368 | 1130.290909 134.376538 l 1369 | 1137.054545 134.065363 l 1370 | 1143.818182 134.169088 l 1371 | 1150.581818 133.754188 l 1372 | 1164.109091 133.339288 l 1373 | 1170.872727 132.924388 l 1374 | 1177.636364 133.028113 l 1375 | 1184.4 132.924388 l 1376 | 1191.163636 132.509488 l 1377 | 1204.690909 132.198312 l 1378 | 1211.454545 132.405763 l 1379 | 1224.981818 132.302037 l 1380 | 1245.272727 131.990862 l 1381 | 1245.272727 131.990862 l 1382 | stroke 1383 | grestore 1384 | 1.000 0.498 0.055 setrgbcolor 1385 | gsave 1386 | 1116 554.4 180 79.2 clipbox 1387 | 230.727273 160.204075 m 1388 | 237.490909 138.214365 l 1389 | 244.254545 136.243589 l 1390 | 251.018182 129.605186 l 1391 | 257.781818 129.294011 l 1392 | 264.545455 129.397736 l 1393 | 278.072727 128.464211 l 1394 | 284.836364 116.32838 l 1395 | 291.6 116.53583 l 1396 | 298.363636 116.53583 l 1397 | 305.127273 115.809755 l 1398 | 325.418182 115.394855 l 1399 | 332.181818 114.87623 l 1400 | 338.945455 114.150154 l 1401 | 345.709091 113.942704 l 1402 | 359.236364 114.253879 l 1403 | 366 114.046429 l 1404 | 372.763636 114.253879 l 1405 | 379.527273 114.046429 l 1406 | 386.290909 113.320354 l 1407 | 393.054545 113.009179 l 1408 | 399.818182 113.216629 l 1409 | 406.581818 111.868203 l 1410 | 413.345455 111.557028 l 1411 | 420.109091 111.868203 l 1412 | 433.636364 111.660753 l 1413 | 440.4 111.349578 l 1414 | 447.163636 111.453303 l 1415 | 453.927273 111.245853 l 1416 | 460.690909 110.623503 l 1417 | 467.454545 110.623503 l 1418 | 474.218182 110.416053 l 1419 | 480.981818 110.001153 l 1420 | 494.509091 110.519778 l 1421 | 508.036364 110.623503 l 1422 | 514.8 110.416053 l 1423 | 541.854545 110.519778 l 1424 | 548.618182 110.830953 l 1425 | 568.909091 110.416053 l 1426 | 575.672727 109.275077 l 1427 | 582.436364 108.860177 l 1428 | 595.963636 109.689977 l 1429 | 602.727273 109.482527 l 1430 | 609.490909 109.482527 l 1431 | 616.254545 109.897427 l 1432 | 629.781818 109.689977 l 1433 | 636.545455 109.171352 l 1434 | 643.309091 108.860177 l 1435 | 650.072727 108.963902 l 1436 | 656.836364 109.275077 l 1437 | 677.127273 109.378802 l 1438 | 683.890909 108.963902 l 1439 | 690.654545 109.067627 l 1440 | 710.945455 108.445277 l 1441 | 717.709091 108.549002 l 1442 | 724.472727 108.860177 l 1443 | 738 108.341552 l 1444 | 751.527273 108.445277 l 1445 | 758.290909 108.134102 l 1446 | 765.054545 108.237827 l 1447 | 771.818182 108.134102 l 1448 | 778.581818 108.341552 l 1449 | 785.345455 108.341552 l 1450 | 792.109091 108.756452 l 1451 | 805.636364 108.549002 l 1452 | 819.163636 108.963902 l 1453 | 832.690909 108.134102 l 1454 | 839.454545 108.341552 l 1455 | 846.218182 108.030377 l 1456 | 852.981818 108.134102 l 1457 | 859.745455 108.549002 l 1458 | 873.272727 108.652727 l 1459 | 886.8 107.822927 l 1460 | 900.327273 107.511751 l 1461 | 907.090909 107.408026 l 1462 | 913.854545 107.926652 l 1463 | 927.381818 107.511751 l 1464 | 940.909091 106.785676 l 1465 | 947.672727 106.681951 l 1466 | 954.436364 106.059601 l 1467 | 961.2 106.370776 l 1468 | 967.963636 105.955876 l 1469 | 981.490909 105.852151 l 1470 | 988.254545 105.955876 l 1471 | 1001.781818 105.748426 l 1472 | 1008.545455 106.163326 l 1473 | 1022.072727 106.267051 l 1474 | 1028.836364 105.852151 l 1475 | 1035.6 105.748426 l 1476 | 1055.890909 106.059601 l 1477 | 1062.654545 105.748426 l 1478 | 1069.418182 105.852151 l 1479 | 1076.181818 105.540976 l 1480 | 1082.945455 105.43725 l 1481 | 1089.709091 105.644701 l 1482 | 1096.472727 105.644701 l 1483 | 1103.236364 105.852151 l 1484 | 1123.527273 105.43725 l 1485 | 1130.290909 105.02235 l 1486 | 1137.054545 105.02235 l 1487 | 1143.818182 105.333525 l 1488 | 1164.109091 104.711175 l 1489 | 1170.872727 104.711175 l 1490 | 1184.4 105.02235 l 1491 | 1191.163636 104.918625 l 1492 | 1197.927273 104.4 l 1493 | 1204.690909 104.60745 l 1494 | 1211.454545 104.503725 l 1495 | 1224.981818 104.8149 l 1496 | 1238.509091 104.8149 l 1497 | 1245.272727 105.126075 l 1498 | 1245.272727 105.126075 l 1499 | stroke 1500 | grestore 1501 | 0.173 0.627 0.173 setrgbcolor 1502 | gsave 1503 | 1116 554.4 180 79.2 clipbox 1504 | 230.727273 391.614653 m 1505 | 237.490909 373.151595 l 1506 | 244.254545 331.350401 l 1507 | 251.018182 320.148096 l 1508 | 257.781818 319.422021 l 1509 | 264.545455 288.615682 l 1510 | 271.309091 287.059807 l 1511 | 278.072727 273.783001 l 1512 | 284.836364 268.596748 l 1513 | 291.6 263.929121 l 1514 | 298.363636 243.910187 l 1515 | 305.127273 241.939411 l 1516 | 311.890909 238.62021 l 1517 | 318.654545 222.335378 l 1518 | 325.418182 219.431076 l 1519 | 332.181818 218.808726 l 1520 | 338.945455 215.282075 l 1521 | 352.472727 215.489525 l 1522 | 359.236364 210.718172 l 1523 | 366 198.582342 l 1524 | 372.763636 181.571434 l 1525 | 379.527273 180.949084 l 1526 | 393.054545 178.770858 l 1527 | 399.818182 178.252233 l 1528 | 406.581818 177.007532 l 1529 | 413.345455 174.310681 l 1530 | 420.109091 171.19893 l 1531 | 426.872727 170.265404 l 1532 | 440.4 167.879728 l 1533 | 453.927273 166.635028 l 1534 | 460.690909 164.456802 l 1535 | 467.454545 163.627001 l 1536 | 474.218182 161.241325 l 1537 | 480.981818 159.8929 l 1538 | 487.745455 159.581725 l 1539 | 494.509091 159.063099 l 1540 | 501.272727 159.581725 l 1541 | 508.036364 158.129574 l 1542 | 514.8 157.092324 l 1543 | 521.563636 152.839597 l 1544 | 528.327273 152.735872 l 1545 | 535.090909 153.358222 l 1546 | 541.854545 150.03902 l 1547 | 555.381818 148.898045 l 1548 | 562.145455 147.549619 l 1549 | 568.909091 147.964519 l 1550 | 582.436364 146.823544 l 1551 | 589.2 145.163943 l 1552 | 602.727273 144.022968 l 1553 | 609.490909 144.334143 l 1554 | 629.781818 143.504342 l 1555 | 636.545455 143.400617 l 1556 | 643.309091 142.778267 l 1557 | 656.836364 142.259642 l 1558 | 663.6 142.467092 l 1559 | 670.363636 141.222391 l 1560 | 677.127273 141.118666 l 1561 | 683.890909 140.600041 l 1562 | 690.654545 140.807491 l 1563 | 704.181818 141.637292 l 1564 | 710.945455 141.118666 l 1565 | 724.472727 142.570817 l 1566 | 738 141.533567 l 1567 | 744.763636 141.741017 l 1568 | 751.527273 141.326116 l 1569 | 758.290909 141.533567 l 1570 | 765.054545 141.533567 l 1571 | 771.818182 140.807491 l 1572 | 778.581818 140.392591 l 1573 | 785.345455 139.770241 l 1574 | 812.4 138.94044 l 1575 | 819.163636 139.562791 l 1576 | 825.927273 139.977691 l 1577 | 832.690909 139.977691 l 1578 | 839.454545 138.94044 l 1579 | 846.218182 138.94044 l 1580 | 859.745455 138.52554 l 1581 | 866.509091 137.90319 l 1582 | 873.272727 137.592015 l 1583 | 900.327273 137.90319 l 1584 | 907.090909 137.384565 l 1585 | 913.854545 137.592015 l 1586 | 920.618182 137.28084 l 1587 | 927.381818 137.384565 l 1588 | 947.672727 136.865939 l 1589 | 954.436364 136.969665 l 1590 | 967.963636 136.658489 l 1591 | 974.727273 136.969665 l 1592 | 981.490909 136.969665 l 1593 | 988.254545 137.28084 l 1594 | 995.018182 137.384565 l 1595 | 1001.781818 136.865939 l 1596 | 1008.545455 137.384565 l 1597 | 1022.072727 136.969665 l 1598 | 1042.363636 135.724964 l 1599 | 1049.127273 135.932414 l 1600 | 1055.890909 135.206339 l 1601 | 1062.654545 134.065363 l 1602 | 1069.418182 134.376538 l 1603 | 1076.181818 134.376538 l 1604 | 1089.709091 133.961638 l 1605 | 1110 134.065363 l 1606 | 1116.763636 134.376538 l 1607 | 1123.527273 134.480263 l 1608 | 1130.290909 134.272813 l 1609 | 1137.054545 133.857913 l 1610 | 1150.581818 133.650463 l 1611 | 1164.109091 131.887137 l 1612 | 1170.872727 131.575962 l 1613 | 1177.636364 131.783412 l 1614 | 1184.4 131.783412 l 1615 | 1191.163636 131.472237 l 1616 | 1197.927273 132.094587 l 1617 | 1204.690909 131.990862 l 1618 | 1218.218182 131.368512 l 1619 | 1238.509091 131.679687 l 1620 | 1245.272727 131.575962 l 1621 | 1245.272727 131.575962 l 1622 | stroke 1623 | grestore 1624 | 0.800 setlinewidth 1625 | 0 setlinejoin 1626 | [] 0 setdash 1627 | 0.000 setgray 1628 | gsave 1629 | 180 79.2 m 1630 | 180 633.6 l 1631 | stroke 1632 | grestore 1633 | gsave 1634 | 1296 79.2 m 1635 | 1296 633.6 l 1636 | stroke 1637 | grestore 1638 | gsave 1639 | 180 79.2 m 1640 | 1296 79.2 l 1641 | stroke 1642 | grestore 1643 | gsave 1644 | 180 633.6 m 1645 | 1296 633.6 l 1646 | stroke 1647 | grestore 1648 | /DejaVuSans findfont 1649 | 18.000 scalefont 1650 | setfont 1651 | gsave 1652 | 570.210938 639.600000 translate 1653 | 0.000000 rotate 1654 | 0.000000 0.000000 m /B glyphshow 1655 | 12.348633 0.000000 m /e glyphshow 1656 | 23.422852 0.000000 m /n glyphshow 1657 | 34.831055 0.000000 m /c glyphshow 1658 | 44.727539 0.000000 m /h glyphshow 1659 | 56.135742 0.000000 m /m glyphshow 1660 | 73.669922 0.000000 m /a glyphshow 1661 | 84.700195 0.000000 m /r glyphshow 1662 | 92.100586 0.000000 m /k glyphshow 1663 | 102.524414 0.000000 m /space glyphshow 1664 | 108.246094 0.000000 m /T glyphshow 1665 | 119.209961 0.000000 m /r glyphshow 1666 | 126.610352 0.000000 m /a glyphshow 1667 | 137.640625 0.000000 m /i glyphshow 1668 | 142.641602 0.000000 m /n glyphshow 1669 | 154.049805 0.000000 m /i glyphshow 1670 | 159.050781 0.000000 m /n glyphshow 1671 | 170.458984 0.000000 m /g glyphshow 1672 | 181.884766 0.000000 m /space glyphshow 1673 | 187.606445 0.000000 m /R glyphshow 1674 | 200.113281 0.000000 m /e glyphshow 1675 | 211.187500 0.000000 m /s glyphshow 1676 | 220.565430 0.000000 m /u glyphshow 1677 | 231.973633 0.000000 m /l glyphshow 1678 | 236.974609 0.000000 m /t glyphshow 1679 | 244.032227 0.000000 m /s glyphshow 1680 | 253.410156 0.000000 m /space glyphshow 1681 | 259.131836 0.000000 m /D glyphshow 1682 | 272.992188 0.000000 m /I glyphshow 1683 | 278.300781 0.000000 m /S glyphshow 1684 | 289.726562 0.000000 m /T glyphshow 1685 | 300.721680 0.000000 m /R glyphshow 1686 | 313.228516 0.000000 m /A glyphshow 1687 | 325.541992 0.000000 m /L glyphshow 1688 | grestore 1689 | 1.000 setlinewidth 1690 | 0 setlinecap 1691 | 0.800 setgray 1692 | gsave 1693 | 1161.765625 529.521875 m 1694 | 1282 529.521875 l 1695 | 1284.666667 529.521875 1286 530.855208 1286 533.521875 c 1696 | 1286 619.6 l 1697 | 1286 622.266667 1284.666667 623.6 1282 623.6 c 1698 | 1161.765625 623.6 l 1699 | 1159.098958 623.6 1157.765625 622.266667 1157.765625 619.6 c 1700 | 1157.765625 533.521875 l 1701 | 1157.765625 530.855208 1159.098958 529.521875 1161.765625 529.521875 c 1702 | cl 1703 | gsave 1704 | 1.000 setgray 1705 | fill 1706 | grestore 1707 | stroke 1708 | grestore 1709 | 1.500 setlinewidth 1710 | 1 setlinejoin 1711 | 2 setlinecap 1712 | [] 0 setdash 1713 | 0.122 0.467 0.706 setrgbcolor 1714 | gsave 1715 | 1165.765625 607.396875 m 1716 | 1205.765625 607.396875 l 1717 | stroke 1718 | grestore 1719 | 0.000 setgray 1720 | /DejaVuSans findfont 1721 | 20.000 scalefont 1722 | setfont 1723 | gsave 1724 | 1221.765625 600.396875 translate 1725 | 0.000000 rotate 1726 | 0.000000 0.000000 m /E glyphshow 1727 | 12.636719 0.000000 m /n glyphshow 1728 | 25.312500 0.000000 m /v glyphshow 1729 | 37.148438 0.000000 m /space glyphshow 1730 | 43.505859 0.000000 m /four glyphshow 1731 | grestore 1732 | 1.000 0.498 0.055 setrgbcolor 1733 | gsave 1734 | 1165.765625 578.0375 m 1735 | 1205.765625 578.0375 l 1736 | stroke 1737 | grestore 1738 | 0.000 setgray 1739 | gsave 1740 | 1221.765625 571.037500 translate 1741 | 0.000000 rotate 1742 | 0.000000 0.000000 m /E glyphshow 1743 | 12.636719 0.000000 m /n glyphshow 1744 | 25.312500 0.000000 m /v glyphshow 1745 | 37.148438 0.000000 m /space glyphshow 1746 | 43.505859 0.000000 m /five glyphshow 1747 | grestore 1748 | 0.173 0.627 0.173 setrgbcolor 1749 | gsave 1750 | 1165.765625 548.678125 m 1751 | 1205.765625 548.678125 l 1752 | stroke 1753 | grestore 1754 | 0.000 setgray 1755 | gsave 1756 | 1221.765625 541.678125 translate 1757 | 0.000000 rotate 1758 | 0.000000 0.000000 m /E glyphshow 1759 | 12.636719 0.000000 m /n glyphshow 1760 | 25.312500 0.000000 m /v glyphshow 1761 | 37.148438 0.000000 m /space glyphshow 1762 | 43.505859 0.000000 m /seven glyphshow 1763 | grestore 1764 | 1765 | end 1766 | showpage 1767 | --------------------------------------------------------------------------------