├── .gitignore
├── LICENSE
├── README.md
├── main.py
├── models
├── agent.py
├── dijkstra.py
└── environment.py
├── network_files
├── 2x3_network.net.xml
└── sunway_network.net.xml
└── requirements.txt
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Grg0rry
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Routing Optimization in Traffic Network using Reinforcement Learning
2 |
3 | Traffic congestion remains a persistent problem in Malaysia, with Kuala Lumpur ranking as one of Asia's most congested cities. This issue leads to significant time delays, increased fuel consumption, and adverse environmental effects. Therefore, route optimisation is a promising solution that aims to guide drivers to take more efficient routes and enables them to reach their destinations quicker.
4 |
5 | This study utilises reinforcement learning algorithms, specifically the `Temporal-Difference Learning` algorithm _(Q-Learning and SARSA)_, in computing the optimised routes with `SUMO` _(Netedit and Sumolib)_ to simulate the Traffic Network Environment. The `Numpy` library is used to construct the q-table used, and this program is coded in `Python` to orchestrate and run the simulator.
6 |
7 | ## Project Scope
8 |
9 | As there are multiple factors involved in selecting the most optimal route, below are the factors that have been preset for this study:
10 | - Traffic network is not updated in real-time with sudden abnormalities like accidents, weather changes, and natural disasters (floods, landslides, etc.)
11 | - Vehicle Speed is constant at 80 km/hr
12 |
13 | ## Method of Evaluation
14 |
15 | 1. Comparision of the routes selected from the agent vs the baseline model (Dijkstra)
16 | 2. Evaluate if the models managed to converge
17 | 3. Comparision of the number of episodes taken to converge (SARSA vs Q_Learning)
18 | 4. Comparision of the time taken for the computation
19 |
20 | ## Method to Run
21 |
22 | 1. Download SUMO (https://sumo.dlr.de/docs/Downloads.php)
23 | 2. Clone this repository to your local machine.
24 | 3. Install the necessary packages
25 | ```python
26 | pip install -r requirements.txt
27 | ```
28 | 4. Update the main.py with your SUMO directory to set the environment variable
29 | ```python
30 | def sumo_configuration():
31 | os.environ["SUMO_HOME"] = "D:/app/SUMO/SUMO/" # -- change to own directory
32 | ...
33 | ```
34 | 5. Upload your netedit file and update the network_file variable
35 | ```python
36 | network_file = './network_files/2x3_network.net.xml'
37 | ```
38 | **More on Netedit:** https://sumo.dlr.de/docs/Netedit/index.html
39 |
40 | 6. Edit to evaluate based on time or distance ("t" or "d")
41 | ```python
42 | env = environment.traffic_env(network_file, congestion, traffic_light, evaluation = "d")
43 | ```
44 | 7. Run the code
45 | ```
46 | > python main.py
47 | ```
48 |
49 | ## Test Cases
50 |
51 | ### Test Case 1 - Ideal Reward Function
52 | In this test case, we examine the models' ability to converge and the number of episodes taken upon tuning and adjusting the reward function with different reward values.
53 |
54 | 1. Set the network settings in `main.py` as the following:
55 | ```python
56 | # 2x3 Traffic Network
57 | network_file = './network_files/2x3_network.net.xml'
58 | ...
59 | start_node = "A"
60 | end_node = "N"
61 |
62 | # Sunway City traffic network
63 | network_file = './network_files/sunway_network.net.xml'
64 | ...
65 | start_node = "101"
66 | end_node = "105"
67 | ```
68 |
69 | 2. Adjust the Reward Function accordingly in `agent.py`. Note to only edit within the reward parameters:
70 | ```python
71 | def step(self, action, state_list, edge_list):
72 | ...
73 |
74 | # Reward Function with Default Reward Function
75 | invalid_action_reward = -50
76 | dead_end_reward = -50
77 | loop_reward = -50
78 | completion_reward = 50
79 | bonus_reward = 50
80 | continue_reward = 0
81 |
82 | # Reward Function with Reduced loop punishment
83 | invalid_action_reward = -50
84 | dead_end_reward = -50
85 | loop_reward = -30
86 | completion_reward = 50
87 | bonus_reward = 50
88 | continue_reward = 0
89 |
90 | # Reward Function with Scaled bonus reward
91 | invalid_action_reward = -50
92 | dead_end_reward = -50
93 | loop_reward = -30
94 | completion_reward = 50
95 | bonus_reward = ((self.best_result-current_result)/self.best_result)*100 + 50
96 | continue_reward = 0
97 | ```
98 |
99 | 3. After adjustment, run the code in the terminal. [Follow Method to Run](#method-to-run)
100 | ```
101 | > python main.py
102 | ```
103 |
104 | ### Test Case 2 - Traffic Density
105 | In this test case, we put the models to a stress test on their ability to maintain performance when the traffic density level increases.
106 |
107 | 1. Adjust the network settings in `main.py` as the following:
108 | ```python
109 | # Sunway City traffic network
110 | network_file = './network_files/sunway_network.net.xml'
111 |
112 | # Sunway University to Taylors University
113 | start_node = "101"
114 | end_node = "105"
115 |
116 | # Monash University to Sunway Pyramid
117 | start_node = "102"
118 | end_node = "106"
119 |
120 | # Sunway Pyramid to Sunway Medical
121 | start_node = "106"
122 | end_node = "104"
123 | ```
124 |
125 | 2. Adjust the traffic density with the congestion level below:
126 |
127 |
128 | Low Traffic Density
129 |
130 |
131 | ##
132 | ```python
133 | congestion = [('gne7248352139_1197884603', 20), ('gne1197879649_1197879633', 14), ('gne1197874335_1197874356', 20), ('gne2302498307_2302498392', 20), ('gne2210030932_2210030940', 11), ('gne2204295706_2204295552', 19), ('gne1197884606_1197884598', 20), ('gne1197879637_8632184543', 12), ('gne678457352_678457366', 18), ('gne1197874473_1197874414', 13), ('gne678457801_678457800', 12), ('gne677583771_677583772', 17), ('gne1197874356_1197874428', 16), ('gne1197874349_1197874386', 14), ('gne1197879623_1197879660', 20), ('gne2124969573_2124969571', 18), ('gne2210030056_2210030374', 13), ('gne677618955_2210029776', 20), ('gne678458018_280459116', 15), ('gne1197874337_7222893122', 10), ('gne678457402_678457371', 13), ('gne1640450753_4921551158', 10), ('gne8632184545_8632184679', 15), ('gne677583769_677583762', 16), ('gne2204295265_2204295275', 14), ('gne677583796_677583797', 11), ('gne678457800_678457801', 13), ('gne2302498389_2302498359', 19), ('gne106_10734244602', 15), ('gne1197888442_1197888420', 13), ('gne677583771_677583800', 20), ('gne4300141715_4300141717', 17), ('gne1197888437_1197888425', 16), ('gne2124949151_2124949155', 20), ('gne4123498096_2210031191', 17), ('gne1197874495_1197874385', 12), ('gne5735834068_1197884604', 14), ('gne1197879630_1197879644', 12), ('gne5236931696_5236931695', 13), ('gne4729109994_1197913467', 18), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 14), ('gne1197874386_1197874349', 19), ('gne677583818_677583814', 16), ('gne678457324_678457269', 19), ('gne1197880917_1197884583', 16), ('gne5735834064_1000000002', 15), ('gne1197874460_1197874388', 13), ('gne678457361_678457402', 12), ('gne269953946_143675841', 18), ('gne7222893125_7222893123', 17), ('gne712814477_1197892771', 11), ('gne5236932237_5236931692', 10), ('gne678457517_678457537', 11), ('gne143676093_143675841', 12), ('gne1197874435_1197874461', 20), ('gne1197874387_1197874435', 12), ('gne2210029837_2210030002', 20), ('gne269953829_143675842', 16), ('gne1197874460_1197874485', 19), ('gne1197879636_1197879651', 11), ('gne677583802_677583801', 16), ('gne7246269656_9122427638', 16), ('gne5472416434_5472416435', 19), ('gne1197892756_1197892767', 17), ('gne5727497444_5727497443', 18), ('gne5236931684_143675326', 14), ('gne1670458830_1670458788', 18)]
134 | ```
135 |
136 |
137 |
138 |
139 | Medium Traffic Density
140 |
141 |
142 | ##
143 | ```python
144 | congestion = [('gne7248352139_1197884603', 14), ('gne1197879649_1197879633', 18), ('gne1197874335_1197874356', 12), ('gne2302498307_2302498392', 18), ('gne2210030932_2210030940', 11), ('gne2204295706_2204295552', 20), ('gne1197884606_1197884598', 14), ('gne1197879637_8632184543', 20), ('gne678457352_678457366', 18), ('gne1197874473_1197874414', 19), ('gne678457801_678457800', 13), ('gne677583771_677583772', 12), ('gne1197874356_1197874428', 15), ('gne1197874349_1197874386', 12), ('gne1197879623_1197879660', 18), ('gne2124969573_2124969571', 18), ('gne2210030056_2210030374', 10), ('gne677618955_2210029776', 19), ('gne678458018_280459116', 15), ('gne1197874337_7222893122', 17), ('gne678457402_678457371', 10), ('gne1640450753_4921551158', 11), ('gne8632184545_8632184679', 15), ('gne677583769_677583762', 14), ('gne2204295265_2204295275', 13), ('gne677583796_677583797', 10), ('gne678457800_678457801', 13), ('gne2302498389_2302498359', 19), ('gne106_10734244602', 11), ('gne1197888442_1197888420', 11), ('gne677583771_677583800', 17), ('gne4300141715_4300141717', 11), ('gne1197888437_1197888425', 18), ('gne2124949151_2124949155', 12), ('gne4123498096_2210031191', 12), ('gne1197874495_1197874385', 20), ('gne5735834068_1197884604', 17), ('gne1197879630_1197879644', 18), ('gne5236931696_5236931695', 12), ('gne4729109994_1197913467', 14), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 19), ('gne1197874386_1197874349', 16), ('gne677583818_677583814', 13), ('gne678457324_678457269', 18), ('gne1197880917_1197884583', 13), ('gne5735834064_1000000002', 14), ('gne1197874460_1197874388', 16), ('gne678457361_678457402', 20), ('gne269953946_143675841', 20), ('gne7222893125_7222893123', 15), ('gne712814477_1197892771', 17), ('gne5236932237_5236931692', 18), ('gne678457517_678457537', 17), ('gne143676093_143675841', 11), ('gne1197874435_1197874461', 13), ('gne1197874387_1197874435', 13), ('gne2210029837_2210030002', 11), ('gne269953829_143675842', 15), ('gne1197874460_1197874485', 10), ('gne1197879636_1197879651', 19), ('gne677583802_677583801', 18), ('gne7246269656_9122427638', 13), ('gne5472416434_5472416435', 19), ('gne1197892756_1197892767', 13), ('gne5727497444_5727497443', 10), ('gne5236931684_143675326', 11), ('gne1670458830_1670458788', 20), ('gne2210826388_2210826253', 10), ('gne8632184543_8632184677', 13), ('gne1197874442_1197874387', 11), ('gne678458063_280458836', 10), ('gne7246269656_1197913486', 15), ('gne1197892781_4729109994', 11), ('gne678457279_678457274', 18), ('gne1197892756_712814477', 13), ('gne677583826_2210826868', 14), ('gne5735834064_677583803', 20), ('gne2210826767_2210826253', 17), ('gne7248352139_5735834069', 13), ('gne678457370_678457375', 18), ('gne2204294872_2204295706', 12), ('gne280465223_678457994', 19), ('gne1197874402_1197874387', 19), ('gne2210029963_2210029752', 17), ('gne280460729_280462229', 13), ('gne5778793362_678457169', 17), ('gne2210826388_677583833', 16), ('gne1197874432_1197874358', 13), ('gne1694168120_269953829', 11), ('gne678457462_678457454', 11), ('gne280460729_1192884325', 20), ('gne1984009884_1984009870', 16), ('gne8759340685_8759340684', 15), ('gne677618895_2210031167', 16), ('gne5778792535_5732957384', 16), ('gne7993603231_7993603234', 17), ('gne677583814_677583819', 10), ('gne1197888419_1197888424', 20), ('gne2210133573_2210133562', 20), ('gne2210031167_2210030414', 20), ('gne678457341_678457339', 11), ('gne2210133494_7993603237', 10), ('gne678457796_678457797', 16), ('gne677583776_677583777', 15), ('gne678457796_678457364', 11), ('gne5778793223_678457273', 13), ('gne5236932237_5236932233', 13), ('gne1197884605_269953766', 13), ('gne678457166_678457164', 18), ('gne677618806_677618821', 17), ('gne1197874490_1197874397', 12), ('gne1197874388_1197874426', 16), ('gne1197879647_1197879644', 12), ('gne1197888435_1197888420', 14), ('gne7222893122_660840279', 17), ('gne1197888445_1197888437', 13), ('gne678458000_678457279', 11), ('gne1197874427_1197874467', 17), ('gne5762708414_5762708412', 18), ('gne5735834073_2000878251', 11), ('gne678458000_678457193', 10), ('gne677583853_677583887', 20), ('gne678457260_678457269', 18), ('gne2210031289_2210029755', 10), ('gne678457364_678457363', 11), ('gne10845816010_10845816005', 13), ('gne1197879652_1197879661', 12), ('gne678457339_280465223', 16), ('gne7682106896_1197879633', 17), ('gne269953946_1984009884', 17), ('gne677583780_677583790', 13), ('gne1197888440_1197888420', 16), ('gne677583803_5735834068', 10), ('gne10311852155_10311852158', 12)]
145 | ```
146 |
147 |
148 |
149 |
150 | High Traffic Density
151 |
152 |
153 | ##
154 | ```python
155 | congestion = [('gne7248352139_1197884603', 11), ('gne1197879649_1197879633', 19), ('gne1197874335_1197874356', 11), ('gne2302498307_2302498392', 20), ('gne2210030932_2210030940', 13), ('gne2204295706_2204295552', 16), ('gne1197884606_1197884598', 11), ('gne1197879637_8632184543', 19), ('gne678457352_678457366', 13), ('gne1197874473_1197874414', 19), ('gne678457801_678457800', 19), ('gne677583771_677583772', 10), ('gne1197874356_1197874428', 19), ('gne1197874349_1197874386', 11), ('gne1197879623_1197879660', 16), ('gne2124969573_2124969571', 20), ('gne2210030056_2210030374', 19), ('gne677618955_2210029776', 19), ('gne678458018_280459116', 18), ('gne1197874337_7222893122', 15), ('gne678457402_678457371', 14), ('gne1640450753_4921551158', 13), ('gne8632184545_8632184679', 20), ('gne677583769_677583762', 15), ('gne2204295265_2204295275', 13), ('gne677583796_677583797', 14), ('gne678457800_678457801', 16), ('gne2302498389_2302498359', 12), ('gne106_10734244602', 20), ('gne1197888442_1197888420', 20), ('gne677583771_677583800', 14), ('gne4300141715_4300141717', 17), ('gne1197888437_1197888425', 15), ('gne2124949151_2124949155', 11), ('gne4123498096_2210031191', 10), ('gne1197874495_1197874385', 17), ('gne5735834068_1197884604', 19), ('gne1197879630_1197879644', 19), ('gne5236931696_5236931695', 11), ('gne4729109994_1197913467', 11), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 13), ('gne1197874386_1197874349', 18), ('gne677583818_677583814', 14), ('gne678457324_678457269', 12), ('gne1197880917_1197884583', 15), ('gne5735834064_1000000002', 11), ('gne1197874460_1197874388', 13), ('gne678457361_678457402', 15), ('gne269953946_143675841', 14), ('gne7222893125_7222893123', 12), ('gne712814477_1197892771', 17), ('gne5236932237_5236931692', 18), ('gne678457517_678457537', 14), ('gne143676093_143675841', 19), ('gne1197874435_1197874461', 20), ('gne1197874387_1197874435', 18), ('gne2210029837_2210030002', 10), ('gne269953829_143675842', 20), ('gne1197874460_1197874485', 18), ('gne1197879636_1197879651', 14), ('gne677583802_677583801', 20), ('gne7246269656_9122427638', 11), ('gne5472416434_5472416435', 12), ('gne1197892756_1197892767', 14), ('gne5727497444_5727497443', 11), ('gne5236931684_143675326', 11), ('gne1670458830_1670458788', 18), ('gne2210826388_2210826253', 12), ('gne8632184543_8632184677', 14), ('gne1197874442_1197874387', 14), ('gne678458063_280458836', 19), ('gne7246269656_1197913486', 13), ('gne1197892781_4729109994', 15), ('gne678457279_678457274', 13), ('gne1197892756_712814477', 20), ('gne677583826_2210826868', 20), ('gne5735834064_677583803', 14), ('gne2210826767_2210826253', 18), ('gne7248352139_5735834069', 17), ('gne678457370_678457375', 14), ('gne2204294872_2204295706', 10), ('gne280465223_678457994', 11), ('gne1197874402_1197874387', 20), ('gne2210029963_2210029752', 16), ('gne280460729_280462229', 14), ('gne5778793362_678457169', 10), ('gne2210826388_677583833', 10), ('gne1197874432_1197874358', 15), ('gne1694168120_269953829', 12), ('gne678457462_678457454', 20), ('gne280460729_1192884325', 14), ('gne1984009884_1984009870', 12), ('gne8759340685_8759340684', 17), ('gne677618895_2210031167', 18), ('gne5778792535_5732957384', 16), ('gne7993603231_7993603234', 18), ('gne677583814_677583819', 10), ('gne1197888419_1197888424', 11), ('gne2210133573_2210133562', 11), ('gne2210031167_2210030414', 12), ('gne678457341_678457339', 18), ('gne2210133494_7993603237', 10), ('gne678457796_678457797', 15), ('gne677583776_677583777', 19), ('gne678457796_678457364', 18), ('gne5778793223_678457273', 12), ('gne5236932237_5236932233', 16), ('gne1197884605_269953766', 12), ('gne678457166_678457164', 10), ('gne677618806_677618821', 14), ('gne1197874490_1197874397', 15), ('gne1197874388_1197874426', 10), ('gne1197879647_1197879644', 15), ('gne1197888435_1197888420', 13), ('gne7222893122_660840279', 20), ('gne1197888445_1197888437', 13), ('gne678458000_678457279', 20), ('gne1197874427_1197874467', 11), ('gne5762708414_5762708412', 15), ('gne5735834073_2000878251', 18), ('gne678458000_678457193', 16), ('gne677583853_677583887', 19), ('gne678457260_678457269', 12), ('gne2210031289_2210029755', 13), ('gne678457364_678457363', 12), ('gne10845816010_10845816005', 12), ('gne1197879652_1197879661', 16), ('gne678457339_280465223', 10), ('gne7682106896_1197879633', 12), ('gne269953946_1984009884', 15), ('gne677583780_677583790', 16), ('gne1197888440_1197888420', 20), ('gne677583803_5735834068', 13), ('gne10311852155_10311852158', 14), ('gne2210133501_2210133223', 12), ('gne677618899_2210031167', 11), ('gne1197913474_4729110010', 16), ('gne677619034_677618890', 10), ('gne1197879642_1197879634', 17), ('gne7211376202_269953935', 13), ('gne2747527091_4921551158', 13), ('gne7248340682_7222893125', 17), ('gne5732957394_678457324', 15), ('gne1197888448_1197892782', 14), ('gne678457341_678457375', 13), ('gne678457269_678457324', 13), ('gne1000000001_5735834058', 10), ('gne678458014_678457222', 20), ('gne280465223_678457339', 13), ('gne677583895_677583856', 16), ('gne1186819607_1197874337', 15), ('gne1197879649_1197879659', 14), ('gne5281743138_1197892781', 11), ('gne2747527105_2747527100', 14), ('gne2210030753_2210031389', 15), ('gne1197874412_1197874403', 20), ('gne2210030796_2210030932', 18), ('gne1197874470_1197874490', 16), ('gne677583887_677583896', 20), ('gne1197874435_1197874387', 18), ('gne678457274_5778793223', 15), ('gne1197880926_1197880927', 10), ('gne1197884570_1197884584', 11), ('gne9354798730_2747527094', 14), ('gne677583872_677583870', 12), ('gne678457357_678457234', 19), ('gne1197892763_1197913456', 14), ('gne678457234_678458031', 10), ('gne678458054_280458836', 11), ('gne677583772_677583771', 19), ('gne1694168207_660840277', 16), ('gne1640452984_108', 15), ('gne280459824_5778792533', 15), ('gne5732957394_678457260', 16), ('gne677583783_677583760', 19), ('gne678457181_5778793364', 18), ('gne677583800_677583806', 11), ('gne1197879662_1197879649', 16), ('gne2210031167_677618899', 19), ('gne2210029752_2210029963', 13), ('gne1197874428_1197874347', 14), ('gne4123498120_4123498116', 10), ('gne1186819608_1197874444', 16), ('gne678457800_1640452980', 10), ('gne678457364_678457796', 18), ('gne2210030002_2210031246', 18), ('gne678457800_678457405', 20), ('gne10734244602_106', 20), ('gne1197874442_1186819607', 13), ('gne7243153330_5236932237', 15), ('gne1197874412_1197874452', 16), ('gne2210029963_1197884606', 11), ('gne1197874432_1197874444', 20), ('gne1197874356_1197874426', 15), ('gne2919814563_678457405', 19), ('gne678457171_678457273', 15), ('gne2210030384_2210030487', 20), ('gne2302498389_2302498384', 11), ('gne677583887_1670458757', 14), ('gne2000878251_677583780', 18), ('gne1197884584_269953766', 14), ('gne678457466_678457502', 20), ('gne678457505_678457502', 16), ('gne677583870_677583871', 15), ('gne2210030870_5762726905', 16), ('gne660840277_712814473', 14), ('gne143675841_1640449330', 18), ('gne1197879624_8004778229', 12), ('gne9209244285_5778223858', 13), ('gne677583777_677583779', 16), ('gne4921555285_2747527095', 20), ('gne677583772_677583773', 16), ('gne677583760_677583763', 20), ('gne677583853_1218366993', 12), ('gne1197874400_1197874353', 19), ('gne8634431542_678457357', 19), ('gne8004778229_1197879661', 14), ('gne1197879633_8004778229', 16), ('gne1197874420_1197874361', 18), ('gne5778793364_678457181', 10), ('gne4300141713_2204295265', 14), ('gne1197879647_1197879641', 14), ('gne2210031191_4123498088', 13), ('gne143675842_7211376202', 16), ('gne677583796_677583776', 19), ('gne1197884608_1197884584', 19), ('gne1197913494_1197913460', 20), ('gne677583826_677583822', 15), ('gne2210031253_2210030870', 17), ('gne1197874450_1197874369', 17), ('gne677583789_677583785', 17), ('gne678457360_678457361', 20), ('gne1197879633_7682106896', 13), ('gne1197874397_1197874490', 18), ('gne8632184679_1197879660', 17), ('gne678457343_678457370', 12), ('gne1186819601_1197879636', 20), ('gne2210030374_2210030056', 11), ('gne1197892763_1197913485', 14), ('gne660840277_712814466', 18), ('gne677583876_677583861', 20), ('gne2000878251_677583771', 20), ('gne1197892762_1197892782', 19), ('gne5735834064_5735834068', 15), ('gne101_2124969571', 11), ('gne5778246029_2968534235', 13), ('gne2210826253_2210826388', 20), ('gne677583804_5735834058', 14), ('gne2688164830_2747527094', 13), ('gne678457370_678457343', 13), ('gne677583892_1670458771', 12), ('gne1197888437_1197888419', 10), ('gne1218366993_677583853', 10), ('gne2747527090_7245815529', 13), ('gne2124969573_101', 17), ('gne1197874412_1197874432', 19), ('gne678457537_678457535', 11), ('gne678457344_678458014', 17), ('gne280460595_5762708412', 16), ('gne1197874403_1197874412', 20), ('gne1197874397_1197874473', 19), ('gne678457796_678457706', 13), ('gne677583873_677583872', 16), ('gne677618908_7248352139', 17), ('gne678457273_5778793223', 16), ('gne1197888438_1197888440', 13), ('gne1197874403_1197874400', 12), ('gne677619044_677618886', 20), ('gne1197874461_1197874386', 10), ('gne1197913508_4729110010', 11)]
156 | ```
157 |
158 |
159 | 3. Set the Reward Function in `agent.py` as the following. Note to only edit within the reward parameters:
160 | ```python
161 | def step(self, action, state_list, edge_list):
162 | ...
163 |
164 | # Reward Function with Scaled bonus reward
165 | invalid_action_reward = -50
166 | dead_end_reward = -50
167 | loop_reward = -30
168 | completion_reward = 50
169 | bonus_reward = 50
170 | continue_reward = 0
171 |
172 | ```
173 |
174 | 4. After adjustment, run the code in the terminal. [Follow Method to Run](#method-to-run)
175 | ```
176 | > python main.py
177 | ```
178 |
179 | ## Graph Plotting
180 |
181 | 1. **Route Map**: In `main.py`, the function below maps the routes produced.
182 | ```python
183 | env.visualize_plot(edge_path)
184 | ```
185 |
186 | 2. **Performance Plot**: In `main.py`, the function below creates a line plot on the performance of each episode. This is also the learning curve of the model.
187 | ```python
188 | env.plot_performance(number_of_episode, logs)
189 | ```
190 |
191 |
--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
1 | import os, sys
2 |
3 | sys.path.append('models/')
4 | import environment
5 | import agent
6 | import dijkstra
7 |
8 |
9 | def sumo_configuration():
10 | os.environ["SUMO_HOME"] = "D:/app/SUMO/SUMO/" # -- change to own directory
11 |
12 | # Check if SUMO sucessfully configured
13 | if 'SUMO_HOME' in os.environ:
14 | tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
15 | sys.path.append(tools)
16 | else:
17 | sys.exit("please declare environment variable 'SUMO_HOME'")
18 |
19 |
20 | if __name__ == '__main__':
21 | # 01 Setup SUMO
22 | sumo_configuration()
23 |
24 |
25 | # 02 Configure network variables
26 | # -------------------
27 | # 2x3 Traffic Network
28 | # [A, B, C, D, E, F, G, H, I, J, K, L, M, N]
29 | # -------------------
30 | network_file = './network_files/2x3_network.net.xml'
31 | congestion = [("gneF_I", 10), ("gneI_F", 10), ("gneB_E", 20), ("gneE_B", 20), ("gneJ_M", 30), ("gneM_J", 30)]
32 | traffic_light = [("B", 5), ("I", 5), ("G", 5)]
33 | start_node = "A"
34 | end_node = "N"
35 |
36 | # -------------------
37 | # Sunway City Traffic Network
38 | # [101: Sunway University, 102: Monash University, 103: Sunway Geo, 104: Sunway Medical, 105: Taylors University, 106: Sunway Pyramid, 107: Sunway Lagoon, 108: PJS10 Park]
39 | # -------------------
40 | network_file = './network_files/sunway_network.net.xml'
41 | congestion = [("gne2124969573_1000000001", 10), ("gne677583745_2302498575", 10), ("gne5236931684_143675326", 20), ("gne1000000001_5735834058", 20), ("gne10734244602_1640449323", 10)]
42 | traffic_light = [(["2124969573", "2124969571"], 5), (["677583896", "1670458823"], 5), (["2210133573", "2210133562", "2210133501", "2210133223"], 5), (["4123498067", "4123498068", "2210132568", "2210132847"], 5), (["1197884608", "1197880914", "1197884584", "269953766"], 5), (["5762726921", "8948947765", "10845806303", "10845816012"], 5), (["677583804", "677583801", "677583803", "677583802"], 5), (["7211376203", "7211376202", "7211376200", "7211376201"], 5), (["2747527085", "1636307448", "678457498", "5780613945", "5780613944"], 5), (["5727497437", "5727497436", "678457587", "678457535"], 5), (["463099148", "1197913517"], 5), ("712814465", 5), ("1197913486", 5), ("9209244285", 5)]
43 | start_node = "101"
44 | end_node = "105"
45 |
46 | # -- Low traffic density
47 | # congestion = [('gne7248352139_1197884603', 20), ('gne1197879649_1197879633', 14), ('gne1197874335_1197874356', 20), ('gne2302498307_2302498392', 20), ('gne2210030932_2210030940', 11), ('gne2204295706_2204295552', 19), ('gne1197884606_1197884598', 20), ('gne1197879637_8632184543', 12), ('gne678457352_678457366', 18), ('gne1197874473_1197874414', 13), ('gne678457801_678457800', 12), ('gne677583771_677583772', 17), ('gne1197874356_1197874428', 16), ('gne1197874349_1197874386', 14), ('gne1197879623_1197879660', 20), ('gne2124969573_2124969571', 18), ('gne2210030056_2210030374', 13), ('gne677618955_2210029776', 20), ('gne678458018_280459116', 15), ('gne1197874337_7222893122', 10), ('gne678457402_678457371', 13), ('gne1640450753_4921551158', 10), ('gne8632184545_8632184679', 15), ('gne677583769_677583762', 16), ('gne2204295265_2204295275', 14), ('gne677583796_677583797', 11), ('gne678457800_678457801', 13), ('gne2302498389_2302498359', 19), ('gne106_10734244602', 15), ('gne1197888442_1197888420', 13), ('gne677583771_677583800', 20), ('gne4300141715_4300141717', 17), ('gne1197888437_1197888425', 16), ('gne2124949151_2124949155', 20), ('gne4123498096_2210031191', 17), ('gne1197874495_1197874385', 12), ('gne5735834068_1197884604', 14), ('gne1197879630_1197879644', 12), ('gne5236931696_5236931695', 13), ('gne4729109994_1197913467', 18), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 14), ('gne1197874386_1197874349', 19), ('gne677583818_677583814', 16), ('gne678457324_678457269', 19), ('gne1197880917_1197884583', 16), ('gne5735834064_1000000002', 15), ('gne1197874460_1197874388', 13), ('gne678457361_678457402', 12), ('gne269953946_143675841', 18), ('gne7222893125_7222893123', 17), ('gne712814477_1197892771', 11), ('gne5236932237_5236931692', 10), ('gne678457517_678457537', 11), ('gne143676093_143675841', 12), ('gne1197874435_1197874461', 20), ('gne1197874387_1197874435', 12), ('gne2210029837_2210030002', 20), ('gne269953829_143675842', 16), ('gne1197874460_1197874485', 19), ('gne1197879636_1197879651', 11), ('gne677583802_677583801', 16), ('gne7246269656_9122427638', 16), ('gne5472416434_5472416435', 19), ('gne1197892756_1197892767', 17), ('gne5727497444_5727497443', 18), ('gne5236931684_143675326', 14), ('gne1670458830_1670458788', 18)]
48 | # -- Medium traffic density
49 | # congestion = [('gne7248352139_1197884603', 14), ('gne1197879649_1197879633', 18), ('gne1197874335_1197874356', 12), ('gne2302498307_2302498392', 18), ('gne2210030932_2210030940', 11), ('gne2204295706_2204295552', 20), ('gne1197884606_1197884598', 14), ('gne1197879637_8632184543', 20), ('gne678457352_678457366', 18), ('gne1197874473_1197874414', 19), ('gne678457801_678457800', 13), ('gne677583771_677583772', 12), ('gne1197874356_1197874428', 15), ('gne1197874349_1197874386', 12), ('gne1197879623_1197879660', 18), ('gne2124969573_2124969571', 18), ('gne2210030056_2210030374', 10), ('gne677618955_2210029776', 19), ('gne678458018_280459116', 15), ('gne1197874337_7222893122', 17), ('gne678457402_678457371', 10), ('gne1640450753_4921551158', 11), ('gne8632184545_8632184679', 15), ('gne677583769_677583762', 14), ('gne2204295265_2204295275', 13), ('gne677583796_677583797', 10), ('gne678457800_678457801', 13), ('gne2302498389_2302498359', 19), ('gne106_10734244602', 11), ('gne1197888442_1197888420', 11), ('gne677583771_677583800', 17), ('gne4300141715_4300141717', 11), ('gne1197888437_1197888425', 18), ('gne2124949151_2124949155', 12), ('gne4123498096_2210031191', 12), ('gne1197874495_1197874385', 20), ('gne5735834068_1197884604', 17), ('gne1197879630_1197879644', 18), ('gne5236931696_5236931695', 12), ('gne4729109994_1197913467', 14), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 19), ('gne1197874386_1197874349', 16), ('gne677583818_677583814', 13), ('gne678457324_678457269', 18), ('gne1197880917_1197884583', 13), ('gne5735834064_1000000002', 14), ('gne1197874460_1197874388', 16), ('gne678457361_678457402', 20), ('gne269953946_143675841', 20), ('gne7222893125_7222893123', 15), ('gne712814477_1197892771', 17), ('gne5236932237_5236931692', 18), ('gne678457517_678457537', 17), ('gne143676093_143675841', 11), ('gne1197874435_1197874461', 13), ('gne1197874387_1197874435', 13), ('gne2210029837_2210030002', 11), ('gne269953829_143675842', 15), ('gne1197874460_1197874485', 10), ('gne1197879636_1197879651', 19), ('gne677583802_677583801', 18), ('gne7246269656_9122427638', 13), ('gne5472416434_5472416435', 19), ('gne1197892756_1197892767', 13), ('gne5727497444_5727497443', 10), ('gne5236931684_143675326', 11), ('gne1670458830_1670458788', 20), ('gne2210826388_2210826253', 10), ('gne8632184543_8632184677', 13), ('gne1197874442_1197874387', 11), ('gne678458063_280458836', 10), ('gne7246269656_1197913486', 15), ('gne1197892781_4729109994', 11), ('gne678457279_678457274', 18), ('gne1197892756_712814477', 13), ('gne677583826_2210826868', 14), ('gne5735834064_677583803', 20), ('gne2210826767_2210826253', 17), ('gne7248352139_5735834069', 13), ('gne678457370_678457375', 18), ('gne2204294872_2204295706', 12), ('gne280465223_678457994', 19), ('gne1197874402_1197874387', 19), ('gne2210029963_2210029752', 17), ('gne280460729_280462229', 13), ('gne5778793362_678457169', 17), ('gne2210826388_677583833', 16), ('gne1197874432_1197874358', 13), ('gne1694168120_269953829', 11), ('gne678457462_678457454', 11), ('gne280460729_1192884325', 20), ('gne1984009884_1984009870', 16), ('gne8759340685_8759340684', 15), ('gne677618895_2210031167', 16), ('gne5778792535_5732957384', 16), ('gne7993603231_7993603234', 17), ('gne677583814_677583819', 10), ('gne1197888419_1197888424', 20), ('gne2210133573_2210133562', 20), ('gne2210031167_2210030414', 20), ('gne678457341_678457339', 11), ('gne2210133494_7993603237', 10), ('gne678457796_678457797', 16), ('gne677583776_677583777', 15), ('gne678457796_678457364', 11), ('gne5778793223_678457273', 13), ('gne5236932237_5236932233', 13), ('gne1197884605_269953766', 13), ('gne678457166_678457164', 18), ('gne677618806_677618821', 17), ('gne1197874490_1197874397', 12), ('gne1197874388_1197874426', 16), ('gne1197879647_1197879644', 12), ('gne1197888435_1197888420', 14), ('gne7222893122_660840279', 17), ('gne1197888445_1197888437', 13), ('gne678458000_678457279', 11), ('gne1197874427_1197874467', 17), ('gne5762708414_5762708412', 18), ('gne5735834073_2000878251', 11), ('gne678458000_678457193', 10), ('gne677583853_677583887', 20), ('gne678457260_678457269', 18), ('gne2210031289_2210029755', 10), ('gne678457364_678457363', 11), ('gne10845816010_10845816005', 13), ('gne1197879652_1197879661', 12), ('gne678457339_280465223', 16), ('gne7682106896_1197879633', 17), ('gne269953946_1984009884', 17), ('gne677583780_677583790', 13), ('gne1197888440_1197888420', 16), ('gne677583803_5735834068', 10), ('gne10311852155_10311852158', 12)]
50 | # -- High traffic density
51 | # congestion = [('gne7248352139_1197884603', 11), ('gne1197879649_1197879633', 19), ('gne1197874335_1197874356', 11), ('gne2302498307_2302498392', 20), ('gne2210030932_2210030940', 13), ('gne2204295706_2204295552', 16), ('gne1197884606_1197884598', 11), ('gne1197879637_8632184543', 19), ('gne678457352_678457366', 13), ('gne1197874473_1197874414', 19), ('gne678457801_678457800', 19), ('gne677583771_677583772', 10), ('gne1197874356_1197874428', 19), ('gne1197874349_1197874386', 11), ('gne1197879623_1197879660', 16), ('gne2124969573_2124969571', 20), ('gne2210030056_2210030374', 19), ('gne677618955_2210029776', 19), ('gne678458018_280459116', 18), ('gne1197874337_7222893122', 15), ('gne678457402_678457371', 14), ('gne1640450753_4921551158', 13), ('gne8632184545_8632184679', 20), ('gne677583769_677583762', 15), ('gne2204295265_2204295275', 13), ('gne677583796_677583797', 14), ('gne678457800_678457801', 16), ('gne2302498389_2302498359', 12), ('gne106_10734244602', 20), ('gne1197888442_1197888420', 20), ('gne677583771_677583800', 14), ('gne4300141715_4300141717', 17), ('gne1197888437_1197888425', 15), ('gne2124949151_2124949155', 11), ('gne4123498096_2210031191', 10), ('gne1197874495_1197874385', 17), ('gne5735834068_1197884604', 19), ('gne1197879630_1197879644', 19), ('gne5236931696_5236931695', 11), ('gne4729109994_1197913467', 11), ('gne678458038_280459824', 18), ('gne2210133562_2210133501', 13), ('gne1197874386_1197874349', 18), ('gne677583818_677583814', 14), ('gne678457324_678457269', 12), ('gne1197880917_1197884583', 15), ('gne5735834064_1000000002', 11), ('gne1197874460_1197874388', 13), ('gne678457361_678457402', 15), ('gne269953946_143675841', 14), ('gne7222893125_7222893123', 12), ('gne712814477_1197892771', 17), ('gne5236932237_5236931692', 18), ('gne678457517_678457537', 14), ('gne143676093_143675841', 19), ('gne1197874435_1197874461', 20), ('gne1197874387_1197874435', 18), ('gne2210029837_2210030002', 10), ('gne269953829_143675842', 20), ('gne1197874460_1197874485', 18), ('gne1197879636_1197879651', 14), ('gne677583802_677583801', 20), ('gne7246269656_9122427638', 11), ('gne5472416434_5472416435', 12), ('gne1197892756_1197892767', 14), ('gne5727497444_5727497443', 11), ('gne5236931684_143675326', 11), ('gne1670458830_1670458788', 18), ('gne2210826388_2210826253', 12), ('gne8632184543_8632184677', 14), ('gne1197874442_1197874387', 14), ('gne678458063_280458836', 19), ('gne7246269656_1197913486', 13), ('gne1197892781_4729109994', 15), ('gne678457279_678457274', 13), ('gne1197892756_712814477', 20), ('gne677583826_2210826868', 20), ('gne5735834064_677583803', 14), ('gne2210826767_2210826253', 18), ('gne7248352139_5735834069', 17), ('gne678457370_678457375', 14), ('gne2204294872_2204295706', 10), ('gne280465223_678457994', 11), ('gne1197874402_1197874387', 20), ('gne2210029963_2210029752', 16), ('gne280460729_280462229', 14), ('gne5778793362_678457169', 10), ('gne2210826388_677583833', 10), ('gne1197874432_1197874358', 15), ('gne1694168120_269953829', 12), ('gne678457462_678457454', 20), ('gne280460729_1192884325', 14), ('gne1984009884_1984009870', 12), ('gne8759340685_8759340684', 17), ('gne677618895_2210031167', 18), ('gne5778792535_5732957384', 16), ('gne7993603231_7993603234', 18), ('gne677583814_677583819', 10), ('gne1197888419_1197888424', 11), ('gne2210133573_2210133562', 11), ('gne2210031167_2210030414', 12), ('gne678457341_678457339', 18), ('gne2210133494_7993603237', 10), ('gne678457796_678457797', 15), ('gne677583776_677583777', 19), ('gne678457796_678457364', 18), ('gne5778793223_678457273', 12), ('gne5236932237_5236932233', 16), ('gne1197884605_269953766', 12), ('gne678457166_678457164', 10), ('gne677618806_677618821', 14), ('gne1197874490_1197874397', 15), ('gne1197874388_1197874426', 10), ('gne1197879647_1197879644', 15), ('gne1197888435_1197888420', 13), ('gne7222893122_660840279', 20), ('gne1197888445_1197888437', 13), ('gne678458000_678457279', 20), ('gne1197874427_1197874467', 11), ('gne5762708414_5762708412', 15), ('gne5735834073_2000878251', 18), ('gne678458000_678457193', 16), ('gne677583853_677583887', 19), ('gne678457260_678457269', 12), ('gne2210031289_2210029755', 13), ('gne678457364_678457363', 12), ('gne10845816010_10845816005', 12), ('gne1197879652_1197879661', 16), ('gne678457339_280465223', 10), ('gne7682106896_1197879633', 12), ('gne269953946_1984009884', 15), ('gne677583780_677583790', 16), ('gne1197888440_1197888420', 20), ('gne677583803_5735834068', 13), ('gne10311852155_10311852158', 14), ('gne2210133501_2210133223', 12), ('gne677618899_2210031167', 11), ('gne1197913474_4729110010', 16), ('gne677619034_677618890', 10), ('gne1197879642_1197879634', 17), ('gne7211376202_269953935', 13), ('gne2747527091_4921551158', 13), ('gne7248340682_7222893125', 17), ('gne5732957394_678457324', 15), ('gne1197888448_1197892782', 14), ('gne678457341_678457375', 13), ('gne678457269_678457324', 13), ('gne1000000001_5735834058', 10), ('gne678458014_678457222', 20), ('gne280465223_678457339', 13), ('gne677583895_677583856', 16), ('gne1186819607_1197874337', 15), ('gne1197879649_1197879659', 14), ('gne5281743138_1197892781', 11), ('gne2747527105_2747527100', 14), ('gne2210030753_2210031389', 15), ('gne1197874412_1197874403', 20), ('gne2210030796_2210030932', 18), ('gne1197874470_1197874490', 16), ('gne677583887_677583896', 20), ('gne1197874435_1197874387', 18), ('gne678457274_5778793223', 15), ('gne1197880926_1197880927', 10), ('gne1197884570_1197884584', 11), ('gne9354798730_2747527094', 14), ('gne677583872_677583870', 12), ('gne678457357_678457234', 19), ('gne1197892763_1197913456', 14), ('gne678457234_678458031', 10), ('gne678458054_280458836', 11), ('gne677583772_677583771', 19), ('gne1694168207_660840277', 16), ('gne1640452984_108', 15), ('gne280459824_5778792533', 15), ('gne5732957394_678457260', 16), ('gne677583783_677583760', 19), ('gne678457181_5778793364', 18), ('gne677583800_677583806', 11), ('gne1197879662_1197879649', 16), ('gne2210031167_677618899', 19), ('gne2210029752_2210029963', 13), ('gne1197874428_1197874347', 14), ('gne4123498120_4123498116', 10), ('gne1186819608_1197874444', 16), ('gne678457800_1640452980', 10), ('gne678457364_678457796', 18), ('gne2210030002_2210031246', 18), ('gne678457800_678457405', 20), ('gne10734244602_106', 20), ('gne1197874442_1186819607', 13), ('gne7243153330_5236932237', 15), ('gne1197874412_1197874452', 16), ('gne2210029963_1197884606', 11), ('gne1197874432_1197874444', 20), ('gne1197874356_1197874426', 15), ('gne2919814563_678457405', 19), ('gne678457171_678457273', 15), ('gne2210030384_2210030487', 20), ('gne2302498389_2302498384', 11), ('gne677583887_1670458757', 14), ('gne2000878251_677583780', 18), ('gne1197884584_269953766', 14), ('gne678457466_678457502', 20), ('gne678457505_678457502', 16), ('gne677583870_677583871', 15), ('gne2210030870_5762726905', 16), ('gne660840277_712814473', 14), ('gne143675841_1640449330', 18), ('gne1197879624_8004778229', 12), ('gne9209244285_5778223858', 13), ('gne677583777_677583779', 16), ('gne4921555285_2747527095', 20), ('gne677583772_677583773', 16), ('gne677583760_677583763', 20), ('gne677583853_1218366993', 12), ('gne1197874400_1197874353', 19), ('gne8634431542_678457357', 19), ('gne8004778229_1197879661', 14), ('gne1197879633_8004778229', 16), ('gne1197874420_1197874361', 18), ('gne5778793364_678457181', 10), ('gne4300141713_2204295265', 14), ('gne1197879647_1197879641', 14), ('gne2210031191_4123498088', 13), ('gne143675842_7211376202', 16), ('gne677583796_677583776', 19), ('gne1197884608_1197884584', 19), ('gne1197913494_1197913460', 20), ('gne677583826_677583822', 15), ('gne2210031253_2210030870', 17), ('gne1197874450_1197874369', 17), ('gne677583789_677583785', 17), ('gne678457360_678457361', 20), ('gne1197879633_7682106896', 13), ('gne1197874397_1197874490', 18), ('gne8632184679_1197879660', 17), ('gne678457343_678457370', 12), ('gne1186819601_1197879636', 20), ('gne2210030374_2210030056', 11), ('gne1197892763_1197913485', 14), ('gne660840277_712814466', 18), ('gne677583876_677583861', 20), ('gne2000878251_677583771', 20), ('gne1197892762_1197892782', 19), ('gne5735834064_5735834068', 15), ('gne101_2124969571', 11), ('gne5778246029_2968534235', 13), ('gne2210826253_2210826388', 20), ('gne677583804_5735834058', 14), ('gne2688164830_2747527094', 13), ('gne678457370_678457343', 13), ('gne677583892_1670458771', 12), ('gne1197888437_1197888419', 10), ('gne1218366993_677583853', 10), ('gne2747527090_7245815529', 13), ('gne2124969573_101', 17), ('gne1197874412_1197874432', 19), ('gne678457537_678457535', 11), ('gne678457344_678458014', 17), ('gne280460595_5762708412', 16), ('gne1197874403_1197874412', 20), ('gne1197874397_1197874473', 19), ('gne678457796_678457706', 13), ('gne677583873_677583872', 16), ('gne677618908_7248352139', 17), ('gne678457273_5778793223', 16), ('gne1197888438_1197888440', 13), ('gne1197874403_1197874400', 12), ('gne677619044_677618886', 20), ('gne1197874461_1197874386', 10), ('gne1197913508_4729110010', 11)]
52 |
53 |
54 | # 03 Initiate Environment
55 | env = environment.traffic_env(network_file, congestion, traffic_light, evaluation = "d")
56 | # env = environment.traffic_env(network_file = network_file, traffic_light = traffic_light, evaluation = "t", congestion_level = "low")
57 | num_episodes = 5000
58 | num_converge = 5
59 |
60 |
61 | # 04 Activate Agent
62 | # -------------------
63 | # Dijkstra Algorithm
64 | # -------------------
65 | print(f'Dijkstra Algorithm{"." * 100}')
66 | Dijkstra = dijkstra.Dijkstra(env, start_node, end_node)
67 | node_path, edge_path, *_ = Dijkstra.search()
68 | env.visualize_plot(edge_path)
69 |
70 | # -------------------
71 | # Q_Learning Algorithm
72 | # -------------------
73 | print(f'\nQ_Learning Algorithm{"." * 100}')
74 | Q_agent = agent.Q_Learning(env, start_node, end_node)
75 | node_path, edge_path, episode, logs = Q_agent.train(num_episodes, num_converge)
76 | env.plot_performance(episode, logs)
77 | env.visualize_plot(edge_path)
78 |
79 | # -------------------
80 | # SARSA Algorithm
81 | # -------------------
82 | print(f'\nSARSA Algorithm{"." * 100}')
83 | S_agent = agent.SARSA(env, start_node, end_node, exploration_rate = 0.1)
84 | node_path, edge_path, episode, logs = S_agent.train(num_episodes, num_converge)
85 | env.plot_performance(episode, logs)
86 | env.visualize_plot(edge_path)
--------------------------------------------------------------------------------
/models/agent.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import sys
3 | import datetime
4 |
5 |
6 | class rl_agent():
7 | def __init__ (self, env, start_node, end_node, learning_rate, discount_factor):
8 | # Define the learning parameters
9 | self.learning_rate = learning_rate
10 | self.discount_factor = discount_factor
11 |
12 | # Initialize environment
13 | self.env = env
14 | self.env.set_start_end(start_node, end_node)
15 |
16 |
17 | # Reset/Initialize agent
18 | def reset(self):
19 | self.q_table = np.zeros((len(self.env.state_space), len(self.env.action_space)))
20 | self.logs = {}
21 | self.best_result = 0
22 |
23 |
24 | def act(self):
25 | pass
26 |
27 |
28 | def step(self, action, state_list, edge_list):
29 | # initialize step
30 | terminate = False
31 | current_state = state_list[-1]
32 | current_edge = edge_list[-1] if edge_list else None
33 |
34 | # reward paramaters
35 | # -------------------
36 | # START OF EDIT
37 | # -------------------
38 | invalid_action_reward = -50
39 | dead_end_reward = -50
40 | # loop_reward = -50
41 | loop_reward = -30
42 | completion_reward = 50
43 | bonus_reward = 50
44 | # bonus_reward = ((self.best_result-current_result)/self.best_result)*100 + 50
45 | continue_reward = 0
46 | # -------------------
47 | # END OF EDIT
48 | # -------------------
49 |
50 | # Get list of outgoing edges
51 | outgoing_edges = self.env.decode_node_to_edges(current_state, direction = 'outgoing')
52 | reward = continue_reward
53 |
54 | # Compute reward and next state
55 | # Out-Of-Bound Action
56 | if action not in self.env.decode_edges_to_actions(outgoing_edges):
57 | reward += invalid_action_reward
58 | next_state = current_state
59 | next_edge = current_edge
60 |
61 | # Valid Action
62 | else:
63 | next_edge = self.env.decode_edges_action_to_edge(outgoing_edges, action)
64 | next_state = self.env.decode_edge_to_node(next_edge, direction = 'end')
65 |
66 | # Completed Route
67 | if next_state in self.env.end_node:
68 | reward += completion_reward
69 | terminate = True
70 |
71 | # check if the route is the shortest distance/time
72 | if self.env.evaluation in ("distance", "d"):
73 | current_result = self.env.get_edge_distance(edge_list + [next_edge])
74 | else:
75 | current_result = self.env.get_edge_time(edge_list + [next_edge])
76 |
77 | # evaluation
78 | if self.best_result == 0:
79 | self.best_result = current_result
80 | elif current_result < self.best_result:
81 | for edge in edge_list:
82 | state_index = self.env.state_space.index(self.env.decode_edge_to_node(edge, direction = 'start'))
83 | action_index = self.env.edge_label[edge]
84 | self.q_table[state_index][action_index] += bonus_reward
85 | self.best_result = current_result
86 |
87 | # Dead-end Route
88 | elif not self.env.decode_node_to_edges(next_state, direction = 'outgoing'):
89 | reward += dead_end_reward
90 | terminate = True
91 |
92 | # Backtrack and find bottleneck
93 | for edge in reversed(edge_list):
94 | if len(self.env.decode_node_to_edges(self.env.decode_edge_to_node(edge, direction = 'end'), direction = 'outgoing')) > 1:
95 | break
96 |
97 | state_index = self.env.state_space.index(self.env.decode_edge_to_node(edge, direction = 'start'))
98 | action_index = self.env.edge_label[edge]
99 | self.q_table[state_index][action_index] += dead_end_reward
100 |
101 | # Travelling
102 | elif current_edge != None:
103 | if (current_edge, next_edge) in [(edge_list[i], edge_list[i+1]) for i in range(len(edge_list)-1)]: # Check if its in a loop
104 | reward += loop_reward
105 |
106 | return next_edge, next_state, reward, terminate
107 |
108 |
109 | def learn(self, current_state, action, next_state, reward):
110 | # Update the Q-table
111 | q_predict = self.q_table[self.env.state_space.index(current_state)][action]
112 | q_target = reward + self.discount_factor * np.max(self.q_table[self.env.state_space.index(next_state)])
113 | self.q_table[self.env.state_space.index(current_state)][action] += self.learning_rate * (q_target - q_predict)
114 |
115 |
116 | def train(self, num_episodes, threshold):
117 | start_time = datetime.datetime.now() # time the training process
118 | self.reset()
119 | # print('Training Started...')
120 |
121 | for episode in range(num_episodes):
122 | # Initialize state
123 | state_journey = [self.env.start_node]
124 | edge_journey = []
125 | terminate = False
126 |
127 | # Iterate till terminate
128 | while True:
129 | last_state = state_journey[-1]
130 | if terminate or last_state in self.env.end_node:
131 | break
132 |
133 | action = self.act(last_state)
134 | next_edge, next_state, reward, terminate = self.step(action, state_journey, edge_journey)
135 |
136 | # Learn from the outcome
137 | self.learn(last_state, action, next_state, reward)
138 |
139 | # Update state
140 | if last_state != next_state:
141 | edge_journey.append(next_edge)
142 | state_journey.append(next_state)
143 |
144 | # Append to logs and print after every episode
145 | self.logs[episode] = [state_journey, edge_journey]
146 |
147 | # print(f'{episode}: {self.logs[episode]}')
148 |
149 | # Compute Convergence
150 | if episode > threshold and self.logs[episode][0][-1] == self.env.end_node:
151 |
152 | # Convergence when 5 consecutive same routes produced
153 | threshold_lst = list(self.logs.values())[-threshold:]
154 | if all(x == threshold_lst[0] for x in threshold_lst):
155 | end_time = datetime.datetime.now()
156 | time_difference = end_time - start_time
157 | processing_seconds = time_difference.total_seconds()
158 |
159 | # --- results output ---
160 | print('Training Completed...\n')
161 | print(f'Episode {episode}:\n-- States: {self.logs[episode][0]} \n-- Edges: {self.logs[episode][1]}')
162 | print(f'-- Processing Time: {processing_seconds} seconds')
163 |
164 | if self.env.evaluation in ("distance", "d"):
165 | print(f'-- Distance travelled: {round(self.env.get_edge_distance(self.logs[episode][1]), 2)} m')
166 | else:
167 | print(f'-- Travelled Time taken: {round(self.env.get_edge_time(self.logs[episode][1]), 2)} mins')
168 |
169 | return self.logs[episode][0], self.logs[episode][1], episode, self.logs
170 |
171 | # Unable to converge
172 | if episode+1 == num_episodes:
173 | print('Training Completed...')
174 | end_time = datetime.datetime.now()
175 | time_difference = end_time - start_time
176 | processing_seconds = time_difference.total_seconds()
177 | print(f'-- Processing Time: {processing_seconds} seconds')
178 | sys.exit(f'Couldnt find shortest path with {num_episodes} episodes')
179 |
180 |
181 | class SARSA(rl_agent):
182 | def __init__ (self, env, start_node, end_node, learning_rate = 0.9, discount_factor = 0.1, exploration_rate = 0.1):
183 | # Inherit from main agent class
184 | super().__init__(env, start_node, end_node, learning_rate, discount_factor)
185 |
186 | # Define additional parameter
187 | self.exploration_rate = exploration_rate
188 |
189 |
190 | def act(self, state):
191 | if np.random.random() < self.exploration_rate:
192 | # Exploration
193 | action = np.random.choice(len(self.env.action_space))
194 | else:
195 | # Exploitation
196 | state_index = self.env.state_space.index(state)
197 | action = np.argmax(self.q_table[state_index])
198 | return action
199 |
200 |
201 | class Q_Learning(rl_agent):
202 | def __init__ (self, env, start_node, end_node, learning_rate = 0.9, discount_factor = 0.1):
203 | # Inherit from main agent class
204 | super().__init__(env, start_node, end_node, learning_rate, discount_factor)
205 |
206 |
207 | def act(self, state):
208 | # Choose action with Highest Q-value
209 | state_index = self.env.state_space.index(state)
210 | action = np.argmax(self.q_table[state_index])
211 | return action
212 |
--------------------------------------------------------------------------------
/models/dijkstra.py:
--------------------------------------------------------------------------------
1 | import heapq
2 | import datetime
3 |
4 |
5 | class Dijkstra:
6 | def __init__ (self, env, start_node, end_node):
7 | # Initialize environment
8 | self.env = env
9 | self.env.set_start_end(start_node, end_node)
10 |
11 |
12 | def reset(self):
13 | # Initialize the distance and predecessor arrays.
14 | self.cost = {node: float('inf') for node in self.env.nodes}
15 | self.predecessor = {node: None for node in self.env.nodes}
16 | self.cost[self.env.start_node] = 0
17 | self.priority_queue = [(0, self.env.start_node)]
18 |
19 |
20 | def cost_funct(self, current_cost, neigh_edge):
21 | if self.env.evaluation in ("distance", "d"):
22 | cost = current_cost + self.env.get_edge_distance(neigh_edge)
23 | else:
24 | cost = current_cost + self.env.get_edge_time(neigh_edge)
25 | return cost
26 |
27 |
28 | def search(self):
29 | start_time = datetime.datetime.now()
30 | self.reset()
31 |
32 | while self.priority_queue:
33 | current_cost, current_node = heapq.heappop(self.priority_queue)
34 |
35 | # If the node is the end node, then stop searching.
36 | if current_node == self.env.end_node:
37 | break
38 |
39 | # Explore the neighbors nodes
40 | for neigh_edge in self.env.decode_node_to_edges(current_node, direction = 'outgoing'):
41 | neigh_node = self.env.decode_edge_to_node(neigh_edge, direction = 'end')
42 |
43 | # Calculate the cost of the neighbor.
44 | tentative_cost = self.cost_funct(current_cost, neigh_edge)
45 |
46 | # If the tentative distance is less than the current distance of the neighbor:
47 | if tentative_cost < self.cost[neigh_node]:
48 | # Update the distance of the neighbor.
49 | self.cost[neigh_node] = tentative_cost
50 |
51 | # Update the predecessor of the neighbor.
52 | self.predecessor[neigh_node] = current_node
53 |
54 | # Add the neighbor to the priority queue.
55 | heapq.heappush(self.priority_queue, (tentative_cost, neigh_node))
56 |
57 | # Construct the path from the start node to the goal node.
58 | node_path = []
59 | edge_path = []
60 | current_node = self.env.end_node
61 | while current_node is not None:
62 | node_path.append(current_node)
63 | current_node = self.predecessor[current_node]
64 | node_path.reverse()
65 |
66 | for index in range(len(node_path)-1):
67 | edge = set(self.env.decode_node_to_edges(node_path[index], "outgoing")) & set(self.env.decode_node_to_edges(node_path[index+1], "incoming"))
68 | edge_path.append(next(iter(edge)))
69 |
70 | # time the search process
71 | end_time = datetime.datetime.now()
72 | time_difference = end_time - start_time
73 | processing_seconds = time_difference.total_seconds()
74 |
75 | # Print out results
76 | print('Search Completed...')
77 | print(f'-- States: {node_path} \n-- Edges: {edge_path}')
78 | print(f'-- Processing Time: {processing_seconds} seconds')
79 |
80 | if self.env.evaluation in ("distance", "d"):
81 | print(f'-- Distance travelled: {round(self.env.get_edge_distance(edge_path), 2)} m')
82 | else:
83 | print(f'-- Travelled Time taken: {round(self.env.get_edge_time(edge_path), 2)} mins')
84 |
85 | return node_path, edge_path
--------------------------------------------------------------------------------
/models/environment.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import sumolib
3 | import math
4 | import networkx as nx
5 | import matplotlib.pyplot as plt
6 | import random
7 |
8 |
9 | class traffic_env:
10 | def __init__ (self, network_file, congested = [], traffic_light = [], evaluation = "", congestion_level = "", travel_speed = 80):
11 | # Parameters
12 | self.network_file = network_file
13 | self.net = sumolib.net.readNet(network_file)
14 | self.nodes = [node.getID().upper() for node in self.net.getNodes()]
15 | self.edges = [edge.getID() for edge in self.net.getEdges()]
16 | self.action_space = [0, 1, 2, 3]
17 | self.state_space = self.nodes
18 | self.edge_label = self.decode_edges_to_label()
19 |
20 | # Define congestions edges
21 | if congested:
22 | self.congested_edges = [item[0] for item in congested]
23 | self.congestion_duration = [item[1] for item in congested]
24 | for edge in self.congested_edges:
25 | if edge not in self.edges:
26 | sys.exit(f'The edge {edge} in congestion_edges provided does not exist')
27 | print(f'Congestion edges are: {self.congested_edges}')
28 | else:
29 | if congestion_level.lower() == "low":
30 | traffic_level = 0.05
31 | elif congestion_level.lower() == "medium":
32 | traffic_level = 0.10
33 | elif congestion_level.lower() == "high":
34 | traffic_level = 0.20
35 |
36 | self.congested_edges = random.sample(self.edges, round(len(self.edges) * traffic_level))
37 | self.congestion_duration = [random.randint(10, 20) for i in range(len(self.congested_edges))]
38 | print(f'traffic: {list(zip(self.congested_edges, self.congestion_duration))}')
39 | print(f'num of congestions: {len(self.congested_edges)}, num of edges: {len(self.edges)}')
40 |
41 | # Define traffic lights nodes
42 | self.tl_nodes = [item[0] for item in traffic_light]
43 | self.tl_duration = [item[1] for item in traffic_light]
44 |
45 | for nodes_lst in self.tl_nodes:
46 | if isinstance(nodes_lst, str):
47 | nodes_lst = [nodes_lst]
48 | for node in nodes_lst:
49 | if node not in self.nodes:
50 | sys.exit(f'The node {node} in traffic_lights provided does not exist')
51 | print(f'Traffic Light nodes are: {self.tl_nodes}')
52 |
53 | # Define evaluation type
54 | if evaluation.lower() not in ('distance', 'd', 'time', 't'):
55 | sys.exit('please provide only distance, d or time, t as the evaluation method')
56 | self.evaluation = evaluation.lower()
57 |
58 | # Calculate time parameters
59 | self.travel_speed = travel_speed
60 |
61 |
62 | # Set starting and ending nodes
63 | def set_start_end(self, start_node, end_node):
64 | """
65 | Validates the input starting and ending node and calls the 'set_action_space'
66 | then returns the action space with it
67 |
68 | Returns:
69 | - A dictionary of the action_space
70 | """
71 |
72 | # Check if the nodes are valid
73 | if start_node not in self.nodes:
74 | sys.exit('Error: Invalid Start Node!')
75 | elif end_node not in self.nodes:
76 | sys.exit('Error: Invalid End Node!')
77 | else:
78 | self.start_node = start_node
79 | self.end_node = end_node
80 |
81 |
82 | # Match node to edges
83 | def decode_node_to_edges(self, node, direction = None):
84 | """
85 | Given a node and direction, returns a list of edges associated with that node.
86 |
87 | Args:
88 | - node (str): The ID of the node to match to edges
89 | - direction (str or None): The direction of the edges to return.
90 | If None, all edges are returned. Otherwise, must be one of the following strings:
91 | - 'incoming': return only edges where the node is the end
92 | - 'outgoing': return only edges where the node is the start
93 |
94 | Returns:
95 | - A list of edges (str) associated with the given node, in the specified direction if specified.
96 | """
97 |
98 | # Check if the direction is valid
99 | if direction not in ('incoming', 'outgoing', None):
100 | sys.exit(f'Invalid direction: {direction}')
101 |
102 | edges = []
103 | net_node = self.net.getNode(node)
104 |
105 | # Match node and direction to return edges
106 | if direction == 'incoming':
107 | for edge in net_node.getIncoming():
108 | if edge.getToNode().getID() == node:
109 | edges.append(edge.getID())
110 |
111 | elif direction == 'outgoing':
112 | for edge in net_node.getOutgoing():
113 | if edge.getFromNode().getID() == node:
114 | edges.append(edge.getID())
115 |
116 | else:
117 | for edge in net_node.getIncoming() + net_node.getOutgoing():
118 | if edge.getToNode().getID() == node or edge.getFromNode().getID() == node:
119 | edges.append(edge.getID())
120 |
121 | return edges
122 |
123 |
124 | # Label edges based of junction from (Right -> Up -> Left -> Down)
125 | def decode_edges_to_label(self):
126 | """
127 | Iterates through the whole state space and returns a dictionary of each state and the direction it is headed.
128 |
129 | Returns:
130 | - A dictionary of states (str) matched with its direction.
131 | """
132 |
133 | edge_labelled = {edge: None for edge in self.edges}
134 |
135 | def get_edge_label(node, outgoing_edges):
136 | # store edge angle
137 | edge_angle = []
138 |
139 | # get the nodes outgoing
140 | start_x, start_y = self.net.getNode(node).getCoord()
141 |
142 | # get outgoing edges
143 | for edge in outgoing_edges:
144 | end_node = self.decode_edge_to_node(edge)
145 | end_x, end_y = self.net.getNode(end_node).getCoord()
146 |
147 | x_diff = end_x - start_x
148 | y_diff = end_y - start_y
149 |
150 | # get their angle
151 | angle = math.degrees(math.atan2(y_diff, x_diff))
152 | edge_angle.append((edge, angle))
153 |
154 | # sort from 0 to 180 to -180 to 0 (Right -> Up -> Left -> Down -> Right)
155 | edge_angle = sorted(edge_angle, key=lambda x: ((x[1] >= 0) * -180, x[1]))
156 |
157 | # label edges
158 | for i in range(len(edge_angle)):
159 | edge_labelled[edge_angle[i][0]] = i
160 |
161 | for node in self.nodes:
162 | outgoing_edges = self.decode_node_to_edges(node, 'outgoing')
163 | if outgoing_edges:
164 | get_edge_label(node, outgoing_edges)
165 | return edge_labelled
166 |
167 |
168 | # Find the actions from a given edges
169 | def decode_edges_to_actions(self, edges):
170 | """
171 | Translate a list of given edges to their actions
172 |
173 | Args:
174 | - edges (list): The list of edges to be translated
175 |
176 | Returns:
177 | - A list of actions (int)
178 | """
179 |
180 | # Check if edges is in the edges list
181 | for edge in edges:
182 | if edge not in self.edges:
183 | sys.exit(f'Error: Edge {edge} not in Edges Space!')
184 |
185 | # Get the label of each edge
186 | edge_label = self.edge_label
187 |
188 | # Returns a list of actions
189 | actions_lst = []
190 | for action in self.action_space:
191 | if action in [edge_label[edge] for edge in edges]:
192 | actions_lst.append(action)
193 | return actions_lst
194 |
195 |
196 | # Find the edge from a given edge and action
197 | def decode_edges_action_to_edge(self, edges, action):
198 | """
199 | Compute the new edge from a given edges and action taken.
200 |
201 | Args:
202 | - edges (list): The list of edges to be translated
203 | - action (int): The action taken
204 |
205 | Returns:
206 | - The new edge (str) or None if no match is found.
207 | """
208 |
209 | # Check if edges is in the edges list
210 | for edge in edges:
211 | if edge not in self.edges:
212 | sys.exit(f'Error: Edge {edge} not in Edges Space!')
213 |
214 | # Get the direction of each edge
215 | edge_label = self.edge_label
216 |
217 | for edge in edges:
218 | if edge_label[edge] == action:
219 | return edge
220 | return None
221 |
222 |
223 | # Find the end node from a given edge
224 | def decode_edge_to_node(self, search_edge, direction = 'end'):
225 | """
226 | Given an edge return the start or ending node of that edge
227 |
228 | Args:
229 | - search_edge (str): The edge to be computed
230 | - direction (str): The direction of the node to return
231 | - 'start': node is the start of the edge
232 | - 'end': node is the end of the edge (default)
233 |
234 | Returns:
235 | - The node (str)
236 | """
237 |
238 | # Check if edges is in the edges list
239 | if search_edge not in self.edges:
240 | sys.exit('Error: Edge not in Edges Space!')
241 |
242 | edge = self.net.getEdge(search_edge)
243 |
244 | if direction == 'start':
245 | node = edge.getFromNode().getID()
246 |
247 | elif direction == 'end':
248 | node = edge.getToNode().getID()
249 |
250 | return node
251 |
252 |
253 | # Find the total distance travelled from a given pathway of nodes
254 | def get_edge_distance(self, travel_edges):
255 | """
256 | Calculates the cost function (distance travelled) through the select pathway/route
257 |
258 | Args:
259 | - travel_edges: The list of edges of the selected route.
260 |
261 | Return:
262 | - total_distance (float): The total distance travelled
263 | """
264 |
265 | total_distance = 0
266 | if isinstance(travel_edges, str):
267 | travel_edges = [travel_edges]
268 |
269 | # Get the length of the edge
270 | for edge in travel_edges:
271 | # Check if edges is in the edges list
272 | if edge not in self.edges:
273 | sys.exit(f'Error: Edge {edge} not in Edges Space!')
274 | total_distance += self.net.getEdge(edge).getLength()
275 |
276 | return total_distance
277 |
278 |
279 | # Find the total time taken from a given pathway of nodes and edges
280 | def get_edge_time(self, travel_edges):
281 | """
282 | Calculates the cost function (time taken) through the select pathway/route
283 |
284 | Args:
285 | - travel_edges: The list of edges of the selected route.
286 | - speed: The speed travel (constant) km/h
287 | - congestion_duration: The time taken for stuck in congestion (in minutes)
288 | - traffic_light_duration: The time taken for stuck in traffic light (in minutes)
289 |
290 | Return:
291 | - total_time (float): The total time taken to travel (in minutes)
292 | """
293 |
294 | total_time = ((self.get_edge_distance(travel_edges)/1000) / self.travel_speed) * 60 # in minutes
295 |
296 | if isinstance(travel_edges, str):
297 | travel_edges = [travel_edges]
298 |
299 | # time punishment
300 | for i in range(len(travel_edges)):
301 | # congested area
302 | if travel_edges[i] in self.congested_edges:
303 | total_time += self.congestion_duration[self.congested_edges.index(travel_edges[i])]
304 |
305 | # traffic light
306 | prev_node = self.decode_edge_to_node(travel_edges[i-1], direction = 'end') if i > 0 else ""
307 | node = self.decode_edge_to_node(travel_edges[i], direction = 'end')
308 |
309 | for index, search_nodes in enumerate(self.tl_nodes):
310 | if node in search_nodes and prev_node not in search_nodes:
311 | total_time += self.tl_duration[index]
312 |
313 | return total_time
314 |
315 |
316 | # ------ Graph Visualization ------
317 | def visualize_plot(self, travel_edges):
318 | """
319 | Plotting of network with selected route
320 |
321 | Args:
322 | - travel_edges (list): The list of edges of the selected route.
323 | - network_files_directory (str): The directory of the network files.
324 | - root_file (str): The initial name of the root file to be converted from network_file.
325 |
326 | Return:
327 | - Plot of network
328 | """
329 |
330 | nodes_dict = {}
331 | for node in self.nodes:
332 | x_coord, y_coord = self.net.getNode(node).getCoord()
333 | nodes_dict[node] = (x_coord, y_coord)
334 |
335 | edges_dict = {}
336 | for edge in self.edges:
337 | from_id = self.net.getEdge(edge).getFromNode().getID()
338 | to_id = self.net.getEdge(edge).getToNode().getID()
339 | edges_dict[edge] = (from_id, to_id)
340 |
341 | # Draws the network layout
342 | G = nx.Graph()
343 | for edge in edges_dict:
344 | G.add_edge(edges_dict[edge][0], edges_dict[edge][1])
345 | pos = {node: nodes_dict[node] for node in nodes_dict}
346 | nx.draw(G, pos, with_labels=False, node_color='black', node_size=200, edge_color='gray')
347 |
348 | # Draws the selected route
349 | route_G = nx.Graph()
350 | for edge in travel_edges:
351 | route_G.add_edge(edges_dict[edge][0], edges_dict[edge][1])
352 | nx.draw(route_G, pos, with_labels=False, node_color='green', node_size=300, edge_color='green', arrowsize = 15, arrows=True, arrowstyle='fancy')
353 |
354 | if self.evaluation in ("time", "t"):
355 | # Highlight traffic light nodes
356 | tl_lst = []
357 | for item in self.tl_nodes:
358 | if isinstance(item, list):
359 | tl_lst.extend(item)
360 | else:
361 | tl_lst.append(item)
362 | nx.draw_networkx_nodes(G, pos, nodelist=tl_lst, node_color='red', node_size=300)
363 |
364 | # Highlight congestion edges
365 | congested_lst = [edges_dict[edge] for edge in self.congested_edges]
366 | nx.draw_networkx_edges(G, pos, edgelist=congested_lst, edge_color='red', width=2)
367 |
368 | plt.show()
369 |
370 |
371 | def plot_performance(self, num_episodes, logs):
372 | """
373 | Plotting of models' performance
374 |
375 | Args:
376 | - num_episodes (int): number of episodes it took for the model to converge.
377 | - logs (dict): the logs of the edges and states it took to converge.
378 |
379 | Return:
380 | - Plot of the evaluation (time/distance) at each episode
381 | """
382 |
383 | if self.evaluation in ("distance", "d"):
384 | plt.ylabel("Distance")
385 | evaluation = [self.get_edge_distance(logs[episode][1]) for episode in range(num_episodes)]
386 | else:
387 | plt.ylabel("Time")
388 | evaluation = [self.get_edge_time(logs[episode][1]) for episode in range(num_episodes)]
389 |
390 | plt.plot(range(num_episodes), evaluation)
391 | plt.xlabel("Episode")
392 | plt.title("Performance of Agent")
393 | plt.show()
--------------------------------------------------------------------------------
/network_files/2x3_network.net.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
408 |
409 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
458 |
459 |
460 |
461 |
462 |
463 |
464 |
465 |
466 |
467 |
468 |
469 |
470 |
471 |
472 |
473 |
474 |
475 |
476 |
477 |
478 |
479 |
480 |
481 |
482 |
483 |
484 |
485 |
486 |
487 |
488 |
489 |
490 |
491 |
492 |
493 |
494 |
495 |
496 |
497 |
498 |
499 |
500 |
501 |
502 |
503 |
504 |
505 |
506 |
507 |
508 |
509 |
510 |
511 |
512 |
513 |
514 |
515 |
516 |
517 |
518 |
519 |
520 |
521 |
522 |
523 |
524 |
525 |
526 |
527 |
528 |
529 |
530 |
531 |
532 |
533 |
534 |
535 |
536 |
537 |
538 |
539 |
540 |
541 |
542 |
543 |
544 |
545 |
546 |
547 |
548 |
549 |
550 |
551 |
552 |
553 |
554 |
555 |
556 |
557 |
558 |
559 |
560 |
561 |
562 |
563 |
564 |
565 |
566 |
567 |
568 |
569 |
570 |
571 |
572 |
573 |
574 |
575 |
576 |
577 |
578 |
579 |
580 |
581 |
582 |
583 |
584 |
585 |
586 |
587 |
588 |
589 |
590 |
591 |
592 |
593 |
594 |
595 |
596 |
597 |
598 |
599 |
600 |
601 |
602 |
603 |
604 |
605 |
606 |
607 |
608 |
609 |
610 |
611 |
612 |
613 |
614 |
615 |
616 |
617 |
618 |
619 |
620 |
621 |
622 |
623 |
624 |
625 |
626 |
627 |
628 |
629 |
630 |
631 |
632 |
633 |
634 |
635 |
636 |
637 |
638 |
639 |
640 |
641 |
642 |
643 |
644 |
645 |
646 |
647 |
648 |
649 |
650 |
651 |
652 |
653 |
654 |
655 |
656 |
657 |
658 |
659 |
660 |
661 |
662 |
663 |
664 |
665 |
666 |
667 |
668 |
669 |
670 |
671 |
672 |
673 |
674 |
675 |
676 |
677 |
678 |
679 |
680 |
681 |
682 |
683 |
684 |
685 |
686 |
687 |
688 |
689 |
690 |
691 |
692 |
693 |
694 |
695 |
696 |
697 |
698 |
699 |
700 |
701 |
702 |
703 |
704 |
705 |
706 |
707 |
708 |
709 |
710 |
711 |
712 |
713 |
714 |
715 |
716 |
717 |
718 |
719 |
720 |
721 |
722 |
723 |
724 |
725 |
726 |
727 |
728 |
729 |
730 |
731 |
732 |
733 |
734 |
735 |
736 |
737 |
738 |
739 |
740 |
741 |
742 |
743 |
744 |
745 |
746 |
747 |
748 |
749 |
750 |
751 |
752 |
753 |
754 |
755 |
756 |
757 |
758 |
759 |
760 |
761 |
762 |
763 |
764 |
765 |
766 |
767 |
768 |
769 |
770 |
771 |
772 |
773 |
774 |
775 |
776 |
777 |
778 |
779 |
780 |
781 |
782 |
783 |
784 |
785 |
786 |
787 |
788 |
789 |
790 |
791 |
792 |
793 |
794 |
795 |
796 |
797 |
798 |
799 |
800 |
801 |
802 |
803 |
804 |
805 |
806 |
807 |
808 |
809 |
810 |
811 |
812 |
813 |
814 |
815 |
816 |
817 |
818 |
819 |
820 |
821 |
822 |
823 |
824 |
825 |
826 |
827 |
828 |
829 |
830 |
831 |
832 |
833 |
834 |
835 |
836 |
837 |
838 |
839 |
840 |
841 |
842 |
843 |
844 |
845 |
846 |
847 |
848 |
849 |
850 |
851 |
852 |
853 |
854 |
855 |
856 |
857 |
858 |
859 |
860 |
861 |
862 |
863 |
864 |
865 |
866 |
867 |
868 |
869 |
870 |
871 |
872 |
873 |
874 |
875 |
876 |
877 |
878 |
879 |
880 |
881 |
882 |
883 |
884 |
885 |
886 |
887 |
888 |
889 |
890 |
891 |
892 |
893 |
894 |
895 |
896 |
897 |
898 |
899 |
900 |
901 |
902 |
903 |
904 |
905 |
906 |
907 |
908 |
909 |
910 |
911 |
912 |
913 |
914 |
915 |
916 |
917 |
918 |
919 |
920 |
921 |
922 |
923 |
924 |
925 |
926 |
927 |
928 |
929 |
930 |
931 |
932 |
933 |
934 |
935 |
936 |
937 |
938 |
939 |
940 |
941 |
942 |
943 |
944 |
945 |
946 |
947 |
948 |
949 |
950 |
951 |
952 |
953 |
954 |
955 |
956 |
957 |
958 |
959 |
960 |
961 |
962 |
963 |
964 |
965 |
966 |
967 |
968 |
969 |
970 |
971 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | networkx
2 | matplotlib
3 | sumolib
4 | numpy
--------------------------------------------------------------------------------