├── .gitattributes ├── .gitignore ├── Demo.py ├── DetailedReportAndArticle.pdf ├── LICENSE ├── Model Folders ├── Cooperative_Trained │ ├── uav0.h5 │ ├── uav1.h5 │ └── uav2.h5 └── Independent_Trained │ ├── uav0.h5 │ ├── uav1.h5 │ └── uav2.h5 ├── README.md ├── SimuClasses.py ├── UAVTraining.ipynb ├── User Data └── test_env.pkl ├── __init__.py └── requirements.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | 147 | # PyCharm 148 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 149 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 150 | # and can be added to the global gitignore or merged into this file. For a more nuclear 151 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 152 | #.idea/ 153 | *.bak 154 | -------------------------------------------------------------------------------- /Demo.py: -------------------------------------------------------------------------------- 1 | from typing import Generator 2 | import matplotlib 3 | 4 | import matplotlib.pyplot as plt 5 | from SimuClasses import * 6 | from tkinter import * 7 | import time 8 | import os 9 | from tkinter import messagebox 10 | from PIL import ImageTk, Image 11 | import sys 12 | import tkinter.font 13 | import requests 14 | from tkinter.filedialog import askopenfilename, askdirectory 15 | from pathlib import Path 16 | from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk 17 | 18 | 19 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 20 | 21 | FONT = "Verdana" 22 | 23 | class DemoGUI: 24 | def split(self): 25 | 26 | return " " 27 | 28 | def startUI(self): 29 | 30 | def set_text(entry, text): 31 | entry.delete(0, END) 32 | entry.insert(0, text) 33 | return None 34 | 35 | def set_label_text(label, text): 36 | label.config(text=text) 37 | return None 38 | 39 | def refresh(self): 40 | self.destroy() 41 | self.__init__() 42 | 43 | def Giris(): 44 | def message(text): 45 | set_label_text(result,text) 46 | # 2 Entry, 3 Button, 1 checkbutton,1 Label olacak 47 | # simulasyon butonuna basıldığında 2 pencere daha çıkacak biri map'ı gösterecek diğeri de step monitor 48 | mainGUI = Tk() 49 | boyut = 800 50 | mainGUI.geometry("%dx%d" % (boyut, boyut)) 51 | mainGUI.title("UAV Simulation Demo") 52 | mainGUI.resizable(False, False) 53 | 54 | 55 | result = None 56 | ModelFolder = None 57 | UserData = None 58 | 59 | UserDataDialog = None 60 | ModelFolderDialog = None 61 | 62 | generateRandomEnv = None 63 | generateOptionCheckBox = None 64 | StartSimuButton = None 65 | 66 | def load_file(): 67 | fname = fname = askopenfilename(defaultextension='.pkl') 68 | set_text(UserData, fname) 69 | 70 | def load_dir(): 71 | fname = askdirectory() 72 | set_text(ModelFolder, fname) 73 | 74 | def simulation_start(): 75 | result_dict = {} 76 | gen = test_env(ModelFolder.get(),UserData.get(), 77 | random_env = generateRandomEnv.get(),step_count=int(StepCountEntry.get()), 78 | plot_trajectories=showTrajectoriesVar.get()) 79 | for i in gen: 80 | plt.pause(0.0000000000001) 81 | result_dict = i 82 | set_label_text(result,"Average Sum Rate Score: {:.2f}\nFinal Sum Rate Score: {:.2f}" 83 | .format(*(result_dict["scores"]))) 84 | 85 | def disable_dialog(): 86 | def switch(obj): 87 | st = obj["state"] 88 | obj["state"] = "normal" if st=="disabled" else "disabled" 89 | 90 | switch(UserData) 91 | switch(UserDataDialog) 92 | 93 | def render(): 94 | # Sol üst 0,0 yani Aşağı gidildikçe y artıyor 95 | def LOC(X,Y,obj): 96 | return {"relx":X,"rely":Y,"obj":obj} 97 | 98 | COL_START_X = 0.2 99 | COL_SPACE_X = 0.4 100 | COL2_START_X = COL_START_X + COL_SPACE_X 101 | 102 | resultLoc = LOC(0.35,0.1,result) 103 | 104 | ModelFolderLoc,ModelFolderDialogLoc = LOC(COL_START_X, 0.4,ModelFolder),LOC(COL2_START_X, 0.39,ModelFolderDialog) 105 | 106 | generateOptionCheckBoxLoc = LOC(COL_START_X,0.5,generateOptionCheckBox) 107 | 108 | UserDataLoc,UserDataDialogLoc = LOC(COL_START_X, 0.6,UserData),LOC(COL2_START_X, 0.59,UserDataDialog) 109 | 110 | stepCountLabelLoc, StepCountEntryLoc = LOC(COL_START_X, 0.7,stepCountLabel),LOC(COL_START_X+0.2, 0.7,StepCountEntry) 111 | 112 | showTrajectoriesCheckBoxLoc = LOC(COL_START_X, 0.8,showTrajectoriesCheckBox) 113 | 114 | StartSimuButtonLoc = LOC(0.35,0.9,StartSimuButton) 115 | 116 | 117 | pack = [resultLoc, 118 | generateOptionCheckBoxLoc, 119 | UserDataLoc,UserDataDialogLoc, 120 | ModelFolderLoc,ModelFolderDialogLoc, 121 | stepCountLabelLoc,StepCountEntryLoc, 122 | showTrajectoriesCheckBoxLoc, 123 | StartSimuButtonLoc] 124 | 125 | for location in pack: 126 | location.pop("obj").place(**location) 127 | 128 | 129 | 130 | result = Label(mainGUI, bg="white", fg="Black", 131 | font=FONT+ " 13", text="") 132 | ModelFolder = Entry( 133 | mainGUI, relief=FLAT, bg="white", fg="black", 134 | font=FONT+" 15 italic", text="") 135 | UserData = Entry( 136 | mainGUI, relief=FLAT, bg="white", fg="black", 137 | font=FONT+" 15 italic", text="") 138 | 139 | UserDataDialog = Button(mainGUI, relief=FLAT, bg="white", fg="black", 140 | font=FONT+" 15 italic", text="Choose User Data", 141 | command=load_file) 142 | ModelFolderDialog = Button(mainGUI, relief=FLAT, bg="white", fg="black", 143 | font=FONT+" 15 italic", text="Choose Model Folder", 144 | command=load_dir) 145 | 146 | generateRandomEnv = tkinter.BooleanVar() 147 | generateOptionCheckBox = tkinter.Checkbutton(mainGUI, text='Random Environment',variable=generateRandomEnv, 148 | font=FONT+ " 15 italic", onvalue=1, offvalue=0, command=disable_dialog) 149 | 150 | stepCountLabel = Label(mainGUI, fg="Black", 151 | font=FONT+ " 13", text="Step Count:") 152 | 153 | StepCountEntry = Entry( 154 | mainGUI, relief=FLAT, bg="white", fg="black", 155 | font=FONT+" 15 italic", text="64") 156 | 157 | showTrajectoriesVar = tkinter.BooleanVar() 158 | showTrajectoriesCheckBox = tkinter.Checkbutton(mainGUI, text='Show Past Trajectories',variable=showTrajectoriesVar, 159 | font=FONT+ " 15 italic", onvalue=1, offvalue=0) 160 | 161 | StartSimuButton = Button(mainGUI, relief=FLAT, bg="white", fg="black", font=FONT+ " 15 italic", text="Start Simulation", 162 | command=simulation_start) 163 | 164 | render() 165 | 166 | Giris() 167 | mainloop() 168 | 169 | 170 | gui = DemoGUI() 171 | gui.startUI() 172 | -------------------------------------------------------------------------------- /DetailedReportAndArticle.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/DetailedReportAndArticle.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Uğur 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Model Folders/Cooperative_Trained/uav0.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Cooperative_Trained/uav0.h5 -------------------------------------------------------------------------------- /Model Folders/Cooperative_Trained/uav1.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Cooperative_Trained/uav1.h5 -------------------------------------------------------------------------------- /Model Folders/Cooperative_Trained/uav2.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Cooperative_Trained/uav2.h5 -------------------------------------------------------------------------------- /Model Folders/Independent_Trained/uav0.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Independent_Trained/uav0.h5 -------------------------------------------------------------------------------- /Model Folders/Independent_Trained/uav1.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Independent_Trained/uav1.h5 -------------------------------------------------------------------------------- /Model Folders/Independent_Trained/uav2.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/Model Folders/Independent_Trained/uav2.h5 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution 2 | 3 | 4 | Video of Execution Process after training. 5 | 6 | (Green Dots: UAVs, Red Dots: Mobile Devices) 7 | 8 | [![IMAGE ALT TEXT](http://img.youtube.com/vi/5OIFD02gico/0.jpg)](http://www.youtube.com/watch?v=5OIFD02gico "Simulation of Multi-Agent UAVs as Aerial Base Stations") 9 | 10 | --- 11 | # System Design 12 | Read report for detailed information about System Model, System Design, Multi-Agent Cooperative Deep Reinforcement Learning Algorithm etc. 13 | 14 | [Detailed Report](DetailedReportAndArticle.pdf) 15 | 16 | --- 17 | # Running Demo: 18 | 19 | 20 | pip install -r requirements.txt 21 | python Demo.py 22 | -------------------------------------------------------------------------------- /SimuClasses.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | import math 4 | import scipy.constants as constant 5 | from scipy.stats import truncnorm 6 | import pickle 7 | import json 8 | import os 9 | import keras 10 | from keras import Model 11 | from keras.layers import AveragePooling2D, Flatten, Conv2D, Input, concatenate, Dense 12 | from keras.losses import Huber 13 | from keras.models import load_model, Sequential 14 | from keras.optimizers import Adam 15 | from random import randint 16 | import random as rand 17 | import tensorflow as tf 18 | from time import sleep 19 | from collections import deque 20 | from statistics import mean 21 | import numpy as np 22 | import matplotlib.pyplot as plt 23 | from IPython.display import clear_output 24 | from mpl_toolkits.mplot3d import Axes3D 25 | import seaborn as sns 26 | from matplotlib.axes import Axes 27 | 28 | sns.set_style("darkgrid") 29 | 30 | # Carrier Frequency 31 | FC = 2e9 32 | 33 | # Speed of Light 34 | C = constant.c 35 | 36 | # Power Spectral Density 37 | N0 = 1e-20 38 | 39 | # Bandwidth 40 | B = 1e6 41 | 42 | # Noise 43 | STD = B * N0 44 | 45 | # Environmental Variable 1 46 | B1 = 0.36 47 | 48 | # Environmental Variable 2 49 | B2 = 0.21 50 | 51 | # Path Loss Exponent 52 | ALPHA = 2 53 | 54 | # Additional path loss for LineOfSight 55 | U_LOS = 10**(3/10) 56 | 57 | # Additional path loss for NonLineOfSight 58 | U_NLOS = 10**(23/10) 59 | 60 | K0 = (4 * math.pi * FC / C) ** 2 61 | 62 | # Environmental Variable 3 63 | ZETA = 0 64 | 65 | # Transmission power of an UAV 66 | P = 0.08 67 | 68 | # Discount Rate 69 | GAMMA = 0.95 70 | 71 | # Learning Rate 72 | LR = 0.001 73 | 74 | # Initial Epsilon Greedy Value 75 | EPSILON = 1 76 | 77 | # Epsilon Greedy Decay Rate 78 | EPSILON_DECAY = 0.95 79 | 80 | # Minimum Epsilon Greedy 81 | EPSILON_MIN = 0.1 82 | 83 | # Minimum altitude to fly 84 | MIN_ALTITUDE = 200 85 | 86 | # Initial Altitude 87 | ALTITUDE = 400 88 | 89 | # Maximum altitude to fly 90 | MAX_ALTITUDE = 800 91 | 92 | # Penalty value 93 | PENALTY = -100 94 | 95 | # UAV speed rate according to humans 96 | UAV_MOVEMENT_SCALE= 20 97 | 98 | STEP_COUNT_TRAINED_ON = 64 99 | UNIT =20 100 | UE_COUNT = 200 101 | 102 | env_dim = (100,100) 103 | 104 | alt = ALTITUDE 105 | mid_height, mid_width = env_dim[1]//2, env_dim[0]//2 106 | 107 | 108 | uav_paths = [ 109 | [mid_width, mid_height,alt ], \ 110 | [mid_width, mid_height,alt-1], \ 111 | [mid_width, mid_height,alt-2], 112 | ] 113 | 114 | # Helper Function for normal distribution 115 | def get_truncated_normal(mean=0, low=0, upp=10): 116 | return truncnorm( 117 | (low - mean), (upp - mean), loc=mean, scale=1).rvs() 118 | 119 | # Normal Distribution 120 | def gaussian(end,begin=0): 121 | normal_random = round(get_truncated_normal(mean=int((begin+end)/2),low=begin,upp=end)) 122 | return max(0,min(normal_random,end)) 123 | 124 | # Function to plot step graph on execution 125 | def plot_step_graph(sum_rates_per_step,ax,lim=100): 126 | #ax.clear() 127 | ax.set_xlabel("Step Number (Time)") 128 | ax.set_ylabel("Sum Rate(bits/s/Hz)") 129 | ax.set_ylim(6,19) 130 | ax.set_xlim(1,lim+1) 131 | t = len(sum_rates_per_step) 132 | if(t>1): 133 | ax.plot([t-1,t],sum_rates_per_step[t-2:],c="blue") 134 | #ax.plot(range(len(sum_rates_per_step)),sum_rates_per_step) 135 | ax.set_title("Sum rates per step") 136 | 137 | # Execution function to execute trained models 138 | def test_env(file_name,user_data_path,random_env=False,fig=None,step_count=64,plot_trajectories=True): 139 | 140 | agent,env = semiConvAgentForTest, UavSimuEnvForTest 141 | 142 | new_env = env(uav_paths,range(UE_COUNT),(100,100),uav_class=agent) 143 | new_env.load(file_name,just_for_test=True) 144 | 145 | sum_rates_per_step = [] 146 | if(fig==None): 147 | fig = plt.figure(figsize=(10,10),dpi=100) 148 | ax = fig.add_subplot(projection='3d'); 149 | fig2,ax1 = plt.subplots(figsize=(5,5),dpi=100) 150 | ax1.set_yticks([i for i in range(5,20)]) 151 | 152 | #TESTING 153 | total_sum_rate = 0 154 | new_env.test_mode = not random_env 155 | new_env.reset(user_data_path) 156 | for counter in range(step_count): 157 | 158 | if counter%UAV_MOVEMENT_SCALE==0: 159 | new_env.step_UEs() 160 | #Step UAVs 161 | new_env.step_UAVs(isTest=True) 162 | 163 | #Get Error 164 | sum_rate = new_env.calculate_sum_rate() 165 | sum_rates_per_step.append(sum_rate) 166 | total_sum_rate += sum_rate 167 | #Step UEs 168 | new_env.render(ax=ax,plot_trajectories=plot_trajectories) 169 | #Plot resource errors by step and interation 170 | plot_step_graph(sum_rates_per_step,ax1,lim=step_count) 171 | 172 | """if(counter>1): 173 | plot_trajectories(new_env,ax3,counter)""" 174 | yield 175 | 176 | 177 | score = (total_sum_rate/counter,new_env.calculate_sum_rate()) 178 | result_dict = {"scores":score 179 | } 180 | yield result_dict 181 | 182 | 183 | # Main Agent 184 | class Agent(): 185 | 186 | def __init__(self,state_size,action_count,batch_size=200,maxlen=10000): 187 | 188 | self.batch_size = batch_size 189 | self.input_size = state_size 190 | self.action_count = action_count 191 | self.lr = LR 192 | self.gamma = GAMMA #Discount factor = continuous tasklarda return'un sonsuza gitmemesi için 193 | self.epsilon = EPSILON 194 | self.epsilon_decay = EPSILON_DECAY 195 | self.epsilon_min = EPSILON_MIN 196 | self.model = self.build_model() 197 | self.memory = deque(maxlen=maxlen) 198 | 199 | def build_model(self): 200 | """ 201 | Create Neural network as Deep Q Network 202 | """ 203 | model = Sequential() 204 | model.add(Dense(128, input_dim= self.input_size, activation = "relu")) 205 | model.add(Dense(64,activation="relu")) 206 | model.add(Dense(64,activation="relu")) 207 | model.add(Dense(64,activation="relu")) 208 | model.add(Dense(64,activation="relu")) 209 | model.add(Dense(self.action_count,activation="linear")) 210 | model.compile(Adam(learning_rate = self.lr ,clipnorm=1.0),loss=Huber()) 211 | return model 212 | 213 | 214 | def store(self, state, action, reward, next_state, done): 215 | self.memory.append((state, action, reward, next_state, done)) 216 | 217 | 218 | def act(self,state,cannot_random=False): 219 | if (rand.uniform(0,1) <= self.epsilon and not cannot_random): 220 | return rand.randint(0,self.action_count-1) 221 | else: 222 | act_values = self.model.predict(state) 223 | return np.argmax(act_values[0]) 224 | 225 | 226 | def is_memory_enough(self): 227 | return not (len(self.memory)< self.batch_size) 228 | 229 | 230 | def replay(self): 231 | 232 | if(not self.is_memory_enough()): 233 | return 234 | 235 | minibatch = rand.sample(self.memory,self.batch_size) 236 | 237 | for state,action, reward,next_state, done in minibatch: 238 | 239 | if done: 240 | target = reward 241 | else: 242 | target = reward + self.gamma*np.amax(self.model.predict(next_state)) 243 | 244 | train_target = self.model.predict(state) 245 | train_target[0][action] = target 246 | 247 | self.model.fit(state,train_target,verbose=0,workers=8,use_multiprocessing=True) 248 | 249 | 250 | def adaptiveEGreedy(self): 251 | if(self.epsilon > self.epsilon_min and self.is_memory_enough()): 252 | self.epsilon *= self.epsilon_decay 253 | 254 | 255 | # General class for moving objects 256 | class MovingObject(): 257 | def __init__(self,initial_location): 258 | self.current_location = initial_location.copy() 259 | self.starting_location = initial_location.copy() 260 | 261 | 262 | def reset(self): 263 | self.current_location = self.starting_location.copy() 264 | 265 | 266 | # Uav class 267 | class UAV(Agent,MovingObject): 268 | 269 | def __init__(self,state_size,action_count,batch_size,initial_location,maxlen=10000): 270 | Agent.__init__(self,state_size,action_count,batch_size=batch_size,maxlen=maxlen) 271 | MovingObject.__init__(self,initial_location) 272 | self.transmission_power = P 273 | 274 | 275 | # User Class 276 | class UE(MovingObject): 277 | 278 | def __init__(self,initial_location,env_dim,movement_vectors,define_path_first = True,step_count = 10000,movement_function=gaussian): 279 | super().__init__(initial_location) 280 | 281 | self.movement_vectors =movement_vectors[:5].copy() # altitude vektörleri hariç 282 | rand.shuffle(self.movement_vectors) 283 | 284 | self.movement_function = movement_function 285 | self.maxY , self.maxX = env_dim 286 | self.path = [] 287 | self.path_determined = define_path_first 288 | 289 | if define_path_first: 290 | self.initial = 0 291 | for _ in range(step_count): 292 | self.path.append(movement_function(len(self.movement_vectors)-1)) 293 | 294 | def move(self): 295 | action_index = 0 296 | if self.path_determined: 297 | action_index = self.path[self.initial] 298 | self.initial+=1 299 | if self.initial==len(self.path): 300 | self.initial = 0 301 | else: 302 | action_index = self.movement_function(len(self.movement_vectors)-1) 303 | 304 | action = self.movement_vectors[int(action_index)] 305 | self.current_location= [self.current_location[i] + action[i] for i in range(3)] 306 | 307 | 308 | # Main Environment 309 | class UavSimuEnv: 310 | actions = [(1,0,0),(-1,0,0),(0,0,0),(0,1,0),(0,-1,0)] #Actionlar 311 | coord_count = 2 312 | 313 | def distance_func(self,v1,v2): 314 | return (sum(((p-q)*UNIT)**2 for p, q in zip(v1[:2], v2[:2])) + (v1[2]-v2[2])**2)** .5 315 | 316 | def get_all_uavs(self): 317 | return [uav for uav in self.map["uav_set"]] 318 | 319 | def save(self,env_name): 320 | for index,uav in enumerate(self.get_all_uavs()): 321 | model = uav.model 322 | path = "modelss/{}/".format(env_name) 323 | if not os.path.exists(path): 324 | os.makedirs(path) 325 | model_name = "{}uav{}".format(path,index) 326 | model.save(model_name+".h5") 327 | pickle.dump(uav.memory, open(model_name+"memory.pkl", 'wb')) 328 | 329 | 330 | def load(self,env_name,just_for_test=True): 331 | """ 332 | Load trained uav models to environment 333 | """ 334 | for index in range(len(self.map["uav_set"])): 335 | path = "modelss/{}/".format(env_name) 336 | model_name = "{}uav{}".format(path,index) 337 | self.getUAV(index).model = keras.models.load_model(model_name+".h5",compile=just_for_test) 338 | self.getUAV(index).memory = pickle.load(open(model_name+"memory.pkl", "rb")) 339 | 340 | 341 | def get_input_size(self,uav_count): 342 | return uav_count*self.coord_count + math.ceil(self.step_count/UAV_MOVEMENT_SCALE) 343 | 344 | def is_collect_step(self): 345 | return self.getUAV(0).is_memory_enough() 346 | 347 | def get_current_epsilon(self): 348 | return self.getUAV(0).epsilon 349 | 350 | def __init__(self,uav_paths,ue_set,env_dim = (100,100),batch_size=200,max_memory_per_agent=10000,uav_class=UAV): 351 | self.step_count = STEP_COUNT_TRAINED_ON 352 | self.uav_count = len(uav_paths) 353 | self.env_dim = env_dim 354 | self.input_size = self.get_input_size(len(uav_paths)) 355 | self.map = {"uav_set": self.init_uav(uav_paths,batch_size,max_memory_per_agent,uav_class), "ue_set": ue_set} 356 | 357 | def init_uav(self,uav_paths,batch_size,max_memory_per_agent,uav_class): 358 | """ 359 | Agent'ları initialize eder 360 | """ 361 | return [uav_class(self.input_size, len(self.actions), batch_size, begin,maxlen=max_memory_per_agent) 362 | for begin in uav_paths] 363 | 364 | 365 | def get_distance_list(self, dest_list, location,isObject=False): 366 | """ 367 | Verilen konum dizisi ile verilen konum arasındaki mesafelerin dizisini döndürür 368 | """ 369 | if (isObject): 370 | return [self.distance_func(location,loc.current_location) for loc in dest_list] 371 | return [self.distance_func(location,loc) for loc in dest_list] 372 | 373 | def reset(self): 374 | """ 375 | Environmentin state'sini en başa alır 376 | """ 377 | def reset_set(key): 378 | for index, val in enumerate(self.map[key]): 379 | self.map[key][index].reset() 380 | 381 | reset_set("ue_set") 382 | reset_set("uav_set") 383 | self.initial_time = 0 384 | 385 | def get_state_input(self,isNext=False): 386 | """ 387 | verilen Agent'in stateye dair inputunu döndürür 388 | 389 | """ 390 | all_uav_coordinates = [] 391 | 392 | for uav in self.map["uav_set"]: 393 | all_uav_coordinates.extend(uav.current_location[:self.coord_count]) 394 | 395 | time_range = [0 for _ in range(math.ceil(self.step_count/UAV_MOVEMENT_SCALE))] 396 | time_range[self.initial_time // UAV_MOVEMENT_SCALE] = 1 397 | return self.reshape(all_uav_coordinates +time_range) 398 | 399 | 400 | def get_distance_matrix(self,map_obj=None): 401 | """ 402 | Tüm UAV-BS ve UE-AV arasındaki mesafe matrislerini ve ilişki sayısı vektörünü döndürür 403 | """ 404 | if(map_obj==None): 405 | map_obj = self.map 406 | 407 | def minIndex(lst): 408 | 409 | return lst.index(min(lst)) 410 | 411 | def get_distances(lst1, lst2, isLst2Object = True): 412 | distance_matrix = [] 413 | assoc_matrix = [] 414 | for _ in range(len(lst2)): 415 | assoc_matrix.append([]) 416 | 417 | for index,member in enumerate(lst1): 418 | distances = self.get_distance_list(lst2,member.current_location,isLst2Object) 419 | distance_matrix.append(distances) 420 | index_of_min = minIndex(distances) 421 | assoc_matrix[index_of_min].append(index) 422 | 423 | return distance_matrix, assoc_matrix 424 | 425 | ue_uav_matrix, assoc_matrix_uav = get_distances(map_obj["ue_set"],map_obj["uav_set"]) 426 | 427 | return {"ue_uav_matrix": ue_uav_matrix, "assoc_matrix_uav" : assoc_matrix_uav} 428 | 429 | 430 | def calculate_sum_rate(self,map_obj=None): 431 | 432 | if(map_obj==None): 433 | map_obj = self.map 434 | 435 | uav_count = len(map_obj["uav_set"]) 436 | ue_count = len(map_obj["ue_set"]) 437 | distance_state_dict = self.get_distance_matrix(map_obj) 438 | sumrate = 0 439 | transmit_powers = [] 440 | channel_gain_matrix = [] 441 | 442 | def calculate_transmit_power(uav_index): 443 | """ 444 | calculate transmit power of uav 445 | """ 446 | uav = map_obj["uav_set"][uav_index] 447 | uav_assoc_count = len(distance_state_dict["assoc_matrix_uav"][uav_index]) 448 | 449 | #Eğer uav hiçbir servis yapmıyorsa transmit gücünü kullanmayacağı için 0 olacak 450 | if(uav_assoc_count==0): 451 | return 0 452 | p = uav.transmission_power / uav_assoc_count 453 | 454 | return p 455 | 456 | 457 | def calculate_channel_gain(uav_index,user_index): 458 | """ 459 | calculate channel gain between uav and user 460 | """ 461 | uav = map_obj["uav_set"][uav_index] 462 | d = distance_state_dict["ue_uav_matrix"][user_index][uav_index] 463 | theta = math.asin(uav.current_location[-1] / d ) 464 | Plos = B1 * (180 * theta / math.pi - ZETA ) ** B2 465 | 466 | Pnlos = 1 - Plos 467 | g = (K0 ** (-1)) * (d ** (- ALPHA)) * ((Plos*U_LOS + Pnlos*U_NLOS) **(-1)) 468 | return g #gainin negarif olmaması için Plos negatif olmalı 469 | 470 | 471 | #First calculate all channel gains and transmit powers of all combination of uav and users 472 | for uav_index in range(uav_count): 473 | 474 | p = calculate_transmit_power(uav_index) 475 | transmit_powers.append(p) 476 | channel_gain_list = [calculate_channel_gain(uav_index,user_index) for user_index in range(ue_count)] 477 | channel_gain_matrix.append(channel_gain_list) 478 | 479 | 480 | def calculate_interference(uav_index,user_index): 481 | """ 482 | Calculate interference between uav and user caused by other uavs 483 | """ 484 | """return sum([transmit_powers[other_uav_index] * calculate_channel_gain(other_uav_index,user_index) for other_uav_index in range(uav_count) if other_uav_index!=uav_index])""" 485 | I = 0 486 | 487 | for other_uav_index in range(uav_count): 488 | if (other_uav_index==uav_index): 489 | continue 490 | p = transmit_powers[other_uav_index] 491 | g = channel_gain_matrix[other_uav_index][user_index] 492 | I += p*g 493 | 494 | return I 495 | 496 | 497 | for uav_index in range(uav_count): 498 | 499 | p = transmit_powers[uav_index] 500 | users_of_uav = distance_state_dict["assoc_matrix_uav"][uav_index] 501 | for user_index in users_of_uav: 502 | 503 | I = calculate_interference(uav_index,user_index) 504 | g = channel_gain_matrix[uav_index][user_index] 505 | SINR = p*g/(I + STD) 506 | sumrateOfUser = B * math.log2(1+SINR) 507 | sumrate += sumrateOfUser 508 | 509 | return sumrate*1e-7 510 | 511 | def step_UEs(self): 512 | """ 513 | Simulasyondaki tüm UE'leri hareket ettirir 514 | """ 515 | ue_set_length = len(self.map["ue_set"]) 516 | for ue in self.map["ue_set"]: 517 | ue.move() 518 | 519 | #Eğer ue'lerin konumları input olarak varsa, bunu yapamayız 520 | if (not self.isInside(ue.current_location,True)): 521 | self.map["ue_set"].remove(ue) 522 | 523 | 524 | def isCollides(self,uav_index,new_location): 525 | for index,uav in enumerate(self.map["uav_set"]): 526 | if(index!=uav_index and np.array_equal(uav.current_location, new_location)): 527 | return True 528 | return False 529 | 530 | 531 | def step_env(self,action_indexes): 532 | 533 | old_sum_rate = self.calculate_sum_rate() 534 | penalty = 0 535 | done = False 536 | for uav_index in range(len(self.map["uav_set"])): 537 | current = self.getUAV(uav_index).current_location 538 | action_index = action_indexes[uav_index] 539 | new_location = [current[i] + self.actions[action_index][i] 540 | for i in range(3)] 541 | if(not self.isInside(new_location)): 542 | penalty += PENALTY 543 | done = True 544 | else: 545 | self.getUAV(uav_index).current_location = new_location 546 | 547 | for uav_index in range(len(self.map["uav_set"])): 548 | if(self.isCollides(uav_index,self.getUAV(uav_index).current_location)): 549 | done = True 550 | penalty +=PENALTY 551 | 552 | new_sum_rate = self.calculate_sum_rate() 553 | 554 | reward = (new_sum_rate-old_sum_rate) + penalty #penalty negatif 555 | done = done or self.initial_time == self.step_count 556 | return self.get_state_input(), reward, done 557 | 558 | 559 | def getUAV(self,uav_index): 560 | return self.map["uav_set"][uav_index] 561 | 562 | 563 | def step_UAVs(self,isTest=False,isCollectStep=False): 564 | """ 565 | Tüm uav'leri hareket ettirir, 566 | hepsinin bu stepteki rewardlarının toplamını 567 | simulasyonun reward listesine ekler 568 | """ 569 | uav_set_length = len(self.map["uav_set"]) 570 | 571 | state = self.get_state_input() 572 | 573 | #Paralel 574 | action_indexes = [self.getUAV(uav_index).act(state,cannot_random = isTest) for uav_index in range(uav_set_length)] 575 | 576 | next_state, reward, Done = self.step_env(action_indexes) 577 | 578 | 579 | #Paralel 580 | if(not isTest): 581 | for uav_index in range(uav_set_length): 582 | self.getUAV(uav_index).store(state,action_indexes[uav_index],reward,next_state,Done) 583 | 584 | if(not isCollectStep): 585 | self.getUAV(uav_index).replay() 586 | if(self.initial_time==self.step_count-1): 587 | self.getUAV(uav_index).adaptiveEGreedy() 588 | self.initial_time +=1 589 | 590 | 591 | 592 | def reshape(self, data): 593 | return np.reshape(data, [1,self.input_size]) 594 | 595 | 596 | def isInside(self,location,isOnGround=False): 597 | """ 598 | Verilen koordinatın simulasyonun dışına çıkmadığını döndürür 599 | """ 600 | return (0 <= location[0] < self.env_dim[1] and 0<= location[1] < self.env_dim[0] ) and (isOnGround or MIN_ALTITUDE <= location[2] <= MAX_ALTITUDE) 601 | 602 | def get3DMap(self): 603 | def getCoords(map): 604 | x,y,z = [],[],[] 605 | for obj in map: 606 | x.append(obj.current_location[0]) 607 | y.append(obj.current_location[1]) 608 | z.append(obj.current_location[2]) 609 | return x,y,z 610 | return getCoords(self.map["uav_set"]),getCoords(self.map["ue_set"]) 611 | 612 | 613 | def render(self): 614 | 615 | """ 616 | Simulasyonu görsel olarak yazdırır 617 | """ 618 | fig = plt.figure(figsize=(10,10)) 619 | ax = fig.add_subplot(projection='3d'); 620 | uav_coords,ue_coords = self.get3DMap() 621 | clear_output(wait=True) 622 | ax.set_xlim3d(0,self.env_dim[0]) 623 | ax.set_ylim3d(0,self.env_dim[1]) 624 | ax.set_zlim3d(0,MAX_ALTITUDE) 625 | ax.scatter(*uav_coords,c="green") 626 | ax.scatter(*ue_coords,c="red") 627 | distance_map = self.get_distance_matrix()["assoc_matrix_uav"] 628 | for uav_index in range(self.uav_count): 629 | coord_of_UAV = self.getUAV(uav_index).current_location 630 | for ue_index in distance_map[uav_index]: 631 | coord_of_UE = self.map["ue_set"][ue_index].current_location 632 | ax.plot([coord_of_UE[0],coord_of_UAV[0]],[coord_of_UE[1],coord_of_UAV[1]],[coord_of_UE[2],coord_of_UAV[2]],c="green",alpha=.1) 633 | plt.show() 634 | 635 | 636 | # Env with independent execution instead of jpint execution 637 | class IndependentAISimuEnv(UavSimuEnv): 638 | 639 | def step(self,uav_index,action_index): 640 | 641 | #Get current and new locations of uav 642 | current_loc = self.getUAV(uav_index).current_location 643 | new_location = [current_loc[i] + self.actions[action_index][i] 644 | for i in range(3)] 645 | 646 | #Calculate Reward 647 | reward = 0 648 | done = False 649 | if(not self.isInside(new_location) or self.isCollides(uav_index,new_location)): 650 | reward = PENALTY 651 | done = True 652 | else: 653 | old_sum = self.calculate_sum_rate() 654 | self.getUAV(uav_index).current_location = new_location 655 | new_sum = self.calculate_sum_rate() 656 | reward = (new_sum - old_sum) 657 | 658 | #If uav is the last one then increase step 659 | if(uav_index==self.uav_count-1): 660 | self.initial_time+=1 661 | 662 | #Get next state (Next uav's next input = next observation of next agent) 663 | next_obs = self.get_state_input(uav_index) 664 | done = done or self.initial_time == self.step_count 665 | return reward,next_obs,done 666 | 667 | def train_agents(self): 668 | for uav_index in range(self.uav_count): 669 | self.getUAV(uav_index).replay() 670 | if(self.initial_time==self.step_count-1): 671 | self.getUAV(uav_index).adaptiveEGreedy() 672 | 673 | def step_UAVs(self,save_reward=True,isTest=False,isCollectStep=False): 674 | 675 | #For each uav, get (observation,action,reward,next_state) 676 | for uav_index in range(self.uav_count): 677 | 678 | observation = self.get_state_input(uav_index) 679 | 680 | #Get action index of uav 681 | action = self.getUAV(uav_index).act(observation, cannot_random=isTest) 682 | 683 | #step env 684 | reward,next_obs,done = self.step(uav_index,action) 685 | 686 | #If training, agent stores 687 | if(not isTest): 688 | self.getUAV(uav_index).store(observation,action,reward,next_obs,done) 689 | 690 | 691 | 692 | #After a step finishes, agent replays 693 | if(not isTest and not isCollectStep): 694 | self.train_agents() 695 | 696 | 697 | # Agent class same as agent classes of trained models 698 | class semiConvAgent(UAV): 699 | 700 | def build_model(self): 701 | inputA = Input(shape=(self.input_size[0],)) 702 | inputB = Input(shape=list(self.input_size[1])+[1]) 703 | 704 | uav_coord_input = Dense(64, activation="relu")(inputA) 705 | uav_coord_input = Model(inputs=inputA, outputs=uav_coord_input) 706 | 707 | ue_coord_input = AveragePooling2D(pool_size=(4, 4),padding="same")(inputB) 708 | ue_coord_input = Flatten()(ue_coord_input) 709 | ue_coord_input = Model(inputs=inputB, outputs=ue_coord_input) 710 | 711 | combined = concatenate([uav_coord_input.output, ue_coord_input.output]) 712 | 713 | z = Dense(512, activation="relu")(combined) 714 | z = Dense(512, activation="relu")(z) 715 | z = Dense(256, activation="relu")(z) 716 | z = Dense(self.action_count,activation="linear")(z) 717 | 718 | model = Model(inputs=[uav_coord_input.input, ue_coord_input.input], outputs=z) 719 | model.compile(Adam(learning_rate = self.lr ,clipnorm=1.0),loss=Huber()) 720 | return model 721 | 722 | 723 | # Environment class that compatible with trained models 724 | class UserScalableEnvWithTimeInfo(IndependentAISimuEnv): 725 | def __init__(self,*args,**kwargs): 726 | super().__init__(*args,**kwargs) 727 | 728 | def reshape(self, data): 729 | return np.reshape(data, (1,) + data.shape) 730 | 731 | def get_input_size(self,uav_count): 732 | return (uav_count*self.coord_count+1,self.env_dim) 733 | 734 | #Point process Distribution 735 | def create_random_user_locations(self): 736 | total_intensity = self.env_dim[0]*self.env_dim[1]*(np.exp(1)-1) / 5 737 | max_intensity = np.exp(1)/5 738 | 739 | number_points = np.random.poisson(self.env_dim[0] * self.env_dim[1] * max_intensity) 740 | ue_coords = [] 741 | ue_count = len(self.map["ue_set"]) 742 | 743 | while(len(ue_coords)= np.random.uniform(0, max_intensity): 748 | ue_coords.append([x,y,0]) 749 | return ue_coords 750 | 751 | 752 | def reset(self,user_data): 753 | """ 754 | Environmentin state'sini en başa alır 755 | """ 756 | def reset_set(key): 757 | for index, val in enumerate(self.map[key]): 758 | self.map[key][index].reset() 759 | 760 | ue_set = [] 761 | 762 | if(self.test_mode==True): 763 | a = UE 764 | with open(user_data, 'rb') as data: 765 | ue_set = pickle.load(data) 766 | else: 767 | random_locs = self.create_random_user_locations() 768 | for coord in random_locs: 769 | ue_set.append(UE(coord,env_dim,UavSimuEnv.actions,define_path_first=True,step_count=self.step_count//UAV_MOVEMENT_SCALE)) 770 | 771 | self.map["ue_set"] = ue_set 772 | reset_set("uav_set") 773 | self.initial_time = 0 774 | 775 | def get_state_input(self,uav_index): 776 | all_uav_coordinates = [] 777 | 778 | for uav in self.map["uav_set"]: 779 | all_uav_coordinates.extend(uav.current_location[:self.coord_count]) 780 | 781 | user_map = np.zeros((env_dim[0],env_dim[1])) 782 | for user in self.map["ue_set"]: 783 | row = user.current_location[0] 784 | col = user.current_location[1] 785 | user_map[row][col] = 1 786 | 787 | return [self.reshape(np.array(all_uav_coordinates + [max(0,self.step_count-self.initial_time)])),self.reshape(user_map)] 788 | 789 | 790 | # Agent class for only execution of trained models 791 | class semiConvAgentForTest(semiConvAgent): 792 | def act(self,*args,**kwargs): 793 | try: 794 | self.location_history 795 | except: 796 | self.location_history = [] 797 | 798 | action_index = semiConvAgent.act(self,*args,**kwargs) 799 | self.location_history.append(self.current_location) 800 | return action_index 801 | 802 | 803 | # Environment class for only execution of trained models 804 | class UavSimuEnvForTest(UserScalableEnvWithTimeInfo): 805 | actions = [(1,0,0),(-1,0,0),(0,0,0),(0,1,0),(0,-1,0),(0,0,-UAV_MOVEMENT_SCALE),(0,0,UAV_MOVEMENT_SCALE)] #Actionlar 806 | 807 | def get_input_size(self,uav_count): 808 | return (uav_count*self.coord_count+1,self.env_dim) 809 | 810 | def load(self,path,just_for_test=True,load_memory=False): 811 | for index in range(len(self.map["uav_set"])): 812 | model_name = "{}/uav{}".format(path,index) 813 | self.getUAV(index).model = keras.models.load_model(model_name+".h5",compile=not just_for_test) 814 | if(load_memory): 815 | self.getUAV(index).memory = pickle.load(open(model_name+"memory.pkl", "rb")) 816 | 817 | 818 | def get_state_input(self,uav_index): 819 | all_uav_coordinates = [] 820 | 821 | for uav in self.map["uav_set"]: 822 | all_uav_coordinates.extend(uav.current_location[:self.coord_count]) 823 | 824 | user_map = np.zeros((env_dim[0],env_dim[1])) 825 | for user in self.map["ue_set"]: 826 | row = user.current_location[0] 827 | col = user.current_location[1] 828 | user_map[row][col] = 1 829 | 830 | return [self.reshape(np.array(all_uav_coordinates + [max(0,self.step_count-self.initial_time)])),self.reshape(user_map)] 831 | 832 | def step_UAVs(self,save_reward=True,isTest=False,isCollectStep=False): 833 | 834 | #For each uav, get (observation,action,reward,next_state) 835 | for uav_index in range(self.uav_count): 836 | 837 | observation = self.get_state_input(uav_index) 838 | 839 | #Get action index of uav 840 | action = self.getUAV(uav_index).act(observation, cannot_random=isTest) 841 | 842 | #step env 843 | self.step(uav_index,action) 844 | 845 | def step(self,uav_index,action_index): 846 | 847 | #Get current and new locations of uav 848 | current_loc = self.getUAV(uav_index).current_location 849 | new_location = [current_loc[i] + self.actions[action_index][i] 850 | for i in range(3)] 851 | 852 | if(not (not self.isInside(new_location) or self.isCollides(uav_index,new_location))): 853 | self.getUAV(uav_index).current_location = new_location 854 | 855 | #If uav is the last one then increase step 856 | if(uav_index==self.uav_count-1): 857 | self.initial_time+=1 858 | def render(self,ax=None,plot_trajectories=True): 859 | 860 | """ 861 | Simulasyonu görsel olarak yazdırır 862 | """ 863 | 864 | ax.clear() 865 | uav_coords,ue_coords = self.get3DMap() 866 | ax.set_xlim3d(0,self.env_dim[0]) 867 | ax.set_ylim3d(0,self.env_dim[1]) 868 | ax.set_zlim3d(0,MAX_ALTITUDE) 869 | 870 | ax.set_xlabel("X (unit)") 871 | ax.set_ylabel("Y (unit)") 872 | ax.set_zlabel("Z (meters)") 873 | ax.scatter(*uav_coords,c="green") 874 | ax.scatter(*ue_coords,c="red") 875 | 876 | 877 | if(plot_trajectories): 878 | for uav_index,uav in enumerate(self.get_all_uavs()): 879 | trajectory_map_X = [] 880 | trajectory_map_Y = [] 881 | trajectory_map_Z = [] 882 | for location in uav.location_history: 883 | trajectory_map_X.append(location[0]) 884 | trajectory_map_Y.append(location[1]) 885 | trajectory_map_Z.append(location[2]) 886 | 887 | ax.scatter(trajectory_map_X,trajectory_map_Y,trajectory_map_Z,c="purple",alpha=.3) 888 | 889 | distance_map = self.get_distance_matrix()["assoc_matrix_uav"] 890 | for uav_index in range(self.uav_count): 891 | coord_of_UAV = self.getUAV(uav_index).current_location 892 | for ue_index in distance_map[uav_index]: 893 | coord_of_UE = self.map["ue_set"][ue_index].current_location 894 | ax.plot([coord_of_UE[0],coord_of_UAV[0]],[coord_of_UE[1],coord_of_UAV[1]],[coord_of_UE[2],coord_of_UAV[2]],c="green",alpha=.1) 895 | -------------------------------------------------------------------------------- /UAVTraining.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "WCMXPdhifmGR" 7 | }, 8 | "source": [ 9 | "## Constants" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "6AQKyacCz0SL" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "import math\n", 21 | "import scipy.constants as constant\n", 22 | "\n", 23 | "# Carrier Frequency \n", 24 | "FC = 2e9 \n", 25 | "\n", 26 | "# Speed of Light\n", 27 | "C = constant.c \n", 28 | "\n", 29 | "# Power Spectral Density\n", 30 | "N0 = 1e-20 \n", 31 | "\n", 32 | "# Bandwidth\n", 33 | "B = 1e6 \n", 34 | "\n", 35 | "# Noise\n", 36 | "STD = B * N0 \n", 37 | "\n", 38 | "# Environmental Variable 1\n", 39 | "B1 = 0.36 \n", 40 | "\n", 41 | "# Environmental Variable 2\n", 42 | "B2 = 0.21 \n", 43 | "\n", 44 | "# Path Loss Exponent\n", 45 | "ALPHA = 2 \n", 46 | "\n", 47 | "# Additional path loss for LineOfSight\n", 48 | "U_LOS = 10**(3/10) \n", 49 | "\n", 50 | "# Additional path loss for NonLineOfSight\n", 51 | "U_NLOS = 10**(23/10) \n", 52 | "\n", 53 | "K0 = (4 * math.pi * FC / C) ** 2\n", 54 | "\n", 55 | "# Environmental Variable 3\n", 56 | "ZETA = 0 \n", 57 | "\n", 58 | "# Transmission power of an UAV\n", 59 | "P = 0.08 \n", 60 | "\n", 61 | "# Discount Rate\n", 62 | "GAMMA = 0.95 \n", 63 | "\n", 64 | "# Learning Rate\n", 65 | "LR = 0.001 \n", 66 | "\n", 67 | "# Initial Epsilon Greedy Value\n", 68 | "EPSILON = 1 \n", 69 | "\n", 70 | "# Epsilon Greedy Decay Rate\n", 71 | "EPSILON_DECAY = 0.95 \n", 72 | "\n", 73 | "# Minimum Epsilon Greedy\n", 74 | "EPSILON_MIN = 0.1 \n", 75 | "\n", 76 | "# Minimum altitude to fly\n", 77 | "MIN_ALTITUDE = 200\n", 78 | "\n", 79 | "# Initial Altitude\n", 80 | "ALTITUDE = 400 \n", 81 | "\n", 82 | "# Maximum altitude to fly\n", 83 | "MAX_ALTITUDE = 800 \n", 84 | "\n", 85 | "# Penalty value\n", 86 | "PENALTY = -100 \n", 87 | "\n", 88 | "# UAV speed rate according to humans\n", 89 | "UAV_MOVEMENT_SCALE= 20\n", 90 | "\n", 91 | "TEST_ENV_FILE = \"User_Data/test_env.pkl\"" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": { 98 | "id": "bgRhcYK3fG_w" 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "import random as rand \n", 103 | "import tensorflow as tf\n", 104 | "from time import sleep\n", 105 | "from keras.layers import Dense, LSTM, Dropout\n", 106 | "from keras.models import Sequential\n", 107 | "from keras.optimizers import Adam\n", 108 | "from collections import deque \n", 109 | "import time\n", 110 | "class Agent(): \n", 111 | " def __init__(self,state_size,action_count,batch_size=200,maxlen=10000):\n", 112 | " self.batch_size = batch_size\n", 113 | " self.input_size = state_size \n", 114 | " self.action_count = action_count \n", 115 | " self.lr = LR\n", 116 | " self.gamma = GAMMA\n", 117 | " self.epsilon = EPSILON\n", 118 | " self.epsilon_decay = EPSILON_DECAY\n", 119 | " self.epsilon_min = EPSILON_MIN\n", 120 | " self.model = self.build_model() # Deep Q Network\n", 121 | " self.memory = deque(maxlen=maxlen) # Experience Memory\n", 122 | "\n", 123 | " def build_model(self):\n", 124 | " \"\"\"\n", 125 | " Create Neural network as Deep Q Network\n", 126 | " \"\"\" \n", 127 | " model = Sequential()\n", 128 | " model.add(Dense(128, input_dim= self.input_size, activation = \"relu\")) \n", 129 | " model.add(Dense(64,activation=\"relu\")) \n", 130 | " model.add(Dense(64,activation=\"relu\")) \n", 131 | " model.add(Dense(64,activation=\"relu\")) \n", 132 | " model.add(Dense(64,activation=\"relu\")) \n", 133 | " model.add(Dense(self.action_count,activation=\"linear\"))\n", 134 | " model.compile(Adam(learning_rate = self.lr ,clipnorm=1.0),loss=Huber())\n", 135 | " return model \n", 136 | "\n", 137 | " def store(self, state, action, reward, next_state, done):\n", 138 | " \"\"\"\n", 139 | " Store experience in memory\n", 140 | " \"\"\"\n", 141 | " self.memory.append((state, action, reward, next_state, done))\n", 142 | "\n", 143 | " def act(self,state,cannot_random=False):\n", 144 | " \"\"\"\n", 145 | " Choosing action either random or using Deep Q Network\n", 146 | " \"\"\"\n", 147 | " if (rand.uniform(0,1) <= self.epsilon and not cannot_random):\n", 148 | " return rand.randint(0,self.action_count-1) \n", 149 | " else:\n", 150 | " act_values = self.model.predict(state) \n", 151 | " return np.argmax(act_values[0])\n", 152 | " \n", 153 | " def is_memory_enough(self):\n", 154 | " \"\"\"\n", 155 | " Check if memory is full enough for a batch\n", 156 | " \"\"\"\n", 157 | " return not (len(self.memory)< self.batch_size)\n", 158 | " \n", 159 | " def replay(self):\n", 160 | " \"\"\"\n", 161 | " Sample random experiences from memory and replay them\n", 162 | " \"\"\"\n", 163 | " if(not self.is_memory_enough()):\n", 164 | " return\n", 165 | " \n", 166 | " minibatch = rand.sample(self.memory,self.batch_size)\n", 167 | " \n", 168 | " for state,action, reward,next_state, done in minibatch: \n", 169 | " \n", 170 | " if done:\n", 171 | " target = reward\n", 172 | " else: \n", 173 | " target = reward + self.gamma*np.amax(self.model.predict(next_state)) \n", 174 | " \n", 175 | " train_target = self.model.predict(state)\n", 176 | " train_target[0][action] = target \n", 177 | " \n", 178 | " self.model.fit(state,train_target,verbose=0,workers=8,use_multiprocessing=True)\n", 179 | "\n", 180 | " def adaptiveEGreedy(self):\n", 181 | " \"\"\"\n", 182 | " Decay epsilon\n", 183 | " \"\"\"\n", 184 | " if(self.epsilon > self.epsilon_min and self.is_memory_enough()):\n", 185 | " self.epsilon *= self.epsilon_decay \n", 186 | " print(\"current_randomness: \", self.epsilon)\n", 187 | "\n", 188 | "from scipy.stats import truncnorm\n", 189 | "\n", 190 | "# Helper Function for normal distribution\n", 191 | "def get_truncated_normal(mean=0, low=0, upp=10):\n", 192 | " return truncnorm(\n", 193 | " (low - mean), (upp - mean), loc=mean, scale=1).rvs()\n", 194 | " \n", 195 | "# Normal Distribution\n", 196 | "def gaussian(end,begin=0):\n", 197 | " normal_random = round(get_truncated_normal(mean=int((begin+end)/2),low=begin,upp=end))\n", 198 | " return max(0,min(normal_random,end))\n", 199 | "\n", 200 | "\n", 201 | "# General Class for moving objects to extend from.\n", 202 | "class MovingObject():\n", 203 | " def __init__(self,initial_location):\n", 204 | " self.current_location = initial_location.copy()\n", 205 | " self.starting_location = initial_location.copy()\n", 206 | "\n", 207 | " def reset(self):\n", 208 | " self.current_location = self.starting_location.copy()\n", 209 | "\n", 210 | "\n", 211 | "#UAV class\n", 212 | "class UAV(Agent,MovingObject): \n", 213 | " def __init__(self,state_size,action_count,batch_size,initial_location,maxlen=10000):\n", 214 | " Agent.__init__(self,state_size,action_count,batch_size=batch_size,maxlen=maxlen)\n", 215 | " MovingObject.__init__(self,initial_location)\n", 216 | " self.transmission_power = P\n", 217 | "\n", 218 | "\n", 219 | "#User class\n", 220 | "class UE(MovingObject):\n", 221 | " def __init__(self,initial_location,env_dim,movement_vectors,define_path_first = True,step_count = 10000,movement_function=gaussian):\n", 222 | " super().__init__(initial_location)\n", 223 | "\n", 224 | " self.movement_vectors =movement_vectors[:5].copy()\n", 225 | " rand.shuffle(self.movement_vectors)\n", 226 | "\n", 227 | " self.movement_function = movement_function\n", 228 | " self.maxY , self.maxX = env_dim\n", 229 | " self.path = []\n", 230 | " self.path_determined = define_path_first\n", 231 | "\n", 232 | " if define_path_first:\n", 233 | " self.initial = 0\n", 234 | " for _ in range(step_count):\n", 235 | " self.path.append(movement_function(len(self.movement_vectors)-1))\n", 236 | " \n", 237 | " def move(self):\n", 238 | " action_index = 0\n", 239 | " if self.path_determined:\n", 240 | " action_index = self.path[self.initial]\n", 241 | " self.initial+=1\n", 242 | " if self.initial==len(self.path):\n", 243 | " self.initial = 0\n", 244 | " else:\n", 245 | " action_index = self.movement_function(len(self.movement_vectors)-1)\n", 246 | "\n", 247 | " action = self.movement_vectors[int(action_index)]\n", 248 | " self.current_location= [self.current_location[i] + action[i] for i in range(3)]" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": { 254 | "id": "RHsK4i8vN-wc" 255 | }, 256 | "source": [ 257 | "# Environment Class" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": null, 263 | "metadata": { 264 | "id": "MVYDnGuofLUA" 265 | }, 266 | "outputs": [], 267 | "source": [ 268 | "from statistics import mean\n", 269 | "import numpy as np\n", 270 | "import matplotlib.pyplot as plt\n", 271 | "from IPython.display import clear_output\n", 272 | "import math\n", 273 | "from mpl_toolkits.mplot3d import Axes3D\n", 274 | "import seaborn as sns\n", 275 | "import time\n", 276 | "from keras.models import load_model\n", 277 | "import pickle\n", 278 | "\n", 279 | "sns.set_style(\"darkgrid\")\n", 280 | "\n", 281 | "\n", 282 | "class UavSimuEnv():\n", 283 | " actions = [(1,0,0),(-1,0,0),(0,0,0),(0,1,0),(0,-1,0)] # Actions\n", 284 | " coord_count = 2\n", 285 | " \n", 286 | " def distance_func(self,v1,v2): \n", 287 | " \"\"\" x and y is unit, altitude is meter so calculated distance this way \"\"\"\n", 288 | " return (sum(((p-q)*UNIT)**2 for p, q in zip(v1[:2], v2[:2])) + (v1[2]-v2[2])**2)** .5\n", 289 | "\n", 290 | " def get_all_uavs(self):\n", 291 | " return [uav for uav in self.map[\"uav_set\"]]\n", 292 | " \n", 293 | " def save(self,env_name): \n", 294 | " \"\"\"\n", 295 | " save models and memories of all UAVs\n", 296 | " \"\"\"\n", 297 | " for index,uav in enumerate(self.get_all_uavs()):\n", 298 | " model = uav.model\n", 299 | " path = \"modelss/{}/\".format(env_name)\n", 300 | " if not os.path.exists(path):\n", 301 | " os.makedirs(path)\n", 302 | " model_name = \"{}uav{}\".format(path,index)\n", 303 | " model.save(model_name+\".h5\")\n", 304 | " pickle.dump(uav.memory, open(model_name+\"memory.pkl\", 'wb'))\n", 305 | " \n", 306 | " \n", 307 | " def load(self,env_name,just_for_test=True):\n", 308 | " \"\"\"\n", 309 | " load models and memories for all UAVs\n", 310 | " \"\"\"\n", 311 | " for index in range(len(self.map[\"uav_set\"])):\n", 312 | " path = \"modelss/{}/\".format(env_name)\n", 313 | " model_name = \"{}uav{}\".format(path,index)\n", 314 | " self.getUAV(index).model = keras.models.load_model(model_name+\".h5\",compile=just_for_test)\n", 315 | " self.getUAV(index).memory = pickle.load(open(model_name+\"memory.pkl\", \"rb\"))\n", 316 | "\n", 317 | " \n", 318 | " def get_input_size(self,uav_count):\n", 319 | " return uav_count*self.coord_count + math.ceil(self.step_count/UAV_MOVEMENT_SCALE)\n", 320 | "\n", 321 | " def is_collect_step(self):\n", 322 | " return self.getUAV(0).is_memory_enough() \n", 323 | " \n", 324 | " def get_current_epsilon(self):\n", 325 | " return self.getUAV(0).epsilon\n", 326 | " \n", 327 | " def __init__(self,uav_paths,ue_set,env_dim = (100,100),batch_size=200,max_memory_per_agent=10000,uav_class=UAV):\n", 328 | " self.step_count = STEP_COUNT\n", 329 | " self.uav_count = len(uav_paths) \n", 330 | " self.env_dim = env_dim \n", 331 | " self.input_size = self.get_input_size(len(uav_paths))\n", 332 | " self.map = {\"uav_set\": self.init_uav(uav_paths,batch_size,max_memory_per_agent,uav_class), \"ue_set\": ue_set}\n", 333 | "\n", 334 | " def init_uav(self,uav_paths,batch_size,max_memory_per_agent,uav_class):\n", 335 | " \"\"\"\n", 336 | " Initialize Agents\n", 337 | " \"\"\"\n", 338 | " print(uav_class)\n", 339 | " return [uav_class(self.input_size, len(self.actions), batch_size, begin,maxlen=max_memory_per_agent) \n", 340 | " for begin in uav_paths]\n", 341 | "\n", 342 | "\n", 343 | " def get_distance_list(self, dest_list, location,isObject=False):\n", 344 | " \"\"\"\n", 345 | " Return distance list of distance between the locations in dest;_list and given location\n", 346 | " \"\"\"\n", 347 | " if (isObject):\n", 348 | " return [self.distance_func(location,loc.current_location) for loc in dest_list]\n", 349 | " return [self.distance_func(location,loc) for loc in dest_list] \n", 350 | " \n", 351 | " def reset(self):\n", 352 | " \"\"\"\n", 353 | " Reset Environment\n", 354 | " \"\"\"\n", 355 | " def reset_set(key):\n", 356 | " for index, val in enumerate(self.map[key]):\n", 357 | " self.map[key][index].reset()\n", 358 | " \n", 359 | " reset_set(\"ue_set\")\n", 360 | " reset_set(\"uav_set\")\n", 361 | " self.initial_time = 0 \n", 362 | "\n", 363 | " def get_state_input(self,isNext=False):\n", 364 | " \"\"\"\n", 365 | " Return state information for UAV\n", 366 | " \"\"\"\n", 367 | " all_uav_coordinates = []\n", 368 | "\n", 369 | " for uav in self.map[\"uav_set\"]:\n", 370 | " all_uav_coordinates.extend(uav.current_location[:self.coord_count]) \n", 371 | " \n", 372 | " time_range = [0 for _ in range(math.ceil(self.step_count/UAV_MOVEMENT_SCALE))]\n", 373 | " print(self.initial_time // UAV_MOVEMENT_SCALE)\n", 374 | " print(self.initial_time)\n", 375 | " time_range[self.initial_time // UAV_MOVEMENT_SCALE] = 1\n", 376 | " return self.reshape(all_uav_coordinates +time_range)\n", 377 | "\n", 378 | " \n", 379 | " def get_distance_matrix(self,map_obj=None):\n", 380 | " \"\"\"\n", 381 | " Return all distance matrix and association count vector between UAV-UE \n", 382 | " \"\"\"\n", 383 | " if(map_obj==None):\n", 384 | " map_obj = self.map\n", 385 | " \n", 386 | " def minIndex(lst):\n", 387 | " return lst.index(min(lst)) \n", 388 | "\n", 389 | " def get_distances(lst1, lst2, isLst2Object = True):\n", 390 | " distance_matrix = []\n", 391 | " assoc_matrix = []\n", 392 | " for _ in range(len(lst2)):\n", 393 | " assoc_matrix.append([])\n", 394 | "\n", 395 | " for index,member in enumerate(lst1):\n", 396 | " distances = self.get_distance_list(lst2,member.current_location,isLst2Object)\n", 397 | " distance_matrix.append(distances)\n", 398 | " index_of_min = minIndex(distances)\n", 399 | " assoc_matrix[index_of_min].append(index)\n", 400 | "\n", 401 | " return distance_matrix, assoc_matrix\n", 402 | "\n", 403 | " ue_uav_matrix, assoc_matrix_uav = get_distances(map_obj[\"ue_set\"],map_obj[\"uav_set\"]) \n", 404 | "\n", 405 | " return {\"ue_uav_matrix\": ue_uav_matrix, \"assoc_matrix_uav\" : assoc_matrix_uav}\n", 406 | "\n", 407 | "\n", 408 | " def calculate_sum_rate(self,map_obj=None):\n", 409 | " \n", 410 | " if(map_obj==None):\n", 411 | " map_obj = self.map\n", 412 | " \n", 413 | " uav_count = len(map_obj[\"uav_set\"])\n", 414 | " ue_count = len(map_obj[\"ue_set\"])\n", 415 | " distance_state_dict = self.get_distance_matrix(map_obj)\n", 416 | " sumrate = 0\n", 417 | " transmit_powers = []\n", 418 | " channel_gain_matrix = []\n", 419 | " \n", 420 | " # Nested Function\n", 421 | " def calculate_transmit_power(uav_index):\n", 422 | " \"\"\"\n", 423 | " calculate transmit power of uav\n", 424 | " \"\"\"\n", 425 | " uav = map_obj[\"uav_set\"][uav_index]\n", 426 | " uav_assoc_count = len(distance_state_dict[\"assoc_matrix_uav\"][uav_index])\n", 427 | " \n", 428 | " #Eğer uav hiçbir servis yapmıyorsa transmit gücünü kullanmayacağı için 0 olacak\n", 429 | " if(uav_assoc_count==0):\n", 430 | " return 0 \n", 431 | " p = uav.transmission_power / uav_assoc_count\n", 432 | "\n", 433 | " return p\n", 434 | "\n", 435 | " # Nested Function\n", 436 | " def calculate_channel_gain(uav_index,user_index):\n", 437 | " \"\"\"\n", 438 | " calculate channel gain between uav and user\n", 439 | " \"\"\"\n", 440 | " uav = map_obj[\"uav_set\"][uav_index]\n", 441 | " d = distance_state_dict[\"ue_uav_matrix\"][user_index][uav_index]\n", 442 | " theta = math.asin(uav.current_location[-1] / d )\n", 443 | " Plos = B1 * (180 * theta / math.pi - ZETA ) ** B2\n", 444 | " \n", 445 | " Pnlos = 1 - Plos\n", 446 | " g = (K0 ** (-1)) * (d ** (- ALPHA)) * ((Plos*U_LOS + Pnlos*U_NLOS) **(-1))\n", 447 | " return g\n", 448 | "\n", 449 | "\n", 450 | " #First calculate all channel gains and transmit powers of all combination of uav and users\n", 451 | " for uav_index in range(uav_count):\n", 452 | "\n", 453 | " p = calculate_transmit_power(uav_index)\n", 454 | " transmit_powers.append(p)\n", 455 | " channel_gain_list = [calculate_channel_gain(uav_index,user_index) \\\n", 456 | " for user_index in range(ue_count)]\n", 457 | " channel_gain_matrix.append(channel_gain_list)\n", 458 | " \n", 459 | " # Nested Function\n", 460 | " def calculate_interference(uav_index,user_index):\n", 461 | " \"\"\"\n", 462 | " Calculate interference between uav and user caused by other uavs\n", 463 | " \"\"\"\n", 464 | " I = 0\n", 465 | "\n", 466 | " for other_uav_index in range(uav_count):\n", 467 | " if (other_uav_index==uav_index):\n", 468 | " continue\n", 469 | " p = transmit_powers[other_uav_index]\n", 470 | " g = channel_gain_matrix[other_uav_index][user_index]\n", 471 | " I += p*g \n", 472 | " \n", 473 | " return I\n", 474 | " \n", 475 | "\n", 476 | " for uav_index in range(uav_count):\n", 477 | "\n", 478 | " p = transmit_powers[uav_index]\n", 479 | " users_of_uav = distance_state_dict[\"assoc_matrix_uav\"][uav_index]\n", 480 | " for user_index in users_of_uav:\n", 481 | " \n", 482 | " I = calculate_interference(uav_index,user_index)\n", 483 | " g = channel_gain_matrix[uav_index][user_index]\n", 484 | " SINR = p*g/(I + STD)\n", 485 | " sumrateOfUser = B * math.log2(1+SINR)\n", 486 | " sumrate += sumrateOfUser \n", 487 | " \n", 488 | " return sumrate*1e-7\n", 489 | "\n", 490 | " def step_UEs(self):\n", 491 | " \"\"\"\n", 492 | " Move all UEs in Simulation\n", 493 | " \"\"\"\n", 494 | " ue_set_length = len(self.map[\"ue_set\"])\n", 495 | " for ue in self.map[\"ue_set\"]:\n", 496 | " ue.move()\n", 497 | "\n", 498 | " if (not self.isInside(ue.current_location,True)):\n", 499 | " self.map[\"ue_set\"].remove(ue)\n", 500 | "\n", 501 | "\n", 502 | " def isCollides(self,uav_index,new_location): \n", 503 | " \"\"\"\n", 504 | " check if location is against limits\n", 505 | " \"\"\"\n", 506 | " for index,uav in enumerate(self.map[\"uav_set\"]): \n", 507 | " if(index!=uav_index and np.array_equal(uav.current_location, new_location)):\n", 508 | " return True\n", 509 | " return False\n", 510 | "\n", 511 | "\n", 512 | " def step_env(self,action_indexes):\n", 513 | " \"\"\"\n", 514 | " Change environment with given joint actions\n", 515 | " \"\"\"\n", 516 | " old_sum_rate = self.calculate_sum_rate()\n", 517 | " penalty = 0\n", 518 | " done = False\n", 519 | " for uav_index in range(len(self.map[\"uav_set\"])):\n", 520 | " current = self.getUAV(uav_index).current_location\n", 521 | " action_index = action_indexes[uav_index]\n", 522 | " new_location = [current[i] + self.actions[action_index][i]\n", 523 | " for i in range(3)]\n", 524 | " if(not self.isInside(new_location)):\n", 525 | " penalty += PENALTY\n", 526 | " done = True\n", 527 | " else:\n", 528 | " self.getUAV(uav_index).current_location = new_location\n", 529 | " \n", 530 | " for uav_index in range(len(self.map[\"uav_set\"])):\n", 531 | " if(self.isCollides(uav_index,self.getUAV(uav_index).current_location)):\n", 532 | " done = True\n", 533 | " penalty +=PENALTY\n", 534 | " \n", 535 | " new_sum_rate = self.calculate_sum_rate()\n", 536 | " \n", 537 | " reward = new_sum_rate-old_sum_rate + penalty #penalty negatif\n", 538 | " done = done or self.initial_time == self.step_count\n", 539 | " return self.get_state_input(), reward, done\n", 540 | " \n", 541 | " \n", 542 | " def getUAV(self,uav_index):\n", 543 | " return self.map[\"uav_set\"][uav_index]\n", 544 | "\n", 545 | "\n", 546 | " def step_UAVs(self,isTest=False,isCollectStep=False):\n", 547 | " \"\"\"\n", 548 | " Move all UAVs\n", 549 | " \"\"\"\n", 550 | " uav_set_length = len(self.map[\"uav_set\"])\n", 551 | " \n", 552 | " state = self.get_state_input()\n", 553 | " \n", 554 | " action_indexes = [self.getUAV(uav_index).act(state,cannot_random = isTest) \\\n", 555 | " for uav_index in range(uav_set_length)]\n", 556 | " \n", 557 | " next_state, reward, Done = self.step_env(action_indexes)\n", 558 | " \n", 559 | " if(not isTest):\n", 560 | " for uav_index in range(uav_set_length):\n", 561 | " self.getUAV(uav_index).store(state,action_indexes[uav_index],reward,next_state,Done) \n", 562 | " \n", 563 | " if(not isCollectStep):\n", 564 | " self.getUAV(uav_index).replay()\n", 565 | " if(self.initial_time==self.step_count-1):\n", 566 | " self.getUAV(uav_index).adaptiveEGreedy() \n", 567 | " self.initial_time +=1\n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " def reshape(self, data):\n", 572 | " return np.reshape(data, [1,self.input_size]) \n", 573 | "\n", 574 | "\n", 575 | " def isInside(self,location,isOnGround=False):\n", 576 | " \"\"\"\n", 577 | " Return if location is against the limits\n", 578 | " \"\"\"\n", 579 | " return (0 <= location[0] < self.env_dim[1] and 0<= location[1] < self.env_dim[0] ) \\\n", 580 | " and (isOnGround or MIN_ALTITUDE <= location[2] <= MAX_ALTITUDE)\n", 581 | "\n", 582 | " def get3DMap(self):\n", 583 | " def getCoords(map):\n", 584 | " x,y,z = [],[],[]\n", 585 | " for obj in map:\n", 586 | " x.append(obj.current_location[0])\n", 587 | " y.append(obj.current_location[1])\n", 588 | " z.append(obj.current_location[2])\n", 589 | " return x,y,z\n", 590 | " return getCoords(self.map[\"uav_set\"]),getCoords(self.map[\"ue_set\"])\n", 591 | "\n", 592 | " \n", 593 | " def render(self):\n", 594 | "\n", 595 | " \"\"\"\n", 596 | " Render simulation visually\n", 597 | " \"\"\"\n", 598 | " fig = plt.figure(figsize=(10,10))\n", 599 | " ax = Axes3D(fig)\n", 600 | " uav_coords,ue_coords = self.get3DMap()\n", 601 | " clear_output(wait=True)\n", 602 | " ax.set_xlim3d(0,self.env_dim[0])\n", 603 | " ax.set_ylim3d(0,self.env_dim[1])\n", 604 | " ax.set_zlim3d(0,MAX_ALTITUDE)\n", 605 | " ax.scatter(*uav_coords,c=\"green\")\n", 606 | " ax.scatter(*ue_coords,c=\"red\")\n", 607 | " distance_map = self.get_distance_matrix()[\"assoc_matrix_uav\"]\n", 608 | " for uav_index in range(self.uav_count):\n", 609 | " coord_of_UAV = self.getUAV(uav_index).current_location\n", 610 | " for ue_index in distance_map[uav_index]: \n", 611 | " coord_of_UE = self.map[\"ue_set\"][ue_index].current_location\n", 612 | " ax.plot([coord_of_UE[0],coord_of_UAV[0]],[coord_of_UE[1],coord_of_UAV[1]],[coord_of_UE[2],coord_of_UAV[2]],c=\"green\",alpha=.1)\n", 613 | " plt.show()" 614 | ] 615 | }, 616 | { 617 | "cell_type": "markdown", 618 | "metadata": { 619 | "id": "umC3ZrHAfSH9" 620 | }, 621 | "source": [ 622 | "# Simulation Functions" 623 | ] 624 | }, 625 | { 626 | "cell_type": "code", 627 | "execution_count": null, 628 | "metadata": { 629 | "id": "H0v-PiXCKj8h" 630 | }, 631 | "outputs": [], 632 | "source": [ 633 | "import numpy as np\n", 634 | "import matplotlib.pyplot as plt\n", 635 | "from collections import deque\n", 636 | "from keras.optimizers import Adam \n", 637 | "import multiprocessing\n", 638 | "from joblib import Parallel, delayed\n", 639 | "\n", 640 | "def plot_iter_graph(ignore_count,sum_rates_per_iteration):\n", 641 | " plt.plot(sum_rates_per_iteration[ignore_count:])\n", 642 | " plt.title(\"Sum rates per iteration\")\n", 643 | " plt.show(sum_rates_per_iteration)\n", 644 | "\n", 645 | "def plot_step_graph(iteration_num, step_num,step_count,sum_rates_per_step):\n", 646 | " if (iteration_num >= 1): \n", 647 | " if (step_num >1):\n", 648 | " begin_iter = max(0,iteration_num-5)\n", 649 | " x_pivot = begin_iter*step_count\n", 650 | " print(begin_iter,x_pivot,iteration_num)\n", 651 | " show_range = range(x_pivot , len(sum_rates_per_step))\n", 652 | " plt.plot(show_range,sum_rates_per_step[x_pivot:])\n", 653 | " plt.title(\"Sum rates per step\") \n", 654 | " for t in range(5):\n", 655 | " plt.axvline(x=(t+begin_iter)*step_count,color=\"red\")\n", 656 | " \n", 657 | " plt.show()\n", 658 | "\n", 659 | "# Train environment\n", 660 | "def simulate(env,iteration_count = 50, step_count = 20,batch_size=200,consider_terminal_state=False,collect_step_size=100,\n", 661 | " env_name=None,onlyForTest=False):\n", 662 | "\n", 663 | " sum_rates_per_iteration = []\n", 664 | " sum_rates_per_step = []\n", 665 | " max_score = 0\n", 666 | " max_other_score = 0\n", 667 | " for iter_num in range(iteration_count + (collect_step_size//step_count)):\n", 668 | " \n", 669 | " \n", 670 | " counter = 0 \n", 671 | " isCollectStep = collect_step_size > (iter_num*step_count)\n", 672 | " env.test_mode = False\n", 673 | " env.reset() \n", 674 | " while (not onlyForTest and counter!=step_count and (isCollectStep == (collect_step_size > (iter_num*step_count + counter) ))):\n", 675 | " \n", 676 | " \n", 677 | " \"\"\"if counter%UAV_MOVEMENT_SCALE==0:\n", 678 | " env.step_UEs()\"\"\"\n", 679 | " \n", 680 | " #Step simulation\n", 681 | " env.step_UAVs(isCollectStep=isCollectStep) \n", 682 | " counter += 1\n", 683 | " \n", 684 | " #Plot sum rates by step and interation\n", 685 | " clear_output(wait=False)\n", 686 | " \n", 687 | " plot_iter_graph(0,sum_rates_per_iteration)\n", 688 | " plot_step_graph(iter_num,counter,step_count,sum_rates_per_step)\n", 689 | " \n", 690 | " print(env.get_current_epsilon())\n", 691 | " print(\"Training, step: \",counter)\n", 692 | " print(\"Iteration: \",iter_num)\n", 693 | " \n", 694 | "\n", 695 | " #TESTING\n", 696 | " counter = 0\n", 697 | " total_sum_rate = 0\n", 698 | " env.test_mode = True\n", 699 | " env.reset()\n", 700 | " while (counter!=step_count and not isCollectStep):\n", 701 | "\n", 702 | " \"\"\"if counter%UAV_MOVEMENT_SCALE==0:\n", 703 | " env.step_UEs()\"\"\"\n", 704 | " #Step UAVs\n", 705 | " env.step_UAVs(isTest=True)\n", 706 | "\n", 707 | " #Get Error\n", 708 | " sum_rate = env.calculate_sum_rate()\n", 709 | " sum_rates_per_step.append(sum_rate)\n", 710 | " total_sum_rate += sum_rate\n", 711 | " counter+=1\n", 712 | " #Step UEs \n", 713 | " env.render() \n", 714 | " \n", 715 | "\n", 716 | " #Plot resource errors by step and interation\n", 717 | " plot_step_graph(iter_num,counter,step_count,sum_rates_per_step)\n", 718 | " \n", 719 | " print(\"Testing...\")\n", 720 | " print(\"Counter: \",counter)\n", 721 | " print(\"Iteration Number: \",iter_num)\n", 722 | " \n", 723 | " print(total_sum_rate/step_count)\n", 724 | " if(not isCollectStep):\n", 725 | " score = total_sum_rate/step_count\n", 726 | " other_score = sum_rates_per_step[-1]\n", 727 | " sum_rates_per_iteration.append(score)\n", 728 | " if(score > max_score):\n", 729 | " env.save(env_name)\n", 730 | " max_score = score \n", 731 | " if(other_score > max_other_score):\n", 732 | " env.save(env_name+\"_bestend\")\n", 733 | " max_other_score = other_score \n", 734 | " return sum_rates_per_iteration, sum_rates_per_step" 735 | ] 736 | }, 737 | { 738 | "cell_type": "markdown", 739 | "metadata": { 740 | "id": "a_gFJRd3Ddho" 741 | }, 742 | "source": [ 743 | "# Extended Environment and Agent Classes" 744 | ] 745 | }, 746 | { 747 | "cell_type": "code", 748 | "execution_count": null, 749 | "metadata": { 750 | "id": "S4JEO27Fa8AQ" 751 | }, 752 | "outputs": [], 753 | "source": [ 754 | "class ThreeDOFUavSimu(UavSimuEnv):\n", 755 | " coord_count = 3\n", 756 | " actions = [(1,0,0),(-1,0,0),(0,0,0),(0,1,0),(0,-1,0),(0,0,-1),(0,0,1)] #Actionlar" 757 | ] 758 | }, 759 | { 760 | "cell_type": "code", 761 | "execution_count": null, 762 | "metadata": {}, 763 | "outputs": [], 764 | "source": [ 765 | "class IndependentAISimuEnv(UavSimuEnv):\n", 766 | " \n", 767 | " def step(self,uav_index,action_index):\n", 768 | " \n", 769 | " #Get current and new locations of uav\n", 770 | " current_loc = self.getUAV(uav_index).current_location\n", 771 | " new_location = [current_loc[i] + self.actions[action_index][i]\n", 772 | " for i in range(3)]\n", 773 | " \n", 774 | " #Calculate Reward\n", 775 | " reward = 0\n", 776 | " done = False\n", 777 | " if(not self.isInside(new_location) or self.isCollides(uav_index,new_location)):\n", 778 | " reward = PENALTY\n", 779 | " done = True\n", 780 | " else:\n", 781 | " old_sum = self.calculate_sum_rate()\n", 782 | " self.getUAV(uav_index).current_location = new_location\n", 783 | " new_sum = self.calculate_sum_rate()\n", 784 | " reward = new_sum - old_sum\n", 785 | " \n", 786 | " #If uav is the last one then increase step\n", 787 | " if(uav_index==self.uav_count-1):\n", 788 | " self.initial_time+=1\n", 789 | " \n", 790 | " #Get next state (Next uav's next input = next observation of next agent)\n", 791 | " next_obs = self.get_state_input(uav_index)\n", 792 | " done = done or self.initial_time == self.step_count\n", 793 | " return reward,next_obs,done \n", 794 | " \n", 795 | " def train_agents(self):\n", 796 | " for uav_index in range(self.uav_count):\n", 797 | " self.getUAV(uav_index).replay() \n", 798 | " print(\"\\nMemory: \\n \",self.getUAV(uav_index).memory[-1])\n", 799 | " print(\"\\nEpsilon: \",self.getUAV(uav_index).epsilon,\"\\n\")\n", 800 | " if(self.initial_time==self.step_count-1):\n", 801 | " self.getUAV(uav_index).adaptiveEGreedy() \n", 802 | " \n", 803 | " def step_UAVs(self,save_reward=True,isTest=False,isCollectStep=False): \n", 804 | " \n", 805 | " #For each uav, get (observation,action,reward,next_state)\n", 806 | " for uav_index in range(self.uav_count):\n", 807 | " \n", 808 | " observation = self.get_state_input(uav_index) \n", 809 | " \n", 810 | " #Get action index of uav\n", 811 | " action = self.getUAV(uav_index).act(observation, cannot_random=isTest)\n", 812 | " \n", 813 | " #step env\n", 814 | " reward,next_obs,done = self.step(uav_index,action)\n", 815 | " \n", 816 | " #If training, agent stores\n", 817 | " if(not isTest):\n", 818 | " self.getUAV(uav_index).store(observation,action,reward,next_obs,done) \n", 819 | " \n", 820 | " #After a step finishes, agent replays\n", 821 | " if(not isTest and not isCollectStep):\n", 822 | " self.train_agents()" 823 | ] 824 | }, 825 | { 826 | "cell_type": "code", 827 | "execution_count": null, 828 | "metadata": {}, 829 | "outputs": [], 830 | "source": [ 831 | "from keras import backend as K\n", 832 | "from keras.losses import Huber\n", 833 | "import tensorflow as tf\n", 834 | "\n", 835 | "class DDQNAgent(UAV):\n", 836 | " \n", 837 | " def update_target_model(self):\n", 838 | " # copy weights from model to target_model\n", 839 | " weights = self.model.get_weights()\n", 840 | " target_weights = self.target_model.get_weights()\n", 841 | " for i in range(len(target_weights)):\n", 842 | " target_weights[i] = weights[i]\n", 843 | " self.target_model.set_weights(target_weights)\n", 844 | " \n", 845 | " def __init__(self,*args,**kwargs): \n", 846 | " super().__init__(*args,**kwargs)\n", 847 | " self.target_model = tf.keras.models.clone_model(self.model)\n", 848 | " \n", 849 | "\n", 850 | "class DDQNAgentDiff(DDQNAgent): \n", 851 | " \n", 852 | " def replay_with_indexes(self,indexes,next_agent_target):\n", 853 | " minibatch = [self.memory[index] for index in indexes]\n", 854 | " self.replay_on_batch(minibatch,next_agent_target)\n", 855 | " \n", 856 | " def replay_on_batch(self,minibatch,next_agent_target):\n", 857 | " if(not self.is_memory_enough() ):\n", 858 | " return\n", 859 | " for state,action, reward,next_state, done in minibatch: \n", 860 | " \n", 861 | " if done:\n", 862 | " target = reward\n", 863 | " else: \n", 864 | " target = reward + self.gamma*np.amax(next_agent_target.predict(next_state)) \n", 865 | " train_target = self.model.predict(state)\n", 866 | " train_target[0][action] = target\n", 867 | " \n", 868 | " self.model.fit(state,train_target,verbose=1,use_multiprocessing=True)\n", 869 | " self.update_target_model()\n", 870 | " \n", 871 | " def replay(self,next_agent_target):\n", 872 | " \n", 873 | " if(not self.is_memory_enough() ):\n", 874 | " return\n", 875 | " minibatch = rand.sample(self.memory,self.batch_size)\n", 876 | " self.replay_on_batch(minibatch,next_agent_target)\n", 877 | " " 878 | ] 879 | }, 880 | { 881 | "cell_type": "code", 882 | "execution_count": null, 883 | "metadata": {}, 884 | "outputs": [], 885 | "source": [ 886 | "from keras.losses import Huber\n", 887 | "from keras.optimizers import RMSprop\n", 888 | " \n", 889 | "class UavSimuEnvWithComm(UavSimuEnv):\n", 890 | " \n", 891 | " def is_collect_step(self):\n", 892 | " return self.getUAV(0).is_memory_enough() \n", 893 | " \n", 894 | " def get_current_epsilon(self):\n", 895 | " return self.getUAV(0).epsilon\n", 896 | " \n", 897 | " def train_agents(self):\n", 898 | " for uav_index in range(self.uav_count):\n", 899 | " next_uav_index = (uav_index+1)%(self.uav_count)\n", 900 | " next_uav_policy = self.getUAV(next_uav_index).model\n", 901 | " self.getUAV(uav_index).replay(next_uav_policy) \n", 902 | "\n", 903 | " print(\"\\nMemory: \\n \",self.getUAV(uav_index).memory[-1])\n", 904 | " print(\"\\nEpsilon: \",self.getUAV(uav_index).epsilon,\"\\n\")\n", 905 | "\n", 906 | " if(self.initial_time==self.step_count-1):\n", 907 | " self.getUAV(uav_index).adaptiveEGreedy() \n", 908 | " \n", 909 | " def isCollides(self,uav_index,new_location): \n", 910 | " for index,uav in enumerate(self.map[\"uav_set\"]): \n", 911 | " if(index>uav_index):\n", 912 | " return False\n", 913 | " elif(index!=uav_index and np.array_equal(uav.current_location, new_location)):\n", 914 | " return True\n", 915 | " return False\n", 916 | "\n", 917 | " def get_state_input(self,uav_index):\n", 918 | " all_uav_coordinates = []\n", 919 | "\n", 920 | " all_uav_coordinates.extend(self.getUAV(uav_index).current_location[:self.coord_count])\n", 921 | " for index,uav in enumerate(self.map[\"uav_set\"]):\n", 922 | " if(uav_index==index):\n", 923 | " continue \n", 924 | " all_uav_coordinates.extend(uav.current_location[:self.coord_count]) \n", 925 | " \n", 926 | " time_range = [0 for _ in range(math.ceil(self.step_count/UAV_MOVEMENT_SCALE))]\n", 927 | " time_range[self.initial_time // UAV_MOVEMENT_SCALE] = 1\n", 928 | " \n", 929 | " return self.reshape(all_uav_coordinates +time_range)\n", 930 | "\n", 931 | " def step(self,uav_index,action_index):\n", 932 | " \n", 933 | " #Get current and new locations of uav\n", 934 | " current_loc = self.getUAV(uav_index).current_location\n", 935 | " new_location = [current_loc[i] + self.actions[action_index][i]\n", 936 | " for i in range(3)]\n", 937 | " \n", 938 | " #Calculate Reward\n", 939 | " reward = 0\n", 940 | " done = False\n", 941 | " if(not self.isInside(new_location) or self.isCollides(uav_index,new_location)):\n", 942 | " reward = PENALTY\n", 943 | " done = True\n", 944 | " else:\n", 945 | " old_sum = self.calculate_sum_rate()\n", 946 | " self.getUAV(uav_index).current_location = new_location\n", 947 | " new_sum = self.calculate_sum_rate()\n", 948 | " reward = new_sum - old_sum\n", 949 | " \n", 950 | " #Get next uav's index\n", 951 | " next_uav_index = (uav_index+1)%(self.uav_count)\n", 952 | " \n", 953 | " #If uav is the last one then increase step\n", 954 | " if(uav_index==self.uav_count-1):\n", 955 | " self.initial_time+=1\n", 956 | " \n", 957 | " #Get next state (Next uav's next input = next observation of next agent)\n", 958 | " next_obs = self.get_state_input(next_uav_index)\n", 959 | " done = done or self.initial_time == self.step_count\n", 960 | " return reward,next_obs,done \n", 961 | "\n", 962 | " def step_UAVs(self,save_reward=True,isTest=False,isCollectStep=False): \n", 963 | " \n", 964 | " \n", 965 | " next_obs = self.get_state_input(0)\n", 966 | " #For each uav, get (observation,action,reward,next_state)\n", 967 | " for uav_index in range(self.uav_count):\n", 968 | " \n", 969 | " observation = next_obs.copy()\n", 970 | " \n", 971 | " #Get action index of uav\n", 972 | " action = self.getUAV(uav_index).act(observation, cannot_random=isTest)\n", 973 | " \n", 974 | " #step env\n", 975 | " reward,next_obs,done = self.step(uav_index,action)\n", 976 | " \n", 977 | " #If training, agent stores\n", 978 | " if(not isTest):\n", 979 | " self.getUAV(uav_index).store(observation,action,reward,next_obs,done) \n", 980 | " \n", 981 | " #After a step finishes, agent replays\n", 982 | " if(not isTest and not isCollectStep):\n", 983 | " self.train_agents()\n", 984 | " \n", 985 | " def __init__(self,uav_paths,ue_set,env_dim = (100,100),batch_size=200,max_memory_per_agent=10000,uav_class=UAV):\n", 986 | " super().__init__(uav_paths,ue_set,env_dim = (100,100),batch_size=batch_size,max_memory_per_agent=max_memory_per_agent,uav_class=uav_class)\n", 987 | " self.map[\"uav_set\"] = self.init_uav(uav_paths,batch_size,max_memory_per_agent,uav_class)\n", 988 | " \n", 989 | "\n", 990 | "class UavSimuEnvWithCommSameReplayIndexesReversedReplay(UavSimuEnvWithComm): \n", 991 | " \n", 992 | " def train_agents(self):\n", 993 | " indexes = rand.sample(range(len(self.getUAV(0).memory)), self.getUAV(0).batch_size)\n", 994 | " for uav_index in reversed(range(self.uav_count)):\n", 995 | " \n", 996 | " next_uav_index = (uav_index+1)%(self.uav_count)\n", 997 | " next_uav_policy = self.getUAV(next_uav_index).model\n", 998 | " self.getUAV(uav_index).replay_with_indexes(indexes,next_uav_policy) \n", 999 | " \n", 1000 | " print(\"\\nMemory: \\n \",self.getUAV(uav_index).memory[-1])\n", 1001 | " print(\"\\nEpsilon: \",self.getUAV(uav_index).epsilon,\"\\n\")\n", 1002 | " \n", 1003 | " if(self.initial_time==self.step_count-1):\n", 1004 | " self.getUAV(uav_index).adaptiveEGreedy() \n" 1005 | ] 1006 | }, 1007 | { 1008 | "cell_type": "code", 1009 | "execution_count": null, 1010 | "metadata": {}, 1011 | "outputs": [], 1012 | "source": [ 1013 | "from keras.layers import AveragePooling2D,Flatten, Conv2D, Input, concatenate\n", 1014 | "from random import randint\n", 1015 | "from keras import Model\n", 1016 | "class semiConvAgent(DDQNAgent):\n", 1017 | " \n", 1018 | " def build_model(self):\n", 1019 | " inputA = Input(shape=(self.input_size[0],))\n", 1020 | " inputB = Input(shape=list(self.input_size[1])+[1])\n", 1021 | " \n", 1022 | " uav_coord_input = Dense(64, activation=\"relu\")(inputA)\n", 1023 | " uav_coord_input = Model(inputs=inputA, outputs=uav_coord_input)\n", 1024 | " \n", 1025 | " ue_coord_input = AveragePooling2D(pool_size=(4, 4),padding=\"same\")(inputB)\n", 1026 | " ue_coord_input = Flatten()(ue_coord_input)\n", 1027 | " ue_coord_input = Model(inputs=inputB, outputs=ue_coord_input)\n", 1028 | "\n", 1029 | " combined = concatenate([uav_coord_input.output, ue_coord_input.output])\n", 1030 | " \n", 1031 | " z = Dense(512, activation=\"relu\")(combined)\n", 1032 | " z = Dense(512, activation=\"relu\")(z)\n", 1033 | " z = Dense(256, activation=\"relu\")(z)\n", 1034 | " z = Dense(self.action_count,activation=\"linear\")(z)\n", 1035 | " \n", 1036 | " model = Model(inputs=[uav_coord_input.input, ue_coord_input.input], outputs=z)\n", 1037 | " model.compile(Adam(learning_rate = self.lr ,clipnorm=1.0),loss=Huber())\n", 1038 | " print(model.summary())\n", 1039 | " return model \n", 1040 | " \n", 1041 | "class semiConvAgentCooperative(DDQNAgentDiff,semiConvAgent):\n", 1042 | " \n", 1043 | " def build_model(self):\n", 1044 | " return semiConvAgent.build_model(self)\n", 1045 | " \n", 1046 | "class UserScalableEnv(IndependentAISimuEnv):\n", 1047 | " def __init__(self,*args,**kwargs):\n", 1048 | " super().__init__(*args,**kwargs)\n", 1049 | " \n", 1050 | " def get_input_size(self,uav_count):\n", 1051 | " return (uav_count*self.coord_count,self.env_dim)\n", 1052 | " \n", 1053 | " #Point process Distribution\n", 1054 | " def create_random_user_locations(self):\n", 1055 | " total_intensity = self.env_dim[0]*self.env_dim[1]*(np.exp(1)-1) / 5\n", 1056 | " max_intensity = np.exp(1)/5\n", 1057 | "\n", 1058 | " number_points = np.random.poisson(self.env_dim[0] * self.env_dim[1] * max_intensity)\n", 1059 | " ue_coords = []\n", 1060 | " ue_count = len(self.map[\"ue_set\"])\n", 1061 | " \n", 1062 | " while(len(ue_coords)= np.random.uniform(0, max_intensity):\n", 1067 | " ue_coords.append([x,y,0])\n", 1068 | " return ue_coords\n", 1069 | " \n", 1070 | " \n", 1071 | " def reset(self):\n", 1072 | " def reset_set(key):\n", 1073 | " for index, val in enumerate(self.map[key]):\n", 1074 | " self.map[key][index].reset()\n", 1075 | " \n", 1076 | " ue_set = []\n", 1077 | " \n", 1078 | " if(self.test_mode==True):\n", 1079 | " with open(TEST_ENV_FILE, 'rb') as data: \n", 1080 | " ue_set = pickle.load(data)\n", 1081 | " else:\n", 1082 | " random_locs = self.create_random_user_locations()\n", 1083 | " for coord in random_locs:\n", 1084 | " ue_set.append(UE(coord,env_dim,UavSimuEnv.actions,define_path_first=True,step_count=self.step_count//UAV_MOVEMENT_SCALE))\n", 1085 | " \n", 1086 | " self.map[\"ue_set\"] = ue_set\n", 1087 | " reset_set(\"uav_set\")\n", 1088 | " self.initial_time = 0 \n", 1089 | "\n", 1090 | " def get_state_input(self,uav_index):\n", 1091 | " all_uav_coordinates = []\n", 1092 | "\n", 1093 | " for uav in self.map[\"uav_set\"]:\n", 1094 | " all_uav_coordinates.extend(uav.current_location[:self.coord_count]) \n", 1095 | "\n", 1096 | " user_map = np.zeros((env_dim[0],env_dim[1]))\n", 1097 | " for user in self.map[\"ue_set\"]:\n", 1098 | " row = user.current_location[0]\n", 1099 | " col = user.current_location[1]\n", 1100 | " user_map[row][col] = 1\n", 1101 | " \n", 1102 | " return [self.reshape(np.array(all_uav_coordinates)),self.reshape(user_map)]\n", 1103 | " \n", 1104 | " def reshape(self, data):\n", 1105 | " return np.reshape(data, (1,) + data.shape) \n", 1106 | " \n", 1107 | "class UserScalableEnvWithTimeInfo(UserScalableEnv):\n", 1108 | " \n", 1109 | " def reshape(self, data):\n", 1110 | " return np.reshape(data, (1,) + data.shape) \n", 1111 | " \n", 1112 | " def get_input_size(self,uav_count):\n", 1113 | " return (uav_count*self.coord_count+1,self.env_dim)\n", 1114 | " \n", 1115 | " def get_state_input(self,uav_index):\n", 1116 | " all_uav_coordinates = []\n", 1117 | "\n", 1118 | " for uav in self.map[\"uav_set\"]:\n", 1119 | " all_uav_coordinates.extend(uav.current_location[:self.coord_count]) \n", 1120 | "\n", 1121 | " user_map = np.zeros((env_dim[0],env_dim[1]))\n", 1122 | " for user in self.map[\"ue_set\"]:\n", 1123 | " row = user.current_location[0]\n", 1124 | " col = user.current_location[1]\n", 1125 | " user_map[row][col] = 1\n", 1126 | " \n", 1127 | " return [self.reshape(np.array(all_uav_coordinates + [self.step_count-self.initial_time])),self.reshape(user_map)]\n", 1128 | " \n", 1129 | "\n", 1130 | "class UserScalableEnvWithTimeInfowithComm(UavSimuEnvWithCommSameReplayIndexesReversedReplay,UserScalableEnvWithTimeInfo):\n", 1131 | " \n", 1132 | " def get_input_size(self,uav_count):\n", 1133 | " return UserScalableEnvWithTimeInfo.get_input_size(self,uav_count)\n", 1134 | " \n", 1135 | " def get_state_input(self,uav_index):\n", 1136 | " return UserScalableEnvWithTimeInfo.get_state_input(self,uav_index)\n", 1137 | " \n", 1138 | " def create_random_user_locations(self):\n", 1139 | " return UserScalableEnvWithTimeInfo.create_random_user_locations(self)\n", 1140 | " \n", 1141 | " def reset(self):\n", 1142 | " UserScalableEnvWithTimeInfo.reset(self)\n", 1143 | " \n", 1144 | " def reshape(self, data):\n", 1145 | " return UserScalableEnvWithTimeInfo.reshape(self,data)\n", 1146 | " " 1147 | ] 1148 | }, 1149 | { 1150 | "cell_type": "code", 1151 | "execution_count": null, 1152 | "metadata": {}, 1153 | "outputs": [], 1154 | "source": [ 1155 | "class UserScalableEnvWithTimeInfowithCommFix(UserScalableEnvWithTimeInfowithComm):\n", 1156 | "\n", 1157 | " def get_state_input(self,uav_index):\n", 1158 | " all_uav_coordinates = []\n", 1159 | "\n", 1160 | " for uav in self.map[\"uav_set\"]:\n", 1161 | " all_uav_coordinates.extend(uav.current_location[:self.coord_count]) \n", 1162 | "\n", 1163 | " user_map = np.zeros((env_dim[0],env_dim[1]))\n", 1164 | " for user in self.map[\"ue_set\"]:\n", 1165 | " row = user.current_location[0]%self.env_dim[0]\n", 1166 | " col = user.current_location[1]%self.env_dim[1]\n", 1167 | " user_map[row][col] = 1\n", 1168 | " \n", 1169 | " return [self.reshape(np.array(all_uav_coordinates + [self.initial_time%20])),self.reshape(user_map)]\n", 1170 | " \n", 1171 | " def step(self,uav_index,action_index):\n", 1172 | " \n", 1173 | " #Get current and new locations of uav\n", 1174 | " current_loc = self.getUAV(uav_index).current_location\n", 1175 | " new_location = [current_loc[i] + self.actions[action_index][i]\n", 1176 | " for i in range(3)]\n", 1177 | " \n", 1178 | " #Calculate Reward\n", 1179 | " reward = 0\n", 1180 | " done = False\n", 1181 | " if(not self.isInside(new_location) or self.isCollides(uav_index,new_location)):\n", 1182 | " reward = PENALTY\n", 1183 | " done = True\n", 1184 | " else:\n", 1185 | " old_sum = self.calculate_sum_rate()\n", 1186 | " self.getUAV(uav_index).current_location = new_location\n", 1187 | " new_sum = self.calculate_sum_rate()\n", 1188 | " reward = new_sum - old_sum\n", 1189 | " \n", 1190 | " #Get next uav's index\n", 1191 | " next_uav_index = (uav_index+1)%(self.uav_count)\n", 1192 | " \n", 1193 | " #If uav is the last one then increase step\n", 1194 | " if(uav_index==self.uav_count-1):\n", 1195 | " self.initial_time+=1\n", 1196 | " \n", 1197 | " if(self.initial_time%UAV_MOVEMENT_SCALE==0):\n", 1198 | " self.step_UEs()\n", 1199 | " \n", 1200 | " #Get next state (Next uav's next input = next observation of next agent)\n", 1201 | " next_obs = self.get_state_input(next_uav_index)\n", 1202 | " done = done or self.initial_time == self.step_count\n", 1203 | " return reward,next_obs,done " 1204 | ] 1205 | }, 1206 | { 1207 | "source": [ 1208 | "# Log Function" 1209 | ], 1210 | "cell_type": "markdown", 1211 | "metadata": {} 1212 | }, 1213 | { 1214 | "cell_type": "code", 1215 | "execution_count": null, 1216 | "metadata": { 1217 | "colab": { 1218 | "base_uri": "https://localhost:8080/" 1219 | }, 1220 | "id": "hPTl9P2_oEGM", 1221 | "outputId": "6baf0300-2a0e-432e-b9a8-d1b333adb94f" 1222 | }, 1223 | "outputs": [], 1224 | "source": [ 1225 | "import pickle\n", 1226 | "import json\n", 1227 | "import os\n", 1228 | "\n", 1229 | "def get_name(*args,**kwargs):\n", 1230 | " return \"{}_{}_{}LR_{}batch__step{}_iter{}_epsilondecay{}_epsilonmin{}_gamma{}\".format(kwargs[\"agent_class\"],\n", 1231 | " kwargs[\"env\"],\n", 1232 | " kwargs[\"lr\"],\n", 1233 | " kwargs[\"batch_size\"],\n", 1234 | " kwargs[\"step_count\"],\n", 1235 | " kwargs[\"iteration_count\"],\n", 1236 | " kwargs[\"epsilon_decay\"],\n", 1237 | " kwargs[\"epsilon_min\"],\n", 1238 | " kwargs[\"gamma\"])\n", 1239 | "def writeLog(*args, **kwargs):\n", 1240 | " \n", 1241 | " name = get_name(*args,**kwargs)\n", 1242 | " with open(\"Logs/\"+name+\".json\", 'w') as f:\n", 1243 | " json.dump(kwargs, f)\n", 1244 | " f.write(os.linesep)\n", 1245 | " return name" 1246 | ] 1247 | }, 1248 | { 1249 | "source": [ 1250 | "# Determine Initial UAV locations" 1251 | ], 1252 | "cell_type": "markdown", 1253 | "metadata": {} 1254 | }, 1255 | { 1256 | "cell_type": "code", 1257 | "execution_count": null, 1258 | "metadata": { 1259 | "id": "JSb3aEfKFPxM" 1260 | }, 1261 | "outputs": [], 1262 | "source": [ 1263 | "import pickle\n", 1264 | "\n", 1265 | "env_dim = (100,100)\n", 1266 | "STEP_COUNT= 64\n", 1267 | "\n", 1268 | "alt = ALTITUDE\n", 1269 | "mid_height, mid_width = env_dim[1]/2, env_dim[0]/2\n", 1270 | "\n", 1271 | "\n", 1272 | "uav_paths = [ \n", 1273 | " [mid_width, mid_height,alt], \\\n", 1274 | " [mid_width, mid_height,alt-1], \\\n", 1275 | " [mid_width, mid_height,alt-2], \n", 1276 | " ]\n", 1277 | "ue_set = []\n", 1278 | "\n", 1279 | "UNIT = 20\n", 1280 | "tmp = UavSimuEnv(uav_paths,ue_set,env_dim)\n", 1281 | "tmp.render() " 1282 | ] 1283 | }, 1284 | { 1285 | "source": [ 1286 | "# Parameter Tuning Loop" 1287 | ], 1288 | "cell_type": "markdown", 1289 | "metadata": {} 1290 | }, 1291 | { 1292 | "cell_type": "code", 1293 | "execution_count": null, 1294 | "metadata": { 1295 | "colab": { 1296 | "base_uri": "https://localhost:8080/", 1297 | "height": 1000 1298 | }, 1299 | "id": "rHRmrGIVcX-L", 1300 | "outputId": "6c3c4c02-a939-4de0-8175-fcf1f71eb475", 1301 | "scrolled": true 1302 | }, 1303 | "outputs": [], 1304 | "source": [ 1305 | "import itertools\n", 1306 | "\n", 1307 | "EPSILON_DECAY = 0.962\n", 1308 | "\n", 1309 | "learning_rates = [1e-6]\n", 1310 | "agents = [semiConvAgentCooperative]\n", 1311 | "Envs=[UserScalableEnvWithTimeInfowithCommFix]\n", 1312 | "ITERATION_COUNT = 100\n", 1313 | "STEP_COUNT= 64\n", 1314 | "COLLECT_STEP_COUNT = STEP_COUNT\n", 1315 | "batch_sizes = [10]\n", 1316 | "UAV_MOVEMENT_SCALE = 20\n", 1317 | "UNIT =20\n", 1318 | "PENALTY = -100\n", 1319 | "EPSILON_MIN = 0.1\n", 1320 | "GAMMA=0.95 # Gamma daha da azaltırsa daha iyi sonuçlar çıkabilir\n", 1321 | "\n", 1322 | "combinations = itertools.product(learning_rates,Envs, \n", 1323 | " batch_sizes,agents) \n", 1324 | "for comb in combinations:\n", 1325 | " print(comb) \n", 1326 | " LR = comb[0] \n", 1327 | " env = comb[1]\n", 1328 | " batch_size = comb[2]\n", 1329 | " agent = comb[3]\n", 1330 | " \n", 1331 | " name = get_name(lr=LR,env=env.__name__,env_dim=env_dim, \n", 1332 | " step_count=STEP_COUNT, iteration_count =ITERATION_COUNT, \n", 1333 | " uav_movement_scale=UAV_MOVEMENT_SCALE, batch_size=batch_size,\n", 1334 | " agent_class=agent.__name__, collect_step_size=COLLECT_STEP_COUNT, \n", 1335 | " epsilon_decay=EPSILON_DECAY, epsilon_min=EPSILON_MIN,\n", 1336 | " gamma=GAMMA)\n", 1337 | " \n", 1338 | " environment = env(uav_paths, ue_set, \n", 1339 | " env_dim, batch_size=batch_size, uav_class=agent)\n", 1340 | "\n", 1341 | " per_iter,per_step = simulate(environment,iteration_count=ITERATION_COUNT, \n", 1342 | " step_count=STEP_COUNT, batch_size = batch_size, \n", 1343 | " collect_step_size=COLLECT_STEP_COUNT, env_name = name)\n", 1344 | "\n", 1345 | " environment.save(name+\"_endofsim\")\n", 1346 | "\n", 1347 | " writeLog(lr=LR, env=env.__name__, per_iter=per_iter, per_step=per_step, env_dim=env_dim, \n", 1348 | " step_count=STEP_COUNT, iteration_count =ITERATION_COUNT, uav_movement_scale=UAV_MOVEMENT_SCALE,\n", 1349 | " LastMap = environment.get3DMap(), batch_size=batch_size, agent_class=agent.__name__,\n", 1350 | " collect_step_size=COLLECT_STEP_COUNT, epsilon_decay=EPSILON_DECAY, epsilon_min=EPSILON_MIN, gamma=GAMMA) " 1351 | ] 1352 | } 1353 | ], 1354 | "metadata": { 1355 | "colab": { 1356 | "collapsed_sections": [], 1357 | "name": "Bitirme.ipynb", 1358 | "provenance": [] 1359 | }, 1360 | "kernelspec": { 1361 | "name": "python3", 1362 | "display_name": "Python 3.8.10 64-bit" 1363 | }, 1364 | "language_info": { 1365 | "codemirror_mode": { 1366 | "name": "ipython", 1367 | "version": 3 1368 | }, 1369 | "file_extension": ".py", 1370 | "mimetype": "text/x-python", 1371 | "name": "python", 1372 | "nbconvert_exporter": "python", 1373 | "pygments_lexer": "ipython3", 1374 | "version": "3.8.10" 1375 | }, 1376 | "interpreter": { 1377 | "hash": "38740d3277777e2cd7c6c2cc9d8addf5118fdf3f82b1b39231fd12aeac8aee8b" 1378 | } 1379 | }, 1380 | "nbformat": 4, 1381 | "nbformat_minor": 1 1382 | } -------------------------------------------------------------------------------- /User Data/test_env.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/User Data/test_env.pkl -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/__init__.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DevMilk/UAV-Based-Cellular-Communication-Multi-Agent-DRL-Solution/9995aaafd95a9b8cdc5a2cc19b284654c4b5181a/requirements.txt --------------------------------------------------------------------------------