├── README.md ├── bash_run_seeds.sh ├── channel.py ├── coverage_plot.ipynb ├── gradient.py ├── main.py ├── main_test.py ├── mobile_env.py ├── sinr_visualisation.py ├── ue_mobility.py └── ue_trace_10k.npy /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | Simulation scripts for the mobility management of UAV base stations project mainly built for paper https://dl.acm.org/citation.cfm?id=3308964. 3 | 4 | # Requirements 5 | * python2.7 6 | * numpy==1.16.2 7 | * tensorflow 8 | * IPython 9 | * matplotlib 10 | 11 | # Files 12 | * main.py 13 | - main simulation script with A3C (V. Mnih et al. 2016. Asynchronous methods for deep reinforcement learning. In ICML. 1928–1937.) implementation 14 | - multi-threading to initialise parallel training of multiple workers in parallel spaces (MobiEnvironments) 15 | - each worker creates a MobiEnvironment instance and starts training in this environment 16 | - there is a pair of global AC nets and local AC nets for worker. Workers train their own nets individually while push the gradients to the global nets periodically, then the global nets optimise uploaded gradients from all workers and distribute the same optimal gradients to all workers. 17 | - choices of CNN and MLP are implimented. Default MLP nets perform as well as CNN in prior work with less training complexity 18 | 19 | 20 | * main_test.py 21 | - load trained model to test (taking input AC model from ./train/Global_A_PARA%.npy where % can be the training step, 2000 by default) 22 | - test is done on controlled UE mobility trace by loading a file ./ue_trace_10k.npy 23 | - at each test step, the output of nn is argmax-ed to make control decisions of UAV movements 24 | - per step reward, SINR, and computation time are recorded for performance evaluation (output to ./test) 25 | 26 | 27 | * mobile_env.py 28 | - followed openAI's gym implementation structure for a wireless mobile environment 29 | - creates a LTE wireless channel which provides computation of SINR values and handover functionality 30 | - step() and step_test() take action from the RL agent and returns updated state, reward, and customisable information to the RL agent. Please be careful here to make the two function consistant. It is not ideal to have two functions one for training and one for testing, but the reason to do this is to enable different user mobility models while keep both training and testing steps computationally cheap (rather than switching between if conditions per step) 31 | - during training the user moves following the group reference model 32 | - during testing the users move using preloaded trace (ue_trace_10k.npy), which is generated from the group reference model 33 | - reward function currently consists of a reward on mean sinr value and a penalty on number of outaged users. which is open for improvement 34 | 35 | * channel.py 36 | - downlink and uplink SINR 37 | - In the WAIN work we take only downlink sinr 38 | 39 | * ue_mobility.py 40 | - a couple of mobility models for UE's movement 41 | - group reference (X. Hong et al. 1999. A group mobility model for ad hoc wireless networks. In ACM MSWiM. 53–60.) model is used in the WAIN paper. please check the WAIN paper for more details 42 | 43 | * sinr_visualisation.py 44 | - utility functions for visualisation during the simulations 45 | 46 | # Build virtual environment 47 | ` virtualenv env ` 48 | ` source env/bin/activate ` 49 | 50 | # Run training 51 | ` mkdir train ` 52 | ` python main.py ` 53 | 54 | # Run testing 55 | ` mkdir test ` 56 | ` python main_test.py ` 57 | 58 | ### Email lirui628@gmail for any questions 59 | ### Have fun :squirrel::octocat::robot::four_leaf_clover: 60 | -------------------------------------------------------------------------------- /bash_run_seeds.sh: -------------------------------------------------------------------------------- 1 | for i in {1..10} 2 | do 3 | echo bash starting seed$i 4 | mkdir seed$i 5 | python main.py -seed=$i 6 | cp -r train seed$i/ 7 | done 8 | -------------------------------------------------------------------------------- /channel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | from IPython import display 4 | #from sinr_visualisation import * 5 | 6 | WATCH_WINDOW = 200 # used for visualisation only 7 | OUT_THRESH = 0 # DL SINR below this value is considered to be user outage 8 | 9 | class LTEChannel: 10 | """ 11 | LTE channel class including simulation of LTE downlink and uplink channels 12 | """ 13 | def __init__(self, nUE, nBS, boundaries, init_ueLoc, init_bsLoc): 14 | 15 | [self.xMin, self.xMax, self.yMin, self.yMax] = boundaries 16 | self.gridX = self.xMax - self.xMin + 1 17 | self.gridY = self.yMax - self.yMin + 1 18 | 19 | self.nUE = nUE 20 | self.nBS = nBS 21 | self.gridWidth = 5 22 | 23 | # FDD ratio 24 | self.alpha = 0.5 25 | # total number of channels 26 | self.K = 120 27 | # freq reuse factor 28 | self.r_freq = 1 29 | # numebr of channels per BS in DL 30 | self.K_DL = self.alpha * self.K / self.r_freq 31 | # numebr of channels per BS in UL 32 | self.K_UL = (1 - self.alpha) * self.K / self.r_freq 33 | # UE Tx power in dBm 34 | self.P_ue_dbm = 23 35 | # BS Tx power in dBm 36 | self.P_bs_dbm = 20 37 | # per channel tx power in DL 38 | #P_b = P_bs / K_DL 39 | # per channel Gaussian noise power in dBm 40 | self.noise_dbm = -121 41 | 42 | #path_loss = a + b*log(d) if d>= path_loss_dis in dB 43 | # self.pathloss_a = 128.1 44 | # self.pathloss_b = 37.6 45 | # self.pathloss_dis = 0.035 #in Km d>= path_loss_dis to have path loss else path loss is 0 46 | self.pathloss_a = 38 47 | self.pathloss_b = 30 48 | self.pathloss_dis = 0 49 | # Antenna Gain in dB 50 | self.antenna_gain = 2 51 | # Equipment/penetrasion loss in dB 52 | self.eq_loss = 0 53 | # shadow fading 54 | self.shadowing_mean = 0 #log normal shadowing N(mean, sd) 55 | self.shadowing_sd = 2 #6log normal shadowing N(mean, sd) 56 | 57 | self.P_ue_watt = 10 **(self.P_ue_dbm / float(10)) * 1e-3 #UE Tx power in W 58 | self.P_bs_watt = 10 **(self.P_bs_dbm/ float(10)) * 1e-3 #BS Tx power in W 59 | self.noise_watt = 10 **(self.noise_dbm / float(10)) * 1e-3 60 | 61 | self.sc_ofdm = 12 #nbr of data subcarriers/subchannel bandwidth 62 | self.sy_ofdm = 14 #nbr of ofdm symbols/subframe 63 | self.t_subframe = 1e-3 #subframe durantion in s 64 | #mapping from sinr to MCS 65 | self.sinr_thresholds = [-float('inf'), -6.5, -4, -2.6, -1, 1, 3, 6.6, 10, 11.4, 11.8, 13, 13.8, 15.6, 16.8, 17.6, float('inf')] 66 | self.sinr_thresholds_watt = [10 ** (s / float(10)) for s in self.sinr_thresholds] 67 | self.efficiency = [1e-16, 0.15, 0.23, 0.38, 0.60, 0.88, 1.18, 1.48, 1.91, 2.41, 2.73, 3.32, 3.90, 4.52, 5.12, 5.55] #bits/symbol 68 | self.rate_thresholds = [(self.sc_ofdm * self.sy_ofdm / float(self.t_subframe)) * e * 1e-6 for e in self.efficiency] #Mb/s 69 | 70 | self.total_channels = 120 71 | # dl_channels_init = (alpha * total_channels)/reuse_factor 72 | self.ul_channels_init = (1-self.alpha) * self.total_channels 73 | 74 | # UL gain average over n number of (imaginary) users from the interfering BS 75 | self.n = 1000 76 | # radius of BS coverage used to generate (imaginary) users for the interfering BS 77 | self.dth = 100 78 | #number of already associated users on each BS 79 | self.ass_per_bs = np.ones((self.nBS,1))#[1, 1, 1, 1, 1, 1] 80 | 81 | self.hoBufDepth = 3 #time to trigger HO 82 | self.hoThresh_db = 1 83 | # self.init_BS = np.zeros((self.nUE)).astype('int32') 84 | 85 | self.interfDL = [range(self.nBS) for bs in range(self.nBS)] 86 | 87 | for bs in range(self.nBS): 88 | self.interfDL[bs].remove(bs) 89 | 90 | self.interfUL = self.interfDL 91 | 92 | self.current_BS, self.current_BS_sinr = self.GetBestDlBS(init_ueLoc, init_bsLoc) #self.init_BS 93 | self.bestBS_buf = [self.current_BS] 94 | 95 | 96 | # rgbColor code for each BS 97 | self.rgbColours = [[255,0,0],[0,255,0],[0,0,255],[0,255,255],[255,255,0]] 98 | # needed for animation/plot the rgb colour code for each UE regarding to which BS it connects to 99 | self.bsColour = np.zeros((nUE,3)) 100 | 101 | # monitor UE 0 102 | self.ue2Watch = 0 103 | self.watch_ue_sinrbest = np.zeros((WATCH_WINDOW)) 104 | self.watch_ue_sinrcurr = np.zeros((WATCH_WINDOW)) 105 | 106 | #for visualising mean rate ul dl 107 | self.watch_ul_rate = np.zeros((WATCH_WINDOW)) 108 | self.watch_dl_rate = np.zeros((WATCH_WINDOW)) 109 | 110 | self.ue_out = np.where(self.current_BS_sinr<= OUT_THRESH) 111 | 112 | 113 | def reset(self, ueLoc, bsLoc): 114 | self.current_BS, self.current_BS_sinr = self.GetBestDlBS(ueLoc, bsLoc) #self.init_BS 115 | self.bestBS_buf = [self.current_BS] 116 | self.ue_out = np.where(self.current_BS_sinr<= OUT_THRESH) 117 | 118 | 119 | def GetBestDlBS(self, ueLoc, bsLoc): 120 | channelGainAll = self.GetChannelGainAll(ueLoc, bsLoc) 121 | dlSinr = self.GetDLSinrAllDb(channelGainAll) 122 | bestDlBS = np.argmax(dlSinr, axis=1) 123 | bestDlSINR = np.max(dlSinr, axis=1) 124 | return bestDlBS, bestDlSINR 125 | 126 | def SetDistanceMultiplier(self, gridWidth): 127 | self.gridWidth = gridWidth 128 | 129 | def SetTxPower (self, txpower): 130 | self.P_bs_dbm = txpower 131 | self.P_bs_watt = 10 **(self.P_bs_dbm/ float(10)) * 1e-3 132 | 133 | def SetHandoverThreshold (self, ho_thresh): 134 | self.ho_thresh = ho_thresh 135 | 136 | 137 | 138 | def UpdateDroneNet(self, ueLoc, bsLoc, ifdisplay=False, time_now=0, get_rate=False): 139 | channelGainAll = self.GetChannelGainAll(ueLoc, bsLoc) 140 | dlSinr = self.GetDLSinrAllDb(channelGainAll) 141 | bestDlBS = np.argmax(dlSinr, axis=1) 142 | bestDlSINR = np.max(dlSinr, axis=1) 143 | # print time_now, "s ue ", ueLoc[10], " dl sinr ", bestDlSINR[10], " from BS ", bestDlBS[10] 144 | 145 | for ue in xrange(self.nUE): 146 | self.current_BS_sinr[ue] = dlSinr[ue, self.current_BS[ue]] 147 | 148 | if np.shape(self.bestBS_buf)[0] < self.hoBufDepth: 149 | self.bestBS_buf = np.append(self.bestBS_buf, [bestDlBS], axis=0) 150 | else: 151 | #FIFO buffer bottom-in 152 | self.bestBS_buf[:-1] = self.bestBS_buf[1:] 153 | self.bestBS_buf[-1] = bestDlBS 154 | # print "bestBS_buf..\n", bestBS_buf 155 | bestRemain = np.all(self.bestBS_buf == self.bestBS_buf[0,:], axis = 0) 156 | bestChanged = self.current_BS != self.bestBS_buf[-1,:] 157 | 158 | ifNeedHO = np.logical_and(bestRemain, bestChanged) 159 | ifNeedHO = np.logical_and(ifNeedHO, bestDlSINR - self.current_BS_sinr > self.hoThresh_db) 160 | # print "if needHO", ifNeedHO 161 | 162 | if np.any(ifNeedHO): 163 | ueNeedHO = np.flatnonzero(ifNeedHO) 164 | for ue in ueNeedHO: 165 | fromBS = self.current_BS[ue] 166 | toBS = self.bestBS_buf[-1][ue] 167 | self.current_BS[ue] = self.bestBS_buf[-1][ue] 168 | 169 | #compute number of ue out of coverage 170 | ue_out = np.array(np.where(self.current_BS_sinr<= OUT_THRESH)) 171 | new_out = ue_out[np.isin(ue_out, self.ue_out, invert=True)] 172 | 173 | self.ue_out = ue_out 174 | n_outage = np.size(new_out) 175 | # print " ", new_out, " ", self.ue_out, " ", n_outage 176 | 177 | 178 | if get_rate or ifdisplay: 179 | #Get DL/Ul rate updates 180 | #DL 181 | dlRatePerChannel = self.GetDLRatePerChannel(dlSinr) 182 | dlRatePerChannel_from_currentBS = np.zeros((self.nUE)) 183 | #UL 184 | ulInterfPower = self.GetULInterference(bsLoc) 185 | # print "UL interference power \n", ulInterfPower 186 | ulRequiredRate = 1 187 | 188 | ulSinr = [] 189 | ulNumChannelNeeded = [] 190 | ulRateUEBS = [] 191 | 192 | for u in range(self.nUE): 193 | tup = self.GetULRateChannels (u, ulRequiredRate, ulInterfPower, channelGainAll) 194 | ulSinr.append(tup[0]) 195 | ulNumChannelNeeded.append(tup[1]) 196 | ulRateUEBS.append(tup[2]) 197 | 198 | ulSinr = np.array(ulSinr) 199 | ulNumChannelNeeded = np.array(ulNumChannelNeeded) 200 | ulRateUEBS = np.array(ulRateUEBS) 201 | 202 | dlRatePerChannel_from_currentBS = np.zeros((self.nUE)) 203 | ulRatePerChannel_from_currentBS = np.zeros((self.nUE)) 204 | for ue_id in range(self.nUE): 205 | dlRatePerChannel_from_currentBS[ue_id] = dlRatePerChannel[ue_id][self.current_BS[ue_id]] #can be accelarated 206 | ulRatePerChannel_from_currentBS[ue_id] = ulRateUEBS[ue_id][self.current_BS[ue_id]] #can be accelarated 207 | # mean rate of all UEs as received from their current BSs(maybe don't care) 208 | dl_rate_mean = np.mean(dlRatePerChannel_from_currentBS) 209 | ul_rate_mean = np.mean(ulRatePerChannel_from_currentBS) 210 | # print "mean DL and UL Rate Per Channel \n", dl_rate_mean, ul_rate_mean 211 | 212 | 213 | association_map = self.GetCurrentAssociationMap(ueLoc) 214 | 215 | # print self.current_BS_sinr, "\n mean sinr", np.mean(self.current_BS_sinr) 216 | return association_map, np.mean(self.current_BS_sinr), n_outage 217 | 218 | 219 | # compute the euclidean distance between 2 nodes 220 | def GetDistance(self, coord1, coord2): 221 | coord1 = coord1[:2]* self.gridWidth 222 | coord2 = coord2[:2]* self.gridWidth 223 | dist = np.linalg.norm(coord1-coord2) 224 | 225 | # dist = math.sqrt((coord1[0] - coord2[0])**2 + (coord1[1] - coord2[1])**2 + (coord1[2] - coord2[2])**2) 226 | return dist 227 | 228 | # compute the pass loss value for the given euclidean distance between BS b and UE i 229 | # based on urban pass loss model as per 3GPP TR 36.814 230 | def GetPassLoss(self, d): 231 | # d = d/1000#work with km 232 | loss = 0 233 | if d > self.pathloss_dis: 234 | loss = self.pathloss_a + self.pathloss_b * math.log10(d) 235 | return loss 236 | 237 | def GetChannelGain(self, coord1, coord2): 238 | d = self.GetDistance(coord1, coord2) 239 | pathLoss = self.GetPassLoss(d) 240 | fading = np.random.normal(self.shadowing_mean, self.shadowing_sd) 241 | # fading = 10 #static fading >> for calibration only!!! 242 | # print fading 243 | # the channel gain between UE and BS accounts for 244 | #1) antenna gain 2) pathloss 3) equipment loss 4) shadow fading 245 | channel_gain_db = self.antenna_gain - pathLoss - fading - self.eq_loss 246 | channel_gain = 10 ** (channel_gain_db / float(10)) 247 | return channel_gain 248 | 249 | def GetChannelGainAll(self, ueLocations, bsLocations): 250 | n_ue = np.shape(ueLocations)[0] 251 | n_bs = np.shape(bsLocations)[0] 252 | channel_gain_all = np.zeros((n_ue, n_bs)) 253 | 254 | for ue_id in range(n_ue): 255 | for bs_id in range(n_bs): 256 | channel_gain_all[ue_id][bs_id] = self.GetChannelGain(ueLocations[ue_id], bsLocations[bs_id]) 257 | return channel_gain_all 258 | 259 | def GetDLSinrAllDb(self, channel_gain_all): 260 | sinr_all = np.zeros((self.nUE,self.nBS)) 261 | for ue_id in range(self.nUE): 262 | for bs_id in range(self.nBS): 263 | interf_bs = self.interfDL[bs_id] 264 | 265 | P_interf = np.sum(self.P_bs_watt * channel_gain_all[ue_id][interf_bs]) 266 | sinr_dl = self.P_bs_watt * channel_gain_all[ue_id, bs_id] / float(self.noise_watt + P_interf) 267 | 268 | sinr_all[ue_id][bs_id] = 10 * math.log10(sinr_dl) 269 | return sinr_all 270 | 271 | #Mapping from SINR to MCS rates 272 | def GetDLRatePerChannel(self, dl_sinr_db): 273 | dl_rate_per_channel = np.zeros((self.nUE,self.nBS)) 274 | for ue_id in range(self.nUE): 275 | for bs_id in range(self.nBS): 276 | for l in range(len(self.sinr_thresholds) - 1): 277 | if (dl_sinr_db[ue_id][bs_id] >= self.sinr_thresholds[l] and dl_sinr_db[ue_id][bs_id] < self.sinr_thresholds[l+1]): 278 | dl_rate_per_channel[ue_id][bs_id] = self.rate_thresholds[l] 279 | break 280 | return dl_rate_per_channel 281 | 282 | def GetNumberDLChannelNeeded(self, requiredRate, dl_rate_per_channel): 283 | dl_needed_channels = np.zeros((self.nUE,self.nBS)) 284 | for ue_id in range(self.nUE): 285 | for bs_id in range(self.nBS): 286 | dl_needed_channels[ue_id][bs_id] = requiredRate/dl_rate_per_channel[ue_id][bs_id] 287 | return dl_needed_channels 288 | 289 | def GetAverageULChannelGainFromInterfBS(self, bs_id, intf_id, bs_loc, bs_intf_loc): 290 | channel_gain = np.zeros((self.n)) 291 | 292 | theta = np.random.uniform(0, 2*math.pi, self.n) 293 | r = np.random.uniform(0, self.dth, self.n) 294 | 295 | #imagine n users attached to bs_intf 296 | vfunc_sin = np.vectorize(math.sin) 297 | vfunc_cos = np.vectorize(math.cos) 298 | 299 | rand_users_loc = np.array([bs_intf_loc[0] + r* vfunc_sin(theta), bs_intf_loc[1] + r* vfunc_cos(theta)]) 300 | #if simulating 3D model (judging by the number of dimensions bs_loc has) 301 | if np.size(bs_loc) == 3: 302 | rand_users_loc = np.append(rand_users_loc, np.ones((1,self.n))*bs_loc[2], axis=0) 303 | 304 | rand_users_loc = np.transpose(rand_users_loc) 305 | 306 | # save the random user location for calibration 307 | # bs_id and intf_id can be removed from the function input if not printing this anymore 308 | # str_name = "rand_users_bs_" + str(bs_id) + "_intf_" + str(intf_id) 309 | # np.save(str_name, rand_users_loc) 310 | 311 | for intf_ue_id in range(self.n): 312 | channel_gain[intf_ue_id] = self.GetChannelGain(bs_loc, rand_users_loc[intf_ue_id]) 313 | 314 | return np.mean(channel_gain) 315 | 316 | 317 | def GetAverageULChannelGain(self, bs_loc): 318 | avg_channel_gain = np.zeros((self.nBS, self.nBS)) 319 | for bs_id in range(self.nBS): 320 | for intf_id in range(self.nBS): 321 | #make avg_channel_gain symmetric 322 | if (intf_id >= bs_id): 323 | if intf_id in self.interfUL[bs_id]: 324 | avg_channel_gain[bs_id][intf_id] = self.GetAverageULChannelGainFromInterfBS(bs_id, intf_id, bs_loc[bs_id], bs_loc[intf_id]) 325 | else: 326 | avg_channel_gain[bs_id][intf_id] = avg_channel_gain[intf_id][bs_id] 327 | 328 | # print "UL channel gain", avg_channel_gain 329 | return avg_channel_gain 330 | 331 | def GetULInterference(self, bs_loc): 332 | ul_interference = np.zeros((self.nBS)) 333 | ulAvgChannelGain= self.GetAverageULChannelGain(bs_loc) 334 | 335 | for bs_id in range(self.nBS): 336 | for intf_id in self.interfUL[bs_id]: 337 | ul_interference[bs_id] += self.P_ue_watt * ulAvgChannelGain[bs_id][intf_id] * self.ass_per_bs[intf_id] / self.ul_channels_init 338 | 339 | return ul_interference 340 | 341 | def GetULRateChannels (self, u, ul_datarate, ul_interference, channel_gain): 342 | 343 | """ 344 | #list of the number of needed channels on the UL from each base station to grant the user u 345 | #the data rate he asks for (when in guaranteed throughput) 346 | #returns both the number of channels needed by user u from each BS (to get the asked data rate), 347 | #and the uplink rate between the user u and each BS 348 | 349 | :param u: user index 350 | :param bs: {b0:[x_b0, y_b0], b1:[x_b1, y_b1], ...} #dictionary of BS coordinates 351 | :param ul_datarate: the value of the requested data rate on the UL 352 | :param ul_interference: uplink_interference(bs, interference_set, ass_per_bs) 353 | :param channel_gain: compute_channel_gain (users, bs) 354 | :return: ul_channels_needed, ul_rate 355 | """ 356 | 357 | ul_sinr_db = [] 358 | ul_channels_needed = [] 359 | ul_rate = [] 360 | 361 | ul_channels_min = [ul_datarate / r for r in self.rate_thresholds] 362 | 363 | for b in range(self.nBS): 364 | ul_channels_threshold = [] 365 | 366 | sinr_ratio = (self.P_ue_watt * channel_gain[u][b]) / (self.noise_watt + ul_interference[b]) #the SINR ratio without dividing by the number of channels needed, for a given user u 367 | 368 | ul_sinr_db.append(10 * math.log10(sinr_ratio)) 369 | 370 | for l in range(1, len(self.sinr_thresholds_watt) - 1): 371 | ul_channels_threshold.append(sinr_ratio/self.sinr_thresholds_watt[l]) 372 | 373 | ul_channels_threshold.insert(0, float('inf')) 374 | ul_channels_threshold.append(0) 375 | 376 | match = [] 377 | for id, val in enumerate(ul_channels_min): 378 | if val <= ul_channels_threshold[id] and val > ul_channels_threshold[id +1]: 379 | match.append(val) 380 | 381 | ul_channels_needed.append(min(match)) 382 | ul_rate.append(ul_datarate/min(match)) #assume it to be rate per channel? 383 | 384 | 385 | return ul_sinr_db, ul_channels_needed, ul_rate 386 | 387 | def GetCurrentAssociationMap(self, ueLoc): 388 | """ 389 | utility func mainly for the class mobile_env 390 | take ue locations as input, 391 | and convert current_BS into a (nBS x, GridX, GridY) heatmap for each BS 392 | NB: current_BS might not be best_bs 393 | Return the association heatmap 394 | TODO: check dim gridX_N and gridY_N 395 | """ 396 | # convert to 2D if given a 3D location 397 | if np.shape(ueLoc)[1] == 3: 398 | ueLoc = ueLoc[:,:-1] 399 | 400 | 401 | association_map = np.zeros((self.nBS, self.gridX, self.gridY)) 402 | # association_sinr_map = np.zeros((self.nBS, gridX, gridY)) 403 | 404 | for ue in range(self.nUE): 405 | bs = self.current_BS[ue] 406 | association_map[bs][ueLoc[ue][0]][ueLoc[ue][1]] += 1 407 | # association_sinr_map[bs][ueLoc[ue][0]][ueLoc[ue][1]] = self.current_BS_sinr[ue] 408 | 409 | return association_map#, association_sinr_map 410 | 411 | def GetSinrInArea (self, bsLoc): 412 | 413 | dl_sinr = np.zeros((self.gridX, self.gridY)) 414 | 415 | loc_id = 0 416 | for x_ in range(self.xMin, self.xMax): 417 | for y_ in range(self.yMin, self.yMax): 418 | dist = [] 419 | for bs in range(np.shape(bsLoc)[0]): 420 | dist.append(self.GetDistance(np.asarray([x_,y_,0]),bsLoc[bs])) 421 | 422 | bs_id = np.argmin(dist) 423 | P_interf = 0 424 | 425 | for i in self.interfDL[bs_id]: 426 | interf_gain = self.GetChannelGain(np.asarray([x_,y_,0]), bsLoc[i]) 427 | P_interf += self.P_bs_watt * interf_gain 428 | 429 | sinr_dl = self.P_bs_watt * self.GetChannelGain(np.asarray([x_,y_,0]), bsLoc[bs_id]) / float(self.noise_watt + P_interf) 430 | 431 | dl_sinr[x_][y_] = 10 * math.log10(sinr_dl) 432 | 433 | return dl_sinr 434 | 435 | -------------------------------------------------------------------------------- /gradient.py: -------------------------------------------------------------------------------- 1 | #from ue_mobility import * 2 | from mobile_env import * 3 | from copy import deepcopy 4 | # from random import randint 5 | import time 6 | from itertools import product 7 | 8 | FILE_NAME_APPEND = "" 9 | OUTPUT_DIR = "gradient/" 10 | OUTPUT_FILE_NAME = OUTPUT_DIR + "reward" + FILE_NAME_APPEND 11 | N_BS = 4 12 | 13 | 14 | def Choose_Act_Gradient(actual_env, s, n_step): 15 | virtual_env = deepcopy(actual_env) 16 | #BS remains but UE moves 17 | _, _, _, _ = virtual_env.step_test(624, False) 18 | # print np.equal(virtual_env.bsLoc, actual_env.bsLoc), virtual_env.bsLoc,"\n", actual_env.bsLoc,"\n" 19 | 20 | current_BS_sinr = virtual_env.channel.current_BS_sinr 21 | bs_loc = virtual_env.bsLoc 22 | ue_loc = virtual_env.ueLoc 23 | 24 | act_all_bs = np.zeros((len(bs_loc))) 25 | 26 | for i_bs in range(len(bs_loc)): 27 | dir_grad = np.zeros((4,1)) 28 | dir_grad[0] = np.mean(current_BS_sinr[np.where(ue_loc[:,0] > bs_loc[i_bs][0])]) 29 | dir_grad[1] = np.mean(current_BS_sinr[np.where(ue_loc[:,0] <= bs_loc[i_bs][0])]) 30 | dir_grad[2] = np.mean(current_BS_sinr[np.where(ue_loc[:,1] > bs_loc[i_bs][1])]) 31 | dir_grad[3] = np.mean(current_BS_sinr[np.where(ue_loc[:,1] <= bs_loc[i_bs][1])]) 32 | act_all_bs[i_bs] = np.nanargmin(dir_grad) 33 | 34 | action = int(act_all_bs[3] + act_all_bs[2]*5 + act_all_bs[1]*(5**2) + act_all_bs[0]*(5**3)) 35 | 36 | # print act_reward, "best action:", best_act 37 | return action 38 | 39 | def Run_Test(reward_file_name): 40 | MAX_STEP = 10000 41 | #if reading mobility trace from file 42 | test_env = MobiEnvironment(N_BS, 40, 100, "read_trace", "./ue_trace_10k.npy") 43 | 44 | s = np.array([np.ravel(test_env.reset())]) 45 | 46 | done = False 47 | step = 0 48 | 49 | outage_buf = [] 50 | reward_buf = [] 51 | sinr_all = [] 52 | n_step_forward = 1 53 | reward_file_name = reward_file_name + str(n_step_forward) 54 | start_time = time.time() 55 | single_step_time = [] 56 | while step <= MAX_STEP: 57 | before_step_time = time.time() 58 | action = Choose_Act_Gradient(test_env, s, n_step_forward) 59 | single_step_time.append(time.time() - before_step_time) 60 | 61 | s_, r, done, info = test_env.step_test(action, False) 62 | 63 | reward_buf.append(info[0]) 64 | sinr_all.append(test_env.channel.current_BS_sinr) 65 | if step % 500 == 0 or step == MAX_STEP: 66 | print "step ", step, " time ellipsed ", time.time() - start_time 67 | start_time = time.time() 68 | np.save(reward_file_name, reward_buf) 69 | np.save(OUTPUT_DIR + "time",single_step_time) 70 | np.save(OUTPUT_DIR + "sinr",sinr_all) 71 | # reset the environment every 2000 steps 72 | if step % 2000 == 0: 73 | s = np.array([np.ravel(test_env.reset())]) 74 | #warm up in 500 steps 75 | for _ in range(500): 76 | action = Choose_Act_Gradient(test_env, s, n_step_forward) 77 | _, _, _, _ = test_env.step_test(action, False) 78 | else: 79 | s = np.array([np.ravel(s_)]) 80 | 81 | step+=1 82 | 83 | np.save(reward_file_name, reward_buf) 84 | np.save(OUTPUT_DIR + "time",single_step_time) 85 | np.save(OUTPUT_DIR + "sinr",sinr_all) 86 | if __name__ == "__main__": 87 | Run_Test(OUTPUT_FILE_NAME) 88 | 89 | 90 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import threading 3 | import tensorflow as tf 4 | import numpy as np 5 | #import gym 6 | import os 7 | import shutil 8 | #import matplotlib.pyplot as plt 9 | from mobile_env import * 10 | import time 11 | 12 | #matplotlib.use('Agg') 13 | 14 | import argparse 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('-seed', type=int, default=6) 17 | args = parser.parse_args() 18 | 19 | OUTPUT_GRAPH = True 20 | LOG_DIR = './log' 21 | N_WORKERS = 4#multiprocessing.cpu_count() 22 | MAX_GLOBAL_EP = 2000 23 | GLOBAL_NET_SCOPE = 'Global_Net' 24 | UPDATE_GLOBAL_ITER = 10 25 | GAMMA = 0.9 26 | ENTROPY_BETA = 0.001 27 | LR_A = 0.0001 # learning rate for actor 28 | LR_C = 0.0001 # learning rate for critic 29 | GLOBAL_RUNNING_R = [] 30 | GLOBAL_EP = 0 31 | TENSOR_SEED= args.seed 32 | CNN_NUM_FILTERS = 10 33 | CNN_KERNEL_SIZE = 5 34 | 35 | N_BS = 4 36 | N_UE = 40 37 | AREA_W = 100 #width of the playground 38 | env = MobiEnvironment(N_BS, N_UE, AREA_W)#gym.make(GAME) 39 | #env.plot_sinr_map() 40 | 41 | N_S = env.observation_space_dim#number of state 42 | N_A = env.action_space_dim 43 | 44 | class ACNet(object): 45 | def __init__(self, scope, globalAC=None, netType='MLP'): 46 | 47 | if scope == GLOBAL_NET_SCOPE: # get global network 48 | with tf.variable_scope(scope): 49 | self.s = tf.placeholder(tf.float32, [None, N_S], 'S') 50 | if netType == 'MLP': 51 | self.a_prob, self.v, self.a_params, self.c_params = self._build_net_mlp(scope) 52 | elif netType == 'CNN': 53 | self.a_prob, self.v, self.a_params, self.c_params = self._build_net_cnn(scope) 54 | else: # local net, calculate losses 55 | with tf.variable_scope(scope): 56 | self.s = tf.placeholder(tf.float32, [None, N_S], 'S') 57 | self.a_his = tf.placeholder(tf.int32, [None, ], 'A') 58 | self.v_target = tf.placeholder(tf.float32, [None, 1], 'Vtarget') 59 | 60 | if netType == 'MLP': 61 | self.a_prob, self.v, self.a_params, self.c_params = self._build_net_mlp(scope) 62 | elif netType == 'CNN': 63 | self.a_prob, self.v, self.a_params, self.c_params = self._build_net_cnn(scope) 64 | 65 | td = tf.subtract(self.v_target, self.v, name='TD_error') 66 | with tf.name_scope('c_loss'): 67 | self.c_loss = tf.reduce_mean(tf.square(td)) 68 | 69 | with tf.name_scope('a_loss'): 70 | log_prob = tf.reduce_sum(tf.log(self.a_prob + 1e-5) * tf.one_hot(self.a_his, N_A, dtype=tf.float32), axis=1, keep_dims=True) 71 | exp_v = log_prob * tf.stop_gradient(td) 72 | entropy = -tf.reduce_sum(self.a_prob * tf.log(self.a_prob + 1e-5), 73 | axis=1, keep_dims=True) # encourage exploration 74 | self.exp_v = ENTROPY_BETA * entropy + exp_v 75 | self.a_loss = tf.reduce_mean(-self.exp_v) 76 | 77 | with tf.name_scope('local_grad'): 78 | self.a_grads = tf.gradients(self.a_loss, self.a_params) 79 | self.c_grads = tf.gradients(self.c_loss, self.c_params) 80 | 81 | with tf.name_scope('sync'): 82 | with tf.name_scope('pull'): 83 | self.pull_a_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.a_params, globalAC.a_params)] 84 | self.pull_c_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.c_params, globalAC.c_params)] 85 | with tf.name_scope('push'): 86 | self.update_a_op = OPT_A.apply_gradients(zip(self.a_grads, globalAC.a_params)) 87 | self.update_c_op = OPT_C.apply_gradients(zip(self.c_grads, globalAC.c_params)) 88 | 89 | def _build_net_cnn(self, scope): 90 | print "build CNN net" 91 | w_init = tf.random_normal_initializer(0., .1, seed=TENSOR_SEED) 92 | with tf.variable_scope('actor'): 93 | l_a = tf.layers.conv2d(tf.transpose(tf.reshape(self.s, shape=[-1, N_BS + 1, AREA_W, AREA_W]), [0, 2, 3, 1]), 94 | filters=CNN_NUM_FILTERS, 95 | kernel_size=CNN_KERNEL_SIZE, 96 | padding='valid', 97 | activation=tf.nn.relu, 98 | kernel_initializer=w_init, 99 | name='a_conv1') 100 | l_a = tf.layers.conv2d(l_a, filters=CNN_NUM_FILTERS, 101 | kernel_size=CNN_KERNEL_SIZE, 102 | padding='valid', 103 | activation=tf.nn.relu, 104 | kernel_initializer=w_init, 105 | name='a_conv2') 106 | l_a = tf.layers.conv2d(l_a, filters=CNN_NUM_FILTERS, 107 | kernel_size=CNN_KERNEL_SIZE, 108 | padding='valid', 109 | activation=tf.nn.relu, 110 | kernel_initializer=w_init, 111 | name='a_conv3') 112 | l_a = tf.contrib.layers.flatten(l_a) 113 | l_a = tf.layers.dense(l_a, 100, tf.nn.relu6, kernel_initializer=w_init, name='la2') 114 | a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap') 115 | 116 | with tf.variable_scope('critic'): 117 | l_c = tf.layers.conv2d(tf.transpose(tf.reshape(self.s, shape=[-1, N_BS + 1, AREA_W, AREA_W]), [0, 2, 3, 1]), 118 | filters=CNN_NUM_FILTERS, 119 | kernel_size=CNN_KERNEL_SIZE, 120 | padding='valid', 121 | activation=tf.nn.relu, 122 | kernel_initializer=w_init, 123 | name='c_conv1') 124 | l_c = tf.layers.conv2d(l_c, filters=CNN_NUM_FILTERS, 125 | kernel_size=CNN_KERNEL_SIZE, 126 | padding='valid', 127 | activation=tf.nn.relu, 128 | kernel_initializer=w_init, 129 | name='c_conv2') 130 | l_c = tf.layers.conv2d(l_c, filters=CNN_NUM_FILTERS, 131 | kernel_size=CNN_KERNEL_SIZE, 132 | padding='valid', 133 | activation=tf.nn.relu, 134 | kernel_initializer=w_init, 135 | name='c_conv3') 136 | l_c = tf.contrib.layers.flatten(l_c) 137 | l_c = tf.layers.dense(l_c, 100, tf.nn.relu6, kernel_initializer=w_init, name='lc2') 138 | v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v') # state value 139 | a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor') 140 | c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic') 141 | return a_prob, v, a_params, c_params 142 | 143 | 144 | def _build_net_mlp(self, scope): 145 | print "build MLP net" 146 | w_init = tf.random_normal_initializer(0., .1, seed = TENSOR_SEED) 147 | with tf.variable_scope('actor'): 148 | l_a = tf.layers.dense(self.s, 200, tf.nn.relu6, kernel_initializer=w_init, name='la') 149 | l_a = tf.layers.dense(l_a, 200, tf.nn.relu6, kernel_initializer=w_init, name='la2') 150 | a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap') 151 | with tf.variable_scope('critic'): 152 | l_c = tf.layers.dense(self.s, 200, tf.nn.relu6, kernel_initializer=w_init, name='lc') 153 | l_c = tf.layers.dense(l_c, 200, tf.nn.relu6, kernel_initializer=w_init, name='lc2') 154 | v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v') # state value 155 | a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor') 156 | c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic') 157 | return a_prob, v, a_params, c_params 158 | 159 | 160 | def update_global(self, feed_dict): # run by a local 161 | SESS.run([self.update_a_op, self.update_c_op], feed_dict) # local grads applies to global net 162 | 163 | def pull_global(self): # run by a local 164 | SESS.run([self.pull_a_params_op, self.pull_c_params_op]) 165 | 166 | def choose_action(self, s): # run by a local 167 | prob_weights = SESS.run(self.a_prob, feed_dict={self.s: s[np.newaxis, :]}) 168 | action = np.random.choice(range(prob_weights.shape[1]), 169 | p=prob_weights.ravel()) # select action w.r.t the actions prob 170 | return action 171 | 172 | class Worker(object): 173 | def __init__(self, name, globalAC): 174 | self.env = MobiEnvironment(N_BS, N_UE, AREA_W)#gym.make(GAME).unwrapped 175 | self.name = name 176 | self.AC = ACNet(name, globalAC) 177 | self.total_steps = 0 178 | self.buf_r_dissect_all_ep = [] 179 | self.step_start = 0 180 | self.step_end = 0 181 | 182 | 183 | def work(self): 184 | global GLOBAL_RUNNING_R, GLOBAL_EP 185 | 186 | buffer_s, buffer_a, buffer_r = [], [], [] 187 | print "worker ", self.name, "starts training" 188 | 189 | while not COORD.should_stop() and GLOBAL_EP < MAX_GLOBAL_EP: 190 | # s = self.env.reset() 191 | s = np.ravel(self.env.reset()) 192 | ep_r = 0 193 | buf_r_dissect = [] 194 | 195 | while True: 196 | a = self.AC.choose_action(s) 197 | 198 | # self.step_start = time.time() 199 | s_, r, done, info = self.env.step(a) 200 | # self.step_end = time.time() 201 | # print self.name," (env) step time ", self.step_end - self.step_start 202 | 203 | s_ = np.ravel(s_) 204 | 205 | ep_r += r 206 | # step_r = r 207 | buffer_s.append(s) 208 | buffer_a.append(a) 209 | buffer_r.append(r) 210 | 211 | if self.name == 'W_0': buf_r_dissect.append(info[0]) 212 | 213 | if self.total_steps % UPDATE_GLOBAL_ITER == 0 or done: # update global and assign to local net 214 | if self.total_steps % (UPDATE_GLOBAL_ITER*50) == 0: 215 | print self.name, "updating GlobalAC at step ", self.total_steps 216 | 217 | # self.update_start= time.time() 218 | if done: 219 | v_s_ = 0 # terminal 220 | else: 221 | v_s_ = SESS.run(self.AC.v, {self.AC.s: s_[np.newaxis, :]})[0, 0] 222 | buffer_v_target = [] 223 | 224 | for r in buffer_r[::-1]: # reverse buffer r 225 | v_s_ = r + GAMMA * v_s_ 226 | buffer_v_target.append(v_s_) 227 | 228 | buffer_v_target.reverse() 229 | 230 | buffer_s, buffer_a, buffer_v_target = np.vstack(buffer_s), np.array(buffer_a), np.vstack(buffer_v_target) 231 | feed_dict = { 232 | self.AC.s: buffer_s, 233 | self.AC.a_his: buffer_a, 234 | self.AC.v_target: buffer_v_target, 235 | } 236 | self.AC.update_global(feed_dict) 237 | 238 | buffer_s, buffer_a, buffer_r = [], [], [] 239 | self.AC.pull_global() 240 | # self.update_end= time.time() 241 | # print self.name," (agent) update time ", self.update_end - self.update_start 242 | 243 | s = s_ 244 | 245 | self.total_steps += 1 246 | 247 | if done: 248 | if len(GLOBAL_RUNNING_R) == 0: # record running episode reward 249 | GLOBAL_RUNNING_R.append(ep_r) 250 | else: 251 | GLOBAL_RUNNING_R.append(0.99 * GLOBAL_RUNNING_R[-1] + 0.01 * ep_r) 252 | print( 253 | self.name, 254 | "Ep:", GLOBAL_EP, 255 | "| Ep_r: %f" % GLOBAL_RUNNING_R[-1], 256 | "| total steps", self.total_steps, 257 | "| step in Ep ", info[1]) 258 | 259 | GLOBAL_EP += 1 260 | 261 | if self.name == 'W_0': 262 | self.buf_r_dissect_all_ep.append(buf_r_dissect) 263 | np.save("train/Reward_dissect", self.buf_r_dissect_all_ep) 264 | 265 | if GLOBAL_EP % 500 == 0: 266 | np.savez("train/Global_A_PARA" + str(GLOBAL_EP), SESS.run(GLOBAL_AC.a_params)) 267 | 268 | np.save("train/Global_return",GLOBAL_RUNNING_R) 269 | # np.savez("train/A_PARA",SESS.run(self.AC.a_params)) 270 | np.savez("train/Global_A_PARA",SESS.run(GLOBAL_AC.a_params)) 271 | 272 | break 273 | 274 | 275 | if __name__ == "__main__": 276 | print ">>>>>>>>>>>>>>>>A3C SIM INFO>>>>>>>>>>>>>>>>>>>>" 277 | print "tensor seed: ", TENSOR_SEED 278 | print "N_S", N_S 279 | print "N_A", N_A 280 | print "LR_C", LR_C 281 | print "N_BS", N_BS 282 | print "N_UE", N_UE 283 | print "AREA_W", AREA_W 284 | print "Num of episodes", MAX_GLOBAL_EP 285 | print "(if cnn), num of filters", CNN_NUM_FILTERS 286 | print "(if cnn), num of filters", CNN_KERNEL_SIZE 287 | print ">>>>>>>>>>>>>>>>>>>>SIM INFO(end)>>>>>>>>>>>>>>>" 288 | 289 | SESS = tf.Session() 290 | 291 | start = time.time() 292 | 293 | with tf.device("/cpu:0"): 294 | OPT_A = tf.train.RMSPropOptimizer(LR_A, name='RMSPropA') 295 | OPT_C = tf.train.RMSPropOptimizer(LR_C, name='RMSPropC') 296 | GLOBAL_AC = ACNet(GLOBAL_NET_SCOPE) # we only need its params 297 | 298 | 299 | workers = [] 300 | # Create worker 301 | for i in range(N_WORKERS): 302 | i_name = 'W_%i' % i # worker namei 303 | print "Creating worker ", i_name 304 | workers.append(Worker(i_name, GLOBAL_AC)) 305 | 306 | COORD = tf.train.Coordinator() 307 | SESS.run(tf.global_variables_initializer()) 308 | np.savez("train/Global_A_PARA_init", SESS.run(GLOBAL_AC.a_params)) 309 | 310 | if OUTPUT_GRAPH: 311 | if os.path.exists(LOG_DIR): 312 | shutil.rmtree(LOG_DIR) 313 | tf.summary.FileWriter(LOG_DIR, SESS.graph) 314 | 315 | worker_threads = [] 316 | for worker in workers: 317 | job = lambda: worker.work() 318 | t = threading.Thread(target=job) 319 | t.start() 320 | worker_threads.append(t) 321 | COORD.join(worker_threads) 322 | 323 | end = time.time() 324 | print "Total time ", (end - start) 325 | 326 | -------------------------------------------------------------------------------- /main_test.py: -------------------------------------------------------------------------------- 1 | import time 2 | TEST_ALGO = "A3C" 3 | 4 | FILE_NAME_APPEND = "2000" 5 | OUTPUT_FILE_NAME = "test/" + FILE_NAME_APPEND + '_' 6 | 7 | def Load_AC_Net(): 8 | """ 9 | Load pre-trained A3C model for testing 10 | """ 11 | file_name = "train/Global_A_PARA" + FILE_NAME_APPEND +".npz" 12 | files = np.load(file_name) 13 | 14 | a_params = files['arr_0'] 15 | 16 | G_AC_TEST = ACNet('Global_Net') 17 | 18 | ops = [] 19 | for idx, param in enumerate(a_params): ops.append(G_AC_TEST.a_params[idx].assign(param)) 20 | SESS.run(ops) 21 | return G_AC_TEST 22 | 23 | def Load_DPPO_Net(): 24 | """ 25 | Load pre-trained DDPO model for testing 26 | """ 27 | 28 | file_name = "test/PI_PARA" + FILE_NAME_APPEND +".npz" 29 | files = np.load(file_name) 30 | 31 | pi_params = files['arr_0'] 32 | 33 | G_PPO_TEST = PPONet() 34 | 35 | ops = [] 36 | for idx, param in enumerate(pi_params): ops.append(G_PPO_TEST.pi_params[idx].assign(param)) 37 | SESS.run(ops) 38 | return G_PPO_TEST 39 | 40 | def Run_Test(g_test_net, reward_file_name): 41 | #maximum training step 42 | MAX_STEP = 10000 43 | 44 | #Reading mobility trace from file 45 | test_env = MobiEnvironment(N_BS, 40, 100, "read_trace", "./ue_trace_10k.npy") 46 | 47 | #reset states 48 | s = np.array([np.ravel(test_env.reset())]) 49 | 50 | done = False 51 | step = 0 52 | 53 | outage_buf = [] 54 | reward_buf = [] 55 | sinr_all = [] 56 | time_all = [] 57 | x = tf.argmax(g_test_net.a_prob, axis = 1) 58 | # ue_walk_trace = [] 59 | while step <= MAX_STEP: 60 | 61 | feed_dict = {g_test_net.s:s} 62 | start_time = time.time() 63 | action = SESS.run(x, feed_dict=feed_dict) 64 | time_all.append(time.time()-start_time) 65 | 66 | s_, r, done, info = test_env.step_test(action, False) 67 | # s_, r, done, info = test_env.step(action, False) 68 | sinr_all.append(test_env.channel.current_BS_sinr) 69 | reward_buf.append(info[0]) 70 | 71 | # ue_walk_trace.append(info[2]) 72 | if step % 500 == 0 or step == MAX_STEP: 73 | print "step ", step 74 | np.save(reward_file_name + "reward", reward_buf) 75 | np.save(reward_file_name +"sinr",sinr_all) 76 | np.save(reward_file_name + "time", time_all) 77 | # np.save("ue_trace_10k", ue_walk_trace) 78 | 79 | #if step % 5 == 0: 80 | #np.save(reward_file_name +"ue_loc" + str(step), test_env.ueLoc) 81 | #np.save(reward_file_name +"sinr_map" + str(step), test_env.sinr_map) 82 | #np.save(reward_file_name +"assoc_sinr" + str(step), test_env.assoc_sinr) 83 | # reset the environment every 2000 steps 84 | if step % 2000 == 0: 85 | s = np.array([np.ravel(test_env.reset())]) 86 | #warm up in 500 steps 87 | for _ in range(500): 88 | _, _, _, _ = test_env.step_test(action, False) 89 | else: 90 | s = np.array([np.ravel(s_)]) 91 | 92 | step+=1 93 | 94 | np.save(reward_file_name + "reward", reward_buf) 95 | np.save(reward_file_name + "sinr",sinr_all) 96 | np.save(reward_file_name + "time", time_all) 97 | # np.save("ue_trace_10k", ue_walk_trace) 98 | 99 | if __name__ == "__main__": 100 | if TEST_ALGO == "A3C": 101 | from main import * 102 | SESS = tf.Session() 103 | 104 | test_net = Load_AC_Net() 105 | elif TEST_ALGO == "DPPO": 106 | from dppo_main import * 107 | SESS = tf.Session() 108 | 109 | test_net = Load_DPPO_Net() 110 | 111 | Run_Test(test_net, OUTPUT_FILE_NAME) 112 | 113 | 114 | -------------------------------------------------------------------------------- /mobile_env.py: -------------------------------------------------------------------------------- 1 | """ 2 | # mobile environment that update channel and association 3 | """ 4 | import numpy as np 5 | import random 6 | import itertools 7 | import math 8 | from channel import * 9 | from ue_mobility import * 10 | import copy 11 | import numpy as np 12 | import sys 13 | 14 | #matplotlib.rcParams.update({'font.size': 14}) 15 | 16 | # defining the number of steps 17 | MAXSTEP = 2000 18 | UE_STEP = 1 19 | 20 | N_ACT = 5 #number of actions of a single agent 21 | 22 | MAX_UE_PER_GRID = 1 # Maximum number of UE per grid 23 | 24 | # relative hight of the BSs to UEs in m assuming plain terrain 25 | H_BS = 10 26 | # min distance between BSs in Grids 27 | MIN_BS_DIST = 2 28 | 29 | R_BS = 50 30 | # 31 | BS_STEP = 2 32 | 33 | 34 | class MobiEnvironment: 35 | 36 | def __init__(self, nBS, nUE, grid_n=200, mobility_model = "group", test_mobi_file_name = ""): 37 | self.nBS = nBS 38 | self.nUE = nUE 39 | self.bs_h = H_BS 40 | 41 | # x,y boundaries 42 | self.grid_n = grid_n 43 | 44 | [xMin, xMax, yMin, yMax] = [1, self.grid_n, 1, self.grid_n] 45 | boundaries = [xMin, xMax, yMin, yMax] 46 | # xBS = np.array([int(xMin +1), int(xMax -1), int(xMin+1), int(xMax-1)]) 47 | # yBS = np.array([int(yMin +1), int(yMax -1), int(yMax-1), int(yMin+1)]) 48 | xBS = np.array([int(xMax/4), int(xMax/4), int(xMax*3/4), int(xMax*3/4)]) 49 | yBS = np.array([int(yMax/4), int(yMax*3/4), int(yMax/4), int(yMax*3/4)]) 50 | 51 | 52 | self.boundaries = boundaries 53 | self.mobility_model = mobility_model 54 | print "mobility model: ", mobility_model, " grid size ", grid_n 55 | # bsLoc is 3D, bsLocGrid is 2D heatmap 56 | #self.bsLoc, self.bsLocGrid = GetRandomLocationInGrid(self.grid_n, self.grid_n, self.nBS, H_BS, MIN_BS_DIST) 57 | self.initBsLoc = np.array([xBS, yBS, np.ones((np.size(xBS)))*self.bs_h], dtype=int).T 58 | self.initBsLocGrid = GetGridMap(self.grid_n, self.grid_n, self.initBsLoc[:,:2]) 59 | self.bsLoc = copy.deepcopy(self.initBsLoc) 60 | self.bsLocGrid = copy.deepcopy(self.initBsLocGrid) 61 | 62 | # self.ueLoc, self.ueLocGrid = GetRandomLocationInGrid(self.grid_n, self.grid_n, self.nUE) 63 | self.ueLoc = [] 64 | self.ueLocGrid = [] 65 | 66 | self.mm = [] 67 | 68 | #mobility trace used for testing 69 | self.test_mobi_trace = [] 70 | 71 | if self.mobility_model == "random_waypoint": 72 | self.mm = random_waypoint(nUE, dimensions=(self.grid_n, self.grid_n), velocity=(1, 1), wt_max=1.0) 73 | elif self.mobility_model == "group": 74 | # self.mm = tvc([10,10,10,10], dimensions=(self.grid_n, self.grid_n), velocity=(1, 1.), aggregation=[0.5,0.2], epoch=[1000,1000]) 75 | self.mm = reference_point_group([10,10,10,10], dimensions=(self.grid_n, self.grid_n), velocity=(0, 1), aggregation=0.8) 76 | for i in range(200): 77 | next(self.mm) 78 | i += 1 #repeat in reset 79 | elif self.mobility_model == "in_coverage": 80 | self.ueLoc, self.ueLocGrid = GetRandomLocationInCellCoverage(self.grid_n, self.grid_n, R_BS, self.bsLoc, self.nUE) 81 | elif self.mobility_model == "read_trace": 82 | print "testing with mobility trace ", test_mobi_file_name 83 | assert test_mobi_file_name 84 | self.ueLoc_trace = np.load(test_mobi_file_name) 85 | 86 | self.ueLoc = self.ueLoc_trace[0] 87 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 88 | 89 | else: 90 | sys.exit("mobility model not defined") 91 | 92 | if (self.mobility_model == "random_waypoint") or (self.mobility_model == "group"): 93 | positions = next(self.mm) 94 | #2D to 3D 95 | z = np.zeros((np.shape(positions)[0],0)) 96 | self.ueLoc = np.concatenate((positions, z), axis=1).astype(int) 97 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 98 | 99 | self.channel = LTEChannel(self.nUE, self.nBS, self.boundaries, self.ueLoc, self.bsLoc) 100 | self.association = self.channel.GetCurrentAssociationMap(self.ueLoc) 101 | 102 | 103 | self.action_space_dim = N_ACT**self.nBS 104 | self.observation_space_dim = self.grid_n * self.grid_n * (nBS + 1) * MAX_UE_PER_GRID 105 | 106 | self.state = np.zeros((nBS + 1, self.grid_n, self.grid_n )) 107 | self.step_n = 0 108 | 109 | 110 | def SetBsH(self, h): 111 | self.bs_h = h 112 | 113 | 114 | def reset(self): 115 | # Get random locations for bs and ue 116 | #self.bsLoc, self.bsLocGrid = GetRandomLocationInGrid(self.grid_n, self.grid_n, self.nBS, H_BS, MIN_BS_DIST) 117 | 118 | self.bsLoc = copy.deepcopy(self.initBsLoc) 119 | self.bsLocGrid = copy.deepcopy(self.initBsLocGrid) 120 | 121 | if (self.mobility_model == "random_waypoint") or (self.mobility_model == "group"): 122 | positions = next(self.mm) 123 | #2D to 3D 124 | z = np.zeros((np.shape(positions)[0],0)) 125 | self.ueLoc = np.concatenate((positions, z), axis=1).astype(int) 126 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 127 | elif self.mobility_model == "read_trace": 128 | print "reseting mobility trace " 129 | self.ueLoc = self.ueLoc_trace[0] 130 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 131 | else: 132 | self.ueLoc, self.ueLocGrid = GetRandomLocationInCellCoverage(self.grid_n, self.grid_n, R_BS, self.bsLoc, self.nUE) 133 | 134 | # reset channel 135 | self.channel.reset(self.ueLoc, self.bsLoc) 136 | # reset association 137 | self.association = self.channel.GetCurrentAssociationMap(self.ueLoc) 138 | 139 | self.state[0] = self.bsLocGrid 140 | self.state[1:] = self.association 141 | 142 | # self.ueLocGrid = np.sum(self.association, axis = 0) 143 | # print np.array_equal(np.sum(self.association, axis = 0), self.ueLocGrid ) 144 | 145 | self.step_n = 0 146 | 147 | return np.array(self.state) 148 | 149 | def step(self, action , ifrender=False): #(step) 150 | 151 | positions = next(self.mm) 152 | #2D to 3D 153 | z = np.zeros((np.shape(positions)[0],0)) 154 | self.ueLoc = np.concatenate((positions, z), axis=1).astype(int) 155 | 156 | self.bsLoc = BS_move(self.bsLoc, self.boundaries, action, BS_STEP, MIN_BS_DIST + BS_STEP, N_ACT) 157 | self.association_map, meanSINR, nOut = self.channel.UpdateDroneNet(self.ueLoc, self.bsLoc, ifrender, self.step_n) 158 | 159 | self.bsLocGrid = GetGridMap(self.grid_n, self.grid_n, self.bsLoc) 160 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 161 | 162 | r_dissect = [] 163 | 164 | r_dissect.append(meanSINR/20) 165 | 166 | r_dissect.append(-1.0 * nOut/self.nUE) 167 | 168 | self.state[0] = self.bsLocGrid 169 | self.state[1:] = self.association_map 170 | 171 | # dist_penalty = Get_loc_penalty(self.bsLoc, 25, self.nUE) 172 | # r_dissect.append(-dist_penalty/self.nUE *0.5) 173 | 174 | done = False 175 | # done = Get_if_collide(self.bsLoc, MIN_BS_DIST) 176 | # collision = done 177 | 178 | self.step_n += 1 179 | 180 | # if collision: 181 | # r_dissect.append(-1) 182 | # else: 183 | # r_dissect.append(0) 184 | 185 | if self.step_n >= MAXSTEP: 186 | done = True 187 | 188 | reward = max(sum(r_dissect), -1) 189 | # print meanSINR, " ",nOut," ", r_dissect, " ", reward 190 | 191 | # info = [r_dissect, self.step_n, self.ueLoc] 192 | info = [r_dissect, self.step_n] 193 | return np.array(self.state), reward, done, info 194 | 195 | def step_test(self, action , ifrender=False): #(step) 196 | """ 197 | similar to step(), but write here an individual function to 198 | avoid "if--else" in the original function to reduce training 199 | time cost 200 | """ 201 | 202 | self.ueLoc = self.ueLoc_trace[self.step_n] 203 | self.bsLoc = BS_move(self.bsLoc, self.boundaries, action, BS_STEP, MIN_BS_DIST + BS_STEP, N_ACT) 204 | self.association_map, meanSINR, nOut = self.channel.UpdateDroneNet(self.ueLoc, self.bsLoc, ifrender, self.step_n) 205 | 206 | self.bsLocGrid = GetGridMap(self.grid_n, self.grid_n, self.bsLoc) 207 | self.ueLocGrid = GetGridMap(self.grid_n, self.grid_n, self.ueLoc) 208 | 209 | r_dissect = [] 210 | 211 | r_dissect.append(meanSINR/20) 212 | 213 | r_dissect.append(-1.0 * nOut/self.nUE) 214 | 215 | self.state[0] = self.bsLocGrid 216 | self.state[1:] = self.association_map 217 | 218 | done = False 219 | 220 | self.step_n += 1 221 | 222 | if self.step_n >= MAXSTEP: 223 | done = True 224 | 225 | reward = max(sum(r_dissect), -1) 226 | 227 | info = [r_dissect, self.step_n, self.ueLoc] 228 | return np.array(self.state), reward, done, info 229 | 230 | 231 | def render(self): 232 | fig = figure(1, figsize=(20,20)) 233 | for bs in range(self.nBS): 234 | subplot(self.nBS +1, 2, bs+1) 235 | title("bs " + str(bs) + "ue distribution") 236 | imshow(self.association[bs], interpolation='nearest', origin='lower') 237 | 238 | subplot(self.nBS +1, 2, self.nBS+ bs+1) 239 | title("bs " + str(bs) + "ue sinr distribution") 240 | imshow(self.association_sinr[bs], interpolation='nearest', origin='lower') 241 | 242 | def plot_sinr_map(self): 243 | fig = figure(1, figsize=(100,100)) 244 | subplot(1, 2, 1) 245 | 246 | sinr_all = self.channel.GetSinrInArea(self.bsLoc) 247 | imshow(sinr_all, interpolation='nearest', origin='lower', vmin= -50, vmax = 100) 248 | colorbar() 249 | xlabel('x[m]') 250 | ylabel('y[m]') 251 | title('DL SINR [dB]') 252 | 253 | subplot(1,2,2) 254 | hist(sinr_all, bins=100, fc='k', ec='k') 255 | ylim((0,20)) 256 | xlabel("SINR") 257 | ylabel("number of UEs") 258 | xlim(-100, 100) 259 | 260 | show() 261 | fig.savefig("sinr_map.pdf") 262 | 263 | np.save("sinr_map",sinr_all) 264 | -------------------------------------------------------------------------------- /sinr_visualisation.py: -------------------------------------------------------------------------------- 1 | from matplotlib import cm 2 | from matplotlib.ticker import LinearLocator, FormatStrFormatter 3 | from matplotlib.pyplot import * 4 | from IPython import display 5 | 6 | matplotlib.rcParams.update({'font.size': 14}) 7 | # 8 | ## used to plot snapshoot of user distribution 9 | #usrDistributionNSteps = np.zeros((nStep, gridX, gridY)) 10 | #for userId in range(nUE): 11 | # (nStepX[userId],nStepY[userId]) = WalkToFixedDirection([xInit[userId],yInit[userId]], boundaries, stepLen, nStep, userId) 12 | # for stepN in range(nStep): 13 | # x = int(nStepX[userId][stepN]) 14 | # y = int(nStepY[userId][stepN]) 15 | # usrDistributionNSteps[stepN][x][y] += 1 16 | # 17 | # 18 | ##plotting user distribution: 19 | ## for time in [1,2,3,nStep/2, nStep-1]: 20 | #for time in [0]: 21 | # title("User Distribution at time "+ str(time) + "s") 22 | # # print usrDistributionNSteps[time][np.nonzero(usrDistributionNSteps[time])] 23 | # imshow(usrDistributionNSteps[time].T, cmap='hot', interpolation='nearest', origin='lower') 24 | # xlabel("x") 25 | # ylabel("y") 26 | # show() 27 | # 28 | # 29 | ## plotting user trajectory (static): 30 | #for ue in range(nUE): 31 | # title("Random Walk of UE " + str(ue) +"($n = " + str(nStep) + "$ steps)") 32 | # ylim(xMax) 33 | # xlim(yMax) 34 | # xlabel("x") 35 | # ylabel("y") 36 | # plot(nStepX[ue],nStepY[ue]) 37 | # show() 38 | # 39 | # 40 | ## visualise UL DL sinrs 41 | ## For best UL SINR 42 | #fig = figure() 43 | #sinrDistrbution = np.zeros((xMax - xMin, yMax - yMin)) 44 | #for userId in range(nUE): 45 | # x = int(ueLocationAll[userId][0]) 46 | # y = int(ueLocationAll[userId][1]) 47 | # sinrDistrbution[y][x] = bestUlSinr[userId] 48 | #title("Best UL SINR") 49 | #pos = imshow(sinrDistrbution, cmap='hot', interpolation='nearest') 50 | #fig.colorbar(pos) 51 | #show 52 | #savefig("BestULSINR", dpi=None, facecolor='w', edgecolor='w', 53 | # orientation='portrait', papertype=None, format=None, 54 | # transparent=False, bbox_inches=None, pad_inches=0.1, 55 | # frameon=None) 56 | # 57 | ## For best DL SINR 58 | #fig = figure() 59 | #sinrDistrbution = np.zeros((xMax - xMin, yMax - yMin)) 60 | #for userId in range(nUE): 61 | # x = int(ueLocationAll[userId][0]) 62 | # y = int(ueLocationAll[userId][1]) 63 | # sinrDistrbution[y][x] = bestDlSinr[userId] 64 | #title("Best DL SINR") 65 | #pos = imshow(sinrDistrbution, cmap='hot', interpolation='nearest') 66 | #fig.colorbar(pos) 67 | #show 68 | #savefig("BestDLSINR", dpi=None, facecolor='w', edgecolor='w', 69 | # orientation='portrait', papertype=None, format=None, 70 | # transparent=False, bbox_inches=None, pad_inches=0.1, 71 | # frameon=None) 72 | # 73 | # 74 | ## For individual BSs 75 | #sinrDistrbution = np.zeros((nBS, xMax - xMin, yMax - yMin)) 76 | # 77 | #for bsId in range(nBS): 78 | # for userId in range(nUE): 79 | # x = int(ueLocationAll[userId][0]) 80 | # y = int(ueLocationAll[userId][1]) 81 | # # Issue with the ueLocationAll indexing when plot the heat map. 82 | # # checked with distance and SINR values all okay but when plotting the heatmap, x, y are inverted 83 | # # plotting fixed values are also okay.. 84 | # # using sinrDistrbution[bsId][y][x] instead of sinrDistrbution[bsId][x][y] resolves the issue 85 | # # sinrDistrbution[bsId][x][y] = GetDistance(ueLocationAll[userId], bsLocationAll[bsId]) 86 | # sinrDistrbution[bsId][y][x] = ulSinr[userId][bsId]#GetDistance(ueLocationAll[userId], bsLocationAll[bsId]) 87 | # 88 | ##plotting user distribution: 89 | #for bsId in range(nBS): 90 | # x = bsLocationAll[bsId][0] 91 | # y = bsLocationAll[bsId][1] 92 | # fig = figure() 93 | # ax = fig.add_subplot(111) 94 | # ax.annotate('BS', xy=(x,y), xytext=(x, y), 95 | # arrowprops=dict(facecolor='black', shrink=0.05)) 96 | # for ueId in range(2): 97 | # ax.annotate('UE', xy=(ueLocationAll[ueId][0],ueLocationAll[ueId][1]), xytext=(ueLocationAll[ueId][0],ueLocationAll[ueId][1]), 98 | # arrowprops=dict(facecolor='white', shrink=0.05)) 99 | # print "UE",ueId," (",ueLocationAll[ueId] ,")", ulSinr[ueId][bsId] 100 | # 101 | # title("DL SINR Distribution from BS"+ str(bsId) + " (" + str(x) + ", " + str(y) + ")") 102 | # imshow(sinrDistrbution[bsId], cmap='hot', interpolation='nearest') 103 | # xlabel("x [m]") 104 | # ylabel("y [m]") 105 | # show() 106 | 107 | 108 | 109 | def draw_UE_HO(ue_loc, numGridX, numGridY, bsLoc, ue2watch, xbestSinr, xcurrSinr, xbestBS, xcurrBS, currentTime, dlRate, ulRate, size=(5, 5), color = []): 110 | fig = figure(1, figsize=size) 111 | fig.subplots_adjust(hspace=.4) 112 | # subplot(4, 1, 1) 113 | ueValGrid = np.zeros((3, numGridX, numGridY)) 114 | if color.any(): 115 | for usr in xrange(len(ue_loc)): 116 | ueValGrid[:, ue_loc[usr][0], ue_loc[usr][1]] = color[usr] 117 | else: 118 | # problem 119 | ueValGrid[:, ue_loc[0], ue_loc[1]] = 1 120 | 121 | for bsId in range(len(bsLoc)): 122 | x = bsLoc[bsId][0] 123 | y = bsLoc[bsId][1] 124 | strBS = "BS " + str(bsId) 125 | text(x, y, strBS, color='white') 126 | 127 | xlabel("x [m]") 128 | ylabel("y [m]") 129 | title("User distribution at time " + str(currentTime)) 130 | imshow(ueValGrid.T, interpolation='nearest', origin='lower') 131 | # 132 | # subplot(4,1,2) 133 | # grid = np.zeros((numGridX, numGridY)) 134 | # grid[ue_loc[ue2watch][0], ue_loc[ue2watch][1]] = 1 135 | # 136 | # for bsId in range(len(bsLoc)): 137 | # x = bsLoc[bsId][0] 138 | # y = bsLoc[bsId][1] 139 | # strBS = "BS " + str(bsId) 140 | # text(x, y, strBS) 141 | # 142 | # xlabel("x [m]") 143 | # ylabel("y [m]") 144 | # title("UE"+ str(ue2watch) + " current location ") 145 | # imshow(grid.T, interpolation='nearest', origin='lower') 146 | # 147 | # # UE 2 watch SINR from current BS and best BS 148 | # subplot(4,1,3) 149 | # xlabel("Time [Steps]") 150 | # ylabel("SINR [dB]") 151 | # strLegendcur = "current SINR (BS" + str(xcurrBS) + ")" 152 | # strLegendbes = "best SINR (BS" + str(xbestBS) + ")" 153 | # 154 | # timeAxis = currentTime - np.array(range(len(xcurrSinr))) 155 | ## print len(xcurrSinr) 156 | # plot(timeAxis, xcurrSinr, label = strLegendcur) 157 | # hold('on') 158 | # plot(timeAxis, xbestSinr, label = strLegendbes) 159 | # 160 | # legend(loc=2) 161 | # ylim(-100, 100) 162 | # title("SINR from best BS and current ass BS. UE"+ str(ue2watch)) 163 | # 164 | # 165 | # subplot(4,1,4) 166 | # xlabel("Time [Steps]") 167 | # ylabel("Mean Rate [Mbps]") 168 | # strLegendcur = "UL" 169 | # strLegendbes = "DL" 170 | # 171 | # timeAxis = currentTime - np.array(range(len(ulRate))) #here 172 | ## print len(ulRate) 173 | # plot(timeAxis, ulRate, label = "UL") 174 | # hold('on') 175 | # plot(timeAxis, dlRate, label = "DL") 176 | # 177 | # legend(loc=2) 178 | # ylim(0, 1) 179 | # title("Mean rate of all UEs from serving BSs") 180 | # 181 | # 182 | show() 183 | 184 | 185 | def plot_sinr_distribution(sinr, x_from, x_to): 186 | hist(sinr, bins=50, fc='k', ec='k') 187 | 188 | xlabel("SINR") 189 | ylabel("number of UEs") 190 | xlim(x_from, x_to) 191 | # savefig("SINR_dist.pdf") 192 | show() 193 | -------------------------------------------------------------------------------- /ue_mobility.py: -------------------------------------------------------------------------------- 1 | # Python code for 2D random walk, fixed direction, and group reference point mobility model. 2 | import numpy as np 3 | #import pylab 4 | import random 5 | import math 6 | from numpy.random import rand 7 | 8 | def WalkNRandomSteps(initCoordinates, boundaries, stepLen, nSteps): 9 | #creating two array for containing x and y coordinate 10 | #of size equals to the number of size and filled up with 0's 11 | [initX, initY] = initCoordinates 12 | x = np.ones(nSteps) * initX 13 | y = np.ones(nSteps) * initY 14 | [xMin, xMax, yMin, yMax] = boundaries 15 | 16 | for i in range(1, nSteps): 17 | x[i] = x[i - 1] 18 | y[i] = y[i - 1] 19 | val = random.randint(1, 4) 20 | 21 | if val == 1: 22 | if x[i] + stepLen >= xMax: 23 | x[i] = xMin 24 | else: 25 | x[i] = x[i] + stepLen 26 | 27 | elif val == 2: 28 | if x[i] - stepLen <= xMin: 29 | x[i] = xMax 30 | else: 31 | x[i] = x[i] - stepLen 32 | 33 | elif val == 3: 34 | if y[i] + stepLen >= yMax: 35 | y[i] = yMin 36 | else: 37 | y[i] = y[i] + stepLen 38 | else: 39 | if y[i] - stepLen <= yMin: 40 | y[i] = yMax 41 | else: 42 | y[i] = y[i] - stepLen 43 | 44 | return (x, y) 45 | 46 | def WalkToFixedDirection(initCoordinates, boundaries, stepLen, nSteps, direction): 47 | #creating two array for containing x and y coordinate 48 | #of size equals to the number of size and filled up with 0's 49 | [initX, initY] = initCoordinates 50 | x = np.ones(nSteps) * initX 51 | y = np.ones(nSteps) * initY 52 | [xMin, xMax, yMin, yMax] = boundaries 53 | 54 | for i in range(1, nSteps): 55 | x[i] = x[i - 1] 56 | y[i] = y[i - 1] 57 | val = direction%4 + 1 58 | 59 | if val == 1: 60 | if x[i] + stepLen >= xMax: 61 | x[i] = xMin 62 | else: 63 | x[i] = x[i] + stepLen 64 | 65 | elif val == 2: 66 | if x[i] - stepLen <= xMin: 67 | x[i] = xMax 68 | else: 69 | x[i] = x[i] - stepLen 70 | 71 | elif val == 3: 72 | if y[i] + stepLen >= yMax: 73 | y[i] = yMin 74 | else: 75 | y[i] = y[i] + stepLen 76 | else: 77 | if y[i] - stepLen <= yMin: 78 | y[i] = yMax 79 | else: 80 | y[i] = y[i] - stepLen 81 | 82 | return (x, y) 83 | 84 | 85 | def GetRandomWalkTraceInbound(numStep, numUE, stepLen, boundaries): 86 | 87 | margin = 5 88 | 89 | [xMin, xMax, yMin, yMax] = boundaries 90 | xInit = np.random.randint(xMin + margin, xMax - margin, size=numUE) 91 | yInit = np.random.randint(yMin + margin, yMax - margin, size=numUE) 92 | 93 | 94 | nStepX = np.zeros((numUE,numStep)) 95 | nStepY = np.zeros((numUE,numStep)) 96 | 97 | 98 | for userId in range(numUE): 99 | # (nStepX[userId],nStepY[userId]) = WalkToFixedDirection([xInit[userId],yInit[userId]], boundaries, stepLen, nStep, userId) 100 | (nStepX[userId],nStepY[userId]) = WalkNRandomSteps([xInit[userId],yInit[userId]], boundaries, stepLen, numStep) 101 | 102 | trace = np.zeros((numStep,numUE,3)).astype(int) 103 | trace[:,:,:2] = np.array([nStepX,nStepY]).T 104 | 105 | return trace 106 | 107 | 108 | 109 | def GetRandomLocationInCellCoverage(gridX_size, gridY_size, cell_size, bsloc, num_node): 110 | """ 111 | #Generate n random locations in coverage of bsLoc ((0, gridX_size),(0, gridY_size)) 112 | input 113 | 1) cell size in x 114 | 2) BS locations (3D) 115 | 3) number of nodes (bs or ue) 116 | 4) height (z) value - default 0 117 | 5) minimum distance between every 2 nodes - default 0 #TODO 118 | 119 | return 1) location of nodes in format (x,y) 120 | 2) heatmap format locations 121 | """ 122 | 123 | nBS = np.shape(bsloc)[0] 124 | rand_users_loc = [] 125 | for ue in range(num_node): 126 | in_bs = np.random.randint(0, nBS) 127 | theta = np.random.uniform(0, 2*math.pi) 128 | r = np.random.uniform(0, cell_size) 129 | 130 | rand_users_loc.append([bsloc[in_bs][0] + r* math.sin(theta), bsloc[in_bs][1] + r* math.cos(theta), 0]) 131 | 132 | loc = np.asarray(rand_users_loc, dtype=int) 133 | 134 | # print loc 135 | 136 | grid = np.zeros((gridX_size, gridY_size)) 137 | 138 | for n in range(num_node): 139 | # print loc[n][0], loc[n][1] 140 | grid[loc[n][0], loc[n][1]] += 1 141 | 142 | return loc, grid 143 | 144 | 145 | def GetRandomLocationInGrid(gridX_size, gridY_size, num_node, h=0, min_dist=0): 146 | """ 147 | #Generate n random locations in range ((0, gridX_size),(0, gridY_size)) 148 | input 1) grid size in x 149 | 2) grid size in y 150 | 3) number of nodes (bs or ue) 151 | 4) height (z) value - default 0 152 | 5) minimum distance between every 2 nodes - default 0 #TODO 153 | 154 | return 1) location of nodes in format (x,y) 155 | 2) heatmap format locations 156 | """ 157 | any_too_close = True 158 | while (any_too_close): 159 | x = np.random.randint(0, gridX_size, size=num_node) 160 | y = np.random.randint(0, gridY_size, size=num_node) 161 | 162 | loc = [x,y,np.ones((num_node)) * h] #3D loc 163 | any_too_close = Get_if_collide(loc, min_dist) 164 | 165 | grid = np.zeros((gridX_size, gridY_size)) 166 | 167 | for n in range(num_node): 168 | grid[x[n], y[n]] += 1 169 | 170 | return np.array(loc, dtype=int).T, grid 171 | 172 | def GetGridMap(gridX_size, gridY_size, nodeLoc): 173 | """ 174 | #Generate n random locations in range ((0, gridX_size),(0, gridY_size)) 175 | input 1) grid size in x 176 | 2) grid size in y 177 | 3) node locations in (x,y) format 178 | 179 | return heatmap format locations 180 | """ 181 | grid = np.zeros((gridX_size, gridY_size)) 182 | 183 | for n in range(np.shape(nodeLoc)[0]): 184 | 185 | grid[nodeLoc[n][0], nodeLoc[n][1]] += 1 186 | 187 | return grid 188 | 189 | 190 | def BS_move(loc, bound, action, stepLen, min_dist, n_action): 191 | """ 192 | BS takes a single move based on "action" value 193 | loc: current BSs location 194 | action: action index (single number for all BS) 195 | stepLen: step length 196 | min_dist: minimum distant between BSs, 197 | (the BS will only move if its distance to all the 198 | other BSs is greater than this value. 199 | n_action: total number of actions for a single BS 200 | return: BSs locations after the step 201 | """ 202 | 203 | nBS = np.shape(loc)[0] 204 | # print "location \n", loc 205 | act_all = Decimal_to_Base_N(action, n_action, nBS) 206 | # print "action", act_all 207 | [xMin, xMax, yMin, yMax] = bound 208 | 209 | #action 5-8 moves with longer stepLen 210 | stepLenLong = stepLen*2 211 | 212 | for i in range(nBS): 213 | 214 | val = act_all[i] 215 | [x, y, z] = loc[i] 216 | 217 | if val == 0: 218 | if x + stepLen < xMax: 219 | x = x + stepLen 220 | 221 | elif val == 1: 222 | if x - stepLen > xMin: 223 | x = x - stepLen 224 | 225 | elif val == 2: 226 | if y + stepLen < yMax: 227 | y = y + stepLen 228 | 229 | elif val == 3: 230 | if y - stepLen > yMin: 231 | y = y - stepLen 232 | 233 | # stay if val == 4 234 | 235 | elif val == 5: 236 | if x + stepLenLong < xMax: 237 | x = x + stepLenLong 238 | 239 | elif val == 6: 240 | if x - stepLenLong > xMin: 241 | x = x - stepLenLong 242 | 243 | elif val == 7: 244 | if y + stepLenLong < yMax: 245 | y = y + stepLenLong 246 | 247 | elif val == 8: 248 | if y - stepLenLong > yMin: 249 | y = y - stepLenLong 250 | 251 | 252 | if_collide = False 253 | 254 | for j in range(nBS): 255 | if i != j: 256 | dist = np.linalg.norm(loc[i]-loc[j]) # verify me 257 | 258 | if dist <= min_dist: 259 | if_collide = True 260 | 261 | if not if_collide: 262 | loc[i] = [x, y, z] 263 | 264 | # print "new location \n", loc 265 | return loc 266 | 267 | def UE_rand_move(loc, bound, stepLen): 268 | 269 | [xMin, xMax, yMin, yMax] = bound 270 | 271 | for i in range(np.shape(loc)[0]): 272 | 273 | [x, y, z] = loc[i] 274 | 275 | val = random.randint(0, 3) 276 | 277 | if val == 0: 278 | if x + stepLen >= xMax: 279 | x = xMin 280 | else: 281 | x = x + stepLen 282 | 283 | elif val == 1: 284 | if x - stepLen <= xMin: 285 | x = xMax 286 | else: 287 | x = x - stepLen 288 | 289 | elif val == 2: 290 | if y + stepLen >= yMax: 291 | y = yMin 292 | else: 293 | y = y + stepLen 294 | elif val == 3: 295 | if y - stepLen <= yMin: 296 | y = yMax 297 | else: 298 | y = y - stepLen 299 | 300 | loc[i] = [x, y, z] 301 | return loc 302 | 303 | 304 | def Decimal_to_Base_N(num, base, digits): 305 | """Change decimal number ``num'' to given base 306 | Upto base 36 is supported. 307 | num: the number to be converted 308 | base: the base 309 | digits: number of output digits 310 | return result_array 311 | """ 312 | 313 | result_array = np.zeros((digits)) 314 | converted_string, modstring = "", "" 315 | # print num 316 | currentnum = num 317 | if not 1 < base < 37: 318 | raise ValueError("base must be between 2 and 36") 319 | if not num: 320 | return result_array 321 | while currentnum: 322 | mod = currentnum % base 323 | currentnum = currentnum // base 324 | converted_string = chr(48 + mod + 7*(mod > 10)) + converted_string 325 | 326 | result = np.array([int(d) for d in str(converted_string)]) 327 | 328 | result_array[digits - len(result):] = result 329 | 330 | return result_array 331 | 332 | def Get_if_collide(locations, threshold): 333 | """ 334 | check if the distance between any 2 of the given locations are below the threshold 335 | """ 336 | any_collide = False 337 | for i in range(len(locations)): 338 | for j in range(len(locations)): 339 | if i == j: 340 | continue 341 | 342 | dist = np.linalg.norm(locations[i]-locations[j]) # verify me 343 | # in number of grids 344 | if dist <= threshold: 345 | any_collide = True 346 | 347 | return any_collide 348 | 349 | def Get_loc_penalty(locations, threshold, nUE): 350 | """ 351 | check if the distance between any 2 of the given locations are below the threshold 352 | """ 353 | penalty = 0 354 | 355 | for i in range(len(locations)): 356 | for j in range(len(locations)): 357 | if i == j: 358 | continue 359 | 360 | dist = np.linalg.norm(locations[i]-locations[j]) 361 | # 362 | if dist <= threshold: 363 | p = nUE - nUE * dist / threshold 364 | penalty += p 365 | 366 | penalty = math.floor(penalty/2) 367 | return penalty 368 | 369 | ''' 370 | Reference Point Group Mobility model, discussed in the following paper: 371 | 372 | Xiaoyan Hong, Mario Gerla, Guangyu Pei, and Ching-Chuan Chiang. 1999. 373 | A group mobility model for ad hoc wireless networks. In Proceedings of the 374 | 2nd ACM international workshop on Modeling, analysis and simulation of 375 | wireless and mobile systems (MSWiM '99). ACM, New York, NY, USA, 53-60. 376 | 377 | In this implementation, group trajectories follow a random direction model, 378 | while nodes follow a random walk around the group center. 379 | The parameter 'aggregation' controls how close the nodes are to the group center. 380 | 381 | Required arguments: 382 | 383 | *nr_nodes*: 384 | list of integers, the number of nodes in each group. 385 | 386 | *dimensions*: 387 | Tuple of Integers, the x and y dimensions of the simulation area. 388 | 389 | keyword arguments: 390 | 391 | *velocity*: 392 | Tuple of Doubles, the minimum and maximum values for group velocity. 393 | 394 | *aggregation*: 395 | Double, parameter (between 0 and 1) used to aggregate the nodes in the group. 396 | Usually between 0 and 1, the more this value approximates to 1, 397 | the nodes will be more aggregated and closer to the group center. 398 | With a value of 0, the nodes are randomly distributed in the simulation area. 399 | With a value of 1, the nodes are close to the group center. 400 | ''' 401 | 402 | U = lambda MIN, MAX, SAMPLES: rand(*SAMPLES.shape) * (MAX - MIN) + MIN 403 | def reference_point_group(nr_nodes, dimensions, velocity=(0.1, 1.), aggregation=0.1): 404 | try: 405 | iter(nr_nodes) 406 | except TypeError: 407 | nr_nodes = [nr_nodes] 408 | 409 | NODES = np.arange(sum(nr_nodes)) 410 | 411 | groups = [] 412 | prev = 0 413 | for (i,n) in enumerate(nr_nodes): 414 | groups.append(np.arange(prev,n+prev)) 415 | prev += n 416 | 417 | g_ref = np.empty(sum(nr_nodes), dtype=np.int) 418 | for (i,g) in enumerate(groups): 419 | for n in g: 420 | g_ref[n] = i 421 | 422 | FL_MAX = max(dimensions) 423 | MIN_V,MAX_V = velocity 424 | FL_DISTR = lambda SAMPLES: U(0, FL_MAX, SAMPLES) 425 | VELOCITY_DISTR = lambda FD: U(MIN_V, MAX_V, FD) 426 | 427 | MAX_X, MAX_Y = dimensions 428 | x = U(0, MAX_X, NODES) 429 | y = U(0, MAX_Y, NODES) 430 | velocity = 1. 431 | theta = U(0, 2*np.pi, NODES) 432 | costheta = np.cos(theta) 433 | sintheta = np.sin(theta) 434 | 435 | GROUPS = np.arange(len(groups)) 436 | g_x = U(0, MAX_X, GROUPS) 437 | g_y = U(0, MAX_X, GROUPS) 438 | g_fl = FL_DISTR(GROUPS) 439 | g_velocity = VELOCITY_DISTR(g_fl) 440 | g_theta = U(0, 2*np.pi, GROUPS) 441 | g_costheta = np.cos(g_theta) 442 | g_sintheta = np.sin(g_theta) 443 | 444 | aggregating = 200 445 | deaggregating = 100 446 | 447 | while True: 448 | 449 | x = x + velocity * costheta 450 | y = y + velocity * sintheta 451 | 452 | g_x = g_x + g_velocity * g_costheta 453 | g_y = g_y + g_velocity * g_sintheta 454 | 455 | if aggregating: 456 | for (i,g) in enumerate(groups): 457 | 458 | # step to group direction + step to group center 459 | x_g = x[g] 460 | y_g = y[g] 461 | c_theta = np.arctan2(g_y[i] - y_g, g_x[i] - x_g) 462 | 463 | x[g] = x_g + g_velocity[i] * g_costheta[i] + aggregation*np.cos(c_theta) 464 | y[g] = y_g + g_velocity[i] * g_sintheta[i] + aggregation*np.sin(c_theta) 465 | 466 | aggregating -= 1 467 | if aggregating == 0: deaggregating = 100 468 | 469 | else: 470 | for (i,g) in enumerate(groups): 471 | 472 | # step to group direction + step to group center 473 | x_g = x[g] 474 | y_g = y[g] 475 | c_theta = np.arctan2(g_y[i] - y_g, g_x[i] - x_g) 476 | 477 | x[g] = x_g + g_velocity[i] * g_costheta[i] 478 | y[g] = y_g + g_velocity[i] * g_sintheta[i] 479 | 480 | deaggregating -= 1 481 | if deaggregating == 0: aggregating = 10 482 | 483 | # node and group bounces on the margins 484 | b = np.where(x<0)[0] 485 | if b.size > 0: 486 | x[b] = - x[b]; costheta[b] = -costheta[b] 487 | g_idx = np.unique(g_ref[b]); g_costheta[g_idx] = -g_costheta[g_idx] 488 | b = np.where(x>MAX_X)[0] 489 | if b.size > 0: 490 | x[b] = 2*MAX_X - x[b]; costheta[b] = -costheta[b] 491 | g_idx = np.unique(g_ref[b]); g_costheta[g_idx] = -g_costheta[g_idx] 492 | b = np.where(y<0)[0] 493 | if b.size > 0: 494 | y[b] = - y[b]; sintheta[b] = -sintheta[b] 495 | g_idx = np.unique(g_ref[b]); g_sintheta[g_idx] = -g_sintheta[g_idx] 496 | b = np.where(y>MAX_Y)[0] 497 | if b.size > 0: 498 | y[b] = 2*MAX_Y - y[b]; sintheta[b] = -sintheta[b] 499 | g_idx = np.unique(g_ref[b]); g_sintheta[g_idx] = -g_sintheta[g_idx] 500 | 501 | # update info for nodes 502 | theta = U(0, 2*np.pi, NODES) 503 | costheta = np.cos(theta) 504 | sintheta = np.sin(theta) 505 | 506 | # update info for arrived groups 507 | g_fl = g_fl - g_velocity 508 | g_arrived = np.where(np.logical_and(g_velocity>0., g_fl<=0.))[0] 509 | 510 | if g_arrived.size > 0: 511 | g_theta = U(0, 2*np.pi, g_arrived) 512 | g_costheta[g_arrived] = np.cos(g_theta) 513 | g_sintheta[g_arrived] = np.sin(g_theta) 514 | g_fl[g_arrived] = FL_DISTR(g_arrived) 515 | g_velocity[g_arrived] = VELOCITY_DISTR(g_fl[g_arrived]) 516 | 517 | yield np.dstack((x,y))[0] 518 | -------------------------------------------------------------------------------- /ue_trace_10k.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruihuili/DRL_UAV_CellularNet/b8e960c827ae296c5ee832fb2706a694d444d4c4/ue_trace_10k.npy --------------------------------------------------------------------------------