├── DQN_env_customized_users_final1.py ├── DQN_env_customized_users_final_test1.py ├── README.md ├── Writedata.py ├── ddpg.py ├── environment_ddpg.py ├── environment_ddpg_tra.py ├── environment_optimize1.py ├── environment_tradition.py ├── test.ipynb ├── 传入用户-窃听 ├── DQN_env_customized_users_final.py ├── Writedata.py ├── __pycache__ │ ├── Writedata.cpython-38.pyc │ ├── environment_optimize1.cpython-38.pyc │ └── environment_tradition.cpython-38.pyc ├── environment_optimize1.py ├── environment_tradition.py ├── epoch_cost_optimize_E25_1 ├── epoch_cost_tradition_E25_1 ├── local_costs_epoch_E25_1 └── test.ipynb └── 改善环境版 ├── DQN_env_wu.py ├── Environment_Wu.py ├── Environment_Wu_tradition.py ├── Writedata.py ├── __pycache__ ├── Environment_Wu.cpython-38.pyc ├── Environment_Wu_tradition.cpython-38.pyc └── Writedata.cpython-38.pyc ├── dataset ├── E_max │ ├── all_costs.xls │ ├── local_costs.xls │ └── optimize_cost.xls ├── E_sum │ ├── all_costs.csv │ ├── local_costs.csv │ └── optimize_cost.csv ├── optimize_cost_colluding.xls ├── optimize_cost_non-colluding.xls └── tra_opi │ ├── all_costs.xls │ ├── all_costs1.xls │ ├── all_costs2.xls │ ├── local_costs.xls │ ├── local_costs1.xls │ ├── local_costs2.xls │ ├── optimize_cost.xls │ ├── optimize_cost1.xls │ ├── r_list.xls │ ├── r_list1.xls │ ├── tradition_costs.xls │ ├── tradition_costs1.xls │ └── tradition_costs2.xls └── test.ipynb /DQN_env_customized_users_final1.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import os 3 | # import Writedata 4 | os.environ['KMP_DUPLICATE_LIB_OK']='True' 5 | #2021。 10。27最终版。优化了算力和功率 6 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2， .....] 7 | # 输出的a为某个用户卸载率增加或者减少 8 | 9 | #2cap 2user 10 | 11 | import torch 12 | import numpy as np 13 | 14 | import matplotlib.pyplot as plt 15 | from environment_optimize1 import environment 16 | from environment_tradition import environment1 17 | 18 | 19 | #%% 20 | 21 | # user_num=input("please input user_num") 22 | 23 | #初始化参数----------------- 24 | user_num=2 25 | cap_num=2 26 | users_num=int(user_num) 27 | 28 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, [user_num, cap_num]) + np.complex('j') * 29 | np.random. normal(1, 1, [user_num, cap_num]))) ** 2 30 | Hn_e = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, user_num) + np.complex('j') * 31 | np.random.normal(1, 1, user_num))) ** 2##窃听者,[user1,user2] 32 | Hn_0=Hn[0] 33 | Hn_1=Hn[1] 34 | 35 | user_l = np.array([[np.random.randint(10,20), np.random.randint(150,200)]]) 36 | 37 | 38 | sus=2000 39 | epochs=100 40 | #---------------- 41 | env=environment(user_num=2,cap_num=2,W=[5,5], 42 | f_local=np.array([[1.4*10e8,0.43*10e8]]), 43 | omega=330 / 8, 44 | F_cap=np.array([[10*10e8,3*10e8]]),p_local=1e-1,p_tran=np.array([[2,3]]),lamuda=1, 45 | noise=0.1,Hn_0=Hn_0,Hn_1=Hn_1,Hn_e=Hn_e,user_l=user_l, 46 | suspend=sus 47 | ) 48 | 49 | #改 50 | #获取环境的相关参数: 51 | n_action,n_state=env.action_state_num() 52 | #其他超参数的定义： 53 | MEMORY_CAPACITY=5000#记忆库的存储大小 54 | greedy=0.9#贪婪概率 55 | discount=0.9#折扣率 56 | batch_size=64#一次训练32个数据 57 | TARGET_REPLACE_ITER=10#目标网络的更新频率 58 | 59 | #%% 60 | 61 | class Net(torch.nn.Module): 62 | def __init__(self): 63 | super(Net, self).__init__() 64 | self.linear1=torch.nn.Linear(n_state,20)#(10,20) 65 | self.linear1.weight.data.normal_(0, 0.1) 66 | self.linear2=torch.nn.Linear(20,30) 67 | self.linear2.weight.data.normal_(0, 0.1) 68 | self.linear3=torch.nn.Linear(30,18) 69 | self.linear3.weight.data.normal_(0, 0.1) 70 | self.linear4=torch.nn.Linear(18,n_action)#(30,12) 71 | self.linear4.weight.data.normal_(0, 0.1) 72 | self.relu=torch.nn.ReLU() 73 | def forward(self,x): 74 | x=self.relu(self.linear1(x)) 75 | x=self.relu(self.linear2(x)) 76 | x = self.relu(self.linear3(x)) 77 | q_values=self.linear4(x) 78 | return q_values 79 | 80 | 81 | #%% 82 | 83 | class DQN (): 84 | def __init__(self):#可能要改成def __init__(self,n_state,n_action) 85 | 86 | #记忆库：存s,a,r,s_ 一共6维 87 | self.memory=np.zeros((MEMORY_CAPACITY,n_state*2+2)) 88 | self.memory_count=0#记忆库计数，用于求index，以及当记忆库计数 > 记忆库容量时 开始学习 89 | 90 | #目标网络,参数更新慢。参数更新跟着agent_network走。用于预测q现实 91 | self.target_network=Net() 92 | 93 | #目标网络更新计数 94 | self.learn_step_counter=0 95 | 96 | #用于训练的网络，参数更新及时。用于预测q估计 97 | self.agent_network=Net() 98 | 99 | #优化器与损失器 100 | self.loss_func=torch.nn.MSELoss() 101 | self.optimizer=torch.optim.Adam(self.agent_network.parameters(),lr=0.001) 102 | def choose_action(self,s):#这里每次只输入一个样本。 103 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2] 104 | # 输出的action为0或1或2或3 105 | if np.random.uniform() < greedy:#选择最优动作 106 | #送入神经网络前，先要处理s成为torch型，且要求shape为(1,4) 107 | #s=torch.tensor(s,dtype=float).reshape(1,4) 108 | #s=s.to(torch.float32) 109 | s=torch.FloatTensor(s).reshape(-1,n_state) 110 | 111 | #把s送入神经网络得到a_value, a_value的shape 为（4，1）。并且得到的是四个动作q值 112 | action_value=self.agent_network.forward(s) 113 | #输出每个样本中预测的最大值，输出[最大值，下标]。只要下标 114 | _,action=torch.max(action_value.data,dim=1) 115 | #在从numpy转为int值 116 | action=action.numpy()[0] 117 | #此时action可能是0、1、2、3 118 | 119 | 120 | else:#随机选择动作 121 | action = np.random.randint(0, n_action) 122 | 123 | return action 124 | 125 | def store_transition(self,s,action,r,s_): 126 | #将四个值打包在一起 127 | 128 | #action为0or 1 or 2 or 3 129 | s=s.reshape(n_state) 130 | s_=s_.reshape(n_state) 131 | 132 | self.transition=np.hstack((s,[action,r],s_)) 133 | 134 | #如果记忆库满了, 就覆盖老数据 135 | index = self.memory_count % MEMORY_CAPACITY 136 | 137 | #传入记忆库 138 | 139 | self.memory[index,:]=self.transition 140 | self.memory_count+=1 141 | #print("目前记忆库存储的总数为",self.memory_count)#要删除 142 | 143 | def learn(self):#从记忆库中选择batch个数据进行训练学习 144 | #首先判断target_network是否需要更新 145 | 146 | if self.learn_step_counter % TARGET_REPLACE_ITER==0: 147 | self.target_network.load_state_dict(self.agent_network.state_dict())#更新目标网络参数 148 | self.learn_step_counter+=1 149 | 150 | #随机从memory中抽32个数据，即batch=32 151 | 152 | sample_index=np.random.choice(MEMORY_CAPACITY,batch_size)#从0-1999中抽32个数字，shape为（32，） 153 | batch_memory=self.memory[sample_index,:]#获取32个数据。 shape 为（32,6） 154 | 155 | 156 | b_s = torch.FloatTensor(batch_memory[:, :n_state]) 157 | b_a = torch.LongTensor(batch_memory[:, n_state:n_state+1].astype(int)) 158 | b_r = torch.FloatTensor(batch_memory[:, n_state+1:n_state+2]) 159 | b_s_ = torch.FloatTensor(batch_memory[:, -n_state:]) 160 | # b_memory shape (batch,6) 161 | # b_s shape (batch,2) 162 | # b_a shape (batch,1) 163 | # b_r shape (batch,1) 164 | # b_s_ shape (batch,2) 165 | 166 | 167 | #把样本state送入agent网络训练，得到q估计 168 | q_predict=self.agent_network(b_s)#shape 为 (32,4) 169 | q_predict=q_predict.gather(1,b_a)#shaoe 为 (32,1)。根据b_a找出q估计 170 | 171 | 172 | # #把样本送入target网络计算，得到Q(s+1) 173 | # q_next=self.target_network(b_s_).detach()# q_next 不进行反向传播, 所以使用detach。shape为 (32,4) 174 | # q_next,_=torch.max(q_next,dim=1)#输出每个样本中预测的最大值，输出[最大值，下标]。只要最大值 175 | # q_next=q_next.reshape((-1,1))#q_next的shape 为（32,1） 176 | # #计算q真实 177 | # q_target=b_r+discount*q_next#q_target的shape 为（32,1） 178 | 179 | #改用ddqn 180 | q_next=self.agent_network(b_s_).detach()# 181 | _,b_a_=torch.max(q_next,dim=1)#由dqn网络选出动作 182 | b_a_=b_a_.reshape((-1,1))#b_a_的shape 为（32,1） 183 | q_next_target=self.target_network(b_s_).detach() 184 | q_next=q_next_target.gather(1,b_a_)#q_next的shape 为（32,1 185 | #计算q真实 186 | q_target=b_r+discount*q_next#q_target的shape 为（32,1） 187 | 188 | 189 | 190 | loss=self.loss_func(q_predict,q_target) 191 | # if self.learn_step_counter%5000==0: 192 | # print("第",self.learn_step_counter,"次强化学习的loss为：",loss.item()) 193 | self.optimizer.zero_grad() 194 | loss.backward() 195 | self.optimizer.step() 196 | return loss.item() 197 | 198 | 199 | 200 | #%% 201 | 202 | dqn=DQN() 203 | dqn_tradition=DQN() 204 | epoch_cost_optimize=[] 205 | epoch_cost_tradition=[] 206 | 207 | 208 | 209 | 210 | epoch_list=[] 211 | loss_list=[] 212 | temp_wu_list=[] 213 | local_costs_epoch=[] 214 | all_costs_epoch=[] 215 | all_costs_epoch_tra=[] 216 | count=0 217 | 218 | 219 | #每次开始游戏前，重开游戏 220 | #s=env.reset()#s是（4,)的numpy 221 | # print("开始第",epoch+1,"轮游戏") 222 | # start = time.time() 223 | for epoch in range(epochs): 224 | print("本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch) 225 | step_loss = [] 226 | step_loss_tradition=[] 227 | local_costs=[] 228 | all_costs=[] 229 | all_costs_traditon = [] 230 | 231 | 232 | #参数随机化 233 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 234 | 235 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 236 | ##窃听者,[user1,user2] 237 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 238 | np.random.normal(1, 0.1, user_num))) ** 2 239 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 240 | np.random.normal(1, 0.1, user_num))) ** 2 241 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 242 | np.random.normal(1, 0.1, user_num))) ** 2 243 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 244 | np.random.normal(1, 0.1, user_num))) ** 2 245 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 246 | np.random.normal(1, 0.1, user_num))) ** 2 247 | Hn_e6 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 248 | np.random.normal(1, 0.1, user_num))) ** 2 249 | Hn_e7 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 250 | np.random.normal(1, 0.1, user_num))) ** 2 251 | Hn_e8 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 252 | np.random.normal(1, 0.1, user_num))) ** 2 253 | Hn_e9 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 254 | np.random.normal(1, 0.1, user_num))) ** 2 255 | Hn_e10 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 256 | np.random.normal(1, 0.1, user_num))) ** 2 257 | Hn_e11 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 258 | np.random.normal(1, 0.1, user_num))) ** 2 259 | Hn_e12 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 260 | np.random.normal(1, 0.1, user_num))) ** 2 261 | Hn_e13 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 262 | np.random.normal(1, 0.1, user_num))) ** 2 263 | Hn_e14 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 264 | np.random.normal(1, 0.1, user_num))) ** 2 265 | Hn_e15 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 266 | np.random.normal(1, 0.1, user_num))) ** 2 267 | 268 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10+Hn_e11 + Hn_e12 + Hn_e13 + Hn_e14 + Hn_e15 269 | # Hn_e=Hn_e1+Hn_e2+Hn_e3+Hn_e4+Hn_e5+Hn_e6+Hn_e7+Hn_e8+Hn_e9+Hn_e10 270 | Hn_e=Hn_e1 271 | # Hn_e=Hn_e1 272 | 273 | # Hn_e=Hn_e1 274 | Hn_0 = Hn[0] 275 | Hn_1 = Hn[1] 276 | 277 | while((Hn_e > Hn_1).any() or (Hn_e > Hn_0).any()): 278 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 279 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 280 | ##窃听者,[user1,user2] 281 | 282 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 283 | np.random.normal(1, 0.1, user_num))) ** 2 284 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 285 | np.random.normal(1, 0.1, user_num))) ** 2 286 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 287 | np.random.normal(1, 0.1, user_num))) ** 2 288 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 289 | np.random.normal(1, 0.1, user_num))) ** 2 290 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 291 | np.random.normal(1, 0.1, user_num))) ** 2 292 | Hn_e6 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 293 | np.random.normal(1, 0.1, user_num))) ** 2 294 | Hn_e7 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 295 | np.random.normal(1, 0.1, user_num))) ** 2 296 | Hn_e8 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 297 | np.random.normal(1, 0.1, user_num))) ** 2 298 | Hn_e9 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 299 | np.random.normal(1, 0.1, user_num))) ** 2 300 | Hn_e10 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 301 | np.random.normal(1, 0.1, user_num))) ** 2 302 | Hn_e11 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 303 | np.random.normal(1, 0.1, user_num))) ** 2 304 | Hn_e12 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 305 | np.random.normal(1, 0.1, user_num))) ** 2 306 | Hn_e13 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 307 | np.random.normal(1, 0.1, user_num))) ** 2 308 | Hn_e14 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 309 | np.random.normal(1, 0.1, user_num))) ** 2 310 | Hn_e15 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 311 | np.random.normal(1, 0.1, user_num))) ** 2 312 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10 + Hn_e11 + Hn_e12 + Hn_e13 + Hn_e14 + Hn_e15 313 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10 314 | Hn_e = Hn_e1 315 | # Hn_e = Hn_e1 316 | 317 | #共谋：sum 318 | Hn_0 = Hn[0] 319 | Hn_1 = Hn[1] 320 | 321 | 322 | # user_l = np.array([[np.random.randint(10,60), np.random.randint(550,600)]]) 323 | user_l=np.array([[10,550]]) 324 | #-------------------------- 325 | #------------优化功率和算力------------- 326 | env = environment(user_num=2, cap_num=2, W=5.0, 327 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 328 | omega=330 / 8, 329 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), lamuda=1, 330 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1,Hn_e=Hn_e,user_l=user_l, 331 | suspend=sus 332 | ) 333 | s = env.reset() 334 | while True:#每次循环执行一个动作 335 | 336 | #通过神经网络，输入状态，获取动作。(不带学习) 337 | action=dqn.choose_action(s) 338 | 339 | #执行，获得环境反馈 340 | s_,r,done,tc_wald,local_cost,all_cost=env.step(action) 341 | step_loss.append(tc_wald) 342 | local_costs.append(local_cost) 343 | all_costs.append(all_cost) 344 | #将环境的反馈s,a,r,s_传入记忆库。用于经验回放 345 | dqn.store_transition(s,action,r,s_) 346 | 347 | #神经网络学习（experience replay 经验回放） 348 | if dqn.memory_count > MEMORY_CAPACITY: 349 | count+=1 350 | loss=dqn.learn() 351 | 352 | 353 | #判断是否结束 354 | if done !=0: 355 | epoch_cost_optimize.append(done) 356 | local_costs_epoch.append(local_cost) 357 | all_costs_epoch.append(all_cost) 358 | break 359 | 360 | #更新状态，进入下一次循环 361 | s=s_ 362 | #-------------------------------------------- 363 | 364 | 365 | # ------------传统方法------------- 366 | env_tradition = environment1(user_num=2, cap_num=2, W=5.0, 367 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 368 | omega=330 / 8, 369 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), 370 | lamuda=1, 371 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1, Hn_e=Hn_e, user_l=user_l, 372 | suspend=sus 373 | ) 374 | s=env_tradition.reset() 375 | while True: # 每次循环执行一个动作 376 | 377 | # 通过神经网络，输入状态，获取动作。(不带学习) 378 | action = dqn_tradition.choose_action(s) 379 | 380 | # 执行，获得环境反馈 381 | s_, r, done, tc_wald, local_cost,all_cost_traditon = env_tradition.step(action) 382 | step_loss_tradition.append(tc_wald) 383 | all_costs_traditon.append(all_cost_traditon) 384 | # local_costs.append(local_cost) 385 | 386 | # 将环境的反馈s,a,r,s_传入记忆库。用于经验回放 387 | dqn_tradition.store_transition(s, action, r, s_) 388 | 389 | # 神经网络学习（experience replay 经验回放） 390 | if dqn_tradition.memory_count > MEMORY_CAPACITY: 391 | count += 1 392 | loss = dqn_tradition.learn() 393 | 394 | # 判断是否结束 395 | if done!=0: 396 | epoch_cost_tradition.append(done) 397 | all_costs_epoch_tra.append(all_cost_traditon) 398 | break 399 | 400 | # 更新状态，进入下一次循环 401 | s = s_ 402 | # -------------------------------------------- 403 | 404 | 405 | plt.plot(range(sus), step_loss, color='skyblue', label='optimize_F_P') 406 | plt.plot(range(sus), step_loss_tradition, color='red', label='tradition') 407 | plt.plot(range(sus),local_costs,color="yellow",label="local") 408 | plt.plot(range(sus), all_costs_traditon, color="green", label='all_off_tra') 409 | plt.legend(loc="best") 410 | 411 | plt.xlabel("step_dqn") 412 | plt.ylabel("costs") 413 | plt.show() 414 | 415 | 416 | # plt.plot(range(epochs),local_costs_epoch, color='yellow', label='locals') 417 | plt.plot(range(epochs),epoch_cost_optimize,color='skyblue', label='optimize_F_P') 418 | plt.plot(range(epochs),epoch_cost_tradition,color='red', label='tradition') 419 | plt.plot(range(epochs),all_costs_epoch,color='purple', label="all_off") 420 | # plt.plot(range(epochs),all_costs_epoch_tra,color='mediumpurple', label="all_off_Tra") 421 | 422 | # Writedata.write_to_excel(epoch_cost_optimize,1,"epoch_cost_optimize_E5_3") 423 | # Writedata.write_to_excel(epoch_cost_tradition,1,"epoch_cost_tradition_E5_3") 424 | # Writedata.write_to_excel(local_costs_epoch,1,"local_costs_epoch_E5_3") 425 | 426 | plt.xlabel("epoch") 427 | plt.ylabel("costs") 428 | plt.legend(loc="best") 429 | plt.show() 430 | 431 | #保存数据 432 | 433 | Writedata.write_to_excel(epoch_cost_optimize,1,"./dataset/tra_opi/optimize_cost2.xls")#优化 434 | Writedata.write_to_excel(all_costs_epoch,1,"./dataset/tra_opi/all_costs2.xls")#优化的全卸载 435 | Writedata.write_to_excel(local_costs_epoch,1,"./dataset/tra_opi/local_costs2.xls")#本地 436 | Writedata.write_to_excel(epoch_cost_tradition,1,"./dataset/tra_opi/tradition_costs2.xls")#传统方法 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net 2 | 基于边缘计算网络的卸载优化及资源优化 3 | 4 | 考虑了一个多用户单窃听者的移动边缘计算(MEC)网络，通过使用深度强化学习算法，并结合凸优化，对MEC网络进行了卸载优化与资源优化。 5 | 6 | 仿真结果如下所示（部分）： 7 | 8 | ![max](https://user-images.githubusercontent.com/77431730/221493000-b2f0a5cd-3c26-48c4-ad89-869bf80f934d.jpg) 9 | 上图展示的是窃听者共谋的结果。从图中可以明显看出，仅仅跑了100epoch，DQN所跑出来的结果明显优于全本地与全卸载。并且趋近于收敛趋势（笔者自己跑的epoch太少了，跑500个epoch应该就能收敛了）。说明DQN从网络中学习到了相关知识，并将网络性能优化。 10 | ![sum](https://user-images.githubusercontent.com/77431730/221493229-ee7f7582-697d-4777-be5c-8606b959b3e0.jpg) 11 | 上图展示的是窃听者非共谋的结果。由于非共谋的窃听者会对网络环境产生较大的干扰，所以从途中可以明显看出此图较上图而言存在着明显的波动。但是在如此大干扰情况下，DQN仍能从中学习到知识，并将网络性能优化。 12 | 13 | ![traditon_compare1](https://user-images.githubusercontent.com/77431730/221493751-227b3460-9bf4-4cbc-99c2-6f49b6ce2df8.jpg) 14 | 最后一张图展示的是，结合了传统的MEC优化算法，在这过程中，我们所提出的DQN分配卸载策略，凸优化解决资源分配的算法仍比传统的MEC优化算法更优。这说明我们所提出的算法相较于传统优化算法有着更好的优势 15 | 16 | 系统模型图如下所示： 17 | fig1 18 | 19 | DQN的结构如下两张图所示： 20 | 21 | ![fig2](https://user-images.githubusercontent.com/77431730/221492887-2c015335-c328-42f6-a2b8-cfab977f744b.png) 22 | 23 | ![fig3](https://user-images.githubusercontent.com/77431730/221492906-7e82e9a6-dd97-454d-aca7-7c4b67f6b7df.png) 24 | 25 | image 26 | -------------------------------------------------------------------------------- /Writedata.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | from xlwt import * 3 | 4 | 5 | # 多次实验时写入数据代码 6 | def write_data(data_list, sheet_name, iteration): 7 | for each_i in range(len(data_list)): 8 | sheet_name.write(each_i, iteration, str(data_list[each_i])) 9 | 10 | 11 | def create_csv(sheet1, data_list, iteration=1): 12 | write_data(data_list=data_list, sheet_name=sheet1, iteration=iteration) 13 | 14 | 15 | def write_to_excel(cost_list, is_one, filename): 16 | wb = Workbook() 17 | sheet1 = wb.add_sheet(sheetname='cost') 18 | if is_one: 19 | create_csv(sheet1=sheet1, data_list=cost_list) 20 | else: 21 | for index in range(len(cost_list)): 22 | create_csv(sheet1=sheet1, data_list=cost_list[index], iteration=index) 23 | wb.save(filename) 24 | 25 | 26 | def read_file(h_file, task_file): 27 | channel_uc = xlrd.open_workbook(h_file) 28 | task_file = xlrd.open_workbook(task_file) 29 | uc_sheet = channel_uc.sheets()[0] 30 | task_sheet = task_file.sheets()[0] 31 | 32 | return uc_sheet, task_sheet 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /ddpg.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import os 3 | os.environ['KMP_DUPLICATE_LIB_OK']='True' 4 | # S=[任务量1✖️卸载率1，任务量2✖️卸载率2] 5 | #A=[卸载率1，卸载率2] 6 | import numpy as np 7 | import torch 8 | # from environment1_customized_users_final import environment 9 | from environment_ddpg import environment 10 | from environment_ddpg_tra import environment1 11 | import matplotlib.pyplot as plt 12 | 13 | #%% 14 | 15 | # user_num=input("please input user_num") 16 | 17 | #初始化参数----------------- 18 | user_num=2 19 | cap_num=2 20 | users_num=int(user_num) 21 | 22 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, [user_num, cap_num]) + np.complex('j') * 23 | np.random. normal(1, 1, [user_num, cap_num]))) ** 2 24 | Hn_e = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, user_num) + np.complex('j') * 25 | np.random.normal(1, 1, user_num))) ** 2##窃听者,[user1,user2] 26 | Hn_0=Hn[0] 27 | Hn_1=Hn[1] 28 | 29 | user_l = np.array([[np.random.randint(10,20), np.random.randint(150,200)]]) 30 | 31 | 32 | sus=2000 33 | epochs=100 34 | #---------------- 35 | env=environment(user_num=2,cap_num=2,W=[5,5], 36 | f_local=np.array([[1.4*10e8,0.43*10e8]]), 37 | omega=330 / 8, 38 | F_cap=np.array([[10*10e8,3*10e8]]),p_local=1e-1,p_tran=np.array([[2,3]]),lamuda=1, 39 | noise=0.1,Hn_0=Hn_0,Hn_1=Hn_1,Hn_e=Hn_e,user_l=user_l, 40 | suspend=sus 41 | ) 42 | 43 | 44 | 45 | 46 | n_action,n_state=env.action_state_num() 47 | 48 | memory_size=5000#记忆库大小 49 | batch_size=64#batch大小 50 | discount=0.9#折扣率 51 | var1=1#var为方差，用于对a添加高斯分布的噪音 52 | var=1 53 | TAU = 0.01 #用于逐步更新target网络 54 | 55 | #%% 56 | 57 | #先构建神经网络： 58 | 59 | #Actor 60 | 61 | class Aactor(torch.nn.Module): 62 | def __init__(self): 63 | super(Aactor,self).__init__() 64 | self.fc1=torch.nn.Linear(n_state,30) 65 | self.fc1.weight.data.normal_(0, 0.1) 66 | 67 | self.fc2_user1=torch.nn.Linear(30,15)#user1 68 | self.fc2_user1.weight.data.normal_(0, 0.1) 69 | 70 | self.fc2_user2=torch.nn.Linear(30,15)##user2 71 | self.fc2_user2.weight.data.normal_(0, 0.1) 72 | 73 | self.fc3_user1=torch.nn.Linear(15,8)#user1 74 | self.fc3_user1.weight.data.normal_(0, 0.1) 75 | 76 | self.fc3_user2 = torch.nn.Linear(15, 8)##user2 77 | self.fc3_user2.weight.data.normal_(0, 0.1) 78 | 79 | self.fc4_user1 = torch.nn.Linear(8, 3)#user1 80 | self.fc4_user1.weight.data.normal_(0, 0.1) 81 | 82 | self.fc4_user2 = torch.nn.Linear(8, 3)#user2 83 | self.fc4_user2.weight.data.normal_(0, 0.1) 84 | 85 | self.relu=torch.nn.ReLU() 86 | self.softmax=torch.nn.Softmax() 87 | self.sigmoid=torch.nn.Sigmoid() 88 | def forward(self,s): 89 | x=self.sigmoid(self.fc1(s)) 90 | x_user1=self.sigmoid(self.fc2_user1(x)) 91 | x_user1 = self.sigmoid(self.fc3_user1(x_user1)) 92 | x_user1=self.fc4_user1(x_user1) 93 | x_user1 = self.softmax(x_user1) 94 | 95 | x_user2=self.sigmoid(self.fc2_user2(x)) 96 | x_user2 = self.sigmoid(self.fc3_user2(x_user2)) 97 | x_user2 = self.softmax(self.fc4_user2(x_user2)) 98 | x = torch.cat((x_user1, x_user2), dim=1) 99 | 100 | return x 101 | 102 | class Critic(torch.nn.Module): 103 | def __init__(self): 104 | super(Critic, self).__init__() 105 | self.fc_a1=torch.nn.Linear(n_state,16) 106 | self.fc_a1.weight.data.normal_(0, 0.1) 107 | 108 | self.fc_b1=torch.nn.Linear(n_action,10) 109 | self.fc_b1.weight.data.normal_(0, 0.1) 110 | 111 | self.fc_ab1=torch.nn.Linear(26,18)#采用简单的加合 112 | self.fc_ab1.weight.data.normal_(0, 0.1) 113 | 114 | self.fc_ab2=torch.nn.Linear(18,1) 115 | self.fc_ab2.weight.data.normal_(0, 0.1) 116 | 117 | self.relu=torch.nn.ReLU() 118 | def forward(self,s,a):#输出s和a，分别输入到一开始不同的网络，然后在第二层的时候加合在一起，在一起训练 119 | s=self.fc_a1(s) 120 | a=self.fc_b1(a) 121 | x=torch.cat((s,a),dim=1) 122 | x=self.relu(x) 123 | x=self.relu(self.fc_ab1(x)) 124 | x=self.fc_ab2(x) 125 | return x 126 | 127 | #%% 128 | 129 | # criterion=torch.nn.CrossEntropyLoss() 130 | # optimizer=torch.optim.Adam(model.parameters(),lr=3e-4) 131 | 132 | class DDPG(): 133 | def __init__(self): 134 | #构建记忆库 135 | self.memory=np.zeros((memory_size,n_state*2+n_action+1),dtype=np.float32) 136 | self.memory_count=0#用于标记存了多少 137 | #构建Actor网络 138 | self.Actor_eval=Aactor() 139 | self.Actor_target=Aactor() 140 | self.optimizer_Actor=torch.optim.Adam(self.Actor_eval.parameters(),lr=3e-4) 141 | 142 | #构建Critic网络 143 | self.Critic_eval=Critic() 144 | self.Critic_target=Critic() 145 | self.optimizer_Critic=torch.optim.Adam(self.Critic_eval.parameters(),lr=3e-4) 146 | self.loss_func=torch.nn.MSELoss() 147 | 148 | def choose_action(self,s):#输入状态s，输出动作 149 | # s=torch.FloatTensor(s).reshape(-1,n_state) 150 | # a=self.Actor_eval(s).detach()#a为[1,1] 151 | # a=a.numpy()[0]#a变为array([0.07249027])，因为step（a）需要传入array 152 | 153 | s = torch.FloatTensor(s) 154 | a=self.Actor_eval(s).detach() 155 | a=a.numpy() 156 | return a 157 | 158 | # return a 159 | 160 | def store_transition(self,s,a,r,s_): 161 | 162 | s=s.reshape(-1,) 163 | a=a.reshape(-1,) 164 | s_=s_.reshape(-1,) 165 | 166 | transition=np.hstack((s,a,[r],s_)) 167 | 168 | #获得索引 169 | self.index=self.memory_count%memory_size 170 | 171 | #存入memory 172 | self.memory[self.index,:]=transition 173 | self.memory_count+=1 174 | 175 | # transition = np.hstack((s, a, [r], s_)) 176 | # index = self.memory_count % memory_size # replace the old memory with new memory 177 | # self.memory[index, :] = transition 178 | # self.memory_count += 1 179 | 180 | if self.memory_count==memory_size: 181 | print("开始学习！！") 182 | 183 | def learn(self): 184 | #具体更新，还没看 185 | for x in self.Actor_target.state_dict().keys(): 186 | eval('self.Actor_target.' + x + '.data.mul_((1-TAU))') 187 | eval('self.Actor_target.' + x + '.data.add_(TAU*self.Actor_eval.' + x + '.data)') 188 | for x in self.Critic_target.state_dict().keys(): 189 | eval('self.Critic_target.' + x + '.data.mul_((1-TAU))') 190 | eval('self.Critic_target.' + x + '.data.add_(TAU*self.Critic_eval.' + x + '.data)') 191 | 192 | #随机抽样 193 | sample_index=np.random.choice(memory_size,size=batch_size)#从0-1999中抽batch_size个数,shape为[32,] 194 | batch_memory=self.memory[sample_index,:]#batch_memory的shape为[32,8] 195 | 196 | b_s = torch.FloatTensor(batch_memory[:, :n_state])#shape为[32,4] 197 | b_a = torch.FloatTensor(batch_memory[:, n_state:n_state+n_action])#shape为[32,4] 198 | b_r = torch.FloatTensor(batch_memory[:, n_state+n_action:n_state+n_action+1])#shape为[32,1] 199 | b_s_ = torch.FloatTensor(batch_memory[:, -n_state:])#shape为[32,3] 200 | 201 | 202 | #计算由Actor网络计算出来的S的a值，并打分 203 | a=self.Actor_eval(b_s)#shape:[32,1] 204 | q=self.Critic_eval(b_s,a)#shape:[32,1] 205 | 206 | #更新Actor_eval网络 207 | loss2=-torch.mean(q) 208 | self.optimizer_Actor.zero_grad() 209 | loss2.backward() 210 | self.optimizer_Actor.step() 211 | 212 | #计算St的q_eval 213 | q_eval=self.Critic_eval(b_s,b_a) 214 | #计算St+1的q_target 215 | a_=self.Actor_target(b_s_).detach()#shape:[32,1] 216 | q_target=self.Critic_target(b_s_,a_).detach()#shape:[32,1] 217 | q_target=b_r+discount*q_target#shape:[32,1] 218 | 219 | # 220 | q_eval=q_eval.reshape(-1,1) 221 | q_target=q_target.reshape(-1,1) 222 | 223 | #更新critic_eval网络 224 | loss=self.loss_func(q_eval, q_target) 225 | self.optimizer_Critic.zero_grad() 226 | loss.backward() 227 | self.optimizer_Critic.step() 228 | 229 | 230 | ddpg=DDPG() 231 | ddpg_tradition=DDPG() 232 | epoch_cost_optimize=[] 233 | epoch_cost_tradition=[] 234 | 235 | epoch_list=[] 236 | loss_list=[] 237 | temp_wu_list=[] 238 | local_costs_epoch=[] 239 | all_costs_epoch=[] 240 | all_costs_epoch_tra=[] 241 | count=0 242 | 243 | 244 | #%% 245 | 246 | for epoch in range(epochs): 247 | print("本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch) 248 | step_loss = [] 249 | step_loss_tradition = [] 250 | local_costs = [] 251 | all_costs = [] 252 | all_costs_traditon = [] 253 | 254 | # 参数随机化 255 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 256 | 257 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 258 | ##窃听者,[user1,user2] 259 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 260 | np.random.normal(1, 0.1, user_num))) ** 2 261 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 262 | np.random.normal(1, 0.1, user_num))) ** 2 263 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 264 | np.random.normal(1, 0.1, user_num))) ** 2 265 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 266 | np.random.normal(1, 0.1, user_num))) ** 2 267 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 268 | np.random.normal(1, 0.1, user_num))) ** 2 269 | Hn_e6 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 270 | np.random.normal(1, 0.1, user_num))) ** 2 271 | Hn_e7 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 272 | np.random.normal(1, 0.1, user_num))) ** 2 273 | Hn_e8 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 274 | np.random.normal(1, 0.1, user_num))) ** 2 275 | Hn_e9 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 276 | np.random.normal(1, 0.1, user_num))) ** 2 277 | Hn_e10 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 278 | np.random.normal(1, 0.1, user_num))) ** 2 279 | Hn_e11 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 280 | np.random.normal(1, 0.1, user_num))) ** 2 281 | Hn_e12 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 282 | np.random.normal(1, 0.1, user_num))) ** 2 283 | Hn_e13 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 284 | np.random.normal(1, 0.1, user_num))) ** 2 285 | Hn_e14 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 286 | np.random.normal(1, 0.1, user_num))) ** 2 287 | Hn_e15 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 288 | np.random.normal(1, 0.1, user_num))) ** 2 289 | 290 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10+Hn_e11 + Hn_e12 + Hn_e13 + Hn_e14 + Hn_e15 291 | # Hn_e=Hn_e1+Hn_e2+Hn_e3+Hn_e4+Hn_e5+Hn_e6+Hn_e7+Hn_e8+Hn_e9+Hn_e10 292 | Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 293 | # Hn_e=Hn_e1 294 | 295 | # Hn_e=Hn_e1 296 | Hn_0 = Hn[0] 297 | Hn_1 = Hn[1] 298 | 299 | while ((Hn_e > Hn_1).any() or (Hn_e > Hn_0).any()): 300 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 301 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 302 | ##窃听者,[user1,user2] 303 | 304 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 305 | np.random.normal(1, 0.1, user_num))) ** 2 306 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 307 | np.random.normal(1, 0.1, user_num))) ** 2 308 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 309 | np.random.normal(1, 0.1, user_num))) ** 2 310 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 311 | np.random.normal(1, 0.1, user_num))) ** 2 312 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 313 | np.random.normal(1, 0.1, user_num))) ** 2 314 | Hn_e6 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 315 | np.random.normal(1, 0.1, user_num))) ** 2 316 | Hn_e7 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 317 | np.random.normal(1, 0.1, user_num))) ** 2 318 | Hn_e8 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 319 | np.random.normal(1, 0.1, user_num))) ** 2 320 | Hn_e9 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 321 | np.random.normal(1, 0.1, user_num))) ** 2 322 | Hn_e10 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 323 | np.random.normal(1, 0.1, user_num))) ** 2 324 | Hn_e11 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 325 | np.random.normal(1, 0.1, user_num))) ** 2 326 | Hn_e12 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 327 | np.random.normal(1, 0.1, user_num))) ** 2 328 | Hn_e13 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 329 | np.random.normal(1, 0.1, user_num))) ** 2 330 | Hn_e14 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 331 | np.random.normal(1, 0.1, user_num))) ** 2 332 | Hn_e15 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 333 | np.random.normal(1, 0.1, user_num))) ** 2 334 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10 + Hn_e11 + Hn_e12 + Hn_e13 + Hn_e14 + Hn_e15 335 | # Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 + Hn_e6 + Hn_e7 + Hn_e8 + Hn_e9 + Hn_e10 336 | Hn_e = Hn_e1 + Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 337 | # Hn_e = Hn_e1 338 | 339 | # 共谋：sum 340 | Hn_0 = Hn[0] 341 | Hn_1 = Hn[1] 342 | 343 | user_l = np.array([[np.random.randint(10, 60), np.random.randint(550, 600)]]) 344 | # ------------优化功率和算力------------- 345 | env = environment(user_num=2, cap_num=2, W=5.0, 346 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 347 | omega=330 / 8, 348 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), lamuda=1, 349 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1, Hn_e=Hn_e, user_l=user_l, 350 | suspend=sus 351 | ) 352 | s = env.reset() 353 | while True: 354 | 355 | #选择动作 356 | action=ddpg.choose_action(s) 357 | 358 | # 让动作加入高斯分布后输入环境 359 | # action = np.clip(np.random.normal(action, var1), 0, 1) 360 | 361 | # 执行，获得环境反馈 362 | s_, r, done, tc_wald, local_cost, all_cost = env.step(action) 363 | step_loss.append(tc_wald) 364 | local_costs.append(local_cost) 365 | all_costs.append(all_cost) 366 | 367 | #存入记忆库 368 | ddpg.store_transition(s,action,r,s_) 369 | 370 | #学习 371 | if ddpg.memory_count>memory_size: 372 | count+=1 373 | va1r=var1*0.9995 #逐步减少噪音 374 | ddpg.learn() 375 | 376 | #判断是否结束 377 | if done: 378 | epoch_cost_optimize.append(done) 379 | local_costs_epoch.append(local_cost) 380 | all_costs_epoch.append(all_cost) 381 | break 382 | 383 | s=s_ 384 | # ------------传统方法------------- 385 | env_tradition = environment1(user_num=2, cap_num=2, W=5.0, 386 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 387 | omega=330 / 8, 388 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), 389 | lamuda=1, 390 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1, Hn_e=Hn_e, user_l=user_l, 391 | suspend=sus 392 | ) 393 | s = env_tradition.reset() 394 | while True: 395 | 396 | # 选择动作 397 | action = ddpg_tradition.choose_action(s) 398 | 399 | # 让动作加入高斯分布后输入环境 400 | # action = np.clip(np.random.normal(action, var), 0, 1) 401 | 402 | # 执行，获得环境反馈 403 | s_, r, done, tc_wald, local_cost,all_cost_traditon = env_tradition.step(action) 404 | step_loss_tradition.append(tc_wald) 405 | all_costs_traditon.append(all_cost_traditon) 406 | 407 | # 存入记忆库 408 | ddpg_tradition.store_transition(s, action, r, s_) 409 | 410 | # 学习 411 | if ddpg_tradition.memory_count > memory_size: 412 | count += 1 413 | ddpg_tradition.learn() 414 | 415 | # 判断是否结束 416 | if done: 417 | epoch_cost_tradition.append(done) 418 | all_costs_epoch_tra.append(all_cost_traditon) 419 | break 420 | 421 | s = s_ 422 | 423 | 424 | plt.plot(range(sus), step_loss, color='skyblue', label='optimize_F_P') 425 | plt.plot(range(sus), step_loss_tradition, color='red', label='tradition') 426 | # plt.plot(range(sus),local_costs,color="yellow",label="local") 427 | # plt.plot(range(sus),all_costs,color="purple",label="all_off") 428 | # plt.plot(range(sus), all_costs_traditon, color="green", label='all_off_tra') 429 | plt.legend(loc="best") 430 | 431 | plt.xlabel("step_dqn_15") 432 | plt.ylabel("costs") 433 | plt.show() 434 | 435 | 436 | plt.plot(range(epochs),local_costs_epoch, color='yellow', label='locals') 437 | plt.plot(range(epochs),epoch_cost_optimize,color='skyblue', label='optimize_F_P') 438 | plt.plot(range(epochs),epoch_cost_tradition,color='red', label='tradition') 439 | plt.plot(range(epochs),all_costs_epoch,color='purple', label="all_off") 440 | plt.plot(range(epochs),all_costs_epoch_tra,color='mediumpurple', label="all_off_Tra") 441 | 442 | # Writedata.write_to_excel(epoch_cost_optimize,1,"epoch_cost_optimize_E5_3") 443 | # Writedata.write_to_excel(epoch_cost_tradition,1,"epoch_cost_tradition_E5_3") 444 | # Writedata.write_to_excel(local_costs_epoch,1,"local_costs_epoch_E5_3") 445 | 446 | plt.xlabel("epoch_e15") 447 | plt.ylabel("costs") 448 | plt.legend(loc="best") 449 | plt.show() 450 | -------------------------------------------------------------------------------- /environment_ddpg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran=p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]])# 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]])# 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道CAPA分给两个用户的信道 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=copy.deepcopy(self.user_offrate) 67 | self.all_off=np.array([[0.0,0.5,0.5,0.0,0.5,0.5]]) 68 | # print("初始化时的卸载率为：", self.user_offrate) 69 | 70 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 71 | 72 | self.lamuda = lamuda 73 | self.noise = noise 74 | self.i = 0 75 | self.cost_list = [] # 记录成本 76 | self.epoch_list = [] 77 | self.quancost_list = [] 78 | self.wucost_list = [] 79 | 80 | self.task_action = 1 # 任务每次变化率（任务卸载率） 81 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 83 | self.p_local = p_local 84 | 85 | 86 | 87 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 88 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 89 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 90 | self.A = 1.0 91 | self.B = 0.0 92 | self.C = 0.0 93 | self.D = 1.0 94 | self.E = 0.0 95 | self.F = 0.0 96 | 97 | self.temp_tc1 = self.total_cost(self.user_offrate)#本地 98 | self.tc_wald = self.total_cost(self.user_offrate) 99 | self.all_cost=self.total_cost(self.all_off)#全卸载 100 | # print("本地的cost为：", self.temp_tc1) 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | def reset(self): # 重开，返回初始化s 110 | 111 | self.user_S = self.l * self.user_offrate #任务量，1x6 112 | #总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 113 | self.P_F=np.append(self.p_tran,self.F_cap,axis=1) 114 | S_=np.append(self.user_S,self.P_F,axis=1) 115 | return S_ 116 | 117 | def step(self, action): 118 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 119 | self.i+=1 120 | self.user_offrate=action 121 | 122 | 123 | 124 | s_ = self.l * self.user_offrate 125 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 126 | s_ = np.append(s_, self.P_F, axis=1) 127 | 128 | # 计算更新卸载率后的成本 129 | tc = self.total_cost(self.user_offrate) 130 | 131 | #本地 132 | self.temp_tc1=self.total_cost(self.judge) 133 | 134 | 135 | # if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 136 | # r = -100 137 | # elif tc < self.tc_wald: 138 | # r = 100 139 | # else: 140 | # r = -100 141 | tc_diff=tc - self.tc_wald 142 | if tc_diff>=5.0: 143 | r=-200 144 | elif tc_diff>0.0 and tc_diff<5.0: 145 | r=-100 146 | elif tc_diff<=-5.0: 147 | r=200 148 | elif tc_diff<0.0 and tc_diff>-5.0: 149 | r=100 150 | elif tc_diff==0.0: 151 | r=-200 152 | 153 | self.user_S = s_ # 更新状态 154 | if self.i % 1000==0: 155 | print("----------------------------------") 156 | print("执行了第", self.i, "次动作") 157 | print("此轮更新卸载率后的成本为:", tc) 158 | print("此时卸载率为：", self.user_offrate) 159 | 160 | 161 | self.tc_wald = tc 162 | 163 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 164 | if self.i == self.suspend: 165 | done = self.tc_wald 166 | self.i=0 167 | 168 | 169 | 170 | return s_, r, done, self.tc_wald,self.temp_tc1,self.all_cost 171 | 172 | #根据动作，改变对应的卸载率 173 | def offrate_normalization(self,number):#如增加，add_subtraction=1 174 | # w=self.i/5.0 175 | global w 176 | if self.i<500: 177 | w=1.0 178 | if self.i<1000: 179 | w=2.0 180 | if self.i<1500: 181 | w=3.0 182 | if self.i<2000: 183 | w=4.0 184 | if number==0: 185 | self.A+=w 186 | elif number==1: 187 | self.B+=w 188 | elif number==2: 189 | self.C+=w 190 | elif number==3: 191 | self.D+=w 192 | elif number==4: 193 | self.E+=w 194 | else: 195 | self.F+=w 196 | 197 | 198 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 199 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 200 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 201 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 202 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 203 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 204 | 205 | 206 | 207 | def action_state_num(self): # 返回动作与环境个数 208 | n_action = self.user_num * (self.cap_num+1)#dqn 209 | n_state = self.user_num*(self.cap_num +1) +self.cap_num*2 210 | return n_action, n_state 211 | 212 | def Time(self, user_offrate): 213 | # 计算香农公式Wn 214 | 215 | # # #-----优化带宽------------- 216 | # if self.user_offrate.all()!=0: 217 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 218 | # else: 219 | # w_ = self.W / self.user_num 220 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 221 | # #------------------- 222 | 223 | #--------均分带宽-------------- 224 | # w_0 = self.W0 / self.user_num#shape[1,2] 225 | # w_1 = self.W1 / self.user_num#shape[1,2] 226 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 227 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 228 | #----------------------------- 229 | 230 | 231 | 232 | 233 | if (user_offrate!=self.judge).any(): 234 | 235 | #----解析------- 236 | a_A=W_A_user*np.log2(self.Hn_0/self.Hn_e) 237 | a_B=W_B_user*np.log2(self.Hn_1/self.Hn_e) 238 | 239 | b_A=W_A_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_0) / math.log(2) 240 | b_B=W_B_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_1) / math.log(2) 241 | 242 | 243 | 244 | #按user分 245 | 246 | 247 | b_user1=np.array([[b_A[0][0],b_B[0][0]]]) 248 | b_user2 = np.array([[b_A[0][1], b_B[0][1]]]) 249 | 250 | self.l1=self.l[0][1:3]#传输与cap计算的任务量 251 | self.l2=self.l[0][4:] 252 | user_offrate_user1=user_offrate[0][1:3]#user1的传输与cap计算的卸载率 253 | user_offrate_user2=user_offrate[0][4:] 254 | 255 | a_user1 =np.array([[a_A[0][0], a_B[0][0]]]) 256 | a_user2 = np.array([[a_A[0][1], a_B[0][1]]]) 257 | 258 | # A_users=np.append(A_user1,A_user2,axis=1) 259 | A_users=a_A*a_B 260 | A_user1=A_users[0][0]#数字 261 | A_user2=A_users[0][1]#数字 262 | 263 | lumuda_user1=pow(np.sum(np.sqrt(self.l1*user_offrate_user1*b_user1)/a_user1)/(self.p_tran[0][0]-np.sum(b_user1/a_user1)),2) 264 | lumuda_user2=pow(np.sum(np.sqrt(self.l2*user_offrate_user2*b_user2)/a_user2)/(self.p_tran[0][1]-np.sum(b_user2/a_user2)),2) 265 | lumuda_users=np.array([[lumuda_user1,lumuda_user2]]) 266 | 267 | 268 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 269 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 270 | 271 | mu_capA = pow(np.sum(np.sqrt(self.offrate_A*self.user_l))/self.F_cap[0][0],2)#检查到这 272 | mu_capB = pow(np.sum(np.sqrt(self.offrate_B * self.user_l)) / self.F_cap[0][1],2) 273 | mu_cpas = np.array([[mu_capA, mu_capB]]) 274 | 275 | #分配传输功率 276 | 277 | self.p_tran_user1=(np.sqrt(self.l1*user_offrate_user1*b_user1/lumuda_user1)+b_user1)/a_user1 278 | self.p_tran_user2=(np.sqrt(self.l2*user_offrate_user2*b_user2/lumuda_user2)+b_user2)/a_user2 279 | 280 | self.p_tran_capA=np.array([[self.p_tran_user1[0][0],self.p_tran_user2[0][0]]]) 281 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 282 | # else: 283 | # p_tran_1 = self.p_tran[0][1] / self.cap_num 284 | # self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) 285 | # self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 286 | 287 | #分配算力 288 | self.F_capA = np.sqrt(self.offrate_A * self.user_l / mu_capA) 289 | self.F_capB = np.sqrt(self.offrate_B * self.user_l / mu_capB) 290 | 291 | 292 | else: 293 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 294 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 295 | 296 | #均分传输功率 297 | p_tran_0 = self.p_tran[0][0] / self.cap_num 298 | p_tran_1 = self.p_tran[0][1] / self.cap_num 299 | 300 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 301 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 302 | 303 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap均分给每一个user的传输功率 304 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap均分给每一个user的传输功率 305 | 306 | #均分算力 307 | self.F_capA = np.full(shape=(1, self.user_num), fill_value=self.F_0) # capA均分给每一个user的算力 308 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) # capB均分给每一个user的算力 309 | 310 | C_A1 = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2)) # [cap_A_user1,cap_A_user2] 311 | C_B1 = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 312 | 313 | # 窃听者 314 | C_A_E = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_e / pow(self.noise, 2)) 315 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 316 | 317 | # 更新速率 318 | C_A = C_A1 - C_A_E 319 | C_B = C_B1 - C_B_E 320 | 321 | 322 | #本地 323 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 324 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 325 | 326 | # ----------CAPA-------------- 327 | #传输 328 | self.T_tran_A=np.zeros(shape=(1,2)) 329 | for i in range(self.user_num): 330 | if(self.offrate_A[0][i]): 331 | self.T_tran_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] / C_A[0][i]#shape[1,2] 332 | #CAP端 333 | 334 | self.T_cap_A=np.zeros(shape=(1,2)) 335 | for i in range(self.user_num): 336 | if(self.offrate_A[0][i]): 337 | self.T_cap_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] * self.omega * self.Mb_to_bit / self.F_capA[0][i]#shape[1,2] 338 | 339 | #----------CAPB-------------- 340 | #传输 341 | 342 | self.T_tran_B = np.zeros(shape=(1, 2)) 343 | for i in range(self.user_num): 344 | if (self.offrate_B[0][i]): 345 | self.T_tran_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] / C_B[0][i] # shape[1,2] 346 | 347 | #CAP端 348 | 349 | # self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 350 | self.T_cap_B = np.zeros(shape=(1, 2)) 351 | for i in range(self.user_num): 352 | if (self.offrate_B[0][i]): 353 | self.T_cap_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] * self.omega * self.Mb_to_bit / self.F_capB[0][i] # shape[1,2] 354 | 355 | # ----------------------------- 356 | 357 | #求T 358 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 359 | 360 | 361 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 362 | # temp = 1 - user_offrate 363 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 364 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 365 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 366 | # T_temp = self.T_tran + self.T_cap 367 | # 368 | # T_max = np.max(T_temp) 369 | # T_max1 = np.max(self.T_local) 370 | # T = max(T_max, T_max1) 371 | 372 | return T 373 | 374 | def Energy(self, ): 375 | self.E_local=self.T_local * self.p_local 376 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 377 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 378 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 379 | 380 | # self.E_local = self.T_local * self.p_local 381 | # self.E_tran = self.T_tran * self.p_tran 382 | # self.E_total = self.E_tran + self.E_local 383 | # E = np.sum(self.E_total) 384 | return E 385 | 386 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 387 | T = self.Time(user_offrate) 388 | # E = self.Energy() 389 | total_cost = T 390 | return total_cost 391 | 392 | def cost_print(self): 393 | self.cost_list1 = self.cost_list # 记录成本 394 | self.epoch_list1 = self.epoch_list 395 | self.quancost_list1 = self.quancost_list 396 | self.wucost_list1 = self.wucost_list 397 | self.cost_list = [] # 记录成本 398 | self.epoch_list = [] 399 | self.quancost_list = [] 400 | self.wucost_list = [] 401 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 402 | -------------------------------------------------------------------------------- /environment_ddpg_tra.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment1(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran=p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]])# 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]])# 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道CAPA分给两个用户的信道 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=copy.deepcopy(self.user_offrate) 67 | self.all_off=np.array([[0.0,0.5,0.5,0.0,0.5,0.5]]) 68 | # print("初始化时的卸载率为：", self.user_offrate) 69 | 70 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 71 | 72 | self.lamuda = lamuda 73 | self.noise = noise 74 | self.i = 0 75 | self.cost_list = [] # 记录成本 76 | self.epoch_list = [] 77 | self.quancost_list = [] 78 | self.wucost_list = [] 79 | 80 | self.task_action = 1 # 任务每次变化率（任务卸载率） 81 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 83 | self.p_local = p_local 84 | 85 | 86 | 87 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 88 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 89 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 90 | self.A = 1.0 91 | self.B = 0.0 92 | self.C = 0.0 93 | self.D = 1.0 94 | self.E = 0.0 95 | self.F = 0.0 96 | 97 | self.temp_tc1 = self.total_cost(self.user_offrate)#本地 98 | self.tc_wald = self.total_cost(self.user_offrate) 99 | self.all_cost=self.total_cost(self.all_off)#全卸载 100 | # print("本地的cost为：", self.temp_tc1) 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | def reset(self): # 重开，返回初始化s 110 | 111 | self.user_S = self.l * self.user_offrate #任务量，1x6 112 | #总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 113 | self.P_F=np.append(self.p_tran,self.F_cap,axis=1) 114 | S_=np.append(self.user_S,self.P_F,axis=1) 115 | return S_ 116 | 117 | def step(self, action): 118 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 119 | 120 | 121 | self.i+=1 122 | self.user_offrate=action 123 | 124 | 125 | 126 | s_ = self.l * self.user_offrate 127 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 128 | s_ = np.append(s_, self.P_F, axis=1) 129 | 130 | # 计算更新卸载率后的成本 131 | tc = self.total_cost(self.user_offrate) 132 | 133 | #本地 134 | self.temp_tc1=self.total_cost(self.judge) 135 | 136 | 137 | # if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 138 | # r = -100 139 | # elif tc < self.tc_wald: 140 | # r = 100 141 | # else: 142 | # r = -100 143 | tc_diff=tc - self.tc_wald 144 | if tc_diff>=5.0: 145 | r=-200 146 | elif tc_diff>0.0 and tc_diff<5.0: 147 | r=-100 148 | elif tc_diff<=-5.0: 149 | r=200 150 | elif tc_diff<0.0 and tc_diff>-5.0: 151 | r=100 152 | elif tc_diff==0.0: 153 | r=-200 154 | 155 | self.user_S = s_ # 更新状态 156 | if self.i % 2000==0: 157 | print("----------------------------------") 158 | print("执行了第", self.i, "次动作") 159 | print("此轮更新卸载率后的成本为:", tc) 160 | print("此时卸载率为：", self.user_offrate) 161 | 162 | 163 | self.tc_wald = tc 164 | 165 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 166 | if self.i == self.suspend: 167 | done = self.tc_wald 168 | self.i=0 169 | 170 | 171 | 172 | return s_, r, done, self.tc_wald,self.temp_tc1,self.all_cost 173 | 174 | #根据动作，改变对应的卸载率 175 | def offrate_normalization(self,number):#如增加，add_subtraction=1 176 | # w=self.i/5.0 177 | global w 178 | if self.i<500: 179 | w=1.0 180 | if self.i<1000: 181 | w=2.0 182 | if self.i<1500: 183 | w=3.0 184 | if self.i<2000: 185 | w=4.0 186 | if number==0: 187 | self.A+=w 188 | elif number==1: 189 | self.B+=w 190 | elif number==2: 191 | self.C+=w 192 | elif number==3: 193 | self.D+=w 194 | elif number==4: 195 | self.E+=w 196 | else: 197 | self.F+=w 198 | 199 | 200 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 201 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 202 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 203 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 204 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 205 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 206 | 207 | 208 | 209 | def action_state_num(self): # 返回动作与环境个数 210 | n_action = self.user_num * (self.cap_num + 1) # dqn 211 | n_state = self.user_num * (self.cap_num + 1) + self.cap_num * 2 212 | return n_action, n_state 213 | 214 | def Time(self, user_offrate): 215 | # 计算香农公式Wn 216 | 217 | # # #-----优化带宽------------- 218 | # if self.user_offrate.all()!=0: 219 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 220 | # else: 221 | # w_ = self.W / self.user_num 222 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 223 | # #------------------- 224 | 225 | #--------均分带宽-------------- 226 | # w_0 = self.W0 / self.user_num#shape[1,2] 227 | # w_1 = self.W1 / self.user_num#shape[1,2] 228 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 229 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 230 | #----------------------------- 231 | 232 | 233 | 234 | 235 | if (user_offrate!=self.judge).any(): 236 | 237 | #----解析------- 238 | a_A=W_A_user*np.log2(self.Hn_0/self.Hn_e) 239 | a_B=W_B_user*np.log2(self.Hn_1/self.Hn_e) 240 | 241 | b_A=W_A_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_0) / math.log(2) 242 | b_B=W_B_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_1) / math.log(2) 243 | 244 | 245 | 246 | #按user分 247 | 248 | 249 | b_user1=np.array([[b_A[0][0],b_B[0][0]]]) 250 | b_user2 = np.array([[b_A[0][1], b_B[0][1]]]) 251 | 252 | self.l1=self.l[0][1:3]#传输与cap计算的任务量 253 | self.l2=self.l[0][4:] 254 | user_offrate_user1=user_offrate[0][1:3]#user1的传输与cap计算的卸载率 255 | user_offrate_user2=user_offrate[0][4:] 256 | 257 | a_user1 =np.array([[a_A[0][0], a_B[0][0]]]) 258 | a_user2 = np.array([[a_A[0][1], a_B[0][1]]]) 259 | 260 | # A_users=np.append(A_user1,A_user2,axis=1) 261 | A_users=a_A*a_B 262 | A_user1=A_users[0][0]#数字 263 | A_user2=A_users[0][1]#数字 264 | 265 | lumuda_user1=pow(np.sum(np.sqrt(self.l1*user_offrate_user1*b_user1)/a_user1)/(self.p_tran[0][0]-np.sum(b_user1/a_user1)),2) 266 | lumuda_user2=pow(np.sum(np.sqrt(self.l2*user_offrate_user2*b_user2)/a_user2)/(self.p_tran[0][1]-np.sum(b_user2/a_user2)),2) 267 | lumuda_users=np.array([[lumuda_user1,lumuda_user2]]) 268 | 269 | 270 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 271 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 272 | 273 | mu_capA = pow(np.sum(np.sqrt(self.offrate_A*self.user_l))/self.F_cap[0][0],2)#检查到这 274 | mu_capB = pow(np.sum(np.sqrt(self.offrate_B * self.user_l)) / self.F_cap[0][1],2) 275 | mu_cpas = np.array([[mu_capA, mu_capB]]) 276 | 277 | #分配传输功率 278 | 279 | self.p_tran_user1=(np.sqrt(self.l1*user_offrate_user1*b_user1/lumuda_user1)+b_user1)/a_user1 280 | self.p_tran_user2=(np.sqrt(self.l2*user_offrate_user2*b_user2/lumuda_user2)+b_user2)/a_user2 281 | 282 | 283 | self.p_tran_capA=np.array([[self.p_tran_user1[0][0],self.p_tran_user2[0][0]]]) 284 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 285 | # else: 286 | # p_tran_1 = self.p_tran[0][1] / self.cap_num 287 | # self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) 288 | # self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 289 | 290 | #分配算力 291 | self.F_capA = np.sqrt(self.offrate_A * self.user_l / mu_capA) 292 | self.F_capB = np.sqrt(self.offrate_B * self.user_l / mu_capB) 293 | 294 | 295 | else: 296 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 297 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 298 | 299 | #均分传输功率 300 | p_tran_0 = self.p_tran[0][0] / self.cap_num 301 | p_tran_1 = self.p_tran[0][1] / self.cap_num 302 | 303 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 304 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 305 | 306 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap均分给每一个user的传输功率 307 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap均分给每一个user的传输功率 308 | 309 | #均分算力 310 | self.F_capA = np.full(shape=(1, self.user_num), fill_value=self.F_0) # capA均分给每一个user的算力 311 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) # capB均分给每一个user的算力 312 | 313 | C_A1 = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2)) # [cap_A_user1,cap_A_user2] 314 | C_B1 = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 315 | 316 | # 窃听者 317 | C_A_E = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_e / pow(self.noise, 2)) 318 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 319 | 320 | # 更新速率 321 | C_A = C_A1 - C_A_E 322 | C_B = C_B1 - C_B_E 323 | 324 | 325 | #本地 326 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 327 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 328 | 329 | # ----------CAPA-------------- 330 | #传输 331 | self.T_tran_A=np.zeros(shape=(1,2)) 332 | for i in range(self.user_num): 333 | if(self.offrate_A[0][i]): 334 | self.T_tran_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] / C_A[0][i]#shape[1,2] 335 | #CAP端 336 | 337 | self.T_cap_A=np.zeros(shape=(1,2)) 338 | for i in range(self.user_num): 339 | if(self.offrate_A[0][i]): 340 | self.T_cap_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] * self.omega * self.Mb_to_bit / self.F_capA[0][i]#shape[1,2] 341 | 342 | #----------CAPB-------------- 343 | #传输 344 | 345 | self.T_tran_B = np.zeros(shape=(1, 2)) 346 | for i in range(self.user_num): 347 | if (self.offrate_B[0][i]): 348 | self.T_tran_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] / C_B[0][i] # shape[1,2] 349 | 350 | #CAP端 351 | 352 | # self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 353 | self.T_cap_B = np.zeros(shape=(1, 2)) 354 | for i in range(self.user_num): 355 | if (self.offrate_B[0][i]): 356 | self.T_cap_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] * self.omega * self.Mb_to_bit / self.F_capB[0][i] # shape[1,2] 357 | 358 | # ----------------------------- 359 | 360 | #求T 361 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 362 | 363 | 364 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 365 | # temp = 1 - user_offrate 366 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 367 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 368 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 369 | # T_temp = self.T_tran + self.T_cap 370 | # 371 | # T_max = np.max(T_temp) 372 | # T_max1 = np.max(self.T_local) 373 | # T = max(T_max, T_max1) 374 | 375 | return T 376 | 377 | def Energy(self, ): 378 | self.E_local=self.T_local * self.p_local 379 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 380 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 381 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 382 | 383 | # self.E_local = self.T_local * self.p_local 384 | # self.E_tran = self.T_tran * self.p_tran 385 | # self.E_total = self.E_tran + self.E_local 386 | # E = np.sum(self.E_total) 387 | return E 388 | 389 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 390 | T = self.Time(user_offrate) 391 | # E = self.Energy() 392 | total_cost = T 393 | return total_cost 394 | 395 | def cost_print(self): 396 | self.cost_list1 = self.cost_list # 记录成本 397 | self.epoch_list1 = self.epoch_list 398 | self.quancost_list1 = self.quancost_list 399 | self.wucost_list1 = self.wucost_list 400 | self.cost_list = [] # 记录成本 401 | self.epoch_list = [] 402 | self.quancost_list = [] 403 | self.wucost_list = [] 404 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 405 | -------------------------------------------------------------------------------- /environment_optimize1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran=p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]])# 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]])# 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道CAPA分给两个用户的信道 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=copy.deepcopy(self.user_offrate) 67 | self.all_off=np.array([[0.0,0.5,0.5,0.0,0.5,0.5]]) 68 | # print("初始化时的卸载率为：", self.user_offrate) 69 | 70 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 71 | 72 | self.lamuda = lamuda 73 | self.noise = noise 74 | self.i = 0 75 | self.cost_list = [] # 记录成本 76 | self.epoch_list = [] 77 | self.quancost_list = [] 78 | self.wucost_list = [] 79 | 80 | self.task_action = 1 # 任务每次变化率（任务卸载率） 81 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 83 | self.p_local = p_local 84 | 85 | 86 | 87 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 88 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 89 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 90 | self.A = 1.0 91 | self.B = 0.01 92 | self.C = 0.01 93 | self.D = 1.0 94 | self.E = 0.01 95 | self.F = 0.01 96 | 97 | self.temp_tc1 = self.total_cost(self.user_offrate)#本地 98 | self.tc_wald = self.total_cost(self.user_offrate) 99 | self.all_cost=self.total_cost(self.all_off)#全卸载 100 | # print("本地的cost为：", self.temp_tc1) 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | def reset(self): # 重开，返回初始化s 110 | 111 | self.user_S = self.l * self.user_offrate #任务量，1x6 112 | #总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 113 | self.P_F=np.append(self.p_tran,self.F_cap,axis=1) 114 | S_=np.append(self.user_S,self.P_F,axis=1) 115 | return S_ 116 | 117 | def step(self, action): 118 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 119 | 120 | 121 | self.i = self.i + 1 122 | 123 | # 判断action执行什么动作,action为一个值 124 | 125 | # self.actions=int(action/2)#对应到卸载率的下标 126 | 127 | # if action%2==0:#增加 128 | # self.offrate_normalization(self.actions,1) 129 | # else:#减少 130 | # self.offrate_normalization(self.actions,0) 131 | 132 | #self.offrate_normalization(1,1)、 133 | 134 | if action==0:#user1本地增加 135 | self.offrate_normalization(0) 136 | elif action==1: 137 | self.offrate_normalization(1) 138 | elif action == 2: # user1-capA增加 139 | self.offrate_normalization(2) 140 | elif action == 3: 141 | self.offrate_normalization(3) 142 | elif action == 4: # user1-capB增加 143 | self.offrate_normalization(4) 144 | else: 145 | self.offrate_normalization(5) 146 | 147 | 148 | 149 | s_ = self.l * self.user_offrate 150 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 151 | s_ = np.append(s_, self.P_F, axis=1) 152 | 153 | # 计算更新卸载率后的成本 154 | tc = self.total_cost(self.user_offrate) 155 | 156 | #本地 157 | self.temp_tc1=self.total_cost(self.judge) 158 | 159 | 160 | # if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 161 | # r = -100 162 | # elif tc < self.tc_wald: 163 | # r = 100 164 | # else: 165 | # r = -100 166 | tc_diff=tc - self.tc_wald 167 | if tc_diff>=5.0: 168 | r=-200 169 | elif tc_diff>0.0 and tc_diff<5.0: 170 | r=-100 171 | elif tc_diff<=-5.0: 172 | r=200 173 | elif tc_diff<0.0 and tc_diff>-5.0: 174 | r=100 175 | elif tc_diff==0.0: 176 | r=-200 177 | 178 | self.user_S = s_ # 更新状态 179 | if self.i % 2000==0: 180 | print("----------------------------------") 181 | print("执行了第", self.i, "次动作") 182 | print("此轮更新卸载率后的成本为:", tc) 183 | print("此时卸载率为：", self.user_offrate) 184 | 185 | 186 | self.tc_wald = tc 187 | 188 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 189 | if self.i == self.suspend: 190 | done = self.tc_wald 191 | self.i=0 192 | 193 | 194 | 195 | return s_, r, done, self.tc_wald,self.temp_tc1,self.all_cost 196 | 197 | #根据动作，改变对应的卸载率 198 | def offrate_normalization(self,number):#如增加，add_subtraction=1 199 | # w=self.i/5.0 200 | global w 201 | if self.i<500: 202 | w=1.0 203 | if self.i<1000: 204 | w=2.0 205 | if self.i<1500: 206 | w=3.0 207 | if self.i<2000: 208 | w=4.0 209 | if number==0: 210 | self.A+=w 211 | elif number==1: 212 | self.B+=w 213 | elif number==2: 214 | self.C+=w 215 | elif number==3: 216 | self.D+=w 217 | elif number==4: 218 | self.E+=w 219 | else: 220 | self.F+=w 221 | 222 | 223 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 224 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 225 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 226 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 227 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 228 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 229 | 230 | 231 | 232 | def action_state_num(self): # 返回动作与环境个数 233 | action_n = self.user_num * (self.cap_num+1)#dqn 234 | state_num = self.user_num*(self.cap_num +1) +self.cap_num+self.user_num 235 | return action_n, state_num 236 | 237 | def Time(self, user_offrate): 238 | # 计算香农公式Wn 239 | 240 | # # #-----优化带宽------------- 241 | # if self.user_offrate.all()!=0: 242 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 243 | # else: 244 | # w_ = self.W / self.user_num 245 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 246 | # #------------------- 247 | 248 | #--------均分带宽-------------- 249 | # w_0 = self.W0 / self.user_num#shape[1,2] 250 | # w_1 = self.W1 / self.user_num#shape[1,2] 251 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 252 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 253 | #----------------------------- 254 | 255 | 256 | 257 | 258 | if (user_offrate!=self.judge).any(): 259 | 260 | #----解析------- 261 | a_A=W_A_user*np.log2(self.Hn_0/self.Hn_e) 262 | a_B=W_B_user*np.log2(self.Hn_1/self.Hn_e) 263 | 264 | b_A=W_A_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_0) / math.log(2) 265 | b_B=W_B_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_1) / math.log(2) 266 | 267 | 268 | 269 | #按user分 270 | 271 | 272 | b_user1=np.array([[b_A[0][0],b_B[0][0]]]) 273 | b_user2 = np.array([[b_A[0][1], b_B[0][1]]]) 274 | 275 | self.l1=self.l[0][1:3]#传输与cap计算的任务量 276 | self.l2=self.l[0][4:] 277 | user_offrate_user1=user_offrate[0][1:3]#user1的传输与cap计算的卸载率 278 | user_offrate_user2=user_offrate[0][4:] 279 | 280 | a_user1 =np.array([[a_A[0][0], a_B[0][0]]]) 281 | a_user2 = np.array([[a_A[0][1], a_B[0][1]]]) 282 | 283 | # A_users=np.append(A_user1,A_user2,axis=1) 284 | A_users=a_A*a_B 285 | A_user1=A_users[0][0]#数字 286 | A_user2=A_users[0][1]#数字 287 | 288 | lumuda_user1=pow(np.sum(np.sqrt(self.l1*user_offrate_user1*b_user1)/a_user1)/(self.p_tran[0][0]-np.sum(b_user1/a_user1)),2) 289 | lumuda_user2=pow(np.sum(np.sqrt(self.l2*user_offrate_user2*b_user2)/a_user2)/(self.p_tran[0][1]-np.sum(b_user2/a_user2)),2) 290 | lumuda_users=np.array([[lumuda_user1,lumuda_user2]]) 291 | 292 | 293 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 294 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 295 | 296 | mu_capA = pow(np.sum(np.sqrt(self.offrate_A*self.user_l))/self.F_cap[0][0],2)#检查到这 297 | mu_capB = pow(np.sum(np.sqrt(self.offrate_B * self.user_l)) / self.F_cap[0][1],2) 298 | mu_cpas = np.array([[mu_capA, mu_capB]]) 299 | 300 | #分配传输功率 301 | 302 | self.p_tran_user1=(np.sqrt(self.l1*user_offrate_user1*b_user1/lumuda_user1)+b_user1)/a_user1 303 | self.p_tran_user2=(np.sqrt(self.l2*user_offrate_user2*b_user2/lumuda_user2)+b_user2)/a_user2 304 | 305 | if np.isnan(self.p_tran_user2).any(): 306 | print("yesssss!") 307 | print(user_offrate) 308 | 309 | self.p_tran_capA=np.array([[self.p_tran_user1[0][0],self.p_tran_user2[0][0]]]) 310 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 311 | # else: 312 | # p_tran_1 = self.p_tran[0][1] / self.cap_num 313 | # self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) 314 | # self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 315 | 316 | #分配算力 317 | self.F_capA = np.sqrt(self.offrate_A * self.user_l / mu_capA) 318 | self.F_capB = np.sqrt(self.offrate_B * self.user_l / mu_capB) 319 | 320 | if np.isnan(self.F_capA).any(): 321 | print("yesssss!self.F_capA") 322 | print(user_offrate) 323 | 324 | else: 325 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 326 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 327 | 328 | #均分传输功率 329 | p_tran_0 = self.p_tran[0][0] / self.cap_num 330 | p_tran_1 = self.p_tran[0][1] / self.cap_num 331 | 332 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 333 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 334 | 335 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap均分给每一个user的传输功率 336 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap均分给每一个user的传输功率 337 | 338 | #均分算力 339 | self.F_capA = np.full(shape=(1, self.user_num), fill_value=self.F_0) # capA均分给每一个user的算力 340 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) # capB均分给每一个user的算力 341 | 342 | C_A1 = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2)) # [cap_A_user1,cap_A_user2] 343 | C_B1 = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 344 | 345 | # 窃听者 346 | C_A_E = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_e / pow(self.noise, 2)) 347 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 348 | 349 | # 更新速率 350 | C_A = C_A1 - C_A_E 351 | C_B = C_B1 - C_B_E 352 | 353 | 354 | #本地 355 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 356 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 357 | 358 | # ----------CAPA-------------- 359 | #传输 360 | self.T_tran_A=np.zeros(shape=(1,2)) 361 | for i in range(self.user_num): 362 | if(self.offrate_A[0][i]): 363 | self.T_tran_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] / C_A[0][i]#shape[1,2] 364 | #CAP端 365 | 366 | self.T_cap_A=np.zeros(shape=(1,2)) 367 | for i in range(self.user_num): 368 | if(self.offrate_A[0][i]): 369 | self.T_cap_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] * self.omega * self.Mb_to_bit / self.F_capA[0][i]#shape[1,2] 370 | 371 | #----------CAPB-------------- 372 | #传输 373 | 374 | self.T_tran_B = np.zeros(shape=(1, 2)) 375 | for i in range(self.user_num): 376 | if (self.offrate_B[0][i]): 377 | self.T_tran_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] / C_B[0][i] # shape[1,2] 378 | 379 | #CAP端 380 | 381 | # self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 382 | self.T_cap_B = np.zeros(shape=(1, 2)) 383 | for i in range(self.user_num): 384 | if (self.offrate_B[0][i]): 385 | self.T_cap_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] * self.omega * self.Mb_to_bit / self.F_capB[0][i] # shape[1,2] 386 | 387 | # ----------------------------- 388 | 389 | #求T 390 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 391 | 392 | 393 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 394 | # temp = 1 - user_offrate 395 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 396 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 397 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 398 | # T_temp = self.T_tran + self.T_cap 399 | # 400 | # T_max = np.max(T_temp) 401 | # T_max1 = np.max(self.T_local) 402 | # T = max(T_max, T_max1) 403 | 404 | return T 405 | 406 | def Energy(self, ): 407 | self.E_local=self.T_local * self.p_local 408 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 409 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 410 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 411 | 412 | # self.E_local = self.T_local * self.p_local 413 | # self.E_tran = self.T_tran * self.p_tran 414 | # self.E_total = self.E_tran + self.E_local 415 | # E = np.sum(self.E_total) 416 | return E 417 | 418 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 419 | T = self.Time(user_offrate) 420 | # E = self.Energy() 421 | total_cost = T 422 | return total_cost 423 | 424 | def cost_print(self): 425 | self.cost_list1 = self.cost_list # 记录成本 426 | self.epoch_list1 = self.epoch_list 427 | self.quancost_list1 = self.quancost_list 428 | self.wucost_list1 = self.wucost_list 429 | self.cost_list = [] # 记录成本 430 | self.epoch_list = [] 431 | self.quancost_list = [] 432 | self.wucost_list = [] 433 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 434 | -------------------------------------------------------------------------------- /environment_tradition.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment1(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran = p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道[CAP] 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=self.user_offrate 67 | self.all_off = np.array([[0.0, 0.5, 0.5, 0.0, 0.5, 0.5]]) 68 | # print("初始化时的卸载率为：", self.user_offrate) 69 | 70 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 71 | 72 | self.lamuda = lamuda 73 | self.noise = noise 74 | self.i = 0 75 | self.cost_list = [] # 记录成本 76 | self.epoch_list = [] 77 | self.quancost_list = [] 78 | self.wucost_list = [] 79 | 80 | self.task_action = 1 # 任务每次变化率（任务卸载率） 81 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 83 | self.p_local = p_local 84 | 85 | #-------------------------- 86 | #窃听者 87 | 88 | 89 | #------------------------- 90 | 91 | 92 | 93 | 94 | 95 | 96 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 97 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 98 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 99 | self.A = 1.0 100 | self.B = 0.01 101 | self.C = 0.01 102 | self.D = 1.0 103 | self.E = 0.01 104 | self.F = 0.01 105 | 106 | self.temp_tc1 = self.total_cost(self.user_offrate) 107 | self.tc_wald = self.total_cost(self.user_offrate) 108 | self.all_cost=self.total_cost(self.all_off) 109 | # print("本地的cost为：", self.temp_tc1) 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | def reset(self): # 重开，返回初始化s 119 | 120 | self.user_S = self.l * self.user_offrate #任务量，1x6 121 | #总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 122 | self.P_F=np.append(self.p_tran,self.F_cap,axis=1) 123 | S_=np.append(self.user_S,self.P_F,axis=1) 124 | return S_ 125 | 126 | def step(self, action): 127 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 128 | 129 | 130 | self.i = self.i + 1 131 | 132 | # 判断action执行什么动作,action为一个值 133 | 134 | # self.actions=int(action/2)#对应到卸载率的下标 135 | 136 | # if action%2==0:#增加 137 | # self.offrate_normalization(self.actions,1) 138 | # else:#减少 139 | # self.offrate_normalization(self.actions,0) 140 | 141 | if action == 0: # user1本地增加 142 | self.offrate_normalization(0) 143 | elif action == 1: 144 | self.offrate_normalization(1) 145 | elif action == 2: # user1-capB增加 146 | self.offrate_normalization(2) 147 | elif action == 3: 148 | self.offrate_normalization(3) 149 | elif action == 4: # user2-capA增加 150 | self.offrate_normalization(4) 151 | else: 152 | self.offrate_normalization(5) 153 | 154 | 155 | s_ = self.l * self.user_offrate 156 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 157 | s_ = np.append(s_, self.P_F, axis=1) 158 | tc = self.total_cost(self.user_offrate) # 更新后的卸载率的成本 159 | 160 | # if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 161 | # r = -100 162 | # elif tc < self.tc_wald: 163 | # r = 100 164 | # else: 165 | # r = -100 166 | tc_diff = tc - self.tc_wald 167 | if tc_diff >= 5.0: 168 | r = -200 169 | elif tc_diff > 0.0 and tc_diff < 5.0: 170 | r = -100 171 | elif tc_diff <= -5.0: 172 | r = 200 173 | elif tc_diff < 0.0 and tc_diff > -5.0: 174 | r = 100 175 | elif tc_diff == 0.0: 176 | r = -200 177 | 178 | 179 | self.tc_wald = tc 180 | 181 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 182 | if self.i == self.suspend: 183 | done = self.tc_wald 184 | self.i = 0 185 | 186 | 187 | 188 | return s_, r, done, self.tc_wald,self.temp_tc1,self.all_cost 189 | 190 | #根据动作，改变对应的卸载率 191 | def offrate_normalization(self,number):#如增加，add_subtraction=1 192 | # w = self.i / 5.0 193 | global w 194 | if self.i < 500: 195 | w = 1.0 196 | if self.i < 1000: 197 | w = 2.0 198 | if self.i < 1500: 199 | w = 3.0 200 | if self.i < 2000: 201 | w = 4.0 202 | if number == 0: 203 | self.A += w 204 | elif number == 1: 205 | self.B += w 206 | elif number == 2: 207 | self.C += w 208 | elif number == 3: 209 | self.D += w 210 | elif number == 4: 211 | self.E += w 212 | else: 213 | self.F += w 214 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 215 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 216 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 217 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 218 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 219 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 220 | 221 | 222 | 223 | def action_state_num(self): # 返回动作与环境个数 224 | action_n = self.user_num * (self.cap_num+1)#dqn 225 | state_num = self.user_num*(self.cap_num +1) +self.cap_num*2 226 | return action_n, state_num 227 | 228 | def Time(self, user_offrate): 229 | # 计算香农公式Wn 230 | 231 | # # #-----优化带宽------------- 232 | # if self.user_offrate.all()!=0: 233 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 234 | # else: 235 | # w_ = self.W / self.user_num 236 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 237 | # #------------------- 238 | 239 | #--------均分带宽-------------- 240 | # w_0 = self.W0 / self.user_num#shape[1,2] 241 | # w_1 = self.W1 / self.user_num#shape[1,2] 242 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 243 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 244 | #----------------------------- 245 | 246 | C_A1=W_A_user*np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2))#[cap_A_user1,cap_A_user2] 247 | C_B1=W_B_user*np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 248 | 249 | #窃听者 250 | C_A_E=W_A_user*np.log2(1+self.p_tran_capA*self.Hn_e / pow(self.noise, 2)) 251 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 252 | 253 | #更新速率 254 | C_A=C_A1 - C_A_E 255 | C_B=C_B1 - C_B_E 256 | 257 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 258 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 259 | 260 | #本地 261 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 262 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 263 | 264 | # ----------CAPA-------------- 265 | #传输 266 | 267 | self.T_tran_A=self.user_l * self.offrate_A / C_A#shape[1,2] 268 | #CAP端 269 | self.T_cap_A=self.user_l * self.offrate_A * self.omega * self.Mb_to_bit / self.F_capA#shape[1,2] 270 | #----------CAPB-------------- 271 | #传输 272 | 273 | self.T_tran_B=self.user_l * self.offrate_B / C_B#shape[1,2] 274 | #CAP端 275 | self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 276 | # ----------------------------- 277 | 278 | #求T 279 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 280 | 281 | T_off_A=self.T_tran_A+self.T_cap_A 282 | 283 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 284 | # temp = 1 - user_offrate 285 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 286 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 287 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 288 | # T_temp = self.T_tran + self.T_cap 289 | # 290 | # T_max = np.max(T_temp) 291 | # T_max1 = np.max(self.T_local) 292 | # T = max(T_max, T_max1) 293 | 294 | return T 295 | 296 | def Energy(self, ): 297 | self.E_local=self.T_local * self.p_local 298 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 299 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 300 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 301 | 302 | # self.E_local = self.T_local * self.p_local 303 | # self.E_tran = self.T_tran * self.p_tran 304 | # self.E_total = self.E_tran + self.E_local 305 | # E = np.sum(self.E_total) 306 | return E 307 | 308 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 309 | T = self.Time(user_offrate) 310 | # E = self.Energy() 311 | total_cost = T 312 | return total_cost 313 | 314 | def cost_print(self): 315 | self.cost_list1 = self.cost_list # 记录成本 316 | self.epoch_list1 = self.epoch_list 317 | self.quancost_list1 = self.quancost_list 318 | self.wucost_list1 = self.wucost_list 319 | self.cost_list = [] # 记录成本 320 | self.epoch_list = [] 321 | self.quancost_list = [] 322 | self.wucost_list = [] 323 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 324 | -------------------------------------------------------------------------------- /test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import torch" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 9, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "text/plain": "tensor([[0., 0., 0., 0.],\n [0., 0., 0., 0.],\n [0., 0., 0., 0.]])" 22 | }, 23 | "execution_count": 9, 24 | "metadata": {}, 25 | "output_type": "execute_result" 26 | } 27 | ], 28 | "source": [ 29 | "w = torch.Tensor(3,4)\n", 30 | "w" 31 | ], 32 | "metadata": { 33 | "collapsed": false, 34 | "pycharm": { 35 | "name": "#%%\n" 36 | } 37 | } 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 34, 42 | "outputs": [ 43 | { 44 | "data": { 45 | "text/plain": "Parameter containing:\ntensor([[-0.4534, 0.3025, -0.2776],\n [-0.4067, -0.0642, -0.1638],\n [ 0.1921, -0.2198, -0.5003],\n [-0.1345, -0.3511, -0.5041],\n [ 0.3314, -0.4759, 0.1622]], requires_grad=True)" 46 | }, 47 | "execution_count": 34, 48 | "metadata": {}, 49 | "output_type": "execute_result" 50 | } 51 | ], 52 | "source": [ 53 | "# tensor([[-37955820.0000, -1592520.5000, -203957.0000]]\n", 54 | "\n", 55 | "a=torch.tensor([[-37955820.0000, -1592520.5000, -203957.0000]])/1000000\n", 56 | "softmax=torch.nn.Softmax()\n", 57 | "relu=torch.nn.ReLU()\n", 58 | "fc1=torch.nn.Linear(3,5)\n", 59 | "fc1.weight" 60 | ], 61 | "metadata": { 62 | "collapsed": false, 63 | "pycharm": { 64 | "name": "#%%\n" 65 | } 66 | } 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 32, 71 | "outputs": [ 72 | { 73 | "name": "stderr", 74 | "output_type": "stream", 75 | "text": [ 76 | ":1: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", 77 | " softmax(a)\n" 78 | ] 79 | }, 80 | { 81 | "data": { 82 | "text/plain": "tensor([[3.2200e-17, 1.9964e-01, 8.0036e-01]])" 83 | }, 84 | "execution_count": 32, 85 | "metadata": {}, 86 | "output_type": "execute_result" 87 | } 88 | ], 89 | "source": [ 90 | "softmax(a)" 91 | ], 92 | "metadata": { 93 | "collapsed": false, 94 | "pycharm": { 95 | "name": "#%%\n" 96 | } 97 | } 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 7, 102 | "outputs": [ 103 | { 104 | "data": { 105 | "text/plain": "tensor([2, 3, 4])" 106 | }, 107 | "execution_count": 7, 108 | "metadata": {}, 109 | "output_type": "execute_result" 110 | } 111 | ], 112 | "source": [ 113 | "a=torch.tensor([2,3,4])\n", 114 | "a" 115 | ], 116 | "metadata": { 117 | "collapsed": false, 118 | "pycharm": { 119 | "name": "#%%\n" 120 | } 121 | } 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 30, 126 | "outputs": [ 127 | { 128 | "ename": "RuntimeError", 129 | "evalue": "\"normal_kernel_cpu\" not implemented for 'Long'", 130 | "output_type": "error", 131 | "traceback": [ 132 | "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", 133 | "\u001B[0;31mRuntimeError\u001B[0m Traceback (most recent call last)", 134 | "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mnormal\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmean\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m0.5\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mstd\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mtorch\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0marange\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;36m6\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", 135 | "\u001B[0;31mRuntimeError\u001B[0m: \"normal_kernel_cpu\" not implemented for 'Long'" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "torch.normal(mean=0.5, std=torch.arange(1, 6))" 141 | ], 142 | "metadata": { 143 | "collapsed": false, 144 | "pycharm": { 145 | "name": "#%%\n" 146 | } 147 | } 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 6, 152 | "outputs": [ 153 | { 154 | "data": { 155 | "text/plain": "array([[0. , 0.5, 0.5, 0.3, 0.2, 0.5]])" 156 | }, 157 | "execution_count": 6, 158 | "metadata": {}, 159 | "output_type": "execute_result" 160 | } 161 | ], 162 | "source": [ 163 | "a=np.array([[0,0.5,0.5,0.3,0.2,0.5]])\n", 164 | "a" 165 | ], 166 | "metadata": { 167 | "collapsed": false, 168 | "pycharm": { 169 | "name": "#%%\n" 170 | } 171 | } 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 13, 176 | "outputs": [ 177 | { 178 | "data": { 179 | "text/plain": "array([[-0.76937837, 2.09532264, 0.18429382, 4.27775308, -2.91123644,\n -1.65041654]])" 180 | }, 181 | "execution_count": 13, 182 | "metadata": {}, 183 | "output_type": "execute_result" 184 | } 185 | ], 186 | "source": [ 187 | "np.random.normal(a,3)" 188 | ], 189 | "metadata": { 190 | "collapsed": false, 191 | "pycharm": { 192 | "name": "#%%\n" 193 | } 194 | } 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 4, 199 | "outputs": [ 200 | { 201 | "data": { 202 | "text/plain": "array([[0.10981178, 0. , 1. , 0.2740658 , 1. ,\n 1. ]])" 203 | }, 204 | "execution_count": 4, 205 | "metadata": {}, 206 | "output_type": "execute_result" 207 | } 208 | ], 209 | "source": [ 210 | "a=np.clip(np.random.normal(a,3),0,1)\n", 211 | "a" 212 | ], 213 | "metadata": { 214 | "collapsed": false, 215 | "pycharm": { 216 | "name": "#%%\n" 217 | } 218 | } 219 | } 220 | ], 221 | "metadata": { 222 | "kernelspec": { 223 | "display_name": "Python 3", 224 | "language": "python", 225 | "name": "python3" 226 | }, 227 | "language_info": { 228 | "codemirror_mode": { 229 | "name": "ipython", 230 | "version": 2 231 | }, 232 | "file_extension": ".py", 233 | "mimetype": "text/x-python", 234 | "name": "python", 235 | "nbconvert_exporter": "python", 236 | "pygments_lexer": "ipython2", 237 | "version": "2.7.6" 238 | } 239 | }, 240 | "nbformat": 4, 241 | "nbformat_minor": 0 242 | } -------------------------------------------------------------------------------- /传入用户-窃听/DQN_env_customized_users_final.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import os 3 | import Writedata 4 | os.environ['KMP_DUPLICATE_LIB_OK']='True' 5 | #2021。9。27最终版。可以自定义用户版。优化了带宽 6 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2， .....] 7 | # 输出的a为某个用户卸载率增加或者减少 8 | 9 | #2cap 2user 10 | 11 | import torch 12 | import numpy as np 13 | 14 | import matplotlib.pyplot as plt 15 | from environment_optimize1 import environment 16 | from environment_tradition import environment1 17 | 18 | 19 | #%% 20 | 21 | # user_num=input("please input user_num") 22 | 23 | #初始化参数----------------- 24 | user_num=2 25 | cap_num=2 26 | users_num=int(user_num) 27 | 28 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, [user_num, cap_num]) + np.complex('j') * 29 | np.random.normal(1, 1, [user_num, cap_num]))) ** 2 30 | Hn_e = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, user_num) + np.complex('j') * 31 | np.random.normal(1, 1, user_num))) ** 2##窃听者,[user1,user2] 32 | Hn_0=Hn[0] 33 | Hn_1=Hn[1] 34 | 35 | user_l = np.array([[np.random.randint(10,20), np.random.randint(150,200)]]) 36 | 37 | e_number=5 38 | sus=2000 39 | epochs=200 40 | #---------------- 41 | env=environment(user_num=2,cap_num=2,W=[5,5], 42 | f_local=np.array([[1.4*10e8,0.43*10e8]]), 43 | omega=330 / 8, 44 | F_cap=np.array([[10*10e8,3*10e8]]),p_local=1e-1,p_tran=np.array([[2,3]]),lamuda=1, 45 | noise=0.1,Hn_0=Hn_0,Hn_1=Hn_1,Hn_e=Hn_e,user_l=user_l, 46 | suspend=sus 47 | ) 48 | 49 | #改 50 | #获取环境的相关参数: 51 | n_action,n_state=env.action_state_num() 52 | #其他超参数的定义： 53 | MEMORY_CAPACITY=5000#记忆库的存储大小 54 | greedy=0.9#贪婪概率 55 | discount=0.9#折扣率 56 | batch_size=64#一次训练32个数据 57 | TARGET_REPLACE_ITER=10#目标网络的更新频率 58 | 59 | #%% 60 | 61 | class Net(torch.nn.Module): 62 | def __init__(self): 63 | super(Net, self).__init__() 64 | self.linear1=torch.nn.Linear(n_state,20)#(10,20) 65 | self.linear1.weight.data.normal_(0, 0.1) 66 | self.linear2=torch.nn.Linear(20,30) 67 | self.linear2.weight.data.normal_(0, 0.1) 68 | self.linear3=torch.nn.Linear(30,18) 69 | self.linear3.weight.data.normal_(0, 0.1) 70 | self.linear4=torch.nn.Linear(18,n_action)#(30,12) 71 | self.linear4.weight.data.normal_(0, 0.1) 72 | self.relu=torch.nn.ReLU() 73 | def forward(self,x): 74 | x=self.relu(self.linear1(x)) 75 | x=self.relu(self.linear2(x)) 76 | x = self.relu(self.linear3(x)) 77 | q_values=self.linear4(x) 78 | return q_values 79 | 80 | 81 | #%% 82 | 83 | class DQN (): 84 | def __init__(self):#可能要改成def __init__(self,n_state,n_action) 85 | 86 | #记忆库：存s,a,r,s_ 一共6维 87 | self.memory=np.zeros((MEMORY_CAPACITY,n_state*2+2)) 88 | self.memory_count=0#记忆库计数，用于求index，以及当记忆库计数 > 记忆库容量时 开始学习 89 | 90 | #目标网络,参数更新慢。参数更新跟着agent_network走。用于预测q现实 91 | self.target_network=Net() 92 | 93 | #目标网络更新计数 94 | self.learn_step_counter=0 95 | 96 | #用于训练的网络，参数更新及时。用于预测q估计 97 | self.agent_network=Net() 98 | 99 | #优化器与损失器 100 | self.loss_func=torch.nn.MSELoss() 101 | self.optimizer=torch.optim.Adam(self.agent_network.parameters(),lr=0.001) 102 | def choose_action(self,s):#这里每次只输入一个样本。 103 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2] 104 | # 输出的action为0或1或2或3 105 | if np.random.uniform() < greedy:#选择最优动作 106 | #送入神经网络前，先要处理s成为torch型，且要求shape为(1,4) 107 | #s=torch.tensor(s,dtype=float).reshape(1,4) 108 | #s=s.to(torch.float32) 109 | s=torch.FloatTensor(s).reshape(-1,n_state) 110 | 111 | #把s送入神经网络得到a_value, a_value的shape 为（4，1）。并且得到的是四个动作q值 112 | action_value=self.agent_network.forward(s) 113 | 114 | #输出每个样本中预测的最大值，输出[最大值，下标]。只要下标 115 | _,action=torch.max(action_value.data,dim=1) 116 | #在从numpy转为int值 117 | action=action.numpy()[0] 118 | #此时action可能是0、1、2、3 119 | 120 | 121 | else:#随机选择动作 122 | action = np.random.randint(0, n_action) 123 | 124 | return action 125 | 126 | def store_transition(self,s,action,r,s_): 127 | #将四个值打包在一起 128 | 129 | #action为0or 1 or 2 or 3 130 | s=s.reshape(n_state) 131 | s_=s_.reshape(n_state) 132 | 133 | self.transition=np.hstack((s,[action,r],s_)) 134 | 135 | #如果记忆库满了, 就覆盖老数据 136 | index = self.memory_count % MEMORY_CAPACITY 137 | 138 | #传入记忆库 139 | 140 | self.memory[index,:]=self.transition 141 | self.memory_count+=1 142 | #print("目前记忆库存储的总数为",self.memory_count)#要删除 143 | 144 | def learn(self):#从记忆库中选择batch个数据进行训练学习 145 | #首先判断target_network是否需要更新 146 | 147 | if self.learn_step_counter % TARGET_REPLACE_ITER==0: 148 | self.target_network.load_state_dict(self.agent_network.state_dict())#更新目标网络参数 149 | self.learn_step_counter+=1 150 | 151 | #随机从memory中抽32个数据，即batch=32 152 | 153 | sample_index=np.random.choice(MEMORY_CAPACITY,batch_size)#从0-1999中抽32个数字，shape为（32，） 154 | batch_memory=self.memory[sample_index,:]#获取32个数据。 shape 为（32,6） 155 | 156 | 157 | b_s = torch.FloatTensor(batch_memory[:, :n_state]) 158 | b_a = torch.LongTensor(batch_memory[:, n_state:n_state+1].astype(int)) 159 | b_r = torch.FloatTensor(batch_memory[:, n_state+1:n_state+2]) 160 | b_s_ = torch.FloatTensor(batch_memory[:, -n_state:]) 161 | # b_memory shape (batch,6) 162 | # b_s shape (batch,2) 163 | # b_a shape (batch,1) 164 | # b_r shape (batch,1) 165 | # b_s_ shape (batch,2) 166 | 167 | 168 | #把样本state送入agent网络训练，得到q估计 169 | q_predict=self.agent_network(b_s)#shape 为 (32,4) 170 | q_predict=q_predict.gather(1,b_a)#shaoe 为 (32,1)。根据b_a找出q估计 171 | 172 | 173 | # #把样本送入target网络计算，得到Q(s+1) 174 | # q_next=self.target_network(b_s_).detach()# q_next 不进行反向传播, 所以使用detach。shape为 (32,4) 175 | # q_next,_=torch.max(q_next,dim=1)#输出每个样本中预测的最大值，输出[最大值，下标]。只要最大值 176 | # q_next=q_next.reshape((-1,1))#q_next的shape 为（32,1） 177 | # #计算q真实 178 | # q_target=b_r+discount*q_next#q_target的shape 为（32,1） 179 | 180 | #改用ddqn 181 | q_next=self.agent_network(b_s_).detach()# 182 | _,b_a_=torch.max(q_next,dim=1)#由dqn网络选出动作 183 | b_a_=b_a_.reshape((-1,1))#b_a_的shape 为（32,1） 184 | q_next_target=self.target_network(b_s_).detach() 185 | q_next=q_next_target.gather(1,b_a_)#q_next的shape 为（32,1 186 | #计算q真实 187 | q_target=b_r+discount*q_next#q_target的shape 为（32,1） 188 | 189 | 190 | 191 | loss=self.loss_func(q_predict,q_target) 192 | # if self.learn_step_counter%5000==0: 193 | # print("第",self.learn_step_counter,"次强化学习的loss为：",loss.item()) 194 | self.optimizer.zero_grad() 195 | loss.backward() 196 | self.optimizer.step() 197 | return loss.item() 198 | 199 | 200 | 201 | #%% 202 | 203 | dqn=DQN() 204 | dqn_tradition=DQN() 205 | epoch_cost_optimize=[] 206 | epoch_cost_tradition=[] 207 | 208 | 209 | 210 | 211 | epoch_list=[] 212 | loss_list=[] 213 | temp_wu_list=[] 214 | local_costs_epoch=[] 215 | count=0 216 | 217 | 218 | 219 | cost=np.zeros([epochs,sus]) 220 | #每次开始游戏前，重开游戏 221 | #s=env.reset()#s是（4,)的numpy 222 | # print("开始第",epoch+1,"轮游戏") 223 | # start = time.time() 224 | e_number=5 225 | for epoch in range(epochs): 226 | print("本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch) 227 | step_loss = [] 228 | step_loss_tradition=[] 229 | local_costs=[] 230 | 231 | #参数随机化 232 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 233 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 234 | 235 | 236 | ##窃听者,[user1,user2] 237 | 238 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 239 | np.random.normal(1, 0.1, user_num))) ** 2 240 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 241 | np.random.normal(1, 0.1, user_num))) ** 2 242 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 243 | np.random.normal(1, 0.1, user_num))) ** 2 244 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 245 | np.random.normal(1, 0.1, user_num))) ** 2 246 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 247 | np.random.normal(1, 0.1, user_num))) ** 2 248 | Hn_e =Hn_e1+ Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 249 | 250 | # Hn_e=Hn_e1 251 | Hn_0 = Hn[0] 252 | Hn_1 = Hn[1] 253 | 254 | while((Hn_e > Hn_1).any() or (Hn_e > Hn_0).any()): 255 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 256 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 257 | ##窃听者,[user1,user2] 258 | Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 259 | np.random.normal(1, 0.1, user_num))) ** 2 260 | Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 261 | np.random.normal(1, 0.1, user_num))) ** 2 262 | Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 263 | np.random.normal(1, 0.1, user_num))) ** 2 264 | Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 265 | np.random.normal(1, 0.1, user_num))) ** 2 266 | Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') * 267 | np.random.normal(1, 0.1, user_num))) ** 2 268 | Hn_e =Hn_e1+ Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5 269 | #共谋：sum 270 | Hn_0 = Hn[0] 271 | Hn_1 = Hn[1] 272 | 273 | 274 | user_l = np.array([[np.random.randint(10,60), np.random.randint(550,600)]]) 275 | #-------------------------- 276 | #------------优化功率和算力------------- 277 | env = environment(user_num=2, cap_num=2, W=5.0, 278 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 279 | omega=330 / 8, 280 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), lamuda=1, 281 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1,Hn_e=Hn_e,user_l=user_l, 282 | suspend=sus 283 | ) 284 | s = env.reset() 285 | while True:#每次循环执行一个动作 286 | 287 | #通过神经网络，输入状态，获取动作。(不带学习) 288 | action=dqn.choose_action(s) 289 | 290 | #执行，获得环境反馈 291 | s_,r,done,tc_wald,local_cost=env.step(action) 292 | step_loss.append(tc_wald) 293 | local_costs.append(local_cost) 294 | #将环境的反馈s,a,r,s_传入记忆库。用于经验回放 295 | dqn.store_transition(s,action,r,s_) 296 | 297 | #神经网络学习（experience replay 经验回放） 298 | if dqn.memory_count > MEMORY_CAPACITY: 299 | count+=1 300 | loss=dqn.learn() 301 | 302 | 303 | #判断是否结束 304 | if done !=0: 305 | epoch_cost_optimize.append(done) 306 | local_costs_epoch.append(local_cost) 307 | break 308 | 309 | #更新状态，进入下一次循环 310 | s=s_ 311 | #-------------------------------------------- 312 | 313 | 314 | # ------------传统方法------------- 315 | env_tradition = environment1(user_num=2, cap_num=2, W=5.0, 316 | f_local=np.array([[1.4 * 10e8, 0.43 * 10e8]]), 317 | omega=330 / 8, 318 | F_cap=np.array([[10 * 10e8, 3 * 10e8]]), p_local=1e-1, p_tran=np.array([[2, 3]]), 319 | lamuda=1, 320 | noise=0.1, Hn_0=Hn_0, Hn_1=Hn_1, Hn_e=Hn_e, user_l=user_l, 321 | suspend=sus 322 | ) 323 | s=env_tradition.reset() 324 | while True: # 每次循环执行一个动作 325 | 326 | # 通过神经网络，输入状态，获取动作。(不带学习) 327 | action = dqn_tradition.choose_action(s) 328 | 329 | # 执行，获得环境反馈 330 | s_, r, done, tc_wald, local_cost = env_tradition.step(action) 331 | step_loss_tradition.append(tc_wald) 332 | # local_costs.append(local_cost) 333 | 334 | # 将环境的反馈s,a,r,s_传入记忆库。用于经验回放 335 | dqn_tradition.store_transition(s, action, r, s_) 336 | 337 | # 神经网络学习（experience replay 经验回放） 338 | if dqn_tradition.memory_count > MEMORY_CAPACITY: 339 | count += 1 340 | loss = dqn_tradition.learn() 341 | 342 | # 判断是否结束 343 | if done!=0: 344 | epoch_cost_tradition.append(done) 345 | break 346 | 347 | # 更新状态，进入下一次循环 348 | s = s_ 349 | # -------------------------------------------- 350 | 351 | 352 | plt.plot(range(sus), step_loss, color='skyblue', label='optimize_F_P') 353 | plt.plot(range(sus), step_loss_tradition, color='red', label='tradition') 354 | plt.plot(range(sus),local_costs,color="yellow",label="local") 355 | plt.legend(loc="best") 356 | 357 | plt.xlabel("step_dqn_e25") 358 | plt.ylabel("costs") 359 | plt.show() 360 | 361 | 362 | plt.plot(range(epochs),local_costs_epoch, color='yellow', label='locals') 363 | plt.plot(range(epochs),epoch_cost_optimize,color='skyblue', label='optimize_F_P') 364 | plt.plot(range(epochs),epoch_cost_tradition,color='red', label='tradition') 365 | 366 | # Writedata.write_to_excel(epoch_cost_optimize,1,"epoch_cost_optimize_E25_1") 367 | # Writedata.write_to_excel(epoch_cost_tradition,1,"epoch_cost_tradition_E25_1") 368 | # Writedata.write_to_excel(local_costs_epoch,1,"local_costs_epoch_E25_1") 369 | 370 | plt.xlabel("epoch_e25") 371 | plt.ylabel("costs") 372 | plt.legend(loc="best") 373 | plt.show() 374 | -------------------------------------------------------------------------------- /传入用户-窃听/Writedata.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | from xlwt import * 3 | 4 | 5 | # 多次实验时写入数据代码 6 | def write_data(data_list, sheet_name, iteration): 7 | for each_i in range(len(data_list)): 8 | sheet_name.write(each_i, iteration, str(data_list[each_i])) 9 | 10 | 11 | def create_csv(sheet1, data_list, iteration=1): 12 | write_data(data_list=data_list, sheet_name=sheet1, iteration=iteration) 13 | 14 | 15 | def write_to_excel(cost_list, is_one, filename): 16 | wb = Workbook() 17 | sheet1 = wb.add_sheet(sheetname='cost') 18 | if is_one: 19 | create_csv(sheet1=sheet1, data_list=cost_list) 20 | else: 21 | for index in range(len(cost_list)): 22 | create_csv(sheet1=sheet1, data_list=cost_list[index], iteration=index) 23 | wb.save(filename) 24 | 25 | 26 | def read_file(h_file, task_file): 27 | channel_uc = xlrd.open_workbook(h_file) 28 | task_file = xlrd.open_workbook(task_file) 29 | uc_sheet = channel_uc.sheets()[0] 30 | task_sheet = task_file.sheets()[0] 31 | 32 | return uc_sheet, task_sheet 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /传入用户-窃听/__pycache__/Writedata.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/__pycache__/Writedata.cpython-38.pyc -------------------------------------------------------------------------------- /传入用户-窃听/__pycache__/environment_optimize1.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/__pycache__/environment_optimize1.cpython-38.pyc -------------------------------------------------------------------------------- /传入用户-窃听/__pycache__/environment_tradition.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/__pycache__/environment_tradition.cpython-38.pyc -------------------------------------------------------------------------------- /传入用户-窃听/environment_optimize1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran=p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]])# 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]])# 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道CAPA分给两个用户的信道 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=copy.deepcopy(self.user_offrate) 67 | # print("初始化时的卸载率为：", self.user_offrate) 68 | 69 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 70 | 71 | self.lamuda = lamuda 72 | self.noise = noise 73 | self.i = 0 74 | self.cost_list = [] # 记录成本 75 | self.epoch_list = [] 76 | self.quancost_list = [] 77 | self.wucost_list = [] 78 | 79 | self.task_action = 1 # 任务每次变化率（任务卸载率） 80 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 81 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | self.p_local = p_local 83 | 84 | 85 | 86 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 87 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 88 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 89 | self.A = 1.0 90 | self.B = 0.0 91 | self.C = 0.0 92 | self.D = 1.0 93 | self.E = 0.0 94 | self.F = 0.0 95 | 96 | self.temp_tc1 = self.total_cost(self.user_offrate) 97 | self.tc_wald = self.total_cost(self.user_offrate) 98 | # print("本地的cost为：", self.temp_tc1) 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | def reset(self): # 重开，返回初始化s 108 | 109 | # 卸载率✖️任务量 110 | self.user_S = self.l * self.user_offrate # 任务量，1x6 111 | # 总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 112 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 113 | # 信道差 114 | self.Hn0_diff = self.Hn_0 - self.Hn_e 115 | self.Hn1_diff = self.Hn_1 - self.Hn_e 116 | self.Hn_diff = np.append(self.Hn0_diff, self.Hn1_diff, axis=1) 117 | 118 | S_ = np.append(self.user_S, self.P_F, axis=1) 119 | 120 | S_ = np.append(S_, self.Hn_diff, axis=1) 121 | return S_ 122 | 123 | def step(self, action): 124 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 125 | 126 | 127 | self.i = self.i + 1 128 | 129 | # 判断action执行什么动作,action为一个值 130 | 131 | # self.actions=int(action/2)#对应到卸载率的下标 132 | 133 | # if action%2==0:#增加 134 | # self.offrate_normalization(self.actions,1) 135 | # else:#减少 136 | # self.offrate_normalization(self.actions,0) 137 | 138 | #self.offrate_normalization(1,1)、 139 | 140 | if action==0:#user1本地增加 141 | self.offrate_normalization(0,1) 142 | elif action==1: 143 | self.offrate_normalization(1,1) 144 | elif action == 2: # user1-capA增加 145 | self.offrate_normalization(2,1) 146 | elif action == 3: 147 | self.offrate_normalization(3,1) 148 | elif action == 4: # user1-capB增加 149 | self.offrate_normalization(4,1) 150 | else: 151 | self.offrate_normalization(5,1) 152 | 153 | 154 | s_ = self.l * self.user_offrate 155 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 156 | s_ = np.append(s_, self.P_F, axis=1) 157 | s_ = np.append(s_, self.Hn_diff, axis=1) 158 | 159 | # 计算更新卸载率后的成本 160 | tc = self.total_cost(self.user_offrate) 161 | 162 | #本地 163 | self.temp_tc1=self.total_cost(self.judge) 164 | 165 | 166 | if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 167 | r = -100 168 | elif tc < self.tc_wald: 169 | r = 100 170 | else: 171 | r = -100 172 | self.user_S = s_ # 更新状态 173 | if self.i % 2000==0: 174 | print("----------------------------------") 175 | print("执行了第", self.i, "次动作") 176 | print("此轮更新卸载率后的成本为:", tc) 177 | print("此时卸载率为：", self.user_offrate) 178 | 179 | 180 | self.tc_wald = tc 181 | 182 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 183 | if self.i == self.suspend: 184 | done = self.tc_wald 185 | self.i=0 186 | 187 | 188 | 189 | return s_, r, done, self.tc_wald,self.temp_tc1 190 | 191 | #根据动作，改变对应的卸载率 192 | def offrate_normalization(self,number,add_subtraction):#如增加，add_subtraction=1 193 | if add_subtraction: 194 | if number==0: 195 | self.A+=1.0 196 | elif number==1: 197 | self.B+=1.0 198 | elif number==2: 199 | self.C+=1.0 200 | elif number==3: 201 | self.D+=1.0 202 | elif number==4: 203 | self.E+=1.0 204 | else: 205 | self.F+=1.0 206 | else: 207 | if number==0: 208 | if self.A>1: 209 | self.A-=1.0 210 | elif number==1: 211 | if self.B>1: 212 | self.B-=1.0 213 | elif number==2: 214 | if self.C>1: 215 | self.C-=1.0 216 | elif number==3: 217 | if self.D>1: 218 | self.D-=1.0 219 | elif number==4: 220 | if self.E>1: 221 | self.E-=1.0 222 | else: 223 | if self.F>1: 224 | self.F-=1.0 225 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 226 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 227 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 228 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 229 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 230 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 231 | 232 | 233 | 234 | def action_state_num(self): # 返回动作与环境个数 235 | #动作个数 236 | action_n = self.user_num * (self.cap_num+1)#dqn 237 | 238 | # 239 | Hn_num = self.cap_num * self.user_num 240 | state_num = self.user_num*(self.cap_num +1) +self.cap_num*2+Hn_num 241 | 242 | return action_n, state_num 243 | 244 | def Time(self, user_offrate): 245 | # 计算香农公式Wn 246 | 247 | # # #-----优化带宽------------- 248 | # if self.user_offrate.all()!=0: 249 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 250 | # else: 251 | # w_ = self.W / self.user_num 252 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 253 | # #------------------- 254 | 255 | #--------均分带宽-------------- 256 | # w_0 = self.W0 / self.user_num#shape[1,2] 257 | # w_1 = self.W1 / self.user_num#shape[1,2] 258 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 259 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 260 | #----------------------------- 261 | 262 | 263 | 264 | 265 | if (user_offrate!=self.judge).any(): 266 | 267 | #----解析------- 268 | a_A=W_A_user*np.log2(self.Hn_0/self.Hn_e) 269 | a_B=W_B_user*np.log2(self.Hn_1/self.Hn_e) 270 | 271 | b_A=W_A_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_0) / math.log(2) 272 | b_B=W_B_user * pow(self.noise, 2) * (1/self.Hn_e - 1/self.Hn_1) / math.log(2) 273 | 274 | 275 | 276 | #按user分 277 | 278 | 279 | b_user1=np.array([[b_A[0][0],b_B[0][0]]]) 280 | b_user2 = np.array([[b_A[0][1], b_B[0][1]]]) 281 | 282 | self.l1=self.l[0][1:3]#传输与cap计算的任务量 283 | self.l2=self.l[0][4:] 284 | user_offrate_user1=user_offrate[0][1:3]#user1的传输与cap计算的卸载率 285 | user_offrate_user2=user_offrate[0][4:] 286 | 287 | a_user1 =np.array([[a_A[0][0], a_B[0][0]]]) 288 | a_user2 = np.array([[a_A[0][1], a_B[0][1]]]) 289 | 290 | # A_users=np.append(A_user1,A_user2,axis=1) 291 | A_users=a_A*a_B 292 | A_user1=A_users[0][0]#数字 293 | A_user2=A_users[0][1]#数字 294 | 295 | lumuda_user1=pow(np.sum(A_user1*np.sqrt(self.l1*user_offrate_user1*b_user1)/a_user1)/(self.p_tran[0][0]*A_user1-np.sum(b_user1*A_user1/a_user1)),2) 296 | lumuda_user2=pow(np.sum(A_user2*np.sqrt(self.l2*user_offrate_user2*b_user2)/a_user2)/(self.p_tran[0][1]*A_user2-np.sum(b_user2*A_user2/a_user2)),2) 297 | lumuda_users=np.array([[lumuda_user1,lumuda_user2]]) 298 | 299 | 300 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 301 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 302 | 303 | mu_capA = pow(np.sum(np.sqrt(self.offrate_A*self.user_l))/self.F_cap[0][0],2)#检查到这 304 | mu_capB = pow(np.sum(np.sqrt(self.offrate_B * self.user_l)) / self.F_cap[0][1],2) 305 | mu_cpas = np.array([[mu_capA, mu_capB]]) 306 | 307 | #分配传输功率 308 | 309 | self.p_tran_user1=(np.sqrt(self.l1*user_offrate_user1*b_user1/lumuda_user1)+b_user1)/a_user1 310 | self.p_tran_user2=(np.sqrt(self.l2*user_offrate_user2*b_user2/lumuda_user2)+b_user2)/a_user2 311 | 312 | self.p_tran_capA=np.array([[self.p_tran_user1[0][0],self.p_tran_user2[0][0]]]) 313 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 314 | # else: 315 | # p_tran_1 = self.p_tran[0][1] / self.cap_num 316 | # self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) 317 | # self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) 318 | 319 | #分配算力 320 | self.F_capA = np.sqrt(self.offrate_A * self.user_l / mu_capA) 321 | self.F_capB = np.sqrt(self.offrate_B * self.user_l / mu_capB) 322 | 323 | 324 | else: 325 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 326 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 327 | 328 | #均分传输功率 329 | p_tran_0 = self.p_tran[0][0] / self.cap_num 330 | p_tran_1 = self.p_tran[0][1] / self.cap_num 331 | 332 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 333 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 334 | 335 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap均分给每一个user的传输功率 336 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap均分给每一个user的传输功率 337 | 338 | #均分算力 339 | self.F_capA = np.full(shape=(1, self.user_num), fill_value=self.F_0) # capA均分给每一个user的算力 340 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) # capB均分给每一个user的算力 341 | 342 | C_A1 = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2)) # [cap_A_user1,cap_A_user2] 343 | C_B1 = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 344 | 345 | # 窃听者 346 | C_A_E = W_A_user * np.log2(1 + self.p_tran_capA * self.Hn_e / pow(self.noise, 2)) 347 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 348 | 349 | # 更新速率 350 | C_A = C_A1 - C_A_E 351 | C_B = C_B1 - C_B_E 352 | 353 | 354 | #本地 355 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 356 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 357 | 358 | # ----------CAPA-------------- 359 | #传输 360 | self.T_tran_A=np.zeros(shape=(1,2)) 361 | for i in range(self.user_num): 362 | if(self.offrate_A[0][i]): 363 | self.T_tran_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] / C_A[0][i]#shape[1,2] 364 | #CAP端 365 | 366 | self.T_cap_A=np.zeros(shape=(1,2)) 367 | for i in range(self.user_num): 368 | if(self.offrate_A[0][i]): 369 | self.T_cap_A[0][i]=self.user_l[0][i] * self.offrate_A[0][i] * self.omega * self.Mb_to_bit / self.F_capA[0][i]#shape[1,2] 370 | 371 | #----------CAPB-------------- 372 | #传输 373 | 374 | self.T_tran_B = np.zeros(shape=(1, 2)) 375 | for i in range(self.user_num): 376 | if (self.offrate_B[0][i]): 377 | self.T_tran_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] / C_B[0][i] # shape[1,2] 378 | 379 | #CAP端 380 | 381 | # self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 382 | self.T_cap_B = np.zeros(shape=(1, 2)) 383 | for i in range(self.user_num): 384 | if (self.offrate_B[0][i]): 385 | self.T_cap_B[0][i] = self.user_l[0][i] * self.offrate_B[0][i] * self.omega * self.Mb_to_bit / self.F_capB[0][i] # shape[1,2] 386 | 387 | # ----------------------------- 388 | 389 | #求T 390 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 391 | 392 | 393 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 394 | # temp = 1 - user_offrate 395 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 396 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 397 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 398 | # T_temp = self.T_tran + self.T_cap 399 | # 400 | # T_max = np.max(T_temp) 401 | # T_max1 = np.max(self.T_local) 402 | # T = max(T_max, T_max1) 403 | 404 | return T 405 | 406 | def Energy(self, ): 407 | self.E_local=self.T_local * self.p_local 408 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 409 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 410 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 411 | 412 | # self.E_local = self.T_local * self.p_local 413 | # self.E_tran = self.T_tran * self.p_tran 414 | # self.E_total = self.E_tran + self.E_local 415 | # E = np.sum(self.E_total) 416 | return E 417 | 418 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 419 | T = self.Time(user_offrate) 420 | # E = self.Energy() 421 | total_cost = self.lamuda * T 422 | return total_cost 423 | 424 | def cost_print(self): 425 | self.cost_list1 = self.cost_list # 记录成本 426 | self.epoch_list1 = self.epoch_list 427 | self.quancost_list1 = self.quancost_list 428 | self.wucost_list1 = self.wucost_list 429 | self.cost_list = [] # 记录成本 430 | self.epoch_list = [] 431 | self.quancost_list = [] 432 | self.wucost_list = [] 433 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 434 | -------------------------------------------------------------------------------- /传入用户-窃听/environment_tradition.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import copy 4 | import math 5 | 6 | # S=[任务量✖️卸载率，总算力，总功率] 7 | # A=[卸载率1增，卸载率1减，卸载率2增，卸载率2减，。。。] 8 | 9 | 10 | class environment1(object): # 奖励r由tc与tc-1比较大小得出。Hn：0.992912817258564, 0.992299817945962 11 | def __init__(self, user_num, cap_num, W, f_local, omega, F_cap, p_local, p_tran, lamuda, noise, Hn_0,Hn_1,Hn_e,user_l, suspend): 12 | # user_num:用户总数, cap_num:cap总数,W:单个CAP的总带宽,f_local为本地用户计算能力,omega:计算时间T的w 13 | # C为计算时间的传输速率,F_cap:分配给user的计算能力,p_local:本地计算功率,p_tran：传输功率;lamuda:时间与能耗之间的比例系数 14 | # noise:噪音的方差，用于计算香农公式的C;Hn:香农公式的hnm，用于计算C。suspend:用于判断执行多少次动作后停止 15 | self.user_num = user_num 16 | self.cap_num = cap_num 17 | 18 | self.f = f_local 19 | self.omega = omega 20 | 21 | #W为总带宽，[capA,capB]，需要取出来 22 | self.W0 = W 23 | self.W1 = W 24 | 25 | # F_cap为算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 26 | 27 | self.F_cap=F_cap 28 | self.F_0 = F_cap[0][0] / self.user_num#均分算力 29 | self.F_1 = F_cap[0][1] / self.user_num#均分算力 30 | 31 | 32 | 33 | self.F_capA = np.full(shape=(1,self.user_num),fill_value=self.F_0)#capA均分给每一个user的算力 34 | self.F_capB = np.full(shape=(1, self.user_num), fill_value=self.F_1) #capB均分给每一个user的算力 35 | 36 | #传输带宽（优化对象）,p_tran总传输功率[user1,user2] 37 | self.p_tran = p_tran 38 | p_tran_0 = self.p_tran[0][0] / self.cap_num 39 | p_tran_1 = self.p_tran[0][1] / self.cap_num 40 | 41 | self.p_tran_user1 = np.full(shape=(1, self.cap_num), fill_value=p_tran_0) # 第一个user分给不同信道的传输功率 42 | self.p_tran_user2 = np.full(shape=(1, self.cap_num), fill_value=p_tran_1) # 第二个user分给不同信道的传输功率 43 | 44 | self.p_tran_capA = np.array([[self.p_tran_user1[0][0], self.p_tran_user2[0][0]]]) # 第一个cap对应每一个user的传输功率 45 | self.p_tran_capB = np.array([[self.p_tran_user1[0][1], self.p_tran_user2[0][1]]]) # 第二个cap对应每一个user的传输功率 46 | 47 | 48 | #信道，Hn为总信道[CAP] 49 | self.Hn_0=Hn_0.reshape(1,2) 50 | self.Hn_1=Hn_1.reshape(1,2) 51 | 52 | self.suspend = suspend 53 | 54 | #分配任务量 55 | self.user_l = user_l#用户最少2个 56 | 57 | #窃听者 58 | self.Hn_e=Hn_e.reshape(1,2) 59 | 60 | #卸载率初始化： 61 | 62 | self.user_offrate = np.zeros(shape=(1,(cap_num + 1)*user_num))#user_offrate的shape[1,(cap_num + 1)*user_num] 63 | for i in range((cap_num + 1)*user_num): 64 | if i % (cap_num + 1)==0: 65 | self.user_offrate[0][i] = 1.0 66 | self.judge=self.user_offrate 67 | # print("初始化时的卸载率为：", self.user_offrate) 68 | 69 | # self.user_offrate_ABC=copy.deepcopy(self.user_offrate) 70 | 71 | self.lamuda = lamuda 72 | self.noise = noise 73 | self.i = 0 74 | self.cost_list = [] # 记录成本 75 | self.epoch_list = [] 76 | self.quancost_list = [] 77 | self.wucost_list = [] 78 | 79 | self.task_action = 1 # 任务每次变化率（任务卸载率） 80 | self.Mb_to_bit = 2 ** 20 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 81 | # self.Mb_to_bit = 1 # Mb to bit 1MB = 1024KB = 1024*1024B = 1024 * 1024 *8 bit =2^23 82 | self.p_local = p_local 83 | 84 | #-------------------------- 85 | #窃听者 86 | 87 | 88 | #------------------------- 89 | 90 | 91 | 92 | 93 | 94 | 95 | # self.l1=np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]) 96 | # self.l2 = np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]) 97 | self.l=np.append(np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][0]),np.full(shape=(1, self.cap_num + 1), fill_value=self.user_l[0][1]),axis=1) 98 | self.A = 1.0 99 | self.B = 0.0 100 | self.C = 0.0 101 | self.D = 1.0 102 | self.E = 0.0 103 | self.F = 0.0 104 | 105 | self.temp_tc1 = self.total_cost(self.user_offrate) 106 | self.tc_wald = self.total_cost(self.user_offrate) 107 | # print("本地的cost为：", self.temp_tc1) 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | def reset(self): # 重开，返回初始化s 117 | 118 | #卸载率✖️任务量 119 | self.user_S = self.l * self.user_offrate #任务量，1x6 120 | #总功率和总算力：p_tran[0],p_tran[1]，F_cap[0]，F_cap[1] 121 | self.P_F=np.append(self.p_tran,self.F_cap,axis=1) 122 | #信道差 123 | self.Hn0_diff=self.Hn_0-self.Hn_e 124 | self.Hn1_diff=self.Hn_1-self.Hn_e 125 | self.Hn_diff=np.append(self.Hn0_diff,self.Hn1_diff,axis=1) 126 | 127 | S_=np.append(self.user_S,self.P_F,axis=1) 128 | 129 | S_=np.append(S_,self.Hn_diff,axis=1) 130 | return S_ 131 | 132 | def step(self, action): 133 | # 输入动作，输出下一步状态s_，奖励r,是否结束done。得根据给的动作，得出下一步s_，还有r，并且r需要求出当前状态下的成本 134 | 135 | 136 | self.i = self.i + 1 137 | 138 | # 判断action执行什么动作,action为一个值 139 | 140 | # self.actions=int(action/2)#对应到卸载率的下标 141 | 142 | # if action%2==0:#增加 143 | # self.offrate_normalization(self.actions,1) 144 | # else:#减少 145 | # self.offrate_normalization(self.actions,0) 146 | 147 | if action == 0: # user1本地增加 148 | self.offrate_normalization(0, 1) 149 | elif action == 1: 150 | self.offrate_normalization(1, 1) 151 | elif action == 2: # user1-capB增加 152 | self.offrate_normalization(2, 1) 153 | elif action == 3: 154 | self.offrate_normalization(3, 1) 155 | elif action == 4: # user2-capA增加 156 | self.offrate_normalization(4, 1) 157 | else: 158 | self.offrate_normalization(5, 1) 159 | 160 | 161 | s_ = self.l * self.user_offrate 162 | self.P_F = np.append(self.p_tran, self.F_cap, axis=1) 163 | s_ = np.append(s_, self.P_F, axis=1) 164 | s_=np.append(s_,self.Hn_diff,axis=1) 165 | tc = self.total_cost(self.user_offrate) # 更新后的卸载率的成本 166 | 167 | if tc > self.tc_wald: # tc为更新后的成本，tc_wald为更新前的成本 168 | r = -100 169 | elif tc < self.tc_wald: 170 | r = 100 171 | else: 172 | r = -100 173 | self.user_S = s_ # 更新状态 174 | # if self.i % 500==0: 175 | # print("----------------------------------") 176 | # print("执行了第", self.i, "次动作") 177 | # print("此轮更新卸载率后的成本为:", tc) 178 | # print("此时卸载率为：", self.user_offrate) 179 | 180 | 181 | self.tc_wald = tc 182 | 183 | done = 0 # 不知道什么情况 done输出Ture，代码也没写 184 | if self.i == self.suspend: 185 | done = self.tc_wald 186 | self.i = 0 187 | 188 | 189 | 190 | return s_, r, done, self.tc_wald,self.temp_tc1 191 | 192 | #根据动作，改变对应的卸载率 193 | def offrate_normalization(self,number,add_subtraction):#如增加，add_subtraction=1 194 | if add_subtraction: 195 | if number==0: 196 | self.A+=1.0 197 | elif number==1: 198 | self.B+=1.0 199 | elif number==2: 200 | self.C+=1.0 201 | elif number==3: 202 | self.D+=1.0 203 | elif number==4: 204 | self.E+=1.0 205 | else: 206 | self.F+=1.0 207 | else: 208 | if number==0: 209 | if self.A>1: 210 | self.A-=1.0 211 | elif number==1: 212 | if self.B>1: 213 | self.B-=1.0 214 | elif number==2: 215 | if self.C>1: 216 | self.C-=1.0 217 | elif number==3: 218 | if self.D>1: 219 | self.D-=1.0 220 | elif number==4: 221 | if self.E>1: 222 | self.E-=1.0 223 | else: 224 | if self.F>1: 225 | self.F-=1.0 226 | self.user_offrate[0][0] = self.A/ (self.A+self.B+self.C) 227 | self.user_offrate[0][1] = self.B / (self.A + self.B + self.C) 228 | self.user_offrate[0][2] = self.C / (self.A + self.B + self.C) 229 | self.user_offrate[0][3] = self.D / (self.D + self.E + self.F) 230 | self.user_offrate[0][4] = self.E / (self.D + self.E + self.F) 231 | self.user_offrate[0][5] = self.F / (self.D + self.E + self.F) 232 | 233 | 234 | 235 | def action_state_num(self): # 返回动作与环境个数 236 | action_n = self.user_num * (self.cap_num+1)#dqn 237 | 238 | Hn_num = self.cap_num * self.user_num 239 | state_num = self.user_num * (self.cap_num + 1) + self.cap_num * 2 + Hn_num 240 | return action_n, state_num 241 | 242 | def Time(self, user_offrate): 243 | # 计算香农公式Wn 244 | 245 | # # #-----优化带宽------------- 246 | # if self.user_offrate.all()!=0: 247 | # W_user=np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))*self.W/np.sum(np.sqrt((self.user_l*self.user_offrate*self.p_tran)/np.log2(1+(self.p_tran*np.square(self.Hn))/pow(self.noise, 2)))) 248 | # else: 249 | # w_ = self.W / self.user_num 250 | # W_user = np.full(shape=(1,self.user_num),fill_value=w_) 251 | # #------------------- 252 | 253 | #--------均分带宽-------------- 254 | # w_0 = self.W0 / self.user_num#shape[1,2] 255 | # w_1 = self.W1 / self.user_num#shape[1,2] 256 | W_A_user = np.full(shape=(1, self.user_num), fill_value=self.W0) 257 | W_B_user = np.full(shape=(1, self.user_num), fill_value=self.W1) 258 | #----------------------------- 259 | 260 | C_A1=W_A_user*np.log2(1 + self.p_tran_capA * self.Hn_0 / pow(self.noise, 2))#[cap_A_user1,cap_A_user2] 261 | C_B1=W_B_user*np.log2(1 + self.p_tran_capB * self.Hn_1 / pow(self.noise, 2)) 262 | 263 | #窃听者 264 | C_A_E=W_A_user*np.log2(1+self.p_tran_capA*self.Hn_e / pow(self.noise, 2)) 265 | C_B_E = W_B_user * np.log2(1 + self.p_tran_capB * self.Hn_e / pow(self.noise, 2)) 266 | 267 | #更新速率 268 | C_A=C_A1 - C_A_E 269 | C_B=C_B1 - C_B_E 270 | 271 | self.offrate_A = np.array([[user_offrate[0][1], user_offrate[0][4]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 272 | self.offrate_B = np.array([[user_offrate[0][2], user_offrate[0][5]]]) # shape:[用户1到capA的卸载率，用户2到capA的卸载率] 273 | 274 | #本地 275 | self.offrate_local=np.array([[user_offrate[0][0],user_offrate[0][3]]])#shape:[本地用户1的卸载率，本地用户2的卸载率] 276 | self.T_local=self.user_l*self.offrate_local * self.omega * self.Mb_to_bit / self.f#shape[1,2] 277 | 278 | # ----------CAPA-------------- 279 | #传输 280 | 281 | self.T_tran_A=self.user_l * self.offrate_A / C_A#shape[1,2] 282 | #CAP端 283 | self.T_cap_A=self.user_l * self.offrate_A * self.omega * self.Mb_to_bit / self.F_capA#shape[1,2] 284 | #----------CAPB-------------- 285 | #传输 286 | 287 | self.T_tran_B=self.user_l * self.offrate_B / C_B#shape[1,2] 288 | #CAP端 289 | self.T_cap_B=self.user_l * self.offrate_B * self.omega * self.Mb_to_bit / self.F_capB#shape[1,2] 290 | # ----------------------------- 291 | 292 | #求T 293 | T=np.sum(self.T_local)+np.sum(self.T_tran_A)+np.sum(self.T_cap_A)+np.sum(self.T_tran_B)+np.sum(self.T_cap_B) 294 | 295 | T_off_A=self.T_tran_A+self.T_cap_A 296 | 297 | # C = W_user * np.log2(1 + self.p_tran * self.Hn / pow(self.noise, 2)) 298 | # temp = 1 - user_offrate 299 | # self.T_local = self.user_l * temp * self.omega * self.Mb_to_bit / self.f # shape(1,num) 300 | # self.T_tran = self.user_l * user_offrate * self.omega * self.Mb_to_bit / C # shape(1,num) 301 | # self.T_cap = self.user_l * user_offrate * self.omega * self.Mb_to_bit / self.F # shape(1,num) 302 | # T_temp = self.T_tran + self.T_cap 303 | # 304 | # T_max = np.max(T_temp) 305 | # T_max1 = np.max(self.T_local) 306 | # T = max(T_max, T_max1) 307 | 308 | return T 309 | 310 | def Energy(self, ): 311 | self.E_local=self.T_local * self.p_local 312 | self.E_tran_A=self.T_tran_A * self.p_tran_capA 313 | self.E_tran_B=self.T_tran_B * self.p_tran_capB 314 | E=np.sum(self.E_local)+np.sum(self.E_tran_A)+np.sum(self.E_tran_B) 315 | 316 | # self.E_local = self.T_local * self.p_local 317 | # self.E_tran = self.T_tran * self.p_tran 318 | # self.E_total = self.E_tran + self.E_local 319 | # E = np.sum(self.E_total) 320 | return E 321 | 322 | def total_cost(self, user_offrate): # total_cost 由T与E决定,返回当前时刻的成本 323 | T = self.Time(user_offrate) 324 | E = self.Energy() 325 | total_cost = self.lamuda * T 326 | return total_cost 327 | 328 | def cost_print(self): 329 | self.cost_list1 = self.cost_list # 记录成本 330 | self.epoch_list1 = self.epoch_list 331 | self.quancost_list1 = self.quancost_list 332 | self.wucost_list1 = self.wucost_list 333 | self.cost_list = [] # 记录成本 334 | self.epoch_list = [] 335 | self.quancost_list = [] 336 | self.wucost_list = [] 337 | return self.cost_list1, self.epoch_list1, self.quancost_list1, self.wucost_list1 338 | -------------------------------------------------------------------------------- /传入用户-窃听/epoch_cost_optimize_E25_1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/epoch_cost_optimize_E25_1 -------------------------------------------------------------------------------- /传入用户-窃听/epoch_cost_tradition_E25_1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/epoch_cost_tradition_E25_1 -------------------------------------------------------------------------------- /传入用户-窃听/local_costs_epoch_E25_1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/传入用户-窃听/local_costs_epoch_E25_1 -------------------------------------------------------------------------------- /传入用户-窃听/test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "user_num=2" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 5, 18 | "outputs": [ 19 | { 20 | "data": { 21 | "text/plain": "array([0.04008809, 0.03647456])" 22 | }, 23 | "execution_count": 5, 24 | "metadata": {}, 25 | "output_type": "execute_result" 26 | } 27 | ], 28 | "source": [ 29 | "Hn_e1 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') *\n", 30 | " np.random.normal(1, 0.1, user_num))) ** 2\n", 31 | "Hn_e2 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') *\n", 32 | " np.random.normal(1, 0.1, user_num))) ** 2\n", 33 | "Hn_e3 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') *\n", 34 | " np.random.normal(1, 0.1, user_num))) ** 2\n", 35 | "Hn_e4 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') *\n", 36 | " np.random.normal(1, 0.1, user_num))) ** 2\n", 37 | "Hn_e5 = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, user_num) + np.complex('j') *\n", 38 | " np.random.normal(1, 0.1, user_num))) ** 2\n", 39 | "Hn_e = Hn_e2 + Hn_e3 + Hn_e4 + Hn_e5\n", 40 | "\n", 41 | "Hn_e" 42 | ], 43 | "metadata": { 44 | "collapsed": false, 45 | "pycharm": { 46 | "name": "#%%\n" 47 | } 48 | } 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": "array([[0.01109678, 0.01234598]])" 57 | }, 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "output_type": "execute_result" 61 | } 62 | ], 63 | "source": [ 64 | "Hn_e=np.zeros(shape=(1,2))\n", 65 | "\n", 66 | "Hn_1=np.zeros(shape=(1,2))\n", 67 | "\n", 68 | "Hn_2=np.zeros(shape=(1,2))\n", 69 | "for i in range(1):\n", 70 | " Hn_e += (np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, 2) + np.complex('j') *\n", 71 | " np.random.normal(1, 0.1, 2))) ** 2).reshape(1,2)\n", 72 | "for i in range(1):\n", 73 | " Hn_1 += (np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, 2) + np.complex('j') *\n", 74 | " np.random.normal(1, 0.1, 2))) ** 2).reshape(1,2)\n", 75 | "for i in range(1):\n", 76 | " Hn_2 += (np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, 2) + np.complex('j') *\n", 77 | " np.random.normal(1, 0.1, 2))) ** 2).reshape(1,2)" 78 | ], 79 | "metadata": { 80 | "collapsed": false, 81 | "pycharm": { 82 | "name": "#%%\n" 83 | } 84 | } 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 4, 89 | "outputs": [ 90 | { 91 | "ename": "NameError", 92 | "evalue": "name 'Hn_1' is not defined", 93 | "output_type": "error", 94 | "traceback": [ 95 | "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", 96 | "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", 97 | "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0mnp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0marray\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mHn_e\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0mHn_1\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0mHn_2\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0maxis\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m1\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m", 98 | "\u001B[0;31mNameError\u001B[0m: name 'Hn_1' is not defined" 99 | ] 100 | } 101 | ], 102 | "source": [ 103 | "np.array(Hn_e,Hn_1,Hn_2,axis=1)" 104 | ], 105 | "metadata": { 106 | "collapsed": false, 107 | "pycharm": { 108 | "name": "#%%\n" 109 | } 110 | } 111 | } 112 | ], 113 | "metadata": { 114 | "kernelspec": { 115 | "display_name": "Python 3", 116 | "language": "python", 117 | "name": "python3" 118 | }, 119 | "language_info": { 120 | "codemirror_mode": { 121 | "name": "ipython", 122 | "version": 2 123 | }, 124 | "file_extension": ".py", 125 | "mimetype": "text/x-python", 126 | "name": "python", 127 | "nbconvert_exporter": "python", 128 | "pygments_lexer": "ipython2", 129 | "version": "2.7.6" 130 | } 131 | }, 132 | "nbformat": 4, 133 | "nbformat_minor": 0 134 | } -------------------------------------------------------------------------------- /改善环境版/DQN_env_wu.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import os 3 | import Writedata 4 | os.environ['KMP_DUPLICATE_LIB_OK']='True' 5 | #2021年11。2。 用了裕鑫的环境,自己的模型 6 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2， .....] 7 | # 输出的a为某个用户卸载率增加或者减少 8 | 9 | #2cap 2user 10 | import torch 11 | import numpy as np 12 | import Writedata 13 | import matplotlib.pyplot as plt 14 | 15 | from Environment_Wu import Environment 16 | from Environment_Wu_tradition import Environment1 17 | 18 | #%% 19 | 20 | # user_num=input("please input user_num") 21 | 22 | #初始化参数----------------- 23 | user_num=2 24 | cap_num=2 25 | users_num=int(user_num) 26 | MAX_EP=2000#步数 27 | epochs=100 28 | e_num=5#窃听数 29 | 30 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, [user_num, cap_num]) + np.complex('j') * 31 | np.random.normal(1, 1, [user_num, cap_num]))) ** 2 32 | 33 | 34 | Hn_e = np.abs(0.1 * np.sqrt(1 / 2) * (np.random.normal(1, 1, user_num) + np.complex('j') * 35 | np.random.normal(1, 1, user_num))) ** 2##窃听者,[user1,user2] 36 | 37 | 38 | user_l=np.array([10, 550]) 39 | 40 | #---------------- 41 | 42 | env = Environment(user_num=2, cap_num=2, W=5e6, 43 | f_local=np.array([1.4 * 10e8, 0.43 * 10e8]), 44 | omega=330 / 8, 45 | F_cap=np.array([10 * 10e8, 3 * 10e8]), p_local=1e-1, p_tran=np.array([2, 3]), lamda_=1.0, 46 | p_noise=0.1, Hn=Hn, Hn_e=Hn_e, user_l=user_l,Mb_to_bit=2 ** 20 47 | ) 48 | 49 | 50 | #获取环境的相关参数: 51 | n_action,n_state=env.action_state_num() 52 | #其他超参数的定义： 53 | MEMORY_CAPACITY=5000#记忆库的存储大小 54 | greedy=0.57#贪婪概率 55 | discount=0.9#折扣率 ff 56 | batch_size=32#一次训练32个数据 57 | TARGET_REPLACE_ITER=100#目标网络的更新频率 58 | 59 | def get_channel_gain_sum():#共谋 60 | # Hn (2,2) 61 | while True: 62 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 63 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 64 | Hn_e = np.abs(0.2 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, e_num]) + np.complex('j') * 65 | np.random.normal(1, 0.1, [user_num, e_num]))) ** 2 66 | Hn_e = np.sum(Hn_e, axis=1) # 共谋取和 67 | if(Hn>Hn_e).all(): 68 | break 69 | 70 | return Hn, Hn_e 71 | 72 | def get_channel_gain_max():#非共谋 73 | while True: 74 | Hn = np.abs(1.1 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, cap_num]) + np.complex('j') * 75 | np.random.normal(1, 0.1, [user_num, cap_num]))) ** 2 76 | 77 | Hn_e = np.abs(0.2 * np.sqrt(1 / 2) * (np.random.normal(1, 0.1, [user_num, e_num]) + np.complex('j') * 78 | np.random.normal(1, 0.1, [user_num, e_num]))) ** 2 79 | Hn_e=np.max(Hn_e, axis=1) 80 | if (Hn > Hn_e).all(): 81 | break 82 | return Hn, Hn_e 83 | 84 | #%% 85 | 86 | class Net(torch.nn.Module): 87 | def __init__(self): 88 | super(Net, self).__init__() 89 | self.linear1=torch.nn.Linear(n_state,64)#(10,20) 90 | self.linear1.weight.data.normal_(0, 0.1) 91 | self.linear2=torch.nn.Linear(64,256) 92 | self.linear2.weight.data.normal_(0, 0.1) 93 | self.linear3=torch.nn.Linear(256,64) 94 | self.linear3.weight.data.normal_(0, 0.1) 95 | self.linear4=torch.nn.Linear(64,n_action)#(30,12) 96 | self.linear4.weight.data.normal_(0, 0.1) 97 | self.relu=torch.nn.ReLU() 98 | self.dropout=torch.nn.Dropout(p=0.5) 99 | def forward(self,x): 100 | x=self.relu(self.linear1(x)) 101 | x=self.dropout(x) 102 | x=self.relu(self.linear2(x)) 103 | x = self.relu(self.linear3(x)) 104 | q_values=self.linear4(x) 105 | return q_values 106 | 107 | 108 | #%% 109 | 110 | class DQN (): 111 | def __init__(self):#可能要改成def __init__(self,n_state,n_action) 112 | 113 | #记忆库：存s,a,r,s_ 一共6维 114 | self.memory=np.zeros((MEMORY_CAPACITY,n_state*2+2)) 115 | self.memory_count=0#记忆库计数，用于求index，以及当记忆库计数 > 记忆库容量时 开始学习 116 | 117 | #目标网络,参数更新慢。参数更新跟着agent_network走。用于预测q现实 118 | self.target_network=Net() 119 | 120 | #目标网络更新计数 121 | self.learn_step_counter=0 122 | 123 | #用于训练的网络，参数更新及时。用于预测q估计 124 | self.agent_network=Net() 125 | 126 | #优化器与损失器 127 | self.loss_func=torch.nn.MSELoss() 128 | self.optimizer=torch.optim.Adam(self.agent_network.parameters(),lr=0.001) 129 | def choose_action(self,s):#这里每次只输入一个样本。 130 | # 输入的s为：[任务量1✖️卸载率1，任务量2✖️卸载率2] 131 | # 输出的action为0或1或2或3 132 | self.agent_network.eval() 133 | if np.random.uniform() < greedy:#选择最优动作 134 | #送入神经网络前，先要处理s成为torch型，且要求shape为(1,4) 135 | #s=torch.tensor(s,dtype=float).reshape(1,4) 136 | #s=s.to(torch.float32) 137 | s=torch.FloatTensor(s).reshape(-1,n_state) 138 | 139 | #把s送入神经网络得到a_value, a_value的shape 为（4，1）。并且得到的是四个动作q值 140 | action_value=self.agent_network.forward(s) 141 | 142 | #输出每个样本中预测的最大值，输出[最大值，下标]。只要下标 143 | _,action=torch.max(action_value.data,dim=1) 144 | #在从numpy转为int值 145 | action=action.numpy()[0] 146 | #此时action可能是0、1、2、3 147 | 148 | 149 | else:#随机选择动作 150 | action = np.random.randint(0, n_action) 151 | 152 | return action 153 | 154 | def store_transition(self,s,action,r,s_): 155 | #将四个值打包在一起 156 | 157 | #action为0or 1 or 2 or 3 158 | s=s.reshape(n_state) 159 | s_=s_.reshape(n_state) 160 | 161 | self.transition=np.hstack((s,[action,r],s_)) 162 | 163 | #如果记忆库满了, 就覆盖老数据 164 | index = self.memory_count % MEMORY_CAPACITY 165 | 166 | #传入记忆库 167 | 168 | self.memory[index,:]=self.transition 169 | self.memory_count+=1 170 | #print("目前记忆库存储的总数为",self.memory_count)#要删除 171 | 172 | def learn(self):#从记忆库中选择batch个数据进行训练学习 173 | #首先判断target_network是否需要更新 174 | 175 | if self.learn_step_counter % TARGET_REPLACE_ITER==0: 176 | self.target_network.load_state_dict(self.agent_network.state_dict())#更新目标网络参数 177 | self.learn_step_counter+=1 178 | 179 | #随机从memory中抽32个数据，即batch=32 180 | sample_index=np.random.choice(MEMORY_CAPACITY,batch_size)#从0-1999中抽32个数字，shape为（32，） 181 | batch_memory=self.memory[sample_index,:]#获取32个数据。 shape 为（32,6） 182 | 183 | self.agent_network.train() 184 | 185 | b_s = torch.FloatTensor(batch_memory[:, :n_state]) 186 | b_a = torch.LongTensor(batch_memory[:, n_state:n_state+1].astype(int)) 187 | b_r = torch.FloatTensor(batch_memory[:, n_state+1:n_state+2]) 188 | b_s_ = torch.FloatTensor(batch_memory[:, -n_state:]) 189 | # b_memory shape (batch,6) 190 | # b_s shape (batch,2) 191 | # b_a shape (batch,1) 192 | # b_r shape (batch,1) 193 | # b_s_ shape (batch,2) 194 | 195 | 196 | #把样本state送入agent网络训练，得到q估计 197 | q_predict=self.agent_network(b_s)#shape 为 (32,4) 198 | q_predict=q_predict.gather(1,b_a)#shaoe 为 (32,1)。根据b_a找出q估计 199 | 200 | 201 | # #把样本送入target网络计算，得到Q(s+1) 202 | # q_next=self.target_network(b_s_).detach()# q_next 不进行反向传播, 所以使用detach。shape为 (32,4) 203 | # q_next,_=torch.max(q_next,dim=1)#输出每个样本中预测的最大值，输出[最大值，下标]。只要最大值 204 | # q_next=q_next.reshape((-1,1))#q_next的shape 为（32,1） 205 | # #计算q真实 206 | # q_target=b_r+discount*q_next#q_target的shape 为（32,1） 207 | 208 | #改用ddqn 209 | q_next=self.agent_network(b_s_).detach()# 210 | _,b_a_=torch.max(q_next,dim=1)#由dqn网络选出动作 211 | b_a_=b_a_.reshape((-1,1))#b_a_的shape 为（32,1） 212 | q_next_target=self.target_network(b_s_).detach() 213 | q_next=q_next_target.gather(1,b_a_)#q_next的shape 为（32,1 214 | #计算q真实 215 | q_target=b_r+discount*q_next#q_target的shape 为（32,1） 216 | 217 | 218 | 219 | loss=self.loss_func(q_predict,q_target) 220 | # if self.learn_step_counter%5000==0: 221 | # print("第",self.learn_step_counter,"次强化学习的loss为：",loss.item()) 222 | self.optimizer.zero_grad() 223 | loss.backward() 224 | self.optimizer.step() 225 | return loss.item() 226 | 227 | 228 | 229 | #%% 230 | 231 | dqn=DQN() 232 | dqn_tradition=DQN() 233 | epoch_cost_optimize=[] 234 | epoch_cost_tradition=[] 235 | r_list_epoch=[] 236 | 237 | local_costs_epoch=[] 238 | all_costs_epoch=[] 239 | all_costs_epoch_tra=[] 240 | count=0 241 | 242 | #每次开始游戏 前，重开游戏 243 | #s=env.reset()#s是（4,)的numpy 244 | # print("开始第",epoch+1,"轮游戏") 245 | # start = time.time() 246 | for epoch in range(epochs): 247 | 248 | print("本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch,"本轮epoch=",epoch) 249 | step_loss = [] 250 | step_loss_tradition=[] 251 | local_costs=[] 252 | all_costs=[] 253 | all_costs_traditon = [] 254 | r_total=0 255 | 256 | Hn, Hn_e = get_channel_gain_sum() 257 | user_l = np.array([np.random.randint(10,20),np.random.randint(550,560)]) 258 | 259 | #-------------------------- 260 | #------------优化功率和算力------------- 261 | env = Environment(user_num=2, cap_num=2, W=5e6, 262 | f_local=np.array([1.4 * 10e8, 0.43 * 10e8]), 263 | omega=330 / 8, 264 | F_cap=np.array([10 * 10e8, 3 * 10e8]), p_local=2, p_tran=np.array([2, 3]), lamda_=1.0, 265 | p_noise=0.1, Hn=Hn, Hn_e=Hn_e, user_l=user_l, Mb_to_bit=2 ** 20 266 | ) 267 | 268 | s = env.reset() 269 | episode_step = 0 270 | while episode_step MEMORY_CAPACITY: 288 | count+=1 289 | loss=dqn.learn() 290 | 291 | 292 | #判断是否结束 293 | if done !=0: 294 | break 295 | 296 | #更新状态，进入下一次循环 297 | s=s_ 298 | #------------------------------------------- 299 | 300 | print("最后一个step的优化PF的cost=",tc_optimized) 301 | 302 | #------------传统方法------------- 303 | env_tradition = Environment1(user_num=2, cap_num=2, W=5e6, 304 | f_local=np.array([1.4 * 10e8, 0.43 * 10e8]), 305 | omega=330 / 8, 306 | F_cap=np.array([10 * 10e8, 3 * 10e8]), p_local=2, p_tran=np.array([2, 3]), lamda_=1.0, 307 | p_noise=0.1, Hn=Hn, Hn_e=Hn_e, user_l=user_l, Mb_to_bit=2 ** 20 308 | ) 309 | s=env_tradition.reset() 310 | episode_step = 0 311 | while episode_step < MAX_EP: # 每次循环执行一个动作 312 | episode_step += 1 313 | # 通过神经网络，输入状态，获取动作。(不带学习) 314 | action = dqn_tradition.choose_action(s) 315 | 316 | # 执行，获得环境反馈 317 | #return self.state, reward, self.tc_cur, self.done 318 | s_, r, tc_tradition, done = env_tradition.step(action, episode_step) 319 | step_loss_tradition.append(tc_tradition) 320 | # 将环境的反馈s,a,r,s_传入记忆库。用于经验回放 321 | dqn_tradition.store_transition(s, action, r, s_) 322 | 323 | # 神经网络学习（experience replay 经验回放） 324 | if dqn_tradition.memory_count > MEMORY_CAPACITY: 325 | count += 1 326 | loss = dqn_tradition.learn() 327 | 328 | # 判断是否结束 329 | if done != 0: 330 | break 331 | 332 | # 更新状态，进入下一次循环 333 | s = s_ 334 | #-------------------------------------------- 335 | epoch_cost_optimize.append(np.mean(step_loss[-200:])) # 强化学习、优化PF、cost 336 | local_costs_epoch.append(local_latency) # 本地 337 | all_costs_epoch.append(all_cost) # 优化的all cap 338 | epoch_cost_tradition.append(np.mean(step_loss_tradition[-200:])) # 强化学习、均分PF、cost 339 | r_list_epoch.append(r_total) 340 | 341 | plt.plot(range(episode_step), step_loss, color='skyblue', label='optimize_F_P') 342 | plt.plot(range(episode_step), step_loss_tradition, color='red', label='tradition') 343 | plt.plot(range(episode_step),local_costs,color="yellow",label="local") 344 | plt.plot(range(episode_step),all_costs,color="purple",label="all_off") 345 | # plt.plot(range(episode_step), all_costs_traditon, color="green", label='all_off_tra') 346 | plt.legend(loc="best") 347 | 348 | plt.xlabel("step_dqn_e1") 349 | plt.ylabel("costs") 350 | plt.show() 351 | if epoch <20: 352 | if greedy<0.73: 353 | if epoch %5 ==0: 354 | greedy+=0.03 355 | if epoch==30: 356 | greedy=0.85 357 | if epoch >30 and epoch <=40: 358 | if epoch %5 == 0: 359 | greedy+=0.01 360 | if epoch > 40: 361 | if epoch %10 ==0: 362 | if greedy<0.98: 363 | greedy+=0.02 364 | 365 | if epoch==98: 366 | greedy=0.99 367 | print("greedy=",greedy) 368 | 369 | plt.plot(range(epochs),local_costs_epoch, color='yellow', label='locals') 370 | plt.plot(range(epochs),epoch_cost_optimize,color='skyblue', label='optimize_F_P') 371 | plt.plot(range(epochs),epoch_cost_tradition,color='red', label='tradition') 372 | plt.plot(range(epochs),all_costs_epoch,color='purple', label="all_off") 373 | # plt.plot(range(epochs),all_costs_epoch_tra,color='mediumpurple', label="all_off_Tra") 374 | 375 | plt.xlabel("epoch_colluding") 376 | plt.ylabel("costs") 377 | plt.legend(loc="best") 378 | plt.show() 379 | 380 | #保存数据 381 | 382 | Writedata.write_to_excel(epoch_cost_optimize,1,"./dataset/tra_opi/optimize_cost2.xls")#优化 383 | Writedata.write_to_excel(all_costs_epoch,1,"./dataset/tra_opi/all_costs2.xls")#优化的全卸载 384 | Writedata.write_to_excel(local_costs_epoch,1,"./dataset/tra_opi/local_costs2.xls")#本地 385 | Writedata.write_to_excel(epoch_cost_tradition,1,"./dataset/tra_opi/tradition_costs2.xls")#传统方法 386 | Writedata.write_to_excel(r_list_epoch,1,"./dataset/tra_opi/r_list.xls")#传统方法 387 | -------------------------------------------------------------------------------- /改善环境版/Environment_Wu.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import numpy 4 | import numpy as np 5 | 6 | 7 | class Environment(object): 8 | 9 | def __init__(self, user_num, cap_num, W, f_local, omega, 10 | F_cap, p_local, p_tran, p_noise, Hn, Hn_e, user_l, Mb_to_bit, lamda_): 11 | 12 | self.user_num = user_num 13 | self.cap_num = cap_num 14 | # self.name = name # 进程名字 15 | 16 | self.f_local = f_local 17 | self.omega = omega 18 | 19 | # 获取信道参数 20 | self.Hn = Hn 21 | self.Hn_e = Hn_e 22 | self.Hn_A = self.Hn[0] 23 | self.Hn_B = self.Hn[1] 24 | 25 | # W为总带宽，[capA,capB]，需要取出来 26 | self.W = W 27 | 28 | 29 | # F_cap为A和B总算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 30 | self.F_cap = F_cap.reshape(1, -1) # (2,1)(F_A,F_B) （10*10e8,3*10e8） 31 | self.F = (np.r_[self.F_cap, self.F_cap] / 2) 32 | self.F_constant = self.F # [[5,1,5],[5,1.5]] 33 | 34 | 35 | # 传输功率 下个版本也许要利用解析解优化 如p_{i,j} 36 | self.p_tran = (p_tran / self.user_num).reshape(-1, 1) 37 | self.P = np.c_[self.p_tran, self.p_tran] 38 | self.p_tran_constant = self.p_tran # [[1, 1], [1.5, 1.5]] 39 | self.p_tran_total = p_tran.reshape(1, -1) # 该参数仅用于传入神经网络寻 40 | 41 | self.F_P_total = np.append(self.F_cap, self.p_tran_total, axis=1) 42 | 43 | self.p_noise = p_noise 44 | self.p_local = p_local 45 | 46 | self.omega = omega 47 | self.Mb_to_bit = Mb_to_bit 48 | self.lamda_ = lamda_ 49 | 50 | self.user_l = user_l # (2,) 51 | 52 | # 卸载率权重 和卸载率的初始化(归一化) 53 | self.off_rate_weight = self.init_offloading_weight() # 卸载率权重(未归一化) 54 | self.offload_rate = np.zeros((self.user_num, self.cap_num + 1)) 55 | self.off_rate_weight_normalization()#卸载率归一化 56 | 57 | # 环境状态 58 | self.state = self.update_state() # (6,1) 59 | 60 | self.tc_pre = self.get_total_cost() 61 | self.tc_cur = 0.0 62 | 63 | self.total_local_latency = self.tc_pre # 本地 64 | self.total_cap_offload_latency = self.get_all_cap_latency_cost() # 全卸载 65 | self.done = False 66 | 67 | # if name == 'w00': 68 | # print("all cap_cost{} all_local_cost{}".format(self.total_cap_offload_latency, self.total_local_latency)) 69 | 70 | def init_offloading_weight(self): 71 | offload_weight = np.zeros((self.user_num, self.cap_num + 1)) 72 | # e.g 73 | # offloading ={[1,0,0],[1,0,0]} 74 | for i in range(self.user_num): 75 | for j in range(self.cap_num+1): 76 | if j ==0: 77 | offload_weight[i][j] = 1.0 78 | else: 79 | offload_weight[i][j] = 0.1 # 全卸载在优化算力会出现问题 分母为0 80 | return offload_weight 81 | 82 | def reset(self): 83 | self.off_rate_weight = self.init_offloading_weight() 84 | self.off_rate_weight_normalization() 85 | 86 | 87 | # s = np.append(self.F_P_total, self.state.reshape(1, -1), axis=1) 加入总功率和总算力self.F_cap，self.p_tran_total 88 | 89 | return self.state 90 | 91 | def action_state_num(self): 92 | action_n = self.user_num * (self.cap_num + 1) # dqn 93 | state_num = self.user_num * (self.cap_num + 1) 94 | return action_n, state_num 95 | 96 | def step(self, a, cur_step, ): 97 | # a [0,1,2,3,4,5]中一个 98 | # [0,1,2] and [3,4,5]分别对应user_1 user2的动作 99 | # 对应a的卸载率 100 | # 总动作数量 = (cap_num+1)*user_num 101 | 102 | # 1.0 更新卸载率权重 103 | # w = (cur_step / 100)+ 1.0 104 | 105 | w = cur_step // 50 + 1.0 106 | 107 | step = w * 1.0 108 | 109 | user_index = a // (self.cap_num + 1)#选出是哪个用户 110 | action_index = a % (self.user_num + 1)#选出是本地还是capA，capB 111 | 112 | self.off_rate_weight[user_index][action_index] += step 113 | 114 | # 1.1 更新卸载率 115 | self.off_rate_weight_normalization() 116 | 117 | # 2. 更新环境状态 118 | self.state = self.update_state() 119 | 120 | # 3.1 优化P和F 121 | self.optimaztion_for_power_and_F_Cap() 122 | self.tc_cur = self.get_total_cost() 123 | 124 | if self.tc_cur > self.tc_pre: 125 | reward = -100 126 | elif self.tc_cur < self.tc_pre: 127 | reward = 100 128 | else: 129 | reward = -50 130 | 131 | # 4.缓存总代价 132 | self.tc_pre = self.tc_cur 133 | 134 | # s = np.append(self.F_P_total, self.state.reshape(1,-1), axis=1) 加入总功率和总算力self.F_cap，self.p_tran_total 135 | 136 | 137 | return self.state, reward, self.total_local_latency, self.tc_cur, self.total_cap_offload_latency, self.done#self.tc_cur为传统方法 138 | 139 | def init_P_and_F(self): 140 | self.F = np.array([[5e9, 1.5e9], [5e9, 1.5e9]]) 141 | self.P = np.array([[1., 1.], [1.5, 1.5]]) 142 | 143 | def update_state(self):#更新环境 144 | temp = np.zeros_like(self.offload_rate) 145 | 146 | for i in range(len(self.user_l)): 147 | temp[i] = self.offload_rate[i] * self.user_l[i] 148 | 149 | state = np.r_[temp[0], temp[1]].reshape(-1, 1).squeeze() 150 | return state 151 | 152 | def off_rate_weight_normalization(self): 153 | for i in range(self.user_num): 154 | total = np.sum(self.off_rate_weight[i]) 155 | for j in range(self.cap_num + 1): 156 | self.offload_rate[i][j] = self.off_rate_weight[i][j] / total 157 | 158 | def get_all_cap_latency_cost(self): 159 | # 全卸载 160 | temp = self.offload_rate 161 | self.offload_rate = np.array([[0., 0.5, 0.5], [0., 0.5, 0.5]]) 162 | all_cap_cost = self.get_total_cost() 163 | self.offload_rate = temp 164 | return all_cap_cost 165 | 166 | def get_all_local_latency(self): 167 | # 全本地计算 168 | temp = self.offload_rate 169 | self.offload_rate = np.array([[1, .0, .0], [1., .0, .0]]) 170 | all_cap_cost = self.get_total_cost() 171 | self.offload_rate = temp 172 | return all_cap_cost 173 | 174 | def get_total_cost(self): 175 | a = self.F 176 | b = self.P 177 | T_total = self.get_total_latency() 178 | E_total = self.get_total_energy() 179 | Cost_total = self.lamda_ * T_total + (1.0 - self.lamda_) * E_total 180 | return Cost_total 181 | 182 | def get_total_energy(self): 183 | 184 | return 0.0 185 | 186 | def get_total_local_latency(self): 187 | # 采用∑方式 188 | 189 | sum = 0 190 | for i in range(self.user_num): 191 | t_local_i = self.user_l[i] * self.offload_rate[i][0] * self.omega * self.Mb_to_bit / self.f_local[i] 192 | sum += t_local_i 193 | return sum 194 | 195 | 196 | 197 | def get_transmit_latency(self): 198 | 199 | hn_e = self.Hn_e.reshape(-1, 1) # (2,1) 200 | hn_e = np.c_[hn_e, hn_e] # (2,2) 201 | 202 | user_l = self.user_l.reshape(-1, 1) # ( 2,1) 203 | user_l = np.c_[user_l, user_l] # (2,2) 204 | 205 | # R(i,j) -->(2,2) 206 | # Shannon formula 207 | R = self.W * np.log2(1 + self.P * self.Hn / (self.p_noise ** 2)) \ 208 | - self.W * np.log2(1 + self.P * hn_e / (self.p_noise ** 2)) 209 | 210 | # T_trans = l_{i,j} *a_{i,j} * Mb_to_bit / R_{i,j} 211 | T_trans = np.multiply(user_l, self.offload_rate[:, 1:]) * self.Mb_to_bit / R 212 | return T_trans 213 | 214 | def get_cap_latency(self): 215 | user_l = self.user_l.reshape(-1, 1) # ( 2,1) 216 | user_l = np.c_[user_l, user_l] # (2,2) 217 | 218 | # (2,2) 219 | T_cap = np.multiply(user_l, self.offload_rate[:, 1:]) * self.Mb_to_bit * self.omega / self.F 220 | return T_cap 221 | 222 | def get_total_latency(self): 223 | T_local = self.get_total_local_latency() # 标量 224 | T_trans = self.get_transmit_latency() # (2,2) 225 | T_cap = self.get_cap_latency() # (2,2) 226 | T_total = T_local + T_trans.sum() + T_cap.sum() 227 | 228 | return T_total 229 | 230 | def optimaztion_for_power_and_F_Cap(self): 231 | hn_e = self.Hn_e.reshape(-1, 1) # (2,1) 232 | hn_e = np.c_[hn_e, hn_e] # (2,2) 233 | 234 | user_l_padding = self.user_l.reshape(-1, 1) 235 | user_l = np.c_[user_l_padding, user_l_padding] 236 | offload_rate = self.offload_rate[:, 1:] 237 | 238 | a = self.W * np.log2((self.Hn / hn_e)) 239 | 240 | b = (self.W * self.p_noise ** 2 * (1 / (hn_e) - 1 / (self.Hn))) / math.log(2) 241 | 242 | # 求lamda_i 243 | fenzi = np.sqrt(offload_rate * user_l * b) / a 244 | # 2个用户直接写死 ，按列加和 245 | row_temp = fenzi[:, 0] + fenzi[:, 1] 246 | fenzi = np.c_[(row_temp, row_temp)] 247 | 248 | row_temp = (b / a)[:, 0] + (b / a)[:, 1] 249 | fenmu = self.p_tran_constant - np.c_[row_temp, row_temp] 250 | lada = np.square(fenzi / fenmu) 251 | 252 | # 求解mu 253 | # 按照行加和 254 | fenzi = np.sqrt(offload_rate * user_l) 255 | row_temp = fenzi[0, :] + fenzi[1, :] 256 | fenzi = np.r_[row_temp.reshape(1, -1), row_temp.reshape(1, -1)] 257 | fenmu = self.F_constant * 2 258 | 259 | mu = np.square(fenzi / fenmu) 260 | 261 | if (lada == 0).any(): 262 | return 263 | 264 | if (mu == 0).any(): 265 | return 266 | 267 | self.P = (np.sqrt(offload_rate * user_l * b / lada) + b) / a 268 | self.F = np.sqrt(offload_rate * user_l / mu) 269 | 270 | if (offload_rate == 0).any(): 271 | a = 0 272 | 273 | 274 | -------------------------------------------------------------------------------- /改善环境版/Environment_Wu_tradition.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import numpy 4 | import numpy as np 5 | 6 | 7 | class Environment1(object): 8 | 9 | def __init__(self, user_num, cap_num, W, f_local, omega, 10 | F_cap, p_local, p_tran, p_noise, Hn, Hn_e, user_l, Mb_to_bit, lamda_): 11 | 12 | self.user_num = user_num 13 | self.cap_num = cap_num 14 | # self.name = name # 进程名字 15 | 16 | self.f_local = f_local 17 | self.omega = omega 18 | 19 | # 获取信道参数 20 | self.Hn = Hn 21 | self.Hn_e = Hn_e 22 | self.Hn_A = self.Hn[0] 23 | self.Hn_B = self.Hn[1] 24 | 25 | # W为总带宽，[capA,capB]，需要取出来 26 | self.W = W 27 | 28 | 29 | # F_cap为A和B总算力（优化对象），[CAPA,CAPB]，需分别均分给n个设备 30 | self.F_cap = F_cap.reshape(1, -1) # (2,1)(F_A,F_B) （10*10e8,3*10e8） 31 | self.F = (np.r_[self.F_cap, self.F_cap] / 2) 32 | self.F_constant = self.F # [[5,1,5],[5,1.5]] 33 | 34 | 35 | # 传输功率 下个版本也许要利用解析解优化 如p_{i,j} 36 | self.p_tran = (p_tran / self.user_num).reshape(-1, 1) 37 | self.P = np.c_[self.p_tran, self.p_tran] 38 | self.p_tran_constant = self.p_tran # [[1, 1], [1.5, 1.5]] 39 | self.p_tran_total = p_tran.reshape(1, -1) # 该参数仅用于传入神经网络寻 40 | 41 | self.F_P_total = np.append(self.F_cap, self.p_tran_total, axis=1) 42 | 43 | self.p_noise = p_noise 44 | self.p_local = p_local 45 | 46 | self.omega = omega 47 | self.Mb_to_bit = Mb_to_bit 48 | self.lamda_ = lamda_ 49 | 50 | self.user_l = user_l # (2,) 51 | 52 | # 卸载率权重 和卸载率的初始化(归一化) 53 | self.off_rate_weight = self.init_offloading_weight() # 卸载率权重(未归一化) 54 | self.offload_rate = np.zeros((self.user_num, self.cap_num + 1)) 55 | self.off_rate_weight_normalization()#卸载率归一化 56 | 57 | # 环境状态 58 | self.state = self.update_state() # (6,1) 59 | 60 | self.tc_pre = self.get_total_cost() 61 | self.tc_cur = 0.0 62 | 63 | self.total_local_latency = self.tc_pre # 本地 64 | self.total_cap_offload_latency = self.get_all_cap_latency_cost() # 全卸载 65 | self.done = False 66 | 67 | # if name == 'w00': 68 | # print("all cap_cost{} all_local_cost{}".format(self.total_cap_offload_latency, self.total_local_latency)) 69 | 70 | def init_offloading_weight(self): 71 | offload_weight = np.zeros((self.user_num, self.cap_num + 1)) 72 | # e.g 73 | # offloading ={[1,0,0],[1,0,0]} 74 | for i in range(self.user_num): 75 | for j in range(self.cap_num+1): 76 | if j ==0: 77 | offload_weight[i][j] = 1.0 78 | else: 79 | offload_weight[i][j] = 0.1 # 全卸载在优化算力会出现问题 分母为0 80 | return offload_weight 81 | 82 | def reset(self): 83 | self.off_rate_weight = self.init_offloading_weight() 84 | self.off_rate_weight_normalization() 85 | 86 | 87 | # s = np.append(self.F_P_total, self.state.reshape(1, -1), axis=1) 加入总功率和总算力self.F_cap，self.p_tran_total 88 | 89 | return self.state 90 | 91 | def action_state_num(self): 92 | action_n = self.user_num * (self.cap_num + 1) # dqn 93 | state_num = self.user_num * (self.cap_num + 1) 94 | return action_n, state_num 95 | 96 | def step(self, a, cur_step, ): 97 | # a [0,1,2,3,4,5]中一个 98 | # [0,1,2] and [3,4,5]分别对应user_1 user2的动作 99 | # 对应a的卸载率 100 | # 总动作数量 = (cap_num+1)*user_num 101 | 102 | # 1.0 更新卸载率权重 103 | # w = (cur_step / 100)+ 1.0 104 | 105 | w = cur_step // 50 + 1.0 106 | 107 | step = w * 1.0 108 | 109 | user_index = a // (self.cap_num + 1)#选出是哪个用户 110 | action_index = a % (self.user_num + 1)#选出是本地还是capA，capB 111 | 112 | self.off_rate_weight[user_index][action_index] += step 113 | 114 | # 1.1 更新卸载率 115 | self.off_rate_weight_normalization() 116 | 117 | # 2. 更新环境状态 118 | self.state = self.update_state() 119 | 120 | # 3.1 优化P和F 121 | self.tc_cur = self.get_total_cost() 122 | 123 | if self.tc_cur > self.tc_pre: 124 | reward = -100 125 | elif self.tc_cur < self.tc_pre: 126 | reward = 100 127 | else: 128 | reward = -50 129 | 130 | # 4.缓存总代价 131 | self.tc_pre = self.tc_cur 132 | 133 | # s = np.append(self.F_P_total, self.state.reshape(1,-1), axis=1) 加入总功率和总算力self.F_cap，self.p_tran_total 134 | 135 | 136 | return self.state, reward,self.tc_cur,self.done#self.tc_cur为传统方法 137 | 138 | def init_P_and_F(self): 139 | self.F = np.array([[5e9, 1.5e9], [5e9, 1.5e9]]) 140 | self.P = np.array([[1., 1.], [1.5, 1.5]]) 141 | 142 | def update_state(self):#更新环境 143 | temp = np.zeros_like(self.offload_rate) 144 | 145 | for i in range(len(self.user_l)): 146 | temp[i] = self.offload_rate[i] * self.user_l[i] 147 | 148 | state = np.r_[temp[0], temp[1]].reshape(-1, 1).squeeze() 149 | return state 150 | 151 | def off_rate_weight_normalization(self): 152 | for i in range(self.user_num): 153 | total = np.sum(self.off_rate_weight[i]) 154 | for j in range(self.cap_num + 1): 155 | self.offload_rate[i][j] = self.off_rate_weight[i][j] / total 156 | 157 | def get_all_cap_latency_cost(self): 158 | # 全卸载 159 | temp = self.offload_rate 160 | self.offload_rate = np.array([[0., 0.5, 0.5], [0., 0.5, 0.5]]) 161 | all_cap_cost = self.get_total_cost() 162 | self.offload_rate = temp 163 | return all_cap_cost 164 | 165 | def get_all_local_latency(self): 166 | # 全本地计算 167 | temp = self.offload_rate 168 | self.offload_rate = np.array([[1, .0, .0], [1., .0, .0]]) 169 | all_cap_cost = self.get_total_cost() 170 | self.offload_rate = temp 171 | return all_cap_cost 172 | 173 | def get_total_cost(self): 174 | a = self.F 175 | b = self.P 176 | T_total = self.get_total_latency() 177 | E_total = self.get_total_energy() 178 | Cost_total = self.lamda_ * T_total + (1.0 - self.lamda_) * E_total 179 | return Cost_total 180 | 181 | def get_total_energy(self): 182 | 183 | return 0.0 184 | 185 | def get_total_local_latency(self): 186 | # 采用∑方式 187 | 188 | sum = 0 189 | for i in range(self.user_num): 190 | t_local_i = self.user_l[i] * self.offload_rate[i][0] * self.omega * self.Mb_to_bit / self.f_local[i] 191 | sum += t_local_i 192 | return sum 193 | 194 | 195 | 196 | def get_transmit_latency(self): 197 | 198 | hn_e = self.Hn_e.reshape(-1, 1) # (2,1) 199 | hn_e = np.c_[hn_e, hn_e] # (2,2) 200 | 201 | user_l = self.user_l.reshape(-1, 1) # ( 2,1) 202 | user_l = np.c_[user_l, user_l] # (2,2) 203 | 204 | # R(i,j) -->(2,2) 205 | # Shannon formula 206 | R = self.W * np.log2(1 + self.P * self.Hn / (self.p_noise ** 2)) \ 207 | - self.W * np.log2(1 + self.P * hn_e / (self.p_noise ** 2)) 208 | 209 | # T_trans = l_{i,j} *a_{i,j} * Mb_to_bit / R_{i,j} 210 | T_trans = np.multiply(user_l, self.offload_rate[:, 1:]) * self.Mb_to_bit / R 211 | return T_trans 212 | 213 | def get_cap_latency(self): 214 | user_l = self.user_l.reshape(-1, 1) # ( 2,1) 215 | user_l = np.c_[user_l, user_l] # (2,2) 216 | 217 | # (2,2) 218 | T_cap = np.multiply(user_l, self.offload_rate[:, 1:]) * self.Mb_to_bit * self.omega / self.F 219 | return T_cap 220 | 221 | def get_total_latency(self): 222 | T_local = self.get_total_local_latency() # 标量 223 | T_trans = self.get_transmit_latency() # (2,2) 224 | T_cap = self.get_cap_latency() # (2,2) 225 | T_total = T_local + T_trans.sum() + T_cap.sum() 226 | 227 | return T_total 228 | 229 | def optimaztion_for_power_and_F_Cap(self): 230 | hn_e = self.Hn_e.reshape(-1, 1) # (2,1) 231 | hn_e = np.c_[hn_e, hn_e] # (2,2) 232 | 233 | user_l_padding = self.user_l.reshape(-1, 1) 234 | user_l = np.c_[user_l_padding, user_l_padding] 235 | offload_rate = self.offload_rate[:, 1:] 236 | 237 | a = self.W * np.log2((self.Hn / hn_e)) 238 | 239 | b = (self.W * self.p_noise ** 2 * (1 / (hn_e) - 1 / (self.Hn))) / math.log(2) 240 | 241 | # 求lamda_i 242 | fenzi = np.sqrt(offload_rate * user_l * b) / a 243 | # 2个用户直接写死 ，按列加和 244 | row_temp = fenzi[:, 0] + fenzi[:, 1] 245 | fenzi = np.c_[(row_temp, row_temp)] 246 | 247 | row_temp = (b / a)[:, 0] + (b / a)[:, 1] 248 | fenmu = self.p_tran_constant - np.c_[row_temp, row_temp] 249 | lada = np.square(fenzi / fenmu) 250 | 251 | # 求解mu 252 | # 按照行加和 253 | fenzi = np.sqrt(offload_rate * user_l) 254 | row_temp = fenzi[0, :] + fenzi[1, :] 255 | fenzi = np.r_[row_temp.reshape(1, -1), row_temp.reshape(1, -1)] 256 | fenmu = self.F_constant * 2 257 | 258 | mu = np.square(fenzi / fenmu) 259 | 260 | if (lada == 0).any(): 261 | return 262 | 263 | if (mu == 0).any(): 264 | return 265 | 266 | self.P = (np.sqrt(offload_rate * user_l * b / lada) + b) / a 267 | self.F = np.sqrt(offload_rate * user_l / mu) 268 | 269 | if (offload_rate == 0).any(): 270 | a = 0 271 | 272 | 273 | -------------------------------------------------------------------------------- /改善环境版/Writedata.py: -------------------------------------------------------------------------------- 1 | import xlrd 2 | from xlwt import * 3 | 4 | 5 | # 多次实验时写入数据代码 6 | def write_data(data_list, sheet_name, iteration): 7 | for each_i in range(len(data_list)): 8 | sheet_name.write(each_i, iteration, str(data_list[each_i])) 9 | 10 | 11 | def create_csv(sheet1, data_list, iteration=1): 12 | write_data(data_list=data_list, sheet_name=sheet1, iteration=iteration) 13 | 14 | 15 | def write_to_excel(cost_list, is_one, filename): 16 | wb = Workbook() 17 | sheet1 = wb.add_sheet(sheetname='cost') 18 | if is_one: 19 | create_csv(sheet1=sheet1, data_list=cost_list) 20 | else: 21 | for index in range(len(cost_list)): 22 | create_csv(sheet1=sheet1, data_list=cost_list[index], iteration=index) 23 | wb.save(filename) 24 | 25 | 26 | def read_file(h_file, task_file): 27 | channel_uc = xlrd.open_workbook(h_file) 28 | task_file = xlrd.open_workbook(task_file) 29 | uc_sheet = channel_uc.sheets()[0] 30 | task_sheet = task_file.sheets()[0] 31 | 32 | return uc_sheet, task_sheet 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /改善环境版/__pycache__/Environment_Wu.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/__pycache__/Environment_Wu.cpython-38.pyc -------------------------------------------------------------------------------- /改善环境版/__pycache__/Environment_Wu_tradition.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/__pycache__/Environment_Wu_tradition.cpython-38.pyc -------------------------------------------------------------------------------- /改善环境版/__pycache__/Writedata.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/__pycache__/Writedata.cpython-38.pyc -------------------------------------------------------------------------------- /改善环境版/dataset/E_max/all_costs.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_max/all_costs.xls -------------------------------------------------------------------------------- /改善环境版/dataset/E_max/local_costs.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_max/local_costs.xls -------------------------------------------------------------------------------- /改善环境版/dataset/E_max/optimize_cost.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_max/optimize_cost.xls -------------------------------------------------------------------------------- /改善环境版/dataset/E_sum/all_costs.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_sum/all_costs.csv -------------------------------------------------------------------------------- /改善环境版/dataset/E_sum/local_costs.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_sum/local_costs.csv -------------------------------------------------------------------------------- /改善环境版/dataset/E_sum/optimize_cost.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/E_sum/optimize_cost.csv -------------------------------------------------------------------------------- /改善环境版/dataset/optimize_cost_colluding.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/optimize_cost_colluding.xls -------------------------------------------------------------------------------- /改善环境版/dataset/optimize_cost_non-colluding.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/optimize_cost_non-colluding.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/all_costs.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/all_costs.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/all_costs1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/all_costs1.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/all_costs2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/all_costs2.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/local_costs.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/local_costs.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/local_costs1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/local_costs1.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/local_costs2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/local_costs2.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/optimize_cost.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/optimize_cost.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/optimize_cost1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/optimize_cost1.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/r_list.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/r_list.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/r_list1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/r_list1.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/tradition_costs.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/tradition_costs.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/tradition_costs1.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/tradition_costs1.xls -------------------------------------------------------------------------------- /改善环境版/dataset/tra_opi/tradition_costs2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiong1998/offloading-optimization-and-resource-optimization-base-on-mobile-edge-computing-net/0c90c5809c0a665f7c9ee2afc74436f2562a789f/改善环境版/dataset/tra_opi/tradition_costs2.xls --------------------------------------------------------------------------------