├── README.md
├── agents
    ├── models.py
    ├── policies.py
    └── utils.py
├── configs
    ├── parameters_der20.py
    ├── parameters_der4.py
    ├── parameters_der40.py
    └── parameters_der6.py
├── docs
    └── microgrid.png
├── envs
    ├── Grid_envs.py
    ├── der20_RL_fn.py
    ├── der40_RL_fn.py
    └── der6_RL_fn.py
├── main.py
├── requirements.txt
└── trainer.py


/README.md:
--------------------------------------------------------------------------------
 1 | # PowerNet: Multi-agent Deep ReinforcementLearning for Scalable Powergrid Control
 2 | 
 3 | An on-policy, cooperative MARL algorithm for voltage control problem in the isolated Microgrid system by incorporating  a  differentiable,  learning-based communication  protocol,  a  spatial  discount factor, and an action smoothing scheme. 
 4 | 
 5 | ## Installation
 6 | - create an python virtual environment: `conda create -n powernet python=3.6`
 7 | - active the virtul environment: `conda activate powernet`
 8 | - install pytorch (torch>=1.2.0): `pip install torch===1.7.0 torchvision===0.8.1 torchaudio===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html`
 9 | - install the requirements: `pip install -r requirements.txt`
10 | 
11 | ## PGSIM
12 | A python implementation of IEEE-34 bus and a 20-DGs Microgrid system with Numba [5] which is an open source JIT compiler that translates a subset of Python and NumPy code into fast machine code.  The simulation platform we developed is based on the line and load specifications detailed in [3].
13 | 
14 | <p align="center">
15 |      <img src="docs/microgrid.png" alt="output_example" width="80%" height="80%">
16 |      <br>Fig.1 System diagrams of the two microgrid simulation platforms: (a) microgrid-6; and (b) microgrid-20.
17 | </p>
18 | 
19 | ## Usage
20 | To run the code, just run it via `python main.py`.  The config files contain the parameters for MARL policies.
21 | 
22 | ## Cite
23 | ```
24 | @misc{chen2020powernet,
25 |       title={PowerNet: Multi-agent Deep Reinforcement Learning for Scalable Powergrid Control}, 
26 |       author={Dong Chen and Zhaojian Li and Tianshu Chu and Rui Yao and Feng Qiu and Kaixiang Lin},
27 |       year={2020},
28 |       eprint={2011.12354},
29 |       archivePrefix={arXiv},
30 |       primaryClass={eess.SY}
31 | }
32 | ```
33 | 
34 | ## Reference
35 | [1] Bidram, Ali, et al. "Distributed cooperative secondary control of microgrids using feedback linearization." IEEE Transactions on Power Systems 28.3 (2013): 3462-3470.
36 | 
37 | [2] Bidram, Ali, et al. "Secondary control of microgrids based on distributed cooperative control of multi-agent systems." IET Generation, Transmission & Distribution 7.8 (2013): 822-831.
38 | 
39 | [3] Mustafa, Aquib, et al. "Detection and Mitigation of Data Manipulation Attacks in AC Microgrids." IEEE Transactions on Smart Grid 11.3 (2019): 2588-2603.
40 | 
41 | [4] Chu, Tianshu, et al. "Multi-agent deep reinforcement learning for large-scale traffic signal control." IEEE Transactions on Intelligent Transportation Systems 21.3 (2019): 1086-1095.
42 | 
43 | [5] [Numba: an open source JIT compiler that translates a subset of Python and NumPy code into fast machine code.](https://numba.pydata.org/)
44 | 


--------------------------------------------------------------------------------
/agents/models.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Model functions
  3 | """
  4 | import os
  5 | import torch
  6 | import torch.nn as nn
  7 | import torch.optim as optim
  8 | from agents.utils import OnPolicyBuffer, MultiAgentOnPolicyBuffer, Scheduler
  9 | from agents.policies import (LstmPolicy, FPPolicy, ConsensusPolicy, NCMultiAgentPolicy,
 10 |                              CommNetMultiAgentPolicy, DIALMultiAgentPolicy)
 11 | import logging
 12 | import numpy as np
 13 | 
 14 | 
 15 | class IA2C:
 16 |     """
 17 |     The basic IA2C implementation with decentralized actor and centralized critic,
 18 |     limited to neighborhood area only.
 19 |     """
 20 | 
 21 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
 22 |                  total_step, model_config, seed=0, use_gpu=False):
 23 |         """
 24 |         n_s_ls: list of agents' state dimension. e.g., n_s_ls = [17, 17, 17, 17]
 25 |         n_a_ls: list of agents' action dimension. e.g., n_a_ls = [5, 5, 5, 5]
 26 |         neighbor_mask: matrix for neighbors: e.g., [[0,1,0,0], [1,0,1,0], [0,1,0,1], [0,0,1,0]]
 27 |         distance_mask: matrix for distance: e.g., [[0,1,2,3], [1,0,1,2], [2,1,0,1], [3,2,1,0]]
 28 |         coop_gamma: coop discount is used to discount the neighbors' impact
 29 |         """
 30 |         self.name = 'ia2c'
 31 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
 32 |                         total_step, seed, use_gpu, model_config)
 33 | 
 34 |     def add_transition(self, ob, naction, action, reward, value, done):
 35 |         if self.reward_norm > 0:
 36 |             reward = reward / self.reward_norm
 37 |         if self.reward_clip > 0:
 38 |             reward = np.clip(reward, -self.reward_clip, self.reward_clip)
 39 |         for i in range(self.n_agent):
 40 |             self.trans_buffer[i].add_transition(ob[i], naction[i], action[i], reward, value[i], done)
 41 | 
 42 |     def backward(self, Rends, dt, summary_writer=None, global_step=None):
 43 |         self.optimizer.zero_grad()
 44 |         for i in range(self.n_agent):
 45 |             obs, nas, acts, dones, Rs, Advs = self.trans_buffer[i].sample_transition(Rends[i], dt)
 46 |             if i == 0:
 47 |                 self.policy[i].backward(obs, nas, acts, dones, Rs, Advs,
 48 |                                         self.e_coef, self.v_coef,
 49 |                                         summary_writer=summary_writer, global_step=global_step)
 50 |             else:
 51 |                 self.policy[i].backward(obs, nas, acts, dones, Rs, Advs,
 52 |                                         self.e_coef, self.v_coef)
 53 |         if self.max_grad_norm > 0:
 54 |             nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
 55 |         self.optimizer.step()
 56 |         if self.lr_decay != 'constant':
 57 |             self._update_lr()
 58 | 
 59 |     def forward(self, obs, done, nactions=None, out_type='p'):
 60 |         out = []
 61 |         if nactions is None:
 62 |             nactions = [None] * self.n_agent
 63 |         for i in range(self.n_agent):
 64 |             cur_out = self.policy[i](obs[i], done, nactions[i], out_type)
 65 |             out.append(cur_out)
 66 |         return out
 67 | 
 68 |     def load(self, model_dir, global_step=None, train_mode=False):
 69 |         save_file = None
 70 |         save_step = 0
 71 |         if os.path.exists(model_dir):
 72 |             if global_step is None:
 73 |                 for file in os.listdir(model_dir):
 74 |                     if file.startswith('checkpoint'):
 75 |                         tokens = file.split('.')[0].split('-')
 76 |                         if len(tokens) != 2:
 77 |                             continue
 78 |                         cur_step = int(tokens[1])
 79 |                         if cur_step > save_step:
 80 |                             save_file = file
 81 |                             save_step = cur_step
 82 |             else:
 83 |                 save_file = 'checkpoint-{:d}.pt'.format(global_step)
 84 |         if save_file is not None:
 85 |             file_path = model_dir + save_file
 86 |             checkpoint = torch.load(file_path)
 87 |             logging.info('Checkpoint loaded: {}'.format(file_path))
 88 |             self.policy.load_state_dict(checkpoint['model_state_dict'])
 89 |             if train_mode:
 90 |                 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 91 |                 self.policy.train()
 92 |             else:
 93 |                 self.policy.eval()
 94 |             return True
 95 |         logging.error('Can not find checkpoint for {}'.format(model_dir))
 96 |         return False
 97 | 
 98 |     def reset(self):
 99 |         for i in range(self.n_agent):
100 |             self.policy[i]._reset()
101 | 
102 |     def save(self, model_dir, global_step):
103 |         file_path = model_dir + 'checkpoint-{:d}.pt'.format(global_step)
104 |         torch.save({'global_step': global_step,
105 |                     'model_state_dict': self.policy.state_dict(),
106 |                     'optimizer_state_dict': self.optimizer.state_dict()},
107 |                    file_path)
108 |         # logging.info('Checkpoint saved: {}'.format(file_path))
109 | 
110 |     def _init_algo(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
111 |                    total_step, seed, use_gpu, model_config):
112 |         # init params
113 |         self.n_s_ls = n_s_ls
114 |         self.n_a_ls = n_a_ls
115 |         self.identical_agent = False
116 |         if max(self.n_a_ls) == min(self.n_a_ls):
117 |             # note for identical IA2C, n_s_ls may have varient dims
118 |             self.identical_agent = True
119 |             self.n_s = n_s_ls[0]
120 |             self.n_a = n_a_ls[0]
121 |         else:
122 |             self.n_s = max(self.n_s_ls)
123 |             self.n_a = max(self.n_a_ls)
124 |         self.neighbor_mask = neighbor_mask
125 |         self.n_agent = len(self.neighbor_mask)
126 |         self.reward_clip = model_config.getfloat('reward_clip')
127 |         self.reward_norm = model_config.getfloat('reward_norm')
128 |         self.n_step = model_config.getint('batch_size')
129 |         self.n_fc = model_config.getint('num_fc')
130 |         self.n_lstm = model_config.getint('num_lstm')
131 |         # init torch
132 |         if use_gpu and torch.cuda.is_available():
133 |             torch.cuda.manual_seed_all(seed)
134 |             torch.backends.cudnn.benchmark = False
135 |             torch.backends.cudnn.deterministic = True
136 |             self.device = torch.device("cuda:0")
137 |             logging.info('Use gpu for pytorch...')
138 |         else:
139 |             torch.manual_seed(seed)
140 |             torch.set_num_threads(1)
141 |             self.device = torch.device("cpu")
142 |             logging.info('Use cpu for pytorch...')
143 | 
144 |         self.policy = self._init_policy()
145 |         self.policy.to(self.device)
146 | 
147 |         # init exp buffer and lr scheduler for training
148 |         if total_step:
149 |             self.total_step = total_step
150 |             self._init_train(model_config, distance_mask, coop_gamma)
151 | 
152 |     def _init_policy(self):
153 |         policy = []
154 |         for i in range(self.n_agent):
155 |             n_n = np.sum(self.neighbor_mask[i])
156 |             if self.identical_agent:
157 |                 local_policy = LstmPolicy(self.n_s_ls[i], self.n_a_ls[i], n_n, self.n_step,
158 |                                           n_fc=self.n_fc, n_lstm=self.n_lstm, name='{:d}'.format(i))
159 |             else:
160 |                 na_dim_ls = []
161 |                 for j in np.where(self.neighbor_mask[i] == 1)[0]:
162 |                     na_dim_ls.append(self.n_a_ls[j])
163 |                 local_policy = LstmPolicy(self.n_s_ls[i], self.n_a_ls[i], n_n, self.n_step,
164 |                                           n_fc=self.n_fc, n_lstm=self.n_lstm, name='{:d}'.format(i),
165 |                                           na_dim_ls=na_dim_ls, identical=False)
166 |                 # local_policy.to(self.device)
167 |             policy.append(local_policy)
168 |         return nn.ModuleList(policy)
169 | 
170 |     def _init_scheduler(self, model_config):
171 |         # init lr scheduler
172 |         self.lr_init = model_config.getfloat('lr_init')
173 |         self.lr_decay = model_config.get('lr_decay')
174 |         if self.lr_decay == 'constant':
175 |             self.lr_scheduler = Scheduler(self.lr_init, decay=self.lr_decay)
176 |         else:
177 |             lr_min = model_config.getfloat('lr_min')
178 |             self.lr_scheduler = Scheduler(self.lr_init, lr_min, self.total_step, decay=self.lr_decay)
179 | 
180 |     def _init_train(self, model_config, distance_mask, coop_gamma):
181 |         # init lr scheduler
182 |         self._init_scheduler(model_config)
183 |         # init parameters for grad computation
184 |         self.v_coef = model_config.getfloat('value_coef')
185 |         self.e_coef = model_config.getfloat('entropy_coef')
186 |         self.max_grad_norm = model_config.getfloat('max_grad_norm')
187 |         # init optimizer
188 |         alpha = model_config.getfloat('rmsp_alpha')
189 |         epsilon = model_config.getfloat('rmsp_epsilon')
190 |         gamma = model_config.getfloat('gamma')
191 |         self.optimizer = optim.RMSprop(self.policy.parameters(), self.lr_init,
192 |                                        eps=epsilon, alpha=alpha)
193 |         # init transition buffer
194 |         self._init_trans_buffer(gamma, distance_mask, coop_gamma)
195 | 
196 |     def _init_trans_buffer(self, gamma, distance_mask, coop_gamma):
197 |         self.trans_buffer = []
198 |         for i in range(self.n_agent):
199 |             # init replay buffer
200 |             self.trans_buffer.append(OnPolicyBuffer(gamma, coop_gamma, distance_mask[i]))
201 | 
202 |     def _update_lr(self):
203 |         # TODO: refactor this using optim.lr_scheduler
204 |         cur_lr = self.lr_scheduler.get(self.n_step)
205 |         for param_group in self.optimizer.param_groups:
206 |             param_group['lr'] = cur_lr
207 | 
208 | 
209 | class IA2C_FP(IA2C):
210 |     """
211 |     In fingerprint IA2C, neighborhood policies (fingerprints) are also included.
212 |     """
213 | 
214 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
215 |                  total_step, model_config, seed=0, use_gpu=False):
216 |         self.name = 'ia2c_fp'
217 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
218 |                         total_step, seed, use_gpu, model_config)
219 | 
220 |     def _init_policy(self):
221 |         policy = []
222 |         for i in range(self.n_agent):
223 |             n_n = np.sum(self.neighbor_mask[i])
224 |             # neighborhood policies are included in local state
225 |             if self.identical_agent:
226 |                 n_s1 = int(self.n_s_ls[i] + self.n_a * n_n)
227 |                 policy.append(FPPolicy(n_s1, self.n_a, int(n_n), self.n_step, n_fc=self.n_fc,
228 |                                        n_lstm=self.n_lstm, name='{:d}'.format(i)))
229 |             else:
230 |                 na_dim_ls = []
231 |                 for j in np.where(self.neighbor_mask[i] == 1)[0]:
232 |                     na_dim_ls.append(self.n_a_ls[j])
233 |                 n_s1 = int(self.n_s_ls[i] + sum(na_dim_ls))
234 |                 policy.append(FPPolicy(n_s1, self.n_a_ls[i], int(n_n), self.n_step, n_fc=self.n_fc,
235 |                                        n_lstm=self.n_lstm, name='{:d}'.format(i),
236 |                                        na_dim_ls=na_dim_ls, identical=False))
237 |         return nn.ModuleList(policy)
238 | 
239 | 
240 | class MA2C_NC(IA2C):
241 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
242 |                  total_step, model_config, seed=0, use_gpu=False):
243 |         """
244 |         n_s_ls: list of agents' state dimension. e.g., n_s_ls = [17, 17, 17, 17]
245 |         n_a_ls: list of agents' action dimension. e.g., n_a_ls = [5, 5, 5, 5]
246 |         neighbor_mask: matrix for neighbors: e.g., [[0,1,0,0], [1,0,1,0], [0,1,0,1], [0,0,1,0]]
247 |         distance_mask: matrix for distance: e.g., [[0,1,2,3], [1,0,1,2], [2,1,0,1], [3,2,1,0]]
248 |         coop_gamma: coop discount is used to discount the neighbors' impact
249 |         """
250 |         self.name = 'ma2c_nc'
251 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
252 |                         total_step, seed, use_gpu, model_config)
253 | 
254 |     def add_transition(self, ob, p, action, reward, value, done):
255 |         if self.reward_norm > 0:
256 |             reward = reward / self.reward_norm
257 |         if self.reward_clip > 0:
258 |             reward = np.clip(reward, -self.reward_clip, self.reward_clip)
259 |         if self.identical_agent:
260 |             self.trans_buffer.add_transition(np.array(ob), np.array(p), action,
261 |                                              reward, value, done)
262 |         else:
263 |             pad_ob, pad_p = self._convert_hetero_states(ob, p)
264 |             self.trans_buffer.add_transition(pad_ob, pad_p, action,
265 |                                              reward, value, done)
266 | 
267 |     def backward(self, Rends, dt, summary_writer=None, global_step=None):
268 |         self.optimizer.zero_grad()
269 |         obs, ps, acts, dones, Rs, Advs = self.trans_buffer.sample_transition(Rends, dt)
270 |         self.policy.backward(obs, ps, acts, dones, Rs, Advs, self.e_coef, self.v_coef,
271 |                              summary_writer=summary_writer, global_step=global_step)
272 |         if self.max_grad_norm > 0:
273 |             nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
274 |         self.optimizer.step()
275 |         if self.lr_decay != 'constant':
276 |             self._update_lr()
277 | 
278 |     def forward(self, obs, done, ps, actions=None, out_type='p'):
279 |         if self.identical_agent:
280 |             return self.policy.forward(np.array(obs), done, np.array(ps),
281 |                                        actions, out_type)
282 |         else:
283 |             pad_ob, pad_p = self._convert_hetero_states(obs, ps)
284 |             return self.policy.forward(pad_ob, done, pad_p,
285 |                                        actions, out_type)
286 | 
287 |     def reset(self):
288 |         self.policy._reset()
289 | 
290 |     def _convert_hetero_states(self, ob, p):
291 |         pad_ob = np.zeros((self.n_agent, self.n_s))
292 |         pad_p = np.zeros((self.n_agent, self.n_a))
293 |         for i in range(self.n_agent):
294 |             pad_ob[i, :len(ob[i])] = ob[i]
295 |             pad_p[i, :len(p[i])] = p[i]
296 |         return pad_ob, pad_p
297 | 
298 |     def _init_policy(self):
299 |         if self.identical_agent:
300 |             return NCMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
301 |                                       self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm)
302 |         else:
303 |             return NCMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
304 |                                       self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm,
305 |                                       n_s_ls=self.n_s_ls, n_a_ls=self.n_a_ls, identical=False)
306 | 
307 |     def _init_trans_buffer(self, gamma, distance_mask, coop_gamma):
308 |         self.trans_buffer = MultiAgentOnPolicyBuffer(gamma, coop_gamma, distance_mask)
309 | 
310 | 
311 | class IA2C_CU(MA2C_NC):
312 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
313 |                  total_step, model_config, seed=0, use_gpu=False):
314 |         self.name = 'ma2c_cu'
315 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
316 |                         total_step, seed, use_gpu, model_config)
317 | 
318 |     def _init_policy(self):
319 |         if self.identical_agent:
320 |             return ConsensusPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
321 |                                    self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm)
322 |         else:
323 |             return ConsensusPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
324 |                                    self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm,
325 |                                    n_s_ls=self.n_s_ls, n_a_ls=self.n_a_ls, identical=False)
326 | 
327 |     def backward(self, Rends, dt, summary_writer=None, global_step=None):
328 |         super(IA2C_CU, self).backward(Rends, dt, summary_writer, global_step)
329 |         self.policy.consensus_update()
330 | 
331 | 
332 | class MA2C_CNET(MA2C_NC):
333 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
334 |                  total_step, model_config, seed=0, use_gpu=False):
335 |         self.name = 'ma2c_ic3'
336 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
337 |                         total_step, seed, use_gpu, model_config)
338 | 
339 |     def _init_policy(self):
340 |         if self.identical_agent:
341 |             return CommNetMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
342 |                                            self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm)
343 |         else:
344 |             return CommNetMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
345 |                                            self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm,
346 |                                            n_s_ls=self.n_s_ls, n_a_ls=self.n_a_ls, identical=False)
347 | 
348 | 
349 | class MA2C_DIAL(MA2C_NC):
350 |     def __init__(self, n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
351 |                  total_step, model_config, seed=0, use_gpu=False):
352 |         self.name = 'ma2c_dial'
353 |         self._init_algo(n_s_ls, n_a_ls, neighbor_mask, distance_mask, coop_gamma,
354 |                         total_step, seed, use_gpu, model_config)
355 | 
356 |     def _init_policy(self):
357 |         if self.identical_agent:
358 |             return DIALMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
359 |                                         self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm)
360 |         else:
361 |             return DIALMultiAgentPolicy(self.n_s, self.n_a, self.n_agent, self.n_step,
362 |                                         self.neighbor_mask, n_fc=self.n_fc, n_h=self.n_lstm,
363 |                                         n_s_ls=self.n_s_ls, n_a_ls=self.n_a_ls, identical=False)
364 | 


--------------------------------------------------------------------------------
/agents/policies.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Policy functions for training and update
  3 | """
  4 | import numpy as np
  5 | import torch
  6 | import torch.nn as nn
  7 | import torch.nn.functional as F
  8 | from agents.utils import batch_to_seq, init_layer, one_hot, run_rnn
  9 | 
 10 | 
 11 | class Policy(nn.Module):
 12 |     def __init__(self, n_a, n_s, n_step, policy_name, agent_name, identical):
 13 |         super(Policy, self).__init__()
 14 |         self.name = policy_name
 15 |         if agent_name is not None:
 16 |             # for multi-agent system
 17 |             self.name += '_' + str(agent_name)
 18 |         self.n_a = n_a
 19 |         self.n_s = n_s
 20 |         self.n_step = n_step
 21 |         self.identical = identical
 22 | 
 23 |     def forward(self, ob, *_args, **_kwargs):
 24 |         raise NotImplementedError()
 25 | 
 26 |     def _init_actor_head(self, n_h, n_a=None):
 27 |         if n_a is None:
 28 |             n_a = self.n_a
 29 |         # only discrete control is supported for now
 30 |         self.actor_head = nn.Linear(n_h, n_a)
 31 |         init_layer(self.actor_head, 'fc')
 32 | 
 33 |     def _init_critic_head(self, n_h, n_n=None):
 34 |         if n_n is None:
 35 |             n_n = int(self.n_n)
 36 |         if n_n:
 37 |             if self.identical:
 38 |                 n_na_sparse = self.n_a * n_n
 39 |             else:
 40 |                 n_na_sparse = sum(self.na_dim_ls)
 41 |             n_h += n_na_sparse
 42 |         self.critic_head = nn.Linear(n_h, 1)
 43 |         init_layer(self.critic_head, 'fc')
 44 | 
 45 |     def _run_critic_head(self, h, na, n_n=None):
 46 |         if n_n is None:
 47 |             n_n = int(self.n_n)
 48 |         if n_n:
 49 |             na = torch.from_numpy(na).long()
 50 |             if self.identical:
 51 |                 na_sparse = one_hot(na, self.n_a)
 52 |                 na_sparse = na_sparse.view(-1, self.n_a * n_n)
 53 |             else:
 54 |                 na_sparse = []
 55 |                 na_ls = torch.chunk(na, n_n, dim=1)
 56 |                 for na_val, na_dim in zip(na_ls, self.na_dim_ls):
 57 |                     na_sparse.append(torch.squeeze(one_hot(na_val, na_dim), dim=1))
 58 |                 na_sparse = torch.cat(na_sparse, dim=1)
 59 |             h = torch.cat([h, na_sparse], dim=1)
 60 |         return self.critic_head(h).squeeze()
 61 | 
 62 |     def _run_loss(self, actor_dist, e_coef, v_coef, vs, As, Rs, Advs):
 63 |         log_probs = actor_dist.log_prob(As)
 64 |         policy_loss = -(log_probs * Advs).mean()
 65 |         entropy_loss = -(actor_dist.entropy()).mean() * e_coef
 66 |         value_loss = (Rs - vs).pow(2).mean() * v_coef
 67 |         return policy_loss, value_loss, entropy_loss
 68 | 
 69 |     def _update_tensorboard(self, summary_writer, global_step):
 70 |         # monitor training
 71 |         summary_writer.add_scalar('loss/entropy_loss', self.entropy_loss,
 72 |                                   global_step=global_step)
 73 |         summary_writer.add_scalar('loss/policy_loss', self.policy_loss,
 74 |                                   global_step=global_step)
 75 |         summary_writer.add_scalar('loss/value_loss', self.value_loss,
 76 |                                   global_step=global_step)
 77 |         summary_writer.add_scalar('loss/total_loss', self.loss,
 78 |                                   global_step=global_step)
 79 | 
 80 | 
 81 | class LstmPolicy(Policy):
 82 |     def __init__(self, n_s, n_a, n_n, n_step, n_fc=64, n_lstm=64, name=None,
 83 |                  na_dim_ls=None, identical=True):
 84 |         super(LstmPolicy, self).__init__(n_a, n_s, n_step, 'lstm', name, identical)
 85 |         if not self.identical:
 86 |             self.na_dim_ls = na_dim_ls
 87 |         self.n_lstm = n_lstm
 88 |         self.n_fc = n_fc
 89 |         self.n_n = n_n
 90 |         self._init_net()
 91 |         self._reset()
 92 | 
 93 |     def backward(self, obs, nactions, acts, dones, Rs, Advs,
 94 |                  e_coef, v_coef, summary_writer=None, global_step=None):
 95 |         obs = torch.from_numpy(obs).float()
 96 |         dones = torch.from_numpy(dones).float()
 97 |         xs = self._encode_ob(obs)
 98 |         hs, new_states = run_rnn(self.lstm_layer, xs, dones, self.states_bw)
 99 |         # backward grad is limited to the minibatch
100 |         self.states_bw = new_states.detach()
101 |         actor_dist = torch.distributions.categorical.Categorical(logits=F.log_softmax(self.actor_head(hs), dim=1))
102 |         vs = self._run_critic_head(hs, nactions)
103 |         self.policy_loss, self.value_loss, self.entropy_loss = \
104 |             self._run_loss(actor_dist, e_coef, v_coef, vs,
105 |                            torch.from_numpy(acts).long(),
106 |                            torch.from_numpy(Rs).float(),
107 |                            torch.from_numpy(Advs).float())
108 |         self.loss = self.policy_loss + self.value_loss + self.entropy_loss
109 |         self.loss.backward()
110 |         if summary_writer is not None:
111 |             self._update_tensorboard(summary_writer, global_step)
112 | 
113 |     def forward(self, ob, done, naction=None, out_type='p'):
114 |         ob = torch.from_numpy(np.expand_dims(ob, axis=0)).float()
115 |         done = torch.from_numpy(np.expand_dims(done, axis=0)).float()
116 |         x = self._encode_ob(ob)
117 |         h, new_states = run_rnn(self.lstm_layer, x, done, self.states_fw)
118 |         if out_type.startswith('p'):
119 |             self.states_fw = new_states.detach()
120 |             return F.softmax(self.actor_head(h), dim=1).squeeze().detach().numpy()
121 |         else:
122 |             return self._run_critic_head(h, np.array([naction])).detach().numpy()
123 | 
124 |     def _encode_ob(self, ob):
125 |         return F.relu(self.fc_layer(ob))
126 | 
127 |     def _init_net(self):
128 |         self.fc_layer = nn.Linear(self.n_s, self.n_fc)
129 |         init_layer(self.fc_layer, 'fc')
130 |         self.lstm_layer = nn.LSTMCell(self.n_fc, self.n_lstm)
131 |         init_layer(self.lstm_layer, 'lstm')
132 |         self._init_actor_head(self.n_lstm)
133 |         self._init_critic_head(self.n_lstm)
134 | 
135 |     def _reset(self):
136 |         # forget the cumulative states every cum_step
137 |         self.states_fw = torch.zeros(self.n_lstm * 2)
138 |         self.states_bw = torch.zeros(self.n_lstm * 2)
139 | 
140 | 
141 | class FPPolicy(LstmPolicy):
142 |     def __init__(self, n_s, n_a, n_n, n_step, n_fc=64, n_lstm=64, name=None,
143 |                  na_dim_ls=None, identical=True):
144 |         super(FPPolicy, self).__init__(n_s, n_a, n_n, n_step, n_fc, n_lstm, name,
145 |                                        na_dim_ls, identical)
146 | 
147 |     def _init_net(self):
148 |         if self.identical:
149 |             # dim of obs except the fingerprints
150 |             self.n_x = self.n_s - self.n_n * self.n_a
151 |         else:
152 |             self.n_x = int(self.n_s - sum(self.na_dim_ls))
153 |         self.fc_x_layer = nn.Linear(self.n_x, self.n_fc)
154 |         init_layer(self.fc_x_layer, 'fc')
155 |         n_h = self.n_fc
156 |         if self.n_n:
157 |             self.fc_p_layer = nn.Linear(self.n_s - self.n_x, self.n_fc)
158 |             init_layer(self.fc_p_layer, 'fc')
159 |             n_h += self.n_fc
160 |         self.lstm_layer = nn.LSTMCell(n_h, self.n_lstm)
161 |         init_layer(self.lstm_layer, 'lstm')
162 |         self._init_actor_head(self.n_lstm)
163 |         self._init_critic_head(self.n_lstm)
164 | 
165 |     def _encode_ob(self, ob):
166 |         x = F.relu(self.fc_x_layer(ob[:, :self.n_x]))
167 |         if self.n_n:
168 |             p = F.relu(self.fc_p_layer(ob[:, self.n_x:]))
169 |             x = torch.cat([x, p], dim=1)
170 |         return x
171 | 
172 | 
173 | class NCMultiAgentPolicy(Policy):
174 |     """ Implemented as a centralized meta-DNN. To simplify the implementation, all input
175 |     and output dimensions are identical among all agents, and invalid values are casted as
176 |     zeros during runtime."""
177 | 
178 |     def __init__(self, n_s, n_a, n_agent, n_step, neighbor_mask, n_fc=64, n_h=64,
179 |                  n_s_ls=None, n_a_ls=None, identical=True):
180 |         """
181 |         n_h: dim of hidden states
182 |         """
183 |         super(NCMultiAgentPolicy, self).__init__(n_a, n_s, n_step, 'nc', None, identical)
184 |         if not self.identical:
185 |             self.n_s_ls = n_s_ls
186 |             self.n_a_ls = n_a_ls
187 |         self.n_agent = n_agent
188 |         self.neighbor_mask = neighbor_mask
189 |         self.n_fc = n_fc
190 |         self.n_h = n_h
191 |         self._init_net()
192 |         self._reset()
193 | 
194 |     def _get_neighbor_dim(self, i_agent):
195 |         # return num of neighbors, dim of overall states, sum of neigbor's action dim,
196 |         # list of neighbor-information's dim, list of neighbor-action's dim
197 |         n_n = int(np.sum(self.neighbor_mask[i_agent]))
198 |         if self.identical:
199 |             return n_n, self.n_s * (n_n + 1), self.n_a * n_n, [self.n_s] * n_n, [self.n_a] * n_n
200 |         else:
201 |             ns_ls = []
202 |             na_ls = []
203 |             for j in np.where(self.neighbor_mask[i_agent])[0]:
204 |                 ns_ls.append(self.n_s_ls[j])
205 |                 na_ls.append(self.n_a_ls[j])
206 |             return n_n, self.n_s_ls[i_agent] + sum(ns_ls), sum(na_ls), ns_ls, na_ls
207 | 
208 |     def _init_actor_head(self, n_a):
209 |         # only discrete control is supported for now
210 |         actor_head = nn.Linear(self.n_h, n_a)
211 |         init_layer(actor_head, 'fc')
212 |         self.actor_heads.append(actor_head)
213 | 
214 |     def _init_critic_head(self, n_na):
215 |         # TODO: can we include more information,like neighbor's hidden states
216 |         critic_head = nn.Linear(self.n_h + n_na, 1)
217 |         init_layer(critic_head, 'fc')
218 |         self.critic_heads.append(critic_head)
219 | 
220 |     def _init_net(self):
221 |         self.fc_x_layers = nn.ModuleList()
222 |         self.fc_p_layers = nn.ModuleList()
223 |         self.fc_m_layers = nn.ModuleList()
224 |         self.lstm_layers = nn.ModuleList()
225 |         self.actor_heads = nn.ModuleList()
226 |         self.critic_heads = nn.ModuleList()
227 |         self.ns_ls_ls = []
228 |         self.na_ls_ls = []
229 |         self.n_n_ls = []
230 |         for i in range(self.n_agent):
231 |             n_n, n_ns, n_na, ns_ls, na_ls = self._get_neighbor_dim(i)
232 |             self.ns_ls_ls.append(ns_ls)
233 |             self.na_ls_ls.append(na_ls)
234 |             self.n_n_ls.append(n_n)
235 |             self._init_comm_layer(n_n, n_ns, n_na)
236 |             n_a = self.n_a if self.identical else self.n_a_ls[i]
237 |             self._init_actor_head(n_a)
238 |             self._init_critic_head(n_na)
239 | 
240 |     def _reset(self):
241 |         self.states_fw = torch.zeros(self.n_agent, self.n_h * 2)
242 |         self.states_bw = torch.zeros(self.n_agent, self.n_h * 2)
243 | 
244 |     def _run_actor_heads(self, hs, detach=False):
245 |         ps = []
246 |         for i in range(self.n_agent):
247 |             if detach:
248 |                 p_i = F.softmax(self.actor_heads[i](hs[i]), dim=1).squeeze().detach().numpy()
249 |             else:
250 |                 p_i = F.log_softmax(self.actor_heads[i](hs[i]), dim=1)
251 |             ps.append(p_i)
252 |         return ps
253 | 
254 |     def _run_critic_heads(self, hs, actions, detach=False):
255 |         # include neighbor's actions (one-hot) to get the value fn
256 |         vs = []
257 |         for i in range(self.n_agent):
258 |             n_n = self.n_n_ls[i]
259 |             if n_n:
260 |                 js = torch.from_numpy(np.where(self.neighbor_mask[i])[0]).long()
261 |                 na_i = torch.index_select(actions, 0, js)
262 |                 na_i_ls = []
263 |                 for j in range(n_n):
264 |                     na_i_ls.append(one_hot(na_i[j], self.na_ls_ls[i][j]))
265 |                 h_i = torch.cat([hs[i]] + na_i_ls, dim=1)
266 |             else:
267 |                 h_i = hs[i]
268 |             v_i = self.critic_heads[i](h_i).squeeze()
269 |             if detach:
270 |                 vs.append(v_i.detach().numpy())
271 |             else:
272 |                 vs.append(v_i)
273 |         return vs
274 | 
275 |     def backward(self, obs, fps, acts, dones, Rs, Advs,
276 |                  e_coef, v_coef, summary_writer=None, global_step=None):
277 |         obs = torch.from_numpy(obs).float().transpose(0, 1)
278 |         dones = torch.from_numpy(dones).float()
279 |         fps = torch.from_numpy(fps).float().transpose(0, 1)
280 |         acts = torch.from_numpy(acts).long()
281 |         hs, new_states = self._run_comm_layers(obs, dones, fps, self.states_bw)
282 |         # backward grad is limited to the minibatch
283 |         self.states_bw = new_states.detach()
284 |         ps = self._run_actor_heads(hs)
285 |         vs = self._run_critic_heads(hs, acts)
286 |         self.policy_loss = 0
287 |         self.value_loss = 0
288 |         self.entropy_loss = 0
289 |         Rs = torch.from_numpy(Rs).float()
290 |         Advs = torch.from_numpy(Advs).float()
291 |         for i in range(self.n_agent):
292 |             actor_dist_i = torch.distributions.categorical.Categorical(logits=ps[i])
293 |             policy_loss_i, value_loss_i, entropy_loss_i = \
294 |                 self._run_loss(actor_dist_i, e_coef, v_coef, vs[i],
295 |                                acts[i], Rs[i], Advs[i])
296 |             self.policy_loss += policy_loss_i
297 |             self.value_loss += value_loss_i
298 |             self.entropy_loss += entropy_loss_i
299 |         self.loss = self.policy_loss + self.value_loss + self.entropy_loss
300 |         self.loss.backward()
301 |         if summary_writer is not None:
302 |             self._update_tensorboard(summary_writer, global_step)
303 | 
304 |     def forward(self, ob, done, fp, action=None, out_type='p'):
305 |         # TxNxm
306 |         ob = torch.from_numpy(np.expand_dims(ob, axis=0)).float()
307 |         done = torch.from_numpy(np.expand_dims(done, axis=0)).float()
308 |         fp = torch.from_numpy(np.expand_dims(fp, axis=0)).float()
309 |         # h dim: NxTxm
310 |         h, new_states = self._run_comm_layers(ob, done, fp, self.states_fw)
311 |         if out_type.startswith('p'):
312 |             self.states_fw = new_states.detach()
313 |             return self._run_actor_heads(h, detach=True)
314 |         else:
315 |             action = torch.from_numpy(np.expand_dims(action, axis=1)).long()
316 |             return self._run_critic_heads(h, action, detach=True)
317 | 
318 |     def _init_comm_layer(self, n_n, n_ns, n_na):
319 |         # num of neighbors, neighbor-information's dim, neighbor-policy's dim
320 |         n_lstm_in = 3 * self.n_fc
321 |         fc_x_layer = nn.Linear(n_ns, self.n_fc)
322 |         init_layer(fc_x_layer, 'fc')
323 |         self.fc_x_layers.append(fc_x_layer)
324 |         if n_n:
325 |             fc_p_layer = nn.Linear(n_na, self.n_fc)
326 |             init_layer(fc_p_layer, 'fc')
327 |             fc_m_layer = nn.Linear(self.n_h * n_n, self.n_fc)
328 |             init_layer(fc_m_layer, 'fc')
329 |             self.fc_m_layers.append(fc_m_layer)
330 |             self.fc_p_layers.append(fc_p_layer)
331 |             lstm_layer = nn.LSTMCell(n_lstm_in, self.n_h)
332 |         else:
333 |             self.fc_m_layers.append(None)
334 |             self.fc_p_layers.append(None)
335 |             lstm_layer = nn.LSTMCell(self.n_fc, self.n_h)
336 |         init_layer(lstm_layer, 'lstm')
337 |         self.lstm_layers.append(lstm_layer)
338 | 
339 |     def _run_comm_layers(self, obs, dones, fps, states):
340 |         # encode comm info(s_i) and (h_(t-1), c_(t-1)) to the h_t and c_t
341 |         # states (h_(t-1), c_(t-1)): t-1 hidden states, (self.n_agent, self.n_h * 2)
342 |         obs = batch_to_seq(obs)
343 |         dones = batch_to_seq(dones)
344 |         fps = batch_to_seq(fps)
345 |         h, c = torch.chunk(states, 2, dim=1)
346 |         outputs = []
347 |         for t, (x, p, done) in enumerate(zip(obs, fps, dones)):
348 |             next_h = []
349 |             next_c = []
350 |             x = x.squeeze(0)
351 |             p = p.squeeze(0)
352 |             for i in range(self.n_agent):
353 |                 n_n = self.n_n_ls[i]
354 |                 if n_n:
355 |                     s_i = self._get_comm_s(i, n_n, x, h, p)
356 |                 else:
357 |                     s_i = F.relu(self.fc_x_layers[i](x[i].unsqueeze(0)))
358 |                 h_i, c_i = h[i].unsqueeze(0) * (1 - done), c[i].unsqueeze(0) * (1 - done)
359 |                 next_h_i, next_c_i = self.lstm_layers[i](s_i, (h_i, c_i))
360 |                 next_h.append(next_h_i)
361 |                 next_c.append(next_c_i)
362 |             h, c = torch.cat(next_h), torch.cat(next_c)
363 |             outputs.append(h.unsqueeze(0))
364 |         outputs = torch.cat(outputs)
365 |         return outputs.transpose(0, 1), torch.cat([h, c], dim=1)
366 | 
367 |     def _get_comm_s(self, i, n_n, x, h, p):
368 |         # get the S_(i-1), h_(i-1), Pi_(i-1)
369 |         js = torch.from_numpy(np.where(self.neighbor_mask[i])[0]).long()
370 |         m_i = torch.index_select(h, 0, js).view(1, self.n_h * n_n)  # neighbor's hidden states
371 |         p_i = torch.index_select(p, 0, js)  # neighbor's policy
372 |         nx_i = torch.index_select(x, 0, js)  # neighbor's obs
373 |         if self.identical:
374 |             p_i = p_i.view(1, self.n_a * n_n)
375 |             nx_i = nx_i.view(1, self.n_s * n_n)
376 |         else:
377 |             p_i_ls = []
378 |             nx_i_ls = []
379 |             for j in range(n_n):
380 |                 p_i_ls.append(p_i[j].narrow(0, 0, self.na_ls_ls[i][j]))
381 |                 nx_i_ls.append(nx_i[j].narrow(0, 0, self.ns_ls_ls[i][j]))
382 |             p_i = torch.cat(p_i_ls).unsqueeze(0)
383 |             nx_i = torch.cat(nx_i_ls).unsqueeze(0)
384 | 
385 |         # encode neighbor's states along with its own states;
386 |         # encode neighbor's policies;
387 |         # encode neighbor's hidden states.
388 |         s_i = [F.relu(self.fc_x_layers[i](torch.cat([x[i].unsqueeze(0), nx_i], dim=1))),
389 |                F.relu(self.fc_p_layers[i](p_i)),
390 |                F.relu(self.fc_m_layers[i](m_i))]
391 |         return torch.cat(s_i, dim=1)
392 | 
393 | 
394 | class ConsensusPolicy(NCMultiAgentPolicy):
395 |     def __init__(self, n_s, n_a, n_agent, n_step, neighbor_mask, n_fc=64, n_h=64,
396 |                  n_s_ls=None, n_a_ls=None, identical=True):
397 |         Policy.__init__(self, n_a, n_s, n_step, 'cu', None, identical)
398 |         if not self.identical:
399 |             self.n_s_ls = n_s_ls
400 |             self.n_a_ls = n_a_ls
401 |         self.n_agent = n_agent
402 |         self.neighbor_mask = neighbor_mask
403 |         self.n_fc = n_fc
404 |         self.n_h = n_h
405 |         self._init_net()
406 |         self._reset()
407 | 
408 |     def consensus_update(self):
409 |         consensus_update = []
410 |         with torch.no_grad():
411 |             for i in range(self.n_agent):
412 |                 mean_wts = self._get_critic_wts(i)
413 |                 for param, wt in zip(self.lstm_layers[i].parameters(), mean_wts):
414 |                     param.copy_(wt)
415 | 
416 |     def _init_net(self):
417 |         self.fc_x_layers = nn.ModuleList()
418 |         self.lstm_layers = nn.ModuleList()
419 |         self.actor_heads = nn.ModuleList()
420 |         self.critic_heads = nn.ModuleList()
421 |         self.na_ls_ls = []
422 |         self.n_n_ls = []
423 |         for i in range(self.n_agent):
424 |             n_n, _, n_na, _, na_ls = self._get_neighbor_dim(i)
425 |             n_s = self.n_s if self.identical else self.n_s_ls[i]
426 |             self.na_ls_ls.append(na_ls)
427 |             self.n_n_ls.append(n_n)
428 |             fc_x_layer = nn.Linear(n_s, self.n_fc)
429 |             init_layer(fc_x_layer, 'fc')
430 |             self.fc_x_layers.append(fc_x_layer)
431 |             lstm_layer = nn.LSTMCell(self.n_fc, self.n_h)
432 |             init_layer(lstm_layer, 'lstm')
433 |             self.lstm_layers.append(lstm_layer)
434 |             n_a = self.n_a if self.identical else self.n_a_ls[i]
435 |             self._init_actor_head(n_a)
436 |             self._init_critic_head(n_na)
437 | 
438 |     def _get_critic_wts(self, i_agent):
439 |         wts = []
440 |         for wt in self.lstm_layers[i_agent].parameters():
441 |             wts.append(wt.detach())
442 |         neighbors = list(np.where(self.neighbor_mask[i_agent] == 1)[0])
443 |         for j in neighbors:
444 |             for k, wt in enumerate(self.lstm_layers[j].parameters()):
445 |                 wts[k] += wt.detach()
446 |         n = 1 + len(neighbors)
447 |         for k in range(len(wts)):
448 |             wts[k] /= n
449 |         return wts
450 | 
451 |     def _run_comm_layers(self, obs, dones, fps, states):
452 |         # NxTxm
453 |         obs = obs.transpose(0, 1)
454 |         hs = []
455 |         new_states = []
456 |         for i in range(self.n_agent):
457 |             xs_i = F.relu(self.fc_x_layers[i](obs[i]))
458 |             hs_i, new_states_i = run_rnn(self.lstm_layers[i], xs_i, dones, states[i])
459 |             hs.append(hs_i.unsqueeze(0))
460 |             new_states.append(new_states_i.unsqueeze(0))
461 |         return torch.cat(hs), torch.cat(new_states)
462 | 
463 | 
464 | class CommNetMultiAgentPolicy(NCMultiAgentPolicy):
465 |     """Reference code: https://github.com/IC3Net/IC3Net/blob/master/comm.py.
466 |        Note in CommNet, the message is generated from hidden state only, so current state
467 |        and neigbor policies are not included in the inputs."""
468 | 
469 |     def __init__(self, n_s, n_a, n_agent, n_step, neighbor_mask, n_fc=64, n_h=64,
470 |                  n_s_ls=None, n_a_ls=None, identical=True):
471 |         Policy.__init__(self, n_a, n_s, n_step, 'cnet', None, identical)
472 |         if not self.identical:
473 |             self.n_s_ls = n_s_ls
474 |             self.n_a_ls = n_a_ls
475 |         self.n_agent = n_agent
476 |         self.neighbor_mask = neighbor_mask
477 |         self.n_fc = n_fc
478 |         self.n_h = n_h
479 |         self._init_net()
480 |         self._reset()
481 | 
482 |     def _init_comm_layer(self, n_n, n_ns, n_na):
483 |         fc_x_layer = nn.Linear(n_ns, self.n_fc)
484 |         init_layer(fc_x_layer, 'fc')
485 |         self.fc_x_layers.append(fc_x_layer)
486 |         if n_n:
487 |             fc_m_layer = nn.Linear(self.n_h, self.n_fc)
488 |             init_layer(fc_m_layer, 'fc')
489 |             self.fc_m_layers.append(fc_m_layer)
490 |         else:
491 |             self.fc_m_layers.append(None)
492 |         lstm_layer = nn.LSTMCell(self.n_fc, self.n_h)
493 |         init_layer(lstm_layer, 'lstm')
494 |         self.lstm_layers.append(lstm_layer)
495 | 
496 |     def _get_comm_s(self, i, n_n, x, h, p):
497 |         js = torch.from_numpy(np.where(self.neighbor_mask[i])[0]).long()
498 |         m_i = torch.index_select(h, 0, js).mean(dim=0, keepdim=True)
499 |         nx_i = torch.index_select(x, 0, js)
500 |         if self.identical:
501 |             nx_i = nx_i.view(1, self.n_s * n_n)
502 |         else:
503 |             nx_i_ls = []
504 |             for j in range(n_n):
505 |                 nx_i_ls.append(nx_i[j].narrow(0, 0, self.ns_ls_ls[i][j]))
506 |             nx_i = torch.cat(nx_i_ls).unsqueeze(0)
507 |         return F.relu(self.fc_x_layers[i](torch.cat([x[i].unsqueeze(0), nx_i], dim=1))) + \
508 |                self.fc_m_layers[i](m_i)
509 | 
510 | 
511 | class DIALMultiAgentPolicy(NCMultiAgentPolicy):
512 |     def __init__(self, n_s, n_a, n_agent, n_step, neighbor_mask, n_fc=64, n_h=64,
513 |                  n_s_ls=None, n_a_ls=None, identical=True):
514 |         Policy.__init__(self, n_a, n_s, n_step, 'dial', None, identical)
515 |         if not self.identical:
516 |             self.n_s_ls = n_s_ls
517 |             self.n_a_ls = n_a_ls
518 |         self.n_agent = n_agent
519 |         self.neighbor_mask = neighbor_mask
520 |         self.n_fc = n_fc
521 |         self.n_h = n_h
522 |         self._init_net()
523 |         self._reset()
524 | 
525 |     def _init_comm_layer(self, n_n, n_ns, n_na):
526 |         fc_x_layer = nn.Linear(n_ns, self.n_fc)
527 |         init_layer(fc_x_layer, 'fc')
528 |         self.fc_x_layers.append(fc_x_layer)
529 |         if n_n:
530 |             fc_m_layer = nn.Linear(self.n_h * n_n, self.n_fc)
531 |             init_layer(fc_m_layer, 'fc')
532 |             self.fc_m_layers.append(fc_m_layer)
533 |         else:
534 |             self.fc_m_layers.append(None)
535 |         lstm_layer = nn.LSTMCell(self.n_fc, self.n_h)
536 |         init_layer(lstm_layer, 'lstm')
537 |         self.lstm_layers.append(lstm_layer)
538 | 
539 |     def _get_comm_s(self, i, n_n, x, h, p):
540 |         # different with NC, include only it's own action, no other's policies.
541 |         # summation all the info
542 |         js = torch.from_numpy(np.where(self.neighbor_mask[i])[0]).long()
543 |         m_i = torch.index_select(h, 0, js).view(1, self.n_h * n_n)
544 |         nx_i = torch.index_select(x, 0, js)
545 |         if self.identical:
546 |             nx_i = nx_i.view(1, self.n_s * n_n)
547 |         else:
548 |             nx_i_ls = []
549 |             for j in range(n_n):
550 |                 nx_i_ls.append(nx_i[j].narrow(0, 0, self.ns_ls_ls[i][j]))
551 |             nx_i = torch.cat(nx_i_ls).unsqueeze(0)
552 |         a_i = one_hot(p[i].argmax().unsqueeze(0), self.n_fc)
553 |         return F.relu(self.fc_x_layers[i](torch.cat([x[i].unsqueeze(0), nx_i], dim=1))) + \
554 |                F.relu(self.fc_m_layers[i](m_i)) + a_i
555 | 


--------------------------------------------------------------------------------
/agents/utils.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import torch
  3 | import torch.nn as nn
  4 | 
  5 | 
  6 | def init_layer(layer, layer_type):
  7 |     """
  8 |     initializers
  9 |     """
 10 |     if layer_type == 'fc':
 11 |         nn.init.orthogonal_(layer.weight.data)
 12 |         nn.init.constant_(layer.bias.data, 0)
 13 |     elif layer_type == 'lstm':
 14 |         nn.init.orthogonal_(layer.weight_ih.data)
 15 |         nn.init.orthogonal_(layer.weight_hh.data)
 16 |         nn.init.constant_(layer.bias_ih.data, 0)
 17 |         nn.init.constant_(layer.bias_hh.data, 0)
 18 | 
 19 | 
 20 | def batch_to_seq(x):
 21 |     """
 22 |     layer helpers
 23 |     """
 24 |     n_step = x.shape[0]
 25 |     if len(x.shape) == 1:
 26 |         x = torch.unsqueeze(x, -1)
 27 |     return torch.chunk(x, n_step)
 28 | 
 29 | 
 30 | def run_rnn(layer, xs, dones, s):
 31 |     # run the LSTM (RNN) module
 32 |     xs = batch_to_seq(xs)
 33 |     # need dones to reset states
 34 |     dones = batch_to_seq(dones)
 35 |     n_in = int(xs[0].shape[1])
 36 |     n_out = int(s.shape[0]) // 2
 37 |     s = torch.unsqueeze(s, 0)
 38 |     h, c = torch.chunk(s, 2, dim=1)
 39 |     outputs = []
 40 |     for _, (x, done) in enumerate(zip(xs, dones)):
 41 |         c = c * (1 - done)
 42 |         h = h * (1 - done)
 43 |         h, c = layer(x, (h, c))
 44 |         outputs.append(h)
 45 |     s = torch.cat([h, c], dim=1)
 46 |     return torch.cat(outputs), torch.squeeze(s)
 47 | 
 48 | 
 49 | def one_hot(x, oh_dim, dim=-1):
 50 |     oh_shape = list(x.shape)
 51 |     if dim == -1:
 52 |         oh_shape.append(oh_dim)
 53 |     else:
 54 |         oh_shape = oh_shape[:dim + 1] + [oh_dim] + oh_shape[dim + 1:]
 55 |     x_oh = torch.zeros(oh_shape)
 56 |     x = torch.unsqueeze(x, -1)
 57 |     if dim == -1:
 58 |         x_oh = x_oh.scatter(dim, x, 1)
 59 |     else:
 60 |         x_oh = x_oh.scatter(dim + 1, x, 1)
 61 |     return x_oh
 62 | 
 63 | 
 64 | class TransBuffer:
 65 |     """
 66 |     buffers
 67 |     """
 68 |     def reset(self):
 69 |         self.buffer = []
 70 | 
 71 |     @property
 72 |     def size(self):
 73 |         return len(self.buffer)
 74 | 
 75 |     def add_transition(self, ob, a, r, *_args, **_kwargs):
 76 |         raise NotImplementedError()
 77 | 
 78 |     def sample_transition(self, *_args, **_kwargs):
 79 |         raise NotImplementedError()
 80 | 
 81 | 
 82 | class OnPolicyBuffer(TransBuffer):
 83 |     def __init__(self, gamma, alpha, distance_mask):
 84 |         self.gamma = gamma
 85 |         self.alpha = alpha
 86 |         if alpha > 0:
 87 |             self.distance_mask = distance_mask
 88 |             self.max_distance = np.max(distance_mask, axis=-1)
 89 |         self.reset()
 90 | 
 91 |     def reset(self, done=False):
 92 |         # the done before each step is required
 93 |         self.obs = []
 94 |         self.acts = []
 95 |         self.rs = []
 96 |         self.vs = []
 97 |         self.adds = []
 98 |         self.dones = [done]
 99 | 
100 |     def add_transition(self, ob, na, a, r, v, done):
101 |         self.obs.append(ob)
102 |         self.adds.append(na)
103 |         self.acts.append(a)
104 |         self.rs.append(r)
105 |         self.vs.append(v)
106 |         self.dones.append(done)
107 | 
108 |     def sample_transition(self, R, dt=0):
109 |         if self.alpha < 0:
110 |             self._add_R_Adv(R)
111 |         else:
112 |             self._add_s_R_Adv(R)
113 |         obs = np.array(self.obs, dtype=np.float32)
114 |         nas = np.array(self.adds, dtype=np.int32)
115 |         acts = np.array(self.acts, dtype=np.int32)
116 |         Rs = np.array(self.Rs, dtype=np.float32)
117 |         Advs = np.array(self.Advs, dtype=np.float32)
118 |         # use pre-step dones here
119 |         dones = np.array(self.dones[:-1], dtype=np.bool)
120 |         self.reset(self.dones[-1])
121 |         return obs, nas, acts, dones, Rs, Advs
122 | 
123 |     def _add_R_Adv(self, R):
124 |         Rs = []
125 |         Advs = []
126 |         # use post-step dones here
127 |         for r, v, done in zip(self.rs[::-1], self.vs[::-1], self.dones[:0:-1]):
128 |             R = r + self.gamma * R * (1. - done)
129 |             Adv = R - v
130 |             Rs.append(R)
131 |             Advs.append(Adv)
132 |         Rs.reverse()
133 |         Advs.reverse()
134 |         self.Rs = Rs
135 |         self.Advs = Advs
136 | 
137 |     def _add_st_R_Adv(self, R, dt):
138 |         Rs = []
139 |         Advs = []
140 |         # use post-step dones here
141 |         tdiff = dt
142 |         for r, v, done in zip(self.rs[::-1], self.vs[::-1], self.dones[:0:-1]):
143 |             R = self.gamma * R * (1. - done)
144 |             if done:
145 |                 tdiff = 0
146 |             # additional spatial rewards
147 |             tmax = min(tdiff, self.max_distance)
148 |             for t in range(tmax + 1):
149 |                 rt = np.sum(r[self.distance_mask == t])
150 |                 R += (self.gamma * self.alpha) ** t * rt
151 |             Adv = R - v
152 |             tdiff += 1
153 |             Rs.append(R)
154 |             Advs.append(Adv)
155 |         Rs.reverse()
156 |         Advs.reverse()
157 |         self.Rs = Rs
158 |         self.Advs = Advs
159 | 
160 |     def _add_s_R_Adv(self, R):
161 |         Rs = []
162 |         Advs = []
163 |         # use post-step dones here
164 |         for r, v, done in zip(self.rs[::-1], self.vs[::-1], self.dones[:0:-1]):
165 |             R = self.gamma * R * (1. - done)
166 |             # additional spatial rewards
167 |             for t in range(self.max_distance + 1):
168 |                 rt = np.sum(r[self.distance_mask == t])
169 |                 R += (self.alpha ** t) * rt
170 |             Adv = R - v
171 |             Rs.append(R)
172 |             Advs.append(Adv)
173 |         Rs.reverse()
174 |         Advs.reverse()
175 |         self.Rs = Rs
176 |         self.Advs = Advs
177 | 
178 | 
179 | class MultiAgentOnPolicyBuffer(OnPolicyBuffer):
180 |     def __init__(self, gamma, alpha, distance_mask):
181 |         super().__init__(gamma, alpha, distance_mask)
182 | 
183 |     def sample_transition(self, R, dt=0):
184 |         if self.alpha < 0:
185 |             self._add_R_Adv(R)
186 |         else:
187 |             self._add_s_R_Adv(R)
188 |         obs = np.transpose(np.array(self.obs, dtype=np.float32), (1, 0, 2))
189 |         policies = np.transpose(np.array(self.adds, dtype=np.float32), (1, 0, 2))
190 |         acts = np.transpose(np.array(self.acts, dtype=np.int32))
191 |         Rs = np.array(self.Rs, dtype=np.float32)
192 |         Advs = np.array(self.Advs, dtype=np.float32)
193 |         dones = np.array(self.dones[:-1], dtype=np.bool)
194 |         self.reset(self.dones[-1])
195 |         return obs, policies, acts, dones, Rs, Advs
196 | 
197 |     def _add_R_Adv(self, R):
198 |         Rs = []
199 |         Advs = []
200 |         vs = np.array(self.vs)
201 |         for i in range(vs.shape[1]):
202 |             cur_Rs = []
203 |             cur_Advs = []
204 |             cur_R = R[i]
205 |             for r, v, done in zip(self.rs[::-1], vs[::-1, i], self.dones[:0:-1]):
206 |                 cur_R = r + self.gamma * cur_R * (1. - done)
207 |                 cur_Adv = cur_R - v
208 |                 cur_Rs.append(cur_R)
209 |                 cur_Advs.append(cur_Adv)
210 |             cur_Rs.reverse()
211 |             cur_Advs.reverse()
212 |             Rs.append(cur_Rs)
213 |             Advs.append(cur_Advs)
214 |         self.Rs = np.array(Rs)
215 |         self.Advs = np.array(Advs)
216 | 
217 |     def _add_st_R_Adv(self, R, dt):
218 |         Rs = []
219 |         Advs = []
220 |         vs = np.array(self.vs)
221 |         for i in range(vs.shape[1]):
222 |             cur_Rs = []
223 |             cur_Advs = []
224 |             cur_R = R[i]
225 |             tdiff = dt
226 |             distance_mask = self.distance_mask[i]
227 |             max_distance = self.max_distance[i]
228 |             for r, v, done in zip(self.rs[::-1], vs[::-1, i], self.dones[:0:-1]):
229 |                 cur_R = self.gamma * cur_R * (1. - done)
230 |                 if done:
231 |                     tdiff = 0
232 |                 # additional spatial rewards
233 |                 tmax = min(tdiff, max_distance)
234 |                 for t in range(tmax + 1):
235 |                     rt = np.sum(r[distance_mask == t])
236 |                     cur_R += (self.gamma * self.alpha) ** t * rt
237 |                 cur_Adv = cur_R - v
238 |                 tdiff += 1
239 |                 cur_Rs.append(cur_R)
240 |                 cur_Advs.append(cur_Adv)
241 |             cur_Rs.reverse()
242 |             cur_Advs.reverse()
243 |             Rs.append(cur_Rs)
244 |             Advs.append(cur_Advs)
245 |         self.Rs = np.array(Rs)
246 |         self.Advs = np.array(Advs)
247 | 
248 |     def _add_s_R_Adv(self, R):
249 |         Rs = []
250 |         Advs = []
251 |         vs = np.array(self.vs)
252 |         for i in range(vs.shape[1]):
253 |             cur_Rs = []
254 |             cur_Advs = []
255 |             cur_R = R[i]
256 |             distance_mask = self.distance_mask[i]
257 |             max_distance = self.max_distance[i]
258 |             for r, v, done in zip(self.rs[::-1], vs[::-1, i], self.dones[:0:-1]):
259 |                 cur_R = self.gamma * cur_R * (1. - done)
260 |                 # additional spatial rewards
261 |                 for t in range(max_distance + 1):
262 |                     rt = np.sum(r[distance_mask == t])
263 |                     cur_R += (self.alpha ** t) * rt
264 |                 cur_Adv = cur_R - v
265 |                 cur_Rs.append(cur_R)
266 |                 cur_Advs.append(cur_Adv)
267 |             cur_Rs.reverse()
268 |             cur_Advs.reverse()
269 |             Rs.append(cur_Rs)
270 |             Advs.append(cur_Advs)
271 |         self.Rs = np.array(Rs)
272 |         self.Advs = np.array(Advs)
273 | 
274 | 
275 | class Scheduler:
276 |     """
277 |     util functions
278 |     """
279 |     def __init__(self, val_init, val_min=0, total_step=0, decay='linear'):
280 |         self.val = val_init
281 |         self.N = float(total_step)
282 |         self.val_min = val_min
283 |         self.decay = decay
284 |         self.n = 0
285 | 
286 |     def get(self, n_step):
287 |         self.n += n_step
288 |         if self.decay == 'linear':
289 |             return max(self.val_min, self.val * (1 - self.n / self.N))
290 |         else:
291 |             return self.val
292 | 


--------------------------------------------------------------------------------
/configs/parameters_der20.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import math
  3 | 
  4 | DER_num = 20
  5 | lines_num = 20
  6 | loads_num = 10
  7 | sampling_time = 0.05
  8 | state_dim = 9
  9 | 
 10 | pi = math.pi
 11 | # --------Parameter values--------------
 12 | Lc = .35e-3
 13 | rLc = .03
 14 | wc = 31.41
 15 | 
 16 | rN = 1e4
 17 | wref = 2 * pi * 60
 18 | Vnom = 480.0
 19 | 
 20 | mp1 = 9.4e-5
 21 | mp2 = mp1
 22 | mp6 = 12.5e-5
 23 | mp7 = mp6
 24 | mp3 = 9.4e-5
 25 | mp4 = mp3
 26 | mp8 = 12.5e-5
 27 | mp9 = mp8
 28 | mp5 = 9.4e-5
 29 | mp11 = mp5
 30 | mp10 = 12.5e-5
 31 | mp16 = mp10
 32 | mp12 = 9.4e-5
 33 | mp13 = mp12
 34 | mp17 = 12.5e-5
 35 | mp18 = mp17
 36 | mp14 = 9.4e-5
 37 | mp15 = mp14
 38 | mp19 = 12.5e-5
 39 | mp20 = mp19
 40 | 
 41 | nq1 = 1.3e-3
 42 | nq2 = nq1
 43 | nq6 = 1.5e-3
 44 | nq7 = nq6
 45 | nq3 = 1.3e-3
 46 | nq4 = nq3
 47 | nq8 = 1.5e-3
 48 | nq9 = nq8
 49 | nq5 = 1.3e-3
 50 | nq11 = nq5
 51 | nq10 = 1.5e-3
 52 | nq16 = nq10
 53 | nq12 = 1.3e-3
 54 | nq13 = nq12
 55 | nq17 = 1.5e-3
 56 | nq18 = nq17
 57 | nq14 = 1.3e-3
 58 | nq15 = nq14
 59 | nq19 = 1.5e-3
 60 | nq20 = nq19
 61 | 
 62 | # ---------------
 63 | kp = 4
 64 | ki = 60
 65 | 
 66 | # ----Network Data-------
 67 | rline1 = .23
 68 | Lline1 = .1 / (2 * pi * 60)
 69 | rline2 = .35
 70 | Lline2 = .58 / (2 * pi * 60)
 71 | rline3 = .23
 72 | Lline3 = .1 / (2 * pi * 60)
 73 | rline4 = .23
 74 | Lline4 = .1 / (2 * pi * 60)
 75 | rline5 = .35
 76 | Lline5 = .58 / (2 * pi * 60)
 77 | rline6 = .23
 78 | Lline6 = .1 / (2 * pi * 60)
 79 | rline7 = .23
 80 | Lline7 = .1 / (2 * pi * 60)
 81 | rline8 = .35
 82 | Lline8 = .58 / (2 * pi * 60)
 83 | rline9 = .23
 84 | Lline9 = .1 / (2 * pi * 60)
 85 | rline10 = .23
 86 | Lline10 = .1 / (2 * pi * 60)
 87 | rline11 = .35
 88 | Lline11 = .58 / (2 * pi * 60)
 89 | rline12 = .23
 90 | Lline12 = .1 / (2 * pi * 60)
 91 | rline13 = .23
 92 | Lline13 = .1 / (2 * pi * 60)
 93 | rline14 = .35
 94 | Lline14 = .58 / (2 * pi * 60)
 95 | rline15 = .23
 96 | Lline15 = .1 / (2 * pi * 60)
 97 | rline16 = .23
 98 | Lline16 = .1 / (2 * pi * 60)
 99 | rline17 = .35
100 | Lline17 = .58 / (2 * pi * 60)
101 | rline18 = .23
102 | Lline18 = .1 / (2 * pi * 60)
103 | rline19 = .23
104 | Lline19 = .1 / (2 * pi * 60)
105 | rline20 = .35
106 | Lline20 = .58 / (2 * pi * 60)
107 | 
108 | QF = 0.5
109 | Rload1 = 2
110 | Lload1 = QF * 2 / (2 * pi * 60)
111 | Rload2 = 2
112 | Lload2 = QF * 1 / (2 * pi * 60)
113 | Rload3 = 2
114 | Lload3 = QF * 2 / (2 * pi * 60)
115 | Rload4 = 2
116 | Lload4 = QF * 1 / (2 * pi * 60)
117 | Rload5 = 2
118 | Lload5 = QF * 2 / (2 * pi * 60)
119 | Rload6 = 2
120 | Lload6 = QF * 1 / (2 * pi * 60)
121 | Rload7 = 2
122 | Lload7 = QF * 2 / (2 * pi * 60)
123 | Rload8 = 2
124 | Lload8 = QF * 1 / (2 * pi * 60)
125 | Rload9 = 2
126 | Lload9 = QF * 2 / (2 * pi * 60)
127 | Rload10 = 2
128 | Lload10 = QF * 1 / (2 * pi * 60)
129 | 
130 | # ------------Controller Parameters------------------------
131 | a_ctrl = 400
132 | 
133 | AP = 60 * np.array([[.0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
134 |                     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
135 |                     [0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
136 |                     [0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
137 |                     [0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
138 |                     [0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
139 |                     [0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
140 |                     [0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
141 |                     [0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
142 |                     [0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
143 |                     [0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
144 |                     [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0],
145 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1],
146 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0],
147 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0],
148 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0],
149 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0],
150 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0],
151 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1],
152 |                     [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0]])
153 | 
154 | D = np.array([[sum(AP[0]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
155 |               [0, sum(AP[1]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
156 |               [0, 0, sum(AP[2]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
157 |               [0, 0, 0, sum(AP[3]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
158 |               [0, 0, 0, 0, sum(AP[4]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
159 |               [0, 0, 0, 0, 0, sum(AP[5]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
160 |               [0, 0, 0, 0, 0, 0, sum(AP[6]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
161 |               [0, 0, 0, 0, 0, 0, 0, sum(AP[7]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
162 |               [0, 0, 0, 0, 0, 0, 0, 0, sum(AP[8]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
163 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[9]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
164 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[10]), 0, 0, 0, 0, 0, 0, 0, 0, 0],
165 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[11]), 0, 0, 0, 0, 0, 0, 0, 0],
166 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[12]), 0, 0, 0, 0, 0, 0, 0],
167 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[13]), 0, 0, 0, 0, 0, 0],
168 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[14]), 0, 0, 0, 0, 0],
169 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[15]), 0, 0, 0, 0],
170 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[16]), 0, 0, 0],
171 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[17]), 0, 0],
172 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[18]), 0],
173 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[19])]])
174 | 
175 | L = D - AP
176 | 
177 | # Pinning gain to the reference frequency
178 | G = np.array([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
179 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
180 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
181 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
182 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
183 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
184 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
185 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
186 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
187 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
188 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
189 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
190 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
191 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
192 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
193 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
194 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
195 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
196 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
197 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
198 | 
199 | # Fig 17
200 | Physical_Graph = np.array([[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 1
201 |                            [0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 2
202 |                            [0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 3
203 |                            [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 4
204 |                            [0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 5
205 |                            [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 6
206 |                            [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 7
207 |                            [0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 8
208 |                            [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 9
209 |                            [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],  # 10
210 |                            [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],  # 11
211 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0],  # 12
212 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0],  # 13
213 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0],  # 14
214 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1],  # 15
215 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],  # 16
216 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0],  # 17
217 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0],  # 18
218 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0],  # 19
219 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]])  # 20
220 | 
221 | Distance_Mask = np.array([[0, 3, 4, 7, 8, 1, 2, 5, 6, 9, 6, 9, 10, 13, 14, 7, 8, 11, 12, 15],  # 1
222 |                           [3, 0, 1, 4, 5, 2, 1, 2, 3, 6, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13],  # 2
223 |                           [4, 1, 0, 3, 4, 3, 2, 1, 2, 5, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11],  # 3
224 |                           [7, 4, 3, 0, 1, 6, 5, 2, 1, 2, 3, 6, 3, 6, 7, 4, 5, 4, 5, 8],  # 4
225 |                           [8, 5, 4, 1, 0, 7, 6, 3, 2, 1, 4, 3, 2, 5, 6, 5, 4, 3, 4, 7],  # 5
226 |                           [1, 2, 3, 6, 7, 0, 1, 4, 5, 8, 5, 8, 9, 12, 13, 6, 7, 10, 11, 14],  # 6
227 |                           [2, 1, 2, 5, 6, 1, 0, 3, 4, 7, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13],  # 7
228 |                           [5, 2, 1, 2, 3, 4, 3, 0, 1, 4, 1, 4, 5, 8, 9, 2, 3, 5, 6, 9],  # 8
229 |                           [6, 3, 2, 1, 2, 5, 4, 1, 0, 3, 2, 5, 4, 7, 8, 3, 4, 5, 6, 9],  # 9
230 |                           [9, 6, 5, 2, 1, 8, 7, 4, 3, 0, 5, 2, 1, 4, 5, 4, 3, 2, 3, 6],  # 10
231 |                           [6, 3, 2, 3, 4, 5, 4, 1, 2, 5, 0, 3, 4, 7, 8, 1, 2, 5, 6, 9],  # 11
232 |                           [9, 6, 5, 4, 3, 8, 7, 4, 5, 2, 3, 0, 1, 4, 5, 2, 1, 2, 3, 6],  # 12
233 |                           [10, 7, 6, 3, 2, 9, 8, 5, 4, 1, 4, 1, 0, 3, 4, 3, 2, 1, 2, 5],  # 13
234 |                           [13, 10, 9, 6, 5, 12, 11, 8, 7, 4, 7, 4, 3, 0, 1, 6, 5, 2, 1, 2],  # 14
235 |                           [14, 11, 10, 7, 6, 13, 12, 9, 8, 5, 8, 5, 4, 1, 0, 7, 6, 3, 2, 1],  # 15
236 |                           [7, 4, 3, 4, 5, 6, 5, 2, 3, 6, 1, 2, 3, 6, 7, 0, 1, 4, 5, 8],  # 16
237 |                           [8, 5, 4, 5, 4, 7, 6, 3, 4, 3, 2, 1, 2, 5, 6, 1, 0, 3, 4, 7],  # 17
238 |                           [11, 8, 7, 4, 3, 10, 9, 6, 5, 2, 5, 2, 1, 2, 3, 4, 3, 0, 1, 4],  # 18
239 |                           [12, 9, 8, 5, 4, 11, 10, 7, 6, 3, 6, 3, 2, 1, 2, 5, 4, 1, 0, 3],  # 19
240 |                           [15, 12, 11, 8, 7, 14, 13, 10, 9, 6, 9, 6, 5, 2, 1, 8, 7, 4, 3, 0]])  # 20
241 | 
242 | # add 0, to meet the difference between MATLAB and python, shape=(201,)
243 | x0 = np.array([0, 0, 0, 0, 0, 0,  # 1
244 |                0, 0, 0, 0, 0,  # 2
245 |                0, 0, 0, 0, 0,  # 3
246 |                0, 0, 0, 0, 0,  # 4
247 |                0, 0, 0, 0, 0,  # 5
248 |                0, 0, 0, 0, 0,  # 6
249 |                0, 0, 0, 0, 0,  # 7
250 |                0, 0, 0, 0, 0,  # 8
251 |                0, 0, 0, 0, 0,  # 9
252 |                0, 0, 0, 0, 0,  # 10
253 |                0, 0, 0, 0, 0,  # 11
254 |                0, 0, 0, 0, 0,  # 12
255 |                0, 0, 0, 0, 0,  # 13
256 |                0, 0, 0, 0, 0,  # 14
257 |                0, 0, 0, 0, 0,  # 15
258 |                0, 0, 0, 0, 0,  # 16
259 |                0, 0, 0, 0, 0,  # 17
260 |                0, 0, 0, 0, 0,  # 18
261 |                0, 0, 0, 0, 0,  # 19
262 |                0, 0, 0, 0, 0,  # 20
263 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
264 |                0, 0, 0,
265 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
266 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
267 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
268 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
269 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
270 |                Vnom, Vnom, Vnom,
271 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
272 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
273 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ])
274 | 
275 | # add 0, to meet the difference between MATLAB and python, shape=(201,)
276 | x_critic = np.array([0, 0, 0, 0, 0, 0,  # 1
277 |                      0, 0, 0, 0, 0,  # 2
278 |                      0, 0, 0, 0, 0,  # 3
279 |                      0, 0, 0, 0, 0,  # 4
280 |                      0, 0, 0, 0, 0,  # 5
281 |                      0, 0, 0, 0, 0,  # 6
282 |                      0, 0, 0, 0, 0,  # 7
283 |                      0, 0, 0, 0, 0,  # 8
284 |                      0, 0, 0, 0, 0,  # 9
285 |                      0, 0, 0, 0, 0,  # 10
286 |                      0, 0, 0, 0, 0,  # 11
287 |                      0, 0, 0, 0, 0,  # 12
288 |                      0, 0, 0, 0, 0,  # 13
289 |                      0, 0, 0, 0, 0,  # 14
290 |                      0, 0, 0, 0, 0,  # 15
291 |                      0, 0, 0, 0, 0,  # 16
292 |                      0, 0, 0, 0, 0,  # 17
293 |                      0, 0, 0, 0, 0,  # 18
294 |                      0, 0, 0, 0, 0,  # 19
295 |                      0, 0, 0, 0, 0,  # 20
296 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
297 |                      0, 0, 0, 0, 0,
298 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
299 |                      2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
300 |                      2 * pi * 60,
301 |                      2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
302 |                      2 * pi * 60,
303 |                      2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
304 |                      Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
305 |                      Vnom, Vnom,
306 |                      Vnom,
307 |                      Vnom, 0])
308 | 
309 | mp = np.array(
310 |     [mp1, mp2, mp3, mp4, mp5, mp6, mp7, mp8, mp9, mp10, mp11, mp12, mp13, mp14, mp15, mp16, mp17, mp18, mp19, mp20])
311 | nq = np.array(
312 |     [nq1, nq2, nq3, nq4, nq5, nq6, nq7, nq8, nq9, nq10, nq11, nq12, nq13, nq14, nq15, nq16, nq17, nq18, nq19, nq20])
313 | 
314 | obs_mean = np.array([[0.00000000e+00, 6.29283743e+04, 3.07978839e+04, 1.32348191e+02,
315 |                       -6.47528904e+01, 1.03796796e+02, -5.21592059e+01, 3.68194779e+02,
316 |                       -1.20152670e+01],
317 |                      [-4.93041786e-02, 6.30173923e+04, 1.68234121e+04, 1.27650177e+02,
318 |                       -3.41183888e+01, 9.99311072e+01, -3.05826380e+01, 3.84754303e+02,
319 |                       -2.75800537e+01],
320 |                      [-4.83026940e-02, 6.30138967e+04, 2.64078047e+04, 1.30929256e+02,
321 |                       -5.48771644e+01, 1.01772581e+02, -4.67882379e+01, 3.73359007e+02,
322 |                       -2.59512455e+01],
323 |                      [-8.82804061e-02, 6.31662493e+04, 1.48097410e+04, 1.27243570e+02,
324 |                       -3.00282412e+01, 9.88151381e+01, -2.75418624e+01, 3.86580620e+02,
325 |                       -3.58185231e+01],
326 |                      [-1.53396305e-01, 6.32074782e+04, 2.45682959e+04, 1.30650136e+02,
327 |                       -5.09303028e+01, 9.88664977e+01, -5.14727440e+01, 3.70418223e+02,
328 |                       -5.65501408e+01],
329 |                      [-1.65011027e-02, 4.73166932e+04, 1.84625127e+04, 9.69607871e+01,
330 |                       -3.78968366e+01, 7.58555115e+01, -3.08587258e+01, 3.81494347e+02,
331 |                       -1.44883493e+01],
332 |                      [-7.03528928e-02, 4.73602159e+04, 2.49348027e+04, 9.90546714e+01,
333 |                       -5.22085498e+01, 7.60480094e+01, -4.77392993e+01, 3.70103481e+02,
334 |                       -3.20859402e+01],
335 |                      [-7.80410413e-02, 4.74392625e+04, 1.68590902e+04, 9.67741550e+01,
336 |                       -3.44178654e+01, 7.50861449e+01, -3.12799559e+01, 3.81936553e+02,
337 |                       -3.18280343e+01],
338 |                      [-9.81964209e-02, 4.74772788e+04, 2.48865169e+04, 9.92333620e+01,
339 |                       -5.21513496e+01, 7.53031111e+01, -4.64068288e+01, 3.70441241e+02,
340 |                       -3.64576923e+01],
341 |                      [-1.52710271e-01, 4.75412263e+04, 1.36380038e+04, 9.60072464e+01,
342 |                       -2.76367498e+01, 7.37299287e+01, -2.95498280e+01, 3.82774251e+02,
343 |                       -5.52793130e+01],
344 |                      [-6.63784012e-02, 6.30905432e+04, 2.42016821e+04, 1.30314870e+02,
345 |                       -5.00216150e+01, 1.01273178e+02, -4.52626412e+01, 3.74384415e+02,
346 |                       -3.10691635e+01],
347 |                      [1.31971170e-02, 6.30522068e+04, 1.06244599e+04, 1.25673273e+02,
348 |                       -2.12483097e+01, 9.79990848e+01, -1.29564891e+01, 3.93142820e+02,
349 |                       -9.59958079e+00],
350 |                      [9.89911093e-04, 6.30372815e+04, 2.22167508e+04, 1.29546967e+02,
351 |                       -4.57372659e+01, 1.00918663e+02, -3.41846513e+01, 3.78791916e+02,
352 |                       -9.59958079e+00],
353 |                      [1.29618049e-01, 6.27656182e+04, 1.44435942e+04, 1.26417018e+02,
354 |                       -2.92002087e+01, 9.71299612e+01, -1.42892440e+01, 3.84691938e+02,
355 |                       2.45889420e+01],
356 |                      [1.43425530e-01, 6.27289664e+04, 1.94882799e+04, 1.28056372e+02,
357 |                       -4.00042387e+01, 9.88248607e+01, -2.35289490e+01, 3.77023871e+02,
358 |                       2.75940875e+01],
359 |                      [-5.75664523e-02, 4.74206821e+04, 1.52145138e+04, 9.62322446e+01,
360 |                       -3.09076068e+01, 7.50619493e+01, -2.75737065e+01, 3.84408967e+02,
361 |                       -2.60070071e+01],
362 |                      [-4.62132310e-02, 4.74006002e+04, 2.24274566e+04, 9.83433097e+01,
363 |                       -4.65498907e+01, 7.62699283e+01, -4.00471076e+01, 3.74414440e+02,
364 |                       -2.20451640e+01],
365 |                      [3.50348076e-02, 4.73514947e+04, 9.46583650e+03, 9.44818202e+01,
366 |                       -1.89488035e+01, 7.35617397e+01, -1.25175327e+01, 3.92288555e+02,
367 |                       3.46725897e+00],
368 |                      [9.29765789e-02, 4.72303747e+04, 2.63115101e+04, 9.92533169e+01,
369 |                       -5.53249064e+01, 7.80638969e+01, -3.85288092e+01, 3.68578201e+02,
370 |                       1.52290973e+01],
371 |                      [2.44430771e-01, 4.71629073e+04, 1.22011703e+03, 9.19220885e+01,
372 |                       -2.49415110e+00, 6.66335088e+01, 1.07921775e+01, 3.94242595e+02,
373 |                       6.97643296e+01]])
374 | 
375 | obs_std = np.array([[1e-12, 4.20343356e+03, 4.85700984e+03, 8.49569114e+00,
376 |                      1.04435108e+01, 3.09905688e+01, 1.64983659e+01, 1.07705269e+02,
377 |                      3.62599666e+00],
378 |                     [3.53208644e-02, 4.07268993e+03, 3.18474039e+03, 7.27486594e+00,
379 |                      6.59024116e+00, 2.94062352e+01, 1.07673233e+01, 1.12420029e+02,
380 |                      1.43182321e+01],
381 |                     [4.50219271e-02, 4.07628579e+03, 3.84826542e+03, 7.78605599e+00,
382 |                      8.02164152e+00, 2.99804139e+01, 1.58040164e+01, 1.08851426e+02,
383 |                      1.57230310e+01],
384 |                     [1.08836450e-01, 4.19079661e+03, 5.57892569e+03, 8.56541981e+00,
385 |                      1.16624186e+01, 2.88013185e+01, 1.74233657e+01, 1.12298972e+02,
386 |                      3.36366445e+01],
387 |                     [1.37059517e-01, 4.33290558e+03, 6.12791477e+03, 9.14265967e+00,
388 |                      1.33285040e+01, 2.80931036e+01, 2.36151694e+01, 1.07532260e+02,
389 |                      4.17724974e+01],
390 |                     [7.36926786e-03, 3.19718414e+03, 2.92929707e+03, 5.97164442e+00,
391 |                      6.09308020e+00, 2.25476621e+01, 9.85390513e+00, 1.11499200e+02,
392 |                      5.05572909e+00],
393 |                     [2.78848864e-02, 3.11031174e+03, 3.97391116e+03, 5.79699603e+00,
394 |                      8.91482441e+00, 2.22632525e+01, 1.49408615e+01, 1.08441720e+02,
395 |                      1.35471693e+01],
396 |                     [8.25305955e-02, 3.04606208e+03, 2.92804158e+03, 5.75932221e+00,
397 |                      6.03682404e+00, 2.19137623e+01, 1.16520326e+01, 1.11262948e+02,
398 |                      2.60898841e+01],
399 |                     [9.79098227e-02, 3.09173758e+03, 5.23944524e+03, 6.58828741e+00,
400 |                      1.15435474e+01, 2.17736019e+01, 1.89274478e+01, 1.07758569e+02,
401 |                      2.97496664e+01],
402 |                     [1.37595975e-01, 3.22258710e+03, 4.08098380e+03, 6.47070128e+00,
403 |                      8.53185325e+00, 2.09757867e+01, 1.47363294e+01, 1.11029807e+02,
404 |                      4.25592585e+01],
405 |                     [8.21029285e-02, 4.00905106e+03, 3.80086720e+03, 7.79257515e+00,
406 |                      8.12332877e+00, 2.96808581e+01, 1.55003639e+01, 1.09240975e+02,
407 |                      2.52447521e+01],
408 |                     [1.02595739e-01, 4.03706055e+03, 2.49241366e+03, 7.33073790e+00,
409 |                      4.97502660e+00, 2.93381675e+01, 9.14063564e+00, 1.14486565e+02,
410 |                      2.99601396e+01],
411 |                     [1.07966938e-01, 4.05754667e+03, 4.11356167e+03, 8.05455891e+00,
412 |                      8.76190441e+00, 3.01553432e+01, 1.54748166e+01, 1.10186587e+02,
413 |                      2.99601396e+01],
414 |                     [1.38518044e-01, 4.22761865e+03, 3.50591976e+03, 6.62981281e+00,
415 |                      7.38719628e+00, 2.95937703e+01, 1.50819504e+01, 1.12070762e+02,
416 |                      4.20976493e+01],
417 |                     [1.44113015e-01, 4.28464751e+03, 6.48124316e+03, 6.51512713e+00,
418 |                      1.41503352e+01, 3.01614611e+01, 1.94874888e+01, 1.10038587e+02,
419 |                      4.35260379e+01],
420 |                     [8.27426818e-02, 3.01275981e+03, 2.29503144e+03, 5.45061519e+00,
421 |                      4.72076804e+00, 2.20351945e+01, 9.73276151e+00, 1.12169970e+02,
422 |                      2.48839111e+01],
423 |                     [8.48676710e-02, 3.01884033e+03, 2.93932933e+03, 5.51880851e+00,
424 |                      6.35914398e+00, 2.24510839e+01, 1.33921664e+01, 1.09320916e+02,
425 |                      2.41679942e+01],
426 |                     [1.09723468e-01, 3.06290265e+03, 2.10642429e+03, 5.12431400e+00,
427 |                      4.27448769e+00, 2.20158690e+01, 9.04678262e+00, 1.14585687e+02,
428 |                      3.18676060e+01],
429 |                     [1.30148700e-01, 3.14843530e+03, 3.05573700e+03, 5.70614986e+00,
430 |                      6.79232126e+00, 2.42166221e+01, 1.54501623e+01, 1.07152775e+02,
431 |                      3.70785000e+01],
432 |                     [1.47446439e-01, 3.23645184e+03, 3.54838371e+03, 4.82558091e+00,
433 |                      7.05528823e+00, 2.06150534e+01, 1.21554693e+01, 1.14626963e+02,
434 |                      5.10992820e+01]])
435 | 


--------------------------------------------------------------------------------
/configs/parameters_der4.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import math
  3 | 
  4 | pi = math.pi
  5 | 
  6 | Lc = .35e-3
  7 | rLc = .03
  8 | wc = 31.41
  9 | 
 10 | DER_num = 4
 11 | lines_num = 3
 12 | loads_num = 2
 13 | 
 14 | mp1 = 9.4 * 1e-5
 15 | nq1 = 1.3 * 1e-3
 16 | mp2 = mp1
 17 | nq2 = nq1
 18 | mp3 = 12.5 * 1e-5
 19 | nq3 = 1.5 * 1e-3
 20 | mp4 = mp3
 21 | nq4 = nq3
 22 | 
 23 | rN = 1e4
 24 | wref = 2 * pi * 60
 25 | Vnom = 380.0
 26 | 
 27 | # for critical bus control
 28 | kp = 4
 29 | ki = 40
 30 | 
 31 | # Network Data
 32 | rline1 = 0.23
 33 | Lline1 = 0.318 / (2 * pi * 60)
 34 | rline2 = 0.35
 35 | Lline2 = 1.847 / (2 * pi * 60)
 36 | rline3 = 0.23
 37 | Lline3 = 0.318 / (2 * pi * 60)
 38 | 
 39 | Rload1 = 2.5
 40 | Lload1 = 1 / (2 * pi * 60)
 41 | Rload2 = 3
 42 | Lload2 = 2 / (2 * pi * 60)
 43 | 
 44 | # Controller Parameters
 45 | a_ctrl = 40
 46 | 
 47 | AP = 4 * np.array([[.0, 0, 0, 0],
 48 |                    [1, 0, 0, 0],
 49 |                    [0, 1, 0, 0],
 50 |                    [0, 0, 1, 0],
 51 |                    ])
 52 | 
 53 | # Pinning gain to the reference frequency
 54 | G = np.array([[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]])
 55 | 
 56 | # matrix for bus and load connection
 57 | BUS_LOAD = np.array([1, 0, 1, 0])
 58 | 
 59 | # matrix for bus connection
 60 | BUSES = np.array([[0, 1, 0, 0],
 61 |                   [1, 0, 1, 0],
 62 |                   [0, 1, 0, 1],
 63 |                   [0, 0, 1, 0]])
 64 | 
 65 | Physical_Graph = np.array([[0, 1, 0, 0],
 66 |                            [1, 0, 1, 0],
 67 |                            [0, 1, 0, 1],
 68 |                            [0, 0, 1, 0]])
 69 | 
 70 | Distance_Mask = np.array([[0, 1, 2, 3],
 71 |                           [1, 0, 1, 2],
 72 |                           [2, 1, 0, 1],
 73 |                           [3, 2, 1, 0]])
 74 | 
 75 | DER_dic = np.array([[Rload1, Lload1, mp1, nq1],
 76 |                     [0, 0, mp2, nq2],
 77 |                     [Rload2, Lload2, mp3, nq3],
 78 |                     [0, 0, mp4, nq4]])
 79 | 
 80 | rline = np.array([rline1, rline2, rline3])
 81 | Lline = np.array([Lline1, Lline2, Lline3])
 82 | 
 83 | # add 0, to meet the difference between MATLAB and python, shape=(35,)
 84 | x0 = np.array([0, 0, 0, 0, 0, 0,
 85 |                0, 0, 0, 0, 0,
 86 |                0, 0, 0, 0, 0,
 87 |                0, 0, 0, 0, 0,
 88 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 89 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
 90 |                Vnom, Vnom, Vnom, Vnom])
 91 | 
 92 | mp = np.array([mp1, mp2, mp3, mp4])
 93 | nq = np.array([nq1, nq2, nq3, nq4])
 94 | 
 95 | obs_mean = np.array([[0.00000000e+00, -1.25472351e+03, -2.12519352e+03, 4.10819053e+00,
 96 |                       -8.73258639e-01, 7.06497949e-01, -4.92866602e-01, 3.72485264e+02,
 97 |                       -6.83244265e+00],
 98 |                      [-3.01979698e-03, 4.14551913e+02, -2.07692677e+03, -2.13223582e+00,
 99 |                       -1.34217855e+00, 0.00000000e+00, 0.00000000e+00, 3.89795909e+02,
100 |                       1.26289101e+01],
101 |                      [-6.87461608e-03, -1.07905479e+02, -1.55205693e+03, 8.49008251e+00,
102 |                       -4.62387385e-01, -3.16647345e-01, -1.19334490e+00, 3.81486416e+02,
103 |                       2.29906559e+01],
104 |                      [-8.41740074e-03, -1.03054684e+03, -8.92739050e+02, -3.68726862e+00,
105 |                       -9.41845252e-01, 0.00000000e+00, 0.00000000e+00, 3.95523327e+02,
106 |                       3.52166136e+01]])
107 | 
108 | obs_std = np.array([[1e-12, 9.03935509e+04, 4.07198509e+04, 3.09694785e+02,
109 |                      3.63693026e+01, 1.14880758e+01, 2.14060149e+01, 1.26208840e+01,
110 |                      1.04869236e+00],
111 |                     [4.90044782e-01, 1.06416791e+05, 2.64131909e+04, 3.33947461e+02,
112 |                      6.00689740e+01, 1e-12, 1e-12, 1.18473960e+01,
113 |                      6.53970631e+00],
114 |                     [3.60300508e-01, 1.23964177e+05, 2.30459256e+04, 3.85219327e+02,
115 |                      9.32747376e+01, 2.21537447e+01, 2.56743784e+01, 1.18200888e+01,
116 |                      1.39858282e+01],
117 |                     [3.52282040e-01, 1.14521148e+05, 2.90240489e+04, 2.66153315e+02,
118 |                      8.41119329e+01, 1e-12, 1e-12, 1.29708915e+01,
119 |                      1.47636637e+01]])
120 | 


--------------------------------------------------------------------------------
/configs/parameters_der40.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import math
  3 | 
  4 | DER_num = 40
  5 | lines_num = 42
  6 | loads_num = 20
  7 | sampling_time = 0.05
  8 | state_dim = 9
  9 | 
 10 | pi = math.pi
 11 | # --------Parameter values--------------
 12 | Lc = .35e-3
 13 | rLc = .03
 14 | wc = 31.41
 15 | 
 16 | rN = 1e4
 17 | wref = 2 * pi * 60
 18 | Vnom = 480.0
 19 | 
 20 | mp1 = 9.4e-5
 21 | mp2 = mp1
 22 | mp6 = 12.5e-5
 23 | mp7 = mp6
 24 | mp3 = 9.4e-5
 25 | mp4 = mp3
 26 | mp8 = 12.5e-5
 27 | mp9 = mp8
 28 | mp5 = 9.4e-5
 29 | mp11 = mp5
 30 | mp10 = 12.5e-5
 31 | mp16 = mp10
 32 | mp12 = 9.4e-5
 33 | mp13 = mp12
 34 | mp17 = 12.5e-5
 35 | mp18 = mp17
 36 | mp14 = 9.4e-5
 37 | mp15 = mp14
 38 | mp19 = 12.5e-5
 39 | mp20 = mp19
 40 | 
 41 | mp21 = 9.4e-5
 42 | mp22 = mp21
 43 | mp26 = 12.5e-5
 44 | mp27 = mp26
 45 | mp23 = 9.4e-5
 46 | mp24 = mp23
 47 | mp28 = 12.5e-5
 48 | mp29 = mp28
 49 | mp25 = 9.4e-5
 50 | mp31 = mp25
 51 | mp30 = 12.5e-5
 52 | mp36 = mp30
 53 | mp32 = 9.4e-5
 54 | mp33 = mp32
 55 | mp37 = 12.5e-5
 56 | mp38 = mp37
 57 | mp34 = 9.4e-5
 58 | mp35 = mp34
 59 | mp39 = 12.5e-5
 60 | mp40 = mp39
 61 | 
 62 | nq1 = 1.3e-3
 63 | nq2 = nq1
 64 | nq6 = 1.5e-3
 65 | nq7 = nq6
 66 | nq3 = 1.3e-3
 67 | nq4 = nq3
 68 | nq8 = 1.5e-3
 69 | nq9 = nq8
 70 | nq5 = 1.3e-3
 71 | nq11 = nq5
 72 | nq10 = 1.5e-3
 73 | nq16 = nq10
 74 | nq12 = 1.3e-3
 75 | nq13 = nq12
 76 | nq17 = 1.5e-3
 77 | nq18 = nq17
 78 | nq14 = 1.3e-3
 79 | nq15 = nq14
 80 | nq19 = 1.5e-3
 81 | nq20 = nq19
 82 | 
 83 | nq21 = 1.3e-3
 84 | nq22 = nq21
 85 | nq26 = 1.5e-3
 86 | nq27 = nq26
 87 | nq23 = 1.3e-3
 88 | nq24 = nq23
 89 | nq28 = 1.5e-3
 90 | nq29 = nq28
 91 | nq25 = 1.3e-3
 92 | nq31 = nq25
 93 | nq30 = 1.5e-3
 94 | nq36 = nq30
 95 | nq32 = 1.3e-3
 96 | nq33 = nq32
 97 | nq37 = 1.5e-3
 98 | nq38 = nq37
 99 | nq34 = 1.3e-3
100 | nq35 = nq34
101 | nq39 = 1.5e-3
102 | nq40 = nq39
103 | 
104 | # ---------------
105 | kp = 4
106 | ki = 60
107 | 
108 | # ----Network Data-------
109 | rline1 = .23
110 | Lline1 = .1 / (2 * pi * 60)
111 | rline2 = .35
112 | Lline2 = .58 / (2 * pi * 60)
113 | rline3 = .23
114 | Lline3 = .1 / (2 * pi * 60)
115 | rline4 = .23
116 | Lline4 = .1 / (2 * pi * 60)
117 | rline5 = .35
118 | Lline5 = .58 / (2 * pi * 60)
119 | rline6 = .23
120 | Lline6 = .1 / (2 * pi * 60)
121 | rline7 = .23
122 | Lline7 = .1 / (2 * pi * 60)
123 | rline8 = .35
124 | Lline8 = .58 / (2 * pi * 60)
125 | rline9 = .23
126 | Lline9 = .1 / (2 * pi * 60)
127 | rline10 = .23
128 | Lline10 = .1 / (2 * pi * 60)
129 | rline11 = .35
130 | Lline11 = .58 / (2 * pi * 60)
131 | rline12 = .23
132 | Lline12 = .1 / (2 * pi * 60)
133 | rline13 = .23
134 | Lline13 = .1 / (2 * pi * 60)
135 | rline14 = .35
136 | Lline14 = .58 / (2 * pi * 60)
137 | rline15 = .23
138 | Lline15 = .1 / (2 * pi * 60)
139 | rline16 = .23
140 | Lline16 = .1 / (2 * pi * 60)
141 | rline17 = .35
142 | Lline17 = .58 / (2 * pi * 60)
143 | rline18 = .23
144 | Lline18 = .1 / (2 * pi * 60)
145 | rline19 = .23
146 | Lline19 = .1 / (2 * pi * 60)
147 | rline20 = .35
148 | Lline20 = .58 / (2 * pi * 60)
149 | 
150 | # ----Network Data-------
151 | rline21 = .23
152 | Lline21 = .1 / (2 * pi * 60)
153 | rline22 = .35
154 | Lline22 = .58 / (2 * pi * 60)
155 | rline23 = .23
156 | Lline23 = .1 / (2 * pi * 60)
157 | rline24 = .23
158 | Lline24 = .1 / (2 * pi * 60)
159 | rline25 = .35
160 | Lline25 = .58 / (2 * pi * 60)
161 | rline26 = .23
162 | Lline26 = .1 / (2 * pi * 60)
163 | rline27 = .23
164 | Lline27 = .1 / (2 * pi * 60)
165 | rline28 = .35
166 | Lline28 = .58 / (2 * pi * 60)
167 | rline29 = .23
168 | Lline29 = .1 / (2 * pi * 60)
169 | rline30 = .23
170 | Lline30 = .1 / (2 * pi * 60)
171 | rline31 = .35
172 | Lline31 = .58 / (2 * pi * 60)
173 | rline32 = .23
174 | Lline32 = .1 / (2 * pi * 60)
175 | rline33 = .23
176 | Lline33 = .1 / (2 * pi * 60)
177 | rline34 = .35
178 | Lline34 = .58 / (2 * pi * 60)
179 | rline35 = .23
180 | Lline35 = .1 / (2 * pi * 60)
181 | rline36 = .23
182 | Lline36 = .1 / (2 * pi * 60)
183 | rline37 = .35
184 | Lline37 = .58 / (2 * pi * 60)
185 | rline38 = .23
186 | Lline38 = .1 / (2 * pi * 60)
187 | rline39 = .23
188 | Lline39 = .1 / (2 * pi * 60)
189 | rline40 = .35
190 | Lline40 = .58 / (2 * pi * 60)
191 | 
192 | # vb 18 --> 21
193 | rline41 = .23
194 | Lline41 = .1 / (2 * pi * 60)
195 | # vb 20 --> 23
196 | rline42 = .35
197 | Lline42 = .58 / (2 * pi * 60)
198 | 
199 | QF = 0.5
200 | Rload1 = 2
201 | Lload1 = QF * 2 / (2 * pi * 60)
202 | Rload2 = 2
203 | Lload2 = QF * 1 / (2 * pi * 60)
204 | Rload3 = 2
205 | Lload3 = QF * 2 / (2 * pi * 60)
206 | Rload4 = 2
207 | Lload4 = QF * 1 / (2 * pi * 60)
208 | Rload5 = 2
209 | Lload5 = QF * 2 / (2 * pi * 60)
210 | Rload6 = 2
211 | Lload6 = QF * 1 / (2 * pi * 60)
212 | Rload7 = 2
213 | Lload7 = QF * 2 / (2 * pi * 60)
214 | Rload8 = 2
215 | Lload8 = QF * 1 / (2 * pi * 60)
216 | Rload9 = 2
217 | Lload9 = QF * 2 / (2 * pi * 60)
218 | Rload10 = 2
219 | Lload10 = QF * 1 / (2 * pi * 60)
220 | Rload11 = 2
221 | Lload11 = QF * 2 / (2 * pi * 60)
222 | Rload12 = 2
223 | Lload12 = QF * 1 / (2 * pi * 60)
224 | Rload13 = 2
225 | Lload13 = QF * 2 / (2 * pi * 60)
226 | Rload14 = 2
227 | Lload14 = QF * 1 / (2 * pi * 60)
228 | Rload15 = 2
229 | Lload15 = QF * 2 / (2 * pi * 60)
230 | Rload16 = 2
231 | Lload16 = QF * 1 / (2 * pi * 60)
232 | Rload17 = 2
233 | Lload17 = QF * 2 / (2 * pi * 60)
234 | Rload18 = 2
235 | Lload18 = QF * 1 / (2 * pi * 60)
236 | Rload19 = 2
237 | Lload19 = QF * 2 / (2 * pi * 60)
238 | Rload20 = 2
239 | Lload20 = QF * 1 / (2 * pi * 60)
240 | 
241 | # ------------Controller Parameters------------------------
242 | a_ctrl = 400
243 | # Physical Graph
244 | AP = 60 * np.array([[.0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
245 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 1
246 |                     [1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
247 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 2
248 |                     [0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
249 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 3
250 |                     [0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
251 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 4
252 |                     [0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
253 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 5
254 |                     [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 6
256 |                     [0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
257 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 7
258 |                     [0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
259 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 8
260 |                     [0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
261 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 9
262 |                     [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
263 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 10
264 |                     [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,
265 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 11
266 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0,
267 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 12
268 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0,
269 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 13
270 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0,
271 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 14
272 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1,
273 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 15
274 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0,
275 |                      1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 16
276 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0,
277 |                      0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 17
278 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0,
279 |                      1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 18
280 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1,
281 |                      0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 19
282 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0,
283 |                      0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 20
284 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
285 |                      0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 21
286 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
287 |                      1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 22
288 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1,
289 |                      0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 23
290 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
291 |                      0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 24
292 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
293 |                      0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 25
294 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
295 |                      1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 26
296 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
297 |                      0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 27
298 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
299 |                      0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 28
300 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
301 |                      0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 29
302 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
303 |                      0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],  # 30
304 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
305 |                      0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0],  # 31
306 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
307 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0],  # 32
308 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
309 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0],  # 33
310 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
311 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0],  # 34
312 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
313 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1],  # 35
314 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
315 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],  # 36
316 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
317 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0],  # 37
318 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
319 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0],  # 38
320 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
321 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1],  # 39
322 |                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
323 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0]  # 40
324 |                     ])
325 | 
326 | D = np.array([[sum(AP[0]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
327 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
328 |               [0, sum(AP[1]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
329 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
330 |               [0, 0, sum(AP[2]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
331 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
332 |               [0, 0, 0, sum(AP[3]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
333 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
334 |               [0, 0, 0, 0, sum(AP[4]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
335 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
336 |               [0, 0, 0, 0, 0, sum(AP[5]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
337 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
338 |               [0, 0, 0, 0, 0, 0, sum(AP[6]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
339 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
340 |               [0, 0, 0, 0, 0, 0, 0, sum(AP[7]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
341 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
342 |               [0, 0, 0, 0, 0, 0, 0, 0, sum(AP[8]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
343 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
344 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[9]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
345 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
346 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[10]), 0, 0, 0, 0, 0, 0, 0, 0, 0,
347 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
348 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[11]), 0, 0, 0, 0, 0, 0, 0, 0,
349 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
350 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[12]), 0, 0, 0, 0, 0, 0, 0,
351 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
352 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[13]), 0, 0, 0, 0, 0, 0,
353 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
354 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[14]), 0, 0, 0, 0, 0,
355 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
356 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[15]), 0, 0, 0, 0,
357 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
358 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[16]), 0, 0, 0,
359 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
360 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[17]), 0, 0,
361 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
362 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[18]), 0,
363 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
364 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[19]),
365 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
366 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
367 |                sum(AP[20]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
368 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
369 |                0, sum(AP[21]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
370 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
371 |                0, 0, sum(AP[22]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
372 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
373 |                0, 0, 0, sum(AP[23]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
374 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
375 |                0, 0, 0, 0, sum(AP[24]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
376 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
377 |                0, 0, 0, 0, 0, sum(AP[25]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
378 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
379 |                0, 0, 0, 0, 0, 0, sum(AP[26]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
380 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
381 |                0, 0, 0, 0, 0, 0, 0, sum(AP[27]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
382 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
383 |                0, 0, 0, 0, 0, 0, 0, 0, sum(AP[28]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
384 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
385 |                0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[29]), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
386 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
387 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[30]), 0, 0, 0, 0, 0, 0, 0, 0, 0],
388 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
389 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[31]), 0, 0, 0, 0, 0, 0, 0, 0],
390 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
391 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[32]), 0, 0, 0, 0, 0, 0, 0],
392 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
393 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[33]), 0, 0, 0, 0, 0, 0],
394 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
395 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[34]), 0, 0, 0, 0, 0],
396 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
397 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[35]), 0, 0, 0, 0],
398 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
399 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[36]), 0, 0, 0],
400 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
401 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[37]), 0, 0],
402 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
403 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[38]), 0],
404 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
405 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, sum(AP[39])]
406 |               ])
407 | 
408 | L = D - AP
409 | 
410 | # Pinning gain to the reference frequency
411 | G = np.array([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
412 |                0, 0, 0, 0, 0],
413 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
414 |                0, 0, 0, 0, 0],
415 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
416 |                0, 0, 0, 0, 0],
417 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
418 |                0, 0, 0, 0, 0],
419 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
420 |                0, 0, 0, 0, 0],
421 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
422 |                0, 0, 0, 0, 0],
423 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
424 |                0, 0, 0, 0, 0],
425 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
426 |                0, 0, 0, 0, 0],
427 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
428 |                0, 0, 0, 0, 0],
429 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
430 |                0, 0, 0, 0, 0],
431 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
432 |                0, 0, 0, 0, 0],
433 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
434 |                0, 0, 0, 0, 0],
435 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
436 |                0, 0, 0, 0, 0],
437 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
438 |                0, 0, 0, 0, 0],
439 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
440 |                0, 0, 0, 0, 0],
441 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
442 |                0, 0, 0, 0, 0],
443 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
444 |                0, 0, 0, 0, 0],
445 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
446 |                0, 0, 0, 0, 0],
447 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
448 |                0, 0, 0, 0, 0],
449 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
450 |                0, 0, 0, 0, 0],
451 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
452 |                0, 0, 0, 0, 0],
453 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
454 |                0, 0, 0, 0, 0],
455 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
456 |                0, 0, 0, 0, 0],
457 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
458 |                0, 0, 0, 0, 0],
459 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
460 |                0, 0, 0, 0, 0],
461 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
462 |                0, 0, 0, 0, 0],
463 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
464 |                0, 0, 0, 0, 0],
465 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
466 |                0, 0, 0, 0, 0],
467 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
468 |                0, 0, 0, 0, 0],
469 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
470 |                0, 0, 0, 0, 0],
471 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
472 |                0, 0, 0, 0, 0],
473 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
474 |                0, 0, 0, 0, 0],
475 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
476 |                0, 0, 0, 0, 0],
477 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
478 |                0, 0, 0, 0, 0],
479 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
480 |                0, 0, 0, 0, 0],
481 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
482 |                0, 0, 0, 0, 0],
483 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
484 |                0, 0, 0, 0, 0],
485 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
486 |                0, 0, 0, 0, 0],
487 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
488 |                0, 0, 0, 0, 0],
489 |               [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
490 |                0, 0, 0, 0, 0]
491 |               ])
492 | 
493 | Physical_Graph = np.array([[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
494 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 1
495 |                            [0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
496 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 2
497 |                            [0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
498 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 3
499 |                            [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
500 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 4
501 |                            [0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
502 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 5
503 |                            [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
504 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 6
505 |                            [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
506 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 7
507 |                            [0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
508 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 8
509 |                            [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
510 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 9
511 |                            [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
512 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 10
513 |                            [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
514 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 11
515 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
516 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 12
517 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0,
518 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 13
519 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0,
520 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 14
521 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1,
522 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 15
523 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0,
524 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 16
525 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,
526 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 17
527 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0,
528 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 18
529 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0,
530 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 19
531 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
532 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 20
533 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
534 |                             0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 21
535 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
536 |                             0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 22
537 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
538 |                             0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 23
539 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
540 |                             0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 24
541 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
542 |                             0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 25
543 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
544 |                             1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 26
545 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
546 |                             0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 27
547 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
548 |                             0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 28
549 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
550 |                             0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # 29
551 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
552 |                             0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],  # 30
553 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
554 |                             0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],  # 31
555 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
556 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0],  # 32
557 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
558 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0],  # 33
559 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
560 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0],  # 34
561 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
562 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1],  # 35
563 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
564 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0],  # 36
565 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
566 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0],  # 37
567 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
568 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0],  # 38
569 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
570 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0],  # 39
571 |                            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
572 |                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]  # 40
573 |                            ])
574 | 
575 | Distance_Mask = np.array([[0, 3, 4, 7, 8, 1, 2, 5, 6, 9, 6, 9, 10, 13, 14, 7, 8, 11, 12, 15,
576 |                            12, 15, 16, 19, 20, 13, 14, 17, 18, 21, 18, 20, 20, 20, 20, 20, 20, 20, 20, 20],  # 1
577 |                           [3, 0, 1, 4, 5, 2, 1, 2, 3, 6, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13,
578 |                            10, 13, 14, 17, 18, 11, 12, 15, 16, 19, 16, 18, 20, 20, 20, 20, 20, 20, 20, 20],  # 2
579 |                           [4, 1, 0, 3, 4, 3, 2, 1, 2, 5, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11,
580 |                            9, 12, 13, 16, 17, 10, 11, 14, 15, 18, 15, 17, 19, 20, 20, 20, 20, 20, 20, 20],  # 3
581 |                           [7, 4, 3, 0, 1, 6, 5, 2, 1, 2, 3, 6, 3, 6, 7, 4, 5, 4, 5, 8,
582 |                            5, 8, 9, 12, 13, 6, 7, 10, 11, 14, 11, 14, 15, 18, 19, 12, 13, 16, 17, 20],  # 4
583 |                           [8, 5, 4, 1, 0, 7, 6, 3, 2, 1, 4, 3, 2, 5, 6, 5, 4, 3, 4, 7,
584 |                            4, 7, 8, 11, 12, 5, 6, 9, 10, 13, 10, 13, 14, 17, 18, 11, 12, 15, 16, 19],  # 5
585 |                           [1, 2, 3, 6, 7, 0, 1, 4, 5, 8, 5, 8, 9, 12, 13, 6, 7, 10, 11, 14,
586 |                            11, 14, 15, 18, 19, 12, 13, 16, 17, 20, 17, 19, 20, 20, 20, 20, 20, 20, 20, 20],  # 6
587 |                           [2, 1, 2, 5, 6, 1, 0, 3, 4, 7, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13,
588 |                            10, 13, 14, 17, 18, 11, 12, 15, 16, 19, 16, 18, 19, 19, 20, 20, 20, 20, 20, 20],  # 7
589 |                           [5, 2, 1, 2, 3, 4, 3, 0, 1, 4, 1, 4, 5, 8, 9, 2, 3, 5, 6, 9,
590 |                            8, 11, 12, 15, 16, 9, 10, 13, 14, 17, 14, 16, 18, 19, 20, 20, 20, 20, 20, 20],  # 8
591 |                           [6, 3, 2, 1, 2, 5, 4, 1, 0, 3, 2, 5, 4, 7, 8, 3, 4, 5, 6, 9,
592 |                            9, 12, 13, 16, 17, 10, 11, 14, 15, 18, 15, 17, 19, 20, 20, 20, 20, 20, 20, 20],  # 9
593 |                           [9, 6, 5, 2, 1, 8, 7, 4, 3, 0, 5, 2, 1, 4, 5, 4, 3, 2, 3, 6,
594 |                            3, 6, 7, 10, 11, 4, 5, 8, 9, 12, 9, 12, 13, 16, 17, 10, 11, 14, 15, 18],  # 10
595 |                           [6, 3, 2, 3, 4, 5, 4, 1, 2, 5, 0, 3, 4, 7, 8, 1, 2, 5, 6, 9,
596 |                            6, 9, 10, 13, 14, 7, 8, 11, 12, 15, 12, 14, 16, 17, 18, 20, 20, 20, 20, 20],  # 11
597 |                           [9, 6, 5, 4, 3, 8, 7, 4, 5, 2, 3, 0, 1, 4, 5, 2, 1, 2, 3, 6,
598 |                            3, 6, 7, 10, 11, 4, 5, 8, 9, 12, 9, 12, 13, 16, 17, 13, 14, 17, 18, 21],  # 12
599 |                           [10, 7, 6, 3, 2, 9, 8, 5, 4, 1, 4, 1, 0, 3, 4, 3, 2, 1, 2, 5,
600 |                            2, 5, 6, 9, 10, 3, 4, 7, 8, 10, 8, 11, 12, 15, 16, 12, 13, 16, 17, 20],  # 13
601 |                           [13, 10, 9, 6, 5, 12, 11, 8, 7, 4, 7, 4, 3, 0, 1, 6, 5, 2, 1, 2,
602 |                            3, 4, 3, 6, 7, 4, 5, 4, 5, 8, 5, 9, 12, 13, 6, 7, 10, 11, 12, 15],  # 14
603 |                           [14, 11, 10, 7, 6, 13, 12, 9, 8, 5, 8, 5, 4, 1, 0, 7, 6, 3, 2, 1,
604 |                            4, 3, 2, 5, 6, 5, 4, 3, 4, 7, 4, 7, 6, 9, 10, 5, 6, 8, 9, 13],  # 15
605 |                           [7, 4, 3, 4, 5, 6, 5, 2, 3, 6, 1, 2, 3, 6, 7, 0, 1, 4, 5, 8,
606 |                            5, 8, 9, 12, 13, 6, 7, 10, 11, 14, 11, 14, 15, 18, 19, 12, 13, 16, 17, 20],  # 16
607 |                           [8, 5, 4, 5, 4, 7, 6, 3, 4, 3, 2, 1, 2, 5, 6, 1, 0, 3, 4, 7,
608 |                            4, 7, 8, 11, 12, 5, 6, 9, 10, 13, 10, 13, 14, 17, 18, 11, 12, 15, 16, 19],  # 17
609 |                           [11, 8, 7, 4, 3, 10, 9, 6, 5, 2, 5, 2, 1, 2, 3, 4, 3, 0, 1, 4,
610 |                            1, 4, 5, 8, 9, 2, 3, 6, 7, 10, 7, 10, 11, 14, 15, 8, 9, 12, 13, 16],  # 18
611 |                           [12, 9, 8, 5, 4, 11, 10, 7, 6, 3, 6, 3, 2, 1, 2, 5, 4, 1, 0, 3,
612 |                            2, 5, 4, 7, 8, 3, 4, 7, 8, 11, 8, 11, 12, 15, 16, 9, 10, 13, 14, 17],  # 19
613 |                           [15, 12, 11, 8, 7, 14, 13, 10, 9, 6, 9, 6, 5, 2, 1, 8, 7, 4, 3, 0,
614 |                            5, 2, 1, 4, 5, 4, 3, 2, 3, 6, 3, 6, 7, 10, 11, 4, 5, 8, 9, 12],  # 20
615 |                           [12, 10, 9, 5, 4, 11, 10, 8, 9, 3, 6, 3, 2, 3, 4, 5, 4, 1, 2, 5,
616 |                            0, 3, 4, 7, 8, 1, 2, 5, 6, 9, 6, 9, 10, 13, 14, 7, 8, 11, 12, 15],  # 21
617 |                           [15, 13, 12, 8, 7, 14, 13, 11, 12, 6, 9, 6, 5, 4, 3, 8, 7, 4, 5, 2,
618 |                            3, 0, 1, 4, 5, 2, 1, 2, 3, 6, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13],  # 22
619 |                           [16, 14, 13, 9, 8, 15, 14, 12, 13, 7, 10, 7, 6, 3, 2, 9, 8, 5, 4, 1,
620 |                            4, 1, 0, 3, 4, 3, 2, 1, 2, 5, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11],  # 23
621 |                           [19, 17, 16, 12, 11, 18, 17, 15, 16, 10, 13, 10, 9, 6, 5, 12, 11, 8, 7, 4,
622 |                            7, 4, 3, 0, 1, 6, 5, 2, 1, 2, 3, 6, 3, 6, 7, 4, 5, 4, 5, 8],  # 24
623 |                           [20, 18, 17, 13, 12, 19, 18, 16, 17, 11, 14, 11, 10, 7, 6, 13, 12, 9, 8, 5,
624 |                            8, 5, 4, 1, 0, 7, 6, 3, 2, 1, 4, 3, 2, 5, 6, 5, 4, 3, 4, 7],  # 25
625 |                           [13, 11, 10, 6, 5, 12, 11, 9, 10, 4, 7, 4, 3, 4, 5, 6, 5, 2, 3, 4,
626 |                            1, 2, 3, 6, 7, 0, 1, 4, 5, 8, 5, 8, 9, 12, 13, 6, 7, 10, 11, 14],  # 26
627 |                           [14, 12, 11, 7, 6, 13, 12, 10, 11, 5, 8, 5, 4, 5, 4, 7, 6, 3, 4, 5,
628 |                            2, 1, 2, 5, 6, 1, 0, 3, 4, 7, 4, 7, 8, 11, 12, 5, 6, 9, 10, 13],  # 27
629 |                           [17, 15, 14, 10, 9, 16, 15, 13, 14, 8, 11, 8, 7, 4, 3, 10, 9, 6, 7, 2,
630 |                            5, 2, 1, 2, 3, 4, 3, 0, 1, 4, 1, 4, 5, 8, 9, 2, 3, 5, 6, 9],  # 28
631 |                           [18, 16, 15, 11, 10, 17, 16, 14, 15, 9, 12, 9, 8, 5, 4, 11, 10, 7, 8, 3,
632 |                            6, 3, 2, 1, 2, 5, 4, 1, 0, 3, 2, 5, 4, 7, 8, 3, 4, 5, 6, 9],  # 29
633 |                           [21, 19, 18, 14, 13, 20, 19, 17, 18, 12, 15, 12, 10, 8, 7, 14, 13, 10, 11, 6,
634 |                            9, 6, 5, 2, 1, 8, 7, 4, 3, 0, 5, 2, 1, 4, 5, 4, 3, 2, 3, 6],  # 30
635 |                           [18, 16, 15, 11, 10, 20, 16, 14, 15, 9, 12, 9, 8, 5, 4, 11, 10, 7, 8, 3,
636 |                            6, 3, 2, 3, 4, 5, 4, 1, 2, 5, 0, 3, 4, 7, 8, 1, 2, 5, 6, 9],  # 31
637 |                           [20, 18, 17, 14, 13, 20, 18, 16, 17, 12, 14, 12, 11, 8, 7, 14, 13, 10, 11, 6,
638 |                            9, 6, 5, 4, 3, 8, 7, 4, 5, 2, 3, 0, 1, 4, 5, 2, 1, 2, 3, 6],  # 32
639 |                           [20, 20, 19, 15, 14, 20, 19, 18, 19, 13, 16, 13, 12, 9, 6, 15, 14, 11, 12, 7,
640 |                            10, 7, 6, 3, 2, 9, 8, 5, 4, 1, 4, 1, 0, 3, 4, 3, 2, 1, 2, 5],  # 33
641 |                           [20, 20, 20, 18, 17, 20, 19, 19, 20, 16, 17, 16, 15, 12, 9, 18, 17, 14, 15, 10,
642 |                            13, 10, 9, 6, 5, 12, 11, 8, 7, 4, 7, 4, 3, 0, 1, 6, 5, 2, 1, 2],  # 34
643 |                           [20, 20, 20, 19, 18, 20, 20, 20, 20, 17, 18, 17, 16, 13, 10, 19, 18, 15, 16, 11,
644 |                            14, 11, 10, 7, 6, 13, 12, 9, 8, 5, 8, 5, 4, 1, 0, 7, 6, 3, 2, 1],  # 35
645 |                           [20, 20, 20, 12, 11, 20, 20, 20, 20, 10, 20, 13, 12, 6, 5, 12, 11, 8, 9, 4,
646 |                            7, 4, 3, 4, 5, 6, 5, 2, 3, 6, 1, 2, 3, 6, 7, 0, 1, 4, 5, 8],  # 36
647 |                           [20, 20, 20, 13, 12, 20, 20, 20, 20, 11, 20, 14, 13, 7, 6, 13, 12, 9, 10, 5,
648 |                            8, 5, 4, 5, 4, 7, 6, 3, 4, 3, 2, 1, 2, 5, 6, 1, 0, 3, 4, 7],  # 37
649 |                           [20, 20, 20, 16, 15, 20, 20, 20, 20, 14, 20, 17, 16, 10, 8, 16, 15, 12, 13, 8,
650 |                            11, 8, 7, 4, 3, 10, 9, 6, 5, 2, 5, 2, 1, 2, 3, 4, 3, 0, 1, 4],  # 38
651 |                           [20, 20, 20, 17, 16, 20, 20, 20, 20, 15, 20, 18, 17, 11, 9, 17, 16, 13, 14, 9,
652 |                            12, 9, 8, 5, 4, 11, 10, 7, 6, 3, 6, 3, 2, 1, 2, 5, 4, 1, 0, 3],  # 39
653 |                           [20, 20, 20, 20, 19, 20, 20, 20, 20, 18, 20, 21, 20, 12, 13, 20, 19, 16, 17, 12,
654 |                            15, 12, 11, 8, 7, 14, 13, 10, 9, 6, 9, 6, 5, 2, 1, 8, 7, 4, 3, 0]  # 40
655 |                           ])
656 | 
657 | # add 0, to meet the difference between MATLAB and python, shape=(201,)
658 | x0 = np.array([0, 0, 0, 0, 0, 0,  # 1
659 |                0, 0, 0, 0, 0,  # 2
660 |                0, 0, 0, 0, 0,  # 3
661 |                0, 0, 0, 0, 0,  # 4
662 |                0, 0, 0, 0, 0,  # 5
663 |                0, 0, 0, 0, 0,  # 6
664 |                0, 0, 0, 0, 0,  # 7
665 |                0, 0, 0, 0, 0,  # 8
666 |                0, 0, 0, 0, 0,  # 9
667 |                0, 0, 0, 0, 0,  # 10
668 |                0, 0, 0, 0, 0,  # 11
669 |                0, 0, 0, 0, 0,  # 12
670 |                0, 0, 0, 0, 0,  # 13
671 |                0, 0, 0, 0, 0,  # 14
672 |                0, 0, 0, 0, 0,  # 15
673 |                0, 0, 0, 0, 0,  # 16
674 |                0, 0, 0, 0, 0,  # 17
675 |                0, 0, 0, 0, 0,  # 18
676 |                0, 0, 0, 0, 0,  # 19
677 |                0, 0, 0, 0, 0,  # 20
678 |                0, 0, 0, 0, 0,  # 21
679 |                0, 0, 0, 0, 0,  # 22
680 |                0, 0, 0, 0, 0,  # 23
681 |                0, 0, 0, 0, 0,  # 24
682 |                0, 0, 0, 0, 0,  # 25
683 |                0, 0, 0, 0, 0,  # 26
684 |                0, 0, 0, 0, 0,  # 27
685 |                0, 0, 0, 0, 0,  # 28
686 |                0, 0, 0, 0, 0,  # 29
687 |                0, 0, 0, 0, 0,  # 30
688 |                0, 0, 0, 0, 0,  # 31
689 |                0, 0, 0, 0, 0,  # 32
690 |                0, 0, 0, 0, 0,  # 33
691 |                0, 0, 0, 0, 0,  # 34
692 |                0, 0, 0, 0, 0,  # 35
693 |                0, 0, 0, 0, 0,  # 36
694 |                0, 0, 0, 0, 0,  # 37
695 |                0, 0, 0, 0, 0,  # 38
696 |                0, 0, 0, 0, 0,  # 39
697 |                0, 0, 0, 0, 0,  # 40
698 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
699 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
700 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
701 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
702 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
703 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
704 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
705 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
706 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
707 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
708 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
709 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
710 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
711 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
712 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
713 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
714 |                2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60, 2 * pi * 60,
715 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
716 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
717 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
718 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
719 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
720 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
721 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
722 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
723 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
724 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
725 | 
726 | mp = np.array(
727 |     [mp1, mp2, mp3, mp4, mp5, mp6, mp7, mp8, mp9, mp10, mp11, mp12, mp13, mp14, mp15, mp16, mp17, mp18, mp19, mp20,
728 |      mp21, mp22, mp23, mp24, mp25, mp26, mp27, mp28, mp29, mp30, mp31, mp32, mp33, mp34, mp35, mp36, mp37, mp38, mp39,
729 |      mp40])
730 | nq = np.array(
731 |     [nq1, nq2, nq3, nq4, nq5, nq6, nq7, nq8, nq9, nq10, nq11, nq12, nq13, nq14, nq15, nq16, nq17, nq18, nq19, nq20,
732 |      nq21, nq22, nq23, nq24, nq25, nq26, nq27, nq28, nq29, nq30, nq31, nq32, nq33, nq34, nq35, nq36, nq37, nq38, nq39,
733 |      nq40])
734 | 
735 | obs_mean = 0
736 | 
737 | obs_std = 1


--------------------------------------------------------------------------------
/configs/parameters_der6.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import math
  3 | 
  4 | DER_num = 6
  5 | lines_num = 10
  6 | loads_num = 4
  7 | sampling_time = 0.05
  8 | state_dim = 9
  9 | 
 10 | pi = math.pi
 11 | 
 12 | # --------Parameter values--------------
 13 | Lc = .35e-3
 14 | rLc = .03
 15 | wc = 31.41
 16 | mp1 = 0.3 * 18.8e-5
 17 | mp2 = mp1
 18 | mp3 = 0.3 * 25e-5
 19 | mp4 = mp3
 20 | mp5 = mp1
 21 | mp6 = mp1
 22 | nq1 = 0.2 * 2.6e-3
 23 | nq2 = nq1
 24 | nq3 = 0.2 * 3e-3
 25 | nq4 = nq3
 26 | nq5 = nq1
 27 | nq6 = nq1
 28 | 
 29 | # --------------
 30 | kp = 4
 31 | ki = 40
 32 | 
 33 | rN = 1e4
 34 | wb = 2 * pi * 60
 35 | wref = 2 * pi * 60
 36 | Vnom = 480
 37 | 
 38 | # -----Network Data-------
 39 | r300 = ((480 / 24900) ** 2) * 1.3368
 40 | x300 = ((480 / 24900) ** 2) * 1.3343
 41 | r301 = ((480 / 24900) ** 2) * 1.9300
 42 | x301 = ((480 / 24900) ** 2) * 1.4115
 43 | r302 = ((480 / 24900) ** 2) * 2.7995
 44 | x302 = ((480 / 24900) ** 2) * 1.4855
 45 | r303 = ((480 / 24900) ** 2) * 2.7995
 46 | x303 = ((480 / 24900) ** 2) * 1.4855
 47 | r304 = ((480 / 24900) ** 2) * 1.9217
 48 | x304 = ((480 / 24900) ** 2) * 1.4212
 49 | 
 50 | rline1 = 0.000189394 * ((37500 + 29730) * r300 + 320 * r301)
 51 | Lline1 = 0.000189394 * ((37500 + 29730) * x300 + 320 * x301) / wref
 52 | rline2 = 0.000189394 * (10210 * r301)
 53 | Lline2 = 0.000189394 * (10210 * x301) / wref
 54 | rline3 = 0.000189394 * (3030 * r303)
 55 | Lline3 = 0.000189394 * (3030 * x303) / wref
 56 | rline4 = 0.000189394 * ((840 + 20440 + 520) * r301)
 57 | Lline4 = 0.000189394 * ((840 + 20440 + 520) * x301) / wref
 58 | rline5 = 0.000189394 * (23330 * r303)
 59 | Lline5 = 0.000189394 * (23330 * x303) / wref
 60 | rline6 = 0.000189394 * ((36830 + 10) * r301)
 61 | Lline6 = 0.000189394 * ((36830 + 10) * x301) / wref
 62 | rline7 = 0.000189394 * (1620 * r303)
 63 | Lline7 = 0.000189394 * (1620 * x303) / wref
 64 | rline8 = 0.000189394 * (5830 * r301)
 65 | Lline8 = 0.000189394 * (5830 * x301) / wref
 66 | rline9 = 0.000189394 * ((280 + 1350 + 3640 + 530) * r301)
 67 | Lline9 = 0.000189394 * ((280 + 1350 + 3640 + 530) * x301) / wref
 68 | rline10 = 0.000189394 * ((2020 + 2680) * r301)
 69 | Lline10 = 0.000189394 * ((2020 + 2680) * x301) / wref
 70 | rline11 = 0.000189394 * (860 * r301)
 71 | Lline11 = 0.000189394 * (860 * x301) / wref
 72 | rline12 = 0.000189394 * (280 * r301 + 4860 * r304)
 73 | Lline12 = 0.000189394 * (280 * x301 + 4860 * x304) / wref
 74 | rline13 = 0.000189394 * ((1710 + 48150 + 13740) * r302)
 75 | Lline13 = 0.000189394 * ((1710 + 48150 + 13740) * x302) / wref
 76 | 
 77 | Rload1 = 1.5
 78 | Lload1 = 1 / wref
 79 | Rload2 = 0.5
 80 | Lload2 = 0.5 / wref
 81 | Rload3 = 1
 82 | Lload3 = 1 / wref
 83 | Rload4 = 0.8
 84 | Lload4 = 0.8 / wref
 85 | 
 86 | # Controller Parameters
 87 | R1 = pi / 2
 88 | R2 = pi / 50
 89 | Etha = 10
 90 | a_ctrl = 40
 91 | 
 92 | 
 93 | # Physical Graph
 94 | AP = 6 * np.array([[.0, 1, 1, 0, 0, 0],
 95 |                    [1, 0, 1, 1, 0, 0],
 96 |                    [1, 1, 0, 0, 1, 0],
 97 |                    [0, 1, 0, 0, 1, 1],
 98 |                    [0, 0, 0, 1, 0, 1],
 99 |                    [0, 0, 0, 1, 1, 0]])
100 | 
101 | Physical_Graph = np.array([[0, 1, 1, 0, 0, 0],  # 1
102 |                            [1, 0, 1, 1, 0, 0],  # 2
103 |                            [1, 1, 0, 0, 1, 0],  # 3
104 |                            [0, 1, 0, 0, 1, 1],  # 4
105 |                            [0, 0, 0, 1, 0, 1],  # 5
106 |                            [0, 0, 0, 1, 1, 0]])  # 6
107 | 
108 | Distance_Mask = np.array([[0, 1, 1, 2, 2, 3],  # 1
109 |                           [1, 0, 1, 1, 2, 2],  # 2
110 |                           [1, 1, 0, 2, 1, 2],  # 3
111 |                           [2, 1, 2, 0, 1, 1],  # 4
112 |                           [2, 2, 1, 1, 0, 1],  # 5
113 |                           [3, 2, 2, 1, 1, 0]])  # 6
114 | 
115 | D = np.array([[sum(AP[0]), 0, 0, 0, 0, 0],
116 |               [0, sum(AP[1]), 0, 0, 0, 0],
117 |               [0, 0, sum(AP[2]), 0, 0, 0],
118 |               [0, 0, 0, sum(AP[3]), 0, 0],
119 |               [0, 0, 0, 0, sum(AP[4]), 0],
120 |               [0, 0, 0, 0, 0, sum(AP[5])]])
121 | 
122 | L = D - AP
123 | 
124 | # Pinning gain to the reference frequency
125 | G = np.array([[1, 0, 0, 0, 0, 0],
126 |               [0, 0, 0, 0, 0, 0],
127 |               [0, 0, 0, 0, 0, 0],
128 |               [0, 0, 0, 0, 0, 0],
129 |               [0, 0, 0, 0, 0, 0],
130 |               [0, 0, 0, 0, 0, 0]])
131 | 
132 | # Initial Conditions
133 | x0 = np.array([0, 0, 0, 0, 0, 0,
134 |                0, 0, 0, 0, 0,
135 |                0, 0, 0, 0, 0,
136 |                0, 0, 0, 0, 0,
137 |                0, 0, 0, 0, 0,
138 |                0, 0, 0, 0, 0,
139 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
140 |                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
141 |                wref, wref, wref, wref, wref, wref,
142 |                Vnom, Vnom, Vnom, Vnom, Vnom, Vnom,
143 |                0, 0, 0, 0,
144 |                0, 0, 0, 0,
145 |                0, 0, 0, 0,
146 |                0, 0, 0, 0,
147 |                0, 0, 0, 0,
148 |                0, 0, 0, 0, ])
149 | 
150 | # Initial Conditions
151 | x_critic = np.array([0, 0, 0, 0, 0, 0,
152 |                      0, 0, 0, 0, 0,
153 |                      0, 0, 0, 0, 0,
154 |                      0, 0, 0, 0, 0,
155 |                      0, 0, 0, 0, 0,
156 |                      0, 0, 0, 0, 0,
157 |                      0, 0, 0, 0, 0,
158 |                      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
159 |                      wref, wref, wref, wref, wref, wref,
160 |                      Vnom, Vnom, Vnom, Vnom, Vnom, Vnom, 0,
161 |                      0, 0, 0, 0,
162 |                      0, 0, 0, 0,
163 |                      0, 0, 0, 0,
164 |                      0, 0, 0, 0,
165 |                      0, 0, 0, 0,
166 |                      0, 0, 0, 0])
167 | 
168 | mp = [mp1, mp2, mp3, mp4, mp5, mp6]
169 | nq = [nq1, nq2, nq3, nq4, nq5, nq6]
170 | 
171 | obs_mean = np.array([[0.00000000e+00, 5.94750957e+04, 5.18826718e+04, 1.59484495e+02,
172 |                       -1.39131987e+02, 1.58977121e+02, -1.37752942e+02, 3.50326969e+02,
173 |                       -1.66857296e+01],
174 |                      [-2.94964702e-03, 5.84485099e+04, 5.03222993e+04, 1.56643756e+02,
175 |                       -1.34910554e+02, 1.56337295e+02, -1.33966792e+02, 3.51290037e+02,
176 |                       -1.74565790e+01],
177 |                      [-1.56892081e-02, 4.39054908e+04, 5.11862523e+04, 1.19121086e+02,
178 |                       -1.38904635e+02, 1.17458840e+02, -1.40130448e+02, 3.47164367e+02,
179 |                       -1.68323952e+01],
180 |                      [-1.58780872e-02, 4.34048292e+04, 5.02168674e+04, 1.17707465e+02,
181 |                       -1.36224549e+02, 1.16293641e+02, -1.37484136e+02, 3.47843176e+02,
182 |                       -1.67953707e+01],
183 |                      [-2.72605340e-04, 5.74754480e+04, 5.96003551e+04, 1.56265833e+02,
184 |                       -1.62105917e+02, 1.56587639e+02, -1.61667359e+02, 3.42553468e+02,
185 |                       -1.56752104e+01],
186 |                      [-3.92235639e-04, 5.74755015e+04, 5.93884888e+04, 1.56220429e+02,
187 |                       -1.61483242e+02, 1.56530411e+02, -1.61057580e+02, 3.42740639e+02,
188 |                       -1.57280775e+01]])
189 | 
190 | obs_std = np.array([[1e-12, 3.86420056e+03, 4.16700891e+03, 1.05387470e+01,
191 |                      1.09296704e+01, 4.65795588e+01, 4.12278816e+01, 1.01437007e+02,
192 |                      4.95697466e+00],
193 |                     [1.16677122e-03, 4.02882556e+03, 4.24164329e+03, 1.01733946e+01,
194 |                      1.09124226e+01, 4.58190749e+01, 4.01796093e+01, 1.01715036e+02,
195 |                      5.17413404e+00],
196 |                     [1.98886559e-03, 3.04426481e+03, 4.03934214e+03, 7.77323806e+00,
197 |                      1.05744803e+01, 3.43026169e+01, 4.16269469e+01, 1.00521725e+02,
198 |                      5.02423347e+00],
199 |                     [2.31861878e-03, 3.22645256e+03, 4.06658900e+03, 7.98839539e+00,
200 |                      1.04757837e+01, 3.39905023e+01, 4.08615239e+01, 1.00718850e+02,
201 |                      5.00494945e+00],
202 |                     [2.81557239e-03, 4.38046968e+03, 4.75347347e+03, 1.08754296e+01,
203 |                      1.22423939e+01, 4.59981011e+01, 4.79306463e+01, 9.91901033e+01,
204 |                      4.74497259e+00],
205 |                     [2.78095118e-03, 4.38070528e+03, 4.75270109e+03, 1.08728819e+01,
206 |                      1.22383163e+01, 4.59788637e+01, 4.77614296e+01, 9.92454851e+01,
207 |                      4.75863648e+00]])
208 | 


--------------------------------------------------------------------------------
/docs/microgrid.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/EnergyQuantResearch/PGSIM/e5ce48bc2730678eb4b119fb10f5e1d24e6b69d8/docs/microgrid.png


--------------------------------------------------------------------------------
/envs/Grid_envs.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | from abc import ABC
  3 | 
  4 | sys.path.append("../")
  5 | from scipy.integrate import odeint
  6 | 
  7 | DER_num = 40
  8 | 
  9 | if DER_num == 4:
 10 |     from configs.parameters_der4 import *
 11 |     from envs.der4_RL_fn import der_fn
 12 | elif DER_num == 6:
 13 |     from configs.parameters_der6 import *
 14 |     from envs.der6_RL_fn import der_fn
 15 | elif DER_num == 19:
 16 |     from configs.parameters_der19 import *
 17 |     from envs.der19_RL_fn import der_fn
 18 | elif DER_num == 20:
 19 |     from configs.parameters_der20 import *
 20 |     from envs.der20_RL_fn import der_fn
 21 | elif DER_num == 21:
 22 |     from configs.parameters_der21 import *
 23 |     from envs.der21_RL_fn import der_fn
 24 | elif DER_num == 40:
 25 |     from configs.parameters_der40 import *
 26 |     from envs.der40_RL_fn import der_fn
 27 | else:
 28 |     pass
 29 | 
 30 | 
 31 | class GridEnv(ABC):
 32 |     def __init__(self, config, discrete=True, rendering=False, episode_length=20, random_seed=0, train_mode=True):
 33 |         # Setup gym environment
 34 |         self.discrete = discrete
 35 |         self.rendering = rendering
 36 |         self.T = episode_length
 37 |         self.seed = random_seed
 38 |         self.agent = config.get('agent')
 39 |         self.dt = config.getfloat('sampling_time')
 40 |         self.coop_gamma = config.getfloat('coop_gamma')
 41 |         self.train_mode = train_mode
 42 |         test_seeds = config.get('test_seeds').split(',')
 43 |         test_seeds = [int(s) for s in test_seeds]
 44 |         self.init_test_seeds(test_seeds)
 45 |         self.min_action = 490
 46 |         self.max_action = 540
 47 |         self.states = []
 48 |         self.n_a = 10
 49 |         self.n_s = 9
 50 |         self.obs_mean = obs_mean
 51 |         self.obs_std = obs_std
 52 |         self.DER_num, self.lines_num, self.loads_num = DER_num, lines_num, loads_num
 53 |         self.n_agent = self.DER_num
 54 |         self.der_fn = der_fn
 55 |         self._init_space()
 56 |         self.step_list = []
 57 |         self.action_smoothing = 0
 58 |         try:
 59 |             self.reset()
 60 |         except Exception as e:
 61 |             self.close()
 62 |             raise e
 63 | 
 64 |     def reset(self, test_ind=0):
 65 |         # for initial random disturbance
 66 |         if self.train_mode:
 67 |             seed = self.seed
 68 |         else:
 69 |             seed = self.test_seeds[test_ind]
 70 |         np.random.seed(seed)
 71 |         # random seed for step function
 72 |         self.seed += 1
 73 |         self.t = 1
 74 |         self.pf_fail = False
 75 |         # initialize fingerprint
 76 |         self.update_fingerprint(self._init_policy())
 77 |         self.index = 0
 78 |         self.vbuses = []
 79 |         self.time = []
 80 |         self.step_list.append(1)  # used to store steps when all the voltages are in normal range
 81 |         self.action_list = [[] for x in range(self.DER_num)]
 82 |         self.x0 = x0
 83 |         # 0.4s for stabling the primary control
 84 |         t = np.linspace(0, 0.4, 11)
 85 |         # random disturbance
 86 |         self.disturbance_R = np.random.rand(self.loads_num) * 0.4 + 0.8
 87 |         self.disturbance_L = np.random.rand(self.loads_num) * 0.4 + 0.8
 88 | 
 89 |         self.x0 = odeint(self.der_fn, self.x0, t, args=(self.disturbance_R, self.disturbance_L), atol=1e-10,
 90 |                          rtol=1e-11, mxstep=5000)
 91 |         for j in range(self.DER_num):
 92 |             # voltage of buses
 93 |             self.vbuses.append(((self.x0[:, self.DER_num * 6 + self.lines_num * 2 + self.loads_num * 2 + j + 1] - nq[j] * self.x0[:,
 94 |                                                                                                                     5 * j + 3]) / Vnom).tolist())
 95 |         self.time.extend(t)
 96 |         self.x0 = (self.x0[-1]).tolist()
 97 |         return self._get_state()
 98 | 
 99 |     def step(self, action, mode='train'):
100 |         if self.discrete:
101 |             # discrete control
102 |             action = (self.max_action - self.min_action) / self.n_a * action + self.min_action
103 |         else:
104 |             # continuous control
105 |             action = 0.5 * (action + 1) * (self.max_action - self.min_action) + self.min_action
106 |         action = np.clip(action, self.min_action, self.max_action).astype(np.float32)
107 | 
108 |         # used to record actions
109 |         for i in range(self.DER_num):
110 |             self.action_list[i].append(action[i])
111 | 
112 |         # action smoothing
113 |         if self.t == 1:
114 |             self.action_past = action
115 |         action = self.action_past * self.action_smoothing + action * (
116 |                 1.0 - self.action_smoothing)
117 | 
118 |         self.x0[(6 * self.DER_num + 2 * self.lines_num + 2 * self.loads_num + 1):(
119 |                     7 * self.DER_num + 2 * self.lines_num + 2 * self.loads_num + 1)] = action.tolist()
120 |         if mode == 'train':
121 |             # random disturbance
122 |             self.disturbance_R = (np.random.rand(self.loads_num) * 0.1 - 0.05) + self.disturbance_R
123 |             self.disturbance_L = (np.random.rand(self.loads_num) * 0.1 - 0.05) + self.disturbance_L
124 |         else:
125 |             self.disturbance_R = 0 + self.disturbance_R
126 |             self.disturbance_L = 0 + self.disturbance_L
127 | 
128 |         try:
129 |             self.x0 = odeint(self.der_fn, self.x0,
130 |                              np.array([(self.t - 1) * self.dt, self.t * self.dt]),
131 |                              args=(self.disturbance_R, self.disturbance_L),
132 |                              atol=1e-10, rtol=1e-11, mxstep=5000)
133 |         except:
134 |             self.pf_fail = True
135 |             print("no solution")
136 |         finally:
137 |             self.x0 = self.x0[-1]
138 |             obs = self._get_state()
139 |             self.states.append(obs)
140 |             reward = self.get_reward()
141 |             done = self._check_termination()
142 |             global_reward = np.sum(reward)
143 |             self.t += 1
144 |             # use original rewards in test
145 |             if not self.train_mode:
146 |                 return obs, reward, done, global_reward
147 |             if (self.agent == 'greedy') or (self.coop_gamma < 0):
148 |                 reward = global_reward
149 |             self.action_past = action
150 |             return obs, reward, done, global_reward
151 | 
152 |     def _check_termination(self, ):
153 |         # achieve the maximum time or no powerflow solution
154 |         if self.t >= self.T or self.pf_fail == True:
155 |             return True
156 |         return False
157 | 
158 |     def get_fingerprint(self):
159 |         return self.fingerprint
160 | 
161 |     def update_fingerprint(self, policy):
162 |         self.fingerprint = policy
163 | 
164 |     def _init_policy(self, ):
165 |         return [np.ones(self.n_a_ls[i]) / self.n_a_ls[i] for i in range(self.n_agent)]
166 | 
167 |     def get_der_state(self):
168 |         self.der_state = np.zeros(shape=(self.n_agent, self.n_s))
169 |         for i in range(self.n_agent):
170 |             self.der_state[i] = (np.array(
171 |                 [self.x0[1 + 5 * i], self.x0[2 + 5 * i], self.x0[3 + 5 * i], self.x0[4 + 5 * i],
172 |                  self.x0[5 + 5 * i], x0[7 * DER_num + 2 * lines_num + 2 * loads_num + 1 + 4 * i],
173 |                  x0[7 * DER_num + 2 * lines_num + 2 * loads_num + 2 + 4 * i],
174 |                  x0[7 * DER_num + 2 * lines_num + 2 * loads_num + 3 + 4 * i],
175 |                  x0[7 * DER_num + 2 * lines_num + 2 * loads_num + 4 + 4 * i]]) -
176 |                                  self.obs_mean[i]) / self.obs_std[i]
177 | 
178 |     def _get_state(self, ):
179 |         state = []
180 |         self.get_der_state()
181 |         if self.agent == 'greedy':
182 |             state = self.der_state
183 |         elif self.agent == 'centralized':
184 |             state = np.array(self.x0[1:(5 * self.DER_num + 2 * self.lines_num + 2 * self.loads_num + 1)])
185 |         else:
186 |             for n in range(self.n_agent):
187 |                 cur_state = self.der_state[n]
188 |                 n_n = np.sum(self.neighbor_mask[n])
189 |                 if self.agent.startswith('ia2c'):
190 |                     for j in range(int(n_n)):
191 |                         index = np.where(self.neighbor_mask[n] == 1)[0]
192 |                         cur_state = np.concatenate((cur_state, self.der_state[index[j]]))
193 |                 if self.agent == 'ia2c_fp':
194 |                     for j in range(int(n_n)):
195 |                         index = np.where(self.neighbor_mask[n] == 1)[0]
196 |                         cur_state = np.concatenate((cur_state, self.fingerprint[index[j]]))
197 |                 state.append(cur_state)
198 |         return state
199 | 
200 |     def get_neighbor_action(self, action):
201 |         naction = []
202 |         for i in range(self.n_agent):
203 |             naction.append(action[self.neighbor_mask[i] == 1])
204 |         return naction
205 | 
206 |     def get_reward(self, ):
207 |         reward = []
208 |         count = 0
209 |         x = np.array(self.x0)
210 |         for j in range(self.DER_num):
211 |             # voltage of buses
212 |             vi = (x[self.DER_num * 6 + self.lines_num * 2 + self.loads_num * 2 + j + 1] - nq[j] * x[5 * j + 3]) / Vnom
213 |             self.vbuses[j].append(vi)
214 |             if self.pf_fail:
215 |                 reward.append(-10)
216 |             elif 0.95 <= vi <= 1.05:
217 |                 count += 1
218 |                 reward.append(0.05 - np.abs(1 - vi))
219 |             elif vi <= 0.8 or vi >= 1.25:
220 |                 reward.append(-10)
221 |             else:
222 |                 reward.append(- np.abs(1 - vi))
223 |         if count == self.n_agent and self.index == 0:
224 |             self.step_list[-1] = self.t
225 |             self.index = 1
226 |         elif (count != self.n_agent) and (self.index == 0) and (self.t == self.T):
227 |             self.step_list[-1] = self.t
228 |         else:
229 |             pass
230 |         self.time.append(self.t * self.dt + 0.4)
231 |         return np.array(reward)
232 | 
233 |     def _init_space(self):
234 |         self.n_s_ls = []
235 |         # declare for IA2C
236 |         self.neighbor_mask = Physical_Graph
237 |         self.distance_mask = Distance_Mask
238 |         for i in range(self.n_agent):
239 |             # initialize state space
240 |             if not self.agent.startswith('ma2c'):
241 |                 n_n = np.sum(self.neighbor_mask[i])
242 |                 self.n_s_ls.append(self.n_s * (1 + n_n))
243 |             else:
244 |                 self.n_s_ls.append(self.n_s)  # DER state*5 + load_state*2 + Bus_state*2
245 |         self.n_a_ls = [self.n_a] * self.n_agent
246 | 
247 |     def init_test_seeds(self, test_seeds):
248 |         self.test_num = len(test_seeds)
249 |         self.test_seeds = test_seeds
250 | 
251 |     def output_data(self, output_path, test_ind):
252 |         np.save(output_path + 'voltage/voltage_' + '{}'.format(test_ind), self.vbuses)
253 |         np.save(output_path + 'voltage/times_' + '{}'.format(test_ind), self.time)
254 |         np.save(output_path + 'voltage/action_list_' + '{}'.format(test_ind), self.action_list)
255 | 
256 |     def terminate(self):
257 |         return
258 | 
259 |     def render(self, mode='human'):
260 |         pass
261 | 


--------------------------------------------------------------------------------
/envs/der20_RL_fn.py:
--------------------------------------------------------------------------------
  1 | """""*******************************************************
  2 |  * Copyright (C) 2020 {Dong Chen} <{chendon9@msu.edu}>
  3 |  * this function is used to interact with RL agent.
  4 | """
  5 | 
  6 | from configs.parameters_der20 import *
  7 | import numba as nb
  8 | 
  9 | 
 10 | @nb.jit
 11 | def der_fn(x, t, disturbance_R, disturbance_L):
 12 |     # ---------Transferring Inv. Output Currents to Global DQ-----------------
 13 |     ioD1 = math.cos(0) * x[4] - math.sin(0) * x[5]
 14 |     ioQ1 = math.sin(0) * x[4] + math.cos(0) * x[5]
 15 |     ioD2 = math.cos(x[6]) * x[9] - math.sin(x[6]) * x[10]
 16 |     ioQ2 = math.sin(x[6]) * x[9] + math.cos(x[6]) * x[10]
 17 |     ioD3 = math.cos(x[11]) * x[14] - math.sin(x[11]) * x[15]
 18 |     ioQ3 = math.sin(x[11]) * x[14] + math.cos(x[11]) * x[15]
 19 |     ioD4 = math.cos(x[16]) * x[19] - math.sin(x[16]) * x[20]
 20 |     ioQ4 = math.sin(x[16]) * x[19] + math.cos(x[16]) * x[20]
 21 |     ioD5 = math.cos(x[21]) * x[24] - math.sin(x[21]) * x[25]
 22 |     ioQ5 = math.sin(x[21]) * x[24] + math.cos(x[21]) * x[25]
 23 |     ioD6 = math.cos(x[26]) * x[29] - math.sin(x[26]) * x[30]
 24 |     ioQ6 = math.sin(x[26]) * x[29] + math.cos(x[26]) * x[30]
 25 |     ioD7 = math.cos(x[31]) * x[34] - math.sin(x[31]) * x[35]
 26 |     ioQ7 = math.sin(x[31]) * x[34] + math.cos(x[31]) * x[35]
 27 |     ioD8 = math.cos(x[36]) * x[39] - math.sin(x[36]) * x[40]
 28 |     ioQ8 = math.sin(x[36]) * x[39] + math.cos(x[36]) * x[40]
 29 |     ioD9 = math.cos(x[41]) * x[44] - math.sin(x[41]) * x[45]
 30 |     ioQ9 = math.sin(x[41]) * x[44] + math.cos(x[41]) * x[45]
 31 |     ioD10 = math.cos(x[46]) * x[49] - math.sin(x[46]) * x[50]
 32 |     ioQ10 = math.sin(x[46]) * x[49] + math.cos(x[46]) * x[50]
 33 |     ioD11 = math.cos(x[51]) * x[54] - math.sin(x[51]) * x[55]
 34 |     ioQ11 = math.sin(x[51]) * x[54] + math.cos(x[51]) * x[55]
 35 |     ioD12 = math.cos(x[56]) * x[59] - math.sin(x[56]) * x[60]
 36 |     ioQ12 = math.sin(x[56]) * x[59] + math.cos(x[56]) * x[60]
 37 |     ioD13 = math.cos(x[61]) * x[64] - math.sin(x[61]) * x[65]
 38 |     ioQ13 = math.sin(x[61]) * x[64] + math.cos(x[61]) * x[65]
 39 |     ioD14 = math.cos(x[66]) * x[69] - math.sin(x[66]) * x[70]
 40 |     ioQ14 = math.sin(x[66]) * x[69] + math.cos(x[66]) * x[70]
 41 |     ioD15 = math.cos(x[71]) * x[74] - math.sin(x[71]) * x[75]
 42 |     ioQ15 = math.sin(x[71]) * x[74] + math.cos(x[71]) * x[75]
 43 |     ioD16 = math.cos(x[76]) * x[79] - math.sin(x[76]) * x[80]
 44 |     ioQ16 = math.sin(x[76]) * x[79] + math.cos(x[76]) * x[80]
 45 |     ioD17 = math.cos(x[81]) * x[84] - math.sin(x[81]) * x[85]
 46 |     ioQ17 = math.sin(x[81]) * x[84] + math.cos(x[81]) * x[85]
 47 |     ioD18 = math.cos(x[86]) * x[89] - math.sin(x[86]) * x[90]
 48 |     ioQ18 = math.sin(x[86]) * x[89] + math.cos(x[86]) * x[90]
 49 |     ioD19 = math.cos(x[91]) * x[94] - math.sin(x[91]) * x[95]
 50 |     ioQ19 = math.sin(x[91]) * x[94] + math.cos(x[91]) * x[95]
 51 |     ioD20 = math.cos(x[96]) * x[99] - math.sin(x[96]) * x[100]
 52 |     ioQ20 = math.sin(x[96]) * x[99] + math.cos(x[96]) * x[100]
 53 | 
 54 |     # ---------Defining Bus Voltages-----------------
 55 |     vbD1 = rN * (ioD1 - x[141] - x[101])
 56 |     vbQ1 = rN * (ioQ1 - x[142] - x[102])
 57 |     vbD2 = rN * (ioD2 + x[105] - x[107])
 58 |     vbQ2 = rN * (ioQ2 + x[106] - x[108])
 59 |     vbD3 = rN * (ioD3 + x[107] - x[109] - x[145])
 60 |     vbQ3 = rN * (ioQ3 + x[108] - x[110] - x[146])
 61 |     vbD4 = rN * (ioD4 + x[113] - x[115])
 62 |     vbQ4 = rN * (ioQ4 + x[114] - x[116])
 63 |     vbD5 = rN * (ioD5 + x[115] - x[117] - x[147])
 64 |     vbQ5 = rN * (ioQ5 + x[116] - x[118] - x[148])
 65 |     vbD6 = rN * (ioD6 + x[101] - x[103])
 66 |     vbQ6 = rN * (ioQ6 + x[102] - x[104])
 67 |     vbD7 = rN * (ioD7 + x[103] - x[105] - x[143])
 68 |     vbQ7 = rN * (ioQ7 + x[104] - x[106] - x[144])
 69 |     vbD8 = rN * (ioD8 + x[109] - x[111] - x[119])
 70 |     vbQ8 = rN * (ioQ8 + x[110] - x[112] - x[120])
 71 |     vbD9 = rN * (ioD9 + x[111] - x[113] - x[147])
 72 |     vbQ9 = rN * (ioQ9 + x[112] - x[114] - x[148])
 73 |     vbD10 = rN * (ioD10 + x[117] - x[119])
 74 |     vbQ10 = rN * (ioQ10 + x[118] - x[120])
 75 |     vbD11 = rN * (ioD11 + x[119] - x[123] - x[151])
 76 |     vbQ11 = rN * (ioQ11 + x[120] - x[124] - x[152])
 77 |     vbD12 = rN * (ioD12 + x[127] - x[129])
 78 |     vbQ12 = rN * (ioQ12 + x[128] - x[130])
 79 |     vbD13 = rN * (ioD13 + x[121] + x[129] - x[131] - x[155])
 80 |     vbQ13 = rN * (ioQ13 + x[122] + x[130] - x[132] - x[156])
 81 |     vbD14 = rN * (ioD14 + x[135] - x[137])
 82 |     vbQ14 = rN * (ioQ14 + x[136] - x[138])
 83 |     vbD15 = rN * (ioD15 + x[137] - x[139] - x[159])
 84 |     vbQ15 = rN * (ioQ15 + x[138] - x[140] - x[160])
 85 |     vbD16 = rN * (ioD16 + x[123] - x[125])
 86 |     vbQ16 = rN * (ioQ16 + x[124] - x[126])
 87 |     vbD17 = rN * (ioD17 + x[125] - x[127] - x[153])
 88 |     vbQ17 = rN * (ioQ17 + x[126] - x[128] - x[154])
 89 |     vbD18 = rN * (ioD18 + x[131] - x[133])
 90 |     vbQ18 = rN * (ioQ18 + x[132] - x[134])
 91 |     vbD19 = rN * (ioD19 + x[133] - x[135] - x[157])
 92 |     vbQ19 = rN * (ioQ19 + x[134] - x[136] - x[158])
 93 |     vbD20 = rN * (ioD20 + x[139])
 94 |     vbQ20 = rN * (ioQ20 + x[140])
 95 | 
 96 |     # ---------Transferring Bus Voltages to Inv. dq-----------------
 97 |     vbd1 = math.cos(0) * vbD1 + math.sin(0) * vbQ1
 98 |     vbq1 = -math.sin(0) * vbD1 + math.cos(0) * vbQ1
 99 |     vbd2 = math.cos(x[6]) * vbD2 + math.sin(x[6]) * vbQ2
100 |     vbq2 = -math.sin(x[6]) * vbD2 + math.cos(x[6]) * vbQ2
101 |     vbd3 = math.cos(x[11]) * vbD3 + math.sin(x[11]) * vbQ3
102 |     vbq3 = -math.sin(x[11]) * vbD3 + math.cos(x[11]) * vbQ3
103 |     vbd4 = math.cos(x[16]) * vbD4 + math.sin(x[16]) * vbQ4
104 |     vbq4 = -math.sin(x[16]) * vbD4 + math.cos(x[16]) * vbQ4
105 |     vbd5 = math.cos(x[21]) * vbD5 + math.sin(x[21]) * vbQ5
106 |     vbq5 = -math.sin(x[21]) * vbD5 + math.cos(x[21]) * vbQ5
107 |     vbd6 = math.cos(x[26]) * vbD6 + math.sin(x[26]) * vbQ6
108 |     vbq6 = -math.sin(x[26]) * vbD6 + math.cos(x[26]) * vbQ6
109 |     vbd7 = math.cos(x[31]) * vbD7 + math.sin(x[31]) * vbQ7
110 |     vbq7 = -math.sin(x[31]) * vbD7 + math.cos(x[31]) * vbQ7
111 |     vbd8 = math.cos(x[36]) * vbD8 + math.sin(x[36]) * vbQ8
112 |     vbq8 = -math.sin(x[36]) * vbD8 + math.cos(x[36]) * vbQ8
113 |     vbd9 = math.cos(x[41]) * vbD9 + math.sin(x[41]) * vbQ9
114 |     vbq9 = -math.sin(x[41]) * vbD9 + math.cos(x[41]) * vbQ9
115 |     vbd10 = math.cos(x[46]) * vbD10 + math.sin(x[46]) * vbQ10
116 |     vbq10 = -math.sin(x[46]) * vbD10 + math.cos(x[46]) * vbQ10
117 |     vbd11 = math.cos(x[51]) * vbD11 + math.sin(x[51]) * vbQ11
118 |     vbq11 = -math.sin(x[51]) * vbD11 + math.cos(x[51]) * vbQ11
119 |     vbd12 = math.cos(x[56]) * vbD12 + math.sin(x[56]) * vbQ12
120 |     vbq12 = -math.sin(x[56]) * vbD12 + math.cos(x[56]) * vbQ12
121 |     vbd13 = math.cos(x[61]) * vbD13 + math.sin(x[61]) * vbQ13
122 |     vbq13 = -math.sin(x[61]) * vbD13 + math.cos(x[61]) * vbQ13
123 |     vbd14 = math.cos(x[66]) * vbD14 + math.sin(x[66]) * vbQ14
124 |     vbq14 = -math.sin(x[66]) * vbD14 + math.cos(x[66]) * vbQ14
125 |     vbd15 = math.cos(x[71]) * vbD15 + math.sin(x[71]) * vbQ15
126 |     vbq15 = -math.sin(x[71]) * vbD15 + math.cos(x[71]) * vbQ15
127 |     vbd16 = math.cos(x[76]) * vbD16 + math.sin(x[76]) * vbQ16
128 |     vbq16 = -math.sin(x[76]) * vbD16 + math.cos(x[76]) * vbQ16
129 |     vbd17 = math.cos(x[81]) * vbD17 + math.sin(x[81]) * vbQ17
130 |     vbq17 = -math.sin(x[81]) * vbD17 + math.cos(x[81]) * vbQ17
131 |     vbd18 = math.cos(x[86]) * vbD18 + math.sin(x[86]) * vbQ18
132 |     vbq18 = -math.sin(x[86]) * vbD18 + math.cos(x[86]) * vbQ18
133 |     vbd19 = math.cos(x[91]) * vbD19 + math.sin(x[91]) * vbQ19
134 |     vbq19 = -math.sin(x[91]) * vbD19 + math.cos(x[91]) * vbQ19
135 |     vbd20 = math.cos(x[96]) * vbD20 + math.sin(x[96]) * vbQ20
136 |     vbq20 = -math.sin(x[96]) * vbD20 + math.cos(x[96]) * vbQ20
137 | 
138 |     wcom = x[161] - mp1 * x[2]
139 |     # ---------------DG1--------------------
140 |     vod1_star = x[181] - nq1 * x[3]
141 |     voq1_star = 0
142 |     xdot1 = x[161] - mp1 * x[2] - wcom
143 |     xdot2 = wc * (vod1_star * x[4] + voq1_star * x[5] - x[2])
144 |     xdot3 = wc * (-vod1_star * x[5] + voq1_star * x[4] - x[3])
145 |     xdot4 = (-rLc / Lc) * x[4] + wcom * x[5] + (1 / Lc) * (vod1_star - vbd1)
146 |     xdot5 = (-rLc / Lc) * x[5] - wcom * x[4] + (1 / Lc) * (voq1_star - vbq1)
147 | 
148 |     # ----------------DG2-------------------
149 |     vod2_star = x[182] - nq2 * x[8]
150 |     voq2_star = 0
151 |     xdot6 = x[162] - mp2 * x[7] - wcom
152 |     xdot7 = wc * (vod2_star * x[9] + voq2_star * x[10] - x[7])
153 |     xdot8 = wc * (-vod2_star * x[10] + voq2_star * x[9] - x[8])
154 |     xdot9 = (-rLc / Lc) * x[9] + wcom * x[10] + (1 / Lc) * (vod2_star - vbd2)
155 |     xdot10 = (-rLc / Lc) * x[10] - wcom * x[9] + (1 / Lc) * (voq2_star - vbq2)
156 | 
157 |     # ----------------DG3-------------------
158 |     vod3_star = x[183] - nq3 * x[13]
159 |     voq3_star = 0
160 |     xdot11 = x[163] - mp3 * x[12] - wcom
161 |     xdot12 = wc * (vod3_star * x[14] + voq3_star * x[15] - x[12])
162 |     xdot13 = wc * (-vod3_star * x[15] + voq3_star * x[14] - x[13])
163 |     xdot14 = (-rLc / Lc) * x[14] + wcom * x[15] + (1 / Lc) * (vod3_star - vbd3)
164 |     xdot15 = (-rLc / Lc) * x[15] - wcom * x[14] + (1 / Lc) * (voq3_star - vbq3)
165 | 
166 |     # ----------------DG4-------------------
167 |     vod4_star = x[184] - nq4 * x[18]
168 |     voq4_star = 0
169 |     xdot16 = x[164] - mp4 * x[17] - wcom
170 |     xdot17 = wc * (vod4_star * x[19] + voq4_star * x[20] - x[17])
171 |     xdot18 = wc * (-vod4_star * x[20] + voq4_star * x[19] - x[18])
172 |     xdot19 = (-rLc / Lc) * x[19] + wcom * x[20] + (1 / Lc) * (vod4_star - vbd4)
173 |     xdot20 = (-rLc / Lc) * x[20] - wcom * x[19] + (1 / Lc) * (voq4_star - vbq4)
174 | 
175 |     # ----------------DG5-------------------
176 |     vod5_star = x[185] - nq5 * x[23]
177 |     voq5_star = 0
178 |     xdot21 = x[165] - mp5 * x[22] - wcom
179 |     xdot22 = wc * (vod5_star * x[24] + voq5_star * x[25] - x[22])
180 |     xdot23 = wc * (-vod5_star * x[25] + voq5_star * x[24] - x[23])
181 |     xdot24 = (-rLc / Lc) * x[24] + wcom * x[25] + (1 / Lc) * (vod5_star - vbd5)
182 |     xdot25 = (-rLc / Lc) * x[25] - wcom * x[24] + (1 / Lc) * (voq5_star - vbq5)
183 | 
184 |     # ----------------DG6-------------------
185 |     vod6_star = x[186] - nq6 * x[28]
186 |     voq6_star = 0
187 |     xdot26 = x[166] - mp6 * x[27] - wcom
188 |     xdot27 = wc * (vod6_star * x[29] + voq6_star * x[30] - x[27])
189 |     xdot28 = wc * (-vod6_star * x[30] + voq6_star * x[29] - x[28])
190 |     xdot29 = (-rLc / Lc) * x[29] + wcom * x[30] + (1 / Lc) * (vod6_star - vbd6)
191 |     xdot30 = (-rLc / Lc) * x[30] - wcom * x[29] + (1 / Lc) * (voq6_star - vbq6)
192 | 
193 |     # ----------------DG7-------------------
194 |     vod7_star = x[187] - nq7 * x[33]
195 |     voq7_star = 0
196 |     xdot31 = x[167] - mp7 * x[32] - wcom
197 |     xdot32 = wc * (vod7_star * x[34] + voq7_star * x[35] - x[32])
198 |     xdot33 = wc * (-vod7_star * x[35] + voq7_star * x[34] - x[33])
199 |     xdot34 = (-rLc / Lc) * x[34] + wcom * x[35] + (1 / Lc) * (vod7_star - vbd7)
200 |     xdot35 = (-rLc / Lc) * x[35] - wcom * x[34] + (1 / Lc) * (voq7_star - vbq7)
201 | 
202 |     # ----------------DG8-------------------
203 |     vod8_star = x[188] - nq8 * x[38]
204 |     voq8_star = 0
205 |     xdot36 = x[168] - mp8 * x[37] - wcom
206 |     xdot37 = wc * (vod8_star * x[39] + voq8_star * x[40] - x[37])
207 |     xdot38 = wc * (-vod8_star * x[40] + voq8_star * x[39] - x[38])
208 |     xdot39 = (-rLc / Lc) * x[39] + wcom * x[40] + (1 / Lc) * (vod8_star - vbd8)
209 |     xdot40 = (-rLc / Lc) * x[40] - wcom * x[39] + (1 / Lc) * (voq8_star - vbq8)
210 | 
211 |     # ----------------DG9-------------------
212 |     vod9_star = x[189] - nq9 * x[43]
213 |     voq9_star = 0
214 |     xdot41 = x[169] - mp9 * x[42] - wcom
215 |     xdot42 = wc * (vod9_star * x[44] + voq9_star * x[45] - x[42])
216 |     xdot43 = wc * (-vod9_star * x[45] + voq9_star * x[44] - x[43])
217 |     xdot44 = (-rLc / Lc) * x[44] + wcom * x[45] + (1 / Lc) * (vod9_star - vbd9)
218 |     xdot45 = (-rLc / Lc) * x[45] - wcom * x[44] + (1 / Lc) * (voq9_star - vbq9)
219 | 
220 |     # ----------------DG10-------------------
221 |     vod10_star = x[190] - nq10 * x[48]
222 |     voq10_star = 0
223 |     xdot46 = x[170] - mp10 * x[47] - wcom
224 |     xdot47 = wc * (vod10_star * x[49] + voq10_star * x[50] - x[47])
225 |     xdot48 = wc * (-vod10_star * x[50] + voq10_star * x[49] - x[48])
226 |     xdot49 = (-rLc / Lc) * x[49] + wcom * x[50] + (1 / Lc) * (vod10_star - vbd10)
227 |     xdot50 = (-rLc / Lc) * x[50] - wcom * x[49] + (1 / Lc) * (voq10_star - vbq10)
228 | 
229 |     # ----------------DG11-------------------
230 |     vod11_star = x[191] - nq11 * x[53]
231 |     voq11_star = 0
232 |     xdot51 = x[171] - mp11 * x[52] - wcom
233 |     xdot52 = wc * (vod11_star * x[54] + voq11_star * x[55] - x[52])
234 |     xdot53 = wc * (-vod11_star * x[55] + voq11_star * x[54] - x[53])
235 |     xdot54 = (-rLc / Lc) * x[54] + wcom * x[55] + (1 / Lc) * (vod11_star - vbd11)
236 |     xdot55 = (-rLc / Lc) * x[55] - wcom * x[54] + (1 / Lc) * (voq11_star - vbq11)
237 | 
238 |     # ----------------DG12-------------------
239 |     vod12_star = x[192] - nq12 * x[58]
240 |     voq12_star = 0
241 |     xdot56 = x[172] - mp12 * x[57] - wcom
242 |     xdot57 = wc * (vod12_star * x[59] + voq12_star * x[60] - x[57])
243 |     xdot58 = wc * (-vod12_star * x[60] + voq12_star * x[59] - x[58])
244 |     xdot59 = (-rLc / Lc) * x[59] + wcom * x[60] + (1 / Lc) * (vod12_star - vbd12)
245 |     xdot60 = (-rLc / Lc) * x[60] - wcom * x[59] + (1 / Lc) * (voq12_star - vbq12)
246 | 
247 |     # ----------------DG13-------------------
248 |     vod13_star = x[193] - nq13 * x[63]
249 |     voq13_star = 0
250 |     xdot61 = x[173] - mp13 * x[62] - wcom
251 |     xdot62 = wc * (vod13_star * x[64] + voq13_star * x[65] - x[62])
252 |     xdot63 = wc * (-vod13_star * x[65] + voq13_star * x[54] - x[63])
253 |     xdot64 = (-rLc / Lc) * x[64] + wcom * x[65] + (1 / Lc) * (vod13_star - vbd13)
254 |     xdot65 = (-rLc / Lc) * x[65] - wcom * x[64] + (1 / Lc) * (voq13_star - vbq13)
255 | 
256 |     # ----------------DG14-------------------
257 |     vod14_star = x[194] - nq14 * x[68]
258 |     voq14_star = 0
259 |     xdot66 = x[174] - mp14 * x[67] - wcom
260 |     xdot67 = wc * (vod14_star * x[69] + voq14_star * x[70] - x[67])
261 |     xdot68 = wc * (-vod14_star * x[70] + voq14_star * x[69] - x[68])
262 |     xdot69 = (-rLc / Lc) * x[69] + wcom * x[70] + (1 / Lc) * (vod14_star - vbd14)
263 |     xdot70 = (-rLc / Lc) * x[70] - wcom * x[69] + (1 / Lc) * (voq14_star - vbq14)
264 | 
265 |     # ----------------DG15-------------------
266 |     vod15_star = x[195] - nq15 * x[73]
267 |     voq15_star = 0
268 |     xdot71 = x[175] - mp15 * x[72] - wcom
269 |     xdot72 = wc * (vod15_star * x[74] + voq15_star * x[75] - x[72])
270 |     xdot73 = wc * (-vod15_star * x[75] + voq15_star * x[74] - x[73])
271 |     xdot74 = (-rLc / Lc) * x[74] + wcom * x[75] + (1 / Lc) * (vod15_star - vbd15)
272 |     xdot75 = (-rLc / Lc) * x[75] - wcom * x[74] + (1 / Lc) * (voq15_star - vbq15)
273 | 
274 |     # ----------------DG16-------------------
275 |     vod16_star = x[196] - nq16 * x[78]
276 |     voq16_star = 0
277 |     xdot76 = x[176] - mp16 * x[77] - wcom
278 |     xdot77 = wc * (vod16_star * x[79] + voq16_star * x[80] - x[77])
279 |     xdot78 = wc * (-vod16_star * x[80] + voq16_star * x[79] - x[78])
280 |     xdot79 = (-rLc / Lc) * x[79] + wcom * x[80] + (1 / Lc) * (vod16_star - vbd16)
281 |     xdot80 = (-rLc / Lc) * x[80] - wcom * x[79] + (1 / Lc) * (voq16_star - vbq16)
282 | 
283 |     # ----------------DG17-------------------
284 |     vod17_star = x[197] - nq17 * x[83]
285 |     voq17_star = 0
286 |     xdot81 = x[177] - mp17 * x[82] - wcom
287 |     xdot82 = wc * (vod17_star * x[84] + voq17_star * x[85] - x[82])
288 |     xdot83 = wc * (-vod17_star * x[85] + voq17_star * x[84] - x[83])
289 |     xdot84 = (-rLc / Lc) * x[84] + wcom * x[85] + (1 / Lc) * (vod17_star - vbd17)
290 |     xdot85 = (-rLc / Lc) * x[85] - wcom * x[84] + (1 / Lc) * (voq17_star - vbq17)
291 | 
292 |     # ----------------DG18-------------------
293 |     vod18_star = x[198] - nq18 * x[88]
294 |     voq18_star = 0
295 |     xdot86 = x[178] - mp18 * x[87] - wcom
296 |     xdot87 = wc * (vod18_star * x[89] + voq18_star * x[90] - x[87])
297 |     xdot88 = wc * (-vod18_star * x[90] + voq18_star * x[89] - x[88])
298 |     xdot89 = (-rLc / Lc) * x[89] + wcom * x[90] + (1 / Lc) * (vod18_star - vbd18)
299 |     xdot90 = (-rLc / Lc) * x[90] - wcom * x[89] + (1 / Lc) * (voq18_star - vbq18)
300 | 
301 |     # ----------------DG19-------------------
302 |     vod19_star = x[199] - nq19 * x[93]
303 |     voq19_star = 0
304 |     xdot91 = x[179] - mp19 * x[92] - wcom
305 |     xdot92 = wc * (vod19_star * x[94] + voq19_star * x[95] - x[92])
306 |     xdot93 = wc * (-vod19_star * x[95] + voq19_star * x[94] - x[93])
307 |     xdot94 = (-rLc / Lc) * x[94] + wcom * x[95] + (1 / Lc) * (vod19_star - vbd19)
308 |     xdot95 = (-rLc / Lc) * x[95] - wcom * x[94] + (1 / Lc) * (voq19_star - vbq19)
309 | 
310 |     # ----------------DG20-------------------
311 |     vod20_star = x[200] - nq20 * x[98]
312 |     voq20_star = 0
313 |     xdot96 = x[180] - mp20 * x[97] - wcom
314 |     xdot97 = wc * (vod20_star * x[99] + voq20_star * x[100] - x[97])
315 |     xdot98 = wc * (-vod20_star * x[100] + voq20_star * x[99] - x[98])
316 |     xdot99 = (-rLc / Lc) * x[99] + wcom * x[100] + (1 / Lc) * (vod20_star - vbd20)
317 |     xdot100 = (-rLc / Lc) * x[100] - wcom * x[99] + (1 / Lc) * (voq20_star - vbq20)
318 | 
319 |     # -------------------------Lines------------------
320 |     # ----1 -> 6-----
321 |     xdot101 = (-rline1 / Lline1) * x[101] + wcom * x[102] + (1 / Lline1) * (vbD1 - vbD6)
322 |     xdot102 = (-rline1 / Lline1) * x[102] - wcom * x[101] + (1 / Lline1) * (vbQ1 - vbQ6)
323 |     # ----6 -> 7-----
324 |     xdot103 = (-rline2 / Lline2) * x[103] + wcom * x[104] + (1 / Lline2) * (vbD6 - vbD7)
325 |     xdot104 = (-rline2 / Lline2) * x[104] - wcom * x[103] + (1 / Lline2) * (vbQ6 - vbQ7)
326 |     # ----7 -> 2-----
327 |     xdot105 = (-rline3 / Lline3) * x[105] + wcom * x[106] + (1 / Lline3) * (vbD7 - vbD2)
328 |     xdot106 = (-rline3 / Lline3) * x[106] - wcom * x[105] + (1 / Lline3) * (vbQ7 - vbQ2)
329 |     # ----2 -> 3-----
330 |     xdot107 = (-rline4 / Lline4) * x[107] + wcom * x[108] + (1 / Lline4) * (vbD2 - vbD3)
331 |     xdot108 = (-rline4 / Lline4) * x[108] - wcom * x[107] + (1 / Lline4) * (vbQ2 - vbQ3)
332 |     # ----3 -> 8-----
333 |     xdot109 = (-rline5 / Lline5) * x[109] + wcom * x[110] + (1 / Lline5) * (vbD3 - vbD8)
334 |     xdot110 = (-rline5 / Lline5) * x[110] - wcom * x[109] + (1 / Lline5) * (vbQ3 - vbQ8)
335 |     # ----8 -> 9-----
336 |     xdot111 = (-rline6 / Lline6) * x[111] + wcom * x[112] + (1 / Lline6) * (vbD8 - vbD9)
337 |     xdot112 = (-rline6 / Lline6) * x[112] - wcom * x[111] + (1 / Lline6) * (vbQ8 - vbQ9)
338 |     # ----9 -> 4-----
339 |     xdot113 = (-rline7 / Lline7) * x[113] + wcom * x[114] + (1 / Lline7) * (vbD9 - vbD4)
340 |     xdot114 = (-rline7 / Lline7) * x[114] - wcom * x[113] + (1 / Lline7) * (vbQ9 - vbQ4)
341 |     # ----4 -> 5-----
342 |     xdot115 = (-rline8 / Lline8) * x[115] + wcom * x[116] + (1 / Lline8) * (vbD4 - vbD5)
343 |     xdot116 = (-rline8 / Lline8) * x[116] - wcom * x[115] + (1 / Lline8) * (vbQ4 - vbQ5)
344 |     # ----5 -> 10-----
345 |     xdot117 = (-rline9 / Lline9) * x[117] + wcom * x[118] + (1 / Lline9) * (vbD5 - vbD10)
346 |     xdot118 = (-rline9 / Lline9) * x[118] - wcom * x[117] + (1 / Lline9) * (vbQ5 - vbQ10)
347 |     # ----8 -> 11-----
348 |     xdot119 = (-rline10 / Lline10) * x[119] + wcom * x[120] + (1 / Lline10) * (vbD8 - vbD11)
349 |     xdot120 = (-rline10 / Lline10) * x[120] - wcom * x[119] + (1 / Lline10) * (vbQ8 - vbQ11)
350 |     # ----10 -> 13-----
351 |     xdot121 = (-rline11 / Lline11) * x[121] + wcom * x[122] + (1 / Lline11) * (vbD10 - vbD13)
352 |     xdot122 = (-rline11 / Lline11) * x[122] - wcom * x[121] + (1 / Lline11) * (vbQ10 - vbQ13)
353 |     # ----11 -> 16-----
354 |     xdot123 = (-rline12 / Lline12) * x[123] + wcom * x[124] + (1 / Lline12) * (vbD11 - vbD16)
355 |     xdot124 = (-rline12 / Lline12) * x[124] - wcom * x[123] + (1 / Lline12) * (vbQ11 - vbQ16)
356 |     # ----16 -> 17-----
357 |     xdot125 = (-rline13 / Lline13) * x[125] + wcom * x[126] + (1 / Lline13) * (vbD16 - vbD17)
358 |     xdot126 = (-rline13 / Lline13) * x[126] - wcom * x[125] + (1 / Lline13) * (vbQ16 - vbQ17)
359 |     # ----17 -> 12-----
360 |     xdot127 = (-rline14 / Lline14) * x[127] + wcom * x[128] + (1 / Lline14) * (vbD17 - vbD12)
361 |     xdot128 = (-rline14 / Lline14) * x[128] - wcom * x[127] + (1 / Lline14) * (vbQ17 - vbQ12)
362 |     # ----12 -> 13-----
363 |     xdot129 = (-rline15 / Lline15) * x[129] + wcom * x[130] + (1 / Lline15) * (vbD12 - vbD13)
364 |     xdot130 = (-rline15 / Lline15) * x[130] - wcom * x[129] + (1 / Lline15) * (vbQ12 - vbQ13)
365 |     # ----13 -> 18-----
366 |     xdot131 = (-rline16 / Lline16) * x[131] + wcom * x[132] + (1 / Lline16) * (vbD13 - vbD18)
367 |     xdot132 = (-rline16 / Lline16) * x[132] - wcom * x[131] + (1 / Lline16) * (vbQ13 - vbQ18)
368 |     # ----18 -> 19-----
369 |     xdot133 = (-rline17 / Lline17) * x[133] + wcom * x[134] + (1 / Lline17) * (vbD18 - vbD19)
370 |     xdot134 = (-rline17 / Lline17) * x[134] - wcom * x[133] + (1 / Lline17) * (vbQ18 - vbQ19)
371 |     # ----19 -> 14-----
372 |     xdot135 = (-rline18 / Lline18) * x[135] + wcom * x[136] + (1 / Lline18) * (vbD19 - vbD14)
373 |     xdot136 = (-rline18 / Lline18) * x[136] - wcom * x[135] + (1 / Lline18) * (vbQ19 - vbQ14)
374 |     # ----14 -> 15-----
375 |     xdot137 = (-rline19 / Lline19) * x[137] + wcom * x[138] + (1 / Lline19) * (vbD14 - vbD15)
376 |     xdot138 = (-rline19 / Lline19) * x[138] - wcom * x[137] + (1 / Lline19) * (vbQ14 - vbQ15)
377 |     # ----15 -> 20-----
378 |     xdot139 = (-rline20 / Lline20) * x[139] + wcom * x[140] + (1 / Lline20) * (vbD15 - vbD20)
379 |     xdot140 = (-rline20 / Lline20) * x[140] - wcom * x[139] + (1 / Lline20) * (vbQ15 - vbQ20)
380 | 
381 |     # -------------------------Loads------------------
382 |     # ------Load1--------
383 |     xdot141 = (-disturbance_R[0] * Rload1 / (disturbance_L[0] * Lload1)) * x[141] + wcom * x[142] + (
384 |                 1 / (disturbance_L[0] * Lload1)) * vbD1
385 |     xdot142 = (-disturbance_R[0] * Rload1 / (disturbance_L[0] * Lload1)) * x[142] - wcom * x[141] + (
386 |                 1 / (disturbance_L[0] * Lload1)) * vbQ1
387 |     # ------Load2--------
388 |     xdot143 = (-disturbance_R[1] * Rload2 / (disturbance_L[1] * Lload2)) * x[143] + wcom * x[144] + (
389 |                 1 / (disturbance_L[1] * Lload2)) * vbD7
390 |     xdot144 = (-disturbance_R[1] * Rload2 / (disturbance_L[1] * Lload2)) * x[144] - wcom * x[143] + (
391 |                 1 / (disturbance_L[1] * Lload2)) * vbQ7
392 |     # ------Load3--------
393 |     xdot145 = (-disturbance_R[2] * Rload3 / (disturbance_L[2] * Lload3)) * x[145] + wcom * x[146] + (
394 |                 1 / (disturbance_L[2] * Lload3)) * vbD3
395 |     xdot146 = (-disturbance_R[2] * Rload3 / (disturbance_L[2] * Lload3)) * x[146] - wcom * x[145] + (
396 |                 1 / (disturbance_L[2] * Lload3)) * vbQ3
397 |     # ------Load4--------
398 |     xdot147 = (-disturbance_R[3] * Rload4 / (disturbance_L[3] * Lload4)) * x[147] + wcom * x[148] + (
399 |                 1 / (disturbance_L[3] * Lload4)) * vbD9
400 |     xdot148 = (-disturbance_R[3] * Rload4 / (disturbance_L[3] * Lload4)) * x[148] - wcom * x[147] + (
401 |                 1 / (disturbance_L[3] * Lload4)) * vbQ9
402 |     # ------Load5--------
403 |     xdot149 = (-disturbance_R[4] * Rload5 / (disturbance_L[4] * Lload5)) * x[149] + wcom * x[150] + (
404 |                 1 / (disturbance_L[4] * Lload5)) * vbD5
405 |     xdot150 = (-disturbance_R[4] * Rload5 / (disturbance_L[4] * Lload5)) * x[150] - wcom * x[149] + (
406 |                 1 / (disturbance_L[4] * Lload5)) * vbQ5
407 |     # ------Load6--------
408 |     xdot151 = (-disturbance_R[5] * Rload6 / (disturbance_L[5] * Lload6)) * x[151] + wcom * x[152] + (
409 |                 1 / (disturbance_L[5] * Lload6)) * vbD11
410 |     xdot152 = (-disturbance_R[5] * Rload6 / (disturbance_L[5] * Lload6)) * x[152] - wcom * x[151] + (
411 |                 1 / (disturbance_L[5] * Lload6)) * vbQ11
412 |     # ------Load7--------
413 |     xdot153 = (-disturbance_R[6] * Rload7 / (disturbance_L[6] * Lload7)) * x[153] + wcom * x[154] + (
414 |                 1 / (disturbance_L[6] * Lload7)) * vbD17
415 |     xdot154 = (-disturbance_R[6] * Rload7 / (disturbance_L[6] * Lload7)) * x[154] - wcom * x[153] + (
416 |                 1 / (disturbance_L[6] * Lload7)) * vbQ17
417 |     # ------Load8--------
418 |     xdot155 = (-disturbance_R[7] * Rload8 / (disturbance_L[7] * Lload8)) * x[155] + wcom * x[156] + (
419 |                 1 / (disturbance_L[7] * Lload8)) * vbD13
420 |     xdot156 = (-disturbance_R[7] * Rload8 / (disturbance_L[7] * Lload8)) * x[156] - wcom * x[155] + (
421 |                 1 / (disturbance_L[7] * Lload8)) * vbQ13
422 |     # ------Load9--------
423 |     xdot157 = (-disturbance_R[8] * Rload9 / (disturbance_L[8] * Lload9)) * x[157] + wcom * x[158] + (
424 |                 1 / (disturbance_L[8] * Lload9)) * vbD19
425 |     xdot158 = (-disturbance_R[8] * Rload9 / (disturbance_L[8] * Lload9)) * x[158] - wcom * x[157] + (
426 |                 1 / (disturbance_L[8] * Lload9)) * vbQ19
427 |     # ------Load10--------
428 |     xdot159 = (-disturbance_R[9] * Rload10 / (disturbance_L[9] * Lload10)) * x[159] + wcom * x[160] + (
429 |                 1 / (disturbance_L[9] * Lload10)) * vbD15
430 |     xdot160 = (-disturbance_R[9] * Rload10 / (disturbance_L[9] * Lload10)) * x[160] - wcom * x[159] + (
431 |                 1 / (disturbance_L[9] * Lload10)) * vbQ15
432 | 
433 |     # ----------------------------------------------------
434 |     # Controller Parameters
435 |     if t <= 0.4:
436 |         # this time is for the primary control
437 |         xdot161 = 0
438 |         xdot162 = 0
439 |         xdot163 = 0
440 |         xdot164 = 0
441 |         xdot165 = 0
442 |         xdot166 = 0
443 |         xdot167 = 0
444 |         xdot168 = 0
445 |         xdot169 = 0
446 |         xdot170 = 0
447 |         xdot171 = 0
448 |         xdot172 = 0
449 |         xdot173 = 0
450 |         xdot174 = 0
451 |         xdot175 = 0
452 |         xdot176 = 0
453 |         xdot177 = 0
454 |         xdot178 = 0
455 |         xdot179 = 0
456 |         xdot180 = 0
457 |         xdot181 = 0
458 |         xdot182 = 0
459 |         xdot183 = 0
460 |         xdot184 = 0
461 |         xdot185 = 0
462 |         xdot186 = 0
463 |         xdot187 = 0
464 |         xdot188 = 0
465 |         xdot189 = 0
466 |         xdot190 = 0
467 |         xdot191 = 0
468 |         xdot192 = 0
469 |         xdot193 = 0
470 |         xdot194 = 0
471 |         xdot195 = 0
472 |         xdot196 = 0
473 |         xdot197 = 0
474 |         xdot198 = 0
475 |         xdot199 = 0
476 |         xdot200 = 0
477 |     else:
478 |         Pratio = np.array([[mp1 * x[2]], [mp2 * x[7]], [mp3 * x[12]], [mp4 * x[17]], [mp5 * x[22]],
479 |                            [mp6 * x[27]], [mp7 * x[37]], [mp8 * x[42]], [mp9 * x[47]], [mp10 * x[52]],
480 |                            [mp11 * x[57]], [mp12 * x[62]], [mp13 * x[67]], [mp14 * x[72]], [mp15 * x[77]],
481 |                            [mp16 * x[82]], [mp17 * x[87]], [mp18 * x[92]], [mp19 * x[97]], [mp20 * x[102]]])
482 | 
483 |         w_array = np.array(
484 |             [[x[161] - Pratio[0][0]], [x[162] - Pratio[1][0]], [x[163] - Pratio[2][0]], [x[164] - Pratio[3][0]],
485 |              [x[165] - Pratio[4][0]], [x[166] - Pratio[5][0]], [x[167] - Pratio[6][0]], [x[168] - Pratio[7][0]],
486 |              [x[169] - Pratio[8][0]], [x[170] - Pratio[9][0]], [x[171] - Pratio[10][0]], [x[172] - Pratio[11][0]],
487 |              [x[173] - Pratio[12][0]], [x[174] - Pratio[13][0]], [x[175] - Pratio[14][0]], [x[176] - Pratio[15][0]],
488 |              [x[177] - Pratio[16][0]], [x[178] - Pratio[17][0]], [x[179] - Pratio[18][0]], [x[180] - Pratio[19][0]]])
489 | 
490 |         # Conventional Freq Control
491 |         Synch_Mat = -1 * a_ctrl * (
492 |                 np.dot(L + G, w_array - np.array([[wref], [wref], [wref], [wref],
493 |                                                   [wref], [wref], [wref], [wref],
494 |                                                   [wref], [wref], [wref], [wref],
495 |                                                   [wref], [wref], [wref], [wref],
496 |                                                   [wref], [wref], [wref], [wref]])) + np.dot(L, Pratio))
497 | 
498 |         # ---frequency input---
499 |         xdot161 = Synch_Mat[0][0]
500 |         xdot162 = Synch_Mat[1][0]
501 |         xdot163 = Synch_Mat[2][0]
502 |         xdot164 = Synch_Mat[3][0]
503 |         xdot165 = Synch_Mat[4][0]
504 |         xdot166 = Synch_Mat[5][0]
505 |         xdot167 = Synch_Mat[6][0]
506 |         xdot168 = Synch_Mat[7][0]
507 |         xdot169 = Synch_Mat[8][0]
508 |         xdot170 = Synch_Mat[9][0]
509 |         xdot171 = Synch_Mat[10][0]
510 |         xdot172 = Synch_Mat[11][0]
511 |         xdot173 = Synch_Mat[12][0]
512 |         xdot174 = Synch_Mat[13][0]
513 |         xdot175 = Synch_Mat[14][0]
514 |         xdot176 = Synch_Mat[15][0]
515 |         xdot177 = Synch_Mat[16][0]
516 |         xdot178 = Synch_Mat[17][0]
517 |         xdot179 = Synch_Mat[18][0]
518 |         xdot180 = Synch_Mat[19][0]
519 |         # ---voltage input---
520 |         xdot181 = 0
521 |         xdot182 = 0
522 |         xdot183 = 0
523 |         xdot184 = 0
524 |         xdot185 = 0
525 |         xdot186 = 0
526 |         xdot187 = 0
527 |         xdot188 = 0
528 |         xdot189 = 0
529 |         xdot190 = 0
530 |         xdot191 = 0
531 |         xdot192 = 0
532 |         xdot193 = 0
533 |         xdot194 = 0
534 |         xdot195 = 0
535 |         xdot196 = 0
536 |         xdot197 = 0
537 |         xdot198 = 0
538 |         xdot199 = 0
539 |         xdot200 = 0
540 | 
541 |     return np.array(
542 |         [0, xdot1, xdot2, xdot3, xdot4, xdot5, xdot6, xdot7, xdot8, xdot9, xdot10, xdot11, xdot12, xdot13, xdot14,
543 |          xdot15, xdot16, xdot17, xdot18, xdot19, xdot20, xdot21, xdot22, xdot23, xdot24, xdot25, xdot26, xdot27,
544 |          xdot28, xdot29, xdot30, xdot31, xdot32, xdot33, xdot34, xdot35, xdot36, xdot37, xdot38, xdot39, xdot40,
545 |          xdot41, xdot42, xdot43, xdot44, xdot45, xdot46, xdot47, xdot48, xdot49, xdot50, xdot51, xdot52, xdot53,
546 |          xdot54, xdot55, xdot56, xdot57, xdot58, xdot59, xdot60, xdot61, xdot62, xdot63, xdot64, xdot65, xdot66,
547 |          xdot67, xdot68, xdot69, xdot70, xdot71, xdot72, xdot73, xdot74, xdot75, xdot76, xdot77, xdot78, xdot79,
548 |          xdot80, xdot81, xdot82, xdot83, xdot84, xdot85, xdot86, xdot87, xdot88, xdot89, xdot90, xdot91, xdot92,
549 |          xdot93, xdot94, xdot95, xdot96, xdot97, xdot98, xdot99, xdot100, xdot101, xdot102, xdot103, xdot104,
550 |          xdot105, xdot106, xdot107, xdot108, xdot109, xdot110, xdot111, xdot112, xdot113, xdot114, xdot115,
551 |          xdot116, xdot117, xdot118, xdot119, xdot120, xdot121, xdot122, xdot123, xdot124, xdot125, xdot126,
552 |          xdot127, xdot128, xdot129, xdot130, xdot131, xdot132, xdot133, xdot134, xdot135, xdot136, xdot137,
553 |          xdot138, xdot139, xdot140, xdot141, xdot142, xdot143, xdot144, xdot145, xdot146, xdot147, xdot148,
554 |          xdot149, xdot150, xdot151, xdot152, xdot153, xdot154, xdot155, xdot156, xdot157, xdot158, xdot159,
555 |          xdot160, xdot161, xdot162, xdot163, xdot164, xdot165, xdot166, xdot167, xdot168, xdot169, xdot170,
556 |          xdot171, xdot172, xdot173, xdot174, xdot175, xdot176, xdot177, xdot178, xdot179, xdot180, xdot181,
557 |          xdot182, xdot183, xdot184, xdot185, xdot186, xdot187, xdot188, xdot189, xdot190, xdot191, xdot192,
558 |          xdot193, xdot194, xdot195, xdot196, xdot197, xdot198, xdot199, xdot200, ioD1, ioQ1, vbD1, vbQ1,
559 |          ioD2, ioQ2, vbD2, vbQ2, ioD3, ioQ3, vbD3, vbQ3, ioD4, ioQ4, vbD4, vbQ4, ioD5, ioQ5, vbD5, vbQ5,
560 |          ioD6, ioQ6, vbD6, vbQ6, ioD7, ioQ7, vbD7, vbQ7, ioD8, ioQ8, vbD8, vbQ8, ioD9, ioQ9, vbD9, vbQ9,
561 |          ioD10, ioQ10, vbD10, vbQ10, ioD11, ioQ11, vbD11, vbQ11, ioD12, ioQ12, vbD12, vbQ13, ioD13, ioQ13, vbD13, vbQ13,
562 |          ioD14, ioQ14, vbD14, vbQ14, ioD15, ioQ15, vbD15, vbQ15, ioD16, ioQ16, vbD16, vbQ16, ioD17, ioQ17, vbD17, vbQ17,
563 |          ioD18, ioQ18, vbD18, vbQ18, ioD19, ioQ19, vbD19, vbQ19, ioD20, ioQ20, vbD20, vbQ20
564 |          ])


--------------------------------------------------------------------------------
/envs/der6_RL_fn.py:
--------------------------------------------------------------------------------
  1 | """""*******************************************************
  2 |  * Copyright (C) 2020 {Dong Chen} <{chendon9@msu.edu}>
  3 |  * this function is used to interact with RL agent.
  4 | """
  5 | 
  6 | from configs.parameters_der6 import *
  7 | import numba as nb
  8 | 
  9 | 
 10 | @nb.jit
 11 | def der_fn(x, t, disturbance_R, disturbance_L):
 12 |     # ----------Transferring Inv. Output Currents to Global DQ-----------------
 13 |     # dq --> \alpha \beta
 14 |     ioD1 = math.cos(0) * x[4] - math.sin(0) * x[5]
 15 |     ioQ1 = math.sin(0) * x[4] + math.cos(0) * x[5]
 16 |     ioD2 = math.cos(x[6]) * x[9] - math.sin(x[6]) * x[10]
 17 |     ioQ2 = math.sin(x[6]) * x[9] + math.cos(x[6]) * x[10]
 18 |     ioD3 = math.cos(x[11]) * x[14] - math.sin(x[11]) * x[15]
 19 |     ioQ3 = math.sin(x[11]) * x[14] + math.cos(x[11]) * x[15]
 20 |     ioD4 = math.cos(x[16]) * x[19] - math.sin(x[16]) * x[20]
 21 |     ioQ4 = math.sin(x[16]) * x[19] + math.cos(x[16]) * x[20]
 22 |     ioD5 = math.cos(x[21]) * x[24] - math.sin(x[21]) * x[25]
 23 |     ioQ5 = math.sin(x[21]) * x[24] + math.cos(x[21]) * x[25]
 24 |     ioD6 = math.cos(x[26]) * x[29] - math.sin(x[26]) * x[30]
 25 |     ioQ6 = math.sin(x[26]) * x[29] + math.cos(x[26]) * x[30]
 26 | 
 27 |     # ----------Defining Bus Voltages-----------------
 28 |     vbD1 = rN * (ioD1 - x[31])
 29 |     vbQ1 = rN * (ioQ1 - x[32])
 30 |     vbD2 = rN * (x[31] + x[49] - x[33])
 31 |     vbQ2 = rN * (x[32] + x[50] - x[34])
 32 |     vbD3 = rN * (ioD2 - x[49])
 33 |     vbQ3 = rN * (ioQ2 - x[50])
 34 |     vbD4 = rN * (x[33] + x[35] - x[37] - x[51])
 35 |     vbQ4 = rN * (x[34] + x[36] - x[38] - x[52])
 36 |     vbD5 = rN * (ioD4 - x[35])
 37 |     vbQ5 = rN * (ioQ4 - x[36])
 38 |     vbD6 = rN * (x[37] + x[39] - x[41])
 39 |     vbQ6 = rN * (x[38] + x[40] - x[42])
 40 |     vbD7 = rN * (ioD6 + x[41] - x[53] - x[43])
 41 |     vbQ7 = rN * (ioQ6 + x[42] - x[54] - x[44])
 42 |     vbD9 = rN * (x[43] + x[45] - x[55])
 43 |     vbQ9 = rN * (x[44] + x[46] - x[56])
 44 |     vbD11 = rN * (x[47] - x[45] - x[57])
 45 |     vbQ11 = rN * (x[48] - x[46] - x[58])
 46 |     vbD13 = rN * (ioD5 - x[47])
 47 |     vbQ13 = rN * (ioQ5 - x[48])
 48 |     vbD14 = rN * (ioD3 - x[39])
 49 |     vbQ14 = rN * (ioQ3 - x[40])
 50 | 
 51 |     # ----------Transferring Bus Voltages to Inv. dq-----------------
 52 |     # \alpha \beta --> dq
 53 |     vbd1 = math.cos(0) * vbD1 + math.sin(0) * vbQ1
 54 |     vbq1 = -math.sin(0) * vbD1 + math.cos(0) * vbQ1
 55 |     vbd3 = math.cos(x[6]) * vbD3 + math.sin(x[6]) * vbQ3
 56 |     vbq3 = -math.sin(x[6]) * vbD3 + math.cos(x[6]) * vbQ3
 57 |     vbd14 = math.cos(x[11]) * vbD14 + math.sin(x[11]) * vbQ14
 58 |     vbq14 = -math.sin(x[11]) * vbD14 + math.cos(x[11]) * vbQ14
 59 |     vbd5 = math.cos(x[16]) * vbD5 + math.sin(x[16]) * vbQ5
 60 |     vbq5 = -math.sin(x[16]) * vbD5 + math.cos(x[16]) * vbQ5
 61 |     vbd13 = math.cos(x[21]) * vbD13 + math.sin(x[21]) * vbQ13
 62 |     vbq13 = -math.sin(x[21]) * vbD13 + math.cos(x[21]) * vbQ13
 63 |     vbd7 = math.cos(x[26]) * vbD7 + math.sin(x[26]) * vbQ7
 64 |     vbq7 = -math.sin(x[26]) * vbD7 + math.cos(x[26]) * vbQ7
 65 | 
 66 |     wcom = x[59] - mp1 * x[2]
 67 |     ## ----------------DG1--------------------
 68 |     vod1_star = x[65] - nq1 * x[3]
 69 |     voq1_star = 0
 70 |     xdot1 = x[59] - mp1 * x[2] - wcom
 71 |     xdot2 = wc * (vod1_star * x[4] + voq1_star * x[5] - x[2])
 72 |     xdot3 = wc * (-vod1_star * x[5] + voq1_star * x[4] - x[3])
 73 |     xdot4 = (-rLc / Lc) * x[4] + wcom * x[5] + (1 / Lc) * (vod1_star - vbd1)
 74 |     xdot5 = (-rLc / Lc) * x[5] - wcom * x[4] + (1 / Lc) * (voq1_star - vbq1)
 75 | 
 76 |     ## ----------------DG2-------------------
 77 |     vod2_star = x[66] - nq2 * x[8]
 78 |     voq2_star = 0
 79 |     xdot6 = x[60] - mp2 * x[7] - wcom
 80 |     xdot7 = wc * (vod2_star * x[9] + voq2_star * x[10] - x[7])
 81 |     xdot8 = wc * (-vod2_star * x[10] + voq2_star * x[9] - x[8])
 82 |     xdot9 = (-rLc / Lc) * x[9] + wcom * x[10] + (1 / Lc) * (vod2_star - vbd3)
 83 |     xdot10 = (-rLc / Lc) * x[10] - wcom * x[9] + (1 / Lc) * (voq2_star - vbq3)
 84 | 
 85 |     ## ----------------DG3-------------------
 86 |     vod3_star = x[67] - nq3 * x[13]
 87 |     voq3_star = 0
 88 |     xdot11 = x[61] - mp3 * x[12] - wcom
 89 |     xdot12 = wc * (vod3_star * x[14] + voq3_star * x[15] - x[12])
 90 |     xdot13 = wc * (-vod3_star * x[15] + voq3_star * x[14] - x[13])
 91 |     xdot14 = (-rLc / Lc) * x[14] + wcom * x[15] + (1 / Lc) * (vod3_star - vbd14)
 92 |     xdot15 = (-rLc / Lc) * x[15] - wcom * x[14] + (1 / Lc) * (voq3_star - vbq14)
 93 | 
 94 |     ## ----------------DG4-------------------
 95 |     vod4_star = x[68] - nq4 * x[18]
 96 |     voq4_star = 0
 97 |     xdot16 = x[62] - mp4 * x[17] - wcom
 98 |     xdot17 = wc * (vod4_star * x[19] + voq4_star * x[20] - x[17])
 99 |     xdot18 = wc * (-vod4_star * x[20] + voq4_star * x[19] - x[18])
100 |     xdot19 = (-rLc / Lc) * x[19] + wcom * x[20] + (1 / Lc) * (vod4_star - vbd5)
101 |     xdot20 = (-rLc / Lc) * x[20] - wcom * x[19] + (1 / Lc) * (voq4_star - vbq5)
102 | 
103 |     # ----------------DG5-------------------
104 |     vod5_star = x[69] - nq5 * x[23]
105 |     voq5_star = 0
106 |     xdot21 = x[63] - mp5 * x[22] - wcom
107 |     xdot22 = wc * (vod5_star * x[24] + voq5_star * x[25] - x[22])
108 |     xdot23 = wc * (-vod5_star * x[25] + voq5_star * x[24] - x[23])
109 |     xdot24 = (-rLc / Lc) * x[24] + wcom * x[25] + (1 / Lc) * (vod5_star - vbd13)
110 |     xdot25 = (-rLc / Lc) * x[25] - wcom * x[24] + (1 / Lc) * (voq5_star - vbq13)
111 | 
112 |     # ----------------DG6-------------------
113 |     vod6_star = x[70] - nq6 * x[28]
114 |     voq6_star = 0
115 |     xdot26 = x[64] - mp6 * x[27] - wcom
116 |     xdot27 = wc * (vod6_star * x[29] + voq6_star * x[30] - x[27])
117 |     xdot28 = wc * (-vod6_star * x[30] + voq6_star * x[29] - x[28])
118 |     xdot29 = (-rLc / Lc) * x[29] + wcom * x[30] + (1 / Lc) * (vod6_star - vbd7)
119 |     xdot30 = (-rLc / Lc) * x[30] - wcom * x[29] + (1 / Lc) * (voq6_star - vbq7)
120 | 
121 |     # ----------------------------------------------------
122 |     # ---------line1
123 |     xdot31 = (-rline1 / Lline1) * x[31] + wcom * x[32] + (1 / Lline1) * (vbD1 - vbD2)
124 |     xdot32 = (-rline1 / Lline1) * x[32] - wcom * x[31] + (1 / Lline1) * (vbQ1 - vbQ2)
125 |     # ---------line2
126 |     xdot33 = (-rline2 / Lline2) * x[33] + wcom * x[34] + (1 / Lline2) * (vbD2 - vbD4)
127 |     xdot34 = (-rline2 / Lline2) * x[34] - wcom * x[33] + (1 / Lline2) * (vbQ2 - vbQ4)
128 |     # ---------line3
129 |     xdot35 = (-rline3 / Lline3) * x[35] + wcom * x[36] + (1 / Lline3) * (vbD5 - vbD4)
130 |     xdot36 = (-rline3 / Lline3) * x[36] - wcom * x[35] + (1 / Lline3) * (vbQ5 - vbQ4)
131 |     # ---------line4
132 |     xdot37 = (-rline4 / Lline4) * x[37] + wcom * x[38] + (1 / Lline4) * (vbD4 - vbD6)
133 |     xdot38 = (-rline4 / Lline4) * x[38] - wcom * x[37] + (1 / Lline4) * (vbQ4 - vbQ6)
134 |     # ---------line5
135 |     xdot39 = (-rline5 / Lline5) * x[39] + wcom * x[40] + (1 / Lline5) * (vbD14 - vbD6)
136 |     xdot40 = (-rline5 / Lline5) * x[40] - wcom * x[39] + (1 / Lline5) * (vbQ14 - vbQ6)
137 |     # ---------line6
138 |     xdot41 = (-rline6 / Lline6) * x[41] + wcom * x[42] + (1 / Lline6) * (vbD6 - vbD7)
139 |     xdot42 = (-rline6 / Lline6) * x[42] - wcom * x[41] + (1 / Lline6) * (vbQ6 - vbQ7)
140 |     # ---------line8
141 |     xdot43 = (-rline8 / Lline8) * x[43] + wcom * x[44] + (1 / Lline8) * (vbD7 - vbD9)
142 |     xdot44 = (-rline8 / Lline8) * x[44] - wcom * x[43] + (1 / Lline8) * (vbQ7 - vbQ9)
143 |     # ---------line10
144 |     xdot45 = (-rline10 / Lline10) * x[45] + wcom * x[46] + (1 / Lline10) * (vbD11 - vbD9)
145 |     xdot46 = (-rline10 / Lline10) * x[46] - wcom * x[45] + (1 / Lline10) * (vbQ11 - vbQ9)
146 |     # ---------line12
147 |     xdot47 = (-rline12 / Lline12) * x[47] + wcom * x[48] + (1 / Lline12) * (vbD13 - vbD11)
148 |     xdot48 = (-rline12 / Lline12) * x[48] - wcom * x[47] + (1 / Lline12) * (vbQ13 - vbQ11)
149 |     # ---------line13
150 |     xdot49 = (-rline13 / Lline13) * x[49] + wcom * x[50] + (1 / Lline13) * (vbD3 - vbD2)
151 |     xdot50 = (-rline13 / Lline13) * x[50] - wcom * x[49] + (1 / Lline13) * (vbQ3 - vbQ2)
152 | 
153 |     # -------------------------Loads------------------
154 |     # ------Load1--------
155 |     xdot51 = (-(disturbance_R[0] * Rload1) / (disturbance_L[0] * Lload1)) * x[51] + wcom * x[52] + (
156 |             1 / (disturbance_L[0] * Lload1)) * vbD4
157 |     xdot52 = (-(disturbance_R[0] * Rload1) / (disturbance_L[0] * Lload1)) * x[52] - wcom * x[51] + (
158 |             1 / (disturbance_L[0] * Lload1)) * vbQ4
159 |     # ------Load2--------
160 |     xdot53 = (-(disturbance_R[1] * Rload2 + rline7) / (disturbance_L[1] * Lload2 + Lline7)) * \
161 |              x[53] + wcom * x[54] + (1 / (disturbance_L[1] * Lload2 + Lline7)) * vbD7
162 |     xdot54 = (-(disturbance_R[1] * Rload2 + rline7) / (disturbance_L[1] * Lload2 + Lline7)) * \
163 |              x[54] - wcom * x[53] + (1 / (disturbance_L[1] * Lload2 + Lline7)) * vbQ7
164 |     # ------Load3--------
165 |     xdot55 = (-(disturbance_R[2] * Rload3 + rline9) / (disturbance_L[2] * Lload3 + Lline9)) * \
166 |              x[55] + wcom * x[56] + (1 / (disturbance_L[2] * Lload3 + Lline9)) * vbD9
167 |     xdot56 = (-(disturbance_R[2] * Rload3 + rline9) / (disturbance_L[2] * Lload3 + Lline9)) * \
168 |              x[56] - wcom * x[55] + (1 / (disturbance_L[2] * Lload3 + Lline9)) * vbQ9
169 |     # ------Load4--------
170 |     xdot57 = (-(disturbance_R[3] * Rload4 + rline11) / (disturbance_L[3] * Lload4 + Lline11)) * \
171 |              x[57] + wcom * x[58] + (1 / (disturbance_L[3] * Lload4 + Lline11)) * vbD11
172 |     xdot58 = (-(disturbance_R[3] * Rload4 + rline11) / (disturbance_L[3] * Lload4 + Lline11)) * \
173 |              x[58] - wcom * x[57] + (1 / (disturbance_L[3] * Lload4 + Lline11)) * vbQ11
174 | 
175 |     if t <= 0.4:
176 |         # this time is for the primary control
177 |         xdot59 = 0
178 |         xdot60 = 0
179 |         xdot61 = 0
180 |         xdot62 = 0
181 |         xdot63 = 0
182 |         xdot64 = 0
183 |         xdot65 = 0
184 |         xdot66 = 0
185 |         xdot67 = 0
186 |         xdot68 = 0
187 |         xdot69 = 0
188 |         xdot70 = 0
189 |     else:
190 |         # --------Controller Parameters--------
191 |         w_array = np.array([[x[59] - mp1 * x[2]],
192 |                             [x[60] - mp2 * x[7]],
193 |                             [x[61] - mp3 * x[12]],
194 |                             [x[62] - mp4 * x[17]],
195 |                             [x[63] - mp5 * x[22]],
196 |                             [x[64] - mp6 * x[27]]])
197 | 
198 |         Pratio = np.array([[mp1 * x[2]], [mp2 * x[7]], [mp3 * x[12]], [mp4 * x[17]], [mp5 * x[22]], [mp6 * x[27]]])
199 | 
200 |         # Conventional Freq Control
201 |         Synch_Mat = -1 * a_ctrl * (np.dot(L + G, w_array - np.array([[wref], [wref], [wref], [wref], [wref], [wref]]))
202 |                                    + np.dot(L, Pratio))
203 | 
204 |         xdot59 = Synch_Mat[0][0]
205 |         xdot60 = Synch_Mat[1][0]
206 |         xdot61 = Synch_Mat[2][0]
207 |         xdot62 = Synch_Mat[3][0]
208 |         xdot63 = Synch_Mat[4][0]
209 |         xdot64 = Synch_Mat[5][0]
210 |         xdot65 = 0
211 |         xdot66 = 0
212 |         xdot67 = 0
213 |         xdot68 = 0
214 |         xdot69 = 0
215 |         xdot70 = 0
216 | 
217 |     return np.array(
218 |         [0, xdot1, xdot2, xdot3, xdot4, xdot5, xdot6, xdot7, xdot8, xdot9, xdot10, xdot11, xdot12, xdot13, xdot14,
219 |          xdot15, xdot16, xdot17, xdot18, xdot19, xdot20, xdot21, xdot22, xdot23, xdot24, xdot25, xdot26, xdot27,
220 |          xdot28, xdot29, xdot30, xdot31, xdot32, xdot33, xdot34, xdot35, xdot36, xdot37, xdot38, xdot39, xdot40,
221 |          xdot41, xdot42, xdot43, xdot44, xdot45, xdot46, xdot47, xdot48, xdot49, xdot50, xdot51, xdot52, xdot53,
222 |          xdot54, xdot55, xdot56, xdot57, xdot58, xdot59, xdot60, xdot61, xdot62, xdot63, xdot64, xdot65, xdot66,
223 |          xdot67, xdot68, xdot69, xdot70, ioD1, ioQ1, vbD1, vbQ1, ioD2, ioQ2, vbD3, vbQ3, ioD3, ioQ3, vbD14, vbQ14, ioD4,
224 |          ioQ4, vbD5, vbQ5, ioD5, ioQ5, vbD13, vbQ13, ioD6, ioQ6, vbD7, vbQ7])
225 | 


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Main function for training and evaluating MARL algorithms in Powernet
  3 | """
  4 | from __future__ import print_function, division
  5 | import argparse
  6 | import configparser
  7 | import logging
  8 | from torch.utils.tensorboard.writer import SummaryWriter
  9 | from envs.Grid_envs import GridEnv
 10 | from agents.models import IA2C, IA2C_FP, MA2C_NC, IA2C_CU, MA2C_CNET, MA2C_DIAL
 11 | from trainer import (Counter, Trainer, Tester, Evaluator,
 12 |                      check_dir, copy_file, find_file,
 13 |                      init_dir, init_log, init_test_flag)
 14 | 
 15 | 
 16 | def parse_args():
 17 |     default_base_dir = './ma2c_cnet_der6'
 18 |     default_config_dir = 'configs/config_ma2c_cnet_DER6.ini'
 19 |     parser = argparse.ArgumentParser(description=('Train or evaluate policy on RL environment '
 20 |                                                   'using A2C'))
 21 |     parser.add_argument('--base-dir', type=str, required=False,
 22 |                         default=default_base_dir, help="experiment base dir")
 23 |     parser.add_argument('--option', type=str, required=False,
 24 |                         default='train', help="train or evaluate")
 25 |     parser.add_argument('--config-dir', type=str, required=False,
 26 |                         default=default_config_dir, help="experiment config path")
 27 |     parser.add_argument('--evaluation-seeds', type=str, required=False,
 28 |                         default=','.join([str(i) for i in range(2000, 2500, 100)]),
 29 |                         help="random seeds for evaluation, split by ,")
 30 |     args = parser.parse_args()
 31 |     return args
 32 | 
 33 | 
 34 | def init_agent(env, config, total_step, seed):
 35 |     if env.agent == 'ia2c':
 36 |         return IA2C(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 37 |                     total_step, config, seed=seed)
 38 |     elif env.agent == 'ia2c_fp':
 39 |         return IA2C_FP(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 40 |                        total_step, config, seed=seed)
 41 |     elif env.agent == 'ma2c_nc':
 42 |         return MA2C_NC(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 43 |                        total_step, config, seed=seed)
 44 |     elif env.agent == 'ma2c_cnet':
 45 |         # CommNet, it calculates the mean of all messages instead of encoding them
 46 |         return MA2C_CNET(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 47 |                          total_step, config, seed=seed)
 48 |     elif env.agent == 'ma2c_cu':
 49 |         """
 50 |         ConseNet: the critic is fully decentralized
 51 |         but each takes global observations and performs consensus updates
 52 |         """
 53 |         return IA2C_CU(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 54 |                        total_step, config, seed=seed)
 55 |     elif env.agent == 'ma2c_dial':
 56 |         """
 57 |         the message is generated together with action-value estimation 
 58 |         by each DQN agent, then it is encoded and summed with other 
 59 |         input signals at the receiver side.
 60 |         """
 61 |         return MA2C_DIAL(env.n_s_ls, env.n_a_ls, env.neighbor_mask, env.distance_mask, env.coop_gamma,
 62 |                          total_step, config, seed=seed)
 63 |     else:
 64 |         return None
 65 | 
 66 | 
 67 | def train(args):
 68 |     base_dir = args.base_dir
 69 |     dirs = init_dir(base_dir)
 70 |     init_log(dirs['log'])
 71 |     config_dir = args.config_dir
 72 |     copy_file(config_dir, dirs['data'])
 73 |     config = configparser.ConfigParser()
 74 |     config.read(config_dir)
 75 | 
 76 |     # init env
 77 |     seed = config.getint('ENV_CONFIG', 'seed')
 78 |     env = GridEnv(config['ENV_CONFIG'], random_seed=seed)
 79 |     logging.info('Training: a dim %r, agent dim: %d' % (env.n_a_ls, env.n_agent))
 80 | 
 81 |     # init step counter
 82 |     total_step = int(config.getfloat('TRAIN_CONFIG', 'total_step'))
 83 |     test_step = int(config.getfloat('TRAIN_CONFIG', 'test_interval'))
 84 |     log_step = int(config.getfloat('TRAIN_CONFIG', 'log_interval'))
 85 |     global_counter = Counter(total_step, test_step, log_step)
 86 | 
 87 |     # init centralized or multi agent
 88 |     torch_seed = config.getint('MODEL_CONFIG', 'torch_seed')
 89 |     model = init_agent(env, config['MODEL_CONFIG'], total_step, torch_seed)
 90 |     model.load(dirs['model'], train_mode=True)
 91 | 
 92 |     # disable multi-threading for safe SUMO implementation
 93 |     summary_writer = SummaryWriter(dirs['log'], flush_secs=10000)
 94 |     trainer = Trainer(env, model, global_counter, summary_writer, output_path=dirs['data'], model_path=dirs['model'])
 95 |     trainer.run()
 96 | 
 97 |     # save model
 98 |     final_step = global_counter.cur_step
 99 |     model.save(dirs['model'], final_step)
100 |     summary_writer.close()
101 | 
102 | 
103 | def evaluate_fn(agent_dir, output_dir, seeds):
104 |     agent = agent_dir.split('/')[-1]
105 |     if not check_dir(agent_dir):
106 |         logging.error('Evaluation: %s does not exist!' % agent)
107 |         return
108 |     # load config file 
109 |     config_dir = find_file(agent_dir + '/data/')
110 |     if not config_dir:
111 |         return
112 |     config = configparser.ConfigParser()
113 |     config.read(config_dir)
114 | 
115 |     # init env
116 |     env = GridEnv(config['ENV_CONFIG'], train_mode=False)
117 |     env.init_test_seeds(seeds)
118 | 
119 |     # load model for agent
120 |     model = init_agent(env, config['MODEL_CONFIG'], 0, 0)
121 |     if model is None:
122 |         return
123 |     model_dir = agent_dir + '/model/'
124 |     if not model.load(model_dir):
125 |         return
126 |     # collect evaluation data
127 |     evaluator = Evaluator(env, model, output_dir)
128 |     evaluator.run()
129 | 
130 | 
131 | def evaluate(args):
132 |     base_dir = args.base_dir
133 |     dirs = init_dir(base_dir, pathes=['eva_data', 'eva_log', 'eva_data/voltage'])
134 |     init_log(dirs['eva_log'])
135 |     output_dir = dirs['eva_data']
136 |     # enforce the same evaluation seeds across agents
137 |     seeds = args.evaluation_seeds
138 |     logging.info('Evaluation: random seeds: %s' % seeds)
139 |     if not seeds:
140 |         seeds = []
141 |     else:
142 |         seeds = [int(s) for s in seeds.split(',')]
143 |     evaluate_fn(base_dir, output_dir, seeds)
144 | 
145 | 
146 | if __name__ == '__main__':
147 |     args = parse_args()
148 |     if args.option == 'train':
149 |         train(args)
150 |     else:
151 |         evaluate(args)
152 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | numba
3 | pandas


--------------------------------------------------------------------------------
/trainer.py:
--------------------------------------------------------------------------------
  1 | import itertools
  2 | import logging
  3 | import numpy as np
  4 | import time
  5 | import os
  6 | import pandas as pd
  7 | import subprocess
  8 | from shutil import copy
  9 | 
 10 | 
 11 | def check_dir(cur_dir):
 12 |     if not os.path.exists(cur_dir):
 13 |         return False
 14 |     return True
 15 | 
 16 | 
 17 | def copy_file(src_dir, tar_dir):
 18 |     copy(src_dir, tar_dir)
 19 |     env = 'envs/Grid_envs.py'
 20 |     copy(env, tar_dir)
 21 |     policies = 'agents/policies.py'
 22 |     copy(policies, tar_dir)
 23 |     models = 'agents/models.py'
 24 |     copy(models, tar_dir)
 25 |     main = 'main.py'
 26 |     copy(main, tar_dir)
 27 | 
 28 | 
 29 | def find_file(cur_dir, suffix='.ini'):
 30 |     for file in os.listdir(cur_dir):
 31 |         if file.endswith(suffix):
 32 |             return cur_dir + '/' + file
 33 |     logging.error('Cannot find %s file' % suffix)
 34 |     return None
 35 | 
 36 | 
 37 | def init_dir(base_dir, pathes=['log', 'data', 'model']):
 38 |     if not os.path.exists(base_dir):
 39 |         os.mkdir(base_dir)
 40 |     dirs = {}
 41 |     for path in pathes:
 42 |         cur_dir = base_dir + '/%s/' % path
 43 |         if not os.path.exists(cur_dir):
 44 |             os.mkdir(cur_dir)
 45 |         dirs[path] = cur_dir
 46 |     return dirs
 47 | 
 48 | 
 49 | def init_log(log_dir):
 50 |     logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
 51 |                         level=logging.INFO,
 52 |                         handlers=[
 53 |                             logging.FileHandler('%s/%d.log' % (log_dir, time.time())),
 54 |                             logging.StreamHandler()
 55 |                         ])
 56 | 
 57 | 
 58 | def init_test_flag(test_mode):
 59 |     if test_mode == 'no_test':
 60 |         return False, False
 61 |     if test_mode == 'in_train_test':
 62 |         return True, False
 63 |     if test_mode == 'after_train_test':
 64 |         return False, True
 65 |     if test_mode == 'all_test':
 66 |         return True, True
 67 |     return False, False
 68 | 
 69 | 
 70 | class Counter:
 71 |     def __init__(self, total_step, test_step, log_step):
 72 |         self.counter = itertools.count(1)
 73 |         self.cur_step = 0
 74 |         self.cur_test_step = 0
 75 |         self.total_step = total_step
 76 |         self.test_step = test_step
 77 |         self.log_step = log_step
 78 |         self.stop = False
 79 | 
 80 |     def next(self):
 81 |         self.cur_step = next(self.counter)
 82 |         return self.cur_step
 83 | 
 84 |     def should_test(self):
 85 |         test = False
 86 |         if (self.cur_step - self.cur_test_step) >= self.test_step:
 87 |             test = True
 88 |             self.cur_test_step = self.cur_step
 89 |         return test
 90 | 
 91 |     def should_log(self):
 92 |         return self.cur_step % self.log_step == 0
 93 | 
 94 |     def should_stop(self):
 95 |         if self.cur_step >= self.total_step:
 96 |             return True
 97 |         return self.stop
 98 | 
 99 | 
100 | class Trainer:
101 |     def __init__(self, env, model, global_counter, summary_writer, output_path=None, model_path=None):
102 |         self.cur_step = 0
103 |         self.global_counter = global_counter
104 |         self.env = env
105 |         self.agent = self.env.agent
106 |         self.model = model
107 |         self.n_step = self.model.n_step
108 |         self.summary_writer = summary_writer
109 |         assert self.env.T % self.n_step == 0
110 |         self.data = []
111 |         self.episode_rewards = [0]
112 |         self.output_path = output_path
113 |         self.model_path = model_path
114 |         self.env.train_mode = True
115 | 
116 |     def _add_summary(self, reward, global_step, is_train=True):
117 |         if is_train:
118 |             self.summary_writer.add_scalar('train_reward', reward, global_step=global_step)
119 |         else:
120 |             self.summary_writer.add_scalar('test_reward', reward, global_step=global_step)
121 | 
122 |     def _get_policy(self, ob, done, mode='train'):
123 |         if self.agent.startswith('ma2c'):
124 |             self.ps = self.env.get_fingerprint()
125 |             policy = self.model.forward(ob, done, self.ps)
126 |         else:
127 |             policy = self.model.forward(ob, done)
128 |         action = []
129 |         for pi in policy:
130 |             if mode == 'train':
131 |                 action.append(np.random.choice(np.arange(len(pi)), p=pi))
132 |             else:
133 |                 action.append(np.argmax(pi))
134 |         return policy, np.array(action)
135 | 
136 |     def _get_value(self, ob, done, action):
137 |         if self.agent.startswith('ma2c'):
138 |             value = self.model.forward(ob, done, self.ps, np.array(action), 'v')
139 |         else:
140 |             self.naction = self.env.get_neighbor_action(action)  # action=[2,3,2,2]; nactions =[[3], [2,2], [3,2], [2]]
141 |             if not self.naction:
142 |                 self.naction = np.nan
143 |             value = self.model.forward(ob, done, self.naction, 'v')
144 |         return value
145 | 
146 |     def _log_episode(self, global_step, mean_reward, std_reward):
147 |         log = {'agent': self.agent,
148 |                'step': global_step,
149 |                'test_id': -1,
150 |                'avg_reward': mean_reward,
151 |                'std_reward': std_reward}
152 |         self.data.append(log)
153 |         self._add_summary(mean_reward, global_step)
154 |         self.summary_writer.flush()
155 | 
156 |     def explore(self, prev_ob, prev_done):
157 |         # run a batch of steps
158 |         ob = prev_ob
159 |         done = prev_done
160 |         for _ in range(self.n_step):
161 |             # pre-decision
162 |             policy, action = self._get_policy(ob, done)
163 |             # post-decision
164 |             value = self._get_value(ob, done, action)
165 |             # transition
166 |             self.env.update_fingerprint(policy)
167 |             next_ob, reward, done, global_reward = self.env.step(action)
168 |             self.episode_rewards[-1] += global_reward
169 |             self.global_counter.next()
170 |             self.cur_step += 1
171 |             # collect experience
172 |             if self.agent.startswith('ma2c'):
173 |                 self.model.add_transition(ob, self.ps, action, reward, value, done)
174 |             else:
175 |                 self.model.add_transition(ob, self.naction, action, reward, value, done)
176 |             if done:
177 |                 break
178 |             ob = next_ob
179 |         if done:
180 |             R = np.zeros(self.model.n_agent)
181 |         else:
182 |             _, action = self._get_policy(ob, done)
183 |             R = self._get_value(ob, done, action)
184 |         return ob, done, R
185 | 
186 |     def perform(self, test_ind):
187 |         # do a test
188 |         ob = self.env.reset(test_ind=test_ind)
189 |         rewards = []
190 |         # note this done is pre-decision to reset LSTM states!
191 |         done = True
192 |         self.model.reset()
193 |         while True:
194 |             if self.agent == 'greedy':
195 |                 action = self.model.forward(ob)
196 |             else:
197 |                 policy, action = self._get_policy(ob, done, mode='test')
198 |                 self.env.update_fingerprint(policy)
199 |             next_ob, reward, done, global_reward = self.env.step(action, mode='train')  # with disturbance
200 |             rewards.append(global_reward)
201 |             if done:
202 |                 break
203 |             ob = next_ob
204 |         mean_reward = np.mean(np.array(rewards))
205 |         std_reward = np.std(np.array(rewards))
206 |         return mean_reward, std_reward
207 | 
208 |     def run(self):
209 |         while not self.global_counter.should_stop():
210 |             ob = self.env.reset()
211 |             # note this done is pre-decision to reset LSTM states!
212 |             done = True
213 |             self.model.reset()
214 |             self.cur_step = 0
215 |             while True:
216 |                 ob, done, R = self.explore(ob, done)
217 |                 dt = self.env.T - self.cur_step
218 |                 global_step = self.global_counter.cur_step
219 |                 self.model.backward(R, dt, self.summary_writer, global_step)
220 |                 # termination
221 |                 if done:
222 |                     if self.global_counter.should_log():
223 |                         # logging
224 |                         mean_reward = round(np.mean(self.episode_rewards[-101:-1]) / self.env.T, 3)
225 |                         logging.info('''Training: global step %d, episode step %d,train r: %.2f, done: %r''' %
226 |                                      (global_step, len(self.episode_rewards), mean_reward, done))
227 |                         np.save('{}'.format('states'), np.array(self.env.states))
228 |                         # save the counter for steps for each epoch when first time reach a normal voltage
229 |                         np.save(self.output_path + '{}'.format('step_count'), self.env.step_list)
230 |                         np.save(self.output_path + '{}'.format('episode_rewards'), self.episode_rewards)
231 |                         # save model
232 |                         self.model.save(self.model_path, global_step)
233 |                     self.env.terminate()
234 |                     self.episode_rewards.append(0)
235 |                     break
236 |             rewards = np.array(self.episode_rewards[-101:-1]) / self.env.T
237 |             mean_reward = np.mean(rewards)
238 |             std_reward = np.std(rewards)
239 |             self._log_episode(global_step, mean_reward, std_reward)
240 |         df = pd.DataFrame(self.data)
241 |         df.to_csv(self.output_path + 'train_reward.csv')
242 | 
243 | 
244 | class Tester(Trainer):
245 |     def __init__(self, env, model, global_counter, summary_writer, output_path):
246 |         super().__init__(env, model, global_counter, summary_writer)
247 |         self.env.train_mode = False
248 |         self.test_num = self.env.test_num
249 |         self.output_path = output_path
250 |         self.data = []
251 |         logging.info('Testing: total test num: %d' % self.test_num)
252 | 
253 |     def run_offline(self):
254 |         self.env.cur_episode = 0
255 |         rewards = []
256 |         for test_ind in range(self.test_num):
257 |             rewards.append(self.perform(test_ind))
258 |         avg_reward = np.mean(np.array(rewards))
259 |         logging.info('Offline testing: avg R: %.2f' % avg_reward)
260 |         self.env.output_data()
261 | 
262 |     def run_online(self, coord):
263 |         self.env.cur_episode = 0
264 |         while not coord.should_stop():
265 |             time.sleep(30)
266 |             if self.global_counter.should_test():
267 |                 rewards = []
268 |                 global_step = self.global_counter.cur_step
269 |                 for test_ind in range(self.test_num):
270 |                     cur_reward = self.perform(test_ind)
271 |                     self.env.terminate()
272 |                     rewards.append(cur_reward)
273 |                     log = {'agent': self.agent,
274 |                            'step': global_step,
275 |                            'test_id': test_ind,
276 |                            'reward': cur_reward}
277 |                     self.data.append(log)
278 |                 avg_reward = np.mean(np.array(rewards))
279 |                 self._add_summary(avg_reward, global_step)
280 |                 logging.info('Testing: global step %d, avg R: %.2f' %
281 |                              (global_step, avg_reward))
282 |                 # self.global_counter.update_test(avg_reward)
283 |         df = pd.DataFrame(self.data)
284 |         df.to_csv(self.output_path + 'train_reward.csv')
285 | 
286 | 
287 | class Evaluator(Tester):
288 |     def __init__(self, env, model, output_path):
289 |         self.env = env
290 |         self.model = model
291 |         self.agent = self.env.agent
292 |         self.env.train_mode = False
293 |         self.test_num = self.env.test_num
294 |         self.output_path = output_path
295 | 
296 |     def run(self):
297 |         self.env.cur_episode = 0
298 |         time.sleep(1)
299 |         rewards = 0
300 |         for test_ind in range(self.test_num):
301 |             reward, _ = self.perform(test_ind)
302 |             logging.info('test %i, avg reward %.2f' % (test_ind, reward))
303 |             rewards += reward
304 |             time.sleep(0.5)
305 |             self.env.output_data(self.output_path, test_ind)
306 |         print('Testing finished, avg reward %.2f' % (rewards / self.test_num))
307 | 


--------------------------------------------------------------------------------