├── omnet └── router │ ├── package.ned │ ├── .tkenvrc │ ├── .oppbuildspec │ ├── NetworkAll.matrix │ ├── omnetpp.ini │ ├── NetworkAll.ports │ ├── messages │ ├── ControlPacket.msg │ ├── TimerNextPacket.msg │ └── DataPacket.msg │ ├── TrafficController.ned │ ├── NodeQueue.ned │ ├── Balancer.ned │ ├── Routing.ned │ ├── Application.ned │ ├── .project │ ├── NodeQueue.h │ ├── Receiver.ned │ ├── Router.ned │ ├── TrafficController.h │ ├── Routing.h │ ├── Balancer.h │ ├── Application.h │ ├── Node.ned │ ├── Server.ned │ ├── Statistic.h │ ├── Routing.cc │ ├── TrafficController.cc │ ├── NodeQueue.cc │ ├── Balancer.cc │ ├── Application.cc │ ├── NetworkAll.ned │ ├── Makefile │ ├── Statistic.cc │ └── .cproject ├── requirements.txt ├── .gitignore ├── sampler ├── NetworkAll.matrix └── path.py ├── DDPG.json ├── OU.py ├── LICENSE.txt ├── ReplayBuffer.py ├── ActorNetwork.py ├── CriticNetwork.py ├── README.md ├── Traffic.py ├── helper.py ├── ddpg.py └── Environment.py /omnet/router/package.ned: -------------------------------------------------------------------------------- 1 | package networksimulator; 2 | 3 | @license(LGPL); 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | h5py==2.7.0 2 | Keras==1.2.2 3 | networkx==1.11 4 | numpy==1.12.1 5 | scipy==0.19.0 6 | tensorflow==1.0.1 7 | -------------------------------------------------------------------------------- /omnet/router/.tkenvrc: -------------------------------------------------------------------------------- 1 | # Partial OMNeT++/Tkenv config file -- see $HOME/.tkenvrc as well 2 | config default-configname {} 3 | config default-runnumber {} 4 | 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | *.pyo 4 | __pycache__ 5 | *.h5 6 | *.json 7 | runs/ 8 | check* 9 | omnet/*/out/ 10 | omnet/*/networkRL 11 | omnet/*/messages/*.cc 12 | omnet/*/messages/*.h 13 | omnet/*/NetworkAll* 14 | traffic/ 15 | -------------------------------------------------------------------------------- /omnet/router/.oppbuildspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /sampler/NetworkAll.matrix: -------------------------------------------------------------------------------- 1 | 0 1 1 1 0 0 0 0 0 0 0 0 0 0 2 | 1 0 1 0 0 0 0 1 0 0 0 0 0 0 3 | 1 1 0 0 0 1 0 0 0 0 0 0 0 0 4 | 1 0 0 0 1 0 0 0 1 0 0 0 0 0 5 | 0 0 0 1 0 1 1 0 0 0 0 0 0 0 6 | 0 0 1 0 1 0 0 0 0 0 1 0 1 0 7 | 0 0 0 0 1 0 0 1 0 0 0 0 0 0 8 | 0 1 0 0 0 0 1 0 0 1 0 0 0 0 9 | 0 0 0 1 0 0 0 0 0 0 0 1 0 1 10 | 0 0 0 0 0 0 0 1 0 0 1 1 0 1 11 | 0 0 0 0 0 1 0 0 0 1 0 0 0 0 12 | 0 0 0 0 0 0 0 0 1 1 0 0 1 0 13 | 0 0 0 0 0 1 0 0 0 0 0 1 0 1 14 | 0 0 0 0 0 0 0 0 1 1 0 0 1 0 15 | -------------------------------------------------------------------------------- /omnet/router/NetworkAll.matrix: -------------------------------------------------------------------------------- 1 | 0 1 1 1 0 0 0 0 0 0 0 0 0 0 2 | 1 0 1 0 0 0 0 1 0 0 0 0 0 0 3 | 1 1 0 0 0 1 0 0 0 0 0 0 0 0 4 | 1 0 0 0 1 0 0 0 1 0 0 0 0 0 5 | 0 0 0 1 0 1 1 0 0 0 0 0 0 0 6 | 0 0 1 0 1 0 0 0 0 0 1 0 1 0 7 | 0 0 0 0 1 0 0 1 0 0 0 0 0 0 8 | 0 1 0 0 0 0 1 0 0 1 0 0 0 0 9 | 0 0 0 1 0 0 0 0 0 0 0 1 0 1 10 | 0 0 0 0 0 0 0 1 0 0 1 1 0 1 11 | 0 0 0 0 0 1 0 0 0 1 0 0 0 0 12 | 0 0 0 0 0 0 0 0 1 1 0 0 1 0 13 | 0 0 0 0 0 1 0 0 0 0 0 1 0 1 14 | 0 0 0 0 0 0 0 0 1 1 0 0 1 0 15 | -------------------------------------------------------------------------------- /omnet/router/omnetpp.ini: -------------------------------------------------------------------------------- 1 | [General] 2 | 3 | repeat = 1 4 | 5 | seed-set = ${runnumber} 6 | simtime-scale = -12 7 | 8 | # Simulation 9 | **.simulationDuration = 1024 # originally: 4050 10 | 11 | # Traffic 12 | **.lambda = 1 13 | **.generation = ${generation = 0} 14 | 15 | warmup-period = 1s 16 | 17 | **.module-eventlog-recording = false 18 | **.vector-recording = false 19 | 20 | **.result-recording-modes = count, -mean, -max, -min 21 | **.app.**.scalar-recording = false 22 | **.routing.**.scalar-recording = false 23 | **.scalar-recording = false 24 | 25 | 26 | network = NetworkAll 27 | -------------------------------------------------------------------------------- /omnet/router/NetworkAll.ports: -------------------------------------------------------------------------------- 1 | -1 0 1 2 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 2 | 0 -1 1 -1 -1 -1 -1 2 -1 -1 -1 -1 -1 -1 3 | 0 1 -1 -1 -1 2 -1 -1 -1 -1 -1 -1 -1 -1 4 | 0 -1 -1 -1 1 -1 -1 -1 2 -1 -1 -1 -1 -1 5 | -1 -1 -1 0 -1 1 2 -1 -1 -1 -1 -1 -1 -1 6 | -1 -1 0 -1 1 -1 -1 -1 -1 -1 2 -1 3 -1 7 | -1 -1 -1 -1 0 -1 -1 1 -1 -1 -1 -1 -1 -1 8 | -1 0 -1 -1 -1 -1 1 -1 -1 2 -1 -1 -1 -1 9 | -1 -1 -1 0 -1 -1 -1 -1 -1 -1 -1 1 -1 2 10 | -1 -1 -1 -1 -1 -1 -1 0 -1 -1 1 2 -1 3 11 | -1 -1 -1 -1 -1 0 -1 -1 -1 1 -1 -1 -1 -1 12 | -1 -1 -1 -1 -1 -1 -1 -1 0 1 -1 -1 2 -1 13 | -1 -1 -1 -1 -1 0 -1 -1 -1 -1 -1 1 -1 2 14 | -1 -1 -1 -1 -1 -1 -1 -1 0 1 -1 -1 2 -1 15 | -------------------------------------------------------------------------------- /DDPG.json: -------------------------------------------------------------------------------- 1 | { 2 | "ACTIVE_NODES": 14, 3 | "ACTUM": "NEW", 4 | "BATCH_SIZE": 32, 5 | "BN": "reward", 6 | "BUFFER_SIZE": 1600, 7 | "ENV": "label", 8 | "EPISODE_COUNT": 100, 9 | "EXPLORE": 0.8, 10 | "GAMMA": 0.99, 11 | "HACTI": "selu", 12 | "HIDDEN1_UNITS": 91, 13 | "HIDDEN2_UNITS": 42, 14 | "LRA": 0.0001, 15 | "LRC": 0.001, 16 | "MAX_DELTA": 0.1, 17 | "MAX_STEPS": 1000, 18 | "MU": 0.0, 19 | "PRAEMIUM": "AVG", 20 | "PRINT": false, 21 | "ROUTING": "Linkweight", 22 | "RSEED": null, 23 | "SIGMA": 0.4, 24 | "STATUM": "T", 25 | "TAU": 0.001, 26 | "THETA": 0.2, 27 | "TRAFFIC": "EXP" 28 | } 29 | -------------------------------------------------------------------------------- /omnet/router/messages/ControlPacket.msg: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | // 17 | // TODO generated message class 18 | // 19 | packet ControlPacket { 20 | 21 | double data; 22 | } 23 | -------------------------------------------------------------------------------- /omnet/router/messages/TimerNextPacket.msg: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | // 17 | // TODO generated message class 18 | // 19 | message TimerNextPacket { 20 | int target; 21 | double lambda; 22 | } 23 | -------------------------------------------------------------------------------- /OU.py: -------------------------------------------------------------------------------- 1 | """ 2 | OU.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | __credits__ = ["https://github.com/yanpanlau", "https://gist.github.com/jimfleming/9a62b2f7ed047ff78e95b5398e955b9e"] 6 | 7 | import numpy as np 8 | from scipy.stats import norm 9 | 10 | 11 | # Ornstein-Uhlenbeck Process 12 | class OU(object): 13 | 14 | def __init__(self, processes, mu=0, theta=0.15, sigma=0.3): 15 | self.dt = 0.1 16 | self.mu = mu 17 | self.theta = theta 18 | self.sigma = sigma 19 | self.processes = processes 20 | self.state = np.ones(self.processes) * self.mu 21 | 22 | def reset(self): 23 | self.state = np.ones(self.processes) * self.mu 24 | 25 | def evolve(self): 26 | X = self.state 27 | dw = norm.rvs(scale=self.dt, size=self.processes) 28 | dx = self.theta * (self.mu - X) * self.dt + self.sigma * dw 29 | self.state = X + dx 30 | return self.state 31 | -------------------------------------------------------------------------------- /sampler/path.py: -------------------------------------------------------------------------------- 1 | """ 2 | path.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | 6 | import numpy as np 7 | from itertools import product 8 | import sys 9 | import networkx as nx 10 | 11 | 12 | G = nx.Graph(np.loadtxt(sys.argv[1], dtype=int)) 13 | 14 | n_edges = len(G.edges()) 15 | 16 | num = 1e5 17 | good = set() 18 | 19 | i = 0 20 | while len(good) < num: 21 | i += 1 22 | matrix = np.full([len(G.nodes())]*2, -1, dtype=int) 23 | 24 | for u, v, d in G.edges(data=True): 25 | d['weight'] = np.random.random() 26 | all_shortest = nx.all_pairs_dijkstra_path(G) 27 | 28 | for s, d in product(G.nodes(), G.nodes()): 29 | if s != d: 30 | matrix[s][d] = all_shortest[s][d][1] 31 | 32 | candidate = ','.join(map(str, matrix.ravel())) 33 | 34 | l = len(good) 35 | good.add(candidate) 36 | 37 | if len(good) == l: 38 | print('dup!') 39 | continue 40 | 41 | print(len(good), i) 42 | with open('path_'+ str(sys.argv[1]) +'.csv', 'a') as file: 43 | file.write(candidate + '\n') 44 | -------------------------------------------------------------------------------- /omnet/router/TrafficController.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | // 19 | // TODO auto-generated module 20 | // 21 | simple TrafficController 22 | { 23 | parameters: 24 | double nodeRatio; 25 | int numNodes; 26 | int id; 27 | string folderName; 28 | 29 | gates: 30 | output out[]; 31 | } 32 | -------------------------------------------------------------------------------- /omnet/router/NodeQueue.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | // 19 | // TODO auto-generated module 20 | // 21 | simple NodeQueue 22 | { 23 | parameters: 24 | int capacity @unit(byte) = default (1024B); 25 | @display("i=block/queue"); 26 | gates: 27 | input in; 28 | output out; 29 | inout line; 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 UPC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /omnet/router/Balancer.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | // 19 | // TODO auto-generated module 20 | // 21 | simple Balancer 22 | { 23 | parameters: 24 | int id; 25 | int numPorts = default(sizeof(in)); 26 | int numTx; 27 | int numNodes; 28 | string folderName; 29 | 30 | gates: 31 | input localIn[]; 32 | output localOut[]; 33 | input in[]; 34 | output out[]; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /omnet/router/Routing.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | // 19 | // TODO auto-generated module 20 | // 21 | simple Routing 22 | { 23 | parameters: 24 | int id; 25 | int numPorts = default(sizeof(in)); 26 | int numTx; 27 | int numNodes; 28 | string folderName; 29 | 30 | gates: 31 | input localIn[]; 32 | output localOut[]; 33 | input in[]; 34 | output out[]; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /omnet/router/Application.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | // 19 | // TODO auto-generated module 20 | // 21 | simple Application 22 | { 23 | 24 | parameters: 25 | int id; 26 | double lambda; 27 | int dest; 28 | int generation; 29 | int numNodes; 30 | 31 | 32 | double simulationDuration; 33 | 34 | 35 | @display("i=block/browser"); 36 | 37 | 38 | gates: 39 | output out; 40 | input tcontrol; 41 | } 42 | -------------------------------------------------------------------------------- /omnet/router/messages/DataPacket.msg: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | // 17 | // TODO generated message class 18 | // 19 | packet DataPacket { 20 | 21 | int srcNode; 22 | int dstNode; 23 | 24 | int ttl; 25 | int lastRouter; 26 | int l2; 27 | int l3; 28 | int l4; 29 | 30 | int lastQueue; 31 | int q2; 32 | int q3; 33 | int q4; 34 | int q5; 35 | 36 | double lastTS; 37 | double t2; 38 | double t3; 39 | double t4; 40 | double t5; 41 | 42 | int routing; 43 | } 44 | -------------------------------------------------------------------------------- /omnet/router/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | JosepSimulator 4 | 5 | 6 | 7 | 8 | 9 | org.omnetpp.cdt.MakefileBuilder 10 | 11 | 12 | 13 | 14 | org.omnetpp.scave.builder.vectorfileindexer 15 | 16 | 17 | 18 | 19 | org.eclipse.cdt.managedbuilder.core.genmakebuilder 20 | clean,full,incremental, 21 | 22 | 23 | 24 | 25 | org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder 26 | full,incremental, 27 | 28 | 29 | 30 | 31 | 32 | org.eclipse.cdt.core.cnature 33 | org.eclipse.cdt.core.ccnature 34 | org.eclipse.cdt.managedbuilder.core.managedBuildNature 35 | org.eclipse.cdt.managedbuilder.core.ScannerConfigNature 36 | org.omnetpp.main.omnetppnature 37 | 38 | 39 | -------------------------------------------------------------------------------- /omnet/router/NodeQueue.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef __NETWORKSIMULATOR_QUEUE_H_ 17 | #define __NETWORKSIMULATOR_QUEUE_H_ 18 | 19 | #include 20 | #include 21 | #include "DataPacket_m.h" 22 | #include "Statistic.h" 23 | 24 | using namespace std; 25 | 26 | 27 | /** 28 | * TODO - Generated class 29 | */ 30 | class NodeQueue : public cSimpleModule 31 | { 32 | 33 | public: 34 | NodeQueue(); 35 | virtual ~NodeQueue(); 36 | 37 | protected: 38 | virtual void initialize(); 39 | virtual void handleMessage(cMessage *msg); 40 | 41 | private: 42 | queue portQueue; 43 | cMessage* endTxMsg; 44 | int deleted; 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /omnet/router/Receiver.ned: -------------------------------------------------------------------------------- 1 | 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | module Receiver 19 | { 20 | parameters: 21 | int id; 22 | int numTx; 23 | int numNodes; 24 | @display("i=misc/node_vs,gold;bgb=224,246"); 25 | 26 | 27 | 28 | 29 | gates: 30 | inout port[]; 31 | 32 | submodules: 33 | routing: Routing { 34 | numTx = numTx; 35 | numNodes = numNodes; 36 | id = id; 37 | }; 38 | queue[sizeof(port)]: NodeQueue; 39 | 40 | connections: 41 | 42 | for i=0..sizeof(port)-1 { 43 | routing.out++ --> queue[i].in; 44 | routing.in++ <-- queue[i].out; 45 | queue[i].line <--> port[i]; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /omnet/router/Router.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | module Router 19 | { 20 | parameters: 21 | int numTx; 22 | int numNodes; 23 | int id; 24 | 25 | @display("i=misc/node_vs,gold;bgb=224,246"); 26 | 27 | 28 | 29 | 30 | gates: 31 | inout port[]; 32 | 33 | submodules: 34 | routing: Routing { 35 | numTx = numTx; 36 | numNodes = numNodes; 37 | id = id; 38 | }; 39 | queue[sizeof(port)]: NodeQueue; 40 | 41 | connections: 42 | 43 | for i=0..sizeof(port)-1 { 44 | routing.out++ --> queue[i].in; 45 | routing.in++ <-- queue[i].out; 46 | queue[i].line <--> port[i]; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /ReplayBuffer.py: -------------------------------------------------------------------------------- 1 | """ 2 | ReplayBuffer.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | __credits__ = "https://github.com/yanpanlau" 6 | 7 | from collections import deque 8 | import numpy as np 9 | 10 | 11 | class ReplayBuffer(object): 12 | 13 | def __init__(self, buffer_size): 14 | self.buffer_size = buffer_size 15 | self.num_experiences = 0 16 | self.buffer = deque() 17 | 18 | def getBatch(self, batch_size): 19 | # Randomly sample batch_size examples 20 | if self.num_experiences < batch_size: 21 | indices = np.random.choice(len(self.buffer), self.num_experiences) 22 | else: 23 | indices = np.random.choice(len(self.buffer), batch_size) 24 | return np.asarray(self.buffer)[indices] 25 | 26 | def size(self): 27 | return self.buffer_size 28 | 29 | def add(self, state, action, reward, new_state, done): 30 | experience = (state, action, reward, new_state, done) 31 | if self.num_experiences < self.buffer_size: 32 | self.buffer.append(experience) 33 | self.num_experiences += 1 34 | else: 35 | self.buffer.popleft() 36 | self.buffer.append(experience) 37 | 38 | def count(self): 39 | # if buffer is full, return buffer size 40 | # otherwise, return experience counter 41 | return self.num_experiences 42 | 43 | def erase(self): 44 | self.buffer = deque() 45 | self.num_experiences = 0 46 | -------------------------------------------------------------------------------- /omnet/router/TrafficController.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef __SCALEFREE_TRAFFICCONTROLLER_H_ 17 | #define __SCALEFREE_TRAFFICCONTROLLER_H_ 18 | 19 | #include 20 | #include "ControlPacket_m.h" 21 | using namespace std; 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | /** 30 | * TODO - Generated class 31 | */ 32 | class TrafficController : public cSimpleModule 33 | { 34 | private: 35 | double nodeRatio; 36 | int numNodes; 37 | int id; 38 | double flowRatio[100]; 39 | string folderName; 40 | 41 | protected: 42 | virtual void initialize(); 43 | virtual void handleMessage(cMessage *msg); 44 | void getTrafficInfo(int id, double rData[]); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /omnet/router/Routing.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef __NETWORKSIMULATOR_NROUTING_H_ 17 | #define __NETWORKSIMULATOR_NROUTING_H_ 18 | 19 | #include 20 | #include "DataPacket_m.h" 21 | #include "Statistic.h" 22 | 23 | using namespace std; 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | 32 | /** 33 | * TODO - Generated class 34 | */ 35 | class Routing : public cSimpleModule 36 | { 37 | private: 38 | int numPorts; 39 | int id; 40 | int numTx; 41 | int numNodes; 42 | int outPort[100]; 43 | string folderName; 44 | 45 | 46 | 47 | public: 48 | Routing(); 49 | virtual ~Routing(); 50 | 51 | protected: 52 | virtual void initialize(); 53 | virtual void handleMessage(cMessage *msg); 54 | void getRoutingInfo(int id, int rData[]); 55 | 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /omnet/router/Balancer.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef __NETWORKSIMULATOR_Balancer_H_ 17 | #define __NETWORKSIMULATOR_Balancer_H_ 18 | 19 | #include 20 | #include "DataPacket_m.h" 21 | #include "Statistic.h" 22 | 23 | using namespace std; 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | 32 | /** 33 | * TODO - Generated class 34 | */ 35 | class Balancer : public cSimpleModule 36 | { 37 | private: 38 | int numPorts; 39 | int id; 40 | int numTx; 41 | int numNodes; 42 | double balance[100]; 43 | string folderName; 44 | 45 | 46 | 47 | public: 48 | Balancer(); 49 | virtual ~Balancer(); 50 | 51 | protected: 52 | virtual void initialize(); 53 | virtual void handleMessage(cMessage *msg); 54 | void getBalancingInfo(int id, double rData[]); 55 | void generateBalancingInfo(int id, double rData[]); 56 | 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /omnet/router/Application.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef __NETWORKSIMULATOR_APPLICATION_H_ 17 | #define __NETWORKSIMULATOR_APPLICATION_H_ 18 | 19 | #include 20 | #include 21 | #include "DataPacket_m.h" 22 | #include "TimerNextPacket_m.h" 23 | #include "ControlPacket_m.h" 24 | #include "Statistic.h" 25 | 26 | 27 | using namespace std; 28 | 29 | /** 30 | * TODO - Generated class 31 | */ 32 | class Application : public cSimpleModule 33 | { 34 | private: 35 | TimerNextPacket *interArrival; 36 | int id; 37 | 38 | int numPackets; 39 | int genT; 40 | double lambdaFactor; 41 | double numRx; 42 | double lambda; 43 | double MAXSIM; 44 | int dest; 45 | 46 | 47 | 48 | public: 49 | Application(); 50 | virtual ~Application(); 51 | 52 | protected: 53 | virtual void initialize(); 54 | virtual void handleMessage(cMessage *msg); 55 | 56 | private: 57 | double nextPacket(int i); 58 | int nextDest(); 59 | int extractId(string name, int pos); 60 | void initSignals(); 61 | }; 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /omnet/router/Node.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | module Node 19 | { 20 | parameters: 21 | int numTx; 22 | int numNodes; 23 | int id; 24 | 25 | @display("i=misc/node_vs,gold;bgb=224,246"); 26 | 27 | 28 | 29 | 30 | gates: 31 | inout port[]; 32 | 33 | submodules: 34 | tcontroller: TrafficController { 35 | id = id; 36 | nodeRatio = uniform(0.1,1); 37 | numNodes = numTx; 38 | }; 39 | app[numNodes]: Application { 40 | id = id; 41 | dest = index; 42 | numNodes = numTx; 43 | }; 44 | routing: Balancer { 45 | numTx = numTx; 46 | numNodes = numTx; 47 | id = id; 48 | }; 49 | queue[sizeof(port)]: NodeQueue; 50 | 51 | connections: 52 | for i=0..numNodes-1 { 53 | routing.localIn++ <-- app[i].out; 54 | tcontroller.out++ --> app[i].tcontrol; 55 | } 56 | for i=0..sizeof(port)-1 { 57 | routing.out++ --> queue[i].in; 58 | routing.in++ <-- queue[i].out; 59 | queue[i].line <--> port[i]; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /omnet/router/Server.ned: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | package networksimulator; 17 | 18 | module Server 19 | { 20 | parameters: 21 | int numTx; 22 | int numNodes; 23 | int id; 24 | 25 | @display("i=misc/node_vs,gold;bgb=224,246"); 26 | 27 | 28 | 29 | 30 | gates: 31 | inout port[]; 32 | 33 | submodules: 34 | tcontroller: TrafficController { 35 | id = id; 36 | nodeRatio = uniform(0.1,1); 37 | numNodes = numTx; 38 | }; 39 | app[numNodes]: Application { 40 | id = id; 41 | dest = index; 42 | numNodes = numTx; 43 | }; 44 | routing: Routing { 45 | numTx = numTx; 46 | numNodes = numTx; 47 | id = id; 48 | }; 49 | queue[sizeof(port)]: NodeQueue; 50 | 51 | connections: 52 | for i=0..numNodes-1 { 53 | routing.localIn++ <-- app[i].out; 54 | tcontroller.out++ --> app[i].tcontrol; 55 | } 56 | for i=0..sizeof(port)-1 { 57 | routing.out++ --> queue[i].in; 58 | routing.in++ <-- queue[i].out; 59 | queue[i].line <--> port[i]; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /omnet/router/Statistic.h: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #ifndef STATISTIC_H_ 17 | 18 | #define STATISTIC_H_ 19 | 20 | 21 | #include "string.h" 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | using namespace std; 29 | 30 | 31 | class Statistic { 32 | 33 | public: 34 | static Statistic *instance(); 35 | void infoTS(simtime_t time); 36 | void setDelay(simtime_t time, int i, int j, double d); 37 | void setTraffic(simtime_t time, int i, int j, double t); 38 | void setRouting(int n, int r, double p); 39 | void setLambda(double l); 40 | void setGeneration(int genType); 41 | void setLost(simtime_t time, int n, int p); 42 | void setLost(simtime_t time); 43 | void setNumTx(int n); 44 | void setNumNodes(int n); 45 | void setRoutingParaam(double r); 46 | void setMaxSim(double r); 47 | void setFolder(string folder); 48 | 49 | 50 | void printStats(); 51 | 52 | protected: 53 | 54 | 55 | Statistic(); 56 | 57 | virtual ~Statistic(); 58 | 59 | 60 | private: 61 | static Statistic *inst; 62 | simtime_t INI; 63 | simtime_t END; 64 | double SIMTIME; 65 | bool collect; 66 | 67 | int numTx; 68 | int numNodes; 69 | int genT; 70 | double lambdaMax; 71 | double routingP; 72 | 73 | string folderName; 74 | 75 | 76 | int drops; 77 | vector< vector< vector > > Traffic; 78 | vector< vector > Routing; 79 | vector< vector< vector > > Delay; 80 | vector< vector > DropsV; 81 | 82 | void initLinkID(); 83 | 84 | }; 85 | 86 | 87 | 88 | #endif /* STATISTIC_H_ */ 89 | 90 | -------------------------------------------------------------------------------- /ActorNetwork.py: -------------------------------------------------------------------------------- 1 | """ 2 | ActorNetwork.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | __credits__ = "https://github.com/yanpanlau" 6 | 7 | from keras.initializations import normal, glorot_normal 8 | from keras.activations import relu 9 | from keras.layers import Dense, Input, BatchNormalization 10 | from keras.models import Model 11 | from keras.regularizers import l2 12 | import keras.backend as K 13 | import tensorflow as tf 14 | 15 | from helper import selu 16 | 17 | 18 | class ActorNetwork(object): 19 | def __init__(self, sess, state_size, action_size, DDPG_config): 20 | self.HIDDEN1_UNITS = DDPG_config['HIDDEN1_UNITS'] 21 | self.HIDDEN2_UNITS = DDPG_config['HIDDEN2_UNITS'] 22 | 23 | self.sess = sess 24 | self.BATCH_SIZE = DDPG_config['BATCH_SIZE'] 25 | self.TAU = DDPG_config['TAU'] 26 | self.LEARNING_RATE = DDPG_config['LRA'] 27 | self.ACTUM = DDPG_config['ACTUM'] 28 | 29 | if self.ACTUM == 'NEW': 30 | self.acti = 'sigmoid' 31 | elif self.ACTUM == 'DELTA': 32 | self.acti = 'tanh' 33 | 34 | self.h_acti = relu 35 | if DDPG_config['HACTI'] == 'selu': 36 | self.h_acti = selu 37 | 38 | K.set_session(sess) 39 | 40 | #Now create the model 41 | self.model, self.weights, self.state = self.create_actor_network(state_size, action_size) 42 | self.target_model, self.target_weights, self.target_state = self.create_actor_network(state_size, action_size) 43 | self.action_gradient = tf.placeholder(tf.float32, [None, action_size]) 44 | self.params_grad = tf.gradients(self.model.output, self.weights, -self.action_gradient) 45 | grads = zip(self.params_grad, self.weights) 46 | self.optimize = tf.train.AdamOptimizer(self.LEARNING_RATE).apply_gradients(grads) 47 | self.sess.run(tf.global_variables_initializer()) 48 | 49 | def train(self, states, action_grads): 50 | self.sess.run(self.optimize, feed_dict={ 51 | self.state: states, 52 | self.action_gradient: action_grads 53 | }) 54 | 55 | def target_train(self): 56 | actor_weights = self.model.get_weights() 57 | actor_target_weights = self.target_model.get_weights() 58 | for i in range(len(actor_weights)): 59 | actor_target_weights[i] = self.TAU * actor_weights[i] + (1 - self.TAU)* actor_target_weights[i] 60 | self.target_model.set_weights(actor_target_weights) 61 | 62 | def create_actor_network(self, state_size, action_dim): 63 | S = Input(shape=[state_size], name='a_S') 64 | h0 = Dense(self.HIDDEN1_UNITS, activation=self.h_acti, init=glorot_normal, name='a_h0')(S) 65 | h1 = Dense(self.HIDDEN2_UNITS, activation=self.h_acti, init=glorot_normal, name='a_h1')(h0) 66 | # https://github.com/fchollet/keras/issues/374 67 | V = Dense(action_dim, activation=self.acti, init=glorot_normal, name='a_V')(h1) 68 | model = Model(input=S, output=V) 69 | return model, model.trainable_weights, S 70 | -------------------------------------------------------------------------------- /CriticNetwork.py: -------------------------------------------------------------------------------- 1 | """ 2 | CriticNetwork.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | __credits__ = "https://github.com/yanpanlau" 6 | 7 | from keras.initializations import normal, glorot_normal 8 | from keras.activations import relu 9 | from keras.layers import Dense, Input, merge, BatchNormalization 10 | from keras.models import Model 11 | from keras.optimizers import Adam 12 | from keras.regularizers import l2 13 | import keras.backend as K 14 | import tensorflow as tf 15 | 16 | from helper import selu 17 | 18 | 19 | class CriticNetwork(object): 20 | def __init__(self, sess, state_size, action_size, DDPG_config): 21 | self.HIDDEN1_UNITS = DDPG_config['HIDDEN1_UNITS'] 22 | self.HIDDEN2_UNITS = DDPG_config['HIDDEN2_UNITS'] 23 | 24 | self.sess = sess 25 | self.BATCH_SIZE = DDPG_config['BATCH_SIZE'] 26 | self.TAU = DDPG_config['TAU'] 27 | self.LEARNING_RATE = DDPG_config['LRC'] 28 | self.action_size = action_size 29 | 30 | self.h_acti = relu 31 | if DDPG_config['HACTI'] == 'selu': 32 | self.h_acti = selu 33 | 34 | K.set_session(sess) 35 | 36 | #Now create the model 37 | self.model, self.action, self.state = self.create_critic_network(state_size, action_size) 38 | self.target_model, self.target_action, self.target_state = self.create_critic_network(state_size, action_size) 39 | self.action_grads = tf.gradients(self.model.output, self.action) #GRADIENTS for policy update 40 | self.sess.run(tf.global_variables_initializer()) 41 | 42 | def gradients(self, states, actions): 43 | return self.sess.run(self.action_grads, feed_dict={ 44 | self.state: states, 45 | self.action: actions 46 | })[0] 47 | 48 | def target_train(self): 49 | critic_weights = self.model.get_weights() 50 | critic_target_weights = self.target_model.get_weights() 51 | for i in range(len(critic_weights)): 52 | critic_target_weights[i] = self.TAU * critic_weights[i] + (1 - self.TAU)* critic_target_weights[i] 53 | self.target_model.set_weights(critic_target_weights) 54 | 55 | def create_critic_network(self, state_size, action_dim): 56 | S = Input(shape=[state_size], name='c_S') 57 | A = Input(shape=[action_dim], name='c_A') 58 | w1 = Dense(self.HIDDEN1_UNITS, activation=self.h_acti, init=glorot_normal, name='c_w1')(S) 59 | a1 = Dense(self.HIDDEN2_UNITS, activation='linear', init=glorot_normal, name='c_a1')(A) 60 | h1 = Dense(self.HIDDEN2_UNITS, activation='linear', init=glorot_normal, name='c_h1')(w1) 61 | h2 = merge([h1, a1], mode='sum', name='c_h2') 62 | h3 = Dense(self.HIDDEN2_UNITS, activation=self.h_acti, init=glorot_normal, name='c_h3')(h2) 63 | V = Dense(action_dim, activation='linear', init=glorot_normal, name='c_V')(h3) 64 | model = Model(input=[S, A], output=V) 65 | adam = Adam(lr=self.LEARNING_RATE) 66 | model.compile(loss='mse', optimizer=adam) 67 | return model, A, S 68 | -------------------------------------------------------------------------------- /omnet/router/Routing.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include "Routing.h" 17 | 18 | 19 | Define_Module(Routing); 20 | 21 | 22 | Routing::Routing() { 23 | } 24 | 25 | Routing::~Routing() { 26 | } 27 | 28 | // -12, number of nodes 29 | void Routing::initialize() 30 | { 31 | numPorts = gateSize("in"); 32 | id = par("id"); 33 | numTx = par("numTx"); 34 | numNodes = par("numNodes"); 35 | folderName = par("folderName").stdstringValue(); 36 | 37 | int diff = numNodes - numTx; 38 | getRoutingInfo(id-diff, outPort); 39 | 40 | Statistic::instance()->setNumNodes(numNodes); 41 | Statistic::instance()->setNumTx(numTx); 42 | Statistic::instance()->setFolder(folderName); 43 | } 44 | 45 | void Routing::handleMessage(cMessage *msg) 46 | { 47 | DataPacket *data = check_and_cast(msg); 48 | 49 | if (id == data->getDstNode()) { 50 | ev << this->getFullPath() << " Message received" << endl; 51 | simtime_t delayPaquet= simTime() - data->getCreationTime(); 52 | Statistic::instance()->setDelay(simTime(), data->getSrcNode(), id, delayPaquet.dbl()); 53 | delete msg; 54 | } 55 | else if (data->getTtl() == 0) { 56 | ev << this->getFullPath() << " TTL = 0. Msg deleted" << endl; 57 | Statistic::instance()->setLost(simTime(), data->getSrcNode(), data->getDstNode()); 58 | delete msg; 59 | } 60 | else { // Tant in com out 61 | int destPort = outPort[data->getDstNode()];; 62 | data->setTtl(data->getTtl()-1); 63 | send(msg, "out", destPort); 64 | 65 | ev << "Routing: " << this->getFullPath() << " Source: " << data->getSrcNode() << " Dest: " << data->getDstNode() 66 | << " using port: "<< destPort << endl; 67 | 68 | } 69 | //if (msg->arrivedOn("localIn")) { 70 | 71 | } 72 | 73 | void Routing::getRoutingInfo(int id, int rData[]) { 74 | 75 | ifstream myfile (folderName + "/Routing.txt"); 76 | double val; 77 | 78 | if (myfile.is_open()) { 79 | int i = 0; 80 | while (id != i) { 81 | for(int k = 0; k < numTx; k++) { 82 | string aux; 83 | getline(myfile, aux, ','); 84 | } 85 | //myfile >> val; 86 | i++; 87 | } 88 | 89 | for(int k = 0; k < numTx; k++) { 90 | string aux; 91 | getline(myfile, aux, ','); 92 | val = stod(aux); 93 | rData[k] = val; 94 | } 95 | 96 | myfile.close(); 97 | } 98 | 99 | 100 | } 101 | -------------------------------------------------------------------------------- /omnet/router/TrafficController.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include "TrafficController.h" 17 | 18 | 19 | Define_Module(TrafficController); 20 | 21 | void TrafficController::initialize() 22 | { 23 | id = par("id"); 24 | nodeRatio = par("nodeRatio"); 25 | numNodes = par("numNodes"); 26 | folderName = par("folderName").stdstringValue(); 27 | 28 | int MODE = 3; 29 | 30 | if (MODE == 1) { 31 | // UNIFORM TRAFFIC PER FLOW 32 | for (int i = 0; i < numNodes; i++) { 33 | double aux; 34 | if (i == id) aux = 0; 35 | else aux = uniform(0.1,1); 36 | 37 | ControlPacket *data = new ControlPacket("trafficInfo"); 38 | data->setData(aux/numNodes); 39 | send(data, "out", i); 40 | } 41 | } 42 | else if (MODE == 2) { 43 | // UNIFOR TRAFFIC PER NODE 44 | double flowRatio[numNodes]; 45 | double sumVal = 0; 46 | for (int i = 0; i < numNodes; i++) { 47 | double aux; 48 | 49 | if (i == id) aux = 0; 50 | else aux = uniform(0.1,1); 51 | 52 | flowRatio[i] = aux; 53 | sumVal += aux; 54 | } 55 | 56 | //ev << "NODE Ratio: " << nodeRatio << endl; 57 | for (int i = 0; i < numNodes; i++) { 58 | ControlPacket *data = new ControlPacket("trafficInfo"); 59 | data->setData(nodeRatio*flowRatio[i]/sumVal); 60 | send(data, "out", i); 61 | } 62 | } 63 | else { 64 | // READED FROM FILE 65 | getTrafficInfo(id, flowRatio); 66 | for (int i = 0; i < numNodes; i++) { 67 | ControlPacket *data = new ControlPacket("trafficInfo"); 68 | data->setData(flowRatio[i]); 69 | send(data, "out", i); 70 | } 71 | } 72 | 73 | } 74 | 75 | void TrafficController::handleMessage(cMessage *msg) 76 | { 77 | // TODO - Generated method body 78 | } 79 | 80 | void TrafficController::getTrafficInfo(int id, double rData[]) { 81 | 82 | string line; 83 | ifstream myfile (folderName + "/Traffic.txt"); 84 | double val; 85 | 86 | if (myfile.is_open()) { 87 | int i = 0; 88 | while (id != i) { 89 | for(int k = 0; k < numNodes; k++) { 90 | string aux; 91 | getline(myfile, aux, ','); 92 | } 93 | //myfile >> val; 94 | i++; 95 | } 96 | 97 | for(int k = 0; k < numNodes; k++) { 98 | string aux; 99 | getline(myfile, aux, ','); 100 | val = stod(aux); 101 | rData[k] = val; 102 | } 103 | 104 | myfile.close(); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /omnet/router/NodeQueue.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include "NodeQueue.h" 17 | 18 | Define_Module(NodeQueue); 19 | 20 | NodeQueue::NodeQueue() { 21 | 22 | 23 | } 24 | 25 | NodeQueue::~NodeQueue() { 26 | cancelAndDelete(endTxMsg); 27 | while (not portQueue.empty()) { 28 | delete portQueue.front(); 29 | portQueue.pop(); 30 | } 31 | 32 | } 33 | 34 | void NodeQueue::initialize() 35 | { 36 | deleted = 0; 37 | endTxMsg = new cMessage("endTxMsg"); 38 | 39 | /* string name = this->getFullPath(); 40 | int r = 0; 41 | if (name[8] == 'r') r = 2; 42 | char c2 = name[13+r]; char c1 = name[12+r]; char p1 = name[20+r]; char p2 = name[21+r]; 43 | if (c2 >= '0' and c2 <= '9') { 44 | idNode = (c1-'0')*10 + (c2 - '0'); 45 | idPort = p2 - '0'; 46 | } 47 | else { 48 | idNode = (c1-'0'); 49 | idPort = p1 - '0'; 50 | }*/ 51 | } 52 | 53 | void NodeQueue::handleMessage(cMessage *msg) 54 | { 55 | if (msg->isSelfMessage()) { 56 | cMessage *packet = portQueue.front(); 57 | portQueue.pop(); 58 | send(packet, "line$o"); 59 | if (not portQueue.empty()) { 60 | cChannel *txChannel = gate("line$o")->getTransmissionChannel(); 61 | simtime_t txFinishTime = txChannel->getTransmissionFinishTime(); 62 | scheduleAt(txFinishTime, endTxMsg); 63 | } 64 | 65 | } 66 | else if (msg->arrivedOn("in")) { 67 | //DataPacket *data = check_and_cast(msg); 68 | 69 | 70 | cChannel *txChannel = gate("line$o")->getTransmissionChannel(); 71 | simtime_t txFinishTime = txChannel->getTransmissionFinishTime(); 72 | if (txFinishTime <= simTime()) { 73 | // channel free; send out packet immediately 74 | send(msg, "line$o"); 75 | } 76 | else { 77 | // store packet and schedule timer; when the timer expires, 78 | // the packet should be removed from the queue and sent out 79 | if (portQueue.empty()) 80 | scheduleAt(txFinishTime, endTxMsg); 81 | if (portQueue.size() < 32) 82 | portQueue.push(msg); 83 | else { 84 | deleted++; 85 | DataPacket *data = check_and_cast(msg); 86 | Statistic::instance()->setLost(simTime(), data->getSrcNode(), data->getDstNode()); 87 | delete msg; 88 | } 89 | } 90 | //ev << "QUEUE INFO " << this->getFullPath() << "--> Queue elements: " << portQueue.size() << endl; 91 | //ev << this->getFullPath() << "--> Queue elements: " << portQueue.size() << endl; 92 | 93 | 94 | } 95 | else { 96 | send(msg,"out"); 97 | } 98 | 99 | 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /omnet/router/Balancer.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include "Balancer.h" 17 | 18 | 19 | Define_Module(Balancer); 20 | 21 | 22 | Balancer::Balancer() { 23 | } 24 | 25 | Balancer::~Balancer() { 26 | } 27 | 28 | void Balancer::initialize() 29 | { 30 | numPorts = gateSize("in"); 31 | id = par("id"); 32 | numTx = par("numTx"); 33 | numNodes = par("numNodes"); 34 | folderName = par("folderName").stdstringValue(); 35 | 36 | //for (int i = 0; i < numTx; i++) { 37 | // balance[i] = uniform(0.05,0.95); 38 | // Statistic::instance()->setRouting(id, i, balance[i]); 39 | //} 40 | 41 | // READ FROM FILE 42 | getBalancingInfo(id, balance); 43 | //generateBalancingInfo(id, balance); 44 | 45 | Statistic::instance()->setNumNodes(numNodes); 46 | Statistic::instance()->setNumTx(numTx); 47 | for (int dest = 0; dest < numTx; dest++) { 48 | Statistic::instance()->setRouting(id, dest, balance[dest]); 49 | } 50 | 51 | } 52 | 53 | void Balancer::handleMessage(cMessage *msg) 54 | { 55 | DataPacket *data = check_and_cast(msg); 56 | 57 | if (id == data->getDstNode()) { 58 | ev << this->getFullPath() << " Message received" << endl; 59 | simtime_t delayPaquet= simTime() - data->getCreationTime(); 60 | Statistic::instance()->setDelay(simTime(), data->getSrcNode(), id, delayPaquet.dbl()); 61 | delete msg; 62 | } 63 | else { // Tant in com out 64 | double aux = uniform(0,1); 65 | int destPort; 66 | if (aux < balance[data->getDstNode()]) 67 | destPort = 0; 68 | else 69 | destPort = 1; 70 | 71 | send(msg, "out", destPort); 72 | 73 | ev << "Balancing: " << this->getFullPath() << " Source: " << data->getSrcNode() << " Dest: " << data->getDstNode() 74 | << " using port: "<< destPort << endl; 75 | 76 | } 77 | //if (msg->arrivedOn("localIn")) { 78 | 79 | } 80 | 81 | 82 | void Balancer::getBalancingInfo(int id, double rData[]) { 83 | 84 | string line; 85 | ifstream myfile (folderName + "/Balancing.txt"); 86 | double val; 87 | 88 | if (myfile.is_open()) { 89 | int i = 0; 90 | while (id != i) { 91 | for(int k = 0; k < numTx; k++) { 92 | string aux; 93 | getline(myfile, aux, ','); 94 | } 95 | //myfile >> val; 96 | i++; 97 | } 98 | 99 | for(int k = 0; k < numTx; k++) { 100 | string aux; 101 | getline(myfile, aux, ','); 102 | val = stod(aux); 103 | rData[k] = val; 104 | } 105 | 106 | myfile.close(); 107 | } 108 | } 109 | 110 | void Balancer::generateBalancingInfo(int id, double rData[]) { 111 | 112 | for(int k = 0; k < numTx; k++) { 113 | rData[k] = uniform(0.05,0.95);; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /omnet/router/Application.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include "Application.h" 17 | 18 | 19 | Define_Module(Application); 20 | 21 | 22 | Application::Application() { 23 | 24 | interArrival = NULL; 25 | 26 | } 27 | 28 | Application::~Application() { 29 | cancelAndDelete(interArrival); 30 | } 31 | 32 | 33 | void Application::initialize() 34 | { 35 | //id = extractId(this->getFullPath(), 12); 36 | id = par("id"); 37 | genT = par("generation"); 38 | lambdaFactor = par("lambda"); 39 | dest = par("dest"); 40 | MAXSIM = par("simulationDuration"); 41 | numRx = par("numNodes"); 42 | numPackets = 0; 43 | 44 | 45 | Statistic::instance()->setGeneration(genT); 46 | Statistic::instance()->setMaxSim(MAXSIM); 47 | Statistic::instance()->setLambda(lambdaFactor); 48 | 49 | 50 | } 51 | 52 | 53 | void Application::handleMessage(cMessage *msg) 54 | { 55 | if (msg->isSelfMessage()) { 56 | 57 | DataPacket *data = new DataPacket("dataPacket"); 58 | 59 | int size; 60 | switch (genT) { 61 | case 0: // Poisson 62 | size = exponential(1000); 63 | if (size > 50000) size = 50000; 64 | break; 65 | case 1: // Deterministic 66 | size = 1000; 67 | break; 68 | case 2: // Uniform 69 | size = uniform(0,2000); 70 | break; 71 | case 3: // Binomial 72 | if (dblrand() < 0.5) size = 300; 73 | else size = 1700; 74 | break; 75 | default: 76 | break; 77 | } 78 | 79 | 80 | data->setBitLength(size); 81 | data->setTtl(numRx); 82 | 83 | data->setDstNode(dest); 84 | data->setSrcNode(id); 85 | data->setLastTS(simTime().dbl()); 86 | 87 | send(data, "out"); 88 | 89 | numPackets++; 90 | Statistic::instance()->setTraffic(simTime(), id, dest, size); 91 | Statistic::instance()->infoTS(simTime()); 92 | 93 | 94 | if (simTime() < MAXSIM) { 95 | simtime_t etime= exponential(1.0/lambda); 96 | scheduleAt(simTime() + etime, msg); 97 | } 98 | else { 99 | EV << "END simulation" << endl; 100 | } 101 | } 102 | 103 | else { 104 | ControlPacket *data = check_and_cast(msg); 105 | double flowRatio = data->getData(); 106 | lambda = lambdaFactor*flowRatio; 107 | //lambda = lambdaMax/numRx; 108 | 109 | interArrival = new TimerNextPacket("timer"); 110 | interArrival->setLambda(1.0/lambda); 111 | if (dest != id) 112 | scheduleAt(simTime() + 1.0/lambda, interArrival); 113 | ev << "Ratio: " << flowRatio << " lambda: " << lambda << endl; 114 | 115 | delete data; 116 | 117 | 118 | } 119 | 120 | 121 | } 122 | 123 | 124 | /* 125 | int Application::extractId(string name, int pos) { 126 | int idt; 127 | char c2 = name[pos+1]; char c1 = name[pos]; 128 | if (c2 >= '0' and c2 <= '9') 129 | idt = (c1-'0')*10 + (c2 - '0'); 130 | else 131 | idt = (c1-'0'); 132 | return idt; 133 | }*/ 134 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Deep-Reinforcement Learning Approach for Software-Defined Networking Routing Optimization 2 | 3 | ###### [1709.07080](https://arxiv.org/abs/1709.07080): Giorgio Stampa, Marta Arias, David Sanchez-Charles, Victor Muntes-Mulero, Albert Cabellos 4 | 5 | In this paper we design and evaluate a Deep-Reinforcement Learning agent that optimizes routing. Our agent adapts automatically to current traffic conditions and proposes tailored configurations that attempt to minimize the network delay. Experiments show very promising performance. Moreover, this approach provides important operational advantages with respect to traditional optimization algorithms. 6 | 7 | --- 8 | 9 | Code and datasets [here](https://github.com/knowledgedefinednetworking/a-deep-rl-approach-for-sdn-routing-optimization/releases). 10 | 11 | --- 12 | 13 | # Keras and Deep Deterministic Policy Gradient to control an OMNeT++ network simulator 14 | 15 | ## How to train? 16 | 17 | ### Single run with fixed parameters 18 | Reads configuration from ```DDPG.json``` 19 | 20 | ``` 21 | python3 ddpg.py 22 | ``` 23 | 24 | ### Experiment with variation in parameters 25 | Reads basic configuration from ```EXPERIMENT.json``` and changes some parameters according to what is defined in function experiment(), ```cluster``` is needed to choose between ```local``` and ```arvei``` execution 26 | 27 | ``` 28 | python3 experiment.py cluster EXPERIMENT.json 29 | ``` 30 | 31 | ## How to play? 32 | 33 | ### Single run with fixed parameters 34 | Reads configuration (ddpg, neural network weights, etc.) from ```folder``` 35 | 36 | ``` 37 | python3 ddpg.py play folder 38 | ``` 39 | 40 | --- 41 | 42 | ### EXAMPLE JSON CONFIG 43 | 44 | ``` 45 | { 46 | "ACTIVE_NODES": 3, # number of active nodes in the network 47 | "ACTUM": "NEW", # action: NEW or DELTA 48 | "BATCH_SIZE": 50, # size of learning batch 49 | "BUFFER_SIZE": 2000, # max size of replay buffer 50 | "ENV": "label", # "label" or "balancing" 51 | "EPISODE_COUNT": 10, # number of episodes 52 | "EXPLORE": 0.8, # exploration: rate if <=1, number of steps otherwise 53 | "GAMMA": 0.99, # discount factor 54 | "HACTI": "selu", # non-linear activation function for hidden layers 55 | "HIDDEN1_UNITS": 300, # neurons of layer 1 56 | "HIDDEN2_UNITS": 600, # neurons of layer 2 57 | "LRA": 0.0001, # learning rate of the actor network 58 | "LRC": 0.001, # learning rate of the critic network 59 | "MAX_STEPS": 1000, # number of steps per episode 60 | "MU": 0.0, # Ornstein-Uhlenbeck process' μ 61 | "PRAEMIUM": "MAX", # reward function 62 | "PRINT": true, # verbosity 63 | "ROUTING": "Linkweight", # "Balancer", "Linkweight", "Pathchoice" 64 | "RSEED": null, # random seed: null or number 65 | "SIGMA": 0.07, # Ornstein-Uhlenbeck process' σ 66 | "STATUM": "T", # state representation: T or RT 67 | "TAU": 0.001, # soft target update 68 | "THETA": 0.03, # Ornstein-Uhlenbeck process' θ 69 | "TRAFFIC": "EXP" # traffic: static or changing (~randomly) 70 | } 71 | ``` 72 | 73 | --- 74 | 75 | author: giorgio@ac.upc.edu 76 | 77 | * [Keras](https://keras.io/) 78 | * [DDPG](https://arxiv.org/abs/1509.02971) 79 | * [OMNeT++](https://omnetpp.org/) 80 | -------------------------------------------------------------------------------- /Traffic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Traffic.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | 6 | import numpy as np 7 | from os import listdir 8 | from re import split 9 | 10 | from OU import OU 11 | from helper import softmax 12 | 13 | 14 | def natural_key(string_): 15 | """See http://www.codinghorror.com/blog/archives/001018.html""" 16 | return [int(s) if s.isdigit() else s for s in split(r'(\d+)', string_)] 17 | 18 | 19 | class Traffic(): 20 | 21 | def __init__(self, nodes_num, type, capacity): 22 | self.nodes_num = nodes_num 23 | self.prev_traffic = None 24 | self.type = type 25 | self.capacity = capacity * nodes_num / (nodes_num - 1) 26 | self.dictionary = {} 27 | self.dictionary['NORM'] = self.normal_traffic 28 | self.dictionary['UNI'] = self.uniform_traffic 29 | self.dictionary['CONTROLLED'] = self.controlled_uniform_traffic 30 | self.dictionary['EXP'] = self.exp_traffic 31 | self.dictionary['OU'] = self.ou_traffic 32 | self.dictionary['STAT'] = self.stat_traffic 33 | self.dictionary['STATEQ'] = self.stat_eq_traffic 34 | self.dictionary['FILE'] = self.file_traffic 35 | self.dictionary['DIR'] = self.dir_traffic 36 | if self.type.startswith('DIR:'): 37 | self.dir = sorted(listdir(self.type.split('DIR:')[-1]), key=lambda x: natural_key((x))) 38 | self.static = None 39 | self.total_ou = OU(1, self.capacity/2, 0.1, self.capacity/2) 40 | self.nodes_ou = OU(self.nodes_num**2, 1, 0.1, 1) 41 | 42 | def normal_traffic(self): 43 | t = np.random.normal(capacity/2, capacity/2) 44 | return np.asarray(t * softmax(np.random.randn(self.nodes_num, self.nodes_num))).clip(min=0.001) 45 | 46 | def uniform_traffic(self): 47 | t = np.random.uniform(0, self.capacity*1.25) 48 | return np.asarray(t * softmax(np.random.uniform(0, 1, size=[self.nodes_num]*2))).clip(min=0.001) 49 | 50 | def controlled_uniform_traffic(self): 51 | t = np.random.uniform(0, self.capacity*1.25) 52 | if self.prev_traffic is None: 53 | self.prev_traffic = np.asarray(t * softmax(np.random.uniform(0, 1, size=[self.nodes_num]*2))).clip(min=0.001) 54 | dist = [1] 55 | dist += [0]*(self.nodes_num**2 - 1) 56 | ch = np.random.choice(dist, [self.nodes_num]*2) 57 | 58 | tt = np.multiply(self.prev_traffic, 1 - ch) 59 | 60 | nt = np.asarray(t * softmax(np.random.uniform(0, 1, size=[self.nodes_num]*2))).clip(min=0.001) 61 | nt = np.multiply(nt, ch) 62 | 63 | self.prev_traffic = tt + nt 64 | 65 | return self.prev_traffic 66 | 67 | def exp_traffic(self): 68 | a = np.random.exponential(size=self.nodes_num) 69 | b = np.random.exponential(size=self.nodes_num) 70 | 71 | T = np.outer(a, b) 72 | 73 | np.fill_diagonal(T, -1) 74 | 75 | T[T!=-1] = np.asarray(np.random.exponential()*T[T!=-1]/np.average(T[T!=-1])).clip(min=0.001) 76 | 77 | return T 78 | 79 | def stat_traffic(self): 80 | if self.static is None: 81 | string = self.type.split('STAT:')[-1] 82 | v = np.asarray(tuple(float(x) for x in string.split(',')[:self.nodes_num**2])) 83 | M = np.split(v, self.nodes_num) 84 | self.static = np.vstack(M) 85 | return self.static 86 | 87 | def stat_eq_traffic(self): 88 | if self.static is None: 89 | value = float(self.type.split('STATEQ:')[-1]) 90 | self.static = np.full([self.nodes_num]*2, value, dtype=float) 91 | return self.static 92 | 93 | def ou_traffic(self): 94 | t = self.total_ou.evolve()[0] 95 | nt = t * softmax(self.nodes_ou.evolve()) 96 | i = np.split(nt, self.nodes_num) 97 | return np.vstack(i).clip(min=0.001) 98 | 99 | def file_traffic(self): 100 | if self.static is None: 101 | fname = 'traffic/' + self.type.split('FILE:')[-1] 102 | v = np.loadtxt(fname, delimiter=',') 103 | self.static = np.split(v, self.nodes_num) 104 | return self.static 105 | 106 | def dir_traffic(self): 107 | while len(self.dir) > 0: 108 | tm = self.dir.pop(0) 109 | if not tm.endswith('.txt'): 110 | continue 111 | fname = self.type.split('DIR:')[-1] + '/' + tm 112 | v = np.loadtxt(fname, delimiter=',') 113 | return np.split(v, self.nodes_num) 114 | return False 115 | 116 | 117 | def generate(self): 118 | return self.dictionary[self.type.split(":")[0]]() 119 | -------------------------------------------------------------------------------- /omnet/router/NetworkAll.ned: -------------------------------------------------------------------------------- 1 | package networksimulator; 2 | 3 | network NetworkAll 4 | { 5 | parameters: 6 | int numNodes = 14; 7 | int numTx = 14; 8 | 9 | types: 10 | channel Channel extends ned.DatarateChannel 11 | { 12 | delay = 0ms; 13 | datarate = 9 kbps; 14 | } 15 | 16 | submodules: 17 | node0: Server { 18 | id = 0; 19 | numTx = numTx; 20 | numNodes = numNodes; 21 | gates: 22 | port[3]; 23 | } 24 | node1: Server { 25 | id = 1; 26 | numTx = numTx; 27 | numNodes = numNodes; 28 | gates: 29 | port[3]; 30 | } 31 | node2: Server { 32 | id = 2; 33 | numTx = numTx; 34 | numNodes = numNodes; 35 | gates: 36 | port[3]; 37 | } 38 | node3: Server { 39 | id = 3; 40 | numTx = numTx; 41 | numNodes = numNodes; 42 | gates: 43 | port[3]; 44 | } 45 | node4: Server { 46 | id = 4; 47 | numTx = numTx; 48 | numNodes = numNodes; 49 | gates: 50 | port[3]; 51 | } 52 | node5: Server { 53 | id = 5; 54 | numTx = numTx; 55 | numNodes = numNodes; 56 | gates: 57 | port[4]; 58 | } 59 | node6: Server { 60 | id = 6; 61 | numTx = numTx; 62 | numNodes = numNodes; 63 | gates: 64 | port[2]; 65 | } 66 | node7: Server { 67 | id = 7; 68 | numTx = numTx; 69 | numNodes = numNodes; 70 | gates: 71 | port[3]; 72 | } 73 | node8: Server { 74 | id = 8; 75 | numTx = numTx; 76 | numNodes = numNodes; 77 | gates: 78 | port[3]; 79 | } 80 | node9: Server { 81 | id = 9; 82 | numTx = numTx; 83 | numNodes = numNodes; 84 | gates: 85 | port[4]; 86 | } 87 | node10: Server { 88 | id = 10; 89 | numTx = numTx; 90 | numNodes = numNodes; 91 | gates: 92 | port[2]; 93 | } 94 | node11: Server { 95 | id = 11; 96 | numTx = numTx; 97 | numNodes = numNodes; 98 | gates: 99 | port[3]; 100 | } 101 | node12: Server { 102 | id = 12; 103 | numTx = numTx; 104 | numNodes = numNodes; 105 | gates: 106 | port[3]; 107 | } 108 | node13: Server { 109 | id = 13; 110 | numTx = numTx; 111 | numNodes = numNodes; 112 | gates: 113 | port[3]; 114 | } 115 | 116 | connections: 117 | node0.port[0] <--> Channel <--> node1.port[0]; 118 | node0.port[1] <--> Channel <--> node2.port[0]; 119 | node0.port[2] <--> Channel <--> node3.port[0]; 120 | node1.port[1] <--> Channel <--> node2.port[1]; 121 | node1.port[2] <--> Channel <--> node7.port[0]; 122 | node2.port[2] <--> Channel <--> node5.port[0]; 123 | node3.port[1] <--> Channel <--> node4.port[0]; 124 | node3.port[2] <--> Channel <--> node8.port[0]; 125 | node4.port[1] <--> Channel <--> node5.port[1]; 126 | node4.port[2] <--> Channel <--> node6.port[0]; 127 | node5.port[2] <--> Channel <--> node10.port[0]; 128 | node5.port[3] <--> Channel <--> node12.port[0]; 129 | node6.port[1] <--> Channel <--> node7.port[1]; 130 | node7.port[2] <--> Channel <--> node9.port[0]; 131 | node8.port[1] <--> Channel <--> node11.port[0]; 132 | node8.port[2] <--> Channel <--> node13.port[0]; 133 | node9.port[1] <--> Channel <--> node10.port[1]; 134 | node9.port[2] <--> Channel <--> node11.port[1]; 135 | node9.port[3] <--> Channel <--> node13.port[1]; 136 | node11.port[2] <--> Channel <--> node12.port[1]; 137 | node12.port[2] <--> Channel <--> node13.port[2]; 138 | } 139 | -------------------------------------------------------------------------------- /omnet/router/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # OMNeT++/OMNEST Makefile for networkRL 3 | # 4 | # This file was generated with the command: 5 | # opp_makemake -f --deep -O out 6 | # 7 | 8 | # Name of target to be created (-o option) 9 | TARGET = networkRL$(EXE_SUFFIX) 10 | 11 | # User interface (uncomment one) (-u option) 12 | #USERIF_LIBS = $(ALL_ENV_LIBS) # that is, $(TKENV_LIBS) $(CMDENV_LIBS) 13 | USERIF_LIBS = $(CMDENV_LIBS) 14 | #USERIF_LIBS = $(TKENV_LIBS) 15 | 16 | # C++ include paths (with -I) 17 | INCLUDE_PATH = -I. -Imessages -Itest 18 | 19 | # Additional object and library files to link with 20 | EXTRA_OBJS = 21 | 22 | # Additional libraries (-L, -l options) 23 | LIBS = 24 | 25 | # Output directory 26 | PROJECT_OUTPUT_DIR = out 27 | PROJECTRELATIVE_PATH = 28 | O = $(PROJECT_OUTPUT_DIR)/$(CONFIGNAME)/$(PROJECTRELATIVE_PATH) 29 | 30 | # Object files for local .cc and .msg files 31 | OBJS = \ 32 | $O/Application.o \ 33 | $O/Balancer.o \ 34 | $O/NodeQueue.o \ 35 | $O/Routing.o \ 36 | $O/Statistic.o \ 37 | $O/TrafficController.o \ 38 | $O/messages/ControlPacket_m.o \ 39 | $O/messages/DataPacket_m.o \ 40 | $O/messages/TimerNextPacket_m.o 41 | 42 | # Message files 43 | MSGFILES = \ 44 | messages/ControlPacket.msg \ 45 | messages/DataPacket.msg \ 46 | messages/TimerNextPacket.msg 47 | 48 | #------------------------------------------------------------------------------ 49 | 50 | # Pull in OMNeT++ configuration (Makefile.inc or configuser.vc) 51 | 52 | ifneq ("$(OMNETPP_CONFIGFILE)","") 53 | CONFIGFILE = $(OMNETPP_CONFIGFILE) 54 | else 55 | ifneq ("$(OMNETPP_ROOT)","") 56 | CONFIGFILE = $(OMNETPP_ROOT)/Makefile.inc 57 | else 58 | CONFIGFILE = $(shell opp_configfilepath) 59 | endif 60 | endif 61 | 62 | ifeq ("$(wildcard $(CONFIGFILE))","") 63 | $(error Config file '$(CONFIGFILE)' does not exist -- add the OMNeT++ bin directory to the path so that opp_configfilepath can be found, or set the OMNETPP_CONFIGFILE variable to point to Makefile.inc) 64 | endif 65 | 66 | include $(CONFIGFILE) 67 | 68 | # Simulation kernel and user interface libraries 69 | OMNETPP_LIB_SUBDIR = $(OMNETPP_LIB_DIR)/$(TOOLCHAIN_NAME) 70 | OMNETPP_LIBS = -L"$(OMNETPP_LIB_SUBDIR)" -L"$(OMNETPP_LIB_DIR)" -loppmain$D $(USERIF_LIBS) $(KERNEL_LIBS) $(SYS_LIBS) 71 | 72 | COPTS = $(CFLAGS) $(INCLUDE_PATH) -I$(OMNETPP_INCL_DIR) 73 | MSGCOPTS = $(INCLUDE_PATH) 74 | 75 | # we want to recompile everything if COPTS changes, 76 | # so we store COPTS into $COPTS_FILE and have object 77 | # files depend on it (except when "make depend" was called) 78 | COPTS_FILE = $O/.last-copts 79 | ifneq ($(MAKECMDGOALS),depend) 80 | ifneq ("$(COPTS)","$(shell cat $(COPTS_FILE) 2>/dev/null || echo '')") 81 | $(shell $(MKPATH) "$O" && echo "$(COPTS)" >$(COPTS_FILE)) 82 | endif 83 | endif 84 | 85 | #------------------------------------------------------------------------------ 86 | # User-supplied makefile fragment(s) 87 | # >>> 88 | # <<< 89 | #------------------------------------------------------------------------------ 90 | 91 | # Main target 92 | all: $O/$(TARGET) 93 | $(Q)$(LN) $O/$(TARGET) . 94 | 95 | $O/$(TARGET): $(OBJS) $(wildcard $(EXTRA_OBJS)) Makefile 96 | @$(MKPATH) $O 97 | @echo Creating executable: $@ 98 | $(Q)$(CXX) $(LDFLAGS) -o $O/$(TARGET) $(OBJS) $(EXTRA_OBJS) $(AS_NEEDED_OFF) $(WHOLE_ARCHIVE_ON) $(LIBS) $(WHOLE_ARCHIVE_OFF) $(OMNETPP_LIBS) 99 | 100 | .PHONY: all clean cleanall depend msgheaders 101 | 102 | .SUFFIXES: .cc 103 | 104 | $O/%.o: %.cc $(COPTS_FILE) 105 | @$(MKPATH) $(dir $@) 106 | $(qecho) "$<" 107 | $(Q)$(CXX) -c $(CXXFLAGS) $(COPTS) -o $@ $< 108 | 109 | %_m.cc %_m.h: %.msg 110 | $(qecho) MSGC: $< 111 | $(Q)$(MSGC) -s _m.cc $(MSGCOPTS) $? 112 | 113 | msgheaders: $(MSGFILES:.msg=_m.h) 114 | 115 | clean: 116 | $(qecho) Cleaning... 117 | $(Q)-rm -rf $O 118 | $(Q)-rm -f networkRL networkRL.exe libnetworkRL.so libnetworkRL.a libnetworkRL.dll libnetworkRL.dylib 119 | $(Q)-rm -f ./*_m.cc ./*_m.h 120 | $(Q)-rm -f messages/*_m.cc messages/*_m.h 121 | $(Q)-rm -f test/*_m.cc test/*_m.h 122 | 123 | cleanall: clean 124 | $(Q)-rm -rf $(PROJECT_OUTPUT_DIR) 125 | 126 | depend: 127 | $(qecho) Creating dependencies... 128 | $(Q)$(MAKEDEPEND) $(INCLUDE_PATH) -f Makefile -P\$$O/ -- $(MSG_CC_FILES) ./*.cc messages/*.cc test/*.cc 129 | 130 | # DO NOT DELETE THIS LINE -- make depend depends on it. 131 | $O/Application.o: Application.cc \ 132 | Application.h \ 133 | Statistic.h \ 134 | messages/ControlPacket_m.h \ 135 | messages/DataPacket_m.h \ 136 | messages/TimerNextPacket_m.h 137 | $O/Balancer.o: Balancer.cc \ 138 | Balancer.h \ 139 | Statistic.h \ 140 | messages/DataPacket_m.h 141 | $O/NodeQueue.o: NodeQueue.cc \ 142 | NodeQueue.h \ 143 | Statistic.h \ 144 | messages/DataPacket_m.h 145 | $O/Routing.o: Routing.cc \ 146 | Routing.h \ 147 | Statistic.h \ 148 | messages/DataPacket_m.h 149 | $O/Statistic.o: Statistic.cc \ 150 | Statistic.h 151 | $O/TrafficController.o: TrafficController.cc \ 152 | TrafficController.h \ 153 | messages/ControlPacket_m.h 154 | $O/messages/ControlPacket_m.o: messages/ControlPacket_m.cc \ 155 | messages/ControlPacket_m.h 156 | $O/messages/DataPacket_m.o: messages/DataPacket_m.cc \ 157 | messages/DataPacket_m.h 158 | $O/messages/TimerNextPacket_m.o: messages/TimerNextPacket_m.cc \ 159 | messages/TimerNextPacket_m.h 160 | -------------------------------------------------------------------------------- /omnet/router/Statistic.cc: -------------------------------------------------------------------------------- 1 | // 2 | // This program is free software: you can redistribute it and/or modify 3 | // it under the terms of the GNU Lesser General Public License as published by 4 | // the Free Software Foundation, either version 3 of the License, or 5 | // (at your option) any later version. 6 | // 7 | // This program is distributed in the hope that it will be useful, 8 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | // GNU Lesser General Public License for more details. 11 | // 12 | // You should have received a copy of the GNU Lesser General Public License 13 | // along with this program. If not, see http://www.gnu.org/licenses/. 14 | // 15 | 16 | #include 17 | #include 18 | 19 | Statistic *Statistic::inst = 0; 20 | 21 | 22 | 23 | Statistic::Statistic() { 24 | 25 | INI = 50; 26 | END = 650; 27 | collect = true; 28 | 29 | Traffic = vector > > (100, vector >(100, vector())); 30 | Routing = vector > (100, vector(100)); 31 | Delay = vector > > (100, vector >(100, vector())); 32 | 33 | DropsV = vector > (100, vector(100, 0)); 34 | drops = 0; 35 | 36 | 37 | } 38 | 39 | 40 | 41 | Statistic::~Statistic() { 42 | 43 | } 44 | 45 | 46 | 47 | Statistic *Statistic::instance() { 48 | if (!inst) 49 | inst = new Statistic(); 50 | return inst; 51 | } 52 | 53 | void Statistic::setMaxSim(double ms) { 54 | END = ms; 55 | SIMTIME = (END.dbl()-INI.dbl())*1000; 56 | } 57 | 58 | void Statistic::setRouting(int src, int dst, double r) { 59 | (Routing)[src][dst] = r; 60 | } 61 | 62 | void Statistic::infoTS(simtime_t time) { 63 | if (time > END and collect) { 64 | collect = false; 65 | printStats(); 66 | } 67 | if (time < INI and not collect) 68 | collect = true; 69 | } 70 | 71 | 72 | void Statistic::setDelay(simtime_t time, int src, int dst, double d) { 73 | if (time > INI and collect) 74 | (Delay)[src][dst].push_back(d); 75 | } 76 | 77 | 78 | 79 | void Statistic::setTraffic(simtime_t time, int src, int dst, double t) { 80 | if (time > INI and collect) 81 | (Traffic)[src][dst].push_back(t); 82 | } 83 | 84 | void Statistic::setLost(simtime_t time, int n, int p) { 85 | if (time > INI and collect) { 86 | drops++; 87 | (DropsV)[n][p]++; 88 | } 89 | } 90 | 91 | void Statistic::setLost(simtime_t time) { 92 | if (time > INI and collect) 93 | drops++; 94 | } 95 | 96 | 97 | /*void Statistic::setRouting(int n, int r, double p) { 98 | }*/ 99 | 100 | void Statistic::setLambda(double l) { 101 | lambdaMax = l; 102 | } 103 | 104 | void Statistic::setGeneration(int genType) { 105 | genT = genType; 106 | } 107 | 108 | void Statistic::setFolder(string folder) { 109 | folderName = folder; 110 | } 111 | 112 | 113 | 114 | void Statistic::setNumTx(int n) { 115 | numTx = n; 116 | } 117 | 118 | void Statistic::setNumNodes(int n) { 119 | numNodes = n; 120 | } 121 | 122 | void Statistic::setRoutingParaam(double r) { 123 | routingP = r; 124 | } 125 | 126 | 127 | void Statistic::printStats() { 128 | string genString; 129 | switch (genT) { 130 | case 0: // Poisson 131 | genString = "M"; //"Poisson"; 132 | break; 133 | case 1: // Deterministic 134 | genString = "D"; //"Deterministic"; 135 | break; 136 | case 2: // Uniform 137 | genString = "U"; //"Uniform"; 138 | break; 139 | case 3: // Binomial 140 | genString = "B"; //"Binomial"; 141 | break; 142 | default: 143 | break; 144 | } 145 | 146 | 147 | 148 | vector features; 149 | // features.push_back(routingP); 150 | 151 | //int firstTx = numNodes - numTx; 152 | // Traffic 153 | /*for (int i = 0; i < numTx; i++) { 154 | for (int j = 0; j < numTx; j++) { 155 | long double d = 0; 156 | unsigned int numPackets = (Traffic)[i][j].size(); 157 | for (unsigned int k = 0; k < numPackets; k++) 158 | d += (Traffic)[i][j][k]; 159 | features.push_back(d/SIMTIME); 160 | } 161 | } 162 | // Routiung 163 | for (int i = 0; i < numTx; i++) { 164 | for (int j = 0; j < numTx; j++) { 165 | features.push_back((Routing)[i][j]); 166 | } 167 | }*/ 168 | 169 | 170 | // Delay 171 | int steps = (SIMTIME/1000)+50; 172 | for (int i = 0; i < numTx; i++) { 173 | for (int j = 0; j < numTx; j++) { 174 | long double d = 0; 175 | unsigned int numPackets = (Delay)[i][j].size(); 176 | for (unsigned int k = 0; k < numPackets; k++) 177 | d += (Delay)[i][j][k]; 178 | if (numPackets == 0) 179 | if (i == j) 180 | features.push_back(-1); 181 | else 182 | features.push_back(std::numeric_limits::infinity()); 183 | else 184 | features.push_back(d/numPackets); 185 | } 186 | } 187 | // Drops 188 | // for (int i = 0; i < numTx; i++) { 189 | // for (int j = 0; j < numTx; j++) { 190 | // features.push_back((DropsV)[i][j]/steps); 191 | // } 192 | // } 193 | features.push_back(drops/steps); 194 | 195 | // Reset 196 | drops = 0; 197 | Traffic.clear(); 198 | Delay.clear(); 199 | 200 | 201 | // Print file 202 | ofstream myfile; 203 | //char * filename = new char[80]; 204 | string filename; 205 | 206 | //sprintf(filename, "dataOU_%d_%d_%d_%d_%s.txt",numNodes, numTx, (int) lambdaMax, (int) SIMTIME/1000, genString.c_str()); 207 | 208 | 209 | // Instant 210 | filename = folderName + "/Delay.txt"; 211 | // sprintf(filename, folderName + "/Delay.txt"); 212 | myfile.open (filename, ios::out | ios::trunc ); 213 | for (unsigned int i = 0; i < features.size(); i++ ) { 214 | double d = features[i]; 215 | myfile << d << ","; 216 | } 217 | myfile << endl; 218 | myfile.close(); 219 | 220 | 221 | } 222 | -------------------------------------------------------------------------------- /helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | helper.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | 6 | import time 7 | import os 8 | import json 9 | import sys 10 | import argparse 11 | from collections import OrderedDict 12 | import numpy as np 13 | 14 | 15 | def experiment_load(): 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("CLUSTER") 18 | parser.add_argument("EXPERIMENT") 19 | args = parser.parse_args() 20 | 21 | # let experiment type (function) come from commandline arg 22 | with open(args.EXPERIMENT) as jconfig: 23 | DDPG_config = json.load(jconfig) 24 | 25 | DDPG_config['CLUSTER'] = args.CLUSTER 26 | DDPG_config['EXPERIMENT'] = args.EXPERIMENT.lower().split('.')[0] 27 | 28 | if DDPG_config['CLUSTER'] == 'local': 29 | import experiment.local 30 | runwrapper = experiment.local.runwrapper 31 | DDPG_config['EXPERIMENT'] = setup_exp(DDPG_config['EXPERIMENT']) 32 | 33 | return DDPG_config, runwrapper 34 | 35 | 36 | def setup_exp(experiment=''): 37 | folder = 'runs/' 38 | os.makedirs(folder, exist_ok=True) 39 | 40 | folder += experiment + '/' 41 | os.makedirs(folder, exist_ok=True) 42 | 43 | return folder 44 | 45 | 46 | def setup_run(DDPG_config): 47 | folder = DDPG_config['EXPERIMENT'] 48 | epoch = 't%.6f/' % time.time() 49 | folder += epoch.replace('.', '') 50 | os.makedirs(folder, exist_ok=True) 51 | 52 | with open(folder + 'folder.ini', 'w') as ifile: 53 | ifile.write('[General]\n') 54 | ifile.write('**.folderName = "' + folder + '"\n') 55 | 56 | with open(folder + 'DDPG.json', 'w') as jconfig: 57 | json.dump(OrderedDict(sorted(DDPG_config.items(), key=lambda t: t[0])), jconfig, indent=4) 58 | 59 | # with open(folder + 'Routing.txt', 'w') as rfile: 60 | # rfile.write(DDPG_config['U_ROUTING'] + '\n') 61 | 62 | if DDPG_config['TRAFFIC'].startswith('STAT:'): 63 | with open(folder + 'Traffic.txt', 'w') as rfile: 64 | rfile.write(DDPG_config['TRAFFIC'].split('STAT:')[-1] + '\n') 65 | 66 | return folder 67 | 68 | 69 | def setup_brute(DDPG_config): 70 | folder = 'runs/brute' 71 | epoch = 't%.6f/' % time.time() 72 | folder += epoch.replace('.', '') 73 | folder += '/' 74 | os.makedirs(folder, exist_ok=True) 75 | 76 | with open(folder + 'folder.ini', 'w') as ifile: 77 | ifile.write('[General]\n') 78 | ifile.write('**.folderName = "' + folder + '"\n') 79 | 80 | # with open(folder + 'Routing.txt', 'w') as rfile: 81 | # rfile.write(DDPG_config['U_ROUTING'] + '\n') 82 | 83 | if DDPG_config['TRAFFIC'].startswith('STAT:'): 84 | with open(folder + 'Traffic.txt', 'w') as rfile: 85 | rfile.write(DDPG_config['TRAFFIC'].split('STAT:')[-1] + '\n') 86 | 87 | with open(folder + 'DDPG.json', 'w') as jconfig: 88 | json.dump(OrderedDict(sorted(DDPG_config.items(), key=lambda t: t[0])), jconfig, indent=4) 89 | 90 | return folder 91 | 92 | 93 | def parser(): 94 | parser = argparse.ArgumentParser() 95 | 96 | parser.add_argument("CLUSTER") 97 | parser.add_argument("EXPERIMENT") 98 | parser.add_argument("--RSEED", type=int, action="store", default=None) 99 | parser.add_argument("--PRINT", action="store_true") 100 | parser.add_argument("--ACTIVE_NODES", type=int, action="store", required=True) 101 | parser.add_argument("--MU", type=float, action="store", required=True) 102 | parser.add_argument("--THETA", type=float, action="store", required=True) 103 | parser.add_argument("--SIGMA", type=float, action="store", required=True) 104 | parser.add_argument("--BUFFER_SIZE", type=int, action="store", required=True) 105 | parser.add_argument("--BATCH_SIZE", type=int, action="store", required=True) 106 | parser.add_argument("--GAMMA", type=float, action="store", required=True) 107 | parser.add_argument("--TAU", type=float, action="store", required=True) 108 | parser.add_argument("--LRA", type=float, action="store", required=True) 109 | parser.add_argument("--LRC", type=float, action="store", required=True) 110 | parser.add_argument("--EXPLORE", type=float, action="store", required=True) 111 | parser.add_argument("--EPISODE_COUNT", type=int, action="store", required=True) 112 | parser.add_argument("--MAX_STEPS", type=int, action="store", required=True) 113 | parser.add_argument("--HACTI", action="store", required=True) 114 | parser.add_argument("--HIDDEN1_UNITS", type=int, action="store", required=True) 115 | parser.add_argument("--HIDDEN2_UNITS", type=int, action="store", required=True) 116 | parser.add_argument("--TRAFFIC", action="store", required=True) 117 | parser.add_argument("--STATUM", action="store", required=True) 118 | parser.add_argument("--PRAEMIUM", action="store", required=True) 119 | parser.add_argument("--ACTUM", action="store", required=True) 120 | parser.add_argument("--MAX_DELTA", type=float, action="store", default=None) 121 | parser.add_argument("--BN", action="store", default=None) 122 | parser.add_argument("--U_ROUTING", action="store", default=None) 123 | parser.add_argument("--ROUTING", action="store", required=True) 124 | parser.add_argument("--ENV", action="store", required=True) 125 | 126 | args = parser.parse_args() 127 | 128 | DDPG_config = vars(args) 129 | 130 | return DDPG_config 131 | 132 | 133 | def pretty(f): 134 | try: 135 | float(f) 136 | return str.format('{0:.3f}', f).rstrip('0').rstrip('.') 137 | except: 138 | return str(f) 139 | 140 | 141 | def scale(array): 142 | mean = array.mean() 143 | std = array.std() 144 | if std == 0: 145 | std = 1 146 | return np.asarray((array - mean)/std) 147 | 148 | 149 | def softmax(x): 150 | return np.exp(x) / np.sum(np.exp(x), axis=0) 151 | 152 | 153 | def selu(x): 154 | from keras.activations import elu 155 | """Scaled Exponential Linear Unit. (Klambauer et al., 2017) 156 | 157 | # Arguments 158 | x: A tensor or variable to compute the activation function for. 159 | 160 | # References 161 | - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515) 162 | """ 163 | alpha = 1.6732632423543772848170429916717 164 | scale = 1.0507009873554804934193349852946 165 | return scale * elu(x, alpha) 166 | -------------------------------------------------------------------------------- /omnet/router/.cproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /ddpg.py: -------------------------------------------------------------------------------- 1 | """ 2 | ddpg.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | __credits__ = "https://github.com/yanpanlau" 6 | 7 | # import simulator as environment env() 8 | from Environment import OmnetBalancerEnv 9 | from Environment import OmnetLinkweightEnv 10 | from Environment import vector_to_file 11 | import numpy as np 12 | import tensorflow as tf 13 | import sys 14 | import json 15 | 16 | from ReplayBuffer import ReplayBuffer 17 | from ActorNetwork import ActorNetwork 18 | from CriticNetwork import CriticNetwork 19 | from OU import OU 20 | from helper import setup_exp, setup_run, parser, pretty, scale 21 | 22 | 23 | def playGame(DDPG_config, train_indicator=1): #1 means Train, 0 means simply Run 24 | # SETUP STARTS HERE 25 | if train_indicator > 0: 26 | folder = setup_run(DDPG_config) 27 | elif train_indicator == 0: 28 | folder = DDPG_config['EXPERIMENT'] 29 | 30 | if DDPG_config['RSEED'] == 0: 31 | DDPG_config['RSEED'] = None 32 | np.random.seed(DDPG_config['RSEED']) 33 | 34 | ACTIVE_NODES = DDPG_config['ACTIVE_NODES'] 35 | 36 | # Generate an environment 37 | if DDPG_config['ENV'] == 'balancing': 38 | env = OmnetBalancerEnv(DDPG_config, folder) 39 | elif DDPG_config['ENV'] == 'label': 40 | env = OmnetLinkweightEnv(DDPG_config, folder) 41 | 42 | action_dim, state_dim = env.a_dim, env.s_dim 43 | 44 | MU = DDPG_config['MU'] 45 | THETA = DDPG_config['THETA'] 46 | SIGMA = DDPG_config['SIGMA'] 47 | 48 | ou = OU(action_dim, MU, THETA, SIGMA) #Ornstein-Uhlenbeck Process 49 | 50 | BUFFER_SIZE = DDPG_config['BUFFER_SIZE'] 51 | BATCH_SIZE = DDPG_config['BATCH_SIZE'] 52 | GAMMA = DDPG_config['GAMMA'] 53 | EXPLORE = DDPG_config['EXPLORE'] 54 | EPISODE_COUNT = DDPG_config['EPISODE_COUNT'] 55 | MAX_STEPS = DDPG_config['MAX_STEPS'] 56 | if EXPLORE <= 1: 57 | EXPLORE = EPISODE_COUNT * MAX_STEPS * EXPLORE 58 | # SETUP ENDS HERE 59 | 60 | reward = 0 61 | done = False 62 | wise = False 63 | step = 0 64 | epsilon = 1 65 | indicator = 0 66 | 67 | #Tensorflow GPU optimization 68 | config = tf.ConfigProto() 69 | config.gpu_options.allow_growth = True 70 | sess = tf.Session(config=config) 71 | from keras import backend as K 72 | K.set_session(sess) 73 | 74 | actor = ActorNetwork(sess, state_dim, action_dim, DDPG_config) 75 | critic = CriticNetwork(sess, state_dim, action_dim, DDPG_config) 76 | buff = ReplayBuffer(BUFFER_SIZE) #Create replay buffer 77 | 78 | ltm = ['a_h0', 'a_h1', 'a_V', 'c_w1', 'c_a1', 'c_h1', 'c_h3', 'c_V'] 79 | layers_to_mind = {} 80 | L2 = {} 81 | 82 | for k in ltm: 83 | layers_to_mind[k] = 0 84 | L2[k] = 0 85 | 86 | vector_to_file(ltm, folder + 'weightsL2' + 'Log.csv', 'w') 87 | 88 | #Now load the weight 89 | try: 90 | actor.model.load_weights(folder + "actormodel.h5") 91 | critic.model.load_weights(folder + "criticmodel.h5") 92 | actor.target_model.load_weights(folder + "actormodel.h5") 93 | critic.target_model.load_weights(folder + "criticmodel.h5") 94 | print("Weight load successfully") 95 | except: 96 | print("Cannot find the weight") 97 | 98 | print("OMNeT++ Experiment Start.") 99 | # initial state of simulator 100 | s_t = env.reset() 101 | loss = 0 102 | for i in range(EPISODE_COUNT): 103 | 104 | print("Episode : " + str(i) + " Replay Buffer " + str(buff.count())) 105 | 106 | total_reward = 0 107 | for j in range(MAX_STEPS): 108 | epsilon -= 1.0 / EXPLORE 109 | a_t = np.zeros([1, action_dim]) 110 | noise_t = np.zeros([1, action_dim]) 111 | 112 | a_t_original = actor.model.predict(s_t.reshape(1, s_t.shape[0])) 113 | 114 | if train_indicator and epsilon > 0 and (step % 1000) // 100 != 9: 115 | noise_t[0] = epsilon * ou.evolve() 116 | 117 | a = a_t_original[0] 118 | n = noise_t[0] 119 | a_t[0] = np.where((a + n > 0) & (a + n < 1), a + n, a - n).clip(min=0, max=1) 120 | 121 | # execute action 122 | s_t1, r_t, done = env.step(a_t[0]) 123 | 124 | buff.add(s_t, a_t[0], r_t, s_t1, done) #Add replay buffer 125 | 126 | scale = lambda x: x 127 | #Do the batch update 128 | batch = buff.getBatch(BATCH_SIZE) 129 | states = scale(np.asarray([e[0] for e in batch])) 130 | actions = scale(np.asarray([e[1] for e in batch])) 131 | rewards = scale(np.asarray([e[2] for e in batch])) 132 | new_states = scale(np.asarray([e[3] for e in batch])) 133 | dones = np.asarray([e[4] for e in batch]) 134 | 135 | y_t = np.zeros([len(batch), action_dim]) 136 | target_q_values = critic.target_model.predict([new_states, actor.target_model.predict(new_states)]) 137 | 138 | for k in range(len(batch)): 139 | if dones[k]: 140 | y_t[k] = rewards[k] 141 | else: 142 | y_t[k] = rewards[k] + GAMMA*target_q_values[k] 143 | 144 | if train_indicator and len(batch) >= BATCH_SIZE: 145 | loss = critic.model.train_on_batch([states, actions], y_t) 146 | a_for_grad = actor.model.predict(states) 147 | grads = critic.gradients(states, a_for_grad) 148 | # does this give an output like train_on_batch above? NO 149 | actor.train(states, grads) 150 | actor.target_train() 151 | critic.target_train() 152 | with open(folder + 'lossLog.csv', 'a') as file: 153 | file.write(pretty(loss) + '\n') 154 | 155 | total_reward += r_t 156 | s_t = s_t1 157 | 158 | for layer in actor.model.layers + critic.model.layers: 159 | if layer.name in layers_to_mind.keys(): 160 | L2[layer.name] = np.linalg.norm(np.ravel(layer.get_weights()[0])-layers_to_mind[layer.name]) 161 | # vector_to_file(np.ravel(layer.get_weights()[0]), folder + 'weights_' + layer.name + 'Log.csv', 'a') 162 | layers_to_mind[layer.name] = np.ravel(layer.get_weights()[0]) 163 | # if max(L2.values()) <= 0.02: 164 | # wise = True 165 | 166 | if train_indicator and len(batch) >= BATCH_SIZE: 167 | vector_to_file([L2[x] for x in ltm], folder + 'weightsL2' + 'Log.csv', 'a') 168 | 169 | vector_to_file(a_t_original[0], folder + 'actionLog.csv', 'a') 170 | vector_to_file(noise_t[0], folder + 'noiseLog.csv', 'a') 171 | 172 | if 'PRINT' in DDPG_config.keys() and DDPG_config['PRINT']: 173 | print("Episode", "%5d" % i, "Step", "%5d" % step, "Reward", "%.6f" % r_t) 174 | print("Epsilon", "%.6f" % max(epsilon, 0)) 175 | 176 | att_ = np.split(a_t[0], ACTIVE_NODES) 177 | for _ in range(ACTIVE_NODES): 178 | att_[_] = np.insert(att_[_], _, -1) 179 | att_ = np.concatenate(att_) 180 | print("Action\n", att_.reshape(ACTIVE_NODES, ACTIVE_NODES)) 181 | print(max(L2, key=L2.get), pretty(max(L2.values()))) 182 | 183 | step += 1 184 | if done or wise: 185 | break 186 | 187 | if np.mod((i+1), 2) == 0: # writes at every 2nd episode 188 | if (train_indicator): 189 | actor.model.save_weights(folder + "actormodel.h5", overwrite=True) 190 | actor.model.save_weights(folder + "actormodel" + str(step) + ".h5") 191 | with open(folder + "actormodel.json", "w") as outfile: 192 | outfile.write(actor.model.to_json(indent=4) + '\n') 193 | 194 | critic.model.save_weights(folder + "criticmodel.h5", overwrite=True) 195 | critic.model.save_weights(folder + "criticmodel" + str(step) + ".h5") 196 | with open(folder + "criticmodel.json", "w") as outfile: 197 | outfile.write(critic.model.to_json(indent=4) + '\n') 198 | 199 | print("TOTAL REWARD @ " + str(i) + "-th Episode : Reward " + str(total_reward)) 200 | print("Total Step: " + str(step)) 201 | print("") 202 | 203 | env.end() # This is for shutting down 204 | print("Finish.") 205 | 206 | 207 | if __name__ == "__main__": 208 | # VANILLA 209 | if len(sys.argv) == 1: 210 | with open('DDPG.json') as jconfig: 211 | DDPG_config = json.load(jconfig) 212 | DDPG_config['EXPERIMENT'] = setup_exp() 213 | playGame(DDPG_config, train_indicator=1) 214 | # PLAY 215 | elif len(sys.argv) == 3: 216 | # WATCH OUT: it appends to *Log.csv files 217 | if sys.argv[1] == 'play': 218 | with open(sys.argv[2] + '/' + 'DDPG.json') as jconfig: 219 | DDPG_config = json.load(jconfig) 220 | # here remove double slash at end if present 221 | experiment = sys.argv[2] if sys.argv[2][-1] == '/' else sys.argv[2] + '/' 222 | DDPG_config['EXPERIMENT'] = experiment 223 | playGame(DDPG_config, train_indicator=0) 224 | # PLAY WITH FILE TRAFFIC 225 | elif len(sys.argv) == 4: 226 | # WATCH OUT: it appends to *Log.csv files 227 | if sys.argv[1] == 'play': 228 | with open(sys.argv[2] + '/' + 'DDPG.json') as jconfig: 229 | DDPG_config = json.load(jconfig) 230 | # here remove double slash at end if present 231 | experiment = sys.argv[2] if sys.argv[2][-1] == '/' else sys.argv[2] + '/' 232 | DDPG_config['EXPERIMENT'] = experiment 233 | # DDPG_config['EPISODE_COUNT'] = 1 234 | # DDPG_config['MAX_STEPS'] = 1 235 | if DDPG_config['TRAFFIC'] == 'DIR:': 236 | DDPG_config['TRAFFIC'] += sys.argv[3] 237 | playGame(DDPG_config, train_indicator=0) 238 | -------------------------------------------------------------------------------- /Environment.py: -------------------------------------------------------------------------------- 1 | """ 2 | Environment.py 3 | """ 4 | __author__ = "giorgio@ac.upc.edu" 5 | 6 | import numpy as np 7 | from scipy import stats 8 | import subprocess 9 | import networkx as nx 10 | 11 | from helper import pretty, softmax 12 | from Traffic import Traffic 13 | 14 | 15 | OMTRAFFIC = 'Traffic.txt' 16 | OMBALANCING = 'Balancing.txt' 17 | OMROUTING = 'Routing.txt' 18 | OMDELAY = 'Delay.txt' 19 | 20 | TRAFFICLOG = 'TrafficLog.csv' 21 | BALANCINGLOG = 'BalancingLog.csv' 22 | REWARDLOG = 'rewardLog.csv' 23 | WHOLELOG = 'Log.csv' 24 | OMLOG = 'omnetLog.csv' 25 | 26 | 27 | # FROM MATRIX 28 | def matrix_to_rl(matrix): 29 | return matrix[(matrix!=-1)] 30 | 31 | matrix_to_log_v = matrix_to_rl 32 | 33 | def matrix_to_omnet_v(matrix): 34 | return matrix.flatten() 35 | 36 | def vector_to_file(vector, file_name, action): 37 | string = ','.join(pretty(_) for _ in vector) 38 | with open(file_name, action) as file: 39 | return file.write(string + '\n') 40 | 41 | 42 | # FROM FILE 43 | def file_to_csv(file_name): 44 | # reads file, outputs csv 45 | with open(file_name, 'r') as file: 46 | return file.readline().strip().strip(',') 47 | 48 | def csv_to_matrix(string, nodes_num): 49 | # reads text, outputs matrix 50 | v = np.asarray(tuple(float(x) for x in string.split(',')[:nodes_num**2])) 51 | M = np.split(v, nodes_num) 52 | return np.vstack(M) 53 | 54 | def csv_to_lost(string): 55 | return float(string.split(',')[-1]) 56 | 57 | 58 | # FROM RL 59 | def rl_to_matrix(vector, nodes_num): 60 | M = np.split(vector, nodes_num) 61 | for _ in range(nodes_num): 62 | M[_] = np.insert(M[_], _, -1) 63 | return np.vstack(M) 64 | 65 | 66 | # TO RL 67 | def rl_state(env): 68 | if env.STATUM == 'RT': 69 | return np.concatenate((matrix_to_rl(env.env_B), matrix_to_rl(env.env_T))) 70 | elif env.STATUM == 'T': 71 | return matrix_to_rl(env.env_T) 72 | 73 | def rl_reward(env): 74 | 75 | delay = np.asarray(env.env_D) 76 | mask = delay == np.inf 77 | delay[mask] = len(delay)*np.max(delay[~mask]) 78 | 79 | if env.PRAEMIUM == 'AVG': 80 | reward = -np.mean(matrix_to_rl(delay)) 81 | elif env.PRAEMIUM == 'MAX': 82 | reward = -np.max(matrix_to_rl(delay)) 83 | elif env.PRAEMIUM == 'AXM': 84 | reward = -(np.mean(matrix_to_rl(delay)) + np.max(matrix_to_rl(delay)))/2 85 | elif env.PRAEMIUM == 'GEO': 86 | reward = -stats.gmean(matrix_to_rl(delay)) 87 | elif env.PRAEMIUM == 'LOST': 88 | reward = -env.env_L 89 | return reward 90 | 91 | 92 | # WRAPPER ITSELF 93 | def omnet_wrapper(env): 94 | if env.ENV == 'label': 95 | sim = 'router' 96 | elif env.ENV == 'balancing': 97 | sim = 'balancer' 98 | 99 | prefix = '' 100 | if env.CLUSTER == 'arvei': 101 | prefix = '/scratch/nas/1/giorgio/rlnet/' 102 | 103 | simexe = prefix + 'omnet/' + sim + '/networkRL' 104 | simfolder = prefix + 'omnet/' + sim + '/' 105 | simini = prefix + 'omnet/' + sim + '/' + 'omnetpp.ini' 106 | 107 | try: 108 | omnet_output = subprocess.check_output([simexe, '-n', simfolder, simini, env.folder + 'folder.ini']).decode() 109 | except Exception as e: 110 | omnet_output = e.stdout.decode() 111 | 112 | if 'Error' in omnet_output: 113 | omnet_output = omnet_output.replace(',', '') 114 | o_u_l = [_.strip() for _ in omnet_output.split('\n') if _ is not ''] 115 | omnet_output = ','.join(o_u_l[4:]) 116 | else: 117 | omnet_output = 'ok' 118 | 119 | vector_to_file([omnet_output], env.folder + OMLOG, 'a') 120 | 121 | 122 | def ned_to_capacity(env): 123 | if env.ENV == 'label': 124 | sim = 'router' 125 | elif env.ENV == 'balancing': 126 | sim = 'balancer' 127 | NED = 'omnet/' + sim + '/NetworkAll.ned' 128 | 129 | capacity = 0 130 | 131 | with open(NED) as nedfile: 132 | for line in nedfile: 133 | if "SlowChannel" in line and "<-->" in line: 134 | capacity += 3 135 | elif "MediumChannel" in line and "<-->" in line: 136 | capacity += 5 137 | elif "FastChannel" in line and "<-->" in line: 138 | capacity += 10 139 | elif "Channel" in line and "<-->" in line: 140 | capacity += 10 141 | 142 | return capacity or None 143 | 144 | 145 | # balancing environment 146 | class OmnetBalancerEnv(): 147 | 148 | def __init__(self, DDPG_config, folder): 149 | self.ENV = 'balancing' 150 | self.ROUTING = 'Balancer' 151 | 152 | self.folder = folder 153 | 154 | self.ACTIVE_NODES = DDPG_config['ACTIVE_NODES'] 155 | 156 | self.ACTUM = DDPG_config['ACTUM'] 157 | self.a_dim = self.ACTIVE_NODES**2 - self.ACTIVE_NODES # routing table minus diagonal 158 | 159 | self.s_dim = self.ACTIVE_NODES**2 - self.ACTIVE_NODES # traffic minus diagonal 160 | 161 | self.STATUM = DDPG_config['STATUM'] 162 | if self.STATUM == 'RT': 163 | self.s_dim *= 2 # traffic + routing table minus diagonals 164 | 165 | if 'MAX_DELTA' in DDPG_config.keys(): 166 | self.MAX_DELTA = DDPG_config['MAX_DELTA'] 167 | 168 | self.PRAEMIUM = DDPG_config['PRAEMIUM'] 169 | 170 | capacity = self.ACTIVE_NODES * (self.ACTIVE_NODES -1) 171 | 172 | self.TRAFFIC = DDPG_config['TRAFFIC'] 173 | self.tgen = Traffic(self.ACTIVE_NODES, self.TRAFFIC, capacity) 174 | 175 | self.CLUSTER = DDPG_config['CLUSTER'] if 'CLUSTER' in DDPG_config.keys() else False 176 | 177 | self.env_T = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=float) # traffic 178 | self.env_B = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=float) # balancing 179 | self.env_D = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=float) # delay 180 | self.env_L = -1.0 # lost packets 181 | 182 | self.counter = 0 183 | 184 | 185 | def upd_env_T(self, matrix): 186 | self.env_T = np.asarray(matrix) 187 | np.fill_diagonal(self.env_T, -1) 188 | 189 | def upd_env_B(self, matrix): 190 | self.env_B = np.asarray(matrix) 191 | np.fill_diagonal(self.env_B, -1) 192 | 193 | def upd_env_D(self, matrix): 194 | self.env_D = np.asarray(matrix) 195 | np.fill_diagonal(self.env_D, -1) 196 | 197 | def upd_env_L(self, number): 198 | self.env_L = number 199 | 200 | 201 | def logheader(self): 202 | nice_matrix = np.chararray([self.ACTIVE_NODES]*2, itemsize=20) 203 | for i in range(self.ACTIVE_NODES): 204 | for j in range(self.ACTIVE_NODES): 205 | nice_matrix[i][j] = str(i) + '-' + str(j) 206 | np.fill_diagonal(nice_matrix, '_') 207 | nice_list = list(nice_matrix[(nice_matrix!=b'_')]) 208 | th = ['t' + _.decode('ascii') for _ in nice_list] 209 | rh = ['r' + _.decode('ascii') for _ in nice_list] 210 | dh = ['d' + _.decode('ascii') for _ in nice_list] 211 | if self.STATUM == 'T': 212 | sh = ['s' + _.decode('ascii') for _ in nice_list] 213 | elif self.STATUM == 'RT': 214 | sh = ['sr' + _.decode('ascii') for _ in nice_list] + ['st' + _.decode('ascii') for _ in nice_list] 215 | ah = ['a' + _.decode('ascii') for _ in nice_list] 216 | header = ['counter'] + th + rh + dh + ['lost'] + sh + ah + ['reward'] 217 | vector_to_file(header, self.folder + WHOLELOG, 'w') 218 | 219 | 220 | def render(self): 221 | return True 222 | 223 | 224 | def reset(self): 225 | if self.counter != 0: 226 | return None 227 | 228 | self.logheader() 229 | 230 | # balancing 231 | self.upd_env_B(np.full([self.ACTIVE_NODES]*2, 0.50, dtype=float)) 232 | if self.ACTUM == 'DELTA': 233 | vector_to_file(matrix_to_omnet_v(self.env_B), self.folder + OMBALANCING, 'w') 234 | 235 | # traffic 236 | self.upd_env_T(self.tgen.generate()) 237 | 238 | vector_to_file(matrix_to_omnet_v(self.env_T), self.folder + OMTRAFFIC, 'w') 239 | 240 | return rl_state(self) 241 | 242 | 243 | def step(self, action): 244 | self.counter += 1 245 | 246 | # define action: NEW or DELTA 247 | if self.ACTUM == 'NEW': 248 | # bound the action 249 | self.upd_env_B(rl_to_matrix(np.clip(action, 0, 1), self.ACTIVE_NODES)) 250 | if self.ACTUM == 'DELTA': 251 | # bound the action 252 | self.upd_env_B(rl_to_matrix(np.clip(action * self.MAX_DELTA + matrix_to_rl(self.env_B), 0, 1), self.ACTIVE_NODES)) 253 | 254 | # write to file input for Omnet: Balancing 255 | vector_to_file(matrix_to_omnet_v(self.env_B), self.folder + OMBALANCING, 'w') 256 | 257 | # execute omnet 258 | omnet_wrapper(self) 259 | 260 | # read Omnet's output: Delay and Lost 261 | om_output = file_to_csv(self.folder + OMDELAY) 262 | self.upd_env_D(csv_to_matrix(om_output, self.ACTIVE_NODES)) 263 | self.upd_env_L(csv_to_lost(om_output)) 264 | 265 | reward = rl_reward(self) 266 | 267 | # log everything to file 268 | vector_to_file([-reward], self.folder + REWARDLOG, 'a') 269 | cur_state = rl_state(self) 270 | log = np.concatenate(([self.counter], matrix_to_log_v(self.env_T), matrix_to_log_v(self.env_B), matrix_to_log_v(self.env_D), [self.env_L], cur_state, action, [-reward])) 271 | vector_to_file(log, self.folder + WHOLELOG, 'a') 272 | 273 | # generate traffic for next iteration 274 | self.upd_env_T(self.tgen.generate()) 275 | # write to file input for Omnet: Traffic, or do nothing if static 276 | if self.TRAFFIC.split(':')[0] not in ('STAT', 'STATEQ', 'FILE'): 277 | vector_to_file(matrix_to_omnet_v(self.env_T), self.folder + OMTRAFFIC, 'w') 278 | 279 | new_state = rl_state(self) 280 | # return new status and reward 281 | return new_state, reward, 0 282 | 283 | 284 | def end(self): 285 | return 286 | 287 | 288 | # label environment 289 | class OmnetLinkweightEnv(): 290 | 291 | def __init__(self, DDPG_config, folder): 292 | self.ENV = 'label' 293 | self.ROUTING = 'Linkweight' 294 | 295 | self.folder = folder 296 | 297 | self.ACTIVE_NODES = DDPG_config['ACTIVE_NODES'] 298 | 299 | self.ACTUM = DDPG_config['ACTUM'] 300 | 301 | topology = 'omnet/router/NetworkAll.matrix' 302 | self.graph = nx.Graph(np.loadtxt(topology, dtype=int)) 303 | if self.ACTIVE_NODES != self.graph.number_of_nodes(): 304 | return False 305 | ports = 'omnet/router/NetworkAll.ports' 306 | self.ports = np.loadtxt(ports, dtype=int) 307 | 308 | self.a_dim = self.graph.number_of_edges() 309 | 310 | self.s_dim = self.ACTIVE_NODES**2 - self.ACTIVE_NODES # traffic minus diagonal 311 | 312 | self.STATUM = DDPG_config['STATUM'] 313 | if self.STATUM == 'RT': 314 | self.s_dim *= 2 # traffic + routing table minus diagonals 315 | 316 | self.PRAEMIUM = DDPG_config['PRAEMIUM'] 317 | 318 | capacity = self.ACTIVE_NODES * (self.ACTIVE_NODES -1) 319 | 320 | self.TRAFFIC = DDPG_config['TRAFFIC'] 321 | self.tgen = Traffic(self.ACTIVE_NODES, self.TRAFFIC, capacity) 322 | 323 | self.CLUSTER = DDPG_config['CLUSTER'] if 'CLUSTER' in DDPG_config.keys() else False 324 | 325 | self.env_T = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=float) # traffic 326 | self.env_W = np.full([self.a_dim], -1.0, dtype=float) # weights 327 | self.env_R = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=int) # routing 328 | self.env_Rn = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=int) # routing (nodes) 329 | self.env_D = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=float) # delay 330 | self.env_L = -1.0 # lost packets 331 | 332 | self.counter = 0 333 | 334 | 335 | def upd_env_T(self, matrix): 336 | self.env_T = np.asarray(matrix) 337 | np.fill_diagonal(self.env_T, -1) 338 | 339 | def upd_env_W(self, vector): 340 | self.env_W = np.asarray(softmax(vector)) 341 | 342 | def upd_env_R(self): 343 | weights = {} 344 | 345 | for e, w in zip(self.graph.edges(), self.env_W): 346 | weights[e] = w 347 | 348 | nx.set_edge_attributes(self.graph, 'weight', weights) 349 | 350 | routing_nodes = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=int) 351 | routing_ports = np.full([self.ACTIVE_NODES]*2, -1.0, dtype=int) 352 | 353 | all_shortest = nx.all_pairs_dijkstra_path(self.graph) 354 | 355 | for s in range(self.ACTIVE_NODES): 356 | for d in range(self.ACTIVE_NODES): 357 | if s != d: 358 | next = all_shortest[s][d][1] 359 | port = self.ports[s][next] 360 | routing_nodes[s][d] = next 361 | routing_ports[s][d] = port 362 | else: 363 | routing_nodes[s][d] = -1 364 | routing_ports[s][d] = -1 365 | 366 | self.env_R = np.asarray(routing_ports) 367 | self.env_Rn = np.asarray(routing_nodes) 368 | 369 | def upd_env_R_from_R(self, routing): 370 | routing_nodes = np.fromstring(routing, sep=',', dtype=int) 371 | M = np.split(np.asarray(routing_nodes), self.ACTIVE_NODES) 372 | routing_nodes = np.vstack(M) 373 | 374 | routing_ports = np.zeros([self.ACTIVE_NODES]*2, dtype=int) 375 | 376 | for s in range(self.ACTIVE_NODES): 377 | for d in range(self.ACTIVE_NODES): 378 | if s != d: 379 | next = routing_nodes[s][d] 380 | port = self.ports[s][next] 381 | routing_ports[s][d] = port 382 | else: 383 | routing_ports[s][d] = -1 384 | 385 | self.env_R = np.asarray(routing_ports) 386 | self.env_Rn = np.asarray(routing_nodes) 387 | 388 | def upd_env_D(self, matrix): 389 | self.env_D = np.asarray(matrix) 390 | np.fill_diagonal(self.env_D, -1) 391 | 392 | def upd_env_L(self, number): 393 | self.env_L = number 394 | 395 | 396 | def logheader(self, easy=False): 397 | nice_matrix = np.chararray([self.ACTIVE_NODES]*2, itemsize=20) 398 | for i in range(self.ACTIVE_NODES): 399 | for j in range(self.ACTIVE_NODES): 400 | nice_matrix[i][j] = str(i) + '-' + str(j) 401 | np.fill_diagonal(nice_matrix, '_') 402 | nice_list = list(nice_matrix[(nice_matrix!=b'_')]) 403 | th = ['t' + _.decode('ascii') for _ in nice_list] 404 | rh = ['r' + _.decode('ascii') for _ in nice_list] 405 | dh = ['d' + _.decode('ascii') for _ in nice_list] 406 | ah = ['a' + str(_[0]) + '-' + str(_[1]) for _ in self.graph.edges()] 407 | header = ['counter'] + th + rh + dh + ['lost'] + ah + ['reward'] 408 | if easy: 409 | header = ['counter', 'lost', 'AVG', 'MAX', 'AXM', 'GEO'] 410 | vector_to_file(header, self.folder + WHOLELOG, 'w') 411 | 412 | 413 | def render(self): 414 | return 415 | 416 | 417 | def reset(self, easy=False): 418 | if self.counter != 0: 419 | return None 420 | 421 | self.logheader(easy) 422 | 423 | # routing 424 | self.upd_env_W(np.full([self.a_dim], 0.50, dtype=float)) 425 | self.upd_env_R() 426 | if self.ACTUM == 'DELTA': 427 | vector_to_file(matrix_to_omnet_v(self.env_R), self.folder + OMROUTING, 'w') 428 | # VERIFY FILE POSITION AND FORMAT (separator, matrix/vector) np.savetxt("tmp.txt", routing, fmt="%d") 429 | 430 | # traffic 431 | self.upd_env_T(self.tgen.generate()) 432 | 433 | vector_to_file(matrix_to_omnet_v(self.env_T), self.folder + OMTRAFFIC, 'w') 434 | 435 | return rl_state(self) 436 | 437 | 438 | def step(self, action): 439 | self.counter += 1 440 | 441 | self.upd_env_W(action) 442 | self.upd_env_R() 443 | 444 | # write to file input for Omnet: Routing 445 | vector_to_file(matrix_to_omnet_v(self.env_R), self.folder + OMROUTING, 'w') 446 | # VERIFY FILE POSITION AND FORMAT (separator, matrix/vector) np.savetxt("tmp.txt", routing, fmt="%d") 447 | 448 | # execute omnet 449 | omnet_wrapper(self) 450 | 451 | # read Omnet's output: Delay and Lost 452 | om_output = file_to_csv(self.folder + OMDELAY) 453 | self.upd_env_D(csv_to_matrix(om_output, self.ACTIVE_NODES)) 454 | self.upd_env_L(csv_to_lost(om_output)) 455 | 456 | reward = rl_reward(self) 457 | 458 | # log everything to file 459 | vector_to_file([-reward], self.folder + REWARDLOG, 'a') 460 | cur_state = rl_state(self) 461 | log = np.concatenate(([self.counter], matrix_to_log_v(self.env_T), matrix_to_log_v(self.env_Rn), matrix_to_log_v(self.env_D), [self.env_L], matrix_to_log_v(self.env_W), [-reward])) 462 | vector_to_file(log, self.folder + WHOLELOG, 'a') 463 | 464 | # generate traffic for next iteration 465 | self.upd_env_T(self.tgen.generate()) 466 | # write to file input for Omnet: Traffic, or do nothing if static 467 | if self.TRAFFIC.split(':')[0] not in ('STAT', 'STATEQ', 'FILE'): 468 | vector_to_file(matrix_to_omnet_v(self.env_T), self.folder + OMTRAFFIC, 'w') 469 | 470 | new_state = rl_state(self) 471 | # return new status and reward 472 | return new_state, reward, 0 473 | 474 | 475 | def easystep(self, action): 476 | self.counter += 1 477 | 478 | self.upd_env_R_from_R(action) 479 | 480 | # write to file input for Omnet: Routing 481 | vector_to_file(matrix_to_omnet_v(self.env_R), self.folder + OMROUTING, 'w') 482 | # VERIFY FILE POSITION AND FORMAT (separator, matrix/vector) np.savetxt("tmp.txt", routing, fmt="%d") 483 | 484 | # execute omnet 485 | omnet_wrapper(self) 486 | 487 | # read Omnet's output: Delay and Lost 488 | om_output = file_to_csv(self.folder + OMDELAY) 489 | self.upd_env_D(csv_to_matrix(om_output, self.ACTIVE_NODES)) 490 | self.upd_env_L(csv_to_lost(om_output)) 491 | 492 | reward = rl_reward(self) 493 | 494 | # log everything to file 495 | vector_to_file([-reward], self.folder + REWARDLOG, 'a') 496 | cur_state = rl_state(self) 497 | log = np.concatenate(([self.counter], [self.env_L], [np.mean(matrix_to_rl(self.env_D))], [np.max(matrix_to_rl(self.env_D))], [(np.mean(matrix_to_rl(self.env_D)) + np.max(matrix_to_rl(self.env_D)))/2], [stats.gmean(matrix_to_rl(self.env_D))])) 498 | vector_to_file(log, self.folder + WHOLELOG, 'a') 499 | 500 | # generate traffic for next iteration 501 | self.upd_env_T(self.tgen.generate()) 502 | # write to file input for Omnet: Traffic, or do nothing if static 503 | if self.TRAFFIC.split(':')[0] not in ('STAT', 'STATEQ', 'FILE', 'DIR'): 504 | vector_to_file(matrix_to_omnet_v(self.env_T), self.folder + OMTRAFFIC, 'w') 505 | 506 | new_state = rl_state(self) 507 | # return new status and reward 508 | return new_state, reward, 0 509 | 510 | 511 | def end(self): 512 | return 513 | --------------------------------------------------------------------------------