├── Distributed Network Intrusion Detection System with Machine Learning ├── Apache Storm cluster │ ├── IDSTopology.java │ ├── bolt │ │ ├── AggregatorBolt.py │ │ ├── BruteForceSShBolt.py │ │ ├── DDosBolt.py │ │ ├── DosBolt.py │ │ └── InfiltrateBolt.py │ └── spout │ │ └── CSVSpout.java ├── Documentation │ ├── Black Book on IDS - Final.pdf │ └── NIDS Paper for IFIP Networking 2018.pdf ├── helper files │ ├── PacketSniffer.py │ ├── Results.py │ └── attach.py ├── machine learning │ ├── csvBruteForceTest.csv │ ├── csvBruteForceTrain.csv │ ├── csvDDosTest.csv │ ├── csvDDosTrain.csv │ ├── csvDosTest.csv │ ├── csvDosTrain.csv │ ├── csvInfiltrateTest.csv │ ├── csvInfiltrateTrain.csv │ ├── load.py │ └── train.py ├── python and parsed data │ ├── BruteForceTrain.pkl │ ├── BruteForceTrain.pkl_01.npy │ ├── BruteForceTrain.pkl_02.npy │ ├── BruteForceTrain.pkl_03.npy │ ├── BruteForceTrain.pkl_04.npy │ ├── DDosTrain.pkl │ ├── DDosTrain.pkl_01.npy │ ├── DDosTrain.pkl_02.npy │ ├── DDosTrain.pkl_03.npy │ ├── DDosTrain.pkl_04.npy │ ├── DosTrain.pkl │ ├── DosTrain.pkl_01.npy │ ├── DosTrain.pkl_02.npy │ ├── DosTrain.pkl_03.npy │ ├── DosTrain.pkl_04.npy │ ├── InfiltrateTrain.pkl │ ├── InfiltrateTrain.pkl_01.npy │ ├── InfiltrateTrain.pkl_02.npy │ ├── InfiltrateTrain.pkl_03.npy │ ├── InfiltrateTrain.pkl_04.npy │ ├── data.dot │ ├── load.py │ ├── load.pyc │ └── train.py └── src │ ├── AggregatorBolt.py │ ├── BruteForceSShBolt.py │ ├── CSVSpout.java │ ├── DDosBolt.py │ ├── DosBolt.py │ ├── IDSTopology.java │ └── InfiltrateBolt.py └── README.md /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/IDSTopology.java: -------------------------------------------------------------------------------- 1 | package storm.IDS; 2 | 3 | import backtype.storm.Config; 4 | import backtype.storm.LocalCluster; 5 | import backtype.storm.StormSubmitter; 6 | import backtype.storm.task.ShellBolt; 7 | import backtype.storm.topology.BasicOutputCollector; 8 | import backtype.storm.topology.IRichBolt; 9 | import backtype.storm.topology.OutputFieldsDeclarer; 10 | import backtype.storm.topology.TopologyBuilder; 11 | import backtype.storm.topology.base.BaseBasicBolt; 12 | import backtype.storm.tuple.Fields; 13 | import backtype.storm.tuple.Tuple; 14 | import backtype.storm.tuple.Values; 15 | import storm.IDS.spout.CSVSpout; 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | /** 20 | * This topology demonstrates Storm's stream groupings and multilang capabilities. 21 | */ 22 | public class IDSTopology { 23 | public static class DosBolt extends ShellBolt implements IRichBolt { 24 | 25 | public DosBolt() { 26 | super("python", "DosBolt.py"); 27 | } 28 | 29 | @Override 30 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 31 | declarer.declare(new Fields(String[])); 32 | } 33 | 34 | @Override 35 | public Map getComponentConfiguration() { 36 | return null; 37 | } 38 | } 39 | 40 | public static class DDosBolt extends ShellBolt implements IRichBolt { 41 | 42 | public DDosBolt() { 43 | super("python", "DDosBolt.py"); 44 | } 45 | 46 | @Override 47 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 48 | declarer.declare(new Fields(String[])); 49 | } 50 | 51 | @Override 52 | public Map getComponentConfiguration() { 53 | return null; 54 | } 55 | } 56 | 57 | public static class InfiltrateBolt extends ShellBolt implements IRichBolt { 58 | 59 | public InfiltrateBolt() { 60 | super("python", "InfiltrateBolt.py"); 61 | } 62 | 63 | @Override 64 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 65 | declarer.declare(new Fields(String[])); 66 | } 67 | 68 | @Override 69 | public Map getComponentConfiguration() { 70 | return null; 71 | } 72 | } 73 | 74 | public static class BruteForceSSHBolt extends ShellBolt implements IRichBolt { 75 | 76 | public DDosBolt() { 77 | super("python", "BruteForceSSHBolt.py"); 78 | } 79 | 80 | @Override 81 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 82 | declarer.declare(new Fields(String[])); 83 | } 84 | 85 | @Override 86 | public Map getComponentConfiguration() { 87 | return null; 88 | } 89 | } 90 | 91 | public static class AggregatorBolt extends ShellBolt implements IRichBolt { 92 | 93 | public AggregatorBolt() { 94 | super("python", "AggregatorBolt.py"); 95 | } 96 | 97 | @Override 98 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 99 | //declarer.declare(new Fields("word")); 100 | } 101 | 102 | @Override 103 | public Map getComponentConfiguration() { 104 | return null; 105 | } 106 | } 107 | 108 | 109 | public static void main(String[] args) throws Exception { 110 | 111 | TopologyBuilder builder = new TopologyBuilder(); 112 | 113 | builder.setSpout("spout", new CSVSpout(), 1); 114 | builder.setBolt("DosBolt", new DosBolt(), 4).shuffleGrouping("spout"); 115 | builder.setBolt("DDosBolt", new DDosBolt(), 4).shuffleGrouping("spout"); 116 | builder.setBolt("InfiltrateBolt", new InfiltrateBolt(), 4).shuffleGrouping("spout"); 117 | builder.setBolt("BruteForceSSHBolt", new BruteForceSSHBolt(), 4).shuffleGrouping("spout"); 118 | builder.setBolt("AggregatorBolt",new AggregatorBolt(), 4).shuffleGrouping("DosBolt",4).shuffleGrouping("DDosBolt",4).shuffleGrouping("InfiltrateBolt",4).shuffleGrouping("BruteForceSSHBolt",4); 119 | Config conf = new Config(); 120 | conf.setDebug(true); 121 | 122 | 123 | if (args != null && args.length > 0) { 124 | conf.setNumWorkers(3); 125 | StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology()); 126 | } 127 | else { 128 | conf.setMaxTaskParallelism(4); 129 | 130 | LocalCluster cluster = new LocalCluster(); 131 | cluster.submitTopology("dist-IDS", conf, builder.createTopology()); 132 | 133 | Thread.sleep(10000); 134 | 135 | cluster.shutdown(); 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/bolt/AggregatorBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | 3 | class AggregatorBolt(storm.BasicBolt): 4 | def process(self,tup): 5 | 6 | 7 | 8 | AggregatorBolt.run() 9 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/bolt/BruteForceSShBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class BruteForceSSHBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/BruteForceTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | BruteForceSSHBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/bolt/DDosBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class DDosBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/DDosTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | DDosBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/bolt/DosBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class DosBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/DosTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | DosBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/bolt/InfiltrateBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class InfiltrateBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/InfiltrateTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | InfiltrateBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Apache Storm cluster/spout/CSVSpout.java: -------------------------------------------------------------------------------- 1 | package storm.IDS.spout; 2 | 3 | import backtype.storm.spout.SpoutOutputCollector; 4 | import backtype.storm.task.TopologyContext; 5 | import backtype.storm.topology.OutputFieldsDeclarer; 6 | import backtype.storm.topology.base.BaseRichSpout; 7 | import backtype.storm.tuple.Fields; 8 | import backtype.storm.tuple.Values; 9 | import backtype.storm.utils.Utils; 10 | 11 | import java.util.Map; 12 | import java.util.Random; 13 | import java.io.BufferedReader; 14 | import java.io.FileNotFoundException; 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | 18 | public class CSVSpout extends BaseRichSpout { 19 | SpoutOutputCollector _collector; 20 | FileReader file = new FileReader(); 21 | BufferedReader br = null; 22 | String cvsSplitBy = ","; 23 | 24 | 25 | @Override 26 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 27 | _collector = collector; 28 | String csvFile = "/home/student/storm/apache-storm-0.10.0/project/data/testData.csv"; 29 | 30 | try { 31 | br = new BufferedReader(new FileReader(csvFile)); 32 | } 33 | 34 | } catch (FileNotFoundException e) { 35 | e.printStackTrace(); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } finally { 39 | if (br != null) { 40 | try { 41 | br.close(); 42 | } catch (IOException e) { 43 | e.printStackTrace(); 44 | } 45 | } 46 | 47 | } 48 | 49 | @Override 50 | public void nextTuple() { 51 | while ((line = br.readLine()) != null) { 52 | // use comma as separator 53 | String[] packet = line.split(cvsSplitBy); 54 | } 55 | 56 | _collector.emit(new Values(packet)); 57 | } 58 | 59 | @Override 60 | public void ack(Object id) { 61 | } 62 | 63 | @Override 64 | public void fail(Object id) { 65 | } 66 | 67 | @Override 68 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 69 | declarer.declare(new Fields("packet")); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Documentation/Black Book on IDS - Final.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/Documentation/Black Book on IDS - Final.pdf -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/Documentation/NIDS Paper for IFIP Networking 2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/Documentation/NIDS Paper for IFIP Networking 2018.pdf -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/helper files/PacketSniffer.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import re 3 | from datetime import datetime as dt 4 | i = 0 5 | path = "C:\\Users\\student\\Project\\sniffer\\" 6 | file = "mypcap1.pcap" 7 | csvfile = "C:\\Users\\student\\Project\\sniffer\\captureCSV.csv" 8 | fcsv = open(csvfile, "ab") 9 | fw = csv.writer(fcsv) 10 | UID = 0 11 | with open(path+file, "r") as fptr: 12 | linetmp = "" 13 | lines = [] 14 | for line in fptr: 15 | tokens = [] 16 | line = line.strip() 17 | linetmp += " " + line 18 | i += 1 19 | if i == 2: 20 | lines.append(linetmp.strip()) 21 | i = 0 22 | linetmp = "" 23 | transactionStart = 0 24 | transaction = [] 25 | tmptrans = [] 26 | for item in lines: 27 | if "(correct)" in item and transactionStart != 1: 28 | transactionStart = 1 29 | tmptrans.append(item) 30 | continue 31 | if "(correct)" in item and transactionStart == 1: 32 | tmptrans.append(item) 33 | transaction.append(tmptrans) 34 | tmptrans = [] 35 | transactionStart = 0 36 | toCSV = [] 37 | for trans in transaction: 38 | startTime = "" 39 | timeFlag = 0 40 | stopTime = "" 41 | source = "" 42 | destination = "" 43 | for item in trans: 44 | tmpIP = [] 45 | IP = [] 46 | UID += 1 47 | toCSV.append(UID) 48 | toCSV.append("SSH") 49 | matchTime = re.search(r"^\d+:\d+:\d+\.\d+\s", item) 50 | toCSV.append(matchTime.group().strip()) 51 | if timeFlag == 0: 52 | startTime = matchTime.group().strip() 53 | startTime = dt.strptime(startTime, "%H:%M:%S.%f") 54 | timeFlag = 1 55 | else: 56 | stopTime = matchTime.group().strip() 57 | stopTime = dt.strptime(stopTime, "%H:%M:%S.%f") 58 | duration = (stopTime - startTime).seconds 59 | timeFlag = 0 60 | matchIP = re.search(r"\s\d+\.\d+\.\d+\.\d+\.\d+\s>\s\d+\.\d+\.\d+\.\d+\.\d+", item) 61 | tmpIP = matchIP.group().split(">") 62 | for i in tmpIP: 63 | IP.append(i.strip()) 64 | source = IP[0] 65 | toCSV.append(source) 66 | destination = IP[1] 67 | toCSV.append(destination) 68 | matchSizeString = re.search(r"length\s\d+\)", item) 69 | matchSize = re.search(r"\d+", matchSizeString.group()) 70 | matchProtoString = re.search(r"\sproto\s[A-Za-z0-9]+\s", item) 71 | protoList = matchProtoString.group().split() 72 | protocol = protoList[1].strip() 73 | toCSV.append(protocol) 74 | toCSV.append("L2L") 75 | fw.writerow(toCSV) 76 | toCSV = [] 77 | fcsv.close() 78 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/helper files/Results.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import precision_score 2 | from sklearn.metrics import classification_report 3 | from sklearn.metrics import confusion_matrix 4 | 5 | f_Dos = open("/home/student/Dosagg_sorted.txt") 6 | f_DDos = open("/home/student/DDosagg_sorted.txt") 7 | f_Infiltrate = open("/home/student/Infiltrateagg_sorted.txt") 8 | f_BruteForceSSH = open("/home/student/BruteForceagg_sorted.txt") 9 | f_testData = open("/home/student/storm/apache-storm-0.10.0/examples/storm-starter/src/jvm/storm/starter/test-data/MasterTest.csv") 10 | 11 | predList = [] 12 | testList = [] 13 | for i in range(0,166766): 14 | dos_line = f_Dos.readline() 15 | ddos_line = f_DDos.readline() 16 | infil_line = f_Infiltrate.readline() 17 | brute_line = f_BruteForceSSH.readline() 18 | testData_line = f_testData.readline() 19 | pred_dos = int(dos_line.split(",")[9]) 20 | pred_ddos = int(ddos_line.split(",")[9]) 21 | pred_infil = int(infil_line.split(",")[9]) 22 | pred_brute = int(brute_line.split(",")[9]) 23 | pred = pred_dos | pred_ddos | pred_brute | pred_infil 24 | pred_test = int(testData_line.split(",")[9]) 25 | if pred_test == 1: 26 | pred_test = 0 27 | else: 28 | pred_test = 1 29 | predList.append(pred) 30 | testList.append(pred_test) 31 | precscore = precision_score(testList, predList) 32 | print "Precision score %f" % (precscore * 100) 33 | 34 | print "----Classification Report----" 35 | report = classification_report(testList, predList) 36 | 37 | 38 | print report 39 | 40 | print "------Confusion matrix------" 41 | 42 | cf = confusion_matrix(testList, predList) 43 | 44 | print cf 45 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/helper files/attach.py: -------------------------------------------------------------------------------- 1 | import smtplib 2 | from email.MIMEMultipart import MIMEMultipart 3 | from email.MIMEText import MIMEText 4 | from email.MIMEBase import MIMEBase 5 | from email import encoders 6 | 7 | fromaddr = "p.blesson@sitpune.edu.in" 8 | toaddr = "psblesson@gmail.com" 9 | 10 | msg = MIMEMultipart() 11 | 12 | msg['From'] = fromaddr 13 | msg['To'] = toaddr 14 | msg['Subject'] = "Intruder Alert!" 15 | 16 | body = "Please check the attachment in relation to suspicious activity on the network" 17 | 18 | msg.attach(MIMEText(body, 'plain')) 19 | 20 | filename = "attacks.txt" 21 | attachment = open("attacks.txt", "rb") 22 | 23 | part = MIMEBase('application', 'octet-stream') 24 | part.set_payload((attachment).read()) 25 | encoders.encode_base64(part) 26 | part.add_header('Content-Disposition', "attachment; filename= %s" % filename) 27 | 28 | msg.attach(part) 29 | 30 | server = smtplib.SMTP('smtp.gmail.com', 587) 31 | server.starttls() 32 | server.login(fromaddr, "onlyfortoday") 33 | text = msg.as_string() 34 | server.sendmail(fromaddr, toaddr, text) 35 | server.quit() 36 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/machine learning/load.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import sys 4 | import shutil 5 | from os import environ 6 | from os.path import join 7 | import sklearn.cross_validation 8 | 9 | import numpy as np 10 | from sklearn.cross_validation import StratifiedKFold 11 | 12 | class Bunch(dict): 13 | """Container object for datasets 14 | Dictionary-like object that exposes its keys as attributes. 15 | >>> b = Bunch(a=1, b=2) 16 | >>> b['b'] 17 | 2 18 | >>> b.b 19 | 2 20 | >>> b.a = 3 21 | >>> b['a'] 22 | 3 23 | >>> b.c = 6 24 | >>> b['c'] 25 | 6 26 | """ 27 | 28 | def __init__(self, **kwargs): 29 | dict.__init__(self, kwargs) 30 | 31 | def __setattr__(self, key, value): 32 | self[key] = value 33 | 34 | def __getattr__(self, key): 35 | try: 36 | return self[key] 37 | except KeyError: 38 | raise AttributeError(key) 39 | 40 | def __setstate__(self, state): 41 | # Bunch pickles generated with scikit-learn 0.16.* have an non 42 | # empty __dict__. This causes a surprising behaviour when 43 | # loading these pickles scikit-learn 0.17: reading bunch.key 44 | # uses __dict__ but assigning to bunch.key use __setattr__ and 45 | # only changes bunch['key']. More details can be found at: 46 | # https://github.com/scikit-learn/scikit-learn/issues/6196. 47 | # Overriding __setstate__ to be a noop has the effect of 48 | # ignoring the pickled __dict__ 49 | pass 50 | 51 | def load_iscx(): 52 | """Load and return the ISCX dataset (classification). 53 | The ISCX dataset is a classic and very easy multi-class classification 54 | dataset. 55 | ================= ============== 56 | Classes 2 57 | Samples per class 58 | Samples total 150 59 | Dimensionality 9 60 | Features real, positive 61 | ================= ============== 62 | ------- 63 | data : Bunch 64 | Dictionary-like object, the interesting attributes are: 65 | 'data', the data to learn, 'target', the classification labels, 66 | 'target_names', the meaning of the labels, 'feature_names', the 67 | meaning of the features, and 'DESCR', the 68 | full description of the dataset. 69 | """ 70 | #These are the 4 CSV Files 71 | 72 | #'csvBruteForceRandom.csv', 'csvDosRandom.csv', csvDDosRandom.csv, csvInfiltrateRandom.csv 73 | 74 | randomcsv = 'csvDDosRandom.csv' 75 | 76 | with open(randomcsv) as csv_file: 77 | data_file = csv.reader(csv_file) 78 | temp = next(data_file) 79 | n_samples = int(temp[0]) 80 | n_features = int(temp[1]) - 2 81 | target_names = np.array(temp[2:]) 82 | data = np.empty((n_samples, n_features)) 83 | target = np.empty((n_samples,), dtype=np.int) 84 | 85 | # print n_samples, n_features, target_names, data, target 86 | 87 | for i, ir in enumerate(data_file): 88 | data[i] = np.asarray(ir[1:-1], dtype=np.int) 89 | target[i] = np.asarray(ir[-1], dtype=np.int) 90 | 91 | # print data, target 92 | 93 | print "For " + randomcsv 94 | 95 | skf = StratifiedKFold(target, n_folds = 2) 96 | 97 | 98 | 99 | return Bunch(data=data, target=target, 100 | target_names=target_names, 101 | feature_names=['appName','totalDestinationPackets','totalSourcePackets','direction', 102 | 'sourceIP','protocolName','destinationIP','duration','target']), skf 103 | 104 | 105 | # printskf. 106 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/machine learning/train.py: -------------------------------------------------------------------------------- 1 | import load 2 | from sklearn import tree 3 | from sklearn.externals.six import StringIO 4 | import sklearn.cross_validation 5 | from sklearn.ensemble import RandomForestClassifier 6 | from sklearn.svm import SVC 7 | from sklearn.cross_validation import train_test_split 8 | #import time 9 | data, skf = load.load_iscx() 10 | 11 | X, y = data.data, data.target 12 | 13 | 14 | clf = tree.DecisionTreeClassifier(criterion='entropy') 15 | #clf = SVC() 16 | 17 | 18 | #scores = sklearn.cross_validation.cross_val_score(clf, data.data, data.target, cv=skf) 19 | 20 | #print scores 21 | 22 | for train_index, test_index in skf: 23 | clf = clf.fit(X[train_index], y[train_index]) 24 | print sklearn.metrics.recall_score(y[test_index],clf.predict(X[test_index])) 25 | 26 | 27 | with open("data.dot", 'w') as f: 28 | f = tree.export_graphviz(clf, out_file=f) 29 | #time.sleep(1) 30 | #exec("dot -Tpng data.dot -o ISCX34.png") 31 | 32 | #print data.target_names 33 | 34 | 35 | 36 | # 22,2,2,2,42,2,23,0,2 -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_01.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_02.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_02.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_03.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_03.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_04.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/BruteForceTrain.pkl_04.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_01.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_02.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_02.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_03.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_03.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_04.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DDosTrain.pkl_04.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_01.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_02.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_02.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_03.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_03.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_04.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/DosTrain.pkl_04.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_01.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_01.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_02.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_02.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_03.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_03.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_04.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/InfiltrateTrain.pkl_04.npy -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/data.dot: -------------------------------------------------------------------------------- 1 | digraph Tree { 2 | node [shape=box] ; 3 | 0 [label="X[3] <= 1.5\nentropy = 0.7221\nsamples = 9427\nvalue = [7541, 1886]"] ; 4 | 1 [label="X[1] <= 1.5\nentropy = 0.0827\nsamples = 6803\nvalue = [6733, 70]"] ; 5 | 0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ; 6 | 2 [label="X[6] <= 2953.5\nentropy = 0.2393\nsamples = 1677\nvalue = [1611, 66]"] ; 7 | 1 -> 2 ; 8 | 3 [label="X[2] <= 14.5\nentropy = 0.0501\nsamples = 1424\nvalue = [1416, 8]"] ; 9 | 2 -> 3 ; 10 | 4 [label="entropy = 0.0\nsamples = 1416\nvalue = [1416, 0]"] ; 11 | 3 -> 4 ; 12 | 5 [label="entropy = 0.0\nsamples = 8\nvalue = [0, 8]"] ; 13 | 3 -> 5 ; 14 | 6 [label="X[6] <= 3002.0\nentropy = 0.7767\nsamples = 253\nvalue = [195, 58]"] ; 15 | 2 -> 6 ; 16 | 7 [label="entropy = 0.0\nsamples = 58\nvalue = [0, 58]"] ; 17 | 6 -> 7 ; 18 | 8 [label="entropy = 0.0\nsamples = 195\nvalue = [195, 0]"] ; 19 | 6 -> 8 ; 20 | 9 [label="X[1] <= 3807.5\nentropy = 0.0092\nsamples = 5126\nvalue = [5122, 4]"] ; 21 | 1 -> 9 ; 22 | 10 [label="X[7] <= 118.5\nentropy = 0.0027\nsamples = 5118\nvalue = [5117, 1]"] ; 23 | 9 -> 10 ; 24 | 11 [label="entropy = 0.0\nsamples = 4894\nvalue = [4894, 0]"] ; 25 | 10 -> 11 ; 26 | 12 [label="X[7] <= 119.5\nentropy = 0.0413\nsamples = 224\nvalue = [223, 1]"] ; 27 | 10 -> 12 ; 28 | 13 [label="X[6] <= 2913.5\nentropy = 0.8113\nsamples = 4\nvalue = [3, 1]"] ; 29 | 12 -> 13 ; 30 | 14 [label="entropy = 0.0\nsamples = 2\nvalue = [2, 0]"] ; 31 | 13 -> 14 ; 32 | 15 [label="X[2] <= 25.5\nentropy = 1.0\nsamples = 2\nvalue = [1, 1]"] ; 33 | 13 -> 15 ; 34 | 16 [label="entropy = 0.0\nsamples = 1\nvalue = [0, 1]"] ; 35 | 15 -> 16 ; 36 | 17 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 37 | 15 -> 17 ; 38 | 18 [label="entropy = 0.0\nsamples = 220\nvalue = [220, 0]"] ; 39 | 12 -> 18 ; 40 | 19 [label="X[4] <= 21.0\nentropy = 0.9544\nsamples = 8\nvalue = [5, 3]"] ; 41 | 9 -> 19 ; 42 | 20 [label="entropy = 0.0\nsamples = 2\nvalue = [0, 2]"] ; 43 | 19 -> 20 ; 44 | 21 [label="X[1] <= 4784.5\nentropy = 0.65\nsamples = 6\nvalue = [5, 1]"] ; 45 | 19 -> 21 ; 46 | 22 [label="entropy = 0.0\nsamples = 1\nvalue = [0, 1]"] ; 47 | 21 -> 22 ; 48 | 23 [label="entropy = 0.0\nsamples = 5\nvalue = [5, 0]"] ; 49 | 21 -> 23 ; 50 | 24 [label="X[5] <= 1.5\nentropy = 0.8908\nsamples = 2624\nvalue = [808, 1816]"] ; 51 | 0 -> 24 [labeldistance=2.5, labelangle=-45, headlabel="False"] ; 52 | 25 [label="entropy = 0.0\nsamples = 448\nvalue = [448, 0]"] ; 53 | 24 -> 25 ; 54 | 26 [label="X[4] <= 43.5\nentropy = 0.6472\nsamples = 2176\nvalue = [360, 1816]"] ; 55 | 24 -> 26 ; 56 | 27 [label="X[4] <= 18.5\nentropy = 0.5224\nsamples = 2058\nvalue = [242, 1816]"] ; 57 | 26 -> 27 ; 58 | 28 [label="entropy = 0.0\nsamples = 102\nvalue = [102, 0]"] ; 59 | 27 -> 28 ; 60 | 29 [label="X[1] <= 7.0\nentropy = 0.3718\nsamples = 1956\nvalue = [140, 1816]"] ; 61 | 27 -> 29 ; 62 | 30 [label="X[6] <= 10.5\nentropy = 0.1159\nsamples = 1476\nvalue = [23, 1453]"] ; 63 | 29 -> 30 ; 64 | 31 [label="X[2] <= 1.5\nentropy = 0.2207\nsamples = 650\nvalue = [23, 627]"] ; 65 | 30 -> 31 ; 66 | 32 [label="X[0] <= 12.5\nentropy = 0.9751\nsamples = 27\nvalue = [16, 11]"] ; 67 | 31 -> 32 ; 68 | 33 [label="X[4] <= 33.0\nentropy = 0.9819\nsamples = 19\nvalue = [8, 11]"] ; 69 | 32 -> 33 ; 70 | 34 [label="X[4] <= 19.5\nentropy = 0.8113\nsamples = 12\nvalue = [3, 9]"] ; 71 | 33 -> 34 ; 72 | 35 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 73 | 34 -> 35 ; 74 | 36 [label="X[4] <= 27.0\nentropy = 0.684\nsamples = 11\nvalue = [2, 9]"] ; 75 | 34 -> 36 ; 76 | 37 [label="X[4] <= 21.0\nentropy = 0.7642\nsamples = 9\nvalue = [2, 7]"] ; 77 | 36 -> 37 ; 78 | 38 [label="entropy = 0.8113\nsamples = 4\nvalue = [1, 3]"] ; 79 | 37 -> 38 ; 80 | 39 [label="entropy = 0.7219\nsamples = 5\nvalue = [1, 4]"] ; 81 | 37 -> 39 ; 82 | 40 [label="entropy = 0.0\nsamples = 2\nvalue = [0, 2]"] ; 83 | 36 -> 40 ; 84 | 41 [label="X[4] <= 39.5\nentropy = 0.8631\nsamples = 7\nvalue = [5, 2]"] ; 85 | 33 -> 41 ; 86 | 42 [label="entropy = 0.0\nsamples = 5\nvalue = [5, 0]"] ; 87 | 41 -> 42 ; 88 | 43 [label="entropy = 0.0\nsamples = 2\nvalue = [0, 2]"] ; 89 | 41 -> 43 ; 90 | 44 [label="entropy = 0.0\nsamples = 8\nvalue = [8, 0]"] ; 91 | 32 -> 44 ; 92 | 45 [label="X[7] <= 89.0\nentropy = 0.0889\nsamples = 623\nvalue = [7, 616]"] ; 93 | 31 -> 45 ; 94 | 46 [label="X[2] <= 4.5\nentropy = 0.0676\nsamples = 621\nvalue = [5, 616]"] ; 95 | 45 -> 46 ; 96 | 47 [label="X[7] <= 10.5\nentropy = 0.1923\nsamples = 169\nvalue = [5, 164]"] ; 97 | 46 -> 47 ; 98 | 48 [label="X[0] <= 7.5\nentropy = 0.0995\nsamples = 155\nvalue = [2, 153]"] ; 99 | 47 -> 48 ; 100 | 49 [label="X[1] <= 5.0\nentropy = 0.0568\nsamples = 153\nvalue = [1, 152]"] ; 101 | 48 -> 49 ; 102 | 50 [label="entropy = 0.0\nsamples = 121\nvalue = [0, 121]"] ; 103 | 49 -> 50 ; 104 | 51 [label="X[7] <= 4.5\nentropy = 0.2006\nsamples = 32\nvalue = [1, 31]"] ; 105 | 49 -> 51 ; 106 | 52 [label="X[2] <= 3.5\nentropy = 0.5436\nsamples = 8\nvalue = [1, 7]"] ; 107 | 51 -> 52 ; 108 | 53 [label="entropy = 0.0\nsamples = 7\nvalue = [0, 7]"] ; 109 | 52 -> 53 ; 110 | 54 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 111 | 52 -> 54 ; 112 | 55 [label="entropy = 0.0\nsamples = 24\nvalue = [0, 24]"] ; 113 | 51 -> 55 ; 114 | 56 [label="X[4] <= 28.5\nentropy = 1.0\nsamples = 2\nvalue = [1, 1]"] ; 115 | 48 -> 56 ; 116 | 57 [label="entropy = 0.0\nsamples = 1\nvalue = [0, 1]"] ; 117 | 56 -> 57 ; 118 | 58 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 119 | 56 -> 58 ; 120 | 59 [label="X[1] <= 3.0\nentropy = 0.7496\nsamples = 14\nvalue = [3, 11]"] ; 121 | 47 -> 59 ; 122 | 60 [label="entropy = 0.0\nsamples = 11\nvalue = [0, 11]"] ; 123 | 59 -> 60 ; 124 | 61 [label="entropy = 0.0\nsamples = 3\nvalue = [3, 0]"] ; 125 | 59 -> 61 ; 126 | 62 [label="entropy = 0.0\nsamples = 452\nvalue = [0, 452]"] ; 127 | 46 -> 62 ; 128 | 63 [label="entropy = 0.0\nsamples = 2\nvalue = [2, 0]"] ; 129 | 45 -> 63 ; 130 | 64 [label="entropy = 0.0\nsamples = 826\nvalue = [0, 826]"] ; 131 | 30 -> 64 ; 132 | 65 [label="X[0] <= 5.0\nentropy = 0.8012\nsamples = 480\nvalue = [117, 363]"] ; 133 | 29 -> 65 ; 134 | 66 [label="X[1] <= 44.0\nentropy = 0.4433\nsamples = 391\nvalue = [36, 355]"] ; 135 | 65 -> 66 ; 136 | 67 [label="X[0] <= 3.0\nentropy = 0.1967\nsamples = 361\nvalue = [11, 350]"] ; 137 | 66 -> 67 ; 138 | 68 [label="entropy = 0.0\nsamples = 5\nvalue = [5, 0]"] ; 139 | 67 -> 68 ; 140 | 69 [label="X[7] <= 20.0\nentropy = 0.1234\nsamples = 356\nvalue = [6, 350]"] ; 141 | 67 -> 69 ; 142 | 70 [label="X[1] <= 31.0\nentropy = 0.2375\nsamples = 154\nvalue = [6, 148]"] ; 143 | 69 -> 70 ; 144 | 71 [label="X[7] <= 13.5\nentropy = 0.1783\nsamples = 149\nvalue = [4, 145]"] ; 145 | 70 -> 71 ; 146 | 72 [label="X[2] <= 6.5\nentropy = 0.143\nsamples = 148\nvalue = [3, 145]"] ; 147 | 71 -> 72 ; 148 | 73 [label="X[1] <= 11.0\nentropy = 0.469\nsamples = 20\nvalue = [2, 18]"] ; 149 | 72 -> 73 ; 150 | 74 [label="entropy = 0.0\nsamples = 12\nvalue = [0, 12]"] ; 151 | 73 -> 74 ; 152 | 75 [label="X[4] <= 27.0\nentropy = 0.8113\nsamples = 8\nvalue = [2, 6]"] ; 153 | 73 -> 75 ; 154 | 76 [label="entropy = 0.0\nsamples = 3\nvalue = [0, 3]"] ; 155 | 75 -> 76 ; 156 | 77 [label="X[4] <= 38.0\nentropy = 0.971\nsamples = 5\nvalue = [2, 3]"] ; 157 | 75 -> 77 ; 158 | 78 [label="X[4] <= 33.5\nentropy = 1.0\nsamples = 4\nvalue = [2, 2]"] ; 159 | 77 -> 78 ; 160 | 79 [label="entropy = 0.9183\nsamples = 3\nvalue = [1, 2]"] ; 161 | 78 -> 79 ; 162 | 80 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 163 | 78 -> 80 ; 164 | 81 [label="entropy = 0.0\nsamples = 1\nvalue = [0, 1]"] ; 165 | 77 -> 81 ; 166 | 82 [label="X[7] <= 11.5\nentropy = 0.0659\nsamples = 128\nvalue = [1, 127]"] ; 167 | 72 -> 82 ; 168 | 83 [label="entropy = 0.0\nsamples = 120\nvalue = [0, 120]"] ; 169 | 82 -> 83 ; 170 | 84 [label="X[1] <= 15.0\nentropy = 0.5436\nsamples = 8\nvalue = [1, 7]"] ; 171 | 82 -> 84 ; 172 | 85 [label="entropy = 0.0\nsamples = 7\nvalue = [0, 7]"] ; 173 | 84 -> 85 ; 174 | 86 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 175 | 84 -> 86 ; 176 | 87 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 177 | 71 -> 87 ; 178 | 88 [label="X[7] <= 3.0\nentropy = 0.971\nsamples = 5\nvalue = [2, 3]"] ; 179 | 70 -> 88 ; 180 | 89 [label="entropy = 0.0\nsamples = 3\nvalue = [0, 3]"] ; 181 | 88 -> 89 ; 182 | 90 [label="entropy = 0.0\nsamples = 2\nvalue = [2, 0]"] ; 183 | 88 -> 90 ; 184 | 91 [label="entropy = 0.0\nsamples = 202\nvalue = [0, 202]"] ; 185 | 69 -> 91 ; 186 | 92 [label="X[7] <= 72.0\nentropy = 0.65\nsamples = 30\nvalue = [25, 5]"] ; 187 | 66 -> 92 ; 188 | 93 [label="entropy = 0.0\nsamples = 23\nvalue = [23, 0]"] ; 189 | 92 -> 93 ; 190 | 94 [label="X[1] <= 73.0\nentropy = 0.8631\nsamples = 7\nvalue = [2, 5]"] ; 191 | 92 -> 94 ; 192 | 95 [label="entropy = 0.0\nsamples = 4\nvalue = [0, 4]"] ; 193 | 94 -> 95 ; 194 | 96 [label="X[2] <= 61.5\nentropy = 0.9183\nsamples = 3\nvalue = [2, 1]"] ; 195 | 94 -> 96 ; 196 | 97 [label="entropy = 0.0\nsamples = 2\nvalue = [2, 0]"] ; 197 | 96 -> 97 ; 198 | 98 [label="entropy = 0.0\nsamples = 1\nvalue = [0, 1]"] ; 199 | 96 -> 98 ; 200 | 99 [label="X[7] <= 3.5\nentropy = 0.4361\nsamples = 89\nvalue = [81, 8]"] ; 201 | 65 -> 99 ; 202 | 100 [label="entropy = 0.0\nsamples = 74\nvalue = [74, 0]"] ; 203 | 99 -> 100 ; 204 | 101 [label="X[4] <= 19.5\nentropy = 0.9968\nsamples = 15\nvalue = [7, 8]"] ; 205 | 99 -> 101 ; 206 | 102 [label="X[2] <= 18.5\nentropy = 0.5033\nsamples = 9\nvalue = [1, 8]"] ; 207 | 101 -> 102 ; 208 | 103 [label="entropy = 0.0\nsamples = 1\nvalue = [1, 0]"] ; 209 | 102 -> 103 ; 210 | 104 [label="entropy = 0.0\nsamples = 8\nvalue = [0, 8]"] ; 211 | 102 -> 104 ; 212 | 105 [label="entropy = 0.0\nsamples = 6\nvalue = [6, 0]"] ; 213 | 101 -> 105 ; 214 | 106 [label="entropy = 0.0\nsamples = 118\nvalue = [118, 0]"] ; 215 | 26 -> 106 ; 216 | } -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/load.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import sys 4 | import shutil 5 | from os import environ 6 | from os.path import join 7 | import sklearn.cross_validation 8 | 9 | import numpy as np 10 | from sklearn.cross_validation import StratifiedKFold 11 | 12 | class Bunch(dict): 13 | """Container object for datasets 14 | Dictionary-like object that exposes its keys as attributes. 15 | >>> b = Bunch(a=1, b=2) 16 | >>> b['b'] 17 | 2 18 | >>> b.b 19 | 2 20 | >>> b.a = 3 21 | >>> b['a'] 22 | 3 23 | >>> b.c = 6 24 | >>> b['c'] 25 | 6 26 | """ 27 | 28 | def __init__(self, **kwargs): 29 | dict.__init__(self, kwargs) 30 | 31 | def __setattr__(self, key, value): 32 | self[key] = value 33 | 34 | def __getattr__(self, key): 35 | try: 36 | return self[key] 37 | except KeyError: 38 | raise AttributeError(key) 39 | 40 | def __setstate__(self, state): 41 | # Bunch pickles generated with scikit-learn 0.16.* have an non 42 | # empty __dict__. This causes a surprising behaviour when 43 | # loading these pickles scikit-learn 0.17: reading bunch.key 44 | # uses __dict__ but assigning to bunch.key use __setattr__ and 45 | # only changes bunch['key']. More details can be found at: 46 | # https://github.com/scikit-learn/scikit-learn/issues/6196. 47 | # Overriding __setstate__ to be a noop has the effect of 48 | # ignoring the pickled __dict__ 49 | pass 50 | 51 | def load_iscx(): 52 | """Load and return the ISCX dataset (classification). 53 | The ISCX dataset is a classic and very easy multi-class classification 54 | dataset. 55 | ================= ============== 56 | Classes 2 57 | Samples per class 58 | Samples total 150 59 | Dimensionality 9 60 | Features real, positive 61 | ================= ============== 62 | ------- 63 | data : Bunch 64 | Dictionary-like object, the interesting attributes are: 65 | 'data', the data to learn, 'target', the classification labels, 66 | 'target_names', the meaning of the labels, 'feature_names', the 67 | meaning of the features, and 'DESCR', the 68 | full description of the dataset. 69 | """ 70 | #These are the 4 CSV Files 71 | 72 | #'csvBruteForceRandom.csv', 'csvDosRandom.csv', csvDDosRandom.csv, csvInfiltrateRandom.csv 73 | 74 | randomcsv = '../data/csvInfiltrateTrain.csv' 75 | 76 | with open(randomcsv) as csv_file: 77 | data_file = csv.reader(csv_file) 78 | temp = next(data_file) 79 | n_samples = int(temp[0]) 80 | n_features = int(temp[1]) - 2 81 | target_names = np.array(temp[2:]) 82 | data = np.empty((n_samples, n_features)) 83 | target = np.empty((n_samples,), dtype=np.int) 84 | 85 | # print n_samples, n_features, target_names, data, target 86 | 87 | for i, ir in enumerate(data_file): 88 | data[i] = np.asarray(ir[1:-1], dtype=np.int) 89 | target[i] = np.asarray(ir[-1], dtype=np.int) 90 | 91 | # print data, target 92 | 93 | print "For " + randomcsv 94 | 95 | skf = StratifiedKFold(target, n_folds = 2) 96 | 97 | 98 | 99 | return Bunch(data=data, target=target, 100 | target_names=target_names, 101 | feature_names=['appName','totalDestinationPackets','totalSourcePackets','direction', 102 | 'sourceIP','protocolName','destinationIP','duration','target']), skf 103 | 104 | 105 | # printskf. 106 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/load.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/indrayudhroy/Distributed-Network-Intrusion-Detection-System-with-Machine-Learning/66d313f22d27a301c828467c042ac777cc0a01ea/Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/load.pyc -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/python and parsed data/train.py: -------------------------------------------------------------------------------- 1 | import load 2 | from sklearn import tree 3 | from sklearn.externals.six import StringIO 4 | import sklearn.cross_validation 5 | from sklearn.ensemble import RandomForestClassifier 6 | from sklearn.svm import SVC 7 | from sklearn.cross_validation import train_test_split 8 | import pickle 9 | from sklearn.externals import joblib 10 | 11 | #import time 12 | data, skf= load.load_iscx() 13 | 14 | X, y = data.data, data.target 15 | 16 | 17 | clf = tree.DecisionTreeClassifier(criterion='entropy') 18 | clf = clf.fit(X,y) 19 | #clf = SVC() 20 | 21 | 22 | scores = sklearn.cross_validation.cross_val_score(clf, data.data, data.target, cv=skf) 23 | 24 | print scores 25 | 26 | #for train_index, test_index in skf: 27 | # clf = clf.fit(X[train_index], y[train_index]) 28 | # print sklearn.metrics.recall_score(y[test_index],clf.predict(X[test_index])) 29 | 30 | 31 | joblib.dump(clf, 'InfiltrateTrain.pkl') 32 | 33 | #clf = joblib.load('filename.pkl') 34 | 35 | with open("data.dot", 'w') as f: 36 | f = tree.export_graphviz(clf, out_file=f) 37 | #time.sleep(1) 38 | #exec("dot -Tpng data.dot -o ISCX34.png") 39 | 40 | #print data.target_names 41 | 42 | 43 | 44 | # 22,2,2,2,42,2,23,0,2 -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/AggregatorBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | 3 | class AggregatorBolt(storm.BasicBolt): 4 | def process(self,tup): 5 | 6 | 7 | 8 | AggregatorBolt.run() 9 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/BruteForceSShBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class BruteForceSSHBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/BruteForceTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | BruteForceSSHBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/CSVSpout.java: -------------------------------------------------------------------------------- 1 | package storm.IDS.spout; 2 | 3 | import backtype.storm.spout.SpoutOutputCollector; 4 | import backtype.storm.task.TopologyContext; 5 | import backtype.storm.topology.OutputFieldsDeclarer; 6 | import backtype.storm.topology.base.BaseRichSpout; 7 | import backtype.storm.tuple.Fields; 8 | import backtype.storm.tuple.Values; 9 | import backtype.storm.utils.Utils; 10 | 11 | import java.util.Map; 12 | import java.util.Random; 13 | import java.io.BufferedReader; 14 | import java.io.FileNotFoundException; 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | 18 | public class CSVSpout extends BaseRichSpout { 19 | SpoutOutputCollector _collector; 20 | FileReader file = new FileReader(); 21 | BufferedReader br = null; 22 | String cvsSplitBy = ","; 23 | 24 | 25 | @Override 26 | public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { 27 | _collector = collector; 28 | String csvFile = "/home/student/storm/apache-storm-0.10.0/project/data/testData.csv"; 29 | 30 | try { 31 | br = new BufferedReader(new FileReader(csvFile)); 32 | } 33 | 34 | } catch (FileNotFoundException e) { 35 | e.printStackTrace(); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } finally { 39 | if (br != null) { 40 | try { 41 | br.close(); 42 | } catch (IOException e) { 43 | e.printStackTrace(); 44 | } 45 | } 46 | 47 | } 48 | 49 | @Override 50 | public void nextTuple() { 51 | while ((line = br.readLine()) != null) { 52 | // use comma as separator 53 | String[] packet = line.split(cvsSplitBy); 54 | } 55 | 56 | _collector.emit(new Values(packet)); 57 | } 58 | 59 | @Override 60 | public void ack(Object id) { 61 | } 62 | 63 | @Override 64 | public void fail(Object id) { 65 | } 66 | 67 | @Override 68 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 69 | declarer.declare(new Fields("packet")); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/DDosBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class DDosBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/DDosTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | DDosBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/DosBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class DosBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/DosTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | DosBolt.run() 12 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/IDSTopology.java: -------------------------------------------------------------------------------- 1 | package storm.IDS; 2 | 3 | import backtype.storm.Config; 4 | import backtype.storm.LocalCluster; 5 | import backtype.storm.StormSubmitter; 6 | import backtype.storm.task.ShellBolt; 7 | import backtype.storm.topology.BasicOutputCollector; 8 | import backtype.storm.topology.IRichBolt; 9 | import backtype.storm.topology.OutputFieldsDeclarer; 10 | import backtype.storm.topology.TopologyBuilder; 11 | import backtype.storm.topology.base.BaseBasicBolt; 12 | import backtype.storm.tuple.Fields; 13 | import backtype.storm.tuple.Tuple; 14 | import backtype.storm.tuple.Values; 15 | import storm.IDS.spout.CSVSpout; 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | /** 20 | * This topology demonstrates Storm's stream groupings and multilang capabilities. 21 | */ 22 | public class IDSTopology { 23 | public static class DosBolt extends ShellBolt implements IRichBolt { 24 | 25 | public DosBolt() { 26 | super("python", "DosBolt.py"); 27 | } 28 | 29 | @Override 30 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 31 | declarer.declare(new Fields(String[])); 32 | } 33 | 34 | @Override 35 | public Map getComponentConfiguration() { 36 | return null; 37 | } 38 | } 39 | 40 | public static class DDosBolt extends ShellBolt implements IRichBolt { 41 | 42 | public DDosBolt() { 43 | super("python", "DDosBolt.py"); 44 | } 45 | 46 | @Override 47 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 48 | declarer.declare(new Fields(String[])); 49 | } 50 | 51 | @Override 52 | public Map getComponentConfiguration() { 53 | return null; 54 | } 55 | } 56 | 57 | public static class InfiltrateBolt extends ShellBolt implements IRichBolt { 58 | 59 | public InfiltrateBolt() { 60 | super("python", "InfiltrateBolt.py"); 61 | } 62 | 63 | @Override 64 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 65 | declarer.declare(new Fields(String[])); 66 | } 67 | 68 | @Override 69 | public Map getComponentConfiguration() { 70 | return null; 71 | } 72 | } 73 | 74 | public static class BruteForceSSHBolt extends ShellBolt implements IRichBolt { 75 | 76 | public DDosBolt() { 77 | super("python", "BruteForceSSHBolt.py"); 78 | } 79 | 80 | @Override 81 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 82 | declarer.declare(new Fields(String[])); 83 | } 84 | 85 | @Override 86 | public Map getComponentConfiguration() { 87 | return null; 88 | } 89 | } 90 | 91 | public static class AggregatorBolt extends ShellBolt implements IRichBolt { 92 | 93 | public AggregatorBolt() { 94 | super("python", "AggregatorBolt.py"); 95 | } 96 | 97 | @Override 98 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 99 | //declarer.declare(new Fields("word")); 100 | } 101 | 102 | @Override 103 | public Map getComponentConfiguration() { 104 | return null; 105 | } 106 | } 107 | 108 | 109 | public static void main(String[] args) throws Exception { 110 | 111 | TopologyBuilder builder = new TopologyBuilder(); 112 | 113 | builder.setSpout("spout", new CSVSpout(), 1); 114 | builder.setBolt("DosBolt", new DosBolt(), 4).shuffleGrouping("spout"); 115 | builder.setBolt("DDosBolt", new DDosBolt(), 4).shuffleGrouping("spout"); 116 | builder.setBolt("InfiltrateBolt", new InfiltrateBolt(), 4).shuffleGrouping("spout"); 117 | builder.setBolt("BruteForceSSHBolt", new BruteForceSSHBolt(), 4).shuffleGrouping("spout"); 118 | builder.setBolt("AggregatorBolt",new AggregatorBolt(), 4).shuffleGrouping("DosBolt",4).shuffleGrouping("DDosBolt",4).shuffleGrouping("InfiltrateBolt",4).shuffleGrouping("BruteForceSSHBolt",4); 119 | Config conf = new Config(); 120 | conf.setDebug(true); 121 | 122 | 123 | if (args != null && args.length > 0) { 124 | conf.setNumWorkers(3); 125 | StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology()); 126 | } 127 | else { 128 | conf.setMaxTaskParallelism(4); 129 | 130 | LocalCluster cluster = new LocalCluster(); 131 | cluster.submitTopology("dist-IDS", conf, builder.createTopology()); 132 | 133 | Thread.sleep(10000); 134 | 135 | cluster.shutdown(); 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /Distributed Network Intrusion Detection System with Machine Learning/src/InfiltrateBolt.py: -------------------------------------------------------------------------------- 1 | import storm 2 | import pickle 3 | from sklearn.externals import joblib 4 | 5 | class InfiltrateBolt(storm.BasicBolt): 6 | def process(self,tup): 7 | clf = joblib.load('/home/student/storm/apache-storm-0.10.0/project/code/InfiltrateTrain.pkl') 8 | predict = clf.predict(tup[1:-1]) 9 | output_tuple = tup[0] + predict 10 | storm.emit([output_tuple]) 11 | InfiltrateBolt.run() 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed-Network-Intrusion-Detection-System-with-Machine-Learning 2 | A research & development project to create and deploy a Network-based Intrusion Detection System (IDS) to detect intruders on a distributed system. That is, it detects and classify threatening or anomalous network traffic as opposed to safe traffic and usage. The project runs on a real-time, distributed cluster on Apache Storm which processes incoming network packets, and uses our novel algorithms and Machine Learning to detect intruders. It uses supervised Machine Learning classifiers such as decision trees, ensemble decision trees, support vector machines, etc. as well as being built with the principles of anomaly-based Intrusion Detection Systems. 3 | --------------------------------------------------------------------------------