├── .gitignore
├── BufferedPackets.py
├── LibNidsReaderThread.py
├── OnlinePcapReaderThread.py
├── PcapReaderThread.py
├── README.md
├── StreamReaderThread.py
├── TcpMessage.py
├── TcpStream.py
├── aeids.conf
├── aeids.py
├── aeids.sql
├── aeids_animation.html
├── aeids_ws.py
├── aeids_ws_online.py
└── pcap_to_csv.py


/.gitignore:
--------------------------------------------------------------------------------
  1 | models/
  2 | 
  3 | # Created by https://www.gitignore.io/api/python
  4 | # Edit at https://www.gitignore.io/?templates=python
  5 | 
  6 | ### Python ###
  7 | # Byte-compiled / optimized / DLL files
  8 | __pycache__/
  9 | *.py[cod]
 10 | *$py.class
 11 | 
 12 | # C extensions
 13 | *.so
 14 | 
 15 | # Distribution / packaging
 16 | .Python
 17 | build/
 18 | develop-eggs/
 19 | dist/
 20 | downloads/
 21 | eggs/
 22 | .eggs/
 23 | lib/
 24 | lib64/
 25 | parts/
 26 | sdist/
 27 | var/
 28 | wheels/
 29 | pip-wheel-metadata/
 30 | share/python-wheels/
 31 | *.egg-info/
 32 | .installed.cfg
 33 | *.egg
 34 | MANIFEST
 35 | 
 36 | # PyInstaller
 37 | #  Usually these files are written by a python script from a template
 38 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 39 | *.manifest
 40 | *.spec
 41 | 
 42 | # Installer logs
 43 | pip-log.txt
 44 | pip-delete-this-directory.txt
 45 | 
 46 | # Unit test / coverage reports
 47 | htmlcov/
 48 | .tox/
 49 | .nox/
 50 | .coverage
 51 | .coverage.*
 52 | .cache
 53 | nosetests.xml
 54 | coverage.xml
 55 | *.cover
 56 | .hypothesis/
 57 | .pytest_cache/
 58 | 
 59 | # Translations
 60 | *.mo
 61 | *.pot
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # pyenv
 73 | .python-version
 74 | 
 75 | # pipenv
 76 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 77 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 78 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 79 | #   install all needed dependencies.
 80 | #Pipfile.lock
 81 | 
 82 | # celery beat schedule file
 83 | celerybeat-schedule
 84 | 
 85 | # SageMath parsed files
 86 | *.sage.py
 87 | 
 88 | # Spyder project settings
 89 | .spyderproject
 90 | .spyproject
 91 | 
 92 | # Rope project settings
 93 | .ropeproject
 94 | 
 95 | # Mr Developer
 96 | .mr.developer.cfg
 97 | .project
 98 | .pydevproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | .dmypy.json
106 | dmypy.json
107 | 
108 | # Pyre type checker
109 | .pyre/
110 | 
111 | # End of https://www.gitignore.io/api/python
112 | 


--------------------------------------------------------------------------------
/BufferedPackets.py:
--------------------------------------------------------------------------------
  1 | import binascii
  2 | 
  3 | from impacket import ImpactDecoder, ImpactPacket
  4 | 
  5 | WINDOW_SIZE = 50
  6 | 
  7 | class BufferedPackets:
  8 |     def __init__(self, first_header, first_frame):
  9 |         packet = first_frame.child()
 10 |         segment = packet.child()
 11 | 
 12 |         if isinstance(segment, ImpactPacket.TCP):
 13 |             self.ready = False
 14 |         else:
 15 |             self.ready = True
 16 | 
 17 |         self.frames = []
 18 |         self.frames.append(first_frame)
 19 |         self.headers = []
 20 |         self.headers.append(first_header)
 21 |         self.id = self.generate_id(first_frame)
 22 |         self.window_counter = WINDOW_SIZE
 23 |         self.start_time = float(str(first_header.getts()[0]) + "." + str(first_header.getts()[1]))
 24 |         self.read = False
 25 | 
 26 |     def check_counter(self):
 27 |         if self.window_counter <= 0:
 28 |             self.ready = True
 29 | 
 30 |     def generate_id(self, frame):
 31 |         packet = frame.child()
 32 |         segment = packet.child()
 33 | 
 34 |         src_addr = packet.get_ip_src()
 35 |         dst_addr = packet.get_ip_dst()
 36 | 
 37 |         if isinstance(segment, ImpactPacket.TCP):
 38 |             protocol = "tcp"
 39 |             src_port = segment.get_th_sport()
 40 |             dst_port = segment.get_th_dport()
 41 |             return "{}-{}-{}-{}-{}".format(src_addr, src_port, dst_addr, dst_port, protocol)
 42 |         elif isinstance(segment, ImpactPacket.UDP):
 43 |             protocol = "udp"
 44 |             src_port = segment.get_uh_sport()
 45 |             dst_port = segment.get_uh_dport()
 46 |             return "{}-{}-{}-{}-{}".format(src_addr, src_port, dst_addr, dst_port, protocol)
 47 |         elif isinstance(segment, ImpactPacket.ICMP):
 48 |             protocol = "icmp"
 49 |             return "{}-0-{}-0-{}".format(src_addr, dst_addr, protocol)
 50 | 
 51 |     def generate_reverse_id(self, frame):
 52 |         packet = frame.child()
 53 |         segment = packet.child()
 54 | 
 55 |         src_addr = packet.get_ip_src()
 56 |         dst_addr = packet.get_ip_dst()
 57 | 
 58 |         if isinstance(segment, ImpactPacket.TCP):
 59 |             protocol = "tcp"
 60 |             src_port = segment.get_th_sport()
 61 |             dst_port = segment.get_th_dport()
 62 |             return "{}-{}-{}-{}-{}".format(dst_addr, dst_port, src_addr, src_port, protocol)
 63 |         elif isinstance(segment, ImpactPacket.UDP):
 64 |             protocol = "udp"
 65 |             src_port = segment.get_uh_sport()
 66 |             dst_port = segment.get_uh_dport()
 67 |             return "{}-{}-{}-{}-{}".format(dst_addr, dst_port, src_addr, src_port, protocol)
 68 |         elif isinstance(segment, ImpactPacket.ICMP):
 69 |             protocol = "icmp"
 70 |             return "{}-0-{}-0-{}".format(dst_addr, src_addr, protocol)
 71 | 
 72 |     def add_frame(self, frame):
 73 |         if self.ready:
 74 |             return False
 75 | 
 76 |         id = self.generate_id(frame)
 77 |         rev_id = self.generate_reverse_id(frame)
 78 |         last_frame = self.frames[-1]
 79 |         new_packet = frame.child()
 80 |         new_segment = new_packet.child()
 81 |         last_packet = last_frame.child()
 82 |         last_segment = last_packet.child()
 83 | 
 84 |         # print id + ">>>>" + self.id
 85 |         self.window_counter -= 1
 86 | 
 87 |         if self.id == id and isinstance(new_segment, ImpactPacket.TCP):
 88 |             if new_segment.get_FIN():
 89 |                 self.ready = True
 90 | 
 91 |             if new_segment.get_th_ack() != last_segment.get_th_ack(): #new frame belongs to a different flow
 92 |                 self.check_counter()
 93 |                 return False
 94 | 
 95 |             for i in range(0, len(self.frames)):
 96 |                 f = self.frames[i]
 97 |                 n = f.child()
 98 |                 s = n.child()
 99 | 
100 |                 if new_segment.get_th_seq() == s.get_th_seq(): #retransmitted packet
101 |                     self.window_counter = WINDOW_SIZE
102 |                     return True
103 |                 elif new_segment.get_th_seq() < s.get_th_seq(): # out of order packet
104 |                     self.frames.insert(i, frame)
105 |                     self.window_counter = WINDOW_SIZE
106 |                     return True
107 | 
108 |             self.frames.append(frame)
109 |             self.window_counter = WINDOW_SIZE
110 |             return True
111 |         elif self.id == rev_id and isinstance(new_segment, ImpactPacket.TCP): #frame belongs to the opposite flow
112 |             if new_segment.get_th_seq() == last_segment.get_th_ack():
113 |                 self.ready = True
114 |             elif new_segment.get_FIN():
115 |                 self.ready = True
116 | 
117 |             return False
118 |         elif not isinstance(new_segment, ImpactPacket.TCP):
119 |             self.check_counter()
120 |             return True
121 |         else:
122 |             self.check_counter()
123 |             return False
124 | 
125 |     def get_payload(self):
126 |         payload = ""
127 |         for frame in self.frames:
128 |             packet = frame.child()
129 |             segment = packet.child()
130 |             if segment.get_data_as_string() is not None:
131 |                 payload += segment.get_data_as_string()
132 | 
133 |         return payload
134 | 
135 |     def get_start_time(self):
136 |         return self.start_time
137 | 
138 |     def get_stop_time(self):
139 |         last_header = self.headers[-1]
140 |         return float(str(last_header.getts()[0]) + "." + str(last_header.getts()[1]))
141 | 
142 |     def get_hexlify_payload(self):
143 |         payload = self.get_payload()
144 |         return binascii.hexlify(payload)
145 | 
146 |     def get_byte_frequency(self):
147 |         byte_frequency = [0] * 256
148 |         payload = self.get_payload()
149 |         length = float(self.get_payload_length())
150 | 
151 |         for i in range(0, 256):
152 |             byte_frequency[i] = float(payload.count(chr(i))) / length
153 | 
154 |         return byte_frequency
155 | 
156 |     def get_payload_length(self):
157 |         length = 0
158 |         for frame in self.frames:
159 |             packet = frame.child()
160 |             segment = packet.child()
161 |             if isinstance(segment, ImpactPacket.TCP):
162 |                 length += len(segment.get_data_as_string())
163 |             elif isinstance(segment, ImpactPacket.UDP):
164 |                 length += segment.get_uh_ulen()
165 | 
166 |         return length
167 | 
168 | 
169 | 


--------------------------------------------------------------------------------
/LibNidsReaderThread.py:
--------------------------------------------------------------------------------
  1 | from collections import deque
  2 | from TcpMessage import TcpMessage
  3 | 
  4 | from inspect import getmembers
  5 | from pprint import pprint
  6 | 
  7 | import binascii
  8 | import nids
  9 | import threading
 10 | 
 11 | end_states = (nids.NIDS_CLOSE, nids.NIDS_TIMEOUT, nids.NIDS_RESET)
 12 | 
 13 | 
 14 | class LibNidsReaderThread(threading.Thread):
 15 |     def __init__(self, filename, protocol, port=None):
 16 |         threading.Thread.__init__(self)
 17 | 
 18 |         self.port = port
 19 |         self.protocol = protocol
 20 |         self.done = False
 21 |         self.connection_list = deque()
 22 |         self.lock = threading.Lock()
 23 |         self.delete_read_connections = False
 24 |         self.last_read_index = -1
 25 | 
 26 |         nids.param("filename", filename)
 27 |         nids.chksum_ctl([('0.0.0.0/0', False)])
 28 |         if port is None:
 29 |             nids.param("pcap_filter", "{}".format(self.protocol))
 30 |         else:
 31 |             nids.param("pcap_filter", "{} port {}".format(self.protocol, self.port))
 32 | 
 33 |     def run(self):
 34 |         nids.init()
 35 |         nids.register_tcp(self.handle_tcp)
 36 | 
 37 |         try:
 38 |             nids.run()
 39 |             print("DONE")
 40 |             self.done = True
 41 |         except nids.error, e:
 42 |             print "[-] Error: %s" % (e)
 43 |         except Exception, e:
 44 |             print "[-] Exception: %s" % (e)
 45 | 
 46 |     def handle_tcp(self, tcp):
 47 |         if tcp.nids_state == nids.NIDS_JUST_EST:
 48 |             ((src, sport), (dst, dport)) = tcp.addr
 49 |             tcp.client.collect = 1
 50 |             tcp.server.collect = 1
 51 |             tcp.start_ts = nids.get_pkt_ts()
 52 | 
 53 |         elif tcp.nids_state == nids.NIDS_DATA:
 54 |             tcp.discard(0)
 55 | 
 56 |         elif tcp.nids_state in end_states:
 57 |             ((src, sport), (dst, dport)) = tcp.addr
 58 |             tcp.stop_ts = nids.get_pkt_ts()
 59 |             print "[+](%s-%s) %s:%s - %s:%s (CTS: %dB | STC: %dB)" % (tcp.start_ts, tcp.stop_ts, src, sport, dst, dport,
 60 |                                                                len(tcp.server.data[:tcp.server.count]),
 61 |                                                                len(tcp.client.data[:tcp.client.count]))
 62 |             # pprint(getmembers(tcp.client))
 63 | 
 64 |             print(binascii.hexlify(tcp.server.data))
 65 |             print(len(tcp.server.data))
 66 |             raw_input("Enter to continue")
 67 |             msg = TcpMessage(tcp.client.data, tcp.server.data, (src, sport, dst, dport), tcp.start_ts, tcp.stop_ts)
 68 |             self.lock.acquire()
 69 |             self.connection_list.append(msg)
 70 |             self.lock.release()
 71 | 
 72 |     def pop_connection(self):
 73 |         self.lock.acquire()
 74 | 
 75 |         if self.delete_read_connections and len(self.connection_list) > 0:
 76 |             msg = self.connection_list.pop()
 77 |             self.lock.release()
 78 |             return msg
 79 |         elif len(self.connection_list) > 0:
 80 |             if self.last_read_index == len(self.connection_list) - 1:
 81 |                 self.lock.release()
 82 |                 return None
 83 |             else:
 84 |                 self.last_read_index += 1
 85 |                 msg = self.connection_list[self.last_read_index]
 86 |                 msg.read = True
 87 |                 self.lock.release()
 88 |                 return msg
 89 |         else:
 90 |             self.lock.release()
 91 |             return None
 92 | 
 93 |     def has_ready_message(self):
 94 |         self.lock.acquire()
 95 | 
 96 |         if self.delete_read_connections and len(self.connection_list) > 0:
 97 |             self.lock.release()
 98 |             return True
 99 |         elif len(self.connection_list) > 0:
100 |             if self.last_read_index >= len(self.connection_list) - 1:
101 |                 self.lock.release()
102 |                 return False
103 |             else:
104 |                 self.lock.release()
105 |                 return True
106 |         else:
107 |             self.lock.release()
108 |             return False
109 | 
110 |     def reset_read_status(self):
111 |         self.lock.acquire()
112 |         # print "Resetting read status"
113 |         for msg in self.connection_list:
114 |             msg.read = False
115 | 
116 |         self.last_read_index = -1
117 |         self.lock.release()
118 | 


--------------------------------------------------------------------------------
/OnlinePcapReaderThread.py:
--------------------------------------------------------------------------------
  1 | from BufferedPackets import BufferedPackets
  2 | from impacket import ImpactDecoder, ImpactPacket
  3 | 
  4 | import pcapy
  5 | import threading
  6 | 
  7 | 
  8 | class OnlinePcapReaderThread(threading.Thread):
  9 |     def __init__(self, protocol, port):
 10 |         threading.Thread.__init__(self)
 11 | 
 12 |         snaplen = 65535
 13 |         promiscious = False
 14 |         read_timeout = 0
 15 | 
 16 |         self.pcap = pcapy.open_live("any", snaplen, promiscious, read_timeout)
 17 |         self.connection_list = []
 18 |         self.ready_connection_list = []
 19 |         self.last_read_index = -1
 20 |         self.done = False
 21 |         self.port = port
 22 |         self.protocol = protocol
 23 |         self.lock = threading.Lock()
 24 |         self.delete_read_connections = False
 25 | 
 26 |         print "{} dst port {}".format(self.protocol, self.port)
 27 |         self.pcap.setfilter("{} dst port {}".format(self.protocol, self.port))
 28 |         #self.pcap.setfilter("{}".format(self.protocol))
 29 | 
 30 |     def run(self):
 31 |         #for i in range (0, 1000):
 32 |         while not self.done:
 33 |             (header, frame) = self.pcap.next()
 34 |             if not header:
 35 |                 break
 36 | 
 37 |             self.parse_packet(header, frame)
 38 |             #self.clean_no_payload()
 39 | 
 40 |         ready_indices = range(0, len(self.connection_list))
 41 |         self.move_ready_packets(ready_indices)
 42 | 
 43 |         #self.clean_no_payload()
 44 | 
 45 |         print "Num of connections : " + str(len(self.connection_list))
 46 |         self.done = True
 47 | 
 48 |     def clean_no_payload(self):
 49 |         try:
 50 |             self.lock.acquire()
 51 |             for i in range(len(self.connection_list) - 1, -1, -1):
 52 |                 if len(self.connection_list) == 0:
 53 |                     break
 54 |                 if self.connection_list[i].get_payload_length() == 0 and self.connection_list[i].ready:
 55 |                     del self.connection_list[i]
 56 |         finally:
 57 |             self.lock.release()
 58 | 
 59 |     def parse_packet(self, header, frame):
 60 |         decoder = ImpactDecoder.LinuxSLLDecoder()
 61 |         ether = decoder.decode(frame)
 62 | 
 63 |         ready_indices = []
 64 | 
 65 |         if ether.get_ether_type() == ImpactPacket.IP.ethertype:
 66 |             self.lock.acquire()
 67 |             for i in range(0, len(self.connection_list)):
 68 |                 buffered_packets = self.connection_list[i]
 69 |                 if buffered_packets.add_frame(ether): #if there's an existing flow
 70 |                     self.lock.release()
 71 |                     if len(ready_indices) > 0:
 72 |                         self.move_ready_packets(ready_indices)
 73 |                     return
 74 | 
 75 |                 if buffered_packets.ready:
 76 |                     ready_indices.append(i)
 77 | 
 78 |             buffered_packets = BufferedPackets(ether)
 79 |             self.connection_list.append(buffered_packets)
 80 |             self.lock.release()
 81 |             if len(ready_indices) > 0:
 82 |                 self.move_ready_packets(ready_indices)
 83 | 
 84 |     def move_ready_packets(self, ready_indices):
 85 |         self.lock.acquire()
 86 | 
 87 |         for i in range(len(ready_indices)-1, -1, -1):
 88 |             if self.connection_list[i].get_payload_length() > 0:
 89 |                 self.ready_connection_list.append(self.connection_list[i])
 90 |             del self.connection_list[i]
 91 | 
 92 |         self.lock.release()
 93 | 
 94 |     def has_ready_message(self):
 95 |         self.lock.acquire()
 96 | 
 97 |         if not self.delete_read_connections:
 98 |             if len(self.ready_connection_list)-1 == self.last_read_index:
 99 |                 self.lock.release()
100 |                 return False
101 |             else:
102 |                 self.lock.release()
103 |                 return True
104 |         else:
105 |             if len(self.ready_connection_list) > 0:
106 |                 self.lock.release()
107 |                 return True
108 |             else:
109 |                 self.lock.release()
110 |                 return False
111 | 
112 |     def has_unread_message(self):
113 |         self.lock.acquire()
114 |         if len(self.ready_connection_list) > 0:
115 |             for buffered_packets in self.connection_list:
116 |                 if not buffered_packets.read:
117 |                     self.lock.release()
118 |                     return True
119 | 
120 |         self.lock.release()
121 |         return False
122 | 
123 |     def pop_connection(self):
124 |         self.lock.acquire()
125 |         # for id in range(0, len(self.connection_list)):
126 |         #     if self.connection_list[id].ready and not self.connection_list[id].read:
127 |         #         bp = self.connection_list[id]
128 |         #         # del self.connection_list[id]
129 |         #         bp.read = True
130 |         #         self.lock.release()
131 |         #         return bp
132 | 
133 |         if not self.delete_read_connections:
134 |             if len(self.ready_connection_list)-1 == self.last_read_index:
135 |                 self.lock.release()
136 |                 return None
137 |             else:
138 |                 self.last_read_index += 1
139 |                 buffered_packets = self.ready_connection_list[self.last_read_index]
140 |                 buffered_packets.read = True
141 |                 self.lock.release()
142 |                 return buffered_packets
143 |         else:
144 |             if len(self.ready_connection_list) <= 0:
145 |                 self.lock.release()
146 |                 return None
147 |             else:
148 |                 buffered_packets = self.ready_connection_list[0]
149 |                 del self.ready_connection_list[0]
150 |                 self.lock.release()
151 |                 return buffered_packets
152 | 
153 |     def reset_read_status(self):
154 |         for buffered_packets in self.connection_list:
155 |             buffered_packets.read = False
156 | 
157 |     def forced_pop_connection(self):
158 |         bp = self.connection_list[0]
159 |         del self.connection_list[0]
160 |         return bp


--------------------------------------------------------------------------------
/PcapReaderThread.py:
--------------------------------------------------------------------------------
  1 | from BufferedPackets import BufferedPackets
  2 | from impacket import ImpactDecoder, ImpactPacket
  3 | 
  4 | import pcapy
  5 | import threading
  6 | import time
  7 | 
  8 | class PcapReaderThread(threading.Thread):
  9 |     def __init__(self, filename, protocol, port):
 10 |         threading.Thread.__init__(self)
 11 |         self.pcap = pcapy.open_offline(filename)
 12 |         self.connection_list = []
 13 |         self.ready_connection_list = []
 14 |         self.last_read_index = -1
 15 |         self.done = False
 16 |         self.port = port
 17 |         self.protocol = protocol
 18 |         self.lock = threading.Lock()
 19 |         self.delete_read_connections = False
 20 | 
 21 |         print "{} dst port {}".format(self.protocol, self.port)
 22 |         self.pcap.setfilter("{} dst port {}".format(self.protocol, self.port))
 23 | 
 24 |     def run(self):
 25 |         #for i in range (0, 1000):
 26 |         while not self.done:
 27 |             (header, frame) = self.pcap.next()
 28 |             if not header:
 29 |                 self.done = True
 30 |                 break
 31 | 
 32 |             self.parse_packet(header, frame)
 33 | 
 34 |         ready_indices = range(0, len(self.connection_list))
 35 |         self.move_ready_packets(ready_indices)
 36 | 
 37 |         #self.clean_no_payload()
 38 | 
 39 |         print "Num of connections : " + str(len(self.ready_connection_list))
 40 |         self.done = True
 41 | 
 42 |     def clean_no_payload(self):
 43 |         try:
 44 |             self.lock.acquire()
 45 |             for i in range(len(self.connection_list) - 1, -1, -1):
 46 |                 if len(self.connection_list) == 0:
 47 |                     break
 48 |                 if self.connection_list[i].get_payload_length() == 0 and self.connection_list[i].ready:
 49 |                     del self.connection_list[i]
 50 |         finally:
 51 |             self.lock.release()
 52 | 
 53 |     def parse_packet(self, header, frame):
 54 |         decoder = ImpactDecoder.EthDecoder()
 55 |         ether = decoder.decode(frame)
 56 | 
 57 |         ready_indices = []
 58 | 
 59 |         if ether.get_ether_type() == ImpactPacket.IP.ethertype:
 60 |             self.lock.acquire()
 61 |             for i in range(0, len(self.connection_list)):
 62 |                 buffered_packets = self.connection_list[i]
 63 |                 if buffered_packets.add_frame(ether): #if there's an existing flow
 64 |                     self.lock.release()
 65 |                     if len(ready_indices) > 0:
 66 |                         self.move_ready_packets(ready_indices)
 67 |                     return
 68 | 
 69 |                 if buffered_packets.ready:
 70 |                     ready_indices.append(i)
 71 | 
 72 |             buffered_packets = BufferedPackets(header, ether)
 73 |             self.connection_list.append(buffered_packets)
 74 |             self.lock.release()
 75 |             if len(ready_indices) > 0:
 76 |                 self.move_ready_packets(ready_indices)
 77 | 
 78 |     def move_ready_packets(self, ready_indices):
 79 |         self.lock.acquire()
 80 | 
 81 |         for i in range(len(ready_indices)-1, -1, -1):
 82 |             if self.connection_list[i].get_payload_length() > 0:
 83 |                 self.ready_connection_list.append(self.connection_list[i])
 84 |             del self.connection_list[i]
 85 | 
 86 |         self.lock.release()
 87 | 
 88 |     def has_ready_message(self):
 89 |         self.lock.acquire()
 90 |         #for buffered_packets in self.ready_connection_list:
 91 |         #    if not buffered_packets.read:
 92 |         #        self.lock.release()
 93 |         #        return True
 94 | 
 95 |         if not self.delete_read_connections:
 96 |             if len(self.ready_connection_list)-1 == self.last_read_index:
 97 |                 self.lock.release()
 98 |                 return False
 99 |             else:
100 |                 self.lock.release()
101 |                 return True
102 |         else:
103 |             if len(self.ready_connection_list) > 0:
104 |                 self.lock.release()
105 |                 return True
106 |             else:
107 |                 self.lock.release()
108 |                 return False
109 | 
110 |         #self.lock.release()
111 |         #return False
112 | 
113 |     def pop_connection(self):
114 |         self.lock.acquire()
115 |         # for id in range(0, len(self.connection_list)):
116 |         #     if self.connection_list[id].ready and not self.connection_list[id].read:
117 |         #         bp = self.connection_list[id]
118 |         #         # del self.connection_list[id]
119 |         #         bp.read = True
120 |         #         self.lock.release()
121 |         #         return bp
122 | 
123 |         if not self.delete_read_connections:
124 |             if len(self.ready_connection_list)-1 == self.last_read_index:
125 |                 self.lock.release()
126 |                 return None
127 |             else:
128 |                 self.last_read_index += 1
129 |                 buffered_packets = self.ready_connection_list[self.last_read_index]
130 |                 buffered_packets.read = True
131 |                 self.lock.release()
132 |                 return buffered_packets
133 |         else:
134 |             if len(self.ready_connection_list) <= 0:
135 |                 self.lock.release()
136 |                 return None
137 |             else:
138 |                 buffered_packets = self.ready_connection_list[0]
139 |                 del self.ready_connection_list[0]
140 |                 self.lock.release()
141 |                 return buffered_packets
142 | 
143 |         # self.lock.release()
144 |         # return None
145 | 
146 |     def reset_read_status(self):
147 |         self.lock.acquire()
148 |         # print "Resetting read status"
149 |         for buffered_packets in self.ready_connection_list:
150 |             buffered_packets.read = False
151 | 
152 |         self.last_read_index = -1
153 |         self.lock.release()
154 | 
155 |     def forced_pop_connection(self):
156 |         bp = self.connection_list[0]
157 |         del self.connection_list[0]
158 |         return bp


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | AEIDS (Unsupervised Approach for Detecting Low Rate Attacks on Network Traffic with Autoencoder)
 2 | ================================================================================================
 3 | 
 4 | AEIDS is a prototype of anomaly-based intrusion detection system which works by remembering the pattern of legitimate network traffic using Autoencoder. The full paper of this approach (Unsupervised Approach for Detecting Low Rate Attacks on Network Traffic with Autoencoder) is available [here](https://ieeexplore.ieee.org/document/8560678)
 5 | 
 6 | Dependencies:
 7 | * Python 2.7
 8 | * Pcapy
 9 | * Keras
10 | * psycopg2 (for database access)
11 | * PostgreSQL 9.5
12 | 
13 | Installation:
14 | 1. Clone this repository and install all necessary libraries and programs
15 | 2. Create a database in PostgreSQL and import the schema in aeids.sql`
16 | 3. Modify `aeids.conf`, put the location of your PCAP file in the`root\_directory variable. Put the name of the PCAP file in the `training\_filename` along with the number of TCP connections to the server using this format `filename:num_connections`. See the examples config provided. Use wireshark or the `counting` phase in AEIDS to get the number of TCP connections.
17 | 4. Modify the database connection configuration in aeids.py. Find the `open_conn()` function.
18 | 


--------------------------------------------------------------------------------
/StreamReaderThread.py:
--------------------------------------------------------------------------------
  1 | # There's bug when an ACK packet is read after FIN has received. The new packet is considered to be part of a new stream not the existing one.
  2 | 
  3 | from TcpStream import *
  4 | from impacket import ImpactDecoder, ImpactPacket
  5 | 
  6 | import pcapy
  7 | import threading
  8 | 
  9 | 
 10 | TIMEOUT = 2
 11 | end_states = (STATE_CLOSE, STATE_RESET, STATE_TIMEOUT)
 12 | 
 13 | 
 14 | class StreamReaderThread(threading.Thread):
 15 |     def __init__(self, filename, protocol, port):
 16 |         threading.Thread.__init__(self)
 17 |         self.pcap = pcapy.open_offline(filename)
 18 |         self.tcp_buffer = {}
 19 |         self.ready_tcp_buffer = []
 20 |         self.last_read_index = -1
 21 |         self.done = False
 22 |         self.port = port
 23 |         self.protocol = protocol
 24 |         self.lock = threading.Lock()
 25 |         self.condition_lock = threading.Condition()
 26 |         self.delete_read_connections = False
 27 |         self.last_timestamp = -1
 28 |         self.packet_counter = 0
 29 | 
 30 |         print("{} dst port {}".format(self.protocol, self.port))
 31 |         self.pcap.setfilter("{} port {}".format(self.protocol, self.port))
 32 | 
 33 |     def run(self):
 34 |         while not self.done:
 35 |             (header, frame) = self.pcap.next()
 36 |             if not header:
 37 |                 break
 38 | 
 39 |             self.parse_packet(header, frame)
 40 | 
 41 |         self.empty_buffer()
 42 |         print("waiting for all threads to finish")
 43 |         while len(self.tcp_buffer) > 0:
 44 |             time.sleep(0.0001)
 45 |         print("main loop finished")
 46 |         self.done = True
 47 | 
 48 |     def parse_packet(self, header, frame):
 49 |         datalink = self.pcap.datalink()
 50 |         if datalink == pcapy.DLT_EN10MB:
 51 |             decoder = ImpactDecoder.EthDecoder()
 52 |         elif datalink == pcapy.DLT_LINUX_SLL:
 53 |             decoder = ImpactDecoder.LinuxSLLDecoder()
 54 |         else:
 55 |             raise Exception("Datalink not supported")
 56 |         ether = decoder.decode(frame)
 57 |         ts = float(str(header.getts()[0]) + "." + str(header.getts()[1]))
 58 |         self.last_timestamp = ts
 59 | 
 60 |         if ether.get_ether_type() == ImpactPacket.IP.ethertype:
 61 |             (id, tcp_tuple) = generate_id(ether)
 62 |             if id == False:
 63 |                 return
 64 | 
 65 |             (rev_id, tcp_tuple) = generate_reverse_id(ether)
 66 | 
 67 |             # print("Buffer", self.tcp_buffer)
 68 |             # print(threading.current_thread().name + "in",)
 69 |             self.acquire_lock("parse")
 70 |             # print(threading.current_thread().name + "out")
 71 |             if id in self.tcp_buffer:
 72 |                 tcp_stream = self.tcp_buffer[id]
 73 |                 to_server = True
 74 |                 # print("[fwd] ID: " + id + ";" + str(ts))
 75 |             elif rev_id in self.tcp_buffer:
 76 |                 tcp_stream = self.tcp_buffer[rev_id]
 77 |                 to_server = False
 78 |                 # print("[rev] ID: " + id + ";" + str(ts))
 79 |             else:
 80 |                 # a new stream has appeared
 81 |                 tcp_stream = TcpStream(id, ts, self)
 82 |                 self.tcp_buffer[id] = tcp_stream
 83 |                 to_server = True
 84 |                 packet = ether.child()
 85 |                 segment = packet.child()
 86 |                 tcp_stream.start()
 87 |                 # print("[new] ID: " + id + ";" + str(ts))
 88 | 
 89 |             tcp_stream.add_packet(ts, to_server, ether)
 90 |             # if tcp_stream.state in end_states:
 91 |             #     tcp_stream.finish()
 92 |             #     self.move_stream(tcp_stream.id)
 93 | 
 94 |             self.packet_counter += 1
 95 |             self.release_lock("parse")
 96 |             # print(threading.current_thread().name + "out2")
 97 | 
 98 |     def move_stream(self, id):
 99 |         # print("[del] ID: " + id + ";" + str(self.tcp_buffer[id].client_data_len) + ";" + str(self.tcp_buffer[id].server_data_len))
100 |         self.acquire_lock("move")
101 |         self.tcp_buffer[id].finish()
102 |         if self.tcp_buffer[id].client_data_len > 0 or self.tcp_buffer[id].server_data_len > 0:
103 |             self.ready_tcp_buffer.append(self.tcp_buffer[id])
104 |         # else:
105 |         #     print("------------------------------")
106 |         #     print(self.tcp_buffer[id].get_hexlify_payload("server"))
107 |         #     print(self.tcp_buffer[id].get_hexlify_payload("client"))
108 |         #     print("------------------------------")
109 |         del(self.tcp_buffer[id])
110 |         self.release_lock("move")
111 |         self.condition_lock.acquire()
112 |         # print("notify")
113 |         self.condition_lock.notify()
114 |         # print("done notifying")
115 |         self.condition_lock.release()
116 | 
117 |     def has_ready_message(self):
118 |         # self.acquire_lock()
119 | 
120 |         if not self.delete_read_connections:
121 |             if len(self.ready_tcp_buffer)-1 == self.last_read_index:
122 |                 # self.release_lock()
123 |                 return False
124 |             else:
125 |                 # self.release_lock()
126 |                 return True
127 |         else:
128 |             if len(self.ready_tcp_buffer) > 0:
129 |                 # self.release_lock()
130 |                 return True
131 |             else:
132 |                 # self.release_lock()
133 |                 return False
134 | 
135 |     def pop_connection(self):
136 |         # print(threading.current_thread().name + "pop in")
137 |         self.acquire_lock("pop")
138 | 
139 |         if not self.delete_read_connections:
140 |             if len(self.ready_tcp_buffer)-1 == self.last_read_index:
141 |                 self.release_lock("pop1")
142 |                 # print(threading.current_thread().name + "pop out1")
143 |                 return None
144 |             else:
145 |                 self.last_read_index += 1
146 |                 tcp_stream = self.ready_tcp_buffer[self.last_read_index]
147 |                 tcp_stream.read = True
148 |                 self.release_lock("pop2")
149 |                 # print(threading.current_thread().name + "pop out2")
150 |                 return tcp_stream
151 |         else:
152 |             if len(self.ready_tcp_buffer) <= 0:
153 |                 self.release_lock("pop3")
154 |                 # print(threading.current_thread().name + "pop out3")
155 |                 return None
156 |             else:
157 |                 tcp_stream = self.ready_tcp_buffer[0]
158 |                 del self.ready_tcp_buffer[0]
159 |                 self.release_lock("pop4")
160 |                 # print(threading.current_thread().name + "pop out4")
161 |                 return tcp_stream
162 | 
163 |     def wait_for_data(self):
164 |         self.condition_lock.acquire()
165 |         # print("wait")
166 |         self.condition_lock.wait(0.1)
167 |         # print("notified")
168 |         self.condition_lock.release()
169 | 
170 |     def reset_read_status(self):
171 |         self.acquire_lock("reset")
172 |         # print "Resetting read status"
173 |         for tcp_stream in self.ready_tcp_buffer:
174 |             tcp_stream.read = False
175 | 
176 |         self.last_read_index = -1
177 |         self.release_lock("reset")
178 | 
179 |     def forced_pop_connection(self):
180 |         bp = self.ready_tcp_buffer[0]
181 |         del self.ready_tcp_buffer[0]
182 |         return bp
183 | 
184 |     def is_timeout(self, stream_last_ts):
185 |         # self.acquire_lock()
186 |         if self.last_timestamp - stream_last_ts  > TIMEOUT:
187 |             # self.release_lock()
188 |             return True
189 |         else:
190 |             # self.release_lock()
191 |             return False
192 | 
193 |     def acquire_lock(self, caller):
194 |         # print(caller + "-try")
195 |         self.lock.acquire()
196 |         # print(caller + "-got")
197 | 
198 |     def release_lock(self, caller):
199 |         # print(caller + "-out")
200 |         self.lock.release()
201 |         # print(caller + "-bye")
202 | 
203 |     def empty_buffer(self):
204 |         ready_indices = []
205 | 
206 |         self.acquire_lock("empty")
207 |         for id, stream in self.tcp_buffer.iteritems():
208 |             stream.state = STATE_TIMEOUT
209 |             # stream.finish()
210 |             # ready_indices.append(id)
211 |         self.release_lock("empty")
212 | 
213 |     def cleanup_all_buffers(self):
214 |         self.acquire_lock("cleanup")
215 |         del self.ready_tcp_buffer
216 |         del self.tcp_buffer
217 |         self.release_lock("cleanup")
218 | 


--------------------------------------------------------------------------------
/TcpMessage.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | import binascii
 3 | 
 4 | class TcpMessage():
 5 |     def __init__(self, client_data, server_data, tcp_tuple, start_ts, stop_ts):
 6 |         self.client_data = client_data
 7 |         self.server_data = server_data
 8 |         self.tcp_tuple = tcp_tuple
 9 |         self.read = False
10 |         self.client_data_len = len(client_data)
11 |         self.server_data_len = len(server_data)
12 |         self.client_bf = self.__calculate_byte_frequency(self.client_data, self.client_data_len)
13 |         self.server_bf = self.__calculate_byte_frequency(self.server_data, self.server_data_len)
14 |         self.id = "{}-{}-{}-{}-tcp".format(self.tcp_tuple[0], self.tcp_tuple[1], self.tcp_tuple[2], self.tcp_tuple[3])
15 |         self.start_ts = start_ts
16 |         self.stop_ts = stop_ts
17 | 
18 |     def get_payload(self, dest):
19 |         if dest == "client":
20 |             return self.client_data
21 |         elif dest == "server":
22 |            return self.server_data
23 | 
24 |     def get_hexlify_payload(self, dest):
25 |         payload = self.get_payload(dest)
26 |         return binascii.hexlify(payload)
27 | 
28 |     def __calculate_byte_frequency(self, payload, length):
29 |         byte_frequency = [0] * 256
30 | 
31 |         if length > 0:
32 |             for i in range(0, 256):
33 |                 byte_frequency[i] = float(payload.count(chr(i))) / length
34 | 
35 |         return byte_frequency
36 | 
37 |     def get_byte_frequency(self, dest):
38 |         if dest == "client":
39 |             return self.client_bf
40 |         elif dest == "server":
41 |            return self.server_bf
42 | 
43 |     def get_payload_length(self, dest):
44 |         if dest == "client":
45 |             return self.client_data_len
46 |         elif dest == "server":
47 |            return self.server_data_len
48 | 
49 |     def get_start_time(self):
50 |         return self.start_ts
51 | 
52 |     def get_stop_time(self):
53 |         return self.stop_ts


--------------------------------------------------------------------------------
/TcpStream.py:
--------------------------------------------------------------------------------
  1 | from impacket import ImpactPacket, ImpactDecoder
  2 | 
  3 | import binascii
  4 | import threading
  5 | import time
  6 | from pprint import pprint
  7 | from inspect import getmembers
  8 | 
  9 | STATE_JUST_EST = 0
 10 | STATE_DATA = 1
 11 | STATE_WAIT_FIN1 = 2
 12 | STATE_WAIT_FIN2 = 3
 13 | STATE_CLOSE = 4
 14 | STATE_TIMEOUT = 5
 15 | STATE_RESET = 6
 16 | 
 17 | end_states = (STATE_CLOSE, STATE_RESET, STATE_TIMEOUT)
 18 | 
 19 | def generate_id(ether):
 20 |     packet = ether.child()
 21 |     segment = packet.child()
 22 | 
 23 |     src_addr = packet.get_ip_src()
 24 |     dst_addr = packet.get_ip_dst()
 25 | 
 26 |     if isinstance(segment, ImpactPacket.TCP):
 27 |         protocol = "tcp"
 28 |         src_port = segment.get_th_sport()
 29 |         dst_port = segment.get_th_dport()
 30 |         return ("{}-{}-{}-{}-{}".format(src_addr, src_port, dst_addr, dst_port, protocol), (src_addr, src_port, dst_addr, dst_port))
 31 |     else:
 32 |         return False
 33 | 
 34 | 
 35 | def generate_reverse_id(ether):
 36 |     packet = ether.child()
 37 |     segment = packet.child()
 38 | 
 39 |     src_addr = packet.get_ip_src()
 40 |     dst_addr = packet.get_ip_dst()
 41 | 
 42 |     if isinstance(segment, ImpactPacket.TCP):
 43 |         protocol = "tcp"
 44 |         src_port = segment.get_th_sport()
 45 |         dst_port = segment.get_th_dport()
 46 |         return ("{}-{}-{}-{}-{}".format(dst_addr, dst_port, src_addr, src_port, protocol), (src_addr, src_port, dst_addr, dst_port))
 47 |     else:
 48 |         return False
 49 | 
 50 | 
 51 | def __calculate_byte_frequency__(payload, length):
 52 |     byte_frequency = [0] * 256
 53 | 
 54 |     if length > 0:
 55 |         for i in range(0, 256):
 56 |             byte_frequency[i] = float(payload.count(chr(i))) / length
 57 | 
 58 |     return byte_frequency
 59 | 
 60 | 
 61 | class TcpStream(threading.Thread):
 62 |     def __init__(self, id, start_time, reader_thread):
 63 |         threading.Thread.__init__(self)
 64 |         info = id.split("-")
 65 |         self.tcp_tuple = (info[0], info[1], info[2], info[3])
 66 |         self.reader_thread = reader_thread
 67 |         self.id = id
 68 |         self.start_time = start_time
 69 |         self.stop_time = -1
 70 |         self.client_buffer = []
 71 |         self.server_buffer = []
 72 |         self.last_packet_time = start_time
 73 |         self.state = STATE_JUST_EST
 74 |         self.client_data = ""
 75 |         self.server_data = ""
 76 |         self.client_last_seq = -1
 77 |         self.server_last_seq = -1
 78 |         self.ready = False
 79 |         self.client_data_len = -1
 80 |         self.server_data_len = -1
 81 |         self.client_bf = None
 82 |         self.server_bf = None
 83 |         self.read = False
 84 | 
 85 |     def run(self):
 86 |         while self.state not in end_states:
 87 |             if self.reader_thread.is_timeout(self.last_packet_time) and self.state not in end_states:
 88 |                 self.state = STATE_TIMEOUT
 89 |             else:
 90 |                 time.sleep(0.0001)
 91 |                 # continue
 92 | 
 93 |         # self.finish()
 94 |         # print(threading.current_thread().name + "move-in")
 95 |         self.reader_thread.move_stream(self.id)
 96 |         # print(threading.current_thread().name + "move-out")
 97 | 
 98 | 
 99 |     # TODO: consider IP fragmentation
100 |     def add_packet(self, ts, to_server, ether):
101 |         packet = ether.child()
102 |         segment = packet.child()
103 | 
104 |         # pprint(getmembers(segment))
105 |         # identify TCP flags
106 |         if segment.get_SYN() and to_server:
107 |             # print("syn")
108 |             self.server_last_seq = segment.get_th_seq()
109 |             # print("syn: ", self.server_last_seq)
110 |             return
111 |         elif segment.get_SYN() and segment.get_ACK() and not to_server:
112 |             # print("syn-ack")
113 |             self.client_last_seq = segment.get_th_seq()
114 |             self.state = STATE_DATA
115 |             return
116 |         elif segment.get_FIN() and self.state < STATE_WAIT_FIN1:
117 |             # print("fin", segment.get_ACK(), to_server, self.id)
118 |             if to_server:
119 |                 self.server_last_seq = segment.get_th_seq()
120 |             else:
121 |                 self.client_last_seq = segment.get_th_seq()
122 |             self.state = STATE_WAIT_FIN1
123 |             self.last_packet_time = ts
124 |             self.stop_time = ts
125 |             return
126 |         elif segment.get_FIN() and self.state == STATE_WAIT_FIN1:
127 |             if to_server:
128 |                 self.server_last_seq = segment.get_th_seq()
129 |             else:
130 |                 self.client_last_seq = segment.get_th_seq()
131 |             self.state = STATE_WAIT_FIN2
132 |             self.last_packet_time = ts
133 |             self.stop_time = ts
134 |             # print("wait fin")
135 |             return
136 |         elif segment.get_ACK() and self.state == STATE_WAIT_FIN2 and to_server and segment.get_th_seq() > self.server_last_seq:
137 |             # print("close1")
138 |             self.last_packet_time = ts
139 |             self.stop_time = ts
140 |             self.state = STATE_CLOSE
141 |             return
142 |         elif segment.get_ACK() and self.state == STATE_WAIT_FIN2 and not to_server and segment.get_th_seq() > self.client_last_seq:
143 |             # print("close2")
144 |             self.last_packet_time = ts
145 |             self.stop_time = ts
146 |             self.state = STATE_CLOSE
147 |             return
148 |         else: # data
149 |             if self.state not in end_states:
150 |                 # print("data")
151 |                 if len(segment.get_data_as_string()) > 0:
152 |                     # print(5)
153 |                     self.last_packet_time = ts
154 |                     self.stop_time = ts
155 |                     if to_server:
156 |                         if self.server_last_seq < segment.get_th_seq():
157 |                             self.server_last_seq = segment.get_th_seq()
158 |                             self.server_buffer.append((segment.get_th_seq(), segment.get_th_ack(), segment.get_data_as_string()))
159 |                             # print(segment.get_data_as_string())
160 |                             return
161 |                         else:
162 |                             # print(8)
163 |                             for i in range(0, len(self.server_buffer)):  # check for retransmission
164 |                                 segment_tuple = self.server_buffer[i]
165 |                                 if segment_tuple[0] == segment.get_th_seq() and segment_tuple[1] == segment.get_th_ack() and len(segment_tuple[2]) == len(segment.get_data_as_string()): # a retransmitted packet
166 |                                     # print("retransmitted")
167 |                                     # print(segment_tuple, segment.get_th_seq(), segment.get_th_ack(), len(segment.get_data_as_string()))
168 |                                     return
169 | 
170 |                             for i in range(0, len(self.server_buffer)):  # check for out of order
171 |                                 segment_tuple = self.server_buffer[i]
172 |                                 if segment_tuple[0] < segment.get_th_seq():  # an out of order packet
173 |                                     self.server_buffer.insert(i, (segment.get_th_seq(), segment.get_th_ack(), segment.get_data_as_string()))
174 |                                     # print(2)
175 |                                     return
176 | 
177 |                     else:
178 |                         # print(7)
179 |                         if self.client_last_seq < segment.get_th_seq():
180 |                             self.client_last_seq = segment.get_th_seq()
181 |                             self.client_buffer.append((segment.get_th_seq(), segment.get_th_ack(), segment.get_data_as_string()))
182 |                             # print(segment.get_data_as_string())
183 |                             # print(3)
184 |                             return
185 |                         else:
186 |                             # print(9)
187 |                             for i in range(0, len(self.client_buffer)):  # check for retransmission
188 |                                 segment_tuple = self.client_buffer[i]
189 |                                 if segment_tuple[0] == segment.get_th_seq() and segment_tuple[1] == segment.get_th_ack() and len(segment_tuple[2]) == len(segment.get_data_as_string()): # a retransmitted packet
190 |                                     # print("retransmitted")
191 |                                     # print(segment_tuple, segment.get_th_seq(), segment.get_th_ack(), len(segment.get_data_as_string()))
192 |                                     return
193 | 
194 |                             for i in range(0, len(self.client_buffer)):  # check for out of order
195 |                                 segment_tuple = self.client_buffer[i]
196 |                                 if segment_tuple[0] < segment.get_th_seq(): # an out of order packet
197 |                                     self.client_buffer.insert(i, (segment.get_th_seq(), segment.get_th_ack(), segment.get_data_as_string()))
198 |                                     # print(4)
199 |                                     return
200 | 
201 |     def finish(self):
202 |         for segment_tuple in self.server_buffer:
203 |             self.server_data += segment_tuple[2]
204 | 
205 |         del self.server_buffer
206 | 
207 |         for segment_tuple in self.client_buffer:
208 |             self.client_data += segment_tuple[2]
209 | 
210 |         del self.client_buffer
211 |         self.ready = True
212 | 
213 |         self.client_data_len = len(self.client_data)
214 |         self.server_data_len = len(self.server_data)
215 |         self.client_bf = __calculate_byte_frequency__(self.client_data, self.client_data_len)
216 |         self.server_bf = __calculate_byte_frequency__(self.server_data, self.server_data_len)
217 | 
218 |     def get_payload(self, dest):
219 |         if dest == "client":
220 |             return self.client_data
221 |         elif dest == "server":
222 |             return self.server_data
223 | 
224 |     def get_hexlify_payload(self, dest):
225 |         payload = self.get_payload(dest)
226 |         return binascii.hexlify(payload)
227 | 
228 |     def get_byte_frequency(self, dest):
229 |         if dest == "client":
230 |             return self.client_bf
231 |         elif dest == "server":
232 |             return self.server_bf
233 | 
234 |     def get_payload_length(self, dest):
235 |         if dest == "client":
236 |             return self.client_data_len
237 |         elif dest == "server":
238 |             return self.server_data_len
239 | 
240 |     def get_start_time(self):
241 |         return self.start_time
242 | 
243 |     def get_stop_time(self):
244 |         return self.stop_time
245 | 


--------------------------------------------------------------------------------
/aeids.conf:
--------------------------------------------------------------------------------
 1 | root_directory=/media/baskoro/HD-LXU3/Datasets/UNSW/UNSW-NB15-Source-Files/UNSW-NB15-pcap-files/pcaps-22-1-2015/attack/
 2 | root_directory=/media/baskoro/HD-LXU3/Datasets/UNSW/UNSW-NB15-Source-Files/UNSW-NB15-pcap-files/pcaps-22-1-2015/normal/
 3 | root_directory=/media/baskoro/HD-LXU3/Datasets/UNSW/UNSW-NB15-Source-Files/UNSW-NB15-pcap-files/pcaps-17-2-2015/attack/
 4 | root_directory=/media/baskoro/HD-LXU3/Datasets/UNSW/UNSW-NB15-Source-Files/UNSW-NB15-pcap-files/pcaps-17-2-2015/normal/
 5 | root_directory=/home/baskoro/Documents/Dataset/ISCX12/without retransmission/
 6 | root_directory=/home/baskoro/Documents/Dataset/HTTP-Attack-Dataset/morphed-shellcode-attacks/
 7 | root_directory=/home/baskoro/Documents/Dataset/HTTP-Attack-Dataset/shellcode-attacks/
 8 | root_directory=/home/baskoro/exwindows/Datasets/Metasploit/FilteredPcap/Pcap/
 9 | tensorboard_log_enabled=False
10 | backend=tensorflow
11 | threshold_method=median
12 | #training_filename=testbed-14jun.pcap:80:136519
13 | training_filename=testbed-14jun.pcap:80:87990
14 | training_filename=testbed-14jun.pcap:22:3263
15 | training_filename=testbed-11jun.pcap:22:10006
16 | training_filename=training-port-80.pcap:80:93406
17 | #training_filename=training-port-80.pcap:80:200
18 | training_filename=training-port-25.pcap:80:41395
19 | 


--------------------------------------------------------------------------------
/aeids.py:
--------------------------------------------------------------------------------
  1 | from BufferedPackets import WINDOW_SIZE
  2 | from keras.callbacks import TensorBoard
  3 | from keras.models import load_model
  4 | from keras.models import Model
  5 | from keras.layers import Dense, Input, Dropout
  6 | from keras.models import model_from_json
  7 | # from LibNidsReaderThread import LibNidsReaderThread
  8 | # from PcapReaderThread import PcapReaderThread
  9 | from StreamReaderThread import StreamReaderThread
 10 | from tensorflow import Tensor
 11 | 
 12 | import binascii
 13 | import math
 14 | import numpy
 15 | import os
 16 | import psycopg2
 17 | import psycopg2.extras
 18 | import sys
 19 | import thread
 20 | import time
 21 | import traceback
 22 | 
 23 | import csv
 24 | 
 25 | # def main(argv):
 26 | #     try:
 27 | #         root_directory = "/home/baskoro/Documents/Dataset/ISCX12/without retransmission/"
 28 | #         filename = root_directory + sys.argv[1]
 29 | #         prt = PcapReaderThread(filename)
 30 | #         prt.run()
 31 | #
 32 | #         while not prt.done:
 33 | #             print "sleeping"
 34 | #             time.sleep(1)
 35 | #
 36 | #         while prt.has_ready_message():
 37 | #             bp = prt.pop_connection()
 38 | #             print bp.get_payload()
 39 | #
 40 | #         print "DIE YOU!!!"
 41 | #
 42 | #     except IndexError:
 43 | #         print "Usage : python aeids.py filename [training|testing]"
 44 | #     except KeyboardInterrupt:
 45 | #         print "Good bye to you my trusted friend"
 46 | # root_directory = "/home/baskoro/Documents/Dataset/ISCX12/without retransmission/"
 47 | # root_directory = "/home/baskoro/Documents/Dataset/HTTP-Attack-Dataset/morphed-shellcode-attacks/"
 48 | # root_directory = "/home/baskoro/Documents/Dataset/HTTP-Attack-Dataset/shellcode-attacks/"
 49 | tensorboard_log_enabled = False
 50 | backend = "tensorflow"
 51 | done = False
 52 | prt = None
 53 | conf = {}
 54 | activation_functions = ["elu", "selu", "softplus", "softsign", "relu", "tanh", "sigmoid", "hard_sigmoid", "linear", "softmax"]
 55 | conn = None
 56 | 
 57 | # possible values: mean, median, zscore
 58 | threshold = "median"
 59 | 
 60 | 
 61 | def main(argv):
 62 |     try:
 63 |         # validate command line arguments
 64 |         if argv[1] != "training" and argv[1] != "predicting" and argv[1] != "testing" and argv[1] != "counting":
 65 |             raise IndexError("Phase {} does not exist.".format(argv[1]))
 66 |         else:
 67 |             phase = argv[1]
 68 | 
 69 |         if argv[2] != "tcp" and argv[2] != "udp":
 70 |             raise IndexError("Protocol {} is not supported.".format(argv[3]))
 71 |         else:
 72 |             protocol = argv[2]
 73 | 
 74 |         if not argv[3].isdigit():
 75 |             raise IndexError("Port must be numeric.")
 76 |         else:
 77 |             port = argv[3]
 78 | 
 79 |         # must be in form of comma separated, representing half of the layers (e.g. 200,100 means there are 3 layers,
 80 |         # with 200, 100, and 200 neurons respectively)
 81 |         if phase != "counting":
 82 |             try:
 83 |                 hidden_layers = argv[4].split(",")
 84 |                 for neurons in hidden_layers:
 85 |                     if not neurons.isdigit():
 86 |                         raise IndexError("Hidden layers must be comma separated numeric values")
 87 |             except ValueError:
 88 |                 raise IndexError("Hidden layers must be comma separated numeric values")
 89 | 
 90 |             if argv[5] not in activation_functions:
 91 |                 raise IndexError("Activation function must be one of the following list")
 92 |             else:
 93 |                 activation_function = argv[5]
 94 | 
 95 |             try:
 96 |                 dropout = float(argv[6])
 97 |             except ValueError:
 98 |                 raise IndexError("Dropout must be numeric.")
 99 | 
100 |             if phase == "training" and not argv[8].isdigit():
101 |                 raise IndexError("Batch size must be numeric.")
102 |             elif phase == "training" or phase == "predicting":
103 |                 batch_size = int(argv[8])
104 | 
105 |             filename = argv[7]
106 |             if phase == "testing":
107 |                 aeids(phase, filename, protocol, port, hidden_layers, activation_function, dropout, argv[8])
108 |             else:
109 |                 aeids(phase, filename, protocol, port, hidden_layers, activation_function, dropout, batch_size=batch_size)
110 |         else:
111 |             count_byte_freq(argv[4], protocol, port)
112 | 
113 |     except IndexError as e:
114 |         print("Usage: python aeids.py <training|predicting|testing|counting> <tcp|udp> <port> <hidden_layers> <activation_function> <dropout> <training filename> [batch_size] [testing filename]")
115 |         print traceback.print_exc()
116 |         exit(0)
117 |     except KeyboardInterrupt:
118 |         print "Interrupted"
119 |         if prt is not None:
120 |             prt.done = True
121 |     except BaseException as e:
122 |         print traceback.print_exc()
123 |         if prt is not None:
124 |             prt.done = True
125 | 
126 | 
127 | def aeids(phase = "training", filename = "", protocol="tcp", port="80", hidden_layers = [200,100], activation_function = "relu", dropout = 0.0, testing_filename = "", batch_size = 1):
128 |     global done
129 |     global prt
130 |     read_conf()
131 | 
132 |     if phase == "training":
133 |         numpy.random.seed(666)
134 | 
135 |         autoencoder = init_model(hidden_layers, activation_function, dropout)
136 | 
137 |         if "{}-{}".format(filename, port) in conf["training_filename"]:
138 |             steps_per_epoch = conf["training_filename"]["{}-{}".format(filename, port)] / batch_size
139 |         else:
140 |             steps_per_epoch = conf["training_filename"]["default-80"] / batch_size
141 | 
142 |         if tensorboard_log_enabled and backend == "tensorflow":
143 |             tensorboard_callback = TensorBoard(log_dir="./logs", batch_size=10000, write_graph=True, write_grads=True,
144 |                                                histogram_freq=1)
145 |             autoencoder.fit_generator(byte_freq_generator(filename, protocol, port, batch_size), steps_per_epoch=100,
146 |                                       epochs=100, verbose=1, callbacks=[tensorboard_callback])
147 |             check_directory(filename, "models")
148 |             autoencoder.save("models/{}/aeids-with-log-{}-hl{}-af{}-do{}.hdf5".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), overwrite=True)
149 |         else:
150 |             autoencoder.fit_generator(byte_freq_generator(filename, protocol, port, batch_size), steps_per_epoch=steps_per_epoch,
151 |                                       epochs=10, verbose=1)
152 |             check_directory(filename, "models")
153 |             autoencoder.save("models/{}/aeids-{}-hl{}-af{}-do{}.hdf5".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), overwrite=True)
154 | 
155 |         print "Training autoencoder finished. Calculating threshold..."
156 |         predict_byte_freq_generator(autoencoder, filename, protocol, port, hidden_layers, activation_function, dropout, phase)
157 |         done = True
158 |         prt.cleanup_all_buffers()
159 |         prt = None
160 |         print "\nFinished."
161 |     elif phase == "predicting":
162 |         autoencoder = load_autoencoder(filename, protocol, port, hidden_layers, activation_function, dropout)
163 |         predict_byte_freq_generator(autoencoder, filename, protocol, port, hidden_layers, activation_function, dropout, phase)
164 |         done = True
165 |         print "\nFinished."
166 |     elif phase == "testing":
167 |         autoencoder = load_autoencoder(filename, protocol, port, hidden_layers, activation_function, dropout)
168 |         predict_byte_freq_generator(autoencoder, filename, protocol, port, hidden_layers, activation_function, dropout, phase, testing_filename)
169 |         prt = None
170 |         print "\nFinished."
171 |     else:
172 |         raise IndexError
173 | 
174 | 
175 | def read_conf():
176 |     global conf
177 | 
178 |     fconf = open("aeids.conf", "r")
179 |     if not fconf:
180 |         print "File aeids.conf does not exist."
181 |         exit(-1)
182 | 
183 |     conf["root_directory"] = []
184 |     conf["training_filename"] = {"default-80": 100000}
185 |     lines = fconf.readlines()
186 |     for line in lines:
187 |         if line.startswith("#"):
188 |             continue
189 |         split = line.split("=", 2)
190 |         print split
191 |         if split[0] == "root_directory":
192 |             conf["root_directory"].append(split[1].strip())
193 |         elif split[0] == "training_filename":
194 |             tmp = split[1].split(":")
195 |             conf["training_filename"]["{}-{}".format(tmp[0], tmp[1])] = int(tmp[2])
196 | 
197 |     fconf.close()
198 | 
199 | 
200 | def init_model(hidden_layers = [200, 100], activation_function ="relu", dropout = 0):
201 |     input_dimension = 256
202 |     input = Input(shape=(input_dimension,))
203 | 
204 |     for i in range(0, len(hidden_layers)):
205 |         if i == 0:
206 |             encoded = Dense(int(hidden_layers[i]), activation=activation_function)(input)
207 |         else:
208 |             encoded = Dense(int(hidden_layers[i]), activation=activation_function)(encoded)
209 | 
210 |         encoded = Dropout(dropout)(encoded)
211 | 
212 |     for i in range(len(hidden_layers) - 1, -1, -1):
213 |         if i == len(hidden_layers) - 1:
214 |             decoded = Dense(int(hidden_layers[i]), activation=activation_function)(encoded)
215 |         else:
216 |             decoded = Dense(int(hidden_layers[i]), activation=activation_function)(decoded)
217 | 
218 |         decoded = Dropout(0.2)(decoded)
219 | 
220 |     if len(hidden_layers) == 1:
221 |         decoded = Dense(input_dimension, activation="sigmoid")(encoded)
222 |     else:
223 |         decoded = Dense(input_dimension, activation="sigmoid")(decoded)
224 |     autoencoder = Model(outputs=decoded, inputs=input)
225 |     autoencoder.compile(loss="binary_crossentropy", optimizer="adadelta")
226 | 
227 |     return autoencoder
228 | 
229 | 
230 | def load_autoencoder(filename, protocol, port, hidden_layers, activation_function, dropout):
231 |     autoencoder = load_model("models/{}/aeids-{}-hl{}-af{}-do{}.hdf5".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout))
232 |     return autoencoder
233 | 
234 | 
235 | def byte_freq_generator(filename, protocol, port, batch_size):
236 |     global prt
237 |     global conf
238 |     global done
239 |     prt = StreamReaderThread(get_pcap_file_fullpath(filename), protocol, port)
240 |     prt.start()
241 |     counter = 0
242 |     done = False
243 | 
244 |     while not done:
245 |         while not prt.done or prt.has_ready_message():
246 |             if not prt.has_ready_message():
247 |                 prt.wait_for_data()
248 |                 continue
249 |             else:
250 |                 buffered_packets = prt.pop_connection()
251 |                 if buffered_packets is None:
252 |                     time.sleep(0.0001)
253 |                     continue
254 |                 if buffered_packets.get_payload_length("server") > 0:
255 |                     byte_frequency = buffered_packets.get_byte_frequency("server")
256 |                     X = numpy.reshape(byte_frequency, (1, 256))
257 | 
258 |                     if counter == 0 or counter % batch_size == 1:
259 |                         dataX = X
260 |                     else:
261 |                         dataX = numpy.r_["0,2", dataX, X]
262 | 
263 |                     counter += 1
264 | 
265 |                     if counter % batch_size == 0:
266 |                         yield dataX, dataX
267 | 
268 |         if dataX.shape[0] > 0:
269 |             yield dataX, dataX
270 | 
271 |         prt.reset_read_status()
272 | 
273 | 
274 | def predict_byte_freq_generator(autoencoder, filename, protocol, port, hidden_layers, activation_function, dropout, phase="training", testing_filename = ""):
275 |     global prt
276 |     global threshold
277 |     print("test_filename", testing_filename)
278 |     if prt is None:
279 |         if phase == "testing":
280 |             prt = StreamReaderThread(get_pcap_file_fullpath(testing_filename), protocol, port)
281 |             print("testing filename: " + testing_filename)
282 |         else:
283 |             prt = StreamReaderThread(get_pcap_file_fullpath(filename), protocol, port)
284 | 
285 |         prt.delete_read_connections = True
286 |         prt.start()
287 |     else:
288 |         prt.reset_read_status()
289 |         prt.delete_read_connections = True
290 | 
291 |     errors_list = []
292 |     counter = 0
293 |     print "predict"
294 | 
295 |     if phase == "testing":
296 |         t1, t2 = load_threshold(filename, protocol, port, hidden_layers, activation_function, dropout)
297 |         check_directory(filename, "results")
298 |         # fresult = open("results/{}/result-{}-hl{}-af{}-do{}-{}.csv".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout, testing_filename), "w")
299 |         open_conn()
300 |         experiment_id = create_experiment(filename, testing_filename, protocol, port, ",".join(hidden_layers), activation_function, dropout)
301 |         # if fresult is None:
302 |         #     raise Exception("Could not create file")
303 | 
304 |     # ftemp = open("results/data.txt", "wb")
305 |     # fcsv = open("results/data.csv", "wb")
306 |     # a = csv.writer(fcsv, quoting=csv.QUOTE_ALL)
307 |     # time.sleep(2)
308 |     i_counter = 0
309 |     # for i in range(0,10):
310 |     while (not prt.done) or (prt.has_ready_message()):
311 |         if not prt.has_ready_message():
312 |             prt.wait_for_data()
313 |         else:
314 |             buffered_packets = prt.pop_connection()
315 |             if buffered_packets is None:
316 |                 continue
317 |             if buffered_packets.get_payload_length("server") == 0:
318 |                 continue
319 | 
320 |             i_counter += 1
321 |             # print "{}-{}".format(i_counter, buffered_packets.id)
322 |             # print "{}-{}: {}".format(i_counter, buffered_packets.id, buffered_packets.get_payload("server")[:100])
323 |             byte_frequency = buffered_packets.get_byte_frequency("server")
324 |             # ftemp.write(buffered_packets.get_payload())
325 |             # a.writerow(byte_frequency)
326 |             data_x = numpy.reshape(byte_frequency, (1, 256))
327 |             decoded_x = autoencoder.predict(data_x)
328 |             # a.writerow(decoded_x[0])
329 | 
330 |             # fcsv.close()
331 |             error = numpy.mean((decoded_x - data_x) ** 2, axis=1)
332 |             # ftemp.write("\r\n\r\n{}".format(error))
333 |             # ftemp.close()
334 |             if phase == "training" or phase == "predicting":
335 |                 errors_list.append(error)
336 |             elif phase == "testing":
337 |                 decision = decide(error[0], t1, t2)
338 |                 # fresult.write("{},{},{},{},{},{}\n".format(buffered_packets.id, error[0], decision[0], decision[1], decision[2], buffered_packets.get_hexlify_payload()))
339 |                 write_results_to_db(experiment_id, buffered_packets, error, decision)
340 | 
341 |             counter += 1
342 |             sys.stdout.write("\rCalculated {} connections.".format(counter))
343 |             sys.stdout.flush()
344 | 
345 |     errors_list = numpy.reshape(errors_list, (1, len(errors_list)))
346 |     if phase == "training" or phase == "predicting":
347 |         save_mean_stdev(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list)
348 |         save_q3_iqr(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list)
349 |         save_median_mad(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list)
350 |     elif phase == "testing":
351 |         # fresult.close()
352 |         return
353 | 
354 | 
355 | def count_byte_freq(filename, protocol, port):
356 |     global prt
357 |     global conf
358 | 
359 |     read_conf()
360 | 
361 |     prt = StreamReaderThread(get_pcap_file_fullpath(filename), protocol, port)
362 |     prt.start()
363 |     prt.delete_read_connections = True
364 |     counter = 0
365 |     missed_counter = 0
366 | 
367 |     while not prt.done or prt.has_ready_message():
368 |         if not prt.has_ready_message():
369 |             # print(1)
370 |             # time.sleep(0.0001)
371 |             missed_counter += 1
372 |             sys.stdout.write("\r1-{} flows. Missed: {}. {} items in buffer. packets: {}. last ts: {}".format(counter, missed_counter, len(prt.tcp_buffer), prt.packet_counter, prt.last_timestamp))
373 |             sys.stdout.flush()
374 |             prt.wait_for_data()
375 |             continue
376 |         else:
377 |             start = time.time()
378 |             buffered_packets = prt.pop_connection()
379 |             end = time.time()
380 |             if buffered_packets is None:
381 |                 # print(2)
382 |                 # time.sleep(0.0001)
383 |                 missed_counter += 1
384 |                 sys.stdout.write("\r2-{} flows. Missed: {}. Time: {}".format(counter, missed_counter, end - start))
385 |                 sys.stdout.flush()
386 |                 prt.wait_for_data()
387 |                 continue
388 |             elif buffered_packets.get_payload_length("server") > 0:
389 |                 counter += 1
390 |                 sys.stdout.write("\r3-{} flows. Missed: {}. Time: {}".format(counter, missed_counter, end-start))
391 |                 sys.stdout.flush()
392 |             else:
393 |                 missed_counter += 1
394 |                 sys.stdout.write("\r4-{} flows. Missed: {}. Time: {}".format(counter, missed_counter, end - start))
395 |                 sys.stdout.flush()
396 | 
397 |     print "Total flows: {}".format(counter)
398 | 
399 | 
400 | def save_mean_stdev(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list):
401 |     mean = numpy.mean(errors_list)
402 |     stdev = numpy.std(errors_list)
403 |     fmean = open("models/{}/mean-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "w")
404 |     fmean.write("{},{}".format(mean, stdev))
405 |     fmean.close()
406 | 
407 | 
408 | def save_q3_iqr(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list):
409 |     qs = numpy.percentile(errors_list, [100, 75, 50, 25, 0])
410 |     iqr = qs[1] - qs[3]
411 |     MC = ((qs[0]-qs[2])-(qs[2]-qs[4]))/(qs[0]-qs[4])
412 |     if MC >= 0:
413 |         constant = 3
414 |     else:
415 |         constant = 4
416 |     iqrplusMC = 1.5 * math.pow(math.e, constant * MC) * iqr
417 |     print "IQR: {}\nMC: {}\nConstant: {}".format(iqr, MC, constant)
418 |     fmean = open("models/{}/median-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "w")
419 |     fmean.write("{},{}".format(qs[1], iqrplusMC))
420 |     fmean.close()
421 | 
422 | 
423 | def save_median_mad(filename, protocol, port, hidden_layers, activation_function, dropout, errors_list):
424 |     median = numpy.median(errors_list)
425 |     mad = numpy.median([numpy.abs(error - median) for error in errors_list])
426 | 
427 |     fmean = open("models/{}/zscore-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "w")
428 |     fmean.write("{},{}".format(median, mad))
429 |     fmean.close()
430 | 
431 | 
432 | def load_threshold(filename, protocol, port, hidden_layers, activation_function, dropout):
433 |     t1 = []
434 |     t2 = []
435 | 
436 |     fmean = open(
437 |         "models/{}/mean-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "r")
438 |     line = fmean.readline()
439 |     split = line.split(",")
440 |     t1.append(split[0])
441 |     t2.append(split[1])
442 |     fmean.close()
443 | 
444 |     fmean = open(
445 |         "models/{}/median-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "r")
446 |     line = fmean.readline()
447 |     split = line.split(",")
448 |     t1.append(split[0])
449 |     t2.append(split[1])
450 |     fmean.close()
451 | 
452 |     fmean = open(
453 |         "models/{}/zscore-{}-hl{}-af{}-do{}.txt".format(filename, protocol + port, ",".join(hidden_layers), activation_function, dropout), "r")
454 |     line = fmean.readline()
455 |     split = line.split(",")
456 |     t1.append(split[0])
457 |     t2.append(split[1])
458 |     fmean.close()
459 | 
460 |     return t1, t2
461 | 
462 | 
463 | def get_threshold(threshold_method, t1, t2):
464 |     if threshold_method == "mean":
465 |         return (float(t1[0]) + 2 * float(t2[0]))
466 |     elif threshold_method == "median":
467 |         return (float(t1[1]) + float(t2[1]))
468 |     elif threshold_method == "zscore":
469 |         return 3.5
470 | 
471 | 
472 | def decide(mse, t1, t2):
473 |     decision = []
474 | 
475 |     if mse > (float(t1[0]) + 2 * float(t2[0])):
476 |         decision.append(True)
477 |     else:
478 |         decision.append(False)
479 | 
480 |     if mse > (float(t1[1]) + float(t2[1])):
481 |         decision.append(True)
482 |     else:
483 |         decision.append(False)
484 | 
485 |     zscore = 0.6745 * (mse - float(t1[2])) / float(t2[2])
486 |     if zscore > 3.5 or zscore < -3.5:
487 |         decision.append(True)
488 |     else:
489 |         decision.append(False)
490 | 
491 |     return decision
492 | 
493 | 
494 | def check_directory(filename, root = "models"):
495 |     if not os.path.isdir("./{}/{}".format(root, filename)):
496 |         os.mkdir("./{}/{}".format(root, filename))
497 | 
498 | 
499 | def get_pcap_file_fullpath(filename):
500 |     global conf
501 |     for i in range(0, len(conf["root_directory"])):
502 |         if os.path.isfile(conf["root_directory"][i] + filename):
503 |             return conf["root_directory"][i] + filename
504 | 
505 | 
506 | def open_conn():
507 |     global conn
508 | 
509 |     conn = psycopg2.connect(host="localhost", database="aeids", user="postgres", password="postgres")
510 |     conn.set_client_encoding('Latin1')
511 | 
512 | 
513 | def create_experiment(training_filename, testing_filename, protocol, port, hidden_layer, activation_function, dropout):
514 |     global conn
515 | 
516 |     cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
517 |     cursor.execute("SELECT * FROM experiments WHERE training_filename=%s AND testing_filename=%s AND protocol=%s AND port=%s AND hidden_layers=%s AND activation_function=%s AND dropout=%s", (training_filename, testing_filename, protocol, port, hidden_layer, activation_function, dropout))
518 | 
519 |     if cursor.rowcount > 0: # There is an existing experiment, get the ID
520 |         row = cursor.fetchone()
521 |         return row["id"]
522 |     else:
523 |         cursor.execute("INSERT INTO experiments(training_filename, testing_filename, protocol, port, hidden_layers, activation_function, dropout) VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING id", (training_filename, testing_filename, protocol, port, hidden_layer, activation_function, dropout))
524 |         if cursor.rowcount == 1:
525 |             row = cursor.fetchone()
526 |             conn.commit()
527 |             return row["id"]
528 |         else:
529 |             raise Exception("Cannot insert a new experiment")
530 | 
531 | 
532 | def get_message_id(buffered_packet):
533 |     global conn
534 | 
535 |     tmp = buffered_packet.id.split("-")
536 |     src_addr = tmp[0]
537 |     src_port = tmp[1]
538 |     dst_addr = tmp[2]
539 |     dst_port = tmp[3]
540 |     protocol = tmp[4]
541 |     start_time = buffered_packet.get_start_time()
542 |     stop_time = buffered_packet.get_stop_time()
543 | 
544 |     cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
545 |     cursor.execute("SELECT * FROM messages WHERE src_ip=%s AND src_port=%s AND dst_ip=%s AND dst_port=%s AND "
546 |                    "protocol=%s AND window_size=%s AND start_time=%s AND stop_time=%s", (src_addr, src_port, dst_addr, dst_port, protocol, WINDOW_SIZE, start_time, stop_time))
547 | 
548 |     if cursor.rowcount > 0:
549 |         row = cursor.fetchone()
550 |         return row["id"]
551 |     else:
552 |         cursor.execute("""INSERT INTO messages (src_ip, src_port, dst_ip, dst_port, protocol, start_time, stop_time, """
553 |                        """payload, window_size) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
554 |                        (src_addr, src_port, dst_addr, dst_port, protocol, start_time, stop_time,
555 |                         psycopg2.Binary(buffered_packet.get_payload("server")), WINDOW_SIZE))
556 |         if cursor.rowcount == 1:
557 |             row = cursor.fetchone()
558 |             conn.commit()
559 |             return row["id"]
560 |         else:
561 |             raise Exception("Cannot insert a new message")
562 | 
563 | 
564 | def write_results_to_db(experiment_id, buffered_packet, error, decision):
565 |     global conn
566 | 
567 |     cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
568 |     message_id = get_message_id(buffered_packet)
569 | 
570 |     cursor.execute("UPDATE mse_results SET mse=%s, decision_mean=%s, decision_median=%s, decision_zscore=%s WHERE messages_id=%s AND experiments_id=%s", (error[0], decision[0], decision[1], decision[2], message_id, experiment_id))
571 |     if cursor.rowcount == 0: # The row doesn't exist
572 |         cursor.execute("INSERT INTO mse_results (experiments_id, messages_id, mse, decision_mean, decision_median, decision_zscore) VALUES (%s, %s, %s, %s, %s, %s)", (experiment_id, message_id, error[0], decision[0], decision[1], decision[2]))
573 | 
574 |     conn.commit()
575 | 
576 | 
577 | if __name__ == '__main__':
578 | 	main(sys.argv)
579 | 


--------------------------------------------------------------------------------
/aeids.sql:
--------------------------------------------------------------------------------
  1 | --
  2 | -- PostgreSQL database dump
  3 | --
  4 | 
  5 | -- Dumped from database version 9.5.16
  6 | -- Dumped by pg_dump version 9.5.16
  7 | 
  8 | SET statement_timeout = 0;
  9 | SET lock_timeout = 0;
 10 | SET client_encoding = 'UTF8';
 11 | SET standard_conforming_strings = on;
 12 | SELECT pg_catalog.set_config('search_path', '', false);
 13 | SET check_function_bodies = false;
 14 | SET client_min_messages = warning;
 15 | SET row_security = off;
 16 | 
 17 | --
 18 | -- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: 
 19 | --
 20 | 
 21 | CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
 22 | 
 23 | 
 24 | --
 25 | -- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: 
 26 | --
 27 | 
 28 | COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
 29 | 
 30 | 
 31 | SET default_tablespace = '';
 32 | 
 33 | SET default_with_oids = false;
 34 | 
 35 | --
 36 | -- Name: experiments; Type: TABLE; Schema: public; Owner: postgres
 37 | --
 38 | 
 39 | CREATE TABLE public.experiments (
 40 |     id integer NOT NULL,
 41 |     protocol character varying(20),
 42 |     port integer,
 43 |     hidden_layers character varying(50),
 44 |     activation_function character varying(20),
 45 |     dropout double precision,
 46 |     training_filename character varying(255),
 47 |     testing_filename character varying(255)
 48 | );
 49 | 
 50 | 
 51 | ALTER TABLE public.experiments OWNER TO postgres;
 52 | 
 53 | --
 54 | -- Name: experiments_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
 55 | --
 56 | 
 57 | CREATE SEQUENCE public.experiments_id_seq
 58 |     START WITH 1
 59 |     INCREMENT BY 1
 60 |     NO MINVALUE
 61 |     NO MAXVALUE
 62 |     CACHE 1;
 63 | 
 64 | 
 65 | ALTER TABLE public.experiments_id_seq OWNER TO postgres;
 66 | 
 67 | --
 68 | -- Name: experiments_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres
 69 | --
 70 | 
 71 | ALTER SEQUENCE public.experiments_id_seq OWNED BY public.experiments.id;
 72 | 
 73 | 
 74 | --
 75 | -- Name: messages; Type: TABLE; Schema: public; Owner: postgres
 76 | --
 77 | 
 78 | CREATE TABLE public.messages (
 79 |     id integer NOT NULL,
 80 |     src_ip inet,
 81 |     src_port integer,
 82 |     dst_ip inet,
 83 |     protocol character varying(20),
 84 |     payload bytea,
 85 |     dst_port integer,
 86 |     window_size integer,
 87 |     start_time double precision,
 88 |     stop_time double precision
 89 | );
 90 | 
 91 | 
 92 | ALTER TABLE public.messages OWNER TO postgres;
 93 | 
 94 | --
 95 | -- Name: messages_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
 96 | --
 97 | 
 98 | CREATE SEQUENCE public.messages_id_seq
 99 |     START WITH 1
100 |     INCREMENT BY 1
101 |     NO MINVALUE
102 |     NO MAXVALUE
103 |     CACHE 1;
104 | 
105 | 
106 | ALTER TABLE public.messages_id_seq OWNER TO postgres;
107 | 
108 | --
109 | -- Name: messages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres
110 | --
111 | 
112 | ALTER SEQUENCE public.messages_id_seq OWNED BY public.messages.id;
113 | 
114 | 
115 | --
116 | -- Name: mse_results; Type: TABLE; Schema: public; Owner: postgres
117 | --
118 | 
119 | CREATE TABLE public.mse_results (
120 |     experiments_id integer NOT NULL,
121 |     messages_id integer NOT NULL,
122 |     mse double precision,
123 |     decision_mean boolean,
124 |     decision_median boolean,
125 |     decision_zscore boolean
126 | );
127 | 
128 | 
129 | ALTER TABLE public.mse_results OWNER TO postgres;
130 | 
131 | --
132 | -- Name: id; Type: DEFAULT; Schema: public; Owner: postgres
133 | --
134 | 
135 | ALTER TABLE ONLY public.experiments ALTER COLUMN id SET DEFAULT nextval('public.experiments_id_seq'::regclass);
136 | 
137 | 
138 | --
139 | -- Name: id; Type: DEFAULT; Schema: public; Owner: postgres
140 | --
141 | 
142 | ALTER TABLE ONLY public.messages ALTER COLUMN id SET DEFAULT nextval('public.messages_id_seq'::regclass);
143 | 
144 | 
145 | --
146 | -- Name: pk_error_results; Type: CONSTRAINT; Schema: public; Owner: postgres
147 | --
148 | 
149 | ALTER TABLE ONLY public.mse_results
150 |     ADD CONSTRAINT pk_error_results PRIMARY KEY (experiments_id, messages_id);
151 | 
152 | 
153 | --
154 | -- Name: pk_experiments; Type: CONSTRAINT; Schema: public; Owner: postgres
155 | --
156 | 
157 | ALTER TABLE ONLY public.experiments
158 |     ADD CONSTRAINT pk_experiments PRIMARY KEY (id);
159 | 
160 | 
161 | --
162 | -- Name: pk_messages; Type: CONSTRAINT; Schema: public; Owner: postgres
163 | --
164 | 
165 | ALTER TABLE ONLY public.messages
166 |     ADD CONSTRAINT pk_messages PRIMARY KEY (id);
167 | 
168 | 
169 | --
170 | -- Name: fk_errorresults_experiments; Type: FK CONSTRAINT; Schema: public; Owner: postgres
171 | --
172 | 
173 | ALTER TABLE ONLY public.mse_results
174 |     ADD CONSTRAINT fk_errorresults_experiments FOREIGN KEY (experiments_id) REFERENCES public.experiments(id);
175 | 
176 | 
177 | --
178 | -- Name: fk_errorresults_messages; Type: FK CONSTRAINT; Schema: public; Owner: postgres
179 | --
180 | 
181 | ALTER TABLE ONLY public.mse_results
182 |     ADD CONSTRAINT fk_errorresults_messages FOREIGN KEY (messages_id) REFERENCES public.messages(id);
183 | 
184 | 
185 | --
186 | -- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres
187 | --
188 | 
189 | REVOKE ALL ON SCHEMA public FROM PUBLIC;
190 | REVOKE ALL ON SCHEMA public FROM postgres;
191 | GRANT ALL ON SCHEMA public TO postgres;
192 | GRANT ALL ON SCHEMA public TO PUBLIC;
193 | 
194 | 
195 | --
196 | -- PostgreSQL database dump complete
197 | --
198 | 
199 | 


--------------------------------------------------------------------------------
/aeids_animation.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | <head>
  4 |     <meta charset="UTF-8">
  5 |     <title>Title</title>
  6 |     <style>
  7 |       .bar{
  8 |         fill: steelblue;
  9 |       }
 10 | 
 11 |       .bar:hover{
 12 |         fill: brown;
 13 |       }
 14 | 
 15 |         .axis {
 16 |           font: 11px sans-serif;
 17 |         }
 18 | 
 19 |         .axis path,
 20 |         .axis line {
 21 |           fill: none;
 22 |           stroke: #000000;
 23 |           stroke-width: 2px;
 24 |           shape-rendering: crispEdges;
 25 |         }
 26 |     </style>
 27 |     <script src="http://d3js.org/d3.v3.min.js"></script>
 28 |     <script src="http://labratrevenge.com/d3-tip/javascripts/d3.tip.v0.6.3.js"></script>
 29 |     <!-- Latest compiled and minified CSS -->
 30 |     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" integrity="sha384-rwoIResjU2yc3z8GV/NPeZWAv56rSmLldC3R/AZzGRnGxQQKnKkoFVhFQhNUwEyJ" crossorigin="anonymous">
 31 | 
 32 |     <script src="https://code.jquery.com/jquery-3.1.1.slim.min.js" integrity="sha384-A7FZj7v+d/sdmMqp/nOQwliLvUsJfDHW+k9Omg/a/EheAdgtzNs3hpfag6Ed950n" crossorigin="anonymous"></script>
 33 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/tether/1.4.0/js/tether.min.js" integrity="sha384-DztdAPBWPRXSA/3eYEEUWrWCy7G5KFbe8fFjk5JAIxUYHKkDx6Qin1DkWx51bBrb" crossorigin="anonymous"></script>
 34 |     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/js/bootstrap.min.js" integrity="sha384-vBWWzlZJ8ea9aCX4pEW3rVHjgjt7zpkNpZk+02D9phzyeVkE+jo0ieGizqPLForn" crossorigin="anonymous"></script>
 35 | </head>
 36 | <body>
 37 |     <div class="row card">
 38 |         <h3 class="card-header">Byte Frequency</h3>
 39 |         <div class="card-block" id="graph_placeholder"></div>
 40 |     </div>
 41 |     <div class="row">
 42 |         <div class="col-md-4" id="buttons">
 43 |             <div class="card">
 44 |                 <h3 class="card-header">Commands</h3>
 45 |                 <div class="card-block">
 46 |                     <input class="btn btn-primary btn-lg btn-block" id="btn_continue" disabled type="button" onclick="start()" value="Continue Visualization">
 47 |                     <input class="btn btn-primary btn-lg btn-block" id="btn_stop" type="button" onclick="stop()" value="Stop Visualization">
 48 |                 </div>
 49 |             </div>
 50 |         </div>
 51 |         <div class="col-md-4">
 52 |             <div class="card">
 53 |                 <h3 class="card-header">HTTP Message</h3>
 54 |                 <div class="card-block">
 55 |                     <textarea rows="20" cols="50" id="txt_payload"></textarea>
 56 |                 </div>
 57 |             </div>
 58 |         </div>
 59 |         <div class="col-md-4">
 60 |             <div class="card">
 61 |                 <h3 class="card-header">Connection Status</h3>
 62 |                 <div class="card-block">
 63 |                     <div id="mse_placeholder"></div>
 64 |                     <h4 class="card-title" id="div_anomaly" style="color: red;">Prediction : </h4>
 65 |                 </div>
 66 |             </div>
 67 |         </div>
 68 |     </div>
 69 | 
 70 |     <script>
 71 | 
 72 |     // set the dimensions of the canvas
 73 |     var margin = {top: 20, right: 20, bottom: 70, left: 40},
 74 |         width = 2048 - margin.left - margin.right,
 75 |         height = 400 - margin.top - margin.bottom;
 76 | 
 77 | 
 78 |     // set the ranges
 79 |     var x = d3.scale.ordinal().rangeRoundBands([0, width], 0.1);
 80 | 
 81 |     var y = d3.scale.linear().range([height, 0]);
 82 | 
 83 |     // define the axis
 84 |     var xAxis = d3.svg.axis()
 85 |         .scale(x)
 86 |         .orient("bottom")
 87 |         .ticks(10);
 88 | 
 89 | 
 90 |     var yAxis = d3.svg.axis()
 91 |         .scale(y)
 92 |         .orient("left")
 93 |         .ticks(10);
 94 | 
 95 | 
 96 |     // add the SVG element
 97 |     var svg = d3.select("#graph_placeholder").append("svg")
 98 |         .attr("width", width + margin.left + margin.right)
 99 |         .attr("height", height + margin.top + margin.bottom)
100 |       .append("g")
101 |         .attr("transform",
102 |               "translate(" + margin.left + "," + margin.top + ")");
103 | 
104 |     // set the dimensions of the canvas
105 |     var mse_margin = {top: 20, right: 20, bottom: 70, left: 40},
106 |         mse_width = 500 - mse_margin.left - mse_margin.right,
107 |         mse_height = 400 - mse_margin.top - mse_margin.bottom;
108 | 
109 | 
110 |     // set the ranges
111 |     var mse_x = d3.scale.ordinal().rangeRoundBands([0, mse_width], 0.1);
112 | 
113 |     var mse_y = d3.scale.linear().range([mse_height, 0]);
114 | 
115 |     // define the axis
116 |     var mse_xAxis = d3.svg.axis()
117 |         .scale(mse_x)
118 |         .orient("bottom");
119 | 
120 | 
121 |     var mse_yAxis = d3.svg.axis()
122 |         .scale(mse_y)
123 |         .orient("left")
124 |         .ticks(10);
125 | 
126 | 
127 |     // add the SVG element
128 |     var mse_svg = d3.select("#mse_placeholder").append("svg")
129 |         .attr("width", mse_width + mse_margin.left + mse_margin.right)
130 |         .attr("height", mse_height + mse_margin.top + mse_margin.bottom)
131 |       .append("g")
132 |         .attr("transform",
133 |               "translate(" + mse_margin.left + "," + mse_margin.top + ")");
134 | 
135 |     function hex2a(hexx) {
136 |         var hex = hexx.toString();//force conversion
137 |         var str = '';
138 |         for (var i = 0; i < hex.length; i += 2)
139 |             str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
140 |         return str;
141 |     }
142 | 
143 |     var done = false;
144 |     console.log(done);
145 |     function draw()
146 |     {
147 |         // load the data
148 |         d3.json("http://localhost:8080/next", function(error, msg) {
149 |             if("error" in msg)
150 |             {
151 |                 alert(msg.error);
152 |                 clearTimeout(timer);
153 |                 return;
154 |             }
155 | 
156 |             data = msg.byte_freq;
157 |             mseData = msg.status;
158 |             document.getElementById("txt_payload").value = hex2a(msg.payload);
159 | 
160 |             if(msg.decision == 1) {
161 |                 document.getElementById("div_anomaly").innerHTML = "Prediction : Anomaly";
162 |             }
163 |             else {
164 |                 document.getElementById("div_anomaly").innerHTML = "Prediction : Normal";
165 |             }
166 | 
167 |             data.forEach(function(d) {
168 |                 d.Letter = d.Letter;
169 |                 d.Freq = d.Freq;
170 |             });
171 | 
172 |             mseData.forEach(function(d) {
173 |                 d.Letter = d.Letter;
174 |                 d.Freq = d.Freq;
175 |             });
176 | 
177 |             d3.selectAll('svg > g > *').remove();
178 | 
179 |               // scale the range of the data
180 |               x.domain(Array.apply(null, {length: 256}).map(Number.call, Number));
181 |               y.domain([0, 0.5]);
182 | 
183 |               var threshold = mseData[0].Freq;
184 |               console.log(threshold);
185 | 
186 |               mse_x.domain(d3.extent(mseData, function(d) {return d.Letter;}));
187 |               mse_y.domain([0, threshold * 3]);
188 | 
189 |               // add axis
190 |               svg.append("g")
191 |                   .attr("class", "x axis")
192 |                   .attr("transform", "translate(0," + height + ")")
193 |                   .call(xAxis)
194 |                 .selectAll("text")
195 |                   .style("text-anchor", "end")
196 |                   .attr("dx", "-.8em")
197 |                   .attr("dy", "-.55em")
198 |                   .attr("transform", "rotate(-90)" );
199 | 
200 |               svg.append("g")
201 |                   .attr("class", "y axis")
202 |                   .call(yAxis)
203 |                 .append("text")
204 |                   .attr("transform", "rotate(-90)")
205 |                   .attr("y", 5)
206 |                   .attr("dy", ".71em")
207 |                   .style("text-anchor", "end")
208 |                   .text("Frequency");
209 | 
210 |                mse_svg.append("g")
211 |                   .attr("class", "x axis")
212 |                   .attr("transform", "translate(0," + mse_height + ")")
213 |                   .call(mse_xAxis)
214 |                 .selectAll("text")
215 |                   .style("text-anchor", "end")
216 |                   .attr("dx", "-.8em")
217 |                   .attr("dy", "-.55em")
218 |                   .attr("transform", "rotate(-90)" );
219 | 
220 |               mse_svg.append("g")
221 |                   .attr("class", "y axis")
222 |                   .call(mse_yAxis)
223 |                 .append("text")
224 |                   .attr("transform", "rotate(-90)")
225 |                   .attr("y", 5)
226 |                   .attr("dy", ".71em")
227 |                   .style("text-anchor", "end")
228 |                   .text("Frequency");
229 | 
230 | 
231 |           // Add bar chart
232 |           svg.selectAll("bar")
233 |               .data(data)
234 |             .enter().append("rect")
235 |               .attr("class", "bar")
236 |               .attr("x", function(d) { return x(d.Letter); })
237 |               .attr("width", x.rangeBand())
238 |               .attr("y", function(d) { return y(d.Freq); })
239 |               .attr("height", function(d) { return height - y(d.Freq); });
240 | 
241 |           mse_svg.selectAll("bar")
242 |               .data(mseData)
243 |             .enter().append("rect")
244 |               .attr("class", "bar")
245 |               .attr("x", function(d) { return mse_x(d.Letter); })
246 |               .attr("width", mse_x.rangeBand())
247 |               .attr("y", function(d) { return mse_y(d.Freq); })
248 |               .attr("height", function(d) { return mse_height - mse_y(d.Freq); });
249 | 
250 |         });
251 |     }
252 | 
253 |     var timer = setInterval(draw, 1000);
254 | 
255 |     function stop()
256 |     {
257 |         alert("stop");
258 |         clearTimeout(timer);
259 |         document.getElementById("btn_continue").disabled = false;
260 |         document.getElementById("btn_stop").disabled = true;
261 |     }
262 | 
263 |     function start()
264 |     {
265 |         timer = setInterval(draw, 1000);
266 |         document.getElementById("btn_continue").disabled = true;
267 |         document.getElementById("btn_stop").disabled = false;
268 |     }
269 | 
270 |     </script>
271 | </body>
272 | </html>


--------------------------------------------------------------------------------
/aeids_ws.py:
--------------------------------------------------------------------------------
  1 | from BufferedPackets import BufferedPackets
  2 | from PcapReaderThread import PcapReaderThread
  3 | from BaseHTTPServer import BaseHTTPRequestHandler
  4 | from aeids import load_mean_stdev, load_autoencoder, decide
  5 | 
  6 | import binascii
  7 | import json
  8 | import numpy
  9 | import sys
 10 | import web
 11 | 
 12 | 
 13 | counter = 1
 14 | prt = None
 15 | filename = "/home/baskoro/Documents/Dataset/HTTP-Attack-Dataset/morphed-shellcode-attacks/all_morphed_shellcode_attacks_payloads.pcap"
 16 | #filename = "/home/baskoro/Documents/Dataset/ISCX12/without retransmission/testbed-13jun-attack.pcap"
 17 | protocol = "tcp"
 18 | port = "80"
 19 | autoencoder = None
 20 | mean = 0
 21 | stdev = 0
 22 | counter = 0
 23 | 
 24 | 
 25 | class AeidsWS():
 26 |     def __init__(self):
 27 |         urls = (
 28 |             '/next', 'GetMessage',
 29 |             '/reset', 'ResetReader'
 30 |         )
 31 |         self.app = web.application(urls, globals())
 32 |         global prt
 33 |         global protocol
 34 |         global port
 35 |         global autoencoder
 36 |         global mean
 37 |         global stdev
 38 | 
 39 |         autoencoder = load_autoencoder(protocol, port)
 40 |         (mean, stdev) = load_mean_stdev(protocol, port)
 41 | 
 42 |     def run(self):
 43 |         self.app.run()
 44 | 
 45 | 
 46 | class GetMessage:
 47 |     def GET(self):
 48 |         global autoencoder
 49 |         global mean
 50 |         global stdev
 51 |         global counter
 52 | 
 53 |         msg = {}
 54 |         web.header("Access-Control-Allow-Origin", "http://localhost:63342")
 55 |         global prt
 56 |         if prt.done and not prt.has_ready_message():
 57 |             msg['error'] = "No more message"
 58 |             print "Connections : " + str(counter)
 59 |             return json.dumps(msg)
 60 |         elif not prt.done and not prt.has_ready_message():
 61 |             msg['error'] = "Waiting for data"
 62 |             return json.dumps(msg)
 63 |         else:
 64 |             buffered_packets = prt.pop_connection()
 65 |             if buffered_packets is None:
 66 |                 msg['error'] = "BP is none"
 67 |                 return json.dumps(msg)
 68 |             else:
 69 |                 byte_frequency = buffered_packets.get_byte_frequency()
 70 |                 bf_json = []
 71 |                 payload_hex = binascii.hexlify(buffered_packets.get_payload())
 72 |                 msg['payload'] = payload_hex
 73 |                 for i in range(0, 256):
 74 |                     bf_json.append({'Letter' : i, 'Freq' : byte_frequency[i]})
 75 | 
 76 |                 data_x = numpy.reshape(byte_frequency, (1, 256))
 77 |                 decoded_x = autoencoder.predict(data_x)
 78 |                 error = numpy.sqrt(numpy.mean((decoded_x - data_x) ** 2, axis=1))
 79 |                 decision = decide(error[0], mean, stdev)
 80 |                 threshold = float(mean) + 2 * float(stdev)
 81 |                 status = []
 82 |                 status.append({'Letter': 'Threshold', 'Freq': threshold})
 83 |                 status.append({'Letter': 'MSE', 'Freq': error[0]})
 84 | 
 85 |                 msg["byte_freq"] = bf_json
 86 |                 msg["decision"] = decision
 87 |                 msg["status"] = status
 88 | 
 89 |                 counter += 1
 90 |                 return json.dumps(msg)
 91 | 
 92 | 
 93 | class ResetReader:
 94 |     def GET(self):
 95 |         global prt
 96 |         global filename
 97 |         global protocol
 98 |         global port
 99 | 
100 |         web.header("Access-Control-Allow-Origin", "http://localhost:63342")
101 | 
102 |         if not prt.done:
103 |             prt.done = True
104 | 
105 |         prt = PcapReaderThread(filename, protocol, port)
106 |         prt.start()
107 | 
108 |         msg = {}
109 |         msg['status'] = "Resetting PCAP reader..."
110 |         return json.dumps(msg)
111 | 
112 | 
113 | def main(argv):
114 |     try:
115 |         global prt
116 |         global filename
117 |         global protocol
118 |         global port
119 |         prt = PcapReaderThread(filename, protocol, port)
120 |         prt.start()
121 | 
122 |         aeids_ws = AeidsWS()
123 |         aeids_ws.run()
124 |     except IndexError:
125 |         print "Usage : python aeids_ws.py"
126 |     except KeyboardInterrupt:
127 |         prt.done = True
128 |         print "Service stopped ..."
129 | 
130 | 
131 | if __name__ == '__main__':
132 |     main(sys.argv)
133 | 
134 | 
135 | 
136 | 
137 | 


--------------------------------------------------------------------------------
/aeids_ws_online.py:
--------------------------------------------------------------------------------
  1 | from BufferedPackets import BufferedPackets
  2 | from OnlinePcapReaderThread import OnlinePcapReaderThread
  3 | from BaseHTTPServer import BaseHTTPRequestHandler
  4 | from aeids import load_threshold, load_autoencoder, decide, get_threshold
  5 | 
  6 | import binascii
  7 | import json
  8 | import numpy
  9 | import sys
 10 | import web
 11 | 
 12 | 
 13 | counter = 1
 14 | prt = None
 15 | protocol = "tcp"
 16 | port = "80"
 17 | autoencoder = None
 18 | t1 = 0
 19 | t2 = 0
 20 | counter = 0
 21 | threshold_method = "zscore"
 22 | training_filename = "testbed-14jun.pcap"
 23 | hidden_layers = ["200","100"]
 24 | activation_function = "relu"
 25 | dropout = "0.2"
 26 | 
 27 | 
 28 | class AeidsWSOnline():
 29 |     def __init__(self):
 30 |         urls = (
 31 |             '/next', 'GetMessage',
 32 |             '/reset', 'ResetReader'
 33 |         )
 34 |         self.app = web.application(urls, globals())
 35 |         global prt
 36 |         global protocol
 37 |         global port
 38 |         global autoencoder
 39 |         global t1
 40 |         global t2
 41 | 
 42 |         autoencoder = load_autoencoder(training_filename, protocol, port, hidden_layers, activation_function, dropout)
 43 |         # Keras bug, have to call function below after loading a model
 44 |         autoencoder._make_predict_function()
 45 |         (t1, t2) = load_threshold(training_filename, protocol, port, hidden_layers, activation_function, dropout)
 46 | 
 47 |     def run(self):
 48 |         self.app.run()
 49 | 
 50 | 
 51 | class GetMessage:
 52 |     def GET(self):
 53 |         global autoencoder
 54 |         global t1
 55 |         global t2
 56 |         global counter
 57 | 
 58 |         msg = {}
 59 |         #web.header("Access-Control-Allow-Origin", "http://localhost:63342")
 60 |         global prt
 61 |         if prt.done and not prt.has_ready_message():
 62 |             msg['error'] = "No more message"
 63 |             print "Connections : " + str(counter)
 64 |             return json.dumps(msg)
 65 |         elif not prt.done and not prt.has_ready_message():
 66 |             msg['error'] = "Waiting for data"
 67 |             return json.dumps(msg)
 68 |         else:
 69 |             buffered_packets = prt.pop_connection()
 70 |             if buffered_packets is None:
 71 |                 msg['error'] = "BP is none"
 72 |                 return json.dumps(msg)
 73 |             else:
 74 |                 byte_frequency = buffered_packets.get_byte_frequency()
 75 |                 input_bf_json = []
 76 |                 output_bf_json = []
 77 |                 payload_hex = binascii.hexlify(buffered_packets.get_payload())
 78 |                 msg['payload'] = payload_hex
 79 |                 for i in range(0, 256):
 80 |                     input_bf_json.append({'Letter' : i, 'Freq' : byte_frequency[i]})
 81 | 
 82 |                 data_x = numpy.reshape(byte_frequency, (1, 256))
 83 |                 decoded_x = autoencoder.predict(data_x)
 84 |                 error = numpy.mean((decoded_x - data_x) ** 2, axis=1)
 85 | 
 86 |                 decision = decide(error[0], t1, t2)
 87 |                 if threshold_method == "zscore":
 88 |                     mse = 0.6745 * (error[0] - float(t1[2])) / float(t2[2])
 89 |                 else:
 90 |                     mse = error[0]
 91 | 
 92 |                 threshold = get_threshold(threshold_method, t1, t2)
 93 |                 status = []
 94 |                 status.append({'Letter': 'Threshold', 'Freq': threshold})
 95 |                 status.append({'Letter': 'MSE', 'Freq': mse})
 96 |                 #decoded_x = numpy.reshape(decoded_x, (256))
 97 | 
 98 |                 for i in range(0, 256):
 99 |                     output_bf_json.append({'Letter': i, 'Freq': float(decoded_x[0][i])})
100 | 
101 |                 msg["input"] = input_bf_json
102 |                 msg["output"] = output_bf_json
103 |                 if threshold_method == "mean":
104 |                     msg["decision"] = decision[0]
105 |                 elif threshold_method == "median":
106 |                     msg["decision"] = decision[1]
107 |                 elif threshold_method == "zscore":
108 |                     msg["decision"] = decision[2]
109 |                 msg["status"] = status
110 | 
111 |                 counter += 1
112 |                 return json.dumps(msg)
113 | 
114 | 
115 | class ResetReader:
116 |     def GET(self):
117 |         global prt
118 |         global protocol
119 |         global port
120 | 
121 |         web.header("Access-Control-Allow-Origin", "http://localhost:63342")
122 | 
123 |         if not prt.done:
124 |             prt.done = True
125 | 
126 |         prt = OnlinePcapReaderThread(protocol, port)
127 |         prt.start()
128 | 
129 |         msg = {}
130 |         msg['status'] = "Resetting PCAP reader..."
131 |         return json.dumps(msg)
132 | 
133 | 
134 | def main(argv):
135 |     try:
136 |         global prt
137 |         global protocol
138 |         global port
139 |         prt = OnlinePcapReaderThread(protocol, port)
140 |         prt.start()
141 | 
142 |         aeids_ws_online = AeidsWSOnline()
143 |         aeids_ws_online.run()
144 |     except IndexError:
145 |         print "Usage : python aeids_ws.py"
146 |     except KeyboardInterrupt:
147 |         prt.done = True
148 |         print "Service stopped ..."
149 | 
150 | 
151 | if __name__ == '__main__':
152 |     main(sys.argv)
153 | 
154 | 
155 | 
156 | 
157 | 


--------------------------------------------------------------------------------
/pcap_to_csv.py:
--------------------------------------------------------------------------------
 1 | from StreamReaderThread import StreamReaderThread
 2 | 
 3 | import sys
 4 | import time
 5 | 
 6 | 
 7 | def main(argv):
 8 |     try:
 9 |         filename = argv[1]
10 |         protocol = "tcp"
11 |         prt = StreamReaderThread(filename, protocol, "80")
12 |         prt.delete_read_connections = True
13 |         prt.start()
14 |         fcsv = open("csv/test.csv", "w")
15 |         counter = 0
16 | 
17 |         while not prt.done or prt.has_ready_message():
18 |             if not prt.has_ready_message():
19 |                 time.sleep(0.0001)
20 |                 continue
21 |             buffered_packets = prt.pop_connection()
22 |             if buffered_packets is not None:
23 |                 #print(buffered_packets.get_byte_frequency("client"))
24 |                 counter += 1
25 |                 byte_frequency = ",".join(str(buffered_packets.get_byte_frequency("server")))
26 |                 fcsv.write("{},{},{},{},{}\n".format(buffered_packets.tcp_tuple[0], buffered_packets.tcp_tuple[1], buffered_packets.tcp_tuple[2], buffered_packets.tcp_tuple[3], byte_frequency))
27 |                 sys.stdout.write("\r{} flows.".format(counter))
28 |                 sys.stdout.flush()
29 | 
30 |         fcsv.close()
31 | 
32 |     except IndexError:
33 |         print("Usage: python pcap_to_csv.py <pcap_filename>")
34 | 
35 | 
36 | if __name__ == '__main__':
37 | 	main(sys.argv)


--------------------------------------------------------------------------------