├── requirements.txt ├── examples └── http-get.fs ├── .gitignore ├── scripts └── make_pcap_poc.py ├── README.md ├── LICENSE └── src └── flowsynth.py /requirements.txt: -------------------------------------------------------------------------------- 1 | scapy 2 | argparse -------------------------------------------------------------------------------- /examples/http-get.fs: -------------------------------------------------------------------------------- 1 | flow default tcp example.com:44123 > google.com:80 (tcp.initialize;); 2 | default > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0aUser-Agent: DogBot\x0d\x0a\x0d\x0a";); 3 | default < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!\x0d\x0a\x0d\x0a";); 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | #Jetbrains IDE 104 | .idea -------------------------------------------------------------------------------- /scripts/make_pcap_poc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | make_pcap_poc.py - A tool that takes a file and creates a pcap of that 4 | file being downloaded over HTTP. Originally created to make 5 | pcaps from proof of concept exploit files related to particular CVEs. 6 | This uses flowsynth to make the pcap. 7 | """ 8 | # Copyright 2017 Secureworks 9 | # 10 | # Licensed under the Apache License, Version 2.0 (the "License"); 11 | # you may not use this file except in compliance with the License. 12 | # You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | # See the License for the specific language governing permissions and 20 | # limitations under the License. 21 | 22 | import os 23 | import glob 24 | import sys 25 | import re 26 | import random 27 | from mimetypes import MimeTypes 28 | import imp 29 | import shlex 30 | import tempfile 31 | 32 | DEBUG = True 33 | 34 | file_from_external_net = True 35 | HTTP_PORT = 80 36 | cve_re = re.compile(r"(?PCVE[\x2D\x5F]?\d{2,4}[\x2D\x5F]?\d{1,6})", re.IGNORECASE) 37 | 38 | def print_debug(msg): 39 | global DEBUG 40 | if msg and DEBUG: 41 | print "\t%s" % msg 42 | 43 | def print_error(msg): 44 | print "ERROR! %s" % msg 45 | sys.exit(1) 46 | 47 | # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # 48 | # change to wherever you have flowsynth.py if not in import path # 49 | flowsynth_script = os.path.join("..", "src", "flowsynth.py") 50 | 51 | try: 52 | import flowsynth 53 | except: 54 | print_debug("Could not import flowsynth, trying load source from \'%s\'." % flowsynth_script) 55 | if not os.path.isfile(flowsynth_script): 56 | print_error("FlowSynth script file \'%s\' does not exist! Update the path in this script (%s)." % (flowsynth_script, sys.argv[0])) 57 | try: 58 | flowsynth = imp.load_source('flowsynth', flowsynth_script) 59 | except Exception, e: 60 | print_error("Could not import flowsynth or load from file \'%s\'. Error:\n%s" % (flowsynth_script, e)) 61 | 62 | def usage(): 63 | print "Usage: make_pcap_poc.py []" 64 | sys.exit(1) 65 | 66 | if len(sys.argv) < 2: 67 | usage() 68 | 69 | poc_file = os.path.abspath(sys.argv[1]) 70 | 71 | if not os.path.isfile(poc_file): 72 | print_error("PoC file \'%s\' does not exist!" % poc_file) 73 | 74 | # try to get CVE based on name and/or path 75 | cve = "CVE-unknown" 76 | result = cve_re.search(os.path.abspath(poc_file)) 77 | if result: 78 | cve = result.group('CVE') 79 | 80 | print "Creating pcap for %s" % cve 81 | 82 | if len(sys.argv) > 2: 83 | pcap_file = sys.argv[2] 84 | else: 85 | pcap_file = "%s_%s.pcap" % (cve, os.getpid()) 86 | 87 | # make the flowsynth file 88 | fs_fh = tempfile.NamedTemporaryFile() 89 | print_debug("FlowSynth file: %s" % fs_fh.name) 90 | 91 | client_ip = "192.168.%d.%d" % (random.randint(0,255), random.randint(0,255)) 92 | server_ip = "172.%d.%d.%d" % (random.randint(16,31), random.randint(0,255), random.randint(0,255)) 93 | if not file_from_external_net: 94 | client_ip_temp = client_ip 95 | client_ip = server_ip 96 | server_ip = client_ip_temp 97 | print_debug("Client IP: %s" % client_ip) 98 | print_debug("Server IP: %s" % server_ip) 99 | 100 | # get file size 101 | file_size = os.path.getsize(poc_file) 102 | print_debug("Using file size: %d" % file_size) 103 | 104 | # get MIME type 105 | mime_type = MimeTypes().guess_type(os.path.basename(poc_file))[0] 106 | if not mime_type: 107 | mime_type = 'text/html' 108 | 109 | print_debug("Using MIME type \'%s\'" % mime_type) 110 | 111 | fs_fh.write("flow default tcp %s:%d > %s:%d (tcp.initialize;);\n" % (client_ip, random.randint(1025, 65535), server_ip, HTTP_PORT)) 112 | fs_fh.write("""default > (content:\"GET /%s/%s HTTP/1.1\\x0d\\x0aUser-Agent: FlowSynth Puncha Yopet Edition (make_pcap_poc.py)\\x0d\\x0aTest-For: %s\\x0d\\x0a\\x0d\\x0a\";);\n""" % (cve, os.path.basename(poc_file), cve)) 113 | fs_fh.write("""default < (content:\"HTTP/1.1 200 OK\\x0D\\x0AServer: FlowSynth (Petty Petter)\\x0D\\x0AContent-Type: %s\\x0D\\x0AContent-Length: %d\\x0D\\x0A\\x0D\\x0A\"; filecontent:\"%s\";);\n""" % (mime_type, file_size, poc_file)) 114 | 115 | # important - reset file pointer so we can read from the top 116 | fs_fh.seek(0) 117 | 118 | fs_args = "flowsynth.py \"%s\" -f pcap -w \"%s\"" % (fs_fh.name, os.path.abspath(pcap_file)) 119 | sys.argv = shlex.split(fs_args) 120 | 121 | flowsynth.main() 122 | 123 | fs_fh.close() 124 | 125 | print "Done. Wrote pcap to:\n%s" % pcap_file 126 | 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flowsynth # 2 | 3 | Flowsynth is a tool for rapidly modeling network traffic. Flowsynth can be used to generate text-based hexdumps of packets as well as native libpcap format packet captures. 4 | 5 | ## Installation ## 6 | 7 | Flowsynth was developed on Python 2.7. The following python modules are required to run Flowsynth: 8 | 9 | + argparse 10 | + scapy 11 | 12 | While it may be feasible to run Flowsynth on other versions of Python, this has not been tested. 13 | 14 | ## How it works ## 15 | 16 | Flowsynth uses a syntax language to describe network flows. The syntax language is made up of individual *instructions* that are parsed by the application and are grouped into *events*, which are a logical representation of the *instructions* in the network domain. After all *instructions* have been parsed, the *events* are iterated over and converted into *packets*, which are the real-world representation of the traffic on the wire. 17 | 18 | These three phases are referred to as the *parsing phase*, *rendering phase*, and the *output phase*. 19 | 20 | Take the following synfile as an example: 21 | 22 | flow default tcp myhost.corp.acme.net:12323 > google.com:80 ( tcp.initialize; ); 23 | default > ( content:"GET / HTTP/1.1\x0d\x0a"; content:"Host: google.com\x0d\x0a\x0d\x0a"; ); 24 | default < ( content:"HTTP/1.1 200 OK"; ); 25 | 26 | This sample contains two types of instructions: Flow declarations and event declarations. The first line (*flow default tcp...*) declares to Flowsynth that a flow is being tracked between myhost.corp.acme.net and google.com. The flow name is *default*. All events that apply to this flow will use this name (*default*) to identify which flow they apply to. The third argument specifies which protocol the flow will use. In this case it's *tcp*. Next we specify the source and destination addresses and ports. Finally, an optional attributes section is included at the end. The *tcp.initialize* attribute is included, which tells Flowsynth to automatically generate a three-way handshake for this flow. It's worth nothing that each attribute and line should be closed with a semicolon (;), as shown above. When this flow declaration instruction is parsed by Flowsynth the application will automatically generate event entries in the compiler timeline to establish a three way handshake. 27 | 28 | Next, Flowsynth will parse the event declaration *default > ( content ...*. Flowsynth will immediately identify that this event declaration belongs to the 'default' flow that was just declared. Once this event is associated with the flow any protocol specific values (like TCP SEQ and ACK numbers) will automatically be applied to the event. The directionality for this specific event is '>', or TO_SERVER. Once the parent flow and directionality have been established Flowsynth will parse the optional attributes section. Just like the flow declaration, each optional attribute must be closed with a semicolon (;). The two 'content' attributes are used to specify the packet's payload. In this case, a HTTP request is being rendered. Flowsynth will read these instructions and generate an entry in the compiler timeline for this event. 29 | 30 | The last event declaration that is parsed by the application shows the server's response to the client. Using the same methods described above, Flowsynth will parse the event declaration and add it to the compiler timeline. 31 | 32 | Once all the instructions have been parsed and processed, Flowsynth iterates over the compiler timeline and renders any events to native packets. In this phase of the application several important things happen: 33 | 34 | 1. Protocol-specific intelligence, like TCP SEQ/ACK calculations, and ACK generation take place. 35 | 2. Specific features of attributes, like converting '*\x3A*' to '*:*' take place. 36 | 37 | Once all of the events have been rendered to native pcaps the output phase occurs. During the output phase the native packets are delivered to the user in one of the two output formats, as a hexdump, or as a native PCAP file. 38 | 39 | ## Usage ## 40 | 41 | flowsynth.py input.syn 42 | 43 | In this most basic example, Flowsynth will read input.syn and output the resulting hexdump to the screen. By default Flowsynth will use 'hex' format. 44 | 45 | flowsynth.py input.syn -f pcap -w /tmp/test.pcap 46 | 47 | In this example, Flowsynth reads input.syn and outputs a libpcap formatted .pcap file to /tmp/test.pcap 48 | 49 | 50 | ## Syntax ## 51 | All Flowsynth syntax files are plain-text files. Currently three types of instructions are supported. 52 | 53 | + Comments 54 | + Flow Declarations 55 | + Event Declarations 56 | 57 | As new features are added, this syntax reference will be updated. 58 | 59 | ### Comments ### 60 | 61 | Comments are supported using the *#* symbol. 62 | 63 | # This is a synfile comment 64 | 65 | ### Flows ### 66 | 67 | #### Declaring a Flow #### 68 | You can declare a flow using the following syntax: 69 | 70 | flow [flow name] [proto] [src]:[srcport] [directionality] [dst]:[dstport] ([flow options]); 71 | 72 | The following flow declaration would describe a flow going from a computer to google.com: 73 | 74 | flow my_connection tcp mydesktop.corp.acme.com:44123 > google.com:80 (tcp.initialize;); 75 | 76 | The following flow declaration would describe a flow going from a computer to a DNS server: 77 | 78 | flow dns_request udp mydesktop.corp.acme.com:11234 > 8.8.8.8:53; 79 | 80 | For the interim, directionality should always be specified as to server: > 81 | 82 | If a DNS record is specified in the flow declaration (instead of an explicit IP address) then Flowsynth will resolve the DNS entry at the time of the flow's declaration. The first A record returned for DNS entry will be used as the IP address throughout the session. The DNS query and response is not included in the output. 83 | 84 | #### Flow Attributes ##### 85 | The following flow attributes are currently supported: 86 | 87 | ##### tcp.initialize ##### 88 | The *tcp.initialize* attribute informs Flowsynth that the flow should have an autogenerated TCP three-way handshake included in the output. The handshake is always added relative to the location of the flow declaration in the synfile. 89 | 90 | ### Events ### 91 | 92 | #### Transferring Data #### 93 | Data can be transferred between hosts using two methods. The example below outlines a data exchange between a client and a webserver: 94 | 95 | my_connection > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0aUser-Agent: DogBot\x0d\x0a\x0d\x0a";); 96 | my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); 97 | 98 | In this example, the flow *my_connection* must have been previously declared. A single packet with the content specified will be transmitted from the client to the server. The following method is also accepted, however, this may change in the future as the syntax is formalized.: 99 | 100 | my_connection.to_server (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0aUser-Agent: DogBot\x0d\x0a\x0d\x0a";); 101 | my_connection.to_client (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); 102 | 103 | Each content keyword within the () should be closed by a semicolon. Each line should also be closed with a semicolon. Failure to do so will generate a lexer error. Multiple content matches can also be used to logically seperate parts of the response, for example: 104 | 105 | # the commands below describe a simple HTTP request 106 | my_connection > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0a\x0d\x0a";); 107 | my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Type: text/html\x0d\x0a\x0d\x0a"; content:"This is my response body.";); 108 | 109 | #### Event Attributes #### 110 | The following event attributes are currently supported: 111 | 112 | + content 113 | + filecontent 114 | + tcp.seq 115 | + tcp.ack 116 | + tcp.noack 117 | + tcp.flags.syn 118 | + tcp.flags.ack 119 | + tcp.flags.rst 120 | 121 | ##### Content Attribute ##### 122 | The *content* attribute is used to specify the payload of a packet. Content attributes must be enclosed in double quotes. Special characters can be expressed in hex, like: *\x0d\x0a*. Anything prefaced with \x will be converted from hex to its ascii representation. These translation takes place during the render phase. 123 | 124 | Example: 125 | 126 | default > ( content: "GET / HTTP/1.1\x0d\x0a"; ); 127 | 128 | ##### Filecontent Attribute ##### 129 | The *filecontent* attribute is used to specify a file that can be used as the payload of a packet. The value of a filecontent attribute is the file that will be read into the payload. 130 | 131 | Example: 132 | 133 | default > ( content: "HTTP/1.1 200 OK\x0d\x0a\x0d\x0a"; filecontent: "index.html"; ); 134 | 135 | ##### tcp.seq Attribute ##### 136 | The *tcp.seq* attribute lets you set the sequence number for the event's packet. 137 | 138 | ##### tcp.ack Attribute ##### 139 | The *tcp.ack* attribute lets you set the acknowledgement number for the event's packet. 140 | 141 | ##### tcp.noack Attribute ##### 142 | The *tcp.noack* attribute tells Flowsynth to not generate an ACK for this event. 143 | 144 | ##### tcp.flags.syn Attribute ##### 145 | The *tcp.flags.syn* attribute tells Flowsynth to force the packet to be a SYN packet. 146 | 147 | ##### tcp.flags.ack Attribute ##### 148 | The *tcp.flags.ack* attribute tells Flowsynth to force the packet to be an ACK packet. 149 | 150 | ##### tcp.flags.rst Attribute ##### 151 | The *tcp.flags.rst* attribute tells Flowsynth to force the packet to be a RST packet. 152 | 153 | ## Authors ### 154 | 155 | + Will Urbanski (will dot urbanski at gmail dot com) 156 | 157 | #### Contributors #### 158 | 159 | + David Wharton 160 | 161 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. -------------------------------------------------------------------------------- /src/flowsynth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | flowsynth - a tool for rapidly modeling network traffic 4 | 5 | Copyright 2014 SecureWorks Corp. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | author: Will Urbanski 20 | """ 21 | 22 | 23 | 24 | import argparse 25 | import logging 26 | import re 27 | import random 28 | import shlex 29 | import sys 30 | import socket 31 | import time 32 | import json 33 | 34 | #include scapy; suppress all errors 35 | logging.getLogger("scapy.runtime").setLevel(logging.ERROR) 36 | logging.getLogger("scapy.interactive").setLevel(logging.ERROR) 37 | logging.getLogger("scapy.loading").setLevel(logging.ERROR) 38 | from scapy.all import Ether, IP, TCP, UDP, RandMAC, hexdump, wrpcap 39 | 40 | #global variables 41 | APP_VERSION_STRING = "1.0.6" 42 | LOGGING_LEVEL = logging.INFO 43 | ARGS = None 44 | 45 | #compiler specific vars 46 | COMPILER_FLOWS = {} # this is a dictionary containing the flow objects 47 | COMPILER_OUTPUT = [] # the output buffer containing a list of packets 48 | COMPILER_TIMELINE = [] # this is a list containing the global compiler timeline 49 | 50 | #timing 51 | START_TIME = 0 52 | END_TIME = 0 53 | 54 | #for recording the build status 55 | BUILD_STATUS = {} 56 | 57 | class SynSyntaxError(Exception): 58 | """ an exception for a syntax error when parsing a synfile """ 59 | def __init__(self, value): 60 | self.value = value 61 | Exception.__init__(self) 62 | def __str__(self): 63 | return repr(self.value) 64 | 65 | class SynTerminalError(Exception): 66 | """ an exception for a terminal error that cannot be recovered from """ 67 | def __init__(self, value): 68 | self.value = value 69 | Exception.__init__(self) 70 | def __str__(self): 71 | return repr(self.value) 72 | 73 | class SynCompileError(Exception): 74 | """a compile-time exception""" 75 | def __init__(self, value): 76 | self.value = value 77 | Exception.__init__(self) 78 | def __str__(self): 79 | return repr(self.value) 80 | 81 | class FSLexer: 82 | """a lexer for the synfile format""" 83 | 84 | LEX_NEW = 0 85 | LEX_EXISTING = 1 86 | 87 | INSTR_FLOW = 0 88 | INSTR_EVENT = 1 89 | 90 | status = 0 #status of the line lex 91 | instr = 0 92 | 93 | instructions = [] 94 | dnscache = {} 95 | 96 | def __init__(self, synfiledata): 97 | 98 | #init 99 | self.instructions = [] 100 | self.dnscache = {} 101 | 102 | lexer = list(shlex.shlex(synfiledata)) 103 | itr_ctr = 0 104 | while len(lexer) > 0: 105 | token = lexer[0] 106 | #should be the start of a new line 107 | #print "New lexer rotation. Starting with Token %s" % token 108 | #print "Tokens are %s" % lexer 109 | if (token.lower() == 'flow'): 110 | (flowdecl, lexer) = self.lex_flow(lexer[1:]) 111 | self.instructions.append(flowdecl) 112 | else: 113 | #treat as an event 114 | (eventdecl, lexer) = self.lex_event(lexer) 115 | self.instructions.append(eventdecl) 116 | itr_ctr = itr_ctr + 1 117 | 118 | def resolve_dns(self, shost): 119 | """Perform DNS lookups once per file, and cache the results. tested.""" 120 | rdns = r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" 121 | if (re.match(rdns, shost) == None): 122 | if shost in self.dnscache: 123 | logging.debug("Host %s in DNSCACHE, returned %s", shost, self.dnscache[shost]) 124 | shost = self.dnscache[shost] 125 | else: 126 | logging.debug("Host %s not in DNSCACHE", shost) 127 | #try socket lookupt 128 | try: 129 | resolved_ip = socket.gethostbyname(shost) 130 | self.dnscache[shost] = resolved_ip 131 | logging.debug("Resolved %s to %s", shost, resolved_ip) 132 | shost = resolved_ip 133 | except socket.gaierror: 134 | compiler_bailout("Cannot resolve %s" % shost) 135 | return shost 136 | 137 | def lex_flow(self, tokens): 138 | """ lex flow declarations""" 139 | logging.debug("lex_flow() called with %s", tokens) 140 | 141 | if (type(tokens) is not list): 142 | parser_bailout("FSLexer tried to flowlex a %s" % type(tokens)) 143 | 144 | #need to read the following mandatory values: 145 | try: 146 | flow_name = tokens[0] 147 | flow_proto = tokens[1] 148 | tokens = tokens[2:] 149 | except IndexError: 150 | raise SynSyntaxError("Corrupt flowdecl") 151 | 152 | flow_src = "" 153 | tok_ctr = 0 154 | for token in tokens: 155 | if (token == ':'): 156 | break 157 | else: 158 | flow_src = "%s%s" % (flow_src, token) 159 | tok_ctr = tok_ctr + 1 160 | tokens = tokens[tok_ctr+1:] 161 | try: 162 | flow_src_port = tokens[0] 163 | except IndexError: 164 | raise SynSyntaxError("No flow source port specified") 165 | 166 | directionality = tokens[1] 167 | if (directionality != ">" and directionality != "<"): 168 | raise SynSyntaxError("Unexpected flow directionality: %s" % directionality) 169 | 170 | tokens = tokens[2:] 171 | flow_dst = "" 172 | tok_ctr = 0 173 | for token in tokens: 174 | if (token == ':'): 175 | break 176 | else: 177 | flow_dst = "%s%s" % (flow_dst, token) 178 | tok_ctr = tok_ctr + 1 179 | tokens = tokens[tok_ctr+1:] 180 | flow_dst_port = tokens[0] 181 | tokens = tokens[1:] 182 | 183 | if (flow_proto.lower() == 'udp'): 184 | flow_proto = Flow.PROTO_UDP 185 | else: 186 | flow_proto = Flow.PROTO_TCP 187 | 188 | #logging.debug("Building flowdecl") 189 | 190 | #start to build our flow decl 191 | flowdecl = {} 192 | flowdecl['type'] = 'flow' 193 | flowdecl['name'] = flow_name 194 | flowdecl['proto'] = flow_proto 195 | flowdecl['src_host'] = self.resolve_dns(flow_src) 196 | flowdecl['src_port'] = flow_src_port 197 | flowdecl['dst_host'] = self.resolve_dns(flow_dst) 198 | flowdecl['dst_port'] = flow_dst_port 199 | flowdecl['flow'] = directionality 200 | flowdecl['attributes'] = {} 201 | 202 | if (tokens[0] == ";"): 203 | tokens = tokens[1:] 204 | 205 | #return flowdecl, tokens 206 | return (flowdecl, tokens) 207 | elif (tokens[0] == '('): 208 | tokens = tokens[1:] 209 | #parse modifiers 210 | 211 | while tokens[0] != ";": 212 | token = tokens[0] 213 | #print "token is %s" % token 214 | if (token == ")"): 215 | #end of attribute spec. jump forward two (should always be ');') 216 | tokens = tokens[1:] 217 | break 218 | modifier_key = "" 219 | tok_ctr = 0 220 | single_modifier = False 221 | for token in tokens: 222 | if (token == ':'): 223 | tokens = tokens[tok_ctr+1:] 224 | break 225 | elif (token == ";"): 226 | tokens = tokens[tok_ctr+1:] 227 | single_modifier = True 228 | break 229 | else: 230 | modifier_key = "%s%s" % (modifier_key, token) 231 | tok_ctr = tok_ctr + 1 232 | 233 | #print "ModKey: %s" % modifier_key 234 | if (single_modifier == False): 235 | modifier_value = "" 236 | tok_ctr = 0 237 | for token in tokens: 238 | if (token == ';' or token == ")"): 239 | tokens = tokens[tok_ctr+1:] 240 | break 241 | else: 242 | modifier_value = "%s%s" % (modifier_value, token) 243 | tok_ctr = tok_ctr + 1 244 | else: 245 | modifier_value = True 246 | 247 | flowdecl['attributes'][modifier_key] = modifier_value 248 | 249 | #print "ModValue: %s" % modifier_value 250 | tokens = tokens[1:] 251 | #print 'Flowdecl: %s' % flowdecl 252 | #print 'Tokens: %s' % tokens 253 | return (flowdecl, tokens) 254 | else: 255 | parser_bailout("Invalid Syntax. unexpected value %s" % tokens[0]) 256 | 257 | 258 | def lex_event(self, tokens): 259 | """ lex an event declarations""" 260 | logging.debug("lex_event() called with %s", tokens) 261 | 262 | flow_name = tokens[0] 263 | try: 264 | if (tokens[1] == '.'): 265 | idx_flowdir = 2 266 | else: 267 | idx_flowdir = 1 268 | except IndexError: 269 | parser_bailout("Invalid Syntax. Unexpected flow directionality.") 270 | 271 | flow_directionality = tokens[idx_flowdir] 272 | tokens = tokens[idx_flowdir+1:] 273 | 274 | #print "Flow name: %s" % flow_name 275 | #print "Flow directionality: %s" % flow_directionality 276 | 277 | eventdecl = {} 278 | eventdecl['name'] = flow_name 279 | eventdecl['type'] = 'event' 280 | eventdecl['attributes'] = {} 281 | eventdecl['contents'] = [] 282 | if (flow_directionality == '>' or flow_directionality == 'to_server'): 283 | eventdecl['flow'] = Flow.FLOW_TO_SERVER 284 | else: 285 | eventdecl['flow'] = Flow.FLOW_TO_CLIENT 286 | 287 | #print "Tokens are %s" % tokens 288 | 289 | if (tokens[0] == '('): 290 | tokens = tokens[1:] 291 | 292 | while tokens[0] != ";": 293 | token = tokens[0] 294 | #print "token is %s" % token 295 | if (token == ")"): 296 | #end of attribute spec. jump forward two (should always be ');') 297 | tokens = tokens[1:] 298 | break 299 | modifier_key = "" 300 | tok_ctr = 0 301 | single_modifier = False 302 | for token in tokens: 303 | if (token == ':'): 304 | tokens = tokens[tok_ctr+1:] 305 | break 306 | elif (token == ";"): 307 | tokens = tokens[tok_ctr+1:] 308 | single_modifier = True 309 | break 310 | else: 311 | modifier_key = "%s%s" % (modifier_key, token) 312 | tok_ctr = tok_ctr + 1 313 | 314 | #print "ModKey: %s" % modifier_key 315 | if (single_modifier == False): 316 | modifier_value = "" 317 | tok_ctr = 0 318 | for token in tokens: 319 | if (token == ';' or token == ")"): 320 | tokens = tokens[tok_ctr+1:] 321 | break 322 | else: 323 | modifier_value = "%s%s" % (modifier_value, token) 324 | tok_ctr = tok_ctr + 1 325 | else: 326 | modifier_value = True 327 | 328 | if (modifier_key.lower() == 'content'): 329 | #content 330 | eventdecl['contents'].append({'type': 'string', 'value': modifier_value}) 331 | elif (modifier_key.lower() == 'filecontent'): 332 | #filecontent 333 | if ARGS.no_filecontent: 334 | # '--no-filecontent' option was passed to flowsynth 335 | compiler_bailout("The 'filecontent' attribute is not supported in this context.") 336 | else: 337 | eventdecl['contents'].append({'type': 'file', 'value': modifier_value}) 338 | elif (modifier_key.lower() == 'uricontent'): 339 | #uricontent 340 | eventdecl['contents'].append({'type': 'uri', 'value': modifier_value}) 341 | else: 342 | eventdecl['attributes'][modifier_key] = modifier_value 343 | 344 | #print "ModValue: %s" % modifier_value 345 | 346 | #skip trailing ; 347 | tokens = tokens[1:] 348 | 349 | 350 | return (eventdecl, tokens) 351 | 352 | 353 | class Flow: 354 | """a class for modeling a specific flow""" 355 | 356 | #consts for different protocols 357 | PROTO_TCP = 0 358 | PROTO_UDP = 1 359 | 360 | #consts for flow directionality 361 | FLOW_TO_SERVER = 0 362 | FLOW_TO_CLIENT = 1 363 | FLOW_BIDIRECTIONAL = 2 364 | 365 | #specific values for the flow 366 | proto = 0 367 | flow = 0 368 | name = "" 369 | src_mac = "" 370 | dst_mac = "" 371 | src_host = "" 372 | src_port = 0 373 | dst_host = "" 374 | dst_port = 0 375 | initialized = False 376 | timeline = [] 377 | 378 | #tcp specific values 379 | to_server_seq = 0 380 | to_client_seq = 0 381 | to_server_ack = 0 382 | to_client_ack = 0 383 | tcp_mss = 1460 384 | 385 | #has test case 386 | def __init__(self, flowdecl = None): 387 | """constructor for the flow class. accepts a flowdecl (dictionary) with flow info""" 388 | if (type(flowdecl) != dict): 389 | parser_bailout("Flowdecl must be a dictionary.") 390 | try: 391 | self.name = flowdecl['name'] 392 | self.proto = flowdecl['proto'] 393 | self.src_host = flowdecl['src_host'] 394 | self.src_port = flowdecl['src_port'] 395 | self.flow = flowdecl['flow'] 396 | self.dst_host = flowdecl['dst_host'] 397 | self.dst_port = flowdecl['dst_port'] 398 | except KeyError: 399 | parser_bailout("Invalid flowdecl passed to Flow.init") 400 | 401 | self.src_mac = RandMAC() 402 | self.dst_mac = RandMAC() 403 | 404 | self.to_server_seq = random.randint(10000, 99999) 405 | self.to_client_seq = random.randint(10000, 99999) 406 | self.to_server_ack = 0 407 | self.to_client_ack = 0 408 | self.tcp_server_bytes = 0 409 | self.tcp_client_bytes = 0 410 | 411 | try: 412 | self.tcp_mss = int(flowdecl['attributes']['mss']) 413 | except KeyError: 414 | self.tcp_mss = 1460 415 | 416 | #has test case 417 | #This function expects all inputs to be enclosed within double quotes 418 | def parse_content(self, content): 419 | """ parse and render a content keyword """ 420 | #pcre_file = r"file\([^\)]+\)" 421 | #pcre_url = r"url\([^\)]+\)" 422 | pcre_text = r'"([^\\"]*(?:\\.[^\\"]*)*)"' 423 | 424 | #print "CONTENT: %s" % content 425 | 426 | #first, check for text 427 | mo_text = re.match(pcre_text, content) 428 | if (mo_text != None): 429 | logging.debug("Content: %s", mo_text.group(1)) 430 | 431 | content_text = mo_text.group(1) 432 | replacements = re.findall(r"\\x[a-fA-F0-9]{2}", content_text) 433 | for replacement in replacements: 434 | content_text = content_text.replace(replacement, chr(int(replacement[2:], 16))) 435 | 436 | # replacements = re.findall(r"\|[A-Fa-f0-9\ ]+\|", content_text) 437 | # for replacement in replacements: 438 | # #remove all spaces 439 | # orig_replacement = replacement 440 | # replacement = replacement.replace(" ","").replace("|","") 441 | # replacement_text = "" 442 | # for sub in [replacement[i:i+2] for i in range(0, len(replacement), 2)]: 443 | # #break into groups of two characters 444 | # replacement_text = "%s%s" % (replacement_text, chr(int(sub, base=16))) 445 | # content_text = content_text.replace(orig_replacement, replacement_text) 446 | 447 | return content_text 448 | 449 | def render_payload(self, event): 450 | """ render all content matches into one payload value """ 451 | str_payload = "" 452 | for modifier in event['attributes']: 453 | #logging.debug("Found modifier: %s", modifier) 454 | keyword = modifier 455 | value = event['attributes'][keyword] 456 | 457 | #logging.debug("Parsed keyword: %s value: %s", keyword, value) 458 | 459 | if 'contents' in event: 460 | for contentobj in event['contents']: 461 | content_value = contentobj['value'] 462 | content_type = contentobj['type'] 463 | if (content_type == 'string'): 464 | str_payload = "%s%s" % (str_payload, self.parse_content(content_value)) 465 | elif (content_type == 'file'): 466 | if ARGS.no_filecontent: 467 | # '--no-filecontent' option was passed to flowsynth 468 | # This is also checked previously in the code path but adding here too 469 | compiler_bailout("The 'filecontent' attribute is not supported in this context.") 470 | else: 471 | str_payload = "%s%s" % (str_payload, self.get_file_content(content_value)) 472 | 473 | return str_payload 474 | 475 | def get_file_content(self, filepath): 476 | #we need to strip quotes from the filepath 477 | filepath = filepath.strip()[1:-1] 478 | 479 | try: 480 | fptr = open(filepath,'r') 481 | fdata = fptr.read() 482 | fptr.close() 483 | return fdata.replace('"','\"') 484 | except IOError: 485 | raise SynCompileError("File not found -- %s" % filepath) 486 | sys.exit(-1) 487 | 488 | def format_port(port): 489 | """format a port specifier""" 490 | if type(port) == int: 491 | return int(port) 492 | elif type(port) == str and port.upper() == 'ANY': 493 | #return a random port between 1024 and 65k 494 | return random.randint(1024, 65000) 495 | elif type(port) == str: 496 | try: 497 | port = int(port) 498 | return port 499 | except ValueError: 500 | raise SynSyntaxError("Invalid Syntax. %s is not a valid port" % port) 501 | 502 | def render(self, eventid): 503 | """ render a specific eventid """ 504 | 505 | event = self.timeline[eventid] 506 | pkts = [] 507 | 508 | #get the payload 509 | hasPayload = False 510 | payload = "" 511 | total_payload = self.render_payload(event) 512 | if len(total_payload) > 0: 513 | hasPayload = True 514 | 515 | # 0-len payloads are OK, but only if no payload at beginning of render() 516 | # +-len payloads are OK, but dont get processed if they are zero-sized 517 | hasIterated = False 518 | while ((len(total_payload) > 0 and hasPayload == True) or (hasPayload == False and hasIterated == False)): 519 | hasIterated = True 520 | 521 | if (hasPayload == True): 522 | #we have a payload and we are using TCP; observe the MSS 523 | if (len(total_payload) > self.tcp_mss and self.proto == Flow.PROTO_TCP): 524 | payload = total_payload[:self.tcp_mss] 525 | total_payload = total_payload[self.tcp_mss:] 526 | else: 527 | payload = total_payload 528 | total_payload = "" 529 | 530 | #figure out what the src/dst port and host are 531 | 532 | if (event['flow'] == Flow.FLOW_TO_SERVER): 533 | #preserve src/dst 534 | src_host = self.src_host 535 | src_port = int(self.src_port) 536 | src_mac = self.src_mac 537 | dst_host = self.dst_host 538 | dst_port = int(self.dst_port) 539 | dst_mac = self.dst_mac 540 | 541 | #use the clients seq/ack 542 | self.tcp_server_bytes = self.tcp_server_bytes + len(payload) 543 | tcp_seq = self.to_server_seq 544 | tcp_ack = self.to_server_ack 545 | logging.debug("*** Flow %s --> S:%s A:%s B:%s", self.name, tcp_seq, tcp_ack, self.tcp_server_bytes) 546 | logging.debug("*** %s", self.timeline[eventid]) 547 | 548 | #nooooooooooo 549 | if (len(payload) > 0): 550 | #set tcp ack to last ack 551 | tcp_ack = self.to_client_seq 552 | 553 | else: 554 | #reverse src/dst 555 | src_host = self.dst_host 556 | src_port = int(self.dst_port) 557 | src_mac = self.dst_mac 558 | dst_host = self.src_host 559 | dst_port = int(self.src_port) 560 | dst_mac = self.src_mac 561 | 562 | #use the servers seq/ack 563 | self.tcp_client_bytes = self.tcp_client_bytes + len(payload) 564 | tcp_seq = self.to_client_seq 565 | tcp_ack = self.to_client_ack 566 | logging.debug("*** Flow %s <-- S:%s A:%s B:%s", self.name, tcp_seq, tcp_ack, self.tcp_client_bytes) 567 | logging.debug("*** %s", self.timeline[eventid]) 568 | 569 | if (len(payload) > 0): 570 | tcp_ack = self.to_server_seq 571 | 572 | 573 | 574 | pkt = None 575 | logging.debug("SRC host: %s", src_host) 576 | logging.debug("DST host: %s", dst_host) 577 | lyr_ip = IP(src = src_host, dst = dst_host) 578 | lyr_eth = Ether(src = src_mac, dst = dst_mac) 579 | if (self.proto == Flow.PROTO_UDP): 580 | #generate udp packet 581 | lyr_udp = UDP(sport = src_port, dport = dst_port) / payload 582 | pkt = lyr_eth / lyr_ip / lyr_udp 583 | pkts.append(pkt) 584 | else: 585 | #generate tcp packet 586 | logging.debug("TCP Packet") 587 | 588 | #handle SEQ 589 | if 'tcp.seq' in event['attributes']: 590 | logging.debug("tcp.seq has been set manually") 591 | tcp_seq = event['attributes']['tcp.seq'] 592 | if (type(tcp_seq) == str): 593 | tcp_seq = int(tcp_seq) 594 | 595 | if 'tcp.ack' in event['attributes']: 596 | logging.debug("tcp.ack has been set manually") 597 | tcp_ack = event['attributes']['tcp.ack'] 598 | if (type(tcp_ack) == str): 599 | tcp_ack = int(tcp_ack) 600 | 601 | #check for tcp flags 602 | if 'tcp.flags.syn' in event['attributes']: 603 | flags = "S" 604 | elif 'tcp.flags.ack' in event['attributes']: 605 | flags = 'A' 606 | elif 'tcp.flags.synack' in event['attributes']: 607 | flags = 'SA' 608 | elif 'tcp.flags.rst' in event['attributes']: 609 | flags = 'R' 610 | #implied noack 611 | event['attributes']['tcp.noack'] = True 612 | else: 613 | flags = 'PA' 614 | 615 | logging.debug('Data packet with inferred flags S:%s A:%s', tcp_seq, tcp_ack) 616 | lyr_tcp = TCP(flags=flags, seq=tcp_seq, ack=tcp_ack, sport = src_port, dport = dst_port) / payload 617 | pkt = lyr_eth / lyr_ip / lyr_tcp 618 | pkts.append(pkt) 619 | 620 | # if (event['flow'] == Flow.FLOW_TO_CLIENT): 621 | # #if flow is to the client, don't increment the PSH ACK, increment the following ack.. 622 | # #increment the SEQ based on payload size 623 | # self.tcp_seq = self.tcp_ack 624 | # self.tck_ack = self.tcp_ack + len(payload) 625 | 626 | logging.debug("Payload size is: %s" % len(payload)) 627 | logging.debug("tcp_seq is %s" % tcp_seq) 628 | logging.debug("tcp_ack is %s" % tcp_ack) 629 | payload_size = len(payload) 630 | 631 | logging.debug("Moving to ACKnowledgement stage") 632 | 633 | #send an ACK 634 | if (event['flow'] == Flow.FLOW_TO_CLIENT): 635 | logging.debug('SERVER requires ACK: Flow is TO_CLIENT') 636 | #flow is SERVER -> CLIENT. Use SERVERs TCP SEQ #s 637 | logging.debug("self.to_client_seq %s" % self.to_client_seq) 638 | logging.debug("self.to_client_ack %s" % self.to_client_ack) 639 | logging.debug("len payload %s" % len(payload)) 640 | 641 | tcp_seq = tcp_ack 642 | tcp_ack = self.to_client_seq + len(payload) 643 | 644 | self.to_client_ack = self.to_client_seq + len(payload) 645 | self.to_client_seq = self.to_client_ack 646 | 647 | #trying to fix this: 648 | #tcp_ack = self.to_client_seq + len(payload) 649 | #tcp_seq = self.to_client_ack 650 | 651 | #self.to_client_ack = self.to_client_seq + len(payload) 652 | #self.to_client_seq = self.to_client_ack 653 | 654 | #previously commented out 655 | #self.tcp_server_bytes = self.tcp_server_bytes + len(payload) 656 | #self.to_server_ack = self.to_server_seq + len(payload) 657 | #tcp_seq = self.to_server_ack 658 | #tcp_seq = self.to_client_seq #None 659 | #tcp_ack = self.to_client_ack 660 | else: 661 | logging.debug('CLIENT requires ACK: Flow is TO_SERVER') 662 | #self.tcp_client_bytes = self.tcp_client_bytes + len(payload) 663 | 664 | tmp_ack = self.to_server_seq 665 | tmp_seq = self.to_server_ack 666 | 667 | #tcp_ack = self.to_server_seq #None 668 | #tcp_seq = self.to_server_ack #reversed 669 | tcp_seq = tcp_ack 670 | tcp_ack = tmp_ack + payload_size 671 | 672 | 673 | self.to_server_ack = self.to_server_seq + payload_size 674 | self.to_server_seq = self.to_server_ack 675 | 676 | if 'tcp.noack' not in event['attributes']: 677 | logging.debug('INFERRED ACK: S:%s A:%s', tcp_seq, tcp_ack) 678 | lyr_eth = Ether(src = dst_mac, dst=src_mac) 679 | lyr_ip = IP(src=dst_host, dst=src_host) 680 | lyr_tcp = TCP(sport = dst_port, dport = src_port, flags='A', seq=tcp_seq, ack=tcp_ack) 681 | pkt = lyr_eth / lyr_ip / lyr_tcp 682 | pkts.append(pkt) 683 | 684 | 685 | logging.debug("*** End Render Flow") 686 | 687 | logging.debug("**Flow State Table **") 688 | logging.debug("to_server S: %s A: %s", self.to_server_seq, self.to_server_ack) 689 | logging.debug("to_client S: %s A: %s", self.to_client_seq, self.to_client_ack) 690 | logging.debug("*********************\n\n") 691 | 692 | 693 | #print hexdump(pkt) 694 | return pkts 695 | 696 | def parse_cmd_line(): 697 | """ use ArgumentParser to parse command line arguments """ 698 | 699 | app_description = "FlowSynth v%s\nWill Urbanski \n\na tool for rapidly modeling network traffic" % APP_VERSION_STRING 700 | 701 | parser = argparse.ArgumentParser(description=app_description, formatter_class = argparse.RawTextHelpFormatter) 702 | parser.add_argument('input', help='input files') 703 | parser.add_argument('-f', dest='output_format', action='store', default="hex", 704 | help='Output format. Valid output formats include: hex, pcap') 705 | parser.add_argument('-w', dest='output_file', action='store', default="", help='Output file.') 706 | parser.add_argument('-q', dest='quiet', action='store_true', default=False, help='Run silently') 707 | parser.add_argument('-d', dest='debug', action='store_true', default=False, help='Run in debug mode') 708 | parser.add_argument('--display', dest='display', action='store', default='text', choices=['text','json'], help='Display format') 709 | parser.add_argument('--no-filecontent', dest='no_filecontent', action='store_true', default=False, help='Disable support for the filecontent attribute') 710 | 711 | args = parser.parse_args() 712 | 713 | if (args.quiet == True): 714 | LOGGING_LEVEL = logging.CRITICAL 715 | if (args.debug == True): 716 | LOGGING_LEVEL = logging.DEBUG 717 | 718 | return args 719 | 720 | def main(): 721 | """ the main function """ 722 | 723 | global ARGS 724 | global LOGGING_LEVEL 725 | global START_TIME 726 | 727 | START_TIME = time.time() 728 | 729 | ARGS = parse_cmd_line() 730 | 731 | logging.basicConfig(format='%(levelname)s: %(message)s', level=LOGGING_LEVEL) 732 | 733 | run(ARGS.input) 734 | 735 | 736 | def run(sFile): 737 | """ executes the compiler """ 738 | global BUILD_STATUS 739 | 740 | #initialize the build status 741 | BUILD_STATUS['app_version'] = APP_VERSION_STRING 742 | BUILD_STATUS['successful'] = False 743 | BUILD_STATUS['compiler'] = {} 744 | BUILD_STATUS['compiler']['start-time'] = START_TIME 745 | BUILD_STATUS['compiler']['end-time'] = -1 746 | BUILD_STATUS['compiler']['instructions'] = -1 747 | BUILD_STATUS['compiler']['events'] = -1 748 | BUILD_STATUS['compiler']['packets'] = -1 749 | 750 | #load the syn file 751 | logging.debug("Entering file loading phase") 752 | filedata = load_syn_file(sFile) 753 | BUILD_STATUS['input-file'] = sFile 754 | BUILD_STATUS['output-file'] = ARGS.output_file 755 | BUILD_STATUS['output-format'] = ARGS.output_format 756 | 757 | #process all instructions 758 | logging.debug("Entering parse phase") 759 | process_instructions(filedata) 760 | 761 | #render all instructions 762 | logging.debug("Entering render phase") 763 | render_timeline() 764 | 765 | #Output handled here 766 | #for now, print to screen 767 | logging.debug("Entering output phase") 768 | output_handler() 769 | BUILD_STATUS['compiler']['end-time'] = END_TIME 770 | BUILD_STATUS['compiler']['time'] = END_TIME - START_TIME 771 | BUILD_STATUS['compiler']['instructions'] = len(COMPILER_INSTRUCTIONS) 772 | BUILD_STATUS['compiler']['events'] = len(COMPILER_TIMELINE) 773 | BUILD_STATUS['compiler']['packets'] = len(COMPILER_OUTPUT) 774 | #ARGS.output_format, len(COMPILER_INSTRUCTIONS), len(COMPILER_TIMELINE), len(COMPILER_OUTPUT) 775 | BUILD_STATUS['successful'] = True 776 | 777 | #print the summary to the screen 778 | output_summary() 779 | 780 | def output_summary(): 781 | """print an output summary""" 782 | global RUNTIME 783 | if (ARGS.quiet == False): 784 | if (ARGS.display == 'text'): 785 | print "\n ~~ Build Summary ~~" 786 | print "Runtime:\t\t%ss\nOutput format:\t\t%s\nRaw instructions:\t%s\nTimeline events:\t%s\nPackets generated:\t%s\n" % ( RUNTIME, ARGS.output_format, len(COMPILER_INSTRUCTIONS), len(COMPILER_TIMELINE), len(COMPILER_OUTPUT)) 787 | else: 788 | print json.dumps(BUILD_STATUS) 789 | 790 | def output_handler(): 791 | """ decide what to do about output """ 792 | global ARGS 793 | global COMPILER_OUTPUT 794 | global COMPILER_TIMELINE 795 | global COMPILER_INSTRUCTIONS 796 | global START_TIME 797 | global END_TIME 798 | global RUNTIME 799 | 800 | if (ARGS.output_format == "hex"): 801 | hex_output() 802 | else: 803 | pcap_output() 804 | 805 | #print the output summary 806 | END_TIME = time.time() 807 | RUNTIME = round(END_TIME - START_TIME, 3) 808 | 809 | 810 | #has test case 811 | def pcap_output(): 812 | """ write a libpcap formatted .pcap file containing the compiler output """ 813 | global ARGS 814 | global COMPILER_OUTPUT 815 | 816 | if (len(COMPILER_OUTPUT) == 0): 817 | compiler_bailout("No output to write to disk.") 818 | 819 | if (ARGS.output_file == ""): 820 | raise SynTerminalError("No output file provided.") 821 | 822 | wrpcap(ARGS.output_file, COMPILER_OUTPUT) 823 | 824 | def hex_output(): 825 | """ produce a hexdump of the compiler output """ 826 | global COMPILER_OUTPUT 827 | for pkt in COMPILER_OUTPUT: 828 | hexdump(pkt) 829 | 830 | def render_timeline(): 831 | """ render the global and flow timelines into COMPILER_OUTPUT """ 832 | global COMPILER_TIMELINE 833 | global COMPILER_FLOWS 834 | 835 | for eventref in COMPILER_TIMELINE: 836 | flowname = eventref['flow'] 837 | eventid = eventref['event'] 838 | 839 | #have the flow render the pkt, and add it to our global output queue 840 | pkts = COMPILER_FLOWS[flowname].render(eventid) 841 | for pkt in pkts: 842 | COMPILER_OUTPUT.append(pkt) 843 | 844 | def process_instructions(instr): 845 | global COMPILER_FLOWS 846 | global COMPILER_OUTPUT 847 | global COMPILER_TIMELINE 848 | global COMPILER_INSTRUCTIONS 849 | 850 | try: 851 | lexer = FSLexer(instr) 852 | #print lexer.instructions 853 | 854 | COMPILER_INSTRUCTIONS = lexer.instructions 855 | 856 | for instr in lexer.instructions: 857 | name = instr['name'] 858 | if instr['type'] == 'flow': 859 | #check if flow exists already ? 860 | if name in COMPILER_FLOWS: 861 | logging.warning("Flow '%s' being redeclared!", name) 862 | 863 | #add the flow to the timeline 864 | add_flow(name, instr) 865 | 866 | if 'tcp.initialize' in instr['attributes']: 867 | #add tcp establishment 868 | autogen_handshake(instr) 869 | 870 | else: 871 | #add an event instead 872 | add_event(name, instr) 873 | except SynSyntaxError as e: 874 | logging.critical("Syntax Error - %s" % (e.value)) 875 | sys.exit(0) 876 | 877 | 878 | def autogen_handshake(flowdecl): 879 | """generate render events for the tcp three-way handshake""" 880 | global COMPILER_TIMELINE 881 | global COMPILER_FLOWS 882 | 883 | parent_flow = COMPILER_FLOWS[flowdecl['name']] 884 | 885 | client_isn = 10 #random.randint(10000, 99999) 886 | server_isn = 100 #random.randint(10000, 99999) 887 | 888 | #send syn 889 | eventdecl = {} 890 | eventdecl['name'] = flowdecl['name'] 891 | eventdecl['type'] = 'event' 892 | eventdecl['flow'] = Flow.FLOW_TO_SERVER 893 | eventdecl['attributes'] = {} 894 | eventdecl['attributes']['tcp.flags.syn'] = True 895 | eventdecl['attributes']['tcp.noack'] = True 896 | eventdecl['attributes']['tcp.seq'] = client_isn 897 | eventdecl['attributes']['tcp.ack'] = None 898 | add_event(flowdecl['name'], eventdecl) 899 | 900 | #send synack 901 | eventdecl = {} 902 | eventdecl['name'] = flowdecl['name'] 903 | eventdecl['type'] = 'event' 904 | eventdecl['flow'] = Flow.FLOW_TO_CLIENT 905 | eventdecl['attributes'] = {} 906 | eventdecl['attributes']['tcp.flags.synack'] = True 907 | eventdecl['attributes']['tcp.noack'] = True 908 | eventdecl['attributes']['tcp.seq'] = server_isn 909 | eventdecl['attributes']['tcp.ack'] = client_isn + 1 910 | add_event(flowdecl['name'], eventdecl) 911 | 912 | #send ack 913 | eventdecl = {} 914 | eventdecl['name'] = flowdecl['name'] 915 | eventdecl['type'] = 'event' 916 | eventdecl['flow'] = Flow.FLOW_TO_SERVER 917 | eventdecl['attributes'] = {} 918 | eventdecl['attributes']['tcp.flags.ack'] = True 919 | eventdecl['attributes']['tcp.noack'] = True 920 | eventdecl['attributes']['tcp.ack'] = server_isn + 1 921 | eventdecl['attributes']['tcp.seq'] = client_isn + 1 922 | add_event(flowdecl['name'], eventdecl) 923 | 924 | #dont set the parent flows SEQ/ACK until the end of this process 925 | parent_flow.to_server_seq = client_isn + 1 926 | parent_flow.to_client_seq = server_isn + 1 927 | parent_flow.to_server_ack = client_isn 928 | parent_flow.to_client_ack = server_isn + 1 929 | 930 | 931 | #has test case 932 | def add_flow(flowname, flowdecl): 933 | """adds a flow to the global flow manager""" 934 | global COMPILER_FLOWS 935 | logging.debug("Declaring flow %s [%s]", flowname, flowdecl) 936 | if (type(flowdecl) is not dict): 937 | compiler_bailout("Invalid flow decl passed to add_flow()") 938 | 939 | new_flow = Flow(flowdecl) 940 | COMPILER_FLOWS[flowname] = new_flow 941 | 942 | 943 | #has test case 944 | def add_event(flowname, eventdecl): 945 | """adds an event to the global timeline""" 946 | global COMPILER_FLOWS 947 | logging.debug("Declaring event in flow %s [%s]", flowname, eventdecl) 948 | 949 | if (type(eventdecl) is not dict): 950 | compiler_bailout("Invalid event decl passed to add_event()") 951 | 952 | #save the eventdecl to the flows local timeline 953 | try: 954 | COMPILER_FLOWS[flowname].timeline.append(eventdecl) 955 | except KeyError: 956 | compiler_bailout("Flow [%s] has not been instantiated." % flowname) 957 | 958 | #create an eventref for the global compiler timeline, and add it to that timeline 959 | eventref = {'flow': flowname, 'event': len(COMPILER_FLOWS[flowname].timeline)-1} 960 | COMPILER_TIMELINE.append(eventref) 961 | 962 | #has test case 963 | def load_syn_file(filename): 964 | """ loads a flowsynth file from disk and returns as a string""" 965 | try: 966 | filedata = "" 967 | fptr = open(filename,'r') 968 | filedata = fptr.read() 969 | fptr.close() 970 | except IOError: 971 | compiler_bailout("Cannot open file ('%s')", filename) 972 | 973 | return filedata 974 | 975 | #helper function to report runtime errors 976 | def compiler_bailout(msg): 977 | try: 978 | if ARGS.display == 'text': 979 | logging.critical("A unrecoverable runtime error was detected.") 980 | logging.critical(">> %s ", msg) 981 | logging.critical("Flowsynth is terminating.") 982 | raise SynTerminalError(msg) 983 | else: 984 | BUILD_STATUS['error'] = msg 985 | output_summary() 986 | sys.exit(-1) 987 | except AttributeError: 988 | raise SynTerminalError(msg) 989 | 990 | #helper function to report syntax errors 991 | def parser_bailout(msg): 992 | try: 993 | if ARGS.display == 'text': 994 | logging.critical("A unrecoverable syntax error was detected.") 995 | logging.critical(">> %s ", msg) 996 | logging.critical("Flowsynth is terminating.") 997 | raise SynSyntaxError(msg) 998 | else: 999 | BUILD_STATUS['error'] = msg 1000 | output_summary() 1001 | sys.exit(-1) 1002 | except AttributeError: 1003 | raise SynSyntaxError(msg) 1004 | 1005 | def show_build_status(): 1006 | """print the build status to screen""" 1007 | print json.dumps(BUILD_STATUS) 1008 | 1009 | #application entrypoint 1010 | if __name__ == '__main__': 1011 | main() 1012 | --------------------------------------------------------------------------------