├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── autotag.yml │ ├── codeql.yml │ ├── docker-image.yml │ └── python-package.yml ├── .gitignore ├── .gitmodules ├── CITATION.cff ├── Dockerfile ├── LICENSE ├── README.md ├── docs ├── CONTRIBUTING.md └── pull_request_template.md ├── examples ├── conn.json.log ├── conn.json.small.log ├── conn.tab.log ├── nfcapd.sample1 ├── nfcapd.sample1.labeled ├── nfcapd.sample2 ├── suricata.json ├── test.binetflow └── test5.tabs.binetflow ├── labels.config ├── netflowlabeler.py └── zeek-files-labeler.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | --- 7 | 8 | **Describe the bug** 9 | A clear and concise description of what the bug is. 10 | 11 | **To Reproduce** 12 | Steps to reproduce the behaviour if applicable. Be brief but make sure it can be reproduced. Bullet points are welcomed here. 13 | 14 | **Expected behaviour** 15 | A clear and concise description of what you expected to happen. 16 | 17 | **Screenshots** 18 | If applicable, add screenshots to help explain your problem. 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEAT]" 5 | labels: enhancement 6 | --- 7 | 8 | **Is your feature request related to a problem? Please describe.** 9 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 10 | 11 | **Describe the solution you'd like** 12 | A clear and concise description of what you want to happen. 13 | 14 | **Describe alternatives you've considered** 15 | A clear and concise description of any alternative solutions or features you've considered. 16 | 17 | **Additional context** 18 | Add any other context or screenshots about the feature request here. 19 | -------------------------------------------------------------------------------- /.github/workflows/autotag.yml: -------------------------------------------------------------------------------- 1 | name: Auto Tag 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | Patch: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | with: 12 | fetch-depth: '0' 13 | - name: Minor version for each merge 14 | id: taggerDryRun 15 | uses: anothrNick/github-tag-action@1.36.0 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | WITH_V: true 19 | DRY_RUN: true 20 | 21 | - name: echo new tag 22 | run: | 23 | echo "The next tag version will be: ${{ steps.taggerDryRun.outputs.new_tag }}" 24 | - name: echo tag 25 | run: | 26 | echo "The current tag is: ${{ steps.taggerDryRun.outputs.tag }}" 27 | - name: echo part 28 | run: | 29 | echo "The version increment was: ${{ steps.taggerDryRun.outputs.part }}" 30 | 31 | - name: Minor version for each merge 32 | id: taggerFinal 33 | uses: anothrNick/github-tag-action@1.36.0 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | WITH_V: true 37 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main", "develop" ] 20 | schedule: 21 | - cron: '34 13 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | with: 74 | category: "/language:${{matrix.language}}" 75 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | jobs: 8 | 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Docker Login 17 | uses: docker/login-action@v1 18 | with: 19 | username: ${{ secrets.DOCKER_USER }} 20 | password: ${{ secrets.DOCKER_PASSWORD }} 21 | 22 | - name: Docker meta 23 | id: docker_meta 24 | uses: docker/metadata-action@v4 25 | with: 26 | images: | 27 | ${{ secrets.DOCKER_USER }}/netflowlabeler 28 | flavor: | 29 | latest=true 30 | tags: | 31 | type=sha 32 | - name: Build and push 33 | id: docker_build 34 | uses: docker/build-push-action@v2 35 | with: 36 | context: . 37 | file: Dockerfile 38 | tags: ${{ steps.docker_meta.outputs.tags }} 39 | labels: ${{ steps.docker_meta.outputs.labels }} 40 | push: true 41 | 42 | - name: Webhook to Slack 43 | uses: 8398a7/action-slack@v3 44 | with: 45 | status: ${{ job.status }} 46 | fields: repo,message 47 | env: 48 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required 49 | if: always() # Pick up events even if the job fails or is canceled. 50 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | build: 9 | 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: ["3.9", "3.10", "3.11"] 14 | 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Set up Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@v4 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install ruff pytest 25 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 26 | - name: Lint with ruff 27 | run: | 28 | # stop the build if there are Python syntax errors or undefined names 29 | ruff --line-length 200 --format=github --select=E9,F63,F7,F82 --target-version=py37 . 30 | # default set of ruff rules with GitHub Annotations 31 | ruff --line-length 200 --format=github --target-version=py37 . 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "dataset"] 2 | path = dataset 3 | url = https://github.com/stratosphereips/security-datasets-for-testing 4 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | title: >- 3 | NetflowLabeler: A configurable rule-based labeling tool for network flow files 4 | message: 'If you use this software, please cite it as below.' 5 | type: software 6 | authors: 7 | - given-names: Sebastian 8 | family-names: Garcia 9 | email: sebastian.garcia@agents.fel.cvut.cz 10 | affiliation: >- 11 | Stratosphere Laboratory, AIC, FEL, Czech 12 | Technical University in Prague 13 | orcid: 'https://orcid.org/0000-0001-6238-9910' 14 | - given-names: Veronica 15 | family-names: Valeros 16 | email: valerver@fel.cvut.cz 17 | affiliation: >- 18 | Stratosphere Laboratory, AIC, FEL, Czech 19 | Technical University in Prague 20 | orcid: 'https://orcid.org/0000-0003-2554-3231' 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | 3 | LABEL org.opencontainers.image.authors="vero.valeros@gmail.com,eldraco@gmail.com" 4 | 5 | ENV DESTINATION_DIR /netflowlabeler 6 | 7 | COPY . ${DESTINATION_DIR}/ 8 | 9 | WORKDIR ${DESTINATION_DIR} 10 | 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NetflowLabeler 2 | [![Docker Image CI](https://github.com/stratosphereips/netflowlabeler/actions/workflows/docker-image.yml/badge.svg)](https://github.com/stratosphereips/netflowlabeler/actions/workflows/docker-image.yml) 3 | ![GitHub last commit (branch)](https://img.shields.io/github/last-commit/stratosphereips/netflowlabeler/main) 4 | ![Docker Pulls](https://img.shields.io/docker/pulls/stratosphereips/netflowlabeler?color=green) 5 | 6 | 7 | _Authors: Sebastian Garcia and Veronica Valeros, Stratosphere Laboratory, CTU in Prague_ 8 | 9 | NetflowLabeler is a Python tool to add labels to text-based network flow files. To label a netflow file, simply add the labels and conditions to a configuration file, then use this tool to assign them. The assignment of labels adheres to our own label ontology, which is structured as a customizable configuration file. Within the configuration file, you have the ability to incorporate both generic and detailed labels. Currently, the tool supports Zeek files that are delimited by TABS. However, future updates will expand its capabilities to include Zeek files in JSON and CSV formats, Argus files in CSV and TABS formats, Nfdump files in CSV format, and Suricata files in JSON format. 10 | 11 | - __netflowlabeler.py__ can label conn.log files based on a configuration file. 12 | - __zeek-files-labeler.py__ can label the rest of the Zeek log files, using the labels in the conn.log file. 13 | 14 | 15 | ## Usage 16 | 17 | To label a conn.log file from a configuration file: 18 | 19 | ```python 20 | netflowlabeler.py -c [-v ] [-d DEBUG] -f [-h] 21 | ``` 22 | To label the rest of the Zeek files using an already labeled conn.log file (conn.log.labeled): 23 | 24 | ```python 25 | zeek-files-labeler.py -l conn.log.labeled -f folder-with-zeek-log-files 26 | ``` 27 | 28 | ## Features 29 | 30 | - You can have AND and OR conditions 31 | - You can have generic labels and detailed labels 32 | - You can use negative conditions 33 | - All columns that can be interpreted as numbers can be compared with <, >, <= and >= 34 | - You can add comments in any place 35 | - You can use CIDR notation for IP ranges 36 | - You can label all the Zeek log files, by using the labels you put in the conn.log file 37 | 38 | ## Example Configuration File of Labels 39 | 40 | An example of the confguration file syntax is shown below: 41 | 42 | ```yaml 43 | Background: 44 | - srcIP=all 45 | # Here the generic label is Background and the detailed label is ARP 46 | Background, ARP: 47 | - Proto=ARP 48 | Malicious, From_Malware: 49 | - srcIP=10.0.0.34 50 | Malicious-More, From_Other_Malware: 51 | - srcIP!=10.0.0.34 & dstPort=23 52 | Malicious-HEre, From_This_Malware: 53 | - srcIP=10.0.0.34 & State=SF 54 | Malicious, From_Local_Link_IPv6: 55 | - srcIP=fe80::1dfe:6c38:93c9:c808 56 | Test-State: 57 | - srcIP=10.0.0.34 & State=S0 58 | Test-largebytes: 59 | - Bytes>=100 60 | Test-smallbytes: 61 | - Bytes<=100 62 | Benign, FromWindows: 63 | - Proto=UDP & srcIP=147.32.84.165 & dstPort=53 # (AND conditions go in one line) 64 | - Proto=TCP & dstIP=1.1.1.1 & dstPort=53 # (all new lines are OR conditions) 65 | ``` 66 | 67 | 0. The first part of the label is the generic label (Benign), after the comma is the detailed description (FromWindows). We encourage not to use : or spaces or , or TABs in the detailed description 68 | 1. If there is no |, then the detailed label is empty. 69 | 2. Don't use quotes for the text. 70 | 3. Labels are assigned from top to bottom 71 | 4. Each new label superseeds and overwrites the previous match 72 | 73 | The position is the priority of the rule. First we check the first rule matches and if it does, then we assign that label. Then we check the second rule, etc. 74 | 75 | 76 | These are the possible fields that you can use in a configuration file to create the rules used for labeling. 77 | 78 | - Date 79 | - start 80 | - Duration 81 | - Proto 82 | - srcIP 83 | - srcPort 84 | - dstIP 85 | - dstPort 86 | - State 87 | - Tos 88 | - Packets 89 | - Bytes 90 | - Flows 91 | 92 | The fields 'Bytes', 'Packets' and 'IPBytes' are computed in Zeek from the fields for the src and dst values. For example, Bytes=srcbytes + dstbytes 93 | 94 | ## Docker Image 95 | 96 | Netflow labeler has a public docker image with the latest version. 97 | 98 | To test the labeler is working correctly, run the following command. The command will run the netflow labeler tool on a Zeek example conn.log file and then cat the labeled file to the standard output. You should see the fresh labels in the output (e.g.: search for the string 'Test-smallbytes'). 99 | 100 | ```bash 101 | docker run --tty -it stratosphereips/netflowlabeler:latest /bin/bash -c 'python3 netflowlabeler.py -c labels.config -f examples/conn.tab.log ; cat examples/conn.tab.log.labeled' 102 | ``` 103 | 104 | To mount your logs path to the container and run the netflow labeler interactively: 105 | ```bash 106 | docker run -v /full/path/to/logs/:/netflowlabeler/data --rm -it stratosphereips/netflowlabeler:latest /bin/bash 107 | ``` 108 | 109 | To mount your logs path to the container and automatically run the netflow labeler on it with your own labels.config file: 110 | ```bash 111 | docker run -v /full/path/to/logs/:/netflowlabeler/data --rm -it stratosphereips/netflowlabeler:latest python3 netflowlabeler.py -c data/labels.config -f data/conn.log 112 | ``` 113 | 114 | ## Netflow Labeler High Level Diagram 115 | 116 | ```mermaid 117 | flowchart LR; 118 | NetFlow["Netflow File"]-->labeler; 119 | Config["Labels Config"]-->labeler; 120 | subgraph ONE["Interpret Input File"] 121 | labeler-->load_conditions; 122 | load_conditions-->process_netflow; 123 | process_netflow-->define_type; 124 | define_type-->define_columns; 125 | end 126 | subgraph TWO["Label NetFlow File"] 127 | define_columns-.->process_argus; 128 | define_columns-.->process_nfdump; 129 | define_columns-->process_zeek; 130 | process_argus-.->output_netflow_line_to_file; 131 | process_nfdump-.->output_netflow_line_to_file; 132 | process_zeek-->output_netflow_line_to_file; 133 | end 134 | output_netflow_line_to_file-->Output["Labeled NetFlow File"]; 135 | ``` 136 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | All contributions are welcomed, thank you for taking the time to contribute to this project! 4 | 5 | ## What branch should you base your contribution? 6 | 7 | As a general rule, base your contribution on the `develop` branch. 8 | 9 | ## Persistent Git Branches 10 | 11 | The following git branches permanent in the repository: 12 | 13 | - `main`: contains the stable version of the repository. 14 | - `develop`: all new features should be based on this branch. 15 | 16 | ## Naming Git branches for Pull Requests 17 | 18 | To keep the Git history clean and facilitate the revision of contributions we 19 | ask all branches to follow concise namings. These are the branch-naming patterns 20 | to follow when contributing: 21 | 22 | - bugfix-<>: pull request branch, contains one bugfix, 23 | - docs-<>: pull request branch, contains documentation work, 24 | - enhance-<>: pull request branch, contains one enhancement (not a new feature, but improvement nonetheless) 25 | - feature-<>: pull request branch, contains a new feature, 26 | - refactor-<>: pull request branch, contains code refactoring, 27 | 28 | ## Creating a pull request 29 | 30 | Commits: 31 | - Commits should do one thing. Keep it simple. 32 | - Commit messages should be easily readable, imperative style ("Fix memory leak in...", not "FixES mem...") 33 | 34 | Pull Requests: 35 | - If you have developed multiple features and/or bugfixes, create separate 36 | branches for each one of them, and request merges for each branch; 37 | - The cleaner you code/change/changeset is, the faster it will be merged. 38 | 39 | ## How can you contribute? 40 | 41 | * Report bugs 42 | * Suggest features and ideas 43 | * Pull requests with a solved GitHub issue and new feature 44 | * Pull request with a new content. 45 | -------------------------------------------------------------------------------- /docs/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Please include a summary of the changes and the related issue. Please also include relevant motivation and context. List any dependencies that are required for this change. 4 | 5 | Fixes # (issue) (delete if not used) 6 | 7 | ## Type of change 8 | 9 | Please delete options that are not relevant. 10 | 11 | - [ ] Bug fix (non-breaking change which fixes an issue) 12 | - [ ] New feature (non-breaking change which adds functionality) 13 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 14 | - [ ] This change requires a documentation update 15 | 16 | # How Has This Been Tested? 17 | 18 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration 19 | 20 | - [ ] Test A 21 | - [ ] Test B 22 | 23 | 24 | # Checklist: 25 | 26 | - [ ] My code follows the style guidelines of this project 27 | - [ ] I have performed a self-review of my code 28 | - [ ] I have commented my code, particularly in hard-to-understand areas 29 | - [ ] I have made corresponding changes to the documentation 30 | - [ ] My changes generate no new warnings 31 | -------------------------------------------------------------------------------- /examples/conn.json.small.log: -------------------------------------------------------------------------------- 1 | {"ts":1626284618.384979,"uid":"CwBR9S17gSfdlxZyu2","id.orig_h":"192.168.1.110","id.orig_p":5353,"id.resp_h":"192.168.1.128","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D^","orig_pkts":1,"orig_ip_bytes":370,"resp_pkts":0,"resp_ip_bytes":0} 2 | {"ts":1626284618.393387,"uid":"Cp1dsj21z1y1eh1Olc","id.orig_h":"192.168.1.102","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":262,"resp_pkts":0,"resp_ip_bytes":0} 3 | {"ts":1626284625.656299,"uid":"C3WyfA1myV4y2yDfi3","id.orig_h":"192.168.1.128","id.orig_p":59241,"id.resp_h":"172.217.23.206","id.resp_p":443,"proto":"tcp","service":"ssl","duration":0.23780393600463868,"orig_bytes":1715,"resp_bytes":8820,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFf","orig_pkts":15,"orig_ip_bytes":2519,"resp_pkts":14,"resp_ip_bytes":9556} 4 | {"ts":1626284626.546695,"uid":"Cb2qA21M8MTVghI6Zh","id.orig_h":"192.168.1.128","id.orig_p":59242,"id.resp_h":"172.217.23.206","id.resp_p":443,"proto":"tcp","service":"ssl","duration":0.2130429744720459,"orig_bytes":1696,"resp_bytes":8818,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFf","orig_pkts":15,"orig_ip_bytes":2512,"resp_pkts":14,"resp_ip_bytes":9554} 5 | {"ts":1626284626.819197,"uid":"C2h2wQ1pdBBygFi972","id.orig_h":"192.168.1.128","id.orig_p":59246,"id.resp_h":"17.253.73.202","id.resp_p":80,"proto":"tcp","service":"http","duration":0.22652912139892579,"orig_bytes":295,"resp_bytes":3312,"conn_state":"SF","missed_bytes":0,"history":"ShADadFf","orig_pkts":7,"orig_ip_bytes":671,"resp_pkts":6,"resp_ip_bytes":3632} 6 | {"ts":1626284626.842219,"uid":"Ctm7VP2JHothMKpS3d","id.orig_h":"192.168.1.128","id.orig_p":59247,"id.resp_h":"17.253.73.202","id.resp_p":80,"proto":"tcp","service":"http","duration":0.20897793769836427,"orig_bytes":295,"resp_bytes":3312,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFf","orig_pkts":9,"orig_ip_bytes":787,"resp_pkts":6,"resp_ip_bytes":3632} 7 | {"ts":1626284630.430249,"uid":"Cd0kY1AAdM385lgY","id.orig_h":"192.168.1.128","id.orig_p":59184,"id.resp_h":"8.8.8.8","id.resp_p":443,"proto":"tcp","duration":0.04250001907348633,"orig_bytes":0,"resp_bytes":0,"conn_state":"SF","missed_bytes":0,"history":"FfA","orig_pkts":2,"orig_ip_bytes":104,"resp_pkts":1,"resp_ip_bytes":52} 8 | {"ts":1626284630.430273,"uid":"C2JmSd46jr0S8mlqkh","id.orig_h":"192.168.1.128","id.orig_p":59185,"id.resp_h":"172.217.23.206","id.resp_p":443,"proto":"tcp","duration":0.04625296592712402,"orig_bytes":0,"resp_bytes":0,"conn_state":"SF","missed_bytes":0,"history":"FfA","orig_pkts":2,"orig_ip_bytes":104,"resp_pkts":1,"resp_ip_bytes":52} 9 | {"ts":1626284630.42861,"uid":"CsoNg212LGCRl5gMb","id.orig_h":"192.168.1.128","id.orig_p":59167,"id.resp_h":"142.250.185.69","id.resp_p":443,"proto":"tcp","duration":0.05095386505126953,"orig_bytes":0,"resp_bytes":0,"conn_state":"SF","missed_bytes":0,"history":"FfA","orig_pkts":2,"orig_ip_bytes":104,"resp_pkts":1,"resp_ip_bytes":52} 10 | {"ts":1626284625.594172,"uid":"CetcLe38jCknZJkKh9","id.orig_h":"192.168.1.128","id.orig_p":62076,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.050440073013305667,"orig_bytes":34,"resp_bytes":106,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":62,"resp_pkts":1,"resp_ip_bytes":134} 11 | {"ts":1626284625.594567,"uid":"CjrAPbiB8gYiDwbDe","id.orig_h":"192.168.1.128","id.orig_p":49614,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.032919883728027347,"orig_bytes":34,"resp_bytes":72,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":62,"resp_pkts":1,"resp_ip_bytes":100} 12 | {"ts":1626284625.629403,"uid":"Cw9agn26P7GdOkxvWd","id.orig_h":"192.168.1.128","id.orig_p":53486,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.052206993103027347,"orig_bytes":36,"resp_bytes":86,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":64,"resp_pkts":1,"resp_ip_bytes":114} 13 | {"ts":1626284626.580861,"uid":"CryPNE42Ilkmc7T6lh","id.orig_h":"192.168.1.128","id.orig_p":63032,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.05651497840881348,"orig_bytes":36,"resp_bytes":105,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":64,"resp_pkts":1,"resp_ip_bytes":133} 14 | {"ts":1626284626.745908,"uid":"CfIX2y2vqQNQjGx1wf","id.orig_h":"192.168.1.128","id.orig_p":54098,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.05612492561340332,"orig_bytes":32,"resp_bytes":159,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":60,"resp_pkts":1,"resp_ip_bytes":187} 15 | {"ts":1626284626.656222,"uid":"CmjPbZ3VVswR0oWZsf","id.orig_h":"192.168.1.128","id.orig_p":56575,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.038156986236572269,"orig_bytes":51,"resp_bytes":67,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":79,"resp_pkts":1,"resp_ip_bytes":95} 16 | {"ts":1626284626.655986,"uid":"CVopFJ3CAUVlGDqSCa","id.orig_h":"192.168.1.128","id.orig_p":58128,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.04579782485961914,"orig_bytes":51,"resp_bytes":108,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":79,"resp_pkts":1,"resp_ip_bytes":136} 17 | {"ts":1626284626.746144,"uid":"CoEG4b4SGicL2LZ9Jj","id.orig_h":"192.168.1.128","id.orig_p":61407,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.07118487358093262,"orig_bytes":32,"resp_bytes":137,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":60,"resp_pkts":1,"resp_ip_bytes":165} 18 | {"ts":1626284626.804596,"uid":"CD9uig1a9zqOxWyNMb","id.orig_h":"192.168.1.128","id.orig_p":53280,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.03987598419189453,"orig_bytes":38,"resp_bytes":98,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":66,"resp_pkts":1,"resp_ip_bytes":126} 19 | {"ts":1626284626.804802,"uid":"C5IEY62DfTN3KXtC92","id.orig_h":"192.168.1.128","id.orig_p":61701,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.051336050033569339,"orig_bytes":38,"resp_bytes":70,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":66,"resp_pkts":1,"resp_ip_bytes":98} 20 | {"ts":1626284627.852022,"uid":"C3RQfWHKxdZpo9RSc","id.orig_h":"192.168.1.128","id.orig_p":54806,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.04652094841003418,"orig_bytes":50,"resp_bytes":123,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":1,"resp_ip_bytes":151} 21 | {"ts":1626284627.783297,"uid":"CfqvmB1gudSoOMrYi6","id.orig_h":"192.168.1.128","id.orig_p":60012,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.04553985595703125,"orig_bytes":48,"resp_bytes":222,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":76,"resp_pkts":1,"resp_ip_bytes":250} 22 | {"ts":1626284618.273316,"uid":"CTkrLhebFMlg5hwo3","id.orig_h":"192.168.1.110","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":1.0101020336151124,"orig_bytes":306,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":362,"resp_pkts":0,"resp_ip_bytes":0} 23 | {"ts":1626284630.720847,"uid":"CrQ8CC2TqBHrySaYTc","id.orig_h":"192.168.1.128","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":68,"resp_pkts":0,"resp_ip_bytes":0} 24 | {"ts":1626284630.721145,"uid":"Cu4r1W14ATT2dVxBVe","id.orig_h":"fe80::96:f45c:832c:bbda","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":88,"resp_pkts":0,"resp_ip_bytes":0} 25 | {"ts":1626284626.703168,"uid":"CykeN91dZxGevTPqZ8","id.orig_h":"192.168.1.128","id.orig_p":59245,"id.resp_h":"172.217.23.234","id.resp_p":443,"proto":"tcp","service":"ssl","duration":13.188737154006958,"orig_bytes":1344,"resp_bytes":4418,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFf","orig_pkts":19,"orig_ip_bytes":2356,"resp_pkts":15,"resp_ip_bytes":5206} 26 | {"ts":1626284641.806115,"uid":"CMfHIv3Uz3vKZeOUO7","id.orig_h":"192.168.1.128","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":0.16997003555297852,"orig_bytes":296,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":352,"resp_pkts":0,"resp_ip_bytes":0} 27 | {"ts":1626284641.806171,"uid":"CVKUJ02LNHHo9q9sZg","id.orig_h":"fe80::96:f45c:832c:bbda","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","service":"dns","duration":0.17004895210266114,"orig_bytes":296,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":392,"resp_pkts":0,"resp_ip_bytes":0} 28 | {"ts":1626284633.099989,"uid":"CtjNPreWx3aSuCFP1","id.orig_h":"192.168.1.100","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":1.6628141403198243,"orig_bytes":1680,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":12,"orig_ip_bytes":2016,"resp_pkts":0,"resp_ip_bytes":0} 29 | {"ts":1626284647.40258,"uid":"CY5BRi1eWxCzgSxXs8","id.orig_h":"192.168.1.102","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":262,"resp_pkts":0,"resp_ip_bytes":0} 30 | {"ts":1626284647.402699,"uid":"CuLxJE3yhxMFTSPwS8","id.orig_h":"192.168.1.110","id.orig_p":5353,"id.resp_h":"192.168.1.128","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D^","orig_pkts":1,"orig_ip_bytes":370,"resp_pkts":0,"resp_ip_bytes":0} 31 | {"ts":1626284639.14219,"uid":"CPRZC57BtIRxdSD45","id.orig_h":"192.168.1.133","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":9.234570026397705,"orig_bytes":1075,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":3,"orig_ip_bytes":1159,"resp_pkts":0,"resp_ip_bytes":0} 32 | {"ts":1626284660.32993,"uid":"CNWjWV3j4uNuqTYKi6","id.orig_h":"192.168.1.128","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":103,"resp_pkts":0,"resp_ip_bytes":0} 33 | {"ts":1626284660.330001,"uid":"CdxgatVmxQudSuumj","id.orig_h":"fe80::96:f45c:832c:bbda","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":123,"resp_pkts":0,"resp_ip_bytes":0} 34 | {"ts":1626284615.92372,"uid":"CFihdA4q8zXkhZs8Rd","id.orig_h":"192.168.1.128","id.orig_p":59236,"id.resp_h":"3.123.248.34","id.resp_p":443,"proto":"tcp","duration":50.95686602592468,"orig_bytes":0,"resp_bytes":40,"conn_state":"SF","missed_bytes":0,"history":"^aAdfF","orig_pkts":9,"orig_ip_bytes":468,"resp_pkts":9,"resp_ip_bytes":507} 35 | {"ts":1626284607.091056,"uid":"CJa2A426cPB0prww1","id.orig_h":"192.168.1.128","id.orig_p":59132,"id.resp_h":"34.216.131.168","id.resp_p":443,"proto":"tcp","duration":64.34389185905457,"orig_bytes":4562,"resp_bytes":424,"conn_state":"SF","missed_bytes":0,"history":"AaDdftF","orig_pkts":16,"orig_ip_bytes":5381,"resp_pkts":12,"resp_ip_bytes":1048} 36 | {"ts":1626284647.314294,"uid":"C5orMFHm5fImeTXya","id.orig_h":"192.168.1.110","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":13.043295860290528,"orig_bytes":454,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":3,"orig_ip_bytes":538,"resp_pkts":0,"resp_ip_bytes":0} 37 | {"ts":1626284676.136344,"uid":"CyThlu1h673sOPxvZ2","id.orig_h":"192.168.1.128","id.orig_p":59259,"id.resp_h":"54.171.205.77","id.resp_p":443,"proto":"tcp","service":"ssl","duration":0.4574100971221924,"orig_bytes":1244,"resp_bytes":5712,"conn_state":"SF","missed_bytes":0,"history":"ShADadtfF","orig_pkts":14,"orig_ip_bytes":2020,"resp_pkts":13,"resp_ip_bytes":7824} 38 | {"ts":1626284673.931959,"uid":"CDatwz4UVyphIJSDZ2","id.orig_h":"192.168.1.104","id.orig_p":5353,"id.resp_h":"192.168.1.128","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D^","orig_pkts":1,"orig_ip_bytes":370,"resp_pkts":0,"resp_ip_bytes":0} 39 | {"ts":1626284674.003761,"uid":"CCsfhg3pL3FfgMdvPd","id.orig_h":"192.168.1.102","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":0.9737138748168945,"orig_bytes":468,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":524,"resp_pkts":0,"resp_ip_bytes":0} 40 | {"ts":1626284618.691165,"uid":"CgMICg3aJwTO1ixCC5","id.orig_h":"192.168.1.128","id.orig_p":59238,"id.resp_h":"3.123.248.34","id.resp_p":443,"proto":"tcp","service":"ssl","duration":61.278794050216678,"orig_bytes":1752,"resp_bytes":1255,"conn_state":"SF","missed_bytes":0,"history":"ShADdafF","orig_pkts":21,"orig_ip_bytes":2856,"resp_pkts":17,"resp_ip_bytes":2147} 41 | {"ts":1626284665.652049,"uid":"CWxuIJ1Sm8YgWhuF9k","id.orig_h":"192.168.1.128","id.orig_p":51148,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":10.421988010406495,"orig_bytes":124,"resp_bytes":115,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":4,"orig_ip_bytes":236,"resp_pkts":1,"resp_ip_bytes":143} 42 | {"ts":1626284676.076418,"uid":"CDO9db40Lc75coBWef","id.orig_h":"192.168.1.128","id.orig_p":56888,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.05761909484863281,"orig_bytes":31,"resp_bytes":63,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":59,"resp_pkts":1,"resp_ip_bytes":91} 43 | {"ts":1626284668.757146,"uid":"ClTK0uf5YCcDiEs71","id.orig_h":"192.168.1.128","id.orig_p":59869,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":1.04966402053833,"orig_bytes":62,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":118,"resp_pkts":0,"resp_ip_bytes":0} 44 | {"ts":1626284629.009101,"uid":"CvNTef11Mc9ZMwzaI4","id.orig_h":"192.168.1.128","id.orig_p":3722,"id.resp_h":"192.168.1.107","id.resp_p":3722,"proto":"udp","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":32,"resp_pkts":0,"resp_ip_bytes":0} 45 | {"ts":1626284606.734904,"uid":"CTuWFL3q7Jbt16Wj75","id.orig_h":"192.168.1.128","id.orig_p":57821,"id.resp_h":"74.125.133.189","id.resp_p":443,"proto":"udp","duration":23.693806886672975,"orig_bytes":1025,"resp_bytes":852,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":19,"orig_ip_bytes":1557,"resp_pkts":19,"resp_ip_bytes":1384} 46 | {"ts":1626284630.168172,"uid":"CfgcavVjVbjeDhVDc","id.orig_h":"192.168.1.128","id.orig_p":59247,"id.resp_h":"172.217.23.206","id.resp_p":443,"proto":"udp","duration":0.29948902130126955,"orig_bytes":8038,"resp_bytes":4584,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":21,"orig_ip_bytes":8626,"resp_pkts":18,"resp_ip_bytes":5088} 47 | {"ts":1626284630.43121,"uid":"Ckei2c1t7qxoPNXsH9","id.orig_h":"192.168.1.128","id.orig_p":3,"id.resp_h":"172.217.23.206","id.resp_p":3,"proto":"icmp","duration":0.036531925201416019,"orig_bytes":280,"resp_bytes":0,"conn_state":"OTH","missed_bytes":0,"orig_pkts":10,"orig_ip_bytes":560,"resp_pkts":0,"resp_ip_bytes":0} 48 | {"ts":1626284627.903653,"uid":"CS8xmk3TMRuwNlyjUj","id.orig_h":"192.168.1.128","id.orig_p":59249,"id.resp_h":"17.248.147.18","id.resp_p":443,"proto":"tcp","service":"ssl","duration":60.46209716796875,"orig_bytes":2187,"resp_bytes":10193,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFf","orig_pkts":17,"orig_ip_bytes":3095,"resp_pkts":14,"resp_ip_bytes":10929} 49 | {"ts":1626284674.948117,"uid":"C350Ye4QEe44AVqMb2","id.orig_h":"192.168.1.133","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":3.0401360988616945,"orig_bytes":702,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":758,"resp_pkts":0,"resp_ip_bytes":0} 50 | {"ts":1626284691.972322,"uid":"CuEoPJ1IACSTg9vJm7","id.orig_h":"192.168.1.128","id.orig_p":57752,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.21979808807373048,"orig_bytes":45,"resp_bytes":79,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":73,"resp_pkts":1,"resp_ip_bytes":107} 51 | {"ts":1626284692.011617,"uid":"CHhByY1WURiwE3jbLh","id.orig_h":"192.168.1.128","id.orig_p":61919,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":0.18278002738952638,"orig_bytes":53,"resp_bytes":114,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":81,"resp_pkts":1,"resp_ip_bytes":142} 52 | {"ts":1626284692.205171,"uid":"C0ctAN2TcJd0oJRp0k","id.orig_h":"192.168.1.128","id.orig_p":55054,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":67,"resp_pkts":0,"resp_ip_bytes":0} 53 | {"ts":1626284683.036768,"uid":"CvElBy2qgBDvaI6wgg","id.orig_h":"192.168.1.100","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":1.0076041221618653,"orig_bytes":410,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":466,"resp_pkts":0,"resp_ip_bytes":0} 54 | {"ts":1626284673.895408,"uid":"Celz2C4xkhAhNV02ba","id.orig_h":"192.168.1.104","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":12.99374008178711,"orig_bytes":712,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":6,"orig_ip_bytes":880,"resp_pkts":0,"resp_ip_bytes":0} 55 | {"ts":1626284700.355687,"uid":"CzSIkd1wvVScEceIsc","id.orig_h":"192.168.1.128","id.orig_p":59201,"id.resp_h":"172.217.23.202","id.resp_p":443,"proto":"tcp","duration":0.14493918418884278,"orig_bytes":63,"resp_bytes":0,"conn_state":"SF","missed_bytes":0,"history":"DFafAr","orig_pkts":5,"orig_ip_bytes":323,"resp_pkts":5,"resp_ip_bytes":248} 56 | {"ts":1626284697.207768,"uid":"C8PKec3RxVX6BAVTlc","id.orig_h":"192.168.1.128","id.orig_p":64898,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":67,"resp_pkts":0,"resp_ip_bytes":0} 57 | {"ts":1626284647.670608,"uid":"CVhdy62LTcwAWG2BGc","id.orig_h":"fe80::818:22f5:e0dc:c9e4","id.orig_p":143,"id.resp_h":"ff02::16","id.resp_p":0,"proto":"icmp","duration":1.0027809143066407,"orig_bytes":40,"resp_bytes":0,"conn_state":"OTH","missed_bytes":0,"orig_pkts":2,"orig_ip_bytes":152,"resp_pkts":0,"resp_ip_bytes":0} 58 | {"ts":1626284700.655897,"uid":"C0VQYcgDJCwBUxB56","id.orig_h":"192.168.1.118","id.orig_p":5353,"id.resp_h":"192.168.1.128","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D^","orig_pkts":1,"orig_ip_bytes":370,"resp_pkts":0,"resp_ip_bytes":0} 59 | {"ts":1626284702.31291,"uid":"CIVHT12D7irWFXRO7b","id.orig_h":"192.168.1.128","id.orig_p":59644,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":67,"resp_pkts":0,"resp_ip_bytes":0} 60 | {"ts":1626284707.212829,"uid":"CG1AaxWuXHcNMkSkk","id.orig_h":"192.168.1.128","id.orig_p":60448,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":1.0474469661712647,"orig_bytes":62,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":118,"resp_pkts":0,"resp_ip_bytes":0} 61 | {"ts":1626284710.325282,"uid":"C2JqwW3pKL2Ae6tJEf","id.orig_h":"192.168.1.128","id.orig_p":59260,"id.resp_h":"199.5.26.160","id.resp_p":80,"proto":"tcp","service":"http","duration":2.561079978942871,"orig_bytes":173,"resp_bytes":6792,"conn_state":"SF","missed_bytes":0,"history":"ShADadtfF","orig_pkts":11,"orig_ip_bytes":793,"resp_pkts":9,"resp_ip_bytes":7264} 62 | {"ts":1626284710.263068,"uid":"C9jcE12WydYI8Syj2l","id.orig_h":"192.168.1.128","id.orig_p":60684,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.03177499771118164,"orig_bytes":31,"resp_bytes":79,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":59,"resp_pkts":1,"resp_ip_bytes":107} 63 | {"ts":1626284700.622647,"uid":"CrHoo12LGo575cJHil","id.orig_h":"192.168.1.118","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":4.019345998764038,"orig_bytes":258,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":314,"resp_pkts":0,"resp_ip_bytes":0} 64 | {"ts":1626284717.963181,"uid":"CtE5t816KN3bszRvu4","id.orig_h":"192.168.1.128","id.orig_p":59261,"id.resp_h":"80.239.148.105","id.resp_p":443,"proto":"tcp","conn_state":"S0","missed_bytes":0,"history":"S","orig_pkts":1,"orig_ip_bytes":64,"resp_pkts":0,"resp_ip_bytes":0} 65 | {"ts":1626284718.256333,"uid":"C5ohmbWfGfFz6xdu6","id.orig_h":"192.168.1.128","id.orig_p":59262,"id.resp_h":"80.239.148.81","id.resp_p":443,"proto":"tcp","service":"ssl","duration":0.2257688045501709,"orig_bytes":1174,"resp_bytes":7826,"conn_state":"SF","missed_bytes":0,"history":"ShADadtFfRr","orig_pkts":16,"orig_ip_bytes":2030,"resp_pkts":14,"resp_ip_bytes":8550} 66 | {"ts":1626284717.901809,"uid":"ChMK4d1OcbD161HGVc","id.orig_h":"192.168.1.128","id.orig_p":53469,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":1.0467379093170167,"orig_bytes":80,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":136,"resp_pkts":0,"resp_ip_bytes":0} 67 | {"ts":1626284717.902002,"uid":"CQmcidj3aa4Qm6Xrj","id.orig_h":"192.168.1.128","id.orig_p":55753,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":1.0469717979431153,"orig_bytes":80,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":136,"resp_pkts":0,"resp_ip_bytes":0} 68 | {"ts":1626284709.182479,"uid":"CMfjca3ftHpz4v2FQf","id.orig_h":"192.168.1.128","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":4.02486515045166,"orig_bytes":360,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":4,"orig_ip_bytes":472,"resp_pkts":0,"resp_ip_bytes":0} 69 | {"ts":1626284709.183233,"uid":"Cb7wsl1T4PHvXxnagj","id.orig_h":"fe80::96:f45c:832c:bbda","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","service":"dns","duration":4.024199962615967,"orig_bytes":540,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":6,"orig_ip_bytes":828,"resp_pkts":0,"resp_ip_bytes":0} 70 | {"ts":1626284721.045973,"uid":"CN1Ord4IBRgtiUO3S9","id.orig_h":"192.168.1.128","id.orig_p":49643,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.026005029678344728,"orig_bytes":40,"resp_bytes":227,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":68,"resp_pkts":1,"resp_ip_bytes":255} 71 | {"ts":1626284721.046254,"uid":"CPkk1t2eAienlUVfBc","id.orig_h":"192.168.1.128","id.orig_p":63322,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.027816057205200197,"orig_bytes":40,"resp_bytes":310,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":68,"resp_pkts":1,"resp_ip_bytes":338} 72 | {"ts":1626284735.110152,"uid":"CH38tN3ImVxbxfYyc6","id.orig_h":"192.168.1.100","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","duration":0.024374008178710939,"orig_bytes":1035,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":8,"orig_ip_bytes":1259,"resp_pkts":0,"resp_ip_bytes":0} 73 | {"ts":1626284684.66754,"uid":"CqQKqh5Jji9PAb3Xc","id.orig_h":"192.168.1.128","id.orig_p":59663,"id.resp_h":"239.255.255.250","id.resp_p":1900,"proto":"udp","duration":3.003342866897583,"orig_bytes":700,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":4,"orig_ip_bytes":812,"resp_pkts":0,"resp_ip_bytes":0} 74 | {"ts":1626284740.732544,"uid":"C21pvhtRjwNWIv3o4","id.orig_h":"192.168.1.118","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":171,"resp_pkts":0,"resp_ip_bytes":0} 75 | {"ts":1626284749.14024,"uid":"CIsTLJ1e8hdylZ1MLj","id.orig_h":"192.168.1.128","id.orig_p":59263,"id.resp_h":"192.168.1.1","id.resp_p":1900,"proto":"tcp","service":"http","duration":0.01100301742553711,"orig_bytes":138,"resp_bytes":3703,"conn_state":"SF","missed_bytes":0,"history":"ShADadfF","orig_pkts":8,"orig_ip_bytes":566,"resp_pkts":7,"resp_ip_bytes":4075} 76 | {"ts":1626284749.213759,"uid":"C3hyoGulJaH3eHKpb","id.orig_h":"192.168.1.128","id.orig_p":59264,"id.resp_h":"192.168.1.119","id.resp_p":44444,"proto":"tcp","service":"http","duration":0.01052713394165039,"orig_bytes":147,"resp_bytes":1464,"conn_state":"SF","missed_bytes":0,"history":"ShADadfF","orig_pkts":6,"orig_ip_bytes":471,"resp_pkts":6,"resp_ip_bytes":1784} 77 | {"ts":1626284750.098141,"uid":"CabxZS2viaFlPNSUb","id.orig_h":"192.168.1.128","id.orig_p":59265,"id.resp_h":"192.168.1.133","id.resp_p":139,"proto":"tcp","duration":0.005897045135498047,"orig_bytes":0,"resp_bytes":0,"conn_state":"REJ","missed_bytes":0,"history":"Sr","orig_pkts":1,"orig_ip_bytes":64,"resp_pkts":1,"resp_ip_bytes":40} 78 | {"ts":1626284749.159896,"uid":"C35dW61kEnsyH0549e","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.1","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 79 | {"ts":1626284749.160725,"uid":"CgQLdO27oq1p6wieE8","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.119","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 80 | {"ts":1626284754.22154,"uid":"Cz2MOO1KnKwbiwMlJ6","id.orig_h":"192.168.1.128","id.orig_p":59266,"id.resp_h":"63.32.197.243","id.resp_p":443,"proto":"tcp","service":"ssl","duration":0.27279210090637209,"orig_bytes":3422,"resp_bytes":18668,"conn_state":"SF","missed_bytes":0,"history":"ShADadtfFR","orig_pkts":25,"orig_ip_bytes":4854,"resp_pkts":21,"resp_ip_bytes":21196} 81 | {"ts":1626284749.634205,"uid":"C4cKep2pybYkUkOuh3","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.100","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 82 | {"ts":1626284749.649107,"uid":"C9j4hu1SCfgsIRqLvc","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.102","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 83 | {"ts":1626284754.664047,"uid":"CGM8Zc36BYz15D47y1","id.orig_h":"192.168.1.128","id.orig_p":59268,"id.resp_h":"172.217.23.196","id.resp_p":443,"proto":"tcp","duration":4.08109188079834,"orig_bytes":0,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"S","orig_pkts":5,"orig_ip_bytes":320,"resp_pkts":0,"resp_ip_bytes":0} 84 | {"ts":1626284749.719106,"uid":"C6NfCM2tO53e2y6Pkd","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.112","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 85 | {"ts":1626284749.786304,"uid":"ClkRhd1Zzaej9YwBS7","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.103","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 86 | {"ts":1626284749.786977,"uid":"CPOIeN2iKIB70alLzd","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.110","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 87 | {"ts":1626284749.787532,"uid":"CMMdp313utKRJZSVZa","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.118","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 88 | {"ts":1626284749.787921,"uid":"CCZyBe1fD0dnkihMKg","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.131","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 89 | {"ts":1626284749.880675,"uid":"CgSZYn3ywcvOIE1Old","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.133","id.resp_p":137,"proto":"udp","service":"dns","duration":0.005272865295410156,"orig_bytes":50,"resp_bytes":121,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":1,"resp_ip_bytes":149} 90 | {"ts":1626284749.96128,"uid":"CDA32GcJ9zphVcoE5","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.107","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 91 | {"ts":1626284753.037299,"uid":"CXSOCYq3wuKo34cR8","id.orig_h":"192.168.1.128","id.orig_p":53947,"id.resp_h":"192.168.1.104","id.resp_p":137,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":78,"resp_pkts":0,"resp_ip_bytes":0} 92 | {"ts":1626284754.193478,"uid":"CL7uMJ2vTEufy3IV76","id.orig_h":"192.168.1.128","id.orig_p":60450,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.02611684799194336,"orig_bytes":29,"resp_bytes":77,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":57,"resp_pkts":1,"resp_ip_bytes":105} 93 | {"ts":1626284754.441943,"uid":"CARoJw3MRn7TylyBuj","id.orig_h":"192.168.1.128","id.orig_p":54163,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.03204703330993652,"orig_bytes":31,"resp_bytes":111,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":59,"resp_pkts":1,"resp_ip_bytes":139} 94 | {"ts":1626284754.602563,"uid":"Cr7nL94Phc8sUoj5Rb","id.orig_h":"192.168.1.128","id.orig_p":55814,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.026275157928466798,"orig_bytes":32,"resp_bytes":60,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":60,"resp_pkts":1,"resp_ip_bytes":88} 95 | {"ts":1626284754.631262,"uid":"CEiLo53GVKv9SxUvG7","id.orig_h":"192.168.1.128","id.orig_p":59318,"id.resp_h":"1.1.1.1","id.resp_p":53,"proto":"udp","service":"dns","duration":0.030751943588256837,"orig_bytes":32,"resp_bytes":48,"conn_state":"SF","missed_bytes":0,"history":"Dd","orig_pkts":1,"orig_ip_bytes":60,"resp_pkts":1,"resp_ip_bytes":76} 96 | {"ts":1626284759.761847,"uid":"CQUQit3pbMOLm6ERC2","id.orig_h":"192.168.1.128","id.orig_p":59268,"id.resp_h":"172.217.23.196","id.resp_p":443,"proto":"tcp","duration":2.0483429431915285,"orig_bytes":0,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"S","orig_pkts":2,"orig_ip_bytes":128,"resp_pkts":0,"resp_ip_bytes":0} 97 | {"ts":1626284755.504011,"uid":"CA4lMF1lI0XyA2R2y","id.orig_h":"192.168.1.128","id.orig_p":49245,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":1,"orig_ip_bytes":80,"resp_pkts":0,"resp_ip_bytes":0} 98 | {"ts":1626284757.955876,"uid":"C1ZYR24Lguyy9JeNxg","id.orig_h":"192.168.1.104","id.orig_p":5353,"id.resp_h":"192.168.1.128","id.resp_p":5353,"proto":"udp","service":"dns","conn_state":"S0","missed_bytes":0,"history":"D^","orig_pkts":1,"orig_ip_bytes":370,"resp_pkts":0,"resp_ip_bytes":0} 99 | {"ts":1626284758.620724,"uid":"C61Iqd2yzjnFZXjoe9","id.orig_h":"192.168.1.128","id.orig_p":59366,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":1.0340039730072022,"orig_bytes":88,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":2,"orig_ip_bytes":144,"resp_pkts":0,"resp_ip_bytes":0} 100 | {"ts":1626284749.107907,"uid":"Cit2bh3asmHRUFDqR2","id.orig_h":"192.168.1.128","id.orig_p":65188,"id.resp_h":"8.8.8.8","id.resp_p":53,"proto":"udp","service":"dns","duration":4.23843789100647,"orig_bytes":465,"resp_bytes":0,"conn_state":"S0","missed_bytes":0,"history":"D","orig_pkts":5,"orig_ip_bytes":605,"resp_pkts":0,"resp_ip_bytes":0} 101 | -------------------------------------------------------------------------------- /examples/conn.tab.log: -------------------------------------------------------------------------------- 1 | #separator \x09 2 | #set_separator , 3 | #empty_field (empty) 4 | #unset_field - 5 | #path conn 6 | #open 2016-10-23-20-10-44 7 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents 8 | #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 9 | 11.537757 CtQjy42V2Sc6pX6qf8 10.0.2.106 62364 8.8.8.8 53 udp dns - - - S0 - - 0 D 1 62 0 0 (empty) 10 | 12.533697 CFA2c43lsoIjrvaurg 10.0.2.106 62364 8.8.4.4 53 udp dns 0.001425 34 50 SF - - 0 Dd 1 62 1 78 (empty) 11 | 12.535643 CRXVlc2NKwlgbQJa29 10.0.2.106 62105 8.8.4.4 53 udp dns 0.001251 34 62 SF - - 0 Dd 1 62 1 90 (empty) 12 | 6.834947 CBVDtj4QMqiJgZauUk :: 135 ff02::1:ffc9:c808 136 icmp - - - - OTH - - 0 - 1 64 0 0 (empty) 13 | 6.835069 CTXTq947RkciXQljGf fe80::1dfe:6c38:93c9:c808 143 ff02::16 0 icmp - 0.500480 40 0 OTH - - 0 - 2 152 0 0 (empty) 14 | 6.835000 CBmX2XMsFIcHGnPb3 fe80::1dfe:6c38:93c9:c808 133 ff02::2 134 icmp - 8.001858 24 0 OTH - - 0 - 3 168 0 0 (empty) 15 | 6.584658 CxDPs71IU4SEnVEjM3 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.531870 588 0 S0 - - 0 D 7 924 0 0 (empty) 16 | 180.218550 CvCAkc3TRsm2WYnoS8 10.0.2.106 49158 95.163.121.33 80 tcp http 4.176622 819 493140 SF - - 0 ShADadfF 155 7031 504 513304 (empty) 17 | 245.549256 CEXhLW2VSGvNyfzKgf 10.0.2.106 49159 115.244.227.73 80 tcp - 2.993107 0 0 S0 - - 0 S 2 104 0 0 (empty) 18 | 254.551240 CWOWja2rNnpqvgU2t9 10.0.2.106 49159 115.244.227.73 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 19 | 255.713252 CSvfQ63Vk1PJrsXtSf 10.0.2.106 49160 188.165.26.237 80 tcp - 3.004384 0 0 S0 - - 0 S 2 104 0 0 (empty) 20 | 264.716423 CQq6g41UYP0l6WwYW2 10.0.2.106 49160 188.165.26.237 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 21 | 263.568237 Cm7fDk2i9DCRYUheJ3 10.0.2.106 58102 8.8.8.8 53 udp dns 0.001398 34 50 SF - - 0 Dd 1 62 1 78 (empty) 22 | 263.570062 CYOoXk4EiEZua8oVMl 10.0.2.106 51605 8.8.8.8 53 udp dns 0.001293 34 62 SF - - 0 Dd 1 62 1 90 (empty) 23 | 267.560547 CiatVc4AYkIiMGbA1l 10.0.2.106 49161 183.99.131.141 80 tcp http 2.525042 616 380 SF - - 0 ShADadfF 6 868 6 624 (empty) 24 | 271.085738 CA7jqWyJgSwoLHALb 10.0.2.106 49162 85.253.42.5 80 tcp - 3.004160 0 0 S0 - - 0 S 2 104 0 0 (empty) 25 | 277.714629 CUEt8r1YGQYSIRred8 10.0.2.106 49163 87.236.215.105 80 tcp http 1.117165 272 380 SF - - 0 ShADadfF 5 484 5 584 (empty) 26 | 278.832401 CSLvQM2EkiSqSVqF0b 10.0.2.106 49164 87.236.215.105 80 tcp http 0.710404 977 686 SF - - 0 ShADadFf 6 1229 6 930 (empty) 27 | 279.491747 CwR8oj1KFfwM6MIyq7 10.0.2.106 49165 87.236.215.105 80 tcp http 0.371674 597 574 SF - - 0 ShADadfF 6 849 6 818 (empty) 28 | 280.088254 Cs3rW2PpwjwyZi3H1 10.0.2.106 49162 85.253.42.5 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 29 | 279.864909 Ci47JE2BoBin7Q7LUg 10.0.2.106 49166 87.236.215.105 80 tcp http 1.155653 808 13902 SF - - 0 ShADadFf 11 1260 18 14626 (empty) 30 | 293.087166 CQThUS3zhS8LDe8Kwh 10.0.2.106 49167 1.164.108.76 80 tcp - 3.004161 0 0 S0 - - 0 S 2 104 0 0 (empty) 31 | 297.680227 CutZidJ2CxZCAash2 10.0.2.106 49168 87.236.215.105 80 tcp http 1.413570 721 132267 SF - - 0 ShADadfF 28 1853 120 137071 (empty) 32 | 299.094851 CqNYV249cKQuYBNEM1 10.0.2.106 49169 87.236.215.105 80 tcp http 0.704770 899 515 SF - - 0 ShADadFf 6 1151 6 759 (empty) 33 | 299.748925 Cl8UXR3BqKGfQN74Hg 10.0.2.106 49170 87.236.215.105 80 tcp http 0.577385 650 40630 SF - - 0 ShADadfF 13 1182 38 42154 (empty) 34 | 300.326677 CZskG6dm8X0HU8az2 10.0.2.106 49171 87.236.215.105 80 tcp http 0.619671 761 510 SF - - 0 ShADadfF 6 1013 6 754 (empty) 35 | 300.947527 C3E7304h0kXofxgush 10.0.2.106 49172 87.236.215.105 80 tcp http 0.365584 606 574 SF - - 0 ShADadfF 6 858 6 818 (empty) 36 | 301.313755 CxDhxeyxuih5EH3D7 10.0.2.106 49173 87.236.215.105 80 tcp http 0.612243 803 510 SF - - 0 ShADadfF 6 1055 6 754 (empty) 37 | 302.089511 CLElSU2STCP71KZXdc 10.0.2.106 49167 1.164.108.76 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 38 | 315.088813 Cj40Yl1G3R4w6pP927 10.0.2.106 49174 183.99.131.141 80 tcp http 2.502365 575 380 SF - - 0 ShADadfF 6 827 6 624 (empty) 39 | 318.583781 CFM4KK2ouVhgooe9Cb 10.0.2.106 49175 1.186.166.248 80 tcp - 3.004126 0 0 S0 - - 0 S 2 104 0 0 (empty) 40 | 327.586102 CNeFDb1QBCXc81hgZ7 10.0.2.106 49175 1.186.166.248 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 41 | 333.569261 CsVJxP22kEwT1auzZb 10.0.2.106 57658 8.8.8.8 53 udp dns 0.001361 34 50 SF - - 0 Dd 1 62 1 78 (empty) 42 | 333.571267 CUxoro1WO7bRnEpUd7 10.0.2.106 53855 8.8.8.8 53 udp dns 0.001483 34 62 SF - - 0 Dd 1 62 1 90 (empty) 43 | 340.585479 CeLNHH2GSpL4wF3jdb 10.0.2.106 49176 108.171.97.48 80 tcp - 3.004049 0 0 S0 - - 0 S 2 104 0 0 (empty) 44 | 349.588191 C0akpg1jeBgPA2O4t1 10.0.2.106 49176 108.171.97.48 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 45 | 362.587538 CEOrBV3TiyOHjfWI8h 10.0.2.106 49177 108.171.97.48 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 46 | 363.572655 C8a9fpLcwBFiPO42 10.0.2.106 57874 8.8.8.8 53 udp dns 0.001314 34 50 SF - - 0 Dd 1 62 1 78 (empty) 47 | 363.574636 CKiSU54LwFzKXU4oJh 10.0.2.106 49800 8.8.8.8 53 udp dns 0.001201 34 62 SF - - 0 Dd 1 62 1 90 (empty) 48 | 435.716244 CO2fka5ZSjtisEpyg fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.046111 588 0 S0 - - 0 D 7 924 0 0 (empty) 49 | 862.770609 CzQ5f8rOliw2AAiV1 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.015674 588 0 S0 - - 0 D 7 924 0 0 (empty) 50 | 1290.002549 CjK0pQ3tr1iq6QWalh fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.098155 588 0 S0 - - 0 D 7 924 0 0 (empty) 51 | 1416.643482 CvBMngVhDZbh94uG2 10.0.2.106 49178 87.236.215.105 80 tcp http 0.402716 510 574 SF - - 0 ShADadfF 6 762 7 858 (empty) 52 | 1417.046820 CxILcY381Dl9IkQj5c 10.0.2.106 49179 87.236.215.105 80 tcp http 0.682466 804 515 SF - - 0 ShADadFf 6 1056 6 759 (empty) 53 | 1565.446557 CBIPwn1awI4n9rsIL7 10.0.2.106 49177 108.171.97.48 80 tcp - - - - S0 - - 0 S 1 52 0 0 (empty) 54 | 1571.455193 CZ4FjT2sRYNxyIWOub 10.0.2.106 49177 108.171.97.48 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 55 | 1584.463390 CzqXtd1k1MpoMxqt97 10.0.2.106 49180 115.244.227.73 80 tcp - 2.994428 0 0 S0 - - 0 S 2 104 0 0 (empty) 56 | 1593.456336 CvCETI2W4dxjJsti04 10.0.2.106 49180 115.244.227.73 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 57 | 1606.465106 C9myMg46mXJtbGrKil 10.0.2.106 49181 134.184.203.154 80 tcp - 3.004655 0 0 S0 - - 0 S 2 104 0 0 (empty) 58 | 1615.468615 Crd3aa18LhONlc3to3 10.0.2.106 49181 134.184.203.154 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 59 | 1628.477146 CZdKu13Bph9DGPTksf 10.0.2.106 49182 117.197.225.81 80 tcp - 3.003660 0 0 S0 - - 0 S 2 104 0 0 (empty) 60 | 1624.023033 CNF4s64M9QGVaWTPHk 10.0.2.106 53478 8.8.8.8 53 udp dns 0.001215 34 50 SF - - 0 Dd 1 62 1 78 (empty) 61 | 1624.024970 CfCD101LqVODUAXnt8 10.0.2.106 62341 8.8.8.8 53 udp dns 0.001255 34 62 SF - - 0 Dd 1 62 1 90 (empty) 62 | 1637.479410 CD8tf721xOH7UmJOe9 10.0.2.106 49182 117.197.225.81 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 63 | 1643.852770 C5APl93tjUDiSvRQeg 10.0.2.106 60105 8.8.8.8 53 udp dns 0.001264 34 50 SF - - 0 Dd 1 62 1 78 (empty) 64 | 1643.854722 CFYOkf2NZYX3vhoopf 10.0.2.106 59830 8.8.8.8 53 udp dns 0.001285 34 62 SF - - 0 Dd 1 62 1 90 (empty) 65 | 1650.478916 CuZ2L82Nde85XoXJJ8 10.0.2.106 49183 134.184.203.154 80 tcp - 3.003465 0 0 S0 - - 0 S 2 104 0 0 (empty) 66 | 1659.481086 CwK2kY2JVvlPcoVNZf 10.0.2.106 49183 134.184.203.154 80 tcp - - - - S0 - - 0 S 1 48 0 0 (empty) 67 | 1672.480919 CiCSMi2xxGtCPICJl9 10.0.2.106 49184 41.150.246.15 80 tcp http 4.086724 639 380 SF - - 0 ShADadfF 6 891 7 664 (empty) 68 | 1717.184090 Cw1KG83QzQiQnebAVl fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.041185 588 0 S0 - - 0 D 7 924 0 0 (empty) 69 | 2144.233155 CGmpH21sbcZSwmbAA3 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.015711 588 0 S0 - - 0 D 7 924 0 0 (empty) 70 | 2604.355583 CEkSzUoFW2XOthEP2 10.0.2.106 49185 87.236.215.105 80 tcp http 0.366840 576 574 SF - - 0 ShADadfF 6 828 6 818 (empty) 71 | 2604.722807 CASKLe4CW2fA1AhM15 10.0.2.106 49186 87.236.215.105 80 tcp http 3.202299 735 515 SF - - 0 ShADadFf 6 987 6 759 (empty) 72 | 2571.254780 CERdwm4mi385HKPT8l fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.018465 588 0 S0 - - 0 D 7 924 0 0 (empty) 73 | 2998.280984 CCoZgD4oHciKUZK3hk fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016121 588 0 S0 - - 0 D 7 924 0 0 (empty) 74 | 3425.304446 CUZXDNojgkDcl5pr4 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016464 588 0 S0 - - 0 D 7 924 0 0 (empty) 75 | 3723.660185 CWREet4wdfweOwK6Fj 10.0.2.106 49187 87.236.215.105 80 tcp http 0.368425 614 574 SF - - 0 ShADadfF 6 866 6 818 (empty) 76 | 3724.029281 CMyHgD9T4jG6SRRaa 10.0.2.106 49188 87.236.215.105 80 tcp http 2.098410 845 759 SF - - 0 ShADadFf 6 1097 6 1003 (empty) 77 | 3852.327309 CSRU3f3ttuV8qEJYKd fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.017800 588 0 S0 - - 0 D 7 924 0 0 (empty) 78 | 4279.352861 Co83JT1pLgzPGdyAXa fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.046008 588 0 S0 - - 0 D 7 924 0 0 (empty) 79 | 4706.422512 C20Xan3dlYwwTniXve fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.020483 588 0 S0 - - 0 D 7 924 0 0 (empty) 80 | 4890.467493 CkXWs12ZbPkUa5Wume 10.0.2.106 49189 87.236.215.105 80 tcp http 0.376224 633 574 SF - - 0 ShADadfF 6 885 6 818 (empty) 81 | 4890.844602 C7GUXL1B7xbFHKGvM9 10.0.2.106 49190 87.236.215.105 80 tcp http 2.014602 851 510 SF - - 0 ShADadfF 6 1103 6 754 (empty) 82 | 5133.450771 CLGq4l3dzFdFlvxhYe fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.015737 588 0 S0 - - 0 D 7 924 0 0 (empty) 83 | 5560.474336 CJEZiW1ZB0QunDJnna fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016820 588 0 S0 - - 0 D 7 924 0 0 (empty) 84 | 5987.498391 CXjndv3ILhoV59WQck fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016225 588 0 S0 - - 0 D 7 924 0 0 (empty) 85 | 6239.377088 CPRJROOgA2f9gkwj5 10.0.2.106 49191 87.236.215.105 80 tcp http 1.358321 651 574 SF - - 0 ShADadfF 6 903 6 818 (empty) 86 | 6240.736052 C5OxnA4uQfU5Twlnrj 10.0.2.106 49192 87.236.215.105 80 tcp http 4.032684 838 751 SF - - 0 ShADadfF 7 1142 6 995 (empty) 87 | 6414.522808 CZNf1HoBXbsDIRXw4 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016002 588 0 S0 - - 0 D 7 924 0 0 (empty) 88 | 6841.546668 C30Jis4ekaTPVixRh6 fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016271 588 0 S0 - - 0 D 7 924 0 0 (empty) 89 | 7308.605122 CTl71C1CSlr7V4q3v5 10.0.2.106 49193 87.236.215.105 80 tcp http 0.367452 568 574 SF - - 0 ShADadfF 6 820 6 818 (empty) 90 | 7308.973179 C76ARt2eQaJ2hh9XVc 10.0.2.106 49194 87.236.215.105 80 tcp http 2.192804 810 752 SF - - 0 ShADadfF 6 1062 6 996 (empty) 91 | 7268.570208 CNyFZB22yikqQH7bHd fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016326 588 0 S0 - - 0 D 7 924 0 0 (empty) 92 | 7695.593209 CFpTdu1xle9JqeRvr fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.017286 588 0 S0 - - 0 D 7 924 0 0 (empty) 93 | 8122.618490 CdDU5z3HAZmVLbaaai fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016273 588 0 S0 - - 0 D 7 924 0 0 (empty) 94 | 8488.911563 CHYVHmR9xAZm0WX11 10.0.2.106 49195 87.236.215.105 80 tcp http 0.681884 611 574 SF - - 0 ShADadfF 6 863 6 818 (empty) 95 | 8489.594121 Cvdo9J3lhl2MOdHmMi 10.0.2.106 49196 87.236.215.105 80 tcp http 1.634115 836 510 SF - - 0 ShADadfF 6 1088 6 754 (empty) 96 | 8549.641804 Cbtt8xDIw9VA64ehi fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016588 588 0 S0 - - 0 D 7 924 0 0 (empty) 97 | 8976.666833 CECR5mHy2LiX2VBc fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.015479 588 0 S0 - - 0 D 7 924 0 0 (empty) 98 | 9403.690605 CCBZAC3BLaL4elgC2j fe80::1dfe:6c38:93c9:c808 546 ff02::1:2 547 udp - 63.016178 588 0 S0 - - 0 D 7 924 0 0 (empty) 99 | 9702.917586 C0TiVtxjzA9fo0VY 10.0.2.106 49197 87.236.215.105 80 tcp http 0.362385 621 574 SF - - 0 ShADadfF 6 873 6 818 (empty) 100 | 9703.280650 CoPuCK3MJgmA9Q988d 10.0.2.106 49198 87.236.215.105 80 tcp http 0.640246 746 750 SF - - 0 ShADadfF 6 998 6 994 (empty) 101 | -------------------------------------------------------------------------------- /examples/nfcapd.sample1: -------------------------------------------------------------------------------- 1 | Date flow start Duration Proto Src IP Addr:Port Dst IP Addr:Port Flags Tos Packets Bytes Flows 2 | 2013-07-09 01:00:23.040 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:49058 ...... 0 1 126 1 3 | 2013-07-09 01:00:23.040 0.028 UDP 10.0.0.150:49058 -> 8.8.8.8:53 ...... 0 2 102 1 4 | 2013-07-09 01:00:23.069 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:52189 ...... 0 1 126 1 5 | 2013-07-09 01:00:23.069 0.028 UDP 10.0.0.150:52189 -> 8.8.8.8:53 ...... 0 2 102 1 6 | 2013-07-09 01:00:28.651 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:55485 ...... 0 1 127 1 7 | 2013-07-09 01:00:28.651 0.027 UDP 10.0.0.150:55485 -> 8.8.8.8:53 ...... 0 2 104 1 8 | 2013-07-09 01:00:28.733 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:47086 ...... 0 1 127 1 9 | 2013-07-09 01:00:28.733 0.028 UDP 10.0.0.150:47086 -> 8.8.8.8:53 ...... 0 2 104 1 10 | 2013-07-09 01:00:28.790 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:56748 ...... 0 1 127 1 11 | 2013-07-09 01:00:28.790 0.027 UDP 10.0.0.150:56748 -> 8.8.8.8:53 ...... 0 2 104 1 12 | 2013-07-09 01:00:28.852 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:41973 ...... 0 1 127 1 13 | 2013-07-09 01:00:28.852 0.027 UDP 10.0.0.150:41973 -> 8.8.8.8:53 ...... 0 2 104 1 14 | 2013-07-09 01:00:26.553 4.781 IGMP 10.0.0.150:0 -> 224.0.0.22:0 ...... 0 4 160 1 15 | 2013-07-09 01:00:26.833 5.155 UDP 10.0.0.150:5353 -> 224.0.0.251:5353 ...... 0 24 5042 1 16 | 2013-07-09 01:03:32.240 0.101 UDP 10.0.0.151:51138 -> 224.0.0.252:5355 ...... 0 4 200 1 17 | 2013-07-09 01:03:34.797 0.097 UDP 10.0.0.151:52928 -> 224.0.0.252:5355 ...... 0 4 200 1 18 | 2013-07-09 01:03:32.541 4.055 UDP 10.0.0.151:137 -> 10.0.0.255:137 ...... 0 12 936 1 19 | 2013-07-09 01:03:37.353 0.145 TCP 10.0.0.150:3128 -> 10.0.0.151:49157 .AP.SF 0 6 539 1 20 | 2013-07-09 01:03:37.353 0.145 TCP 10.0.0.151:49157 -> 10.0.0.150:3128 .AP.SF 0 6 372 1 21 | 2013-07-09 01:03:54.799 0.145 UDP 10.0.0.151:52929 -> 239.255.255.250:3702 ...... 0 4 4068 1 22 | 2013-07-09 01:03:54.789 0.215 IGMP 10.0.0.151:0 -> 224.0.0.22:0 ...... 0 4 160 1 23 | 2013-07-09 01:03:55.024 0.150 UDP 10.0.0.151:52931 -> 239.255.255.250:3702 ...... 0 4 2608 1 24 | 2013-07-09 01:04:25.259 0.302 UDP 10.0.0.152:57993 -> 239.255.255.250:3702 ...... 0 8 8140 1 25 | 2013-07-09 01:04:25.230 0.471 IGMP 10.0.0.152:0 -> 224.0.0.22:0 ...... 0 4 160 1 26 | 2013-07-09 01:04:25.731 0.099 UDP 10.0.0.152:57997 -> 239.255.255.250:3702 ...... 0 4 2736 1 27 | 2013-07-09 01:04:25.740 0.199 UDP 10.0.0.151:3702 -> 10.0.0.152:57997 ...... 0 2 2522 1 28 | 2013-07-09 01:04:25.852 0.137 TCP 10.0.0.151:5357 -> 10.0.0.152:49697 .AP.SF 0 4 2524 1 29 | 2013-07-09 01:04:25.852 0.137 TCP 10.0.0.152:49697 -> 10.0.0.151:5357 .AP.SF 0 6 1203 1 30 | 2013-07-09 01:04:27.510 0.164 UDP 10.0.0.152:57999 -> 239.255.255.250:3702 ...... 0 4 2608 1 31 | 2013-07-09 01:04:28.761 0.095 UDP 10.0.0.152:56214 -> 224.0.0.252:5355 ...... 0 4 200 1 32 | 2013-07-09 01:04:29.845 0.240 UDP 10.0.0.151:3702 -> 10.0.0.152:57999 ...... 0 2 2502 1 33 | 2013-07-09 01:04:31.313 0.097 UDP 10.0.0.152:50728 -> 224.0.0.252:5355 ...... 0 4 200 1 34 | 2013-07-09 01:04:29.059 4.053 UDP 10.0.0.152:137 -> 10.0.0.255:137 ...... 0 12 936 1 35 | 2013-07-09 01:04:33.868 0.097 TCP 10.0.0.150:3128 -> 10.0.0.152:49700 .AP.SF 0 6 539 1 36 | 2013-07-09 01:04:33.868 0.097 TCP 10.0.0.152:49700 -> 10.0.0.150:3128 .AP.SF 0 5 332 1 37 | -------------------------------------------------------------------------------- /examples/nfcapd.sample1.labeled: -------------------------------------------------------------------------------- 1 | Date flow start Duration Proto Src IP Addr:Port Dst IP Addr:Port Flags Tos Packets Bytes Flows Label 2 | 2013-07-09 01:00:23.040 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:49058 ...... 0 1 126 1 Background 3 | 2013-07-09 01:00:23.040 0.028 UDP 10.0.0.150:49058 -> 8.8.8.8:53 ...... 0 2 102 1 Background 4 | 2013-07-09 01:00:23.069 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:52189 ...... 0 1 126 1 Background 5 | 2013-07-09 01:00:23.069 0.028 UDP 10.0.0.150:52189 -> 8.8.8.8:53 ...... 0 2 102 1 Background 6 | 2013-07-09 01:00:28.651 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:55485 ...... 0 1 127 1 Background 7 | 2013-07-09 01:00:28.651 0.027 UDP 10.0.0.150:55485 -> 8.8.8.8:53 ...... 0 2 104 1 Background 8 | 2013-07-09 01:00:28.733 0.028 UDP 8.8.8.8:53 -> 10.0.0.150:47086 ...... 0 1 127 1 Background 9 | 2013-07-09 01:00:28.733 0.028 UDP 10.0.0.150:47086 -> 8.8.8.8:53 ...... 0 2 104 1 Background 10 | 2013-07-09 01:00:28.790 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:56748 ...... 0 1 127 1 Background 11 | 2013-07-09 01:00:28.790 0.027 UDP 10.0.0.150:56748 -> 8.8.8.8:53 ...... 0 2 104 1 Background 12 | 2013-07-09 01:00:28.852 0.027 UDP 8.8.8.8:53 -> 10.0.0.150:41973 ...... 0 1 127 1 Background 13 | 2013-07-09 01:00:28.852 0.027 UDP 10.0.0.150:41973 -> 8.8.8.8:53 ...... 0 2 104 1 Background 14 | 2013-07-09 01:00:26.553 4.781 IGMP 10.0.0.150:0:0 -> 224.0.0.22:0:0 ...... 0 4 160 1 Background 15 | 2013-07-09 01:00:26.833 5.155 UDP 10.0.0.150:5353 -> 224.0.0.251:5353 ...... 0 24 5042 1 Background 16 | 2013-07-09 01:03:32.240 0.101 UDP 10.0.0.151:51138 -> 224.0.0.252:5355 ...... 0 4 200 1 Background 17 | 2013-07-09 01:03:34.797 0.097 UDP 10.0.0.151:52928 -> 224.0.0.252:5355 ...... 0 4 200 1 Background 18 | 2013-07-09 01:03:32.541 4.055 UDP 10.0.0.151:137 -> 10.0.0.255:137 ...... 0 12 936 1 Background 19 | 2013-07-09 01:03:37.353 0.145 TCP 10.0.0.150:3128 -> 10.0.0.151:49157 .AP.SF 0 6 539 1 Background 20 | 2013-07-09 01:03:37.353 0.145 TCP 10.0.0.151:49157 -> 10.0.0.150:3128 .AP.SF 0 6 372 1 Background 21 | 2013-07-09 01:03:54.799 0.145 UDP 10.0.0.151:52929 -> 239.255.255.250:3702 ...... 0 4 4068 1 Background 22 | 2013-07-09 01:03:54.789 0.215 IGMP 10.0.0.151:0:0 -> 224.0.0.22:0:0 ...... 0 4 160 1 Background 23 | 2013-07-09 01:03:55.024 0.150 UDP 10.0.0.151:52931 -> 239.255.255.250:3702 ...... 0 4 2608 1 Background 24 | 2013-07-09 01:04:25.259 0.302 UDP 10.0.0.152:57993 -> 239.255.255.250:3702 ...... 0 8 8140 1 Background 25 | 2013-07-09 01:04:25.230 0.471 IGMP 10.0.0.152:0:0 -> 224.0.0.22:0:0 ...... 0 4 160 1 Background 26 | 2013-07-09 01:04:25.731 0.099 UDP 10.0.0.152:57997 -> 239.255.255.250:3702 ...... 0 4 2736 1 Background 27 | 2013-07-09 01:04:25.740 0.199 UDP 10.0.0.151:3702 -> 10.0.0.152:57997 ...... 0 2 2522 1 Test-Label 28 | 2013-07-09 01:04:25.852 0.137 TCP 10.0.0.151:5357 -> 10.0.0.152:49697 .AP.SF 0 4 2524 1 Test-Label 29 | 2013-07-09 01:04:25.852 0.137 TCP 10.0.0.152:49697 -> 10.0.0.151:5357 .AP.SF 0 6 1203 1 Background 30 | 2013-07-09 01:04:27.510 0.164 UDP 10.0.0.152:57999 -> 239.255.255.250:3702 ...... 0 4 2608 1 Background 31 | 2013-07-09 01:04:28.761 0.095 UDP 10.0.0.152:56214 -> 224.0.0.252:5355 ...... 0 4 200 1 Background 32 | 2013-07-09 01:04:29.845 0.240 UDP 10.0.0.151:3702 -> 10.0.0.152:57999 ...... 0 2 2502 1 Test-Label 33 | 2013-07-09 01:04:31.313 0.097 UDP 10.0.0.152:50728 -> 224.0.0.252:5355 ...... 0 4 200 1 Background 34 | 2013-07-09 01:04:29.059 4.053 UDP 10.0.0.152:137 -> 10.0.0.255:137 ...... 0 12 936 1 Background 35 | 2013-07-09 01:04:33.868 0.097 TCP 10.0.0.150:3128 -> 10.0.0.152:49700 .AP.SF 0 6 539 1 Test-Label 36 | 2013-07-09 01:04:33.868 0.097 TCP 10.0.0.152:49700 -> 10.0.0.150:3128 .AP.SF 0 5 332 1 Background 37 | -------------------------------------------------------------------------------- /examples/nfcapd.sample2: -------------------------------------------------------------------------------- 1 | Date flow start Duration Proto Src IP Addr:Port Dst IP Addr:Port Flags Tos Packets Bytes Flows 2 | 2013-07-09 01:15:33.005 0.548 TCP 10.0.0.151:49161 -> 10.0.0.150:3128 .APRS. 0 5 333 1 3 | 2013-07-09 01:15:35.189 7.719 TCP 10.0.0.150:3128 -> 10.0.0.151:49162 .AP.S. 0 9 2816 1 4 | 2013-07-09 01:15:35.189 7.719 TCP 10.0.0.151:49162 -> 10.0.0.150:3128 .APRS. 0 8 763 1 5 | 2013-07-09 01:15:33.045 10.173 TCP 10.0.0.150:39476 -> 54.242.92.108:80 .APRSF 0 28 2998 1 6 | 2013-07-09 01:15:33.045 10.173 TCP 54.242.92.108:80 -> 10.0.0.150:39476 .AP.SF 0 11 5697 1 7 | 2013-07-09 01:15:42.909 9.216 TCP 10.0.0.150:3128 -> 10.0.0.151:49163 .AP.S. 0 6 1152 1 8 | 2013-07-09 01:15:42.909 9.216 TCP 10.0.0.151:49163 -> 10.0.0.150:3128 .APRS. 0 6 558 1 9 | 2013-07-09 01:15:52.687 0.230 TCP 10.0.0.150:3128 -> 10.0.0.151:49164 .AP.S. 0 5 2533 1 10 | 2013-07-09 01:15:52.687 0.230 TCP 10.0.0.151:49164 -> 10.0.0.150:3128 .APRS. 0 5 333 1 11 | 2013-07-09 01:15:52.918 10.024 TCP 10.0.0.150:3128 -> 10.0.0.151:49165 .AP.S. 0 10 3638 1 12 | 2013-07-09 01:15:52.918 10.024 TCP 10.0.0.151:49165 -> 10.0.0.150:3128 .APRS. 0 8 763 1 13 | 2013-07-09 01:16:02.943 0.946 TCP 10.0.0.150:3128 -> 10.0.0.151:49166 .AP.S. 0 6 1153 1 14 | 2013-07-09 01:16:02.943 0.946 TCP 10.0.0.151:49166 -> 10.0.0.150:3128 .APRS. 0 6 558 1 15 | 2013-07-09 01:16:04.630 0.043 TCP 10.0.0.150:3128 -> 10.0.0.151:49167 .AP.SF 0 6 2143 1 16 | 2013-07-09 01:16:04.630 0.043 TCP 10.0.0.151:49167 -> 10.0.0.150:3128 .AP.SF 0 6 488 1 17 | 2013-07-09 01:16:12.706 0.227 TCP 10.0.0.150:3128 -> 10.0.0.151:49168 .AP.S. 0 5 2533 1 18 | 2013-07-09 01:16:12.706 0.227 TCP 10.0.0.151:49168 -> 10.0.0.150:3128 .APRS. 0 5 333 1 19 | 2013-07-09 01:16:12.934 10.005 TCP 10.0.0.150:3128 -> 10.0.0.151:49169 .AP.S. 0 10 3638 1 20 | 2013-07-09 01:16:12.934 10.005 TCP 10.0.0.151:49169 -> 10.0.0.150:3128 .APRS. 0 8 763 1 21 | 2013-07-09 01:16:25.137 0.257 TCP 10.0.0.150:3128 -> 10.0.0.152:49702 .AP.S. 0 5 2533 1 22 | 2013-07-09 01:16:25.137 0.257 TCP 10.0.0.152:49702 -> 10.0.0.150:3128 .APRS. 0 5 333 1 23 | 2013-07-09 01:16:22.939 10.005 TCP 10.0.0.150:3128 -> 10.0.0.151:49170 .AP.S. 0 10 3634 1 24 | 2013-07-09 01:16:22.939 10.005 TCP 10.0.0.151:49170 -> 10.0.0.150:3128 .APRS. 0 8 759 1 25 | 2013-07-09 01:16:26.511 8.577 TCP 10.0.0.150:3128 -> 10.0.0.152:49703 .AP.S. 0 10 3638 1 26 | 2013-07-09 01:16:26.511 8.577 TCP 10.0.0.152:49703 -> 10.0.0.150:3128 .APRS. 0 8 763 1 27 | 2013-07-09 01:16:35.089 5.899 TCP 10.0.0.150:3128 -> 10.0.0.152:49704 .AP.S. 0 6 1157 1 28 | 2013-07-09 01:16:35.089 5.899 TCP 10.0.0.152:49704 -> 10.0.0.150:3128 .APRS. 0 6 562 1 29 | 2013-07-09 01:16:32.946 10.002 TCP 10.0.0.150:3128 -> 10.0.0.151:49171 .AP.S. 0 10 3638 1 30 | 2013-07-09 01:16:32.946 10.002 TCP 10.0.0.151:49171 -> 10.0.0.150:3128 .APRS. 0 9 803 1 31 | 2013-07-09 01:16:44.871 0.234 TCP 10.0.0.150:3128 -> 10.0.0.152:49705 .AP.S. 0 4 1718 1 32 | 2013-07-09 01:16:44.871 0.234 TCP 10.0.0.152:49705 -> 10.0.0.150:3128 .APRS. 0 5 333 1 33 | 2013-07-09 01:16:45.105 5.197 TCP 10.0.0.150:3128 -> 10.0.0.152:49706 .AP.S. 0 6 1157 1 34 | 2013-07-09 01:16:45.105 5.197 TCP 10.0.0.152:49706 -> 10.0.0.150:3128 .APRS. 0 6 562 1 35 | 2013-07-09 01:16:50.974 0.052 TCP 10.0.0.150:3128 -> 10.0.0.152:49707 .AP.SF 0 6 2143 1 36 | 2013-07-09 01:16:50.974 0.052 TCP 10.0.0.152:49707 -> 10.0.0.150:3128 .AP.SF 0 6 488 1 37 | 2013-07-09 01:16:42.949 10.006 TCP 10.0.0.150:3128 -> 10.0.0.151:49172 .AP.S. 0 10 3634 1 38 | 2013-07-09 01:16:42.949 10.006 TCP 10.0.0.151:49172 -> 10.0.0.150:3128 .APRS. 0 9 799 1 39 | 2013-07-09 01:16:54.869 0.237 TCP 10.0.0.150:3128 -> 10.0.0.152:49708 .AP.S. 0 5 2533 1 40 | 2013-07-09 01:16:54.869 0.237 TCP 10.0.0.152:49708 -> 10.0.0.150:3128 .APRS. 0 5 333 1 41 | 2013-07-09 01:16:52.955 9.998 TCP 10.0.0.150:3128 -> 10.0.0.151:49173 .AP.S. 0 10 3638 1 42 | 2013-07-09 01:16:52.955 9.998 TCP 10.0.0.151:49173 -> 10.0.0.150:3128 .APRS. 0 8 763 1 43 | 2013-07-09 01:16:55.107 9.996 TCP 10.0.0.150:3128 -> 10.0.0.152:49709 .AP.S. 0 10 3638 1 44 | 2013-07-09 01:16:55.107 9.996 TCP 10.0.0.152:49709 -> 10.0.0.150:3128 .APRS. 0 9 803 1 45 | 2013-07-09 01:17:02.954 10.007 TCP 10.0.0.150:3128 -> 10.0.0.151:49174 .AP.S. 0 10 3634 1 46 | 2013-07-09 01:17:02.954 10.007 TCP 10.0.0.151:49174 -> 10.0.0.150:3128 .APRS. 0 8 759 1 47 | 2013-07-09 01:17:05.104 9.996 TCP 10.0.0.150:3128 -> 10.0.0.152:49710 .AP.S. 0 10 3638 1 48 | 2013-07-09 01:17:05.104 9.996 TCP 10.0.0.152:49710 -> 10.0.0.150:3128 .APRS. 0 8 763 1 49 | 2013-07-09 01:17:12.962 10.014 TCP 10.0.0.150:3128 -> 10.0.0.151:49175 .AP.S. 0 10 3638 1 50 | 2013-07-09 01:17:12.962 10.014 TCP 10.0.0.151:49175 -> 10.0.0.150:3128 .APRS. 0 8 763 1 51 | 2013-07-09 01:17:15.101 10.020 TCP 10.0.0.150:3128 -> 10.0.0.152:49711 .AP.S. 0 10 3639 1 52 | 2013-07-09 01:17:15.101 10.020 TCP 10.0.0.152:49711 -> 10.0.0.150:3128 .APRS. 0 8 763 1 53 | 2013-07-09 01:17:26.007 0.252 TCP 10.0.0.150:3128 -> 10.0.0.153:49705 .AP.S. 0 5 2533 1 54 | 2013-07-09 01:17:26.007 0.252 TCP 10.0.0.153:49705 -> 10.0.0.150:3128 .APRS. 0 5 333 1 55 | 2013-07-09 01:17:22.977 10.003 TCP 10.0.0.150:3128 -> 10.0.0.151:49176 .AP.S. 0 10 3635 1 56 | 2013-07-09 01:17:22.977 10.003 TCP 10.0.0.151:49176 -> 10.0.0.150:3128 .APRS. 0 8 759 1 57 | 2013-07-09 01:17:25.122 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49712 .AP.S. 0 10 3639 1 58 | 2013-07-09 01:17:25.122 10.004 TCP 10.0.0.152:49712 -> 10.0.0.150:3128 .APRS. 0 9 803 1 59 | 2013-07-09 01:17:27.275 8.673 TCP 10.0.0.150:3128 -> 10.0.0.153:49706 .AP.S. 0 10 3639 1 60 | 2013-07-09 01:17:27.275 8.673 TCP 10.0.0.153:49706 -> 10.0.0.150:3128 .APRS. 0 8 763 1 61 | 2013-07-09 01:17:35.949 4.807 TCP 10.0.0.150:3128 -> 10.0.0.153:49707 .AP.S. 0 6 1158 1 62 | 2013-07-09 01:17:35.949 4.807 TCP 10.0.0.153:49707 -> 10.0.0.150:3128 .APRS. 0 6 562 1 63 | 2013-07-09 01:17:32.981 9.992 TCP 10.0.0.150:3128 -> 10.0.0.151:49177 .AP.S. 0 10 3639 1 64 | 2013-07-09 01:17:32.981 9.992 TCP 10.0.0.151:49177 -> 10.0.0.150:3128 .APRS. 0 9 803 1 65 | 2013-07-09 01:17:35.127 10.003 TCP 10.0.0.150:3128 -> 10.0.0.152:49713 .AP.S. 0 10 3639 1 66 | 2013-07-09 01:17:35.127 10.003 TCP 10.0.0.152:49713 -> 10.0.0.150:3128 .APRS. 0 8 763 1 67 | 2013-07-09 01:17:45.732 0.228 TCP 10.0.0.150:3128 -> 10.0.0.153:49708 .AP.S. 0 5 2533 1 68 | 2013-07-09 01:17:45.732 0.228 TCP 10.0.0.153:49708 -> 10.0.0.150:3128 .APRS. 0 6 373 1 69 | 2013-07-09 01:17:45.960 3.654 TCP 10.0.0.150:3128 -> 10.0.0.153:49709 .AP.S. 0 6 1158 1 70 | 2013-07-09 01:17:45.960 3.654 TCP 10.0.0.153:49709 -> 10.0.0.150:3128 .APRS. 0 6 562 1 71 | 2013-07-09 01:17:50.213 0.030 TCP 10.0.0.150:3128 -> 10.0.0.153:49710 .AP.SF 0 6 2143 1 72 | 2013-07-09 01:17:50.213 0.030 TCP 10.0.0.153:49710 -> 10.0.0.150:3128 .AP.SF 0 6 488 1 73 | 2013-07-09 01:17:42.975 10.008 TCP 10.0.0.150:3128 -> 10.0.0.151:49178 .AP.S. 0 10 3635 1 74 | 2013-07-09 01:17:42.975 10.008 TCP 10.0.0.151:49178 -> 10.0.0.150:3128 .APRS. 0 8 759 1 75 | 2013-07-09 01:17:45.130 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49714 .AP.S. 0 10 3639 1 76 | 2013-07-09 01:17:45.130 10.004 TCP 10.0.0.152:49714 -> 10.0.0.150:3128 .APRS. 0 8 763 1 77 | 2013-07-09 01:17:55.728 0.230 TCP 10.0.0.150:3128 -> 10.0.0.153:49711 .AP.S. 0 5 2533 1 78 | 2013-07-09 01:17:55.728 0.230 TCP 10.0.0.153:49711 -> 10.0.0.150:3128 .APRS. 0 5 333 1 79 | 2013-07-09 01:17:52.984 10.011 TCP 10.0.0.150:3128 -> 10.0.0.151:49179 .AP.S. 0 10 3639 1 80 | 2013-07-09 01:17:52.984 10.011 TCP 10.0.0.151:49179 -> 10.0.0.150:3128 .APRS. 0 8 763 1 81 | 2013-07-09 01:17:55.135 10.001 TCP 10.0.0.150:3128 -> 10.0.0.152:49715 .AP.S. 0 9 2824 1 82 | 2013-07-09 01:17:55.135 10.001 TCP 10.0.0.152:49715 -> 10.0.0.150:3128 .APRSF 0 8 763 1 83 | 2013-07-09 01:17:55.959 10.006 TCP 10.0.0.150:3128 -> 10.0.0.153:49712 .AP.S. 0 10 3639 1 84 | 2013-07-09 01:17:55.959 10.006 TCP 10.0.0.153:49712 -> 10.0.0.150:3128 .APRS. 0 8 763 1 85 | 2013-07-09 01:18:02.996 9.997 TCP 10.0.0.150:3128 -> 10.0.0.151:49180 .AP.S. 0 10 3635 1 86 | 2013-07-09 01:18:02.996 9.997 TCP 10.0.0.151:49180 -> 10.0.0.150:3128 .APRS. 0 9 799 1 87 | 2013-07-09 01:18:05.137 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49716 .AP.S. 0 10 3639 1 88 | 2013-07-09 01:18:05.137 10.004 TCP 10.0.0.152:49716 -> 10.0.0.150:3128 .APRS. 0 8 763 1 89 | 2013-07-09 01:18:05.966 10.000 TCP 10.0.0.150:3128 -> 10.0.0.153:49713 .AP.S. 0 10 3639 1 90 | 2013-07-09 01:18:05.966 10.000 TCP 10.0.0.153:49713 -> 10.0.0.150:3128 .APRS. 0 8 763 1 91 | 2013-07-09 01:18:12.994 10.007 TCP 10.0.0.150:3128 -> 10.0.0.151:49181 .AP.S. 0 10 3639 1 92 | 2013-07-09 01:18:12.994 10.007 TCP 10.0.0.151:49181 -> 10.0.0.150:3128 .APRS. 0 9 803 1 93 | 2013-07-09 01:18:15.142 10.000 TCP 10.0.0.150:3128 -> 10.0.0.152:49717 .AP.S. 0 10 3639 1 94 | 2013-07-09 01:18:15.142 10.000 TCP 10.0.0.152:49717 -> 10.0.0.150:3128 .APRS. 0 9 803 1 95 | 2013-07-09 01:18:15.967 10.003 TCP 10.0.0.150:3128 -> 10.0.0.153:49714 .AP.S. 0 10 3639 1 96 | 2013-07-09 01:18:15.967 10.003 TCP 10.0.0.153:49714 -> 10.0.0.150:3128 .APRS. 0 8 763 1 97 | 2013-07-09 01:18:23.002 9.999 TCP 10.0.0.150:3128 -> 10.0.0.151:49182 .AP.S. 0 10 3635 1 98 | 2013-07-09 01:18:23.002 9.999 TCP 10.0.0.151:49182 -> 10.0.0.150:3128 .APRS. 0 9 799 1 99 | 2013-07-09 01:18:25.143 10.008 TCP 10.0.0.150:3128 -> 10.0.0.152:49718 .AP.S. 0 10 3639 1 100 | 2013-07-09 01:18:25.143 10.008 TCP 10.0.0.152:49718 -> 10.0.0.150:3128 .APRS. 0 8 763 1 101 | 2013-07-09 01:18:25.971 10.004 TCP 10.0.0.150:3128 -> 10.0.0.153:49715 .AP.S. 0 10 3639 1 102 | 2013-07-09 01:18:25.971 10.004 TCP 10.0.0.153:49715 -> 10.0.0.150:3128 .APRS. 0 8 763 1 103 | 2013-07-09 01:18:33.001 10.011 TCP 10.0.0.150:3128 -> 10.0.0.151:49183 .AP.S. 0 10 3639 1 104 | 2013-07-09 01:18:33.001 10.011 TCP 10.0.0.151:49183 -> 10.0.0.150:3128 .APRS. 0 9 803 1 105 | 2013-07-09 01:18:35.152 9.998 TCP 10.0.0.150:3128 -> 10.0.0.152:49719 .AP.S. 0 10 3639 1 106 | 2013-07-09 01:18:35.152 9.998 TCP 10.0.0.152:49719 -> 10.0.0.150:3128 .APRS. 0 8 763 1 107 | 2013-07-09 01:18:35.976 10.005 TCP 10.0.0.150:3128 -> 10.0.0.153:49716 .AP.S. 0 10 3639 1 108 | 2013-07-09 01:18:35.976 10.005 TCP 10.0.0.153:49716 -> 10.0.0.150:3128 .APRS. 0 9 803 1 109 | 2013-07-09 01:18:43.013 9.996 TCP 10.0.0.150:3128 -> 10.0.0.151:49184 .AP.S. 0 10 3635 1 110 | 2013-07-09 01:18:43.013 9.996 TCP 10.0.0.151:49184 -> 10.0.0.150:3128 .APRS. 0 8 759 1 111 | 2013-07-09 01:18:45.151 10.009 TCP 10.0.0.150:3128 -> 10.0.0.152:49720 .AP.S. 0 10 3639 1 112 | 2013-07-09 01:18:45.151 10.009 TCP 10.0.0.152:49720 -> 10.0.0.150:3128 .APRS. 0 9 803 1 113 | 2013-07-09 01:18:45.982 10.002 TCP 10.0.0.150:3128 -> 10.0.0.153:49717 .AP.S. 0 10 3639 1 114 | 2013-07-09 01:18:45.982 10.002 TCP 10.0.0.153:49717 -> 10.0.0.150:3128 .APRS. 0 9 803 1 115 | 2013-07-09 01:18:53.011 10.017 TCP 10.0.0.150:3128 -> 10.0.0.151:49185 .AP.S. 0 10 3639 1 116 | 2013-07-09 01:18:53.011 10.017 TCP 10.0.0.151:49185 -> 10.0.0.150:3128 .APRS. 0 9 803 1 117 | 2013-07-09 01:18:55.161 10.441 TCP 10.0.0.150:3128 -> 10.0.0.152:49721 .AP.S. 0 9 2825 1 118 | 2013-07-09 01:18:55.161 10.441 TCP 10.0.0.152:49721 -> 10.0.0.150:3128 .APRS. 0 8 763 1 119 | 2013-07-09 01:15:42.910 202.914 TCP 10.0.0.150:56243 -> 54.242.92.108:80 .APRSF 0 480 64300 1 120 | 2013-07-09 01:15:42.910 202.914 TCP 54.242.92.108:80 -> 10.0.0.150:56243 .AP.SF 0 149 121835 1 121 | 2013-07-09 01:18:55.984 10.017 TCP 10.0.0.150:3128 -> 10.0.0.153:49718 .AP.S. 0 10 3639 1 122 | 2013-07-09 01:18:55.984 10.017 TCP 10.0.0.153:49718 -> 10.0.0.150:3128 .APRS. 0 8 763 1 123 | 2013-07-09 01:19:03.028 9.890 TCP 10.0.0.150:3128 -> 10.0.0.151:49186 .AP.S. 0 10 3639 1 124 | 2013-07-09 01:19:03.028 9.890 TCP 10.0.0.151:49186 -> 10.0.0.150:3128 .APRS. 0 8 763 1 125 | 2013-07-09 01:19:05.603 9.460 TCP 10.0.0.150:3128 -> 10.0.0.152:49722 .AP.S. 0 10 3639 1 126 | 2013-07-09 01:19:05.603 9.460 TCP 10.0.0.152:49722 -> 10.0.0.150:3128 .APRS. 0 9 803 1 127 | 2013-07-09 01:19:06.002 9.891 TCP 10.0.0.150:3128 -> 10.0.0.153:49719 .AP.S. 0 10 3639 1 128 | 2013-07-09 01:19:06.002 9.891 TCP 10.0.0.153:49719 -> 10.0.0.150:3128 .APRS. 0 9 803 1 129 | 2013-07-09 01:19:05.637 15.711 TCP 10.0.0.150:47993 -> 54.242.92.108:80 .AP.SF 0 10 1004 1 130 | 2013-07-09 01:19:05.637 15.711 TCP 54.242.92.108:80 -> 10.0.0.150:47993 .AP.SF 0 5 481 1 131 | 2013-07-09 01:19:12.919 10.003 TCP 10.0.0.150:3128 -> 10.0.0.151:49187 .AP.S. 0 10 3639 1 132 | 2013-07-09 01:19:12.919 10.003 TCP 10.0.0.151:49187 -> 10.0.0.150:3128 .APRS. 0 8 763 1 133 | 2013-07-09 01:19:15.064 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49723 .AP.S. 0 10 3639 1 134 | 2013-07-09 01:19:15.064 10.004 TCP 10.0.0.152:49723 -> 10.0.0.150:3128 .APRS. 0 9 803 1 135 | 2013-07-09 01:19:15.893 9.999 TCP 10.0.0.150:3128 -> 10.0.0.153:49720 .AP.S. 0 10 3639 1 136 | 2013-07-09 01:19:15.893 9.999 TCP 10.0.0.153:49720 -> 10.0.0.150:3128 .APRS. 0 9 803 1 137 | 2013-07-09 01:19:22.923 10.004 TCP 10.0.0.150:3128 -> 10.0.0.151:49188 .AP.S. 0 10 3639 1 138 | 2013-07-09 01:19:22.923 10.004 TCP 10.0.0.151:49188 -> 10.0.0.150:3128 .APRS. 0 9 803 1 139 | 2013-07-09 01:19:25.069 10.003 TCP 10.0.0.150:3128 -> 10.0.0.152:49724 .AP.S. 0 10 3639 1 140 | 2013-07-09 01:19:25.069 10.003 TCP 10.0.0.152:49724 -> 10.0.0.150:3128 .APRS. 0 9 803 1 141 | 2013-07-09 01:19:25.892 10.005 TCP 10.0.0.150:3128 -> 10.0.0.153:49721 .AP.S. 0 10 3639 1 142 | 2013-07-09 01:19:25.892 10.005 TCP 10.0.0.153:49721 -> 10.0.0.150:3128 .APRS. 0 9 803 1 143 | 2013-07-09 01:19:32.928 10.008 TCP 10.0.0.150:3128 -> 10.0.0.151:49189 .AP.S. 0 10 3639 1 144 | 2013-07-09 01:19:32.928 10.008 TCP 10.0.0.151:49189 -> 10.0.0.150:3128 .APRS. 0 8 763 1 145 | 2013-07-09 01:19:35.073 10.003 TCP 10.0.0.150:3128 -> 10.0.0.152:49725 .AP.S. 0 10 3639 1 146 | 2013-07-09 01:19:35.073 10.003 TCP 10.0.0.152:49725 -> 10.0.0.150:3128 .APRS. 0 8 763 1 147 | 2013-07-09 01:19:35.898 10.002 TCP 10.0.0.150:3128 -> 10.0.0.153:49722 .AP.S. 0 10 3639 1 148 | 2013-07-09 01:19:35.898 10.002 TCP 10.0.0.153:49722 -> 10.0.0.150:3128 .APRS. 0 8 763 1 149 | 2013-07-09 01:19:42.937 9.999 TCP 10.0.0.150:3128 -> 10.0.0.151:49190 .AP.S. 0 10 3639 1 150 | 2013-07-09 01:19:42.937 9.999 TCP 10.0.0.151:49190 -> 10.0.0.150:3128 .APRS. 0 8 763 1 151 | 2013-07-09 01:19:45.077 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49726 .AP.S. 0 10 3639 1 152 | 2013-07-09 01:19:45.077 10.004 TCP 10.0.0.152:49726 -> 10.0.0.150:3128 .APRS. 0 9 803 1 153 | 2013-07-09 01:19:45.901 10.003 TCP 10.0.0.150:3128 -> 10.0.0.153:49723 .AP.S. 0 10 3639 1 154 | 2013-07-09 01:19:45.901 10.003 TCP 10.0.0.153:49723 -> 10.0.0.150:3128 .APRS. 0 9 803 1 155 | 2013-07-09 01:19:52.937 10.002 TCP 10.0.0.150:3128 -> 10.0.0.151:49191 .AP.S. 0 10 3639 1 156 | 2013-07-09 01:19:52.937 10.002 TCP 10.0.0.151:49191 -> 10.0.0.150:3128 .APRS. 0 9 803 1 157 | 2013-07-09 01:19:55.082 10.004 TCP 10.0.0.150:3128 -> 10.0.0.152:49727 .AP.S. 0 10 3639 1 158 | 2013-07-09 01:19:55.082 10.004 TCP 10.0.0.152:49727 -> 10.0.0.150:3128 .APRS. 0 8 763 1 159 | 2013-07-09 01:19:55.905 10.006 TCP 10.0.0.150:3128 -> 10.0.0.153:49724 .AP.S. 0 10 3639 1 160 | 2013-07-09 01:19:55.905 10.006 TCP 10.0.0.153:49724 -> 10.0.0.150:3128 .APRS. 0 8 763 1 161 | 2013-07-09 01:20:02.940 10.004 TCP 10.0.0.150:3128 -> 10.0.0.151:49192 .AP.S. 0 10 3639 1 162 | 2013-07-09 01:20:02.940 10.004 TCP 10.0.0.151:49192 -> 10.0.0.150:3128 .APRS. 0 8 763 1 163 | 2013-07-09 01:20:05.087 10.003 TCP 10.0.0.150:3128 -> 10.0.0.152:49728 .AP.S. 0 10 3639 1 164 | 2013-07-09 01:20:05.087 10.003 TCP 10.0.0.152:49728 -> 10.0.0.150:3128 .APRS. 0 8 763 1 165 | 2013-07-09 01:20:05.912 10.001 TCP 10.0.0.150:3128 -> 10.0.0.153:49725 .AP.S. 0 10 3639 1 166 | 2013-07-09 01:20:05.912 10.001 TCP 10.0.0.153:49725 -> 10.0.0.150:3128 .APRS. 0 8 763 1 167 | 2013-07-09 01:20:12.945 10.004 TCP 10.0.0.150:3128 -> 10.0.0.151:49193 .AP.S. 0 10 3639 1 168 | 2013-07-09 01:20:12.945 10.004 TCP 10.0.0.151:49193 -> 10.0.0.150:3128 .APRSF 0 9 803 1 169 | -------------------------------------------------------------------------------- /examples/suricata.json: -------------------------------------------------------------------------------- 1 | {"timestamp":"2021-06-06T15:59:46.457984+0200","flow_id":1374195536821504,"pcap_cnt":313,"event_type":"alert","src_ip":"78.36.196.213","src_port":51698,"dest_ip":"192.168.1.129","dest_port":445,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403388,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 89","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T15:59:46.457984+0200"},"host":"stratosphere.org"} 2 | {"timestamp":"2021-06-06T15:58:31.793839+0200","flow_id":1406059894217967,"pcap_cnt":149,"event_type":"alert","src_ip":"45.143.200.10","src_port":8080,"dest_ip":"192.168.1.129","dest_port":16804,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403335,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 36","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T15:58:31.793839+0200"},"host":"stratosphere.org"} 3 | {"timestamp":"2021-06-06T16:05:22.830309+0200","flow_id":590295307823973,"pcap_cnt":1015,"event_type":"alert","src_ip":"51.79.215.126","src_port":45329,"dest_ip":"192.168.1.129","dest_port":3389,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403359,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 60","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:05:22.830309+0200"},"host":"stratosphere.org"} 4 | {"timestamp":"2021-06-06T16:08:12.413136+0200","flow_id":1255701717208528,"pcap_cnt":1414,"event_type":"alert","src_ip":"193.27.229.184","src_port":40401,"dest_ip":"192.168.1.129","dest_port":33891,"proto":"TCP","metadata":{"flowbits":["ET.Evil","ET.DshieldIP"]},"alert":{"action":"allowed","gid":1,"signature_id":2402000,"rev":5934,"signature":"ET DROP Dshield Block Listed Source group 1","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2010_12_30"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["Dshield"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:08:12.413136+0200"},"host":"stratosphere.org"} 5 | {"timestamp":"2021-06-06T16:13:15.892979+0200","flow_id":512792653963315,"pcap_cnt":2059,"event_type":"alert","src_ip":"45.143.203.12","src_port":42498,"dest_ip":"192.168.1.129","dest_port":61000,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403335,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 36","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:13:15.892979+0200"},"host":"stratosphere.org"} 6 | {"timestamp":"2021-06-06T16:11:52.265714+0200","flow_id":1882792726629874,"pcap_cnt":1839,"event_type":"alert","src_ip":"87.103.208.30","src_port":45919,"dest_ip":"192.168.1.129","dest_port":1433,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2010935,"rev":3,"signature":"ET SCAN Suspicious inbound to MSSQL port 1433","category":"Potentially Bad Traffic","severity":2,"metadata":{"created_at":["2010_07_30"],"former_category":["HUNTING"],"updated_at":["2018_03_27"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:11:52.265714+0200"},"host":"stratosphere.org"} 7 | {"timestamp":"2021-06-06T16:10:07.671554+0200","flow_id":1269467094925122,"pcap_cnt":1656,"event_type":"alert","src_ip":"5.101.0.194","src_port":45892,"dest_ip":"192.168.1.129","dest_port":4470,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403304,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 5","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:10:07.671554+0200"},"host":"stratosphere.org"} 8 | {"timestamp":"2021-06-06T16:14:38.245904+0200","flow_id":1791859689963664,"pcap_cnt":2247,"event_type":"alert","src_ip":"92.63.197.71","src_port":45810,"dest_ip":"192.168.1.129","dest_port":3320,"proto":"TCP","metadata":{"flowbits":["ET.Evil","ET.DshieldIP"]},"alert":{"action":"allowed","gid":1,"signature_id":2402000,"rev":5934,"signature":"ET DROP Dshield Block Listed Source group 1","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2010_12_30"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["Dshield"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:14:38.245904+0200"},"host":"stratosphere.org"} 9 | {"timestamp":"2021-06-06T16:11:52.265714+0200","flow_id":1882792726629874,"pcap_cnt":1839,"event_type":"alert","src_ip":"87.103.208.30","src_port":45919,"dest_ip":"192.168.1.129","dest_port":1433,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403394,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 95","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:11:52.265714+0200"},"host":"stratosphere.org"} 10 | {"timestamp":"2021-06-06T16:11:52.265719+0200","flow_id":1882792726629874,"pcap_cnt":1840,"event_type":"alert","src_ip":"87.103.208.30","src_port":45919,"dest_ip":"192.168.1.129","dest_port":1433,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2010935,"rev":3,"signature":"ET SCAN Suspicious inbound to MSSQL port 1433","category":"Potentially Bad Traffic","severity":2,"metadata":{"created_at":["2010_07_30"],"former_category":["HUNTING"],"updated_at":["2018_03_27"]}},"flow":{"pkts_toserver":2,"pkts_toclient":0,"bytes_toserver":120,"bytes_toclient":0,"start":"2021-06-06T16:11:52.265714+0200"},"host":"stratosphere.org"} 11 | {"timestamp":"2021-06-06T16:14:29.534418+0200","flow_id":720720615516050,"pcap_cnt":2221,"event_type":"alert","src_ip":"74.120.14.27","src_port":40648,"dest_ip":"192.168.1.129","dest_port":50995,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403385,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 86","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:14:29.534418+0200"},"host":"stratosphere.org"} 12 | {"timestamp":"2021-06-06T16:17:41.491928+0200","flow_id":473137238296542,"pcap_cnt":2719,"event_type":"alert","src_ip":"175.143.54.101","src_port":53084,"dest_ip":"192.168.1.129","dest_port":22,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2001219,"rev":20,"signature":"ET SCAN Potential SSH Scan","category":"Attempted Information Leak","severity":2,"metadata":{"created_at":["2010_07_30"],"updated_at":["2010_07_30"]}},"flow":{"pkts_toserver":5,"pkts_toclient":4,"bytes_toserver":326,"bytes_toclient":240,"start":"2021-06-06T16:17:40.051166+0200"},"host":"stratosphere.org"} 13 | {"timestamp":"2021-06-06T16:19:29.350408+0200","flow_id":1982191184730312,"pcap_cnt":2982,"event_type":"alert","src_ip":"65.49.20.101","src_port":50519,"dest_ip":"192.168.1.129","dest_port":50075,"proto":"TCP","alert":{"action":"allowed","gid":1,"signature_id":2403379,"rev":66418,"signature":"ET CINS Active Threat Intelligence Poor Reputation IP group 80","category":"Misc Attack","severity":2,"metadata":{"affected_product":["Any"],"attack_target":["Any"],"created_at":["2013_10_08"],"deployment":["Perimeter"],"signature_severity":["Major"],"tag":["CINS"],"updated_at":["2021_06_07"]}},"flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":60,"bytes_toclient":0,"start":"2021-06-06T16:19:29.350408+0200"},"host":"stratosphere.org"} 14 | -------------------------------------------------------------------------------- /labels.config: -------------------------------------------------------------------------------- 1 | ################################################################################################################# 2 | # This is the labels configuration file of the netflowlabeler tool 3 | # Example Usage 4 | # 5 | # Benign, FromWindows: 6 | # - Proto=UDP & srcIP=147.32.84.165 & dstPort=53 # (AND conditions go in one line) 7 | # - Proto=TCP & dstIP=1.1.1.1 & dstPort=53 # (all new lines are OR conditions) 8 | # 9 | # 0. The first part of the label is the generic label (Benign), after the comma is the detailed description (FromWindows). We encourage not to use : or spaces or , or TABs in the detailed description 10 | # 1. If there is no |, then the detailed label is empty. 11 | # 2. Don't use quotes for the text. 12 | # 3. Labels are assigned from top to bottom 13 | # 4. Each new label superseeds and overwrites the previous match 14 | # 15 | # This are the possible fields that you can use in a configuration file to create the rules used for labeling. 16 | # Date 17 | # start 18 | # Duration 19 | # Proto 20 | # srcIP 21 | # srcPort 22 | # dstIP 23 | # dstPort 24 | # State 25 | # Tos 26 | # Packets 27 | # Bytes 28 | # Flows 29 | ################################################################################################################# 30 | # Background labels should go first 31 | Background: 32 | - srcIP=all 33 | Background, ARP: 34 | - Proto=ARP 35 | # Malicious labels should go next 36 | # No detailed label 37 | Malicious, From_Malware: 38 | - srcIP=10.0.0.34 39 | Malicious, From_Local_Link_IPv6: 40 | - srcIP=fe80::1dfe:6c38:93c9:c808 41 | Malicious, From-Benign-DNS-Server: 42 | - Proto=UDP & dstIP=147.32.84.192 & srcPort=53 43 | Malicious, IPv6-Web: 44 | - dstIP=2a00:1450:400c:c05::64 & dstPort=80 45 | Malicious, DNS-Access: 46 | - srcIP=10.0.0.34 & dstIP=8.8.8.8 & dstPort=53 47 | Test-Negative: 48 | - srcIP!=fe80::2acf:e9ff:fe17:3079 & dstPort=5353 49 | Test-Negative2: 50 | - srcIP=fe80::2acf:e9ff:fe17:3079 & dstPort!=5354 51 | Test-State: 52 | - srcIP=10.0.0.34 & State=S0 53 | Test-largebytes: 54 | - Bytes>=100 55 | Test-smallbytes: 56 | - Bytes<=100 57 | # Normal labels go last 58 | Benign, FromInfectedComputer: 59 | - srcIP=fe80::50be:8acf:be49:8c14 & dstIP=ff02::1:2 60 | Benign, Windows: 61 | - srcIP=77.67.96.222 62 | - dstIP=77.67.96.222 63 | Benign, Unknown: 64 | - dstIP=2a00:1450:400c:c05::69 & dstPort=443 65 | 66 | -------------------------------------------------------------------------------- /netflowlabeler.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # Copyright (C) 2009 Sebastian Garcia, Veronica Valeros 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 | # 18 | # 19 | # Authors: 20 | # Sebastian Garcia, sebastian.garcia@agents.fel.cvut.cz, eldraco@gmail.com 21 | # Veronica Valeros, valerver@fel.cvut.cz, vero.valeros@gmail.com 22 | # Stratosphere Laboratory, AIC, FEL, Czech Technical University in Prague 23 | 24 | # Description 25 | # A tool to add labels in netflow files based on a configuration. 26 | # Flow file include Zeek, Argus, and NFdump. Both in CSV and TSV 27 | 28 | """ 29 | netflowlabeler.py is a tool to add labels in netflow files based on a 30 | configuration file. 31 | """ 32 | import sys 33 | import json 34 | import argparse 35 | import ipaddress 36 | 37 | version = "0.5" 38 | 39 | 40 | class labeler(): 41 | """ 42 | This class handles the adding of new labeling conditions 43 | and the return of the labels 44 | 45 | conditionsGroup = [ 46 | {'Background': [ 47 | [ {'srcIP': 'all'} ] 48 | ] }, 49 | {'Normal': [ 50 | [ {'Proto':'IGMP'} ], 51 | [ {'Proto':'ARP'} ] 52 | ] }, 53 | {'Botnet-CC': [ 54 | [ {'srcIP': '10.0.0.151'} ], 55 | [ {'dstIP': '10.0.0.151'} ] 56 | ] }, 57 | {'Botnet-SPAM': [ 58 | [ {'Proto': 'TCP'}, {'srcPort': '25'} ], 59 | [ {'Proto': 'TCP'}, {'dstPort': '25'} ] 60 | ] }, 61 | {'Botnet-DGA': [ 62 | [ {'Proto':'UDP'}, {'srcPort':'53'} ] , 63 | [ {'Proto':'UDP'}, {'dstPort':'53'} ] 64 | ] } 65 | ] 66 | """ 67 | conditionsGroup = [] 68 | 69 | def addCondition(self, condition): 70 | """ 71 | Add a condition. 72 | Input: condition is a string? 73 | """ 74 | try: 75 | self.conditionsGroup.append(condition) 76 | 77 | if args.debug > 0: 78 | print(f'\tCondition added: {condition}') 79 | 80 | except Exception as inst: 81 | print('[!] Error in class labeler addCondition(): unable to add a condition') 82 | print(type(inst)) # the exception instance 83 | print(inst.args) # arguments stored in .args 84 | print(inst) # __str__ allows args to printed directly 85 | sys.exit(-1) 86 | 87 | def getLabel(self, column_values): 88 | """ 89 | Get the values of the columns of a netflow line, 90 | matche the labels conditions, and return a label. 91 | 92 | Input: 93 | - column_values is a dict, where each key is the standard field in a netflow 94 | Output: 95 | - labelToReturn: return a tuple containing a generic and detailed label 96 | """ 97 | try: 98 | # Default to empty genericlabel and detailedlabel 99 | labelToReturn = ("(empty)", "(empty)") 100 | 101 | # Process all the conditions 102 | for group in self.conditionsGroup: 103 | # The first key of the group is the label to put 104 | # Example: {'Botnet-SPAM': [[{'Proto': 'TCP'}, {'srcPort': '25'}], [{'Proto': 'TCP'}, {'dstPort': '25'}]]} 105 | labelline = list(group.keys())[0] 106 | genericlabelToVerify = labelline.split(',')[0].strip() 107 | 108 | # The detailed label may not be there, try to obtain it 109 | try: 110 | detailedlabelToVerify = labelline.split(',')[1].strip() 111 | except IndexError: 112 | # There is no detailedlabel 113 | detailedlabelToVerify = '(empty)' 114 | 115 | if args.debug > 0: 116 | print(f'\tLabel to verify {labelline}: {genericlabelToVerify} {detailedlabelToVerify}') 117 | 118 | orConditions = group[labelline] 119 | if args.debug > 0: 120 | print(f'\t\tOr conditions group : {orConditions}') 121 | 122 | # orConditions is an array. 123 | # Each position of this array should be ORed with the next position 124 | for andcondition in orConditions: 125 | # If any of these andConditions groups is true, 126 | # just return the label, because this for is an 'OR' 127 | if args.debug > 0: 128 | print(f'\t\tAnd condition group : {andcondition}') 129 | 130 | # With this we keep control of how each part of the and is going... 131 | allTrue = True 132 | for acond in andcondition: 133 | if args.debug > 0: 134 | print(f'\t\t\tAnd this with : {acond}') 135 | 136 | condColumn = list(acond.keys())[0] 137 | condValue = acond[condColumn].lower() 138 | condColumn = condColumn.lower() 139 | 140 | if condColumn.find('!') != -1: 141 | # Negation condition 142 | condColumn = condColumn.replace('!', '') 143 | netflowValue = column_values[condColumn] 144 | if args.debug > 0: 145 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 146 | 147 | if (condValue != netflowValue) or (condValue == 'all'): 148 | allTrue = True 149 | if args.debug > 0: 150 | print('\t\t\tTrue (negative)') 151 | continue 152 | else: 153 | if args.debug > 0: 154 | print('\t\t\tFalse (negative)') 155 | allTrue = False 156 | break 157 | elif condColumn.find('!') == -1: 158 | # Normal condition, no negation 159 | 160 | # Is the column a number? 161 | # if ('bytes' in condColumn) or ('packets' in condColumn) or ('srcport' in condColumn) or 162 | # ('dstport' in condColumn) or ('sbytes' in condColumn) or ('dbyets' in condColumn) or 163 | # ('spkts' in condColumn) or ('dpkts' in condColumn) or ('ip_orig_bytes' in condColumn) or 164 | # ('ip_resp_bytes' in condColumn): 165 | column_num_keywords = ['bytes', 'packets', 'srcport', 'dstport', 166 | 'sbytes', 'dbytes', 'spkts', 'dpkts', 167 | 'ip_orig_bytes', 'ip_resp_bytes'] 168 | if any(keyword in condColumn for keyword in column_num_keywords): 169 | # It is a colum that we can treat as a number 170 | # Find if there is <, > or = in the condition 171 | if '>' in condColumn[-1]: 172 | condColumn = condColumn[:-1] 173 | netflowValue = column_values[condColumn] 174 | if args.debug > 0: 175 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 176 | 177 | # Pay attention to directionality of condition 'condValue < flowvalue' 178 | if (int(condValue) < int(netflowValue)) or (condValue == 'all'): 179 | allTrue = True 180 | if args.debug > 0: 181 | print('\t\t\tTrue') 182 | continue 183 | else: 184 | if args.debug > 0: 185 | print('\t\t\tFalse') 186 | allTrue = False 187 | break 188 | elif '<' in condColumn[-1]: 189 | condColumn = condColumn[:-1] 190 | netflowValue = column_values[condColumn] 191 | if args.debug > 0: 192 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 193 | # Pay attention to directionality of condition 'condValue > flowvalue' 194 | if (int(condValue) > int(netflowValue)) or (condValue == 'all'): 195 | allTrue = True 196 | if args.debug > 0: 197 | print('\t\t\tTrue') 198 | continue 199 | else: 200 | if args.debug > 0: 201 | print('\t\t\tFalse') 202 | allTrue = False 203 | break 204 | elif '<=' in condColumn[-2]: 205 | condColumn = condColumn[:-2] 206 | netflowValue = column_values[condColumn] 207 | if args.debug > 0: 208 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 209 | # Pay attention to directionality of condition 'condValue >= flowvalue' 210 | if (int(condValue) >= int(netflowValue)) or (condValue == 'all'): 211 | allTrue = True 212 | if args.debug > 0: 213 | print('\t\t\tTrue') 214 | continue 215 | else: 216 | if args.debug > 0: 217 | print('\t\t\tFalse') 218 | allTrue = False 219 | break 220 | elif '>=' in condColumn[-2]: 221 | condColumn = condColumn[:-2] 222 | netflowValue = column_values[condColumn] 223 | if args.debug > 0: 224 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 225 | # Pay attention to directionality of condition 'condValue <= flowvalue' 226 | if (int(condValue) <= int(netflowValue)) or (condValue == 'all'): 227 | allTrue = True 228 | if args.debug > 0: 229 | print('\t\t\tTrue') 230 | continue 231 | else: 232 | if args.debug > 0: 233 | print('\t\t\tFalse') 234 | allTrue = False 235 | break 236 | else: 237 | netflowValue = column_values[condColumn] 238 | if args.debug > 0: 239 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 240 | if (int(condValue) == int(netflowValue)) or (condValue == 'all'): 241 | allTrue = True 242 | if args.debug > 0: 243 | print('\t\t\tTrue') 244 | continue 245 | else: 246 | if args.debug > 0: 247 | print('\t\t\tFalse') 248 | allTrue = False 249 | break 250 | 251 | # Is the column related to an IP and the value has a / for the CIDR? 252 | elif ('ip' in condColumn) and ('/' in condValue): 253 | netflowValue = column_values[condColumn] 254 | if ':' in condValue: 255 | temp_net = ipaddress.IPv6Network(condValue) 256 | else: 257 | temp_net = ipaddress.IPv4Network(condValue) 258 | temp_ip = ipaddress.ip_address(netflowValue) 259 | if temp_ip in temp_net: 260 | if args.debug > 0: 261 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 262 | allTrue = True 263 | if args.debug > 0: 264 | print('\t\t\tTrue') 265 | continue 266 | else: 267 | if args.debug > 0: 268 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 269 | if args.debug > 0: 270 | print('\t\t\tFalse') 271 | allTrue = False 272 | break 273 | 274 | # It is not a colum that we can treat as a number 275 | else: 276 | netflowValue = column_values[condColumn] 277 | if (condValue == netflowValue) or (condValue == 'all'): 278 | netflowValue = column_values[condColumn] 279 | if args.debug > 0: 280 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 281 | allTrue = True 282 | if args.debug > 0: 283 | print('\t\t\tTrue') 284 | continue 285 | else: 286 | netflowValue = column_values[condColumn] 287 | if args.debug > 0: 288 | print(f'\t\tTo compare field: {condColumn}, Condition value: {condValue}, Netflow value: {netflowValue}') 289 | if args.debug > 0: 290 | print('\t\t\tFalse') 291 | allTrue = False 292 | break 293 | 294 | if allTrue: 295 | labelToReturn = (genericlabelToVerify, detailedlabelToVerify) 296 | if args.debug > 0: 297 | print(f'\tNew label assigned: {genericlabelToVerify} {detailedlabelToVerify}') 298 | 299 | if args.verbose > 1: 300 | if 'Background' in labelToReturn: 301 | print(f'\tFinal label assigned: {labelToReturn}') 302 | else: 303 | print(f'\tFinal label assigned: \x1b\x5b1;31;40m{labelToReturn}\x1b\x5b0;0;40m') 304 | 305 | return labelToReturn 306 | 307 | except Exception as inst: 308 | print('[!] Error in class labeler getLabel(): unable to label the given column values') 309 | print(column_values) 310 | print(type(inst)) # the exception instance 311 | print(inst.args) # arguments stored in .args 312 | print(inst) # __str__ allows args to printed directly 313 | sys.exit(-1) 314 | 315 | 316 | def output_netflow_line_to_file(outputfile, originalline, filetype='', genericlabel='', detailedlabel=''): 317 | """ 318 | Get data and store it on a new file 319 | If genericlabel is empty, it is a headler line to process 320 | """ 321 | try: 322 | if 'csv' in filetype: 323 | separator = ',' 324 | elif 'tab' in filetype: 325 | separator = '\t' 326 | 327 | if type(originalline) == str and genericlabel == '': 328 | # It is a headerline 329 | 330 | # Should we add the 'label' string? Zeek has many headerlines 331 | if '#fields' in originalline: 332 | outputline = originalline.strip() + separator + 'label' + separator + 'detailedlabel' + '\n' 333 | outputfile.writelines(outputline) 334 | elif '#types' in originalline: 335 | outputline = originalline.strip() + separator + 'string' + separator + 'string' + '\n' 336 | outputfile.writelines(outputline) 337 | else: 338 | outputfile.writelines(originalline) 339 | # We are not putting the 'label' string in the header! 340 | elif type(originalline) == str and genericlabel != '': 341 | # These are values to store 342 | outputline = originalline.strip() + separator + genericlabel + separator + detailedlabel + '\n' 343 | outputfile.writelines(outputline) 344 | if args.debug > 1: 345 | print(f'Just outputed label {genericlabel} in line {outputline}') 346 | # keep it open! 347 | 348 | except Exception as inst: 349 | print('Problem in output_labeled_netflow_file()') 350 | print(type(inst)) # the exception instance 351 | print(inst.args) # arguments stored in .args 352 | print(inst) # __str__ allows args to printed directly 353 | sys.exit(-1) 354 | 355 | 356 | def process_nfdump(f, headers, labelmachine): 357 | """ 358 | DEPRECATED!! NEEDS UPDATE COMPLETELY 359 | Process and label an nfdump file 360 | """ 361 | pass 362 | """ 363 | # Just to monitor how many lines we read 364 | amount_lines_processed = 0 365 | 366 | # Parse the file into an array of dictionaries. 367 | # We will use the columns names as dictionary keys 368 | # Example: [ {'Date': '10/10/2013} , {'SrcIp':'1.1.1.1} , , ] 369 | netflowArray = [] 370 | columnDict = {} 371 | 372 | # Replace the TABs for spaces, if it has them..., 373 | # and replace the : in the ports to spaces also, 374 | # and strip the \n, and the word flow 375 | temp2 = headers.replace('flow', '') 376 | temp = re.sub('\s+', ' ', temp2).replace(':', ' ').strip() 377 | columnNames = temp.split(' ') 378 | 379 | # Only to separate src ip from dst ip 380 | addressType = '' 381 | 382 | # if args.debug > 0: 383 | # print(f'Columns names: {columnNames}') 384 | 385 | for cN in columnNames: 386 | # Separate between src ip and dst ip 387 | if 'Src' in cN: 388 | addressType = 'src' 389 | elif 'Dst' in cN: 390 | addressType = 'dst' 391 | elif 'IP' in cN: 392 | columnDict[addressType+cN] = "" 393 | netflowArray.append(columnDict) 394 | columnDict = {} 395 | # Separate ports 396 | elif 'Port' in cN: 397 | columnDict[addressType+cN] = "" 398 | netflowArray.append(columnDict) 399 | columnDict = {} 400 | elif 'Addr' in cN: 401 | pass 402 | elif 'Prot' in cN: 403 | columnDict['Proto'] = "" 404 | netflowArray.append(columnDict) 405 | columnDict = {} 406 | elif 'Durat' in cN: 407 | columnDict['Duration'] = "" 408 | netflowArray.append(columnDict) 409 | columnDict = {} 410 | elif 'Flow' in cN: 411 | columnDict['Flows'] = "" 412 | netflowArray.append(columnDict) 413 | columnDict = {} 414 | else: 415 | columnDict[cN] = "" 416 | netflowArray.append(columnDict) 417 | columnDict = {} 418 | 419 | columnDict['Label'] = "" 420 | netflowArray.append(columnDict) 421 | columnDict = {} 422 | 423 | # if args.debug > 0: 424 | # print('netflowArray') 425 | # print(netflowArray) 426 | 427 | # Create the output file with the header 428 | outputfile = open(args.netflowFile+'.labeled', 'w+') 429 | 430 | # Write the column names 431 | columnnames = "Date flow start Duration Proto Src IP Addr:Port Dst IP Addr:Port Flags Tos Packets Bytes Flows Label\n" 432 | outputfile.writelines(columnnames) 433 | 434 | # Read the second line to start processing 435 | line = f.readline() 436 | amount_lines_processed += 1 437 | while line: 438 | if args.verbose > 0: 439 | print(f'Netflow line: {line}', end=' ') 440 | 441 | # Parse the columns 442 | # Strip and replace ugly stuff 443 | temp2 = line.replace('->', '') 444 | temp = re.sub('\s+', ' ', temp2).strip() 445 | columnValues = temp.split(' ') 446 | 447 | # Date 448 | date = columnValues[0] 449 | # Store the value in the dict 450 | dict = netflowArray[0] 451 | columnName = list(dict.keys())[0] 452 | dict[columnName] = date 453 | netflowArray[0] = dict 454 | 455 | hour = columnValues[1] 456 | # Store the value in the dict 457 | dict = netflowArray[1] 458 | columnName = list(dict.keys())[0] 459 | dict[columnName] = hour 460 | netflowArray[1] = dict 461 | 462 | duration = columnValues[2] 463 | # Store the value in the dict 464 | dict = netflowArray[2] 465 | columnName = list(dict.keys())[0] 466 | dict[columnName] = duration 467 | netflowArray[2] = dict 468 | 469 | protocol = columnValues[3].upper() 470 | # Store the value in the dict 471 | dict = netflowArray[3] 472 | columnName = list(dict.keys())[0] 473 | # columnName = 'Proto' 474 | dict[columnName] = protocol 475 | netflowArray[3] = dict 476 | 477 | if 'TCP' in protocol or 'UDP' in protocol or 'RTP' in protocol: 478 | temp = columnValues[4] 479 | if len(temp.split(':')) <= 2: 480 | # It is IPV4 481 | srcip = temp.split(':')[0] 482 | # Store the value in the dict 483 | dict = netflowArray[4] 484 | columnName = list(dict.keys())[0] 485 | dict[columnName] = srcip 486 | netflowArray[4] = dict 487 | 488 | srcport = temp.split(':')[1] 489 | # Store the value in the dict 490 | dict = netflowArray[5] 491 | columnName = list(dict.keys())[0] 492 | dict[columnName] = srcport 493 | netflowArray[5] = dict 494 | 495 | temp2 = columnValues[5] 496 | dstip = temp2.split(':')[0] 497 | # Store the value in the dict 498 | dict = netflowArray[6] 499 | columnName = list(dict.keys())[0] 500 | dict[columnName] = dstip 501 | netflowArray[6] = dict 502 | 503 | dstport = temp2.split(':')[1] 504 | # Store the value in the dict 505 | dict = netflowArray[7] 506 | columnName = list(dict.keys())[0] 507 | dict[columnName] = dstport 508 | netflowArray[7] = dict 509 | elif len(temp.split(':')) > 2: 510 | # We are using ipv6! THIS DEPENDS A LOT ON THE 511 | # program that created the netflow.. 512 | srcip = temp[0:temp.rfind(':')] 513 | # Store the value in the dict 514 | dict = netflowArray[4] 515 | columnName = list(dict.keys())[0] 516 | dict[columnName] = srcip 517 | netflowArray[4] = dict 518 | 519 | srcport = temp[temp.rfind(':')+1:] 520 | # Store the value in the dict 521 | dict = netflowArray[5] 522 | columnName = list(dict.keys())[0] 523 | dict[columnName] = srcport 524 | netflowArray[5] = dict 525 | 526 | temp2 = columnValues[5] 527 | dstip = temp2[0:temp2.rfind(':')] 528 | # Store the value in the dict 529 | dict = netflowArray[6] 530 | columnName = list(dict.keys())[0] 531 | dict[columnName] = dstip 532 | netflowArray[6] = dict 533 | 534 | dstport = temp2[temp2.rfind(':')+1:] 535 | # Store the value in the dict 536 | dict = netflowArray[7] 537 | columnName = list(dict.keys())[0] 538 | dict[columnName] = dstport 539 | netflowArray[7] = dict 540 | else: 541 | print() 542 | print('Please implement this protocol!') 543 | print(line) 544 | sys.exit(-1) 545 | elif protocol == 'IPNIP' or protocol == 'RSVP' or protocol == 'GRE' or protocol == 'UDT' or / 546 | protocol == 'ARP' or protocol == 'ICMP' or protocol == 'PIM' or protocol == 'ESP' or / 547 | protocol == 'UNAS' or protocol == 'IGMP' or 'IPX' in protocol or 'RARP' in protocol / 548 | or 'LLC' in protocol or 'IPV6' in protocol: 549 | srcip = temp = columnValues[4] 550 | # Store the value in the dict 551 | dict = netflowArray[4] 552 | columnName = list(dict.keys())[0] 553 | dict[columnName] = srcip 554 | netflowArray[4] = dict 555 | 556 | srcport = '0' 557 | # Store the value in the dict 558 | dict = netflowArray[5] 559 | columnName = list(dict.keys())[0] 560 | dict[columnName] = srcport 561 | netflowArray[5] = dict 562 | 563 | dstip = temp = columnValues[5] 564 | # Store the value in the dict 565 | dict = netflowArray[6] 566 | columnName = list(dict.keys())[0] 567 | dict[columnName] = dstip 568 | netflowArray[6] = dict 569 | 570 | dstport = '0' 571 | # Store the value in the dict 572 | dict = netflowArray[7] 573 | columnName = list(dict.keys())[0] 574 | dict[columnName] = dstport 575 | netflowArray[7] = dict 576 | 577 | flags = columnValues[6].upper() 578 | # Store the value in the dict 579 | dict = netflowArray[8] 580 | columnName = list(dict.keys())[0] 581 | dict[columnName] = flags 582 | netflowArray[8] = dict 583 | 584 | tos = columnValues[7] 585 | # Store the value in the dict 586 | dict = netflowArray[9] 587 | columnName = list(dict.keys())[0] 588 | dict[columnName] = tos 589 | netflowArray[9] = dict 590 | 591 | packets = columnValues[8] 592 | # Store the value in the dict 593 | dict = netflowArray[10] 594 | columnName = list(dict.keys())[0] 595 | dict[columnName] = packets 596 | netflowArray[10] = dict 597 | 598 | bytes = columnValues[9] 599 | # Store the value in the dict 600 | dict = netflowArray[11] 601 | columnName = list(dict.keys())[0] 602 | dict[columnName] = bytes 603 | netflowArray[11] = dict 604 | 605 | flows = columnValues[10] 606 | # Store the value in the dict 607 | dict = netflowArray[12] 608 | columnName = list(dict.keys())[0] 609 | dict[columnName] = flows 610 | netflowArray[12] = dict 611 | 612 | # Empty the label in the dict 613 | dict = netflowArray[13] 614 | columnName = list(dict.keys())[0] 615 | dict[columnName] = "" 616 | netflowArray[13] = dict 617 | 618 | # if args.debug > 0: 619 | # print(date,hour,duration,protocol, srcip, srcport, dstip, dstport, flags, tos, packets, bytes, flows) 620 | # print(netflowArray) 621 | 622 | # Request a label 623 | genericlabel, detailedlabel = labelmachine.getLabel(netflowArray) 624 | 625 | # Store the value in the dict 626 | dict = netflowArray[13] 627 | columnName = list(dict.keys())[0] 628 | dict[columnName] = genericlabel 629 | netflowArray[13] = dict 630 | 631 | # if args.debug > 0: 632 | # print(netflowArray) 633 | 634 | # Ask to store the netflow 635 | output_netflow_line_to_file(outputfile, netflowArray) 636 | 637 | line = f.readline() 638 | amount_lines_processed += 1 639 | 640 | # Close the outputfile 641 | outputfile.close() 642 | """ 643 | 644 | 645 | def define_columns(headerline, filetype): 646 | """ Define the columns for Argus and Zeek-tab from the line received """ 647 | # These are the indexes for later fast processing 648 | column_idx = {} 649 | column_idx['starttime'] = False 650 | column_idx['endtime'] = False 651 | column_idx['dur'] = False 652 | column_idx['proto'] = False 653 | column_idx['appproto'] = False 654 | column_idx['srcip'] = False 655 | column_idx['srcport'] = False 656 | column_idx['dir'] = False 657 | column_idx['dstip'] = False 658 | column_idx['dstport'] = False 659 | column_idx['state'] = False 660 | column_idx['pkts'] = False 661 | column_idx['spkts'] = False 662 | column_idx['dpkts'] = False 663 | column_idx['bytes'] = False 664 | column_idx['sbytes'] = False 665 | column_idx['dbytes'] = False 666 | column_idx['orig_ip_bytes'] = False 667 | column_idx['resp_ip_bytes'] = False 668 | column_idx['history'] = False 669 | column_idx['event_type'] = False 670 | column_idx['uid'] = False 671 | column_idx['local_orig'] = False 672 | column_idx['local_resp'] = False 673 | column_idx['missed_bytes'] = False 674 | column_idx['tunnel_parents'] = False 675 | 676 | try: 677 | if 'csv' in filetype or 'tab' in filetype: 678 | # This should work for zeek-csv, zeek-tab, argus-csv, nfdump-csv 679 | if 'csv' in filetype: 680 | separator = ',' 681 | elif 'tab' in filetype: 682 | separator = '\t' 683 | nline = headerline.strip().split(separator) 684 | try: 685 | # Remove the extra column of zeek if it is there 686 | nline.remove('#fields') 687 | except ValueError: 688 | # ignore if #fields is not there 689 | pass 690 | if args.debug > 1: 691 | print(f'Headers line: {nline}') 692 | for field in nline: 693 | if args.debug > 2: 694 | print(f'Field: {field.lower()}, index: {nline.index(field)}') 695 | if 'time' in field.lower() or field.lower() == 'ts': 696 | column_idx['starttime'] = nline.index(field) 697 | elif 'uid' in field.lower(): 698 | column_idx['uid'] = nline.index(field) 699 | elif 'dur' in field.lower(): 700 | column_idx['dur'] = nline.index(field) 701 | elif 'proto' in field.lower(): 702 | column_idx['proto'] = nline.index(field) 703 | elif 'srca' in field.lower() or 'id.orig_h' in field.lower(): 704 | column_idx['srcip'] = nline.index(field) 705 | elif 'srcport' in field.lower() or 'id.orig_p' in field.lower(): 706 | column_idx['srcport'] = nline.index(field) 707 | elif 'dir' in field.lower(): 708 | column_idx['dir'] = nline.index(field) 709 | elif 'dsta' in field.lower() or 'id.resp_h' in field.lower(): 710 | column_idx['dstip'] = nline.index(field) 711 | elif 'dstport' in field.lower() or 'id.resp_p' in field.lower(): 712 | column_idx['dstport'] = nline.index(field) 713 | elif 'state' in field.lower(): 714 | column_idx['state'] = nline.index(field) 715 | elif 'srcbytes' in field.lower() or 'orig_bytes' in field.lower(): 716 | column_idx['sbytes'] = nline.index(field) 717 | elif 'destbytes' in field.lower() or 'resp_bytes' in field.lower(): 718 | column_idx['dbytes'] = nline.index(field) 719 | elif 'service' in field.lower(): 720 | column_idx['appproto'] = nline.index(field) 721 | elif 'srcpkts' in field.lower() or 'orig_pkts' in field.lower(): 722 | column_idx['spkts'] = nline.index(field) 723 | elif 'destpkts' in field.lower() or 'resp_pkts' in field.lower(): 724 | column_idx['dpkts'] = nline.index(field) 725 | elif 'totpkts' in field.lower(): 726 | column_idx['pkts'] = nline.index(field) 727 | elif 'totbytes' in field.lower(): 728 | column_idx['bytes'] = nline.index(field) 729 | elif 'history' in field.lower(): 730 | column_idx['history'] = nline.index(field) 731 | elif 'orig_ip_bytes' in field.lower(): 732 | column_idx['orig_ip_bytes'] = nline.index(field) 733 | elif 'resp_ip_bytes' in field.lower(): 734 | column_idx['resp_ip_bytes'] = nline.index(field) 735 | elif 'local_orig' in field.lower(): 736 | column_idx['local_orig'] = nline.index(field) 737 | elif 'local_resp' in field.lower(): 738 | column_idx['local_resp'] = nline.index(field) 739 | elif 'missed_bytes' in field.lower(): 740 | column_idx['missed_bytes'] = nline.index(field) 741 | elif 'tunnel_parents' in field.lower(): 742 | column_idx['tunnel_parents'] = nline.index(field) 743 | elif 'json' in filetype: 744 | if 'timestamp' in headerline: 745 | # Suricata json 746 | column_idx['starttime'] = 'timestamp' 747 | column_idx['srcip'] = 'src_ip' 748 | column_idx['dur'] = False 749 | column_idx['proto'] = 'proto' 750 | column_idx['srcport'] = 'src_port' 751 | column_idx['dstip'] = 'dst_ip' 752 | column_idx['dstport'] = 'dest_port' 753 | column_idx['spkts'] = 'flow/pkts_toserver' 754 | column_idx['dpkts'] = 'flow/pkts_toclient' 755 | column_idx['sbytes'] = 'flow/bytes_toserver' 756 | column_idx['dbytes'] = 'flow/bytes_toclient' 757 | column_idx['event_type'] = 'event_type' 758 | elif 'ts' in headerline: 759 | # Zeek json 760 | column_idx['starttime'] = 'ts' 761 | column_idx['srcip'] = 'id.orig_h' 762 | column_idx['endtime'] = '' 763 | column_idx['dur'] = 'duration' 764 | column_idx['proto'] = 'proto' 765 | column_idx['appproto'] = 'service' 766 | column_idx['srcport'] = 'id.orig_p' 767 | column_idx['dstip'] = 'id.resp_h' 768 | column_idx['dstport'] = 'id.resp_p' 769 | column_idx['state'] = 'conn_state' 770 | column_idx['pkts'] = '' 771 | column_idx['spkts'] = 'orig_pkts' 772 | column_idx['dpkts'] = 'resp_pkts' 773 | column_idx['bytes'] = '' 774 | column_idx['sbytes'] = 'orig_bytes' 775 | column_idx['dbytes'] = 'resp_bytes' 776 | column_idx['orig_ip_bytes'] = 'orig_ip_bytes' 777 | column_idx['resp_ip_bytes'] = 'resp_ip_bytes' 778 | column_idx['history'] = 'history' 779 | 780 | # Some of the fields were not found probably, 781 | # so just delete them from the index if their value is False. 782 | # If not we will believe that we have data on them 783 | # We need a temp dict because we can not change the size of dict while analyzing it 784 | temp_dict = {} 785 | for i in column_idx: 786 | if type(column_idx[i]) == bool and column_idx[i] is False: 787 | continue 788 | temp_dict[i] = column_idx[i] 789 | column_idx = temp_dict 790 | 791 | return column_idx 792 | except Exception as inst: 793 | exception_line = sys.exc_info()[2].tb_lineno 794 | print(f'\tProblem in define_columns() line {exception_line}', 0, 1) 795 | print(str(type(inst)), 0, 1) 796 | print(str(inst), 0, 1) 797 | sys.exit(1) 798 | 799 | 800 | def define_type(data): 801 | """ 802 | Try to define very fast the type of input from :Zeek file, 803 | Suricata json, Argus binetflow CSV, Argus binetflow TSV 804 | Using a Heuristic detection 805 | Input: The first line after the headers if there were some, as 'data' 806 | Outputs types can be can be: zeek-json, suricata, argus-tab, argus-csv, zeek-tab 807 | """ 808 | try: 809 | # If line json, it can be Zeek or suricata 810 | # If line CSV, it can be Argus 811 | # If line TSV, it can be Argus or zeek 812 | 813 | input_type = 'unknown' 814 | 815 | # Is it json? 816 | try: 817 | json_line = json.loads(data) 818 | # json 819 | try: 820 | # Zeek? 821 | _ = json_line['ts'] 822 | input_type = 'zeek-json' 823 | return input_type 824 | except KeyError: 825 | # Suricata? 826 | _ = json_line['timestamp'] 827 | input_type = 'suricata-json' 828 | return input_type 829 | except json.JSONDecodeError: 830 | # No json 831 | if type(data) == str: 832 | # string 833 | nr_commas = len(data.split(',')) 834 | nr_tabs = len(data.split(' ')) 835 | if nr_commas > nr_tabs: 836 | # Commas is the separator 837 | if nr_commas > 40: 838 | input_type = 'nfdump-csv' 839 | else: 840 | # comma separated argus file 841 | input_type = 'argus-csv' 842 | elif nr_tabs >= nr_commas: 843 | # Tabs is the separator or it can be also equal number of commas and tabs, including both 0 844 | # Can be Zeek conn.log with TABS 845 | # Can be Argus binetflow with TABS 846 | # Can be Nfdump binetflow with TABS 847 | if '->' in data or 'StartTime' in data: 848 | input_type = 'argus-tab' 849 | elif 'separator' in data: 850 | input_type = 'zeek-tab' 851 | elif 'Date' in data: 852 | input_type = 'nfdump-tab' 853 | 854 | return input_type 855 | 856 | except Exception as inst: 857 | exception_line = sys.exc_info()[2].tb_lineno 858 | print(f'\tProblem in define_type() line {exception_line}', 0, 1) 859 | print(str(type(inst)), 0, 1) 860 | print(str(inst), 0, 1) 861 | sys.exit(1) 862 | 863 | 864 | def process_zeek(column_idx, input_file, output_file, labelmachine, filetype): 865 | """ 866 | Process and label a Zeek file using the label configuration. 867 | Zeek files can have three distinct field separators: 868 | - 'tab': currently supported 869 | - 'csv': currently supported 870 | - 'json': not implemented yet 871 | """ 872 | try: 873 | amount_lines_processed = 0 874 | column_values = {} 875 | 876 | if args.verbose > 0: 877 | print(f'[+] Labeling the flow file {args.netflowFile}') 878 | 879 | # Read firstlines 880 | line = input_file.readline() 881 | 882 | # Delete headerlines 883 | while '#' in line: 884 | line = input_file.readline() 885 | 886 | # Process each flow in input file 887 | while line: 888 | # Count the flows processed 889 | amount_lines_processed += 1 890 | 891 | if args.verbose > 1: 892 | print(f'Netflow line: {line}', end='') 893 | 894 | # Zeek files can be in csv, tab or JSON format 895 | # Labeling CSV and TAB uses the same method 896 | if 'csv' in filetype or 'tab' in filetype: 897 | # Work with csv and tabs 898 | if 'csv' in filetype: 899 | separator = ',' 900 | elif 'tab' in filetype: 901 | separator = '\t' 902 | 903 | # Transform the line into an array 904 | line_values = line.split(separator) 905 | 906 | # Read values from the flow line 907 | for key in column_idx: 908 | column_values[key] = line_values[column_idx[key]] 909 | 910 | # Create the hand-made columns that are a sum of other columns 911 | # First empty them 912 | column_values['bytes'] = '' 913 | column_values['pkts'] = '' 914 | column_values['ipbytes'] = '' 915 | 916 | # bytes: total bytes. Calculated as the SUM of sbytes and dbytes 917 | # We do it like this because sometimes the column can be - or 0 918 | if column_values['sbytes'] == '-': 919 | sbytes = 0 920 | else: 921 | sbytes = int(column_values['sbytes']) 922 | if column_values['dbytes'] == '-': 923 | dbytes = 0 924 | else: 925 | dbytes = int(column_values['dbytes']) 926 | column_values['bytes'] = str(sbytes + dbytes) 927 | # print(f'New column bytes = {column_values["bytes"]}') 928 | 929 | # pkts: total packets. Calculated as the SUM of spkts and dpkts 930 | # We do it like this because sometimes the column can be - or 0 931 | if column_values['spkts'] == '-': 932 | spkts = 0 933 | else: 934 | spkts = int(column_values['spkts']) 935 | if column_values['dpkts'] == '-': 936 | dpkts = 0 937 | else: 938 | dpkts = int(column_values['dpkts']) 939 | column_values['pkts'] = str(spkts + dpkts) 940 | # print(f'New column pkst = {column_values["pkts"]}') 941 | 942 | # ipbytes: total transferred bytes. 943 | # Calculated as the SUM of orig_ip_bytes and resp_ip_bytes. 944 | # We do it like this because sometimes the column can be - or 0 945 | if column_values['orig_ip_bytes'] == '-': 946 | sip_bytes = 0 947 | else: 948 | sip_bytes = int(column_values['orig_ip_bytes']) 949 | if column_values['resp_ip_bytes'] == '-': 950 | dip_bytes = 0 951 | else: 952 | dip_bytes = int(column_values['resp_ip_bytes']) 953 | column_values['ipbytes'] = str(sip_bytes + dip_bytes) 954 | # print(f'New column ipbytes = {column_values["ipbytes"]}') 955 | 956 | # Request a label 957 | genericlabel, detailedlabel = labelmachine.getLabel(column_values) 958 | if args.debug > 1: 959 | print(f'Label {genericlabel} assigned in line {line}') 960 | 961 | # Store the netflow 962 | output_netflow_line_to_file(output_file, line, filetype, genericlabel=genericlabel, detailedlabel=detailedlabel) 963 | 964 | # Read next flow line ignoring comments 965 | line = input_file.readline() 966 | while '#' in line: 967 | line = input_file.readline() 968 | 969 | elif 'json' in filetype: 970 | # Count the first line 971 | amount_lines_processed += 1 972 | pass 973 | 974 | # Returned number of labeled flows 975 | return amount_lines_processed 976 | except Exception as inst: 977 | exception_line = sys.exc_info()[2].tb_lineno 978 | print(f'\t[!] Error in process_zeek(): exception in line {exception_line}', 0, 1) 979 | print(str(type(inst)), 0, 1) 980 | print(str(inst), 0, 1) 981 | sys.exit(1) 982 | 983 | 984 | def process_argus(column_idx, output_file, labelmachine, filetype): 985 | """ 986 | DEPRECATED!! NEEDS UPDATE COMPLETELY 987 | Process an Argus file 988 | """ 989 | try: 990 | pass 991 | """ 992 | print(column_idx) 993 | return 0 994 | 995 | # This is argus files... 996 | amount_lines_processed = 0 997 | 998 | # Parse the file into an array of dictionaries. We will use the columns names as dictionary keys 999 | # Example: [ {'Date': '10/10/2013} , {'SrcIp':'1.1.1.1} , , ] 1000 | netflowArray = [] 1001 | columnDict = {} 1002 | 1003 | # Replace the TABs for spaces, if it has them..., and replace the : in the ports to spaces also, and strip the \n, and the word flow 1004 | temp = re.sub('\s+', ' ', headers).strip() 1005 | columnNames = temp.split(' ') 1006 | 1007 | # if args.debug > 0: 1008 | # print(f'Columns names: {columnNames}') 1009 | 1010 | # So far argus does no have a column Date 1011 | columnDict['Date'] = "" 1012 | netflowArray.append(columnDict) 1013 | columnDict = {} 1014 | 1015 | columnDict['start'] = "" 1016 | netflowArray.append(columnDict) 1017 | columnDict = {} 1018 | 1019 | columnDict['Duration'] = "" 1020 | netflowArray.append(columnDict) 1021 | columnDict = {} 1022 | 1023 | columnDict['Proto'] = "" 1024 | netflowArray.append(columnDict) 1025 | columnDict = {} 1026 | 1027 | columnDict['srcIP'] = "" 1028 | netflowArray.append(columnDict) 1029 | columnDict = {} 1030 | 1031 | columnDict['srcPort'] = "" 1032 | netflowArray.append(columnDict) 1033 | columnDict = {} 1034 | 1035 | columnDict['dstIP'] = "" 1036 | netflowArray.append(columnDict) 1037 | columnDict = {} 1038 | 1039 | columnDict['dstPort'] = "" 1040 | netflowArray.append(columnDict) 1041 | columnDict = {} 1042 | 1043 | columnDict['Flags'] = "" 1044 | netflowArray.append(columnDict) 1045 | columnDict = {} 1046 | 1047 | columnDict['Tos'] = "" 1048 | netflowArray.append(columnDict) 1049 | columnDict = {} 1050 | 1051 | columnDict['Packets'] = "" 1052 | netflowArray.append(columnDict) 1053 | columnDict = {} 1054 | 1055 | columnDict['Bytes'] = "" 1056 | netflowArray.append(columnDict) 1057 | columnDict = {} 1058 | 1059 | columnDict['Label'] = "" 1060 | netflowArray.append(columnDict) 1061 | columnDict = {} 1062 | 1063 | # Write the column names 1064 | columnnames = "Date Time Dur Proto SrcAddr Sport Dir DstAddr Dport State sTos TotPkts TotBytes Label\n" 1065 | output_file.writelines(columnnames) 1066 | 1067 | # Read the second line to start processing 1068 | line = f.readline() 1069 | amount_lines_processed += 1 1070 | while line: 1071 | if args.verbose > 0: 1072 | print(f'Netflow line: {line}', end=' ') 1073 | 1074 | # Parse the columns 1075 | # Strip and replace ugly stuff 1076 | temp2 = line.replace('->', '') 1077 | temp = re.sub('\s+', ' ', temp2).strip() 1078 | columnValues = temp.split(' ') 1079 | 1080 | # if args.debug > 0: 1081 | # print(columnValues) 1082 | 1083 | # Date 1084 | date = columnValues[0] 1085 | # Store the value in the dict 1086 | dict = netflowArray[0] 1087 | columnName = list(dict.keys())[0] 1088 | dict[columnName] = date 1089 | netflowArray[0] = dict 1090 | 1091 | hour = columnValues[1] 1092 | # Store the value in the dict 1093 | dict = netflowArray[1] 1094 | columnName = list(dict.keys())[0] 1095 | dict[columnName] = hour 1096 | netflowArray[1] = dict 1097 | 1098 | duration = columnValues[2] 1099 | # Store the value in the dict 1100 | dict = netflowArray[2] 1101 | columnName = list(dict.keys())[0] 1102 | dict[columnName] = duration 1103 | netflowArray[2] = dict 1104 | 1105 | protocol = columnValues[3].upper() 1106 | # Store the value in the dict 1107 | dict = netflowArray[3] 1108 | columnName = list(dict.keys())[0] 1109 | dict[columnName] = protocol 1110 | netflowArray[3] = dict 1111 | 1112 | srcIP = columnValues[4] 1113 | # Store the value in the dict 1114 | dict = netflowArray[4] 1115 | columnName = list(dict.keys())[0] 1116 | dict[columnName] = srcIP 1117 | netflowArray[4] = dict 1118 | 1119 | if 'ARP' in protocol: 1120 | srcPort = '0' 1121 | # Store the value in the dict 1122 | dict = netflowArray[5] 1123 | columnName = list(dict.keys())[0] 1124 | dict[columnName] = srcPort 1125 | netflowArray[5] = dict 1126 | else: 1127 | srcPort = columnValues[5] 1128 | # Store the value in the dict 1129 | dict = netflowArray[5] 1130 | columnName = list(dict.keys())[0] 1131 | dict[columnName] = srcPort 1132 | netflowArray[5] = dict 1133 | 1134 | dstIP = columnValues[6] 1135 | # Store the value in the dict 1136 | dict = netflowArray[6] 1137 | columnName = list(dict.keys())[0] 1138 | dict[columnName] = dstIP 1139 | netflowArray[6] = dict 1140 | 1141 | if 'ARP' in protocol: 1142 | dstPort = '0' 1143 | # Store the value in the dict 1144 | dict = netflowArray[7] 1145 | columnName = list(dict.keys())[0] 1146 | dict[columnName] = dstPort 1147 | netflowArray[7] = dict 1148 | 1149 | Flags = columnValues[8] 1150 | # Store the value in the dict 1151 | dict = netflowArray[8] 1152 | columnName = list(dict.keys())[0] 1153 | dict[columnName] = Flags 1154 | netflowArray[8] = dict 1155 | 1156 | else: 1157 | dstPort = columnValues[7] 1158 | # Store the value in the dict 1159 | dict = netflowArray[7] 1160 | columnName = list(dict.keys())[0] 1161 | dict[columnName] = dstPort 1162 | netflowArray[7] = dict 1163 | 1164 | Flags = columnValues[8] 1165 | # Store the value in the dict 1166 | dict = netflowArray[8] 1167 | columnName = list(dict.keys())[0] 1168 | dict[columnName] = Flags 1169 | netflowArray[8] = dict 1170 | 1171 | if 'LLC' in protocol: 1172 | Tos = '0' 1173 | # Store the value in the dict 1174 | dict = netflowArray[9] 1175 | columnName = list(dict.keys())[0] 1176 | dict[columnName] = Tos 1177 | netflowArray[9] = dict 1178 | 1179 | Packets = columnValues[9] 1180 | # Store the value in the dict 1181 | dict = netflowArray[10] 1182 | columnName = list(dict.keys())[0] 1183 | dict[columnName] = Packets 1184 | netflowArray[10] = dict 1185 | 1186 | Bytes = columnValues[10] 1187 | # Store the value in the dict 1188 | dict = netflowArray[11] 1189 | columnName = list(dict.keys())[0] 1190 | dict[columnName] = Bytes 1191 | netflowArray[11] = dict 1192 | 1193 | # Request a label 1194 | genericlabellabel, detailedlabel = labelmachine.getLabel(netflowArray) 1195 | # Store the value in the dict 1196 | dict = netflowArray[12] 1197 | columnName = list(dict.keys())[0] 1198 | dict[columnName] = genericlabellabel 1199 | netflowArray[12] = dict 1200 | elif 'ARP' in protocol: 1201 | Tos = '0' 1202 | # Store the value in the dict 1203 | dict = netflowArray[9] 1204 | columnName = list(dict.keys())[0] 1205 | dict[columnName] = Tos 1206 | netflowArray[9] = dict 1207 | 1208 | Packets = columnValues[8] 1209 | # Store the value in the dict 1210 | dict = netflowArray[10] 1211 | columnName = list(dict.keys())[0] 1212 | dict[columnName] = Packets 1213 | netflowArray[10] = dict 1214 | 1215 | Bytes = columnValues[9] 1216 | # Store the value in the dict 1217 | dict = netflowArray[11] 1218 | columnName = list(dict.keys())[0] 1219 | dict[columnName] = Bytes 1220 | netflowArray[11] = dict 1221 | 1222 | # Request a label 1223 | genericlabellabel, detailedlabel = labelmachine.getLabel(netflowArray) 1224 | # Store the value in the dict 1225 | dict = netflowArray[12] 1226 | columnName = list(dict.keys())[0] 1227 | dict[columnName] = genericlabellabel 1228 | netflowArray[12] = dict 1229 | else: 1230 | Tos = columnValues[9] 1231 | # Store the value in the dict 1232 | dict = netflowArray[9] 1233 | columnName = list(dict.keys())[0] 1234 | dict[columnName] = Tos 1235 | netflowArray[9] = dict 1236 | 1237 | Packets = columnValues[10] 1238 | # Store the value in the dict 1239 | dict = netflowArray[10] 1240 | columnName = list(dict.keys())[0] 1241 | dict[columnName] = Packets 1242 | netflowArray[10] = dict 1243 | 1244 | Bytes = columnValues[11] 1245 | # Store the value in the dict 1246 | dict = netflowArray[11] 1247 | columnName = list(dict.keys())[0] 1248 | dict[columnName] = Bytes 1249 | netflowArray[11] = dict 1250 | 1251 | # Request a label 1252 | genericlabellabel, detailedlabel = labelmachine.getLabel(netflowArray) 1253 | # Store the value in the dict 1254 | dict = netflowArray[12] 1255 | columnName = list(dict.keys())[0] 1256 | dict[columnName] = genericlabellabel 1257 | netflowArray[12] = dict 1258 | 1259 | # if args.debug > 0: 1260 | # print(netflowArray) 1261 | 1262 | # Ask to store the netflow 1263 | output_netflow_line_to_file(output_file, netflowArray) 1264 | 1265 | line = f.readline() 1266 | amount_lines_processed += 1 1267 | """ 1268 | except Exception as inst: 1269 | exception_line = sys.exc_info()[2].tb_lineno 1270 | print(f'\tProblem in process_argus() line {exception_line}', 0, 1) 1271 | print(str(type(inst)), 0, 1) 1272 | print(str(inst), 0, 1) 1273 | sys.exit(1) 1274 | 1275 | 1276 | def process_netflow(labelmachine): 1277 | """ 1278 | This function takes the flowFile and parse it. 1279 | Then it asks for a label and finally it calls 1280 | a function to store the netflow in a file. 1281 | """ 1282 | try: 1283 | if args.verbose > 0: 1284 | print(f'[+] Processing the flow file {args.netflowFile}') 1285 | 1286 | # Open flows file 1287 | try: 1288 | input_file = open(args.netflowFile, 'r') 1289 | except Exception as inst: 1290 | print('[!] Error in process_netflow: cannot open the input netflow file.') 1291 | print(type(inst)) # the exception instance 1292 | print(inst.args) # arguments stored in .args 1293 | print(inst) # __str__ allows args to printed directly 1294 | sys.exit(-1) 1295 | 1296 | # Define the type of file 1297 | headerline = input_file.readline() 1298 | 1299 | # If there are no headers, do not process the file: 1300 | # - Zeek headers start with '#' 1301 | # - Argus headers start with 'StartTime' 1302 | # - nfdump headers start with 'Date' 1303 | # if '#' not in headerline[0] and 'Date' not in headerline and 'StartTime' not in headerline and 'ts' not in headerline and 'timestamp' not in headerline: 1304 | header_keywords = ['#', 'Date', 'StarTime', 'ts', 'timestamp'] 1305 | if not any(headerline.startswith(keyword) for keyword in header_keywords): 1306 | print('[!] Error in process_netflow: the input netflow file has not headers.') 1307 | sys.exit(-1) 1308 | 1309 | # Attempt to automatically identify the type of file 1310 | # from the header of the netflow file 1311 | filetype = define_type(headerline) 1312 | if args.verbose > 0: 1313 | print(f'[+] The input netflow file to label was identified as: {filetype}') 1314 | 1315 | # Create the output file to store the labeled netflows 1316 | output_file = open(args.netflowFile+'.labeled', 'w+') 1317 | if args.verbose > 0: 1318 | print(f"[+] The netflow file labeled can be found at: {args.netflowFile+'.labeled'}") 1319 | 1320 | # Store the headers in the output file 1321 | output_netflow_line_to_file(output_file, headerline) 1322 | 1323 | # Define the columns based on the type of the input netflow file 1324 | # and call the labeler function based on the detected type 1325 | if filetype == 'zeek-json': 1326 | column_idx = define_columns(headerline, filetype='json') 1327 | amount_lines_processed = 0 1328 | elif filetype == 'suricata-json': 1329 | column_idx = define_columns(headerline, filetype='json') 1330 | amount_lines_processed = 0 1331 | elif filetype == 'nfdump-csv': 1332 | column_idx = define_columns(headerline, filetype='csv') 1333 | amount_lines_processed = 0 1334 | elif filetype == 'argus-csv': 1335 | column_idx = define_columns(headerline, filetype='csv') 1336 | amount_lines_processed = process_argus(column_idx, input_file, output_file, labelmachine, filetype='csv') 1337 | elif filetype == 'argus-tab': 1338 | column_idx = define_columns(headerline, filetype='tab') 1339 | amount_lines_processed = process_argus(column_idx, input_file, output_file, labelmachine, filetype='tab') 1340 | elif filetype == 'zeek-tab': 1341 | # Get all the other headers first 1342 | while '#types' not in headerline: 1343 | # Go through all the # headers, but rememeber the #fields one 1344 | if '#fields' in headerline: 1345 | fields_headerline = headerline 1346 | headerline = input_file.readline() 1347 | 1348 | # Store the rest of the zeek headers in the output file 1349 | output_netflow_line_to_file(output_file, headerline, filetype='tab') 1350 | 1351 | # Get the columns indexes 1352 | column_idx = define_columns(fields_headerline, filetype='tab') 1353 | 1354 | # Process the whole file 1355 | amount_lines_processed = process_zeek(column_idx, input_file, output_file, labelmachine, filetype='tab') 1356 | 1357 | elif filetype == 'nfdump-tab': 1358 | column_idx = define_columns(headerline, filetype='tab') 1359 | amount_lines_processed = process_nfdump(column_idx, input_file, output_file, headerline, labelmachine) 1360 | else: 1361 | print(f"[!] Error in process_netflow: filetype not supported {filetype}") 1362 | 1363 | # Close the input file 1364 | input_file.close() 1365 | 1366 | # Close the output file 1367 | output_file.close() 1368 | 1369 | print(f"[+] Labeling completed. Total number of flows read: {amount_lines_processed}") 1370 | 1371 | except Exception as inst: 1372 | exception_line = sys.exc_info()[2].tb_lineno 1373 | print(f'[!] Error in process_netflow() line {exception_line}', 0, 1) 1374 | print(type(inst)) # the exception instance 1375 | print(inst.args) # arguments stored in .args 1376 | print(inst) # __str__ allows args to printed directly 1377 | sys.exit(-1) 1378 | 1379 | 1380 | def load_conditions(labelmachine): 1381 | """ 1382 | Load the labeling conditions from a configuration file. 1383 | Input: labelmachine is a labeler object 1384 | Output: modified labelmachine object. No return instruction. 1385 | """ 1386 | try: 1387 | conf = open(args.configFile) 1388 | 1389 | if args.debug > 0: 1390 | print('Loading the conditions from the configuration file ') 1391 | 1392 | # Read the labeling configuration file 1393 | line = conf.readline().strip() 1394 | conditions = {} 1395 | 1396 | # Process each line of the labeling configuration file 1397 | # There are three possible options here: 1398 | # - We read a comment: # 1399 | # - We read a label: does not start with symbols 1400 | # - We read a label condition: starts with '-' 1401 | while line: 1402 | # Ignore comments marked with '#' 1403 | if line.strip().find('#') == 0: 1404 | line = conf.readline().strip() 1405 | continue 1406 | 1407 | # Read a label 1408 | if line.strip()[0] != '-': 1409 | label = line.split(':')[0] 1410 | # if args.debug > 0: 1411 | # print(f'Label: {label}') 1412 | conditions[label] = [] 1413 | 1414 | # Now read all the conditions for this label 1415 | line = conf.readline().strip() 1416 | while line: 1417 | # If line starts with '-' is a condition 1418 | if line.strip()[0] == '-': 1419 | # Parse the condition 1420 | tempAndConditions = line.strip().split('-')[1] 1421 | if args.debug > 1: 1422 | print(f'Condition: {tempAndConditions}') 1423 | 1424 | # Check if the condition is composed, 1425 | # e.g.: srcIP=xxx.xxx.xxx.xxx & dstPort=xx 1426 | andConditions = [] 1427 | for andCond in tempAndConditions.split('&'): 1428 | tempdict = {} 1429 | tempdict[andCond.strip().split('=')[0]] = andCond.strip().split('=')[1] 1430 | andConditions.append(tempdict) 1431 | 1432 | conditions[label].append(andConditions) 1433 | 1434 | line = conf.readline().strip() 1435 | else: 1436 | # Finished reading all conditions for a given label 1437 | break 1438 | labelmachine.addCondition(conditions) 1439 | conditions = {} 1440 | 1441 | except KeyboardInterrupt: 1442 | # CTRL-C pretty handling. 1443 | print("Keyboard Interruption!. Exiting.") 1444 | sys.exit(1) 1445 | except Exception as inst: 1446 | print('Problem in main() function at load_conditions ') 1447 | print(type(inst)) # the exception instance 1448 | print(inst.args) # arguments stored in .args 1449 | print(inst) # __str__ allows args to printed directly 1450 | sys.exit(-1) 1451 | return False 1452 | 1453 | 1454 | if __name__ == '__main__': 1455 | print(f'NetFlow Labeler v{version}') 1456 | print('Stratosphere Research Laboratory (https://stratosphereips.org)') 1457 | print() 1458 | 1459 | # Parse the parameters 1460 | parser = argparse.ArgumentParser(description="A configurable rule-based labeling tool for network flow files") 1461 | parser.add_argument('-c', '--configFile', metavar='', action='store', 1462 | required=True, help='path to labeling configuration.') 1463 | parser.add_argument('-f', '--netflowFile', metavar='', action='store', 1464 | required=True, help='path to the file to label.') 1465 | parser.add_argument('-v', '--verbose', action='store', 1466 | required=False, type=int, default=0, help='set verbosity level.') 1467 | parser.add_argument('-d', '--debug', action='store', required=False, type=int, default=0, 1468 | help='set debugging level.') 1469 | args = parser.parse_args() 1470 | 1471 | try: 1472 | # Create an instance of the labeler 1473 | labelmachine = labeler() 1474 | 1475 | # Load labeling conditions from config file 1476 | load_conditions(labelmachine) 1477 | 1478 | # Direct process of netflow flows 1479 | process_netflow(labelmachine) 1480 | except KeyboardInterrupt: 1481 | # CTRL-C pretty handling. 1482 | print("Keyboard Interruption!. Exiting.") 1483 | sys.exit(1) 1484 | except Exception as inst: 1485 | # Notify of any other exception 1486 | print('Exception in __main__') 1487 | print(type(inst)) # the exception instance 1488 | print(inst.args) # arguments stored in .args 1489 | print(inst) # __str__ allows args to printed directly 1490 | -------------------------------------------------------------------------------- /zeek-files-labeler.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # Copyright (C) 2009 Sebastian Garcia, Veronica Valeros 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program; if not, write to the Free Software 16 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 | # 18 | # 19 | """ 20 | Zeek files labeler 21 | 22 | A tool that effortlessly adds labels to netflow files. With support for Zeek, Argus, and NFdump 23 | formats in both CSV and TSV. 24 | 25 | Authors: 26 | Sebastian Garcia, sebastian.garcia@agents.fel.cvut.cz, eldraco@gmail.com 27 | Veronica Valeros, vero.valeros@gmail.com 28 | Stratosphere Laboratory, Czech Technical University in Prague 29 | """ 30 | 31 | 32 | import sys 33 | import json 34 | import argparse 35 | import os 36 | from os import listdir 37 | from os.path import isfile, join 38 | import subprocess 39 | 40 | VERSION = "0.1" 41 | 42 | 43 | def output_netflow_line_to_file(outputfile, originalline, filetype='', genericlabel='', detailedlabel=''): 44 | """ 45 | Store the input line with its labels into the output file. If 'genericlabel' is empty, it means 46 | the input line is a header line, and requires special processing. 47 | - Input: outpuffile, originalline, filetype, genericlabel and detailedlabel 48 | - Output: no output 49 | """ 50 | try: 51 | # Configure the field separator 52 | if 'csv' in filetype: 53 | separator = ',' 54 | elif 'tab' in filetype: 55 | separator = '\t' 56 | 57 | # Validate if input line is a header line. Write all header lines back without change, 58 | # except those Zeek headers that define fields and types. 59 | if isinstance(originalline, str) and genericlabel == '': 60 | if '#fields' in originalline: 61 | outputline = originalline.strip() + separator + 'label' + separator + 'detailedlabel' + '\n' 62 | outputfile.writelines(outputline) 63 | elif '#types' in originalline: 64 | outputline = originalline.strip() + separator + 'string' + separator + 'string' + '\n' 65 | outputfile.writelines(outputline) 66 | else: 67 | outputfile.writelines(originalline) 68 | # Validate if input line is a netflow line and store along with the new fields 69 | elif isinstance(originalline,str) and genericlabel != '': 70 | outputline = originalline.strip() + separator + genericlabel + separator + detailedlabel + '\n' 71 | outputfile.writelines(outputline) 72 | if args.debug > 1: 73 | print(f' [+] Wrote line: {outputline}') 74 | # do not close the file 75 | 76 | except Exception as inst: 77 | print('Problem in output_labeled_netflow_file()') 78 | print(type(inst)) # the exception instance 79 | print(inst.args) # arguments stored in .args 80 | print(inst) # __str__ allows args to printed directly 81 | sys.exit(-1) 82 | 83 | 84 | def define_columns(headerline, filetype): 85 | """ 86 | Define the columns for Argus and Zeek-tab from the line received 87 | - Input: headerline, filetype 88 | - Output: column_idx 89 | """ 90 | # These are the indexes for later fast processing 91 | column_idx = {} 92 | column_idx['starttime'] = False 93 | column_idx['endtime'] = False 94 | column_idx['dur'] = False 95 | column_idx['proto'] = False 96 | column_idx['appproto'] = False 97 | column_idx['srcip'] = False 98 | column_idx['srcport'] = False 99 | column_idx['dir'] = False 100 | column_idx['dstip'] = False 101 | column_idx['dstport'] = False 102 | column_idx['state'] = False 103 | column_idx['pkts'] = False 104 | column_idx['spkts'] = False 105 | column_idx['dpkts'] = False 106 | column_idx['bytes'] = False 107 | column_idx['sbytes'] = False 108 | column_idx['dbytes'] = False 109 | column_idx['orig_ip_bytes'] = False 110 | column_idx['resp_ip_bytes'] = False 111 | column_idx['history'] = False 112 | column_idx['event_type'] = False 113 | column_idx['uid'] = False 114 | column_idx['conn_uids'] = False 115 | column_idx['local_orig'] = False 116 | column_idx['local_resp'] = False 117 | column_idx['missed_bytes'] = False 118 | column_idx['tunnel_parents'] = False 119 | column_idx['label'] = False 120 | column_idx['detailedlabel'] = False 121 | column_idx['fingerprint'] = False 122 | column_idx['id'] = False 123 | column_idx['uids'] = False 124 | 125 | try: 126 | if 'csv' in filetype or 'tab' in filetype: 127 | # This should work for zeek-csv, zeek-tab, argus-csv, nfdump-csv 128 | if 'csv' in filetype: 129 | separator = ',' 130 | elif 'tab' in filetype: 131 | separator = '\t' 132 | 133 | nline = headerline.strip().split(separator) 134 | try: 135 | # Remove the extra column of zeek if it is there 136 | nline.remove('#fields') 137 | except ValueError: 138 | # ignore if #fields is not there 139 | pass 140 | 141 | if args.debug > 1: 142 | print(f'Headers line: {nline}') 143 | 144 | for field in nline: 145 | if args.debug > 2: 146 | print(f'Field: {field.lower()}, index: {nline.index(field)}') 147 | 148 | if 'time' in field.lower() or field.lower() == 'ts': 149 | column_idx['starttime'] = nline.index(field) 150 | elif field.lower() == 'uid': 151 | column_idx['uid'] = nline.index(field) 152 | elif 'conn_uids' in field.lower(): 153 | column_idx['conn_uids'] = nline.index(field) 154 | elif 'dur' in field.lower(): 155 | column_idx['dur'] = nline.index(field) 156 | elif 'proto' in field.lower(): 157 | column_idx['proto'] = nline.index(field) 158 | elif 'srca' in field.lower() or 'id.orig_h' in field.lower(): 159 | column_idx['srcip'] = nline.index(field) 160 | elif 'srcport' in field.lower() or 'id.orig_p' in field.lower(): 161 | column_idx['srcport'] = nline.index(field) 162 | elif 'dir' in field.lower(): 163 | column_idx['dir'] = nline.index(field) 164 | elif 'dsta' in field.lower() or 'id.resp_h' in field.lower(): 165 | column_idx['dstip'] = nline.index(field) 166 | elif 'dstport' in field.lower() or 'id.resp_p' in field.lower(): 167 | column_idx['dstport'] = nline.index(field) 168 | elif 'state' in field.lower(): 169 | column_idx['state'] = nline.index(field) 170 | elif 'srcbytes' in field.lower() or 'orig_bytes' in field.lower(): 171 | column_idx['sbytes'] = nline.index(field) 172 | elif 'destbytes' in field.lower() or 'resp_bytes' in field.lower(): 173 | column_idx['dbytes'] = nline.index(field) 174 | elif 'service' in field.lower(): 175 | column_idx['appproto'] = nline.index(field) 176 | elif 'srcpkts' in field.lower() or 'orig_pkts' in field.lower(): 177 | column_idx['spkts'] = nline.index(field) 178 | elif 'destpkts' in field.lower() or 'resp_pkts' in field.lower(): 179 | column_idx['dpkts'] = nline.index(field) 180 | elif 'totpkts' in field.lower(): 181 | column_idx['pkts'] = nline.index(field) 182 | elif 'totbytes' in field.lower(): 183 | column_idx['bytes'] = nline.index(field) 184 | elif 'history' in field.lower(): 185 | column_idx['history'] = nline.index(field) 186 | elif 'orig_ip_bytes' in field.lower(): 187 | column_idx['orig_ip_bytes'] = nline.index(field) 188 | elif 'resp_ip_bytes' in field.lower(): 189 | column_idx['resp_ip_bytes'] = nline.index(field) 190 | elif 'local_orig' in field.lower(): 191 | column_idx['local_orig'] = nline.index(field) 192 | elif 'local_resp' in field.lower(): 193 | column_idx['local_resp'] = nline.index(field) 194 | elif 'missed_bytes' in field.lower(): 195 | column_idx['missed_bytes'] = nline.index(field) 196 | elif 'tunnel_parents' in field.lower(): 197 | column_idx['tunnel_parents'] = nline.index(field) 198 | elif 'detailedlabel' in field.lower(): 199 | column_idx['detailedlabel'] = nline.index(field) 200 | elif 'label' in field.lower(): 201 | column_idx['label'] = nline.index(field) 202 | elif 'fingerprint' in field.lower(): 203 | column_idx['fingerprint'] = nline.index(field) 204 | elif 'uids' in field.lower(): 205 | column_idx['uids'] = nline.index(field) 206 | elif 'id' in field.lower(): 207 | column_idx['id'] = nline.index(field) 208 | elif 'json' in filetype: 209 | if 'timestamp' in headerline: 210 | # Suricata json 211 | column_idx['starttime'] = 'timestamp' 212 | column_idx['srcip'] = 'src_ip' 213 | column_idx['dur'] = False 214 | column_idx['proto'] = 'proto' 215 | column_idx['srcport'] = 'src_port' 216 | column_idx['dstip'] = 'dst_ip' 217 | column_idx['dstport'] = 'dest_port' 218 | column_idx['spkts'] = 'flow/pkts_toserver' 219 | column_idx['dpkts'] = 'flow/pkts_toclient' 220 | column_idx['sbytes'] = 'flow/bytes_toserver' 221 | column_idx['dbytes'] = 'flow/bytes_toclient' 222 | column_idx['event_type'] = 'event_type' 223 | elif 'ts' in headerline: 224 | # Zeek json 225 | column_idx['starttime'] = 'ts' 226 | column_idx['conn_uids'] = 'conn_uids' 227 | column_idx['srcip'] = 'id.orig_h' 228 | column_idx['endtime'] = '' 229 | column_idx['dur'] = 'duration' 230 | column_idx['proto'] = 'proto' 231 | column_idx['appproto'] = 'service' 232 | column_idx['srcport'] = 'id.orig_p' 233 | column_idx['dstip'] = 'id.resp_h' 234 | column_idx['dstport'] = 'id.resp_p' 235 | column_idx['state'] = 'conn_state' 236 | column_idx['pkts'] = '' 237 | column_idx['spkts'] = 'orig_pkts' 238 | column_idx['dpkts'] = 'resp_pkts' 239 | column_idx['bytes'] = '' 240 | column_idx['sbytes'] = 'orig_bytes' 241 | column_idx['dbytes'] = 'resp_bytes' 242 | column_idx['orig_ip_bytes'] = 'orig_ip_bytes' 243 | column_idx['resp_ip_bytes'] = 'resp_ip_bytes' 244 | column_idx['history'] = 'history' 245 | column_idx['fingerprint'] = 'fingerprint' 246 | 247 | # Some of the fields were not found probably, 248 | # so just delete them from the index if their value is False. 249 | # If not we will believe that we have data on them 250 | # We need a temp dict because we can not change the size of dict while analyzing it 251 | temp_dict = {} 252 | for key, value in column_idx.items(): 253 | if isinstance(value,bool) and value is False: 254 | continue 255 | temp_dict[key] = value 256 | 257 | column_idx = temp_dict 258 | 259 | return column_idx 260 | except Exception as inst: 261 | exception_line = sys.exc_info()[2].tb_lineno 262 | print(f'\tProblem in define_columns() line {exception_line}', 0, 1) 263 | print(str(type(inst)), 0, 1) 264 | print(str(inst), 0, 1) 265 | sys.exit(1) 266 | 267 | 268 | def define_type(data): 269 | """ 270 | Using heuristic detection, quickly determine the input type from the following options: 271 | Zeek file, Suricata JSON, Argus binetflow CSV, or Argus binetflow TSV. 272 | - Input: The first line after the headers if there were some, as 'data' 273 | - Outputs types can be can be: zeek-json, suricata, argus-tab, argus-csv, zeek-tab 274 | If input is JSON, it can be Zeek or Suricata 275 | If input is CSV, it can be Argus 276 | If input is TSV, it can be Argus or zeek 277 | """ 278 | input_type = 'unknown' 279 | try: 280 | # Validate if input is JSON 281 | try: 282 | json_line = json.loads(data) 283 | 284 | # Determine if logs are Zeek or Suricata 285 | try: 286 | # Validate if input are Zeek JSON logs 287 | _ = json_line['ts'] 288 | input_type = 'zeek-json' 289 | except KeyError: 290 | # Validate if input are Suricata JSON logs? 291 | _ = json_line['timestamp'] 292 | input_type = 'suricata-json' 293 | # Validate if input is CSV or TSV 294 | except json.JSONDecodeError: 295 | # Validate if input is text based 296 | if isinstance(data,str): 297 | nr_commas = len(data.split(',')) 298 | nr_tabs = len(data.split(' ')) 299 | if nr_commas > nr_tabs: 300 | # Commas is the separator 301 | if nr_commas > 40: 302 | input_type = 'nfdump-csv' 303 | else: 304 | # comma separated argus file 305 | input_type = 'argus-csv' 306 | elif nr_tabs >= nr_commas: 307 | # Tabs is the separator or it can be also equal number of commas and tabs, including both 0 308 | # Can be Zeek conn.log with TABS 309 | # Can be Argus binetflow with TABS 310 | # Can be Nfdump binetflow with TABS 311 | if '->' in data or 'StartTime' in data: 312 | input_type = 'argus-tab' 313 | elif 'separator' in data: 314 | input_type = 'zeek-tab' 315 | elif 'Date' in data: 316 | input_type = 'nfdump-tab' 317 | else: 318 | print("Exception in define_type(): unknown logs type.") 319 | sys.exit(1) 320 | 321 | # Returned guessed input log type 322 | return input_type 323 | except Exception as inst: 324 | exception_line = sys.exc_info()[2].tb_lineno 325 | print(f'\tProblem in define_type() line {exception_line}', 0, 1) 326 | print(str(type(inst)), 0, 1) 327 | print(str(inst), 0, 1) 328 | sys.exit(1) 329 | 330 | 331 | def process_zeek(column_idx, input_file, output_file, labelmachine, filetype): 332 | """ 333 | Process a Zeek file 334 | The filetype can be: 'tab', 'csv', 'json' 335 | """ 336 | try: 337 | amount_lines_processed = 0 338 | column_values = {} 339 | 340 | # Read firstlines 341 | line = input_file.readline() 342 | 343 | # Delete headerlines 344 | while '#' in line: 345 | line = input_file.readline() 346 | 347 | while line: 348 | # Count the first line 349 | amount_lines_processed += 1 350 | 351 | if args.verbose > 0: 352 | print(f'Netflow line: {line}', end='') 353 | 354 | if 'csv' in filetype or 'tab' in filetype: 355 | # Work with csv and tabs 356 | if 'csv' in filetype: 357 | separator = ',' 358 | elif 'tab' in filetype: 359 | separator = '\t' 360 | # Transform the line into an array 361 | line_values = line.split(separator) 362 | 363 | # Read values from the flow line 364 | for key in column_idx: 365 | column_values[key] = line_values[column_idx[key]] 366 | 367 | # Create the hand-made columns that are a sum of other columns 368 | # First empty them 369 | column_values['bytes'] = '' 370 | column_values['pkts'] = '' 371 | column_values['ipbytes'] = '' 372 | 373 | # Sum bytes 374 | # We do it like this because sometimes the column can be - or 0 375 | if column_values['sbytes'] == '-': 376 | sbytes = 0 377 | else: 378 | sbytes = int(column_values['sbytes']) 379 | if column_values['dbytes'] == '-': 380 | dbytes = 0 381 | else: 382 | dbytes = int(column_values['dbytes']) 383 | column_values['bytes'] = str(sbytes + dbytes) 384 | #print(f'New column bytes = {column_values["bytes"]}') 385 | 386 | # Sum packets 387 | # We do it like this because sometimes the column can be - or 0 388 | if column_values['spkts'] == '-': 389 | spkts = 0 390 | else: 391 | spkts = int(column_values['spkts']) 392 | if column_values['dpkts'] == '-': 393 | dpkts = 0 394 | else: 395 | dpkts = int(column_values['dpkts']) 396 | column_values['pkts'] = str(spkts + dpkts) 397 | #print(f'New column pkst = {column_values["pkts"]}') 398 | 399 | # Sum ip_bytes 400 | # We do it like this because sometimes the column can be - or 0 401 | if column_values['orig_ip_bytes'] == '-': 402 | sip_bytes = 0 403 | else: 404 | sip_bytes = int(column_values['orig_ip_bytes']) 405 | if column_values['resp_ip_bytes'] == '-': 406 | dip_bytes = 0 407 | else: 408 | dip_bytes = int(column_values['resp_ip_bytes']) 409 | column_values['ipbytes'] = str(sip_bytes + dip_bytes) 410 | #print(f'New column ipbytes = {column_values["ipbytes"]}') 411 | 412 | # Request a label 413 | genericlabel, detailedlabel = labelmachine.getLabel(column_values) 414 | if args.debug > 1: 415 | print(f'Label {genericlabel} assigned in line {line}') 416 | 417 | # Store the netflow 418 | output_netflow_line_to_file(output_file, line, filetype, genericlabel=genericlabel, detailedlabel=detailedlabel) 419 | 420 | line = input_file.readline() 421 | while '#' in line: 422 | line = input_file.readline() 423 | 424 | elif 'json' in filetype: 425 | # Count the first line 426 | amount_lines_processed += 1 427 | 428 | return amount_lines_processed 429 | except Exception as inst: 430 | exception_line = sys.exc_info()[2].tb_lineno 431 | print(f'\tProblem in process_zeek() line {exception_line}', 0, 1) 432 | print(str(type(inst)), 0, 1) 433 | print(str(inst), 0, 1) 434 | sys.exit(1) 435 | 436 | 437 | def cache_labeled_file(): 438 | """ 439 | Read the labeled file and store the uid and labels in a dictionary. 440 | - Input: global variable 'args.labeledfile' 441 | - Output: labels_dict 442 | """ 443 | try: 444 | if args.verbose > 0: 445 | print(f'[+] Labeled file to use: {args.labeledfile}') 446 | 447 | # Open labeled flows file and get the columns 448 | try: 449 | input_labeled_file = open(args.labeledfile,'r', encoding='utf-8') 450 | except Exception as inst: 451 | print('Some problem opening the input labeled netflow file. In cache_labeled_file()') 452 | print(type(inst)) # the exception instance 453 | print(inst.args) # arguments stored in .args 454 | print(inst) # __str__ allows args to printed directly 455 | sys.exit(-1) 456 | 457 | # Get the first header line to find the type 458 | headerline = input_labeled_file.readline() 459 | 460 | # If there are no headers, get out. Most start with '#' but Argus starts with 'StartTime' and nfdump with 'Date' 461 | if '#' not in headerline[0]: 462 | print('The labeled file has not headers. Please add them.') 463 | sys.exit(-1) 464 | 465 | # Define the type of file 466 | filetype = define_type(headerline) 467 | 468 | if args.verbose > 3: 469 | print(f'[+] Type of labeled file to use: {filetype}') 470 | 471 | # Define the columns 472 | if filetype == 'zeek-json': 473 | input_labeled_file_column_idx = define_columns(headerline, filetype='json') 474 | elif filetype == 'zeek-tab': 475 | # Get all the other headers first 476 | while '#types' not in headerline: 477 | # Go through all the # headers, but rememeber the #fields one 478 | if '#fields' in headerline: 479 | fields_headerline = headerline 480 | headerline = input_labeled_file.readline() 481 | # Get the columns indexes 482 | input_labeled_file_column_idx = define_columns(fields_headerline, filetype='tab') 483 | # Set the fields separator 484 | input_labeled_file_separator = '\t' 485 | 486 | # Read and Cache the labeled file 487 | labels_dict = {} 488 | 489 | inputline = input_labeled_file.readline() 490 | lines_with_labels_read = 0 491 | while inputline and '#' not in inputline: 492 | # Transform the line into an array 493 | line_values = inputline.split(input_labeled_file_separator) 494 | if args.debug > 8: 495 | print(f"[+] Line values: {line_values}") 496 | # Read column values from the flow line 497 | try: 498 | uid = line_values[input_labeled_file_column_idx['uid']] 499 | generic_label = line_values[input_labeled_file_column_idx['label']].strip() 500 | detailed_label = line_values[input_labeled_file_column_idx['detailedlabel']].strip() 501 | # Store the labels for this uid in the dict 502 | labels_dict[uid] = [generic_label, detailed_label] 503 | lines_with_labels_read += 1 504 | if args.debug > 6: 505 | print(f"[+] UID: {uid}. Label: {generic_label}. Detailed label: {detailed_label}") 506 | except IndexError: 507 | # Some zeek log files can have the headers only and no data. 508 | # Because we create them sometimes from larger zeek files that were filtered 509 | continue 510 | inputline = input_labeled_file.readline() 511 | 512 | if args.verbose > 1: 513 | print(f"[+] Finished reading labeled file. Read {lines_with_labels_read} lines with labels.\n") 514 | return labels_dict 515 | 516 | except Exception as inst: 517 | exception_line = sys.exc_info()[2].tb_lineno 518 | print(f'\tProblem in cache_labeled_file() line {exception_line}', 0, 1) 519 | print(str(type(inst)), 0, 1) 520 | print(str(inst), 0, 1) 521 | sys.exit(1) 522 | 523 | 524 | def process_zeekfolder(): 525 | """ 526 | This function takes the labeled conn.log file and parses it. 527 | Then it asks for a label and finally it calls a function to store the netflow in a file. 528 | 529 | Method: 530 | 1. Read the labeled file once and store the uid and labels in a dictionary 531 | 2. Read each of the files in the zeek folder, read their uid, and assign the label given to that uid in the labeled file 532 | """ 533 | try: 534 | # Get and load the dict with uid and labels 535 | labels_dict = cache_labeled_file() 536 | 537 | if args.verbose > 0: 538 | print(f"\n[+] Processing the zeek folder {args.zeekfolder} for files to label") 539 | 540 | 541 | # ----- Second, open each file in the folder, and label them. 542 | # Get the list of files in this folder 543 | zeekfiles = [f for f in listdir(args.zeekfolder) if isfile(join(args.zeekfolder, f))] 544 | 545 | lines_labeled = 0 546 | uid_without_label = 0 547 | 548 | for zeekfile_name in zeekfiles: 549 | 550 | # Ignore the following files 551 | ignore_keywords = ['.labeled', 'services', 'summary', 'conn.log', 'capture_loss.log', 552 | 'loaded_scripts.log', 'packet_filter.log', 'stats.log', 'reporter.log'] 553 | 554 | if any(keyword in zeekfile_name for keyword in ignore_keywords): 555 | continue 556 | 557 | # Ignore empty files 558 | if os.stat(join(args.zeekfolder, zeekfile_name)).st_size == 0: 559 | continue 560 | 561 | if args.verbose > 0: 562 | print(f'[+] Processing zeek file: {zeekfile_name}') 563 | 564 | try: 565 | zeekfile = open(join(args.zeekfolder, zeekfile_name),'r', encoding='utf-8') 566 | except Exception as inst: 567 | print(f'Some problem opening a zeek file {zeekfile_name}. In process_zeekfolder()') 568 | print(type(inst)) # the exception instance 569 | print(inst.args) # arguments stored in .args 570 | print(inst) # __str__ allows args to printed directly 571 | sys.exit(-1) 572 | 573 | # Get the first header line to find the type 574 | headerline = zeekfile.readline() 575 | 576 | # If there are no headers, get out. Most start with '#' but Argus starts with 'StartTime' and nfdump with 'Date' 577 | if '#' not in headerline[0]: 578 | print('The file has not headers. Please add them.') 579 | sys.exit(-1) 580 | 581 | # ---- Define the type of file 582 | filetype = define_type(headerline) 583 | if args.verbose > 3: 584 | print(f'[+] Type of flow file to label: {filetype}') 585 | 586 | # Create the output file for all cases 587 | output_file = open(join(args.zeekfolder, zeekfile_name+'.labeled'),'w', encoding='utf-8') 588 | if args.debug > 1: 589 | print(f"[+] Output file created: {join(args.zeekfolder, zeekfile_name+'.labeled')}") 590 | 591 | # Store the first header line in the output file 592 | output_netflow_line_to_file(output_file, headerline) 593 | 594 | # ---- Define the columns of this file 595 | if filetype == 'zeek-json': 596 | column_idx = define_columns(headerline, filetype='json') 597 | elif filetype == 'zeek-tab': 598 | # ---- Get all the headers lines and store them in the output file 599 | while '#types' not in headerline: 600 | # Go through all the # headers, but rememeber the #fields one 601 | if '#fields' in headerline: 602 | fields_headerline = headerline 603 | headerline = zeekfile.readline() 604 | # Store the rest of the zeek headers in the output file 605 | output_netflow_line_to_file(output_file, headerline, filetype='tab') 606 | # ---- Get the columns indexes for each colum 607 | column_idx = define_columns(fields_headerline, filetype='tab') 608 | zeek_file_file_separator = '\t' 609 | 610 | # ---- For the majority of zeek log files, using the UID from conn.log to find the related flow is ok 611 | # ---- But it is not for x509.log and files.log. 612 | 613 | if zeekfile_name == 'x509.log': 614 | line_to_label = zeekfile.readline().strip() 615 | while line_to_label and '#' not in line_to_label[0]: 616 | # Transform the line into an array 617 | line_values = line_to_label.split(zeek_file_file_separator) 618 | if args.debug > 5: 619 | print(f"[+] line values: {line_values}") 620 | 621 | # Read column values from the line to label 622 | try: 623 | fingerprint = line_values[column_idx['fingerprint']] 624 | if args.debug > 5: 625 | print(f"[+] got the fingerprint: {fingerprint}") 626 | 627 | #if args.verbose > 5: 628 | #print(f"[+] Greping {fingerprint} in file {join(args.zeekfolder, zeekfile_name)}") 629 | command = 'grep ' + fingerprint + ' ' + join(args.zeekfolder, 'ssl.log') 630 | result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=True) 631 | result = result.stdout.decode('utf-8') 632 | #if args.verbose > 5: 633 | #print(f"\t[+] Result {result}") 634 | 635 | # Using this fingerprint find the uid of the ssl line 636 | uid = result.split('\t')[1] 637 | 638 | # Using this uid, find the label for the conn.log line 639 | try: 640 | # Get the labels 641 | generic_label_to_assign = labels_dict[uid][0] 642 | detailed_label_to_assign = labels_dict[uid][1] 643 | except KeyError: 644 | # There is no label for this uid! 645 | generic_label_to_assign = '(empty)' 646 | detailed_label_to_assign = '(empty)' 647 | uid_without_label += 1 648 | if args.debug > 1: 649 | print(f"There is no label for this uid: {uid}") 650 | 651 | if args.debug > 3: 652 | print(f"[+] To label UID: {uid}. Label: {generic_label_to_assign}. Detailed label: {detailed_label_to_assign}") 653 | # Store the rest of the zeek line in the output file 654 | output_netflow_line_to_file(output_file, line_to_label, filetype='tab', genericlabel=generic_label_to_assign, detailedlabel=detailed_label_to_assign) 655 | lines_labeled += 1 656 | except (IndexError, KeyError): 657 | # Some zeek log files can have the headers only and no data. 658 | # Because we create them sometimes from larger zeek files that were filtered 659 | pass 660 | line_to_label = zeekfile.readline().strip() 661 | if zeekfile_name in ('ocsp.log', 'pe.log'): 662 | line_to_label = zeekfile.readline().strip() 663 | while line_to_label and '#' not in line_to_label[0]: 664 | # Transform the line into an array 665 | line_values = line_to_label.split(zeek_file_file_separator) 666 | if args.debug > 5: 667 | print(f"[+] line values: {line_values}") 668 | 669 | # Read column values from the line to label 670 | try: 671 | file_id = line_values[column_idx['id']] 672 | if args.debug > 5: 673 | print(f"[+] got the file ID: {file_id}") 674 | 675 | #if args.verbose > 5: 676 | #print(f"[+] Greping {file_id} in file {join(args.zeekfolder, zeekfile_name)}") 677 | command = 'grep ' + file_id + ' ' + join(args.zeekfolder, 'files.log') 678 | result = subprocess.run(command.split(), stdout=subprocess.PIPE, check=True) 679 | result = result.stdout.decode('utf-8') 680 | #if args.verbose > 5: 681 | #print(f"\t[+] Result {result}") 682 | 683 | # Using this file_id find the uid of the ssl line 684 | uid = result.split('\t')[4] 685 | 686 | # Using this uid, find the label for the conn.log line 687 | try: 688 | # Get the labels 689 | generic_label_to_assign = labels_dict[uid][0] 690 | detailed_label_to_assign = labels_dict[uid][1] 691 | except KeyError: 692 | # There is no label for this uid! 693 | generic_label_to_assign = '(empty)' 694 | detailed_label_to_assign = '(empty)' 695 | uid_without_label += 1 696 | if args.debug > 1: 697 | print(f"There is no label for this uid: {uid}") 698 | 699 | if args.debug > 3: 700 | print(f"[+] To label UID: {uid}. Label: {generic_label_to_assign}. Detailed label: {detailed_label_to_assign}") 701 | # Store the rest of the zeek line in the output file 702 | output_netflow_line_to_file(output_file, line_to_label, filetype='tab', genericlabel=generic_label_to_assign, detailedlabel=detailed_label_to_assign) 703 | lines_labeled += 1 704 | except (IndexError, KeyError): 705 | # Some zeek log files can have the headers only and no data. 706 | # Because we create them sometimes from larger zeek files that were filtered 707 | pass 708 | line_to_label = zeekfile.readline().strip() 709 | 710 | else: 711 | # ---- Read the lines from the rest of log files to label 712 | 713 | # Read each line of the labeled file and get the zeek uid 714 | line_to_label = zeekfile.readline().strip() 715 | 716 | while line_to_label and '#' not in line_to_label[0]: 717 | # Transform the line into an array 718 | line_values = line_to_label.split(zeek_file_file_separator) 719 | if args.debug > 5: 720 | print(f"[+] Line values: {line_values}") 721 | 722 | # Read column values from the zeek line 723 | try: 724 | if zeekfile_name == 'files.log': 725 | uid = line_values[column_idx['conn_uids']] 726 | elif zeekfile_name == 'dhcp.log': 727 | uid = line_values[column_idx['uids']] 728 | else: 729 | uid = line_values[column_idx['uid']] 730 | 731 | lines_labeled += 1 732 | 733 | try: 734 | # Get the labels 735 | generic_label_to_assign = labels_dict[uid][0] 736 | detailed_label_to_assign = labels_dict[uid][1] 737 | except KeyError: 738 | # There is no label for this uid! 739 | generic_label_to_assign = '(empty)' 740 | detailed_label_to_assign = '(empty)' 741 | uid_without_label += 1 742 | if args.debug > 1: 743 | print(f"There is no label for this uid: {uid}") 744 | 745 | if args.debug > 3: 746 | print(f"[+] To label UID: {uid}. Label: {generic_label_to_assign}. Detailed label: {detailed_label_to_assign}") 747 | # Store the rest of the zeek line in the output file 748 | output_netflow_line_to_file(output_file, line_to_label, filetype='tab', genericlabel=generic_label_to_assign, detailedlabel=detailed_label_to_assign) 749 | except (IndexError, KeyError): 750 | # Some zeek log files can have the headers only and no data. 751 | # Because we create them sometimes from larger zeek files that were filtered 752 | pass 753 | line_to_label = zeekfile.readline().strip() 754 | 755 | # Store the last header back 756 | if line_to_label and '#' in headerline[0]: 757 | # Store the rest of the zeek headers in the output file 758 | output_netflow_line_to_file(output_file, line_to_label, filetype='tab') 759 | 760 | 761 | 762 | if args.verbose > 0: 763 | print(f"[+] Read all labeled files. Labeled {lines_labeled} lines in total. UID without label {uid_without_label}") 764 | 765 | # Close the input file 766 | zeekfile.close() 767 | # Close the outputfile 768 | output_file.close() 769 | 770 | except Exception as inst: 771 | exception_line = sys.exc_info()[2].tb_lineno 772 | print(f'Problem in process_zeekfolder() line {exception_line}', 0, 1) 773 | print(type(inst)) # the exception instance 774 | print(inst.args) # arguments stored in .args 775 | print(inst) # __str__ allows args to printed directly 776 | sys.exit(-1) 777 | 778 | 779 | if __name__ == '__main__': 780 | print(f"Zeek Files labeler from labeled conn.log.labeled file. Version {VERSION}") 781 | print("https://stratosphereips.org") 782 | 783 | # Parse the parameters 784 | parser = argparse.ArgumentParser(description="Given a conn.log.labeled file, copy those labels to the rest of the Zeek log files", add_help=False) 785 | parser.add_argument('-l','--labeledfile', metavar='', action='store', required=True, help='path to labeled conn.log file.') 786 | parser.add_argument('-v', '--verbose',metavar='',action='store', required=False, type=int, default=0, help='amount of verbosity. This shows more info about the results.') 787 | parser.add_argument('-d', '--debug', action='store', required=False, type=int, default=0, help='amount of debugging. This shows inner information about the program.') 788 | parser.add_argument('-f', '--zeekfolder',metavar='', action='store', required=True, help='folder with Zeek files to label.') 789 | parser.add_argument("-h", "--help", action="help", help="command line help") 790 | args = parser.parse_args() 791 | 792 | try: 793 | # Process zeek files in the folder 794 | process_zeekfolder() 795 | 796 | except KeyboardInterrupt: 797 | # CTRL-C pretty handling. 798 | print("Keyboard Interruption!. Exiting.") 799 | sys.exit(1) 800 | --------------------------------------------------------------------------------