├── CHANGELOG.md ├── Dockerfile.nprobe_py ├── Dockerfile.ntopng ├── LICENSE ├── README.md ├── docker-compose.yml ├── install_ntop.sh └── nprobe_py.py /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Changelog 2 | ### 0.0.4 3 | * securing nprobe_py image 4 | ### 0.0.3 5 | * adding -T parameter with @NTOPNG@ template 6 | * adding verbose dependent printing 7 | * removing redunant code 8 | * Netflow/set == 2 and Netflow/set == 3 are sharing record scan 9 | * NetFlow IPFIX 10 | * adding expected sequence number, need to wait for set=2 or 3 to properly handle it 11 | * one NetFlow message means now one zmq message 12 | * fixed performance timers in case of multiple sets within NetFlow message 13 | * general performance fixes 14 | * switching from json to orjson 15 | ### 0.0.2 16 | * protocols support 17 | * NetfFow/set == 3 18 | * support for scopes 19 | * ntop/zmq with v1 header 20 | * source_id - to identifiy nprobe_py within ntop 21 | * msg_id - to monitor dropping of ntop/zmq messages 22 | * parameter: 23 | * --zmq-source-id (default = 1) 24 | ### 0.0.1 25 | * initial release 26 | -------------------------------------------------------------------------------- /Dockerfile.nprobe_py: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm 2 | 3 | ENV LANG C.UTF-8 4 | 5 | RUN \ 6 | apt-get update \ 7 | && apt-get upgrade -y \ 8 | && apt-get install -y -q \ 9 | python3 \ 10 | python3-pip \ 11 | && pip install --target=/app --break-system-packages \ 12 | pyzmq \ 13 | orjson \ 14 | && pip cache purge \ 15 | && apt-get clean \ 16 | && rm -rf /tmp/* \ 17 | && rm -rf /var/tmp/* \ 18 | && rm -rf /var/lib/apt/lists/* 19 | 20 | FROM debian:bookworm 21 | 22 | RUN \ 23 | apt-get update \ 24 | && apt-get upgrade -y \ 25 | && apt-get install -y -q python3 \ 26 | && apt-get clean \ 27 | && rm -rf /tmp/* \ 28 | && rm -rf /var/tmp/* \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | COPY --from=0 /app /app 32 | COPY nprobe_py.py /app/nprobe_py.py 33 | 34 | USER nobody 35 | 36 | -------------------------------------------------------------------------------- /Dockerfile.ntopng: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm 2 | 3 | ENV LANG C.UTF-8 4 | 5 | COPY install_ntop.sh /install_ntop.sh 6 | 7 | RUN \ 8 | apt update \ 9 | && apt upgrade -y \ 10 | && apt install -y -q wget lsb-release gnupg \ 11 | && cd /tmp \ 12 | && /install_ntop.sh \ 13 | && apt clean \ 14 | && rm -rf /tmp/* \ 15 | && rm -rf /var/tmp/* \ 16 | && rm -rf /var/lib/apt/lists/* 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nprobe_py 0.0.4 2 | 3 | License: http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | To some extent it is replacement to nprobe for ntop.org. 6 | 7 | It takes NetFlow/IPFIX stream and makes it available as ZMQ for ntop. 8 | 9 | Code found here is like "Cutting the grass with machete" and definitely could be prettier, but it simply gets the job done, so don't complain about it too much. 10 | 11 | ## Usage: 12 | ### Build images 13 | 14 | docker-compose build 15 | #or 16 | docker compose build 17 | 18 | ### Start 19 | 20 | docker-compose up 21 | #or 22 | docker compose up 23 | 24 | or in the backround 25 | 26 | docker-compose up -d 27 | #or 28 | docker compose up -d 29 | 30 | ## stop 31 | docker-compose stop 32 | #or 33 | docker compose stop 34 | 35 | ### Remove 36 | docker-compose rm 37 | or 38 | docker compose rm 39 | 40 | ### Access 41 | 42 | #### ntop WebUI address 43 | 44 | http://{host}:3000 45 | 46 | 47 | #### NetFlow/IPFIX collector 48 | 49 | udp://{host}:2055 50 | 51 | ### Additional Configuration 52 | 53 | #### ntopng and NAT 54 | 55 | \1 Go to Web ui 56 | 57 | http://{host}:3000 58 | 59 | \2 navigate through menu 60 | 61 | Settings -> Preferences 62 | 63 | \3 click Expert View 64 | 65 | \4 go further through menu 66 | 67 | Network Interfaces 68 | 69 | \5 check options 70 | 71 | Use Post-Nat Destination IPv4 Addresses and Ports: [ X ] 72 | Use Post-Nat Source IPv4 Addresses and Ports: [ X ] 73 | 74 | \6 click save 75 | 76 | #### Mikrotik 7.X for NetFlow/IPFIX stream 77 | 78 | \1 Go to Web ui 79 | 80 | http://{mikrotik} 81 | 82 | \2 Navigate through menu 83 | 84 | WebFig -> IP -> Traffic Flow -> Targets -> Add New 85 | 86 | \3 Set options 87 | 88 | Enabled: [x] 89 | Src. Address: {ip of Mikrotik} 90 | Dst. Address: {ip of host} 91 | port: 2055 92 | Version: IPFIX 93 | v9/IPFIX Template Refresh: 20 94 | v9/IPFIX Template Timeout: 1800 95 | 96 | \4 Click OK 97 | 98 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | nprobe_py: 5 | image: nprobe_py:latest 6 | command: [ 7 | "/app/nprobe_py.py", 8 | "--performance", 9 | "--ntopng", "tcp://*:1234", 10 | "--collector-port", "udp://0.0.0.0:2055", 11 | "-T", "@NTOPNG@" 12 | ] 13 | ports: 14 | - 2055:2055/udp 15 | build: 16 | context: ./ 17 | dockerfile: Dockerfile.nprobe_py 18 | 19 | redis: 20 | image: "redis:alpine" 21 | volumes: 22 | - ./vol_redis:/data:rw 23 | 24 | ntopng: 25 | image: "ntopng:latest" 26 | command: [ 27 | "ntopng", 28 | '--community', 29 | "--redis", "redis", 30 | "--interface", "tcp://nprobe_py:1234" , 31 | '--local-networks', '192.168.1.0/24=LAN,10.1.1.0/24=LAN_SENSORS' 32 | ] 33 | ports: 34 | - "3000:3000" 35 | depends_on: 36 | - redis 37 | - nprobe_py 38 | volumes: 39 | - ./vol_ntopng:/var/lib/ntopng:rw 40 | build: 41 | context: ./ 42 | dockerfile: Dockerfile.ntopng 43 | 44 | -------------------------------------------------------------------------------- /install_ntop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | arch=`dpkg --print-architecture` 6 | 7 | case $arch in 8 | arm64) 9 | #currently only in unstable channels 10 | channel=apt 11 | pkg=apt-ntop.deb 12 | ;; 13 | amd64) 14 | channel=apt-stable 15 | pkg=apt-ntop-stable.deb 16 | ;; 17 | *) 18 | echo unnknown architecture 19 | exit 1 20 | ;; 21 | esac 22 | 23 | wget https://packages.ntop.org/${channel}/bookworm/all/${pkg} 24 | apt install -y ./${pkg} 25 | apt update 26 | 27 | apt install -y ntopng 28 | 29 | -------------------------------------------------------------------------------- /nprobe_py.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024 Rafal Prasal 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import socket 18 | import os 19 | import sys 20 | import time 21 | import zmq 22 | import multiprocessing as mp 23 | import orjson 24 | import zlib 25 | import ipaddress 26 | import argparse 27 | import re 28 | 29 | #https://stackoverflow.com/questions/107705/disable-output-buffering 30 | buf_arg = 0 31 | if sys.version_info[0] == 3: 32 | os.environ['PYTHONUNBUFFERED'] = '1' 33 | buf_arg = 1 34 | 35 | sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buf_arg) 36 | sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buf_arg) 37 | 38 | #https://www.rfc-editor.org/rfc/rfc7011 39 | #https://www.rfc-editor.org/rfc/rfc7012 40 | #https://www.iana.org/assignments/ipfix/ipfix.xhtml 41 | 42 | def unknown(bytes): 43 | return f'0x{bytes.hex()}' 44 | 45 | def hexadecimal(bytes): 46 | return f'0x{bytes.hex()}' 47 | 48 | def unsigned(bytes): 49 | return int.from_bytes(bytes, 'big') 50 | 51 | def macaddress(bytes): 52 | return ':'.join(f'{b:02x}' for b in bytes) 53 | 54 | def ipv4address(bytes): 55 | return ipaddress.IPv4Address(bytes).exploded 56 | 57 | def ipv6address(bytes): 58 | return ipaddress.IPv6Address(bytes).exploded 59 | 60 | #https://www.ntop.org/guides/nprobe/flow_information_elements.html 61 | #https://www.iana.org/assignments/ipfix/ipfix.xhtml 62 | IPFIX_fields = { 63 | -1: { #default Enterprise 64 | 1: { 65 | 'name': 'octetDeltaCount', 66 | 'from_bytes': unsigned, 67 | 'nprobe': 'IN_BYTES' 68 | }, 69 | 2: { 70 | 'name': 'packetDeltaCount', 71 | 'from_bytes': unsigned, 72 | 'nprobe': 'IN_PKTS' 73 | }, 74 | 4: { 75 | 'name': 'protocolIdentifier', 76 | 'from_bytes': unsigned, 77 | 'nprobe': 'PROTOCOL' 78 | }, 79 | 5: { 80 | 'name': 'ipClassOfService', 81 | 'from_bytes': unsigned, 82 | 'nprobe': 'SRC_TOS' 83 | }, 84 | 6: { 85 | 'name': 'tcpControlBits', 86 | 'from_bytes': hexadecimal, 87 | 'nprobe': 'TCP_FLAGS' 88 | }, 89 | 7: { 90 | 'name': 'sourceTransportPort', 91 | 'from_bytes': unsigned, 92 | 'nprobe': 'L4_SRC_PORT' 93 | }, 94 | 8: { 95 | 'name': 'sourceIPv4Address', 96 | 'from_bytes': ipv4address, 97 | 'nprobe': 'IPV4_SRC_ADDR' 98 | }, 99 | 9: { 100 | 'name': 'sourceIPv4PrefixLength', 101 | 'from_bytes': unsigned, 102 | 'nprobe': 'IPV4_SRC_MASK' 103 | }, 104 | 10: { 105 | 'name': 'ingressInterface', 106 | 'from_bytes': unsigned, 107 | 'nprobe': 'INPUT_SNMP' 108 | }, 109 | 11: { 110 | 'name': 'destinationTransportPort', 111 | 'from_bytes': unsigned, 112 | 'nprobe': 'L4_DST_PORT' 113 | }, 114 | 12: { 115 | 'name': 'destinationIPv4Address', 116 | 'from_bytes': ipv4address, 117 | 'nprobe': 'IPV4_DST_ADDR' 118 | }, 119 | 13: { 120 | 'name': 'destinationIPv4PrefixLength', 121 | 'from_bytes': unsigned, 122 | 'nprobe': 'IPV4_DST_MASK' 123 | }, 124 | 14: { 125 | 'name': 'egressInterface', 126 | 'from_bytes': unsigned, 127 | 'nprobe': 'OUTPUT_SNMP' 128 | }, 129 | 15: { 130 | 'name': 'ipNextHopIPv4Address', 131 | 'from_bytes': ipv4address, 132 | 'nprobe': 'IPV4_NEXT_HOP' 133 | }, 134 | 16: { 135 | 'name': 'bgpSourceAsNumber', 136 | 'from_bytes': unsigned, 137 | 'nprobe': 'SRC_AS' 138 | }, 139 | 17: { 140 | 'name': 'DST_AS', 141 | 'from_bytes': unsigned, 142 | 'nprobe': 'DST_AS' 143 | }, 144 | 18: { 145 | 'name': 'bgpNexthopIPv4Address', 146 | 'from_bytes': ipv4address, 147 | 'nprobe': 'BGP_IPV4_NEXT_HOP' 148 | }, 149 | 21: { 150 | 'name': 'flowEndSysUpTime', 151 | 'from_bytes': unsigned, 152 | 'nprobe': 'LAST_SWITCHED' 153 | }, 154 | 22: { 155 | 'name': 'flowStartSysUpTime', 156 | 'from_bytes': unsigned, 157 | 'nprobe': 'FIRST_SWITCHED' 158 | }, 159 | 27: { 160 | 'name': 'sourceIPv6Address', 161 | 'from_bytes': ipv6address, 162 | 'nprobe': 'IPV6_SRC_ADDR' 163 | }, 164 | 28: { 165 | 'name': 'destinationIPv6Address', 166 | 'from_bytes': ipv6address, 167 | 'nprobe': 'IPV6_DST_ADDR' 168 | }, 169 | 29: { 170 | 'name': 'sourceIPv6PrefixLength', 171 | 'from_bytes': unsigned, 172 | 'nprobe': 'IPV6_SRC_MASK' 173 | }, 174 | 30: { 175 | 'name': 'destinationIPv6PrefixLength', 176 | 'from_bytes': unsigned, 177 | 'nprobe': 'IPV6_DST_MASK' 178 | }, 179 | 31: { 180 | 'name': 'flowLabelIPv6', 181 | 'from_bytes': hexadecimal 182 | }, 183 | 32: { 184 | 'name': 'icmpTypeCodeIPv4', 185 | 'from_bytes': unsigned, 186 | 'nprobe': 'ICMP_TYPE' 187 | }, 188 | 33: { 189 | 'name': 'igmpType', 190 | 'from_bytes': unsigned 191 | }, 192 | 34: { 193 | 'name': 'SAMPLING_INTERVAL', 194 | 'from_bytes': unsigned, 195 | 'nprobe': 'SAMPLING_INTERVAL' 196 | }, 197 | 35: { 198 | 'name': 'SAMPLING_ALGORITHM', 199 | 'from_bytes': unsigned, 200 | 'nprobe': 'SAMPLING_ALGORITHM' 201 | }, 202 | 36: { 203 | 'name': 'flowActiveTimeout', 204 | 'from_bytes': unsigned, 205 | 'nprobe': 'FLOW_ACTIVE_TIMEOUT' 206 | }, 207 | 37: { 208 | 'name': 'flowIdleTimeout', 209 | 'from_bytes': unsigned, 210 | 'nprobe': 'FLOW_INACTIVE_TIMEOUT' 211 | }, 212 | 38: { 213 | 'name': 'ENGINE_TYPE', 214 | 'from_bytes': unsigned, 215 | 'nprobe': 'ENGINE_TYPE' 216 | }, 217 | 39: { 218 | 'name': 'ENGINE_ID', 219 | 'from_bytes': unsigned, 220 | 'nprobe': 'ENGINE_ID' 221 | }, 222 | 40: { 223 | 'name': 'exportedOctetTotalCount', 224 | 'from_bytes': unsigned, 225 | 'nprobe': 'TOTAL_BYTES_EXP' 226 | }, 227 | 41: { 228 | 'name': 'exportedMessageTotalCount', 229 | 'from_bytes': unsigned, 230 | 'nprobe': 'TOTAL_PKTS_EXP' 231 | }, 232 | 42: { 233 | 'name': 'exportedFlowRecordTotalCount', 234 | 'from_bytes': unsigned, 235 | 'nprobe': 'TOTAL_FLOWS_EXP' 236 | }, 237 | 44: { 238 | 'name': 'sourceIPv4Prefix', 239 | 'from_bytes': ipv4address, 240 | }, 241 | 45: { 242 | 'name': 'destinationIPv4Prefix', 243 | 'from_bytes': ipv4address, 244 | }, 245 | 46: { 246 | 'name': 'mplsTopLabelType', 247 | 'from_bytes': unsigned, 248 | }, 249 | 52: { 250 | 'name': 'minimumTTL', 251 | 'from_bytes': unsigned, 252 | 'nprobe': 'TOTAL_FLOWS_EXP' 253 | }, 254 | 53: { 255 | 'name': 'MIN_TTL', 256 | 'from_bytes': unsigned, 257 | 'nprobe': 'MAX_TTL' 258 | }, 259 | 55: { 260 | 'name': 'ipClassOfService', 261 | 'from_bytes': unsigned, 262 | 'nprobe': 'DST_TOS' 263 | }, 264 | 56: { 265 | 'name': 'sourceMacAddress', 266 | 'from_bytes': macaddress, 267 | 'nprobe': 'IN_SRC_MAC' 268 | }, 269 | 57: { 270 | 'name': 'postDestinationMacAddress', 271 | 'from_bytes': macaddress, 272 | 'nprobe': 'OUT_DST_MAC' 273 | }, 274 | 58: { 275 | 'name': 'vlanId', 276 | 'from_bytes': unsigned, 277 | 'nprobe': 'SRC_VLAN' 278 | }, 279 | 59: { 280 | 'name': 'postVlanId', 281 | 'from_bytes': unsigned, 282 | 'nprobe': 'DST_VLAN' 283 | }, 284 | 60: { 285 | 'name': 'ipVersion', 286 | 'from_bytes': unsigned, 287 | 'nprobe': 'IP_PROTOCOL_VERSION' 288 | }, 289 | 61: { 290 | 'name': 'flowDirection', 291 | 'from_bytes': unsigned, 292 | 'nprobe': 'DIRECTION' 293 | }, 294 | 62: { 295 | 'name': 'ipNextHopIPv6Address', 296 | 'from_bytes': ipv6address, 297 | 'nprobe': 'IPV6_NEXT_HOP' 298 | }, 299 | 70: { 300 | 'name': 'mplsTopLabelStackSection', 301 | 'from_bytes': unsigned, 302 | 'nprobe': 'MPLS_LABEL_1' 303 | }, 304 | 71: { 305 | 'name': 'mplsLabelStackSection2', 306 | 'from_bytes': unsigned, 307 | 'nprobe': 'MPLS_LABEL_2' 308 | }, 309 | 72: { 310 | 'name': 'mplsLabelStackSection3', 311 | 'from_bytes': unsigned, 312 | 'nprobe': 'MPLS_LABEL_3' 313 | }, 314 | 73: { 315 | 'name': 'mplsLabelStackSection4', 316 | 'from_bytes': unsigned, 317 | 'nprobe': 'MPLS_LABEL_4' 318 | }, 319 | 74: { 320 | 'name': 'mplsLabelStackSection5', 321 | 'from_bytes': unsigned, 322 | 'nprobe': 'MPLS_LABEL_5' 323 | }, 324 | 75: { 325 | 'name': 'mplsLabelStackSection6', 326 | 'from_bytes': unsigned, 327 | 'nprobe': 'MPLS_LABEL_6' 328 | }, 329 | 76: { 330 | 'name': 'mplsLabelStackSection7', 331 | 'from_bytes': unsigned, 332 | 'nprobe': 'MPLS_LABEL_7' 333 | }, 334 | 77: { 335 | 'name': 'mplsLabelStackSection8', 336 | 'from_bytes': unsigned, 337 | 'nprobe': 'MPLS_LABEL_8' 338 | }, 339 | 78: { 340 | 'name': 'mplsLabelStackSection9', 341 | 'from_bytes': unsigned, 342 | 'nprobe': 'MPLS_LABEL_9' 343 | }, 344 | 79: { 345 | 'name': 'mplsLabelStackSection10', 346 | 'from_bytes': unsigned, 347 | 'nprobe': 'MPLS_LABEL_10' 348 | }, 349 | 80: { 350 | 'name': 'destinationMacAddress', 351 | 'from_bytes': macaddress, 352 | 'nprobe': 'IN_DST_MACADDRESS' 353 | }, 354 | 81: { 355 | 'name': 'postSourceMacAddress', 356 | 'from_bytes': macaddress, 357 | 'nprobe': 'OUT_SRC_MAC' 358 | }, 359 | 82: { 360 | 'name': 'interfaceName', 361 | 'from_bytes': str, 362 | 'nprobe': 'INTERFACE_NAME' 363 | }, 364 | 95: { 365 | 'name': 'application_id', 366 | 'from_bytes': unsigned, 367 | 'nprobe': 'APPLICATION_ID' 368 | }, 369 | 102: { 370 | 'name': 'PACKET_SECTION_OFFSET', 371 | 'from_bytes': unsigned, 372 | 'nprobe': 'PACKET_SECTION_OFFSET' 373 | }, 374 | 103: { 375 | 'name': 'SAMPLED_PACKET_SIZE', 376 | 'from_bytes': unsigned, 377 | 'nprobe': 'SAMPLED_PACKET_SIZE' 378 | }, 379 | 104: { 380 | 'name': 'SAMPLED_PACKET_ID', 381 | 'from_bytes': unsigned, 382 | 'nprobe': 'SAMPLED_PACKET_ID' 383 | }, 384 | 130: { 385 | 'nprobe': 'EXPORTER_IPV4_ADDRESS', 386 | 'from_bytes': ipv4address, 387 | 'name': 'exporterIPv4Address' 388 | }, 389 | 131: { 390 | 'nprobe': 'EXPORTER_IPV6_ADDRESS', 391 | 'from_bytes': ipv6address, 392 | 'name': 'exporterIPv6Address' 393 | }, 394 | 148: { 395 | 'name': 'flowId', 396 | 'from_bytes': unsigned, 397 | 'nprobe': 'FLOW_ID' 398 | }, 399 | 150: { 400 | 'name': 'flowStartSeconds', 401 | 'from_bytes': unsigned, 402 | 'nprobe': 'FLOW_START_SEC' 403 | }, 404 | 151: { 405 | 'name': 'flowEndSeconds', 406 | 'from_bytes': unsigned, 407 | 'nprobe': 'FLOW_END_SEC' 408 | }, 409 | 152: { 410 | 'name': 'flowStartMilliseconds', 411 | 'from_bytes': unsigned, 412 | 'nprobe': 'FLOW_START_MILLISECONDS' 413 | }, 414 | 153: { 415 | 'name': 'flowEndMilliseconds', 416 | 'from_bytes': unsigned, 417 | 'nprobe': 'FLOW_END_MILLISECONDS' 418 | }, 419 | 154: { 420 | 'name': 'flowStartMicroseconds', 421 | 'from_bytes': unsigned, 422 | 'nprobe': 'FLOW_START_MILLISECONDS' 423 | }, 424 | 155: { 425 | 'name': 'flowEndMicroseconds', 426 | 'from_bytes': unsigned, 427 | 'nprobe': 'FLOW_END_MICROSECONDS' 428 | }, 429 | 160: { 430 | 'name': 'systemInitTimeMilliseconds', 431 | 'from_bytes': unsigned, 432 | }, 433 | 161: { 434 | 'name': 'flowDurationMilliseconds', 435 | 'from_bytes': unsigned, 436 | 'nprobe': 'FLOW_DURATION_MILLISECONDS' 437 | }, 438 | 162: { 439 | 'name': 'flowDurationMicroseconds', 440 | 'from_bytes': unsigned, 441 | 'nprobe': 'FLOW_DURATION_MICROSECONDS' 442 | }, 443 | 176: { 444 | 'name': 'icmpTypeIPv4', 445 | 'from_bytes': unsigned, 446 | 'nprobe': 'ICMP_IPV4_TYPE' 447 | }, 448 | 177: { 449 | 'name': 'icmpCodeIPv4', 450 | 'from_bytes': unsigned, 451 | 'nprobe': 'ICMP_IPV4_TYPE' 452 | 453 | }, 454 | 178: { 455 | 'name': 'icmpTypeIPv6', 456 | 'from_bytes': unsigned 457 | }, 458 | 179: { 459 | 'name': 'icmpCodeIPv6', 460 | 'from_bytes': unsigned 461 | }, 462 | 184: { 463 | 'name': 'tcpSequenceNumber', 464 | 'from_bytes': unsigned 465 | }, 466 | 185: { 467 | 'name': 'tcpAcknowledgementNumber', 468 | 'from_bytes': unsigned 469 | }, 470 | 186: { 471 | 'name': 'tcpWindowSize', 472 | 'from_bytes': unsigned 473 | }, 474 | 189: { 475 | 'name': 'ipHeaderLength', 476 | 'from_bytes': unsigned, 477 | }, 478 | 192: { 479 | 'name': 'ipTTL', 480 | 'from_bytes': unsigned, 481 | }, 482 | 205: { 483 | 'name': 'udpMessageLength', 484 | 'from_bytes': unsigned, 485 | }, 486 | 206: { 487 | 'name': 'isMulticast', 488 | 'from_bytes': unsigned 489 | }, 490 | 224: { 491 | 'name': 'ipTotalLength', 492 | 'from_bytes': unsigned 493 | }, 494 | 225: { 495 | 'name': 'postNATSourceIPv4Address', 496 | 'from_bytes': ipv4address, 497 | 'nprobe': 'POST_NAT_SRC_IPV4_ADDR' 498 | }, 499 | 226: { 500 | 'name': 'postNATDestinationIPv4Address', 501 | 'from_bytes': ipv4address, 502 | 'nprobe': 'POST_NAT_DST_IPV4_ADDR' 503 | }, 504 | 227: { 505 | 'name': 'postNAPTSourceTransportPort', 506 | 'from_bytes': unsigned, 507 | 'nprobe': 'POST_NAPT_SRC_TRANSPORT_PORT' 508 | }, 509 | 228: { 510 | 'name': 'postNAPTDestinationTransportPort', 511 | 'from_bytes': unsigned, 512 | 'nprobe': 'POST_NAPT_DST_TRANSPORT_PORT' 513 | }, 514 | 229: { 515 | 'name': 'natOriginatingAddressRealm', 516 | 'from_bytes': unsigned, 517 | 'nprobe': 'NAT_ORIGINATING_ADDRESS_REALM' 518 | }, 519 | 230: { 520 | 'name': 'natEvent', 521 | 'from_bytes': unsigned, 522 | 'nprobe': 'NAT_EVENT' 523 | }, 524 | 233: { 525 | 'name': 'firewallEvent', 526 | 'from_bytes': unsigned, 527 | 'nprobe': 'FIREWALL_EVENT' 528 | }, 529 | 234: { 530 | 'name': 'ingressVRFID', 531 | 'from_bytes': unsigned, 532 | 'nprobe': 'INGRESS_VRFID' 533 | }, 534 | 239: { 535 | 'name': 'biflow_direction', 536 | 'from_bytes': unsigned, 537 | 'nprobe': 'BIFLOW_DIRECTION' 538 | }, 539 | 243: { 540 | 'name': 'dot1qVlanId', 541 | 'from_bytes': unsigned, 542 | 'nprobe': 'DOT1Q_SRC_VLAN' 543 | }, 544 | 254: { 545 | 'name': 'postdot1qVlanId', 546 | 'from_bytes': unsigned, 547 | 'nprobe': 'DOT1Q_DST_VLAN' 548 | }, 549 | 277: { 550 | 'name': 'OBSERVATION_POINT_TYPE', 551 | 'from_bytes': unsigned, 552 | 'nprobe': 'OBSERVATION_POINT_TYPE' 553 | }, 554 | 300: { 555 | 'name': 'OBSERVATION_POINT_ID', 556 | 'from_bytes': unsigned, 557 | 'nprobe': 'OBSERVATION_POINT_ID' 558 | }, 559 | 302: { 560 | 'name': 'SELECTOR_ID', 561 | 'from_bytes': unsigned, 562 | 'nprobe': 'SELECTOR_ID' 563 | }, 564 | 304: { 565 | 'name': 'IPFIX_SAMPLING_ALGORITHM', 566 | 'from_bytes': unsigned, 567 | 'nprobe': 'IPFIX_SAMPLING_ALGORITHM' 568 | }, 569 | 309: { 570 | 'name': 'SAMPLING_SIZE', 571 | 'from_bytes': unsigned, 572 | 'nprobe': 'SAMPLING_SIZE' 573 | }, 574 | 310: { 575 | 'name': 'SAMPLING_POPULATION', 576 | 'from_bytes': unsigned, 577 | 'nprobe': 'SAMPLING_POPULATION' 578 | }, 579 | 312: { 580 | 'name': 'FRAME_LENGTH', 581 | 'from_bytes': unsigned, 582 | 'nprobe': 'FRAME_LENGTH' 583 | }, 584 | 318: { 585 | 'name': 'PACKETS_OBSERVED', 586 | 'from_bytes': unsigned, 587 | 'nprobe': 'PACKETS_OBSERVED' 588 | }, 589 | 319: { 590 | 'name': 'PACKETS_SELECTED', 591 | 'from_bytes': unsigned, 592 | 'nprobe': 'PACKETS_SELECTED' 593 | }, 594 | 335: { 595 | 'name': 'SELECTOR_NAME', 596 | 'from_bytes': unsigned, 597 | 'nprobe': 'SELECTOR_NAME' 598 | }, 599 | }, 600 | 35632: { #NTOP 601 | 1028: { 602 | 'name': 'PROTOCOL_MAP', 603 | 'from_bytes': str, 604 | 'nprobe': 'PROTOCOL_MAP' 605 | }, 606 | 1031: { 607 | 'name': 'L4_SRC_PORT_MAP', 608 | 'from_bytes': str, 609 | 'nprobe': 'L4_SRC_PORT_MAP' 610 | }, 611 | 1035: { 612 | 'name': 'L4_DST_PORT_MAP', 613 | 'from_bytes': str, 614 | 'nprobe': 'L4_DST_PORT_MAP' 615 | }, 616 | 1036: { 617 | 'name': 'L4_SRV_PORT', 618 | 'from_bytes': unsigned, 619 | 'nprobe': 'L4_SRV_PORT' 620 | }, 621 | 1037: { 622 | 'name': 'L4_SRV_PORT_MAP', 623 | 'from_bytes': str, 624 | 'nprobe': 'L4_SRV_PORT_MAP' 625 | }, 626 | 80: { 627 | 'name': 'SRC_FRAGMENTS', 628 | 'from_bytes': unsigned, 629 | 'nprobe': 'SRC_FRAGMENTS' 630 | }, 631 | 81: { 632 | 'name': 'DST_FRAGMENTS', 633 | 'from_bytes': unsigned, 634 | 'nprobe': 'DST_FRAGMENTS' 635 | }, 636 | 123: { 637 | 'name': 'CLIENT_NW_LATENCY_MS', 638 | 'from_bytes': unsigned, 639 | 'nprobe': 'CLIENT_NW_LATENCY_MS' 640 | }, 641 | 124: { 642 | 'name': 'SERVER_NW_LATENCY_MS', 643 | 'from_bytes': unsigned, 644 | 'nprobe': 'SERVER_NW_LATENCY_MS' 645 | }, 646 | 78: { 647 | 'name': 'CLIENT_TCP_FLAGS', 648 | 'from_bytes': unsigned, 649 | 'nprobe': 'CLIENT_TCP_FLAGS' 650 | }, 651 | 79: { 652 | 'name': 'SERVER_TCP_FLAGS', 653 | 'from_bytes': unsigned, 654 | 'nprobe': 'SERVER_TCP_FLAGS' 655 | }, 656 | 125: { 657 | 'name': 'APPL_LATENCY_MS', 658 | 'from_bytes': unsigned, 659 | 'nprobe': 'APPL_LATENCY_MS' 660 | }, 661 | 471: { 662 | 'name': 'NPROBE_IPV4_ADDRESS', 663 | 'from_bytes': ipv4address, 664 | 'nprobe': 'NPROBE_IPV4_ADDRESS' 665 | }, 666 | 82: { 667 | 'name': 'SRC_TO_DST_MAX_THROUGHPUT', 668 | 'from_bytes': unsigned, 669 | 'nprobe': 'SRC_TO_DST_MAX_THROUGHPUT' 670 | }, 671 | 83: { 672 | 'name': 'SRC_TO_DST_MIN_THROUGHPUT', 673 | 'from_bytes': unsigned, 674 | 'nprobe': 'SRC_TO_DST_MIN_THROUGHPUT' 675 | }, 676 | 84: { 677 | 'name': 'SRC_TO_DST_AVG_THROUGHPUT', 678 | 'from_bytes': unsigned, 679 | 'nprobe': 'SRC_TO_DST_AVG_THROUGHPUT' 680 | }, 681 | 85: { 682 | 'name': 'DST_TO_SRC_MAX_THROUGHPUT', 683 | 'from_bytes': unsigned, 684 | 'nprobe': 'DST_TO_SRC_MAX_THROUGHPUT' 685 | }, 686 | 86: { 687 | 'name': 'DST_TO_SRC_MIN_THROUGHPUT', 688 | 'from_bytes': unsigned, 689 | 'nprobe': 'DST_TO_SRC_MIN_THROUGHPUT' 690 | }, 691 | 87: { 692 | 'name': 'DST_TO_SRC_AVG_THROUGHPUT', 693 | 'from_bytes': unsigned, 694 | 'nprobe': 'DST_TO_SRC_AVG_THROUGHPUT' 695 | }, 696 | 88: { 697 | 'name': 'NUM_PKTS_UP_TO_128_BYTES', 698 | 'from_bytes': unsigned, 699 | 'nprobe': 'NUM_PKTS_UP_TO_128_BYTES' 700 | }, 701 | 89: { 702 | 'name': 'NUM_PKTS_128_TO_256_BYTES', 703 | 'from_bytes': unsigned, 704 | 'nprobe': 'NUM_PKTS_128_TO_256_BYTES' 705 | }, 706 | 90: { 707 | 'name': 'NUM_PKTS_256_TO_512_BYTES', 708 | 'from_bytes': unsigned, 709 | 'nprobe': 'NUM_PKTS_256_TO_512_BYTES' 710 | }, 711 | 91: { 712 | 'name': 'NUM_PKTS_512_TO_1024_BYTES', 713 | 'from_bytes': unsigned, 714 | 'nprobe': 'NUM_PKTS_512_TO_1024_BYTES' 715 | }, 716 | 92: { 717 | 'name': 'NUM_PKTS_1024_TO_1514_BYTES', 718 | 'from_bytes': unsigned, 719 | 'nprobe': 'NUM_PKTS_1024_TO_1514_BYTES' 720 | }, 721 | 93: { 722 | 'name': 'NUM_PKTS_OVER_1514_BYTES', 723 | 'from_bytes': unsigned, 724 | 'nprobe': 'NUM_PKTS_OVER_1514_BYTES' 725 | }, 726 | 98: { 727 | 'name': 'CUMULATIVE_ICMP_TYPE', 728 | 'from_bytes': unsigned, 729 | 'nprobe': 'CUMULATIVE_ICMP_TYPE' 730 | }, 731 | 101: { 732 | 'name': 'SRC_IP_COUNTRY', 733 | 'from_bytes': unsigned, 734 | 'nprobe': 'SRC_IP_COUNTRY' 735 | }, 736 | 102: { 737 | 'name': 'SRC_IP_CITY', 738 | 'from_bytes': str, 739 | 'nprobe': 'SRC_IP_CITY' 740 | }, 741 | 103: { 742 | 'name': 'DST_IP_COUNTRY', 743 | 'from_bytes': unsigned, 744 | 'nprobe': 'DST_IP_COUNTRY' 745 | }, 746 | 104: { 747 | 'name': 'DST_IP_CITY', 748 | 'from_bytes': str, 749 | 'nprobe': 'DST_IP_CITY' 750 | }, 751 | 448: { 752 | 'name': 'SRC_IP_LONG', 753 | 'from_bytes': str, 754 | 'nprobe': 'SRC_IP_LONG' 755 | }, 756 | 449: { 757 | 'name': 'SRC_IP_LAT', 758 | 'from_bytes': str, 759 | 'nprobe': 'SRC_IP_LAT' 760 | }, 761 | 450: { 762 | 'name': 'DST_IP_LONG', 763 | 'from_bytes': str, 764 | 'nprobe': 'DST_IP_LONG' 765 | }, 766 | 451: { 767 | 'name': 'DST_IP_LAT', 768 | 'from_bytes': str, 769 | 'nprobe': 'DST_IP_LAT' 770 | }, 771 | 105: { 772 | 'name': 'FLOW_PROTO_PORT', 773 | 'from_bytes': unsigned, 774 | 'nprobe': 'FLOW_PROTO_PORT' 775 | }, 776 | 106: { 777 | 'name': 'UPSTREAM_TUNNEL_ID', 778 | 'from_bytes': unsigned, 779 | 'nprobe': 'UPSTREAM_TUNNEL_ID' 780 | }, 781 | 446: { 782 | 'name': 'UPSTREAM_SESSION_ID', 783 | 'from_bytes': unsigned, 784 | 'nprobe': 'UPSTREAM_SESSION_ID' 785 | }, 786 | 107: { 787 | 'name': 'LONGEST_FLOW_PKT', 788 | 'from_bytes': unsigned, 789 | 'nprobe': 'LONGEST_FLOW_PKT' 790 | }, 791 | 108: { 792 | 'name': 'SHORTEST_FLOW_PKT', 793 | 'from_bytes': unsigned, 794 | 'nprobe': 'SHORTEST_FLOW_PKT' 795 | }, 796 | 127: { 797 | 'name': 'RETRANSMITTED_IN_BYTES', 798 | 'from_bytes': unsigned, 799 | 'nprobe': 'RETRANSMITTED_IN_BYTES' 800 | }, 801 | 109: { 802 | 'name': 'RETRANSMITTED_IN_PKTS', 803 | 'from_bytes': unsigned, 804 | 'nprobe': 'RETRANSMITTED_IN_PKTS' 805 | }, 806 | 128: { 807 | 'name': 'RETRANSMITTED_OUT_BYTES', 808 | 'from_bytes': unsigned, 809 | 'nprobe': 'RETRANSMITTED_OUT_BYTES' 810 | }, 811 | 110: { 812 | 'name': 'RETRANSMITTED_OUT_PKTS', 813 | 'from_bytes': unsigned, 814 | 'nprobe': 'RETRANSMITTED_OUT_PKTS' 815 | }, 816 | 111: { 817 | 'name': 'OOORDER_IN_PKTS', 818 | 'from_bytes': unsigned, 819 | 'nprobe': 'OOORDER_IN_PKTS' 820 | }, 821 | 112: { 822 | 'name': 'OOORDER_OUT_PKTS', 823 | 'from_bytes': unsigned, 824 | 'nprobe': 'OOORDER_OUT_PKTS' 825 | }, 826 | 113: { 827 | 'name': 'UNTUNNELED_PROTOCOL', 828 | 'from_bytes': unsigned, 829 | 'nprobe': 'UNTUNNELED_PROTOCOL' 830 | }, 831 | 114: { 832 | 'name': 'UNTUNNELED_IPV4_SRC_ADDR', 833 | 'from_bytes': unsigned, 834 | 'nprobe': 'UNTUNNELED_IPV4_SRC_ADDR' 835 | }, 836 | 115: { 837 | 'name': 'UNTUNNELED_L4_SRC_PORT', 838 | 'from_bytes': unsigned, 839 | 'nprobe': 'UNTUNNELED_L4_SRC_PORT' 840 | }, 841 | 116: { 842 | 'name': 'UNTUNNELED_IPV4_DST_ADDR', 843 | 'from_bytes': unsigned, 844 | 'nprobe': 'UNTUNNELED_IPV4_DST_ADDR' 845 | }, 846 | 117: { 847 | 'name': 'UNTUNNELED_L4_DST_PORT', 848 | 'from_bytes': unsigned, 849 | 'nprobe': 'UNTUNNELED_L4_DST_PORT' 850 | }, 851 | 118: { 852 | 'name': 'L7_PROTO', 853 | 'from_bytes': unsigned, 854 | 'nprobe': 'L7_PROTO' 855 | }, 856 | 119: { 857 | 'name': 'L7_PROTO_NAME', 858 | 'from_bytes': unsigned, 859 | 'nprobe': 'L7_PROTO_NAME' 860 | }, 861 | 120: { 862 | 'name': 'DOWNSTREAM_TUNNEL_ID', 863 | 'from_bytes': unsigned, 864 | 'nprobe': 'DOWNSTREAM_TUNNEL_ID' 865 | }, 866 | 447: { 867 | 'name': 'DOWNSTREAM_SESSION_ID', 868 | 'from_bytes': unsigned, 869 | 'nprobe': 'DOWNSTREAM_SESSION_ID' 870 | }, 871 | 188: { 872 | 'name': 'SSL_SERVER_NAME', 873 | 'from_bytes': str, 874 | 'nprobe': 'SSL_SERVER_NAME' 875 | }, 876 | 189: { 877 | 'name': 'BITTORRENT_HASH', 878 | 'from_bytes': str, 879 | 'nprobe': 'BITTORRENT_HASH' 880 | }, 881 | 121: { 882 | 'name': 'FLOW_USER_NAME', 883 | 'from_bytes': str, 884 | 'nprobe': 'FLOW_USER_NAME' 885 | }, 886 | 122: { 887 | 'name': 'FLOW_SERVER_NAME', 888 | 'from_bytes': str, 889 | 'nprobe': 'FLOW_SERVER_NAME' 890 | }, 891 | 126: { 892 | 'name': 'PLUGIN_NAME', 893 | 'from_bytes': str, 894 | 'nprobe': 'PLUGIN_NAME' 895 | }, 896 | 396: { 897 | 'name': 'UNTUNNELED_IPV6_SRC_ADDR', 898 | 'from_bytes': ipv6address, 899 | 'nprobe': 'UNTUNNELED_IPV6_SRC_ADDR' 900 | }, 901 | 397: { 902 | 'name': 'UNTUNNELED_IPV6_DST_ADDR', 903 | 'from_bytes': ipv6address, 904 | 'nprobe': 'UNTUNNELED_IPV6_DST_ADDR' 905 | }, 906 | 347: { 907 | 'name': 'NUM_PKTS_TTL_EQ_1', 908 | 'from_bytes': unsigned, 909 | 'nprobe': 'NUM_PKTS_TTL_EQ_1' 910 | }, 911 | 346: { 912 | 'name': 'NUM_PKTS_TTL_2_5', 913 | 'from_bytes': unsigned, 914 | 'nprobe': 'NUM_PKTS_TTL_2_5' 915 | }, 916 | 334: { 917 | 'name': 'NUM_PKTS_TTL_5_32', 918 | 'from_bytes': unsigned, 919 | 'nprobe': 'NUM_PKTS_TTL_5_32' 920 | }, 921 | 335: { 922 | 'name': 'NUM_PKTS_TTL_32_64', 923 | 'from_bytes': unsigned, 924 | 'nprobe': 'NUM_PKTS_TTL_32_64' 925 | }, 926 | 336: { 927 | 'name': 'NUM_PKTS_TTL_64_96', 928 | 'from_bytes': unsigned, 929 | 'nprobe': 'NUM_PKTS_TTL_64_96' 930 | }, 931 | 337: { 932 | 'name': 'NUM_PKTS_TTL_96_128', 933 | 'from_bytes': unsigned, 934 | 'nprobe': 'NUM_PKTS_TTL_96_128' 935 | }, 936 | 338: { 937 | 'name': 'NUM_PKTS_TTL_128_160', 938 | 'from_bytes': unsigned, 939 | 'nprobe': 'NUM_PKTS_TTL_128_160' 940 | }, 941 | 339: { 942 | 'name': 'NUM_PKTS_TTL_160_192', 943 | 'from_bytes': unsigned, 944 | 'nprobe': 'NUM_PKTS_TTL_160_192' 945 | }, 946 | 340: { 947 | 'name': 'NUM_PKTS_TTL_192_224', 948 | 'from_bytes': unsigned, 949 | 'nprobe': 'NUM_PKTS_TTL_192_224' 950 | }, 951 | 341: { 952 | 'name': 'NUM_PKTS_TTL_224_255', 953 | 'from_bytes': unsigned, 954 | 'nprobe': 'NUM_PKTS_TTL_224_255' 955 | }, 956 | 349: { 957 | 'name': 'IN_SRC_OSI_SAP', 958 | 'from_bytes': str, 959 | 'nprobe': 'IN_SRC_OSI_SAP' 960 | }, 961 | 350: { 962 | 'name': 'OUT_DST_OSI_SAP', 963 | 'from_bytes': 350, 964 | 'nprobe': 'OUT_DST_OSI_SAP' 965 | }, 966 | 391: { 967 | 'name': 'DURATION_IN', 968 | 'from_bytes': unsigned, 969 | 'nprobe': 'DURATION_IN' 970 | }, 971 | 392: { 972 | 'name': 'DURATION_OUT', 973 | 'from_bytes': unsigned, 974 | 'nprobe': 'DURATION_OUT' 975 | }, 976 | 415: { 977 | 'name': 'TCP_WIN_MIN_IN', 978 | 'from_bytes': unsigned, 979 | 'nprobe': 'TCP_WIN_MIN_IN' 980 | }, 981 | 416: { 982 | 'name': 'TCP_WIN_MAX_IN', 983 | 'from_bytes': unsigned, 984 | 'nprobe': 'TCP_WIN_MAX_IN' 985 | }, 986 | 417: { 987 | 'name': 'TCP_WIN_MSS_IN', 988 | 'from_bytes': unsigned, 989 | 'nprobe': 'TCP_WIN_MSS_IN' 990 | }, 991 | 418: { 992 | 'name': 'TCP_WIN_SCALE_IN', 993 | 'from_bytes': unsigned, 994 | 'nprobe': 'TCP_WIN_SCALE_IN' 995 | }, 996 | 419: { 997 | 'name': 'TCP_WIN_MIN_OUT', 998 | 'from_bytes': unsigned, 999 | 'nprobe': 'TCP_WIN_MIN_OUT' 1000 | }, 1001 | 420: { 1002 | 'name': 'TCP_WIN_MAX_OUT', 1003 | 'from_bytes': unsigned, 1004 | 'nprobe': 'TCP_WIN_MAX_OUT' 1005 | }, 1006 | 421: { 1007 | 'name': 'TCP_WIN_MSS_OUT', 1008 | 'from_bytes': unsigned, 1009 | 'nprobe': 'TCP_WIN_MSS_OUT' 1010 | }, 1011 | 422: { 1012 | 'name': 'TCP_WIN_SCALE_OUT', 1013 | 'from_bytes': unsigned, 1014 | 'nprobe': 'TCP_WIN_SCALE_OUT' 1015 | }, 1016 | 438: { 1017 | 'name': 'PAYLOAD_HASH', 1018 | 'from_bytes': unsigned, 1019 | 'nprobe': 'PAYLOAD_HASH' 1020 | }, 1021 | 443: { 1022 | 'name': 'SRC_AS_MAP', 1023 | 'from_bytes': str, 1024 | 'nprobe': 'SRC_AS_MAP' 1025 | }, 1026 | 444: { 1027 | 'name': 'DST_AS_MAP', 1028 | 'from_bytes': str, 1029 | 'nprobe': 'DST_AS_MAP' 1030 | }, 1031 | 472: { 1032 | 'name': 'SRC_TO_DST_SECOND_BYTES', 1033 | 'from_bytes': unsigned, 1034 | 'nprobe': 'SRC_TO_DST_SECOND_BYTES' 1035 | }, 1036 | 473: { 1037 | 'name': 'DST_TO_SRC_SECOND_BYTES', 1038 | 'from_bytes': unsigned, 1039 | 'nprobe': 'DST_TO_SRC_SECOND_BYTES' 1040 | }, 1041 | 489: { 1042 | 'name': 'JA3C_HASH', 1043 | 'from_bytes': unsigned, 1044 | 'nprobe': 'JA3C_HASH' 1045 | }, 1046 | 490: { 1047 | 'name': 'JA3S_HASH', 1048 | 'from_bytes': unsigned, 1049 | 'nprobe': 'JA3S_HASH' 1050 | }, 1051 | 491: { 1052 | 'name': 'SRC_HOST_NAME', 1053 | 'from_bytes': str, 1054 | 'nprobe': 'SRC_HOST_NAME' 1055 | }, 1056 | 492: { 1057 | 'name': 'DST_HOST_NAME', 1058 | 'from_bytes': str, 1059 | 'nprobe': 'DST_HOST_NAME' 1060 | }, 1061 | 493: { 1062 | 'name': 'SSL_CIPHER', 1063 | 'from_bytes': unsigned, 1064 | 'nprobe': 'SSL_CIPHER' 1065 | }, 1066 | 494: { 1067 | 'name': 'SSL_UNSAFE_CIPHER', 1068 | 'from_bytes': unsigned, 1069 | 'nprobe': 'SSL_UNSAFE_CIPHER' 1070 | }, 1071 | 495: { 1072 | 'name': 'SSL_VERSION', 1073 | 'from_bytes': unsigned, 1074 | 'nprobe': 'SSL_VERSION' 1075 | } 1076 | } 1077 | } 1078 | 1079 | def parseAddress(address): 1080 | m=re.match('^udp:\/\/([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}):([0-9]+)$', address) 1081 | 1082 | if m is None: 1083 | raise Exception(address +" not in format +udp://IPv4:port") 1084 | 1085 | ipv4=ipaddress.IPv4Address(m.group(1)).exploded 1086 | port=int(m.group(2)) 1087 | 1088 | return (ipv4, port) 1089 | 1090 | def IPFIX_scan_record(message, netflow_fields, field_offset, field_count): 1091 | 1092 | field_list=[] 1093 | rec_len=0 1094 | for i in range(field_count): 1095 | 1096 | field_id=int.from_bytes(message[field_offset:field_offset+2],'big') 1097 | field_length=int.from_bytes(message[field_offset+2:field_offset+4],'big') 1098 | field_enterprise=-1 1099 | 1100 | if field_id & 0x8000: 1101 | field_enterprise = int.from_bytes(message[field_offset+4:field_offset+8],'big') 1102 | field_id = field_id ^ 0x8000 1103 | 1104 | if field_enterprise in IPFIX_fields \ 1105 | and field_id in IPFIX_fields[ field_enterprise ] \ 1106 | and 'nprobe' in IPFIX_fields[ field_enterprise ][ field_id ]: 1107 | 1108 | IPFIX_field = IPFIX_fields[ field_enterprise ][ field_id ] 1109 | 1110 | if IPFIX_field['nprobe'] in netflow_fields: 1111 | field_list.append( 1112 | (field_id, rec_len, rec_len+field_length, field_enterprise, IPFIX_field['nprobe'], IPFIX_field['from_bytes']) 1113 | ) 1114 | 1115 | rec_len=rec_len+field_length 1116 | 1117 | field_offset=field_offset+4 1118 | if(field_enterprise) != -1: 1119 | field_offset=field_offset+4 1120 | 1121 | return (field_offset, rec_len, field_list) 1122 | 1123 | def verbose_print(args_verbose, num, msg, **kwargs): 1124 | if num<=args_verbose: 1125 | print( 1126 | msg.format( 1127 | **kwargs 1128 | ) 1129 | ) 1130 | 1131 | def collector_fields(args_fields): 1132 | ret = {} 1133 | for f in args_fields: 1134 | ret[ f ] = '' 1135 | 1136 | return ret 1137 | 1138 | def netflow_collector(args_collector_port, args_performance, args_fields, args_verbose, q): 1139 | templates={} 1140 | 1141 | netflow_fields = collector_fields(args_fields) 1142 | 1143 | server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 1144 | 1145 | server_socket.bind( parseAddress(args_collector_port) ) 1146 | 1147 | print("Netflow: loop") 1148 | print("Netflow fields: "+ ", ".join(netflow_fields)) 1149 | 1150 | pkt_cnt=-1 1151 | next_sequence=0 1152 | 1153 | while True: 1154 | message, address = server_socket.recvfrom(1500) 1155 | start_processing=time.time_ns() 1156 | pkt_cnt=pkt_cnt+1 1157 | 1158 | msg_version = int.from_bytes(message[0:2], 'big') 1159 | 1160 | if msg_version != 10: 1161 | print("#WARNING: unknown version "+str(msg_version)) 1162 | continue 1163 | 1164 | msg_length = int.from_bytes(message[2:4],'big') 1165 | msg_export_time = int.from_bytes(message[4:8],'big') 1166 | msg_sequence = int.from_bytes(message[8:12],'big') 1167 | msg_observation_domain_id = int.from_bytes(message[12:16],'big') 1168 | 1169 | verbose_print( 1170 | args_verbose, 1171 | 0, 1172 | 'NetFlow:{cnt} ver:{version} seq:{sequence} exp:{next_sequence} dom:{domain_id} len:{length} drp:{dropped}', 1173 | cnt=pkt_cnt, 1174 | version=msg_version, 1175 | sequence=msg_sequence, 1176 | next_sequence=next_sequence, 1177 | domain_id=msg_observation_domain_id, 1178 | length=msg_length, 1179 | dropped = (msg_sequence - next_sequence) 1180 | ) 1181 | 1182 | data_offset=16 1183 | records = [] 1184 | while(data_offset0): 1296 | q.put(records) 1297 | 1298 | end_queueing = time.time_ns() 1299 | 1300 | if args_performance: 1301 | verbose_print( 1302 | args_verbose, 1303 | 0, 1304 | "netflow_timers(ns) {processing} / {parsing} / {queueing}", 1305 | processing=end_queueing-start_processing, 1306 | parsing=end_parsing-start_processing, 1307 | queueing=end_queueing-end_parsing 1308 | ) 1309 | 1310 | def zmq_broker(args_ntopng, args_zmq_disable_compression, args_performance, args_zmq_source_id, args_verbose, q): 1311 | 1312 | context = zmq.Context.instance() 1313 | socket = context.socket(zmq.PUB) 1314 | 1315 | socket.setsockopt(zmq.TCP_KEEPALIVE,1) 1316 | socket.setsockopt(zmq.TCP_KEEPALIVE_CNT,30) 1317 | socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE,3) 1318 | socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL,3) 1319 | 1320 | socket.bind(args_ntopng.replace('zmq','tcp')) 1321 | 1322 | poller = zmq.Poller() 1323 | poller.register(socket, zmq.POLLIN | zmq.POLLOUT) 1324 | 1325 | socks = dict(poller.poll()) 1326 | assert socks[socket] == zmq.POLLOUT 1327 | assert poller not in socks 1328 | 1329 | version=2 1330 | 1331 | zmq_msg_hdr_v1={ 1332 | 'url': bytearray([ 102 ,108,111,119, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]), 1333 | 'version': version.to_bytes(1,'little'), 1334 | 'source_id': args_zmq_source_id.to_bytes(1, 'little'), 1335 | 'size': bytearray([0, 0]), 1336 | 'msg_id': 0, 1337 | } 1338 | 1339 | print("ZMQ: loop") 1340 | 1341 | msg_id=0 1342 | 1343 | zmq_msg_hdr=zmq_msg_hdr_v1['url'] \ 1344 | +zmq_msg_hdr_v1['version'] \ 1345 | +zmq_msg_hdr_v1['source_id'] \ 1346 | +zmq_msg_hdr_v1['size'] 1347 | 1348 | while True: 1349 | records = q.get() 1350 | start_processing=time.time_ns() 1351 | 1352 | if records is None: 1353 | break 1354 | 1355 | verbose_print( 1356 | args_verbose, 1357 | 0, 1358 | 'ZMQ:{msg_id}', 1359 | msg_id=msg_id 1360 | ) 1361 | 1362 | rec_comp=orjson.dumps(records) 1363 | end_jsoning=time.time_ns() 1364 | 1365 | if not args_zmq_disable_compression: 1366 | len_json = len(rec_comp) 1367 | rec_comp=(0).to_bytes(1, 'big')+zlib.compress(rec_comp, 6) 1368 | len_comp=len(rec_comp) 1369 | verbose_print( 1370 | args_verbose, 1371 | 0, 1372 | 'ZMQ:{msg_id} comp: {after} / {before}', 1373 | msg_id=msg_id, 1374 | after=len_comp, 1375 | before=len_json 1376 | ) 1377 | 1378 | end_compressing=time.time_ns() 1379 | 1380 | socket.send_multipart([ 1381 | zmq_msg_hdr \ 1382 | +msg_id.to_bytes(4,'little') 1383 | ,rec_comp 1384 | ]) 1385 | 1386 | msg_id=(msg_id+1) & 0xffffffff 1387 | poller.poll() 1388 | 1389 | end_processing=time.time_ns() 1390 | 1391 | if args_performance: 1392 | verbose_print( 1393 | args_verbose, 1394 | 0, 1395 | 'zmq_timers(ns) {processing} / {jsoning} / {compressing} / {sending}', 1396 | processing=end_processing-start_processing, 1397 | jsoning=end_jsoning-start_processing, 1398 | compressing=end_compressing-end_jsoning, 1399 | sending=end_processing-end_compressing 1400 | ) 1401 | 1402 | poller.unregister(socket) 1403 | 1404 | 1405 | if __name__ == '__main__': 1406 | 1407 | parser = argparse.ArgumentParser() 1408 | parser.add_argument('--collector-port', default="udp://0.0.0.0:2055") 1409 | parser.add_argument('--ntopng', default='zmq://0.0.0.0:1234') 1410 | parser.add_argument('--zmq-disable-compression', action='store_true', default=False) 1411 | parser.add_argument('--zmq-source-id', default="1", type=int) 1412 | parser.add_argument('--verbose', default="0", type=int) 1413 | parser.add_argument('--performance', action='store_true', default=False) 1414 | parser.add_argument('-T', default='%IPV4_SRC_ADDR %IPV4_DST_ADDR %INPUT_SNMP %OUTPUT_SNMP %IN_PKTS %IN_BYTES %FIRST_SWITCHED %LAST_SWITCHED %L4_SRC_PORT %L4_DST_PORT %TCP_FLAGS %PROTOCOL %SRC_TOS %SRC_AS %DST_AS') 1415 | parser.add_argument('--version', action='store_true', default=False) 1416 | 1417 | args=parser.parse_args() 1418 | 1419 | if args.version: 1420 | print("0.0.4") 1421 | sys.exit(0) 1422 | 1423 | mp.set_start_method('spawn') 1424 | rec_queue = mp.Queue() 1425 | p=mp.Process( 1426 | target=zmq_broker, 1427 | args=( 1428 | args.ntopng, 1429 | args.zmq_disable_compression, 1430 | args.performance, 1431 | args.zmq_source_id, 1432 | args.verbose, 1433 | rec_queue, 1434 | ) 1435 | ) 1436 | 1437 | p.start() 1438 | 1439 | try: 1440 | netflow_collector( 1441 | args.collector_port, 1442 | args.performance, 1443 | args.T.upper().replace( 1444 | '@NTOPNG@', 1445 | "%IN_SRC_MAC %OUT_DST_MAC %INPUT_SNMP %OUTPUT_SNMP %SRC_VLAN %IPV4_SRC_ADDR %IPV4_DST_ADDR %L4_SRC_PORT %L4_DST_PORT %IPV6_SRC_ADDR %IPV6_DST_ADDR %IP_PROTOCOL_VERSION %PROTOCOL %L7_PROTO %L7_CONFIDENCE %IN_BYTES %IN_PKTS %OUT_BYTES %OUT_PKTS %FIRST_SWITCHED %LAST_SWITCHED %CLIENT_TCP_FLAGS %SERVER_TCP_FLAGS %L7_PROTO_RISK %L7_RISK_SCORE %EXPORTER_IPV4_ADDRESS %DIRECTION %SAMPLING_INTERVAL %TOTAL_FLOWS_EXP %NPROBE_IPV4_ADDRESS %NPROBE_INSTANCE_NAME %POST_NAT_SRC_IPV4_ADDR %POST_NAT_DST_IPV4_ADDR %POST_NAPT_SRC_TRANSPORT_PORT %POST_NAPT_DST_TRANSPORT_PORT" 1446 | ).replace('%','').split(' '), 1447 | args.verbose, 1448 | rec_queue, 1449 | ) 1450 | except Exception as e: 1451 | print(str(e)) 1452 | pass 1453 | 1454 | rec_queue.put(None) 1455 | p.join() 1456 | 1457 | print("Exiting") 1458 | 1459 | --------------------------------------------------------------------------------