├── .github └── workflows │ └── semgrep.yml ├── Dockerfile.inserter ├── Dockerfile.mocker ├── LICENSE ├── Makefile ├── README.md ├── compose ├── clickhouse │ └── create.sh ├── docker-compose-clickhouse-collect.yml ├── docker-compose-clickhouse-mock.yml ├── docker-compose-postgres-collect.yml ├── docker-compose-postgres-mock.yml ├── grafana │ ├── dashboards.yml │ ├── dashboards │ │ ├── perfs.json │ │ ├── viz-ch.json │ │ └── viz.json │ ├── datasources-ch.yml │ └── datasources.yml ├── postgres │ └── create.sh └── prometheus │ └── prometheus.yml ├── go.mod ├── go.sum ├── grafana └── Dockerfile ├── inserter └── inserter.go ├── mocker └── mocker.go └── pb-ext ├── flow.pb.go └── flow.proto /.github/workflows/semgrep.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: {} 3 | workflow_dispatch: {} 4 | push: 5 | branches: 6 | - main 7 | - master 8 | schedule: 9 | - cron: '0 0 * * *' 10 | name: Semgrep config 11 | jobs: 12 | semgrep: 13 | name: semgrep/ci 14 | runs-on: ubuntu-latest 15 | env: 16 | SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} 17 | SEMGREP_URL: https://cloudflare.semgrep.dev 18 | SEMGREP_APP_URL: https://cloudflare.semgrep.dev 19 | SEMGREP_VERSION_CHECK_URL: https://cloudflare.semgrep.dev/api/check-version 20 | container: 21 | image: semgrep/semgrep 22 | steps: 23 | - uses: actions/checkout@v4 24 | - run: semgrep ci 25 | -------------------------------------------------------------------------------- /Dockerfile.inserter: -------------------------------------------------------------------------------- 1 | FROM golang:alpine as builder 2 | ARG src_dir 3 | 4 | RUN apk --update --no-cache add git 5 | 6 | RUN mkdir /builder 7 | WORKDIR /builder 8 | COPY . /builder 9 | 10 | RUN cd /builder/inserter && go build 11 | 12 | FROM alpine:latest 13 | 14 | RUN apk update --no-cache && \ 15 | adduser -S -D -H -h / flow 16 | USER flow 17 | COPY --from=builder /builder/inserter / 18 | 19 | ENTRYPOINT ["./inserter"] 20 | -------------------------------------------------------------------------------- /Dockerfile.mocker: -------------------------------------------------------------------------------- 1 | FROM golang:alpine as builder 2 | ARG src_dir 3 | 4 | RUN apk --update --no-cache add git 5 | 6 | RUN mkdir /builder 7 | WORKDIR /builder 8 | COPY . /builder 9 | 10 | RUN cd /builder/mocker && go build 11 | 12 | FROM alpine:latest 13 | 14 | RUN apk update --no-cache && \ 15 | adduser -S -D -H -h / flow 16 | USER flow 17 | COPY --from=builder /builder/mocker / 18 | 19 | ENTRYPOINT ["./mocker"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: proto 2 | proto: 3 | @echo generating protobuf 4 | protoc --go_out=. --plugin=$(PROTOCPATH)protoc-gen-go pb-ext/*.proto -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flow-pipeline 2 | 3 | This repository contains a set of tools and examples for [GoFlow](https://github.com/cloudflare/goflow), 4 | a NetFlow/IPFIX/sFlow collector by [Cloudflare](https://www.cloudflare.com). 5 | 6 | ## Start a flow pipeline 7 | 8 | The demo directory contains a startup file for an example pipeline including: 9 | * GoFlow: an sFlow collector 10 | * A mock collector 11 | * Kafka/Zookeeper 12 | * A database (Postgres/clickhouse) 13 | * An inserter: to insert the flows in a database (for Postgres) 14 | 15 | It will listen on port 6343/UDP for sFlow and 2055/UDP for NetFlow. 16 | 17 | The protobuf provided in this repository is a light version of 18 | the GoFlow original one. Only a handful of fields will be inserted. 19 | 20 | A basic pipeline looks like this: 21 | 22 | ``` 23 | 24 | 25 | 26 | +------+ +-----+ 27 | sFlow/NetFlow |goflow+--------->Kafka| 28 | +------+ +-----+ 29 | | 30 | +--------------+ 31 | Topic: flows | | 32 | | | 33 | +-----v----+ +-v---------+ 34 | | inserter | |new service| 35 | +----------+ +-----------+ 36 | | 37 | | 38 | +--v--+ 39 | | DB | 40 | +-----+ 41 | 42 | ``` 43 | 44 | You can add a _processor_ that would enrich the data 45 | by consuming from Kafka and re-injecting the data into Kafka 46 | or directly into the database. 47 | 48 | For instance, IP addresses can be mapped to countries, ASN 49 | or customer information. 50 | 51 | A suggestion is extending the GoFlow protobuf with new fields. 52 | 53 | ## Run a mock insertion 54 | 55 | A mock insertion replaces the GoFlow decoding part. A _mocker_ generates 56 | protobuf messages and sends them to Kafka. 57 | 58 | Clone the repository, then run the following (for Postgres): 59 | 60 | ``` 61 | $ cd compose 62 | $ docker-compose -f docker-compose-postgres-mock.yml up 63 | ``` 64 | 65 | Wait a minute for all the components to start. 66 | 67 | You can connect on the local Grafana http://localhost:3000 (admin/admin) to look at the flows being collected. 68 | 69 | ## Run a GoFlow insertion 70 | 71 | If you want to send sFlow/NetFlow/IPFIX to a GoFlow, run the following: 72 | 73 | Using Postgres: 74 | ``` 75 | $ cd compose 76 | $ docker-compose -f docker-compose-postgres-collect.yml up 77 | ``` 78 | 79 | Using Clickhouse (see next section): 80 | ``` 81 | $ cd compose 82 | $ docker-compose -f docker-compose-clickhouse-collect.yml up 83 | ``` 84 | 85 | Keep in mind this is a development/prototype setup. 86 | Some components will likely not be able to process more than a few 87 | thousands rows per second. 88 | You will likely have to tweak configuration statements, 89 | number of workers. 90 | 91 | Using a production setup, GoFlow was able to process more than +100k flows 92 | per seconds and insert them in a Clickhouse database. 93 | 94 | ## About the Clickhouse setup 95 | 96 | If you choose to visualize in Grafana, you will need a 97 | [Clickhouse Data source plugin](https://grafana.com/grafana/plugins/vertamedia-clickhouse-datasource). 98 | You can connect to the compose Grafana which has the plugin installed. 99 | 100 | The insertion is handled natively by Clickhouse: 101 | * Creates a table with a [Kafka Engine](https://clickhouse.tech/docs/en/operations/table_engines/kafka/). 102 | * Uses [Protobuf format](https://clickhouse.tech/docs/en/interfaces/formats/#protobuf). 103 | 104 | Note: the protobuf messages to be written with their lengths. 105 | 106 | Clickhouse will connect to Kafka periodically and fetch the content. Materialized views 107 | allow to store the data persistently and aggregate over fields. 108 | 109 | To connect to the database, you have to run the following: 110 | ``` 111 | $ docker exec -ti compose_db_1 clickhouse-client 112 | ``` 113 | 114 | Once in the client CLI, a handful of tables are available: 115 | * `flows` is directly connected to Kafka, it fetches from the current offset 116 | * `flows_raw` contains the materialized view of `flows` 117 | * `flows_5m` contains 5-minutes aggregates of ASN 118 | 119 | Commands example: 120 | ``` 121 | :) DESCRIBE flows_raw 122 | 123 | DESCRIBE TABLE flows_raw 124 | 125 | ┌─name───────────┬─type────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐ 126 | │ Date │ Date │ │ │ │ │ │ 127 | │ TimeReceived │ DateTime │ │ │ │ │ │ 128 | │ TimeFlowStart │ DateTime │ │ │ │ │ │ 129 | │ SequenceNum │ UInt32 │ │ │ │ │ │ 130 | │ SamplingRate │ UInt64 │ │ │ │ │ │ 131 | │ SamplerAddress │ FixedString(16) │ │ │ │ │ │ 132 | │ SrcAddr │ FixedString(16) │ │ │ │ │ │ 133 | │ DstAddr │ FixedString(16) │ │ │ │ │ │ 134 | │ SrcAS │ UInt32 │ │ │ │ │ │ 135 | │ DstAS │ UInt32 │ │ │ │ │ │ 136 | │ EType │ UInt32 │ │ │ │ │ │ 137 | │ Proto │ UInt32 │ │ │ │ │ │ 138 | │ SrcPort │ UInt32 │ │ │ │ │ │ 139 | │ DstPort │ UInt32 │ │ │ │ │ │ 140 | │ Bytes │ UInt64 │ │ │ │ │ │ 141 | │ Packets │ UInt64 │ │ │ │ │ │ 142 | └────────────────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘ 143 | 144 | :) SELECT Date,TimeReceived,IPv6NumToString(SrcAddr), IPv6NumToString(DstAddr), Bytes, Packets FROM flows_raw; 145 | 146 | SELECT 147 | Date, 148 | TimeReceived, 149 | IPv6NumToString(SrcAddr), 150 | IPv6NumToString(DstAddr), 151 | Bytes, 152 | Packets 153 | FROM flows_raw 154 | 155 | ┌───────Date─┬────────TimeReceived─┬─IPv6NumToString(SrcAddr)─┬─IPv6NumToString(DstAddr)─┬─Bytes─┬─Packets─┐ 156 | │ 2020-03-22 │ 2020-03-22 21:26:38 │ 2001:db8:0:1::80 │ 2001:db8:0:1::20 │ 105 │ 63 │ 157 | │ 2020-03-22 │ 2020-03-22 21:26:38 │ 2001:db8:0:1::c2 │ 2001:db8:0:1:: │ 386 │ 43 │ 158 | │ 2020-03-22 │ 2020-03-22 21:26:38 │ 2001:db8:0:1::6b │ 2001:db8:0:1::9c │ 697 │ 29 │ 159 | │ 2020-03-22 │ 2020-03-22 21:26:38 │ 2001:db8:0:1::81 │ 2001:db8:0:1:: │ 1371 │ 54 │ 160 | │ 2020-03-22 │ 2020-03-22 21:26:39 │ 2001:db8:0:1::87 │ 2001:db8:0:1::32 │ 123 │ 23 │ 161 | 162 | ``` 163 | 164 | To look at aggregates (optimizing will run the summing operation). 165 | The Nested structure allows to have sum per structures (in our case, per Ethernet-Type). 166 | 167 | ``` 168 | :) OPTIMIZE TABLE flows_5m; 169 | 170 | OPTIMIZE TABLE flows_5m 171 | 172 | Ok. 173 | 174 | :) SELECT * FROM flows_5m WHERE SrcAS = 65001; 175 | 176 | SELECT * 177 | FROM flows_5m 178 | WHERE SrcAS = 65001 179 | 180 | ┌───────Date─┬────────────Timeslot─┬─SrcAS─┬─DstAS─┬─ETypeMap.EType─┬─ETypeMap.Bytes─┬─ETypeMap.Packets─┬─ETypeMap.Count─┬─Bytes─┬─Packets─┬─Count─┐ 181 | │ 2020-03-22 │ 2020-03-22 21:25:00 │ 65001 │ 65000 │ [34525] │ [2930] │ [152] │ [4] │ 2930 │ 152 │ 4 │ 182 | │ 2020-03-22 │ 2020-03-22 21:25:00 │ 65001 │ 65001 │ [34525] │ [1935] │ [190] │ [3] │ 1935 │ 190 │ 3 │ 183 | │ 2020-03-22 │ 2020-03-22 21:25:00 │ 65001 │ 65002 │ [34525] │ [4820] │ [288] │ [6] │ 4820 │ 288 │ 6 │ 184 | ``` 185 | 186 | **Regarding the storage of IP addresses:** 187 | At the moment, the current Clickhouse table does not perform any transformation of the addresses before insertion. 188 | The bytes are inserted in a `FixedString(16)` regardless of the family (IPv4, IPv6). 189 | In the dashboards, the function `IPv6NumToString(SrcAddr)` is used. 190 | 191 | For example, **192.168.1.1** will end up being **101:a8c0::** 192 | ```sql 193 | WITH toFixedString(reinterpretAsString(ipv4), 16) AS ipv4c 194 | SELECT 195 | '192.168.1.1' AS ip, 196 | IPv4StringToNum(ip) AS ipv4, 197 | IPv6NumToString(ipv4c) AS ipv6 198 | 199 | ┌─ip──────────┬───────ipv4─┬─ipv6───────┐ 200 | │ 192.168.1.1 │ 3232235777 │ 101:a8c0:: │ 201 | └─────────────┴────────────┴────────────┘ 202 | ``` 203 | 204 | In order to convert it: 205 | ```sql 206 | WITH IPv6StringToNum(ip) AS ipv6 207 | SELECT 208 | '101:a8c0::' AS ip, 209 | reinterpretAsUInt32(ipv6) AS ipv6c, 210 | IPv4NumToString(ipv6c) AS ipv4 211 | 212 | ┌─ip─────────┬──────ipv6c─┬─ipv4────────┐ 213 | │ 101:a8c0:: │ 3232235777 │ 192.168.1.1 │ 214 | └────────────┴────────────┴─────────────┘ 215 | ``` 216 | 217 | Which for instance to display either IPv4 or IPv6 in a single query: 218 | ```sql 219 | SELECT 220 | if(EType = 0x800, IPv4NumToString(reinterpretAsUInt32(SrcAddr)), IPv6NumToString(SrcAddr) AS SrcIP 221 | ``` 222 | 223 | This will be fixed in future dashboard/db schema version. 224 | 225 | ## Information and roadmap 226 | 227 | This repository is an example and does not offer any warranties. I try to update it whenever I can. 228 | Contributions are welcome. 229 | 230 | The main purpose is for users to get started quickly and provide a basic system. 231 | This should not be used in production. 232 | 233 | I received requests to publish the Flink aggregator source code as you may have seen it 234 | being used in GoFlow presentations. 235 | Unfortunately, we moved entirely towards Clickhouse, the old code has not been updated in a while. 236 | It may get published at some point but this is currently low priority. 237 | 238 | ## Issue troubleshooting 239 | 240 | The compose files don't bind to specific versions of the containers. You will likely need to `down` in order to clean the setup (volumes, network), `push` to resynchronize repositories like GoFlow and `build` to rebuild components like inserter . 241 | 242 | ```bash 243 | $ docker-compose -f some-yaml-listed-above.yml down 244 | $ docker-compose -f some-yaml-listed-above.yml pull 245 | $ docker-compose -f some-yaml-listed-above.yml build 246 | $ docker-compose -f some-yaml-listed-above.yml up 247 | ``` 248 | -------------------------------------------------------------------------------- /compose/clickhouse/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | clickhouse client -n <<-EOSQL 5 | CREATE TABLE IF NOT EXISTS flows 6 | ( 7 | TimeReceived UInt64, 8 | TimeFlowStart UInt64, 9 | 10 | SequenceNum UInt32, 11 | SamplingRate UInt64, 12 | SamplerAddress FixedString(16), 13 | 14 | SrcAddr FixedString(16), 15 | DstAddr FixedString(16), 16 | 17 | SrcAS UInt32, 18 | DstAS UInt32, 19 | 20 | EType UInt32, 21 | Proto UInt32, 22 | 23 | SrcPort UInt32, 24 | DstPort UInt32, 25 | 26 | Bytes UInt64, 27 | Packets UInt64 28 | ) ENGINE = Kafka() 29 | SETTINGS 30 | kafka_broker_list = 'kafka:9092', 31 | kafka_topic_list = 'flows', 32 | kafka_group_name = 'clickhouse', 33 | kafka_format = 'Protobuf', 34 | kafka_schema = 'flow.proto:FlowMessage'; 35 | 36 | CREATE TABLE IF NOT EXISTS flows_raw 37 | ( 38 | Date Date, 39 | TimeReceived DateTime, 40 | TimeFlowStart DateTime, 41 | 42 | SequenceNum UInt32, 43 | SamplingRate UInt64, 44 | SamplerAddress FixedString(16), 45 | 46 | SrcAddr FixedString(16), 47 | DstAddr FixedString(16), 48 | 49 | SrcAS UInt32, 50 | DstAS UInt32, 51 | 52 | EType UInt32, 53 | Proto UInt32, 54 | 55 | SrcPort UInt32, 56 | DstPort UInt32, 57 | 58 | Bytes UInt64, 59 | Packets UInt64 60 | ) ENGINE = MergeTree() 61 | PARTITION BY Date 62 | ORDER BY TimeReceived; 63 | 64 | CREATE MATERIALIZED VIEW IF NOT EXISTS flows_raw_view TO flows_raw 65 | AS SELECT 66 | toDate(TimeReceived) AS Date, 67 | * 68 | FROM flows; 69 | 70 | CREATE TABLE IF NOT EXISTS flows_5m 71 | ( 72 | Date Date, 73 | Timeslot DateTime, 74 | 75 | SrcAS UInt32, 76 | DstAS UInt32, 77 | 78 | ETypeMap Nested ( 79 | EType UInt32, 80 | Bytes UInt64, 81 | Packets UInt64, 82 | Count UInt64 83 | ), 84 | 85 | Bytes UInt64, 86 | Packets UInt64, 87 | Count UInt64 88 | ) ENGINE = SummingMergeTree() 89 | PARTITION BY Date 90 | ORDER BY (Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`); 91 | 92 | CREATE MATERIALIZED VIEW IF NOT EXISTS flows_5m_view TO flows_5m 93 | AS 94 | SELECT 95 | Date, 96 | toStartOfFiveMinute(TimeReceived) AS Timeslot, 97 | SrcAS, 98 | DstAS, 99 | 100 | [EType] AS \`ETypeMap.EType\`, 101 | [Bytes] AS \`ETypeMap.Bytes\`, 102 | [Packets] AS \`ETypeMap.Packets\`, 103 | [Count] AS \`ETypeMap.Count\`, 104 | 105 | sum(Bytes) AS Bytes, 106 | sum(Packets) AS Packets, 107 | count() AS Count 108 | 109 | FROM flows_raw 110 | GROUP BY Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`; 111 | 112 | EOSQL -------------------------------------------------------------------------------- /compose/docker-compose-clickhouse-collect.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | zookeeper: 4 | image: 'bitnami/zookeeper:latest' 5 | ports: 6 | - '2181:2181' 7 | environment: 8 | - ALLOW_ANONYMOUS_LOGIN=yes 9 | restart: always 10 | kafka: 11 | image: 'bitnami/kafka:latest' 12 | ports: 13 | - '9092:9092' 14 | environment: 15 | - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 16 | - ALLOW_PLAINTEXT_LISTENER=yes 17 | - KAFKA_DELETE_TOPIC_ENABLE=true 18 | restart: always 19 | depends_on: 20 | - 'zookeeper' 21 | initializer: 22 | image: 'bitnami/kafka:latest' 23 | depends_on: 24 | - 'kafka' 25 | entrypoint: '/bin/bash' 26 | command: > 27 | -c "sleep 15 ; 28 | kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 2 --topic flows ;" 29 | grafana: 30 | build: ../grafana 31 | environment: 32 | - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource 33 | ports: 34 | - '3000:3000' 35 | restart: always 36 | volumes: 37 | - ./grafana/datasources-ch.yml:/etc/grafana/provisioning/datasources/datasources-ch.yml 38 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 39 | - ./grafana/dashboards:/var/lib/grafana/dashboards 40 | prometheus: 41 | image: 'prom/prometheus' 42 | ports: 43 | - '9090:9090' 44 | restart: always 45 | volumes: 46 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 47 | goflow: 48 | image: 'cloudflare/goflow:latest' 49 | depends_on: 50 | - 'initializer' 51 | ports: 52 | - '8080:8080' 53 | - '6343:6343/udp' 54 | - '2055:2055/udp' 55 | restart: always 56 | entrypoint: '/bin/sh' 57 | command: > 58 | -c "sleep 15 ; 59 | /goflow -kafka.brokers kafka:9092 60 | -kafka.topic flows 61 | -proto.fixedlen=true 62 | " 63 | db: 64 | image: yandex/clickhouse-server 65 | ports: 66 | - 8123:8123 67 | volumes: 68 | - ./clickhouse:/docker-entrypoint-initdb.d/ 69 | - ../pb-ext/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto 70 | depends_on: 71 | - 'kafka' -------------------------------------------------------------------------------- /compose/docker-compose-clickhouse-mock.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | kafka: 4 | image: 'bitnami/kafka:latest' 5 | ports: 6 | - '9092:9092' 7 | environment: 8 | - ALLOW_PLAINTEXT_LISTENER=yes 9 | - KAFKA_DELETE_TOPIC_ENABLE=true 10 | restart: always 11 | initializer: 12 | image: 'bitnami/kafka:latest' 13 | depends_on: 14 | - 'kafka' 15 | entrypoint: '/bin/bash' 16 | command: > 17 | -c "sleep 15 ; 18 | kafka-topics.sh --create --bootstrap-server kafka:9092 --replication-factor 1 --partitions 2 --topic flows ;" 19 | grafana: 20 | build: ../grafana 21 | environment: 22 | - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource 23 | ports: 24 | - '3000:3000' 25 | restart: always 26 | volumes: 27 | - ./grafana/datasources-ch.yml:/etc/grafana/provisioning/datasources/datasources-ch.yml 28 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 29 | - ./grafana/dashboards:/var/lib/grafana/dashboards 30 | mocker: 31 | build: 32 | context: ../ 33 | dockerfile: Dockerfile.mocker 34 | restart: always 35 | command: > 36 | -kafka.brokers kafka:9092 -kafka.topic flows -proto.fixedlen=true 37 | db: 38 | image: yandex/clickhouse-server 39 | ports: 40 | - 8123:8123 41 | volumes: 42 | - ./clickhouse:/docker-entrypoint-initdb.d/ 43 | - ../pb-ext/flow.proto:/var/lib/clickhouse/format_schemas/flow.proto 44 | depends_on: 45 | - 'kafka' -------------------------------------------------------------------------------- /compose/docker-compose-postgres-collect.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zookeeper: 4 | image: 'bitnami/zookeeper:latest' 5 | ports: 6 | - '2181:2181' 7 | environment: 8 | - ALLOW_ANONYMOUS_LOGIN=yes 9 | restart: always 10 | kafka: 11 | image: 'bitnami/kafka:latest' 12 | ports: 13 | - '9092:9092' 14 | environment: 15 | - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 16 | - ALLOW_PLAINTEXT_LISTENER=yes 17 | - KAFKA_DELETE_TOPIC_ENABLE=true 18 | restart: always 19 | depends_on: 20 | - 'zookeeper' 21 | initializer: 22 | image: 'bitnami/kafka:latest' 23 | depends_on: 24 | - 'kafka' 25 | entrypoint: '/bin/bash' 26 | command: > 27 | -c "sleep 15 ; 28 | kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 2 --topic flows ;" 29 | goflow: 30 | image: 'cloudflare/goflow:latest' 31 | depends_on: 32 | - 'initializer' 33 | ports: 34 | - '8080:8080' 35 | - '6343:6343/udp' 36 | - '2055:2055/udp' 37 | restart: always 38 | entrypoint: '/bin/sh' 39 | command: > 40 | -c "sleep 15 ; 41 | /goflow -kafka.brokers kafka:9092 42 | -kafka.topic flows" 43 | grafana: 44 | image: 'grafana/grafana:latest' 45 | ports: 46 | - '3000:3000' 47 | restart: always 48 | volumes: 49 | - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml 50 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 51 | - ./grafana/dashboards:/var/lib/grafana/dashboards 52 | prometheus: 53 | image: 'prom/prometheus' 54 | ports: 55 | - '9090:9090' 56 | restart: always 57 | volumes: 58 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 59 | postgres: 60 | image: postgres:latest 61 | ports: 62 | - '5432:5432' 63 | environment: 64 | POSTGRES_PASSWORD: flows 65 | restart: always 66 | volumes: 67 | - ./postgres:/docker-entrypoint-initdb.d 68 | inserter: 69 | build: 70 | context: ../ 71 | dockerfile: Dockerfile.inserter 72 | restart: always 73 | command: 74 | -kafka.brokers kafka:9092 -kafka.topic flows -postgres.host postgres -postgres.port 5432 -postgres.pass flows -------------------------------------------------------------------------------- /compose/docker-compose-postgres-mock.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zookeeper: 4 | image: 'bitnami/zookeeper:latest' 5 | ports: 6 | - '2181:2181' 7 | environment: 8 | - ALLOW_ANONYMOUS_LOGIN=yes 9 | restart: always 10 | kafka: 11 | image: 'bitnami/kafka:latest' 12 | ports: 13 | - '9092:9092' 14 | environment: 15 | - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 16 | - ALLOW_PLAINTEXT_LISTENER=yes 17 | - KAFKA_DELETE_TOPIC_ENABLE=true 18 | restart: always 19 | depends_on: 20 | - 'zookeeper' 21 | initializer: 22 | image: 'bitnami/kafka:latest' 23 | depends_on: 24 | - 'kafka' 25 | entrypoint: '/bin/bash' 26 | command: > 27 | -c "sleep 15 ; 28 | kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 2 --topic flows" 29 | grafana: 30 | image: 'grafana/grafana:latest' 31 | ports: 32 | - '3000:3000' 33 | restart: always 34 | volumes: 35 | - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml 36 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 37 | - ./grafana/dashboards:/var/lib/grafana/dashboards 38 | prometheus: 39 | image: 'prom/prometheus' 40 | ports: 41 | - '9090:9090' 42 | restart: always 43 | volumes: 44 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 45 | postgres: 46 | image: postgres:latest 47 | ports: 48 | - '5432:5432' 49 | environment: 50 | POSTGRES_PASSWORD: flows 51 | restart: always 52 | volumes: 53 | - ./postgres:/docker-entrypoint-initdb.d 54 | inserter: 55 | build: 56 | context: ../ 57 | dockerfile: Dockerfile.inserter 58 | restart: always 59 | command: > 60 | -kafka.brokers kafka:9092 -kafka.topic flows 61 | -postgres.host postgres -postgres.port 5432 -postgres.pass flows 62 | mocker: 63 | build: 64 | context: ../ 65 | dockerfile: Dockerfile.mocker 66 | restart: always 67 | command: > 68 | -kafka.brokers kafka:9092 -kafka.topic flows -------------------------------------------------------------------------------- /compose/grafana/dashboards.yml: -------------------------------------------------------------------------------- 1 | - name: 'default' 2 | org_id: 1 3 | folder: '' 4 | type: file 5 | options: 6 | folder: /var/lib/grafana/dashboards -------------------------------------------------------------------------------- /compose/grafana/dashboards/perfs.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "description": "Metrics about the NetFlow+sFlow collector", 16 | "editable": true, 17 | "gnetId": null, 18 | "graphTooltip": 0, 19 | "id": 2, 20 | "links": [], 21 | "panels": [ 22 | { 23 | "collapsed": false, 24 | "gridPos": { 25 | "h": 1, 26 | "w": 24, 27 | "x": 0, 28 | "y": 0 29 | }, 30 | "id": 53, 31 | "panels": [], 32 | "repeat": null, 33 | "title": "Totals", 34 | "type": "row" 35 | }, 36 | { 37 | "cacheTimeout": null, 38 | "colorBackground": false, 39 | "colorValue": false, 40 | "colors": [ 41 | "rgba(245, 54, 54, 0.9)", 42 | "rgba(237, 129, 40, 0.89)", 43 | "rgba(50, 172, 45, 0.97)" 44 | ], 45 | "datasource": "Prometheus", 46 | "format": "pps", 47 | "gauge": { 48 | "maxValue": 100, 49 | "minValue": 0, 50 | "show": false, 51 | "thresholdLabels": false, 52 | "thresholdMarkers": true 53 | }, 54 | "gridPos": { 55 | "h": 7, 56 | "w": 12, 57 | "x": 0, 58 | "y": 1 59 | }, 60 | "id": 9, 61 | "interval": null, 62 | "links": [], 63 | "mappingType": 1, 64 | "mappingTypes": [ 65 | { 66 | "name": "value to text", 67 | "value": 1 68 | }, 69 | { 70 | "name": "range to text", 71 | "value": 2 72 | } 73 | ], 74 | "maxDataPoints": 100, 75 | "nullPointMode": "connected", 76 | "nullText": null, 77 | "postfix": "", 78 | "postfixFontSize": "50%", 79 | "prefix": "", 80 | "prefixFontSize": "50%", 81 | "rangeMaps": [ 82 | { 83 | "from": "null", 84 | "text": "N/A", 85 | "to": "null" 86 | } 87 | ], 88 | "sparkline": { 89 | "fillColor": "rgba(31, 118, 189, 0.18)", 90 | "full": true, 91 | "lineColor": "rgb(31, 120, 193)", 92 | "show": true 93 | }, 94 | "tableColumn": "", 95 | "targets": [ 96 | { 97 | "expr": "sum(rate(flow_process_nf_flowset_records_sum[5m]))", 98 | "format": "time_series", 99 | "intervalFactor": 1, 100 | "refId": "A" 101 | } 102 | ], 103 | "thresholds": "", 104 | "title": "NetFlows Total", 105 | "type": "singlestat", 106 | "valueFontSize": "80%", 107 | "valueMaps": [ 108 | { 109 | "op": "=", 110 | "text": "N/A", 111 | "value": "null" 112 | } 113 | ], 114 | "valueName": "avg" 115 | }, 116 | { 117 | "cacheTimeout": null, 118 | "colorBackground": false, 119 | "colorValue": false, 120 | "colors": [ 121 | "rgba(245, 54, 54, 0.9)", 122 | "rgba(237, 129, 40, 0.89)", 123 | "rgba(50, 172, 45, 0.97)" 124 | ], 125 | "datasource": "Prometheus", 126 | "format": "pps", 127 | "gauge": { 128 | "maxValue": 100, 129 | "minValue": 0, 130 | "show": false, 131 | "thresholdLabels": false, 132 | "thresholdMarkers": true 133 | }, 134 | "gridPos": { 135 | "h": 7, 136 | "w": 12, 137 | "x": 12, 138 | "y": 1 139 | }, 140 | "id": 10, 141 | "interval": null, 142 | "links": [], 143 | "mappingType": 1, 144 | "mappingTypes": [ 145 | { 146 | "name": "value to text", 147 | "value": 1 148 | }, 149 | { 150 | "name": "range to text", 151 | "value": 2 152 | } 153 | ], 154 | "maxDataPoints": 100, 155 | "nullPointMode": "connected", 156 | "nullText": null, 157 | "postfix": "", 158 | "postfixFontSize": "50%", 159 | "prefix": "", 160 | "prefixFontSize": "50%", 161 | "rangeMaps": [ 162 | { 163 | "from": "null", 164 | "text": "N/A", 165 | "to": "null" 166 | } 167 | ], 168 | "sparkline": { 169 | "fillColor": "rgba(31, 118, 189, 0.18)", 170 | "full": true, 171 | "lineColor": "rgb(31, 120, 193)", 172 | "show": true 173 | }, 174 | "tableColumn": "", 175 | "targets": [ 176 | { 177 | "expr": "sum(rate(flow_process_sf_samples_sum[5m]))", 178 | "format": "time_series", 179 | "intervalFactor": 1, 180 | "refId": "A" 181 | } 182 | ], 183 | "thresholds": "", 184 | "title": "sFlows Total", 185 | "type": "singlestat", 186 | "valueFontSize": "80%", 187 | "valueMaps": [ 188 | { 189 | "op": "=", 190 | "text": "N/A", 191 | "value": "null" 192 | } 193 | ], 194 | "valueName": "avg" 195 | }, 196 | { 197 | "collapsed": false, 198 | "gridPos": { 199 | "h": 1, 200 | "w": 24, 201 | "x": 0, 202 | "y": 8 203 | }, 204 | "id": 54, 205 | "panels": [], 206 | "repeat": null, 207 | "title": "Flows UDP Metrics", 208 | "type": "row" 209 | }, 210 | { 211 | "aliasColors": {}, 212 | "bars": false, 213 | "dashLength": 10, 214 | "dashes": false, 215 | "datasource": "Prometheus", 216 | "fill": 10, 217 | "gridPos": { 218 | "h": 7, 219 | "w": 12, 220 | "x": 0, 221 | "y": 9 222 | }, 223 | "id": 51, 224 | "legend": { 225 | "alignAsTable": true, 226 | "avg": true, 227 | "current": true, 228 | "max": false, 229 | "min": false, 230 | "rightSide": true, 231 | "show": true, 232 | "sort": "current", 233 | "sortDesc": true, 234 | "total": false, 235 | "values": true 236 | }, 237 | "lines": true, 238 | "linewidth": 1, 239 | "links": [], 240 | "nullPointMode": "null", 241 | "percentage": false, 242 | "pointradius": 5, 243 | "points": false, 244 | "renderer": "flot", 245 | "seriesOverrides": [ 246 | { 247 | "alias": "/Total*/", 248 | "color": "#DEDAF7", 249 | "fill": 0, 250 | "stack": false 251 | } 252 | ], 253 | "spaceLength": 10, 254 | "stack": true, 255 | "steppedLine": false, 256 | "targets": [ 257 | { 258 | "expr": "sum(rate(udp_traffic_bytes[5m])) by (remote_ip)", 259 | "format": "time_series", 260 | "intervalFactor": 1, 261 | "legendFormat": "{{ remote_ip }}", 262 | "refId": "A" 263 | }, 264 | { 265 | "expr": "sum(rate(udp_traffic_bytes[5m]))", 266 | "format": "time_series", 267 | "intervalFactor": 1, 268 | "legendFormat": "Total", 269 | "refId": "B" 270 | } 271 | ], 272 | "thresholds": [], 273 | "timeFrom": null, 274 | "timeRegions": [], 275 | "timeShift": null, 276 | "title": "UDP2Kafka Bandwidth", 277 | "tooltip": { 278 | "shared": false, 279 | "sort": 0, 280 | "value_type": "individual" 281 | }, 282 | "type": "graph", 283 | "xaxis": { 284 | "buckets": null, 285 | "mode": "time", 286 | "name": null, 287 | "show": true, 288 | "values": [] 289 | }, 290 | "yaxes": [ 291 | { 292 | "format": "Bps", 293 | "label": null, 294 | "logBase": 1, 295 | "max": null, 296 | "min": "0", 297 | "show": true 298 | }, 299 | { 300 | "format": "short", 301 | "label": null, 302 | "logBase": 1, 303 | "max": null, 304 | "min": null, 305 | "show": true 306 | } 307 | ], 308 | "yaxis": { 309 | "align": false, 310 | "alignLevel": null 311 | } 312 | }, 313 | { 314 | "aliasColors": {}, 315 | "bars": false, 316 | "dashLength": 10, 317 | "dashes": false, 318 | "datasource": "Prometheus", 319 | "fill": 10, 320 | "gridPos": { 321 | "h": 7, 322 | "w": 12, 323 | "x": 12, 324 | "y": 9 325 | }, 326 | "id": 52, 327 | "legend": { 328 | "alignAsTable": true, 329 | "avg": true, 330 | "current": true, 331 | "max": false, 332 | "min": false, 333 | "rightSide": true, 334 | "show": true, 335 | "sort": "current", 336 | "sortDesc": true, 337 | "total": false, 338 | "values": true 339 | }, 340 | "lines": true, 341 | "linewidth": 1, 342 | "links": [], 343 | "nullPointMode": "null", 344 | "percentage": false, 345 | "pointradius": 5, 346 | "points": false, 347 | "renderer": "flot", 348 | "seriesOverrides": [ 349 | { 350 | "alias": "/Total*/", 351 | "color": "#DEDAF7", 352 | "fill": 0, 353 | "stack": false 354 | } 355 | ], 356 | "spaceLength": 10, 357 | "stack": true, 358 | "steppedLine": false, 359 | "targets": [ 360 | { 361 | "expr": "sum(rate(udp_traffic_packets[5m])) by (remote_ip)", 362 | "format": "time_series", 363 | "intervalFactor": 1, 364 | "legendFormat": "{{ remote_ip }}", 365 | "refId": "A" 366 | }, 367 | { 368 | "expr": "sum(rate(udp_traffic_packets[5m]))", 369 | "format": "time_series", 370 | "intervalFactor": 1, 371 | "legendFormat": "Total", 372 | "refId": "B" 373 | } 374 | ], 375 | "thresholds": [], 376 | "timeFrom": null, 377 | "timeRegions": [], 378 | "timeShift": null, 379 | "title": "UDP2Kafka packets", 380 | "tooltip": { 381 | "shared": false, 382 | "sort": 0, 383 | "value_type": "individual" 384 | }, 385 | "type": "graph", 386 | "xaxis": { 387 | "buckets": null, 388 | "mode": "time", 389 | "name": null, 390 | "show": true, 391 | "values": [] 392 | }, 393 | "yaxes": [ 394 | { 395 | "format": "pps", 396 | "label": "", 397 | "logBase": 1, 398 | "max": null, 399 | "min": "0", 400 | "show": true 401 | }, 402 | { 403 | "format": "short", 404 | "label": null, 405 | "logBase": 1, 406 | "max": null, 407 | "min": null, 408 | "show": true 409 | } 410 | ], 411 | "yaxis": { 412 | "align": false, 413 | "alignLevel": null 414 | } 415 | }, 416 | { 417 | "aliasColors": {}, 418 | "bars": false, 419 | "dashLength": 10, 420 | "dashes": false, 421 | "datasource": "Prometheus", 422 | "fill": 10, 423 | "gridPos": { 424 | "h": 7, 425 | "w": 12, 426 | "x": 0, 427 | "y": 16 428 | }, 429 | "id": 1, 430 | "legend": { 431 | "alignAsTable": true, 432 | "avg": true, 433 | "current": true, 434 | "max": false, 435 | "min": false, 436 | "rightSide": true, 437 | "show": true, 438 | "sort": "current", 439 | "sortDesc": true, 440 | "total": false, 441 | "values": true 442 | }, 443 | "lines": true, 444 | "linewidth": 1, 445 | "links": [], 446 | "nullPointMode": "null", 447 | "percentage": false, 448 | "pointradius": 5, 449 | "points": false, 450 | "renderer": "flot", 451 | "seriesOverrides": [ 452 | { 453 | "alias": "/Total*/", 454 | "color": "#DEDAF7", 455 | "fill": 0, 456 | "stack": false 457 | } 458 | ], 459 | "spaceLength": 10, 460 | "stack": true, 461 | "steppedLine": false, 462 | "targets": [ 463 | { 464 | "expr": "sum(rate(flow_traffic_bytes{type=\"sFlow\"}[5m])) by (remote_ip)", 465 | "format": "time_series", 466 | "intervalFactor": 1, 467 | "legendFormat": "{{ remote_ip }}", 468 | "refId": "A" 469 | }, 470 | { 471 | "aggregator": "sum", 472 | "alias": "$tag_type | Total", 473 | "currentTagKey": "", 474 | "currentTagValue": "", 475 | "downsampleAggregator": "avg", 476 | "downsampleFillPolicy": "none", 477 | "downsampleInterval": "", 478 | "expr": "sum(rate(flow_traffic_bytes{type=\"sFlow\"}[5m]))", 479 | "format": "time_series", 480 | "intervalFactor": 1, 481 | "legendFormat": "sFlow | Total", 482 | "metric": "goflow.flow_traffic_bytes", 483 | "refId": "B", 484 | "shouldComputeRate": true, 485 | "tags": { 486 | "type": "sFlow" 487 | } 488 | } 489 | ], 490 | "thresholds": [], 491 | "timeFrom": null, 492 | "timeRegions": [], 493 | "timeShift": null, 494 | "title": "sFlow UDP Bandwidth", 495 | "tooltip": { 496 | "shared": false, 497 | "sort": 0, 498 | "value_type": "individual" 499 | }, 500 | "type": "graph", 501 | "xaxis": { 502 | "buckets": null, 503 | "mode": "time", 504 | "name": null, 505 | "show": true, 506 | "values": [] 507 | }, 508 | "yaxes": [ 509 | { 510 | "format": "Bps", 511 | "label": null, 512 | "logBase": 1, 513 | "max": null, 514 | "min": "0", 515 | "show": true 516 | }, 517 | { 518 | "format": "short", 519 | "label": null, 520 | "logBase": 1, 521 | "max": null, 522 | "min": null, 523 | "show": true 524 | } 525 | ], 526 | "yaxis": { 527 | "align": false, 528 | "alignLevel": null 529 | } 530 | }, 531 | { 532 | "aliasColors": {}, 533 | "bars": false, 534 | "dashLength": 10, 535 | "dashes": false, 536 | "datasource": "Prometheus", 537 | "fill": 10, 538 | "gridPos": { 539 | "h": 7, 540 | "w": 12, 541 | "x": 12, 542 | "y": 16 543 | }, 544 | "id": 2, 545 | "legend": { 546 | "alignAsTable": true, 547 | "avg": true, 548 | "current": true, 549 | "max": false, 550 | "min": false, 551 | "rightSide": true, 552 | "show": true, 553 | "sort": "current", 554 | "sortDesc": true, 555 | "total": false, 556 | "values": true 557 | }, 558 | "lines": true, 559 | "linewidth": 1, 560 | "links": [], 561 | "nullPointMode": "null", 562 | "percentage": false, 563 | "pointradius": 5, 564 | "points": false, 565 | "renderer": "flot", 566 | "seriesOverrides": [ 567 | { 568 | "alias": "/Total*/", 569 | "color": "#DEDAF7", 570 | "fill": 0, 571 | "stack": false 572 | } 573 | ], 574 | "spaceLength": 10, 575 | "stack": true, 576 | "steppedLine": false, 577 | "targets": [ 578 | { 579 | "expr": "sum(rate(flow_traffic_packets{type=\"sFlow\"}[5m])) by (remote_ip)", 580 | "format": "time_series", 581 | "intervalFactor": 1, 582 | "legendFormat": "{{ remote_ip }}", 583 | "refId": "A" 584 | }, 585 | { 586 | "aggregator": "sum", 587 | "alias": "$tag_type | Total", 588 | "currentTagKey": "", 589 | "currentTagValue": "", 590 | "downsampleAggregator": "avg", 591 | "downsampleFillPolicy": "none", 592 | "downsampleInterval": "", 593 | "expr": "sum(rate(flow_traffic_packets{type=\"sFlow\"}[5m]))", 594 | "format": "time_series", 595 | "intervalFactor": 1, 596 | "legendFormat": "sFlow | Total", 597 | "metric": "goflow-kafka.flow_traffic_packets", 598 | "refId": "B", 599 | "shouldComputeRate": true, 600 | "tags": { 601 | "type": "sFlow" 602 | } 603 | } 604 | ], 605 | "thresholds": [], 606 | "timeFrom": null, 607 | "timeRegions": [], 608 | "timeShift": null, 609 | "title": "sFlow UDP Packets", 610 | "tooltip": { 611 | "shared": false, 612 | "sort": 0, 613 | "value_type": "individual" 614 | }, 615 | "type": "graph", 616 | "xaxis": { 617 | "buckets": null, 618 | "mode": "time", 619 | "name": null, 620 | "show": true, 621 | "values": [] 622 | }, 623 | "yaxes": [ 624 | { 625 | "format": "pps", 626 | "label": "", 627 | "logBase": 1, 628 | "max": null, 629 | "min": "0", 630 | "show": true 631 | }, 632 | { 633 | "format": "short", 634 | "label": null, 635 | "logBase": 1, 636 | "max": null, 637 | "min": null, 638 | "show": true 639 | } 640 | ], 641 | "yaxis": { 642 | "align": false, 643 | "alignLevel": null 644 | } 645 | }, 646 | { 647 | "aliasColors": {}, 648 | "bars": false, 649 | "dashLength": 10, 650 | "dashes": false, 651 | "datasource": "Prometheus", 652 | "fill": 10, 653 | "gridPos": { 654 | "h": 7, 655 | "w": 12, 656 | "x": 0, 657 | "y": 23 658 | }, 659 | "id": 48, 660 | "legend": { 661 | "alignAsTable": true, 662 | "avg": true, 663 | "current": true, 664 | "max": false, 665 | "min": false, 666 | "rightSide": true, 667 | "show": true, 668 | "sort": "current", 669 | "sortDesc": true, 670 | "total": false, 671 | "values": true 672 | }, 673 | "lines": true, 674 | "linewidth": 1, 675 | "links": [], 676 | "nullPointMode": "null", 677 | "percentage": false, 678 | "pointradius": 5, 679 | "points": false, 680 | "renderer": "flot", 681 | "seriesOverrides": [ 682 | { 683 | "alias": "/Total*/", 684 | "color": "#DEDAF7", 685 | "fill": 0, 686 | "stack": false 687 | } 688 | ], 689 | "spaceLength": 10, 690 | "stack": true, 691 | "steppedLine": false, 692 | "targets": [ 693 | { 694 | "expr": "sum(rate(flow_traffic_bytes{type=\"NetFlow\"}[5m])) by (remote_ip)", 695 | "format": "time_series", 696 | "intervalFactor": 1, 697 | "legendFormat": "{{ remote_ip }}", 698 | "refId": "A" 699 | }, 700 | { 701 | "expr": "sum(rate(flow_traffic_bytes{type=\"NetFlow\"}[5m]))", 702 | "format": "time_series", 703 | "intervalFactor": 1, 704 | "legendFormat": "NetFlow | Total", 705 | "refId": "B" 706 | } 707 | ], 708 | "thresholds": [], 709 | "timeFrom": null, 710 | "timeRegions": [], 711 | "timeShift": null, 712 | "title": "NetFlow UDP Bandwidth", 713 | "tooltip": { 714 | "shared": false, 715 | "sort": 0, 716 | "value_type": "individual" 717 | }, 718 | "type": "graph", 719 | "xaxis": { 720 | "buckets": null, 721 | "mode": "time", 722 | "name": null, 723 | "show": true, 724 | "values": [] 725 | }, 726 | "yaxes": [ 727 | { 728 | "format": "Bps", 729 | "label": null, 730 | "logBase": 1, 731 | "max": null, 732 | "min": "0", 733 | "show": true 734 | }, 735 | { 736 | "format": "short", 737 | "label": null, 738 | "logBase": 1, 739 | "max": null, 740 | "min": null, 741 | "show": true 742 | } 743 | ], 744 | "yaxis": { 745 | "align": false, 746 | "alignLevel": null 747 | } 748 | }, 749 | { 750 | "aliasColors": {}, 751 | "bars": false, 752 | "dashLength": 10, 753 | "dashes": false, 754 | "datasource": "Prometheus", 755 | "fill": 10, 756 | "gridPos": { 757 | "h": 7, 758 | "w": 12, 759 | "x": 12, 760 | "y": 23 761 | }, 762 | "id": 47, 763 | "legend": { 764 | "alignAsTable": true, 765 | "avg": true, 766 | "current": true, 767 | "max": false, 768 | "min": false, 769 | "rightSide": true, 770 | "show": true, 771 | "sort": "current", 772 | "sortDesc": true, 773 | "total": false, 774 | "values": true 775 | }, 776 | "lines": true, 777 | "linewidth": 1, 778 | "links": [], 779 | "nullPointMode": "null", 780 | "percentage": false, 781 | "pointradius": 5, 782 | "points": false, 783 | "renderer": "flot", 784 | "seriesOverrides": [ 785 | { 786 | "alias": "/Total*/", 787 | "color": "#DEDAF7", 788 | "fill": 0, 789 | "stack": false 790 | } 791 | ], 792 | "spaceLength": 10, 793 | "stack": true, 794 | "steppedLine": false, 795 | "targets": [ 796 | { 797 | "expr": "sum(rate(flow_traffic_packets{type=\"NetFlow\"}[5m])) by (remote_ip)", 798 | "format": "time_series", 799 | "intervalFactor": 1, 800 | "legendFormat": "{{ remote_ip }}", 801 | "refId": "A" 802 | }, 803 | { 804 | "aggregator": "sum", 805 | "alias": "$tag_type | Total", 806 | "currentTagKey": "", 807 | "currentTagValue": "", 808 | "downsampleAggregator": "avg", 809 | "downsampleFillPolicy": "none", 810 | "downsampleInterval": "", 811 | "expr": "sum(rate(flow_traffic_packets{type=\"NetFlow\"}[5m]))", 812 | "format": "time_series", 813 | "intervalFactor": 1, 814 | "legendFormat": "NetFlow | Total", 815 | "metric": "goflow-kafka.flow_traffic_packets", 816 | "refId": "B", 817 | "shouldComputeRate": true, 818 | "tags": { 819 | "type": "NetFlow" 820 | } 821 | } 822 | ], 823 | "thresholds": [], 824 | "timeFrom": null, 825 | "timeRegions": [], 826 | "timeShift": null, 827 | "title": "NetFlow UDP Packets", 828 | "tooltip": { 829 | "shared": false, 830 | "sort": 0, 831 | "value_type": "individual" 832 | }, 833 | "type": "graph", 834 | "xaxis": { 835 | "buckets": null, 836 | "mode": "time", 837 | "name": null, 838 | "show": true, 839 | "values": [] 840 | }, 841 | "yaxes": [ 842 | { 843 | "format": "pps", 844 | "label": "", 845 | "logBase": 1, 846 | "max": null, 847 | "min": "0", 848 | "show": true 849 | }, 850 | { 851 | "format": "short", 852 | "label": null, 853 | "logBase": 1, 854 | "max": null, 855 | "min": null, 856 | "show": true 857 | } 858 | ], 859 | "yaxis": { 860 | "align": false, 861 | "alignLevel": null 862 | } 863 | }, 864 | { 865 | "collapsed": false, 866 | "gridPos": { 867 | "h": 1, 868 | "w": 24, 869 | "x": 0, 870 | "y": 30 871 | }, 872 | "id": 55, 873 | "panels": [], 874 | "repeat": null, 875 | "title": "NetFlow metrics", 876 | "type": "row" 877 | }, 878 | { 879 | "aliasColors": {}, 880 | "bars": false, 881 | "dashLength": 10, 882 | "dashes": false, 883 | "datasource": "Prometheus", 884 | "fill": 0, 885 | "gridPos": { 886 | "h": 7, 887 | "w": 12, 888 | "x": 0, 889 | "y": 31 890 | }, 891 | "id": 3, 892 | "legend": { 893 | "alignAsTable": true, 894 | "avg": true, 895 | "current": true, 896 | "max": true, 897 | "min": true, 898 | "show": true, 899 | "total": false, 900 | "values": true 901 | }, 902 | "lines": true, 903 | "linewidth": 1, 904 | "links": [], 905 | "nullPointMode": "null", 906 | "percentage": false, 907 | "pointradius": 5, 908 | "points": false, 909 | "renderer": "flot", 910 | "seriesOverrides": [], 911 | "spaceLength": 10, 912 | "stack": false, 913 | "steppedLine": false, 914 | "targets": [ 915 | { 916 | "expr": "sum(rate(flow_process_nf_flowset_records_sum[5m])) by (version,type)", 917 | "format": "time_series", 918 | "intervalFactor": 1, 919 | "legendFormat": "{{ version }} | {{ type }} ", 920 | "refId": "A" 921 | } 922 | ], 923 | "thresholds": [], 924 | "timeFrom": null, 925 | "timeRegions": [], 926 | "timeShift": null, 927 | "title": "NetFlows by type and version", 928 | "tooltip": { 929 | "shared": true, 930 | "sort": 0, 931 | "value_type": "individual" 932 | }, 933 | "type": "graph", 934 | "xaxis": { 935 | "buckets": null, 936 | "mode": "time", 937 | "name": null, 938 | "show": true, 939 | "values": [] 940 | }, 941 | "yaxes": [ 942 | { 943 | "format": "pps", 944 | "label": null, 945 | "logBase": 1, 946 | "max": null, 947 | "min": null, 948 | "show": true 949 | }, 950 | { 951 | "format": "short", 952 | "label": null, 953 | "logBase": 1, 954 | "max": null, 955 | "min": null, 956 | "show": true 957 | } 958 | ], 959 | "yaxis": { 960 | "align": false, 961 | "alignLevel": null 962 | } 963 | }, 964 | { 965 | "aliasColors": {}, 966 | "bars": false, 967 | "dashLength": 10, 968 | "dashes": false, 969 | "datasource": "Prometheus", 970 | "fill": 1, 971 | "gridPos": { 972 | "h": 7, 973 | "w": 12, 974 | "x": 12, 975 | "y": 31 976 | }, 977 | "id": 4, 978 | "legend": { 979 | "alignAsTable": true, 980 | "avg": true, 981 | "current": true, 982 | "max": true, 983 | "min": true, 984 | "show": true, 985 | "total": false, 986 | "values": true 987 | }, 988 | "lines": true, 989 | "linewidth": 1, 990 | "links": [], 991 | "nullPointMode": "null", 992 | "percentage": false, 993 | "pointradius": 5, 994 | "points": false, 995 | "renderer": "flot", 996 | "seriesOverrides": [], 997 | "spaceLength": 10, 998 | "stack": false, 999 | "steppedLine": false, 1000 | "targets": [ 1001 | { 1002 | "expr": "sum(rate(flow_process_nf_errors_count[5m])) by (error)", 1003 | "format": "time_series", 1004 | "intervalFactor": 1, 1005 | "legendFormat": "{{ error }}", 1006 | "metric": "goflow-kafka.flow_process_nf_errors_count", 1007 | "refId": "A" 1008 | } 1009 | ], 1010 | "thresholds": [], 1011 | "timeFrom": null, 1012 | "timeRegions": [], 1013 | "timeShift": null, 1014 | "title": "NetFlows errors", 1015 | "tooltip": { 1016 | "shared": true, 1017 | "sort": 0, 1018 | "value_type": "individual" 1019 | }, 1020 | "type": "graph", 1021 | "xaxis": { 1022 | "buckets": null, 1023 | "mode": "time", 1024 | "name": null, 1025 | "show": true, 1026 | "values": [] 1027 | }, 1028 | "yaxes": [ 1029 | { 1030 | "format": "hertz", 1031 | "label": null, 1032 | "logBase": 1, 1033 | "max": null, 1034 | "min": null, 1035 | "show": true 1036 | }, 1037 | { 1038 | "format": "short", 1039 | "label": null, 1040 | "logBase": 1, 1041 | "max": null, 1042 | "min": null, 1043 | "show": true 1044 | } 1045 | ], 1046 | "yaxis": { 1047 | "align": false, 1048 | "alignLevel": null 1049 | } 1050 | }, 1051 | { 1052 | "aliasColors": {}, 1053 | "bars": false, 1054 | "dashLength": 10, 1055 | "dashes": false, 1056 | "datasource": "Prometheus", 1057 | "fill": 0, 1058 | "gridPos": { 1059 | "h": 7, 1060 | "w": 12, 1061 | "x": 0, 1062 | "y": 38 1063 | }, 1064 | "id": 6, 1065 | "legend": { 1066 | "alignAsTable": true, 1067 | "avg": false, 1068 | "current": true, 1069 | "max": false, 1070 | "min": false, 1071 | "rightSide": true, 1072 | "show": true, 1073 | "sort": "current", 1074 | "sortDesc": true, 1075 | "total": false, 1076 | "values": true 1077 | }, 1078 | "lines": true, 1079 | "linewidth": 1, 1080 | "links": [], 1081 | "nullPointMode": "null", 1082 | "percentage": false, 1083 | "pointradius": 5, 1084 | "points": false, 1085 | "renderer": "flot", 1086 | "seriesOverrides": [], 1087 | "spaceLength": 10, 1088 | "stack": false, 1089 | "steppedLine": false, 1090 | "targets": [ 1091 | { 1092 | "expr": "sum(rate(flow_process_nf_templates_count[5m])) by (version,router,type)", 1093 | "format": "time_series", 1094 | "intervalFactor": 1, 1095 | "legendFormat": "{{ version }} | {{ router }} | {{ type }}", 1096 | "refId": "A" 1097 | } 1098 | ], 1099 | "thresholds": [], 1100 | "timeFrom": null, 1101 | "timeRegions": [], 1102 | "timeShift": null, 1103 | "title": "NetFlow templates", 1104 | "tooltip": { 1105 | "shared": false, 1106 | "sort": 0, 1107 | "value_type": "individual" 1108 | }, 1109 | "type": "graph", 1110 | "xaxis": { 1111 | "buckets": null, 1112 | "mode": "time", 1113 | "name": null, 1114 | "show": true, 1115 | "values": [] 1116 | }, 1117 | "yaxes": [ 1118 | { 1119 | "format": "pps", 1120 | "label": null, 1121 | "logBase": 1, 1122 | "max": null, 1123 | "min": null, 1124 | "show": true 1125 | }, 1126 | { 1127 | "format": "short", 1128 | "label": null, 1129 | "logBase": 1, 1130 | "max": null, 1131 | "min": null, 1132 | "show": true 1133 | } 1134 | ], 1135 | "yaxis": { 1136 | "align": false, 1137 | "alignLevel": null 1138 | } 1139 | }, 1140 | { 1141 | "aliasColors": {}, 1142 | "bars": false, 1143 | "dashLength": 10, 1144 | "dashes": false, 1145 | "datasource": "Prometheus", 1146 | "fill": 0, 1147 | "gridPos": { 1148 | "h": 7, 1149 | "w": 12, 1150 | "x": 12, 1151 | "y": 38 1152 | }, 1153 | "id": 7, 1154 | "legend": { 1155 | "alignAsTable": true, 1156 | "avg": false, 1157 | "current": true, 1158 | "max": false, 1159 | "min": false, 1160 | "rightSide": true, 1161 | "show": true, 1162 | "sort": "current", 1163 | "sortDesc": true, 1164 | "total": false, 1165 | "values": true 1166 | }, 1167 | "lines": true, 1168 | "linewidth": 1, 1169 | "links": [], 1170 | "nullPointMode": "null", 1171 | "percentage": false, 1172 | "pointradius": 5, 1173 | "points": false, 1174 | "renderer": "flot", 1175 | "seriesOverrides": [], 1176 | "spaceLength": 10, 1177 | "stack": false, 1178 | "steppedLine": false, 1179 | "targets": [ 1180 | { 1181 | "expr": "sum(rate(flow_process_nf_flowset_records_sum[5m])) by (router,version)", 1182 | "format": "time_series", 1183 | "intervalFactor": 1, 1184 | "legendFormat": "{{ router }} | {{ version }}", 1185 | "refId": "A" 1186 | } 1187 | ], 1188 | "thresholds": [], 1189 | "timeFrom": null, 1190 | "timeRegions": [], 1191 | "timeShift": null, 1192 | "title": "NetFlows - DataFlowSets by router and version", 1193 | "tooltip": { 1194 | "shared": false, 1195 | "sort": 0, 1196 | "value_type": "individual" 1197 | }, 1198 | "type": "graph", 1199 | "xaxis": { 1200 | "buckets": null, 1201 | "mode": "time", 1202 | "name": null, 1203 | "show": true, 1204 | "values": [] 1205 | }, 1206 | "yaxes": [ 1207 | { 1208 | "format": "pps", 1209 | "label": null, 1210 | "logBase": 1, 1211 | "max": null, 1212 | "min": null, 1213 | "show": true 1214 | }, 1215 | { 1216 | "format": "short", 1217 | "label": null, 1218 | "logBase": 1, 1219 | "max": null, 1220 | "min": null, 1221 | "show": true 1222 | } 1223 | ], 1224 | "yaxis": { 1225 | "align": false, 1226 | "alignLevel": null 1227 | } 1228 | }, 1229 | { 1230 | "aliasColors": {}, 1231 | "bars": false, 1232 | "dashLength": 10, 1233 | "dashes": false, 1234 | "datasource": "Prometheus", 1235 | "fill": 0, 1236 | "gridPos": { 1237 | "h": 7, 1238 | "w": 24, 1239 | "x": 0, 1240 | "y": 45 1241 | }, 1242 | "id": 14, 1243 | "legend": { 1244 | "alignAsTable": true, 1245 | "avg": true, 1246 | "current": true, 1247 | "max": true, 1248 | "min": true, 1249 | "rightSide": true, 1250 | "show": true, 1251 | "total": false, 1252 | "values": true 1253 | }, 1254 | "lines": true, 1255 | "linewidth": 1, 1256 | "links": [], 1257 | "nullPointMode": "null", 1258 | "percentage": false, 1259 | "pointradius": 5, 1260 | "points": false, 1261 | "renderer": "flot", 1262 | "seriesOverrides": [], 1263 | "spaceLength": 10, 1264 | "stack": false, 1265 | "steppedLine": false, 1266 | "targets": [ 1267 | { 1268 | "expr": "sum(rate(flow_process_nf_delay_summary_seconds[5m])) by (quantile,router,version)", 1269 | "format": "time_series", 1270 | "intervalFactor": 1, 1271 | "legendFormat": "Q{{ quantile }} | {{ router }} | {{ version }}", 1272 | "refId": "A" 1273 | } 1274 | ], 1275 | "thresholds": [], 1276 | "timeFrom": null, 1277 | "timeRegions": [], 1278 | "timeShift": null, 1279 | "title": "Time between flow and processing", 1280 | "tooltip": { 1281 | "shared": false, 1282 | "sort": 0, 1283 | "value_type": "individual" 1284 | }, 1285 | "type": "graph", 1286 | "xaxis": { 1287 | "buckets": null, 1288 | "mode": "time", 1289 | "name": null, 1290 | "show": true, 1291 | "values": [] 1292 | }, 1293 | "yaxes": [ 1294 | { 1295 | "format": "s", 1296 | "label": "", 1297 | "logBase": 1, 1298 | "max": null, 1299 | "min": "0", 1300 | "show": true 1301 | }, 1302 | { 1303 | "format": "short", 1304 | "label": null, 1305 | "logBase": 1, 1306 | "max": null, 1307 | "min": null, 1308 | "show": true 1309 | } 1310 | ], 1311 | "yaxis": { 1312 | "align": false, 1313 | "alignLevel": null 1314 | } 1315 | }, 1316 | { 1317 | "collapsed": false, 1318 | "gridPos": { 1319 | "h": 1, 1320 | "w": 24, 1321 | "x": 0, 1322 | "y": 52 1323 | }, 1324 | "id": 56, 1325 | "panels": [], 1326 | "repeat": null, 1327 | "title": "sFlow metrics", 1328 | "type": "row" 1329 | }, 1330 | { 1331 | "aliasColors": {}, 1332 | "bars": false, 1333 | "dashLength": 10, 1334 | "dashes": false, 1335 | "datasource": "Prometheus", 1336 | "fill": 0, 1337 | "gridPos": { 1338 | "h": 7, 1339 | "w": 12, 1340 | "x": 0, 1341 | "y": 53 1342 | }, 1343 | "id": 5, 1344 | "legend": { 1345 | "alignAsTable": true, 1346 | "avg": true, 1347 | "current": true, 1348 | "max": true, 1349 | "min": true, 1350 | "show": true, 1351 | "total": false, 1352 | "values": true 1353 | }, 1354 | "lines": true, 1355 | "linewidth": 1, 1356 | "links": [], 1357 | "nullPointMode": "null", 1358 | "percentage": false, 1359 | "pointradius": 5, 1360 | "points": false, 1361 | "renderer": "flot", 1362 | "seriesOverrides": [], 1363 | "spaceLength": 10, 1364 | "stack": false, 1365 | "steppedLine": false, 1366 | "targets": [ 1367 | { 1368 | "expr": "sum(rate(flow_process_sf_samples_sum[5m])) by (version,type)", 1369 | "format": "time_series", 1370 | "intervalFactor": 1, 1371 | "legendFormat": "{{ version }} | {{ type }}", 1372 | "refId": "A" 1373 | } 1374 | ], 1375 | "thresholds": [], 1376 | "timeFrom": null, 1377 | "timeRegions": [], 1378 | "timeShift": null, 1379 | "title": "sFlows by type", 1380 | "tooltip": { 1381 | "shared": true, 1382 | "sort": 0, 1383 | "value_type": "individual" 1384 | }, 1385 | "type": "graph", 1386 | "xaxis": { 1387 | "buckets": null, 1388 | "mode": "time", 1389 | "name": null, 1390 | "show": true, 1391 | "values": [] 1392 | }, 1393 | "yaxes": [ 1394 | { 1395 | "format": "pps", 1396 | "label": null, 1397 | "logBase": 1, 1398 | "max": null, 1399 | "min": null, 1400 | "show": true 1401 | }, 1402 | { 1403 | "format": "short", 1404 | "label": null, 1405 | "logBase": 1, 1406 | "max": null, 1407 | "min": null, 1408 | "show": true 1409 | } 1410 | ], 1411 | "yaxis": { 1412 | "align": false, 1413 | "alignLevel": null 1414 | } 1415 | }, 1416 | { 1417 | "aliasColors": {}, 1418 | "bars": false, 1419 | "dashLength": 10, 1420 | "dashes": false, 1421 | "datasource": "Prometheus", 1422 | "fill": 0, 1423 | "gridPos": { 1424 | "h": 7, 1425 | "w": 12, 1426 | "x": 12, 1427 | "y": 53 1428 | }, 1429 | "id": 8, 1430 | "legend": { 1431 | "alignAsTable": true, 1432 | "avg": false, 1433 | "current": true, 1434 | "max": false, 1435 | "min": false, 1436 | "rightSide": true, 1437 | "show": true, 1438 | "sort": "current", 1439 | "sortDesc": true, 1440 | "total": false, 1441 | "values": true 1442 | }, 1443 | "lines": true, 1444 | "linewidth": 1, 1445 | "links": [], 1446 | "nullPointMode": "null", 1447 | "percentage": false, 1448 | "pointradius": 5, 1449 | "points": false, 1450 | "renderer": "flot", 1451 | "seriesOverrides": [], 1452 | "spaceLength": 10, 1453 | "stack": false, 1454 | "steppedLine": false, 1455 | "targets": [ 1456 | { 1457 | "expr": "sum(rate(flow_process_sf_samples_records_sum{type=\"FlowSample\"}[5m])) by (agent,version)", 1458 | "format": "time_series", 1459 | "intervalFactor": 1, 1460 | "legendFormat": "{{ agent }} | {{ version }}", 1461 | "refId": "A" 1462 | } 1463 | ], 1464 | "thresholds": [], 1465 | "timeFrom": null, 1466 | "timeRegions": [], 1467 | "timeShift": null, 1468 | "title": "sFlows Flow records by agent and version", 1469 | "tooltip": { 1470 | "shared": false, 1471 | "sort": 0, 1472 | "value_type": "individual" 1473 | }, 1474 | "type": "graph", 1475 | "xaxis": { 1476 | "buckets": null, 1477 | "mode": "time", 1478 | "name": null, 1479 | "show": true, 1480 | "values": [] 1481 | }, 1482 | "yaxes": [ 1483 | { 1484 | "format": "pps", 1485 | "label": null, 1486 | "logBase": 1, 1487 | "max": null, 1488 | "min": null, 1489 | "show": true 1490 | }, 1491 | { 1492 | "format": "short", 1493 | "label": null, 1494 | "logBase": 1, 1495 | "max": null, 1496 | "min": null, 1497 | "show": true 1498 | } 1499 | ], 1500 | "yaxis": { 1501 | "align": false, 1502 | "alignLevel": null 1503 | } 1504 | }, 1505 | { 1506 | "collapsed": false, 1507 | "gridPos": { 1508 | "h": 1, 1509 | "w": 24, 1510 | "x": 0, 1511 | "y": 60 1512 | }, 1513 | "id": 57, 1514 | "panels": [], 1515 | "repeat": null, 1516 | "title": "NetFlow decoder metrics", 1517 | "type": "row" 1518 | }, 1519 | { 1520 | "aliasColors": {}, 1521 | "bars": false, 1522 | "dashLength": 10, 1523 | "dashes": false, 1524 | "datasource": "Prometheus", 1525 | "fill": 0, 1526 | "gridPos": { 1527 | "h": 7, 1528 | "w": 8, 1529 | "x": 0, 1530 | "y": 61 1531 | }, 1532 | "id": 11, 1533 | "legend": { 1534 | "alignAsTable": true, 1535 | "avg": true, 1536 | "current": true, 1537 | "max": true, 1538 | "min": true, 1539 | "rightSide": false, 1540 | "show": true, 1541 | "sort": "current", 1542 | "sortDesc": false, 1543 | "total": false, 1544 | "values": true 1545 | }, 1546 | "lines": true, 1547 | "linewidth": 1, 1548 | "links": [], 1549 | "nullPointMode": "null", 1550 | "percentage": false, 1551 | "pointradius": 5, 1552 | "points": false, 1553 | "renderer": "flot", 1554 | "seriesOverrides": [], 1555 | "spaceLength": 10, 1556 | "stack": false, 1557 | "steppedLine": false, 1558 | "targets": [ 1559 | { 1560 | "expr": "avg(flow_summary_decoding_time_us{name=\"NetFlow\"}) by (quantile)", 1561 | "format": "time_series", 1562 | "intervalFactor": 1, 1563 | "legendFormat": "Q{{ quantile }}", 1564 | "refId": "A" 1565 | } 1566 | ], 1567 | "thresholds": [], 1568 | "timeFrom": null, 1569 | "timeRegions": [], 1570 | "timeShift": null, 1571 | "title": "NetFlow decoding time quantiles", 1572 | "tooltip": { 1573 | "shared": true, 1574 | "sort": 0, 1575 | "value_type": "individual" 1576 | }, 1577 | "type": "graph", 1578 | "xaxis": { 1579 | "buckets": null, 1580 | "mode": "time", 1581 | "name": null, 1582 | "show": true, 1583 | "values": [] 1584 | }, 1585 | "yaxes": [ 1586 | { 1587 | "format": "µs", 1588 | "label": null, 1589 | "logBase": 1, 1590 | "max": null, 1591 | "min": null, 1592 | "show": true 1593 | }, 1594 | { 1595 | "format": "short", 1596 | "label": null, 1597 | "logBase": 1, 1598 | "max": null, 1599 | "min": null, 1600 | "show": true 1601 | } 1602 | ], 1603 | "yaxis": { 1604 | "align": false, 1605 | "alignLevel": null 1606 | } 1607 | }, 1608 | { 1609 | "aliasColors": {}, 1610 | "bars": false, 1611 | "dashLength": 10, 1612 | "dashes": false, 1613 | "datasource": "Prometheus", 1614 | "fill": 0, 1615 | "gridPos": { 1616 | "h": 7, 1617 | "w": 8, 1618 | "x": 8, 1619 | "y": 61 1620 | }, 1621 | "id": 12, 1622 | "legend": { 1623 | "alignAsTable": true, 1624 | "avg": true, 1625 | "current": true, 1626 | "max": true, 1627 | "min": true, 1628 | "rightSide": false, 1629 | "show": true, 1630 | "sort": "current", 1631 | "sortDesc": false, 1632 | "total": false, 1633 | "values": true 1634 | }, 1635 | "lines": true, 1636 | "linewidth": 1, 1637 | "links": [], 1638 | "nullPointMode": "null", 1639 | "percentage": false, 1640 | "pointradius": 5, 1641 | "points": false, 1642 | "renderer": "flot", 1643 | "seriesOverrides": [], 1644 | "spaceLength": 10, 1645 | "stack": false, 1646 | "steppedLine": false, 1647 | "targets": [ 1648 | { 1649 | "expr": "avg(flow_summary_processing_time_us{name=\"NetFlow\"}) by (quantile)", 1650 | "format": "time_series", 1651 | "intervalFactor": 1, 1652 | "legendFormat": "Q{{ quantile }}", 1653 | "refId": "A" 1654 | } 1655 | ], 1656 | "thresholds": [], 1657 | "timeFrom": null, 1658 | "timeRegions": [], 1659 | "timeShift": null, 1660 | "title": "NetFlow processing time quantiles", 1661 | "tooltip": { 1662 | "shared": true, 1663 | "sort": 0, 1664 | "value_type": "individual" 1665 | }, 1666 | "type": "graph", 1667 | "xaxis": { 1668 | "buckets": null, 1669 | "mode": "time", 1670 | "name": null, 1671 | "show": true, 1672 | "values": [] 1673 | }, 1674 | "yaxes": [ 1675 | { 1676 | "format": "µs", 1677 | "label": null, 1678 | "logBase": 1, 1679 | "max": null, 1680 | "min": null, 1681 | "show": true 1682 | }, 1683 | { 1684 | "format": "short", 1685 | "label": null, 1686 | "logBase": 1, 1687 | "max": null, 1688 | "min": null, 1689 | "show": true 1690 | } 1691 | ], 1692 | "yaxis": { 1693 | "align": false, 1694 | "alignLevel": null 1695 | } 1696 | }, 1697 | { 1698 | "aliasColors": {}, 1699 | "bars": false, 1700 | "dashLength": 10, 1701 | "dashes": false, 1702 | "datasource": "Prometheus", 1703 | "fill": 0, 1704 | "gridPos": { 1705 | "h": 7, 1706 | "w": 8, 1707 | "x": 16, 1708 | "y": 61 1709 | }, 1710 | "id": 13, 1711 | "legend": { 1712 | "alignAsTable": true, 1713 | "avg": true, 1714 | "current": true, 1715 | "max": true, 1716 | "min": true, 1717 | "rightSide": false, 1718 | "show": true, 1719 | "sort": null, 1720 | "sortDesc": null, 1721 | "total": false, 1722 | "values": true 1723 | }, 1724 | "lines": true, 1725 | "linewidth": 1, 1726 | "links": [], 1727 | "nullPointMode": "null", 1728 | "percentage": false, 1729 | "pointradius": 5, 1730 | "points": false, 1731 | "renderer": "flot", 1732 | "seriesOverrides": [], 1733 | "spaceLength": 10, 1734 | "stack": false, 1735 | "steppedLine": false, 1736 | "targets": [ 1737 | { 1738 | "expr": "sum(rate(flow_decoder_count{name=\"NetFlow\"}[5m])) by (worker)", 1739 | "format": "time_series", 1740 | "intervalFactor": 1, 1741 | "legendFormat": "Worker {{ worker }}", 1742 | "refId": "A" 1743 | } 1744 | ], 1745 | "thresholds": [], 1746 | "timeFrom": null, 1747 | "timeRegions": [], 1748 | "timeShift": null, 1749 | "title": "NetFlow worker rate", 1750 | "tooltip": { 1751 | "shared": true, 1752 | "sort": 0, 1753 | "value_type": "individual" 1754 | }, 1755 | "type": "graph", 1756 | "xaxis": { 1757 | "buckets": null, 1758 | "mode": "time", 1759 | "name": null, 1760 | "show": true, 1761 | "values": [] 1762 | }, 1763 | "yaxes": [ 1764 | { 1765 | "format": "pps", 1766 | "label": null, 1767 | "logBase": 1, 1768 | "max": null, 1769 | "min": null, 1770 | "show": true 1771 | }, 1772 | { 1773 | "format": "short", 1774 | "label": null, 1775 | "logBase": 1, 1776 | "max": null, 1777 | "min": null, 1778 | "show": true 1779 | } 1780 | ], 1781 | "yaxis": { 1782 | "align": false, 1783 | "alignLevel": null 1784 | } 1785 | }, 1786 | { 1787 | "collapsed": false, 1788 | "gridPos": { 1789 | "h": 1, 1790 | "w": 24, 1791 | "x": 0, 1792 | "y": 68 1793 | }, 1794 | "id": 58, 1795 | "panels": [], 1796 | "repeat": null, 1797 | "title": "sFlow decoder metrics", 1798 | "type": "row" 1799 | }, 1800 | { 1801 | "aliasColors": {}, 1802 | "bars": false, 1803 | "dashLength": 10, 1804 | "dashes": false, 1805 | "datasource": "Prometheus", 1806 | "fill": 0, 1807 | "gridPos": { 1808 | "h": 7, 1809 | "w": 8, 1810 | "x": 0, 1811 | "y": 69 1812 | }, 1813 | "id": 15, 1814 | "legend": { 1815 | "alignAsTable": true, 1816 | "avg": true, 1817 | "current": true, 1818 | "max": true, 1819 | "min": true, 1820 | "show": true, 1821 | "sort": "current", 1822 | "sortDesc": false, 1823 | "total": false, 1824 | "values": true 1825 | }, 1826 | "lines": true, 1827 | "linewidth": 1, 1828 | "links": [], 1829 | "nullPointMode": "null", 1830 | "percentage": false, 1831 | "pointradius": 5, 1832 | "points": false, 1833 | "renderer": "flot", 1834 | "seriesOverrides": [], 1835 | "spaceLength": 10, 1836 | "stack": false, 1837 | "steppedLine": false, 1838 | "targets": [ 1839 | { 1840 | "expr": "avg(flow_summary_decoding_time_us{name=\"sFlow\"}) by (quantile)", 1841 | "format": "time_series", 1842 | "intervalFactor": 1, 1843 | "legendFormat": "Q{{ quantile }}", 1844 | "refId": "A" 1845 | } 1846 | ], 1847 | "thresholds": [], 1848 | "timeFrom": null, 1849 | "timeRegions": [], 1850 | "timeShift": null, 1851 | "title": "sFlow decoding time quantiles", 1852 | "tooltip": { 1853 | "shared": true, 1854 | "sort": 0, 1855 | "value_type": "individual" 1856 | }, 1857 | "type": "graph", 1858 | "xaxis": { 1859 | "buckets": null, 1860 | "mode": "time", 1861 | "name": null, 1862 | "show": true, 1863 | "values": [] 1864 | }, 1865 | "yaxes": [ 1866 | { 1867 | "format": "µs", 1868 | "label": null, 1869 | "logBase": 1, 1870 | "max": null, 1871 | "min": null, 1872 | "show": true 1873 | }, 1874 | { 1875 | "format": "short", 1876 | "label": null, 1877 | "logBase": 1, 1878 | "max": null, 1879 | "min": null, 1880 | "show": true 1881 | } 1882 | ], 1883 | "yaxis": { 1884 | "align": false, 1885 | "alignLevel": null 1886 | } 1887 | }, 1888 | { 1889 | "aliasColors": {}, 1890 | "bars": false, 1891 | "dashLength": 10, 1892 | "dashes": false, 1893 | "datasource": "Prometheus", 1894 | "fill": 0, 1895 | "gridPos": { 1896 | "h": 7, 1897 | "w": 8, 1898 | "x": 8, 1899 | "y": 69 1900 | }, 1901 | "id": 16, 1902 | "legend": { 1903 | "alignAsTable": true, 1904 | "avg": true, 1905 | "current": true, 1906 | "max": true, 1907 | "min": true, 1908 | "show": true, 1909 | "sort": "current", 1910 | "sortDesc": false, 1911 | "total": false, 1912 | "values": true 1913 | }, 1914 | "lines": true, 1915 | "linewidth": 1, 1916 | "links": [], 1917 | "nullPointMode": "null", 1918 | "percentage": false, 1919 | "pointradius": 5, 1920 | "points": false, 1921 | "renderer": "flot", 1922 | "seriesOverrides": [], 1923 | "spaceLength": 10, 1924 | "stack": false, 1925 | "steppedLine": false, 1926 | "targets": [ 1927 | { 1928 | "expr": "avg(flow_summary_decoding_time_us{name=\"sFlow\"}) by (quantile)", 1929 | "format": "time_series", 1930 | "intervalFactor": 1, 1931 | "legendFormat": "Q{{ quantile }}", 1932 | "refId": "A" 1933 | } 1934 | ], 1935 | "thresholds": [], 1936 | "timeFrom": null, 1937 | "timeRegions": [], 1938 | "timeShift": null, 1939 | "title": "sFlow processing time quantiles", 1940 | "tooltip": { 1941 | "shared": true, 1942 | "sort": 0, 1943 | "value_type": "individual" 1944 | }, 1945 | "type": "graph", 1946 | "xaxis": { 1947 | "buckets": null, 1948 | "mode": "time", 1949 | "name": null, 1950 | "show": true, 1951 | "values": [] 1952 | }, 1953 | "yaxes": [ 1954 | { 1955 | "format": "µs", 1956 | "label": null, 1957 | "logBase": 1, 1958 | "max": null, 1959 | "min": null, 1960 | "show": true 1961 | }, 1962 | { 1963 | "format": "short", 1964 | "label": null, 1965 | "logBase": 1, 1966 | "max": null, 1967 | "min": null, 1968 | "show": true 1969 | } 1970 | ], 1971 | "yaxis": { 1972 | "align": false, 1973 | "alignLevel": null 1974 | } 1975 | }, 1976 | { 1977 | "aliasColors": {}, 1978 | "bars": false, 1979 | "dashLength": 10, 1980 | "dashes": false, 1981 | "datasource": "Prometheus", 1982 | "fill": 0, 1983 | "gridPos": { 1984 | "h": 7, 1985 | "w": 8, 1986 | "x": 16, 1987 | "y": 69 1988 | }, 1989 | "id": 17, 1990 | "legend": { 1991 | "alignAsTable": true, 1992 | "avg": true, 1993 | "current": true, 1994 | "max": true, 1995 | "min": true, 1996 | "rightSide": false, 1997 | "show": true, 1998 | "sort": null, 1999 | "sortDesc": null, 2000 | "total": false, 2001 | "values": true 2002 | }, 2003 | "lines": true, 2004 | "linewidth": 1, 2005 | "links": [], 2006 | "nullPointMode": "null", 2007 | "percentage": false, 2008 | "pointradius": 5, 2009 | "points": false, 2010 | "renderer": "flot", 2011 | "seriesOverrides": [], 2012 | "spaceLength": 10, 2013 | "stack": false, 2014 | "steppedLine": false, 2015 | "targets": [ 2016 | { 2017 | "expr": "sum(rate(flow_decoder_count{name=\"sFlow\"}[5m])) by (worker)", 2018 | "format": "time_series", 2019 | "intervalFactor": 1, 2020 | "legendFormat": "Worker {{ worker }}", 2021 | "refId": "A" 2022 | } 2023 | ], 2024 | "thresholds": [], 2025 | "timeFrom": null, 2026 | "timeRegions": [], 2027 | "timeShift": null, 2028 | "title": "sFlow worker rate", 2029 | "tooltip": { 2030 | "shared": true, 2031 | "sort": 0, 2032 | "value_type": "individual" 2033 | }, 2034 | "type": "graph", 2035 | "xaxis": { 2036 | "buckets": null, 2037 | "mode": "time", 2038 | "name": null, 2039 | "show": true, 2040 | "values": [] 2041 | }, 2042 | "yaxes": [ 2043 | { 2044 | "format": "pps", 2045 | "label": null, 2046 | "logBase": 1, 2047 | "max": null, 2048 | "min": null, 2049 | "show": true 2050 | }, 2051 | { 2052 | "format": "short", 2053 | "label": null, 2054 | "logBase": 1, 2055 | "max": null, 2056 | "min": null, 2057 | "show": true 2058 | } 2059 | ], 2060 | "yaxis": { 2061 | "align": false, 2062 | "alignLevel": null 2063 | } 2064 | } 2065 | ], 2066 | "refresh": false, 2067 | "schemaVersion": 16, 2068 | "style": "dark", 2069 | "tags": [], 2070 | "templating": { 2071 | "list": [] 2072 | }, 2073 | "time": { 2074 | "from": "now-24h", 2075 | "to": "now" 2076 | }, 2077 | "timepicker": { 2078 | "refresh_intervals": [ 2079 | "5s", 2080 | "10s", 2081 | "30s", 2082 | "1m", 2083 | "5m", 2084 | "15m", 2085 | "30m", 2086 | "1h", 2087 | "2h", 2088 | "1d" 2089 | ], 2090 | "time_options": [ 2091 | "5m", 2092 | "15m", 2093 | "1h", 2094 | "6h", 2095 | "12h", 2096 | "24h", 2097 | "2d", 2098 | "7d", 2099 | "30d" 2100 | ] 2101 | }, 2102 | "timezone": "utc", 2103 | "title": "GoFlow - Internals", 2104 | "uid": "4U5xQZPmz", 2105 | "version": 2 2106 | } -------------------------------------------------------------------------------- /compose/grafana/dashboards/viz-ch.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "$$hashKey": "object:631", 6 | "builtIn": 1, 7 | "datasource": "-- Grafana --", 8 | "enable": true, 9 | "hide": true, 10 | "iconColor": "rgba(0, 211, 255, 1)", 11 | "name": "Annotations & Alerts", 12 | "type": "dashboard" 13 | } 14 | ] 15 | }, 16 | "editable": true, 17 | "gnetId": null, 18 | "graphTooltip": 0, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "ClickHouse", 27 | "fill": 1, 28 | "fillGradient": 0, 29 | "gridPos": { 30 | "h": 9, 31 | "w": 24, 32 | "x": 0, 33 | "y": 0 34 | }, 35 | "hiddenSeries": false, 36 | "id": 2, 37 | "legend": { 38 | "avg": false, 39 | "current": false, 40 | "max": false, 41 | "min": false, 42 | "show": true, 43 | "total": false, 44 | "values": false 45 | }, 46 | "lines": true, 47 | "linewidth": 1, 48 | "links": [], 49 | "nullPointMode": "null", 50 | "options": { 51 | "dataLinks": [] 52 | }, 53 | "percentage": false, 54 | "pointradius": 5, 55 | "points": false, 56 | "renderer": "flot", 57 | "seriesOverrides": [], 58 | "spaceLength": 10, 59 | "stack": false, 60 | "steppedLine": false, 61 | "targets": [ 62 | { 63 | "database": "default", 64 | "dateColDataType": "Date", 65 | "dateLoading": false, 66 | "dateTimeColDataType": "TimeFlowStart", 67 | "dateTimeType": "DATETIME", 68 | "datetimeLoading": false, 69 | "format": "time_series", 70 | "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", 71 | "group": [], 72 | "intervalFactor": 1, 73 | "metricColumn": "none", 74 | "query": "SELECT\n toUInt64(toStartOfMinute($dateTimeCol))*1000 as t,\n sum(Bytes*SamplingRate) as sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY t\nORDER BY t", 75 | "rawQuery": "SELECT toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t, sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1585445405) AND TimeFlowStart >= toDateTime(1585445405) GROUP BY t ORDER BY t", 76 | "rawSql": "SELECT\n (cast(extract(epoch from time_flow) as integer)/30)*30 AS \"time\",\n sum(bytes*sampling_rate*8)/30\nFROM flows\nWHERE\n $__timeFilter(date_inserted)\nGROUP BY \"time\"\nORDER BY \"time\"", 77 | "refId": "A", 78 | "round": "0s", 79 | "select": [ 80 | [ 81 | { 82 | "params": [ 83 | "bytes" 84 | ], 85 | "type": "column" 86 | } 87 | ] 88 | ], 89 | "table": "flows_raw", 90 | "tableLoading": false, 91 | "timeColumn": "date_inserted", 92 | "timeColumnType": "timestamp", 93 | "where": [ 94 | { 95 | "name": "$__timeFilter", 96 | "params": [], 97 | "type": "macro" 98 | } 99 | ] 100 | } 101 | ], 102 | "thresholds": [], 103 | "timeFrom": null, 104 | "timeRegions": [], 105 | "timeShift": null, 106 | "title": "Instant traffic", 107 | "tooltip": { 108 | "shared": true, 109 | "sort": 0, 110 | "value_type": "individual" 111 | }, 112 | "type": "graph", 113 | "xaxis": { 114 | "buckets": null, 115 | "mode": "time", 116 | "name": null, 117 | "show": true, 118 | "values": [] 119 | }, 120 | "yaxes": [ 121 | { 122 | "format": "bps", 123 | "label": null, 124 | "logBase": 1, 125 | "max": null, 126 | "min": null, 127 | "show": true 128 | }, 129 | { 130 | "format": "short", 131 | "label": null, 132 | "logBase": 1, 133 | "max": null, 134 | "min": null, 135 | "show": true 136 | } 137 | ], 138 | "yaxis": { 139 | "align": false, 140 | "alignLevel": null 141 | } 142 | }, 143 | { 144 | "columns": [], 145 | "datasource": "ClickHouse", 146 | "fontSize": "100%", 147 | "gridPos": { 148 | "h": 9, 149 | "w": 12, 150 | "x": 0, 151 | "y": 9 152 | }, 153 | "id": 7, 154 | "links": [], 155 | "pageSize": null, 156 | "scroll": true, 157 | "showHeader": true, 158 | "sort": { 159 | "col": 1, 160 | "desc": true 161 | }, 162 | "styles": [ 163 | { 164 | "alias": "Time", 165 | "align": "auto", 166 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 167 | "pattern": "Time", 168 | "type": "date" 169 | }, 170 | { 171 | "alias": "", 172 | "align": "auto", 173 | "colorMode": null, 174 | "colors": [ 175 | "rgba(245, 54, 54, 0.9)", 176 | "rgba(237, 129, 40, 0.89)", 177 | "rgba(50, 172, 45, 0.97)" 178 | ], 179 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 180 | "decimals": 0, 181 | "mappingType": 1, 182 | "pattern": ".*_port", 183 | "thresholds": [], 184 | "type": "number", 185 | "unit": "none" 186 | }, 187 | { 188 | "alias": "", 189 | "align": "auto", 190 | "colorMode": null, 191 | "colors": [ 192 | "rgba(245, 54, 54, 0.9)", 193 | "rgba(237, 129, 40, 0.89)", 194 | "rgba(50, 172, 45, 0.97)" 195 | ], 196 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 197 | "decimals": 2, 198 | "mappingType": 1, 199 | "pattern": "sumbytes", 200 | "thresholds": [], 201 | "type": "number", 202 | "unit": "decbytes" 203 | }, 204 | { 205 | "alias": "", 206 | "align": "auto", 207 | "colorMode": null, 208 | "colors": [ 209 | "rgba(245, 54, 54, 0.9)", 210 | "rgba(237, 129, 40, 0.89)", 211 | "rgba(50, 172, 45, 0.97)" 212 | ], 213 | "decimals": 0, 214 | "pattern": "/.*/", 215 | "thresholds": [], 216 | "type": "number", 217 | "unit": "short" 218 | } 219 | ], 220 | "targets": [ 221 | { 222 | "database": "default", 223 | "dateColDataType": "", 224 | "dateLoading": false, 225 | "dateTimeColDataType": "TimeFlowStart", 226 | "dateTimeType": "DATETIME", 227 | "datetimeLoading": false, 228 | "format": "table", 229 | "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", 230 | "group": [], 231 | "intervalFactor": 1, 232 | "metricColumn": "none", 233 | "query": "SELECT\n if(EType = 0x800, IPv4NumToString(reinterpretAsUInt32(substring(reverse(SrcAddr), 13,4))), IPv6NumToString(SrcAddr)) as srcip,\n sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY srcip\nORDER BY sumbytes DESC", 234 | "rawQuery": "SELECT toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t, sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1593315015) AND TimeFlowStart >= toDateTime(1593315015) GROUP BY t ORDER BY t", 235 | "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip", 236 | "refId": "A", 237 | "round": "0s", 238 | "select": [ 239 | [ 240 | { 241 | "params": [ 242 | "value" 243 | ], 244 | "type": "column" 245 | } 246 | ] 247 | ], 248 | "table": "flows_raw", 249 | "tableLoading": false, 250 | "timeColumn": "time", 251 | "where": [ 252 | { 253 | "name": "$__timeFilter", 254 | "params": [], 255 | "type": "macro" 256 | } 257 | ] 258 | } 259 | ], 260 | "title": "Top source IPs", 261 | "transform": "table", 262 | "type": "table" 263 | }, 264 | { 265 | "columns": [], 266 | "datasource": "ClickHouse", 267 | "fontSize": "100%", 268 | "gridPos": { 269 | "h": 9, 270 | "w": 12, 271 | "x": 12, 272 | "y": 9 273 | }, 274 | "id": 9, 275 | "links": [], 276 | "pageSize": null, 277 | "scroll": true, 278 | "showHeader": true, 279 | "sort": { 280 | "col": 2, 281 | "desc": true 282 | }, 283 | "styles": [ 284 | { 285 | "$$hashKey": "object:1506", 286 | "alias": "Time", 287 | "align": "auto", 288 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 289 | "pattern": "Time", 290 | "type": "date" 291 | }, 292 | { 293 | "$$hashKey": "object:1507", 294 | "alias": "", 295 | "align": "auto", 296 | "colorMode": null, 297 | "colors": [ 298 | "rgba(245, 54, 54, 0.9)", 299 | "rgba(237, 129, 40, 0.89)", 300 | "rgba(50, 172, 45, 0.97)" 301 | ], 302 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 303 | "decimals": 0, 304 | "mappingType": 1, 305 | "pattern": "port", 306 | "thresholds": [], 307 | "type": "number", 308 | "unit": "none" 309 | }, 310 | { 311 | "$$hashKey": "object:1508", 312 | "alias": "", 313 | "align": "auto", 314 | "colorMode": null, 315 | "colors": [ 316 | "rgba(245, 54, 54, 0.9)", 317 | "rgba(237, 129, 40, 0.89)", 318 | "rgba(50, 172, 45, 0.97)" 319 | ], 320 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 321 | "decimals": 2, 322 | "mappingType": 1, 323 | "pattern": "sumbytes", 324 | "thresholds": [], 325 | "type": "number", 326 | "unit": "decbytes" 327 | }, 328 | { 329 | "$$hashKey": "object:1509", 330 | "alias": "", 331 | "align": "auto", 332 | "colorMode": null, 333 | "colors": [ 334 | "rgba(245, 54, 54, 0.9)", 335 | "rgba(237, 129, 40, 0.89)", 336 | "rgba(50, 172, 45, 0.97)" 337 | ], 338 | "decimals": 0, 339 | "pattern": "/.*/", 340 | "thresholds": [], 341 | "type": "number", 342 | "unit": "short" 343 | } 344 | ], 345 | "targets": [ 346 | { 347 | "database": "default", 348 | "dateColDataType": "", 349 | "dateLoading": false, 350 | "dateTimeColDataType": "TimeFlowStart", 351 | "dateTimeType": "DATETIME", 352 | "datetimeLoading": false, 353 | "format": "table", 354 | "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", 355 | "group": [], 356 | "intervalFactor": 1, 357 | "metricColumn": "none", 358 | "query": "SELECT\n SrcPort as port,\n sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY port\nORDER BY sumbytes DESC", 359 | "rawQuery": "SELECT toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t, sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1585442922) AND TimeFlowStart >= toDateTime(1585442922) GROUP BY t ORDER BY t", 360 | "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip", 361 | "refId": "A", 362 | "round": "0s", 363 | "select": [ 364 | [ 365 | { 366 | "params": [ 367 | "value" 368 | ], 369 | "type": "column" 370 | } 371 | ] 372 | ], 373 | "table": "flows_raw", 374 | "tableLoading": false, 375 | "timeColumn": "time", 376 | "where": [ 377 | { 378 | "name": "$__timeFilter", 379 | "params": [], 380 | "type": "macro" 381 | } 382 | ] 383 | } 384 | ], 385 | "title": "Top source ports", 386 | "transform": "table", 387 | "type": "table" 388 | }, 389 | { 390 | "columns": [], 391 | "datasource": "ClickHouse", 392 | "fontSize": "100%", 393 | "gridPos": { 394 | "h": 9, 395 | "w": 12, 396 | "x": 0, 397 | "y": 18 398 | }, 399 | "id": 10, 400 | "links": [], 401 | "pageSize": null, 402 | "scroll": true, 403 | "showHeader": true, 404 | "sort": { 405 | "col": 1, 406 | "desc": true 407 | }, 408 | "styles": [ 409 | { 410 | "alias": "Time", 411 | "align": "auto", 412 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 413 | "pattern": "Time", 414 | "type": "date" 415 | }, 416 | { 417 | "alias": "", 418 | "align": "auto", 419 | "colorMode": null, 420 | "colors": [ 421 | "rgba(245, 54, 54, 0.9)", 422 | "rgba(237, 129, 40, 0.89)", 423 | "rgba(50, 172, 45, 0.97)" 424 | ], 425 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 426 | "decimals": 0, 427 | "mappingType": 1, 428 | "pattern": ".*_port", 429 | "thresholds": [], 430 | "type": "number", 431 | "unit": "none" 432 | }, 433 | { 434 | "alias": "", 435 | "align": "auto", 436 | "colorMode": null, 437 | "colors": [ 438 | "rgba(245, 54, 54, 0.9)", 439 | "rgba(237, 129, 40, 0.89)", 440 | "rgba(50, 172, 45, 0.97)" 441 | ], 442 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 443 | "decimals": 2, 444 | "mappingType": 1, 445 | "pattern": "sumbytes", 446 | "thresholds": [], 447 | "type": "number", 448 | "unit": "decbytes" 449 | }, 450 | { 451 | "alias": "", 452 | "align": "auto", 453 | "colorMode": null, 454 | "colors": [ 455 | "rgba(245, 54, 54, 0.9)", 456 | "rgba(237, 129, 40, 0.89)", 457 | "rgba(50, 172, 45, 0.97)" 458 | ], 459 | "decimals": 0, 460 | "pattern": "/.*/", 461 | "thresholds": [], 462 | "type": "number", 463 | "unit": "short" 464 | } 465 | ], 466 | "targets": [ 467 | { 468 | "database": "default", 469 | "dateColDataType": "", 470 | "dateLoading": false, 471 | "dateTimeColDataType": "TimeFlowStart", 472 | "dateTimeType": "DATETIME", 473 | "datetimeLoading": false, 474 | "format": "table", 475 | "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", 476 | "group": [], 477 | "intervalFactor": 1, 478 | "metricColumn": "none", 479 | "query": "SELECT\n if(EType = 0x800, IPv4NumToString(reinterpretAsUInt32(substring(reverse(DstAddr), 13,4))), IPv6NumToString(DstAddr)) as dstip,\n sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY dstip\nORDER BY sumbytes DESC", 480 | "rawQuery": "SELECT toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t, sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1593317660) AND TimeFlowStart >= toDateTime(1593317660) GROUP BY t ORDER BY t", 481 | "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip", 482 | "refId": "A", 483 | "round": "0s", 484 | "select": [ 485 | [ 486 | { 487 | "params": [ 488 | "value" 489 | ], 490 | "type": "column" 491 | } 492 | ] 493 | ], 494 | "table": "flows_raw", 495 | "tableLoading": false, 496 | "timeColumn": "time", 497 | "where": [ 498 | { 499 | "name": "$__timeFilter", 500 | "params": [], 501 | "type": "macro" 502 | } 503 | ] 504 | } 505 | ], 506 | "title": "Top destination IPs", 507 | "transform": "table", 508 | "type": "table" 509 | }, 510 | { 511 | "columns": [], 512 | "datasource": "ClickHouse", 513 | "fontSize": "100%", 514 | "gridPos": { 515 | "h": 9, 516 | "w": 12, 517 | "x": 12, 518 | "y": 18 519 | }, 520 | "id": 11, 521 | "links": [], 522 | "pageSize": null, 523 | "scroll": true, 524 | "showHeader": true, 525 | "sort": { 526 | "col": 2, 527 | "desc": true 528 | }, 529 | "styles": [ 530 | { 531 | "$$hashKey": "object:1428", 532 | "alias": "Time", 533 | "align": "auto", 534 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 535 | "pattern": "Time", 536 | "type": "date" 537 | }, 538 | { 539 | "$$hashKey": "object:1429", 540 | "alias": "", 541 | "align": "auto", 542 | "colorMode": null, 543 | "colors": [ 544 | "rgba(245, 54, 54, 0.9)", 545 | "rgba(237, 129, 40, 0.89)", 546 | "rgba(50, 172, 45, 0.97)" 547 | ], 548 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 549 | "decimals": 0, 550 | "mappingType": 1, 551 | "pattern": "port", 552 | "thresholds": [], 553 | "type": "number", 554 | "unit": "none" 555 | }, 556 | { 557 | "$$hashKey": "object:1430", 558 | "alias": "", 559 | "align": "auto", 560 | "colorMode": null, 561 | "colors": [ 562 | "rgba(245, 54, 54, 0.9)", 563 | "rgba(237, 129, 40, 0.89)", 564 | "rgba(50, 172, 45, 0.97)" 565 | ], 566 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 567 | "decimals": 2, 568 | "mappingType": 1, 569 | "pattern": "sumbytes", 570 | "thresholds": [], 571 | "type": "number", 572 | "unit": "decbytes" 573 | }, 574 | { 575 | "$$hashKey": "object:1431", 576 | "alias": "", 577 | "align": "auto", 578 | "colorMode": null, 579 | "colors": [ 580 | "rgba(245, 54, 54, 0.9)", 581 | "rgba(237, 129, 40, 0.89)", 582 | "rgba(50, 172, 45, 0.97)" 583 | ], 584 | "decimals": 0, 585 | "pattern": "/.*/", 586 | "thresholds": [], 587 | "type": "number", 588 | "unit": "short" 589 | } 590 | ], 591 | "targets": [ 592 | { 593 | "database": "default", 594 | "dateColDataType": "", 595 | "dateLoading": false, 596 | "dateTimeColDataType": "TimeFlowStart", 597 | "dateTimeType": "DATETIME", 598 | "datetimeLoading": false, 599 | "format": "table", 600 | "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", 601 | "group": [], 602 | "intervalFactor": 1, 603 | "metricColumn": "none", 604 | "query": "SELECT\n DstPort as port,\n sum(Bytes*SamplingRate) AS sumbytes\nFROM $table\nWHERE $timeFilter\nGROUP BY port\nORDER BY sumbytes DESC", 605 | "rawQuery": "SELECT toUInt64(toStartOfMinute(TimeFlowStart))*1000 as t, sum(Bytes*SamplingRate) as sumbytes FROM default.flows_raw WHERE Date >= toDate(1585442721) AND TimeFlowStart >= toDateTime(1585442721) GROUP BY t ORDER BY t", 606 | "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip", 607 | "refId": "A", 608 | "round": "0s", 609 | "select": [ 610 | [ 611 | { 612 | "params": [ 613 | "value" 614 | ], 615 | "type": "column" 616 | } 617 | ] 618 | ], 619 | "table": "flows_raw", 620 | "tableLoading": false, 621 | "timeColumn": "time", 622 | "where": [ 623 | { 624 | "name": "$__timeFilter", 625 | "params": [], 626 | "type": "macro" 627 | } 628 | ] 629 | } 630 | ], 631 | "title": "Top destination ports", 632 | "transform": "table", 633 | "type": "table" 634 | } 635 | ], 636 | "refresh": false, 637 | "schemaVersion": 22, 638 | "style": "dark", 639 | "tags": [], 640 | "templating": { 641 | "list": [] 642 | }, 643 | "time": { 644 | "from": "now-1h", 645 | "to": "now" 646 | }, 647 | "timepicker": { 648 | "refresh_intervals": [ 649 | "5s", 650 | "10s", 651 | "30s", 652 | "1m", 653 | "5m", 654 | "15m", 655 | "30m", 656 | "1h", 657 | "2h", 658 | "1d" 659 | ], 660 | "time_options": [ 661 | "5m", 662 | "15m", 663 | "1h", 664 | "6h", 665 | "12h", 666 | "24h", 667 | "2d", 668 | "7d", 669 | "30d" 670 | ] 671 | }, 672 | "timezone": "", 673 | "title": "Traffic (ClickHouse)", 674 | "uid": "tkNEAd9Zk", 675 | "variables": { 676 | "list": [] 677 | }, 678 | "version": 1 679 | } -------------------------------------------------------------------------------- /compose/grafana/dashboards/viz.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 3, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "PostgreSQL", 27 | "fill": 1, 28 | "gridPos": { 29 | "h": 9, 30 | "w": 24, 31 | "x": 0, 32 | "y": 0 33 | }, 34 | "id": 2, 35 | "legend": { 36 | "avg": false, 37 | "current": false, 38 | "max": false, 39 | "min": false, 40 | "show": true, 41 | "total": false, 42 | "values": false 43 | }, 44 | "lines": true, 45 | "linewidth": 1, 46 | "links": [], 47 | "nullPointMode": "null", 48 | "percentage": false, 49 | "pointradius": 5, 50 | "points": false, 51 | "renderer": "flot", 52 | "seriesOverrides": [], 53 | "spaceLength": 10, 54 | "stack": false, 55 | "steppedLine": false, 56 | "targets": [ 57 | { 58 | "format": "time_series", 59 | "group": [], 60 | "metricColumn": "none", 61 | "rawQuery": true, 62 | "rawSql": "SELECT\n (cast(extract(epoch from time_flow) as integer)/30)*30 AS \"time\",\n sum(bytes*sampling_rate*8)/30\nFROM flows\nWHERE\n $__timeFilter(date_inserted)\nGROUP BY \"time\"\nORDER BY \"time\"", 63 | "refId": "A", 64 | "select": [ 65 | [ 66 | { 67 | "params": [ 68 | "bytes" 69 | ], 70 | "type": "column" 71 | } 72 | ] 73 | ], 74 | "table": "flows", 75 | "timeColumn": "date_inserted", 76 | "timeColumnType": "timestamp", 77 | "where": [ 78 | { 79 | "name": "$__timeFilter", 80 | "params": [], 81 | "type": "macro" 82 | } 83 | ] 84 | } 85 | ], 86 | "thresholds": [], 87 | "timeFrom": null, 88 | "timeRegions": [], 89 | "timeShift": null, 90 | "title": "Instant traffic", 91 | "tooltip": { 92 | "shared": true, 93 | "sort": 0, 94 | "value_type": "individual" 95 | }, 96 | "type": "graph", 97 | "xaxis": { 98 | "buckets": null, 99 | "mode": "time", 100 | "name": null, 101 | "show": true, 102 | "values": [] 103 | }, 104 | "yaxes": [ 105 | { 106 | "format": "bps", 107 | "label": null, 108 | "logBase": 1, 109 | "max": null, 110 | "min": null, 111 | "show": true 112 | }, 113 | { 114 | "format": "short", 115 | "label": null, 116 | "logBase": 1, 117 | "max": null, 118 | "min": null, 119 | "show": true 120 | } 121 | ], 122 | "yaxis": { 123 | "align": false, 124 | "alignLevel": null 125 | } 126 | }, 127 | { 128 | "columns": [], 129 | "datasource": "PostgreSQL", 130 | "fontSize": "100%", 131 | "gridPos": { 132 | "h": 9, 133 | "w": 12, 134 | "x": 0, 135 | "y": 9 136 | }, 137 | "id": 7, 138 | "links": [], 139 | "pageSize": null, 140 | "scroll": true, 141 | "showHeader": true, 142 | "sort": { 143 | "col": 2, 144 | "desc": true 145 | }, 146 | "styles": [ 147 | { 148 | "alias": "Time", 149 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 150 | "pattern": "Time", 151 | "type": "date" 152 | }, 153 | { 154 | "alias": "", 155 | "colorMode": null, 156 | "colors": [ 157 | "rgba(245, 54, 54, 0.9)", 158 | "rgba(237, 129, 40, 0.89)", 159 | "rgba(50, 172, 45, 0.97)" 160 | ], 161 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 162 | "decimals": 0, 163 | "mappingType": 1, 164 | "pattern": ".*_port", 165 | "thresholds": [], 166 | "type": "number", 167 | "unit": "none" 168 | }, 169 | { 170 | "alias": "", 171 | "colorMode": null, 172 | "colors": [ 173 | "rgba(245, 54, 54, 0.9)", 174 | "rgba(237, 129, 40, 0.89)", 175 | "rgba(50, 172, 45, 0.97)" 176 | ], 177 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 178 | "decimals": 2, 179 | "mappingType": 1, 180 | "pattern": "sumbytes", 181 | "thresholds": [], 182 | "type": "number", 183 | "unit": "decbytes" 184 | }, 185 | { 186 | "alias": "", 187 | "colorMode": null, 188 | "colors": [ 189 | "rgba(245, 54, 54, 0.9)", 190 | "rgba(237, 129, 40, 0.89)", 191 | "rgba(50, 172, 45, 0.97)" 192 | ], 193 | "decimals": 0, 194 | "pattern": "/.*/", 195 | "thresholds": [], 196 | "type": "number", 197 | "unit": "short" 198 | } 199 | ], 200 | "targets": [ 201 | { 202 | "format": "table", 203 | "group": [], 204 | "metricColumn": "none", 205 | "rawQuery": true, 206 | "rawSql": "SELECT src_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_ip", 207 | "refId": "A", 208 | "select": [ 209 | [ 210 | { 211 | "params": [ 212 | "value" 213 | ], 214 | "type": "column" 215 | } 216 | ] 217 | ], 218 | "timeColumn": "time", 219 | "where": [ 220 | { 221 | "name": "$__timeFilter", 222 | "params": [], 223 | "type": "macro" 224 | } 225 | ] 226 | } 227 | ], 228 | "title": "Top source IPs", 229 | "transform": "table", 230 | "type": "table" 231 | }, 232 | { 233 | "columns": [], 234 | "datasource": "PostgreSQL", 235 | "fontSize": "100%", 236 | "gridPos": { 237 | "h": 9, 238 | "w": 12, 239 | "x": 12, 240 | "y": 9 241 | }, 242 | "id": 5, 243 | "links": [], 244 | "pageSize": null, 245 | "scroll": true, 246 | "showHeader": true, 247 | "sort": { 248 | "col": 2, 249 | "desc": true 250 | }, 251 | "styles": [ 252 | { 253 | "alias": "Time", 254 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 255 | "pattern": "Time", 256 | "type": "date" 257 | }, 258 | { 259 | "alias": "", 260 | "colorMode": null, 261 | "colors": [ 262 | "rgba(245, 54, 54, 0.9)", 263 | "rgba(237, 129, 40, 0.89)", 264 | "rgba(50, 172, 45, 0.97)" 265 | ], 266 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 267 | "decimals": 0, 268 | "mappingType": 1, 269 | "pattern": ".*_port", 270 | "thresholds": [], 271 | "type": "number", 272 | "unit": "none" 273 | }, 274 | { 275 | "alias": "", 276 | "colorMode": null, 277 | "colors": [ 278 | "rgba(245, 54, 54, 0.9)", 279 | "rgba(237, 129, 40, 0.89)", 280 | "rgba(50, 172, 45, 0.97)" 281 | ], 282 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 283 | "decimals": 2, 284 | "mappingType": 1, 285 | "pattern": "sumbytes", 286 | "thresholds": [], 287 | "type": "number", 288 | "unit": "decbytes" 289 | }, 290 | { 291 | "alias": "", 292 | "colorMode": null, 293 | "colors": [ 294 | "rgba(245, 54, 54, 0.9)", 295 | "rgba(237, 129, 40, 0.89)", 296 | "rgba(50, 172, 45, 0.97)" 297 | ], 298 | "decimals": 0, 299 | "pattern": "/.*/", 300 | "thresholds": [], 301 | "type": "number", 302 | "unit": "short" 303 | } 304 | ], 305 | "targets": [ 306 | { 307 | "format": "table", 308 | "group": [], 309 | "metricColumn": "none", 310 | "rawQuery": true, 311 | "rawSql": "SELECT src_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY src_port", 312 | "refId": "A", 313 | "select": [ 314 | [ 315 | { 316 | "params": [ 317 | "value" 318 | ], 319 | "type": "column" 320 | } 321 | ] 322 | ], 323 | "timeColumn": "time", 324 | "where": [ 325 | { 326 | "name": "$__timeFilter", 327 | "params": [], 328 | "type": "macro" 329 | } 330 | ] 331 | } 332 | ], 333 | "title": "Top source ports", 334 | "transform": "table", 335 | "type": "table" 336 | }, 337 | { 338 | "columns": [], 339 | "datasource": "PostgreSQL", 340 | "fontSize": "100%", 341 | "gridPos": { 342 | "h": 9, 343 | "w": 12, 344 | "x": 0, 345 | "y": 18 346 | }, 347 | "id": 8, 348 | "links": [], 349 | "pageSize": null, 350 | "scroll": true, 351 | "showHeader": true, 352 | "sort": { 353 | "col": 2, 354 | "desc": true 355 | }, 356 | "styles": [ 357 | { 358 | "alias": "Time", 359 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 360 | "pattern": "Time", 361 | "type": "date" 362 | }, 363 | { 364 | "alias": "", 365 | "colorMode": null, 366 | "colors": [ 367 | "rgba(245, 54, 54, 0.9)", 368 | "rgba(237, 129, 40, 0.89)", 369 | "rgba(50, 172, 45, 0.97)" 370 | ], 371 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 372 | "decimals": 0, 373 | "mappingType": 1, 374 | "pattern": ".*_port", 375 | "thresholds": [], 376 | "type": "number", 377 | "unit": "none" 378 | }, 379 | { 380 | "alias": "", 381 | "colorMode": null, 382 | "colors": [ 383 | "rgba(245, 54, 54, 0.9)", 384 | "rgba(237, 129, 40, 0.89)", 385 | "rgba(50, 172, 45, 0.97)" 386 | ], 387 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 388 | "decimals": 2, 389 | "mappingType": 1, 390 | "pattern": "sumbytes", 391 | "thresholds": [], 392 | "type": "number", 393 | "unit": "decbytes" 394 | }, 395 | { 396 | "alias": "", 397 | "colorMode": null, 398 | "colors": [ 399 | "rgba(245, 54, 54, 0.9)", 400 | "rgba(237, 129, 40, 0.89)", 401 | "rgba(50, 172, 45, 0.97)" 402 | ], 403 | "decimals": 0, 404 | "pattern": "/.*/", 405 | "thresholds": [], 406 | "type": "number", 407 | "unit": "short" 408 | } 409 | ], 410 | "targets": [ 411 | { 412 | "format": "table", 413 | "group": [], 414 | "metricColumn": "none", 415 | "rawQuery": true, 416 | "rawSql": "SELECT dst_ip, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_ip", 417 | "refId": "A", 418 | "select": [ 419 | [ 420 | { 421 | "params": [ 422 | "value" 423 | ], 424 | "type": "column" 425 | } 426 | ] 427 | ], 428 | "timeColumn": "time", 429 | "where": [ 430 | { 431 | "name": "$__timeFilter", 432 | "params": [], 433 | "type": "macro" 434 | } 435 | ] 436 | } 437 | ], 438 | "title": "Top destination IPs", 439 | "transform": "table", 440 | "type": "table" 441 | }, 442 | { 443 | "columns": [], 444 | "datasource": "PostgreSQL", 445 | "fontSize": "100%", 446 | "gridPos": { 447 | "h": 9, 448 | "w": 12, 449 | "x": 12, 450 | "y": 18 451 | }, 452 | "id": 6, 453 | "links": [], 454 | "pageSize": null, 455 | "scroll": true, 456 | "showHeader": true, 457 | "sort": { 458 | "col": 2, 459 | "desc": true 460 | }, 461 | "styles": [ 462 | { 463 | "alias": "Time", 464 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 465 | "pattern": "Time", 466 | "type": "date" 467 | }, 468 | { 469 | "alias": "", 470 | "colorMode": null, 471 | "colors": [ 472 | "rgba(245, 54, 54, 0.9)", 473 | "rgba(237, 129, 40, 0.89)", 474 | "rgba(50, 172, 45, 0.97)" 475 | ], 476 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 477 | "decimals": 0, 478 | "mappingType": 1, 479 | "pattern": ".*_port", 480 | "thresholds": [], 481 | "type": "number", 482 | "unit": "none" 483 | }, 484 | { 485 | "alias": "", 486 | "colorMode": null, 487 | "colors": [ 488 | "rgba(245, 54, 54, 0.9)", 489 | "rgba(237, 129, 40, 0.89)", 490 | "rgba(50, 172, 45, 0.97)" 491 | ], 492 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 493 | "decimals": 2, 494 | "mappingType": 1, 495 | "pattern": "sumbytes", 496 | "thresholds": [], 497 | "type": "number", 498 | "unit": "decbytes" 499 | }, 500 | { 501 | "alias": "", 502 | "colorMode": null, 503 | "colors": [ 504 | "rgba(245, 54, 54, 0.9)", 505 | "rgba(237, 129, 40, 0.89)", 506 | "rgba(50, 172, 45, 0.97)" 507 | ], 508 | "decimals": 0, 509 | "pattern": "/.*/", 510 | "thresholds": [], 511 | "type": "number", 512 | "unit": "short" 513 | } 514 | ], 515 | "targets": [ 516 | { 517 | "format": "table", 518 | "group": [], 519 | "metricColumn": "none", 520 | "rawQuery": true, 521 | "rawSql": "SELECT dst_port, count(*), sum(bytes) AS sumBytes FROM flows GROUP BY dst_port", 522 | "refId": "A", 523 | "select": [ 524 | [ 525 | { 526 | "params": [ 527 | "value" 528 | ], 529 | "type": "column" 530 | } 531 | ] 532 | ], 533 | "timeColumn": "time", 534 | "where": [ 535 | { 536 | "name": "$__timeFilter", 537 | "params": [], 538 | "type": "macro" 539 | } 540 | ] 541 | } 542 | ], 543 | "title": "Top destination ports", 544 | "transform": "table", 545 | "type": "table" 546 | } 547 | ], 548 | "schemaVersion": 16, 549 | "style": "dark", 550 | "tags": [], 551 | "templating": { 552 | "list": [] 553 | }, 554 | "time": { 555 | "from": "now-3h", 556 | "to": "now" 557 | }, 558 | "timepicker": { 559 | "refresh_intervals": [ 560 | "5s", 561 | "10s", 562 | "30s", 563 | "1m", 564 | "5m", 565 | "15m", 566 | "30m", 567 | "1h", 568 | "2h", 569 | "1d" 570 | ], 571 | "time_options": [ 572 | "5m", 573 | "15m", 574 | "1h", 575 | "6h", 576 | "12h", 577 | "24h", 578 | "2d", 579 | "7d", 580 | "30d" 581 | ] 582 | }, 583 | "timezone": "", 584 | "title": "Traffic", 585 | "uid": "HdAEBnPiz", 586 | "version": 3 587 | } -------------------------------------------------------------------------------- /compose/grafana/datasources-ch.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | orgId: 1 8 | url: http://prometheus:9090 9 | version: 1 10 | editable: true 11 | - name: ClickHouse 12 | type: vertamedia-clickhouse-datasource 13 | typeLogoUrl: '' 14 | access: proxy 15 | url: http://db:8123 16 | password: '' 17 | user: '' 18 | database: '' 19 | basicAuth: false 20 | basicAuthUser: '' 21 | basicAuthPassword: '' 22 | withCredentials: false 23 | isDefault: true 24 | jsonData: {} 25 | secureJsonFields: {} 26 | version: 3 27 | readOnly: false -------------------------------------------------------------------------------- /compose/grafana/datasources.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | orgId: 1 8 | url: http://prometheus:9090 9 | version: 1 10 | editable: true 11 | - name: PostgreSQL 12 | type: postgres 13 | access: proxy 14 | orgId: 1 15 | url: postgres:5432 16 | database: postgres 17 | user: postgres 18 | jsonData: 19 | sslmode: disable 20 | secureJsonData: 21 | password: flows 22 | version: 1 23 | editable: true -------------------------------------------------------------------------------- /compose/postgres/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 5 | CREATE TABLE IF NOT EXISTS flows ( 6 | id bigserial PRIMARY KEY, 7 | date_inserted timestamp default NULL, 8 | 9 | time_flow timestamp default NULL, 10 | type integer, 11 | sampling_rate integer, 12 | src_as bigint, 13 | dst_as bigint, 14 | src_ip inet, 15 | dst_ip inet, 16 | 17 | bytes bigint, 18 | packets bigint, 19 | 20 | etype integer, 21 | proto integer, 22 | src_port integer, 23 | dst_port integer 24 | ); 25 | EOSQL 26 | -------------------------------------------------------------------------------- /compose/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 3 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 4 | alerting: 5 | alertmanagers: 6 | - static_configs: 7 | - targets: 8 | 9 | rule_files: 10 | 11 | scrape_configs: 12 | - job_name: 'prometheus' 13 | static_configs: 14 | - targets: ['localhost:9090', 'goflow:8080'] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cloudflare/flow-pipeline 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.26.1 7 | github.com/golang/protobuf v1.4.3 8 | github.com/lib/pq v1.3.0 9 | github.com/prometheus/client_golang v1.11.1 10 | github.com/sirupsen/logrus v1.6.0 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= 3 | github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= 4 | github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= 5 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 6 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 7 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 8 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 9 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 10 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 11 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 12 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 13 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 14 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 15 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 16 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= 21 | github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 22 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 23 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 24 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 25 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 26 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 27 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 28 | github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= 29 | github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= 30 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 31 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 32 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 33 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 34 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 35 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 36 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 37 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 38 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 39 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 40 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 41 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 42 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 43 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 44 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 45 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 46 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 47 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 48 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 49 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 50 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 52 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 53 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 54 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 55 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 56 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 57 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 58 | github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= 59 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 60 | github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= 61 | github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= 62 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 63 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 64 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 65 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 66 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 67 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 68 | github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= 69 | github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 70 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 71 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= 72 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 73 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 74 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 75 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 76 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 77 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 78 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 79 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 80 | github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= 81 | github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 82 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 83 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 84 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 85 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 86 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 87 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 88 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 89 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 90 | github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= 91 | github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 92 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 93 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 94 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 95 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 96 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 97 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 98 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 99 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 100 | github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= 101 | github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 102 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 103 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 104 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 105 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 106 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 107 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 108 | github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= 109 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 110 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 111 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 112 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 113 | github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= 114 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 115 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= 116 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 117 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 118 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 119 | github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= 120 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 121 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 122 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 123 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 124 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 125 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 126 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 127 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 128 | github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 129 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 130 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 131 | golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 132 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= 133 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 134 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 135 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 136 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 137 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 138 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 139 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 140 | golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= 141 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 142 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 143 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 144 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 145 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 146 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 147 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 148 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 149 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 150 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 151 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 152 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 153 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 154 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 155 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= 158 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 159 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 160 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 161 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 162 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 163 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 164 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 165 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 166 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 167 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 168 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 169 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 170 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 171 | google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= 172 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 173 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 174 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 175 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 176 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 177 | gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= 178 | gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= 179 | gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= 180 | gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= 181 | gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= 182 | gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= 183 | gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= 184 | gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= 185 | gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= 186 | gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= 187 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 188 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 189 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 190 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 191 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 192 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 193 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 194 | -------------------------------------------------------------------------------- /grafana/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu AS builder 2 | 3 | RUN apt-get update && apt-get install -y git 4 | RUN git clone https://github.com/Vertamedia/clickhouse-grafana.git 5 | 6 | FROM grafana/grafana 7 | 8 | COPY --from=builder /clickhouse-grafana /var/lib/grafana/plugins -------------------------------------------------------------------------------- /inserter/inserter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "flag" 7 | "fmt" 8 | "github.com/Shopify/sarama" 9 | flow "github.com/cloudflare/flow-pipeline/pb-ext" 10 | proto "github.com/golang/protobuf/proto" 11 | _ "github.com/lib/pq" 12 | "github.com/prometheus/client_golang/prometheus" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | log "github.com/sirupsen/logrus" 15 | "net" 16 | "net/http" 17 | "os" 18 | "os/signal" 19 | "strings" 20 | "sync" 21 | "syscall" 22 | "time" 23 | ) 24 | 25 | var ( 26 | LogLevel = flag.String("loglevel", "info", "Log level") 27 | 28 | MetricsAddr = flag.String("metrics.addr", ":8081", "Metrics address") 29 | MetricsPath = flag.String("metrics.path", "/metrics", "Metrics path") 30 | 31 | KafkaVersion = flag.String("kafka.version", "2.1.1", "Kafka version") 32 | KafkaTopic = flag.String("kafka.topic", "flows-processed", "Kafka topic to consume from") 33 | KafkaBrk = flag.String("kafka.brokers", "127.0.0.1:9092,[::1]:9092", "Kafka brokers list separated by commas") 34 | KafkaGroup = flag.String("kafka.group", "postgres-inserter", "Kafka group id") 35 | FlushTime = flag.Duration("flush.dur", time.Second*5, "Flush duration") 36 | FlushCount = flag.Int("flush.count", 100, "Flush count") 37 | 38 | PostgresUser = flag.String("postgres.user", "postgres", "Postgres user") 39 | PostgresPass = flag.String("postgres.pass", "", "Postgres password") 40 | PostgresHost = flag.String("postgres.host", "127.0.0.1", "Postgres host") 41 | PostgresPort = flag.Int("postgres.port", 5432, "Postgres port") 42 | PostgresDbName = flag.String("postgres.dbname", "postgres", "Postgres database") 43 | 44 | Inserts = prometheus.NewCounter( 45 | prometheus.CounterOpts{ 46 | Name: "insert_count", 47 | Help: "Inserts made to Postgres.", 48 | }, 49 | ) 50 | 51 | flow_fields = []string{ 52 | "date_inserted", 53 | "time_flow", 54 | "type", 55 | "sampling_rate", 56 | "src_ip", 57 | "dst_ip", 58 | "bytes", 59 | "packets", 60 | "src_port", 61 | "dst_port", 62 | "etype", 63 | "proto", 64 | "src_as", 65 | "dst_as", 66 | } 67 | ) 68 | 69 | func (s *state) metricsHTTP() { 70 | prometheus.MustRegister(Inserts) 71 | http.Handle(*MetricsPath, promhttp.Handler()) 72 | log.Fatal(http.ListenAndServe(*MetricsAddr, nil)) 73 | } 74 | 75 | type state struct { 76 | ready chan bool 77 | 78 | msgCount int 79 | last time.Time 80 | dur time.Duration 81 | 82 | db *sql.DB 83 | 84 | lock *sync.RWMutex 85 | flows [][]interface{} 86 | 87 | flushTimer <-chan time.Time 88 | } 89 | 90 | func (s *state) flush() bool { 91 | log.Infof("Processed %d records in the last iteration.", s.msgCount) 92 | s.lock.Lock() 93 | s.msgCount = 0 94 | 95 | flows_replace := make([]string, len(flow_fields)) 96 | for i := range flow_fields { 97 | flows_replace[i] = fmt.Sprintf("$%v", i+1) 98 | } 99 | query := fmt.Sprintf("INSERT INTO flows (%v) VALUES (%v)", strings.Join(flow_fields, ", "), strings.Join(flows_replace, ", ")) 100 | for _, curFlow := range s.flows { 101 | _, err := s.db.Exec(query, curFlow...) 102 | if err != nil { 103 | log.Debugf("TEST %v", curFlow) 104 | log.Fatal(err) 105 | } 106 | } 107 | 108 | s.flows = make([][]interface{}, 0) 109 | s.lock.Unlock() 110 | return true 111 | } 112 | 113 | func (s *state) buffer(msg *sarama.ConsumerMessage, cur time.Time) (bool, error, time.Time) { 114 | var flush bool 115 | s.lock.Lock() 116 | s.msgCount++ 117 | 118 | if s.msgCount == *FlushCount { 119 | flush = true 120 | } 121 | 122 | var fmsg flow.FlowMessage 123 | 124 | err := proto.Unmarshal(msg.Value, &fmsg) 125 | if err != nil { 126 | log.Printf("unmarshalling error: ", err) 127 | } else { 128 | log.Debug(fmsg) 129 | ts := time.Unix(int64(fmsg.TimeFlowStart), 0) 130 | 131 | srcip := net.IP(fmsg.SrcAddr) 132 | dstip := net.IP(fmsg.DstAddr) 133 | srcipstr := srcip.String() 134 | dstipstr := dstip.String() 135 | if srcipstr == "" { 136 | srcipstr = "0.0.0.0" 137 | } 138 | if dstipstr == "" { 139 | dstipstr = "0.0.0.0" 140 | } 141 | 142 | extract := []interface{}{ 143 | "NOW()", 144 | ts, 145 | fmsg.Type, 146 | fmsg.SamplingRate, 147 | srcipstr, 148 | dstipstr, 149 | fmsg.Bytes, 150 | fmsg.Packets, 151 | fmsg.SrcPort, 152 | fmsg.DstPort, 153 | fmsg.Etype, 154 | fmsg.Proto, 155 | fmsg.SrcAS, 156 | fmsg.DstAS, 157 | } 158 | s.flows = append(s.flows, extract) 159 | } 160 | s.lock.Unlock() 161 | if flush { 162 | s.flush() 163 | } 164 | return false, nil, cur 165 | } 166 | 167 | func (s *state) Setup(sarama.ConsumerGroupSession) error { 168 | close(s.ready) 169 | return nil 170 | } 171 | 172 | func (s *state) Cleanup(sarama.ConsumerGroupSession) error { 173 | return nil 174 | } 175 | 176 | func (s *state) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { 177 | for { 178 | select { 179 | case message := <-claim.Messages(): 180 | log.Debugf("%s/%d/%d\t%s\t", message.Topic, message.Partition, message.Offset, message.Key) 181 | flush, err, _ := s.buffer(message, time.Now().UTC()) 182 | if flush { 183 | s.flush() 184 | } 185 | if err != nil { 186 | log.Errorf("Error while processing: %v", err) 187 | } 188 | session.MarkMessage(message, "") 189 | case <-s.flushTimer: 190 | s.flush() 191 | s.flushTimer = time.After(*FlushTime) 192 | } 193 | } 194 | 195 | return nil 196 | } 197 | 198 | func main() { 199 | flag.Parse() 200 | 201 | lvl, _ := log.ParseLevel(*LogLevel) 202 | log.SetLevel(lvl) 203 | 204 | s := &state{ 205 | last: time.Time{}, 206 | lock: &sync.RWMutex{}, 207 | flushTimer: time.After(*FlushTime), 208 | ready: make(chan bool), 209 | } 210 | go s.metricsHTTP() 211 | 212 | kafkaVersion, err := sarama.ParseKafkaVersion(*KafkaVersion) 213 | if err != nil { 214 | log.Fatal(err) 215 | } 216 | 217 | config := sarama.NewConfig() 218 | config.Version = kafkaVersion 219 | 220 | pg_pass := *PostgresPass 221 | if pg_pass == "" { 222 | log.Debugf("Postgres password argument unset, using environment variable $POSTGRES_PASSWORD") 223 | pg_pass = os.Getenv("POSTGRES_PASSWORD") 224 | } 225 | 226 | info := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", 227 | *PostgresHost, *PostgresPort, *PostgresUser, pg_pass, *PostgresDbName) 228 | db, err := sql.Open("postgres", info) 229 | if err != nil { 230 | log.Fatal(err) 231 | } 232 | s.db = db 233 | 234 | signals := make(chan os.Signal, 1) 235 | signal.Notify(signals, os.Interrupt) 236 | 237 | ctx, cancel := context.WithCancel(context.Background()) 238 | client, err := sarama.NewConsumerGroup(strings.Split(*KafkaBrk, ","), *KafkaGroup, config) 239 | if err != nil { 240 | log.Fatalf("Error creating consumer group client: %v", err) 241 | } 242 | 243 | wg := &sync.WaitGroup{} 244 | wg.Add(1) 245 | go func() { 246 | defer wg.Done() 247 | for { 248 | if err := client.Consume(ctx, strings.Split(*KafkaTopic, ","), s); err != nil { 249 | log.Fatalf("Error from consumer: %v", err) 250 | } 251 | if ctx.Err() != nil { 252 | return 253 | } 254 | s.ready = make(chan bool) 255 | } 256 | }() 257 | 258 | <-s.ready 259 | 260 | sigterm := make(chan os.Signal, 1) 261 | signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM) 262 | select { 263 | case <-ctx.Done(): 264 | case <-sigterm: 265 | } 266 | cancel() 267 | wg.Wait() 268 | if err = client.Close(); err != nil { 269 | log.Fatalf("Error closing client: %v", err) 270 | } 271 | } 272 | -------------------------------------------------------------------------------- /mocker/mocker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/Shopify/sarama" 6 | flow "github.com/cloudflare/flow-pipeline/pb-ext" 7 | proto "github.com/golang/protobuf/proto" 8 | log "github.com/sirupsen/logrus" 9 | "math/rand" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | var ( 15 | LogLevel = flag.String("loglevel", "info", "Log level") 16 | 17 | ProduceFreq = flag.Int("produce.freq", 100, "Produce interval in ms") 18 | ProduceRandom = flag.Int("produce.random", 300, "Add randomness") 19 | 20 | KafkaTopic = flag.String("kafka.topic", "flows", "Kafka topic to produce to") 21 | KafkaBrk = flag.String("kafka.brokers", "127.0.0.1:9092,[::1]:9092", "Kafka brokers list separated by commas") 22 | 23 | FixedLength = flag.Bool("proto.fixedlen", false, "Enable fixed length protobuf") 24 | ) 25 | 26 | func main() { 27 | flag.Parse() 28 | 29 | lvl, _ := log.ParseLevel(*LogLevel) 30 | log.SetLevel(lvl) 31 | 32 | brokers := strings.Split(*KafkaBrk, ",") 33 | config := sarama.NewConfig() 34 | config.Producer.Return.Errors = true 35 | config.Producer.Return.Successes = false 36 | producer, err := sarama.NewAsyncProducer(brokers, config) 37 | log.Infof("Trying to connect to Kafka: %v", brokers) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | defer producer.Close() 43 | 44 | go func() { 45 | for { 46 | select { 47 | case err := <-producer.Errors(): 48 | log.Error(err) 49 | } 50 | } 51 | }() 52 | 53 | var i uint32 54 | for { 55 | select { 56 | case <-time.After(time.Millisecond * time.Duration(*ProduceFreq+(rand.Int()%(*ProduceRandom)))): 57 | ts := time.Now().UTC().UnixNano() / 1000000000 58 | 59 | bytes := rand.Int() % 1500 60 | packets := rand.Int() % 100 61 | srcas := rand.Int() % 3 62 | dstas := rand.Int() % 3 63 | 64 | srcip := []byte{0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} 65 | dstip := []byte{0x20, 0x01, 0x0d, 0xb8, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} 66 | 67 | tmp := make([]byte, 1) 68 | rand.Read(tmp) 69 | srcip = append(srcip, tmp...) 70 | rand.Read(tmp) 71 | dstip = append(dstip, tmp...) 72 | 73 | srcport := rand.Int() 74 | dstport := rand.Int() 75 | 76 | fmsg := &flow.FlowMessage{ 77 | SamplingRate: 1, 78 | Bytes: uint64(bytes), 79 | Packets: uint64(packets), 80 | SrcAS: uint32(65000 + srcas), 81 | DstAS: uint32(65000 + dstas), 82 | Etype: 0x86dd, 83 | SrcAddr: srcip, 84 | DstAddr: dstip, 85 | TimeFlowStart: uint64(ts), 86 | TimeReceived: uint64(ts), 87 | SrcPort: uint32(srcport & 0xFFFF), 88 | DstPort: uint32(dstport & 0xFFFF), 89 | SequenceNum: i, 90 | } 91 | i++ 92 | 93 | log.Debugf("Sending to %v: %v", *KafkaTopic, fmsg) 94 | 95 | var b []byte 96 | if !*FixedLength { 97 | b, _ = proto.Marshal(fmsg) 98 | } else { 99 | buf := proto.NewBuffer([]byte{}) 100 | buf.EncodeMessage(fmsg) 101 | b = buf.Bytes() 102 | } 103 | producer.Input() <- &sarama.ProducerMessage{ 104 | Topic: *KafkaTopic, 105 | Value: sarama.ByteEncoder(b), 106 | } 107 | } 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /pb-ext/flow.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: pb-ext/flow.proto 3 | 4 | package flowprotob 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/golang/protobuf/proto" 9 | math "math" 10 | ) 11 | 12 | // Reference imports to suppress errors if they are not otherwise used. 13 | var _ = proto.Marshal 14 | var _ = fmt.Errorf 15 | var _ = math.Inf 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the proto package it is being compiled against. 19 | // A compilation error at this line likely means your copy of the 20 | // proto package needs to be updated. 21 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 22 | 23 | type FlowMessage_FlowType int32 24 | 25 | const ( 26 | FlowMessage_FLOWUNKNOWN FlowMessage_FlowType = 0 27 | FlowMessage_SFLOW_5 FlowMessage_FlowType = 1 28 | FlowMessage_NETFLOW_V5 FlowMessage_FlowType = 2 29 | FlowMessage_NETFLOW_V9 FlowMessage_FlowType = 3 30 | FlowMessage_IPFIX FlowMessage_FlowType = 4 31 | ) 32 | 33 | var FlowMessage_FlowType_name = map[int32]string{ 34 | 0: "FLOWUNKNOWN", 35 | 1: "SFLOW_5", 36 | 2: "NETFLOW_V5", 37 | 3: "NETFLOW_V9", 38 | 4: "IPFIX", 39 | } 40 | 41 | var FlowMessage_FlowType_value = map[string]int32{ 42 | "FLOWUNKNOWN": 0, 43 | "SFLOW_5": 1, 44 | "NETFLOW_V5": 2, 45 | "NETFLOW_V9": 3, 46 | "IPFIX": 4, 47 | } 48 | 49 | func (x FlowMessage_FlowType) String() string { 50 | return proto.EnumName(FlowMessage_FlowType_name, int32(x)) 51 | } 52 | 53 | func (FlowMessage_FlowType) EnumDescriptor() ([]byte, []int) { 54 | return fileDescriptor_3864ec3df39bbe77, []int{0, 0} 55 | } 56 | 57 | type FlowMessage struct { 58 | Type FlowMessage_FlowType `protobuf:"varint,1,opt,name=Type,proto3,enum=flowprotob.FlowMessage_FlowType" json:"Type,omitempty"` 59 | TimeReceived uint64 `protobuf:"varint,2,opt,name=TimeReceived,proto3" json:"TimeReceived,omitempty"` 60 | SequenceNum uint32 `protobuf:"varint,4,opt,name=SequenceNum,proto3" json:"SequenceNum,omitempty"` 61 | SamplingRate uint64 `protobuf:"varint,3,opt,name=SamplingRate,proto3" json:"SamplingRate,omitempty"` 62 | FlowDirection uint32 `protobuf:"varint,42,opt,name=FlowDirection,proto3" json:"FlowDirection,omitempty"` 63 | // Sampler information 64 | SamplerAddress []byte `protobuf:"bytes,11,opt,name=SamplerAddress,proto3" json:"SamplerAddress,omitempty"` 65 | // Found inside packet 66 | TimeFlowStart uint64 `protobuf:"varint,38,opt,name=TimeFlowStart,proto3" json:"TimeFlowStart,omitempty"` 67 | TimeFlowEnd uint64 `protobuf:"varint,5,opt,name=TimeFlowEnd,proto3" json:"TimeFlowEnd,omitempty"` 68 | // Size of the sampled packet 69 | Bytes uint64 `protobuf:"varint,9,opt,name=Bytes,proto3" json:"Bytes,omitempty"` 70 | Packets uint64 `protobuf:"varint,10,opt,name=Packets,proto3" json:"Packets,omitempty"` 71 | // Source/destination addresses 72 | SrcAddr []byte `protobuf:"bytes,6,opt,name=SrcAddr,proto3" json:"SrcAddr,omitempty"` 73 | DstAddr []byte `protobuf:"bytes,7,opt,name=DstAddr,proto3" json:"DstAddr,omitempty"` 74 | // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...) 75 | Etype uint32 `protobuf:"varint,30,opt,name=Etype,proto3" json:"Etype,omitempty"` 76 | // Layer 4 protocol 77 | Proto uint32 `protobuf:"varint,20,opt,name=Proto,proto3" json:"Proto,omitempty"` 78 | // Ports for UDP and TCP 79 | SrcPort uint32 `protobuf:"varint,21,opt,name=SrcPort,proto3" json:"SrcPort,omitempty"` 80 | DstPort uint32 `protobuf:"varint,22,opt,name=DstPort,proto3" json:"DstPort,omitempty"` 81 | // Interfaces 82 | InIf uint32 `protobuf:"varint,18,opt,name=InIf,proto3" json:"InIf,omitempty"` 83 | OutIf uint32 `protobuf:"varint,19,opt,name=OutIf,proto3" json:"OutIf,omitempty"` 84 | // Ethernet information 85 | SrcMac uint64 `protobuf:"varint,27,opt,name=SrcMac,proto3" json:"SrcMac,omitempty"` 86 | DstMac uint64 `protobuf:"varint,28,opt,name=DstMac,proto3" json:"DstMac,omitempty"` 87 | // Vlan 88 | SrcVlan uint32 `protobuf:"varint,33,opt,name=SrcVlan,proto3" json:"SrcVlan,omitempty"` 89 | DstVlan uint32 `protobuf:"varint,34,opt,name=DstVlan,proto3" json:"DstVlan,omitempty"` 90 | // 802.1q VLAN in sampled packet 91 | VlanId uint32 `protobuf:"varint,29,opt,name=VlanId,proto3" json:"VlanId,omitempty"` 92 | // VRF 93 | IngressVrfID uint32 `protobuf:"varint,39,opt,name=IngressVrfID,proto3" json:"IngressVrfID,omitempty"` 94 | EgressVrfID uint32 `protobuf:"varint,40,opt,name=EgressVrfID,proto3" json:"EgressVrfID,omitempty"` 95 | // IP and TCP special flags 96 | IPTos uint32 `protobuf:"varint,23,opt,name=IPTos,proto3" json:"IPTos,omitempty"` 97 | ForwardingStatus uint32 `protobuf:"varint,24,opt,name=ForwardingStatus,proto3" json:"ForwardingStatus,omitempty"` 98 | IPTTL uint32 `protobuf:"varint,25,opt,name=IPTTL,proto3" json:"IPTTL,omitempty"` 99 | TCPFlags uint32 `protobuf:"varint,26,opt,name=TCPFlags,proto3" json:"TCPFlags,omitempty"` 100 | IcmpType uint32 `protobuf:"varint,31,opt,name=IcmpType,proto3" json:"IcmpType,omitempty"` 101 | IcmpCode uint32 `protobuf:"varint,32,opt,name=IcmpCode,proto3" json:"IcmpCode,omitempty"` 102 | IPv6FlowLabel uint32 `protobuf:"varint,37,opt,name=IPv6FlowLabel,proto3" json:"IPv6FlowLabel,omitempty"` 103 | // Fragments (IPv4/IPv6) 104 | FragmentId uint32 `protobuf:"varint,35,opt,name=FragmentId,proto3" json:"FragmentId,omitempty"` 105 | FragmentOffset uint32 `protobuf:"varint,36,opt,name=FragmentOffset,proto3" json:"FragmentOffset,omitempty"` 106 | BiFlowDirection uint32 `protobuf:"varint,41,opt,name=BiFlowDirection,proto3" json:"BiFlowDirection,omitempty"` 107 | // Autonomous system information 108 | SrcAS uint32 `protobuf:"varint,14,opt,name=SrcAS,proto3" json:"SrcAS,omitempty"` 109 | DstAS uint32 `protobuf:"varint,15,opt,name=DstAS,proto3" json:"DstAS,omitempty"` 110 | NextHop []byte `protobuf:"bytes,12,opt,name=NextHop,proto3" json:"NextHop,omitempty"` 111 | NextHopAS uint32 `protobuf:"varint,13,opt,name=NextHopAS,proto3" json:"NextHopAS,omitempty"` 112 | // Prefix size 113 | SrcNet uint32 `protobuf:"varint,16,opt,name=SrcNet,proto3" json:"SrcNet,omitempty"` 114 | DstNet uint32 `protobuf:"varint,17,opt,name=DstNet,proto3" json:"DstNet,omitempty"` 115 | // IP encapsulation information 116 | HasEncap bool `protobuf:"varint,43,opt,name=HasEncap,proto3" json:"HasEncap,omitempty"` 117 | SrcAddrEncap []byte `protobuf:"bytes,44,opt,name=SrcAddrEncap,proto3" json:"SrcAddrEncap,omitempty"` 118 | DstAddrEncap []byte `protobuf:"bytes,45,opt,name=DstAddrEncap,proto3" json:"DstAddrEncap,omitempty"` 119 | ProtoEncap uint32 `protobuf:"varint,46,opt,name=ProtoEncap,proto3" json:"ProtoEncap,omitempty"` 120 | EtypeEncap uint32 `protobuf:"varint,47,opt,name=EtypeEncap,proto3" json:"EtypeEncap,omitempty"` 121 | IPTosEncap uint32 `protobuf:"varint,48,opt,name=IPTosEncap,proto3" json:"IPTosEncap,omitempty"` 122 | IPTTLEncap uint32 `protobuf:"varint,49,opt,name=IPTTLEncap,proto3" json:"IPTTLEncap,omitempty"` 123 | IPv6FlowLabelEncap uint32 `protobuf:"varint,50,opt,name=IPv6FlowLabelEncap,proto3" json:"IPv6FlowLabelEncap,omitempty"` 124 | FragmentIdEncap uint32 `protobuf:"varint,51,opt,name=FragmentIdEncap,proto3" json:"FragmentIdEncap,omitempty"` 125 | FragmentOffsetEncap uint32 `protobuf:"varint,52,opt,name=FragmentOffsetEncap,proto3" json:"FragmentOffsetEncap,omitempty"` 126 | // MPLS information 127 | HasMPLS bool `protobuf:"varint,53,opt,name=HasMPLS,proto3" json:"HasMPLS,omitempty"` 128 | MPLSCount uint32 `protobuf:"varint,54,opt,name=MPLSCount,proto3" json:"MPLSCount,omitempty"` 129 | MPLS1TTL uint32 `protobuf:"varint,55,opt,name=MPLS1TTL,proto3" json:"MPLS1TTL,omitempty"` 130 | MPLS1Label uint32 `protobuf:"varint,56,opt,name=MPLS1Label,proto3" json:"MPLS1Label,omitempty"` 131 | MPLS2TTL uint32 `protobuf:"varint,57,opt,name=MPLS2TTL,proto3" json:"MPLS2TTL,omitempty"` 132 | MPLS2Label uint32 `protobuf:"varint,58,opt,name=MPLS2Label,proto3" json:"MPLS2Label,omitempty"` 133 | MPLS3TTL uint32 `protobuf:"varint,59,opt,name=MPLS3TTL,proto3" json:"MPLS3TTL,omitempty"` 134 | MPLS3Label uint32 `protobuf:"varint,60,opt,name=MPLS3Label,proto3" json:"MPLS3Label,omitempty"` 135 | MPLSLastTTL uint32 `protobuf:"varint,61,opt,name=MPLSLastTTL,proto3" json:"MPLSLastTTL,omitempty"` 136 | MPLSLastLabel uint32 `protobuf:"varint,62,opt,name=MPLSLastLabel,proto3" json:"MPLSLastLabel,omitempty"` 137 | // PPP information 138 | HasPPP bool `protobuf:"varint,63,opt,name=HasPPP,proto3" json:"HasPPP,omitempty"` 139 | PPPAddressControl uint32 `protobuf:"varint,64,opt,name=PPPAddressControl,proto3" json:"PPPAddressControl,omitempty"` 140 | SrcCountry string `protobuf:"bytes,100,opt,name=SrcCountry,proto3" json:"SrcCountry,omitempty"` 141 | DstCountry string `protobuf:"bytes,101,opt,name=DstCountry,proto3" json:"DstCountry,omitempty"` 142 | SrcASDB uint32 `protobuf:"varint,102,opt,name=SrcASDB,proto3" json:"SrcASDB,omitempty"` 143 | DstASDB uint32 `protobuf:"varint,103,opt,name=DstASDB,proto3" json:"DstASDB,omitempty"` 144 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 145 | XXX_unrecognized []byte `json:"-"` 146 | XXX_sizecache int32 `json:"-"` 147 | } 148 | 149 | func (m *FlowMessage) Reset() { *m = FlowMessage{} } 150 | func (m *FlowMessage) String() string { return proto.CompactTextString(m) } 151 | func (*FlowMessage) ProtoMessage() {} 152 | func (*FlowMessage) Descriptor() ([]byte, []int) { 153 | return fileDescriptor_3864ec3df39bbe77, []int{0} 154 | } 155 | 156 | func (m *FlowMessage) XXX_Unmarshal(b []byte) error { 157 | return xxx_messageInfo_FlowMessage.Unmarshal(m, b) 158 | } 159 | func (m *FlowMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 160 | return xxx_messageInfo_FlowMessage.Marshal(b, m, deterministic) 161 | } 162 | func (m *FlowMessage) XXX_Merge(src proto.Message) { 163 | xxx_messageInfo_FlowMessage.Merge(m, src) 164 | } 165 | func (m *FlowMessage) XXX_Size() int { 166 | return xxx_messageInfo_FlowMessage.Size(m) 167 | } 168 | func (m *FlowMessage) XXX_DiscardUnknown() { 169 | xxx_messageInfo_FlowMessage.DiscardUnknown(m) 170 | } 171 | 172 | var xxx_messageInfo_FlowMessage proto.InternalMessageInfo 173 | 174 | func (m *FlowMessage) GetType() FlowMessage_FlowType { 175 | if m != nil { 176 | return m.Type 177 | } 178 | return FlowMessage_FLOWUNKNOWN 179 | } 180 | 181 | func (m *FlowMessage) GetTimeReceived() uint64 { 182 | if m != nil { 183 | return m.TimeReceived 184 | } 185 | return 0 186 | } 187 | 188 | func (m *FlowMessage) GetSequenceNum() uint32 { 189 | if m != nil { 190 | return m.SequenceNum 191 | } 192 | return 0 193 | } 194 | 195 | func (m *FlowMessage) GetSamplingRate() uint64 { 196 | if m != nil { 197 | return m.SamplingRate 198 | } 199 | return 0 200 | } 201 | 202 | func (m *FlowMessage) GetFlowDirection() uint32 { 203 | if m != nil { 204 | return m.FlowDirection 205 | } 206 | return 0 207 | } 208 | 209 | func (m *FlowMessage) GetSamplerAddress() []byte { 210 | if m != nil { 211 | return m.SamplerAddress 212 | } 213 | return nil 214 | } 215 | 216 | func (m *FlowMessage) GetTimeFlowStart() uint64 { 217 | if m != nil { 218 | return m.TimeFlowStart 219 | } 220 | return 0 221 | } 222 | 223 | func (m *FlowMessage) GetTimeFlowEnd() uint64 { 224 | if m != nil { 225 | return m.TimeFlowEnd 226 | } 227 | return 0 228 | } 229 | 230 | func (m *FlowMessage) GetBytes() uint64 { 231 | if m != nil { 232 | return m.Bytes 233 | } 234 | return 0 235 | } 236 | 237 | func (m *FlowMessage) GetPackets() uint64 { 238 | if m != nil { 239 | return m.Packets 240 | } 241 | return 0 242 | } 243 | 244 | func (m *FlowMessage) GetSrcAddr() []byte { 245 | if m != nil { 246 | return m.SrcAddr 247 | } 248 | return nil 249 | } 250 | 251 | func (m *FlowMessage) GetDstAddr() []byte { 252 | if m != nil { 253 | return m.DstAddr 254 | } 255 | return nil 256 | } 257 | 258 | func (m *FlowMessage) GetEtype() uint32 { 259 | if m != nil { 260 | return m.Etype 261 | } 262 | return 0 263 | } 264 | 265 | func (m *FlowMessage) GetProto() uint32 { 266 | if m != nil { 267 | return m.Proto 268 | } 269 | return 0 270 | } 271 | 272 | func (m *FlowMessage) GetSrcPort() uint32 { 273 | if m != nil { 274 | return m.SrcPort 275 | } 276 | return 0 277 | } 278 | 279 | func (m *FlowMessage) GetDstPort() uint32 { 280 | if m != nil { 281 | return m.DstPort 282 | } 283 | return 0 284 | } 285 | 286 | func (m *FlowMessage) GetInIf() uint32 { 287 | if m != nil { 288 | return m.InIf 289 | } 290 | return 0 291 | } 292 | 293 | func (m *FlowMessage) GetOutIf() uint32 { 294 | if m != nil { 295 | return m.OutIf 296 | } 297 | return 0 298 | } 299 | 300 | func (m *FlowMessage) GetSrcMac() uint64 { 301 | if m != nil { 302 | return m.SrcMac 303 | } 304 | return 0 305 | } 306 | 307 | func (m *FlowMessage) GetDstMac() uint64 { 308 | if m != nil { 309 | return m.DstMac 310 | } 311 | return 0 312 | } 313 | 314 | func (m *FlowMessage) GetSrcVlan() uint32 { 315 | if m != nil { 316 | return m.SrcVlan 317 | } 318 | return 0 319 | } 320 | 321 | func (m *FlowMessage) GetDstVlan() uint32 { 322 | if m != nil { 323 | return m.DstVlan 324 | } 325 | return 0 326 | } 327 | 328 | func (m *FlowMessage) GetVlanId() uint32 { 329 | if m != nil { 330 | return m.VlanId 331 | } 332 | return 0 333 | } 334 | 335 | func (m *FlowMessage) GetIngressVrfID() uint32 { 336 | if m != nil { 337 | return m.IngressVrfID 338 | } 339 | return 0 340 | } 341 | 342 | func (m *FlowMessage) GetEgressVrfID() uint32 { 343 | if m != nil { 344 | return m.EgressVrfID 345 | } 346 | return 0 347 | } 348 | 349 | func (m *FlowMessage) GetIPTos() uint32 { 350 | if m != nil { 351 | return m.IPTos 352 | } 353 | return 0 354 | } 355 | 356 | func (m *FlowMessage) GetForwardingStatus() uint32 { 357 | if m != nil { 358 | return m.ForwardingStatus 359 | } 360 | return 0 361 | } 362 | 363 | func (m *FlowMessage) GetIPTTL() uint32 { 364 | if m != nil { 365 | return m.IPTTL 366 | } 367 | return 0 368 | } 369 | 370 | func (m *FlowMessage) GetTCPFlags() uint32 { 371 | if m != nil { 372 | return m.TCPFlags 373 | } 374 | return 0 375 | } 376 | 377 | func (m *FlowMessage) GetIcmpType() uint32 { 378 | if m != nil { 379 | return m.IcmpType 380 | } 381 | return 0 382 | } 383 | 384 | func (m *FlowMessage) GetIcmpCode() uint32 { 385 | if m != nil { 386 | return m.IcmpCode 387 | } 388 | return 0 389 | } 390 | 391 | func (m *FlowMessage) GetIPv6FlowLabel() uint32 { 392 | if m != nil { 393 | return m.IPv6FlowLabel 394 | } 395 | return 0 396 | } 397 | 398 | func (m *FlowMessage) GetFragmentId() uint32 { 399 | if m != nil { 400 | return m.FragmentId 401 | } 402 | return 0 403 | } 404 | 405 | func (m *FlowMessage) GetFragmentOffset() uint32 { 406 | if m != nil { 407 | return m.FragmentOffset 408 | } 409 | return 0 410 | } 411 | 412 | func (m *FlowMessage) GetBiFlowDirection() uint32 { 413 | if m != nil { 414 | return m.BiFlowDirection 415 | } 416 | return 0 417 | } 418 | 419 | func (m *FlowMessage) GetSrcAS() uint32 { 420 | if m != nil { 421 | return m.SrcAS 422 | } 423 | return 0 424 | } 425 | 426 | func (m *FlowMessage) GetDstAS() uint32 { 427 | if m != nil { 428 | return m.DstAS 429 | } 430 | return 0 431 | } 432 | 433 | func (m *FlowMessage) GetNextHop() []byte { 434 | if m != nil { 435 | return m.NextHop 436 | } 437 | return nil 438 | } 439 | 440 | func (m *FlowMessage) GetNextHopAS() uint32 { 441 | if m != nil { 442 | return m.NextHopAS 443 | } 444 | return 0 445 | } 446 | 447 | func (m *FlowMessage) GetSrcNet() uint32 { 448 | if m != nil { 449 | return m.SrcNet 450 | } 451 | return 0 452 | } 453 | 454 | func (m *FlowMessage) GetDstNet() uint32 { 455 | if m != nil { 456 | return m.DstNet 457 | } 458 | return 0 459 | } 460 | 461 | func (m *FlowMessage) GetHasEncap() bool { 462 | if m != nil { 463 | return m.HasEncap 464 | } 465 | return false 466 | } 467 | 468 | func (m *FlowMessage) GetSrcAddrEncap() []byte { 469 | if m != nil { 470 | return m.SrcAddrEncap 471 | } 472 | return nil 473 | } 474 | 475 | func (m *FlowMessage) GetDstAddrEncap() []byte { 476 | if m != nil { 477 | return m.DstAddrEncap 478 | } 479 | return nil 480 | } 481 | 482 | func (m *FlowMessage) GetProtoEncap() uint32 { 483 | if m != nil { 484 | return m.ProtoEncap 485 | } 486 | return 0 487 | } 488 | 489 | func (m *FlowMessage) GetEtypeEncap() uint32 { 490 | if m != nil { 491 | return m.EtypeEncap 492 | } 493 | return 0 494 | } 495 | 496 | func (m *FlowMessage) GetIPTosEncap() uint32 { 497 | if m != nil { 498 | return m.IPTosEncap 499 | } 500 | return 0 501 | } 502 | 503 | func (m *FlowMessage) GetIPTTLEncap() uint32 { 504 | if m != nil { 505 | return m.IPTTLEncap 506 | } 507 | return 0 508 | } 509 | 510 | func (m *FlowMessage) GetIPv6FlowLabelEncap() uint32 { 511 | if m != nil { 512 | return m.IPv6FlowLabelEncap 513 | } 514 | return 0 515 | } 516 | 517 | func (m *FlowMessage) GetFragmentIdEncap() uint32 { 518 | if m != nil { 519 | return m.FragmentIdEncap 520 | } 521 | return 0 522 | } 523 | 524 | func (m *FlowMessage) GetFragmentOffsetEncap() uint32 { 525 | if m != nil { 526 | return m.FragmentOffsetEncap 527 | } 528 | return 0 529 | } 530 | 531 | func (m *FlowMessage) GetHasMPLS() bool { 532 | if m != nil { 533 | return m.HasMPLS 534 | } 535 | return false 536 | } 537 | 538 | func (m *FlowMessage) GetMPLSCount() uint32 { 539 | if m != nil { 540 | return m.MPLSCount 541 | } 542 | return 0 543 | } 544 | 545 | func (m *FlowMessage) GetMPLS1TTL() uint32 { 546 | if m != nil { 547 | return m.MPLS1TTL 548 | } 549 | return 0 550 | } 551 | 552 | func (m *FlowMessage) GetMPLS1Label() uint32 { 553 | if m != nil { 554 | return m.MPLS1Label 555 | } 556 | return 0 557 | } 558 | 559 | func (m *FlowMessage) GetMPLS2TTL() uint32 { 560 | if m != nil { 561 | return m.MPLS2TTL 562 | } 563 | return 0 564 | } 565 | 566 | func (m *FlowMessage) GetMPLS2Label() uint32 { 567 | if m != nil { 568 | return m.MPLS2Label 569 | } 570 | return 0 571 | } 572 | 573 | func (m *FlowMessage) GetMPLS3TTL() uint32 { 574 | if m != nil { 575 | return m.MPLS3TTL 576 | } 577 | return 0 578 | } 579 | 580 | func (m *FlowMessage) GetMPLS3Label() uint32 { 581 | if m != nil { 582 | return m.MPLS3Label 583 | } 584 | return 0 585 | } 586 | 587 | func (m *FlowMessage) GetMPLSLastTTL() uint32 { 588 | if m != nil { 589 | return m.MPLSLastTTL 590 | } 591 | return 0 592 | } 593 | 594 | func (m *FlowMessage) GetMPLSLastLabel() uint32 { 595 | if m != nil { 596 | return m.MPLSLastLabel 597 | } 598 | return 0 599 | } 600 | 601 | func (m *FlowMessage) GetHasPPP() bool { 602 | if m != nil { 603 | return m.HasPPP 604 | } 605 | return false 606 | } 607 | 608 | func (m *FlowMessage) GetPPPAddressControl() uint32 { 609 | if m != nil { 610 | return m.PPPAddressControl 611 | } 612 | return 0 613 | } 614 | 615 | func (m *FlowMessage) GetSrcCountry() string { 616 | if m != nil { 617 | return m.SrcCountry 618 | } 619 | return "" 620 | } 621 | 622 | func (m *FlowMessage) GetDstCountry() string { 623 | if m != nil { 624 | return m.DstCountry 625 | } 626 | return "" 627 | } 628 | 629 | func (m *FlowMessage) GetSrcASDB() uint32 { 630 | if m != nil { 631 | return m.SrcASDB 632 | } 633 | return 0 634 | } 635 | 636 | func (m *FlowMessage) GetDstASDB() uint32 { 637 | if m != nil { 638 | return m.DstASDB 639 | } 640 | return 0 641 | } 642 | 643 | func init() { 644 | proto.RegisterEnum("flowprotob.FlowMessage_FlowType", FlowMessage_FlowType_name, FlowMessage_FlowType_value) 645 | proto.RegisterType((*FlowMessage)(nil), "flowprotob.FlowMessage") 646 | } 647 | 648 | func init() { proto.RegisterFile("pb-ext/flow.proto", fileDescriptor_3864ec3df39bbe77) } 649 | 650 | var fileDescriptor_3864ec3df39bbe77 = []byte{ 651 | // 986 bytes of a gzipped FileDescriptorProto 652 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x96, 0xfb, 0x73, 0xda, 0x46, 653 | 0x10, 0xc7, 0x4b, 0xe2, 0xf8, 0x71, 0xf8, 0x81, 0x2f, 0xa9, 0xbb, 0x75, 0xd3, 0x94, 0xba, 0x69, 654 | 0x4a, 0xf3, 0x20, 0x09, 0xc4, 0x69, 0xd3, 0x77, 0x30, 0x30, 0xd6, 0x14, 0x63, 0x0d, 0xa2, 0x4e, 655 | 0x7f, 0xeb, 0x1c, 0xd2, 0xc1, 0x30, 0x05, 0x89, 0x4a, 0x47, 0x1c, 0xff, 0xf3, 0x9d, 0xcc, 0xee, 656 | 0x9e, 0x90, 0xe4, 0xe4, 0x27, 0xf4, 0xfd, 0x7c, 0x77, 0x97, 0xd3, 0xed, 0x9e, 0x24, 0xb1, 0xbf, 657 | 0x18, 0x3d, 0xd1, 0xef, 0xcc, 0xd3, 0xf1, 0x2c, 0xba, 0xac, 0x2f, 0xe2, 0xc8, 0x44, 0x52, 0xe0, 658 | 0x35, 0x5d, 0x8e, 0x8e, 0xfe, 0xaf, 0x88, 0x72, 0x77, 0x16, 0x5d, 0x9e, 0xe9, 0x24, 0x51, 0x13, 659 | 0x2d, 0x5f, 0x88, 0xb5, 0xe1, 0xd5, 0x42, 0x43, 0xa9, 0x5a, 0xaa, 0xed, 0x36, 0xaa, 0xf5, 0x2c, 660 | 0xb4, 0x9e, 0x0b, 0xa3, 0x6b, 0x8c, 0x1b, 0x50, 0xb4, 0x3c, 0x12, 0xdb, 0xc3, 0xe9, 0x5c, 0x0f, 661 | 0xb4, 0xaf, 0xa7, 0x6f, 0x75, 0x00, 0x37, 0xaa, 0xa5, 0xda, 0xda, 0xa0, 0xc0, 0x64, 0x55, 0x94, 662 | 0x3d, 0xfd, 0xdf, 0x52, 0x87, 0xbe, 0xee, 0x2f, 0xe7, 0xb0, 0x56, 0x2d, 0xd5, 0x76, 0x06, 0x79, 663 | 0x84, 0x55, 0x3c, 0x35, 0x5f, 0xcc, 0xa6, 0xe1, 0x64, 0xa0, 0x8c, 0x86, 0x9b, 0x5c, 0x25, 0xcf, 664 | 0xe4, 0x7d, 0xb1, 0x83, 0xff, 0xdd, 0x9e, 0xc6, 0xda, 0x37, 0xd3, 0x28, 0x84, 0x87, 0x54, 0xa7, 665 | 0x08, 0xe5, 0x03, 0xb1, 0x4b, 0x59, 0x3a, 0x7e, 0x1d, 0x04, 0xb1, 0x4e, 0x12, 0x28, 0x57, 0x4b, 666 | 0xb5, 0xed, 0xc1, 0x35, 0x8a, 0xd5, 0x70, 0x8d, 0x98, 0xec, 0x19, 0x15, 0x1b, 0x78, 0x40, 0x7f, 667 | 0x59, 0x84, 0xb8, 0xf2, 0x14, 0x74, 0xc2, 0x00, 0x6e, 0x51, 0x4c, 0x1e, 0xc9, 0x3b, 0xe2, 0x56, 668 | 0xeb, 0xca, 0xe8, 0x04, 0xb6, 0xc8, 0x63, 0x21, 0x41, 0x6c, 0xb8, 0xca, 0xff, 0x57, 0x9b, 0x04, 669 | 0x04, 0xf1, 0x54, 0xa2, 0xe3, 0xc5, 0x3e, 0xae, 0x02, 0xd6, 0x69, 0x61, 0xa9, 0x44, 0xa7, 0x9d, 670 | 0x18, 0x72, 0x36, 0xd8, 0xb1, 0x12, 0xff, 0xa3, 0x63, 0xb0, 0x35, 0xf7, 0xe8, 0x8e, 0x59, 0x20, 671 | 0x75, 0xb1, 0x3d, 0x70, 0x87, 0x29, 0x09, 0x5b, 0xdf, 0x8d, 0x62, 0x03, 0x9f, 0x12, 0x4f, 0xa5, 672 | 0xad, 0x4f, 0xce, 0x01, 0x3b, 0x56, 0x4a, 0x29, 0xd6, 0x9c, 0xd0, 0x19, 0x83, 0x24, 0x4c, 0xd7, 673 | 0x58, 0xfd, 0x7c, 0x69, 0x9c, 0x31, 0xdc, 0xe6, 0xea, 0x24, 0xe4, 0x81, 0x58, 0xf7, 0x62, 0xff, 674 | 0x4c, 0xf9, 0xf0, 0x05, 0xdd, 0x96, 0x55, 0xc8, 0xdb, 0x89, 0x41, 0x7e, 0x97, 0x39, 0x2b, 0xbb, 675 | 0x9a, 0x8b, 0x99, 0x0a, 0xe1, 0xeb, 0xd5, 0x6a, 0x50, 0xda, 0xd5, 0x90, 0x73, 0xb4, 0x5a, 0x0d, 676 | 0x39, 0x07, 0x62, 0x1d, 0x7f, 0x9d, 0x00, 0xbe, 0x24, 0xc3, 0x2a, 0x9c, 0x11, 0x27, 0x9c, 0x60, 677 | 0xf3, 0x2e, 0xe2, 0xb1, 0xd3, 0x86, 0xef, 0xc8, 0x2d, 0x30, 0xec, 0x57, 0x27, 0x17, 0x52, 0xe3, 678 | 0x49, 0xcb, 0x21, 0xbc, 0x2f, 0xc7, 0x1d, 0x46, 0x09, 0x7c, 0xc6, 0xf7, 0x45, 0x42, 0x3e, 0x14, 679 | 0x95, 0x6e, 0x14, 0x5f, 0xaa, 0x38, 0x98, 0x86, 0x13, 0xcf, 0x28, 0xb3, 0x4c, 0x00, 0x28, 0xe0, 680 | 0x03, 0x6e, 0x2b, 0x0c, 0x7b, 0xf0, 0xf9, 0xaa, 0xc2, 0xb0, 0x27, 0x0f, 0xc5, 0xe6, 0xf0, 0xc4, 681 | 0xed, 0xce, 0xd4, 0x24, 0x81, 0x43, 0x32, 0x56, 0x1a, 0x3d, 0xc7, 0x9f, 0x2f, 0xe8, 0x74, 0x7d, 682 | 0xc5, 0x5e, 0xaa, 0x53, 0xef, 0x24, 0x0a, 0x34, 0x54, 0x33, 0x0f, 0x35, 0xce, 0xa8, 0xe3, 0xbe, 683 | 0x7d, 0x89, 0xa3, 0xd6, 0x53, 0x23, 0x3d, 0x83, 0x6f, 0x79, 0xe2, 0x0b, 0x50, 0xde, 0x13, 0xa2, 684 | 0x1b, 0xab, 0xc9, 0x5c, 0x87, 0xc6, 0x09, 0xe0, 0x1b, 0x0a, 0xc9, 0x11, 0x3c, 0x11, 0xa9, 0x3a, 685 | 0x1f, 0x8f, 0x13, 0x6d, 0xe0, 0x3e, 0xc5, 0x5c, 0xa3, 0xb2, 0x26, 0xf6, 0x5a, 0xd3, 0xe2, 0x09, 686 | 0xfb, 0x9e, 0x02, 0xaf, 0x63, 0xdc, 0x01, 0x1c, 0x5a, 0x0f, 0x76, 0x79, 0x07, 0x48, 0x20, 0xc5, 687 | 0x81, 0xf5, 0x60, 0x8f, 0x29, 0x09, 0xec, 0x73, 0x5f, 0xbf, 0x33, 0xa7, 0xd1, 0x02, 0xb6, 0x79, 688 | 0xaa, 0xad, 0x94, 0x77, 0xc5, 0x96, 0xbd, 0x7c, 0xed, 0xc1, 0x0e, 0xe5, 0x64, 0xc0, 0x4e, 0x5a, 689 | 0x5f, 0x1b, 0xa8, 0xf0, 0x14, 0xb0, 0xb2, 0x93, 0x86, 0x7c, 0x9f, 0x39, 0x2b, 0xdc, 0xc7, 0x53, 690 | 0x95, 0x74, 0x42, 0x5f, 0x2d, 0xe0, 0x51, 0xb5, 0x54, 0xdb, 0x1c, 0xac, 0x34, 0x3d, 0x5d, 0xf8, 691 | 0x90, 0xb1, 0xff, 0x98, 0x16, 0x52, 0x60, 0x18, 0x63, 0x8f, 0x1b, 0xc7, 0x3c, 0xe1, 0x98, 0x3c, 692 | 0xc3, 0x9d, 0xa6, 0x43, 0xc6, 0x11, 0x75, 0xde, 0xe9, 0x8c, 0xa0, 0x4f, 0x47, 0x93, 0xfd, 0xa7, 693 | 0xec, 0x67, 0x04, 0x7d, 0x1a, 0x37, 0xf6, 0x9f, 0xb1, 0x9f, 0x11, 0xeb, 0x0f, 0x7b, 0xec, 0x3f, 694 | 0x5f, 0xf9, 0x96, 0xc8, 0xba, 0x90, 0x85, 0xd6, 0x73, 0x5c, 0x83, 0xe2, 0x3e, 0xe2, 0x60, 0x47, 695 | 0xb3, 0x39, 0xe0, 0xe0, 0x26, 0x77, 0xf4, 0x1a, 0x96, 0xcf, 0xc4, 0xed, 0xe2, 0x34, 0x70, 0xf4, 696 | 0x0b, 0x8a, 0xfe, 0x98, 0x85, 0x7d, 0x3d, 0x55, 0xc9, 0x99, 0xdb, 0xf3, 0xe0, 0x98, 0xb6, 0x3b, 697 | 0x95, 0xd8, 0x57, 0xfc, 0x3d, 0x89, 0x96, 0xa1, 0x81, 0x97, 0xdc, 0xd7, 0x15, 0xc0, 0x3e, 0xa1, 698 | 0x78, 0x8e, 0x07, 0xe8, 0x07, 0x9e, 0xf7, 0x54, 0xe3, 0xfd, 0xd3, 0x35, 0x0f, 0xfb, 0x8f, 0x7c, 699 | 0xff, 0x19, 0x49, 0x73, 0x1b, 0x98, 0xfb, 0x2a, 0xcb, 0x6d, 0xe4, 0x72, 0x1b, 0x9c, 0xfb, 0x53, 700 | 0x96, 0xdb, 0x28, 0xe4, 0x36, 0x31, 0xf7, 0xe7, 0x2c, 0xb7, 0x99, 0xcb, 0x6d, 0x72, 0xee, 0x2f, 701 | 0x59, 0x2e, 0x13, 0x7c, 0xaa, 0xa0, 0xea, 0xa9, 0xc4, 0x60, 0xfa, 0xaf, 0xfc, 0x54, 0xc9, 0x21, 702 | 0x3c, 0xa9, 0xa9, 0xe4, 0x22, 0xbf, 0xf1, 0x49, 0x2d, 0x40, 0x9c, 0xdd, 0x53, 0x95, 0xb8, 0xae, 703 | 0x0b, 0xbf, 0xd3, 0x96, 0x59, 0x25, 0x1f, 0x8b, 0x7d, 0xd7, 0x75, 0xed, 0x9b, 0xe9, 0x24, 0x0a, 704 | 0x4d, 0x1c, 0xcd, 0xe0, 0x0f, 0xaa, 0xf0, 0xa1, 0x81, 0xab, 0xf5, 0x62, 0x9f, 0x76, 0x33, 0xbe, 705 | 0x82, 0xa0, 0x5a, 0xaa, 0x6d, 0x0d, 0x72, 0x04, 0xfd, 0x76, 0x62, 0x52, 0x5f, 0xb3, 0x9f, 0x91, 706 | 0xf4, 0x0d, 0xe4, 0xb5, 0x5b, 0x30, 0x5e, 0x3d, 0x93, 0x51, 0xa6, 0x6f, 0x20, 0x74, 0x26, 0xab, 707 | 0x67, 0x32, 0xca, 0x23, 0x4f, 0x6c, 0xa6, 0xef, 0x7d, 0xb9, 0x27, 0xca, 0xdd, 0xde, 0xf9, 0x9b, 708 | 0xbf, 0xfa, 0x7f, 0xf6, 0xcf, 0xdf, 0xf4, 0x2b, 0x9f, 0xc8, 0xb2, 0xd8, 0xf0, 0x90, 0xfc, 0x73, 709 | 0x5c, 0x29, 0xc9, 0x5d, 0x21, 0xfa, 0x9d, 0x21, 0xc9, 0x8b, 0xe3, 0xca, 0x8d, 0x82, 0x7e, 0x55, 710 | 0xb9, 0x29, 0xb7, 0xf0, 0xe9, 0xd9, 0x75, 0xfe, 0xae, 0xac, 0xb5, 0x1e, 0x89, 0x43, 0x3f, 0x9a, 711 | 0xd7, 0xfd, 0x59, 0xb4, 0x0c, 0xc6, 0x33, 0x15, 0xeb, 0x7a, 0xa8, 0x0d, 0x7d, 0x76, 0xa8, 0xc9, 712 | 0xa4, 0xb5, 0x93, 0xfb, 0xe8, 0x70, 0x47, 0xa3, 0x75, 0xfa, 0x14, 0x69, 0xbe, 0x0f, 0x00, 0x00, 713 | 0xff, 0xff, 0x57, 0x93, 0x96, 0x00, 0xd5, 0x08, 0x00, 0x00, 714 | } 715 | -------------------------------------------------------------------------------- /pb-ext/flow.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package flowprotob; 3 | 4 | option java_package = "com.cloudflare.net.flowagg"; 5 | option java_outer_classname = "FlowMessagePb"; 6 | 7 | message FlowMessage { 8 | 9 | enum FlowType { 10 | FLOWUNKNOWN = 0; 11 | SFLOW_5 = 1; 12 | NETFLOW_V5 = 2; 13 | NETFLOW_V9 = 3; 14 | IPFIX = 4; 15 | } 16 | FlowType Type = 1; 17 | 18 | uint64 TimeReceived = 2; 19 | uint32 SequenceNum = 4; 20 | uint64 SamplingRate = 3; 21 | 22 | uint32 FlowDirection = 42; 23 | 24 | // Sampler information 25 | bytes SamplerAddress = 11; 26 | 27 | // Found inside packet 28 | uint64 TimeFlowStart = 38; 29 | uint64 TimeFlowEnd = 5; 30 | 31 | // Size of the sampled packet 32 | uint64 Bytes = 9; 33 | uint64 Packets = 10; 34 | 35 | // Source/destination addresses 36 | bytes SrcAddr = 6; 37 | bytes DstAddr = 7; 38 | 39 | // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...) 40 | uint32 Etype = 30; 41 | 42 | // Layer 4 protocol 43 | uint32 Proto = 20; 44 | 45 | // Ports for UDP and TCP 46 | uint32 SrcPort = 21; 47 | uint32 DstPort = 22; 48 | 49 | // Interfaces 50 | uint32 InIf = 18; 51 | uint32 OutIf = 19; 52 | 53 | // IP and TCP special flags 54 | uint32 IPTos = 23; 55 | uint32 ForwardingStatus = 24; 56 | uint32 IPTTL = 25; 57 | uint32 TCPFlags = 26; 58 | uint32 IcmpType = 31; 59 | uint32 IcmpCode = 32; 60 | uint32 IPv6FlowLabel = 37; 61 | 62 | // Autonomous system information 63 | uint32 SrcAS = 14; 64 | uint32 DstAS = 15; 65 | } 66 | --------------------------------------------------------------------------------