13 |
28 | """#
29 | mode: "html"
30 | }
31 | type: "text"
32 | title: ""
33 | }, {
34 | gridPos: {h: 6, w: 12, x: 0, y: 3}
35 | options: {
36 | maxItems: 0
37 | tags: ["netmeta"]
38 | }
39 | title: "NetMeta Dashboards"
40 | type: "dashlist"
41 | }, {
42 | gridPos: {h: 9, w: 12, x: 0, y: 9}
43 | options: {
44 | query: "Clickhouse -"
45 | }
46 | title: "Clickhouse Dashboards"
47 | type: "dashlist"
48 | }, {
49 | gridPos: {h: 15, w: 12, x: 12, y: 3}
50 | options: {
51 | feedUrl: "https://netmeta-cache.leoluk.de/v1/releases.atom"
52 | showImage: true
53 | }
54 | title: "NetMeta News"
55 | type: "news"
56 | }]
57 | }
58 |
--------------------------------------------------------------------------------
/cmd/portmirror/frame.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "net"
5 |
6 | "github.com/gopacket/gopacket"
7 | "github.com/gopacket/gopacket/layers"
8 | )
9 |
10 | func macToUint64(mac net.HardwareAddr) (u uint64) {
11 | return uint64(mac[0])<<40 | uint64(mac[1])<<32 | (uint64(mac[2])<<24 | uint64(mac[3])<<16 | uint64(mac[4])<<8 | uint64(mac[5]))
12 | }
13 |
14 | type frameInfo struct {
15 | EthernetType uint32
16 | SrcMAC net.HardwareAddr
17 | DstMAC net.HardwareAddr
18 |
19 | Protocol uint32
20 | SrcIP net.IP
21 | DstIP net.IP
22 | IPTOS uint8
23 | IPTTL uint8
24 | FlowLabel uint32
25 |
26 | TCPFlags uint32
27 | SrcPort uint32
28 | DstPort uint32
29 | SeqNum uint32
30 | }
31 |
32 | func readFrameInfo(packet gopacket.Packet) (info frameInfo) {
33 | for _, layer := range packet.Layers() {
34 | switch layer.(type) {
35 | case *layers.Ethernet:
36 | info.EthernetType = uint32(layer.(*layers.Ethernet).EthernetType)
37 | info.SrcMAC = layer.(*layers.Ethernet).SrcMAC
38 | info.DstMAC = layer.(*layers.Ethernet).DstMAC
39 |
40 | case *layers.IPv4:
41 | info.Protocol = uint32(layer.(*layers.IPv4).Protocol)
42 | info.SrcIP = layer.(*layers.IPv4).SrcIP
43 | info.DstIP = layer.(*layers.IPv4).DstIP
44 | info.IPTOS = layer.(*layers.IPv4).TOS
45 | info.IPTTL = layer.(*layers.IPv4).TTL
46 |
47 | case *layers.IPv6:
48 | info.Protocol = uint32(layer.(*layers.IPv6).NextHeader)
49 | info.SrcIP = layer.(*layers.IPv6).SrcIP
50 | info.DstIP = layer.(*layers.IPv6).DstIP
51 | info.FlowLabel = layer.(*layers.IPv6).FlowLabel
52 |
53 | case *layers.TCP:
54 | info.SrcPort = uint32(layer.(*layers.TCP).SrcPort)
55 | info.DstPort = uint32(layer.(*layers.TCP).DstPort)
56 | info.TCPFlags = uint32(layer.(*layers.TCP).Contents[13])
57 | info.SeqNum = layer.(*layers.TCP).Seq
58 |
59 | case *layers.UDP:
60 | info.SrcPort = uint32(layer.(*layers.UDP).SrcPort)
61 | info.DstPort = uint32(layer.(*layers.UDP).DstPort)
62 | }
63 | }
64 |
65 | return
66 | }
67 |
--------------------------------------------------------------------------------
/.aspect/bazelrc/performance.bazelrc:
--------------------------------------------------------------------------------
1 | # Don't apply `--noremote_upload_local_results` and `--noremote_accept_cached` to the disk cache.
2 | # If you have both `--noremote_upload_local_results` and `--disk_cache`, then this fixes a bug where
3 | # Bazel doesn't write to the local disk cache as it treats as a remote cache.
4 | # Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_remote_results_ignore_disk
5 | build --incompatible_remote_results_ignore_disk
6 |
7 | # Directories used by sandboxed non-worker execution may be reused to avoid unnecessary setup costs.
8 | # Save time on Sandbox creation and deletion when many of the same kind of action run during the
9 | # build.
10 | # No longer experimental in Bazel 6: https://github.com/bazelbuild/bazel/commit/c1a95501a5611878e5cc43a3cc531f2b9e47835b
11 | # Docs: https://bazel.build/reference/command-line-reference#flag--reuse_sandbox_directories
12 | build --experimental_reuse_sandbox_directories
13 |
14 | # Do not build runfiles symlink forests for external repositories under
15 | # `.runfiles/wsname/external/repo` (in addition to `.runfiles/repo`). This reduces runfiles &
16 | # sandbox creation times & prevents accidentally depending on this feature which may flip to off by
17 | # default in the future. Note, some rules may fail under this flag, please file issues with the rule
18 | # author.
19 | # Docs: https://bazel.build/reference/command-line-reference#flag--legacy_external_runfiles
20 | build --nolegacy_external_runfiles
21 |
22 | # Avoid creating a runfiles tree for binaries or tests until it is needed.
23 | # Docs: https://bazel.build/reference/command-line-reference#flag--build_runfile_links
24 | # See https://github.com/bazelbuild/bazel/issues/6627
25 | #
26 | # This may break local workflows that `build` a binary target, then run the resulting program
27 | # outside of `bazel run`. In those cases, the script will need to call
28 | # `bazel build --build_runfile_links //my/binary:target` and then execute the resulting program.
29 | build --nobuild_runfile_links
30 |
--------------------------------------------------------------------------------
/deploy/single-node/schema/0001_create_flows_raw.sql:
--------------------------------------------------------------------------------
1 | -- +goose Up
2 | CREATE TABLE IF NOT EXISTS flows_raw
3 | (
4 | -- Generated Date field for partitioning
5 | Date Date,
6 |
7 | -- Raw fields
8 |
9 | FlowType Enum8(
10 | 'FLOWUNKNOWN' = 0,
11 | 'SFLOW_5' = 1,
12 | 'NETFLOW_V5' = 2,
13 | 'NETFLOW_V9' = 3,
14 | 'IPFIX' = 4
15 | ),
16 |
17 | SequenceNum UInt64,
18 |
19 | TimeReceived UInt64,
20 | SamplingRate UInt64,
21 |
22 | FlowDirection UInt8,
23 |
24 | SamplerAddress IPv6,
25 |
26 | TimeFlowStart UInt64,
27 | TimeFlowEnd UInt64,
28 |
29 | Bytes UInt64,
30 | Packets UInt64,
31 |
32 | SrcAddr IPv6,
33 | DstAddr IPv6,
34 |
35 | EType UInt16,
36 |
37 | Proto UInt8,
38 |
39 | SrcPort UInt32,
40 | DstPort UInt32,
41 |
42 | InIf UInt32,
43 | OutIf UInt32,
44 |
45 | SrcMac UInt64,
46 | DstMac UInt64,
47 |
48 | SrcVlan UInt32,
49 | DstVlan UInt32,
50 | VlanId UInt32,
51 |
52 | IngressVrfId UInt32,
53 | EgressVrfId UInt32,
54 |
55 | IPTos UInt8,
56 | ForwardingStatus UInt8,
57 | IPTTL UInt8,
58 | TCPFlags UInt8,
59 | IcmpType UInt8,
60 | IcmpCode UInt8,
61 | IPv6FlowLabel UInt32,
62 |
63 | FragmentId UInt32,
64 | FragmentOffset UInt32,
65 |
66 | BiFlowDirection UInt8,
67 |
68 | SrcAS UInt32,
69 | DstAS UInt32,
70 |
71 | NextHop IPv6,
72 | NextHopAS UInt32,
73 |
74 | SrcNet UInt8,
75 | DstNet UInt8
76 | ) ENGINE = MergeTree()
77 | PARTITION BY Date
78 | ORDER BY (TimeReceived, FlowDirection, SamplerAddress, SrcAS, DstAS, SrcAddr, DstAddr)
79 | TTL Date + INTERVAL 1 WEEK;
80 |
81 | -- +goose Down
82 | DROP TABLE IF EXISTS flows_raw;
--------------------------------------------------------------------------------
/cue.mod/gen/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_go_gen.cue:
--------------------------------------------------------------------------------
1 | // Code generated by cue get go. DO NOT EDIT.
2 |
3 | //cue:generate cue get go k8s.io/apimachinery/pkg/apis/meta/v1
4 |
5 | package v1
6 |
7 | // GroupResource specifies a Group and a Resource, but does not force a version. This is useful for identifying
8 | // concepts during lookup stages without having partially valid types
9 | //
10 | // +protobuf.options.(gogoproto.goproto_stringer)=false
11 | #GroupResource: {
12 | group: string @go(Group) @protobuf(1,bytes,opt)
13 | resource: string @go(Resource) @protobuf(2,bytes,opt)
14 | }
15 |
16 | // GroupVersionResource unambiguously identifies a resource. It doesn't anonymously include GroupVersion
17 | // to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling
18 | //
19 | // +protobuf.options.(gogoproto.goproto_stringer)=false
20 | #GroupVersionResource: {
21 | group: string @go(Group) @protobuf(1,bytes,opt)
22 | version: string @go(Version) @protobuf(2,bytes,opt)
23 | resource: string @go(Resource) @protobuf(3,bytes,opt)
24 | }
25 |
26 | // GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying
27 | // concepts during lookup stages without having partially valid types
28 | //
29 | // +protobuf.options.(gogoproto.goproto_stringer)=false
30 | #GroupKind: {
31 | group: string @go(Group) @protobuf(1,bytes,opt)
32 | kind: string @go(Kind) @protobuf(2,bytes,opt)
33 | }
34 |
35 | // GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion
36 | // to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling
37 | //
38 | // +protobuf.options.(gogoproto.goproto_stringer)=false
39 | #GroupVersionKind: {
40 | group: string @go(Group) @protobuf(1,bytes,opt)
41 | version: string @go(Version) @protobuf(2,bytes,opt)
42 | kind: string @go(Kind) @protobuf(3,bytes,opt)
43 | }
44 |
45 | // GroupVersion contains the "group" and the "version", which uniquely identifies the API.
46 | //
47 | // +protobuf.options.(gogoproto.goproto_stringer)=false
48 | #GroupVersion: _
49 |
--------------------------------------------------------------------------------
/deploy/single-node/k8s/kafka/kafka.cue:
--------------------------------------------------------------------------------
1 | package kafka
2 |
3 | #Config: {
4 | goflowTopicRetention: int
5 | enableExternalKafkaListener: bool
6 | advertisedKafkaHost: string
7 | fastNetMon?: {
8 | topicRetention: int
9 | ...
10 | }
11 | }
12 |
13 | KafkaTopic: "flow-messages": {
14 | metadata: labels: "strimzi.io/cluster": "netmeta"
15 | spec: {
16 | partitions: 1
17 | replicas: 1
18 | config: "retention.bytes": "\(#Config.goflowTopicRetention)"
19 | }
20 | }
21 |
22 | if #Config.fastNetMon.topicRetention != _|_ {
23 | KafkaTopic: "fastnetmon": {
24 | metadata: labels: "strimzi.io/cluster": "netmeta"
25 | spec: {
26 | partitions: 1
27 | replicas: 1
28 | config: "retention.bytes": "\(#Config.fastNetMon.topicRetention)"
29 | }
30 | }
31 | }
32 |
33 | Kafka: "netmeta": spec: {
34 | kafka: {
35 | version: "3.2.3"
36 | replicas: 1
37 | listeners: [{
38 | name: "plain"
39 | port: 9092
40 | type: "internal"
41 | tls: false
42 | }, {
43 | name: "tls"
44 | port: 9093
45 | type: "internal"
46 | tls: true
47 | authentication: type: "tls"
48 | },
49 | if #Config.enableExternalKafkaListener {
50 | {
51 | name: "external"
52 | port: 9094
53 | type: "nodeport"
54 | tls: false
55 | configuration: bootstrap: nodePort: 32100
56 | configuration: brokers: [{broker: 0, advertisedHost: #Config.advertisedKafkaHost}]
57 | }
58 | },
59 | ]
60 | config: {
61 | "offsets.topic.replication.factor": 1
62 | "transaction.state.log.replication.factor": 1
63 | "transaction.state.log.min.isr": 1
64 | "log.message.format.version": "2.4"
65 | }
66 | storage: {
67 | type: "jbod"
68 | volumes: [{
69 | id: 0
70 | type: "persistent-claim"
71 | size: "100Gi"
72 | deleteClaim: false
73 | }]
74 | }
75 | jvmOptions: javaSystemProperties: [{name: "log4j2.formatMsgNoLookups", value: "true"}]
76 | }
77 | zookeeper: {
78 | replicas: 1
79 | storage: {
80 | type: "persistent-claim"
81 | size: "100Gi"
82 | deleteClaim: false
83 | }
84 | jvmOptions: javaSystemProperties: [{name: "log4j2.formatMsgNoLookups", value: "true"}]
85 | }
86 | entityOperator: {
87 | topicOperator: {
88 | jvmOptions: javaSystemProperties: [{name: "log4j2.formatMsgNoLookups", value: "true"}]
89 | }
90 | userOperator: {
91 | jvmOptions: javaSystemProperties: [{name: "log4j2.formatMsgNoLookups", value: "true"}]
92 | }
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/deploy/single-node/schema/traffic_data.proto:
--------------------------------------------------------------------------------
1 | // FastNetMon traffic format
2 | // https://github.com/pavel-odintsov/fastnetmon/blob/master/src/traffic_output_formats/protobuf/traffic_data.proto
3 | syntax = "proto3";
4 |
5 | enum TrafficDirection {
6 | // Value is not set
7 | TRAFFIC_DIRECTION_UNKNOWN = 0;
8 |
9 | // Traffic is coming to our address space
10 | TRAFFIC_DIRECTION_INCOMING = 1;
11 |
12 | // Traffic is coming from our address space
13 | TRAFFIC_DIRECTION_OUTGOING = 2;
14 |
15 | // Traffic where both source and destination IPs do not belong to our address space or non IP traffic (for example ARP)
16 | TRAFFIC_DIRECTION_OTHER = 3;
17 |
18 | // Traffic is going from our address space to our address space
19 | TRAFFIC_DIRECTION_INTERNAL = 4;
20 | };
21 |
22 | enum TelemetryType {
23 | TELEMETRY_TYPE_UNKNOWN = 0;
24 | TELEMETRY_TYPE_MIRROR = 1;
25 | TELEMETRY_TYPE_SFLOW = 2;
26 | TELEMETRY_TYPE_NETFLOW = 3;
27 | TELEMETRY_TYPE_TERA_FLOW = 4;
28 | }
29 |
30 | // Our unified flow - packet message
31 | message TrafficData {
32 | // Timestamp in seconds
33 | uint64 timestamp_seconds = 1;
34 |
35 | // Timestamp in milliseconds
36 | uint64 timestamp_milliseconds = 2;
37 |
38 | // Type of plugin which received traffic
39 | TelemetryType telemetry_type = 3;
40 |
41 | // IP protocol version: 4 or 6
42 | uint32 ip_version = 4;
43 |
44 | TrafficDirection traffic_direction = 5;
45 |
46 | // Sampling ratio
47 | uint64 sampling_ratio = 6;
48 |
49 | // Protocol field from IP packet
50 | uint32 protocol = 7;
51 |
52 | // Source and destination IPs for IPv4 (4 bytes) and IPv6 (16 bytes)
53 | bytes source_ip = 8;
54 | bytes destination_ip = 9;
55 |
56 | // Ports for UDP and TCP protocols
57 | uint32 source_port = 10;
58 | uint32 destination_port = 11;
59 |
60 | // Number of transferred packets
61 | uint64 packets = 12;
62 |
63 | // Total length in bytes for transferred packets
64 | uint64 octets = 13;
65 |
66 | // TTL for IPv4 or Hop Limit for IPv6
67 | uint32 ttl = 14;
68 |
69 | // TCP flags encoded in bit set
70 | uint32 tcp_flags = 15;
71 |
72 | bool ip_fragmented = 16;
73 | bool ip_dont_fragment = 17;
74 |
75 | // Input and output interfaces
76 | uint64 input_interface = 18;
77 | uint64 output_interface = 19;
78 |
79 | // Autonomous system numbers
80 | uint32 source_asn = 20;
81 | uint32 destination_asn = 21;
82 |
83 | // IPv4 or IPv6 address of device which sent traffic data
84 | bytes agent_address = 22;
85 | }
86 |
--------------------------------------------------------------------------------
/third_party/grafana/defs.bzl:
--------------------------------------------------------------------------------
1 | load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load")
2 | load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
3 |
4 | # TODO(tim): Migrate this to a bazel extension
5 | _grafana_plugin_attrs = {
6 | "version": attr.string(
7 | mandatory = True,
8 | doc = "The plugin version",
9 | ),
10 | "plugin_name": attr.string(
11 | mandatory = True,
12 | doc = "The plugin name",
13 | ),
14 | }
15 |
16 | def _grafana_plugin_impl(ctx):
17 | ctx.download_and_extract(
18 | url = "https://grafana.com/api/plugins/%s/versions/%s/download?os=%s&arch=%s" % (ctx.attr.plugin_name, ctx.attr.version, os(ctx), arch(ctx)),
19 | type = "zip",
20 | )
21 |
22 | ctx.file("BUILD.bazel", """
23 | package(default_visibility = ["//visibility:public"])
24 |
25 | filegroup(
26 | name = "files",
27 | srcs = glob(["**"]),
28 | )
29 | """)
30 |
31 | grafana_plugin = repository_rule(
32 | attrs = _grafana_plugin_attrs,
33 | implementation = _grafana_plugin_impl,
34 | )
35 |
36 | def _grafana_impl(ctx):
37 | ctx.download_and_extract(
38 | url = "https://dl.grafana.com/oss/release/grafana-11.4.0.{OS}-{ARCH}.tar.gz".format(
39 | OS = os(ctx),
40 | ARCH = arch(ctx),
41 | ),
42 | stripPrefix = "grafana-v11.4.0",
43 | )
44 | ctx.file("BUILD.bazel", 'exports_files(["bin/grafana-server"])')
45 |
46 | grafana = repository_rule(
47 | implementation = _grafana_impl,
48 | )
49 |
50 | def os(ctx):
51 | os = ctx.os.name.lower()
52 | if os == "mac os x":
53 | return "darwin"
54 | return os
55 |
56 | def arch(ctx):
57 | arch = ctx.os.arch.lower()
58 | if arch == "aarch64":
59 | return "arm64"
60 | return arch
61 |
62 | def grafana_plugin_layer(name):
63 | pkg_tar(
64 | name = name,
65 | srcs = ["@{}//:files".format(name)],
66 | package_dir = "/var/lib/grafana/plugins/{}".format(name),
67 | )
68 |
69 | return ":{}".format(name)
70 |
71 | def grafana_image():
72 | oci_image(
73 | name = "grafana_image",
74 | base = "@grafana",
75 | tars = [
76 | grafana_plugin_layer("netsage-sankey-panel"),
77 | grafana_plugin_layer("grafana-clickhouse-datasource"),
78 | ],
79 | visibility = ["//visibility:public"],
80 | )
81 |
82 | oci_load(
83 | name = "grafana_load",
84 | image = ":grafana_image",
85 | repo_tags = ["github.com/monogon-dev/netmeta/grafana:latest"],
86 | visibility = ["//visibility:public"],
87 | )
88 |
--------------------------------------------------------------------------------
/cmd/reconciler/quote.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "strconv"
5 | "unicode/utf8"
6 | _ "unsafe"
7 | )
8 |
9 | // QuoteSingleQuote has the same functionality as strconv.Quote but uses single quotes
10 | func QuoteSingleQuote(s string) string {
11 | return quoteWith(s, '\'')
12 | }
13 |
14 | // The code followed below this comment is copied from strconv/quote.go
15 | // and slightly modified by removing unused features in our usage.
16 |
17 | const (
18 | lowerhex = "0123456789abcdef"
19 | )
20 |
21 | func quoteWith(s string, quote byte) string {
22 | return string(appendQuotedWith(make([]byte, 0, 3*len(s)/2), s, quote))
23 | }
24 |
25 | func appendQuotedWith(buf []byte, s string, quote byte) []byte {
26 | // Often called with big strings, so preallocate. If there's quoting,
27 | // this is conservative but still helps a lot.
28 | if cap(buf)-len(buf) < len(s) {
29 | nBuf := make([]byte, len(buf), len(buf)+1+len(s)+1)
30 | copy(nBuf, buf)
31 | buf = nBuf
32 | }
33 | buf = append(buf, quote)
34 | for width := 0; len(s) > 0; s = s[width:] {
35 | r := rune(s[0])
36 | width = 1
37 | if r >= utf8.RuneSelf {
38 | r, width = utf8.DecodeRuneInString(s)
39 | }
40 | if width == 1 && r == utf8.RuneError {
41 | buf = append(buf, `\x`...)
42 | buf = append(buf, lowerhex[s[0]>>4])
43 | buf = append(buf, lowerhex[s[0]&0xF])
44 | continue
45 | }
46 | buf = appendEscapedRune(buf, r, quote)
47 | }
48 | buf = append(buf, quote)
49 | return buf
50 | }
51 |
52 | func appendEscapedRune(buf []byte, r rune, quote byte) []byte {
53 | if r == rune(quote) || r == '\\' { // always backslashed
54 | buf = append(buf, '\\')
55 | buf = append(buf, byte(r))
56 | return buf
57 | }
58 | if strconv.IsPrint(r) {
59 | return utf8.AppendRune(buf, r)
60 | }
61 | switch r {
62 | case '\a':
63 | buf = append(buf, `\a`...)
64 | case '\b':
65 | buf = append(buf, `\b`...)
66 | case '\f':
67 | buf = append(buf, `\f`...)
68 | case '\n':
69 | buf = append(buf, `\n`...)
70 | case '\r':
71 | buf = append(buf, `\r`...)
72 | case '\t':
73 | buf = append(buf, `\t`...)
74 | case '\v':
75 | buf = append(buf, `\v`...)
76 | default:
77 | switch {
78 | case r < ' ' || r == 0x7f:
79 | buf = append(buf, `\x`...)
80 | buf = append(buf, lowerhex[byte(r)>>4])
81 | buf = append(buf, lowerhex[byte(r)&0xF])
82 | case !utf8.ValidRune(r):
83 | r = 0xFFFD
84 | fallthrough
85 | case r < 0x10000:
86 | buf = append(buf, `\u`...)
87 | for s := 12; s >= 0; s -= 4 {
88 | buf = append(buf, lowerhex[r>>uint(s)&0xF])
89 | }
90 | default:
91 | buf = append(buf, `\U`...)
92 | for s := 28; s >= 0; s -= 4 {
93 | buf = append(buf, lowerhex[r>>uint(s)&0xF])
94 | }
95 | }
96 | }
97 | return buf
98 | }
99 |
--------------------------------------------------------------------------------
/cmd/portmirror/portmirror.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "flag"
6 | "net"
7 | "os"
8 | "os/signal"
9 | "sync"
10 | "time"
11 |
12 | _ "github.com/netsampler/goflow2/format/protobuf"
13 | "github.com/netsampler/goflow2/transport"
14 | _ "github.com/netsampler/goflow2/transport/kafka"
15 | "github.com/sirupsen/logrus"
16 | )
17 |
18 | var (
19 | interfaces string
20 | sampleRate int
21 | logLevel string
22 | workerCount int
23 | fanoutBase int
24 | samplerAddressString string
25 | samplerAddress net.IP
26 | )
27 |
28 | func init() {
29 | flag.StringVar(&interfaces, "iface", "", "which interface to use in the following format: RX_NAME:TX_NAME,RX_NAME:TX_NAME")
30 | flag.IntVar(&sampleRate, "samplerate", 1000, "the samplerate to use")
31 | flag.StringVar(&logLevel, "loglevel", "info", "Log level")
32 | flag.IntVar(&workerCount, "workercount", 8, "Number of workers per interface")
33 | flag.IntVar(&fanoutBase, "fanoutBase", 42, "fanout group base id")
34 | flag.StringVar(&samplerAddressString, "sampler-address", "127.0.0.1", "The address the instance use as SamplerAddress")
35 | }
36 |
37 | func main() {
38 | flag.Parse()
39 |
40 | lvl, _ := logrus.ParseLevel(logLevel)
41 | logrus.SetLevel(lvl)
42 |
43 | samplerAddress = net.ParseIP(samplerAddressString)
44 | if samplerAddress == nil {
45 | logrus.Fatalf("invalid sampler-address provided: %q", samplerAddressString)
46 | }
47 |
48 | tapPairs := loadConfig()
49 |
50 | kafka, err := transport.FindTransport(context.Background(), "kafka")
51 | if err != nil {
52 | logrus.Fatal(err)
53 | }
54 |
55 | var startGroup, endGroup sync.WaitGroup
56 | startGroup.Add(workerCount * len(tapPairs) * 2)
57 | endGroup.Add(workerCount * len(tapPairs) * 2)
58 |
59 | ctx, cancelFunc := context.WithCancel(context.Background())
60 | for _, tp := range tapPairs {
61 | logrus.Infof("Starting workers on pair: RX: %q - TX: %q", tp.RX.name, tp.TX.name)
62 | for i := 0; i < workerCount; i++ {
63 | go tp.TX.Worker(ctx, &startGroup, &endGroup, kafka)
64 | }
65 |
66 | for i := 0; i < workerCount; i++ {
67 | go tp.RX.Worker(ctx, &startGroup, &endGroup, kafka)
68 | }
69 | }
70 | logrus.Infof("Waiting for workers to become ready...")
71 | startGroup.Wait()
72 | logrus.Infof("Lets go!")
73 |
74 | c := make(chan os.Signal, 1)
75 | signal.Notify(c, os.Interrupt)
76 |
77 | <-c
78 | logrus.Println("Got Interrupt. Exiting...")
79 | cancelFunc()
80 |
81 | logrus.Println("Waiting a maxiumum of 10 Seconds for goroutines to shutdown")
82 | timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
83 | go func() {
84 | endGroup.Wait()
85 | cancel()
86 | }()
87 |
88 | defer cancel()
89 | <-timeout.Done()
90 | }
91 |
--------------------------------------------------------------------------------
/cmd/reconciler/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "flag"
7 | "fmt"
8 | "github.com/ClickHouse/clickhouse-go/v2"
9 | "github.com/emicklei/proto"
10 | "github.com/huandu/go-sqlbuilder"
11 | "log"
12 | "os"
13 | "strings"
14 | "time"
15 | )
16 |
17 | var (
18 | cfg = flag.String("config", "config.json", "The config file to use")
19 | dbHost = mustEnv("DB_HOST")
20 | dbUser = mustEnv("DB_USER")
21 | dbPass = mustEnv("DB_PASS")
22 | )
23 |
24 | func loadConfig() (*Config, error) {
25 | cfgFile, err := os.Open(*cfg)
26 | if err != nil {
27 | return nil, err
28 | }
29 | defer cfgFile.Close()
30 |
31 | var cfg Config
32 | d := json.NewDecoder(cfgFile)
33 | d.DisallowUnknownFields()
34 | if err := d.Decode(&cfg); err != nil {
35 | return nil, err
36 | }
37 |
38 | return &cfg, nil
39 | }
40 |
41 | func mustEnv(name string) string {
42 | v, found := os.LookupEnv(name)
43 | if !found {
44 | log.Fatalf("missing env: %s", name)
45 | }
46 |
47 | return v
48 | }
49 |
50 | func main() {
51 | flag.Parse()
52 |
53 | cfg, err := loadConfig()
54 | if err != nil {
55 | log.Fatalf("loading config: %v", err)
56 | }
57 |
58 | conn, err := clickhouse.Open(&clickhouse.Options{
59 | Addr: []string{dbHost},
60 | Auth: clickhouse.Auth{
61 | Database: cfg.Database,
62 | Username: dbUser,
63 | Password: dbPass,
64 | },
65 | })
66 | if err != nil {
67 | log.Fatal(err)
68 | }
69 |
70 | if err := conn.Ping(context.Background()); err != nil {
71 | log.Fatal(err)
72 | }
73 |
74 | r := &Reconciler{
75 | conn: conn,
76 | cfg: cfg,
77 | }
78 |
79 | for {
80 | if err := r.Reconcile(); err != nil {
81 | log.Println(err)
82 | }
83 | log.Println("reconcile done. sleeping for some time")
84 | time.Sleep(1 * time.Minute)
85 | }
86 | }
87 |
88 | func loadTableSchema(schema string, builder *sqlbuilder.CreateTableBuilder) error {
89 | n := strings.SplitN(schema, ":", 2)
90 | if len(n) != 2 {
91 | return fmt.Errorf("invalid source table schema: %v", schema)
92 | }
93 |
94 | path, msgName := n[0], n[1]
95 | f, err := os.Open(path)
96 | if err != nil {
97 | return err
98 | }
99 | defer f.Close()
100 |
101 | parser := proto.NewParser(f)
102 | definition, err := parser.Parse()
103 | if err != nil {
104 | return err
105 | }
106 |
107 | cv := &clickhouseVisitor{
108 | enumTypes: make(map[string]map[string]int),
109 | builder: builder,
110 | }
111 |
112 | var msg *proto.Message
113 | proto.Walk(definition,
114 | proto.WithEnum(func(e *proto.Enum) {
115 | e.Accept(cv)
116 | }),
117 | proto.WithMessage(func(message *proto.Message) {
118 | if message.Name == msgName {
119 | msg = message
120 | }
121 | }))
122 | if msg == nil {
123 | return fmt.Errorf("can't find message inside %q: %v", path, msgName)
124 | }
125 |
126 | // we have to evaluate the message after the proto.Walk
127 | // else it's not guaranteed that all enums are discovered
128 | msg.Accept(cv)
129 |
130 | return nil
131 | }
132 |
--------------------------------------------------------------------------------
/cmd/reconciler/visitor.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "log"
6 | "math"
7 | "sort"
8 | "strings"
9 |
10 | "github.com/emicklei/proto"
11 | "github.com/huandu/go-sqlbuilder"
12 | )
13 |
14 | type clickhouseVisitor struct {
15 | proto.NoopVisitor
16 | enumTypes map[string]map[string]int
17 | builder *sqlbuilder.CreateTableBuilder
18 | }
19 |
20 | func (c *clickhouseVisitor) VisitMessage(m *proto.Message) {
21 | for _, v := range m.Elements {
22 | v.Accept(c)
23 | }
24 | }
25 |
26 | func (c *clickhouseVisitor) VisitNormalField(i *proto.NormalField) {
27 | columnName := i.Name
28 | columnType := c.ToColumnType(i.Type)
29 |
30 | for _, option := range i.Options {
31 | switch option.Name {
32 | case "(column_type)":
33 | columnType = option.Constant.Source
34 | case "(column_name)":
35 | columnName = option.Constant.Source
36 | case "(column_skip)":
37 | // just exit the visitor and ignore the rest of the logic
38 | return
39 | default:
40 | log.Printf("unknown option %q on %q! Skipping!", option.Name, i.Name)
41 | }
42 | }
43 |
44 | c.builder.Define(columnName, columnType)
45 | }
46 |
47 | func (c *clickhouseVisitor) VisitEnumField(i *proto.EnumField) {
48 | c.enumTypes[i.Parent.(*proto.Enum).Name][i.Name] = i.Integer
49 | }
50 |
51 | func (c *clickhouseVisitor) VisitEnum(e *proto.Enum) {
52 | c.enumTypes[e.Name] = make(map[string]int)
53 | for _, v := range e.Elements {
54 | v.Accept(c)
55 | }
56 | }
57 |
58 | type enumOption struct {
59 | value int
60 | name string
61 | }
62 |
63 | func (e enumOption) String() string {
64 | return fmt.Sprintf("%s = %d", QuoteSingleQuote(e.name), e.value)
65 | }
66 |
67 | func (c *clickhouseVisitor) ToColumnType(t string) string {
68 | if enum, found := c.enumTypes[t]; found {
69 | var enumOptions []enumOption
70 |
71 | for s, i := range enum {
72 | enumOptions = append(enumOptions, enumOption{
73 | value: i,
74 | name: s,
75 | })
76 | }
77 |
78 | size := int(math.Log2(float64(len(enumOptions))))
79 | switch {
80 | case size <= 8:
81 | size = 8
82 | case size <= 16:
83 | size = 16
84 | default:
85 | size = 0
86 | }
87 |
88 | t := fmt.Sprintf("Enum%d", size)
89 | if size == 0 {
90 | t = "Enum"
91 | }
92 |
93 | // Enums are sorted inside clickhouse
94 | sort.SliceStable(enumOptions, func(i, j int) bool {
95 | return enumOptions[i].value < enumOptions[j].value
96 | })
97 |
98 | var enumOptionStrings []string
99 | for _, option := range enumOptions {
100 | enumOptionStrings = append(enumOptionStrings, option.String())
101 | }
102 |
103 | return fmt.Sprintf("%s(%s)", t, strings.Join(enumOptionStrings, ", "))
104 | }
105 |
106 | switch t {
107 | case "int32":
108 | return "Int32"
109 | case "uint32":
110 | return "UInt32"
111 | case "int64":
112 | return "Int64"
113 | case "uint64":
114 | return "UInt64"
115 | case "bytes":
116 | return "String"
117 | case "bool":
118 | return "Bool"
119 | case "string":
120 | return "String"
121 | default:
122 | panic("type not implemented: " + t)
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/cue.mod/gen/k8s.io/api/core/v1/well_known_labels_go_gen.cue:
--------------------------------------------------------------------------------
1 | // Code generated by cue get go. DO NOT EDIT.
2 |
3 | //cue:generate cue get go k8s.io/api/core/v1
4 |
5 | package v1
6 |
7 | #LabelHostname: "kubernetes.io/hostname"
8 | #LabelTopologyZone: "topology.kubernetes.io/zone"
9 | #LabelTopologyRegion: "topology.kubernetes.io/region"
10 |
11 | // These label have been deprecated since 1.17, but will be supported for
12 | // the foreseeable future, to accommodate things like long-lived PVs that
13 | // use them. New users should prefer the "topology.kubernetes.io/*"
14 | // equivalents.
15 | #LabelFailureDomainBetaZone: "failure-domain.beta.kubernetes.io/zone"
16 | #LabelFailureDomainBetaRegion: "failure-domain.beta.kubernetes.io/region"
17 |
18 | // Retained for compat when vendored. Do not use these consts in new code.
19 | #LabelZoneFailureDomain: "failure-domain.beta.kubernetes.io/zone"
20 | #LabelZoneRegion: "failure-domain.beta.kubernetes.io/region"
21 | #LabelZoneFailureDomainStable: "topology.kubernetes.io/zone"
22 | #LabelZoneRegionStable: "topology.kubernetes.io/region"
23 | #LabelInstanceType: "beta.kubernetes.io/instance-type"
24 | #LabelInstanceTypeStable: "node.kubernetes.io/instance-type"
25 | #LabelOSStable: "kubernetes.io/os"
26 | #LabelArchStable: "kubernetes.io/arch"
27 |
28 | // LabelWindowsBuild is used on Windows nodes to specify the Windows build number starting with v1.17.0.
29 | // It's in the format MajorVersion.MinorVersion.BuildNumber (for ex: 10.0.17763)
30 | #LabelWindowsBuild: "node.kubernetes.io/windows-build"
31 |
32 | // LabelNamespaceSuffixKubelet is an allowed label namespace suffix kubelets can self-set ([*.]kubelet.kubernetes.io/*)
33 | #LabelNamespaceSuffixKubelet: "kubelet.kubernetes.io"
34 |
35 | // LabelNamespaceSuffixNode is an allowed label namespace suffix kubelets can self-set ([*.]node.kubernetes.io/*)
36 | #LabelNamespaceSuffixNode: "node.kubernetes.io"
37 |
38 | // LabelNamespaceNodeRestriction is a forbidden label namespace that kubelets may not self-set when the NodeRestriction admission plugin is enabled
39 | #LabelNamespaceNodeRestriction: "node-restriction.kubernetes.io"
40 |
41 | // IsHeadlessService is added by Controller to an Endpoint denoting if its parent
42 | // Service is Headless. The existence of this label can be used further by other
43 | // controllers and kube-proxy to check if the Endpoint objects should be replicated when
44 | // using Headless Services
45 | #IsHeadlessService: "service.kubernetes.io/headless"
46 |
47 | // LabelNodeExcludeBalancers specifies that the node should not be considered as a target
48 | // for external load-balancers which use nodes as a second hop (e.g. many cloud LBs which only
49 | // understand nodes). For services that use externalTrafficPolicy=Local, this may mean that
50 | // any backends on excluded nodes are not reachable by those external load-balancers.
51 | // Implementations of this exclusion may vary based on provider.
52 | #LabelNodeExcludeBalancers: "node.kubernetes.io/exclude-from-external-load-balancers"
53 |
54 | // LabelMetadataName is the label name which, in-tree, is used to automatically label namespaces, so they can be selected easily by tools which require definitive labels
55 | #LabelMetadataName: "kubernetes.io/metadata.name"
56 |
--------------------------------------------------------------------------------
/deploy/single-node/k8s/traefik/deployment.cue:
--------------------------------------------------------------------------------
1 | package traefik
2 |
3 | import (
4 | "list"
5 | )
6 |
7 | #Config: {
8 | ports: {
9 | http: int
10 | https: int
11 | clickhouse: int
12 | }
13 |
14 | letsencryptMode: "staging" | "production" | "off"
15 | letsencryptAccountMail: string
16 | enableClickhouseIngress: bool
17 | }
18 |
19 | ServiceAccount: "traefik-ingress-controller": {}
20 |
21 | PersistentVolumeClaim: "traefik-data": {}
22 |
23 | TLSOption: default: spec: {
24 | minVersion: "VersionTLS12"
25 | cipherSuites: [
26 | "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
27 | "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
28 | "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
29 | "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
30 | "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
31 | "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
32 | ]
33 | }
34 |
35 | Deployment: traefik: {
36 | metadata: labels: app: "traefik"
37 | spec: {
38 | replicas: 1
39 | strategy: type: "Recreate"
40 | selector: matchLabels: app: "traefik"
41 | template: {
42 | metadata: labels: app: "traefik"
43 | spec: {
44 | // The poor-man's LoadBalancer implementation in k3s does not support IPv6 and is limited to legacy IPv4.
45 | // Bypass this inconvenience by running traefik in the host network namespace.
46 | hostNetwork: true
47 | serviceAccountName: "traefik-ingress-controller"
48 | containers: [{
49 | name: "traefik"
50 | image: "docker.io/traefik:v2.9.1@sha256:3eaf50b7874f63567ee5d18498536928faef199302c805cb6b766e06b649302d"
51 |
52 | let _args = [
53 | [
54 | "--accesslog",
55 | "--entrypoints.web.Address=:\(#Config.ports.http)",
56 | "--entrypoints.websecure.Address=:\(#Config.ports.https)",
57 | "--providers.kubernetescrd",
58 | ],
59 | if #Config.letsencryptMode != "off" {
60 | [
61 | "--certificatesresolvers.publicHostnameResolver.acme.tlschallenge",
62 | "--certificatesresolvers.publicHostnameResolver.acme.email=\(#Config.letsencryptAccountMail)",
63 | "--certificatesresolvers.publicHostnameResolver.acme.storage=/data/acme-\(#Config.letsencryptMode).json",
64 | ]
65 | },
66 | if #Config.letsencryptMode == "staging" {
67 | [
68 | "--certificatesresolvers.publicHostnameResolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory",
69 | ]
70 | },
71 | if #Config.enableClickhouseIngress {
72 | [
73 | "--entrypoints.clickhouse.Address=:\(#Config.ports.clickhouse)",
74 | ]
75 | },
76 | ]
77 |
78 | args: list.FlattenN(_args, 1)
79 |
80 | ports: [{
81 | name: "web"
82 | containerPort: #Config.ports.http
83 | protocol: "TCP"
84 | }, {
85 | name: "websecure"
86 | containerPort: #Config.ports.https
87 | protocol: "TCP"
88 | }, {
89 | name: "clickhouse"
90 | containerPort: #Config.ports.clickhouse
91 | protocol: "TCP"
92 | }]
93 |
94 | volumeMounts: [{
95 | mountPath: "/data"
96 | name: "traefik-data"
97 | }]
98 | }]
99 | restartPolicy: "Always"
100 | volumes: [{
101 | name: "traefik-data"
102 | persistentVolumeClaim: claimName: "traefik-data"
103 | }]
104 | }
105 | }
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/cue.mod/gen/k8s.io/apimachinery/pkg/runtime/types_go_gen.cue:
--------------------------------------------------------------------------------
1 | // Code generated by cue get go. DO NOT EDIT.
2 |
3 | //cue:generate cue get go k8s.io/apimachinery/pkg/runtime
4 |
5 | package runtime
6 |
7 | // TypeMeta is shared by all top level objects. The proper way to use it is to inline it in your type,
8 | // like this:
9 | //
10 | // type MyAwesomeAPIObject struct {
11 | // runtime.TypeMeta `json:",inline"`
12 | // ... // other fields
13 | // }
14 | //
15 | // func (obj *MyAwesomeAPIObject) SetGroupVersionKind(gvk *metav1.GroupVersionKind) { metav1.UpdateTypeMeta(obj,gvk) }; GroupVersionKind() *GroupVersionKind
16 | //
17 | // TypeMeta is provided here for convenience. You may use it directly from this package or define
18 | // your own with the same fields.
19 | //
20 | // +k8s:deepcopy-gen=false
21 | // +protobuf=true
22 | // +k8s:openapi-gen=true
23 | #TypeMeta: {
24 | // +optional
25 | apiVersion?: string @go(APIVersion) @protobuf(1,bytes,opt)
26 |
27 | // +optional
28 | kind?: string @go(Kind) @protobuf(2,bytes,opt)
29 | }
30 |
31 | #ContentTypeJSON: "application/json"
32 | #ContentTypeYAML: "application/yaml"
33 | #ContentTypeProtobuf: "application/vnd.kubernetes.protobuf"
34 |
35 | // RawExtension is used to hold extensions in external versions.
36 | //
37 | // To use this, make a field which has RawExtension as its type in your external, versioned
38 | // struct, and Object in your internal struct. You also need to register your
39 | // various plugin types.
40 | //
41 | // // Internal package:
42 | //
43 | // type MyAPIObject struct {
44 | // runtime.TypeMeta `json:",inline"`
45 | // MyPlugin runtime.Object `json:"myPlugin"`
46 | // }
47 | //
48 | // type PluginA struct {
49 | // AOption string `json:"aOption"`
50 | // }
51 | //
52 | // // External package:
53 | //
54 | // type MyAPIObject struct {
55 | // runtime.TypeMeta `json:",inline"`
56 | // MyPlugin runtime.RawExtension `json:"myPlugin"`
57 | // }
58 | //
59 | // type PluginA struct {
60 | // AOption string `json:"aOption"`
61 | // }
62 | //
63 | // // On the wire, the JSON will look something like this:
64 | //
65 | // {
66 | // "kind":"MyAPIObject",
67 | // "apiVersion":"v1",
68 | // "myPlugin": {
69 | // "kind":"PluginA",
70 | // "aOption":"foo",
71 | // },
72 | // }
73 | //
74 | // So what happens? Decode first uses json or yaml to unmarshal the serialized data into
75 | // your external MyAPIObject. That causes the raw JSON to be stored, but not unpacked.
76 | // The next step is to copy (using pkg/conversion) into the internal struct. The runtime
77 | // package's DefaultScheme has conversion functions installed which will unpack the
78 | // JSON stored in RawExtension, turning it into the correct object type, and storing it
79 | // in the Object. (TODO: In the case where the object is of an unknown type, a
80 | // runtime.Unknown object will be created and stored.)
81 | //
82 | // +k8s:deepcopy-gen=true
83 | // +protobuf=true
84 | // +k8s:openapi-gen=true
85 | #RawExtension: _
86 |
87 | // Unknown allows api objects with unknown types to be passed-through. This can be used
88 | // to deal with the API objects from a plug-in. Unknown objects still have functioning
89 | // TypeMeta features-- kind, version, etc.
90 | // TODO: Make this object have easy access to field based accessors and settors for
91 | // metadata and field mutatation.
92 | //
93 | // +k8s:deepcopy-gen=true
94 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
95 | // +protobuf=true
96 | // +k8s:openapi-gen=true
97 | #Unknown: _
98 |
--------------------------------------------------------------------------------
/UPGRADE.md:
--------------------------------------------------------------------------------
1 | # Upgrade guidance
2 | ## Upgrade from stable to main
3 |
4 | Upgrading from the stable branch to the main branch, or upgrading on the main branch from a commit before Oct 20, 2022, requires a manual migration with downtime.
5 |
6 | ```
7 | # Be sure to have apparmor-utils installed (apparmor_parser is required by k3s)
8 |
9 | ./install.sh
10 |
11 | ./scripts/build_containers.sh
12 | cd deploy/single-node/
13 |
14 | # Edit your config_local.cue
15 | # Replace clickhouseOperatorPassword with clickhouseAdminPassword
16 |
17 | kubectl delete crd \
18 | kafkaconnectors.kafka.strimzi.io \
19 | kafkaconnectors.kafka.strimzi.io \
20 | kafkaconnects.kafka.strimzi.io \
21 | kafkamirrormaker2s.kafka.strimzi.io \
22 | kafkas.kafka.strimzi.io \
23 | kafkamirrormakers.kafka.strimzi.io \
24 | kafkabridges.kafka.strimzi.io
25 |
26 | kubectl patch clickhouseinstallation netmeta --type json -p='[{"op": "remove", "path": "/spec/templates/podTemplates/0/podDistribution"}]'
27 | kubectl patch clickhouseinstallation netmeta --type json -p='[{"op": "remove", "path": "/spec/templates/podTemplates/0/zone/values"}]'
28 | kubectl patch clickhouseinstallation netmeta --type json -p='[{"op": "remove", "path": "/status/errors"}]'
29 |
30 | cue apply
31 |
32 | # Wait for pods to rollout
33 | kubectl get pods -w
34 |
35 | kubectl exec -i chi-netmeta-netmeta-0-0-0 -c clickhouse -- clickhouse-client -q 'DROP TABLE flows_queue'
36 | kubectl exec -i chi-netmeta-netmeta-0-0-0 -c clickhouse -- clickhouse-client -q 'DROP VIEW flows_raw_view'
37 |
38 | GOOSE_DRIVER=clickhouse \
39 | GOOSE_DB_USER=admin \
40 | GOOSE_DB_PASS=$(cue export -e netmeta.config.clickhouseAdminPassword --out text) \
41 | GOOSE_DB_ADDR=$(kubectl get pod chi-netmeta-netmeta-0-0-0 --template '{{.status.podIP}}') \
42 | GOOSE_DBSTRING="tcp://$GOOSE_DB_USER:$GOOSE_DB_PASS@$GOOSE_DB_ADDR:9000/default" \
43 | GOOSE_MIGRATION_DIR="schema/" \
44 | goose up
45 |
46 | # If you access NetMeta from external infrastructure (e.g. your own Grafana), you have to change the username from `clickhouse_operator` to `admin`
47 | ```
48 |
49 | ## Migrate to IPv4-mapped IPv6 addresses
50 | This can take a long time...
51 |
52 | ```clickhouse
53 | INSERT INTO flows_raw
54 | WITH
55 | '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' AS IPv6v4NullPadding,
56 | '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' AS IPv6Null,
57 | '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff' AS IPv6v4RFCPadding
58 | SELECT * REPLACE (
59 | if(startsWith(SamplerAddress, IPv6v4NullPadding),
60 | reinterpret(toFixedString(IPv6v4RFCPadding || substr(SamplerAddress, 13, 16), 16), 'IPv6'), SamplerAddress) AS SamplerAddress,
61 | if(startsWith(SrcAddr, IPv6v4NullPadding),
62 | reinterpret(toFixedString(IPv6v4RFCPadding || substr(SrcAddr, 13, 16), 16), 'IPv6'), SrcAddr) AS SrcAddr,
63 | if(startsWith(DstAddr, IPv6v4NullPadding),
64 | reinterpret(toFixedString(IPv6v4RFCPadding || substr(DstAddr, 13, 16), 16), 'IPv6'), DstAddr) AS DstAddr,
65 | if(startsWith(NextHop, IPv6v4NullPadding) and NextHop != IPv6Null,
66 | reinterpret(toFixedString(IPv6v4RFCPadding || substr(NextHop, 13, 16), 16), 'IPv6'), NextHop) AS NextHop
67 | )
68 | FROM flows_raw;
69 |
70 | -- Delete data with the old non-compliant mapping
71 | ALTER TABLE flows_raw
72 | DELETE WHERE
73 | startsWith(SamplerAddress, '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') OR
74 | startsWith(SrcAddr, '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') OR
75 | startsWith(DstAddr, '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00');
76 | ```
--------------------------------------------------------------------------------
/cmd/reconciler/config.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "sort"
6 | "strings"
7 |
8 | "github.com/huandu/go-sqlbuilder"
9 | )
10 |
11 | // Function contains all information to create or benchmark a UDF inside Clickhouse
12 | type Function struct {
13 | Name string `json:"name"`
14 | Arguments []string `json:"arguments"`
15 | Query string `json:"query"`
16 | }
17 |
18 | func (f *Function) CreateQuery() string {
19 | return fmt.Sprintf("CREATE FUNCTION %s AS (%s) -> %s", f.Name, strings.Join(f.Arguments, ", "), f.Query)
20 | }
21 |
22 | func (f *Function) CreateOrReplaceQuery() string {
23 | return fmt.Sprintf("CREATE OR REPLACE FUNCTION %s AS (%s) -> %s", f.Name, strings.Join(f.Arguments, ", "), f.Query)
24 | }
25 |
26 | // MaterializedView contains all information to create a MaterializedView inside Clickhouse
27 | // It also allows to benchmark the select statement inside the MV
28 | type MaterializedView struct {
29 | Name string `json:"name"`
30 | To string `json:"to"`
31 | From string `json:"from"`
32 | Query string `json:"query"`
33 | }
34 |
35 | func (mv MaterializedView) DropQuery(database string) string {
36 | return fmt.Sprintf("DROP VIEW %s.%s", database, mv.Name)
37 | }
38 |
39 | func (mv MaterializedView) SelectQuery(database string) string {
40 | return strings.Replace(mv.Query, "%%from%%", database+"."+mv.From, 1)
41 | }
42 |
43 | func (mv MaterializedView) CreateQuery(database string) string {
44 | s := fmt.Sprintf("CREATE MATERIALIZED VIEW %s TO %s AS %s", mv.Name, mv.To, mv.Query)
45 | s = strings.Replace(s, "%%from%%", database+"."+mv.From, 1)
46 | return s
47 | }
48 |
49 | type Settings map[string]any
50 |
51 | func (s Settings) String() string {
52 | var settings []string
53 | for k, v := range s {
54 | switch v.(type) {
55 | case int:
56 | settings = append(settings, fmt.Sprintf("%s = %d", k, v))
57 | case float64:
58 | settings = append(settings, fmt.Sprintf("%s = %.0f", k, v))
59 | case string:
60 | settings = append(settings, fmt.Sprintf("%s = %s", k, QuoteSingleQuote(v.(string))))
61 | default:
62 | settings = append(settings, fmt.Sprintf("%s = %s", k, v))
63 | }
64 | }
65 |
66 | // clickhouse sorts the settings internally
67 | sort.Strings(settings)
68 |
69 | return strings.Join(settings, ", ")
70 | }
71 |
72 | // Table contains the Name, the Type, additional Settings and
73 | // a reference to a Message inside the Protobuf file
74 | type Table struct {
75 | Name string `json:"name"`
76 | Schema string `json:"schema"`
77 | Engine string `json:"engine"`
78 | Settings Settings `json:"settings"`
79 | }
80 |
81 | func (t Table) CreateQuery(database string) (string, error) {
82 | builder := sqlbuilder.
83 | CreateTable(database + "." + t.Name)
84 |
85 | if err := loadTableSchema(t.Schema, builder); err != nil {
86 | return "", err
87 | }
88 |
89 | builder.SQL("ENGINE = " + t.Engine)
90 |
91 | if len(t.Settings) != 0 {
92 | builder.SQL("SETTINGS " + t.Settings.String())
93 | }
94 |
95 | return builder.String(), nil
96 | }
97 |
98 | func (t Table) DropQuery(database string) string {
99 | return fmt.Sprintf("DROP TABLE %s.%s", database, t.Name)
100 | }
101 |
102 | type Config struct {
103 | Database string `json:"database"`
104 | Functions []Function `json:"functions"`
105 | MaterializedViews []MaterializedView `json:"materialized_views"`
106 | SourceTables []Table `json:"source_tables"`
107 | }
108 |
--------------------------------------------------------------------------------
/.aspect/bazelrc/ci.bazelrc:
--------------------------------------------------------------------------------
1 | # Set this flag to enable re-tries of failed tests on CI.
2 | # When any test target fails, try one or more times. This applies regardless of whether the "flaky"
3 | # tag appears on the target definition.
4 | # This is a tradeoff: legitimately failing tests will take longer to report,
5 | # but we can paper over flaky tests that pass most of the time.
6 | # The alternative is to mark every flaky test with the `flaky = True` attribute, but this requires
7 | # the buildcop to make frequent code edits.
8 | # Not recommended for local builds so that the flakiness is observed during development and thus
9 | # is more likely to get fixed.
10 | # Note that when passing after the first attempt, Bazel will give a special "FLAKY" status.
11 | # Docs: https://bazel.build/docs/user-manual#flaky-test-attempts
12 | test --flaky_test_attempts=2
13 |
14 | # Announce all announces command options read from the bazelrc file(s) when starting up at the
15 | # beginning of each Bazel invocation. This is very useful on CI to be able to inspect what Bazel rc
16 | # settings are being applied on each run.
17 | # Docs: https://bazel.build/docs/user-manual#announce-rc
18 | build --announce_rc
19 |
20 | # Add a timestamp to each message generated by Bazel specifying the time at which the message was
21 | # displayed.
22 | # Docs: https://bazel.build/docs/user-manual#show-timestamps
23 | build --show_timestamps
24 |
25 | # Only show progress every 60 seconds on CI.
26 | # We want to find a compromise between printing often enough to show that the build isn't stuck,
27 | # but not so often that we produce a long log file that requires a lot of scrolling.
28 | # https://bazel.build/reference/command-line-reference#flag--show_progress_rate_limit
29 | build --show_progress_rate_limit=60
30 |
31 | # Use cursor controls in screen output.
32 | # Docs: https://bazel.build/docs/user-manual#curses
33 | build --curses=yes
34 |
35 | # Use colors to highlight output on the screen. Set to `no` if your CI does not display colors.
36 | # Docs: https://bazel.build/docs/user-manual#color
37 | build --color=yes
38 |
39 | # The terminal width in columns. Configure this to override the default value based on what your CI system renders.
40 | # Docs: https://github.com/bazelbuild/bazel/blob/1af61b21df99edc2fc66939cdf14449c2661f873/src/main/java/com/google/devtools/build/lib/runtime/UiOptions.java#L151
41 | build --terminal_columns=143
42 |
43 | ######################################
44 | # Generic remote cache configuration #
45 | ######################################
46 |
47 | # Only download remote outputs of top level targets to the local machine.
48 | # Docs: https://bazel.build/reference/command-line-reference#flag--remote_download_toplevel
49 | build --remote_download_toplevel
50 |
51 | # The maximum amount of time to wait for remote execution and cache calls.
52 | # https://bazel.build/reference/command-line-reference#flag--remote_timeout
53 | build --remote_timeout=3600
54 |
55 | # Upload locally executed action results to the remote cache.
56 | # Docs: https://bazel.build/reference/command-line-reference#flag--remote_upload_local_results
57 | build --remote_upload_local_results
58 |
59 | # Fall back to standalone local execution strategy if remote execution fails. If the grpc remote
60 | # cache connection fails, it will fail the build, add this so it falls back to the local cache.
61 | # Docs: https://bazel.build/reference/command-line-reference#flag--remote_local_fallback
62 | build --remote_local_fallback
63 |
64 | # Fixes builds hanging on CI that get the TCP connection closed without sending RST packets.
65 | # Docs: https://bazel.build/reference/command-line-reference#flag--grpc_keepalive_time
66 | build --grpc_keepalive_time=30s
67 |
--------------------------------------------------------------------------------
/deploy/single-node/k8s/clickhouse/clickhouse.cue:
--------------------------------------------------------------------------------
1 | package clickhouse
2 |
3 | import (
4 | "crypto/sha256"
5 | "encoding/hex"
6 | )
7 |
8 | // A stripped down version of the #SamplerConfig found in deploy/single-node/config.cue
9 | #SamplerConfig: [string]: {
10 | device: string
11 | samplingRate: int
12 | anonymizeAddresses: bool
13 | description: string
14 | interface: [string]: {
15 | id: int
16 | description: string
17 | }
18 | vlan: [string]: {
19 | id: int
20 | description: string
21 | }
22 | host: [string]: {
23 | device: string
24 | description: string
25 | }
26 | ...
27 | }
28 |
29 | #UserData: {
30 | autnums: [string]: {
31 | asn: int
32 | name: string
33 | country: string
34 | }
35 | }
36 |
37 | #Config: {
38 | clickhouseAdminPassword: string
39 | clickhouseReadonlyPassword: string
40 | enableClickhouseIngress: bool | *false
41 | sampler: #SamplerConfig
42 | userData: #UserData
43 | }
44 |
45 | ClickHouseInstallation: netmeta: spec: {
46 | defaults: templates: {
47 | dataVolumeClaimTemplate: "local-pv"
48 | logVolumeClaimTemplate: "local-pv"
49 | serviceTemplate: "chi-service-internal"
50 | podTemplate: "clickhouse-static"
51 | }
52 | configuration: {
53 | settings: {
54 | format_schema_path: "/etc/clickhouse-server/config.d/"
55 | dictionaries_config: "config.d/*.conf"
56 | user_defined_executable_functions_config: "config.d/*_function.xml"
57 | http_port: 8123
58 | "access_control_improvements/settings_constraints_replace_previous": true
59 | }
60 | clusters: [
61 | {
62 | name: "netmeta"
63 | layout: {
64 | shardsCount: 1
65 | replicasCount: 1
66 | }
67 | },
68 | ]
69 | users: {
70 | "default/access_management": 1
71 | "admin/password_sha256_hex": hex.Encode(sha256.Sum256(#Config.clickhouseAdminPassword))
72 | "admin/networks/ip": "::/0"
73 | "readonly/password_sha256_hex": hex.Encode(sha256.Sum256(#Config.clickhouseReadonlyPassword))
74 | "readonly/networks/ip": "::/0"
75 | "readonly/profile": "readonly"
76 | }
77 | profiles: {
78 | "readonly/readonly": "1"
79 | "readonly/constraints/additional_table_filters/changeable_in_readonly": ""
80 | }
81 | files: [string]: string
82 | }
83 | templates: {
84 | volumeClaimTemplates: [{
85 | name: "local-pv"
86 | spec: {
87 | accessModes: [
88 | "ReadWriteOnce",
89 | ]
90 | resources: requests: storage: "100Gi"
91 | }
92 | }]
93 | serviceTemplates: [{
94 | name: "chi-service-internal"
95 | generateName: "clickhouse-{chi}"
96 | spec: ports: [
97 | {name: "http", port: 8123, targetPort: port, protocol: "TCP"},
98 | {name: "tcp", port: 9000, targetPort: port, protocol: "TCP"},
99 | ]
100 | }]
101 | podTemplates: [{
102 | name: "clickhouse-static"
103 | spec: {
104 | securityContext: {
105 | runAsUser: 101
106 | runAsGroup: 101
107 | fsGroup: 101
108 | }
109 | containers: [{
110 | name: "clickhouse"
111 | image: "docker.io/clickhouse/clickhouse-server:23.2.4.12-alpine@sha256:0a63183a4da2923868cc0e58285e404a4a60e1be657c5b7005e736e91cfd4da9"
112 | resources: {}
113 | }]
114 | }
115 | }]
116 | }
117 | }
118 |
119 | if #Config.enableClickhouseIngress {
120 | IngressRoute: "clickhouse-ingress": spec: {
121 | entryPoints: ["clickhouse"]
122 | routes: [
123 | {
124 | match: "PathPrefix(`/`)"
125 | kind: "Rule"
126 | services: [
127 | {
128 | name: "clickhouse-netmeta"
129 | port: "http"
130 | },
131 | ]
132 | },
133 | ]
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/deploy/single-node/schema/FlowMessage.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | package netmeta;
3 |
4 | // Adapter from/compatible with cloudflare/goflow.
5 |
6 | message FlowMessage {
7 | enum FlowType {
8 | FLOWUNKNOWN = 0;
9 | SFLOW_5 = 1;
10 | NETFLOW_V5 = 2;
11 | NETFLOW_V9 = 3;
12 | IPFIX = 4;
13 | }
14 | FlowType Type = 1 [(column_name) = "FlowType"];
15 |
16 | uint64 TimeReceived = 2;
17 | uint32 SequenceNum = 4;
18 | uint64 SamplingRate = 3;
19 |
20 | uint32 FlowDirection = 42 [(column_type) = "UInt8"];
21 |
22 | // Sampler information
23 | bytes SamplerAddress = 11 [(column_type) = "FixedString(16)"];
24 |
25 | // Found inside packet
26 | uint64 TimeFlowStart = 38;
27 | uint64 TimeFlowEnd = 5;
28 |
29 | // Size of the sampled packet
30 | uint64 Bytes = 9;
31 | uint64 Packets = 10;
32 |
33 | // Source/destination addresses
34 | bytes SrcAddr = 6 [(column_type) = "FixedString(16)"];
35 | bytes DstAddr = 7 [(column_type) = "FixedString(16)"];
36 |
37 | // Layer 3 protocol (IPv4/IPv6/ARP/MPLS...)
38 | uint32 Etype = 30 [(column_type) = "UInt16"];
39 |
40 | // Layer 4 protocol
41 | uint32 Proto = 20 [(column_type) = "UInt8"];
42 |
43 | // Ports for UDP and TCP
44 | uint32 SrcPort = 21;
45 | uint32 DstPort = 22;
46 |
47 | // Interfaces
48 | uint32 InIf = 18;
49 | uint32 OutIf = 19;
50 |
51 | // Ethernet information
52 | uint64 SrcMac = 27;
53 | uint64 DstMac = 28;
54 |
55 | // Vlan
56 | uint32 SrcVlan = 33;
57 | uint32 DstVlan = 34;
58 | // 802.1q VLAN in sampled packet
59 | uint32 VlanId = 29;
60 |
61 | // VRF
62 | uint32 IngressVrfID = 39;
63 | uint32 EgressVrfID = 40;
64 |
65 | // IP and TCP special flags
66 | uint32 IPTos = 23 [(column_type) = "UInt8"];
67 | uint32 ForwardingStatus = 24 [(column_type) = "UInt8"];
68 | uint32 IPTTL = 25 [(column_type) = "UInt8"];
69 | uint32 TCPFlags = 26 [(column_type) = "UInt8"];
70 | uint32 IcmpType = 31 [(column_type) = "UInt8"];
71 | uint32 IcmpCode = 32 [(column_type) = "UInt8"];
72 | uint32 IPv6FlowLabel = 37;
73 |
74 | // Fragments (IPv4/IPv6)
75 | uint32 FragmentId = 35;
76 | uint32 FragmentOffset = 36;
77 |
78 | uint32 BiFlowDirection = 41 [(column_type) = "UInt8"];
79 |
80 | // Autonomous system information
81 | uint32 SrcAS = 14;
82 | uint32 DstAS = 15;
83 |
84 | bytes NextHop = 12 [(column_type) = "FixedString(16)"];
85 | uint32 NextHopAS = 13;
86 |
87 | // Prefix size
88 | uint32 SrcNet = 16 [(column_type) = "UInt8"];
89 | uint32 DstNet = 17 [(column_type) = "UInt8"];
90 |
91 | // IP encapsulation information
92 | bool HasEncap = 43 [(column_skip) = true];
93 | bytes SrcAddrEncap = 44 [(column_skip) = true];
94 | bytes DstAddrEncap = 45 [(column_skip) = true];
95 | uint32 ProtoEncap = 46 [(column_skip) = true];
96 | uint32 EtypeEncap = 47 [(column_skip) = true];
97 |
98 | uint32 IPTosEncap = 48 [(column_skip) = true];
99 | uint32 IPTTLEncap = 49 [(column_skip) = true];
100 | uint32 IPv6FlowLabelEncap = 50 [(column_skip) = true];
101 | uint32 FragmentIdEncap = 51 [(column_skip) = true];
102 | uint32 FragmentOffsetEncap = 52 [(column_skip) = true];
103 |
104 | // MPLS information
105 | bool HasMPLS = 53 [(column_skip) = true];
106 | uint32 MPLSCount = 54 [(column_skip) = true];
107 | uint32 MPLS1TTL = 55 [(column_skip) = true]; // First TTL
108 | uint32 MPLS1Label = 56 [(column_skip) = true]; // First Label
109 | uint32 MPLS2TTL = 57 [(column_skip) = true]; // Second TTL
110 | uint32 MPLS2Label = 58 [(column_skip) = true]; // Second Label
111 | uint32 MPLS3TTL = 59 [(column_skip) = true]; // Third TTL
112 | uint32 MPLS3Label = 60 [(column_skip) = true]; // Third Label
113 | uint32 MPLSLastTTL = 61 [(column_skip) = true]; // Last TTL
114 | uint32 MPLSLastLabel = 62 [(column_skip) = true]; // Last Label
115 |
116 | // PPP information
117 | bool HasPPP = 63 [(column_skip) = true];
118 | uint32 PPPAddressControl = 64 [(column_skip) = true];
119 |
120 | // Custom fields: start after ID 1000:
121 | // uint32 MyCustomField = 1000;
122 | }
123 |
--------------------------------------------------------------------------------
/MODULE.bazel:
--------------------------------------------------------------------------------
1 | bazel_dep(name = "gazelle")
2 | single_version_override(
3 | module_name = "gazelle",
4 | patch_strip = 1,
5 | patches = [
6 | # Snatched from monogon-dev/monogon to support patches to
7 | # goflow for adding flowdirection.
8 | "//third_party/gazelle:add-prepatching.patch",
9 | ],
10 | version = "0.40.0",
11 | )
12 |
13 | bazel_dep(name = "rules_go")
14 | single_version_override(
15 | module_name = "rules_go",
16 | patch_strip = 1,
17 | patches = [
18 | # Kick out cgo support from stdlib as the zig toolchain
19 | # doesn't like darwin right now.
20 | "//third_party/rules_go:disable-cgo.patch",
21 | "//third_party/rules_go:rules_go_absolute_embedsrc.patch",
22 | ],
23 | version = "0.50.1",
24 | )
25 |
26 | bazel_dep(name = "rules_oci", version = "2.0.1")
27 | bazel_dep(name = "rules_pkg", version = "1.0.1")
28 | bazel_dep(name = "aspect_bazel_lib", version = "2.9.0")
29 | bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0")
30 | bazel_dep(name = "toolchains_protoc", version = "0.3.3")
31 | bazel_dep(name = "rules_multirun", version = "0.9.0")
32 | bazel_dep(name = "buildifier_prebuilt", version = "8.0.1")
33 |
34 | toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains")
35 | use_repo(toolchains, "zig_sdk")
36 |
37 | register_toolchains(
38 | "@zig_sdk//toolchain:linux_amd64_musl",
39 | "@zig_sdk//toolchain:linux_arm64_musl",
40 | "@zig_sdk//toolchain:windows_amd64",
41 | "@zig_sdk//toolchain:windows_arm64",
42 | "@zig_sdk//toolchain:darwin_amd64",
43 | "@zig_sdk//toolchain:darwin_arm64",
44 | )
45 |
46 | go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
47 | go_sdk.download(version = "1.23.1")
48 |
49 | go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
50 | go_deps.from_file(go_mod = "//:go.mod")
51 | use_repo(
52 | go_deps,
53 | "com_github_clickhouse_clickhouse_go_v2",
54 | "com_github_emicklei_proto",
55 | "com_github_gopacket_gopacket",
56 | "com_github_huandu_go_sqlbuilder",
57 | "com_github_netsampler_goflow2",
58 | "com_github_netsampler_goflow2_v2",
59 | "com_github_osrg_gobgp",
60 | "com_github_pressly_goose_v3",
61 | "com_github_sirupsen_logrus",
62 | "com_github_vishvananda_netlink",
63 | "io_k8s_klog_v2",
64 | "org_cuelang_go",
65 | )
66 |
67 | oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")
68 | oci.pull(
69 | name = "distroless_base",
70 | digest = "sha256:7a4bffcb07307d97aa731b50cb6ab22a68a8314426f4e4428335939b5b1943a5",
71 | image = "gcr.io/distroless/base",
72 | platforms = [
73 | "linux/amd64",
74 | "linux/arm/v7",
75 | "linux/arm64/v8",
76 | "linux/ppc64le",
77 | "linux/s390x",
78 | ],
79 | )
80 | oci.pull(
81 | name = "grafana",
82 | digest = "sha256:2a73ae33c9f0c51af6eced2ef185d5d3682b4c378c4fdd6941a14e8ea4a3e95b",
83 | image = "index.docker.io/grafana/grafana",
84 | platforms = [
85 | "linux/amd64",
86 | "linux/arm64/v8",
87 | "linux/arm/v7",
88 | ],
89 | )
90 | use_repo(
91 | oci,
92 | "distroless_base",
93 | "distroless_base_linux_amd64",
94 | "distroless_base_linux_arm64_v8",
95 | "distroless_base_linux_arm_v7",
96 | "distroless_base_linux_ppc64le",
97 | "distroless_base_linux_s390x",
98 | "grafana",
99 | "grafana_linux_amd64",
100 | "grafana_linux_arm64_v8",
101 | "grafana_linux_arm_v7",
102 | )
103 |
104 | grafana_plugin = use_repo_rule("//third_party/grafana:defs.bzl", "grafana_plugin")
105 |
106 | grafana_plugin(
107 | name = "netsage-sankey-panel",
108 | plugin_name = "netsage-sankey-panel",
109 | version = "1.1.3",
110 | )
111 |
112 | grafana_plugin(
113 | name = "grafana-clickhouse-datasource",
114 | plugin_name = "grafana-clickhouse-datasource",
115 | version = "4.5.0",
116 | )
117 |
118 | grafana = use_repo_rule("//third_party/grafana:defs.bzl", "grafana")
119 |
120 | grafana(
121 | name = "grafana_bin",
122 | )
123 |
--------------------------------------------------------------------------------
/cue.mod/gen/k8s.io/apimachinery/pkg/api/resource/quantity_go_gen.cue:
--------------------------------------------------------------------------------
1 | // Code generated by cue get go. DO NOT EDIT.
2 |
3 | //cue:generate cue get go k8s.io/apimachinery/pkg/api/resource
4 |
5 | package resource
6 |
7 | // Quantity is a fixed-point representation of a number.
8 | // It provides convenient marshaling/unmarshaling in JSON and YAML,
9 | // in addition to String() and AsInt64() accessors.
10 | //
11 | // The serialization format is:
12 | //
13 | // ```
14 | // Welcome to NetMeta
14 |
15 |
27 | Need help?
16 |
17 |
19 | Issue Tracker
20 |
21 |
23 | Contact Us
24 |
25 |
26 |
3 |
4 | 
49 | 
50 | 
51 |
52 | ## 💥 API Stability
53 |
54 | NetMeta is **beta software** and subject to change. It exposes the following APIs:
55 |
56 | * The cluster configuration file for single-node deployments.
57 | * ClickHouse SQL schema for raw database access.
58 | * Protobuf schemas for ingestion for writing custom processors.
59 |
60 | One NetMeta has stabilized, these APIs will be stable and backwards compatible.
61 |
62 | ## 🛠 Deployment
63 | ### [Single-node deployment](deploy/single-node/README.md)
64 |
65 | NetMeta includes a production-ready single node deployment that scales to up to ~100k events/s and billions of database
66 | rows. More infos can be found [here](deploy/single-node/README.md)
67 |
68 | Ingestion performance is limited by CPU performance and disk bandwidth.
69 | Query performance is limited by disk and memory bandwidth, as well as total amount of available memory for larger
70 | in-memory aggregations.
71 |
72 | Most reads/writes are sequential due to heavy use of batching in all parts of the stack,
73 | and it works fine even on network storage or spinning disks. We recommend local NVMe drives for best performance.
74 |
75 | NetMeta can scale to millions of events per seconds in multi-node deployments.
76 |
77 | ### Multi-node deployment
78 |
79 | We are currently finalizing the design for multi-node deployments. Please contact us if you're interested in
80 | large-scale deployments - we want your feedback!
81 |
82 | ### Monogon OS
83 |
84 | NetMeta will be a first-class citizen on [Monogon OS](https://monogon.tech/monogon_os.html) - stay tuned!
85 |
86 | ### ☸️ Kubernetes
87 |
88 | NetMeta works on any Kubernetes cluster that supports LoadBalancer and Ingress objects and can provision storage.
89 | It's up to you to carefully read the deployment code and cluster role assigments to make sure it works with your
90 | cluster.
91 | Note that we use two operators, which require cluster-admin permissions since CRDs are global
92 | ([Strimzi](https://strimzi.io/docs/master) for Kafka
93 | and [clickhouse-operator](https://github.com/Altinity/clickhouse-operator)).
94 |
95 | All pieces of NetMeta are installed into a single namespace. By default, this is ``default``, which is
96 | probably not what
97 | you want.
98 | You can change the target namespace in the deployment config.
99 |
100 | Please contact us if you need help porting NetMeta to an existing k8s cluster.
101 |
102 | ## 💼 Support
103 |
104 | [Please contact us](https://monogon.tech/pricing.html)
105 | for support and consulting. If you are using NetMeta in production, we'd love to hear from you!
106 |
107 | ## 🧩 Related
108 |
109 | NetMeta is powered by a number of great open source projects, we use:
110 |
111 | - [ClickHouse](https://clickhouse.tech) as the main database
112 | - [Kafka](https://kafka.apache.org) as a queue in front of ClickHouse
113 | - [Grafana](https://grafana.com/) with
114 | - [clickhouse-grafana](https://github.com/Vertamedia/clickhouse-grafana) as frontend
115 | - [goflow](https://github.com/cloudflare/goflow) as the sFlow/Netflow collector
116 | - [Strimzi](https://strimzi.io/) to deploy Kafka,
117 | - [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) to deploy ClickHouse, as well as
118 | - [Kubernetes](https://kubernetes.io/) and Rancher's [k3s](https://k3s.io/).
119 |
120 | ## 🏰 Architecture
121 |
122 | ```mermaid
123 | flowchart TD;
124 | sFlow --> goflow
125 | IPFIX --> goflow
126 | Netflow --> goflow
127 |
128 | kafka[Kafka Broker]
129 | clickhouse["ClickHouse ServerMergeTree(hourly partitions)"]
130 |
131 | goflow --> kafka
132 | ntm-agent --> kafka
133 | kafka --> clickhouse
134 | clickhouse --> |SQL| grafana[Grafana]
135 | asmap --> clickhouse
136 | nexthop --> clickhouse
137 | resolver --> clickhouse
138 |
139 | ```
140 |
141 | ---
142 |
143 | (C) 2022 [Monogon SE](https://monogon.tech).
144 |
145 | This software is provided "as-is" and
146 | without any express or implied warranties, including, without limitation, the implied warranties of
147 | merchantability and fitness for a particular purpose.
148 |
--------------------------------------------------------------------------------
/deploy/single-node/k8s/clickhouse/files.cue:
--------------------------------------------------------------------------------
1 | package clickhouse
2 |
3 | import (
4 | "strings"
5 | "strconv"
6 | "netmeta.monogon.tech/xml"
7 | )
8 |
9 | // template for TSV dictionaries
10 | _files: [NAME=string]: {
11 | cfg: {
12 | layout: _
13 | structure: _
14 | }
15 | data: _
16 |
17 | _cfg: yandex: dictionary: {
18 | cfg
19 |
20 | name: NAME
21 | source: [{
22 | file: {
23 | path: "/etc/clickhouse-server/config.d/\(NAME).tsv"
24 | format: "TSV"
25 | }
26 | settings: format_tsv_null_representation: "NULL"
27 | }]
28 | lifetime: 60
29 | }
30 | }
31 |
32 | // Iterate over all defined files in _files and generate the config files for clickhouse
33 | ClickHouseInstallation: netmeta: spec: configuration: files: {
34 | for k, v in _files {
35 | "\(k).conf": (xml.#Marshal & {in: v._cfg}).out
36 | "\(k).tsv": v.data
37 | }
38 | }
39 |
40 | // Dictionary for user-defined interface name lookup
41 | _files: InterfaceNames: {
42 | data: strings.Join([ for s in #Config.sampler for i in s.interface {
43 | strings.Join([s.device, "\(i.id)", i.description], "\t")
44 | }], "\n")
45 |
46 | cfg: {
47 | layout: complex_key_hashed: null
48 | structure: {
49 | key: [{
50 | attribute: {
51 | name: "Device"
52 | type: "String"
53 | }
54 | }, {
55 | attribute: {
56 | name: "Index"
57 | type: "UInt32"
58 | }
59 | }]
60 | attribute: {
61 | name: "Description"
62 | type: "String"
63 | null_value: null
64 | }
65 | }
66 | }
67 | }
68 |
69 | // Dictionary for user-defined sampler settings lookup
70 | _files: SamplerConfig: {
71 | data: strings.Join([ for s in #Config.sampler {
72 | let samplingRate = [
73 | if s.samplingRate == 0 {
74 | "NULL"
75 | },
76 | "\(s.samplingRate)",
77 | ][0]
78 |
79 | let description = [
80 | if s.description == "" {
81 | "NULL"
82 | },
83 | "\(s.description)",
84 | ][0]
85 |
86 | strings.Join([s.device, samplingRate, description, strconv.FormatBool(s.anonymizeAddresses)], "\t")
87 | }], "\n")
88 |
89 | cfg: {
90 | layout: complex_key_hashed: null
91 | structure: [{
92 | key: [{
93 | attribute: {
94 | name: "Device"
95 | type: "String"
96 | }
97 | }]
98 | }, {
99 | attribute: {
100 | name: "SamplingRate"
101 | type: "Nullable(UInt64)"
102 | null_value: null
103 | }
104 | }, {
105 | attribute: {
106 | name: "Description"
107 | type: "Nullable(String)"
108 | null_value: null
109 | }
110 | }, {
111 | attribute: {
112 | name: "AnonymizeAddresses"
113 | type: "Bool"
114 | null_value: false
115 | }
116 | }]
117 | }
118 | }
119 |
120 | // Dictionary for user-defined vlan name lookup
121 | _files: VlanNames: {
122 | data: strings.Join([ for s in #Config.sampler for v in s.vlan {
123 | strings.Join([s.device, "\(v.id)", v.description], "\t")
124 | }], "\n")
125 |
126 | cfg: {
127 | layout: complex_key_hashed: null
128 | structure: {
129 | key: [{
130 | attribute: {
131 | name: "Device"
132 | type: "String"
133 | }
134 | }, {
135 | attribute: {
136 | name: "Index"
137 | type: "UInt32"
138 | }
139 | }]
140 | attribute: {
141 | name: "Description"
142 | type: "String"
143 | null_value: null
144 | }
145 | }
146 | }
147 | }
148 |
149 | // Dictionary for user-defined host name lookup
150 | _files: HostNames: {
151 | data: strings.Join([ for s in #Config.sampler for h in s.host {
152 | strings.Join([s.device, h.device, h.description], "\t")
153 | }], "\n")
154 |
155 | cfg: {
156 | layout: complex_key_hashed: null
157 | structure: {
158 | key: [{
159 | attribute: {
160 | name: "Sampler"
161 | type: "String"
162 | }
163 | }, {
164 | attribute: {
165 | name: "Device"
166 | type: "String"
167 | }
168 | }]
169 | attribute: {
170 | name: "Description"
171 | type: "String"
172 | null_value: null
173 | }
174 | }
175 | }
176 | }
177 |
178 | _files: user_autnums: {
179 | data: strings.Join([ for _, e in #Config.userData.autnums {
180 | strings.Join(["\(e.asn)", e.name, e.country], "\t")
181 | }], "\n")
182 |
183 | cfg: {
184 | layout: flat: null
185 | structure: [{
186 | id: name: "asnum"
187 | }, {
188 | attribute: {
189 | name: "name"
190 | type: "String"
191 | null_value: null
192 | }
193 | }, {
194 | attribute: {
195 | name: "country"
196 | type: "String"
197 | null_value: null
198 | }
199 | }]
200 | }
201 | }
202 |
203 | ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo.conf": (xml.#Marshal & {in: {
204 | yandex: dictionary: {
205 | name: "risinfo"
206 | source: http: {
207 | url: "http://risinfo/rib.tsv"
208 | format: "TabSeparated"
209 | }
210 | lifetime: 3600
211 | layout: ip_trie: access_to_key_from_attributes: true
212 | structure: key: attribute: {
213 | name: "prefix"
214 | type: "String"
215 | }
216 | structure: attribute: {
217 | name: "asnum"
218 | type: "UInt32"
219 | null_value: 0
220 | }
221 | }
222 | }}).out
223 |
224 | ClickHouseInstallation: netmeta: spec: configuration: files: "autnums.conf": (xml.#Marshal & {in: {
225 | yandex: dictionary: {
226 | name: "autnums"
227 | source: clickhouse: {
228 | query:
229 | #"""
230 | SELECT * FROM dictionaries.risinfo_autnums
231 | UNION ALL
232 | SELECT * FROM dictionaries.user_autnums
233 | """#
234 | }
235 | lifetime: 3600
236 | layout: flat: null
237 | structure: [{
238 | id: name: "asnum"
239 | }, {
240 | attribute: {
241 | name: "name"
242 | type: "String"
243 | null_value: null
244 | }
245 | }, {
246 | attribute: {
247 | name: "country"
248 | type: "String"
249 | null_value: null
250 | }
251 | }]
252 | }
253 | }}).out
254 |
255 | ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo_autnums.conf": (xml.#Marshal & {in: {
256 | yandex: dictionary: {
257 | name: "risinfo_autnums"
258 | source: http: {
259 | url: "http://risinfo/autnums.tsv"
260 | format: "TabSeparated"
261 | }
262 | lifetime: 86400
263 | layout: flat: null
264 | structure: [{
265 | id: name: "asnum"
266 | }, {
267 | attribute: {
268 | name: "name"
269 | type: "String"
270 | null_value: null
271 | }
272 | }, {
273 | attribute: {
274 | name: "country"
275 | type: "String"
276 | null_value: null
277 | }
278 | }]
279 | }
280 | }}).out
281 |
282 | ClickHouseInstallation: netmeta: spec: configuration: files: "format_function.xml": (xml.#Marshal & {in: {
283 | yandex: functions: {
284 | type: "executable"
285 | name: "formatQuery"
286 | return_type: "String"
287 | argument: [{
288 | type: "String"
289 | name: "query"
290 | }]
291 | format: "LineAsString"
292 | command: "clickhouse format --oneline"
293 | execute_direct: "0"
294 | }
295 | }}).out
296 |
--------------------------------------------------------------------------------
/deploy/dashboards/NetMeta_Relations.cue:
--------------------------------------------------------------------------------
1 | package dashboards
2 |
3 | import "list"
4 |
5 | _asRelationQueries: {
6 | "Inbound traffic relations (Top 20)":
7 | #"""
8 | SELECT
9 | ASNToString(SrcAS) AS SrcASName,
10 | ASNToString(DstAS) AS DstASName,
11 | (sum(Bytes * SamplingRate) / 1024) as Bytes
12 | FROM flows_raw
13 | WHERE $__timeFilter(TimeReceived)
14 | \#(_filtersWithHost)
15 | AND isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
16 | GROUP BY SrcAS, DstAS
17 | ORDER BY Bytes DESC
18 | LIMIT 20
19 | """#
20 |
21 | "Outbound traffic relations (Top 20)":
22 | #"""
23 | SELECT
24 | ASNToString(DstAS) AS DstASName,
25 | ASNToString(SrcAS) AS SrcASName,
26 | (sum(Bytes * SamplingRate) / 1024) as Bytes
27 | FROM flows_raw
28 | WHERE $__timeFilter(TimeReceived)
29 | \#(_filtersWithHost)
30 | AND NOT isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
31 | GROUP BY SrcAS, DstAS
32 | ORDER BY Bytes DESC
33 | LIMIT 20
34 | """#
35 |
36 | "Inbound traffic relations via interface (Top 30)":
37 | #"""
38 | SELECT
39 | ASNToString(SrcAS) AS SrcASName,
40 | InterfaceToString(SamplerAddress, OutIf) AS OutIfName,
41 | ASNToString(DstAS) AS DstASName,
42 | (sum(Bytes * SamplingRate) / 1024) as Bytes
43 | FROM flows_raw
44 | WHERE $__timeFilter(TimeReceived)
45 | \#(_filtersWithHost)
46 | AND isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
47 | GROUP BY SrcAS, DstAS, SamplerAddress, OutIf
48 | ORDER BY Bytes DESC
49 | LIMIT 30
50 | """#
51 |
52 | "Outbound traffic relations via interface (Top 30)":
53 | #"""
54 | SELECT
55 | ASNToString(DstAS) AS DstASName,
56 | InterfaceToString(SamplerAddress, OutIf) AS OutIfName,
57 | ASNToString(SrcAS) AS SrcASName,
58 | (sum(Bytes * SamplingRate) / 1024) as Bytes
59 | FROM flows_raw
60 | WHERE $__timeFilter(TimeReceived)
61 | \#(_filtersWithHost)
62 | AND NOT isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
63 | GROUP BY SrcAS, DstAS, SamplerAddress, OutIf
64 | ORDER BY Bytes DESC
65 | LIMIT 30
66 | """#
67 | }
68 |
69 | _asRelations: [{
70 | title: "AS Relations"
71 | gridPos: y: 6
72 | type: "row"
73 | }, {
74 | title: "Inbound traffic relations (Top 20)"
75 | type: "netsage-sankey-panel"
76 | gridPos: {h: 24, w: 12, x: 0, y: 7}
77 | options: nodePadding: 6
78 | options: nodeWidth: 23
79 | options: iteration: 15
80 | targets: [{
81 | rawSql: _asRelationQueries[title]
82 | }]
83 | }, {
84 | title: "Outbound traffic relations (Top 20)"
85 | type: "netsage-sankey-panel"
86 | gridPos: {h: 24, w: 12, x: 12, y: 7}
87 | options: nodePadding: 6
88 | options: nodeWidth: 23
89 | options: iteration: 15
90 | targets: [{
91 | rawSql: _asRelationQueries[title]
92 | }]
93 | }, {
94 | title: "Inbound traffic relations via interface (Top 30)"
95 | type: "netsage-sankey-panel"
96 | gridPos: {h: 24, w: 12, x: 0, y: 31}
97 | options: nodePadding: 6
98 | options: nodeWidth: 23
99 | options: iteration: 15
100 | targets: [{
101 | rawSql: _asRelationQueries[title]
102 | }]
103 | }, {
104 | title: "Outbound traffic relations via interface (Top 30)"
105 | type: "netsage-sankey-panel"
106 | gridPos: {h: 24, w: 12, x: 12, y: 31}
107 | options: nodePadding: 6
108 | options: nodeWidth: 23
109 | options: iteration: 15
110 | targets: [{
111 | rawSql: _asRelationQueries[title]
112 | }]
113 | }]
114 |
115 | _topFlowSankeyQueries: {
116 | "Top 30 Flows (per IP)":
117 | #"""
118 | SELECT
119 | if($showHostnames, HostToString(SamplerAddress, SrcAddr), IPv6ToString(SrcAddr)) AS Src,
120 | if($showHostnames, HostToString(SamplerAddress, DstAddr), IPv6ToString(DstAddr)) AS Dst,
121 | (sum(Bytes * SamplingRate) / 1024) as Bytes
122 | FROM flows_raw
123 | WHERE $__timeFilter(TimeReceived)
124 | \#(_filtersWithHost)
125 | GROUP BY SamplerAddress, SrcAddr, DstAddr
126 | ORDER BY Bytes DESC
127 | LIMIT 30
128 | """#
129 |
130 | "Top 30 Flows (per IP+Port)":
131 | #"""
132 | SELECT
133 | if($showHostnames, HostToString(SamplerAddress, SrcAddr), IPv6ToString(SrcAddr)) || ' ' || dictGetString('IPProtocols', 'Name', toUInt64(Proto)) || toString(SrcPort) as Src,
134 | if($showHostnames, HostToString(SamplerAddress, DstAddr), IPv6ToString(DstAddr)) || ' ' || dictGetString('IPProtocols', 'Name', toUInt64(Proto)) || toString(DstPort) as Dst,
135 | (sum(Bytes * SamplingRate) / 1024) as Bytes
136 | FROM flows_raw
137 | WHERE $__timeFilter(TimeReceived)
138 | \#(_filtersWithHost)
139 | GROUP BY SamplerAddress, SrcAddr, SrcPort, Proto, DstAddr, DstPort
140 | ORDER BY Bytes DESC
141 | LIMIT 30
142 | """#
143 | }
144 |
145 | _topFlowSankey: [{
146 | title: "AS Relations"
147 | gridPos: y: 32
148 | type: "row"
149 | }, {
150 | title: "Top 30 Flows (per IP)"
151 | type: "netsage-sankey-panel"
152 | gridPos: {h: 24, w: 12, x: 0, y: 33}
153 | options: nodePadding: 6
154 | options: nodeWidth: 28
155 | targets: [{
156 | rawSql: _topFlowSankeyQueries[title]
157 | }]
158 | }, {
159 | title: "Top 30 Flows (per IP+Port)"
160 | type: "netsage-sankey-panel"
161 | gridPos: {h: 24, w: 12, x: 12, y: 33}
162 | options: nodePadding: 11
163 | options: nodeWidth: 28
164 | targets: [{
165 | rawSql: _topFlowSankeyQueries[title]
166 | }]
167 | }]
168 |
169 | _flowsPerASNQueries: {
170 | "Top 30 ASN per service (inbound)":
171 | #"""
172 | SELECT
173 | ASNToString(SrcAS) AS SrcASName,
174 | if($showHostnames, HostToString(SamplerAddress, DstAddr), IPv6ToString(DstAddr)) || ' ' || dictGetString('IPProtocols', 'Name', toUInt64(Proto)) || toString(DstPort) as Dst,
175 | (sum(Bytes * SamplingRate) / 1024) as Bytes
176 | FROM flows_raw
177 | WHERE $__timeFilter(TimeReceived)
178 | \#(_filtersWithHost)
179 | AND isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
180 | GROUP BY SrcAS, SamplerAddress, DstAddr, Proto, DstPort
181 | ORDER BY Bytes DESC
182 | LIMIT 30
183 | """#
184 |
185 | "Top 30 ASN per service (outbound)":
186 | #"""
187 | SELECT
188 | ASNToString(SrcAS) AS SrcASName,
189 | if($showHostnames, HostToString(SamplerAddress, DstAddr), IPv6ToString(DstAddr)) || ' ' || dictGetString('IPProtocols', 'Name', toUInt64(Proto)) || toString(DstPort) as Dst,
190 | (sum(Bytes * SamplingRate) / 1024) as Bytes
191 | FROM flows_raw
192 | WHERE $__timeFilter(TimeReceived)
193 | \#(_filtersWithHost)
194 | AND NOT isIncomingFlow(SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection)
195 | GROUP BY SrcAS, SamplerAddress, DstAddr, Proto, DstPort
196 | ORDER BY Bytes DESC
197 | LIMIT 30
198 | """#
199 | }
200 |
201 | _flowsPerASN: [{
202 | title: "Flows per ASN"
203 | gridPos: y: 34
204 | type: "row"
205 | }, {
206 | title: "Top 30 ASN per service (inbound)"
207 | type: "netsage-sankey-panel"
208 | gridPos: {h: 24, w: 12, x: 0, y: 60}
209 | options: nodePadding: 11
210 | options: nodeWidth: 28
211 | targets: [{
212 | rawSql: _flowsPerASNQueries[title]
213 | }]
214 | }, {
215 | title: "Top 30 ASN per service (outbound)"
216 | type: "netsage-sankey-panel"
217 | gridPos: {h: 24, w: 12, x: 12, y: 60}
218 | options: nodePadding: 11
219 | options: nodeWidth: 28
220 | targets: [{
221 | rawSql: _flowsPerASNQueries[title]
222 | }]
223 | }]
224 |
225 | dashboards: "Traffic Relations": {
226 | #folder: "NetMeta"
227 | title: "Traffic Relations"
228 | uid: "5pH2j5ank"
229 | _panels: list.Concat([
230 | _disclaimerPanels,
231 | _infoPanels,
232 | _asRelations,
233 | _topFlowSankey,
234 | _flowsPerASN,
235 | ])
236 | }
237 |
--------------------------------------------------------------------------------