├── VERSION ├── .gitignore ├── docs ├── traffic_source_by_as.png ├── traffic_destination_by_country.png └── config.yaml ├── .golangci.yaml ├── pkg ├── collector │ ├── log.go │ ├── producer.go │ ├── enrich_host_alias_test.go │ ├── types_test.go │ ├── filter_test.go │ ├── enrich_host_alias.go │ ├── types.go │ ├── filter.go │ ├── enrich_test.go │ ├── metric_test.go │ ├── metric.go │ ├── server_test.go │ ├── server.go │ └── enrich.go └── public │ └── types.go ├── .github ├── dependabot.yml └── workflows │ ├── release-on-tag.yaml │ ├── build.yaml │ ├── codeql.yml │ ├── stale.yaml │ ├── scorecards.yml │ └── publish-image.yml ├── .pre-commit-config.yaml ├── cmd └── main.go ├── Dockerfile ├── go.mod ├── Makefile ├── testdata └── config.yaml ├── README.md └── go.sum /VERSION: -------------------------------------------------------------------------------- 1 | 1.1.1 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /config.yaml 2 | coverage.out 3 | .idea 4 | netflow-collector 5 | netflow-collector.exe 6 | -------------------------------------------------------------------------------- /docs/traffic_source_by_as.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rkosegi/netflow-collector/HEAD/docs/traffic_source_by_as.png -------------------------------------------------------------------------------- /docs/traffic_destination_by_country.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rkosegi/netflow-collector/HEAD/docs/traffic_destination_by_country.png -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | linters: 16 | enable: 17 | - staticcheck 18 | disable-all: true 19 | -------------------------------------------------------------------------------- /pkg/collector/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "log/slog" 19 | ) 20 | 21 | var ( 22 | baseLogger = slog.Default() 23 | ) 24 | 25 | func SetBaseLogger(logger *slog.Logger) { 26 | baseLogger = logger 27 | } 28 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | version: 2 16 | updates: 17 | - package-ecosystem: gomod 18 | directory: / 19 | schedule: 20 | interval: daily 21 | - package-ecosystem: github-actions 22 | directory: / 23 | schedule: 24 | interval: daily 25 | -------------------------------------------------------------------------------- /.github/workflows/release-on-tag.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: Create release on tag push 16 | on: 17 | push: 18 | tags: 19 | - v* 20 | 21 | permissions: 22 | packages: write 23 | contents: write 24 | 25 | jobs: 26 | release: 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 30 | - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 31 | with: 32 | generate_release_notes: true 33 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | repos: 16 | - repo: https://github.com/pre-commit/pre-commit-hooks 17 | rev: v6.0.0 18 | hooks: 19 | - id: trailing-whitespace 20 | - id: check-merge-conflict 21 | - id: end-of-file-fixer 22 | - id: mixed-line-ending 23 | - repo: https://github.com/dnephin/pre-commit-golang 24 | rev: v0.5.1 25 | hooks: 26 | - id: go-mod-tidy 27 | - id: go-fmt 28 | - id: go-imports 29 | - repo: https://github.com/gitleaks/gitleaks 30 | rev: v8.30.0 31 | hooks: 32 | - id: gitleaks 33 | -------------------------------------------------------------------------------- /pkg/collector/producer.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "github.com/netsampler/goflow2/v2/decoders/netflowlegacy" 5 | flowpb "github.com/netsampler/goflow2/v2/pb" 6 | "github.com/netsampler/goflow2/v2/producer" 7 | protoproducer "github.com/netsampler/goflow2/v2/producer/proto" 8 | ) 9 | 10 | type messageConsumer interface { 11 | Consume(msg *flowpb.FlowMessage) 12 | } 13 | 14 | type producerMetricAdapter struct { 15 | consumer messageConsumer 16 | } 17 | 18 | func (p *producerMetricAdapter) Produce(msg interface{}, args *producer.ProduceArgs) ([]producer.ProducerMessage, error) { 19 | tr := uint64(args.TimeReceived.UnixNano()) 20 | sa, _ := args.SamplerAddress.Unmap().MarshalBinary() 21 | if rpt, ok := msg.(*netflowlegacy.PacketNetFlowV5); ok { 22 | rpt, err := protoproducer.ProcessMessageNetFlowLegacy(rpt) 23 | for _, x := range rpt { 24 | fmsg, ok := x.(*protoproducer.ProtoProducerMessage) 25 | if !ok { 26 | continue 27 | } 28 | fmsg.TimeReceivedNs = tr 29 | fmsg.SamplerAddress = sa 30 | } 31 | return rpt, err 32 | } 33 | return []producer.ProducerMessage{}, nil 34 | } 35 | 36 | func (p *producerMetricAdapter) Commit(messages []producer.ProducerMessage) { 37 | for _, msg := range messages { 38 | p.consumer.Consume(&(msg.(*protoproducer.ProtoProducerMessage)).FlowMessage) 39 | } 40 | } 41 | 42 | func (p *producerMetricAdapter) Close() {} 43 | -------------------------------------------------------------------------------- /pkg/collector/enrich_host_alias_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/rkosegi/ipfix-collector/pkg/public" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestEnrichHostAlias(t *testing.T) { 25 | e := getEnricher("host_alias") 26 | assert.NoError(t, e.Start()) 27 | e.Configure(map[string]interface{}{ 28 | "alias_map": map[string]interface{}{ 29 | "192.168.0.1": "gateway", 30 | }, 31 | }) 32 | defer func(e public.Enricher) { 33 | _ = e.Close() 34 | }(e) 35 | f := &public.Flow{} 36 | f.AddAttr("source_ip", []byte{192, 168, 0, 1}) 37 | e.Enrich(f) 38 | assert.Equal(t, "gateway", *f.AsString("source_host_alias")) 39 | 40 | f = &public.Flow{} 41 | f.AddAttr("source_ip", []byte{192, 168, 0, 10}) 42 | e.Enrich(f) 43 | 44 | assert.Equal(t, "unknown", *f.AsString("source_host_alias")) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/collector/types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | ) 22 | 23 | func TestParseConfig(t *testing.T) { 24 | cfg, err := LoadConfig("../../testdata/config.yaml") 25 | assert.NoError(t, err) 26 | assert.NotNil(t, cfg) 27 | assert.Equal(t, 120, cfg.FlushInterval) 28 | // filter 29 | filt0 := *cfg.Pipeline.Filter 30 | assert.Equal(t, "source_ip", filt0[1].Match) 31 | // enrich 32 | assert.Equal(t, 4, len(*cfg.Pipeline.Enrich)) 33 | // metrics 34 | assert.Equal(t, 1, len(cfg.Pipeline.Metrics.Items)) 35 | // extensions 36 | assert.Equal(t, "/usr/share/GeoIP/", cfg.Extensions["maxmind_asn"]["mmdb_dir"]) 37 | assert.Equal(t, "wan0", cfg.Extensions["interface_mapper"]["1"]) 38 | met1 := cfg.Pipeline.Metrics.Items[0] 39 | assert.Equal(t, "Traffic detail", met1.Description) 40 | assert.Equal(t, "proto_name", met1.Labels[1].Value) 41 | assert.Equal(t, "empty_str", met1.Labels[4].OnMissing) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/collector/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/rkosegi/ipfix-collector/pkg/public" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestCidrFn(t *testing.T) { 25 | subnet1 := "192.168.1.0/24" 26 | flow1 := &public.Flow{} 27 | flow1.AddAttr("source_ip", []byte{192, 168, 1, 14}) 28 | flow2 := &public.Flow{} 29 | flow2.AddAttr("source_ip", []byte{10, 11, 12, 13}) 30 | fn, err := getFilterFn(&public.FlowMatchRule{ 31 | Match: "source_ip", 32 | Cidr: &subnet1, 33 | }) 34 | assert.NoError(t, err) 35 | assert.True(t, fn(flow1)) 36 | assert.False(t, fn(flow2)) 37 | } 38 | 39 | func TestIsFn(t *testing.T) { 40 | ip := "192.168.1.14" 41 | flow1 := &public.Flow{} 42 | flow1.AddAttr("source_ip", []byte{192, 168, 1, 14}) 43 | flow2 := &public.Flow{} 44 | flow2.AddAttr("source_ip", []byte{10, 11, 12, 13}) 45 | fn, err := getFilterFn(&public.FlowMatchRule{ 46 | Match: "source_ip", 47 | Is: &ip, 48 | }) 49 | assert.NoError(t, err) 50 | assert.True(t, fn(flow1)) 51 | assert.False(t, fn(flow2)) 52 | } 53 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/alecthomas/kingpin/v2" 21 | "github.com/prometheus/common/promslog" 22 | "github.com/prometheus/common/promslog/flag" 23 | "github.com/prometheus/common/version" 24 | "github.com/rkosegi/ipfix-collector/pkg/collector" 25 | ) 26 | 27 | const progName = "netflow_collector" 28 | 29 | var ( 30 | configFile = kingpin.Flag("config", "Path to the configuration file.").Default("config.yaml").String() 31 | ) 32 | 33 | func main() { 34 | promlogConfig := &promslog.Config{ 35 | Style: promslog.GoKitStyle, 36 | } 37 | flag.AddFlags(kingpin.CommandLine, promlogConfig) 38 | kingpin.Version(version.Print(progName)) 39 | kingpin.HelpFlag.Short('h') 40 | kingpin.Parse() 41 | 42 | logger := promslog.New(promlogConfig) 43 | logger.Info(fmt.Sprintf("Starting %s", progName), "version", version.Info(), "config", *configFile) 44 | collector.SetBaseLogger(logger) 45 | if cfg, err := collector.LoadConfig(*configFile); err != nil { 46 | panic(err) 47 | } else { 48 | panic(collector.New(cfg, logger).Run()) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.25 as builder 16 | 17 | WORKDIR /build 18 | COPY . /build 19 | 20 | RUN make build-local 21 | 22 | FROM cgr.dev/chainguard/static:latest 23 | 24 | ARG VERSION 25 | ARG BUILD_DATE 26 | ARG GIT_COMMIT 27 | 28 | WORKDIR / 29 | COPY --from=builder /build/netflow-collector . 30 | 31 | LABEL org.opencontainers.image.url="https://github.com/rkosegi/netflow-collector" \ 32 | org.opencontainers.image.documentation="https://github.com/rkosegi/netflow-collector/blob/main/README.md" \ 33 | org.opencontainers.image.source="https://github.com/rkosegi/netflow-collector.git" \ 34 | org.opencontainers.image.title="Netflow collector" \ 35 | org.opencontainers.image.licenses="Apache-2.0" \ 36 | org.opencontainers.image.vendor="rkosegi" \ 37 | org.opencontainers.image.description="Simple Netflow V5 exporter for prometheus" \ 38 | org.opencontainers.image.created="${BUILD_DATE}" \ 39 | org.opencontainers.image.revision="${GIT_COMMIT}" \ 40 | org.opencontainers.image.version="${VERSION}" 41 | 42 | USER 65532:65532 43 | 44 | ENTRYPOINT ["/netflow-collector"] 45 | -------------------------------------------------------------------------------- /pkg/collector/enrich_host_alias.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "github.com/rkosegi/ipfix-collector/pkg/public" 19 | ) 20 | 21 | type enrichHostAlias struct { 22 | aliases map[string]string 23 | } 24 | 25 | func (e *enrichHostAlias) Close() error { return nil } 26 | func (e *enrichHostAlias) Start() error { return nil } 27 | 28 | func (e *enrichHostAlias) Configure(cfg map[string]interface{}) { 29 | e.aliases = map[string]string{} 30 | if _, ok := cfg["alias_map"]; ok { 31 | m := cfg["alias_map"].(map[string]interface{}) 32 | for k, v := range m { 33 | e.aliases[k] = v.(string) 34 | } 35 | } 36 | } 37 | 38 | func (e *enrichHostAlias) enrichAliasAttr(flow *public.Flow, attr, dest string) { 39 | if ip := flow.AsIp(attr); ip != nil { 40 | if alias, ok := e.aliases[ip.String()]; ok { 41 | flow.AddAttr(dest, alias) 42 | return 43 | } 44 | } 45 | flow.AddAttr(dest, "unknown") 46 | } 47 | 48 | func (e *enrichHostAlias) Enrich(flow *public.Flow) { 49 | e.enrichAliasAttr(flow, "source_ip", "source_host_alias") 50 | e.enrichAliasAttr(flow, "destination_ip", "destination_host_alias") 51 | } 52 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: Build 16 | 17 | on: 18 | push: 19 | branches: [ main ] 20 | pull_request: 21 | branches: [ main ] 22 | workflow_dispatch: 23 | permissions: 24 | contents: read 25 | 26 | jobs: 27 | build: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - name: Set up Go 31 | uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 32 | with: 33 | go-version: '1.25' 34 | 35 | - name: Install lint tools 36 | run: | 37 | go install golang.org/x/tools/cmd/goimports@latest 38 | 39 | - name: Check out code 40 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 41 | with: 42 | fetch-depth: 0 43 | 44 | - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 45 | with: 46 | python-version: '3.x' 47 | - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 48 | 49 | - name: Build 50 | run: go build -v ./... 51 | 52 | - name: Test 53 | run: go test -v ./... -coverprofile=coverage.out 54 | -------------------------------------------------------------------------------- /pkg/collector/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "os" 19 | 20 | "github.com/jellydator/ttlcache/v3" 21 | "github.com/prometheus/client_golang/prometheus" 22 | "github.com/rkosegi/ipfix-collector/pkg/public" 23 | "gopkg.in/yaml.v3" 24 | ) 25 | 26 | type metricEntry struct { 27 | counter *prometheus.CounterVec 28 | opts prometheus.CounterOpts 29 | labels []*labelProcessor 30 | metrics *ttlcache.Cache[string, prometheus.Counter] 31 | } 32 | 33 | type FilterFn func(flow *public.Flow) bool 34 | 35 | type FlowMatcher struct { 36 | rule *public.FlowMatchRule 37 | fn FilterFn 38 | } 39 | 40 | type labelProcessor struct { 41 | attr string 42 | name string 43 | applyFn func(flow *public.Flow) string 44 | onMissingFn func(flow *public.Flow) string 45 | converterFn func(interface{}) string 46 | } 47 | 48 | func LoadConfig(file string) (*public.Config, error) { 49 | var cfg public.Config 50 | data, err := os.ReadFile(file) 51 | if err != nil { 52 | return nil, err 53 | } 54 | err = yaml.Unmarshal(data, &cfg) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return &cfg, nil 59 | } 60 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the License); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an AS IS BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: CodeQL 16 | 17 | on: 18 | push: 19 | branches: 20 | - main 21 | pull_request: 22 | branches: 23 | - main 24 | schedule: 25 | - cron: 0 0 * * 1 26 | 27 | permissions: 28 | contents: read 29 | 30 | jobs: 31 | analyze: 32 | name: Analyze 33 | runs-on: ubuntu-latest 34 | permissions: 35 | actions: read 36 | contents: read 37 | security-events: write 38 | 39 | strategy: 40 | fail-fast: false 41 | matrix: 42 | language: 43 | - go 44 | steps: 45 | - name: Checkout repository 46 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 47 | 48 | - name: Initialize CodeQL 49 | uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 50 | with: 51 | languages: ${{ matrix.language }} 52 | 53 | - name: Autobuild 54 | uses: github/codeql-action/autobuild@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 55 | 56 | - name: Perform CodeQL Analysis 57 | uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 58 | with: 59 | category: /language:${{matrix.language}} 60 | -------------------------------------------------------------------------------- /.github/workflows/stale.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: Close inactive issues 16 | on: 17 | schedule: 18 | - cron: 30 2 * * 2 19 | workflow_dispatch: 20 | permissions: 21 | contents: read 22 | 23 | jobs: 24 | close-issues: 25 | runs-on: ubuntu-latest 26 | permissions: 27 | issues: write 28 | pull-requests: write 29 | steps: 30 | - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 31 | with: 32 | days-before-stale: 30 33 | days-before-close: 7 34 | exempt-issue-labels: bug 35 | exempt-pr-labels: bug 36 | stale-issue-label: stale 37 | stale-pr-label: stale 38 | stale-issue-message: This issue has been marked 'stale' due to lack of activity. The issue will be closed in another 7 days. 39 | close-issue-message: This issue has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 40 | stale-pr-message: This pr has been marked 'stale' due to lack of recent activity.The issue will be closed in another 7 days. 41 | close-pr-message: This pr has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 42 | repo-token: ${{ secrets.GITHUB_TOKEN }} 43 | -------------------------------------------------------------------------------- /.github/workflows/scorecards.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the License); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an AS IS BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: Scorecard supply-chain security 16 | on: 17 | branch_protection_rule: 18 | schedule: 19 | - cron: 20 7 * * 2 20 | push: 21 | branches: 22 | - main 23 | 24 | permissions: read-all 25 | 26 | jobs: 27 | analysis: 28 | name: Scorecard analysis 29 | runs-on: ubuntu-latest 30 | permissions: 31 | security-events: write 32 | id-token: write 33 | contents: read 34 | actions: read 35 | issues: read 36 | pull-requests: read 37 | checks: read 38 | 39 | steps: 40 | - name: Checkout code 41 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 42 | with: 43 | persist-credentials: false 44 | 45 | - name: Run analysis 46 | uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 47 | with: 48 | results_file: results.sarif 49 | results_format: sarif 50 | publish_results: true 51 | 52 | - name: Upload artifact 53 | uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 54 | with: 55 | name: SARIF file 56 | path: results.sarif 57 | retention-days: 5 58 | 59 | - name: Upload to code-scanning 60 | uses: github/codeql-action/upload-sarif@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 61 | with: 62 | sarif_file: results.sarif 63 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/rkosegi/ipfix-collector 16 | 17 | go 1.25 18 | 19 | require ( 20 | github.com/alecthomas/kingpin/v2 v2.4.0 21 | github.com/jellydator/ttlcache/v3 v3.4.0 22 | github.com/maxmind/mmdbwriter v1.1.0 23 | github.com/netsampler/goflow2/v2 v2.2.3 24 | github.com/oschwald/geoip2-golang v1.13.0 25 | github.com/prometheus/client_golang v1.23.2 26 | github.com/prometheus/client_model v0.6.2 27 | github.com/prometheus/common v0.67.4 28 | github.com/stretchr/testify v1.11.1 29 | gopkg.in/yaml.v3 v3.0.1 30 | ) 31 | 32 | require ( 33 | github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 36 | github.com/davecgh/go-spew v1.1.1 // indirect 37 | github.com/libp2p/go-reuseport v0.4.0 // indirect 38 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 39 | github.com/oschwald/maxminddb-golang v1.13.0 // indirect 40 | github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.10 // indirect 41 | github.com/pmezard/go-difflib v1.0.0 // indirect 42 | github.com/prometheus/procfs v0.16.1 // indirect 43 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 44 | go.yaml.in/yaml/v2 v2.4.3 // indirect 45 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect 46 | golang.org/x/sync v0.15.0 // indirect 47 | golang.org/x/sys v0.37.0 // indirect 48 | google.golang.org/protobuf v1.36.10 // indirect 49 | ) 50 | -------------------------------------------------------------------------------- /docs/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | netflow_endpoint: 0.0.0.0:30000 16 | telemetry_endpoint: 0.0.0.0:30001 17 | flush_interval: 120 18 | pipeline: 19 | filter: 20 | - local-to-local: true 21 | - match: source_ip 22 | is: 0.0.0.0 23 | - match: source_ip 24 | is: 255.255.255.255 25 | - match: destination_ip 26 | is: 0.0.0.0 27 | - match: destination_ip 28 | is: 255.255.255.255 29 | enrich: 30 | - interface_mapper 31 | - maxmind_country 32 | - maxmind_asn 33 | - protocol_name 34 | metrics: 35 | prefix: netflow 36 | items: 37 | - name: traffic_detail 38 | description: Traffic detail 39 | labels: 40 | - name: sampler 41 | value: sampler 42 | converter: ipv4 43 | - name: protocol 44 | value: proto_name 45 | converter: str 46 | - name: source_country 47 | value: source_country 48 | converter: str 49 | - name: destination_country 50 | value: destination_country 51 | converter: str 52 | - name: input_interface 53 | value: input_interface 54 | converter: uint32 55 | - name: output_interface 56 | value: output_interface 57 | converter: uint32 58 | - name: source_asn_org 59 | value: source_asn_org 60 | converter: str 61 | - name: destination_asn_org 62 | value: destination_asn_org 63 | converter: str 64 | extensions: 65 | maxmind_country: 66 | mmdb_dir: /usr/share/GeoIP/ 67 | maxmind_asn: 68 | mmdb_dir: /usr/share/GeoIP/ 69 | interface_mapper: 70 | "0": wan0 71 | "4": lan1 72 | -------------------------------------------------------------------------------- /.github/workflows/publish-image.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | name: Publish container image on tag push 16 | on: 17 | push: 18 | tags: 19 | - v* 20 | workflow_dispatch: 21 | 22 | env: 23 | REGISTRY: ghcr.io 24 | 25 | permissions: 26 | contents: read 27 | 28 | jobs: 29 | image: 30 | runs-on: ubuntu-latest 31 | permissions: 32 | contents: read 33 | packages: write 34 | 35 | steps: 36 | - name: Checkout 37 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 38 | 39 | - name: Set up QEMU 40 | uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 41 | 42 | - name: Set up Docker Buildx 43 | uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 44 | 45 | - name: Log in to the Container registry 46 | uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef 47 | with: 48 | registry: ${{ env.REGISTRY }} 49 | username: ${{ github.actor }} 50 | password: ${{ secrets.GITHUB_TOKEN }} 51 | 52 | - name: Extract image metadata (tags, labels) 53 | id: meta 54 | uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 55 | with: 56 | images: ${{ env.REGISTRY }}/${{ github.repository }} 57 | 58 | - name: Get build timestamp 59 | run: echo "{now}=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT 60 | 61 | - name: Build and push image 62 | uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 63 | with: 64 | context: . 65 | push: true 66 | platforms: linux/amd64,linux/arm64 67 | build-args: | 68 | GIT_COMMIT=${{ github.sha }} 69 | VERSION=${{ github.ref_name }} 70 | BUILD_DATE=${{ steps.build-timestamp.outputs.now }} 71 | tags: ${{ env.REGISTRY }}/${{ github.repository }}:${{ github.ref_name }} 72 | labels: ${{ steps.meta.outputs.labels }} 73 | -------------------------------------------------------------------------------- /pkg/collector/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "net" 19 | "strconv" 20 | 21 | "github.com/rkosegi/ipfix-collector/pkg/public" 22 | ) 23 | 24 | func getFilterMatcher(rule public.FlowMatchRule) (*FlowMatcher, error) { 25 | ret := &FlowMatcher{ 26 | rule: &rule, 27 | } 28 | fn, err := getFilterFn(&rule) 29 | if err != nil { 30 | return nil, err 31 | } 32 | ret.fn = fn 33 | return ret, nil 34 | } 35 | 36 | func getCidrFilterFn(rule *public.FlowMatchRule) (FilterFn, error) { 37 | _, ipnet, err := net.ParseCIDR(*rule.Cidr) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return func(flow *public.Flow) bool { 42 | ip := flow.AsIp(rule.Match) 43 | if ip == nil { 44 | return false 45 | } 46 | return ipnet.Contains(ip) 47 | }, nil 48 | } 49 | 50 | func getL2LFilterFn(rule *public.FlowMatchRule) (FilterFn, error) { 51 | return func(flow *public.Flow) bool { 52 | return isLocalIp(flow.AsIp("source_ip")) && isLocalIp(flow.AsIp("destination_ip")) 53 | }, nil 54 | } 55 | 56 | func getIsFilterFn(rule *public.FlowMatchRule) (FilterFn, error) { 57 | ip := net.ParseIP(*rule.Is) 58 | return func(flow *public.Flow) bool { 59 | v := flow.AsIp(rule.Match) 60 | if v == nil && ip == nil { 61 | return true 62 | } 63 | if v == nil || ip == nil { 64 | return false 65 | } 66 | return ip.Equal(v) 67 | }, nil 68 | } 69 | 70 | func getIsUint32FilterFn(rule *public.FlowMatchRule) (FilterFn, error) { 71 | i, err := strconv.Atoi(*rule.IsUint32) 72 | if err != nil { 73 | return nil, err 74 | } 75 | return func(flow *public.Flow) bool { 76 | v := flow.AsUint32(rule.Match) 77 | return v != nil && *v == uint32(i) 78 | }, nil 79 | } 80 | 81 | func getFilterFn(rule *public.FlowMatchRule) (FilterFn, error) { 82 | if rule.Local2Local != nil && *rule.Local2Local { 83 | return getL2LFilterFn(rule) 84 | } 85 | if rule.Cidr != nil { 86 | return getCidrFilterFn(rule) 87 | } 88 | if rule.Is != nil { 89 | return getIsFilterFn(rule) 90 | } 91 | if rule.IsUint32 != nil { 92 | return getIsUint32FilterFn(rule) 93 | } 94 | return nil, nil 95 | } 96 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | REGISTRY ?= ghcr.io/rkosegi 16 | DOCKER ?= docker 17 | IMAGE_NAME := $(REGISTRY)"/netflow-collector" 18 | VERSION := $(shell cat VERSION) 19 | VER_PARTS := $(subst ., ,$(VERSION)) 20 | VER_MAJOR := $(word 1,$(VER_PARTS)) 21 | VER_MINOR := $(word 2,$(VER_PARTS)) 22 | VER_PATCH := $(word 3,$(VER_PARTS)) 23 | VER_NEXT_PATCH := $(VER_MAJOR).$(VER_MINOR).$(shell echo $$(($(VER_PATCH)+1))) 24 | BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') 25 | GIT_COMMIT ?= $(shell git rev-parse --short HEAD) 26 | BRANCH ?= $(strip $(shell git rev-parse --abbrev-ref HEAD)) 27 | PKG := github.com/prometheus/common 28 | ARCH ?= $(shell go env GOARCH) 29 | OS ?= $(shell uname -s | tr A-Z a-z) 30 | LDFLAGS = -s -w 31 | LDFLAGS += -X ${PKG}/version.Version=v${VERSION} 32 | LDFLAGS += -X ${PKG}/version.Revision=${GIT_COMMIT} 33 | LDFLAGS += -X ${PKG}/version.Branch=${BRANCH} 34 | LDFLAGS += -X ${PKG}/version.BuildUser=$(shell id -u -n)@$(shell hostname) 35 | LDFLAGS += -X ${PKG}/version.BuildDate=${BUILD_DATE} 36 | 37 | .DEFAULT_GOAL := build-local 38 | 39 | bump-patch-version: 40 | @echo Current: $(VERSION) 41 | @echo Next: $(VER_NEXT_PATCH) 42 | @echo "$(VER_NEXT_PATCH)" > VERSION 43 | git add -- VERSION 44 | git commit -sm "Bump version to $(VER_NEXT_PATCH)" 45 | 46 | git-tag: 47 | git tag -am "Release $(VERSION)" $(VERSION) 48 | 49 | git-push-tag: 50 | git push --tags 51 | 52 | new-release: bump-patch-version git-tag 53 | 54 | update-go-deps: 55 | @for m in $$(go list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ 56 | go get $$m; \ 57 | done 58 | go mod tidy 59 | 60 | build-docker: 61 | docker build -t "$(IMAGE_NAME):v$(VERSION)" \ 62 | --build-arg VERSION=$(VERSION) \ 63 | --build-arg GIT_COMMIT=$(GIT_COMMIT) \ 64 | --build-arg BUILD_DATE=$(BUILD_DATE) \ 65 | . 66 | 67 | push-docker: 68 | docker push $(IMAGE_NAME):$(VERSION) 69 | 70 | build-local: 71 | go fmt ./... 72 | go mod tidy 73 | GOOS=$(OS) GOARCH=$(ARCH) CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o netflow-collector cmd/main.go 74 | 75 | lint: 76 | pre-commit run --all-files 77 | 78 | test: 79 | go test -v ./... 80 | -------------------------------------------------------------------------------- /pkg/collector/enrich_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/rkosegi/ipfix-collector/pkg/public" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestInterfaceMapper(t *testing.T) { 25 | e := getEnricher("interface_mapper") 26 | assert.NoError(t, e.Start()) 27 | e.Configure(map[string]interface{}{ 28 | "0": "wan0", 29 | "1": "eth0", 30 | }) 31 | defer func(e public.Enricher) { 32 | _ = e.Close() 33 | }(e) 34 | f := &public.Flow{} 35 | f.AddAttr("input_interface", uint32(0)) 36 | f.AddAttr("output_interface", uint32(1)) 37 | e.Enrich(f) 38 | assert.Equal(t, "wan0", *f.AsString("input_interface_name")) 39 | assert.Equal(t, "eth0", *f.AsString("output_interface_name")) 40 | } 41 | 42 | func TestProtocolName(t *testing.T) { 43 | e := getEnricher("protocol_name") 44 | assert.NoError(t, e.Start()) 45 | e.Configure(map[string]interface{}{}) 46 | defer func(e public.Enricher) { 47 | _ = e.Close() 48 | }(e) 49 | f := &public.Flow{} 50 | f.AddAttr("proto", uint32(1)) 51 | e.Enrich(f) 52 | assert.Equal(t, "icmp", *f.AsString("proto_name")) 53 | } 54 | 55 | func TestReverseLookup(t *testing.T) { 56 | f := &public.Flow{} 57 | 58 | // NOTE: This IP address is assumed not to have a reverse DNS record. 59 | // If the test is run on a network where this is defined, it will fail. 60 | f.AddAttr("source_ip", []byte{192, 168, 255, 255}) 61 | f.AddAttr("destination_ip", []byte{1, 1, 1, 1}) 62 | 63 | nolookup := getEnricher("reverse_dns") 64 | assert.NoError(t, nolookup.Start()) 65 | nolookup.Configure(map[string]interface{}{ 66 | "lookup_local": false, 67 | "lookup_remote": false, 68 | }) 69 | defer func(e public.Enricher) { 70 | _ = e.Close() 71 | }(nolookup) 72 | nolookup.Enrich(f) 73 | assert.Equal(t, "local", f.Raw("source_dns")) 74 | assert.Equal(t, "remote", f.Raw("destination_dns")) 75 | 76 | lookup := getEnricher("reverse_dns") 77 | assert.NoError(t, lookup.Start()) 78 | lookup.Configure(map[string]interface{}{ 79 | "lookup_remote": true, 80 | "lookup_local": true, 81 | "ip_as_unknown": true, 82 | }) 83 | defer func(e public.Enricher) { 84 | _ = e.Close() 85 | }(nolookup) 86 | lookup.Enrich(f) 87 | assert.Equal(t, "192.168.255.255", f.Raw("source_dns")) 88 | assert.Equal(t, "one.one.one.one", f.Raw("destination_dns")) 89 | } 90 | -------------------------------------------------------------------------------- /testdata/config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Richard Kosegi 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | netflow_endpoint: 0.0.0.0:30000 16 | telemetry_endpoint: 0.0.0.0:30001 17 | flush_interval: 120 18 | pipeline: 19 | filter: 20 | - local-to-local: true 21 | - match: source_ip 22 | is: 0.0.0.0 23 | - match: source_ip 24 | is: 255.255.255.255 25 | - match: destination_ip 26 | is: 0.0.0.0 27 | - match: destination_ip 28 | is: 255.255.255.255 29 | enrich: 30 | - interface_mapper 31 | - maxmind_country 32 | - maxmind_asn 33 | - protocol_name 34 | metrics: 35 | prefix: netflow 36 | items: 37 | - name: traffic_detail 38 | description: Traffic detail 39 | labels: 40 | - name: sampler 41 | value: sampler 42 | converter: ipv4 43 | - name: protocol 44 | value: proto_name 45 | converter: str 46 | - name: source_port 47 | value: source_port 48 | converter: uint32 49 | - name: destination_port 50 | value: destination_port 51 | converter: uint32 52 | - name: source_country 53 | value: source_country 54 | converter: str 55 | on_missing: empty_str 56 | - name: destination_country 57 | value: destination_country 58 | converter: str 59 | - name: source 60 | value: source_ip 61 | converter: ipv4 62 | - name: destination 63 | value: destination_ip 64 | converter: ipv4 65 | - name: input_interface 66 | value: input_interface 67 | converter: uint32 68 | - name: output_interface 69 | value: output_interface 70 | converter: uint32 71 | - name: source_asn_org 72 | value: source_asn_org 73 | converter: str 74 | - name: destination_asn_org 75 | value: destination_asn_org 76 | converter: str 77 | - name: output_interface_name 78 | converter: str 79 | value: output_interface_name 80 | - name: input_interface_name 81 | converter: str 82 | value: input_interface_name 83 | extensions: 84 | maxmind_country: 85 | mmdb_dir: /usr/share/GeoIP/ 86 | maxmind_asn: 87 | mmdb_dir: /usr/share/GeoIP/ 88 | interface_mapper: 89 | "1": wan0 90 | "2": lan 91 | "7": bridge1 92 | -------------------------------------------------------------------------------- /pkg/public/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package public 16 | 17 | import ( 18 | "io" 19 | "net" 20 | ) 21 | 22 | type Collector interface { 23 | Run() error 24 | } 25 | 26 | type Flow struct { 27 | attrs map[string]interface{} 28 | } 29 | 30 | type Enricher interface { 31 | io.Closer 32 | Configure(map[string]interface{}) 33 | Start() error 34 | Enrich(*Flow) 35 | } 36 | 37 | // AddAttr adds or updates attribute value 38 | func (f *Flow) AddAttr(attr string, v interface{}) { 39 | if f.attrs == nil { 40 | f.attrs = make(map[string]interface{}, 0) 41 | } 42 | f.attrs[attr] = v 43 | } 44 | 45 | // AsIp attempts to get attribute value as net.IP 46 | func (f *Flow) AsIp(attr string) net.IP { 47 | if v, ok := f.attrs[attr]; ok { 48 | b := v.([]byte) 49 | return net.IPv4(b[0], b[1], b[2], b[3]) 50 | } 51 | return nil 52 | } 53 | 54 | // AsString attempts to get attribute value as string 55 | func (f *Flow) AsString(attr string) *string { 56 | if v, ok := f.attrs[attr]; ok { 57 | x := v.(string) 58 | return &x 59 | } 60 | return nil 61 | } 62 | 63 | // Raw attempts to get attribute value 64 | func (f *Flow) Raw(attr string) interface{} { 65 | return f.attrs[attr] 66 | } 67 | 68 | // AsUint32 attempts to get attribute value as uint32 69 | func (f *Flow) AsUint32(attr string) *uint32 { 70 | if v, ok := f.attrs[attr]; ok { 71 | x := v.(uint32) 72 | return &x 73 | } 74 | return nil 75 | } 76 | 77 | type Config struct { 78 | NetflowEndpoint string `yaml:"netflow_endpoint"` 79 | TelemetryEndpoint *string `yaml:"telemetry_endpoint"` 80 | Pipeline Pipeline `yaml:"pipeline"` 81 | FlushInterval int `yaml:"flush_interval"` 82 | Extensions map[string]map[string]interface{} `yaml:"extensions"` 83 | } 84 | 85 | type Pipeline struct { 86 | Filter *[]FlowMatchRule `yaml:"filter,omitempty"` 87 | Enrich *[]string `yaml:"enrich,omitempty"` 88 | Metrics MetricsConfig `yaml:"metrics"` 89 | } 90 | 91 | type MetricsConfig struct { 92 | Prefix string `yaml:"prefix"` 93 | Items []MetricSpec `yaml:"items"` 94 | } 95 | 96 | type MetricSpec struct { 97 | Name string `yaml:"name"` 98 | Description string `yaml:"description"` 99 | Labels []MetricLabel `yaml:"labels"` 100 | } 101 | 102 | type MetricLabel struct { 103 | Name string `yaml:"name"` 104 | Value string `yaml:"value"` 105 | OnMissing string `yaml:"on_missing,omitempty"` 106 | Converter string `yaml:"converter"` 107 | } 108 | 109 | type FlowMatchRule struct { 110 | Match string `yaml:"match"` 111 | Cidr *string `yaml:"cidr,omitempty"` 112 | Is *string `yaml:"is,omitempty"` 113 | IsUint32 *string `yaml:"isUint32,omitempty"` 114 | Local2Local *bool `yaml:"local-to-local"` 115 | } 116 | -------------------------------------------------------------------------------- /pkg/collector/metric_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/jellydator/ttlcache/v3" 22 | "github.com/prometheus/client_golang/prometheus" 23 | dto "github.com/prometheus/client_model/go" 24 | "github.com/rkosegi/ipfix-collector/pkg/public" 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func TestMetrics(t *testing.T) { 29 | s := &public.MetricSpec{ 30 | Name: "test1", 31 | Description: "Test metric 1", 32 | Labels: []public.MetricLabel{ 33 | { 34 | Name: "source", 35 | Value: "source_ip", 36 | OnMissing: "empty_str", 37 | Converter: "ipv4", 38 | }, 39 | }, 40 | } 41 | m := &metricEntry{} 42 | m.init("netflow", s, 60) 43 | f := &public.Flow{} 44 | f.AddAttr("source_ip", []byte{10, 11, 12, 13}) 45 | f.AddAttr("bytes", uint64(30)) 46 | m.apply(f) 47 | 48 | assert.Equal(t, float64(30), getMetric(t, m, "10.11.12.13")) 49 | } 50 | 51 | // This test is a bit awful because it has time.Sleep() in it and takes approx 2 seconds 52 | // This is to verify that metrics expire as expected 53 | func TestMetricExpiration(t *testing.T) { 54 | s := &public.MetricSpec{ 55 | Name: "test1", 56 | Description: "Test metric 1", 57 | Labels: []public.MetricLabel{ 58 | { 59 | Name: "source", 60 | Value: "source_ip", 61 | OnMissing: "empty_str", 62 | Converter: "ipv4", 63 | }, 64 | }, 65 | } 66 | m := &metricEntry{} 67 | m.init("netflow", s, 1) 68 | f := &public.Flow{} 69 | f.AddAttr("source_ip", []byte{10, 11, 12, 13}) 70 | f.AddAttr("bytes", uint64(1)) 71 | 72 | // start tests 73 | 74 | // t = 0.0 - add a thing, verify we get the stat back 75 | assert.Equal(t, 0, countMetrics(m)) 76 | m.apply(f) 77 | assert.Equal(t, 1, countMetrics(m)) 78 | assert.Equal(t, true, metricExists(m, "10.11.12.13")) 79 | assert.Equal(t, float64(1), getMetric(t, m, "10.11.12.13")) 80 | time.Sleep(time.Millisecond * 500) 81 | 82 | // t = 0.5 - first thing should still be validated as we have 1 sec TTL, now add second thing 83 | m.apply(f) 84 | assert.Equal(t, 1, countMetrics(m)) 85 | assert.Equal(t, true, metricExists(m, "10.11.12.13")) 86 | assert.Equal(t, float64(2), getMetric(t, m, "10.11.12.13")) 87 | time.Sleep(time.Millisecond * 600) 88 | 89 | // t = 1.1 - adding second thing should have extended TTL, so verify we still have both things 90 | assert.Equal(t, true, metricExists(m, "10.11.12.13")) 91 | assert.Equal(t, float64(2), getMetric(t, m, "10.11.12.13")) 92 | time.Sleep(time.Millisecond * 500) 93 | 94 | // t = 1.6 - now it should have expired, verify it has gone 95 | assert.Equal(t, 0, countMetrics(m)) 96 | assert.Equal(t, false, metricExists(m, "10.11.12.13")) 97 | 98 | // t = 1.6 - add it again, verify counter has reset 99 | m.apply(f) 100 | assert.Equal(t, 1, countMetrics(m)) 101 | assert.Equal(t, true, metricExists(m, "10.11.12.13")) 102 | assert.Equal(t, float64(1), getMetric(t, m, "10.11.12.13")) 103 | } 104 | 105 | func metricExists(m *metricEntry, v string) bool { 106 | return m.metrics.Get(v, ttlcache.WithDisableTouchOnHit[string, prometheus.Counter]()) != nil 107 | } 108 | 109 | func getMetric(t *testing.T, m *metricEntry, v string) float64 { 110 | vv := dto.Metric{} 111 | assert.NoError(t, (m.metrics.Get(v, ttlcache.WithDisableTouchOnHit[string, prometheus.Counter]()).Value()).Write(&vv)) 112 | return *vv.Counter.Value 113 | } 114 | 115 | func countMetrics(m *metricEntry) int { 116 | return m.metrics.Len() 117 | } 118 | -------------------------------------------------------------------------------- /pkg/collector/metric.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "net" 19 | "strconv" 20 | "strings" 21 | "time" 22 | 23 | "github.com/jellydator/ttlcache/v3" 24 | "github.com/prometheus/client_golang/prometheus" 25 | "github.com/rkosegi/ipfix-collector/pkg/public" 26 | ) 27 | 28 | func (m *metricEntry) init(prefix string, spec *public.MetricSpec, flushInterval int) { 29 | labels := make([]*labelProcessor, 0) 30 | labelNames := make([]string, 0) 31 | for _, label := range spec.Labels { 32 | labelNames = append(labelNames, label.Name) 33 | lp := &labelProcessor{} 34 | lp.init(label) 35 | labels = append(labels, lp) 36 | } 37 | m.labels = labels 38 | m.opts = prometheus.CounterOpts{ 39 | Namespace: prefix, 40 | Subsystem: "flow", 41 | Name: spec.Name, 42 | Help: spec.Description, 43 | } 44 | m.counter = prometheus.NewCounterVec(m.opts, labelNames) 45 | m.metrics = ttlcache.New( 46 | ttlcache.WithTTL[string, prometheus.Counter](time.Duration(flushInterval) * time.Second), 47 | ) 48 | go m.metrics.Start() 49 | } 50 | 51 | func (m *metricEntry) Collect(ch chan<- prometheus.Metric) { 52 | // we don't use the Range function on the cache as it doesn't seem to be 53 | // very thread-safe, ie it emits the same value more than once which 54 | // results in 500 errors when being scraped. So instead we take a copy... 55 | for _, item := range m.metrics.Items() { 56 | ch <- item.Value() 57 | } 58 | } 59 | 60 | func (m *metricEntry) Describe(ch chan<- *prometheus.Desc) { 61 | m.counter.Describe(ch) 62 | } 63 | 64 | func (m *metricEntry) apply(flow *public.Flow) { 65 | labelValues := make([]string, 0) 66 | for _, lp := range m.labels { 67 | labelValues = append(labelValues, lp.apply(flow)) 68 | } 69 | m.metrics.Get(strings.Join(labelValues, "|"), 70 | ttlcache.WithLoader[string, prometheus.Counter](ttlcache.LoaderFunc[string, prometheus.Counter]( 71 | func(c *ttlcache.Cache[string, prometheus.Counter], key string) *ttlcache.Item[string, prometheus.Counter] { 72 | opts := m.opts 73 | opts.ConstLabels = make(prometheus.Labels) 74 | for i, lp := range m.labels { 75 | opts.ConstLabels[lp.name] = labelValues[i] 76 | } 77 | return c.Set(key, prometheus.NewCounter(opts), ttlcache.DefaultTTL) 78 | }, 79 | ))).Value().Add(float64(flow.Raw("bytes").(uint64))) 80 | } 81 | 82 | func (lp *labelProcessor) init(label public.MetricLabel) { 83 | lp.attr = label.Value 84 | lp.name = label.Name 85 | lp.applyFn = lp.apply 86 | lp.onMissingFn = func(flow *public.Flow) string { 87 | return "" 88 | } 89 | 90 | switch label.Converter { 91 | case "ipv4": 92 | lp.converterFn = func(v interface{}) string { 93 | data := v.([]byte) 94 | return net.IPv4(data[0], data[1], data[2], data[3]).String() 95 | } 96 | 97 | case "str": 98 | lp.converterFn = func(v interface{}) string { 99 | return v.(string) 100 | } 101 | 102 | case "uint32": 103 | lp.converterFn = func(v interface{}) string { 104 | return strconv.FormatUint(uint64(v.(uint32)), 10) 105 | } 106 | 107 | case "uint64": 108 | lp.converterFn = func(v interface{}) string { 109 | return strconv.FormatUint(v.(uint64), 10) 110 | } 111 | 112 | case "static": 113 | lp.applyFn = func(flow *public.Flow) string { 114 | return label.Value 115 | } 116 | } 117 | } 118 | 119 | func (lp *labelProcessor) apply(flow *public.Flow) string { 120 | if attr := flow.Raw(lp.attr); attr != nil { 121 | return lp.converterFn(attr) 122 | } else { 123 | return lp.onMissingFn(flow) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /pkg/collector/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "os" 21 | "testing" 22 | 23 | "github.com/maxmind/mmdbwriter" 24 | "github.com/maxmind/mmdbwriter/mmdbtype" 25 | flowpb "github.com/netsampler/goflow2/v2/pb" 26 | dto "github.com/prometheus/client_model/go" 27 | "github.com/rkosegi/ipfix-collector/pkg/public" 28 | "github.com/stretchr/testify/assert" 29 | ) 30 | 31 | func strPtr(str string) *string { 32 | return &str 33 | } 34 | 35 | func genMockmmdb(path string, t *testing.T) { 36 | db, err := mmdbwriter.New(mmdbwriter.Options{ 37 | RecordSize: 24, 38 | DatabaseType: "GeoLite2-ASN", 39 | }) 40 | if err != nil { 41 | t.Fatalf("unable to create new MMDB: %v", err) 42 | } 43 | 44 | err = db.Insert(&net.IPNet{ 45 | IP: net.IP{8, 8, 8, 8}, 46 | Mask: net.IPv4Mask(255, 255, 255, 0), 47 | }, mmdbtype.Map{ 48 | "autonomous_system_number": mmdbtype.Uint32(15169), 49 | "autonomous_system_organization": mmdbtype.String("Google LLC"), 50 | }) 51 | if err != nil { 52 | t.Fatalf("unable to insert mock data: %v", err) 53 | } 54 | f, err := os.OpenFile(fmt.Sprintf("%s/%s", path, "GeoLite2-ASN.mmdb"), os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.FileMode(0o600)) 55 | if err != nil { 56 | t.Fatalf("unable to open file for writing %v", err) 57 | } 58 | _, err = db.WriteTo(f) 59 | if err != nil { 60 | t.Fatalf("unable to write %v", err) 61 | } 62 | defer func() { 63 | err = f.Close() 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | }() 68 | 69 | db, err = mmdbwriter.New(mmdbwriter.Options{ 70 | RecordSize: 24, 71 | DatabaseType: "GeoLite2-Country", 72 | }) 73 | if err != nil { 74 | t.Fatalf("unable to create new MMDB: %v", err) 75 | } 76 | 77 | err = db.Insert(&net.IPNet{ 78 | IP: net.IP{8, 8, 8, 8}, 79 | Mask: net.IPv4Mask(255, 255, 255, 0), 80 | }, mmdbtype.Map{ 81 | "continent": mmdbtype.Map{ 82 | "code": mmdbtype.String("NA"), 83 | "geoname_id": mmdbtype.Int32(6255149), 84 | "names": mmdbtype.Map{ 85 | "en": mmdbtype.String("North America"), 86 | }, 87 | }, 88 | "registered_country": mmdbtype.Map{ 89 | "geoname_id": mmdbtype.Int32(6252001), 90 | "iso_code": mmdbtype.String("US"), 91 | "names": mmdbtype.Map{ 92 | "en": mmdbtype.String("USA"), 93 | }, 94 | }, 95 | "country": mmdbtype.Map{ 96 | "geoname_id": mmdbtype.Int32(6252001), 97 | "iso_code": mmdbtype.String("US"), 98 | "names": mmdbtype.Map{ 99 | "en": mmdbtype.String("USA"), 100 | }, 101 | }, 102 | }) 103 | 104 | f2, err := os.OpenFile(fmt.Sprintf("%s/%s", path, "GeoLite2-Country.mmdb"), os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.FileMode(0o600)) 105 | if err != nil { 106 | t.Fatalf("unable to open file for writing %v", err) 107 | } 108 | _, err = db.WriteTo(f2) 109 | if err != nil { 110 | t.Fatalf("unable to write %v", err) 111 | } 112 | defer func() { 113 | err = f2.Close() 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | }() 118 | 119 | } 120 | 121 | func getFreePort(proto string, t *testing.T) int { 122 | if proto == "udp" { 123 | addr, err := net.ResolveUDPAddr(proto, "127.0.0.1:0") 124 | if err != nil { 125 | t.Fatalf("unable to resolve loopback udp address: %v", err) 126 | } 127 | l, err := net.ListenUDP(proto, addr) 128 | if err != nil { 129 | t.Fatalf("unable to listen on udp address: %v", err) 130 | } 131 | defer func(l *net.UDPConn) { 132 | _ = l.Close() 133 | }(l) 134 | return l.LocalAddr().(*net.UDPAddr).Port 135 | } else { 136 | addr, err := net.ResolveTCPAddr(proto, "127.0.0.1:0") 137 | if err != nil { 138 | t.Fatalf("unable to resolve loopback tcp address: %v", err) 139 | } 140 | l, err := net.ListenTCP(proto, addr) 141 | if err != nil { 142 | t.Fatalf("unable to listen on tcp address: %v", err) 143 | } 144 | defer func(l *net.TCPListener) { 145 | _ = l.Close() 146 | }(l) 147 | return l.Addr().(*net.TCPAddr).Port 148 | } 149 | } 150 | 151 | func TestServer(t *testing.T) { 152 | f, err := os.CreateTemp("", "nf.*.yaml") 153 | assert.NoError(t, err) 154 | defer func() { 155 | _ = os.Remove(f.Name()) 156 | }() 157 | cfg, err := LoadConfig("../../testdata/config.yaml") 158 | assert.NoError(t, err) 159 | assert.NotNil(t, cfg) 160 | cfg.TelemetryEndpoint = strPtr(fmt.Sprintf("0.0.0.0:%d", getFreePort("tcp", t))) 161 | cfg.NetflowEndpoint = fmt.Sprintf("0.0.0.0:%d", getFreePort("udp", t)) 162 | *cfg.Pipeline.Filter = append(*cfg.Pipeline.Filter, public.FlowMatchRule{ 163 | IsUint32: strPtr("10"), 164 | Match: "source_as", 165 | }) 166 | 167 | d, err := os.MkdirTemp("", "geoip") 168 | assert.NoError(t, err) 169 | defer func() { 170 | _ = os.RemoveAll(d) 171 | }() 172 | cfg.Extensions["maxmind_asn"]["mmdb_dir"] = d 173 | cfg.Extensions["maxmind_country"]["mmdb_dir"] = d 174 | genMockmmdb(d, t) 175 | 176 | c := New(cfg, baseLogger) 177 | go func() { 178 | _ = c.Run() 179 | }() 180 | c.(*col).waitUntilReady() 181 | c.(*col).Publish([]*flowpb.FlowMessage{{ 182 | Type: flowpb.FlowMessage_NETFLOW_V5, 183 | Packets: 1, 184 | SamplerAddress: []byte{127, 0, 0, 1}, 185 | SrcAddr: []byte{8, 8, 8, 8}, 186 | DstAddr: []byte{192, 168, 1, 2}, 187 | SrcPort: 53, 188 | DstPort: 31034, 189 | Proto: 0x11, 190 | SrcAs: 20, 191 | }}) 192 | m := &dto.Metric{} 193 | assert.NoError(t, c.(*col).totalFlowsCounter.WithLabelValues("127.0.0.1").Write(m)) 194 | assert.Equal(t, float64(1), m.Counter.GetValue()) 195 | } 196 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Netflow exporter 2 | 3 | Have you ever wondered where is your internet traffic going? 4 | 5 | ![traffic destination by country](docs/traffic_destination_by_country.png) 6 | 7 | ![traffic source by AS](docs/traffic_source_by_as.png) 8 | 9 | ## How it works 10 | 11 | Simply put, it uses netflow protocol, specifically it uses V5 version (IPv4 only) for simplicity. 12 | In order for your setup to work, you will either need [nfdump](https://github.com/phaag/nfdump) 13 | or dedicated hardware such as [Mikrotik RB941](https://mikrotik.com/product/RB941-2nD) 14 | Flows are then fed into collector that aggregates them as metrics. 15 | Geolocation info is gathered from Maxmind GeoIP Lite database. 16 | Necessary files can be obtained on RHEL OS (or similar) with `sudo dnf install geolite2-country geolite2-asn` 17 | 18 | 19 | Example configuration for routerboard 20 | ``` 21 | /ip traffic-flow 22 | set enabled=yes interfaces=wan,bridge1 23 | /ip traffic-flow target 24 | add dst-address=192.168.0.10 port=30000 version=5 25 | ``` 26 | 27 | _Note `192.168.0.10` is address of machine where exporter is running_ 28 | 29 | ## Configurable metrics 30 | 31 | Flows are aggregated into metrics in fully configurable manner. 32 | 33 | Example metric 34 | ```yaml 35 | - name: traffic_detail 36 | description: Traffic detail 37 | labels: 38 | - name: sampler 39 | value: sampler 40 | converter: ipv4 41 | - name: protocol 42 | value: proto_name 43 | converter: str 44 | - name: source_country 45 | value: source_country 46 | converter: str 47 | - name: destination_asn_org 48 | value: destination_asn_org 49 | converter: str 50 | ``` 51 | 52 | Full example can be found [here](docs/config.yaml) 53 | 54 | ## Supported enrichers 55 | 56 | 57 | - `maxmind_country` 58 | 59 | MaxMind GeoLite country data are used to add source and destination country (if applicable) 60 | - used attributes: `source_ip`, `destination_ip` 61 | - added attributes: `source_country`, `destination_country` 62 | - configuration options: 63 | - `mmdb_dir` - path to directory which holds [MaxMind GeoIP DB files](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) 64 | 65 | - `maxmind_asn` 66 | 67 | MaxMind GeoLite country data are used to add source and destination autonomous system (if applicable) 68 | - used attributes: `source_ip`, `destination_ip` 69 | - added attributes: `source_asn_org`, `destination_asn_org` 70 | - configuration options: 71 | - `mmdb_dir` - path to directory which holds [MaxMind GeoIP DB files](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) 72 | 73 | - `interface_mapper` 74 | - `protocol_name` 75 | 76 | - `host_alias` 77 | 78 | Allows to alias IP address to some human-memorable name. This is kind of similar to `reverse_dns`, 79 | but it uses static map of aliases instead of actual DNS service. 80 | - used attributes: `source_ip`, `destination_ip` 81 | - added attributes: `source_host_alias`, `destination_host_alias` 82 | - configuration options: 83 | 84 | - `alias_map` - mapping of IPv4 address to host alias 85 | 86 | Example config 87 | 88 | ```yaml 89 | pipline: 90 | enrich: 91 | - host_alias 92 | metrics: 93 | items: 94 | - name: source_host_alias 95 | description: Human-memorable alias of source host 96 | labels: 97 | - name: source_host_alias 98 | value: source_host_alias 99 | converter: str 100 | on_missing: empty_str 101 | - name: destination_host_alias 102 | value: destination_host_alias 103 | converter: str 104 | on_missing: empty_str 105 | ..... 106 | extensions: 107 | ..... 108 | host_alias: 109 | alias_map: 110 | 192.168.0.1: My gateway 111 | 192.168.0.10: TVBox 112 | 192.168.0.20: SmartPlug1 113 | ``` 114 | 115 | - `reverse_dns` 116 | 117 | Does a reverse DNS lookup for IP and selects the first entry returned. `unknown` set if none found and ip_as_unknown is not enabled. Results (including missing) cached per `cache_duration`. 118 | 119 | - used attributes: `source_ip`, `destination_ip` 120 | - added attributes: `source_dns`, `destination_dns` 121 | - configuration options: 122 | - `cache_duration` - how long to cache result for. Default `1h`. 123 | - `tail_pihole` - useful if running on a server which is running `pihole`. If set, read from `pihole -t` to populate DNS cache. This cache will be used instead of a reverse DNS lookup if available. By tailing the PiHole log, we can see the original query before `CNAME` redirection and thus give a more interesting answer. Ensure that additional logging entries are enabled, e.g. `echo log-queries=extra | sudo tee /etc/dnsmasq.d/42-add-query-ids.conf ; pihole restartdns` 124 | - `lookup_local` - enable looking up local addresses. Default `false`. 125 | - `lookup_remote` - enable looking up remote addresses. Default `true`. 126 | - `ip_as_unknown` - if a reverse record is not available, uses the IP address itself rather than "unknown" string. Default `false`. 127 | 128 | e.g. add `reverse_dns` under `enrich:` and the following under `labels:`: 129 | 130 | ```yaml 131 | - name: source_ip 132 | value: source_ip 133 | converter: ipv4 134 | - name: destination_ip 135 | value: destination_ip 136 | converter: ipv4 137 | - name: source_dns 138 | value: source_dns 139 | converter: str 140 | - name: destination_dns 141 | value: destination_dns 142 | converter: str 143 | ``` 144 | 145 | e.g. gather in/out statistics for hosts on the local network: 146 | ```yaml 147 | pipline: 148 | ... 149 | enrich: 150 | - reverse_dns 151 | metrics: 152 | ... 153 | items: 154 | - name: traffic_in 155 | description: Traffic in per host 156 | labels: 157 | - name: host 158 | value: destination_dns 159 | converter: str 160 | - name: traffic_out 161 | description: Traffic out per host 162 | labels: 163 | - name: host 164 | value: source_dns 165 | converter: str 166 | 167 | ... 168 | extensions: 169 | reverse_dns: 170 | lookup_local: true 171 | lookup_remote: false 172 | ip_as_unknown: true 173 | ``` 174 | 175 | ## Run using podman/docker 176 | 177 | ```bash 178 | podman run -ti -p 30000:30000/udp -p 30001:30001/tcp -u 1000 \ 179 | -v $(pwd)/config.yaml:/config.yaml:ro \ 180 | -v /usr/share/GeoIP:/usr/share/GeoIP:ro \ 181 | ghcr.io/rkosegi/netflow-collector:latest 182 | ``` 183 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= 2 | github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= 3 | github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= 4 | github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= 5 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 6 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 7 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 8 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 13 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 14 | github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= 15 | github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= 16 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 17 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 18 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 19 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 20 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 21 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 22 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 23 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 24 | github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= 25 | github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= 26 | github.com/maxmind/mmdbwriter v1.1.0 h1:/A7oLq07eKIOp2cP3w6N9nV5X1Aa6KqK3kHy6B5bxbo= 27 | github.com/maxmind/mmdbwriter v1.1.0/go.mod h1:hWm/woy2UXZMuHs9GBB6KMmEclvjMZstQ7pJ+KmTqMM= 28 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 29 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 30 | github.com/netsampler/goflow2/v2 v2.2.3 h1:uItOl69jDHuNJR+LGZ1JFs4/9qzBgbm95SP0QTMzGwo= 31 | github.com/netsampler/goflow2/v2 v2.2.3/go.mod h1:qC4yiY8Rw7SEwrpPy+w2ktnXc403Vilt2ZyBEYE5iJQ= 32 | github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI= 33 | github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= 34 | github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= 35 | github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= 36 | github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.10 h1:d9tiCD1ueYjGStkagZmLYMbItMnJPpmn27jBctlyRg8= 37 | github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.10/go.mod h1:EkyB0XWibbE1/+tXyR+ZehlGg66bRtMzxQSPotYH2EA= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= 41 | github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= 42 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 43 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 44 | github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= 45 | github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= 46 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 47 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 48 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 49 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 50 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 51 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 52 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 53 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 54 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 55 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 56 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 57 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 58 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 59 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 60 | github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= 61 | github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= 62 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 63 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 64 | go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= 65 | go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= 66 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= 67 | go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= 68 | golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= 69 | golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 70 | golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 71 | golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 72 | google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 73 | google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 74 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 75 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 76 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /pkg/collector/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "log/slog" 21 | 22 | flowpb "github.com/netsampler/goflow2/v2/pb" 23 | "github.com/netsampler/goflow2/v2/utils" 24 | "github.com/prometheus/client_golang/prometheus" 25 | "github.com/prometheus/client_golang/prometheus/collectors" 26 | "github.com/prometheus/client_golang/prometheus/promhttp" 27 | "github.com/rkosegi/ipfix-collector/pkg/public" 28 | 29 | "net" 30 | "net/http" 31 | "strconv" 32 | "sync" 33 | "time" 34 | ) 35 | 36 | type col struct { 37 | logger *slog.Logger 38 | ready sync.WaitGroup 39 | cfg *public.Config 40 | filters []FlowMatcher 41 | enrichers []public.Enricher 42 | metrics []*metricEntry 43 | droppedFlowsCounter *prometheus.CounterVec 44 | totalFlowsCounter *prometheus.CounterVec 45 | scrapingSum *prometheus.SummaryVec 46 | ap *utils.AutoFlowPipe 47 | recv *utils.UDPReceiver 48 | } 49 | 50 | func (c *col) Close() error { 51 | if c.ap != nil { 52 | c.ap.Close() 53 | } 54 | if c.recv != nil { 55 | return c.recv.Stop() 56 | } 57 | return nil 58 | } 59 | 60 | func (c *col) Describe(descs chan<- *prometheus.Desc) { 61 | c.droppedFlowsCounter.Describe(descs) 62 | c.totalFlowsCounter.Describe(descs) 63 | c.scrapingSum.Describe(descs) 64 | for _, m := range c.metrics { 65 | m.Describe(descs) 66 | } 67 | } 68 | 69 | func (c *col) Collect(ch chan<- prometheus.Metric) { 70 | start := time.Now() 71 | defer func() { 72 | c.scrapingSum.WithLabelValues().Observe(float64(time.Now().UnixMicro() - start.UnixMicro())) 73 | c.scrapingSum.Collect(ch) 74 | }() 75 | 76 | c.droppedFlowsCounter.Collect(ch) 77 | c.totalFlowsCounter.Collect(ch) 78 | for _, m := range c.metrics { 79 | m.Collect(ch) 80 | } 81 | } 82 | 83 | func (c *col) Publish(messages []*flowpb.FlowMessage) { 84 | for _, msg := range messages { 85 | c.Consume(msg) 86 | } 87 | } 88 | 89 | func (c *col) Consume(msg *flowpb.FlowMessage) { 90 | if msg.Type == flowpb.FlowMessage_NETFLOW_V5 { 91 | flow := c.mapMsg(msg) 92 | c.processFlow(flow) 93 | c.totalFlowsCounter.WithLabelValues(flow.AsIp("sampler").String()).Inc() 94 | } 95 | } 96 | 97 | func (c *col) waitUntilReady() { 98 | c.ready.Wait() 99 | } 100 | 101 | func (c *col) Run() (err error) { 102 | err = c.start() 103 | if err != nil { 104 | return err 105 | } 106 | if c.recv, err = utils.NewUDPReceiver(&utils.UDPReceiverConfig{ 107 | Workers: 2, 108 | Sockets: 1, 109 | Blocking: false, 110 | QueueSize: 100, 111 | }); err != nil { 112 | return err 113 | } 114 | c.ap = utils.NewFlowPipe(&utils.PipeConfig{ 115 | Producer: &producerMetricAdapter{consumer: c}, 116 | }) 117 | 118 | host, port, err := net.SplitHostPort(c.cfg.NetflowEndpoint) 119 | if err != nil { 120 | return err 121 | } 122 | iport, err := strconv.Atoi(port) 123 | if err != nil { 124 | return err 125 | } 126 | c.logger.Info("starting Netflow V5 listener", "host", host, "port", iport) 127 | 128 | defer func() { 129 | _ = c.Close() 130 | }() 131 | 132 | err = c.recv.Start(host, iport, c.ap.DecodeFlow) 133 | if err != nil { 134 | return err 135 | } 136 | <-make(chan struct{}) 137 | return nil 138 | } 139 | 140 | func (c *col) startEnrichers() (err error) { 141 | if c.cfg.Pipeline.Enrich != nil { 142 | c.logger.Debug("starting enrichers", "enrichers", len(*c.cfg.Pipeline.Enrich)) 143 | for _, name := range *c.cfg.Pipeline.Enrich { 144 | c.logger.Info("starting enricher", "name", name) 145 | e := getEnricher(name) 146 | if e == nil { 147 | return fmt.Errorf("unknown enricher : %s", name) 148 | } 149 | 150 | if ext, ok := c.cfg.Extensions[name]; ok { 151 | e.Configure(ext) 152 | } 153 | err = e.Start() 154 | if err != nil { 155 | return err 156 | } 157 | c.enrichers = append(c.enrichers, e) 158 | } 159 | } 160 | return nil 161 | } 162 | 163 | func (c *col) startFilters() error { 164 | if c.cfg.Pipeline.Filter != nil { 165 | c.logger.Info("starting filters", "rules", len(*c.cfg.Pipeline.Filter)) 166 | for _, rule := range *c.cfg.Pipeline.Filter { 167 | m, err := getFilterMatcher(rule) 168 | if err != nil { 169 | return err 170 | } 171 | c.filters = append(c.filters, *m) 172 | } 173 | } 174 | return nil 175 | } 176 | 177 | func (c *col) start() (err error) { 178 | defer c.ready.Done() 179 | 180 | c.totalFlowsCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ 181 | Namespace: c.cfg.Pipeline.Metrics.Prefix, 182 | Subsystem: "server", 183 | Name: "total_flows", 184 | Help: "The total number of ingested flows.", 185 | }, []string{"sampler"}) 186 | c.droppedFlowsCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ 187 | Namespace: c.cfg.Pipeline.Metrics.Prefix, 188 | Subsystem: "server", 189 | Name: "dropped_flows", 190 | Help: "The total number of dropped flows.", 191 | }, []string{"sampler"}) 192 | c.scrapingSum = prometheus.NewSummaryVec(prometheus.SummaryOpts{ 193 | Namespace: c.cfg.Pipeline.Metrics.Prefix, 194 | Subsystem: "server", 195 | Name: "scrape", 196 | Help: "The summary of time spent by scraping in microseconds", 197 | }, []string{}) 198 | if err = c.startFilters(); err != nil { 199 | return err 200 | } 201 | if err = c.startEnrichers(); err != nil { 202 | return err 203 | } 204 | if c.cfg.FlushInterval == 0 { 205 | c.cfg.FlushInterval = 180 206 | } 207 | c.logger.Info("creating metric items", "count", len(c.cfg.Pipeline.Metrics.Items)) 208 | for _, metric := range c.cfg.Pipeline.Metrics.Items { 209 | me := &metricEntry{} 210 | me.init(c.cfg.Pipeline.Metrics.Prefix, &metric, c.cfg.FlushInterval) 211 | c.metrics = append(c.metrics, me) 212 | } 213 | 214 | if c.cfg.TelemetryEndpoint != nil { 215 | prometheus.MustRegister(c) 216 | prometheus.MustRegister(collectors.NewBuildInfoCollector()) 217 | http.Handle("/metrics", promhttp.Handler()) 218 | http.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { 219 | w.WriteHeader(http.StatusOK) 220 | _, _ = w.Write([]byte("OK")) 221 | }) 222 | 223 | c.logger.Info("starting metrics server", "address", *c.cfg.TelemetryEndpoint) 224 | go func() { 225 | if err = http.ListenAndServe(*c.cfg.TelemetryEndpoint, nil); err != nil { 226 | if !errors.Is(err, http.ErrServerClosed) { 227 | panic(err) 228 | } else { 229 | c.logger.Info("metrics server closed") 230 | } 231 | } 232 | }() 233 | } 234 | 235 | return nil 236 | } 237 | 238 | func (c *col) processFlow(flow *public.Flow) { 239 | for _, m := range c.filters { 240 | if m.fn(flow) { 241 | c.droppedFlowsCounter.WithLabelValues(flow.AsIp("sampler").String()).Inc() 242 | return 243 | } 244 | } 245 | for _, en := range c.enrichers { 246 | en.Enrich(flow) 247 | } 248 | for _, m := range c.metrics { 249 | m.apply(flow) 250 | } 251 | } 252 | 253 | func (c *col) mapMsg(msg *flowpb.FlowMessage) *public.Flow { 254 | f := &public.Flow{} 255 | f.AddAttr("source_ip", msg.SrcAddr) 256 | f.AddAttr("destination_ip", msg.DstAddr) 257 | if msg.SrcAs != 0 { 258 | f.AddAttr("source_as", msg.SrcAs) 259 | } 260 | if msg.DstAs != 0 { 261 | f.AddAttr("destination_as", msg.DstAs) 262 | } 263 | f.AddAttr("proto", msg.Proto) 264 | f.AddAttr("source_port", msg.SrcPort) 265 | f.AddAttr("destination_port", msg.DstPort) 266 | f.AddAttr("input_interface", msg.InIf) 267 | f.AddAttr("output_interface", msg.OutIf) 268 | f.AddAttr("next_hop", msg.NextHop) 269 | f.AddAttr("sampler", msg.SamplerAddress) 270 | f.AddAttr("bytes", msg.Bytes) 271 | f.AddAttr("packets", msg.Packets) 272 | return f 273 | } 274 | 275 | func New(cfg *public.Config, logger *slog.Logger) public.Collector { 276 | c := &col{ 277 | logger: logger, 278 | cfg: cfg, 279 | filters: []FlowMatcher{}, 280 | enrichers: []public.Enricher{}, 281 | metrics: []*metricEntry{}, 282 | } 283 | c.ready.Add(1) 284 | return c 285 | } 286 | -------------------------------------------------------------------------------- /pkg/collector/enrich.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Richard Kosegi 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package collector 16 | 17 | import ( 18 | "bufio" 19 | "context" 20 | "fmt" 21 | "io" 22 | "log/slog" 23 | "net" 24 | "os/exec" 25 | "strconv" 26 | "strings" 27 | "time" 28 | 29 | "github.com/jellydator/ttlcache/v3" 30 | "github.com/oschwald/geoip2-golang" 31 | "github.com/rkosegi/ipfix-collector/pkg/public" 32 | ) 33 | 34 | var ( 35 | localCidrsStr = []string{ 36 | "0.0.0.0/8,10.0.0.0/8,100.64.0.0/10,127.0.0.0/8", 37 | "169.254.0.0/16,172.16.0.0/12,192.0.0.0/24,192.0.2.0/24", 38 | "192.88.99.0/24,192.168.0.0/16,198.18.0.0/15,198.51.100.0/24", 39 | "203.0.113.0/24,224.0.0.0/4,233.252.0.0/24,240.0.0.0/4,255.255.255.255/32", 40 | } 41 | enrichers = map[string]public.Enricher{ 42 | "maxmind_country": &maxmindCountry{}, 43 | "maxmind_asn": &maxmindAsn{}, 44 | "interface_mapper": &interfaceName{}, 45 | "protocol_name": &protocolName{}, 46 | "reverse_dns": &reverseDNS{lookupRemote: true}, 47 | "host_alias": &enrichHostAlias{}, 48 | } 49 | localCidrs []*net.IPNet 50 | ) 51 | 52 | func init() { 53 | localCidrs = make([]*net.IPNet, 0) 54 | for _, s := range localCidrsStr { 55 | for _, ips := range strings.Split(s, ",") { 56 | _, ipnet, err := net.ParseCIDR(ips) 57 | if err == nil { 58 | localCidrs = append(localCidrs, ipnet) 59 | } 60 | } 61 | } 62 | } 63 | 64 | func getEnricher(name string) public.Enricher { 65 | return enrichers[name] 66 | } 67 | 68 | type maxmindCountry struct { 69 | logger *slog.Logger 70 | isOpen bool 71 | dir string 72 | db *geoip2.Reader 73 | } 74 | 75 | func (m *maxmindCountry) Configure(cfg map[string]interface{}) { 76 | if dir, ok := cfg["mmdb_dir"]; !ok { 77 | m.dir = "/usr/share/GeoIP" 78 | } else { 79 | m.dir = dir.(string) 80 | } 81 | m.logger = baseLogger.With("component", "geoip_country") 82 | m.logger.Info(fmt.Sprintf("using directory %s for Country GeoIP", m.dir)) 83 | } 84 | 85 | func (m *maxmindCountry) Close() error { 86 | if m.db != nil { 87 | return m.db.Close() 88 | } 89 | return nil 90 | } 91 | 92 | func isLocalIp(addr net.IP) bool { 93 | for _, cidr := range localCidrs { 94 | if cidr.Contains(addr) { 95 | return true 96 | } 97 | } 98 | return false 99 | } 100 | 101 | func (m *maxmindCountry) Enrich(flow *public.Flow) { 102 | if m.isOpen { 103 | sourceIp := flow.AsIp("source_ip") 104 | destIp := flow.AsIp("destination_ip") 105 | if isLocalIp(sourceIp) { 106 | flow.AddAttr("source_country", "local") 107 | } else { 108 | country, _ := m.db.Country(sourceIp) 109 | if country != nil { 110 | if len(country.Country.IsoCode) == 0 { 111 | country.Country.IsoCode = "Unknown" 112 | } 113 | flow.AddAttr("source_country", country.Country.IsoCode) 114 | } 115 | } 116 | if isLocalIp(destIp) { 117 | flow.AddAttr("destination_country", "local") 118 | } else { 119 | country, _ := m.db.Country(flow.AsIp("destination_ip")) 120 | if country != nil { 121 | if len(country.Country.IsoCode) == 0 { 122 | country.Country.IsoCode = "Unknown" 123 | } 124 | flow.AddAttr("destination_country", country.Country.IsoCode) 125 | } 126 | } 127 | } 128 | } 129 | 130 | func (m *maxmindCountry) Start() error { 131 | db, err := geoip2.Open(fmt.Sprintf("%s/GeoLite2-Country.mmdb", m.dir)) 132 | if err != nil { 133 | return err 134 | } 135 | m.isOpen = true 136 | m.db = db 137 | return nil 138 | } 139 | 140 | type interfaceName struct { 141 | mapping map[string]string 142 | } 143 | 144 | func (i *interfaceName) Close() error { 145 | return nil 146 | } 147 | 148 | func (i *interfaceName) Configure(cfg map[string]interface{}) { 149 | i.mapping = map[string]string{} 150 | for k, v := range cfg { 151 | i.mapping[k] = v.(string) 152 | } 153 | } 154 | 155 | func (i *interfaceName) Start() error { 156 | return nil 157 | } 158 | 159 | func (i *interfaceName) Enrich(flow *public.Flow) { 160 | ii := flow.AsUint32("input_interface") 161 | if ii != nil { 162 | if name, ok := i.mapping[strconv.FormatUint(uint64(*ii), 10)]; ok { 163 | flow.AddAttr("input_interface_name", name) 164 | } 165 | } 166 | 167 | ii = flow.AsUint32("output_interface") 168 | if ii != nil { 169 | if name, ok := i.mapping[strconv.FormatUint(uint64(*ii), 10)]; ok { 170 | flow.AddAttr("output_interface_name", name) 171 | } 172 | } 173 | } 174 | 175 | type protocolName struct { 176 | } 177 | 178 | func (p *protocolName) Close() error { 179 | return nil 180 | } 181 | 182 | func (p *protocolName) Configure(_ map[string]interface{}) { 183 | // not used in this enricher 184 | } 185 | 186 | func (p *protocolName) Start() error { 187 | return nil 188 | } 189 | 190 | func (p *protocolName) Enrich(flow *public.Flow) { 191 | protoName := "" 192 | proto := flow.AsUint32("proto") 193 | switch *proto { 194 | case 0x01: 195 | protoName = "icmp" 196 | 197 | case 0x02: 198 | protoName = "igmp" 199 | 200 | case 0x06: 201 | protoName = "tcp" 202 | 203 | case 0x11: 204 | protoName = "udp" 205 | 206 | default: 207 | protoName = fmt.Sprintf("other (%d)", *proto) 208 | } 209 | flow.AddAttr("proto_name", protoName) 210 | } 211 | 212 | type maxmindAsn struct { 213 | logger *slog.Logger 214 | isOpen bool 215 | dir string 216 | db *geoip2.Reader 217 | } 218 | 219 | func (m *maxmindAsn) Configure(cfg map[string]interface{}) { 220 | if dir, ok := cfg["mmdb_dir"]; !ok { 221 | m.dir = "/usr/share/GeoIP" 222 | } else { 223 | m.dir = dir.(string) 224 | } 225 | m.logger = baseLogger.With("component", "geoip_asn") 226 | m.logger.Info(fmt.Sprintf("using directory %s for ASN GeoIP", m.dir)) 227 | } 228 | 229 | func (m *maxmindAsn) Close() error { 230 | if m.db != nil { 231 | return m.db.Close() 232 | } 233 | return nil 234 | } 235 | 236 | func (m *maxmindAsn) Start() error { 237 | db, err := geoip2.Open(fmt.Sprintf("%s/GeoLite2-ASN.mmdb", m.dir)) 238 | if err != nil { 239 | return err 240 | } 241 | m.isOpen = true 242 | m.db = db 243 | return nil 244 | } 245 | 246 | func (m *maxmindAsn) Enrich(flow *public.Flow) { 247 | if m.isOpen { 248 | for _, dir := range []string{"source", "destination"} { 249 | ip := flow.AsIp(dir + "_ip") 250 | if !isLocalIp(ip) { 251 | asn, _ := m.db.ASN(ip) 252 | if asn != nil { 253 | if len(asn.AutonomousSystemOrganization) > 0 { 254 | flow.AddAttr(dir+"_asn_org", asn.AutonomousSystemOrganization) 255 | flow.AddAttr(dir+"_asn_num", asn.AutonomousSystemNumber) 256 | } 257 | } 258 | } 259 | } 260 | } 261 | } 262 | 263 | type reverseDNS struct { 264 | ttl time.Duration 265 | cache *ttlcache.Cache[string, string] 266 | 267 | tailPiHole bool 268 | piHoleResults *ttlcache.Cache[string, string] 269 | 270 | lookupLocal bool 271 | lookupRemote bool 272 | ipAsUnknown bool 273 | logger *slog.Logger 274 | } 275 | 276 | func (m *reverseDNS) Configure(cfg map[string]interface{}) { 277 | tailPiholeObject, ok := cfg["tail_pihole"] 278 | if ok { 279 | m.tailPiHole, ok = tailPiholeObject.(bool) 280 | if !ok { 281 | panic("tail_pihole (if specified) must be a boolean, e.g. true (or false)") 282 | } 283 | } 284 | 285 | lookupLocal, ok := cfg["lookup_local"] 286 | if ok { 287 | m.lookupLocal, ok = lookupLocal.(bool) 288 | if !ok { 289 | panic("lookup_local (if specified) must be a boolean, e.g. true (or false)") 290 | } 291 | } 292 | 293 | lookupRemote, ok := cfg["lookup_remote"] 294 | if ok { 295 | m.lookupRemote, ok = lookupRemote.(bool) 296 | if !ok { 297 | panic("lookup_remote (if specified) must be a boolean, e.g. true (or false)") 298 | } 299 | } 300 | 301 | ipAsUnknown, ok := cfg["ip_as_unknown"] 302 | if ok { 303 | m.ipAsUnknown, ok = ipAsUnknown.(bool) 304 | if !ok { 305 | panic("ip_as_unknown (if specified) must be a boolean, e.g. true (or false)") 306 | } 307 | } 308 | 309 | durationResult, ok := cfg["cache_duration"] 310 | if !ok { 311 | // if not found, set default 312 | m.ttl = time.Hour 313 | return 314 | } 315 | 316 | durationString, ok := durationResult.(string) 317 | if !ok { 318 | panic("cache_duration (if specified) must be a Go duration string, e.g. 1h") 319 | } 320 | 321 | var err error 322 | m.ttl, err = time.ParseDuration(durationString) 323 | if err != nil { 324 | panic("cache_duration (if specified) must be a Go duration string, e.g. 1h") 325 | } 326 | } 327 | 328 | func (m *reverseDNS) populateCacheWithPiHoleEntries(msgs io.Reader) { 329 | // cache to hold the results 330 | m.piHoleResults = ttlcache.New( 331 | ttlcache.WithTTL[string, string](m.ttl), // IP to name 332 | ) 333 | go m.piHoleResults.Start() 334 | 335 | // cache to hold the DNS masq query session 336 | dnsMasqCache := ttlcache.New( 337 | ttlcache.WithTTL[string, string](time.Minute), // session ID to original query 338 | ) 339 | go dnsMasqCache.Start() 340 | 341 | scanner := bufio.NewScanner(msgs) 342 | for scanner.Scan() { 343 | bits := strings.Split(scanner.Text(), " ") 344 | if len(bits) < 7 { 345 | continue 346 | } 347 | sessionId := bits[1] 348 | action := bits[3] 349 | switch { 350 | case strings.HasPrefix(action, "query"): 351 | dnsMasqCache.Set(sessionId, bits[4], ttlcache.DefaultTTL) 352 | case action == "cached", action == "reply": 353 | resultIP := bits[6] 354 | if bits[5] == "is" && resultIP != "" { 355 | origQuery := dnsMasqCache.Get(sessionId) 356 | if origQuery != nil { 357 | m.piHoleResults.Set(resultIP, origQuery.Value(), ttlcache.DefaultTTL) 358 | m.logger.Info("got entry", "tph-query", origQuery.Value(), "tph-result", resultIP) 359 | } 360 | } 361 | } 362 | } 363 | } 364 | 365 | func (m *reverseDNS) Close() error { 366 | return nil 367 | } 368 | 369 | func (m *reverseDNS) reverseLookup(ip net.IP) string { 370 | if isLocalIp(ip) { 371 | if !m.lookupLocal { 372 | return "local" 373 | } 374 | } else { 375 | if !m.lookupRemote { 376 | return "remote" 377 | } 378 | } 379 | 380 | s := ip.String() 381 | if m.tailPiHole { 382 | piHoleResult := m.piHoleResults.Get(s) 383 | if piHoleResult != nil { 384 | return piHoleResult.Value() 385 | } 386 | } 387 | return m.cache.Get(s).Value() 388 | } 389 | 390 | func (m *reverseDNS) Enrich(flow *public.Flow) { 391 | flow.AddAttr("source_dns", m.reverseLookup(flow.AsIp("source_ip"))) 392 | flow.AddAttr("destination_dns", m.reverseLookup(flow.AsIp("destination_ip"))) 393 | } 394 | 395 | func (m *reverseDNS) Start() error { 396 | m.logger = baseLogger.With("component", "reverse_dns") 397 | ctx := context.Background() 398 | m.cache = ttlcache.New( 399 | ttlcache.WithTTL[string, string](m.ttl), 400 | ttlcache.WithDisableTouchOnHit[string, string](), 401 | ttlcache.WithLoader[string, string](ttlcache.LoaderFunc[string, string]( 402 | func(c *ttlcache.Cache[string, string], key string) *ttlcache.Item[string, string] { 403 | m.logger.Debug("cache lookup", "key", key) 404 | result := "unknown" 405 | if m.ipAsUnknown { 406 | result = key 407 | } 408 | names, err := net.DefaultResolver.LookupAddr(ctx, key) 409 | if err == nil && len(names) != 0 { 410 | result = strings.TrimRight(names[0], ".") 411 | } 412 | m.logger.Debug("lookup result", "result", result) 413 | return c.Set(key, result, ttlcache.DefaultTTL) 414 | }, 415 | )), 416 | ) 417 | go m.cache.Start() 418 | 419 | if m.tailPiHole { 420 | m.logger.Info("tailing pihole") 421 | tph := exec.CommandContext(ctx, "pihole", "-t") 422 | stdout, err := tph.StdoutPipe() 423 | if err != nil { 424 | return err 425 | } 426 | err = tph.Start() 427 | if err != nil { 428 | return err 429 | } 430 | go m.populateCacheWithPiHoleEntries(stdout) 431 | } 432 | 433 | return nil 434 | } 435 | --------------------------------------------------------------------------------