├── docs ├── docs │ ├── .pages │ ├── salt-live │ │ ├── .pages │ │ ├── usage.md │ │ └── quickstart.md │ ├── demo │ │ ├── tui-usage.gif │ │ ├── tui-usage.webm │ │ ├── tui-overview.gif │ │ └── tui-overview.webm │ ├── salt-exporter │ │ ├── .pages │ │ ├── performance.md │ │ ├── quickstart.md │ │ ├── configuration.md │ │ └── metrics.md │ ├── stylesheets │ │ └── extra.css │ └── index.md ├── requirements.txt └── mkdocs.yml ├── e2e_test ├── states │ └── test │ │ ├── fail.sls │ │ └── succeed.sls ├── exec_commands.sh ├── docker-compose.demo.yaml ├── docker-compose.yaml └── e2e_test.go ├── Dockerfile ├── tui_overview_demo.tape ├── internal ├── tui │ ├── utils.go │ ├── styles.go │ ├── item.go │ ├── filters.go │ ├── keymap.go │ └── tui.go ├── logging │ └── logging.go ├── filters │ ├── filters_test.go │ └── filters.go └── metrics │ ├── config.go │ ├── metrics.go │ └── registry.go ├── .gitignore ├── record_demo.sh ├── .github ├── FUNDING.yml └── workflows │ ├── goreleaser.yml │ ├── doc.yml │ └── go.yml ├── cmd ├── salt-exporter │ ├── flag.go │ ├── config_test.yml │ ├── main.go │ ├── config.go │ └── config_test.go └── salt-live │ └── main.go ├── prometheus_alerts └── highstate.yaml ├── LICENSE ├── .goreleaser.yaml ├── pkg ├── parser │ ├── fake_data_test.go │ ├── fake_beacon_data_test.go │ ├── parser_test.go │ ├── parser.go │ ├── fake_exec_data_test.go │ └── fake_state_data_test.go ├── event │ ├── event_test.go │ └── event.go └── listener │ ├── pkiwatcher.go │ └── listener.go ├── tui_usage_demo.tape ├── .golangci.yml ├── go.mod ├── README.md ├── go.sum └── grafana └── Saltstack-1682711767600.json /docs/docs/.pages: -------------------------------------------------------------------------------- 1 | nav: -------------------------------------------------------------------------------- /docs/docs/salt-live/.pages: -------------------------------------------------------------------------------- 1 | nav: 2 | - quickstart.md 3 | - ... -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs-material 2 | mkdocs-awesome-pages-plugin 3 | mkdocs-glightbox -------------------------------------------------------------------------------- /docs/docs/demo/tui-usage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kpetremann/salt-exporter/HEAD/docs/docs/demo/tui-usage.gif -------------------------------------------------------------------------------- /docs/docs/demo/tui-usage.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kpetremann/salt-exporter/HEAD/docs/docs/demo/tui-usage.webm -------------------------------------------------------------------------------- /e2e_test/states/test/fail.sls: -------------------------------------------------------------------------------- 1 | --- 2 | fail: 3 | test.fail_with_changes: 4 | - name: "It's ok to fail" 5 | -------------------------------------------------------------------------------- /docs/docs/demo/tui-overview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kpetremann/salt-exporter/HEAD/docs/docs/demo/tui-overview.gif -------------------------------------------------------------------------------- /e2e_test/states/test/succeed.sls: -------------------------------------------------------------------------------- 1 | succeed: 2 | test.succeed_with_changes: 3 | - name: "Always a success" 4 | -------------------------------------------------------------------------------- /docs/docs/demo/tui-overview.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kpetremann/salt-exporter/HEAD/docs/docs/demo/tui-overview.webm -------------------------------------------------------------------------------- /docs/docs/salt-exporter/.pages: -------------------------------------------------------------------------------- 1 | nav: 2 | - quickstart.md 3 | - configuration.md 4 | - metrics.md 5 | - ... 6 | - performance.md -------------------------------------------------------------------------------- /docs/docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | [data-md-color-scheme="default"] { 2 | --md-primary-fg-color: #711d1d; 3 | --md-accent-fg-color: #A13939; 4 | --md-typeset-a-color: #c45858; 5 | } 6 | 7 | [data-md-color-scheme="slate"] { 8 | --md-primary-fg-color: #A13939; 9 | --md-accent-fg-color: #c45858; 10 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:bookworm AS builder 2 | 3 | WORKDIR /go/src/ 4 | COPY ./ /go/src/ 5 | RUN mkdir build 6 | RUN go build -o /go/src/build/salt-exporter 7 | 8 | FROM debian:bullseye-slim AS runner 9 | WORKDIR /app/salt-exporter/ 10 | COPY --from=builder /go/src/build/salt-exporter ./ 11 | CMD ["/app/salt-exporter/salt-exporter"] 12 | -------------------------------------------------------------------------------- /e2e_test/exec_commands.sh: -------------------------------------------------------------------------------- 1 | # execution module 2 | salt foo test.true 3 | salt foo test.exception 4 | 5 | # state module 6 | salt foo state.single test.succeed_with_changes name="succeed" 7 | salt foo state.single test.fail_with_changes name="fails" 8 | 9 | # state 10 | salt foo state.sls test.succeed 11 | salt foo state.sls test.fail 12 | 13 | exit 0 -------------------------------------------------------------------------------- /tui_overview_demo.tape: -------------------------------------------------------------------------------- 1 | Output docs/docs/demo/tui-overview.gif 2 | Output docs/docs/demo/tui-overview.webm 3 | 4 | Set Padding 0 5 | 6 | Set FontSize 16 7 | Set Framerate 60 8 | 9 | Set Width 1000 10 | Set Height 600 11 | 12 | # Start 13 | Hide 14 | Type@0s "SALT_DEMO=true ./salt-live -ipc-file ./e2e_test/ipc.ignore/master_event_pub.ipc -hard-filter='!ping_master'" 15 | Enter 16 | Sleep 5s 17 | Show 18 | 19 | # Admire the output for a bit 20 | Sleep 3s 21 | 22 | Down@1s 3 23 | 24 | Sleep 1s 25 | -------------------------------------------------------------------------------- /internal/tui/utils.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/alecthomas/chroma/quick" 8 | ) 9 | 10 | const nbFormat = 3 11 | const ( 12 | YAML format = iota 13 | JSON 14 | PARSED 15 | ) 16 | 17 | func Highlight(content, extension, syntaxTheme string) (string, error) { 18 | buf := new(bytes.Buffer) 19 | if err := quick.Highlight(buf, content, extension, "terminal256", syntaxTheme); err != nil { 20 | return "", fmt.Errorf("%w", err) 21 | } 22 | 23 | return buf.String(), nil 24 | } 25 | -------------------------------------------------------------------------------- /docs/docs/salt-exporter/performance.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Performance 3 | --- 4 | 5 | # Estimated performance 6 | 7 | According to a simple benchmark, for a single event it takes: 8 | 9 | * ~60µs for parsing 10 | * ~9µs for converting to Prometheus metric 11 | 12 | With a security margin, we can estimate processing an event should take 100µs maximum. 13 | 14 | Roughly, the exporter should be able to handle about 10kQps. 15 | 16 | For a base of 1000 Salt minions, it should be able to sustain 10 jobs per minion per second, which is quite high for Salt. 17 | -------------------------------------------------------------------------------- /e2e_test/docker-compose.demo.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | salt_master: 3 | volumes: 4 | - ../e2e_test/:/test/:ro 5 | - ./states:/srv/salt/:ro 6 | - ./ipc.ignore:/var/run/salt/master/ 7 | 8 | exporter: 9 | volumes: 10 | - ../:/app/:ro 11 | - ./ipc.ignore:/var/run/salt/master/:ro 12 | 13 | # recorder: 14 | # image: ghcr.io/charmbracelet/vhs:v0.5.1-devel 15 | # entrypoint: tail -f /dev/null 16 | # working_dir: /app 17 | # user: "${UID}:${GID}" 18 | # volumes: 19 | # - ../:/app/:rw 20 | # - ipc:/var/run/salt/master/:ro 21 | # depends_on: 22 | # minion: 23 | # condition: service_healthy 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # Visual Studio Code 18 | .vscode 19 | 20 | # Tagged file 21 | *.ignore 22 | *.patch 23 | 24 | # Logs 25 | debug.log 26 | 27 | # Artifacts 28 | dist/ 29 | /salt-exporter 30 | /salt-live 31 | !/salt-exporter/ 32 | !/salt-live/ 33 | 34 | # Build 35 | .venv 36 | 37 | # Configuration 38 | settings.yml 39 | settings.yaml 40 | 41 | -------------------------------------------------------------------------------- /record_demo.sh: -------------------------------------------------------------------------------- 1 | CGO_ENABLED=0 go build ./cmd/salt-live 2 | docker compose -f ./e2e_test/docker-compose.yaml -f ./e2e_test/docker-compose.demo.yaml up -d --wait 3 | 4 | docker compose -f ./e2e_test/docker-compose.yaml -f ./e2e_test/docker-compose.demo.yaml exec -d salt_master sh -c 'sleep 3 && sh /test/exec_commands.sh' 5 | ~/go/bin/vhs tui_usage_demo.tape 6 | 7 | docker compose -f ./e2e_test/docker-compose.yaml -f ./e2e_test/docker-compose.demo.yaml exec -d salt_master sh -c 'sleep 3 && sh /test/exec_commands.sh' 8 | ~/go/bin/vhs tui_overview_demo.tape 9 | 10 | docker compose -f ./e2e_test/docker-compose.yaml -f ./e2e_test/docker-compose.demo.yaml down 11 | rm ./salt-live 12 | -------------------------------------------------------------------------------- /internal/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/rs/zerolog" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | // Configure load log default configuration (like format, output target etc...). 12 | func Configure() { 13 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnix 14 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 15 | } 16 | 17 | // SetLogLevel configures the loglevel. 18 | // 19 | // logLevel: The log level to use, in zerolog format. 20 | func SetLevel(logLevel string) { 21 | level, err := zerolog.ParseLevel(logLevel) 22 | fmt.Println(logLevel) 23 | if err != nil { 24 | fmt.Println("Failed to parse log level") 25 | os.Exit(1) 26 | } 27 | zerolog.SetGlobalLevel(level) 28 | } 29 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [kpetremann] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /cmd/salt-exporter/flag.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "reflect" 6 | 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | type viperFlag struct { 11 | original flag.Flag 12 | alias string 13 | } 14 | 15 | func (f viperFlag) HasChanged() bool { return true } // TODO: fix? 16 | 17 | func (f viperFlag) Name() string { 18 | if f.alias != "" { 19 | return f.alias 20 | } 21 | return f.original.Name 22 | } 23 | 24 | func (f viperFlag) ValueString() string { return f.original.Value.String() } 25 | 26 | func (f viperFlag) ValueType() string { 27 | t := reflect.TypeOf(f.original.Value) 28 | if t.Kind() == reflect.Ptr { 29 | return t.Elem().Kind().String() 30 | } 31 | return t.Kind().String() 32 | } 33 | 34 | type viperFlagSet struct { 35 | flags []viperFlag 36 | } 37 | 38 | func (f viperFlagSet) VisitAll(fn func(viper.FlagValue)) { 39 | for _, flag := range f.flags { 40 | fn(flag) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /prometheus_alerts/highstate.yaml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: saltstack 3 | rules: 4 | - alert: SaltExporterLastHighstateSuccess 5 | expr: sum by(minion) (salt_function_health{function="state.highstate", state="highstate"} == 0) 6 | for: 60m 7 | labels: 8 | severity: critical 9 | minion: "{{ $labels.minion }}" 10 | annotations: 11 | summary: "Salt Last Successful Highstate Failed (minion {{ $labels.minion }})" 12 | description: "Salt Last Successful Highstate failed since > 60m" 13 | - alert: SaltExporterLastHighstateSuccessInfo 14 | expr: sum by(minion) (salt_function_health{function="state.highstate", state="highstate"} == 0) 15 | for: 10m 16 | labels: 17 | severity: info 18 | minion: "{{ $labels.minion }}" 19 | annotations: 20 | summary: "Salt Last Successful Highstate Failed (minion {{ $labels.minion }})" -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | # run only against tags 6 | tags: 7 | - '*' 8 | 9 | permissions: 10 | contents: write 11 | # packages: write 12 | # issues: write 13 | 14 | jobs: 15 | goreleaser: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - 19 | name: Checkout 20 | uses: actions/checkout@v5 21 | with: 22 | fetch-depth: 0 23 | - 24 | name: Fetch all tags 25 | run: git fetch --force --tags 26 | - 27 | name: Set up Go 28 | uses: actions/setup-go@v6 29 | with: 30 | go-version-file: 'go.mod' 31 | - 32 | name: Run GoReleaser 33 | uses: goreleaser/goreleaser-action@v4 34 | with: 35 | distribution: goreleaser 36 | version: latest 37 | args: release --clean 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | -------------------------------------------------------------------------------- /cmd/salt-exporter/config_test.yml: -------------------------------------------------------------------------------- 1 | listen-address: "127.0.0.1" 2 | listen-port: 2113 3 | 4 | ipc-file: /dev/null 5 | pki-dir: /tmp/pki 6 | 7 | log-level: "info" 8 | tls: 9 | enabled: true 10 | key: "/path/to/key" 11 | certificate: "/path/to/certificate" 12 | 13 | metrics: 14 | global: 15 | filters: 16 | ignore-test: true 17 | ignore-mock: false 18 | 19 | salt_new_job_total: 20 | enabled: true 21 | 22 | salt_expected_responses_total: 23 | enabled: true 24 | 25 | salt_function_responses_total: 26 | enabled: true 27 | add-minion-label: true # not recommended in production 28 | 29 | salt_scheduled_job_return_total: 30 | enabled: true 31 | add-minion-label: true # not recommended in production 32 | 33 | salt_responses_total: 34 | enabled: true 35 | 36 | salt_function_status: 37 | enabled: true 38 | filters: 39 | functions: 40 | - "state.sls" 41 | states: 42 | - "test" -------------------------------------------------------------------------------- /internal/tui/styles.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import "github.com/charmbracelet/lipgloss" 4 | 5 | var ( 6 | appTitleStyle = lipgloss.NewStyle(). 7 | Padding(0, 2). 8 | Foreground(lipgloss.Color("#FFFDF5")). 9 | Background(lipgloss.Color("#255aa0")). 10 | Border(lipgloss.NormalBorder()). 11 | BorderForeground(lipgloss.Color("#255aa0")). 12 | BorderBackground(lipgloss.Color("#255aa0")) 13 | 14 | topBarStyle = lipgloss.NewStyle().Padding(0, 1). 15 | BorderStyle(lipgloss.InnerHalfBlockBorder()). 16 | BorderBottom(true).BorderForeground(lipgloss.Color("#255aa0")) 17 | 18 | listTitleStyle = lipgloss.NewStyle(). 19 | Padding(0, 1). 20 | MarginLeft(2). 21 | Border(lipgloss.RoundedBorder()) 22 | 23 | leftPanelStyle = lipgloss.NewStyle().Padding(0, 1) 24 | 25 | rightPanelTitleStyle = lipgloss.NewStyle().Padding(0, 1).MarginBottom(1). 26 | Border(lipgloss.RoundedBorder()) 27 | 28 | rightPanelStyle = lipgloss.NewStyle(). 29 | Padding(0, 2). 30 | BorderStyle(lipgloss.NormalBorder()).BorderLeft(true).BorderForeground(lipgloss.Color("#255aa0")) 31 | ) 32 | -------------------------------------------------------------------------------- /internal/tui/item.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | ) 8 | 9 | type item struct { 10 | title string 11 | description string 12 | event event.SaltEvent 13 | datetime string 14 | sender string 15 | state string 16 | eventJSON string 17 | eventYAML string 18 | } 19 | 20 | func (i item) Title() string { 21 | if i.event.Data.Retcode > 0 { 22 | return fmt.Sprintf("/!\\ %s", i.event.Tag) 23 | } else { 24 | return i.event.Tag 25 | } 26 | } 27 | 28 | func (i item) Description() string { 29 | out := fmt.Sprintf("%s - %s - %s", i.datetime, i.sender, i.event.Data.Fun) 30 | if i.state != "" { 31 | out = fmt.Sprintf("%s %s", out, i.state) 32 | } 33 | if i.event.TargetNumber > 0 { 34 | target := "targets" 35 | if i.event.TargetNumber == 1 { 36 | target = "target" 37 | } 38 | out = fmt.Sprintf("%s - %d %s", out, i.event.TargetNumber, target) 39 | } 40 | return out 41 | } 42 | func (i item) FilterValue() string { 43 | return i.title + " " + i.Description() + " " + i.eventJSON 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Kevin Petremann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | 5 | builds: 6 | - id: salt-exporter 7 | binary: salt-exporter 8 | main: ./cmd/salt-exporter 9 | env: 10 | - CGO_ENABLED=0 11 | goos: 12 | - linux 13 | - id: salt-live 14 | binary: salt-live 15 | main: ./cmd/salt-live 16 | env: 17 | - CGO_ENABLED=0 18 | goos: 19 | - linux 20 | 21 | archives: 22 | - name_template: "{{ .ProjectName }}_{{ .Version }}_{{- title .Os }}_{{ .Arch }}" 23 | 24 | checksum: 25 | name_template: 'checksums.txt' 26 | 27 | snapshot: 28 | name_template: "{{ incpatch .Version }}-next" 29 | 30 | changelog: 31 | use: github 32 | sort: asc 33 | abbrev: -1 34 | filters: 35 | exclude: 36 | - '^docs:' 37 | - '^test:' 38 | groups: 39 | - title: 'Breaking changes' 40 | regexp: '^.*?(\([[:word:]]+\))??!:.+$' 41 | order: 0 42 | - title: 'Enhancements' 43 | regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' 44 | order: 1 45 | - title: 'Fixes' 46 | regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' 47 | order: 2 48 | - title: 'Internal' 49 | regexp: '^.+$' 50 | order: 999 51 | -------------------------------------------------------------------------------- /docs/docs/salt-live/usage.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - toc 4 | --- 5 | # Usage 6 | 7 | ## Tutorial 8 | 9 | [![tui.gif](../demo/tui-usage.gif)](../demo/tui-usage.webm) 10 | 11 | ## Hard filter 12 | 13 | You can run `Salt Live` with the `-hard-filter` flag. 14 | 15 | Unlike the filter in the TUI (using ++slash++), all events not matching the filter are definitely discarded. 16 | 17 | ## Keyboard shortcuts 18 | 19 | | Key | Effect | 20 | |-------------------|-----------------------------------------------------------------------| 21 | | ++q++ | Exit. | 22 | | ++slash++ | Display the prompt to edit the filter. | 23 | | ++up++ / ++down++ | Navigate in the list. This stops the refresh of the list. | 24 | | ++f++ | Follow mode: resume the refresh of the event list. | 25 | | ++m++ | Change output format of the side panel (YAML, JSON, Golang structure).| 26 | | ++w++ | Toggle word wrap (only in JSON mode). | 27 | -------------------------------------------------------------------------------- /pkg/parser/fake_data_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/vmihailenco/msgpack/v5" 7 | ) 8 | 9 | type FakeData struct { 10 | Arg []interface{} `msgpack:"arg"` 11 | Cmd string `msgpack:"cmd"` 12 | Fun string `msgpack:"fun"` 13 | FunArgs []interface{} `msgpack:"fun_args"` 14 | ID string `msgpack:"id"` 15 | Jid string `msgpack:"jid"` 16 | Minions []string `msgpack:"minions"` 17 | Missing []string `msgpack:"missing"` 18 | Retcode int `msgpack:"retcode"` 19 | Return interface{} `msgpack:"return"` 20 | Schedule string `msgpack:"schedule"` 21 | Success bool `msgpack:"success"` 22 | Tgt interface{} `msgpack:"tgt"` 23 | TgtType string `msgpack:"tgt_type"` 24 | Timestamp string `msgpack:"_stamp"` 25 | User string `msgpack:"user"` 26 | Out string `msgpack:"out"` 27 | } 28 | 29 | func fakeEventAsMap(event []byte) map[string]interface{} { 30 | var m interface{} 31 | 32 | if err := msgpack.Unmarshal(event, &m); err != nil { 33 | log.Fatalln(err) 34 | } 35 | 36 | return map[string]interface{}{"body": event} 37 | } 38 | -------------------------------------------------------------------------------- /internal/filters/filters_test.go: -------------------------------------------------------------------------------- 1 | package filters_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/kpetremann/salt-exporter/internal/filters" 7 | ) 8 | 9 | func TestMatchesFilters(t *testing.T) { 10 | var tests = []struct { 11 | value string 12 | filters []string 13 | want bool 14 | }{ 15 | {"foo", []string{"foo"}, true}, 16 | {"foo", []string{"bar"}, false}, 17 | {"foo", []string{"*"}, true}, 18 | {"foo", []string{"*o"}, true}, 19 | {"foo", []string{"f*"}, true}, 20 | {"foo", []string{"*o*"}, true}, 21 | {"foo", []string{"foo", "bar"}, true}, 22 | {"foo", []string{"bar", "baz"}, false}, 23 | {"foo", []string{"*o", "bar"}, true}, 24 | {"foo", []string{"bar", "*o"}, true}, 25 | {"test.ping", []string{"test"}, false}, 26 | {"test.ping", []string{"test.*"}, true}, 27 | {"test.ping", []string{"test.ping*"}, true}, 28 | {"state.sls", []string{"state.*"}, true}, 29 | {"state.sls", []string{"state.*", "test.ping"}, true}, 30 | {"state.sls", []string{"state", "test.ping"}, false}, 31 | } 32 | 33 | for _, tt := range tests { 34 | got := filters.Match(tt.value, tt.filters) 35 | if got != tt.want { 36 | t.Errorf("'MatchFilters(%q, %q)' wants '%v' got '%v'", tt.value, tt.filters, tt.want, got) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /e2e_test/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | salt_master: 3 | image: saltstack/salt:3006.4 4 | environment: 5 | SALT_MASTER_CONFIG: '{"interface": "0.0.0.0", "auto_accept": true}' 6 | volumes: 7 | - ../e2e_test/:/test/:ro 8 | - ./states:/srv/salt/:ro 9 | - ipc:/var/run/salt/master/ 10 | networks: 11 | - e2e 12 | 13 | minion: 14 | image: saltstack/salt:3006.4 15 | environment: 16 | SALT_MINION_CONFIG: '{"id": "foo", "master": "salt_master"}' 17 | healthcheck: 18 | test: ["CMD-SHELL", "salt-call status.ping_master salt_master"] 19 | interval: 1s 20 | start_period: 1s 21 | networks: 22 | - e2e 23 | 24 | exporter: 25 | image: golang:bookworm 26 | command: "go run ./cmd/salt-exporter" 27 | working_dir: "/app" 28 | environment: 29 | CGO_ENABLED: 0 30 | volumes: 31 | - ../:/app/:ro 32 | - ipc:/var/run/salt/master/:ro 33 | ports: 34 | - 127.0.0.1:2112:2112 35 | healthcheck: 36 | test: ["CMD-SHELL", "curl --fail http://127.0.0.1:2112/metrics"] 37 | interval: 1s 38 | retries: 60 39 | start_period: 1s 40 | depends_on: 41 | minion: 42 | condition: service_healthy 43 | 44 | volumes: 45 | ipc: 46 | 47 | networks: 48 | e2e: 49 | -------------------------------------------------------------------------------- /tui_usage_demo.tape: -------------------------------------------------------------------------------- 1 | Output docs/docs/demo/tui-usage.gif 2 | Output docs/docs/demo/tui-usage.webm 3 | 4 | Set Padding 0 5 | 6 | Set FontSize 16 7 | Set Framerate 60 8 | 9 | Set Width 1000 10 | Set Height 600 11 | 12 | # Start 13 | Hide 14 | Type@0s "SALT_DEMO=true ./salt-live -ipc-file ./e2e_test/ipc.ignore/master_event_pub.ipc -hard-filter='!ping_master'" 15 | Enter 16 | Sleep 100ms 17 | Type "$" 18 | Sleep 100ms 19 | 20 | Show 21 | Type@50ms "Simply start salt-live" 22 | Sleep 2s 23 | Type "$" 24 | 25 | # Admire the output for a bit 26 | Sleep 6s 27 | 28 | # Navigate 29 | Type@60ms "$Use the arrows to navigate in the event list" 30 | Sleep 3s 31 | Type "$" 32 | Sleep 1s 33 | Down@1s 3 34 | Sleep 1s 35 | Up 1 36 | Sleep 2s 37 | 38 | Type@60ms "$While navigating, the list is frozen. Press 'f' to get the events in real time again." 39 | Sleep 3s 40 | Type "$" 41 | Sleep 1s 42 | Type "f" 43 | Sleep 1s 44 | 45 | # Switch side view output format 46 | Type@60ms "$Press 'm' to change the output format of the event details on the right" 47 | Sleep 3s 48 | Type "$" 49 | Hide 50 | Down 1 51 | Up 1 52 | Show 53 | Sleep 1s 54 | Type@2s "mmmm" 55 | Sleep 2s 56 | 57 | # Filter 58 | Type@60ms "$Press '/' to filter the events" 59 | Sleep 3s 60 | Type "$" 61 | Sleep 1s 62 | Type '/' 63 | Type@10ms 'foo state.sls' 64 | Sleep 2s 65 | Enter 66 | 67 | Sleep 5s 68 | -------------------------------------------------------------------------------- /internal/metrics/config.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | type Config struct { 4 | // HealtMinions enable/disable the health functions/states metrics 5 | HealthMinions bool `mapstructure:"health-minions"` 6 | 7 | Global struct { 8 | Filters struct { 9 | IgnoreTest bool `mapstructure:"ignore-test"` 10 | IgnoreMock bool `mapstructure:"ignore-mock"` 11 | } 12 | } 13 | 14 | /* 15 | New job metrics 16 | */ 17 | 18 | SaltNewJobTotal struct { 19 | Enabled bool 20 | } `mapstructure:"salt_new_job_total"` 21 | 22 | SaltExpectedResponsesTotal struct { 23 | Enabled bool 24 | } `mapstructure:"salt_expected_responses_total"` 25 | 26 | /* 27 | Response metrics 28 | */ 29 | 30 | SaltFunctionResponsesTotal struct { 31 | Enabled bool 32 | AddMinionLabel bool `mapstructure:"add-minion-label"` 33 | } `mapstructure:"salt_function_responses_total"` 34 | 35 | SaltScheduledJobReturnTotal struct { 36 | Enabled bool 37 | AddMinionLabel bool `mapstructure:"add-minion-label"` 38 | } `mapstructure:"salt_scheduled_job_return_total"` 39 | 40 | SaltResponsesTotal struct { 41 | Enabled bool 42 | } `mapstructure:"salt_responses_total"` 43 | 44 | SaltFunctionStatus struct { 45 | Enabled bool 46 | Filters struct { 47 | Functions []string 48 | States []string 49 | } 50 | } `mapstructure:"salt_function_status"` 51 | } 52 | -------------------------------------------------------------------------------- /internal/filters/filters.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func matchTerm(s string, pattern string) bool { 8 | switch { 9 | case s == pattern: 10 | return true 11 | 12 | case pattern == "*": 13 | return true 14 | 15 | case strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*"): 16 | return strings.Contains(s, pattern[1:len(pattern)-1]) 17 | 18 | case strings.HasPrefix(pattern, "*"): 19 | return strings.HasSuffix(s, pattern[1:]) 20 | 21 | case strings.HasSuffix(pattern, "*"): 22 | return strings.HasPrefix(s, pattern[:len(pattern)-1]) 23 | 24 | default: 25 | return false 26 | } 27 | } 28 | 29 | // Function to check if a string exists in a slice of strings 30 | // 31 | // It supports the following wildcards as a prefix or suffix. 32 | // Wildcard is not supported in the middle of the string. 33 | // 34 | // Examples: 35 | // - Match("foo", []string{"foo"}) -> true 36 | // - Match("foo", []string{"bar"}) -> false 37 | // - Match("foo", []string{"*"}) -> true 38 | // - Match("foo", []string{"*o"}) -> true 39 | // - Match("foo", []string{"f*"}) -> true 40 | // - Match("foo", []string{"*f"}) -> false 41 | // - Match("foo", []string{"*o*"}) -> true 42 | func Match(value string, filters []string) bool { 43 | for _, pattern := range filters { 44 | if matchTerm(value, pattern) { 45 | return true 46 | } 47 | } 48 | return false 49 | } 50 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: Documentation generation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: write 13 | 14 | jobs: 15 | build-doc: 16 | 17 | runs-on: ubuntu-latest 18 | 19 | strategy: 20 | matrix: 21 | python-version: ["3.13"] 22 | 23 | steps: 24 | - uses: actions/checkout@v5 25 | 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v5 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | 31 | - name: Install dependencies 32 | run: | 33 | python -m pip install --upgrade pip 34 | pip install -r docs/requirements.txt 35 | 36 | - name: Generate doc 37 | run: | 38 | cd docs 39 | mkdocs build 40 | cp -R site/ ~/html 41 | cd .. 42 | 43 | - name: Deploy doc 44 | if: github.ref == 'refs/heads/main' 45 | run: | 46 | git config --local user.email "action@github.com" 47 | git config --local user.name "GitHub Action" 48 | git fetch origin gh-pages && git checkout gh-pages 49 | rm -rf * 50 | cp -R ~/html/* . 51 | touch .nojekyll 52 | git add . 53 | git commit --allow-empty -m "update doc" 54 | git push 55 | -------------------------------------------------------------------------------- /docs/docs/salt-live/quickstart.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Quickstart 3 | --- 4 | 5 | # Salt Live 6 | 7 | ## Quickstart 8 | 9 | `Salt Live` is a Terminal UI tool to watch events in real time. 10 | 11 | It includes the following features: 12 | 13 | * Hard filter from the CLI: filtered out events are discarded forever. 14 | * Soft filter from the TUI: filtered out events are still kept in the buffer. 15 | * Event details can be displayed in: 16 | * YAML 17 | * JSON 18 | * Golang structure 19 | * The list is frozen when navigating the events. 20 | * It prevents annoying list updates when checking event details. 21 | * New events are still received and kept in the buffer. 22 | * Once the freeze is removed, the events are displayed in real-time. 23 | 24 | ## Installation 25 | 26 | You can download the binary from the [Github releases](https://github.com/kpetremann/salt-exporter/releases) page. 27 | 28 | Or install from source: 29 | 30 | * latest published version: 31 | ``` { .sh .copy } 32 | go install github.com/kpetremann/salt-exporter/cmd/salt-live@latest 33 | ``` 34 | * latest commit (unstable): 35 | ``` { .sh .copy } 36 | go install github.com/kpetremann/salt-exporter/cmd/salt-live@main 37 | ``` 38 | 39 | ## Credits 40 | 41 | This tool uses these amazing libraries: 42 | 43 | * [Bubble tea](https://github.com/charmbracelet/bubbletea) 44 | * [Bubbles](https://github.com/charmbracelet/bubbles) 45 | -------------------------------------------------------------------------------- /internal/tui/filters.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/charmbracelet/bubbles/list" 7 | ) 8 | 9 | const negativePrefix = "!" 10 | 11 | // Filter via multiple fields and keywords. 12 | func WordsFilter(term string, targets []string) []list.Rank { 13 | var ranks []int 14 | 15 | term = strings.ToLower(term) 16 | splitedTerm := strings.Split(term, " ") 17 | 18 | for i, target := range targets { 19 | match := true 20 | 21 | for _, word := range splitedTerm { 22 | // check if excluding a substring 23 | negated := strings.HasPrefix(word, negativePrefix) 24 | if negated { 25 | word = strings.TrimPrefix(word, negativePrefix) 26 | } 27 | 28 | // we ignore empty terms 29 | if word == "" { 30 | continue 31 | } 32 | 33 | // look for the substring 34 | matching := strings.Contains(strings.ToLower(target), word) 35 | 36 | // if excluding the substring, invert the result 37 | if negated { 38 | matching = !matching 39 | } 40 | 41 | // if one of the words is not matching, the whole target is not matching 42 | if !matching { 43 | match = false 44 | break 45 | } 46 | } 47 | 48 | if match { 49 | ranks = append(ranks, i) 50 | } 51 | } 52 | 53 | result := make([]list.Rank, len(ranks)) 54 | for i, indexMatching := range ranks { 55 | result[i] = list.Rank{ 56 | Index: indexMatching, 57 | MatchedIndexes: []int{}, 58 | } 59 | } 60 | return result 61 | } 62 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | enable: 4 | - asasalint 5 | - asciicheck 6 | - bodyclose 7 | - dupword 8 | - durationcheck 9 | - errname 10 | - errorlint 11 | - exhaustive 12 | - gocheckcompilerdirectives 13 | - gocritic 14 | - godot 15 | - goprintffuncname 16 | - gosec 17 | - grouper 18 | - makezero 19 | - nilerr 20 | - nilnil 21 | - nolintlint 22 | - nosprintfhostport 23 | - prealloc 24 | - predeclared 25 | - reassign 26 | - staticcheck 27 | - testableexamples 28 | - thelper 29 | - tparallel 30 | - unconvert 31 | - unparam 32 | - usestdlibvars 33 | - wastedassign 34 | - whitespace 35 | - zerologlint 36 | exclusions: 37 | generated: lax 38 | presets: 39 | - comments 40 | - common-false-positives 41 | - legacy 42 | - std-error-handling 43 | rules: 44 | - linters: 45 | - goconst 46 | path: (.+)_test\.go 47 | - linters: 48 | - govet 49 | path: (.+)_test\.go 50 | text: 'fieldalignment: .*' 51 | paths: 52 | - third_party$ 53 | - builtin$ 54 | - examples$ 55 | issues: 56 | max-issues-per-linter: 0 57 | max-same-issues: 0 58 | fix: false 59 | formatters: 60 | enable: 61 | - gofmt 62 | - goimports 63 | exclusions: 64 | generated: lax 65 | paths: 66 | - third_party$ 67 | - builtin$ 68 | - examples$ 69 | -------------------------------------------------------------------------------- /pkg/parser/fake_beacon_data_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | "github.com/vmihailenco/msgpack/v5" 8 | ) 9 | 10 | /* 11 | Fake new beacon message of type /status 12 | 13 | salt/beacon/host1.example.com/status/2023-10-09T11:36:02.182345 { 14 | { 15 | "id": "host1.example.com", 16 | "data": { 17 | "loadavg": { 18 | "1-min": 0.35, 19 | "5-min": 0.48, 20 | "15-min": 0.26 21 | } 22 | }, 23 | "_stamp": "2023-10-09T11:36:02.205686" 24 | } 25 | } 26 | */ 27 | var expectedBeacon = event.SaltEvent{ 28 | Tag: "salt/beacon/host1.example.com/status/2023-10-09T11:36:02.182345", 29 | Type: "status", 30 | Module: event.BeaconModule, 31 | TargetNumber: 0, 32 | Data: event.EventData{ 33 | Timestamp: "2023-10-09T11:36:02.205686", 34 | ID: "host1.example.com", 35 | Minions: []string{}, 36 | }, 37 | IsScheduleJob: false, 38 | } 39 | 40 | func fakeBeaconEvent() []byte { 41 | // Marshal the data using MsgPack 42 | fake := FakeData{ 43 | Timestamp: "2023-10-09T11:36:02.205686", 44 | Minions: []string{}, 45 | ID: "host1.example.com", 46 | } 47 | 48 | fakeBody, err := msgpack.Marshal(fake) 49 | if err != nil { 50 | log.Fatalln(err) 51 | } 52 | 53 | fakeMessage := []byte("salt/beacon/host1.example.com/status/2023-10-09T11:36:02.182345\n\n") 54 | fakeMessage = append(fakeMessage, fakeBody...) 55 | 56 | return fakeMessage 57 | } 58 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | COMPOSE_FILE: e2e_test/docker-compose.yaml 11 | 12 | jobs: 13 | tests: 14 | name: "Lint and test" 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v5 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v6 21 | with: 22 | go-version-file: 'go.mod' 23 | 24 | - name: Run golangci-lint 25 | uses: golangci/golangci-lint-action@v9 26 | 27 | - name: Test 28 | run: go test -v ./... 29 | 30 | - name: Build salt-exporter 31 | run: go build -v ./cmd/salt-exporter 32 | 33 | - name: Build salt-live 34 | run: go build -v ./cmd/salt-live 35 | 36 | e2e: 37 | name: "End-to-end tests" 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v5 41 | 42 | - name: Set up Go 43 | uses: actions/setup-go@v6 44 | with: 45 | go-version-file: 'go.mod' 46 | 47 | - name: Set up environment 48 | run: docker compose -f $COMPOSE_FILE up -d --wait --wait-timeout 60 49 | 50 | - name: Run some Salt commands 51 | run: docker compose -f $COMPOSE_FILE exec salt_master sh /test/exec_commands.sh 52 | 53 | - name: Test 54 | run: go test -v -tags=e2e ./e2e_test/... 55 | 56 | - name: Print metrics if failed 57 | if: failure() 58 | run: curl 127.0.0.1:2112/metrics | grep salt_ 59 | 60 | - name: "Clean up environment" 61 | if: always() 62 | run: docker compose -f $COMPOSE_FILE down 63 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Salt tools 2 | repo_url: https://github.com/kpetremann/salt-exporter 3 | repo_name: kpetremann/salt-exporter 4 | site_description: Salt Exporter/Live documentation 5 | site_author: Kevin Petremann 6 | 7 | plugins: 8 | - awesome-pages 9 | - search 10 | 11 | markdown_extensions: 12 | - admonition 13 | - attr_list 14 | - md_in_html 15 | - pymdownx.details 16 | - pymdownx.inlinehilite 17 | - pymdownx.snippets 18 | - pymdownx.superfences 19 | - pymdownx.keys 20 | - pymdownx.emoji: 21 | emoji_index: !!python/name:materialx.emoji.twemoji 22 | emoji_generator: !!python/name:materialx.emoji.to_svg 23 | - pymdownx.highlight: 24 | anchor_linenums: true 25 | line_spans: __span 26 | pygments_lang_class: true 27 | - pymdownx.superfences: 28 | custom_fences: 29 | - name: mermaid 30 | class: mermaid 31 | format: !!python/name:pymdownx.superfences.fence_code_format 32 | 33 | theme: 34 | name: material 35 | features: 36 | - navigation.instant 37 | - navigation.trackings 38 | - navigation.expand 39 | 40 | icon: 41 | repo: fontawesome/brands/github-alt 42 | # logo: assets/mini-logo.png 43 | # favicon: assets/logo-enter-small.png 44 | palette: 45 | - media: "(prefers-color-scheme: light)" 46 | scheme: default 47 | toggle: 48 | icon: material/brightness-7 49 | name: Switch to dark mode 50 | - media: "(prefers-color-scheme: dark)" 51 | scheme: slate 52 | toggle: 53 | icon: material/brightness-4 54 | name: Switch to light mode 55 | 56 | extra_css: 57 | - stylesheets/extra.css 58 | -------------------------------------------------------------------------------- /docs/docs/salt-exporter/quickstart.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Quickstart 3 | --- 4 | 5 | # Salt Exporter 6 | 7 | 8 | ## Installation 9 | 10 | You can download the binary from the [Github releases](https://github.com/kpetremann/salt-exporter/releases) page. 11 | 12 | Or install from source: 13 | 14 | * latest published version: 15 | ``` { .sh .copy } 16 | go install github.com/kpetremann/salt-exporter/cmd/salt-exporter@latest 17 | ``` 18 | 19 | * latest commit (unstable): 20 | ``` { .sh .copy } 21 | go install github.com/kpetremann/salt-exporter/cmd/salt-exporter@main 22 | ``` 23 | 24 | !!! warning "Deprecation notice" 25 | 26 | The following flags are deprecated: 27 | 28 | * `-health-minions` 29 | * `-health-functions-filter` 30 | * `-health-states-filter` 31 | 32 | They should be replaced by metrics configuration in the `config.yml` file. 33 | 34 | The equivalent of: 35 | ``` shell 36 | ./salt-exporter -health-minions -health-functions-filter "func1,func2" -health-states-filter "state1,state2"` 37 | ``` 38 | 39 | is: 40 | ``` { .yaml .copy } 41 | metrics: 42 | salt_responses_total: 43 | enabled: true 44 | 45 | salt_function_status: 46 | enabled: true 47 | filters: 48 | functions: 49 | - "func1" 50 | - "func2" 51 | states: 52 | - "state1" 53 | - "state2" 54 | ``` 55 | 56 | 57 | ## Usage 58 | 59 | The exporter runs out of the box: 60 | ```./salt-exporter``` 61 | 62 | !!! note 63 | 64 | You need to run the exporter with the user running the Salt master. 65 | 66 | !!! example "Examples of configuration options" 67 | 68 | * All metrics can be either enabled or disabled. 69 | * You can add a minion label to some metrics (not recommended on large environment as it could lead to cardinality issues). 70 | * You can filter out `test=true`/`mock=true` events, useful to ignore tests. 71 | * ... more options can be found in the [configuration page](./configuration.md) 72 | -------------------------------------------------------------------------------- /docs/docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Overview 3 | hide: 4 | - toc 5 | --- 6 | 7 | [![Latest](https://img.shields.io/github/v/release/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter/releases) 8 | [![Go](https://img.shields.io/github/go-mod/go-version/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter) 9 | [![CI](https://github.com/kpetremann/salt-exporter/actions/workflows/go.yml/badge.svg)](https://github.com/kpetremann/salt-exporter/actions/workflows/go.yml) 10 | [![GitHub](https://img.shields.io/github/license/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter/blob/main/LICENSE) 11 | 12 | # Salt tools 13 | 14 | ## Salt Exporter 15 | 16 | `Salt Exporter` is a Prometheus exporter for [Saltstack](https://github.com/saltstack/salt) events. It exposes relevant metrics regarding jobs and results. 17 | 18 | This exporter is passive. It does not use the Salt API. 19 | 20 | It works out of the box: you just need to run the exporter on the same user as the Salt Master. 21 | 22 | ``` 23 | $ ./salt-exporter 24 | ``` 25 | 26 | ``` promql 27 | $ curl -s 127.0.0.1:2112/metrics 28 | 29 | salt_expected_responses_total{function="cmd.run", state=""} 6 30 | salt_expected_responses_total{function="state.sls",state="test"} 1 31 | 32 | salt_function_responses_total{function="cmd.run",state="",success="true"} 6 33 | salt_function_responses_total{function="state.sls",state="test",success="true"} 1 34 | 35 | salt_function_status{minion="node1",function="state.highstate",state="highstate"} 1 36 | 37 | salt_new_job_total{function="cmd.run",state="",success="false"} 3 38 | salt_new_job_total{function="state.sls",state="test",success="false"} 1 39 | 40 | salt_responses_total{minion="local",success="true"} 6 41 | salt_responses_total{minion="node1",success="true"} 6 42 | 43 | salt_scheduled_job_return_total{function="state.sls",minion="local",state="test",success="true"} 2 44 | ``` 45 | 46 | ## Salt Live 47 | 48 | !!! tip "`salt-run state.event pretty=True` under steroids" 49 | 50 | `Salt Live` is a Terminal UI tool to watch events in real time. 51 | 52 | Check out the full demo **[here](./salt-live/usage.md)**. 53 | 54 | [![tui.gif](./demo/tui-overview.gif)](./demo/tui-overview.webm) 55 | -------------------------------------------------------------------------------- /cmd/salt-live/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | 11 | "github.com/kpetremann/salt-exporter/internal/tui" 12 | "github.com/kpetremann/salt-exporter/pkg/event" 13 | "github.com/kpetremann/salt-exporter/pkg/listener" 14 | "github.com/kpetremann/salt-exporter/pkg/parser" 15 | "github.com/rs/zerolog/log" 16 | 17 | tea "github.com/charmbracelet/bubbletea" 18 | ) 19 | 20 | var version = "unknown" 21 | var commit = "unknown" 22 | var date = "unknown" 23 | 24 | func printVersion() { 25 | if version == "unknown" { 26 | version = fmt.Sprintf("v%s", version) 27 | } 28 | fmt.Println("Version:", version) 29 | fmt.Println("Build date:", date) 30 | fmt.Println("Commit:", commit) 31 | } 32 | 33 | func main() { 34 | maxItems := flag.Int("max-events", 1000, "maximum events to keep in memory") 35 | bufferSize := flag.Int("buffer-size", 1000, "buffer size in number of events") 36 | filter := flag.String("hard-filter", "", "filter when received (filtered out events are discarded forever)") 37 | ipcFilepath := flag.String("ipc-file", listener.DefaultIPCFilepath, "file location of the salt-master event bus") 38 | versionCmd := flag.Bool("version", false, "print version") 39 | debug := flag.Bool("debug", false, "enable debug mode (log to debug.log)") 40 | flag.Parse() 41 | 42 | if *debug { 43 | f, err := tea.LogToFile("debug.log", "debug") 44 | if err != nil { 45 | fmt.Println("fatal:", err) 46 | os.Exit(1) 47 | } 48 | defer f.Close() 49 | } 50 | 51 | if *versionCmd { 52 | printVersion() 53 | return 54 | } 55 | 56 | log.Logger = log.Output(nil) 57 | ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) 58 | defer stop() 59 | 60 | eventChan := make(chan event.SaltEvent, *bufferSize) 61 | parser := parser.NewEventParser(true) 62 | eventListener := listener.NewEventListener(ctx, parser, eventChan) 63 | eventListener.SetIPCFilepath(*ipcFilepath) 64 | go eventListener.ListenEvents() 65 | 66 | p := tea.NewProgram(tui.NewModel(eventChan, *maxItems, *filter), tea.WithMouseCellMotion()) 67 | if _, err := p.Run(); err != nil { 68 | fmt.Printf("Alas, there's been an error: %v", err) 69 | os.Exit(1) //nolint:gocritic // force exit 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "github.com/kpetremann/salt-exporter/pkg/event" 8 | "github.com/kpetremann/salt-exporter/pkg/parser" 9 | ) 10 | 11 | func TestParseEvent(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | args map[string]interface{} 15 | want event.SaltEvent 16 | }{ 17 | { 18 | name: "new job", 19 | args: fakeEventAsMap(fakeNewJobEvent()), 20 | want: expectedNewJob, 21 | }, 22 | { 23 | name: "return job", 24 | args: fakeEventAsMap(fakeRetJobEvent()), 25 | want: expectedReturnJob, 26 | }, 27 | { 28 | name: "new schedule job", 29 | args: fakeEventAsMap(fakeNewScheduleJobEvent()), 30 | want: expectedNewScheduleJob, 31 | }, 32 | { 33 | name: "return ack schedule job", 34 | args: fakeEventAsMap(fakeAckScheduleJobEvent()), 35 | want: expectedAckScheduleJob, 36 | }, 37 | { 38 | name: "return schedule job", 39 | args: fakeEventAsMap(fakeScheduleJobReturnEvent()), 40 | want: expectedScheduleJobReturn, 41 | }, 42 | { 43 | name: "new state.sls", 44 | args: fakeEventAsMap(fakeNewStateSlsJobEvent()), 45 | want: expectedNewStateSlsJob, 46 | }, 47 | { 48 | name: "return state.sls", 49 | args: fakeEventAsMap(fakeStateSlsReturnEvent()), 50 | want: expectedStateSlsReturn, 51 | }, 52 | { 53 | name: "new state.single", 54 | args: fakeEventAsMap(fakeNewStateSingleEvent()), 55 | want: expectedNewStateSingle, 56 | }, 57 | { 58 | name: "return state.single", 59 | args: fakeEventAsMap(fakeStateSingleReturnEvent()), 60 | want: expectedStateSingleReturn, 61 | }, 62 | { 63 | name: "new state.sls test=True mock=True", 64 | args: fakeEventAsMap(fakeNewTestMockStateSlsJobEvent()), 65 | want: expectedNewTestMockStateSlsJob, 66 | }, 67 | { 68 | name: "return state.sls test=True mock=True", 69 | args: fakeEventAsMap(fakeTestMockStateSlsReturnEvent()), 70 | want: expectedTestMockStateSlsReturn, 71 | }, 72 | { 73 | name: "beacon", 74 | args: fakeEventAsMap(fakeBeaconEvent()), 75 | want: expectedBeacon, 76 | }, 77 | } 78 | 79 | p := parser.NewEventParser(false) 80 | for _, test := range tests { 81 | parsed, err := p.Parse(test.args) 82 | if err != nil { 83 | t.Errorf("Unexpected error %s", err.Error()) 84 | } 85 | 86 | if diff := cmp.Diff(parsed, test.want); diff != "" { 87 | t.Errorf("Mismatch for '%s' test:\n%s", test.name, diff) 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /internal/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func boolToFloat64(b bool) float64 { 11 | if b { 12 | return 1.0 13 | } 14 | return 0.0 15 | } 16 | 17 | func eventToMetrics(e event.SaltEvent, r *Registry) { 18 | if e.Module == event.BeaconModule { 19 | if e.Type != "status" { 20 | return 21 | } 22 | r.UpdateLastHeartbeat(e.Data.ID) 23 | return 24 | } 25 | 26 | switch e.Type { 27 | case "new": 28 | state := e.ExtractState() 29 | r.IncreaseNewJobTotal(e.Data.Fun, state) 30 | r.IncreaseExpectedResponsesTotal(e.Data.Fun, state, float64(e.TargetNumber)) 31 | 32 | case "ret": 33 | state := e.ExtractState() 34 | success := e.Data.Success 35 | 36 | if e.IsScheduleJob { 37 | // for scheduled job, when the states in the job actually failed 38 | // - the global "success" value is always true 39 | // - the state module success is false, but the global retcode is > 0 40 | // - if defined, the "result" of a state module in event.Return covers 41 | // the corner case when retccode is not properly computed by Salt. 42 | // 43 | // using retcode and state module success could be enough, but we combine all values 44 | // in case there are other corner cases. 45 | success = e.Data.Success && (e.Data.Retcode == 0) 46 | if e.StateModuleSuccess != nil { 47 | success = success && *e.StateModuleSuccess 48 | } 49 | r.IncreaseScheduledJobReturnTotal(e.Data.Fun, state, e.Data.ID, success) 50 | } else { 51 | r.IncreaseFunctionResponsesTotal(e.Data.Fun, state, e.Data.ID, success) 52 | } 53 | 54 | r.IncreaseResponseTotal(e.Data.ID, success) 55 | r.SetFunctionStatus(e.Data.ID, e.Data.Fun, state, success) 56 | } 57 | } 58 | 59 | func ExposeMetrics(ctx context.Context, eventChan <-chan event.SaltEvent, watchChan <-chan event.WatchEvent, config Config) { 60 | registry := NewRegistry(config) 61 | 62 | for { 63 | select { 64 | case <-ctx.Done(): 65 | log.Info().Msg("stopping event listener") 66 | return 67 | case e := <-watchChan: 68 | if e.Op == event.Accepted { 69 | registry.AddObservableMinion(e.MinionName) 70 | } 71 | if e.Op == event.Removed { 72 | registry.DeleteObservableMinion(e.MinionName) 73 | } 74 | case e := <-eventChan: 75 | if config.Global.Filters.IgnoreTest && e.IsTest { 76 | continue 77 | } 78 | if config.Global.Filters.IgnoreMock && e.IsMock { 79 | continue 80 | } 81 | 82 | eventToMetrics(e, ®istry) 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /internal/tui/keymap.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "github.com/charmbracelet/bubbles/key" 5 | teaList "github.com/charmbracelet/bubbles/list" 6 | ) 7 | 8 | type keyMap struct { 9 | enableFollow key.Binding 10 | toggleJSONYAML key.Binding 11 | toggleWordwrap key.Binding 12 | demoText key.Binding 13 | } 14 | 15 | func defaultKeyMap() *keyMap { 16 | return &keyMap{ 17 | enableFollow: key.NewBinding( 18 | key.WithKeys("f", "F"), 19 | key.WithHelp("f", "follow"), 20 | ), 21 | toggleWordwrap: key.NewBinding( 22 | key.WithKeys("w", "W"), 23 | key.WithHelp("w", "JSON word wrap"), 24 | ), 25 | toggleJSONYAML: key.NewBinding( 26 | key.WithKeys("m", "M"), 27 | key.WithHelp("m", "JSON/YAML/parsed"), 28 | ), 29 | demoText: key.NewBinding( 30 | key.WithKeys("$"), 31 | ), 32 | } 33 | } 34 | 35 | func bubblesListKeyMap() teaList.KeyMap { 36 | return teaList.KeyMap{ 37 | // Browsing. 38 | CursorUp: key.NewBinding( 39 | key.WithKeys("up", "k"), 40 | key.WithHelp("↑/k", "up"), 41 | ), 42 | CursorDown: key.NewBinding( 43 | key.WithKeys("down", "j"), 44 | key.WithHelp("↓/j", "down"), 45 | ), 46 | PrevPage: key.NewBinding( 47 | key.WithKeys("left", "h", "pgup", "b", "u"), 48 | key.WithHelp("←/h/pgup", "prev page"), 49 | ), 50 | NextPage: key.NewBinding( 51 | key.WithKeys("right", "l", "pgdown", "d"), 52 | key.WithHelp("→/l/pgdn", "next page"), 53 | ), 54 | GoToStart: key.NewBinding( 55 | key.WithKeys("home", "g"), 56 | key.WithHelp("g/home", "go to start"), 57 | ), 58 | GoToEnd: key.NewBinding( 59 | key.WithKeys("end", "G"), 60 | key.WithHelp("G/end", "go to end"), 61 | ), 62 | Filter: key.NewBinding( 63 | key.WithKeys("/"), 64 | key.WithHelp("/", "filter"), 65 | ), 66 | ClearFilter: key.NewBinding( 67 | key.WithKeys("esc"), 68 | key.WithHelp("esc", "clear filter"), 69 | ), 70 | 71 | // Filtering. 72 | CancelWhileFiltering: key.NewBinding( 73 | key.WithKeys("esc"), 74 | key.WithHelp("esc", "cancel"), 75 | ), 76 | AcceptWhileFiltering: key.NewBinding( 77 | key.WithKeys("enter", "tab", "shift+tab", "ctrl+k", "up", "ctrl+j", "down"), 78 | key.WithHelp("enter", "apply filter"), 79 | ), 80 | 81 | // Toggle help. 82 | ShowFullHelp: key.NewBinding( 83 | key.WithKeys("?"), 84 | key.WithHelp("?", "more"), 85 | ), 86 | CloseFullHelp: key.NewBinding( 87 | key.WithKeys("?"), 88 | key.WithHelp("?", "close help"), 89 | ), 90 | 91 | // Quitting. 92 | Quit: key.NewBinding( 93 | key.WithKeys("q", "Q", "esc"), 94 | key.WithHelp("q", "quit"), 95 | ), 96 | ForceQuit: key.NewBinding(key.WithKeys("ctrl+c")), 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /pkg/event/event_test.go: -------------------------------------------------------------------------------- 1 | package event_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | ) 8 | 9 | func getNewStateEvent() event.SaltEvent { 10 | return event.SaltEvent{ 11 | Tag: "salt/job/20220630000f000000000/new", 12 | Type: "new", 13 | Module: event.JobModule, 14 | TargetNumber: 1, 15 | Data: event.EventData{ 16 | Timestamp: "2022-06-30T00:00:00.000000", 17 | Fun: "state.sls", 18 | Arg: []interface{}{"test"}, 19 | Jid: "20220630000000000000", 20 | Minions: []string{"node1"}, 21 | Missing: []string{}, 22 | Tgt: "node1", 23 | TgtType: "glob", 24 | User: "salt_user", 25 | }, 26 | IsScheduleJob: false, 27 | } 28 | } 29 | 30 | func TestExtractState(t *testing.T) { 31 | stateSls := getNewStateEvent() 32 | 33 | stateSlsFunArg := getNewStateEvent() 34 | stateSlsFunArg.Data.Arg = nil 35 | stateSlsFunArg.Data.FunArgs = []interface{}{"test", map[string]bool{"dry_run": true}} 36 | 37 | stateSlsFunArgMap := getNewStateEvent() 38 | stateSlsFunArgMap.Data.Arg = nil 39 | stateSlsFunArgMap.Data.FunArgs = []interface{}{map[string]interface{}{"mods": "test", "dry_run": true}} 40 | 41 | stateApplyArg := getNewStateEvent() 42 | stateApplyArg.Data.Fun = "state.apply" 43 | 44 | stateApplyHighstate := getNewStateEvent() 45 | stateApplyHighstate.Data.Fun = "state.apply" 46 | stateApplyHighstate.Data.Arg = nil 47 | 48 | stateHighstate := getNewStateEvent() 49 | stateHighstate.Data.Fun = "state.highstate" 50 | stateHighstate.Data.Arg = nil 51 | 52 | tests := []struct { 53 | name string 54 | event event.SaltEvent 55 | want string 56 | }{ 57 | { 58 | name: "state via state.sls", 59 | event: stateSls, 60 | want: "test", 61 | }, 62 | { 63 | name: "state via state.sls args + kwargs", 64 | event: stateSlsFunArg, 65 | want: "test", 66 | }, 67 | { 68 | name: "state via state.sls kwargs only", 69 | event: stateSlsFunArgMap, 70 | want: "test", 71 | }, 72 | { 73 | name: "state via state.apply args only", 74 | event: stateApplyArg, 75 | want: "test", 76 | }, 77 | { 78 | name: "state via state.apply", 79 | event: stateApplyArg, 80 | want: "test", 81 | }, 82 | { 83 | name: "highstate via state.apply", 84 | event: stateApplyHighstate, 85 | want: "highstate", 86 | }, 87 | { 88 | name: "state.highstate", 89 | event: stateHighstate, 90 | want: "highstate", 91 | }, 92 | } 93 | 94 | for _, test := range tests { 95 | if res := test.event.ExtractState(); res != test.want { 96 | t.Errorf("Mismatch for '%s', wants '%s' got '%s' ", test.name, test.want, res) 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kpetremann/salt-exporter 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.25.4 6 | 7 | require ( 8 | github.com/alecthomas/chroma v0.10.0 9 | github.com/charmbracelet/bubbles v0.20.0 10 | github.com/charmbracelet/bubbletea v1.2.4 11 | github.com/charmbracelet/lipgloss v1.0.0 12 | github.com/fsnotify/fsnotify v1.8.0 13 | github.com/google/go-cmp v0.6.0 14 | github.com/k0kubun/pp/v3 v3.4.1 15 | github.com/prometheus/client_golang v1.20.5 16 | github.com/prometheus/client_model v0.6.1 17 | github.com/prometheus/common v0.61.0 18 | github.com/rs/zerolog v1.33.0 19 | github.com/spf13/viper v1.19.0 20 | github.com/vmihailenco/msgpack/v5 v5.4.1 21 | gopkg.in/yaml.v3 v3.0.1 22 | ) 23 | 24 | require ( 25 | github.com/atotto/clipboard v0.1.4 // indirect 26 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 27 | github.com/beorn7/perks v1.0.1 // indirect 28 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 29 | github.com/charmbracelet/x/ansi v0.6.0 // indirect 30 | github.com/charmbracelet/x/term v0.2.1 // indirect 31 | github.com/dlclark/regexp2 v1.11.4 // indirect 32 | github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect 33 | github.com/hashicorp/hcl v1.0.0 // indirect 34 | github.com/klauspost/compress v1.17.11 // indirect 35 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 36 | github.com/magiconair/properties v1.8.9 // indirect 37 | github.com/mattn/go-colorable v0.1.13 // indirect 38 | github.com/mattn/go-isatty v0.0.20 // indirect 39 | github.com/mattn/go-localereader v0.0.1 // indirect 40 | github.com/mattn/go-runewidth v0.0.16 // indirect 41 | github.com/mitchellh/mapstructure v1.5.0 // indirect 42 | github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect 43 | github.com/muesli/cancelreader v0.2.2 // indirect 44 | github.com/muesli/termenv v0.15.2 // indirect 45 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 46 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 47 | github.com/prometheus/procfs v0.15.1 // indirect 48 | github.com/rivo/uniseg v0.4.7 // indirect 49 | github.com/sagikazarmark/locafero v0.6.0 // indirect 50 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 51 | github.com/sahilm/fuzzy v0.1.1 // indirect 52 | github.com/sourcegraph/conc v0.3.0 // indirect 53 | github.com/spf13/afero v1.11.0 // indirect 54 | github.com/spf13/cast v1.7.1 // indirect 55 | github.com/spf13/pflag v1.0.5 // indirect 56 | github.com/subosito/gotenv v1.6.0 // indirect 57 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 58 | go.uber.org/multierr v1.11.0 // indirect 59 | golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect 60 | golang.org/x/sync v0.10.0 // indirect 61 | golang.org/x/sys v0.28.0 // indirect 62 | golang.org/x/text v0.21.0 // indirect 63 | google.golang.org/protobuf v1.36.0 // indirect 64 | gopkg.in/ini.v1 v1.67.0 // indirect 65 | ) 66 | -------------------------------------------------------------------------------- /pkg/listener/pkiwatcher.go: -------------------------------------------------------------------------------- 1 | package listener 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/fsnotify/fsnotify" 12 | "github.com/kpetremann/salt-exporter/pkg/event" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | const DefaultPKIDirpath = "/etc/salt/pki/master" 17 | 18 | type PKIWatcher struct { 19 | ctx context.Context 20 | pkiDirPath string 21 | watcher *fsnotify.Watcher 22 | eventChan chan<- event.WatchEvent 23 | lock sync.RWMutex 24 | } 25 | 26 | func NewPKIWatcher(ctx context.Context, pkiDirPath string, eventChan chan event.WatchEvent) (*PKIWatcher, error) { 27 | watcher, err := fsnotify.NewWatcher() 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | w := &PKIWatcher{ 33 | ctx: ctx, 34 | pkiDirPath: pkiDirPath, 35 | watcher: watcher, 36 | eventChan: eventChan, 37 | lock: sync.RWMutex{}, 38 | } 39 | 40 | return w, nil 41 | } 42 | 43 | // SetPKIDirectory sets the filepath to the salt-master pki directory 44 | // 45 | // The directory must be readable by the user running the exporter (usually salt). 46 | // 47 | // Default: /etc/salt/pki. 48 | func (w *PKIWatcher) SetPKIDirectory(filepath string) { 49 | w.pkiDirPath = filepath 50 | } 51 | 52 | func (w *PKIWatcher) open() { 53 | for { 54 | select { 55 | case <-w.ctx.Done(): 56 | return 57 | default: 58 | } 59 | 60 | minionsDir := path.Join(w.pkiDirPath, "minions") 61 | 62 | log.Info().Msg("loading currently accepted minions") 63 | entries, err := os.ReadDir(minionsDir) 64 | if err != nil { 65 | log.Error().Str("error", err.Error()).Msg("failed to list PKI directory") 66 | time.Sleep(5 * time.Second) 67 | } else { 68 | for _, e := range entries { 69 | if !e.IsDir() { 70 | w.eventChan <- event.WatchEvent{ 71 | MinionName: e.Name(), 72 | Op: event.Accepted, 73 | } 74 | log.Info().Msgf("minion %s loaded", e.Name()) 75 | } 76 | } 77 | 78 | // Add a path. 79 | err = w.watcher.Add(minionsDir) 80 | if err != nil { 81 | log.Error().Str("error", err.Error()).Msg("failed to watch PKI directory") 82 | time.Sleep(time.Second * 5) 83 | } else { 84 | return 85 | } 86 | } 87 | } 88 | } 89 | 90 | func (w *PKIWatcher) StartWatching() { 91 | w.open() 92 | 93 | for { 94 | select { 95 | case <-w.ctx.Done(): 96 | w.Stop() 97 | return 98 | case evt := <-w.watcher.Events: 99 | minionName := path.Base(evt.Name) 100 | if minionName == ".key_cache" || strings.HasPrefix(minionName, ".___atomic_write") { 101 | continue 102 | } 103 | if evt.Op == fsnotify.Create { 104 | w.eventChan <- event.WatchEvent{ 105 | MinionName: minionName, 106 | Op: event.Accepted, 107 | } 108 | log.Info().Msgf("minion %s accepted by master", minionName) 109 | } 110 | if evt.Op == fsnotify.Remove { 111 | w.eventChan <- event.WatchEvent{ 112 | MinionName: minionName, 113 | Op: event.Removed, 114 | } 115 | log.Info().Msgf("minion %s removed from master", minionName) 116 | } 117 | case err := <-w.watcher.Errors: 118 | log.Error().Str("error", err.Error()).Msg("fail processing watch event") 119 | } 120 | } 121 | } 122 | 123 | func (w *PKIWatcher) Stop() { 124 | log.Info().Msg("stop listening for PKI changes") 125 | w.watcher.Close() 126 | } 127 | -------------------------------------------------------------------------------- /cmd/salt-exporter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/kpetremann/salt-exporter/internal/logging" 13 | "github.com/kpetremann/salt-exporter/internal/metrics" 14 | "github.com/kpetremann/salt-exporter/pkg/event" 15 | "github.com/kpetremann/salt-exporter/pkg/listener" 16 | "github.com/kpetremann/salt-exporter/pkg/parser" 17 | "github.com/prometheus/client_golang/prometheus/promhttp" 18 | "github.com/rs/zerolog/log" 19 | ) 20 | 21 | var ( 22 | version = "unknown" 23 | commit = "unknown" 24 | date = "unknown" 25 | ) 26 | 27 | func quit() { 28 | log.Warn().Msg("Bye.") 29 | } 30 | 31 | func printInfo(config Config) { 32 | log.Info().Str("Version", version).Send() 33 | log.Info().Str("Commit", commit).Send() 34 | log.Info().Str("Build time", date).Send() 35 | 36 | if config.Metrics.HealthMinions { 37 | log.Info().Msgf("health-minions: functions filters: %s", config.Metrics.SaltFunctionStatus.Filters.Functions) 38 | log.Info().Msgf("health-minions: states filters: %s", config.Metrics.SaltFunctionStatus.Filters.States) 39 | } 40 | 41 | if config.Metrics.Global.Filters.IgnoreTest { 42 | log.Info().Msg("test=True events will be ignored") 43 | } 44 | if config.Metrics.Global.Filters.IgnoreMock { 45 | log.Info().Msg("mock=True events will be ignored") 46 | } 47 | } 48 | 49 | func start(config Config) { 50 | listenSocket := fmt.Sprint(config.ListenAddress, ":", config.ListenPort) 51 | 52 | ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) 53 | defer stop() 54 | 55 | log.Info().Msg("listening for events...") 56 | eventChan := make(chan event.SaltEvent) 57 | watchChan := make(chan event.WatchEvent) 58 | 59 | // listen and expose metric 60 | parser := parser.NewEventParser(false) 61 | eventListener := listener.NewEventListener(ctx, parser, eventChan) 62 | eventListener.SetIPCFilepath(config.IPCFile) 63 | 64 | if config.Metrics.HealthMinions { 65 | pkiWatcher, err := listener.NewPKIWatcher(ctx, config.PKIDir, watchChan) 66 | if err != nil { 67 | log.Fatal().Msgf("unable to watch PKI for minions change: %v", err) //nolint:gocritic // force exit 68 | } 69 | 70 | go pkiWatcher.StartWatching() 71 | } 72 | go eventListener.ListenEvents() 73 | go metrics.ExposeMetrics(ctx, eventChan, watchChan, config.Metrics) 74 | 75 | // start http server 76 | log.Info().Msg("exposing metrics on " + listenSocket + "/metrics") 77 | 78 | mux := http.NewServeMux() 79 | mux.Handle("/metrics", promhttp.Handler()) 80 | httpServer := http.Server{Addr: listenSocket, Handler: mux, ReadHeaderTimeout: 2 * time.Second} 81 | 82 | go func() { 83 | var err error 84 | 85 | if !config.TLS.Enabled { 86 | err = httpServer.ListenAndServe() 87 | } else { 88 | err = httpServer.ListenAndServeTLS(config.TLS.Certificate, config.TLS.Key) 89 | } 90 | 91 | if err != nil { 92 | log.Error().Err(err).Send() 93 | stop() 94 | } 95 | }() 96 | 97 | // exiting 98 | <-ctx.Done() 99 | if err := httpServer.Shutdown(context.Background()); err != nil { 100 | log.Error().Err(err).Send() 101 | } 102 | } 103 | 104 | func main() { 105 | defer quit() 106 | logging.Configure() 107 | 108 | config, err := ReadConfig() 109 | if err != nil { 110 | log.Fatal().Err(err).Msg("failed to load settings during initialization") //nolint:gocritic // force exit 111 | } 112 | 113 | logging.SetLevel(config.LogLevel) 114 | printInfo(config) 115 | start(config) 116 | } 117 | -------------------------------------------------------------------------------- /pkg/listener/listener.go: -------------------------------------------------------------------------------- 1 | package listener 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net" 7 | "time" 8 | 9 | "github.com/kpetremann/salt-exporter/pkg/event" 10 | "github.com/rs/zerolog/log" 11 | "github.com/vmihailenco/msgpack/v5" 12 | ) 13 | 14 | type eventParser interface { 15 | Parse(message map[string]interface{}) (event.SaltEvent, error) 16 | } 17 | 18 | const DefaultIPCFilepath = "/var/run/salt/master/master_event_pub.ipc" 19 | 20 | // EventListener listens to the salt-master event bus and sends events to the event channel. 21 | type EventListener struct { 22 | // ctx specificies the context used mainly for cancellation 23 | ctx context.Context 24 | 25 | // eventChan is the channel to send events to 26 | eventChan chan event.SaltEvent 27 | 28 | // iPCFilepath is filepath to the salt-master event bus 29 | iPCFilepath string 30 | 31 | // saltEventBus keeps the connection to the salt-master event bus 32 | saltEventBus net.Conn 33 | 34 | // decoder is msgpack decoder for parsing the event bus messages 35 | decoder *msgpack.Decoder 36 | 37 | eventParser eventParser 38 | } 39 | 40 | // Open opens the salt-master event bus. 41 | func (e *EventListener) Open() { 42 | log.Info().Str("file", e.iPCFilepath).Msg("connecting to salt-master event bus") 43 | var err error 44 | 45 | for { 46 | select { 47 | case <-e.ctx.Done(): 48 | return 49 | default: 50 | } 51 | 52 | e.saltEventBus, err = net.Dial("unix", e.iPCFilepath) 53 | if err != nil { 54 | log.Error().Msg("failed to connect to event bus, retrying in 5 seconds") 55 | time.Sleep(time.Second * 5) 56 | } else { 57 | log.Info().Msg("successfully connected to event bus") 58 | e.decoder = msgpack.NewDecoder(e.saltEventBus) 59 | return 60 | } 61 | } 62 | } 63 | 64 | // Close closes the salt-master event bus. 65 | func (e *EventListener) Close() error { 66 | log.Info().Msg("disconnecting from salt-master event bus") 67 | if e.saltEventBus != nil { 68 | return e.saltEventBus.Close() 69 | } else { 70 | return errors.New("trying to close already closed bus") 71 | } 72 | } 73 | 74 | // Reconnect reconnects to the salt-master event bus. 75 | func (e *EventListener) Reconnect() { 76 | select { 77 | case <-e.ctx.Done(): 78 | return 79 | default: 80 | e.Close() 81 | e.Open() 82 | } 83 | } 84 | 85 | // NewEventListener creates a new EventListener 86 | // 87 | // The events will be sent to eventChan. 88 | func NewEventListener(ctx context.Context, eventParser eventParser, eventChan chan event.SaltEvent) *EventListener { 89 | e := EventListener{ 90 | ctx: ctx, 91 | eventChan: eventChan, 92 | eventParser: eventParser, 93 | iPCFilepath: DefaultIPCFilepath, 94 | } 95 | return &e 96 | } 97 | 98 | // SetIPCFilepath sets the filepath to the salt-master event bus 99 | // 100 | // The IPC file must be readable by the user running the exporter. 101 | // 102 | // Default: /var/run/salt/master/master_event_pub.ipc. 103 | func (e *EventListener) SetIPCFilepath(filepath string) { 104 | e.iPCFilepath = filepath 105 | } 106 | 107 | // ListenEvents listens to the salt-master event bus and sends events to the event channel. 108 | func (e *EventListener) ListenEvents() { 109 | e.Open() 110 | 111 | for { 112 | select { 113 | case <-e.ctx.Done(): 114 | log.Info().Msg("stop listening events") 115 | e.Close() 116 | return 117 | default: 118 | message, err := e.decoder.DecodeMap() 119 | if err != nil { 120 | log.Error().Str("error", err.Error()).Msg("unable to read event") 121 | log.Error().Msg("event bus may be closed, trying to reconnect") 122 | 123 | e.Reconnect() 124 | 125 | continue 126 | } 127 | if event, err := e.eventParser.Parse(message); err == nil { 128 | e.eventChan <- event 129 | } 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Latest](https://img.shields.io/github/v/release/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter/releases) 2 | [![Go](https://img.shields.io/github/go-mod/go-version/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter) 3 | [![CI](https://github.com/kpetremann/salt-exporter/actions/workflows/go.yml/badge.svg)](https://github.com/kpetremann/salt-exporter/actions/workflows/go.yml) 4 | [![GitHub](https://img.shields.io/github/license/kpetremann/salt-exporter)](https://github.com/kpetremann/salt-exporter/blob/main/LICENSE) 5 | 6 | [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/kpetremann) 7 | 8 | Buy Me A Coffee 9 | 10 | ## Salt Live 11 | 12 | > _`salt-run state.event pretty=True` under steroids_ 13 | 14 | Salt Exporter comes with `Salt Live`. This is a Terminal UI tool to watch events in real time. 15 | 16 | demo 17 | 18 | 19 | ## Salt Exporter 20 | 21 | `Salt Exporter` is a Prometheus exporter for [Saltstack](https://github.com/saltstack/salt) events. It exposes relevant metrics regarding jobs and results. 22 | 23 | This exporter is passive. It does not use the Salt API. 24 | 25 | It works out of the box: you just need to run the exporter on the same user as the Salt Master. 26 | 27 | ``` 28 | $ ./salt-exporter 29 | ``` 30 | 31 | ``` 32 | $ curl -s 127.0.0.1:2112/metrics 33 | 34 | salt_expected_responses_total{function="cmd.run", state=""} 6 35 | salt_expected_responses_total{function="state.sls",state="test"} 1 36 | 37 | salt_function_responses_total{function="cmd.run",state="",success="true"} 6 38 | salt_function_responses_total{function="state.sls",state="test",success="true"} 1 39 | 40 | salt_function_status{minion="node1",function="state.highstate",state="highstate"} 1 41 | 42 | salt_new_job_total{function="cmd.run",state="",success="false"} 3 43 | salt_new_job_total{function="state.sls",state="test",success="false"} 1 44 | 45 | salt_responses_total{minion="local",success="true"} 6 46 | salt_responses_total{minion="node1",success="true"} 6 47 | 48 | salt_scheduled_job_return_total{function="state.sls",minion="local",state="test",success="true"} 2 49 | 50 | salt_health_last_heartbeat{minion="local"} 1703053536 51 | salt_health_last_heartbeat{minion="node1"} 1703053536 52 | 53 | salt_health_minions_total{} 2 54 | ``` 55 | 56 | ### Deprecation notice 57 | 58 | `-health-minions`, `health-functions-filter` and `health-states-filter` are deprecated. 59 | They should be replaced by metrics configuration in the `config.yml` file. 60 | 61 | The equivalent of `./salt-exporter -health-minions -health-functions-filter "func1,func2" -health-states-filter "state1,state2"` is: 62 | 63 | ```yaml 64 | metrics: 65 | salt_responses_total: 66 | enabled: true 67 | 68 | salt_function_status: 69 | enabled: true 70 | filters: 71 | functions: 72 | - "func1" 73 | - "func2" 74 | states: 75 | - "state1" 76 | - "state2" 77 | ``` 78 | 79 | ### Installation 80 | 81 | Just use the binary from [Github releases](https://github.com/kpetremann/salt-exporter/releases) page. 82 | 83 | Or, install from source: 84 | - latest published version: `go install github.com/kpetremann/salt-exporter/cmd/salt-exporter@latest` 85 | - latest commit (unstable): `go install github.com/kpetremann/salt-exporter/cmd/salt-exporter@main` 86 | 87 | ### Usage 88 | 89 | Simply run: 90 | ```./salt-exporter``` 91 | 92 | The exporter can be configured in different ways, with the following precedence order: 93 | * flags 94 | * environment variables 95 | * configuration file (config.yml) 96 | 97 | See the [official documentation](https://kpetremann.github.io/salt-exporter) for more details 98 | -------------------------------------------------------------------------------- /pkg/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/kpetremann/salt-exporter/pkg/event" 9 | "github.com/rs/zerolog/log" 10 | "github.com/vmihailenco/msgpack/v5" 11 | ) 12 | 13 | const testArg = "test" 14 | const mockArg = "mock" 15 | 16 | type Event struct { 17 | KeepRewBody bool 18 | } 19 | 20 | func NewEventParser(keepRawBody bool) Event { 21 | return Event{KeepRewBody: keepRawBody} 22 | } 23 | 24 | // isDryRun checks if an event is run with test=True 25 | // 26 | // Salt stores can store this info at two locations: 27 | // 28 | // in args: 29 | // 30 | // "arg": [ 31 | // "somestate", 32 | // { 33 | // "__kwarg__": true, 34 | // "test": true 35 | // } 36 | // ] 37 | // 38 | // or in fun_args: 39 | // 40 | // "fun_args": [ 41 | // "somestate", 42 | // { 43 | // "test": true 44 | // } 45 | // ] 46 | func getBoolKwarg(event event.SaltEvent, field string) bool { 47 | for _, arg := range event.Data.Arg { 48 | if fields, ok := arg.(map[string]interface{}); ok { 49 | if val, ok := fields[field].(bool); ok { 50 | return val 51 | } 52 | } 53 | } 54 | 55 | for _, funArg := range event.Data.FunArgs { 56 | if fields, ok := funArg.(map[string]interface{}); ok { 57 | if val, ok := fields[field].(bool); ok { 58 | return val 59 | } 60 | } 61 | } 62 | 63 | return false 64 | } 65 | 66 | func statemoduleResult(event event.SaltEvent) *bool { 67 | substates, ok := event.Data.Return.(map[string]interface{}) 68 | if !ok { 69 | return nil 70 | } 71 | 72 | for _, ret := range substates { 73 | substate, ok := ret.(map[string]interface{}) 74 | if !ok { 75 | return nil 76 | } 77 | 78 | result, ok := substate["result"] 79 | if !ok { 80 | return nil 81 | } 82 | 83 | r, ok := result.(bool) 84 | if !ok { 85 | return nil 86 | } 87 | 88 | if !r { 89 | return &r 90 | } 91 | } 92 | 93 | success := true 94 | return &success 95 | } 96 | 97 | // ParseEvent parses a salt event. 98 | func (e Event) Parse(message map[string]interface{}) (event.SaltEvent, error) { 99 | var body string 100 | 101 | if raw, ok := message["body"].([]byte); ok { 102 | body = string(raw) 103 | } else { 104 | body = message["body"].(string) 105 | } 106 | lines := strings.SplitN(body, "\n\n", 2) 107 | 108 | tag := lines[0] 109 | if !(strings.HasPrefix(tag, "salt/")) { 110 | return event.SaltEvent{}, errors.New("tag not supported") 111 | } 112 | log.Debug().Str("tag", tag).Msg("new event") 113 | 114 | parts := strings.Split(tag, "/") 115 | 116 | if len(parts) < 3 { 117 | return event.SaltEvent{}, errors.New("tag not supported") 118 | } 119 | 120 | eventModule := event.GetEventModule(tag) 121 | 122 | if eventModule == event.UnknownModule { 123 | return event.SaltEvent{}, errors.New("tag not supported. Module unknown") 124 | } 125 | 126 | // Extract job type from the tag 127 | if len(tag) < 4 { 128 | return event.SaltEvent{}, fmt.Errorf("invalid salt tag: %s", tag) 129 | } 130 | jobType := strings.Split(tag, "/")[3] 131 | 132 | // Parse message body 133 | byteResult := []byte(lines[1]) 134 | ev := event.SaltEvent{Tag: tag, Type: jobType, Module: eventModule} 135 | 136 | if e.KeepRewBody { 137 | ev.RawBody = byteResult 138 | } 139 | 140 | if err := msgpack.Unmarshal(byteResult, &ev.Data); err != nil { 141 | log.Warn().Str("error", err.Error()).Str("tag", tag).Msg("decoding_failure") 142 | return event.SaltEvent{}, err 143 | } 144 | 145 | // Extract other info 146 | ev.TargetNumber = len(ev.Data.Minions) 147 | ev.IsScheduleJob = ev.Data.Schedule != "" 148 | ev.IsTest = getBoolKwarg(ev, testArg) 149 | ev.IsMock = getBoolKwarg(ev, mockArg) 150 | ev.StateModuleSuccess = statemoduleResult(ev) 151 | 152 | // A runner are executed on the master but they do not provide their ID in the event 153 | if strings.HasPrefix(tag, "salt/run") && ev.Data.ID == "" { 154 | ev.Data.ID = "master" 155 | } 156 | 157 | return ev, nil 158 | } 159 | -------------------------------------------------------------------------------- /pkg/event/event.go: -------------------------------------------------------------------------------- 1 | package event 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "strings" 7 | 8 | "github.com/vmihailenco/msgpack/v5" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type EventModule int 13 | 14 | type WatchOp uint32 15 | 16 | const ( 17 | UnknownModule EventModule = iota 18 | RunnerModule 19 | JobModule 20 | BeaconModule 21 | ) 22 | 23 | const ( 24 | Accepted WatchOp = iota 25 | Removed 26 | ) 27 | 28 | type WatchEvent struct { 29 | MinionName string 30 | Op WatchOp 31 | } 32 | 33 | type EventData struct { 34 | Arg []interface{} `msgpack:"arg"` 35 | Cmd string `msgpack:"cmd"` 36 | Fun string `msgpack:"fun"` 37 | FunArgs []interface{} `msgpack:"fun_args"` 38 | ID string `msgpack:"id"` 39 | Jid string `msgpack:"jid"` 40 | JidStamp string `msgpack:"jid_stamp"` 41 | Minions []string `msgpack:"minions"` 42 | Missing []string `msgpack:"missing"` 43 | Out string `msgpack:"out"` 44 | Retcode int `msgpack:"retcode"` 45 | Return interface{} `msgpack:"return"` 46 | Tgt interface{} `msgpack:"tgt"` 47 | TgtType string `msgpack:"tgt_type"` 48 | Timestamp string `msgpack:"_stamp"` 49 | User string `msgpack:"user"` 50 | Schedule string `msgpack:"schedule"` 51 | Success bool `msgpack:"success"` 52 | } 53 | 54 | type SaltEvent struct { 55 | Tag string 56 | Type string 57 | Module EventModule 58 | TargetNumber int 59 | Data EventData 60 | IsScheduleJob bool 61 | RawBody []byte 62 | IsTest bool 63 | IsMock bool 64 | StateModuleSuccess *bool 65 | } 66 | 67 | // RawToJSON converts raw body to JSON 68 | // 69 | // If indent is true, the JSON will be indented. 70 | func (e SaltEvent) RawToJSON(indent bool) ([]byte, error) { 71 | if e.RawBody == nil { 72 | return nil, errors.New("raw body not registered") 73 | } 74 | 75 | var data interface{} 76 | if err := msgpack.Unmarshal(e.RawBody, &data); err != nil { 77 | return nil, err 78 | } 79 | if indent { 80 | return json.MarshalIndent(data, "", " ") 81 | } else { 82 | return json.Marshal(data) 83 | } 84 | } 85 | 86 | // RawToYAML converts raw body to YAML. 87 | func (e SaltEvent) RawToYAML() ([]byte, error) { 88 | if e.RawBody == nil { 89 | return nil, errors.New("raw body not registered") 90 | } 91 | 92 | var data interface{} 93 | if err := msgpack.Unmarshal(e.RawBody, &data); err != nil { 94 | return nil, err 95 | } 96 | 97 | return yaml.Marshal(data) 98 | } 99 | 100 | func GetEventModule(tag string) EventModule { 101 | tagParts := strings.Split(tag, "/") 102 | if len(tagParts) < 2 { 103 | return UnknownModule 104 | } 105 | switch tagParts[1] { 106 | case "run": 107 | return RunnerModule 108 | case "job": 109 | return JobModule 110 | case "beacon": 111 | return BeaconModule 112 | default: 113 | return UnknownModule 114 | } 115 | } 116 | 117 | // extractStateFromArgs extracts embedded state info. 118 | func extractStateFromArgs(args interface{}, key string) string { 119 | // args only 120 | if v, ok := args.(string); ok { 121 | return v 122 | } 123 | 124 | // kwargs 125 | if v, ok := args.(map[string]interface{}); ok { 126 | if _, keyExists := v[key]; !keyExists { 127 | return "" 128 | } 129 | if ret, isString := v[key].(string); isString { 130 | return ret 131 | } 132 | } 133 | 134 | return "" 135 | } 136 | 137 | // Extract state info from event. 138 | func (e *SaltEvent) ExtractState() string { 139 | switch e.Data.Fun { 140 | case "state.sls", "state.apply": 141 | switch { 142 | case len(e.Data.Arg) > 0: 143 | return extractStateFromArgs(e.Data.Arg[0], "mods") 144 | case len(e.Data.FunArgs) > 0: 145 | return extractStateFromArgs(e.Data.FunArgs[0], "mods") 146 | case e.Data.Fun == "state.apply": 147 | return "highstate" 148 | } 149 | case "state.single": 150 | if len(e.Data.Arg) > 0 { 151 | return extractStateFromArgs(e.Data.Arg[0], "fun") 152 | } else if len(e.Data.FunArgs) > 0 { 153 | return extractStateFromArgs(e.Data.FunArgs[0], "fun") 154 | } 155 | case "state.highstate": 156 | return "highstate" 157 | } 158 | return "" 159 | } 160 | -------------------------------------------------------------------------------- /internal/metrics/registry.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | "github.com/kpetremann/salt-exporter/internal/filters" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promauto" 10 | ) 11 | 12 | type Registry struct { 13 | config Config 14 | 15 | observedMinions int32 16 | 17 | newJobTotal *prometheus.CounterVec 18 | expectedResponsesTotal *prometheus.CounterVec 19 | 20 | functionResponsesTotal *prometheus.CounterVec 21 | scheduledJobReturnTotal *prometheus.CounterVec 22 | 23 | responseTotal *prometheus.CounterVec 24 | functionStatus *prometheus.GaugeVec 25 | 26 | statusLastResponse *prometheus.GaugeVec 27 | minionsTotal *prometheus.GaugeVec 28 | } 29 | 30 | func NewRegistry(config Config) Registry { 31 | functionResponsesTotalLabels := []string{"function", "state", "success"} 32 | if config.SaltFunctionResponsesTotal.AddMinionLabel { 33 | functionResponsesTotalLabels = append([]string{"minion"}, functionResponsesTotalLabels...) 34 | } 35 | 36 | scheduledJobReturnTotalLabels := []string{"function", "state", "success"} 37 | if config.SaltScheduledJobReturnTotal.AddMinionLabel { 38 | scheduledJobReturnTotalLabels = append([]string{"minion"}, scheduledJobReturnTotalLabels...) 39 | } 40 | 41 | return Registry{ 42 | config: config, 43 | 44 | observedMinions: 0, 45 | newJobTotal: promauto.NewCounterVec( 46 | prometheus.CounterOpts{ 47 | Name: "salt_new_job_total", 48 | Help: "Total number of new jobs processed", 49 | }, 50 | []string{"function", "state"}, 51 | ), 52 | 53 | expectedResponsesTotal: promauto.NewCounterVec( 54 | prometheus.CounterOpts{ 55 | Name: "salt_expected_responses_total", 56 | Help: "Total number of expected minions responses", 57 | }, 58 | []string{"function", "state"}, 59 | ), 60 | 61 | functionResponsesTotal: promauto.NewCounterVec( 62 | prometheus.CounterOpts{ 63 | Name: "salt_function_responses_total", 64 | Help: "Total number of responses per function processed", 65 | }, 66 | functionResponsesTotalLabels, 67 | ), 68 | 69 | scheduledJobReturnTotal: promauto.NewCounterVec( 70 | prometheus.CounterOpts{ 71 | Name: "salt_scheduled_job_return_total", 72 | Help: "Total number of scheduled job responses", 73 | }, 74 | scheduledJobReturnTotalLabels, 75 | ), 76 | 77 | responseTotal: promauto.NewCounterVec( 78 | prometheus.CounterOpts{ 79 | Name: "salt_responses_total", 80 | Help: "Total number of responses", 81 | }, 82 | []string{"minion", "success"}, 83 | ), 84 | 85 | functionStatus: promauto.NewGaugeVec( 86 | prometheus.GaugeOpts{ 87 | Name: "salt_function_status", 88 | Help: "Last function/state success, 0=Failed, 1=Success", 89 | }, 90 | []string{"minion", "function", "state"}, 91 | ), 92 | statusLastResponse: promauto.NewGaugeVec( 93 | prometheus.GaugeOpts{ 94 | Name: "salt_health_last_heartbeat", 95 | Help: "Last status beacon received. Unix timestamp", 96 | }, 97 | []string{"minion"}, 98 | ), 99 | minionsTotal: promauto.NewGaugeVec( 100 | prometheus.GaugeOpts{ 101 | Name: "salt_health_minions_total", 102 | Help: "Total number of observed minions via status beacon", 103 | }, []string{}, 104 | ), 105 | } 106 | } 107 | 108 | func (r *Registry) UpdateLastHeartbeat(minion string) { 109 | timestamp := time.Now().Unix() 110 | r.statusLastResponse.WithLabelValues(minion).Set(float64(timestamp)) 111 | } 112 | 113 | func (r *Registry) AddObservableMinion(minion string) { 114 | r.observedMinions += 1 115 | r.UpdateLastHeartbeat(minion) 116 | r.minionsTotal.WithLabelValues().Set(float64(r.observedMinions)) 117 | } 118 | 119 | func (r *Registry) DeleteObservableMinion(minion string) { 120 | r.statusLastResponse.DeleteLabelValues(minion) 121 | r.observedMinions -= 1 122 | r.minionsTotal.WithLabelValues().Set(float64(r.observedMinions)) 123 | } 124 | 125 | func (r *Registry) IncreaseNewJobTotal(function, state string) { 126 | if r.config.SaltNewJobTotal.Enabled { 127 | r.newJobTotal.WithLabelValues(function, state).Inc() 128 | } 129 | } 130 | 131 | func (r *Registry) IncreaseExpectedResponsesTotal(function, state string, value float64) { 132 | if r.config.SaltExpectedResponsesTotal.Enabled { 133 | r.expectedResponsesTotal.WithLabelValues(function, state).Add(value) 134 | } 135 | } 136 | 137 | func (r *Registry) IncreaseFunctionResponsesTotal(function, state, minion string, success bool) { 138 | labels := []string{function, state, strconv.FormatBool(success)} 139 | if r.config.SaltFunctionResponsesTotal.AddMinionLabel { 140 | labels = append([]string{minion}, labels...) 141 | } 142 | 143 | if r.config.SaltFunctionResponsesTotal.Enabled { 144 | r.functionResponsesTotal.WithLabelValues(labels...).Inc() 145 | } 146 | } 147 | 148 | func (r *Registry) IncreaseScheduledJobReturnTotal(function, state, minion string, success bool) { 149 | labels := []string{function, state, strconv.FormatBool(success)} 150 | if r.config.SaltScheduledJobReturnTotal.AddMinionLabel { 151 | labels = append([]string{minion}, labels...) 152 | } 153 | 154 | if r.config.SaltScheduledJobReturnTotal.Enabled { 155 | r.scheduledJobReturnTotal.WithLabelValues(labels...).Inc() 156 | } 157 | } 158 | 159 | func (r *Registry) IncreaseResponseTotal(minion string, success bool) { 160 | if r.config.SaltResponsesTotal.Enabled { 161 | r.responseTotal.WithLabelValues(minion, strconv.FormatBool(success)).Inc() 162 | } 163 | } 164 | 165 | func (r *Registry) SetFunctionStatus(minion, function, state string, success bool) { 166 | if !r.config.SaltFunctionStatus.Enabled { 167 | return 168 | } 169 | if !filters.Match(function, r.config.SaltFunctionStatus.Filters.Functions) { 170 | return 171 | } 172 | if !filters.Match(state, r.config.SaltFunctionStatus.Filters.States) { 173 | return 174 | } 175 | 176 | r.functionStatus.WithLabelValues(minion, function, state).Set(boolToFloat64(success)) 177 | } 178 | -------------------------------------------------------------------------------- /cmd/salt-exporter/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/kpetremann/salt-exporter/internal/metrics" 12 | "github.com/kpetremann/salt-exporter/pkg/listener" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | const defaultConfigFilename = "config.yml" 17 | const defaultLogLevel = "info" 18 | const defaultPort = 2112 19 | const defaultHealthMinion = true 20 | const defaultHealthFunctionsFilter = "state.highstate" 21 | const defaultHealthStatesFilter = "highstate" 22 | 23 | var flagConfigMapping = map[string]string{ 24 | "host": "listen-address", 25 | "port": "listen-port", 26 | "tls": "tls.enabled", 27 | "tls-cert": "tls.certificate", 28 | "tls-key": "tls.key", 29 | "ignore-test": "metrics.global.filters.ignore-test", 30 | "ignore-mock": "metrics.global.filters.ignore-mock", 31 | "health-minions": "metrics.health-minions", 32 | "health-functions-filter": "metrics.salt_function_status.filters.functions", 33 | "health-states-filter": "metrics.salt_function_status.filters.states", 34 | } 35 | 36 | type Config struct { 37 | LogLevel string `mapstructure:"log-level"` 38 | 39 | ListenAddress string `mapstructure:"listen-address"` 40 | ListenPort int `mapstructure:"listen-port"` 41 | IPCFile string `mapstructure:"ipc-file"` 42 | PKIDir string `mapstructure:"pki-dir"` 43 | TLS struct { 44 | Enabled bool 45 | Key string 46 | Certificate string 47 | } 48 | 49 | Metrics metrics.Config 50 | } 51 | 52 | func parseFlags() (string, bool) { 53 | // flags 54 | versionCmd := flag.Bool("version", false, "print version") 55 | configFile := flag.String("config-file", defaultConfigFilename, "config filepath") 56 | flag.String("log-level", defaultLogLevel, "log level (debug, info, warn, error, fatal, panic, disabled)") 57 | flag.String("host", "", "listen address") 58 | flag.Int("port", defaultPort, "listen port") 59 | flag.String("ipc-file", listener.DefaultIPCFilepath, "file location of the salt-master event bus") 60 | flag.Bool("tls", false, "enable TLS") 61 | flag.String("tls-cert", "", "TLS certificated") 62 | flag.String("tls-key", "", "TLS private key") 63 | 64 | flag.Bool("ignore-test", false, "ignore test=True events") 65 | flag.Bool("ignore-mock", false, "ignore mock=True events") 66 | 67 | // deprecated flag 68 | healthMinions := flag.Bool("health-minions", defaultHealthMinion, "[DEPRECATED] enable minion metrics") 69 | flag.String("health-functions-filter", defaultHealthStatesFilter, 70 | "[DEPRECATED] apply filter on functions to monitor, separated by a comma") 71 | flag.String("health-states-filter", defaultHealthStatesFilter, 72 | "[DEPRECATED] apply filter on states to monitor, separated by a comma") 73 | flag.Parse() 74 | 75 | if *versionCmd { 76 | if version == "unknown" { 77 | version = fmt.Sprintf("v%s", version) 78 | } 79 | fmt.Println("Version:", version) 80 | fmt.Println("Build date:", date) 81 | fmt.Println("Commit:", commit) 82 | os.Exit(0) 83 | } 84 | 85 | return *configFile, *healthMinions 86 | } 87 | 88 | func setDefaults(healthMinions bool) { 89 | viper.SetDefault("log-level", defaultLogLevel) 90 | viper.SetDefault("listen-port", defaultPort) 91 | viper.SetDefault("ipc-file", listener.DefaultIPCFilepath) 92 | viper.SetDefault("pki-dir", listener.DefaultPKIDirpath) 93 | viper.SetDefault("metrics.health-minions", defaultHealthMinion) 94 | viper.SetDefault("metrics.salt_new_job_total.enabled", true) 95 | viper.SetDefault("metrics.salt_expected_responses_total.enabled", true) 96 | viper.SetDefault("metrics.salt_function_responses_total.enabled", true) 97 | viper.SetDefault("metrics.salt_scheduled_job_return_total.enabled", true) 98 | viper.SetDefault("metrics.salt_function_status.enabled", healthMinions) // TODO: true once health-minions will be removed 99 | viper.SetDefault("metrics.salt_responses_total.enabled", healthMinions) // TODO: true once health-minions will be removed 100 | viper.SetDefault("metrics.salt_function_status.filters.functions", []string{defaultHealthFunctionsFilter}) 101 | viper.SetDefault("metrics.salt_function_status.filters.states", []string{defaultHealthStatesFilter}) 102 | } 103 | 104 | func getConfig(configFileName string, healthMinions bool) (Config, error) { 105 | setDefaults(healthMinions) 106 | 107 | // bind flags 108 | var allFlags []viperFlag 109 | flag.Visit(func(f *flag.Flag) { 110 | m := viperFlag{original: *f, alias: flagConfigMapping[f.Name]} 111 | allFlags = append(allFlags, m) 112 | }) 113 | 114 | fSet := viperFlagSet{ 115 | flags: allFlags, 116 | } 117 | if err := viper.BindFlagValues(fSet); err != nil { 118 | return Config{}, fmt.Errorf("flag binding failure: %w", err) 119 | } 120 | 121 | // bind configuration file 122 | if filepath.IsAbs(configFileName) { 123 | viper.SetConfigFile(configFileName) 124 | } else { 125 | ext := filepath.Ext(configFileName) 126 | viper.SetConfigName(strings.TrimSuffix(configFileName, ext)) 127 | viper.SetConfigType(strings.TrimPrefix(ext, ".")) 128 | viper.AddConfigPath(".") 129 | } 130 | 131 | viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "__")) 132 | viper.SetEnvPrefix("SALT") 133 | viper.AutomaticEnv() 134 | 135 | err := viper.ReadInConfig() 136 | if err != nil { 137 | if _, ok := err.(viper.ConfigFileNotFoundError); !ok { //nolint: errorlint // ConfigFileNotFoundError not implementing error 138 | return Config{}, fmt.Errorf("invalid config file: %w", err) 139 | } 140 | } 141 | 142 | // extract configuration 143 | var cfg Config 144 | if err := viper.Unmarshal(&cfg); err != nil { 145 | return Config{}, fmt.Errorf("failed to load configuration: %w", err) 146 | } 147 | 148 | return cfg, nil 149 | } 150 | 151 | func checkRequirements(cfg Config) error { 152 | if cfg.TLS.Enabled { 153 | if cfg.TLS.Certificate == "" { 154 | return errors.New("TLS Certificate not specified") 155 | } 156 | if cfg.TLS.Key == "" { 157 | return errors.New("TLS Private Key not specified") 158 | } 159 | } 160 | 161 | return nil 162 | } 163 | 164 | func ReadConfig() (Config, error) { 165 | var err error 166 | 167 | configFileName, healthMinions := parseFlags() 168 | 169 | cfg, err := getConfig(configFileName, healthMinions) 170 | if err != nil { 171 | return Config{}, err 172 | } 173 | 174 | err = checkRequirements(cfg) 175 | if err != nil { 176 | return Config{}, err 177 | } 178 | 179 | return cfg, nil 180 | } 181 | -------------------------------------------------------------------------------- /docs/docs/salt-exporter/configuration.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Configuration 3 | --- 4 | 5 | # Configuration 6 | 7 | The salt-exporter can be configured with flags, environments variables and configuration file. 8 | 9 | !!! info 10 | 11 | The precedence order for the different methods is: 12 | 13 | * flags 14 | * environment variables 15 | * configuration file (config.yml) 16 | 17 | ## Configuration file 18 | 19 | The exporter is looking for `config.yml`. 20 | 21 | Note: You can specify a specific config filepath using `--config-file`, i.e. `--config-file="/srv/salt-exporter/config.yml"` 22 | 23 | See below a full example of a configuration file: 24 | 25 | ``` { .yaml .copy } 26 | log-level: "info" 27 | 28 | listen-address: "" 29 | listen-port: 2112 30 | 31 | pki-dir: /etc/salt/pki/master 32 | ipc-file: /var/run/salt/master/master_event_pub.ipc 33 | 34 | tls: 35 | enabled: true 36 | key: "/path/to/key" 37 | certificate: "/path/to/certificate" 38 | 39 | metrics: 40 | global: 41 | filters: 42 | ignore-test: false 43 | ignore-mock: false 44 | 45 | salt_new_job_total: 46 | enabled: true 47 | 48 | salt_expected_responses_total: 49 | enabled: true 50 | 51 | salt_function_responses_total: 52 | enabled: true 53 | add-minion-label: false # not recommended in production 54 | 55 | salt_scheduled_job_return_total: 56 | enabled: true 57 | add-minion-label: false # not recommended in production 58 | 59 | salt_responses_total: 60 | enabled: true 61 | 62 | salt_function_status: 63 | enabled: true 64 | filters: 65 | functions: 66 | - "state.highstate" 67 | states: 68 | - "highstate" 69 | ``` 70 | 71 | ### Global parameters 72 | 73 | | Parameter | Default | Description | 74 | |----------------|-----------|--------------------------------------------------------------------| 75 | | log-level | `info` | log level can be: debug, info, warn, error, fatal, panic, disabled | 76 | | listen-address | `0.0.0.0` | listening address | 77 | | listen-port | `2112` | listening port | 78 | | pki-dir | `/etc/salt/pki/master` | path to Salt master's PKI directory | 79 | 80 | ### TLS settings 81 | 82 | All parameters below are in the `tls` section of the configuration. 83 | 84 | | Parameter | Default | Description | 85 | |-------------|---------|---------------------------------------------| 86 | | enabled | `false` | enables/disables TLS on the metrics webserver | 87 | | key | | TLS key for the metrics webserver | 88 | | certificate | | TLS certificate for the metrics webserver | 89 | 90 | ### Metrics global settings 91 | 92 | All parameters below are in the `metrics.global` section of the configuration. 93 | 94 | | Parameter | Default | Description | 95 | |---------------------|---------|---------------------------| 96 | | filters.ignore-test | `false` | ignores `test=True` events | 97 | | filters.ignore-mock | `false` | ignores `mock=True` events | 98 | 99 | ### Metrics configuration 100 | 101 | All parameters below are in the `metrics` section of the configuration. 102 | 103 | | Parameter | Default | Description | 104 | |-----------|-------------------|-------------------------------------------------------------------| 105 | | ``.enabled | `true` | enables or disables a metric | 106 | | ``.add-minion-label

Only for:
| `false` | adds minion label
_not recommended
can lead to cardinality issues_ | 107 | | salt_function_status.filters.function | `state.highstate` | updates the metric only if the event function matches the filter | 108 | | salt_function_status.filters.states | `highstate` | updates the metric only if the event state matches the filter | 109 | 110 | ### Minions health detection 111 | 112 | In most of the cases all that you need to configure is to enable [`status` beacon](https://docs.saltproject.io/en/latest/ref/beacons/all/salt.beacons.status.html#:~:text=salt.-,beacons.,presence%20to%20be%20set%20up.) on Salt minions. 113 | However, if you change the [pki directory](https://docs.saltproject.io/en/latest/ref/configuration/master.html#pki-dir) for Salt master, you'll need to make a change in the exporter side too by changing it in the configuration 114 | ```yaml 115 | log-level: "info" 116 | 117 | pki-dir: /path/as/set/in/master/config 118 | 119 | tls: 120 | ... 121 | ``` 122 | 123 | ## Alternative methods 124 | 125 | ### Environment variables 126 | 127 | All settings available in the configuration file can be set as environment variables, but: 128 | 129 | * all variables must be prefixed by `SALT_` 130 | * uppercase only 131 | * `-` in the configuration file becomes a `_` 132 | * `__` is the level separator 133 | 134 | For example, the equivalent of this config file: 135 | 136 | ``` yaml 137 | log-level: "info" 138 | tls: 139 | enabled: true 140 | metrics: 141 | global: 142 | filters: 143 | ignore-test: true 144 | ``` 145 | 146 | is: 147 | 148 | ``` shell 149 | SALT_LOG_LEVEL="info" 150 | SALT_TLS__ENABLED=true 151 | SALT_METRICS__GLOBAL__FILTERS__IGNORE_TEST=true 152 | ``` 153 | 154 | ### Flags 155 | 156 | ``` 157 | ./salt-exporter -help 158 | -config-file string 159 | config filepath (default "config.yml") 160 | -health-functions-filter string 161 | [DEPRECATED] apply filter on functions to monitor, separated by a comma (default "highstate") 162 | -health-minions 163 | [DEPRECATED] enable minion metrics (default true) 164 | -health-states-filter string 165 | [DEPRECATED] apply filter on states to monitor, separated by a comma (default "highstate") 166 | -host string 167 | listen address 168 | -ignore-mock 169 | ignore mock=True events 170 | -ignore-test 171 | ignore test=True events 172 | -ipc-file string 173 | file location of the salt-master event bus (default "/var/run/salt/master/master_event_pub.ipc") 174 | -log-level string 175 | log level (debug, info, warn, error, fatal, panic, disabled) (default "info") 176 | -port int 177 | listen port (default 2112) 178 | -tls 179 | enable TLS 180 | -tls-cert string 181 | TLS certificated 182 | -tls-key string 183 | TLS private key 184 | -version 185 | print version 186 | ``` 187 | 188 | -------------------------------------------------------------------------------- /e2e_test/e2e_test.go: -------------------------------------------------------------------------------- 1 | //go:build e2e 2 | 3 | package e2e_test 4 | 5 | import ( 6 | "fmt" 7 | "net/http" 8 | "testing" 9 | 10 | prom "github.com/prometheus/client_model/go" 11 | 12 | "github.com/prometheus/common/expfmt" 13 | ) 14 | 15 | const exporterURL = "http://127.0.0.1:2112/metrics" 16 | 17 | type Metric struct { 18 | Name string 19 | Labels map[string]string 20 | Value float64 21 | } 22 | 23 | // scrapeMetrics fetches the metrics from the running exporter 24 | func scrapeMetrics(url string) (map[string]*prom.MetricFamily, error) { 25 | // Fetch the metrics from the running exporter 26 | resp, err := http.Get(url) 27 | if err != nil { 28 | return nil, err 29 | } 30 | defer resp.Body.Close() 31 | 32 | // Read the response body and parse the metrics 33 | var parser expfmt.TextParser 34 | parsed, err := parser.TextToMetricFamilies(resp.Body) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return parsed, nil 40 | } 41 | 42 | // getValue returns the value of a metric depending on its type 43 | func getValue(metric *prom.Metric, metrictType prom.MetricType) float64 { 44 | switch metrictType { 45 | case prom.MetricType_COUNTER: 46 | return metric.Counter.GetValue() 47 | case prom.MetricType_GAUGE: 48 | return metric.Gauge.GetValue() 49 | case prom.MetricType_SUMMARY: 50 | return metric.Summary.GetSampleSum() 51 | case prom.MetricType_HISTOGRAM: 52 | return metric.Histogram.GetSampleSum() 53 | default: 54 | return 0 55 | } 56 | } 57 | 58 | // getMetrics returns the metrics from the running exporter 59 | func getMetrics(url string) ([]Metric, error) { 60 | // Fetch the metrics from the running exporter 61 | parsed, err := scrapeMetrics(url) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | // Parse the metrics 67 | metrics := []Metric{} 68 | for _, metric := range parsed { 69 | for _, m := range metric.Metric { 70 | labels := map[string]string{} 71 | for _, l := range m.Label { 72 | labels[*l.Name] = *l.Value 73 | } 74 | metrics = append(metrics, Metric{*metric.Name, labels, getValue(m, *metric.Type)}) 75 | } 76 | } 77 | 78 | return metrics, nil 79 | } 80 | 81 | // getValueForMetric returns the value of a metric with the given name and labels from the parsed metrics 82 | func getValueForMetric(metrics []Metric, name string, labels map[string]string) (float64, error) { 83 | // Check if the metric exists 84 | for _, m := range metrics { 85 | if m.Name == name { 86 | // Check if the labels match 87 | match := true 88 | for k, v := range labels { 89 | if m.Labels[k] != v { 90 | match = false 91 | break 92 | } 93 | } 94 | if match { 95 | return m.Value, nil 96 | } 97 | } 98 | } 99 | 100 | // Return an error 101 | return 0, fmt.Errorf("%s with labels %v doesn't exist", name, labels) 102 | } 103 | 104 | // TestMetrics tests the metrics exposed by the exporter 105 | func TestMetrics(t *testing.T) { 106 | // Fetch the metrics from the running exporter 107 | parsed, err := getMetrics(exporterURL) 108 | if err != nil { 109 | t.Fatal(err) 110 | } 111 | 112 | // Define the expected metrics 113 | expected := map[string][]Metric{ 114 | "total responses": { 115 | {"salt_responses_total", map[string]string{"minion": "foo", "success": "false"}, 3}, 116 | {"salt_responses_total", map[string]string{"minion": "foo", "success": "true"}, 3}, 117 | }, 118 | "execution modules": { 119 | {"salt_new_job_total", map[string]string{"function": "test.exception", "state": ""}, 1}, 120 | {"salt_new_job_total", map[string]string{"function": "test.true", "state": ""}, 1}, 121 | {"salt_expected_responses_total", map[string]string{"function": "test.exception", "state": ""}, 1}, 122 | {"salt_expected_responses_total", map[string]string{"function": "test.true", "state": ""}, 1}, 123 | {"salt_function_responses_total", map[string]string{"function": "test.exception", "state": "", "success": "false"}, 1}, 124 | {"salt_function_responses_total", map[string]string{"function": "test.true", "state": "", "success": "true"}, 1}, 125 | }, 126 | "state modules": { 127 | {"salt_new_job_total", map[string]string{"function": "state.single", "state": "test.succeed_with_changes"}, 1}, 128 | {"salt_new_job_total", map[string]string{"function": "state.single", "state": "test.fail_with_changes"}, 1}, 129 | {"salt_expected_responses_total", map[string]string{"function": "state.single", "state": "test.succeed_with_changes"}, 1}, 130 | {"salt_expected_responses_total", map[string]string{"function": "state.single", "state": "test.fail_with_changes"}, 1}, 131 | {"salt_function_responses_total", map[string]string{"function": "state.single", "state": "test.succeed_with_changes", "success": "true"}, 1}, 132 | {"salt_function_responses_total", map[string]string{"function": "state.single", "state": "test.fail_with_changes", "success": "false"}, 1}, 133 | }, 134 | "states": { 135 | {"salt_new_job_total", map[string]string{"function": "state.sls", "state": "test.succeed"}, 1}, 136 | {"salt_new_job_total", map[string]string{"function": "state.sls", "state": "test.fail"}, 1}, 137 | {"salt_expected_responses_total", map[string]string{"function": "state.sls", "state": "test.succeed"}, 1}, 138 | {"salt_expected_responses_total", map[string]string{"function": "state.sls", "state": "test.fail"}, 1}, 139 | {"salt_function_responses_total", map[string]string{"function": "state.sls", "state": "test.succeed", "success": "true"}, 1}, 140 | {"salt_function_responses_total", map[string]string{"function": "state.sls", "state": "test.fail", "success": "false"}, 1}, 141 | }, 142 | } 143 | 144 | // Get values from healthcheck 145 | healthcheckLabelsSuccess := map[string]string{"function": "status.ping_master", "state": "", "success": "true"} 146 | healthcheckValueTrue, err := getValueForMetric(parsed, "salt_function_responses_total", healthcheckLabelsSuccess) 147 | if err != nil { 148 | healthcheckValueTrue = 0 149 | } 150 | healthcheckLabelsFailed := map[string]string{"function": "status.ping_master", "state": "", "success": "false"} 151 | healthcheckValueFalse, err := getValueForMetric(parsed, "salt_function_responses_total", healthcheckLabelsFailed) 152 | if err != nil { 153 | healthcheckValueFalse = 0 154 | } 155 | healthcheckFindJobLabels := map[string]string{"function": "saltutil.find_job", "state": "", "success": "true"} 156 | healthcheckFindJob, err := getValueForMetric(parsed, "salt_function_responses_total", healthcheckFindJobLabels) 157 | if err != nil { 158 | healthcheckFindJob = 0 159 | } 160 | 161 | // Check if the expected metrics are present 162 | for testName, metrics := range expected { 163 | for _, e := range metrics { 164 | value, err := getValueForMetric(parsed, e.Name, e.Labels) 165 | 166 | // Remove events coming from docker healthcheck 167 | if testName == "total responses" { 168 | if e.Labels["success"] == "true" { 169 | value -= healthcheckValueTrue + healthcheckFindJob 170 | } else if e.Labels["success"] == "false" { 171 | value -= healthcheckValueFalse 172 | } 173 | } 174 | 175 | if err != nil { 176 | t.Error(err) 177 | } else if value != e.Value { 178 | t.Errorf("[%s] %s with labels %v = %f, got %f", testName, e.Name, e.Labels, e.Value, value) 179 | } 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /docs/docs/salt-exporter/metrics.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Metrics 3 | --- 4 | 5 | # Exposed metrics 6 | 7 | ## Metrics 8 | 9 | ??? info "Supported Salt event tags" 10 | 11 | Each Salt event having a tag in this list will update the metrics: 12 | 13 | * `salt/job//new` 14 | * `salt/job//ret/<*>` 15 | * `salt/run//new` 16 | * `salt/run//ret/<*>` 17 | 18 | | Metric | Labels | Description | 19 | |-----------------------------------|-----------------------------------------------------|---------------------------------------------------------------------------| 20 | | `salt_new_job_total` | `function`, `state` | Total number of new jobs | 21 | | `salt_expected_responses_total` | `function`, `state` | Counter incremented by the number of targeted minion for each new job | 22 | | `salt_function_responses_total` | `function`, `state`, `success`
(opt: `minion`) | Total number of job responses by function, state and success
| 23 | | `salt_scheduled_job_return_total` | `function`, `state`, `success`
(opt: `minion`) | Counter incremented each time a minion sends a scheduled job result | 24 | | `salt_responses_total` | `minion`, `success` | Total number of job responses
_including scheduled_job responses_ | 25 | | `salt_function_status` | `function`, `state`, `minion` | Last status of a job execution* | 26 | | `salt_health_last_heartbeat` | `minion` | Last heartbeat from minion in UNIX timestamp 27 | | `salt_health_minions_total` | | Total number of registered minions 28 | 29 | \* more details in the section below. 30 | 31 | 32 | 33 | ## Labels details 34 | 35 | The exporter exposes the label for both classic jobs and runners. 36 | 37 | | Prometheus label | Salt information | 38 | |------------------|--------------------------------| 39 | | `function` | execution module | 40 | | `state` | state and state module | 41 | | `minion` | minion sending the response | 42 | | `success` | job status | 43 | 44 | ## Function status 45 | 46 | By default, a Salt highstate generates the following metric: 47 | ``` promql 48 | salt_function_status{function="state.highstate",minion="node1",state="highstate"} 1 49 | ``` 50 | 51 | The value can be: 52 | 53 | * `1` the last function/state execution was `successful` 54 | * `0` the last function/state execution has `failed` 55 | 56 | You can find an example of Prometheus alerts that could be used [here](https://github.com/kpetremann/salt-exporter/blob/main/prometheus_alerts/highstate.yaml). 57 | 58 | See the [configuration page](./configuration.md) if you want to watch other functions/states, or if you want to disable this metric. 59 | 60 | ## Minions health 61 | 62 | The exporter is supporting "hearbeat"-ing detection from minions which can be used to monitor for non-responding/dead minions. Under the hood it depends on Salt's beacons. 63 | To ensure that all required minions are reported (even if there is no heartbeat from them yet), exporter needs access to the PKI directory of the Salt Master (by default `/etc/salt/pki/master`) where it watches for accepted minion's public keys (located under `/etc/salt/pki/master/minions`). 64 | On startup, all currently accepted minions are added with last heartbeat set to current time. From this point forward, exporter is using __fsnotify__ to detect added or removed minions. This will ensure that once minion is added, it will be monitored for heartbeat and metric will be removed once minion is deleted from Salt master. 65 | 66 | To use this functionality you'll need to add [`status` beacon](https://docs.saltproject.io/en/latest/ref/beacons/all/salt.beacons.status.html#:~:text=salt.-,beacons.,presence%20to%20be%20set%20up.) to each minion. It doesn't mater what functions will returned or the period. Exporter will just detect such events (in the format `salt/beacon//status`) and register the timestamp as last heartbeat. 67 | 68 | ### Detecting dead minions 69 | 70 | The most simple way is (e.g. no heartbeat in last hour): 71 | ``` { .promql .copy } 72 | (time() - salt_health_last_heartbeat) > 3600 73 | ``` 74 | > __NOTE__: Above is assuming beacon interval is set to < 3600 seconds 75 | 76 | ## How to estimate missing responses 77 | 78 | Simple way: 79 | ``` { .promql .copy } 80 | salt_expected_responses_total - on(function) salt_function_responses_total 81 | ``` 82 | 83 | More advanced: 84 | ``` { .promql .copy } 85 | sum by (instance, function, state) ( 86 | increase(salt_expected_responses_total{function=~"$function", state=~"$state"}[$__rate_interval]) 87 | ) 88 | - sum by (instance, function, state) ( 89 | increase(salt_function_responses_total{function=~"$function", state=~"$state"}[$__rate_interval]) 90 | ) 91 | ``` 92 | 93 | ## Examples 94 | 95 | ??? example "Execution modules" 96 | 97 | ``` promql 98 | # HELP salt_expected_responses_total Total number of expected minions responses 99 | # TYPE salt_expected_responses_total counter 100 | salt_expected_responses_total{function="cmd.run", state=""} 6 101 | salt_expected_responses_total{function="test.ping", state=""} 6 102 | 103 | # HELP salt_function_responses_total Total number of responses per function processed 104 | # TYPE salt_function_responses_total counter 105 | salt_function_responses_total{function="cmd.run",state="",success="true"} 6 106 | salt_function_responses_total{function="test.ping",state="",success="true"} 6 107 | 108 | # HELP salt_new_job_total Total number of new jobs processed 109 | # TYPE salt_new_job_total counter 110 | salt_new_job_total{function="cmd.run",state=""} 3 111 | salt_new_job_total{function="test.ping",state=""} 3 112 | 113 | # HELP salt_responses_total Total number of responses 114 | # TYPE salt_responses_total counter 115 | salt_responses_total{minion="local",success="true"} 6 116 | salt_responses_total{minion="node1",success="true"} 6 117 | 118 | # HELP salt_scheduled_job_return_total Total number of scheduled job responses 119 | # TYPE salt_scheduled_job_return_total counter 120 | salt_scheduled_job_return_total{function="cmd.run",minion="local",state="",success="true"} 2 121 | ``` 122 | 123 | ??? example "States and state modules" 124 | 125 | States (state.sls/apply/highstate) and state module (state.single): 126 | 127 | ``` promql 128 | salt_expected_responses_total{function="state.apply",state="highstate"} 1 129 | salt_expected_responses_total{function="state.highstate",state="highstate"} 2 130 | salt_expected_responses_total{function="state.sls",state="test"} 1 131 | salt_expected_responses_total{function="state.single",state="test.nop"} 3 132 | 133 | salt_function_responses_total{function="state.apply",state="highstate",success="true"} 1 134 | salt_function_responses_total{function="state.highstate",state="highstate",success="true"} 2 135 | salt_function_responses_total{function="state.sls",state="test",success="true"} 1 136 | salt_function_responses_total{function="state.single",state="test.nop",success="true"} 3 137 | 138 | salt_function_status{minion="node1",function="state.highstate",state="highstate"} 1 139 | 140 | salt_new_job_total{function="state.apply",state="highstate",success="false"} 1 141 | salt_new_job_total{function="state.highstate",state="highstate",success="false"} 2 142 | salt_new_job_total{function="state.sls",state="test",success="false"} 1 143 | salt_new_job_total{function="state.single",state="test.nop",success="true"} 3 144 | 145 | salt_scheduled_job_return_total{function="state.sls",minion="local",state="test",success="true"} 3 146 | ``` 147 | ??? example "Minions heartbeat" 148 | 149 | ```promql 150 | salt_health_last_heartbeat{minion="local"} 1703053536 151 | salt_health_last_heartbeat{minion="node1"} 1703052536 152 | 153 | salt_health_minions_total{} 2 154 | ``` -------------------------------------------------------------------------------- /internal/tui/tui.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | "github.com/charmbracelet/bubbles/key" 10 | teaList "github.com/charmbracelet/bubbles/list" 11 | "github.com/charmbracelet/bubbles/textinput" 12 | teaViewport "github.com/charmbracelet/bubbles/viewport" 13 | tea "github.com/charmbracelet/bubbletea" 14 | "github.com/charmbracelet/lipgloss" 15 | "github.com/k0kubun/pp/v3" 16 | "github.com/kpetremann/salt-exporter/pkg/event" 17 | ) 18 | 19 | const theme = "solarized-dark" 20 | 21 | type format int 22 | 23 | type Mode int 24 | 25 | const ( 26 | Following Mode = iota 27 | Frozen 28 | ) 29 | 30 | type model struct { 31 | eventList teaList.Model 32 | itemsBuffer []teaList.Item 33 | sideBlock teaViewport.Model 34 | demoText textinput.Model 35 | eventChan <-chan event.SaltEvent 36 | hardFilter string 37 | keys *keyMap 38 | sideInfos string 39 | sideTitle string 40 | terminalWidth int 41 | terminalHeight int 42 | maxItems int 43 | outputFormat format 44 | currentMode Mode 45 | wordWrap bool 46 | demoMode bool 47 | demoEnabled bool 48 | } 49 | 50 | func NewModel(eventChan <-chan event.SaltEvent, maxItems int, filter string) model { 51 | var listKeys = defaultKeyMap() 52 | 53 | list := teaList.NewDefaultDelegate() 54 | 55 | selColor := lipgloss.Color("#fcc203") 56 | list.Styles.SelectedTitle = list.Styles.SelectedTitle.Foreground(selColor).BorderLeftForeground(selColor) 57 | list.Styles.SelectedDesc = list.Styles.SelectedTitle 58 | 59 | eventList := teaList.New([]teaList.Item{}, list, 0, 0) 60 | eventList.Title = "Events" 61 | eventList.Styles.Title = listTitleStyle 62 | eventList.AdditionalFullHelpKeys = func() []key.Binding { 63 | return []key.Binding{ 64 | listKeys.enableFollow, 65 | listKeys.toggleWordwrap, 66 | listKeys.toggleJSONYAML, 67 | } 68 | } 69 | eventList.AdditionalShortHelpKeys = func() []key.Binding { 70 | return []key.Binding{ 71 | listKeys.enableFollow, 72 | listKeys.toggleJSONYAML, 73 | } 74 | } 75 | eventList.SetShowHelp(false) 76 | eventList.SetShowTitle(false) 77 | eventList.Filter = WordsFilter 78 | eventList.KeyMap = bubblesListKeyMap() 79 | 80 | rawView := teaViewport.New(1, 1) 81 | rawView.KeyMap = teaViewport.KeyMap{} 82 | 83 | m := model{ 84 | eventList: eventList, 85 | sideBlock: rawView, 86 | keys: listKeys, 87 | eventChan: eventChan, 88 | hardFilter: filter, 89 | currentMode: Following, 90 | maxItems: maxItems, 91 | } 92 | 93 | if os.Getenv("SALT_DEMO") == "true" { 94 | m.demoEnabled = true 95 | m.demoText = textinput.New() 96 | m.demoText.Focus() 97 | } 98 | 99 | return m 100 | } 101 | 102 | func watchEvent(m model) tea.Cmd { 103 | return func() tea.Msg { 104 | for { 105 | e := <-m.eventChan 106 | sender := "master" 107 | if e.Data.ID != "" { 108 | sender = e.Data.ID 109 | } 110 | eventJSON, err := e.RawToJSON(true) 111 | if err != nil { 112 | log.Fatalln(err) 113 | } 114 | eventYAML, err := e.RawToYAML() 115 | if err != nil { 116 | log.Fatalln(err) 117 | } 118 | datetime, _ := time.Parse("2006-01-02T15:04:05.999999", e.Data.Timestamp) 119 | item := item{ 120 | title: e.Tag, 121 | description: e.Type, 122 | datetime: datetime.Format("15:04"), 123 | event: e, 124 | sender: sender, 125 | state: e.ExtractState(), 126 | eventJSON: string(eventJSON), 127 | eventYAML: string(eventYAML), 128 | } 129 | 130 | // No hard filter set 131 | if m.hardFilter == "" { 132 | return item 133 | } 134 | 135 | // Hard filter set 136 | if rank := m.eventList.Filter(m.hardFilter, []string{item.FilterValue()}); len(rank) > 0 { 137 | return item 138 | } 139 | } 140 | } 141 | } 142 | 143 | func (m model) Init() tea.Cmd { 144 | return watchEvent(m) 145 | } 146 | 147 | func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { 148 | var cmds []tea.Cmd 149 | 150 | if m.demoMode { 151 | var cmd tea.Cmd 152 | m.demoText, cmd = m.demoText.Update(msg) 153 | cmds = append(cmds, cmd) 154 | } 155 | 156 | // Ensure the mode is Frozen if we are currently navigating 157 | if m.eventList.Index() > 0 { 158 | m.currentMode = Frozen 159 | } 160 | 161 | /* 162 | Manage events 163 | */ 164 | switch msg := msg.(type) { 165 | case item: 166 | switch m.currentMode { 167 | case Following: 168 | // In follow mode (default), we update both the list and the buffer 169 | currentList := m.eventList.Items() 170 | if len(currentList) >= m.maxItems { 171 | m.eventList.RemoveItem(len(currentList) - 1) 172 | } 173 | cmds = append(cmds, m.eventList.InsertItem(0, msg)) 174 | m.itemsBuffer = m.eventList.Items() 175 | case Frozen: 176 | // In Frozen mode, we only update the buffer and keep the item list as is 177 | m.itemsBuffer = append([]teaList.Item{msg}, m.itemsBuffer...) 178 | if len(m.itemsBuffer) > m.maxItems { 179 | m.itemsBuffer = m.itemsBuffer[:len(m.itemsBuffer)-1] 180 | } 181 | } 182 | 183 | cmds = append(cmds, watchEvent(m)) 184 | 185 | case tea.WindowSizeMsg: 186 | m.terminalWidth = msg.Width 187 | m.terminalHeight = msg.Height 188 | 189 | // Enforce width here to avoid filter overflow 190 | m.eventList.SetWidth(m.terminalWidth/2 - leftPanelStyle.GetHorizontalFrameSize()) 191 | m.eventList.Help.Width = m.terminalWidth 192 | 193 | case tea.KeyMsg: 194 | // Don't match any of the keys below if we're actively filtering. 195 | if m.eventList.FilterState() == teaList.Filtering { 196 | break 197 | } 198 | 199 | if m.demoEnabled && key.Matches(msg, m.keys.demoText) { 200 | m.demoMode = !m.demoMode 201 | m.demoText.SetValue("") 202 | } 203 | if m.demoMode { 204 | return m, tea.Batch(cmds...) 205 | } 206 | 207 | switch { 208 | case key.Matches(msg, m.keys.enableFollow): 209 | m.currentMode = Following 210 | m.eventList.ResetSelected() 211 | cmds = append(cmds, m.eventList.SetItems(m.itemsBuffer)) 212 | case key.Matches(msg, m.keys.toggleWordwrap): 213 | m.wordWrap = !m.wordWrap 214 | case key.Matches(msg, m.keys.toggleJSONYAML): 215 | m.outputFormat = (m.outputFormat + 1) % nbFormat 216 | } 217 | } 218 | 219 | /* 220 | Update embedded components 221 | */ 222 | var cmd tea.Cmd 223 | m.eventList, cmd = m.eventList.Update(msg) 224 | cmds = append(cmds, cmd) 225 | 226 | m.updateSideInfos() 227 | m.sideBlock, cmd = m.sideBlock.Update(msg) 228 | cmds = append(cmds, cmd) 229 | 230 | if m.eventList.Index() > 0 { 231 | m.currentMode = Frozen 232 | } 233 | 234 | m.updateTitle() 235 | 236 | return m, tea.Batch(cmds...) 237 | } 238 | 239 | func (m *model) updateSideInfos() { 240 | if sel := m.eventList.SelectedItem(); sel != nil { 241 | switch m.outputFormat { 242 | case YAML: 243 | m.sideTitle = "Raw event (YAML)" 244 | m.sideInfos = sel.(item).eventYAML 245 | if m.wordWrap { 246 | m.sideInfos = strings.ReplaceAll(m.sideInfos, "\\n", " \\\n") 247 | } 248 | if info, err := Highlight(m.sideInfos, "yaml", theme); err != nil { 249 | m.sideBlock.SetContent(m.sideInfos) 250 | } else { 251 | m.sideBlock.SetContent(info) 252 | } 253 | case JSON: 254 | m.sideTitle = "Raw event (JSON)" 255 | m.sideInfos = sel.(item).eventJSON 256 | if m.wordWrap { 257 | m.sideInfos = strings.ReplaceAll(m.sideInfos, "\\n", " \\\n") 258 | } 259 | if info, err := Highlight(m.sideInfos, "json", theme); err != nil { 260 | m.sideBlock.SetContent(m.sideInfos) 261 | } else { 262 | m.sideBlock.SetContent(info) 263 | } 264 | case PARSED: 265 | m.sideTitle = "Parsed event (Golang)" 266 | eventLite := sel.(item).event 267 | eventLite.RawBody = nil 268 | m.sideInfos = pp.Sprint(eventLite) 269 | m.sideBlock.SetContent(m.sideInfos) 270 | } 271 | } 272 | } 273 | 274 | func (m *model) updateTitle() { 275 | switch m.currentMode { 276 | case Following: 277 | m.eventList.Title = "Events" 278 | case Frozen: 279 | m.eventList.Title = "Events (frozen)" 280 | } 281 | } 282 | 283 | func (m model) View() string { 284 | if m.demoMode { 285 | return lipgloss.Place(m.terminalWidth, m.terminalHeight, lipgloss.Center, lipgloss.Center, m.demoText.View()) 286 | } 287 | 288 | /* 289 | Bottom 290 | */ 291 | helpView := m.eventList.Help.View(m.eventList) 292 | 293 | /* 294 | Top bar 295 | */ 296 | topBarStyle.Width(m.terminalWidth) 297 | topBar := topBarStyle.Render(appTitleStyle.Render("Salt Live")) 298 | 299 | // Calculate content height for left and right panels 300 | var content []string 301 | contentHeight := m.terminalHeight - lipgloss.Height(topBar) - lipgloss.Height(helpView) 302 | contentWidth := m.terminalWidth / 2 303 | 304 | /* 305 | Left panel 306 | */ 307 | 308 | if m.currentMode == Frozen { 309 | listTitleStyle.Background(lipgloss.Color("#a02725")) 310 | listTitleStyle.Foreground(lipgloss.Color("#ffffff")) 311 | } else { 312 | listTitleStyle.UnsetBackground() 313 | listTitleStyle.UnsetForeground() 314 | } 315 | listTitle := listTitleStyle.Render(m.eventList.Title) 316 | 317 | leftPanelStyle.Width(contentWidth) 318 | leftPanelStyle.Height(contentHeight) 319 | 320 | m.eventList.SetSize( 321 | contentWidth-leftPanelStyle.GetHorizontalFrameSize(), 322 | contentHeight-lipgloss.Height(listTitle)-leftPanelStyle.GetVerticalFrameSize(), 323 | ) 324 | 325 | listWithTitle := lipgloss.JoinVertical(0, listTitle, m.eventList.View()) 326 | 327 | content = append(content, leftPanelStyle.Render(listWithTitle)) 328 | 329 | /* 330 | Right panel 331 | */ 332 | 333 | if m.sideInfos != "" { 334 | rawTitle := rightPanelTitleStyle.Render(m.sideTitle) 335 | 336 | rightPanelStyle.Width(contentWidth) 337 | rightPanelStyle.Height(contentHeight) 338 | 339 | m.sideBlock.Width = contentWidth - rightPanelStyle.GetHorizontalFrameSize() 340 | m.sideBlock.Height = contentHeight - lipgloss.Height(rawTitle) - rightPanelStyle.GetVerticalFrameSize() 341 | 342 | sideInfos := rightPanelStyle.Render(lipgloss.JoinVertical(0, rawTitle, m.sideBlock.View())) 343 | content = append(content, sideInfos) 344 | } 345 | 346 | /* 347 | Final rendering 348 | */ 349 | return lipgloss.JoinVertical(0, topBar, lipgloss.JoinHorizontal(0, content...), helpView) 350 | } 351 | -------------------------------------------------------------------------------- /pkg/parser/fake_exec_data_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | "github.com/vmihailenco/msgpack/v5" 8 | ) 9 | 10 | /* 11 | Fake new job message of type /new 12 | 13 | salt/job/20220630000000000000/new/localhost { 14 | "_stamp": "2022-06-30T00:00:00.000000", 15 | "arg": [], 16 | "fun": "test.ping", 17 | "jid": "20220630000000000000", 18 | "minions": [ 19 | "localhost" 20 | ], 21 | "missing": [], 22 | "tgt": "localhost", 23 | "tgt_type": "glob", 24 | "user": "sudo_user" 25 | } 26 | */ 27 | 28 | var expectedNewJob = event.SaltEvent{ 29 | Tag: "salt/job/20220630000000000000/new", 30 | Type: "new", 31 | Module: event.JobModule, 32 | TargetNumber: 1, 33 | Data: event.EventData{ 34 | Timestamp: "2022-06-30T00:00:00.000000", 35 | Fun: "test.ping", 36 | Jid: "20220630000000000000", 37 | Minions: []string{"localhost"}, 38 | Tgt: "localhost", 39 | TgtType: "glob", 40 | User: "salt_user", 41 | }, 42 | IsScheduleJob: false, 43 | } 44 | 45 | func fakeNewJobEvent() []byte { 46 | // Marshal the data using MsgPack 47 | fake := FakeData{ 48 | Timestamp: "2022-06-30T00:00:00.000000", 49 | Fun: "test.ping", 50 | Jid: "20220630000000000000", 51 | Minions: []string{"localhost"}, 52 | Tgt: "localhost", 53 | TgtType: "glob", 54 | User: "salt_user", 55 | } 56 | 57 | fakeBody, err := msgpack.Marshal(fake) 58 | if err != nil { 59 | log.Fatalln(err) 60 | } 61 | 62 | fakeMessage := []byte("salt/job/20220630000000000000/new\n\n") 63 | fakeMessage = append(fakeMessage, fakeBody...) 64 | 65 | return fakeMessage 66 | } 67 | 68 | /* 69 | Fake new job message of type /ret 70 | 71 | salt/job/20220630000000000000/ret/localhost { 72 | "_stamp": "2022-06-30T00:00:00.000000", 73 | "cmd": "_return", 74 | "fun": "test.ping", 75 | "fun_args": [], 76 | "id": "localhost", 77 | "jid": "20220630000000000000", 78 | "retcode": 0, 79 | "return": true, 80 | "success": true 81 | } 82 | 83 | */ 84 | 85 | var expectedReturnJob = event.SaltEvent{ 86 | Tag: "salt/job/20220630000000000000/ret/localhost", 87 | Type: "ret", 88 | Module: event.JobModule, 89 | TargetNumber: 0, 90 | Data: event.EventData{ 91 | Timestamp: "2022-06-30T00:00:00.000000", 92 | Cmd: "_return", 93 | Fun: "test.ping", 94 | ID: "localhost", 95 | Jid: "20220630000000000000", 96 | Retcode: 0, 97 | Return: true, 98 | Success: true, 99 | }, 100 | IsScheduleJob: false, 101 | } 102 | 103 | func fakeRetJobEvent() []byte { 104 | // Marshal the data using MsgPack 105 | fake := FakeData{ 106 | Timestamp: "2022-06-30T00:00:00.000000", 107 | Cmd: "_return", 108 | Fun: "test.ping", 109 | ID: "localhost", 110 | Jid: "20220630000000000000", 111 | Retcode: 0, 112 | Return: true, 113 | Success: true, 114 | } 115 | 116 | fakeBody, err := msgpack.Marshal(fake) 117 | if err != nil { 118 | log.Fatalln(err) 119 | } 120 | 121 | fakeMessage := []byte("salt/job/20220630000000000000/ret/localhost\n\n") 122 | fakeMessage = append(fakeMessage, fakeBody...) 123 | 124 | return fakeMessage 125 | } 126 | 127 | /* 128 | Fake manual scheduled job trigger 129 | salt/job/20220630000000000000/new { 130 | "_stamp": "2022-06-30T00:00:00.000000", 131 | "arg": [ 132 | "sync_all" 133 | ], 134 | "fun": "schedule.run_job", 135 | "jid": "20220630000000000000", 136 | "minions": [ 137 | "localhost" 138 | ], 139 | "missing": [], 140 | "tgt": "localhost", 141 | "tgt_type": "glob", 142 | "user": "salt_user" 143 | } 144 | */ 145 | 146 | var expectedNewScheduleJob = event.SaltEvent{ 147 | Tag: "salt/job/20220630000000000000/new", 148 | Type: "new", 149 | Module: event.JobModule, 150 | TargetNumber: 1, 151 | Data: event.EventData{ 152 | Timestamp: "2022-06-30T00:00:00.000000", 153 | FunArgs: []interface{}{"sync_all"}, 154 | Fun: "schedule.run_job", 155 | Jid: "20220630000000000000", 156 | Minions: []string{"localhost"}, 157 | Tgt: "localhost", 158 | TgtType: "glob", 159 | User: "salt_user", 160 | }, 161 | IsScheduleJob: false, 162 | } 163 | 164 | func fakeNewScheduleJobEvent() []byte { 165 | // Marshal the data using MsgPack 166 | fake := FakeData{ 167 | Timestamp: "2022-06-30T00:00:00.000000", 168 | FunArgs: []interface{}{"sync_all"}, 169 | Fun: "schedule.run_job", 170 | Jid: "20220630000000000000", 171 | Minions: []string{"localhost"}, 172 | Tgt: "localhost", 173 | TgtType: "glob", 174 | User: "salt_user", 175 | } 176 | 177 | fakeBody, err := msgpack.Marshal(fake) 178 | if err != nil { 179 | log.Fatalln(err) 180 | } 181 | 182 | fakeMessage := []byte("salt/job/20220630000000000000/new\n\n") 183 | fakeMessage = append(fakeMessage, fakeBody...) 184 | 185 | return fakeMessage 186 | } 187 | 188 | /* 189 | Fake ack of manual triggered schedule job 190 | 191 | salt/job/20220630000000000000/ret/localhost { 192 | "_stamp": "2022-06-30T00:00:00.000000", 193 | "cmd": "_return", 194 | "fun": "schedule.run_job", 195 | "fun_args": [ 196 | "sync_all" 197 | ], 198 | "id": "localhost", 199 | "jid": "20220630000000000000", 200 | "retcode": 0, 201 | "return": { 202 | "comment": "Scheduling Job sync_all on minion.", 203 | "result": true 204 | }, 205 | "success": true 206 | } 207 | */ 208 | 209 | var expectedAckScheduleJob = event.SaltEvent{ 210 | Tag: "salt/job/20220630000000000000/ret/localhost", 211 | Type: "ret", 212 | Module: event.JobModule, 213 | TargetNumber: 0, 214 | Data: event.EventData{ 215 | Timestamp: "2022-06-30T00:00:00.000000", 216 | Cmd: "_return", 217 | Fun: "schedule.run_job", 218 | FunArgs: []interface{}{"sync_all"}, 219 | ID: "localhost", 220 | Jid: "20220630000000000000", 221 | Retcode: 0, 222 | Return: map[string]interface{}{ 223 | "comment": "Scheduling Job sync_all on minion.", 224 | "result": true, 225 | }, 226 | Success: true, 227 | }, 228 | IsScheduleJob: false, 229 | } 230 | 231 | func fakeAckScheduleJobEvent() []byte { 232 | // Marshal the data using MsgPack 233 | fake := FakeData{ 234 | Timestamp: "2022-06-30T00:00:00.000000", 235 | Cmd: "_return", 236 | Fun: "schedule.run_job", 237 | FunArgs: []interface{}{"sync_all"}, 238 | ID: "localhost", 239 | Jid: "20220630000000000000", 240 | Retcode: 0, 241 | Return: map[string]interface{}{ 242 | "comment": "Scheduling Job sync_all on minion.", 243 | "result": true, 244 | }, 245 | Success: true, 246 | } 247 | 248 | fakeBody, err := msgpack.Marshal(fake) 249 | if err != nil { 250 | log.Fatalln(err) 251 | } 252 | 253 | fakeMessage := []byte("salt/job/20220630000000000000/ret/localhost\n\n") 254 | fakeMessage = append(fakeMessage, fakeBody...) 255 | 256 | return fakeMessage 257 | } 258 | 259 | /* 260 | Fake schedule job return 261 | 262 | salt/job/20220630000000000000/ret/localhost { 263 | "_stamp": "2022-06-30T00:00:00.000000", 264 | "arg": [], 265 | "cmd": "_return", 266 | "fun": "saltutil.sync_all", 267 | "fun_args": [], 268 | "id": "localhost", 269 | "jid": "20220630000000000000", 270 | "pid": 3969911, 271 | "retcode": 0, 272 | "return": { 273 | "beacons": [], 274 | "clouds": [], 275 | "engines": [], 276 | "executors": [], 277 | "grains": [], 278 | "log_handlers": [], 279 | "matchers": [], 280 | "modules": [], 281 | "output": [], 282 | "proxymodules": [], 283 | "renderers": [], 284 | "returners": [], 285 | "sdb": [], 286 | "serializers": [], 287 | "states": [], 288 | "thorium": [], 289 | "utils": [] 290 | }, 291 | "schedule": "sync_all", 292 | "success": true, 293 | "tgt": "localhost", 294 | "tgt_type": "glob" 295 | } 296 | */ 297 | 298 | var expectedScheduleJobReturn = event.SaltEvent{ 299 | Tag: "salt/job/20220630000000000000/ret/localhost", 300 | Type: "ret", 301 | Module: event.JobModule, 302 | TargetNumber: 0, 303 | Data: event.EventData{ 304 | Timestamp: "2022-06-30T00:00:00.000000", 305 | Cmd: "_return", 306 | Fun: "saltutil.sync_all", 307 | ID: "localhost", 308 | Jid: "20220630000000000000", 309 | Retcode: 0, 310 | Return: map[string]interface{}{ 311 | "beacons": []interface{}{}, 312 | "clouds": []interface{}{}, 313 | "engines": []interface{}{}, 314 | "executors": []interface{}{}, 315 | "grains": []interface{}{}, 316 | "log_handlers": []interface{}{}, 317 | "matchers": []interface{}{}, 318 | "modules": []interface{}{}, 319 | "output": []interface{}{}, 320 | "proxymodules": []interface{}{}, 321 | "renderers": []interface{}{}, 322 | "returners": []interface{}{}, 323 | "sdb": []interface{}{}, 324 | "serializers": []interface{}{}, 325 | "states": []interface{}{}, 326 | "thorium": []interface{}{}, 327 | "utils": []interface{}{}, 328 | }, 329 | Schedule: "sync_all", 330 | Success: true, 331 | Tgt: "localhost", 332 | TgtType: "glob", 333 | }, 334 | IsScheduleJob: true, 335 | } 336 | 337 | func fakeScheduleJobReturnEvent() []byte { 338 | // Marshal the data using MsgPack 339 | fake := FakeData{ 340 | Timestamp: "2022-06-30T00:00:00.000000", 341 | Cmd: "_return", 342 | Fun: "saltutil.sync_all", 343 | ID: "localhost", 344 | Jid: "20220630000000000000", 345 | Retcode: 0, 346 | Return: map[string]interface{}{ 347 | "beacons": []interface{}{}, 348 | "clouds": []interface{}{}, 349 | "engines": []interface{}{}, 350 | "executors": []interface{}{}, 351 | "grains": []interface{}{}, 352 | "log_handlers": []interface{}{}, 353 | "matchers": []interface{}{}, 354 | "modules": []interface{}{}, 355 | "output": []interface{}{}, 356 | "proxymodules": []interface{}{}, 357 | "renderers": []interface{}{}, 358 | "returners": []interface{}{}, 359 | "sdb": []interface{}{}, 360 | "serializers": []interface{}{}, 361 | "states": []interface{}{}, 362 | "thorium": []interface{}{}, 363 | "utils": []interface{}{}, 364 | }, 365 | Schedule: "sync_all", 366 | Success: true, 367 | Tgt: "localhost", 368 | TgtType: "glob", 369 | } 370 | 371 | fakeBody, err := msgpack.Marshal(fake) 372 | if err != nil { 373 | log.Fatalln(err) 374 | } 375 | 376 | fakeMessage := []byte("salt/job/20220630000000000000/ret/localhost\n\n") 377 | fakeMessage = append(fakeMessage, fakeBody...) 378 | 379 | return fakeMessage 380 | } 381 | -------------------------------------------------------------------------------- /cmd/salt-exporter/config_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "testing" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | "github.com/kpetremann/salt-exporter/internal/metrics" 10 | "github.com/kpetremann/salt-exporter/pkg/listener" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | func TestReadConfigFlagOnly(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | flags []string 18 | want Config 19 | }{ 20 | { 21 | name: "simple config, flags only", 22 | flags: []string{ 23 | "-host=127.0.0.1", 24 | "-port=8080", 25 | }, 26 | want: Config{ 27 | LogLevel: defaultLogLevel, 28 | ListenAddress: "127.0.0.1", 29 | ListenPort: 8080, 30 | IPCFile: listener.DefaultIPCFilepath, 31 | PKIDir: listener.DefaultPKIDirpath, 32 | TLS: struct { 33 | Enabled bool 34 | Key string 35 | Certificate string 36 | }{ 37 | Enabled: false, 38 | Key: "", 39 | Certificate: "", 40 | }, 41 | Metrics: metrics.Config{ 42 | HealthMinions: true, 43 | Global: struct { 44 | Filters struct { 45 | IgnoreTest bool `mapstructure:"ignore-test"` 46 | IgnoreMock bool `mapstructure:"ignore-mock"` 47 | } 48 | }{ 49 | Filters: struct { 50 | IgnoreTest bool `mapstructure:"ignore-test"` 51 | IgnoreMock bool `mapstructure:"ignore-mock"` 52 | }{ 53 | IgnoreTest: false, 54 | IgnoreMock: false, 55 | }, 56 | }, 57 | SaltNewJobTotal: struct{ Enabled bool }{ 58 | Enabled: true, 59 | }, 60 | SaltExpectedResponsesTotal: struct{ Enabled bool }{ 61 | Enabled: true, 62 | }, 63 | SaltFunctionResponsesTotal: struct { 64 | Enabled bool 65 | AddMinionLabel bool `mapstructure:"add-minion-label"` 66 | }{ 67 | Enabled: true, 68 | AddMinionLabel: false, 69 | }, 70 | SaltScheduledJobReturnTotal: struct { 71 | Enabled bool 72 | AddMinionLabel bool `mapstructure:"add-minion-label"` 73 | }{ 74 | Enabled: true, 75 | AddMinionLabel: false, 76 | }, 77 | SaltResponsesTotal: struct{ Enabled bool }{ 78 | Enabled: true, 79 | }, 80 | SaltFunctionStatus: struct { 81 | Enabled bool 82 | Filters struct { 83 | Functions []string 84 | States []string 85 | } 86 | }{ 87 | Enabled: true, 88 | Filters: struct { 89 | Functions []string 90 | States []string 91 | }{ 92 | Functions: []string{ 93 | "state.highstate", 94 | }, 95 | States: []string{ 96 | "highstate", 97 | }, 98 | }, 99 | }, 100 | }, 101 | }, 102 | }, 103 | { 104 | name: "advanced config, flags only", 105 | flags: []string{ 106 | "-host=127.0.0.1", 107 | "-port=8080", 108 | "-ipc-file=/dev/null", 109 | "-health-minions=false", 110 | "-health-functions-filter=test.sls", 111 | "-health-states-filter=nop", 112 | "-ignore-test", 113 | "-ignore-mock", 114 | "-tls", 115 | "-tls-cert=./cert", 116 | "-tls-key=./key", 117 | }, 118 | want: Config{ 119 | LogLevel: defaultLogLevel, 120 | ListenAddress: "127.0.0.1", 121 | ListenPort: 8080, 122 | IPCFile: "/dev/null", 123 | PKIDir: "/etc/salt/pki/master", 124 | TLS: struct { 125 | Enabled bool 126 | Key string 127 | Certificate string 128 | }{ 129 | Enabled: true, 130 | Key: "./key", 131 | Certificate: "./cert", 132 | }, 133 | Metrics: metrics.Config{ 134 | HealthMinions: false, 135 | Global: struct { 136 | Filters struct { 137 | IgnoreTest bool `mapstructure:"ignore-test"` 138 | IgnoreMock bool `mapstructure:"ignore-mock"` 139 | } 140 | }{ 141 | Filters: struct { 142 | IgnoreTest bool `mapstructure:"ignore-test"` 143 | IgnoreMock bool `mapstructure:"ignore-mock"` 144 | }{ 145 | IgnoreTest: true, 146 | IgnoreMock: true, 147 | }, 148 | }, 149 | SaltNewJobTotal: struct{ Enabled bool }{ 150 | Enabled: true, 151 | }, 152 | SaltExpectedResponsesTotal: struct{ Enabled bool }{ 153 | Enabled: true, 154 | }, 155 | SaltFunctionResponsesTotal: struct { 156 | Enabled bool 157 | AddMinionLabel bool `mapstructure:"add-minion-label"` 158 | }{ 159 | Enabled: true, 160 | AddMinionLabel: false, 161 | }, 162 | SaltScheduledJobReturnTotal: struct { 163 | Enabled bool 164 | AddMinionLabel bool `mapstructure:"add-minion-label"` 165 | }{ 166 | Enabled: true, 167 | AddMinionLabel: false, 168 | }, 169 | SaltResponsesTotal: struct{ Enabled bool }{ 170 | Enabled: false, 171 | }, 172 | SaltFunctionStatus: struct { 173 | Enabled bool 174 | Filters struct { 175 | Functions []string 176 | States []string 177 | } 178 | }{ 179 | Enabled: false, 180 | Filters: struct { 181 | Functions []string 182 | States []string 183 | }{ 184 | Functions: []string{ 185 | "test.sls", 186 | }, 187 | States: []string{ 188 | "nop", 189 | }, 190 | }, 191 | }, 192 | }, 193 | }, 194 | }, 195 | } 196 | 197 | name := os.Args[0] 198 | backupArgs := os.Args 199 | backupCommandLine := flag.CommandLine 200 | defer func() { 201 | flag.CommandLine = backupCommandLine 202 | os.Args = backupArgs 203 | viper.Reset() 204 | }() 205 | 206 | for _, test := range tests { 207 | t.Run(test.name, func(t *testing.T) { 208 | os.Args = append([]string{name}, test.flags...) 209 | flag.CommandLine = flag.NewFlagSet(name, flag.ContinueOnError) 210 | viper.Reset() 211 | 212 | cfg, err := ReadConfig() 213 | 214 | if diff := cmp.Diff(cfg, test.want); diff != "" { 215 | t.Errorf("Mismatch for '%s' test:\n%s", test.name, diff) 216 | } 217 | 218 | if err != nil { 219 | t.Errorf("Unexpected error for '%s': '%s'", test.name, err) 220 | } 221 | }) 222 | } 223 | } 224 | 225 | func TestConfigFileOnly(t *testing.T) { 226 | name := os.Args[0] 227 | backupArgs := os.Args 228 | backupCommandLine := flag.CommandLine 229 | defer func() { 230 | flag.CommandLine = backupCommandLine 231 | os.Args = backupArgs 232 | viper.Reset() 233 | }() 234 | 235 | flags := []string{ 236 | "-config-file=config_test.yml", 237 | } 238 | 239 | os.Args = append([]string{name}, flags...) 240 | flag.CommandLine = flag.NewFlagSet(name, flag.ContinueOnError) 241 | viper.Reset() 242 | 243 | cfg, err := ReadConfig() 244 | 245 | want := Config{ 246 | LogLevel: "info", 247 | ListenAddress: "127.0.0.1", 248 | ListenPort: 2113, 249 | IPCFile: "/dev/null", 250 | PKIDir: "/tmp/pki", 251 | TLS: struct { 252 | Enabled bool 253 | Key string 254 | Certificate string 255 | }{ 256 | Enabled: true, 257 | Key: "/path/to/key", 258 | Certificate: "/path/to/certificate", 259 | }, 260 | Metrics: metrics.Config{ 261 | HealthMinions: true, 262 | Global: struct { 263 | Filters struct { 264 | IgnoreTest bool `mapstructure:"ignore-test"` 265 | IgnoreMock bool `mapstructure:"ignore-mock"` 266 | } 267 | }{ 268 | Filters: struct { 269 | IgnoreTest bool `mapstructure:"ignore-test"` 270 | IgnoreMock bool `mapstructure:"ignore-mock"` 271 | }{ 272 | IgnoreTest: true, 273 | IgnoreMock: false, 274 | }, 275 | }, 276 | SaltNewJobTotal: struct{ Enabled bool }{ 277 | Enabled: true, 278 | }, 279 | SaltExpectedResponsesTotal: struct{ Enabled bool }{ 280 | Enabled: true, 281 | }, 282 | SaltFunctionResponsesTotal: struct { 283 | Enabled bool 284 | AddMinionLabel bool `mapstructure:"add-minion-label"` 285 | }{ 286 | Enabled: true, 287 | AddMinionLabel: true, 288 | }, 289 | SaltScheduledJobReturnTotal: struct { 290 | Enabled bool 291 | AddMinionLabel bool `mapstructure:"add-minion-label"` 292 | }{ 293 | Enabled: true, 294 | AddMinionLabel: true, 295 | }, 296 | SaltResponsesTotal: struct{ Enabled bool }{ 297 | Enabled: true, 298 | }, 299 | SaltFunctionStatus: struct { 300 | Enabled bool 301 | Filters struct { 302 | Functions []string 303 | States []string 304 | } 305 | }{ 306 | Enabled: true, 307 | Filters: struct { 308 | Functions []string 309 | States []string 310 | }{ 311 | Functions: []string{ 312 | "state.sls", 313 | }, 314 | States: []string{ 315 | "test", 316 | }, 317 | }, 318 | }, 319 | }, 320 | } 321 | 322 | if diff := cmp.Diff(cfg, want); diff != "" { 323 | t.Errorf("Mismatch:\n%s", diff) 324 | } 325 | 326 | if err != nil { 327 | t.Errorf("Unexpected error: '%s'", err) 328 | } 329 | } 330 | 331 | func TestConfigFileWithFlags(t *testing.T) { 332 | name := os.Args[0] 333 | backupArgs := os.Args 334 | backupCommandLine := flag.CommandLine 335 | defer func() { 336 | flag.CommandLine = backupCommandLine 337 | os.Args = backupArgs 338 | viper.Reset() 339 | }() 340 | 341 | flags := []string{ 342 | "-config-file=config_test.yml", 343 | "-host=127.0.0.1", 344 | "-port=8080", 345 | "-health-minions=false", 346 | "-health-functions-filter=test.sls", 347 | "-health-states-filter=nop", 348 | "-ignore-mock", 349 | "-ipc-file=/somewhere", 350 | } 351 | 352 | os.Args = append([]string{name}, flags...) 353 | flag.CommandLine = flag.NewFlagSet(name, flag.ContinueOnError) 354 | viper.Reset() 355 | 356 | cfg, err := ReadConfig() 357 | want := Config{ 358 | LogLevel: "info", 359 | ListenAddress: "127.0.0.1", 360 | ListenPort: 8080, 361 | IPCFile: "/somewhere", 362 | PKIDir: "/tmp/pki", 363 | TLS: struct { 364 | Enabled bool 365 | Key string 366 | Certificate string 367 | }{ 368 | Enabled: true, 369 | Key: "/path/to/key", 370 | Certificate: "/path/to/certificate", 371 | }, 372 | Metrics: metrics.Config{ 373 | HealthMinions: false, 374 | Global: struct { 375 | Filters struct { 376 | IgnoreTest bool `mapstructure:"ignore-test"` 377 | IgnoreMock bool `mapstructure:"ignore-mock"` 378 | } 379 | }{ 380 | Filters: struct { 381 | IgnoreTest bool `mapstructure:"ignore-test"` 382 | IgnoreMock bool `mapstructure:"ignore-mock"` 383 | }{ 384 | IgnoreTest: true, 385 | IgnoreMock: true, 386 | }, 387 | }, 388 | SaltNewJobTotal: struct{ Enabled bool }{ 389 | Enabled: true, 390 | }, 391 | SaltExpectedResponsesTotal: struct{ Enabled bool }{ 392 | Enabled: true, 393 | }, 394 | SaltFunctionResponsesTotal: struct { 395 | Enabled bool 396 | AddMinionLabel bool `mapstructure:"add-minion-label"` 397 | }{ 398 | Enabled: true, 399 | AddMinionLabel: true, 400 | }, 401 | SaltScheduledJobReturnTotal: struct { 402 | Enabled bool 403 | AddMinionLabel bool `mapstructure:"add-minion-label"` 404 | }{ 405 | Enabled: true, 406 | AddMinionLabel: true, 407 | }, 408 | SaltResponsesTotal: struct{ Enabled bool }{ 409 | Enabled: true, 410 | }, 411 | SaltFunctionStatus: struct { 412 | Enabled bool 413 | Filters struct { 414 | Functions []string 415 | States []string 416 | } 417 | }{ 418 | Enabled: true, 419 | Filters: struct { 420 | Functions []string 421 | States []string 422 | }{ 423 | Functions: []string{ 424 | "test.sls", 425 | }, 426 | States: []string{ 427 | "nop", 428 | }, 429 | }, 430 | }, 431 | }, 432 | } 433 | 434 | if diff := cmp.Diff(cfg, want); diff != "" { 435 | t.Errorf("Mismatch:\n%s", diff) 436 | } 437 | 438 | if err != nil { 439 | t.Errorf("Unexpected error: '%s'", err) 440 | } 441 | } 442 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= 2 | github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= 3 | github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= 4 | github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= 5 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 6 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 10 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 11 | github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= 12 | github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= 13 | github.com/charmbracelet/bubbletea v1.1.1 h1:KJ2/DnmpfqFtDNVTvYZ6zpPFL9iRCRr0qqKOCvppbPY= 14 | github.com/charmbracelet/bubbletea v1.1.1/go.mod h1:9Ogk0HrdbHolIKHdjfFpyXJmiCzGwy+FesYkZr7hYU4= 15 | github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE= 16 | github.com/charmbracelet/bubbletea v1.2.4/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM= 17 | github.com/charmbracelet/lipgloss v0.13.1 h1:Oik/oqDTMVA01GetT4JdEC033dNzWoQHdWnHnQmXE2A= 18 | github.com/charmbracelet/lipgloss v0.13.1/go.mod h1:zaYVJ2xKSKEnTEEbX6uAHabh2d975RJ+0yfkFpRBz5U= 19 | github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= 20 | github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= 21 | github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY= 22 | github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= 23 | github.com/charmbracelet/x/ansi v0.6.0 h1:qOznutrb93gx9oMiGf7caF7bqqubh6YIM0SWKyA08pA= 24 | github.com/charmbracelet/x/ansi v0.6.0/go.mod h1:KBUFw1la39nl0dLl10l5ORDAqGXaeurTQmwyyVKse/Q= 25 | github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4hwm0x0= 26 | github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= 27 | github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= 28 | github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= 29 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 30 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 31 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 32 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 33 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 34 | github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= 35 | github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= 36 | github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= 37 | github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= 38 | github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= 39 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 40 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 41 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 42 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 43 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 44 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 45 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 46 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 47 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 48 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 49 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 50 | github.com/k0kubun/pp/v3 v3.2.0 h1:h33hNTZ9nVFNP3u2Fsgz8JXiF5JINoZfFq4SvKJwNcs= 51 | github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapdz1EwA= 52 | github.com/k0kubun/pp/v3 v3.4.1 h1:1WdFZDRRqe8UsR61N/2RoOZ3ziTEqgTPVqKrHeb779Y= 53 | github.com/k0kubun/pp/v3 v3.4.1/go.mod h1:+SiNiqKnBfw1Nkj82Lh5bIeKQOAkPy6Xw9CAZUZ8npI= 54 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 55 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 56 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 57 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 58 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 59 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 60 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 61 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 62 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 63 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 64 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 65 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 66 | github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= 67 | github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 68 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 69 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 70 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 71 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 72 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 73 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 74 | github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= 75 | github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= 76 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 77 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 78 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 79 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 80 | github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= 81 | github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= 82 | github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= 83 | github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= 84 | github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= 85 | github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= 86 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 87 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 88 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 89 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 90 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 91 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 92 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 93 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 94 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= 95 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 96 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 97 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 98 | github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA= 99 | github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= 100 | github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= 101 | github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= 102 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 103 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 104 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 105 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 106 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 107 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 108 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 109 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 110 | github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= 111 | github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= 112 | github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= 113 | github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= 114 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 115 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 116 | github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= 117 | github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= 118 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 119 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 120 | github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= 121 | github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= 122 | github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= 123 | github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 124 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 125 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 126 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 127 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 128 | github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= 129 | github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= 130 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 131 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 132 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 133 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 134 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 135 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 136 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 137 | github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= 138 | github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= 139 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 140 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 141 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 142 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 143 | golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= 144 | golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= 145 | golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= 146 | golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= 147 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 148 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 149 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 150 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 151 | golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 152 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 153 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 154 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 155 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= 156 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 157 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 158 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 159 | golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= 160 | golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= 161 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 162 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 163 | google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= 164 | google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 165 | google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= 166 | google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 167 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 168 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 169 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 170 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 171 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 172 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 173 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 174 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 175 | -------------------------------------------------------------------------------- /pkg/parser/fake_state_data_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/kpetremann/salt-exporter/pkg/event" 7 | "github.com/vmihailenco/msgpack/v5" 8 | ) 9 | 10 | var False = false 11 | var True = true 12 | 13 | /* 14 | Fake state.sls job 15 | 16 | salt/job/20220630000000000000/new { 17 | "_stamp": "2022-06-30T00:00:00.000000", 18 | "arg": [ 19 | "test" 20 | ], 21 | "fun": "state.sls", 22 | "jid": "20220630000000000000", 23 | "minions": [ 24 | "node1" 25 | ], 26 | "missing": [], 27 | "tgt": "node1", 28 | "tgt_type": "glob", 29 | "user": "salt_user" 30 | } 31 | */ 32 | 33 | var expectedNewStateSlsJob = event.SaltEvent{ 34 | Tag: "salt/job/20220630000000000000/new", 35 | Type: "new", 36 | Module: event.JobModule, 37 | TargetNumber: 1, 38 | Data: event.EventData{ 39 | Timestamp: "2022-06-30T00:00:00.000000", 40 | Fun: "state.sls", 41 | Arg: []interface{}{"test"}, 42 | Jid: "20220630000000000000", 43 | Minions: []string{"node1"}, 44 | Missing: []string{}, 45 | Tgt: "node1", 46 | TgtType: "glob", 47 | User: "salt_user", 48 | }, 49 | IsScheduleJob: false, 50 | IsTest: false, 51 | IsMock: false, 52 | } 53 | 54 | func fakeNewStateSlsJobEvent() []byte { 55 | // Marshal the data using MsgPack 56 | fake := FakeData{ 57 | Timestamp: "2022-06-30T00:00:00.000000", 58 | Fun: "state.sls", 59 | Arg: []interface{}{"test"}, 60 | Jid: "20220630000000000000", 61 | Minions: []string{"node1"}, 62 | Missing: []string{}, 63 | Tgt: "node1", 64 | TgtType: "glob", 65 | User: "salt_user", 66 | } 67 | 68 | fakeBody, err := msgpack.Marshal(fake) 69 | if err != nil { 70 | log.Fatalln(err) 71 | } 72 | 73 | fakeMessage := []byte("salt/job/20220630000000000000/new\n\n") 74 | fakeMessage = append(fakeMessage, fakeBody...) 75 | 76 | return fakeMessage 77 | } 78 | 79 | /* 80 | Fake state.sls ret 81 | 82 | salt/job/20220630000000000000/ret/node1 { 83 | "_stamp": "2022-06-30T00:00:00.000000", 84 | "cmd": "_return", 85 | "fun": "state.sls", 86 | "fun_args": [ 87 | "test" 88 | ], 89 | "id": "node1", 90 | "jid": "20220630000000000000", 91 | "out": "highstate", 92 | "retcode": 0, 93 | "return": { 94 | "test_|-dummy test_|-Dummy test_|-nop": { 95 | "__id__": "dummy test", 96 | "__run_num__": 0, 97 | "__sls__": "test", 98 | "changes": {}, 99 | "comment": "Success!", 100 | "duration": 0.481, 101 | "name": "Dummy test", 102 | "result": true, 103 | "start_time": "09:17:08.822722" 104 | } 105 | }, 106 | "success": true 107 | } 108 | 109 | */ 110 | 111 | var expectedStateSlsReturn = event.SaltEvent{ 112 | Tag: "salt/job/20220630000000000000/ret/node1", 113 | Type: "ret", 114 | Module: event.JobModule, 115 | TargetNumber: 0, 116 | Data: event.EventData{ 117 | Timestamp: "2022-06-30T00:00:00.000000", 118 | Cmd: "_return", 119 | Fun: "state.sls", 120 | FunArgs: []interface{}{"test"}, 121 | ID: "node1", 122 | Jid: "20220630000000000000", 123 | Out: "highstate", 124 | Retcode: 0, 125 | Return: map[string]interface{}{ 126 | "test_|-dummy test_|-Dummy test_|-nop": map[string]interface{}{ 127 | "__id__": "dummy test", 128 | "__run_num__": int8(0), 129 | "__sls__": "test", 130 | "changes": map[string]interface{}{}, 131 | "comment": "Success!", 132 | "duration": 0.481, 133 | "name": "Dummy test", 134 | "result": true, 135 | "start_time": "09:17:08.822722", 136 | }, 137 | }, 138 | Success: true, 139 | }, 140 | IsScheduleJob: false, 141 | IsTest: false, 142 | IsMock: false, 143 | StateModuleSuccess: &True, 144 | } 145 | 146 | func fakeStateSlsReturnEvent() []byte { 147 | // Marshal the data using MsgPack 148 | fake := FakeData{ 149 | Timestamp: "2022-06-30T00:00:00.000000", 150 | Cmd: "_return", 151 | Fun: "state.sls", 152 | FunArgs: []interface{}{"test"}, 153 | ID: "node1", 154 | Out: "highstate", 155 | Jid: "20220630000000000000", 156 | Retcode: 0, 157 | Return: map[string]interface{}{ 158 | "test_|-dummy test_|-Dummy test_|-nop": map[string]interface{}{ 159 | "__id__": "dummy test", 160 | "__run_num__": 0, 161 | "__sls__": "test", 162 | "changes": map[string]interface{}{}, 163 | "comment": "Success!", 164 | "duration": 0.481, 165 | "name": "Dummy test", 166 | "result": true, 167 | "start_time": "09:17:08.822722", 168 | }, 169 | }, 170 | Success: true, 171 | } 172 | 173 | fakeBody, err := msgpack.Marshal(fake) 174 | if err != nil { 175 | log.Fatalln(err) 176 | } 177 | 178 | fakeMessage := []byte("salt/job/20220630000000000000/ret/node1\n\n") 179 | fakeMessage = append(fakeMessage, fakeBody...) 180 | 181 | return fakeMessage 182 | } 183 | 184 | /* 185 | 186 | Fake state.single 187 | 188 | salt/job/20220630000000000000/new { 189 | "_stamp": "2022-06-30T00:00:00.000000", 190 | "arg": [ 191 | { 192 | "__kwarg__": true, 193 | "fun": "test.nop", 194 | "name": "toto" 195 | } 196 | ], 197 | "fun": "state.single", 198 | "jid": "20220630000000000000", 199 | "minions": [ 200 | "node1" 201 | ], 202 | "missing": [], 203 | "tgt": "node1", 204 | "tgt_type": "glob", 205 | "user": "salt_user" 206 | } 207 | 208 | */ 209 | 210 | var expectedNewStateSingle = event.SaltEvent{ 211 | Tag: "salt/job/20220630000000000000/new", 212 | Type: "new", 213 | Module: event.JobModule, 214 | TargetNumber: 1, 215 | Data: event.EventData{ 216 | Timestamp: "2022-06-30T00:00:00.000000", 217 | Arg: []interface{}{ 218 | map[string]interface{}{ 219 | "__kwarg__": true, 220 | "fun": "test.nop", 221 | "name": "toto", 222 | }, 223 | }, 224 | Fun: "state.single", 225 | Jid: "20220630000000000000", 226 | Minions: []string{"node1"}, 227 | Missing: []string{}, 228 | Tgt: "node1", 229 | TgtType: "glob", 230 | User: "salt_user", 231 | }, 232 | IsScheduleJob: false, 233 | IsTest: false, 234 | IsMock: false, 235 | } 236 | 237 | func fakeNewStateSingleEvent() []byte { 238 | // Marshal the data using MsgPack 239 | fake := FakeData{ 240 | Timestamp: "2022-06-30T00:00:00.000000", 241 | Arg: []interface{}{ 242 | map[string]interface{}{ 243 | "__kwarg__": true, 244 | "fun": "test.nop", 245 | "name": "toto", 246 | }, 247 | }, 248 | Fun: "state.single", 249 | Jid: "20220630000000000000", 250 | Minions: []string{"node1"}, 251 | Missing: []string{}, 252 | Tgt: "node1", 253 | TgtType: "glob", 254 | User: "salt_user", 255 | } 256 | 257 | fakeBody, err := msgpack.Marshal(fake) 258 | if err != nil { 259 | log.Fatalln(err) 260 | } 261 | 262 | fakeMessage := []byte("salt/job/20220630000000000000/new\n\n") 263 | fakeMessage = append(fakeMessage, fakeBody...) 264 | 265 | return fakeMessage 266 | } 267 | 268 | /* 269 | 270 | Fake state.single return 271 | 272 | salt/job/20220630000000000000/ret/node1 { 273 | "_stamp": "2022-06-30T00:00:00.000000", 274 | "cmd": "_return", 275 | "fun": "state.single", 276 | "fun_args": [ 277 | { 278 | "fun": "test.nop", 279 | "name": "toto" 280 | } 281 | ], 282 | "id": "node1", 283 | "jid": "20220630000000000000", 284 | "out": "highstate", 285 | "retcode": 0, 286 | "return": { 287 | "test_|-toto_|-toto_|-nop": { 288 | "__id__": "toto", 289 | "__run_num__": 0, 290 | "__sls__": null, 291 | "changes": {}, 292 | "comment": "Success!", 293 | "duration": 0.49, 294 | "name": "toto", 295 | "result": true, 296 | "start_time": "09:20:38.462572" 297 | } 298 | }, 299 | "success": true 300 | } 301 | 302 | */ 303 | 304 | var expectedStateSingleReturn = event.SaltEvent{ 305 | Tag: "salt/job/20220630000000000000/ret/node1", 306 | Type: "ret", 307 | Module: event.JobModule, 308 | TargetNumber: 0, 309 | Data: event.EventData{ 310 | Timestamp: "2022-06-30T00:00:00.000000", 311 | Cmd: "_return", 312 | Fun: "state.single", 313 | FunArgs: []interface{}{ 314 | map[string]interface{}{ 315 | "fun": "test.nop", 316 | "name": "toto", 317 | }, 318 | }, 319 | ID: "node1", 320 | Jid: "20220630000000000000", 321 | Out: "highstate", 322 | Retcode: 0, 323 | Return: map[string]interface{}{ 324 | "test_|-toto_|-toto_|-nop": map[string]interface{}{ 325 | "__id__": "toto", 326 | "__run_num__": int8(0), 327 | "__sls__": nil, 328 | "changes": map[string]interface{}{}, 329 | "comment": "Success!", 330 | "duration": 0.49, 331 | "name": "toto", 332 | "result": true, 333 | "start_time": "09:20:38.462572", 334 | }, 335 | }, 336 | Success: true, 337 | }, 338 | IsScheduleJob: false, 339 | IsTest: false, 340 | IsMock: false, 341 | StateModuleSuccess: &True, 342 | } 343 | 344 | func fakeStateSingleReturnEvent() []byte { 345 | // Marshal the data using MsgPack 346 | fake := FakeData{ 347 | Timestamp: "2022-06-30T00:00:00.000000", 348 | Cmd: "_return", 349 | Fun: "state.single", 350 | FunArgs: []interface{}{ 351 | map[string]interface{}{ 352 | "fun": "test.nop", 353 | "name": "toto", 354 | }, 355 | }, 356 | ID: "node1", 357 | Jid: "20220630000000000000", 358 | Out: "highstate", 359 | Retcode: 0, 360 | Return: map[string]interface{}{ 361 | "test_|-toto_|-toto_|-nop": map[string]interface{}{ 362 | "__id__": "toto", 363 | "__run_num__": 0, 364 | "__sls__": nil, 365 | "changes": map[string]interface{}{}, 366 | "comment": "Success!", 367 | "duration": 0.49, 368 | "name": "toto", 369 | "result": true, 370 | "start_time": "09:20:38.462572", 371 | }, 372 | }, 373 | Success: true, 374 | } 375 | 376 | fakeBody, err := msgpack.Marshal(fake) 377 | if err != nil { 378 | log.Fatalln(err) 379 | } 380 | 381 | fakeMessage := []byte("salt/job/20220630000000000000/ret/node1\n\n") 382 | fakeMessage = append(fakeMessage, fakeBody...) 383 | 384 | return fakeMessage 385 | } 386 | 387 | /* 388 | Fake state.sls job test=True mock=True 389 | 390 | salt/job/20220630000000000000/new { 391 | "_stamp": "2022-06-30T00:00:00.000000", 392 | "fun": "state.sls", 393 | "arg": [ 394 | "somestate", 395 | { 396 | "__kwarg__": true, 397 | "test": true, 398 | "mock": true 399 | } 400 | ], 401 | "jid": "20220630000000000000", 402 | "minions": [ 403 | "node1" 404 | ], 405 | "missing": [], 406 | "tgt": "node1", 407 | "tgt_type": "glob", 408 | "user": "salt_user" 409 | } 410 | */ 411 | 412 | var expectedNewTestMockStateSlsJob = event.SaltEvent{ 413 | Tag: "salt/job/20220630000000000000/new", 414 | Type: "new", 415 | Module: event.JobModule, 416 | TargetNumber: 1, 417 | Data: event.EventData{ 418 | Timestamp: "2022-06-30T00:00:00.000000", 419 | Fun: "state.sls", 420 | Arg: []interface{}{ 421 | "somestate", 422 | map[string]interface{}{ 423 | "test": true, 424 | "mock": true, 425 | }, 426 | }, 427 | Jid: "20220630000000000000", 428 | Minions: []string{"node1"}, 429 | Missing: []string{}, 430 | Tgt: "node1", 431 | TgtType: "glob", 432 | User: "salt_user", 433 | }, 434 | IsScheduleJob: false, 435 | IsTest: true, 436 | IsMock: true, 437 | } 438 | 439 | func fakeNewTestMockStateSlsJobEvent() []byte { 440 | // Marshal the data using MsgPack 441 | fake := FakeData{ 442 | Timestamp: "2022-06-30T00:00:00.000000", 443 | Fun: "state.sls", 444 | Arg: []interface{}{ 445 | "somestate", 446 | map[string]interface{}{ 447 | "test": true, 448 | "mock": true, 449 | }, 450 | }, 451 | Jid: "20220630000000000000", 452 | Minions: []string{"node1"}, 453 | Missing: []string{}, 454 | Tgt: "node1", 455 | TgtType: "glob", 456 | User: "salt_user", 457 | } 458 | 459 | fakeBody, err := msgpack.Marshal(fake) 460 | if err != nil { 461 | log.Fatalln(err) 462 | } 463 | 464 | fakeMessage := []byte("salt/job/20220630000000000000/new\n\n") 465 | fakeMessage = append(fakeMessage, fakeBody...) 466 | 467 | return fakeMessage 468 | } 469 | 470 | /* 471 | Fake state.sls ret 472 | 473 | salt/job/20220630000000000000/ret/node1 { 474 | "_stamp": "2022-06-30T00:00:00.000000", 475 | "cmd": "_return", 476 | "fun": "state.sls", 477 | "fun_args": [ 478 | "somestate", 479 | { 480 | "test": true, 481 | "mock": true 482 | } 483 | ], 484 | "id": "node1", 485 | "jid": "20220630000000000000", 486 | "out": "highstate", 487 | "retcode": 1, 488 | "return": { 489 | "somestate_|-dummy somestate_|-Dummy somestate_|-nop": { 490 | "__id__": "dummy somestate", 491 | "__run_num__": 0, 492 | "__sls__": "somestate", 493 | "changes": {}, 494 | "comment": "Success!", 495 | "duration": 0.481, 496 | "name": "Dummy somestate", 497 | "result": true, 498 | "start_time": "09:17:08.822722" 499 | }, 500 | "somestate_|-failed_|-failed_|-fail_with_changes": { 501 | "__id__": "failed", 502 | "__run_num__": 2, 503 | "__sls__": "test", 504 | "changes": { 505 | "testing": { 506 | "new": "Something pretended to change", 507 | "old": "Unchanged" 508 | } 509 | }, 510 | "comment": "Failure!", 511 | "duration": 0.579, 512 | "name": "failed", 513 | "result": false, 514 | "start_time": "09:17:02.812345" 515 | }, 516 | }, 517 | "success": true 518 | } 519 | 520 | */ 521 | 522 | var expectedTestMockStateSlsReturn = event.SaltEvent{ 523 | Tag: "salt/job/20220630000000000000/ret/node1", 524 | Type: "ret", 525 | Module: event.JobModule, 526 | TargetNumber: 0, 527 | Data: event.EventData{ 528 | Timestamp: "2022-06-30T00:00:00.000000", 529 | Cmd: "_return", 530 | Fun: "state.sls", 531 | FunArgs: []interface{}{ 532 | "somestate", 533 | map[string]interface{}{ 534 | "test": true, 535 | "mock": true, 536 | }, 537 | }, 538 | ID: "node1", 539 | Jid: "20220630000000000000", 540 | Out: "highstate", 541 | Retcode: 1, 542 | Return: map[string]interface{}{ 543 | "somestate_|-dummy somestate_|-Dummy somestate_|-nop": map[string]interface{}{ 544 | "__id__": "dummy somestate", 545 | "__run_num__": int8(0), 546 | "__sls__": "somestate", 547 | "changes": map[string]interface{}{}, 548 | "comment": "Success!", 549 | "duration": 0.481, 550 | "name": "Dummy somestate", 551 | "result": true, 552 | "start_time": "09:17:08.822722", 553 | }, 554 | "somestate_|-failed_|-failed_|-fail_with_changes": map[string]interface{}{ 555 | "__id__": "dummy somestate", 556 | "__run_num__": int8(2), 557 | "__sls__": "somestate", 558 | "changes": map[string]interface{}{}, 559 | "comment": "Failure!", 560 | "duration": 0.579, 561 | "name": "failed", 562 | "result": false, 563 | "start_time": "09:17:08.812345", 564 | }, 565 | }, 566 | Success: true, 567 | }, 568 | IsScheduleJob: false, 569 | IsTest: true, 570 | IsMock: true, 571 | StateModuleSuccess: &False, 572 | } 573 | 574 | func fakeTestMockStateSlsReturnEvent() []byte { 575 | // Marshal the data using MsgPack 576 | fake := FakeData{ 577 | Timestamp: "2022-06-30T00:00:00.000000", 578 | Cmd: "_return", 579 | Fun: "state.sls", 580 | FunArgs: []interface{}{ 581 | "somestate", 582 | map[string]interface{}{ 583 | "test": true, 584 | "mock": true, 585 | }, 586 | }, 587 | ID: "node1", 588 | Out: "highstate", 589 | Jid: "20220630000000000000", 590 | Retcode: 1, 591 | Return: map[string]interface{}{ 592 | "somestate_|-dummy somestate_|-Dummy somestate_|-nop": map[string]interface{}{ 593 | "__id__": "dummy somestate", 594 | "__run_num__": 0, 595 | "__sls__": "somestate", 596 | "changes": map[string]interface{}{}, 597 | "comment": "Success!", 598 | "duration": 0.481, 599 | "name": "Dummy somestate", 600 | "result": true, 601 | "start_time": "09:17:08.822722", 602 | }, 603 | "somestate_|-failed_|-failed_|-fail_with_changes": map[string]interface{}{ 604 | "__id__": "dummy somestate", 605 | "__run_num__": int8(2), 606 | "__sls__": "somestate", 607 | "changes": map[string]interface{}{}, 608 | "comment": "Failure!", 609 | "duration": 0.579, 610 | "name": "failed", 611 | "result": false, 612 | "start_time": "09:17:08.812345", 613 | }, 614 | }, 615 | Success: true, 616 | } 617 | 618 | fakeBody, err := msgpack.Marshal(fake) 619 | if err != nil { 620 | log.Fatalln(err) 621 | } 622 | 623 | fakeMessage := []byte("salt/job/20220630000000000000/ret/node1\n\n") 624 | fakeMessage = append(fakeMessage, fakeBody...) 625 | 626 | return fakeMessage 627 | } 628 | -------------------------------------------------------------------------------- /grafana/Saltstack-1682711767600.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS", 5 | "label": "Prometheus", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "grafana", 16 | "id": "grafana", 17 | "name": "Grafana", 18 | "version": "9.5.1" 19 | }, 20 | { 21 | "type": "datasource", 22 | "id": "prometheus", 23 | "name": "Prometheus", 24 | "version": "1.0.0" 25 | }, 26 | { 27 | "type": "panel", 28 | "id": "timeseries", 29 | "name": "Time series", 30 | "version": "" 31 | } 32 | ], 33 | "annotations": { 34 | "list": [ 35 | { 36 | "builtIn": 1, 37 | "datasource": { 38 | "type": "grafana", 39 | "uid": "-- Grafana --" 40 | }, 41 | "enable": true, 42 | "hide": true, 43 | "iconColor": "rgba(0, 211, 255, 1)", 44 | "name": "Annotations & Alerts", 45 | "target": { 46 | "limit": 100, 47 | "matchAny": false, 48 | "tags": [], 49 | "type": "dashboard" 50 | }, 51 | "type": "dashboard" 52 | } 53 | ] 54 | }, 55 | "editable": true, 56 | "fiscalYearStartMonth": 0, 57 | "graphTooltip": 0, 58 | "id": null, 59 | "links": [], 60 | "liveNow": false, 61 | "panels": [ 62 | { 63 | "collapsed": false, 64 | "gridPos": { 65 | "h": 1, 66 | "w": 24, 67 | "x": 0, 68 | "y": 0 69 | }, 70 | "id": 5, 71 | "panels": [], 72 | "title": "Jobs", 73 | "type": "row" 74 | }, 75 | { 76 | "datasource": { 77 | "type": "prometheus", 78 | "uid": "${DS_PROMETHEUS}" 79 | }, 80 | "fieldConfig": { 81 | "defaults": { 82 | "color": { 83 | "mode": "palette-classic" 84 | }, 85 | "custom": { 86 | "axisCenteredZero": false, 87 | "axisColorMode": "text", 88 | "axisLabel": "", 89 | "axisPlacement": "auto", 90 | "barAlignment": 0, 91 | "drawStyle": "line", 92 | "fillOpacity": 100, 93 | "gradientMode": "hue", 94 | "hideFrom": { 95 | "legend": false, 96 | "tooltip": false, 97 | "viz": false 98 | }, 99 | "lineInterpolation": "stepBefore", 100 | "lineWidth": 1, 101 | "pointSize": 5, 102 | "scaleDistribution": { 103 | "type": "linear" 104 | }, 105 | "showPoints": "auto", 106 | "spanNulls": false, 107 | "stacking": { 108 | "group": "A", 109 | "mode": "none" 110 | }, 111 | "thresholdsStyle": { 112 | "mode": "off" 113 | } 114 | }, 115 | "mappings": [], 116 | "min": 0, 117 | "thresholds": { 118 | "mode": "absolute", 119 | "steps": [ 120 | { 121 | "color": "green", 122 | "value": null 123 | }, 124 | { 125 | "color": "red", 126 | "value": 80 127 | } 128 | ] 129 | } 130 | }, 131 | "overrides": [] 132 | }, 133 | "gridPos": { 134 | "h": 9, 135 | "w": 12, 136 | "x": 0, 137 | "y": 1 138 | }, 139 | "id": 2, 140 | "options": { 141 | "legend": { 142 | "calcs": [], 143 | "displayMode": "table", 144 | "placement": "right", 145 | "showLegend": true 146 | }, 147 | "tooltip": { 148 | "mode": "single", 149 | "sort": "none" 150 | } 151 | }, 152 | "targets": [ 153 | { 154 | "datasource": { 155 | "type": "prometheus", 156 | "uid": "${DS_PROMETHEUS}" 157 | }, 158 | "editorMode": "code", 159 | "expr": "sum by (function, state) (increase(salt_new_job_total{function=~\"$function\", state=~\"$state\"}[$__rate_interval]))", 160 | "hide": false, 161 | "legendFormat": "\"{{function}} {{state}}\"", 162 | "range": true, 163 | "refId": "A" 164 | } 165 | ], 166 | "title": "New job", 167 | "type": "timeseries" 168 | }, 169 | { 170 | "datasource": { 171 | "type": "prometheus", 172 | "uid": "${DS_PROMETHEUS}" 173 | }, 174 | "description": "", 175 | "fieldConfig": { 176 | "defaults": { 177 | "color": { 178 | "mode": "palette-classic" 179 | }, 180 | "custom": { 181 | "axisCenteredZero": false, 182 | "axisColorMode": "text", 183 | "axisLabel": "", 184 | "axisPlacement": "auto", 185 | "barAlignment": 0, 186 | "drawStyle": "line", 187 | "fillOpacity": 100, 188 | "gradientMode": "hue", 189 | "hideFrom": { 190 | "legend": false, 191 | "tooltip": false, 192 | "viz": false 193 | }, 194 | "lineInterpolation": "stepBefore", 195 | "lineWidth": 1, 196 | "pointSize": 5, 197 | "scaleDistribution": { 198 | "type": "linear" 199 | }, 200 | "showPoints": "auto", 201 | "spanNulls": false, 202 | "stacking": { 203 | "group": "A", 204 | "mode": "none" 205 | }, 206 | "thresholdsStyle": { 207 | "mode": "off" 208 | } 209 | }, 210 | "mappings": [], 211 | "min": 0, 212 | "thresholds": { 213 | "mode": "absolute", 214 | "steps": [ 215 | { 216 | "color": "green", 217 | "value": null 218 | }, 219 | { 220 | "color": "red", 221 | "value": 80 222 | } 223 | ] 224 | } 225 | }, 226 | "overrides": [] 227 | }, 228 | "gridPos": { 229 | "h": 9, 230 | "w": 12, 231 | "x": 12, 232 | "y": 1 233 | }, 234 | "id": 3, 235 | "options": { 236 | "legend": { 237 | "calcs": [], 238 | "displayMode": "table", 239 | "placement": "right", 240 | "showLegend": true 241 | }, 242 | "tooltip": { 243 | "mode": "single", 244 | "sort": "none" 245 | } 246 | }, 247 | "targets": [ 248 | { 249 | "datasource": { 250 | "type": "prometheus", 251 | "uid": "${DS_PROMETHEUS}" 252 | }, 253 | "editorMode": "code", 254 | "expr": "sum by (instance, function, state) (increase(salt_expected_responses_total{function=~\"$function\", state=~\"$state\"}[$__rate_interval])) - sum by (instance, function, state) (increase(salt_function_responses_total{function=~\"$function\", state=~\"$state\"}[$__rate_interval]))", 255 | "hide": false, 256 | "legendFormat": "\"{{function}} {{state}}\"", 257 | "range": true, 258 | "refId": "A" 259 | } 260 | ], 261 | "title": "Missing responses", 262 | "type": "timeseries" 263 | }, 264 | { 265 | "datasource": { 266 | "type": "prometheus", 267 | "uid": "${DS_PROMETHEUS}" 268 | }, 269 | "description": "", 270 | "fieldConfig": { 271 | "defaults": { 272 | "color": { 273 | "mode": "palette-classic" 274 | }, 275 | "custom": { 276 | "axisCenteredZero": true, 277 | "axisColorMode": "text", 278 | "axisLabel": "", 279 | "axisPlacement": "auto", 280 | "barAlignment": 0, 281 | "drawStyle": "line", 282 | "fillOpacity": 100, 283 | "gradientMode": "hue", 284 | "hideFrom": { 285 | "legend": false, 286 | "tooltip": false, 287 | "viz": false 288 | }, 289 | "lineInterpolation": "stepBefore", 290 | "lineWidth": 1, 291 | "pointSize": 1, 292 | "scaleDistribution": { 293 | "type": "linear" 294 | }, 295 | "showPoints": "auto", 296 | "spanNulls": false, 297 | "stacking": { 298 | "group": "A", 299 | "mode": "none" 300 | }, 301 | "thresholdsStyle": { 302 | "mode": "off" 303 | } 304 | }, 305 | "mappings": [], 306 | "min": 0, 307 | "thresholds": { 308 | "mode": "absolute", 309 | "steps": [ 310 | { 311 | "color": "green", 312 | "value": null 313 | }, 314 | { 315 | "color": "red", 316 | "value": 80 317 | } 318 | ] 319 | } 320 | }, 321 | "overrides": [] 322 | }, 323 | "gridPos": { 324 | "h": 9, 325 | "w": 12, 326 | "x": 0, 327 | "y": 10 328 | }, 329 | "id": 12, 330 | "options": { 331 | "legend": { 332 | "calcs": [], 333 | "displayMode": "table", 334 | "placement": "right", 335 | "showLegend": true 336 | }, 337 | "tooltip": { 338 | "mode": "single", 339 | "sort": "none" 340 | } 341 | }, 342 | "pluginVersion": "9.4.7", 343 | "targets": [ 344 | { 345 | "datasource": { 346 | "type": "prometheus", 347 | "uid": "${DS_PROMETHEUS}" 348 | }, 349 | "editorMode": "code", 350 | "expr": "sum by (instance, function, state) (increase(salt_function_responses_total{function=~\"$function\", state=~\"$state\",success=\"true\"}[$__rate_interval]))", 351 | "legendFormat": "\"{{function}} {{state}}\"", 352 | "range": true, 353 | "refId": "A" 354 | } 355 | ], 356 | "title": "Response success", 357 | "type": "timeseries" 358 | }, 359 | { 360 | "datasource": { 361 | "type": "prometheus", 362 | "uid": "${DS_PROMETHEUS}" 363 | }, 364 | "description": "", 365 | "fieldConfig": { 366 | "defaults": { 367 | "color": { 368 | "mode": "palette-classic" 369 | }, 370 | "custom": { 371 | "axisCenteredZero": true, 372 | "axisColorMode": "text", 373 | "axisLabel": "", 374 | "axisPlacement": "auto", 375 | "barAlignment": 0, 376 | "drawStyle": "line", 377 | "fillOpacity": 100, 378 | "gradientMode": "hue", 379 | "hideFrom": { 380 | "legend": false, 381 | "tooltip": false, 382 | "viz": false 383 | }, 384 | "lineInterpolation": "stepBefore", 385 | "lineWidth": 1, 386 | "pointSize": 5, 387 | "scaleDistribution": { 388 | "type": "linear" 389 | }, 390 | "showPoints": "auto", 391 | "spanNulls": false, 392 | "stacking": { 393 | "group": "A", 394 | "mode": "none" 395 | }, 396 | "thresholdsStyle": { 397 | "mode": "off" 398 | } 399 | }, 400 | "mappings": [], 401 | "min": 0, 402 | "thresholds": { 403 | "mode": "absolute", 404 | "steps": [ 405 | { 406 | "color": "green", 407 | "value": null 408 | }, 409 | { 410 | "color": "red", 411 | "value": 80 412 | } 413 | ] 414 | } 415 | }, 416 | "overrides": [] 417 | }, 418 | "gridPos": { 419 | "h": 9, 420 | "w": 12, 421 | "x": 12, 422 | "y": 10 423 | }, 424 | "id": 13, 425 | "options": { 426 | "legend": { 427 | "calcs": [], 428 | "displayMode": "table", 429 | "placement": "right", 430 | "showLegend": true 431 | }, 432 | "tooltip": { 433 | "mode": "single", 434 | "sort": "none" 435 | } 436 | }, 437 | "pluginVersion": "9.4.7", 438 | "targets": [ 439 | { 440 | "datasource": { 441 | "type": "prometheus", 442 | "uid": "${DS_PROMETHEUS}" 443 | }, 444 | "editorMode": "code", 445 | "expr": "- sum by (instance, function, state) (increase(salt_function_responses_total{function=~\"$function\", state=~\"$state\", success=\"false\"}[$__rate_interval]))", 446 | "legendFormat": "\"{{function}} {{state}}\"", 447 | "range": true, 448 | "refId": "A" 449 | } 450 | ], 451 | "title": "Response success", 452 | "type": "timeseries" 453 | }, 454 | { 455 | "collapsed": false, 456 | "gridPos": { 457 | "h": 1, 458 | "w": 24, 459 | "x": 0, 460 | "y": 19 461 | }, 462 | "id": 11, 463 | "panels": [], 464 | "title": "Scheduled jobs", 465 | "type": "row" 466 | }, 467 | { 468 | "datasource": { 469 | "type": "prometheus", 470 | "uid": "${DS_PROMETHEUS}" 471 | }, 472 | "fieldConfig": { 473 | "defaults": { 474 | "color": { 475 | "mode": "palette-classic" 476 | }, 477 | "custom": { 478 | "axisCenteredZero": true, 479 | "axisColorMode": "text", 480 | "axisLabel": "", 481 | "axisPlacement": "auto", 482 | "barAlignment": 0, 483 | "drawStyle": "line", 484 | "fillOpacity": 100, 485 | "gradientMode": "hue", 486 | "hideFrom": { 487 | "legend": false, 488 | "tooltip": false, 489 | "viz": false 490 | }, 491 | "lineInterpolation": "stepBefore", 492 | "lineWidth": 1, 493 | "pointSize": 1, 494 | "scaleDistribution": { 495 | "type": "linear" 496 | }, 497 | "showPoints": "auto", 498 | "spanNulls": false, 499 | "stacking": { 500 | "group": "A", 501 | "mode": "none" 502 | }, 503 | "thresholdsStyle": { 504 | "mode": "off" 505 | } 506 | }, 507 | "mappings": [], 508 | "min": 0, 509 | "thresholds": { 510 | "mode": "absolute", 511 | "steps": [ 512 | { 513 | "color": "green", 514 | "value": null 515 | }, 516 | { 517 | "color": "red", 518 | "value": 80 519 | } 520 | ] 521 | } 522 | }, 523 | "overrides": [] 524 | }, 525 | "gridPos": { 526 | "h": 8, 527 | "w": 24, 528 | "x": 0, 529 | "y": 20 530 | }, 531 | "id": 7, 532 | "options": { 533 | "legend": { 534 | "calcs": [], 535 | "displayMode": "list", 536 | "placement": "bottom", 537 | "showLegend": true 538 | }, 539 | "tooltip": { 540 | "mode": "single", 541 | "sort": "none" 542 | } 543 | }, 544 | "targets": [ 545 | { 546 | "datasource": { 547 | "type": "prometheus", 548 | "uid": "${DS_PROMETHEUS}" 549 | }, 550 | "editorMode": "code", 551 | "exemplar": false, 552 | "expr": "sum by (function, state) (increase(salt_scheduled_job_return_total{function=~\"$function\", state=~\"$state\", success=\"true\"}[$__rate_interval]))", 553 | "instant": false, 554 | "legendFormat": "\"{{function}} {{state}}\"", 555 | "range": true, 556 | "refId": "A" 557 | }, 558 | { 559 | "datasource": { 560 | "type": "prometheus", 561 | "uid": "${DS_PROMETHEUS}" 562 | }, 563 | "editorMode": "code", 564 | "expr": "- sum by (function, state) (increase(salt_scheduled_job_return_total{function=~\"$function\", state=~\"$state\", success=\"false\"}[$__rate_interval]))", 565 | "hide": false, 566 | "legendFormat": "\"{{function}} {{state}}\"", 567 | "range": true, 568 | "refId": "B" 569 | } 570 | ], 571 | "title": "Scheduled job responses", 572 | "type": "timeseries" 573 | } 574 | ], 575 | "refresh": "", 576 | "revision": 1, 577 | "schemaVersion": 38, 578 | "style": "dark", 579 | "tags": [], 580 | "templating": { 581 | "list": [ 582 | { 583 | "allValue": ".*", 584 | "current": {}, 585 | "datasource": { 586 | "type": "prometheus", 587 | "uid": "${DS_PROMETHEUS}" 588 | }, 589 | "definition": "query_result(salt_scheduled_job_return_total or salt_new_job_total or salt_function_responses_total)", 590 | "hide": 0, 591 | "includeAll": true, 592 | "multi": false, 593 | "name": "function", 594 | "options": [], 595 | "query": { 596 | "query": "query_result(salt_scheduled_job_return_total or salt_new_job_total or salt_function_responses_total)", 597 | "refId": "StandardVariableQuery" 598 | }, 599 | "refresh": 1, 600 | "regex": "/.*function=\"([^\"]+)\".*/", 601 | "skipUrlSync": false, 602 | "sort": 1, 603 | "type": "query" 604 | }, 605 | { 606 | "allValue": ".*", 607 | "current": {}, 608 | "datasource": { 609 | "type": "prometheus", 610 | "uid": "${DS_PROMETHEUS}" 611 | }, 612 | "definition": "query_result(salt_scheduled_job_return_total{function=~\"$function\"} or salt_new_job_total{function=~\"$function\"} or salt_function_responses_total{function=~\"$function\"})", 613 | "hide": 0, 614 | "includeAll": true, 615 | "multi": false, 616 | "name": "state", 617 | "options": [], 618 | "query": { 619 | "query": "query_result(salt_scheduled_job_return_total{function=~\"$function\"} or salt_new_job_total{function=~\"$function\"} or salt_function_responses_total{function=~\"$function\"})", 620 | "refId": "StandardVariableQuery" 621 | }, 622 | "refresh": 1, 623 | "regex": "/.*state=\"([^\"]+)\".*/", 624 | "skipUrlSync": false, 625 | "sort": 1, 626 | "type": "query" 627 | } 628 | ] 629 | }, 630 | "time": { 631 | "from": "now-1h", 632 | "to": "now" 633 | }, 634 | "timepicker": {}, 635 | "timezone": "", 636 | "title": "Saltstack", 637 | "uid": "PBxX3EE4z", 638 | "version": 17, 639 | "weekStart": "" 640 | } --------------------------------------------------------------------------------