├── .gitignore ├── .github ├── env ├── dependabot.yml └── workflows │ ├── tests.yaml │ ├── checks.yaml │ └── publish.yaml ├── pkg ├── options │ ├── doc.go │ ├── options_test.go │ ├── options.go │ └── types.go ├── instr │ ├── doc.go │ └── metrics.go ├── auth │ ├── doc.go │ ├── token.go │ └── roundtripper.go ├── transport │ ├── doc.go │ ├── body.go │ ├── transport.go │ └── tls.go ├── logs │ ├── doc.go │ ├── types.go │ ├── write.go │ ├── read.go │ └── query.go ├── metrics │ ├── doc.go │ ├── query.go │ ├── read.go │ └── write.go └── api │ └── api.go ├── .bingo ├── go.mod ├── mdox.mod ├── jsonnet.mod ├── jsonnetfmt.mod ├── golangci-lint.mod ├── kubeval.mod ├── .gitignore ├── gojsontoyaml.mod ├── variables.env ├── README.md ├── thanos.mod └── Variables.mk ├── Dockerfile ├── examples └── manifests │ ├── up-service.yaml │ ├── up-job.yaml │ ├── up-deployment.yaml │ ├── up-job-with-logs.yaml │ └── up-job-with-get-token.yaml ├── test ├── config │ └── loki.yml └── integration.sh ├── .golangci.yml ├── go.mod ├── jsonnet ├── main.jsonnet ├── up.libsonnet └── job │ └── up.libsonnet ├── README.md ├── Makefile ├── LICENSE ├── go.sum └── cmd └── up └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | /up 2 | /.tags 3 | /vendor 4 | /tmp 5 | /.idea 6 | -------------------------------------------------------------------------------- /.github/env: -------------------------------------------------------------------------------- 1 | golang-version=1.24 2 | image-repository=quay.io/observatorium/up 3 | -------------------------------------------------------------------------------- /pkg/options/doc.go: -------------------------------------------------------------------------------- 1 | // Package options handles the command line flags. 2 | package options 3 | -------------------------------------------------------------------------------- /pkg/instr/doc.go: -------------------------------------------------------------------------------- 1 | // Package instr provides the instrumentation facility for metrics. 2 | package instr 3 | -------------------------------------------------------------------------------- /pkg/auth/doc.go: -------------------------------------------------------------------------------- 1 | // Package auth provides the token and roundtripper for bearer based authentication. 2 | package auth 3 | -------------------------------------------------------------------------------- /pkg/transport/doc.go: -------------------------------------------------------------------------------- 1 | // Package transport provides the transport layer security configuration for logs and metrics endpoints. 2 | package transport 3 | -------------------------------------------------------------------------------- /pkg/logs/doc.go: -------------------------------------------------------------------------------- 1 | // Package logs represents the reader and writer interface 2 | // to query and push logs to Loki for a set of labels. 3 | package logs 4 | -------------------------------------------------------------------------------- /.bingo/go.mod: -------------------------------------------------------------------------------- 1 | module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files. -------------------------------------------------------------------------------- /.bingo/mdox.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.24.10 4 | 5 | require github.com/bwplotka/mdox v0.9.0 6 | -------------------------------------------------------------------------------- /pkg/metrics/doc.go: -------------------------------------------------------------------------------- 1 | // Package metrics represents the reader and writer interface 2 | // to remote write to and query from Prometheus for a set of labels. 3 | package metrics 4 | -------------------------------------------------------------------------------- /.bingo/jsonnet.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/google/go-jsonnet v0.16.0 // cmd/jsonnet 6 | -------------------------------------------------------------------------------- /.bingo/jsonnetfmt.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/google/go-jsonnet v0.16.0 // cmd/jsonnetfmt 6 | -------------------------------------------------------------------------------- /.bingo/golangci-lint.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/golangci/golangci-lint v1.45.2 // cmd/golangci-lint 6 | -------------------------------------------------------------------------------- /.bingo/kubeval.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/instrumenta/kubeval v0.0.0-20201005082916-38668c6c5b23 6 | -------------------------------------------------------------------------------- /.bingo/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Ignore everything 3 | * 4 | 5 | # But not these files: 6 | !.gitignore 7 | !*.mod 8 | !*.sum 9 | !README.md 10 | !Variables.mk 11 | !variables.env 12 | 13 | *tmp.mod 14 | -------------------------------------------------------------------------------- /.bingo/gojsontoyaml.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.17 4 | 5 | require github.com/brancz/gojsontoyaml v0.0.0-20200602132005-3697ded27e8c 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: github-actions 8 | directory: / 9 | schedule: 10 | interval: weekly 11 | groups: 12 | patch-updates: 13 | update-types: 14 | - "patch" 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-alpine as builder 2 | 3 | RUN apk add ca-certificates --no-cache make && update-ca-certificates 4 | 5 | WORKDIR /workspace 6 | 7 | COPY . . 8 | 9 | RUN make build 10 | 11 | FROM scratch 12 | 13 | COPY --from=builder /workspace/up /usr/bin/up 14 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 15 | 16 | ENTRYPOINT ["/usr/bin/up"] 17 | -------------------------------------------------------------------------------- /pkg/logs/types.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | type queryResponse struct { 4 | Status string `json:"status"` 5 | Data struct { 6 | ResultType string `json:"resultType"` 7 | Result []stream `json:"result"` 8 | } `json:"data"` 9 | } 10 | 11 | // PushRequest represents the payload to push logs to Loki. 12 | type PushRequest struct { 13 | Streams []stream `json:"streams"` 14 | } 15 | 16 | type stream struct { 17 | Stream map[string]string `json:"stream"` 18 | Values [][]string `json:"values"` 19 | } 20 | -------------------------------------------------------------------------------- /pkg/transport/body.go: -------------------------------------------------------------------------------- 1 | package transport 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | 7 | "github.com/go-kit/log" 8 | "github.com/go-kit/log/level" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func ExhaustCloseWithLogOnErr(l log.Logger, rc io.ReadCloser) { 13 | if _, err := io.Copy(ioutil.Discard, rc); err != nil { 14 | level.Warn(l).Log("msg", "failed to exhaust reader, performance may be impeded", "err", err) 15 | } 16 | 17 | if err := rc.Close(); err != nil { 18 | level.Warn(l).Log("msg", "detected close error", "err", errors.Wrap(err, "response body close")) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/manifests/up-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: blackbox-prober 6 | app.kubernetes.io/instance: observatorium-up 7 | app.kubernetes.io/name: observatorium-up 8 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 9 | name: observatorium-up 10 | namespace: observatorium 11 | spec: 12 | ports: 13 | - name: http 14 | port: 8080 15 | targetPort: 8080 16 | selector: 17 | app.kubernetes.io/component: blackbox-prober 18 | app.kubernetes.io/instance: observatorium-up 19 | app.kubernetes.io/name: observatorium-up 20 | -------------------------------------------------------------------------------- /.bingo/variables.env: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. 4 | GOBIN=${GOBIN:=$(go env GOBIN)} 5 | 6 | if [ -z "$GOBIN" ]; then 7 | GOBIN="$(go env GOPATH)/bin" 8 | fi 9 | 10 | 11 | GOJSONTOYAML="${GOBIN}/gojsontoyaml-v0.0.0-20200602132005-3697ded27e8c" 12 | 13 | GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.45.2" 14 | 15 | JSONNET="${GOBIN}/jsonnet-v0.16.0" 16 | 17 | JSONNETFMT="${GOBIN}/jsonnetfmt-v0.16.0" 18 | 19 | KUBEVAL="${GOBIN}/kubeval-v0.0.0-20201005082916-38668c6c5b23" 20 | 21 | MDOX="${GOBIN}/mdox-v0.9.0" 22 | 23 | THANOS="${GOBIN}/thanos-v0.39.2" 24 | 25 | -------------------------------------------------------------------------------- /test/config/loki.yml: -------------------------------------------------------------------------------- 1 | auth_enabled: false 2 | 3 | server: 4 | http_listen_port: 3100 5 | 6 | querier: 7 | engine: 8 | max_look_back_period: 2h 9 | 10 | ingester: 11 | lifecycler: 12 | address: 127.0.0.1 13 | ring: 14 | kvstore: 15 | store: inmemory 16 | replication_factor: 1 17 | final_sleep: 0s 18 | chunk_idle_period: 5m 19 | chunk_retain_period: 1h 20 | 21 | schema_config: 22 | configs: 23 | - from: 1995-01-01 24 | store: boltdb 25 | object_store: filesystem 26 | schema: v11 27 | index: 28 | prefix: index_ 29 | period: 168h 30 | 31 | storage_config: 32 | boltdb: 33 | directory: /tmp/loki/index 34 | 35 | filesystem: 36 | directory: /tmp/loki/chunks 37 | 38 | limits_config: 39 | enforce_metric_name: false 40 | reject_old_samples: false 41 | -------------------------------------------------------------------------------- /pkg/auth/token.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | ) 7 | 8 | type TokenProvider interface { 9 | Get() (string, error) 10 | } 11 | 12 | func NewNoOpTokenProvider() *StaticToken { 13 | return &StaticToken{token: ""} 14 | } 15 | 16 | type StaticToken struct { 17 | token string 18 | } 19 | 20 | func NewStaticToken(token string) *StaticToken { 21 | return &StaticToken{token: token} 22 | } 23 | 24 | func (t *StaticToken) Get() (string, error) { 25 | return t.token, nil 26 | } 27 | 28 | type FileToken struct { 29 | file string 30 | } 31 | 32 | func NewFileToken(file string) *FileToken { 33 | return &FileToken{file: file} 34 | } 35 | 36 | func (t *FileToken) Get() (string, error) { 37 | b, err := ioutil.ReadFile(t.file) 38 | if err != nil { 39 | return "", err 40 | } 41 | 42 | return strings.TrimSpace(string(b)), nil 43 | } 44 | -------------------------------------------------------------------------------- /.bingo/README.md: -------------------------------------------------------------------------------- 1 | # Project Development Dependencies. 2 | 3 | This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. 4 | 5 | * Run `bingo get` to install all tools having each own module file in this directory. 6 | * Run `bingo get ` to install that have own module file in this directory. 7 | * For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. 8 | * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool. 9 | * For go: Import `.bingo/variables.go` to for variable names. 10 | * See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. 11 | 12 | ## Requirements 13 | 14 | * Go 1.14+ 15 | -------------------------------------------------------------------------------- /.bingo/thanos.mod: -------------------------------------------------------------------------------- 1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT 2 | 3 | go 1.24.10 4 | 5 | replace capnproto.org/go/capnp/v3 => capnproto.org/go/capnp/v3 v3.0.0-alpha.30 6 | 7 | replace github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab 8 | 9 | replace github.com/prometheus/prometheus => github.com/thanos-io/thanos-prometheus v0.0.0-20250610133519-082594458a88 10 | 11 | replace github.com/sercand/kuberesolver/v4 => github.com/sercand/kuberesolver/v5 v5.1.1 12 | 13 | replace github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e 14 | 15 | replace google.golang.org/grpc => google.golang.org/grpc v1.63.2 16 | 17 | replace gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497 18 | 19 | require github.com/thanos-io/thanos v0.39.2 // cmd/thanos 20 | -------------------------------------------------------------------------------- /pkg/transport/transport.go: -------------------------------------------------------------------------------- 1 | package transport 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/go-kit/log" 9 | "github.com/observatorium/up/pkg/options" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | func NewTLSTransport(l log.Logger, tls options.TLS) (*http.Transport, error) { 14 | tlsConfig, err := newTLSConfig(l, tls.Cert, tls.Key, tls.CACert) 15 | if err != nil { 16 | return nil, errors.Wrap(err, "tls config") 17 | } 18 | 19 | return &http.Transport{ 20 | Proxy: http.ProxyFromEnvironment, 21 | DialContext: (&net.Dialer{ 22 | Timeout: 30 * time.Second, 23 | KeepAlive: 30 * time.Second, 24 | DualStack: true, 25 | }).DialContext, 26 | ForceAttemptHTTP2: true, 27 | MaxIdleConns: 100, 28 | IdleConnTimeout: 90 * time.Second, 29 | TLSHandshakeTimeout: 10 * time.Second, 30 | ExpectContinueTimeout: 1 * time.Second, 31 | TLSClientConfig: tlsConfig, 32 | }, nil 33 | } 34 | -------------------------------------------------------------------------------- /pkg/auth/roundtripper.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/go-kit/log" 7 | ) 8 | 9 | type BearerTokenRoundTripper struct { 10 | l log.Logger 11 | r http.RoundTripper 12 | t TokenProvider 13 | TraceID string 14 | } 15 | 16 | func NewBearerTokenRoundTripper(l log.Logger, t TokenProvider, r http.RoundTripper) *BearerTokenRoundTripper { 17 | if r == nil { 18 | r = http.DefaultTransport 19 | } 20 | 21 | return &BearerTokenRoundTripper{ 22 | l: l, 23 | t: t, 24 | r: r, 25 | } 26 | } 27 | 28 | func (r *BearerTokenRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 29 | token, err := r.t.Get() 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | if token != "" { 35 | req.Header.Add("Authorization", "Bearer "+token) 36 | } 37 | 38 | resp, err := r.r.RoundTrip(req) 39 | if err != nil { 40 | return resp, err 41 | } 42 | 43 | r.TraceID = resp.Header.Get("X-Thanos-Trace-Id") 44 | 45 | return resp, err 46 | } 47 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - 'master' 7 | - 'main' 8 | tags: 9 | - 'v*' 10 | jobs: 11 | unit-tests: 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | os: 16 | - ubuntu-latest 17 | name: Unit tests 18 | steps: 19 | - uses: actions/checkout@v6 20 | - name: Import environment variables from file 21 | run: cat ".github/env" >> $GITHUB_ENV 22 | - uses: actions/setup-go@v6 23 | with: 24 | go-version: '${{ env.golang-version }}' 25 | - run: make test 26 | integration-tests: 27 | runs-on: ${{ matrix.os }} 28 | strategy: 29 | matrix: 30 | os: 31 | - ubuntu-latest 32 | name: Integration tests 33 | steps: 34 | - uses: actions/checkout@v6 35 | - name: Import environment variables from file 36 | run: cat ".github/env" >> $GITHUB_ENV 37 | - uses: actions/setup-go@v6 38 | with: 39 | go-version: '${{ env.golang-version }}' 40 | cache: true 41 | - run: make test-integration 42 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # default concurrency is a available CPU number 3 | concurrency: 4 4 | # timeout for analysis, e.g. 30s, 5m, default is 1m 5 | deadline: 3m 6 | tests: true 7 | 8 | # output configuration options 9 | output: 10 | # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" 11 | format: colored-line-number 12 | 13 | # print lines of code with issue, default is true 14 | print-issued-lines: true 15 | 16 | # print linter name in the end of issue text, default is true 17 | print-linter-name: true 18 | 19 | linters: 20 | enable: 21 | # Sorted alphabetically. 22 | - bodyclose 23 | - deadcode 24 | - depguard 25 | - errcheck 26 | - exportloopref 27 | - funlen 28 | - gocognit 29 | - goconst 30 | - godot 31 | - gofmt 32 | - goimports 33 | - gosimple 34 | - govet 35 | - ineffassign 36 | - lll 37 | - misspell 38 | - staticcheck 39 | - structcheck 40 | - typecheck 41 | - unparam 42 | - unused 43 | - varcheck 44 | 45 | linters-settings: 46 | errcheck: 47 | exclude-functions: 48 | - '(github.com/go-kit/log.Logger).Log' 49 | lll: 50 | line-length: 140 51 | funlen: 52 | lines: 120 53 | statements: 45 54 | gocognit: 55 | min-complexity: 40 56 | -------------------------------------------------------------------------------- /examples/manifests/up-job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: test 6 | app.kubernetes.io/instance: observatorium-up 7 | app.kubernetes.io/name: observatorium-up 8 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 9 | name: observatorium-up 10 | spec: 11 | backoffLimit: 5 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: test 16 | app.kubernetes.io/instance: observatorium-up 17 | app.kubernetes.io/name: observatorium-up 18 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 19 | spec: 20 | containers: 21 | - args: 22 | - --endpoint-type=metrics 23 | - --endpoint-write=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/receive 24 | - --endpoint-read=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/query 25 | - --period=1s 26 | - --duration=2m 27 | - --name=foo 28 | - --labels=bar="baz" 29 | - --latency=10s 30 | - --initial-query-delay=5s 31 | - --threshold=0.90 32 | image: quay.io/observatorium/up:master-2020-06-03-8a20b4e 33 | name: observatorium-up 34 | resources: {} 35 | volumeMounts: [] 36 | initContainers: [] 37 | restartPolicy: OnFailure 38 | volumes: [] 39 | -------------------------------------------------------------------------------- /pkg/options/options_test.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/efficientgo/tools/core/pkg/testutil" 8 | "github.com/prometheus/prometheus/prompb" 9 | ) 10 | 11 | func TestLabelArg_Sort(t *testing.T) { 12 | testCases := []struct { 13 | original labelArg 14 | expected labelArg 15 | }{ 16 | { 17 | // Test sorting with lower-case label names (__name__ should be first). 18 | labelArg{ 19 | prompb.Label{Name: "z", Value: "1"}, 20 | prompb.Label{Name: "a", Value: "1"}, 21 | prompb.Label{Name: "__name__", Value: "test"}, 22 | }, 23 | labelArg{ 24 | prompb.Label{Name: "__name__", Value: "test"}, 25 | prompb.Label{Name: "a", Value: "1"}, 26 | prompb.Label{Name: "z", Value: "1"}, 27 | }, 28 | }, 29 | { 30 | // Test sorting with upper-case label names (upper case should be first). 31 | labelArg{ 32 | prompb.Label{Name: "__name__", Value: "test"}, 33 | prompb.Label{Name: "A", Value: "1"}, 34 | prompb.Label{Name: "b", Value: "1"}, 35 | }, 36 | labelArg{ 37 | prompb.Label{Name: "A", Value: "1"}, 38 | prompb.Label{Name: "__name__", Value: "test"}, 39 | prompb.Label{Name: "b", Value: "1"}, 40 | }, 41 | }, 42 | } 43 | 44 | for i, tc := range testCases { 45 | t.Run(fmt.Sprintf("case #%d", i), func(t *testing.T) { 46 | tc.original.Sort() 47 | testutil.Equals(t, tc.expected, tc.original) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /examples/manifests/up-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: blackbox-prober 6 | app.kubernetes.io/instance: observatorium-up 7 | app.kubernetes.io/name: observatorium-up 8 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 9 | name: observatorium-up 10 | namespace: observatorium 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app.kubernetes.io/component: blackbox-prober 16 | app.kubernetes.io/instance: observatorium-up 17 | app.kubernetes.io/name: observatorium-up 18 | template: 19 | metadata: 20 | labels: 21 | app.kubernetes.io/component: blackbox-prober 22 | app.kubernetes.io/instance: observatorium-up 23 | app.kubernetes.io/name: observatorium-up 24 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 25 | spec: 26 | containers: 27 | - args: 28 | - --duration=0 29 | - --log.level=debug 30 | - --endpoint-type=metrics 31 | - --endpoint-read=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/query 32 | - --endpoint-write=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/receive 33 | image: quay.io/observatorium/up:master-2020-06-03-8a20b4e 34 | name: observatorium-up 35 | ports: 36 | - containerPort: 8080 37 | name: http 38 | resources: {} 39 | volumeMounts: [] 40 | volumes: [] 41 | -------------------------------------------------------------------------------- /.github/workflows/checks.yaml: -------------------------------------------------------------------------------- 1 | name: checks 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - 'master' 7 | - 'main' 8 | tags: 9 | - 'v*' 10 | jobs: 11 | lint: 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | os: 16 | - ubuntu-latest 17 | name: Lint 18 | steps: 19 | - uses: actions/checkout@v6 20 | - name: Import environment variables from file 21 | run: cat ".github/env" >> $GITHUB_ENV 22 | - uses: actions/setup-go@v6 23 | with: 24 | go-version: '${{ env.golang-version }}' 25 | - run: make tidy && git diff --exit-code 26 | - run: make --always-make tidy && git diff --exit-code 27 | generate: 28 | runs-on: ubuntu-latest 29 | name: Generate 30 | steps: 31 | - uses: actions/checkout@v6 32 | - name: Import environment variables from file 33 | run: cat ".github/env" >> $GITHUB_ENV 34 | - uses: actions/setup-go@v6 35 | with: 36 | go-version: '${{ env.golang-version }}' 37 | - run: make --always-make generate && git diff --exit-code 38 | build: 39 | runs-on: ${{ matrix.os }} 40 | strategy: 41 | matrix: 42 | os: 43 | - ubuntu-latest 44 | name: Build the binary 45 | steps: 46 | - uses: actions/checkout@v6 47 | - name: Import environment variables from file 48 | run: cat ".github/env" >> $GITHUB_ENV 49 | - uses: actions/setup-go@v6 50 | with: 51 | go-version: '${{ env.golang-version }}' 52 | - run: make build 53 | -------------------------------------------------------------------------------- /pkg/transport/tls.go: -------------------------------------------------------------------------------- 1 | package transport 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "io/ioutil" 7 | 8 | "github.com/go-kit/log" 9 | "github.com/go-kit/log/level" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | const HTTPS = "https" 14 | 15 | func newTLSConfig(logger log.Logger, certFile, keyFile, caCertFile string) (*tls.Config, error) { 16 | var certPool *x509.CertPool 17 | 18 | if caCertFile != "" { 19 | caPEM, err := ioutil.ReadFile(caCertFile) 20 | if err != nil { 21 | return nil, errors.Wrap(err, "reading client CA") 22 | } 23 | 24 | certPool = x509.NewCertPool() 25 | if !certPool.AppendCertsFromPEM(caPEM) { 26 | return nil, errors.Wrap(err, "building client CA") 27 | } 28 | 29 | level.Info(logger).Log("msg", "TLS client using provided certificate pool") 30 | } else { 31 | var err error 32 | certPool, err = x509.SystemCertPool() 33 | if err != nil { 34 | return nil, errors.Wrap(err, "reading system certificate pool") 35 | } 36 | 37 | level.Info(logger).Log("msg", "TLS client using system certificate pool") 38 | } 39 | 40 | tlsCfg := &tls.Config{RootCAs: certPool} 41 | 42 | if (keyFile != "") != (certFile != "") { 43 | return nil, errors.Errorf("both client key and certificate must be provided") 44 | } 45 | 46 | if certFile != "" { 47 | cert, err := tls.LoadX509KeyPair(certFile, keyFile) 48 | if err != nil { 49 | return nil, errors.Wrap(err, "client credentials") 50 | } 51 | 52 | tlsCfg.Certificates = []tls.Certificate{cert} 53 | 54 | level.Info(logger).Log("msg", "TLS client authentication enabled") 55 | } 56 | 57 | return tlsCfg, nil 58 | } 59 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/observatorium/up 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.10 6 | 7 | require ( 8 | github.com/efficientgo/tools/core v0.0.0-20230505153745-6b7392939a60 9 | github.com/go-kit/log v0.2.1 10 | github.com/gogo/protobuf v1.3.2 11 | github.com/golang/snappy v1.0.0 12 | github.com/oklog/run v1.2.0 13 | github.com/pkg/errors v0.9.1 14 | github.com/prometheus/client_golang v1.23.2 15 | github.com/prometheus/client_model v0.6.2 16 | github.com/prometheus/common v0.67.4 17 | github.com/prometheus/prometheus v0.308.0 18 | gopkg.in/yaml.v2 v2.4.0 19 | ) 20 | 21 | require ( 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 24 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 25 | github.com/dennwc/varint v1.0.0 // indirect 26 | github.com/go-logfmt/logfmt v0.6.0 // indirect 27 | github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect 28 | github.com/json-iterator/go v1.1.12 // indirect 29 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 30 | github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect 31 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 32 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 33 | github.com/prometheus/procfs v0.16.1 // indirect 34 | go.uber.org/atomic v1.11.0 // indirect 35 | go.uber.org/goleak v1.3.0 // indirect 36 | go.yaml.in/yaml/v2 v2.4.3 // indirect 37 | golang.org/x/sys v0.37.0 // indirect 38 | golang.org/x/text v0.30.0 // indirect 39 | google.golang.org/protobuf v1.36.10 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /jsonnet/main.jsonnet: -------------------------------------------------------------------------------- 1 | local commonConfig = { 2 | local cfg = self, 3 | namespace: 'observatorium', 4 | name: 'observatorium-up', 5 | version: 'master-2020-06-03-8a20b4e', 6 | image: 'quay.io/observatorium/up:' + cfg.version, 7 | replicas: 1, 8 | endpointType: 'metrics', 9 | writeEndpoint: 'http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/receive', 10 | readEndpoint: 'http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/query', 11 | }; 12 | 13 | local up = (import 'up.libsonnet')(commonConfig); 14 | local job = (import 'job/up.libsonnet')(commonConfig { backoffLimit: 5 }); 15 | local jobWithGetToken = (import 'job/up.libsonnet')(commonConfig { 16 | backoffLimit: 5, 17 | getToken: { 18 | image: 'docker.io/curlimages/curl', 19 | endpoint: 'http://FAKE.svc.cluster.local:%d/dex/token', 20 | username: 'admin@example.com', 21 | password: 'password', 22 | clientID: 'test', 23 | clientSecret: 'ZXhhbXBsZS1hcHAtc2VjcmV0', 24 | }, 25 | }); 26 | local jobWithLogs = (import 'job/up.libsonnet')(commonConfig { 27 | backoffLimit: 5, 28 | sendLogs: { 29 | // Note: Keep debian here because we need coreutils' date 30 | // for timestamp generation in nanoseconds. 31 | image: 'docker.io/debian', 32 | }, 33 | }); 34 | 35 | { ['up-' + name]: up[name] for name in std.objectFields(up) if up[name] != null } + 36 | { ['up-' + name]: job[name] for name in std.objectFields(job) if job[name] != null } + 37 | { ['up-%s-with-get-token' % name]: jobWithGetToken[name] for name in std.objectFields(jobWithGetToken) if jobWithGetToken[name] != null } + 38 | { ['up-%s-with-logs' % name]: jobWithLogs[name] for name in std.objectFields(jobWithLogs) if jobWithLogs[name] != null } 39 | -------------------------------------------------------------------------------- /pkg/metrics/query.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "time" 8 | 9 | "github.com/observatorium/up/pkg/auth" 10 | "github.com/observatorium/up/pkg/options" 11 | "github.com/observatorium/up/pkg/transport" 12 | 13 | "github.com/go-kit/log" 14 | "github.com/go-kit/log/level" 15 | "github.com/pkg/errors" 16 | promapi "github.com/prometheus/client_golang/api" 17 | promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 18 | ) 19 | 20 | // Query executes a query specification, a set of queries, against Prometheus. 21 | func Query( 22 | ctx context.Context, 23 | l log.Logger, 24 | endpoint *url.URL, 25 | t auth.TokenProvider, 26 | query options.Query, 27 | tls options.TLS, 28 | defaultStep time.Duration, 29 | ) (int, promapiv1.Warnings, error) { 30 | var ( 31 | warn promapiv1.Warnings 32 | err error 33 | rt *auth.BearerTokenRoundTripper 34 | ) 35 | 36 | level.Debug(l).Log("msg", "running specified query", "name", query.GetName(), "query", query.GetQuery()) 37 | 38 | // Copy URL to avoid modifying the passed value. 39 | u := new(url.URL) 40 | *u = *endpoint 41 | 42 | if u.Scheme == transport.HTTPS { 43 | tp, err := transport.NewTLSTransport(l, tls) 44 | if err != nil { 45 | return 0, warn, errors.Wrap(err, "create round tripper") 46 | } 47 | 48 | rt = auth.NewBearerTokenRoundTripper(l, t, tp) 49 | } else { 50 | rt = auth.NewBearerTokenRoundTripper(l, t, nil) 51 | } 52 | 53 | c, err := promapi.NewClient(promapi.Config{ 54 | Address: u.String(), 55 | RoundTripper: rt, 56 | }) 57 | if err != nil { 58 | err = fmt.Errorf("create new API client: %w", err) 59 | return 0, warn, err 60 | } 61 | 62 | return query.Run(ctx, c, l, rt.TraceID, defaultStep) 63 | } 64 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Container build & push jobs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - main 8 | tags: 9 | - "v[0-9]+.[0-9]+.[0-9]+*" 10 | 11 | jobs: 12 | build_push_container: 13 | runs-on: ubuntu-latest 14 | name: Build and push the container image for a commit or tag. 15 | steps: 16 | - name: Checkout code into the Go module directory. 17 | uses: actions/checkout@v6 18 | 19 | - name: Login to image registry 20 | uses: docker/login-action@v3 21 | with: 22 | registry: quay.io 23 | username: ${{ secrets.QUAY_USERNAME }} 24 | password: ${{ secrets.QUAY_PASSWORD }} 25 | 26 | - name: Set up QEMU 27 | uses: docker/setup-qemu-action@v3 28 | 29 | - name: Set up Docker Buildx 30 | id: buildx 31 | uses: docker/setup-buildx-action@v3 32 | 33 | - name: Cache for Docker's buildx 34 | uses: actions/cache@v5 35 | with: 36 | path: .buildxcache/ 37 | key: ${{ runner.os }}-buildx-${{ hashFiles('**/*.go', 'Dockerfile', 'go.sum') }} 38 | restore-keys: | 39 | ${{ runner.os }}-buildx- 40 | - name: Snapshot container buid & push 41 | run: make conditional-container-build-push 42 | 43 | - name: Check semver tag 44 | id: check-semver-tag 45 | # The regex below comes from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. 46 | run: | 47 | if [[ ${{ github.event.ref }} =~ ^refs/tags/v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ ]]; then 48 | echo ::set-output name=match::true 49 | fi 50 | - name: Release container build & push 51 | if: steps.check-semver-tag.outputs.match == 'true' 52 | run: make container-release-build-push 53 | -------------------------------------------------------------------------------- /examples/manifests/up-job-with-logs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: test 6 | app.kubernetes.io/instance: observatorium-up 7 | app.kubernetes.io/name: observatorium-up 8 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 9 | name: observatorium-up 10 | spec: 11 | backoffLimit: 5 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: test 16 | app.kubernetes.io/instance: observatorium-up 17 | app.kubernetes.io/name: observatorium-up 18 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 19 | spec: 20 | containers: 21 | - args: 22 | - --endpoint-type=metrics 23 | - --endpoint-write=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/receive 24 | - --endpoint-read=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/query 25 | - --period=1s 26 | - --duration=2m 27 | - --name=foo 28 | - --labels=bar="baz" 29 | - --latency=10s 30 | - --initial-query-delay=5s 31 | - --threshold=0.90 32 | - --logs-file=/var/logs-file/logs.yaml 33 | image: quay.io/observatorium/up:master-2020-06-03-8a20b4e 34 | name: observatorium-up 35 | resources: {} 36 | volumeMounts: 37 | - mountPath: /var/logs-file 38 | name: logs-file 39 | readOnly: true 40 | initContainers: 41 | - command: 42 | - /bin/sh 43 | - -c 44 | - | 45 | cat > /var/logs-file/logs.yaml << EOF 46 | spec: 47 | logs: [ [ "$(date '+%s%N')", "log line"] ] 48 | EOF 49 | image: docker.io/debian 50 | name: logs-file 51 | volumeMounts: 52 | - mountPath: /var/logs-file 53 | name: logs-file 54 | readOnly: false 55 | restartPolicy: OnFailure 56 | volumes: 57 | - emptyDir: {} 58 | name: logs-file 59 | -------------------------------------------------------------------------------- /test/integration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This test spins up one Thanos receiver for ingestion and one querier for querying. 4 | # The up binary is then run against them. 5 | 6 | set -euo pipefail 7 | 8 | result=1 9 | trap 'kill $(jobs -p); exit $result' EXIT 10 | 11 | ( 12 | ${THANOS} receive \ 13 | --grpc-address=127.0.0.1:10901 \ 14 | --http-address=127.0.0.1:10902 \ 15 | --remote-write.address=127.0.0.1:19291 \ 16 | --log.level=info \ 17 | --label=receive_replica=\"0\" \ 18 | --tsdb.path="$(mktemp -d)" 19 | ) & 20 | 21 | ( 22 | ${THANOS} query \ 23 | --grpc-address=127.0.0.1:10911 \ 24 | --http-address=127.0.0.1:9091 \ 25 | --store=127.0.0.1:10901 \ 26 | --log.level=info \ 27 | --query.replica-label=receive_replica 28 | ) & 29 | 30 | ( 31 | ./tmp/bin/loki \ 32 | -log.level=debug \ 33 | -target=all \ 34 | -config.file=./test/config/loki.yml 35 | ) & 36 | 37 | echo "## waiting for dependencies to come up..." 38 | sleep 10 39 | 40 | if ./up \ 41 | --listen=0.0.0.0:8888 \ 42 | --endpoint-type=metrics \ 43 | --endpoint-read=http://127.0.0.1:9091 \ 44 | --endpoint-write=http://127.0.0.1:19291/api/v1/receive \ 45 | --period=500ms \ 46 | --initial-query-delay=250ms \ 47 | --threshold=1 \ 48 | --latency=10s \ 49 | --duration=10s \ 50 | --log.level=debug \ 51 | --name=up_test \ 52 | --labels='foo="bar"'; then 53 | result=0 54 | echo "## metrics tests: ok" 55 | else 56 | result=1 57 | printf "## metrics tests: failed\n\n" 58 | exit 1 59 | fi 60 | 61 | if ./up \ 62 | --listen=0.0.0.0:8888 \ 63 | --endpoint-type=logs \ 64 | --endpoint-read=http://127.0.0.1:3100/loki/api/v1/query \ 65 | --endpoint-write=http://127.0.0.1:3100/loki/api/v1/push \ 66 | --period=500ms \ 67 | --initial-query-delay=250ms \ 68 | --threshold=1 \ 69 | --latency=10s \ 70 | --duration=10s \ 71 | --log.level=debug \ 72 | --name=up_test \ 73 | --labels='foo="bar"'\ 74 | --logs="[\"$(date '+%s%N')\",\"log line 1\"]"; then 75 | result=0 76 | echo "## logs tests: ok" 77 | else 78 | result=1 79 | printf "## logs tests: failed\n\n" 80 | exit 1 81 | fi 82 | 83 | printf "\t## all tests: ok\n\n" 1>&2 84 | exit 0 85 | -------------------------------------------------------------------------------- /pkg/logs/write.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "net/http" 8 | "net/url" 9 | 10 | "github.com/observatorium/up/pkg/auth" 11 | "github.com/observatorium/up/pkg/options" 12 | "github.com/observatorium/up/pkg/transport" 13 | 14 | "github.com/go-kit/log" 15 | "github.com/pkg/errors" 16 | "github.com/prometheus/prometheus/prompb" 17 | ) 18 | 19 | // Write executes a push against Loki sending a set of labels and log entries to store. 20 | func Write(ctx context.Context, endpoint *url.URL, t auth.TokenProvider, wreq *PushRequest, l log.Logger, tls options.TLS) (int, error) { 21 | var ( 22 | buf []byte 23 | err error 24 | req *http.Request 25 | res *http.Response 26 | rt http.RoundTripper 27 | ) 28 | 29 | if endpoint.Scheme == transport.HTTPS { 30 | rt, err = transport.NewTLSTransport(l, tls) 31 | if err != nil { 32 | return 0, errors.Wrap(err, "create round tripper") 33 | } 34 | 35 | rt = auth.NewBearerTokenRoundTripper(l, t, rt) 36 | } else { 37 | rt = auth.NewBearerTokenRoundTripper(l, t, nil) 38 | } 39 | 40 | client := &http.Client{Transport: rt} 41 | 42 | buf, err = json.Marshal(wreq) 43 | if err != nil { 44 | return 0, errors.Wrap(err, "marshalling payload") 45 | } 46 | 47 | req, err = http.NewRequest(http.MethodPost, endpoint.String(), bytes.NewBuffer(buf)) 48 | if err != nil { 49 | return 0, errors.Wrap(err, "creating request") 50 | } 51 | 52 | req.Header.Add("Content-Type", "application/json") 53 | 54 | res, err = client.Do(req.WithContext(ctx)) //nolint:bodyclose 55 | if err != nil { 56 | return 0, errors.Wrap(err, "making request") 57 | } 58 | 59 | defer transport.ExhaustCloseWithLogOnErr(l, res.Body) 60 | 61 | if res.StatusCode != http.StatusNoContent { 62 | err = errors.New(res.Status) 63 | return res.StatusCode, errors.Wrap(err, "non-204 status") 64 | } 65 | 66 | return res.StatusCode, nil 67 | } 68 | 69 | // Generate takes a set of labels and log lines and returns the payload to push logs to Loki. 70 | func Generate(labels []prompb.Label, values [][]string) *PushRequest { 71 | s := make(map[string]string) 72 | for _, label := range labels { 73 | s[label.Name] = label.Value 74 | } 75 | 76 | return &PushRequest{ 77 | Streams: []stream{ 78 | { 79 | Stream: s, 80 | Values: values, 81 | }, 82 | }, 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /examples/manifests/up-job-with-get-token.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: test 6 | app.kubernetes.io/instance: observatorium-up 7 | app.kubernetes.io/name: observatorium-up 8 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 9 | name: observatorium-up 10 | spec: 11 | backoffLimit: 5 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: test 16 | app.kubernetes.io/instance: observatorium-up 17 | app.kubernetes.io/name: observatorium-up 18 | app.kubernetes.io/version: master-2020-06-03-8a20b4e 19 | spec: 20 | containers: 21 | - args: 22 | - --endpoint-type=metrics 23 | - --endpoint-write=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/receive 24 | - --endpoint-read=http://FAKE.svc.cluster.local:8080/api/metrics/v1/test/api/v1/query 25 | - --period=1s 26 | - --duration=2m 27 | - --name=foo 28 | - --labels=bar="baz" 29 | - --latency=10s 30 | - --initial-query-delay=5s 31 | - --threshold=0.90 32 | - --token-file=/var/shared/token 33 | image: quay.io/observatorium/up:master-2020-06-03-8a20b4e 34 | name: observatorium-up 35 | resources: {} 36 | volumeMounts: 37 | - mountPath: /var/shared 38 | name: shared 39 | readOnly: true 40 | initContainers: 41 | - command: 42 | - /bin/sh 43 | - -c 44 | - | 45 | curl --request POST \ 46 | --silent \ 47 | \ 48 | --url http://FAKE.svc.cluster.local:%d/dex/token \ 49 | --header 'content-type: application/x-www-form-urlencoded' \ 50 | --data grant_type=password \ 51 | --data username=admin@example.com \ 52 | --data password=password \ 53 | --data client_id=test \ 54 | --data client_secret=ZXhhbXBsZS1hcHAtc2VjcmV0 \ 55 | --data scope="openid email" | sed 's/^{.*"id_token":[^"]*"\([^"]*\)".*}/\1/' > /var/shared/token 56 | image: docker.io/curlimages/curl 57 | name: curl 58 | volumeMounts: 59 | - mountPath: /var/shared 60 | name: shared 61 | readOnly: false 62 | restartPolicy: OnFailure 63 | volumes: 64 | - emptyDir: {} 65 | name: shared 66 | -------------------------------------------------------------------------------- /pkg/metrics/read.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | "time" 10 | 11 | "github.com/observatorium/up/pkg/api" 12 | "github.com/observatorium/up/pkg/auth" 13 | "github.com/observatorium/up/pkg/instr" 14 | "github.com/observatorium/up/pkg/options" 15 | "github.com/observatorium/up/pkg/transport" 16 | 17 | "github.com/go-kit/log" 18 | "github.com/pkg/errors" 19 | promapi "github.com/prometheus/client_golang/api" 20 | "github.com/prometheus/common/model" 21 | "github.com/prometheus/prometheus/prompb" 22 | ) 23 | 24 | // Read executes query against Prometheus with the same labels to retrieve the written metrics back. 25 | func Read( 26 | ctx context.Context, 27 | endpoint *url.URL, 28 | tp auth.TokenProvider, 29 | labels []prompb.Label, 30 | ago, latency time.Duration, 31 | m instr.Metrics, 32 | l log.Logger, 33 | tls options.TLS, 34 | ) (int, error) { 35 | var ( 36 | rt http.RoundTripper 37 | err error 38 | ) 39 | 40 | if endpoint.Scheme == transport.HTTPS { 41 | rt, err = transport.NewTLSTransport(l, tls) 42 | if err != nil { 43 | return 0, errors.Wrap(err, "create round tripper") 44 | } 45 | 46 | rt = auth.NewBearerTokenRoundTripper(l, tp, rt) 47 | } else { 48 | rt = auth.NewBearerTokenRoundTripper(l, tp, nil) 49 | } 50 | 51 | client, err := promapi.NewClient(promapi.Config{ 52 | Address: endpoint.String(), 53 | RoundTripper: rt, 54 | }) 55 | if err != nil { 56 | return 0, err 57 | } 58 | 59 | labelSelectors := make([]string, len(labels)) 60 | for i, label := range labels { 61 | labelSelectors[i] = fmt.Sprintf(`%s="%s"`, label.Name, label.Value) 62 | } 63 | 64 | query := fmt.Sprintf("{%s}", strings.Join(labelSelectors, ",")) 65 | ts := time.Now().Add(ago) 66 | 67 | value, httpCode, _, err := api.Query(ctx, client, query, ts, false) 68 | if err != nil { 69 | return httpCode, errors.Wrap(err, "query request failed") 70 | } 71 | 72 | vec := value.(model.Vector) 73 | if len(vec) != 1 { 74 | return httpCode, errors.Errorf("expected one metric, got %d", len(vec)) 75 | } 76 | 77 | t := time.Unix(int64(vec[0].Value/1000), 0) 78 | 79 | diffSeconds := time.Since(t).Seconds() 80 | 81 | m.MetricValueDifference.Observe(diffSeconds) 82 | 83 | if diffSeconds > latency.Seconds() { 84 | return httpCode, errors.Errorf("metric value is too old: %2.fs", diffSeconds) 85 | } 86 | 87 | return httpCode, nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/metrics/write.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "net/http" 7 | "net/url" 8 | "time" 9 | 10 | "github.com/observatorium/up/pkg/auth" 11 | "github.com/observatorium/up/pkg/options" 12 | "github.com/observatorium/up/pkg/transport" 13 | 14 | "github.com/go-kit/log" 15 | "github.com/gogo/protobuf/proto" 16 | "github.com/golang/snappy" 17 | "github.com/pkg/errors" 18 | "github.com/prometheus/prometheus/prompb" 19 | ) 20 | 21 | // Write executes a remote-write against Prometheus sending a set of labels and metrics to store. 22 | func Write(ctx context.Context, endpoint *url.URL, t auth.TokenProvider, wreq proto.Message, l log.Logger, tls options.TLS, 23 | tenantHeader string, tenant string) (int, error) { 24 | var ( 25 | buf []byte 26 | err error 27 | req *http.Request 28 | res *http.Response 29 | rt http.RoundTripper 30 | ) 31 | 32 | if endpoint.Scheme == transport.HTTPS { 33 | rt, err = transport.NewTLSTransport(l, tls) 34 | if err != nil { 35 | return 0, errors.Wrap(err, "create round tripper") 36 | } 37 | } else { 38 | rt = http.DefaultTransport 39 | } 40 | 41 | client := &http.Client{Transport: rt} 42 | 43 | buf, err = proto.Marshal(wreq) 44 | if err != nil { 45 | return 0, errors.Wrap(err, "marshalling proto") 46 | } 47 | 48 | req, err = http.NewRequest("POST", endpoint.String(), bytes.NewBuffer(snappy.Encode(nil, buf))) 49 | if err != nil { 50 | return 0, errors.Wrap(err, "creating request") 51 | } 52 | 53 | token, err := t.Get() 54 | if err != nil { 55 | return 0, errors.Wrap(err, "retrieving token") 56 | } 57 | 58 | if token != "" { 59 | req.Header.Add("Authorization", "Bearer "+token) 60 | } 61 | 62 | if tenant != "" { 63 | req.Header.Add(tenantHeader, tenant) 64 | } 65 | 66 | res, err = client.Do(req.WithContext(ctx)) //nolint:bodyclose 67 | if err != nil { 68 | return 0, errors.Wrap(err, "making request") 69 | } 70 | 71 | defer transport.ExhaustCloseWithLogOnErr(l, res.Body) 72 | 73 | if res.StatusCode != http.StatusOK { 74 | err = errors.New(res.Status) 75 | return res.StatusCode, errors.Wrap(err, "non-200 status") 76 | } 77 | 78 | return res.StatusCode, nil 79 | } 80 | 81 | // Generate takes a set of labels and metrics key-value pairs and returns the payload to write metrics to Prometheus. 82 | func Generate(labels []prompb.Label) *prompb.WriteRequest { 83 | timestamp := time.Now().UnixNano() / int64(time.Millisecond) 84 | 85 | return &prompb.WriteRequest{ 86 | Timeseries: []prompb.TimeSeries{ 87 | { 88 | Labels: labels, 89 | Samples: []prompb.Sample{ 90 | { 91 | Value: float64(timestamp), 92 | Timestamp: timestamp, 93 | }, 94 | }, 95 | }, 96 | }, 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /pkg/logs/read.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "time" 12 | 13 | "github.com/observatorium/up/pkg/auth" 14 | "github.com/observatorium/up/pkg/instr" 15 | "github.com/observatorium/up/pkg/options" 16 | "github.com/observatorium/up/pkg/transport" 17 | 18 | "github.com/go-kit/log" 19 | "github.com/pkg/errors" 20 | "github.com/prometheus/prometheus/prompb" 21 | ) 22 | 23 | // Read executes query against Loki with the same labels to retrieve the written logs back. 24 | func Read( 25 | ctx context.Context, 26 | endpoint *url.URL, 27 | tp auth.TokenProvider, 28 | labels []prompb.Label, // change to Loki ProtoBufs 29 | ago, latency time.Duration, 30 | m instr.Metrics, 31 | l log.Logger, 32 | tls options.TLS, 33 | ) (int, error) { 34 | var ( 35 | rt http.RoundTripper 36 | err error 37 | ) 38 | 39 | if endpoint.Scheme == transport.HTTPS { 40 | rt, err = transport.NewTLSTransport(l, tls) 41 | if err != nil { 42 | return 0, errors.Wrap(err, "create round tripper") 43 | } 44 | 45 | rt = auth.NewBearerTokenRoundTripper(l, tp, rt) 46 | } else { 47 | rt = auth.NewBearerTokenRoundTripper(l, tp, nil) 48 | } 49 | 50 | client := &http.Client{Transport: rt} 51 | 52 | labelSelectors := make([]string, len(labels)) 53 | for i, label := range labels { 54 | labelSelectors[i] = fmt.Sprintf(`%s="%s"`, label.Name, label.Value) 55 | } 56 | 57 | query := fmt.Sprintf("{%s}", strings.Join(labelSelectors, ",")) 58 | 59 | params := url.Values{} 60 | params.Add("query", query) 61 | endpoint.RawQuery = params.Encode() 62 | 63 | req, err := http.NewRequest(http.MethodGet, endpoint.String(), nil) 64 | if err != nil { 65 | return 0, errors.Wrap(err, "creating request") 66 | } 67 | 68 | res, err := client.Do(req.WithContext(ctx)) 69 | if err != nil { 70 | if res == nil { 71 | //Unknown error. 72 | return 0, errors.Wrap(err, "making request") 73 | } 74 | 75 | return res.StatusCode, errors.Wrap(err, "making request") 76 | } 77 | 78 | if res.StatusCode != http.StatusOK { 79 | err = errors.New(res.Status) 80 | return res.StatusCode, errors.Wrap(err, "non-200 status") 81 | } 82 | 83 | defer res.Body.Close() 84 | 85 | body, err := ioutil.ReadAll(res.Body) 86 | if err != nil { 87 | return res.StatusCode, errors.Wrap(err, "reading response body") 88 | } 89 | 90 | rr := &queryResponse{} 91 | 92 | err = json.Unmarshal(body, rr) 93 | if err != nil { 94 | return res.StatusCode, errors.Wrap(err, "unmarshalling response") 95 | } 96 | 97 | rl := len(rr.Data.Result) 98 | if rl != 1 { 99 | return res.StatusCode, errors.Errorf("expected one log entry, got %d", rl) 100 | } 101 | 102 | return res.StatusCode, nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/instr/metrics.go: -------------------------------------------------------------------------------- 1 | package instr 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | type Metrics struct { 9 | RemoteWriteRequests *prometheus.CounterVec 10 | RemoteWriteRequestDuration prometheus.Histogram 11 | QueryResponses *prometheus.CounterVec 12 | QueryResponseDuration prometheus.Histogram 13 | MetricValueDifference prometheus.Histogram 14 | CustomQueryExecuted *prometheus.CounterVec 15 | CustomQueryErrors *prometheus.CounterVec 16 | CustomQueryRequestDuration *prometheus.HistogramVec 17 | CustomQueryLastDuration *prometheus.GaugeVec 18 | } 19 | 20 | func RegisterMetrics(reg *prometheus.Registry) Metrics { 21 | m := Metrics{ 22 | RemoteWriteRequests: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ 23 | Name: "up_remote_writes_total", 24 | Help: "Total number of remote write requests.", 25 | }, []string{"result", "http_code"}), 26 | RemoteWriteRequestDuration: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ 27 | Name: "up_remote_writes_duration_seconds", 28 | Help: "Duration of remote write requests.", 29 | }), 30 | QueryResponses: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ 31 | Name: "up_queries_total", 32 | Help: "The total number of queries made.", 33 | }, []string{"result", "http_code"}), 34 | QueryResponseDuration: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ 35 | Name: "up_queries_duration_seconds", 36 | Help: "Duration of up queries.", 37 | }), 38 | MetricValueDifference: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ 39 | Name: "up_metric_value_difference", 40 | Help: "The time difference between the current timestamp and the timestamp in the metrics value.", 41 | Buckets: prometheus.LinearBuckets(4, 0.25, 16), 42 | }), 43 | CustomQueryExecuted: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ 44 | Name: "up_custom_query_executed_total", 45 | Help: "The total number of custom specified queries executed.", 46 | }, []string{"type", "query", "http_code"}), 47 | CustomQueryRequestDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ 48 | Name: "up_custom_query_duration_seconds", 49 | Help: "Duration of custom specified queries", 50 | // We deliberately chose quite large buckets as we want to be able to accurately measure heavy queries. 51 | Buckets: []float64{0.1, 0.25, 0.5, 1, 5, 10, 20, 30, 45, 60, 100, 120}, 52 | }, []string{"type", "query", "http_code"}), 53 | CustomQueryErrors: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ 54 | Name: "up_custom_query_errors_total", 55 | Help: "The total number of custom specified queries executed.", 56 | }, []string{"type", "query", "http_code"}), 57 | CustomQueryLastDuration: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ 58 | Name: "up_custom_query_last_duration", 59 | Help: "The duration of the query execution last time the query was executed successfully.", 60 | }, []string{"type", "query", "http_code"}), 61 | } 62 | 63 | return m 64 | } 65 | -------------------------------------------------------------------------------- /pkg/logs/query.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/observatorium/up/pkg/auth" 12 | "github.com/observatorium/up/pkg/options" 13 | "github.com/observatorium/up/pkg/transport" 14 | 15 | "github.com/go-kit/log" 16 | "github.com/go-kit/log/level" 17 | "github.com/pkg/errors" 18 | 19 | promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 20 | ) 21 | 22 | func Query( 23 | ctx context.Context, 24 | l log.Logger, 25 | endpoint *url.URL, 26 | t auth.TokenProvider, 27 | q options.Query, 28 | tls options.TLS, 29 | defaultStep time.Duration, 30 | ) (int, promapiv1.Warnings, error) { 31 | // TODO: avoid type casting when we need to support all query endpoints for logs. 32 | query, ok := q.(*options.QuerySpec) 33 | if !ok { 34 | return 0, nil, errors.New("Incorrect query type for logs queries") 35 | } 36 | 37 | level.Debug(l).Log("msg", "running specified query", "name", query.Name, "query", query.Query) 38 | 39 | var ( 40 | rt http.RoundTripper 41 | warn promapiv1.Warnings 42 | err error 43 | ) 44 | 45 | if endpoint.Scheme == transport.HTTPS { 46 | rt, err = transport.NewTLSTransport(l, tls) 47 | if err != nil { 48 | return 0, warn, errors.Wrap(err, "create round tripper") 49 | } 50 | 51 | rt = auth.NewBearerTokenRoundTripper(l, t, rt) 52 | } else { 53 | rt = auth.NewBearerTokenRoundTripper(l, t, nil) 54 | } 55 | 56 | client := &http.Client{Transport: rt} 57 | 58 | params := url.Values{} 59 | params.Add("query", query.Query) 60 | 61 | if query.Duration > 0 { 62 | step := defaultStep 63 | if query.Step > 0 { 64 | step = query.Step 65 | } 66 | 67 | params.Add("start", time.Now().Add(-time.Duration(query.Duration)).String()) 68 | params.Add("end", time.Now().String()) 69 | params.Add("step", step.String()) 70 | } 71 | 72 | endpoint.RawQuery = params.Encode() 73 | 74 | req, err := http.NewRequest(http.MethodGet, endpoint.String(), nil) 75 | if err != nil { 76 | return 0, warn, errors.Wrap(err, "creating request") 77 | } 78 | 79 | res, err := client.Do(req.WithContext(ctx)) 80 | if err != nil { 81 | if res == nil { 82 | return 0, warn, errors.Wrap(err, "making request") 83 | } 84 | 85 | return res.StatusCode, warn, errors.Wrap(err, "making request") 86 | } 87 | 88 | if res.StatusCode != http.StatusOK { 89 | err = errors.New(res.Status) 90 | return res.StatusCode, warn, errors.Wrap(err, "non-200 status") 91 | } 92 | 93 | defer res.Body.Close() 94 | 95 | body, err := ioutil.ReadAll(res.Body) 96 | if err != nil { 97 | return res.StatusCode, warn, errors.Wrap(err, "reading response body") 98 | } 99 | 100 | rr := &queryResponse{} 101 | 102 | err = json.Unmarshal(body, rr) 103 | if err != nil { 104 | return res.StatusCode, warn, errors.Wrap(err, "unmarshalling response") 105 | } 106 | 107 | if len(rr.Data.Result) == 0 { 108 | return res.StatusCode, warn, errors.Errorf("expected at min one log entry, got none") 109 | } 110 | 111 | return res.StatusCode, warn, nil 112 | } 113 | -------------------------------------------------------------------------------- /pkg/options/options.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "net/url" 5 | "sort" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/go-kit/log/level" 11 | "github.com/observatorium/up/pkg/auth" 12 | "github.com/pkg/errors" 13 | "github.com/prometheus/common/model" 14 | "github.com/prometheus/prometheus/prompb" 15 | ) 16 | 17 | type TLS struct { 18 | Cert string 19 | Key string 20 | CACert string 21 | } 22 | 23 | type Options struct { 24 | LogLevel level.Option 25 | EndpointType EndpointType 26 | WriteEndpoint *url.URL 27 | ReadEndpoint *url.URL 28 | Labels labelArg 29 | Logs logs 30 | Listen string 31 | Name string 32 | Token auth.TokenProvider 33 | Queries []Query 34 | Period time.Duration 35 | Duration time.Duration 36 | Latency time.Duration 37 | InitialQueryDelay time.Duration 38 | SuccessThreshold float64 39 | TLS TLS 40 | DefaultStep time.Duration 41 | Tenant string 42 | TenantHeader string 43 | } 44 | 45 | type EndpointType string 46 | 47 | const ( 48 | LogsEndpointType EndpointType = "logs" 49 | MetricsEndpointType EndpointType = "metrics" 50 | ) 51 | 52 | type LogsSpec struct { 53 | Logs logs `yaml:"logs"` 54 | } 55 | 56 | type labelArg []prompb.Label 57 | 58 | func (la *labelArg) String() string { 59 | ls := make([]string, len(*la)) 60 | for i, l := range *la { 61 | ls[i] = l.Name + "=" + l.Value 62 | } 63 | 64 | return strings.Join(ls, ", ") 65 | } 66 | 67 | func (la *labelArg) Set(v string) error { 68 | labels := strings.Split(v, ",") 69 | lset := make([]prompb.Label, len(labels)) 70 | 71 | for i, l := range labels { 72 | parts := strings.SplitN(l, "=", 2) 73 | if len(parts) != 2 { 74 | return errors.Errorf("unrecognized label %q", l) 75 | } 76 | 77 | if !model.LabelName.IsValid(model.LabelName(parts[0])) { 78 | return errors.Errorf("unsupported format for label %s", l) 79 | } 80 | 81 | val, err := strconv.Unquote(parts[1]) 82 | if err != nil { 83 | return errors.Wrap(err, "unquote label value") 84 | } 85 | 86 | lset[i] = prompb.Label{Name: parts[0], Value: val} 87 | } 88 | 89 | *la = lset 90 | 91 | return nil 92 | } 93 | 94 | // Sort ensures all labels are ordered, in line with how upstream Prometheus code guarantees 95 | // ordering. See https://github.com/prometheus/prometheus/pull/5372. 96 | func (la *labelArg) Sort() { 97 | sort.Sort(la) 98 | } 99 | 100 | func (la *labelArg) Len() int { return len(*la) } 101 | func (la *labelArg) Swap(i, j int) { (*la)[i], (*la)[j] = (*la)[j], (*la)[i] } 102 | func (la *labelArg) Less(i, j int) bool { return (*la)[i].Name < (*la)[j].Name } 103 | 104 | type logs [][]string 105 | 106 | func (va *logs) String() string { 107 | s := make([]string, len(*va)) 108 | 109 | for i, l := range *va { 110 | s[i] = strings.Join(l, ",") 111 | } 112 | 113 | return strings.Join(s, ",") 114 | } 115 | 116 | func (va *logs) Set(v string) error { 117 | vas := strings.Split(v, "],[") 118 | vset := make(logs, len(vas)) 119 | 120 | for i, v := range vas { 121 | v = strings.TrimLeft(v, "[") 122 | v = strings.TrimRight(v, "]") 123 | vs := strings.Split(v, ",") 124 | 125 | for i, s := range vs { 126 | vs[i] = strings.Trim(s, `"`) 127 | } 128 | 129 | vset[i] = vs 130 | } 131 | 132 | *va = vset 133 | 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /.bingo/Variables.mk: -------------------------------------------------------------------------------- 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. 2 | # All tools are designed to be build inside $GOBIN. 3 | BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST))) 4 | GOPATH ?= $(shell go env GOPATH) 5 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin 6 | GO ?= $(shell which go) 7 | 8 | # Below generated variables ensure that every time a tool under each variable is invoked, the correct version 9 | # will be used; reinstalling only if needed. 10 | # For example for gojsontoyaml variable: 11 | # 12 | # In your main Makefile (for non array binaries): 13 | # 14 | #include .bingo/Variables.mk # Assuming -dir was set to .bingo . 15 | # 16 | #command: $(GOJSONTOYAML) 17 | # @echo "Running gojsontoyaml" 18 | # @$(GOJSONTOYAML) 19 | # 20 | GOJSONTOYAML := $(GOBIN)/gojsontoyaml-v0.0.0-20200602132005-3697ded27e8c 21 | $(GOJSONTOYAML): $(BINGO_DIR)/gojsontoyaml.mod 22 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 23 | @echo "(re)installing $(GOBIN)/gojsontoyaml-v0.0.0-20200602132005-3697ded27e8c" 24 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=gojsontoyaml.mod -o=$(GOBIN)/gojsontoyaml-v0.0.0-20200602132005-3697ded27e8c "github.com/brancz/gojsontoyaml" 25 | 26 | GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.45.2 27 | $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod 28 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 29 | @echo "(re)installing $(GOBIN)/golangci-lint-v1.45.2" 30 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.45.2 "github.com/golangci/golangci-lint/cmd/golangci-lint" 31 | 32 | JSONNET := $(GOBIN)/jsonnet-v0.16.0 33 | $(JSONNET): $(BINGO_DIR)/jsonnet.mod 34 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 35 | @echo "(re)installing $(GOBIN)/jsonnet-v0.16.0" 36 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=jsonnet.mod -o=$(GOBIN)/jsonnet-v0.16.0 "github.com/google/go-jsonnet/cmd/jsonnet" 37 | 38 | JSONNETFMT := $(GOBIN)/jsonnetfmt-v0.16.0 39 | $(JSONNETFMT): $(BINGO_DIR)/jsonnetfmt.mod 40 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 41 | @echo "(re)installing $(GOBIN)/jsonnetfmt-v0.16.0" 42 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=jsonnetfmt.mod -o=$(GOBIN)/jsonnetfmt-v0.16.0 "github.com/google/go-jsonnet/cmd/jsonnetfmt" 43 | 44 | KUBEVAL := $(GOBIN)/kubeval-v0.0.0-20201005082916-38668c6c5b23 45 | $(KUBEVAL): $(BINGO_DIR)/kubeval.mod 46 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 47 | @echo "(re)installing $(GOBIN)/kubeval-v0.0.0-20201005082916-38668c6c5b23" 48 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=kubeval.mod -o=$(GOBIN)/kubeval-v0.0.0-20201005082916-38668c6c5b23 "github.com/instrumenta/kubeval" 49 | 50 | MDOX := $(GOBIN)/mdox-v0.9.0 51 | $(MDOX): $(BINGO_DIR)/mdox.mod 52 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 53 | @echo "(re)installing $(GOBIN)/mdox-v0.9.0" 54 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox" 55 | 56 | THANOS := $(GOBIN)/thanos-v0.39.2 57 | $(THANOS): $(BINGO_DIR)/thanos.mod 58 | @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. 59 | @echo "(re)installing $(GOBIN)/thanos-v0.39.2" 60 | @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=thanos.mod -o=$(GOBIN)/thanos-v0.39.2 "github.com/thanos-io/thanos/cmd/thanos" 61 | 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # UP 2 | 3 | UP is a simple client for testing Prometheus remote-write and Loki write requests. 4 | 5 | For a specified metric the client writes the metric at a chosen interval, where the value of the metric is always the current timestamp in milliseconds. 6 | 7 | It can also read the metric back from a specified endpoint and compare its value against the current time to determine the total write-read latency. 8 | 9 | For a specified log entry the client writes the log entry at a chosen interval for the value given. 10 | 11 | It can also read the log back from a specified endpoint and compare the number of results. 12 | 13 | When the given duration is greater than 0s, UP will evaluate number of errors and will exit with a non-zero code if the ratio is greater than the specified threshold. 14 | 15 | [![Build Status](https://github.com/observatorium/up/actions/workflows/checks.yaml/badge.svg?branch=master)](https://github.com/observatorium/up/actions/workflows/checks.yaml) 16 | 17 | ## Getting Started 18 | 19 | The easiest way to begin making remote write requests is to run the UP container. For example, to report an `up` metric every 10 seconds, run: 20 | 21 | ```shell 22 | docker run --rm -p 8080:8080 quay.io/observatorium/up --endpoint-write=https://example.com/api/v1/receive --period=10s 23 | ``` 24 | 25 | Note that the metric name and labels are customizable. For example, to report a metric named `foo` with a custom `bar` label, run: 26 | 27 | ```shell 28 | docker run --rm -p 8080:8080 quay.io/observatorium/up --endpoint-write=https://example.com/api/v1/receive --period=10s --name foo --labels 'bar="baz"' 29 | ``` 30 | 31 | ## Usage 32 | 33 | ```txt mdox-exec="./up --help" mdox-expect-exit-code=2 34 | Usage of ./up: 35 | -duration duration 36 | The duration of the up command to run until it stops. If 0 it will not stop until the process is terminated. (default 5m0s) 37 | -endpoint-read string 38 | The endpoint to which to make query requests. 39 | -endpoint-type string 40 | The endpoint type. Options: 'logs', 'metrics'. (default "metrics") 41 | -endpoint-write string 42 | The endpoint to which to make remote-write requests. 43 | -initial-query-delay duration 44 | The time to wait before executing the first query. (default 10s) 45 | -labels value 46 | The labels in addition to '__name__' that should be applied to remote-write requests. 47 | -latency duration 48 | The maximum allowable latency between writing and reading. (default 15s) 49 | -listen string 50 | The address on which internal server runs. (default ":8080") 51 | -log.level string 52 | The log filtering level. Options: 'error', 'warn', 'info', 'debug'. (default "info") 53 | -logs value 54 | The logs that should be sent to remote-write requests. 55 | -logs-file string 56 | A file containing logs to send against the logs write endpoint. 57 | -name string 58 | The name of the metric to send in remote-write requests. (default "up") 59 | -period duration 60 | The time to wait between remote-write requests. (default 5s) 61 | -queries-file string 62 | A file containing queries to run against the read endpoint. 63 | -step duration 64 | Default step duration for range queries. Can be overridden if step is set in query spec. (default 5m0s) 65 | -tenant string 66 | Tenant ID to used to determine tenant for write requests. 67 | -tenant-header string 68 | Name of HTTP header used to determine tenant for write requests. (default "tenant_id") 69 | -threshold float 70 | The percentage of successful requests needed to succeed overall. 0 - 1. (default 0.9) 71 | -tls-ca-file string 72 | File containing the TLS CA to use against servers for verification. If no CA is specified, there won't be any verification. 73 | -tls-client-cert-file string 74 | File containing the default x509 Certificate for HTTPS. Leave blank to disable TLS. 75 | -tls-client-private-key-file string 76 | File containing the default x509 private key matching --tls-cert-file. Leave blank to disable TLS. 77 | -token string 78 | The bearer token to set in the authorization header on requests. Takes predence over --token-file if set. 79 | -token-file string 80 | The file from which to read a bearer token to set in the authorization header on requests. 81 | ``` 82 | -------------------------------------------------------------------------------- /jsonnet/up.libsonnet: -------------------------------------------------------------------------------- 1 | // These are the defaults for this components configuration. 2 | // When calling the function to generate the component's manifest, 3 | // you can pass an object structured like the default to overwrite default values. 4 | local defaults = { 5 | local defaults = self, 6 | name: error 'must provide name', 7 | namespace: error 'must provide namespace', 8 | version: error 'must provide version', 9 | image: error 'must provide image', 10 | endpointType: error 'must provide endpoint type', 11 | replicas: error 'must provide replicas', 12 | queryConfig: {}, 13 | readEndpoint: '', 14 | writeEndpoint: '', 15 | logs: '', 16 | ports: { http: 8080 }, 17 | resources: {}, 18 | serviceMonitor: false, 19 | 20 | commonLabels:: { 21 | 'app.kubernetes.io/name': 'observatorium-up', 22 | 'app.kubernetes.io/instance': defaults.name, 23 | 'app.kubernetes.io/version': defaults.version, 24 | 'app.kubernetes.io/component': 'blackbox-prober', 25 | }, 26 | 27 | podLabelSelector:: { 28 | [labelName]: defaults.commonLabels[labelName] 29 | for labelName in std.objectFields(defaults.commonLabels) 30 | if !std.setMember(labelName, ['app.kubernetes.io/version']) 31 | }, 32 | }; 33 | 34 | function(params) { 35 | local up = self, 36 | 37 | // Combine the defaults and the passed params to make the component's config. 38 | config:: defaults + params, 39 | // Safety checks for combined config of defaults and params 40 | assert std.isNumber(up.config.replicas) && up.config.replicas >= 0 : 'observatorium up replicas has to be number >= 0', 41 | assert std.isObject(up.config.resources), 42 | assert std.isObject(up.config.queryConfig), 43 | assert std.isBoolean(up.config.serviceMonitor), 44 | 45 | service: { 46 | apiVersion: 'v1', 47 | kind: 'Service', 48 | metadata: { 49 | name: up.config.name, 50 | namespace: up.config.namespace, 51 | labels: up.config.commonLabels, 52 | }, 53 | spec: { 54 | ports: [ 55 | { 56 | assert std.isString(name), 57 | assert std.isNumber(up.config.ports[name]), 58 | 59 | name: name, 60 | port: up.config.ports[name], 61 | targetPort: up.config.ports[name], 62 | } 63 | for name in std.objectFields(up.config.ports) 64 | ], 65 | selector: up.config.podLabelSelector, 66 | }, 67 | }, 68 | 69 | deployment: 70 | local c = { 71 | name: 'observatorium-up', 72 | image: up.config.image, 73 | args: [ 74 | '--duration=0', 75 | '--log.level=debug', 76 | '--endpoint-type=' + up.config.endpointType, 77 | ] + 78 | (if up.config.queryConfig != {} then ['--queries-file=/etc/up/queries.yaml'] else []) + 79 | (if up.config.readEndpoint != '' then ['--endpoint-read=' + up.config.readEndpoint] else []) + 80 | (if up.config.writeEndpoint != '' then ['--endpoint-write=' + up.config.writeEndpoint] else []) + 81 | (if up.config.logs != '' then ['--logs=' + up.config.logs] else []), 82 | ports: [ 83 | { name: port.name, containerPort: port.port } 84 | for port in up.service.spec.ports 85 | ], 86 | volumeMounts: if up.config.queryConfig != {} then [ 87 | { mountPath: '/etc/up/', name: 'query-config', readOnly: false }, 88 | ] else [], 89 | resources: if up.config.resources != {} then up.config.resources else {}, 90 | }; 91 | 92 | { 93 | apiVersion: 'apps/v1', 94 | kind: 'Deployment', 95 | metadata: { 96 | name: up.config.name, 97 | namespace: up.config.namespace, 98 | labels: up.config.commonLabels, 99 | }, 100 | spec: { 101 | replicas: up.config.replicas, 102 | selector: { matchLabels: up.config.podLabelSelector }, 103 | template: { 104 | metadata: { 105 | labels: up.config.commonLabels, 106 | }, 107 | spec: { 108 | containers: [c], 109 | volumes: if up.config.queryConfig != {} then 110 | [{ configMap: { name: up.config.name }, name: 'query-config' }] 111 | else [], 112 | }, 113 | }, 114 | }, 115 | }, 116 | 117 | configmap: if up.config.queryConfig != {} then { 118 | apiVersion: 'v1', 119 | data: { 120 | 'queries.yaml': std.manifestYamlDoc(up.config.queryConfig), 121 | }, 122 | kind: 'ConfigMap', 123 | metadata: { 124 | labels: up.config.commonLabels, 125 | name: up.config.name, 126 | namespace: up.config.namespace, 127 | }, 128 | } else null, 129 | 130 | serviceMonitor: if up.config.serviceMonitor == true then { 131 | apiVersion: 'monitoring.coreos.com/v1', 132 | kind: 'ServiceMonitor', 133 | metadata+: { 134 | name: up.config.name, 135 | namespace: up.config.namespace, 136 | }, 137 | spec: { 138 | selector: { 139 | matchLabels: up.config.podLabelSelector, 140 | }, 141 | endpoints: [ 142 | { port: 'http' }, 143 | ], 144 | }, 145 | } else null, 146 | } 147 | -------------------------------------------------------------------------------- /pkg/options/types.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/go-kit/log" 10 | "github.com/go-kit/log/level" 11 | "github.com/observatorium/up/pkg/api" 12 | promapi "github.com/prometheus/client_golang/api" 13 | promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 14 | "github.com/prometheus/common/model" 15 | ) 16 | 17 | const ( 18 | // Labels for query types. 19 | labelQuery = "query" 20 | labelQueryRange = "query_range" 21 | labelSeries = "series" 22 | labelNames = "label_names" 23 | labelValues = "label_values" 24 | ) 25 | 26 | // Query represents different types of queries. 27 | type Query interface { 28 | // GetName gets the name of the query. 29 | GetName() string 30 | // GetType gets the query type. 31 | GetType() string 32 | // GetQuery gets the query statement (promql) or label/matchers of the query. 33 | GetQuery() string 34 | // Run executes the query. 35 | Run(ctx context.Context, c promapi.Client, logger log.Logger, traceID string, 36 | defaultStep time.Duration) (int, promapiv1.Warnings, error) 37 | } 38 | 39 | type QuerySpec struct { 40 | Name string `yaml:"name"` 41 | Query string `yaml:"query"` 42 | Duration model.Duration `yaml:"duration,omitempty"` 43 | Step time.Duration `yaml:"step,omitempty"` 44 | Cache bool `yaml:"cache,omitempty"` 45 | } 46 | 47 | func (q QuerySpec) GetName() string { 48 | return q.Name 49 | } 50 | 51 | func (q QuerySpec) GetType() string { 52 | if q.Duration > 0 { 53 | return labelQueryRange 54 | } 55 | 56 | return labelQuery 57 | } 58 | 59 | func (q QuerySpec) GetQuery() string { return q.Query } 60 | 61 | func (q QuerySpec) Run(ctx context.Context, c promapi.Client, logger log.Logger, traceID string, 62 | defaultStep time.Duration) (int, promapiv1.Warnings, error) { 63 | var ( 64 | warn promapiv1.Warnings 65 | err error 66 | ) 67 | 68 | if q.Duration > 0 { 69 | step := defaultStep 70 | if q.Step > 0 { 71 | step = q.Step 72 | } 73 | 74 | _, httpCode, warn, err := api.QueryRange(ctx, c, q.Query, promapiv1.Range{ 75 | Start: time.Now().Add(-time.Duration(q.Duration)), 76 | End: time.Now(), 77 | Step: step, 78 | }, q.Cache) 79 | if err != nil { 80 | err = fmt.Errorf("querying: %w", err) 81 | return httpCode, warn, err 82 | } 83 | 84 | // Don't log response in range query case because there are a lot. 85 | level.Debug(logger).Log("msg", "request finished", "name", q.Name, "trace-id", traceID) 86 | 87 | return httpCode, warn, err 88 | } 89 | 90 | _, httpCode, warn, err := api.Query(ctx, c, q.Query, time.Now(), q.Cache) 91 | if err != nil { 92 | err = fmt.Errorf("querying: %w", err) 93 | return httpCode, warn, err 94 | } 95 | 96 | level.Debug(logger).Log("msg", "request finished", "name", q.Name, "response code ", httpCode, "trace-id", traceID) 97 | 98 | return httpCode, warn, err 99 | } 100 | 101 | type LabelSpec struct { 102 | Name string `yaml:"name"` 103 | Label string `yaml:"label"` 104 | Duration model.Duration `yaml:"duration"` 105 | Cache bool `yaml:"cache"` 106 | } 107 | 108 | func (q LabelSpec) GetName() string { return q.Name } 109 | 110 | func (q LabelSpec) GetType() string { 111 | if len(q.Label) > 0 { 112 | return labelValues 113 | } 114 | 115 | return labelNames 116 | } 117 | 118 | func (q LabelSpec) GetQuery() string { return q.Label } 119 | 120 | func (q LabelSpec) Run(ctx context.Context, c promapi.Client, logger log.Logger, traceID string, 121 | _ time.Duration) (int, promapiv1.Warnings, error) { 122 | var ( 123 | warn promapiv1.Warnings 124 | err error 125 | httpCode int 126 | ) 127 | 128 | if len(q.Label) > 0 { 129 | _, httpCode, warn, err = api.LabelValues(ctx, c, q.Label, time.Now().Add(-time.Duration(q.Duration)), time.Now(), q.Cache) 130 | } else { 131 | _, httpCode, warn, err = api.LabelNames(ctx, c, time.Now().Add(-time.Duration(q.Duration)), time.Now(), q.Cache) 132 | } 133 | 134 | if err != nil { 135 | err = fmt.Errorf("querying: %w", err) 136 | return httpCode, warn, err 137 | } 138 | 139 | // Don't log responses because there are a lot. 140 | level.Debug(logger).Log("msg", "request finished", "name", q.Name, "trace-id", traceID) 141 | 142 | return httpCode, warn, err 143 | } 144 | 145 | type SeriesSpec struct { 146 | Name string `yaml:"name"` 147 | Matchers []string `yaml:"matchers"` 148 | Duration model.Duration `yaml:"duration"` 149 | Cache bool `yaml:"cache"` 150 | } 151 | 152 | func (q SeriesSpec) GetName() string { return q.Name } 153 | 154 | func (q SeriesSpec) GetType() string { return labelSeries } 155 | 156 | func (q SeriesSpec) GetQuery() string { return strings.Join(q.Matchers, ", ") } 157 | 158 | func (q SeriesSpec) Run(ctx context.Context, c promapi.Client, logger log.Logger, traceID string, 159 | _ time.Duration) (int, promapiv1.Warnings, error) { 160 | _, httpCode, warn, err := api.Series(ctx, c, q.Matchers, time.Now().Add(-time.Duration(q.Duration)), time.Now(), q.Cache) 161 | if err != nil { 162 | err = fmt.Errorf("querying: %w", err) 163 | return httpCode, warn, err 164 | } 165 | 166 | // Don't log responses because there are a lot. 167 | level.Debug(logger).Log("msg", "request finished", "name", q.Name, "trace-id", traceID) 168 | 169 | return httpCode, warn, err 170 | } 171 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include .bingo/Variables.mk 2 | 3 | BIN_DIR ?= ./tmp/bin 4 | LOKI ?= $(BIN_DIR)/loki 5 | LOKI_VERSION ?= 1.5.0 6 | 7 | OS ?= $(shell uname -s | tr '[A-Z]' '[a-z]') 8 | ARCH ?= $(shell uname -m) 9 | GOARCH ?= $(shell go env GOARCH) 10 | VERSION := $(strip $(shell [ -d .git ] && git describe --always --tags --dirty)) 11 | BUILD_DATE := $(shell date -u +"%Y-%m-%d") 12 | BUILD_TIMESTAMP := $(shell date -u +"%Y-%m-%dT%H:%M:%S%Z") 13 | VCS_BRANCH := $(strip $(shell git rev-parse --abbrev-ref HEAD)) 14 | VCS_REF := $(strip $(shell [ -d .git ] && git rev-parse --short HEAD)) 15 | DOCKER_REPO ?= quay.io/observatorium/up 16 | 17 | EXAMPLES := examples 18 | MANIFESTS := ${EXAMPLES}/manifests 19 | 20 | all: build generate validate 21 | 22 | build: up README.md 23 | 24 | .PHONY: up 25 | up: 26 | CGO_ENABLED=0 go build -v -ldflags '-w -extldflags '-static'' ./cmd/up 27 | 28 | .PHONY: generate 29 | generate: jsonnet-fmt ${MANIFESTS} README.md 30 | 31 | .PHONY: validate 32 | validate: $(KUBEVAL) $(MANIFESTS) 33 | $(KUBEVAL) --ignore-missing-schemas $(MANIFESTS)/*.yaml 34 | 35 | .PHONY: tidy 36 | go mod tidy -v 37 | 38 | .PHONY: go-fmt 39 | go-fmt: 40 | @fmt_res=$$(gofmt -d -s $$(find . -type f -name '*.go' -not -path './jsonnet/vendor/*')); if [ -n "$$fmt_res" ]; then printf '\nGofmt found style issues. Please check the reported issues\nand fix them if necessary before submitting the code for review:\n\n%s' "$$fmt_res"; exit 1; fi 41 | 42 | .PHONY: lint 43 | lint: $(GOLANGCI_LINT) 44 | $(GOLANGCI_LINT) run -v -c .golangci.yml 45 | 46 | .PHONY: container-dev 47 | container-dev: 48 | @docker build \ 49 | --build-arg BUILD_DATE="$(BUILD_TIMESTAMP)" \ 50 | --build-arg VERSION="$(VERSION)" \ 51 | --build-arg VCS_REF="$(VCS_REF)" \ 52 | --build-arg VCS_BRANCH="$(VCS_BRANCH)" \ 53 | --build-arg DOCKERFILE_PATH="/Dockerfile" \ 54 | -t $(DOCKER_REPO):$(VCS_BRANCH)-$(BUILD_DATE)-$(VERSION) \ 55 | . 56 | docker tag $(DOCKER_REPO):$(VCS_BRANCH)-$(BUILD_DATE)-$(VERSION) $(DOCKER_REPO):latest 57 | 58 | .PHONY: clean 59 | clean: 60 | -rm tmp/help.txt 61 | -rm ./up 62 | 63 | .PHONY: README.md 64 | README.md: $(MDOX) up 65 | $(MDOX) fmt $(@) 66 | 67 | .PHONY: test 68 | test: 69 | CGO_ENABLED=1 go test -v -race ./... 70 | 71 | .PHONY: test-integration 72 | test-integration: build test/integration.sh | $(LOKI) $(THANOS) 73 | PATH=$$PATH:$$(pwd)/$(BIN_DIR) THANOS=$(THANOS) ./test/integration.sh 74 | 75 | .PHONY: ${MANIFESTS} 76 | ${MANIFESTS}: jsonnet/main.jsonnet jsonnet/*.libsonnet $(JSONNET) $(GOJSONTOYAML) 77 | @rm -rf ${MANIFESTS} 78 | @mkdir -p ${MANIFESTS} 79 | $(JSONNET) -J jsonnet/vendor -m ${MANIFESTS} jsonnet/main.jsonnet | xargs -I{} sh -c 'cat {} | $(GOJSONTOYAML) > {}.yaml && rm -f {}' -- {} 80 | 81 | JSONNET_SRC = $(shell find . -name 'vendor' -prune -o -name 'examples/vendor' -prune -o -name 'tmp' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print) 82 | JSONNETFMT_CMD := $(JSONNETFMT) -n 2 --max-blank-lines 2 --string-style s --comment-style s 83 | 84 | .PHONY: jsonnet-fmt 85 | jsonnet-fmt: | $(JSONNETFMT) 86 | echo ${JSONNET_SRC} | xargs -n 1 -- $(JSONNETFMT_CMD) -i 87 | 88 | .PHONY: format 89 | format: $(GOLANGCI_LINT) go-fmt jsonnet-fmt 90 | $(GOLANGCI_LINT) run --fix -c .golangci.yml 91 | 92 | $(BIN_DIR): 93 | mkdir -p $(BIN_DIR) 94 | 95 | $(LOKI): $(BIN_DIR) 96 | loki_pkg="loki-$$(go env GOOS)-$$(go env GOARCH)" && \ 97 | cd $(BIN_DIR) && curl -O -L "https://github.com/grafana/loki/releases/download/v$(LOKI_VERSION)/$$loki_pkg.zip" && \ 98 | unzip $$loki_pkg.zip && \ 99 | mv $$loki_pkg loki && \ 100 | rm $$loki_pkg.zip 101 | 102 | .PHONY: container-build 103 | container-build: 104 | git update-index --refresh 105 | docker buildx build \ 106 | --platform linux/amd64,linux/arm64 \ 107 | --cache-to type=local,dest=./.buildxcache/ \ 108 | --build-arg BUILD_DATE="$(BUILD_TIMESTAMP)" \ 109 | --build-arg VERSION="$(VERSION)" \ 110 | --build-arg VCS_REF="$(VCS_REF)" \ 111 | --build-arg VCS_BRANCH="$(VCS_BRANCH)" \ 112 | --build-arg DOCKERFILE_PATH="/Dockerfile" \ 113 | -t $(DOCKER_REPO):$(VCS_BRANCH)-$(BUILD_DATE)-$(VERSION) \ 114 | -t $(DOCKER_REPO):latest \ 115 | . 116 | 117 | .PHONY: container-build-push 118 | container-build-push: 119 | git update-index --refresh 120 | @docker buildx build \ 121 | --push \ 122 | --platform linux/amd64,linux/arm64 \ 123 | --cache-to type=local,dest=./.buildxcache/ \ 124 | --build-arg BUILD_DATE="$(BUILD_TIMESTAMP)" \ 125 | --build-arg VERSION="$(VERSION)" \ 126 | --build-arg VCS_REF="$(VCS_REF)" \ 127 | --build-arg VCS_BRANCH="$(VCS_BRANCH)" \ 128 | --build-arg DOCKERFILE_PATH="/Dockerfile" \ 129 | -t $(DOCKER_REPO):$(VCS_BRANCH)-$(BUILD_DATE)-$(VERSION) \ 130 | -t $(DOCKER_REPO):latest \ 131 | . 132 | .PHONY: conditional-container-build-push 133 | conditional-container-build-push: 134 | build/conditional-container-push.sh $(DOCKER_REPO):$(VCS_BRANCH)-$(BUILD_DATE)-$(VERSION) 135 | .PHONY: container-release-build-push 136 | container-release-build-push: VERSION_TAG = $(strip $(shell [ -d .git ] && git tag --points-at HEAD)) 137 | container-release-build-push: container-build-push 138 | # https://git-scm.com/docs/git-tag#Documentation/git-tag.txt---points-atltobjectgt 139 | @docker buildx build \ 140 | --push \ 141 | --platform linux/amd64,linux/arm64 \ 142 | --cache-from type=local,src=./.buildxcache/ \ 143 | --build-arg BUILD_DATE="$(BUILD_TIMESTAMP)" \ 144 | --build-arg VERSION="$(VERSION)" \ 145 | --build-arg VCS_REF="$(VCS_REF)" \ 146 | --build-arg VCS_BRANCH="$(VCS_BRANCH)" \ 147 | --build-arg DOCKERFILE_PATH="/Dockerfile" \ 148 | -t $(DOCKER_REPO):$(VERSION_TAG) \ 149 | -t $(DOCKER_REPO):latest \ 150 | . 151 | -------------------------------------------------------------------------------- /jsonnet/job/up.libsonnet: -------------------------------------------------------------------------------- 1 | // These are the defaults for this components configuration. 2 | // When calling the function to generate the component's manifest, 3 | // you can pass an object structured like the default to overwrite default values. 4 | local defaults = { 5 | local defaults = self, 6 | name: error 'must provide name', 7 | namespace: error 'must provide namespace', 8 | version: error 'must provide version', 9 | image: error 'must provide image', 10 | endpointType: error 'must provide endpoint type', 11 | writeEndpoint: error 'must provide writeEndpoint', 12 | readEndpoint: error 'must provide readEndpoint', 13 | backoffLimit: error 'must provide backoffLimit', 14 | replicas: 1, 15 | tls: {}, 16 | resources: {}, 17 | getToken: {}, 18 | sendLogs: {}, 19 | 20 | commonLabels:: { 21 | 'app.kubernetes.io/name': 'observatorium-up', 22 | 'app.kubernetes.io/instance': defaults.name, 23 | 'app.kubernetes.io/version': defaults.version, 24 | 'app.kubernetes.io/component': 'test', 25 | }, 26 | }; 27 | 28 | function(params) { 29 | local up = self, 30 | 31 | // Combine the defaults and the passed params to make the component's config. 32 | config:: defaults + params, 33 | // Safety checks for combined config of defaults and params 34 | assert std.isNumber(up.config.replicas) && up.config.replicas >= 0 : 'observatorium up job replicas has to be number >= 0', 35 | assert std.isObject(up.config.resources), 36 | assert std.isObject(up.config.tls), 37 | 38 | job: 39 | local bash = { 40 | name: 'logs-file', 41 | image: up.config.sendLogs.image, 42 | command: [ 43 | '/bin/sh', 44 | '-c', 45 | ||| 46 | cat > /var/logs-file/logs.yaml << EOF 47 | spec: 48 | logs: [ [ "$(date '+%s%N')", "log line"] ] 49 | EOF 50 | |||, 51 | ], 52 | volumeMounts: [ 53 | { name: 'logs-file', mountPath: '/var/logs-file', readOnly: false }, 54 | ], 55 | }; 56 | local curl = { 57 | name: 'curl', 58 | image: up.config.getToken.image, 59 | command: [ 60 | '/bin/sh', 61 | '-c', 62 | ||| 63 | curl --request POST \ 64 | --silent \ 65 | %s \ 66 | --url %s \ 67 | --header 'content-type: application/x-www-form-urlencoded' \ 68 | --data grant_type=password \ 69 | --data username=%s \ 70 | --data password=%s \ 71 | --data client_id=%s \ 72 | --data client_secret=%s \ 73 | --data scope="openid email" | sed 's/^{.*"id_token":[^"]*"\([^"]*\)".*}/\1/' > /var/shared/token 74 | ||| % [ 75 | (if std.objectHas(up.config.getToken, 'oidc') then '--cacert /mnt/oidc-tls/%s' % [up.config.getToken.oidc.caKey] else ''), 76 | up.config.getToken.endpoint, 77 | up.config.getToken.username, 78 | up.config.getToken.password, 79 | up.config.getToken.clientID, 80 | up.config.getToken.clientSecret, 81 | ], 82 | ], 83 | volumeMounts: 84 | [{ name: 'shared', mountPath: '/var/shared', readOnly: false }] + 85 | (if std.objectHas(up.config.getToken, 'oidc') then [{ name: 'oidc-tls', mountPath: '/mnt/oidc-tls', readOnly: true }] else []), 86 | }; 87 | local c = { 88 | name: 'observatorium-up', 89 | image: up.config.image, 90 | args: [ 91 | '--endpoint-type=' + up.config.endpointType, 92 | '--endpoint-write=' + up.config.writeEndpoint, 93 | '--endpoint-read=' + up.config.readEndpoint, 94 | '--period=1s', 95 | '--duration=2m', 96 | '--name=foo', 97 | '--labels=bar="baz"', 98 | '--latency=10s', 99 | '--initial-query-delay=5s', 100 | '--threshold=0.90', 101 | ] + 102 | (if up.config.tls != {} then ['--tls-ca-file=/mnt/tls/' + up.config.tls.caKey] else []) + 103 | (if up.config.getToken != {} then ['--token-file=/var/shared/token'] else []) + 104 | (if up.config.sendLogs != {} then ['--logs-file=/var/logs-file/logs.yaml'] else []), 105 | volumeMounts: 106 | (if up.config.tls != {} then [{ name: 'tls', mountPath: '/mnt/tls', readOnly: true }] else []) + 107 | (if up.config.getToken != {} then [{ name: 'shared', mountPath: '/var/shared', readOnly: true }] else []) + 108 | (if up.config.sendLogs != {} then [{ name: 'logs-file', mountPath: '/var/logs-file', readOnly: true }] else []), 109 | resources: if up.config.resources != {} then up.config.resources else {}, 110 | }; 111 | { 112 | apiVersion: 'batch/v1', 113 | kind: 'Job', 114 | metadata: { 115 | name: up.config.name, 116 | labels: up.config.commonLabels, 117 | }, 118 | spec: { 119 | backoffLimit: up.config.backoffLimit, 120 | template: { 121 | metadata: { 122 | labels: up.config.commonLabels, 123 | }, 124 | spec: { 125 | initContainers+: 126 | (if up.config.getToken != {} then [curl] else []) + 127 | (if up.config.sendLogs != {} then [bash] else []), 128 | containers: [c], 129 | restartPolicy: 'OnFailure', 130 | volumes: 131 | (if up.config.tls != {} then [{ configMap: { name: up.config.tls.configMapName }, name: 'tls' }] else []) + 132 | (if up.config.getToken != {} then [{ emptyDir: {}, name: 'shared' }] else []) + 133 | (if up.config.sendLogs != {} then [{ emptyDir: {}, name: 'logs-file' }] else []) + 134 | (if std.objectHas(up.config.getToken, 'oidc') then [{ configMap: { name: up.config.getToken.oidc.configMapName }, name: 'oidc-tls' }] else []), 135 | }, 136 | }, 137 | }, 138 | }, 139 | } 140 | -------------------------------------------------------------------------------- /pkg/api/api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // This is a modified copy from https://github.com/prometheus/client_golang/blob/master/api/prometheus/v1/api.go. 15 | 16 | package api 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "net/http" 23 | "net/url" 24 | "strconv" 25 | "strings" 26 | "time" 27 | 28 | promapi "github.com/prometheus/client_golang/api" 29 | promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 30 | "github.com/prometheus/common/model" 31 | ) 32 | 33 | const ( 34 | statusAPIError = 422 35 | 36 | // Possible values for ErrorType. 37 | ErrBadResponse promapiv1.ErrorType = "bad_response" 38 | ErrServer promapiv1.ErrorType = "server_error" 39 | ErrClient promapiv1.ErrorType = "client_error" 40 | 41 | epQuery = "/api/v1/query" 42 | epQueryRange = "/api/v1/query_range" 43 | epSeries = "/api/v1/series" 44 | epLabels = "/api/v1/labels" 45 | epLabelValues = "/api/v1/label/:name/values" 46 | ) 47 | 48 | func errorTypeAndMsgFor(resp *http.Response) (promapiv1.ErrorType, string) { 49 | switch resp.StatusCode / 100 { 50 | case 4: 51 | return ErrClient, fmt.Sprintf("client error: %d", resp.StatusCode) 52 | case 5: 53 | return ErrServer, fmt.Sprintf("server error: %d", resp.StatusCode) 54 | } 55 | 56 | return ErrBadResponse, fmt.Sprintf("bad response code %d", resp.StatusCode) 57 | } 58 | 59 | func apiError(code int) bool { 60 | // These are the codes that Prometheus sends when it returns an error. 61 | return code == statusAPIError || code == http.StatusBadRequest 62 | } 63 | 64 | // queryResult contains result data for a query. 65 | type queryResult struct { 66 | Type model.ValueType `json:"resultType"` 67 | Result interface{} `json:"result"` 68 | 69 | // The decoded value. 70 | v model.Value 71 | } 72 | 73 | func (qr *queryResult) UnmarshalJSON(b []byte) error { 74 | v := struct { 75 | Type model.ValueType `json:"resultType"` 76 | Result json.RawMessage `json:"result"` 77 | }{} 78 | 79 | err := json.Unmarshal(b, &v) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | switch v.Type { 85 | case model.ValScalar: 86 | var sv model.Scalar 87 | err = json.Unmarshal(v.Result, &sv) 88 | qr.v = &sv 89 | 90 | case model.ValVector: 91 | var vv model.Vector 92 | err = json.Unmarshal(v.Result, &vv) 93 | qr.v = vv 94 | 95 | case model.ValMatrix: 96 | var mv model.Matrix 97 | err = json.Unmarshal(v.Result, &mv) 98 | qr.v = mv 99 | 100 | default: 101 | err = fmt.Errorf("unexpected value type %q", v.Type) 102 | } 103 | 104 | return err 105 | } 106 | 107 | type apiResponse struct { 108 | Status string `json:"status"` 109 | Data json.RawMessage `json:"data"` 110 | ErrorType promapiv1.ErrorType `json:"errorType"` 111 | Error string `json:"error"` 112 | Warnings []string `json:"warnings,omitempty"` 113 | } 114 | 115 | func do(ctx context.Context, client promapi.Client, req *http.Request) (*http.Response, []byte, promapiv1.Warnings, error) { 116 | resp, body, err := client.Do(ctx, req) 117 | if err != nil { 118 | return resp, body, nil, err 119 | } 120 | 121 | code := resp.StatusCode 122 | 123 | if code/100 != 2 && !apiError(code) { 124 | errorType, errorMsg := errorTypeAndMsgFor(resp) 125 | 126 | return resp, body, nil, &promapiv1.Error{ 127 | Type: errorType, 128 | Msg: errorMsg, 129 | Detail: string(body), 130 | } 131 | } 132 | 133 | var result apiResponse 134 | 135 | if http.StatusNoContent != code { 136 | if jsonErr := json.Unmarshal(body, &result); jsonErr != nil { 137 | return resp, body, nil, &promapiv1.Error{ 138 | Type: ErrBadResponse, 139 | Msg: jsonErr.Error(), 140 | } 141 | } 142 | } 143 | 144 | if apiError(code) && result.Status == "success" { 145 | err = &promapiv1.Error{ 146 | Type: ErrBadResponse, 147 | Msg: "inconsistent body for response code", 148 | } 149 | } 150 | 151 | if result.Status == "error" { 152 | err = &promapiv1.Error{ 153 | Type: result.ErrorType, 154 | Msg: result.Error, 155 | } 156 | } 157 | 158 | return resp, result.Data, result.Warnings, err 159 | } 160 | 161 | // doGetFallback will attempt to do the request as-is, and on a 405 it will fallback to a GET request. 162 | func doGetFallback( 163 | ctx context.Context, 164 | client promapi.Client, 165 | u *url.URL, 166 | args url.Values, 167 | cache bool, 168 | ) (*http.Response, []byte, promapiv1.Warnings, error) { //nolint:unparam 169 | req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(args.Encode())) 170 | if err != nil { 171 | return nil, nil, nil, err 172 | } 173 | 174 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 175 | 176 | if !cache { 177 | req.Header.Set("Cache-Control", "no-store") 178 | } 179 | 180 | resp, data, warnings, err := do(ctx, client, req) 181 | 182 | if resp != nil && resp.StatusCode == http.StatusMethodNotAllowed { 183 | u.RawQuery = args.Encode() 184 | req, err = http.NewRequest(http.MethodGet, u.String(), nil) 185 | 186 | if err != nil { 187 | return nil, nil, warnings, err 188 | } 189 | 190 | if !cache { 191 | req.Header.Set("Cache-Control", "no-store") 192 | } 193 | 194 | return do(ctx, client, req) 195 | } 196 | 197 | return resp, data, warnings, err 198 | } 199 | 200 | func QueryRange(ctx context.Context, client promapi.Client, query string, r promapiv1.Range, 201 | cache bool) (model.Value, int, promapiv1.Warnings, error) { 202 | u := client.URL(epQueryRange, nil) 203 | q := u.Query() 204 | q.Set("query", query) 205 | q.Set("start", formatTime(r.Start)) 206 | q.Set("end", formatTime(r.End)) 207 | q.Set("step", strconv.FormatFloat(r.Step.Seconds(), 'f', -1, 64)) 208 | 209 | resp, data, warnings, err := doGetFallback(ctx, client, u, q, cache) //nolint:bodyclose 210 | if err != nil { 211 | if resp == nil { 212 | return nil, 0, warnings, err 213 | } 214 | 215 | return nil, resp.StatusCode, warnings, err 216 | } 217 | 218 | var qres queryResult 219 | 220 | return qres.v, resp.StatusCode, warnings, json.Unmarshal(data, &qres) 221 | } 222 | 223 | func Query(ctx context.Context, client promapi.Client, query string, ts time.Time, 224 | cache bool) (model.Value, int, promapiv1.Warnings, error) { 225 | u := client.URL(epQuery, nil) 226 | q := u.Query() 227 | 228 | q.Set("query", query) 229 | 230 | if !ts.IsZero() { 231 | q.Set("time", formatTime(ts)) 232 | } 233 | 234 | resp, data, warnings, err := doGetFallback(ctx, client, u, q, cache) //nolint:bodyclose 235 | if err != nil { 236 | if resp == nil { 237 | // Unknown error. 238 | return nil, 0, warnings, err 239 | } 240 | 241 | return nil, resp.StatusCode, warnings, err 242 | } 243 | 244 | var qres queryResult 245 | 246 | return qres.v, resp.StatusCode, warnings, json.Unmarshal(data, &qres) 247 | } 248 | 249 | func Series(ctx context.Context, client promapi.Client, matches []string, startTime time.Time, endTime time.Time, 250 | cache bool) ([]model.LabelSet, int, promapiv1.Warnings, error) { 251 | u := client.URL(epSeries, nil) 252 | q := u.Query() 253 | 254 | for _, m := range matches { 255 | q.Add("match[]", m) 256 | } 257 | 258 | q.Set("start", formatTime(startTime)) 259 | q.Set("end", formatTime(endTime)) 260 | 261 | resp, body, warnings, err := doGetFallback(ctx, client, u, q, cache) //nolint:bodyclose 262 | if err != nil { 263 | if resp == nil { 264 | return nil, 0, warnings, err 265 | } 266 | 267 | return nil, resp.StatusCode, warnings, err 268 | } 269 | 270 | var mset []model.LabelSet 271 | 272 | return mset, resp.StatusCode, warnings, json.Unmarshal(body, &mset) 273 | } 274 | 275 | func LabelNames(ctx context.Context, client promapi.Client, startTime time.Time, endTime time.Time, 276 | cache bool) ([]string, int, promapiv1.Warnings, error) { 277 | u := client.URL(epLabels, nil) 278 | q := u.Query() 279 | q.Set("start", formatTime(startTime)) 280 | q.Set("end", formatTime(endTime)) 281 | 282 | u.RawQuery = q.Encode() 283 | 284 | resp, body, warnings, err := doGetFallback(ctx, client, u, q, cache) //nolint:bodyclose 285 | if err != nil { 286 | if resp == nil { 287 | return nil, 0, warnings, err 288 | } 289 | 290 | return nil, resp.StatusCode, warnings, err 291 | } 292 | 293 | var labelNames []string 294 | 295 | if resp == nil { 296 | // Unknown error. 297 | return labelNames, 0, warnings, json.Unmarshal(body, &labelNames) 298 | } 299 | 300 | return labelNames, resp.StatusCode, warnings, json.Unmarshal(body, &labelNames) 301 | } 302 | 303 | func LabelValues(ctx context.Context, client promapi.Client, label string, startTime time.Time, endTime time.Time, 304 | cache bool) (model.LabelValues, int, promapiv1.Warnings, error) { 305 | u := client.URL(epLabelValues, map[string]string{"name": label}) 306 | q := u.Query() 307 | q.Set("start", formatTime(startTime)) 308 | q.Set("end", formatTime(endTime)) 309 | 310 | u.RawQuery = q.Encode() 311 | 312 | resp, body, warnings, err := doGetFallback(ctx, client, u, q, cache) //nolint:bodyclose 313 | if err != nil { 314 | if resp == nil { 315 | // Unknown error. 316 | return nil, 0, warnings, err 317 | } 318 | 319 | return nil, resp.StatusCode, warnings, err 320 | } 321 | 322 | var labelValues model.LabelValues 323 | if resp == nil { 324 | // Unknown error. 325 | return labelValues, 0, warnings, json.Unmarshal(body, &labelValues) 326 | } 327 | 328 | return labelValues, resp.StatusCode, warnings, json.Unmarshal(body, &labelValues) 329 | } 330 | 331 | func formatTime(t time.Time) string { 332 | return strconv.FormatFloat(float64(t.Unix())+float64(t.Nanosecond())/1e9, 'f', -1, 64) 333 | } 334 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= 2 | cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= 3 | cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= 4 | cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= 5 | cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= 6 | cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= 7 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w= 8 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= 9 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.12.0 h1:wL5IEG5zb7BVv1Kv0Xm92orq+5hB5Nipn3B5tn4Rqfk= 10 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.12.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I= 11 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= 12 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= 13 | github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI= 14 | github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= 15 | github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= 16 | github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= 17 | github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= 18 | github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= 19 | github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y= 20 | github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c= 21 | github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA= 22 | github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA= 23 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk= 24 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= 25 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= 26 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= 27 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= 28 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= 29 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= 30 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= 31 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= 32 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= 33 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= 34 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= 35 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0= 36 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= 37 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0= 38 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= 39 | github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs= 40 | github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= 41 | github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= 42 | github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= 43 | github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= 44 | github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= 45 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 46 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 47 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 48 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 49 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 50 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 51 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 52 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 53 | github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= 54 | github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= 55 | github.com/efficientgo/tools/core v0.0.0-20230505153745-6b7392939a60 h1:2tYT4FnRj0hAAkGpDVmIU2/PndCwhfSnI0onr9tvv7k= 56 | github.com/efficientgo/tools/core v0.0.0-20230505153745-6b7392939a60/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= 57 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 58 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 59 | github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= 60 | github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= 61 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 62 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 63 | github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= 64 | github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 65 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 66 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 67 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 68 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 69 | github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= 70 | github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= 71 | github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= 72 | github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 73 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 74 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 75 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 76 | github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= 77 | github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= 78 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 79 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 80 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= 81 | github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= 82 | github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= 83 | github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= 84 | github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= 85 | github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= 86 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 87 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 88 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 89 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 90 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 91 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 92 | github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= 93 | github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= 94 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 95 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 96 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 97 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 98 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 99 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 100 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 101 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 102 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 103 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 104 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 105 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 106 | github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= 107 | github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 108 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 109 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 110 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= 111 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 112 | github.com/oklog/run v1.2.0 h1:O8x3yXwah4A73hJdlrwo/2X6J62gE5qTMusH0dvz60E= 113 | github.com/oklog/run v1.2.0/go.mod h1:mgDbKRSwPhJfesJ4PntqFUbKQRZ50NgmZTSPlFA0YFk= 114 | github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= 115 | github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= 116 | github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= 117 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= 118 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= 119 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 120 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 121 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 122 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 123 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 124 | github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= 125 | github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= 126 | github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a h1:RF1vfKM34/3DbGNis22BGd6sDDY3XBi0eM7pYqmOEO0= 127 | github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a/go.mod h1:FGJuwvfcPY0V5enm+w8zF1RNS062yugQtPPQp1c4Io4= 128 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 129 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 130 | github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= 131 | github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= 132 | github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= 133 | github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= 134 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 135 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 136 | github.com/prometheus/prometheus v0.308.0 h1:kVh/5m1n6m4cSK9HYTDEbMxzuzCWyEdPdKSxFRxXj04= 137 | github.com/prometheus/prometheus v0.308.0/go.mod h1:xXYKzScyqyFHihpS0UsXpC2F3RA/CygOs7wb4mpdusE= 138 | github.com/prometheus/sigv4 v0.3.0 h1:QIG7nTbu0JTnNidGI1Uwl5AGVIChWUACxn2B/BQ1kms= 139 | github.com/prometheus/sigv4 v0.3.0/go.mod h1:fKtFYDus2M43CWKMNtGvFNHGXnAJJEGZbiYCmVp/F8I= 140 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 141 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 142 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 143 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 144 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 145 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 146 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 147 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 148 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 149 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 150 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 151 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= 152 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= 153 | go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= 154 | go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= 155 | go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= 156 | go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= 157 | go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= 158 | go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= 159 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 160 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 161 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 162 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 163 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 164 | go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= 165 | go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= 166 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 167 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 168 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 169 | golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= 170 | golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= 171 | golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90= 172 | golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg= 173 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 174 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 175 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 176 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 177 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 178 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 179 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 180 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 181 | golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= 182 | golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= 183 | golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= 184 | golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= 185 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 186 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 187 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 | golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= 189 | golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 190 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 191 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 192 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 193 | golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 194 | golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 195 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 196 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 197 | golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= 198 | golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= 199 | golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= 200 | golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= 201 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 202 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 203 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 204 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 205 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 206 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 207 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 208 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 209 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 210 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 211 | google.golang.org/api v0.252.0 h1:xfKJeAJaMwb8OC9fesr369rjciQ704AjU/psjkKURSI= 212 | google.golang.org/api v0.252.0/go.mod h1:dnHOv81x5RAmumZ7BWLShB/u7JZNeyalImxHmtTHxqw= 213 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797 h1:CirRxTOwnRWVLKzDNrs0CXAaVozJoR4G9xvdRecrdpk= 214 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251002232023-7c0ddcbb5797/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ= 215 | google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= 216 | google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= 217 | google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 218 | google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 219 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 220 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 221 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 222 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 223 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 224 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 225 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 226 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 227 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 228 | k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= 229 | k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= 230 | k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= 231 | k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= 232 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 233 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 234 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 235 | k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= 236 | k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 237 | -------------------------------------------------------------------------------- /cmd/up/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | stdlog "log" 9 | "net/http" 10 | "net/http/pprof" 11 | "net/url" 12 | "os" 13 | "os/signal" 14 | "strconv" 15 | "syscall" 16 | "time" 17 | 18 | "github.com/observatorium/up/pkg/auth" 19 | "github.com/observatorium/up/pkg/instr" 20 | "github.com/observatorium/up/pkg/logs" 21 | "github.com/observatorium/up/pkg/metrics" 22 | "github.com/observatorium/up/pkg/options" 23 | 24 | "github.com/go-kit/log" 25 | "github.com/go-kit/log/level" 26 | "github.com/oklog/run" 27 | "github.com/pkg/errors" 28 | promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 29 | "github.com/prometheus/client_golang/prometheus" 30 | "github.com/prometheus/client_golang/prometheus/collectors" 31 | "github.com/prometheus/client_golang/prometheus/promhttp" 32 | dto "github.com/prometheus/client_model/go" 33 | "github.com/prometheus/common/model" 34 | "github.com/prometheus/prometheus/prompb" 35 | "github.com/prometheus/prometheus/promql/parser" 36 | "gopkg.in/yaml.v2" 37 | ) 38 | 39 | const ( 40 | numOfEndpoints = 2 41 | timeoutBetweenQueries = 100 * time.Millisecond 42 | 43 | labelSuccess = "success" 44 | labelError = "error" 45 | ) 46 | 47 | // CallsFile is a struct that represents the YAML file format for queries. 48 | // It is exported for other third party packages to use when generating their queries. 49 | type CallsFile struct { 50 | Queries []options.QuerySpec `yaml:"queries"` 51 | Labels []options.LabelSpec `yaml:"labels"` 52 | Series []options.SeriesSpec `yaml:"series"` 53 | } 54 | 55 | type logsFile struct { 56 | Spec options.LogsSpec `yaml:"spec"` 57 | } 58 | 59 | func main() { //nolint:golint,funlen 60 | l := log.WithPrefix(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), "name", "up") 61 | l = log.WithPrefix(l, "ts", log.DefaultTimestampUTC) 62 | l = log.WithPrefix(l, "caller", log.DefaultCaller) 63 | 64 | opts, err := parseFlags(l) 65 | if err != nil { 66 | level.Error(l).Log("msg", "could not parse command line flags", "err", err) 67 | os.Exit(1) 68 | } 69 | 70 | l = level.NewFilter(l, opts.LogLevel) 71 | l = log.WithPrefix(l, "caller", log.DefaultCaller) 72 | 73 | reg := prometheus.NewRegistry() 74 | reg.MustRegister( 75 | collectors.NewGoCollector(), 76 | collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), 77 | ) 78 | 79 | m := instr.RegisterMetrics(reg) 80 | 81 | // Error channel to gather failures 82 | ch := make(chan error, numOfEndpoints) 83 | 84 | g := &run.Group{} 85 | { 86 | // Signal chans must be buffered. 87 | sig := make(chan os.Signal, 1) 88 | g.Add(func() error { 89 | signal.Notify(sig, os.Interrupt, syscall.SIGTERM) 90 | <-sig 91 | level.Info(l).Log("msg", "caught interrupt") 92 | return nil 93 | }, func(_ error) { 94 | close(sig) 95 | }) 96 | } 97 | // Schedule HTTP server 98 | scheduleHTTPServer(l, opts, reg, g) 99 | 100 | ctx := context.Background() 101 | 102 | var cancel context.CancelFunc 103 | if opts.Duration != 0 { 104 | ctx, cancel = context.WithTimeout(ctx, opts.Duration) 105 | } else { 106 | ctx, cancel = context.WithCancel(ctx) 107 | } 108 | 109 | if opts.WriteEndpoint != nil { 110 | g.Add(func() error { 111 | l := log.With(l, "component", "writer") 112 | level.Info(l).Log("msg", "starting the writer") 113 | 114 | return runPeriodically(ctx, opts, m.RemoteWriteRequests, l, ch, func(rCtx context.Context) { 115 | t := time.Now() 116 | httpCode, err := write(rCtx, l, opts) 117 | duration := time.Since(t).Seconds() 118 | m.RemoteWriteRequestDuration.Observe(duration) 119 | if err != nil { 120 | m.RemoteWriteRequests.WithLabelValues(labelError, strconv.Itoa(httpCode)).Inc() 121 | level.Error(l).Log("msg", "failed to make request", "err", err) 122 | } else { 123 | m.RemoteWriteRequests.WithLabelValues(labelSuccess, strconv.Itoa(httpCode)).Inc() 124 | } 125 | }) 126 | }, func(_ error) { 127 | cancel() 128 | }) 129 | } 130 | 131 | if opts.ReadEndpoint != nil && opts.WriteEndpoint != nil { 132 | g.Add(func() error { 133 | l := log.With(l, "component", "reader") 134 | level.Info(l).Log("msg", "starting the reader") 135 | 136 | // Wait for at least one period before start reading metrics. 137 | level.Info(l).Log("msg", "waiting for initial delay before querying", "type", opts.EndpointType) 138 | select { 139 | case <-ctx.Done(): 140 | return nil 141 | case <-time.After(opts.InitialQueryDelay): 142 | } 143 | 144 | level.Info(l).Log("msg", "start querying", "type", opts.EndpointType) 145 | 146 | return runPeriodically(ctx, opts, m.QueryResponses, l, ch, func(rCtx context.Context) { 147 | t := time.Now() 148 | httpCode, err := read(rCtx, l, m, opts) 149 | duration := time.Since(t).Seconds() 150 | m.QueryResponseDuration.Observe(duration) 151 | if err != nil { 152 | if httpCode != 0 { 153 | m.QueryResponses.WithLabelValues(labelError, strconv.Itoa(httpCode)).Inc() 154 | } 155 | level.Error(l).Log("msg", "failed to query", "err", err) 156 | } else { 157 | if httpCode != 0 { 158 | m.QueryResponses.WithLabelValues(labelSuccess, strconv.Itoa(httpCode)).Inc() 159 | } 160 | } 161 | }) 162 | }, func(_ error) { 163 | cancel() 164 | }) 165 | } 166 | 167 | if opts.ReadEndpoint != nil && opts.Queries != nil { 168 | addCustomQueryRunGroup(ctx, g, l, opts, m, cancel) 169 | } 170 | 171 | if err := g.Run(); err != nil { 172 | level.Info(l).Log("msg", "run group exited with error", "err", err) 173 | } 174 | 175 | close(ch) 176 | 177 | fail := false 178 | for err := range ch { 179 | fail = true 180 | 181 | level.Error(l).Log("err", err) 182 | } 183 | 184 | if fail { 185 | level.Error(l).Log("msg", "up failed") 186 | os.Exit(1) 187 | } 188 | 189 | level.Info(l).Log("msg", "up completed its mission!") 190 | } 191 | 192 | func write(ctx context.Context, l log.Logger, opts options.Options) (int, error) { 193 | switch opts.EndpointType { 194 | case options.MetricsEndpointType: 195 | return metrics.Write(ctx, opts.WriteEndpoint, opts.Token, metrics.Generate(opts.Labels), l, opts.TLS, 196 | opts.TenantHeader, opts.Tenant) 197 | case options.LogsEndpointType: 198 | return logs.Write(ctx, opts.WriteEndpoint, opts.Token, logs.Generate(opts.Labels, opts.Logs), l, opts.TLS) 199 | } 200 | 201 | return 0, fmt.Errorf("invalid endpoint-type: %v", opts.EndpointType) 202 | } 203 | 204 | func read(ctx context.Context, l log.Logger, m instr.Metrics, opts options.Options) (int, error) { 205 | switch opts.EndpointType { 206 | case options.MetricsEndpointType: 207 | return metrics.Read(ctx, opts.ReadEndpoint, opts.Token, opts.Labels, -1*opts.InitialQueryDelay, opts.Latency, m, l, opts.TLS) 208 | case options.LogsEndpointType: 209 | return logs.Read(ctx, opts.ReadEndpoint, opts.Token, opts.Labels, -1*opts.InitialQueryDelay, opts.Latency, m, l, opts.TLS) 210 | } 211 | 212 | return 0, fmt.Errorf("invalid endpoint-type: %v", opts.EndpointType) 213 | } 214 | 215 | func query(ctx context.Context, l log.Logger, q options.Query, opts options.Options) (int, promapiv1.Warnings, error) { 216 | switch opts.EndpointType { 217 | case options.MetricsEndpointType: 218 | return metrics.Query(ctx, l, opts.ReadEndpoint, opts.Token, q, opts.TLS, opts.DefaultStep) 219 | case options.LogsEndpointType: 220 | return logs.Query(ctx, l, opts.ReadEndpoint, opts.Token, q, opts.TLS, opts.DefaultStep) 221 | } 222 | 223 | return 0, nil, fmt.Errorf("invalid endpoint-type: %v", opts.EndpointType) 224 | } 225 | 226 | func addCustomQueryRunGroup(ctx context.Context, g *run.Group, l log.Logger, opts options.Options, m instr.Metrics, cancel func()) { 227 | g.Add(func() error { 228 | l := log.With(l, "component", "query-reader") 229 | level.Info(l).Log("msg", "starting the reader for queries") 230 | 231 | // Wait for at least one period before start reading metrics. 232 | level.Info(l).Log("msg", "waiting for initial delay before querying specified queries") 233 | select { 234 | case <-ctx.Done(): 235 | return nil 236 | case <-time.After(opts.InitialQueryDelay): 237 | } 238 | 239 | level.Info(l).Log("msg", "start querying for specified queries") 240 | 241 | for { 242 | select { 243 | case <-ctx.Done(): 244 | return nil 245 | default: 246 | for _, q := range opts.Queries { 247 | select { 248 | case <-ctx.Done(): 249 | return nil 250 | default: 251 | t := time.Now() 252 | httpCode, warn, err := query(ctx, l, q, opts) 253 | duration := time.Since(t).Seconds() 254 | queryType := q.GetType() 255 | name := q.GetName() 256 | if err != nil { 257 | level.Info(l).Log( 258 | "msg", "failed to execute specified query", 259 | "type", queryType, 260 | "name", name, 261 | "duration", duration, 262 | "warnings", fmt.Sprintf("%#+v", warn), 263 | "err", err, 264 | ) 265 | if httpCode != 0 { 266 | m.CustomQueryErrors.WithLabelValues(queryType, name, strconv.Itoa(httpCode)).Inc() 267 | } 268 | 269 | } else { 270 | level.Debug(l).Log("msg", "successfully executed specified query", 271 | "type", queryType, 272 | "name", name, 273 | "duration", duration, 274 | "warnings", fmt.Sprintf("%#+v", warn), 275 | ) 276 | 277 | m.CustomQueryLastDuration.WithLabelValues(queryType, name, strconv.Itoa(httpCode)).Set(duration) 278 | } 279 | if httpCode != 0 { 280 | m.CustomQueryExecuted.WithLabelValues(queryType, name, strconv.Itoa(httpCode)).Inc() 281 | m.CustomQueryRequestDuration.WithLabelValues(queryType, name, strconv.Itoa(httpCode)).Observe(duration) 282 | } 283 | } 284 | time.Sleep(timeoutBetweenQueries) 285 | } 286 | time.Sleep(timeoutBetweenQueries) 287 | } 288 | } 289 | }, func(_ error) { 290 | cancel() 291 | }) 292 | } 293 | 294 | func runPeriodically(ctx context.Context, opts options.Options, c *prometheus.CounterVec, l log.Logger, ch chan error, 295 | f func(rCtx context.Context)) error { 296 | var ( 297 | t = time.NewTicker(opts.Period) 298 | deadline time.Time 299 | rCtx context.Context 300 | rCancel context.CancelFunc 301 | ) 302 | 303 | for { 304 | select { 305 | case <-t.C: 306 | // NOTICE: Do not propagate parent context to prevent cancellation of in-flight request. 307 | // It will be cancelled after the deadline. 308 | deadline = time.Now().Add(opts.Period) 309 | rCtx, rCancel = context.WithDeadline(context.Background(), deadline) 310 | 311 | // Will only get scheduled once per period and guaranteed to get cancelled after deadline. 312 | go func() { 313 | defer rCancel() // Make sure context gets cancelled even if execution panics. 314 | 315 | f(rCtx) 316 | }() 317 | case <-ctx.Done(): 318 | t.Stop() 319 | 320 | if rCtx != nil { 321 | select { 322 | // If it gets immediately cancelled, zero value of deadline won't cause a lock! 323 | case <-time.After(time.Until(deadline)): 324 | rCancel() 325 | case <-rCtx.Done(): 326 | } 327 | } 328 | 329 | return reportResults(l, ch, c, opts.SuccessThreshold) 330 | } 331 | } 332 | } 333 | 334 | func reportResults(l log.Logger, ch chan error, c *prometheus.CounterVec, threshold float64) error { 335 | metrics := make(chan prometheus.Metric, numOfEndpoints) 336 | c.Collect(metrics) 337 | close(metrics) 338 | 339 | var success, failures float64 340 | 341 | for m := range metrics { 342 | m1 := &dto.Metric{} 343 | if err := m.Write(m1); err != nil { 344 | level.Warn(l).Log("msg", "cannot read success and error count from prometheus counter", "err", err) 345 | } 346 | 347 | for _, l := range m1.Label { 348 | switch *l.Value { 349 | case labelError: 350 | failures = m1.GetCounter().GetValue() 351 | case labelSuccess: 352 | success = m1.GetCounter().GetValue() 353 | } 354 | } 355 | } 356 | 357 | level.Info(l).Log("msg", "number of requests", "success", success, "errors", failures) 358 | 359 | ratio := success / (success + failures) 360 | if ratio < threshold { 361 | level.Error(l).Log("msg", "ratio is below threshold") 362 | 363 | err := errors.Errorf("failed with less than %2.f%% success ratio - actual %2.f%%", threshold*100, ratio*100) 364 | ch <- err 365 | 366 | return err 367 | } 368 | 369 | return nil 370 | } 371 | 372 | // Helpers 373 | 374 | func parseFlags(l log.Logger) (options.Options, error) { 375 | var ( 376 | rawEndpointType string 377 | rawWriteEndpoint string 378 | rawReadEndpoint string 379 | rawLogLevel string 380 | queriesFileName string 381 | logsFileName string 382 | tokenFile string 383 | token string 384 | ) 385 | 386 | opts := options.Options{} 387 | 388 | flag.StringVar(&rawLogLevel, "log.level", "info", "The log filtering level. Options: 'error', 'warn', 'info', 'debug'.") 389 | flag.StringVar(&rawEndpointType, "endpoint-type", "metrics", "The endpoint type. Options: 'logs', 'metrics'.") 390 | flag.StringVar(&rawWriteEndpoint, "endpoint-write", "", "The endpoint to which to make remote-write requests.") 391 | flag.StringVar(&rawReadEndpoint, "endpoint-read", "", "The endpoint to which to make query requests.") 392 | flag.Var(&opts.Labels, "labels", "The labels in addition to '__name__' that should be applied to remote-write requests.") 393 | flag.StringVar(&opts.Listen, "listen", ":8080", "The address on which internal server runs.") 394 | flag.Var(&opts.Logs, "logs", "The logs that should be sent to remote-write requests.") 395 | flag.StringVar(&logsFileName, "logs-file", "", "A file containing logs to send against the logs write endpoint.") 396 | flag.StringVar(&opts.Name, "name", "up", "The name of the metric to send in remote-write requests.") 397 | flag.StringVar(&token, "token", "", 398 | "The bearer token to set in the authorization header on requests. Takes predence over --token-file if set.") 399 | flag.StringVar(&tokenFile, "token-file", "", 400 | "The file from which to read a bearer token to set in the authorization header on requests.") 401 | flag.StringVar(&queriesFileName, "queries-file", "", "A file containing queries to run against the read endpoint.") 402 | flag.DurationVar(&opts.Period, "period", 5*time.Second, "The time to wait between remote-write requests.") 403 | flag.DurationVar(&opts.Duration, "duration", 5*time.Minute, 404 | "The duration of the up command to run until it stops. If 0 it will not stop until the process is terminated.") 405 | flag.Float64Var(&opts.SuccessThreshold, "threshold", 0.9, "The percentage of successful requests needed to succeed overall. 0 - 1.") 406 | flag.DurationVar(&opts.Latency, "latency", 15*time.Second, 407 | "The maximum allowable latency between writing and reading.") 408 | flag.DurationVar(&opts.InitialQueryDelay, "initial-query-delay", 10*time.Second, 409 | "The time to wait before executing the first query.") 410 | flag.DurationVar(&opts.DefaultStep, "step", 5*time.Minute, "Default step duration for range queries. "+ 411 | "Can be overridden if step is set in query spec.") 412 | 413 | flag.StringVar(&opts.TLS.Cert, "tls-client-cert-file", "", 414 | "File containing the default x509 Certificate for HTTPS. Leave blank to disable TLS.") 415 | flag.StringVar(&opts.TLS.Key, "tls-client-private-key-file", "", 416 | "File containing the default x509 private key matching --tls-cert-file. Leave blank to disable TLS.") 417 | flag.StringVar(&opts.TLS.CACert, "tls-ca-file", "", 418 | "File containing the TLS CA to use against servers for verification. If no CA is specified, there won't be any verification.") 419 | flag.StringVar(&opts.TenantHeader, "tenant-header", "tenant_id", 420 | "Name of HTTP header used to determine tenant for write requests.") 421 | flag.StringVar(&opts.Tenant, "tenant", "", "Tenant ID to used to determine tenant for write requests.") 422 | flag.Parse() 423 | 424 | return buildOptionsFromFlags( 425 | l, opts, rawLogLevel, rawEndpointType, rawWriteEndpoint, rawReadEndpoint, queriesFileName, logsFileName, token, tokenFile, 426 | ) 427 | } 428 | 429 | func buildOptionsFromFlags( 430 | l log.Logger, 431 | opts options.Options, 432 | rawLogLevel, rawEndpointType, rawWriteEndpoint, rawReadEndpoint, queriesFileName, logsFileName, token, tokenFile string, 433 | ) (options.Options, error) { 434 | var err error 435 | 436 | err = parseLogLevel(&opts, rawLogLevel) 437 | if err != nil { 438 | return opts, errors.Wrap(err, "parsing log level") 439 | } 440 | 441 | err = parseEndpointType(&opts, rawEndpointType) 442 | if err != nil { 443 | return opts, errors.Wrap(err, "parsing endpoint type") 444 | } 445 | 446 | err = parseWriteEndpoint(&opts, l, rawWriteEndpoint) 447 | if err != nil { 448 | return opts, errors.Wrap(err, "parsing write endpoint") 449 | } 450 | 451 | err = parseReadEndpoint(&opts, l, rawReadEndpoint) 452 | if err != nil { 453 | return opts, errors.Wrap(err, "parsing read endpoint") 454 | } 455 | 456 | err = parseQueriesFileName(&opts, l, queriesFileName) 457 | if err != nil { 458 | return opts, errors.Wrap(err, "parsing queries file name") 459 | } 460 | 461 | err = parseLogsFileName(&opts, l, logsFileName) 462 | if err != nil { 463 | return opts, errors.Wrap(err, "parsing logs file name") 464 | } 465 | 466 | if opts.Latency <= opts.Period { 467 | return opts, errors.Errorf("--latency cannot be less than period") 468 | } 469 | 470 | opts.Labels = append(opts.Labels, prompb.Label{ 471 | Name: "__name__", 472 | Value: opts.Name, 473 | }) 474 | // We need to ensure labels are sorted before we proceed. 475 | opts.Labels.Sort() 476 | 477 | opts.Token = tokenProvider(token, tokenFile) 478 | 479 | return opts, err 480 | } 481 | 482 | func parseLogLevel(opts *options.Options, rawLogLevel string) error { 483 | switch rawLogLevel { 484 | case "error": 485 | opts.LogLevel = level.AllowError() 486 | case "warn": 487 | opts.LogLevel = level.AllowWarn() 488 | case "info": 489 | opts.LogLevel = level.AllowInfo() 490 | case "debug": 491 | opts.LogLevel = level.AllowDebug() 492 | default: 493 | return errors.Errorf("unexpected log level") 494 | } 495 | 496 | return nil 497 | } 498 | 499 | func parseEndpointType(opts *options.Options, rawEndpointType string) error { 500 | switch options.EndpointType(rawEndpointType) { 501 | case options.LogsEndpointType: 502 | opts.EndpointType = options.LogsEndpointType 503 | case options.MetricsEndpointType: 504 | opts.EndpointType = options.MetricsEndpointType 505 | default: 506 | return errors.Errorf("unexpected endpoint type") 507 | } 508 | 509 | return nil 510 | } 511 | 512 | func parseWriteEndpoint(opts *options.Options, l log.Logger, rawWriteEndpoint string) error { 513 | if rawWriteEndpoint != "" { 514 | writeEndpoint, err := url.ParseRequestURI(rawWriteEndpoint) 515 | if err != nil { 516 | return fmt.Errorf("--endpoint-write is invalid: %w", err) 517 | } 518 | 519 | opts.WriteEndpoint = writeEndpoint 520 | } else { 521 | l.Log("msg", "no write endpoint specified, no write tests being performed") 522 | } 523 | 524 | return nil 525 | } 526 | 527 | func parseReadEndpoint(opts *options.Options, l log.Logger, rawReadEndpoint string) error { 528 | if rawReadEndpoint != "" { 529 | readEndpoint, err := url.ParseRequestURI(rawReadEndpoint) 530 | if err != nil { 531 | return fmt.Errorf("--endpoint-read is invalid: %w", err) 532 | } 533 | 534 | opts.ReadEndpoint = readEndpoint 535 | } else { 536 | l.Log("msg", "no read endpoint specified, no read tests being performed") 537 | } 538 | 539 | return nil 540 | } 541 | 542 | func parseQueriesFileName(opts *options.Options, l log.Logger, queriesFileName string) error { 543 | if queriesFileName != "" { 544 | b, err := ioutil.ReadFile(queriesFileName) 545 | if err != nil { 546 | return fmt.Errorf("--queries-file is invalid: %w", err) 547 | } 548 | 549 | qf := CallsFile{} 550 | err = yaml.Unmarshal(b, &qf) //nolint:typecheck 551 | 552 | if err != nil { 553 | return fmt.Errorf("--queries-file content is invalid: %w", err) 554 | } 555 | 556 | l.Log("msg", fmt.Sprintf("%d queries configured to be queried periodically", len(qf.Queries))) 557 | 558 | // validate queries 559 | for _, q := range qf.Queries { 560 | _, err = parser.ParseExpr(q.Query) 561 | if err != nil { 562 | return fmt.Errorf("query %q in --queries-file content is invalid: %w", q.Name, err) 563 | } 564 | 565 | opts.Queries = append(opts.Queries, q) 566 | } 567 | 568 | for _, q := range qf.Series { 569 | if len(q.Matchers) == 0 { 570 | return fmt.Errorf("series query %q in --queries-file matchers cannot be empty", q.Name) 571 | } 572 | 573 | if len(q.Matchers) > 0 { 574 | for _, s := range q.Matchers { 575 | if _, err := parser.ParseMetricSelector(s); err != nil { 576 | return fmt.Errorf("series query %q in --queries-file matchers are invalid: %w", q.Name, err) 577 | } 578 | } 579 | } 580 | 581 | opts.Queries = append(opts.Queries, q) 582 | } 583 | 584 | for _, q := range qf.Labels { 585 | if len(q.Label) > 0 && !model.LabelNameRE.MatchString(q.Label) { 586 | return fmt.Errorf("label_values query %q in --queries-file label is invalid: %w", q.Name, err) 587 | } 588 | 589 | opts.Queries = append(opts.Queries, q) 590 | } 591 | } 592 | 593 | return nil 594 | } 595 | 596 | func parseLogsFileName(opts *options.Options, l log.Logger, logsFileName string) error { 597 | if logsFileName != "" { 598 | b, err := ioutil.ReadFile(logsFileName) 599 | if err != nil { 600 | return fmt.Errorf("--logs-file is invalid: %w", err) 601 | } 602 | 603 | lf := logsFile{} 604 | err = yaml.Unmarshal(b, &lf) //nolint:typecheck 605 | 606 | if err != nil { 607 | return fmt.Errorf("--logs-file content is invalid: %w", err) 608 | } 609 | 610 | l.Log("msg", fmt.Sprintf("%d logs configured to be written periodically", len(lf.Spec.Logs))) 611 | 612 | opts.Logs = lf.Spec.Logs 613 | } 614 | 615 | return nil 616 | } 617 | 618 | func tokenProvider(token, tokenFile string) auth.TokenProvider { 619 | var res auth.TokenProvider 620 | 621 | res = auth.NewNoOpTokenProvider() 622 | if tokenFile != "" { 623 | res = auth.NewFileToken(tokenFile) 624 | } 625 | 626 | if token != "" { 627 | res = auth.NewStaticToken(token) 628 | } 629 | 630 | return res 631 | } 632 | 633 | func scheduleHTTPServer(l log.Logger, opts options.Options, reg *prometheus.Registry, g *run.Group) { 634 | logger := log.With(l, "component", "http") 635 | router := http.NewServeMux() 636 | router.Handle("/metrics", promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))) 637 | router.HandleFunc("/debug/pprof/", pprof.Index) 638 | 639 | srv := &http.Server{Addr: opts.Listen, Handler: router} 640 | 641 | g.Add(func() error { 642 | level.Info(logger).Log("msg", "starting the HTTP server", "address", opts.Listen) 643 | return srv.ListenAndServe() 644 | }, func(err error) { 645 | if errors.Is(err, http.ErrServerClosed) { 646 | level.Warn(logger).Log("msg", "internal server closed unexpectedly") 647 | return 648 | } 649 | level.Info(logger).Log("msg", "shutting down internal server") 650 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 651 | defer cancel() 652 | if err := srv.Shutdown(ctx); err != nil { 653 | stdlog.Fatal(err) 654 | } 655 | }) 656 | } 657 | --------------------------------------------------------------------------------