├── .buildkite ├── .dockerignore ├── lib │ └── release_dry_run.sh ├── steps │ ├── release-s3-version.sh │ ├── build-binary.sh │ ├── upload-release-steps.sh │ ├── build-lambda.sh │ ├── upload-to-s3.sh │ └── release-github.sh ├── Dockerfile ├── docker-compose.yaml ├── pipeline.release.yml └── pipeline.yml ├── .gitignore ├── version └── version.go ├── .github └── dependabot.yml ├── Dockerfile.lambda ├── backend ├── backends.go ├── cloudwatch_test.go ├── newrelic_test.go ├── newrelic.go ├── statsd.go ├── prometheus.go ├── cloudwatch.go ├── stackdriver_test.go ├── stackdriver.go └── prometheus_test.go ├── token ├── memory.go ├── memory_test.go ├── token.go ├── ssm.go ├── mock │ ├── ssm_client.go │ └── secretsmanager_client.go ├── ssm_test.go ├── secretsmanager.go └── secretsmanager_test.go ├── Dockerfile ├── LICENSE.md ├── RELEASE.md ├── go.mod ├── main.go ├── collector ├── collector_test.go └── collector.go ├── lambda └── main.go ├── README.md ├── go.sum └── CHANGELOG.md /.buildkite/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | build/ 3 | handler.zip 4 | bin/ 5 | lambda/handler 6 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // Version the library version number 4 | const Version = "5.9.3" 5 | -------------------------------------------------------------------------------- /.buildkite/lib/release_dry_run.sh: -------------------------------------------------------------------------------- 1 | release_dry_run() { 2 | if [[ "${RELEASE_DRY_RUN:-false}" == "true" ]]; then 3 | echo "$@" 4 | else 5 | "$@" 6 | fi 7 | } 8 | -------------------------------------------------------------------------------- /.buildkite/steps/release-s3-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | source .buildkite/lib/release_dry_run.sh 5 | 6 | release_dry_run .buildkite/steps/upload-to-s3.sh release 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: / 5 | schedule: 6 | interval: "weekly" 7 | open-pull-requests-limit: 2 8 | reviewers: 9 | - "buildkite/pipelines-dispatch" 10 | 11 | -------------------------------------------------------------------------------- /.buildkite/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM buildkite/agent:3.45.0 AS agent 2 | 3 | FROM public.ecr.aws/docker/library/golang:1.20.2-alpine3.17 4 | COPY --from=agent /usr/local/bin/buildkite-agent /usr/local/bin/buildkite-agent 5 | RUN apk --no-cache add bash zip curl aws-cli github-cli 6 | -------------------------------------------------------------------------------- /Dockerfile.lambda: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/provided:al2 2 | 3 | RUN yum install -y unzip wget && \ 4 | wget "https://github.com/buildkite/buildkite-agent-metrics/releases/latest/download/handler.zip" && \ 5 | unzip handler.zip && rm -f handler.zip 6 | 7 | ENTRYPOINT ["./bootstrap"] 8 | -------------------------------------------------------------------------------- /.buildkite/steps/build-binary.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -eu 4 | 5 | GOOS=${1:-linux} 6 | GOARCH=${2:-amd64} 7 | 8 | export GOOS 9 | export GOARCH 10 | export CGO_ENABLED=0 11 | 12 | go build -o "buildkite-agent-metrics-${GOOS}-${GOARCH}" . 13 | buildkite-agent artifact upload "buildkite-agent-metrics-${GOOS}-${GOARCH}" 14 | -------------------------------------------------------------------------------- /.buildkite/steps/upload-release-steps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [[ "${RELEASE_DRY_RUN:-false}" != "true" && "$BUILDKITE_BRANCH" != "${BUILDKITE_TAG:-}" ]]; then 5 | echo "Skipping release for a non-tag build on $BUILDKITE_BRANCH" >&2 6 | exit 0 7 | fi 8 | 9 | buildkite-agent pipeline upload .buildkite/pipeline.release.yml 10 | -------------------------------------------------------------------------------- /backend/backends.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import "github.com/buildkite/buildkite-agent-metrics/v5/collector" 4 | 5 | // Backend is a receiver of metrics 6 | type Backend interface { 7 | Collect(r *collector.Result) error 8 | } 9 | 10 | // Closer is an interface for backends that need to dispose of resources 11 | type Closer interface { 12 | Close() error 13 | } 14 | -------------------------------------------------------------------------------- /.buildkite/steps/build-lambda.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -eu 4 | 5 | docker run --rm --volume "$PWD:/code" \ 6 | --workdir /code \ 7 | --rm \ 8 | --env CGO_ENABLED=0 \ 9 | golang:1.21 \ 10 | go build -tags lambda.norpc -o lambda/bootstrap ./lambda 11 | 12 | chmod +x lambda/bootstrap 13 | 14 | mkdir -p dist/ 15 | zip -j handler.zip lambda/bootstrap 16 | 17 | buildkite-agent artifact upload handler.zip 18 | -------------------------------------------------------------------------------- /token/memory.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | type inMemoryProvider struct { 4 | // The token value to provide on each Get call. 5 | Token string 6 | } 7 | 8 | // NewInMemory constructs a Buildkite API token provider backed by a in-memory string. 9 | func NewInMemory(token string) (Provider, error) { 10 | return &inMemoryProvider{Token: token}, nil 11 | } 12 | 13 | func (p inMemoryProvider) Get() (string, error) { 14 | return p.Token, nil 15 | } 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21 as builder 2 | WORKDIR /go/src/github.com/buildkite/buildkite-agent-metrics/ 3 | COPY . . 4 | RUN GO111MODULE=on GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o buildkite-agent-metrics . 5 | 6 | FROM alpine:3.18 7 | RUN apk update && apk add curl ca-certificates 8 | COPY --from=builder /go/src/github.com/buildkite/buildkite-agent-metrics/buildkite-agent-metrics . 9 | EXPOSE 8080 8125 10 | ENTRYPOINT ["./buildkite-agent-metrics"] 11 | -------------------------------------------------------------------------------- /.buildkite/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | agent: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | volumes: 9 | - ../:/workspace:cached 10 | working_dir: /workspace 11 | environment: 12 | - BUILDKITE_BUILD_NUMBER 13 | - BUILDKITE_BUILD_ID 14 | - BUILDKITE_JOB_ID 15 | - BUILDKITE_BRANCH 16 | - BUILDKITE_TAG 17 | - BUILDKITE_AGENT_ACCESS_TOKEN 18 | - GITHUB_RELEASE_ACCESS_TOKEN 19 | - RELEASE_DRY_RUN 20 | -------------------------------------------------------------------------------- /token/memory_test.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | const ( 8 | inMemoryTestToken = "some-token" 9 | ) 10 | 11 | func TestInMemoryProvider_Get(t *testing.T) { 12 | provider, err := NewInMemory(inMemoryTestToken) 13 | if err != nil { 14 | t.Fatalf("no errors were expected to be returned by NewInMemory but got: %v", err) 15 | } 16 | token, err := provider.Get() 17 | if err != nil { 18 | t.Fatalf("no errors were expected to be returned by InMemoryProvider.Get() but got: %v", err) 19 | } 20 | 21 | if token != inMemoryTestToken { 22 | t.Fatalf("expecting '%s' to be returned by InMemoryProvider.Get() but got: %s", inMemoryTestToken, token) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /token/token.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // Provider represents the behaviour of obtaining a Buildkite token. 9 | type Provider interface { 10 | Get() (string, error) 11 | } 12 | 13 | // Must is a helper function to ensure a Provider object can be successfully instantiated when calling any of the 14 | // constructor functions provided by this package. 15 | // 16 | // This helper is intended to be used at program startup to load the Provider implementation to be used. Such as: 17 | // 18 | // var provider := token.Must(token.NewSSMProvider()) 19 | func Must(prov Provider, err error) Provider { 20 | if err != nil { 21 | _, _ = fmt.Fprintf(os.Stderr, "failed to initialize Buildkite token provider: %v", err) 22 | os.Exit(1) 23 | } 24 | return prov 25 | } 26 | -------------------------------------------------------------------------------- /backend/cloudwatch_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestParseCloudWatchDimensions(t *testing.T) { 9 | for _, tc := range []struct { 10 | s string 11 | expected []CloudWatchDimension 12 | }{ 13 | {"", []CloudWatchDimension{}}, 14 | {" ", []CloudWatchDimension{}}, 15 | {"Key=Value", []CloudWatchDimension{{"Key", "Value"}}}, 16 | {"Key=Value,Another=Value", []CloudWatchDimension{{"Key", "Value"}, {"Another", "Value"}}}, 17 | {"Key=Value, Another=Value", []CloudWatchDimension{{"Key", "Value"}, {"Another", "Value"}}}, 18 | {"Key=Value, Another=Value ", []CloudWatchDimension{{"Key", "Value"}, {"Another", "Value"}}}, 19 | } { 20 | t.Run(tc.s, func(t *testing.T) { 21 | d, err := ParseCloudWatchDimensions(tc.s) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | if !reflect.DeepEqual(d, tc.expected) { 26 | t.Errorf("Expected %s to parse to %#v, got %#v", tc.s, tc.expected, d) 27 | } 28 | }) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Buildkite Pty Ltd 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /.buildkite/pipeline.release.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - block: ":rocket: Release ${BUILDKITE_TAG}?" 3 | key: block-release 4 | 5 | - label: ":s3: Upload" 6 | command: ".buildkite/steps/release-s3-version.sh" 7 | depends_on: 8 | - block-release 9 | agents: 10 | queue: "elastic-runners" 11 | concurrency: 1 12 | concurrency_group: 'release_buildkite_metrics_github' 13 | plugins: 14 | - aws-assume-role-with-web-identity: 15 | role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-metrics 16 | 17 | - label: ":github: Release" 18 | command: ".buildkite/steps/release-github.sh" 19 | depends_on: 20 | - block-release 21 | agents: 22 | queue: "elastic-runners" 23 | plugins: 24 | - aws-assume-role-with-web-identity: 25 | role-arn: arn:aws:iam::445615400570:role/pipeline-buildkite-buildkite-agent-metrics 26 | - aws-ssm#v1.0.0: 27 | parameters: 28 | GITHUB_RELEASE_ACCESS_TOKEN: /pipelines/buildkite/buildkite-agent-metrics/GITHUB_RELEASE_ACCESS_TOKEN 29 | - docker-compose#v4.14.0: 30 | config: .buildkite/docker-compose.yaml 31 | run: agent 32 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Instructions 2 | 3 | 1. Determine the SemVer version for the new release: `$VERSION`. Note: [semver does not include a `v` prefix](https://github.com/semver/semver/blob/master/semver.md#is-v123-a-semantic-version). 4 | 1. Generate a changelog using [ghch](https://github.com/buildkite/ghch) `ghch --format=markdown --next-version=v$VERSION`. 5 | 1. Create a new branch pointing to the trunk `git fetch origin && git switch -c release/$VERSION origin/master`. 6 | 1. Update [CHANGELOG.md](CHANEGLOG.md) with the generated changelog. 7 | 1. Update [`version/version.go`](version/version.go) with the new version. 8 | 1. Push your branch and wait for CI to pass. 9 | 1. Merge your branch. 10 | 1. Switch back to the trunk branch: `git switch master`. 11 | 1. Pull the trunk branch with tags, and ensure it is pointing to the commit you want to release. 12 | 1. Tag the release with the new version: `git tag -sm v$VERSION v$VERSION`. 13 | 1. Push the tag to GitHub: `git push --tags`. 14 | 1. A tag build will commence on the [pipeline](https://buildkite.com/buildkite/buildkite-agent-metrics). Wait for and unblock the release steps. 15 | 1. Check that the [draft release](https://github.com/buildkite/buildkite-agent-metrics/releases) that the build creates is acceptable and release it. Note: the job log for release step on the tag build will contain a link to the draft release. 16 | -------------------------------------------------------------------------------- /.buildkite/steps/upload-to-s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | export AWS_DEFAULT_REGION=us-east-1 5 | 6 | # Updated from https://docs.aws.amazon.com/general/latest/gr/rande.html#lambda_region 7 | EXTRA_REGIONS=( 8 | us-east-2 9 | us-west-1 10 | us-west-2 11 | af-south-1 12 | ap-east-1 13 | ap-south-1 14 | ap-northeast-2 15 | ap-northeast-1 16 | ap-southeast-2 17 | ap-southeast-1 18 | ca-central-1 19 | eu-central-1 20 | eu-west-1 21 | eu-west-2 22 | eu-south-1 23 | eu-west-3 24 | eu-north-1 25 | me-south-1 26 | sa-east-1 27 | ) 28 | 29 | VERSION=$(awk -F\" '/const Version/ {print $2}' version/version.go) 30 | BASE_BUCKET=buildkite-lambdas 31 | BUCKET_PATH="buildkite-agent-metrics" 32 | 33 | if [[ "${1:-}" == "release" ]] ; then 34 | BUCKET_PATH="${BUCKET_PATH}/v${VERSION}" 35 | else 36 | BUCKET_PATH="${BUCKET_PATH}/builds/${BUILDKITE_BUILD_NUMBER}" 37 | fi 38 | 39 | echo "~~~ :buildkite: Downloading artifacts" 40 | buildkite-agent artifact download handler.zip . 41 | 42 | echo "--- :s3: Uploading lambda to ${BASE_BUCKET}/${BUCKET_PATH}/ in ${AWS_DEFAULT_REGION}" 43 | aws s3 cp --acl public-read handler.zip "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip" 44 | 45 | for region in "${EXTRA_REGIONS[@]}" ; do 46 | bucket="${BASE_BUCKET}-${region}" 47 | echo "--- :s3: Copying files to ${bucket}" 48 | aws --region "${region}" s3 cp --acl public-read "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip" "s3://${bucket}/${BUCKET_PATH}/handler.zip" 49 | done 50 | -------------------------------------------------------------------------------- /token/ssm.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/ssm" 8 | ) 9 | 10 | // SSMService represents the minimal subset of interactions required to retrieve a Buildkite API token from 11 | // AWS Systems Manager parameter store. 12 | type SSMClient interface { 13 | GetParameter(*ssm.GetParameterInput) (*ssm.GetParameterOutput, error) 14 | } 15 | 16 | type ssmProvider struct { 17 | Client SSMClient 18 | Name string 19 | } 20 | 21 | // SSMProviderOpt represents a configuration option for the AWS SSM Buildkite token provider. 22 | type SSMProviderOpt func(prov *ssmProvider) error 23 | 24 | // NewEnvironment constructs a Buildkite API token provider backed by AWS Systems Manager parameter store. 25 | func NewSSM(client SSMClient, name string, opts ...SSMProviderOpt) (Provider, error) { 26 | provider := &ssmProvider{ 27 | Client: client, 28 | Name: name, 29 | } 30 | 31 | for _, opt := range opts { 32 | err := opt(provider) 33 | if err != nil { 34 | return nil, err 35 | } 36 | } 37 | 38 | return provider, nil 39 | } 40 | 41 | func (p ssmProvider) Get() (string, error) { 42 | output, err := p.Client.GetParameter(&ssm.GetParameterInput{ 43 | Name: aws.String(p.Name), 44 | WithDecryption: aws.Bool(true), 45 | }) 46 | 47 | if err != nil { 48 | return "", fmt.Errorf("failed to retrieve Buildkite token (%s) from AWS SSM: %v", p.Name, err) 49 | } 50 | 51 | return *output.Parameter.Value, nil 52 | } 53 | -------------------------------------------------------------------------------- /token/mock/ssm_client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ssm.go 3 | 4 | // Package mock is a generated GoMock package. 5 | package mock 6 | 7 | import ( 8 | ssm "github.com/aws/aws-sdk-go/service/ssm" 9 | gomock "github.com/golang/mock/gomock" 10 | reflect "reflect" 11 | ) 12 | 13 | // SSMClient is a mock of SSMClient interface 14 | type SSMClient struct { 15 | ctrl *gomock.Controller 16 | recorder *SSMClientMockRecorder 17 | } 18 | 19 | // SSMClientMockRecorder is the mock recorder for SSMClient 20 | type SSMClientMockRecorder struct { 21 | mock *SSMClient 22 | } 23 | 24 | // NewSSMClient creates a new mock instance 25 | func NewSSMClient(ctrl *gomock.Controller) *SSMClient { 26 | mock := &SSMClient{ctrl: ctrl} 27 | mock.recorder = &SSMClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *SSMClient) EXPECT() *SSMClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // GetParameter mocks base method 37 | func (m *SSMClient) GetParameter(arg0 *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "GetParameter", arg0) 40 | ret0, _ := ret[0].(*ssm.GetParameterOutput) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // GetParameter indicates an expected call of GetParameter 46 | func (mr *SSMClientMockRecorder) GetParameter(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetParameter", reflect.TypeOf((*SSMClient)(nil).GetParameter), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: ":test_tube: Test" 3 | key: test 4 | command: go test -v -race ./... 5 | plugins: 6 | - docker#v5.9.0: 7 | image: golang:1.21 8 | 9 | - group: ":hammer_and_wrench: Binary builds" 10 | steps: 11 | - name: ":{{matrix.os}}: Build {{matrix.os}} {{matrix.arch}} binary" 12 | command: .buildkite/steps/build-binary.sh {{matrix.os}} {{matrix.arch}} 13 | key: build-binary 14 | depends_on: 15 | - test 16 | plugins: 17 | - docker#v5.9.0: 18 | image: golang:1.21 19 | mount-buildkite-agent: true 20 | matrix: 21 | setup: 22 | os: 23 | - darwin 24 | - linux 25 | - windows 26 | arch: 27 | - amd64 28 | - arm64 29 | 30 | - name: ":lambda: Build Lambda" 31 | key: build-lambda 32 | depends_on: 33 | - test 34 | command: .buildkite/steps/build-lambda.sh 35 | 36 | - name: ":s3: Upload to S3" 37 | key: upload-to-s3 38 | depends_on: 39 | - test 40 | - build-lambda 41 | command: ".buildkite/steps/upload-to-s3.sh" 42 | branches: master 43 | agents: 44 | queue: "elastic-runners" 45 | concurrency: 1 46 | concurrency_group: "release_buildkite_metrics_s3" 47 | plugins: 48 | - aws-assume-role-with-web-identity: 49 | role-arn: arn:aws:iam::032379705303:role/pipeline-buildkite-buildkite-agent-metrics 50 | 51 | - name: ":pipeline:" 52 | key: upload-release-steps 53 | depends_on: 54 | - build-binary 55 | - upload-to-s3 56 | command: .buildkite/steps/upload-release-steps.sh 57 | -------------------------------------------------------------------------------- /token/mock/secretsmanager_client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: secretsmanager.go 3 | 4 | // Package mock is a generated GoMock package. 5 | package mock 6 | 7 | import ( 8 | secretsmanager "github.com/aws/aws-sdk-go/service/secretsmanager" 9 | gomock "github.com/golang/mock/gomock" 10 | reflect "reflect" 11 | ) 12 | 13 | // SecretsManagerClient is a mock of SecretsManagerClient interface 14 | type SecretsManagerClient struct { 15 | ctrl *gomock.Controller 16 | recorder *SecretsManagerClientMockRecorder 17 | } 18 | 19 | // SecretsManagerClientMockRecorder is the mock recorder for SecretsManagerClient 20 | type SecretsManagerClientMockRecorder struct { 21 | mock *SecretsManagerClient 22 | } 23 | 24 | // NewSecretsManagerClient creates a new mock instance 25 | func NewSecretsManagerClient(ctrl *gomock.Controller) *SecretsManagerClient { 26 | mock := &SecretsManagerClient{ctrl: ctrl} 27 | mock.recorder = &SecretsManagerClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *SecretsManagerClient) EXPECT() *SecretsManagerClientMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // GetSecretValue mocks base method 37 | func (m *SecretsManagerClient) GetSecretValue(arg0 *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "GetSecretValue", arg0) 40 | ret0, _ := ret[0].(*secretsmanager.GetSecretValueOutput) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // GetSecretValue indicates an expected call of GetSecretValue 46 | func (mr *SecretsManagerClientMockRecorder) GetSecretValue(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecretValue", reflect.TypeOf((*SecretsManagerClient)(nil).GetSecretValue), arg0) 49 | } 50 | -------------------------------------------------------------------------------- /token/ssm_test.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/ssm" 9 | "github.com/buildkite/buildkite-agent-metrics/v5/token/mock" 10 | "github.com/golang/mock/gomock" 11 | ) 12 | 13 | //go:generate mockgen -source ssm.go -mock_names SSMClient=SSMClient -package mock -destination mock/ssm_client.go 14 | 15 | const ( 16 | ssmTestParameterName = "test-param" 17 | ssmTestParameterValue = "some-value" 18 | ) 19 | 20 | func TestSSMProvider_New_WithErroringOpt(t *testing.T) { 21 | expectedErr := fmt.Errorf("some-error") 22 | 23 | errFunc := func(provider *ssmProvider) error { 24 | return expectedErr 25 | } 26 | 27 | _, err := NewSSM(nil, ssmTestParameterName, errFunc) 28 | 29 | if err == nil { 30 | t.Fatalf("expected error to be '%s' but found 'nil'", expectedErr.Error()) 31 | } 32 | 33 | if err != expectedErr { 34 | t.Fatalf("expected error to be '%s' but found '%s'", expectedErr.Error(), err.Error()) 35 | } 36 | } 37 | 38 | func TestSSMProvider_Get(t *testing.T) { 39 | req := ssm.GetParameterInput{ 40 | Name: aws.String(ssmTestParameterName), 41 | WithDecryption: aws.Bool(true), 42 | } 43 | 44 | res := ssm.GetParameterOutput{ 45 | Parameter: &ssm.Parameter{ 46 | Value: aws.String(ssmTestParameterValue), 47 | }, 48 | } 49 | 50 | ctrl := gomock.NewController(t) 51 | defer ctrl.Finish() 52 | 53 | client := mock.NewSSMClient(ctrl) 54 | client.EXPECT().GetParameter(gomock.Eq(&req)).Return(&res, nil) 55 | 56 | provider, err := NewSSM(client, ssmTestParameterName) 57 | if err != nil { 58 | t.Fatalf("failed to create SSMProvider: %v", err) 59 | } 60 | 61 | token, err := provider.Get() 62 | if err != nil { 63 | t.Fatalf("failed to call 'Get()' on SSMProvider: %v", err) 64 | } 65 | 66 | if token != ssmTestParameterValue { 67 | t.Fatalf("expected token to be '%s' but found '%s'", ssmTestParameterValue, token) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/buildkite/buildkite-agent-metrics/v5 2 | 3 | go 1.20 4 | 5 | require ( 6 | cloud.google.com/go/monitoring v1.17.0 7 | github.com/DataDog/datadog-go v4.8.3+incompatible 8 | github.com/aws/aws-lambda-go v1.42.0 9 | github.com/aws/aws-sdk-go v1.48.16 10 | github.com/golang/mock v1.6.0 11 | github.com/google/go-cmp v0.6.0 12 | github.com/newrelic/go-agent v3.24.1+incompatible 13 | github.com/prometheus/client_golang v1.17.0 14 | github.com/prometheus/client_model v0.5.0 15 | google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b 16 | google.golang.org/protobuf v1.31.0 17 | ) 18 | 19 | require ( 20 | cloud.google.com/go/compute v1.23.1 // indirect 21 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 22 | github.com/Microsoft/go-winio v0.6.0 // indirect 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 25 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 26 | github.com/golang/protobuf v1.5.3 // indirect 27 | github.com/google/s2a-go v0.1.7 // indirect 28 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 29 | github.com/googleapis/gax-go/v2 v2.12.0 // indirect 30 | github.com/jmespath/go-jmespath v0.4.0 // indirect 31 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 32 | github.com/prometheus/common v0.44.0 // indirect 33 | github.com/prometheus/procfs v0.11.1 // indirect 34 | go.opencensus.io v0.24.0 // indirect 35 | golang.org/x/crypto v0.17.0 // indirect 36 | golang.org/x/mod v0.9.0 // indirect 37 | golang.org/x/net v0.17.0 // indirect 38 | golang.org/x/oauth2 v0.13.0 // indirect 39 | golang.org/x/sync v0.4.0 // indirect 40 | golang.org/x/sys v0.15.0 // indirect 41 | golang.org/x/text v0.14.0 // indirect 42 | golang.org/x/tools v0.7.0 // indirect 43 | google.golang.org/api v0.149.0 // indirect 44 | google.golang.org/appengine v1.6.7 // indirect 45 | google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect 46 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect 47 | google.golang.org/grpc v1.59.0 // indirect 48 | ) 49 | -------------------------------------------------------------------------------- /backend/newrelic_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestToCustomEvent(t *testing.T) { 10 | tcs := []struct { 11 | desc string 12 | clusterName string 13 | queueName string // input queue 14 | metrics map[string]int // input metrics 15 | want map[string]any // output shaped data 16 | }{ 17 | { 18 | desc: "partial data", 19 | clusterName: "test-cluster", 20 | queueName: "partial-data-test", 21 | metrics: map[string]int{ 22 | "BusyAgentCount": 0, 23 | "BusyAgentPercentage": 0, 24 | "IdleAgentCount": 3, 25 | "TotalAgentCount": 3, 26 | "RunningJobsCount": 0, 27 | }, 28 | want: map[string]any{ 29 | "Cluster": "test-cluster", 30 | "Queue": "partial-data-test", 31 | "BusyAgentCount": 0, 32 | "BusyAgentPercentage": 0, 33 | "IdleAgentCount": 3, 34 | "TotalAgentCount": 3, 35 | "RunningJobsCount": 0, 36 | }, 37 | }, 38 | { 39 | desc: "complete data", 40 | clusterName: "test-cluster", 41 | queueName: "complete-data-test", 42 | metrics: map[string]int{ 43 | "BusyAgentCount": 2, 44 | "BusyAgentPercentage": 20, 45 | "IdleAgentCount": 8, 46 | "TotalAgentCount": 10, 47 | "RunningJobsCount": 2, 48 | "ScheduledJobsCount": 0, 49 | "WaitingJobsCount": 0, 50 | }, 51 | want: map[string]any{ 52 | "Cluster": "test-cluster", 53 | "Queue": "complete-data-test", 54 | "BusyAgentCount": 2, 55 | "BusyAgentPercentage": 20, 56 | "IdleAgentCount": 8, 57 | "TotalAgentCount": 10, 58 | "RunningJobsCount": 2, 59 | "ScheduledJobsCount": 0, 60 | "WaitingJobsCount": 0, 61 | }, 62 | }, 63 | } 64 | 65 | for _, tc := range tcs { 66 | t.Run(tc.desc, func(t *testing.T) { 67 | got := toCustomEvent(tc.clusterName, tc.queueName, tc.metrics) 68 | if diff := cmp.Diff(got, tc.want); diff != "" { 69 | t.Errorf("toCustomEvent output diff (-got +want):\n%s", diff) 70 | } 71 | }) 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /.buildkite/steps/release-github.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eufo pipefail 4 | 5 | source .buildkite/lib/release_dry_run.sh 6 | 7 | if [[ "$GITHUB_RELEASE_ACCESS_TOKEN" == "" ]]; then 8 | echo "Error: Missing \$GITHUB_RELEASE_ACCESS_TOKEN" >&2 9 | exit 1 10 | fi 11 | 12 | echo --- Fetching tags 13 | git fetch --prune --force origin "+refs/tags/*:refs/tags/*" 14 | 15 | echo --- Downloading binaries 16 | rm -rf dist 17 | mkdir -p dist 18 | buildkite-agent artifact download handler.zip ./dist 19 | buildkite-agent artifact download "buildkite-agent-metrics-*" ./dist 20 | 21 | echo --- Checking tags 22 | version=$(awk -F\" '/const Version/ {print $2}' version/version.go) 23 | tag="v${version#v}" 24 | 25 | if [[ "${RELEASE_DRY_RUN:-false}" != true && $tag != "$BUILDKITE_TAG" ]]; then 26 | echo "Error: version.go has not been updated to ${BUILDKITE_TAG#v}" 27 | exit 1 28 | fi 29 | 30 | last_tag=$(git describe --tags --abbrev=0 --exclude "$tag") 31 | 32 | # escape . so we can use in regex 33 | escaped_tag="${tag//\./\\.}" 34 | escaped_last_tag="${last_tag//\./\\.}" 35 | 36 | if [[ "${RELEASE_DRY_RUN:-false}" != true ]] && ! grep "^## \[$escaped_tag\]" CHANGELOG.md; then 37 | echo "Error: CHANGELOG.md has not been updated for $tag" >&2 38 | exit 1 39 | fi 40 | 41 | echo --- Creating checksum file 42 | pushd dist &>/dev/null 43 | set +f 44 | sha256sum -- * > sha256sums.txt 45 | set -f 46 | popd &>/dev/null 47 | 48 | echo --- The following notes will accompany the release: 49 | # The sed commands below: 50 | # Find lines between headers of the changelogs (inclusive) 51 | # Delete the lines included from the headers 52 | # The command substituion will then delete the empty lines from the end 53 | notes=$(sed -n "/^## \[${escaped_tag}\]/,/^## \[${escaped_last_tag}\]/p" CHANGELOG.md | sed '$d') 54 | 55 | echo --- The following notes will accompany the release: 56 | echo "$notes" 57 | 58 | echo --- :github: Publishing draft release 59 | # TODO: add the following flag once github-cli in alpine repo hits v2.27+ 60 | # --verify-tag \ 61 | set +f 62 | GITHUB_TOKEN="$GITHUB_RELEASE_ACCESS_TOKEN" \ 63 | release_dry_run gh release create \ 64 | --draft \ 65 | --notes "$notes" \ 66 | "$tag" \ 67 | dist/* 68 | set -f 69 | -------------------------------------------------------------------------------- /backend/newrelic.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 8 | newrelic "github.com/newrelic/go-agent" 9 | ) 10 | 11 | const newRelicConnectionTimeout = time.Second * 30 12 | 13 | // NewRelicBackend sends metrics to New Relic Insights 14 | type NewRelicBackend struct { 15 | client newrelic.Application 16 | } 17 | 18 | // NewNewRelicBackend returns a backend for New Relic 19 | // Where appName is your desired application name in New Relic 20 | // 21 | // and licenseKey is your New Relic license key 22 | func NewNewRelicBackend(appName string, licenseKey string) (*NewRelicBackend, error) { 23 | config := newrelic.NewConfig(appName, licenseKey) 24 | app, err := newrelic.NewApplication(config) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | // Waiting for connection is essential or no data will make it during short-lived execution (e.g. Lambda) 30 | err = app.WaitForConnection(newRelicConnectionTimeout) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &NewRelicBackend{ 36 | client: app, 37 | }, nil 38 | } 39 | 40 | // Collect metrics 41 | func (nr *NewRelicBackend) Collect(r *collector.Result) error { 42 | // Publish event for each queue 43 | for queue, metrics := range r.Queues { 44 | data := toCustomEvent(r.Cluster, queue, metrics) 45 | err := nr.client.RecordCustomEvent("BuildkiteQueueMetrics", data) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | nr.client.RecordCustomEvent("queue_agent_metrics", data) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // toCustomEvent converts a map of metrics to a valid New Relic event body 57 | func toCustomEvent(clusterName, queueName string, queueMetrics map[string]int) map[string]any { 58 | eventData := map[string]any{ 59 | "Queue": queueName, 60 | } 61 | 62 | if clusterName != "" { 63 | eventData["Cluster"] = clusterName 64 | } 65 | 66 | for k, v := range queueMetrics { 67 | eventData[k] = v 68 | } 69 | 70 | return eventData 71 | } 72 | 73 | // Close by shutting down NR client 74 | func (nr *NewRelicBackend) Close() error { 75 | nr.client.Shutdown(newRelicConnectionTimeout) 76 | log.Printf("Disposed New Relic client") 77 | 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /backend/statsd.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/DataDog/datadog-go/statsd" 7 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 8 | ) 9 | 10 | // StatsD sends metrics to StatsD (Datadog spec) 11 | type StatsD struct { 12 | client *statsd.Client 13 | tagsSupported bool 14 | } 15 | 16 | func NewStatsDBackend(host string, tagsSupported bool) (*StatsD, error) { 17 | client, err := statsd.NewBuffered(host, 100) 18 | if err != nil { 19 | return nil, err 20 | } 21 | // prefix every metric with the app name 22 | client.Namespace = "buildkite." 23 | return &StatsD{ 24 | client: client, 25 | tagsSupported: tagsSupported, 26 | }, nil 27 | } 28 | 29 | func (cb *StatsD) Collect(r *collector.Result) error { 30 | collectFunc := cb.collectWithoutTags 31 | if cb.tagsSupported { 32 | collectFunc = cb.collectWithTags 33 | } 34 | 35 | if err := collectFunc(r); err != nil { 36 | return err 37 | } 38 | 39 | return cb.client.Flush() 40 | } 41 | 42 | // collectWithTags tags clusters and queues. 43 | func (cb *StatsD) collectWithTags(r *collector.Result) error { 44 | commonTags := make([]string, 0, 2) 45 | prefix := "" 46 | if r.Cluster != "" { 47 | commonTags = append(commonTags, "cluster:"+r.Cluster) 48 | prefix = "clusters." 49 | } 50 | 51 | for name, value := range r.Totals { 52 | if err := cb.client.Gauge(prefix+name, float64(value), commonTags, 1.0); err != nil { 53 | return err 54 | } 55 | } 56 | 57 | for queue, counts := range r.Queues { 58 | tags := append(commonTags, "queue:"+queue) 59 | 60 | for name, value := range counts { 61 | if err := cb.client.Gauge(prefix+"queues."+name, float64(value), tags, 1.0); err != nil { 62 | return err 63 | } 64 | } 65 | } 66 | 67 | return cb.client.Flush() 68 | } 69 | 70 | // collectWithoutTags embeds clusters and queues into metric names. 71 | func (cb *StatsD) collectWithoutTags(r *collector.Result) error { 72 | prefix := "" 73 | if r.Cluster != "" { 74 | prefix = fmt.Sprintf("clusters.%s.", r.Cluster) 75 | } 76 | 77 | for name, value := range r.Totals { 78 | if err := cb.client.Gauge(prefix+name, float64(value), nil, 1.0); err != nil { 79 | return err 80 | } 81 | } 82 | 83 | for queue, counts := range r.Queues { 84 | prefix := fmt.Sprintf("queues.%s.", queue) 85 | if r.Cluster != "" { 86 | prefix = fmt.Sprintf("clusters.%s.queues.%s.", r.Cluster, queue) 87 | } 88 | 89 | for name, value := range counts { 90 | if err := cb.client.Gauge(prefix+name, float64(value), nil, 1.0); err != nil { 91 | return err 92 | } 93 | } 94 | } 95 | 96 | return cb.client.Flush() 97 | } 98 | -------------------------------------------------------------------------------- /backend/prometheus.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 11 | 12 | "github.com/prometheus/client_golang/prometheus" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | ) 15 | 16 | var ( 17 | camel = regexp.MustCompile("(^[^A-Z0-9]*|[A-Z0-9]*)([A-Z0-9][^A-Z]+|$)") 18 | ) 19 | 20 | type Prometheus struct { 21 | totals map[string]*prometheus.GaugeVec 22 | queues map[string]*prometheus.GaugeVec 23 | pipelines map[string]*prometheus.GaugeVec 24 | } 25 | 26 | func NewPrometheusBackend() *Prometheus { 27 | return &Prometheus{ 28 | totals: make(map[string]*prometheus.GaugeVec), 29 | queues: make(map[string]*prometheus.GaugeVec), 30 | pipelines: make(map[string]*prometheus.GaugeVec), 31 | } 32 | } 33 | 34 | // Serve runs a Prometheus metrics HTTP server. 35 | func (p *Prometheus) Serve(path, addr string) { 36 | m := http.NewServeMux() 37 | m.Handle(path, promhttp.Handler()) 38 | log.Fatal(http.ListenAndServe(addr, m)) 39 | } 40 | 41 | func (p *Prometheus) Collect(r *collector.Result) error { 42 | // Clear the gauges to prevent stale values from persisting forever. 43 | for _, gauge := range p.queues { 44 | gauge.Reset() 45 | } 46 | 47 | for name, value := range r.Totals { 48 | labelNames := []string{} 49 | if r.Cluster != "" { 50 | labelNames = append(labelNames, "cluster") 51 | } 52 | gauge, ok := p.totals[name] 53 | if !ok { 54 | gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 55 | Name: fmt.Sprintf("buildkite_total_%s", camelToUnderscore(name)), 56 | Help: fmt.Sprintf("Buildkite Total: %s", name), 57 | }, labelNames) 58 | prometheus.MustRegister(gauge) 59 | p.totals[name] = gauge 60 | } 61 | 62 | if r.Cluster != "" { 63 | gauge.WithLabelValues(r.Cluster).Set(float64(value)) 64 | } else { 65 | gauge.WithLabelValues().Set(float64(value)) 66 | } 67 | } 68 | 69 | for queue, counts := range r.Queues { 70 | for name, value := range counts { 71 | gauge, ok := p.queues[name] 72 | if !ok { 73 | labelNames := []string{"queue"} 74 | if r.Cluster != "" { 75 | labelNames = append(labelNames, "cluster") 76 | } 77 | gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 78 | Name: fmt.Sprintf("buildkite_queues_%s", camelToUnderscore(name)), 79 | Help: fmt.Sprintf("Buildkite Queues: %s", name), 80 | }, labelNames) 81 | prometheus.MustRegister(gauge) 82 | p.queues[name] = gauge 83 | } 84 | if r.Cluster != "" { 85 | gauge.WithLabelValues(queue, r.Cluster).Set(float64(value)) 86 | } else { 87 | gauge.WithLabelValues(queue).Set(float64(value)) 88 | } 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func camelToUnderscore(s string) string { 96 | var a []string 97 | for _, sub := range camel.FindAllStringSubmatch(s, -1) { 98 | if sub[1] != "" { 99 | a = append(a, sub[1]) 100 | } 101 | if sub[2] != "" { 102 | a = append(a, sub[2]) 103 | } 104 | } 105 | return strings.ToLower(strings.Join(a, "_")) 106 | } 107 | -------------------------------------------------------------------------------- /token/secretsmanager.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/secretsmanager" 10 | ) 11 | 12 | // SecretsManagerOpt represents a configuration option for the AWS SecretsManager Buildkite token provider. 13 | type SecretsManagerOpt func(opts *secretsManagerProvider) error 14 | 15 | // SecretsManagerClient represents the minimal interactions required to retrieve a Buildkite API token from 16 | // AWS Secrets Manager. 17 | type SecretsManagerClient interface { 18 | GetSecretValue(*secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) 19 | } 20 | 21 | type secretsManagerProvider struct { 22 | Client SecretsManagerClient 23 | SecretID string 24 | JSONKey string 25 | } 26 | 27 | // WithSecretsManagerJSONSecret instructs SecretsManager Buidlkite token provider that the token is stored within a JSON 28 | // payload. The key parameter specifies the JSON field holding the secret value within the JSON blob. 29 | // 30 | // This configuration option works for both AWS supported secret formats (SecretString and SecretBinary). However, for 31 | // the later case, the binary payload must be a valid JSON document containing the 'key' field. 32 | func WithSecretsManagerJSONSecret(key string) SecretsManagerOpt { 33 | return func(provider *secretsManagerProvider) error { 34 | provider.JSONKey = key 35 | return nil 36 | } 37 | } 38 | 39 | // NewSecretsManager constructs a Buildkite API token provider backed by AWS Secrets Manager. 40 | func NewSecretsManager(client SecretsManagerClient, secretID string, opts ...SecretsManagerOpt) (Provider, error) { 41 | provider := &secretsManagerProvider{ 42 | Client: client, 43 | SecretID: secretID, 44 | } 45 | 46 | for _, opt := range opts { 47 | err := opt(provider) 48 | if err != nil { 49 | return nil, err 50 | } 51 | } 52 | 53 | return provider, nil 54 | } 55 | 56 | func (p secretsManagerProvider) Get() (string, error) { 57 | res, err := p.Client.GetSecretValue(&secretsmanager.GetSecretValueInput{ 58 | SecretId: aws.String(p.SecretID), 59 | }) 60 | 61 | if err != nil { 62 | return "", fmt.Errorf("failed to retrieve secret '%s' from SecretsManager: %v", p.SecretID, err) 63 | } 64 | 65 | secret, err := p.parseResponse(res) 66 | if err != nil { 67 | return "", fmt.Errorf("failed to parse SecretsManager's response for '%s': %v", p.SecretID, err) 68 | } 69 | 70 | return secret, nil 71 | } 72 | 73 | func (p secretsManagerProvider) parseResponse(res *secretsmanager.GetSecretValueOutput) (string, error) { 74 | var err error 75 | var secretBytes []byte 76 | 77 | if res.SecretString != nil { 78 | secretBytes = []byte(*res.SecretString) 79 | } else { 80 | secretBytes, err = decodeBase64(res.SecretBinary) 81 | if err != nil { 82 | return "", err 83 | } 84 | } 85 | 86 | if p.JSONKey != "" { 87 | secret, err := extractStringKeyFromJSON(secretBytes, p.JSONKey) 88 | if err != nil { 89 | return "", err 90 | } 91 | return secret, nil 92 | } 93 | 94 | return string(secretBytes), nil 95 | } 96 | 97 | func extractStringKeyFromJSON(data []byte, key string) (string, error) { 98 | contents := map[string]interface{}{} 99 | err := json.Unmarshal(data, &contents) 100 | if err != nil { 101 | return "", err 102 | } 103 | 104 | // Checks whether the provided data is a valid JSON, contains the requested key and its corresponding value 105 | // is a string. 106 | if secretValue, ok := contents[key].(string); ok { 107 | return secretValue, nil 108 | } else { 109 | return "", fmt.Errorf("key '%s' doesn't exist or isn't a string value", key) 110 | } 111 | } 112 | 113 | func decodeBase64(data []byte) ([]byte, error) { 114 | decodedBytes := make([]byte, base64.StdEncoding.DecodedLen(len(data))) 115 | size, err := base64.StdEncoding.Decode(decodedBytes, data) 116 | if err != nil { 117 | return nil, err 118 | } 119 | return decodedBytes[:size], nil 120 | } 121 | -------------------------------------------------------------------------------- /backend/cloudwatch.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strings" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/cloudwatch" 11 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 12 | ) 13 | 14 | // CloudWatchDimension is a dimension to add to metrics 15 | type CloudWatchDimension struct { 16 | Key string 17 | Value string 18 | } 19 | 20 | func ParseCloudWatchDimensions(ds string) ([]CloudWatchDimension, error) { 21 | dimensions := []CloudWatchDimension{} 22 | 23 | if strings.TrimSpace(ds) == "" { 24 | return dimensions, nil 25 | } 26 | 27 | for _, dimension := range strings.Split(strings.TrimSpace(ds), ",") { 28 | parts := strings.SplitN(strings.TrimSpace(dimension), "=", 2) 29 | if len(parts) != 2 { 30 | return nil, fmt.Errorf("Failed to parse dimension of %q", dimension) 31 | } 32 | dimensions = append(dimensions, CloudWatchDimension{ 33 | Key: parts[0], 34 | Value: parts[1], 35 | }) 36 | } 37 | 38 | return dimensions, nil 39 | } 40 | 41 | // CloudWatchBackend sends metrics to AWS CloudWatch 42 | type CloudWatchBackend struct { 43 | region string 44 | dimensions []CloudWatchDimension 45 | } 46 | 47 | // NewCloudWatchBackend returns a new CloudWatchBackend with optional dimensions 48 | func NewCloudWatchBackend(region string, dimensions []CloudWatchDimension) *CloudWatchBackend { 49 | return &CloudWatchBackend{ 50 | region: region, 51 | dimensions: dimensions, 52 | } 53 | } 54 | 55 | func (cb *CloudWatchBackend) Collect(r *collector.Result) error { 56 | sess, err := session.NewSession(&aws.Config{ 57 | Region: aws.String(cb.region), 58 | }) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | svc := cloudwatch.New(sess) 64 | metrics := []*cloudwatch.MetricDatum{} 65 | 66 | // Set the baseline org dimension 67 | dimensions := []*cloudwatch.Dimension{ 68 | { 69 | Name: aws.String("Org"), 70 | Value: aws.String(r.Org), 71 | }, 72 | } 73 | 74 | // Add cluster dimension if a cluster token was used 75 | if r.Cluster != "" { 76 | dimensions = append(dimensions, &cloudwatch.Dimension{ 77 | Name: aws.String("Cluster"), 78 | Value: aws.String(r.Cluster), 79 | }) 80 | } 81 | 82 | // Add custom dimension if provided 83 | for _, d := range cb.dimensions { 84 | log.Printf("Using custom Cloudwatch dimension of [ %s = %s ]", d.Key, d.Value) 85 | 86 | dimensions = append(dimensions, &cloudwatch.Dimension{ 87 | Name: aws.String(d.Key), Value: aws.String(d.Value), 88 | }) 89 | } 90 | 91 | // Add total metrics 92 | metrics = append(metrics, cloudwatchMetrics(r.Totals, nil)...) 93 | 94 | for name, c := range r.Queues { 95 | queueDimensions := dimensions 96 | 97 | // Add an queue dimension 98 | queueDimensions = append(queueDimensions, 99 | &cloudwatch.Dimension{Name: aws.String("Queue"), Value: aws.String(name)}, 100 | ) 101 | 102 | // Add per-queue metrics 103 | metrics = append(metrics, cloudwatchMetrics(c, queueDimensions)...) 104 | } 105 | 106 | log.Printf("Extracted %d cloudwatch metrics from results", len(metrics)) 107 | 108 | // Chunk into batches of 10 metrics 109 | for _, chunk := range chunkCloudwatchMetrics(10, metrics) { 110 | log.Printf("Submitting chunk of %d metrics to Cloudwatch", len(chunk)) 111 | _, err := svc.PutMetricData(&cloudwatch.PutMetricDataInput{ 112 | MetricData: chunk, 113 | Namespace: aws.String("Buildkite"), 114 | }) 115 | if err != nil { 116 | return err 117 | } 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func cloudwatchMetrics(counts map[string]int, dimensions []*cloudwatch.Dimension) []*cloudwatch.MetricDatum { 124 | m := []*cloudwatch.MetricDatum{} 125 | 126 | for k, v := range counts { 127 | m = append(m, &cloudwatch.MetricDatum{ 128 | MetricName: aws.String(k), 129 | Dimensions: dimensions, 130 | Value: aws.Float64(float64(v)), 131 | Unit: aws.String("Count"), 132 | }) 133 | } 134 | 135 | return m 136 | } 137 | 138 | func chunkCloudwatchMetrics(size int, data []*cloudwatch.MetricDatum) [][]*cloudwatch.MetricDatum { 139 | var chunks = [][]*cloudwatch.MetricDatum{} 140 | for i := 0; i < len(data); i += size { 141 | end := i + size 142 | if end > len(data) { 143 | end = len(data) 144 | } 145 | chunks = append(chunks, data[i:end]) 146 | } 147 | return chunks 148 | } 149 | -------------------------------------------------------------------------------- /backend/stackdriver_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" 8 | "github.com/google/go-cmp/cmp" 9 | "google.golang.org/genproto/googleapis/api/label" 10 | "google.golang.org/genproto/googleapis/api/metric" 11 | "google.golang.org/protobuf/testing/protocmp" 12 | "google.golang.org/protobuf/types/known/timestamppb" 13 | ) 14 | 15 | func Test_createCustomMetricRequest(t *testing.T) { 16 | type args struct { 17 | projectID string 18 | metricType string 19 | } 20 | tests := []struct { 21 | name string 22 | args args 23 | want *monitoringpb.CreateMetricDescriptorRequest 24 | }{ 25 | { 26 | name: "HappyPath", 27 | args: args{ 28 | projectID: "test-project-id", 29 | metricType: "test-metric-type", 30 | }, 31 | want: &monitoringpb.CreateMetricDescriptorRequest{ 32 | Name: "projects/test-project-id", 33 | MetricDescriptor: &metric.MetricDescriptor{ 34 | Name: "test-metric-type", 35 | Type: "test-metric-type", 36 | MetricKind: metric.MetricDescriptor_GAUGE, 37 | ValueType: metric.MetricDescriptor_INT64, 38 | Description: "Buildkite metric: [test-metric-type]", 39 | DisplayName: "test-metric-type", 40 | Labels: []*label.LabelDescriptor{ 41 | { 42 | Key: clusterLabelKey, 43 | ValueType: label.LabelDescriptor_STRING, 44 | Description: clusterDescription, 45 | }, 46 | { 47 | Key: queueLabelKey, 48 | ValueType: label.LabelDescriptor_STRING, 49 | Description: queueDescription, 50 | }, 51 | }, 52 | }, 53 | }, 54 | }, 55 | } 56 | for _, tt := range tests { 57 | t.Run(tt.name, func(t *testing.T) { 58 | got := createCustomMetricRequest(&tt.args.projectID, &tt.args.metricType) 59 | if diff := cmp.Diff(got, tt.want, protocmp.Transform()); diff != "" { 60 | t.Errorf("createCustomMetricRequest diff (-got +want):\n%s", diff) 61 | } 62 | }) 63 | } 64 | } 65 | 66 | func Test_createTimeSeriesValueRequest(t *testing.T) { 67 | now := ×tamppb.Timestamp{ 68 | Seconds: time.Now().Unix(), 69 | } 70 | 71 | type args struct { 72 | projectID string 73 | metricType string 74 | cluster string 75 | queue string 76 | value int 77 | time *timestamppb.Timestamp 78 | } 79 | tests := []struct { 80 | name string 81 | args args 82 | want *monitoringpb.CreateTimeSeriesRequest 83 | }{ 84 | { 85 | name: "Happy Path", 86 | args: args{ 87 | projectID: "test-project-id", 88 | metricType: "test-metric-type", 89 | cluster: "test-cluster", 90 | queue: "test-queue", 91 | value: 13, 92 | time: now, 93 | }, 94 | want: &monitoringpb.CreateTimeSeriesRequest{ 95 | Name: "projects/test-project-id", 96 | TimeSeries: []*monitoringpb.TimeSeries{{ 97 | Metric: &metric.Metric{ 98 | Type: "test-metric-type", 99 | Labels: map[string]string{ 100 | clusterLabelKey: "test-cluster", 101 | queueLabelKey: "test-queue", 102 | }, 103 | }, 104 | Points: []*monitoringpb.Point{{ 105 | Interval: &monitoringpb.TimeInterval{ 106 | StartTime: now, 107 | EndTime: now, 108 | }, 109 | Value: &monitoringpb.TypedValue{ 110 | Value: &monitoringpb.TypedValue_Int64Value{ 111 | Int64Value: int64(13), 112 | }, 113 | }, 114 | }}, 115 | }}, 116 | }, 117 | }, 118 | { 119 | name: "Bad Queue Name", 120 | args: args{ 121 | projectID: "test-project-id", 122 | metricType: "test-metric-type", 123 | queue: "${BUILDKITE_QUEUE:-default}", 124 | value: 13, 125 | time: now, 126 | }, 127 | want: &monitoringpb.CreateTimeSeriesRequest{ 128 | Name: "projects/test-project-id", 129 | TimeSeries: []*monitoringpb.TimeSeries{{ 130 | Metric: &metric.Metric{ 131 | Type: "test-metric-type", 132 | Labels: map[string]string{ 133 | clusterLabelKey: "", 134 | queueLabelKey: "${BUILDKITE_QUEUE:-default}", 135 | }, 136 | }, 137 | Points: []*monitoringpb.Point{{ 138 | Interval: &monitoringpb.TimeInterval{ 139 | StartTime: now, 140 | EndTime: now, 141 | }, 142 | Value: &monitoringpb.TypedValue{ 143 | Value: &monitoringpb.TypedValue_Int64Value{ 144 | Int64Value: int64(13), 145 | }, 146 | }, 147 | }}, 148 | }}, 149 | }, 150 | }, 151 | } 152 | for _, tt := range tests { 153 | t.Run(tt.name, func(t *testing.T) { 154 | got := createTimeSeriesValueRequest(&tt.args.projectID, &tt.args.metricType, tt.args.cluster, tt.args.queue, tt.args.value, tt.args.time) 155 | if diff := cmp.Diff(got, tt.want, protocmp.Transform()); diff != "" { 156 | t.Errorf("createTimeSeriesValueRequest diff (-got +want):\n%s", diff) 157 | } 158 | }) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /backend/stackdriver.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "time" 9 | 10 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 11 | "google.golang.org/genproto/googleapis/api/label" 12 | "google.golang.org/protobuf/types/known/timestamppb" 13 | 14 | monitoring "cloud.google.com/go/monitoring/apiv3/v2" 15 | "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" 16 | "google.golang.org/genproto/googleapis/api/metric" 17 | ) 18 | 19 | const ( 20 | metricTypeFmt = "custom.googleapis.com/buildkite/%s/%s" 21 | clusterLabelKey = "Cluster" 22 | clusterDescription = "Name of the Buildkite Cluster, or empty" 23 | queueLabelKey = "Queue" 24 | queueDescription = "Name of the Queue" 25 | totalMetricsQueue = "Total" 26 | ) 27 | 28 | // Stackdriver does not allow dashes in metric names 29 | var dashReplacer = strings.NewReplacer("-", "_") 30 | 31 | // StackDriverBackend sends metrics to GCP Stackdriver 32 | type StackDriverBackend struct { 33 | projectID string 34 | client *monitoring.MetricClient 35 | metricTypes map[string]string 36 | } 37 | 38 | // NewStackDriverBackend returns a new StackDriverBackend for the specified project 39 | func NewStackDriverBackend(gcpProjectID string) (*StackDriverBackend, error) { 40 | ctx := context.Background() 41 | c, err := monitoring.NewMetricClient(ctx) 42 | if err != nil { 43 | return nil, fmt.Errorf("[NewStackDriverBackend] could not create stackdriver client: %v", err) 44 | } 45 | 46 | return &StackDriverBackend{ 47 | projectID: gcpProjectID, 48 | client: c, 49 | metricTypes: make(map[string]string), 50 | }, nil 51 | } 52 | 53 | // Collect metrics 54 | func (sd *StackDriverBackend) Collect(r *collector.Result) error { 55 | ctx := context.Background() 56 | now := ×tamppb.Timestamp{ 57 | Seconds: time.Now().Unix(), 58 | } 59 | orgName := dashReplacer.Replace(r.Org) 60 | metricTypeFunc := func(name string) string { 61 | return fmt.Sprintf(metricTypeFmt, orgName, name) 62 | } 63 | 64 | for name, value := range r.Totals { 65 | mt, present := sd.metricTypes[name] 66 | if !present { 67 | mt = metricTypeFunc(name) 68 | metricReq := createCustomMetricRequest(&sd.projectID, &mt) 69 | _, err := sd.client.CreateMetricDescriptor(ctx, metricReq) 70 | if err != nil { 71 | retErr := fmt.Errorf("[Collect] could not create custom metric [%s]: %v", mt, err) 72 | log.Println(retErr) 73 | return retErr 74 | } 75 | log.Printf("[Collect] created custom metric [%s]", mt) 76 | sd.metricTypes[name] = mt 77 | } 78 | req := createTimeSeriesValueRequest(&sd.projectID, &mt, r.Cluster, totalMetricsQueue, value, now) 79 | err := sd.client.CreateTimeSeries(ctx, req) 80 | if err != nil { 81 | retErr := fmt.Errorf("[Collect] could not write metric [%s] value [%d], %v ", mt, value, err) 82 | log.Println(retErr) 83 | return retErr 84 | } 85 | } 86 | 87 | for queue, counts := range r.Queues { 88 | for name, value := range counts { 89 | mt := metricTypeFunc(name) 90 | req := createTimeSeriesValueRequest(&sd.projectID, &mt, r.Cluster, queue, value, now) 91 | err := sd.client.CreateTimeSeries(ctx, req) 92 | if err != nil { 93 | retErr := fmt.Errorf("[Collect] could not write metric [%s] value [%d], %v ", mt, value, err) 94 | log.Println(retErr) 95 | return retErr 96 | } 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | 103 | // createCustomMetricRequest creates a custom metric request as specified by the metric type. 104 | func createCustomMetricRequest(projectID *string, metricType *string) *monitoringpb.CreateMetricDescriptorRequest { 105 | clusterLabel := &label.LabelDescriptor{ 106 | Key: clusterLabelKey, 107 | ValueType: label.LabelDescriptor_STRING, 108 | Description: clusterDescription, 109 | } 110 | queueLabel := &label.LabelDescriptor{ 111 | Key: queueLabelKey, 112 | ValueType: label.LabelDescriptor_STRING, 113 | Description: queueDescription, 114 | } 115 | labels := []*label.LabelDescriptor{ 116 | clusterLabel, 117 | queueLabel, 118 | } 119 | md := &metric.MetricDescriptor{ 120 | Name: *metricType, 121 | Type: *metricType, 122 | MetricKind: metric.MetricDescriptor_GAUGE, 123 | ValueType: metric.MetricDescriptor_INT64, 124 | Description: fmt.Sprintf("Buildkite metric: [%s]", *metricType), 125 | DisplayName: *metricType, 126 | Labels: labels, 127 | } 128 | req := &monitoringpb.CreateMetricDescriptorRequest{ 129 | Name: "projects/" + *projectID, 130 | MetricDescriptor: md, 131 | } 132 | 133 | return req 134 | } 135 | 136 | // createTimeSeriesValueRequest creates a StackDriver value request for the specified metric 137 | func createTimeSeriesValueRequest(projectID *string, metricType *string, cluster, queue string, value int, time *timestamppb.Timestamp) *monitoringpb.CreateTimeSeriesRequest { 138 | req := &monitoringpb.CreateTimeSeriesRequest{ 139 | Name: "projects/" + *projectID, 140 | TimeSeries: []*monitoringpb.TimeSeries{{ 141 | Metric: &metric.Metric{ 142 | Type: *metricType, 143 | Labels: map[string]string{ 144 | clusterLabelKey: cluster, 145 | queueLabelKey: queue, 146 | }, 147 | }, 148 | Points: []*monitoringpb.Point{{ 149 | Interval: &monitoringpb.TimeInterval{ 150 | StartTime: time, 151 | EndTime: time, 152 | }, 153 | Value: &monitoringpb.TypedValue{ 154 | Value: &monitoringpb.TypedValue_Int64Value{ 155 | Int64Value: int64(value), 156 | }, 157 | }, 158 | }}, 159 | }}, 160 | } 161 | return req 162 | } 163 | -------------------------------------------------------------------------------- /backend/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 8 | "github.com/google/go-cmp/cmp" 9 | "github.com/prometheus/client_golang/prometheus" 10 | dto "github.com/prometheus/client_model/go" 11 | ) 12 | 13 | const ( 14 | runningBuildsCount = iota 15 | scheduledBuildsCount 16 | runningJobsCount 17 | scheduledJobsCount 18 | unfinishedJobsCount 19 | idleAgentCount 20 | busyAgentCount 21 | totalAgentCount 22 | ) 23 | 24 | func newTestResult(t *testing.T) *collector.Result { 25 | t.Helper() 26 | totals := map[string]int{ 27 | "RunningBuildsCount": runningBuildsCount, 28 | "ScheduledBuildsCount": scheduledBuildsCount, 29 | "RunningJobsCount": runningJobsCount, 30 | "ScheduledJobsCount": scheduledJobsCount, 31 | "UnfinishedJobsCount": unfinishedJobsCount, 32 | "IdleAgentCount": idleAgentCount, 33 | "BusyAgentCount": busyAgentCount, 34 | "TotalAgentCount": totalAgentCount, 35 | } 36 | 37 | res := &collector.Result{ 38 | Totals: totals, 39 | Cluster: "test_cluster", 40 | Queues: map[string]map[string]int{ 41 | "default": totals, 42 | "deploy": totals, 43 | }, 44 | } 45 | return res 46 | } 47 | 48 | // gatherMetrics runs the Prometheus gatherer, and returns the metric families 49 | // grouped by name. 50 | func gatherMetrics(t *testing.T) map[string]*dto.MetricFamily { 51 | t.Helper() 52 | 53 | oldRegisterer := prometheus.DefaultRegisterer 54 | defer func() { 55 | prometheus.DefaultRegisterer = oldRegisterer 56 | }() 57 | r := prometheus.NewRegistry() 58 | prometheus.DefaultRegisterer = r 59 | 60 | p := NewPrometheusBackend() 61 | p.Collect(newTestResult(t)) 62 | 63 | mfs, err := r.Gather() 64 | if err != nil { 65 | t.Fatalf("prometheus.Registry.Gather() = %v", err) 66 | return nil 67 | } 68 | 69 | mfsm := make(map[string]*dto.MetricFamily) 70 | for _, mf := range mfs { 71 | mfsm[*mf.Name] = mf 72 | } 73 | return mfsm 74 | } 75 | 76 | func TestCollect(t *testing.T) { 77 | metricFamilies := gatherMetrics(t) 78 | 79 | if got, want := len(metricFamilies), 16; got != want { 80 | t.Errorf("len(metricFamilies) = %d, want %d", got, want) 81 | } 82 | 83 | type promMetric struct { 84 | Labels map[string]string 85 | Value float64 86 | } 87 | 88 | tcs := []struct { 89 | group string 90 | metricName string 91 | wantHelp string 92 | wantType dto.MetricType 93 | wantMetrics []promMetric 94 | }{ 95 | { 96 | group: "Total", 97 | metricName: "buildkite_total_running_jobs_count", 98 | wantHelp: "Buildkite Total: RunningJobsCount", 99 | wantType: dto.MetricType_GAUGE, 100 | wantMetrics: []promMetric{ 101 | { 102 | Labels: map[string]string{"cluster": "test_cluster"}, 103 | Value: runningJobsCount, 104 | }, 105 | }, 106 | }, 107 | { 108 | group: "Total", 109 | metricName: "buildkite_total_scheduled_jobs_count", 110 | wantHelp: "Buildkite Total: ScheduledJobsCount", 111 | wantType: dto.MetricType_GAUGE, 112 | wantMetrics: []promMetric{ 113 | { 114 | Labels: map[string]string{"cluster": "test_cluster"}, 115 | Value: scheduledJobsCount, 116 | }, 117 | }, 118 | }, 119 | { 120 | group: "Queues", 121 | metricName: "buildkite_queues_scheduled_builds_count", 122 | wantHelp: "Buildkite Queues: ScheduledBuildsCount", 123 | wantType: dto.MetricType_GAUGE, 124 | wantMetrics: []promMetric{ 125 | { 126 | Labels: map[string]string{ 127 | "cluster": "test_cluster", 128 | "queue": "default", 129 | }, 130 | Value: scheduledBuildsCount, 131 | }, 132 | { 133 | Labels: map[string]string{ 134 | "cluster": "test_cluster", 135 | "queue": "deploy", 136 | }, 137 | Value: scheduledBuildsCount, 138 | }, 139 | }, 140 | }, 141 | { 142 | group: "Queues", 143 | metricName: "buildkite_queues_idle_agent_count", 144 | wantHelp: "Buildkite Queues: IdleAgentCount", 145 | wantType: dto.MetricType_GAUGE, 146 | wantMetrics: []promMetric{ 147 | { 148 | Labels: map[string]string{ 149 | "cluster": "test_cluster", 150 | "queue": "default", 151 | }, 152 | Value: idleAgentCount, 153 | }, 154 | { 155 | Labels: map[string]string{ 156 | "cluster": "test_cluster", 157 | "queue": "deploy", 158 | }, 159 | Value: idleAgentCount, 160 | }, 161 | }, 162 | }, 163 | } 164 | 165 | for _, tc := range tcs { 166 | t.Run(fmt.Sprintf("%s/%s", tc.group, tc.metricName), func(t *testing.T) { 167 | mf, ok := metricFamilies[tc.metricName] 168 | if !ok { 169 | t.Errorf("no metric found for name %s", tc.metricName) 170 | } 171 | 172 | if got, want := mf.GetHelp(), tc.wantHelp; got != want { 173 | t.Errorf("mf.GetHelp() = %q, want %q", got, want) 174 | } 175 | 176 | if got, want := mf.GetType(), tc.wantType; got != want { 177 | t.Errorf("mf.GetType() = %q, want %q", got, want) 178 | } 179 | 180 | // Convert the metric family into an easier-to-diff form. 181 | ms := mf.GetMetric() 182 | var gotMetrics []promMetric 183 | for _, m := range ms { 184 | gotMetric := promMetric{ 185 | Value: m.Gauge.GetValue(), 186 | Labels: make(map[string]string), 187 | } 188 | for _, label := range m.Label { 189 | gotMetric.Labels[label.GetName()] = label.GetValue() 190 | } 191 | gotMetrics = append(gotMetrics, gotMetric) 192 | } 193 | 194 | if diff := cmp.Diff(gotMetrics, tc.wantMetrics); diff != "" { 195 | t.Errorf("metrics diff (-got +want):\n%s", diff) 196 | } 197 | }) 198 | } 199 | } 200 | 201 | func TestCamelToUnderscore(t *testing.T) { 202 | tcs := []struct { 203 | input string 204 | want string 205 | }{ 206 | { 207 | input: "TotalAgentCount", 208 | want: "total_agent_count", 209 | }, 210 | { 211 | input: "Total@#4JobsCount", 212 | want: "total@#4_jobs_count", 213 | }, 214 | { 215 | input: "BuildkiteQueuesIdleAgentCount1_11", 216 | want: "buildkite_queues_idle_agent_count1_11", 217 | }, 218 | } 219 | 220 | for _, tc := range tcs { 221 | if got := camelToUnderscore(tc.input); got != tc.want { 222 | t.Errorf("camelToUnderscore(%q) = %q, want %q", tc.input, got, tc.want) 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | "github.com/buildkite/buildkite-agent-metrics/v5/backend" 14 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 15 | "github.com/buildkite/buildkite-agent-metrics/v5/version" 16 | ) 17 | 18 | // Where we send metrics 19 | var metricsBackend backend.Backend 20 | 21 | func main() { 22 | var ( 23 | interval = flag.Duration("interval", 0, "Update metrics every interval, rather than once") 24 | showVersion = flag.Bool("version", false, "Show the version") 25 | quiet = flag.Bool("quiet", false, "Only print errors") 26 | debug = flag.Bool("debug", false, "Show debug output") 27 | debugHttp = flag.Bool("debug-http", false, "Show full http traces") 28 | dryRun = flag.Bool("dry-run", false, "Whether to only print metrics") 29 | endpoint = flag.String("endpoint", "https://agent.buildkite.com/v3", "A custom Buildkite Agent API endpoint") 30 | timeout = flag.Int("timeout", 15, "Timeout, in seconds, for HTTP requests to Buildkite API") 31 | 32 | // backend config 33 | backendOpt = flag.String("backend", "cloudwatch", "Specify the backend to use: cloudwatch, newrelic, prometheus, stackdriver, statsd") 34 | statsdHost = flag.String("statsd-host", "127.0.0.1:8125", "Specify the StatsD server") 35 | statsdTags = flag.Bool("statsd-tags", false, "Whether your StatsD server supports tagging like Datadog") 36 | prometheusAddr = flag.String("prometheus-addr", ":8080", "Prometheus metrics transport bind address") 37 | prometheusPath = flag.String("prometheus-path", "/metrics", "Prometheus metrics transport path") 38 | clwRegion = flag.String("cloudwatch-region", "", "AWS Region to connect to, defaults to $AWS_REGION or us-east-1") 39 | clwDimensions = flag.String("cloudwatch-dimensions", "", "Cloudwatch dimensions to index metrics under, in the form of Key=Value, Other=Value") 40 | gcpProjectID = flag.String("stackdriver-projectid", "", "Specify Stackdriver Project ID") 41 | nrAppName = flag.String("newrelic-app-name", "", "New Relic application name for metric events") 42 | nrLicenseKey = flag.String("newrelic-license-key", "", "New Relic license key for publishing events") 43 | ) 44 | 45 | // custom config for multiple tokens and queues 46 | var tokens, queues stringSliceFlag 47 | flag.Var(&tokens, "token", "Buildkite Agent registration tokens. At least one is required. Multiple cluster tokens can be used to gather metrics for multiple clusters.") 48 | flag.Var(&queues, "queue", "Specific queues to process") 49 | 50 | flag.Parse() 51 | 52 | if *showVersion { 53 | fmt.Printf("buildkite-agent-metrics %s\n", version.Version) 54 | os.Exit(0) 55 | } 56 | 57 | if len(tokens) == 0 { 58 | envTokens := strings.Split(os.Getenv("BUILDKITE_AGENT_TOKEN"), ",") 59 | for _, t := range envTokens { 60 | t = strings.TrimSpace(t) 61 | if t == "" { 62 | continue 63 | } 64 | tokens = append(tokens, t) 65 | } 66 | } 67 | 68 | if len(tokens) == 0 { 69 | fmt.Println("Must provide at least one token with either --token or BUILDKITE_AGENT_TOKEN") 70 | os.Exit(1) 71 | } 72 | 73 | var err error 74 | switch strings.ToLower(*backendOpt) { 75 | case "cloudwatch": 76 | region := *clwRegion 77 | if envRegion := os.Getenv(`AWS_REGION`); region == "" && envRegion != "" { 78 | region = envRegion 79 | } else { 80 | region = `us-east-1` 81 | } 82 | dimensions, err := backend.ParseCloudWatchDimensions(*clwDimensions) 83 | if err != nil { 84 | fmt.Println(err) 85 | os.Exit(1) 86 | } 87 | metricsBackend = backend.NewCloudWatchBackend(region, dimensions) 88 | 89 | case "statsd": 90 | metricsBackend, err = backend.NewStatsDBackend(*statsdHost, *statsdTags) 91 | if err != nil { 92 | fmt.Printf("Error starting StatsD, err: %v\n", err) 93 | os.Exit(1) 94 | } 95 | 96 | case "prometheus": 97 | prom := backend.NewPrometheusBackend() 98 | go prom.Serve(*prometheusPath, *prometheusAddr) 99 | metricsBackend = prom 100 | 101 | case "stackdriver": 102 | if *gcpProjectID == "" { 103 | *gcpProjectID = os.Getenv(`GCP_PROJECT_ID`) 104 | } 105 | metricsBackend, err = backend.NewStackDriverBackend(*gcpProjectID) 106 | if err != nil { 107 | fmt.Printf("Error starting Stackdriver backend, err: %v\n", err) 108 | os.Exit(1) 109 | } 110 | 111 | case "newrelic": 112 | metricsBackend, err = backend.NewNewRelicBackend(*nrAppName, *nrLicenseKey) 113 | if err != nil { 114 | fmt.Printf("Error starting New Relic client: %v\n", err) 115 | os.Exit(1) 116 | } 117 | 118 | default: 119 | fmt.Println("Must provide a supported backend: cloudwatch, newrelic, prometheus, stackdriver, statsd") 120 | os.Exit(1) 121 | } 122 | 123 | if *quiet { 124 | log.SetOutput(io.Discard) 125 | } 126 | 127 | userAgent := fmt.Sprintf("buildkite-agent-metrics/%s buildkite-agent-metrics-cli", version.Version) 128 | if *interval > 0 { 129 | userAgent += fmt.Sprintf(" interval=%s", *interval) 130 | } 131 | 132 | // Queues passed as flags take precedence. But if no queues are passed in we 133 | // check env vars. If no env vars are defined we default to ingesting metrics 134 | // for all queues. 135 | // NOTE: `BUILDKITE_QUEUE` is a comma separated string of queues 136 | // i.e. "default,deploy,test" 137 | if len(queues) == 0 { 138 | if q, exists := os.LookupEnv(`BUILDKITE_QUEUE`); exists { 139 | queues = strings.Split(q, ",") 140 | } 141 | } 142 | 143 | collectors := make([]*collector.Collector, 0, len(tokens)) 144 | for _, token := range tokens { 145 | collectors = append(collectors, &collector.Collector{ 146 | UserAgent: userAgent, 147 | Endpoint: *endpoint, 148 | Token: token, 149 | Queues: []string(queues), 150 | Quiet: *quiet, 151 | Debug: *debug, 152 | DebugHttp: *debugHttp, 153 | Timeout: *timeout, 154 | }) 155 | } 156 | 157 | collectFunc := func() (time.Duration, error) { 158 | start := time.Now() 159 | 160 | // minimum result.PollDuration across collectors 161 | var pollDuration time.Duration 162 | 163 | for _, c := range collectors { 164 | result, err := c.Collect() 165 | if err != nil { 166 | fmt.Printf("Error collecting agent metrics, err: %s\n", err) 167 | if errors.Is(err, collector.ErrUnauthorized) { 168 | // Unique exit code to signal HTTP 401 169 | os.Exit(4) 170 | } 171 | return time.Duration(0), err 172 | } 173 | 174 | if *dryRun { 175 | continue 176 | } 177 | 178 | if err := metricsBackend.Collect(result); err != nil { 179 | return time.Duration(0), err 180 | } 181 | if result.PollDuration > pollDuration { 182 | pollDuration = result.PollDuration 183 | } 184 | } 185 | 186 | log.Printf("Finished in %s", time.Since(start)) 187 | return pollDuration, nil 188 | } 189 | 190 | minPollDuration, err := collectFunc() 191 | if err != nil { 192 | fmt.Println(err) 193 | } 194 | 195 | if *interval > 0 { 196 | for { 197 | waitTime := *interval 198 | 199 | // Respect the min poll duration returned by the API 200 | if *interval < minPollDuration { 201 | log.Printf("Increasing poll duration based on rate-limit headers") 202 | waitTime = minPollDuration 203 | } 204 | 205 | log.Printf("Waiting for %v (minimum of %v)", waitTime, minPollDuration) 206 | time.Sleep(waitTime) 207 | 208 | minPollDuration, err = collectFunc() 209 | if err != nil { 210 | fmt.Println(err) 211 | } 212 | } 213 | } 214 | } 215 | 216 | type stringSliceFlag []string 217 | 218 | func (i *stringSliceFlag) String() string { 219 | return fmt.Sprintf("%v", *i) 220 | } 221 | 222 | func (i *stringSliceFlag) Set(value string) error { 223 | *i = append(*i, value) 224 | return nil 225 | } 226 | -------------------------------------------------------------------------------- /token/secretsmanager_test.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/secretsmanager" 10 | "github.com/buildkite/buildkite-agent-metrics/v5/token/mock" 11 | "github.com/golang/mock/gomock" 12 | ) 13 | 14 | //go:generate mockgen -source secretsmanager.go -mock_names SecretsManagerClient=SecretsManagerClient -package mock -destination mock/secretsmanager_client.go 15 | 16 | const ( 17 | secretsManagerSecretID = "some-secret-id" 18 | secretsManagerSecretJSONKey = "some_json_key" 19 | secretsManagerSecretValue = "super-secret-value" 20 | secretsManagerSecretJSONValue = `{"some_json_key" : "super-secret-value"}` 21 | ) 22 | 23 | func TestSecretsManagerProvider_WithSecretsManagerJSONSecret(t *testing.T) { 24 | provider := secretsManagerProvider{} 25 | 26 | err := WithSecretsManagerJSONSecret(secretsManagerSecretJSONKey)(&provider) 27 | if err != nil { 28 | t.Fatalf("failed to apply WithJSONSecret: %v", err) 29 | } 30 | 31 | if provider.JSONKey != secretsManagerSecretJSONKey { 32 | t.Fatalf( 33 | "expected secretsManagerProvider.JSONKey to be %s but found %s", 34 | secretsManagerSecretJSONKey, provider.JSONKey) 35 | } 36 | } 37 | 38 | func TestSecretsManagerProvider_New_WithErroringOpt(t *testing.T) { 39 | expectedErr := fmt.Errorf("some-error") 40 | 41 | errFunc := func(provider *secretsManagerProvider) error { 42 | return expectedErr 43 | } 44 | 45 | _, err := NewSecretsManager(nil, secretsManagerSecretID, errFunc) 46 | 47 | if err == nil { 48 | t.Fatalf("expected error to be '%s' but found 'nil'", expectedErr.Error()) 49 | } 50 | 51 | if err != expectedErr { 52 | t.Fatalf("expected error to be '%s' but found '%s'", expectedErr.Error(), err.Error()) 53 | } 54 | } 55 | 56 | func TestSecretsManagerProvider_Get_WithPlainTextSecret(t *testing.T) { 57 | req := secretsmanager.GetSecretValueInput{ 58 | SecretId: aws.String(secretsManagerSecretID), 59 | } 60 | res := secretsmanager.GetSecretValueOutput{ 61 | SecretString: aws.String(secretsManagerSecretValue), 62 | SecretBinary: nil, 63 | } 64 | 65 | ctrl := gomock.NewController(t) 66 | defer ctrl.Finish() 67 | 68 | client := mock.NewSecretsManagerClient(ctrl) 69 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 70 | 71 | provider, err := NewSecretsManager(client, secretsManagerSecretID) 72 | if err != nil { 73 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 74 | } 75 | 76 | token, err := provider.Get() 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | 81 | if token != secretsManagerSecretValue { 82 | t.Fatalf("expected token to be '%s' but found '%s'", secretsManagerSecretValue, token) 83 | } 84 | } 85 | 86 | func TestSecretsManagerProvider_Get_WithBinarySecret(t *testing.T) { 87 | req := secretsmanager.GetSecretValueInput{ 88 | SecretId: aws.String(secretsManagerSecretID), 89 | } 90 | res := secretsmanager.GetSecretValueOutput{ 91 | SecretString: nil, 92 | SecretBinary: stringToBase64(secretsManagerSecretValue), 93 | } 94 | 95 | ctrl := gomock.NewController(t) 96 | defer ctrl.Finish() 97 | 98 | client := mock.NewSecretsManagerClient(ctrl) 99 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 100 | 101 | provider, err := NewSecretsManager(client, secretsManagerSecretID) 102 | if err != nil { 103 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 104 | } 105 | 106 | token, err := provider.Get() 107 | if err != nil { 108 | t.Error(err) 109 | } 110 | 111 | if token != secretsManagerSecretValue { 112 | t.Fatalf("expected token to be '%s' but found '%s'", secretsManagerSecretValue, token) 113 | } 114 | } 115 | 116 | func TestSecretsManagerProvider_Get_WithExistingJSONKey(t *testing.T) { 117 | req := secretsmanager.GetSecretValueInput{ 118 | SecretId: aws.String(secretsManagerSecretID), 119 | } 120 | res := secretsmanager.GetSecretValueOutput{ 121 | SecretString: aws.String(secretsManagerSecretJSONValue), 122 | SecretBinary: nil, 123 | } 124 | 125 | ctrl := gomock.NewController(t) 126 | defer ctrl.Finish() 127 | 128 | client := mock.NewSecretsManagerClient(ctrl) 129 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 130 | 131 | provider, err := NewSecretsManager(client, secretsManagerSecretID, 132 | WithSecretsManagerJSONSecret(secretsManagerSecretJSONKey)) 133 | 134 | if err != nil { 135 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 136 | } 137 | 138 | token, err := provider.Get() 139 | if err != nil { 140 | t.Error(err) 141 | } 142 | 143 | if token != secretsManagerSecretValue { 144 | t.Fatalf("expected token to be '%s' but found '%s'", secretsManagerSecretValue, token) 145 | } 146 | } 147 | 148 | func TestSecretsManagerProvider_Get_WithBinaryJSONSecret(t *testing.T) { 149 | req := secretsmanager.GetSecretValueInput{ 150 | SecretId: aws.String(secretsManagerSecretID), 151 | } 152 | res := secretsmanager.GetSecretValueOutput{ 153 | SecretString: nil, 154 | SecretBinary: stringToBase64(secretsManagerSecretJSONValue), 155 | } 156 | 157 | ctrl := gomock.NewController(t) 158 | defer ctrl.Finish() 159 | 160 | client := mock.NewSecretsManagerClient(ctrl) 161 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 162 | 163 | provider, err := NewSecretsManager(client, secretsManagerSecretID, 164 | WithSecretsManagerJSONSecret(secretsManagerSecretJSONKey)) 165 | 166 | if err != nil { 167 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 168 | } 169 | 170 | token, err := provider.Get() 171 | if err != nil { 172 | t.Error(err) 173 | } 174 | 175 | if token != secretsManagerSecretValue { 176 | t.Fatalf("expected token to be '%s' but found '%s'", secretsManagerSecretValue, token) 177 | } 178 | } 179 | 180 | func TestSecretsManagerProvider_Get_WithNonJSONPayload(t *testing.T) { 181 | req := secretsmanager.GetSecretValueInput{ 182 | SecretId: aws.String(secretsManagerSecretID), 183 | } 184 | res := secretsmanager.GetSecretValueOutput{ 185 | SecretString: aws.String("this is not a JSON payload"), 186 | SecretBinary: nil, 187 | } 188 | 189 | ctrl := gomock.NewController(t) 190 | defer ctrl.Finish() 191 | 192 | client := mock.NewSecretsManagerClient(ctrl) 193 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 194 | 195 | provider, err := NewSecretsManager(client, secretsManagerSecretID, 196 | WithSecretsManagerJSONSecret(secretsManagerSecretJSONKey)) 197 | 198 | if err != nil { 199 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 200 | } 201 | 202 | _, err = provider.Get() 203 | if err == nil { 204 | t.Fatalf("expecting error when extracting JSON key from a non-JSON payload") 205 | } 206 | } 207 | 208 | func TestSecretsManagerProvider_Get_WithNonStringValue(t *testing.T) { 209 | req := secretsmanager.GetSecretValueInput{ 210 | SecretId: aws.String(secretsManagerSecretID), 211 | } 212 | res := secretsmanager.GetSecretValueOutput{ 213 | SecretString: aws.String(`{ "non_string_value": true }`), 214 | SecretBinary: nil, 215 | } 216 | 217 | ctrl := gomock.NewController(t) 218 | defer ctrl.Finish() 219 | 220 | client := mock.NewSecretsManagerClient(ctrl) 221 | client.EXPECT().GetSecretValue(gomock.Eq(&req)).Return(&res, nil) 222 | 223 | provider, err := NewSecretsManager(client, secretsManagerSecretID, 224 | WithSecretsManagerJSONSecret("non_string_value")) 225 | 226 | if err != nil { 227 | t.Fatalf("failed to create SecretsManagerProvider: %v", err) 228 | } 229 | 230 | _, err = provider.Get() 231 | if err == nil { 232 | t.Fatalf("expecting error when extracting a non-string value from JSON payload") 233 | } 234 | } 235 | 236 | func stringToBase64(text string) []byte { 237 | data := base64.StdEncoding.EncodeToString([]byte(text)) 238 | return []byte(data) 239 | } 240 | -------------------------------------------------------------------------------- /collector/collector_test.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | ) 10 | 11 | func TestCollectorWithEmptyResponseForAllQueues(t *testing.T) { 12 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | if r.URL.Path == "/metrics" { 14 | w.WriteHeader(http.StatusOK) 15 | _, _ = io.WriteString(w, `{ 16 | "organization": { 17 | "slug": "test" 18 | }, 19 | "jobs": {}, 20 | "agents": {} 21 | }`) 22 | } else { 23 | w.WriteHeader(http.StatusNotFound) 24 | } 25 | })) 26 | c := &Collector{ 27 | Endpoint: s.URL, 28 | Token: "abc123", 29 | UserAgent: "some-client/1.2.3", 30 | } 31 | res, err := c.Collect() 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | testCases := []struct { 36 | Group string 37 | Counts map[string]int 38 | Key string 39 | Expected int 40 | }{ 41 | {"Totals", res.Totals, RunningJobsCount, 0}, 42 | {"Totals", res.Totals, ScheduledJobsCount, 0}, 43 | {"Totals", res.Totals, UnfinishedJobsCount, 0}, 44 | {"Totals", res.Totals, TotalAgentCount, 0}, 45 | {"Totals", res.Totals, BusyAgentCount, 0}, 46 | {"Totals", res.Totals, IdleAgentCount, 0}, 47 | {"Totals", res.Totals, BusyAgentPercentage, 0}, 48 | } 49 | 50 | for _, tc := range testCases { 51 | t.Run(fmt.Sprintf("%s/%s", tc.Group, tc.Key), func(t *testing.T) { 52 | if tc.Counts[tc.Key] != tc.Expected { 53 | t.Fatalf("%s was %d; want %d", tc.Key, tc.Counts[tc.Key], tc.Expected) 54 | } 55 | }) 56 | } 57 | 58 | if len(res.Queues) > 0 { 59 | t.Fatalf("Unexpected queues in response: %v", res.Queues) 60 | } 61 | } 62 | 63 | func TestCollectorWithNoJobsForAllQueues(t *testing.T) { 64 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 65 | if r.URL.Path == "/metrics" { 66 | w.WriteHeader(http.StatusOK) 67 | _, _ = io.WriteString(w, `{ 68 | "organization": { 69 | "slug": "test" 70 | }, 71 | "jobs": { 72 | "scheduled": 0, 73 | "running": 0, 74 | "total": 0, 75 | "queues": {} 76 | }, 77 | "agents": { 78 | "idle": 0, 79 | "busy": 0, 80 | "total": 0, 81 | "queues": {} 82 | } 83 | }`) 84 | } else { 85 | w.WriteHeader(http.StatusNotFound) 86 | } 87 | })) 88 | c := &Collector{ 89 | Endpoint: s.URL, 90 | Token: "abc123", 91 | UserAgent: "some-client/1.2.3", 92 | } 93 | res, err := c.Collect() 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | testCases := []struct { 98 | Group string 99 | Counts map[string]int 100 | Key string 101 | Expected int 102 | }{ 103 | {"Totals", res.Totals, RunningJobsCount, 0}, 104 | {"Totals", res.Totals, ScheduledJobsCount, 0}, 105 | {"Totals", res.Totals, UnfinishedJobsCount, 0}, 106 | {"Totals", res.Totals, TotalAgentCount, 0}, 107 | {"Totals", res.Totals, BusyAgentCount, 0}, 108 | {"Totals", res.Totals, IdleAgentCount, 0}, 109 | {"Totals", res.Totals, BusyAgentPercentage, 0}, 110 | } 111 | 112 | for _, tc := range testCases { 113 | t.Run(fmt.Sprintf("%s/%s", tc.Group, tc.Key), func(t *testing.T) { 114 | if tc.Counts[tc.Key] != tc.Expected { 115 | t.Fatalf("%s was %d; want %d", tc.Key, tc.Counts[tc.Key], tc.Expected) 116 | } 117 | }) 118 | } 119 | 120 | if len(res.Queues) > 0 { 121 | t.Fatalf("Unexpected queues in response: %v", res.Queues) 122 | } 123 | } 124 | 125 | func TestCollectorWithSomeJobsAndAgentsForAllQueues(t *testing.T) { 126 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 127 | if r.URL.Path == "/metrics" { 128 | w.WriteHeader(http.StatusOK) 129 | _, _ = io.WriteString(w, `{ 130 | "organization": { 131 | "slug": "test" 132 | }, 133 | "jobs": { 134 | "scheduled": 3, 135 | "running": 1, 136 | "total": 4, 137 | "queues": { 138 | "default": { 139 | "scheduled": 2, 140 | "running": 1, 141 | "total": 3 142 | }, 143 | "deploy": { 144 | "scheduled": 1, 145 | "running": 0, 146 | "total": 1 147 | } 148 | } 149 | }, 150 | "agents": { 151 | "idle": 0, 152 | "busy": 1, 153 | "total": 1, 154 | "queues": { 155 | "default": { 156 | "idle": 0, 157 | "busy": 1, 158 | "total": 1 159 | } 160 | } 161 | } 162 | }`) 163 | } else { 164 | w.WriteHeader(http.StatusNotFound) 165 | } 166 | })) 167 | c := &Collector{ 168 | Endpoint: s.URL, 169 | Token: "abc123", 170 | UserAgent: "some-client/1.2.3", 171 | } 172 | res, err := c.Collect() 173 | if err != nil { 174 | t.Fatal(err) 175 | } 176 | testCases := []struct { 177 | Group string 178 | Counts map[string]int 179 | Key string 180 | Expected int 181 | }{ 182 | {"Totals", res.Totals, RunningJobsCount, 1}, 183 | {"Totals", res.Totals, ScheduledJobsCount, 3}, 184 | {"Totals", res.Totals, UnfinishedJobsCount, 4}, 185 | {"Totals", res.Totals, TotalAgentCount, 1}, 186 | {"Totals", res.Totals, BusyAgentCount, 1}, 187 | {"Totals", res.Totals, IdleAgentCount, 0}, 188 | {"Totals", res.Totals, BusyAgentPercentage, 100}, 189 | 190 | {"Queue.default", res.Queues["default"], RunningJobsCount, 1}, 191 | {"Queue.default", res.Queues["default"], ScheduledJobsCount, 2}, 192 | {"Queue.default", res.Queues["default"], UnfinishedJobsCount, 3}, 193 | {"Queue.default", res.Queues["default"], TotalAgentCount, 1}, 194 | {"Queue.default", res.Queues["default"], BusyAgentCount, 1}, 195 | {"Queue.default", res.Queues["default"], IdleAgentCount, 0}, 196 | 197 | {"Queue.deploy", res.Queues["deploy"], RunningJobsCount, 0}, 198 | {"Queue.deploy", res.Queues["deploy"], ScheduledJobsCount, 1}, 199 | {"Queue.deploy", res.Queues["deploy"], UnfinishedJobsCount, 1}, 200 | {"Queue.deploy", res.Queues["deploy"], TotalAgentCount, 0}, 201 | {"Queue.deploy", res.Queues["deploy"], BusyAgentCount, 0}, 202 | {"Queue.deploy", res.Queues["deploy"], IdleAgentCount, 0}, 203 | } 204 | 205 | for queue := range res.Queues { 206 | switch queue { 207 | case "default", "deploy": 208 | continue 209 | default: 210 | t.Fatalf("Unexpected queue %s", queue) 211 | } 212 | } 213 | 214 | for _, tc := range testCases { 215 | t.Run(fmt.Sprintf("%s/%s", tc.Group, tc.Key), func(t *testing.T) { 216 | if tc.Counts[tc.Key] != tc.Expected { 217 | t.Fatalf("%s was %d; want %d", tc.Key, tc.Counts[tc.Key], tc.Expected) 218 | } 219 | }) 220 | } 221 | } 222 | 223 | func TestCollectorWithSomeJobsAndAgentsForAQueue(t *testing.T) { 224 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 225 | if r.URL.Path == "/metrics/queue" && r.URL.Query().Get("name") == "deploy" { 226 | w.WriteHeader(http.StatusOK) 227 | _, _ = io.WriteString(w, `{ 228 | "organization": { 229 | "slug": "test" 230 | }, 231 | "jobs": { 232 | "scheduled": 3, 233 | "running": 1, 234 | "total": 4 235 | }, 236 | "agents": { 237 | "idle": 0, 238 | "busy": 1, 239 | "total": 1 240 | } 241 | }`) 242 | } else { 243 | w.WriteHeader(http.StatusNotFound) 244 | } 245 | })) 246 | c := &Collector{ 247 | Endpoint: s.URL, 248 | Token: "abc123", 249 | UserAgent: "some-client/1.2.3", 250 | Queues: []string{"deploy"}, 251 | } 252 | res, err := c.Collect() 253 | if err != nil { 254 | t.Fatal(err) 255 | } 256 | if len(res.Totals) > 0 { 257 | t.Fatalf("Expected no Totals but found: %v", res.Totals) 258 | } 259 | testCases := []struct { 260 | Group string 261 | Counts map[string]int 262 | Key string 263 | Expected int 264 | }{ 265 | {"Queue.deploy", res.Queues["deploy"], RunningJobsCount, 1}, 266 | {"Queue.deploy", res.Queues["deploy"], ScheduledJobsCount, 3}, 267 | {"Queue.deploy", res.Queues["deploy"], UnfinishedJobsCount, 4}, 268 | {"Queue.deploy", res.Queues["deploy"], TotalAgentCount, 1}, 269 | {"Queue.deploy", res.Queues["deploy"], BusyAgentCount, 1}, 270 | {"Queue.deploy", res.Queues["deploy"], IdleAgentCount, 0}, 271 | {"Queue.deploy", res.Queues["deploy"], BusyAgentPercentage, 100}, 272 | } 273 | 274 | for _, tc := range testCases { 275 | t.Run(fmt.Sprintf("%s/%s", tc.Group, tc.Key), func(t *testing.T) { 276 | if tc.Counts[tc.Key] != tc.Expected { 277 | t.Fatalf("%s was %d; want %d", tc.Key, tc.Counts[tc.Key], tc.Expected) 278 | } 279 | }) 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /lambda/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | "github.com/aws/aws-lambda-go/lambda" 15 | "github.com/aws/aws-sdk-go/aws" 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | "github.com/aws/aws-sdk-go/service/secretsmanager" 18 | "github.com/aws/aws-sdk-go/service/ssm" 19 | 20 | "github.com/buildkite/buildkite-agent-metrics/v5/backend" 21 | "github.com/buildkite/buildkite-agent-metrics/v5/collector" 22 | "github.com/buildkite/buildkite-agent-metrics/v5/token" 23 | "github.com/buildkite/buildkite-agent-metrics/v5/version" 24 | ) 25 | 26 | const ( 27 | BKAgentTokenEnvVar = "BUILDKITE_AGENT_TOKEN" 28 | BKAgentTokenSSMKeyEnvVar = "BUILDKITE_AGENT_TOKEN_SSM_KEY" 29 | BKAgentTokenSecretsManagerSecretIDEnvVar = "BUILDKITE_AGENT_SECRETS_MANAGER_SECRET_ID" 30 | BKAgentTokenSecretsManagerJSONKeyEnvVar = "BUILDKITE_AGENT_SECRETS_MANAGER_JSON_KEY" 31 | ) 32 | 33 | var ( 34 | nextPollTime time.Time 35 | ) 36 | 37 | func main() { 38 | if os.Getenv(`DEBUG`) != "" { 39 | _, err := Handler(context.Background(), json.RawMessage([]byte{})) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | } else { 44 | lambda.Start(Handler) 45 | } 46 | } 47 | 48 | func Handler(ctx context.Context, evt json.RawMessage) (string, error) { 49 | // Where we send metrics 50 | var metricsBackend backend.Backend 51 | 52 | awsRegion := os.Getenv("AWS_REGION") 53 | backendOpt := os.Getenv("BUILDKITE_BACKEND") 54 | queue := os.Getenv("BUILDKITE_QUEUE") 55 | clwDimensions := os.Getenv("BUILDKITE_CLOUDWATCH_DIMENSIONS") 56 | quietString := os.Getenv("BUILDKITE_QUIET") 57 | quiet := quietString == "1" || quietString == "true" 58 | timeout := os.Getenv("BUILDKITE_AGENT_METRICS_TIMEOUT") 59 | 60 | debugEnvVar := os.Getenv("BUILDKITE_AGENT_METRICS_DEBUG") 61 | debug := debugEnvVar == "1" || debugEnvVar == "true" 62 | 63 | debugHTTPEnvVar := os.Getenv("BUILDKITE_AGENT_METRICS_DEBUG_HTTP") 64 | debugHTTP := debugHTTPEnvVar == "1" || debugHTTPEnvVar == "true" 65 | 66 | if quiet { 67 | log.SetOutput(io.Discard) 68 | } 69 | 70 | startTime := time.Now() 71 | 72 | if !nextPollTime.IsZero() && nextPollTime.After(startTime) { 73 | log.Printf("Skipping polling, next poll time is in %v", 74 | nextPollTime.Sub(startTime)) 75 | return "", nil 76 | } 77 | 78 | providers, err := initTokenProvider(awsRegion) 79 | if err != nil { 80 | return "", err 81 | } 82 | 83 | tokens := make([]string, 0) 84 | for _, provider := range providers { 85 | bkToken, err := provider.Get() 86 | if err != nil { 87 | return "", err 88 | } 89 | tokens = append(tokens, bkToken) 90 | } 91 | 92 | queues := []string{} 93 | if queue != "" { 94 | queues = strings.Split(queue, ",") 95 | } 96 | 97 | if timeout == "" { 98 | timeout = "15" 99 | } 100 | 101 | configuredTimeout, err := strconv.Atoi(timeout) 102 | 103 | if err != nil { 104 | return "", err 105 | } 106 | 107 | userAgent := fmt.Sprintf("buildkite-agent-metrics/%s buildkite-agent-metrics-lambda", version.Version) 108 | 109 | collectors := make([]*collector.Collector, 0, len(tokens)) 110 | for _, token := range tokens { 111 | collectors = append(collectors, &collector.Collector{ 112 | UserAgent: userAgent, 113 | Endpoint: "https://agent.buildkite.com/v3", 114 | Token: token, 115 | Queues: queues, 116 | Quiet: quiet, 117 | Debug: debug, 118 | DebugHttp: debugHTTP, 119 | Timeout: configuredTimeout, 120 | }) 121 | } 122 | 123 | switch strings.ToLower(backendOpt) { 124 | case "statsd": 125 | statsdHost := os.Getenv("STATSD_HOST") 126 | statsdTags := strings.EqualFold(os.Getenv("STATSD_TAGS"), "true") 127 | metricsBackend, err = backend.NewStatsDBackend(statsdHost, statsdTags) 128 | if err != nil { 129 | return "", err 130 | } 131 | 132 | case "newrelic": 133 | nrAppName := os.Getenv("NEWRELIC_APP_NAME") 134 | nrLicenseKey := os.Getenv("NEWRELIC_LICENSE_KEY") 135 | metricsBackend, err = backend.NewNewRelicBackend(nrAppName, nrLicenseKey) 136 | if err != nil { 137 | fmt.Printf("Error starting New Relic client: %v\n", err) 138 | os.Exit(1) 139 | } 140 | 141 | default: 142 | dimensions, err := backend.ParseCloudWatchDimensions(clwDimensions) 143 | if err != nil { 144 | return "", err 145 | } 146 | metricsBackend = backend.NewCloudWatchBackend(awsRegion, dimensions) 147 | } 148 | 149 | // minimum res.PollDuration across collectors 150 | var pollDuration time.Duration 151 | 152 | for _, c := range collectors { 153 | res, err := c.Collect() 154 | if err != nil { 155 | return "", err 156 | } 157 | 158 | if res.PollDuration > pollDuration { 159 | pollDuration = res.PollDuration 160 | } 161 | 162 | res.Dump() 163 | 164 | if err := metricsBackend.Collect(res); err != nil { 165 | return "", err 166 | } 167 | } 168 | 169 | original, ok := metricsBackend.(backend.Closer) 170 | if ok { 171 | err := original.Close() 172 | if err != nil { 173 | return "", err 174 | } 175 | } 176 | 177 | log.Printf("Finished in %s", time.Since(startTime)) 178 | 179 | // Store the next acceptable poll time in global state 180 | nextPollTime = time.Now().Add(pollDuration) 181 | 182 | return "", nil 183 | } 184 | 185 | func initTokenProvider(awsRegion string) ([]token.Provider, error) { 186 | err := checkMutuallyExclusiveEnvVars( 187 | BKAgentTokenEnvVar, 188 | BKAgentTokenSSMKeyEnvVar, 189 | BKAgentTokenSecretsManagerSecretIDEnvVar, 190 | ) 191 | if err != nil { 192 | return nil, err 193 | } 194 | 195 | var providers []token.Provider 196 | if bkTokenEnvVar := os.Getenv(BKAgentTokenEnvVar); bkTokenEnvVar != "" { 197 | bkTokens := strings.Split(bkTokenEnvVar, ",") 198 | for _, bkToken := range bkTokens { 199 | provider, err := token.NewInMemory(bkToken) 200 | if err != nil { 201 | return nil, err 202 | } 203 | providers = append(providers, provider) 204 | } 205 | } 206 | 207 | if ssmKey := os.Getenv(BKAgentTokenSSMKeyEnvVar); ssmKey != "" { 208 | sess, err := session.NewSession(&aws.Config{Region: aws.String(awsRegion)}) 209 | if err != nil { 210 | return nil, err 211 | } 212 | client := ssm.New(sess) 213 | provider, err := token.NewSSM(client, ssmKey) 214 | if err != nil { 215 | return nil, err 216 | } 217 | providers = append(providers, provider) 218 | } 219 | 220 | if secretsManagerSecretID := os.Getenv(BKAgentTokenSecretsManagerSecretIDEnvVar); secretsManagerSecretID != "" { 221 | jsonKey := os.Getenv(BKAgentTokenSecretsManagerJSONKeyEnvVar) 222 | sess, err := session.NewSession(&aws.Config{Region: aws.String(awsRegion)}) 223 | if err != nil { 224 | return nil, err 225 | } 226 | client := secretsmanager.New(sess) 227 | if jsonKey == "" { 228 | secretIDs := strings.Split(secretsManagerSecretID, ",") 229 | for _, secretID := range secretIDs { 230 | secretManager, err := token.NewSecretsManager(client, secretID) 231 | if err != nil { 232 | return nil, err 233 | } 234 | providers = append(providers, secretManager) 235 | } 236 | } else { 237 | secretManager, err := token.NewSecretsManager(client, secretsManagerSecretID, token.WithSecretsManagerJSONSecret(jsonKey)) 238 | if err != nil { 239 | return nil, err 240 | } 241 | providers = append(providers, secretManager) 242 | } 243 | } 244 | 245 | if len(providers) == 0 { 246 | // This should be very unlikely or even impossible (famous last words): 247 | // - There was exactly one of the mutually-exclusive env vars 248 | // - If a token provider above failed to use its value, it should error 249 | // - Otherwise, each if-branch appends to providers, so... 250 | return nil, fmt.Errorf("no Buildkite token providers could be created") 251 | } 252 | 253 | return providers, nil 254 | } 255 | 256 | func checkMutuallyExclusiveEnvVars(varNames ...string) error { 257 | foundVars := make([]string, 0) 258 | for _, varName := range varNames { 259 | value := os.Getenv(varName) 260 | if value != "" { 261 | foundVars = append(foundVars, value) 262 | } 263 | } 264 | switch len(foundVars) { 265 | case 0: 266 | return fmt.Errorf("one of the environment variables [%s] must be provided", strings.Join(varNames, ",")) 267 | 268 | case 1: 269 | return nil // that's what we want 270 | 271 | default: 272 | return fmt.Errorf("the environment variables [%s] are mutually exclusive", strings.Join(foundVars, ",")) 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /collector/collector.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "net/http/httputil" 10 | "net/url" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | const ( 17 | ScheduledJobsCount = "ScheduledJobsCount" 18 | RunningJobsCount = "RunningJobsCount" 19 | UnfinishedJobsCount = "UnfinishedJobsCount" 20 | WaitingJobsCount = "WaitingJobsCount" 21 | IdleAgentCount = "IdleAgentCount" 22 | BusyAgentCount = "BusyAgentCount" 23 | TotalAgentCount = "TotalAgentCount" 24 | BusyAgentPercentage = "BusyAgentPercentage" 25 | 26 | PollDurationHeader = `Buildkite-Agent-Metrics-Poll-Duration` 27 | ) 28 | 29 | var ( 30 | ErrUnauthorized = errors.New("unauthorized") 31 | ) 32 | 33 | type Collector struct { 34 | Endpoint string 35 | Token string 36 | UserAgent string 37 | Queues []string 38 | Quiet bool 39 | Debug bool 40 | DebugHttp bool 41 | Timeout int 42 | } 43 | 44 | type Result struct { 45 | Totals map[string]int 46 | Queues map[string]map[string]int 47 | Org string 48 | Cluster string 49 | PollDuration time.Duration 50 | } 51 | 52 | type organizationResponse struct { 53 | Slug string `json:"slug"` 54 | } 55 | 56 | type clusterResponse struct { 57 | Name string `json:"name"` 58 | } 59 | 60 | type metricsAgentsResponse struct { 61 | Idle int `json:"idle"` 62 | Busy int `json:"busy"` 63 | Total int `json:"total"` 64 | } 65 | 66 | type metricsJobsResponse struct { 67 | Scheduled int `json:"scheduled"` 68 | Running int `json:"running"` 69 | Waiting int `json:"waiting"` 70 | Total int `json:"total"` 71 | } 72 | 73 | type queueMetricsResponse struct { 74 | Agents metricsAgentsResponse `json:"agents"` 75 | Jobs metricsJobsResponse `json:"jobs"` 76 | Organization organizationResponse `json:"organization"` 77 | Cluster clusterResponse `json:"cluster"` 78 | } 79 | 80 | type allMetricsAgentsResponse struct { 81 | metricsAgentsResponse 82 | Queues map[string]metricsAgentsResponse `json:"queues"` 83 | } 84 | 85 | type allMetricsJobsResponse struct { 86 | metricsJobsResponse 87 | Queues map[string]metricsJobsResponse `json:"queues"` 88 | } 89 | 90 | type allMetricsResponse struct { 91 | Agents allMetricsAgentsResponse `json:"agents"` 92 | Jobs allMetricsJobsResponse `json:"jobs"` 93 | Organization organizationResponse `json:"organization"` 94 | Cluster clusterResponse `json:"cluster"` 95 | } 96 | 97 | func (c *Collector) Collect() (*Result, error) { 98 | result := &Result{ 99 | Totals: map[string]int{}, 100 | Queues: map[string]map[string]int{}, 101 | } 102 | 103 | httpClient := &http.Client{ 104 | Timeout: time.Duration(c.Timeout) * time.Second, 105 | } 106 | 107 | if len(c.Queues) == 0 { 108 | if err := c.collectAllQueues(httpClient, result); err != nil { 109 | return nil, err 110 | } 111 | } else { 112 | for _, queue := range c.Queues { 113 | if err := c.collectQueue(httpClient, result, queue); err != nil { 114 | return nil, err 115 | } 116 | } 117 | } 118 | 119 | if !c.Quiet { 120 | result.Dump() 121 | } 122 | 123 | return result, nil 124 | } 125 | 126 | func (c *Collector) collectAllQueues(httpClient *http.Client, result *Result) error { 127 | log.Println("Collecting agent metrics for all queues") 128 | 129 | endpoint, err := url.Parse(c.Endpoint) 130 | if err != nil { 131 | return err 132 | } 133 | 134 | endpoint.Path += "/metrics" 135 | 136 | req, err := http.NewRequest("GET", endpoint.String(), nil) 137 | if err != nil { 138 | return err 139 | } 140 | 141 | req.Header.Set("User-Agent", c.UserAgent) 142 | req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token)) 143 | 144 | if c.DebugHttp { 145 | if dump, err := httputil.DumpRequest(req, true); err == nil { 146 | log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump) 147 | } 148 | } 149 | 150 | res, err := httpClient.Do(req) 151 | if err != nil { 152 | return err 153 | } 154 | defer res.Body.Close() 155 | 156 | if res.StatusCode == 401 { 157 | return fmt.Errorf("http 401 response received %w", ErrUnauthorized) 158 | } 159 | 160 | if c.DebugHttp { 161 | if dump, err := httputil.DumpResponse(res, true); err == nil { 162 | log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump) 163 | } 164 | } 165 | 166 | // Handle any errors 167 | if res.StatusCode != http.StatusOK { 168 | // If it's json response, show the error message 169 | if strings.HasPrefix(res.Header.Get("Content-Type"), "application/json") { 170 | var errStruct struct { 171 | Message string `json:"message"` 172 | } 173 | err := json.NewDecoder(res.Body).Decode(&errStruct) 174 | if err == nil { 175 | return errors.New(errStruct.Message) 176 | } else { 177 | log.Printf("Failed to decode error: %v", err) 178 | } 179 | } 180 | 181 | return fmt.Errorf("Request failed with %s (%d)", res.Status, res.StatusCode) 182 | } 183 | 184 | var allMetrics allMetricsResponse 185 | 186 | // Check if we get a poll duration header from server 187 | if pollSeconds := res.Header.Get(PollDurationHeader); pollSeconds != "" { 188 | pollSecondsInt, err := strconv.ParseInt(pollSeconds, 10, 64) 189 | if err != nil { 190 | log.Printf("Failed to parse %s header: %v", PollDurationHeader, err) 191 | } else { 192 | result.PollDuration = time.Duration(pollSecondsInt) * time.Second 193 | } 194 | } 195 | 196 | err = json.NewDecoder(res.Body).Decode(&allMetrics) 197 | if err != nil { 198 | return err 199 | } 200 | 201 | if allMetrics.Organization.Slug == "" { 202 | return fmt.Errorf("No organization slug was found in the metrics response") 203 | } 204 | 205 | log.Printf("Found organization %q, cluster %q", allMetrics.Organization.Slug, allMetrics.Cluster.Name) 206 | result.Org = allMetrics.Organization.Slug 207 | result.Cluster = allMetrics.Cluster.Name 208 | 209 | result.Totals[ScheduledJobsCount] = allMetrics.Jobs.Scheduled 210 | result.Totals[RunningJobsCount] = allMetrics.Jobs.Running 211 | result.Totals[UnfinishedJobsCount] = allMetrics.Jobs.Total 212 | result.Totals[WaitingJobsCount] = allMetrics.Jobs.Waiting 213 | result.Totals[IdleAgentCount] = allMetrics.Agents.Idle 214 | result.Totals[BusyAgentCount] = allMetrics.Agents.Busy 215 | result.Totals[TotalAgentCount] = allMetrics.Agents.Total 216 | result.Totals[BusyAgentPercentage] = busyAgentPercentage(allMetrics.Agents.metricsAgentsResponse) 217 | 218 | for queueName, queueJobMetrics := range allMetrics.Jobs.Queues { 219 | if _, ok := result.Queues[queueName]; !ok { 220 | result.Queues[queueName] = map[string]int{} 221 | } 222 | result.Queues[queueName][ScheduledJobsCount] = queueJobMetrics.Scheduled 223 | result.Queues[queueName][RunningJobsCount] = queueJobMetrics.Running 224 | result.Queues[queueName][UnfinishedJobsCount] = queueJobMetrics.Total 225 | result.Queues[queueName][WaitingJobsCount] = queueJobMetrics.Waiting 226 | } 227 | 228 | for queueName, queueAgentMetrics := range allMetrics.Agents.Queues { 229 | if _, ok := result.Queues[queueName]; !ok { 230 | result.Queues[queueName] = map[string]int{} 231 | } 232 | result.Queues[queueName][IdleAgentCount] = queueAgentMetrics.Idle 233 | result.Queues[queueName][BusyAgentCount] = queueAgentMetrics.Busy 234 | result.Queues[queueName][TotalAgentCount] = queueAgentMetrics.Total 235 | result.Queues[queueName][BusyAgentPercentage] = busyAgentPercentage(queueAgentMetrics) 236 | } 237 | 238 | return nil 239 | } 240 | 241 | func (c *Collector) collectQueue(httpClient *http.Client, result *Result, queue string) error { 242 | log.Printf("Collecting agent metrics for queue '%s'", queue) 243 | 244 | endpoint, err := url.Parse(c.Endpoint) 245 | if err != nil { 246 | return err 247 | } 248 | 249 | endpoint.Path += "/metrics/queue" 250 | endpoint.RawQuery = url.Values{"name": {queue}}.Encode() 251 | 252 | req, err := http.NewRequest("GET", endpoint.String(), nil) 253 | if err != nil { 254 | return err 255 | } 256 | 257 | req.Header.Set("User-Agent", c.UserAgent) 258 | req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.Token)) 259 | 260 | if c.DebugHttp { 261 | if dump, err := httputil.DumpRequest(req, true); err == nil { 262 | log.Printf("DEBUG request uri=%s\n%s\n", req.URL, dump) 263 | } 264 | } 265 | 266 | res, err := httpClient.Do(req) 267 | if err != nil { 268 | return err 269 | } 270 | defer res.Body.Close() 271 | 272 | if res.StatusCode == 401 { 273 | return fmt.Errorf("http 401 response received %w", ErrUnauthorized) 274 | } 275 | 276 | if c.DebugHttp { 277 | if dump, err := httputil.DumpResponse(res, true); err == nil { 278 | log.Printf("DEBUG response uri=%s\n%s\n", req.URL, dump) 279 | } 280 | } 281 | 282 | // Handle any errors 283 | if res.StatusCode != http.StatusOK { 284 | // If it's json response, show the error message 285 | if strings.HasPrefix(res.Header.Get("Content-Type"), "application/json") { 286 | var errStruct struct { 287 | Message string `json:"message"` 288 | } 289 | err := json.NewDecoder(res.Body).Decode(&errStruct) 290 | if err == nil { 291 | return errors.New(errStruct.Message) 292 | } else { 293 | log.Printf("Failed to decode error: %v", err) 294 | } 295 | } 296 | 297 | return fmt.Errorf("Request failed with %s (%d)", res.Status, res.StatusCode) 298 | } 299 | 300 | var queueMetrics queueMetricsResponse 301 | err = json.NewDecoder(res.Body).Decode(&queueMetrics) 302 | if err != nil { 303 | return err 304 | } 305 | 306 | if queueMetrics.Organization.Slug == "" { 307 | return fmt.Errorf("No organization slug was found in the metrics response") 308 | } 309 | 310 | log.Printf("Found organization %q, cluster %q", queueMetrics.Organization.Slug, queueMetrics.Cluster.Name) 311 | result.Org = queueMetrics.Organization.Slug 312 | result.Cluster = queueMetrics.Cluster.Name 313 | 314 | result.Queues[queue] = map[string]int{ 315 | ScheduledJobsCount: queueMetrics.Jobs.Scheduled, 316 | RunningJobsCount: queueMetrics.Jobs.Running, 317 | UnfinishedJobsCount: queueMetrics.Jobs.Total, 318 | WaitingJobsCount: queueMetrics.Jobs.Waiting, 319 | IdleAgentCount: queueMetrics.Agents.Idle, 320 | BusyAgentCount: queueMetrics.Agents.Busy, 321 | TotalAgentCount: queueMetrics.Agents.Total, 322 | BusyAgentPercentage: busyAgentPercentage(queueMetrics.Agents), 323 | } 324 | return nil 325 | } 326 | 327 | func busyAgentPercentage(agents metricsAgentsResponse) int { 328 | if agents.Total > 0 { 329 | return int(100 * agents.Busy / agents.Total) 330 | } 331 | return 0 332 | } 333 | 334 | func (r Result) Dump() { 335 | for name, c := range r.Totals { 336 | log.Printf("Buildkite > Org=%s > %s=%d", r.Org, name, c) 337 | } 338 | 339 | for name, c := range r.Queues { 340 | for k, v := range c { 341 | log.Printf("Buildkite > Org=%s > Queue=%s > %s=%d", r.Org, name, k, v) 342 | } 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Buildkite Agent Metrics 2 | 3 | A command-line tool for collecting [Buildkite](https://buildkite.com/) agent metrics, focusing on enabling auto-scaling. Currently [AWS Cloudwatch](http://aws.amazon.com/cloudwatch/), [StatsD](https://github.com/etsy/statsd), [Prometheus](https://prometheus.io), [Stackdriver](https://cloud.google.com/stackdriver/) and [New Relic](https://newrelic.com/products/insights) are supported. 4 | 5 | [![Build status](https://badge.buildkite.com/80d04fcde3a306bef44e77aadb1f1ffdc20ebb3c8f1f585a60.svg)](https://buildkite.com/buildkite/buildkite-agent-metrics) 6 | 7 | ## Installing 8 | 9 | Either download the latest binary from 10 | [Github Releases](https://github.com/buildkite/buildkite-agent-metrics/releases) or install with: 11 | 12 | ```bash 13 | go install github.com/buildkite/buildkite-agent-metrics/v5@latest 14 | ``` 15 | 16 | ## Running 17 | 18 | Several running modes are supported. All of them require an Agent Registration 19 | Token, found on the 20 | [Buildkite Agents page](https://buildkite.com/organizations/-/agents). 21 | 22 | ### Running as a Daemon 23 | 24 | The simplest deployment is to run as a long-running daemon that collects metrics 25 | across all queues in an organization. 26 | 27 | ```shell 28 | buildkite-agent-metrics -token abc123 -interval 30s 29 | ``` 30 | 31 | Restrict it to a single queue with `-queue`: 32 | 33 | ```shell 34 | buildkite-agent-metrics -token abc123 -interval 30s -queue my-queue 35 | ``` 36 | 37 | Restrict it to multiple queues by repeating `-queue`: 38 | 39 | ```shell 40 | buildkite-agent-metrics -token abc123 -interval 30s -queue my-queue1 -queue my-queue2 41 | ``` 42 | 43 | When using clusters, you can pass a cluster registration token to gather metrics 44 | only for that cluster: 45 | 46 | ```shell 47 | buildkite-agent-metrics -token clustertoken ... 48 | ``` 49 | 50 | You can repeat `-token` to gather metrics for multiple clusters: 51 | 52 | ```shell 53 | buildkite-agent-metrics -token clusterAtoken -token clusterBtoken ... 54 | ``` 55 | 56 | ### Running as an AWS Lambda 57 | 58 | An AWS Lambda bundle is created and published as part of the build process. The 59 | lambda will require the 60 | [`cloudwatch:PutMetricData`](https://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/publishingMetrics.html) 61 | IAM permission. 62 | 63 | It requires a `provided.al2` environment and respects the following env vars: 64 | 65 | - `BUILDKITE_BACKEND` : The name of the backend to use (e.g. `cloudwatch`, 66 | `statsd`, `newrelic`. For the lambda, `prometheus` and `stackdriver` are not 67 | supported). 68 | - `BUILDKITE_QUEUE` : A comma separated list of Buildkite queues to process 69 | (e.g. `backend-deploy,ui-deploy`). 70 | - `BUILDKITE_QUIET` : A boolean specifying that only `ERROR` log lines must be 71 | printed. (e.g. `1`, `true`). 72 | - `BUILDKITE_CLOUDWATCH_DIMENSIONS` : A comma separated list in the form of 73 | `Key=Value,Other=Value` containing the Cloudwatch dimensions to index metrics 74 | under. 75 | 76 | Additionally, one of the following groups of environment variables must be set 77 | in order to define how the Lambda function should obtain the required Buildkite 78 | Agent API token: 79 | 80 | #### Option 1 - Provide the token(s) as plain-text 81 | 82 | - `BUILDKITE_AGENT_TOKEN` : The Buildkite Agent API token to use. You can supply 83 | multiple tokens comma-separated. 84 | 85 | #### Option 2 - Retrieve token from AWS Systems Manager 86 | 87 | - `BUILDKITE_AGENT_TOKEN_SSM_KEY` : The parameter name which contains the token 88 | value in AWS Systems Manager. You can supply multiple names comma-separated. 89 | 90 | **Note**: Parameters stored as `String` and `SecureString` are currently 91 | supported. 92 | 93 | #### Option 3 - Retrieve token from AWS Secrets Manager 94 | 95 | - `BUILDKITE_AGENT_SECRETS_MANAGER_SECRET_ID`: The id of the secret which 96 | contains the token value in AWS Secrets Manager. You can supply 97 | multiple ids comma-separated. 98 | - (Optional) `BUILDKITE_AGENT_SECRETS_MANAGER_JSON_KEY`: The JSON key containing 99 | the token value in the secret JSON blob. 100 | 101 | **Note 1**: Both `SecretBinary` and `SecretString` are supported. In the case of 102 | `SecretBinary`, the secret payload will be automatically decoded and returned as 103 | a plain-text string. 104 | 105 | **Note 2**: `BUILDKITE_AGENT_SECRETS_MANAGER_JSON_KEY` can be used on secrets of 106 | type `SecretBinary` only if their binary payload corresponds to a valid JSON 107 | object containing the provided key. 108 | 109 | ```bash 110 | aws lambda create-function \ 111 | --function-name buildkite-agent-metrics \ 112 | --memory 128 \ 113 | --role arn:aws:iam::account-id:role/execution_role \ 114 | --runtime provided.al2 \ 115 | --zip-file fileb://handler.zip \ 116 | --handler handler 117 | ``` 118 | 119 | ### Running as a Container 120 | 121 | You can build a docker image for the `buildkite-agent-metrics` following: 122 | 123 | ```shell 124 | docker build -t buildkite-agent-metrics . 125 | ``` 126 | 127 | This will create a local docker image named as `buildkite-agent-metrics` that 128 | you can tag and push to your own registry. 129 | 130 | You can use the command-line arguments in a docker execution in the same way as 131 | described before: 132 | 133 | ```shell 134 | docker run --rm buildkite-agent-metrics -token abc123 -interval 30s -queue my-queue 135 | ``` 136 | 137 | ### Supported command line flags 138 | 139 | ```shell 140 | $ buildkite-agent-metrics --help 141 | Usage of buildkite-agent-metrics: 142 | -backend string 143 | Specify the backend to use: cloudwatch, statsd, prometheus, stackdriver (default "cloudwatch") 144 | -cloudwatch-dimensions string 145 | Cloudwatch dimensions to index metrics under, in the form of Key=Value, Other=Value 146 | -cloudwatch-region string 147 | AWS Region to connect to, defaults to $AWS_REGION or us-east-1 148 | -debug 149 | Show debug output 150 | -debug-http 151 | Show full http traces 152 | -dry-run 153 | Whether to only print metrics 154 | -endpoint string 155 | A custom Buildkite Agent API endpoint (default "https://agent.buildkite.com/v3") 156 | -interval duration 157 | Update metrics every interval, rather than once 158 | -newrelic-app-name string 159 | New Relic application name for metric events 160 | -newrelic-license-key string 161 | New Relic license key for publishing events 162 | -prometheus-addr string 163 | Prometheus metrics transport bind address (default ":8080") 164 | -prometheus-path string 165 | Prometheus metrics transport path (default "/metrics") 166 | -queue value 167 | Specific queues to process 168 | -quiet 169 | Only print errors 170 | -stackdriver-projectid string 171 | Specify Stackdriver Project ID 172 | -statsd-host string 173 | Specify the StatsD server (default "127.0.0.1:8125") 174 | -statsd-tags 175 | Whether your StatsD server supports tagging like Datadog 176 | -token string 177 | A Buildkite Agent Registration Token 178 | -version 179 | Show the version 180 | ``` 181 | 182 | ### Backends 183 | 184 | By default metrics will be submitted to CloudWatch but the backend can be switched to StatsD or Prometheus using the command-line argument `-backend statsd` or `-backend prometheus` respectively. 185 | 186 | #### Cloudwatch 187 | 188 | The Cloudwatch backend supports the following arguments: 189 | 190 | - `-cloudwatch-dimensions`: A optional custom dimension in the form of `Key=Value, Key=Value` 191 | 192 | #### StatsD (Datadog) 193 | 194 | The StatsD backend supports the following arguments: 195 | 196 | - `-statsd-host HOST`: The StatsD host and port (defaults to `127.0.0.1:8125`). 197 | - `-statsd-tags`: Some StatsD servers like the agent provided by Datadog support 198 | tags. If specified, metrics will be tagged by `queue` otherwise metrics will 199 | include the queue name in the metric. Only enable this option if you know 200 | your StatsD server supports tags. 201 | 202 | #### Prometheus 203 | 204 | The Prometheus backend supports the following arguments: 205 | 206 | - `-prometheus-addr`: The local address to listen on (defaults to `:8080`). 207 | - `-prometheus-path`: The path under `prometheus-addr` to expose metrics on 208 | (defaults to `/metrics`). 209 | 210 | #### Stackdriver 211 | 212 | The Stackdriver backend supports the following arguments: 213 | 214 | - `-stackdriver-projectid`: The Google Cloud Platform project to report metrics 215 | for. 216 | 217 | The New Relic backend supports the following arguments: 218 | 219 | - `-newrelic-app-name`: String for the New Relic app name 220 | - `-newrelic-license-key`: The New Relic license key. Must be of type `INGEST` 221 | 222 | ### Upgrading from v2 to v3 223 | 224 | 1. The `-org` argument is no longer needed 225 | 2. The `-token` argument is now an _Agent Registration Token_ — the same used in 226 | the Buildkite Agent configuration file, and found on the 227 | [Buildkite Agents page](https://buildkite.com/organizations/-/agents). 228 | 3. Build and pipeline metrics have been removed, focusing on agents and jobs by 229 | queue for auto–scaling. 230 | If you have a compelling reason to gather build or pipeline metrics please 231 | continue to use the 232 | [previous version](https://github.com/buildkite/buildkite-agent-metrics/releases/tag/v2.1.0) 233 | or [open an issue](https://github.com/buildkite/buildkite-agent-metrics/issues) 234 | with details. 235 | 236 | ## Development 237 | 238 | This tool is built with Go 1.20+ and assumes 239 | [Go Modules](https://github.com/golang/go/wiki/Modules) by default. 240 | 241 | You can build and run the binary tool locally with Go installed: 242 | 243 | ```shell 244 | go run *.go -token [buildkite agent registration token] 245 | ``` 246 | 247 | Currently this will publish metrics to Cloudwatch under the custom metric prefix 248 | of `Buildkite`, using AWS credentials from your environment. The machine will 249 | require the 250 | [`cloudwatch:PutMetricData`](https://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/publishingMetrics.html) 251 | IAM permission. 252 | 253 | ### The `token` package 254 | 255 | It is an abstraction layer enabling the retrieval of a Buildkite Agent API token 256 | from different sources. 257 | 258 | The current supported sources are: 259 | 260 | - AWS Systems Manager (a.k.a parameter store). 261 | - AWS Secrets Manager. 262 | - OS environment variable. 263 | 264 | #### Tests 265 | 266 | All the tests for AWS dependant resources require their corresponding auto-generated mocks. Thus, 267 | before running them, you need to generate such mocks by executing: 268 | 269 | ```bash 270 | go generate token/secretsmanager_test.go 271 | go generate token/ssm_test.go 272 | ``` 273 | 274 | ## Metrics 275 | 276 | The following metrics are gathered when no specific queue is supplied: 277 | 278 | ```plain 279 | Buildkite > (Org) > RunningJobsCount 280 | Buildkite > (Org) > ScheduledJobsCount 281 | Buildkite > (Org) > UnfinishedJobsCount 282 | Buildkite > (Org) > WaitingJobsCount 283 | Buildkite > (Org) > IdleAgentsCount 284 | Buildkite > (Org) > BusyAgentsCount 285 | Buildkite > (Org) > BusyAgentPercentage 286 | Buildkite > (Org) > TotalAgentsCount 287 | 288 | Buildkite > (Org, Queue) > RunningJobsCount 289 | Buildkite > (Org, Queue) > ScheduledJobsCount 290 | Buildkite > (Org, Queue) > UnfinishedJobsCount 291 | Buildkite > (Org, Queue) > WaitingJobsCount 292 | Buildkite > (Org, Queue) > IdleAgentsCount 293 | Buildkite > (Org, Queue) > BusyAgentsCount 294 | Buildkite > (Org, Queue) > BusyAgentPercentage 295 | Buildkite > (Org, Queue) > TotalAgentsCount 296 | ``` 297 | 298 | When a queue is specified, only that queue's metrics are published. 299 | 300 | We send metrics for Jobs in the following states: 301 | 302 | - **Scheduled**: the job hasn't been assigned to an agent yet. If you have agent 303 | capacity, this value should be close to 0. 304 | - **Waiting**: the job is known to exist but isn't schedulable yet due to 305 | dependencies, `wait` statements, etc. This information is mostly useful to an 306 | autoscaler, since it represents work that will start soon. 307 | - **Running**: an agent is actively executing this job. 308 | 309 | ## License 310 | 311 | See [LICENSE.md](LICENSE.md) (MIT) 312 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= 3 | cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= 4 | cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= 5 | cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= 6 | cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= 7 | cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= 8 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 9 | github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q= 10 | github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 11 | github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= 12 | github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= 13 | github.com/aws/aws-lambda-go v1.42.0 h1:U4QKkxLp/il15RJGAANxiT9VumQzimsUER7gokqA0+c= 14 | github.com/aws/aws-lambda-go v1.42.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= 15 | github.com/aws/aws-sdk-go v1.48.16 h1:mcj2/9J/MJ55Dov+ocMevhR8Jv6jW/fAxbrn4a1JFc8= 16 | github.com/aws/aws-sdk-go v1.48.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= 17 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 18 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 19 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 20 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 21 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 22 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 23 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 24 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 25 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 26 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 27 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 28 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 29 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 30 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 31 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 32 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 33 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 34 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 35 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 36 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 37 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 38 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 39 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 40 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 41 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 42 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 43 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 44 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 45 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 46 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 47 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 48 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 49 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 50 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 51 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 52 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 53 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 54 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 55 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 56 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 57 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 58 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 59 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 60 | github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= 61 | github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= 62 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 63 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= 64 | github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= 65 | github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= 66 | github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= 67 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 68 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 69 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 70 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 71 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 72 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 73 | github.com/newrelic/go-agent v3.24.1+incompatible h1:Dvt6dtEafdrNydNsEHbtSMljWUt8pyWmcaO/wVJmuf8= 74 | github.com/newrelic/go-agent v3.24.1+incompatible/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ= 75 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 76 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 77 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= 78 | github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= 79 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 80 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= 81 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= 82 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 83 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 84 | github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= 85 | github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 88 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 89 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 90 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 91 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 92 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 93 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 94 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 95 | go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= 96 | go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= 97 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 98 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 99 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 100 | golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= 101 | golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 102 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 103 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 104 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 105 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 106 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 107 | golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= 108 | golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 109 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 110 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 111 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 112 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 113 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 114 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 115 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 116 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 117 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 118 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 119 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 120 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 121 | golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= 122 | golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= 123 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 124 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 125 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 126 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 127 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 128 | golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= 129 | golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 130 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 131 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 132 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 133 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 134 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 135 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 136 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 137 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 138 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 139 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 140 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 141 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 142 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 143 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 144 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 145 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 146 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 147 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 148 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 149 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 150 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 151 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 152 | golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= 153 | golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= 154 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 155 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 156 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 157 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 158 | google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= 159 | google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= 160 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 161 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 162 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 163 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 164 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 165 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 166 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 167 | google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= 168 | google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= 169 | google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= 170 | google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= 171 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= 172 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= 173 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 174 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 175 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 176 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 177 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= 178 | google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= 179 | google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= 180 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 181 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 182 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 183 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 184 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 185 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 186 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 187 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 188 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 189 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 190 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 191 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 192 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 193 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 194 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 195 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 196 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 197 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 198 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 199 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 200 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 201 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ## [v5.9.3](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.9.3) (2023-12-19) 8 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.9.2...v5.9.3) 9 | 10 | ### Changed 11 | - Add v5 to module path [#248](https://github.com/buildkite/buildkite-agent-metrics/pull/248) (@DrJosh9000) 12 | 13 | ### Dependencies 14 | - build(deps): bump golang.org/x/crypto from 0.14.0 to 0.17.0 [#246](https://github.com/buildkite/buildkite-agent-metrics/pull/246) (@dependabot[bot]) 15 | - build(deps): bump cloud.google.com/go/monitoring from 1.16.3 to 1.17.0 [#245](https://github.com/buildkite/buildkite-agent-metrics/pull/245) (@dependabot[bot]) 16 | - build(deps): bump github.com/aws/aws-lambda-go from 1.41.0 to 1.42.0 [#244](https://github.com/buildkite/buildkite-agent-metrics/pull/244) (@dependabot[bot]) 17 | 18 | ## [v5.9.2](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.9.2) (2023-12-12) 19 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.9.1...v5.9.2) 20 | 21 | ### Fixed 22 | - Fix non-Secrets Manager token providers [#243](https://github.com/buildkite/buildkite-agent-metrics/pull/243) (@DrJosh9000) 23 | 24 | ### Changed 25 | - Allow env vars to control debug logging for the lambda [#238](https://github.com/buildkite/buildkite-agent-metrics/pull/238) (@triarius) 26 | 27 | ### Dependencies 28 | - Bump github.com/aws/aws-sdk-go from 1.48.3 to 1.48.4 to 1.48.16 [#237](https://github.com/buildkite/buildkite-agent-metrics/pull/237), [#241](https://github.com/buildkite/buildkite-agent-metrics/pull/241) (@dependabot[bot]) 29 | 30 | ## [v5.9.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.9.1) (2023-11-27) 31 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.9.0...v5.9.1) 32 | 33 | ### Changed 34 | - Support for multiple secrets manager secrets command seperated [#233](https://github.com/buildkite/buildkite-agent-metrics/pull/233) (@lucylura) 35 | 36 | ### Fixed 37 | - Ignore Cluster label/dimension/tag for empty unclustered queues. This may fix continuity errors when clusters are not used [#234](https://github.com/buildkite/buildkite-agent-metrics/pull/234) (@triarius) 38 | 39 | ### Internal 40 | - Document SSM Parameters names may be comma separated [#235](https://github.com/buildkite/buildkite-agent-metrics/pull/235) (@triarius) 41 | 42 | ### Dependencies 43 | - build(deps): bump github.com/aws/aws-sdk-go from 1.47.3 to 1.48.3 [#232](https://github.com/buildkite/buildkite-agent-metrics/pull/232) (@dependabot[bot]) 44 | 45 | ## [v5.9.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.9.0) (2023-11-22) 46 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.8.0...v5.9.0) 47 | 48 | > [!WARNING] 49 | > This release adds a new Cluster label/tag/dimension, which is populated when using agent cluster tokens. This may break continuity with existing time series! 50 | 51 | ### Added 52 | - Collect from multiple clusters [#227](https://github.com/buildkite/buildkite-agent-metrics/pull/227) (@DrJosh9000) 53 | - feat(gcp): add env vars for buildkite queues and gcp project id [#212](https://github.com/buildkite/buildkite-agent-metrics/pull/212) (@NotArpit) 54 | 55 | ### Fixed 56 | - Change build process to better support `provided.al2` [#225](https://github.com/buildkite/buildkite-agent-metrics/pull/225) (@triarius) 57 | - fix(collector): exit on 401 when queues specified [#211](https://github.com/buildkite/buildkite-agent-metrics/pull/211) (@NotArpit) 58 | - Fix another reference to go1.x [#230](https://github.com/buildkite/buildkite-agent-metrics/pull/230) (@jradtilbrook) 59 | 60 | ### Internal 61 | - Split Collect [#226](https://github.com/buildkite/buildkite-agent-metrics/pull/226) (@DrJosh9000) 62 | - Various dependency updates [#206](https://github.com/buildkite/buildkite-agent-metrics/pull/206), [#208](https://github.com/buildkite/buildkite-agent-metrics/pull/208), [#213](https://github.com/buildkite/buildkite-agent-metrics/pull/213), [#215](https://github.com/buildkite/buildkite-agent-metrics/pull/215), [#216](https://github.com/buildkite/buildkite-agent-metrics/pull/216), [#217](https://github.com/buildkite/buildkite-agent-metrics/pull/217), [#218](https://github.com/buildkite/buildkite-agent-metrics/pull/218), [#219](https://github.com/buildkite/buildkite-agent-metrics/pull/219), [#220](https://github.com/buildkite/buildkite-agent-metrics/pull/220), [#221](https://github.com/buildkite/buildkite-agent-metrics/pull/221), [#222](https://github.com/buildkite/buildkite-agent-metrics/pull/222), [#223](https://github.com/buildkite/buildkite-agent-metrics/pull/223) (@dependabot[bot]) 63 | 64 | ## [v5.8.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.8.0) (2023-09-15) 65 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.7.0...v5.8.0) 66 | 67 | ### Changed 68 | - Exit with code 4 on 401 response from the API [#203](https://github.com/buildkite/buildkite-agent-metrics/pull/203) (@NotArpit) 69 | - Bump github.com/aws/aws-sdk-go to 1.45.6 [#191](https://github.com/buildkite/buildkite-agent-metrics/pull/191) [#194](https://github.com/buildkite/buildkite-agent-metrics/pull/194) [#195](https://github.com/buildkite/buildkite-agent-metrics/pull/195) [#197](https://github.com/buildkite/buildkite-agent-metrics/pull/197) [#198](https://github.com/buildkite/buildkite-agent-metrics/pull/198) [#199](https://github.com/buildkite/buildkite-agent-metrics/pull/199) [#201](https://github.com/buildkite/buildkite-agent-metrics/pull/201) (@dependabot[bot]) 70 | - Bump github.com/newrelic/go-agent to 3.24.1+incompatible [#193](https://github.com/buildkite/buildkite-agent-metrics/pull/193) [#196](https://github.com/buildkite/buildkite-agent-metrics/pull/196) (@dependabot[bot]) 71 | 72 | ## [v5.7.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.7.0) (2023-07-24) 73 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.6.0...v5.7.0) 74 | 75 | ### Changed 76 | - Make the timeout configurable [#184](https://github.com/buildkite/buildkite-agent-metrics/pull/184) (@mcncl) 77 | - Update the role ARN used during releases [#162](https://github.com/buildkite/buildkite-agent-metrics/pull/162) (@yob) 78 | - Many dependency version bumps (@dependabot[bot]) 79 | 80 | ## [v5.6.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.6.0) (2023-04-11) 81 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.6...v5.6.0) 82 | 83 | ### Changed 84 | - Bump github.com/aws/aws-sdk-go from 1.44.234 to 1.44.239 [#157](https://github.com/buildkite/buildkite-agent-metrics/pull/157) (@dependabot[bot]) 85 | 86 | ### Fixed 87 | - Handle API errors when querying queue [#139](https://github.com/buildkite/buildkite-agent-metrics/pull/139) (@dyson) 88 | 89 | ## [v5.5.6](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.6) (2023-04-10) 90 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.5...v5.5.6) 91 | 92 | ### Changed 93 | - Remove comments in the middle of a bash command in the release script [#155](https://github.com/buildkite/buildkite-agent-metrics/pull/155) (@triarius) 94 | 95 | ## [v5.5.5](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.5) (2023-04-10) 96 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.4...v5.5.5) 97 | 98 | ### Changed 99 | - add notes about what the job states mean [#130](https://github.com/buildkite/buildkite-agent-metrics/pull/130) (@edmund-huber) 100 | - More fixes to the automated release [#153](https://github.com/buildkite/buildkite-agent-metrics/pull/153) (@triarius) 101 | 102 | ## [v5.5.4](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.4) (2023-04-10) 103 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.3...v5.5.4) 104 | 105 | ### Changed 106 | - Fix `--verify-tag` not available in github-cli 2.20 for release automation [#151](https://github.com/buildkite/buildkite-agent-metrics/pull/151) (@triarius) 107 | 108 | ## [v5.5.3](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.3) (2023-04-10) 109 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.2...v5.5.3) 110 | 111 | ### Changed 112 | - More fixes to release automation [#149](https://github.com/buildkite/buildkite-agent-metrics/pull/149) (@triarius) 113 | 114 | ## [v5.5.2](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.2) (2023-04-09) 115 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.1...v5.5.2) 116 | 117 | ### Changed 118 | - Attempt to fix release process [#147](https://github.com/buildkite/buildkite-agent-metrics/pull/147) (@triarius) 119 | 120 | ## [v5.5.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.1) (2023-04-05) 121 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.5.0...v5.5.1) 122 | 123 | ### Changed 124 | - Update release process to generate checksums [#145](https://github.com/buildkite/buildkite-agent-metrics/pull/145) (@triarius) 125 | - Allow dependabot to slowly keep gomod up to date [#135](https://github.com/buildkite/buildkite-agent-metrics/pull/135) (@yob) 126 | - Dependency updates (@dependabot[bot]) 127 | 128 | ## [v5.5.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.5.0) (2023-03-16) 129 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.4.0...v5.5.0) 130 | 131 | ### Changed 132 | - Fixed release process with OIDC [#133](https://github.com/buildkite/buildkite-agent-metrics/pull/133) [#134](https://github.com/buildkite/buildkite-agent-metrics/pull/134) (@yob) 133 | - Update Go (1.20), Alpine (3.17), and all modules [#131](https://github.com/buildkite/buildkite-agent-metrics/pull/131) (@DrJosh9000) 134 | 135 | ## [v5.4.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.4.0) (2022-06-10) 136 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.3.0...v5.4.0) 137 | 138 | ### Changed 139 | - Standardize http.Client collector configurations [#121](https://github.com/buildkite/buildkite-agent-metrics/pull/121) (@alloveras) 140 | - Update AWS Lambda SDK v1.6.0 -> v1.28.0, add a lambda-specific dockerfile [#120](https://github.com/buildkite/buildkite-agent-metrics/pull/120) (@ohookins) 141 | 142 | ## [v5.3.0](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.2.1...v5.3.0) (2021-07-16) 143 | 144 | ### Addded 145 | 146 | * Support reading an agent token from the environment [#116](https://github.com/buildkite/buildkite-agent-metrics/pull/116) ([@cole-h](https://github.com/cole-h)) 147 | 148 | ## [v5.2.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.2.0) (2021-07-01) 149 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.2.0...v5.2.1) 150 | 151 | ### Added 152 | 153 | * Support for more AWS Regions (af-south-1, ap-east-1, ap-southeast-2, ap-southeast-1, eu-south-1, me-south-1) [#109](https://github.com/buildkite/buildkite-agent-metrics/pull/109) 154 | * ARM64 binaries for Linux and macOS 155 | 156 | ### Changed 157 | 158 | * Build using golang 1.16 159 | * Update newrelic/go-agent from v2.7.0 to v3.0.0 [#111](https://github.com/buildkite/buildkite-agent-metrics/pull/111) (@mallyvai) 160 | 161 | ## [v5.2.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.2.0) (2020-03-05) 162 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.1.0...v5.2.0) 163 | 164 | ### Changed 165 | - Add support for AWS SecretsManager as BK token provider [#98](https://github.com/buildkite/buildkite-agent-metrics/pull/98) (@alloveras) 166 | - Don't exit on when error is encountered [#94](https://github.com/buildkite/buildkite-agent-metrics/pull/94) (@amalucelli) 167 | - Stackdriver: Use organization specific metric names. [#87](https://github.com/buildkite/buildkite-agent-metrics/pull/87) (@philwo) 168 | - Fix typo in README.md. [#88](https://github.com/buildkite/buildkite-agent-metrics/pull/88) (@philwo) 169 | 170 | ## [v5.1.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.1.0) (2019-05-18) 171 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v5.0.0...v5.1.0) 172 | 173 | ### Changed 174 | - Support multiple queue params [#86](https://github.com/buildkite/buildkite-agent-metrics/pull/86) (@lox) 175 | - Add New Relic backend [#85](https://github.com/buildkite/buildkite-agent-metrics/pull/85) (@chloehutchinson) 176 | - Add Stackdriver Backend [#78](https://github.com/buildkite/buildkite-agent-metrics/pull/78) (@winfieldj) 177 | 178 | ## [v5.0.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v5.0.0) (2019-05-05) 179 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v4.1.2...v5.0.0) 180 | 181 | ### Changed 182 | - Add BusyAgentPercentage metric [#80](https://github.com/buildkite/buildkite-agent-metrics/pull/80) (@arromer) 183 | - Drop metrics with only queue dimension [#82](https://github.com/buildkite/buildkite-agent-metrics/pull/82) (@lox) 184 | - Add WaitingJobsCount metric [#81](https://github.com/buildkite/buildkite-agent-metrics/pull/81) (@lox) 185 | - Read AWS_REGION for cloudwatch, default to us-east-1 [#79](https://github.com/buildkite/buildkite-agent-metrics/pull/79) (@lox) 186 | - Add a Dockerfile [#77](https://github.com/buildkite/buildkite-agent-metrics/pull/77) (@amalucelli) 187 | - Enforce Buildkite-Agent-Metrics-Poll-Duration header [#83](https://github.com/buildkite/buildkite-agent-metrics/pull/83) (@lox) 188 | - Add support for reading buildkite token from ssm [#76](https://github.com/buildkite/buildkite-agent-metrics/pull/76) (@arromer) 189 | - Update bucket publishing for new regions [#74](https://github.com/buildkite/buildkite-agent-metrics/pull/74) (@lox) 190 | - Update the readme to have the correct Environment variables and expla… [#73](https://github.com/buildkite/buildkite-agent-metrics/pull/73) (@bmbentson) 191 | 192 | ## [v4.1.3](https://github.com/buildkite/buildkite-agent-metrics/tree/v4.1.3) (2019-03-26) 193 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v4.1.2...v4.1.3) 194 | 195 | ### Changed 196 | - Update bucket publishing for new regions [#74](https://github.com/buildkite/buildkite-agent-metrics/pull/74) (@lox) 197 | - Update the readme to have the correct Environment variables and expla… [#73](https://github.com/buildkite/buildkite-agent-metrics/pull/73) (@bmbentson) 198 | 199 | ## [v4.1.2](https://github.com/buildkite/buildkite-agent-metrics/tree/v4.1.2) (2019-01-21) 200 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v4.1.1...v4.1.2) 201 | 202 | ### Fixed 203 | - Add back cloudwatch metric with only Queue dimension [#69](https://github.com/buildkite/buildkite-agent-metrics/pull/69) (@lox) 204 | 205 | ## [v4.1.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v4.1.1) (2019-01-21) 206 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v4.1.0...v4.1.1) 207 | 208 | ### Fixed 209 | - Add missing organization dimension to per-queue metrics [#68](https://github.com/buildkite/buildkite-agent-metrics/pull/68) (@lox) 210 | 211 | ## [v4.1.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v4.1.0) (2019-01-03) 212 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v4.0.0...v4.1.0) 213 | 214 | ### Changed 215 | - Expose org slug as a cloudwatch dimension [#67](https://github.com/buildkite/buildkite-agent-metrics/pull/67) (@lox) 216 | - Clarify lambda handler in README, add example [#66](https://github.com/buildkite/buildkite-agent-metrics/pull/66) (@lox) 217 | 218 | ## [v4.0.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v4.0.0) (2018-11-01) 219 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v3.1.0...v4.0.0) 220 | 221 | ### Changed 222 | - Update dependencies [#62](https://github.com/buildkite/buildkite-agent-metrics/pull/62) (@lox) 223 | - Move to golang 1.11 [#61](https://github.com/buildkite/buildkite-agent-metrics/pull/61) (@lox) 224 | - Move to aws lambda go [#60](https://github.com/buildkite/buildkite-agent-metrics/pull/60) (@lox) 225 | - Remove unused vendors [#57](https://github.com/buildkite/buildkite-agent-metrics/pull/57) (@paulolai) 226 | - Update references to github.com/buildkite/buildkite-metrics [#56](https://github.com/buildkite/buildkite-agent-metrics/pull/56) (@paulolai) 227 | - Update readme to reflect elastic stack's changed paths [#54](https://github.com/buildkite/buildkite-agent-metrics/pull/54) (@lox) 228 | - Update capitalization on Datadog [#52](https://github.com/buildkite/buildkite-agent-metrics/pull/52) (@irabinovitch) 229 | 230 | ## [v3.1.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v3.1.0) (2018-08-17) 231 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v3.0.1...v3.1.0) 232 | 233 | ### Changed 234 | - Add a 5 second timeout for metrics requests [#50](https://github.com/buildkite/buildkite-agent-metrics/pull/50) (@lox) 235 | - Improve running docs [#49](https://github.com/buildkite/buildkite-agent-metrics/pull/49) (@lox) 236 | - Allow a custom cloudwatch dimension flag [#46](https://github.com/buildkite/buildkite-agent-metrics/pull/46) (@lox) 237 | 238 | ## [v3.0.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v3.0.1) (2018-07-12) 239 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v3.0.0...v3.0.1) 240 | 241 | ### Changed 242 | - Reset prometheus queue gauges to prevent stale values persisting [#45](https://github.com/buildkite/buildkite-agent-metrics/pull/45) (@majolo) 243 | 244 | ## [v3.0.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v3.0.0) (2018-04-17) 245 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v2.1.0...v3.0.0) 246 | 247 | ### Changed 248 | - Update buildkite-metrics to use the agent metrics api [#40](https://github.com/buildkite/buildkite-agent-metrics/pull/40) (@sj26) 249 | 250 | ## [v2.1.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v2.1.0) (2018-03-07) 251 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v2.0.2...v2.1.0) 252 | 253 | ### Changed 254 | - Add Prometheus metrics backend [#39](https://github.com/buildkite/buildkite-agent-metrics/pull/39) (@martinbaillie) 255 | - Ensure statsd commands are flushed after each run [#38](https://github.com/buildkite/buildkite-agent-metrics/pull/38) (@theist) 256 | - Small typo in readme [#35](https://github.com/buildkite/buildkite-agent-metrics/pull/35) (@theist) 257 | 258 | ## [v2.0.2](https://github.com/buildkite/buildkite-agent-metrics/tree/v2.0.2) (2018-01-07) 259 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v2.0.1...v2.0.2) 260 | 261 | Skipped version due to release issues. 262 | 263 | ## [v2.0.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v2.0.1) (2018-01-07) 264 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v2.0.0...v2.0.1) 265 | 266 | Skipped version due to release issues. 267 | 268 | ## [v2.0.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v2.0.0) (2017-11-27) 269 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.6.0...v2.0.0) 270 | 271 | ### Changed 272 | 273 | ## [v1.6.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.6.0) (2017-11-22) 274 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.5.0...v1.6.0) 275 | 276 | ### Changed 277 | - Add an endpoint and better user-agent information [#34](https://github.com/buildkite/buildkite-agent-metrics/pull/34) (@lox) 278 | - Punycode pipeline names [#33](https://github.com/buildkite/buildkite-agent-metrics/pull/33) (@rbvigilante) 279 | 280 | ## [v1.5.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.5.0) (2017-08-11) 281 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.4.2...v1.5.0) 282 | 283 | ### Changed 284 | - Add retry for failed bk calls to lambda [#30](https://github.com/buildkite/buildkite-agent-metrics/pull/30) (@lox) 285 | 286 | ## [v1.4.2](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.4.2) (2017-03-07) 287 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.4.1...v1.4.2) 288 | 289 | ### Changed 290 | - Add BUILDKITE_QUIET support to lambda [#28](https://github.com/buildkite/buildkite-agent-metrics/pull/28) (@lox) 291 | - Upload lambda to region specific buckets [#26](https://github.com/buildkite/buildkite-agent-metrics/pull/26) (@lox) 292 | 293 | ## [v1.4.1](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.4.1) (2016-12-20) 294 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.4.0...v1.4.1) 295 | 296 | ### Changed 297 | - Support the queue parameter and logs in lambda func [#25](https://github.com/buildkite/buildkite-agent-metrics/pull/25) (@lox) 298 | 299 | ## [v1.4.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.4.0) (2016-12-19) 300 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.3.0...v1.4.0) 301 | 302 | ### Changed 303 | - Add StatsD support [#24](https://github.com/buildkite/buildkite-agent-metrics/pull/24) (@callumj) 304 | 305 | ## [v1.3.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.3.0) (2016-12-19) 306 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.2.0...v1.3.0) 307 | 308 | ### Changed 309 | - Correctly filter stats by queue [#23](https://github.com/buildkite/buildkite-agent-metrics/pull/23) (@lox) 310 | - Moved collector into subpackage with tests [#22](https://github.com/buildkite/buildkite-agent-metrics/pull/22) (@lox) 311 | - Debug flag now shows useful debugging, added dry-run [#20](https://github.com/buildkite/buildkite-agent-metrics/pull/20) (@lox) 312 | - Add a lambda function for executing stats [#18](https://github.com/buildkite/buildkite-agent-metrics/pull/18) (@lox) 313 | - Add a quiet flag to close #9 [#14](https://github.com/buildkite/buildkite-agent-metrics/pull/14) (@lox) 314 | - Revert "Support multiple queues via --queue" [#16](https://github.com/buildkite/buildkite-agent-metrics/pull/16) (@sj26) 315 | - Support multiple queues via --queue [#13](https://github.com/buildkite/buildkite-agent-metrics/pull/13) (@lox) 316 | - Increase page size [#12](https://github.com/buildkite/buildkite-agent-metrics/pull/12) (@lox) 317 | - Replace glide with govendor, bump vendors [#11](https://github.com/buildkite/buildkite-agent-metrics/pull/11) (@lox) 318 | - Improve error logging [#7](https://github.com/buildkite/buildkite-agent-metrics/pull/7) (@yeungda-rea) 319 | 320 | ## [v1.2.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.2.0) (2016-06-22) 321 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.1.0...v1.2.0) 322 | 323 | ### Changed 324 | - OpenJobsCount [#3](https://github.com/buildkite/buildkite-agent-metrics/pull/3) (@eliank) 325 | - Add a -queue flag to allow filtering metrics by queue [#1](https://github.com/buildkite/buildkite-agent-metrics/pull/1) (@lox) 326 | 327 | ## [v1.1.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.1.0) (2016-04-07) 328 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/v1.0.0...v1.1.0) 329 | 330 | ### Changed 331 | 332 | ## [v1.0.0](https://github.com/buildkite/buildkite-agent-metrics/tree/v1.0.0) (2016-04-07) 333 | [Full Changelog](https://github.com/buildkite/buildkite-agent-metrics/compare/78a3ded05dcf...v1.0.0) 334 | 335 | Initial release 336 | --------------------------------------------------------------------------------