├── .github ├── prerelease.sh └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.pre.yml ├── .goreleaser.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── gitlab-ci-pipelines-exporter │ └── main.go └── tools │ └── man │ └── main.go ├── docs ├── configuration_syntax.md ├── images │ ├── grafana_dashboard_environments.jpg │ ├── grafana_dashboard_environments_example.png │ ├── grafana_dashboard_example.png │ ├── grafana_dashboard_jobs.jpg │ ├── grafana_dashboard_jobs_example.png │ ├── grafana_dashboard_pipelines.jpg │ ├── grafana_dashboard_pipelines_example.png │ ├── monitor_cli_example.gif │ ├── prometheus_metrics_list_example.png │ ├── prometheus_pipeline_status_metric_example.png │ ├── prometheus_targets_example.png │ ├── webhook_configuration.png │ └── webhook_trigger_test.png └── metrics.md ├── examples ├── ha-setup │ ├── README.md │ ├── docker-compose.yml │ └── gitlab-ci-pipelines-exporter.yml ├── opentelemetry │ ├── README.md │ ├── docker-compose.yml │ ├── gitlab-ci-pipelines-exporter.yml │ ├── grafana │ │ └── datasources.yml │ ├── otel-collector-config.yml │ └── prometheus │ │ └── config.yml ├── quickstart │ ├── README.md │ ├── docker-compose.yml │ ├── gitlab-ci-pipelines-exporter.yml │ ├── grafana │ │ ├── dashboards.yml │ │ ├── dashboards │ │ │ ├── dashboard_environments.json │ │ │ ├── dashboard_jobs.json │ │ │ └── dashboard_pipelines.json │ │ └── datasources.yml │ └── prometheus │ │ └── config.yml └── webhooks │ ├── README.md │ ├── docker-compose.yml │ └── gitlab-ci-pipelines-exporter.yml ├── go.mod ├── go.sum ├── internal ├── cli │ ├── cli.go │ └── cli_test.go └── cmd │ ├── monitor.go │ ├── run.go │ ├── run_test.go │ ├── utils.go │ ├── utils_test.go │ └── validate.go ├── pkg ├── config │ ├── config.go │ ├── config_test.go │ ├── global.go │ ├── parser.go │ ├── parser_test.go │ ├── project.go │ ├── project_test.go │ ├── wildcard.go │ └── wildcard_test.go ├── controller │ ├── collectors.go │ ├── collectors_test.go │ ├── controller.go │ ├── controller_test.go │ ├── environments.go │ ├── environments_test.go │ ├── garbage_collector.go │ ├── garbage_collector_test.go │ ├── handlers.go │ ├── handlers_test.go │ ├── jobs.go │ ├── jobs_test.go │ ├── metadata.go │ ├── metadata_test.go │ ├── metrics.go │ ├── metrics_test.go │ ├── pipelines.go │ ├── pipelines_test.go │ ├── projects.go │ ├── projects_test.go │ ├── refs.go │ ├── refs_test.go │ ├── scheduler.go │ ├── scheduler_test.go │ ├── store.go │ ├── store_test.go │ ├── webhooks.go │ └── webhooks_test.go ├── gitlab │ ├── branches.go │ ├── branches_test.go │ ├── client.go │ ├── client_test.go │ ├── environments.go │ ├── environments_test.go │ ├── jobs.go │ ├── jobs_test.go │ ├── pipelines.go │ ├── pipelines_test.go │ ├── projects.go │ ├── projects_test.go │ ├── repositories.go │ ├── repositories_test.go │ ├── tags.go │ ├── tags_test.go │ ├── version.go │ └── version_test.go ├── monitor │ ├── client │ │ └── client.go │ ├── monitor.go │ ├── protobuf │ │ ├── monitor.pb.go │ │ ├── monitor.proto │ │ └── monitor_grpc.pb.go │ ├── server │ │ └── server.go │ └── ui │ │ └── ui.go ├── ratelimit │ ├── local.go │ ├── local_test.go │ ├── ratelimit.go │ ├── ratelimit_test.go │ ├── redis.go │ └── redis_test.go ├── schemas │ ├── deployments.go │ ├── deployments_test.go │ ├── environments.go │ ├── environments_test.go │ ├── jobs.go │ ├── jobs_test.go │ ├── metric.go │ ├── metric_test.go │ ├── pipelines.go │ ├── pipelines_test.go │ ├── projects.go │ ├── projects_test.go │ ├── ref.go │ ├── ref_test.go │ ├── tasks.go │ └── tasks_test.go └── store │ ├── local.go │ ├── local_test.go │ ├── redis.go │ ├── redis_test.go │ ├── store.go │ └── store_test.go └── renovate.json5 /.github/prerelease.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | RELEASE_ID=$(curl -sL https://api.github.com/repos/${REPOSITORY}/releases/tags/edge | jq -r .id) 6 | HEAD_SHA=$(curl -sL https://api.github.com/repos/${REPOSITORY}/git/refs/heads/main | jq -r .object.sha) 7 | PRERELEASE_TAG=$(git describe --always --abbrev=7 --tags --exclude=edge) 8 | 9 | # Bump the edge tag to the head of main 10 | curl -sL \ 11 | -X PATCH \ 12 | -u "_:${GITHUB_TOKEN}" \ 13 | -H "Accept: application/vnd.github.v3+json" \ 14 | -d '{"sha":"'${HEAD_SHA}'","force":true}' \ 15 | "https://api.github.com/repos/${REPOSITORY}/git/refs/tags/edge" 16 | 17 | # Ensure we execute some cleanup functions on exit 18 | function cleanup { 19 | git tag -d ${PRERELEASE_TAG} || true 20 | git fetch --tags -f || true 21 | } 22 | trap cleanup EXIT 23 | 24 | # Create some directories to avoid race errors on snap packages build 25 | mkdir -p ${HOME}/.cache/snapcraft/{download,stage-packages} 26 | 27 | # Build the binaries using a prerelease tag 28 | git tag -d edge 29 | git tag -f ${PRERELEASE_TAG} 30 | goreleaser release \ 31 | --clean \ 32 | --skip=validate \ 33 | -f .goreleaser.pre.yml 34 | 35 | # Delete existing assets from the edge prerelease on GitHub 36 | for asset_url in $(curl -sL -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${REPOSITORY}/releases/tags/edge | jq -r ".assets[].url"); do 37 | echo "deleting edge release asset: ${asset_url}" 38 | curl -sL \ 39 | -X DELETE \ 40 | -u "_:${GITHUB_TOKEN}" \ 41 | ${asset_url} 42 | done 43 | 44 | # Upload new assets onto the edge prerelease on GitHub 45 | for asset in $(find dist -type f -name "${NAME}_edge*"); do 46 | echo "uploading ${asset}.." 47 | curl -sL \ 48 | -u "_:${GITHUB_TOKEN}" \ 49 | -H "Accept: application/vnd.github.v3+json" \ 50 | -H "Content-Type: $(file -b --mime-type ${asset})" \ 51 | --data-binary @${asset} \ 52 | "https://uploads.github.com/repos/${REPOSITORY}/releases/${RELEASE_ID}/assets?name=$(basename $asset)" 53 | done 54 | 55 | # Upload snaps to the edge channel 56 | find dist -type f -name "*.snap" -exec snapcraft upload --release edge '{}' \; 57 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: release 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v[0-9]+.[0-9]+.[0-9]+' 8 | branches: 9 | - main 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-24.04 14 | 15 | env: 16 | DOCKER_CLI_EXPERIMENTAL: 'enabled' 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3 26 | 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3 29 | 30 | - name: docker.io Login 31 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3 32 | with: 33 | registry: docker.io 34 | username: ${{ github.repository_owner }} 35 | password: ${{ secrets.DOCKER_HUB_TOKEN }} 36 | 37 | - name: ghcr.io login 38 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3 39 | with: 40 | registry: ghcr.io 41 | username: ${{ github.repository_owner }} 42 | password: ${{ secrets.GH_PAT }} 43 | 44 | - name: quay.io Login 45 | uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3 46 | with: 47 | registry: quay.io 48 | username: ${{ github.repository_owner }} 49 | password: ${{ secrets.QUAY_TOKEN }} 50 | 51 | - name: Snapcraft config 52 | uses: samuelmeuli/action-snapcraft@fceeb3c308e76f3487e72ef608618de625fb7fe8 # v3 53 | 54 | - name: Set up Go 55 | uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 56 | with: 57 | go-version: '1.22' 58 | 59 | - name: Import GPG key 60 | uses: crazy-max/ghaction-import-gpg@cb9bde2e2525e640591a934b1fd28eef1dcaf5e5 # v6 61 | with: 62 | gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} 63 | passphrase: ${{ secrets.GPG_PASSPHRASE }} 64 | 65 | - name: Install goreleaser 66 | uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6 67 | with: 68 | version: v1.24.0 69 | install-only: true 70 | 71 | - name: Run goreleaser 72 | run: make ${{ github.ref == 'refs/heads/main' && 'pre' || '' }}release 73 | env: 74 | GITHUB_TOKEN: ${{ secrets.GH_PAT }} 75 | SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }} 76 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: test 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - 'v[0-9]+.[0-9]+.[0-9]+' 10 | pull_request: 11 | branches: 12 | - main 13 | 14 | jobs: 15 | test: 16 | strategy: 17 | matrix: 18 | os: 19 | - ubuntu-24.04 20 | - macos-13 21 | - windows-2022 22 | 23 | runs-on: ${{ matrix.os }} 24 | 25 | steps: 26 | - name: Checkout code 27 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 28 | 29 | - name: Install Go 30 | uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 31 | with: 32 | go-version: '1.23' 33 | 34 | - name: Lint 35 | if: ${{ matrix.os == 'ubuntu-24.04' }} 36 | run: make lint 37 | 38 | - name: Test 39 | run: make test 40 | 41 | - name: Publish coverage to coveralls.io 42 | uses: shogo82148/actions-goveralls@785c9d68212c91196d3994652647f8721918ba11 # v1 43 | if: ${{ matrix.os == 'ubuntu-24.04' }} 44 | with: 45 | path-to-profile: coverage.out 46 | 47 | - name: Build 48 | run: make build 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | helpers 3 | vendor 4 | **/*.tgz 5 | /gitlab-ci-pipelines-exporter.yml 6 | **/.DS_Store 7 | gitlab-ci-pipelines-exporter 8 | !cmd/gitlab-ci-pipelines-exporter 9 | !examples/**/gitlab-ci-pipelines-exporter 10 | coverage.out 11 | .*.sock 12 | .idea 13 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable-all: true 3 | disable: 4 | # Deprecated 5 | - gomnd 6 | 7 | # We don't want these ones 8 | - forcetypeassert 9 | - gochecknoglobals 10 | - godox 11 | - ireturn 12 | - nakedret 13 | - testpackage 14 | - varnamelen 15 | - interfacebloat 16 | - wsl 17 | 18 | # TODO 19 | - tagliatelle 20 | - promlinter 21 | - paralleltest 22 | - gocognit 23 | - gomoddirectives 24 | - forbidigo 25 | - goconst 26 | - mnd 27 | - lll 28 | - dupl 29 | - depguard 30 | - tagalign 31 | 32 | linters-settings: 33 | funlen: 34 | lines: -1 # (disabled) 35 | statements: 100 36 | 37 | cyclop: 38 | max-complexity: 20 39 | 40 | lll: 41 | line-length: 140 42 | 43 | nestif: 44 | min-complexity: 18 45 | 46 | gci: 47 | sections: 48 | - standard 49 | - default 50 | - prefix(github.com/mvisonneau) 51 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ## 2 | # BUILD CONTAINER 3 | ## 4 | 5 | FROM alpine:3.21@sha256:21dc6063fd678b478f57c0e13f47560d0ea4eeba26dfc947b2a4f81f686b9f45 as certs 6 | 7 | RUN \ 8 | apk add --no-cache ca-certificates 9 | 10 | ## 11 | # RELEASE CONTAINER 12 | ## 13 | 14 | FROM busybox:1.37-glibc@sha256:c598938e58d0efcc5a01efe9059d113f22970914e05e39ab2a597a10f9db9bdc 15 | 16 | WORKDIR / 17 | 18 | COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 19 | COPY gitlab-ci-pipelines-exporter /usr/local/bin/ 20 | 21 | # Run as nobody user 22 | USER 65534 23 | 24 | EXPOSE 8080 25 | 26 | ENTRYPOINT ["/usr/local/bin/gitlab-ci-pipelines-exporter"] 27 | CMD ["run"] 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME := gitlab-ci-pipelines-exporter 2 | FILES := $(shell git ls-files */*.go) 3 | COVERAGE_FILE := coverage.out 4 | REPOSITORY := mvisonneau/$(NAME) 5 | .DEFAULT_GOAL := help 6 | GOLANG_VERSION := 1.23 7 | 8 | .PHONY: fmt 9 | fmt: ## Format source code 10 | go run mvdan.cc/gofumpt@v0.7.0 -w $(shell git ls-files **/*.go) 11 | go run github.com/daixiang0/gci@v0.13.5 write -s standard -s default -s "prefix(github.com/mvisonneau)" . 12 | 13 | .PHONY: lint 14 | lint: ## Run all lint related tests upon the codebase 15 | go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2 run -v --fast 16 | 17 | .PHONY: test 18 | test: ## Run the tests against the codebase 19 | @rm -rf $(COVERAGE_FILE) 20 | go test -v -count=1 -race ./... -coverprofile=$(COVERAGE_FILE) 21 | @go tool cover -func $(COVERAGE_FILE) | awk '/^total/ {print "coverage: " $$3}' 22 | 23 | .PHONY: coverage 24 | coverage: ## Prints coverage report 25 | go tool cover -func $(COVERAGE_FILE) 26 | 27 | .PHONY: install 28 | install: ## Build and install locally the binary (dev purpose) 29 | go install ./cmd/$(NAME) 30 | 31 | .PHONY: build 32 | build: ## Build the binaries using local GOOS 33 | go build ./cmd/$(NAME) 34 | 35 | .PHONY: release 36 | release: ## Build & release the binaries (stable) 37 | mkdir -p ${HOME}/.cache/snapcraft/download 38 | mkdir -p ${HOME}/.cache/snapcraft/stage-packages 39 | git tag -d edge 40 | goreleaser release --clean 41 | find dist -type f -name "*.snap" -exec snapcraft upload --release stable,edge '{}' \; 42 | 43 | .PHONY: protoc 44 | protoc: ## Generate golang from .proto files 45 | @command -v protoc 2>&1 >/dev/null || (echo "protoc needs to be available in PATH: https://github.com/protocolbuffers/protobuf/releases"; false) 46 | @command -v protoc-gen-go 2>&1 >/dev/null || go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 47 | protoc \ 48 | --go_out=. --go_opt=paths=source_relative \ 49 | --go-grpc_out=. --go-grpc_opt=paths=source_relative \ 50 | pkg/monitor/protobuf/monitor.proto 51 | 52 | .PHONY: prerelease 53 | prerelease: ## Build & prerelease the binaries (edge) 54 | @\ 55 | REPOSITORY=$(REPOSITORY) \ 56 | NAME=$(NAME) \ 57 | GITHUB_TOKEN=$(GITHUB_TOKEN) \ 58 | .github/prerelease.sh 59 | 60 | .PHONY: clean 61 | clean: ## Remove binary if it exists 62 | rm -f $(NAME) 63 | 64 | .PHONY: coverage-html 65 | coverage-html: ## Generates coverage report and displays it in the browser 66 | go tool cover -html=coverage.out 67 | 68 | .PHONY: dev-env 69 | dev-env: ## Build a local development environment using Docker 70 | @docker run -it --rm \ 71 | -v $(shell pwd):/go/src/github.com/mvisonneau/$(NAME) \ 72 | -w /go/src/github.com/mvisonneau/$(NAME) \ 73 | -p 8080:8080 \ 74 | golang:$(GOLANG_VERSION) \ 75 | /bin/bash -c '\ 76 | git config --global --add safe.directory $$(pwd);\ 77 | make install;\ 78 | bash\ 79 | ' 80 | 81 | .PHONY: is-git-dirty 82 | is-git-dirty: ## Tests if git is in a dirty state 83 | @git status --porcelain 84 | @test $(shell git status --porcelain | grep -c .) -eq 0 85 | 86 | .PHONY: man-pages 87 | man-pages: ## Generates man pages 88 | rm -rf helpers/manpages 89 | mkdir -p helpers/manpages 90 | go run ./cmd/tools/man | gzip -c -9 >helpers/manpages/$(NAME).1.gz 91 | 92 | .PHONY: autocomplete-scripts 93 | autocomplete-scripts: ## Download CLI autocompletion scripts 94 | rm -rf helpers/autocomplete 95 | mkdir -p helpers/autocomplete 96 | curl -sL https://raw.githubusercontent.com/urfave/cli/v2.27.1/autocomplete/bash_autocomplete > helpers/autocomplete/bash 97 | curl -sL https://raw.githubusercontent.com/urfave/cli/v2.27.1/autocomplete/zsh_autocomplete > helpers/autocomplete/zsh 98 | curl -sL https://raw.githubusercontent.com/urfave/cli/v2.27.1/autocomplete/powershell_autocomplete.ps1 > helpers/autocomplete/ps1 99 | 100 | .PHONY: all 101 | all: lint test build coverage ## Test, builds and ship package for all supported platforms 102 | 103 | .PHONY: help 104 | help: ## Displays this help 105 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 106 | -------------------------------------------------------------------------------- /cmd/gitlab-ci-pipelines-exporter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/internal/cli" 7 | ) 8 | 9 | var version = "devel" 10 | 11 | func main() { 12 | cli.Run(version, os.Args) 13 | } 14 | -------------------------------------------------------------------------------- /cmd/tools/man/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/internal/cli" 8 | ) 9 | 10 | var version = "devel" 11 | 12 | func main() { 13 | fmt.Println(cli.NewApp(version, time.Now()).ToMan()) 14 | } 15 | -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_environments.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_environments.jpg -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_environments_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_environments_example.png -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_example.png -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_jobs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_jobs.jpg -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_jobs_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_jobs_example.png -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_pipelines.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_pipelines.jpg -------------------------------------------------------------------------------- /docs/images/grafana_dashboard_pipelines_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/grafana_dashboard_pipelines_example.png -------------------------------------------------------------------------------- /docs/images/monitor_cli_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/monitor_cli_example.gif -------------------------------------------------------------------------------- /docs/images/prometheus_metrics_list_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/prometheus_metrics_list_example.png -------------------------------------------------------------------------------- /docs/images/prometheus_pipeline_status_metric_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/prometheus_pipeline_status_metric_example.png -------------------------------------------------------------------------------- /docs/images/prometheus_targets_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/prometheus_targets_example.png -------------------------------------------------------------------------------- /docs/images/webhook_configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/webhook_configuration.png -------------------------------------------------------------------------------- /docs/images/webhook_trigger_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvisonneau/gitlab-ci-pipelines-exporter/088f5f947722b3021b84cb829e6f1fdf3cfda753/docs/images/webhook_trigger_test.png -------------------------------------------------------------------------------- /examples/ha-setup/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.8' 3 | services: 4 | redis: 5 | image: docker.io/bitnami/redis:6.2 6 | ports: 7 | - 6379:6379 8 | environment: 9 | ALLOW_EMPTY_PASSWORD: 'yes' 10 | 11 | gitlab-ci-pipelines-exporter-1: &gitlab-ci-pipelines-exporter 12 | image: quay.io/mvisonneau/gitlab-ci-pipelines-exporter:v0.5.10 13 | # You can comment out the image name and use the following statement 14 | # to build the image against the current version of the repository 15 | #build: ../.. 16 | ports: 17 | - 8081:8080 18 | links: 19 | - redis 20 | environment: 21 | GCPE_CONFIG: /etc/gitlab-ci-pipelines-exporter.yml 22 | GCPE_INTERNAL_MONITORING_LISTENER_ADDRESS: tcp://127.0.0.1:8082 23 | volumes: 24 | - type: bind 25 | source: ./gitlab-ci-pipelines-exporter.yml 26 | target: /etc/gitlab-ci-pipelines-exporter.yml 27 | 28 | gitlab-ci-pipelines-exporter-2: 29 | <<: *gitlab-ci-pipelines-exporter 30 | ports: 31 | - 8082:8080 32 | 33 | gitlab-ci-pipelines-exporter-3: 34 | <<: *gitlab-ci-pipelines-exporter 35 | ports: 36 | - 8083:8080 37 | 38 | networks: 39 | default: 40 | driver: bridge 41 | -------------------------------------------------------------------------------- /examples/ha-setup/gitlab-ci-pipelines-exporter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | log: 3 | level: debug 4 | 5 | gitlab: 6 | url: https://gitlab.com 7 | token: 8 | 9 | redis: 10 | url: redis://redis:6379 11 | 12 | # Example public projects to monitor 13 | projects: 14 | - name: gitlab-org/gitlab-runner 15 | - name: gitlab-org/charts/auto-deploy-app 16 | -------------------------------------------------------------------------------- /examples/opentelemetry/README.md: -------------------------------------------------------------------------------- 1 | # Example monitoring of gitlab-ci-pipelines-exporter with Jaeger 2 | 3 | ## Requirements 4 | 5 | - **~5 min of your time** 6 | - A personal access token on [gitlab.com](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) (or your own instance) with `read_api` scope 7 | - [git](https://git-scm.com/) & [docker-compose](https://docs.docker.com/compose/) 8 | 9 | ## 🚀 10 | 11 | ```bash 12 | # Clone this repository 13 | ~$ git clone https://github.com/mvisonneau/gitlab-ci-pipelines-exporter.git 14 | ~$ cd gitlab-ci-pipelines-exporter/examples/opentelemetry 15 | 16 | # Provide your personal GitLab API access token (needs read_api permissions) 17 | ~$ sed -i 's//xXF_xxjV_xxyzxzz/' gitlab-ci-pipelines-exporter.yml 18 | 19 | # Start gitlab-ci-pipelines-exporter, prometheus and grafana containers ! 20 | ~$ docker-compose up -d 21 | Creating network "opentelemetry_default" with driver "bridge" 22 | Creating opentelemetry_jaeger_1 ... done 23 | Creating opentelemetry_redis_1 ... done 24 | Creating opentelemetry_otel-collector_1 ... done 25 | Creating opentelemetry_gitlab-ci-pipelines-exporter_1 ... done 26 | Creating opentelemetry_prometheus_1 ... done 27 | Creating opentelemetry_grafana_1 ... done 28 | ``` 29 | 30 | You should now have a stack completely configured and accessible at these locations: 31 | 32 | - `gitlab-ci-pipelines-exporter`: [http://localhost:8080/metrics](http://localhost:8080/metrics) 33 | - `jaeger`: [http://localhost:16686](http://localhost:16686) 34 | - `prometheus`: [http://localhost:9090](http://localhost:9090) 35 | - `grafana`: [http://localhost:3000](http://localhost:3000) (if you want/need to login, creds are _admin/admin_) 36 | 37 | ## Use and troubleshoot 38 | 39 | ### Validate that containers are running 40 | 41 | ```bash 42 | ~$ docker-compose ps 43 | Name Command State Ports 44 | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- 45 | opentelemetry_gitlab-ci-pipelines-exporter_1 /usr/local/bin/gitlab-ci-p ... Up 0.0.0.0:8080->8080/tcp 46 | opentelemetry_grafana_1 /run.sh Up 0.0.0.0:3000->3000/tcp 47 | opentelemetry_jaeger_1 /go/bin/all-in-one-linux Up 14250/tcp, 14268/tcp, 0.0.0.0:16686->16686/tcp, 5775/udp, 5778/tcp, 48 | 6831/udp, 6832/udp 49 | opentelemetry_otel-collector_1 /otelcontribcol --config=/ ... Up 0.0.0.0:4317->4317/tcp, 55679/tcp, 55680/tcp 50 | opentelemetry_prometheus_1 /bin/prometheus --config.f ... Up 0.0.0.0:9090->9090/tcp 51 | opentelemetry_redis_1 /opt/bitnami/scripts/redis ... Up 0.0.0.0:6379->6379/tcp 52 | ``` 53 | 54 | ## Cleanup 55 | 56 | ```bash 57 | ~$ docker-compose down 58 | ``` 59 | -------------------------------------------------------------------------------- /examples/opentelemetry/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.8' 3 | services: 4 | redis: 5 | image: docker.io/bitnami/redis:6.2 6 | ports: 7 | - 6379:6379 8 | environment: 9 | ALLOW_EMPTY_PASSWORD: 'yes' 10 | 11 | jaeger: 12 | image: docker.io/jaegertracing/all-in-one:1.33 13 | volumes: 14 | - ./prometheus/config.yml:/etc/prometheus/prometheus.yml 15 | ports: 16 | - 16686:16686 17 | 18 | otel-collector: 19 | image: docker.io/otel/opentelemetry-collector-contrib-dev:latest 20 | command: ["--config=/etc/otel-collector-config.yml"] 21 | volumes: 22 | - ./otel-collector-config.yml:/etc/otel-collector-config.yml 23 | ports: 24 | - 4317:4317 25 | links: 26 | - jaeger 27 | 28 | gitlab-ci-pipelines-exporter: 29 | image: quay.io/mvisonneau/gitlab-ci-pipelines-exporter:v0.5.10 30 | # You can comment out the image name and use the following statement 31 | # to build the image against the current version of the repository 32 | # build: ../.. 33 | ports: 34 | - 8080:8080 35 | environment: 36 | GCPE_GITLAB_TOKEN: ${GCPE_GITLAB_TOKEN} 37 | GCPE_CONFIG: /etc/gitlab-ci-pipelines-exporter.yml 38 | GCPE_INTERNAL_MONITORING_LISTENER_ADDRESS: tcp://127.0.0.1:8082 39 | links: 40 | - redis 41 | - otel-collector 42 | volumes: 43 | - type: bind 44 | source: ./gitlab-ci-pipelines-exporter.yml 45 | target: /etc/gitlab-ci-pipelines-exporter.yml 46 | 47 | prometheus: 48 | image: docker.io/prom/prometheus:v2.44.0 49 | ports: 50 | - 9090:9090 51 | links: 52 | - gitlab-ci-pipelines-exporter 53 | volumes: 54 | - ./prometheus/config.yml:/etc/prometheus/prometheus.yml 55 | 56 | grafana: 57 | image: docker.io/grafana/grafana:9.5.2 58 | ports: 59 | - 3000:3000 60 | environment: 61 | GF_AUTH_ANONYMOUS_ENABLED: 'true' 62 | GF_INSTALL_PLUGINS: grafana-polystat-panel,yesoreyeram-boomtable-panel 63 | links: 64 | - prometheus 65 | - jaeger 66 | volumes: 67 | - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/default.yml 68 | 69 | networks: 70 | default: 71 | driver: bridge 72 | -------------------------------------------------------------------------------- /examples/opentelemetry/gitlab-ci-pipelines-exporter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | log: 3 | level: trace 4 | format: json 5 | 6 | opentelemetry: 7 | grpc_endpoint: otel-collector:4317 8 | 9 | gitlab: 10 | url: https://gitlab.com 11 | token: 12 | 13 | redis: 14 | url: redis://redis:6379 15 | 16 | # Example public projects to monitor 17 | projects: 18 | - name: gitlab-org/gitlab-runner 19 | # Pull environments related metrics prefixed with 'stable' for this project 20 | pull: 21 | environments: 22 | enabled: true 23 | name_regexp: '^stable.*' 24 | 25 | - name: gitlab-org/charts/auto-deploy-app -------------------------------------------------------------------------------- /examples/opentelemetry/grafana/datasources.yml: -------------------------------------------------------------------------------- 1 | datasources: 2 | - name: 'prometheus' 3 | type: 'prometheus' 4 | access: 'proxy' 5 | org_id: 1 6 | url: 'http://prometheus:9090' 7 | is_default: true 8 | version: 1 9 | editable: true 10 | - name: 'jaeger' 11 | type: 'jaeger' 12 | access: 'proxy' 13 | org_id: 1 14 | url: 'http://jaeger:16686' 15 | is_default: false 16 | version: 1 17 | editable: true -------------------------------------------------------------------------------- /examples/opentelemetry/otel-collector-config.yml: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | 6 | exporters: 7 | jaeger: 8 | endpoint: jaeger:14250 9 | tls: 10 | insecure: true 11 | 12 | processors: 13 | batch: 14 | 15 | service: 16 | pipelines: 17 | traces: 18 | receivers: [otlp] 19 | processors: [batch] 20 | exporters: [jaeger] 21 | -------------------------------------------------------------------------------- /examples/opentelemetry/prometheus/config.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: 'gitlab-ci-pipelines-exporter' 7 | scrape_interval: 10s 8 | scrape_timeout: 5s 9 | static_configs: 10 | - targets: ['gitlab-ci-pipelines-exporter:8080'] -------------------------------------------------------------------------------- /examples/quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Example usage of gitlab-ci-pipelines-exporter with Prometheus & Grafana 2 | 3 | ## Requirements 4 | 5 | - **~5 min of your time** 6 | - A personal access token on [gitlab.com](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) (or your own instance) with `read_api` scope 7 | - [git](https://git-scm.com/) & [docker-compose](https://docs.docker.com/compose/) 8 | 9 | ## 🚀 10 | 11 | ```bash 12 | # Clone this repository 13 | ~$ git clone https://github.com/mvisonneau/gitlab-ci-pipelines-exporter.git 14 | ~$ cd gitlab-ci-pipelines-exporter/examples/quickstart 15 | 16 | # Provide your personal GitLab API access token (needs read_api permissions) 17 | ~$ sed -i 's//xXF_xxjV_xxyzxzz/' gitlab-ci-pipelines-exporter.yml 18 | 19 | # Start gitlab-ci-pipelines-exporter, prometheus and grafana containers ! 20 | ~$ docker-compose up -d 21 | Creating network "quickstart_default" with driver "bridge" 22 | Creating quickstart_gitlab-ci-pipelines-exporter_1 ... done 23 | Creating quickstart_prometheus_1 ... done 24 | Creating quickstart_grafana_1 ... done 25 | ``` 26 | 27 | You should now have a stack completely configured and accessible at these locations: 28 | 29 | - `gitlab-ci-pipelines-exporter`: [http://localhost:8080/metrics](http://localhost:8080/metrics) 30 | - `prometheus`: [http://localhost:9090](http://localhost:9090) 31 | - `grafana`: [http://localhost:3000](http://localhost:3000) (if you want/need to login, creds are _admin/admin_) 32 | 33 | ## Use and troubleshoot 34 | 35 | ### Validate that containers are running 36 | 37 | ```bash 38 | ~$ docker ps 39 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 40 | c9aedfdefe41 grafana/grafana:latest "/run.sh" 6 seconds ago Up 4 seconds 0.0.0.0:3000->3000/tcp quickstart_grafana_1 41 | b3500bff6038 prom/prometheus:latest "/bin/prometheus --c…" 7 seconds ago Up 5 seconds 0.0.0.0:9090->9090/tcp quickstart_prometheus_1 42 | 930b76005b13 mvisonneau/gitlab-ci-pipelines-exporter:latest "/usr/local/bin/gitl…" 8 seconds ago Up 6 seconds 0.0.0.0:8080->8080/tcp quickstart_gitlab-ci-pipelines-exporter_1 43 | ``` 44 | 45 | ### Check logs from the gitlab-ci-pipelines-exporter container 46 | 47 | ```bash 48 | ~$ docker logs -f quickstart_gitlab-ci-pipelines-exporter_1 49 | time="2020-04-28T23:09:01Z" level=info msg="starting exporter" discover-projects-refs-interval=300s discover-wildcard-projects-interval=1800s gitlab-endpoint="https://gitlab.com" on-init-fetch-refs-from-pipelines=false pulling-projects-refs-interval=30s rate-limit=10rps 50 | time="2020-04-28T23:09:01Z" level=info msg="started, now serving requests" listen-address=":8080" 51 | time="2020-04-28T23:09:01Z" level=info msg="discover wildcards" count=0 52 | time="2020-04-28T23:09:14Z" level=info msg="discovered new project ref" project-id=250833 project-path-with-namespace=gitlab-org/gitlab-runner project-ref=master project-ref-kind=branch 53 | time="2020-04-28T23:09:15Z" level=info msg="discovered new project ref" project-id=11915984 project-path-with-namespace=gitlab-org/charts/auto-deploy-app project-ref=master project-ref-kind=branch 54 | time="2020-04-28T23:09:15Z" level=info msg="pulling metrics from projects refs" count=2 55 | ``` 56 | 57 | ### Check we can fetch metrics from the exporter container 58 | 59 | ```bash 60 | # How many metrics we can get 61 | ~$ curl -s http://localhost:8080/metrics | grep project | wc -l 62 | 616 63 | 64 | # Some specific metrics 65 | ~$ curl -s http://localhost:8080/metrics | grep project | grep gitlab_ci_pipeline_timestamp 66 | gitlab_ci_pipeline_timestamp{kind="branch",project="gitlab-org/charts/auto-deploy-app",ref="master",topics="",variables=""} 1.595330197e+09 67 | gitlab_ci_pipeline_timestamp{kind="branch",project="gitlab-org/gitlab-runner",ref="master",topics="",variables=""} 1.604520738e+09 68 | ``` 69 | 70 | ### Checkout prometheus targets and available metrics 71 | 72 | You can open this URL in your browser and should see the exporter is being configured and pulled correctly: 73 | 74 | [http://localhost:9090/targets](http://localhost:9090/targets) 75 | 76 | ![prometheus_targets](/docs/images/prometheus_targets_example.png) 77 | 78 | You should then be able to see the following metrics under the `gitlab_ci_` prefix: 79 | 80 | [http://localhost:9090/new/graph](http://localhost:9090/new/graph) 81 | 82 | ![prometheus_metrics_list](/docs/images/prometheus_metrics_list_example.png) 83 | 84 | You can then validate that you get the expected values for your projects metrics, eg `gitlab_ci_pipeline_status`: 85 | 86 | [http://localhost:9090/new/graph?g0.expr=gitlab_ci_pipeline_status&g0.tab=1&g0.stacked=0&g0.range_input=1h](http://localhost:9090/new/graph?g0.expr=gitlab_ci_pipeline_status&g0.tab=1&g0.stacked=0&g0.range_input=1h) 87 | 88 | ![prometheus_pipeline_status_metric_example](/docs/images/prometheus_pipeline_status_metric_example.png) 89 | 90 | ### Checkout the grafana example dashboards 91 | 92 | Example dashboards should be available at these addresses: 93 | 94 | - **Pipelines dashboard** - [http://localhost:3000/d/gitlab_ci_pipelines](http://localhost:3000/d/gitlab_ci_pipelines) 95 | 96 | ![grafana_dashboard_pipelines_example](/docs/images/grafana_dashboard_pipelines_example.png) 97 | 98 | - **Jobs dashboard** - [http://localhost:3000/d/gitlab_ci_jobs](http://localhost:3000/d/gitlab_ci_jobs) 99 | 100 | ![grafana_dashboard_jobs_example](/docs/images/grafana_dashboard_jobs_example.png) 101 | 102 | - **Environments / Deployments dashboard** - [http://localhost:3000/d/gitlab_ci_environment_deployments](http://localhost:3000/d/gitlab_ci_environment_deployments) 103 | 104 | ![grafana_dashboard_environments_example](/docs/images/grafana_dashboard_environments_example.png) 105 | 106 | ## Perform configuration changes 107 | 108 | I believe it would be more interesting for you to be monitoring your own projects. To perform configuration changes, there are 2 simple steps: 109 | 110 | ```bash 111 | # Edit the configuration file for the exporter 112 | ~$ vi ./gitlab-ci-pipelines-exporter/config.yml 113 | 114 | # Restart the exporter container 115 | ~$ docker-compose restart gitlab-ci-pipelines-exporter 116 | ``` 117 | 118 | ## Cleanup 119 | 120 | ```bash 121 | ~$ docker-compose down 122 | ``` 123 | -------------------------------------------------------------------------------- /examples/quickstart/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.8' 3 | services: 4 | gitlab-ci-pipelines-exporter: 5 | image: quay.io/mvisonneau/gitlab-ci-pipelines-exporter:v0.5.10 6 | # You can comment out the image name and use the following statement 7 | # to build the image against the current version of the repository 8 | # build: ../.. 9 | ports: 10 | - 8080:8080 11 | environment: 12 | GCPE_GITLAB_TOKEN: ${GCPE_GITLAB_TOKEN} 13 | GCPE_CONFIG: /etc/gitlab-ci-pipelines-exporter.yml 14 | GCPE_INTERNAL_MONITORING_LISTENER_ADDRESS: tcp://127.0.0.1:8082 15 | volumes: 16 | - type: bind 17 | source: ./gitlab-ci-pipelines-exporter.yml 18 | target: /etc/gitlab-ci-pipelines-exporter.yml 19 | 20 | prometheus: 21 | image: docker.io/prom/prometheus:v2.44.0 22 | ports: 23 | - 9090:9090 24 | links: 25 | - gitlab-ci-pipelines-exporter 26 | volumes: 27 | - ./prometheus/config.yml:/etc/prometheus/prometheus.yml 28 | 29 | grafana: 30 | image: docker.io/grafana/grafana:9.5.2 31 | ports: 32 | - 3000:3000 33 | environment: 34 | GF_AUTH_ANONYMOUS_ENABLED: 'true' 35 | GF_INSTALL_PLUGINS: grafana-polystat-panel,yesoreyeram-boomtable-panel 36 | links: 37 | - prometheus 38 | volumes: 39 | - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/default.yml 40 | - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/default.yml 41 | - ./grafana/dashboards:/var/lib/grafana/dashboards 42 | 43 | networks: 44 | default: 45 | driver: bridge 46 | -------------------------------------------------------------------------------- /examples/quickstart/gitlab-ci-pipelines-exporter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | log: 3 | level: debug 4 | 5 | gitlab: 6 | url: https://gitlab.com 7 | token: 8 | 9 | # Pull jobs related metrics on all projects 10 | project_defaults: 11 | pull: 12 | pipeline: 13 | jobs: 14 | enabled: true 15 | 16 | # Example public projects to monitor 17 | projects: 18 | - name: gitlab-org/gitlab-runner 19 | # Pull environments related metrics prefixed with 'stable' for this project 20 | pull: 21 | environments: 22 | enabled: true 23 | name_regexp: '^stable.*' 24 | 25 | - name: gitlab-org/charts/auto-deploy-app 26 | -------------------------------------------------------------------------------- /examples/quickstart/grafana/dashboards.yml: -------------------------------------------------------------------------------- 1 | - name: 'default' 2 | org_id: 1 3 | folder: '' 4 | type: 'file' 5 | options: 6 | folder: '/var/lib/grafana/dashboards' -------------------------------------------------------------------------------- /examples/quickstart/grafana/datasources.yml: -------------------------------------------------------------------------------- 1 | datasources: 2 | - name: 'prometheus' 3 | type: 'prometheus' 4 | access: 'proxy' 5 | org_id: 1 6 | url: 'http://prometheus:9090' 7 | is_default: true 8 | version: 1 9 | editable: true -------------------------------------------------------------------------------- /examples/quickstart/prometheus/config.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: 'gitlab-ci-pipelines-exporter' 7 | scrape_interval: 10s 8 | scrape_timeout: 5s 9 | static_configs: 10 | - targets: ['gitlab-ci-pipelines-exporter:8080'] -------------------------------------------------------------------------------- /examples/webhooks/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '3.8' 3 | services: 4 | gitlab-ci-pipelines-exporter: 5 | image: quay.io/mvisonneau/gitlab-ci-pipelines-exporter:v0.5.10 6 | # You can comment out the image name and use the following statement 7 | # to build the image against the current version of the repository 8 | # build: ../.. 9 | ports: 10 | - 8080:8080 11 | environment: 12 | GCPE_CONFIG: /etc/gitlab-ci-pipelines-exporter.yml 13 | GCPE_INTERNAL_MONITORING_LISTENER_ADDRESS: tcp://127.0.0.1:8082 14 | volumes: 15 | - type: bind 16 | source: ./gitlab-ci-pipelines-exporter.yml 17 | target: /etc/gitlab-ci-pipelines-exporter.yml 18 | 19 | networks: 20 | default: 21 | driver: bridge 22 | -------------------------------------------------------------------------------- /examples/webhooks/gitlab-ci-pipelines-exporter.yml: -------------------------------------------------------------------------------- 1 | --- 2 | log: 3 | level: debug 4 | 5 | gitlab: 6 | url: https://gitlab.com 7 | token: 8 | 9 | server: 10 | webhook: 11 | enabled: true 12 | secret_token: 13 | 14 | pull: 15 | projects_from_wildcards: 16 | on_init: false 17 | scheduled: false 18 | 19 | environments_from_projects: 20 | on_init: false 21 | scheduled: false 22 | 23 | refs_from_projects: 24 | on_init: false 25 | scheduled: false 26 | 27 | metrics: 28 | on_init: false 29 | scheduled: false 30 | 31 | projects: 32 | # Configure a project on which you are authorized to configure webhooks 33 | - name: 34 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mvisonneau/gitlab-ci-pipelines-exporter 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | dario.cat/mergo v1.0.1 9 | github.com/alicebob/miniredis/v2 v2.34.0 10 | github.com/charmbracelet/bubbles v0.20.0 11 | github.com/charmbracelet/bubbletea v1.2.4 12 | github.com/charmbracelet/lipgloss v1.0.0 13 | github.com/creasty/defaults v1.8.0 14 | github.com/go-logr/stdr v1.2.2 15 | github.com/go-playground/validator/v10 v10.24.0 16 | github.com/go-redis/redis_rate/v10 v10.0.1 17 | github.com/google/uuid v1.6.0 18 | github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb 19 | github.com/mvisonneau/go-helpers v0.0.1 20 | github.com/paulbellamy/ratecounter v0.2.0 21 | github.com/pkg/errors v0.9.1 22 | github.com/prometheus/client_golang v1.20.5 23 | github.com/redis/go-redis/extra/redisotel/v9 v9.7.0 24 | github.com/redis/go-redis/v9 v9.7.0 25 | github.com/sirupsen/logrus v1.9.3 26 | github.com/stretchr/testify v1.10.0 27 | github.com/uptrace/opentelemetry-go-extra/otellogrus v0.3.2 28 | github.com/urfave/cli/v2 v2.27.5 29 | github.com/vmihailenco/msgpack/v5 v5.4.1 30 | github.com/vmihailenco/taskq/memqueue/v4 v4.0.0-beta.4 31 | github.com/vmihailenco/taskq/redisq/v4 v4.0.0-beta.4 32 | github.com/vmihailenco/taskq/v4 v4.0.0-beta.4 33 | github.com/xanzy/go-gitlab v0.115.0 34 | github.com/xeonx/timeago v1.0.0-rc5 35 | go.openly.dev/pointy v1.3.0 36 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 37 | go.opentelemetry.io/otel v1.33.0 38 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 39 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 40 | go.opentelemetry.io/otel/sdk v1.33.0 41 | go.opentelemetry.io/otel/trace v1.33.0 42 | golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 43 | golang.org/x/mod v0.22.0 44 | golang.org/x/time v0.9.0 45 | google.golang.org/grpc v1.69.4 46 | google.golang.org/protobuf v1.36.2 47 | gopkg.in/yaml.v3 v3.0.1 48 | ) 49 | 50 | require ( 51 | github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect 52 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 53 | github.com/beorn7/perks v1.0.1 // indirect 54 | github.com/bsm/redislock v0.9.4 // indirect 55 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 56 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 57 | github.com/charmbracelet/harmonica v0.2.0 // indirect 58 | github.com/charmbracelet/x/ansi v0.6.0 // indirect 59 | github.com/charmbracelet/x/term v0.2.1 // indirect 60 | github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect 61 | github.com/davecgh/go-spew v1.1.1 // indirect 62 | github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect 63 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 64 | github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect 65 | github.com/felixge/httpsnoop v1.0.4 // indirect 66 | github.com/gabriel-vasile/mimetype v1.4.8 // indirect 67 | github.com/go-logr/logr v1.4.2 // indirect 68 | github.com/go-playground/locales v0.14.1 // indirect 69 | github.com/go-playground/universal-translator v0.18.1 // indirect 70 | github.com/google/go-querystring v1.1.0 // indirect 71 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect 72 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 73 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 74 | github.com/hashicorp/golang-lru v1.0.2 // indirect 75 | github.com/klauspost/compress v1.17.11 // indirect 76 | github.com/leodido/go-urn v1.4.0 // indirect 77 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 78 | github.com/mattn/go-isatty v0.0.20 // indirect 79 | github.com/mattn/go-localereader v0.0.1 // indirect 80 | github.com/mattn/go-runewidth v0.0.16 // indirect 81 | github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect 82 | github.com/muesli/cancelreader v0.2.2 // indirect 83 | github.com/muesli/termenv v0.15.2 // indirect 84 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 85 | github.com/pmezard/go-difflib v1.0.0 // indirect 86 | github.com/prometheus/client_model v0.6.1 // indirect 87 | github.com/prometheus/common v0.61.0 // indirect 88 | github.com/prometheus/procfs v0.15.1 // indirect 89 | github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0 // indirect 90 | github.com/rivo/uniseg v0.4.7 // indirect 91 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 92 | github.com/uptrace/opentelemetry-go-extra/otelutil v0.3.2 // indirect 93 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 94 | github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect 95 | github.com/yuin/gopher-lua v1.1.1 // indirect 96 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 97 | go.opentelemetry.io/otel/log v0.9.0 // indirect 98 | go.opentelemetry.io/otel/metric v1.33.0 // indirect 99 | go.opentelemetry.io/proto/otlp v1.5.0 // indirect 100 | golang.org/x/crypto v0.32.0 // indirect 101 | golang.org/x/net v0.34.0 // indirect 102 | golang.org/x/oauth2 v0.25.0 // indirect 103 | golang.org/x/sync v0.10.0 // indirect 104 | golang.org/x/sys v0.29.0 // indirect 105 | golang.org/x/text v0.21.0 // indirect 106 | google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect 107 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422 // indirect 108 | gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect 109 | ) 110 | -------------------------------------------------------------------------------- /internal/cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/urfave/cli/v2" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/internal/cmd" 11 | ) 12 | 13 | // Run handles the instanciation of the CLI application. 14 | func Run(version string, args []string) { 15 | err := NewApp(version, time.Now()).Run(args) 16 | if err != nil { 17 | fmt.Println(err) 18 | os.Exit(1) 19 | } 20 | } 21 | 22 | // NewApp configures the CLI application. 23 | func NewApp(version string, start time.Time) (app *cli.App) { 24 | app = cli.NewApp() 25 | app.Name = "gitlab-ci-pipelines-exporter" 26 | app.Version = version 27 | app.Usage = "Export metrics about GitLab CI pipelines statuses" 28 | app.EnableBashCompletion = true 29 | 30 | app.Flags = cli.FlagsByName{ 31 | &cli.StringFlag{ 32 | Name: "internal-monitoring-listener-address", 33 | Aliases: []string{"m"}, 34 | EnvVars: []string{"GCPE_INTERNAL_MONITORING_LISTENER_ADDRESS"}, 35 | Usage: "internal monitoring listener address", 36 | }, 37 | } 38 | 39 | app.Commands = cli.CommandsByName{ 40 | { 41 | Name: "run", 42 | Usage: "start the exporter", 43 | Action: cmd.ExecWrapper(cmd.Run), 44 | Flags: cli.FlagsByName{ 45 | &cli.StringFlag{ 46 | Name: "config", 47 | Aliases: []string{"c"}, 48 | EnvVars: []string{"GCPE_CONFIG"}, 49 | Usage: "config `file`", 50 | Value: "./gitlab-ci-pipelines-exporter.yml", 51 | }, 52 | &cli.StringFlag{ 53 | Name: "redis-url", 54 | EnvVars: []string{"GCPE_REDIS_URL"}, 55 | Usage: "redis `url` for an HA setup (format: redis[s]://[:password@]host[:port][/db-number][?option=value]) (overrides config file parameter)", 56 | }, 57 | &cli.StringFlag{ 58 | Name: "gitlab-token", 59 | EnvVars: []string{"GCPE_GITLAB_TOKEN"}, 60 | Usage: "GitLab API access `token` (overrides config file parameter)", 61 | }, 62 | &cli.StringFlag{ 63 | Name: "webhook-secret-token", 64 | EnvVars: []string{"GCPE_WEBHOOK_SECRET_TOKEN"}, 65 | Usage: "`token` used to authenticate legitimate requests (overrides config file parameter)", 66 | }, 67 | &cli.StringFlag{ 68 | Name: "gitlab-health-url", 69 | EnvVars: []string{"GCPE_GITLAB_HEALTH_URL"}, 70 | Usage: "GitLab health URL (overrides config file parameter)", 71 | }, 72 | }, 73 | }, 74 | { 75 | Name: "validate", 76 | Usage: "validate the configuration file", 77 | Action: cmd.ExecWrapper(cmd.Validate), 78 | Flags: cli.FlagsByName{ 79 | &cli.StringFlag{ 80 | Name: "config", 81 | Aliases: []string{"c"}, 82 | EnvVars: []string{"GCPE_CONFIG"}, 83 | Usage: "config `file`", 84 | Value: "./gitlab-ci-pipelines-exporter.yml", 85 | }, 86 | }, 87 | }, 88 | { 89 | Name: "monitor", 90 | Usage: "display information about the currently running exporter", 91 | Action: cmd.ExecWrapper(cmd.Monitor), 92 | }, 93 | } 94 | 95 | app.Metadata = map[string]interface{}{ 96 | "startTime": start, 97 | } 98 | 99 | return 100 | } 101 | -------------------------------------------------------------------------------- /internal/cli/cli_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestRun(t *testing.T) { 11 | assert.NotPanics(t, func() { Run("0.0.0", []string{"gitlab-ci-pipelines-exporter", "--version"}) }) 12 | } 13 | 14 | func TestNewApp(t *testing.T) { 15 | app := NewApp("0.0.0", time.Now()) 16 | assert.Equal(t, "gitlab-ci-pipelines-exporter", app.Name) 17 | assert.Equal(t, "0.0.0", app.Version) 18 | } 19 | -------------------------------------------------------------------------------- /internal/cmd/monitor.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | 6 | monitorUI "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor/ui" 7 | ) 8 | 9 | // Monitor .. 10 | func Monitor(ctx *cli.Context) (int, error) { 11 | cfg, err := parseGlobalFlags(ctx) 12 | if err != nil { 13 | return 1, err 14 | } 15 | 16 | monitorUI.Start( 17 | ctx.App.Version, 18 | cfg.InternalMonitoringListenerAddress, 19 | ) 20 | 21 | return 0, nil 22 | } 23 | -------------------------------------------------------------------------------- /internal/cmd/run.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/pprof" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | log "github.com/sirupsen/logrus" 13 | "github.com/urfave/cli/v2" 14 | 15 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/controller" 16 | monitoringServer "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor/server" 17 | ) 18 | 19 | // Run launches the exporter. 20 | func Run(cliCtx *cli.Context) (int, error) { 21 | cfg, err := configure(cliCtx) 22 | if err != nil { 23 | return 1, err 24 | } 25 | 26 | ctx, ctxCancel := context.WithCancel(context.Background()) 27 | defer ctxCancel() 28 | 29 | c, err := controller.New(ctx, cfg, cliCtx.App.Version) 30 | if err != nil { 31 | return 1, err 32 | } 33 | 34 | // Start the monitoring RPC server 35 | go func(c *controller.Controller) { 36 | s := monitoringServer.NewServer( 37 | c.Gitlab, 38 | c.Config, 39 | c.Store, 40 | c.TaskController.TaskSchedulingMonitoring, 41 | ) 42 | s.Serve() 43 | }(&c) 44 | 45 | // Graceful shutdowns 46 | onShutdown := make(chan os.Signal, 1) 47 | signal.Notify(onShutdown, syscall.SIGINT, syscall.SIGTERM, syscall.SIGABRT) 48 | 49 | // HTTP server 50 | mux := http.NewServeMux() 51 | srv := &http.Server{ 52 | Addr: cfg.Server.ListenAddress, 53 | Handler: mux, 54 | } 55 | 56 | // health endpoints 57 | health := c.HealthCheckHandler(ctx) 58 | mux.HandleFunc("/health/live", health.LiveEndpoint) 59 | mux.HandleFunc("/health/ready", health.ReadyEndpoint) 60 | 61 | // metrics endpoint 62 | if cfg.Server.Metrics.Enabled { 63 | mux.HandleFunc("/metrics", c.MetricsHandler) 64 | } 65 | 66 | // pprof/debug endpoints 67 | if cfg.Server.EnablePprof { 68 | mux.HandleFunc("/debug/pprof/", pprof.Index) 69 | mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 70 | mux.HandleFunc("/debug/pprof/profile", pprof.Profile) 71 | mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 72 | mux.HandleFunc("/debug/pprof/trace", pprof.Trace) 73 | } 74 | 75 | // webhook endpoints 76 | if cfg.Server.Webhook.Enabled { 77 | mux.HandleFunc("/webhook", c.WebhookHandler) 78 | } 79 | 80 | go func() { 81 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { 82 | log.WithContext(ctx). 83 | WithError(err). 84 | Fatal() 85 | } 86 | }() 87 | 88 | log.WithFields( 89 | log.Fields{ 90 | "listen-address": cfg.Server.ListenAddress, 91 | "pprof-endpoint-enabled": cfg.Server.EnablePprof, 92 | "metrics-endpoint-enabled": cfg.Server.Metrics.Enabled, 93 | "webhook-endpoint-enabled": cfg.Server.Webhook.Enabled, 94 | "openmetrics-encoding-enabled": cfg.Server.Metrics.EnableOpenmetricsEncoding, 95 | "controller-uuid": c.UUID, 96 | }, 97 | ).Info("http server started") 98 | 99 | <-onShutdown 100 | log.Info("received signal, attempting to gracefully exit..") 101 | ctxCancel() 102 | 103 | httpServerContext, forceHTTPServerShutdown := context.WithTimeout(context.Background(), 5*time.Second) 104 | defer forceHTTPServerShutdown() 105 | 106 | if err := srv.Shutdown(httpServerContext); err != nil { 107 | return 1, err 108 | } 109 | 110 | log.Info("stopped!") 111 | 112 | return 0, nil 113 | } 114 | -------------------------------------------------------------------------------- /internal/cmd/run_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | // func TestRunWrongLogLevel(t *testing.T) { 10 | // ctx, flags := NewTestContext() 11 | // flags.String("log-format", "foo", "") 12 | // exitCode, err := Run(ctx) 13 | // assert.Equal(t, 1, exitCode) 14 | // assert.Error(t, err) 15 | // } 16 | 17 | func TestRunInvalidConfigFile(t *testing.T) { 18 | ctx, flags := NewTestContext() 19 | 20 | flags.String("config", "path_does_not_exist", "") 21 | 22 | exitCode, err := Run(ctx) 23 | assert.Equal(t, 1, exitCode) 24 | assert.Error(t, err) 25 | } 26 | -------------------------------------------------------------------------------- /internal/cmd/utils.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | stdlibLog "log" 6 | "net/url" 7 | "os" 8 | "time" 9 | 10 | "github.com/go-logr/stdr" 11 | log "github.com/sirupsen/logrus" 12 | "github.com/uptrace/opentelemetry-go-extra/otellogrus" 13 | "github.com/urfave/cli/v2" 14 | "github.com/vmihailenco/taskq/v4" 15 | 16 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 17 | "github.com/mvisonneau/go-helpers/logger" 18 | ) 19 | 20 | var start time.Time 21 | 22 | func configure(ctx *cli.Context) (cfg config.Config, err error) { 23 | start = ctx.App.Metadata["startTime"].(time.Time) 24 | 25 | assertStringVariableDefined(ctx, "config") 26 | 27 | cfg, err = config.ParseFile(ctx.String("config")) 28 | if err != nil { 29 | return 30 | } 31 | 32 | cfg.Global, err = parseGlobalFlags(ctx) 33 | if err != nil { 34 | return 35 | } 36 | 37 | configCliOverrides(ctx, &cfg) 38 | 39 | if err = cfg.Validate(); err != nil { 40 | return 41 | } 42 | 43 | // Configure logger 44 | if err = logger.Configure(logger.Config{ 45 | Level: cfg.Log.Level, 46 | Format: cfg.Log.Format, 47 | }); err != nil { 48 | return 49 | } 50 | 51 | log.AddHook(otellogrus.NewHook(otellogrus.WithLevels( 52 | log.PanicLevel, 53 | log.FatalLevel, 54 | log.ErrorLevel, 55 | log.WarnLevel, 56 | ))) 57 | 58 | // This hack is to embed taskq logs with logrus 59 | taskq.SetLogger(stdr.New(stdlibLog.New(log.StandardLogger().WriterLevel(log.WarnLevel), "taskq", 0))) 60 | 61 | log.WithFields( 62 | log.Fields{ 63 | "gitlab-endpoint": cfg.Gitlab.URL, 64 | "gitlab-rate-limit": fmt.Sprintf("%drps", cfg.Gitlab.MaximumRequestsPerSecond), 65 | }, 66 | ).Info("configured") 67 | 68 | log.WithFields(config.SchedulerConfig(cfg.Pull.ProjectsFromWildcards).Log()).Info("pull projects from wildcards") 69 | log.WithFields(config.SchedulerConfig(cfg.Pull.EnvironmentsFromProjects).Log()).Info("pull environments from projects") 70 | log.WithFields(config.SchedulerConfig(cfg.Pull.RefsFromProjects).Log()).Info("pull refs from projects") 71 | log.WithFields(config.SchedulerConfig(cfg.Pull.Metrics).Log()).Info("pull metrics") 72 | 73 | log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Projects).Log()).Info("garbage collect projects") 74 | log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Environments).Log()).Info("garbage collect environments") 75 | log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Refs).Log()).Info("garbage collect refs") 76 | log.WithFields(config.SchedulerConfig(cfg.GarbageCollect.Metrics).Log()).Info("garbage collect metrics") 77 | 78 | return 79 | } 80 | 81 | func parseGlobalFlags(ctx *cli.Context) (cfg config.Global, err error) { 82 | if listenerAddr := ctx.String("internal-monitoring-listener-address"); listenerAddr != "" { 83 | cfg.InternalMonitoringListenerAddress, err = url.Parse(listenerAddr) 84 | } 85 | 86 | return 87 | } 88 | 89 | func exit(exitCode int, err error) cli.ExitCoder { 90 | defer log.WithFields( 91 | log.Fields{ 92 | "execution-time": time.Since(start), 93 | }, 94 | ).Debug("exited..") 95 | 96 | if err != nil { 97 | log.WithError(err).Error() 98 | } 99 | 100 | return cli.Exit("", exitCode) 101 | } 102 | 103 | // ExecWrapper gracefully logs and exits our `run` functions. 104 | func ExecWrapper(f func(ctx *cli.Context) (int, error)) cli.ActionFunc { 105 | return func(ctx *cli.Context) error { 106 | return exit(f(ctx)) 107 | } 108 | } 109 | 110 | func configCliOverrides(ctx *cli.Context, cfg *config.Config) { 111 | if ctx.String("gitlab-token") != "" { 112 | cfg.Gitlab.Token = ctx.String("gitlab-token") 113 | } 114 | 115 | if cfg.Server.Webhook.Enabled { 116 | if ctx.String("webhook-secret-token") != "" { 117 | cfg.Server.Webhook.SecretToken = ctx.String("webhook-secret-token") 118 | } 119 | } 120 | 121 | if ctx.String("redis-url") != "" { 122 | cfg.Redis.URL = ctx.String("redis-url") 123 | } 124 | 125 | if healthURL := ctx.String("gitlab-health-url"); healthURL != "" { 126 | cfg.Gitlab.HealthURL = healthURL 127 | cfg.Gitlab.EnableHealthCheck = true 128 | } 129 | } 130 | 131 | func assertStringVariableDefined(ctx *cli.Context, k string) { 132 | if len(ctx.String(k)) == 0 { 133 | _ = cli.ShowAppHelp(ctx) 134 | 135 | log.Errorf("'--%s' must be set!", k) 136 | os.Exit(2) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /internal/cmd/utils_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/urfave/cli/v2" 13 | 14 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 15 | ) 16 | 17 | func NewTestContext() (ctx *cli.Context, flags *flag.FlagSet) { 18 | app := cli.NewApp() 19 | app.Name = "gitlab-ci-pipelines-exporter" 20 | 21 | app.Metadata = map[string]interface{}{ 22 | "startTime": time.Now(), 23 | } 24 | 25 | flags = flag.NewFlagSet("test", flag.ContinueOnError) 26 | ctx = cli.NewContext(app, flags, nil) 27 | 28 | return 29 | } 30 | 31 | func TestConfigure(t *testing.T) { 32 | var ( 33 | cfg config.Config 34 | err error 35 | ) 36 | 37 | f, err := ioutil.TempFile(".", "test-*.yml") 38 | assert.NoError(t, err) 39 | 40 | defer os.Remove(f.Name()) 41 | 42 | // Webhook endpoint enabled 43 | ioutil.WriteFile(f.Name(), []byte(`wildcards: [{}]`), 0o644) 44 | 45 | ctx, flags := NewTestContext() 46 | flags.String("log-format", "text", "") 47 | flags.String("log-level", "debug", "") 48 | flags.String("config", f.Name(), "") 49 | 50 | // Undefined gitlab-token 51 | flags.String("gitlab-token", "", "") 52 | 53 | _, err = configure(ctx) 54 | assert.Error(t, err) 55 | 56 | // Valid configuration 57 | flags.Set("gitlab-token", "secret") 58 | 59 | cfg, err = configure(ctx) 60 | assert.NoError(t, err) 61 | assert.Equal(t, "secret", cfg.Gitlab.Token) 62 | 63 | // Invalid config file syntax 64 | ioutil.WriteFile(f.Name(), []byte("["), 0o644) 65 | 66 | cfg, err = configure(ctx) 67 | assert.Error(t, err) 68 | 69 | // Webhook endpoint enabled 70 | ioutil.WriteFile(f.Name(), []byte(` 71 | wildcards: [{}] 72 | server: 73 | webhook: 74 | enabled: true 75 | `), 0o644) 76 | 77 | // No secret token defined for the webhook endpoint 78 | cfg, err = configure(ctx) 79 | assert.Error(t, err) 80 | 81 | // Defining the webhook secret token 82 | flags.String("webhook-secret-token", "secret", "") 83 | 84 | cfg, err = configure(ctx) 85 | assert.NoError(t, err) 86 | assert.Equal(t, "secret", cfg.Server.Webhook.SecretToken) 87 | 88 | // Test health url flag 89 | healthURL := "https://gitlab.com/-/readiness?token" 90 | flags.String("gitlab-health-url", healthURL, "") 91 | 92 | cfg, err = configure(ctx) 93 | assert.NoError(t, err) 94 | assert.Equal(t, cfg.Gitlab.HealthURL, healthURL) 95 | assert.True(t, cfg.Gitlab.EnableHealthCheck) 96 | } 97 | 98 | func TestExit(t *testing.T) { 99 | err := exit(20, fmt.Errorf("test")) 100 | assert.Equal(t, "", err.Error()) 101 | assert.Equal(t, 20, err.ExitCode()) 102 | } 103 | 104 | func TestExecWrapper(t *testing.T) { 105 | function := func(ctx *cli.Context) (int, error) { 106 | return 0, nil 107 | } 108 | assert.Equal(t, exit(function(&cli.Context{})), ExecWrapper(function)(&cli.Context{})) 109 | } 110 | -------------------------------------------------------------------------------- /internal/cmd/validate.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | func Validate(cliCtx *cli.Context) (int, error) { 9 | log.Debug("Validating configuration..") 10 | 11 | if _, err := configure(cliCtx); err != nil { 12 | log.WithError(err).Error("Failed to configure") 13 | 14 | return 1, err 15 | } 16 | 17 | log.Debug("Configuration is valid") 18 | 19 | return 0, nil 20 | } 21 | -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNew(t *testing.T) { 11 | c := Config{} 12 | 13 | c.Log.Level = "info" 14 | c.Log.Format = "text" 15 | 16 | c.OpenTelemetry.GRPCEndpoint = "" 17 | 18 | c.Server.ListenAddress = ":8080" 19 | c.Server.Metrics.Enabled = true 20 | 21 | c.Gitlab.URL = "https://gitlab.com" 22 | c.Gitlab.HealthURL = "https://gitlab.com/explore" 23 | c.Gitlab.EnableHealthCheck = true 24 | c.Gitlab.EnableTLSVerify = true 25 | c.Gitlab.MaximumRequestsPerSecond = 1 26 | c.Gitlab.BurstableRequestsPerSecond = 5 27 | c.Gitlab.MaximumJobsQueueSize = 1000 28 | 29 | c.Pull.ProjectsFromWildcards.OnInit = true 30 | c.Pull.ProjectsFromWildcards.Scheduled = true 31 | c.Pull.ProjectsFromWildcards.IntervalSeconds = 1800 32 | 33 | c.Pull.EnvironmentsFromProjects.OnInit = true 34 | c.Pull.EnvironmentsFromProjects.Scheduled = true 35 | c.Pull.EnvironmentsFromProjects.IntervalSeconds = 1800 36 | 37 | c.Pull.RefsFromProjects.OnInit = true 38 | c.Pull.RefsFromProjects.Scheduled = true 39 | c.Pull.RefsFromProjects.IntervalSeconds = 300 40 | 41 | c.Pull.Metrics.OnInit = true 42 | c.Pull.Metrics.Scheduled = true 43 | c.Pull.Metrics.IntervalSeconds = 30 44 | 45 | c.GarbageCollect.Projects.Scheduled = true 46 | c.GarbageCollect.Projects.IntervalSeconds = 14400 47 | 48 | c.GarbageCollect.Environments.Scheduled = true 49 | c.GarbageCollect.Environments.IntervalSeconds = 14400 50 | 51 | c.GarbageCollect.Refs.Scheduled = true 52 | c.GarbageCollect.Refs.IntervalSeconds = 1800 53 | 54 | c.GarbageCollect.Metrics.Scheduled = true 55 | c.GarbageCollect.Metrics.IntervalSeconds = 600 56 | 57 | c.ProjectDefaults.OutputSparseStatusMetrics = true 58 | 59 | c.ProjectDefaults.Pull.Environments.Regexp = `.*` 60 | c.ProjectDefaults.Pull.Environments.ExcludeStopped = true 61 | 62 | c.ProjectDefaults.Pull.Refs.Branches.Enabled = true 63 | c.ProjectDefaults.Pull.Refs.Branches.Regexp = `^(?:main|master)$` 64 | c.ProjectDefaults.Pull.Refs.Branches.ExcludeDeleted = true 65 | 66 | c.ProjectDefaults.Pull.Refs.Tags.Enabled = true 67 | c.ProjectDefaults.Pull.Refs.Tags.Regexp = `.*` 68 | c.ProjectDefaults.Pull.Refs.Tags.ExcludeDeleted = true 69 | 70 | c.ProjectDefaults.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true 71 | c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true 72 | c.ProjectDefaults.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com` 73 | c.ProjectDefaults.Pull.Pipeline.Variables.Regexp = `.*` 74 | 75 | assert.Equal(t, c, New()) 76 | } 77 | 78 | func TestValidConfig(t *testing.T) { 79 | cfg := New() 80 | 81 | cfg.Gitlab.Token = "foo" 82 | cfg.Projects = append(cfg.Projects, NewProject("bar")) 83 | 84 | assert.NoError(t, cfg.Validate()) 85 | } 86 | 87 | func TestSchedulerConfigLog(t *testing.T) { 88 | sc := SchedulerConfig{ 89 | OnInit: true, 90 | Scheduled: true, 91 | IntervalSeconds: 300, 92 | } 93 | 94 | assert.Equal(t, log.Fields{ 95 | "on-init": "yes", 96 | "scheduled": "every 300s", 97 | }, sc.Log()) 98 | } 99 | -------------------------------------------------------------------------------- /pkg/config/global.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "net/url" 5 | ) 6 | 7 | // Global is used for globally shared exporter config. 8 | type Global struct { 9 | // InternalMonitoringListenerAddress can be used to access 10 | // some metrics related to the exporter internals 11 | InternalMonitoringListenerAddress *url.URL 12 | } 13 | -------------------------------------------------------------------------------- /pkg/config/parser.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | 8 | "gopkg.in/yaml.v3" 9 | ) 10 | 11 | // Format represents the format of the config file. 12 | type Format uint8 13 | 14 | const ( 15 | // FormatYAML represents a Config written in yaml format. 16 | FormatYAML Format = iota 17 | ) 18 | 19 | // ParseFile reads the content of a file and attempt to unmarshal it 20 | // into a Config. 21 | func ParseFile(filename string) (c Config, err error) { 22 | var ( 23 | t Format 24 | fileBytes []byte 25 | ) 26 | 27 | // Figure out what type of config file we provided 28 | t, err = GetTypeFromFileExtension(filename) 29 | if err != nil { 30 | return 31 | } 32 | 33 | // Read the content of the config file 34 | fileBytes, err = ioutil.ReadFile(filepath.Clean(filename)) 35 | if err != nil { 36 | return 37 | } 38 | 39 | // Parse the content and return Config 40 | return Parse(t, fileBytes) 41 | } 42 | 43 | // Parse unmarshal provided bytes with given ConfigType into a Config object. 44 | func Parse(f Format, bytes []byte) (cfg Config, err error) { 45 | switch f { 46 | case FormatYAML: 47 | err = yaml.Unmarshal(bytes, &cfg) 48 | default: 49 | err = fmt.Errorf("unsupported config type '%+v'", f) 50 | } 51 | 52 | // hack: automatically update the cfg.GitLab.HealthURL for self-hosted GitLab 53 | if cfg.Gitlab.URL != "https://gitlab.com" && 54 | cfg.Gitlab.HealthURL == "https://gitlab.com/explore" { 55 | cfg.Gitlab.HealthURL = fmt.Sprintf("%s/-/health", cfg.Gitlab.URL) 56 | } 57 | 58 | return 59 | } 60 | 61 | // GetTypeFromFileExtension returns the ConfigType based upon the extension of 62 | // the file. 63 | func GetTypeFromFileExtension(filename string) (f Format, err error) { 64 | switch ext := filepath.Ext(filename); ext { 65 | case ".yml", ".yaml": 66 | f = FormatYAML 67 | default: 68 | err = fmt.Errorf("unsupported config type '%s', expected .y(a)ml", ext) 69 | } 70 | 71 | return 72 | } 73 | -------------------------------------------------------------------------------- /pkg/config/project_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewProject(t *testing.T) { 10 | p := Project{} 11 | 12 | p.Name = "foo/bar" 13 | 14 | p.OutputSparseStatusMetrics = true 15 | 16 | p.Pull.Environments.Regexp = `.*` 17 | p.Pull.Environments.ExcludeStopped = true 18 | 19 | p.Pull.Refs.Branches.Enabled = true 20 | p.Pull.Refs.Branches.Regexp = `^(?:main|master)$` 21 | p.Pull.Refs.Branches.ExcludeDeleted = true 22 | 23 | p.Pull.Refs.Tags.Enabled = true 24 | p.Pull.Refs.Tags.Regexp = `.*` 25 | p.Pull.Refs.Tags.ExcludeDeleted = true 26 | 27 | p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true 28 | p.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true 29 | p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com` 30 | p.Pull.Pipeline.Variables.Regexp = `.*` 31 | 32 | assert.Equal(t, p, NewProject("foo/bar")) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/config/wildcard.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/creasty/defaults" 5 | ) 6 | 7 | // Wildcard is a specific handler to dynamically search projects. 8 | type Wildcard struct { 9 | // ProjectParameters holds parameters specific to the projects which 10 | // will be discovered using this wildcard. 11 | ProjectParameters `yaml:",inline"` 12 | 13 | Search string `yaml:"search"` 14 | Owner WildcardOwner `yaml:"owner"` 15 | Archived bool `yaml:"archived"` 16 | } 17 | 18 | // WildcardOwner .. 19 | type WildcardOwner struct { 20 | Name string `yaml:"name"` 21 | Kind string `yaml:"kind"` 22 | IncludeSubgroups bool `yaml:"include_subgroups"` 23 | } 24 | 25 | // Wildcards .. 26 | type Wildcards []Wildcard 27 | 28 | // NewWildcard returns a new wildcard with the default parameters. 29 | func NewWildcard() (w Wildcard) { 30 | defaults.MustSet(&w) 31 | 32 | return 33 | } 34 | -------------------------------------------------------------------------------- /pkg/config/wildcard_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewWildcard(t *testing.T) { 10 | w := Wildcard{} 11 | 12 | w.OutputSparseStatusMetrics = true 13 | 14 | w.Pull.Environments.Regexp = `.*` 15 | w.Pull.Environments.ExcludeStopped = true 16 | 17 | w.Pull.Refs.Branches.Enabled = true 18 | w.Pull.Refs.Branches.Regexp = `^(?:main|master)$` 19 | w.Pull.Refs.Branches.ExcludeDeleted = true 20 | 21 | w.Pull.Refs.Tags.Enabled = true 22 | w.Pull.Refs.Tags.Regexp = `.*` 23 | w.Pull.Refs.Tags.ExcludeDeleted = true 24 | 25 | w.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = true 26 | w.Pull.Pipeline.Jobs.RunnerDescription.Enabled = true 27 | w.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `shared-runners-manager-(\d*)\.gitlab\.com` 28 | w.Pull.Pipeline.Variables.Regexp = `.*` 29 | 30 | assert.Equal(t, w, NewWildcard()) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/controller/collectors_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewCollectorFunctions(t *testing.T) { 11 | for _, f := range []func() prometheus.Collector{ 12 | NewInternalCollectorCurrentlyQueuedTasksCount, 13 | NewInternalCollectorEnvironmentsCount, 14 | NewInternalCollectorExecutedTasksCount, 15 | NewInternalCollectorGitLabAPIRequestsCount, 16 | NewInternalCollectorMetricsCount, 17 | NewInternalCollectorProjectsCount, 18 | NewInternalCollectorRefsCount, 19 | NewCollectorCoverage, 20 | NewCollectorDurationSeconds, 21 | NewCollectorEnvironmentBehindCommitsCount, 22 | NewCollectorEnvironmentBehindDurationSeconds, 23 | NewCollectorEnvironmentDeploymentDurationSeconds, 24 | NewCollectorEnvironmentDeploymentJobID, 25 | NewCollectorEnvironmentDeploymentStatus, 26 | NewCollectorEnvironmentDeploymentTimestamp, 27 | NewCollectorEnvironmentInformation, 28 | NewCollectorID, 29 | NewCollectorJobArtifactSizeBytes, 30 | NewCollectorJobDurationSeconds, 31 | NewCollectorJobID, 32 | NewCollectorJobQueuedDurationSeconds, 33 | NewCollectorJobStatus, 34 | NewCollectorJobTimestamp, 35 | NewCollectorQueuedDurationSeconds, 36 | NewCollectorStatus, 37 | NewCollectorTimestamp, 38 | } { 39 | c := f() 40 | assert.NotNil(t, c) 41 | assert.IsType(t, &prometheus.GaugeVec{}, c) 42 | } 43 | 44 | for _, f := range []func() prometheus.Collector{ 45 | NewCollectorJobRunCount, 46 | NewCollectorRunCount, 47 | NewCollectorEnvironmentDeploymentCount, 48 | } { 49 | c := f() 50 | assert.NotNil(t, c) 51 | assert.IsType(t, &prometheus.CounterVec{}, c) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/controller/controller_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 12 | ) 13 | 14 | func newMockedGitlabAPIServer() (mux *http.ServeMux, srv *httptest.Server) { 15 | mux = http.NewServeMux() 16 | srv = httptest.NewServer(mux) 17 | 18 | return 19 | } 20 | 21 | func newTestController(cfg config.Config) (ctx context.Context, c Controller, mux *http.ServeMux, srv *httptest.Server) { 22 | ctx = context.Background() 23 | mux, srv = newMockedGitlabAPIServer() 24 | 25 | cfg.Gitlab.URL = srv.URL 26 | if cfg.Gitlab.MaximumRequestsPerSecond < 1 { 27 | cfg.Gitlab.MaximumRequestsPerSecond = 1000 28 | } 29 | 30 | if cfg.Gitlab.BurstableRequestsPerSecond < 1 { 31 | cfg.Gitlab.BurstableRequestsPerSecond = 1 32 | } 33 | 34 | c, _ = New(context.Background(), cfg, "0.0.0-ci") 35 | 36 | return 37 | } 38 | 39 | func TestConfigureGitlab(t *testing.T) { 40 | c := Controller{} 41 | assert.NoError(t, c.configureGitlab( 42 | config.Gitlab{ 43 | MaximumRequestsPerSecond: 5, 44 | }, 45 | "0.0.0", 46 | )) 47 | assert.NotNil(t, c.Gitlab) 48 | } 49 | 50 | // func TestConfigureRedisClient(t *testing.T) { 51 | 52 | // s, err := miniredis.Run() 53 | // if err != nil { 54 | // panic(err) 55 | // } 56 | // defer s.Close() 57 | 58 | // c := redis.NewClient(&redis.Options{Addr: s.Addr()}) 59 | // assert.NoError(t, ConfigureRedisClient(c)) 60 | // assert.Equal(t, redisClient, c) 61 | 62 | // s.Close() 63 | // assert.Error(t, ConfigureRedisClient(c)) 64 | // } 65 | 66 | // func TestConfigureStore(t *testing.T) { 67 | // cfg = config.Config{ 68 | // Projects: []config.Project{ 69 | // { 70 | // Name: "foo/bar", 71 | // }, 72 | // }, 73 | // } 74 | 75 | // // Test with local storage 76 | // configureStore() 77 | // assert.NotNil(t, store) 78 | 79 | // projects, err := store.Projects() 80 | // assert.NoError(t, err) 81 | 82 | // expectedProjects := config.Projects{ 83 | // "3861188962": config.Project{ 84 | // Name: "foo/bar", 85 | // }, 86 | // } 87 | // assert.Equal(t, expectedProjects, projects) 88 | 89 | // // Test with redis storage 90 | // s, err := miniredis.Run() 91 | // if err != nil { 92 | // panic(err) 93 | // } 94 | // defer s.Close() 95 | 96 | // c := redis.NewClient(&redis.Options{Addr: s.Addr()}) 97 | // assert.NoError(t, ConfigureRedisClient(c)) 98 | 99 | // configureStore() 100 | // projects, err = store.Projects() 101 | // assert.NoError(t, err) 102 | // assert.Equal(t, expectedProjects, projects) 103 | // } 104 | -------------------------------------------------------------------------------- /pkg/controller/environments.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | log "github.com/sirupsen/logrus" 7 | 8 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 9 | ) 10 | 11 | // PullEnvironmentsFromProject .. 12 | func (c *Controller) PullEnvironmentsFromProject(ctx context.Context, p schemas.Project) (err error) { 13 | var envs schemas.Environments 14 | 15 | envs, err = c.Gitlab.GetProjectEnvironments(ctx, p) 16 | if err != nil { 17 | return 18 | } 19 | 20 | for k := range envs { 21 | var exists bool 22 | 23 | exists, err = c.Store.EnvironmentExists(ctx, k) 24 | if err != nil { 25 | return 26 | } 27 | 28 | if !exists { 29 | env := envs[k] 30 | if err = c.UpdateEnvironment(ctx, &env); err != nil { 31 | return 32 | } 33 | 34 | log.WithFields(log.Fields{ 35 | "project-name": env.ProjectName, 36 | "environment-id": env.ID, 37 | "environment-name": env.Name, 38 | }).Info("discovered new environment") 39 | 40 | c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env) 41 | } 42 | } 43 | 44 | return 45 | } 46 | 47 | // UpdateEnvironment .. 48 | func (c *Controller) UpdateEnvironment(ctx context.Context, env *schemas.Environment) error { 49 | pulledEnv, err := c.Gitlab.GetEnvironment(ctx, env.ProjectName, env.ID) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | env.Available = pulledEnv.Available 55 | env.ExternalURL = pulledEnv.ExternalURL 56 | env.LatestDeployment = pulledEnv.LatestDeployment 57 | 58 | return c.Store.SetEnvironment(ctx, *env) 59 | } 60 | 61 | // PullEnvironmentMetrics .. 62 | func (c *Controller) PullEnvironmentMetrics(ctx context.Context, env schemas.Environment) (err error) { 63 | // At scale, the scheduled environment may be behind the actual state being stored 64 | // to avoid issues, we refresh it from the store before manipulating it 65 | if err := c.Store.GetEnvironment(ctx, &env); err != nil { 66 | return err 67 | } 68 | 69 | // Save the existing deployment ID before we updated environment from the API 70 | deploymentJobID := env.LatestDeployment.JobID 71 | 72 | if err = c.UpdateEnvironment(ctx, &env); err != nil { 73 | return 74 | } 75 | 76 | var ( 77 | infoLabels = env.InformationLabelsValues() 78 | commitDate float64 79 | ) 80 | 81 | switch env.LatestDeployment.RefKind { 82 | case schemas.RefKindBranch: 83 | infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetBranchLatestCommit(ctx, env.ProjectName, env.LatestDeployment.RefName) 84 | case schemas.RefKindTag: 85 | // TODO: Review how to manage this in a nicier fashion 86 | infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetProjectMostRecentTagCommit(ctx, env.ProjectName, ".*") 87 | default: 88 | infoLabels["latest_commit_short_id"] = env.LatestDeployment.CommitShortID 89 | commitDate = env.LatestDeployment.Timestamp 90 | } 91 | 92 | if err != nil { 93 | return err 94 | } 95 | 96 | var ( 97 | envBehindDurationSeconds float64 98 | envBehindCommitCount float64 99 | ) 100 | 101 | behindCommitsCountMetric := schemas.Metric{ 102 | Kind: schemas.MetricKindEnvironmentBehindCommitsCount, 103 | Labels: env.DefaultLabelsValues(), 104 | } 105 | 106 | // To reduce the amount of compare requests being made, we check if the labels are unchanged since 107 | // the latest emission of the information metric 108 | if infoLabels["latest_commit_short_id"] != infoLabels["current_commit_short_id"] { 109 | infoMetric := schemas.Metric{ 110 | Kind: schemas.MetricKindEnvironmentInformation, 111 | Labels: env.DefaultLabelsValues(), 112 | } 113 | 114 | var commitCount int 115 | 116 | if err = c.Store.GetMetric(ctx, &infoMetric); err != nil { 117 | return err 118 | } 119 | 120 | if infoMetric.Labels["latest_commit_short_id"] != infoLabels["latest_commit_short_id"] || 121 | infoMetric.Labels["current_commit_short_id"] != infoLabels["current_commit_short_id"] { 122 | commitCount, err = c.Gitlab.GetCommitCountBetweenRefs(ctx, env.ProjectName, infoLabels["current_commit_short_id"], infoLabels["latest_commit_short_id"]) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | envBehindCommitCount = float64(commitCount) 128 | } else { 129 | // TODO: Find a more efficient way 130 | if err = c.Store.GetMetric(ctx, &behindCommitsCountMetric); err != nil { 131 | return err 132 | } 133 | 134 | envBehindCommitCount = behindCommitsCountMetric.Value 135 | } 136 | } 137 | 138 | storeSetMetric(ctx, c.Store, schemas.Metric{ 139 | Kind: schemas.MetricKindEnvironmentBehindCommitsCount, 140 | Labels: env.DefaultLabelsValues(), 141 | Value: envBehindCommitCount, 142 | }) 143 | 144 | if commitDate-env.LatestDeployment.Timestamp > 0 { 145 | envBehindDurationSeconds = commitDate - env.LatestDeployment.Timestamp 146 | } 147 | 148 | envDeploymentCount := schemas.Metric{ 149 | Kind: schemas.MetricKindEnvironmentDeploymentCount, 150 | Labels: env.DefaultLabelsValues(), 151 | } 152 | 153 | storeGetMetric(ctx, c.Store, &envDeploymentCount) 154 | 155 | if env.LatestDeployment.JobID > deploymentJobID { 156 | envDeploymentCount.Value++ 157 | } 158 | 159 | storeSetMetric(ctx, c.Store, envDeploymentCount) 160 | 161 | storeSetMetric(ctx, c.Store, schemas.Metric{ 162 | Kind: schemas.MetricKindEnvironmentBehindDurationSeconds, 163 | Labels: env.DefaultLabelsValues(), 164 | Value: envBehindDurationSeconds, 165 | }) 166 | 167 | storeSetMetric(ctx, c.Store, schemas.Metric{ 168 | Kind: schemas.MetricKindEnvironmentDeploymentDurationSeconds, 169 | Labels: env.DefaultLabelsValues(), 170 | Value: env.LatestDeployment.DurationSeconds, 171 | }) 172 | 173 | storeSetMetric(ctx, c.Store, schemas.Metric{ 174 | Kind: schemas.MetricKindEnvironmentDeploymentJobID, 175 | Labels: env.DefaultLabelsValues(), 176 | Value: float64(env.LatestDeployment.JobID), 177 | }) 178 | 179 | emitStatusMetric( 180 | ctx, 181 | c.Store, 182 | schemas.MetricKindEnvironmentDeploymentStatus, 183 | env.DefaultLabelsValues(), 184 | statusesList[:], 185 | env.LatestDeployment.Status, 186 | env.OutputSparseStatusMetrics, 187 | ) 188 | 189 | storeSetMetric(ctx, c.Store, schemas.Metric{ 190 | Kind: schemas.MetricKindEnvironmentDeploymentTimestamp, 191 | Labels: env.DefaultLabelsValues(), 192 | Value: env.LatestDeployment.Timestamp, 193 | }) 194 | 195 | storeSetMetric(ctx, c.Store, schemas.Metric{ 196 | Kind: schemas.MetricKindEnvironmentInformation, 197 | Labels: infoLabels, 198 | Value: 1, 199 | }) 200 | 201 | return nil 202 | } 203 | -------------------------------------------------------------------------------- /pkg/controller/environments_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestPullEnvironmentsFromProject(t *testing.T) { 15 | ctx, c, mux, srv := newTestController(config.Config{}) 16 | defer srv.Close() 17 | 18 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/environments"), 19 | func(w http.ResponseWriter, r *http.Request) { 20 | fmt.Fprint(w, `[{"name":"main"},{"id":1337,"name":"prod"}]`) 21 | }) 22 | 23 | mux.HandleFunc("/api/v4/projects/foo/environments/1337", 24 | func(w http.ResponseWriter, r *http.Request) { 25 | fmt.Fprint(w, ` 26 | { 27 | "id": 1, 28 | "name": "prod", 29 | "external_url": "https://foo.example.com", 30 | "state": "available", 31 | "last_deployment": { 32 | "ref": "bar", 33 | "created_at": "2019-03-25T18:55:13.252Z", 34 | "deployable": { 35 | "id": 2, 36 | "status": "success", 37 | "tag": false, 38 | "duration": 21623.13423, 39 | "user": { 40 | "username": "alice" 41 | }, 42 | "commit": { 43 | "short_id": "416d8ea1" 44 | } 45 | } 46 | } 47 | }`) 48 | }) 49 | 50 | p := schemas.NewProject("foo") 51 | p.Pull.Environments.Regexp = "^prod" 52 | assert.NoError(t, c.PullEnvironmentsFromProject(ctx, p)) 53 | 54 | storedEnvironments, _ := c.Store.Environments(ctx) 55 | expectedEnvironments := schemas.Environments{ 56 | "54146361": schemas.Environment{ 57 | ProjectName: "foo", 58 | ID: 1337, 59 | Name: "prod", 60 | ExternalURL: "https://foo.example.com", 61 | Available: true, 62 | LatestDeployment: schemas.Deployment{ 63 | JobID: 2, 64 | RefKind: schemas.RefKindBranch, 65 | RefName: "bar", 66 | Username: "alice", 67 | Timestamp: 1553540113, 68 | DurationSeconds: 21623.13423, 69 | CommitShortID: "416d8ea1", 70 | Status: "success", 71 | }, 72 | OutputSparseStatusMetrics: true, 73 | }, 74 | } 75 | assert.Equal(t, expectedEnvironments, storedEnvironments) 76 | } 77 | 78 | func TestPullEnvironmentMetricsSucceed(t *testing.T) { 79 | ctx, c, mux, srv := newTestController(config.Config{}) 80 | defer srv.Close() 81 | 82 | mux.HandleFunc("/api/v4/projects/foo/environments/1", 83 | func(w http.ResponseWriter, r *http.Request) { 84 | fmt.Fprint(w, ` 85 | { 86 | "id": 1, 87 | "name": "prod", 88 | "external_url": "https://foo.example.com", 89 | "state": "available", 90 | "last_deployment": { 91 | "ref": "bar", 92 | "created_at": "2019-03-25T18:55:13.252Z", 93 | "deployable": { 94 | "id": 2, 95 | "status": "success", 96 | "tag": false, 97 | "duration": 21623.13423, 98 | "user": { 99 | "public_email": "foo@bar.com" 100 | }, 101 | "commit": { 102 | "short_id": "416d8ea1" 103 | } 104 | } 105 | } 106 | }`) 107 | }) 108 | 109 | mux.HandleFunc("/api/v4/projects/foo/repository/branches/bar", 110 | func(w http.ResponseWriter, r *http.Request) { 111 | fmt.Fprint(w, ` 112 | { 113 | "commit": { 114 | "short_id": "416d8ea1", 115 | "committed_date": "2019-03-25T18:55:13.252Z" 116 | } 117 | }`) 118 | }) 119 | 120 | env := schemas.Environment{ 121 | ProjectName: "foo", 122 | Name: "prod", 123 | ID: 1, 124 | } 125 | 126 | // Metrics pull shall succeed 127 | assert.NoError(t, c.PullEnvironmentMetrics(ctx, env)) 128 | 129 | // Check if all the metrics exist 130 | metrics, _ := c.Store.Metrics(ctx) 131 | labels := map[string]string{ 132 | "project": "foo", 133 | "environment": "prod", 134 | } 135 | 136 | environmentBehindCommitsCount := schemas.Metric{ 137 | Kind: schemas.MetricKindEnvironmentBehindCommitsCount, 138 | Labels: labels, 139 | Value: 0, 140 | } 141 | assert.Equal(t, environmentBehindCommitsCount, metrics[environmentBehindCommitsCount.Key()]) 142 | 143 | environmentBehindCommitsDurationSeconds := schemas.Metric{ 144 | Kind: schemas.MetricKindEnvironmentBehindDurationSeconds, 145 | Labels: labels, 146 | Value: 0, 147 | } 148 | assert.Equal(t, environmentBehindCommitsDurationSeconds, metrics[environmentBehindCommitsDurationSeconds.Key()]) 149 | 150 | environmentDeploymentDurationSeconds := schemas.Metric{ 151 | Kind: schemas.MetricKindEnvironmentDeploymentDurationSeconds, 152 | Labels: labels, 153 | Value: 21623.13423, 154 | } 155 | assert.Equal(t, environmentDeploymentDurationSeconds, metrics[environmentDeploymentDurationSeconds.Key()]) 156 | 157 | labels["status"] = "success" 158 | status := schemas.Metric{ 159 | Kind: schemas.MetricKindEnvironmentDeploymentStatus, 160 | Labels: labels, 161 | Value: 1, 162 | } 163 | assert.Equal(t, status, metrics[status.Key()]) 164 | } 165 | -------------------------------------------------------------------------------- /pkg/controller/garbage_collector_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 13 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 14 | ) 15 | 16 | func TestGarbageCollectProjects(t *testing.T) { 17 | p1 := schemas.NewProject("cfg/p1") 18 | p2 := schemas.NewProject("cfg/p2") 19 | p3 := schemas.NewProject("wc/p3") 20 | p4 := schemas.NewProject("wc/p4") 21 | 22 | ctx, c, mux, srv := newTestController(config.Config{ 23 | Projects: []config.Project{p1.Project}, 24 | Wildcards: config.Wildcards{ 25 | config.Wildcard{ 26 | Owner: config.WildcardOwner{ 27 | Kind: "group", 28 | Name: "wc", 29 | }, 30 | }, 31 | }, 32 | }) 33 | defer srv.Close() 34 | 35 | mux.HandleFunc("/api/v4/groups/wc/projects", 36 | func(w http.ResponseWriter, r *http.Request) { 37 | fmt.Fprint(w, `[{"id":1, "path_with_namespace": "wc/p3", "jobs_enabled": true}]`) 38 | }) 39 | 40 | c.Store.SetProject(ctx, p1) 41 | c.Store.SetProject(ctx, p2) 42 | c.Store.SetProject(ctx, p3) 43 | c.Store.SetProject(ctx, p4) 44 | 45 | assert.NoError(t, c.GarbageCollectProjects(context.Background())) 46 | storedProjects, err := c.Store.Projects(ctx) 47 | assert.NoError(t, err) 48 | 49 | expectedProjects := schemas.Projects{ 50 | p1.Key(): p1, 51 | p3.Key(): p3, 52 | } 53 | assert.Equal(t, expectedProjects, storedProjects) 54 | } 55 | 56 | func TestGarbageCollectEnvironments(t *testing.T) { 57 | ctx, c, mux, srv := newTestController(config.Config{}) 58 | defer srv.Close() 59 | 60 | mux.HandleFunc("/api/v4/projects/p2/environments", 61 | func(w http.ResponseWriter, r *http.Request) { 62 | fmt.Fprint(w, `[{"name": "main"}]`) 63 | }) 64 | 65 | p2 := schemas.NewProject("p2") 66 | p2.Pull.Environments.Enabled = true 67 | p2.Pull.Environments.Regexp = "^main$" 68 | 69 | envp1main := schemas.Environment{ProjectName: "p1", Name: "main"} 70 | envp2dev := schemas.Environment{ProjectName: "p2", Name: "dev"} 71 | envp2main := schemas.Environment{ProjectName: "p2", Name: "main"} 72 | 73 | c.Store.SetProject(ctx, p2) 74 | c.Store.SetEnvironment(ctx, envp1main) 75 | c.Store.SetEnvironment(ctx, envp2dev) 76 | c.Store.SetEnvironment(ctx, envp2main) 77 | 78 | assert.NoError(t, c.GarbageCollectEnvironments(context.Background())) 79 | storedEnvironments, err := c.Store.Environments(ctx) 80 | assert.NoError(t, err) 81 | 82 | expectedEnvironments := schemas.Environments{ 83 | envp2main.Key(): schemas.Environment{ 84 | ProjectName: "p2", 85 | Name: "main", 86 | OutputSparseStatusMetrics: true, 87 | }, 88 | } 89 | assert.Equal(t, expectedEnvironments, storedEnvironments) 90 | } 91 | 92 | func TestGarbageCollectRefs(t *testing.T) { 93 | ctx, c, mux, srv := newTestController(config.Config{}) 94 | defer srv.Close() 95 | 96 | mux.HandleFunc("/api/v4/projects/p2/repository/branches", 97 | func(w http.ResponseWriter, r *http.Request) { 98 | fmt.Fprint(w, `[{"name": "main"}]`) 99 | }) 100 | 101 | mux.HandleFunc("/api/v4/projects/p2/repository/tags", 102 | func(w http.ResponseWriter, r *http.Request) { 103 | fmt.Fprint(w, `[{"name": "main"}]`) 104 | }) 105 | 106 | pr1dev := schemas.NewRef(schemas.NewProject("p1"), schemas.RefKindBranch, "dev") 107 | pr1main := schemas.NewRef(schemas.NewProject("p1"), schemas.RefKindBranch, "main") 108 | 109 | p2 := schemas.NewProject("p2") 110 | p2.Pull.Environments.Regexp = "^main$" 111 | 112 | pr2dev := schemas.NewRef(p2, schemas.RefKindBranch, "dev") 113 | pr2main := schemas.NewRef(p2, schemas.RefKindBranch, "main") 114 | 115 | c.Store.SetProject(ctx, p2) 116 | c.Store.SetRef(ctx, pr1dev) 117 | c.Store.SetRef(ctx, pr1main) 118 | c.Store.SetRef(ctx, pr2dev) 119 | c.Store.SetRef(ctx, pr2main) 120 | 121 | assert.NoError(t, c.GarbageCollectRefs(context.Background())) 122 | storedRefs, err := c.Store.Refs(ctx) 123 | assert.NoError(t, err) 124 | 125 | newPR2main := schemas.NewRef(p2, schemas.RefKindBranch, "main") 126 | expectedRefs := schemas.Refs{ 127 | newPR2main.Key(): newPR2main, 128 | } 129 | assert.Equal(t, expectedRefs, storedRefs) 130 | } 131 | 132 | func TestGarbageCollectMetrics(t *testing.T) { 133 | ctx, c, _, srv := newTestController(config.Config{}) 134 | srv.Close() 135 | 136 | p1 := schemas.NewProject("p1") 137 | p1.Pull.Pipeline.Jobs.Enabled = true 138 | 139 | ref1 := schemas.NewRef(p1, schemas.RefKindBranch, "foo") 140 | 141 | ref1m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"project": "p1", "ref": "foo", "kind": "branch"}} 142 | ref1m2 := schemas.Metric{Kind: schemas.MetricKindStatus, Labels: prometheus.Labels{"project": "p1", "ref": "foo", "kind": "branch"}} 143 | ref1m3 := schemas.Metric{Kind: schemas.MetricKindJobDurationSeconds, Labels: prometheus.Labels{"project": "p1", "ref": "foo", "kind": "branch"}} 144 | 145 | ref2m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"project": "p2", "ref": "bar", "kind": "branch"}} 146 | ref3m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"project": "foo", "kind": "branch"}} 147 | ref4m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"ref": "bar", "kind": "branch"}} 148 | 149 | c.Store.SetRef(ctx, ref1) 150 | c.Store.SetMetric(ctx, ref1m1) 151 | c.Store.SetMetric(ctx, ref1m2) 152 | c.Store.SetMetric(ctx, ref1m3) 153 | c.Store.SetMetric(ctx, ref2m1) 154 | c.Store.SetMetric(ctx, ref3m1) 155 | c.Store.SetMetric(ctx, ref4m1) 156 | 157 | assert.NoError(t, c.GarbageCollectMetrics(context.Background())) 158 | storedMetrics, err := c.Store.Metrics(ctx) 159 | assert.NoError(t, err) 160 | 161 | expectedMetrics := schemas.Metrics{ 162 | ref1m1.Key(): ref1m1, 163 | ref1m3.Key(): ref1m3, 164 | } 165 | assert.Equal(t, expectedMetrics, storedMetrics) 166 | } 167 | -------------------------------------------------------------------------------- /pkg/controller/handlers.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "reflect" 9 | 10 | "github.com/heptiolabs/healthcheck" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | log "github.com/sirupsen/logrus" 13 | "github.com/xanzy/go-gitlab" 14 | "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" 15 | "go.opentelemetry.io/otel/trace" 16 | ) 17 | 18 | // HealthCheckHandler .. 19 | func (c *Controller) HealthCheckHandler(ctx context.Context) (h healthcheck.Handler) { 20 | h = healthcheck.NewHandler() 21 | if c.Config.Gitlab.EnableHealthCheck { 22 | h.AddReadinessCheck("gitlab-reachable", c.Gitlab.ReadinessCheck(ctx)) 23 | } else { 24 | log.WithContext(ctx). 25 | Warn("GitLab health check has been disabled. Readiness checks won't be operated.") 26 | } 27 | 28 | return 29 | } 30 | 31 | // MetricsHandler .. 32 | func (c *Controller) MetricsHandler(w http.ResponseWriter, r *http.Request) { 33 | ctx := r.Context() 34 | span := trace.SpanFromContext(ctx) 35 | 36 | defer span.End() 37 | 38 | registry := NewRegistry(ctx) 39 | 40 | metrics, err := c.Store.Metrics(ctx) 41 | if err != nil { 42 | log.WithContext(ctx). 43 | WithError(err). 44 | Error() 45 | } 46 | 47 | if err := registry.ExportInternalMetrics( 48 | ctx, 49 | c.Gitlab, 50 | c.Store, 51 | ); err != nil { 52 | log.WithContext(ctx). 53 | WithError(err). 54 | Warn() 55 | } 56 | 57 | registry.ExportMetrics(metrics) 58 | 59 | otelhttp.NewHandler( 60 | promhttp.HandlerFor(registry, promhttp.HandlerOpts{ 61 | Registry: registry, 62 | EnableOpenMetrics: c.Config.Server.Metrics.EnableOpenmetricsEncoding, 63 | }), 64 | "/metrics", 65 | ).ServeHTTP(w, r) 66 | } 67 | 68 | // WebhookHandler .. 69 | func (c *Controller) WebhookHandler(w http.ResponseWriter, r *http.Request) { 70 | span := trace.SpanFromContext(r.Context()) 71 | defer span.End() 72 | 73 | // We create a new background context instead of relying on the request one which has a short cancellation TTL 74 | ctx := trace.ContextWithSpan(context.Background(), span) 75 | 76 | logger := log. 77 | WithContext(ctx). 78 | WithFields(log.Fields{ 79 | "ip-address": r.RemoteAddr, 80 | "user-agent": r.UserAgent(), 81 | }) 82 | 83 | logger.Debug("webhook request") 84 | 85 | if r.Header.Get("X-Gitlab-Token") != c.Config.Server.Webhook.SecretToken { 86 | logger.Debug("invalid token provided for a webhook request") 87 | w.WriteHeader(http.StatusForbidden) 88 | fmt.Fprint(w, "{\"error\": \"invalid token\"}") 89 | 90 | return 91 | } 92 | 93 | if r.Body == http.NoBody { 94 | logger. 95 | WithError(fmt.Errorf("nil body")). 96 | Warn("unable to read body of a received webhook") 97 | 98 | w.WriteHeader(http.StatusBadRequest) 99 | 100 | return 101 | } 102 | 103 | payload, err := io.ReadAll(r.Body) 104 | if err != nil { 105 | logger. 106 | WithError(err). 107 | Warn("unable to read body of a received webhook") 108 | 109 | w.WriteHeader(http.StatusBadRequest) 110 | 111 | return 112 | } 113 | 114 | event, err := gitlab.ParseHook(gitlab.HookEventType(r), payload) 115 | if err != nil { 116 | logger. 117 | WithError(err). 118 | Warn("unable to parse body of a received webhook") 119 | 120 | w.WriteHeader(http.StatusBadRequest) 121 | 122 | return 123 | } 124 | 125 | switch event := event.(type) { 126 | case *gitlab.PipelineEvent: 127 | go c.processPipelineEvent(ctx, *event) 128 | case *gitlab.JobEvent: 129 | go c.processJobEvent(ctx, *event) 130 | case *gitlab.DeploymentEvent: 131 | go c.processDeploymentEvent(ctx, *event) 132 | case *gitlab.PushEvent: 133 | go c.processPushEvent(ctx, *event) 134 | case *gitlab.TagEvent: 135 | go c.processTagEvent(ctx, *event) 136 | case *gitlab.MergeEvent: 137 | go c.processMergeEvent(ctx, *event) 138 | default: 139 | logger. 140 | WithField("event-type", reflect.TypeOf(event).String()). 141 | Warn("received a non supported event type as a webhook") 142 | 143 | w.WriteHeader(http.StatusUnprocessableEntity) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /pkg/controller/handlers_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 13 | ) 14 | 15 | func TestWebhookHandler(t *testing.T) { 16 | _, c, _, srv := newTestController(config.Config{ 17 | Server: config.Server{ 18 | Webhook: config.ServerWebhook{ 19 | Enabled: true, 20 | SecretToken: "secret", 21 | }, 22 | }, 23 | }) 24 | srv.Close() 25 | 26 | req := httptest.NewRequest(http.MethodPost, "/webhook", nil) 27 | 28 | // Test without auth token, should return a 403 29 | w := httptest.NewRecorder() 30 | c.WebhookHandler(w, req) 31 | assert.Equal(t, http.StatusForbidden, w.Result().StatusCode) 32 | 33 | // Provide correct authentication header 34 | req.Header.Add("X-Gitlab-Token", "secret") 35 | 36 | // Test with empty body, should return a 400 37 | w = httptest.NewRecorder() 38 | c.WebhookHandler(w, req) 39 | assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode) 40 | 41 | // Provide an invalid body 42 | req.Body = ioutil.NopCloser(strings.NewReader(`[`)) 43 | 44 | // Test with invalid body, should return a 400 45 | w = httptest.NewRecorder() 46 | c.WebhookHandler(w, req) 47 | assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode) 48 | 49 | // Provide an invalid event type 50 | req.Body = ioutil.NopCloser(strings.NewReader(`{"object_kind": "wiki_page"}`)) 51 | req.Header.Set("X-Gitlab-Event", "Wiki Page Hook") 52 | 53 | // Test with invalid event type, should return a 422 54 | w = httptest.NewRecorder() 55 | c.WebhookHandler(w, req) 56 | assert.Equal(t, http.StatusUnprocessableEntity, w.Result().StatusCode) 57 | 58 | // Provide an valid event type: pipeline 59 | req.Body = ioutil.NopCloser(strings.NewReader(`{"object_kind": "pipeline"}`)) 60 | req.Header.Set("X-Gitlab-Event", "Pipeline Hook") 61 | 62 | // Test with pipeline event type, should return a 200 63 | w = httptest.NewRecorder() 64 | c.WebhookHandler(w, req) 65 | assert.Equal(t, http.StatusOK, w.Result().StatusCode) 66 | 67 | // Provide an valid event type: deployment 68 | req.Body = ioutil.NopCloser(strings.NewReader(`{"object_kind": "deployment"}`)) 69 | req.Header.Set("X-Gitlab-Event", "Deployment Hook") 70 | 71 | // Test with deployment event type, should return a 200 72 | w = httptest.NewRecorder() 73 | c.WebhookHandler(w, req) 74 | assert.Equal(t, http.StatusOK, w.Result().StatusCode) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/controller/jobs.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "regexp" 7 | 8 | log "github.com/sirupsen/logrus" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 11 | ) 12 | 13 | // PullRefPipelineJobsMetrics .. 14 | func (c *Controller) PullRefPipelineJobsMetrics(ctx context.Context, ref schemas.Ref) error { 15 | jobs, err := c.Gitlab.ListRefPipelineJobs(ctx, ref) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | for _, job := range jobs { 21 | c.ProcessJobMetrics(ctx, ref, job) 22 | } 23 | 24 | return nil 25 | } 26 | 27 | // PullRefMostRecentJobsMetrics .. 28 | func (c *Controller) PullRefMostRecentJobsMetrics(ctx context.Context, ref schemas.Ref) error { 29 | if !ref.Project.Pull.Pipeline.Jobs.Enabled { 30 | return nil 31 | } 32 | 33 | jobs, err := c.Gitlab.ListRefMostRecentJobs(ctx, ref) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | for _, job := range jobs { 39 | c.ProcessJobMetrics(ctx, ref, job) 40 | } 41 | 42 | return nil 43 | } 44 | 45 | // ProcessJobMetrics .. 46 | func (c *Controller) ProcessJobMetrics(ctx context.Context, ref schemas.Ref, job schemas.Job) { 47 | projectRefLogFields := log.Fields{ 48 | "project-name": ref.Project.Name, 49 | "job-name": job.Name, 50 | "job-id": job.ID, 51 | } 52 | 53 | labels := ref.DefaultLabelsValues() 54 | labels["stage"] = job.Stage 55 | labels["job_name"] = job.Name 56 | labels["tag_list"] = job.TagList 57 | labels["failure_reason"] = job.FailureReason 58 | 59 | if ref.Project.Pull.Pipeline.Jobs.RunnerDescription.Enabled { 60 | re, err := regexp.Compile(ref.Project.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp) 61 | if err != nil { 62 | log.WithContext(ctx). 63 | WithFields(projectRefLogFields). 64 | WithError(err). 65 | Error("invalid job runner description aggregation regexp") 66 | } 67 | 68 | if re.MatchString(job.Runner.Description) { 69 | labels["runner_description"] = ref.Project.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp 70 | } else { 71 | labels["runner_description"] = job.Runner.Description 72 | } 73 | } else { 74 | // TODO: Figure out how to completely remove it from the exporter instead of keeping it empty 75 | labels["runner_description"] = "" 76 | } 77 | 78 | // Refresh ref state from the store 79 | if err := c.Store.GetRef(ctx, &ref); err != nil { 80 | log.WithContext(ctx). 81 | WithFields(projectRefLogFields). 82 | WithError(err). 83 | Error("getting ref from the store") 84 | 85 | return 86 | } 87 | 88 | // In case a job gets restarted, it will have an ID greated than the previous one(s) 89 | // jobs in new pipelines should get greated IDs too 90 | lastJob, lastJobExists := ref.LatestJobs[job.Name] 91 | if lastJobExists && reflect.DeepEqual(lastJob, job) { 92 | return 93 | } 94 | 95 | // Update the ref in the store 96 | if ref.LatestJobs == nil { 97 | ref.LatestJobs = make(schemas.Jobs) 98 | } 99 | 100 | ref.LatestJobs[job.Name] = job 101 | 102 | if err := c.Store.SetRef(ctx, ref); err != nil { 103 | log.WithContext(ctx). 104 | WithFields(projectRefLogFields). 105 | WithError(err). 106 | Error("writing ref in the store") 107 | 108 | return 109 | } 110 | 111 | log.WithFields(projectRefLogFields).Trace("processing job metrics") 112 | 113 | storeSetMetric(ctx, c.Store, schemas.Metric{ 114 | Kind: schemas.MetricKindJobID, 115 | Labels: labels, 116 | Value: float64(job.ID), 117 | }) 118 | 119 | storeSetMetric(ctx, c.Store, schemas.Metric{ 120 | Kind: schemas.MetricKindJobTimestamp, 121 | Labels: labels, 122 | Value: job.Timestamp, 123 | }) 124 | 125 | storeSetMetric(ctx, c.Store, schemas.Metric{ 126 | Kind: schemas.MetricKindJobDurationSeconds, 127 | Labels: labels, 128 | Value: job.DurationSeconds, 129 | }) 130 | 131 | storeSetMetric(ctx, c.Store, schemas.Metric{ 132 | Kind: schemas.MetricKindJobQueuedDurationSeconds, 133 | Labels: labels, 134 | Value: job.QueuedDurationSeconds, 135 | }) 136 | 137 | jobRunCount := schemas.Metric{ 138 | Kind: schemas.MetricKindJobRunCount, 139 | Labels: labels, 140 | } 141 | 142 | // If the metric does not exist yet, start with 0 instead of 1 143 | // this could cause some false positives in prometheus 144 | // when restarting the exporter otherwise 145 | jobRunCountExists, err := c.Store.MetricExists(ctx, jobRunCount.Key()) 146 | if err != nil { 147 | log.WithContext(ctx). 148 | WithFields(projectRefLogFields). 149 | WithError(err). 150 | Error("checking if metric exists in the store") 151 | 152 | return 153 | } 154 | 155 | // We want to increment this counter only once per job ID if: 156 | // - the metric is already set 157 | // - the job has been triggered 158 | jobTriggeredRegexp := regexp.MustCompile("^(skipped|manual|scheduled)$") 159 | lastJobTriggered := !jobTriggeredRegexp.MatchString(lastJob.Status) 160 | jobTriggered := !jobTriggeredRegexp.MatchString(job.Status) 161 | 162 | if jobRunCountExists && ((lastJob.ID != job.ID && jobTriggered) || (lastJob.ID == job.ID && jobTriggered && !lastJobTriggered)) { 163 | storeGetMetric(ctx, c.Store, &jobRunCount) 164 | 165 | jobRunCount.Value++ 166 | } 167 | 168 | storeSetMetric(ctx, c.Store, jobRunCount) 169 | 170 | storeSetMetric(ctx, c.Store, schemas.Metric{ 171 | Kind: schemas.MetricKindJobArtifactSizeBytes, 172 | Labels: labels, 173 | Value: job.ArtifactSize, 174 | }) 175 | 176 | emitStatusMetric( 177 | ctx, 178 | c.Store, 179 | schemas.MetricKindJobStatus, 180 | labels, 181 | statusesList[:], 182 | job.Status, 183 | ref.Project.OutputSparseStatusMetrics, 184 | ) 185 | } 186 | -------------------------------------------------------------------------------- /pkg/controller/jobs_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestPullRefPipelineJobsMetrics(t *testing.T) { 15 | ctx, c, mux, srv := newTestController(config.Config{}) 16 | defer srv.Close() 17 | 18 | mux.HandleFunc("/api/v4/projects/foo/pipelines/1/jobs", 19 | func(w http.ResponseWriter, r *http.Request) { 20 | fmt.Fprint(w, `[{"id":1,"created_at":"2016-08-11T11:28:34.085Z","started_at":"2016-08-11T11:28:56.085Z"},{"id":2,"created_at":"2016-08-11T11:28:34.085Z","started_at":"2016-08-11T11:28:58.085Z"}]`) 21 | }) 22 | 23 | p := schemas.NewProject("foo") 24 | p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = false 25 | 26 | ref := schemas.NewRef(p, schemas.RefKindBranch, "bar") 27 | ref.LatestPipeline.ID = 1 28 | 29 | // TODO: assert the results? 30 | assert.NoError(t, c.PullRefPipelineJobsMetrics(ctx, ref)) 31 | srv.Close() 32 | assert.Error(t, c.PullRefPipelineJobsMetrics(ctx, ref)) 33 | } 34 | 35 | func TestPullRefMostRecentJobsMetrics(t *testing.T) { 36 | ctx, c, mux, srv := newTestController(config.Config{}) 37 | defer srv.Close() 38 | 39 | mux.HandleFunc("/api/v4/projects/foo/jobs", 40 | func(w http.ResponseWriter, r *http.Request) { 41 | fmt.Fprint(w, `[{"id":1,"created_at":"2016-08-11T11:28:34.085Z"},{"id":2,"created_at":"2016-08-11T11:28:34.085Z"}]`) 42 | }) 43 | 44 | ref := schemas.Ref{ 45 | Project: schemas.NewProject("foo"), 46 | Name: "bar", 47 | LatestJobs: schemas.Jobs{ 48 | "bar": { 49 | ID: 1, 50 | }, 51 | }, 52 | } 53 | 54 | // Test with FetchPipelineJobMetrics disabled 55 | assert.NoError(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) 56 | 57 | // Enable FetchPipelineJobMetrics 58 | ref.Project.Pull.Pipeline.Jobs.Enabled = true 59 | assert.NoError(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) 60 | srv.Close() 61 | assert.Error(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) 62 | } 63 | 64 | func TestProcessJobMetrics(t *testing.T) { 65 | ctx, c, _, srv := newTestController(config.Config{}) 66 | srv.Close() 67 | 68 | oldJob := schemas.Job{ 69 | ID: 1, 70 | Name: "foo", 71 | Timestamp: 1, 72 | } 73 | 74 | newJob := schemas.Job{ 75 | ID: 2, 76 | Name: "foo", 77 | Timestamp: 2, 78 | DurationSeconds: 15, 79 | Status: "failed", 80 | Stage: "🚀", 81 | TagList: "", 82 | ArtifactSize: 150, 83 | Runner: schemas.Runner{ 84 | Description: "foo-123-bar", 85 | }, 86 | } 87 | 88 | p := schemas.NewProject("foo") 89 | p.Topics = "first,second" 90 | p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `foo-(.*)-bar` 91 | 92 | ref := schemas.NewRef(p, schemas.RefKindBranch, "foo") 93 | ref.LatestPipeline.ID = 1 94 | ref.LatestPipeline.Variables = "none" 95 | ref.LatestJobs = schemas.Jobs{ 96 | "foo": oldJob, 97 | } 98 | 99 | c.Store.SetRef(ctx, ref) 100 | 101 | // If we run it against the same job, nothing should change in the store 102 | c.ProcessJobMetrics(ctx, ref, oldJob) 103 | refs, _ := c.Store.Refs(ctx) 104 | assert.Equal(t, schemas.Jobs{ 105 | "foo": oldJob, 106 | }, refs[ref.Key()].LatestJobs) 107 | 108 | // Update the ref 109 | c.ProcessJobMetrics(ctx, ref, newJob) 110 | refs, _ = c.Store.Refs(ctx) 111 | assert.Equal(t, schemas.Jobs{ 112 | "foo": newJob, 113 | }, refs[ref.Key()].LatestJobs) 114 | 115 | // Check if all the metrics exist 116 | metrics, _ := c.Store.Metrics(ctx) 117 | labels := map[string]string{ 118 | "project": ref.Project.Name, 119 | "topics": ref.Project.Topics, 120 | "ref": ref.Name, 121 | "kind": string(ref.Kind), 122 | "variables": ref.LatestPipeline.Variables, 123 | "source": ref.LatestPipeline.Source, 124 | "stage": newJob.Stage, 125 | "tag_list": newJob.TagList, 126 | "failure_reason": newJob.FailureReason, 127 | "job_name": newJob.Name, 128 | "runner_description": ref.Project.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp, 129 | } 130 | 131 | lastJobRunID := schemas.Metric{ 132 | Kind: schemas.MetricKindJobID, 133 | Labels: labels, 134 | Value: 2, 135 | } 136 | assert.Equal(t, lastJobRunID, metrics[lastJobRunID.Key()]) 137 | 138 | timeSinceLastJobRun := schemas.Metric{ 139 | Kind: schemas.MetricKindJobTimestamp, 140 | Labels: labels, 141 | Value: 2, 142 | } 143 | assert.Equal(t, timeSinceLastJobRun, metrics[timeSinceLastJobRun.Key()]) 144 | 145 | lastRunJobDuration := schemas.Metric{ 146 | Kind: schemas.MetricKindJobDurationSeconds, 147 | Labels: labels, 148 | Value: newJob.DurationSeconds, 149 | } 150 | assert.Equal(t, lastRunJobDuration, metrics[lastRunJobDuration.Key()]) 151 | 152 | jobRunCount := schemas.Metric{ 153 | Kind: schemas.MetricKindJobRunCount, 154 | Labels: labels, 155 | Value: 0, 156 | } 157 | assert.Equal(t, jobRunCount, metrics[jobRunCount.Key()]) 158 | 159 | artifactSize := schemas.Metric{ 160 | Kind: schemas.MetricKindJobArtifactSizeBytes, 161 | Labels: labels, 162 | Value: float64(150), 163 | } 164 | assert.Equal(t, artifactSize, metrics[artifactSize.Key()]) 165 | 166 | labels["status"] = newJob.Status 167 | status := schemas.Metric{ 168 | Kind: schemas.MetricKindJobStatus, 169 | Labels: labels, 170 | Value: float64(1), 171 | } 172 | assert.Equal(t, status, metrics[status.Key()]) 173 | } 174 | -------------------------------------------------------------------------------- /pkg/controller/metadata.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | goGitlab "github.com/xanzy/go-gitlab" 7 | 8 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab" 9 | ) 10 | 11 | func (c *Controller) GetGitLabMetadata(ctx context.Context) error { 12 | options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)} 13 | 14 | metadata, _, err := c.Gitlab.Metadata.GetMetadata(options...) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | if metadata.Version != "" { 20 | c.Gitlab.UpdateVersion(gitlab.NewGitLabVersion(metadata.Version)) 21 | } 22 | 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/controller/metadata_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab" 12 | ) 13 | 14 | func TestGetGitLabMetadataSuccess(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | data string 18 | expectedVersion gitlab.GitLabVersion 19 | }{ 20 | { 21 | name: "successful parse", 22 | data: ` 23 | { 24 | "version":"16.7.0-pre", 25 | "revision":"3fe364fe754", 26 | "kas":{ 27 | "enabled":true, 28 | "externalUrl":"wss://kas.gitlab.com", 29 | "version":"v16.7.0-rc2" 30 | }, 31 | "enterprise":true 32 | } 33 | `, 34 | expectedVersion: gitlab.NewGitLabVersion("v16.7.0-pre"), 35 | }, 36 | { 37 | name: "unsuccessful parse", 38 | data: ` 39 | { 40 | "revision":"3fe364fe754" 41 | } 42 | `, 43 | expectedVersion: gitlab.NewGitLabVersion(""), 44 | }, 45 | } 46 | 47 | for _, tc := range tests { 48 | t.Run(tc.name, func(t *testing.T) { 49 | ctx, c, mux, srv := newTestController(config.Config{}) 50 | defer srv.Close() 51 | 52 | mux.HandleFunc("/api/v4/metadata", 53 | func(w http.ResponseWriter, r *http.Request) { 54 | fmt.Fprint(w, tc.data) 55 | }) 56 | 57 | assert.NoError(t, c.GetGitLabMetadata(ctx)) 58 | assert.Equal(t, tc.expectedVersion, c.Gitlab.Version()) 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/controller/metrics_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 13 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 14 | ) 15 | 16 | func TestNewRegistry(t *testing.T) { 17 | r := NewRegistry(context.Background()) 18 | assert.NotNil(t, r.Registry) 19 | assert.NotNil(t, r.Collectors) 20 | } 21 | 22 | // introduce a test to check the /metrics endpoint body. 23 | func TestMetricsHandler(t *testing.T) { 24 | _, c, _, srv := newTestController(config.Config{}) 25 | srv.Close() 26 | 27 | w := httptest.NewRecorder() 28 | r := httptest.NewRequest(http.MethodGet, "/", nil) 29 | c.MetricsHandler(w, r) 30 | 31 | // TODO: Find a way to see if expected metrics are present 32 | assert.Equal(t, http.StatusOK, w.Result().StatusCode) 33 | } 34 | 35 | func TestRegistryGetCollector(t *testing.T) { 36 | r := NewRegistry(context.Background()) 37 | assert.Equal(t, r.Collectors[schemas.MetricKindCoverage], r.GetCollector(schemas.MetricKindCoverage)) 38 | assert.Nil(t, r.GetCollector(150)) 39 | } 40 | 41 | func TestExportMetrics(_ *testing.T) { 42 | r := NewRegistry(context.Background()) 43 | 44 | m1 := schemas.Metric{ 45 | Kind: schemas.MetricKindCoverage, 46 | Labels: prometheus.Labels{ 47 | "project": "foo", 48 | "topics": "alpha", 49 | "ref": "bar", 50 | "kind": "branch", 51 | "source": "schedule", 52 | "variables": "beta", 53 | }, 54 | Value: float64(107.7), 55 | } 56 | 57 | m2 := schemas.Metric{ 58 | Kind: schemas.MetricKindRunCount, 59 | Labels: prometheus.Labels{ 60 | "project": "foo", 61 | "topics": "alpha", 62 | "ref": "bar", 63 | "kind": "branch", 64 | "source": "schedule", 65 | "variables": "beta", 66 | }, 67 | Value: float64(10), 68 | } 69 | 70 | metrics := schemas.Metrics{ 71 | m1.Key(): m1, 72 | m2.Key(): m2, 73 | } 74 | 75 | // TODO: Assert that we have the correct metrics being rendered by the exporter 76 | r.ExportMetrics(metrics) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/controller/projects.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | log "github.com/sirupsen/logrus" 7 | 8 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 9 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 10 | ) 11 | 12 | // PullProject .. 13 | func (c *Controller) PullProject(ctx context.Context, name string, pull config.ProjectPull) error { 14 | gp, err := c.Gitlab.GetProject(ctx, name) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | p := schemas.NewProject(gp.PathWithNamespace) 20 | p.Pull = pull 21 | 22 | projectExists, err := c.Store.ProjectExists(ctx, p.Key()) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if !projectExists { 28 | log.WithFields(log.Fields{ 29 | "project-name": p.Name, 30 | }).Info("discovered new project") 31 | 32 | if err := c.Store.SetProject(ctx, p); err != nil { 33 | log.WithContext(ctx). 34 | WithError(err). 35 | Error() 36 | } 37 | 38 | c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p) 39 | c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p) 40 | } 41 | 42 | return nil 43 | } 44 | 45 | // PullProjectsFromWildcard .. 46 | func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wildcard) error { 47 | foundProjects, err := c.Gitlab.ListProjects(ctx, w) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | for _, p := range foundProjects { 53 | projectExists, err := c.Store.ProjectExists(ctx, p.Key()) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | if !projectExists { 59 | log.WithFields(log.Fields{ 60 | "wildcard-search": w.Search, 61 | "wildcard-owner-kind": w.Owner.Kind, 62 | "wildcard-owner-name": w.Owner.Name, 63 | "wildcard-owner-include-subgroups": w.Owner.IncludeSubgroups, 64 | "wildcard-archived": w.Archived, 65 | "project-name": p.Name, 66 | }).Info("discovered new project") 67 | 68 | if err := c.Store.SetProject(ctx, p); err != nil { 69 | log.WithContext(ctx). 70 | WithError(err). 71 | Error() 72 | } 73 | 74 | c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p) 75 | c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p) 76 | } 77 | } 78 | 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /pkg/controller/projects_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestPullProjectsFromWildcard(t *testing.T) { 15 | ctx, c, mux, srv := newTestController(config.Config{}) 16 | defer srv.Close() 17 | 18 | mux.HandleFunc("/api/v4/projects", 19 | func(w http.ResponseWriter, r *http.Request) { 20 | fmt.Fprint(w, `[{"id":2,"path_with_namespace":"bar","jobs_enabled":true}]`) 21 | }) 22 | 23 | w := config.NewWildcard() 24 | assert.NoError(t, c.PullProjectsFromWildcard(ctx, w)) 25 | 26 | projects, _ := c.Store.Projects(ctx) 27 | p1 := schemas.NewProject("bar") 28 | 29 | expectedProjects := schemas.Projects{ 30 | p1.Key(): p1, 31 | } 32 | assert.Equal(t, expectedProjects, projects) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/controller/refs.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | "dario.cat/mergo" 7 | log "github.com/sirupsen/logrus" 8 | 9 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 10 | ) 11 | 12 | // GetRefs .. 13 | func (c *Controller) GetRefs(ctx context.Context, p schemas.Project) ( 14 | refs schemas.Refs, 15 | err error, 16 | ) { 17 | var pulledRefs schemas.Refs 18 | 19 | refs = make(schemas.Refs) 20 | 21 | if p.Pull.Refs.Branches.Enabled { 22 | // If one of these parameter is set, we will need to fetch the branches from the 23 | // pipelines API instead of the branches one 24 | if !p.Pull.Refs.Branches.ExcludeDeleted || 25 | p.Pull.Refs.Branches.MostRecent > 0 || 26 | p.Pull.Refs.Branches.MaxAgeSeconds > 0 { 27 | if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(ctx, p, schemas.RefKindBranch); err != nil { 28 | return 29 | } 30 | } else { 31 | if pulledRefs, err = c.Gitlab.GetProjectBranches(ctx, p); err != nil { 32 | return 33 | } 34 | } 35 | 36 | if err = mergo.Merge(&refs, pulledRefs); err != nil { 37 | return 38 | } 39 | } 40 | 41 | if p.Pull.Refs.Tags.Enabled { 42 | // If one of these parameter is set, we will need to fetch the tags from the 43 | // pipelines API instead of the tags one 44 | if !p.Pull.Refs.Tags.ExcludeDeleted || 45 | p.Pull.Refs.Tags.MostRecent > 0 || 46 | p.Pull.Refs.Tags.MaxAgeSeconds > 0 { 47 | if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(ctx, p, schemas.RefKindTag); err != nil { 48 | return 49 | } 50 | } else { 51 | if pulledRefs, err = c.Gitlab.GetProjectTags(ctx, p); err != nil { 52 | return 53 | } 54 | } 55 | 56 | if err = mergo.Merge(&refs, pulledRefs); err != nil { 57 | return 58 | } 59 | } 60 | 61 | if p.Pull.Refs.MergeRequests.Enabled { 62 | if pulledRefs, err = c.Gitlab.GetRefsFromPipelines( 63 | ctx, 64 | p, 65 | schemas.RefKindMergeRequest, 66 | ); err != nil { 67 | return 68 | } 69 | 70 | if err = mergo.Merge(&refs, pulledRefs); err != nil { 71 | return 72 | } 73 | } 74 | 75 | return 76 | } 77 | 78 | // PullRefsFromProject .. 79 | func (c *Controller) PullRefsFromProject(ctx context.Context, p schemas.Project) error { 80 | refs, err := c.GetRefs(ctx, p) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | for _, ref := range refs { 86 | refExists, err := c.Store.RefExists(ctx, ref.Key()) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | if !refExists { 92 | log.WithFields(log.Fields{ 93 | "project-name": ref.Project.Name, 94 | "ref": ref.Name, 95 | "ref-kind": ref.Kind, 96 | }).Info("discovered new ref") 97 | 98 | if err = c.Store.SetRef(ctx, ref); err != nil { 99 | return err 100 | } 101 | 102 | c.ScheduleTask(ctx, schemas.TaskTypePullRefMetrics, string(ref.Key()), ref) 103 | } 104 | } 105 | 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/controller/refs_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestGetRefs(t *testing.T) { 15 | ctx, c, mux, srv := newTestController(config.Config{}) 16 | defer srv.Close() 17 | 18 | mux.HandleFunc("/api/v4/projects/foo/repository/branches", 19 | func(w http.ResponseWriter, r *http.Request) { 20 | fmt.Fprint(w, `[{"name":"dev"},{"name":"main"}]`) 21 | }) 22 | 23 | mux.HandleFunc("/api/v4/projects/foo/repository/tags", 24 | func(w http.ResponseWriter, r *http.Request) { 25 | fmt.Fprint(w, `[{"name":"0.0.1"},{"name":"v0.0.2"}]`) 26 | }) 27 | 28 | mux.HandleFunc("/api/v4/projects/foo/pipelines", 29 | func(w http.ResponseWriter, r *http.Request) { 30 | fmt.Fprint(w, `[{"ref":"refs/merge-requests/1234/head"}]`) 31 | }) 32 | 33 | p := schemas.NewProject("foo") 34 | p.Pull.Refs.Branches.Regexp = `^m` 35 | p.Pull.Refs.Tags.Regexp = `^v` 36 | p.Pull.Refs.MergeRequests.Enabled = true 37 | 38 | foundRefs, err := c.GetRefs(ctx, p) 39 | assert.NoError(t, err) 40 | 41 | ref1 := schemas.NewRef(p, schemas.RefKindBranch, "main") 42 | ref2 := schemas.NewRef(p, schemas.RefKindTag, "v0.0.2") 43 | ref3 := schemas.NewRef(p, schemas.RefKindMergeRequest, "1234") 44 | expectedRefs := schemas.Refs{ 45 | ref1.Key(): ref1, 46 | ref2.Key(): ref2, 47 | ref3.Key(): ref3, 48 | } 49 | assert.Equal(t, expectedRefs, foundRefs) 50 | } 51 | 52 | func TestPullRefsFromProject(t *testing.T) { 53 | ctx, c, mux, srv := newTestController(config.Config{}) 54 | defer srv.Close() 55 | 56 | mux.HandleFunc("/api/v4/projects/foo", 57 | func(w http.ResponseWriter, r *http.Request) { 58 | fmt.Fprint(w, `{"name":"foo"}`) 59 | }) 60 | 61 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/branches"), 62 | func(w http.ResponseWriter, r *http.Request) { 63 | fmt.Fprint(w, `[{"name":"main"},{"name":"nope"}]`) 64 | }) 65 | 66 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/tags"), 67 | func(w http.ResponseWriter, r *http.Request) { 68 | fmt.Fprint(w, `[]`) 69 | }) 70 | 71 | p1 := schemas.NewProject("foo") 72 | assert.NoError(t, c.PullRefsFromProject(ctx, p1)) 73 | 74 | ref1 := schemas.NewRef(p1, schemas.RefKindBranch, "main") 75 | expectedRefs := schemas.Refs{ 76 | ref1.Key(): ref1, 77 | } 78 | 79 | projectsRefs, _ := c.Store.Refs(ctx) 80 | assert.Equal(t, expectedRefs, projectsRefs) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/controller/scheduler_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | // TODO 4 | -------------------------------------------------------------------------------- /pkg/controller/store.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | 6 | log "github.com/sirupsen/logrus" 7 | 8 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 9 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/store" 10 | ) 11 | 12 | func metricLogFields(m schemas.Metric) log.Fields { 13 | return log.Fields{ 14 | "metric-kind": m.Kind, 15 | "metric-labels": m.Labels, 16 | } 17 | } 18 | 19 | func storeGetMetric(ctx context.Context, s store.Store, m *schemas.Metric) { 20 | if err := s.GetMetric(ctx, m); err != nil { 21 | log.WithContext(ctx). 22 | WithFields(metricLogFields(*m)). 23 | WithError(err). 24 | Errorf("reading metric from the store") 25 | } 26 | } 27 | 28 | func storeSetMetric(ctx context.Context, s store.Store, m schemas.Metric) { 29 | if err := s.SetMetric(ctx, m); err != nil { 30 | log.WithContext(ctx). 31 | WithFields(metricLogFields(m)). 32 | WithError(err). 33 | Errorf("writing metric from the store") 34 | } 35 | } 36 | 37 | func storeDelMetric(ctx context.Context, s store.Store, m schemas.Metric) { 38 | if err := s.DelMetric(ctx, m.Key()); err != nil { 39 | log.WithContext(ctx). 40 | WithFields(metricLogFields(m)). 41 | WithError(err). 42 | Errorf("deleting metric from the store") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pkg/controller/store_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | log "github.com/sirupsen/logrus" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 11 | ) 12 | 13 | func TestMetricLogFields(t *testing.T) { 14 | m := schemas.Metric{ 15 | Kind: schemas.MetricKindDurationSeconds, 16 | Labels: prometheus.Labels{ 17 | "foo": "bar", 18 | }, 19 | } 20 | expected := log.Fields{ 21 | "metric-kind": schemas.MetricKindDurationSeconds, 22 | "metric-labels": prometheus.Labels{"foo": "bar"}, 23 | } 24 | assert.Equal(t, expected, metricLogFields(m)) 25 | } 26 | 27 | func TestStoreGetSetDelMetric(_ *testing.T) { 28 | // TODO: implement correctly 29 | } 30 | -------------------------------------------------------------------------------- /pkg/controller/webhooks_test.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 9 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 10 | ) 11 | 12 | func TestTriggerRefMetricsPull(t *testing.T) { 13 | ctx, c, _, srv := newTestController(config.Config{}) 14 | srv.Close() 15 | 16 | ref1 := schemas.Ref{ 17 | Project: schemas.NewProject("group/foo"), 18 | Name: "main", 19 | } 20 | 21 | p2 := schemas.NewProject("group/bar") 22 | ref2 := schemas.Ref{ 23 | Project: p2, 24 | Name: "main", 25 | } 26 | 27 | assert.NoError(t, c.Store.SetRef(ctx, ref1)) 28 | assert.NoError(t, c.Store.SetProject(ctx, p2)) 29 | 30 | // TODO: Assert results somehow 31 | c.triggerRefMetricsPull(ctx, ref1) 32 | c.triggerRefMetricsPull(ctx, ref2) 33 | } 34 | 35 | func TestTriggerEnvironmentMetricsPull(t *testing.T) { 36 | ctx, c, _, srv := newTestController(config.Config{}) 37 | srv.Close() 38 | 39 | p1 := schemas.NewProject("foo/bar") 40 | env1 := schemas.Environment{ 41 | ProjectName: p1.Name, 42 | Name: "dev", 43 | } 44 | 45 | env2 := schemas.Environment{ 46 | ProjectName: "foo/baz", 47 | Name: "prod", 48 | } 49 | 50 | assert.NoError(t, c.Store.SetProject(ctx, p1)) 51 | assert.NoError(t, c.Store.SetEnvironment(ctx, env1)) 52 | assert.NoError(t, c.Store.SetEnvironment(ctx, env2)) 53 | 54 | // TODO: Assert results somehow 55 | c.triggerEnvironmentMetricsPull(ctx, env1) 56 | c.triggerEnvironmentMetricsPull(ctx, env2) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/gitlab/branches.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | 7 | log "github.com/sirupsen/logrus" 8 | goGitlab "github.com/xanzy/go-gitlab" 9 | "go.opentelemetry.io/otel" 10 | "go.opentelemetry.io/otel/attribute" 11 | 12 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 13 | ) 14 | 15 | // GetProjectBranches .. 16 | func (c *Client) GetProjectBranches(ctx context.Context, p schemas.Project) ( 17 | refs schemas.Refs, 18 | err error, 19 | ) { 20 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectBranches") 21 | defer span.End() 22 | span.SetAttributes(attribute.String("project_name", p.Name)) 23 | 24 | refs = make(schemas.Refs) 25 | 26 | options := &goGitlab.ListBranchesOptions{ 27 | ListOptions: goGitlab.ListOptions{ 28 | Page: 1, 29 | PerPage: 100, 30 | }, 31 | } 32 | 33 | var re *regexp.Regexp 34 | 35 | if re, err = regexp.Compile(p.Pull.Refs.Branches.Regexp); err != nil { 36 | return 37 | } 38 | 39 | for { 40 | c.rateLimit(ctx) 41 | 42 | var ( 43 | branches []*goGitlab.Branch 44 | resp *goGitlab.Response 45 | ) 46 | 47 | branches, resp, err = c.Branches.ListBranches(p.Name, options, goGitlab.WithContext(ctx)) 48 | if err != nil { 49 | return 50 | } 51 | 52 | c.requestsRemaining(resp) 53 | 54 | for _, branch := range branches { 55 | if re.MatchString(branch.Name) { 56 | ref := schemas.NewRef(p, schemas.RefKindBranch, branch.Name) 57 | refs[ref.Key()] = ref 58 | } 59 | } 60 | 61 | if resp.CurrentPage >= resp.NextPage { 62 | break 63 | } 64 | 65 | options.Page = resp.NextPage 66 | } 67 | 68 | return 69 | } 70 | 71 | // GetBranchLatestCommit .. 72 | func (c *Client) GetBranchLatestCommit(ctx context.Context, project, branch string) (string, float64, error) { 73 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetBranchLatestCommit") 74 | defer span.End() 75 | span.SetAttributes(attribute.String("project_name", project)) 76 | span.SetAttributes(attribute.String("branch_name", branch)) 77 | 78 | log.WithFields(log.Fields{ 79 | "project-name": project, 80 | "branch": branch, 81 | }).Debug("reading project branch") 82 | 83 | c.rateLimit(ctx) 84 | 85 | b, resp, err := c.Branches.GetBranch(project, branch, goGitlab.WithContext(ctx)) 86 | if err != nil { 87 | return "", 0, err 88 | } 89 | 90 | c.requestsRemaining(resp) 91 | 92 | return b.Commit.ShortID, float64(b.Commit.CommittedDate.Unix()), nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/gitlab/branches_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestGetProjectBranches(t *testing.T) { 15 | ctx, mux, server, c := getMockedClient() 16 | defer server.Close() 17 | 18 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/branches"), 19 | func(w http.ResponseWriter, r *http.Request) { 20 | assert.Equal(t, "GET", r.Method) 21 | assert.Equal(t, []string{"100"}, r.URL.Query()["per_page"]) 22 | currentPage, err := strconv.Atoi(r.URL.Query()["page"][0]) 23 | assert.NoError(t, err) 24 | 25 | nextPage := currentPage + 1 26 | 27 | if currentPage == 2 { 28 | nextPage = currentPage 29 | } 30 | 31 | w.Header().Add("X-Page", strconv.Itoa(currentPage)) 32 | w.Header().Add("X-Next-Page", strconv.Itoa(nextPage)) 33 | 34 | if currentPage == 1 { 35 | fmt.Fprint(w, `[{"name":"main"},{"name":"dev"}]`) 36 | 37 | return 38 | } 39 | 40 | fmt.Fprint(w, `[]`) 41 | }) 42 | 43 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/0/repository/branches"), 44 | func(w http.ResponseWriter, r *http.Request) { 45 | w.WriteHeader(http.StatusNotFound) 46 | }) 47 | 48 | p := schemas.NewProject("foo") 49 | expectedRef := schemas.NewRef(p, schemas.RefKindBranch, "main") 50 | refs, err := c.GetProjectBranches(ctx, p) 51 | assert.NoError(t, err) 52 | assert.Len(t, refs, 1) 53 | assert.Equal(t, schemas.Refs{ 54 | expectedRef.Key(): expectedRef, 55 | }, refs) 56 | 57 | // Test invalid project name 58 | p.Name = "invalid" 59 | _, err = c.GetProjectBranches(ctx, p) 60 | assert.Error(t, err) 61 | 62 | // Test invalid regexp 63 | p.Name = "foo" 64 | p.Pull.Refs.Branches.Regexp = `[` 65 | _, err = c.GetProjectBranches(ctx, p) 66 | assert.Error(t, err) 67 | } 68 | 69 | func TestGetBranchLatestCommit(t *testing.T) { 70 | ctx, mux, server, c := getMockedClient() 71 | defer server.Close() 72 | 73 | mux.HandleFunc("/api/v4/projects/1/repository/branches/main", 74 | func(w http.ResponseWriter, r *http.Request) { 75 | assert.Equal(t, r.Method, "GET") 76 | fmt.Fprint(w, ` 77 | { 78 | "commit": { 79 | "short_id": "7b5c3cc", 80 | "committed_date": "2019-03-25T18:55:13.252Z" 81 | } 82 | }`) 83 | }) 84 | 85 | commitShortID, commitCreatedAt, err := c.GetBranchLatestCommit(ctx, "1", "main") 86 | assert.NoError(t, err) 87 | assert.Equal(t, "7b5c3cc", commitShortID) 88 | assert.Equal(t, float64(1553540113), commitCreatedAt) 89 | } 90 | -------------------------------------------------------------------------------- /pkg/gitlab/client.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net/http" 8 | "strconv" 9 | "sync" 10 | "sync/atomic" 11 | "time" 12 | 13 | "github.com/heptiolabs/healthcheck" 14 | "github.com/paulbellamy/ratecounter" 15 | goGitlab "github.com/xanzy/go-gitlab" 16 | "go.opentelemetry.io/otel" 17 | 18 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/ratelimit" 19 | ) 20 | 21 | const ( 22 | userAgent = "gitlab-ci-pipelines-exporter" 23 | tracerName = "gitlab-ci-pipelines-exporter" 24 | ) 25 | 26 | // Client .. 27 | type Client struct { 28 | *goGitlab.Client 29 | 30 | Readiness struct { 31 | URL string 32 | HTTPClient *http.Client 33 | } 34 | 35 | RateLimiter ratelimit.Limiter 36 | RateCounter *ratecounter.RateCounter 37 | RequestsCounter atomic.Uint64 38 | RequestsLimit int 39 | RequestsRemaining int 40 | 41 | version GitLabVersion 42 | mutex sync.RWMutex 43 | } 44 | 45 | // ClientConfig .. 46 | type ClientConfig struct { 47 | URL string 48 | Token string 49 | UserAgentVersion string 50 | DisableTLSVerify bool 51 | ReadinessURL string 52 | 53 | RateLimiter ratelimit.Limiter 54 | } 55 | 56 | // NewHTTPClient .. 57 | func NewHTTPClient(disableTLSVerify bool) *http.Client { 58 | // http.DefaultTransport contains useful settings such as the correct values for the picking 59 | // up proxy informations from env variables 60 | transport := http.DefaultTransport.(*http.Transport).Clone() 61 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: disableTLSVerify} 62 | 63 | return &http.Client{ 64 | Transport: transport, 65 | } 66 | } 67 | 68 | // NewClient .. 69 | func NewClient(cfg ClientConfig) (*Client, error) { 70 | opts := []goGitlab.ClientOptionFunc{ 71 | goGitlab.WithHTTPClient(NewHTTPClient(cfg.DisableTLSVerify)), 72 | goGitlab.WithBaseURL(cfg.URL), 73 | goGitlab.WithoutRetries(), 74 | } 75 | 76 | gc, err := goGitlab.NewOAuthClient(cfg.Token, opts...) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | gc.UserAgent = fmt.Sprintf("%s-%s", userAgent, cfg.UserAgentVersion) 82 | 83 | readinessCheckHTTPClient := NewHTTPClient(cfg.DisableTLSVerify) 84 | readinessCheckHTTPClient.Timeout = 5 * time.Second 85 | 86 | return &Client{ 87 | Client: gc, 88 | RateLimiter: cfg.RateLimiter, 89 | Readiness: struct { 90 | URL string 91 | HTTPClient *http.Client 92 | }{ 93 | URL: cfg.ReadinessURL, 94 | HTTPClient: readinessCheckHTTPClient, 95 | }, 96 | RateCounter: ratecounter.NewRateCounter(time.Second), 97 | }, nil 98 | } 99 | 100 | // ReadinessCheck .. 101 | func (c *Client) ReadinessCheck(ctx context.Context) healthcheck.Check { 102 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ReadinessCheck") 103 | defer span.End() 104 | 105 | return func() error { 106 | if c.Readiness.HTTPClient == nil { 107 | return fmt.Errorf("readiness http client not configured") 108 | } 109 | 110 | req, err := http.NewRequestWithContext( 111 | ctx, 112 | http.MethodGet, 113 | c.Readiness.URL, 114 | nil, 115 | ) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | resp, err := c.Readiness.HTTPClient.Do(req) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | if resp == nil { 126 | return fmt.Errorf("HTTP error: empty response") 127 | } 128 | 129 | if err == nil && resp.StatusCode != http.StatusOK { 130 | return fmt.Errorf("HTTP error: %d", resp.StatusCode) 131 | } 132 | 133 | return nil 134 | } 135 | } 136 | 137 | func (c *Client) rateLimit(ctx context.Context) { 138 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:rateLimit") 139 | defer span.End() 140 | 141 | ratelimit.Take(ctx, c.RateLimiter) 142 | // Used for monitoring purposes 143 | c.RateCounter.Incr(1) 144 | c.RequestsCounter.Add(1) 145 | } 146 | 147 | func (c *Client) UpdateVersion(version GitLabVersion) { 148 | c.mutex.Lock() 149 | defer c.mutex.Unlock() 150 | c.version = version 151 | } 152 | 153 | func (c *Client) Version() GitLabVersion { 154 | c.mutex.RLock() 155 | defer c.mutex.RUnlock() 156 | 157 | return c.version 158 | } 159 | 160 | func (c *Client) requestsRemaining(response *goGitlab.Response) { 161 | if response == nil { 162 | return 163 | } 164 | 165 | if remaining := response.Header.Get("ratelimit-remaining"); remaining != "" { 166 | c.RequestsRemaining, _ = strconv.Atoi(remaining) 167 | } 168 | 169 | if limit := response.Header.Get("ratelimit-limit"); limit != "" { 170 | c.RequestsLimit, _ = strconv.Atoi(limit) 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /pkg/gitlab/client_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | "time" 10 | 11 | "github.com/paulbellamy/ratecounter" 12 | "github.com/stretchr/testify/assert" 13 | goGitlab "github.com/xanzy/go-gitlab" 14 | 15 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/ratelimit" 16 | ) 17 | 18 | // Mocking helpers. 19 | func getMockedClient() (context.Context, *http.ServeMux, *httptest.Server, *Client) { 20 | mux := http.NewServeMux() 21 | server := httptest.NewServer(mux) 22 | 23 | opts := []goGitlab.ClientOptionFunc{ 24 | goGitlab.WithBaseURL(server.URL), 25 | goGitlab.WithoutRetries(), 26 | } 27 | 28 | gc, _ := goGitlab.NewClient("", opts...) 29 | 30 | c := &Client{ 31 | Client: gc, 32 | RateLimiter: ratelimit.NewLocalLimiter(100, 1), 33 | RateCounter: ratecounter.NewRateCounter(time.Second), 34 | } 35 | 36 | return context.Background(), mux, server, c 37 | } 38 | 39 | func TestNewHTTPClient(t *testing.T) { 40 | c := NewHTTPClient(true) 41 | assert.True(t, c.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) 42 | } 43 | 44 | func TestNewClient(t *testing.T) { 45 | cfg := ClientConfig{ 46 | URL: "https://gitlab.example.com", 47 | Token: "supersecret", 48 | UserAgentVersion: "0.0.0", 49 | DisableTLSVerify: true, 50 | ReadinessURL: "https://gitlab.example.com/amialive", 51 | RateLimiter: ratelimit.NewLocalLimiter(10, 1), 52 | } 53 | 54 | c, err := NewClient(cfg) 55 | assert.NoError(t, err) 56 | assert.NotNil(t, c.Client) 57 | assert.Equal(t, "gitlab-ci-pipelines-exporter-0.0.0", c.Client.UserAgent) 58 | assert.Equal(t, "https", c.Client.BaseURL().Scheme) 59 | assert.Equal(t, "gitlab.example.com", c.Client.BaseURL().Host) 60 | assert.Equal(t, "https://gitlab.example.com/amialive", c.Readiness.URL) 61 | assert.True(t, c.Readiness.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify) 62 | assert.Equal(t, 5*time.Second, c.Readiness.HTTPClient.Timeout) 63 | } 64 | 65 | func TestReadinessCheck(t *testing.T) { 66 | ctx, mux, server, c := getMockedClient() 67 | mux.HandleFunc( 68 | "/200", 69 | func(w http.ResponseWriter, r *http.Request) { 70 | assert.Equal(t, "GET", r.Method) 71 | w.WriteHeader(http.StatusOK) 72 | }, 73 | ) 74 | mux.HandleFunc( 75 | "/500", 76 | func(w http.ResponseWriter, r *http.Request) { 77 | w.WriteHeader(http.StatusInternalServerError) 78 | }, 79 | ) 80 | 81 | readinessCheck := c.ReadinessCheck(ctx) 82 | assert.Error(t, readinessCheck()) 83 | 84 | c.Readiness.HTTPClient = NewHTTPClient(false) 85 | c.Readiness.URL = fmt.Sprintf("%s/200", server.URL) 86 | 87 | assert.NoError(t, readinessCheck()) 88 | 89 | c.Readiness.URL = fmt.Sprintf("%s/500", server.URL) 90 | 91 | assert.Error(t, readinessCheck()) 92 | } 93 | -------------------------------------------------------------------------------- /pkg/gitlab/environments.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | 7 | log "github.com/sirupsen/logrus" 8 | goGitlab "github.com/xanzy/go-gitlab" 9 | "go.opentelemetry.io/otel" 10 | "go.opentelemetry.io/otel/attribute" 11 | 12 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 13 | ) 14 | 15 | // GetProjectEnvironments .. 16 | func (c *Client) GetProjectEnvironments(ctx context.Context, p schemas.Project) ( 17 | envs schemas.Environments, 18 | err error, 19 | ) { 20 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectEnvironments") 21 | defer span.End() 22 | span.SetAttributes(attribute.String("project_name", p.Name)) 23 | 24 | envs = make(schemas.Environments) 25 | 26 | options := &goGitlab.ListEnvironmentsOptions{ 27 | ListOptions: goGitlab.ListOptions{ 28 | Page: 1, 29 | PerPage: 100, 30 | }, 31 | } 32 | 33 | if p.Pull.Environments.ExcludeStopped { 34 | options.States = goGitlab.String("available") 35 | } 36 | 37 | re, err := regexp.Compile(p.Pull.Environments.Regexp) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | for { 43 | c.rateLimit(ctx) 44 | 45 | var ( 46 | glenvs []*goGitlab.Environment 47 | resp *goGitlab.Response 48 | ) 49 | 50 | glenvs, resp, err = c.Environments.ListEnvironments(p.Name, options, goGitlab.WithContext(ctx)) 51 | if err != nil { 52 | return 53 | } 54 | 55 | c.requestsRemaining(resp) 56 | 57 | for _, glenv := range glenvs { 58 | if re.MatchString(glenv.Name) { 59 | env := schemas.Environment{ 60 | ProjectName: p.Name, 61 | ID: glenv.ID, 62 | Name: glenv.Name, 63 | OutputSparseStatusMetrics: p.OutputSparseStatusMetrics, 64 | } 65 | 66 | if glenv.State == "available" { 67 | env.Available = true 68 | } 69 | 70 | envs[env.Key()] = env 71 | } 72 | } 73 | 74 | if resp.CurrentPage >= resp.NextPage { 75 | break 76 | } 77 | 78 | options.Page = resp.NextPage 79 | } 80 | 81 | return 82 | } 83 | 84 | // GetEnvironment .. 85 | func (c *Client) GetEnvironment( 86 | ctx context.Context, 87 | project string, 88 | environmentID int, 89 | ) ( 90 | environment schemas.Environment, 91 | err error, 92 | ) { 93 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetEnvironment") 94 | defer span.End() 95 | span.SetAttributes(attribute.String("project_name", project)) 96 | span.SetAttributes(attribute.Int("environment_id", environmentID)) 97 | 98 | environment = schemas.Environment{ 99 | ProjectName: project, 100 | ID: environmentID, 101 | } 102 | 103 | c.rateLimit(ctx) 104 | 105 | var ( 106 | e *goGitlab.Environment 107 | resp *goGitlab.Response 108 | ) 109 | 110 | e, resp, err = c.Environments.GetEnvironment(project, environmentID, goGitlab.WithContext(ctx)) 111 | if err != nil || e == nil { 112 | return 113 | } 114 | 115 | c.requestsRemaining(resp) 116 | 117 | environment.Name = e.Name 118 | environment.ExternalURL = e.ExternalURL 119 | 120 | if e.State == "available" { 121 | environment.Available = true 122 | } 123 | 124 | if e.LastDeployment == nil { 125 | log.WithContext(ctx). 126 | WithFields(log.Fields{ 127 | "project-name": project, 128 | "environment-name": e.Name, 129 | }). 130 | Debug("no deployments found for the environment") 131 | 132 | return 133 | } 134 | 135 | if e.LastDeployment.Deployable.Tag { 136 | environment.LatestDeployment.RefKind = schemas.RefKindTag 137 | } else { 138 | environment.LatestDeployment.RefKind = schemas.RefKindBranch 139 | } 140 | 141 | environment.LatestDeployment.RefName = e.LastDeployment.Ref 142 | environment.LatestDeployment.JobID = e.LastDeployment.Deployable.ID 143 | environment.LatestDeployment.DurationSeconds = e.LastDeployment.Deployable.Duration 144 | environment.LatestDeployment.Status = e.LastDeployment.Deployable.Status 145 | 146 | if e.LastDeployment.Deployable.User != nil { 147 | environment.LatestDeployment.Username = e.LastDeployment.Deployable.User.Username 148 | } 149 | 150 | if e.LastDeployment.Deployable.Commit != nil { 151 | environment.LatestDeployment.CommitShortID = e.LastDeployment.Deployable.Commit.ShortID 152 | } 153 | 154 | if e.LastDeployment.CreatedAt != nil { 155 | environment.LatestDeployment.Timestamp = float64(e.LastDeployment.CreatedAt.Unix()) 156 | } 157 | 158 | return 159 | } 160 | -------------------------------------------------------------------------------- /pkg/gitlab/environments_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestGetProjectEnvironments(t *testing.T) { 15 | ctx, mux, server, c := getMockedClient() 16 | defer server.Close() 17 | 18 | mux.HandleFunc( 19 | "/api/v4/projects/foo/environments", 20 | func(w http.ResponseWriter, r *http.Request) { 21 | assert.Equal(t, "GET", r.Method) 22 | assert.Equal(t, []string{"100"}, r.URL.Query()["per_page"]) 23 | currentPage, err := strconv.Atoi(r.URL.Query()["page"][0]) 24 | assert.NoError(t, err) 25 | nextPage := currentPage + 1 26 | if currentPage == 2 { 27 | nextPage = currentPage 28 | } 29 | 30 | w.Header().Add("X-Page", strconv.Itoa(currentPage)) 31 | w.Header().Add("X-Next-Page", strconv.Itoa(nextPage)) 32 | 33 | if scope, ok := r.URL.Query()["states"]; ok && len(scope) == 1 && scope[0] == "available" { 34 | fmt.Fprint(w, `[{"id":1338,"name":"main"}]`) 35 | 36 | return 37 | } 38 | 39 | if currentPage == 1 { 40 | fmt.Fprint(w, `[{"id":1338,"name":"main"},{"id":1337,"name":"dev"}]`) 41 | 42 | return 43 | } 44 | 45 | fmt.Fprint(w, `[]`) 46 | }, 47 | ) 48 | 49 | mux.HandleFunc( 50 | "/api/v4/projects/0/environments", 51 | func(w http.ResponseWriter, r *http.Request) { 52 | w.WriteHeader(http.StatusNotFound) 53 | }, 54 | ) 55 | 56 | p := schemas.NewProject("foo") 57 | p.Pull.Environments.Regexp = "^dev" 58 | p.Pull.Environments.ExcludeStopped = false 59 | 60 | xenv := schemas.Environment{ 61 | ProjectName: "foo", 62 | Name: "dev", 63 | ID: 1337, 64 | OutputSparseStatusMetrics: true, 65 | } 66 | 67 | xenvs := schemas.Environments{ 68 | xenv.Key(): xenv, 69 | } 70 | 71 | envs, err := c.GetProjectEnvironments(ctx, p) 72 | assert.NoError(t, err) 73 | assert.Equal(t, xenvs, envs) 74 | 75 | // Test invalid project 76 | p.Name = "" 77 | _, err = c.GetProjectEnvironments(ctx, p) 78 | assert.Error(t, err) 79 | 80 | // Test invalid regexp 81 | p.Name = "foo" 82 | p.Pull.Environments.Regexp = "[" 83 | _, err = c.GetProjectEnvironments(ctx, p) 84 | assert.Error(t, err) 85 | 86 | // Test exclude stopped 87 | xenv = schemas.Environment{ 88 | ProjectName: "foo", 89 | Name: "main", 90 | ID: 1338, 91 | OutputSparseStatusMetrics: true, 92 | } 93 | 94 | xenvs = schemas.Environments{ 95 | xenv.Key(): xenv, 96 | } 97 | 98 | p.Pull.Environments.Regexp = ".*" 99 | p.Pull.Environments.ExcludeStopped = true 100 | envs, err = c.GetProjectEnvironments(ctx, p) 101 | assert.NoError(t, err) 102 | assert.Equal(t, xenvs, envs) 103 | } 104 | 105 | func TestGetEnvironment(t *testing.T) { 106 | ctx, mux, server, c := getMockedClient() 107 | defer server.Close() 108 | 109 | mux.HandleFunc("/api/v4/projects/foo/environments/1", 110 | func(w http.ResponseWriter, r *http.Request) { 111 | assert.Equal(t, r.Method, "GET") 112 | fmt.Fprint(w, ` 113 | { 114 | "id": 1, 115 | "name": "foo", 116 | "external_url": "https://foo.example.com", 117 | "state": "available", 118 | "last_deployment": { 119 | "ref": "bar", 120 | "created_at": "2019-03-25T18:55:13.252Z", 121 | "deployable": { 122 | "id": 23, 123 | "status": "success", 124 | "tag": false, 125 | "duration": 21623.13423, 126 | "user": { 127 | "username": "alice" 128 | }, 129 | "commit": { 130 | "short_id": "416d8ea1" 131 | } 132 | } 133 | } 134 | }`) 135 | }) 136 | 137 | e, err := c.GetEnvironment(ctx, "foo", 1) 138 | assert.NoError(t, err) 139 | assert.NotNil(t, e) 140 | 141 | expectedEnv := schemas.Environment{ 142 | ProjectName: "foo", 143 | ID: 1, 144 | Name: "foo", 145 | ExternalURL: "https://foo.example.com", 146 | Available: true, 147 | LatestDeployment: schemas.Deployment{ 148 | JobID: 23, 149 | RefKind: schemas.RefKindBranch, 150 | RefName: "bar", 151 | Username: "alice", 152 | Timestamp: 1553540113, 153 | DurationSeconds: 21623.13423, 154 | CommitShortID: "416d8ea1", 155 | Status: "success", 156 | }, 157 | } 158 | assert.Equal(t, expectedEnv, e) 159 | } 160 | -------------------------------------------------------------------------------- /pkg/gitlab/jobs_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestListRefPipelineJobs(t *testing.T) { 15 | ctx, mux, server, c := getMockedClient() 16 | defer server.Close() 17 | 18 | ref := schemas.Ref{ 19 | Project: schemas.NewProject("foo"), 20 | Name: "yay", 21 | } 22 | 23 | // Test with no most recent pipeline defined 24 | jobs, err := c.ListRefPipelineJobs(ctx, ref) 25 | assert.NoError(t, err) 26 | assert.Len(t, jobs, 0) 27 | 28 | mux.HandleFunc("/api/v4/projects/foo/pipelines/1/jobs", 29 | func(w http.ResponseWriter, r *http.Request) { 30 | fmt.Fprint(w, `[{"id":10}]`) 31 | }) 32 | 33 | mux.HandleFunc("/api/v4/projects/11/pipelines/2/jobs", 34 | func(w http.ResponseWriter, r *http.Request) { 35 | fmt.Fprint(w, `[{"id":20}]`) 36 | }) 37 | 38 | mux.HandleFunc("/api/v4/projects/12/pipelines/3/jobs", 39 | func(w http.ResponseWriter, r *http.Request) { 40 | fmt.Fprint(w, `[{"id":30}]`) 41 | }) 42 | 43 | mux.HandleFunc("/api/v4/projects/foo/pipelines/1/bridges", 44 | func(w http.ResponseWriter, r *http.Request) { 45 | fmt.Fprint(w, `[{"id":1,"downstream_pipeline":{"id":2, "project_id": 11}}]`) 46 | }) 47 | 48 | mux.HandleFunc("/api/v4/projects/11/pipelines/2/bridges", 49 | func(w http.ResponseWriter, r *http.Request) { 50 | fmt.Fprint(w, `[{"id":1,"downstream_pipeline":{"id":3, "project_id": 12}}]`) 51 | }) 52 | 53 | mux.HandleFunc("/api/v4/projects/12/pipelines/3/bridges", 54 | func(w http.ResponseWriter, r *http.Request) { 55 | fmt.Fprint(w, `[]`) 56 | }) 57 | 58 | ref.LatestPipeline = schemas.Pipeline{ 59 | ID: 1, 60 | } 61 | 62 | jobs, err = c.ListRefPipelineJobs(ctx, ref) 63 | assert.NoError(t, err) 64 | assert.Equal(t, []schemas.Job{ 65 | {ID: 10}, 66 | {ID: 20}, 67 | {ID: 30}, 68 | }, jobs) 69 | 70 | // Test invalid project id 71 | ref.Project.Name = "bar" 72 | _, err = c.ListRefPipelineJobs(ctx, ref) 73 | assert.Error(t, err) 74 | } 75 | 76 | func TestListPipelineJobs(t *testing.T) { 77 | ctx, mux, server, c := getMockedClient() 78 | defer server.Close() 79 | 80 | mux.HandleFunc("/api/v4/projects/foo/pipelines/1/jobs", 81 | func(w http.ResponseWriter, r *http.Request) { 82 | assert.Equal(t, "GET", r.Method) 83 | expectedQueryParams := url.Values{ 84 | "page": []string{"1"}, 85 | "per_page": []string{"100"}, 86 | } 87 | assert.Equal(t, expectedQueryParams, r.URL.Query()) 88 | fmt.Fprint(w, `[{"id":1},{"id":2}]`) 89 | }) 90 | 91 | mux.HandleFunc("/api/v4/projects/bar/pipelines/1/jobs", 92 | func(w http.ResponseWriter, r *http.Request) { 93 | w.WriteHeader(http.StatusNotFound) 94 | }) 95 | 96 | jobs, err := c.ListPipelineJobs(ctx, "foo", 1) 97 | assert.NoError(t, err) 98 | assert.Len(t, jobs, 2) 99 | 100 | // Test invalid project id 101 | _, err = c.ListPipelineJobs(ctx, "bar", 1) 102 | assert.Error(t, err) 103 | } 104 | 105 | func TestListPipelineBridges(t *testing.T) { 106 | ctx, mux, server, c := getMockedClient() 107 | defer server.Close() 108 | 109 | mux.HandleFunc("/api/v4/projects/foo/pipelines/1/bridges", 110 | func(w http.ResponseWriter, r *http.Request) { 111 | assert.Equal(t, "GET", r.Method) 112 | expectedQueryParams := url.Values{ 113 | "page": []string{"1"}, 114 | "per_page": []string{"100"}, 115 | } 116 | assert.Equal(t, expectedQueryParams, r.URL.Query()) 117 | fmt.Fprint(w, `[{"id":1,"pipeline":{"id":100}}]`) 118 | }) 119 | 120 | mux.HandleFunc("/api/v4/projects/bar/pipelines/1/bridges", 121 | func(w http.ResponseWriter, r *http.Request) { 122 | w.WriteHeader(http.StatusNotFound) 123 | }) 124 | 125 | bridges, err := c.ListPipelineBridges(ctx, "foo", 1) 126 | assert.NoError(t, err) 127 | assert.Len(t, bridges, 1) 128 | 129 | // Test invalid project id 130 | _, err = c.ListPipelineBridges(ctx, "bar", 1) 131 | assert.Error(t, err) 132 | } 133 | 134 | func TestListRefMostRecentJobs(t *testing.T) { 135 | tests := []struct { 136 | name string 137 | keysetPagination bool 138 | expectedQueryParams url.Values 139 | }{ 140 | { 141 | name: "offset pagination", 142 | keysetPagination: false, 143 | expectedQueryParams: url.Values{ 144 | "page": []string{"1"}, 145 | "per_page": []string{"100"}, 146 | }, 147 | }, 148 | { 149 | name: "keyset pagination", 150 | keysetPagination: true, 151 | expectedQueryParams: url.Values{ 152 | "pagination": []string{"keyset"}, 153 | "per_page": []string{"100"}, 154 | }, 155 | }, 156 | } 157 | 158 | for _, tc := range tests { 159 | t.Run(tc.name, func(t *testing.T) { 160 | ctx, mux, server, c := getMockedClient() 161 | defer server.Close() 162 | 163 | if tc.keysetPagination { 164 | c.UpdateVersion(NewGitLabVersion("16.0.0")) 165 | } else { 166 | c.UpdateVersion(NewGitLabVersion("15.0.0")) 167 | } 168 | 169 | ref := schemas.Ref{ 170 | Project: schemas.NewProject("foo"), 171 | Name: "yay", 172 | } 173 | 174 | jobs, err := c.ListRefMostRecentJobs(ctx, ref) 175 | assert.NoError(t, err) 176 | assert.Len(t, jobs, 0) 177 | 178 | mux.HandleFunc("/api/v4/projects/foo/jobs", 179 | func(w http.ResponseWriter, r *http.Request) { 180 | assert.Equal(t, "GET", r.Method) 181 | assert.Equal(t, tc.expectedQueryParams, r.URL.Query()) 182 | fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`) 183 | }) 184 | 185 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"), 186 | func(w http.ResponseWriter, r *http.Request) { 187 | w.WriteHeader(http.StatusNotFound) 188 | }) 189 | 190 | ref.LatestJobs = schemas.Jobs{ 191 | "foo": { 192 | ID: 1, 193 | Name: "foo", 194 | }, 195 | "bar": { 196 | ID: 2, 197 | Name: "bar", 198 | }, 199 | } 200 | 201 | jobs, err = c.ListRefMostRecentJobs(ctx, ref) 202 | assert.NoError(t, err) 203 | assert.Len(t, jobs, 2) 204 | assert.Equal(t, 3, jobs[0].ID) 205 | assert.Equal(t, 4, jobs[1].ID) 206 | 207 | ref.LatestJobs["baz"] = schemas.Job{ 208 | ID: 5, 209 | Name: "baz", 210 | } 211 | 212 | jobs, err = c.ListRefMostRecentJobs(ctx, ref) 213 | assert.NoError(t, err) 214 | assert.Len(t, jobs, 2) 215 | assert.Equal(t, 3, jobs[0].ID) 216 | assert.Equal(t, 4, jobs[1].ID) 217 | 218 | // Test invalid project id 219 | ref.Project.Name = "bar" 220 | _, err = c.ListRefMostRecentJobs(ctx, ref) 221 | assert.Error(t, err) 222 | }) 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /pkg/gitlab/projects.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "regexp" 7 | 8 | log "github.com/sirupsen/logrus" 9 | goGitlab "github.com/xanzy/go-gitlab" 10 | "go.openly.dev/pointy" 11 | "go.opentelemetry.io/otel" 12 | "go.opentelemetry.io/otel/attribute" 13 | 14 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 15 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 16 | ) 17 | 18 | // GetProject .. 19 | func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project, error) { 20 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProject") 21 | defer span.End() 22 | span.SetAttributes(attribute.String("project_name", name)) 23 | 24 | log.WithFields(log.Fields{ 25 | "project-name": name, 26 | }).Debug("reading project") 27 | 28 | c.rateLimit(ctx) 29 | p, resp, err := c.Projects.GetProject(name, &goGitlab.GetProjectOptions{}, goGitlab.WithContext(ctx)) 30 | c.requestsRemaining(resp) 31 | 32 | return p, err 33 | } 34 | 35 | // ListProjects .. 36 | func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas.Project, error) { 37 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ListProjects") 38 | defer span.End() 39 | span.SetAttributes(attribute.String("wildcard_search", w.Search)) 40 | span.SetAttributes(attribute.String("wildcard_owner_kind", w.Owner.Kind)) 41 | span.SetAttributes(attribute.String("wildcard_owner_name", w.Owner.Name)) 42 | span.SetAttributes(attribute.Bool("wildcard_owner_include_subgroups", w.Owner.IncludeSubgroups)) 43 | span.SetAttributes(attribute.Bool("wildcard_archived", w.Archived)) 44 | 45 | logFields := log.Fields{ 46 | "wildcard-search": w.Search, 47 | "wildcard-owner-kind": w.Owner.Kind, 48 | "wildcard-owner-name": w.Owner.Name, 49 | "wildcard-owner-include-subgroups": w.Owner.IncludeSubgroups, 50 | "wildcard-archived": w.Archived, 51 | } 52 | log.WithFields(logFields).Debug("listing all projects from wildcard") 53 | 54 | var projects []schemas.Project 55 | 56 | listOptions := goGitlab.ListOptions{ 57 | Page: 1, 58 | PerPage: 100, 59 | } 60 | 61 | // As a result, the API will return the projects that the owner has access onto. 62 | // This is not necessarily what we the end-user would intend when leveraging a 63 | // scoped wildcard. Therefore, if the wildcard owner name is set, we want to filter 64 | // out to project actually *belonging* to the owner. 65 | var ownerRegexp *regexp.Regexp 66 | 67 | if len(w.Owner.Name) > 0 { 68 | ownerRegexp = regexp.MustCompile(fmt.Sprintf(`^%s\/`, w.Owner.Name)) 69 | } else { 70 | ownerRegexp = regexp.MustCompile(`.*`) 71 | } 72 | 73 | for { 74 | var ( 75 | gps []*goGitlab.Project 76 | resp *goGitlab.Response 77 | err error 78 | ) 79 | 80 | c.rateLimit(ctx) 81 | 82 | switch w.Owner.Kind { 83 | case "user": 84 | gps, resp, err = c.Projects.ListUserProjects( 85 | w.Owner.Name, 86 | &goGitlab.ListProjectsOptions{ 87 | Archived: &w.Archived, 88 | ListOptions: listOptions, 89 | Search: &w.Search, 90 | Simple: pointy.Bool(true), 91 | }, 92 | goGitlab.WithContext(ctx), 93 | ) 94 | case "group": 95 | gps, resp, err = c.Groups.ListGroupProjects( 96 | w.Owner.Name, 97 | &goGitlab.ListGroupProjectsOptions{ 98 | Archived: &w.Archived, 99 | WithShared: pointy.Bool(false), 100 | IncludeSubGroups: &w.Owner.IncludeSubgroups, 101 | ListOptions: listOptions, 102 | Search: &w.Search, 103 | Simple: pointy.Bool(true), 104 | }, 105 | goGitlab.WithContext(ctx), 106 | ) 107 | default: 108 | // List all visible projects 109 | gps, resp, err = c.Projects.ListProjects( 110 | &goGitlab.ListProjectsOptions{ 111 | ListOptions: listOptions, 112 | Archived: &w.Archived, 113 | Search: &w.Search, 114 | Simple: pointy.Bool(true), 115 | }, 116 | goGitlab.WithContext(ctx), 117 | ) 118 | } 119 | 120 | if err != nil { 121 | return projects, fmt.Errorf("unable to list projects with search pattern '%s' from the GitLab API : %v", w.Search, err.Error()) 122 | } 123 | 124 | c.requestsRemaining(resp) 125 | 126 | // Copy relevant settings from wildcard into created project 127 | for _, gp := range gps { 128 | if !ownerRegexp.MatchString(gp.PathWithNamespace) { 129 | log.WithFields(logFields).WithFields(log.Fields{ 130 | "project-id": gp.ID, 131 | "project-name": gp.PathWithNamespace, 132 | }).Debug("project path not matching owner's name, skipping") 133 | 134 | continue 135 | } 136 | 137 | p := schemas.NewProject(gp.PathWithNamespace) 138 | p.ProjectParameters = w.ProjectParameters 139 | projects = append(projects, p) 140 | } 141 | 142 | if resp.CurrentPage >= resp.NextPage { 143 | break 144 | } 145 | 146 | listOptions.Page = resp.NextPage 147 | } 148 | 149 | return projects, nil 150 | } 151 | -------------------------------------------------------------------------------- /pkg/gitlab/projects_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 12 | ) 13 | 14 | func TestGetProject(t *testing.T) { 15 | ctx, mux, server, c := getMockedClient() 16 | defer server.Close() 17 | 18 | mux.HandleFunc("/api/v4/projects/foo%2Fbar", 19 | func(w http.ResponseWriter, r *http.Request) { 20 | assert.Equal(t, r.Method, "GET") 21 | _, _ = fmt.Fprint(w, `{"id":1}`) 22 | }) 23 | 24 | p, err := c.GetProject(ctx, "foo/bar") 25 | assert.NoError(t, err) 26 | require.NotNil(t, p) 27 | assert.Equal(t, 1, p.ID) 28 | } 29 | 30 | func TestListUserProjects(t *testing.T) { 31 | ctx, mux, server, c := getMockedClient() 32 | defer server.Close() 33 | 34 | w := config.Wildcard{ 35 | Search: "bar", 36 | Owner: config.WildcardOwner{ 37 | Name: "foo", 38 | Kind: "user", 39 | IncludeSubgroups: false, 40 | }, 41 | Archived: false, 42 | } 43 | 44 | mux.HandleFunc(fmt.Sprintf("/api/v4/users/%s/projects", w.Owner.Name), 45 | func(w http.ResponseWriter, r *http.Request) { 46 | assert.Equal(t, r.Method, "GET") 47 | _, _ = fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar"},{"id":2,"path_with_namespace":"bar/baz"}]`) 48 | }) 49 | 50 | projects, err := c.ListProjects(ctx, w) 51 | assert.NoError(t, err) 52 | assert.Len(t, projects, 1) 53 | assert.Equal(t, "foo/bar", projects[0].Name) 54 | } 55 | 56 | func TestListGroupProjects(t *testing.T) { 57 | ctx, mux, server, c := getMockedClient() 58 | defer server.Close() 59 | 60 | w := config.Wildcard{ 61 | Search: "bar", 62 | Owner: config.WildcardOwner{ 63 | Name: "foo", 64 | Kind: "group", 65 | IncludeSubgroups: false, 66 | }, 67 | Archived: false, 68 | } 69 | 70 | mux.HandleFunc(fmt.Sprintf("/api/v4/groups/%s/projects", w.Owner.Name), 71 | func(w http.ResponseWriter, r *http.Request) { 72 | assert.Equal(t, r.Method, "GET") 73 | _, _ = fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar"},{"id":2,"path_with_namespace":"bar/baz"}]`) 74 | }) 75 | 76 | projects, err := c.ListProjects(ctx, w) 77 | assert.NoError(t, err) 78 | assert.Len(t, projects, 1) 79 | assert.Equal(t, "foo/bar", projects[0].Name) 80 | } 81 | 82 | func TestListProjects(t *testing.T) { 83 | ctx, mux, server, c := getMockedClient() 84 | defer server.Close() 85 | 86 | w := config.Wildcard{ 87 | Search: "bar", 88 | Owner: config.WildcardOwner{ 89 | Name: "", 90 | Kind: "", 91 | IncludeSubgroups: false, 92 | }, 93 | Archived: false, 94 | } 95 | 96 | mux.HandleFunc("/api/v4/projects", 97 | func(w http.ResponseWriter, r *http.Request) { 98 | assert.Equal(t, r.Method, "GET") 99 | _, _ = fmt.Fprint(w, `[{"id":2,"path_with_namespace":"bar"}]`) 100 | }) 101 | 102 | projects, err := c.ListProjects(ctx, w) 103 | assert.NoError(t, err) 104 | assert.Len(t, projects, 1) 105 | assert.Equal(t, "bar", projects[0].Name) 106 | } 107 | 108 | func TestListProjectsAPIError(t *testing.T) { 109 | ctx, mux, server, c := getMockedClient() 110 | defer server.Close() 111 | 112 | w := config.Wildcard{ 113 | Search: "bar", 114 | Owner: config.WildcardOwner{ 115 | Name: "foo", 116 | Kind: "user", 117 | }, 118 | Archived: false, 119 | } 120 | 121 | mux.HandleFunc(fmt.Sprintf("/api/v4/users/%s/projects", w.Owner.Name), 122 | func(w http.ResponseWriter, r *http.Request) { 123 | w.WriteHeader(http.StatusInternalServerError) 124 | _, _ = w.Write([]byte("500 - Something bad happened!")) 125 | }) 126 | 127 | _, err := c.ListProjects(ctx, w) 128 | assert.Error(t, err) 129 | assert.Contains(t, err.Error(), "unable to list projects with search pattern") 130 | } 131 | -------------------------------------------------------------------------------- /pkg/gitlab/repositories.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | log "github.com/sirupsen/logrus" 8 | goGitlab "github.com/xanzy/go-gitlab" 9 | "go.openly.dev/pointy" 10 | "go.opentelemetry.io/otel" 11 | "go.opentelemetry.io/otel/attribute" 12 | ) 13 | 14 | // GetCommitCountBetweenRefs .. 15 | func (c *Client) GetCommitCountBetweenRefs(ctx context.Context, project, from, to string) (int, error) { 16 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetCommitCountBetweenRefs") 17 | defer span.End() 18 | span.SetAttributes(attribute.String("project_name", project)) 19 | span.SetAttributes(attribute.String("from_ref", from)) 20 | span.SetAttributes(attribute.String("to_ref", to)) 21 | 22 | log.WithFields(log.Fields{ 23 | "project-name": project, 24 | "from-ref": from, 25 | "to-ref": to, 26 | }).Debug("comparing refs") 27 | 28 | c.rateLimit(ctx) 29 | 30 | cmp, resp, err := c.Repositories.Compare(project, &goGitlab.CompareOptions{ 31 | From: &from, 32 | To: &to, 33 | Straight: pointy.Bool(true), 34 | }, goGitlab.WithContext(ctx)) 35 | if err != nil { 36 | return 0, err 37 | } 38 | 39 | c.requestsRemaining(resp) 40 | 41 | if cmp == nil { 42 | return 0, fmt.Errorf("could not compare refs successfully") 43 | } 44 | 45 | return len(cmp.Commits), nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/gitlab/repositories_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestGetCommitCountBetweenRefs(t *testing.T) { 12 | ctx, mux, server, c := getMockedClient() 13 | defer server.Close() 14 | 15 | mux.HandleFunc("/api/v4/projects/foo/repository/compare", 16 | func(w http.ResponseWriter, r *http.Request) { 17 | assert.Equal(t, "GET", r.Method) 18 | fmt.Fprint(w, `{"commits":[{},{},{}]}`) 19 | }) 20 | 21 | mux.HandleFunc("/api/v4/projects/bar/repository/compare", 22 | func(w http.ResponseWriter, r *http.Request) { 23 | fmt.Fprint(w, `{`) 24 | }) 25 | 26 | commitCount, err := c.GetCommitCountBetweenRefs(ctx, "foo", "bar", "baz") 27 | assert.NoError(t, err) 28 | assert.Equal(t, 3, commitCount) 29 | 30 | commitCount, err = c.GetCommitCountBetweenRefs(ctx, "bar", "", "") 31 | assert.Error(t, err) 32 | assert.Equal(t, 0, commitCount) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/gitlab/tags.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | 7 | goGitlab "github.com/xanzy/go-gitlab" 8 | "go.opentelemetry.io/otel" 9 | "go.opentelemetry.io/otel/attribute" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | // GetProjectTags .. 15 | func (c *Client) GetProjectTags(ctx context.Context, p schemas.Project) ( 16 | refs schemas.Refs, 17 | err error, 18 | ) { 19 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectTags") 20 | defer span.End() 21 | span.SetAttributes(attribute.String("project_name", p.Name)) 22 | 23 | refs = make(schemas.Refs) 24 | 25 | options := &goGitlab.ListTagsOptions{ 26 | ListOptions: goGitlab.ListOptions{ 27 | Page: 1, 28 | PerPage: 100, 29 | }, 30 | } 31 | 32 | var re *regexp.Regexp 33 | 34 | if re, err = regexp.Compile(p.Pull.Refs.Tags.Regexp); err != nil { 35 | return 36 | } 37 | 38 | for { 39 | c.rateLimit(ctx) 40 | 41 | var ( 42 | tags []*goGitlab.Tag 43 | resp *goGitlab.Response 44 | ) 45 | 46 | tags, resp, err = c.Tags.ListTags(p.Name, options, goGitlab.WithContext(ctx)) 47 | if err != nil { 48 | return 49 | } 50 | 51 | c.requestsRemaining(resp) 52 | 53 | for _, tag := range tags { 54 | if re.MatchString(tag.Name) { 55 | ref := schemas.NewRef(p, schemas.RefKindTag, tag.Name) 56 | refs[ref.Key()] = ref 57 | } 58 | } 59 | 60 | if resp.CurrentPage >= resp.NextPage { 61 | break 62 | } 63 | 64 | options.Page = resp.NextPage 65 | } 66 | 67 | return 68 | } 69 | 70 | // GetProjectMostRecentTagCommit .. 71 | func (c *Client) GetProjectMostRecentTagCommit(ctx context.Context, projectName, filterRegexp string) (string, float64, error) { 72 | ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectTags") 73 | defer span.End() 74 | span.SetAttributes(attribute.String("project_name", projectName)) 75 | span.SetAttributes(attribute.String("regexp", filterRegexp)) 76 | 77 | options := &goGitlab.ListTagsOptions{ 78 | ListOptions: goGitlab.ListOptions{ 79 | Page: 1, 80 | PerPage: 100, 81 | }, 82 | } 83 | 84 | re, err := regexp.Compile(filterRegexp) 85 | if err != nil { 86 | return "", 0, err 87 | } 88 | 89 | for { 90 | c.rateLimit(ctx) 91 | 92 | tags, resp, err := c.Tags.ListTags(projectName, options, goGitlab.WithContext(ctx)) 93 | if err != nil { 94 | return "", 0, err 95 | } 96 | 97 | c.requestsRemaining(resp) 98 | 99 | for _, tag := range tags { 100 | if re.MatchString(tag.Name) { 101 | return tag.Commit.ShortID, float64(tag.Commit.CommittedDate.Unix()), nil 102 | } 103 | } 104 | 105 | if resp.CurrentPage >= resp.NextPage { 106 | break 107 | } 108 | 109 | options.Page = resp.NextPage 110 | } 111 | 112 | return "", 0, nil 113 | } 114 | -------------------------------------------------------------------------------- /pkg/gitlab/tags_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | func TestGetProjectTags(t *testing.T) { 15 | ctx, mux, server, c := getMockedClient() 16 | defer server.Close() 17 | 18 | mux.HandleFunc("/api/v4/projects/foo/repository/tags", 19 | func(w http.ResponseWriter, r *http.Request) { 20 | assert.Equal(t, "GET", r.Method) 21 | expectedQueryParams := url.Values{ 22 | "page": []string{"1"}, 23 | "per_page": []string{"100"}, 24 | } 25 | assert.Equal(t, expectedQueryParams, r.URL.Query()) 26 | fmt.Fprint(w, `[{"name":"foo"},{"name":"bar"}]`) 27 | }) 28 | 29 | p := schemas.NewProject("foo") 30 | p.Pull.Refs.Tags.Regexp = `^f` 31 | 32 | expectedRef := schemas.NewRef(p, schemas.RefKindTag, "foo") 33 | refs, err := c.GetProjectTags(ctx, p) 34 | assert.NoError(t, err) 35 | assert.Len(t, refs, 1) 36 | assert.Equal(t, schemas.Refs{ 37 | expectedRef.Key(): expectedRef, 38 | }, refs) 39 | 40 | // Test invalid project name 41 | p.Name = "invalid" 42 | _, err = c.GetProjectTags(ctx, p) 43 | assert.Error(t, err) 44 | 45 | // Test invalid regexp 46 | p.Name = "foo" 47 | p.Pull.Refs.Tags.Regexp = `[` 48 | _, err = c.GetProjectTags(ctx, p) 49 | assert.Error(t, err) 50 | } 51 | 52 | func TestGetProjectMostRecentTagCommit(t *testing.T) { 53 | ctx, mux, server, c := getMockedClient() 54 | defer server.Close() 55 | 56 | mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/tags"), 57 | func(w http.ResponseWriter, r *http.Request) { 58 | assert.Equal(t, "GET", r.Method) 59 | expectedQueryParams := url.Values{ 60 | "page": []string{"1"}, 61 | "per_page": []string{"100"}, 62 | } 63 | assert.Equal(t, expectedQueryParams, r.URL.Query()) 64 | fmt.Fprint(w, ` 65 | [ 66 | { 67 | "name": "foo", 68 | "commit": { 69 | "short_id": "7b5c3cc", 70 | "committed_date": "2019-03-25T18:55:13.252Z" 71 | } 72 | }, 73 | { 74 | "name": "bar" 75 | } 76 | ]`) 77 | }) 78 | 79 | _, _, err := c.GetProjectMostRecentTagCommit(ctx, "foo", "[") 80 | assert.Error(t, err) 81 | assert.Contains(t, err.Error(), "error parsing regexp") 82 | 83 | commitShortID, commitCreatedAt, err := c.GetProjectMostRecentTagCommit(ctx, "foo", "^f") 84 | assert.NoError(t, err) 85 | assert.Equal(t, "7b5c3cc", commitShortID) 86 | assert.Equal(t, float64(1553540113), commitCreatedAt) 87 | } 88 | -------------------------------------------------------------------------------- /pkg/gitlab/version.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "strings" 5 | 6 | "golang.org/x/mod/semver" 7 | ) 8 | 9 | type GitLabVersion struct { 10 | Version string 11 | } 12 | 13 | func NewGitLabVersion(version string) GitLabVersion { 14 | ver := "" 15 | if strings.HasPrefix(version, "v") { 16 | ver = version 17 | } else if version != "" { 18 | ver = "v" + version 19 | } 20 | 21 | return GitLabVersion{Version: ver} 22 | } 23 | 24 | // PipelineJobsKeysetPaginationSupported returns true if the GitLab instance 25 | // is running 15.9 or later. 26 | func (v GitLabVersion) PipelineJobsKeysetPaginationSupported() bool { 27 | if v.Version == "" { 28 | return false 29 | } 30 | 31 | return semver.Compare(v.Version, "v15.9.0") >= 0 32 | } 33 | -------------------------------------------------------------------------------- /pkg/gitlab/version_test.go: -------------------------------------------------------------------------------- 1 | package gitlab 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestPipelineJobsKeysetPaginationSupported(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | version GitLabVersion 13 | expectedResult bool 14 | }{ 15 | { 16 | name: "unknown", 17 | version: NewGitLabVersion(""), 18 | expectedResult: false, 19 | }, 20 | { 21 | name: "15.8.0", 22 | version: NewGitLabVersion("15.8.0"), 23 | expectedResult: false, 24 | }, 25 | { 26 | name: "v15.8.0", 27 | version: NewGitLabVersion("v15.8.0"), 28 | expectedResult: false, 29 | }, 30 | { 31 | name: "15.9.0", 32 | version: NewGitLabVersion("15.9.0"), 33 | expectedResult: true, 34 | }, 35 | { 36 | name: "v15.9.0", 37 | version: NewGitLabVersion("v15.9.0"), 38 | expectedResult: true, 39 | }, 40 | { 41 | name: "15.9.1", 42 | version: NewGitLabVersion("15.9.1"), 43 | expectedResult: true, 44 | }, 45 | { 46 | name: "15.10.2", 47 | version: NewGitLabVersion("15.10.2"), 48 | expectedResult: true, 49 | }, 50 | { 51 | name: "16.0.0", 52 | version: NewGitLabVersion("16.0.0"), 53 | expectedResult: true, 54 | }, 55 | } 56 | 57 | for _, tc := range tests { 58 | t.Run(tc.name, func(t *testing.T) { 59 | result := tc.version.PipelineJobsKeysetPaginationSupported() 60 | 61 | assert.Equal(t, tc.expectedResult, result) 62 | }) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /pkg/monitor/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "google.golang.org/grpc" 9 | "google.golang.org/grpc/credentials/insecure" 10 | 11 | pb "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor/protobuf" 12 | ) 13 | 14 | // Client .. 15 | type Client struct { 16 | pb.MonitorClient 17 | } 18 | 19 | // NewClient .. 20 | func NewClient(ctx context.Context, endpoint *url.URL) *Client { 21 | log.WithField("endpoint", endpoint.String()).Debug("establishing gRPC connection to the server..") 22 | 23 | conn, err := grpc.DialContext( 24 | ctx, 25 | endpoint.String(), 26 | grpc.WithTransportCredentials(insecure.NewCredentials()), 27 | ) 28 | if err != nil { 29 | log.WithField("endpoint", endpoint.String()).WithField("error", err).Fatal("could not connect to the server") 30 | } 31 | 32 | log.Debug("gRPC connection established") 33 | 34 | return &Client{ 35 | MonitorClient: pb.NewMonitorClient(conn), 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/monitor/monitor.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import "time" 4 | 5 | type TaskSchedulingStatus struct { 6 | Last time.Time 7 | Next time.Time 8 | } 9 | -------------------------------------------------------------------------------- /pkg/monitor/protobuf/monitor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/timestamp.proto"; 4 | 5 | option go_package = "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor/protobuf"; 6 | 7 | package monitor; 8 | 9 | service Monitor { 10 | rpc GetConfig(Empty) returns (Config) {} 11 | rpc GetTelemetry(Empty) returns (stream Telemetry) {} 12 | } 13 | 14 | message Empty {} 15 | 16 | message Config { 17 | string content = 1; 18 | } 19 | 20 | message Telemetry { 21 | double gitlab_api_usage = 1; 22 | uint64 gitlab_api_requests_count = 2; 23 | double gitlab_api_rate_limit = 3; 24 | uint64 gitlab_api_limit_remaining = 4; 25 | double tasks_buffer_usage = 5; 26 | uint64 tasks_executed_count = 6; 27 | Entity projects = 7; 28 | Entity refs = 8; 29 | Entity envs = 9; 30 | Entity metrics = 10; 31 | } 32 | 33 | message Entity { 34 | int64 count = 1; 35 | google.protobuf.Timestamp last_gc = 2; 36 | google.protobuf.Timestamp last_pull = 3; 37 | google.protobuf.Timestamp next_gc = 4; 38 | google.protobuf.Timestamp next_pull = 5; 39 | } 40 | -------------------------------------------------------------------------------- /pkg/monitor/protobuf/monitor_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v3.21.0 5 | // source: pkg/monitor/protobuf/monitor.proto 6 | 7 | package protobuf 8 | 9 | import ( 10 | context "context" 11 | 12 | grpc "google.golang.org/grpc" 13 | codes "google.golang.org/grpc/codes" 14 | status "google.golang.org/grpc/status" 15 | ) 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the grpc package it is being compiled against. 19 | // Requires gRPC-Go v1.32.0 or later. 20 | const _ = grpc.SupportPackageIsVersion7 21 | 22 | // MonitorClient is the client API for Monitor service. 23 | // 24 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 25 | type MonitorClient interface { 26 | GetConfig(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Config, error) 27 | GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Monitor_GetTelemetryClient, error) 28 | } 29 | 30 | type monitorClient struct { 31 | cc grpc.ClientConnInterface 32 | } 33 | 34 | func NewMonitorClient(cc grpc.ClientConnInterface) MonitorClient { 35 | return &monitorClient{cc} 36 | } 37 | 38 | func (c *monitorClient) GetConfig(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Config, error) { 39 | out := new(Config) 40 | err := c.cc.Invoke(ctx, "/monitor.Monitor/GetConfig", in, out, opts...) 41 | if err != nil { 42 | return nil, err 43 | } 44 | return out, nil 45 | } 46 | 47 | func (c *monitorClient) GetTelemetry(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Monitor_GetTelemetryClient, error) { 48 | stream, err := c.cc.NewStream(ctx, &Monitor_ServiceDesc.Streams[0], "/monitor.Monitor/GetTelemetry", opts...) 49 | if err != nil { 50 | return nil, err 51 | } 52 | x := &monitorGetTelemetryClient{stream} 53 | if err := x.ClientStream.SendMsg(in); err != nil { 54 | return nil, err 55 | } 56 | if err := x.ClientStream.CloseSend(); err != nil { 57 | return nil, err 58 | } 59 | return x, nil 60 | } 61 | 62 | type Monitor_GetTelemetryClient interface { 63 | Recv() (*Telemetry, error) 64 | grpc.ClientStream 65 | } 66 | 67 | type monitorGetTelemetryClient struct { 68 | grpc.ClientStream 69 | } 70 | 71 | func (x *monitorGetTelemetryClient) Recv() (*Telemetry, error) { 72 | m := new(Telemetry) 73 | if err := x.ClientStream.RecvMsg(m); err != nil { 74 | return nil, err 75 | } 76 | return m, nil 77 | } 78 | 79 | // MonitorServer is the server API for Monitor service. 80 | // All implementations must embed UnimplementedMonitorServer 81 | // for forward compatibility 82 | type MonitorServer interface { 83 | GetConfig(context.Context, *Empty) (*Config, error) 84 | GetTelemetry(*Empty, Monitor_GetTelemetryServer) error 85 | mustEmbedUnimplementedMonitorServer() 86 | } 87 | 88 | // UnimplementedMonitorServer must be embedded to have forward compatible implementations. 89 | type UnimplementedMonitorServer struct{} 90 | 91 | func (UnimplementedMonitorServer) GetConfig(context.Context, *Empty) (*Config, error) { 92 | return nil, status.Errorf(codes.Unimplemented, "method GetConfig not implemented") 93 | } 94 | 95 | func (UnimplementedMonitorServer) GetTelemetry(*Empty, Monitor_GetTelemetryServer) error { 96 | return status.Errorf(codes.Unimplemented, "method GetTelemetry not implemented") 97 | } 98 | func (UnimplementedMonitorServer) mustEmbedUnimplementedMonitorServer() {} 99 | 100 | // UnsafeMonitorServer may be embedded to opt out of forward compatibility for this service. 101 | // Use of this interface is not recommended, as added methods to MonitorServer will 102 | // result in compilation errors. 103 | type UnsafeMonitorServer interface { 104 | mustEmbedUnimplementedMonitorServer() 105 | } 106 | 107 | func RegisterMonitorServer(s grpc.ServiceRegistrar, srv MonitorServer) { 108 | s.RegisterService(&Monitor_ServiceDesc, srv) 109 | } 110 | 111 | func _Monitor_GetConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 112 | in := new(Empty) 113 | if err := dec(in); err != nil { 114 | return nil, err 115 | } 116 | if interceptor == nil { 117 | return srv.(MonitorServer).GetConfig(ctx, in) 118 | } 119 | info := &grpc.UnaryServerInfo{ 120 | Server: srv, 121 | FullMethod: "/monitor.Monitor/GetConfig", 122 | } 123 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 124 | return srv.(MonitorServer).GetConfig(ctx, req.(*Empty)) 125 | } 126 | return interceptor(ctx, in, info, handler) 127 | } 128 | 129 | func _Monitor_GetTelemetry_Handler(srv interface{}, stream grpc.ServerStream) error { 130 | m := new(Empty) 131 | if err := stream.RecvMsg(m); err != nil { 132 | return err 133 | } 134 | return srv.(MonitorServer).GetTelemetry(m, &monitorGetTelemetryServer{stream}) 135 | } 136 | 137 | type Monitor_GetTelemetryServer interface { 138 | Send(*Telemetry) error 139 | grpc.ServerStream 140 | } 141 | 142 | type monitorGetTelemetryServer struct { 143 | grpc.ServerStream 144 | } 145 | 146 | func (x *monitorGetTelemetryServer) Send(m *Telemetry) error { 147 | return x.ServerStream.SendMsg(m) 148 | } 149 | 150 | // Monitor_ServiceDesc is the grpc.ServiceDesc for Monitor service. 151 | // It's only intended for direct use with grpc.RegisterService, 152 | // and not to be introspected or modified (even as a copy) 153 | var Monitor_ServiceDesc = grpc.ServiceDesc{ 154 | ServiceName: "monitor.Monitor", 155 | HandlerType: (*MonitorServer)(nil), 156 | Methods: []grpc.MethodDesc{ 157 | { 158 | MethodName: "GetConfig", 159 | Handler: _Monitor_GetConfig_Handler, 160 | }, 161 | }, 162 | Streams: []grpc.StreamDesc{ 163 | { 164 | StreamName: "GetTelemetry", 165 | Handler: _Monitor_GetTelemetry_Handler, 166 | ServerStreams: true, 167 | }, 168 | }, 169 | Metadata: "pkg/monitor/protobuf/monitor.proto", 170 | } 171 | -------------------------------------------------------------------------------- /pkg/ratelimit/local.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "golang.org/x/time/rate" 9 | ) 10 | 11 | // Local .. 12 | type Local struct { 13 | *rate.Limiter 14 | } 15 | 16 | // NewLocalLimiter .. 17 | func NewLocalLimiter(maximumRPS, burstableRPS int) Limiter { 18 | return Local{ 19 | rate.NewLimiter(rate.Limit(maximumRPS), burstableRPS), 20 | } 21 | } 22 | 23 | // Take .. 24 | func (l Local) Take(ctx context.Context) time.Duration { 25 | start := time.Now() 26 | 27 | if err := l.Limiter.Wait(ctx); err != nil { 28 | log.WithContext(ctx). 29 | WithError(err). 30 | Fatal() 31 | } 32 | 33 | return start.Sub(time.Now()) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/ratelimit/local_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewLocalLimiter(t *testing.T) { 10 | assert.IsType(t, Local{}, NewLocalLimiter(10, 1)) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/ratelimit/ratelimit.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // Limiter .. 9 | type Limiter interface { 10 | Take(ctx context.Context) time.Duration 11 | } 12 | 13 | // Take .. 14 | func Take(ctx context.Context, l Limiter) { 15 | l.Take(ctx) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/ratelimit/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/exec" 7 | "testing" 8 | "time" 9 | 10 | "github.com/alicebob/miniredis/v2" 11 | "github.com/redis/go-redis/v9" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func MeasureTakeDuration(l Limiter) int64 { 16 | start := time.Now() 17 | 18 | Take(context.TODO(), l) 19 | 20 | return int64(time.Since(start)) 21 | } 22 | 23 | func TestLocalTake(t *testing.T) { 24 | l := NewLocalLimiter(1, 1) 25 | 26 | assert.LessOrEqual(t, MeasureTakeDuration(l), int64(100*time.Millisecond)) 27 | assert.GreaterOrEqual(t, MeasureTakeDuration(l), int64(time.Second)) 28 | } 29 | 30 | func TestRedisTake(t *testing.T) { 31 | s, err := miniredis.Run() 32 | if err != nil { 33 | panic(err) 34 | } 35 | 36 | defer s.Close() 37 | 38 | l := NewRedisLimiter( 39 | redis.NewClient(&redis.Options{Addr: s.Addr()}), 40 | 1, 41 | ) 42 | 43 | assert.LessOrEqual(t, MeasureTakeDuration(l), int64(250*time.Millisecond)) 44 | assert.GreaterOrEqual(t, MeasureTakeDuration(l), int64(900*time.Millisecond)) 45 | } 46 | 47 | func TestRedisTakeError(t *testing.T) { 48 | if os.Getenv("SHOULD_ERROR") == "1" { 49 | l := NewRedisLimiter( 50 | redis.NewClient(&redis.Options{Addr: "doesnotexist"}), 51 | 1, 52 | ) 53 | 54 | Take(context.TODO(), l) 55 | 56 | return 57 | } 58 | 59 | cmd := exec.Command(os.Args[0], "-test.run=TestRedisTakeError") 60 | cmd.Env = append(os.Environ(), "SHOULD_ERROR=1") 61 | 62 | err := cmd.Run() 63 | if e, ok := err.(*exec.ExitError); ok && !e.Success() { 64 | return 65 | } 66 | 67 | t.Fatal("process ran successfully, wanted exit status 1") 68 | } 69 | -------------------------------------------------------------------------------- /pkg/ratelimit/redis.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/go-redis/redis_rate/v10" 8 | "github.com/redis/go-redis/v9" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | const redisKey string = `gcpe:gitlab:api` 13 | 14 | // Redis .. 15 | type Redis struct { 16 | *redis_rate.Limiter 17 | MaxRPS int 18 | } 19 | 20 | // NewRedisLimiter .. 21 | func NewRedisLimiter(redisClient *redis.Client, maxRPS int) Limiter { 22 | return Redis{ 23 | Limiter: redis_rate.NewLimiter(redisClient), 24 | MaxRPS: maxRPS, 25 | } 26 | } 27 | 28 | // Take .. 29 | func (r Redis) Take(ctx context.Context) time.Duration { 30 | start := time.Now() 31 | 32 | for { 33 | res, err := r.Allow(ctx, redisKey, redis_rate.PerSecond(r.MaxRPS)) 34 | if err != nil { 35 | log.WithContext(ctx). 36 | WithError(err). 37 | Fatal() 38 | } 39 | 40 | if res.Allowed > 0 { 41 | break 42 | } else { 43 | log.WithFields( 44 | log.Fields{ 45 | "for": res.RetryAfter.String(), 46 | }, 47 | ).Debug("throttled GitLab requests") 48 | time.Sleep(res.RetryAfter) 49 | } 50 | } 51 | 52 | return start.Sub(time.Now()) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/ratelimit/redis_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-redis/redis_rate/v10" 7 | "github.com/redis/go-redis/v9" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewRedisLimiter(t *testing.T) { 12 | redisClient := redis.NewClient(&redis.Options{}) 13 | l := NewRedisLimiter( 14 | redisClient, 15 | 10, 16 | ) 17 | 18 | expectedValue := Redis{ 19 | Limiter: redis_rate.NewLimiter(redisClient), 20 | MaxRPS: 10, 21 | } 22 | 23 | assert.Equal(t, expectedValue, l) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/schemas/deployments.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | // Deployment .. 4 | type Deployment struct { 5 | JobID int 6 | RefKind RefKind 7 | RefName string 8 | Username string 9 | Timestamp float64 10 | DurationSeconds float64 11 | CommitShortID string 12 | Status string 13 | } 14 | -------------------------------------------------------------------------------- /pkg/schemas/deployments_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | -------------------------------------------------------------------------------- /pkg/schemas/environments.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "hash/crc32" 5 | "strconv" 6 | ) 7 | 8 | // Environment .. 9 | type Environment struct { 10 | ProjectName string 11 | ID int 12 | Name string 13 | ExternalURL string 14 | Available bool 15 | LatestDeployment Deployment 16 | 17 | OutputSparseStatusMetrics bool 18 | } 19 | 20 | // EnvironmentKey .. 21 | type EnvironmentKey string 22 | 23 | // Key .. 24 | func (e Environment) Key() EnvironmentKey { 25 | return EnvironmentKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(e.ProjectName + e.Name))))) 26 | } 27 | 28 | // Environments allows us to keep track of all the Environment objects we have discovered. 29 | type Environments map[EnvironmentKey]Environment 30 | 31 | // Count returns the amount of environments in the map. 32 | func (envs Environments) Count() int { 33 | return len(envs) 34 | } 35 | 36 | // DefaultLabelsValues .. 37 | func (e Environment) DefaultLabelsValues() map[string]string { 38 | return map[string]string{ 39 | "project": e.ProjectName, 40 | "environment": e.Name, 41 | } 42 | } 43 | 44 | // InformationLabelsValues .. 45 | func (e Environment) InformationLabelsValues() (v map[string]string) { 46 | v = e.DefaultLabelsValues() 47 | v["environment_id"] = strconv.Itoa(e.ID) 48 | v["external_url"] = e.ExternalURL 49 | v["kind"] = string(e.LatestDeployment.RefKind) 50 | v["ref"] = e.LatestDeployment.RefName 51 | v["current_commit_short_id"] = e.LatestDeployment.CommitShortID 52 | v["latest_commit_short_id"] = "" 53 | v["available"] = strconv.FormatBool(e.Available) 54 | v["username"] = e.LatestDeployment.Username 55 | 56 | return 57 | } 58 | -------------------------------------------------------------------------------- /pkg/schemas/environments_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEnvironmentKey(t *testing.T) { 10 | e := Environment{ 11 | ProjectName: "foo", 12 | Name: "bar", 13 | } 14 | 15 | assert.Equal(t, EnvironmentKey("2666930069"), e.Key()) 16 | } 17 | 18 | func TestEnvironmentsCount(t *testing.T) { 19 | assert.Equal(t, 2, Environments{ 20 | EnvironmentKey("foo"): Environment{}, 21 | EnvironmentKey("bar"): Environment{}, 22 | }.Count()) 23 | } 24 | 25 | func TestEnvironmentDefaultLabelsValues(t *testing.T) { 26 | e := Environment{ 27 | ProjectName: "foo", 28 | Name: "bar", 29 | } 30 | 31 | expectedValue := map[string]string{ 32 | "project": "foo", 33 | "environment": "bar", 34 | } 35 | 36 | assert.Equal(t, expectedValue, e.DefaultLabelsValues()) 37 | } 38 | 39 | func TestEnvironmentInformationLabelsValues(t *testing.T) { 40 | e := Environment{ 41 | ProjectName: "foo", 42 | Name: "bar", 43 | ID: 10, 44 | ExternalURL: "http://genial", 45 | Available: true, 46 | LatestDeployment: Deployment{ 47 | RefKind: RefKindBranch, 48 | RefName: "foo", 49 | CommitShortID: "123abcde", 50 | Username: "bob", 51 | }, 52 | } 53 | 54 | expectedValue := map[string]string{ 55 | "project": "foo", 56 | "environment": "bar", 57 | "environment_id": "10", 58 | "external_url": "http://genial", 59 | "kind": "branch", 60 | "ref": "foo", 61 | "current_commit_short_id": "123abcde", 62 | "latest_commit_short_id": "", 63 | "available": "true", 64 | "username": "bob", 65 | } 66 | 67 | assert.Equal(t, expectedValue, e.InformationLabelsValues()) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/schemas/jobs.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "strings" 5 | 6 | goGitlab "github.com/xanzy/go-gitlab" 7 | ) 8 | 9 | // Job .. 10 | type Job struct { 11 | ID int 12 | Name string 13 | Stage string 14 | Timestamp float64 15 | DurationSeconds float64 16 | QueuedDurationSeconds float64 17 | Status string 18 | TagList string 19 | ArtifactSize float64 20 | FailureReason string 21 | Runner Runner 22 | } 23 | 24 | // Runner .. 25 | type Runner struct { 26 | Description string 27 | } 28 | 29 | // Jobs .. 30 | type Jobs map[string]Job 31 | 32 | // NewJob .. 33 | func NewJob(gj goGitlab.Job) Job { 34 | var ( 35 | artifactSize float64 36 | timestamp float64 37 | ) 38 | 39 | for _, artifact := range gj.Artifacts { 40 | artifactSize += float64(artifact.Size) 41 | } 42 | 43 | if gj.CreatedAt != nil { 44 | timestamp = float64(gj.CreatedAt.Unix()) 45 | } 46 | 47 | return Job{ 48 | ID: gj.ID, 49 | Name: gj.Name, 50 | Stage: gj.Stage, 51 | Timestamp: timestamp, 52 | DurationSeconds: gj.Duration, 53 | QueuedDurationSeconds: gj.QueuedDuration, 54 | Status: gj.Status, 55 | TagList: strings.Join(gj.TagList, ","), 56 | ArtifactSize: artifactSize, 57 | FailureReason: gj.FailureReason, 58 | 59 | Runner: Runner{ 60 | Description: gj.Runner.Description, 61 | }, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pkg/schemas/jobs_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | goGitlab "github.com/xanzy/go-gitlab" 9 | ) 10 | 11 | func TestNewJob(t *testing.T) { 12 | createdAt := time.Date(2020, 10, 1, 13, 5, 5, 0, time.UTC) 13 | startedAt := time.Date(2020, 10, 1, 13, 5, 35, 0, time.UTC) 14 | 15 | gitlabJob := goGitlab.Job{ 16 | ID: 2, 17 | Name: "foo", 18 | CreatedAt: &createdAt, 19 | StartedAt: &startedAt, 20 | Duration: 15, 21 | QueuedDuration: 10, 22 | Status: "failed", 23 | Stage: "🚀", 24 | TagList: []string{"test-tag"}, 25 | Runner: struct { 26 | ID int "json:\"id\"" 27 | Description string "json:\"description\"" 28 | Active bool "json:\"active\"" 29 | IsShared bool "json:\"is_shared\"" 30 | Name string "json:\"name\"" 31 | }{ 32 | Description: "xxx", 33 | }, 34 | Artifacts: []struct { 35 | FileType string "json:\"file_type\"" 36 | Filename string "json:\"filename\"" 37 | Size int "json:\"size\"" 38 | FileFormat string "json:\"file_format\"" 39 | }{ 40 | { 41 | Size: 100, 42 | }, 43 | { 44 | Size: 50, 45 | }, 46 | }, 47 | } 48 | 49 | expectedJob := Job{ 50 | ID: 2, 51 | Name: "foo", 52 | Stage: "🚀", 53 | Timestamp: 1.601557505e+09, 54 | DurationSeconds: 15, 55 | QueuedDurationSeconds: 10, 56 | Status: "failed", 57 | TagList: "test-tag", 58 | ArtifactSize: 150, 59 | 60 | Runner: Runner{ 61 | Description: "xxx", 62 | }, 63 | } 64 | 65 | assert.Equal(t, expectedJob, NewJob(gitlabJob)) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/schemas/metric.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | "strconv" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | const ( 12 | // MetricKindCoverage refers to the coerage of a job/pipeline. 13 | MetricKindCoverage MetricKind = iota 14 | 15 | // MetricKindDurationSeconds .. 16 | MetricKindDurationSeconds 17 | 18 | // MetricKindEnvironmentBehindCommitsCount .. 19 | MetricKindEnvironmentBehindCommitsCount 20 | 21 | // MetricKindEnvironmentBehindDurationSeconds .. 22 | MetricKindEnvironmentBehindDurationSeconds 23 | 24 | // MetricKindEnvironmentDeploymentCount .. 25 | MetricKindEnvironmentDeploymentCount 26 | 27 | // MetricKindEnvironmentDeploymentDurationSeconds .. 28 | MetricKindEnvironmentDeploymentDurationSeconds 29 | 30 | // MetricKindEnvironmentDeploymentJobID .. 31 | MetricKindEnvironmentDeploymentJobID 32 | 33 | // MetricKindEnvironmentDeploymentStatus .. 34 | MetricKindEnvironmentDeploymentStatus 35 | 36 | // MetricKindEnvironmentDeploymentTimestamp .. 37 | MetricKindEnvironmentDeploymentTimestamp 38 | 39 | // MetricKindEnvironmentInformation .. 40 | MetricKindEnvironmentInformation 41 | 42 | // MetricKindID .. 43 | MetricKindID 44 | 45 | // MetricKindJobArtifactSizeBytes .. 46 | MetricKindJobArtifactSizeBytes 47 | 48 | // MetricKindJobDurationSeconds .. 49 | MetricKindJobDurationSeconds 50 | 51 | // MetricKindJobID .. 52 | MetricKindJobID 53 | 54 | // MetricKindJobQueuedDurationSeconds .. 55 | MetricKindJobQueuedDurationSeconds 56 | 57 | // MetricKindJobRunCount .. 58 | MetricKindJobRunCount 59 | 60 | // MetricKindJobStatus .. 61 | MetricKindJobStatus 62 | 63 | // MetricKindJobTimestamp .. 64 | MetricKindJobTimestamp 65 | 66 | // MetricKindQueuedDurationSeconds .. 67 | MetricKindQueuedDurationSeconds 68 | 69 | // MetricKindRunCount .. 70 | MetricKindRunCount 71 | 72 | // MetricKindStatus .. 73 | MetricKindStatus 74 | 75 | // MetricKindTimestamp .. 76 | MetricKindTimestamp 77 | 78 | // MetricKindTestReportTotalTime .. 79 | MetricKindTestReportTotalTime 80 | 81 | // MetricKindTestReportTotalCount .. 82 | MetricKindTestReportTotalCount 83 | 84 | // MetricKindTestReportSuccessCount .. 85 | MetricKindTestReportSuccessCount 86 | 87 | // MetricKindTestReportFailedCount .. 88 | MetricKindTestReportFailedCount 89 | 90 | // MetricKindTestReportSkippedCount .. 91 | MetricKindTestReportSkippedCount 92 | 93 | // MetricKindTestReportErrorCount .. 94 | MetricKindTestReportErrorCount 95 | 96 | // MetricKindTestSuiteTotalTime .. 97 | MetricKindTestSuiteTotalTime 98 | 99 | // MetricKindTestSuiteTotalCount .. 100 | MetricKindTestSuiteTotalCount 101 | 102 | // MetricKindTestSuiteSuccessCount .. 103 | MetricKindTestSuiteSuccessCount 104 | 105 | // MetricKindTestSuiteFailedCount .. 106 | MetricKindTestSuiteFailedCount 107 | 108 | // MetricKindTestSuiteSkippedCount .. 109 | MetricKindTestSuiteSkippedCount 110 | 111 | // MetricKindTestSuiteErrorCount .. 112 | MetricKindTestSuiteErrorCount 113 | 114 | // MetricKindTestCaseExecutionTime .. 115 | MetricKindTestCaseExecutionTime 116 | 117 | // MetricKindTestCaseStatus .. 118 | MetricKindTestCaseStatus 119 | ) 120 | 121 | // MetricKind .. 122 | type MetricKind int32 123 | 124 | // Metric .. 125 | type Metric struct { 126 | Kind MetricKind 127 | Labels prometheus.Labels 128 | Value float64 129 | } 130 | 131 | // MetricKey .. 132 | type MetricKey string 133 | 134 | // Metrics .. 135 | type Metrics map[MetricKey]Metric 136 | 137 | // Key .. 138 | func (m Metric) Key() MetricKey { 139 | key := strconv.Itoa(int(m.Kind)) 140 | 141 | switch m.Kind { 142 | case MetricKindCoverage, MetricKindDurationSeconds, MetricKindID, MetricKindQueuedDurationSeconds, MetricKindRunCount, MetricKindStatus, MetricKindTimestamp, MetricKindTestReportTotalCount, MetricKindTestReportErrorCount, MetricKindTestReportFailedCount, MetricKindTestReportSkippedCount, MetricKindTestReportSuccessCount, MetricKindTestReportTotalTime: 143 | key += fmt.Sprintf("%v", []string{ 144 | m.Labels["project"], 145 | m.Labels["kind"], 146 | m.Labels["ref"], 147 | m.Labels["source"], 148 | }) 149 | 150 | case MetricKindJobArtifactSizeBytes, MetricKindJobDurationSeconds, MetricKindJobID, MetricKindJobQueuedDurationSeconds, MetricKindJobRunCount, MetricKindJobStatus, MetricKindJobTimestamp: 151 | key += fmt.Sprintf("%v", []string{ 152 | m.Labels["project"], 153 | m.Labels["kind"], 154 | m.Labels["ref"], 155 | m.Labels["stage"], 156 | m.Labels["tag_list"], 157 | m.Labels["job_name"], 158 | m.Labels["failure_reason"], 159 | }) 160 | 161 | case MetricKindEnvironmentBehindCommitsCount, MetricKindEnvironmentBehindDurationSeconds, MetricKindEnvironmentDeploymentCount, MetricKindEnvironmentDeploymentDurationSeconds, MetricKindEnvironmentDeploymentJobID, MetricKindEnvironmentDeploymentStatus, MetricKindEnvironmentDeploymentTimestamp, MetricKindEnvironmentInformation: 162 | key += fmt.Sprintf("%v", []string{ 163 | m.Labels["project"], 164 | m.Labels["environment"], 165 | }) 166 | 167 | case MetricKindTestSuiteErrorCount, MetricKindTestSuiteFailedCount, MetricKindTestSuiteSkippedCount, MetricKindTestSuiteSuccessCount, MetricKindTestSuiteTotalCount, MetricKindTestSuiteTotalTime: 168 | key += fmt.Sprintf("%v", []string{ 169 | m.Labels["project"], 170 | m.Labels["kind"], 171 | m.Labels["ref"], 172 | m.Labels["test_suite_name"], 173 | }) 174 | 175 | case MetricKindTestCaseExecutionTime, MetricKindTestCaseStatus: 176 | key += fmt.Sprintf("%v", []string{ 177 | m.Labels["project"], 178 | m.Labels["kind"], 179 | m.Labels["ref"], 180 | m.Labels["test_suite_name"], 181 | m.Labels["test_case_name"], 182 | m.Labels["test_case_classname"], 183 | }) 184 | } 185 | 186 | // If the metric is a "status" one, add the status label 187 | switch m.Kind { 188 | case MetricKindJobStatus, MetricKindEnvironmentDeploymentStatus, MetricKindStatus, MetricKindTestCaseStatus: 189 | key += m.Labels["status"] 190 | } 191 | 192 | return MetricKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(key))))) 193 | } 194 | -------------------------------------------------------------------------------- /pkg/schemas/metric_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestMetricKey(t *testing.T) { 11 | assert.Equal(t, MetricKey("3797596385"), Metric{ 12 | Kind: MetricKindCoverage, 13 | Labels: prometheus.Labels{ 14 | "foo": "bar", 15 | }, 16 | }.Key()) 17 | 18 | assert.Equal(t, MetricKey("77312310"), Metric{ 19 | Kind: MetricKindEnvironmentInformation, 20 | Labels: prometheus.Labels{ 21 | "project": "foo", 22 | "environment": "bar", 23 | "foo": "bar", 24 | }, 25 | }.Key()) 26 | 27 | assert.Equal(t, MetricKey("1288741005"), Metric{ 28 | Kind: MetricKindEnvironmentInformation, 29 | }.Key()) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/schemas/pipelines.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | 7 | log "github.com/sirupsen/logrus" 8 | goGitlab "github.com/xanzy/go-gitlab" 9 | ) 10 | 11 | // Pipeline .. 12 | type Pipeline struct { 13 | ID int 14 | Coverage float64 15 | Timestamp float64 16 | DurationSeconds float64 17 | QueuedDurationSeconds float64 18 | Source string 19 | Status string 20 | Variables string 21 | TestReport TestReport 22 | } 23 | 24 | // TestReport .. 25 | type TestReport struct { 26 | TotalTime float64 27 | TotalCount int 28 | SuccessCount int 29 | FailedCount int 30 | SkippedCount int 31 | ErrorCount int 32 | TestSuites []TestSuite 33 | } 34 | 35 | // TestSuite .. 36 | type TestSuite struct { 37 | Name string 38 | TotalTime float64 39 | TotalCount int 40 | SuccessCount int 41 | FailedCount int 42 | SkippedCount int 43 | ErrorCount int 44 | TestCases []TestCase 45 | } 46 | 47 | // TestCase .. 48 | type TestCase struct { 49 | Name string 50 | Classname string 51 | ExecutionTime float64 52 | Status string 53 | } 54 | 55 | // NewPipeline .. 56 | func NewPipeline(ctx context.Context, gp goGitlab.Pipeline) Pipeline { 57 | var ( 58 | coverage float64 59 | err error 60 | timestamp float64 61 | ) 62 | 63 | if gp.Coverage != "" { 64 | coverage, err = strconv.ParseFloat(gp.Coverage, 64) 65 | if err != nil { 66 | log.WithContext(ctx). 67 | WithField("error", err.Error()). 68 | Warnf("could not parse coverage string returned from GitLab API '%s' into Float64", gp.Coverage) 69 | } 70 | } 71 | 72 | if gp.UpdatedAt != nil { 73 | timestamp = float64(gp.UpdatedAt.Unix()) 74 | } 75 | 76 | pipeline := Pipeline{ 77 | ID: gp.ID, 78 | Coverage: coverage, 79 | Timestamp: timestamp, 80 | DurationSeconds: float64(gp.Duration), 81 | QueuedDurationSeconds: float64(gp.QueuedDuration), 82 | Source: gp.Source, 83 | } 84 | 85 | if gp.DetailedStatus != nil { 86 | pipeline.Status = gp.DetailedStatus.Group 87 | } else { 88 | pipeline.Status = gp.Status 89 | } 90 | 91 | return pipeline 92 | } 93 | 94 | // NewTestReport .. 95 | func NewTestReport(gtr goGitlab.PipelineTestReport) TestReport { 96 | testSuites := []TestSuite{} 97 | 98 | for _, x := range gtr.TestSuites { 99 | testSuites = append(testSuites, NewTestSuite(x)) 100 | } 101 | 102 | return TestReport{ 103 | TotalTime: gtr.TotalTime, 104 | TotalCount: gtr.TotalCount, 105 | SuccessCount: gtr.SuccessCount, 106 | FailedCount: gtr.FailedCount, 107 | SkippedCount: gtr.SkippedCount, 108 | ErrorCount: gtr.ErrorCount, 109 | TestSuites: testSuites, 110 | } 111 | } 112 | 113 | // NewTestSuite .. 114 | func NewTestSuite(gts *goGitlab.PipelineTestSuites) TestSuite { 115 | testCases := []TestCase{} 116 | 117 | for _, x := range gts.TestCases { 118 | testCases = append(testCases, NewTestCase(x)) 119 | } 120 | 121 | return TestSuite{ 122 | Name: gts.Name, 123 | TotalTime: gts.TotalTime, 124 | TotalCount: gts.TotalCount, 125 | SuccessCount: gts.SuccessCount, 126 | FailedCount: gts.FailedCount, 127 | SkippedCount: gts.SkippedCount, 128 | ErrorCount: gts.ErrorCount, 129 | TestCases: testCases, 130 | } 131 | } 132 | 133 | // NewTestCase .. 134 | func NewTestCase(gtc *goGitlab.PipelineTestCases) TestCase { 135 | return TestCase{ 136 | Name: gtc.Name, 137 | Classname: gtc.Classname, 138 | ExecutionTime: gtc.ExecutionTime, 139 | Status: gtc.Status, 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /pkg/schemas/pipelines_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | goGitlab "github.com/xanzy/go-gitlab" 10 | ) 11 | 12 | func TestNewPipeline(t *testing.T) { 13 | createdAt := time.Date(2020, 10, 1, 13, 4, 10, 0, time.UTC) 14 | startedAt := time.Date(2020, 10, 1, 13, 5, 10, 0, time.UTC) 15 | updatedAt := time.Date(2020, 10, 1, 13, 5, 50, 0, time.UTC) 16 | 17 | testCases := []struct { 18 | status string 19 | detailedStatus goGitlab.DetailedStatus 20 | expectedStatus string 21 | }{ 22 | { 23 | "running", 24 | goGitlab.DetailedStatus{ 25 | Text: "Running", 26 | Label: "running", 27 | Group: "running", 28 | }, 29 | "running", 30 | }, 31 | { 32 | "success", 33 | goGitlab.DetailedStatus{ 34 | Text: "Passed", 35 | Label: "passed", 36 | Group: "success", 37 | }, 38 | "success", 39 | }, 40 | { 41 | "canceled", 42 | goGitlab.DetailedStatus{ 43 | Text: "Canceled", 44 | Label: "canceled", 45 | Group: "canceled", 46 | }, 47 | "canceled", 48 | }, 49 | { 50 | "success", 51 | goGitlab.DetailedStatus{ 52 | Text: "Warning", 53 | Label: "passed with warnings", 54 | Group: "success-with-warnings", 55 | }, 56 | "success-with-warnings", 57 | }, 58 | } 59 | 60 | for _, tc := range testCases { 61 | t.Run(tc.status, func(t *testing.T) { 62 | gitlabPipeline := goGitlab.Pipeline{ 63 | ID: 21, 64 | Coverage: "25.6", 65 | CreatedAt: &createdAt, 66 | StartedAt: &startedAt, 67 | UpdatedAt: &updatedAt, 68 | Duration: 15, 69 | QueuedDuration: 5, 70 | Source: "schedule", 71 | Status: tc.status, 72 | DetailedStatus: &tc.detailedStatus, 73 | } 74 | 75 | expectedPipeline := Pipeline{ 76 | ID: 21, 77 | Coverage: 25.6, 78 | Timestamp: 1.60155755e+09, 79 | DurationSeconds: 15, 80 | QueuedDurationSeconds: 5, 81 | Source: "schedule", 82 | Status: tc.expectedStatus, 83 | } 84 | 85 | assert.Equal(t, expectedPipeline, NewPipeline(context.Background(), gitlabPipeline)) 86 | }) 87 | } 88 | } 89 | 90 | func TestNewTestReport(t *testing.T) { 91 | gitlabTestReport := goGitlab.PipelineTestReport{ 92 | TotalTime: 10, 93 | TotalCount: 2, 94 | SuccessCount: 1, 95 | FailedCount: 1, 96 | SkippedCount: 0, 97 | ErrorCount: 0, 98 | TestSuites: []*goGitlab.PipelineTestSuites{ 99 | { 100 | Name: "First", 101 | TotalTime: 3, 102 | TotalCount: 1, 103 | SuccessCount: 1, 104 | FailedCount: 0, 105 | SkippedCount: 0, 106 | ErrorCount: 0, 107 | TestCases: []*goGitlab.PipelineTestCases{ 108 | { 109 | Name: "First", 110 | Classname: "ClassFirst", 111 | ExecutionTime: 4, 112 | Status: "success", 113 | }, 114 | }, 115 | }, 116 | { 117 | Name: "Second", 118 | TotalTime: 2, 119 | TotalCount: 1, 120 | SuccessCount: 0, 121 | FailedCount: 1, 122 | SkippedCount: 0, 123 | ErrorCount: 0, 124 | TestCases: []*goGitlab.PipelineTestCases{ 125 | { 126 | Name: "First", 127 | Classname: "ClassFirst", 128 | ExecutionTime: 4, 129 | Status: "success", 130 | }, 131 | }, 132 | }, 133 | }, 134 | } 135 | 136 | expectedTestReport := TestReport{ 137 | TotalTime: 10, 138 | TotalCount: 2, 139 | SuccessCount: 1, 140 | FailedCount: 1, 141 | SkippedCount: 0, 142 | ErrorCount: 0, 143 | TestSuites: []TestSuite{ 144 | { 145 | Name: "First", 146 | TotalTime: 3, 147 | TotalCount: 1, 148 | SuccessCount: 1, 149 | FailedCount: 0, 150 | SkippedCount: 0, 151 | ErrorCount: 0, 152 | TestCases: []TestCase{ 153 | { 154 | Name: "First", 155 | Classname: "ClassFirst", 156 | ExecutionTime: 4, 157 | Status: "success", 158 | }, 159 | }, 160 | }, 161 | { 162 | Name: "Second", 163 | TotalTime: 2, 164 | TotalCount: 1, 165 | SuccessCount: 0, 166 | FailedCount: 1, 167 | SkippedCount: 0, 168 | ErrorCount: 0, 169 | TestCases: []TestCase{ 170 | { 171 | Name: "First", 172 | Classname: "ClassFirst", 173 | ExecutionTime: 4, 174 | Status: "success", 175 | }, 176 | }, 177 | }, 178 | }, 179 | } 180 | assert.Equal(t, expectedTestReport, NewTestReport(gitlabTestReport)) 181 | } 182 | 183 | func TestNewTestSuite(t *testing.T) { 184 | gitlabTestSuite := &goGitlab.PipelineTestSuites{ 185 | Name: "Suite", 186 | TotalTime: 4, 187 | TotalCount: 6, 188 | SuccessCount: 2, 189 | FailedCount: 2, 190 | SkippedCount: 1, 191 | ErrorCount: 1, 192 | TestCases: []*goGitlab.PipelineTestCases{ 193 | { 194 | Name: "First", 195 | Classname: "ClassFirst", 196 | ExecutionTime: 4, 197 | Status: "success", 198 | }, 199 | }, 200 | } 201 | 202 | expectedTestSuite := TestSuite{ 203 | Name: "Suite", 204 | TotalTime: 4, 205 | TotalCount: 6, 206 | SuccessCount: 2, 207 | FailedCount: 2, 208 | SkippedCount: 1, 209 | ErrorCount: 1, 210 | TestCases: []TestCase{ 211 | { 212 | Name: "First", 213 | Classname: "ClassFirst", 214 | ExecutionTime: 4, 215 | Status: "success", 216 | }, 217 | }, 218 | } 219 | assert.Equal(t, expectedTestSuite, NewTestSuite(gitlabTestSuite)) 220 | } 221 | -------------------------------------------------------------------------------- /pkg/schemas/projects.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "hash/crc32" 5 | "strconv" 6 | 7 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 8 | ) 9 | 10 | // Project .. 11 | type Project struct { 12 | config.Project 13 | 14 | Topics string 15 | } 16 | 17 | // ProjectKey .. 18 | type ProjectKey string 19 | 20 | // Projects .. 21 | type Projects map[ProjectKey]Project 22 | 23 | // Key .. 24 | func (p Project) Key() ProjectKey { 25 | return ProjectKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(p.Name))))) 26 | } 27 | 28 | // NewProject .. 29 | func NewProject(name string) Project { 30 | return Project{Project: config.NewProject(name)} 31 | } 32 | -------------------------------------------------------------------------------- /pkg/schemas/projects_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestProjectKey(t *testing.T) { 10 | assert.Equal(t, ProjectKey("2356372769"), NewProject("foo").Key()) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/schemas/ref.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | "regexp" 7 | "strconv" 8 | 9 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 10 | ) 11 | 12 | const ( 13 | mergeRequestRegexp string = `^((\d+)|refs/merge-requests/(\d+)/head)$` 14 | 15 | // RefKindBranch refers to a branch. 16 | RefKindBranch RefKind = "branch" 17 | 18 | // RefKindTag refers to a tag. 19 | RefKindTag RefKind = "tag" 20 | 21 | // RefKindMergeRequest refers to a tag. 22 | RefKindMergeRequest RefKind = "merge-request" 23 | ) 24 | 25 | // RefKind is used to determine the kind of the ref. 26 | type RefKind string 27 | 28 | // Ref is what we will use a metrics entity on which we will 29 | // perform regular pulling operations. 30 | type Ref struct { 31 | Kind RefKind 32 | Name string 33 | Project Project 34 | LatestPipeline Pipeline 35 | LatestJobs Jobs 36 | } 37 | 38 | // RefKey .. 39 | type RefKey string 40 | 41 | // Key .. 42 | func (ref Ref) Key() RefKey { 43 | return RefKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(string(ref.Kind) + ref.Project.Name + ref.Name))))) 44 | } 45 | 46 | // Refs allows us to keep track of all the Ref 47 | // we have configured/discovered. 48 | type Refs map[RefKey]Ref 49 | 50 | // Count returns the amount of projects refs in the map. 51 | func (refs Refs) Count() int { 52 | return len(refs) 53 | } 54 | 55 | // DefaultLabelsValues .. 56 | func (ref Ref) DefaultLabelsValues() map[string]string { 57 | return map[string]string{ 58 | "kind": string(ref.Kind), 59 | "project": ref.Project.Name, 60 | "ref": ref.Name, 61 | "topics": ref.Project.Topics, 62 | "variables": ref.LatestPipeline.Variables, 63 | "source": ref.LatestPipeline.Source, 64 | } 65 | } 66 | 67 | // NewRef is an helper which returns a new Ref. 68 | func NewRef( 69 | project Project, 70 | kind RefKind, 71 | name string, 72 | ) Ref { 73 | return Ref{ 74 | Kind: kind, 75 | Name: name, 76 | Project: project, 77 | LatestJobs: make(Jobs), 78 | } 79 | } 80 | 81 | // GetRefRegexp returns the expected regexp given a ProjectPullRefs config and a RefKind. 82 | func GetRefRegexp(ppr config.ProjectPullRefs, rk RefKind) (re *regexp.Regexp, err error) { 83 | switch rk { 84 | case RefKindBranch: 85 | return regexp.Compile(ppr.Branches.Regexp) 86 | case RefKindTag: 87 | return regexp.Compile(ppr.Tags.Regexp) 88 | case RefKindMergeRequest: 89 | return regexp.Compile(mergeRequestRegexp) 90 | } 91 | 92 | return nil, fmt.Errorf("invalid ref kind (%v)", rk) 93 | } 94 | 95 | // GetMergeRequestIIDFromRefName parse a refName to extract a merge request IID. 96 | func GetMergeRequestIIDFromRefName(refName string) (string, error) { 97 | re := regexp.MustCompile(mergeRequestRegexp) 98 | if matches := re.FindStringSubmatch(refName); len(matches) == 4 { 99 | if len(matches[2]) > 0 { 100 | return matches[2], nil 101 | } 102 | 103 | if len(matches[3]) > 0 { 104 | return matches[3], nil 105 | } 106 | } 107 | 108 | return refName, fmt.Errorf("unable to extract the merge-request ID from the ref (%s)", refName) 109 | } 110 | -------------------------------------------------------------------------------- /pkg/schemas/ref_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRefKey(t *testing.T) { 10 | assert.Equal(t, RefKey("1690074537"), NewRef( 11 | NewProject("foo/bar"), 12 | RefKindBranch, 13 | "baz", 14 | ).Key()) 15 | } 16 | 17 | func TestRefsCount(t *testing.T) { 18 | assert.Equal(t, 2, Refs{ 19 | RefKey("foo"): Ref{}, 20 | RefKey("bar"): Ref{}, 21 | }.Count()) 22 | } 23 | 24 | func TestRefDefaultLabelsValues(t *testing.T) { 25 | p := NewProject("foo/bar") 26 | p.Topics = "amazing,project" 27 | ref := Ref{ 28 | Project: p, 29 | Kind: RefKindBranch, 30 | Name: "feature", 31 | LatestPipeline: Pipeline{ 32 | Variables: "blah", 33 | Source: "schedule", 34 | }, 35 | LatestJobs: make(Jobs), 36 | } 37 | 38 | expectedValue := map[string]string{ 39 | "kind": "branch", 40 | "project": "foo/bar", 41 | "ref": "feature", 42 | "topics": "amazing,project", 43 | "variables": "blah", 44 | "source": "schedule", 45 | } 46 | 47 | assert.Equal(t, expectedValue, ref.DefaultLabelsValues()) 48 | } 49 | 50 | func TestNewRef(t *testing.T) { 51 | p := NewProject("foo/bar") 52 | p.Topics = "bar,baz" 53 | p.OutputSparseStatusMetrics = false 54 | p.Pull.Pipeline.Jobs.Enabled = true 55 | p.Pull.Pipeline.Jobs.FromChildPipelines.Enabled = false 56 | p.Pull.Pipeline.Jobs.RunnerDescription.Enabled = false 57 | p.Pull.Pipeline.Variables.Enabled = true 58 | p.Pull.Pipeline.Variables.Regexp = `.*` 59 | p.Pull.Pipeline.Jobs.RunnerDescription.AggregationRegexp = `.*` 60 | 61 | expectedValue := Ref{ 62 | Project: p, 63 | Kind: RefKindTag, 64 | Name: "v0.0.7", 65 | LatestJobs: make(Jobs), 66 | } 67 | 68 | assert.Equal(t, expectedValue, NewRef( 69 | p, 70 | RefKindTag, 71 | "v0.0.7", 72 | )) 73 | } 74 | 75 | func TestGetMergeRequestIIDFromRefName(t *testing.T) { 76 | name, err := GetMergeRequestIIDFromRefName("1234") 77 | assert.NoError(t, err) 78 | assert.Equal(t, "1234", name) 79 | 80 | name, err = GetMergeRequestIIDFromRefName("refs/merge-requests/5678/head") 81 | assert.NoError(t, err) 82 | assert.Equal(t, "5678", name) 83 | 84 | name, err = GetMergeRequestIIDFromRefName("20.0.1") 85 | assert.Error(t, err) 86 | assert.Equal(t, "20.0.1", name) 87 | 88 | name, err = GetMergeRequestIIDFromRefName("x") 89 | assert.Error(t, err) 90 | assert.Equal(t, "x", name) 91 | } 92 | -------------------------------------------------------------------------------- /pkg/schemas/tasks.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | // TaskType represents the type of a task. 4 | type TaskType string 5 | 6 | const ( 7 | // TaskTypePullProject .. 8 | TaskTypePullProject TaskType = "PullProject" 9 | 10 | // TaskTypePullProjectsFromWildcard .. 11 | TaskTypePullProjectsFromWildcard TaskType = "PullProjectsFromWildcard" 12 | 13 | // TaskTypePullProjectsFromWildcards .. 14 | TaskTypePullProjectsFromWildcards TaskType = "PullProjectsFromWildcards" 15 | 16 | // TaskTypePullEnvironmentsFromProject .. 17 | TaskTypePullEnvironmentsFromProject TaskType = "PullEnvironmentsFromProject" 18 | 19 | // TaskTypePullEnvironmentsFromProjects .. 20 | TaskTypePullEnvironmentsFromProjects TaskType = "PullEnvironmentsFromProjects" 21 | 22 | // TaskTypePullEnvironmentMetrics .. 23 | TaskTypePullEnvironmentMetrics TaskType = "PullEnvironmentMetrics" 24 | 25 | // TaskTypePullMetrics .. 26 | TaskTypePullMetrics TaskType = "PullMetrics" 27 | 28 | // TaskTypePullRefsFromProject .. 29 | TaskTypePullRefsFromProject TaskType = "PullRefsFromProject" 30 | 31 | // TaskTypePullRefsFromProjects .. 32 | TaskTypePullRefsFromProjects TaskType = "PullRefsFromProjects" 33 | 34 | // TaskTypePullRefMetrics .. 35 | TaskTypePullRefMetrics TaskType = "PullRefMetrics" 36 | 37 | // TaskTypeGarbageCollectProjects .. 38 | TaskTypeGarbageCollectProjects TaskType = "GarbageCollectProjects" 39 | 40 | // TaskTypeGarbageCollectEnvironments .. 41 | TaskTypeGarbageCollectEnvironments TaskType = "GarbageCollectEnvironments" 42 | 43 | // TaskTypeGarbageCollectRefs .. 44 | TaskTypeGarbageCollectRefs TaskType = "GarbageCollectRefs" 45 | 46 | // TaskTypeGarbageCollectMetrics .. 47 | TaskTypeGarbageCollectMetrics TaskType = "GarbageCollectMetrics" 48 | ) 49 | 50 | // Tasks can be used to keep track of tasks. 51 | type Tasks map[TaskType]map[string]interface{} 52 | -------------------------------------------------------------------------------- /pkg/schemas/tasks_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | -------------------------------------------------------------------------------- /pkg/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/redis/go-redis/v9" 7 | log "github.com/sirupsen/logrus" 8 | "go.opentelemetry.io/otel" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | // Store .. 15 | type Store interface { 16 | SetProject(ctx context.Context, p schemas.Project) error 17 | DelProject(ctx context.Context, pk schemas.ProjectKey) error 18 | GetProject(ctx context.Context, p *schemas.Project) error 19 | ProjectExists(ctx context.Context, pk schemas.ProjectKey) (bool, error) 20 | Projects(ctx context.Context) (schemas.Projects, error) 21 | ProjectsCount(ctx context.Context) (int64, error) 22 | SetEnvironment(ctx context.Context, e schemas.Environment) error 23 | DelEnvironment(ctx context.Context, ek schemas.EnvironmentKey) error 24 | GetEnvironment(ctx context.Context, e *schemas.Environment) error 25 | EnvironmentExists(ctx context.Context, ek schemas.EnvironmentKey) (bool, error) 26 | Environments(ctx context.Context) (schemas.Environments, error) 27 | EnvironmentsCount(ctx context.Context) (int64, error) 28 | SetRef(ctx context.Context, r schemas.Ref) error 29 | DelRef(ctx context.Context, rk schemas.RefKey) error 30 | GetRef(ctx context.Context, r *schemas.Ref) error 31 | RefExists(ctx context.Context, rk schemas.RefKey) (bool, error) 32 | Refs(ctx context.Context) (schemas.Refs, error) 33 | RefsCount(ctx context.Context) (int64, error) 34 | SetMetric(ctx context.Context, m schemas.Metric) error 35 | DelMetric(ctx context.Context, mk schemas.MetricKey) error 36 | GetMetric(ctx context.Context, m *schemas.Metric) error 37 | MetricExists(ctx context.Context, mk schemas.MetricKey) (bool, error) 38 | Metrics(ctx context.Context) (schemas.Metrics, error) 39 | MetricsCount(ctx context.Context) (int64, error) 40 | 41 | // Helpers to keep track of currently queued tasks and avoid scheduling them 42 | // twice at the risk of ending up with loads of dangling goroutines being locked 43 | QueueTask(ctx context.Context, tt schemas.TaskType, taskUUID, processUUID string) (bool, error) 44 | UnqueueTask(ctx context.Context, tt schemas.TaskType, taskUUID string) error 45 | CurrentlyQueuedTasksCount(ctx context.Context) (uint64, error) 46 | ExecutedTasksCount(ctx context.Context) (uint64, error) 47 | } 48 | 49 | // NewLocalStore .. 50 | func NewLocalStore() Store { 51 | return &Local{ 52 | projects: make(schemas.Projects), 53 | environments: make(schemas.Environments), 54 | refs: make(schemas.Refs), 55 | metrics: make(schemas.Metrics), 56 | } 57 | } 58 | 59 | // NewRedisStore .. 60 | func NewRedisStore(client *redis.Client) Store { 61 | return &Redis{ 62 | Client: client, 63 | } 64 | } 65 | 66 | // New creates a new store and populates it with 67 | // provided []schemas.Project. 68 | func New( 69 | ctx context.Context, 70 | r *redis.Client, 71 | projects config.Projects, 72 | ) (s Store) { 73 | ctx, span := otel.Tracer("gitlab-ci-pipelines-exporter").Start(ctx, "store:New") 74 | defer span.End() 75 | 76 | if r != nil { 77 | s = NewRedisStore(r) 78 | } else { 79 | s = NewLocalStore() 80 | } 81 | 82 | // Load all the configured projects in the store 83 | for _, p := range projects { 84 | sp := schemas.Project{Project: p} 85 | 86 | exists, err := s.ProjectExists(ctx, sp.Key()) 87 | if err != nil { 88 | log.WithContext(ctx). 89 | WithFields(log.Fields{ 90 | "project-name": p.Name, 91 | }). 92 | WithError(err). 93 | Error("reading project from the store") 94 | } 95 | 96 | if !exists { 97 | if err = s.SetProject(ctx, sp); err != nil { 98 | log.WithContext(ctx). 99 | WithFields(log.Fields{ 100 | "project-name": p.Name, 101 | }). 102 | WithError(err). 103 | Error("writing project in the store") 104 | } 105 | } 106 | } 107 | 108 | return 109 | } 110 | -------------------------------------------------------------------------------- /pkg/store/store_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/redis/go-redis/v9" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" 11 | "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" 12 | ) 13 | 14 | var testCtx = context.Background() 15 | 16 | func TestNewLocalStore(t *testing.T) { 17 | expectedValue := &Local{ 18 | projects: make(schemas.Projects), 19 | environments: make(schemas.Environments), 20 | refs: make(schemas.Refs), 21 | metrics: make(schemas.Metrics), 22 | } 23 | assert.Equal(t, expectedValue, NewLocalStore()) 24 | } 25 | 26 | func TestNewRedisStore(t *testing.T) { 27 | redisClient := redis.NewClient(&redis.Options{}) 28 | expectedValue := &Redis{ 29 | Client: redisClient, 30 | } 31 | 32 | assert.Equal(t, expectedValue, NewRedisStore(redisClient)) 33 | } 34 | 35 | func TestNew(t *testing.T) { 36 | localStore := New(testCtx, nil, config.Projects{}) 37 | assert.IsType(t, &Local{}, localStore) 38 | 39 | redisClient := redis.NewClient(&redis.Options{}) 40 | redisStore := New(testCtx, redisClient, config.Projects{}) 41 | assert.IsType(t, &Redis{}, redisStore) 42 | 43 | localStore = New(testCtx, nil, config.Projects{ 44 | { 45 | Name: "foo", 46 | }, 47 | { 48 | Name: "foo", 49 | }, 50 | { 51 | Name: "bar", 52 | }, 53 | }) 54 | count, _ := localStore.ProjectsCount(testCtx) 55 | assert.Equal(t, int64(2), count) 56 | } 57 | -------------------------------------------------------------------------------- /renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | $schema: "https://docs.renovatebot.com/renovate-schema.json", 3 | extends: ["config:best-practices"], 4 | 5 | postUpdateOptions: [ 6 | "gomodTidy", // Run go mod tidy after Go module updates. 7 | ], 8 | 9 | customManagers: [ 10 | // Update Makefile's go dependencies 11 | { 12 | customType: "regex", 13 | fileMatch: ["^Makefile$"], 14 | matchStrings: ["go run (?.*?)@(?.*?) "], 15 | datasourceTemplate: "go", 16 | }, 17 | ], 18 | 19 | packageRules: [ 20 | // Group all patch updates into a single PR 21 | { 22 | groupName: "all patch and minor", 23 | matchPackageNames: ["*"], 24 | matchUpdateTypes: ["minor", "patch", "digest"], 25 | automerge: true, 26 | }, 27 | ], 28 | } 29 | --------------------------------------------------------------------------------