├── .github ├── ISSUE_TEMPLATE │ ├── 1-bug-report.yml │ └── 2-feature-request.yml ├── actions │ └── setup-goversion │ │ └── action.yml ├── dependabot.yml └── workflows │ ├── docker.yml │ ├── release.yml │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yaml ├── .mockery.yaml ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── catalog-info.yaml ├── cloudcost-exporter-dashboards ├── README.md ├── convertor │ ├── README.md │ ├── convert_dashboard.go │ └── dashboard.json ├── dashboards │ ├── dashboards.go │ └── operations_dashboard.go └── grafana │ └── cloudcost-exporter-operations-dashboard.json ├── cmd ├── dashboards │ ├── main.go │ └── main_test.go └── exporter │ ├── config │ ├── config.go │ ├── string_slice_flag.go │ └── string_slice_flag_test.go │ ├── exporter.go │ └── web │ ├── web.go │ └── web_test.go ├── docs ├── contribute │ ├── creating-a-new-module.md │ ├── developer-guide.md │ ├── logging.md │ └── releases.md ├── deploying │ └── aws │ │ ├── README.md │ │ ├── cross-account-access-diagram.png │ │ ├── permissions-policy.json │ │ └── role-trust-policy.json └── metrics │ ├── aws │ ├── ec2.md │ └── s3.md │ ├── azure │ └── aks.md │ ├── gcp │ ├── gcs.md │ └── gke.md │ └── providers.md ├── go.mod ├── go.sum ├── main.go ├── mocks └── pkg │ ├── aws │ └── services │ │ ├── costexplorer │ │ └── CostExplorer.go │ │ ├── ec2 │ │ └── EC2.go │ │ └── pricing │ │ └── Pricing.go │ ├── azure │ └── azureClientWrapper │ │ └── azureClientWrapper.go │ ├── google │ └── gcs │ │ ├── CloudCatalogClient.go │ │ ├── RegionsClient.go │ │ └── StorageClientInterface.go │ └── provider │ ├── Collector.go │ ├── Provider.go │ └── Registry.go ├── pkg ├── aws │ ├── README.md │ ├── aws.go │ ├── aws_test.go │ ├── ec2 │ │ ├── README.md │ │ ├── compute.go │ │ ├── compute_test.go │ │ ├── disk.go │ │ ├── disk_test.go │ │ ├── ec2.go │ │ ├── ec2_test.go │ │ ├── pricing_map.go │ │ └── pricing_map_test.go │ ├── s3 │ │ ├── s3.go │ │ ├── s3_test.go │ │ └── testdata │ │ │ └── dimensions.csv │ └── services │ │ ├── README.md │ │ ├── costexplorer │ │ └── costexplorer.go │ │ ├── ec2 │ │ └── ec2.go │ │ └── pricing │ │ └── pricing.go ├── azure │ ├── README.md │ ├── aks │ │ ├── README.md │ │ ├── aks.go │ │ ├── aks_test.go │ │ ├── machine_store.go │ │ ├── machine_store_test.go │ │ ├── price_store.go │ │ └── price_store_test.go │ ├── azure.go │ ├── azureClientWrapper │ │ └── azureClientWrapper.go │ └── azure_test.go ├── google │ ├── README.md │ ├── billing │ │ ├── billing.go │ │ └── billing_test_helpers.go │ ├── gcp.go │ ├── gcp_test.go │ ├── gcs │ │ ├── bucket.go │ │ ├── bucket_cache.go │ │ ├── bucket_cache_test.go │ │ ├── bucket_test.go │ │ ├── gcs.go │ │ └── gcs_test.go │ └── gke │ │ ├── README.md │ │ ├── disk.go │ │ ├── disk_test.go │ │ ├── gke.go │ │ ├── gke_test.go │ │ ├── machinespec.go │ │ ├── machinespec_test.go │ │ ├── pricing_map.go │ │ └── pricing_map_test.go ├── logger │ └── level.go ├── provider │ ├── mocks │ │ └── provider.go │ └── provider.go └── utils │ ├── consts.go │ ├── consts_test.go │ ├── metrics.go │ └── metrics_test.go └── scripts ├── aws-spot-pricing └── main.go └── gcp-fetch-skus └── gcp-fetch-skus.go /.github/ISSUE_TEMPLATE/1-bug-report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report 3 | type: "bug" 4 | projects: ["grafana/513"] # Platform Monitoring 5 | body: 6 | - type: input 7 | id: cloudcost-exporter-version 8 | attributes: 9 | label: CloudCost Exporter Version 10 | - type: input 11 | id: csp 12 | attributes: 13 | label: Cloud Provider 14 | - type: textarea 15 | id: affected_modules 16 | attributes: 17 | label: Affected Modules(s) 18 | placeholder: | 19 | * eks|gke|aks 20 | * s3|gcs 21 | - type: textarea 22 | id: expected_behavior 23 | attributes: 24 | label: Expected Behavior 25 | placeholder: | 26 | What should have happened? 27 | - type: textarea 28 | id: actual_behavior 29 | attributes: 30 | label: Actual Behavior 31 | placeholder: | 32 | What actually happened? 33 | - type: textarea 34 | id: steps_to_reproduce 35 | attributes: 36 | label: Steps to Reproduce 37 | placeholder: | 38 | Please list the steps required to reproduce the issue. 39 | - type: textarea 40 | id: important_factoids 41 | attributes: 42 | label: Important Factoids 43 | placeholder: | 44 | Are there anything atypical about your cloudcost-exporter setup? In which kind of Kubernetes cluster does it run? 45 | - type: textarea 46 | id: references 47 | attributes: 48 | label: References 49 | placeholder: | 50 | Include a list of references to support the bug. 51 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Suggest an idea for CloudCost Exporter 3 | type: "enhancement" 4 | projects: ["grafana/513"] # Platform Monitoring 5 | body: 6 | - type: input 7 | id: csp 8 | attributes: 9 | label: Cloud Provider 10 | - type: textarea 11 | id: request 12 | attributes: 13 | label: Feature Request 14 | placeholder: | 15 | What would you like to see added to the Grafana provider? 16 | -------------------------------------------------------------------------------- /.github/actions/setup-goversion/action.yml: -------------------------------------------------------------------------------- 1 | # This action extracts the go version from Dockerfile and uses this version setup go 2 | name: setup-goversion 3 | description: Extracts the go version from Dockerfile and uses this version setup go 4 | runs: 5 | using: composite 6 | steps: 7 | - id: goversion 8 | run: | 9 | cat Dockerfile | awk 'BEGIN{IGNORECASE=1} /^FROM golang:.* AS build$/ {v=$2;split(v,a,":|-")}; END {printf("version=%s", a[2])}' >> $GITHUB_OUTPUT 10 | shell: bash 11 | - uses: actions/setup-go@v5 12 | with: 13 | go-version: "${{steps.goversion.outputs.version}}" 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | - package-ecosystem: "docker" # See documentation for possible values 13 | directory: "/" # Location of package manifests 14 | schedule: 15 | interval: "weekly" 16 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.docker.com/build/ci/github-actions/multi-platform/ 2 | name: Build and Push Image 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - v* 10 | 11 | permissions: 12 | contents: read 13 | id-token: write 14 | 15 | env: 16 | REGISTRY_IMAGE: grafana/cloudcost-exporter 17 | # Docker image tags. See https://github.com/docker/metadata-action for format 18 | TAGS_CONFIG: | 19 | type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} 20 | type=sha,prefix={{ branch }}-,format=short,enable=${{ github.ref == 'refs/heads/main' }} 21 | type=semver,pattern={{ version }} 22 | 23 | jobs: 24 | build: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Checkout 28 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 29 | with: 30 | persist-credentials: false 31 | - name: Build and push 32 | uses: grafana/shared-workflows/actions/build-push-to-dockerhub@402975d84dd3fac9ba690f994f412d0ee2f51cf4 # build-push-to-dockerhub-v0.1.1 33 | with: 34 | repository: ${{ env.REGISTRY_IMAGE }} 35 | context: . 36 | push: true 37 | platforms: linux/amd64,linux/arm64 38 | tags: ${{ env.TAGS_CONFIG }} 39 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | 7 | permissions: 8 | contents: write 9 | 10 | jobs: 11 | goreleaser: 12 | name: Release 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 17 | with: 18 | fetch-depth: 0 19 | persist-credentials: false 20 | - name: Set up Go 21 | uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # 5.1.0 22 | with: 23 | cache: false 24 | - name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # 6.1.0 26 | with: 27 | distribution: goreleaser 28 | version: 2 29 | args: release --clean 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Tests 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | build-lint-test: 14 | permissions: 15 | pull-requests: read 16 | contents: read 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 21 | with: 22 | persist-credentials: false 23 | - uses: ./.github/actions/setup-goversion 24 | - name: Build 25 | run: go build -v ./... 26 | - name: Lint 27 | uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # 6.1.1 28 | with: 29 | version: v1.64.6 30 | - name: Test 31 | run: go test -v ./... 32 | - name: Install make 33 | run: sudo apt-get update && sudo apt-get install -y make 34 | shell: bash 35 | - name: Check for Dashboards Drift 36 | run: | 37 | make build-dashboards > /dev/null 38 | if ! git diff --exit-code; then 39 | echo "Dashboards are out of sync. Please run 'make build-dashboards' and commit the changes." 40 | exit 1 41 | fi 42 | shell: bash 43 | 44 | 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/go 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=go 3 | 4 | ### Go ### 5 | # If you prefer the allow list template instead of the deny list, see community template: 6 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 7 | # 8 | # Binaries for programs and plugins 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | 15 | # Test binary, built with `go test -c` 16 | *.test 17 | 18 | # Output of the go coverage tool, specifically when used with LiteIDE 19 | *.out 20 | 21 | # Dependency directories (remove the comment below to include it) 22 | # vendor/ 23 | 24 | # Go workspace file 25 | go.work 26 | 27 | # End of https://www.toptal.com/developers/gitignore/api/go 28 | 29 | .run 30 | .idea 31 | .vscode 32 | 33 | cloudcost-exporter 34 | 35 | dist/ 36 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | output: 2 | formats: 3 | - format: colored-line-number 4 | 5 | linters: 6 | disable-all: true 7 | enable: 8 | - goimports 9 | - gofmt 10 | - misspell 11 | - errorlint 12 | - unused 13 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # Make sure to check the documentation at https://goreleaser.com 2 | 3 | version: 2 4 | 5 | project_name: cloudcost-explorer 6 | builds: 7 | - id: cloudcost-explorer 8 | binary: cloudcost-explorer 9 | main: ./cmd/exporter/exporter.go 10 | env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | goarch: 15 | - amd64 16 | - arm64 17 | 18 | archives: 19 | - format: tar.gz 20 | # this name template makes the OS and Arch compatible with the results of uname. 21 | name_template: >- 22 | {{ .ProjectName }}_ 23 | {{- title .Os }}_ 24 | {{- if eq .Arch "amd64" }}x86_64 25 | {{- else }}{{ .Arch }}{{ end }} 26 | {{- if .Arm }}v{{ .Arm }}{{ end }} 27 | checksum: 28 | name_template: 'checksums.txt' 29 | changelog: 30 | sort: asc 31 | filters: 32 | exclude: 33 | - '^docs' 34 | - '^chore(deps):' 35 | -------------------------------------------------------------------------------- /.mockery.yaml: -------------------------------------------------------------------------------- 1 | with-expecter: True 2 | inpackage: True 3 | dir: mocks/{{ replaceAll .InterfaceDirRelative "internal" "internal_" }} 4 | mockname: "{{.InterfaceName}}" 5 | outpkg: "{{.PackageName}}" 6 | filename: "{{.InterfaceName}}.go" 7 | all: True 8 | packages: 9 | github.com/grafana/cloudcost-exporter: 10 | config: 11 | recursive: True 12 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default all reviews to be Grafana's Monitoring squad 2 | * @grafana/platform-monitoring 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at contact@grafana.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for your interest in contributing to Grafana `cloudcost-exporter` project! 4 | We welcome all people who want to contribute in a healthy and constructive manner within our community. To help us create a safe and positive community experience for all, we require all participants to adhere to the [Code of Conduct](CODE_OF_CONDUCT.md). 5 | 6 | ## Become a contributor 7 | 8 | You can contribute to Grafana `cloudcost-exporter` in several ways. Here are some examples: 9 | 10 | - Contribute to the Grafana `cloudcost-exporter` codebase 11 | - Report bugs and enhancements 12 | - Implement new collectors for Cloud Service Providers that are not yet supported 13 | 14 | For more ways to contribute, check out the [Open Source Guides](https://opensource.guide/how-to-contribute/). 15 | 16 | ## Report bugs 17 | 18 | Report a bug by submitting a [bug report](https://github.com/grafana/cloudcost-exporter/issues/new?labels=bug&template=1-bug_report.md). Make sure that you provide as much information as possible on how to reproduce the bug. 19 | 20 | Before submitting a new issue, try to make sure someone hasn't already reported the problem. Look through the [existing issues](https://github.com/grafana/cloudcost-exporter/issues) for similar issues. 21 | 22 | ## Security issues 23 | 24 | If you believe you've found a security vulnerability, please read our [security policy](https://github.com/grafana/cloudcost-exporter/security/policy) for more details. 25 | 26 | ## Suggest enhancements 27 | 28 | If you have an idea of how to improve Grafana's CloudCost Exporter, submit an [enhancement request](https://github.com/grafana/cloudcost-exporter/issues/new?labels=enhancement&template=2-enhancement_request.md). 29 | 30 | ## Where do I go from here? 31 | 32 | - Set up your [development environment](docs/contribute/developer-guide.md). 33 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build Go Binary 2 | FROM golang:1.24.3 AS build 3 | 4 | WORKDIR /app 5 | COPY ["go.mod", "go.sum", "./"] 6 | RUN go mod download 7 | 8 | COPY . . 9 | 10 | ENV GOCACHE=/go/pkg/mod/ 11 | RUN --mount=type=cache,target="/go/pkg/mod/" make build-binary 12 | 13 | # Build Image 14 | FROM scratch 15 | COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 16 | COPY --from=build /etc/passwd /etc/passwd 17 | 18 | WORKDIR /root 19 | 20 | COPY --from=build /app/cloudcost-exporter ./ 21 | ENTRYPOINT ["./cloudcost-exporter"] 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build-image build-binary build test push push-dev 2 | 3 | VERSION=$(shell git describe --tags --dirty --always) 4 | 5 | IMAGE_PREFIX=grafana 6 | 7 | IMAGE_NAME=cloudcost-exporter 8 | IMAGE_NAME_LATEST=${IMAGE_PREFIX}/${IMAGE_NAME}:latest 9 | IMAGE_NAME_VERSION=$(IMAGE_PREFIX)/$(IMAGE_NAME):$(VERSION) 10 | 11 | WORKFLOW_TEMPLATE=cloudcost-exporter 12 | 13 | PROM_VERSION_PKG ?= github.com/prometheus/common/version 14 | BUILD_USER ?= $(shell whoami)@$(shell hostname) 15 | BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") 16 | GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) 17 | GIT_REVISION ?= $(shell git rev-parse --short HEAD) 18 | GO_LDFLAGS = -X $(PROM_VERSION_PKG).Branch=$(GIT_BRANCH) -X $(PROM_VERSION_PKG).Version=$(VERSION) -X $(PROM_VERSION_PKG).Revision=$(GIT_REVISION) -X ${PROM_VERSION_PKG}.BuildUser=${BUILD_USER} -X ${PROM_VERSION_PKG}.BuildDate=${BUILD_DATE} 19 | 20 | build-image: 21 | docker build --build-arg GO_LDFLAGS="$(GO_LDFLAGS)" -t $(IMAGE_PREFIX)/$(IMAGE_NAME) -t $(IMAGE_NAME_VERSION) . 22 | 23 | build-binary: 24 | CGO_ENABLED=0 go build -v -ldflags "$(GO_LDFLAGS)" -o cloudcost-exporter ./cmd/exporter 25 | 26 | build: build-binary build-image 27 | 28 | generate-mocks: 29 | mockgen -source=pkg/provider/provider.go -destination pkg/provider/mocks/provider.go 30 | mockgen -source=pkg/azure/azureClientWrapper/azureClientWrapper.go -destination mocks/pkg/azure/azureClientWrapper/azureClientWrapper.go 31 | 32 | test: build 33 | go test -v ./... 34 | 35 | lint: 36 | golangci-lint run ./... 37 | 38 | push-dev: build test 39 | docker push $(IMAGE_NAME_VERSION) 40 | 41 | push: build test push-dev 42 | docker push $(IMAGE_NAME_LATEST) 43 | 44 | grizzly-serve: 45 | grr serve -p 8088 -w -S "go run ./cloudcost-exporter-dashboards/main.go" 46 | 47 | build-dashboards: 48 | go run ./cmd/dashboards/main.go --output=file 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloud Cost Exporter 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/grafana/cloudcost-exporter.svg)](https://pkg.go.dev/github.com/grafana/cloudcost-exporter) 4 | 5 | Cloud Cost exporter is a tool designed to collect cost data from cloud providers and export the data in Prometheus format. 6 | The cost data can then be combined with usage data from tools such as stackdriver, yace, and promitor to measure the spend of resources at a granular level. 7 | 8 | ## Goals 9 | 10 | The goal of this project is to provide a consistent interface for collecting the rate of cost data from multiple cloud providers and exporting the data in Prometheus format. 11 | There was a need to track the costs of both kubernetes and non-kubernetes resources across multiple cloud providers at a per minute interval. 12 | Billing data for each cloud provider takes hours to days for it to be fully accurate, and we needed a way of having a more real-time view of costs. 13 | 14 | Primary goals: 15 | - Track the rate(IE, $/cpu/hr) for resources across 16 | - Export the rate in Prometheus format 17 | - Support the major cloud providers(AWS, GCP, Azure) 18 | 19 | Non Goals: 20 | - Billing level accuracy 21 | - Measure the spend of resources 22 | - Take into account CUDs/Discounts/Reservations pricing information 23 | 24 | ## Supported Cloud Providers 25 | 26 | - AWS 27 | - GCP 28 | - Azure 29 | 30 | ## Installation 31 | 32 | Each tagged version of the Cloud Cost Exporter will publish a Docker image to https://hub.docker.com/r/grafana/cloudcost-exporter and a Helm chart. 33 | 34 | ### Local usage 35 | 36 | The image can be used to deploy Cloud Cost Exporter to a Kubernetes cluster or to run it locally. 37 | 38 | #### Use the image 39 | 40 | Cloud Cost Exporter has an opinionated way of authenticating against each cloud provider: 41 | 42 | | Provider | Notes | 43 | |-|-| 44 | | GCP | Depends on [default credentials](https://cloud.google.com/docs/authentication/application-default-credentials) | 45 | | AWS | Uses profile names from your [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) or `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` env variables | 46 | | Azure | Uses the [default azure credential chain](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash), e.g. enviornment variables: `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and `AZURE_CLIENT_SECRET` | 47 | 48 | ### Deployment to Kubernetes 49 | 50 | When running in a Kubernetes cluster, it is recommended to create an IAM role for a Service Account (IRSA) with the necessary permissions for the cloud provider. 51 | 52 | Documentation about the necessary permission for AWS can be found [here](./docs/deploying/aws/README.md#1-setup-the-iam-role). Documentation for GCP and Azure are under development. 53 | 54 | #### Use the Helm chart 55 | 56 | When deploying to Kubernetes, it is recommended to use the Helm chart, which can be found here: https://github.com/grafana/helm-charts/tree/main/charts/cloudcost-exporter 57 | 58 | Additional Helm chart configuration for AWS can be found [here](./docs/deploying/aws/README.md#2-configure-the-helm-chart) 59 | 60 | ## Metrics 61 | 62 | Check out the follow docs for metrics: 63 | - [provider level](docs/metrics/providers.md) 64 | - gcp 65 | - [gke](docs/metrics/gcp/gke.md) 66 | - [gcs](docs/metrics/gcp/gcs.md) 67 | - aws 68 | - [s3](docs/metrics/aws/s3.md) 69 | - [ec2](docs/metrics/aws/ec2.md) 70 | - azure 71 | - [aks](docs/metrics/azure/aks.md) 72 | 73 | ## Maturity 74 | 75 | This project is in the early stages of development and is subject to change. 76 | Grafana Labs builds and maintains this project as part of our commitment to the open-source community, but we do not provide support for it. 77 | In its current state, the exporter exports rates for resources and not the total spend. 78 | 79 | For a better understanding of how we view measuring costs, view a talk given at [KubeCon NA 2023](https://youtu.be/8eiLXtL3oLk?si=wm-43ZQ9Fr51wS4a&t=1) 80 | 81 | In the future, we intend to opensource recording rules we use internally to measure the spend of resources. 82 | 83 | ## Contributing 84 | 85 | Grafana Labs is always looking to support new contributors! 86 | Please take a look at our [contributing guide](CONTRIBUTING.md) for more information on how to get started. 87 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting security issues 2 | 3 | If you think you have found a security vulnerability, please send a report to [security@grafana.com](mailto:security@grafana.com). This address can be used for all of Grafana Labs's open source and commercial products (including but not limited to Grafana, Grafana Cloud, Grafana Enterprise, and grafana.com). We can accept only vulnerability reports at this address. 4 | 5 | Please encrypt your message to us; please use our PGP key. The key fingerprint is: 6 | 7 | F988 7BEA 027A 049F AE8E 5CAA D125 8932 BE24 C5CA 8 | 9 | The key is available from [pgp.mit.edu](https://pgp.mit.edu/pks/lookup?op=get&search=0xF9887BEA027A049FAE8E5CAAD1258932BE24C5CA) by searching for [grafana](https://pgp.mit.edu/pks/lookup?search=grafana&op=index). 10 | 11 | Grafana Labs will send you a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. 12 | 13 | **Important:** We ask you to not disclose the vulnerability before it have been fixed and announced, unless you received a response from the Grafana Labs security team that you can do so. 14 | 15 | ## Security announcements 16 | 17 | We maintain a category on the community site called [Security Announcements](https://community.grafana.com/c/security-announcements), 18 | where we will post a summary, remediation, and mitigation details for any patch containing security fixes. 19 | 20 | You can also subscribe to email updates to this category if you have a grafana.com account and sign on to the community site or track updates via an [RSS feed](https://community.grafana.com/c/security-announcements.rss). 21 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: cloudcost-exporter 5 | title: Cloudcost Exporter 6 | annotations: 7 | github.com/project-slug: grafana/cloudcost-exporter 8 | links: 9 | - title: Community Slack Channel 10 | url: https://grafana.slack.com/archives/C064EMEB4E6 11 | - title: Internal Slack Channel 12 | url: https://raintank-corp.slack.com/archives/C067MGQ98AW 13 | description: | 14 | Prometheus Exporter for Cloud Provider agnostic cost metrics 15 | spec: 16 | type: service 17 | owner: group:default/platform-monitoring 18 | lifecycle: production 19 | -------------------------------------------------------------------------------- /cloudcost-exporter-dashboards/README.md: -------------------------------------------------------------------------------- 1 | # Grafana Dashboards 2 | 3 | > [!WARNING] 4 | > This is still highly experimental as engineers at Grafana Labs are learning how to generate dashboards as code. 5 | > The main goal is for us to be able to generate and use the same internal dashboards as we recommend OSS users to use. 6 | 7 | This container a set of Grafana Dashboards that are generated using the [Grafana Foundation SDK](https://github.com/grafana/grafana-foundation-sdk). 8 | 9 | ## Getting Started 10 | 11 | > [!INFO] 12 | > If you want to develop these dashboards and view them against a live Grafana instance, 13 | > install and configure [grizzly](https://grafana.github.io/grizzly/installation/) 14 | 15 | To generate the dashboards: 16 | 17 | ```shell 18 | make build-dashboards 19 | ``` 20 | 21 | To iteratively develop dashboards with live reload: 22 | 23 | ```shell 24 | make grizzly-serve 25 | ``` 26 | -------------------------------------------------------------------------------- /cloudcost-exporter-dashboards/convertor/README.md: -------------------------------------------------------------------------------- 1 | # Convertor 2 | 3 | This is a small script that's intended to be used to convert a json blob dashboard to Go code using Grafana's Foundation SDK to manage dashboards. 4 | 5 | ## Usage 6 | 7 | First get a dashboard represented as a [json blob](https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json) and output the contents into `dashboard.json` 8 | 9 | Then execute the script with the following command: 10 | ```bash 11 | go run convert_dashboard.go 12 | ``` 13 | Copy the output and paste it into a new file in the `config/dashboards` directory. 14 | 15 | -------------------------------------------------------------------------------- /cloudcost-exporter-dashboards/convertor/convert_dashboard.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/grafana/grafana-foundation-sdk/go/cog/plugins" 9 | "github.com/grafana/grafana-foundation-sdk/go/dashboard" 10 | ) 11 | 12 | func main() { 13 | 14 | // Required to correctly unmarshal panels and dataqueries 15 | plugins.RegisterDefaultPlugins() 16 | 17 | dashboardJSON, err := os.ReadFile("dashboard.json") 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | dash := dashboard.Dashboard{} 23 | 24 | if err = json.Unmarshal(dashboardJSON, &dash); err != nil { 25 | panic(err) 26 | } 27 | 28 | converted := dashboard.DashboardConverter(dash) 29 | fmt.Println(converted) 30 | } 31 | -------------------------------------------------------------------------------- /cloudcost-exporter-dashboards/dashboards/dashboards.go: -------------------------------------------------------------------------------- 1 | package dashboards 2 | 3 | import ( 4 | "github.com/grafana/grafana-foundation-sdk/go/dashboard" 5 | ) 6 | 7 | func BuildDashboards() []*dashboard.DashboardBuilder { 8 | operationDashboard := OperationsDashboard() 9 | return []*dashboard.DashboardBuilder{ 10 | operationDashboard, 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /cloudcost-exporter-dashboards/dashboards/operations_dashboard.go: -------------------------------------------------------------------------------- 1 | package dashboards 2 | 3 | import ( 4 | "github.com/grafana/grafana-foundation-sdk/go/cog" 5 | "github.com/grafana/grafana-foundation-sdk/go/common" 6 | "github.com/grafana/grafana-foundation-sdk/go/dashboard" 7 | "github.com/grafana/grafana-foundation-sdk/go/prometheus" 8 | "github.com/grafana/grafana-foundation-sdk/go/stat" 9 | "github.com/grafana/grafana-foundation-sdk/go/timeseries" 10 | ) 11 | 12 | func OperationsDashboard() *dashboard.DashboardBuilder { 13 | builder := dashboard.NewDashboardBuilder("CloudCost Exporter Operations Dashboard"). 14 | // leaving this for BC reasons, but a proper human-readable UID would be better. 15 | Uid("1a9c0de366458599246184cf0ae8b468"). 16 | Editable(). 17 | Tooltip(dashboard.DashboardCursorSyncCrosshair). 18 | Refresh("30s"). 19 | WithVariable(dashboard.NewDatasourceVariableBuilder("datasource"). 20 | Label("Data Source"). 21 | Type("prometheus"), 22 | ). 23 | WithVariable(dashboard.NewQueryVariableBuilder("cluster"). 24 | // TODO: this query is grafana-specific. We should not expect every user to have `tanka_environment_info` metric. 25 | Query(dashboard.StringOrMap{ 26 | String: cog.ToPtr("label_values(tanka_environment_info{app=\"cloudcost-exporter\"},exported_cluster)"), 27 | }). 28 | Datasource(prometheusDatasourceRef()). 29 | Multi(true). 30 | Refresh(dashboard.VariableRefreshOnDashboardLoad). 31 | IncludeAll(true). 32 | AllValue(".*"), 33 | ). 34 | Annotation(dashboard.NewAnnotationQueryBuilder(). 35 | Name("Annotations & Alerts"). 36 | Datasource(dashboard.DataSourceRef{ 37 | Type: cog.ToPtr[string]("grafana"), 38 | Uid: cog.ToPtr[string]("-- Grafana --"), 39 | }). 40 | Hide(true). 41 | IconColor("rgba(0, 211, 255, 1)"). 42 | Type("dashboard"). 43 | BuiltIn(1), 44 | ). 45 | WithRow(dashboard.NewRowBuilder("Overview")). 46 | WithPanel(collectorStatusCurrent().Height(6).Span(12)). 47 | WithPanel(collectorScrapeDurationOverTime().Height(8).Span(24)). 48 | WithRow(dashboard.NewRowBuilder("AWS")). 49 | WithPanel(costExplorerAPIRequestsOverTime().Height(6).Span(12)). 50 | WithPanel(awsS3NextPricingMapRefreshOverTime().Height(6).Span(12)). 51 | WithRow(dashboard.NewRowBuilder("GCP")). 52 | WithPanel(gcpListBucketsRPSOverTime().Height(6).Span(12)). 53 | WithPanel(gcpNextScrapeOverTime().Height(6).Span(12)) 54 | return builder 55 | } 56 | 57 | func prometheusDatasourceRef() dashboard.DataSourceRef { 58 | return dashboard.DataSourceRef{ 59 | Type: cog.ToPtr[string]("prometheus"), 60 | Uid: cog.ToPtr[string]("${datasource}"), 61 | } 62 | } 63 | 64 | func prometheusQuery(expression string, legendFormat string) *prometheus.DataqueryBuilder { 65 | return prometheus.NewDataqueryBuilder(). 66 | Expr(expression). 67 | Range(). 68 | LegendFormat(legendFormat) 69 | } 70 | 71 | func collectorScrapeDurationOverTime() *timeseries.PanelBuilder { 72 | return timeseries.NewPanelBuilder(). 73 | Title("Collector Scrape Duration"). 74 | Description("Duration of scrapes by provider and collector"). 75 | Datasource(prometheusDatasourceRef()). 76 | WithTarget( 77 | prometheusQuery("cloudcost_exporter_collector_last_scrape_duration_seconds", "{{provider}}:{{collector}}"), 78 | ). 79 | Unit("s"). 80 | ThresholdsStyle(common.NewGraphThresholdsStyleConfigBuilder().Mode(common.GraphThresholdsStyleModeLine)). 81 | Thresholds(dashboard.NewThresholdsConfigBuilder(). 82 | Mode(dashboard.ThresholdsModeAbsolute). 83 | Steps([]dashboard.Threshold{ 84 | {Color: "green"}, 85 | {Value: cog.ToPtr[float64](60), Color: "red"}, 86 | }), 87 | ). 88 | ColorScheme(dashboard.NewFieldColorBuilder().Mode("palette-classic")). 89 | Legend(common.NewVizLegendOptionsBuilder(). 90 | DisplayMode(common.LegendDisplayModeTable). 91 | Placement(common.LegendPlacementBottom). 92 | ShowLegend(true). 93 | SortBy("Last *"). 94 | SortDesc(true). 95 | Calcs([]string{"lastNotNull", 96 | "min", 97 | "max"}), 98 | ) 99 | } 100 | 101 | func costExplorerAPIRequestsOverTime() *timeseries.PanelBuilder { 102 | return timeseries.NewPanelBuilder(). 103 | Title("CostExplorer API Requests"). 104 | Datasource(prometheusDatasourceRef()). 105 | WithTarget( 106 | prometheusQuery( 107 | "sum by (cluster) (increase(cloudcost_exporter_aws_s3_cost_api_requests_total{cluster=~\"$cluster\"}[5m]))", 108 | "__auto", 109 | ), 110 | ). 111 | Unit("reqps") 112 | } 113 | 114 | func awsS3NextPricingMapRefreshOverTime() *timeseries.PanelBuilder { 115 | return timeseries.NewPanelBuilder(). 116 | Title("Next pricing map refresh"). 117 | Description("The AWS s3 module uses cost data pulled from Cost Explorer, which costs $0.01 per API call. The cost metrics are refreshed every hour, so if this value goes below 0, it indicates a problem with refreshing the pricing map and thus needs investigation."). 118 | Datasource(prometheusDatasourceRef()). 119 | Unit("s"). 120 | WithTarget( 121 | prometheusQuery( 122 | "max by (cluster) (cloudcost_exporter_aws_s3_next_scrape{cluster=~\"$cluster\"}) - time() ", 123 | "__auto", 124 | ), 125 | ) 126 | } 127 | 128 | func gcpListBucketsRPSOverTime() *timeseries.PanelBuilder { 129 | return timeseries.NewPanelBuilder(). 130 | Title("GCS List Buckets Requests Per Second"). 131 | Description("The number of requests per second to list buckets in GCS."). 132 | Datasource(prometheusDatasourceRef()). 133 | Unit("reqps"). 134 | WithTarget(prometheusQuery("sum by (cluster, status) (increase(cloudcost_exporter_gcp_gcs_bucket_list_status_total[5m]))", "{{cluster}}:{{status}}")) 135 | } 136 | 137 | func gcpNextScrapeOverTime() *timeseries.PanelBuilder { 138 | return timeseries.NewPanelBuilder(). 139 | Title("GCS Pricing Map Refresh Time"). 140 | Description("The amount of time before the next refresh of the GCS pricing map. "). 141 | Datasource(prometheusDatasourceRef()). 142 | Unit("s"). 143 | WithTarget( 144 | prometheusQuery("max by (cluster) (cloudcost_exporter_gcp_gcs_next_scrape{cluster=~\"$cluster\"}) - time() ", "__auto"), 145 | ) 146 | } 147 | 148 | func collectorStatusCurrent() *stat.PanelBuilder { 149 | return stat.NewPanelBuilder(). 150 | Title("Collector Status"). 151 | Description("Display the status of all the collectors running."). 152 | Datasource(prometheusDatasourceRef()). 153 | Unit("short"). 154 | WithTarget( 155 | prometheusQuery("max by (provider, collector) (cloudcost_exporter_collector_last_scrape_error == 0)", "{{provider}}:{{collector}}"). 156 | Instant(), 157 | ). 158 | JustifyMode(common.BigValueJustifyModeAuto). 159 | TextMode(common.BigValueTextModeAuto). 160 | Orientation(common.VizOrientationAuto). 161 | ReduceOptions(common.NewReduceDataOptionsBuilder(). 162 | Values(false). 163 | Calcs([]string{"lastNotNull"}), 164 | ). 165 | Mappings([]dashboard.ValueMapping{ 166 | { 167 | ValueMap: cog.ToPtr[dashboard.ValueMap](dashboard.ValueMap{ 168 | Type: "value", 169 | Options: map[string]dashboard.ValueMappingResult{ 170 | "0": {Text: cog.ToPtr[string]("Up"), Index: cog.ToPtr[int32](0)}, 171 | }, 172 | }), 173 | }, 174 | { 175 | SpecialValueMap: cog.ToPtr[dashboard.SpecialValueMap](dashboard.SpecialValueMap{ 176 | Type: "special", 177 | Options: dashboard.DashboardSpecialValueMapOptions{ 178 | Match: "null+nan", 179 | Result: dashboard.ValueMappingResult{ 180 | Text: cog.ToPtr[string]("Down"), 181 | Color: cog.ToPtr[string]("red"), 182 | Index: cog.ToPtr[int32](1), 183 | }, 184 | }, 185 | }), 186 | }, 187 | }) 188 | } 189 | -------------------------------------------------------------------------------- /cmd/dashboards/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | "github.com/grafana/grafana-foundation-sdk/go/dashboard" 12 | 13 | "github.com/grafana/cloudcost-exporter/cloudcost-exporter-dashboards/dashboards" 14 | ) 15 | 16 | func main() { 17 | output := flag.String("output", "console", "Where to write output to. Can be console or file") 18 | outputDir := flag.String("output-dir", "./cloudcost-exporter-dashboards/grafana", "output directory") 19 | flag.Parse() 20 | dashes := dashboards.BuildDashboards() 21 | 22 | err := run(dashes, output, outputDir) 23 | if err != nil { 24 | log.Fatalf("error generating dashboards: %s", err.Error()) 25 | } 26 | } 27 | 28 | func run(dashes []*dashboard.DashboardBuilder, output *string, outputDir *string) error { 29 | for _, dash := range dashes { 30 | build, err := dash.Build() 31 | if err != nil { 32 | return err 33 | } 34 | data, err := json.MarshalIndent(build, "", " ") 35 | if err != nil { 36 | return err 37 | } 38 | if *output == "console" { 39 | fmt.Println(string(data)) 40 | continue 41 | } 42 | 43 | err = os.WriteFile(fmt.Sprintf("%s/%s.json", *outputDir, sluggify(*build.Title)), data, 0644) 44 | if err != nil { 45 | return err 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | // sluggify will take a string and convert the string 52 | func sluggify(s string) string { 53 | s = strings.TrimSpace(s) 54 | return strings.ReplaceAll(strings.ToLower(s), " ", "-") 55 | } 56 | -------------------------------------------------------------------------------- /cmd/dashboards/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_sluggify(t *testing.T) { 8 | tests := map[string]struct { 9 | s string 10 | want string 11 | }{ 12 | "empty string should return gracefully": {s: "", want: ""}, 13 | "single word should return as is": {s: "word", want: "word"}, 14 | "multiple words should be hyphenated": {s: "multiple words", want: "multiple-words"}, 15 | "mixed case should be lowercased": {s: "Mixed Case", want: "mixed-case"}, 16 | "leading and trailing spaces should be removed": {s: " leading and trailing spaces ", want: "leading-and-trailing-spaces"}, 17 | } 18 | for name, tt := range tests { 19 | t.Run(name, func(t *testing.T) { 20 | if got := sluggify(tt.s); got != tt.want { 21 | t.Errorf("sluggify() = %v, want %v", got, tt.want) 22 | } 23 | }) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cmd/exporter/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "log/slog" 5 | "time" 6 | ) 7 | 8 | type Config struct { 9 | Provider string 10 | ProjectID string 11 | Providers struct { 12 | AWS struct { 13 | Profile string 14 | Region string 15 | Services StringSliceFlag 16 | RoleARN string 17 | } 18 | GCP struct { 19 | DefaultGCSDiscount int 20 | Projects StringSliceFlag 21 | Region string 22 | Services StringSliceFlag 23 | } 24 | Azure struct { 25 | Services StringSliceFlag 26 | SubscriptionId string 27 | } 28 | } 29 | Collector struct { 30 | ScrapeInterval time.Duration 31 | Timeout time.Duration 32 | } 33 | 34 | Server struct { 35 | Address string 36 | Path string 37 | Timeout time.Duration 38 | } 39 | LoggerOpts struct { 40 | Level string // Maps to slog levels: debug, info, warn, error 41 | Output string // io.Writer interface to write out to: stdout, stderr, file 42 | Type string // How to write out the logs: json, text 43 | } 44 | 45 | Logger *slog.Logger 46 | } 47 | -------------------------------------------------------------------------------- /cmd/exporter/config/string_slice_flag.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "strings" 4 | 5 | type StringSliceFlag []string 6 | 7 | func (f *StringSliceFlag) String() string { 8 | return strings.Join(*f, ",") 9 | } 10 | 11 | func (f *StringSliceFlag) Set(value string) error { 12 | *f = append(*f, value) 13 | return nil 14 | } 15 | -------------------------------------------------------------------------------- /cmd/exporter/config/string_slice_flag_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "flag" 5 | "testing" 6 | ) 7 | 8 | func TestStringSliceFlag_Set(t *testing.T) { 9 | tests := map[string]struct { 10 | values []string 11 | exp int 12 | }{ 13 | "empty": { 14 | values: []string{}, 15 | exp: 0, 16 | }, 17 | "single": { 18 | values: []string{"-test", "test1"}, 19 | exp: 1, 20 | }, 21 | "multiple": { 22 | values: []string{"-test", "test1", "-test", "test2"}, 23 | exp: 2, 24 | }, 25 | } 26 | 27 | for name, test := range tests { 28 | t.Run(name, func(t *testing.T) { 29 | var ssf StringSliceFlag 30 | fs := flag.NewFlagSet("test", flag.ContinueOnError) 31 | fs.Var(&ssf, "test", "test") 32 | if err := fs.Parse(test.values); err != nil { 33 | t.Fatalf("unexpected error: %v", err) 34 | } 35 | if exp, got := test.exp, len(ssf); exp != got { 36 | t.Fatalf("expected %d, got %d", exp, got) 37 | } 38 | }) 39 | } 40 | } 41 | 42 | func TestStringSliceFlag_String(t *testing.T) { 43 | tests := map[string]struct { 44 | values []string 45 | exp string 46 | }{ 47 | "empty": { 48 | values: []string{}, 49 | exp: "", 50 | }, "single": { 51 | values: []string{"test1"}, 52 | exp: "test1", 53 | }, "multiple": { 54 | values: []string{"test1", "test2"}, 55 | exp: "test1,test2", 56 | }, 57 | } 58 | for name, test := range tests { 59 | t.Run(name, func(t *testing.T) { 60 | var ssf StringSliceFlag 61 | for _, v := range test.values { 62 | if err := ssf.Set(v); err != nil { 63 | t.Fatalf("unexpected error: %v", err) 64 | } 65 | } 66 | if exp, got := test.exp, ssf.String(); exp != got { 67 | t.Fatalf("expected %q, got %q", exp, got) 68 | } 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /cmd/exporter/web/web.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | const homepageTemplate = ` 9 | 10 | Cloudcost Exporter 11 | 12 |

Cloudcost Exporter

13 |

Metrics

14 | 15 | ` 16 | 17 | func HomePageHandler(metricsPath string) func(w http.ResponseWriter, r *http.Request) { 18 | return func(w http.ResponseWriter, r *http.Request) { 19 | if r.URL.Path == "/" { 20 | fmt.Fprintf(w, homepageTemplate, metricsPath) 21 | } else { 22 | http.NotFound(w, r) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cmd/exporter/web/web_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestLandingPage(t *testing.T) { 12 | tests := map[string]struct { 13 | reqMethod string 14 | reqPath string 15 | expectedResCode int 16 | expectedResTexts []string 17 | }{ 18 | "simple get": {reqMethod: "GET", reqPath: "/", expectedResCode: 200, expectedResTexts: []string{"", "Cloudcost Exporter", "href=\"/metrics\"", ""}}, 19 | "get bad path": {reqMethod: "GET", reqPath: "/asdf", expectedResCode: 404, expectedResTexts: []string{"not found"}}, 20 | } 21 | 22 | handler := http.HandlerFunc(HomePageHandler("/metrics")) 23 | 24 | for name, test := range tests { 25 | t.Run(name, func(t *testing.T) { 26 | req, _ := http.NewRequest(test.reqMethod, test.reqPath, nil) 27 | resRecorder := httptest.NewRecorder() 28 | 29 | handler.ServeHTTP(resRecorder, req) 30 | gotStatus := resRecorder.Code 31 | resBody := resRecorder.Body.String() 32 | 33 | assert.Equalf(t, test.expectedResCode, gotStatus, "Wrong status code! Expected: %v, got: %v", test.expectedResCode, gotStatus) 34 | for _, expected := range test.expectedResTexts { 35 | assert.Containsf(t, resBody, expected, "Response body does not contain expected text: %v", expected) 36 | } 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /docs/contribute/creating-a-new-module.md: -------------------------------------------------------------------------------- 1 | # Creating a New Module 2 | 3 | This document outlines the process for creating a new module for the Cloud Cost Exporter. 4 | The current architecture of the exporter is designed to be modular and extensible. 5 | 6 | Steps: 7 | 1. Create a new module in the `pkg/${CLOUD_SERVICE_PROVIDER}/${MODULE_NAME}` directory 8 | 1. For example: `pkg/aws/eks/eks.go` 9 | 1. Implement the `Collector` [interface](https://github.com/grafana/cloudcost-exporter/blob/main/pkg/provider/provider.go) in the new module 10 | 1. Create a `PricingMap` for the new module 11 | 1. `PricingMap` should be a `map[string]Pricing` where key is the region and Pricing is the cost of the resource in that region 12 | 2. Gather pricing information from the cloud provider's pricing API 13 | 3. For example: `pkg/aws/eks/pricing.go` 14 | 3. Implement a cache for the `PricingMap` since pricing typically stays static for ~24 hours 15 | 1. Implement a `List{Resource}` function that will list all resources of the type 16 | 1. For example: `pkg/aws/eks/instances.go` 17 | 2. The function should return a list of `Resource` structs 18 | 3. Implement a `GetCost` function that will calculate the cost of the resource 19 | 1. For example: `pkg/aws/eks/cost.go` 20 | 2. The function should return the cost of the resource 21 | 3. Implement a `GetLabels` function that will return the labels for the resource 22 | 1. For example: `pkg/aws/eks/labels.go` 23 | 2. The function should return the labels for the resource 24 | 25 | -------------------------------------------------------------------------------- /docs/contribute/developer-guide.md: -------------------------------------------------------------------------------- 1 | # Developer Guide 2 | 3 | This guide will help you get started with the development environment and how to contribute to the project. 4 | 5 | ## External Dependencies 6 | 7 | There are two tools that we use to generate mocks for testing that need to be installed independently: 8 | 1. [mockery](https://vektra.github.io/mockery/latest/installation/): See the note about _not_ using go tools to install 9 | 2. [mockgen](https://github.com/uber-go/mock?tab=readme-ov-file#installation) 10 | 11 | Use the latest version available for both. 12 | The tools are used by `make generate-mocks` in the [Makefile](https://github.com/grafana/cloudcost-exporter/blob/66b83baacf9ad4408f0ad7c7b1738ac3b2c179b2/Makefile#L28) 13 | 14 | ## Running Locally 15 | 16 | Prior to running the exporter, you will need to ensure you have the appropriate credentials for the cloud provider you are trying to export data for. 17 | - AWS 18 | - `aws sso login --profile $AWS_PROFILE` 19 | - GCP 20 | - `gcloud auth application-default login` 21 | - Azure 22 | - `az login` 23 | 24 | 25 | > [!WARNING] 26 | > AWS costexplorer costs $0.01 _per_ request! 27 | > The default settings will keep it to 1 request per hour. 28 | > Each restart of the exporter will trigger a new request. 29 | 30 | ```shell 31 | # Usage 32 | go run cmd/exporter/exporter.go --help 33 | 34 | # GCP 35 | go run cmd/exporter/exporter.go -provider gcp -project-id=$GCP_PROJECT_ID 36 | 37 | # GCP - with custom bucket projects 38 | 39 | go run cmd/exporter/exporter.go -provider gcp -project-id=$GCP_PROJECT_ID -gcp.bucket-projects=$GPC_PROJECT_ID -gcp.bucket-projects=$GPC_PROJECT_ID 40 | 41 | # AWS - Prod 42 | go run cmd/exporter/exporter.go -provider aws -aws.profile $AWS_PROFILE 43 | 44 | # Azure 45 | go run cmd/exporter/exporter.go -provider azure -azure.subscription-id $AZ_SUBSCRIPTION_ID 46 | ``` 47 | 48 | ## Project Structure 49 | 50 | The main entrypoint for the cloudcost exporter is [exporter.go](../../cmd/exporter/exporter.go). 51 | When running the application, there is a flag that is used to determine which cloud service provider(csp) to use. 52 | `cloudcost-exporter` currently supports three csp's: 53 | - `gcp` 54 | - `aws` 55 | - `azure` 56 | 57 | Each csp has an entrypoint in `./pkg/{aws,azure,gcp}/{aws,azure,gcp}.go` that is responsible for initializing the provider and a set of collectors. 58 | A collector is a modules for a single CSP that collects cost data for a specific service and emits the data as a set of Prometheus metrics. 59 | A provider can run multiple collectors at once. 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /docs/contribute/logging.md: -------------------------------------------------------------------------------- 1 | # Logging 2 | 3 | `cloudcost-exporter` codebase has grown to the point where we need to have some form of structured logging. 4 | The initial commit was introduced `db82ae9ccdfddc010492f4739724c8e67ef40851` with a fairly detailed messaged on the requirements and the approach. 5 | The very short is we needed: 6 | 7 | 1. Structured logging with consistent labels across providers and collectors 8 | 2. Ability to define a log level and runtime that limits the logs emitted 9 | 10 | With `slog` being part of Go's stdlib since 1.21, we decided to use it as the logging library with a wrapper so that we can get log levels as well. 11 | 12 | ## Guidelines 13 | 14 | 1. Every provider _must_ accept a `*slog.Logger` in the constructor 15 | 1. Every collector _must_ accept a `*slog.Logger` in the constructor 16 | 1. Each provider and collector _must_ add a `provider` or `collector` group when initializing using the [slog.Logger.With](https://pkg.go.dev/golang.org/x/exp/slog#Logger.With) method, specifying the collector or provider used 17 | 1. Always prefer to use the `logger.WithAttr(...)` method to add structured data to the log message for both performance and consistency(see [slog blog post](https://go.dev/blog/slog) and search Performance section for more information) 18 | - NB: If you do not have any additional fields to log out, then you can use the `logger.Info("message")` methods 19 | 20 | ## Expanding the logging 21 | 22 | If you need more flexibility or need to expand the logger, please file an [issue](https://github.com/grafana/cloudcost-exporter/issues/new) with the requirements, and we can discuss the best way to implement it. 23 | 24 | -------------------------------------------------------------------------------- /docs/contribute/releases.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | ## Building images 4 | 5 | There is a [github workflow](../../.github/workflows/docker.yml) that publishes images on changes to `main` and new tags. 6 | 7 | ## Versioning 8 | 9 | We follow [semver](https://semver.org/) and generate new tags manually. 10 | 11 | To cut a release, clone `cloudcost-exporter` locally and pull latest from main. 12 | Determine if the next release is a major, minor, or hotfix. 13 | Then increment the relevant version label. 14 | 15 | For instance, let's say we're on `v0.2.2` and determined the next release is a minor change. 16 | The next version would then be `v0.3.0`. 17 | Execute the following command to generate the tag and push it: 18 | 19 | ```sh 20 | git tag v0.3.0 21 | # Optionally, add a message on why the specific version label was updated: git tag v0.3.0 -m "Adds liveness probes with backwards compatibility" 22 | git push origin tag v0.3.0 23 | ``` 24 | 25 | ## Releases 26 | 27 | Creating and pushing a new tag will trigger the `goreleaser` workflow in [./.github/workflows/release.yml](https://github.com/grafana/cloudcost-exporter/tree/main/.github/workflows/release.yml). 28 | 29 | The configuration for `goreleaser` itself can be found in [./.goreleaser.yaml](https://github.com/grafana/cloudcost-exporter/blob/main/.goreleaser.yaml). 30 | 31 | See https://github.com/grafana/cloudcost-exporter/issues/18 for progress on our path to automating releases. 32 | 33 | ## GitHub Actions 34 | 35 | When adding or upgrading a GitHub Actions `actions`, please set the full length commit SHA instead of the version: 36 | 37 | ``` 38 | jobs: 39 | myjob: 40 | runs-on: ubuntu-latest 41 | steps: 42 | - uses: foo/baraction@abcdef1234567890abcdef1234567890abcdef12 # v1.2.3 43 | ``` 44 | 45 | Granular control of the version helps with security since commit SHAs are immutable. 46 | 47 | ## Helm chart 48 | 49 | The `cloudcost-exporter`'s Helm chart can be found here: https://github.com/grafana/helm-charts/tree/main/charts/cloudcost-exporter 50 | 51 | ### Helm chart release process 52 | 53 | If making changes to the Chart template/values (optional): 54 | 1. Make changes to the Helm chart [templates](https://github.com/grafana/helm-charts/tree/main/charts/cloudcost-exporter/templates/) if needed 55 | 1. Update the [values.yaml](https://github.com/grafana/helm-charts/tree/main/charts/cloudcost-exporter/values.yaml) if needed 56 | 57 | Once changes have been made to the Chart itself (see above) and/or **there is a new release of cloudcost-exporter** (required): 58 | 1. Update the [Chart.yaml](https://github.com/grafana/helm-charts/tree/main/charts/cloudcost-exporter/Chart.yaml): 59 | * Make sure that the `appVersion` matches the new cloudcost-explorer release version 60 | * Bump the Helm chart's `version` too 61 | 1. [Generate the Helm chart's README](https://github.com/grafana/helm-charts/blob/main/CONTRIBUTING.md#generate-readme) 62 | -------------------------------------------------------------------------------- /docs/deploying/aws/README.md: -------------------------------------------------------------------------------- 1 | # AWS CloudCost Exporter Deployment 2 | 3 | ## Setup the required IRSA authentication 4 | 5 | ### 1. Setup the primary IAM role 6 | 7 | cloudcost-exporter uses [AWS SDK for Go V2](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/getting-started.html) 8 | and supports providing authentication via the [AWS SDK's default credential provider chain](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). 9 | This means that the CloudCost Exporter can be deployed on an EC2 instance, ECS, EKS, or any other AWS service that supports IAM roles for service accounts. 10 | 11 | First, create an IAM Policy the minimum required permissions for cloudcost-exporter to work on AWS, an example of which can be found in [./permissions-policy.json](./permissions-policy.json). 12 | 13 | Next, [create and associate an IAM role for the cloudcost-exporter Service Account](https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html). 14 | 15 | The role ARN will be passed as an [annotation to the Service Account](#serviceaccountannotations-required) in the values file of the Helm chart. 16 | 17 | The role's trust policy should look like this: [./role-trust-policy.json](./role-trust-policy.json). 18 | 19 | >[!IMPORTANT] 20 | > In the STS `Condition` element of the JSON policy, make sure that the Service Account name matches exactly the Service Account name that is deployed. 21 | > For example, this could be as simple as `"system:serviceaccount:cloudcost-exporter:cloudcost-exporter"`. 22 | > If using Helm to create a release such as `my-release`, the Service Account name should be updated to `"system:serviceaccount:cloudcost-exporter:my-release-cloudcost-exporter"`. 23 | 24 | ### 2. Configure the Helm chart 25 | 26 | The Helm chart can be deployed after creating the necessary role and policy described above in [Authentication](#authentication). 27 | 28 | An example values file with the additional AWS-specific values is provided here: https://github.com/grafana/helm-charts/blob/main/charts/cloudcost-exporter/values.aws.yaml 29 | 30 | The AWS-specific values can also be set like this: 31 | ```console 32 | helm install my-release grafana/cloudcost-exporter \ 33 | --set 'containerArgs[0]=--provider=aws' \ 34 | --set 'containerArgs[1]=--aws.region=us-east-1' \ 35 | --set 'containerArgs[2]=--aws.services=s3\,ec2' \ 36 | --set-string serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::123456789012:role/CloudCostExporterRole" \ 37 | --namespace cloudcost-exporter --create-namespace 38 | ``` 39 | 40 | ### `containerArgs` (required) 41 | 42 | Set AWS as the provider: 43 | ``` 44 | - "--provider=aws" 45 | ``` 46 | 47 | Set the region that the client should authenticate with. 48 | Check which region is used by the [AWS IAM endpoint](https://docs.aws.amazon.com/general/latest/gr/iam-service.html). 49 | This is often a global endpoint so only needs to be set to one region. 50 | ``` 51 | - "--aws.region=us-east-1" 52 | ``` 53 | 54 | Set which AWS service to use: 55 | ``` 56 | - "--aws.services=s3,ec2" 57 | ``` 58 | 59 | ### `serviceAccount.annotations` (required) 60 | 61 | Annotate the `serviceAccount` with the ARN of the role created above. 62 | This should look like the following: 63 | 64 | ```yaml 65 | serviceAccount: 66 | annotations: 67 | eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/CloudCostExporterRole 68 | ``` 69 | 70 | ## Access resources in a separate account 71 | 72 | There are two roles that cloudcost-exporter can use: the first one is the IRSA role, which is required. 73 | The setup for this is described in the section [Setup the required IRSA authentication](#setup-the-required-irsa-authentication). 74 | This is the role ARN that is added as a tag to the Service Account. 75 | This authenticates cloudcost-exporter running in the cluster with the cluster's AWS account via OIDC. 76 | 77 | cloudcost-exporter is also able to pull metrics from an account that is different to the account where the cluster is running. 78 | This feature is **optional**. 79 | 80 | The authentication setup involves the following (see AWS docs: [Cross-account access using roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies-cross-account-resource-access.html#access_policies-cross-account-using-roles)): 81 | * Set up the IRSA role in the cluster account 82 | * Set up the role to be assumed in the account where resources will be accessed 83 | 84 | ![Cross-account IAM access diagram](./cross-account-access-diagram.png "Cross-account IAM access diagram") 85 | 86 | Once the roles are setup, set the role ARN of the role to be assumed through the following flag: `--aws.roleARN=` 87 | 88 | This can be added to the Helm configuration as an additional [containerArgs](#containerargs-required). 89 | 90 | ## Troubleshooting 91 | 92 | ### Issue with Service Account authentication 93 | 94 | The following logs suggest that there is an error with the Service Account setup. Please double-check the docs above. Feel free to open an issue in the repository if you encounter any issues. 95 | 96 | ``` 97 | level=ERROR msg="Error selecting provider" message="error getting regions: operation error EC2: DescribeRegions, get identity: get credentials: failed to refresh cached credentials, failed to retrieve credentials, operation error STS: AssumeRoleWithWebIdentity, https response error StatusCode: 403, RequestID: , api error AccessDenied: Not authorized to perform sts:AssumeRoleWithWebIdentity" provider=aws 98 | ``` 99 | -------------------------------------------------------------------------------- /docs/deploying/aws/cross-account-access-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafana/cloudcost-exporter/844c955840dfdab6ced16d7c7360a249091de02a/docs/deploying/aws/cross-account-access-diagram.png -------------------------------------------------------------------------------- /docs/deploying/aws/permissions-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "ce:*", 8 | "ec2:DescribeRegions", 9 | "ec2:DescribeInstances", 10 | "ec2:DescribeSpotPriceHistory", 11 | "ec2:DescribeVolumes", 12 | "pricing:GetProducts" 13 | ], 14 | "Resource": "*" 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /docs/deploying/aws/role-trust-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Principal": { 7 | "Federated": "arn:aws:iam::123456789012:oidc-provider/oidc.eks..amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE" 8 | }, 9 | "Action": "sts:AssumeRoleWithWebIdentity", 10 | "Condition": { 11 | "StringEquals": { 12 | "oidc.eks..amazonaws.com/id/EXAMPLED539D4633E53DE1B71EXAMPLE:sub": "system:serviceaccount:cloudcost-exporter:cloudcost-exporter" 13 | } 14 | } 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /docs/metrics/aws/ec2.md: -------------------------------------------------------------------------------- 1 | # EKS compute Metrics 2 | 3 | | Metric name | Metric type | Description | Labels | 4 | |----------------------------------------------------|-------------|----------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | cloudcost_aws_ec2_instance_cpu_usd_per_core_hour | Gauge | The processing cost of a EC2 Compute Instance in USD/(core*h) | `cluster_name`=<name of the cluster the instance is associated with, if it exists. Can be empty>
`instance`=<name of the compute instance>
`instance_id`=<The unique id associated with the compute instance>
`region`=<AWS region code>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: m7a.large>
`price_tier`=<spot\|ondemand> `architecture`=<arm64\|x86_64 > | 6 | | cloudcost_aws_ec2_instance_memory_usd_per_gib_hour | Gauge | The memory cost of a EC2 Compute Instance in USD/(GiB*h) | `cluster_name`=<name of the cluster the instance is associated with, if it exists. Can be empty>
`instance`=<name of the compute instance>
`instance_id`=<The unique id associated with the compute instance>
`region`=<AWS region code>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: m7a.large>
`price_tier`=<spot\|ondemand> `architecture`=<arm64\|x86_64 > | 7 | | cloudcost_aws_ec2_instance_total_usd_per_hour | Gauge | The total cost of an EC2 Compute Instance in USD/*h) | `cluster_name`=<name of the cluster the instance is associated with, if it exists. Can be empty>
`instance`=<name of the compute instance>
`instance_id`=<The unique id associated with the compute instance>
`region`=<AWS region code>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: m7a.large>
`price_tier`=<spot\|ondemand> `architecture`=<arm64\|x86_64 > | 8 | 9 | ## EBS Metrics 10 | 11 | | Metric name | Metric type | Description | Labels | 12 | |----------------------------------------------------|-------------|----------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 13 | | cloudcost_aws_ec2_persistent_volume_usd_per_hour | Gauge | The cost of an EBS Volume in USD/h | `availability_zone`=<AWS AZ code>
`disk`=<EBS volume ID>
`persistentvolume`=<k8s persistent volume ID>
`region`=<AWS region code>
`size_gib`=<volume size in GiB, can always be parsed to an integer>
`state`=<volume state, eg: available, in-use;
`type`=<volume type, eg: gp2, gp3> | 14 | 15 | 16 | ## Pricing Source 17 | 18 | The pricing data is sourced from the [AWS Pricing API](https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_pricing_GetProducts.html) and is updated every 24 hours. 19 | There are a few assumptions that we're making specific to Grafana Labs: 20 | 1. All costs are in USD 21 | 2. Only consider Linux based instances 22 | 3. `cloudcost-exporter` emits the list price and does not take into account any discounts or savings plans 23 | -------------------------------------------------------------------------------- /docs/metrics/aws/s3.md: -------------------------------------------------------------------------------- 1 | # AWS S3 Metrics 2 | 3 | | Metric name | Metric type | Description | Labels | 4 | |----------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | cloudcost_aws_s3_storage_by_location_usd_per_gibyte_hour | Gauge | Storage cost of S3 objects by region, class, and tier. Cost represented in USD/(GiB*h) | `region`=<AWS region>
`class`=<[AWS S3 storage class](https://aws.amazon.com/s3/storage-classes/)> | 6 | | cloudcost_aws_s3_operation_by_location_usd_per_krequest | Gauge | Operation cost of S3 objects by region, class, and tier. Cost represented in USD/(1k req) | `region`=<AWS region>
`class`=<[AWS S3 storage class](https://aws.amazon.com/s3/storage-classes/)>
`tier`=<[AWS S3 request tier](https://aws.amazon.com/s3/pricing/)> | -------------------------------------------------------------------------------- /docs/metrics/azure/aks.md: -------------------------------------------------------------------------------- 1 | # AKS Metrics 2 | 3 | | Metric Name | Metric Type | Description | Labels | 4 | |------------------------------------------------------|-------------|------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | cloudcost_azure_aks_instance_total_usd_per_hour | Gauge | The total cost of an Azure VM used in an AKS cluster in USD/h | `cluster_name` = <name of the cluster the instance is associated with>
`instance`=<name of the compute instance>
`region`=<Azure region of the compute instance>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: Standard_D4s_v3>
`price_tier`=<spot\|ondemand>
`operating_system`=<Windows\|Linux> | 6 | | cloudcost_azure_aks_instance_cpu_usd_per_core_hour | Gauge | The compute cost of an Azure VM used in an AKS cluster in USD/(core*h) | `cluster_name`=<name of the cluster the instance is associated with>
`instance`=<name of the compute instance>
`region`=<Azure region of the compute instance>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: Standard_D4s_v3>
`price_tier`=<spot\|ondemand>
`operating_system`=<Windows\|Linux> | 7 | | cloudcost_azure_aks_instance_memory_usd_per_gib_hour | Gauge | The memory cost of an Azure VM used in an AKS cluster in USD/(GiB*h) | `cluster_name`=<name of the cluster the instance is associated with>
`instance`=<name of the compute instance>
`region`=<Azure region of the compute instance>
`family`=<broader compute family (General Purpose, Compute Optimized, Memory Optimized, ...) >
`machine_type`=<specific machine type, e.g.: Standard_D4s_v3>
`price_tier`=<spot\|ondemand>
`operating_system`=<Windows\|Linux> | 8 | -------------------------------------------------------------------------------- /docs/metrics/gcp/gcs.md: -------------------------------------------------------------------------------- 1 | # GCP GCS Metrics 2 | 3 | | Metric name | Metric type | Description | Labels | 4 | |--------------------------------------------------------------------|-------------|----------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | cloudcost_gcp_gcs_storage_by_location_usd_per_gibyte_hour | Gauge | Storage cost of GCS objects by location and storage_class. Cost represented in USD/(GiB*h) | `location`=<GCP region>
`storage_class`=<[GCP GCS storage class](https://cloud.google.com/storage/docs/storage-classes)> | 6 | | cloudcost_gcp_gcs_storage_discount_by_location_usd_per_gibyte_hour | Gauge | Discount for storage cost of GCS objects by location and storage_class. Cost represented in USD/(GiB*h) | `location`=<GCP region>
`storage_class`=<[GCP GCS storage class](https://cloud.google.com/storage/docs/storage-classes)> | 7 | | cloudcost_gcp_gcs_operation_by_location_usd_per_krequest | Gauge | Operation cost of GCS objects by location, storage_class, and opclass. Cost represented in USD/(1k req) | `location`=<GCP region>
`storage_class`=<[GCP GCS storage class](https://cloud.google.com/storage/docs/storage-classes)>
`opclass`=<[GCP GCS request operations](https://cloud.google.com/storage/pricing#process-pricing)> | 8 | | cloudcost_gcp_gcs_operation_discount_by_location_usd_per_krequest | Gauge | Discount for operation cost of GCS objects by location, storage_class, and opclass. Cost represented in USD/(1k req) | `location`=<GCP region>
`storage_class`=<[GCP GCS storage class](https://cloud.google.com/storage/docs/storage-classes)>
`opclass`=<[GCP GCS request operations](https://cloud.google.com/storage/pricing#process-pricing)> | 9 | | cloudcost_gcp_gcs_bucket_info | Gauge | Location, location_type and storage class information for a GCS object by bucket_name | `location`=<GCP region>
`location_type`=<multi-region\|region\|dual-region>
`storage_class`=<[GCP GCS storage class](https://cloud.google.com/storage/docs/storage-classes)>
`bucket_name`=<name of the bucket> | -------------------------------------------------------------------------------- /docs/metrics/gcp/gke.md: -------------------------------------------------------------------------------- 1 | # GKE Compute Metrics 2 | 3 | | Metric name | Metric type | Description | Labels | 4 | |------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 5 | | cloudcost_gcp_gke_instance_cpu_usd_per_core_hour | Gauge | The processing cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(core*h) | `cluster_name`=<name of the cluster the instance is associated with>
`instance`=<name of the compute instance>
`region`=<GCP region code>
`family`=<broader compute family (n1, n2, c3 ...) >
`machine_type`=<specific machine type, e.g.: n2-standard-2>
`project`=<GCP project, where the instance is provisioned>
`price_tier`=<spot\|ondemand> | 6 | | cloudcost_gcp_gke_compute_instance_memory_usd_per_gib_hour | Gauge | The memory cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(GiB*h) | `cluster_name`=<name of the cluster the instance is associated with>
`instance`=<name of the compute instance>
`region`=<GCP region code>
`family`=<broader compute family (n1, n2, c3 ...) >
`machine_type`=<specific machine type, e.g.: n2-standard-2>
`project`=<GCP project, where the instance is provisioned>
`price_tier`=<spot\|ondemand> | 7 | | cloudcost_gcp_gke_persistent_volume_usd_per_hour | Gauge | The cost of a GKE Persistent Volume in USD/(GiB*h) | `cluster_name`=<name of the cluster the instance is associated with>
`namespace`=<The namespace the pvc was created for>
`persistentvolume`=<Name of the persistent volume>
`region`=<The region the pvc was created in>
`project`=<GCP project, where the instance is provisioned>
`storage_class`=<pd-standard\|pd-ssd\|pd-balanced\|pd-extreme>
`disk_type`=<boot_disk\|persistent_volume>
`use_status`=<in-use\|idle> | 8 | 9 | ## Persistent Volumes 10 | 11 | There's two sources of data for persistent volumes: 12 | - Skus from the Billing API 13 | - Disk metadata from compute API 14 | 15 | There's a bit of a disconnect between the two. 16 | Sku's descriptions have the following format: 17 | ``` 18 | Balanced PD Capacity in 19 | Commitment v1: Local SSD In 20 | Extreme PD Capacity in 21 | Extreme PD IOPS in 22 | Hyperdisk Balanced Capacity in 23 | Hyperdisk Balanced IOPS in 24 | Hyperdisk Balanced Throughput in 25 | Hyperdisk Extreme Capacity in 26 | Hyperdisk Extreme IOPS in 27 | Hyperdisk Throughput Capacity in 28 | Hyperdisk Throughput Throughput Capacity in 29 | Regional Balanced PD Capacity in 30 | Regional SSD backed PD Capacity in 31 | Regional Storage PD Capacity in 32 | SSD backed Local Storage attached to Spot Preemptible VM in 33 | SSD backed Local Storage in 35 | Storage PD Capacity in 36 | ``` 37 | 38 | Generically, the sku descriptions have the following format: 39 | ``` 40 | PD Capacity in 41 | ``` 42 | 43 | Disk metadata has the following format: 44 | ``` 45 | projects//zones//disks/ 46 | ``` 47 | 48 | To map the sku to the disk type, we can use the following mapping: 49 | 50 | - Storage PD Capacity -> pd-standard 51 | - SSD backed PD Capacity -> pd-ssd 52 | - Balanced PD Capacity -> pd-balanced 53 | - Extreme PD Capacity -> pd-extreme 54 | - Hyperdisk Balanced -> hyperdisk-balanced 55 | 56 | > [!WARNING] 57 | > The following storage classes are experimental 58 | 59 | ## Experimental Storage Costs 60 | 61 | Cloudcost Exporter needs to support the following hyperdisk pricing dimensions: 62 | - [x] provisioned space 63 | - [ ] Network throughput 64 | - [ ] IOps 65 | - [ ] high availability 66 | 67 | [#344](https://github.com/grafana/cloudcost-exporter/pull/344) introduced experimental support for provisioned space for [hyperdisk class](https://cloud.google.com/compute/disks-image-pricing#persistentdisk) 68 | -------------------------------------------------------------------------------- /docs/metrics/providers.md: -------------------------------------------------------------------------------- 1 | # Provider Metrics 2 | 3 | Baseline metrics that are exported when you run `cloudcost-exporter` with any of the supported providers. 4 | These metrics are meant to be generic enough to create an operational dashboard and alerting rules for the exporter. 5 | Each provider _must_ implement these metrics. 6 | 7 | | Metric name | Metric type | Description | Labels | 8 | |---------------------------------------------------------------|-------------|-----------------------------------------------|-----------------------------------------------------------------------------------------------| 9 | | cloudcost_exporter_scrapes_total | Counter | Total number of scrapes for the gcp provider. | `provider`=<name of the provider>
| 10 | | cloudcost_exporter_last_scrape_error | Gauge | Was the last scrape an error. 1 is an error. | `provider`=<name of the provider>
| 11 | | cloudcost_exporter_last_scrape_duration_seconds | Gauge | Duration of the last scrape in seconds. | `provider`=<name of the provider>
| 12 | | cloudcost_exporter_collector_scrapes_total | Counter | Total number of scrapes, by collector. | `provider`=<name of the provider>
`collector`=<name of the collector>
| 13 | | cloudcost_exporter_collector_last_scrape_duration_seconds | Gauge | Duration of the last scrape in seconds. | `provider`=<name of the provider>
`collector`=<name of the collector>
| 14 | | cloudcost_exporter_collector_last_scrape_error | Gauge | Was the last scrape an error. 1 is an error. | `provider`=<name of the provider>
`collector`=<name of the collector>
| 15 | 16 | 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/grafana/cloudcost-exporter 2 | 3 | go 1.24 4 | 5 | require ( 6 | cloud.google.com/go/billing v1.20.2 7 | cloud.google.com/go/compute v1.38.0 8 | cloud.google.com/go/storage v1.52.0 9 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.2 10 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 11 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 12 | github.com/Azure/go-autorest/autorest/to v0.4.1 13 | github.com/aws/aws-sdk-go-v2 v1.36.3 14 | github.com/aws/aws-sdk-go-v2/config v1.29.8 15 | github.com/aws/aws-sdk-go-v2/credentials v1.17.61 16 | github.com/aws/aws-sdk-go-v2/service/costexplorer v1.49.0 17 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.217.0 18 | github.com/aws/aws-sdk-go-v2/service/pricing v1.34.3 19 | github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 20 | github.com/google/go-cmp v0.7.0 21 | github.com/googleapis/gax-go/v2 v2.14.1 22 | github.com/grafana/grafana-foundation-sdk/go v0.0.0-20241213004919-4516aa9d3732 23 | github.com/prometheus/client_golang v1.21.1 24 | github.com/prometheus/client_model v0.6.1 25 | github.com/prometheus/common v0.63.0 26 | github.com/stretchr/testify v1.10.0 27 | go.uber.org/mock v0.5.0 28 | golang.org/x/sync v0.14.0 29 | gomodules.xyz/azure-retail-prices-sdk-for-go v0.0.3 30 | google.golang.org/api v0.232.0 31 | google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb 32 | google.golang.org/grpc v1.72.0 33 | gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e 34 | ) 35 | 36 | require ( 37 | cel.dev/expr v0.20.0 // indirect 38 | cloud.google.com/go v0.121.0 // indirect 39 | cloud.google.com/go/auth v0.16.1 // indirect 40 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 41 | cloud.google.com/go/compute/metadata v0.6.0 // indirect 42 | cloud.google.com/go/iam v1.5.0 // indirect 43 | cloud.google.com/go/monitoring v1.24.0 // indirect 44 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect 45 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect 46 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect 47 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 48 | github.com/AzureAD/microsoft-authentication-library-for-go v1.3.3 // indirect 49 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect 50 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect 51 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect 52 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect 53 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect 54 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect 55 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect 56 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect 57 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect 58 | github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 // indirect 59 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 // indirect 60 | github.com/aws/smithy-go v1.22.2 // indirect 61 | github.com/beorn7/perks v1.0.1 // indirect 62 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 63 | github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect 64 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 65 | github.com/davecgh/go-spew v1.1.1 // indirect 66 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 67 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 68 | github.com/felixge/httpsnoop v1.0.4 // indirect 69 | github.com/go-jose/go-jose/v4 v4.0.4 // indirect 70 | github.com/go-logr/logr v1.4.2 // indirect 71 | github.com/go-logr/stdr v1.2.2 // indirect 72 | github.com/golang-jwt/jwt/v5 v5.2.1 // indirect 73 | github.com/google/s2a-go v0.1.9 // indirect 74 | github.com/google/uuid v1.6.0 // indirect 75 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect 76 | github.com/klauspost/compress v1.17.11 // indirect 77 | github.com/kylelemons/godebug v1.1.0 // indirect 78 | github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect 79 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 80 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 81 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 82 | github.com/pmezard/go-difflib v1.0.0 // indirect 83 | github.com/prometheus/procfs v0.15.1 // indirect 84 | github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect 85 | github.com/stretchr/objx v0.5.2 // indirect 86 | github.com/zeebo/errs v1.4.0 // indirect 87 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 88 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect 89 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect 90 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 91 | go.opentelemetry.io/otel v1.35.0 // indirect 92 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 93 | go.opentelemetry.io/otel/sdk v1.35.0 // indirect 94 | go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect 95 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 96 | golang.org/x/crypto v0.37.0 // indirect 97 | golang.org/x/net v0.39.0 // indirect 98 | golang.org/x/oauth2 v0.30.0 // indirect 99 | golang.org/x/sys v0.32.0 // indirect 100 | golang.org/x/text v0.24.0 // indirect 101 | golang.org/x/time v0.11.0 // indirect 102 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect 103 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 // indirect 104 | google.golang.org/protobuf v1.36.6 // indirect 105 | gopkg.in/yaml.v3 v3.0.1 // indirect 106 | ) 107 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package cloudcost_exporter 2 | 3 | // Exporting a mostly empty main.go file is required for mockery to work: https://vektra.github.io/mockery/v2.38/notes/#error-no-go-files-found-in-root-search-path 4 | 5 | const ( 6 | ExporterName = "cloudcost_exporter" 7 | MetricPrefix = "cloudcost" 8 | ) 9 | -------------------------------------------------------------------------------- /mocks/pkg/aws/services/costexplorer/CostExplorer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package costexplorer 4 | 5 | import ( 6 | context "context" 7 | 8 | servicecostexplorer "github.com/aws/aws-sdk-go-v2/service/costexplorer" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // CostExplorer is an autogenerated mock type for the CostExplorer type 13 | type CostExplorer struct { 14 | mock.Mock 15 | } 16 | 17 | type CostExplorer_Expecter struct { 18 | mock *mock.Mock 19 | } 20 | 21 | func (_m *CostExplorer) EXPECT() *CostExplorer_Expecter { 22 | return &CostExplorer_Expecter{mock: &_m.Mock} 23 | } 24 | 25 | // GetCostAndUsage provides a mock function with given fields: ctx, params, optFns 26 | func (_m *CostExplorer) GetCostAndUsage(ctx context.Context, params *servicecostexplorer.GetCostAndUsageInput, optFns ...func(*servicecostexplorer.Options)) (*servicecostexplorer.GetCostAndUsageOutput, error) { 27 | _va := make([]interface{}, len(optFns)) 28 | for _i := range optFns { 29 | _va[_i] = optFns[_i] 30 | } 31 | var _ca []interface{} 32 | _ca = append(_ca, ctx, params) 33 | _ca = append(_ca, _va...) 34 | ret := _m.Called(_ca...) 35 | 36 | if len(ret) == 0 { 37 | panic("no return value specified for GetCostAndUsage") 38 | } 39 | 40 | var r0 *servicecostexplorer.GetCostAndUsageOutput 41 | var r1 error 42 | if rf, ok := ret.Get(0).(func(context.Context, *servicecostexplorer.GetCostAndUsageInput, ...func(*servicecostexplorer.Options)) (*servicecostexplorer.GetCostAndUsageOutput, error)); ok { 43 | return rf(ctx, params, optFns...) 44 | } 45 | if rf, ok := ret.Get(0).(func(context.Context, *servicecostexplorer.GetCostAndUsageInput, ...func(*servicecostexplorer.Options)) *servicecostexplorer.GetCostAndUsageOutput); ok { 46 | r0 = rf(ctx, params, optFns...) 47 | } else { 48 | if ret.Get(0) != nil { 49 | r0 = ret.Get(0).(*servicecostexplorer.GetCostAndUsageOutput) 50 | } 51 | } 52 | 53 | if rf, ok := ret.Get(1).(func(context.Context, *servicecostexplorer.GetCostAndUsageInput, ...func(*servicecostexplorer.Options)) error); ok { 54 | r1 = rf(ctx, params, optFns...) 55 | } else { 56 | r1 = ret.Error(1) 57 | } 58 | 59 | return r0, r1 60 | } 61 | 62 | // CostExplorer_GetCostAndUsage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCostAndUsage' 63 | type CostExplorer_GetCostAndUsage_Call struct { 64 | *mock.Call 65 | } 66 | 67 | // GetCostAndUsage is a helper method to define mock.On call 68 | // - ctx context.Context 69 | // - params *servicecostexplorer.GetCostAndUsageInput 70 | // - optFns ...func(*servicecostexplorer.Options) 71 | func (_e *CostExplorer_Expecter) GetCostAndUsage(ctx interface{}, params interface{}, optFns ...interface{}) *CostExplorer_GetCostAndUsage_Call { 72 | return &CostExplorer_GetCostAndUsage_Call{Call: _e.mock.On("GetCostAndUsage", 73 | append([]interface{}{ctx, params}, optFns...)...)} 74 | } 75 | 76 | func (_c *CostExplorer_GetCostAndUsage_Call) Run(run func(ctx context.Context, params *servicecostexplorer.GetCostAndUsageInput, optFns ...func(*servicecostexplorer.Options))) *CostExplorer_GetCostAndUsage_Call { 77 | _c.Call.Run(func(args mock.Arguments) { 78 | variadicArgs := make([]func(*servicecostexplorer.Options), len(args)-2) 79 | for i, a := range args[2:] { 80 | if a != nil { 81 | variadicArgs[i] = a.(func(*servicecostexplorer.Options)) 82 | } 83 | } 84 | run(args[0].(context.Context), args[1].(*servicecostexplorer.GetCostAndUsageInput), variadicArgs...) 85 | }) 86 | return _c 87 | } 88 | 89 | func (_c *CostExplorer_GetCostAndUsage_Call) Return(_a0 *servicecostexplorer.GetCostAndUsageOutput, _a1 error) *CostExplorer_GetCostAndUsage_Call { 90 | _c.Call.Return(_a0, _a1) 91 | return _c 92 | } 93 | 94 | func (_c *CostExplorer_GetCostAndUsage_Call) RunAndReturn(run func(context.Context, *servicecostexplorer.GetCostAndUsageInput, ...func(*servicecostexplorer.Options)) (*servicecostexplorer.GetCostAndUsageOutput, error)) *CostExplorer_GetCostAndUsage_Call { 95 | _c.Call.Return(run) 96 | return _c 97 | } 98 | 99 | // NewCostExplorer creates a new instance of CostExplorer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 100 | // The first argument is typically a *testing.T value. 101 | func NewCostExplorer(t interface { 102 | mock.TestingT 103 | Cleanup(func()) 104 | }) *CostExplorer { 105 | mock := &CostExplorer{} 106 | mock.Mock.Test(t) 107 | 108 | t.Cleanup(func() { mock.AssertExpectations(t) }) 109 | 110 | return mock 111 | } 112 | -------------------------------------------------------------------------------- /mocks/pkg/aws/services/pricing/Pricing.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package pricing 4 | 5 | import ( 6 | context "context" 7 | 8 | servicepricing "github.com/aws/aws-sdk-go-v2/service/pricing" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // Pricing is an autogenerated mock type for the Pricing type 13 | type Pricing struct { 14 | mock.Mock 15 | } 16 | 17 | type Pricing_Expecter struct { 18 | mock *mock.Mock 19 | } 20 | 21 | func (_m *Pricing) EXPECT() *Pricing_Expecter { 22 | return &Pricing_Expecter{mock: &_m.Mock} 23 | } 24 | 25 | // GetProducts provides a mock function with given fields: ctx, params, optFns 26 | func (_m *Pricing) GetProducts(ctx context.Context, params *servicepricing.GetProductsInput, optFns ...func(*servicepricing.Options)) (*servicepricing.GetProductsOutput, error) { 27 | _va := make([]interface{}, len(optFns)) 28 | for _i := range optFns { 29 | _va[_i] = optFns[_i] 30 | } 31 | var _ca []interface{} 32 | _ca = append(_ca, ctx, params) 33 | _ca = append(_ca, _va...) 34 | ret := _m.Called(_ca...) 35 | 36 | if len(ret) == 0 { 37 | panic("no return value specified for GetProducts") 38 | } 39 | 40 | var r0 *servicepricing.GetProductsOutput 41 | var r1 error 42 | if rf, ok := ret.Get(0).(func(context.Context, *servicepricing.GetProductsInput, ...func(*servicepricing.Options)) (*servicepricing.GetProductsOutput, error)); ok { 43 | return rf(ctx, params, optFns...) 44 | } 45 | if rf, ok := ret.Get(0).(func(context.Context, *servicepricing.GetProductsInput, ...func(*servicepricing.Options)) *servicepricing.GetProductsOutput); ok { 46 | r0 = rf(ctx, params, optFns...) 47 | } else { 48 | if ret.Get(0) != nil { 49 | r0 = ret.Get(0).(*servicepricing.GetProductsOutput) 50 | } 51 | } 52 | 53 | if rf, ok := ret.Get(1).(func(context.Context, *servicepricing.GetProductsInput, ...func(*servicepricing.Options)) error); ok { 54 | r1 = rf(ctx, params, optFns...) 55 | } else { 56 | r1 = ret.Error(1) 57 | } 58 | 59 | return r0, r1 60 | } 61 | 62 | // Pricing_GetProducts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProducts' 63 | type Pricing_GetProducts_Call struct { 64 | *mock.Call 65 | } 66 | 67 | // GetProducts is a helper method to define mock.On call 68 | // - ctx context.Context 69 | // - params *servicepricing.GetProductsInput 70 | // - optFns ...func(*servicepricing.Options) 71 | func (_e *Pricing_Expecter) GetProducts(ctx interface{}, params interface{}, optFns ...interface{}) *Pricing_GetProducts_Call { 72 | return &Pricing_GetProducts_Call{Call: _e.mock.On("GetProducts", 73 | append([]interface{}{ctx, params}, optFns...)...)} 74 | } 75 | 76 | func (_c *Pricing_GetProducts_Call) Run(run func(ctx context.Context, params *servicepricing.GetProductsInput, optFns ...func(*servicepricing.Options))) *Pricing_GetProducts_Call { 77 | _c.Call.Run(func(args mock.Arguments) { 78 | variadicArgs := make([]func(*servicepricing.Options), len(args)-2) 79 | for i, a := range args[2:] { 80 | if a != nil { 81 | variadicArgs[i] = a.(func(*servicepricing.Options)) 82 | } 83 | } 84 | run(args[0].(context.Context), args[1].(*servicepricing.GetProductsInput), variadicArgs...) 85 | }) 86 | return _c 87 | } 88 | 89 | func (_c *Pricing_GetProducts_Call) Return(_a0 *servicepricing.GetProductsOutput, _a1 error) *Pricing_GetProducts_Call { 90 | _c.Call.Return(_a0, _a1) 91 | return _c 92 | } 93 | 94 | func (_c *Pricing_GetProducts_Call) RunAndReturn(run func(context.Context, *servicepricing.GetProductsInput, ...func(*servicepricing.Options)) (*servicepricing.GetProductsOutput, error)) *Pricing_GetProducts_Call { 95 | _c.Call.Return(run) 96 | return _c 97 | } 98 | 99 | // NewPricing creates a new instance of Pricing. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 100 | // The first argument is typically a *testing.T value. 101 | func NewPricing(t interface { 102 | mock.TestingT 103 | Cleanup(func()) 104 | }) *Pricing { 105 | mock := &Pricing{} 106 | mock.Mock.Test(t) 107 | 108 | t.Cleanup(func() { mock.AssertExpectations(t) }) 109 | 110 | return mock 111 | } 112 | -------------------------------------------------------------------------------- /mocks/pkg/azure/azureClientWrapper/azureClientWrapper.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: pkg/azure/azureClientWrapper/azureClientWrapper.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -source=pkg/azure/azureClientWrapper/azureClientWrapper.go -destination mocks/pkg/azure/azureClientWrapper/azureClientWrapper.go 7 | // 8 | // Package mock_azureClientWrapper is a generated GoMock package. 9 | package mock_azureClientWrapper 10 | 11 | import ( 12 | context "context" 13 | reflect "reflect" 14 | 15 | armcompute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4" 16 | armcontainerservice "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5" 17 | gomock "go.uber.org/mock/gomock" 18 | sdk "gomodules.xyz/azure-retail-prices-sdk-for-go/sdk" 19 | ) 20 | 21 | // MockAzureClient is a mock of AzureClient interface. 22 | type MockAzureClient struct { 23 | ctrl *gomock.Controller 24 | recorder *MockAzureClientMockRecorder 25 | } 26 | 27 | // MockAzureClientMockRecorder is the mock recorder for MockAzureClient. 28 | type MockAzureClientMockRecorder struct { 29 | mock *MockAzureClient 30 | } 31 | 32 | // NewMockAzureClient creates a new mock instance. 33 | func NewMockAzureClient(ctrl *gomock.Controller) *MockAzureClient { 34 | mock := &MockAzureClient{ctrl: ctrl} 35 | mock.recorder = &MockAzureClientMockRecorder{mock} 36 | return mock 37 | } 38 | 39 | // EXPECT returns an object that allows the caller to indicate expected use. 40 | func (m *MockAzureClient) EXPECT() *MockAzureClientMockRecorder { 41 | return m.recorder 42 | } 43 | 44 | // ListClustersInSubscription mocks base method. 45 | func (m *MockAzureClient) ListClustersInSubscription(arg0 context.Context) ([]*armcontainerservice.ManagedCluster, error) { 46 | m.ctrl.T.Helper() 47 | ret := m.ctrl.Call(m, "ListClustersInSubscription", arg0) 48 | ret0, _ := ret[0].([]*armcontainerservice.ManagedCluster) 49 | ret1, _ := ret[1].(error) 50 | return ret0, ret1 51 | } 52 | 53 | // ListClustersInSubscription indicates an expected call of ListClustersInSubscription. 54 | func (mr *MockAzureClientMockRecorder) ListClustersInSubscription(arg0 any) *gomock.Call { 55 | mr.mock.ctrl.T.Helper() 56 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListClustersInSubscription", reflect.TypeOf((*MockAzureClient)(nil).ListClustersInSubscription), arg0) 57 | } 58 | 59 | // ListMachineTypesByLocation mocks base method. 60 | func (m *MockAzureClient) ListMachineTypesByLocation(arg0 context.Context, arg1 string) ([]*armcompute.VirtualMachineSize, error) { 61 | m.ctrl.T.Helper() 62 | ret := m.ctrl.Call(m, "ListMachineTypesByLocation", arg0, arg1) 63 | ret0, _ := ret[0].([]*armcompute.VirtualMachineSize) 64 | ret1, _ := ret[1].(error) 65 | return ret0, ret1 66 | } 67 | 68 | // ListMachineTypesByLocation indicates an expected call of ListMachineTypesByLocation. 69 | func (mr *MockAzureClientMockRecorder) ListMachineTypesByLocation(arg0, arg1 any) *gomock.Call { 70 | mr.mock.ctrl.T.Helper() 71 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMachineTypesByLocation", reflect.TypeOf((*MockAzureClient)(nil).ListMachineTypesByLocation), arg0, arg1) 72 | } 73 | 74 | // ListPrices mocks base method. 75 | func (m *MockAzureClient) ListPrices(arg0 context.Context, arg1 *sdk.RetailPricesClientListOptions) ([]*sdk.ResourceSKU, error) { 76 | m.ctrl.T.Helper() 77 | ret := m.ctrl.Call(m, "ListPrices", arg0, arg1) 78 | ret0, _ := ret[0].([]*sdk.ResourceSKU) 79 | ret1, _ := ret[1].(error) 80 | return ret0, ret1 81 | } 82 | 83 | // ListPrices indicates an expected call of ListPrices. 84 | func (mr *MockAzureClientMockRecorder) ListPrices(arg0, arg1 any) *gomock.Call { 85 | mr.mock.ctrl.T.Helper() 86 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPrices", reflect.TypeOf((*MockAzureClient)(nil).ListPrices), arg0, arg1) 87 | } 88 | 89 | // ListVirtualMachineScaleSetsFromResourceGroup mocks base method. 90 | func (m *MockAzureClient) ListVirtualMachineScaleSetsFromResourceGroup(arg0 context.Context, arg1 string) ([]*armcompute.VirtualMachineScaleSet, error) { 91 | m.ctrl.T.Helper() 92 | ret := m.ctrl.Call(m, "ListVirtualMachineScaleSetsFromResourceGroup", arg0, arg1) 93 | ret0, _ := ret[0].([]*armcompute.VirtualMachineScaleSet) 94 | ret1, _ := ret[1].(error) 95 | return ret0, ret1 96 | } 97 | 98 | // ListVirtualMachineScaleSetsFromResourceGroup indicates an expected call of ListVirtualMachineScaleSetsFromResourceGroup. 99 | func (mr *MockAzureClientMockRecorder) ListVirtualMachineScaleSetsFromResourceGroup(arg0, arg1 any) *gomock.Call { 100 | mr.mock.ctrl.T.Helper() 101 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVirtualMachineScaleSetsFromResourceGroup", reflect.TypeOf((*MockAzureClient)(nil).ListVirtualMachineScaleSetsFromResourceGroup), arg0, arg1) 102 | } 103 | 104 | // ListVirtualMachineScaleSetsOwnedVms mocks base method. 105 | func (m *MockAzureClient) ListVirtualMachineScaleSetsOwnedVms(arg0 context.Context, arg1, arg2 string) ([]*armcompute.VirtualMachineScaleSetVM, error) { 106 | m.ctrl.T.Helper() 107 | ret := m.ctrl.Call(m, "ListVirtualMachineScaleSetsOwnedVms", arg0, arg1, arg2) 108 | ret0, _ := ret[0].([]*armcompute.VirtualMachineScaleSetVM) 109 | ret1, _ := ret[1].(error) 110 | return ret0, ret1 111 | } 112 | 113 | // ListVirtualMachineScaleSetsOwnedVms indicates an expected call of ListVirtualMachineScaleSetsOwnedVms. 114 | func (mr *MockAzureClientMockRecorder) ListVirtualMachineScaleSetsOwnedVms(arg0, arg1, arg2 any) *gomock.Call { 115 | mr.mock.ctrl.T.Helper() 116 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVirtualMachineScaleSetsOwnedVms", reflect.TypeOf((*MockAzureClient)(nil).ListVirtualMachineScaleSetsOwnedVms), arg0, arg1, arg2) 117 | } 118 | -------------------------------------------------------------------------------- /mocks/pkg/google/gcs/CloudCatalogClient.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.38.0. DO NOT EDIT. 2 | 3 | package gcs 4 | 5 | import ( 6 | billing "cloud.google.com/go/billing/apiv1" 7 | billingpb "cloud.google.com/go/billing/apiv1/billingpb" 8 | 9 | context "context" 10 | 11 | gax "github.com/googleapis/gax-go/v2" 12 | 13 | mock "github.com/stretchr/testify/mock" 14 | ) 15 | 16 | // CloudCatalogClient is an autogenerated mock type for the CloudCatalogClient type 17 | type CloudCatalogClient struct { 18 | mock.Mock 19 | } 20 | 21 | type CloudCatalogClient_Expecter struct { 22 | mock *mock.Mock 23 | } 24 | 25 | func (_m *CloudCatalogClient) EXPECT() *CloudCatalogClient_Expecter { 26 | return &CloudCatalogClient_Expecter{mock: &_m.Mock} 27 | } 28 | 29 | // ListServices provides a mock function with given fields: ctx, req, opts 30 | func (_m *CloudCatalogClient) ListServices(ctx context.Context, req *billingpb.ListServicesRequest, opts ...gax.CallOption) *billing.ServiceIterator { 31 | _va := make([]interface{}, len(opts)) 32 | for _i := range opts { 33 | _va[_i] = opts[_i] 34 | } 35 | var _ca []interface{} 36 | _ca = append(_ca, ctx, req) 37 | _ca = append(_ca, _va...) 38 | ret := _m.Called(_ca...) 39 | 40 | if len(ret) == 0 { 41 | panic("no return value specified for ListServices") 42 | } 43 | 44 | var r0 *billing.ServiceIterator 45 | if rf, ok := ret.Get(0).(func(context.Context, *billingpb.ListServicesRequest, ...gax.CallOption) *billing.ServiceIterator); ok { 46 | r0 = rf(ctx, req, opts...) 47 | } else { 48 | if ret.Get(0) != nil { 49 | r0 = ret.Get(0).(*billing.ServiceIterator) 50 | } 51 | } 52 | 53 | return r0 54 | } 55 | 56 | // CloudCatalogClient_ListServices_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListServices' 57 | type CloudCatalogClient_ListServices_Call struct { 58 | *mock.Call 59 | } 60 | 61 | // ListServices is a helper method to define mock.On call 62 | // - ctx context.Context 63 | // - req *billingpb.ListServicesRequest 64 | // - opts ...gax.CallOption 65 | func (_e *CloudCatalogClient_Expecter) ListServices(ctx interface{}, req interface{}, opts ...interface{}) *CloudCatalogClient_ListServices_Call { 66 | return &CloudCatalogClient_ListServices_Call{Call: _e.mock.On("ListServices", 67 | append([]interface{}{ctx, req}, opts...)...)} 68 | } 69 | 70 | func (_c *CloudCatalogClient_ListServices_Call) Run(run func(ctx context.Context, req *billingpb.ListServicesRequest, opts ...gax.CallOption)) *CloudCatalogClient_ListServices_Call { 71 | _c.Call.Run(func(args mock.Arguments) { 72 | variadicArgs := make([]gax.CallOption, len(args)-2) 73 | for i, a := range args[2:] { 74 | if a != nil { 75 | variadicArgs[i] = a.(gax.CallOption) 76 | } 77 | } 78 | run(args[0].(context.Context), args[1].(*billingpb.ListServicesRequest), variadicArgs...) 79 | }) 80 | return _c 81 | } 82 | 83 | func (_c *CloudCatalogClient_ListServices_Call) Return(_a0 *billing.ServiceIterator) *CloudCatalogClient_ListServices_Call { 84 | _c.Call.Return(_a0) 85 | return _c 86 | } 87 | 88 | func (_c *CloudCatalogClient_ListServices_Call) RunAndReturn(run func(context.Context, *billingpb.ListServicesRequest, ...gax.CallOption) *billing.ServiceIterator) *CloudCatalogClient_ListServices_Call { 89 | _c.Call.Return(run) 90 | return _c 91 | } 92 | 93 | // ListSkus provides a mock function with given fields: ctx, req, opts 94 | func (_m *CloudCatalogClient) ListSkus(ctx context.Context, req *billingpb.ListSkusRequest, opts ...gax.CallOption) *billing.SkuIterator { 95 | _va := make([]interface{}, len(opts)) 96 | for _i := range opts { 97 | _va[_i] = opts[_i] 98 | } 99 | var _ca []interface{} 100 | _ca = append(_ca, ctx, req) 101 | _ca = append(_ca, _va...) 102 | ret := _m.Called(_ca...) 103 | 104 | if len(ret) == 0 { 105 | panic("no return value specified for ListSkus") 106 | } 107 | 108 | var r0 *billing.SkuIterator 109 | if rf, ok := ret.Get(0).(func(context.Context, *billingpb.ListSkusRequest, ...gax.CallOption) *billing.SkuIterator); ok { 110 | r0 = rf(ctx, req, opts...) 111 | } else { 112 | if ret.Get(0) != nil { 113 | r0 = ret.Get(0).(*billing.SkuIterator) 114 | } 115 | } 116 | 117 | return r0 118 | } 119 | 120 | // CloudCatalogClient_ListSkus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSkus' 121 | type CloudCatalogClient_ListSkus_Call struct { 122 | *mock.Call 123 | } 124 | 125 | // ListSkus is a helper method to define mock.On call 126 | // - ctx context.Context 127 | // - req *billingpb.ListSkusRequest 128 | // - opts ...gax.CallOption 129 | func (_e *CloudCatalogClient_Expecter) ListSkus(ctx interface{}, req interface{}, opts ...interface{}) *CloudCatalogClient_ListSkus_Call { 130 | return &CloudCatalogClient_ListSkus_Call{Call: _e.mock.On("ListSkus", 131 | append([]interface{}{ctx, req}, opts...)...)} 132 | } 133 | 134 | func (_c *CloudCatalogClient_ListSkus_Call) Run(run func(ctx context.Context, req *billingpb.ListSkusRequest, opts ...gax.CallOption)) *CloudCatalogClient_ListSkus_Call { 135 | _c.Call.Run(func(args mock.Arguments) { 136 | variadicArgs := make([]gax.CallOption, len(args)-2) 137 | for i, a := range args[2:] { 138 | if a != nil { 139 | variadicArgs[i] = a.(gax.CallOption) 140 | } 141 | } 142 | run(args[0].(context.Context), args[1].(*billingpb.ListSkusRequest), variadicArgs...) 143 | }) 144 | return _c 145 | } 146 | 147 | func (_c *CloudCatalogClient_ListSkus_Call) Return(_a0 *billing.SkuIterator) *CloudCatalogClient_ListSkus_Call { 148 | _c.Call.Return(_a0) 149 | return _c 150 | } 151 | 152 | func (_c *CloudCatalogClient_ListSkus_Call) RunAndReturn(run func(context.Context, *billingpb.ListSkusRequest, ...gax.CallOption) *billing.SkuIterator) *CloudCatalogClient_ListSkus_Call { 153 | _c.Call.Return(run) 154 | return _c 155 | } 156 | 157 | // NewCloudCatalogClient creates a new instance of CloudCatalogClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 158 | // The first argument is typically a *testing.T value. 159 | func NewCloudCatalogClient(t interface { 160 | mock.TestingT 161 | Cleanup(func()) 162 | }) *CloudCatalogClient { 163 | mock := &CloudCatalogClient{} 164 | mock.Mock.Test(t) 165 | 166 | t.Cleanup(func() { mock.AssertExpectations(t) }) 167 | 168 | return mock 169 | } 170 | -------------------------------------------------------------------------------- /mocks/pkg/google/gcs/RegionsClient.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package gcs 4 | 5 | import ( 6 | compute "cloud.google.com/go/compute/apiv1" 7 | computepb "cloud.google.com/go/compute/apiv1/computepb" 8 | 9 | context "context" 10 | 11 | gax "github.com/googleapis/gax-go/v2" 12 | 13 | mock "github.com/stretchr/testify/mock" 14 | ) 15 | 16 | // RegionsClient is an autogenerated mock type for the RegionsClient type 17 | type RegionsClient struct { 18 | mock.Mock 19 | } 20 | 21 | type RegionsClient_Expecter struct { 22 | mock *mock.Mock 23 | } 24 | 25 | func (_m *RegionsClient) EXPECT() *RegionsClient_Expecter { 26 | return &RegionsClient_Expecter{mock: &_m.Mock} 27 | } 28 | 29 | // List provides a mock function with given fields: ctx, req, opts 30 | func (_m *RegionsClient) List(ctx context.Context, req *computepb.ListRegionsRequest, opts ...gax.CallOption) *compute.RegionIterator { 31 | _va := make([]interface{}, len(opts)) 32 | for _i := range opts { 33 | _va[_i] = opts[_i] 34 | } 35 | var _ca []interface{} 36 | _ca = append(_ca, ctx, req) 37 | _ca = append(_ca, _va...) 38 | ret := _m.Called(_ca...) 39 | 40 | if len(ret) == 0 { 41 | panic("no return value specified for List") 42 | } 43 | 44 | var r0 *compute.RegionIterator 45 | if rf, ok := ret.Get(0).(func(context.Context, *computepb.ListRegionsRequest, ...gax.CallOption) *compute.RegionIterator); ok { 46 | r0 = rf(ctx, req, opts...) 47 | } else { 48 | if ret.Get(0) != nil { 49 | r0 = ret.Get(0).(*compute.RegionIterator) 50 | } 51 | } 52 | 53 | return r0 54 | } 55 | 56 | // RegionsClient_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' 57 | type RegionsClient_List_Call struct { 58 | *mock.Call 59 | } 60 | 61 | // List is a helper method to define mock.On call 62 | // - ctx context.Context 63 | // - req *computepb.ListRegionsRequest 64 | // - opts ...gax.CallOption 65 | func (_e *RegionsClient_Expecter) List(ctx interface{}, req interface{}, opts ...interface{}) *RegionsClient_List_Call { 66 | return &RegionsClient_List_Call{Call: _e.mock.On("List", 67 | append([]interface{}{ctx, req}, opts...)...)} 68 | } 69 | 70 | func (_c *RegionsClient_List_Call) Run(run func(ctx context.Context, req *computepb.ListRegionsRequest, opts ...gax.CallOption)) *RegionsClient_List_Call { 71 | _c.Call.Run(func(args mock.Arguments) { 72 | variadicArgs := make([]gax.CallOption, len(args)-2) 73 | for i, a := range args[2:] { 74 | if a != nil { 75 | variadicArgs[i] = a.(gax.CallOption) 76 | } 77 | } 78 | run(args[0].(context.Context), args[1].(*computepb.ListRegionsRequest), variadicArgs...) 79 | }) 80 | return _c 81 | } 82 | 83 | func (_c *RegionsClient_List_Call) Return(_a0 *compute.RegionIterator) *RegionsClient_List_Call { 84 | _c.Call.Return(_a0) 85 | return _c 86 | } 87 | 88 | func (_c *RegionsClient_List_Call) RunAndReturn(run func(context.Context, *computepb.ListRegionsRequest, ...gax.CallOption) *compute.RegionIterator) *RegionsClient_List_Call { 89 | _c.Call.Return(run) 90 | return _c 91 | } 92 | 93 | // NewRegionsClient creates a new instance of RegionsClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 94 | // The first argument is typically a *testing.T value. 95 | func NewRegionsClient(t interface { 96 | mock.TestingT 97 | Cleanup(func()) 98 | }) *RegionsClient { 99 | mock := &RegionsClient{} 100 | mock.Mock.Test(t) 101 | 102 | t.Cleanup(func() { mock.AssertExpectations(t) }) 103 | 104 | return mock 105 | } 106 | -------------------------------------------------------------------------------- /mocks/pkg/google/gcs/StorageClientInterface.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package gcs 4 | 5 | import ( 6 | context "context" 7 | 8 | storage "cloud.google.com/go/storage" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // StorageClientInterface is an autogenerated mock type for the StorageClientInterface type 13 | type StorageClientInterface struct { 14 | mock.Mock 15 | } 16 | 17 | type StorageClientInterface_Expecter struct { 18 | mock *mock.Mock 19 | } 20 | 21 | func (_m *StorageClientInterface) EXPECT() *StorageClientInterface_Expecter { 22 | return &StorageClientInterface_Expecter{mock: &_m.Mock} 23 | } 24 | 25 | // Buckets provides a mock function with given fields: ctx, projectID 26 | func (_m *StorageClientInterface) Buckets(ctx context.Context, projectID string) *storage.BucketIterator { 27 | ret := _m.Called(ctx, projectID) 28 | 29 | if len(ret) == 0 { 30 | panic("no return value specified for Buckets") 31 | } 32 | 33 | var r0 *storage.BucketIterator 34 | if rf, ok := ret.Get(0).(func(context.Context, string) *storage.BucketIterator); ok { 35 | r0 = rf(ctx, projectID) 36 | } else { 37 | if ret.Get(0) != nil { 38 | r0 = ret.Get(0).(*storage.BucketIterator) 39 | } 40 | } 41 | 42 | return r0 43 | } 44 | 45 | // StorageClientInterface_Buckets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Buckets' 46 | type StorageClientInterface_Buckets_Call struct { 47 | *mock.Call 48 | } 49 | 50 | // Buckets is a helper method to define mock.On call 51 | // - ctx context.Context 52 | // - projectID string 53 | func (_e *StorageClientInterface_Expecter) Buckets(ctx interface{}, projectID interface{}) *StorageClientInterface_Buckets_Call { 54 | return &StorageClientInterface_Buckets_Call{Call: _e.mock.On("Buckets", ctx, projectID)} 55 | } 56 | 57 | func (_c *StorageClientInterface_Buckets_Call) Run(run func(ctx context.Context, projectID string)) *StorageClientInterface_Buckets_Call { 58 | _c.Call.Run(func(args mock.Arguments) { 59 | run(args[0].(context.Context), args[1].(string)) 60 | }) 61 | return _c 62 | } 63 | 64 | func (_c *StorageClientInterface_Buckets_Call) Return(_a0 *storage.BucketIterator) *StorageClientInterface_Buckets_Call { 65 | _c.Call.Return(_a0) 66 | return _c 67 | } 68 | 69 | func (_c *StorageClientInterface_Buckets_Call) RunAndReturn(run func(context.Context, string) *storage.BucketIterator) *StorageClientInterface_Buckets_Call { 70 | _c.Call.Return(run) 71 | return _c 72 | } 73 | 74 | // NewStorageClientInterface creates a new instance of StorageClientInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 75 | // The first argument is typically a *testing.T value. 76 | func NewStorageClientInterface(t interface { 77 | mock.TestingT 78 | Cleanup(func()) 79 | }) *StorageClientInterface { 80 | mock := &StorageClientInterface{} 81 | mock.Mock.Test(t) 82 | 83 | t.Cleanup(func() { mock.AssertExpectations(t) }) 84 | 85 | return mock 86 | } 87 | -------------------------------------------------------------------------------- /mocks/pkg/provider/Collector.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package provider 4 | 5 | import ( 6 | prometheus "github.com/prometheus/client_golang/prometheus" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // Collector is an autogenerated mock type for the Collector type 11 | type Collector struct { 12 | mock.Mock 13 | } 14 | 15 | type Collector_Expecter struct { 16 | mock *mock.Mock 17 | } 18 | 19 | func (_m *Collector) EXPECT() *Collector_Expecter { 20 | return &Collector_Expecter{mock: &_m.Mock} 21 | } 22 | 23 | // Collect provides a mock function with given fields: _a0 24 | func (_m *Collector) Collect(_a0 chan<- prometheus.Metric) error { 25 | ret := _m.Called(_a0) 26 | 27 | if len(ret) == 0 { 28 | panic("no return value specified for Collect") 29 | } 30 | 31 | var r0 error 32 | if rf, ok := ret.Get(0).(func(chan<- prometheus.Metric) error); ok { 33 | r0 = rf(_a0) 34 | } else { 35 | r0 = ret.Error(0) 36 | } 37 | 38 | return r0 39 | } 40 | 41 | // Collector_Collect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Collect' 42 | type Collector_Collect_Call struct { 43 | *mock.Call 44 | } 45 | 46 | // Collect is a helper method to define mock.On call 47 | // - _a0 chan<- prometheus.Metric 48 | func (_e *Collector_Expecter) Collect(_a0 interface{}) *Collector_Collect_Call { 49 | return &Collector_Collect_Call{Call: _e.mock.On("Collect", _a0)} 50 | } 51 | 52 | func (_c *Collector_Collect_Call) Run(run func(_a0 chan<- prometheus.Metric)) *Collector_Collect_Call { 53 | _c.Call.Run(func(args mock.Arguments) { 54 | run(args[0].(chan<- prometheus.Metric)) 55 | }) 56 | return _c 57 | } 58 | 59 | func (_c *Collector_Collect_Call) Return(_a0 error) *Collector_Collect_Call { 60 | _c.Call.Return(_a0) 61 | return _c 62 | } 63 | 64 | func (_c *Collector_Collect_Call) RunAndReturn(run func(chan<- prometheus.Metric) error) *Collector_Collect_Call { 65 | _c.Call.Return(run) 66 | return _c 67 | } 68 | 69 | // CollectMetrics provides a mock function with given fields: _a0 70 | func (_m *Collector) CollectMetrics(_a0 chan<- prometheus.Metric) float64 { 71 | ret := _m.Called(_a0) 72 | 73 | if len(ret) == 0 { 74 | panic("no return value specified for CollectMetrics") 75 | } 76 | 77 | var r0 float64 78 | if rf, ok := ret.Get(0).(func(chan<- prometheus.Metric) float64); ok { 79 | r0 = rf(_a0) 80 | } else { 81 | r0 = ret.Get(0).(float64) 82 | } 83 | 84 | return r0 85 | } 86 | 87 | // Collector_CollectMetrics_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CollectMetrics' 88 | type Collector_CollectMetrics_Call struct { 89 | *mock.Call 90 | } 91 | 92 | // CollectMetrics is a helper method to define mock.On call 93 | // - _a0 chan<- prometheus.Metric 94 | func (_e *Collector_Expecter) CollectMetrics(_a0 interface{}) *Collector_CollectMetrics_Call { 95 | return &Collector_CollectMetrics_Call{Call: _e.mock.On("CollectMetrics", _a0)} 96 | } 97 | 98 | func (_c *Collector_CollectMetrics_Call) Run(run func(_a0 chan<- prometheus.Metric)) *Collector_CollectMetrics_Call { 99 | _c.Call.Run(func(args mock.Arguments) { 100 | run(args[0].(chan<- prometheus.Metric)) 101 | }) 102 | return _c 103 | } 104 | 105 | func (_c *Collector_CollectMetrics_Call) Return(_a0 float64) *Collector_CollectMetrics_Call { 106 | _c.Call.Return(_a0) 107 | return _c 108 | } 109 | 110 | func (_c *Collector_CollectMetrics_Call) RunAndReturn(run func(chan<- prometheus.Metric) float64) *Collector_CollectMetrics_Call { 111 | _c.Call.Return(run) 112 | return _c 113 | } 114 | 115 | // Describe provides a mock function with given fields: _a0 116 | func (_m *Collector) Describe(_a0 chan<- *prometheus.Desc) error { 117 | ret := _m.Called(_a0) 118 | 119 | if len(ret) == 0 { 120 | panic("no return value specified for Describe") 121 | } 122 | 123 | var r0 error 124 | if rf, ok := ret.Get(0).(func(chan<- *prometheus.Desc) error); ok { 125 | r0 = rf(_a0) 126 | } else { 127 | r0 = ret.Error(0) 128 | } 129 | 130 | return r0 131 | } 132 | 133 | // Collector_Describe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Describe' 134 | type Collector_Describe_Call struct { 135 | *mock.Call 136 | } 137 | 138 | // Describe is a helper method to define mock.On call 139 | // - _a0 chan<- *prometheus.Desc 140 | func (_e *Collector_Expecter) Describe(_a0 interface{}) *Collector_Describe_Call { 141 | return &Collector_Describe_Call{Call: _e.mock.On("Describe", _a0)} 142 | } 143 | 144 | func (_c *Collector_Describe_Call) Run(run func(_a0 chan<- *prometheus.Desc)) *Collector_Describe_Call { 145 | _c.Call.Run(func(args mock.Arguments) { 146 | run(args[0].(chan<- *prometheus.Desc)) 147 | }) 148 | return _c 149 | } 150 | 151 | func (_c *Collector_Describe_Call) Return(_a0 error) *Collector_Describe_Call { 152 | _c.Call.Return(_a0) 153 | return _c 154 | } 155 | 156 | func (_c *Collector_Describe_Call) RunAndReturn(run func(chan<- *prometheus.Desc) error) *Collector_Describe_Call { 157 | _c.Call.Return(run) 158 | return _c 159 | } 160 | 161 | // Name provides a mock function with given fields: 162 | func (_m *Collector) Name() string { 163 | ret := _m.Called() 164 | 165 | if len(ret) == 0 { 166 | panic("no return value specified for Name") 167 | } 168 | 169 | var r0 string 170 | if rf, ok := ret.Get(0).(func() string); ok { 171 | r0 = rf() 172 | } else { 173 | r0 = ret.Get(0).(string) 174 | } 175 | 176 | return r0 177 | } 178 | 179 | // Collector_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' 180 | type Collector_Name_Call struct { 181 | *mock.Call 182 | } 183 | 184 | // Name is a helper method to define mock.On call 185 | func (_e *Collector_Expecter) Name() *Collector_Name_Call { 186 | return &Collector_Name_Call{Call: _e.mock.On("Name")} 187 | } 188 | 189 | func (_c *Collector_Name_Call) Run(run func()) *Collector_Name_Call { 190 | _c.Call.Run(func(args mock.Arguments) { 191 | run() 192 | }) 193 | return _c 194 | } 195 | 196 | func (_c *Collector_Name_Call) Return(_a0 string) *Collector_Name_Call { 197 | _c.Call.Return(_a0) 198 | return _c 199 | } 200 | 201 | func (_c *Collector_Name_Call) RunAndReturn(run func() string) *Collector_Name_Call { 202 | _c.Call.Return(run) 203 | return _c 204 | } 205 | 206 | // Register provides a mock function with given fields: r 207 | func (_m *Collector) Register(r Registry) error { 208 | ret := _m.Called(r) 209 | 210 | if len(ret) == 0 { 211 | panic("no return value specified for Register") 212 | } 213 | 214 | var r0 error 215 | if rf, ok := ret.Get(0).(func(Registry) error); ok { 216 | r0 = rf(r) 217 | } else { 218 | r0 = ret.Error(0) 219 | } 220 | 221 | return r0 222 | } 223 | 224 | // Collector_Register_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Register' 225 | type Collector_Register_Call struct { 226 | *mock.Call 227 | } 228 | 229 | // Register is a helper method to define mock.On call 230 | // - r Registry 231 | func (_e *Collector_Expecter) Register(r interface{}) *Collector_Register_Call { 232 | return &Collector_Register_Call{Call: _e.mock.On("Register", r)} 233 | } 234 | 235 | func (_c *Collector_Register_Call) Run(run func(r Registry)) *Collector_Register_Call { 236 | _c.Call.Run(func(args mock.Arguments) { 237 | run(args[0].(Registry)) 238 | }) 239 | return _c 240 | } 241 | 242 | func (_c *Collector_Register_Call) Return(_a0 error) *Collector_Register_Call { 243 | _c.Call.Return(_a0) 244 | return _c 245 | } 246 | 247 | func (_c *Collector_Register_Call) RunAndReturn(run func(Registry) error) *Collector_Register_Call { 248 | _c.Call.Return(run) 249 | return _c 250 | } 251 | 252 | // NewCollector creates a new instance of Collector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 253 | // The first argument is typically a *testing.T value. 254 | func NewCollector(t interface { 255 | mock.TestingT 256 | Cleanup(func()) 257 | }) *Collector { 258 | mock := &Collector{} 259 | mock.Mock.Test(t) 260 | 261 | t.Cleanup(func() { mock.AssertExpectations(t) }) 262 | 263 | return mock 264 | } 265 | -------------------------------------------------------------------------------- /mocks/pkg/provider/Provider.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.43.2. DO NOT EDIT. 2 | 3 | package provider 4 | 5 | import ( 6 | prometheus "github.com/prometheus/client_golang/prometheus" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // Provider is an autogenerated mock type for the Provider type 11 | type Provider struct { 12 | mock.Mock 13 | } 14 | 15 | type Provider_Expecter struct { 16 | mock *mock.Mock 17 | } 18 | 19 | func (_m *Provider) EXPECT() *Provider_Expecter { 20 | return &Provider_Expecter{mock: &_m.Mock} 21 | } 22 | 23 | // Collect provides a mock function with given fields: _a0 24 | func (_m *Provider) Collect(_a0 chan<- prometheus.Metric) { 25 | _m.Called(_a0) 26 | } 27 | 28 | // Provider_Collect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Collect' 29 | type Provider_Collect_Call struct { 30 | *mock.Call 31 | } 32 | 33 | // Collect is a helper method to define mock.On call 34 | // - _a0 chan<- prometheus.Metric 35 | func (_e *Provider_Expecter) Collect(_a0 interface{}) *Provider_Collect_Call { 36 | return &Provider_Collect_Call{Call: _e.mock.On("Collect", _a0)} 37 | } 38 | 39 | func (_c *Provider_Collect_Call) Run(run func(_a0 chan<- prometheus.Metric)) *Provider_Collect_Call { 40 | _c.Call.Run(func(args mock.Arguments) { 41 | run(args[0].(chan<- prometheus.Metric)) 42 | }) 43 | return _c 44 | } 45 | 46 | func (_c *Provider_Collect_Call) Return() *Provider_Collect_Call { 47 | _c.Call.Return() 48 | return _c 49 | } 50 | 51 | func (_c *Provider_Collect_Call) RunAndReturn(run func(chan<- prometheus.Metric)) *Provider_Collect_Call { 52 | _c.Call.Return(run) 53 | return _c 54 | } 55 | 56 | // Describe provides a mock function with given fields: _a0 57 | func (_m *Provider) Describe(_a0 chan<- *prometheus.Desc) { 58 | _m.Called(_a0) 59 | } 60 | 61 | // Provider_Describe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Describe' 62 | type Provider_Describe_Call struct { 63 | *mock.Call 64 | } 65 | 66 | // Describe is a helper method to define mock.On call 67 | // - _a0 chan<- *prometheus.Desc 68 | func (_e *Provider_Expecter) Describe(_a0 interface{}) *Provider_Describe_Call { 69 | return &Provider_Describe_Call{Call: _e.mock.On("Describe", _a0)} 70 | } 71 | 72 | func (_c *Provider_Describe_Call) Run(run func(_a0 chan<- *prometheus.Desc)) *Provider_Describe_Call { 73 | _c.Call.Run(func(args mock.Arguments) { 74 | run(args[0].(chan<- *prometheus.Desc)) 75 | }) 76 | return _c 77 | } 78 | 79 | func (_c *Provider_Describe_Call) Return() *Provider_Describe_Call { 80 | _c.Call.Return() 81 | return _c 82 | } 83 | 84 | func (_c *Provider_Describe_Call) RunAndReturn(run func(chan<- *prometheus.Desc)) *Provider_Describe_Call { 85 | _c.Call.Return(run) 86 | return _c 87 | } 88 | 89 | // RegisterCollectors provides a mock function with given fields: r 90 | func (_m *Provider) RegisterCollectors(r Registry) error { 91 | ret := _m.Called(r) 92 | 93 | if len(ret) == 0 { 94 | panic("no return value specified for RegisterCollectors") 95 | } 96 | 97 | var r0 error 98 | if rf, ok := ret.Get(0).(func(Registry) error); ok { 99 | r0 = rf(r) 100 | } else { 101 | r0 = ret.Error(0) 102 | } 103 | 104 | return r0 105 | } 106 | 107 | // Provider_RegisterCollectors_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RegisterCollectors' 108 | type Provider_RegisterCollectors_Call struct { 109 | *mock.Call 110 | } 111 | 112 | // RegisterCollectors is a helper method to define mock.On call 113 | // - r Registry 114 | func (_e *Provider_Expecter) RegisterCollectors(r interface{}) *Provider_RegisterCollectors_Call { 115 | return &Provider_RegisterCollectors_Call{Call: _e.mock.On("RegisterCollectors", r)} 116 | } 117 | 118 | func (_c *Provider_RegisterCollectors_Call) Run(run func(r Registry)) *Provider_RegisterCollectors_Call { 119 | _c.Call.Run(func(args mock.Arguments) { 120 | run(args[0].(Registry)) 121 | }) 122 | return _c 123 | } 124 | 125 | func (_c *Provider_RegisterCollectors_Call) Return(_a0 error) *Provider_RegisterCollectors_Call { 126 | _c.Call.Return(_a0) 127 | return _c 128 | } 129 | 130 | func (_c *Provider_RegisterCollectors_Call) RunAndReturn(run func(Registry) error) *Provider_RegisterCollectors_Call { 131 | _c.Call.Return(run) 132 | return _c 133 | } 134 | 135 | // NewProvider creates a new instance of Provider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 136 | // The first argument is typically a *testing.T value. 137 | func NewProvider(t interface { 138 | mock.TestingT 139 | Cleanup(func()) 140 | }) *Provider { 141 | mock := &Provider{} 142 | mock.Mock.Test(t) 143 | 144 | t.Cleanup(func() { mock.AssertExpectations(t) }) 145 | 146 | return mock 147 | } 148 | -------------------------------------------------------------------------------- /pkg/aws/README.md: -------------------------------------------------------------------------------- 1 | # AWS 2 | 3 | This module is responsible for collecting and exporting costs associated with AWS resources. 4 | `aws.go` is the entrypoint for the module and is responsible for setting up the AWS session and starting the collection process. 5 | The module is built upon the aws-sdk-go library and uses the Cost Explorer API to collect cost data. 6 | 7 | -------------------------------------------------------------------------------- /pkg/aws/aws_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log/slog" 7 | "os" 8 | "testing" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/stretchr/testify/require" 12 | "go.uber.org/mock/gomock" 13 | 14 | "github.com/grafana/cloudcost-exporter/pkg/provider" 15 | mock_provider "github.com/grafana/cloudcost-exporter/pkg/provider/mocks" 16 | ) 17 | 18 | var logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) 19 | 20 | func Test_New(t *testing.T) { 21 | for _, tc := range []struct { 22 | name string 23 | expectedError error 24 | }{ 25 | { 26 | name: "no error", 27 | }, 28 | } { 29 | t.Run(tc.name, func(t *testing.T) { 30 | // TODO refactor New() 31 | t.SkipNow() 32 | 33 | a, err := New(context.Background(), &Config{}) 34 | if tc.expectedError != nil { 35 | require.EqualError(t, err, tc.expectedError.Error()) 36 | return 37 | } 38 | require.NoError(t, err) 39 | require.NotNil(t, a) 40 | }) 41 | } 42 | } 43 | 44 | func Test_RegisterCollectors(t *testing.T) { 45 | for _, tc := range []struct { 46 | name string 47 | numCollectors int 48 | register func(r provider.Registry) error 49 | expectedError error 50 | }{ 51 | { 52 | name: "no error if no collectors", 53 | }, 54 | { 55 | name: "bubble-up single collector error", 56 | numCollectors: 1, 57 | register: func(r provider.Registry) error { 58 | return fmt.Errorf("test register error") 59 | }, 60 | expectedError: fmt.Errorf("test register error"), 61 | }, 62 | { 63 | name: "two collectors with no errors", 64 | numCollectors: 2, 65 | register: func(r provider.Registry) error { return nil }, 66 | }, 67 | } { 68 | t.Run(tc.name, func(t *testing.T) { 69 | ctrl := gomock.NewController(t) 70 | r := mock_provider.NewMockRegistry(ctrl) 71 | r.EXPECT().MustRegister(gomock.Any()).AnyTimes() 72 | c := mock_provider.NewMockCollector(ctrl) 73 | if tc.register != nil { 74 | c.EXPECT().Register(r).DoAndReturn(tc.register).Times(tc.numCollectors) 75 | } 76 | 77 | a := AWS{ 78 | Config: nil, 79 | collectors: []provider.Collector{}, 80 | logger: logger, 81 | } 82 | for i := 0; i < tc.numCollectors; i++ { 83 | a.collectors = append(a.collectors, c) 84 | } 85 | 86 | err := a.RegisterCollectors(r) 87 | if tc.expectedError != nil { 88 | require.EqualError(t, err, tc.expectedError.Error()) 89 | return 90 | } 91 | require.NoError(t, err) 92 | }) 93 | } 94 | } 95 | 96 | func Test_CollectMetrics(t *testing.T) { 97 | for _, tc := range []struct { 98 | name string 99 | numCollectors int 100 | collect func(chan<- prometheus.Metric) error 101 | }{ 102 | { 103 | name: "no error if no collectors", 104 | }, 105 | { 106 | name: "bubble-up single collector error", 107 | numCollectors: 1, 108 | collect: func(chan<- prometheus.Metric) error { 109 | return nil 110 | }, 111 | }, 112 | { 113 | name: "two collectors with no errors", 114 | numCollectors: 2, 115 | collect: func(chan<- prometheus.Metric) error { return nil }, 116 | }, 117 | } { 118 | t.Run(tc.name, func(t *testing.T) { 119 | ch := make(chan prometheus.Metric) 120 | go func() { 121 | for range ch { 122 | // This is necessary to ensure the test doesn't hang 123 | } 124 | }() 125 | ctrl := gomock.NewController(t) 126 | c := mock_provider.NewMockCollector(ctrl) 127 | if tc.collect != nil { 128 | c.EXPECT().Collect(ch).DoAndReturn(tc.collect).Times(tc.numCollectors) 129 | c.EXPECT().Name().Return("test").AnyTimes() 130 | } 131 | 132 | a := AWS{ 133 | Config: nil, 134 | collectors: []provider.Collector{}, 135 | logger: logger, 136 | } 137 | for i := 0; i < tc.numCollectors; i++ { 138 | a.collectors = append(a.collectors, c) 139 | } 140 | 141 | a.Collect(ch) 142 | close(ch) 143 | }) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /pkg/aws/ec2/README.md: -------------------------------------------------------------------------------- 1 | # ec2 cost module 2 | 3 | This module is responsible for collecting pricing information for EC2 instances. 4 | See [metrics](/docs/metrics/aws/ec2.md) for more information on the metrics that are collected. 5 | 6 | ## Overview 7 | 8 | EC2 instances are a foundational component in the AWS ecosystem. 9 | They can be used as bare bone virtual machines, or used as the underlying infrastructure for services like 10 | 1. [EC2 instances](https://aws.amazon.com/ec2/pricing/on-demand/) 11 | 1. [ECS Clusters](https://aws.amazon.com/ecs/pricing/) that use ec2 instances* 12 | 1. [EKS Clusters](https://aws.amazon.com/eks/pricing/) 13 | 14 | This module aims to emit metrics generically for ec2 instances that can be used for the services above. 15 | A conscious decision was made the keep ec2 + eks implementations coupled. 16 | See [#215](https://github.com/grafana/cloudcost-exporter/pull/215) for more details on _why_, as this decision _can_ be reversed in the future. 17 | 18 | *Fargate is a serverless product which builds upon ec2 instances, but with a specific caveat: [Pricing is based upon the requests by workloads](https://aws.amazon.com/fargate/pricing/) 19 | > ![WARNING] 20 | > Even though Fargate uses ec2 instances under the hood, it would require a separate module since the pricing comes from a different end point 21 | 22 | ## Pricing Map 23 | 24 | The pricing map is generated based on the machine type and the region where the instance is running. 25 | 26 | Here's how the data structure looks like: 27 | 28 | ``` 29 | --> root 30 | --> region 31 | --> machine type 32 | --> reservation type(on-demand, spot) 33 | --> price 34 | ``` 35 | 36 | Regions are populated by making a [describe region](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeRegions.html) api call to find the regions enabled for the account. 37 | The [price](https://github.com/grafana/cloudcost-exporter/blob/eb6b3ed9e0d4ab4eb27bda71ada091730c95f709/pkg/aws/ec2/pricing_map.go#L62) keeps track of the hourly cost per: 38 | 1. price per cpu 39 | 2. price per GiB of ram 40 | 3. total price 41 | 42 | The pricing information for the compute instances is collected from the AWS Pricing API. 43 | Detailed documentation around the pricing API can be found [here](https://aws.amazon.com/ec2/pricing/on-demand/). 44 | One of the main challenges with EKS compute instance pricing is that the pricing is for the full instance and not broken down by resource. 45 | This means that the pricing information is not available for the CPU and memory separately. 46 | `cloudcost-exporter` makes the assumption that the ratio of costs is relatively similar to that of GKE instances. 47 | When fetching the list prices, `cloudcost-exporter` will use the ratio from GCP to break down the cost of the instance into CPU and memory. 48 | 49 | ## Collecting Machines 50 | 51 | The following attributes must be available from the ec2 instance to make the lookup: 52 | - region 53 | - machine type 54 | - reservation type 55 | 56 | Every time the collector is scraped, a list of machines is collected _by region_ in a seperate goroutine. 57 | This allows the collector to scrape each region in parallel, making the largest region be the bottleneck. 58 | For simplicity, there is no cache, though this is a nice feature to add in the future. 59 | 60 | ## Cost Calculations 61 | 62 | Here's some example PromQL queries that can be used to calculate the costs of ec2 instances: 63 | 64 | ```PromQL 65 | // Calculate the total hourly cost of all ec2 instances 66 | sum(cloudcost_aws_ec2_instance_total_usd_per_houry) 67 | // Calculate the total hourly cost by region 68 | sum by (region) (cloudcost_aws_ec2_instance_total_usd_per_houry) 69 | // Calculate the total hourly cost by machine type 70 | sum by (machine_type) (cloudcost_aws_ec2_instance_total_usd_per_houry) 71 | // Calculate the total hourly cost by reservation type 72 | sum by (reservation) (cloudcost_aws_ec2_instance_total_usd_per_houry) 73 | ``` 74 | 75 | You can do more interesting queries if you run [yet-another-cloudwatch-exporter](https://github.com/nerdswords/yet-another-cloudwatch-exporter) and export the following metrics: 76 | - `aws_ec2_info` 77 | 78 | All of these examples assume that you have created the tag name referenced in the examples. 79 | 80 | ```PromQL 81 | // Calculate the total hourly cost by team 82 | // Assumes a tag called `Team` has been created on the ec2 instances 83 | sum by (team) ( 84 | cloudcost_aws_ec2_instance_total_usd_per_houry 85 | * on (instance_id) group_right() 86 | label_join(aws_ec2_info, "team", "tag_Team") 87 | ) 88 | 89 | // Calculate the total hourly cost by team and environment 90 | // Assumes a tag called `Team` has been created on the ec2 instances 91 | // Assumes a tag called `Environment` has been created on the ec2 instances 92 | sum by (team, environment) ( 93 | cloudcost_aws_ec2_instance_total_usd_per_houry 94 | * on (instance_id) group_right() 95 | label_join( 96 | label_join(aws_ec2_info, "environment", "tag_Environment") 97 | "team", "tag_Team") 98 | ) 99 | ``` 100 | -------------------------------------------------------------------------------- /pkg/aws/ec2/compute.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/aws" 7 | ec22 "github.com/aws/aws-sdk-go-v2/service/ec2" 8 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 9 | 10 | "github.com/grafana/cloudcost-exporter/pkg/aws/services/ec2" 11 | ) 12 | 13 | const maxResults = 1000 14 | 15 | func ListComputeInstances(ctx context.Context, client ec2.EC2) ([]types.Reservation, error) { 16 | dii := &ec22.DescribeInstancesInput{ 17 | // 1000 max results was decided arbitrarily. This can likely be tuned. 18 | MaxResults: aws.Int32(maxResults), 19 | } 20 | var instances []types.Reservation 21 | for { 22 | resp, err := client.DescribeInstances(ctx, dii) 23 | if err != nil { 24 | return nil, err 25 | } 26 | instances = append(instances, resp.Reservations...) 27 | if resp.NextToken == nil || *resp.NextToken == "" { 28 | break 29 | } 30 | dii.NextToken = resp.NextToken 31 | } 32 | 33 | return instances, nil 34 | } 35 | 36 | var clusterTags = []string{"cluster", "eks:cluster-name", "aws:eks:cluster-name"} 37 | 38 | func ClusterNameFromInstance(instance types.Instance) string { 39 | for _, tag := range instance.Tags { 40 | for _, key := range clusterTags { 41 | if *tag.Key == key { 42 | return *tag.Value 43 | } 44 | } 45 | } 46 | return "" 47 | } 48 | -------------------------------------------------------------------------------- /pkg/aws/ec2/compute_test.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go-v2/aws" 8 | "github.com/aws/aws-sdk-go-v2/service/ec2" 9 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | 13 | ec22 "github.com/grafana/cloudcost-exporter/mocks/pkg/aws/services/ec2" 14 | ) 15 | 16 | func TestListComputeInstances(t *testing.T) { 17 | tests := map[string]struct { 18 | ctx context.Context 19 | DescribeInstances func(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) 20 | err error 21 | want []types.Reservation 22 | expectedCalls int 23 | }{ 24 | "No instance should return nothing": { 25 | ctx: context.Background(), 26 | DescribeInstances: func(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 27 | return &ec2.DescribeInstancesOutput{}, nil 28 | }, 29 | err: nil, 30 | want: nil, 31 | expectedCalls: 1, 32 | }, 33 | "Single instance should return a single instance": { 34 | ctx: context.Background(), 35 | DescribeInstances: func(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 36 | return &ec2.DescribeInstancesOutput{ 37 | Reservations: []types.Reservation{ 38 | { 39 | Instances: []types.Instance{ 40 | { 41 | InstanceId: aws.String("i-1234567890abcdef0"), 42 | InstanceType: types.InstanceTypeA1Xlarge, 43 | }, 44 | }, 45 | }, 46 | }, 47 | }, nil 48 | }, 49 | err: nil, 50 | want: []types.Reservation{ 51 | { 52 | Instances: []types.Instance{ 53 | { 54 | InstanceId: aws.String("i-1234567890abcdef0"), 55 | InstanceType: types.InstanceTypeA1Xlarge, 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | "Ensure errors propagate": { 62 | ctx: context.Background(), 63 | DescribeInstances: func(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 64 | return nil, assert.AnError 65 | }, 66 | err: assert.AnError, 67 | want: nil, 68 | }, 69 | "NextToken should return multiple instances": { 70 | ctx: context.Background(), 71 | DescribeInstances: func(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { 72 | if e.NextToken == nil { 73 | return &ec2.DescribeInstancesOutput{ 74 | NextToken: aws.String("token"), 75 | Reservations: []types.Reservation{ 76 | { 77 | Instances: []types.Instance{ 78 | { 79 | InstanceId: aws.String("i-1234567890abcdef0"), 80 | InstanceType: types.InstanceTypeA1Xlarge, 81 | }, 82 | }, 83 | }, 84 | }, 85 | }, nil 86 | } 87 | return &ec2.DescribeInstancesOutput{ 88 | Reservations: []types.Reservation{ 89 | { 90 | Instances: []types.Instance{ 91 | { 92 | InstanceId: aws.String("i-1234567890abcdef0"), 93 | InstanceType: types.InstanceTypeA1Xlarge, 94 | }, 95 | }, 96 | }, 97 | }, 98 | }, nil 99 | }, 100 | 101 | err: nil, 102 | want: []types.Reservation{ 103 | { 104 | Instances: []types.Instance{ 105 | { 106 | InstanceId: aws.String("i-1234567890abcdef0"), 107 | InstanceType: types.InstanceTypeA1Xlarge, 108 | }, 109 | }, 110 | }, 111 | { 112 | Instances: []types.Instance{ 113 | { 114 | InstanceId: aws.String("i-1234567890abcdef0"), 115 | InstanceType: types.InstanceTypeA1Xlarge, 116 | }, 117 | }, 118 | }, 119 | }, 120 | expectedCalls: 2, 121 | }, 122 | } 123 | for name, tt := range tests { 124 | t.Run(name, func(t *testing.T) { 125 | client := ec22.NewEC2(t) 126 | client.EXPECT(). 127 | DescribeInstances(mock.Anything, mock.Anything, mock.Anything). 128 | RunAndReturn(tt.DescribeInstances). 129 | Times(tt.expectedCalls) 130 | 131 | got, err := ListComputeInstances(tt.ctx, client) 132 | assert.Equal(t, tt.err, err) 133 | assert.Equalf(t, tt.want, got, "ListComputeInstances(%v, %v)", tt.ctx, client) 134 | }) 135 | } 136 | } 137 | 138 | func Test_clusterNameFromInstance(t *testing.T) { 139 | tests := map[string]struct { 140 | instance types.Instance 141 | want string 142 | }{ 143 | "Instance with no tags should return an empty string": { 144 | instance: types.Instance{}, 145 | want: "", 146 | }, 147 | "Instance with a tag should return the cluster name": { 148 | instance: types.Instance{ 149 | Tags: []types.Tag{ 150 | { 151 | Key: aws.String("cluster"), 152 | Value: aws.String("cluster-name"), 153 | }, 154 | }, 155 | }, 156 | want: "cluster-name", 157 | }, 158 | "Instance with eks:clustername should return the cluster name": { 159 | instance: types.Instance{ 160 | Tags: []types.Tag{ 161 | { 162 | Key: aws.String("eks:cluster-name"), 163 | Value: aws.String("cluster-name"), 164 | }, 165 | }, 166 | }, 167 | want: "cluster-name", 168 | }, 169 | "Instance with aws:eks:cluster-name should return the cluster name": { 170 | instance: types.Instance{ 171 | Tags: []types.Tag{ 172 | { 173 | Key: aws.String("eks:cluster-name"), 174 | Value: aws.String("cluster-name"), 175 | }, 176 | }, 177 | }, 178 | want: "cluster-name", 179 | }, 180 | } 181 | for name, tt := range tests { 182 | t.Run(name, func(t *testing.T) { 183 | assert.Equalf(t, tt.want, ClusterNameFromInstance(tt.instance), "ClusterNameFromInstance(%v)", tt.instance) 184 | }) 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /pkg/aws/ec2/disk.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/aws" 7 | ec22 "github.com/aws/aws-sdk-go-v2/service/ec2" 8 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 9 | 10 | "github.com/grafana/cloudcost-exporter/pkg/aws/services/ec2" 11 | ) 12 | 13 | const eksPVTagName = "kubernetes.io/created-for/pv/name" 14 | 15 | func ListEBSVolumes(ctx context.Context, client ec2.EC2) ([]types.Volume, error) { 16 | params := &ec22.DescribeVolumesInput{ 17 | Filters: []types.Filter{ 18 | // excludes volumes created from snapshots 19 | { 20 | Name: aws.String("snapshot-id"), 21 | Values: []string{""}, 22 | }, 23 | }, 24 | } 25 | 26 | pager := ec22.NewDescribeVolumesPaginator(client, params) 27 | var volumes []types.Volume 28 | 29 | for pager.HasMorePages() { 30 | resp, err := pager.NextPage(ctx) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | volumes = append(volumes, resp.Volumes...) 36 | if resp.NextToken == nil || *resp.NextToken == "" { 37 | break 38 | } 39 | } 40 | 41 | return volumes, nil 42 | } 43 | 44 | func NameFromVolume(volume types.Volume) string { 45 | for _, tag := range volume.Tags { 46 | if *tag.Key == eksPVTagName { 47 | return *tag.Value 48 | } 49 | } 50 | 51 | return "" 52 | } 53 | -------------------------------------------------------------------------------- /pkg/aws/ec2/disk_test.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aws/aws-sdk-go-v2/aws" 8 | "github.com/aws/aws-sdk-go-v2/service/ec2" 9 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | 13 | ec22 "github.com/grafana/cloudcost-exporter/mocks/pkg/aws/services/ec2" 14 | ) 15 | 16 | func TestListEBSVolumes(t *testing.T) { 17 | tests := map[string]struct { 18 | DescribeVolumes func(ctx context.Context, e *ec2.DescribeVolumesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) 19 | err error 20 | expected []types.Volume 21 | expectedCalls int 22 | }{ 23 | "no volumes should return empty": { 24 | DescribeVolumes: func(ctx context.Context, e *ec2.DescribeVolumesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) { 25 | return &ec2.DescribeVolumesOutput{}, nil 26 | }, 27 | expectedCalls: 1, 28 | }, 29 | "ensure errors propagate": { 30 | DescribeVolumes: func(ctx context.Context, e *ec2.DescribeVolumesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) { 31 | return nil, assert.AnError 32 | }, 33 | err: assert.AnError, 34 | expectedCalls: 1, 35 | }, 36 | "returns volumes": { 37 | DescribeVolumes: func(ctx context.Context, e *ec2.DescribeVolumesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) { 38 | return &ec2.DescribeVolumesOutput{ 39 | Volumes: []types.Volume{ 40 | { 41 | VolumeId: aws.String("vol-111111111"), 42 | }, 43 | }, 44 | }, nil 45 | }, 46 | expected: []types.Volume{ 47 | { 48 | VolumeId: aws.String("vol-111111111"), 49 | }, 50 | }, 51 | expectedCalls: 1, 52 | }, 53 | "paginator iterates over pages": { 54 | DescribeVolumes: func(ctx context.Context, e *ec2.DescribeVolumesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) { 55 | if e.NextToken == nil { 56 | return &ec2.DescribeVolumesOutput{ 57 | NextToken: aws.String("token"), 58 | Volumes: []types.Volume{ 59 | { 60 | VolumeId: aws.String("vol-111111111"), 61 | }, 62 | }, 63 | }, nil 64 | } 65 | return &ec2.DescribeVolumesOutput{ 66 | Volumes: []types.Volume{ 67 | { 68 | VolumeId: aws.String("vol-2222222222"), 69 | }, 70 | }, 71 | }, nil 72 | }, 73 | expected: []types.Volume{ 74 | { 75 | VolumeId: aws.String("vol-111111111"), 76 | }, 77 | { 78 | VolumeId: aws.String("vol-2222222222"), 79 | }, 80 | }, 81 | expectedCalls: 2, 82 | }, 83 | } 84 | 85 | for name, tt := range tests { 86 | t.Run(name, func(t *testing.T) { 87 | client := ec22.NewEC2(t) 88 | client.EXPECT(). 89 | DescribeVolumes(mock.Anything, mock.Anything, mock.Anything). 90 | RunAndReturn(tt.DescribeVolumes). 91 | Times(tt.expectedCalls) 92 | ctx := context.Background() 93 | 94 | resp, err := ListEBSVolumes(ctx, client) 95 | assert.Equal(t, tt.err, err) 96 | assert.Equal(t, tt.expected, resp) 97 | }) 98 | } 99 | } 100 | 101 | func TestNameFromVolume(t *testing.T) { 102 | tests := map[string]struct { 103 | volume types.Volume 104 | expected string 105 | }{ 106 | "no tags returns empty string": { 107 | volume: types.Volume{}, 108 | expected: "", 109 | }, 110 | "tags exist but not the pv name one": { 111 | volume: types.Volume{ 112 | Tags: []types.Tag{ 113 | { 114 | Key: aws.String("asdf"), 115 | Value: aws.String("asdf"), 116 | }, 117 | }, 118 | }, 119 | expected: "", 120 | }, 121 | "tags slice contains the tag for the pv name eks adds to PVs": { 122 | volume: types.Volume{ 123 | Tags: []types.Tag{ 124 | { 125 | Key: aws.String(eksPVTagName), 126 | Value: aws.String("pvc-1234567890"), 127 | }, 128 | }, 129 | }, 130 | expected: "pvc-1234567890", 131 | }, 132 | } 133 | 134 | for name, tt := range tests { 135 | t.Run(name, func(t *testing.T) { 136 | assert.Equal(t, tt.expected, NameFromVolume(tt.volume)) 137 | }) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/aws/s3/testdata/dimensions.csv: -------------------------------------------------------------------------------- 1 | APS1-DataTransfer-Out-Bytes,APS1,DataTransfer,Out,Bytes, 2 | APS1-Requests-Tier1,APS1,Requests-Tier1,Tier1,, 3 | APS1-Requests-Tier2,APS1,Requests-Tier2,Tier2,, 4 | APS1-TimedStorage-ByteHrs,APS1,TimedStorage,ByteHrs,, 5 | CAN1-DataTransfer-Out-Bytes,CAN1,DataTransfer,Out,Bytes, 6 | CAN1-Requests-Tier1,CAN1,Requests-Tier1,Tier1,, 7 | CAN1-Requests-Tier2,CAN1,Requests-Tier2,Tier2,, 8 | CAN1-TimedStorage-ByteHrs,CAN1,TimedStorage,ByteHrs,, 9 | EUC1-DataTransfer-Out-Bytes,EUC1,DataTransfer,Out,Bytes, 10 | EUC1-Requests-Tier1,EUC1,Requests-Tier1,Tier1,, 11 | EUC1-Requests-Tier2,EUC1,Requests-Tier2,Tier2,, 12 | EUC1-TimedStorage-ByteHrs,EUC1,TimedStorage,ByteHrs,, 13 | Requests-Tier1,,,,, 14 | Requests-Tier2,,,,, 15 | USE1-USE2-AWS-Out-Bytes,USE1,,AWS,Out,Bytes 16 | USE2-DataTransfer-Out-Bytes,USE2,DataTransfer,Out,Bytes, 17 | USE2-Requests-Tier1,USE2,Requests-Tier1,Tier1,, 18 | USE2-Requests-Tier2,USE2,Requests-Tier2,Tier2,, 19 | USE2-TimedStorage-ByteHrs,USE2,TimedStorage,ByteHrs,, 20 | USW2-DataTransfer-Out-Bytes,USW2,DataTransfer,Out,Bytes, 21 | USW2-Requests-Tier1,USW2,Requests-Tier1,Tier1,, 22 | USW2-Requests-Tier2,USW2,Requests-Tier2,Tier2,, 23 | USW2-TimedStorage-ByteHrs,USW2,TimedStorage,ByteHrs,, 24 | APS1-DataTransfer-In-Bytes,APS1,DataTransfer,In,Bytes, 25 | DataTransfer-In-Bytes,DataTransfer,In,Bytes,, 26 | DataTransfer-Out-Bytes,DataTransfer,Out,Bytes,, 27 | USE2-DataTransfer-In-Bytes,USE2,DataTransfer,In,Bytes, 28 | USW2-DataTransfer-In-Bytes,USW2,DataTransfer,In,Bytes, 29 | APS1-USE1-AWS-Out-Bytes,APS1,,AWS,Out,Bytes 30 | CAN1-USE1-AWS-Out-Bytes,CAN1,,AWS,Out,Bytes 31 | EUC1-USE1-AWS-Out-Bytes,EUC1,,AWS,Out,Bytes 32 | USE2-USE1-AWS-Out-Bytes,USE2,,AWS,Out,Bytes 33 | USW2-USE1-AWS-Out-Bytes,USW2,,AWS,Out,Bytes 34 | APS2-DataTransfer-In-Bytes,APS2,DataTransfer,In,Bytes, 35 | APS2-DataTransfer-Out-Bytes,APS2,DataTransfer,Out,Bytes, 36 | APS2-Requests-Tier1,APS2,Requests-Tier1,Tier1,, 37 | APS2-Requests-Tier2,APS2,Requests-Tier2,Tier2,, 38 | CAN1-DataTransfer-In-Bytes,CAN1,DataTransfer,In,Bytes, 39 | EUC1-DataTransfer-In-Bytes,EUC1,DataTransfer,In,Bytes, 40 | EUN1-DataTransfer-In-Bytes,EUN1,DataTransfer,In,Bytes, 41 | EUN1-DataTransfer-Out-Bytes,EUN1,DataTransfer,Out,Bytes, 42 | EUN1-Requests-Tier1,EUN1,Requests-Tier1,Tier1,, 43 | EUN1-Requests-Tier2,EUN1,Requests-Tier2,Tier2,, 44 | SAE1-DataTransfer-In-Bytes,SAE1,DataTransfer,In,Bytes, 45 | SAE1-DataTransfer-Out-Bytes,SAE1,DataTransfer,Out,Bytes, 46 | SAE1-Requests-Tier1,SAE1,Requests-Tier1,Tier1,, 47 | SAE1-Requests-Tier2,SAE1,Requests-Tier2,Tier2,, 48 | APS2-USE1-AWS-Out-Bytes,APS2,,AWS,Out,Bytes 49 | EUN1-USE1-AWS-Out-Bytes,EUN1,,AWS,Out,Bytes 50 | SAE1-USE1-AWS-Out-Bytes,SAE1,,AWS,Out,Bytes 51 | APS2-TimedStorage-ByteHrs,APS2,TimedStorage,ByteHrs,, 52 | EUN1-TimedStorage-ByteHrs,EUN1,TimedStorage,ByteHrs,, 53 | SAE1-TimedStorage-ByteHrs,SAE1,TimedStorage,ByteHrs,, -------------------------------------------------------------------------------- /pkg/aws/services/README.md: -------------------------------------------------------------------------------- 1 | # Services 2 | 3 | This package contains a subset of AWS services that are being used from AWS SDK v2. 4 | The services should be interfaces that define the methods that are being used from the AWS SDK v2. 5 | We do this so that we can generate mocks for these services and use them in our tests. 6 | For example, see: 7 | - [mocks](../../mocks/aws/services) 8 | - [tests](../../pkg/aws/services/s3/s3_test.go) 9 | 10 | ## Generated Mocks 11 | 12 | The mocks for these services are generated using [mockery]() 13 | To generate mocks for these services, run the following command: 14 | 15 | ```bash 16 | make generate-mocks 17 | ``` 18 | -------------------------------------------------------------------------------- /pkg/aws/services/costexplorer/costexplorer.go: -------------------------------------------------------------------------------- 1 | package costexplorer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/costexplorer" 7 | ) 8 | 9 | type CostExplorer interface { 10 | GetCostAndUsage(ctx context.Context, params *costexplorer.GetCostAndUsageInput, optFns ...func(*costexplorer.Options)) (*costexplorer.GetCostAndUsageOutput, error) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/aws/services/ec2/ec2.go: -------------------------------------------------------------------------------- 1 | package ec2 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/ec2" 7 | ) 8 | 9 | type EC2 interface { 10 | DescribeInstances(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) 11 | DescribeRegions(ctx context.Context, e *ec2.DescribeRegionsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeRegionsOutput, error) 12 | DescribeSpotPriceHistory(ctx context.Context, input *ec2.DescribeSpotPriceHistoryInput, optFns ...func(*ec2.Options)) (*ec2.DescribeSpotPriceHistoryOutput, error) 13 | DescribeVolumes(context.Context, *ec2.DescribeVolumesInput, ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/aws/services/pricing/pricing.go: -------------------------------------------------------------------------------- 1 | package pricing 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go-v2/service/pricing" 7 | ) 8 | 9 | type Pricing interface { 10 | GetProducts(ctx context.Context, params *pricing.GetProductsInput, optFns ...func(*pricing.Options)) (*pricing.GetProductsOutput, error) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/azure/README.md: -------------------------------------------------------------------------------- 1 | # Azure 2 | 3 | This package is responsible for collecting and exporting cost metrics associated with Azure. 4 | -------------------------------------------------------------------------------- /pkg/azure/aks/README.md: -------------------------------------------------------------------------------- 1 | # AKS Module 2 | 3 | This module is responsible for collecting pricing information for AKS clusters. 4 | 5 | 6 | ## Pricing Map 7 | 8 | Because Azure has not yet implemented the pricing API into it's SDK :shame:, this package uses the 3rd party [Azure Retail Prices SDK](https://github.com/gomodules/azure-retail-prices-sdk-for-go) to grab the prices. 9 | 10 | This is based on [Azure's pricing model](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/windows/), where different prices are determined by a combination of those factors. 11 | 12 | ### Price Stratification 13 | 14 | The PricingMap is built out with the following structure: 15 | 16 | ``` 17 | root -> { 18 | regionName -> { 19 | machinePriority -> { 20 | operatingSystem -> { 21 | skuName -> information 22 | } 23 | } 24 | } 25 | } 26 | ``` 27 | 28 | That way, in order to uniquely identify a price, we will have to have the following attributes of any VM: 29 | - the region it is deployed into 30 | - it's priority (spot or on-demand) 31 | - the operating system it is running 32 | - it's SKU (e.g. `E8-4as_v4`) 33 | 34 | ## Machine Map 35 | 36 | In order to collect the VMs that are relevant for AKS, this package grabs a list of relevant machines in the following way: 37 | 38 | - A list of AKS clusters for the subscription are obtained 39 | - Each VMSS (Virtual Machine Scale Set) that creates worker nodes is collected for the resource groups that Azure uses to provision VMs 40 | - Each VM for the VMSS is collected 41 | - VMSS and their metadata (namely their pricing SKU) is stored in a map with the following structure: 42 | 43 | ``` 44 | root -> { 45 | vmUniqueName -> information 46 | } 47 | ``` 48 | 49 | In parallel, machine types and their relevant info are collected, stored in a map with the following structure: 50 | 51 | ``` 52 | root -> { 53 | region -> { 54 | sizeIdentifier -> sizingInformation 55 | } 56 | } 57 | 58 | 59 | ``` 60 | 61 | The information contained on the VM Information is enough to uniquely identify both the machine itself and the price that accompanies it. The sizing information allows CPU and Memory price calculation. 62 | -------------------------------------------------------------------------------- /pkg/azure/aks/aks.go: -------------------------------------------------------------------------------- 1 | package aks 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log/slog" 8 | "time" 9 | 10 | "github.com/grafana/cloudcost-exporter/pkg/utils" 11 | 12 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 13 | "github.com/prometheus/client_golang/prometheus" 14 | 15 | "github.com/grafana/cloudcost-exporter/pkg/azure/azureClientWrapper" 16 | "github.com/grafana/cloudcost-exporter/pkg/provider" 17 | 18 | cloudcost_exporter "github.com/grafana/cloudcost-exporter" 19 | ) 20 | 21 | const ( 22 | subsystem = "azure_aks" 23 | AZ_API_VERSION string = "2023-01-01-preview" // using latest API Version https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices 24 | ) 25 | 26 | type MachineOperatingSystem int 27 | 28 | const ( 29 | Linux MachineOperatingSystem = iota 30 | Windows 31 | ) 32 | 33 | var machineOperatingSystemNames [2]string = [2]string{"Linux", "Windows"} 34 | 35 | func (mo MachineOperatingSystem) String() string { 36 | return machineOperatingSystemNames[mo] 37 | } 38 | 39 | type MachinePriority int 40 | 41 | const ( 42 | OnDemand MachinePriority = iota 43 | Spot 44 | ) 45 | 46 | var machinePriorityNames [2]string = [2]string{"ondemand", "spot"} 47 | 48 | func (mp MachinePriority) String() string { 49 | return machinePriorityNames[mp] 50 | } 51 | 52 | // Errors 53 | var ( 54 | ErrClientCreationFailure = errors.New("failed to create client") 55 | ErrPageAdvanceFailure = errors.New("failed to advance page") 56 | ErrPriceStorePopulationFailure = errors.New("failed to populate price store") 57 | ErrMachineStorePopulationFailure = errors.New("failed to populate machine store") 58 | ErrVmPriceRetrievalFailure = errors.New("failed to retrieve price info for VM") 59 | ) 60 | 61 | // Prometheus Metrics 62 | var ( 63 | InstanceCPUHourlyCostDesc = utils.GenerateDesc( 64 | cloudcost_exporter.MetricPrefix, 65 | subsystem, 66 | utils.InstanceCPUCostSuffix, 67 | "The cpu cost a a compute instance in USD/(core*h)", 68 | []string{"instance", "region", "machine_type", "family", "cluster_name", "price_tier", "operating_system"}, 69 | ) 70 | InstanceMemoryHourlyCostDesc = utils.GenerateDesc( 71 | cloudcost_exporter.MetricPrefix, 72 | subsystem, 73 | utils.InstanceMemoryCostSuffix, 74 | "The memory cost of a compute instance in USD/(GiB*h)", 75 | []string{"instance", "region", "machine_type", "family", "cluster_name", "price_tier", "operating_system"}, 76 | ) 77 | InstanceTotalHourlyCostDesc = utils.GenerateDesc( 78 | cloudcost_exporter.MetricPrefix, 79 | subsystem, 80 | utils.InstanceTotalCostSuffix, 81 | "The total cost of a compute instance in USD/h", 82 | []string{"instance", "region", "machine_type", "family", "cluster_name", "price_tier", "operating_system"}, 83 | ) 84 | ) 85 | 86 | // Collector is a prometheus collector that collects metrics from AKS clusters. 87 | type Collector struct { 88 | context context.Context 89 | logger *slog.Logger 90 | 91 | PriceStore *PriceStore 92 | MachineStore *MachineStore 93 | } 94 | 95 | type Config struct { 96 | Logger *slog.Logger 97 | Credentials *azidentity.DefaultAzureCredential 98 | 99 | SubscriptionId string 100 | } 101 | 102 | func New(ctx context.Context, cfg *Config, azClientWrapper azureClientWrapper.AzureClient) (*Collector, error) { 103 | logger := cfg.Logger.With("collector", "aks") 104 | priceStore := NewPricingStore(ctx, logger, azClientWrapper) 105 | machineStore, err := NewMachineStore(ctx, logger, azClientWrapper) 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | priceTicker := time.NewTicker(priceRefreshInterval) 111 | machineTicker := time.NewTicker(machineRefreshInterval) 112 | 113 | go func(ctx context.Context) { 114 | for { 115 | select { 116 | case <-ctx.Done(): 117 | return 118 | case <-priceTicker.C: 119 | priceStore.PopulatePriceStore(ctx) 120 | } 121 | } 122 | }(ctx) 123 | go func(ctx context.Context) { 124 | for { 125 | select { 126 | case <-ctx.Done(): 127 | return 128 | case <-machineTicker.C: 129 | machineStore.PopulateMachineStore(ctx) 130 | } 131 | } 132 | }(ctx) 133 | 134 | return &Collector{ 135 | context: ctx, 136 | logger: logger, 137 | 138 | PriceStore: priceStore, 139 | MachineStore: machineStore, 140 | }, nil 141 | } 142 | 143 | // CollectMetrics is a no-op function that satisfies the provider.Collector interface. 144 | // Deprecated: CollectMetrics is deprecated and will be removed in a future release. 145 | func (c *Collector) CollectMetrics(_ chan<- prometheus.Metric) float64 { 146 | return 0 147 | } 148 | 149 | func (c *Collector) getMachinePrices(vmId string) (*MachineSku, error) { 150 | vmInfo, err := c.MachineStore.getVmInfoByVmId(vmId) 151 | if err != nil { 152 | return nil, err 153 | } 154 | 155 | prices, err := c.PriceStore.getPriceInfoFromVmInfo(vmInfo) 156 | if err != nil { 157 | return nil, fmt.Errorf("%w: %w", err, ErrVmPriceRetrievalFailure) 158 | } 159 | 160 | return prices, nil 161 | } 162 | 163 | // Collect satisfies the provider.Collector interface. 164 | func (c *Collector) Collect(ch chan<- prometheus.Metric) error { 165 | c.logger.Info("collecting metrics") 166 | now := time.Now() 167 | 168 | machineList := c.MachineStore.GetListOfVmsForSubscription() 169 | 170 | for _, vmInfo := range machineList { 171 | vmId := vmInfo.Id 172 | price, err := c.getMachinePrices(vmId) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | labelValues := []string{ 178 | vmInfo.Name, 179 | vmInfo.Region, 180 | vmInfo.MachineTypeSku, 181 | vmInfo.MachineFamily, 182 | vmInfo.OwningCluster, 183 | vmInfo.Priority.String(), 184 | vmInfo.OperatingSystem.String(), 185 | } 186 | 187 | ch <- prometheus.MustNewConstMetric(InstanceCPUHourlyCostDesc, prometheus.GaugeValue, price.MachinePricesBreakdown.PricePerCore, labelValues...) 188 | ch <- prometheus.MustNewConstMetric(InstanceMemoryHourlyCostDesc, prometheus.GaugeValue, price.MachinePricesBreakdown.PricePerGiB, labelValues...) 189 | ch <- prometheus.MustNewConstMetric(InstanceTotalHourlyCostDesc, prometheus.GaugeValue, price.RetailPrice, labelValues...) 190 | } 191 | 192 | c.logger.LogAttrs(c.context, slog.LevelInfo, "metrics collected", slog.Duration("duration", time.Since(now))) 193 | return nil 194 | } 195 | 196 | func (c *Collector) Describe(ch chan<- *prometheus.Desc) error { 197 | ch <- InstanceCPUHourlyCostDesc 198 | ch <- InstanceMemoryHourlyCostDesc 199 | ch <- InstanceTotalHourlyCostDesc 200 | return nil 201 | } 202 | 203 | func (c *Collector) Name() string { 204 | return subsystem 205 | } 206 | 207 | func (c *Collector) Register(_ provider.Registry) error { 208 | c.logger.LogAttrs(c.context, slog.LevelInfo, "registering collector") 209 | return nil 210 | } 211 | -------------------------------------------------------------------------------- /pkg/azure/azure.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log/slog" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 12 | "github.com/prometheus/client_golang/prometheus" 13 | 14 | "github.com/grafana/cloudcost-exporter/pkg/azure/aks" 15 | "github.com/grafana/cloudcost-exporter/pkg/azure/azureClientWrapper" 16 | "github.com/grafana/cloudcost-exporter/pkg/provider" 17 | 18 | cloudcost_exporter "github.com/grafana/cloudcost-exporter" 19 | ) 20 | 21 | const ( 22 | subsystem = "azure" 23 | ) 24 | 25 | var ( 26 | InvalidSubscriptionId = errors.New("subscription id was invalid") 27 | ) 28 | 29 | var ( 30 | collectorDurationDesc = prometheus.NewDesc( 31 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "collector", "last_scrape_duration_seconds"), 32 | "Duration of the last scrape in seconds.", 33 | []string{"provider", "collector"}, 34 | nil, 35 | ) 36 | collectorLastScrapeErrorDesc = prometheus.NewDesc( 37 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "collector", "last_scrape_error"), 38 | "Was the last scrape an error. 1 indicates an error.", 39 | []string{"provider", "collector"}, 40 | nil, 41 | ) 42 | collectorLastScrapeTime = prometheus.NewDesc( 43 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "collector", "last_scrape_time"), 44 | "Time of the last scrape.", 45 | []string{"provider", "collector"}, 46 | nil, 47 | ) 48 | collectorScrapesTotalCounter = prometheus.NewCounterVec( 49 | prometheus.CounterOpts{ 50 | Name: prometheus.BuildFQName(cloudcost_exporter.ExporterName, "collector", "scrapes_total"), 51 | Help: "Total number of scrapes for a collector.", 52 | }, 53 | []string{"provider", "collector"}, 54 | ) 55 | collectorSuccessDesc = prometheus.NewDesc( 56 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, subsystem, "collector_success"), 57 | "Was the last scrape of the Azure metrics successful.", 58 | []string{"collector"}, 59 | nil, 60 | ) 61 | providerLastScrapeDurationDesc = prometheus.NewDesc( 62 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "", "last_scrape_duration_seconds"), 63 | "Duration of the last scrape in seconds.", 64 | []string{"provider"}, 65 | nil, 66 | ) 67 | providerLastScrapeErrorDesc = prometheus.NewDesc( 68 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "", "last_scrape_error"), 69 | "Was the last scrape an error. 1 indicates an error.", 70 | []string{"provider"}, 71 | nil, 72 | ) 73 | providerLastScrapeTime = prometheus.NewDesc( 74 | prometheus.BuildFQName(cloudcost_exporter.ExporterName, "", "last_scrape_time"), 75 | "Time of the last scrape.", 76 | []string{"provider"}, 77 | nil, 78 | ) 79 | providerScrapesTotalCounter = prometheus.NewCounterVec( 80 | prometheus.CounterOpts{ 81 | Name: prometheus.BuildFQName(cloudcost_exporter.ExporterName, "", "scrapes_total"), 82 | Help: "Total number of scrapes.", 83 | }, 84 | []string{"provider"}, 85 | ) 86 | ) 87 | 88 | type Azure struct { 89 | context context.Context 90 | logger *slog.Logger 91 | 92 | subscriptionId string 93 | azCredentials *azidentity.DefaultAzureCredential 94 | 95 | collectorTimeout time.Duration 96 | collectors []provider.Collector 97 | } 98 | 99 | type Config struct { 100 | Logger *slog.Logger 101 | 102 | SubscriptionId string 103 | 104 | CollectorTimeout time.Duration 105 | Services []string 106 | } 107 | 108 | func New(ctx context.Context, config *Config) (*Azure, error) { 109 | logger := config.Logger.With("provider", subsystem) 110 | collectors := []provider.Collector{} 111 | 112 | if config.SubscriptionId == "" { 113 | logger.LogAttrs(ctx, slog.LevelError, "subscription id was invalid") 114 | return nil, InvalidSubscriptionId 115 | } 116 | 117 | creds, err := azidentity.NewDefaultAzureCredential(nil) 118 | if err != nil { 119 | logger.LogAttrs(ctx, slog.LevelError, "failed to create azure credentials", slog.String("err", err.Error())) 120 | return nil, err 121 | } 122 | 123 | azClientWrapper, err := azureClientWrapper.NewAzureClientWrapper(logger, config.SubscriptionId, creds) 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | // Collector Registration 129 | for _, svc := range config.Services { 130 | switch strings.ToUpper(svc) { 131 | case "AKS": 132 | collector, err := aks.New(ctx, &aks.Config{ 133 | Credentials: creds, 134 | SubscriptionId: config.SubscriptionId, 135 | Logger: logger, 136 | }, azClientWrapper) 137 | if err != nil { 138 | return nil, err 139 | } 140 | collectors = append(collectors, collector) 141 | default: 142 | logger.LogAttrs(ctx, slog.LevelInfo, "unknown service", slog.String("service", svc)) 143 | } 144 | } 145 | 146 | return &Azure{ 147 | context: ctx, 148 | logger: logger, 149 | 150 | subscriptionId: config.SubscriptionId, 151 | azCredentials: creds, 152 | 153 | collectorTimeout: config.CollectorTimeout, 154 | collectors: collectors, 155 | }, nil 156 | } 157 | 158 | func (a *Azure) RegisterCollectors(registry provider.Registry) error { 159 | a.logger.LogAttrs(a.context, slog.LevelInfo, "registering collectors", slog.Int("NumOfCollectors", len(a.collectors))) 160 | 161 | registry.MustRegister(collectorScrapesTotalCounter) 162 | for _, c := range a.collectors { 163 | err := c.Register(registry) 164 | if err != nil { 165 | return err 166 | } 167 | } 168 | 169 | return nil 170 | } 171 | 172 | func (a *Azure) Describe(ch chan<- *prometheus.Desc) { 173 | ch <- collectorLastScrapeErrorDesc 174 | ch <- collectorDurationDesc 175 | ch <- providerLastScrapeErrorDesc 176 | ch <- providerLastScrapeDurationDesc 177 | ch <- collectorLastScrapeTime 178 | ch <- providerLastScrapeTime 179 | ch <- collectorSuccessDesc 180 | for _, c := range a.collectors { 181 | if err := c.Describe(ch); err != nil { 182 | a.logger.LogAttrs(a.context, slog.LevelInfo, "error describing collector", slog.String("collector", c.Name()), slog.String("error", err.Error())) 183 | } 184 | } 185 | } 186 | 187 | func (a *Azure) Collect(ch chan<- prometheus.Metric) { 188 | // TODO - implement collector context 189 | _, cancel := context.WithTimeout(a.context, a.collectorTimeout) 190 | defer cancel() 191 | 192 | providerStart := time.Now() 193 | wg := &sync.WaitGroup{} 194 | wg.Add(len(a.collectors)) 195 | 196 | for _, c := range a.collectors { 197 | go func(c provider.Collector) { 198 | collectorStart := time.Now() 199 | defer wg.Done() 200 | collectorErrors := 0.0 201 | if err := c.Collect(ch); err != nil { 202 | collectorErrors = 1.0 203 | a.logger.LogAttrs(a.context, slog.LevelInfo, "error collecting metrics from collector", slog.String("collector", c.Name()), slog.String("error", err.Error())) 204 | } 205 | ch <- prometheus.MustNewConstMetric(collectorLastScrapeErrorDesc, prometheus.GaugeValue, collectorErrors, subsystem, c.Name()) 206 | ch <- prometheus.MustNewConstMetric(collectorDurationDesc, prometheus.GaugeValue, time.Since(collectorStart).Seconds(), subsystem, c.Name()) 207 | ch <- prometheus.MustNewConstMetric(collectorLastScrapeTime, prometheus.GaugeValue, float64(time.Now().Unix()), subsystem, c.Name()) 208 | ch <- prometheus.MustNewConstMetric(collectorSuccessDesc, prometheus.GaugeValue, collectorErrors, c.Name()) 209 | collectorScrapesTotalCounter.WithLabelValues(subsystem, c.Name()).Inc() 210 | }(c) 211 | 212 | } 213 | wg.Wait() 214 | 215 | ch <- prometheus.MustNewConstMetric(providerLastScrapeErrorDesc, prometheus.GaugeValue, 0.0, subsystem) 216 | ch <- prometheus.MustNewConstMetric(providerLastScrapeDurationDesc, prometheus.GaugeValue, time.Since(providerStart).Seconds(), subsystem) 217 | ch <- prometheus.MustNewConstMetric(providerLastScrapeTime, prometheus.GaugeValue, float64(time.Now().Unix()), subsystem) 218 | providerScrapesTotalCounter.WithLabelValues(subsystem).Inc() 219 | } 220 | -------------------------------------------------------------------------------- /pkg/azure/azureClientWrapper/azureClientWrapper.go: -------------------------------------------------------------------------------- 1 | package azureClientWrapper 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log/slog" 8 | 9 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 10 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4" 11 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5" 12 | "github.com/Azure/go-autorest/autorest/to" 13 | 14 | retailPriceSdk "gomodules.xyz/azure-retail-prices-sdk-for-go/sdk" 15 | ) 16 | 17 | var ( 18 | ErrClientCreationFailure = errors.New("failed to create client") 19 | ErrPageAdvanceFailure = errors.New("failed to advance page") 20 | ) 21 | 22 | type AzureClient interface { 23 | // Machine Store 24 | ListClustersInSubscription(context.Context) ([]*armcontainerservice.ManagedCluster, error) 25 | ListVirtualMachineScaleSetsOwnedVms(context.Context, string, string) ([]*armcompute.VirtualMachineScaleSetVM, error) 26 | ListVirtualMachineScaleSetsFromResourceGroup(context.Context, string) ([]*armcompute.VirtualMachineScaleSet, error) 27 | ListMachineTypesByLocation(context.Context, string) ([]*armcompute.VirtualMachineSize, error) 28 | 29 | // Price Store 30 | ListPrices(context.Context, *retailPriceSdk.RetailPricesClientListOptions) ([]*retailPriceSdk.ResourceSKU, error) 31 | } 32 | 33 | type AzClientWrapper struct { 34 | logger *slog.Logger 35 | 36 | azVMSizesClient *armcompute.VirtualMachineSizesClient 37 | azVMSSClient *armcompute.VirtualMachineScaleSetsClient 38 | azVMSSVmClient *armcompute.VirtualMachineScaleSetVMsClient 39 | azAksClient *armcontainerservice.ManagedClustersClient 40 | 41 | retailPricesClient *retailPriceSdk.RetailPricesClient 42 | } 43 | 44 | func NewAzureClientWrapper(logger *slog.Logger, subscriptionId string, credentials *azidentity.DefaultAzureCredential) (*AzClientWrapper, error) { 45 | ctx := context.TODO() 46 | 47 | computeClientFactory, err := armcompute.NewClientFactory(subscriptionId, credentials, nil) 48 | if err != nil { 49 | logger.LogAttrs(ctx, slog.LevelError, "unable to create compute client factory", slog.String("err", err.Error())) 50 | return nil, ErrClientCreationFailure 51 | } 52 | 53 | containerClientFactory, err := armcontainerservice.NewClientFactory(subscriptionId, credentials, nil) 54 | if err != nil { 55 | logger.LogAttrs(ctx, slog.LevelError, "unable to create container client factory", slog.String("err", err.Error())) 56 | return nil, ErrClientCreationFailure 57 | } 58 | 59 | retailPricesClient, err := retailPriceSdk.NewRetailPricesClient(nil) 60 | if err != nil { 61 | return nil, ErrClientCreationFailure 62 | } 63 | 64 | return &AzClientWrapper{ 65 | logger: logger.With("client", "azure"), 66 | 67 | azVMSizesClient: computeClientFactory.NewVirtualMachineSizesClient(), 68 | azVMSSClient: computeClientFactory.NewVirtualMachineScaleSetsClient(), 69 | azVMSSVmClient: computeClientFactory.NewVirtualMachineScaleSetVMsClient(), 70 | azAksClient: containerClientFactory.NewManagedClustersClient(), 71 | 72 | retailPricesClient: retailPricesClient, 73 | }, nil 74 | } 75 | 76 | func (a *AzClientWrapper) ListVirtualMachineScaleSetsOwnedVms(ctx context.Context, rgName, vmssName string) ([]*armcompute.VirtualMachineScaleSetVM, error) { 77 | logger := a.logger.With("pager", "listVirtualMachineScaleSetsOwnedVms") 78 | 79 | vmList := []*armcompute.VirtualMachineScaleSetVM{} 80 | 81 | opts := &armcompute.VirtualMachineScaleSetVMsClientListOptions{ 82 | Expand: to.StringPtr("instanceView"), 83 | } 84 | pager := a.azVMSSVmClient.NewListPager(rgName, vmssName, opts) 85 | for pager.More() { 86 | nextResult, err := pager.NextPage(ctx) 87 | if err != nil { 88 | logger.LogAttrs(ctx, slog.LevelError, "unable to advance page", slog.String("err", err.Error())) 89 | return nil, fmt.Errorf("%w: %w", ErrPageAdvanceFailure, err) 90 | } 91 | 92 | vmList = append(vmList, nextResult.Value...) 93 | } 94 | 95 | return vmList, nil 96 | } 97 | 98 | func (a *AzClientWrapper) ListVirtualMachineScaleSetsFromResourceGroup(ctx context.Context, rgName string) ([]*armcompute.VirtualMachineScaleSet, error) { 99 | logger := a.logger.With("pager", "listVirtualMachineScaleSetsFromResourceGroup") 100 | 101 | vmssList := []*armcompute.VirtualMachineScaleSet{} 102 | 103 | pager := a.azVMSSClient.NewListPager(rgName, nil) 104 | for pager.More() { 105 | nextResult, err := pager.NextPage(ctx) 106 | if err != nil { 107 | logger.LogAttrs(ctx, slog.LevelError, "unable to advance page", slog.String("err", err.Error())) 108 | return nil, fmt.Errorf("%w: %w", ErrPageAdvanceFailure, err) 109 | } 110 | 111 | vmssList = append(vmssList, nextResult.Value...) 112 | } 113 | 114 | return vmssList, nil 115 | } 116 | 117 | func (a *AzClientWrapper) ListClustersInSubscription(ctx context.Context) ([]*armcontainerservice.ManagedCluster, error) { 118 | logger := a.logger.With("pager", "listClustersInSubscription") 119 | 120 | clusterList := []*armcontainerservice.ManagedCluster{} 121 | 122 | pager := a.azAksClient.NewListPager(nil) 123 | for pager.More() { 124 | page, err := pager.NextPage(ctx) 125 | if err != nil { 126 | logger.LogAttrs(ctx, slog.LevelError, "unable to advance page", slog.String("err", err.Error())) 127 | return nil, fmt.Errorf("%w: %w", ErrPageAdvanceFailure, err) 128 | } 129 | clusterList = append(clusterList, page.Value...) 130 | } 131 | 132 | return clusterList, nil 133 | } 134 | 135 | func (a *AzClientWrapper) ListMachineTypesByLocation(ctx context.Context, region string) ([]*armcompute.VirtualMachineSize, error) { 136 | logger := a.logger.With("pager", "listMachineTypesByLocation") 137 | 138 | machineList := []*armcompute.VirtualMachineSize{} 139 | 140 | pager := a.azVMSizesClient.NewListPager(region, nil) 141 | for pager.More() { 142 | nextResult, err := pager.NextPage(ctx) 143 | if err != nil { 144 | logger.LogAttrs(ctx, slog.LevelError, "unable to advance page", slog.String("err", err.Error())) 145 | return nil, fmt.Errorf("%w: %w", ErrPageAdvanceFailure, err) 146 | } 147 | 148 | machineList = append(machineList, nextResult.Value...) 149 | } 150 | 151 | return machineList, nil 152 | } 153 | 154 | func (a *AzClientWrapper) ListPrices(ctx context.Context, searchOptions *retailPriceSdk.RetailPricesClientListOptions) ([]*retailPriceSdk.ResourceSKU, error) { 155 | logger := a.logger.With("pager", "listPrices") 156 | 157 | logger.LogAttrs(ctx, slog.LevelDebug, "populating prices with opts", slog.String("opts", fmt.Sprintf("%+v", searchOptions))) 158 | prices := []*retailPriceSdk.ResourceSKU{} 159 | 160 | pager := a.retailPricesClient.NewListPager(searchOptions) 161 | for pager.More() { 162 | page, err := pager.NextPage(ctx) 163 | if err != nil { 164 | logger.LogAttrs(ctx, slog.LevelError, "unable to advance page", slog.String("err", err.Error())) 165 | return nil, fmt.Errorf("%w: %w", ErrPageAdvanceFailure, err) 166 | } 167 | 168 | for _, v := range page.Items { 169 | prices = append(prices, &v) 170 | } 171 | } 172 | 173 | return prices, nil 174 | } 175 | -------------------------------------------------------------------------------- /pkg/azure/azure_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "os" 7 | "testing" 8 | 9 | mock_provider "github.com/grafana/cloudcost-exporter/pkg/provider/mocks" 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/stretchr/testify/assert" 12 | "go.uber.org/mock/gomock" 13 | ) 14 | 15 | var ( 16 | parentCtx context.Context = context.TODO() 17 | testLogger *slog.Logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) 18 | ) 19 | 20 | func Test_New(t *testing.T) { 21 | testTable := map[string]struct { 22 | expectedErr error 23 | subId string 24 | }{ 25 | "no subscription ID": { 26 | expectedErr: InvalidSubscriptionId, 27 | subId: "", 28 | }, 29 | 30 | "base case": { 31 | expectedErr: nil, 32 | subId: "asdf-1234", 33 | }, 34 | } 35 | 36 | for name, tc := range testTable { 37 | t.Run(name, func(t *testing.T) { 38 | a, err := New(parentCtx, &Config{ 39 | Logger: testLogger, 40 | SubscriptionId: tc.subId, 41 | }) 42 | if tc.expectedErr != nil { 43 | assert.ErrorIs(t, err, tc.expectedErr) 44 | return 45 | } 46 | assert.NoError(t, err) 47 | assert.NotNil(t, a) 48 | }) 49 | } 50 | } 51 | 52 | func Test_RegisterCollectors(t *testing.T) { 53 | ctrl := gomock.NewController(t) 54 | defer ctrl.Finish() 55 | 56 | mockRegistry := mock_provider.NewMockRegistry(ctrl) 57 | 58 | testCases := map[string]struct { 59 | mockCollectors []*mock_provider.MockCollector 60 | expectedErr error 61 | }{ 62 | "no collectors": { 63 | mockCollectors: []*mock_provider.MockCollector{}, 64 | }, 65 | "AKS collector": { 66 | mockCollectors: []*mock_provider.MockCollector{ 67 | mock_provider.NewMockCollector(ctrl), 68 | }, 69 | }, 70 | "AKS and future storage collector": { 71 | mockCollectors: []*mock_provider.MockCollector{ 72 | mock_provider.NewMockCollector(ctrl), 73 | mock_provider.NewMockCollector(ctrl), 74 | }, 75 | }, 76 | } 77 | 78 | for name, tc := range testCases { 79 | t.Run(name, func(t *testing.T) { 80 | azProvider := &Azure{ 81 | logger: testLogger, 82 | context: parentCtx, 83 | } 84 | for _, c := range tc.mockCollectors { 85 | call := c.EXPECT().Register(gomock.Any()).AnyTimes() 86 | call.Return(nil) 87 | 88 | azProvider.collectors = append(azProvider.collectors, c) 89 | } 90 | 91 | mockRegistry.EXPECT().MustRegister(gomock.Any()).Times(1) 92 | err := azProvider.RegisterCollectors(mockRegistry) 93 | assert.Equal(t, err, tc.expectedErr) 94 | }) 95 | } 96 | } 97 | 98 | func Test_CollectMetrics(t *testing.T) { 99 | ctrl := gomock.NewController(t) 100 | defer ctrl.Finish() 101 | 102 | ch := make(chan prometheus.Metric) 103 | testCases := map[string]struct { 104 | mockCollectors []*mock_provider.MockCollector 105 | expectedErr error 106 | }{ 107 | "base case": { 108 | mockCollectors: []*mock_provider.MockCollector{ 109 | mock_provider.NewMockCollector(ctrl), 110 | }, 111 | }, 112 | } 113 | 114 | for name, tc := range testCases { 115 | t.Run(name, func(t *testing.T) { 116 | go func() { 117 | // no metrics are generated here, so loop through to avoid 118 | // process hang waiting on metrics that will never come 119 | for range ch { 120 | } 121 | }() 122 | 123 | azProvider := &Azure{ 124 | logger: testLogger, 125 | context: parentCtx, 126 | } 127 | for _, c := range tc.mockCollectors { 128 | c.EXPECT().Collect(gomock.Any()).Times(1) 129 | c.EXPECT().Name().AnyTimes() 130 | 131 | azProvider.collectors = append(azProvider.collectors, c) 132 | } 133 | 134 | azProvider.Collect(ch) 135 | }) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pkg/google/README.md: -------------------------------------------------------------------------------- 1 | # Google 2 | 3 | This package contains the Google Cloud Platform (GCP) exporter and reporter. 4 | `gcp.go` is responsible for setting up the GCP session and starting the collection process. 5 | The module is built upon the [google-cloud-go](https://github.com/googleapis/google-cloud-go) library and uses the GCP Billing API to collect cost data. 6 | Pricing data is fetched from the [GCP Pricing API](Pricing data is fetched from the [GCP Pricing API](https://cloud.google.com/billing/docs/how-to/understanding-costs#pricing). 7 | -------------------------------------------------------------------------------- /pkg/google/billing/billing.go: -------------------------------------------------------------------------------- 1 | package billing 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | 8 | billingv1 "cloud.google.com/go/billing/apiv1" 9 | "cloud.google.com/go/billing/apiv1/billingpb" 10 | "google.golang.org/api/iterator" 11 | ) 12 | 13 | var ServiceNotFound = errors.New("service not found") 14 | 15 | // GetServiceName will search for a service by the display name and return the full name. 16 | // The full name is need by the GetPricing method to collect all the pricing information for a given service. 17 | func GetServiceName(ctx context.Context, billingService *billingv1.CloudCatalogClient, name string) (string, error) { 18 | serviceIterator := billingService.ListServices(ctx, &billingpb.ListServicesRequest{PageSize: 5000}) 19 | for { 20 | service, err := serviceIterator.Next() 21 | if err != nil { 22 | if errors.Is(err, iterator.Done) { 23 | break 24 | } 25 | return "", err 26 | } 27 | if service.DisplayName == name { 28 | return service.Name, nil 29 | } 30 | } 31 | return "", ServiceNotFound 32 | } 33 | 34 | // GetPricing will collect all the pricing information for a given service and return a list of skus. 35 | func GetPricing(ctx context.Context, billingService *billingv1.CloudCatalogClient, serviceName string) []*billingpb.Sku { 36 | var skus []*billingpb.Sku 37 | skuIterator := billingService.ListSkus(ctx, &billingpb.ListSkusRequest{Parent: serviceName}) 38 | for { 39 | sku, err := skuIterator.Next() 40 | if err != nil { 41 | if errors.Is(err, iterator.Done) { 42 | break 43 | } 44 | log.Println(err) // keep going if we get an error 45 | } 46 | skus = append(skus, sku) 47 | } 48 | return skus 49 | } 50 | -------------------------------------------------------------------------------- /pkg/google/gcp_test.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | "os" 7 | "strings" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | "go.uber.org/mock/gomock" 15 | 16 | "github.com/grafana/cloudcost-exporter/pkg/provider" 17 | mock_provider "github.com/grafana/cloudcost-exporter/pkg/provider/mocks" 18 | "github.com/grafana/cloudcost-exporter/pkg/utils" 19 | ) 20 | 21 | var logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) 22 | 23 | func Test_RegisterCollectors(t *testing.T) { 24 | tests := map[string]struct { 25 | numCollectors int 26 | register func(r provider.Registry) error 27 | expectedError error 28 | }{ 29 | "no error if no collectors": {}, 30 | "bubble-up single collector error": { 31 | numCollectors: 1, 32 | register: func(r provider.Registry) error { 33 | return fmt.Errorf("test register error") 34 | }, 35 | expectedError: fmt.Errorf("test register error"), 36 | }, 37 | "two collectors with no errors": { 38 | numCollectors: 2, 39 | register: func(r provider.Registry) error { return nil }, 40 | }, 41 | } 42 | for name, tt := range tests { 43 | t.Run(name, func(t *testing.T) { 44 | ctrl := gomock.NewController(t) 45 | r := mock_provider.NewMockRegistry(ctrl) 46 | r.EXPECT().MustRegister(gomock.Any()).AnyTimes() 47 | 48 | c := mock_provider.NewMockCollector(ctrl) 49 | if tt.register != nil { 50 | c.EXPECT().Register(r).DoAndReturn(tt.register).Times(tt.numCollectors) 51 | } 52 | gcp := &GCP{ 53 | config: &Config{}, 54 | collectors: []provider.Collector{}, 55 | logger: logger, 56 | } 57 | for range tt.numCollectors { 58 | gcp.collectors = append(gcp.collectors, c) 59 | } 60 | err := gcp.RegisterCollectors(r) 61 | if tt.expectedError != nil { 62 | require.EqualError(t, err, tt.expectedError.Error()) 63 | return 64 | } 65 | require.NoError(t, err) 66 | }) 67 | } 68 | } 69 | 70 | func TestGCP_CollectMetrics(t *testing.T) { 71 | tests := map[string]struct { 72 | numCollectors int 73 | collect func(chan<- prometheus.Metric) error 74 | expectedMetrics []*utils.MetricResult 75 | }{ 76 | "no error if no collectors": { 77 | numCollectors: 0, 78 | expectedMetrics: []*utils.MetricResult{ 79 | { 80 | FqName: "cloudcost_exporter_last_scrape_error", 81 | Labels: utils.LabelMap{"provider": "gcp"}, 82 | Value: 0, 83 | MetricType: prometheus.GaugeValue, 84 | }, 85 | }, 86 | }, 87 | "bubble-up single collector error": { 88 | numCollectors: 1, 89 | collect: func(chan<- prometheus.Metric) error { 90 | return fmt.Errorf("test collect error") 91 | }, 92 | expectedMetrics: []*utils.MetricResult{ 93 | { 94 | FqName: "cloudcost_exporter_collector_last_scrape_error", 95 | Labels: utils.LabelMap{"provider": "gcp", "collector": "test"}, 96 | Value: 1, 97 | MetricType: prometheus.GaugeValue, 98 | }, 99 | { 100 | FqName: "cloudcost_exporter_last_scrape_error", 101 | Labels: utils.LabelMap{"provider": "gcp"}, 102 | Value: 0, 103 | MetricType: prometheus.GaugeValue, 104 | }, 105 | }, 106 | }, 107 | "two collectors with no errors": { 108 | numCollectors: 2, 109 | collect: func(chan<- prometheus.Metric) error { return nil }, 110 | expectedMetrics: []*utils.MetricResult{ 111 | { 112 | FqName: "cloudcost_exporter_collector_last_scrape_error", 113 | Labels: utils.LabelMap{"provider": "gcp", "collector": "test"}, 114 | Value: 0, 115 | MetricType: prometheus.GaugeValue, 116 | }, 117 | { 118 | FqName: "cloudcost_exporter_collector_last_scrape_error", 119 | Labels: utils.LabelMap{"provider": "gcp", "collector": "test"}, 120 | Value: 0, 121 | MetricType: prometheus.GaugeValue, 122 | }, 123 | { 124 | FqName: "cloudcost_exporter_last_scrape_error", 125 | Labels: utils.LabelMap{"provider": "gcp"}, 126 | Value: 0, 127 | MetricType: prometheus.GaugeValue, 128 | }}, 129 | }, 130 | } 131 | for name, tt := range tests { 132 | t.Run(name, func(t *testing.T) { 133 | ch := make(chan prometheus.Metric) 134 | 135 | ctrl := gomock.NewController(t) 136 | c := mock_provider.NewMockCollector(ctrl) 137 | registry := mock_provider.NewMockRegistry(ctrl) 138 | registry.EXPECT().MustRegister(gomock.Any()).AnyTimes() 139 | if tt.collect != nil { 140 | c.EXPECT().Name().Return("test").AnyTimes() 141 | // TODO: @pokom need to figure out why _sometimes_ this fails if we set it to *.Times(tt.numCollectors) 142 | c.EXPECT().Collect(ch).DoAndReturn(tt.collect).AnyTimes() 143 | c.EXPECT().Register(registry).Return(nil).AnyTimes() 144 | } 145 | gcp := &GCP{ 146 | config: &Config{}, 147 | collectors: []provider.Collector{}, 148 | logger: logger, 149 | } 150 | 151 | for range tt.numCollectors { 152 | gcp.collectors = append(gcp.collectors, c) 153 | } 154 | 155 | wg := sync.WaitGroup{} 156 | 157 | wg.Add(1) 158 | go func() { 159 | gcp.Collect(ch) 160 | close(ch) 161 | }() 162 | wg.Done() 163 | 164 | wg.Wait() 165 | var metrics []*utils.MetricResult 166 | var ignoreMetric = func(metricName string) bool { 167 | ignoredMetricSuffix := []string{ 168 | "duration_seconds", 169 | "last_scrape_time", 170 | } 171 | for _, suffix := range ignoredMetricSuffix { 172 | if strings.Contains(metricName, suffix) { 173 | return true 174 | } 175 | } 176 | 177 | return false 178 | } 179 | for m := range ch { 180 | metric := utils.ReadMetrics(m) 181 | if ignoreMetric(metric.FqName) { 182 | continue 183 | } 184 | metrics = append(metrics, metric) 185 | } 186 | assert.ElementsMatch(t, metrics, tt.expectedMetrics) 187 | }) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /pkg/google/gcs/bucket.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | 8 | "cloud.google.com/go/storage" 9 | "google.golang.org/api/iterator" 10 | ) 11 | 12 | type StorageClientInterface interface { 13 | Buckets(ctx context.Context, projectID string) *storage.BucketIterator 14 | } 15 | 16 | type BucketClient struct { 17 | client StorageClientInterface 18 | } 19 | 20 | func NewBucketClient(client StorageClientInterface) *BucketClient { 21 | return &BucketClient{ 22 | client: client, 23 | } 24 | } 25 | 26 | func (bc *BucketClient) list(ctx context.Context, project string) ([]*storage.BucketAttrs, error) { 27 | log.Printf("Listing buckets for project %s", project) 28 | buckets := make([]*storage.BucketAttrs, 0) 29 | it := bc.client.Buckets(ctx, project) 30 | for { 31 | bucketAttrs, err := it.Next() 32 | if errors.Is(err, iterator.Done) { 33 | break 34 | } 35 | if err != nil { 36 | return buckets, err 37 | } 38 | buckets = append(buckets, bucketAttrs) 39 | } 40 | 41 | return buckets, nil 42 | } 43 | 44 | // TODO: Return an interface of the storage.BucketAttrs 45 | func (bc *BucketClient) List(ctx context.Context, project string) ([]*storage.BucketAttrs, error) { 46 | return bc.list(ctx, project) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/google/gcs/bucket_cache.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "sync" 5 | 6 | "cloud.google.com/go/storage" 7 | ) 8 | 9 | type BucketCache struct { 10 | Buckets map[string][]*storage.BucketAttrs 11 | m sync.RWMutex 12 | } 13 | 14 | func (c *BucketCache) Get(project string) []*storage.BucketAttrs { 15 | c.m.RLock() 16 | buckets := c.Buckets[project] 17 | c.m.RUnlock() 18 | return buckets 19 | } 20 | 21 | func (c *BucketCache) Set(project string, buckets []*storage.BucketAttrs) { 22 | c.m.Lock() 23 | c.Buckets[project] = buckets 24 | c.m.Unlock() 25 | } 26 | 27 | func NewBucketCache() *BucketCache { 28 | return &BucketCache{ 29 | Buckets: make(map[string][]*storage.BucketAttrs), 30 | m: sync.RWMutex{}, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/google/gcs/bucket_cache_test.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "cloud.google.com/go/storage" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestBucketCache_Get(t *testing.T) { 12 | tests := map[string]struct { 13 | want int 14 | error assert.ErrorAssertionFunc 15 | projects []string 16 | buckets []*storage.BucketAttrs 17 | }{ 18 | "empty": { 19 | want: 0, 20 | error: assert.Error, 21 | }, 22 | "one": { 23 | want: 1, 24 | projects: []string{"test"}, 25 | buckets: generateNBuckets(1), 26 | }, 27 | "ten": { 28 | want: 10, 29 | projects: []string{"test"}, 30 | buckets: generateNBuckets(10), 31 | }, 32 | } 33 | for name, tt := range tests { 34 | t.Run(name, func(t *testing.T) { 35 | bucketCache := NewBucketCache() 36 | for _, project := range tt.projects { 37 | bucketCache.Set(project, tt.buckets) 38 | } 39 | for _, project := range tt.projects { 40 | buckets := bucketCache.Get(project) 41 | assert.Equal(t, tt.want, len(buckets)) 42 | } 43 | }) 44 | } 45 | } 46 | 47 | func BenchmarkBucketCache_Get(b *testing.B) { 48 | bucketCache := NewBucketCache() 49 | buckets := generateNBuckets(1000) 50 | bucketCache.Set("test", buckets) 51 | for i := 0; i < b.N; i++ { 52 | bucketCache.Get("test") 53 | } 54 | } 55 | 56 | func BenchmarkBucketCache_GetParallel(b *testing.B) { 57 | bucketCache := NewBucketCache() 58 | buckets := generateNBuckets(1000) 59 | bucketCache.Set("test", buckets) 60 | b.RunParallel(func(pb *testing.PB) { 61 | for pb.Next() { 62 | bucketCache.Get("test") 63 | } 64 | }) 65 | } 66 | 67 | func BenchmarkBucketCache_Set(b *testing.B) { 68 | bucketCache := NewBucketCache() 69 | buckets := generateNBuckets(1000) 70 | for i := 0; i < b.N; i++ { 71 | bucketCache.Set(strconv.Itoa(i), buckets) 72 | } 73 | } 74 | 75 | func BenchmarkBucketCache_SetParallel(b *testing.B) { 76 | bucketCache := NewBucketCache() 77 | buckets := generateNBuckets(1000) 78 | b.RunParallel(func(pb *testing.PB) { 79 | for pb.Next() { 80 | bucketCache.Set("test", buckets) 81 | } 82 | }) 83 | } 84 | 85 | func generateNBuckets(n int) []*storage.BucketAttrs { 86 | buckets := make([]*storage.BucketAttrs, n) 87 | for i := 0; i < n; i++ { 88 | buckets[i] = &storage.BucketAttrs{ 89 | Name: "test", 90 | } 91 | } 92 | return buckets 93 | } 94 | -------------------------------------------------------------------------------- /pkg/google/gcs/bucket_test.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "cloud.google.com/go/storage" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | "google.golang.org/api/option" 13 | 14 | "github.com/grafana/cloudcost-exporter/mocks/pkg/google/gcs" 15 | ) 16 | 17 | func TestNewBucketClient(t *testing.T) { 18 | tests := map[string]struct { 19 | client StorageClientInterface 20 | }{ 21 | "Empty cloudCatalogClient": { 22 | client: gcs.NewStorageClientInterface(t), 23 | }, 24 | } 25 | for name, test := range tests { 26 | t.Run(name, func(t *testing.T) { 27 | client := NewBucketClient(test.client) 28 | if client == nil { 29 | t.Errorf("expected cloudCatalogClient to be non-nil") 30 | } 31 | }) 32 | } 33 | } 34 | 35 | // note: not checking this error because we don't care if w.Write() fails, 36 | // that's not our code to fix :) 37 | // 38 | //nolint:errcheck 39 | func TestBucketClient_List(t *testing.T) { 40 | tests := map[string]struct { 41 | server *httptest.Server 42 | projects []string 43 | want int 44 | wantErr bool 45 | }{ 46 | "no projects should result in no results": { 47 | projects: []string{"project-1"}, 48 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 49 | w.WriteHeader(http.StatusOK) 50 | w.Write([]byte(`{"items": []}`)) 51 | })), 52 | want: 0, 53 | wantErr: false, 54 | }, 55 | "one item should result in one bucket": { 56 | projects: []string{"project-1"}, 57 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 58 | w.WriteHeader(http.StatusOK) 59 | w.Write([]byte(`{"items": [{"name": "testing-123"}]}`)) 60 | })), 61 | want: 1, 62 | wantErr: false, 63 | }, 64 | "An error should be handled": { 65 | projects: []string{"project-1"}, 66 | server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 67 | w.WriteHeader(http.StatusOK) 68 | w.Write([]byte(``)) 69 | }, 70 | )), 71 | want: 0, 72 | wantErr: true, 73 | }, 74 | } 75 | for name, test := range tests { 76 | t.Run(name, func(t *testing.T) { 77 | for _, project := range test.projects { 78 | sc, err := storage.NewClient(context.Background(), option.WithEndpoint(test.server.URL), option.WithAPIKey("hunter2")) 79 | require.NoError(t, err) 80 | bc := NewBucketClient(sc) 81 | got, err := bc.List(context.Background(), project) 82 | assert.Equal(t, test.wantErr, err != nil) 83 | assert.NotNil(t, got) 84 | assert.Equal(t, test.want, len(got)) 85 | } 86 | }) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /pkg/google/gke/README.md: -------------------------------------------------------------------------------- 1 | # GKE Module 2 | 3 | Collects and exports costs associated with GKE instances. 4 | It's built on top of main of the same primitives as the GCP module. 5 | Specifically we share 6 | - PricingMap 7 | - MachineSpec 8 | - ListInstances 9 | 10 | What differs between the two is that the module will filter out instances that are not GKE instances. 11 | This is done by checking the `labels` field of the instance and looking for the cluster name. 12 | If no cluster name is found, the instance is not considered a GKE instance and is filtered out. 13 | 14 | The primary motivation for this module was to ensure we could support the following cases with ease: 15 | 1. Collecting costs for GKE instances 16 | 2. Collecting costs for Compute instances that _may not_ be a GKE instance 17 | 3. Collecting costs for Persistent Volumes that may be attached to a GKE instance 18 | 19 | See the [Design Doc](https://docs.google.com/document/d/1nCU1SVsuJ4HpV6R-N-AFBaDI5AJmSS3q9jH8_h-_Y8s/edit) for the rationale for a separate module. 20 | TL;DR; We do not want to emit metrics with a `exporter_cluster` label that is empty or make the setup process more complex needed. 21 | 22 | ## Disk Pricing 23 | 24 | Running the `gke` module also collects costs associated with Persistent Volumes. 25 | Persistent Volumes are attached to GKE instances and are billed as [disks](https://cloud.google.com/compute/disks-image-pricing). 26 | The price is based off of a combination of the following attributes: 27 | - region 28 | - disk type 29 | - disk size 30 | 31 | For simplicity, `cloudcost-exporter` has implemented the following disk types: 32 | - Standard(hard disk drives) 33 | - SSD(solid state drives) 34 | - Local SSD 35 | 36 | According to the [documentation](https://cloud.google.com/compute/disks-image-pricing#disk-and-image-pricing), pricing for storage is for [JEDEC Binary GB or IEC gibibytes(GiB)](https://en.wikipedia.org/wiki/Gigabyte). 37 | One of the more confusing bits is that the documentation for [disk](https://pkg.go.dev/google.golang.org/api/compute/v1#Disk) implies that the size is in GB, but doesn't specify if it's a [decimal GB or Binary GB](https://en.wikipedia.org/wiki/Gigabyte). 38 | `cloudcost-exporter` is assuming that the size is in binary GB which aligns with the pricing documentation. 39 | 40 | 41 | -------------------------------------------------------------------------------- /pkg/google/gke/disk.go: -------------------------------------------------------------------------------- 1 | package gke 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "strings" 7 | 8 | "google.golang.org/api/compute/v1" 9 | ) 10 | 11 | const ( 12 | BootDiskLabel = "goog-gke-node" 13 | pvcNamespaceKey = "kubernetes.io/created-for/pvc/namespace" 14 | pvcNamespaceShortKey = "kubernetes.io-created-for/pvc-namespace" 15 | pvNameKey = "kubernetes.io/created-for/pv/name" 16 | pvNameShortKey = "kubernetes.io-created-for/pv-name" 17 | idleDisk = "idle" 18 | inUseDisk = "in-use" 19 | ) 20 | 21 | type Disk struct { 22 | Cluster string 23 | 24 | Project string 25 | name string // Name of the disk as it appears in the GCP console. Used as a backup if the name can't be extracted from the description 26 | zone string 27 | labels map[string]string 28 | description map[string]string 29 | diskType string // type is a reserved word, which is why we're using diskType 30 | Size int64 31 | users []string 32 | } 33 | 34 | func NewDisk(disk *compute.Disk, project string) *Disk { 35 | clusterName := disk.Labels[GkeClusterLabel] 36 | d := &Disk{ 37 | Cluster: clusterName, 38 | Project: project, 39 | name: disk.Name, 40 | zone: disk.Zone, 41 | diskType: disk.Type, 42 | labels: disk.Labels, 43 | description: make(map[string]string), 44 | Size: disk.SizeGb, 45 | users: disk.Users, 46 | } 47 | err := extractLabelsFromDesc(disk.Description, d.description) 48 | if err != nil { 49 | log.Printf("error extracting labels from disk(%s) description: %v", d.Name(), err) 50 | } 51 | return d 52 | } 53 | 54 | // Namespace will search through the description fields for the namespace of the disk. If the namespace can't be determined 55 | // An empty string is return. 56 | func (d Disk) Namespace() string { 57 | return coalesce(d.description, pvcNamespaceKey, pvcNamespaceShortKey) 58 | } 59 | 60 | // Region will return the region of the disk by search through the zone field and returning the region. If the region can't be determined 61 | // It will return an empty string 62 | func (d Disk) Region() string { 63 | zone := d.labels[GkeRegionLabel] 64 | if zone == "" { 65 | // This would be a case where the disk is no longer mounted _or_ the disk is associated with a Compute instance 66 | zone = d.zone[strings.LastIndex(d.zone, "/")+1:] 67 | } 68 | // If zone _still_ is empty we can't determine the region, so we return an empty string 69 | // This prevents an index out of bounds error 70 | if zone == "" { 71 | return "" 72 | } 73 | if strings.Count(zone, "-") < 2 { 74 | return zone 75 | } 76 | return zone[:strings.LastIndex(zone, "-")] 77 | } 78 | 79 | // Name will return the name of the disk. If the disk has a label "kubernetes.io/created-for/pv/name" it will return the value stored in that key. 80 | // otherwise it will return the disk name that is directly associated with the disk. 81 | func (d Disk) Name() string { 82 | if d.description == nil { 83 | return d.name 84 | } 85 | // first check that the key exists in the map, if it does return the value 86 | name := coalesce(d.description, pvNameKey, pvNameShortKey) 87 | if name != "" { 88 | return name 89 | } 90 | return d.name 91 | } 92 | 93 | // coalesce will take a map and a list of keys and return the first value that is found in the map. If no value is found it will return an empty string 94 | func coalesce(desc map[string]string, keys ...string) string { 95 | for _, key := range keys { 96 | if val, ok := desc[key]; ok { 97 | return val 98 | } 99 | } 100 | return "" 101 | } 102 | 103 | // extractLabelsFromDesc will take a description string and extract the labels from it. GKE disks store their description as 104 | // a json blob in the description field. This function will extract the labels from that json blob and return them as a map 105 | // Some useful information about the json blob are name, cluster, namespace, and pvc's that the disk is associated with 106 | func extractLabelsFromDesc(description string, labels map[string]string) error { 107 | if description == "" { 108 | return nil 109 | } 110 | if err := json.Unmarshal([]byte(description), &labels); err != nil { 111 | return err 112 | } 113 | return nil 114 | } 115 | 116 | // StorageClass will return the storage class of the disk by looking at the type. Type in GCP is represented as a URL and as such 117 | // we're looking for the last part of the URL to determine the storage class 118 | func (d Disk) StorageClass() string { 119 | diskType := strings.Split(d.diskType, "/") 120 | return diskType[len(diskType)-1] 121 | } 122 | 123 | // DiskType will search through the labels to determine the type of disk. If the disk has a label "goog-gke-node" it will return "boot_disk" 124 | // Otherwise it returns persistent_volume 125 | func (d Disk) DiskType() string { 126 | if _, ok := d.labels[BootDiskLabel]; ok { 127 | return "boot_disk" 128 | } 129 | return "persistent_volume" 130 | } 131 | 132 | // UseStatus will return two constant strings to tell apart disks that are sitting idle from those that are mounted to a pod 133 | // It's named UseStatus and not just Status because the GCP API already has a field Status that holds a different concept that 134 | // we don't want to overwrite. From their docs: 135 | // Status: [Output Only] The status of disk creation. - CREATING: Disk is 136 | // provisioning. - RESTORING: Source data is being copied into the disk. - 137 | // FAILED: Disk creation failed. - READY: Disk is ready for use. - DELETING: 138 | // Disk is deleting. UNAVAILABLE - Disk is currently unavailable and cannot be accessed, 139 | func (d Disk) UseStatus() string { 140 | if len(d.users) == 0 { 141 | return idleDisk 142 | } 143 | 144 | return inUseDisk 145 | } 146 | -------------------------------------------------------------------------------- /pkg/google/gke/machinespec.go: -------------------------------------------------------------------------------- 1 | package gke 2 | 3 | import ( 4 | "log" 5 | "regexp" 6 | "strings" 7 | 8 | "google.golang.org/api/compute/v1" 9 | ) 10 | 11 | var ( 12 | re = regexp.MustCompile(`\bin\b`) 13 | GkeClusterLabel = "goog-k8s-cluster-name" 14 | GkeRegionLabel = "goog-k8s-cluster-location" 15 | ) 16 | 17 | // MachineSpec is a slimmed down representation of a google compute.Instance struct 18 | type MachineSpec struct { 19 | Instance string 20 | Zone string 21 | Region string 22 | Family string 23 | MachineType string 24 | SpotInstance bool 25 | Labels map[string]string 26 | PriceTier string 27 | } 28 | 29 | // NewMachineSpec will create a new MachineSpec from compute.Instance objects. 30 | // It's responsible for determining the machine family and region that it operates in 31 | func NewMachineSpec(instance *compute.Instance) *MachineSpec { 32 | zone := instance.Zone[strings.LastIndex(instance.Zone, "/")+1:] 33 | region := getRegionFromZone(zone) 34 | machineType := getMachineTypeFromURL(instance.MachineType) 35 | family := getMachineFamily(machineType) 36 | spot := isSpotInstance(instance.Scheduling.ProvisioningModel) 37 | priceTier := priceTierForInstance(spot) 38 | 39 | return &MachineSpec{ 40 | Instance: instance.Name, 41 | Zone: zone, 42 | Region: region, 43 | MachineType: machineType, 44 | Family: family, 45 | SpotInstance: spot, 46 | Labels: instance.Labels, 47 | PriceTier: priceTier, 48 | } 49 | } 50 | 51 | func isSpotInstance(model string) bool { 52 | return model == "SPOT" 53 | } 54 | 55 | func getRegionFromZone(zone string) string { 56 | return zone[:strings.LastIndex(zone, "-")] 57 | } 58 | 59 | func getMachineTypeFromURL(url string) string { 60 | return url[strings.LastIndex(url, "/")+1:] 61 | } 62 | 63 | func getMachineFamily(machineType string) string { 64 | if !strings.Contains(machineType, "-") { 65 | log.Printf("Machine type %s doesn't contain a -", machineType) 66 | return "" 67 | } 68 | split := strings.Split(machineType, "-") 69 | return strings.ToLower(split[0]) 70 | } 71 | 72 | func stripOutKeyFromDescription(description string) string { 73 | // Except for commitments, the description will have running in it 74 | runningInIndex := strings.Index(description, "running in") 75 | 76 | if runningInIndex > 0 { 77 | description = description[:runningInIndex] 78 | return strings.Trim(description, " ") 79 | } 80 | // If we can't find running in, try to find Commitment v1: 81 | splitString := strings.Split(description, "Commitment v1:") 82 | if len(splitString) == 1 { 83 | log.Printf("No running in or commitment found in description: %s", description) 84 | return "" 85 | } 86 | // Take everything after the Commitment v1 87 | // TODO: Evaluate if we want to consider leaving in Commitment V1 88 | split := splitString[1] 89 | // Now something a bit more tricky, we need to find an exact match of "in" 90 | // Turns out that locations such as Berlin break this assumption 91 | // SO we need to use a regexp to find the first instance of "in" 92 | foundIndex := re.FindStringIndex(split) 93 | if len(foundIndex) == 0 { 94 | log.Printf("No in found in description: %s", description) 95 | return "" 96 | } 97 | str := split[:foundIndex[0]] 98 | return strings.Trim(str, " ") 99 | } 100 | 101 | func priceTierForInstance(spotInstance bool) string { 102 | if spotInstance { 103 | return "spot" 104 | } 105 | // TODO: Handle if it's a commitment 106 | return "ondemand" 107 | } 108 | 109 | func (m *MachineSpec) GetClusterName() string { 110 | if clusterName, ok := m.Labels[GkeClusterLabel]; ok { 111 | return clusterName 112 | } 113 | return "" 114 | } 115 | -------------------------------------------------------------------------------- /pkg/google/gke/machinespec_test.go: -------------------------------------------------------------------------------- 1 | package gke 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_stripOutKeyFromDescription(t *testing.T) { 8 | tests := map[string]struct { 9 | description string 10 | want string 11 | }{ 12 | "simple": { 13 | description: "N1 Predefined Instance Core running in Americas", 14 | want: "N1 Predefined Instance Core", 15 | }, 16 | "commitment v1: empty": { 17 | description: "Commitment v1:", 18 | want: "", 19 | }, 20 | "commitment v1": { 21 | description: "Commitment v1: N2 Predefined Instance Core in Americas", 22 | want: "N2 Predefined Instance Core", 23 | }, 24 | "commitment v2": { 25 | description: "Commitment v1: N2D AMD Ram in Americin for 1 year", 26 | want: "N2D AMD Ram", 27 | }, 28 | "commitment berlin": { 29 | description: "Commitment v1: G2 Ram in Berlin for 1 year", 30 | want: "G2 Ram", 31 | }, 32 | } 33 | for name, tt := range tests { 34 | t.Run(name, func(t *testing.T) { 35 | if got := stripOutKeyFromDescription(tt.description); got != tt.want { 36 | t.Errorf("stripOutKeyFromDescription() = %v, want %v", got, tt.want) 37 | } 38 | }) 39 | } 40 | } 41 | func Test_getMachineInfoFromMachineType(t *testing.T) { 42 | type result struct { 43 | wantCpu int 44 | wantRam int 45 | wantZone string 46 | wantType string 47 | wantMachineType string 48 | } 49 | tests := map[string]struct { 50 | machineType string 51 | want result 52 | }{ 53 | "simple": { 54 | machineType: "https://www.googleapis.com/compute/v1/projects/grafanalabs-dev/zones/us-central1-a/machineTypes/n2-standard-8", 55 | want: result{ 56 | wantCpu: 2, 57 | wantRam: 8, 58 | wantZone: "us-central1-a", 59 | wantMachineType: "n2-standard-8", 60 | wantType: "n2", 61 | }, 62 | }, 63 | } 64 | for name, test := range tests { 65 | t.Run(name, func(t *testing.T) { 66 | if got := getMachineTypeFromURL(test.machineType); got != test.want.wantMachineType { 67 | t.Errorf("getMachineTypeFromURL() = %v, want %v", got, test.want.wantMachineType) 68 | } 69 | }) 70 | } 71 | } 72 | 73 | func Test_GetMachineFamily(t *testing.T) { 74 | tests := map[string]struct { 75 | machineType string 76 | want string 77 | }{ 78 | "n1": { 79 | machineType: "n1-standard-8", 80 | want: "n1", 81 | }, 82 | "n2": { 83 | machineType: "n2-standard-8", 84 | want: "n2", 85 | }, 86 | "n2-bad": { 87 | machineType: "n2_standard", 88 | want: "", 89 | }, 90 | "n2d": { 91 | machineType: "n2d-standard-8", 92 | want: "n2d", 93 | }, 94 | "e1": { 95 | machineType: "e2-standard-8", 96 | want: "e2", 97 | }, 98 | "g1": { 99 | machineType: "g1-standard-8", 100 | want: "g1", 101 | }, 102 | } 103 | 104 | for name, test := range tests { 105 | t.Run(name, func(t *testing.T) { 106 | if got := getMachineFamily(test.machineType); got != test.want { 107 | t.Errorf("stripOutKeyFromDescription() = %v, want %v", got, test.want) 108 | } 109 | }) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /pkg/logger/level.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log/slog" 7 | "os" 8 | ) 9 | 10 | // A LevelHandler wraps a Handler with an Enabled method 11 | // that returns false for levels below a minimum. 12 | type LevelHandler struct { 13 | level slog.Leveler 14 | handler slog.Handler 15 | } 16 | 17 | // NewLevelHandler returns a LevelHandler with the given level. 18 | // All methods except Enabled delegate to h. 19 | func NewLevelHandler(level slog.Leveler, h slog.Handler) *LevelHandler { 20 | // Optimization: avoid chains of LevelHandlers. 21 | if lh, ok := h.(*LevelHandler); ok { 22 | h = lh.Handler() 23 | } 24 | return &LevelHandler{level, h} 25 | } 26 | 27 | // Enabled implements Handler.Enabled by reporting whether 28 | // level is at least as large as h's level. 29 | func (h *LevelHandler) Enabled(_ context.Context, level slog.Level) bool { 30 | return level >= h.level.Level() 31 | } 32 | 33 | // Handle implements Handler.Handle. 34 | func (h *LevelHandler) Handle(ctx context.Context, r slog.Record) error { 35 | return h.handler.Handle(ctx, r) 36 | } 37 | 38 | // WithAttrs implements Handler.WithAttrs. 39 | func (h *LevelHandler) WithAttrs(attrs []slog.Attr) slog.Handler { 40 | return NewLevelHandler(h.level, h.handler.WithAttrs(attrs)) 41 | } 42 | 43 | // WithGroup implements Handler.WithGroup. 44 | func (h *LevelHandler) WithGroup(name string) slog.Handler { 45 | return NewLevelHandler(h.level, h.handler.WithGroup(name)) 46 | } 47 | 48 | // Handler returns the Handler wrapped by h. 49 | func (h *LevelHandler) Handler() slog.Handler { 50 | return h.handler 51 | } 52 | 53 | // GetLogLevel parses a string and returns the corresponding slog.Leveler. Returns slog.LevelInfo if the string is not recognized. 54 | func GetLogLevel(level string) slog.Leveler { 55 | switch level { 56 | case "debug": 57 | return slog.LevelDebug 58 | case "info": 59 | return slog.LevelInfo 60 | case "warn": 61 | return slog.LevelWarn 62 | case "error": 63 | return slog.LevelError 64 | default: 65 | return slog.LevelInfo 66 | } 67 | } 68 | 69 | // WriterForOutput returns an io.Writer based on the output string. Returns os.Stdout if the string is not recognized. 70 | func WriterForOutput(output string) io.Writer { 71 | switch output { 72 | case "stdout": 73 | return os.Stdout 74 | case "stderr": 75 | return os.Stderr 76 | default: 77 | return os.Stdout 78 | } 79 | } 80 | 81 | // HandlerForOutput returns a slog.Handler based on the output string. Returns a slog.NewTextHandler if the string is not recognized. 82 | func HandlerForOutput(output string, w io.Writer) slog.Handler { 83 | switch output { 84 | case "json": 85 | return slog.NewJSONHandler(w, nil) 86 | default: 87 | return slog.NewTextHandler(w, nil) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/provider/provider.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | //go:generate mockgen -source=provider.go -destination mocks/provider.go 8 | 9 | type Registry interface { 10 | prometheus.Registerer 11 | prometheus.Gatherer 12 | prometheus.Collector 13 | } 14 | 15 | type Collector interface { 16 | Register(r Registry) error 17 | CollectMetrics(chan<- prometheus.Metric) float64 18 | Collect(chan<- prometheus.Metric) error 19 | Describe(chan<- *prometheus.Desc) error 20 | Name() string 21 | } 22 | 23 | type Provider interface { 24 | prometheus.Collector 25 | RegisterCollectors(r Registry) error 26 | } 27 | -------------------------------------------------------------------------------- /pkg/utils/consts.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | const ( 6 | HoursInMonth = 24.35 * 30 // 24.35 is the average amount of hours in a day over a year 7 | InstanceCPUCostSuffix = "instance_cpu_usd_per_core_hour" 8 | InstanceMemoryCostSuffix = "instance_memory_usd_per_gib_hour" 9 | InstanceTotalCostSuffix = "instance_total_usd_per_hour" 10 | PersistentVolumeCostSuffix = "persistent_volume_usd_per_hour" 11 | ) 12 | 13 | func GenerateDesc(prefix, subsystem, suffix, description string, labels []string) *prometheus.Desc { 14 | return prometheus.NewDesc( 15 | prometheus.BuildFQName(prefix, subsystem, suffix), 16 | description, 17 | labels, 18 | nil, 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/utils/consts_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | func TestGenerateDesc(t *testing.T) { 11 | prefix := "test_prefix" 12 | subsystem := "test_subsystem" 13 | suffix := "test_suffix" 14 | description := "This is a test description" 15 | labels := []string{"label1", "label2"} 16 | 17 | desc := GenerateDesc(prefix, subsystem, suffix, description, labels) 18 | 19 | // Expected values 20 | expectedFQName := prometheus.BuildFQName(prefix, subsystem, suffix) 21 | 22 | if !strings.Contains(desc.String(), expectedFQName) { 23 | t.Errorf("Expected FQName %s in desc, but got %s", expectedFQName, desc.String()) 24 | } 25 | 26 | if !strings.Contains(desc.String(), description) { 27 | t.Errorf("Expected description %s in desc, but got %s", description, desc.String()) 28 | } 29 | 30 | for _, label := range labels { 31 | if !strings.Contains(desc.String(), label) { 32 | t.Errorf("Expected label %s in desc, but got %s", label, desc.String()) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/utils/metrics.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | io_prometheus_client "github.com/prometheus/client_model/go" 8 | ) 9 | 10 | type LabelMap map[string]string 11 | 12 | type MetricResult struct { 13 | FqName string 14 | Labels LabelMap 15 | Value float64 16 | MetricType prometheus.ValueType 17 | } 18 | 19 | var ( 20 | re = regexp.MustCompile(`fqName:\s*"([^"]+)"`) 21 | ) 22 | 23 | func ReadMetrics(metric prometheus.Metric) *MetricResult { 24 | if metric == nil { 25 | return nil 26 | } 27 | m := &io_prometheus_client.Metric{} 28 | err := metric.Write(m) 29 | if err != nil { 30 | return nil 31 | } 32 | labels := make(LabelMap, len(m.Label)) 33 | for _, l := range m.Label { 34 | labels[l.GetName()] = l.GetValue() 35 | } 36 | fqName := parseFqNameFromMetric(metric.Desc().String()) 37 | if m.Gauge != nil { 38 | return &MetricResult{ 39 | FqName: fqName, 40 | Labels: labels, 41 | Value: m.GetGauge().GetValue(), 42 | MetricType: prometheus.GaugeValue, 43 | } 44 | } 45 | if m.Counter != nil { 46 | return &MetricResult{ 47 | Labels: labels, 48 | Value: m.GetCounter().GetValue(), 49 | MetricType: prometheus.CounterValue, 50 | } 51 | } 52 | if m.Untyped != nil { 53 | return &MetricResult{ 54 | Labels: labels, 55 | Value: m.GetUntyped().GetValue(), 56 | MetricType: prometheus.UntypedValue, 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | func parseFqNameFromMetric(desc string) string { 63 | if desc == "" { 64 | return "" 65 | } 66 | return re.FindStringSubmatch(desc)[1] 67 | } 68 | -------------------------------------------------------------------------------- /pkg/utils/metrics_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_parseFqNameFromMetric(t *testing.T) { 8 | tests := map[string]struct { 9 | arg string 10 | want string 11 | }{ 12 | "empty metric": { 13 | arg: "", 14 | want: "", 15 | }, 16 | "metric with fqName": { 17 | arg: "fqName: \"aws_s3_bucket_size_bytes\"", 18 | want: "aws_s3_bucket_size_bytes", 19 | }, 20 | "metric with fqName and help": { 21 | arg: "FqName:\"Desc{fqName: \"cloudcost_exporter_gcp_collector_success\", help: \"Was the last scrape of the GCP metrics successful.\", constLabels: {}, variableLabels: {collector}}\"", 22 | want: "cloudcost_exporter_gcp_collector_success", 23 | }, 24 | } 25 | for name, tt := range tests { 26 | t.Run(name, func(t *testing.T) { 27 | if got := parseFqNameFromMetric(tt.arg); got != tt.want { 28 | t.Errorf("parseFqNameFromMetric() = %v, want %v", got, tt.want) 29 | } 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /scripts/aws-spot-pricing/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "time" 9 | 10 | "github.com/aws/aws-sdk-go-v2/config" 11 | "github.com/aws/aws-sdk-go-v2/service/ec2" 12 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 13 | ) 14 | 15 | func main() { 16 | options := []func(*config.LoadOptions) error{config.WithEC2IMDSRegion()} 17 | options = append(options, config.WithRegion("us-east-2")) 18 | options = append(options, config.WithSharedConfigProfile(os.Getenv("AWS_PROFILE"))) 19 | cfg, err := config.LoadDefaultConfig(context.TODO(), options...) 20 | if err != nil { 21 | log.Fatalf("unable to load SDK config, %v", err) 22 | } 23 | 24 | client := ec2.NewFromConfig(cfg) 25 | 26 | // Call DescribeSpotPriceHistory 27 | var spotPrices []types.SpotPrice 28 | starTime := time.Now().Add(-time.Hour * 24) 29 | endTime := time.Now() 30 | sphi := &ec2.DescribeSpotPriceHistoryInput{ 31 | ProductDescriptions: []string{ 32 | "Linux/UNIX (Amazon VPC)", // replace with your product description 33 | }, 34 | StartTime: &starTime, 35 | EndTime: &endTime, 36 | } 37 | for { 38 | resp, err := client.DescribeSpotPriceHistory(context.TODO(), sphi) 39 | if err != nil { 40 | break 41 | } 42 | spotPrices = append(spotPrices, resp.SpotPriceHistory...) 43 | if resp.NextToken == nil || *resp.NextToken == "" { 44 | break 45 | } 46 | sphi.NextToken = resp.NextToken 47 | } 48 | 49 | spotPriceMap := map[string]map[string]types.SpotPrice{} 50 | // Print the spot prices 51 | for _, spotPrice := range spotPrices { 52 | az := *spotPrice.AvailabilityZone 53 | instanceType := string(spotPrice.InstanceType) 54 | if _, ok := spotPriceMap[az]; !ok { 55 | spotPriceMap[az] = map[string]types.SpotPrice{} 56 | } 57 | if _, ok := spotPriceMap[az][instanceType]; ok { 58 | // Check to see if the price is newer 59 | if spotPriceMap[az][instanceType].Timestamp.After(*spotPrice.Timestamp) { 60 | continue 61 | } 62 | } 63 | spotPriceMap[az][instanceType] = spotPrice 64 | } 65 | for region, prices := range spotPriceMap { 66 | fmt.Printf("Region: %s\n", region) 67 | for instanceType, price := range prices { 68 | fmt.Printf("Instance type: %s, price: %s\n", instanceType, *price.SpotPrice) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /scripts/gcp-fetch-skus/gcp-fetch-skus.go: -------------------------------------------------------------------------------- 1 | // Usage: go run gcp-fetch-skus.go 2 | // THis is a useful utility if you want to fetch a set of sku's for a particular service and export them to CSV. 3 | package main 4 | 5 | import ( 6 | "context" 7 | "encoding/csv" 8 | "flag" 9 | "fmt" 10 | "log" 11 | "os" 12 | "strconv" 13 | 14 | billingv1 "cloud.google.com/go/billing/apiv1" 15 | 16 | "github.com/grafana/cloudcost-exporter/pkg/google/billing" 17 | ) 18 | 19 | type Config struct { 20 | Service string 21 | OutputFile string 22 | } 23 | 24 | func main() { 25 | var config *Config 26 | flag.StringVar(&config.Service, "service", "Compute Engine", "The service to fetch skus for") 27 | flag.StringVar(&config.OutputFile, "output-file", "skus.csv", "The file to write the skus to") 28 | flag.Parse() 29 | if err := run(config); err != nil { 30 | log.Printf("error: %v", err) 31 | os.Exit(1) 32 | } 33 | } 34 | 35 | func run(config *Config) error { 36 | ctx := context.Background() 37 | client, err := billingv1.NewCloudCatalogClient(ctx) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | defer client.Close() 42 | svcid, err := billing.GetServiceName(ctx, client, config.Service) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | skus := billing.GetPricing(ctx, client, svcid) 47 | file, err := os.Create(config.OutputFile) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | defer file.Close() 52 | writer := csv.NewWriter(file) 53 | err = writer.Write([]string{"sku_id", "description", "category", "region", "pricing_info"}) 54 | if err != nil { 55 | return fmt.Errorf("error writing record to csv: %w", err) 56 | } 57 | for _, sku := range skus { 58 | for _, region := range sku.ServiceRegions { 59 | price := "" 60 | if len(sku.PricingInfo) != 0 { 61 | if len(sku.PricingInfo[0].PricingExpression.TieredRates) != 0 { 62 | rates := len(sku.PricingInfo[0].PricingExpression.TieredRates) 63 | rateIdx := 0 64 | if rates > 1 { 65 | rateIdx = rates - 1 66 | } 67 | price = strconv.FormatFloat(float64(sku.PricingInfo[0].PricingExpression.TieredRates[rateIdx].UnitPrice.Nanos)*1e-9, 'f', -1, 64) 68 | } 69 | } 70 | err = writer.Write([]string{sku.SkuId, sku.Description, sku.Category.ResourceFamily, region, price}) 71 | if err != nil { 72 | return fmt.Errorf("error writing record to csv: %w", err) 73 | } 74 | } 75 | } 76 | writer.Flush() 77 | return writer.Error() 78 | } 79 | --------------------------------------------------------------------------------