├── .dockerignore ├── .gcloudignore ├── .github ├── CODEOWNERS ├── minty.yaml └── workflows │ ├── ci.yml │ ├── cleanup.yml │ ├── draft-release.yml │ └── release.yml ├── .gitignore ├── .goreleaser.docker.yaml ├── AUTHORS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── RELEASEING.md ├── VERSION ├── abc.templates ├── deployments │ ├── contents │ │ └── deploy.sh │ ├── spec.yaml │ └── workflows │ │ └── deploy-github-metrics-aggregator.yaml └── infra │ ├── contents │ ├── main.tf │ ├── outputs.tf │ └── terraform.tf │ └── spec.yaml ├── assets └── architecture.svg ├── cmd └── github-metrics-aggregator │ └── main.go ├── config ├── artifacts_metadata.json └── commit-review-status-metadata.json ├── docs └── playbooks │ └── alerts │ ├── BadRequests.md │ ├── ContainerUsage.md │ ├── ForwardProgressFailed.md │ ├── RequestLatency.md │ └── ServerFaults.md ├── go.mod ├── go.sum ├── integration ├── bigquery.go ├── config.go └── main_test.go ├── pkg ├── artifact │ ├── config.go │ ├── ingest_logs.go │ ├── ingest_logs_test.go │ ├── job.go │ ├── query.go │ └── storage.go ├── bq │ ├── bigquery.go │ └── bigquery_test.go ├── cli │ ├── artifact.go │ ├── retry.go │ ├── retry_test.go │ ├── review.go │ ├── root.go │ ├── root_test.go │ ├── webhook.go │ └── webhook_test.go ├── githubclient │ └── githubclient.go ├── retry │ ├── bigquery.go │ ├── bigquery_mock.go │ ├── config.go │ ├── config_test.go │ ├── github_mock.go │ ├── lock_mock.go │ ├── retry.go │ ├── retry_test.go │ └── server.go ├── review │ ├── breakglass_query.go │ ├── breakglass_query_test.go │ ├── commit_query.go │ ├── commit_query_test.go │ ├── commit_review_status.go │ ├── commit_review_status_test.go │ ├── config.go │ ├── issue_fetcher.go │ └── job.go ├── teeth │ ├── bigquery.go │ ├── bigquery_test.go │ ├── publish_logs.go │ └── sql │ │ └── publisher_source.sql ├── version │ └── version.go └── webhook │ ├── bigquery.go │ ├── bigquery_mock.go │ ├── config.go │ ├── config_test.go │ ├── pubsub.go │ ├── server.go │ ├── webhook.go │ └── webhook_test.go ├── protos ├── protos.go └── pubsub_schemas │ ├── event.pb.go │ └── event.proto ├── terraform ├── bigquery.tf ├── dashboard.tf ├── dashboards │ └── default.json ├── main.tf ├── modules │ ├── artifacts │ │ ├── cloud_run_job.tf │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── terraform.tf │ │ └── variables.tf │ ├── bigquery_metrics_views │ │ ├── data │ │ │ ├── bq_tvf │ │ │ │ ├── events │ │ │ │ │ ├── issue_events_by_date.sql │ │ │ │ │ ├── pull_request_events_by_date.sql │ │ │ │ │ └── push_events_by_date.sql │ │ │ │ └── resources │ │ │ │ │ ├── issues_by_date.sql │ │ │ │ │ └── pull_requests_by_date.sql │ │ │ └── bq_views │ │ │ │ ├── events │ │ │ │ ├── check_run_events.sql │ │ │ │ ├── deployment_events.sql │ │ │ │ ├── deployment_status_events.sql │ │ │ │ ├── issue_comment_events.sql │ │ │ │ ├── issue_events.sql │ │ │ │ ├── pull_request_events.sql │ │ │ │ ├── pull_request_review_comment_events.sql │ │ │ │ ├── pull_request_review_events.sql │ │ │ │ ├── push_events.sql │ │ │ │ ├── release_events.sql │ │ │ │ ├── team_events.sql │ │ │ │ └── workflow_run_events.sql │ │ │ │ └── resources │ │ │ │ ├── check_runs.sql │ │ │ │ ├── deployment_statuses.sql │ │ │ │ ├── deployments.sql │ │ │ │ ├── issue_comment.sql │ │ │ │ ├── issues.sql │ │ │ │ ├── pull_request_review_comments.sql │ │ │ │ ├── pull_request_reviews.sql │ │ │ │ ├── pull_requests.sql │ │ │ │ ├── releases.sql │ │ │ │ ├── teams.sql │ │ │ │ └── workflow_runs.sql │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── terraform.tf │ │ ├── tvf.tf │ │ └── variables.tf │ ├── commit_review_status │ │ ├── cloud_run_job.tf │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── terraform.tf │ │ └── variables.tf │ └── invocation_comment │ │ ├── main.tf │ │ ├── terraform.tf │ │ └── variables.tf ├── notification.tf ├── outputs.tf ├── pubsub.tf ├── service_retry.tf ├── service_webhook.tf ├── terraform.tf └── variables.tf └── testdata ├── issues.json ├── pull_request.json └── workflow_job.json /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.env 2 | **/.git 3 | **/.gitignore 4 | **/.vscode 5 | **/bin 6 | **/docker-compose* 7 | **/Dockerfile* 8 | **/terraform* 9 | LICENSE 10 | AUTHORS 11 | CONTRIBUTING.md 12 | README.md 13 | 14 | # Ignore generated credentials from google-github-actions/auth 15 | gha-creds-*.json 16 | -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | #!include:.gitignore 2 | 3 | # Ignore source directories 4 | **/.git 5 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default fallback 2 | * @abcxyz/breakglass 3 | 4 | # abcxyz/infrastructure owns all files 5 | * @abcxyz/infrastructure-team 6 | 7 | # Go readability 8 | *.go @abcxyz/go-readability 9 | 10 | # terraform-readability owns all terraform files 11 | 12 | *.tf @abcxyz/terraform-readability 13 | -------------------------------------------------------------------------------- /.github/minty.yaml: -------------------------------------------------------------------------------- 1 | version: 'minty.abcxyz.dev/v2' 2 | 3 | rule: 4 | if: |- 5 | assertion.iss == issuers.github && 6 | assertion.repository_owner_id == '93787867' && 7 | assertion.repository_id == '566990094' && 8 | assertion.ref == 'refs/heads/main' 9 | 10 | scope: 11 | draft-release: 12 | rule: 13 | if: |- 14 | assertion.workflow_ref == assertion.job_workflow_ref && 15 | assertion.workflow_ref.startsWith("abcxyz/github-metrics-aggregator/.github/workflows/draft-release.yml") && 16 | assertion.event_name == 'workflow_dispatch' 17 | repositories: 18 | - 'github-metrics-aggregator' 19 | permissions: 20 | contents: 'write' 21 | pull_requests: 'write' 22 | 23 | release: 24 | rule: 25 | if: |- 26 | assertion.workflow_ref == assertion.job_workflow_ref && 27 | assertion.workflow_ref.startsWith("abcxyz/github-metrics-aggregator/.github/workflows/release.yml") && 28 | assertion.event_name == 'workflow_run' 29 | repositories: 30 | - 'github-metrics-aggregator' 31 | permissions: 32 | contents: 'write' 33 | 34 | -------------------------------------------------------------------------------- /.github/workflows/cleanup.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: 'cleanup' 16 | 17 | on: 18 | pull_request: 19 | paths: 20 | - '.github/workflows/cleanup.yml' 21 | schedule: 22 | - cron: '0 0 */1 * *' 23 | workflow_dispatch: 24 | 25 | env: 26 | WIF_PROVIDER: 'projects/1022440990968/locations/global/workloadIdentityPools/github-automation/providers/gma-ci-i' 27 | WIF_SERVICE_ACCOUNT: 'github-automation-bot@gha-gma-ci-i-8258a4.iam.gserviceaccount.com' 28 | GAR_REPO: 'us-docker.pkg.dev/github-metrics-ci/ci-images' 29 | INTEGRATION_PROJECT_ID: 'github-metrics-aggreg-i-64d426' 30 | INTEGRATION_REGION: 'us-central1' 31 | 32 | permissions: 33 | contents: 'read' 34 | id-token: 'write' 35 | 36 | jobs: 37 | # cleanup_cloudrun_revisions deletes all Webhook and Retry Cloud Run revisions 38 | # for the given service that are more than 5 hours old. 39 | cleanup_cloudrun_revisions: 40 | runs-on: 'ubuntu-latest' 41 | strategy: 42 | matrix: 43 | service_name: 44 | - 'github-metrics-webhook-c764' 45 | - 'github-metrics-retry-4d31' 46 | steps: 47 | - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 48 | 49 | - uses: 'google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f' # ratchet:google-github-actions/auth@v2 50 | with: 51 | workload_identity_provider: '${{ env.WIF_PROVIDER }}' 52 | service_account: '${{ env.WIF_SERVICE_ACCOUNT }}' 53 | 54 | - uses: 'google-github-actions/setup-gcloud@6189d56e4096ee891640bb02ac264be376592d6a' # ratchet:google-github-actions/setup-gcloud@v2 55 | 56 | - name: 'Remove old Webhook Cloud Run revisions' 57 | shell: 'bash' 58 | run: |- 59 | # List all revisions that were deployed 5 hours ago or later, excluding the latest revision 60 | # There must be at least one revisions running at all times 61 | REVISIONS=$(gcloud run revisions list \ 62 | --project="$INTEGRATION_PROJECT_ID" \ 63 | --region="$INTEGRATION_REGION" \ 64 | --service="${{ matrix.service_name }}" \ 65 | --format="value(name)" \ 66 | --sort-by="~metadata.creationTimestamp" \ 67 | --filter="metadata.creationTimestamp < '-p5h'" | sed 1d) 68 | 69 | IFS=$'\n'; for NAME in $REVISIONS; do 70 | echo "Deleting ${NAME}..." 71 | gcloud run revisions delete "${NAME}" --project="$INTEGRATION_PROJECT_ID" --region="$INTEGRATION_REGION" --quiet --async 72 | done 73 | 74 | # cleanup_container_images deletes all untagged container images that are more 75 | # than 14 days old. 76 | cleanup_container_images: 77 | runs-on: 'ubuntu-latest' 78 | steps: 79 | - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 80 | 81 | - uses: 'google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f' # ratchet:google-github-actions/auth@v2 82 | with: 83 | workload_identity_provider: '${{ env.WIF_PROVIDER }}' 84 | service_account: '${{ env.WIF_SERVICE_ACCOUNT }}' 85 | 86 | - name: 'Remove old container images' 87 | uses: 'docker://us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli' # ratchet:exclude old image versions are removed 88 | with: 89 | args: >- 90 | -repo="${{ env.GAR_REPO }}" 91 | -recursive="true" 92 | -grace="336h" 93 | -tag-filter-any="(?i)[0-9a-f]{40}(-amd64|-arm64)?" 94 | -------------------------------------------------------------------------------- /.github/workflows/draft-release.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: 'draft-release' 16 | 17 | on: 18 | workflow_dispatch: 19 | inputs: 20 | version_strategy: 21 | description: |- 22 | Update strategy 23 | default: 'patch' 24 | type: 'choice' 25 | options: 26 | - 'major' 27 | - 'minor' 28 | - 'patch' 29 | - 'prerelease' 30 | required: true 31 | 32 | env: 33 | PR_BRANCH: 'automation/draft-release-${{ github.ref_name }}' 34 | 35 | jobs: 36 | draft-release: 37 | runs-on: 'ubuntu-latest' 38 | permissions: 39 | contents: 'read' 40 | id-token: 'write' 41 | steps: 42 | - name: 'Increment version' 43 | id: 'increment-version' 44 | uses: 'abcxyz/actions/.github/actions/increment-version@main' # ratchet:exclude 45 | with: 46 | version_strategy: '${{ inputs.version_strategy }}' 47 | 48 | - name: 'Mint token' 49 | id: 'mint-token' 50 | uses: 'abcxyz/github-token-minter/.github/actions/mint-token@main' # ratchet:exclude 51 | with: 52 | wif_provider: '${{ vars.TOKEN_MINTER_WIF_PROVIDER }}' 53 | wif_service_account: '${{ vars.TOKEN_MINTER_WIF_SERVICE_ACCOUNT }}' 54 | service_audience: '${{ vars.TOKEN_MINTER_SERVICE_AUDIENCE }}' 55 | service_url: '${{ vars.TOKEN_MINTER_SERVICE_URL }}' 56 | requested_permissions: |- 57 | { 58 | "scope": "draft-release", 59 | "repositories": ["${{ github.event.repository.name }}"], 60 | "permissions": { 61 | "pull_requests": "write", 62 | "contents": "write" 63 | } 64 | } 65 | 66 | - uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' # ratchet:actions/github-script@v7 67 | id: 'generate-release-notes' 68 | env: 69 | CURRENT_VERSION: '${{ steps.increment-version.outputs.current_version }}' 70 | NEXT_VERSION: '${{ steps.increment-version.outputs.next_version }}' 71 | with: 72 | github-token: '${{ steps.mint-token.outputs.token }}' 73 | script: |- 74 | const releaseNotesResponse = await github.rest.repos.generateReleaseNotes({ 75 | owner: context.repo.owner, 76 | repo: context.repo.repo, 77 | tag_name: `v${process.env.NEXT_VERSION}`, 78 | // Bootstrapping problem for first release, as 0.0.0 tag doesn't exist. 79 | ... (process.env.CURRENT_VERSION != '0.0.0') && { previous_tag_name: `v${process.env.CURRENT_VERSION}`}, 80 | }); 81 | core.setOutput('release-notes', releaseNotesResponse.data.body) 82 | 83 | - name: 'Update Pull Request' 84 | uses: 'abcxyz/actions/.github/actions/create-pull-request@main' # ratchet:exclude 85 | with: 86 | token: '${{ steps.mint-token.outputs.token }}' 87 | base_branch: '${{ github.event.repository.default_branch }}' 88 | head_branch: '${{ env.PR_BRANCH }}' 89 | title: 'Release: v${{ steps.increment-version.outputs.next_version }}' 90 | body: '${{ steps.generate-release-notes.outputs.release-notes }}' 91 | changed_paths: |- 92 | ["VERSION"] 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Local .terraform directories 3 | **/.terraform/* 4 | 5 | # terraform lock file 6 | terraform/.terraform.lock.hcl 7 | 8 | # .tfstate files 9 | *.tfstate 10 | *.tfstate.* 11 | 12 | # Ignore override files as they are usually used to override resources locally and so 13 | # are not checked in 14 | override.tf 15 | override.tf.json 16 | *_override.tf 17 | *_override.tf.json 18 | 19 | ### macOS ### 20 | # General 21 | .DS_Store 22 | .AppleDouble 23 | .LSOverride 24 | 25 | ### Maven ### 26 | target/ 27 | pom.xml.tag 28 | pom.xml.releaseBackup 29 | pom.xml.versionsBackup 30 | pom.xml.next 31 | release.properties 32 | dependency-reduced-pom.xml 33 | buildNumber.properties 34 | .mvn/timing.properties 35 | .mvn/wrapper/maven-wrapper.jar 36 | !**/src/main/**/target/ 37 | !**/src/test/**/target/ 38 | 39 | ### Maven Patch ### 40 | # Eclipse m2e generated files 41 | # Eclipse Core 42 | .project 43 | # JDT-specific (Eclipse Java Development Tools) 44 | .classpath 45 | 46 | ### Java ### 47 | # Compiled class file 48 | *.class 49 | 50 | # Log file 51 | *.log 52 | 53 | # Package Files # 54 | *.jar 55 | *.war 56 | 57 | ### IntelliJ IDEA ### 58 | .idea 59 | *.iws 60 | *.iml 61 | *.ipr 62 | .factorypath 63 | .settings 64 | 65 | # Go workspace file 66 | go.work 67 | 68 | ### Visual Studio Code IDEA ### 69 | .vscode 70 | coverage.out 71 | 72 | *.tfvars 73 | 74 | bin/ 75 | dist/ 76 | 77 | # Env Vars 78 | .env* 79 | 80 | # Ignore generated credentials from google-github-actions/auth 81 | gha-creds-*.json 82 | -------------------------------------------------------------------------------- /.goreleaser.docker.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | env: 16 | # Global env vars for Go build. 17 | - 'CGO_ENABLED=0' 18 | - 'GO111MODULE=on' 19 | - 'GOPROXY=https://proxy.golang.org,direct' 20 | 21 | before: 22 | hooks: 23 | - 'go mod tidy' 24 | 25 | builds: 26 | - id: 'github-metrics-aggregator' 27 | main: './cmd/github-metrics-aggregator' 28 | binary: 'github-metrics-aggregator' 29 | mod_timestamp: '{{ .CommitTimestamp }}' 30 | flags: 31 | - '-a' 32 | - '-trimpath' 33 | ldflags: 34 | - '-s' 35 | - '-w' 36 | - '-X={{ .ModulePath }}/pkg/version.Name=github-metrics-aggregator' 37 | - '-X={{ .ModulePath }}/pkg/version.Version={{ .Version }}' 38 | - '-X={{ .ModulePath }}/pkg/version.Commit={{ .Commit }}' 39 | - '-extldflags=-static' 40 | goos: 41 | - 'linux' 42 | goarch: 43 | - 'amd64' 44 | - 'arm64' 45 | 46 | dockers: 47 | - ids: 48 | - 'github-metrics-aggregator' 49 | use: 'buildx' 50 | goos: 'linux' 51 | goarch: 'amd64' 52 | image_templates: 53 | - '{{ .Env.DOCKER_REPO }}/github-metrics-aggregator:{{ .Env.DOCKER_TAG }}-amd64' 54 | build_flag_templates: 55 | - '--platform=linux/amd64' 56 | - '--pull' 57 | - '--label=org.opencontainers.image.created={{ .CommitTimestamp }}' 58 | - '--label=org.opencontainers.image.description=GitHub Metrics Aggregator command line.' 59 | - '--label=org.opencontainers.image.licenses=Apache-2.0' 60 | - '--label=org.opencontainers.image.name=github-metrics-aggregator' 61 | - '--label=org.opencontainers.image.revision={{ .FullCommit }}' 62 | - '--label=org.opencontainers.image.source={{ .GitURL }}' 63 | - '--label=org.opencontainers.image.title=github-metrics-aggregator' 64 | - '--label=org.opencontainers.image.version={{ .Version }}' 65 | - ids: 66 | - 'github-metrics-aggregator' 67 | use: 'buildx' 68 | goos: 'linux' 69 | goarch: 'arm64' 70 | image_templates: 71 | - '{{ .Env.DOCKER_REPO }}/github-metrics-aggregator:{{ .Env.DOCKER_TAG }}-arm64' 72 | build_flag_templates: 73 | - '--platform=linux/arm64' 74 | - '--pull' 75 | - '--label=org.opencontainers.image.created={{ .CommitTimestamp }}' 76 | - '--label=org.opencontainers.image.description=GitHub Metrics Aggregator command line.' 77 | - '--label=org.opencontainers.image.licenses=Apache-2.0' 78 | - '--label=org.opencontainers.image.name=github-metrics-aggregator' 79 | - '--label=org.opencontainers.image.revision={{ .FullCommit }}' 80 | - '--label=org.opencontainers.image.source={{ .GitURL }}' 81 | - '--label=org.opencontainers.image.title=github-metrics-aggregator' 82 | - '--label=org.opencontainers.image.version={{ .Version }}' 83 | docker_manifests: 84 | - image_templates: 85 | - '{{ .Env.DOCKER_REPO }}/github-metrics-aggregator:{{ .Env.DOCKER_TAG }}-amd64' 86 | - '{{ .Env.DOCKER_REPO }}/github-metrics-aggregator:{{ .Env.DOCKER_TAG }}-arm64' 87 | 88 | # TODO: Follow up on signing. 89 | 90 | # Disable SCM release we only want docker release here. 91 | release: 92 | disable: true 93 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the list of significant contributors. 2 | # 3 | # This does not necessarily list everyone who has contributed code, 4 | # especially since many employees of one corporation may be contributing. 5 | # To see the full list of contributors, see the revision history in 6 | # source control. 7 | 8 | Alphabet 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | We do not accept outside contributions. 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Use distroless for ca certs. 16 | FROM gcr.io/distroless/static 17 | 18 | COPY github-metrics-aggregator /bin/github-metrics-aggregator 19 | 20 | # Normally we would set this to run as "nobody". 21 | # But goreleaser builds the binary locally and sometimes it will mess up the permission 22 | # and cause "exec user process caused: permission denied". 23 | # 24 | # USER nobody 25 | 26 | # Run the CLI 27 | ENTRYPOINT ["/bin/github-metrics-aggregator"] 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # build generates the server go binary 2 | build: 3 | @go build \ 4 | -a \ 5 | -trimpath \ 6 | -ldflags "-s -w -extldflags='-static'" \ 7 | -o ./bin/server \ 8 | ./cmd/webhook 9 | .PHONY: build 10 | 11 | # protoc generates the protos 12 | protoc: 13 | @go generate ./protos 14 | .PHONY: protoc 15 | -------------------------------------------------------------------------------- /RELEASEING.md: -------------------------------------------------------------------------------- 1 | # Releasing 2 | 3 | To create a release for this project follow the following steps: 4 | 5 | 1. Go to the [Draft Release](https://github.com/abcxyz/github-metrics-aggregator/actions/workflows/draft-release.yml) action and click `Run workflow` 6 | 2. Enter the update strategy you would like (`major`, `minor`, `patch`). This project uses [semver](https://semver.org/) for its 7 | versioning which is where these terms originate. 8 | 3. A pull request will then be created with the new version number and release notes. Have this PR reviewed for correctness. 9 | 4. Once the PR is approved and merged, a release will be created with the new version and published to GitHub. 10 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.0.31 2 | -------------------------------------------------------------------------------- /abc.templates/deployments/contents/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Expected to be set externally. 5 | declare -r GITHUB_SHA WEBHOOK_SERVICE_NAME RETRY_SERVICE_NAME PROJECT_ID REGION 6 | 7 | GITHUB_METRICS_VERSION='REPLACE_GITHUB_METRICS_VERSION_TAG' 8 | PLATFORM='amd64' 9 | VERSION="${GITHUB_METRICS_VERSION}-${PLATFORM}" 10 | UPSTREAM_IMAGE_NAME="us-docker.pkg.dev/abcxyz-artifacts/docker-images/github-metrics-aggregator:${VERSION}" 11 | IMAGE_NAME="REPLACE_FULL_IMAGE_NAME:${GITHUB_METRICS_VERSION}-${GITHUB_SHA}" 12 | 13 | echo "Copying ${UPSTREAM_IMAGE_NAME} to ${IMAGE_NAME}..." 14 | crane copy "${UPSTREAM_IMAGE_NAME}" "${IMAGE_NAME}" 15 | 16 | gcloud run services update "${WEBHOOK_SERVICE_NAME}" \ 17 | --quiet \ 18 | --project="${PROJECT_ID}" \ 19 | --region="${REGION}" \ 20 | --image="${IMAGE_NAME}" 21 | 22 | gcloud run services update "${RETRY_SERVICE_NAME}" \ 23 | --quiet \ 24 | --project="${PROJECT_ID}" \ 25 | --region="${REGION}" \ 26 | --image="${IMAGE_NAME}" 27 | -------------------------------------------------------------------------------- /abc.templates/deployments/spec.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: 'cli.abcxyz.dev/v1beta3' 16 | kind: 'Template' 17 | 18 | desc: 'Generate the components necessary to deploy GitHub Metrics Aggregator' 19 | 20 | inputs: 21 | - name: 'custom_name' 22 | desc: 'A custom name for GitHub Metrics Aggregator' 23 | default: 'github-metrics' 24 | - name: 'wif_provider' 25 | desc: 'The Google Cloud workload identity federation provider for GitHub Metrics Aggregator' 26 | - name: 'wif_service_account' 27 | desc: 'The Google Cloud service account for GitHub Metrics Aggregator' 28 | rules: 29 | - rule: 'gcp_matches_service_account(wif_service_account)' 30 | - name: 'project_id' 31 | desc: 'The Google Cloud project ID containing GitHub Metrics Aggregator' 32 | rules: 33 | - rule: 'gcp_matches_project_id(project_id)' 34 | - name: 'full_image_name' 35 | desc: 'The full image name, excluding the version tag' 36 | - name: 'region' 37 | desc: 'The region of the GitHub Metrics Aggregator Cloud Run service' 38 | - name: 'webhook_service_name' 39 | desc: 'The name of the GitHub Metrics Aggregator webhook Cloud Run service' 40 | - name: 'retry_service_name' 41 | desc: 'The name of the GitHub Metrics Aggregator retry Cloud Run service' 42 | 43 | steps: 44 | - desc: 'Include required Dockerfiles and configs' 45 | action: 'include' 46 | params: 47 | paths: 48 | - 'contents' 49 | - 'workflows/deploy-github-metrics-aggregator.yaml' 50 | as: 51 | - '{{toLowerHyphenCase .custom_name}}/deployments' 52 | - '.github/workflows/deploy-{{toLowerHyphenCase .custom_name}}.yaml' 53 | - desc: 'Replace variables' 54 | action: 'string_replace' 55 | params: 56 | paths: 57 | - '{{toLowerHyphenCase .custom_name}}/deployments' 58 | - '.github/workflows' 59 | replacements: 60 | - to_replace: 'REPLACE_CUSTOM_NAME' 61 | with: '{{toLowerHyphenCase .custom_name}}' 62 | - to_replace: 'REPLACE_SUBDIRECTORY' 63 | with: '{{toLowerHyphenCase .custom_name}}/deployments' 64 | - to_replace: 'REPLACE_WIF_PROVIDER' 65 | with: '{{.wif_provider}}' 66 | - to_replace: 'REPLACE_WIF_SERVICE_ACCOUNT' 67 | with: '{{.wif_service_account}}' 68 | - to_replace: 'REPLACE_FULL_IMAGE_NAME' 69 | with: '{{.full_image_name}}' 70 | - to_replace: 'REPLACE_WEBHOOK_SERVICE_NAME' 71 | with: '{{.webhook_service_name}}' 72 | - to_replace: 'REPLACE_RETRY_SERVICE_NAME' 73 | with: '{{.retry_service_name}}' 74 | - to_replace: 'REPLACE_PROJECT_ID' 75 | with: '{{.project_id}}' 76 | - to_replace: 'REPLACE_REGION' 77 | with: '{{.region}}' 78 | - to_replace: 'REPLACE_GITHUB_METRICS_VERSION_TAG' 79 | with: '{{._git_tag}}' 80 | -------------------------------------------------------------------------------- /abc.templates/deployments/workflows/deploy-github-metrics-aggregator.yaml: -------------------------------------------------------------------------------- 1 | name: 'deploy to REPLACE_CUSTOM_NAME' 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | paths: 8 | - 'REPLACE_SUBDIRECTORY/**' 9 | workflow_dispatch: 10 | 11 | env: 12 | WIF_PROVIDER: 'REPLACE_WIF_PROVIDER' 13 | WIF_SERVICE_ACCOUNT: 'REPLACE_WIF_SERVICE_ACCOUNT' 14 | 15 | 16 | # Don't cancel in progress since we don't want to have half-baked releases. 17 | concurrency: 18 | group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}-release' 19 | 20 | permissions: 21 | contents: 'read' 22 | id-token: 'write' 23 | 24 | jobs: 25 | release_and_deploy: 26 | runs-on: 'ubuntu-latest' 27 | environment: 'REPLACE_CUSTOM_NAME' 28 | steps: 29 | - name: 'Checkout' 30 | uses: 'actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b' # ratchet:actions/checkout@v4 31 | 32 | - name: 'Install crane' 33 | env: 34 | VERSION: 'v0.19.1' 35 | run: |- 36 | curl -fsL https://github.com/google/go-containerregistry/releases/download/${VERSION}/go-containerregistry_Linux_x86_64.tar.gz | sudo tar xzf - -C /usr/local/bin crane 37 | 38 | - id: 'auth' 39 | name: 'Authenticate to Google Cloud' 40 | uses: 'google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c' # ratchet:google-github-actions/auth@v2 41 | with: 42 | workload_identity_provider: '${{ env.WIF_PROVIDER }}' 43 | service_account: '${{ env.WIF_SERVICE_ACCOUNT }}' 44 | token_format: 'access_token' 45 | 46 | - name: 'Setup gcloud' 47 | uses: 'google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200' # ratchet:google-github-actions/setup-gcloud@v2 48 | 49 | - uses: 'docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20' # ratchet:docker/login-action@v3 50 | with: 51 | username: 'oauth2accesstoken' 52 | password: '${{ steps.auth.outputs.access_token }}' 53 | registry: 'us-docker.pkg.dev' 54 | 55 | - name: 'Deploy' 56 | run: 'REPLACE_SUBDIRECTORY/deploy.sh' 57 | env: 58 | PROJECT_ID: 'REPLACE_PROJECT_ID' 59 | REGION: 'REPLACE_REGION' 60 | WEBHOOK_SERVICE_NAME: 'REPLACE_WEBHOOK_SERVICE_NAME' 61 | RETRY_SERVICE_NAME: 'REPLACE_RETRY_SERVICE_NAME' 62 | -------------------------------------------------------------------------------- /abc.templates/infra/contents/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | project_id = "REPLACE_PROJECT_ID" 3 | automation_service_account_member = "REPLACE_AUTOMATION_SERVICE_ACCOUNT_MEMBER" 4 | } 5 | 6 | module "REPLACE_MODULE_NAME" { 7 | source = "git::https://github.com/abcxyz/github-metrics-aggregator.git//terraform?ref=REPLACE_GITHUB_METRICS_VERSION_SHA" # REPLACE_GITHUB_METRICS_VERSION_TAG 8 | 9 | project_id = local.project_id 10 | 11 | image = "gcr.io/cloudrun/placeholder@sha256:f1586972ac147796d60ee2c5a0d6cc78067fc862b6d715d6d2a96826455c3423" 12 | bigquery_project_id = local.project_id 13 | automation_service_account_member = local.automation_service_account_member 14 | webhook_domains = ["REPLACE_DOMAIN"] 15 | github_app_id = "REPLACE_GITHUB_APP_ID" 16 | log_level = "info" 17 | retry_service_iam = { 18 | admins = [] 19 | developers = [] 20 | invokers = [] 21 | } 22 | webhook_service_iam = { 23 | admins = [] 24 | developers = [] 25 | invokers = [ 26 | "allUsers", # public access, called by github webhook 27 | ] 28 | } 29 | dataset_iam = { 30 | owners = [] 31 | editors = [] 32 | viewers = [] 33 | } 34 | github_metrics_dashboard = { 35 | enabled = false 36 | viewers = [] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /abc.templates/infra/contents/outputs.tf: -------------------------------------------------------------------------------- 1 | output "gclb_external_ip_name" { 2 | description = "The external IPv4 name assigned to the global fowarding rule for the global load balancer." 3 | value = module.REPLACE_MODULE_NAME.gclb_external_ip_name 4 | } 5 | 6 | output "gclb_external_ip_address" { 7 | description = "The external IPv4 assigned to the global fowarding rule for the global load balancer." 8 | value = module.REPLACE_MODULE_NAME.gclb_external_ip_address 9 | } 10 | 11 | output "webhook_run_service_name" { 12 | description = "The webhook Cloud Run service name." 13 | value = module.REPLACE_MODULE_NAME.webhook_run_service.service_name 14 | } 15 | 16 | output "webhook_run_service_name_url" { 17 | description = "The webhook Cloud Run service url." 18 | value = module.REPLACE_MODULE_NAME.webhook_run_service.service_url 19 | } 20 | 21 | output "retry_run_service_name" { 22 | description = "The retry Cloud Run service name." 23 | value = module.REPLACE_MODULE_NAME.retry_run_service.service_name 24 | } 25 | 26 | output "retry_run_service_url" { 27 | description = "The retry Cloud Run service url." 28 | value = module.REPLACE_MODULE_NAME.retry_run_service.service_url 29 | } 30 | 31 | output "github_metrics_looker_studio_report_link" { 32 | description = "The Looker Studio Linking API link for connecting the data sources for the GitHub Metrics dashboard." 33 | value = module.REPLACE_MODULE_NAME.github_metrics_looker_studio_report_link 34 | } 35 | -------------------------------------------------------------------------------- /abc.templates/infra/contents/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | backend "gcs" { 19 | bucket = "REPLACE_BUCKET_NAME" 20 | prefix = "REPLACE_BUCKET_PREFIX" 21 | } 22 | 23 | required_providers { 24 | google = { 25 | version = "~>5.19" 26 | source = "hashicorp/google" 27 | } 28 | 29 | github = { 30 | source = "integrations/github" 31 | version = ">= 6.0" 32 | } 33 | } 34 | } 35 | 36 | provider "google" { 37 | user_project_override = true 38 | } 39 | -------------------------------------------------------------------------------- /abc.templates/infra/spec.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: 'cli.abcxyz.dev/v1beta3' 16 | kind: 'Template' 17 | 18 | desc: 'Generate the infrastructure necessary to run GitHub Metrics Aggregator' 19 | 20 | inputs: 21 | - name: 'custom_name' 22 | desc: 'A custom name for Github Metrics Aggregator' 23 | default: 'github-metrics' 24 | rules: 25 | - rule: 'matches(custom_name, "^[A-Za-z][0-9A-Za-z-]+[0-9A-Za-z]$")' 26 | message: 'Name can only contain letters, numbers, hyphens(-) and must start with letter.' 27 | - name: 'project_id' 28 | desc: 'The Google Cloud project ID to deploy GitHub Token Minter' 29 | rules: 30 | - rule: 'gcp_matches_project_id(project_id)' 31 | - name: 'automation_service_account_email' 32 | desc: 'The CI service account email for deploying GitHub Metrics Aggregator' 33 | rules: 34 | - rule: 'gcp_matches_service_account(automation_service_account_email)' 35 | - name: 'domain' 36 | desc: 'A custom domain for the Google Cloud Load Balancer' 37 | - name: 'terraform_state_bucket' 38 | desc: 'The Google Cloud storage bucket for Terraform backend state' 39 | - name: 'github_app_id' 40 | desc: 'The ID of the GitHub app' 41 | 42 | steps: 43 | - desc: 'Include required files and directories' 44 | action: 'include' 45 | params: 46 | paths: 47 | - 'contents' 48 | as: 49 | - '{{toLowerHyphenCase .custom_name}}/infra' 50 | 51 | - desc: 'Replace variables' 52 | action: 'string_replace' 53 | params: 54 | paths: 55 | - '{{toLowerHyphenCase .custom_name}}/infra' 56 | replacements: 57 | - to_replace: 'REPLACE_CUSTOM_NAME' 58 | with: '{{toLowerHyphenCase .custom_name}}' 59 | - to_replace: 'REPLACE_MODULE_NAME' 60 | with: '{{toLowerSnakeCase .custom_name}}' 61 | - to_replace: 'REPLACE_PROJECT_ID' 62 | with: '{{.project_id}}' 63 | - to_replace: 'REPLACE_AUTOMATION_SERVICE_ACCOUNT_MEMBER' 64 | with: 'serviceAccount:{{.automation_service_account_email}}' 65 | - to_replace: 'REPLACE_DOMAIN' 66 | with: '{{.domain}}' 67 | - to_replace: 'REPLACE_GITHUB_APP_ID' 68 | with: '{{.github_app_id}}' 69 | - to_replace: 'REPLACE_BUCKET_NAME' 70 | with: '{{.terraform_state_bucket}}' 71 | - to_replace: 'REPLACE_BUCKET_PREFIX' 72 | with: '{{toLowerHyphenCase .custom_name}}/infra' 73 | - to_replace: '# REPLACE_GITHUB_METRICS_VERSION_TAG' 74 | with: |- 75 | {{- if ._git_tag -}} 76 | # {{._git_tag}} 77 | {{- else -}} 78 | {{._git_tag}} 79 | {{- end }} 80 | - to_replace: 'REPLACE_GITHUB_METRICS_VERSION_SHA' 81 | with: '{{._git_sha}}' 82 | 83 | - desc: 'Print warning if rendered without git tag' 84 | action: 'print' 85 | if: '_git_tag == ""' 86 | params: 87 | message: |- 88 | # 89 | # WARNING ############################## 90 | # 91 | # The template was rendered without a valid git tag. For best compatibility, we recommended 92 | # re-rendering this template using one of the latest tags at https://github.com/abcxyz/github-metrics-aggregator/tags. 93 | # 94 | ######################################## 95 | -------------------------------------------------------------------------------- /cmd/github-metrics-aggregator/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Entry point of the application. 16 | package main 17 | 18 | import ( 19 | "context" 20 | "os" 21 | "os/signal" 22 | "syscall" 23 | 24 | "github.com/abcxyz/github-metrics-aggregator/pkg/cli" 25 | "github.com/abcxyz/pkg/logging" 26 | ) 27 | 28 | func main() { 29 | ctx, done := signal.NotifyContext(context.Background(), 30 | syscall.SIGINT, syscall.SIGTERM) 31 | defer done() 32 | 33 | logger := logging.NewFromEnv("") 34 | ctx = logging.WithLogger(ctx, logger) 35 | 36 | if err := realMain(ctx); err != nil { 37 | done() 38 | logger.ErrorContext(ctx, "process exited with error", "error", err) 39 | os.Exit(1) 40 | } 41 | } 42 | 43 | func realMain(ctx context.Context) error { 44 | return cli.Run(ctx, os.Args[1:]) //nolint:wrapcheck // Want passthrough 45 | } 46 | -------------------------------------------------------------------------------- /config/artifacts_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Artifact pipeline definition that reads events from BQ and ingests workflow logs to GCS.", 3 | "name": "artifact_integration", 4 | "parameters": [ 5 | { 6 | "name": "batch-size", 7 | "helpText": "Number of items to process in this execution", 8 | "label": "Batch Size" 9 | }, 10 | { 11 | "name": "events-project-id", 12 | "helpText": "Project ID of the events table", 13 | "label": "Events Table Project ID" 14 | }, 15 | { 16 | "name": "events-table", 17 | "helpText": "Dataset + name of the events table. Must be in the format, i.e. gma.events_table", 18 | "label": "Events Table Name" 19 | }, 20 | { 21 | "name": "leech-project-id", 22 | "helpText": "Project ID of the leech status table", 23 | "label": "Lecch Status Table Project ID" 24 | }, 25 | { 26 | "name": "leech-table", 27 | "helpText": "Dataset + name of the leech table. Must be in the format, ie. gma.leech_status", 28 | "label": "Leech Status Table Name" 29 | }, 30 | { 31 | "name": "logs-bucket-name", 32 | "helpText": "Name of the Cloud Storage bucket to store GitHub logs", 33 | "label": "Logs Bucket Name" 34 | }, 35 | { 36 | "name": "github-app-id", 37 | "helpText": "The GitHub app ID", 38 | "label": "GitHub App ID" 39 | }, 40 | { 41 | "name": "github-install-id", 42 | "helpText": "The GitHub app's installation ID", 43 | "label": "GitHub Installation ID" 44 | }, 45 | { 46 | "name": "github-private-key-secret", 47 | "helpText": "Fully qualified path to a secret containing the GitHub app's private key, i.e. projects//secrets//versions/", 48 | "label": "GitHub Private Key Secret" 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /config/commit-review-status-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Pipeline definition that reads push events from BigQuery and outputs the approval status of commits", 3 | "name": "commit-review-status", 4 | "parameters": [ 5 | { 6 | "helpText": "The token to use when authenticating with GitHub", 7 | "label": "GitHub Token", 8 | "name": "github-token", 9 | "isOptional": true 10 | }, 11 | { 12 | "helpText": "The provisioned GitHub App reference", 13 | "label": "GitHub App ID", 14 | "name": "github-app-id", 15 | "isOptional": true 16 | }, 17 | { 18 | "helpText": "The provisioned GitHub App Installation reference", 19 | "label": "GitHub App Installation ID", 20 | "name": "github-app-installation-id", 21 | "isOptional": true 22 | }, 23 | { 24 | "helpText": "The resource name for the secret manager resource containing the GitHub App private key", 25 | "label": "GitHub App Private Key Resource Name", 26 | "name": "github-app-private-key-resource-name", 27 | "isOptional": true 28 | }, 29 | { 30 | "helpText": "The name of the push events table. The value provided must be a fully qualified BigQuery table name of the form :.", 31 | "label": "Push Events Table", 32 | "name": "push-events-table" 33 | }, 34 | { 35 | "helpText": "The name of the commit review status table. The value provided must be a fully qualified BigQuery table name of the form :.
", 36 | "label": "Commit Review Status Table", 37 | "name": "commit-review-status-table" 38 | }, 39 | { 40 | "helpText": "The name of the issues table. The value provided must be a fully qualified BigQuery table name of the form :.
", 41 | "label": "Issues Table", 42 | "name": "issues-table" 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /docs/playbooks/alerts/BadRequests.md: -------------------------------------------------------------------------------- 1 | # Bad Requests 2 | 3 | ## Webhook Service Bad Requests High 4 | 5 | This alert fires when the webhook Cloud Run service is experiencing a high level of bad requests. The alert policy monitors the `request-count` metric and checks for the response code class 4xx. 6 | 7 | The service throws a 400 when the payload from GitHub is empty which is unexpected. 8 | 9 | ### Triage Steps 10 | 11 | 1. Navigate to Log Explorer and set the date range to match the range when the error was observed. 12 | 2. Query for `severity=ERROR jsonPayload.code=~"4[0-9][0-9]+" resource.type="cloud_run_revision resource.labels.service_name=~"github-metrics-webhook-....""` 13 | 14 | ## Retry Service Bad Requests High 15 | 16 | This alert fires when the retry Cloud Run service is experiencing a high level of bad requests. The alert policy monitors the `request-count` metric and checks for the response code class 4xx. 17 | 18 | ### Triage Steps 19 | 20 | 1. Navigate to Log Explorer and set the date range to match the range when the error was observed. 21 | 2. Query for `severity=ERROR jsonPayload.code=~"4[0-9][0-9]+" resource.type="cloud_run_revision resource.labels.service_name=~"github-metrics-retry-....""` 22 | -------------------------------------------------------------------------------- /docs/playbooks/alerts/ContainerUsage.md: -------------------------------------------------------------------------------- 1 | # ContainerUsage 2 | 3 | ## High CPU Utilization 4 | 5 | This alert fires when a Cloud Run service or job is experiencing high CPU utilization across all container instances. The alert policy monitors the P99 sampling metric. We consider any lower p value to be too relaxed. 6 | 7 | ### Triage Steps 8 | 9 | 1. Observe the metric over a longer period of time (days or weeks) to see if the CPU utilization has been steadily increasing. 10 | 2. Review recent deployments to see if a recent change may have caused the increase. 11 | -------------------------------------------------------------------------------- /docs/playbooks/alerts/ForwardProgressFailed.md: -------------------------------------------------------------------------------- 1 | # ForwardProgressFailed 2 | 3 | ## Services 4 | 5 | This alert fires when cloud run services have not made forward progress in an acceptable amount of time. 6 | 7 | - `github-metrics-retry-XXXX` - Checks for failed event deliveries since the last run and retries delivery. The default cadence for this service is to run hourly and is configured by Cloud Scheduler. 8 | - `github-metrics-webhook-XXXX` - Triggered by GitHub webhook events. 9 | 10 | 11 | ### Retry Service Triage Steps 12 | 13 | When retry service does not execute within a configured interval, this alert will fire. 14 | 15 | To begin triage, find the root cause by doing the following: 16 | 17 | 1. Go to the Cloud Run page in your GCP project. 18 | 2. Confirm the selected tab is Services (this is the default). 19 | 3. Select the retry service and select the Revisions tab. 20 | 4. Ensure the latest deployment is successful, if not review the logs under the Logs tab and review for errors. 21 | 6. If the latest service revision is successful, navigate to Cloud Scheduler by searching for it Cloud Scheduler in the search bar and confirm the cadence is aligned with the expected alert policy interval. If it is not, adjust the alert policy in terraform titled `revision_alert_policy` to better align to the new cadence. 22 | 23 | ### Webhook Service Triage Steps 24 | 25 | The webhook service execution rate is tightly coupled to how active a team is on GitHub. This alert will fire anytime there are no events within a configured interval. 26 | 27 | 1. Go to the Cloud Run page in your GCP project. 28 | 2. Confirm the selected tab is Services (this is the default). 29 | 3. Select the retry service and select the Revisions tab. 30 | 4. Ensure the latest deployment is successful, if not review the logs under the Logs tab and review for errors. 31 | 5. If the latest service revision is successful, or there are no errors, then consider increasing the time window. Alternatively write a workflow in a GitHub repo that executes on an hourly basis. This event should get processed by the webhook service and bridge these gaps of missing data. 32 | 33 | ## Jobs 34 | 35 | This alert fires when background jobs have not made forward progress in an acceptable amount of time. The alert will include the name of the job that is failing to make forward progress. The jobs are invoked in the background. 36 | 37 | - `gma-artifacts` - Writes workflow logs from GitHub to Cloud Storage. 38 | 39 | - `commit-review-status-job` - Audits pull requests and writes the results to BigQuery 40 | 41 | Each job runs on a different interval. Check your Terraform configuration to see how frequently a specific job runs. 42 | 43 | ### Cloud Run Job Triage Steps 44 | 45 | When one of the jobs does not return success within a configured interval, this alert will fire. For most cases, this means the job has already failed 2+ times. 46 | 47 | To begin triage, identify the offending job and its root cause by doing the following: 48 | 49 | 1. Go to the Cloud Run page in your GCP project. 50 | 2. Click on the Jobs tab. 51 | 3. Under the History tab, identify the failing execution(s) of the job and select one. 52 | 4. Once selected, a second screen of the execution details will appear. 53 | 5. Under the Tasks tab, select View Logs. 54 | 6. Review logs for errors. 55 | -------------------------------------------------------------------------------- /docs/playbooks/alerts/RequestLatency.md: -------------------------------------------------------------------------------- 1 | # Request Latency 2 | 3 | ## Webhook Service P99 Latency High 4 | 5 | This alert fires when the webhook Cloud Run service is experiencing a high level of request latency. The alert policy monitors the `request_latencies` metric and sample for the P99 latency. 6 | 7 | ### Triage Steps 8 | 9 | 1. Go to the Cloud Run page in your GCP project. 10 | 2. Confirm the selected tab is Services (this is the default). 11 | 3. Select the webhook service and select the Metrics tab. 12 | 4. Review the request count graph and look for the latent time frame. 13 | 5. Navigate to Trace Explorer in Cloud Monitoring. 14 | 6. Sort by highest latency and ensure the relative time frame includes the time when the spike occurred. 15 | 7. Select the failing span and view the Logs & Events tab and view the error log. 16 | 17 | ## Retry Service P99 Latency High 18 | 19 | This alert fires when the retry Cloud Run service is experiencing a high level of request latency. The alert policy monitors the `request_latencies` metric and sample for the P99 latency. 20 | 21 | ### Triage Steps 22 | 23 | 1. Go to the Cloud Run page in your GCP project. 24 | 2. Confirm the selected tab is Services (this is the default). 25 | 3. Select the webhook service and select the Metrics tab. 26 | 4. Review the request count graph and look for the latent time frame. 27 | 5. Navigate to Trace Explorer in Cloud Monitoring. 28 | 6. Sort by highest latency and ensure the relative time frame includes the time when the spike occurred. 29 | 7. Select the failing span and view the Logs & Events tab and view the error log. 30 | -------------------------------------------------------------------------------- /docs/playbooks/alerts/ServerFaults.md: -------------------------------------------------------------------------------- 1 | # Server Faults 2 | 3 | ## Webhook Service Faults High 4 | 5 | This alert fires when the webhook Cloud Run service is experiencing server faults. The alert policy monitors the `request-count` metric and checks for the response code class 5xx. 6 | 7 | Event failures emitted by the webhook service are later processed by the retry service. This means users do not have to manually reattempt event delivery. 8 | 9 | ### Triage Steps 10 | 11 | 1. Navigate to Log Explorer and set the date range to match the range when the error was observed. 12 | 2. Query for `severity=ERROR jsonPayload.code=~"5[0-9][0-9]+" resource.type="cloud_run_revision resource.labels.service_name=~"github-metrics-webhook-....""` 13 | 3. Write failures to BigQuery will be retried, no further action to take. 14 | 15 | 16 | ## Retry Service Faults High 17 | 18 | This alert fires when the retry Cloud Run service is experiencing server faults. The alert policy monitors the `request-count` metric and checks for the response code class 5xx. 19 | 20 | ### Triage Steps 21 | 22 | 1. Navigate to Log Explorer and set the date range to match the range when the error was observed. 23 | 2. Query for `severity=ERROR jsonPayload.code=~"4[0-9][0-9]+" resource.type="cloud_run_revision resource.labels.service_name=~"github-metrics-retry-....""` 24 | 3. Review logs for recent runs to identify the failure. 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/abcxyz/github-metrics-aggregator 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.0 6 | 7 | require ( 8 | cloud.google.com/go/bigquery v1.66.2 9 | cloud.google.com/go/pubsub v1.47.0 10 | cloud.google.com/go/storage v1.50.0 11 | github.com/abcxyz/pkg v1.4.0 12 | github.com/google/go-cmp v0.6.0 13 | github.com/google/go-github/v61 v61.0.0 14 | github.com/google/uuid v1.6.0 15 | github.com/sethvargo/go-envconfig v1.1.1 16 | github.com/sethvargo/go-gcslock v0.1.3 17 | github.com/sethvargo/go-retry v0.3.0 18 | github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 19 | golang.org/x/oauth2 v0.26.0 20 | google.golang.org/api v0.221.0 21 | google.golang.org/grpc v1.70.0 22 | google.golang.org/protobuf v1.36.5 23 | ) 24 | 25 | require ( 26 | cel.dev/expr v0.20.0 // indirect 27 | cloud.google.com/go v0.118.2 // indirect 28 | cloud.google.com/go/auth v0.14.1 // indirect 29 | cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect 30 | cloud.google.com/go/compute/metadata v0.6.0 // indirect 31 | cloud.google.com/go/iam v1.4.0 // indirect 32 | cloud.google.com/go/monitoring v1.24.0 // indirect 33 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0 // indirect 34 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect 35 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect 36 | github.com/apache/arrow/go/v15 v15.0.2 // indirect 37 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 38 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 39 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 40 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 41 | github.com/felixge/httpsnoop v1.0.4 // indirect 42 | github.com/go-logr/logr v1.4.2 // indirect 43 | github.com/go-logr/stdr v1.2.2 // indirect 44 | github.com/goccy/go-json v0.10.5 // indirect 45 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 46 | github.com/google/flatbuffers v25.2.10+incompatible // indirect 47 | github.com/google/go-querystring v1.1.0 // indirect 48 | github.com/google/s2a-go v0.1.9 // indirect 49 | github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect 50 | github.com/googleapis/gax-go/v2 v2.14.1 // indirect 51 | github.com/klauspost/compress v1.17.11 // indirect 52 | github.com/klauspost/cpuid/v2 v2.2.9 // indirect 53 | github.com/kr/text v0.2.0 // indirect 54 | github.com/mattn/go-isatty v0.0.20 // indirect 55 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 56 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 57 | github.com/posener/complete/v2 v2.1.0 // indirect 58 | github.com/posener/script v1.2.0 // indirect 59 | github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect 60 | github.com/zeebo/xxh3 v1.0.2 // indirect 61 | go.einride.tech/aip v0.68.1 // indirect 62 | go.opencensus.io v0.24.0 // indirect 63 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 64 | go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect 65 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect 66 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect 67 | go.opentelemetry.io/otel v1.34.0 // indirect 68 | go.opentelemetry.io/otel/metric v1.34.0 // indirect 69 | go.opentelemetry.io/otel/sdk v1.34.0 // indirect 70 | go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect 71 | go.opentelemetry.io/otel/trace v1.34.0 // indirect 72 | golang.org/x/crypto v0.36.0 // indirect 73 | golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect 74 | golang.org/x/mod v0.23.0 // indirect 75 | golang.org/x/net v0.38.0 // indirect 76 | golang.org/x/sync v0.12.0 // indirect 77 | golang.org/x/sys v0.31.0 // indirect 78 | golang.org/x/text v0.23.0 // indirect 79 | golang.org/x/time v0.10.0 // indirect 80 | golang.org/x/tools v0.30.0 // indirect 81 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect 82 | google.golang.org/genproto v0.0.0-20250212204824-5a70512c5d8b // indirect 83 | google.golang.org/genproto/googleapis/api v0.0.0-20250212204824-5a70512c5d8b // indirect 84 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b // indirect 85 | gopkg.in/yaml.v3 v3.0.1 // indirect 86 | ) 87 | -------------------------------------------------------------------------------- /integration/bigquery.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "testing" 21 | "time" 22 | 23 | "cloud.google.com/go/bigquery" 24 | "github.com/sethvargo/go-retry" 25 | ) 26 | 27 | // makeBigQueryClient creates a new client and automatically closes the 28 | // connection when the tests finish. 29 | func makeBigQueryClient(ctx context.Context, tb testing.TB, projectID string) *bigquery.Client { 30 | tb.Helper() 31 | 32 | client, err := bigquery.NewClient(ctx, projectID) 33 | if err != nil { 34 | tb.Fatal(err) 35 | } 36 | 37 | tb.Cleanup(func() { 38 | if err := client.Close(); err != nil { 39 | tb.Errorf("failed to close the biquery client: %v", err) 40 | } 41 | }) 42 | 43 | return client 44 | } 45 | 46 | // makeQuery generates a bigquery Query from a query string and parameters. 47 | func makeQuery(bqClient bigquery.Client, queryString string, queryParameters *[]bigquery.QueryParameter) *bigquery.Query { 48 | bqQuery := bqClient.Query(queryString) 49 | bqQuery.Parameters = *queryParameters 50 | return bqQuery 51 | } 52 | 53 | // queryIfNumRowsExistWithRetries queries the DB and checks if the expected 54 | // number of rows exists or not. If not, the query will be retried with the 55 | // specified retry inputs. 56 | func queryIfNumRowsExistWithRetries(ctx context.Context, tb testing.TB, bqQuery *bigquery.Query, retryWaitDuration time.Duration, retryLimit uint64, testName string, expectedNum int64) { 57 | tb.Helper() 58 | 59 | b := retry.NewExponential(retryWaitDuration) 60 | if err := retry.Do(ctx, retry.WithMaxRetries(retryLimit, b), func(ctx context.Context) error { 61 | found, err := queryIfNumRowsExist(ctx, tb, bqQuery, expectedNum) 62 | if found { 63 | // Early exit retry if rows already found. 64 | return nil 65 | } 66 | 67 | tb.Log("matching entry not found, retrying...") 68 | 69 | if err != nil { 70 | tb.Logf("query error: %v.", err) 71 | } 72 | return retry.RetryableError(fmt.Errorf("no matching records found in bigquery after timeout for %q", testName)) 73 | }); err != nil { 74 | tb.Errorf("Retry failed: %v.", err) 75 | } 76 | } 77 | 78 | // queryIfNumRowsExist queries the DB and checks if the expected number of rows 79 | // exists or not. 80 | func queryIfNumRowsExist(ctx context.Context, tb testing.TB, query *bigquery.Query, expectedNum int64) (bool, error) { 81 | tb.Helper() 82 | 83 | q, err := query.Run(ctx) 84 | if err != nil { 85 | return false, fmt.Errorf("failed to run query: %w", err) 86 | } 87 | 88 | it, err := q.Read(ctx) 89 | if err != nil { 90 | return false, fmt.Errorf("failed to read query results: %w", err) 91 | } 92 | 93 | var row []bigquery.Value 94 | if err := it.Next(&row); err != nil { 95 | return false, fmt.Errorf("failed to get next row: %w", err) 96 | } 97 | 98 | // Check if the matching row count is equal to expected, if yes, then the rows exists. 99 | tb.Logf("Found %d matching rows", row[0]) 100 | result, ok := row[0].(int64) 101 | if !ok { 102 | return false, fmt.Errorf("error converting query results to integer value (got %T)", row[0]) 103 | } 104 | return result == expectedNum, nil 105 | } 106 | -------------------------------------------------------------------------------- /integration/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/sethvargo/go-envconfig" 23 | ) 24 | 25 | type config struct { 26 | ProjectID string `env:"PROJECT_ID,required"` 27 | DatasetID string `env:"DATASET_ID,required"` 28 | IDToken string `env:"ID_TOKEN,required"` 29 | GitHubWebhookSecret string `env:"GITHUB_WEBHOOK_SECRET,required"` 30 | EndpointURL string `env:"ENDPOINT_URL,required"` 31 | HTTPRequestTimeout time.Duration `env:"HTTP_REQUEST_TIMEOUT,default=60s"` 32 | QueryRetryWaitDuration time.Duration `env:"QUERY_RETRY_WAIT_DURATION,default=5s"` 33 | QueryRetryLimit uint64 `env:"QUERY_RETRY_COUNT,default=5"` 34 | } 35 | 36 | func newTestConfig(ctx context.Context) (*config, error) { 37 | var c config 38 | if err := envconfig.Process(ctx, &c); err != nil { 39 | return nil, fmt.Errorf("failed to process environment: %w", err) 40 | } 41 | return &c, nil 42 | } 43 | -------------------------------------------------------------------------------- /integration/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integration 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "crypto/hmac" 21 | "crypto/sha256" 22 | "encoding/hex" 23 | "fmt" 24 | "net/http" 25 | "os" 26 | "path" 27 | "testing" 28 | 29 | "cloud.google.com/go/bigquery" 30 | "github.com/google/uuid" 31 | 32 | "github.com/abcxyz/github-metrics-aggregator/pkg/webhook" 33 | "github.com/abcxyz/pkg/testutil" 34 | ) 35 | 36 | func validateCfg(t *testing.T) *config { 37 | t.Helper() 38 | 39 | cfg, err := newTestConfig(t.Context()) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | return cfg 45 | } 46 | 47 | // createSignature creates a HMAC 256 signature for the test request payload. 48 | func createSignature(key, payload []byte) string { 49 | mac := hmac.New(sha256.New, key) 50 | mac.Write(payload) 51 | return hex.EncodeToString(mac.Sum(nil)) 52 | } 53 | 54 | func TestHTTPEndpoints(t *testing.T) { 55 | t.Parallel() 56 | testutil.SkipIfNotIntegration(t) 57 | 58 | cfg := validateCfg(t) 59 | ctx := t.Context() 60 | requestID := uuid.New().String() 61 | 62 | resp, err := makeHTTPRequest(requestID, cfg.EndpointURL, cfg) 63 | if err != nil { 64 | t.Fatalf("error calling service url: %v", err) 65 | } 66 | defer resp.Body.Close() 67 | 68 | if resp.StatusCode != http.StatusCreated { 69 | t.Fatalf("invalid http response code got: %d, want: %d", resp.StatusCode, http.StatusCreated) 70 | } 71 | 72 | bqClient := makeBigQueryClient(ctx, t, cfg.ProjectID) 73 | query := generateQuery(bqClient, requestID, cfg.ProjectID, cfg.DatasetID) 74 | queryIfNumRowsExistWithRetries(ctx, t, query, cfg.QueryRetryWaitDuration, cfg.QueryRetryLimit, "test-main", 1) 75 | } 76 | 77 | func makeHTTPRequest(requestID, endpointURL string, cfg *config) (*http.Response, error) { 78 | payload, err := os.ReadFile(path.Join("..", "testdata", "pull_request.json")) 79 | if err != nil { 80 | return nil, fmt.Errorf("failed to create payload from file: %w", err) 81 | } 82 | 83 | req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, endpointURL, bytes.NewReader(payload)) 84 | if err != nil { 85 | return nil, fmt.Errorf("failed to create audit log http request: %w", err) 86 | } 87 | 88 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.IDToken)) 89 | req.Header.Add(webhook.DeliveryIDHeader, requestID) 90 | req.Header.Add(webhook.EventTypeHeader, "pull_request") 91 | req.Header.Add(webhook.SHA256SignatureHeader, fmt.Sprintf("sha256=%s", createSignature([]byte(cfg.GitHubWebhookSecret), payload))) 92 | 93 | httpClient := &http.Client{Timeout: cfg.HTTPRequestTimeout} 94 | resp, err := httpClient.Do(req) 95 | if err != nil { 96 | return nil, fmt.Errorf("failed to execute audit log request: %w", err) 97 | } 98 | return resp, nil 99 | } 100 | 101 | func generateQuery(bqClient *bigquery.Client, requestID, projectID, datasetID string) *bigquery.Query { 102 | queryString := fmt.Sprintf("SELECT COUNT(1) FROM `%s.%s`", projectID, datasetID) 103 | queryString += ` WHERE delivery_id = @request_id` 104 | return makeQuery(*bqClient, queryString, &[]bigquery.QueryParameter{{Name: "request_id", Value: requestID}}) 105 | } 106 | -------------------------------------------------------------------------------- /pkg/artifact/job.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package artifact 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "runtime" 21 | 22 | "github.com/abcxyz/github-metrics-aggregator/pkg/bq" 23 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 24 | "github.com/abcxyz/pkg/logging" 25 | "github.com/abcxyz/pkg/workerpool" 26 | ) 27 | 28 | // ExecuteJob runs the ingestion pipeline job to read GitHub 29 | // action workflow logs from GitHub and store them into GCS. 30 | func ExecuteJob(ctx context.Context, cfg *Config) error { 31 | logger := logging.FromContext(ctx) 32 | 33 | bqClient, err := bq.NewBigQuery(ctx, cfg.ProjectID, cfg.DatasetID) 34 | if err != nil { 35 | return fmt.Errorf("failed to create bigquery client: %w", err) 36 | } 37 | defer bqClient.Close() 38 | 39 | // Create a pool of workers to manage all of the log ingestions 40 | pool := workerpool.New[ArtifactRecord](&workerpool.Config{ 41 | Concurrency: int64(runtime.NumCPU()), 42 | StopOnError: false, 43 | }) 44 | 45 | // Setup a log ingester to process ingestion events 46 | logsFn, err := NewLogIngester(ctx, cfg.ProjectID, cfg.BucketName, cfg.GitHubAppID, cfg.GitHubInstallID, cfg.GitHubPrivateKeySecret) 47 | if err != nil { 48 | return fmt.Errorf("failed to create log ingester: %w", err) 49 | } 50 | 51 | logger.InfoContext(ctx, "ingestion job starting", 52 | "name", version.Name, 53 | "commit", version.Commit, 54 | "version", version.Version) 55 | 56 | // Read up to `BatchSize` number of events that need to be processed 57 | query, err := makeQuery(bqClient, cfg.EventsTableID, cfg.ArtifactsTableID, cfg.BatchSize) 58 | if err != nil { 59 | return fmt.Errorf("failed to populate query template: %w", err) 60 | } 61 | events, err := bq.Query[EventRecord](ctx, bqClient, query) 62 | if err != nil { 63 | return fmt.Errorf("failed to query bigquery for events: %w", err) 64 | } 65 | 66 | // Fan out the work of processing all of the events that were found 67 | for _, event := range events { 68 | if err := pool.Do(ctx, func() (ArtifactRecord, error) { 69 | artifact := logsFn.ProcessElement(ctx, *event) 70 | // Errors are handled by the element processor and are flagged as special 71 | // artifact records. There is no possible error returned from processing. 72 | return artifact, nil 73 | }); err != nil { 74 | return fmt.Errorf("failed to submit job to worker pool: %w", err) 75 | } 76 | } 77 | // When all of the workers are complete, extract the result values 78 | results, err := pool.Done(ctx) 79 | if err != nil { 80 | return fmt.Errorf("failed to ingest logs for events: %w", err) 81 | } 82 | artifacts := make([]*ArtifactRecord, 0, len(results)) 83 | for _, v := range results { 84 | artifacts = append(artifacts, &v.Value) 85 | } 86 | 87 | // Save all of the result records to the output table 88 | if err := bq.Write[ArtifactRecord](ctx, bqClient, cfg.ArtifactsTableID, artifacts); err != nil { 89 | return fmt.Errorf("failed to write artifacts to bigquery: %w", err) 90 | } 91 | 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/artifact/query.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package artifact 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "text/template" 21 | 22 | "github.com/abcxyz/github-metrics-aggregator/pkg/bq" 23 | ) 24 | 25 | // sourceQuery is the driving BigQuery query that selects events 26 | // that need to be processed. 27 | const sourceQuery = ` 28 | SELECT 29 | delivery_id, 30 | JSON_VALUE(payload, "$.repository.full_name") repo_slug, 31 | JSON_VALUE(payload, "$.repository.name") repo_name, 32 | JSON_VALUE(payload, "$.repository.owner.login") org_name, 33 | JSON_VALUE(payload, "$.workflow_run.logs_url") logs_url, 34 | JSON_VALUE(payload, "$.workflow_run.actor.login") github_actor, 35 | JSON_VALUE(payload, "$.workflow_run.html_url") workflow_url, 36 | JSON_VALUE(payload, "$.workflow_run.id") workflow_run_id, 37 | JSON_VALUE(payload, "$.workflow_run.run_attempt") workflow_run_attempt, 38 | ARRAY( 39 | SELECT 40 | JSON_QUERY(pull_request, "$.number") 41 | FROM UNNEST( 42 | JSON_QUERY_ARRAY(payload, "$.workflow_run.pull_requests") 43 | ) pull_request 44 | ) pull_request_numbers 45 | FROM {{.BT}}{{.ProjectID}}.{{.DatasetID}}.{{.EventTableID}}{{.BT}} 46 | WHERE 47 | event = "workflow_run" 48 | AND JSON_VALUE(payload, "$.workflow_run.status") = "completed" 49 | AND delivery_id NOT IN ( 50 | SELECT 51 | delivery_id 52 | FROM {{.BT}}{{.ProjectID}}.{{.DatasetID}}.{{.ArtifactTableID}}{{.BT}} 53 | ) 54 | LIMIT {{.BatchSize}} 55 | ` 56 | 57 | type queryParameters struct { 58 | ProjectID string 59 | DatasetID string 60 | EventTableID string 61 | ArtifactTableID string 62 | BatchSize int 63 | BT string 64 | } 65 | 66 | // makeQuery renders a string template representing the SQL query. 67 | func makeQuery(client *bq.BigQuery, eventsTable, artifactTable string, batchSize int) (string, error) { 68 | tmpl, err := template.New("query").Parse(sourceQuery) 69 | if err != nil { 70 | return "", fmt.Errorf("failed to parse query template: %w", err) 71 | } 72 | 73 | var sb strings.Builder 74 | if err := tmpl.Execute(&sb, &queryParameters{ 75 | ProjectID: client.ProjectID, 76 | DatasetID: client.DatasetID, 77 | EventTableID: eventsTable, 78 | ArtifactTableID: artifactTable, 79 | BatchSize: batchSize, 80 | BT: "`", 81 | }); err != nil { 82 | return "", fmt.Errorf("failed to apply query template parameters: %w", err) 83 | } 84 | return sb.String(), nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/artifact/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package artifact 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io" 21 | "regexp" 22 | "strings" 23 | 24 | "cloud.google.com/go/storage" 25 | ) 26 | 27 | // ObjectWriter is an interface for writing a object/blob to a storage medium. 28 | type ObjectWriter interface { 29 | Write(ctx context.Context, content io.Reader, descriptor string) error 30 | } 31 | 32 | // ObjectStore is an implementation of the ObjectWriter interface that 33 | // writes to Cloud Storage. 34 | type ObjectStore struct { 35 | client *storage.Client 36 | } 37 | 38 | // NewObjectStore creates a ObjectWriter implementation that uses cloud storage 39 | // to store its objects. 40 | func NewObjectStore(ctx context.Context) (*ObjectStore, error) { 41 | sc, err := storage.NewClient(ctx) 42 | if err != nil { 43 | return nil, fmt.Errorf("error initializaing cloud storage client: %w", err) 44 | } 45 | return &ObjectStore{client: sc}, nil 46 | } 47 | 48 | // Write writes an object to Google Cloud Storage. 49 | func (s *ObjectStore) Write(ctx context.Context, content io.Reader, objectDescriptor string) error { 50 | // Split the descriptor into chunks 51 | bucketName, objectName, _, err := parseGCSURI(objectDescriptor) 52 | if err != nil { 53 | return fmt.Errorf("failed to parse gcs uri: %w", err) 54 | } 55 | 56 | // Connect to bucket 57 | bucket := s.client.Bucket(bucketName) 58 | // Setup the GCS object with the filename to write to 59 | obj := bucket.Object(objectName) 60 | 61 | writer := obj.NewWriter(ctx) 62 | 63 | if _, err := io.Copy(writer, content); err != nil { 64 | return fmt.Errorf("failed to copy contents of reader to cloud storage object: %w", err) 65 | } 66 | 67 | // File appears in GCS after Close 68 | if err := writer.Close(); err != nil { 69 | return fmt.Errorf("failed to close gcs file: %w", err) 70 | } 71 | 72 | return nil 73 | } 74 | 75 | // parseGCSURI parses a gcs uri of the type gs://blah/blah/blah.blah 76 | // The parts are: 77 | // 78 | // bucket name 79 | // object path 80 | // file name 81 | // 82 | // Throws an error if the uri cannot be parsed. 83 | func parseGCSURI(objectURI string) (string, string, string, error) { 84 | // First verify that all of the parts exist 85 | r, _ := regexp.Compile("gs://(.*)/(.*)") 86 | if !r.MatchString(objectURI) { 87 | return "", "", "", fmt.Errorf("invalid uri: [%s]", objectURI) 88 | } 89 | // Extract bucket name by splitting string by '/' 90 | // take the 3rd item in the list (index position 2) which is the bucket name 91 | parts := strings.Split(objectURI, "/") 92 | bucket := parts[2] 93 | 94 | // Extract object name by splitting string to remove gs:// prefix and bucket name 95 | // rejoin to rebuild the file path 96 | objectName := strings.Join(parts[3:], "/") 97 | // Extract the last segment as the filename 98 | fileName := parts[len(parts)-1] 99 | return bucket, objectName, fileName, nil 100 | } 101 | -------------------------------------------------------------------------------- /pkg/bq/bigquery.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bq 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "fmt" 21 | 22 | "cloud.google.com/go/bigquery" 23 | "google.golang.org/api/iterator" 24 | "google.golang.org/api/option" 25 | 26 | "github.com/abcxyz/pkg/logging" 27 | ) 28 | 29 | // BigQuery provides a client and dataset identifiers. 30 | type BigQuery struct { 31 | ProjectID string 32 | DatasetID string 33 | client *bigquery.Client 34 | } 35 | 36 | // NewBigQuery creates a new instance of a BigQuery client. 37 | func NewBigQuery(ctx context.Context, projectID, datasetID string, opts ...option.ClientOption) (*BigQuery, error) { 38 | client, err := bigquery.NewClient(ctx, projectID, opts...) 39 | if err != nil { 40 | return nil, fmt.Errorf("failed to create new bigquery client: %w", err) 41 | } 42 | 43 | return &BigQuery{ 44 | ProjectID: projectID, 45 | DatasetID: datasetID, 46 | client: client, 47 | }, nil 48 | } 49 | 50 | // Close releases any resources held by the BigQuery client. 51 | func (bq *BigQuery) Close() error { 52 | if err := bq.client.Close(); err != nil { 53 | return fmt.Errorf("failed to close BigQuery client: %w", err) 54 | } 55 | return nil 56 | } 57 | 58 | // Query takes a queryString (assumed to be valid SQL) and executes it against 59 | // BigQuery using the given client. The results are then mapped to a slice of T, 60 | // where each row in the result is mapped to a struct of type T. 61 | func Query[T any](ctx context.Context, bq *BigQuery, queryString string) ([]*T, error) { 62 | logger := logging.FromContext(ctx) 63 | logger.DebugContext(ctx, "executing query", 64 | "project_id", bq.client.Project(), 65 | "query", queryString, 66 | ) 67 | query := bq.client.Query(queryString) 68 | job, err := query.Run(ctx) 69 | if err != nil { 70 | return nil, fmt.Errorf("query.Run failed: %w", err) 71 | } 72 | rows, err := job.Read(ctx) 73 | if err != nil { 74 | return nil, fmt.Errorf("job.Read failed: %w", err) 75 | } 76 | return rowsToSlice[T](rows, rows.TotalRows) 77 | } 78 | 79 | func Write[T any](ctx context.Context, bq *BigQuery, tableID string, rows []*T) error { 80 | logger := logging.FromContext(ctx) 81 | logger.DebugContext(ctx, "writing rows", 82 | "project_id", bq.client.Project(), 83 | "dataset_id", bq.DatasetID, 84 | "table_id", tableID, 85 | "num_rows", len(rows), 86 | ) 87 | if err := bq.client.Dataset(bq.DatasetID).Table(tableID).Inserter().Put(ctx, rows); err != nil { 88 | return fmt.Errorf("failed to write to BigQuery: %w", err) 89 | } 90 | return nil 91 | } 92 | 93 | type rowItr[T any] interface { 94 | Next(t interface{}) error 95 | } 96 | 97 | func rowsToSlice[T any](rows rowItr[T], totalRows uint64) ([]*T, error) { 98 | items := make([]*T, 0, totalRows) 99 | for { 100 | var t T 101 | err := rows.Next(&t) 102 | if errors.Is(err, iterator.Done) { 103 | break 104 | } 105 | if err != nil { 106 | return nil, fmt.Errorf("rows.Next failed: %w", err) 107 | } 108 | items = append(items, &t) 109 | } 110 | return items, nil 111 | } 112 | -------------------------------------------------------------------------------- /pkg/bq/bigquery_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bq 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/google/go-cmp/cmp" 22 | "google.golang.org/api/iterator" 23 | ) 24 | 25 | type TestStruct struct { 26 | StringField string 27 | IntField int 28 | } 29 | 30 | type TestRowIterator struct { 31 | rows []TestStruct 32 | } 33 | 34 | func (ri *TestRowIterator) Next(i interface{}) error { 35 | if len(ri.rows) == 0 { 36 | return iterator.Done 37 | } 38 | t, ok := i.(*TestStruct) 39 | if !ok { 40 | return fmt.Errorf("invalid type %T", i) 41 | } 42 | t.StringField = ri.rows[0].StringField 43 | t.IntField = ri.rows[0].IntField 44 | ri.rows = ri.rows[1:] 45 | return nil 46 | } 47 | 48 | func TestRowsToSlice(t *testing.T) { 49 | t.Parallel() 50 | cases := []struct { 51 | name string 52 | rows rowItr[TestStruct] 53 | totalRows uint64 54 | want []*TestStruct 55 | }{ 56 | { 57 | name: "no_items", 58 | rows: &TestRowIterator{ 59 | rows: []TestStruct{}, 60 | }, 61 | totalRows: 0, 62 | want: []*TestStruct{}, 63 | }, 64 | { 65 | name: "one_item", 66 | rows: &TestRowIterator{ 67 | rows: []TestStruct{ 68 | { 69 | StringField: "a", 70 | IntField: 1, 71 | }, 72 | }, 73 | }, 74 | totalRows: 1, 75 | want: []*TestStruct{ 76 | { 77 | StringField: "a", 78 | IntField: 1, 79 | }, 80 | }, 81 | }, 82 | { 83 | name: "several_items", 84 | rows: &TestRowIterator{ 85 | rows: []TestStruct{ 86 | { 87 | StringField: "a", 88 | IntField: 1, 89 | }, 90 | { 91 | StringField: "b", 92 | IntField: 2, 93 | }, 94 | { 95 | StringField: "c", 96 | IntField: 3, 97 | }, 98 | }, 99 | }, 100 | totalRows: 2, 101 | want: []*TestStruct{ 102 | { 103 | StringField: "a", 104 | IntField: 1, 105 | }, 106 | { 107 | StringField: "b", 108 | IntField: 2, 109 | }, 110 | { 111 | StringField: "c", 112 | IntField: 3, 113 | }, 114 | }, 115 | }, 116 | } 117 | 118 | for _, tc := range cases { 119 | t.Run(tc.name, func(t *testing.T) { 120 | t.Parallel() 121 | got, _ := rowsToSlice[TestStruct](tc.rows, tc.totalRows) 122 | if diff := cmp.Diff(got, tc.want); diff != "" { 123 | t.Errorf("unexpected result (-got, +want):\n%s", diff) 124 | } 125 | }) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /pkg/cli/artifact.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cli 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/abcxyz/github-metrics-aggregator/pkg/artifact" 22 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 23 | "github.com/abcxyz/pkg/cli" 24 | "github.com/abcxyz/pkg/logging" 25 | ) 26 | 27 | var _ cli.Command = (*ArtifactJobCommand)(nil) 28 | 29 | // The ArtifactJobCommand is a Cloud Run job that will read workflow event 30 | // records from BigQuery and ingest any available logs into cloud storage. 31 | // 32 | // The job acts as a GitHub App for authentication purposes. 33 | type ArtifactJobCommand struct { 34 | cli.BaseCommand 35 | 36 | cfg *artifact.Config 37 | 38 | // testFlagSetOpts is only used for testing. 39 | testFlagSetOpts []cli.Option 40 | } 41 | 42 | func (c *ArtifactJobCommand) Desc() string { 43 | return `Execute an artifact retrieval job for GitHub Metrics Aggregator` 44 | } 45 | 46 | func (c *ArtifactJobCommand) Help() string { 47 | return ` 48 | Usage: {{ COMMAND }} [options] 49 | Execute an artifact retrieval job for GitHub Metrics Aggregator 50 | ` 51 | } 52 | 53 | func (c *ArtifactJobCommand) Flags() *cli.FlagSet { 54 | c.cfg = &artifact.Config{} 55 | set := cli.NewFlagSet(c.testFlagSetOpts...) 56 | return c.cfg.ToFlags(set) 57 | } 58 | 59 | func (c *ArtifactJobCommand) Run(ctx context.Context, args []string) error { 60 | f := c.Flags() 61 | if err := f.Parse(args); err != nil { 62 | return fmt.Errorf("failed to parse flags: %w", err) 63 | } 64 | args = f.Args() 65 | if len(args) > 0 { 66 | return fmt.Errorf("unexpected arguments: %q", args) 67 | } 68 | 69 | logger := logging.FromContext(ctx) 70 | logger.DebugContext(ctx, "running job", 71 | "name", version.Name, 72 | "commit", version.Commit, 73 | "version", version.Version) 74 | 75 | if err := c.cfg.Validate(); err != nil { 76 | return fmt.Errorf("invalid configuration: %w", err) 77 | } 78 | logger.DebugContext(ctx, "loaded configuration", "config", c.cfg) 79 | 80 | if err := artifact.ExecuteJob(ctx, c.cfg); err != nil { 81 | logger.ErrorContext(ctx, "error executing artifact job", "error", err) 82 | return fmt.Errorf("job execution failed: %w", err) 83 | } 84 | 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /pkg/cli/retry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cli 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "net/http" 21 | 22 | "google.golang.org/api/option" 23 | 24 | "github.com/abcxyz/github-metrics-aggregator/pkg/retry" 25 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 26 | "github.com/abcxyz/pkg/cli" 27 | "github.com/abcxyz/pkg/logging" 28 | "github.com/abcxyz/pkg/renderer" 29 | "github.com/abcxyz/pkg/serving" 30 | ) 31 | 32 | var _ cli.Command = (*RetryServerCommand)(nil) 33 | 34 | type RetryServerCommand struct { 35 | cli.BaseCommand 36 | 37 | cfg *retry.Config 38 | 39 | // testFlagSetOpts is only used for testing. 40 | testFlagSetOpts []cli.Option 41 | 42 | // testDatastore is only used for testing 43 | testDatastore retry.Datastore 44 | 45 | // testGCSLockClientOptions is only used for testing 46 | testGCSLockClientOptions []option.ClientOption 47 | 48 | // testGitHub is only used for testing 49 | testGitHub retry.GitHubSource 50 | } 51 | 52 | func (c *RetryServerCommand) Desc() string { 53 | return `Start a retry server for GitHub Metrics Aggregator` 54 | } 55 | 56 | func (c *RetryServerCommand) Help() string { 57 | return ` 58 | Usage: {{ COMMAND }} [options] 59 | Start a retry server for GitHub Metrics Aggregator. 60 | ` 61 | } 62 | 63 | func (c *RetryServerCommand) Flags() *cli.FlagSet { 64 | c.cfg = &retry.Config{} 65 | set := cli.NewFlagSet(c.testFlagSetOpts...) 66 | return c.cfg.ToFlags(set) 67 | } 68 | 69 | func (c *RetryServerCommand) Run(ctx context.Context, args []string) error { 70 | server, mux, err := c.RunUnstarted(ctx, args) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | return server.StartHTTPHandler(ctx, mux) 76 | } 77 | 78 | func (c *RetryServerCommand) RunUnstarted(ctx context.Context, args []string) (*serving.Server, http.Handler, error) { 79 | f := c.Flags() 80 | if err := f.Parse(args); err != nil { 81 | return nil, nil, fmt.Errorf("failed to parse flags: %w", err) 82 | } 83 | args = f.Args() 84 | if len(args) > 0 { 85 | return nil, nil, fmt.Errorf("unexpected arguments: %q", args) 86 | } 87 | 88 | logger := logging.FromContext(ctx) 89 | logger.DebugContext(ctx, "server starting", 90 | "name", version.Name, 91 | "commit", version.Commit, 92 | "version", version.Version) 93 | 94 | h, err := renderer.New(ctx, nil, 95 | renderer.WithOnError(func(err error) { 96 | logger.ErrorContext(ctx, "failed to render", "error", err) 97 | })) 98 | if err != nil { 99 | return nil, nil, fmt.Errorf("failed to create renderer: %w", err) 100 | } 101 | 102 | if err := c.cfg.Validate(); err != nil { 103 | return nil, nil, fmt.Errorf("invalid configuration: %w", err) 104 | } 105 | logger.DebugContext(ctx, "loaded configuration", "config", c.cfg) 106 | 107 | retryClientOptions := &retry.RetryClientOptions{} 108 | 109 | if c.testDatastore != nil { 110 | retryClientOptions.DatastoreClientOverride = c.testDatastore 111 | } 112 | 113 | if c.testGCSLockClientOptions != nil { 114 | retryClientOptions.GCSLockClientOpts = c.testGCSLockClientOptions 115 | } 116 | 117 | if c.testGitHub != nil { 118 | retryClientOptions.GitHubOverride = c.testGitHub 119 | } 120 | 121 | retryServer, err := retry.NewServer(ctx, h, c.cfg, retryClientOptions) 122 | if err != nil { 123 | return nil, nil, fmt.Errorf("failed to create server: %w", err) 124 | } 125 | 126 | mux := retryServer.Routes(ctx) 127 | 128 | server, err := serving.New(c.cfg.Port) 129 | if err != nil { 130 | return nil, nil, fmt.Errorf("failed to create serving infrastructure: %w", err) 131 | } 132 | 133 | return server, mux, nil 134 | } 135 | -------------------------------------------------------------------------------- /pkg/cli/review.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cli 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/abcxyz/github-metrics-aggregator/pkg/review" 22 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 23 | "github.com/abcxyz/pkg/cli" 24 | "github.com/abcxyz/pkg/logging" 25 | ) 26 | 27 | var _ cli.Command = (*ReviewJobCommand)(nil) 28 | 29 | // The ReviewJobCommand is a Cloud Run job that will read commits 30 | // from BigQuery and verify that they received proper review. 31 | // 32 | // The job acts as a GitHub App for authentication purposes. 33 | type ReviewJobCommand struct { 34 | cli.BaseCommand 35 | 36 | cfg *review.Config 37 | 38 | // testFlagSetOpts is only used for testing. 39 | testFlagSetOpts []cli.Option 40 | } 41 | 42 | func (c *ReviewJobCommand) Desc() string { 43 | return `Execute an artifact retrieval job for GitHub Metrics Aggregator` 44 | } 45 | 46 | func (c *ReviewJobCommand) Help() string { 47 | return ` 48 | Usage: {{ COMMAND }} [options] 49 | Execute an artifact retrieval job for GitHub Metrics Aggregator 50 | ` 51 | } 52 | 53 | func (c *ReviewJobCommand) Flags() *cli.FlagSet { 54 | c.cfg = &review.Config{} 55 | set := cli.NewFlagSet(c.testFlagSetOpts...) 56 | return c.cfg.ToFlags(set) 57 | } 58 | 59 | func (c *ReviewJobCommand) Run(ctx context.Context, args []string) error { 60 | f := c.Flags() 61 | if err := f.Parse(args); err != nil { 62 | return fmt.Errorf("failed to parse flags: %w", err) 63 | } 64 | args = f.Args() 65 | if len(args) > 0 { 66 | return fmt.Errorf("unexpected arguments: %q", args) 67 | } 68 | 69 | logger := logging.FromContext(ctx) 70 | logger.DebugContext(ctx, "running job", 71 | "name", version.Name, 72 | "commit", version.Commit, 73 | "version", version.Version) 74 | 75 | if err := c.cfg.Validate(); err != nil { 76 | return fmt.Errorf("invalid configuration: %w", err) 77 | } 78 | logger.DebugContext(ctx, "loaded configuration", "config", c.cfg) 79 | 80 | if err := review.ExecuteJob(ctx, c.cfg); err != nil { 81 | return fmt.Errorf("job execution failed: %w", err) 82 | } 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/cli/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package cli implements the commands for the github-metrics-aggregator CLI. 16 | package cli 17 | 18 | import ( 19 | "context" 20 | 21 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 22 | "github.com/abcxyz/pkg/cli" 23 | ) 24 | 25 | var rootCmd = func() cli.Command { 26 | return &cli.RootCommand{ 27 | Name: "github-metrics-aggregator", 28 | Version: version.HumanVersion, 29 | Commands: map[string]cli.CommandFactory{ 30 | "retry": func() cli.Command { 31 | return &cli.RootCommand{ 32 | Name: "retry", 33 | Description: "Perform retry operations", 34 | Commands: map[string]cli.CommandFactory{ 35 | "server": func() cli.Command { 36 | return &RetryServerCommand{} 37 | }, 38 | }, 39 | } 40 | }, 41 | "webhook": func() cli.Command { 42 | return &cli.RootCommand{ 43 | Name: "webhook", 44 | Description: "Perform webhook operations", 45 | Commands: map[string]cli.CommandFactory{ 46 | "server": func() cli.Command { 47 | return &WebhookServerCommand{} 48 | }, 49 | }, 50 | } 51 | }, 52 | "job": func() cli.Command { 53 | return &cli.RootCommand{ 54 | Name: "job", 55 | Description: "Execute a Cloud Run job", 56 | Commands: map[string]cli.CommandFactory{ 57 | "artifact": func() cli.Command { 58 | return &ArtifactJobCommand{} 59 | }, 60 | "review": func() cli.Command { 61 | return &ReviewJobCommand{} 62 | }, 63 | }, 64 | } 65 | }, 66 | }, 67 | } 68 | } 69 | 70 | // Run executes the CLI. 71 | func Run(ctx context.Context, args []string) error { 72 | return rootCmd().Run(ctx, args) //nolint:wrapcheck // Want passthrough 73 | } 74 | -------------------------------------------------------------------------------- /pkg/cli/root_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cli 16 | 17 | import ( 18 | "strings" 19 | "testing" 20 | ) 21 | 22 | func TestRootCommand_Help(t *testing.T) { 23 | t.Parallel() 24 | 25 | exp := ` 26 | Usage: github-metrics-aggregator COMMAND 27 | 28 | job Execute a Cloud Run job 29 | retry Perform retry operations 30 | webhook Perform webhook operations 31 | ` 32 | 33 | cmd := rootCmd() 34 | if got, want := strings.TrimSpace(cmd.Help()), strings.TrimSpace(exp); got != want { 35 | t.Errorf("expected\n\n%s\n\nto be\n\n%s\n\n", got, want) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/cli/webhook.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cli 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "net/http" 21 | 22 | "google.golang.org/api/option" 23 | 24 | "github.com/abcxyz/github-metrics-aggregator/pkg/version" 25 | "github.com/abcxyz/github-metrics-aggregator/pkg/webhook" 26 | "github.com/abcxyz/pkg/cli" 27 | "github.com/abcxyz/pkg/logging" 28 | "github.com/abcxyz/pkg/renderer" 29 | "github.com/abcxyz/pkg/serving" 30 | ) 31 | 32 | var _ cli.Command = (*WebhookServerCommand)(nil) 33 | 34 | type WebhookServerCommand struct { 35 | cli.BaseCommand 36 | 37 | cfg *webhook.Config 38 | 39 | // testFlagSetOpts is only used for testing. 40 | testFlagSetOpts []cli.Option 41 | 42 | testPubSubClientOptions []option.ClientOption 43 | 44 | // testDatastore is only used for testing 45 | testDatastore webhook.Datastore 46 | } 47 | 48 | func (c *WebhookServerCommand) Desc() string { 49 | return `Start a webhook server for GitHub Metrics Aggregator` 50 | } 51 | 52 | func (c *WebhookServerCommand) Help() string { 53 | return ` 54 | Usage: {{ COMMAND }} [options] 55 | Start a webhook server for GitHub Metrics Aggregator. 56 | ` 57 | } 58 | 59 | func (c *WebhookServerCommand) Flags() *cli.FlagSet { 60 | c.cfg = &webhook.Config{} 61 | set := cli.NewFlagSet(c.testFlagSetOpts...) 62 | return c.cfg.ToFlags(set) 63 | } 64 | 65 | func (c *WebhookServerCommand) Run(ctx context.Context, args []string) error { 66 | server, mux, err := c.RunUnstarted(ctx, args) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | return server.StartHTTPHandler(ctx, mux) 72 | } 73 | 74 | func (c *WebhookServerCommand) RunUnstarted(ctx context.Context, args []string) (*serving.Server, http.Handler, error) { 75 | f := c.Flags() 76 | if err := f.Parse(args); err != nil { 77 | return nil, nil, fmt.Errorf("failed to parse flags: %w", err) 78 | } 79 | args = f.Args() 80 | if len(args) > 0 { 81 | return nil, nil, fmt.Errorf("unexpected arguments: %q", args) 82 | } 83 | 84 | logger := logging.FromContext(ctx) 85 | logger.DebugContext(ctx, "server starting", 86 | "name", version.Name, 87 | "commit", version.Commit, 88 | "version", version.Version) 89 | 90 | h, err := renderer.New(ctx, nil, 91 | renderer.WithOnError(func(err error) { 92 | logger.ErrorContext(ctx, "failed to render", "error", err) 93 | })) 94 | if err != nil { 95 | return nil, nil, fmt.Errorf("failed to create renderer: %w", err) 96 | } 97 | 98 | if err := c.cfg.Validate(); err != nil { 99 | return nil, nil, fmt.Errorf("invalid configuration: %w", err) 100 | } 101 | logger.DebugContext(ctx, "loaded configuration", "config", c.cfg) 102 | 103 | agent := fmt.Sprintf("abcxyz:github-metrics-aggregator/%s", version.Version) 104 | opts := append([]option.ClientOption{option.WithUserAgent(agent)}, c.testPubSubClientOptions...) 105 | webhookClientOptions := &webhook.WebhookClientOptions{ 106 | DLQEventPubsubClientOpts: opts, 107 | EventPubsubClientOpts: opts, 108 | } 109 | 110 | // expect tests to pass this attribute 111 | if c.testDatastore != nil { 112 | webhookClientOptions.DatastoreClientOverride = c.testDatastore 113 | } 114 | 115 | webhookServer, err := webhook.NewServer(ctx, h, c.cfg, webhookClientOptions) 116 | if err != nil { 117 | return nil, nil, fmt.Errorf("failed to create server: %w", err) 118 | } 119 | 120 | mux := webhookServer.Routes(ctx) 121 | 122 | server, err := serving.New(c.cfg.Port) 123 | if err != nil { 124 | return nil, nil, fmt.Errorf("failed to create serving infrastructure: %w", err) 125 | } 126 | 127 | return server, mux, nil 128 | } 129 | -------------------------------------------------------------------------------- /pkg/githubclient/githubclient.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package githubclient 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/google/go-github/v61/github" 22 | "golang.org/x/oauth2" 23 | 24 | "github.com/abcxyz/pkg/githubauth" 25 | ) 26 | 27 | type GitHub struct { 28 | client *github.Client 29 | } 30 | 31 | // New creates a new instance of a GitHub client. 32 | func New(ctx context.Context, appID, rsaPrivateKeyPEM string) (*GitHub, error) { 33 | signer, err := githubauth.NewPrivateKeySigner(rsaPrivateKeyPEM) 34 | if err != nil { 35 | return nil, fmt.Errorf("failed to create private key signer: %w", err) 36 | } 37 | app, err := githubauth.NewApp(appID, signer) 38 | if err != nil { 39 | return nil, fmt.Errorf("failed to create github app: %w", err) 40 | } 41 | 42 | ts := app.OAuthAppTokenSource() 43 | client := github.NewClient(oauth2.NewClient(ctx, ts)) 44 | 45 | return &GitHub{ 46 | client: client, 47 | }, nil 48 | } 49 | 50 | // ListDeliveries lists a paginated result of event deliveries. 51 | func (gh *GitHub) ListDeliveries(ctx context.Context, opts *github.ListCursorOptions) ([]*github.HookDelivery, *github.Response, error) { 52 | deliveries, resp, err := gh.client.Apps.ListHookDeliveries(ctx, opts) 53 | if err != nil { 54 | return deliveries, resp, fmt.Errorf("failed to list deliveries: %w", err) 55 | } 56 | return deliveries, resp, nil 57 | } 58 | 59 | // RedeliverEvent redelivers a failed event which will be picked up by the webhook service. 60 | func (gh *GitHub) RedeliverEvent(ctx context.Context, deliveryID int64) error { 61 | _, _, err := gh.client.Apps.RedeliverHookDelivery(ctx, deliveryID) 62 | if err != nil { 63 | return fmt.Errorf("failed to redeliver event: %w", err) 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/retry/bigquery_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package retry 16 | 17 | import "context" 18 | 19 | type retrieveCheckpointIDRes struct { 20 | res string 21 | err error 22 | } 23 | 24 | type writeCheckpointIDRes struct { 25 | err error 26 | } 27 | 28 | type deliveryEventExistsRes struct { 29 | res bool 30 | err error 31 | } 32 | 33 | type MockDatastore struct { 34 | retrieveCheckpointID *retrieveCheckpointIDRes 35 | writeCheckpointID *writeCheckpointIDRes 36 | deliveryEventExists *deliveryEventExistsRes 37 | } 38 | 39 | func (f *MockDatastore) WriteFailureEvent(ctx context.Context, failureEventTableID, deliveryID, createdAt string) error { 40 | return nil 41 | } 42 | 43 | func (f *MockDatastore) RetrieveCheckpointID(ctx context.Context, checkpointTableID string) (string, error) { 44 | if f.retrieveCheckpointID != nil { 45 | return f.retrieveCheckpointID.res, f.retrieveCheckpointID.err 46 | } 47 | return "", nil 48 | } 49 | 50 | func (f *MockDatastore) WriteCheckpointID(ctx context.Context, checkpointTableID, deliveryID, createdAt string) error { 51 | if f.writeCheckpointID != nil { 52 | return f.writeCheckpointID.err 53 | } 54 | return nil 55 | } 56 | 57 | func (f *MockDatastore) DeliveryEventExists(ctx context.Context, eventsTableID, deliveryID string) (bool, error) { 58 | if f.deliveryEventExists != nil { 59 | return f.deliveryEventExists.res, f.deliveryEventExists.err 60 | } 61 | return false, nil 62 | } 63 | 64 | func (f *MockDatastore) Close() error { 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/retry/github_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package retry 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/google/go-github/v61/github" 21 | ) 22 | 23 | type listDeliveriesRes struct { 24 | deliveries []*github.HookDelivery 25 | res *github.Response 26 | err error 27 | } 28 | 29 | type redeliverEventRes struct { 30 | err error 31 | } 32 | 33 | type MockGitHub struct { 34 | listDeliveries *listDeliveriesRes 35 | redeliverEvent *redeliverEventRes 36 | } 37 | 38 | func (m *MockGitHub) ListDeliveries(ctx context.Context, opts *github.ListCursorOptions) ([]*github.HookDelivery, *github.Response, error) { 39 | if m.listDeliveries != nil { 40 | return m.listDeliveries.deliveries, m.listDeliveries.res, m.listDeliveries.err 41 | } 42 | return []*github.HookDelivery{}, nil, nil 43 | } 44 | 45 | func (m *MockGitHub) RedeliverEvent(ctx context.Context, deliveryID int64) error { 46 | if m.redeliverEvent != nil { 47 | return m.redeliverEvent.err 48 | } 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/retry/lock_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package retry 16 | 17 | import ( 18 | "context" 19 | "time" 20 | ) 21 | 22 | type acquireRes struct { 23 | err error 24 | } 25 | 26 | type closeRes struct { 27 | err error 28 | } 29 | 30 | type MockLock struct { 31 | acquire *acquireRes 32 | close *closeRes 33 | } 34 | 35 | func (m *MockLock) Acquire(context.Context, time.Duration) error { 36 | return m.acquire.err 37 | } 38 | 39 | func (m *MockLock) Close(context.Context) error { 40 | return m.close.err 41 | } 42 | -------------------------------------------------------------------------------- /pkg/review/breakglass_query.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package review 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "text/template" 21 | "time" 22 | ) 23 | 24 | // breakGlassIssueSQL is the BigQuery query that searches for a 25 | // break glass issues created by given user and within a specified time frame. 26 | const breakGlassIssueSQL = ` 27 | SELECT 28 | issues.html_url html_url 29 | FROM 30 | {{.BT}}{{.ProjectID}}.{{.DatasetID}}.{{.IssuesTableID}}{{.BT}} issues 31 | WHERE 32 | issues.repository = 'breakglass' 33 | AND author = '{{.Author}}' 34 | AND issues.created_at <= TIMESTAMP('{{.Timestamp}}') 35 | AND issues.closed_at >= TIMESTAMP('{{.Timestamp}}') 36 | ` 37 | 38 | type bgQueryParameters struct { 39 | ProjectID string 40 | DatasetID string 41 | IssuesTableID string 42 | Author string 43 | Timestamp string 44 | BT string 45 | } 46 | 47 | // makeBreakglassQuery returns a BigQuery query that searches for a break glass 48 | // issue created by given user and within a specified time frame. 49 | func makeBreakglassQuery(cfg *Config, author string, timestamp *time.Time) (string, error) { 50 | tmpl, err := template.New("breakglass-query").Parse(breakGlassIssueSQL) 51 | if err != nil { 52 | return "", fmt.Errorf("failed to parse query template: %w", err) 53 | } 54 | 55 | var sb strings.Builder 56 | if err := tmpl.Execute(&sb, &bgQueryParameters{ 57 | ProjectID: cfg.ProjectID, 58 | DatasetID: cfg.DatasetID, 59 | IssuesTableID: cfg.IssuesTableID, 60 | Author: author, 61 | Timestamp: timestamp.Format(time.RFC3339), 62 | BT: "`", 63 | }); err != nil { 64 | return "", fmt.Errorf("failed to apply query template parameters: %w", err) 65 | } 66 | return sb.String(), nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/review/breakglass_query_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package review 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/google/go-cmp/cmp" 22 | ) 23 | 24 | func TestGetBreakGlassIssueQuery(t *testing.T) { 25 | t.Parallel() 26 | cases := []struct { 27 | name string 28 | cfg *Config 29 | user string 30 | timestamp time.Time 31 | want string 32 | }{ 33 | { 34 | name: "query_template_populated_correctly", 35 | cfg: defaultConfig, 36 | user: "bbechtel", 37 | timestamp: time.Date(2023, 8, 15, 23, 21, 34, 0, time.UTC), 38 | want: ` 39 | SELECT 40 | issues.html_url html_url 41 | FROM 42 | ` + "`my_project.my_dataset.issues`" + ` issues 43 | WHERE 44 | issues.repository = 'breakglass' 45 | AND author = 'bbechtel' 46 | AND issues.created_at <= TIMESTAMP('2023-08-15T23:21:34Z') 47 | AND issues.closed_at >= TIMESTAMP('2023-08-15T23:21:34Z') 48 | `, 49 | }, 50 | } 51 | for _, tc := range cases { 52 | t.Run(tc.name, func(t *testing.T) { 53 | t.Parallel() 54 | 55 | got, err := makeBreakglassQuery(tc.cfg, tc.user, &tc.timestamp) 56 | if err != nil { 57 | t.Errorf("unexpected error making breakglass query: %v", err) 58 | } 59 | if diff := cmp.Diff(got, tc.want); diff != "" { 60 | t.Errorf("GetBreakGlassIssueQuery unexpected result (-got,+want):\n%s", diff) 61 | } 62 | }) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /pkg/review/commit_query.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package review 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "text/template" 21 | ) 22 | 23 | // commitSQL is the BigQuery query that selects the commits that need 24 | // to be processed. The criteria for a commit that needs to be processed are: 25 | // 1. The commit was pushed to the repository's default branch. 26 | // 2. We do not have a record for the commit in the commit_review_status table. 27 | const commitSQL = ` 28 | WITH 29 | commits AS ( 30 | SELECT 31 | push_events.pusher author, 32 | push_events.organization, 33 | push_events.repository, 34 | push_events.repository_default_branch branch, 35 | push_events.repository_visibility visibility, 36 | JSON_VALUE(commit_json, '$.id') commit_sha, 37 | TIMESTAMP(JSON_VALUE(commit_json, '$.timestamp')) commit_timestamp, 38 | FROM 39 | {{.BT}}{{.ProjectID}}.{{.DatasetID}}.{{.PushEventsTableID}}{{.BT}} push_events, 40 | UNNEST(push_events.commits) commit_json 41 | WHERE 42 | push_events.ref = CONCAT('refs/heads/', push_events.repository_default_branch) ) 43 | SELECT 44 | commits.author, 45 | commits.organization, 46 | commits.repository, 47 | commits.branch, 48 | commits.visibility, 49 | commits.commit_sha, 50 | commits.commit_timestamp 51 | FROM 52 | commits 53 | LEFT JOIN 54 | {{.BT}}{{.ProjectID}}.{{.DatasetID}}.{{.CommitReviewStatusTableID}}{{.BT}} commit_review_status 55 | ON 56 | commit_review_status.commit_sha = commits.commit_sha 57 | WHERE 58 | commit_review_status.commit_sha IS NULL 59 | ` 60 | 61 | type queryParameters struct { 62 | ProjectID string 63 | DatasetID string 64 | PushEventsTableID string 65 | CommitReviewStatusTableID string 66 | BT string 67 | } 68 | 69 | // makeCommitQuery returns a BigQuery query that selects the commits that need to be 70 | // processed. 71 | func makeCommitQuery(cfg *Config) (string, error) { 72 | tmpl, err := template.New("commit-query").Parse(commitSQL) 73 | if err != nil { 74 | return "", fmt.Errorf("failed to parse query template: %w", err) 75 | } 76 | 77 | var sb strings.Builder 78 | if err := tmpl.Execute(&sb, &queryParameters{ 79 | ProjectID: cfg.ProjectID, 80 | DatasetID: cfg.DatasetID, 81 | PushEventsTableID: cfg.PushEventsTableID, 82 | CommitReviewStatusTableID: cfg.CommitReviewStatusTableID, 83 | BT: "`", 84 | }); err != nil { 85 | return "", fmt.Errorf("failed to apply query template parameters: %w", err) 86 | } 87 | return sb.String(), nil 88 | } 89 | -------------------------------------------------------------------------------- /pkg/review/commit_query_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package review 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/google/go-cmp/cmp" 21 | ) 22 | 23 | func TestGetCommitQuery(t *testing.T) { 24 | t.Parallel() 25 | cases := []struct { 26 | name string 27 | cfg *Config 28 | want string 29 | }{ 30 | { 31 | name: "query_template_populated_correctly", 32 | cfg: defaultConfig, 33 | want: ` 34 | WITH 35 | commits AS ( 36 | SELECT 37 | push_events.pusher author, 38 | push_events.organization, 39 | push_events.repository, 40 | push_events.repository_default_branch branch, 41 | push_events.repository_visibility visibility, 42 | JSON_VALUE(commit_json, '$.id') commit_sha, 43 | TIMESTAMP(JSON_VALUE(commit_json, '$.timestamp')) commit_timestamp, 44 | FROM 45 | ` + "`my_project.my_dataset.push_events`" + ` push_events, 46 | UNNEST(push_events.commits) commit_json 47 | WHERE 48 | push_events.ref = CONCAT('refs/heads/', push_events.repository_default_branch) ) 49 | SELECT 50 | commits.author, 51 | commits.organization, 52 | commits.repository, 53 | commits.branch, 54 | commits.visibility, 55 | commits.commit_sha, 56 | commits.commit_timestamp 57 | FROM 58 | commits 59 | LEFT JOIN 60 | ` + "`my_project.my_dataset.commit_review_status`" + ` commit_review_status 61 | ON 62 | commit_review_status.commit_sha = commits.commit_sha 63 | WHERE 64 | commit_review_status.commit_sha IS NULL 65 | `, 66 | }, 67 | } 68 | for _, tc := range cases { 69 | t.Run(tc.name, func(t *testing.T) { 70 | t.Parallel() 71 | 72 | got, _ := makeCommitQuery(tc.cfg) 73 | if diff := cmp.Diff(got, tc.want); diff != "" { 74 | t.Errorf("GetCommitQuery got unexpected result (-got,+want):\n%s", diff) 75 | } 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /pkg/review/issue_fetcher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package review 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/abcxyz/github-metrics-aggregator/pkg/bq" 23 | ) 24 | 25 | // BreakGlassIssueFetcher fetches break glass issues from a data source. 26 | type BreakGlassIssueFetcher interface { 27 | // getBreakGlassIssues retrieves all break glass issues created by the given 28 | // author and whose open duration contains the specified timestamp. 29 | // The issue's open duration contains the timestamp if 30 | // issue.created_at <= timestamp <= issue.closed_at holds. 31 | fetch(ctx context.Context, cfg *Config, author string, timestamp *time.Time) ([]*breakGlassIssue, error) 32 | } 33 | 34 | // BigQueryBreakGlassIssueFetcher implements the BreakGlassIssueFetcher 35 | // interface and fetches the break glass issue data from BigQuery. 36 | type BigQueryBreakGlassIssueFetcher struct { 37 | client *bq.BigQuery 38 | } 39 | 40 | func (bqif *BigQueryBreakGlassIssueFetcher) fetch(ctx context.Context, cfg *Config, author string, timestamp *time.Time) ([]*breakGlassIssue, error) { 41 | issueQuery, err := makeBreakglassQuery(cfg, author, timestamp) 42 | if err != nil { 43 | return nil, fmt.Errorf("failed to create breakglass query: %w", err) 44 | } 45 | items, err := bq.Query[breakGlassIssue](ctx, bqif.client, issueQuery) 46 | if err != nil { 47 | return nil, fmt.Errorf("client.Query failed: %w", err) 48 | } 49 | return items, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/teeth/bigquery_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package teeth 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/google/go-cmp/cmp" 21 | ) 22 | 23 | const ( 24 | testProjectID = "github_metrics_aggregator" 25 | testDatasetID = "1234-asdf-9876" 26 | testPullRequestEventsTable = "pull_request_events" 27 | testEventsTable = "events" 28 | testLeechTable = "leech_status" 29 | testInvocationCommentTable = "invocation_comment_status" 30 | ) 31 | 32 | func TestPopulatePublisherSourceQuery(t *testing.T) { 33 | t.Parallel() 34 | 35 | want := `-- Copyright 2024 The Authors (see AUTHORS file) 36 | -- 37 | -- Licensed under the Apache License, Version 2.0 (the "License"); 38 | -- you may not use this file except in compliance with the License. 39 | -- You may obtain a copy of the License at 40 | -- 41 | -- http://www.apache.org/licenses/LICENSE-2.0 42 | -- 43 | -- Unless required by applicable law or agreed to in writing, software 44 | -- distributed under the License is distributed on an "AS IS" BASIS, 45 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 46 | -- See the License for the specific language governing permissions and 47 | -- limitations under the License. 48 | SELECT 49 | pull_request_events.delivery_id, 50 | delivery_events.pull_request_id, 51 | html_url AS pull_request_html_url, 52 | delivery_events.received, 53 | logs_uri, 54 | head_sha 55 | FROM 56 | ` + "`" + testProjectID + "." + testDatasetID + "." + testPullRequestEventsTable + "`" + ` AS pull_request_events 57 | JOIN ( 58 | SELECT 59 | delivery_id, 60 | received, 61 | logs_uri, 62 | SAFE.INT64(pull_request.id) AS pull_request_id, 63 | LAX_STRING(pull_request.url) AS pull_request_url, 64 | LAX_STRING(events.payload.workflow_run.head_sha) AS head_sha, 65 | FROM 66 | ` + "`" + testProjectID + "." + testDatasetID + "." + testLeechTable + "`" + ` leech_status 67 | JOIN ( 68 | SELECT 69 | * 70 | FROM 71 | ` + "`" + testProjectID + "." + testDatasetID + "." + testEventsTable + "`" + ` events, 72 | UNNEST(JSON_EXTRACT_ARRAY(events.payload.workflow_run.pull_requests)) AS pull_request 73 | WHERE 74 | received >= TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -30 DAY)) AS events 75 | USING 76 | (delivery_id)) AS delivery_events 77 | ON 78 | pull_request_events.id = delivery_events.pull_request_id 79 | WHERE 80 | pull_request_events.id NOT IN ( 81 | SELECT 82 | DISTINCT pull_request_id 83 | FROM 84 | ` + "`" + testProjectID + "." + testDatasetID + "." + testInvocationCommentTable + "`" + ` invocation_comment_status) 85 | AND merged_at BETWEEN TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -30 DAY) 86 | AND TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -1 HOUR) 87 | ORDER BY 88 | received, 89 | pull_request_events.id ASC 90 | ` 91 | 92 | config := &BQConfig{ 93 | ProjectID: testProjectID, 94 | DatasetID: testDatasetID, 95 | PullRequestEventsTable: testPullRequestEventsTable, 96 | InvocationCommentStatusTable: testInvocationCommentTable, 97 | EventsTable: testEventsTable, 98 | LeechStatusTable: testLeechTable, 99 | } 100 | q, err := populatePublisherSourceQuery(t.Context(), config) 101 | if err != nil { 102 | t.Errorf("SetUpPublisherSourceQuery returned unexpected error: %v", err) 103 | } 104 | if diff := cmp.Diff(want, q); diff != "" { 105 | t.Errorf("embedded source query mismatch (-want +got):\n%s", diff) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /pkg/teeth/publish_logs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package teeth contains a job that will read leech and pull request event 16 | // records from BigQuery and publish any available log invocations in a PR 17 | // comment. 18 | package teeth 19 | 20 | import ( 21 | "context" 22 | "fmt" 23 | ) 24 | 25 | // BigQueryClient defines the spec for calls to read from and write to 26 | // BigQuery tables. 27 | type BigQueryClient interface { 28 | QueryLatest(context.Context) ([]*PublisherSourceRecord, error) 29 | Insert(context.Context, []*InvocationCommentStatusRecord) error 30 | } 31 | 32 | // GetLatestSourceRecords gets the latest publisher source records. 33 | func GetLatestSourceRecords(ctx context.Context, bqClient BigQueryClient) ([]*PublisherSourceRecord, error) { 34 | res, err := bqClient.QueryLatest(ctx) 35 | if err != nil { 36 | return nil, fmt.Errorf("failed to query with bigquery client: %w", err) 37 | } 38 | return res, nil 39 | } 40 | 41 | // SaveInvocationCommentStatus inserts the statuses into the 42 | // InvocationCommentStatus table. 43 | func SaveInvocationCommentStatus(ctx context.Context, bqClient BigQueryClient, statuses []*InvocationCommentStatusRecord) error { 44 | if err := bqClient.Insert(ctx, statuses); err != nil { 45 | return fmt.Errorf("failed to insert with bigquery client: %w", err) 46 | } 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/teeth/sql/publisher_source.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | SELECT 15 | pull_request_events.delivery_id, 16 | delivery_events.pull_request_id, 17 | html_url AS pull_request_html_url, 18 | delivery_events.received, 19 | logs_uri, 20 | head_sha 21 | FROM 22 | `{{.PullRequestEventsTable}}` AS pull_request_events 23 | JOIN ( 24 | SELECT 25 | delivery_id, 26 | received, 27 | logs_uri, 28 | SAFE.INT64(pull_request.id) AS pull_request_id, 29 | LAX_STRING(pull_request.url) AS pull_request_url, 30 | LAX_STRING(events.payload.workflow_run.head_sha) AS head_sha, 31 | FROM 32 | `{{.LeechStatusTable}}` leech_status 33 | JOIN ( 34 | SELECT 35 | * 36 | FROM 37 | `{{.EventsTable}}` events, 38 | UNNEST(JSON_EXTRACT_ARRAY(events.payload.workflow_run.pull_requests)) AS pull_request 39 | WHERE 40 | received >= TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -30 DAY)) AS events 41 | USING 42 | (delivery_id)) AS delivery_events 43 | ON 44 | pull_request_events.id = delivery_events.pull_request_id 45 | WHERE 46 | pull_request_events.id NOT IN ( 47 | SELECT 48 | DISTINCT pull_request_id 49 | FROM 50 | `{{.InvocationCommentStatusTable}}` invocation_comment_status) 51 | AND merged_at BETWEEN TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -30 DAY) 52 | AND TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -1 HOUR) 53 | ORDER BY 54 | received, 55 | pull_request_events.id ASC 56 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package version 16 | 17 | import ( 18 | "runtime" 19 | "runtime/debug" 20 | ) 21 | 22 | var ( 23 | // Name is the name of the binary. This can be overridden by the build 24 | // process. 25 | Name = "github-metrics-aggregator" 26 | 27 | // Version is the main package version. This can be overridden by the build 28 | // process. 29 | Version = "source" 30 | 31 | // Commit is the git sha. This can be overridden by the build process. 32 | Commit = func() string { 33 | if info, ok := debug.ReadBuildInfo(); ok { 34 | for _, setting := range info.Settings { 35 | if setting.Key == "vcs.revision" { 36 | return setting.Value 37 | } 38 | } 39 | } 40 | return "HEAD" 41 | }() 42 | 43 | // OSArch is the operating system and architecture combination. 44 | OSArch = runtime.GOOS + "/" + runtime.GOARCH 45 | 46 | // HumanVersion is the compiled version. 47 | HumanVersion = Name + " " + Version + " (" + Commit + ", " + OSArch + ")" 48 | ) 49 | -------------------------------------------------------------------------------- /pkg/webhook/bigquery_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package webhook 16 | 17 | import "context" 18 | 19 | type deliveryEventExistsRes struct { 20 | res bool 21 | err error 22 | } 23 | 24 | type failureEventsExceedsRetryLimitRes struct { 25 | res bool 26 | err error 27 | } 28 | 29 | type MockDatastore struct { 30 | deliveryEventExists *deliveryEventExistsRes 31 | failureEventsExceedsRetryLimit *failureEventsExceedsRetryLimitRes 32 | } 33 | 34 | func (m *MockDatastore) DeliveryEventExists(ctx context.Context, eventsTableID, deliveryID string) (bool, error) { 35 | if m.deliveryEventExists != nil { 36 | return m.deliveryEventExists.res, m.deliveryEventExists.err 37 | } 38 | return false, nil 39 | } 40 | 41 | func (m *MockDatastore) FailureEventsExceedsRetryLimit(ctx context.Context, failureEventTableID, deliveryID string, retryLimit int) (bool, error) { 42 | if m.failureEventsExceedsRetryLimit != nil { 43 | return m.failureEventsExceedsRetryLimit.res, m.failureEventsExceedsRetryLimit.err 44 | } 45 | return false, nil 46 | } 47 | 48 | func (m *MockDatastore) WriteFailureEvent(ctx context.Context, failureEventTableID, deliveryID, createdAt string) error { 49 | return nil 50 | } 51 | 52 | func (m *MockDatastore) Close() error { 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/webhook/pubsub.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package webhook 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "cloud.google.com/go/pubsub" 22 | "google.golang.org/api/option" 23 | ) 24 | 25 | // PubSubMessenger implements the Messenger interface for Google Cloud pubsub. 26 | type PubSubMessenger struct { 27 | projectID string 28 | topicID string 29 | 30 | client *pubsub.Client 31 | topic *pubsub.Topic 32 | } 33 | 34 | // NewPubSubMessenger creates a new instance of the PubSubMessenger. 35 | func NewPubSubMessenger(ctx context.Context, projectID, topicID string, opts ...option.ClientOption) (*PubSubMessenger, error) { 36 | // pubsub client forces you to provide a projectID 37 | client, err := pubsub.NewClient(ctx, projectID, opts...) 38 | if err != nil { 39 | return nil, fmt.Errorf("failed to create new pubsub client: %w", err) 40 | } 41 | 42 | topic := client.Topic(topicID) 43 | 44 | return &PubSubMessenger{ 45 | projectID: projectID, 46 | topicID: topicID, 47 | client: client, 48 | topic: topic, 49 | }, nil 50 | } 51 | 52 | // Send sends a message to a Google Cloud pubsub topic. 53 | func (p *PubSubMessenger) Send(ctx context.Context, msg []byte) error { 54 | result := p.topic.Publish(ctx, &pubsub.Message{ 55 | Data: msg, 56 | }) 57 | 58 | if _, err := result.Get(ctx); err != nil { 59 | return fmt.Errorf("pubsub: failed to get result: %w", err) 60 | } 61 | return nil 62 | } 63 | 64 | // Close handles the graceful shutdown of the pubsub client. 65 | func (p *PubSubMessenger) Close() error { 66 | p.topic.Stop() 67 | if err := p.client.Close(); err != nil { 68 | return fmt.Errorf("failed to close pubsub client: %w", err) 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /protos/protos.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:generate protoc --proto_path=. --go_out=. --go_opt=paths=source_relative pubsub_schemas/event.proto 16 | 17 | // Package protos contains versioned protos for the github-metrics-aggregator. 18 | package protos 19 | -------------------------------------------------------------------------------- /protos/pubsub_schemas/event.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The Authors (see AUTHORS file) 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | option go_package = "./protos"; 18 | 19 | message Event { 20 | string delivery_id = 1; 21 | string signature = 2; 22 | string received = 3; 23 | string event = 4; 24 | string payload = 5; 25 | } 26 | -------------------------------------------------------------------------------- /terraform/dashboard.tf: -------------------------------------------------------------------------------- 1 | resource "google_monitoring_dashboard" "default" { 2 | count = var.enable_monitoring_dashboard ? 1 : 0 3 | 4 | project = var.project_id 5 | 6 | dashboard_json = file("${path.module}/dashboards/default.json") 7 | 8 | depends_on = [ 9 | module.webhook_alerts, 10 | module.retry_alerts, 11 | module.leech, 12 | module.commit_review_status, 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /terraform/modules/artifacts/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | resource "google_bigquery_table" "leech_table" { 16 | project = var.project_id 17 | 18 | deletion_protection = false 19 | table_id = var.leech_table_id 20 | dataset_id = var.dataset_id 21 | schema = jsonencode([ 22 | { 23 | "name" : "delivery_id", 24 | "type" : "STRING", 25 | "mode" : "REQUIRED", 26 | "description" : "GUID that represents the event that was ingested." 27 | }, 28 | { 29 | "name" : "processed_at", 30 | "type" : "TIMESTAMP", 31 | "mode" : "REQUIRED", 32 | "description" : "Timestamp of when the event was processed." 33 | }, 34 | { 35 | "name" : "status", 36 | "type" : "STRING", 37 | "mode" : "REQUIRED", 38 | "description" : "The status of the log ingestion." 39 | }, 40 | { 41 | "name" : "workflow_uri", 42 | "type" : "STRING", 43 | "mode" : "REQUIRED", 44 | "description" : "The original workflow uri that trigger the ingestion." 45 | }, 46 | { 47 | "name" : "logs_uri", 48 | "type" : "STRING", 49 | "mode" : "REQUIRED", 50 | "description" : "The GCS uri of the logs." 51 | }, 52 | { 53 | "name" : "github_actor", 54 | "type" : "STRING", 55 | "mode" : "REQUIRED", 56 | "description" : "GitHub user that triggered the workflow event." 57 | }, 58 | { 59 | "name" : "organization_name", 60 | "type" : "STRING", 61 | "mode" : "REQUIRED", 62 | "description" : "GitHub organization name." 63 | }, 64 | { 65 | "name" : "repository_name", 66 | "type" : "STRING", 67 | "mode" : "REQUIRED", 68 | "description" : "GitHub repository name." 69 | }, 70 | { 71 | "name" : "repository_slug", 72 | "type" : "STRING", 73 | "mode" : "REQUIRED", 74 | "description" : "Combined org/repo_name of the repository." 75 | }, 76 | { 77 | "name" : "job_name", 78 | "type" : "STRING", 79 | "mode" : "REQUIRED", 80 | "description" : "Apache Beam job name of the pipeline that processed this event." 81 | }, 82 | ]) 83 | } 84 | 85 | resource "google_bigquery_table_iam_member" "leech_owners" { 86 | for_each = toset(var.leech_table_iam.owners) 87 | 88 | project = var.project_id 89 | 90 | dataset_id = var.dataset_id 91 | table_id = google_bigquery_table.leech_table.id 92 | role = "roles/bigquery.dataOwner" 93 | member = each.value 94 | } 95 | 96 | resource "google_bigquery_table_iam_member" "leech_editors" { 97 | for_each = toset(var.leech_table_iam.editors) 98 | 99 | project = var.project_id 100 | 101 | dataset_id = var.dataset_id 102 | table_id = google_bigquery_table.leech_table.id 103 | role = "roles/bigquery.dataEditor" 104 | member = each.value 105 | } 106 | 107 | resource "google_bigquery_table_iam_member" "leech_viewers" { 108 | for_each = toset(var.leech_table_iam.viewers) 109 | 110 | project = var.project_id 111 | 112 | dataset_id = var.dataset_id 113 | table_id = google_bigquery_table.leech_table.id 114 | role = "roles/bigquery.dataViewer" 115 | member = each.value 116 | } 117 | 118 | resource "google_storage_bucket" "leech_storage_bucket" { 119 | project = var.project_id 120 | 121 | name = var.leech_bucket_name 122 | location = var.leech_bucket_location 123 | 124 | uniform_bucket_level_access = true 125 | public_access_prevention = "enforced" 126 | } 127 | -------------------------------------------------------------------------------- /terraform/modules/artifacts/outputs.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | output "google_service_account" { 16 | value = google_service_account.default 17 | } 18 | 19 | output "job_id" { 20 | value = google_cloud_run_v2_job.default.id 21 | } 22 | 23 | output "job_name" { 24 | value = google_cloud_run_v2_job.default.name 25 | } 26 | -------------------------------------------------------------------------------- /terraform/modules/artifacts/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | required_providers { 19 | google = { 20 | version = "~>5.19" 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_tvf/events/issue_events_by_date.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- TVF queries cannot end in ; as it will be wrapped in a CREATE TABLE FUNCTION 16 | -- and the end of the query is not the end of the compiled SQL statement. 17 | 18 | SELECT 19 | received, 20 | event, 21 | delivery_id, 22 | LAX_STRING(payload.action) action, 23 | ORGANIZATION, 24 | organization_id, 25 | repository_full_name, 26 | repository_id, 27 | repository, 28 | repository_visibility, 29 | sender, 30 | sender_id, 31 | LAX_STRING(payload.issue.active_lock_reason) active_lock_reason, 32 | LAX_STRING(payload.issue.assignee.login) assignee, 33 | LAX_STRING(payload.issue.user.login) author, 34 | SAFE.INT64(payload.issue.user.id) author_id, 35 | LAX_STRING(payload.issue.author_association) author_association, 36 | LAX_STRING(payload.issue.body) body, 37 | TIMESTAMP(LAX_STRING(payload.issue.closed_at)) closed_at, 38 | SAFE.INT64(payload.issue.comments) comments, 39 | TIMESTAMP(LAX_STRING(payload.issue.created_at)) created_at, 40 | SAFE.BOOL(payload.issue.draft) draft, 41 | LAX_STRING(payload.issue.html_url) html_url, 42 | SAFE.INT64(payload.issue.id) id, 43 | SAFE.BOOL(payload.issue.locked) locked, 44 | SAFE.INT64(payload.issue.number) number, 45 | TIMESTAMP_DIFF(TIMESTAMP(LAX_STRING(payload.issue.closed_at)), TIMESTAMP(LAX_STRING(payload.issue.created_at)), SECOND) open_duration_seconds, 46 | LAX_STRING(payload.issue.state) state, 47 | LAX_STRING(payload.issue.state_reason) state_reason, 48 | LAX_STRING(payload.issue.title) title, 49 | TIMESTAMP(LAX_STRING(payload.issue.updated_at)) updated_at, 50 | FROM 51 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, endTimestamp, 'issues') 52 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_tvf/events/pull_request_events_by_date.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- TVF queries cannot end in ; as it will be wrapped in a CREATE TABLE FUNCTION 16 | -- and the end of the query is not the end of the compiled SQL statement. 17 | 18 | SELECT 19 | received, 20 | event, 21 | delivery_id, 22 | JSON_VALUE(payload, "$.action") action, 23 | organization, 24 | organization_id, 25 | repository_full_name, 26 | repository_id, 27 | repository, 28 | repository_visibility, 29 | sender, 30 | sender_id, 31 | LAX_STRING(payload.pull_request.active_lock_reason) active_lock_reason, 32 | SAFE.INT64(payload.pull_request.additions) additions, 33 | LAX_STRING(payload.pull_request.base.ref) base_ref, 34 | SAFE.INT64(payload.pull_request.changed_files) changed_files, 35 | TIMESTAMP(LAX_STRING(payload.pull_request.closed_at)) closed_at, 36 | SAFE.INT64(payload.pull_request.comments) comments, 37 | SAFE.INT64(payload.pull_request.commits) commits, 38 | TIMESTAMP(LAX_STRING(payload.pull_request.created_at)) created_at, 39 | SAFE.INT64(payload.pull_request.deletions) deletions, 40 | SAFE.BOOL(payload.pull_request.draft) draft, 41 | LAX_STRING(payload.pull_request.head.ref) head_ref, 42 | LAX_STRING(payload.pull_request.html_url) html_url, 43 | SAFE.INT64(payload.pull_request.id) id, 44 | SAFE.BOOL(payload.pull_request.locked) locked, 45 | SAFE.BOOL(payload.pull_request.maintainer_can_modify) maintainer_can_modify, 46 | LAX_STRING(payload.pull_request.merge_commit_sha) merge_commit_sha, 47 | LAX_STRING(payload.pull_request.mergeable_state) mergeable_state, 48 | SAFE.BOOL(payload.pull_request.merged) merged, 49 | TIMESTAMP(LAX_STRING(payload.pull_request.merged_at)) merged_at, 50 | LAX_STRING(payload.pull_request.merged_by.login) merged_by, 51 | SAFE.INT64(payload.pull_request.number) number, 52 | SAFE.INT64(payload.pull_request.review_comments) review_comments, 53 | LAX_STRING(payload.pull_request.state) state, 54 | LAX_STRING(payload.pull_request.title) title, 55 | TIMESTAMP(LAX_STRING(payload.pull_request.updated_at)) updated_at, 56 | SAFE.INT64(payload.pull_request.user.id) author_id, 57 | LAX_STRING(payload.pull_request.user.login) author, 58 | TIMESTAMP_DIFF( TIMESTAMP(LAX_STRING(payload.pull_request.closed_at)), TIMESTAMP(LAX_STRING(payload.pull_request.created_at)), SECOND) open_duration_seconds 59 | FROM 60 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, endTimestamp, "pull_request") 61 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_tvf/events/push_events_by_date.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the 'License'); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an 'AS IS' BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- TVF queries cannot end in ; as it will be wrapped in a CREATE TABLE FUNCTION 16 | -- and the end of the query is not the end of the compiled SQL statement. 17 | 18 | SELECT 19 | received, 20 | event, 21 | delivery_id, 22 | organization, 23 | organization_id, 24 | repository_full_name, 25 | repository_id, 26 | repository, 27 | repository_visibility, 28 | sender, 29 | sender_id, 30 | LAX_STRING(payload.after) after_sha, 31 | LAX_STRING(payload.before) before_sha, 32 | LAX_STRING(payload.compare) compare_url, 33 | JSON_QUERY_ARRAY(payload.commits) commits, 34 | ARRAY_LENGTH(JSON_QUERY_ARRAY(payload.commits)) commit_count, 35 | SAFE.BOOL(payload.created) created, 36 | SAFE.BOOL(payload.deleted) deleted, 37 | SAFE.BOOL(payload.forced) forced, 38 | LAX_STRING(payload.pusher.name) pusher, 39 | LAX_STRING(payload.ref) ref, 40 | LAX_STRING(payload.repository.default_branch) repository_default_branch 41 | FROM 42 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, endTimestamp, 'push') 43 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_tvf/resources/issues_by_date.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- TVF queries cannot end in ; as it will be wrapped in a CREATE TABLE FUNCTION 16 | -- and the end of the query is not the end of the compiled SQL statement. 17 | 18 | SELECT 19 | issue_events.active_lock_reason, 20 | issue_events.assignee, 21 | issue_events.author, 22 | issue_events.author_association, 23 | issue_events.body, 24 | issue_events.closed_at, 25 | issue_events.comments, 26 | issue_events.created_at, 27 | issue_events.delivery_id, 28 | issue_events.draft, 29 | issue_events.html_url, 30 | issue_events.id, 31 | issue_events.locked, 32 | issue_events.number, 33 | issue_events.open_duration_seconds, 34 | issue_events.organization, 35 | issue_events.organization_id, 36 | issue_events.repository, 37 | issue_events.repository_id, 38 | issue_events.state, 39 | issue_events.state_reason, 40 | issue_events.title, 41 | issue_events.updated_at, 42 | FROM 43 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, 44 | endTimestamp) issue_events 45 | INNER JOIN ( 46 | SELECT 47 | id, 48 | MAX(received) received 49 | FROM 50 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, 51 | endTimestamp) 52 | GROUP BY 53 | id ) unique_issue_ids 54 | ON 55 | issue_events.id = unique_issue_ids.id 56 | AND issue_events.received = unique_issue_ids.received 57 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_tvf/resources/pull_requests_by_date.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- TVF queries cannot end in ; as it will be wrapped in a CREATE TABLE FUNCTION 16 | -- and the end of the query is not the end of the compiled SQL statement. 17 | 18 | SELECT 19 | pull_request_events.active_lock_reason, 20 | pull_request_events.additions, 21 | pull_request_events.author, 22 | pull_request_events.author_id, 23 | pull_request_events.base_ref, 24 | pull_request_events.changed_files, 25 | pull_request_events.closed_at, 26 | pull_request_events.comments, 27 | pull_request_events.commits, 28 | pull_request_events.created_at, 29 | pull_request_events.deletions, 30 | pull_request_events.delivery_id, 31 | pull_request_events.draft, 32 | pull_request_events.head_ref, 33 | pull_request_events.html_url, 34 | pull_request_events.id, 35 | pull_request_events.locked, 36 | pull_request_events.maintainer_can_modify, 37 | pull_request_events.merge_commit_sha, 38 | pull_request_events.mergeable_state, 39 | pull_request_events.merged, 40 | pull_request_events.merged_at, 41 | pull_request_events.merged_by, 42 | pull_request_events.number, 43 | pull_request_events.open_duration_seconds, 44 | pull_request_events.organization, 45 | pull_request_events.organization_id, 46 | pull_request_events.repository, 47 | pull_request_events.repository_full_name, 48 | pull_request_events.repository_id, 49 | pull_request_events.repository_visibility, 50 | pull_request_events.state, 51 | pull_request_events.title, 52 | pull_request_events.updated_at, 53 | (CASE 54 | WHEN pull_request_events.additions + pull_request_events.deletions <= 9 THEN 'XS' 55 | WHEN pull_request_events.additions + pull_request_events.deletions <= 49 THEN 'S' 56 | WHEN pull_request_events.additions + pull_request_events.deletions <= 249 THEN 'M' 57 | WHEN pull_request_events.additions + pull_request_events.deletions <= 999 THEN 'L' 58 | ELSE 59 | 'XL' 60 | END 61 | ) AS pr_size, 62 | (CASE 63 | WHEN pull_request_events.open_duration_seconds < 600 THEN '< 10m' 64 | WHEN pull_request_events.open_duration_seconds < 1800 THEN '< 30m' 65 | WHEN pull_request_events.open_duration_seconds < 3600 THEN '< 1h' 66 | WHEN pull_request_events.open_duration_seconds < 10800 THEN '< 3h' 67 | WHEN pull_request_events.open_duration_seconds < 21600 THEN '< 6h' 68 | WHEN pull_request_events.open_duration_seconds < 43200 THEN '< 12h' 69 | WHEN pull_request_events.open_duration_seconds < 86400 THEN '< 1d' 70 | WHEN pull_request_events.open_duration_seconds < 172800 THEN '< 2d' 71 | WHEN pull_request_events.open_duration_seconds < 345600 THEN '< 4d' 72 | WHEN pull_request_events.open_duration_seconds < 604800 THEN '< 7d' 73 | WHEN pull_request_events.open_duration_seconds < 1209600 THEN '< 14d' 74 | WHEN pull_request_events.open_duration_seconds < 2592000 THEN '< 30d' 75 | ELSE 76 | '>= 30d' 77 | END 78 | ) AS submission_time, 79 | FROM 80 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, endTimestamp) pull_request_events 81 | INNER JOIN ( 82 | SELECT 83 | id, 84 | MAX(received) received 85 | FROM 86 | `${parent_project_id}.${parent_dataset_id}.${parent_routine_id}`(startTimestamp, endTimestamp) 87 | GROUP BY 88 | id ) unique_pull_request_ids 89 | ON 90 | pull_request_events.id = unique_pull_request_ids.id 91 | AND pull_request_events.received = unique_pull_request_ids.received 92 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/check_run_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to check_runs. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#check_run 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.check_run.app.name") app, 33 | SAFE_CAST(JSON_VALUE(payload, "$.check_run.app.id") AS INT64) app_id, 34 | SAFE_CAST(JSON_VALUE(payload, "$.check_run.check_suite.id") AS INT64) check_suite_id, 35 | TIMESTAMP(JSON_VALUE(payload, "$.check_run.completed_at")) completed_at, 36 | JSON_VALUE(payload, "$.check_run.conclusion") conclusion, 37 | SAFE_CAST(JSON_VALUE(payload, "$.check_run.deployment.id") AS INT64) deployment_id, 38 | JSON_VALUE(payload, "$.check_run.details_url") details_url, 39 | JSON_VALUE(payload, "$.check_run.external_id") external_id, 40 | JSON_VALUE(payload, "$.check_run.head_sha") head_sha, 41 | JSON_VALUE(payload, "$.check_run.html_url") html_url, 42 | SAFE_CAST(JSON_VALUE(payload, "$.check_run.id") AS INT64) id, 43 | JSON_VALUE(payload, "$.check_run.name") name, 44 | JSON_VALUE(payload, "$.check_run.output.summary") output_summary, 45 | JSON_VALUE(payload, "$.check_run.output.text") output_text, 46 | JSON_VALUE(payload, "$.check_run.output.title") output_title, 47 | TIMESTAMP(JSON_VALUE(payload, "$.check_run.started_at")) started_at, 48 | JSON_VALUE(payload, "$.check_run.status") status, 49 | FROM 50 | `${dataset_id}.${table_id}` 51 | WHERE 52 | event = "check_run"; 53 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/deployment_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to deployments. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#deployment 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | TIMESTAMP(JSON_VALUE(payload, "$.deployment.created_at")) created_at, 33 | JSON_QUERY(payload, "$.deployment.creator.login") creator, 34 | SAFE_CAST(JSON_VALUE(payload, "$.deployment.creator.id") AS INT64) creator_id, 35 | JSON_VALUE(payload, "$.deployment.description") description, 36 | JSON_VALUE(payload, "$.deployment.environment") environment, 37 | SAFE_CAST(JSON_VALUE(payload, "$.deployment.id") AS INT64) id, 38 | JSON_VALUE(payload, "$.deployment.original_environment") original_environment, 39 | JSON_VALUE(payload, "$.deployment.payload") deployment_payload, 40 | SAFE_CAST(JSON_VALUE(payload, "$.deployment.production_environment") AS BOOL) production_environment, 41 | JSON_VALUE(payload, "$.deployment.ref") ref, 42 | JSON_VALUE(payload, "$.deployment.sha") sha, 43 | JSON_VALUE(payload, "$.deployment.task") task, 44 | SAFE_CAST(JSON_VALUE(payload, "$.deployment.transient_environment") AS BOOL) transient_environment, 45 | TIMESTAMP(JSON_VALUE(payload, "$.deployment.updated_at")) updated_at, 46 | FROM 47 | `${dataset_id}.${table_id}` 48 | WHERE 49 | event = "deployment"; 50 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/deployment_status_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to deployments. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#deployment_status 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | SAFE_CAST(JSON_VALUE(payload, "$.check_run.id") AS INT64) check_run_id, 33 | TIMESTAMP(JSON_VALUE(payload, "$.deployment_status.created_at")) created_at, 34 | JSON_VALUE(payload, "$.deployment_status.creator.login") creator, 35 | SAFE_CAST(JSON_VALUE(payload, "$.deployment_status.creator.id") AS INT64) creator_id, 36 | SAFE_CAST(JSON_VALUE(payload, "$.deployment.id") AS INT64) deployment_id, 37 | JSON_VALUE(payload, "$.deployment_status.description") description, 38 | JSON_VALUE(payload, "$.deployment_status.environment") environment, 39 | SAFE_CAST(JSON_VALUE(payload, "$.deployment_status.id") AS INT64) id, 40 | JSON_VALUE(payload, "$.deployment_status.state") state, 41 | JSON_VALUE(payload, "$.deployment_status.target_url") target_url, 42 | SAFE_CAST(JSON_VALUE(payload, "$.workflow.id") AS INT64) workflow_id, 43 | SAFE_CAST(JSON_VALUE(payload, "$.workflow_run.id") AS INT64) workflow_run_id, 44 | TIMESTAMP(JSON_VALUE(payload, "$.deployment_status.updated_at")) updated_at, 45 | FROM 46 | `${dataset_id}.${table_id}` 47 | WHERE 48 | event = "deployment_status"; 49 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/issue_comment_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Filters events to those just pertaining to issue comments. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#issue_comment 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.comment.body") body, 33 | SAFE_CAST(JSON_VALUE(payload, "$.comment.user.id") AS INT64) commenter_id, 34 | JSON_VALUE(payload, "$.comment.user.login") commenter, 35 | TIMESTAMP(JSON_VALUE(payload, "$.comment.created_at")) created_at, 36 | JSON_VALUE(payload, "$.comment.html_url") html_url, 37 | SAFE_CAST(JSON_VALUE(payload, "$.comment.id") AS INT64) id, 38 | SAFE_CAST(JSON_VALUE(payload, "$.issue.id") AS INT64) issue_id, 39 | SAFE_CAST(JSON_VALUE(payload, "$.comment.line") AS INT64) line, 40 | JSON_VALUE(payload, "$.comment.path") path, 41 | TIMESTAMP(JSON_VALUE(payload, "$.comment.updated_at")) updated_at, 42 | FROM 43 | `${dataset_id}.${table_id}` 44 | WHERE 45 | event = "issue_comment"; 46 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/issue_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Filters events to those just pertaining to issues. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#issues 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.issue.active_lock_reason") active_lock_reason, 33 | JSON_VALUE(payload, "$.issue.assignee.login") assignee, 34 | JSON_VALUE(payload, "$.issue.user.login") author, 35 | SAFE_CAST(JSON_VALUE(payload, "$.issue.user.id") AS INT64) author_id, 36 | JSON_VALUE(payload, "$.issue.author_association") author_association, 37 | JSON_VALUE(payload, "$.issue.body") body, 38 | TIMESTAMP(JSON_VALUE(payload, "$.issue.closed_at")) closed_at, 39 | SAFE_CAST(JSON_VALUE(payload, "$.issue.comments") AS INT64) comments, 40 | TIMESTAMP(JSON_VALUE(payload, "$.issue.created_at")) created_at, 41 | SAFE_CAST(JSON_VALUE(payload, "$.issue.draft") AS BOOL) draft, 42 | JSON_VALUE(payload, "$.issue.html_url") html_url, 43 | SAFE_CAST(JSON_VALUE(payload, "$.issue.id") AS INT64) id, 44 | SAFE_CAST(JSON_VALUE(payload, "$.issue.locked") AS BOOL) locked, 45 | SAFE_CAST(JSON_VALUE(payload, "$.issue.number") AS INT64) number, 46 | TIMESTAMP_DIFF(TIMESTAMP(JSON_VALUE(payload, "$.issue.closed_at")), TIMESTAMP(JSON_VALUE(payload, "$.issue.created_at")), SECOND) open_duration_seconds, 47 | JSON_VALUE(payload, "$.issue.state") state, 48 | JSON_VALUE(payload, "$.issue.state_reason") state_reason, 49 | JSON_VALUE(payload, "$.issue.title") title, 50 | TIMESTAMP(JSON_VALUE(payload, "$.issue.updated_at")) updated_at, 51 | FROM 52 | `${dataset_id}.${table_id}` 53 | WHERE 54 | event = "issues"; 55 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/pull_request_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | SELECT 16 | received, 17 | event, 18 | delivery_id, 19 | JSON_VALUE(payload, "$.action") action, 20 | organization, 21 | organization_id, 22 | repository_full_name, 23 | repository_id, 24 | repository, 25 | repository_visibility, 26 | sender, 27 | sender_id, 28 | JSON_VALUE(payload, "$.pull_request.active_lock_reason") active_lock_reason, 29 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.additions") AS INT64) additions, 30 | JSON_VALUE(payload, "$.pull_request.base.ref") base_ref, 31 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.changed_files") AS INT64) changed_files, 32 | TIMESTAMP(JSON_VALUE(payload, "$.pull_request.closed_at")) closed_at, 33 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.comments") AS INT64) comments, 34 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.commits") AS INT64) commits, 35 | TIMESTAMP(JSON_VALUE(payload, "$.pull_request.created_at")) created_at, 36 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.deletions") AS INT64) deletions, 37 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.draft") AS BOOL) draft, 38 | JSON_VALUE(payload, "$.pull_request.head.ref") head_ref, 39 | JSON_VALUE(payload, "$.pull_request.html_url") html_url, 40 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.id") AS INT64) id, 41 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.locked") AS BOOL) locked, 42 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.maintainer_can_modify") AS BOOL) maintainer_can_modify, 43 | JSON_VALUE(payload, "$.pull_request.merge_commit_sha") merge_commit_sha, 44 | JSON_VALUE(payload, "$.pull_request.mergeable_state") mergeable_state, 45 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.merged") AS BOOL) merged, 46 | TIMESTAMP(JSON_VALUE(payload, "$.pull_request.merged_at")) merged_at, 47 | JSON_VALUE(payload, "$.pull_request.merged_by.login") merged_by, 48 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.number") AS INT64) number, 49 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.review_comments") AS INT64) review_comments, 50 | JSON_VALUE(payload, "$.pull_request.state") state, 51 | JSON_VALUE(payload, "$.pull_request.title") title, 52 | TIMESTAMP(JSON_VALUE(payload, "$.pull_request.updated_at")) updated_at, 53 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.user.id") AS INT64) author_id, 54 | JSON_VALUE(payload, "$.pull_request.user.login") author, 55 | TIMESTAMP_DIFF( TIMESTAMP(JSON_VALUE(payload, "$.pull_request.closed_at")), TIMESTAMP(JSON_VALUE(payload, "$.pull_request.created_at")), SECOND) open_duration_seconds 56 | FROM 57 | `${dataset_id}.${table_id}` 58 | WHERE 59 | event = "pull_request"; 60 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/pull_request_review_comment_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to pull request review comments. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request_review_comment 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.comment.body") body, 33 | SAFE_CAST(JSON_VALUE(payload, "$.comment.user.id") AS INT64) commenter_id, 34 | JSON_VALUE(payload, "$.comment.user.login") commenter, 35 | JSON_VALUE(payload, "$.comment.commit_id") commit_sha, 36 | TIMESTAMP(JSON_VALUE(payload, "$.comment.created_at")) created_at, 37 | JSON_VALUE(payload, "$.comment.diff_hunk") diff_hunk, 38 | JSON_VALUE(payload, "$.comment.html_url") html_url, 39 | SAFE_CAST(JSON_VALUE(payload, "$.comment.id") AS INT64) id, 40 | SAFE_CAST(JSON_VALUE(payload, "$.comment.in_reply_to_id") AS INT64) in_reply_to_id, 41 | SAFE_CAST(JSON_VALUE(payload, "$.comment.line") AS INT64) line, 42 | JSON_VALUE(payload, "$.comment.path") path, 43 | SAFE_CAST(JSON_VALUE(payload, "$.pull_request.id") AS INT64) pull_request_id, 44 | SAFE_CAST(JSON_VALUE(payload, "$.comment.pull_request_review_id") AS INT64) pull_request_review_id, 45 | TIMESTAMP(JSON_VALUE(payload, "$.comment.updated_at")) updated_at, 46 | FROM 47 | `${dataset_id}.${table_id}` 48 | WHERE 49 | event = "pull_request_review_comment"; 50 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/pull_request_review_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Filters events to those just pertaining to pull request reviews. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request_review 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.id") AS INT64) pull_request_id, 33 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.number") AS INT64) pull_request_number, 34 | JSON_VALUE(payload, "$.pull_request.state") pull_request_state, 35 | JSON_VALUE(payload, "$.pull_request.url") pull_request_url, 36 | SAFE_CAST(JSON_QUERY(payload, "$.pull_request.user.id") AS INT64) pull_request_author_id, 37 | JSON_VALUE(payload, "$.pull_request.user.login") pull_request_author, 38 | SAFE_CAST(JSON_QUERY(payload, "$.review.id") AS INT64) id, 39 | JSON_VALUE(payload, "$.review.body") body, 40 | JSON_VALUE(payload, "$.review.commit_id") commit_id, 41 | JSON_VALUE(payload, "$.review.html_url") html_url, 42 | JSON_VALUE(payload, "$.review.state") state, 43 | TIMESTAMP(JSON_VALUE(payload, "$.review.submitted_at")) submitted_at, 44 | JSON_VALUE(payload, "$.review.user.login") reviewer, 45 | SAFE_CAST(JSON_QUERY(payload, "$.review.user.id") AS INT64) reviewer_id, 46 | FROM 47 | `${dataset_id}.${table_id}` 48 | WHERE 49 | event = "pull_request_review"; 50 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/push_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to pushes. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | organization, 24 | organization_id, 25 | repository_full_name, 26 | repository_id, 27 | repository, 28 | repository_visibility, 29 | sender, 30 | sender_id, 31 | JSON_VALUE(payload, "$.after") after_sha, 32 | JSON_VALUE(payload, "$.before") before_sha, 33 | JSON_VALUE(payload, "$.compare") compare_url, 34 | JSON_QUERY_ARRAY(payload, "$.commits") commits, 35 | ARRAY_LENGTH(JSON_QUERY_ARRAY(payload, "$.commits")) commit_count, 36 | SAFE_CAST(JSON_VALUE(payload, "$.created") AS BOOL) created, 37 | SAFE_CAST(JSON_VALUE(payload, "$.deleted") AS BOOL) deleted, 38 | SAFE_CAST(JSON_VALUE(payload, "$.forced") AS BOOL) forced, 39 | JSON_VALUE(payload, "$.pusher.name") pusher, 40 | JSON_VALUE(payload, "$.ref") ref, 41 | JSON_VALUE(payload, '$.repository.default_branch') repository_default_branch 42 | FROM 43 | `${dataset_id}.${table_id}` 44 | WHERE 45 | event = 'push' 46 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/release_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Filters events to those just pertaining to releases. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#release 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.release.author.login") author, 33 | SAFE_CAST(JSON_VALUE(payload, "$.release.author.id") AS INT64) author_id, 34 | JSON_QUERY(payload, "$.release.body") body, 35 | TIMESTAMP(JSON_VALUE(payload, "$.release.created_at")) created_at, 36 | SAFE_CAST(JSON_VALUE(payload, "$.release.draft") AS BOOL) draft, 37 | JSON_VALUE(payload, "$.release.html_url") html_url, 38 | SAFE_CAST(JSON_VALUE(payload, "$.release.id") AS INT64) id, 39 | JSON_VALUE(payload, "$.release.name") name, 40 | SAFE_CAST(JSON_VALUE(payload, "$.release.prerelease") AS BOOL) prerelease, 41 | TIMESTAMP(JSON_VALUE(payload, "$.release.published_at")) published_at, 42 | JSON_VALUE(payload, "$.release.tag_name") tag_name, 43 | JSON_VALUE(payload, "$.release.tarball_url") tarball_url, 44 | JSON_VALUE(payload, "$.release.target_commitish") target_commitish, 45 | JSON_VALUE(payload, "$.release.upload_url") upload_url, 46 | JSON_VALUE(payload, "$.release.zipball_url") zipball_url, 47 | FROM 48 | `${dataset_id}.${table_id}` 49 | WHERE 50 | event = "release"; 51 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/team_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Filters events to those just pertaining to teams. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#team 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | SAFE_CAST(JSON_VALUE(payload, "$.team.deleted") AS BOOL) deleted, 33 | JSON_VALUE(payload, "$.team.description") description, 34 | JSON_VALUE(payload, "$.team.html_url") html_url, 35 | SAFE_CAST(JSON_VALUE(payload, "$.team.id") AS INT64) id, 36 | JSON_VALUE(payload, "$.team.members_url") members_url, 37 | JSON_VALUE(payload, "$.team.name") name, 38 | SAFE_CAST(JSON_VALUE(payload, "$.team.parent.id") AS INT64) parent_id, 39 | JSON_VALUE(payload, "$.team.parent.name") parent_name, 40 | JSON_VALUE(payload, "$.team.permission") permission, 41 | JSON_VALUE(payload, "$.team.privacy") privacy, 42 | JSON_VALUE(payload, "$.team.notification_setting") notification_setting, 43 | JSON_VALUE(payload, "$.team.slug") slug, 44 | FROM 45 | `${dataset_id}.${table_id}` 46 | WHERE 47 | event = "team"; 48 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/events/workflow_run_events.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Filters events to those just pertaining to workflow runs. 16 | -- Relevant GitHub Docs: 17 | -- https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_run 18 | 19 | SELECT 20 | received, 21 | event, 22 | delivery_id, 23 | JSON_VALUE(payload, "$.action") action, 24 | organization, 25 | organization_id, 26 | repository_full_name, 27 | repository_id, 28 | repository, 29 | repository_visibility, 30 | sender, 31 | sender_id, 32 | JSON_VALUE(payload, "$.workflow_run.actor.login") actor, 33 | JSON_VALUE(payload, "$.workflow_run.conclusion") conclusion, 34 | TIMESTAMP(JSON_VALUE(payload, "$.workflow_run.created_at")) created_at, 35 | JSON_VALUE(payload, "$.workflow_run.display_title") display_title, 36 | TIMESTAMP_DIFF(TIMESTAMP(JSON_VALUE(payload, "$.workflow_run.updated_at")), TIMESTAMP(JSON_VALUE(payload, "$.workflow_run.run_started_at")), SECOND) duration_seconds, 37 | JSON_VALUE(payload, "$.workflow_run.event") workflow_event, 38 | JSON_VALUE(payload, "$.workflow_run.head_branch") head_branch, 39 | JSON_VALUE(payload, "$.workflow_run.head_sha") head_sha, 40 | JSON_VALUE(payload, "$.workflow_run.html_url") html_url, 41 | SAFE_CAST(JSON_VALUE(payload, "$.workflow_run.id") AS INT64) id, 42 | JSON_VALUE(payload, "$.workflow_run.path") path, 43 | SAFE_CAST(JSON_VALUE(payload, "$.workflow_run.run_attempt") AS INT64) run_attempt, 44 | SAFE_CAST(JSON_VALUE(payload, "$.workflow_run.run_number") AS INT64) run_number, 45 | TIMESTAMP(JSON_VALUE(payload, "$.workflow_run.run_started_at")) run_started_at, 46 | JSON_VALUE(payload, "$.workflow_run.status") status, 47 | TIMESTAMP(JSON_VALUE(payload, "$.workflow_run.updated_at")) updated_at, 48 | JSON_VALUE(payload, "$.workflow_run.name") workflow_name, 49 | SAFE_CAST(JSON_VALUE(payload, "$.workflow_run.workflow_id") AS INT64) workflow_id, 50 | JSON_VALUE(payload, "$.workflow.html_url") workflow_html_url, 51 | FROM 52 | `${dataset_id}.${table_id}` 53 | WHERE 54 | event = "workflow_run"; 55 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/check_runs.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Extracts all distinct check_runs from the check_run_events view. 16 | -- The most recent event for each distinct check_run id is used to extract 17 | -- the check_run data. This ensures that the check_run data is up to date. 18 | 19 | SELECT 20 | check_run_events.organization, 21 | check_run_events.organization_id, 22 | check_run_events.repository_id, 23 | check_run_events.repository, 24 | check_run_events.app, 25 | check_run_events.app_id, 26 | check_run_events.check_suite_id, 27 | check_run_events.completed_at, 28 | check_run_events.conclusion, 29 | check_run_events.delivery_id, 30 | check_run_events.deployment_id, 31 | check_run_events.details_url, 32 | check_run_events.external_id, 33 | check_run_events.head_sha, 34 | check_run_events.html_url, 35 | check_run_events.id, 36 | check_run_events.name, 37 | check_run_events.output_summary, 38 | check_run_events.output_text, 39 | check_run_events.output_title, 40 | check_run_events.started_at, 41 | check_run_events.status, 42 | FROM 43 | `${dataset_id}.check_run_events` check_run_events 44 | JOIN ( 45 | SELECT 46 | id, 47 | MAX(received) received 48 | FROM 49 | `${dataset_id}.check_run_events` 50 | GROUP BY 51 | id ) unique_check_run_ids 52 | ON 53 | check_run_events.id = unique_check_run_ids.id 54 | AND check_run_events.received = unique_check_run_ids.received; 55 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/deployment_statuses.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Extracts all distinct deployment_statuses from the deployment_status_events view. 16 | -- The most recent event for each distinct deployment_status id is used to extract 17 | -- the deployment_status data. This ensures that the deployment_status data is up to date. 18 | 19 | SELECT 20 | deployment_status_events.organization, 21 | deployment_status_events.organization_id, 22 | deployment_status_events.repository_id, 23 | deployment_status_events.repository, 24 | deployment_status_events.check_run_id, 25 | deployment_status_events.created_at, 26 | deployment_status_events.creator, 27 | deployment_status_events.creator_id, 28 | deployment_status_events.delivery_id, 29 | deployment_status_events.deployment_id, 30 | deployment_status_events.description, 31 | deployment_status_events.environment, 32 | deployment_status_events.id, 33 | deployment_status_events.state, 34 | deployment_status_events.target_url, 35 | deployment_status_events.workflow_id, 36 | deployment_status_events.workflow_run_id, 37 | deployment_status_events.updated_at, 38 | FROM 39 | `${dataset_id}.deployment_status_events` deployment_status_events 40 | JOIN ( 41 | SELECT 42 | id, 43 | MAX(received) received 44 | FROM 45 | `${dataset_id}.deployment_status_events` 46 | GROUP BY 47 | id ) unique_deployment_status_ids 48 | ON 49 | deployment_status_events.id = unique_deployment_status_ids.id 50 | AND deployment_status_events.received = unique_deployment_status_ids.received 51 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/deployments.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct deployments from the deployment_events view. 16 | -- The most recent event for each distinct deployment id is used to extract 17 | -- the deployment data. This ensures that the deployment data is up to date. 18 | 19 | SELECT 20 | deployment_events.organization, 21 | deployment_events.organization_id, 22 | deployment_events.repository_id, 23 | deployment_events.repository, 24 | deployment_events.created_at, 25 | deployment_events.creator, 26 | deployment_events.creator_id, 27 | deployment_events.delivery_id, 28 | deployment_events.description, 29 | deployment_events.environment, 30 | deployment_events.id, 31 | deployment_events.original_environment, 32 | deployment_events.deployment_payload, 33 | deployment_events.production_environment, 34 | deployment_events.ref, 35 | deployment_events.sha, 36 | deployment_events.task, 37 | deployment_events.transient_environment, 38 | deployment_events.updated_at, 39 | FROM `${dataset_id}.deployment_events` deployment_events 40 | JOIN ( 41 | SELECT id, MAX(received) received 42 | FROM `${dataset_id}.deployment_events` 43 | GROUP BY id 44 | ) unique_deployment_ids 45 | ON deployment_events.id = unique_deployment_ids.id 46 | AND deployment_events.received = unique_deployment_ids.received; 47 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/issue_comment.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct pull_requests from the issue_comments. 16 | -- The most recent event for each distinct comment id is used to extract 17 | -- the comment data. This ensures that the comment data is up to date. 18 | 19 | SELECT 20 | issue_comment_events.organization, 21 | issue_comment_events.organization_id, 22 | issue_comment_events.repository, 23 | issue_comment_events.repository_id, 24 | issue_comment_events.body, 25 | issue_comment_events.commenter_id, 26 | issue_comment_events.commenter, 27 | issue_comment_events.created_at, 28 | issue_comment_events.delivery_id, 29 | issue_comment_events.html_url, 30 | issue_comment_events.id, 31 | issue_comment_events.issue_id, 32 | issue_comment_events.line, 33 | issue_comment_events.path, 34 | issue_comment_events.updated_at, 35 | FROM 36 | `${dataset_id}.issue_comment_events` issue_comment_events 37 | JOIN ( 38 | SELECT id, max(received) received 39 | FROM `${dataset_id}.issue_comment_events` 40 | GROUP BY id 41 | ) unique_issue_comment_ids 42 | ON issue_comment_events.id = unique_issue_comment_ids.id 43 | AND issue_comment_events.received = unique_issue_comment_ids.received; 44 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/issues.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct issues from the issue_events view. 16 | -- The most recent event for each distinct issue id is used to extract 17 | -- the issue data. This ensures that the issue data is up to date. 18 | 19 | SELECT 20 | issue_events.active_lock_reason, 21 | issue_events.assignee, 22 | issue_events.author, 23 | issue_events.author_association, 24 | issue_events.body, 25 | issue_events.closed_at, 26 | issue_events.comments, 27 | issue_events.created_at, 28 | issue_events.delivery_id, 29 | issue_events.draft, 30 | issue_events.html_url, 31 | issue_events.id, 32 | issue_events.locked, 33 | issue_events.number, 34 | issue_events.open_duration_seconds, 35 | issue_events.organization, 36 | issue_events.organization_id, 37 | issue_events.repository, 38 | issue_events.repository_id, 39 | issue_events.state, 40 | issue_events.state_reason, 41 | issue_events.title, 42 | issue_events.updated_at, 43 | FROM 44 | `${dataset_id}.issue_events` issue_events 45 | INNER JOIN ( 46 | SELECT 47 | id, 48 | MAX(received) received 49 | FROM 50 | `${dataset_id}.issue_events` 51 | GROUP BY 52 | id ) unique_issue_ids 53 | ON 54 | issue_events.id = unique_issue_ids.id 55 | AND issue_events.received = unique_issue_ids.received; 56 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/pull_request_review_comments.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- Extracts all distinct pull_request_reviews from the pull_request_review_comment_events view. 16 | -- The most recent event for each distinct comment id is used to extract 17 | -- the comment data. This ensures that the comment data is up to date. 18 | 19 | SELECT 20 | pull_request_review_comment_events.body, 21 | pull_request_review_comment_events.commenter, 22 | pull_request_review_comment_events.commenter_id, 23 | pull_request_review_comment_events.commit_sha, 24 | pull_request_review_comment_events.created_at, 25 | pull_request_review_comment_events.delivery_id, 26 | pull_request_review_comment_events.diff_hunk, 27 | pull_request_review_comment_events.html_url, 28 | pull_request_review_comment_events.id, 29 | pull_request_review_comment_events.in_reply_to_id, 30 | pull_request_review_comment_events.line, 31 | pull_request_review_comment_events.organization, 32 | pull_request_review_comment_events.organization_id, 33 | pull_request_review_comment_events.path, 34 | pull_request_review_comment_events.pull_request_id, 35 | pull_request_review_comment_events.pull_request_review_id, 36 | pull_request_review_comment_events.repository, 37 | pull_request_review_comment_events.repository_id, 38 | pull_request_review_comment_events.updated_at, 39 | FROM 40 | `${dataset_id}.pull_request_review_comment_events` pull_request_review_comment_events 41 | INNER JOIN ( 42 | SELECT 43 | id, 44 | MAX(received) received 45 | FROM 46 | `${dataset_id}.pull_request_review_comment_events` 47 | GROUP BY 48 | id ) unique_pull_request_review_comment_ids 49 | ON 50 | pull_request_review_comment_events.id = unique_pull_request_review_comment_ids.id 51 | AND pull_request_review_comment_events.received = unique_pull_request_review_comment_ids.received; 52 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/pull_request_reviews.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct pull_request_reviews from the pull_request_review_events view. 16 | -- The most recent event for each distinct review id is used to extract 17 | -- the review data. This ensures that the review data is up to date. 18 | 19 | SELECT 20 | pull_request_review_events.body, 21 | pull_request_review_events.commit_id, 22 | pull_request_review_events.delivery_id, 23 | pull_request_review_events.html_url, 24 | pull_request_review_events.id, 25 | pull_request_review_events.organization, 26 | pull_request_review_events.organization_id, 27 | pull_request_review_events.pull_request_author, 28 | pull_request_review_events.pull_request_author_id, 29 | pull_request_review_events.pull_request_id, 30 | pull_request_review_events.pull_request_number, 31 | pull_request_review_events.pull_request_url, 32 | pull_request_review_events.repository, 33 | pull_request_review_events.repository_full_name, 34 | pull_request_review_events.repository_id, 35 | pull_request_review_events.repository_visibility, 36 | pull_request_review_events.reviewer, 37 | pull_request_review_events.reviewer_id, 38 | (pull_request_review_events.reviewer = pull_request_review_events.pull_request_author) AS reviewer_is_author, 39 | pull_request_review_events.state, 40 | pull_request_review_events.submitted_at, 41 | FROM 42 | `${dataset_id}.pull_request_review_events` pull_request_review_events 43 | INNER JOIN ( 44 | SELECT 45 | id, 46 | MAX(received) received 47 | FROM 48 | `${dataset_id}.pull_request_review_events` 49 | GROUP BY 50 | id ) unique_pull_request_review_ids 51 | ON 52 | pull_request_review_events.id = unique_pull_request_review_ids.id 53 | AND pull_request_review_events.received = unique_pull_request_review_ids.received; 54 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/pull_requests.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct pull_requests from the pull_request_events_view. 16 | -- The most recent event for each distinct pull request id is used to extract 17 | -- the pull request data. This ensures that the pull request data is up to date. 18 | 19 | SELECT 20 | pull_request_events.active_lock_reason, 21 | pull_request_events.additions, 22 | pull_request_events.author, 23 | pull_request_events.author_id, 24 | pull_request_events.base_ref, 25 | pull_request_events.changed_files, 26 | pull_request_events.closed_at, 27 | pull_request_events.comments, 28 | pull_request_events.commits, 29 | pull_request_events.created_at, 30 | pull_request_events.deletions, 31 | pull_request_events.delivery_id, 32 | pull_request_events.draft, 33 | pull_request_events.head_ref, 34 | pull_request_events.html_url, 35 | pull_request_events.id, 36 | pull_request_events.locked, 37 | pull_request_events.maintainer_can_modify, 38 | pull_request_events.merge_commit_sha, 39 | pull_request_events.mergeable_state, 40 | pull_request_events.merged, 41 | pull_request_events.merged_at, 42 | pull_request_events.merged_by, 43 | pull_request_events.number, 44 | pull_request_events.open_duration_seconds, 45 | pull_request_events.organization, 46 | pull_request_events.organization_id, 47 | pull_request_events.repository, 48 | pull_request_events.repository_full_name, 49 | pull_request_events.repository_id, 50 | pull_request_events.repository_visibility, 51 | pull_request_events.state, 52 | pull_request_events.title, 53 | pull_request_events.updated_at, 54 | (CASE 55 | WHEN pull_request_events.additions + pull_request_events.deletions <= 9 THEN 'XS' 56 | WHEN pull_request_events.additions + pull_request_events.deletions <= 49 THEN 'S' 57 | WHEN pull_request_events.additions + pull_request_events.deletions <= 249 THEN 'M' 58 | WHEN pull_request_events.additions + pull_request_events.deletions <= 999 THEN 'L' 59 | ELSE 60 | 'XL' 61 | END 62 | ) AS pr_size, 63 | (CASE 64 | WHEN pull_request_events.open_duration_seconds < 600 THEN '< 10m' 65 | WHEN pull_request_events.open_duration_seconds < 1800 THEN '< 30m' 66 | WHEN pull_request_events.open_duration_seconds < 3600 THEN '< 1h' 67 | WHEN pull_request_events.open_duration_seconds < 10800 THEN '< 3h' 68 | WHEN pull_request_events.open_duration_seconds < 21600 THEN '< 6h' 69 | WHEN pull_request_events.open_duration_seconds < 43200 THEN '< 12h' 70 | WHEN pull_request_events.open_duration_seconds < 86400 THEN '< 1d' 71 | WHEN pull_request_events.open_duration_seconds < 172800 THEN '< 2d' 72 | WHEN pull_request_events.open_duration_seconds < 345600 THEN '< 4d' 73 | WHEN pull_request_events.open_duration_seconds < 604800 THEN '< 7d' 74 | WHEN pull_request_events.open_duration_seconds < 1209600 THEN '< 14d' 75 | WHEN pull_request_events.open_duration_seconds < 2592000 THEN '< 30d' 76 | ELSE 77 | '>= 30d' 78 | END 79 | ) AS submission_time, 80 | FROM 81 | `${dataset_id}.pull_request_events` pull_request_events 82 | INNER JOIN ( 83 | SELECT 84 | id, 85 | MAX(received) received 86 | FROM 87 | `${dataset_id}.pull_request_events` 88 | GROUP BY 89 | id ) unique_pull_request_ids 90 | ON 91 | pull_request_events.id = unique_pull_request_ids.id 92 | AND pull_request_events.received = unique_pull_request_ids.received; 93 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/releases.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct releases from the release_events view. 16 | -- The most recent event for each distinct release id is used to extract 17 | -- the release data. This ensures that the release data is up to date. 18 | 19 | SELECT 20 | releases_events.author, 21 | releases_events.author_id, 22 | releases_events.body, 23 | releases_events.created_at, 24 | releases_events.delivery_id, 25 | releases_events.draft, 26 | releases_events.html_url, 27 | releases_events.id, 28 | releases_events.name, 29 | releases_events.organization, 30 | releases_events.organization_id, 31 | releases_events.prerelease, 32 | releases_events.published_at, 33 | releases_events.repository, 34 | releases_events.repository_id, 35 | releases_events.tag_name, 36 | releases_events.tarball_url, 37 | releases_events.target_commitish, 38 | releases_events.upload_url, 39 | releases_events.zipball_url, 40 | FROM 41 | `${dataset_id}.release_events` releases_events 42 | INNER JOIN ( 43 | SELECT 44 | id, 45 | MAX(received) received 46 | FROM 47 | `${dataset_id}.release_events` 48 | GROUP BY 49 | id ) unique_release_ids 50 | ON 51 | releases_events.id = unique_release_ids.id 52 | AND releases_events.received = unique_release_ids.received; 53 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/teams.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct teams from the team_events view. 16 | -- The most recent event for each distinct team id is used to extract 17 | -- the team data. This ensures that the team data is up to date. 18 | 19 | SELECT 20 | team_events.deleted, 21 | team_events.delivery_id, 22 | team_events.description, 23 | team_events.html_url, 24 | team_events.id, 25 | team_events.members_url, 26 | team_events.name, 27 | team_events.organization, 28 | team_events.organization_id, 29 | team_events.parent_id, 30 | team_events.parent_name, 31 | team_events.permission, 32 | team_events.privacy, 33 | team_events.repository, 34 | team_events.repository_id, 35 | team_events.notification_setting, 36 | team_events.slug, 37 | FROM 38 | `${dataset_id}.team_events` team_events 39 | JOIN( 40 | SELECT id, MAX(received) received 41 | FROM `${dataset_id}.team_events` 42 | GROUP BY id 43 | ) unique_team_ids 44 | ON team_events.id = unique_team_ids.id 45 | AND team_events.received = unique_team_ids.received; 46 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/data/bq_views/resources/workflow_runs.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 The Authors (see AUTHORS file) 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | -- 15 | -- Extracts all distinct workflow_runs from the workflow_run_events view. 16 | -- The most recent event for each distinct workflow_run id is used to extract 17 | -- the workflow_run data. This ensures that the workflow_run data is up to date. 18 | 19 | SELECT 20 | workflow_run_events.actor, 21 | workflow_run_events.conclusion, 22 | workflow_run_events.created_at, 23 | workflow_run_events.delivery_id, 24 | workflow_run_events.display_title, 25 | workflow_run_events.duration_seconds, 26 | workflow_run_events.workflow_event, 27 | workflow_run_events.head_branch, 28 | workflow_run_events.head_sha, 29 | workflow_run_events.html_url, 30 | workflow_run_events.id, 31 | workflow_run_events.organization, 32 | workflow_run_events.organization_id, 33 | workflow_run_events.path, 34 | workflow_run_events.repository, 35 | workflow_run_events.repository_id, 36 | workflow_run_events.run_attempt, 37 | workflow_run_events.run_number, 38 | workflow_run_events.run_started_at, 39 | workflow_run_events.status, 40 | workflow_run_events.updated_at, 41 | workflow_run_events.workflow_name, 42 | workflow_run_events.workflow_id, 43 | workflow_run_events.workflow_html_url, 44 | FROM 45 | `${dataset_id}.workflow_run_events` workflow_run_events 46 | INNER JOIN ( 47 | SELECT 48 | id, 49 | MAX(received) received 50 | FROM 51 | `${dataset_id}.workflow_run_events` 52 | GROUP BY 53 | id ) unique_workflow_run_ids 54 | ON 55 | workflow_run_events.id = unique_workflow_run_ids.id 56 | AND workflow_run_events.received = unique_workflow_run_ids.received; 57 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | resource "google_bigquery_table" "event_views" { 16 | for_each = fileset("${path.module}/data/bq_views/events", "*.sql") 17 | 18 | project = var.project_id 19 | 20 | deletion_protection = false 21 | dataset_id = var.dataset_id 22 | friendly_name = replace(each.value, ".sql", "") 23 | table_id = replace(each.value, ".sql", "") 24 | view { 25 | query = templatefile("${path.module}/data/bq_views/events/${each.value}", { 26 | dataset_id = var.dataset_id 27 | table_id = var.base_table_id 28 | }) 29 | use_legacy_sql = false 30 | } 31 | } 32 | 33 | resource "google_bigquery_table" "resource_views" { 34 | for_each = fileset("${path.module}/data/bq_views/resources", "*.sql") 35 | 36 | project = var.project_id 37 | 38 | deletion_protection = false 39 | dataset_id = var.dataset_id 40 | friendly_name = replace(each.value, ".sql", "") 41 | table_id = replace(each.value, ".sql", "") 42 | view { 43 | query = templatefile("${path.module}/data/bq_views/resources/${each.value}", { 44 | dataset_id = var.dataset_id, 45 | }) 46 | use_legacy_sql = false 47 | } 48 | 49 | # Must wait for all events tables before creating views on them 50 | depends_on = [google_bigquery_table.event_views] 51 | } 52 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/outputs.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | output "bigquery_event_views" { 16 | description = "BigQuery event view resources." 17 | value = { for k, v in google_bigquery_table.event_views : k => v.table_id } 18 | } 19 | 20 | output "bigquery_resource_views" { 21 | description = "BigQuery resource view resources." 22 | value = { for k, v in google_bigquery_table.resource_views : k => v.table_id } 23 | } 24 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | required_providers { 19 | google = { 20 | version = "~>5.19" 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /terraform/modules/bigquery_metrics_views/variables.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | variable "project_id" { 16 | description = "The GCP project ID." 17 | type = string 18 | } 19 | 20 | variable "dataset_id" { 21 | type = string 22 | description = "The BigQuery dataset id to create views in." 23 | } 24 | 25 | variable "base_table_id" { 26 | type = string 27 | description = "The BigQuery base table name for creating metrics views from." 28 | } 29 | 30 | variable "base_tvf_id" { 31 | type = string 32 | description = "The BigQuery base routine for creating TVF metrics from." 33 | } 34 | -------------------------------------------------------------------------------- /terraform/modules/commit_review_status/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | resource "google_bigquery_table" "commit_review_status_table" { 16 | project = var.project_id 17 | 18 | deletion_protection = false 19 | table_id = var.commit_review_status_table_id 20 | dataset_id = var.dataset_id 21 | schema = jsonencode([ 22 | { 23 | name : "author", 24 | type : "STRING", 25 | mode : "REQUIRED", 26 | description : "The author of the commit." 27 | }, 28 | { 29 | name : "organization", 30 | type : "STRING", 31 | mode : "REQUIRED", 32 | description : "The GitHub organization to which the commit belongs." 33 | }, 34 | { 35 | name : "repository", 36 | type : "STRING", 37 | mode : "REQUIRED", 38 | description : "The GitHub repository to which the commit belongs." 39 | }, 40 | { 41 | name : "branch", 42 | type : "STRING", 43 | mode : "REQUIRED", 44 | description : "The GitHub branch to which the commit belongs." 45 | }, 46 | { 47 | name : "visibility", 48 | type : "STRING", 49 | mode : "NULLABLE", 50 | description : "The repository visibility" 51 | }, 52 | { 53 | name : "commit_sha", 54 | type : "STRING", 55 | mode : "REQUIRED", 56 | description : "The SHA Hash for the commit." 57 | }, 58 | { 59 | name : "commit_timestamp", 60 | type : "TIMESTAMP", 61 | mode : "REQUIRED", 62 | description : "The Timestamp when the commit was made" 63 | }, 64 | { 65 | name : "commit_html_url", 66 | type : "STRING", 67 | mode : "REQUIRED", 68 | description : "The URL for the commit in GitHub" 69 | }, 70 | { 71 | name : "pull_request_id", 72 | type : "INT64", 73 | mode : "NULLABLE", 74 | description : "The id of the pull request that introduced the commit." 75 | }, 76 | { 77 | name : "pull_request_number", 78 | type : "INT64", 79 | mode : "NULLABLE", 80 | description : "The number of the pull request that introduced the commit." 81 | }, 82 | { 83 | name : "pull_request_html_url", 84 | type : "STRING", 85 | mode : "NULLABLE", 86 | description : "The html url of the pull request that introduced the commit." 87 | }, 88 | { 89 | name : "approval_status", 90 | type : "STRING", 91 | mode : "REQUIRED", 92 | description : "The approval status of the commit in GitHub." 93 | }, 94 | { 95 | name : "break_glass_issue_urls", 96 | type : "STRING", 97 | mode : "REPEATED", 98 | description : "The URLs of the break glass issues that the author had open during the time the commit was made." 99 | }, 100 | { 101 | name : "note", 102 | type : "STRING", 103 | mode : "NULLABLE", 104 | description : "Optional context on the about the commit (e.g. a processing error message)" 105 | }, 106 | ]) 107 | } 108 | 109 | resource "google_bigquery_table_iam_member" "commit_review_status_owners" { 110 | for_each = toset(var.commit_review_status_table_iam.owners) 111 | 112 | project = var.project_id 113 | 114 | dataset_id = var.dataset_id 115 | table_id = google_bigquery_table.commit_review_status_table.id 116 | role = "roles/bigquery.dataOwner" 117 | member = each.value 118 | } 119 | 120 | resource "google_bigquery_table_iam_member" "commit_review_status_editors" { 121 | for_each = toset(var.commit_review_status_table_iam.editors) 122 | 123 | project = var.project_id 124 | 125 | dataset_id = var.dataset_id 126 | table_id = google_bigquery_table.commit_review_status_table.id 127 | role = "roles/bigquery.dataEditor" 128 | member = each.value 129 | } 130 | 131 | resource "google_bigquery_table_iam_member" "commit_review_status_viewers" { 132 | for_each = toset(var.commit_review_status_table_iam.viewers) 133 | 134 | project = var.project_id 135 | 136 | dataset_id = var.dataset_id 137 | table_id = google_bigquery_table.commit_review_status_table.id 138 | role = "roles/bigquery.dataViewer" 139 | member = each.value 140 | } 141 | -------------------------------------------------------------------------------- /terraform/modules/commit_review_status/outputs.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | output "commit_review_status_table_id" { 16 | value = google_bigquery_table.commit_review_status_table.table_id 17 | } 18 | 19 | output "google_service_account" { 20 | value = google_service_account.default 21 | } 22 | 23 | output "job_id" { 24 | value = google_cloud_run_v2_job.default.id 25 | } 26 | 27 | output "job_name" { 28 | value = google_cloud_run_v2_job.default.name 29 | } 30 | -------------------------------------------------------------------------------- /terraform/modules/commit_review_status/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | required_providers { 19 | google = { 20 | version = "~>5.19" 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /terraform/modules/invocation_comment/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | resource "google_bigquery_table" "invocation_comment_table" { 16 | project = var.project_id 17 | 18 | deletion_protection = false 19 | table_id = var.invocation_comment_table_id 20 | dataset_id = var.dataset_id 21 | schema = jsonencode([ 22 | { 23 | "name" : "pull_request_id", 24 | "type" : "INT64", 25 | "mode" : "REQUIRED", 26 | "description" : "ID of pull request." 27 | }, 28 | { 29 | "name" : "pull_request_html_url", 30 | "type" : "STRING", 31 | "mode" : "REQUIRED", 32 | "description" : "URL of pull request." 33 | }, 34 | { 35 | "name" : "processed_at", 36 | "type" : "TIMESTAMP", 37 | "mode" : "REQUIRED", 38 | "description" : "Timestamp of when the analyzer pipeline processed the PR." 39 | }, 40 | { 41 | "name" : "comment_id", 42 | "type" : "INT64", 43 | "mode" : "NULLABLE", 44 | "description" : "ID of pull request comment." 45 | }, 46 | { 47 | "name" : "status", 48 | "type" : "STRING", 49 | "mode" : "REQUIRED", 50 | "description" : "The status of invocation comment operation." 51 | }, 52 | { 53 | "name" : "job_name", 54 | "type" : "STRING", 55 | "mode" : "REQUIRED", 56 | "description" : "Job name of the analyzer that processed this event." 57 | }, 58 | ]) 59 | } 60 | 61 | resource "google_bigquery_table_iam_member" "invocation_comment_owners" { 62 | for_each = toset(var.invocation_comment_table_iam.owners) 63 | 64 | project = var.project_id 65 | 66 | dataset_id = var.dataset_id 67 | table_id = google_bigquery_table.invocation_comment_table.id 68 | role = "roles/bigquery.dataOwner" 69 | member = each.value 70 | } 71 | 72 | resource "google_bigquery_table_iam_member" "invocation_comment_editors" { 73 | for_each = toset(var.invocation_comment_table_iam.editors) 74 | 75 | project = var.project_id 76 | 77 | dataset_id = var.dataset_id 78 | table_id = google_bigquery_table.invocation_comment_table.id 79 | role = "roles/bigquery.dataEditor" 80 | member = each.value 81 | } 82 | 83 | resource "google_bigquery_table_iam_member" "invocation_comment_viewers" { 84 | for_each = toset(var.invocation_comment_table_iam.viewers) 85 | 86 | project = var.project_id 87 | 88 | dataset_id = var.dataset_id 89 | table_id = google_bigquery_table.invocation_comment_table.id 90 | role = "roles/bigquery.dataViewer" 91 | member = each.value 92 | } 93 | -------------------------------------------------------------------------------- /terraform/modules/invocation_comment/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | required_providers { 19 | google = { 20 | version = "~>5.19" 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /terraform/modules/invocation_comment/variables.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | variable "project_id" { 16 | description = "The GCP project ID." 17 | type = string 18 | } 19 | 20 | variable "dataset_id" { 21 | type = string 22 | description = "The BigQuery dataset id to create." 23 | } 24 | 25 | variable "invocation_comment_table_id" { 26 | description = "The BigQuery invocation comment table id to create." 27 | type = string 28 | default = "invocation_comment_status" 29 | nullable = false 30 | } 31 | 32 | variable "invocation_comment_table_iam" { 33 | description = "IAM member bindings for the BigQuery invocation comment table." 34 | type = object({ 35 | owners = optional(list(string), []) 36 | editors = optional(list(string), []) 37 | viewers = optional(list(string), []) 38 | }) 39 | default = { 40 | owners = [] 41 | editors = [] 42 | viewers = [] 43 | } 44 | nullable = false 45 | } 46 | -------------------------------------------------------------------------------- /terraform/notification.tf: -------------------------------------------------------------------------------- 1 | resource "google_monitoring_notification_channel" "non_paging" { 2 | for_each = { 3 | for k, v in var.alert_notification_channel_non_paging : k => v 4 | if k == "email" && try(v.labels.email_address, "") != "" 5 | } 6 | 7 | project = var.project_id 8 | 9 | display_name = "Non-paging Notification Channel" 10 | type = each.key 11 | labels = each.value.labels 12 | } 13 | -------------------------------------------------------------------------------- /terraform/terraform.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The Authors (see AUTHORS file) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | required_version = ">= 1.7" 17 | 18 | required_providers { 19 | google = { 20 | version = "~> 5.19" 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | --------------------------------------------------------------------------------