├── .bazelignore ├── .bazelrc ├── .bazelversion ├── .gitignore ├── .kazelcfg.json ├── .prow.yaml ├── AUTHORS ├── BUILD.bazel ├── CONTRIBUTING.md ├── LICENSE ├── OWNERS ├── README.md ├── WORKSPACE ├── benchmark ├── BUILD.bazel ├── tabulator.sh └── tabulator_bench_test.go ├── build_test_update.md ├── cc ├── BUILD ├── WORKSPACE ├── armeabi_cc_toolchain_config.bzl ├── builtin_include_directory_paths ├── cc_toolchain_config.bzl ├── cc_wrapper.sh ├── module.modulemap └── tools │ └── cpp │ └── empty.cc ├── cluster ├── README.md ├── bind-service-accounts.sh ├── canary │ ├── BUILD.bazel │ ├── api.yaml │ ├── config_merger.yaml │ ├── monitoring.yaml │ ├── namespace.yaml │ ├── summarizer.yaml │ ├── tabulator.yaml │ └── updater.yaml ├── create-subscription.sh ├── create-topic.sh ├── deploy.sh ├── list-gcs-prefixes.sh ├── prod │ ├── BUILD.bazel │ ├── api.yaml │ ├── config_merger.yaml │ ├── knative │ │ ├── OWNERS │ │ ├── namespace.yaml │ │ ├── setup-pubsub.sh │ │ ├── setup-testgrid-pubsub.sh │ │ ├── summarizer.yaml │ │ ├── tabulator.yaml │ │ └── updater.yaml │ ├── monitoring.yaml │ ├── namespace.yaml │ ├── summarizer.yaml │ ├── tabulator.yaml │ └── updater.yaml ├── setup-pubsub.sh ├── setup-testgrid-pubsub.sh ├── setup.sh ├── testgrid-canary-autobump-config.yaml └── testgrid-prod-autobump-config.yaml ├── cmd ├── README.md ├── api │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── config_merger │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── state_comparer │ ├── BUILD.bazel │ ├── README.md │ ├── main.go │ └── main_test.go ├── summarizer │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── tabulator │ ├── BUILD.bazel │ ├── README.md │ └── main.go └── updater │ ├── BUILD.bazel │ ├── README.md │ ├── main.go │ └── main_test.go ├── config.md ├── config ├── BUILD.bazel ├── cache.go ├── cache_test.go ├── config.go ├── config_test.go ├── converge.go ├── converge_test.go ├── fields.go ├── fields_test.go ├── print │ ├── BUILD.bazel │ ├── README.md │ └── main.go ├── queue.go ├── queue_test.go ├── snapshot │ ├── BUILD.bazel │ ├── config_snapshot.go │ └── config_snapshot_test.go └── yamlcfg │ ├── BUILD.bazel │ ├── yaml2proto.go │ └── yaml2proto_test.go ├── def.bzl ├── extension └── testgrid_alerter │ ├── css │ └── testgrid_alerter.css │ ├── images │ └── icon.png │ ├── js │ ├── popup.js │ ├── settings.js │ └── updater.js │ ├── manifest.json │ └── static │ ├── popup.html │ └── settings.html ├── go.mod ├── go.sum ├── hack ├── BUILD.bazel ├── autodeps.sh ├── bazel.sh ├── boilerplate │ ├── boilerplate.Dockerfile.txt │ ├── boilerplate.Makefile.txt │ ├── boilerplate.bzl.txt │ ├── boilerplate.generated.go.txt │ ├── boilerplate.go.txt │ ├── boilerplate.py.txt │ └── boilerplate.sh.txt ├── check-pr.sh ├── coalesce.py ├── coalesce_test.py ├── print-workspace-status.sh ├── tools.go ├── update-all.sh ├── update-file-perms.sh ├── update-protos.sh ├── update-spelling.sh ├── verify-all.sh ├── verify-file-perms.sh ├── verify-protos.sh ├── verify-spelling.sh └── verify_boilerplate.py ├── images ├── BUILD.bazel ├── gcloud-bazel │ ├── .gitignore │ └── push.sh └── push.sh ├── internal └── result │ ├── BUILD.bazel │ ├── results.go │ └── results_test.go ├── java └── BUILD ├── metadata ├── BUILD.bazel ├── README.md ├── job.go ├── job_test.go └── junit │ ├── BUILD.bazel │ ├── junit.go │ └── junit_test.go ├── pb ├── BUILD.bazel ├── README.md ├── api │ └── v1 │ │ ├── BUILD.bazel │ │ ├── data.pb.go │ │ └── data.proto ├── config │ ├── BUILD.bazel │ ├── config.pb.go │ └── config.proto ├── custom_evaluator │ ├── BUILD.bazel │ ├── custom_evaluator.pb.go │ └── custom_evaluator.proto ├── issue_state │ ├── BUILD.bazel │ ├── issue_state.pb.go │ └── issue_state.proto ├── state │ ├── BUILD.bazel │ ├── state.pb.go │ └── state.proto ├── summary │ ├── BUILD.bazel │ ├── summary.pb.go │ └── summary.proto └── test_status │ ├── BUILD.bazel │ ├── test_status.pb.go │ └── test_status.proto ├── pkg ├── api │ ├── BUILD.bazel │ ├── README.md │ ├── router.go │ ├── router_http_test.go │ └── v1 │ │ ├── BUILD.bazel │ │ ├── config.go │ │ ├── config_cache.go │ │ ├── config_cache_test.go │ │ ├── config_http_test.go │ │ ├── config_test.go │ │ ├── json.go │ │ ├── server.go │ │ ├── server_fake.go │ │ ├── state.go │ │ ├── state_test.go │ │ ├── summary.go │ │ └── summary_test.go ├── merger │ ├── BUILD.bazel │ ├── merger.go │ └── merger_test.go ├── pubsub │ ├── BUILD.bazel │ ├── pubsub.go │ └── pubsub_test.go ├── summarizer │ ├── BUILD.bazel │ ├── analyzers │ │ ├── BUILD.bazel │ │ ├── baseanalyzer.go │ │ ├── baseanalyzer_test.go │ │ ├── flipanalyzer.go │ │ └── flipanalyzer_test.go │ ├── common │ │ ├── BUILD.bazel │ │ └── common.go │ ├── flakiness.go │ ├── flakiness_test.go │ ├── persist.go │ ├── pubsub.go │ ├── pubsub_test.go │ ├── summary.go │ └── summary_test.go ├── tabulator │ ├── BUILD.bazel │ ├── filter.go │ ├── filter_test.go │ ├── persist.go │ ├── pubsub.go │ ├── pubsub_test.go │ ├── tabstate.go │ └── tabstate_test.go └── updater │ ├── BUILD.bazel │ ├── eval.go │ ├── eval_test.go │ ├── gcs.go │ ├── gcs_test.go │ ├── inflate.go │ ├── inflate_test.go │ ├── persist.go │ ├── persist_test.go │ ├── pubsub.go │ ├── pubsub_test.go │ ├── read.go │ ├── read_test.go │ ├── resultstore │ ├── BUILD.bazel │ ├── client.go │ ├── query │ │ ├── BUILD.bazel │ │ ├── query.go │ │ └── query_test.go │ ├── resultstore.go │ └── resultstore_test.go │ ├── updater.go │ └── updater_test.go ├── platform └── BUILD.bazel ├── repos.bzl ├── resultstore ├── BUILD.bazel ├── README.md ├── client.go ├── client_test.go ├── resultstore.go └── resultstore_test.go ├── standalone.md ├── terraform ├── Makefile ├── README.md ├── main.tf ├── modules │ └── alerts │ │ ├── main.tf │ │ ├── probers.tf │ │ └── variables.tf └── provider.tf └── util ├── BUILD.bazel ├── gcs ├── BUILD.bazel ├── client.go ├── fake │ ├── BUILD.bazel │ ├── fake.go │ ├── fake_test.go │ └── sort_test.go ├── gcs.go ├── gcs_test.go ├── local_gcs.go ├── local_gcs_test.go ├── read.go ├── read_test.go ├── real_gcs.go ├── real_gcs_test.go ├── sort.go └── sort_test.go ├── links.go ├── links_test.go ├── log.go ├── metrics ├── BUILD.bazel ├── logmetrics │ ├── BUILD.bazel │ ├── log.go │ └── log_test.go ├── metrics.go ├── metrics_test.go └── prometheus │ ├── BUILD.bazel │ ├── prometheus.go │ └── prometheus_test.go ├── queue ├── BUILD.bazel ├── persist.go ├── persist_test.go ├── queue.go └── queue_test.go └── strings.go /.bazelignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | # populate env used for stamping builds etc 2 | build --workspace_status_command=./hack/print-workspace-status.sh 3 | run --workspace_status_command=./hack/print-workspace-status.sh 4 | 5 | # enable data race detection 6 | test --@io_bazel_rules_go//go/config:race --test_output=errors 7 | 8 | # only build tests when testing 9 | test --build_tests_only 10 | 11 | # Note needs an instance name 12 | # See --config=ci-instance for a concrete example 13 | # https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-0.27.0.bazelrc 14 | build:remote --jobs=500 15 | build:remote --java_runtime_version=rbe_jdk 16 | build:remote --tool_java_runtime_version=rbe_jdk 17 | build:remote --extra_toolchains=//java:all 18 | 19 | build:remote --crosstool_top=@rbe_default//cc:toolchain 20 | build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 21 | 22 | build:remote --extra_toolchains=//platform:cc-toolchain 23 | build:remote --extra_execution_platforms=//platform:with_network 24 | build:remote --host_platform=//platform:with_network 25 | build:remote --platforms=//platform:with_network 26 | #build:remote --extra_execution_platforms=@io_k8s_repo_infra//:rbe_with_network 27 | #build:remote --host_platform=@io_k8s_repo_infra//:rbe_with_network 28 | #build:remote --platforms=@io_k8s_repo_infra//:rbe_with_network 29 | 30 | build:remote --define=EXECUTOR=remote 31 | build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com 32 | build:remote --remote_timeout=3600 33 | 34 | # --google_credentials=some_file.json 35 | build:remote --google_default_credentials=true 36 | build:remote --config=toplevel 37 | 38 | run:remote --remote_download_outputs=all --noexperimental_inmemory_jdeps_files --noexperimental_inmemory_dotd_files 39 | 40 | # Improve cache hit rate 41 | build:remote --incompatible_strict_action_env=true 42 | 43 | # Minimize what is downloaded 44 | build:inmemory --experimental_inmemory_jdeps_files 45 | build:inmemory --experimental_inmemory_dotd_files 46 | 47 | build:toplevel --config=inmemory 48 | build:toplevel --remote_download_outputs=toplevel 49 | 50 | build:minimal --config=inmemory 51 | build:minimal --remote_download_outputs=minimal 52 | 53 | # Compose the remote configs with an instance name 54 | # A couple examples below: 55 | 56 | # --config=ci-instance adds the instance name 57 | build:ci-instance --remote_instance_name=projects/oss-prow-builds/instances/default_instance 58 | build:trusted-instance --remote_instance_name=projects/k8s-prow/instances/default_instance 59 | 60 | # Config we want to use in ci 61 | build:ci --config=remote --config=ci-instance 62 | 63 | # Used for non-interactive ci builds 64 | build:ci --noshow_progress # reduce log spam 65 | test:ci --nobuild_tests_only # yes, build everything 66 | 67 | # https://github.com/bazelbuild/rules_go/pull/2110#issuecomment-508713878 68 | build --stamp=true 69 | -------------------------------------------------------------------------------- /.bazelversion: -------------------------------------------------------------------------------- 1 | 5.2.0 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.sublime-project 4 | *.sublime-workspace 5 | *~ 6 | /bazel-* 7 | /_artifacts 8 | /_output 9 | node_modules/ 10 | .DS_Store 11 | # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA 12 | .idea/ 13 | .ijwb/ 14 | *.iml 15 | .vscode/* 16 | /vendor/ 17 | # profiling outputs 18 | benchmark.test 19 | mem.out 20 | cpu.out 21 | ## terraform output 22 | *.tfvars 23 | *.terraform/ 24 | prow/oss/terraform/state.tf 25 | *.terraform.lock.hcl 26 | -------------------------------------------------------------------------------- /.kazelcfg.json: -------------------------------------------------------------------------------- 1 | { 2 | "GoPrefix": "github.com/GoogleCloudPlatform/testgrid", 3 | "AddSourcesRules": true, 4 | "SkippedPaths": ["node_modules"] 5 | } 6 | -------------------------------------------------------------------------------- /.prow.yaml: -------------------------------------------------------------------------------- 1 | presubmits: 2 | - name: test-testgrid-all 3 | decorate: true 4 | always_run: true 5 | spec: 6 | serviceAccountName: presubmits 7 | containers: 8 | - image: gcr.io/k8s-testgrid/gcloud-bazel:v20220613-v0.6-58-ge5a50c7 9 | command: 10 | - bazel 11 | args: 12 | - test 13 | - --config=ci 14 | - //... 15 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the list of TestGrid authors for copyright purposes. 2 | # 3 | # This does not necessarily list everyone who has contributed code, since in 4 | # some cases, their employer may be the copyright holder. To see the full list 5 | # of contributors, see the revision history in source control. 6 | Google LLC 7 | -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | # gazelle:prefix github.com/GoogleCloudPlatform/testgrid 2 | load("@io_k8s_repo_infra//defs:run_in_workspace.bzl", "workspace_binary") 3 | 4 | workspace_binary( 5 | name = "go", 6 | cmd = "@go_sdk//:bin/go", 7 | ) 8 | 9 | filegroup( 10 | name = "package-srcs", 11 | srcs = glob( 12 | ["**"], 13 | exclude = [ 14 | "bazel-*/**", 15 | ".git/**", 16 | ], 17 | ), 18 | visibility = ["//visibility:private"], 19 | ) 20 | 21 | filegroup( 22 | name = "all-srcs", 23 | srcs = [ 24 | ":package-srcs", 25 | "//benchmark:all-srcs", 26 | "//cc:all-srcs", 27 | "//cluster/canary:all-srcs", 28 | "//cluster/prod:all-srcs", 29 | "//cmd/api:all-srcs", 30 | "//cmd/config_merger:all-srcs", 31 | "//cmd/state_comparer:all-srcs", 32 | "//cmd/summarizer:all-srcs", 33 | "//cmd/tabulator:all-srcs", 34 | "//cmd/updater:all-srcs", 35 | "//config:all-srcs", 36 | "//hack:all-srcs", 37 | "//images:all-srcs", 38 | "//internal/result:all-srcs", 39 | "//java:all-srcs", 40 | "//metadata:all-srcs", 41 | "//pb:all-srcs", 42 | "//pkg/api:all-srcs", 43 | "//pkg/merger:all-srcs", 44 | "//pkg/pubsub:all-srcs", 45 | "//pkg/summarizer:all-srcs", 46 | "//pkg/tabulator:all-srcs", 47 | "//pkg/updater:all-srcs", 48 | "//platform:all-srcs", 49 | "//resultstore:all-srcs", 50 | "//util:all-srcs", 51 | ], 52 | tags = ["automanaged"], 53 | visibility = ["//visibility:public"], 54 | ) 55 | 56 | exports_files([ 57 | "go.mod", 58 | "go.sum", 59 | ]) 60 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google.com/conduct/). 29 | 30 | ## Source Code Headers 31 | 32 | Every file containing source code must include copyright and license 33 | information. This includes any JS/CSS files that you might be serving out to 34 | browsers. (This is to help well-intentioned people avoid accidental copying that 35 | doesn't comply with the license.) 36 | 37 | Apache header: 38 | 39 | Copyright 2023 The TestGrid Authors. 40 | 41 | Licensed under the Apache License, Version 2.0 (the "License"); 42 | you may not use this file except in compliance with the License. 43 | You may obtain a copy of the License at 44 | 45 | https://www.apache.org/licenses/LICENSE-2.0 46 | 47 | Unless required by applicable law or agreed to in writing, software 48 | distributed under the License is distributed on an "AS IS" BASIS, 49 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 50 | See the License for the specific language governing permissions and 51 | limitations under the License. 52 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | reviewers: 4 | - chases2 5 | - sultan-duisenbay 6 | - michelle192837 7 | 8 | approvers: 9 | - chases2 10 | - sultan-duisenbay 11 | - michelle192837 12 | 13 | emeritus_approvers: 14 | - fejta 15 | -------------------------------------------------------------------------------- /benchmark/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_test") 2 | 3 | filegroup( 4 | name = "package-srcs", 5 | srcs = glob(["**"]), 6 | tags = ["automanaged"], 7 | visibility = ["//visibility:private"], 8 | ) 9 | 10 | filegroup( 11 | name = "all-srcs", 12 | srcs = [":package-srcs"], 13 | tags = ["automanaged"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | 17 | go_test( 18 | name = "go_default_test", 19 | srcs = ["tabulator_bench_test.go"], 20 | deps = [ 21 | "//pkg/tabulator:go_default_library", 22 | "//util/gcs:go_default_library", 23 | "//util/metrics/prometheus:go_default_library", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /benchmark/tabulator.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2022 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | # Time the tabultor to compare algorithm differences. 21 | # Runs across the entire configuration once; ignoring events. 22 | 23 | SRC_BUCKET="gs://testgrid-canary" # An example state to operate on 24 | WORK_BUCKET="/tmp/testgrid-state" # A bucket to work in. Can be GCS or Local 25 | 26 | # Tabulator uses config and testgroup state as inputs 27 | gsutil -m cp ${SRC_BUCKET}/config ${WORK_BUCKET}/config 28 | gsutil -m rsync -r ${SRC_BUCKET}/grid ${WORK_BUCKET}/grid 29 | go test --bench="BenchmarkTabulator" --benchmem --cpuprofile cpu.out --memprofile mem.out --config="${WORK_BUCKET}/config" --confirm 30 | -------------------------------------------------------------------------------- /benchmark/tabulator_bench_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package benchmark 18 | 19 | import ( 20 | "context" 21 | "flag" 22 | "runtime" 23 | "testing" 24 | "time" 25 | 26 | "github.com/GoogleCloudPlatform/testgrid/pkg/tabulator" 27 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 28 | "github.com/GoogleCloudPlatform/testgrid/util/metrics/prometheus" 29 | ) 30 | 31 | var config = flag.String("config", "", "Config file of working testgrid directory. The directory may be modified.") 32 | 33 | func BenchmarkTabulator(b *testing.B) { 34 | ctx, cancel := context.WithCancel(context.Background()) 35 | defer cancel() 36 | 37 | if config == nil || *config == "" { 38 | b.Fatalf("needs --config flag") 39 | } 40 | 41 | cfgPath, err := gcs.NewPath(*config) 42 | if err != nil { 43 | b.Fatalf("bad config path %q: %v", *config, err) 44 | } 45 | 46 | storageClient, err := gcs.ClientWithCreds(ctx, "") 47 | if err != nil { 48 | b.Fatalf("create client: %v", err) 49 | } 50 | defer storageClient.Close() 51 | 52 | client := gcs.NewClient(storageClient) 53 | 54 | mets := tabulator.CreateMetrics(prometheus.NewFactory()) // TODO(chases2): replace with logging metrics factory or teach tabulator to tolerate missing metrics 55 | 56 | opts := &tabulator.UpdateOptions{ 57 | ConfigPath: *cfgPath, 58 | ReadConcurrency: 2 * runtime.NumCPU(), 59 | WriteConcurrency: 4 * runtime.NumCPU(), 60 | GridPathPrefix: "grid", 61 | TabsPathPrefix: "tabs", 62 | AllowedGroups: nil, 63 | Confirm: true, 64 | CalculateStats: true, 65 | UseTabAlertSettings: true, 66 | ExtendState: false, 67 | Freq: time.Duration(0), 68 | } 69 | if err = tabulator.Update(ctx, client, mets, opts); err != nil { 70 | b.Fatalf("update error: %v", err) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /build_test_update.md: -------------------------------------------------------------------------------- 1 | # Updating TestGrid 2 | 3 | If you're looking to develop for TestGrid, welcome! 4 | 5 | While the front-end UI is not currently open source, it can be used as a [service](./standalone.md). 6 | If you would like to propose a UI change or feature, please file a bug describing how you would like the UI to behave. 7 | 8 | ## YAML configuration and config.proto 9 | 10 | While TestGrid configuration is located in YAML files, TestGrid doesn't natively 11 | read YAML. Instead, it expects configuration data in [`config.proto`]. This file 12 | is commented, and should be treated as the authoritative "input" schema to 13 | TestGrid. 14 | 15 | [`config.proto`] is generated primarily with 16 | [Configurator](https://github.com/kubernetes/test-infra/blob/master/testgrid/cmd/configurator). 17 | Updates to the [testgrid.k8s.io config] are automatically Configurated when a change is 18 | merged. 19 | 20 | You can convert a yaml file to the config proto with: 21 | ``` 22 | bazel run //testgrid/cmd/configurator -- \ 23 | --yaml=testgrid/config.yaml \ 24 | --print-text \ 25 | --oneshot \ 26 | --output=/tmp/config.pb \ 27 | # Or push to gcs 28 | # --output=gs://my-bucket/config 29 | # --gcp-service-account=/path/to/foo.json 30 | ``` 31 | 32 | For our production instance of TestGrid (https://testgrid.k8s.io), this file is read to a Google 33 | Cloud Storage (GCS) location, and read from there. 34 | 35 | ## Changing a .proto file 36 | 37 | If you modify a .proto file, you'll also need to generate and check in the 38 | .pb.go files. 39 | 40 | Run `bazel run //hack:update-protos` to generate, and `bazel run //hack:verify-protos.sh` 41 | to verify. This command requires `python` to be installed. 42 | 43 | ## Testing 44 | 45 | Run `bazel test //...` to run all unit tests in TestGrid. Note that this does not validate 46 | the [testgrid.k8s.io config]; those tests are in `bazel test //config/tests/testgrids/...` 47 | 48 | [`config.proto`]: ./config/config.proto 49 | [testgrid.k8s.io config]: https://github.com/kubernetes/test-infra/blob/master/config/testgrids 50 | -------------------------------------------------------------------------------- /cc/WORKSPACE: -------------------------------------------------------------------------------- 1 | # DO NOT EDIT: automatically generated WORKSPACE file for cc_autoconf rule 2 | workspace(name = "local_config_cc") 3 | -------------------------------------------------------------------------------- /cc/builtin_include_directory_paths: -------------------------------------------------------------------------------- 1 | This file is generated by cc_configure and contains builtin include directories 2 | that /usr/local/bin/clang-12 reported. This file is a dependency of every compilation action and 3 | changes to it will be reflected in the action cache key. When some of these 4 | paths change, Bazel will make sure to rerun the action, even though none of 5 | declared action inputs or the action commandline changes. 6 | 7 | /usr/local/include 8 | /usr/local/lib/clang/12.0.0/include 9 | /usr/include/x86_64-linux-gnu 10 | /usr/include 11 | /usr/local/lib/clang/12.0.0/share 12 | /usr/include/c++/7.5.0 13 | /usr/include/x86_64-linux-gnu/c++/7.5.0 14 | /usr/include/c++/7.5.0/backward 15 | /usr/local/include/c++/v1 16 | -------------------------------------------------------------------------------- /cc/cc_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # 17 | # Ship the environment to the C++ action 18 | # 19 | set -eu 20 | 21 | # Set-up the environment 22 | 23 | 24 | # Call the C++ compiler 25 | /usr/local/bin/clang-12 "$@" 26 | -------------------------------------------------------------------------------- /cc/tools/cpp/empty.cc: -------------------------------------------------------------------------------- 1 | int main() {} -------------------------------------------------------------------------------- /cluster/README.md: -------------------------------------------------------------------------------- 1 | # Add a new component to the release 2 | 3 | 1. Create the component, such that no other components depend on it. (If it fails, you can debug without worrying about breaking production.) 4 | 1. **PR**: Add the component to the BUILD file, and send a PR for this. (Example: https://github.com/GoogleCloudPlatform/testgrid/pull/822). 5 | 1. Wait until Container Registry has a date-versioned image for the component. 6 | - e.g. https://github.com/GoogleCloudPlatform/testgrid/pull/822 -> tabulator image with version`v20220210-v0.0.121-16-g5bceb0c` 7 | 1. **PR**: Create a configuration `.yaml` file under `cluster/canary`, referencing the image version all components are using in production (so the autobumper can catch it). 8 | 1. Bind the service account(s) for the component in the `testgrid-canary` namespace: 9 | 10 | ``` 11 | NAMESPACE=testgrid-canary 12 | PROJECT= 13 | COMPONENT= 14 | SERVICE_ACCOUNT= 15 | gcloud iam service-accounts add-iam-policy-binding --project=$PROJECT --role=roles/iam.workloadIdentityUser --member=serviceAccount:k8s-testgrid.svc.id.goog[$NAMESPACE/$COMPONENT] $SERVICE_ACCOUNT 16 | ``` 17 | 1. **PR**: Create `.yaml` under `cluster/prod`. 18 | 1. Bind the service account(s) for the component in the `testgrid` namespace: 19 | 20 | ``` 21 | NAMESPACE=testgrid 22 | PROJECT= 23 | COMPONENT= 24 | SERVICE_ACCOUNT= 25 | gcloud iam service-accounts add-iam-policy-binding --project=$PROJECT --role=roles/iam.workloadIdentityUser --member=serviceAccount:k8s-testgrid.svc.id.goog[$NAMESPACE/$COMPONENT] $SERVICE_ACCOUNT 26 | ``` -------------------------------------------------------------------------------- /cluster/canary/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@io_bazel_rules_k8s//k8s:object.bzl", "k8s_object") 16 | load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") 17 | 18 | k8s_objects( 19 | name = "canary", 20 | objects = [ 21 | ":namespace", # Must be first 22 | ":api", 23 | ":summarizer", 24 | ":updater", 25 | ":config_merger", 26 | "tabulator", 27 | ], 28 | ) 29 | 30 | CLUSTER = "{STABLE_TESTGRID_CLUSTER}" 31 | 32 | MULTI_KIND = None 33 | 34 | k8s_object( 35 | name = "api", 36 | cluster = CLUSTER, 37 | kind = MULTI_KIND, 38 | template = "api.yaml", 39 | ) 40 | 41 | k8s_object( 42 | name = "updater", 43 | cluster = CLUSTER, 44 | kind = MULTI_KIND, 45 | template = "updater.yaml", 46 | ) 47 | 48 | k8s_object( 49 | name = "summarizer", 50 | cluster = CLUSTER, 51 | kind = MULTI_KIND, 52 | template = "summarizer.yaml", 53 | ) 54 | 55 | k8s_object( 56 | name = "config_merger", 57 | cluster = CLUSTER, 58 | kind = MULTI_KIND, 59 | template = "config_merger.yaml", 60 | ) 61 | 62 | k8s_object( 63 | name = "tabulator", 64 | cluster = CLUSTER, 65 | kind = MULTI_KIND, 66 | template = "tabulator.yaml", 67 | ) 68 | 69 | k8s_object( 70 | name = "namespace", 71 | cluster = CLUSTER, 72 | kind = "Namespace", 73 | template = "namespace.yaml", 74 | ) 75 | 76 | filegroup( 77 | name = "package-srcs", 78 | srcs = glob(["**"]), 79 | tags = ["automanaged"], 80 | visibility = ["//visibility:private"], 81 | ) 82 | 83 | filegroup( 84 | name = "all-srcs", 85 | srcs = [":package-srcs"], 86 | tags = ["automanaged"], 87 | visibility = ["//visibility:public"], 88 | ) 89 | -------------------------------------------------------------------------------- /cluster/canary/api.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-api 6 | namespace: testgrid-canary 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: api 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: api 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: api 24 | spec: 25 | serviceAccountName: api 26 | containers: 27 | - name: api 28 | image: gcr.io/k8s-testgrid/api:v20250417-v0.0.174-18-g6730e7b4 29 | args: 30 | - --allowed-origin=* 31 | - --scope=gs://k8s-testgrid-canary 32 | - --port=8080 33 | --- 34 | apiVersion: v1 35 | kind: ServiceAccount 36 | metadata: 37 | annotations: 38 | iam.gke.io/gcp-service-account: testgrid-canary-api@k8s-testgrid.iam.gserviceaccount.com 39 | name: api 40 | namespace: testgrid-canary 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: api 46 | namespace: testgrid-canary 47 | spec: 48 | type: NodePort 49 | selector: 50 | app: testgrid 51 | component: api 52 | ports: 53 | - protocol: TCP 54 | port: 80 55 | targetPort: 8080 56 | --- 57 | apiVersion: networking.k8s.io/v1 58 | kind: Ingress 59 | metadata: 60 | name: testgrid-api-ingress 61 | namespace: testgrid-canary 62 | annotations: 63 | kubernetes.io/ingress.class: "gce" 64 | spec: 65 | rules: 66 | - http: 67 | paths: 68 | - path: /* 69 | pathType: ImplementationSpecific 70 | backend: 71 | service: 72 | name: api 73 | port: 74 | number: 80 75 | -------------------------------------------------------------------------------- /cluster/canary/config_merger.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-config-merger 6 | namespace: testgrid-canary 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: config-merger 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: config-merger 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: config-merger 24 | spec: 25 | serviceAccountName: config-merger 26 | containers: 27 | - name: config-merger 28 | image: gcr.io/k8s-testgrid/config_merger:v20250417-v0.0.174-18-g6730e7b4 29 | ports: 30 | - name: metrics 31 | containerPort: 2112 32 | args: 33 | - --config-url=https://raw.githubusercontent.com/kubernetes/test-infra/master/config/mergelists/canary.yaml 34 | - --confirm 35 | - --wait=15m 36 | --- 37 | apiVersion: v1 38 | kind: ServiceAccount 39 | metadata: 40 | annotations: 41 | # Uses same as updater 42 | iam.gke.io/gcp-service-account: testgrid-canary@k8s-testgrid.iam.gserviceaccount.com 43 | name: config-merger 44 | namespace: testgrid-canary 45 | -------------------------------------------------------------------------------- /cluster/canary/monitoring.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: monitoring.gke.io/v1alpha1 3 | kind: PodMonitor 4 | metadata: 5 | labels: 6 | app: testgrid-metrics 7 | component: config-merger 8 | name: config-merger 9 | namespace: testgrid-canary 10 | spec: 11 | podMetricsEndpoints: 12 | - interval: 30s 13 | port: metrics 14 | scheme: http 15 | namespaceSelector: 16 | matchNames: 17 | - testgrid-canary 18 | selector: 19 | matchLabels: 20 | app: testgrid 21 | component: config-merger 22 | --- 23 | apiVersion: monitoring.gke.io/v1alpha1 24 | kind: PodMonitor 25 | metadata: 26 | labels: 27 | app: testgrid-metrics 28 | component: summarizer 29 | name: summarizer 30 | namespace: testgrid-canary 31 | spec: 32 | podMetricsEndpoints: 33 | - interval: 30s 34 | port: metrics 35 | scheme: http 36 | namespaceSelector: 37 | matchNames: 38 | - testgrid-canary 39 | selector: 40 | matchLabels: 41 | app: testgrid 42 | component: summarizer 43 | --- 44 | apiVersion: monitoring.gke.io/v1alpha1 45 | kind: PodMonitor 46 | metadata: 47 | labels: 48 | app: testgrid-metrics 49 | component: updater 50 | name: updater 51 | namespace: testgrid-canary 52 | spec: 53 | podMetricsEndpoints: 54 | - interval: 30s 55 | port: metrics 56 | scheme: http 57 | namespaceSelector: 58 | matchNames: 59 | - testgrid-canary 60 | selector: 61 | matchLabels: 62 | app: testgrid 63 | component: updater 64 | --- 65 | # These will be consumed by GKE Managed Prometheus(GMP) services in the cluster. 66 | # See: https://cloud.google.com/stackdriver/docs/managed-prometheus. 67 | apiVersion: monitoring.googleapis.com/v1 68 | kind: PodMonitoring 69 | metadata: 70 | labels: 71 | app: testgrid-metrics 72 | component: config-merger 73 | name: config-merger 74 | namespace: testgrid-canary 75 | spec: 76 | endpoints: 77 | - interval: 30s 78 | port: metrics 79 | scheme: http 80 | selector: 81 | matchLabels: 82 | app: testgrid 83 | component: config-merger 84 | --- 85 | apiVersion: monitoring.googleapis.com/v1 86 | kind: PodMonitoring 87 | metadata: 88 | labels: 89 | app: testgrid-metrics 90 | component: summarizer 91 | name: summarizer 92 | namespace: testgrid-canary 93 | spec: 94 | endpoints: 95 | - interval: 30s 96 | port: metrics 97 | scheme: http 98 | selector: 99 | matchLabels: 100 | app: testgrid 101 | component: summarizer 102 | --- 103 | apiVersion: monitoring.googleapis.com/v1 104 | kind: PodMonitoring 105 | metadata: 106 | labels: 107 | app: testgrid-metrics 108 | component: updater 109 | name: updater 110 | namespace: testgrid-canary 111 | spec: 112 | endpoints: 113 | - interval: 30s 114 | port: metrics 115 | scheme: http 116 | selector: 117 | matchLabels: 118 | app: testgrid 119 | component: updater 120 | -------------------------------------------------------------------------------- /cluster/canary/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: testgrid-canary 5 | -------------------------------------------------------------------------------- /cluster/canary/summarizer.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | annotations: 6 | # Uses same as updater 7 | iam.gke.io/gcp-service-account: testgrid-canary@k8s-testgrid.iam.gserviceaccount.com 8 | name: summarizer 9 | namespace: testgrid-canary 10 | --- 11 | apiVersion: apps/v1 12 | kind: Deployment 13 | metadata: 14 | name: testgrid-summarizer-tabs 15 | namespace: testgrid-canary 16 | labels: 17 | app: testgrid 18 | channel: stable 19 | component: summarizer-tabs 20 | spec: 21 | replicas: 1 22 | selector: 23 | matchLabels: 24 | app: testgrid 25 | channel: stable 26 | component: summarizer-tabs 27 | template: 28 | metadata: 29 | labels: 30 | app: testgrid 31 | channel: stable 32 | component: summarizer-tabs 33 | spec: 34 | serviceAccountName: summarizer 35 | containers: 36 | - name: summarizer 37 | image: gcr.io/k8s-testgrid/summarizer:v20250417-v0.0.174-18-g6730e7b4 38 | ports: 39 | - name: metrics 40 | containerPort: 2112 41 | args: 42 | - --config=gs://k8s-testgrid-canary/config 43 | - --confirm 44 | - --json-logs 45 | - --persist-queue=gs://k8s-testgrid-canary/queue/summarizer-tabs.json 46 | - --pubsub=k8s-testgrid/canary-tab-updates 47 | - --wait=1h 48 | resources: 49 | requests: 50 | cpu: "1" 51 | memory: "50G" 52 | limits: 53 | cpu: "2" 54 | memory: "100G" 55 | --- 56 | -------------------------------------------------------------------------------- /cluster/canary/tabulator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-tabulator 6 | namespace: testgrid-canary 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: tabulator 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: tabulator 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: tabulator 24 | spec: 25 | serviceAccountName: tabulator 26 | containers: 27 | - name: tabulator 28 | image: gcr.io/k8s-testgrid/tabulator:v20250417-v0.0.174-18-g6730e7b4 29 | ports: 30 | - name: metrics 31 | containerPort: 2112 32 | args: 33 | - --column-stats 34 | - --config=gs://k8s-testgrid-canary/config 35 | - --confirm 36 | - --json-logs 37 | - --persist-queue=gs://k8s-testgrid-canary/queue/tabulator.json 38 | - --pubsub=k8s-testgrid/canary-test-group-updates 39 | - --read-concurrency=10 40 | - --wait=15m 41 | - --write-concurrency=10 42 | resources: 43 | requests: 44 | cpu: "30" 45 | memory: "25G" 46 | limits: 47 | cpu: "40" 48 | memory: "50G" 49 | --- 50 | apiVersion: v1 51 | kind: ServiceAccount 52 | metadata: 53 | annotations: 54 | # Uses same as updater 55 | iam.gke.io/gcp-service-account: testgrid-canary@k8s-testgrid.iam.gserviceaccount.com 56 | name: tabulator 57 | namespace: testgrid-canary 58 | -------------------------------------------------------------------------------- /cluster/canary/updater.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | annotations: 6 | iam.gke.io/gcp-service-account: testgrid-canary@k8s-testgrid.iam.gserviceaccount.com 7 | name: updater 8 | namespace: testgrid-canary 9 | --- 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | metadata: 13 | name: testgrid-updater 14 | namespace: testgrid-canary 15 | labels: 16 | component: updater 17 | app: testgrid 18 | spec: 19 | replicas: 1 # https://github.com/GoogleCloudPlatform/testgrid/issues/663 20 | selector: 21 | matchLabels: 22 | app: testgrid 23 | component: updater 24 | template: 25 | metadata: 26 | labels: 27 | component: updater 28 | app: testgrid 29 | spec: 30 | serviceAccountName: updater 31 | containers: 32 | - name: updater 33 | image: gcr.io/k8s-testgrid/updater:v20250417-v0.0.174-18-g6730e7b4 34 | ports: 35 | - name: metrics 36 | containerPort: 2112 37 | args: 38 | - --build-concurrency=10 39 | - --build-timeout=5m 40 | - --config=gs://k8s-testgrid-canary/config 41 | - --confirm 42 | - --group-concurrency=10 43 | - --group-timeout=20m 44 | - --json-logs 45 | - --persist-queue=gs://k8s-testgrid-canary/queue/updater.json 46 | - --subscribe=kubernetes-jenkins=kubernetes-jenkins/testgrid-canary 47 | - --subscribe=oss-prow=k8s-testgrid/testgrid-canary 48 | - --subscribe=istio-prow=k8s-testgrid/testgrid-canary 49 | - --subscribe=kubernetes-ci-logs=k8s-testgrid/testgrid-canary-kubernetes-ci-logs 50 | - --wait=4h 51 | resources: 52 | requests: 53 | cpu: "30" 54 | memory: "75G" 55 | limits: 56 | cpu: "40" 57 | memory: "150G" 58 | --- 59 | -------------------------------------------------------------------------------- /cluster/create-topic.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -o nounset 18 | set -o errexit 19 | 20 | # See gsutil notification --help 21 | 22 | args=() 23 | topic= 24 | prefixes=() 25 | 26 | while getopts "e:t:p:b:" flag; do 27 | case "$flag" in 28 | e) args+=(-e "$OPTARG");; 29 | t) topic=$OPTARG;; 30 | p) prefixes+=("$OPTARG");; 31 | esac 32 | done 33 | 34 | shift $((OPTIND -1)) 35 | 36 | if [[ -z "$topic" || $# == 0 ]]; then 37 | echo "Usage: $(basename "$0") [-e EVENT [-e ...]] [-p PREFIX] <-t TOPIC> " >&2 38 | echo >&2 39 | echo " -e EVENT: OBJECT_FINALIZE|OBJECT_METADATA_UPDATE|OBJECT_DELETE|OBJECT_ARCHIVE (repeatable)" >&2 40 | echo " -p PREFIX: only publish messages for objects that start with this name, such as logs/" >&2 41 | echo " -t TOPIC: foo or projects/proj/topics/foo" >&2 42 | echo >&2 43 | echo " More info: gsutil notification --help" >&2 44 | exit 1 45 | fi 46 | 47 | log() { 48 | ( 49 | set -o xtrace 50 | "$@" 51 | ) 52 | } 53 | 54 | for bucket in "$@"; do 55 | existing=( $(gsutil notification list "$bucket" 2>/dev/null | grep -B 1 "$topic" | grep notificationConfigs || true) ) 56 | for prefix in "${prefixes[@]}"; do 57 | log gsutil notification create -f json -t "$topic" -p "$prefix" "${args[@]}" "$bucket" 58 | done 59 | if [[ ${#existing[@]} -gt 0 ]]; then 60 | log gsutil notification delete "${existing[@]}" 61 | fi 62 | done 63 | -------------------------------------------------------------------------------- /cluster/list-gcs-prefixes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | if [[ $# -lt 1 ]]; then 18 | echo "Usage: $(basename "$0") " >&2 19 | exit 1 20 | fi 21 | # Find all the "gcs_prefix": "bucket/path/to/job" and output bucket/path 22 | go run github.com/GoogleCloudPlatform/testgrid/config/print --path="$1" \ 23 | | grep '"gcs_prefix"' \ 24 | | cut -d '"' -f 4 \ 25 | | cut -d / -f 1-2 \ 26 | | sort -u 27 | -------------------------------------------------------------------------------- /cluster/prod/api.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-api 6 | namespace: testgrid 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: api 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: api 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: api 24 | spec: 25 | serviceAccountName: api 26 | containers: 27 | - name: api 28 | image: gcr.io/k8s-testgrid/api:v20240917-v0.0.174-2-g70f42770 29 | args: 30 | - --allowed-origin=* 31 | - --scope=gs://k8s-testgrid 32 | - --port=8080 33 | --- 34 | apiVersion: v1 35 | kind: ServiceAccount 36 | metadata: 37 | annotations: 38 | iam.gke.io/gcp-service-account: testgrid-api@k8s-testgrid.iam.gserviceaccount.com 39 | name: api 40 | namespace: testgrid 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: api 46 | namespace: testgrid 47 | spec: 48 | type: NodePort 49 | selector: 50 | app: testgrid 51 | component: api 52 | ports: 53 | - protocol: TCP 54 | port: 80 55 | targetPort: 8080 56 | --- 57 | apiVersion: networking.k8s.io/v1 58 | kind: Ingress 59 | metadata: 60 | name: testgrid-api-ingress 61 | namespace: testgrid 62 | annotations: 63 | kubernetes.io/ingress.class: "gce" 64 | spec: 65 | rules: 66 | - http: 67 | paths: 68 | - path: /* 69 | pathType: ImplementationSpecific 70 | backend: 71 | service: 72 | name: api 73 | port: 74 | number: 80 75 | -------------------------------------------------------------------------------- /cluster/prod/config_merger.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-config-merger 6 | namespace: testgrid 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: config-merger 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: config-merger 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: config-merger 24 | spec: 25 | serviceAccountName: config-merger 26 | containers: 27 | - name: config-merger 28 | image: gcr.io/k8s-testgrid/config_merger:v20240917-v0.0.174-2-g70f42770 29 | ports: 30 | - name: metrics 31 | containerPort: 2112 32 | args: 33 | - --config-url=https://raw.githubusercontent.com/kubernetes/test-infra/master/config/mergelists/prod.yaml 34 | - --confirm 35 | - --wait=15m 36 | --- 37 | apiVersion: v1 38 | kind: ServiceAccount 39 | metadata: 40 | annotations: 41 | # Uses same as updater 42 | iam.gke.io/gcp-service-account: updater@k8s-testgrid.iam.gserviceaccount.com 43 | name: config-merger 44 | namespace: testgrid 45 | -------------------------------------------------------------------------------- /cluster/prod/knative/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - chaodaiG 3 | - fejta 4 | -------------------------------------------------------------------------------- /cluster/prod/knative/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: knative 5 | -------------------------------------------------------------------------------- /cluster/prod/knative/setup-pubsub.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -o nounset 18 | set -o errexit 19 | 20 | buckets=( 21 | gs://knative-prow 22 | ) 23 | 24 | topic=projects/knative-tests/topics/prow-updates 25 | 26 | dir=$(dirname "$0")/../.. 27 | list=$dir/list-gcs-prefixes.sh 28 | create_topic=$dir/create-topic.sh 29 | create_sub=$dir/create-subscription.sh 30 | 31 | 32 | log() { 33 | ( 34 | set -o xtrace 35 | "$@" 36 | ) 37 | } 38 | 39 | apply-subscription() { 40 | topic=$1 41 | project=$2 42 | sub=$3 43 | # Prod 44 | log "$create_sub" -t "$topic" \ 45 | -b serviceAccount:testgrid-updater@knative-tests.iam.gserviceaccount.com \ 46 | -p "$project" "$sub" 47 | } 48 | 49 | apply-topic() { 50 | topic=$1 51 | shift 52 | buckets=("$@") 53 | log "$create_topic" -t "$topic" -p logs/ "${buckets[@]}" 54 | } 55 | 56 | do-list() { 57 | "$list" gs://knative-own-testgrid/config 58 | echo "NOTICE: edit this file ($(basename "$0")) to add any additional paths" >&2 59 | } 60 | 61 | something= 62 | while getopts "lst" flag; do 63 | case "$flag" in 64 | s) 65 | apply-subscription "$topic" knative-tests testgrid 66 | something=yes 67 | ;; 68 | t) 69 | apply-topic "$topic" "${buckets[@]}" 70 | something=yes 71 | ;; 72 | l) 73 | do-list 74 | something=yes 75 | ;; 76 | esac 77 | done 78 | 79 | if [[ -z "$something" ]]; then 80 | echo "Usage: $(basename "$0") [-u] [-s] [-l]" >&2 81 | echo >&2 82 | echo " -l: list buckets in use" >&2 83 | echo " -t: configure topics for ${buckets[@]}" >&2 84 | echo " -s: configure subscriptions" >&2 85 | exit 1 86 | fi 87 | -------------------------------------------------------------------------------- /cluster/prod/knative/setup-testgrid-pubsub.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2022 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | ############# 17 | # Sets up the internal queues that TestGrid uses to speak to other parts of itself 18 | ############# 19 | set -o nounset 20 | set -o errexit 21 | 22 | 23 | dir=$(dirname "$0")/../.. 24 | create_topic=$dir/create-topic.sh 25 | create_sub=$dir/create-subscription.sh 26 | 27 | log() { 28 | ( 29 | set -o xtrace 30 | "$@" 31 | ) 32 | } 33 | 34 | 35 | apply() { 36 | log "$create_topic" -t "${topic_prefix}-grid" -p 'grid/' "$bucket" 37 | log "$create_topic" -t "${topic_prefix}-tabs" -p 'tabs/' "$bucket" 38 | 39 | log "$create_sub" -t "${topic_prefix}-grid" -b "$bot" -p "${project}" "$group_sub" 40 | log "$create_sub" -t "${topic_prefix}-tabs" -b "$bot" -p "${project}" "$tab_sub" 41 | } 42 | 43 | project=knative-tests 44 | bucket=gs://knative-own-testgrid 45 | topic_prefix="projects/$project/topics/knative-own-testgrid" 46 | group_sub=test-group-updates 47 | tab_sub=tab-updates 48 | bot=serviceAccount:testgrid-updater@knative-tests.iam.gserviceaccount.com 49 | apply 50 | -------------------------------------------------------------------------------- /cluster/prod/knative/summarizer.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-summarizer-tabs 6 | namespace: knative 7 | labels: 8 | app: testgrid 9 | component: summarizer-tabs 10 | spec: 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: testgrid 15 | component: summarizer-tabs 16 | template: 17 | metadata: 18 | labels: 19 | app: testgrid 20 | component: summarizer-tabs 21 | spec: 22 | serviceAccountName: summarizer 23 | containers: 24 | - name: summarizer 25 | image: gcr.io/k8s-testgrid/summarizer:v20240917-v0.0.174-2-g70f42770 26 | ports: 27 | - name: metrics 28 | containerPort: 2112 29 | args: 30 | - --config=gs://knative-own-testgrid/config 31 | - --confirm 32 | - --json-logs 33 | - --persist-queue=gs://knative-own-testgrid/queue/summarizer-tabs.json 34 | - --pubsub=knative-tests/tab-updates 35 | - --wait=5m 36 | --- 37 | apiVersion: v1 38 | kind: ServiceAccount 39 | metadata: 40 | annotations: 41 | # Uses same as updater 42 | iam.gke.io/gcp-service-account: testgrid-updater@knative-tests.iam.gserviceaccount.com 43 | name: summarizer 44 | namespace: knative 45 | -------------------------------------------------------------------------------- /cluster/prod/knative/tabulator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-tabulator 6 | namespace: knative 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: tabulator 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: tabulator 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: tabulator 24 | spec: 25 | serviceAccountName: tabulator 26 | containers: 27 | - name: tabulator 28 | image: gcr.io/k8s-testgrid/tabulator:v20240917-v0.0.174-2-g70f42770 29 | ports: 30 | - name: metrics 31 | containerPort: 2112 32 | args: 33 | - --column-stats 34 | - --config=gs://knative-own-testgrid/config 35 | - --confirm 36 | - --json-logs 37 | - --persist-queue=gs://knative-own-testgrid/queue/tabulator.json 38 | - --pubsub=knative-tests/test-group-updates 39 | - --wait=1h 40 | resources: 41 | requests: 42 | cpu: "30" 43 | memory: "25G" 44 | limits: 45 | cpu: "40" 46 | memory: "50G" 47 | --- 48 | apiVersion: v1 49 | kind: ServiceAccount 50 | metadata: 51 | annotations: 52 | # Uses same as updater 53 | iam.gke.io/gcp-service-account: testgrid-updater@knative-tests.iam.gserviceaccount.com 54 | name: tabulator 55 | namespace: knative 56 | -------------------------------------------------------------------------------- /cluster/prod/knative/updater.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-updater 6 | namespace: knative 7 | labels: 8 | component: updater 9 | app: testgrid 10 | spec: 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: testgrid 15 | component: updater 16 | template: 17 | metadata: 18 | labels: 19 | component: updater 20 | app: testgrid 21 | spec: 22 | serviceAccountName: updater 23 | containers: 24 | - name: updater 25 | image: gcr.io/k8s-testgrid/updater:v20240917-v0.0.174-2-g70f42770 26 | args: 27 | - --build-timeout=10m 28 | - --config=gs://knative-own-testgrid/config 29 | - --confirm 30 | - --group-timeout=20m 31 | - --json-logs 32 | - --persist-queue=gs://knative-own-testgrid/queue/updater.json 33 | - --subscribe=knative-prow=knative-tests/testgrid 34 | - --wait=1h 35 | --- 36 | apiVersion: v1 37 | kind: ServiceAccount 38 | metadata: 39 | annotations: 40 | iam.gke.io/gcp-service-account: testgrid-updater@knative-tests.iam.gserviceaccount.com 41 | name: updater 42 | namespace: knative 43 | -------------------------------------------------------------------------------- /cluster/prod/monitoring.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: monitoring.gke.io/v1alpha1 3 | kind: PodMonitor 4 | metadata: 5 | labels: 6 | app: testgrid-metrics 7 | component: config-merger 8 | name: config-merger 9 | namespace: testgrid 10 | spec: 11 | podMetricsEndpoints: 12 | - interval: 30s 13 | port: metrics 14 | scheme: http 15 | namespaceSelector: 16 | matchNames: 17 | - testgrid 18 | selector: 19 | matchLabels: 20 | app: testgrid 21 | component: config-merger 22 | --- 23 | apiVersion: monitoring.gke.io/v1alpha1 24 | kind: PodMonitor 25 | metadata: 26 | labels: 27 | app: testgrid-metrics 28 | component: summarizer 29 | name: summarizer 30 | namespace: testgrid 31 | spec: 32 | podMetricsEndpoints: 33 | - interval: 30s 34 | port: metrics 35 | scheme: http 36 | namespaceSelector: 37 | matchNames: 38 | - testgrid 39 | selector: 40 | matchLabels: 41 | app: testgrid 42 | component: summarizer 43 | --- 44 | apiVersion: monitoring.gke.io/v1alpha1 45 | kind: PodMonitor 46 | metadata: 47 | labels: 48 | app: testgrid-metrics 49 | component: updater 50 | name: updater 51 | namespace: testgrid 52 | spec: 53 | podMetricsEndpoints: 54 | - interval: 30s 55 | port: metrics 56 | scheme: http 57 | namespaceSelector: 58 | matchNames: 59 | - testgrid 60 | selector: 61 | matchLabels: 62 | app: testgrid 63 | component: updater 64 | --- 65 | # These will be consumed by GKE Managed Prometheus(GMP) services in the cluster. 66 | # See: https://cloud.google.com/stackdriver/docs/managed-prometheus. 67 | apiVersion: monitoring.googleapis.com/v1 68 | kind: PodMonitoring 69 | metadata: 70 | labels: 71 | app: testgrid-metrics 72 | component: config-merger 73 | name: config-merger 74 | namespace: testgrid 75 | spec: 76 | endpoints: 77 | - interval: 30s 78 | port: metrics 79 | scheme: http 80 | selector: 81 | matchLabels: 82 | app: testgrid 83 | component: config-merger 84 | --- 85 | apiVersion: monitoring.googleapis.com/v1 86 | kind: PodMonitoring 87 | metadata: 88 | labels: 89 | app: testgrid-metrics 90 | component: summarizer 91 | name: summarizer 92 | namespace: testgrid 93 | spec: 94 | endpoints: 95 | - interval: 30s 96 | port: metrics 97 | scheme: http 98 | selector: 99 | matchLabels: 100 | app: testgrid 101 | component: summarizer 102 | --- 103 | apiVersion: monitoring.googleapis.com/v1 104 | kind: PodMonitoring 105 | metadata: 106 | labels: 107 | app: testgrid-metrics 108 | component: updater 109 | name: updater 110 | namespace: testgrid 111 | spec: 112 | endpoints: 113 | - interval: 30s 114 | port: metrics 115 | scheme: http 116 | selector: 117 | matchLabels: 118 | app: testgrid 119 | component: updater 120 | -------------------------------------------------------------------------------- /cluster/prod/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: testgrid 5 | -------------------------------------------------------------------------------- /cluster/prod/summarizer.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | annotations: 6 | # Uses same as updater 7 | iam.gke.io/gcp-service-account: updater@k8s-testgrid.iam.gserviceaccount.com 8 | name: summarizer 9 | namespace: testgrid 10 | --- 11 | apiVersion: apps/v1 12 | kind: Deployment 13 | metadata: 14 | name: testgrid-summarizer-tabs 15 | namespace: testgrid 16 | labels: 17 | app: testgrid 18 | component: summarizer-tabs 19 | spec: 20 | replicas: 1 21 | selector: 22 | matchLabels: 23 | app: testgrid 24 | component: summarizer-tabs 25 | template: 26 | metadata: 27 | labels: 28 | app: testgrid 29 | component: summarizer-tabs 30 | spec: 31 | serviceAccountName: summarizer 32 | containers: 33 | - name: summarizer 34 | image: gcr.io/k8s-testgrid/summarizer:v20240917-v0.0.174-2-g70f42770 35 | ports: 36 | - name: metrics 37 | containerPort: 2112 38 | args: 39 | - --config=gs://k8s-testgrid/config 40 | - --confirm 41 | - --json-logs 42 | - --persist-queue=gs://k8s-testgrid/queue/summarizer-tabs.json 43 | - --pubsub=k8s-testgrid/tab-updates 44 | - --wait=5m 45 | resources: 46 | requests: 47 | cpu: "1" 48 | memory: "50G" 49 | limits: 50 | cpu: "2" 51 | memory: "100G" 52 | --- 53 | -------------------------------------------------------------------------------- /cluster/prod/tabulator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: testgrid-tabulator 6 | namespace: testgrid 7 | labels: 8 | app: testgrid 9 | channel: stable 10 | component: tabulator 11 | spec: 12 | replicas: 1 13 | selector: 14 | matchLabels: 15 | app: testgrid 16 | channel: stable 17 | component: tabulator 18 | template: 19 | metadata: 20 | labels: 21 | app: testgrid 22 | channel: stable 23 | component: tabulator 24 | spec: 25 | serviceAccountName: tabulator 26 | containers: 27 | - name: tabulator 28 | image: gcr.io/k8s-testgrid/tabulator:v20240917-v0.0.174-2-g70f42770 29 | ports: 30 | - name: metrics 31 | containerPort: 2112 32 | args: 33 | - --column-stats 34 | - --config=gs://k8s-testgrid/config 35 | - --confirm 36 | - --json-logs 37 | - --persist-queue=gs://k8s-testgrid/queue/tabulator.json 38 | - --pubsub=k8s-testgrid/test-group-updates 39 | - --read-concurrency=10 40 | - --wait=1h 41 | - --write-concurrency=10 42 | resources: 43 | requests: 44 | cpu: "30" 45 | memory: "25G" 46 | limits: 47 | cpu: "40" 48 | memory: "50G" 49 | --- 50 | apiVersion: v1 51 | kind: ServiceAccount 52 | metadata: 53 | annotations: 54 | # Uses same as updater 55 | iam.gke.io/gcp-service-account: updater@k8s-testgrid.iam.gserviceaccount.com 56 | name: tabulator 57 | namespace: testgrid 58 | -------------------------------------------------------------------------------- /cluster/prod/updater.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | annotations: 6 | iam.gke.io/gcp-service-account: updater@k8s-testgrid.iam.gserviceaccount.com 7 | name: updater 8 | namespace: testgrid 9 | --- 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | metadata: 13 | name: testgrid-updater 14 | namespace: testgrid 15 | labels: 16 | component: updater 17 | app: testgrid 18 | spec: 19 | replicas: 1 # https://github.com/GoogleCloudPlatform/testgrid/issues/663 20 | selector: 21 | matchLabels: 22 | app: testgrid 23 | component: updater 24 | template: 25 | metadata: 26 | labels: 27 | component: updater 28 | app: testgrid 29 | spec: 30 | serviceAccountName: updater 31 | containers: 32 | - name: updater 33 | image: gcr.io/k8s-testgrid/updater:v20240917-v0.0.174-2-g70f42770 34 | ports: 35 | - name: metrics 36 | containerPort: 2112 37 | args: 38 | - --build-concurrency=10 39 | - --build-timeout=5m 40 | - --config=gs://k8s-testgrid/config 41 | - --confirm 42 | - --group-concurrency=10 43 | - --group-timeout=20m 44 | - --json-logs 45 | - --persist-queue=gs://k8s-testgrid/queue/updater.json 46 | - --subscribe=kubernetes-jenkins=kubernetes-jenkins/testgrid 47 | - --subscribe=oss-prow=k8s-testgrid/testgrid 48 | - --subscribe=istio-prow=k8s-testgrid/testgrid 49 | - --subscribe=kubernetes-ci-logs=k8s-testgrid/testgrid-kubernetes-ci-logs 50 | - --wait=2h 51 | resources: 52 | requests: 53 | cpu: "30" 54 | memory: "75G" 55 | limits: 56 | cpu: "40" 57 | memory: "150G" 58 | --- 59 | -------------------------------------------------------------------------------- /cluster/setup-pubsub.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -o nounset 18 | set -o errexit 19 | 20 | goog_buckets=( 21 | gs://oss-prow 22 | gs://istio-prow 23 | ) 24 | 25 | goog_topic=projects/k8s-testgrid/topics/prow-updates 26 | 27 | k8s_topic=projects/k8s-infra-prow/topics/kubernetes-ci-logs-updates 28 | 29 | k8s_buckets=( 30 | gs://kubernetes-ci-logs 31 | ) 32 | 33 | dir=$(dirname "$0") 34 | list=$dir/list-gcs-prefixes.sh 35 | create_topic=$dir/create-topic.sh 36 | create_sub=$dir/create-subscription.sh 37 | 38 | 39 | log() { 40 | ( 41 | set -o xtrace 42 | "$@" 43 | ) 44 | } 45 | 46 | apply-subscription() { 47 | topic=$1 48 | project=$2 49 | sub=$3 50 | canary=$4 51 | # Prod 52 | log "$create_sub" -t "$topic" \ 53 | -b serviceAccount:updater@k8s-testgrid.iam.gserviceaccount.com \ 54 | -p "$project" "$sub" 55 | # Canary 56 | log "$create_sub" -t "$topic" \ 57 | -b serviceAccount:testgrid-canary@k8s-testgrid.iam.gserviceaccount.com \ 58 | -p "$project" "$canary" 59 | } 60 | 61 | apply-topic() { 62 | topic=$1 63 | shift 64 | buckets=("$@") 65 | log "$create_topic" -t "$topic" -p logs/ -p pr-logs/ "${buckets[@]}" 66 | } 67 | 68 | do-list() { 69 | "$list" gs://k8s-testgrid-canary/config 70 | echo "NOTICE: edit this file ($(basename "$0")) to add any additional paths" >&2 71 | } 72 | 73 | something= 74 | while getopts "lst" flag; do 75 | case "$flag" in 76 | s) 77 | apply-subscription "$goog_topic" k8s-testgrid testgrid testgrid-canary 78 | apply-subscription "$k8s_topic" k8s-testgrid testgrid-kubernetes-ci-logs testgrid-canary-kubernetes-ci-logs 79 | something=yes 80 | ;; 81 | t) 82 | apply-topic "$goog_topic" "${goog_buckets[@]}" 83 | apply-topic "$k8s_topic" "${k8s_buckets[@]}" 84 | something=yes 85 | ;; 86 | l) 87 | do-list 88 | something=yes 89 | ;; 90 | esac 91 | done 92 | 93 | if [[ -z "$something" ]]; then 94 | echo "Usage: $(basename "$0") [-u] [-s] [-l]" >&2 95 | echo >&2 96 | echo " -l: list buckets in use" >&2 97 | echo " -t: configure topics for ${buckets[@]}" >&2 98 | echo " -s: configure subscriptions" >&2 99 | exit 1 100 | fi 101 | -------------------------------------------------------------------------------- /cluster/setup-testgrid-pubsub.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The TestGrid Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -o nounset 18 | set -o errexit 19 | 20 | 21 | dir=$(dirname "$0") 22 | create_topic=$dir/create-topic.sh 23 | create_sub=$dir/create-subscription.sh 24 | 25 | log() { 26 | ( 27 | set -o xtrace 28 | "$@" 29 | ) 30 | } 31 | 32 | 33 | apply() { 34 | log "$create_topic" -t "$topic" -p '' "$bucket" 35 | 36 | log "$create_sub" -t "$topic" -b "$bot" -p "$project" -f "hasPrefix(attributes.name, \"$group_prefix\")" "$group_sub" 37 | log "$create_sub" -t "$topic" -b "$bot" -p "$project" -f "hasPrefix(attributes.name, \"$tab_prefix\")" "$tab_sub" 38 | } 39 | 40 | project=k8s-testgrid 41 | 42 | bucket=gs://k8s-testgrid 43 | topic="projects/$project/topics/testgrid" 44 | group_sub=test-group-updates 45 | tab_sub=tab-updates 46 | group_prefix="grid/" 47 | tab_prefix="tabs/" 48 | bot=serviceAccount:updater@k8s-testgrid.iam.gserviceaccount.com 49 | apply 50 | 51 | 52 | bucket=gs://k8s-testgrid-canary 53 | topic="projects/$project/topics/canary-testgrid" 54 | group_sub=canary-test-group-updates 55 | tab_sub=canary-tab-updates 56 | group_prefix="grid/" 57 | tab_prefix="tabs/" 58 | bot=serviceAccount:testgrid-canary@k8s-testgrid.iam.gserviceaccount.com 59 | apply 60 | -------------------------------------------------------------------------------- /cluster/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | 18 | # Ensure namespace exists 19 | echo -n 'testgrid namespace: ' >&2 20 | kubectl get namespaces/testgrid &>/dev/null || kubectl create namespace testgrid 21 | echo 'PRESENT' >&2 22 | # Ensure secrets exists 23 | for i in github-service-account testgrid-service-account; do 24 | echo -n "${i} secret: " >&2 25 | kubectl get secrets/${i} --namespace=testgrid &>/dev/null \ 26 | && echo 'PRESENT' >&2 \ 27 | && continue 28 | echo 'MISSING' >&2 29 | echo 'Fix with the following:' >&2 30 | echo " kubectl create secret generic ${i} \\" >&2 31 | echo ' --from-file=service-account.json=PATH/TO/SERVICE-ACCOUNT.json' >&2 32 | exit 1 33 | done 34 | echo 'READY to deploy with the following command:' >&2 35 | #TODO(chases2): Make this run using Bazel 36 | #echo ' bazel run //cluster:dev.create' 37 | echo ' kubectl apply -f cluster/' 38 | -------------------------------------------------------------------------------- /cluster/testgrid-canary-autobump-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | gitHubLogin: "google-oss-robot" 3 | gitHubToken: "/etc/github-token/oauth" 4 | onCallAddress: "https://storage.googleapis.com/kubernetes-jenkins/oncall.json" 5 | skipPullRequest: false 6 | skipOncallAssignment: true 7 | gitHubOrg: "GoogleCloudPlatform" 8 | gitHubRepo: "testgrid" 9 | remoteName: "testgrid" 10 | headBranchName: "autobump-canary" 11 | upstreamURLBase: "https://raw.githubusercontent.com/GoogleCloudPlatform/testgrid/master" 12 | includedConfigPaths: 13 | - "cluster/canary" 14 | targetVersion: "latest" 15 | prefixes: 16 | - name: "Testgrid Canary" 17 | prefix: "gcr.io/k8s-testgrid/" 18 | repo: "https://github.com/GoogleCloudPlatform/testgrid" 19 | summarise: true 20 | consistentImages: true 21 | -------------------------------------------------------------------------------- /cluster/testgrid-prod-autobump-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | gitHubLogin: "google-oss-robot" 3 | gitHubToken: "/etc/github-token/oauth" 4 | onCallAddress: "https://storage.googleapis.com/kubernetes-jenkins/oncall.json" 5 | skipPullRequest: false 6 | skipOncallAssignment: true 7 | gitHubOrg: "GoogleCloudPlatform" 8 | gitHubRepo: "testgrid" 9 | remoteName: "testgrid" 10 | headBranchName: "autobump-prod" 11 | upstreamURLBase: "https://raw.githubusercontent.com/GoogleCloudPlatform/testgrid/master" 12 | includedConfigPaths: 13 | - "cluster/prod" 14 | targetVersion: "upstream" 15 | prefixes: 16 | - name: "Testgrid Prod" 17 | prefix: "gcr.io/k8s-testgrid/" 18 | refConfigFile: "cluster/canary/summarizer.yaml" 19 | repo: "https://github.com/GoogleCloudPlatform/testgrid" 20 | summarise: true 21 | consistentImages: true 22 | -------------------------------------------------------------------------------- /cmd/README.md: -------------------------------------------------------------------------------- 1 | # TestGrid Components 2 | 3 | The `cmd` directory contains the main files for TestGrid components, and documentation for how to 4 | run them. For specifics on a particular component, see the subdirectory for that component. 5 | 6 | ## Tips for running locally 7 | 8 | ### Google Cloud Storage (GCS) Authentication 9 | 10 | If you're using files from GCS and you hit an authentication or 'unauthorized' error in logs, run: 11 | ```gcloud auth application-default login``` 12 | 13 | Most components will automatically find and use the credentials generated from this. 14 | 15 | ### Common Flags 16 | 17 | Most components have similar flags and defaults. Here's a few you can expect to use: 18 | - Run `bazelisk run //cmd/$COMPONENT -- --help` for full flag list and descriptions. 19 | - `--config` specifies the path to a valid [config proto]. It can take a: 20 | - local file: e.g. `/tmp/testgrid/my-config.pb` 21 | - GCS (Google Cloud Storage) file: e.g. `gs://my-bucket/my-config.pb` 22 | - `--confirm`, if true, writes data to the same base path as your `--config`. 23 | - Without this, it's "dry-run` by default and doesn't write data anywhere. 24 | - `--wait`, if true, runs the component continuously until force-quit, waiting the specified time 25 | before beginning another cycle. 26 | - Without this, it will run once and exit. 27 | - `--debug` or `--trace` print debug-level or trace-level logs, respectively. 28 | - Without this, only logs at info-level or higher will be displayed. 29 | - Debug logs tend to be useful for development or debugging. 30 | - Trace logs can be _very_ noisy, and tend to be useful for debugging something very specific. 31 | 32 | ### Developing and Testing 33 | 34 | Most of the TestGrid libraries used for these components live under the `pkg/` directory. 35 | - e.g. `cmd/updater` library code is under `pkg/updater` 36 | 37 | To run unit tests for a particular component, run: 38 | 39 | ```shell 40 | COMPONENT={component} # e.g. 'updater', 'summarizer', etc. 41 | bazelisk test //cmd/$COMPONENT/... # Runs unit tests for the main/binary, like flag-parsing or default arguments 42 | bazelisk test //pkg/$COMPONENT/... # Runs unit tests for library code 43 | ``` -------------------------------------------------------------------------------- /cmd/api/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("//:def.bzl", "go_image") 3 | 4 | go_image( 5 | name = "image", 6 | directory = "/", 7 | files = [":api"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_library( 12 | name = "go_default_library", 13 | srcs = ["main.go"], 14 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/api", 15 | visibility = ["//visibility:private"], 16 | deps = [ 17 | "//pkg/api:go_default_library", 18 | "@com_github_sirupsen_logrus//:go_default_library", 19 | ], 20 | ) 21 | 22 | go_binary( 23 | name = "api", 24 | embed = [":go_default_library"], 25 | visibility = ["//visibility:public"], 26 | ) 27 | 28 | filegroup( 29 | name = "package-srcs", 30 | srcs = glob(["**"]), 31 | tags = ["automanaged"], 32 | visibility = ["//visibility:private"], 33 | ) 34 | 35 | filegroup( 36 | name = "all-srcs", 37 | srcs = [":package-srcs"], 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:public"], 40 | ) 41 | -------------------------------------------------------------------------------- /cmd/api/README.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | This component exposes TestGrid data publicly that could otherwise be viewed through the UI. 4 | 5 | ## Local development 6 | See also [common tips](/cmd/README.md) for running locally. 7 | 8 | The surface of the API is described in this (proto definition)[/pb/api/v1/data.proto]. Usage is similar between protocols. 9 | 10 | You may want to set `--scope=gs://your-bucket`. This will set this as the server's default; otherwise you'll be required to specify with each call. 11 | 12 | If you're using this for developing the UI (https://github.com/kubernetes-sigs/testgrid), optionally set `--allowed-origin=*` to avoid CORS issues. 13 | 14 | ```bash 15 | bazelisk run //cmd/api -- \ 16 | # --scope=gs://your-bucket 17 | # --allowed-origin=* 18 | ``` 19 | 20 | ### HTTP 21 | 22 | Use the `--http-port` option to set the listening port. Default is 8080. 23 | 24 | You can specify further with URL parameters: `?scope=gs://your-bucket`. 25 | 26 | `curl localhost:8080/api/v1/dashboards` 27 | 28 | ### gRPC 29 | 30 | Use the `--grpc-port` option to set the listening port. Default is 50051. 31 | 32 | `grpc_cli call localhost:50051 testgrid.api.v1.TestGridData.ListDashboard` 33 | 34 | Reflection is enabled in gRPC, allowing you to ask the server what methods are available. 35 | 36 | `grpc_cli ls localhost:50051 testgrid.api.v1.TestGridData` 37 | -------------------------------------------------------------------------------- /cmd/config_merger/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("//:def.bzl", "go_image") 3 | 4 | go_image( 5 | name = "image", 6 | directory = "/", 7 | files = [":config_merger"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_binary( 12 | name = "config_merger", 13 | embed = [":go_default_library"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | 17 | go_library( 18 | name = "go_default_library", 19 | srcs = ["main.go"], 20 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/config_merger", 21 | visibility = ["//visibility:public"], 22 | deps = [ 23 | "//pkg/merger:go_default_library", 24 | "//util/gcs:go_default_library", 25 | "//util/metrics/prometheus:go_default_library", 26 | "@com_github_sirupsen_logrus//:go_default_library", 27 | ], 28 | ) 29 | 30 | filegroup( 31 | name = "package-srcs", 32 | srcs = glob(["**"]), 33 | tags = ["automanaged"], 34 | visibility = ["//visibility:private"], 35 | ) 36 | 37 | filegroup( 38 | name = "all-srcs", 39 | srcs = [":package-srcs"], 40 | tags = ["automanaged"], 41 | visibility = ["//visibility:public"], 42 | ) 43 | -------------------------------------------------------------------------------- /cmd/config_merger/README.md: -------------------------------------------------------------------------------- 1 | # Config Merger 2 | This component validates and combines two or more [config proto] files into a single TestGrid 3 | configuration (also a config proto). This config is used by basically every other component in TestGrid. 4 | 5 | ## Local development 6 | See also [common tips](/cmd/README.md) for running locally. 7 | 8 | ```bash 9 | --config-list=/tmp/testgrid/my-config-list.yaml \ # See 'Configuration List' below for details. 10 | # --confirm \ 11 | ``` 12 | 13 | ## Configuration List 14 | The config merger requires a YAML file containing: 15 | - a list of the configurations its trying to merge 16 | - a location to put the final configuration 17 | 18 | The `--config-list` flag should point to a file like this: 19 | 20 | ```yaml 21 | target: "gs://path/to/write/config" # Final result goes here 22 | sources: 23 | - name: "red" # Used in renaming 24 | location: "gs://example/red-team/config" 25 | contact: "red-admin@example.com" # Used for cross-team communication, not yet by config_merger 26 | - name: "blue" 27 | location: "gs://example/blue-team/config" 28 | contact: "blue.team.contact@example.com" 29 | ``` 30 | 31 | ### Renaming 32 | Test Groups, Dashboards, and Dashboard Groups may be renamed to prevent 33 | duplicates in the final config. In this case, the `name` in the config list 34 | is added as a prefix, giving precedence by alphabetical order. 35 | 36 | For example, if both configurations in the example above contain a dashboard 37 | named `"foo"`, the red dashboard will be renamed to `"red-foo"`. 38 | 39 | [config proto]: /pb/config/config.proto -------------------------------------------------------------------------------- /cmd/state_comparer/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") 2 | 3 | go_binary( 4 | name = "state_comparer", 5 | embed = [":go_default_library"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/compare_states", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | go_library( 11 | name = "go_default_library", 12 | srcs = ["main.go"], 13 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/state_comparer", 14 | visibility = ["//visibility:private"], 15 | deps = [ 16 | "//config:go_default_library", 17 | "//pb/state:go_default_library", 18 | "//util/gcs:go_default_library", 19 | "@com_github_google_go_cmp//cmp:go_default_library", 20 | "@com_github_sirupsen_logrus//:go_default_library", 21 | "@org_golang_google_api//iterator:go_default_library", 22 | ], 23 | ) 24 | 25 | go_test( 26 | name = "go_default_test", 27 | srcs = ["main_test.go"], 28 | embed = [":go_default_library"], 29 | deps = [ 30 | "//pb/state:go_default_library", 31 | "//util/gcs:go_default_library", 32 | ], 33 | ) 34 | 35 | filegroup( 36 | name = "package-srcs", 37 | srcs = glob(["**"]), 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:private"], 40 | ) 41 | 42 | filegroup( 43 | name = "all-srcs", 44 | srcs = [":package-srcs"], 45 | tags = ["automanaged"], 46 | visibility = ["//visibility:public"], 47 | ) 48 | -------------------------------------------------------------------------------- /cmd/state_comparer/README.md: -------------------------------------------------------------------------------- 1 | # Compare States 2 | Compares TestGrid states of the same names between two directories. Useful for validating that 3 | changes to the updater or other code don't unexpectedly change the resulting state protos. 4 | 5 | ## Running 6 | You should have two directories set up: one each for the states you want to compare (e.g. "/tmp/cmp/first" and "/tmp/cmp/second") 7 | 8 | ```shell 9 | # Example basic run: 10 | bazelisk run //cmd/state_comparer -- --first="/tmp/cmp/first/" --second="/tmp/cmp/second/" --diff-ratio-ok=0.3 11 | 12 | # Example detailed/advanced run: 13 | bazelisk run //cmd/state_comparer -- --first="/tmp/tgcmp/first/" --second="/tmp/tgcmp/second/" --diff-ratio-ok=0.3 --test-group-url="http://testgrid-canary/q/testgroup/" --config="/tmp/cmp/config" --debug 14 | ``` -------------------------------------------------------------------------------- /cmd/summarizer/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("//:def.bzl", "go_image") 3 | 4 | go_image( 5 | name = "image", 6 | directory = "/", 7 | files = [":summarizer"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_binary( 12 | name = "summarizer", 13 | embed = [":go_default_library"], 14 | visibility = ["//visibility:public"], 15 | ) 16 | 17 | go_library( 18 | name = "go_default_library", 19 | srcs = ["main.go"], 20 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/summarizer", 21 | visibility = ["//visibility:private"], 22 | deps = [ 23 | "//pkg/pubsub:go_default_library", 24 | "//pkg/summarizer:go_default_library", 25 | "//util:go_default_library", 26 | "//util/gcs:go_default_library", 27 | "//util/metrics/prometheus:go_default_library", 28 | "@com_github_sirupsen_logrus//:go_default_library", 29 | "@com_google_cloud_go_pubsub//:go_default_library", 30 | "@org_golang_google_api//option:go_default_library", 31 | ], 32 | ) 33 | 34 | filegroup( 35 | name = "package-srcs", 36 | srcs = glob(["**"]), 37 | tags = ["automanaged"], 38 | visibility = ["//visibility:private"], 39 | ) 40 | 41 | filegroup( 42 | name = "all-srcs", 43 | srcs = [":package-srcs"], 44 | tags = ["automanaged"], 45 | visibility = ["//visibility:public"], 46 | ) 47 | -------------------------------------------------------------------------------- /cmd/summarizer/README.md: -------------------------------------------------------------------------------- 1 | # Summarizer 2 | 3 | This component reads dashboard-tab-based [state proto] files and summarizes them into a [summary proto]. 4 | 5 | These protos are read by the frontend (e.g. https://testgrid.k8s.io) 6 | 7 | ## Local development 8 | See also [common tips](/cmd/README.md) for running locally. 9 | 10 | ```bash 11 | # --config can take a local file (e.g. `/tmp/testgrid/config`) or GCS file (e.g. `gs://my-testgrid-bucket/config`) 12 | bazelisk run //cmd/summarizer -- \ 13 | --config=gs://my-testgrid-bucket/somewhere/config \ 14 | # --dashboards=foo,bar \ # If specified, only summarize these dashboards. 15 | # --debug \ 16 | # --confirm \ 17 | ``` 18 | 19 | [summary proto]: pb/summary/summary.proto 20 | -------------------------------------------------------------------------------- /cmd/tabulator/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | load("//:def.bzl", "go_image") 3 | 4 | go_image( 5 | name = "image", 6 | directory = "/", 7 | files = [":tabulator"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_library( 12 | name = "go_default_library", 13 | srcs = ["main.go"], 14 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/tabulator", 15 | visibility = ["//visibility:private"], 16 | deps = [ 17 | "//pkg/pubsub:go_default_library", 18 | "//pkg/tabulator:go_default_library", 19 | "//util:go_default_library", 20 | "//util/gcs:go_default_library", 21 | "//util/metrics/prometheus:go_default_library", 22 | "@com_github_sirupsen_logrus//:go_default_library", 23 | "@com_google_cloud_go_pubsub//:go_default_library", 24 | "@org_golang_google_api//option:go_default_library", 25 | ], 26 | ) 27 | 28 | go_binary( 29 | name = "tabulator", 30 | embed = [":go_default_library"], 31 | visibility = ["//visibility:public"], 32 | ) 33 | 34 | filegroup( 35 | name = "package-srcs", 36 | srcs = glob(["**"]), 37 | tags = ["automanaged"], 38 | visibility = ["//visibility:private"], 39 | ) 40 | 41 | filegroup( 42 | name = "all-srcs", 43 | srcs = [":package-srcs"], 44 | tags = ["automanaged"], 45 | visibility = ["//visibility:public"], 46 | ) 47 | -------------------------------------------------------------------------------- /cmd/tabulator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Tabulator 3 | 4 | This component reads test-group-based [state proto] files, combines them with dashboard tab 5 | configuration, and produces tab-based [state proto] files. 6 | 7 | The TestGrid frontend reads these protos and converts them to JSON, which the 8 | javascript UI reads and renders on the screen. 9 | 10 | These protos are read by: 11 | - The frontend (e.g. [testgrid.k8s.io](https://testgrid.k8s.io)) 12 | - The [Summarizer] component 13 | 14 | ## Local development 15 | See also [common tips](/cmd/README.md) for running locally. 16 | 17 | ```bash 18 | # --config can take a local file (e.g. `/tmp/testgrid/config`) or GCS file (e.g. `gs://my-testgrid-bucket/config`) 19 | bazelisk run //cmd/tabulator -- \ 20 | --config=gs://my-testgrid-bucket/somewhere/config \ 21 | # --groups=foo,bar \ # If specified, only tabulate these test groups. 22 | # --debug \ 23 | # --confirm \ 24 | ``` 25 | 26 | [state proto]: /pb/state/state.proto 27 | [Summarizer]: /cmd/summarizer -------------------------------------------------------------------------------- /cmd/updater/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") 2 | load("//:def.bzl", "go_image") 3 | 4 | go_image( 5 | name = "image", 6 | directory = "/", 7 | files = [":updater"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_binary( 12 | name = "updater", 13 | embed = [":go_default_library"], 14 | pure = "on", 15 | visibility = ["//visibility:public"], 16 | ) 17 | 18 | go_library( 19 | name = "go_default_library", 20 | srcs = ["main.go"], 21 | importpath = "github.com/GoogleCloudPlatform/testgrid/cmd/updater", 22 | visibility = ["//visibility:private"], 23 | deps = [ 24 | "//pb/config:go_default_library", 25 | "//pkg/pubsub:go_default_library", 26 | "//pkg/updater:go_default_library", 27 | "//pkg/updater/resultstore:go_default_library", 28 | "//util:go_default_library", 29 | "//util/gcs:go_default_library", 30 | "//util/metrics/prometheus:go_default_library", 31 | "@com_github_sirupsen_logrus//:go_default_library", 32 | "@com_google_cloud_go_pubsub//:go_default_library", 33 | "@org_golang_google_api//option:go_default_library", 34 | ], 35 | ) 36 | 37 | filegroup( 38 | name = "package-srcs", 39 | srcs = glob(["**"]), 40 | tags = ["automanaged"], 41 | visibility = ["//visibility:private"], 42 | ) 43 | 44 | filegroup( 45 | name = "all-srcs", 46 | srcs = [":package-srcs"], 47 | tags = ["automanaged"], 48 | visibility = ["//visibility:public"], 49 | ) 50 | 51 | go_test( 52 | name = "go_default_test", 53 | srcs = ["main_test.go"], 54 | embed = [":go_default_library"], 55 | deps = [ 56 | "//pb/config:go_default_library", 57 | "//util:go_default_library", 58 | "//util/gcs:go_default_library", 59 | "@com_github_google_go_cmp//cmp:go_default_library", 60 | ], 61 | ) 62 | -------------------------------------------------------------------------------- /cmd/updater/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Testgrid Updater 3 | 4 | This component is responsible for compiling collated GCS results into a [state proto]. 5 | 6 | These protos are read by the [Tabulator] component. 7 | 8 | ## Local development 9 | See also [common tips](/cmd/README.md) for running locally. 10 | 11 | ```bash 12 | # --config can take a local file (e.g. `/tmp/testgrid/config`) or GCS file (e.g. `gs://my-testgrid-bucket/config`) 13 | bazelisk run //cmd/updater -- \ 14 | --config=gs://my-testgrid-bucket/somewhere/config \ 15 | # --test-group=foo,bar \ # If specified, only update these test groups. 16 | # --debug \ 17 | # --confirm \ 18 | ``` 19 | 20 | ### Debugging 21 | 22 | The two most useful flags are `--test-group=foo` and `--confirm=false` (default). 23 | 24 | Setting the `--test-group` flag tells the client to only update a specific group. 25 | This is useful when debugging problems in a specific group. 26 | 27 | The `--confirm` flag controls whether any data is written. Nothing is written by default. 28 | 29 | 30 | ## Update cycles 31 | 32 | Each update cycle the updater: 33 | 34 | * Downloads the specified config proto to get the list of test groups. 35 | * Iterates through each group 36 | - Downloads the existing state proto if present 37 | * Drops the oldest and newest columns 38 | * Old ones are no longer relevant 39 | * New columns may still change 40 | - Grabs the gcs prefix 41 | - Scans GCS under that prefix for results greater than existing ones 42 | * Each job is in a unique GCS\_PREFIX/JOB\_ID folder 43 | * New folders must be monotonically greater than old ones 44 | - Compiles the job result in each folder into a cell mapping 45 | - Converts the cell into the existing state grid proto. 46 | * Appends a new column into the state grid. 47 | * Creates any new rows. 48 | * Appends data to existing rows. 49 | * Determines which (if any) rows have alerts 50 | * Optionally uploads the proto to GCS 51 | 52 | If the `--wait` flag is unset, the job returns at this time. 53 | 54 | Otherwise it repeats after sleeping for that duration. 55 | 56 | [state proto]: /pb/state/state.proto 57 | [Tabulator]: /cmd/tabulator -------------------------------------------------------------------------------- /config/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "cache.go", 7 | "config.go", 8 | "converge.go", 9 | "fields.go", 10 | "queue.go", 11 | ], 12 | importpath = "github.com/GoogleCloudPlatform/testgrid/config", 13 | visibility = ["//visibility:public"], 14 | deps = [ 15 | "//pb/config:go_default_library", 16 | "//pkg/updater/resultstore/query:go_default_library", 17 | "//util/gcs:go_default_library", 18 | "//util/queue:go_default_library", 19 | "@com_github_golang_protobuf//proto:go_default_library", 20 | "@com_github_hashicorp_go_multierror//:go_default_library", 21 | "@com_github_sirupsen_logrus//:go_default_library", 22 | "@com_google_cloud_go_storage//:go_default_library", 23 | "@org_bitbucket_creachadair_stringset//:go_default_library", 24 | "@org_golang_google_protobuf//reflect/protoreflect:go_default_library", 25 | ], 26 | ) 27 | 28 | filegroup( 29 | name = "package-srcs", 30 | srcs = glob(["**"]), 31 | tags = ["automanaged"], 32 | visibility = ["//visibility:private"], 33 | ) 34 | 35 | filegroup( 36 | name = "all-srcs", 37 | srcs = [ 38 | ":package-srcs", 39 | "//config/print:all-srcs", 40 | "//config/snapshot:all-srcs", 41 | "//config/yamlcfg:all-srcs", 42 | ], 43 | tags = ["automanaged"], 44 | visibility = ["//visibility:public"], 45 | ) 46 | 47 | go_test( 48 | name = "go_default_test", 49 | srcs = [ 50 | "cache_test.go", 51 | "config_test.go", 52 | "converge_test.go", 53 | "fields_test.go", 54 | "queue_test.go", 55 | ], 56 | embed = [":go_default_library"], 57 | deps = [ 58 | "//pb/config:go_default_library", 59 | "//util/gcs:go_default_library", 60 | "//util/gcs/fake:go_default_library", 61 | "@com_github_golang_protobuf//proto:go_default_library", 62 | "@com_github_google_go_cmp//cmp:go_default_library", 63 | "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", 64 | "@com_github_hashicorp_go_multierror//:go_default_library", 65 | "@com_github_sirupsen_logrus//:go_default_library", 66 | "@com_google_cloud_go_storage//:go_default_library", 67 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 68 | ], 69 | ) 70 | 71 | # for repo-infra hack 72 | platform( 73 | name = "platform", 74 | constraint_values = [ 75 | "@bazel_tools//platforms:linux", 76 | "@bazel_tools//platforms:x86_64", 77 | "@bazel_tools//tools/cpp:clang", 78 | ], 79 | parents = ["@local_config_platform//:host"], 80 | visibility = ["//visibility:public"], 81 | ) 82 | -------------------------------------------------------------------------------- /config/fields.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "fmt" 21 | 22 | configpb "github.com/GoogleCloudPlatform/testgrid/pb/config" 23 | "google.golang.org/protobuf/reflect/protoreflect" 24 | ) 25 | 26 | type protoPath struct { 27 | name string 28 | count map[string]int64 29 | } 30 | 31 | func (pp protoPath) countRecursive(fd protoreflect.FieldDescriptor, val protoreflect.Value) bool { 32 | if pp.name != "" { 33 | pp.name = fmt.Sprintf("%s.%s", pp.name, fd.Name()) 34 | } else { 35 | pp.name = string(fd.Name()) 36 | } 37 | 38 | if fd.Kind() == protoreflect.MessageKind { 39 | if fd.IsList() { 40 | for i := 0; i < val.List().Len(); i++ { 41 | val.List().Get(i).Message().Range(pp.countRecursive) 42 | } 43 | return true 44 | } 45 | // Does not support maps of messages; there aren't any in config.proto 46 | val.Message().Range(pp.countRecursive) 47 | return true 48 | } 49 | pp.count[pp.name]++ 50 | return true 51 | } 52 | 53 | // Fields returns counts of all of the primitive fields in this configuration 54 | // 55 | // Field names in the map are qualified with where they are nested; this is not the same as a message's "FullName" 56 | // Ex: A dashboard tab's name is "dashboards.dashboard_tab.name" 57 | func Fields(cfg *configpb.Configuration) map[string]int64 { 58 | pp := protoPath{ 59 | count: map[string]int64{}, 60 | } 61 | 62 | cfg.ProtoReflect().Range(pp.countRecursive) 63 | return pp.count 64 | } 65 | -------------------------------------------------------------------------------- /config/print/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["main.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/config/print", 7 | visibility = ["//visibility:private"], 8 | deps = [ 9 | "//config:go_default_library", 10 | "//util/gcs:go_default_library", 11 | "@com_github_sirupsen_logrus//:go_default_library", 12 | ], 13 | ) 14 | 15 | go_binary( 16 | name = "print", 17 | embed = [":go_default_library"], 18 | visibility = ["//visibility:public"], 19 | ) 20 | 21 | filegroup( 22 | name = "package-srcs", 23 | srcs = glob(["**"]), 24 | tags = ["automanaged"], 25 | visibility = ["//visibility:private"], 26 | ) 27 | 28 | filegroup( 29 | name = "all-srcs", 30 | srcs = [":package-srcs"], 31 | tags = ["automanaged"], 32 | visibility = ["//visibility:public"], 33 | ) 34 | -------------------------------------------------------------------------------- /config/print/README.md: -------------------------------------------------------------------------------- 1 | # Config Printer 2 | 3 | The config printer is a debugging utility that prints a TestGrid configuration in a 4 | human-readable format. It will read from the local filesystem or Google Cloud Storage. 5 | 6 | ## Usage and installation 7 | 8 | The tool can be built and run with Bazel. 9 | 10 | ```sh 11 | bazel run //config/print -- gs://example/config 12 | ``` 13 | 14 | The tool can be installed via go install. You may want to rename the 15 | resulting binary so it doesn't shadow your shell's `print` utility. 16 | 17 | ```sh 18 | go install ./config/print 19 | print gs://example/config 20 | ``` 21 | -------------------------------------------------------------------------------- /config/snapshot/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["config_snapshot.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/config/snapshot", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//config:go_default_library", 10 | "//pb/config:go_default_library", 11 | "//util/gcs:go_default_library", 12 | "@com_github_sethvargo_go_retry//:go_default_library", 13 | "@com_github_sirupsen_logrus//:go_default_library", 14 | "@com_google_cloud_go_storage//:go_default_library", 15 | ], 16 | ) 17 | 18 | go_test( 19 | name = "go_default_test", 20 | srcs = ["config_snapshot_test.go"], 21 | embed = [":go_default_library"], 22 | deps = [ 23 | "//pb/config:go_default_library", 24 | "//util/gcs:go_default_library", 25 | "//util/gcs/fake:go_default_library", 26 | "@com_github_google_go_cmp//cmp:go_default_library", 27 | "@com_google_cloud_go_storage//:go_default_library", 28 | "@org_golang_google_protobuf//proto:go_default_library", 29 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 30 | ], 31 | ) 32 | 33 | filegroup( 34 | name = "package-srcs", 35 | srcs = glob(["**"]), 36 | tags = ["automanaged"], 37 | visibility = ["//visibility:private"], 38 | ) 39 | 40 | filegroup( 41 | name = "all-srcs", 42 | srcs = [":package-srcs"], 43 | tags = ["automanaged"], 44 | visibility = ["//visibility:public"], 45 | ) 46 | -------------------------------------------------------------------------------- /config/yamlcfg/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["yaml2proto.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/config/yamlcfg", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//config:go_default_library", 10 | "//pb/config:go_default_library", 11 | "@io_k8s_sigs_yaml//:go_default_library", 12 | ], 13 | ) 14 | 15 | go_test( 16 | name = "go_default_test", 17 | srcs = ["yaml2proto_test.go"], 18 | embed = [":go_default_library"], 19 | deps = [ 20 | "//pb/config:go_default_library", 21 | "@com_github_google_go_cmp//cmp:go_default_library", 22 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 23 | ], 24 | ) 25 | 26 | filegroup( 27 | name = "package-srcs", 28 | srcs = glob(["**"]), 29 | tags = ["automanaged"], 30 | visibility = ["//visibility:private"], 31 | ) 32 | 33 | filegroup( 34 | name = "all-srcs", 35 | srcs = [":package-srcs"], 36 | tags = ["automanaged"], 37 | visibility = ["//visibility:public"], 38 | ) 39 | -------------------------------------------------------------------------------- /def.bzl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@io_bazel_rules_docker//container:image.bzl", "container_image") 16 | load("@io_bazel_rules_docker//container:bundle.bzl", "container_bundle") 17 | load("@io_bazel_rules_docker//contrib:push-all.bzl", "container_push") 18 | load("@io_bazel_rules_docker//go:image.bzl", _go_image = "go_image") 19 | 20 | ## make_image is a macro for creating :app and :image targets 21 | def go_image( 22 | name, # use "image" 23 | base = None, 24 | stamp = "@io_bazel_rules_docker//stamp:always", # stamp by default, but allow overrides 25 | app_name = "app", 26 | **kwargs): 27 | _go_image( 28 | name = app_name, 29 | base = base, 30 | embed = [":go_default_library"], 31 | goarch = "amd64", 32 | goos = "linux", 33 | pure = "on", 34 | ) 35 | 36 | container_image( 37 | name = name, 38 | base = ":" + app_name, 39 | stamp = stamp, 40 | **kwargs 41 | ) 42 | 43 | # push_image creates a bundle of container images, and a target to push them. 44 | def push_image( 45 | name, 46 | bundle_name = "bundle", 47 | images = None): 48 | container_bundle( 49 | name = bundle_name, 50 | images = images, 51 | ) 52 | container_push( 53 | name = name, 54 | bundle = ":" + bundle_name, 55 | format = "Docker", # TODO(fejta): consider OCI? 56 | ) 57 | 58 | # tags appends default tags to name 59 | # 60 | # In particular, names is a {image_prefix: image_target} mapping, which gets 61 | # expanded into three full image paths: 62 | # image_prefix:latest 63 | # image_prefix:latest-{BUILD_USER} 64 | # image_prefix:{DOCKER_TAG} 65 | # (See hack/print-workspace-status.sh for how BUILD_USER and DOCKER_TAG are created. 66 | def tags(targets): 67 | outs = {} 68 | for img, target in targets.items(): 69 | outs["%s:{DOCKER_TAG}" % img] = target 70 | outs["%s:latest-{BUILD_USER}" % img] = target 71 | outs["%s:latest" % img] = target 72 | return outs 73 | -------------------------------------------------------------------------------- /extension/testgrid_alerter/css/testgrid_alerter.css: -------------------------------------------------------------------------------- 1 | .small-popup { 2 | min-width: 500px; 3 | } 4 | 5 | .right-button { 6 | float:right; 7 | } 8 | 9 | 10 | .bar { 11 | overflow: auto; 12 | padding: 5px; 13 | width: 100%; 14 | } 15 | 16 | body { 17 | margin: 10px; 18 | } 19 | 20 | .dashboard-div { 21 | border: 2px solid black; 22 | padding-left: 5px; 23 | } 24 | 25 | .tab-ul { 26 | padding: 0 5px 5px 20px; 27 | } 28 | 29 | .test-li { 30 | margin: 0 0 5px 0; 31 | } 32 | 33 | .dashboard { 34 | font-weight: bold; 35 | } 36 | 37 | .failing { 38 | color: red; 39 | } 40 | 41 | .stale { 42 | color: orange; 43 | } 44 | 45 | .collapsed { 46 | display: none; 47 | } 48 | 49 | .line { 50 | display: inline-block; 51 | } 52 | -------------------------------------------------------------------------------- /extension/testgrid_alerter/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/testgrid/6494ec10fd753b42d7046fe1a01209b0b9c3d62c/extension/testgrid_alerter/images/icon.png -------------------------------------------------------------------------------- /extension/testgrid_alerter/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 2, 3 | 4 | "name": "TestGrid Alerter", 5 | "description": "This extension shows TestGrid alerts.", 6 | "version": "1.0", 7 | 8 | "browser_action": { 9 | "default_icon": "images/icon.png", 10 | "default_popup": "static/popup.html" 11 | }, 12 | "permissions": [ 13 | "https://ajax.googleapis.com/", 14 | "https://testgrid.k8s.io/*", 15 | "storage" 16 | ], 17 | "background": { 18 | "scripts": ["js/updater.js"] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /extension/testgrid_alerter/static/popup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | TestGrid Extension 5 | 6 | 7 | 8 | 9 |
10 | 11 |
12 |

Alerts

13 |
14 | 15 | 16 | 17 | 18 |
19 |

20 |
21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /extension/testgrid_alerter/static/settings.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | TestGrid Extension Settings 5 | 6 | 7 | 8 | 9 |
10 | 11 | 12 |
13 |

Settings

14 |

Select Dashboards

15 | 17 |

Select URLs

18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/GoogleCloudPlatform/testgrid 2 | 3 | require ( 4 | bitbucket.org/creachadair/stringset v0.0.11 5 | cloud.google.com/go v0.110.7 // indirect 6 | cloud.google.com/go/pubsub v1.33.0 7 | cloud.google.com/go/storage v1.31.0 8 | github.com/client9/misspell v0.3.4 9 | github.com/fvbommel/sortorder v1.1.0 10 | github.com/go-chi/chi v1.5.4 11 | github.com/go-logr/logr v1.2.4 // indirect 12 | github.com/golang/protobuf v1.5.3 13 | github.com/google/go-cmp v0.5.9 14 | github.com/google/gofuzz v1.2.0 // indirect 15 | github.com/google/uuid v1.3.0 16 | github.com/hashicorp/errwrap v1.1.0 // indirect 17 | github.com/hashicorp/go-multierror v1.1.1 18 | github.com/prometheus/client_golang v1.11.1 19 | github.com/prometheus/client_model v0.3.0 20 | github.com/sethvargo/go-retry v0.2.4 21 | github.com/sirupsen/logrus v1.9.3 22 | google.golang.org/api v0.134.0 23 | google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf 24 | google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect 25 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect 26 | google.golang.org/grpc v1.57.0 27 | google.golang.org/protobuf v1.31.0 28 | k8s.io/api v0.27.4 29 | k8s.io/klog/v2 v2.100.1 // indirect 30 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 31 | sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect 32 | sigs.k8s.io/yaml v1.4.0 33 | ) 34 | 35 | go 1.14 36 | -------------------------------------------------------------------------------- /hack/autodeps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o nounset 17 | set -o errexit 18 | set -o pipefail 19 | 20 | cd $(git rev-parse --show-toplevel) 21 | 22 | if [[ $# -lt 2 ]]; then 23 | echo "Usage: $(basename "$0") [git-name] [git-email] [--patch|--minor]" >&2 24 | exit 1 25 | fi 26 | user=$1 27 | token=$2 28 | shift 2 29 | if [[ $# -ge 2 ]]; then 30 | echo "git config user.name=$1 user.email=$2..." >&2 31 | git config user.name "$1" 32 | git config user.email "$2" 33 | shift 2 34 | fi 35 | if ! git config user.name &>/dev/null && git config user.email &>/dev/null; then 36 | echo "ERROR: git config user.name, user.email unset. No defaults provided" >&2 37 | exit 1 38 | fi 39 | 40 | export GO111MODULE=on 41 | ./hack/update-deps.sh "$@" # --patch or --minor 42 | 43 | 44 | git add -A 45 | if git diff --name-only --exit-code HEAD; then 46 | echo "Nothing changed" >&2 47 | exit 0 48 | fi 49 | 50 | if ! bazel test //... -- -//vendor/...; then 51 | echo "ERROR: update fails unit tests." >&2 52 | exit 1 53 | fi 54 | 55 | title="Run ./hack/update-deps.sh $@" 56 | git commit -m "${title}" 57 | git push -f "git@github.com:${user}/testgrid.git" HEAD:autoupdate 58 | 59 | echo "Creating PR to merge ${user}:autoupdate into master..." >&2 60 | bazel run //robots/pr-creator -- \ 61 | --github-token-path="${token}" \ 62 | --org=GoogleCloudPlatform --repo=testgrid --branch=master \ 63 | --title="${title}" --match-title="${title}" \ 64 | --body="Automatic go module update. Please review" \ 65 | --source="${user}":autoupdate \ 66 | --confirm 67 | -------------------------------------------------------------------------------- /hack/bazel.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # runs bazel and then coalesce.py (to convert results to junit) 17 | 18 | set -o nounset 19 | set -o errexit 20 | set -o pipefail 21 | 22 | code=0 23 | (set -o xtrace && bazel "$@") || code=$? 24 | coalesce=$(dirname "${BASH_SOURCE[0]}")/coalesce.py 25 | (set -o xtrace && "$coalesce") || true 26 | set -o xtrace 27 | exit "$code" 28 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.Dockerfile.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The AUTHOR Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.Makefile.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The AUTHOR Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.bzl.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The AUTHOR Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.generated.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright YEAR The AUTHOR Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.py.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The AUTHOR Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.sh.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The AUTHOR Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/check-pr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Usage: check-pr [ref] 17 | # 18 | # Ideally if check-pr passes then your golang PR will pass presubmit tests. 19 | 20 | set -o nounset 21 | set -o errexit 22 | set -o pipefail 23 | 24 | cd $(git rev-parse --show-toplevel) 25 | 26 | dirs=() 27 | tests=() 28 | ref="${1:-HEAD}" 29 | echo -n "Packages changed since $ref: " 30 | for d in $(git diff --name-only "$ref" | xargs -n 1 dirname | sort -u); do 31 | if ! ls "./$d/"*.go &> /dev/null; then 32 | continue 33 | fi 34 | echo -n "$d " 35 | dirs+=("./$d") 36 | tests+=("//$d:all") 37 | done 38 | 39 | if [[ ${#dirs[@]} == 0 ]]; then 40 | echo NONE 41 | exit 0 42 | fi 43 | echo 44 | 45 | failing=() 46 | # step runs command and prints the output if it fails. 47 | # if no is specified, is used as the command. 48 | step() { 49 | name="$1" 50 | shift 51 | cmd="$@" 52 | 53 | echo -n "Running ${name}... " 54 | tmp="$(mktemp)" 55 | if [[ -z "${cmd}" ]]; then 56 | cmd="${name}" 57 | fi 58 | if ! ${cmd} &> "$tmp"; then 59 | echo FAIL: 60 | cat "$tmp" 61 | rm -f "$tmp" 62 | failing+=("${name}") 63 | return 0 64 | fi 65 | rm -f "$tmp" 66 | echo PASS 67 | return 0 68 | } 69 | 70 | step "//hack:verify-all" bazel test //hack:verify-all 71 | step "bazel test" bazel test --build_tests_only "${tests[@]}" 72 | 73 | if [[ "${#failing[@]}" != 0 ]]; then 74 | echo "FAILURE: ${#failing[@]} steps failed: ${failing[@]}" 75 | exit 1 76 | fi 77 | echo "SUCCESS" 78 | -------------------------------------------------------------------------------- /hack/print-workspace-status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | git_commit="$(git describe --tags --always --dirty)" 21 | build_date="$(date -u '+%Y%m%d')" 22 | docker_tag="v${build_date}-${git_commit}" 23 | 24 | cat <&2 23 | else 24 | bazel=$(command -v bazelisk || command -v bazel || true) 25 | if [[ -z "$bazel" ]]; then 26 | echo "Install bazel at https://bazel.build" >&2 27 | exit 1 28 | fi 29 | ( 30 | set -o xtrace 31 | "$bazel" run //hack:update-protos 32 | ) 33 | exit 0 34 | fi 35 | 36 | protoc=$1 37 | plugin=$2 38 | boiler=$3 39 | grpc=$4 40 | importmap=$5 41 | dest=$BUILD_WORKSPACE_DIRECTORY 42 | 43 | if [[ -z "${_virtual_imports:-}" ]]; then 44 | export _virtual_imports="$0.runfiles/com_google_protobuf/_virtual_imports" 45 | fi 46 | 47 | genproto() { 48 | dir=$(dirname "$1") 49 | base=$(basename "$1") 50 | out=$dest/github.com/GoogleCloudPlatform/testgrid/$dir/${base%.proto}.pb.go 51 | final=$dest/$dir/${base%.proto}.pb.go 52 | rm -f "$final" "$out" # mac will complain otherwise 53 | ( 54 | # TODO(fejta): this _virtual_imports piece is super fragile 55 | # Add any extra well-known imports to data and then add a new path 56 | "$protoc" \ 57 | "--plugin=$plugin" \ 58 | "--proto_path=$dir" \ 59 | "--proto_path=$dest" \ 60 | "--proto_path=$_virtual_imports/timestamp_proto" \ 61 | "--go_out=${grpc},${importmap}:$dest" \ 62 | "$1" 63 | ) 64 | tmp=$(mktemp) 65 | mv "$out" "$tmp" 66 | cat "$boiler" "$tmp" > "$final" 67 | } 68 | 69 | echo -n "Generating protos: " >&2 70 | for p in $(find . -not '(' -path './vendor' -o -path './node_modules' -o -path './external' -prune ')' -name '*.proto'); do 71 | echo -n "$p " 72 | genproto "$p" 73 | done 74 | echo 75 | 76 | -------------------------------------------------------------------------------- /hack/update-spelling.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then 21 | echo "Updating spelling..." >&2 22 | elif ! command -v bazel &>/dev/null; then 23 | echo "Install bazel at https://bazel.build" >&2 24 | exit 1 25 | else 26 | ( 27 | set -o xtrace 28 | bazel run --test_output=streamed //hack:update-spelling 29 | ) 30 | exit 0 31 | fi 32 | 33 | find -L . -type f -not \( \ 34 | \( \ 35 | -path '*/vendor/*' \ 36 | -o -path '*/static/*' \ 37 | -o -path '*/third_party/*' \ 38 | -o -path '*/node_modules/*' \ 39 | -o -path '*/localdata/*' \ 40 | \) -prune \ 41 | \) -exec "$@" '{}' + 42 | -------------------------------------------------------------------------------- /hack/verify-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | cd "$(git rev-parse --show-toplevel)" 21 | find hack -name 'verify-*.sh' -not -name "$(basename "$0")" \( -print -exec '{}' ';' -o -quit \) 22 | -------------------------------------------------------------------------------- /hack/verify-file-perms.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | # BSD sed doesn't have --version, and needs + instead of / 21 | # GNU sed deprecated and removed + 22 | desired_perm="/111" 23 | # we're not actually running a search so SC2185 doesn't apply 24 | # shellcheck disable=SC2185 25 | if ! find --version >/dev/null 2>&1; then 26 | desired_perm="+111" 27 | fi 28 | 29 | # find all files named *.sh (approximate shell script detection ...) 30 | # - ignoring .git 31 | # - that are not executable by all 32 | files=$(find . -type f -name '*.sh' -not -perm "${desired_perm}" -not -path './.git/*') 33 | if [[ -n "${files}" ]]; then 34 | echo "${files}" 35 | echo 36 | echo "Please run hack/update-file-perms.sh" 37 | exit 1 38 | fi 39 | -------------------------------------------------------------------------------- /hack/verify-protos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | if [[ -n "${TEST_WORKSPACE:-}" ]]; then # Running inside bazel 21 | echo "Checking protos for changes..." >&2 22 | elif ! command -v bazel &>/dev/null; then 23 | echo "Install bazel at https://bazel.build" >&2 24 | exit 1 25 | else 26 | ( 27 | set -o xtrace 28 | bazel test //hack:verify-protos 29 | ) 30 | exit 0 31 | fi 32 | 33 | TESTINFRA_ROOT=$PWD 34 | 35 | _tmpdir="$(mktemp -d -t verify-deps.XXXXXX)" 36 | trap "rm -rf ${_tmpdir}" EXIT 37 | 38 | cp -a "${TESTINFRA_ROOT}/." "${_tmpdir}" 39 | 40 | # Update protos, outputting to $_tmpdir 41 | ( 42 | update_protos=$1 43 | protoc=$2 44 | plugin=$3 45 | boiler=$4 46 | grpc=$5 47 | importmap=$6 48 | 49 | export _virtual_imports=$TEST_SRCDIR/com_google_protobuf/_virtual_imports 50 | export BUILD_WORKSPACE_DIRECTORY=${_tmpdir} 51 | "$update_protos" "$protoc" "$plugin" "$boiler" "$grpc" "$importmap" 52 | ) 53 | 54 | 55 | # Ensure nothing changed 56 | diff=$(diff -Nupr \ 57 | -x ".git" \ 58 | -x "bazel-*" \ 59 | -x "_output" \ 60 | "${TESTINFRA_ROOT}" "${_tmpdir}" 2>/dev/null || true) 61 | 62 | if [[ -n "${diff}" ]]; then 63 | echo "${diff}" >&2 64 | echo >&2 65 | echo "ERROR: protos changed. Run bazel run //hack:update-protos" >&2 66 | exit 1 67 | fi 68 | -------------------------------------------------------------------------------- /hack/verify-spelling.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2018 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | 20 | if [[ -n "${TEST_WORKSPACE:-}" ]]; then 21 | echo "Validating spelling..." >&2 22 | elif ! command -v bazel &> /dev/null; then 23 | echo "Install bazel at https://bazel.build" >&2 24 | exit 1 25 | else 26 | ( 27 | set -o xtrace 28 | bazel test --test_output=streamed //hack:verify-spelling 29 | ) 30 | exit 0 31 | fi 32 | 33 | trap 'echo ERROR: found unexpected instance of "Git"hub, use github or GitHub' ERR 34 | 35 | # Unit test: Git"hub (remove ") 36 | # Appear to need to use this if statement on mac to get the not grep to work 37 | if find -L . -type f -not \( \ 38 | \( \ 39 | -path '*/vendor/*' \ 40 | -o -path '*/external/*' \ 41 | -o -path '*/static/*' \ 42 | -o -path '*/third_party/*' \ 43 | -o -path '*/node_modules/*' \ 44 | -o -path '*/localdata/*' \ 45 | -o -path '*/gubernator/*' \ 46 | -o -path '*/prow/bugzilla/client_test.go' \ 47 | \) -prune \ 48 | \) -exec grep -Hn 'Git'hub '{}' '+' ; then 49 | false 50 | fi 51 | 52 | 53 | trap 'echo ERROR: bad spelling, fix with hack/update-spelling.sh' ERR 54 | 55 | # Unit test: lang auge (remove space) 56 | find -L . -type f -not \( \ 57 | \( \ 58 | -path '*/vendor/*' \ 59 | -o -path '*/external/*' \ 60 | -o -path '*/static/*' \ 61 | -o -path '*/third_party/*' \ 62 | -o -path '*/node_modules/*' \ 63 | -o -path '*/localdata/*' \ 64 | \) -prune \ 65 | \) -exec "$@" '{}' '+' 66 | -------------------------------------------------------------------------------- /images/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("//:def.bzl", "push_image", "tags") 16 | 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | push_image( 20 | name = "push", 21 | images = tags({ 22 | "{STABLE_TESTGRID_REPO}/updater": "//cmd/updater:image", 23 | "{STABLE_TESTGRID_REPO}/summarizer": "//cmd/summarizer:image", 24 | "{STABLE_TESTGRID_REPO}/config_merger": "//cmd/config_merger:image", 25 | "{STABLE_TESTGRID_REPO}/api": "//cmd/api:image", 26 | "{STABLE_TESTGRID_REPO}/tabulator": "//cmd/tabulator:image", 27 | }), 28 | ) 29 | 30 | filegroup( 31 | name = "package-srcs", 32 | srcs = glob(["**"]), 33 | tags = ["automanaged"], 34 | visibility = ["//visibility:private"], 35 | ) 36 | 37 | filegroup( 38 | name = "all-srcs", 39 | srcs = [":package-srcs"], 40 | tags = ["automanaged"], 41 | visibility = ["//visibility:public"], 42 | ) 43 | -------------------------------------------------------------------------------- /images/gcloud-bazel/.gitignore: -------------------------------------------------------------------------------- 1 | rules_k8s/ 2 | -------------------------------------------------------------------------------- /images/gcloud-bazel/push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | NEW=3.4.1 18 | OLD=3.0.0 19 | 20 | cd "$(dirname "${BASH_SOURCE[0]}")" || exit 21 | rm -rf rules_k8s 22 | git clone https://github.com/bazelbuild/rules_k8s.git 23 | make -C rules_k8s/images/gcloud-bazel push PROJECT=k8s-testgrid "OLD=$OLD" "NEW=$NEW" 24 | rm -rf rules_k8s 25 | -------------------------------------------------------------------------------- /images/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # bump.sh will 17 | # * ensure there are no pending changes 18 | # * activate GOOGLE_APPLICATION_CREDENTIALS and configure-docker if set 19 | # * run //images:push, retrying if necessary 20 | 21 | set -o errexit 22 | set -o nounset 23 | set -o pipefail 24 | 25 | # Coloring Macros 26 | # See https://misc.flogisoft.com/bash/tip_colors_and_formatting 27 | 28 | color-version() { # Bold blue 29 | echo -e "\x1B[1;34m${@}\x1B[0m" 30 | } 31 | 32 | color-error() { # Light red 33 | echo -e "\x1B[91m${@}\x1B[0m" 34 | } 35 | 36 | color-target() { # Bold cyan 37 | echo -e "\x1B[1;33m${@}\x1B[0m" 38 | } 39 | 40 | gcloud auth configure-docker 41 | 42 | # Build and push the current commit, failing on any uncommitted changes. 43 | new_version="v$(date -u '+%Y%m%d')-$(git describe --tags --always --dirty)" 44 | echo -e "version: $(color-version ${new_version})" >&2 45 | if [[ "${new_version}" == *-dirty ]]; then 46 | echo -e "$(color-error ERROR): uncommitted changes to repo" >&2 47 | echo " Fix with git commit" >&2 48 | exit 1 49 | fi 50 | 51 | bazel=$(command -v bazelisk || command -v bazel) 52 | 53 | echo -e "Pushing $(color-version ${new_version})" >&2 54 | # Remove retries after https://github.com/bazelbuild/rules_docker/issues/673 55 | for i in {1..3}; do 56 | if "$bazel" run --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //images:push; then 57 | exit 0 58 | elif [[ "$i" == 3 ]]; then 59 | echo "Failed" 60 | exit 1 61 | fi 62 | echo "Failed attempt $i, retrying..." 63 | sleep 5 64 | done 65 | -------------------------------------------------------------------------------- /internal/result/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["results.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/internal/result", 7 | visibility = ["//:__subpackages__"], 8 | deps = [ 9 | "//pb/config:go_default_library", 10 | "//pb/state:go_default_library", 11 | "//pb/test_status:go_default_library", 12 | ], 13 | ) 14 | 15 | filegroup( 16 | name = "package-srcs", 17 | srcs = glob(["**"]), 18 | tags = ["automanaged"], 19 | visibility = ["//visibility:private"], 20 | ) 21 | 22 | filegroup( 23 | name = "all-srcs", 24 | srcs = [":package-srcs"], 25 | tags = ["automanaged"], 26 | visibility = ["//visibility:public"], 27 | ) 28 | 29 | go_test( 30 | name = "go_default_test", 31 | srcs = ["results_test.go"], 32 | embed = [":go_default_library"], 33 | deps = [ 34 | "//pb/config:go_default_library", 35 | "//pb/test_status:go_default_library", 36 | ], 37 | ) 38 | -------------------------------------------------------------------------------- /java/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Bazel Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This file is auto-generated by github.com/bazelbuild/bazel-toolchains/pkg/rbeconfigsgen 16 | # and should not be modified directly. 17 | 18 | load("@bazel_tools//tools/jdk:local_java_repository.bzl", "local_java_runtime") 19 | 20 | package(default_visibility = ["//visibility:public"]) 21 | 22 | alias( 23 | name = "jdk", 24 | actual = "rbe_jdk", 25 | ) 26 | 27 | local_java_runtime( 28 | name = "rbe_jdk", 29 | java_home = "/usr/lib/jvm/java-8-openjdk-amd64", 30 | version = "1.8.0_275", 31 | ) 32 | 33 | filegroup( 34 | name = "package-srcs", 35 | srcs = glob(["**"]), 36 | tags = ["automanaged"], 37 | visibility = ["//visibility:private"], 38 | ) 39 | 40 | filegroup( 41 | name = "all-srcs", 42 | srcs = [":package-srcs"], 43 | tags = ["automanaged"], 44 | visibility = ["//visibility:public"], 45 | ) 46 | -------------------------------------------------------------------------------- /metadata/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["job.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/metadata", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | filegroup( 11 | name = "package-srcs", 12 | srcs = glob(["**"]), 13 | tags = ["automanaged"], 14 | visibility = ["//visibility:private"], 15 | ) 16 | 17 | filegroup( 18 | name = "all-srcs", 19 | srcs = [ 20 | ":package-srcs", 21 | "//metadata/junit:all-srcs", 22 | ], 23 | tags = ["automanaged"], 24 | visibility = ["//visibility:public"], 25 | ) 26 | 27 | go_test( 28 | name = "go_default_test", 29 | srcs = ["job_test.go"], 30 | embed = [":go_default_library"], 31 | ) 32 | -------------------------------------------------------------------------------- /metadata/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Skeleton 3 | 4 | TODO(fejta): improve this documentation. 5 | 6 | See: 7 | * [job.go](/metadata/job.go) for information about `started.json` and `finished.json`. 8 | * [junit subpackage](/metadata/junit) for information about the junit files. 9 | * [prow](https://github.com/kubernetes/test-infra/tree/master/prow), which typically creates these results. 10 | - In particular its [pod utilities](https://github.com/kubernetes/test-infra/blob/master/prow/pod-utilities.md) 11 | which create these files as testgrid expects them. 12 | 13 | # Pubsub 14 | 15 | See documentation for [pubsub](https://cloud.google.com/pubsub) and [GCS' integration](https://cloud.google.com/storage/docs/pubsub-notifications). 16 | 17 | Testgrid can provide near realtime results by configuring GCS to send notifications of newly written results to a pubsub topic. 18 | 19 | TODO(fejta): improve documentation, link to setup script. 20 | -------------------------------------------------------------------------------- /metadata/junit/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["junit.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/metadata/junit", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | filegroup( 11 | name = "package-srcs", 12 | srcs = glob(["**"]), 13 | tags = ["automanaged"], 14 | visibility = ["//visibility:private"], 15 | ) 16 | 17 | filegroup( 18 | name = "all-srcs", 19 | srcs = [":package-srcs"], 20 | tags = ["automanaged"], 21 | visibility = ["//visibility:public"], 22 | ) 23 | 24 | go_test( 25 | name = "go_default_test", 26 | srcs = ["junit_test.go"], 27 | embed = [":go_default_library"], 28 | deps = ["@com_github_google_go_cmp//cmp:go_default_library"], 29 | ) 30 | -------------------------------------------------------------------------------- /pb/BUILD.bazel: -------------------------------------------------------------------------------- 1 | filegroup( 2 | name = "package-srcs", 3 | srcs = glob(["**"]), 4 | tags = ["automanaged"], 5 | visibility = ["//visibility:private"], 6 | ) 7 | 8 | filegroup( 9 | name = "all-srcs", 10 | srcs = [ 11 | ":package-srcs", 12 | "//pb/api/v1:all-srcs", 13 | "//pb/config:all-srcs", 14 | "//pb/custom_evaluator:all-srcs", 15 | "//pb/issue_state:all-srcs", 16 | "//pb/state:all-srcs", 17 | "//pb/summary:all-srcs", 18 | "//pb/test_status:all-srcs", 19 | ], 20 | tags = ["automanaged"], 21 | visibility = ["//visibility:public"], 22 | ) 23 | -------------------------------------------------------------------------------- /pb/README.md: -------------------------------------------------------------------------------- 1 | # Protocol Buffers in TestGrid 2 | 3 | TestGrid stores its configuration, state, and other information in cloud storage 4 | encoded via these protocol buffers. 5 | 6 | ## Reading a Protocol Buffer 7 | 8 | Protocol buffers can be read using the proto compiler `protoc`. Be sure your 9 | working directory is this repository. 10 | 11 | This example uses gsutil to read a Configuration from Google Cloud Storage. Then, 12 | it uses protoc to decode it. 13 | ```bash 14 | gsutil cat gs://example-bucket/config | protoc --decode=Configuration pb/config/config.proto 15 | ``` 16 | 17 | You need to pass protoc the proto name and file used to encode the file. 18 | 19 | These components generally generate these types of protos: 20 | 21 | | Component | Message | Source | 22 | |-----------|---------|--------| 23 | | Configurator or [Config Merger](/cmd/config_merger) | `Configuration` | [config.proto](./config/config.proto) | 24 | | [Summarizer](/cmd/summarizer) | `DashboardSummary` | [summary.proto](./summary/summary.proto) | 25 | | [Updater](/cmd/updater) | `Grid` (see [Reading a Grid](#reading-a-grid))| [state.proto](./state/state.proto) | 26 | | [Tabulator](/cmd/tabulator) | `Grid` (see [Reading a Grid](#reading-a-grid)) | [state.proto](./state/state.proto) | 27 | 28 | ### Reading a Grid 29 | 30 | The Updater and Tabulator will compress its state as well as encoding it. To read it, you'll 31 | need to do one of the following: 32 | - In Go: Use [DownloadGrid()](/util/gcs/gcs.go) or `zlib.NewReader(reader)` 33 | - In shell: Use a script that will uncompress zlib, then pipe that result to `protoc` 34 | 35 | ### Reading an Unknown Protocol Buffer 36 | 37 | ```bash 38 | gsutil cat gs://example-bucket/config | protoc --decode_raw 39 | ``` 40 | 41 | The result will use message numbers instead of message names. For example, `1` 42 | instead of `test_groups` 43 | 44 | ## Changing a Protocol Buffer Definition 45 | 46 | If you want to change one of the .proto files in this repository, you'll also 47 | need to regenerate the .pb.go files. Do so with this command: 48 | ```bash 49 | bazel run //hack:update-protos 50 | ``` 51 | -------------------------------------------------------------------------------- /pb/api/v1/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "testgrid_api_v1_proto", 7 | srcs = ["data.proto"], 8 | visibility = ["//visibility:public"], 9 | deps = [ 10 | "//pb/config:config_proto", 11 | "//pb/state:state_proto", 12 | "//pb/summary:summary_proto", 13 | "@com_google_protobuf//:timestamp_proto", 14 | ], 15 | ) 16 | 17 | go_proto_library( 18 | name = "testgrid_api_v1_go_proto", 19 | compilers = ["@io_bazel_rules_go//proto:go_grpc"], 20 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/api/v1", 21 | proto = ":testgrid_api_v1_proto", 22 | visibility = ["//visibility:public"], 23 | deps = [ 24 | "//pb/config:go_default_library", 25 | "//pb/state:go_default_library", 26 | "//pb/summary:go_default_library", 27 | ], 28 | ) 29 | 30 | go_library( 31 | name = "go_default_library", 32 | embed = [":testgrid_api_v1_go_proto"], 33 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/api/v1", 34 | visibility = ["//visibility:public"], 35 | ) 36 | 37 | filegroup( 38 | name = "package-srcs", 39 | srcs = glob(["**"]), 40 | tags = ["automanaged"], 41 | visibility = ["//visibility:private"], 42 | ) 43 | 44 | filegroup( 45 | name = "all-srcs", 46 | srcs = [":package-srcs"], 47 | tags = ["automanaged"], 48 | visibility = ["//visibility:public"], 49 | ) 50 | -------------------------------------------------------------------------------- /pb/config/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "config_proto", 7 | srcs = ["config.proto"], 8 | visibility = ["//visibility:public"], 9 | deps = ["//pb/custom_evaluator:custom_evaluator_proto"], 10 | ) 11 | 12 | go_proto_library( 13 | name = "config_go_proto", 14 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/config", 15 | proto = ":config_proto", 16 | visibility = ["//visibility:public"], 17 | deps = ["//pb/custom_evaluator:go_default_library"], 18 | ) 19 | 20 | go_library( 21 | name = "go_default_library", 22 | embed = [":config_go_proto"], 23 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/config", 24 | visibility = ["//visibility:public"], 25 | ) 26 | 27 | filegroup( 28 | name = "package-srcs", 29 | srcs = glob(["**"]), 30 | tags = ["automanaged"], 31 | visibility = ["//visibility:private"], 32 | ) 33 | 34 | filegroup( 35 | name = "all-srcs", 36 | srcs = [":package-srcs"], 37 | tags = ["automanaged"], 38 | visibility = ["//visibility:public"], 39 | ) 40 | -------------------------------------------------------------------------------- /pb/custom_evaluator/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "custom_evaluator_proto", 7 | srcs = ["custom_evaluator.proto"], 8 | visibility = ["//visibility:public"], 9 | deps = ["//pb/test_status:test_status_proto"], 10 | ) 11 | 12 | go_proto_library( 13 | name = "custom_evaluator_go_proto", 14 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/custom_evaluator", 15 | proto = ":custom_evaluator_proto", 16 | visibility = ["//visibility:public"], 17 | deps = ["//pb/test_status:go_default_library"], 18 | ) 19 | 20 | go_library( 21 | name = "go_default_library", 22 | embed = [":custom_evaluator_go_proto"], 23 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/custom_evaluator", 24 | visibility = ["//visibility:public"], 25 | ) 26 | 27 | filegroup( 28 | name = "package-srcs", 29 | srcs = glob(["**"]), 30 | tags = ["automanaged"], 31 | visibility = ["//visibility:private"], 32 | ) 33 | 34 | filegroup( 35 | name = "all-srcs", 36 | srcs = [":package-srcs"], 37 | tags = ["automanaged"], 38 | visibility = ["//visibility:public"], 39 | ) 40 | -------------------------------------------------------------------------------- /pb/issue_state/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "issue_state_proto", 7 | srcs = ["issue_state.proto"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_proto_library( 12 | name = "issue_state_go_proto", 13 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/issue_state", 14 | proto = ":issue_state_proto", 15 | visibility = ["//visibility:public"], 16 | ) 17 | 18 | go_library( 19 | name = "go_default_library", 20 | embed = [":issue_state_go_proto"], 21 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/issue_state", 22 | visibility = ["//visibility:public"], 23 | ) 24 | 25 | filegroup( 26 | name = "package-srcs", 27 | srcs = glob(["**"]), 28 | tags = ["automanaged"], 29 | visibility = ["//visibility:private"], 30 | ) 31 | 32 | filegroup( 33 | name = "all-srcs", 34 | srcs = [":package-srcs"], 35 | tags = ["automanaged"], 36 | visibility = ["//visibility:public"], 37 | ) 38 | -------------------------------------------------------------------------------- /pb/issue_state/issue_state.proto: -------------------------------------------------------------------------------- 1 | // Backing state for issues associated with a TestGrid test group. 2 | 3 | syntax = "proto3"; 4 | 5 | package testgrid.issue_state; 6 | 7 | option go_package = "github.com/GoogleCloudPlatform/testgrid/pb/issue_state"; 8 | 9 | message TargetAndMethods { 10 | string target_name = 1; 11 | repeated string method_names = 2; 12 | } 13 | 14 | message IssueInfo { 15 | string issue_id = 1; 16 | string title = 2; // Issue title or description. 17 | bool is_autobug = 3; // True if auto-created by TestGrid for a failing test. 18 | bool is_flakiness_bug = 19 | 8; // True if auto-created by TestGrid for a flaky test. 20 | double last_modified = 4; // In seconds since epoch. 21 | repeated string row_ids = 5; // Associated row IDs (mentioned in the issue). 22 | 23 | // Run IDs used to associate this issue with a particular target (in case of 24 | // repeats, or across runs on different dashboards). 25 | repeated string run_ids = 6; 26 | 27 | // Targets + methods associated with this issue. 28 | // Only set if test group's `link_bugs_by_test_methods` is True, else all 29 | // targets + methods will be linked to this issue. 30 | repeated TargetAndMethods targets_and_methods = 7; 31 | } 32 | 33 | message IssueState { 34 | // List of collected info for bugs. 35 | repeated IssueInfo issue_info = 1; 36 | 37 | reserved 2; 38 | 39 | reserved 3; 40 | } 41 | -------------------------------------------------------------------------------- /pb/state/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "state_proto", 7 | srcs = ["state.proto"], 8 | visibility = ["//visibility:public"], 9 | deps = [ 10 | "//pb/config:config_proto", 11 | "@com_google_protobuf//:timestamp_proto", 12 | ], 13 | ) 14 | 15 | go_proto_library( 16 | name = "state_go_proto", 17 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/state", 18 | proto = ":state_proto", 19 | visibility = ["//visibility:public"], 20 | deps = [ 21 | "//pb/config:go_default_library", 22 | ], 23 | ) 24 | 25 | go_library( 26 | name = "go_default_library", 27 | embed = [":state_go_proto"], 28 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/state", 29 | visibility = ["//visibility:public"], 30 | ) 31 | 32 | filegroup( 33 | name = "package-srcs", 34 | srcs = glob(["**"]), 35 | tags = ["automanaged"], 36 | visibility = ["//visibility:private"], 37 | ) 38 | 39 | filegroup( 40 | name = "all-srcs", 41 | srcs = [":package-srcs"], 42 | tags = ["automanaged"], 43 | visibility = ["//visibility:public"], 44 | ) 45 | -------------------------------------------------------------------------------- /pb/summary/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "summary_proto", 7 | srcs = ["summary.proto"], 8 | visibility = ["//visibility:public"], 9 | deps = [ 10 | "@com_google_protobuf//:timestamp_proto", 11 | ], 12 | ) 13 | 14 | go_proto_library( 15 | name = "summary_go_proto", 16 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/summary", 17 | proto = ":summary_proto", 18 | visibility = ["//visibility:public"], 19 | ) 20 | 21 | go_library( 22 | name = "go_default_library", 23 | embed = [":summary_go_proto"], 24 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/summary", 25 | visibility = ["//visibility:public"], 26 | ) 27 | 28 | filegroup( 29 | name = "package-srcs", 30 | srcs = glob(["**"]), 31 | tags = ["automanaged"], 32 | visibility = ["//visibility:private"], 33 | ) 34 | 35 | filegroup( 36 | name = "all-srcs", 37 | srcs = [":package-srcs"], 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:public"], 40 | ) 41 | -------------------------------------------------------------------------------- /pb/test_status/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@rules_proto//proto:defs.bzl", "proto_library") 2 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 3 | load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") 4 | 5 | proto_library( 6 | name = "test_status_proto", 7 | srcs = ["test_status.proto"], 8 | visibility = ["//visibility:public"], 9 | ) 10 | 11 | go_proto_library( 12 | name = "test_status_go_proto", 13 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/test_status", 14 | proto = ":test_status_proto", 15 | visibility = ["//visibility:public"], 16 | ) 17 | 18 | filegroup( 19 | name = "package-srcs", 20 | srcs = glob(["**"]), 21 | tags = ["automanaged"], 22 | visibility = ["//visibility:private"], 23 | ) 24 | 25 | filegroup( 26 | name = "all-srcs", 27 | srcs = [":package-srcs"], 28 | tags = ["automanaged"], 29 | visibility = ["//visibility:public"], 30 | ) 31 | 32 | go_library( 33 | name = "go_default_library", 34 | embed = [":test_status_go_proto"], 35 | importpath = "github.com/GoogleCloudPlatform/testgrid/pb/test_status", 36 | visibility = ["//visibility:public"], 37 | ) 38 | -------------------------------------------------------------------------------- /pb/test_status/test_status.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package testgrid.test_status; 4 | 5 | option go_package = "github.com/GoogleCloudPlatform/testgrid/pb/test_status"; 6 | 7 | enum TestStatus { 8 | // Proto versions of test_status.py's GathererStatus 9 | // Note that: NO_RESULT is used to signal that there should be no change. 10 | // This must be updated every time a new GathererStatus is added. 11 | NO_RESULT = 0; 12 | PASS = 1; 13 | PASS_WITH_ERRORS = 2; 14 | PASS_WITH_SKIPS = 3; 15 | RUNNING = 4; 16 | CATEGORIZED_ABORT = 5; 17 | UNKNOWN = 6; 18 | CANCEL = 7; 19 | BLOCKED = 8; 20 | TIMED_OUT = 9; 21 | CATEGORIZED_FAIL = 10; 22 | BUILD_FAIL = 11; 23 | FAIL = 12; 24 | FLAKY = 13; 25 | TOOL_FAIL = 14; 26 | BUILD_PASSED = 15; 27 | } 28 | -------------------------------------------------------------------------------- /pkg/api/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["router.go"], 6 | data = ["README.md"], 7 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/api", 8 | visibility = ["//visibility:public"], 9 | deps = [ 10 | "//pb/api/v1:go_default_library", 11 | "//pkg/api/v1:go_default_library", 12 | "//util/gcs:go_default_library", 13 | "@com_github_go_chi_chi//:go_default_library", 14 | "@com_google_cloud_go_storage//:go_default_library", 15 | "@org_golang_google_grpc//:go_default_library", 16 | "@org_golang_google_grpc//reflection:go_default_library", 17 | ], 18 | ) 19 | 20 | filegroup( 21 | name = "package-srcs", 22 | srcs = glob(["**"]), 23 | tags = ["automanaged"], 24 | visibility = ["//visibility:private"], 25 | ) 26 | 27 | filegroup( 28 | name = "all-srcs", 29 | srcs = [ 30 | ":package-srcs", 31 | "//pkg/api/v1:all-srcs", 32 | ], 33 | tags = ["automanaged"], 34 | visibility = ["//visibility:public"], 35 | ) 36 | 37 | go_test( 38 | name = "go_default_test", 39 | srcs = ["router_http_test.go"], 40 | data = ["README.md"], 41 | embed = [":go_default_library"], 42 | ) 43 | -------------------------------------------------------------------------------- /pkg/api/README.md: -------------------------------------------------------------------------------- 1 | # TestGrid HTTP API 2 | Valid responses are all in JSON. Error responses may not be in JSON. Replace things in curly braces. 3 | 4 | Exact API definitions can be found on [GitHub](https://github.com/GoogleCloudPlatform/testgrid/blob/master/pb/api/v1/data.proto). 5 | 6 | ## LIST 7 | "List" methods use the GET HTTP verb. See https://cloud.google.com/apis/design/standard_methods for details. 8 | 9 | - /api/v1/dashboards - List dashboards 10 | - /api/v1/dashboard-groups - List dashboard groups 11 | - /api/v1/dashboards/{dashboard}/tabs - List a dashboard's tabs 12 | - /api/v1/dashboard-groups/{dashboard-group} - List the dashboards in a dashboard group 13 | - /api/v1/dashboards/{dashboard}/tab-summaries - List the tab summaries for the dashboard (data rendered in dashboard view) 14 | - /api/v1/dashboard-groups/{dashboard-group}/dashboard-summaries - List the dashboard summaries for the dashboard group (data rendered in dashboard group view) 15 | 16 | ## GET 17 | - /api/v1/dashboards/{dashboard} - Returns a dashboard's configuration. Often empty; dashboard-level configuration is rare. 18 | - /api/v1/dashboards/{dashboard}/tabs/{tab}/headers - Returns the headers for a tab's grid result 19 | - /api/v1/dashboards/{dashboard}/tabs/{tab}/rows - Returns information on a tab's rows and the data within those rows. 20 | - /api/v1/dashboards/{dashboard}/tab-summaries/{tab} - Returns the summary for a particular tab in the given dashboard 21 | - /api/v1/dashboards/{dashboard}/summary - Returns the aggregated summary for a particular dashboard. -------------------------------------------------------------------------------- /pkg/api/router_http_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package api 18 | 19 | import ( 20 | "net/http" 21 | "net/http/httptest" 22 | "testing" 23 | ) 24 | 25 | func TestHealth(t *testing.T) { 26 | tests := []struct { 27 | name string 28 | path string 29 | expectedCode int 30 | }{ 31 | { 32 | name: "Returns a healthiness check at '/'", 33 | path: "/", 34 | expectedCode: http.StatusOK, 35 | }, 36 | { 37 | name: "Return 404 to nonsense", 38 | path: "/ipa/v1/derp", 39 | expectedCode: http.StatusNotFound, 40 | }, 41 | } 42 | 43 | // Tests are run from the local directory, while the image is built from the repository root 44 | healthCheckFile = "README.md" 45 | 46 | for _, test := range tests { 47 | t.Run(test.name, func(t *testing.T) { 48 | router, _, err := GetRouters(RouterOptions{}, nil) 49 | if err != nil { 50 | t.Fatalf("Unexpected error: %v", err) 51 | } 52 | request, err := http.NewRequest("GET", test.path, nil) 53 | if err != nil { 54 | t.Fatalf("Can't form request: %v", err) 55 | } 56 | response := httptest.NewRecorder() 57 | router.ServeHTTP(response, request) 58 | if response.Code != test.expectedCode { 59 | t.Errorf("Expected %d, but got %d", test.expectedCode, response.Code) 60 | } 61 | }) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pkg/api/v1/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "config.go", 7 | "config_cache.go", 8 | "json.go", 9 | "server.go", 10 | "server_fake.go", 11 | "state.go", 12 | "summary.go", 13 | ], 14 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/api/v1", 15 | visibility = ["//visibility:public"], 16 | deps = [ 17 | "//config:go_default_library", 18 | "//config/snapshot:go_default_library", 19 | "//pb/api/v1:go_default_library", 20 | "//pb/config:go_default_library", 21 | "//pb/state:go_default_library", 22 | "//pb/summary:go_default_library", 23 | "//pkg/summarizer:go_default_library", 24 | "//pkg/tabulator:go_default_library", 25 | "//util/gcs:go_default_library", 26 | "@com_github_go_chi_chi//:go_default_library", 27 | "@com_github_sirupsen_logrus//:go_default_library", 28 | "@com_google_cloud_go_storage//:go_default_library", 29 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 30 | "@org_golang_google_protobuf//encoding/protojson:go_default_library", 31 | "@org_golang_google_protobuf//proto:go_default_library", 32 | ], 33 | ) 34 | 35 | filegroup( 36 | name = "package-srcs", 37 | srcs = glob(["**"]), 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:private"], 40 | ) 41 | 42 | filegroup( 43 | name = "all-srcs", 44 | srcs = [":package-srcs"], 45 | tags = ["automanaged"], 46 | visibility = ["//visibility:public"], 47 | ) 48 | 49 | go_test( 50 | name = "go_default_test", 51 | srcs = [ 52 | "config_cache_test.go", 53 | "config_http_test.go", 54 | "config_test.go", 55 | "state_test.go", 56 | "summary_test.go", 57 | ], 58 | embed = [":go_default_library"], 59 | deps = [ 60 | "//config/snapshot:go_default_library", 61 | "//pb/api/v1:go_default_library", 62 | "//pb/config:go_default_library", 63 | "//pb/state:go_default_library", 64 | "//pb/summary:go_default_library", 65 | "//util/gcs:go_default_library", 66 | "@com_github_google_go_cmp//cmp:go_default_library", 67 | "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", 68 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 69 | "@org_golang_google_protobuf//encoding/protojson:go_default_library", 70 | "@org_golang_google_protobuf//proto:go_default_library", 71 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 72 | "@org_golang_google_protobuf//types/known/timestamppb:go_default_library", 73 | ], 74 | ) 75 | -------------------------------------------------------------------------------- /pkg/api/v1/json.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1 18 | 19 | import ( 20 | "net/http" 21 | 22 | "google.golang.org/protobuf/encoding/protojson" 23 | "google.golang.org/protobuf/proto" 24 | ) 25 | 26 | // writeJSON will write obj to w as JSON, or will write the JSON marshalling error 27 | // Includes headers that are universal to all API responses 28 | func (s Server) writeJSON(w http.ResponseWriter, msg proto.Message) { 29 | 30 | opts := protojson.MarshalOptions{UseProtoNames: true} 31 | resp, err := opts.Marshal(msg) 32 | if err != nil { 33 | http.Error(w, err.Error(), http.StatusInternalServerError) 34 | return 35 | } 36 | 37 | w.Header().Set("Content-Type", "application/json") 38 | if s.AccessControlAllowOrigin != "" { 39 | w.Header().Set("Access-Control-Allow-Origin", s.AccessControlAllowOrigin) 40 | if s.AccessControlAllowOrigin != "*" { 41 | w.Header().Set("Vary", "Origin") 42 | } 43 | } 44 | w.Write(resp) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/api/v1/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1 (api/v1) is the first versioned implementation of the API 18 | package v1 19 | 20 | import ( 21 | "time" 22 | 23 | apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1" 24 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 25 | "github.com/go-chi/chi" 26 | ) 27 | 28 | // Server contains the necessary settings and i/o objects needed to serve this api 29 | type Server struct { 30 | Client gcs.ConditionalClient 31 | DefaultBucket string 32 | TabPathPrefix string 33 | SummaryPathPrefix string 34 | AccessControlAllowOrigin string 35 | Timeout time.Duration 36 | defaultCache *cachedConfig 37 | } 38 | 39 | // Ensure the server implementation conforms to the API 40 | var _ apipb.TestGridDataServer = (*Server)(nil) 41 | 42 | // Route applies all the v1 API functions provided by the Server to the Router given. 43 | // If the router is nil, a new one is instantiated. 44 | func Route(r *chi.Mux, s Server) *chi.Mux { 45 | if r == nil { 46 | r = chi.NewRouter() 47 | } 48 | r.Get("/dashboard-groups", s.ListDashboardGroupHTTP) 49 | r.Get("/dashboard-groups/{dashboard-group}", s.GetDashboardGroupHTTP) 50 | r.Get("/dashboards", s.ListDashboardsHTTP) 51 | r.Get("/dashboards/{dashboard}/tabs", s.ListDashboardTabsHTTP) 52 | r.Get("/dashboards/{dashboard}", s.GetDashboardHTTP) 53 | 54 | r.Get("/dashboards/{dashboard}/tabs/{tab}/headers", s.ListHeadersHTTP) 55 | r.Get("/dashboards/{dashboard}/tabs/{tab}/rows", s.ListRowsHTTP) 56 | 57 | r.Get("/dashboards/{dashboard}/tab-summaries", s.ListTabSummariesHTTP) 58 | r.Get("/dashboards/{dashboard}/tab-summaries/{tab}", s.GetTabSummaryHTTP) 59 | 60 | r.Get("/dashboard-groups/{dashboard-group}/dashboard-summaries", s.ListDashboardSummariesHTTP) 61 | r.Get("/dashboards/{dashboard}/summary", s.GetDashboardSummaryHTTP) 62 | return r 63 | } 64 | -------------------------------------------------------------------------------- /pkg/merger/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["merger.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/merger", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//config:go_default_library", 10 | "//pb/config:go_default_library", 11 | "//util/gcs:go_default_library", 12 | "//util/metrics:go_default_library", 13 | "@com_github_golang_protobuf//proto:go_default_library", 14 | "@com_github_sirupsen_logrus//:go_default_library", 15 | "@com_google_cloud_go_storage//:go_default_library", 16 | "@io_k8s_sigs_yaml//goyaml.v2:go_default_library", 17 | ], 18 | ) 19 | 20 | go_test( 21 | name = "go_default_test", 22 | srcs = ["merger_test.go"], 23 | embed = [":go_default_library"], 24 | deps = [ 25 | "//pb/config:go_default_library", 26 | "//util/gcs:go_default_library", 27 | "@com_github_golang_protobuf//proto:go_default_library", 28 | "@com_github_google_go_cmp//cmp:go_default_library", 29 | "@com_google_cloud_go_storage//:go_default_library", 30 | ], 31 | ) 32 | 33 | filegroup( 34 | name = "package-srcs", 35 | srcs = glob(["**"]), 36 | tags = ["automanaged"], 37 | visibility = ["//visibility:private"], 38 | ) 39 | 40 | filegroup( 41 | name = "all-srcs", 42 | srcs = [":package-srcs"], 43 | tags = ["automanaged"], 44 | visibility = ["//visibility:public"], 45 | ) 46 | -------------------------------------------------------------------------------- /pkg/pubsub/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["pubsub.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/pubsub", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//util/gcs:go_default_library", 10 | "@com_github_sirupsen_logrus//:go_default_library", 11 | "@com_google_cloud_go_pubsub//:go_default_library", 12 | ], 13 | ) 14 | 15 | go_test( 16 | name = "go_default_test", 17 | srcs = ["pubsub_test.go"], 18 | embed = [":go_default_library"], 19 | deps = [ 20 | "//util/gcs:go_default_library", 21 | "@com_github_google_go_cmp//cmp:go_default_library", 22 | "@com_github_sirupsen_logrus//:go_default_library", 23 | "@com_google_cloud_go_pubsub//:go_default_library", 24 | ], 25 | ) 26 | 27 | filegroup( 28 | name = "package-srcs", 29 | srcs = glob(["**"]), 30 | tags = ["automanaged"], 31 | visibility = ["//visibility:private"], 32 | ) 33 | 34 | filegroup( 35 | name = "all-srcs", 36 | srcs = [":package-srcs"], 37 | tags = ["automanaged"], 38 | visibility = ["//visibility:public"], 39 | ) 40 | -------------------------------------------------------------------------------- /pkg/summarizer/analyzers/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "baseanalyzer.go", 7 | "flipanalyzer.go", 8 | ], 9 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/analyzers", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "//pb/summary:go_default_library", 13 | "//pkg/summarizer/common:go_default_library", 14 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 15 | ], 16 | ) 17 | 18 | go_test( 19 | name = "go_default_test", 20 | srcs = [ 21 | "baseanalyzer_test.go", 22 | "flipanalyzer_test.go", 23 | ], 24 | embed = [":go_default_library"], 25 | deps = [ 26 | "//pb/summary:go_default_library", 27 | "//pkg/summarizer/common:go_default_library", 28 | "@com_github_golang_protobuf//proto:go_default_library", 29 | "@com_github_google_go_cmp//cmp:go_default_library", 30 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 31 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 32 | ], 33 | ) 34 | 35 | filegroup( 36 | name = "package-srcs", 37 | srcs = glob(["**"]), 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:private"], 40 | ) 41 | 42 | filegroup( 43 | name = "all-srcs", 44 | srcs = [":package-srcs"], 45 | tags = ["automanaged"], 46 | visibility = ["//visibility:public"], 47 | ) 48 | -------------------------------------------------------------------------------- /pkg/summarizer/common/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["common.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/summarizer/common", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | filegroup( 11 | name = "package-srcs", 12 | srcs = glob(["**"]), 13 | tags = ["automanaged"], 14 | visibility = ["//visibility:private"], 15 | ) 16 | 17 | filegroup( 18 | name = "all-srcs", 19 | srcs = [":package-srcs"], 20 | tags = ["automanaged"], 21 | visibility = ["//visibility:public"], 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/summarizer/common/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | // GridMetrics contains the gathered metrics such as passed and failed test count 20 | // for a state.proto Grid 21 | type GridMetrics struct { 22 | Name string 23 | Passed int 24 | Failed int 25 | FlakyCount int 26 | AverageFlakiness float64 27 | FailedInfraCount int 28 | InfraFailures map[string]int 29 | } 30 | 31 | // NewGridMetrics constructs a new GridMetrics struct with nil default values 32 | // reassigned to working versions. 33 | func NewGridMetrics(name string) *GridMetrics { 34 | gridMetrics := GridMetrics{Name: name, InfraFailures: make(map[string]int, 0)} 35 | return &gridMetrics 36 | } 37 | -------------------------------------------------------------------------------- /pkg/summarizer/persist.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package summarizer 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/testgrid/config" 24 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 25 | "github.com/GoogleCloudPlatform/testgrid/util/queue" 26 | "github.com/sirupsen/logrus" 27 | ) 28 | 29 | // FixPersistent persists the updater queue using queue.FixPersistent. 30 | func FixPersistent(log logrus.FieldLogger, client queue.PersistClient, path gcs.Path, tick <-chan time.Time) Fixer { 31 | log = log.WithField("path", path) 32 | fix := queue.FixPersistent(log, client, path, tick) 33 | return func(ctx context.Context, iq *config.DashboardQueue) error { 34 | return fix(ctx, &iq.Queue) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/tabulator/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "filter.go", 7 | "persist.go", 8 | "pubsub.go", 9 | "tabstate.go", 10 | ], 11 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/tabulator", 12 | visibility = ["//visibility:public"], 13 | deps = [ 14 | "//config:go_default_library", 15 | "//config/snapshot:go_default_library", 16 | "//pb/config:go_default_library", 17 | "//pb/state:go_default_library", 18 | "//pb/test_status:go_default_library", 19 | "//pkg/pubsub:go_default_library", 20 | "//pkg/updater:go_default_library", 21 | "//util/gcs:go_default_library", 22 | "//util/metrics:go_default_library", 23 | "//util/queue:go_default_library", 24 | "@com_github_sirupsen_logrus//:go_default_library", 25 | "@org_golang_google_protobuf//proto:go_default_library", 26 | ], 27 | ) 28 | 29 | go_test( 30 | name = "go_default_test", 31 | srcs = [ 32 | "filter_test.go", 33 | "pubsub_test.go", 34 | "tabstate_test.go", 35 | ], 36 | embed = [":go_default_library"], 37 | deps = [ 38 | "//pb/config:go_default_library", 39 | "//pb/state:go_default_library", 40 | "//pb/test_status:go_default_library", 41 | "//pkg/pubsub:go_default_library", 42 | "//pkg/updater:go_default_library", 43 | "//util/gcs:go_default_library", 44 | "//util/gcs/fake:go_default_library", 45 | "@com_github_google_go_cmp//cmp:go_default_library", 46 | "@com_github_sirupsen_logrus//:go_default_library", 47 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 48 | ], 49 | ) 50 | 51 | filegroup( 52 | name = "package-srcs", 53 | srcs = glob(["**"]), 54 | tags = ["automanaged"], 55 | visibility = ["//visibility:private"], 56 | ) 57 | 58 | filegroup( 59 | name = "all-srcs", 60 | srcs = [":package-srcs"], 61 | tags = ["automanaged"], 62 | visibility = ["//visibility:public"], 63 | ) 64 | -------------------------------------------------------------------------------- /pkg/tabulator/filter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tabulator 18 | 19 | import ( 20 | "fmt" 21 | "net/url" 22 | "regexp" 23 | 24 | statepb "github.com/GoogleCloudPlatform/testgrid/pb/state" 25 | ) 26 | 27 | const ( 28 | includeFilter = "include-filter-by-regex" 29 | excludeFilter = "exclude-filter-by-regex" 30 | ) 31 | 32 | // filterGrid filters the grid to rows matching the include/exclude list in the base options 33 | func filterGrid(baseOptions string, rows []*statepb.Row) ([]*statepb.Row, error) { 34 | 35 | vals, err := url.ParseQuery(baseOptions) 36 | if err != nil { 37 | return nil, fmt.Errorf("parse %q: %v", baseOptions, err) 38 | } 39 | 40 | for _, include := range vals[includeFilter] { 41 | if rows, err = includeRows(rows, include); err != nil { 42 | return nil, fmt.Errorf("bad %s=%s: %v", includeFilter, include, err) 43 | } 44 | } 45 | 46 | for _, exclude := range vals[excludeFilter] { 47 | if rows, err = excludeRows(rows, exclude); err != nil { 48 | return nil, fmt.Errorf("bad %s=%s: %v", excludeFilter, exclude, err) 49 | } 50 | } 51 | 52 | // TODO(chases2): drop columns that are now empty due to filters 53 | // TODO(fejta): grouping, which is not used by testgrid.k8s.io 54 | // TODO(fejta): sorting, unused by testgrid.k8s.io 55 | // TODO(fejta): graph, unused by testgrid.k8s.io 56 | // TODO(fejta): tabuluar, unused by testgrid.k8s.io 57 | return rows, nil 58 | } 59 | 60 | // includeRows returns the subset of rows that match the regex 61 | func includeRows(in []*statepb.Row, include string) ([]*statepb.Row, error) { 62 | re, err := regexp.Compile(include) 63 | if err != nil { 64 | return nil, err 65 | } 66 | var rows []*statepb.Row 67 | for _, r := range in { 68 | if !re.MatchString(r.Name) { 69 | continue 70 | } 71 | rows = append(rows, r) 72 | } 73 | return rows, nil 74 | } 75 | 76 | // excludeRows returns the subset of rows that do not match the regex 77 | func excludeRows(in []*statepb.Row, exclude string) ([]*statepb.Row, error) { 78 | re, err := regexp.Compile(exclude) 79 | if err != nil { 80 | return nil, err 81 | } 82 | var rows []*statepb.Row 83 | for _, r := range in { 84 | if re.MatchString(r.Name) { 85 | continue 86 | } 87 | rows = append(rows, r) 88 | } 89 | return rows, nil 90 | } 91 | -------------------------------------------------------------------------------- /pkg/tabulator/persist.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tabulator 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/testgrid/config" 24 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 25 | "github.com/GoogleCloudPlatform/testgrid/util/queue" 26 | "github.com/sirupsen/logrus" 27 | ) 28 | 29 | // FixPersistent persists the updater queue using queue.FixPersistent. 30 | func FixPersistent(log logrus.FieldLogger, client queue.PersistClient, path gcs.Path, tick <-chan time.Time) Fixer { 31 | log = log.WithField("path", path) 32 | fix := queue.FixPersistent(log, client, path, tick) 33 | return func(ctx context.Context, iq *config.TestGroupQueue) error { 34 | return fix(ctx, &iq.Queue) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/tabulator/pubsub_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package tabulator 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/GoogleCloudPlatform/testgrid/pkg/pubsub" 23 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 24 | "github.com/google/go-cmp/cmp" 25 | ) 26 | 27 | func TestProcessNotification(t *testing.T) { 28 | mustPath := func(s string) gcs.Path { 29 | p, err := gcs.NewPath(s) 30 | if err != nil { 31 | t.Fatalf("gcs.NewPath(%q): %v", s, err) 32 | } 33 | return *p 34 | } 35 | pstr := func(s string) *string { return &s } 36 | cases := []struct { 37 | name string 38 | prefix gcs.Path 39 | notice *pubsub.Notification 40 | want *string 41 | }{ 42 | { 43 | name: "basically works", 44 | prefix: mustPath("gs://bucket/prefix/"), 45 | notice: &pubsub.Notification{}, 46 | }, 47 | { 48 | name: "wrong bucket", 49 | prefix: mustPath("gs://bucket/prefix/"), 50 | notice: &pubsub.Notification{ 51 | Path: mustPath("gs://elsewhere/prefix/foo"), 52 | }, 53 | }, 54 | { 55 | name: "wrong prefix", 56 | prefix: mustPath("gs://bucket/prefix/"), 57 | notice: &pubsub.Notification{ 58 | Path: mustPath("gs://bucket/foo"), 59 | }, 60 | }, 61 | { 62 | name: "require trailing slash", 63 | prefix: mustPath("gs://bucket/prefix"), // missing / 64 | notice: &pubsub.Notification{ 65 | Path: mustPath("gs://bucket/prefix/foo"), 66 | }, 67 | }, 68 | { 69 | name: "match", 70 | prefix: mustPath("gs://bucket/prefix/"), 71 | notice: &pubsub.Notification{ 72 | Path: mustPath("gs://bucket/prefix/foo"), 73 | }, 74 | want: pstr("foo"), 75 | }, 76 | } 77 | 78 | for _, tc := range cases { 79 | t.Run(tc.name, func(t *testing.T) { 80 | got := processNotification(tc.prefix, tc.notice) 81 | if diff := cmp.Diff(tc.want, got); diff != "" { 82 | t.Errorf("processNotification() got unexpected diff (-want +got):\n%s", diff) 83 | } 84 | }) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /pkg/updater/persist.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package updater 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/GoogleCloudPlatform/testgrid/config" 24 | configpb "github.com/GoogleCloudPlatform/testgrid/pb/config" 25 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 26 | "github.com/GoogleCloudPlatform/testgrid/util/queue" 27 | "github.com/sirupsen/logrus" 28 | ) 29 | 30 | // FixPersistent persists the updater queue using queue.FixPersistent. 31 | func FixPersistent(log logrus.FieldLogger, client queue.PersistClient, path gcs.Path, tick <-chan time.Time) Fixer { 32 | fix := queue.FixPersistent(log, client, path, tick) 33 | return func(ctx context.Context, _ logrus.FieldLogger, q *config.TestGroupQueue, _ []*configpb.TestGroup) error { 34 | return fix(ctx, &q.Queue) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/updater/resultstore/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "client.go", 7 | "resultstore.go", 8 | ], 9 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "//pb/config:go_default_library", 13 | "//pb/custom_evaluator:go_default_library", 14 | "//pb/state:go_default_library", 15 | "//pb/test_status:go_default_library", 16 | "//pkg/updater:go_default_library", 17 | "//pkg/updater/resultstore/query:go_default_library", 18 | "//util/gcs:go_default_library", 19 | "@com_github_sirupsen_logrus//:go_default_library", 20 | "@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto", 21 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 22 | "@org_golang_google_grpc//:go_default_library", 23 | "@org_golang_google_grpc//credentials:go_default_library", 24 | "@org_golang_google_grpc//credentials/oauth:go_default_library", 25 | "@org_golang_google_grpc//metadata:go_default_library", 26 | ], 27 | ) 28 | 29 | go_test( 30 | name = "go_default_test", 31 | srcs = ["resultstore_test.go"], 32 | embed = [":go_default_library"], 33 | deps = [ 34 | "//pb/config:go_default_library", 35 | "//pb/custom_evaluator:go_default_library", 36 | "//pb/state:go_default_library", 37 | "//pb/test_status:go_default_library", 38 | "//pkg/updater:go_default_library", 39 | "@com_github_google_go_cmp//cmp:go_default_library", 40 | "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", 41 | "@com_github_sirupsen_logrus//:go_default_library", 42 | "@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto", 43 | "@io_bazel_rules_go//proto/wkt:duration_go_proto", 44 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 45 | "@org_golang_google_grpc//:go_default_library", 46 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 47 | ], 48 | ) 49 | 50 | filegroup( 51 | name = "package-srcs", 52 | srcs = glob(["**"]), 53 | tags = ["automanaged"], 54 | visibility = ["//visibility:private"], 55 | ) 56 | 57 | filegroup( 58 | name = "all-srcs", 59 | srcs = [ 60 | ":package-srcs", 61 | "//pkg/updater/resultstore/query:all-srcs", 62 | ], 63 | tags = ["automanaged"], 64 | visibility = ["//visibility:public"], 65 | ) 66 | -------------------------------------------------------------------------------- /pkg/updater/resultstore/query/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["query.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/pkg/updater/resultstore/query", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | go_test( 11 | name = "go_default_test", 12 | srcs = ["query_test.go"], 13 | embed = [":go_default_library"], 14 | ) 15 | 16 | filegroup( 17 | name = "package-srcs", 18 | srcs = glob(["**"]), 19 | tags = ["automanaged"], 20 | visibility = ["//visibility:private"], 21 | ) 22 | 23 | filegroup( 24 | name = "all-srcs", 25 | srcs = [":package-srcs"], 26 | tags = ["automanaged"], 27 | visibility = ["//visibility:public"], 28 | ) 29 | -------------------------------------------------------------------------------- /pkg/updater/resultstore/query/query.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package query 18 | 19 | import ( 20 | "fmt" 21 | "regexp" 22 | "strings" 23 | ) 24 | 25 | func translateAtom(simpleAtom string) (string, error) { 26 | if simpleAtom == "" { 27 | return "", nil 28 | } 29 | // For now, we expect an atom with the exact form `target:""` 30 | // Split the `key:value` atom. 31 | parts := strings.SplitN(simpleAtom, ":", 2) 32 | if len(parts) != 2 { 33 | return "", fmt.Errorf("unrecognized atom %q", simpleAtom) 34 | } 35 | key := strings.TrimSpace(parts[0]) 36 | val := strings.Trim(strings.TrimSpace(parts[1]), `"`) 37 | 38 | switch { 39 | case key == "target": 40 | return fmt.Sprintf(`id.target_id="%s"`, val), nil 41 | default: 42 | return "", fmt.Errorf("unrecognized atom key %q", key) 43 | } 44 | } 45 | 46 | var ( 47 | queryRe = regexp.MustCompile(`^target:".*"$`) 48 | ) 49 | 50 | func TranslateQuery(simpleQuery string) (string, error) { 51 | if simpleQuery == "" { 52 | return "", nil 53 | } 54 | // For now, we expect a query with a single atom, with the exact form `target:""` 55 | if !queryRe.MatchString(simpleQuery) { 56 | return "", fmt.Errorf("invalid query %q: must match %q", simpleQuery, queryRe.String()) 57 | } 58 | query, err := translateAtom(simpleQuery) 59 | if err != nil { 60 | return "", fmt.Errorf("invalid query %q: %v", simpleQuery, err) 61 | } 62 | return query, nil 63 | } 64 | -------------------------------------------------------------------------------- /platform/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # TODO(fejta): describe this package. 2 | 3 | toolchain( 4 | name = "cc-toolchain", 5 | exec_compatible_with = [ 6 | "@bazel_tools//platforms:linux", 7 | "@bazel_tools//platforms:x86_64", 8 | "@bazel_tools//tools/cpp:clang", 9 | ], 10 | target_compatible_with = [ 11 | "@bazel_tools//platforms:linux", 12 | "@bazel_tools//platforms:x86_64", 13 | ], 14 | toolchain = "//cc:cc-compiler-k8", 15 | toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", 16 | visibility = ["//visibility:public"], 17 | ) 18 | 19 | platform( 20 | name = "platform", 21 | constraint_values = [ 22 | "@bazel_tools//platforms:linux", 23 | "@bazel_tools//platforms:x86_64", 24 | "@bazel_tools//tools/cpp:clang", 25 | ], 26 | exec_properties = { 27 | "container-image": "docker://l.gcr.io/google/rbe-ubuntu18-04@sha256:48b67b41118dbcdfc265e7335f454fbefa62681ab8d47200971fc7a52fb32054", 28 | "OSFamily": "Linux", 29 | }, 30 | parents = ["@local_config_platform//:host"], 31 | visibility = ["//visibility:public"], 32 | ) 33 | 34 | # TODO(fejta): https://github.com/bazelbuild/bazel-toolchains/blob/dac71231098d891e5c4b74a2078fe9343feef510/rules/exec_properties/exec_properties.bzl#L143 35 | platform( 36 | name = "with_network", 37 | exec_properties = { 38 | "dockerNetwork": "standard", 39 | }, 40 | parents = [":platform"], 41 | visibility = ["//visibility:public"], 42 | ) 43 | 44 | filegroup( 45 | name = "package-srcs", 46 | srcs = glob(["**"]), 47 | tags = ["automanaged"], 48 | visibility = ["//visibility:private"], 49 | ) 50 | 51 | filegroup( 52 | name = "all-srcs", 53 | srcs = [":package-srcs"], 54 | tags = ["automanaged"], 55 | visibility = ["//visibility:public"], 56 | ) 57 | -------------------------------------------------------------------------------- /resultstore/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "client.go", 7 | "resultstore.go", 8 | ], 9 | importpath = "github.com/GoogleCloudPlatform/testgrid/resultstore", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "@com_github_google_uuid//:go_default_library", 13 | "@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto", 14 | "@io_bazel_rules_go//proto/wkt:duration_go_proto", 15 | "@io_bazel_rules_go//proto/wkt:field_mask_go_proto", 16 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 17 | "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", 18 | "@org_golang_google_grpc//:go_default_library", 19 | "@org_golang_google_grpc//credentials:go_default_library", 20 | "@org_golang_google_grpc//credentials/oauth:go_default_library", 21 | "@org_golang_google_grpc//metadata:go_default_library", 22 | ], 23 | ) 24 | 25 | filegroup( 26 | name = "package-srcs", 27 | srcs = glob(["**"]), 28 | tags = ["automanaged"], 29 | visibility = ["//visibility:private"], 30 | ) 31 | 32 | filegroup( 33 | name = "all-srcs", 34 | srcs = [":package-srcs"], 35 | tags = ["automanaged"], 36 | visibility = ["//visibility:public"], 37 | ) 38 | 39 | go_test( 40 | name = "go_default_test", 41 | srcs = [ 42 | "client_test.go", 43 | "resultstore_test.go", 44 | ], 45 | embed = [":go_default_library"], 46 | deps = [ 47 | "@go_googleapis//google/devtools/resultstore/v2:resultstore_go_proto", 48 | "@io_bazel_rules_go//proto/wkt:duration_go_proto", 49 | "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", 50 | ], 51 | ) 52 | -------------------------------------------------------------------------------- /resultstore/README.md: -------------------------------------------------------------------------------- 1 | # TestGrid ResultStore 2 | The ResultStore client creates a CRUD interface for users to interact with the Google ResultStore. 3 | 4 | ## ResultStore invocation searching 5 | The ResultStore client allows users to directly search invocations stored in a 6 | GCP project given a query. The query format is documented in the 7 | [SearchInvocationsRequest 8 | type](https://godoc.org/google.golang.org/genproto/googleapis/devtools/resultstore/v2#SearchInvocationsRequest). 9 | Search will return a list of `resultstore.Invocation` that satisfies the query condition. 10 | 11 | Sample search code snippet 12 | ```go 13 | conn, err := resultstore.Connect(ctx, serviceAccountPath) 14 | if err != nil { 15 | // error handling 16 | } 17 | client := resultstore.NewClient(conn).WithContext(ctx) 18 | invocationClient := client.Invocations() 19 | 20 | projectID := "GCP Project ID" 21 | queryTime := time.Unix(1567800000, 0).Format(time.RFC3339) 22 | query := fmt.Sprintf("timing.start_time>%q", queryTime) 23 | invocationsFieldMask := []string{ 24 | "invocations.name", 25 | "invocations.timing", 26 | "next_page_token", 27 | } 28 | result, err := invocationClient.Search(ctx, projectID, query, invocationsFieldMask...) 29 | ``` 30 | 31 | -------------------------------------------------------------------------------- /resultstore/client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package resultstore 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | resultstore "google.golang.org/genproto/googleapis/devtools/resultstore/v2" 24 | ) 25 | 26 | func TestConvertToInvocations(t *testing.T) { 27 | // Resultstore stores time using the protobuf type. It differs from the 28 | // Golang's Time and Duration type and requires conversion. 29 | now := stamp(time.Now()) 30 | duration := dur(time.Second) 31 | cases := []struct { 32 | name string 33 | response *resultstore.SearchInvocationsResponse 34 | expected []*Invocation 35 | }{ 36 | { 37 | name: "empty response", 38 | response: &resultstore.SearchInvocationsResponse{}, 39 | expected: []*Invocation{}, 40 | }, 41 | { 42 | name: "single response", 43 | response: &resultstore.SearchInvocationsResponse{ 44 | Invocations: []*resultstore.Invocation{ 45 | { 46 | Name: "invocations/fakeid-12345", 47 | Id: &resultstore.Invocation_Id{ 48 | InvocationId: "fakeid-12345", 49 | }, 50 | StatusAttributes: &resultstore.StatusAttributes{ 51 | Status: resultstore.Status_PASSED, 52 | }, 53 | Timing: &resultstore.Timing{ 54 | StartTime: now, 55 | Duration: duration, 56 | }, 57 | }, 58 | }, 59 | }, 60 | expected: []*Invocation{ 61 | { 62 | Name: "invocations/fakeid-12345", 63 | Duration: protoDurationToGoDuration(duration), 64 | Start: protoTimeToGoTime(now), 65 | Status: resultstore.Status_PASSED, 66 | }, 67 | }, 68 | }, 69 | } 70 | 71 | for _, tc := range cases { 72 | t.Run(tc.name, func(t *testing.T) { 73 | got := convertToInvocations(tc.response) 74 | if !deepEqual(got, tc.expected) { 75 | t.Errorf(diff(got, tc.expected)) 76 | } 77 | }) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /terraform/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TestGrid Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | tf = "/google/data/ro/teams/terraform/bin/terraform" 16 | 17 | .PHONY: check-install 18 | check-install: 19 | ifeq ($(shell $(tf) -version),) 20 | @echo "ERROR: terraform is not installed"; exit 1 21 | endif 22 | 23 | .PHONY: init-no-backend 24 | init-no-backend: check-install 25 | $(tf) init --backend=false 26 | 27 | .PHONY: validate 28 | validate: init-no-backend 29 | $(tf) validate -------------------------------------------------------------------------------- /terraform/README.md: -------------------------------------------------------------------------------- 1 | (Modified from https://github.com/GoogleCloudPlatform/oss-test-infra/tree/master/prow/oss/terraform) 2 | 3 | ## Terraform 4 | 5 | This directory contains terrafrom configurations for provisioning monitoring and alerting stacks on GCP for TestGrid. These are applied manually. 6 | 7 | ### Prerequisite For Provisioning 8 | 9 | - Terraform 0.13.1 10 | [Installation guide](https://www.terraform.io/downloads.html) 11 | 12 | - Authenticate with GCP 13 | 14 | ```shell 15 | gcloud auth login && gcloud auth application-default login 16 | ``` 17 | 18 | ### Initial Setup (One time action) 19 | 20 | This is done once before initial provisioning of monitoring and alerting stacks. 21 | 22 | ```shell 23 | gsutil mb -p k8s-testgrid gs://k8s-testgrid-metrics-terraform 24 | gsutil versioning set on gs://k8s-testgrid-metrics-terraform 25 | ``` 26 | 27 | ### Provisioning 28 | 29 | 1. Run `terraform init`. Terraform will automatically download the plugins 30 | required to execute this code. You only need to do this once per machine. 31 | 32 | ```shell 33 | terraform init 34 | ``` 35 | 36 | 1. Validate with: 37 | 38 | ```shell 39 | make validate 40 | ``` 41 | 42 | 1. Execute Terraform: 43 | 44 | ```shell 45 | terraform apply 46 | ``` 47 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TestGrid Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Store terraform states in GCS 16 | terraform { 17 | backend "gcs" { 18 | bucket = "k8s-testgrid-metrics-terraform" 19 | } 20 | } 21 | 22 | module "alert" { 23 | source = "./modules/alerts" 24 | 25 | project = "k8s-testgrid" 26 | # gcloud alpha monitoring channels list --project=k8s-testgrid 27 | # grep displayName: Michelle 28 | notification_channel_id = "12611470047778396886" 29 | 30 | blackbox_probers = [ 31 | // Production 32 | // Check both the original and the aliased URLs 33 | "k8s-testgrid.appspot.com", 34 | "testgrid.k8s.io", 35 | // Canary 36 | "external-canary-dot-k8s-testgrid.appspot.com", 37 | ] 38 | 39 | pubsub_topics = [ 40 | "canary-testgrid", 41 | "testgrid", 42 | ] 43 | } -------------------------------------------------------------------------------- /terraform/modules/alerts/probers.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TestGrid Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | resource "google_monitoring_uptime_check_config" "https" { 16 | project = var.project 17 | selected_regions = [] 18 | 19 | for_each = var.blackbox_probers 20 | 21 | display_name = each.key 22 | timeout = "10s" 23 | period = "60s" 24 | 25 | http_check { 26 | port = "443" 27 | use_ssl = true 28 | validate_ssl = true 29 | } 30 | 31 | monitored_resource { 32 | type = "uptime_url" 33 | labels = { 34 | project_id = var.project 35 | host = each.key 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /terraform/modules/alerts/variables.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TestGrid Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | variable "project" { 16 | type = string 17 | } 18 | 19 | variable "notification_channel_id" { 20 | type = string 21 | } 22 | 23 | variable "blackbox_probers" { 24 | type = set(string) 25 | default = [] 26 | } 27 | 28 | variable "pubsub_topics" { 29 | type = set(string) 30 | default = [] 31 | } -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The TestGrid Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | provider "google" { 16 | project = "k8s-testgrid" 17 | region = "us-west1" 18 | } 19 | provider "google-beta" { 20 | project = "k8s-testgrid" 21 | region = "us-west1" 22 | } -------------------------------------------------------------------------------- /util/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "links.go", 7 | "log.go", 8 | "strings.go", 9 | ], 10 | importpath = "github.com/GoogleCloudPlatform/testgrid/util", 11 | visibility = ["//visibility:public"], 12 | deps = [ 13 | "//pb/config:go_default_library", 14 | "@com_github_sirupsen_logrus//:go_default_library", 15 | ], 16 | ) 17 | 18 | filegroup( 19 | name = "package-srcs", 20 | srcs = glob(["**"]), 21 | tags = ["automanaged"], 22 | visibility = ["//visibility:private"], 23 | ) 24 | 25 | filegroup( 26 | name = "all-srcs", 27 | srcs = [ 28 | ":package-srcs", 29 | "//util/gcs:all-srcs", 30 | "//util/metrics:all-srcs", 31 | "//util/queue:all-srcs", 32 | ], 33 | tags = ["automanaged"], 34 | visibility = ["//visibility:public"], 35 | ) 36 | 37 | go_test( 38 | name = "go_default_test", 39 | srcs = ["links_test.go"], 40 | embed = [":go_default_library"], 41 | deps = [ 42 | "//pb/config:go_default_library", 43 | "@com_github_google_go_cmp//cmp:go_default_library", 44 | "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /util/gcs/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "client.go", 7 | "gcs.go", 8 | "local_gcs.go", 9 | "read.go", 10 | "real_gcs.go", 11 | "sort.go", 12 | ], 13 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/gcs", 14 | visibility = ["//visibility:public"], 15 | deps = [ 16 | "//metadata:go_default_library", 17 | "//metadata/junit:go_default_library", 18 | "//pb/state:go_default_library", 19 | "@com_github_fvbommel_sortorder//:go_default_library", 20 | "@com_github_golang_protobuf//proto:go_default_library", 21 | "@com_github_sirupsen_logrus//:go_default_library", 22 | "@com_google_cloud_go_storage//:go_default_library", 23 | "@io_k8s_api//core/v1:go_default_library", 24 | "@org_golang_google_api//googleapi:go_default_library", 25 | "@org_golang_google_api//iterator:go_default_library", 26 | "@org_golang_google_api//option:go_default_library", 27 | ], 28 | ) 29 | 30 | go_test( 31 | name = "go_default_test", 32 | srcs = [ 33 | "gcs_test.go", 34 | "local_gcs_test.go", 35 | "read_test.go", 36 | "real_gcs_test.go", 37 | "sort_test.go", 38 | ], 39 | embed = [":go_default_library"], 40 | deps = [ 41 | "//metadata:go_default_library", 42 | "//metadata/junit:go_default_library", 43 | "//pb/state:go_default_library", 44 | "@com_github_golang_protobuf//proto:go_default_library", 45 | "@com_github_google_go_cmp//cmp:go_default_library", 46 | "@com_google_cloud_go_storage//:go_default_library", 47 | "@io_k8s_api//core/v1:go_default_library", 48 | "@org_golang_google_api//googleapi:go_default_library", 49 | "@org_golang_google_api//iterator:go_default_library", 50 | ], 51 | ) 52 | 53 | filegroup( 54 | name = "package-srcs", 55 | srcs = glob(["**"]), 56 | tags = ["automanaged"], 57 | visibility = ["//visibility:private"], 58 | ) 59 | 60 | filegroup( 61 | name = "all-srcs", 62 | srcs = [ 63 | ":package-srcs", 64 | "//util/gcs/fake:all-srcs", 65 | ], 66 | tags = ["automanaged"], 67 | visibility = ["//visibility:public"], 68 | ) 69 | -------------------------------------------------------------------------------- /util/gcs/fake/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["fake.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/gcs/fake", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//util/gcs:go_default_library", 10 | "@com_google_cloud_go_storage//:go_default_library", 11 | "@org_golang_google_api//googleapi:go_default_library", 12 | "@org_golang_google_api//iterator:go_default_library", 13 | ], 14 | ) 15 | 16 | go_test( 17 | name = "go_default_test", 18 | srcs = [ 19 | "fake_test.go", 20 | "sort_test.go", 21 | ], 22 | embed = [":go_default_library"], 23 | deps = [ 24 | "//util/gcs:go_default_library", 25 | "@com_github_google_go_cmp//cmp:go_default_library", 26 | "@com_github_sirupsen_logrus//:go_default_library", 27 | "@com_google_cloud_go_storage//:go_default_library", 28 | "@org_golang_google_api//googleapi:go_default_library", 29 | ], 30 | ) 31 | 32 | filegroup( 33 | name = "package-srcs", 34 | srcs = glob(["**"]), 35 | tags = ["automanaged"], 36 | visibility = ["//visibility:private"], 37 | ) 38 | 39 | filegroup( 40 | name = "all-srcs", 41 | srcs = [":package-srcs"], 42 | tags = ["automanaged"], 43 | visibility = ["//visibility:public"], 44 | ) 45 | -------------------------------------------------------------------------------- /util/gcs/fake/fake_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package fake 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/GoogleCloudPlatform/testgrid/util/gcs" 23 | ) 24 | 25 | func TestInterfaces(t *testing.T) { 26 | var ( 27 | _ gcs.Uploader = &Uploader{} 28 | _ gcs.Downloader = &Client{} 29 | _ gcs.Lister = &Lister{} 30 | _ gcs.Iterator = &Iterator{} 31 | _ gcs.Opener = &Opener{} 32 | _ gcs.Copier = &Uploader{} 33 | _ gcs.Client = &UploadClient{} 34 | _ gcs.ConditionalClient = &UploadClient{} 35 | ) 36 | } 37 | -------------------------------------------------------------------------------- /util/gcs/real_gcs_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package gcs 18 | 19 | import ( 20 | "errors" 21 | "net/http" 22 | "testing" 23 | 24 | "cloud.google.com/go/storage" 25 | "github.com/google/go-cmp/cmp" 26 | "google.golang.org/api/googleapi" 27 | ) 28 | 29 | func TestWrapGoogleAPIError(t *testing.T) { 30 | create := func(code int) error { 31 | var e googleapi.Error 32 | e.Code = code 33 | return &e 34 | } 35 | cases := []struct { 36 | name string 37 | err error 38 | want string 39 | is error 40 | }{ 41 | { 42 | name: "nil", 43 | }, 44 | { 45 | name: "basic", 46 | err: errors.New("hi"), 47 | want: errors.New("hi").Error(), 48 | }, 49 | { 50 | name: "random code", 51 | err: create(http.StatusInternalServerError), 52 | want: create(http.StatusInternalServerError).Error(), 53 | }, 54 | { 55 | name: "404", 56 | err: create(http.StatusNotFound), 57 | want: create(http.StatusNotFound).Error(), 58 | is: storage.ErrObjectNotExist, 59 | }, 60 | } 61 | 62 | for _, tc := range cases { 63 | t.Run(tc.name, func(t *testing.T) { 64 | gotErr := wrapGoogleAPIError(tc.err) 65 | var got string 66 | if gotErr != nil { 67 | got = gotErr.Error() 68 | } 69 | if diff := cmp.Diff(tc.want, got); diff != "" { 70 | t.Errorf("wrapGoogleAPIError() got unexpected diff (-want +got):\n%s", diff) 71 | } 72 | if tc.is != nil && !errors.Is(gotErr, tc.is) { 73 | t.Errorf("errors.Is(%v, %v) returned false", gotErr, tc.is) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /util/gcs/sort_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package gcs 18 | 19 | import ( 20 | "net/url" 21 | "testing" 22 | 23 | "github.com/google/go-cmp/cmp" 24 | ) 25 | 26 | // TestLeastRecentlyUpdated in util/gcs/fake/sort_test 27 | // TestTouch in util/gcs/fake/sort_test 28 | 29 | func TestSort(t *testing.T) { 30 | cases := []struct { 31 | name string 32 | builds []Build 33 | want []Build 34 | }{ 35 | { 36 | name: "basic", 37 | }, 38 | { 39 | name: "sorted", 40 | builds: []Build{ 41 | {baseName: "c"}, 42 | {baseName: "b"}, 43 | {baseName: "a"}, 44 | }, 45 | want: []Build{ 46 | {baseName: "c"}, 47 | {baseName: "b"}, 48 | {baseName: "a"}, 49 | }, 50 | }, 51 | { 52 | name: "stable", // already sorted elements do not move 53 | builds: []Build{ 54 | { 55 | Path: Path{url: url.URL{Host: "foo"}}, 56 | baseName: "c", 57 | }, 58 | { 59 | Path: Path{url: url.URL{Host: "bar"}}, 60 | baseName: "c", 61 | }, 62 | { 63 | Path: Path{url: url.URL{Path: "other"}}, 64 | baseName: "c", 65 | }, 66 | {baseName: "a"}, 67 | {baseName: "b"}, 68 | }, 69 | want: []Build{ 70 | { 71 | Path: Path{url: url.URL{Host: "foo"}}, 72 | baseName: "c", 73 | }, 74 | { 75 | Path: Path{url: url.URL{Host: "bar"}}, 76 | baseName: "c", 77 | }, 78 | { 79 | Path: Path{url: url.URL{Path: "other"}}, 80 | baseName: "c", 81 | }, 82 | {baseName: "b"}, 83 | {baseName: "a"}, 84 | }, 85 | }, 86 | { 87 | name: "resort", 88 | builds: []Build{ 89 | {baseName: "b"}, 90 | {baseName: "c"}, 91 | {baseName: "a"}, 92 | }, 93 | want: []Build{ 94 | {baseName: "c"}, 95 | {baseName: "b"}, 96 | {baseName: "a"}, 97 | }, 98 | }, 99 | { 100 | name: "numerics", 101 | builds: []Build{ 102 | {baseName: "a1b"}, 103 | {baseName: "a10b"}, 104 | {baseName: "a2b"}, 105 | {baseName: "a3b"}, 106 | }, 107 | want: []Build{ 108 | {baseName: "a10b"}, 109 | {baseName: "a3b"}, 110 | {baseName: "a2b"}, 111 | {baseName: "a1b"}, 112 | }, 113 | }, 114 | } 115 | 116 | for _, tc := range cases { 117 | t.Run(tc.name, func(t *testing.T) { 118 | Sort(tc.builds) 119 | if diff := cmp.Diff(tc.want, tc.builds, cmp.AllowUnexported(Build{}, Path{})); diff != "" { 120 | t.Errorf("Sort() got unexpected diff (-want +got):\n%s", diff) 121 | } 122 | }) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /util/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The TestGrid Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package util has convenience functions for use throughout TestGrid. 18 | package util 19 | 20 | import ( 21 | "context" 22 | "time" 23 | 24 | "github.com/sirupsen/logrus" 25 | ) 26 | 27 | // Progress log every duration, including an ETA for completion. 28 | // Returns a function for updating the current index 29 | func Progress(ctx context.Context, log logrus.FieldLogger, every time.Duration, total int, msg string) func(int) { 30 | start := time.Now() 31 | ch := make(chan int, 1) 32 | go func() { 33 | timer := time.NewTimer(every) 34 | defer timer.Stop() 35 | var current int 36 | for { 37 | select { 38 | case <-ctx.Done(): 39 | return 40 | case current = <-ch: 41 | // updated index 42 | case now := <-timer.C: 43 | elapsed := now.Sub(start) 44 | var rate time.Duration 45 | if current > 0 { 46 | rate = elapsed / time.Duration(current) 47 | } 48 | eta := time.Duration(total-current) * rate 49 | 50 | log.WithFields(logrus.Fields{ 51 | "current": current, 52 | "total": total, 53 | "percent": (100 * current) / total, 54 | "remain": eta.Round(time.Minute), 55 | "eta": now.Add(eta).Round(time.Minute), 56 | "start": start.Round(time.Minute), 57 | }).Info(msg) 58 | timer.Reset(every) 59 | } 60 | } 61 | }() 62 | 63 | return func(idx int) { 64 | select { 65 | case ch <- idx: 66 | default: 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /util/metrics/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["metrics.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/metrics", 7 | visibility = ["//visibility:public"], 8 | ) 9 | 10 | filegroup( 11 | name = "package-srcs", 12 | srcs = glob(["**"]), 13 | tags = ["automanaged"], 14 | visibility = ["//visibility:private"], 15 | ) 16 | 17 | filegroup( 18 | name = "all-srcs", 19 | srcs = [ 20 | ":package-srcs", 21 | "//util/metrics/logmetrics:all-srcs", 22 | "//util/metrics/prometheus:all-srcs", 23 | ], 24 | tags = ["automanaged"], 25 | visibility = ["//visibility:public"], 26 | ) 27 | 28 | go_test( 29 | name = "go_default_test", 30 | srcs = ["metrics_test.go"], 31 | embed = [":go_default_library"], 32 | ) 33 | -------------------------------------------------------------------------------- /util/metrics/logmetrics/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["log.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/metrics/logmetrics", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//util/metrics:go_default_library", 10 | "@com_github_sirupsen_logrus//:go_default_library", 11 | "@org_bitbucket_creachadair_stringset//:go_default_library", 12 | ], 13 | ) 14 | 15 | filegroup( 16 | name = "package-srcs", 17 | srcs = glob(["**"]), 18 | tags = ["automanaged"], 19 | visibility = ["//visibility:private"], 20 | ) 21 | 22 | filegroup( 23 | name = "all-srcs", 24 | srcs = [":package-srcs"], 25 | tags = ["automanaged"], 26 | visibility = ["//visibility:public"], 27 | ) 28 | 29 | go_test( 30 | name = "go_default_test", 31 | srcs = ["log_test.go"], 32 | embed = [":go_default_library"], 33 | deps = [ 34 | "@com_github_google_go_cmp//cmp:go_default_library", 35 | "@com_github_sirupsen_logrus//:go_default_library", 36 | ], 37 | ) 38 | -------------------------------------------------------------------------------- /util/metrics/prometheus/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = ["prometheus.go"], 6 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/metrics/prometheus", 7 | visibility = ["//visibility:public"], 8 | deps = [ 9 | "//util/metrics:go_default_library", 10 | "@com_github_prometheus_client_golang//prometheus:go_default_library", 11 | "@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library", 12 | "@com_github_prometheus_client_model//go:go_default_library", 13 | ], 14 | ) 15 | 16 | filegroup( 17 | name = "package-srcs", 18 | srcs = glob(["**"]), 19 | tags = ["automanaged"], 20 | visibility = ["//visibility:private"], 21 | ) 22 | 23 | filegroup( 24 | name = "all-srcs", 25 | srcs = [":package-srcs"], 26 | tags = ["automanaged"], 27 | visibility = ["//visibility:public"], 28 | ) 29 | 30 | go_test( 31 | name = "go_default_test", 32 | srcs = ["prometheus_test.go"], 33 | embed = [":go_default_library"], 34 | deps = ["@com_github_google_go_cmp//cmp:go_default_library"], 35 | ) 36 | -------------------------------------------------------------------------------- /util/queue/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") 2 | 3 | go_library( 4 | name = "go_default_library", 5 | srcs = [ 6 | "persist.go", 7 | "queue.go", 8 | ], 9 | importpath = "github.com/GoogleCloudPlatform/testgrid/util/queue", 10 | visibility = ["//visibility:public"], 11 | deps = [ 12 | "//util/gcs:go_default_library", 13 | "@com_github_sirupsen_logrus//:go_default_library", 14 | "@com_google_cloud_go_storage//:go_default_library", 15 | ], 16 | ) 17 | 18 | go_test( 19 | name = "go_default_test", 20 | srcs = [ 21 | "persist_test.go", 22 | "queue_test.go", 23 | ], 24 | embed = [":go_default_library"], 25 | deps = [ 26 | "//util/gcs:go_default_library", 27 | "//util/gcs/fake:go_default_library", 28 | "@com_github_google_go_cmp//cmp:go_default_library", 29 | "@com_github_sirupsen_logrus//:go_default_library", 30 | "@com_google_cloud_go_storage//:go_default_library", 31 | "@org_golang_google_protobuf//testing/protocmp:go_default_library", 32 | ], 33 | ) 34 | 35 | filegroup( 36 | name = "package-srcs", 37 | srcs = glob(["**"]), 38 | tags = ["automanaged"], 39 | visibility = ["//visibility:private"], 40 | ) 41 | 42 | filegroup( 43 | name = "all-srcs", 44 | srcs = [":package-srcs"], 45 | tags = ["automanaged"], 46 | visibility = ["//visibility:public"], 47 | ) 48 | -------------------------------------------------------------------------------- /util/strings.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "strings" 21 | ) 22 | 23 | // Strings represents the value of a flag that accept multiple strings. 24 | type Strings struct { 25 | vals []string 26 | } 27 | 28 | // Strings returns the slice of strings set for this value instance. 29 | func (s *Strings) Strings() []string { 30 | return s.vals 31 | } 32 | 33 | // String returns a concatenated string of all the values joined by commas. 34 | func (s *Strings) String() string { 35 | return strings.Join(s.vals, ",") 36 | } 37 | 38 | // Set records the value passed 39 | func (s *Strings) Set(value string) error { 40 | s.vals = append(s.vals, value) 41 | return nil 42 | } 43 | --------------------------------------------------------------------------------