├── .gitignore ├── .go-version ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── RELEASE.md ├── SECURITY_CONTACTS ├── code-of-conduct.md ├── go.mod ├── go.sum ├── hack ├── boilerplate │ ├── boilerplate.Dockerfile.txt │ ├── boilerplate.Makefile.txt │ ├── boilerplate.generatego.txt │ ├── boilerplate.go.txt │ ├── boilerplate.py │ ├── boilerplate.py.txt │ ├── boilerplate.sh.txt │ ├── boilerplate_test.py │ └── test │ │ ├── fail.go │ │ ├── fail.py │ │ ├── pass.go │ │ └── pass.py ├── build │ ├── ci-binaries.sh │ ├── goinstalldir.sh │ └── setup-go.sh ├── ci │ ├── push-binaries │ │ ├── cloudbuild.yaml │ │ └── push-binaries.sh │ └── unit.sh ├── third_party │ └── gimme │ │ ├── LICENSE │ │ ├── README.md │ │ └── gimme ├── tools │ ├── .golangci.yml │ ├── go.mod │ ├── go.sum │ └── tools.go ├── update │ ├── gofmt.sh │ └── tidy.sh └── verify │ ├── boilerplate.sh │ ├── go-version.sh │ ├── lint.sh │ ├── shellcheck.sh │ ├── tidy.sh │ └── tidy_quiet.sh ├── kubetest2-gce ├── README.md ├── ci-tests │ ├── buildupdown-legacy.sh │ ├── buildupdown.sh │ └── updown-legacy.sh ├── deployer │ ├── build.go │ ├── build_test.go │ ├── common.go │ ├── deployer.go │ ├── down.go │ ├── dumplogs.go │ ├── firewall.go │ ├── options │ │ └── build.go │ └── up.go └── main.go ├── kubetest2-gke ├── OWNERS ├── ci-tests │ ├── buildupdown.sh │ └── test.sh ├── deployer │ ├── build.go │ ├── build │ │ └── gke_make.go │ ├── build_test.go │ ├── commandutils.go │ ├── common.go │ ├── deployer.go │ ├── deployer_test.go │ ├── down.go │ ├── dumplogs.go │ ├── firewall.go │ ├── network.go │ ├── network_test.go │ ├── options │ │ ├── build.go │ │ ├── cluster.go │ │ ├── common.go │ │ ├── network.go │ │ └── project.go │ ├── post_tester.go │ ├── up.go │ ├── up_test.go │ ├── utils │ │ ├── util.go │ │ └── util_test.go │ ├── version.go │ └── version_test.go └── main.go ├── kubetest2-kind ├── README.md ├── ci-tests │ └── buildupdown.sh ├── deployer │ ├── build.go │ ├── deployer.go │ ├── down.go │ ├── dumplogs.go │ └── up.go └── main.go ├── kubetest2-noop ├── deployer │ └── deployer.go └── main.go ├── kubetest2-tester-clusterloader2 └── main.go ├── kubetest2-tester-exec └── main.go ├── kubetest2-tester-ginkgo └── main.go ├── kubetest2-tester-node ├── ci-tests │ └── gce-test.sh └── main.go ├── main.go └── pkg ├── app ├── app.go ├── cmd.go ├── doc.go └── shim │ ├── const.go │ ├── deployers.go │ ├── doc.go │ ├── shim.go │ └── testers.go ├── artifacts └── paths.go ├── boskos └── boskos.go ├── build ├── bazel.go ├── build.go ├── krel.go ├── make.go ├── options.go └── stage.go ├── exec ├── doc.go ├── exec.go └── local.go ├── fs └── fs.go ├── metadata ├── junit.go ├── metadata.go ├── metadata_test.go ├── writer.go └── writer_test.go ├── process ├── doc.go ├── exec.go ├── junitexec.go └── mutexwriter.go ├── testers ├── clusterloader2 │ ├── cl2.go │ └── suite │ │ └── suite.go ├── exec │ ├── exec.go │ └── exec_test.go ├── ginkgo │ ├── ginkgo.go │ ├── kubectl │ │ └── kubectl.go │ └── package.go ├── metadata.go └── node │ └── node.go ├── types ├── helpers.go └── types.go └── util ├── util.go └── util_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Intellij specific files 2 | .idea/ 3 | *.iml 4 | 5 | # kubetest2 default artifacts 6 | _artifacts/ 7 | bin/ 8 | 9 | # VisualStudio Code specific files 10 | .vscode/ 11 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.22.10 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: 4 | 5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ 6 | 7 | ## Getting Started 8 | 9 | We have full documentation on how to get started contributing here: 10 | 11 | 14 | 15 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests 16 | - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) 17 | - [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet) - Common resources for existing developers 18 | 19 | ## Mentorship 20 | 21 | - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! 22 | 23 | ## Contact Information 24 | 25 | - [Slack](https://kubernetes.slack.com/messages/sig-testing) 26 | - [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-testing) 27 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Common uses: 16 | # - installing kubetest2: `make install INSTALL_DIR=$HOME/go/bin` 17 | # installing a deployer: `make install-deployer-$(deployer-name) INSTALL_DIR=$HOME/go/bin` 18 | # - cleaning up and starting over: `make clean` 19 | 20 | # get the repo root and output path 21 | REPO_ROOT:=$(shell pwd) 22 | export REPO_ROOT 23 | OUT_DIR=$(REPO_ROOT)/bin 24 | # record the source commit in the binary, overridable 25 | COMMIT?=$(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty 2>/dev/null) 26 | INSTALL?=install 27 | # make install will place binaries here 28 | # the default path attempts to mimic go install 29 | INSTALL_DIR?=$(shell $(REPO_ROOT)/hack/build/goinstalldir.sh) 30 | # the output binary name, overridden when cross compiling 31 | BINARY_NAME?=kubetest2 32 | BINARY_PATH?=. 33 | # the container cli to use e.g. docker,podman 34 | DOCKER?=$(shell which docker || which podman || echo "docker") 35 | export DOCKER 36 | # ========================= Setup Go With Gimme ================================ 37 | # go version to use for build etc. 38 | # setup correct go version with gimme 39 | PATH:=$(shell . hack/build/setup-go.sh && echo "$${PATH}") 40 | # go1.9+ can autodetect GOROOT, but if some other tool sets it ... 41 | GOROOT:= 42 | # enable modules 43 | GO111MODULE=on 44 | # disable CGO by default for static binaries 45 | CGO_ENABLED=0 46 | export PATH GOROOT GO111MODULE CGO_ENABLED 47 | # work around broken PATH export 48 | SPACE:=$(subst ,, ) 49 | SHELL:=env PATH=$(subst $(SPACE),\$(SPACE),$(PATH)) $(SHELL) 50 | # ============================================================================== 51 | # flags for reproducible go builds 52 | BUILD_FLAGS?=-trimpath -ldflags="-buildid=" 53 | 54 | build-all: 55 | go build -v $(BUILD_FLAGS) ./... 56 | 57 | install: BUILD_FLAGS=-trimpath -ldflags="-buildid= -X=sigs.k8s.io/kubetest2/pkg/app/shim.GitTag=$(COMMIT)" 58 | install: 59 | go build -v $(BUILD_FLAGS) -o $(OUT_DIR)/$(BINARY_NAME) $(BINARY_PATH) 60 | $(INSTALL) -d $(INSTALL_DIR) 61 | $(INSTALL) $(OUT_DIR)/$(BINARY_NAME) $(INSTALL_DIR)/$(BINARY_NAME) 62 | 63 | install-deployer-%: BINARY_PATH=./kubetest2-$* 64 | install-deployer-%: BINARY_NAME=kubetest2-$* 65 | install-deployer-%: BUILD_FLAGS=-trimpath -ldflags="-buildid= -X=sigs.k8s.io/kubetest2/kubetest2-$*/deployer.GitTag=$(COMMIT)" 66 | install-deployer-%: 67 | go build -v $(BUILD_FLAGS) -o $(OUT_DIR)/$(BINARY_NAME) $(BINARY_PATH) 68 | $(INSTALL) -d $(INSTALL_DIR) 69 | $(INSTALL) $(OUT_DIR)/$(BINARY_NAME) $(INSTALL_DIR)/$(BINARY_NAME) 70 | 71 | install-tester-%: BINARY_PATH=./kubetest2-tester-$* 72 | install-tester-%: BINARY_NAME=kubetest2-tester-$* 73 | install-tester-%: BUILD_FLAGS=-trimpath -ldflags="-buildid= -X=sigs.k8s.io/kubetest2/pkg/testers/$*.GitTag=$(COMMIT)" 74 | install-tester-%: 75 | go build $(BUILD_FLAGS) -v $(BUILD_OPTS) -o $(OUT_DIR)/$(BINARY_NAME) $(BINARY_PATH) 76 | $(INSTALL) -d $(INSTALL_DIR) 77 | $(INSTALL) $(OUT_DIR)/$(BINARY_NAME) $(INSTALL_DIR)/$(BINARY_NAME) 78 | 79 | install-all: TESTERS := $(wildcard kubetest2-tester-*) 80 | install-all: DEPLOYERS := $(filter-out $(TESTERS), $(wildcard kubetest2-*)) 81 | install-all: TESTER_TARGETS := $(subst kubetest2-tester-,install-tester-, $(TESTERS)) 82 | install-all: DEPLOYER_TARGETS := $(subst kubetest2-,install-deployer-, $(DEPLOYERS)) 83 | install-all: install 84 | $(MAKE) -j $(DEPLOYER_TARGETS) $(TESTER_TARGETS) 85 | 86 | quick-verify: install install-deployer-kind install-tester-exec 87 | kubetest2 kind --up --down --test=exec -- kubectl get all -A 88 | 89 | ci-binaries: 90 | ./hack/build/ci-binaries.sh 91 | 92 | push-ci-binaries: 93 | ./hack/ci/push-binaries/push-binaries.sh 94 | 95 | # cleans the output directory 96 | clean-output: 97 | rm -rf $(OUT_DIR)/ 98 | 99 | # standard cleanup target 100 | clean: clean-output 101 | 102 | fix: 103 | ./hack/update/gofmt.sh 104 | ./hack/update/tidy.sh 105 | 106 | boilerplate: 107 | ./hack/verify/boilerplate.sh 108 | 109 | go-version: 110 | ./hack/verify/go-version.sh 111 | 112 | lint: 113 | ./hack/verify/lint.sh 114 | 115 | shellcheck: 116 | ./hack/verify/shellcheck.sh 117 | 118 | tidy: 119 | ./hack/verify/tidy.sh 120 | 121 | unit: 122 | ./hack/ci/unit.sh 123 | 124 | verify: 125 | $(MAKE) -j lint shellcheck unit tidy boilerplate go-version 126 | 127 | .PHONY: build-all install install-deployer-% install-tester-% install-all ci-binaries push-ci-binaries quick-verify clean-output clean verify lint shellcheck 128 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | reviewers: 4 | - aojea 5 | - richackard 6 | - upodroid 7 | - dims 8 | approvers: 9 | - aojea 10 | - jbpratt # chair 11 | - michelle192837 # chair 12 | - upodroid 13 | - xmcqueen # chair 14 | emeritus_approvers: 15 | - amwat 16 | - BenTheElder 17 | - MushuEE 18 | - spiffxp 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubetest2 2 | 3 | Kubetest2 is a framework for deploying Kubernetes clusters and running end-to-end tests against them. 4 | 5 | It is intended to be the next significant iteration of [kubetest] 6 | 7 | ## Concepts 8 | 9 | kubetest2 is effectively split into three independent executables: 10 | 11 | - `kubetest2`: discovers and invokes deployers and testers in `PATH` 12 | - `kubetest2-DEPLOYER`: manages the lifecycle of a Kubernetes cluster 13 | - `kubetest2-tester-TESTER`: tests a Kubernetes cluster 14 | 15 | The intent behind this design is: 16 | - minimize coupling between deployers and testers 17 | - encourage implementation of new deployers and testers out-of-tree 18 | - keep dependencies / surface area of kubetest2 small 19 | 20 | We provide [reference implementations](#reference-implementations) but all 21 | all new implementations should be [external implementations](#external-implementations) 22 | 23 | ## Installation 24 | 25 | To install kubetest2 and all reference deployers and testers: 26 | `go install sigs.k8s.io/kubetest2/...@latest` 27 | 28 | To install a specific deployer: 29 | `go install sigs.k8s.io/kubetest2/kubetest2-DEPLOYER@latest` (DEPLOYER can be `gce`, `gke`, etc.) 30 | 31 | To install a sepcific tester: 32 | `go install sigs.k8s.io/kubetest2/kubetest2-tester-TESTER@latest` (TESTER can be `ginkgo`, `exec`, etc.) 33 | 34 | ## Usage 35 | 36 | General usage is of the form: 37 | ``` 38 | kubetest2 [Flags] [DeployerFlags] -- [TesterArgs] 39 | ``` 40 | 41 | **Example**: list all flags for the `noop` deployer and `ginkgo` tester 42 | ``` 43 | kubetest2 noop --test=ginkgo --help 44 | ``` 45 | 46 | **Example**: deploy a cluster using a local checkout of `kubernetes/kubernetes`, run Conformance tests 47 | ``` 48 | kubetest2 gce -v 2 \ 49 | --repo-root $KK_REPO_ROOT \ 50 | --gcp-project $YOUR_GCP_PROJECT \ 51 | --legacy-mode \ 52 | --build \ 53 | --up \ 54 | --down \ 55 | --test=ginkgo \ 56 | -- \ 57 | --focus-regex='\[Conformance\]' 58 | ``` 59 | 60 | ## Reference Implementations 61 | 62 | See individual READMEs for more information 63 | 64 | **Deployers** 65 | - [`kubetest2-gce`](/kubetest2-gce) - use scripts in `kubernetes/cloud-provider-gcp` or `kubernetes/kubernetes` 66 | - [`kubetest2-gke`](/kubetest2-gke) - use `gcloud containers` 67 | - [`kubetest2-kind`](/kubetest2-kind) - use `kind` 68 | - [`kubetest2-noop`](/kubetest2-noop) - do nothing (to use a pre-existing cluster) 69 | 70 | **Testers** 71 | - [`kubetest2-tester-clusterloader2`](/kubetest2-tester-clusterloader2) - use clusterloader2 72 | - [`kubetest2-tester-exec`](/kubetest2-tester-exec) - exec a given command with the given args / flags 73 | - [`kubetest2-tester-ginkgo`](/kubetest2-tester-ginkgo) - runs e2e tests from `kubernetes/kubernetes` 74 | - [`kubetest2-tester-node`](/kubetest2-tester-node) - runs node e2e tests from `kubernetes/kubernetes` 75 | 76 | ## External Implementations 77 | 78 | **Deployers** 79 | - [`kubetest2-aks`][kubetest2-aks] 80 | - [`kubetest2-kops`][kubetest2-kops] 81 | - [`kubetest2-tf`][kubetest2-tf] 82 | - [`kubetest2-ec2`][kubetest2-ec2] 83 | 84 | **Testers** 85 | - [`kubetest2-tester-kops`][kubetest2-tester-kops] 86 | 87 | ## Support 88 | 89 | This project is currently unversioned and unreleased. We make a best-effort attempt to enforce the following: 90 | - `kubetest2` and its reference implementations must work with the in-development version of kubernetes and all [currently supported kubernetes releases][k8s-supported-releases] 91 | - e.g. no generics until older supported kubernetes version supports generics 92 | - e.g. ginkgo tester must work with both ginkgo v1 and ginkgo v2 93 | - changes to the following testers must not break jobs in the kubernetes project 94 | - `kubetest2-tester-exec` 95 | - `kubetest2-tester-ginkgo` 96 | 97 | ### Contact 98 | 99 | Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). 100 | 101 | You can reach the maintainers of this project at: 102 | 103 | - [Slack](https://kubernetes.slack.com/messages/sig-testing) 104 | - [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-testing) 105 | 106 | ### Code of conduct 107 | 108 | Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). 109 | 110 | 111 | [kubetest]: https://git.k8s.io/test-infra/kubetest 112 | [kubetest2-aks]: https://sigs.k8s.io/cloud-provider-azure/kubetest2-aks 113 | [kubetest2-kops]: https://git.k8s.io/kops/tests/e2e/kubetest2-kops 114 | [kubetest2-tf]: https://github.com/ppc64le-cloud/kubetest2-plugins/tree/master/kubetest2-tf 115 | [kubetest2-ec2]: https://github.com/kubernetes-sigs/provider-aws-test-infra/tree/main/kubetest2-ec2 116 | [kubetest2-tester-kops]: https://git.k8s.io/kops/tests/e2e/kubetest2-tester-kops 117 | [k8s-supported-releases]: https://kubernetes.io/releases/patch-releases/#support-period 118 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | The Kubernetes Template Project is released on an as-needed basis. The process is as follows: 4 | 5 | 1. An issue is proposing a new release with a changelog since the last release 6 | 1. All [OWNERS](OWNERS) must LGTM this release 7 | 1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` 8 | 1. The release issue is closed 9 | 1. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` 10 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Committee to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | amwat 14 | BenTheElder 15 | spiffxp 16 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.Dockerfile.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.Makefile.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.generatego.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright YEAR The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.py.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.sh.txt: -------------------------------------------------------------------------------- 1 | # Copyright YEAR The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2016 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import boilerplate 18 | import unittest 19 | from io import StringIO 20 | import os 21 | import sys 22 | 23 | class TestBoilerplate(unittest.TestCase): 24 | """ 25 | Note: run this test from the hack/boilerplate directory. 26 | 27 | $ python -m unittest boilerplate_test 28 | """ 29 | 30 | def test_boilerplate(self): 31 | os.chdir("test/") 32 | 33 | class Args(object): 34 | def __init__(self): 35 | self.filenames = [] 36 | self.rootdir = "." 37 | self.boilerplate_dir = "../" 38 | self.verbose = True 39 | 40 | # capture stdout 41 | old_stdout = sys.stdout 42 | sys.stdout = StringIO.StringIO() 43 | 44 | boilerplate.args = Args() 45 | ret = boilerplate.main() 46 | 47 | output = sorted(sys.stdout.getvalue().split()) 48 | 49 | sys.stdout = old_stdout 50 | 51 | self.assertEqual( 52 | output, ['././fail.go', '././fail.py']) 53 | -------------------------------------------------------------------------------- /hack/boilerplate/test/fail.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Kubernetes Authors. 3 | 4 | fail 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package test 20 | -------------------------------------------------------------------------------- /hack/boilerplate/test/fail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2015 The Kubernetes Authors. 4 | # 5 | # failed 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /hack/boilerplate/test/pass.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package test 18 | -------------------------------------------------------------------------------- /hack/boilerplate/test/pass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2015 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | True 18 | -------------------------------------------------------------------------------- /hack/build/ci-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o xtrace 20 | 21 | # this script is invoked in GCB as the entrypoint 22 | # avoid prow potentially not being in the right working directory 23 | REPO_ROOT="${REPO_ROOT:-"$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &> /dev/null && pwd -P)"}" 24 | cd "${REPO_ROOT}" &> /dev/null 25 | source hack/build/setup-go.sh 26 | 27 | make clean 28 | 29 | OS_ARCHES=( 30 | linux_amd64 31 | linux_arm64 32 | linux_ppc64le 33 | darwin_amd64 34 | darwin_arm64 35 | ) 36 | 37 | build_os_arch() { 38 | os="$(echo "$1" | cut -d '_' -f 1)" 39 | arch="$(echo "$1" | cut -d '_' -f 2)" 40 | make install-all GOOS="${os}" GOARCH="${arch}" OUT_DIR="${REPO_ROOT}/bin/${os}/${arch}" 41 | tar -czvf "${REPO_ROOT}/bin/${os}-${arch}.tgz" -C "${REPO_ROOT}/bin/${os}/${arch}" . 42 | } 43 | 44 | export -f build_os_arch 45 | 46 | # NOTE: disable SC2016 because we _intend_ for these to evaluate later 47 | # shellcheck disable=SC2016 48 | printf '%s\0' "${OS_ARCHES[@]}" | xargs -0 -n 1 -P "${PARALLELISM:-0}" bash -c 'build_os_arch $0' 49 | -------------------------------------------------------------------------------- /hack/build/goinstalldir.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # this utility prints out the golang install dir, even if go is not installed 17 | # IE it prints the directory where `go install ...` would theoretically place 18 | # binaries 19 | 20 | # if we have go, just ask go! 21 | if which go >/dev/null 2>&1; then 22 | DIR=$(go env GOBIN) 23 | if [ -n "${DIR}" ]; then 24 | echo "${DIR}" 25 | exit 0 26 | fi 27 | DIR=$(go env GOPATH) 28 | if [ -n "${DIR}" ]; then 29 | echo "${DIR}/bin" 30 | exit 0 31 | fi 32 | fi 33 | 34 | # mimic go behavior 35 | 36 | # check if GOBIN is set anyhow 37 | if [ -n "${GOBIN}" ]; then 38 | echo "GOBIN" 39 | exit 0 40 | fi 41 | 42 | # check if GOPATH is set anyhow 43 | if [ -n "${GOPATH}" ]; then 44 | echo "${GOPATH}/bin" 45 | exit 0 46 | fi 47 | 48 | # finally use default for no $GOPATH or $GOBIN 49 | echo "${HOME}/go/bin" 50 | -------------------------------------------------------------------------------- /hack/build/setup-go.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2020 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to setup go version with gimme as needed 17 | # MUST BE RUN FROM THE REPO ROOT DIRECTORY 18 | 19 | # read go-version file unless GO_VERSION is set 20 | GO_VERSION="${GO_VERSION:-"$(cat .go-version)"}" 21 | 22 | # we don't actually care where the .env files are 23 | # however, GIMME_SILENT_ENV doesn't trigger re-generating a .env if it 24 | # already exists and isn't "silent" (no `go version` command in it) 25 | # so we fix that by changing where the .env is written, ensuring ours 26 | # is generated from this repo and silent. 27 | export GIMME_ENV_PREFIX=./bin/.gimme/ 28 | export GIMME_SILENT_ENV=y 29 | 30 | # only setup go if we haven't set FORCE_HOST_GO, or `go version` doesn't match 31 | # go version output looks like: 32 | # go version go1.14.5 darwin/amd64 33 | if ! ([ -n "${FORCE_HOST_GO:-}" ] || \ 34 | (command -v go >/dev/null && [ "$(go version | cut -d' ' -f3)" = "go${GO_VERSION}" ])); then 35 | # eval because the output of this is shell to set PATH etc. 36 | eval "$(hack/third_party/gimme/gimme "${GO_VERSION}")" 37 | fi 38 | 39 | # force go modules 40 | export GO111MODULE=on -------------------------------------------------------------------------------- /hack/ci/push-binaries/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # See https://cloud.google.com/cloud-build/docs/build-config 2 | options: 3 | substitution_option: ALLOW_LOOSE 4 | machineType: 'E2_HIGHCPU_8' 5 | steps: 6 | - name: gcr.io/k8s-staging-test-infra/kubekins-e2e:latest-master 7 | env: 8 | - PULL_BASE_SHA=$_PULL_BASE_SHA 9 | entrypoint: hack/ci/push-binaries/push-binaries.sh 10 | substitutions: 11 | _GIT_TAG: '12345' 12 | _PULL_BASE_SHA: 'invalid' 13 | -------------------------------------------------------------------------------- /hack/ci/push-binaries/push-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2021 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit 17 | set -o nounset 18 | set -o pipefail 19 | set -o xtrace 20 | 21 | # this script is invoked in GCB as the entrypoint 22 | # avoid prow potentially not being in the right working directory 23 | REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." &> /dev/null && pwd -P)" 24 | # export REPO_ROOT for gcs_upload_version to use in subshells 25 | export REPO_ROOT 26 | cd "${REPO_ROOT}" &> /dev/null 27 | 28 | # pass through git details from prow / image builder 29 | if [ -n "${PULL_BASE_SHA:-}" ]; then 30 | export COMMIT="${PULL_BASE_SHA:?}" 31 | else 32 | COMMIT="$(git rev-parse HEAD 2>/dev/null)" 33 | export COMMIT 34 | fi 35 | 36 | # short commit is currently 8 characters 37 | SHORT_COMMIT="${COMMIT:0:8}" 38 | 39 | # we upload here 40 | BUCKET="${BUCKET:-k8s-staging-kubetest2}" 41 | export BUCKET 42 | 43 | # under each of these 44 | VERSIONS=( 45 | latest 46 | "${SHORT_COMMIT}" 47 | ) 48 | 49 | # build the ci binaries 50 | make ci-binaries 51 | 52 | gcs_upload_version() { 53 | echo "uploading CI binaries to gs://${BUCKET}/$1/ ..." 54 | # gsutil -m cp -P -r "${REPO_ROOT}/bin" "gs://${BUCKET}/$1/" 55 | # only copy the tarballs 56 | gsutil -m cp -P "${REPO_ROOT}/bin/*.tgz" "gs://${BUCKET}/$1/" 57 | } 58 | 59 | export -f gcs_upload_version 60 | 61 | # NOTE: disable SC2016 because we _intend_ for these to evaluate later 62 | # shellcheck disable=SC2016 63 | printf '%s\0' "${VERSIONS[@]}" | xargs -0 -n 1 -P "${PARALLELISM:-0}" bash -c 'gcs_upload_version $0' 64 | -------------------------------------------------------------------------------- /hack/ci/unit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to run unit tests, with coverage enabled and junit xml output 17 | set -o errexit -o nounset -o pipefail 18 | 19 | # cd to the repo root and setup go 20 | REPO_ROOT="$(git rev-parse --show-toplevel)" 21 | cd "${REPO_ROOT}" &> /dev/null 22 | source hack/build/setup-go.sh 23 | 24 | # build gotestsum 25 | cd "${REPO_ROOT}/hack/tools" &> /dev/null 26 | go build -o "${REPO_ROOT}/bin/gotestsum" gotest.tools/gotestsum 27 | cd "${REPO_ROOT}" &> /dev/null 28 | 29 | # run unit tests with coverage enabled and junit output 30 | "${REPO_ROOT}/bin/gotestsum" --junitfile="${REPO_ROOT}/bin/junit.xml" -- \ 31 | -coverprofile="${REPO_ROOT}/bin/unit.cov" -covermode count -coverpkg sigs.k8s.io/kubetest2/... ./... 32 | 33 | # filter out generated files 34 | sed '/zz_generated/d' "${REPO_ROOT}/bin/unit.cov" > "${REPO_ROOT}/bin/filtered.cov" 35 | 36 | # generate cover html 37 | go tool cover -html="${REPO_ROOT}/bin/filtered.cov" -o "${REPO_ROOT}/bin/filtered.html" 38 | 39 | # if we are in CI, copy to the artifact upload location 40 | if [[ -n "${ARTIFACTS:-}" ]]; then 41 | cp "${REPO_ROOT}/bin/junit.xml" "${REPO_ROOT}/bin/filtered.cov" "${REPO_ROOT}/bin/filtered.html" "${ARTIFACTS:?}/" 42 | fi 43 | -------------------------------------------------------------------------------- /hack/third_party/gimme/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2018 gimme contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /hack/third_party/gimme/README.md: -------------------------------------------------------------------------------- 1 | # gimme 2 | 3 | This is an unmodified copy of [gimme], so we don't have to download it 4 | from the internet. 5 | 6 | [gimme]: https://github.com/travis-ci/gimme -------------------------------------------------------------------------------- /hack/tools/.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 15m 3 | 4 | linters: 5 | disable-all: true 6 | enable: 7 | # default golangci-lint lints 8 | - errcheck 9 | - gosimple 10 | - govet 11 | - ineffassign 12 | - staticcheck 13 | - typecheck 14 | - unused 15 | 16 | # additional lints 17 | - gochecknoinits 18 | - gofmt 19 | - revive 20 | - misspell 21 | - unparam 22 | -------------------------------------------------------------------------------- /hack/tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | /* 5 | Copyright 2020 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | /* 21 | Package tools is used to track binary dependencies with go modules 22 | https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 23 | */ 24 | package tools 25 | 26 | import ( 27 | // linter(s) 28 | _ "github.com/golangci/golangci-lint/cmd/golangci-lint" 29 | 30 | // test runner 31 | _ "gotest.tools/gotestsum" 32 | ) 33 | -------------------------------------------------------------------------------- /hack/update/gofmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # script to run gofmt over our code (not vendor) 18 | set -o errexit -o nounset -o pipefail 19 | 20 | # cd to the repo root and setup go 21 | REPO_ROOT="$(git rev-parse --show-toplevel)" 22 | cd "${REPO_ROOT}" &> /dev/null 23 | source hack/build/setup-go.sh 24 | 25 | find . -name '*.go' -type f -print0 | xargs -0 gofmt -s -w 26 | -------------------------------------------------------------------------------- /hack/update/tidy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to run linters 17 | set -o errexit -o nounset -o pipefail 18 | 19 | # cd to the repo root and setup go 20 | REPO_ROOT="$(git rev-parse --show-toplevel)" 21 | cd "${REPO_ROOT}" &> /dev/null 22 | source hack/build/setup-go.sh 23 | 24 | # first for the repo in general 25 | go mod tidy 26 | -------------------------------------------------------------------------------- /hack/verify/boilerplate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script checks boilerplate header for all files. 18 | # Usage: `hack/verify-boilerplate.sh`. 19 | 20 | set -o errexit 21 | set -o nounset 22 | set -o pipefail 23 | 24 | REPO_ROOT="$(git rev-parse --show-toplevel)" 25 | 26 | boilerDir="${REPO_ROOT}/hack/boilerplate" 27 | boiler="${boilerDir}/boilerplate.py" 28 | 29 | files_need_boilerplate=() 30 | while IFS=$'\n' read -r line; do 31 | files_need_boilerplate+=( "$line" ) 32 | done < <("${boiler}" "$@") 33 | 34 | # Run boilerplate check 35 | if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then 36 | for file in "${files_need_boilerplate[@]}"; do 37 | echo "Boilerplate header is wrong for: ${file}" >&2 38 | done 39 | 40 | exit 1 41 | fi 42 | -------------------------------------------------------------------------------- /hack/verify/go-version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2022 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to verify go.mod version is <= .go-version 17 | set -o errexit -o nounset -o pipefail 18 | 19 | # cd to the repo root 20 | REPO_ROOT="$(git rev-parse --show-toplevel)" 21 | cd "${REPO_ROOT}" &> /dev/null 22 | 23 | go_mod_version=$(&2 28 | exit 1 29 | fi 30 | -------------------------------------------------------------------------------- /hack/verify/lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to run linters 17 | set -o errexit -o nounset -o pipefail 18 | 19 | # cd to the repo root and setup go 20 | REPO_ROOT="$(git rev-parse --show-toplevel)" 21 | cd "${REPO_ROOT}" &> /dev/null 22 | source hack/build/setup-go.sh 23 | 24 | # build golangci-lint 25 | cd "${REPO_ROOT}/hack/tools" &> /dev/null 26 | go build -o "${REPO_ROOT}/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint 27 | cd "${REPO_ROOT}" &> /dev/null 28 | 29 | # first for the repo in general 30 | "${REPO_ROOT}"/bin/golangci-lint --config "${REPO_ROOT}"/hack/tools/.golangci.yml run ./... 31 | -------------------------------------------------------------------------------- /hack/verify/shellcheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2019 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # CI script to run shellcheck 18 | set -o errexit 19 | set -o nounset 20 | set -o pipefail 21 | 22 | # cd to the repo root 23 | REPO_ROOT="$(git rev-parse --show-toplevel)" 24 | cd "${REPO_ROOT}" &> /dev/null 25 | 26 | if ! docker info >/dev/null 2>&1; then 27 | echo -e "\033[1;33mWARN\033[0m: shellcheck requires docker to be running" 28 | exit 0 29 | fi 30 | 31 | # upstream shellcheck latest stable image as of February 2nd, 2021 32 | SHELLCHECK_IMAGE='koalaman/shellcheck:v0.7.1' 33 | 34 | # Find all shell scripts excluding: 35 | # - Anything git-ignored - No need to lint untracked files. 36 | # - ./.git/* - Ignore anything in the git object store. 37 | # - ./vendor/* - Ignore vendored contents. 38 | # - ./bin/* - No need to lint output directories. 39 | all_shell_scripts=() 40 | while IFS=$'\n' read -r script; 41 | do git check-ignore -q "$script" || all_shell_scripts+=("$script"); 42 | done < <(find . -not -path './.git/*' -not -path './vendor/*' -not -path './bin/*' -type f -exec \ 43 | awk -v pattern='#!.*sh' 'FNR==1{if ($0~pattern) print FILENAME;}' '{}' \;) 44 | 45 | # common arguments we'll pass to shellcheck 46 | SHELLCHECK_OPTIONS=( 47 | # allow following sourced files that are not specified in the command, 48 | # we need this because we specify one file at at time in order to trivially 49 | # detect which files are failing 50 | '--external-sources' 51 | # disabled lint codes 52 | # 2330 - disabled due to https://github.com/koalaman/shellcheck/issues/1162 53 | '--exclude=2230' 54 | # 2126 - disabled because grep -c exits error when there are zero matches, 55 | # unlike grep | wc -l 56 | '--exclude=2126' 57 | # set colorized output 58 | '--color=auto' 59 | ) 60 | 61 | # actually shellcheck 62 | docker run \ 63 | --rm -t -v "${REPO_ROOT}:${REPO_ROOT}" -w "${REPO_ROOT}" \ 64 | "${SHELLCHECK_IMAGE}" \ 65 | "${SHELLCHECK_OPTIONS[@]}" "${all_shell_scripts[@]}" 66 | -------------------------------------------------------------------------------- /hack/verify/tidy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # script to run linters 17 | set -o errexit -o nounset -o pipefail 18 | 19 | # cd to the repo root and setup go 20 | REPO_ROOT="$(git rev-parse --show-toplevel)" 21 | cd "${REPO_ROOT}" &> /dev/null 22 | source hack/build/setup-go.sh 23 | 24 | # first for the repo in general 25 | ./hack/verify/tidy_quiet.sh 26 | -------------------------------------------------------------------------------- /hack/verify/tidy_quiet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright 2019 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -o errexit -o nounset -o pipefail 17 | 18 | # cd to the repo root and setup go 19 | REPO_ROOT="$(git rev-parse --show-toplevel)" 20 | cd "${REPO_ROOT}" &> /dev/null 21 | source hack/build/setup-go.sh 22 | 23 | function cleanup { 24 | exit_code=$? 25 | if [[ "${exit_code}" -ne 0 ]]; then 26 | echo "Modules are not tidied. Please run: make fix" 27 | fi 28 | rm -rf "${tmp}" 29 | exit "${exit_code}" 30 | } 31 | 32 | tmp="$(mktemp -d)" 33 | trap 'cleanup' EXIT 34 | cp -r . "${tmp}" 35 | cd "${tmp}" &> /dev/null 36 | git add . 37 | go mod tidy 38 | git diff --quiet -- go.mod go.sum 39 | -------------------------------------------------------------------------------- /kubetest2-gce/README.md: -------------------------------------------------------------------------------- 1 | # Kubetest2 GCE Deployer 2 | 3 | This component of kubetest2 is responsible for test cluster lifecycles for clusters deployed to GCE VMs. 4 | 5 | The [original design proposal](https://docs.google.com/document/d/157nSQNyy9cOjw4izG0rUs_9z9Suy31JQtng_g2peTsw/edit#heading=h.5irk4csrpu0y) has a great deal of detail about the motivations for the deployer and detailed evaluations of the original (kubetest) deployer it replaces. 6 | 7 | ## Usage 8 | 9 | Currently, the GCE deployer must be running on a system with a version of k/k or the cloud-provider-gcp repository cloned (necessary because of the reliance on scripts, see Implementation). A simple run without running tests looks as follows: 10 | 11 | ``` 12 | kubetest2 gce --gcp-project $TARGETPROJECT --repo-root $CLONEDREPOPATH --build --up --down 13 | ``` 14 | 15 | If targeting k/k instead of cloud-provider-gcp, you must add `--legacy-mode` so the deployer knows how to build the code. 16 | 17 | The deployer supports Boskos, so `--gcp-project` can be skipped if there is an available Boskos instance running. 18 | 19 | See the usage (`--help`) for more options. 20 | 21 | ## Implementation 22 | The deployer is essentially a Golang wrapper for `kube-up.sh` and `kube-down.sh` located [here](https://github.com/kubernetes/kubernetes/tree/master/cluster) in k/k and [here](https://github.com/kubernetes/cloud-provider-gcp/tree/master/cluster) in cloud-provider-gcp. It replaces bash scripts located [here](https://github.com/kubernetes/kubernetes/tree/master/hack/e2e-internal). See the design proposal for a deeper explanation. 23 | -------------------------------------------------------------------------------- /kubetest2-gce/ci-tests/buildupdown-legacy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | make install 23 | make install-deployer-gce 24 | make install-tester-ginkgo 25 | 26 | cd "${GOPATH}/src/k8s.io/kubernetes" 27 | # kubetest2 against k/k 28 | kubetest2 gce \ 29 | -v=2 \ 30 | --repo-root=. \ 31 | --build \ 32 | --up \ 33 | --down \ 34 | --legacy-mode \ 35 | --test=ginkgo \ 36 | --target-build-arch=linux/amd64 \ 37 | --master-size=e2-standard-2 \ 38 | --node-size=e2-standard-2 \ 39 | --env=KUBE_MASTER_OS_DISTRIBUTION=ubuntu \ 40 | --env=KUBE_GCE_MASTER_IMAGE=ubuntu-2204-jammy-v20230531 \ 41 | --env=KUBE_GCE_MASTER_PROJECT=ubuntu-os-cloud \ 42 | --env=KUBE_NODE_OS_DISTRIBUTION=ubuntu \ 43 | --env=KUBE_GCE_NODE_IMAGE=ubuntu-2204-jammy-v20230531 \ 44 | --env=KUBE_GCE_NODE_PROJECT=ubuntu-os-cloud \ 45 | -- \ 46 | --focus-regex='Secrets should be consumable via the environment' \ 47 | --skip-regex='\[Driver:.gcepd\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]' \ 48 | --use-built-binaries=true \ 49 | --timeout=30m 50 | -------------------------------------------------------------------------------- /kubetest2-gce/ci-tests/buildupdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | REPO_ROOT=$(git rev-parse --show-toplevel) 23 | cd "${REPO_ROOT}" &> /dev/null || exit 1 24 | 25 | make install 26 | make install-deployer-gce 27 | make install-tester-ginkgo 28 | 29 | REPO_ROOT="${GOPATH}"/src/k8s.io/cloud-provider-gcp; 30 | 31 | # TODO(spiffxp): remove this when gce-build-up-down job updated to do this, 32 | # or when bazel 5.3.0 is preinstalled on kubekins image 33 | if [ "${CI}" == "true" ]; then 34 | go install github.com/bazelbuild/bazelisk@latest 35 | mkdir -p /tmp/use-bazelisk 36 | ln -s "$(go env GOPATH)/bin/bazelisk" /tmp/use-bazelisk/bazel 37 | export PATH="/tmp/use-bazelisk:${PATH}" 38 | fi 39 | 40 | if [[ -f "${REPO_ROOT}/ginko-test-package-version.env" ]]; then 41 | TEST_PACKAGE_VERSION=$(cat "${REPO_ROOT}/ginko-test-package-version.env") 42 | export TEST_PACKAGE_VERSION 43 | echo "TEST_PACKAGE_VERSION set to ${TEST_PACKAGE_VERSION}" 44 | else 45 | export TEST_PACKAGE_VERSION="v1.25.0" 46 | echo "TEST_PACKAGE_VERSION - Falling back to v1.25.0" 47 | fi; 48 | 49 | kubetest2 gce \ 50 | -v=2 \ 51 | --repo-root="$REPO_ROOT" \ 52 | --build \ 53 | --up \ 54 | --down \ 55 | --test=ginkgo \ 56 | --master-size=e2-standard-2 \ 57 | --node-size=e2-standard-2 \ 58 | -- \ 59 | --test-package-version="${TEST_PACKAGE_VERSION}" \ 60 | --focus-regex='Secrets should be consumable via the environment' \ 61 | --skip-regex='\[Driver:.gcepd\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]' \ 62 | --timeout=60m 63 | -------------------------------------------------------------------------------- /kubetest2-gce/ci-tests/updown-legacy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | make install 23 | make install-deployer-gce 24 | make install-tester-ginkgo 25 | 26 | cd "${GOPATH}/src/k8s.io/kubernetes" 27 | # kubetest2 against k/k 28 | kubetest2 gce \ 29 | --v=9 \ 30 | --up \ 31 | --down \ 32 | --legacy-mode \ 33 | --test=ginkgo \ 34 | --target-build-arch=linux/amd64 \ 35 | --master-size=e2-standard-2 \ 36 | --node-size=e2-standard-2 \ 37 | --kubernetes-version=https://dl.k8s.io/release/stable.txt \ 38 | --env=KUBE_MASTER_OS_DISTRIBUTION=ubuntu \ 39 | --env=KUBE_GCE_MASTER_IMAGE=ubuntu-2204-jammy-v20230531 \ 40 | --env=KUBE_GCE_MASTER_PROJECT=ubuntu-os-cloud \ 41 | --env=KUBE_NODE_OS_DISTRIBUTION=ubuntu \ 42 | --env=KUBE_GCE_NODE_IMAGE=ubuntu-2204-jammy-v20230531 \ 43 | --env=KUBE_GCE_NODE_PROJECT=ubuntu-os-cloud \ 44 | -- \ 45 | --test-package-url=https://dl.k8s.io \ 46 | --test-package-marker=stable.txt \ 47 | --focus-regex='Secrets should be consumable via the environment' \ 48 | --skip-regex='\[Driver:.gcepd\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]' \ 49 | --timeout=30m 50 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "path" 23 | "strings" 24 | 25 | "k8s.io/klog/v2" 26 | 27 | "sigs.k8s.io/kubetest2/pkg/build" 28 | "sigs.k8s.io/kubetest2/pkg/exec" 29 | ) 30 | 31 | func (d *deployer) Build() error { 32 | klog.V(1).Info("GCE deployer starting Build()") 33 | 34 | if err := d.init(); err != nil { 35 | return fmt.Errorf("build failed to init: %s", err) 36 | } 37 | 38 | if d.LegacyMode { 39 | // this supports the kubernetes/kubernetes build 40 | klog.V(2).Info("starting the legacy kubernetes/kubernetes build") 41 | version, err := d.BuildOptions.Build() 42 | if err != nil { 43 | return err 44 | } 45 | 46 | // append the kubetest2 run id 47 | // avoid double + in the version 48 | // so they are valid docker tags 49 | if strings.Contains(version, "+") { 50 | version += "-" + d.commonOptions.RunID() 51 | } else { 52 | version += "+" + d.commonOptions.RunID() 53 | } 54 | 55 | // stage build if requested 56 | if d.BuildOptions.CommonBuildOptions.StageLocation != "" { 57 | if err := d.BuildOptions.Stage(version); err != nil { 58 | return fmt.Errorf("error staging build: %v", err) 59 | } 60 | } 61 | build.StoreCommonBinaries(d.RepoRoot, d.commonOptions.RunDir()) 62 | } else { 63 | // this code path supports the kubernetes/cloud-provider-gcp build 64 | klog.V(2).Info("starting the build") 65 | 66 | var cmd exec.Cmd 67 | // determine the build system for kubernetes/cloud-provider-gcp 68 | if _, err := os.Stat(path.Join(d.RepoRoot, "Makefile")); err == nil { 69 | // For releases that uses Makefile 70 | cmd = exec.Command("make", "release-tars") 71 | } else if _, err := os.Stat(path.Join(d.RepoRoot, "BUILD")); err == nil { 72 | // For releases that uses Bazel 73 | cmd = exec.Command("bazel", "build", "//release:release-tars") 74 | } else { 75 | return fmt.Errorf("cannot determine build system") 76 | } 77 | 78 | exec.InheritOutput(cmd) 79 | cmd.SetDir(d.RepoRoot) 80 | err := cmd.Run() 81 | if err != nil { 82 | return fmt.Errorf("error during make step of build: %s", err) 83 | } 84 | } 85 | 86 | // no untarring, uploading, etc is necessary because 87 | // kube-up/down use find-release-tars and upload-tars 88 | // which know how to find the tars, assuming KUBE_ROOT 89 | // is set 90 | 91 | return nil 92 | } 93 | 94 | func (d *deployer) setRepoPathIfNotSet() error { 95 | if d.RepoRoot != "" { 96 | return nil 97 | } 98 | 99 | path, err := os.Getwd() 100 | if err != nil { 101 | return fmt.Errorf("failed to get current working directory for setting Kubernetes root path: %s", err) 102 | } 103 | klog.V(1).Infof("defaulting repo root to the current directory: %s", path) 104 | d.RepoRoot = path 105 | 106 | return nil 107 | } 108 | 109 | // verifyBuildFlags only checks flags that are needed for Build 110 | func (d *deployer) verifyBuildFlags() error { 111 | if err := d.setRepoPathIfNotSet(); err != nil { 112 | return err 113 | } 114 | d.BuildOptions.CommonBuildOptions.RepoRoot = d.RepoRoot 115 | return d.BuildOptions.Validate() 116 | } 117 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/build_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "os" 21 | "path/filepath" 22 | "testing" 23 | ) 24 | 25 | func TestSetRepoPathIfNotSet(t *testing.T) { 26 | tmpdir, err := filepath.EvalSymlinks(os.TempDir()) 27 | if err != nil { 28 | t.Errorf("failed to read tempdir") 29 | } 30 | cases := []struct { 31 | name string 32 | 33 | initialDeployer deployer 34 | expectedRepoPath string 35 | }{ 36 | { 37 | name: "set empty repo path", 38 | expectedRepoPath: tmpdir, 39 | }, 40 | { 41 | name: "set preset repo path", 42 | initialDeployer: deployer{ 43 | RepoRoot: "/test/path", 44 | }, 45 | expectedRepoPath: "/test/path", 46 | }, 47 | } 48 | 49 | err = os.Chdir(os.TempDir()) 50 | if err != nil { 51 | t.Errorf("failed to chdir for test: %s", err) 52 | } 53 | 54 | for i := range cases { 55 | c := &cases[i] 56 | t.Run(c.name, func(t *testing.T) { 57 | t.Parallel() 58 | 59 | d := &c.initialDeployer 60 | err := d.setRepoPathIfNotSet() 61 | if err != nil { 62 | t.Errorf("failed to set repo path: %s", err) 63 | } 64 | 65 | if d.RepoRoot != c.expectedRepoPath { 66 | t.Errorf("expected repo path to be %s but it was %s", c.expectedRepoPath, d.RepoRoot) 67 | } 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/down.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "path/filepath" 22 | 23 | "k8s.io/klog/v2" 24 | "sigs.k8s.io/kubetest2/pkg/boskos" 25 | "sigs.k8s.io/kubetest2/pkg/exec" 26 | ) 27 | 28 | func (d *deployer) Down() error { 29 | klog.V(1).Info("GCE deployer starting Down()") 30 | 31 | if err := d.DumpClusterLogs(); err != nil { 32 | klog.Warningf("Dumping cluster logs at the begin of Down() failed: %s", err) 33 | } 34 | 35 | if err := d.init(); err != nil { 36 | return fmt.Errorf("down failed to init: %s", err) 37 | } 38 | 39 | path, err := d.verifyKubectl() 40 | if err != nil { 41 | return err 42 | } 43 | d.kubectlPath = path 44 | 45 | env := d.buildEnv() 46 | script := filepath.Join(d.RepoRoot, "cluster", "kube-down.sh") 47 | klog.V(2).Infof("About to run script at: %s", script) 48 | 49 | cmd := exec.Command(script) 50 | cmd.SetEnv(env...) 51 | exec.InheritOutput(cmd) 52 | 53 | if err := cmd.Run(); err != nil { 54 | return fmt.Errorf("error encountered during %s: %s", script, err) 55 | } 56 | 57 | klog.V(2).Info("about to delete nodeport firewall rule") 58 | // best-effort try to delete the explicitly created firewall rules 59 | // ideally these should already be deleted by kube-down 60 | d.deleteFirewallRuleNodePort() 61 | 62 | if d.boskos != nil { 63 | klog.V(2).Info("releasing boskos project") 64 | err := boskos.Release( 65 | d.boskos, 66 | []string{d.GCPProject}, 67 | d.boskosHeartbeatClose, 68 | ) 69 | if err != nil { 70 | return fmt.Errorf("down failed to release boskos project: %s", err) 71 | } 72 | } 73 | 74 | return nil 75 | } 76 | 77 | func (d *deployer) verifyDownFlags() error { 78 | if err := d.setRepoPathIfNotSet(); err != nil { 79 | return err 80 | } 81 | 82 | if d.GCPProject == "" { 83 | return fmt.Errorf("gcp project must be set") 84 | } 85 | 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/dumplogs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | 24 | "k8s.io/klog/v2" 25 | 26 | "sigs.k8s.io/kubetest2/pkg/exec" 27 | ) 28 | 29 | func (d *deployer) DumpClusterLogs() error { 30 | klog.V(1).Info("GCE deployer starting DumpClusterLogs()") 31 | 32 | if err := d.init(); err != nil { 33 | return fmt.Errorf("dump cluster logs failed to init: %s", err) 34 | } 35 | 36 | klog.V(2).Info("making logs directory") 37 | if err := d.makeLogsDir(); err != nil { 38 | return fmt.Errorf("couldn't make logs dir: %s", err) 39 | } 40 | 41 | // Run sshDump before kubectlDump because kubectl could fail if master kube 42 | // didn't come up successfully but the instances were still created and setup 43 | // proceeded to a certain point. This allows retrieval of logs that could 44 | // indicate why master coming up failed. 45 | if err := d.sshDump(); err != nil { 46 | return fmt.Errorf("failed to dump logs from instance log files: %s", err) 47 | } 48 | 49 | if err := d.kubectlDump(); err != nil { 50 | return fmt.Errorf("failed to dump cluster info with kubectl: %s", err) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func (d *deployer) makeLogsDir() error { 57 | _, err := os.Stat(d.logsDir) 58 | 59 | if os.IsNotExist(err) { 60 | err := os.Mkdir(d.logsDir, os.ModePerm) 61 | if err != nil { 62 | return fmt.Errorf("failed to create %s: %s", d.logsDir, err) 63 | } 64 | return nil 65 | } 66 | if err != nil { 67 | return fmt.Errorf("unexpected exception when making cluster logs directory: %s", err) 68 | } 69 | 70 | // file definitely exists, overwrite if requested 71 | 72 | if d.OverwriteLogsDir { 73 | klog.V(2).Infof("logs directory %s already exists, removing and recreating", d.logsDir) 74 | 75 | if err := os.RemoveAll(d.logsDir); err != nil { 76 | return fmt.Errorf("failed to delete existing logs directory: %s", err) 77 | } 78 | 79 | err := os.Mkdir(d.logsDir, os.ModePerm) 80 | if err != nil { 81 | return fmt.Errorf("failed to create %s: %s", d.logsDir, err) 82 | } 83 | return nil 84 | } 85 | 86 | return fmt.Errorf("cluster logs directory %s already exists, please clean up manually or use the overwrite flag before continuing", d.logsDir) 87 | } 88 | 89 | func (d *deployer) sshDump() error { 90 | env := d.buildEnv() 91 | 92 | args := []string{ 93 | filepath.Join(d.RepoRoot, "cluster", "log-dump", "log-dump.sh"), 94 | d.logsDir, 95 | } 96 | klog.V(2).Infof("About to run: %s", args) 97 | 98 | cmd := exec.Command(args[0], args[1:]...) 99 | cmd.SetEnv(env...) 100 | exec.InheritOutput(cmd) 101 | if err := cmd.Run(); err != nil { 102 | return fmt.Errorf("failed to use log-dump.sh for cluster logs: %s", err) 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func (d *deployer) kubectlDump() error { 109 | env := d.buildEnv() 110 | outfile, err := os.Create(filepath.Join(d.logsDir, "cluster-info.log")) 111 | if err != nil { 112 | return fmt.Errorf("failed to create cluster-info log file: %s", err) 113 | } 114 | defer outfile.Close() 115 | 116 | args := []string{ 117 | d.kubectlPath, 118 | "cluster-info", 119 | "dump", 120 | } 121 | klog.V(2).Infof("About to run: %s", args) 122 | 123 | cmd := exec.Command(args[0], args[1:]...) 124 | cmd.SetEnv(env...) 125 | cmd.SetStderr(os.Stderr) 126 | cmd.SetStdout(outfile) 127 | err = cmd.Run() 128 | if err != nil { 129 | return fmt.Errorf("couldn't use kubectl to dump cluster info: %s", err) 130 | } 131 | 132 | return nil 133 | } 134 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/firewall.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | 22 | "k8s.io/klog/v2" 23 | "sigs.k8s.io/kubetest2/pkg/exec" 24 | ) 25 | 26 | // kube-up.sh builds NODE_TAG based on KUBE_GCE_INSTANCE_PREFIX which the deployer 27 | // sets as d.instacePrefix. This function replicates NODE_TAG string construction 28 | // because it is needed for firewall rules 29 | func (d *deployer) nodeTag() string { 30 | return fmt.Sprintf("%s-minion", d.instancePrefix) 31 | } 32 | 33 | func (d *deployer) nodePortRuleName() string { 34 | return fmt.Sprintf("%s-nodeports", d.nodeTag()) 35 | } 36 | 37 | func (d *deployer) createFirewallRuleNodePort() error { 38 | cmd := exec.Command( 39 | "gcloud", "compute", "firewall-rules", "create", 40 | "--project", d.GCPProject, 41 | "--target-tags", d.nodeTag(), 42 | "--allow", "tcp:30000-32767,udp:30000-32767", 43 | "--network", d.network, 44 | d.nodePortRuleName(), 45 | ) 46 | exec.InheritOutput(cmd) 47 | if err := cmd.Run(); err != nil { 48 | return fmt.Errorf("failed to create nodeports firewall rule: %s", err) 49 | } 50 | 51 | return nil 52 | } 53 | 54 | func (d *deployer) deleteFirewallRuleNodePort() { 55 | cmd := exec.Command( 56 | "gcloud", "compute", "firewall-rules", "delete", 57 | "--project", d.GCPProject, 58 | d.nodePortRuleName(), 59 | ) 60 | exec.InheritOutput(cmd) 61 | if err := cmd.Run(); err != nil { 62 | klog.Warning("failed to delete nodeports firewall rules: might be deleted already?") 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /kubetest2-gce/deployer/options/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/build" 21 | ) 22 | 23 | type BuildOptions struct { 24 | CommonBuildOptions *build.Options 25 | } 26 | 27 | var _ build.Builder = &BuildOptions{} 28 | var _ build.Stager = &BuildOptions{} 29 | 30 | func (bo *BuildOptions) Validate() error { 31 | return bo.CommonBuildOptions.Validate() 32 | } 33 | 34 | func (bo *BuildOptions) Build() (string, error) { 35 | return bo.CommonBuildOptions.Build() 36 | } 37 | 38 | func (bo *BuildOptions) Stage(version string) error { 39 | return bo.CommonBuildOptions.Stage(version) 40 | } 41 | -------------------------------------------------------------------------------- /kubetest2-gce/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/app" 21 | 22 | "sigs.k8s.io/kubetest2/kubetest2-gce/deployer" 23 | ) 24 | 25 | func main() { 26 | app.Main(deployer.Name, deployer.New) 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-gke/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | reviewers: 4 | - chizhg 5 | -------------------------------------------------------------------------------- /kubetest2-gke/ci-tests/buildupdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | REPO_ROOT=$(git rev-parse --show-toplevel) 23 | cd "${REPO_ROOT}" &> /dev/null || exit 1 24 | 25 | make install 26 | make install-deployer-gke 27 | make install-tester-exec 28 | 29 | function main() { 30 | CLUSTER_TOPOLOGY="singlecluster" 31 | 32 | while [ $# -gt 0 ]; do 33 | case "$1" in 34 | --cluster-topology) 35 | shift 36 | CLUSTER_TOPOLOGY="$1" 37 | ;; 38 | *) 39 | echo "Invalid argument" 40 | exit 1 41 | ;; 42 | esac 43 | shift 44 | done 45 | 46 | NUM_CLUSTERS=0 47 | case "${CLUSTER_TOPOLOGY}" in 48 | "singlecluster") 49 | NUM_CLUSTERS=1 50 | ;; 51 | "multicluster") 52 | NUM_CLUSTERS=2 53 | ;; 54 | *) 55 | echo "Invalid cluster topology ${CLUSTER_TOPOLOGY}" 56 | exit 1 57 | ;; 58 | esac 59 | 60 | kubetest2 gke \ 61 | -v 2 \ 62 | --boskos-resource-type gce-project \ 63 | --num-clusters "${NUM_CLUSTERS}" \ 64 | --num-nodes 1 \ 65 | --zone us-central1-c,us-west1-a,us-east1-b \ 66 | --network ci-tests-network \ 67 | --up \ 68 | --down \ 69 | --test=exec -- "${REPO_ROOT}/kubetest2-gke/ci-tests/test.sh" 70 | } 71 | 72 | main "$@" 73 | -------------------------------------------------------------------------------- /kubetest2-gke/ci-tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | echo "Dumping relevant info for the master and cluster services" 18 | kubectl cluster-info dump 19 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "regexp" 23 | "strings" 24 | 25 | "k8s.io/klog/v2" 26 | 27 | "sigs.k8s.io/kubetest2/pkg/build" 28 | ) 29 | 30 | var ( 31 | // (1.18.10-gke.12.34)(...) 32 | // $1 == prefix 33 | // $2 == suffix 34 | gkeCIBuildPrefix = regexp.MustCompile(`^(\d\.\d+\.\d+-gke\.\d+\.\d+)([+-].*)?$`) 35 | 36 | // (1.18.10-gke.1234)(...) 37 | // $1 == prefix 38 | // $2 == suffix 39 | gkeBuildPrefix = regexp.MustCompile(`^(\d\.\d+\.\d+-gke\.\d+)([+-].*)?$`) 40 | 41 | // (1.18.10)(...) 42 | // $1 == prefix 43 | // $2 == suffix 44 | buildPrefix = regexp.MustCompile(`^(\d\.\d+\.\d+)([+-].*)?$`) 45 | 46 | defaultImageTag = "gke.gcr.io" 47 | ) 48 | 49 | func (d *Deployer) Build() error { 50 | imageTag := defaultImageTag 51 | if d.BuildOptions.CommonBuildOptions.ImageLocation != "" { 52 | imageTag = d.BuildOptions.CommonBuildOptions.ImageLocation 53 | } 54 | 55 | klog.V(2).Infof("setting KUBE_DOCKER_REGISTRY to %s for tagging images", imageTag) 56 | if err := os.Setenv("KUBE_DOCKER_REGISTRY", imageTag); err != nil { 57 | return err 58 | } 59 | if err := d.VerifyBuildFlags(); err != nil { 60 | return err 61 | } 62 | version, err := d.BuildOptions.Build() 63 | if err != nil { 64 | return err 65 | } 66 | klog.V(2).Infof("got build version: %s", version) 67 | version = strings.TrimPrefix(version, "v") 68 | if version, err = normalizeVersion(version); err != nil { 69 | return err 70 | } 71 | 72 | // append the kubetest2 run id 73 | // avoid double + in the version 74 | // so they are valid docker tags 75 | if !strings.HasSuffix(version, d.Kubetest2CommonOptions.RunID()) { 76 | if strings.Contains(version, "+") { 77 | version += "-" + d.Kubetest2CommonOptions.RunID() 78 | } else { 79 | version += "+" + d.Kubetest2CommonOptions.RunID() 80 | } 81 | } 82 | 83 | // stage build if requested 84 | if d.BuildOptions.CommonBuildOptions.StageLocation != "" { 85 | if err := d.BuildOptions.Stage(version); err != nil { 86 | return fmt.Errorf("error staging build: %v", err) 87 | } 88 | } 89 | d.ClusterVersion = version 90 | build.StoreCommonBinaries(d.RepoRoot, d.Kubetest2CommonOptions.RunDir()) 91 | return nil 92 | } 93 | 94 | func (d *Deployer) VerifyBuildFlags() error { 95 | if d.RepoRoot == "" { 96 | return fmt.Errorf("required repo-root when building from source") 97 | } 98 | d.BuildOptions.CommonBuildOptions.RepoRoot = d.RepoRoot 99 | if d.Kubetest2CommonOptions.ShouldBuild() && d.Kubetest2CommonOptions.ShouldUp() && d.BuildOptions.CommonBuildOptions.StageLocation == "" { 100 | return fmt.Errorf("creating a gke cluster from built sources requires staging them to a specific GCS bucket, use --stage=gs://") 101 | } 102 | // force extra GCP files to be staged 103 | d.BuildOptions.CommonBuildOptions.StageExtraGCPFiles = true 104 | // add kubetest2 runid as the version suffix 105 | d.BuildOptions.CommonBuildOptions.VersionSuffix = d.Kubetest2CommonOptions.RunID() 106 | return d.BuildOptions.Validate() 107 | } 108 | 109 | // ensure that the version is a valid gke version 110 | func normalizeVersion(version string) (string, error) { 111 | 112 | finalVersion := "" 113 | if matches := gkeCIBuildPrefix.FindStringSubmatch(version); matches != nil { 114 | // prefix is usable as-is 115 | finalVersion = matches[1] 116 | // preserve suffix if present 117 | if suffix := strings.TrimLeft(matches[2], "+-"); len(suffix) > 0 { 118 | finalVersion += "+" + suffix 119 | } 120 | } else if matches := gkeBuildPrefix.FindStringSubmatch(version); matches != nil { 121 | // prefix needs .0 appended 122 | finalVersion = matches[1] + ".0" 123 | // preserve suffix if present 124 | if suffix := strings.TrimLeft(matches[2], "+-"); len(suffix) > 0 { 125 | finalVersion += "+" + suffix 126 | } 127 | } else if matches := buildPrefix.FindStringSubmatch(version); matches != nil { 128 | // prefix needs -gke.99.0 appended 129 | finalVersion = matches[1] + "-gke.99.0" 130 | // preserve suffix if present 131 | if suffix := strings.TrimLeft(matches[2], "+-"); len(suffix) > 0 { 132 | finalVersion += "+" + suffix 133 | } 134 | } else { 135 | return "", fmt.Errorf("could not construct version from %s", version) 136 | } 137 | 138 | if finalVersion != version { 139 | klog.V(2).Infof("modified version %q to %q", version, finalVersion) 140 | } 141 | return finalVersion, nil 142 | } 143 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/build/gke_make.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io" 23 | "os" 24 | "strings" 25 | 26 | "k8s.io/klog/v2" 27 | 28 | util "sigs.k8s.io/kubetest2/kubetest2-gke/deployer/utils" 29 | "sigs.k8s.io/kubetest2/pkg/build" 30 | "sigs.k8s.io/kubetest2/pkg/exec" 31 | ) 32 | 33 | type gkeBuildAction string 34 | 35 | const ( 36 | compile gkeBuildAction = "compile" 37 | pack gkeBuildAction = "package" 38 | stage gkeBuildAction = "push-gcs" 39 | printVersion gkeBuildAction = "print-version" 40 | ) 41 | 42 | const ( 43 | // GKEMakeStrategy builds and stages using the gke_make build 44 | GKEMakeStrategy build.BuildAndStageStrategy = "gke_make" 45 | latestBuildMarkerPrefix string = "latest" 46 | ) 47 | 48 | type GKEMake struct { 49 | RepoRoot string 50 | BuildScript string 51 | VersionSuffix string 52 | StageLocation string 53 | UpdateLatest bool 54 | } 55 | 56 | func gkeBuildActions(actions []gkeBuildAction) string { 57 | stringActions := make([]string, len(actions)) 58 | for i, action := range actions { 59 | stringActions[i] = string(action) 60 | } 61 | return strings.Join(stringActions, ",") 62 | } 63 | 64 | func arg(key, value string) string { 65 | return fmt.Sprintf("%s=%s", key, value) 66 | } 67 | 68 | func (gmb *GKEMake) runWithActions(stdout, stderr io.Writer, actions []gkeBuildAction, extraArgs ...string) error { 69 | 70 | args := []string{arg("GKE_BUILD_ACTIONS", gkeBuildActions(actions))} 71 | args = append(args, extraArgs...) 72 | cmd := exec.Command(gmb.BuildScript, args...) 73 | cmd.SetDir(gmb.RepoRoot) 74 | cmd.SetStdout(stdout) 75 | cmd.SetStderr(stderr) 76 | return cmd.Run() 77 | } 78 | 79 | func (gmb *GKEMake) Build() (string, error) { 80 | klog.V(2).Infof("starting gke build ...") 81 | 82 | stdout := &bytes.Buffer{} 83 | stderr := &bytes.Buffer{} 84 | if err := gmb.runWithActions(stdout, stderr, []gkeBuildAction{printVersion}, arg("VERSION_SUFFIX", gmb.VersionSuffix)); err != nil { 85 | klog.Errorf("failed to get version: %s\n%v", stderr.String(), err) 86 | return "", err 87 | } 88 | 89 | version := strings.TrimSpace(stdout.String()) 90 | if version == "" { 91 | klog.Error(stderr.String()) 92 | return "", fmt.Errorf("failed to get version: got empty version") 93 | } 94 | 95 | // Skip validation for faster builds 96 | // TODO: add support for a separate validate mode 97 | if err := gmb.runWithActions(os.Stdout, os.Stderr, []gkeBuildAction{compile, pack}, arg("VERSION", version)); err != nil { 98 | return "", err 99 | } 100 | 101 | return version, nil 102 | } 103 | 104 | var _ build.Builder = &GKEMake{} 105 | 106 | func (gmb *GKEMake) Stage(version string) error { 107 | klog.V(2).Infof("staging gke builds ...") 108 | if !strings.HasPrefix(version, "v") { 109 | version = "v" + version 110 | } 111 | args := []string{arg("VERSION", version)} 112 | if gmb.StageLocation != "" { 113 | args = append(args, arg("GCS_BUCKET", gmb.StageLocation)) 114 | } 115 | 116 | if err := gmb.runWithActions(os.Stdout, os.Stderr, []gkeBuildAction{stage}, args...); err != nil { 117 | return err 118 | } 119 | 120 | if gmb.UpdateLatest { 121 | if err := util.StageGKEBuildMarker(version, gmb.StageLocation, latestBuildMarkerPrefix); err != nil { 122 | return fmt.Errorf("error during build marker staging: %s", err) 123 | } 124 | } 125 | 126 | return nil 127 | } 128 | 129 | var _ build.Stager = &GKEMake{} 130 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/build_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import "testing" 20 | 21 | func TestNormalizeVersion(t *testing.T) { 22 | testCases := []struct { 23 | name string 24 | version string 25 | expectedVersion string 26 | expectError bool 27 | }{ 28 | { 29 | name: "empty input", 30 | version: "", 31 | expectedVersion: "", 32 | expectError: true, 33 | }, 34 | { 35 | name: "only core version", 36 | version: "1.19.4", 37 | expectedVersion: "1.19.4-gke.99.0", 38 | expectError: false, 39 | }, 40 | { 41 | name: "oss rc tag", 42 | version: "1.19.4-rc.0", 43 | expectedVersion: "1.19.4-gke.99.0+rc.0", 44 | expectError: false, 45 | }, 46 | { 47 | name: "oss rc tag + change", 48 | version: "1.19.4-rc.0-1-g37677a1347a", 49 | expectedVersion: "1.19.4-gke.99.0+rc.0-1-g37677a1347a", 50 | expectError: false, 51 | }, 52 | { 53 | name: "oss beta tag", 54 | version: "1.19.4-beta.0", 55 | expectedVersion: "1.19.4-gke.99.0+beta.0", 56 | expectError: false, 57 | }, 58 | { 59 | name: "oss beta tag + change", 60 | version: "1.19.4-beta.0-1-g37677a1347a", 61 | expectedVersion: "1.19.4-gke.99.0+beta.0-1-g37677a1347a", 62 | expectError: false, 63 | }, 64 | { 65 | name: "oss alpha tag", 66 | version: "1.19.4-alpha.0", 67 | expectedVersion: "1.19.4-gke.99.0+alpha.0", 68 | expectError: false, 69 | }, 70 | { 71 | name: "oss alpha tag + change", 72 | version: "1.19.4-alpha.0-1-g37677a1347a", 73 | expectedVersion: "1.19.4-gke.99.0+alpha.0-1-g37677a1347a", 74 | expectError: false, 75 | }, 76 | { 77 | name: "gke build", 78 | version: "1.18.10-gke.601", 79 | expectedVersion: "1.18.10-gke.601.0", 80 | expectError: false, 81 | }, 82 | { 83 | name: "gke build + change", 84 | version: "1.18.10-gke.601-1-geebff6c215c", 85 | expectedVersion: "1.18.10-gke.601.0+1-geebff6c215c", 86 | expectError: false, 87 | }, 88 | { 89 | name: "core version, pre-existing suffix", 90 | version: "1.17.12+foobar", 91 | expectedVersion: "1.17.12-gke.99.0+foobar", 92 | expectError: false, 93 | }, 94 | { 95 | name: "full version, pre-existing suffix", 96 | version: "1.16.13-gke.401+qwerty", 97 | expectedVersion: "1.16.13-gke.401.0+qwerty", 98 | expectError: false, 99 | }, 100 | { 101 | name: "full version, multiple pre-existing suffix", 102 | version: "1.16.13-gke.401+qwerty+foobar", 103 | expectedVersion: "1.16.13-gke.401.0+qwerty+foobar", 104 | expectError: false, 105 | }, 106 | { 107 | name: "full version, multiple patch, multiple pre-existing suffix", 108 | version: "1.16.13-gke.401.123+qwerty+foobar", 109 | expectedVersion: "1.16.13-gke.401.123+qwerty+foobar", 110 | expectError: false, 111 | }, 112 | { 113 | name: "alpha version with patch", 114 | version: "1.16.13-alpha.123", 115 | expectedVersion: "1.16.13-gke.99.0+alpha.123", 116 | expectError: false, 117 | }, 118 | { 119 | name: "beta version, no patch, pre-existing suffix", 120 | version: "1.20.0-beta+qwe123", 121 | expectedVersion: "1.20.0-gke.99.0+beta+qwe123", 122 | expectError: false, 123 | }, 124 | } 125 | 126 | for _, tc := range testCases { 127 | tc := tc 128 | t.Run(tc.name, func(t *testing.T) { 129 | t.Parallel() 130 | actualVersion, err := normalizeVersion(tc.version) 131 | if err != nil && !tc.expectError { 132 | t.Errorf("did not expect an error but got: %v", err) 133 | } 134 | if err == nil && tc.expectError { 135 | t.Errorf("expected an error but got none") 136 | } 137 | if actualVersion != tc.expectedVersion { 138 | t.Errorf("expected version: %q, but got: %q", tc.expectedVersion, actualVersion) 139 | } 140 | }) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/commandutils.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io" 23 | "os" 24 | osexec "os/exec" 25 | 26 | "path/filepath" 27 | "strings" 28 | 29 | "k8s.io/klog/v2" 30 | 31 | "sigs.k8s.io/kubetest2/pkg/exec" 32 | ) 33 | 34 | func (d *Deployer) PrepareGcpIfNeeded(projectID string) error { 35 | // TODO(RonWeber): This is an almost direct copy/paste from kubetest's prepareGcp() 36 | // It badly needs refactored. 37 | 38 | var endpoint string 39 | switch env := d.Environment; { 40 | case env == "test": 41 | endpoint = "https://test-container.sandbox.googleapis.com/" 42 | case env == "staging": 43 | endpoint = "https://staging-container.sandbox.googleapis.com/" 44 | case env == "staging2": 45 | endpoint = "https://staging2-container.sandbox.googleapis.com/" 46 | case env == "prod": 47 | endpoint = "https://container.googleapis.com/" 48 | case urlRe.MatchString(env): 49 | endpoint = env 50 | default: 51 | return fmt.Errorf("--environment must be one of {test,staging,staging2,prod} or match %v, found %q", urlRe, env) 52 | } 53 | 54 | if err := os.Setenv("CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS", "1"); err != nil { 55 | return fmt.Errorf("could not set CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS=1: %v", err) 56 | } 57 | if err := os.Setenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER", endpoint); err != nil { 58 | return err 59 | } 60 | 61 | if err := runWithOutput(exec.RawCommand("gcloud config set project " + projectID)); err != nil { 62 | return fmt.Errorf("failed to set project %s: %w", projectID, err) 63 | } 64 | 65 | // gcloud creds may have changed 66 | if err := activateServiceAccount(d.GCPServiceAccount); err != nil { 67 | return err 68 | } 69 | 70 | if !d.GCPSSHKeyIgnored { 71 | // Ensure ssh keys exist 72 | klog.V(1).Info("Checking existing of GCP ssh keys...") 73 | k := filepath.Join(home(".ssh"), "google_compute_engine") 74 | if _, err := os.Stat(k); err != nil { 75 | return err 76 | } 77 | pk := k + ".pub" 78 | if _, err := os.Stat(pk); err != nil { 79 | return err 80 | } 81 | } 82 | 83 | //TODO(RonWeber): kubemark 84 | return nil 85 | } 86 | 87 | // Activate service account if set or do nothing. 88 | func activateServiceAccount(path string) error { 89 | if path == "" { 90 | return nil 91 | } 92 | return runWithOutput(exec.RawCommand("gcloud auth activate-service-account --key-file=" + path)) 93 | } 94 | 95 | // Get the project number for the given project ID. 96 | func getProjectNumber(projectID string) (string, error) { 97 | // Get the service project number. 98 | projectNum, err := exec.Output(exec.RawCommand( 99 | fmt.Sprintf("gcloud projects describe %s --format=value(projectNumber)", projectID))) 100 | if err != nil { 101 | return "", err 102 | } 103 | 104 | return strings.TrimSpace(string(projectNum)), nil 105 | } 106 | 107 | // home returns $HOME/part/part/part 108 | func home(parts ...string) string { 109 | p := append([]string{os.Getenv("HOME")}, parts...) 110 | return filepath.Join(p...) 111 | } 112 | 113 | func getClusterCredentials(project, loc, cluster string) error { 114 | // Get gcloud to create the file. 115 | if err := runWithOutput(exec.Command("gcloud", 116 | containerArgs("clusters", "get-credentials", cluster, "--project="+project, loc)...), 117 | ); err != nil { 118 | return fmt.Errorf("error executing get-credentials: %v", err) 119 | } 120 | 121 | return nil 122 | } 123 | 124 | func containerArgs(args ...string) []string { 125 | return append(append([]string{}, "container"), args...) 126 | } 127 | 128 | func runWithNoOutput(cmd exec.Cmd) error { 129 | exec.NoOutput(cmd) 130 | return cmd.Run() 131 | } 132 | 133 | func runWithOutput(cmd exec.Cmd) error { 134 | exec.InheritOutput(cmd) 135 | return cmd.Run() 136 | } 137 | 138 | func runWithOutputAndReturn(cmd exec.Cmd) (string, error) { 139 | var buf bytes.Buffer 140 | 141 | exec.SetOutput(cmd, io.MultiWriter(os.Stdout, &buf), io.MultiWriter(os.Stderr, &buf)) 142 | if err := cmd.Run(); err != nil { 143 | return buf.String(), err 144 | } 145 | return buf.String(), nil 146 | } 147 | 148 | // execError returns a string format of err including stderr if the 149 | // err is an ExitError, useful for errors from e.g. exec.Cmd.Output(). 150 | func execError(err error) string { 151 | if ee, ok := err.(*osexec.ExitError); ok { 152 | return fmt.Sprintf("%v (output: %q)", err, string(ee.Stderr)) 153 | } 154 | return err.Error() 155 | } 156 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/deployer_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import "testing" 20 | 21 | func TestLocationFlag(t *testing.T) { 22 | testCases := []struct { 23 | regions []string 24 | zones []string 25 | retryCount int 26 | expected string 27 | }{ 28 | { 29 | regions: []string{"us-central1"}, 30 | zones: []string{}, 31 | retryCount: 0, 32 | expected: "--region=us-central1", 33 | }, 34 | { 35 | regions: []string{}, 36 | zones: []string{"us-central1-c"}, 37 | retryCount: 0, 38 | expected: "--zone=us-central1-c", 39 | }, 40 | { 41 | regions: []string{"us-central1", "us-east"}, 42 | zones: []string{}, 43 | retryCount: 1, 44 | expected: "--region=us-east", 45 | }, 46 | } 47 | 48 | for _, tc := range testCases { 49 | got := locationFlag(tc.regions, tc.zones, tc.retryCount) 50 | if got != tc.expected { 51 | t.Errorf("expected %q but got %q", tc.expected, got) 52 | } 53 | } 54 | } 55 | 56 | func TestRegionFromLocation(t *testing.T) { 57 | testCases := []struct { 58 | regions []string 59 | zones []string 60 | retryCount int 61 | expected string 62 | }{ 63 | { 64 | regions: []string{"us-central1"}, 65 | zones: []string{}, 66 | retryCount: 0, 67 | expected: "us-central1", 68 | }, 69 | { 70 | regions: []string{}, 71 | zones: []string{"us-central1-c"}, 72 | retryCount: 0, 73 | expected: "us-central1", 74 | }, 75 | { 76 | regions: []string{"us-central1", "us-east"}, 77 | zones: []string{}, 78 | retryCount: 1, 79 | expected: "us-east", 80 | }, 81 | } 82 | 83 | for _, tc := range testCases { 84 | got := regionFromLocation(tc.regions, tc.zones, tc.retryCount) 85 | if got != tc.expected { 86 | t.Errorf("expected %q but got %q", tc.expected, got) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/down.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | 23 | "k8s.io/klog/v2" 24 | 25 | "sigs.k8s.io/kubetest2/pkg/boskos" 26 | "sigs.k8s.io/kubetest2/pkg/exec" 27 | ) 28 | 29 | func (d *Deployer) Down() error { 30 | if err := d.Init(); err != nil { 31 | return err 32 | } 33 | // Nothing to clean if there is no GCP project. 34 | // This edge case happens e.g. when Up fails to acquire the Boskos project. 35 | if len(d.Projects) == 0 { 36 | return nil 37 | } 38 | 39 | if err := d.DumpClusterLogs(); err != nil { 40 | klog.Warningf("Dumping cluster logs at the end of Up() failed: %v", err) 41 | } 42 | 43 | // If the GCP projects are acquired from Boskos, release the projects and 44 | // rely on boskos-janitor to do clean-ups for them. 45 | if d.totalBoskosProjectsRequested > 0 { 46 | return boskos.Release(d.boskos, d.Projects, d.boskosHeartbeatClose) 47 | } 48 | 49 | d.DeleteClusters(d.retryCount) 50 | 51 | numDeletedFWRules, errCleanFirewalls := d.CleanupNetworkFirewalls(d.Projects[0], d.Network) 52 | if errCleanFirewalls != nil { 53 | klog.Errorf("Error cleaning-up firewall rules: %v", errCleanFirewalls) 54 | } else { 55 | klog.V(1).Infof("Deleted %d network firewall rules", numDeletedFWRules) 56 | } 57 | 58 | if err := d.TeardownNetwork(); err != nil { 59 | return err 60 | } 61 | if err := d.DeleteSubnets(d.retryCount); err != nil { 62 | return err 63 | } 64 | return d.DeleteNetwork() 65 | } 66 | 67 | func (d *Deployer) DeleteClusters(retryCount int) { 68 | // We best-effort try all of these and report errors as appropriate. 69 | var wg sync.WaitGroup 70 | for i := range d.Projects { 71 | project := d.Projects[i] 72 | for j := range d.projectClustersLayout[project] { 73 | cluster := d.projectClustersLayout[project][j] 74 | loc := locationFlag(d.Regions, d.Zones, retryCount) 75 | 76 | wg.Add(1) 77 | go func() { 78 | defer wg.Done() 79 | d.DeleteCluster(project, loc, cluster) 80 | }() 81 | } 82 | } 83 | wg.Wait() 84 | } 85 | 86 | func (d *Deployer) DeleteCluster(project, loc string, cluster cluster) { 87 | if err := runWithOutput(exec.Command( 88 | "gcloud", containerArgs("clusters", "delete", "-q", cluster.name, 89 | "--project="+project, 90 | loc)...)); err != nil { 91 | klog.Errorf("Error deleting cluster: %v", err) 92 | } 93 | } 94 | 95 | // VerifyDownFlags validates flags for down phase. 96 | func (d *Deployer) VerifyDownFlags() error { 97 | if len(d.Clusters) == 0 { 98 | return fmt.Errorf("--cluster-name must be set for GKE deployment") 99 | } 100 | if len(d.Projects) == 0 { 101 | return fmt.Errorf("--project must be set for GKE deployment") 102 | } 103 | return d.VerifyLocationFlags() 104 | } 105 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/dumplogs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "strings" 23 | 24 | "sigs.k8s.io/kubetest2/pkg/exec" 25 | ) 26 | 27 | // DumpClusterLogs for GKE generates a small script that wraps 28 | // log-dump.sh with the appropriate shell-fu to get the cluster 29 | // dumped. 30 | // 31 | // TODO(RonWeber): This whole path is really gross, but this seemed 32 | // the least gross hack to get this done. 33 | // 34 | // TODO(RonWeber): Make this work with multizonal and regional clusters. 35 | func (d *Deployer) DumpClusterLogs() error { 36 | if len(d.Zones) <= 0 { 37 | return fmt.Errorf("DumpClusterLogs is currently only supported for zonal clusters") 38 | } 39 | // gkeLogDumpTemplate is a template of a shell script where 40 | // - %[1]s is the project 41 | // - %[2]s is the zone 42 | // - %[3]s is the KUBE_NODE_OS_DISTRIBUTION 43 | // - %[4]s is a filter composed of the instance groups 44 | // - %[5]s is the log-dump.sh command line 45 | const gkeLogDumpTemplate = ` 46 | function log_dump_custom_get_instances() { 47 | if [[ $1 == "master" ]]; then 48 | return 0 49 | fi 50 | 51 | gcloud compute instances list '--project=%[1]s' '--filter=%[4]s' '--format=get(name)' 52 | } 53 | export -f log_dump_custom_get_instances 54 | # Set below vars that log-dump.sh expects in order to use scp with gcloud. 55 | export PROJECT=%[1]s 56 | export ZONE='%[2]s' 57 | export KUBERNETES_PROVIDER=gke 58 | export KUBE_NODE_OS_DISTRIBUTION='%[3]s' 59 | %[5]s 60 | ` 61 | for _, project := range d.Projects { 62 | // Prevent an obvious injection. 63 | if strings.Contains(d.localLogsDir, "'") || strings.Contains(d.gcsLogsDir, "'") { 64 | return fmt.Errorf("%q or %q contain single quotes - nice try", d.localLogsDir, d.gcsLogsDir) 65 | } 66 | 67 | // Generate a slice of filters to be OR'd together below 68 | var filters []string 69 | for _, cluster := range d.projectClustersLayout[project] { 70 | if err := d.GetInstanceGroups(); err != nil { 71 | return err 72 | } 73 | for _, ig := range d.instanceGroups[project][cluster.name] { 74 | filters = append(filters, fmt.Sprintf("(metadata.created-by:*%s)", ig.path)) 75 | } 76 | } 77 | 78 | // Generate the log-dump.sh command-line 79 | dumpCmd := fmt.Sprintf("./cluster/log-dump/log-dump.sh '%s'", d.localLogsDir) 80 | if d.gcsLogsDir != "" { 81 | dumpCmd += " " + d.gcsLogsDir 82 | } 83 | cmd := exec.Command("bash", "-c", fmt.Sprintf(gkeLogDumpTemplate, 84 | project, 85 | d.Zones[d.retryCount], 86 | os.Getenv("NODE_OS_DISTRIBUTION"), 87 | strings.Join(filters, " OR "), 88 | dumpCmd)) 89 | cmd.SetDir(d.RepoRoot) 90 | if err := runWithOutput(cmd); err != nil { 91 | return err 92 | } 93 | } 94 | 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/options/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | gkeBuild "sigs.k8s.io/kubetest2/kubetest2-gke/deployer/build" 24 | "sigs.k8s.io/kubetest2/pkg/build" 25 | ) 26 | 27 | type BuildOptions struct { 28 | CommonBuildOptions *build.Options 29 | UpdateLatestGreenMarker bool `flag:"~update-latest-green-marker" desc:"When set to true, will update the latest-green-x.y.txt marker on GCS."` 30 | BuildScript string `flag:"~build-script" desc:"Only used with the gke_make build. Absolute path to the gke_make build script."` 31 | } 32 | 33 | var _ build.Builder = &BuildOptions{} 34 | var _ build.Stager = &BuildOptions{} 35 | 36 | func (bo *BuildOptions) Validate() error { 37 | if bo.CommonBuildOptions.Strategy == string(gkeBuild.GKEMakeStrategy) { 38 | if bo.BuildScript != "" { 39 | if _, err := os.Stat(bo.BuildScript); err != nil { 40 | return fmt.Errorf("failed to validate --build-script, required with --strategy=gke_make: %w", err) 41 | } 42 | gkeMake := &gkeBuild.GKEMake{ 43 | RepoRoot: bo.CommonBuildOptions.RepoRoot, 44 | BuildScript: bo.BuildScript, 45 | VersionSuffix: bo.CommonBuildOptions.VersionSuffix, 46 | StageLocation: bo.CommonBuildOptions.StageLocation, 47 | UpdateLatest: bo.CommonBuildOptions.UpdateLatest, 48 | } 49 | bo.CommonBuildOptions.Builder = gkeMake 50 | bo.CommonBuildOptions.Stager = gkeMake 51 | return nil 52 | } 53 | } 54 | return bo.CommonBuildOptions.Validate() 55 | } 56 | 57 | func (bo *BuildOptions) Build() (string, error) { 58 | return bo.CommonBuildOptions.Build() 59 | } 60 | 61 | func (bo *BuildOptions) Stage(version string) error { 62 | return bo.CommonBuildOptions.Stage(version) 63 | } 64 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/options/cluster.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | import "fmt" 20 | 21 | type ExtraNodePoolOptions struct { 22 | Name string 23 | MachineType string 24 | ImageType string 25 | NumNodes int 26 | } 27 | 28 | type ClusterOptions struct { 29 | Environment string `flag:"~environment" desc:"Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided"` 30 | 31 | GcloudCommandGroup string `flag:"~gcloud-command-group" desc:"gcloud command group, can be one of empty, alpha, beta."` 32 | Autopilot bool `flag:"~autopilot" desc:"Whether to create GKE Autopilot clusters or not."` 33 | GcloudExtraFlags string `flag:"~gcloud-extra-flags" desc:"Extra gcloud flags to pass when creating the clusters."` 34 | CreateCommandFlag string `flag:"~create-command" desc:"gcloud subcommand and additional flags used to create a cluster, such as container clusters create --quiet. If it's specified, --gcloud-command-group, --autopilot, --gcloud-extra-flags will be ignored."` 35 | 36 | Regions []string `flag:"~region" desc:"Comma separated list for use with gcloud commands to specify the cluster region(s). The first region will be considered the primary region, and the rest will be considered the backup regions."` 37 | Zones []string `flag:"~zone" desc:"Comma separated list for use with gcloud commands to specify the cluster zone(s). The first zone will be considered the primary zone, and the rest will be considered the backup zones."` 38 | 39 | NumClusters int `flag:"~num-clusters" desc:"Number of clusters to create, will auto-generate names as (kt2--)."` 40 | Clusters []string `flag:"~cluster-name" desc:"Cluster names separated by comma. Must be set. For multi-project profile, it should be in the format of clusterA:0,clusterB:1,clusterC:2, where the index means the index of the project."` 41 | MachineType string `flag:"~machine-type" desc:"For use with gcloud commands to specify the machine type for the cluster."` 42 | NumNodes int `flag:"~num-nodes" desc:"For use with gcloud commands to specify the number of nodes for each of the cluster's zones."` 43 | ImageType string `flag:"~image-type" desc:"The image type to use for the cluster."` 44 | ReleaseChannel string `desc:"Use a GKE release channel, could be one of empty, rapid, regular and stable - https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels"` 45 | LegacyClusterVersion string `flag:"~version,deprecated" desc:"Use --cluster-version instead"` 46 | ClusterVersion string `desc:"Use a specific GKE version e.g. 1.16.13.gke-400, 'latest' or ''. If --build is specified it will default to building kubernetes from source."` 47 | WorkloadIdentityEnabled bool `flag:"~enable-workload-identity" desc:"Whether enable workload identity for the cluster or not. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity."` 48 | FirewallRuleAllow string `desc:"A list of protocols and ports whose traffic will be allowed for the firewall rules created for the cluster."` 49 | 50 | WindowsEnabled bool `flag:"~enable-windows" desc:"Whether enable Windows node pool in the cluster or not."` 51 | WindowsNumNodes int `flag:"~windows-num-nodes" desc:"For use with gcloud commands to specify the number of nodes for Windows node pools in the cluster."` 52 | WindowsMachineType string `flag:"~windows-machine-type" desc:"For use with gcloud commands to specify the machine type for Windows node in the cluster."` 53 | WindowsImageType string `flag:"~windows-image-type" desc:"The Windows image type to use for the cluster."` 54 | 55 | NodePoolCreateConcurrency int `flag:"~nodepool-create-concurrency" desc:"Number of nodepools to create concurrently, default is 1"` 56 | ExtraNodePool []string `flag:"~extra-nodepool" desc:"create an extra nodepool. repeat the flag for another nodepool. options as key=value&key=value... supported options are name,machine-type,image-type,num-nodes. "` 57 | 58 | RetryableErrorPatterns []string `flag:"~retryable-error-patterns" desc:"Comma separated list of regex match patterns for retryable errors during cluster creation."` 59 | } 60 | 61 | func (uo *ClusterOptions) Validate() error { 62 | // allow max 99 clusters (should be sufficient for most use cases) 63 | if uo.NumClusters < 1 || uo.NumClusters > 99 { 64 | return fmt.Errorf("need to specify between 1 and 99 clusters got %q: ", uo.NumClusters) 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/options/common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | type CommonOptions struct { 20 | RepoRoot string `desc:"Path to root of the kubernetes repo. Used with --build and for dumping cluster logs."` 21 | GCPServiceAccount string `flag:"~gcp-service-account" desc:"Service account to activate before using gcloud."` 22 | GCPSSHKeyIgnored bool `flag:"~ignore-gcp-ssh-key" desc:"Whether the GCP SSH key should be ignored or not for bringing up the cluster."` 23 | } 24 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/options/network.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | type NetworkOptions struct { 20 | Network string `flag:"~network" desc:"Cluster network. Defaults to the default network if not provided. For multi-project use cases, this will be the Shared VPC network name."` 21 | 22 | PrivateClusterAccessLevel string `flag:"~private-cluster-access-level" desc:"Private cluster access level, if not empty, must be one of 'no', 'limited' or 'unrestricted'. See the details in https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters."` 23 | PrivateClusterMasterIPRanges []string `flag:"~private-cluster-master-ip-range" desc:"Private cluster master IP ranges. It should be IPv4 CIDR(s), and its length must be the same as the number of clusters if private cluster is requested."` 24 | SubnetworkRanges []string `flag:"~subnetwork-ranges" desc:"Subnetwork ranges as required for shared VPC setup as described in https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_network_and_two_subnets. For multi-project profile, it is required and should be in the format of 10.0.4.0/22 10.0.32.0/20 10.4.0.0/14,172.16.4.0/22 172.16.16.0/20 172.16.4.0/22, where the subnetworks configuration for different project are separated by comma, and the ranges of each subnetwork configuration is separated by space."` 25 | } 26 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/options/project.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | type ProjectOptions struct { 20 | Projects []string `flag:"~project" desc:"Comma separated list of GCP Project(s) to use for creating the cluster."` 21 | 22 | BoskosLocation string `flag:"~boskos-location" desc:"If set, manually specifies the location of the Boskos server."` 23 | BoskosAcquireTimeoutSeconds int `flag:"~boskos-acquire-timeout-seconds" desc:"How long (in seconds) to hang on a request to Boskos to acquire a resource before erroring."` 24 | BoskosHeartbeatIntervalSeconds int `flag:"~boskos-heartbeat-interval-seconds" desc:"How often (in seconds) to send a heartbeat to Boskos to hold the acquired resource. 0 means no heartbeat."` 25 | BoskosResourceType []string `flag:"~boskos-resource-type" desc:"If set, manually specifies the resource type(s) of GCP projects to acquire from Boskos."` 26 | BoskosProjectsRequested []int `flag:"~projects-requested" desc:"Number of projects to request from Boskos. It is only respected if projects is empty, and must be larger than zero."` 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/post_tester.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | 22 | util "sigs.k8s.io/kubetest2/kubetest2-gke/deployer/utils" 23 | ) 24 | 25 | var ( 26 | greenMarkerPrefix = "latest-green" 27 | ) 28 | 29 | // PostTest will check if there's any error in the test. If there's no 30 | // error in the test, and the --update-latest-green-marker is set to true, 31 | // this method will stage the build marker to the GCS bucket. 32 | func (d *Deployer) PostTest(testErr error) error { 33 | if testErr != nil || !d.BuildOptions.UpdateLatestGreenMarker { 34 | return nil 35 | } 36 | // If we are here, that means the new build passes the smoke test. 37 | if err := util.StageGKEBuildMarker(d.ClusterVersion, d.CommonBuildOptions.StageLocation, greenMarkerPrefix); err != nil { 38 | return fmt.Errorf("error during green build marker staging: %s", err) 39 | } 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/utils/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "regexp" 23 | "strings" 24 | 25 | "k8s.io/klog/v2" 26 | "sigs.k8s.io/kubetest2/pkg/exec" 27 | ) 28 | 29 | var ( 30 | gkeMinorVersionRegex = regexp.MustCompile(`^v?(\d\.\d+).*$`) 31 | ) 32 | 33 | // StageGKEBuildMarker stages the build marker to the stage location. 34 | func StageGKEBuildMarker(version, stageLocation, markerPrefix string) error { 35 | destinationURL := stageLocation + "/" + fileName(version, markerPrefix) 36 | pushCmd := fmt.Sprintf("gsutil -h 'Content-Type:text/plain' cp - %s", destinationURL) 37 | cmd := exec.RawCommand(pushCmd) 38 | if !strings.HasPrefix(version, "v") { 39 | version = "v" + version 40 | } 41 | cmd.SetStdin(strings.NewReader(version)) 42 | exec.SetOutput(cmd, os.Stdout, os.Stderr) 43 | if err := cmd.Run(); err != nil { 44 | return fmt.Errorf("failed to upload the latest version number: %s", err) 45 | } 46 | return nil 47 | } 48 | 49 | func fileName(version, markerPrefix string) string { 50 | m := gkeMinorVersionRegex.FindStringSubmatch(version) 51 | if len(m) < 2 { 52 | klog.Warningf("can't find the minor version of %s, defaulting to %s.txt %v", version, markerPrefix, m) 53 | return fmt.Sprintf("%s.txt", markerPrefix) 54 | } 55 | 56 | minor := m[1] 57 | return fmt.Sprintf("%s-%s.txt", markerPrefix, minor) 58 | } 59 | -------------------------------------------------------------------------------- /kubetest2-gke/deployer/utils/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestFileName(t *testing.T) { 24 | testCases := []struct { 25 | desc string 26 | version string 27 | markerPrefix string 28 | expectedFileName string 29 | }{ 30 | { 31 | desc: "latest-green, 1.23", 32 | version: "1.23", 33 | markerPrefix: "latest-green", 34 | expectedFileName: "latest-green-1.23.txt", 35 | }, 36 | { 37 | desc: "latest, v1.22.0-alpha.3", 38 | version: "v1.22.0-alpha.3", 39 | markerPrefix: "latest", 40 | expectedFileName: "latest-1.22.txt", 41 | }, 42 | { 43 | desc: "latest, invalid tag", 44 | version: "beta", 45 | markerPrefix: "latest", 46 | expectedFileName: "latest.txt", 47 | }, 48 | } 49 | 50 | for _, tc := range testCases { 51 | tc := tc 52 | t.Run(tc.desc, func(st *testing.T) { 53 | st.Parallel() 54 | fileName := fileName(tc.version, tc.markerPrefix) 55 | if fileName != tc.expectedFileName { 56 | t.Errorf("FileName mismatch: got %s, want %s.", fileName, tc.expectedFileName) 57 | } 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /kubetest2-gke/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/app" 21 | 22 | "sigs.k8s.io/kubetest2/kubetest2-gke/deployer" 23 | ) 24 | 25 | func main() { 26 | app.Main(deployer.Name, deployer.New) 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-kind/README.md: -------------------------------------------------------------------------------- 1 | # Kubetest2 KIND Deployer 2 | 3 | This component of kubetest2 is responsible for test cluster lifecycles for clusters deployed using kind. 4 | 5 | ## Usage 6 | 7 | Currently, the kind deployer is capable of building and deploying a cluster using kind. 8 | 9 | ``` 10 | kubetest2 kind --build --up --down --test=exec -- kubectl get all -A 11 | ``` 12 | 13 | See the usage (`--help`) for more options. 14 | 15 | ## Implementation 16 | The deployer builds a kind node image and is essentially a Golang wrapper for building e2e dependencies as in `e2e-k8s.sh` located [here](https://github.com/kubernetes-sigs/kind/blob/main/hack/ci/e2e-k8s.sh#L72-L86) in [kind](https://github.com/kubernetes-sigs/kind/tree/main) 17 | -------------------------------------------------------------------------------- /kubetest2-kind/ci-tests/buildupdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2018 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | make install 23 | make install-deployer-kind 24 | make install-tester-ginkgo 25 | go install sigs.k8s.io/kind@v0.24.0 26 | 27 | cd "${GOPATH}/src/k8s.io/kubernetes" 28 | 29 | # kubetest2 against k/k 30 | kubetest2 kind \ 31 | -v=2 \ 32 | --build \ 33 | --up \ 34 | --down \ 35 | --test=ginkgo \ 36 | -- \ 37 | --focus-regex='Secrets should be consumable via the environment' \ 38 | --skip-regex='\[Driver:.gcepd\]|\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]' \ 39 | --use-built-binaries=true \ 40 | --timeout=30m 41 | -------------------------------------------------------------------------------- /kubetest2-kind/deployer/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | "runtime" 24 | 25 | "k8s.io/klog/v2" 26 | 27 | "sigs.k8s.io/kubetest2/pkg/build" 28 | "sigs.k8s.io/kubetest2/pkg/exec" 29 | "sigs.k8s.io/kubetest2/pkg/fs" 30 | "sigs.k8s.io/kubetest2/pkg/process" 31 | ) 32 | 33 | const ( 34 | target = "all" 35 | ) 36 | 37 | func (d *deployer) Build() error { 38 | args := []string{ 39 | "build", "node-image", 40 | } 41 | if d.KubeRoot != "" { 42 | args = append(args, d.KubeRoot) 43 | } 44 | if d.BuildType != "" { 45 | args = append(args, "--type", d.BuildType) 46 | } 47 | if d.BaseImage != "" { 48 | args = append(args, "--base-image", d.BaseImage) 49 | } 50 | // set the explicitly specified image name if set 51 | if d.NodeImage != "" { 52 | args = append(args, "--image", d.NodeImage) 53 | } else if d.commonOptions.ShouldBuild() { 54 | // otherwise if we just built an image, use that 55 | args = append(args, "--image", kindDefaultBuiltImageName) 56 | } 57 | 58 | klog.V(0).Infof("Build(): building kind node image...\n") 59 | // we want to see the output so use process.ExecJUnit 60 | if err := process.ExecJUnit("kind", args, os.Environ()); err != nil { 61 | return err 62 | } 63 | klog.V(0).Infof("Build(): build e2e requirements...\n") 64 | e2ePath := "test/e2e/e2e.test" 65 | kubectlPath := "cmd/kubectl" 66 | ginkgoPath := "vendor/github.com/onsi/ginkgo/v2/ginkgo" 67 | 68 | // make sure we have e2e requirements 69 | cmd := exec.Command("make", target, 70 | fmt.Sprintf("WHAT=%s %s %s", kubectlPath, e2ePath, ginkgoPath)) 71 | cmd.SetDir(d.KubeRoot) 72 | exec.InheritOutput(cmd) 73 | if err := cmd.Run(); err != nil { 74 | return err 75 | } 76 | 77 | //move files 78 | const dockerizedOutput = "_output/dockerized" 79 | for _, binary := range build.CommonTestBinaries { 80 | source := filepath.Join(d.KubeRoot, "_output/bin", binary) 81 | dest := filepath.Join(d.KubeRoot, dockerizedOutput, "bin", runtime.GOOS, runtime.GOARCH, binary) 82 | if err := fs.CopyFile(source, dest); err != nil { 83 | klog.Warningf("failed to copy %s to %s: %v", source, dest, err) 84 | } 85 | } 86 | 87 | build.StoreCommonBinaries(d.KubeRoot, d.commonOptions.RunDir()) 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /kubetest2-kind/deployer/deployer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package deployer implements the kubetest2 kind deployer 18 | package deployer 19 | 20 | import ( 21 | "flag" 22 | "os" 23 | "path/filepath" 24 | 25 | "github.com/octago/sflags/gen/gpflag" 26 | "github.com/spf13/pflag" 27 | "k8s.io/klog/v2" 28 | 29 | "sigs.k8s.io/kubetest2/pkg/artifacts" 30 | "sigs.k8s.io/kubetest2/pkg/types" 31 | ) 32 | 33 | // Name is the name of the deployer 34 | const Name = "kind" 35 | 36 | var GitTag string 37 | 38 | // New implements deployer.New for kind 39 | func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { 40 | // create a deployer object and set fields that are not flag controlled 41 | d := &deployer{ 42 | commonOptions: opts, 43 | logsDir: filepath.Join(artifacts.BaseDir(), "logs"), 44 | } 45 | // register flags and return 46 | return d, bindFlags(d) 47 | } 48 | 49 | // assert that New implements types.NewDeployer 50 | var _ types.NewDeployer = New 51 | 52 | type deployer struct { 53 | // generic parts 54 | commonOptions types.Options 55 | // kind specific details 56 | NodeImage string `flag:"image-name" desc:"the image name to use for build and up"` 57 | BaseImage string `flag:"base-image" desc:"the base image name to use for the build"` 58 | ClusterName string `flag:"cluster-name" desc:"the kind cluster --name"` 59 | BuildType string `desc:"--type for kind build node-image"` 60 | ConfigPath string `flag:"config" desc:"--config for kind create cluster"` 61 | KubeconfigPath string `flag:"kubeconfig" desc:"--kubeconfig flag for kind create cluster"` 62 | KubeRoot string `desc:"the Kubernetes source for kind build node-image"` 63 | 64 | logsDir string 65 | } 66 | 67 | func (d *deployer) Kubeconfig() (string, error) { 68 | if d.KubeconfigPath != "" { 69 | return d.KubeconfigPath, nil 70 | } 71 | home, err := os.UserHomeDir() 72 | if err != nil { 73 | return "", err 74 | } 75 | return filepath.Join(home, ".kube", "config"), nil 76 | } 77 | 78 | func (d *deployer) Version() string { 79 | return GitTag 80 | } 81 | 82 | // helper used to create & bind a flagset to the deployer 83 | func bindFlags(d *deployer) *pflag.FlagSet { 84 | flags, err := gpflag.Parse(d) 85 | if err != nil { 86 | klog.Fatalf("unable to generate flags from deployer") 87 | return nil 88 | } 89 | 90 | // initing the klog flags adds them to goflag.CommandLine 91 | // they can then be added to the built pflag set 92 | klog.InitFlags(nil) 93 | flags.AddGoFlagSet(flag.CommandLine) 94 | 95 | return flags 96 | } 97 | 98 | // assert that deployer implements types.DeployerWithKubeconfig 99 | var _ types.DeployerWithKubeconfig = &deployer{} 100 | 101 | // well-known kind related constants 102 | const kindDefaultBuiltImageName = "kindest/node:latest" 103 | -------------------------------------------------------------------------------- /kubetest2-kind/deployer/down.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "os" 21 | 22 | "k8s.io/klog/v2" 23 | 24 | "sigs.k8s.io/kubetest2/pkg/process" 25 | ) 26 | 27 | func (d *deployer) Down() error { 28 | if err := d.DumpClusterLogs(); err != nil { 29 | klog.Warningf("Dumping cluster logs at the start of Down() failed: %v", err) 30 | } 31 | args := []string{ 32 | "delete", "cluster", 33 | "--name", d.ClusterName, 34 | } 35 | 36 | klog.V(0).Infof("Down(): deleting kind cluster...%s\n", d.ClusterName) 37 | // we want to see the output so use process.ExecJUnit 38 | return process.ExecJUnit("kind", args, os.Environ()) 39 | } 40 | -------------------------------------------------------------------------------- /kubetest2-kind/deployer/dumplogs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "os" 21 | 22 | "k8s.io/klog/v2" 23 | 24 | "sigs.k8s.io/kubetest2/pkg/process" 25 | ) 26 | 27 | func (d *deployer) DumpClusterLogs() error { 28 | args := []string{ 29 | "export", "logs", 30 | "--name", d.ClusterName, 31 | d.logsDir, 32 | } 33 | 34 | klog.V(0).Infof("DumpClusterLogs(): exporting kind cluster logs...\n") 35 | // we want to see the output so use process.ExecJUnit 36 | return process.ExecJUnit("kind", args, os.Environ()) 37 | } 38 | -------------------------------------------------------------------------------- /kubetest2-kind/deployer/up.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package deployer 18 | 19 | import ( 20 | "os" 21 | "strings" 22 | 23 | "k8s.io/klog/v2" 24 | 25 | "sigs.k8s.io/kubetest2/pkg/exec" 26 | "sigs.k8s.io/kubetest2/pkg/metadata" 27 | "sigs.k8s.io/kubetest2/pkg/process" 28 | ) 29 | 30 | func (d *deployer) IsUp() (up bool, err error) { 31 | // naively assume that if the api server reports nodes, the cluster is up 32 | lines, err := exec.CombinedOutputLines( 33 | exec.Command("kubectl", "get", "nodes", "-o=name"), 34 | ) 35 | if err != nil { 36 | return false, metadata.NewJUnitError(err, strings.Join(lines, "\n")) 37 | } 38 | return len(lines) > 0, nil 39 | } 40 | 41 | func (d *deployer) Up() error { 42 | args := []string{ 43 | "create", "cluster", 44 | "--name", d.ClusterName, 45 | } 46 | 47 | // set the explicitly specified image name if set 48 | if d.NodeImage != "" { 49 | args = append(args, "--image", d.NodeImage) 50 | } else if d.commonOptions.ShouldBuild() { 51 | // otherwise if we just built an image, use that 52 | // NOTE: this is safe in the face of upstream changes, because 53 | // we use the same logic / constant for Build() 54 | args = append(args, "--image", kindDefaultBuiltImageName) 55 | } 56 | if d.ConfigPath != "" { 57 | args = append(args, "--config", d.ConfigPath) 58 | } 59 | if d.KubeconfigPath != "" { 60 | args = append(args, "--kubeconfig", d.KubeconfigPath) 61 | } 62 | 63 | klog.V(0).Infof("Up(): creating kind cluster...\n") 64 | // we want to see the output so use process.ExecJUnit 65 | return process.ExecJUnit("kind", args, os.Environ()) 66 | } 67 | -------------------------------------------------------------------------------- /kubetest2-kind/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/app" 21 | 22 | "sigs.k8s.io/kubetest2/kubetest2-kind/deployer" 23 | ) 24 | 25 | func main() { 26 | app.Main(deployer.Name, deployer.New) 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-noop/deployer/deployer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package deployer implements the kubetest2 kind deployer 18 | package deployer 19 | 20 | import ( 21 | "flag" 22 | "os" 23 | "path/filepath" 24 | 25 | "github.com/octago/sflags/gen/gpflag" 26 | "github.com/spf13/pflag" 27 | "k8s.io/klog/v2" 28 | 29 | "sigs.k8s.io/kubetest2/pkg/types" 30 | ) 31 | 32 | // Name is the name of the deployer 33 | const Name = "noop" 34 | 35 | var GitTag string 36 | 37 | // New implements deployer.New for kind 38 | func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { 39 | // create a deployer object and set fields that are not flag controlled 40 | d := &deployer{ 41 | commonOptions: opts, 42 | } 43 | // register flags and return 44 | return d, bindFlags(d) 45 | } 46 | 47 | // assert that New implements types.NewDeployer 48 | var _ types.NewDeployer = New 49 | 50 | type deployer struct { 51 | // generic parts 52 | commonOptions types.Options 53 | 54 | KubeconfigPath string `flag:"kubeconfig" desc:"Absolute path to existing kubeconfig for cluster"` 55 | } 56 | 57 | func (d *deployer) Up() error { 58 | return nil 59 | } 60 | 61 | func (d *deployer) Down() error { 62 | return nil 63 | } 64 | 65 | func (d *deployer) IsUp() (up bool, err error) { 66 | return false, nil 67 | } 68 | 69 | func (d *deployer) DumpClusterLogs() error { 70 | return nil 71 | } 72 | 73 | func (d *deployer) Build() error { 74 | // TODO: build should probably still exist with common options 75 | return nil 76 | } 77 | 78 | func (d *deployer) Kubeconfig() (string, error) { 79 | // noop deployer is specifically used with an existing cluster and KUBECONFIG 80 | if d.KubeconfigPath != "" { 81 | return d.KubeconfigPath, nil 82 | } 83 | if kconfig, ok := os.LookupEnv("KUBECONFIG"); ok { 84 | return kconfig, nil 85 | } 86 | home, err := os.UserHomeDir() 87 | if err != nil { 88 | return "", err 89 | } 90 | return filepath.Join(home, ".kube", "config"), nil 91 | } 92 | 93 | func (d *deployer) Version() string { 94 | return GitTag 95 | } 96 | 97 | // helper used to create & bind a flagset to the deployer 98 | func bindFlags(d *deployer) *pflag.FlagSet { 99 | flags, err := gpflag.Parse(d) 100 | if err != nil { 101 | klog.Fatalf("unable to generate flags from deployer") 102 | return nil 103 | } 104 | 105 | // initing the klog flags adds them to goflag.CommandLine 106 | // they can then be added to the built pflag set 107 | klog.InitFlags(nil) 108 | flags.AddGoFlagSet(flag.CommandLine) 109 | 110 | return flags 111 | } 112 | 113 | // assert that deployer implements types.DeployerWithKubeconfig 114 | var _ types.DeployerWithKubeconfig = &deployer{} 115 | -------------------------------------------------------------------------------- /kubetest2-noop/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/app" 21 | 22 | "sigs.k8s.io/kubetest2/kubetest2-noop/deployer" 23 | ) 24 | 25 | func main() { 26 | app.Main(deployer.Name, deployer.New) 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-tester-clusterloader2/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/testers/clusterloader2" 21 | ) 22 | 23 | func main() { 24 | clusterloader2.Main() 25 | } 26 | -------------------------------------------------------------------------------- /kubetest2-tester-exec/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package exec implements a kubetest2 tester that simply executes the arguments 18 | // as a subprocess 19 | package main 20 | 21 | import ( 22 | "sigs.k8s.io/kubetest2/pkg/testers/exec" 23 | ) 24 | 25 | func main() { 26 | exec.Main() 27 | } 28 | -------------------------------------------------------------------------------- /kubetest2-tester-ginkgo/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | ginkgotester "sigs.k8s.io/kubetest2/pkg/testers/ginkgo" 21 | ) 22 | 23 | func main() { 24 | ginkgotester.Main() 25 | } 26 | -------------------------------------------------------------------------------- /kubetest2-tester-node/ci-tests/gce-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2023 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | set -o xtrace 21 | 22 | make install-all 23 | 24 | kubetest2 noop \ 25 | --test=node \ 26 | -- \ 27 | --provider=gce \ 28 | --repo-root="${GOPATH}/src/k8s.io/kubernetes" \ 29 | --gcp-zone=us-west1-b \ 30 | --instance-type=e2-standard-2 \ 31 | --node-env=PULL_REFS="${PULL_REFS}" \ 32 | --focus-regex="\[NodeConformance\]" \ 33 | --test-args='--kubelet-flags="--cgroup-driver=systemd --runtime-cgroups=/system.slice/containerd.service"' \ 34 | --image-config-file="${GOPATH}/src/k8s.io/test-infra/jobs/e2e_node/containerd/image-config-systemd.yaml" 35 | -------------------------------------------------------------------------------- /kubetest2-tester-node/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | nodetester "sigs.k8s.io/kubetest2/pkg/testers/node" 21 | ) 22 | 23 | func main() { 24 | nodetester.Main() 25 | } 26 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "sigs.k8s.io/kubetest2/pkg/app/shim" 21 | ) 22 | 23 | func main() { 24 | shim.Main() 25 | } 26 | -------------------------------------------------------------------------------- /pkg/app/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package app implements the kubetest2 high level application logic 18 | package app 19 | 20 | // TODO(bentheelder): actually implement! 21 | -------------------------------------------------------------------------------- /pkg/app/shim/const.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package shim 18 | 19 | // BinaryName is the name of the binary 20 | const BinaryName = "kubetest2" 21 | -------------------------------------------------------------------------------- /pkg/app/shim/deployers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package shim 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "os/exec" 23 | "path/filepath" 24 | "strings" 25 | ) 26 | 27 | // FindDeployer locates the binary implementing the named deployer 28 | // TODO(bentheelder): move this to another package? 29 | func FindDeployer(name string) (path string, err error) { 30 | binary := fmt.Sprintf("%s-%s", BinaryName, name) 31 | path, err = exec.LookPath(binary) 32 | if err != nil { 33 | return "", fmt.Errorf("%#v not found in PATH, could not locate %#v deployer", binary, name) 34 | } 35 | return path, err 36 | } 37 | 38 | // FindDeployers looks for all deployers in PATH, returning a map of the 39 | // deployer name to the first matching binary found in path 40 | func FindDeployers() map[string]string { 41 | nameToPath := make(map[string]string) 42 | prefix := fmt.Sprintf("%s-", BinaryName) 43 | testerPrefix := fmt.Sprintf("%s-tester-", BinaryName) 44 | // search every directory in PATH for kubetest2-* binaries 45 | searchPaths := filepath.SplitList(os.Getenv("PATH")) 46 | for _, dir := range searchPaths { 47 | // mimic LookPath() for nicer results 48 | if dir == "" { 49 | // Unix shell semantics: path element "" means "." 50 | dir = "." 51 | } 52 | 53 | // list all files in the directory 54 | files, err := os.ReadDir(dir) 55 | 56 | // ignore bad directories in PATH 57 | if os.IsNotExist(err) { 58 | continue 59 | } 60 | 61 | // check every file in the directory against the prefix 62 | for _, f := range files { 63 | // ignore directories 64 | if f.IsDir() { 65 | continue 66 | } 67 | // ensure the prefix matches 68 | fileName := f.Name() 69 | if !strings.HasPrefix(fileName, prefix) { 70 | continue 71 | } 72 | // ignore if it is a tester 73 | if strings.HasPrefix(fileName, testerPrefix) { 74 | continue 75 | } 76 | // convert the file name to a deployer name 77 | // TODO(bentheelder): handle PATHEXT on windows 78 | name := strings.TrimPrefix(fileName, prefix) 79 | // only keep the first result 80 | if _, foundAlready := nameToPath[name]; foundAlready { 81 | continue 82 | } 83 | // use FindDeployer / LookPath to ensure consistency 84 | path, err := FindDeployer(name) 85 | if err != nil { 86 | continue 87 | } 88 | nameToPath[name] = path 89 | } 90 | } 91 | return nameToPath 92 | } 93 | -------------------------------------------------------------------------------- /pkg/app/shim/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package shim implements the kubetest2 root command logic, "shimming" to 18 | // deployer specific binaries 19 | package shim 20 | -------------------------------------------------------------------------------- /pkg/app/shim/shim.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package shim 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "github.com/spf13/cobra" 24 | "github.com/spf13/pflag" 25 | 26 | "sigs.k8s.io/kubetest2/pkg/process" 27 | ) 28 | 29 | // GitTag captures the git commit SHA of the build. This gets printed by all the deployers and testers 30 | var GitTag string 31 | 32 | // Main implements the kubetest2 root binary entrypoint 33 | func Main() { 34 | if err := Run(); err != nil { 35 | os.Exit(1) 36 | } 37 | } 38 | 39 | // Run instantiates and executes the shim cobra command, returning the result 40 | func Run() error { 41 | return NewCommand().Execute() 42 | } 43 | 44 | var usageLong = `kubetest2 is a tool for kubernetes end to end testing. 45 | 46 | It orchestrates creating clusters, building kubernetes, deleting clusters, running tests, etc. 47 | 48 | kubetest2 should be called with a deployer like: 'kubetest2 kind --help' 49 | 50 | For more information see: https://github.com/kubernetes-sigs/kubetest2` 51 | 52 | // NewCommand returns a new cobra.Command for building the base image 53 | func NewCommand() *cobra.Command { 54 | cmd := &cobra.Command{ 55 | Use: fmt.Sprintf("%s [deployer]", BinaryName), 56 | SilenceUsage: true, 57 | SilenceErrors: true, 58 | RunE: runE, 59 | } 60 | 61 | // enable our custom help function, which will list known deployers 62 | cmd.SetHelpFunc(help) 63 | 64 | // we want all flags to be passed through without parsing 65 | cmd.DisableFlagParsing = true 66 | 67 | return cmd 68 | } 69 | 70 | // runE implements the actual command logic 71 | func runE(cmd *cobra.Command, args []string) error { 72 | // there should be at least one argument (the deployer) unless the user 73 | // is asking for help on the shim itself 74 | if len(args) < 1 { 75 | return cmd.Help() 76 | } 77 | 78 | // gracefully handle help or version command if it is the only argument 79 | if len(args) == 1 { 80 | // check for -h, --help 81 | flags := pflag.NewFlagSet(BinaryName, pflag.ContinueOnError) 82 | help := flags.BoolP("help", "h", false, "") 83 | // check for -v, --version 84 | ver := flags.BoolP("version", "v", false, fmt.Sprintf("prints %s version", BinaryName)) 85 | // we don't care about errors, only if -h / --help was set 86 | _ = flags.Parse(args) 87 | if *help { 88 | return cmd.Help() 89 | } 90 | if *ver { 91 | fmt.Printf("%s version %s\n", BinaryName, GitTag) 92 | return nil 93 | } 94 | } 95 | 96 | // otherwise find and execute the deployer with the remaining arguments 97 | deployerName := args[0] 98 | deployer, err := FindDeployer(deployerName) 99 | if err != nil { 100 | cmd.Printf("Error: could not find kubetest2 deployer %#v\n", deployerName) 101 | cmd.Println() 102 | usage(cmd) 103 | return err 104 | } 105 | 106 | env := os.Environ() 107 | version := fmt.Sprintf("kubetest2 version %s", GitTag) 108 | env = append(env, fmt.Sprintf("KUBETEST2_VERSION=%s", version)) 109 | return process.Exec(deployer, args[1:], env) 110 | } 111 | 112 | // custom help info, includes usage() 113 | // 114 | //nolint:revive 115 | func help(cmd *cobra.Command, args []string) { 116 | cmd.Println(usageLong) 117 | cmd.Println() 118 | usage(cmd) 119 | } 120 | 121 | // the usage subset of help info, attempts to identify and list known deployers 122 | func usage(cmd *cobra.Command) { 123 | deployers := FindDeployers() 124 | cmd.Println("Usage:") 125 | cmd.Printf(" %s [deployer] [flags]\n", BinaryName) 126 | cmd.Println() 127 | cmd.Println("Detected Deployers:") 128 | for deployer := range deployers { 129 | cmd.Printf(" %s\n", deployer) 130 | } 131 | cmd.Println() 132 | testers := FindTesters() 133 | cmd.Println("Detected Testers:") 134 | for tester := range testers { 135 | cmd.Printf(" %s\n", tester) 136 | } 137 | cmd.Println() 138 | cmd.Println("For more help, run kubetest2 [deployer] --help") 139 | } 140 | -------------------------------------------------------------------------------- /pkg/app/shim/testers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package shim 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "os/exec" 23 | "path/filepath" 24 | "strings" 25 | ) 26 | 27 | // FindTester locates the binary implementing the named tester 28 | // TODO(bentheelder): move this to another package? 29 | func FindTester(name string) (path string, err error) { 30 | binary := fmt.Sprintf("%s-tester-%s", BinaryName, name) 31 | path, err = exec.LookPath(binary) 32 | if err != nil { 33 | return "", fmt.Errorf("%#v not found in PATH, could not locate %#v tester", binary, name) 34 | } 35 | return path, err 36 | } 37 | 38 | // FindTesters looks for all testers in PATH, returning a map of the 39 | // tester name to the first matching binary found in path 40 | func FindTesters() map[string]string { 41 | nameToPath := make(map[string]string) 42 | prefix := fmt.Sprintf("%s-tester-", BinaryName) 43 | 44 | // search every directory in PATH for kubetest2-tester-* binaries 45 | searchPaths := filepath.SplitList(os.Getenv("PATH")) 46 | for _, dir := range searchPaths { 47 | // mimic LookPath() for nicer results 48 | if dir == "" { 49 | // Unix shell semantics: path element "" means "." 50 | dir = "." 51 | } 52 | 53 | // list all files in the directory 54 | files, err := os.ReadDir(dir) 55 | 56 | // ignore bad directories in PATH 57 | if os.IsNotExist(err) { 58 | continue 59 | } 60 | 61 | // check every file in the directory against the prefix 62 | for _, f := range files { 63 | // ignore directories 64 | if f.IsDir() { 65 | continue 66 | } 67 | // ensure the prefix matches 68 | fileName := f.Name() 69 | if !strings.HasPrefix(fileName, prefix) { 70 | continue 71 | } 72 | // convert the file name to a deployer name 73 | // TODO(bentheelder): handle PATHEXT on windows 74 | name := strings.TrimPrefix(fileName, prefix) 75 | // only keep the first result 76 | if _, foundAlready := nameToPath[name]; foundAlready { 77 | continue 78 | } 79 | // use FindTester / LookPath to ensure consistency 80 | path, err := FindTester(name) 81 | if err != nil { 82 | continue 83 | } 84 | nameToPath[name] = path 85 | } 86 | } 87 | return nameToPath 88 | } 89 | -------------------------------------------------------------------------------- /pkg/artifacts/paths.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package artifacts 18 | 19 | import ( 20 | "fmt" 21 | "github.com/spf13/pflag" 22 | "k8s.io/klog/v2" 23 | "os" 24 | "path/filepath" 25 | ) 26 | 27 | var baseDir string 28 | var RunDirFlag string 29 | 30 | // BaseDir returns the path to the directory where artifacts should be written 31 | // (including metadata files like junit_runner.xml) 32 | func BaseDir() string { 33 | d := baseDir 34 | if d == "" { 35 | def, err := defaultArtifactsDir() 36 | if err != nil { 37 | klog.Fatalf("failed to get default artifacts directory: %s", err) 38 | } 39 | d = def 40 | } 41 | return d 42 | } 43 | 44 | // RunDir returns the path to the directory used for storing all files in general 45 | // specific to a single run of kubetest2 46 | func RunDir() string { 47 | d := RunDirFlag 48 | if d == "" { 49 | def, err := defaultRunDir() 50 | if err != nil { 51 | klog.Fatalf("failed to get default Run directory: %s", err) 52 | } 53 | d = def 54 | } 55 | return d 56 | } 57 | 58 | // the default is $ARTIFACTS if set, otherwise ./_artifacts 59 | // constructed as an absolute path to help the ginkgo tester because 60 | // for some reason it needs an absolute path to the kubeconfig 61 | func defaultArtifactsDir() (string, error) { 62 | if path, set := os.LookupEnv("ARTIFACTS"); set { 63 | absPath, err := filepath.Abs(path) 64 | if err != nil { 65 | return "", fmt.Errorf("failed to convert filepath from $ARTIFACTS (%s) to absolute path: %s", path, err) 66 | } 67 | return absPath, nil 68 | } 69 | 70 | absPath, err := filepath.Abs("_artifacts") 71 | if err != nil { 72 | return "", fmt.Errorf("when constructing default artifacts dir, failed to get absolute path: %s", err) 73 | } 74 | return absPath, nil 75 | } 76 | 77 | // the default is $KUBETEST2_RUN_DIR if set, otherwise ./_rundir 78 | // constructed as an absolute path to help the ginkgo tester because 79 | // for some reason it needs an absolute path to the kubeconfig 80 | func defaultRunDir() (string, error) { 81 | if path, set := os.LookupEnv("KUBETEST2_RUN_DIR"); set { 82 | absPath, err := filepath.Abs(path) 83 | if err != nil { 84 | return "", fmt.Errorf("failed to convert filepath from $KUBETEST2_RUN_DIR (%s) to absolute path: %s", path, err) 85 | } 86 | return absPath, nil 87 | } 88 | absPath, err := filepath.Abs("_rundir") 89 | if err != nil { 90 | return "", fmt.Errorf("when constructing default rundir, failed to get absolute path: %s", err) 91 | } 92 | return absPath, nil 93 | } 94 | 95 | // BindFlags binds the artifact and rundir related flags. 96 | func BindFlags(flags *pflag.FlagSet) error { 97 | defaultArtifacts, err := defaultArtifactsDir() 98 | if err != nil { 99 | return err 100 | } 101 | flags.StringVar(&baseDir, "artifacts", defaultArtifacts, `top-level directory to put artifacts under for each kubetest2 run, defaulting to "${ARTIFACTS:-./_artifacts}". If using the ginkgo tester, this must be an absolute path.`) 102 | flags.StringVar(&RunDirFlag, "rundir", "", `directory to put run related test binaries like e2e.test, ginkgo, kubectl for each kubetest2 run, defaulting to "${KUBETEST2_RUN_DIR:-./_rundir}". If using the ginkgo tester, this must be an absolute path.`) 103 | return nil 104 | } 105 | 106 | // MustBindFlags calls BindFlags and panics on an error 107 | func MustBindFlags(flags *pflag.FlagSet) { 108 | err := BindFlags(flags) 109 | if err != nil { 110 | klog.Fatalf("failed to get default artifacts || rundir directory: %s", err) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /pkg/boskos/boskos.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package boskos 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "os" 23 | "time" 24 | 25 | "k8s.io/klog/v2" 26 | "sigs.k8s.io/boskos/client" 27 | "sigs.k8s.io/boskos/common" 28 | ) 29 | 30 | // const (for the run) owner string for consistency between up and down 31 | var boskosOwner = os.Getenv("JOB_NAME") + "-kubetest2" 32 | 33 | // NewClient creates a boskos client for kubetest2 deployers. 34 | func NewClient(boskosLocation string) (*client.Client, error) { 35 | boskos, err := client.NewClient( 36 | boskosOwner, 37 | boskosLocation, 38 | "", 39 | "", 40 | ) 41 | if err != nil { 42 | return nil, fmt.Errorf("failed to create boskos client: %s", err) 43 | } 44 | 45 | return boskos, nil 46 | } 47 | 48 | // Acquire acquires a resource for the given type and starts a heartbeat goroutine to keep the resource reserved. 49 | func Acquire(boskosClient *client.Client, resourceType string, timeout, heartbeatInterval time.Duration, heartbeatClose chan struct{}) (*common.Resource, error) { 50 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 51 | defer cancel() 52 | 53 | boskosResource, err := boskosClient.AcquireWait(ctx, resourceType, "free", "busy") 54 | if err != nil { 55 | return nil, fmt.Errorf("failed to get a %q from boskos: %s", resourceType, err) 56 | } 57 | if boskosResource == nil { 58 | return nil, fmt.Errorf("boskos had no %s available", resourceType) 59 | } 60 | 61 | if heartbeatInterval != 0 { 62 | startBoskosHeartbeat( 63 | boskosClient, 64 | boskosResource, 65 | heartbeatInterval, 66 | heartbeatClose, 67 | ) 68 | } 69 | 70 | return boskosResource, nil 71 | } 72 | 73 | // startBoskosHeartbeat starts a goroutine that sends periodic updates to boskos 74 | // about the provided resource until the channel is closed. This prevents 75 | // reaper from taking the resource from the deployer while it is still in use. 76 | func startBoskosHeartbeat(boskosClient *client.Client, resource *common.Resource, interval time.Duration, heartbeatClose chan struct{}) { 77 | go func(c *client.Client, resource *common.Resource) { 78 | klog.V(2).Info("boskos hearbeat starting") 79 | 80 | for { 81 | select { 82 | case <-heartbeatClose: 83 | klog.V(2).Info("Boskos heartbeat func received signal to close") 84 | return 85 | case <-time.NewTicker(interval).C: 86 | klog.V(2).Info("Sending heartbeat to Boskos") 87 | if err := c.UpdateOne(resource.Name, "busy", nil); err != nil { 88 | klog.Warningf("[Boskos] Update of %s failed with %v", resource.Name, err) 89 | } 90 | } 91 | } 92 | }(boskosClient, resource) 93 | } 94 | 95 | // Release releases a resource. 96 | func Release(client *client.Client, resourceNames []string, heartbeatClose chan struct{}) error { 97 | for _, name := range resourceNames { 98 | if err := client.Release(name, "dirty"); err != nil { 99 | return fmt.Errorf("failed to release %s: %s", name, err) 100 | } 101 | } 102 | close(heartbeatClose) 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /pkg/build/bazel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | import ( 20 | "fmt" 21 | 22 | "k8s.io/klog/v2" 23 | 24 | "sigs.k8s.io/kubetest2/pkg/exec" 25 | ) 26 | 27 | type Bazel struct { 28 | RepoRoot string 29 | StageLocation string 30 | ImageLocation string 31 | } 32 | 33 | var _ Builder = &Bazel{} 34 | var _ Stager = &Bazel{} 35 | 36 | func (b *Bazel) Stage(version string) error { 37 | location := b.StageLocation + "/v" + version 38 | klog.V(0).Infof("Staging builds to %s ...", location) 39 | cmd := exec.Command("bazel", "run", "//:push-build", "--", location) 40 | cmd.SetDir(b.RepoRoot) 41 | exec.InheritOutput(cmd) 42 | return cmd.Run() 43 | } 44 | 45 | func (b *Bazel) Build() (string, error) { 46 | klog.V(0).Infof("Building kubernetes from %s ...", b.RepoRoot) 47 | version, err := sourceVersion(b.RepoRoot) 48 | if err != nil { 49 | return "", fmt.Errorf("failed to get version: %v", err) 50 | } 51 | cmd := exec.Command("bazel", "build", "//build/release-tars") 52 | cmd = cmd.SetDir(b.RepoRoot) 53 | setSourceDateEpoch(b.RepoRoot, cmd) 54 | exec.InheritOutput(cmd) 55 | return version, cmd.Run() 56 | } 57 | -------------------------------------------------------------------------------- /pkg/build/build.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package build implements a common system for building kubernetes for deployers to use. 18 | package build 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "runtime" 25 | "strings" 26 | 27 | "k8s.io/klog/v2" 28 | "sigs.k8s.io/kubetest2/pkg/exec" 29 | "sigs.k8s.io/kubetest2/pkg/fs" 30 | ) 31 | 32 | type Builder interface { 33 | // Build determines how kubernetes artifacts are built from sources or existing artifacts 34 | // and returns the version being built 35 | Build() (string, error) 36 | } 37 | 38 | type NoopBuilder struct{} 39 | 40 | var _ Builder = &NoopBuilder{} 41 | 42 | func (n *NoopBuilder) Build() (string, error) { 43 | return "", nil 44 | } 45 | 46 | // sourceVersion the kubernetes git version based on hack/print-workspace-status.sh 47 | // the raw version is also returned 48 | func sourceVersion(kubeRoot string) (string, error) { 49 | // get the version output 50 | cmd := exec.Command("sh", "-c", "hack/print-workspace-status.sh") 51 | cmd.SetDir(kubeRoot) 52 | output, err := exec.CombinedOutputLines(cmd) 53 | if err != nil { 54 | return "", err 55 | } 56 | 57 | // parse it, and populate it into _output/git_version 58 | version := "" 59 | for _, line := range output { 60 | parts := strings.SplitN(line, " ", 2) 61 | if len(parts) != 2 { 62 | return "", fmt.Errorf("could not parse kubernetes version: %q", strings.Join(output, "\n")) 63 | } 64 | if parts[0] == "gitVersion" { 65 | version = parts[1] 66 | return version, nil 67 | } 68 | } 69 | if version == "" { 70 | return "", fmt.Errorf("could not obtain kubernetes version: %q", strings.Join(output, "\n")) 71 | 72 | } 73 | return "", fmt.Errorf("could not find kubernetes version in output: %q", strings.Join(output, "\n")) 74 | } 75 | 76 | var ( 77 | CommonTestBinaries = []string{ 78 | "kubectl", 79 | "e2e.test", 80 | "ginkgo", 81 | } 82 | ) 83 | 84 | // StoreCommonBinaries will best effort try to store commonly built binaries 85 | // to the output directory 86 | func StoreCommonBinaries(kuberoot string, outroot string) { 87 | const dockerizedOutput = "_output/dockerized" 88 | root := filepath.Join(kuberoot, dockerizedOutput, "bin", runtime.GOOS, runtime.GOARCH) 89 | for _, binary := range CommonTestBinaries { 90 | source := filepath.Join(root, binary) 91 | dest := filepath.Join(outroot, binary) 92 | if _, err := os.Stat(source); err == nil { 93 | klog.V(2).Infof("copying %s to %s ...", source, dest) 94 | if err := fs.CopyFile(source, dest); err != nil { 95 | klog.Warningf("failed to copy %s to %s: %v", source, dest, err) 96 | } 97 | } else { 98 | klog.Warningf("could not find %s: %v", source, err) 99 | } 100 | } 101 | } 102 | 103 | // setSourceDateEpoch sets the SOURCE_DATE_EPOCH env to the commit timestamp of the latest commit in the 104 | // kubernetes repository, specified under kubeRoot, for reproducible builds 105 | // https://github.com/kubernetes/kubernetes/blob/7eae33cb0e1ead51c80ad517bc670113d77fa28d/build/README.md#reproducibility 106 | func setSourceDateEpoch(kubeRoot string, cmd exec.Cmd) { 107 | if os.Getenv("SOURCE_DATE_EPOCH") != "" { 108 | return 109 | } 110 | gitCmd := exec.Command("git", "log", "-1", "--pretty=%ct") 111 | gitCmd.SetDir(kubeRoot) 112 | if output, err := exec.CombinedOutputLines(gitCmd); err == nil { 113 | env := append(os.Environ(), fmt.Sprintf("SOURCE_DATE_EPOCH=%s", output[0])) 114 | cmd.SetEnv(env...) 115 | } else { 116 | klog.Warningf("failed to compute SOURCE_DATE_EPOCH from kubernetes repository: %v", err) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /pkg/build/krel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | import ( 20 | "fmt" 21 | "regexp" 22 | "strings" 23 | 24 | rbuild "k8s.io/release/pkg/build" 25 | ) 26 | 27 | type Krel struct { 28 | StageLocation string 29 | ImageLocation string 30 | RepoRoot string 31 | StageExtraFiles bool 32 | UpdateLatest bool 33 | } 34 | 35 | var _ Stager = &Krel{} 36 | 37 | // Stage stages the build to GCS using 38 | // essentially release/push-build.sh --bucket=B --ci --gcs-suffix=S --noupdatelatest 39 | func (rpb *Krel) Stage(version string) error { 40 | if !strings.HasPrefix(version, "v") { 41 | version = "v" + version 42 | } 43 | re := regexp.MustCompile(`^gs://([\w-]+)/(devel|ci)(/.*)?`) 44 | mat := re.FindStringSubmatch(rpb.StageLocation) 45 | if mat == nil || len(mat) < 4 { 46 | return fmt.Errorf("invalid stage location: %v. Use gs:////", rpb.StageLocation) 47 | } 48 | 49 | if err := rbuild.NewInstance(&rbuild.Options{ 50 | Bucket: mat[1], 51 | GCSRoot: mat[3], 52 | AllowDup: true, 53 | CI: mat[2] == "ci", 54 | NoUpdateLatest: !rpb.UpdateLatest, 55 | Registry: rpb.ImageLocation, 56 | Version: version, 57 | StageExtraFiles: rpb.StageExtraFiles, 58 | RepoRoot: rpb.RepoRoot, 59 | }).Push(); err != nil { 60 | 61 | return fmt.Errorf("stage via krel push: %w", err) 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /pkg/build/make.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | import ( 20 | "fmt" 21 | 22 | "k8s.io/klog/v2" 23 | "sigs.k8s.io/kubetest2/pkg/exec" 24 | ) 25 | 26 | type MakeBuilder struct { 27 | RepoRoot string 28 | TargetBuildArch string 29 | } 30 | 31 | var _ Builder = &MakeBuilder{} 32 | 33 | const ( 34 | target = "quick-release" 35 | ) 36 | 37 | // Build builds kubernetes with the quick-release make target 38 | func (m *MakeBuilder) Build() (string, error) { 39 | version, err := sourceVersion(m.RepoRoot) 40 | if err != nil { 41 | return "", fmt.Errorf("failed to get version: %v", err) 42 | } 43 | cmd := exec.Command("make", target, 44 | fmt.Sprintf("KUBE_BUILD_PLATFORMS=%s", m.TargetBuildArch)) 45 | klog.Infof("running build %s using: KUBE_BUILD_PLATFORMS=%s", target, m.TargetBuildArch) 46 | cmd.SetDir(m.RepoRoot) 47 | setSourceDateEpoch(m.RepoRoot, cmd) 48 | exec.InheritOutput(cmd) 49 | if err = cmd.Run(); err != nil { 50 | return "", err 51 | } 52 | return version, nil 53 | } 54 | -------------------------------------------------------------------------------- /pkg/build/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | import ( 20 | "fmt" 21 | ) 22 | 23 | // ignore package name stutter 24 | type BuildAndStageStrategy string //nolint:revive 25 | 26 | const ( 27 | // bazelStrategy builds and (optionally) stages using bazel 28 | bazelStrategy BuildAndStageStrategy = "bazel" 29 | // MakeStrategy builds using make and (optionally) stages using krel 30 | MakeStrategy BuildAndStageStrategy = "make" 31 | ) 32 | 33 | type Options struct { 34 | Strategy string `flag:"~strategy" desc:"Determines the build strategy to use either make or bazel."` 35 | StageLocation string `flag:"~stage" desc:"Upload binaries to gs://bucket/ci/job-suffix if set"` 36 | RepoRoot string `flag:"-"` 37 | ImageLocation string `flag:"~image-location" desc:"Image registry where built images are stored."` 38 | StageExtraGCPFiles bool `flag:"-"` 39 | VersionSuffix string `flag:"-"` 40 | UpdateLatest bool `flag:"~update-latest" desc:"Whether should upload the build number to the GCS"` 41 | TargetBuildArch string `flag:"~target-build-arch" desc:"Target architecture for the test artifacts for dockerized build"` 42 | Builder 43 | Stager 44 | } 45 | 46 | func (o *Options) Validate() error { 47 | return o.implementationFromStrategy() 48 | } 49 | 50 | func (o *Options) implementationFromStrategy() error { 51 | switch BuildAndStageStrategy(o.Strategy) { 52 | case bazelStrategy: 53 | bazel := &Bazel{ 54 | RepoRoot: o.RepoRoot, 55 | StageLocation: o.StageLocation, 56 | ImageLocation: o.ImageLocation, 57 | } 58 | o.Builder = bazel 59 | o.Stager = bazel 60 | case MakeStrategy: 61 | o.Builder = &MakeBuilder{ 62 | RepoRoot: o.RepoRoot, 63 | TargetBuildArch: o.TargetBuildArch, 64 | } 65 | o.Stager = &Krel{ 66 | RepoRoot: o.RepoRoot, 67 | StageLocation: o.StageLocation, 68 | ImageLocation: o.ImageLocation, 69 | StageExtraFiles: o.StageExtraGCPFiles, 70 | UpdateLatest: o.UpdateLatest, 71 | } 72 | default: 73 | return fmt.Errorf("unknown build strategy: %v", o.Strategy) 74 | } 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /pkg/build/stage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package build 18 | 19 | type Stager interface { 20 | // Stage determines how kubernetes artifacts will be staged (e.g. to say a GCS bucket) 21 | // for the specified version 22 | Stage(version string) error 23 | } 24 | 25 | type NoopStager struct{} 26 | 27 | var _ Stager = &NoopStager{} 28 | 29 | func (n *NoopStager) Stage(string) error { 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/exec/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package exec contains an interface for executing commands, along with helpers 18 | // TODO(bentheelder): add standardized timeout functionality & a default timeout 19 | // so that commands cannot hang indefinitely (!) 20 | package exec 21 | -------------------------------------------------------------------------------- /pkg/exec/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package exec 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "context" 23 | "io" 24 | "os" 25 | 26 | shell "github.com/kballard/go-shellquote" 27 | ) 28 | 29 | // Cmd abstracts over running a command somewhere, this is useful for testing 30 | type Cmd interface { 31 | Run() error 32 | // Each entry should be of the form "key=value" 33 | SetEnv(...string) Cmd 34 | SetStdin(io.Reader) Cmd 35 | SetStdout(io.Writer) Cmd 36 | SetStderr(io.Writer) Cmd 37 | SetDir(string) Cmd 38 | } 39 | 40 | // Cmder abstracts over creating commands 41 | type Cmder interface { 42 | // command, args..., just like os/exec.Cmd 43 | Command(string, ...string) Cmd 44 | CommandContext(context.Context, string, ...string) Cmd 45 | } 46 | 47 | // DefaultCmder is a LocalCmder instance used for convenience, packages 48 | // originally using os/exec.Command can instead use pkg/kind/exec.Command 49 | // which forwards to this instance 50 | // TODO(bentheelder): swap this for testing 51 | // TODO(bentheelder): consider not using a global for this :^) 52 | var DefaultCmder = &LocalCmder{} 53 | 54 | // Command is a convenience wrapper over DefaultCmder.Command 55 | func Command(command string, args ...string) Cmd { 56 | return DefaultCmder.Command(command, args...) 57 | } 58 | 59 | func CommandContext(ctx context.Context, command string, args ...string) Cmd { 60 | return DefaultCmder.CommandContext(ctx, command, args...) 61 | } 62 | 63 | func RawCommand(raw string) Cmd { 64 | cmdSplit, err := shell.Split(raw) 65 | // If failed to split, just return the raw string as the command. 66 | if len(cmdSplit) == 0 || err != nil { 67 | return DefaultCmder.Command(raw) 68 | } 69 | return DefaultCmder.Command(cmdSplit[0], cmdSplit[1:]...) 70 | } 71 | 72 | func RawCommandContext(ctx context.Context, raw string) Cmd { 73 | cmdSplit, err := shell.Split(raw) 74 | // If failed to split, just return the raw string as the command. 75 | if len(cmdSplit) == 0 || err != nil { 76 | return DefaultCmder.CommandContext(ctx, raw) 77 | } 78 | return DefaultCmder.CommandContext(ctx, cmdSplit[0], cmdSplit[1:]...) 79 | } 80 | 81 | // Output is for compatibility with cmd.Output. 82 | func Output(cmd Cmd) ([]byte, error) { 83 | var buff bytes.Buffer 84 | cmd.SetStdout(&buff) 85 | err := cmd.Run() 86 | return buff.Bytes(), err 87 | } 88 | 89 | // OutputLines is like os/exec's cmd.Output(), 90 | // but over our Cmd interface, and instead of returning the byte buffer of 91 | // stdout, it scans it for lines and returns a slice of output lines 92 | func OutputLines(cmd Cmd) (lines []string, err error) { 93 | var buff bytes.Buffer 94 | cmd.SetStdout(&buff) 95 | err = cmd.Run() 96 | scanner := bufio.NewScanner(&buff) 97 | for scanner.Scan() { 98 | lines = append(lines, scanner.Text()) 99 | } 100 | return lines, err 101 | } 102 | 103 | // CombinedOutputLines is like os/exec's cmd.CombinedOutput(), 104 | // but over our Cmd interface, and instead of returning the byte buffer of 105 | // stderr + stdout, it scans these for lines and returns a slice of output lines 106 | func CombinedOutputLines(cmd Cmd) (lines []string, err error) { 107 | var buff bytes.Buffer 108 | cmd.SetStdout(&buff) 109 | cmd.SetStderr(&buff) 110 | err = cmd.Run() 111 | scanner := bufio.NewScanner(&buff) 112 | for scanner.Scan() { 113 | lines = append(lines, scanner.Text()) 114 | } 115 | return lines, err 116 | } 117 | 118 | // SetOutput sets cmd's output to write to the given Writer. 119 | func SetOutput(cmd Cmd, stdoutWriter, stderrWriter io.Writer) { 120 | cmd.SetStdout(stdoutWriter) 121 | cmd.SetStderr(stderrWriter) 122 | } 123 | 124 | // InheritOutput sets cmd's output to write to the current process's stdout and stderr 125 | func InheritOutput(cmd Cmd) { 126 | cmd.SetStderr(os.Stderr) 127 | cmd.SetStdout(os.Stdout) 128 | } 129 | 130 | // NoOutput ignores all output from the command. 131 | func NoOutput(cmd Cmd) { 132 | cmd.SetStdout(io.Discard) 133 | cmd.SetStderr(io.Discard) 134 | } 135 | -------------------------------------------------------------------------------- /pkg/exec/local.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package exec 18 | 19 | import ( 20 | "context" 21 | "io" 22 | osexec "os/exec" 23 | "strings" 24 | 25 | "k8s.io/klog/v2" 26 | ) 27 | 28 | // LocalCmd wraps os/exec.Cmd, implementing the exec.Cmd interface 29 | type LocalCmd struct { 30 | *osexec.Cmd 31 | } 32 | 33 | var _ Cmd = &LocalCmd{} 34 | 35 | // LocalCmder is a factory for LocalCmd, implementing Cmder 36 | type LocalCmder struct{} 37 | 38 | var _ Cmder = &LocalCmder{} 39 | 40 | // Command returns a new exec.Cmd backed by Cmd 41 | func (c *LocalCmder) Command(name string, arg ...string) Cmd { 42 | klog.V(2).Infof("⚙️ %s %s", name, strings.Join(arg, " ")) 43 | return &LocalCmd{ 44 | Cmd: osexec.Command(name, arg...), 45 | } 46 | } 47 | 48 | // CommandContext returns a new exec.Cmd with the context, backed by Cmd 49 | func (c *LocalCmder) CommandContext(ctx context.Context, name string, arg ...string) Cmd { 50 | klog.V(2).Infof("⚙️ %s %s", name, strings.Join(arg, " ")) 51 | return &LocalCmd{ 52 | Cmd: osexec.CommandContext(ctx, name, arg...), 53 | } 54 | } 55 | 56 | // SetEnv sets env 57 | func (cmd *LocalCmd) SetEnv(env ...string) Cmd { 58 | cmd.Env = env 59 | return cmd 60 | } 61 | 62 | // SetStdin sets stdin 63 | func (cmd *LocalCmd) SetStdin(r io.Reader) Cmd { 64 | cmd.Stdin = r 65 | return cmd 66 | } 67 | 68 | // SetStdout set stdout 69 | func (cmd *LocalCmd) SetStdout(w io.Writer) Cmd { 70 | cmd.Stdout = w 71 | return cmd 72 | } 73 | 74 | // SetStderr sets stderr 75 | func (cmd *LocalCmd) SetStderr(w io.Writer) Cmd { 76 | cmd.Stderr = w 77 | return cmd 78 | } 79 | 80 | func (cmd *LocalCmd) SetDir(dir string) Cmd { 81 | cmd.Dir = dir 82 | return cmd 83 | } 84 | 85 | // Run runs 86 | func (cmd *LocalCmd) Run() error { 87 | return cmd.Cmd.Run() 88 | } 89 | -------------------------------------------------------------------------------- /pkg/fs/fs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package fs 18 | 19 | import ( 20 | "io" 21 | "os" 22 | "path/filepath" 23 | ) 24 | 25 | // CopyFile copies a file from src to dst 26 | func CopyFile(src, dst string) (err error) { 27 | // get source information 28 | info, err := os.Stat(src) 29 | if err != nil { 30 | return err 31 | } 32 | return copyFile(src, dst, info) 33 | } 34 | 35 | func copyFile(src, dst string, info os.FileInfo) error { 36 | // open src for reading 37 | in, err := os.Open(src) 38 | if err != nil { 39 | return err 40 | } 41 | defer func() { 42 | closeErr := in.Close() 43 | // if we weren't returning an error 44 | if err == nil { 45 | err = closeErr 46 | } 47 | }() 48 | if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { 49 | return err 50 | } 51 | // create dst file 52 | // this is like f, err := os.Create(dst); os.Chmod(f.Name(), src.Mode()) 53 | out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode()) 54 | if err != nil { 55 | return err 56 | } 57 | // make sure we close the file 58 | defer func() { 59 | closeErr := out.Close() 60 | // if we weren't returning an error 61 | if err == nil { 62 | err = closeErr 63 | } 64 | }() 65 | // actually copy 66 | if _, err = io.Copy(out, in); err != nil { 67 | return err 68 | } 69 | err = out.Sync() 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /pkg/metadata/junit.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metadata 18 | 19 | import ( 20 | "encoding/xml" 21 | "io" 22 | ) 23 | 24 | // JUnitError represents an error with extra metadata suitable for structured 25 | // (JUnit.xml) output written by kubetest2 26 | // If this type is returned as an error by a Deployer, kubetest2 may write the 27 | // metadata to the output junit_runner.xml 28 | type JUnitError interface { 29 | error 30 | // this should be the command output (stdout/stderr) 31 | SystemOut() string 32 | } 33 | 34 | type simpleJUnitError struct { 35 | error 36 | systemOut string 37 | } 38 | 39 | // ensure simpleJUnitError implements JUnitError 40 | var _ JUnitError = &simpleJUnitError{} 41 | 42 | func (s *simpleJUnitError) SystemOut() string { 43 | return s.systemOut 44 | } 45 | 46 | // NewJUnitError returns a simple instance of JUnitError wrapping systemOut 47 | func NewJUnitError(inner error, systemOut string) error { 48 | return &simpleJUnitError{ 49 | error: inner, 50 | systemOut: systemOut, 51 | } 52 | } 53 | 54 | // testSuite holds a slice of TestCase and other summary metadata. 55 | // 56 | // A build (column in testgrid) is composed of one or more TestSuites. 57 | type testSuite struct { 58 | XMLName xml.Name `xml:"testsuite"` 59 | Name string `xml:"name,attr"` 60 | Failures int `xml:"failures,attr"` 61 | Tests int `xml:"tests,attr"` 62 | Time float64 `xml:"time,attr"` 63 | Cases []testCase 64 | } 65 | 66 | func (t *testSuite) Write(writer io.Writer) error { 67 | // write xml header 68 | _, _ = io.WriteString(writer, ``) 69 | // write indented suite 70 | e := xml.NewEncoder(writer) 71 | e.Indent("", " ") 72 | return e.Encode(t) 73 | } 74 | 75 | func (t *testSuite) AddTestCase(tc testCase) { 76 | t.Tests++ 77 | if tc.Failure != "" { 78 | t.Failures++ 79 | } 80 | t.Cases = append(t.Cases, tc) 81 | } 82 | 83 | // testCase holds the result of a test/step/command. 84 | // 85 | // This will become a row in testgrid. 86 | type testCase struct { 87 | XMLName xml.Name `xml:"testcase"` 88 | Name string `xml:"name,attr"` 89 | ClassName string `xml:"classname,attr"` 90 | Time float64 `xml:"time,attr"` 91 | Failure string `xml:"failure,omitempty"` 92 | Skipped string `xml:"skipped,omitempty"` 93 | SystemOut string `xml:"system-out,omitempty"` 94 | } 95 | -------------------------------------------------------------------------------- /pkg/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metadata 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | ) 24 | 25 | type CustomJSON struct { 26 | data map[string]string 27 | } 28 | 29 | func NewCustomJSON(from io.Reader) (*CustomJSON, error) { 30 | meta := &CustomJSON{} 31 | if from != nil { 32 | dataBytes, err := io.ReadAll(from) 33 | if err != nil { 34 | return nil, err 35 | } 36 | if err := json.Unmarshal(dataBytes, &meta.data); err != nil { 37 | return nil, err 38 | } 39 | } 40 | return meta, nil 41 | } 42 | 43 | func (m *CustomJSON) Add(key, value string) error { 44 | if m.data == nil { 45 | m.data = map[string]string{} 46 | } 47 | if _, exists := m.data[key]; exists { 48 | return fmt.Errorf("key %s already exists in the metadata", key) 49 | } 50 | m.data[key] = value 51 | return nil 52 | } 53 | 54 | func (m *CustomJSON) Write(writer io.Writer) error { 55 | data, err := json.Marshal(m.data) 56 | if err == nil { 57 | _, err = writer.Write(data) 58 | } 59 | return err 60 | } 61 | -------------------------------------------------------------------------------- /pkg/metadata/metadata_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metadata 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "reflect" 23 | "strings" 24 | "testing" 25 | ) 26 | 27 | func TestCustomJSON_AddWrite(t *testing.T) { 28 | meta := CustomJSON{} 29 | if err := meta.Add("foo", "bar"); err != nil { 30 | t.Errorf("did not expect an error, but got: %v", err) 31 | } 32 | if err := meta.Add("baz", "qwe"); err != nil { 33 | t.Errorf("did not expect an error, but got: %v", err) 34 | } 35 | 36 | var buff bytes.Buffer 37 | if err := meta.Write(&buff); err != nil { 38 | t.Errorf("did not expect an error, but got: %v", err) 39 | } 40 | 41 | actualBytes := buff.Bytes() 42 | 43 | expectedBytes, err := json.Marshal(meta.data) 44 | if err != nil { 45 | t.Errorf("did not expect an error, but got: %v", err) 46 | } 47 | if !bytes.Equal(buff.Bytes(), expectedBytes) { 48 | t.Errorf("mismatched metadata bytes, got: %v, want: %v", actualBytes, expectedBytes) 49 | } 50 | } 51 | 52 | func TestNewCustomJSON(t *testing.T) { 53 | json := `{"baz":"qwe","foo":"bar"}` 54 | meta, err := NewCustomJSON(strings.NewReader(json)) 55 | if err != nil { 56 | t.Errorf("did not expect an error, but got: %v", err) 57 | } 58 | expectedData := map[string]string{ 59 | "foo": "bar", 60 | "baz": "qwe", 61 | } 62 | if !reflect.DeepEqual(meta.data, expectedData) { 63 | t.Errorf("mismatched metadata bytes, got: %v, want: %v", meta.data, expectedData) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/metadata/writer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metadata 18 | 19 | import ( 20 | "io" 21 | "time" 22 | ) 23 | 24 | // Writer manages writing out kubetest2 metadata, namely JUnit 25 | type Writer struct { 26 | suite testSuite 27 | start time.Time 28 | runnerOut io.Writer 29 | // for faking out time when testing 30 | timeNow func() time.Time 31 | } 32 | 33 | // NewWriter constructs a new writer, the junit_runner.xml contents 34 | // will be written to runnerOut, with the top level kubetest2 stages as 35 | // metadata (Up, Down, etc.) 36 | func NewWriter(suiteName string, runnerOut io.Writer) *Writer { 37 | suite := testSuite{Name: suiteName} 38 | return &Writer{ 39 | suite: suite, 40 | runnerOut: runnerOut, 41 | start: time.Now(), 42 | timeNow: time.Now, 43 | } 44 | } 45 | 46 | // WrapStep executes doStep and captures the output to be written to the 47 | // kubetest2 runner metadata. If doStep returns a JUnitError this metadata 48 | // will be captured 49 | func (w *Writer) WrapStep(name string, doStep func() error) error { 50 | start := w.timeNow() 51 | err := doStep() 52 | finish := w.timeNow() 53 | tc := testCase{ 54 | Name: name, 55 | ClassName: w.suite.Name, 56 | Time: finish.Sub(start).Seconds(), 57 | } 58 | if err != nil { 59 | tc.Failure = err.Error() 60 | } 61 | if v, ok := err.(JUnitError); ok { 62 | tc.SystemOut = v.SystemOut() 63 | } 64 | w.suite.AddTestCase(tc) 65 | return err 66 | } 67 | 68 | // Finish finalizes the metadata (time) and writes it out 69 | func (w *Writer) Finish() error { 70 | w.suite.Time = w.timeNow().Sub(w.start).Seconds() 71 | return w.suite.Write(w.runnerOut) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/process/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package process contains helpers for executing processes in ways that behave 18 | // similarly to syscall.Exec, but using child processes instead 19 | package process 20 | -------------------------------------------------------------------------------- /pkg/process/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package process 18 | 19 | import ( 20 | "os" 21 | "os/exec" 22 | "os/signal" 23 | ) 24 | 25 | // Exec generally mimics syscall.Exec behavior, but using a child process 26 | // isntead to make testing etc. easier 27 | func Exec(argv0 string, args []string, env []string) error { 28 | // construct command from inputs 29 | cmd := exec.Command(argv0, args...) 30 | cmd.Env = env 31 | 32 | // inherit some standard file descriptors, as if `syscall.Exec`ed 33 | cmd.Stdin = os.Stdin 34 | cmd.Stdout = os.Stdout 35 | cmd.Stderr = os.Stderr 36 | 37 | return execCmdWithSignals(cmd) 38 | } 39 | 40 | func execCmdWithSignals(cmd *exec.Cmd) error { 41 | // setup listener to forward all signals 42 | // TODO(bentheelder): what should this buffer size be? 43 | signals := make(chan os.Signal, 5) 44 | signal.Notify(signals) 45 | defer signal.Stop(signals) 46 | 47 | // start the process 48 | if err := cmd.Start(); err != nil { 49 | return err 50 | } 51 | 52 | // set up a channel to monitor for when it exits 53 | wait := make(chan error, 1) 54 | go func() { 55 | wait <- cmd.Wait() 56 | close(wait) 57 | }() 58 | 59 | // pass all signals to the subcommand until it exits, return the result 60 | for { 61 | select { 62 | case sig := <-signals: 63 | // TODO(bentheelder): can this actually fail? should we log this? 64 | _ = cmd.Process.Signal(sig) 65 | case err := <-wait: 66 | return err 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /pkg/process/junitexec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package process 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "io" 23 | "os" 24 | "os/exec" 25 | 26 | "sigs.k8s.io/kubetest2/pkg/metadata" 27 | ) 28 | 29 | type execJunitError struct { 30 | error 31 | systemout string 32 | } 33 | 34 | func (e *execJunitError) SystemOut() string { 35 | return e.systemout 36 | } 37 | 38 | var _ metadata.JUnitError = &execJunitError{} 39 | 40 | // ExecJUnit is like Exec, except that it tees the output and captures it 41 | // for returning a metadata.JUnitError if the process does not exit success 42 | func ExecJUnit(argv0 string, args []string, env []string) error { 43 | // construct command from inputs 44 | cmd := exec.Command(argv0, args...) 45 | return execJUnit(cmd, env) 46 | } 47 | 48 | func ExecJUnitContext(ctx context.Context, argv0 string, args []string, env []string) error { 49 | cmd := exec.CommandContext(ctx, argv0, args...) 50 | return execJUnit(cmd, env) 51 | } 52 | 53 | func execJUnit(cmd *exec.Cmd, env []string) error { 54 | cmd.Env = env 55 | 56 | // inherit some standard file descriptors, as if `syscall.Exec`ed 57 | cmd.Stdin = os.Stdin 58 | // ensure we also capture output 59 | var systemout bytes.Buffer 60 | syncSystemOut := &mutexWriter{ 61 | writer: &systemout, 62 | } 63 | cmd.Stdout = io.MultiWriter(syncSystemOut, os.Stdout) 64 | cmd.Stderr = io.MultiWriter(syncSystemOut, os.Stderr) 65 | 66 | // actually execute, return a JUnit error if the command errors 67 | if err := execCmdWithSignals(cmd); err != nil { 68 | return &execJunitError{ 69 | error: err, 70 | systemout: systemout.String(), 71 | } 72 | } 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /pkg/process/mutexwriter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package process 18 | 19 | import ( 20 | "io" 21 | "sync" 22 | ) 23 | 24 | // mutexWriter is a simple synchronized wrapper around an io.Writer 25 | type mutexWriter struct { 26 | writer io.Writer 27 | mu sync.Mutex 28 | } 29 | 30 | func (m *mutexWriter) Write(b []byte) (int, error) { 31 | m.mu.Lock() 32 | defer m.mu.Unlock() 33 | return m.writer.Write(b) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/testers/clusterloader2/suite/suite.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package suite 18 | 19 | type Suite struct { 20 | TestConfigs []string 21 | TestOverrides []string 22 | } 23 | 24 | // GetSuite returns the default configurations for well-known testing setups. 25 | func GetSuite(suite string) *Suite { 26 | const ( 27 | load = "load" 28 | density = "density" 29 | nodeThroughput = "node-throughput" 30 | ) 31 | 32 | var supportedSuites = map[string]*Suite{ 33 | load: { 34 | TestConfigs: []string{ 35 | "testing/load/config.yaml", 36 | }, 37 | }, 38 | 39 | density: { 40 | TestConfigs: []string{ 41 | "testing/density/config.yaml", 42 | }, 43 | }, 44 | 45 | nodeThroughput: { 46 | TestConfigs: []string{ 47 | "testing/node-throughput/config.yaml", 48 | }, 49 | }, 50 | } 51 | return supportedSuites[suite] 52 | } 53 | -------------------------------------------------------------------------------- /pkg/testers/exec/exec.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package exec 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "strings" 23 | 24 | "github.com/octago/sflags/gen/gpflag" 25 | "k8s.io/klog/v2" 26 | 27 | "sigs.k8s.io/kubetest2/pkg/process" 28 | "sigs.k8s.io/kubetest2/pkg/testers" 29 | ) 30 | 31 | var GitTag string 32 | 33 | type Tester struct { 34 | argv []string 35 | } 36 | 37 | const usage = `kubetest2 --test=exec -- [TestCommand] [TestArgs] 38 | TestCommand: the command to invoke for testing 39 | TestArgs: arguments passed to test command 40 | ` 41 | 42 | func (t *Tester) Execute() error { 43 | fs, err := gpflag.Parse(t) 44 | if err != nil { 45 | return fmt.Errorf("failed to initialize tester: %v", err) 46 | } 47 | 48 | fs.Usage = func() { 49 | fmt.Print(usage) 50 | } 51 | 52 | if len(os.Args) < 2 { 53 | fs.Usage() 54 | return nil 55 | } 56 | 57 | // gracefully handle -h or --help if it is the only argument 58 | help := fs.BoolP("help", "h", false, "") 59 | // we don't care about errors, only if -h / --help was set 60 | _ = fs.Parse(os.Args[1:2]) 61 | 62 | if *help { 63 | fs.Usage() 64 | return nil 65 | } 66 | 67 | t.argv = os.Args[1:] 68 | if err := testers.WriteVersionToMetadata(GitTag); err != nil { 69 | return err 70 | } 71 | return t.Test() 72 | } 73 | 74 | func expandEnv(args []string) []string { 75 | expandedArgs := make([]string, len(args)) 76 | for i, arg := range args { 77 | // best effort handle literal dollar for backward compatibility 78 | // this is not an all-purpose shell special character handler 79 | if strings.Contains(arg, `\$`) { 80 | expandedArgs[i] = strings.ReplaceAll(arg, `\$`, `$`) 81 | } else { 82 | expandedArgs[i] = os.ExpandEnv(arg) 83 | } 84 | } 85 | return expandedArgs 86 | } 87 | 88 | func (t *Tester) Test() error { 89 | expandedArgs := expandEnv(t.argv) 90 | return process.ExecJUnit(expandedArgs[0], expandedArgs[1:], os.Environ()) 91 | } 92 | 93 | func NewDefaultTester() *Tester { 94 | return &Tester{} 95 | } 96 | 97 | func Main() { 98 | t := NewDefaultTester() 99 | if err := t.Execute(); err != nil { 100 | klog.Fatalf("failed to run exec tester: %v", err) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /pkg/testers/exec/exec_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package exec 18 | 19 | import ( 20 | "os" 21 | "reflect" 22 | "testing" 23 | ) 24 | 25 | func TestExpandEnv(t *testing.T) { 26 | testCases := []struct { 27 | name string 28 | args []string 29 | envs map[string]string 30 | expectedExpandedArgs []string 31 | }{ 32 | { 33 | name: "empty args", 34 | args: []string{}, 35 | expectedExpandedArgs: []string{}, 36 | }, 37 | { 38 | name: "empty string arg", 39 | args: []string{"", ""}, 40 | expectedExpandedArgs: []string{"", ""}, 41 | }, 42 | { 43 | name: "single env to expand", 44 | args: []string{"$FOO"}, 45 | envs: map[string]string{"FOO": "foo"}, 46 | expectedExpandedArgs: []string{"foo"}, 47 | }, 48 | { 49 | name: "escaped dollar", 50 | args: []string{`\$FOO`}, 51 | envs: map[string]string{"FOO": "foo"}, 52 | expectedExpandedArgs: []string{`$FOO`}, 53 | }, 54 | { 55 | name: "escaped dollar and backslash", 56 | args: []string{`\\$FOO`}, 57 | envs: map[string]string{"FOO": "foo"}, 58 | expectedExpandedArgs: []string{`\$FOO`}, 59 | }, 60 | } 61 | 62 | for _, tc := range testCases { 63 | tc := tc 64 | t.Run(tc.name, func(t *testing.T) { 65 | t.Parallel() 66 | for key, val := range tc.envs { 67 | if err := os.Setenv(key, val); err != nil { 68 | t.Errorf("failed to set env: %v", err) 69 | } 70 | } 71 | actualExpandedArgs := expandEnv(tc.args) 72 | if !reflect.DeepEqual(tc.expectedExpandedArgs, actualExpandedArgs) { 73 | t.Errorf("mismatched expanded args: expected: %v, but got: %v", tc.expectedExpandedArgs, actualExpandedArgs) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pkg/testers/ginkgo/kubectl/kubectl.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package kubectl 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "sigs.k8s.io/kubetest2/pkg/exec" 24 | ) 25 | 26 | const ( 27 | kubectl = "kubectl" 28 | ) 29 | 30 | // APIServerURL obtains the URL of the k8s master from kubectl 31 | func APIServerURL() (string, error) { 32 | kubecontext, err := execAndResult(kubectl, "config", "view", "-o", "jsonpath=\"{.current-context}\"") 33 | if err != nil { 34 | return "", fmt.Errorf("Could not get kube context: %v", err) 35 | } 36 | 37 | clustername, err := execAndResult(kubectl, "config", "view", "-o", 38 | fmt.Sprintf("jsonpath=\"{.contexts[?(@.name == %s)].context.cluster}\"", kubecontext)) 39 | if err != nil { 40 | return "", fmt.Errorf("Could not get cluster name: %v", err) 41 | } 42 | 43 | apiServerURL, err := execAndResult(kubectl, "config", "view", "-o", 44 | fmt.Sprintf("jsonpath={.clusters[?(@.name == %s)].cluster.server}", clustername)) 45 | if err != nil { 46 | return "", err 47 | } 48 | return apiServerURL, nil 49 | } 50 | 51 | // execAndResult runs command with args and returns the entire output (or error) 52 | func execAndResult(command string, args ...string) (string, error) { 53 | cmd := exec.Command(command, args...) 54 | cmd.SetStderr(os.Stderr) 55 | bytes, err := exec.Output(cmd) 56 | return string(bytes), err 57 | } 58 | -------------------------------------------------------------------------------- /pkg/testers/metadata.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package testers 18 | 19 | import ( 20 | "os" 21 | "path/filepath" 22 | 23 | "sigs.k8s.io/kubetest2/pkg/artifacts" 24 | "sigs.k8s.io/kubetest2/pkg/metadata" 25 | ) 26 | 27 | func WriteVersionToMetadata(version string) error { 28 | var meta *metadata.CustomJSON 29 | // check existing metadata and initialize it if it exists 30 | metadataPath := filepath.Join(artifacts.BaseDir(), "metadata.json") 31 | if _, err := os.Stat(metadataPath); err == nil { 32 | metadataJSON, err := os.Open(metadataPath) 33 | if err != nil { 34 | return err 35 | } 36 | meta, err = metadata.NewCustomJSON(metadataJSON) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | if err := metadataJSON.Sync(); err != nil { 42 | return err 43 | } 44 | if err := metadataJSON.Close(); err != nil { 45 | return err 46 | } 47 | } else { 48 | meta, err = metadata.NewCustomJSON(nil) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | 54 | if err := meta.Add("tester-version", version); err != nil { 55 | return err 56 | } 57 | 58 | metadataJSON, err := os.Create(metadataPath) 59 | if err != nil { 60 | return err 61 | } 62 | if err := meta.Write(metadataJSON); err != nil { 63 | return err 64 | } 65 | 66 | if err := metadataJSON.Sync(); err != nil { 67 | return err 68 | } 69 | return metadataJSON.Close() 70 | } 71 | -------------------------------------------------------------------------------- /pkg/types/helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package types 18 | 19 | type incorrectUsageImpl struct { 20 | helpText string 21 | } 22 | 23 | var _ IncorrectUsage = &incorrectUsageImpl{} 24 | 25 | func (i incorrectUsageImpl) Error() string { 26 | // TODO(bentheelder): possibly this should wrap the string a bit 27 | return i.helpText 28 | } 29 | 30 | func (i *incorrectUsageImpl) HelpText() string { 31 | return i.helpText 32 | } 33 | 34 | // NewIncorrectUsage returns a simple IncorrectUsage implementation wrapping 35 | // helpText 36 | func NewIncorrectUsage(helpText string) error { 37 | return &incorrectUsageImpl{helpText} 38 | } 39 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "fmt" 21 | "net/url" 22 | "strings" 23 | 24 | "github.com/blang/semver/v4" 25 | "github.com/go-resty/resty/v2" 26 | ) 27 | 28 | func ParseKubernetesMarker(version string) (string, error) { 29 | if _, err := semver.ParseTolerant(version); err == nil { 30 | return version, nil 31 | } 32 | if u, err := url.Parse(version); err == nil { 33 | resp, err := resty.New().R().Get(version) 34 | if err != nil { 35 | return "", err 36 | } 37 | 38 | // Replace the last part of the version URL path with the contents of the URL's body 39 | // Example: 40 | // https://storage.googleapis.com/k8s-release-dev/ci/latest.txt -> v1.21.0-beta.1.112+576aa2d2470b28%0A 41 | // becomes https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.112+576aa2d2470b28%0A 42 | pathParts := strings.Split(u.Path, "/") 43 | pathParts[len(pathParts)-1] = resp.String() 44 | u.Path = strings.Join(pathParts, "/") 45 | return strings.TrimSpace(u.String()), nil 46 | } 47 | return "", fmt.Errorf("unexpected kubernetes version: %v", version) 48 | } 49 | 50 | // PseudoUniqueSubstring returns a substring of a UUID 51 | // that can be reasonably used in resource names 52 | // where length is constrained 53 | // e.g https://cloud.google.com/compute/docs/naming-resources 54 | // but still retain as much uniqueness as possible 55 | // also easily lets us tie it back to a run 56 | func PseudoUniqueSubstring(uuid string) string { 57 | // both KUBETEST2_RUN_ID and PROW_JOB_ID uuids are generated 58 | // following RFC 4122 https://tools.ietf.org/html/rfc4122 59 | // e.g. 09a2565a-7ac6-11eb-a603-2218f636630c 60 | // extract the first 13 characters (09a2565a-7ac6) as they are the ones that depend on 61 | // timestamp and has the best avalanche effect (https://en.wikipedia.org/wiki/Avalanche_effect) 62 | // as compared to the other bytes 63 | // 13 characters is also <= the no. of character being used previously 64 | const maxResourceNamePrefixLength = 13 65 | if len(uuid) <= maxResourceNamePrefixLength { 66 | return uuid 67 | } 68 | return uuid[:maxResourceNamePrefixLength] 69 | } 70 | -------------------------------------------------------------------------------- /pkg/util/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import "testing" 20 | 21 | func TestPseudoUniqueSubstring(t *testing.T) { 22 | testCases := []struct { 23 | name string 24 | uuid string 25 | expectedSubstring string 26 | }{ 27 | { 28 | name: "actual uuid", 29 | uuid: "09a2565a-7ac6-11eb-a603-2218f636630c", 30 | expectedSubstring: "09a2565a-7ac6", 31 | }, 32 | { 33 | name: "<= 13 length uuid", 34 | uuid: "09a2565a-7ac6", 35 | expectedSubstring: "09a2565a-7ac6", 36 | }, 37 | { 38 | name: "empty string", 39 | uuid: "", 40 | expectedSubstring: "", 41 | }, 42 | } 43 | 44 | for _, tc := range testCases { 45 | tc := tc 46 | t.Run(tc.name, func(t *testing.T) { 47 | t.Parallel() 48 | actualSubstring := PseudoUniqueSubstring(tc.uuid) 49 | if actualSubstring != tc.expectedSubstring { 50 | t.Errorf("invalid substring: expected %s, but got %s", tc.expectedSubstring, actualSubstring) 51 | } 52 | }) 53 | } 54 | } 55 | --------------------------------------------------------------------------------