├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── SECURITY_CONTACTS ├── bin ├── consider-early-travis-exit.sh ├── install-test-dependencies.sh ├── pre-commit.sh └── test-on-prow.sh ├── code-of-conduct.md ├── go.mod ├── go.sum └── integration ├── .gitignore ├── README.md ├── addr ├── addr_suite_test.go ├── manager.go └── manager_test.go ├── apiserver.go ├── assets └── bin │ └── .gitkeep ├── control_plane.go ├── doc.go ├── etcd.go ├── integration_suite_test.go ├── internal ├── apiserver.go ├── apiserver_test.go ├── arguments.go ├── arguments_test.go ├── bin_path_finder.go ├── bin_path_finder_test.go ├── etcd.go ├── etcd_test.go ├── integration_tests │ ├── apiserver_integration_test.go │ ├── doc.go │ ├── etcd_integration_test.go │ ├── integration_suite_test.go │ └── integration_test.go ├── internal_suite_test.go ├── process.go └── process_test.go ├── kubectl.go ├── kubectl_test.go └── scripts └── download-binaries.sh /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.13.x 5 | 6 | go_import_path: sigs.k8s.io/testing_frameworks 7 | 8 | before_install: 9 | - source ./bin/consider-early-travis-exit.sh 10 | - ./bin/install-test-dependencies.sh 11 | 12 | # Install must be set to prevent default `go get` to run. 13 | # The dependencies have already been vendored by `dep` so 14 | # we don't need to fetch them. 15 | install: 16 | - 17 | 18 | script: 19 | - ./bin/pre-commit.sh 20 | 21 | # TBD. Suppressing for now. 22 | notifications: 23 | email: false 24 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## Sign the CLA 4 | 5 | Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests. Please see https://git.k8s.io/community/CLA.md for more info 6 | 7 | ### Contributing A Patch 8 | 9 | 1. Submit an issue describing your proposed change to the repo in question. 10 | 1. The [repo owners](OWNERS) will respond to your issue promptly. 11 | 1. If your proposed change is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). 12 | 1. Fork the desired repo, develop and test your code changes. 13 | 1. Submit a pull request. 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 The Kubernetes Authors 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs: https://git.k8s.io/community/contributors/devel/owners.md 2 | 3 | approvers: 4 | - sig-testing-leads 5 | - frameworks-admins 6 | -------------------------------------------------------------------------------- /OWNERS_ALIASES: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs: https://git.k8s.io/community/contributors/devel/owners.md 2 | 3 | aliases: 4 | sig-testing-leads: 5 | - fejta 6 | - spiffxp 7 | - stevekuznetsov 8 | - timothysc 9 | frameworks-admins: 10 | - apelisse 11 | - hoegaarden 12 | - totherme 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Frameworks 2 | 3 | Test frameworks for testing kubernetes 4 | 5 | This was created as a result of [kubernetes/community#1524](https://github.com/kubernetes/community/pull/1524) 6 | 7 | ## What lives here? 8 | 9 | - The [integration test framework](integration/) 10 | 11 | ## What is allowed to live here? 12 | 13 | Any test framework for testing any part of kubernetes is welcome so long as we 14 | can avoid vendor loops. 15 | 16 | Right now, the only things vendored into this repo are 17 | [ginkgo](https://github.com/onsi/ginkgo) and 18 | [gomega](https://github.com/onsi/gomega). We would like to keep vendored 19 | libraries to a minimum in order to make it as easy as possible to import these 20 | frameworks into other kubernetes repos. Code in this repo should certainly 21 | never import `k8s.io/kubernetes`. 22 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Team to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | hoegaarden 14 | spiffxp 15 | timothysc 16 | -------------------------------------------------------------------------------- /bin/consider-early-travis-exit.sh: -------------------------------------------------------------------------------- 1 | # Exits with status 0 if it can be determined that the 2 | # current PR should not trigger all travis checks. 3 | # 4 | # This could be done with a "git ...|grep -vqE" oneliner 5 | # but as travis triggering is refined it's useful to check 6 | # travis logs to see how branch files were considered. 7 | function consider-early-travis-exit { 8 | if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 9 | echo "Unknown pull request." 10 | return 11 | fi 12 | # Might use this to improve checks on multi-commit PRs. 13 | echo "TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE" 14 | echo "Branch Files ('T'==trigger tests, ' '=ignore):" 15 | echo "---" 16 | local triggers=0 17 | local invisibles=0 18 | for fn in $(git diff --name-only HEAD origin/master); do 19 | if [[ "$fn" =~ (\.md$)|(^docs/) ]]; then 20 | echo " $fn" 21 | let invisibles+=1 22 | else 23 | echo " T $fn" 24 | let triggers+=1 25 | fi 26 | done 27 | echo "---" 28 | printf >&2 "%6d files invisible to travis.\n" $invisibles 29 | printf >&2 "%6d files trigger travis.\n" $triggers 30 | if [ $triggers -eq 0 ]; then 31 | echo "No files triggered travis test, exiting early." 32 | # see https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh 33 | travis_terminate 0 34 | fi 35 | } 36 | consider-early-travis-exit 37 | unset -f consider-early-travis-exit 38 | -------------------------------------------------------------------------------- /bin/install-test-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -u 5 | 6 | go get -u golang.org/x/tools/cmd/goimports 7 | -------------------------------------------------------------------------------- /bin/pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Make sure, we run in the root of the repo and 4 | # therefore run the tests on all packages 5 | base_dir="$( cd "$(dirname "$0")/.." && pwd )" 6 | cd "$base_dir" || { 7 | echo "Cannot cd to '$base_dir'. Aborting." >&2 8 | exit 1 9 | } 10 | 11 | rc=0 12 | 13 | go_dirs() { 14 | go list -f '{{.Dir}}' ./... | tr '\n' '\0' 15 | } 16 | 17 | echo "Running go fmt" 18 | diff <(echo -n) <(go_dirs | xargs -0 gofmt -s -d -l) 19 | rc=$((rc || $?)) 20 | 21 | echo "Running goimports" 22 | diff -u <(echo -n) <(go_dirs | xargs -0 goimports -l) 23 | rc=$((rc || $?)) 24 | 25 | echo "Running go vet" 26 | go vet -all ./... 27 | rc=$((rc || $?)) 28 | 29 | echo "Installing test binaries" 30 | ./integration/scripts/download-binaries.sh 31 | rc=$((rc || $?)) 32 | 33 | echo "Running go test" 34 | go test -v ./... 35 | rc=$((rc || $?)) 36 | 37 | exit $rc 38 | -------------------------------------------------------------------------------- /bin/test-on-prow.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -u 5 | 6 | export GO111MODULE=on 7 | ./bin/install-test-dependencies.sh 8 | ./bin/pre-commit.sh 9 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module sigs.k8s.io/testing_frameworks 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/golang/protobuf v1.0.0 // indirect 7 | github.com/kr/pretty v0.1.0 // indirect 8 | github.com/onsi/ginkgo v1.4.0 9 | github.com/onsi/gomega v1.3.0 10 | golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b // indirect 11 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect 12 | golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b // indirect 13 | golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984 // indirect 14 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 15 | gopkg.in/yaml.v2 v2.0.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/golang/protobuf v1.0.0 h1:lsek0oXi8iFE9L+EXARyHIjU5rlWIhhTkjDz3vHhWWQ= 2 | github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 4 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 5 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 6 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 7 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 8 | github.com/onsi/ginkgo v1.4.0 h1:n60/4GZK0Sr9O2iuGKq876Aoa0ER2ydgpMOBwzJ8e2c= 9 | github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 10 | github.com/onsi/gomega v1.3.0 h1:yPHEatyQC4jN3vdfvqJXG7O9vfC6LhaAV1NEdYpP+h0= 11 | github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 12 | golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b h1:Xu6Gf1IrU0c8CSJqWR43Bh8vb+Ft3jVIUahRiqL1oaI= 13 | golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 14 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 15 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 16 | golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b h1:mxo/dXmtEd5rXc/ZzMKg0qDhMT+51+LvV65S9dP6nh4= 17 | golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 18 | golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984 h1:4S3Dic2vY09agWhKAjYa6buMB7HsLkVrliEHZclmmSU= 19 | golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 20 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 21 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 22 | gopkg.in/yaml.v2 v2.0.0 h1:uUkhRGrsEyx/laRdeS6YIQKIys8pg+lRSRdVMTYjivs= 23 | gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 24 | -------------------------------------------------------------------------------- /integration/.gitignore: -------------------------------------------------------------------------------- 1 | assets/bin 2 | -------------------------------------------------------------------------------- /integration/README.md: -------------------------------------------------------------------------------- 1 | # Integration Testing Framework 2 | 3 | A framework for integration testing components of kubernetes. This framework is 4 | intended to work properly both in CI, and on a local dev machine. It therefore 5 | explicitly supports both Linux and Darwin. 6 | 7 | For detailed documentation see the 8 | [![GoDoc](https://godoc.org/github.com/kubernetes-sigs/testing_frameworks/integration?status.svg)](https://godoc.org/github.com/kubernetes-sigs/testing_frameworks/integration). 9 | -------------------------------------------------------------------------------- /integration/addr/addr_suite_test.go: -------------------------------------------------------------------------------- 1 | package addr_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestAddr(t *testing.T) { 11 | t.Parallel() 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Addr Suite") 14 | } 15 | -------------------------------------------------------------------------------- /integration/addr/manager.go: -------------------------------------------------------------------------------- 1 | package addr 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | const ( 11 | portReserveTime = 1 * time.Minute 12 | portConflictRetry = 100 13 | ) 14 | 15 | type portCache struct { 16 | lock sync.Mutex 17 | ports map[int]time.Time 18 | } 19 | 20 | func (c *portCache) add(port int) bool { 21 | c.lock.Lock() 22 | defer c.lock.Unlock() 23 | // remove outdated port 24 | for p, t := range c.ports { 25 | if time.Since(t) > portReserveTime { 26 | delete(c.ports, p) 27 | } 28 | } 29 | // try allocating new port 30 | if _, ok := c.ports[port]; ok { 31 | return false 32 | } 33 | c.ports[port] = time.Now() 34 | return true 35 | } 36 | 37 | var cache = &portCache{ 38 | ports: make(map[int]time.Time), 39 | } 40 | 41 | func suggest() (port int, resolvedHost string, err error) { 42 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 43 | if err != nil { 44 | return 45 | } 46 | l, err := net.ListenTCP("tcp", addr) 47 | if err != nil { 48 | return 49 | } 50 | port = l.Addr().(*net.TCPAddr).Port 51 | defer func() { 52 | err = l.Close() 53 | }() 54 | resolvedHost = addr.IP.String() 55 | return 56 | } 57 | 58 | // Suggest suggests an address a process can listen on. It returns 59 | // a tuple consisting of a free port and the hostname resolved to its IP. 60 | // It makes sure that new port allocated does not conflict with old ports 61 | // allocated within 1 minute. 62 | func Suggest() (port int, resolvedHost string, err error) { 63 | for i := 0; i < portConflictRetry; i++ { 64 | port, resolvedHost, err = suggest() 65 | if err != nil { 66 | return 67 | } 68 | if cache.add(port) { 69 | return 70 | } 71 | } 72 | err = fmt.Errorf("no free ports found after %d retries", portConflictRetry) 73 | return 74 | } 75 | -------------------------------------------------------------------------------- /integration/addr/manager_test.go: -------------------------------------------------------------------------------- 1 | package addr_test 2 | 3 | import ( 4 | "sigs.k8s.io/testing_frameworks/integration/addr" 5 | 6 | "net" 7 | "strconv" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("SuggestAddress", func() { 14 | It("returns a free port and an address to bind to", func() { 15 | port, host, err := addr.Suggest() 16 | 17 | Expect(err).NotTo(HaveOccurred()) 18 | Expect(host).To(Or(Equal("127.0.0.1"), Equal("::1"))) 19 | Expect(port).NotTo(Equal(0)) 20 | 21 | addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, strconv.Itoa(port))) 22 | Expect(err).NotTo(HaveOccurred()) 23 | l, err := net.ListenTCP("tcp", addr) 24 | defer func() { 25 | Expect(l.Close()).To(Succeed()) 26 | }() 27 | Expect(err).NotTo(HaveOccurred()) 28 | }) 29 | }) 30 | -------------------------------------------------------------------------------- /integration/apiserver.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/url" 7 | "time" 8 | 9 | "sigs.k8s.io/testing_frameworks/integration/addr" 10 | "sigs.k8s.io/testing_frameworks/integration/internal" 11 | ) 12 | 13 | // APIServer knows how to run a kubernetes apiserver. 14 | type APIServer struct { 15 | // URL is the address the ApiServer should listen on for client connections. 16 | // 17 | // If this is not specified, we default to a random free port on localhost. 18 | URL *url.URL 19 | 20 | // SecurePort is the additional secure port that the APIServer should listen on. 21 | SecurePort int 22 | 23 | // Path is the path to the apiserver binary. 24 | // 25 | // If this is left as the empty string, we will attempt to locate a binary, 26 | // by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and 27 | // the default test assets directory. See the "Binaries" section above (in 28 | // doc.go) for details. 29 | Path string 30 | 31 | // Args is a list of arguments which will passed to the APIServer binary. 32 | // Before they are passed on, they will be evaluated as go-template strings. 33 | // This means you can use fields which are defined and exported on this 34 | // APIServer struct (e.g. "--cert-dir={{ .Dir }}"). 35 | // Those templates will be evaluated after the defaulting of the APIServer's 36 | // fields has already happened and just before the binary actually gets 37 | // started. Thus you have access to caluclated fields like `URL` and others. 38 | // 39 | // If not specified, the minimal set of arguments to run the APIServer will 40 | // be used. 41 | Args []string 42 | 43 | // CertDir is a path to a directory containing whatever certificates the 44 | // APIServer will need. 45 | // 46 | // If left unspecified, then the Start() method will create a fresh temporary 47 | // directory, and the Stop() method will clean it up. 48 | CertDir string 49 | 50 | // EtcdURL is the URL of the Etcd the APIServer should use. 51 | // 52 | // If this is not specified, the Start() method will return an error. 53 | EtcdURL *url.URL 54 | 55 | // StartTimeout, StopTimeout specify the time the APIServer is allowed to 56 | // take when starting and stoppping before an error is emitted. 57 | // 58 | // If not specified, these default to 20 seconds. 59 | StartTimeout time.Duration 60 | StopTimeout time.Duration 61 | 62 | // Out, Err specify where APIServer should write its StdOut, StdErr to. 63 | // 64 | // If not specified, the output will be discarded. 65 | Out io.Writer 66 | Err io.Writer 67 | 68 | processState *internal.ProcessState 69 | } 70 | 71 | // Start starts the apiserver, waits for it to come up, and returns an error, 72 | // if occurred. 73 | func (s *APIServer) Start() error { 74 | if s.processState == nil { 75 | if err := s.setProcessState(); err != nil { 76 | return err 77 | } 78 | } 79 | return s.processState.Start(s.Out, s.Err) 80 | } 81 | 82 | func (s *APIServer) setProcessState() error { 83 | if s.EtcdURL == nil { 84 | return fmt.Errorf("expected EtcdURL to be configured") 85 | } 86 | 87 | var err error 88 | 89 | s.processState = &internal.ProcessState{} 90 | 91 | s.processState.DefaultedProcessInput, err = internal.DoDefaulting( 92 | "kube-apiserver", 93 | s.URL, 94 | s.CertDir, 95 | s.Path, 96 | s.StartTimeout, 97 | s.StopTimeout, 98 | ) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | // Defaulting the secure port 104 | if s.SecurePort == 0 { 105 | s.SecurePort, _, err = addr.Suggest() 106 | if err != nil { 107 | return err 108 | } 109 | } 110 | 111 | s.processState.HealthCheckEndpoint = "/healthz" 112 | 113 | s.URL = &s.processState.URL 114 | s.CertDir = s.processState.Dir 115 | s.Path = s.processState.Path 116 | s.StartTimeout = s.processState.StartTimeout 117 | s.StopTimeout = s.processState.StopTimeout 118 | 119 | s.processState.Args, err = internal.RenderTemplates( 120 | internal.DoAPIServerArgDefaulting(s.Args), s, 121 | ) 122 | return err 123 | } 124 | 125 | // Stop stops this process gracefully, waits for its termination, and cleans up 126 | // the CertDir if necessary. 127 | func (s *APIServer) Stop() error { 128 | return s.processState.Stop() 129 | } 130 | 131 | // APIServerDefaultArgs exposes the default args for the APIServer so that you 132 | // can use those to append your own additional arguments. 133 | // 134 | // The internal default arguments are explicitely copied here, we don't want to 135 | // allow users to change the internal ones. 136 | var APIServerDefaultArgs = append([]string{}, internal.APIServerDefaultArgs...) 137 | -------------------------------------------------------------------------------- /integration/assets/bin/.gitkeep: -------------------------------------------------------------------------------- 1 | This directory will be the home of some binaries which are downloaded with `pkg/framework/test/scripts/download-binaries`. 2 | -------------------------------------------------------------------------------- /integration/control_plane.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | ) 7 | 8 | // ControlPlane is a struct that knows how to start your test control plane. 9 | // 10 | // Right now, that means Etcd and your APIServer. This is likely to increase in 11 | // future. 12 | type ControlPlane struct { 13 | APIServer *APIServer 14 | Etcd *Etcd 15 | } 16 | 17 | // Start will start your control plane processes. To stop them, call Stop(). 18 | func (f *ControlPlane) Start() error { 19 | if f.Etcd == nil { 20 | f.Etcd = &Etcd{} 21 | } 22 | if err := f.Etcd.Start(); err != nil { 23 | return err 24 | } 25 | 26 | if f.APIServer == nil { 27 | f.APIServer = &APIServer{} 28 | } 29 | f.APIServer.EtcdURL = f.Etcd.URL 30 | return f.APIServer.Start() 31 | } 32 | 33 | // Stop will stop your control plane processes, and clean up their data. 34 | func (f *ControlPlane) Stop() error { 35 | if f.APIServer != nil { 36 | if err := f.APIServer.Stop(); err != nil { 37 | return err 38 | } 39 | } 40 | if f.Etcd != nil { 41 | if err := f.Etcd.Stop(); err != nil { 42 | return err 43 | } 44 | } 45 | return nil 46 | } 47 | 48 | // APIURL returns the URL you should connect to to talk to your API. 49 | func (f *ControlPlane) APIURL() *url.URL { 50 | return f.APIServer.URL 51 | } 52 | 53 | // KubeCtl returns a pre-configured KubeCtl, ready to connect to this 54 | // ControlPlane. 55 | func (f *ControlPlane) KubeCtl() *KubeCtl { 56 | k := &KubeCtl{} 57 | k.Opts = append(k.Opts, fmt.Sprintf("--server=%s", f.APIURL())) 58 | return k 59 | } 60 | -------------------------------------------------------------------------------- /integration/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package integration implements an integration testing framework for kubernetes. 4 | 5 | It provides components for standing up a kubernetes API, against which you can test a 6 | kubernetes client, or other kubernetes components. The lifecycle of the components 7 | needed to provide this API is managed by this framework. 8 | 9 | Quickstart 10 | 11 | If you want to test a kubernetes client against the latest kubernetes APIServer 12 | and Etcd, you can use `./scripts/download-binaries.sh` to download APIServer 13 | and Etcd binaries for your platform. Then add something like the following to 14 | your tests: 15 | 16 | cp := &integration.ControlPlane{} 17 | cp.Start() 18 | kubeCtl := cp.KubeCtl() 19 | stdout, stderr, err := kubeCtl.Run("get", "pods") 20 | // You can check on err, stdout & stderr and build up 21 | // your tests 22 | cp.Stop() 23 | 24 | Components 25 | 26 | Currently the framework provides the following components: 27 | 28 | ControlPlane: The ControlPlane wraps Etcd & APIServer (see below) and wires 29 | them together correctly. A ControlPlane can be stopped & started and can 30 | provide the URL to connect to the API. The ControlPlane can also be asked for a 31 | KubeCtl which is already correctly configured for this ControlPlane. The 32 | ControlPlane is a good entry point for default setups. 33 | 34 | Etcd: Manages an Etcd binary, which can be started, stopped and connected to. 35 | By default Etcd will listen on a random port for http connections and will 36 | create a temporary directory for its data. To configure it differently, see the 37 | Etcd type documentation below. 38 | 39 | APIServer: Manages an Kube-APIServer binary, which can be started, stopped and 40 | connected to. By default APIServer will listen on a random port for http 41 | connections and will create a temporary directory to store the (auto-generated) 42 | certificates. To configure it differently, see the APIServer type 43 | documentation below. 44 | 45 | KubeCtl: Wraps around a `kubectl` binary and can `Run(...)` arbitrary commands 46 | against a kubernetes control plane. 47 | 48 | Binaries 49 | 50 | Etcd, APIServer & KubeCtl use the same mechanism to determine which binaries to 51 | use when they get started. 52 | 53 | 1. If the component is configured with a `Path` the framework tries to run that 54 | binary. 55 | For example: 56 | 57 | myEtcd := &Etcd{ 58 | Path: "/some/other/etcd", 59 | } 60 | cp := &integration.ControlPlane{ 61 | Etcd: myEtcd, 62 | } 63 | cp.Start() 64 | 65 | 2. If the Path field on APIServer, Etcd or KubeCtl is left unset and an 66 | environment variable named `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD` or 67 | `TEST_ASSET_KUBECTL` is set, its value is used as a path to the binary for the 68 | APIServer, Etcd or KubeCtl. 69 | 70 | 3. If neither the `Path` field, nor the environment variable is set, the 71 | framework tries to use the binaries `kube-apiserver`, `etcd` or `kubectl` in 72 | the directory `${FRAMEWORK_DIR}/assets/bin/`. 73 | 74 | For convenience this framework ships with 75 | `${FRAMEWORK_DIR}/scripts/download-binaries.sh` which can be used to download 76 | pre-compiled versions of the needed binaries and place them in the default 77 | location (`${FRAMEWORK_DIR}/assets/bin/`). 78 | 79 | Arguments for Etcd and APIServer 80 | 81 | Those components will start without any configuration. However, if you want or 82 | need to, you can override certain configuration -- one of which are the 83 | arguments used when calling the binary. 84 | 85 | When you choose to specify your own set of arguments, those won't be appended 86 | to the default set of arguments, it is your responsibility to provide all the 87 | arguments needed for the binary to start successfully. 88 | 89 | However, the default arguments for APIServer and Etcd are exported as 90 | `APIServerDefaultArgs` and `EtcdDefaultArgs` from this package. Treat those 91 | variables as read-only constants. Internally we have a set of default 92 | arguments for defaulting, the `APIServerDefaultArgs` and `EtcdDefaultArgs` are 93 | just copies of those. So when you override them you loose access to the actual 94 | internal default arguments, but your override won't affect the defaulting. 95 | 96 | All arguments are interpreted as go templates. Those templates have access to 97 | all exported fields of the `APIServer`/`Etcd` struct. It does not matter if 98 | those fields where explicitly set up or if they were defaulted by calling the 99 | `Start()` method, the template evaluation runs just before the binary is 100 | executed and right after the defaulting of all the struct's fields has 101 | happened. 102 | 103 | // When you want to append additional arguments ... 104 | etcd := &Etcd{ 105 | // Additional custom arguments will appended to the set of default 106 | // arguments 107 | Args: append(EtcdDefaultArgs, "--additional=arg"), 108 | DataDir: "/my/special/data/dir", 109 | } 110 | 111 | // When you want to use a custom set of arguments ... 112 | etcd := &Etcd{ 113 | // Only custom arguments will be passed to the binary 114 | Args: []string{"--one=1", "--two=2", "--three=3"}, 115 | DataDir: "/my/special/data/dir", 116 | } 117 | 118 | */ 119 | package integration 120 | -------------------------------------------------------------------------------- /integration/etcd.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | "net/url" 8 | 9 | "sigs.k8s.io/testing_frameworks/integration/internal" 10 | ) 11 | 12 | // Etcd knows how to run an etcd server. 13 | type Etcd struct { 14 | // URL is the address the Etcd should listen on for client connections. 15 | // 16 | // If this is not specified, we default to a random free port on localhost. 17 | URL *url.URL 18 | 19 | // Path is the path to the etcd binary. 20 | // 21 | // If this is left as the empty string, we will attempt to locate a binary, 22 | // by checking for the TEST_ASSET_ETCD environment variable, and the default 23 | // test assets directory. See the "Binaries" section above (in doc.go) for 24 | // details. 25 | Path string 26 | 27 | // Args is a list of arguments which will passed to the Etcd binary. Before 28 | // they are passed on, the`y will be evaluated as go-template strings. This 29 | // means you can use fields which are defined and exported on this Etcd 30 | // struct (e.g. "--data-dir={{ .Dir }}"). 31 | // Those templates will be evaluated after the defaulting of the Etcd's 32 | // fields has already happened and just before the binary actually gets 33 | // started. Thus you have access to caluclated fields like `URL` and others. 34 | // 35 | // If not specified, the minimal set of arguments to run the Etcd will be 36 | // used. 37 | Args []string 38 | 39 | // DataDir is a path to a directory in which etcd can store its state. 40 | // 41 | // If left unspecified, then the Start() method will create a fresh temporary 42 | // directory, and the Stop() method will clean it up. 43 | DataDir string 44 | 45 | // StartTimeout, StopTimeout specify the time the Etcd is allowed to 46 | // take when starting and stopping before an error is emitted. 47 | // 48 | // If not specified, these default to 20 seconds. 49 | StartTimeout time.Duration 50 | StopTimeout time.Duration 51 | 52 | // Out, Err specify where Etcd should write its StdOut, StdErr to. 53 | // 54 | // If not specified, the output will be discarded. 55 | Out io.Writer 56 | Err io.Writer 57 | 58 | processState *internal.ProcessState 59 | } 60 | 61 | // Start starts the etcd, waits for it to come up, and returns an error, if one 62 | // occoured. 63 | func (e *Etcd) Start() error { 64 | if e.processState == nil { 65 | if err := e.setProcessState(); err != nil { 66 | return err 67 | } 68 | } 69 | return e.processState.Start(e.Out, e.Err) 70 | } 71 | 72 | func (e *Etcd) setProcessState() error { 73 | var err error 74 | 75 | e.processState = &internal.ProcessState{} 76 | 77 | e.processState.DefaultedProcessInput, err = internal.DoDefaulting( 78 | "etcd", 79 | e.URL, 80 | e.DataDir, 81 | e.Path, 82 | e.StartTimeout, 83 | e.StopTimeout, 84 | ) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | e.processState.StartMessage = internal.GetEtcdStartMessage(e.processState.URL) 90 | 91 | e.URL = &e.processState.URL 92 | e.DataDir = e.processState.Dir 93 | e.Path = e.processState.Path 94 | e.StartTimeout = e.processState.StartTimeout 95 | e.StopTimeout = e.processState.StopTimeout 96 | 97 | e.processState.Args, err = internal.RenderTemplates( 98 | internal.DoEtcdArgDefaulting(e.Args), e, 99 | ) 100 | return err 101 | } 102 | 103 | // Stop stops this process gracefully, waits for its termination, and cleans up 104 | // the DataDir if necessary. 105 | func (e *Etcd) Stop() error { 106 | return e.processState.Stop() 107 | } 108 | 109 | // EtcdDefaultArgs exposes the default args for Etcd so that you 110 | // can use those to append your own additional arguments. 111 | // 112 | // The internal default arguments are explicitely copied here, we don't want to 113 | // allow users to change the internal ones. 114 | var EtcdDefaultArgs = append([]string{}, internal.EtcdDefaultArgs...) 115 | -------------------------------------------------------------------------------- /integration/integration_suite_test.go: -------------------------------------------------------------------------------- 1 | package integration_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestIntegration(t *testing.T) { 11 | t.Parallel() 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Integration Framework Unit Tests") 14 | } 15 | -------------------------------------------------------------------------------- /integration/internal/apiserver.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | var APIServerDefaultArgs = []string{ 4 | "--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}", 5 | "--cert-dir={{ .CertDir }}", 6 | "--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}", 7 | "--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}", 8 | "--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}", 9 | } 10 | 11 | func DoAPIServerArgDefaulting(args []string) []string { 12 | if len(args) != 0 { 13 | return args 14 | } 15 | 16 | return APIServerDefaultArgs 17 | } 18 | -------------------------------------------------------------------------------- /integration/internal/apiserver_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | . "sigs.k8s.io/testing_frameworks/integration/internal" 7 | ) 8 | 9 | var _ = Describe("Apiserver", func() { 10 | It("defaults Args if they are empty", func() { 11 | initialArgs := []string{} 12 | defaultedArgs := DoAPIServerArgDefaulting(initialArgs) 13 | Expect(defaultedArgs).To(BeEquivalentTo(APIServerDefaultArgs)) 14 | }) 15 | 16 | It("keeps Args as is if they are not empty", func() { 17 | initialArgs := []string{"--one", "--two=2"} 18 | defaultedArgs := DoAPIServerArgDefaulting(initialArgs) 19 | Expect(defaultedArgs).To(BeEquivalentTo([]string{ 20 | "--one", "--two=2", 21 | })) 22 | }) 23 | }) 24 | -------------------------------------------------------------------------------- /integration/internal/arguments.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "bytes" 5 | "html/template" 6 | ) 7 | 8 | func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) { 9 | var t *template.Template 10 | 11 | for _, arg := range argTemplates { 12 | t, err = template.New(arg).Parse(arg) 13 | if err != nil { 14 | args = nil 15 | return 16 | } 17 | 18 | buf := &bytes.Buffer{} 19 | err = t.Execute(buf, data) 20 | if err != nil { 21 | args = nil 22 | return 23 | } 24 | args = append(args, buf.String()) 25 | } 26 | 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /integration/internal/arguments_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | "net/url" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | "sigs.k8s.io/testing_frameworks/integration" 10 | . "sigs.k8s.io/testing_frameworks/integration/internal" 11 | ) 12 | 13 | var _ = Describe("Arguments", func() { 14 | It("templates URLs", func() { 15 | templates := []string{ 16 | "plain URL: {{ .SomeURL }}", 17 | "method on URL: {{ .SomeURL.Hostname }}", 18 | "empty URL: {{ .EmptyURL }}", 19 | "handled empty URL: {{- if .EmptyURL }}{{ .EmptyURL }}{{ end }}", 20 | } 21 | data := struct { 22 | SomeURL *url.URL 23 | EmptyURL *url.URL 24 | }{ 25 | &url.URL{Scheme: "https", Host: "the.host.name:3456"}, 26 | nil, 27 | } 28 | 29 | out, err := RenderTemplates(templates, data) 30 | Expect(err).NotTo(HaveOccurred()) 31 | Expect(out).To(BeEquivalentTo([]string{ 32 | "plain URL: https://the.host.name:3456", 33 | "method on URL: the.host.name", 34 | "empty URL: <nil>", 35 | "handled empty URL:", 36 | })) 37 | }) 38 | 39 | It("templates strings", func() { 40 | templates := []string{ 41 | "a string: {{ .SomeString }}", 42 | "empty string: {{- .EmptyString }}", 43 | } 44 | data := struct { 45 | SomeString string 46 | EmptyString string 47 | }{ 48 | "this is some random string", 49 | "", 50 | } 51 | 52 | out, err := RenderTemplates(templates, data) 53 | Expect(err).NotTo(HaveOccurred()) 54 | Expect(out).To(BeEquivalentTo([]string{ 55 | "a string: this is some random string", 56 | "empty string:", 57 | })) 58 | }) 59 | 60 | It("has no access to unexported fields", func() { 61 | templates := []string{ 62 | "this is just a string", 63 | "this blows up {{ .test }}", 64 | } 65 | data := struct{ test string }{"ooops private"} 66 | 67 | out, err := RenderTemplates(templates, data) 68 | Expect(out).To(BeEmpty()) 69 | Expect(err).To(MatchError( 70 | ContainSubstring("is an unexported field of struct"), 71 | )) 72 | }) 73 | 74 | It("errors when field cannot be found", func() { 75 | templates := []string{"this does {{ .NotExist }}"} 76 | data := struct{ Unused string }{"unused"} 77 | 78 | out, err := RenderTemplates(templates, data) 79 | Expect(out).To(BeEmpty()) 80 | Expect(err).To(MatchError( 81 | ContainSubstring("can't evaluate field"), 82 | )) 83 | }) 84 | 85 | Context("When overriding external default args", func() { 86 | It("does not change the internal default args for APIServer", func() { 87 | integration.APIServerDefaultArgs[0] = "oh no!" 88 | Expect(APIServerDefaultArgs).NotTo(BeEquivalentTo(integration.APIServerDefaultArgs)) 89 | }) 90 | It("does not change the internal default args for Etcd", func() { 91 | integration.EtcdDefaultArgs[0] = "oh no!" 92 | Expect(EtcdDefaultArgs).NotTo(BeEquivalentTo(integration.EtcdDefaultArgs)) 93 | }) 94 | }) 95 | }) 96 | -------------------------------------------------------------------------------- /integration/internal/bin_path_finder.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "regexp" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | var assetsPath string 12 | 13 | func init() { 14 | _, thisFile, _, ok := runtime.Caller(0) 15 | if !ok { 16 | panic("Could not determine the path of the BinPathFinder") 17 | } 18 | assetsPath = filepath.Join(filepath.Dir(thisFile), "..", "assets", "bin") 19 | } 20 | 21 | // BinPathFinder checks the an environment variable, derived from the symbolic name, 22 | // and falls back to a default assets location when this variable is not set 23 | func BinPathFinder(symbolicName string) (binPath string) { 24 | punctuationPattern := regexp.MustCompile("[^A-Z0-9]+") 25 | sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_") 26 | leadingNumberPattern := regexp.MustCompile("^[0-9]+") 27 | sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "") 28 | envVar := "TEST_ASSET_" + sanitizedName 29 | 30 | if val, ok := os.LookupEnv(envVar); ok { 31 | return val 32 | } 33 | 34 | return filepath.Join(assetsPath, symbolicName) 35 | } 36 | -------------------------------------------------------------------------------- /integration/internal/bin_path_finder_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "os" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("BinPathFinder", func() { 11 | Context("when relying on the default assets path", func() { 12 | var ( 13 | previousAssetsPath string 14 | ) 15 | BeforeEach(func() { 16 | previousAssetsPath = assetsPath 17 | assetsPath = "/some/path/assets/bin" 18 | }) 19 | AfterEach(func() { 20 | assetsPath = previousAssetsPath 21 | }) 22 | It("returns the default path when no env var is configured", func() { 23 | binPath := BinPathFinder("some_bin") 24 | Expect(binPath).To(Equal("/some/path/assets/bin/some_bin")) 25 | }) 26 | }) 27 | 28 | Context("when environment is configured", func() { 29 | var ( 30 | previousValue string 31 | wasSet bool 32 | ) 33 | BeforeEach(func() { 34 | envVarName := "TEST_ASSET_ANOTHER_SYMBOLIC_NAME" 35 | if val, ok := os.LookupEnv(envVarName); ok { 36 | previousValue = val 37 | wasSet = true 38 | } 39 | os.Setenv(envVarName, "/path/to/some_bin.exe") 40 | }) 41 | AfterEach(func() { 42 | if wasSet { 43 | os.Setenv("TEST_ASSET_ANOTHER_SYMBOLIC_NAME", previousValue) 44 | } else { 45 | os.Unsetenv("TEST_ASSET_ANOTHER_SYMBOLIC_NAME") 46 | } 47 | }) 48 | It("returns the path from the env", func() { 49 | binPath := BinPathFinder("another_symbolic_name") 50 | Expect(binPath).To(Equal("/path/to/some_bin.exe")) 51 | }) 52 | 53 | It("sanitizes the environment variable name", func() { 54 | By("cleaning all non-underscore punctuation") 55 | binPath := BinPathFinder("another-symbolic name") 56 | Expect(binPath).To(Equal("/path/to/some_bin.exe")) 57 | binPath = BinPathFinder("another+symbolic\\name") 58 | Expect(binPath).To(Equal("/path/to/some_bin.exe")) 59 | binPath = BinPathFinder("another=symbolic.name") 60 | Expect(binPath).To(Equal("/path/to/some_bin.exe")) 61 | By("removing numbers from the beginning of the name") 62 | binPath = BinPathFinder("12another_symbolic_name") 63 | Expect(binPath).To(Equal("/path/to/some_bin.exe")) 64 | }) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /integration/internal/etcd.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "net/url" 5 | ) 6 | 7 | var EtcdDefaultArgs = []string{ 8 | "--listen-peer-urls=http://localhost:0", 9 | "--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}", 10 | "--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}", 11 | "--data-dir={{ .DataDir }}", 12 | } 13 | 14 | func DoEtcdArgDefaulting(args []string) []string { 15 | if len(args) != 0 { 16 | return args 17 | } 18 | 19 | return EtcdDefaultArgs 20 | } 21 | 22 | func isSecureScheme(scheme string) bool { 23 | // https://github.com/coreos/etcd/blob/d9deeff49a080a88c982d328ad9d33f26d1ad7b6/pkg/transport/listener.go#L53 24 | if scheme == "https" || scheme == "unixs" { 25 | return true 26 | } 27 | return false 28 | } 29 | 30 | func GetEtcdStartMessage(listenUrl url.URL) string { 31 | if isSecureScheme(listenUrl.Scheme) { 32 | // https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L167 33 | return "serving client requests on " 34 | } 35 | 36 | // https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L124 37 | return "serving insecure client requests on " 38 | } 39 | -------------------------------------------------------------------------------- /integration/internal/etcd_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | "net/url" 5 | 6 | . "sigs.k8s.io/testing_frameworks/integration/internal" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var _ = Describe("Etcd", func() { 13 | It("defaults Args if they are empty", func() { 14 | initialArgs := []string{} 15 | defaultedArgs := DoEtcdArgDefaulting(initialArgs) 16 | Expect(defaultedArgs).To(BeEquivalentTo(EtcdDefaultArgs)) 17 | }) 18 | 19 | It("keeps Args as is if they are not empty", func() { 20 | initialArgs := []string{"--eins", "--zwei=2"} 21 | defaultedArgs := DoEtcdArgDefaulting(initialArgs) 22 | Expect(defaultedArgs).To(BeEquivalentTo([]string{ 23 | "--eins", "--zwei=2", 24 | })) 25 | }) 26 | }) 27 | 28 | var _ = Describe("GetEtcdStartMessage()", func() { 29 | Context("when using a non tls URL", func() { 30 | It("generates valid start message", func() { 31 | url := url.URL{ 32 | Scheme: "http", 33 | Host: "some.insecure.host:1234", 34 | } 35 | message := GetEtcdStartMessage(url) 36 | Expect(message).To(Equal("serving insecure client requests on ")) 37 | }) 38 | }) 39 | Context("when using a tls URL", func() { 40 | It("generates valid start message", func() { 41 | url := url.URL{ 42 | Scheme: "https", 43 | Host: "some.secure.host:8443", 44 | } 45 | message := GetEtcdStartMessage(url) 46 | Expect(message).To(Equal("serving client requests on ")) 47 | }) 48 | }) 49 | }) 50 | -------------------------------------------------------------------------------- /integration/internal/integration_tests/apiserver_integration_test.go: -------------------------------------------------------------------------------- 1 | package integration_tests 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | . "sigs.k8s.io/testing_frameworks/integration" 8 | ) 9 | 10 | var _ = Describe("APIServer", func() { 11 | Context("when no EtcdURL is provided", func() { 12 | It("does not panic", func() { 13 | apiServer := &APIServer{} 14 | 15 | starter := func() { 16 | Expect(apiServer.Start()).To( 17 | MatchError(ContainSubstring("expected EtcdURL to be configured")), 18 | ) 19 | } 20 | 21 | Expect(starter).NotTo(Panic()) 22 | }) 23 | }) 24 | }) 25 | -------------------------------------------------------------------------------- /integration/internal/integration_tests/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package integrarion_tests is holding the integration tests to run against the 3 | framework. 4 | 5 | This file's only purpose is to make godep happy. 6 | */ 7 | package integration_tests 8 | -------------------------------------------------------------------------------- /integration/internal/integration_tests/etcd_integration_test.go: -------------------------------------------------------------------------------- 1 | package integration_tests 2 | 3 | import ( 4 | "bytes" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | 10 | . "sigs.k8s.io/testing_frameworks/integration" 11 | ) 12 | 13 | var _ = Describe("Etcd", func() { 14 | It("sets the properties after defaulting", func() { 15 | etcd := &Etcd{} 16 | 17 | Expect(etcd.URL).To(BeZero()) 18 | Expect(etcd.DataDir).To(BeZero()) 19 | Expect(etcd.Path).To(BeZero()) 20 | Expect(etcd.StartTimeout).To(BeZero()) 21 | Expect(etcd.StopTimeout).To(BeZero()) 22 | 23 | Expect(etcd.Start()).To(Succeed()) 24 | defer func() { 25 | Expect(etcd.Stop()).To(Succeed()) 26 | }() 27 | 28 | Expect(etcd.URL).NotTo(BeZero()) 29 | Expect(etcd.DataDir).NotTo(BeZero()) 30 | Expect(etcd.Path).NotTo(BeZero()) 31 | Expect(etcd.StartTimeout).NotTo(BeZero()) 32 | Expect(etcd.StopTimeout).NotTo(BeZero()) 33 | }) 34 | 35 | It("can inspect IO", func() { 36 | stderr := &bytes.Buffer{} 37 | etcd := &Etcd{ 38 | Err: stderr, 39 | } 40 | 41 | Expect(etcd.Start()).To(Succeed()) 42 | defer func() { 43 | Expect(etcd.Stop()).To(Succeed()) 44 | }() 45 | 46 | Expect(stderr.String()).NotTo(BeEmpty()) 47 | }) 48 | 49 | It("can use user specified Args", func() { 50 | stdout := &bytes.Buffer{} 51 | stderr := &bytes.Buffer{} 52 | etcd := &Etcd{ 53 | Args: []string{"--help"}, 54 | Out: stdout, 55 | Err: stderr, 56 | StartTimeout: 500 * time.Millisecond, 57 | } 58 | 59 | // it will timeout, as we'll never see the "startup message" we are waiting 60 | // for on StdErr 61 | Expect(etcd.Start()).To(MatchError(ContainSubstring("timeout"))) 62 | 63 | Expect(stdout.String()).To(ContainSubstring("member flags")) 64 | Expect(stderr.String()).To(ContainSubstring("usage: etcd")) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /integration/internal/integration_tests/integration_suite_test.go: -------------------------------------------------------------------------------- 1 | package integration_tests 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestIntegration(t *testing.T) { 11 | t.Parallel() 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Integration Framework Integration Tests") 14 | } 15 | -------------------------------------------------------------------------------- /integration/internal/integration_tests/integration_test.go: -------------------------------------------------------------------------------- 1 | package integration_tests 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net" 7 | "time" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | "sigs.k8s.io/testing_frameworks/integration" 12 | ) 13 | 14 | var _ = Describe("The Testing Framework", func() { 15 | var controlPlane *integration.ControlPlane 16 | 17 | AfterEach(func() { 18 | Expect(controlPlane.Stop()).To(Succeed()) 19 | }) 20 | 21 | It("Successfully manages the control plane lifecycle", func() { 22 | var err error 23 | 24 | controlPlane = &integration.ControlPlane{} 25 | 26 | By("Starting all the control plane processes") 27 | err = controlPlane.Start() 28 | Expect(err).NotTo(HaveOccurred(), "Expected controlPlane to start successfully") 29 | 30 | apiServerURL := controlPlane.APIURL() 31 | etcdClientURL := controlPlane.APIServer.EtcdURL 32 | 33 | isEtcdListeningForClients := isSomethingListeningOnPort(etcdClientURL.Host) 34 | isAPIServerListening := isSomethingListeningOnPort(apiServerURL.Host) 35 | 36 | By("Ensuring Etcd is listening") 37 | Expect(isEtcdListeningForClients()).To(BeTrue(), 38 | fmt.Sprintf("Expected Etcd to listen for clients on %s,", etcdClientURL.Host)) 39 | 40 | By("Ensuring APIServer is listening") 41 | CheckAPIServerIsReady(controlPlane.KubeCtl()) 42 | 43 | By("getting a kubectl & run it against the control plane") 44 | kubeCtl := controlPlane.KubeCtl() 45 | stdout, stderr, err := kubeCtl.Run("get", "pods") 46 | Expect(err).NotTo(HaveOccurred()) 47 | bytes, err := ioutil.ReadAll(stdout) 48 | Expect(err).NotTo(HaveOccurred()) 49 | Expect(bytes).To(BeEmpty()) 50 | Expect(stderr).To(ContainSubstring("No resources found")) 51 | 52 | By("Stopping all the control plane processes") 53 | err = controlPlane.Stop() 54 | Expect(err).NotTo(HaveOccurred(), "Expected controlPlane to stop successfully") 55 | 56 | By("Ensuring Etcd is not listening anymore") 57 | Expect(isEtcdListeningForClients()).To(BeFalse(), "Expected Etcd not to listen for clients anymore") 58 | 59 | By("Ensuring APIServer is not listening anymore") 60 | Expect(isAPIServerListening()).To(BeFalse(), "Expected APIServer not to listen anymore") 61 | 62 | By("Not erroring when stopping a stopped ControlPlane") 63 | Expect(func() { 64 | Expect(controlPlane.Stop()).To(Succeed()) 65 | }).NotTo(Panic()) 66 | }) 67 | 68 | Context("when Stop() is called on the control plane", func() { 69 | Context("but the control plane is not started yet", func() { 70 | It("does not error", func() { 71 | controlPlane = &integration.ControlPlane{} 72 | 73 | stoppingTheControlPlane := func() { 74 | Expect(controlPlane.Stop()).To(Succeed()) 75 | } 76 | 77 | Expect(stoppingTheControlPlane).NotTo(Panic()) 78 | }) 79 | }) 80 | }) 81 | 82 | Context("when the control plane is configured with its components", func() { 83 | It("it does not default them", func() { 84 | myEtcd, myAPIServer := 85 | &integration.Etcd{StartTimeout: 15 * time.Second}, 86 | &integration.APIServer{StopTimeout: 16 * time.Second} 87 | 88 | controlPlane = &integration.ControlPlane{ 89 | Etcd: myEtcd, 90 | APIServer: myAPIServer, 91 | } 92 | 93 | Expect(controlPlane.Start()).To(Succeed()) 94 | Expect(controlPlane.Etcd).To(BeIdenticalTo(myEtcd)) 95 | Expect(controlPlane.APIServer).To(BeIdenticalTo(myAPIServer)) 96 | Expect(controlPlane.Etcd.StartTimeout).To(Equal(15 * time.Second)) 97 | Expect(controlPlane.APIServer.StopTimeout).To(Equal(16 * time.Second)) 98 | }) 99 | }) 100 | 101 | Context("when etcd already started", func() { 102 | It("starts the control plane successfully", func() { 103 | myEtcd := &integration.Etcd{} 104 | Expect(myEtcd.Start()).To(Succeed()) 105 | 106 | controlPlane = &integration.ControlPlane{ 107 | Etcd: myEtcd, 108 | } 109 | 110 | Expect(controlPlane.Start()).To(Succeed()) 111 | }) 112 | }) 113 | 114 | Context("when control plane is already started", func() { 115 | It("can attempt to start again without errors", func() { 116 | controlPlane = &integration.ControlPlane{} 117 | Expect(controlPlane.Start()).To(Succeed()) 118 | Expect(controlPlane.Start()).To(Succeed()) 119 | }) 120 | }) 121 | 122 | Context("when control plane starts and stops", func() { 123 | It("can attempt to start again without errors", func() { 124 | controlPlane = &integration.ControlPlane{} 125 | Expect(controlPlane.Start()).To(Succeed()) 126 | Expect(controlPlane.Stop()).To(Succeed()) 127 | Expect(controlPlane.Start()).To(Succeed()) 128 | }) 129 | }) 130 | 131 | Measure("It should be fast to bring up and tear down the control plane", func(b Benchmarker) { 132 | b.Time("lifecycle", func() { 133 | controlPlane = &integration.ControlPlane{} 134 | 135 | controlPlane.Start() 136 | controlPlane.Stop() 137 | }) 138 | }, 10) 139 | }) 140 | 141 | type portChecker func() bool 142 | 143 | func isSomethingListeningOnPort(hostAndPort string) portChecker { 144 | return func() bool { 145 | conn, err := net.DialTimeout("tcp", hostAndPort, 1*time.Second) 146 | 147 | if err != nil { 148 | return false 149 | } 150 | conn.Close() 151 | return true 152 | } 153 | } 154 | 155 | // CheckAPIServerIsReady checks if the APIServer is really ready and not only 156 | // listening. 157 | // 158 | // While porting some tests in k/k 159 | // (https://github.com/hoegaarden/kubernetes/blob/287fdef1bd98646bc521f4433c1009936d5cf7a2/hack/make-rules/test-cmd-util.sh#L1524-L1535) 160 | // we found, that the APIServer was 161 | // listening but not serving certain APIs yet. 162 | // 163 | // We changed the readiness detection in the PR at 164 | // https://github.com/kubernetes-sigs/testing_frameworks/pull/48. To confirm 165 | // this changed behaviour does what it should do, we used the same test as in 166 | // k/k's test-cmd (see link above) and test if certain well-known known APIs 167 | // are actually available. 168 | func CheckAPIServerIsReady(kubeCtl *integration.KubeCtl) { 169 | expectedAPIS := []string{ 170 | "/api/v1/namespaces/default/pods 200 OK", 171 | "/api/v1/namespaces/default/replicationcontrollers 200 OK", 172 | "/api/v1/namespaces/default/services 200 OK", 173 | "/apis/apps/v1/namespaces/default/daemonsets 200 OK", 174 | "/apis/apps/v1/namespaces/default/deployments 200 OK", 175 | "/apis/apps/v1/namespaces/default/replicasets 200 OK", 176 | "/apis/apps/v1/namespaces/default/statefulsets 200 OK", 177 | "/apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers 200", 178 | "/apis/batch/v1/namespaces/default/jobs 200 OK", 179 | } 180 | 181 | _, output, err := kubeCtl.Run("--v=6", "--namespace", "default", "get", "all", "--chunk-size=0") 182 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 183 | 184 | stdoutBytes, err := ioutil.ReadAll(output) 185 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 186 | 187 | for _, api := range expectedAPIS { 188 | ExpectWithOffset(1, string(stdoutBytes)).To(ContainSubstring(api)) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /integration/internal/internal_suite_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestInternal(t *testing.T) { 11 | t.Parallel() 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "Internal Suite") 14 | } 15 | -------------------------------------------------------------------------------- /integration/internal/process.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "os/exec" 12 | "path" 13 | "strconv" 14 | "time" 15 | 16 | "github.com/onsi/gomega/gbytes" 17 | "github.com/onsi/gomega/gexec" 18 | 19 | "sigs.k8s.io/testing_frameworks/integration/addr" 20 | ) 21 | 22 | type ProcessState struct { 23 | DefaultedProcessInput 24 | Session *gexec.Session 25 | // Healthcheck Endpoint. If we get http.StatusOK from this endpoint, we 26 | // assume the process is ready to operate. E.g. "/healthz". If this is set, 27 | // we ignore StartMessage. 28 | HealthCheckEndpoint string 29 | // HealthCheckPollInterval is the interval which will be used for polling the 30 | // HealthCheckEndpoint. 31 | // If left empty it will default to 100 Milliseconds. 32 | HealthCheckPollInterval time.Duration 33 | // StartMessage is the message to wait for on stderr. If we recieve this 34 | // message, we assume the process is ready to operate. Ignored if 35 | // HealthCheckEndpoint is specified. 36 | // 37 | // The usage of StartMessage is discouraged, favour HealthCheckEndpoint 38 | // instead! 39 | // 40 | // Deprecated: Use HealthCheckEndpoint in favour of StartMessage 41 | StartMessage string 42 | Args []string 43 | 44 | // ready holds wether the process is currently in ready state (hit the ready condition) or not. 45 | // It will be set to true on a successful `Start()` and set to false on a successful `Stop()` 46 | ready bool 47 | } 48 | 49 | type DefaultedProcessInput struct { 50 | URL url.URL 51 | Dir string 52 | DirNeedsCleaning bool 53 | Path string 54 | StopTimeout time.Duration 55 | StartTimeout time.Duration 56 | } 57 | 58 | func DoDefaulting( 59 | name string, 60 | listenUrl *url.URL, 61 | dir string, 62 | path string, 63 | startTimeout time.Duration, 64 | stopTimeout time.Duration, 65 | ) (DefaultedProcessInput, error) { 66 | defaults := DefaultedProcessInput{ 67 | Dir: dir, 68 | Path: path, 69 | StartTimeout: startTimeout, 70 | StopTimeout: stopTimeout, 71 | } 72 | 73 | if listenUrl == nil { 74 | port, host, err := addr.Suggest() 75 | if err != nil { 76 | return DefaultedProcessInput{}, err 77 | } 78 | defaults.URL = url.URL{ 79 | Scheme: "http", 80 | Host: net.JoinHostPort(host, strconv.Itoa(port)), 81 | } 82 | } else { 83 | defaults.URL = *listenUrl 84 | } 85 | 86 | if dir == "" { 87 | newDir, err := ioutil.TempDir("", "k8s_test_framework_") 88 | if err != nil { 89 | return DefaultedProcessInput{}, err 90 | } 91 | defaults.Dir = newDir 92 | defaults.DirNeedsCleaning = true 93 | } 94 | 95 | if path == "" { 96 | if name == "" { 97 | return DefaultedProcessInput{}, fmt.Errorf("must have at least one of name or path") 98 | } 99 | defaults.Path = BinPathFinder(name) 100 | } 101 | 102 | if startTimeout == 0 { 103 | defaults.StartTimeout = 20 * time.Second 104 | } 105 | 106 | if stopTimeout == 0 { 107 | defaults.StopTimeout = 20 * time.Second 108 | } 109 | 110 | return defaults, nil 111 | } 112 | 113 | type stopChannel chan struct{} 114 | 115 | func (ps *ProcessState) Start(stdout, stderr io.Writer) (err error) { 116 | if ps.ready { 117 | return nil 118 | } 119 | 120 | command := exec.Command(ps.Path, ps.Args...) 121 | 122 | ready := make(chan bool) 123 | timedOut := time.After(ps.StartTimeout) 124 | var pollerStopCh stopChannel 125 | 126 | if ps.HealthCheckEndpoint != "" { 127 | healthCheckURL := ps.URL 128 | healthCheckURL.Path = ps.HealthCheckEndpoint 129 | pollerStopCh = make(stopChannel) 130 | go pollURLUntilOK(healthCheckURL, ps.HealthCheckPollInterval, ready, pollerStopCh) 131 | } else { 132 | startDetectStream := gbytes.NewBuffer() 133 | ready = startDetectStream.Detect(ps.StartMessage) 134 | stderr = safeMultiWriter(stderr, startDetectStream) 135 | } 136 | 137 | ps.Session, err = gexec.Start(command, stdout, stderr) 138 | if err != nil { 139 | return err 140 | } 141 | 142 | select { 143 | case <-ready: 144 | ps.ready = true 145 | return nil 146 | case <-timedOut: 147 | if pollerStopCh != nil { 148 | close(pollerStopCh) 149 | } 150 | if ps.Session != nil { 151 | ps.Session.Terminate() 152 | } 153 | return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path)) 154 | } 155 | } 156 | 157 | func safeMultiWriter(writers ...io.Writer) io.Writer { 158 | safeWriters := []io.Writer{} 159 | for _, w := range writers { 160 | if w != nil { 161 | safeWriters = append(safeWriters, w) 162 | } 163 | } 164 | return io.MultiWriter(safeWriters...) 165 | } 166 | 167 | func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) { 168 | if interval <= 0 { 169 | interval = 100 * time.Millisecond 170 | } 171 | for { 172 | res, err := http.Get(url.String()) 173 | if err == nil && res.StatusCode == http.StatusOK { 174 | ready <- true 175 | return 176 | } 177 | 178 | select { 179 | case <-stopCh: 180 | return 181 | default: 182 | time.Sleep(interval) 183 | } 184 | } 185 | } 186 | 187 | func (ps *ProcessState) Stop() error { 188 | if ps.Session == nil { 189 | return nil 190 | } 191 | 192 | // gexec's Session methods (Signal, Kill, ...) do not check if the Process is 193 | // nil, so we are doing this here for now. 194 | // This should probably be fixed in gexec. 195 | if ps.Session.Command.Process == nil { 196 | return nil 197 | } 198 | 199 | detectedStop := ps.Session.Terminate().Exited 200 | timedOut := time.After(ps.StopTimeout) 201 | 202 | select { 203 | case <-detectedStop: 204 | break 205 | case <-timedOut: 206 | return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path)) 207 | } 208 | ps.ready = false 209 | if ps.DirNeedsCleaning { 210 | return os.RemoveAll(ps.Dir) 211 | } 212 | 213 | return nil 214 | } 215 | -------------------------------------------------------------------------------- /integration/internal/process_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | "time" 13 | 14 | . "github.com/onsi/ginkgo" 15 | . "github.com/onsi/gomega" 16 | "github.com/onsi/gomega/gexec" 17 | "github.com/onsi/gomega/ghttp" 18 | "sigs.k8s.io/testing_frameworks/integration/addr" 19 | . "sigs.k8s.io/testing_frameworks/integration/internal" 20 | ) 21 | 22 | var _ = Describe("Start method", func() { 23 | var ( 24 | processState *ProcessState 25 | ) 26 | BeforeEach(func() { 27 | processState = &ProcessState{} 28 | processState.Path = "bash" 29 | processState.Args = simpleBashScript 30 | }) 31 | 32 | It("can start a process", func() { 33 | processState.StartTimeout = 10 * time.Second 34 | processState.StartMessage = "loop 5" 35 | 36 | err := processState.Start(nil, nil) 37 | Expect(err).NotTo(HaveOccurred()) 38 | 39 | Consistently(processState.Session.ExitCode).Should(BeNumerically("==", -1)) 40 | }) 41 | 42 | Context("when a health check endpoint is provided", func() { 43 | var server *ghttp.Server 44 | BeforeEach(func() { 45 | server = ghttp.NewServer() 46 | }) 47 | AfterEach(func() { 48 | server.Close() 49 | }) 50 | 51 | Context("when the healthcheck returns ok", func() { 52 | BeforeEach(func() { 53 | server.RouteToHandler("GET", "/healthz", ghttp.RespondWith(http.StatusOK, "")) 54 | }) 55 | 56 | It("hits the endpoint, and successfully starts", func() { 57 | processState.HealthCheckEndpoint = "/healthz" 58 | processState.StartTimeout = 100 * time.Millisecond 59 | processState.URL = getServerURL(server) 60 | 61 | err := processState.Start(nil, nil) 62 | Expect(err).NotTo(HaveOccurred()) 63 | Expect(server.ReceivedRequests()).To(HaveLen(1)) 64 | Consistently(processState.Session.ExitCode).Should(BeNumerically("==", -1)) 65 | }) 66 | }) 67 | 68 | Context("when the healthcheck always returns failure", func() { 69 | BeforeEach(func() { 70 | server.RouteToHandler("GET", "/healthz", ghttp.RespondWith(http.StatusInternalServerError, "")) 71 | }) 72 | It("returns a timeout error and stops health API checker", func() { 73 | processState.HealthCheckEndpoint = "/healthz" 74 | processState.StartTimeout = 500 * time.Millisecond 75 | processState.URL = getServerURL(server) 76 | 77 | err := processState.Start(nil, nil) 78 | Expect(err).To(MatchError(ContainSubstring("timeout"))) 79 | 80 | nrReceivedRequests := len(server.ReceivedRequests()) 81 | Expect(nrReceivedRequests).To(Equal(5)) 82 | time.Sleep(200 * time.Millisecond) 83 | Expect(nrReceivedRequests).To(Equal(5)) 84 | }) 85 | }) 86 | 87 | Context("when the healthcheck isn't even listening", func() { 88 | BeforeEach(func() { 89 | server.Close() 90 | }) 91 | 92 | It("returns a timeout error", func() { 93 | processState.HealthCheckEndpoint = "/healthz" 94 | processState.StartTimeout = 500 * time.Millisecond 95 | 96 | port, host, err := addr.Suggest() 97 | Expect(err).NotTo(HaveOccurred()) 98 | 99 | processState.URL = url.URL{ 100 | Scheme: "http", 101 | Host: net.JoinHostPort(host, strconv.Itoa(port)), 102 | } 103 | 104 | err = processState.Start(nil, nil) 105 | Expect(err).To(MatchError(ContainSubstring("timeout"))) 106 | }) 107 | }) 108 | 109 | Context("when the healthcheck fails initially but succeeds eventually", func() { 110 | BeforeEach(func() { 111 | server.AppendHandlers( 112 | ghttp.RespondWith(http.StatusInternalServerError, ""), 113 | ghttp.RespondWith(http.StatusInternalServerError, ""), 114 | ghttp.RespondWith(http.StatusInternalServerError, ""), 115 | ghttp.RespondWith(http.StatusOK, ""), 116 | ) 117 | }) 118 | 119 | It("hits the endpoint repeatedly, and successfully starts", func() { 120 | processState.HealthCheckEndpoint = "/healthz" 121 | processState.StartTimeout = 20 * time.Second 122 | processState.URL = getServerURL(server) 123 | 124 | err := processState.Start(nil, nil) 125 | Expect(err).NotTo(HaveOccurred()) 126 | Expect(server.ReceivedRequests()).To(HaveLen(4)) 127 | Consistently(processState.Session.ExitCode).Should(BeNumerically("==", -1)) 128 | }) 129 | 130 | Context("when the polling interval is not configured", func() { 131 | It("uses the default interval for polling", func() { 132 | processState.HealthCheckEndpoint = "/helathz" 133 | processState.StartTimeout = 300 * time.Millisecond 134 | processState.URL = getServerURL(server) 135 | 136 | Expect(processState.Start(nil, nil)).To(MatchError(ContainSubstring("timeout"))) 137 | Expect(server.ReceivedRequests()).To(HaveLen(3)) 138 | }) 139 | }) 140 | 141 | Context("when the polling interval is configured", func() { 142 | BeforeEach(func() { 143 | processState.HealthCheckPollInterval = time.Millisecond * 20 144 | }) 145 | 146 | It("hits the endpoint in the configured interval", func() { 147 | processState.HealthCheckEndpoint = "/healthz" 148 | processState.StartTimeout = 3 * processState.HealthCheckPollInterval 149 | processState.URL = getServerURL(server) 150 | 151 | Expect(processState.Start(nil, nil)).To(MatchError(ContainSubstring("timeout"))) 152 | Expect(server.ReceivedRequests()).To(HaveLen(3)) 153 | }) 154 | }) 155 | }) 156 | }) 157 | 158 | Context("when a health check endpoint is not provided", func() { 159 | 160 | Context("when process takes too long to start", func() { 161 | It("returns a timeout error", func() { 162 | processState.StartTimeout = 200 * time.Millisecond 163 | processState.StartMessage = "loop 5000" 164 | 165 | err := processState.Start(nil, nil) 166 | Expect(err).To(MatchError(ContainSubstring("timeout"))) 167 | 168 | Eventually(processState.Session.ExitCode).Should(Equal(143)) 169 | }) 170 | }) 171 | 172 | Context("when the command cannot be started", func() { 173 | BeforeEach(func() { 174 | processState = &ProcessState{} 175 | processState.Path = "/nonexistent" 176 | }) 177 | 178 | It("propagates the error", func() { 179 | err := processState.Start(nil, nil) 180 | 181 | Expect(os.IsNotExist(err)).To(BeTrue()) 182 | }) 183 | 184 | Context("but Stop() is called on it", func() { 185 | It("does not panic", func() { 186 | processState.Start(nil, nil) 187 | 188 | stoppingFailedProcess := func() { 189 | Expect(processState.Stop()).To(Succeed()) 190 | } 191 | 192 | Expect(stoppingFailedProcess).NotTo(Panic()) 193 | }) 194 | }) 195 | }) 196 | }) 197 | 198 | Context("when IO is configured", func() { 199 | It("can inspect stdout & stderr", func() { 200 | stdout := &bytes.Buffer{} 201 | stderr := &bytes.Buffer{} 202 | 203 | processState.Args = []string{ 204 | "-c", 205 | ` 206 | echo 'this is stderr' >&2 207 | echo 'that is stdout' 208 | echo 'i started' >&2 209 | `, 210 | } 211 | processState.StartMessage = "i started" 212 | processState.StartTimeout = 1 * time.Second 213 | 214 | Expect(processState.Start(stdout, stderr)).To(Succeed()) 215 | 216 | Expect(stdout.String()).To(Equal("that is stdout\n")) 217 | Expect(stderr.String()).To(Equal("this is stderr\ni started\n")) 218 | }) 219 | }) 220 | }) 221 | 222 | var _ = Describe("Stop method", func() { 223 | Context("when Stop() is called", func() { 224 | var ( 225 | processState *ProcessState 226 | ) 227 | BeforeEach(func() { 228 | var err error 229 | processState = &ProcessState{} 230 | processState.Session, err = gexec.Start(getSimpleCommand(), nil, nil) 231 | Expect(err).NotTo(HaveOccurred()) 232 | processState.StopTimeout = 10 * time.Second 233 | }) 234 | 235 | It("stops the process", func() { 236 | Expect(processState.Stop()).To(Succeed()) 237 | }) 238 | 239 | Context("multiple times", func() { 240 | It("does not error or panic on consecutive calls", func() { 241 | stoppingTheProcess := func() { 242 | Expect(processState.Stop()).To(Succeed()) 243 | } 244 | Expect(stoppingTheProcess).NotTo(Panic()) 245 | Expect(stoppingTheProcess).NotTo(Panic()) 246 | Expect(stoppingTheProcess).NotTo(Panic()) 247 | }) 248 | }) 249 | }) 250 | 251 | Context("when the command cannot be stopped", func() { 252 | It("returns a timeout error", func() { 253 | var err error 254 | 255 | processState := &ProcessState{} 256 | processState.Session, err = gexec.Start(getSimpleCommand(), nil, nil) 257 | Expect(err).NotTo(HaveOccurred()) 258 | processState.Session.Exited = make(chan struct{}) 259 | processState.StopTimeout = 200 * time.Millisecond 260 | 261 | Expect(processState.Stop()).To(MatchError(ContainSubstring("timeout"))) 262 | }) 263 | }) 264 | 265 | Context("when the directory needs to be cleaned up", func() { 266 | It("removes the directory", func() { 267 | var err error 268 | 269 | processState := &ProcessState{} 270 | processState.Session, err = gexec.Start(getSimpleCommand(), nil, nil) 271 | Expect(err).NotTo(HaveOccurred()) 272 | processState.Dir, err = ioutil.TempDir("", "k8s_test_framework_") 273 | Expect(err).NotTo(HaveOccurred()) 274 | processState.DirNeedsCleaning = true 275 | processState.StopTimeout = 400 * time.Millisecond 276 | 277 | Expect(processState.Stop()).To(Succeed()) 278 | Expect(processState.Dir).NotTo(BeAnExistingFile()) 279 | }) 280 | }) 281 | }) 282 | 283 | var _ = Describe("DoDefaulting", func() { 284 | Context("when all inputs are provided", func() { 285 | It("passes them through", func() { 286 | defaults, err := DoDefaulting( 287 | "some name", 288 | &url.URL{Host: "some.host.to.listen.on"}, 289 | "/some/dir", 290 | "/some/path/to/some/bin", 291 | 20*time.Hour, 292 | 65537*time.Millisecond, 293 | ) 294 | Expect(err).NotTo(HaveOccurred()) 295 | 296 | Expect(defaults.URL).To(Equal(url.URL{Host: "some.host.to.listen.on"})) 297 | Expect(defaults.Dir).To(Equal("/some/dir")) 298 | Expect(defaults.DirNeedsCleaning).To(BeFalse()) 299 | Expect(defaults.Path).To(Equal("/some/path/to/some/bin")) 300 | Expect(defaults.StartTimeout).To(Equal(20 * time.Hour)) 301 | Expect(defaults.StopTimeout).To(Equal(65537 * time.Millisecond)) 302 | }) 303 | }) 304 | 305 | Context("when inputs are empty", func() { 306 | It("defaults them", func() { 307 | defaults, err := DoDefaulting( 308 | "some name", 309 | nil, 310 | "", 311 | "", 312 | 0, 313 | 0, 314 | ) 315 | Expect(err).NotTo(HaveOccurred()) 316 | 317 | Expect(defaults.Dir).To(BeADirectory()) 318 | Expect(os.RemoveAll(defaults.Dir)).To(Succeed()) 319 | Expect(defaults.DirNeedsCleaning).To(BeTrue()) 320 | 321 | Expect(defaults.URL).NotTo(BeZero()) 322 | Expect(defaults.URL.Scheme).To(Equal("http")) 323 | Expect(defaults.URL.Hostname()).NotTo(BeEmpty()) 324 | Expect(defaults.URL.Port()).NotTo(BeEmpty()) 325 | 326 | Expect(defaults.Path).NotTo(BeEmpty()) 327 | 328 | Expect(defaults.StartTimeout).NotTo(BeZero()) 329 | Expect(defaults.StopTimeout).NotTo(BeZero()) 330 | }) 331 | }) 332 | 333 | Context("when neither name nor path are provided", func() { 334 | It("returns an error", func() { 335 | _, err := DoDefaulting( 336 | "", 337 | nil, 338 | "", 339 | "", 340 | 0, 341 | 0, 342 | ) 343 | Expect(err).To(MatchError("must have at least one of name or path")) 344 | }) 345 | }) 346 | }) 347 | 348 | var simpleBashScript = []string{ 349 | "-c", 350 | ` 351 | i=0 352 | while true 353 | do 354 | echo "loop $i" >&2 355 | let 'i += 1' 356 | sleep 0.2 357 | done 358 | `, 359 | } 360 | 361 | func getSimpleCommand() *exec.Cmd { 362 | return exec.Command("bash", simpleBashScript...) 363 | } 364 | 365 | func getServerURL(server *ghttp.Server) url.URL { 366 | url, err := url.Parse(server.URL()) 367 | Expect(err).NotTo(HaveOccurred()) 368 | return *url 369 | } 370 | -------------------------------------------------------------------------------- /integration/kubectl.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os/exec" 7 | 8 | "sigs.k8s.io/testing_frameworks/integration/internal" 9 | ) 10 | 11 | // KubeCtl is a wrapper around the kubectl binary. 12 | type KubeCtl struct { 13 | // Path where the kubectl binary can be found. 14 | // 15 | // If this is left empty, we will attempt to locate a binary, by checking for 16 | // the TEST_ASSET_KUBECTL environment variable, and the default test assets 17 | // directory. See the "Binaries" section above (in doc.go) for details. 18 | Path string 19 | 20 | // Opts can be used to configure additional flags which will be used each 21 | // time the wrapped binary is called. 22 | // 23 | // For example, you might want to use this to set the URL of the APIServer to 24 | // connect to. 25 | Opts []string 26 | } 27 | 28 | // Run executes the wrapped binary with some preconfigured options and the 29 | // arguments given to this method. It returns Readers for the stdout and 30 | // stderr. 31 | func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) { 32 | if k.Path == "" { 33 | k.Path = internal.BinPathFinder("kubectl") 34 | } 35 | 36 | stdoutBuffer := &bytes.Buffer{} 37 | stderrBuffer := &bytes.Buffer{} 38 | allArgs := append(k.Opts, args...) 39 | 40 | cmd := exec.Command(k.Path, allArgs...) 41 | cmd.Stdout = stdoutBuffer 42 | cmd.Stderr = stderrBuffer 43 | 44 | err = cmd.Run() 45 | 46 | return stdoutBuffer, stderrBuffer, err 47 | } 48 | -------------------------------------------------------------------------------- /integration/kubectl_test.go: -------------------------------------------------------------------------------- 1 | package integration_test 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | . "sigs.k8s.io/testing_frameworks/integration" 10 | ) 11 | 12 | var _ = Describe("Kubectl", func() { 13 | It("runs kubectl", func() { 14 | k := &KubeCtl{Path: "bash"} 15 | args := []string{"-c", "echo 'something'"} 16 | stdout, stderr, err := k.Run(args...) 17 | Expect(err).NotTo(HaveOccurred()) 18 | Expect(stdout).To(ContainSubstring("something")) 19 | bytes, err := ioutil.ReadAll(stderr) 20 | Expect(err).NotTo(HaveOccurred()) 21 | Expect(bytes).To(BeEmpty()) 22 | }) 23 | 24 | Context("when the command returns a non-zero exit code", func() { 25 | It("returns an error", func() { 26 | k := &KubeCtl{Path: "bash"} 27 | args := []string{ 28 | "-c", "echo 'this is StdErr' >&2; echo 'but this is StdOut' >&1; exit 66", 29 | } 30 | 31 | stdout, stderr, err := k.Run(args...) 32 | 33 | Expect(err).To(MatchError(ContainSubstring("exit status 66"))) 34 | 35 | Expect(stdout).To(ContainSubstring("but this is StdOut")) 36 | Expect(stderr).To(ContainSubstring("this is StdErr")) 37 | }) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /integration/scripts/download-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | # Use DEBUG=1 ./scripts/download-binaries.sh to get debug output 5 | quiet="-s" 6 | [[ -z "${DEBUG:-""}" ]] || { 7 | set -x 8 | quiet="" 9 | } 10 | 11 | logEnd() { 12 | local msg='done.' 13 | [ "$1" -eq 0 ] || msg='Error downloading assets' 14 | echo "$msg" 15 | } 16 | trap 'logEnd $?' EXIT 17 | 18 | # Use BASE_URL=https://my/binaries/url ./scripts/download-binaries to download 19 | # from a different bucket 20 | : "${BASE_URL:="https://storage.googleapis.com/k8s-c10s-test-binaries"}" 21 | 22 | test_framework_dir="$(cd "$(dirname "$0")/.." ; pwd)" 23 | os="$(uname -s)" 24 | os_lowercase="$(echo "$os" | tr '[:upper:]' '[:lower:]' )" 25 | arch="$(uname -m)" 26 | 27 | dest_dir="${1:-"${test_framework_dir}/assets/bin"}" 28 | etcd_dest="${dest_dir}/etcd" 29 | kubectl_dest="${dest_dir}/kubectl" 30 | kube_apiserver_dest="${dest_dir}/kube-apiserver" 31 | 32 | echo "About to download a couple of binaries. This might take a while..." 33 | 34 | curl $quiet "${BASE_URL}/etcd-${os}-${arch}" --output "$etcd_dest" 35 | curl $quiet "${BASE_URL}/kube-apiserver-${os}-${arch}" --output "$kube_apiserver_dest" 36 | 37 | kubectl_version="$(curl $quiet https://storage.googleapis.com/kubernetes-release/release/stable.txt)" 38 | kubectl_url="https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/${os_lowercase}/amd64/kubectl" 39 | curl $quiet "$kubectl_url" --output "$kubectl_dest" 40 | 41 | chmod +x "$etcd_dest" "$kubectl_dest" "$kube_apiserver_dest" 42 | 43 | echo "# destination:" 44 | echo "# ${dest_dir}" 45 | echo "# versions:" 46 | echo -n "# etcd: "; "$etcd_dest" --version | head -n 1 47 | echo -n "# kube-apiserver: "; "$kube_apiserver_dest" --version 48 | echo -n "# kubectl: "; "$kubectl_dest" version --client --short 49 | --------------------------------------------------------------------------------