├── .github ├── dependabot.yml └── workflows │ ├── build.yaml │ ├── codeql-analysis.yml │ ├── dependabot-auto-approve.yaml │ ├── dependabot-auto-merge.yaml │ └── semgrep.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── apis └── kubeapplier │ ├── doc.go │ └── v1alpha1 │ ├── groupversion_info.go │ ├── meta_types.go │ ├── waybill_types.go │ └── zz_generated.deepcopy.go ├── client ├── client.go ├── client_test.go └── suite_test.go ├── git └── repository.go ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt ├── kubectl ├── client.go └── client_test.go ├── log └── logger.go ├── main.go ├── manifests ├── base │ ├── client │ │ ├── kustomization.yaml │ │ └── service-account.yaml │ ├── cluster │ │ ├── clusterrole.yaml │ │ ├── kube-applier.io_waybills.yaml │ │ └── kustomization.yaml │ └── server │ │ ├── kube-applier.yaml │ │ └── kustomization.yaml └── example │ ├── kube-applier-ingress.yaml │ ├── kube-applier-patch.yaml │ ├── kustomization.yaml │ ├── resources │ └── known_hosts │ ├── secrets │ ├── ssh │ └── strongbox-keyring │ └── waybill.yaml ├── metrics ├── prometheus.go └── prometheus_test.go ├── run ├── runner.go ├── runner_test.go ├── scheduler.go ├── scheduler_test.go ├── strongbox.go └── suite_test.go ├── static ├── bootstrap │ ├── css │ │ ├── bootstrap-theme.css │ │ ├── bootstrap-theme.css.map │ │ ├── bootstrap-theme.min.css │ │ ├── bootstrap-theme.min.css.map │ │ ├── bootstrap.css │ │ ├── bootstrap.css.map │ │ ├── bootstrap.min.css │ │ └── bootstrap.min.css.map │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ └── js │ │ ├── bootstrap.js │ │ ├── bootstrap.min.js │ │ ├── jquery.min.js │ │ └── npm.js ├── img │ └── favicon.ico ├── js │ └── main.js └── stylesheets │ └── main.css ├── sysutil └── clock.go ├── templates └── status.html ├── testdata ├── bases │ └── simple-deployment │ │ ├── deployment.yaml │ │ └── kustomization.yaml ├── manifests │ ├── app-a-kustomize │ │ ├── 00-namespace.yaml │ │ ├── deployment.yaml │ │ ├── kustomization.yaml │ │ └── secret.yaml │ ├── app-a │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml │ ├── app-b-kustomize │ │ ├── 00-namespace.yaml │ │ └── kustomization.yaml │ ├── app-b │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml │ ├── app-c-kustomize │ │ ├── 00-namespace.yaml │ │ └── kustomization.yaml │ ├── app-c │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml │ ├── app-d-kustomize │ │ ├── 00-namespace.yaml │ │ └── kustomization.yaml │ ├── app-d │ │ ├── .gitattributes │ │ ├── .strongbox-keyid │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml │ ├── app-e │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml │ └── strongbox-age │ │ ├── .gitattributes │ │ ├── .strongbox_recipient │ │ ├── 00-namespace.yaml │ │ └── deployment.yaml └── web │ └── testStatusPage.html └── webserver ├── oidc └── oidc.go ├── operational.go ├── result.go ├── result_test.go ├── suite_test.go ├── template.go ├── template_test.go ├── webserver.go └── webserver_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # See GitHub's docs for more information on this file: 2 | # https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/configuration-options-for-dependency-updates 3 | version: 2 4 | updates: 5 | # Maintain dependencies for GitHub Actions 6 | - package-ecosystem: "github-actions" 7 | directory: "/" 8 | schedule: 9 | interval: "monthly" 10 | 11 | # Maintain dependencies for Go modules 12 | - package-ecosystem: "gomod" 13 | directory: "/" 14 | schedule: 15 | interval: "monthly" 16 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | tags: 8 | - "v*" 9 | pull_request: 10 | branches: 11 | - "master" 12 | 13 | env: 14 | REGISTRY: quay.io 15 | IMAGE_NAME: ${{ github.repository }} 16 | 17 | jobs: 18 | docker: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: Checkout 22 | uses: actions/checkout@v2 23 | with: 24 | fetch-depth: 0 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v3 27 | - name: Set up Docker Buildx 28 | uses: docker/setup-buildx-action@v3 29 | - name: Login to Quay.io Container Registry 30 | uses: docker/login-action@v3 31 | with: 32 | registry: quay.io 33 | username: utilitywarehouse+drone_ci 34 | password: ${{ secrets.SYSTEM_QUAY_TOKEN }} 35 | - name: Extract metadata (tags, labels) for Docker 36 | id: meta 37 | uses: docker/metadata-action@v5 38 | with: 39 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 40 | - name: Build and push Docker image 41 | uses: docker/build-push-action@v6 42 | with: 43 | context: . 44 | push: true 45 | tags: ${{ steps.meta.outputs.tags }} 46 | labels: ${{ steps.meta.outputs.labels }} 47 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '30 22 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go', 'javascript' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-auto-approve.yaml: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#approve-a-pull-request 2 | name: Dependabot auto-approve 3 | on: pull_request 4 | 5 | permissions: 6 | pull-requests: write 7 | 8 | jobs: 9 | dependabot: 10 | runs-on: ubuntu-latest 11 | if: ${{ github.actor == 'dependabot[bot]' }} 12 | steps: 13 | - name: Dependabot metadata 14 | id: metadata 15 | uses: dependabot/fetch-metadata@v1.3.6 16 | with: 17 | github-token: "${{ secrets.GITHUB_TOKEN }}" 18 | - name: Approve a PR 19 | run: gh pr review --approve "$PR_URL" 20 | env: 21 | PR_URL: ${{github.event.pull_request.html_url}} 22 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 23 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-auto-merge.yaml: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request 2 | name: Dependabot auto-merge 3 | on: pull_request 4 | 5 | permissions: 6 | pull-requests: write 7 | contents: write 8 | 9 | jobs: 10 | dependabot: 11 | runs-on: ubuntu-latest 12 | if: ${{ github.actor == 'dependabot[bot]' }} 13 | steps: 14 | - name: Dependabot metadata 15 | id: metadata 16 | uses: dependabot/fetch-metadata@v1.3.6 17 | with: 18 | github-token: "${{ secrets.GITHUB_TOKEN }}" 19 | - name: Enable auto-merge for Dependabot PRs 20 | run: gh pr merge --auto --merge "$PR_URL" 21 | env: 22 | PR_URL: ${{github.event.pull_request.html_url}} 23 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 24 | -------------------------------------------------------------------------------- /.github/workflows/semgrep.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name of this GitHub Actions workflow. 3 | name: Semgrep 4 | 5 | on: 6 | # Scan changed files in PRs (diff-aware scanning): 7 | pull_request: {} 8 | # Scan on-demand through GitHub Actions interface: 9 | workflow_dispatch: {} 10 | # Scan mainline branches and report all findings: 11 | push: 12 | branches: 13 | - main 14 | - master 15 | # Schedule the CI job (this method uses cron syntax): 16 | schedule: 17 | - cron: '30 14 * * *' 18 | # or whatever time works best for your team. 19 | 20 | jobs: 21 | semgrep: 22 | # User definable name of this GitHub Actions job. 23 | name: semgrep/ci 24 | # If you are self-hosting, change the following `runs-on` value: 25 | runs-on: ubuntu-latest 26 | 27 | container: 28 | # A Docker image with Semgrep installed. Do not change this. 29 | image: returntocorp/semgrep 30 | 31 | # Skip any PR created by dependabot to avoid permission issues: 32 | if: (github.actor != 'dependabot[bot]') 33 | 34 | steps: 35 | # Fetch project source with GitHub Actions Checkout. 36 | - uses: actions/checkout@v4 37 | # Run the "semgrep ci" command on the command line of the docker image. 38 | - run: semgrep scan --config auto 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | kube-applier 2 | kubebuilder-bindir/ 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-alpine AS build 2 | 3 | WORKDIR /src 4 | 5 | RUN apk --no-cache add git gcc make musl-dev curl bash openssh-client 6 | 7 | ENV \ 8 | KUBECTL_VERSION=v1.33.0 \ 9 | KUSTOMIZE_VERSION=v5.5.0 \ 10 | STRONGBOX_VERSION=2.0.0-RC4 11 | 12 | RUN os=$(go env GOOS) && arch=$(go env GOARCH) \ 13 | && curl -Ls -o /usr/local/bin/kubectl https://dl.k8s.io/${KUBECTL_VERSION}/bin/${os}/${arch}/kubectl \ 14 | && chmod +x /usr/local/bin/kubectl \ 15 | && curl -Ls https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_${os}_${arch}.tar.gz \ 16 | | tar xz -C /usr/local/bin/ \ 17 | && chmod +x /usr/local/bin/kustomize \ 18 | && curl -Ls -o /usr/local/bin/strongbox https://github.com/uw-labs/strongbox/releases/download/v${STRONGBOX_VERSION}/strongbox_${STRONGBOX_VERSION}_${os}_${arch} \ 19 | && chmod +x /usr/local/bin/strongbox \ 20 | && strongbox -git-config 21 | 22 | COPY go.mod go.sum /src/ 23 | RUN go mod download 24 | 25 | COPY . /src 26 | RUN go get -t ./... \ 27 | && make test \ 28 | && CGO_ENABLED=0 && go build -o /kube-applier . 29 | 30 | FROM alpine:3.20 31 | RUN apk --no-cache add git openssh-client tini 32 | COPY templates/ /templates/ 33 | COPY static/ /static/ 34 | COPY --from=build \ 35 | /usr/local/bin/kubectl \ 36 | /usr/local/bin/kustomize \ 37 | /usr/local/bin/strongbox \ 38 | /usr/local/bin/ 39 | COPY --from=build /root/.gitconfig /root/.gitconfig 40 | COPY --from=build /kube-applier /kube-applier 41 | ENTRYPOINT ["/sbin/tini", "--"] 42 | CMD [ "/kube-applier" ] 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | "License" shall mean the terms and conditions for use, reproduction, 9 | and distribution as defined by Sections 1 through 9 of this document. 10 | "Licensor" shall mean the copyright owner or entity authorized by 11 | the copyright owner that is granting the License. 12 | "Legal Entity" shall mean the union of the acting entity and all 13 | other entities that control, are controlled by, or are under common 14 | control with that entity. For the purposes of this definition, 15 | "control" means (i) the power, direct or indirect, to cause the 16 | direction or management of such entity, whether by contract or 17 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 18 | outstanding shares, or (iii) beneficial ownership of such entity. 19 | "You" (or "Your") shall mean an individual or Legal Entity 20 | exercising permissions granted by this License. 21 | "Source" form shall mean the preferred form for making modifications, 22 | including but not limited to software source code, documentation 23 | source, and configuration files. 24 | "Object" form shall mean any form resulting from mechanical 25 | transformation or translation of a Source form, including but 26 | not limited to compiled object code, generated documentation, 27 | and conversions to other media types. 28 | "Work" shall mean the work of authorship, whether in Source or 29 | Object form, made available under the License, as indicated by a 30 | copyright notice that is included in or attached to the work 31 | (an example is provided in the Appendix below). 32 | "Derivative Works" shall mean any work, whether in Source or Object 33 | form, that is based on (or derived from) the Work and for which the 34 | editorial revisions, annotations, elaborations, or other modifications 35 | represent, as a whole, an original work of authorship. For the purposes 36 | of this License, Derivative Works shall not include works that remain 37 | separable from, or merely link (or bind by name) to the interfaces of, 38 | the Work and Derivative Works thereof. 39 | "Contribution" shall mean any work of authorship, including 40 | the original version of the Work and any modifications or additions 41 | to that Work or Derivative Works thereof, that is intentionally 42 | submitted to Licensor for inclusion in the Work by the copyright owner 43 | or by an individual or Legal Entity authorized to submit on behalf of 44 | the copyright owner. For the purposes of this definition, "submitted" 45 | means any form of electronic, verbal, or written communication sent 46 | to the Licensor or its representatives, including but not limited to 47 | communication on electronic mailing lists, source code control systems, 48 | and issue tracking systems that are managed by, or on behalf of, the 49 | Licensor for the purpose of discussing and improving the Work, but 50 | excluding communication that is conspicuously marked or otherwise 51 | designated in writing by the copyright owner as "Not a Contribution." 52 | "Contributor" shall mean Licensor and any individual or Legal Entity 53 | on behalf of whom a Contribution has been received by Licensor and 54 | subsequently incorporated within the Work. 55 | 56 | 2. Grant of Copyright License. Subject to the terms and conditions of 57 | this License, each Contributor hereby grants to You a perpetual, 58 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 59 | copyright license to reproduce, prepare Derivative Works of, 60 | publicly display, publicly perform, sublicense, and distribute the 61 | Work and such Derivative Works in Source or Object form. 62 | 63 | 3. Grant of Patent License. Subject to the terms and conditions of 64 | this License, each Contributor hereby grants to You a perpetual, 65 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 66 | (except as stated in this section) patent license to make, have made, 67 | use, offer to sell, sell, import, and otherwise transfer the Work, 68 | where such license applies only to those patent claims licensable 69 | by such Contributor that are necessarily infringed by their 70 | Contribution(s) alone or by combination of their Contribution(s) 71 | with the Work to which such Contribution(s) was submitted. If You 72 | institute patent litigation against any entity (including a 73 | cross-claim or counterclaim in a lawsuit) alleging that the Work 74 | or a Contribution incorporated within the Work constitutes direct 75 | or contributory patent infringement, then any patent licenses 76 | granted to You under this License for that Work shall terminate 77 | as of the date such litigation is filed. 78 | 79 | 4. Redistribution. You may reproduce and distribute copies of the 80 | Work or Derivative Works thereof in any medium, with or without 81 | modifications, and in Source or Object form, provided that You 82 | meet the following conditions: 83 | 84 | (a) You must give any other recipients of the Work or 85 | Derivative Works a copy of this License; and 86 | 87 | (b) You must cause any modified files to carry prominent notices 88 | stating that You changed the files; and 89 | 90 | (c) You must retain, in the Source form of any Derivative Works 91 | that You distribute, all copyright, patent, trademark, and 92 | attribution notices from the Source form of the Work, 93 | excluding those notices that do not pertain to any part of 94 | the Derivative Works; and 95 | 96 | (d) If the Work includes a "NOTICE" text file as part of its 97 | distribution, then any Derivative Works that You distribute must 98 | include a readable copy of the attribution notices contained 99 | within such NOTICE file, excluding those notices that do not 100 | pertain to any part of the Derivative Works, in at least one 101 | of the following places: within a NOTICE text file distributed 102 | as part of the Derivative Works; within the Source form or 103 | documentation, if provided along with the Derivative Works; or, 104 | within a display generated by the Derivative Works, if and 105 | wherever such third-party notices normally appear. The contents 106 | of the NOTICE file are for informational purposes only and 107 | do not modify the License. You may add Your own attribution 108 | notices within Derivative Works that You distribute, alongside 109 | or as an addendum to the NOTICE text from the Work, provided 110 | that such additional attribution notices cannot be construed 111 | as modifying the License. 112 | 113 | You may add Your own copyright statement to Your modifications and 114 | may provide additional or different license terms and conditions 115 | for use, reproduction, or distribution of Your modifications, or 116 | for any such Derivative Works as a whole, provided Your use, 117 | reproduction, and distribution of the Work otherwise complies with 118 | the conditions stated in this License. 119 | 120 | 5. Submission of Contributions. Unless You explicitly state otherwise, 121 | any Contribution intentionally submitted for inclusion in the Work 122 | by You to the Licensor shall be under the terms and conditions of 123 | this License, without any additional terms or conditions. 124 | Notwithstanding the above, nothing herein shall supersede or modify 125 | the terms of any separate license agreement you may have executed 126 | with Licensor regarding such Contributions. 127 | 128 | 6. Trademarks. This License does not grant permission to use the trade 129 | names, trademarks, service marks, or product names of the Licensor, 130 | except as required for reasonable and customary use in describing the 131 | origin of the Work and reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. Unless required by applicable law or 134 | agreed to in writing, Licensor provides the Work (and each 135 | Contributor provides its Contributions) on an "AS IS" BASIS, 136 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 137 | implied, including, without limitation, any warranties or conditions 138 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 139 | PARTICULAR PURPOSE. You are solely responsible for determining the 140 | appropriateness of using or redistributing the Work and assume any 141 | risks associated with Your exercise of permissions under this License. 142 | 143 | 8. Limitation of Liability. In no event and under no legal theory, 144 | whether in tort (including negligence), contract, or otherwise, 145 | unless required by applicable law (such as deliberate and grossly 146 | negligent acts) or agreed to in writing, shall any Contributor be 147 | liable to You for damages, including any direct, indirect, special, 148 | incidental, or consequential damages of any character arising as a 149 | result of this License or out of the use or inability to use the 150 | Work (including but not limited to damages for loss of goodwill, 151 | work stoppage, computer failure or malfunction, or any and all 152 | other commercial damages or losses), even if such Contributor 153 | has been advised of the possibility of such damages. 154 | 155 | 9. Accepting Warranty or Additional Liability. While redistributing 156 | the Work or Derivative Works thereof, You may choose to offer, 157 | and charge a fee for, acceptance of support, warranty, indemnity, 158 | or other liability obligations and/or rights consistent with this 159 | License. However, in accepting such obligations, You may act only 160 | on Your own behalf and on Your sole responsibility, not on behalf 161 | of any other Contributor, and only if You agree to indemnify, 162 | defend, and hold each Contributor harmless for any liability 163 | incurred by, or claims asserted against, such Contributor by reason 164 | of your accepting any such warranty or additional liability. 165 | 166 | END OF TERMS AND CONDITIONS 167 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | IMAGE := quay.io/utilitywarehouse/kube-applier 3 | 4 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 5 | ifeq (,$(shell go env GOBIN)) 6 | GOBIN=$(shell go env GOPATH)/bin 7 | else 8 | GOBIN=$(shell go env GOBIN) 9 | endif 10 | 11 | .PHONY: manifests generate controller-gen-install test build run release 12 | 13 | # Generate manifests e.g. CRD, RBAC etc. 14 | manifests: controller-gen-install 15 | controller-gen \ 16 | crd:crdVersions=v1 \ 17 | paths="./..." \ 18 | output:crd:artifacts:config=manifests/base/cluster 19 | @{ \ 20 | cd manifests/base/cluster ;\ 21 | kustomize edit add resource kube-applier.io_* ;\ 22 | } 23 | 24 | # Generate code 25 | generate: controller-gen-install 26 | controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..." 27 | 28 | # Make sure controller-gen is installed. This should build and install packages 29 | # in module-aware mode, ignoring any local go.mod file 30 | controller-gen-install: 31 | go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3 32 | 33 | KUBEBUILDER_BINDIR=$${PWD}/kubebuilder-bindir 34 | KUBEBUILDER_VERSION="1.30.x" 35 | test: 36 | command -v setup-envtest || go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest 37 | mkdir -p $(KUBEBUILDER_BINDIR) 38 | setup-envtest --bin-dir $(KUBEBUILDER_BINDIR) use -p env $(KUBEBUILDER_VERSION) 39 | source <(setup-envtest --bin-dir $(KUBEBUILDER_BINDIR) use -i -p env $(KUBEBUILDER_VERSION)); CGO_ENABLED=1; go test -v -race -count=1 -cover ./... 40 | 41 | build: 42 | docker build -t kube-applier . 43 | 44 | BJS_VERSION="5.1.0" 45 | update-bootstrap-js: 46 | (cd /tmp/ && curl -L -O https://github.com/twbs/bootstrap/releases/download/v$(BJS_VERSION)/bootstrap-$(BJS_VERSION)-dist.zip) 47 | (cd /tmp/ && unzip bootstrap-$(BJS_VERSION)-dist.zip) 48 | cp /tmp/bootstrap-$(BJS_VERSION)-dist/js/bootstrap.js static/bootstrap/js/bootstrap.js 49 | 50 | update-jquery-js: 51 | curl -o static/bootstrap/js/jquery.min.js https://code.jquery.com/jquery-3.6.0.min.js 52 | 53 | release: 54 | @sd "$(IMAGE):master" "$(IMAGE):$(VERSION)" $$(rg -l -- $(IMAGE) manifests/) 55 | @git add -- manifests/ 56 | @git commit -m "Release $(VERSION)" 57 | @sd "$(IMAGE):$(VERSION)" "$(IMAGE):master" $$(rg -l -- "$(IMAGE)" manifests/) 58 | @git add -- manifests/ 59 | @git commit -m "Clean up release $(VERSION)" 60 | -------------------------------------------------------------------------------- /apis/kubeapplier/doc.go: -------------------------------------------------------------------------------- 1 | package kubeapplier 2 | 3 | // GroupName is the group name used in this package 4 | const ( 5 | GroupName = "kube-applier.io" 6 | ) 7 | -------------------------------------------------------------------------------- /apis/kubeapplier/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Package v1alpha1 contains API Schema definitions for the kube-applier v1alpha1 API group 2 | // +kubebuilder:object:generate=true 3 | // +groupName=kube-applier.io 4 | package v1alpha1 5 | 6 | import ( 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "sigs.k8s.io/controller-runtime/pkg/scheme" 9 | 10 | kubeapplier "github.com/utilitywarehouse/kube-applier/apis/kubeapplier" 11 | ) 12 | 13 | var ( 14 | // GroupVersion is group version used to register these objects 15 | GroupVersion = schema.GroupVersion{Group: kubeapplier.GroupName, Version: "v1alpha1"} 16 | 17 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 18 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 19 | 20 | // AddToScheme adds the types in this group-version to the given scheme. 21 | AddToScheme = SchemeBuilder.AddToScheme 22 | ) 23 | -------------------------------------------------------------------------------- /apis/kubeapplier/v1alpha1/meta_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | // ObjectReference is a reference to an object with a given name, in a given 4 | // namespace. If Namespace is not specified, it implies the same namespace as 5 | // the Waybill itself. 6 | type ObjectReference struct { 7 | // Name of the resource being referred to. 8 | // +required 9 | Name string `json:"name"` 10 | 11 | // Namespace of the resource being referred to. 12 | // +optional 13 | Namespace string `json:"namespace,omitempty"` 14 | } 15 | -------------------------------------------------------------------------------- /apis/kubeapplier/v1alpha1/waybill_types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // WaybillSpec defines the desired state of Waybill 8 | type WaybillSpec struct { 9 | // AutoApply determines whether this Waybill will be automatically applied 10 | // by scheduled or polling runs. 11 | // +optional 12 | // +kubebuilder:default=true 13 | AutoApply *bool `json:"autoApply,omitempty"` 14 | 15 | // DelegateServiceAccountSecretRef references a Secret of type 16 | // kubernetes.io/service-account-token in the same namespace as the Waybill 17 | // that will be passed by kube-applier to kubectl when performing apply 18 | // runs. 19 | // +optional 20 | // +kubebuilder:default=kube-applier-delegate-token 21 | // +kubebuilder:validation:MinLength=1 22 | DelegateServiceAccountSecretRef string `json:"delegateServiceAccountSecretRef,omitempty"` 23 | 24 | // DryRun enables the dry-run flag when applying this Waybill. 25 | // +optional 26 | // +kubebuilder:default=false 27 | DryRun bool `json:"dryRun,omitempty"` 28 | 29 | // GitSSHSecretRef will override the default Git SSH key passed as a 30 | // flag. It references a Secret that contains an item named `key` and 31 | // optionally an item named `known_hosts`. If present, these are passed to 32 | // the apply runtime and are used by `kustomize` when cloning remote bases. 33 | // This allows the use of bases from private repositories that the default 34 | // key will not have access to. 35 | // +optional 36 | GitSSHSecretRef *ObjectReference `json:"gitSSHSecretRef,omitempty"` 37 | 38 | // Prune determines whether pruning is enabled for this Waybill. 39 | // +optional 40 | // +kubebuilder:default=true 41 | Prune *bool `json:"prune,omitempty"` 42 | 43 | // PruneClusterResources determines whether pruning is enabled for cluster 44 | // resources, as part of this Waybill. 45 | // +optional 46 | // +kubebuilder:default=false 47 | PruneClusterResources bool `json:"pruneClusterResources,omitempty"` 48 | 49 | // PruneBlacklist can be used to specify a list of resources that are exempt 50 | // from pruning. 51 | // +optional 52 | PruneBlacklist []string `json:"pruneBlacklist,omitempty"` 53 | 54 | // RepositoryPath defines the relative path inside the Repository where the 55 | // configuration for this Waybill is stored. Accepted values are absolute 56 | // or relative paths (relative to the root of the repository), such as: 57 | // 'foo', '/foo', 'foo/bar', '/foo/bar' etc., as well as an empty string. 58 | // If not specified, it will default to the name of the namespace where the 59 | // Waybill is created. 60 | // +optional 61 | // +kubebuilder:validation:Pattern=^(\/?[a-zA-Z0-9.\_\-]+(\/[a-zA-Z0-9.\_\-]+)*\/?)?$ 62 | RepositoryPath string `json:"repositoryPath"` 63 | 64 | // RunInterval determines how often this Waybill is applied in seconds. 65 | // +optional 66 | // +kubebuilder:default=3600 67 | RunInterval int `json:"runInterval,omitempty"` 68 | 69 | // RunTimeout specifies the timeout for performing an apply run. 70 | // +optional 71 | // +kubebuilder:default=900 72 | RunTimeout int `json:"runTimeout,omitempty"` 73 | 74 | // ServerSideApply determines whether the server-side apply flag is enabled 75 | // for this Waybill. 76 | // +optional 77 | // +kubebuilder:default=false 78 | ServerSideApply bool `json:"serverSideApply,omitempty"` 79 | 80 | // StrongboxKeyringSecretRef references a Secret that contains an item named 81 | // '.strongbox_keyring' with any strongbox keys required to decrypt the 82 | // files before applying. See the strongbox documentation for the format of 83 | // the keyring data. 84 | // +optional 85 | StrongboxKeyringSecretRef *ObjectReference `json:"strongboxKeyringSecretRef,omitempty"` 86 | } 87 | 88 | // WaybillStatus defines the observed state of Waybill 89 | type WaybillStatus struct { 90 | // LastRun contains the last apply run's information. 91 | // +nullable 92 | // +optional 93 | LastRun *WaybillStatusRun `json:"lastRun,omitempty"` 94 | } 95 | 96 | // WaybillStatusRun contains information about an apply run of a Waybill 97 | // resource. 98 | type WaybillStatusRun struct { 99 | // Command is the command used during the apply run. 100 | Command string `json:"command"` 101 | 102 | // Commit is the git commit hash on which this apply run operated. 103 | Commit string `json:"commit"` 104 | 105 | // ErrorMessage describes any errors that occured during the apply run. 106 | ErrorMessage string `json:"errorMessage"` 107 | 108 | // Finished is the time that the apply run finished applying this Waybill. 109 | Finished metav1.Time `json:"finished"` 110 | 111 | // Output is the stdout of the Command. 112 | Output string `json:"output"` 113 | 114 | // Started is the time that the apply run started applying this Waybill. 115 | Started metav1.Time `json:"started"` 116 | 117 | // Success denotes whether the apply run was successful or not. 118 | Success bool `json:"success"` 119 | 120 | // Type is a short description of the kind of apply run that was attempted. 121 | // +kubebuilder:default="unknown" 122 | Type string `json:"type"` 123 | } 124 | 125 | // +kubebuilder:object:root=true 126 | 127 | // Waybill is the Schema for the Waybills API of kube-applier. A Waybill is 128 | // defined as a namespace associated with a path in a remote git repository 129 | // where kubernetes configuration is stored. 130 | // +kubebuilder:resource:shortName=wb;wbs 131 | // +kubebuilder:subresource:status 132 | // +kubebuilder:printcolumn:name="Success",type=boolean,JSONPath=`.status.lastRun.success` 133 | // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.lastRun.type` 134 | // +kubebuilder:printcolumn:name="Commit",type=string,JSONPath=`.status.lastRun.commit` 135 | // +kubebuilder:printcolumn:name="Last Applied",type=date,JSONPath=`.status.lastRun.finished` 136 | // +kubebuilder:printcolumn:name="Auto Apply",type=boolean,JSONPath=`.spec.autoApply`,priority=10 137 | // +kubebuilder:printcolumn:name="Dry Run",type=boolean,JSONPath=`.spec.dryRun`,priority=10 138 | // +kubebuilder:printcolumn:name="Prune",type=boolean,JSONPath=`.spec.prune`,priority=10 139 | // +kubebuilder:printcolumn:name="Run Interval",type=number,JSONPath=`.spec.runInterval`,priority=10 140 | // +kubebuilder:printcolumn:name="Repository Path",type=string,JSONPath=`.spec.repositoryPath`,priority=20 141 | type Waybill struct { 142 | metav1.TypeMeta `json:",inline"` 143 | metav1.ObjectMeta `json:"metadata,omitempty"` 144 | 145 | // +kubebuilder:default={autoApply:true} 146 | // +optional 147 | Spec WaybillSpec `json:"spec,omitempty"` 148 | Status WaybillStatus `json:"status,omitempty"` 149 | } 150 | 151 | // +kubebuilder:object:root=true 152 | 153 | // WaybillList contains a list of Waybill 154 | type WaybillList struct { 155 | metav1.TypeMeta `json:",inline"` 156 | metav1.ListMeta `json:"metadata,omitempty"` 157 | 158 | Items []Waybill `json:"items"` 159 | } 160 | 161 | func init() { 162 | SchemeBuilder.Register(&Waybill{}, &WaybillList{}) 163 | } 164 | -------------------------------------------------------------------------------- /apis/kubeapplier/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by controller-gen. DO NOT EDIT. 5 | 6 | package v1alpha1 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { 14 | *out = *in 15 | } 16 | 17 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReference. 18 | func (in *ObjectReference) DeepCopy() *ObjectReference { 19 | if in == nil { 20 | return nil 21 | } 22 | out := new(ObjectReference) 23 | in.DeepCopyInto(out) 24 | return out 25 | } 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Waybill) DeepCopyInto(out *Waybill) { 29 | *out = *in 30 | out.TypeMeta = in.TypeMeta 31 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 32 | in.Spec.DeepCopyInto(&out.Spec) 33 | in.Status.DeepCopyInto(&out.Status) 34 | } 35 | 36 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Waybill. 37 | func (in *Waybill) DeepCopy() *Waybill { 38 | if in == nil { 39 | return nil 40 | } 41 | out := new(Waybill) 42 | in.DeepCopyInto(out) 43 | return out 44 | } 45 | 46 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 47 | func (in *Waybill) DeepCopyObject() runtime.Object { 48 | if c := in.DeepCopy(); c != nil { 49 | return c 50 | } 51 | return nil 52 | } 53 | 54 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 55 | func (in *WaybillList) DeepCopyInto(out *WaybillList) { 56 | *out = *in 57 | out.TypeMeta = in.TypeMeta 58 | in.ListMeta.DeepCopyInto(&out.ListMeta) 59 | if in.Items != nil { 60 | in, out := &in.Items, &out.Items 61 | *out = make([]Waybill, len(*in)) 62 | for i := range *in { 63 | (*in)[i].DeepCopyInto(&(*out)[i]) 64 | } 65 | } 66 | } 67 | 68 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaybillList. 69 | func (in *WaybillList) DeepCopy() *WaybillList { 70 | if in == nil { 71 | return nil 72 | } 73 | out := new(WaybillList) 74 | in.DeepCopyInto(out) 75 | return out 76 | } 77 | 78 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 79 | func (in *WaybillList) DeepCopyObject() runtime.Object { 80 | if c := in.DeepCopy(); c != nil { 81 | return c 82 | } 83 | return nil 84 | } 85 | 86 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 87 | func (in *WaybillSpec) DeepCopyInto(out *WaybillSpec) { 88 | *out = *in 89 | if in.AutoApply != nil { 90 | in, out := &in.AutoApply, &out.AutoApply 91 | *out = new(bool) 92 | **out = **in 93 | } 94 | if in.GitSSHSecretRef != nil { 95 | in, out := &in.GitSSHSecretRef, &out.GitSSHSecretRef 96 | *out = new(ObjectReference) 97 | **out = **in 98 | } 99 | if in.Prune != nil { 100 | in, out := &in.Prune, &out.Prune 101 | *out = new(bool) 102 | **out = **in 103 | } 104 | if in.PruneBlacklist != nil { 105 | in, out := &in.PruneBlacklist, &out.PruneBlacklist 106 | *out = make([]string, len(*in)) 107 | copy(*out, *in) 108 | } 109 | if in.StrongboxKeyringSecretRef != nil { 110 | in, out := &in.StrongboxKeyringSecretRef, &out.StrongboxKeyringSecretRef 111 | *out = new(ObjectReference) 112 | **out = **in 113 | } 114 | } 115 | 116 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaybillSpec. 117 | func (in *WaybillSpec) DeepCopy() *WaybillSpec { 118 | if in == nil { 119 | return nil 120 | } 121 | out := new(WaybillSpec) 122 | in.DeepCopyInto(out) 123 | return out 124 | } 125 | 126 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 127 | func (in *WaybillStatus) DeepCopyInto(out *WaybillStatus) { 128 | *out = *in 129 | if in.LastRun != nil { 130 | in, out := &in.LastRun, &out.LastRun 131 | *out = new(WaybillStatusRun) 132 | (*in).DeepCopyInto(*out) 133 | } 134 | } 135 | 136 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaybillStatus. 137 | func (in *WaybillStatus) DeepCopy() *WaybillStatus { 138 | if in == nil { 139 | return nil 140 | } 141 | out := new(WaybillStatus) 142 | in.DeepCopyInto(out) 143 | return out 144 | } 145 | 146 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 147 | func (in *WaybillStatusRun) DeepCopyInto(out *WaybillStatusRun) { 148 | *out = *in 149 | in.Finished.DeepCopyInto(&out.Finished) 150 | in.Started.DeepCopyInto(&out.Started) 151 | } 152 | 153 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaybillStatusRun. 154 | func (in *WaybillStatusRun) DeepCopy() *WaybillStatusRun { 155 | if in == nil { 156 | return nil 157 | } 158 | out := new(WaybillStatusRun) 159 | in.DeepCopyInto(out) 160 | return out 161 | } 162 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | // Package client defines a custom kubernetes Client for use with kube-applier 2 | // and its custom resources. 3 | package client 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "log" 9 | "slices" 10 | "sort" 11 | "strings" 12 | 13 | authorizationv1 "k8s.io/api/authorization/v1" 14 | corev1 "k8s.io/api/core/v1" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/apimachinery/pkg/fields" 17 | "k8s.io/apimachinery/pkg/runtime" 18 | "k8s.io/apimachinery/pkg/runtime/schema" 19 | "k8s.io/client-go/kubernetes" 20 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 21 | "k8s.io/client-go/rest" 22 | "sigs.k8s.io/controller-runtime/pkg/cache" 23 | "sigs.k8s.io/controller-runtime/pkg/client" 24 | "sigs.k8s.io/controller-runtime/pkg/client/config" 25 | "sigs.k8s.io/controller-runtime/pkg/cluster" 26 | 27 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 28 | // +kubebuilder:scaffold:imports 29 | // For local dev 30 | //_ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 31 | ) 32 | 33 | const ( 34 | // Name identifies this client and is used for ownership-related fields. 35 | Name = "kube-applier" 36 | ) 37 | 38 | var ( 39 | scheme = runtime.NewScheme() 40 | 41 | defaultUpdateOptions = &client.UpdateOptions{FieldManager: "kube-applier"} 42 | ) 43 | 44 | func init() { 45 | if err := clientgoscheme.AddToScheme(scheme); err != nil { 46 | log.Fatalf("Cannot setup client scheme: %v", err) 47 | } 48 | 49 | if err := kubeapplierv1alpha1.AddToScheme(scheme); err != nil { 50 | log.Fatalf("Cannot setup client scheme: %v", err) 51 | } 52 | // +kubebuilder:scaffold:scheme 53 | } 54 | 55 | // Client encapsulates a kubernetes client for interacting with the apiserver. 56 | type Client struct { 57 | cluster.Cluster 58 | clientset kubernetes.Interface 59 | shutdown func() 60 | } 61 | 62 | // New returns a new kubernetes client. 63 | func New(opts ...cluster.Option) (*Client, error) { 64 | cfg, err := config.GetConfig() 65 | if err != nil { 66 | return nil, fmt.Errorf("Cannot get kubernetes config: %v", err) 67 | } 68 | return newClient(cfg, opts...) 69 | } 70 | 71 | // NewWithConfig returns a new kubernetes client initialised with the provided 72 | // configuration. 73 | func NewWithConfig(cfg *rest.Config, opts ...cluster.Option) (*Client, error) { 74 | return newClient(cfg, opts...) 75 | } 76 | 77 | func newClient(cfg *rest.Config, opts ...cluster.Option) (*Client, error) { 78 | c, err := cluster.New(cfg, func(options *cluster.Options) { 79 | options.Scheme = scheme 80 | options.NewCache = func(config *rest.Config, opts cache.Options) (cache.Cache, error) { 81 | // Only cache events for Waybills 82 | if opts.ByObject == nil { 83 | opts.ByObject = make(map[client.Object]cache.ByObject) 84 | } 85 | opts.ByObject[&corev1.Event{}] = cache.ByObject{ 86 | Field: fields.SelectorFromSet(fields.Set{"involvedObject.kind": "Waybill"}), 87 | } 88 | return cache.New(config, opts) 89 | } 90 | for _, opt := range opts { 91 | opt(options) 92 | } 93 | }) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | clientset, err := kubernetes.NewForConfig(cfg) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | ctx, shutdown := context.WithCancel(context.Background()) 104 | go c.Start(ctx) 105 | 106 | return &Client{ 107 | Cluster: c, 108 | clientset: clientset, 109 | shutdown: shutdown, 110 | }, nil 111 | } 112 | 113 | // Shutdown shuts down the client 114 | func (c *Client) Shutdown() { 115 | c.shutdown() 116 | } 117 | 118 | // CloneConfig copies the client's config into a new rest.Config, it does not 119 | // copy user credentials 120 | func (c *Client) CloneConfig() *rest.Config { 121 | return rest.AnonymousClientConfig(c.GetConfig()) 122 | } 123 | 124 | // EmitWaybillEvent creates an Event for the provided Waybill. 125 | func (c *Client) EmitWaybillEvent(waybill *kubeapplierv1alpha1.Waybill, eventType, reason, messageFmt string, args ...interface{}) { 126 | c.GetEventRecorderFor(Name).Eventf(waybill, eventType, reason, messageFmt, args...) 127 | } 128 | 129 | // HasAccess returns a boolean depending on whether the email address provided 130 | // corresponds to a user who has edit access to the specified Waybill. 131 | func (c *Client) HasAccess(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill, email, verb string) (bool, error) { 132 | gvk := waybill.GroupVersionKind() 133 | plural, err := c.pluralName(gvk) 134 | if err != nil { 135 | return false, err 136 | } 137 | response, err := c.clientset.AuthorizationV1().SubjectAccessReviews().Create( 138 | ctx, 139 | &authorizationv1.SubjectAccessReview{ 140 | Spec: authorizationv1.SubjectAccessReviewSpec{ 141 | ResourceAttributes: &authorizationv1.ResourceAttributes{ 142 | Namespace: waybill.Namespace, 143 | Verb: verb, 144 | Group: gvk.Group, 145 | Version: gvk.Version, 146 | Resource: plural, 147 | }, 148 | User: email, 149 | }, 150 | }, 151 | metav1.CreateOptions{}, 152 | ) 153 | if err != nil { 154 | return false, err 155 | } 156 | return response.Status.Allowed, nil 157 | } 158 | 159 | // ListWaybillEvents returns a list of Waybill events. The events are sorted by 160 | // the LastTimestamp field. 161 | func (c *Client) ListWaybillEvents(ctx context.Context) ([]corev1.Event, error) { 162 | eventList := &corev1.EventList{} 163 | if err := c.GetClient().List(ctx, eventList); err != nil { 164 | return nil, err 165 | } 166 | 167 | var events []corev1.Event 168 | for _, e := range eventList.Items { 169 | if e.InvolvedObject.Kind == "Waybill" && e.InvolvedObject.GroupVersionKind().Group == kubeapplierv1alpha1.GroupVersion.Group { 170 | events = append(events, e) 171 | } 172 | } 173 | sort.Slice(events, func(i, j int) bool { 174 | return events[i].LastTimestamp.Before(&events[j].LastTimestamp) 175 | }) 176 | 177 | return events, nil 178 | } 179 | 180 | // ListWaybills returns a list of all the Waybill resources. 181 | func (c *Client) ListWaybills(ctx context.Context) ([]kubeapplierv1alpha1.Waybill, error) { 182 | waybills := &kubeapplierv1alpha1.WaybillList{} 183 | if err := c.GetClient().List(ctx, waybills); err != nil { 184 | return nil, err 185 | } 186 | // ensure that the list of Waybills is sorted alphabetically 187 | sortedWaybills := make([]kubeapplierv1alpha1.Waybill, len(waybills.Items)) 188 | for i, wb := range waybills.Items { 189 | sortedWaybills[i] = wb 190 | } 191 | sort.Slice(sortedWaybills, func(i, j int) bool { 192 | return sortedWaybills[i].Namespace+sortedWaybills[i].Name < sortedWaybills[j].Namespace+sortedWaybills[j].Name 193 | }) 194 | 195 | unique := map[string]kubeapplierv1alpha1.Waybill{} 196 | for _, wb := range sortedWaybills { 197 | if v, ok := unique[wb.Namespace]; ok { 198 | c.EmitWaybillEvent(&wb, corev1.EventTypeWarning, "MultipleWaybillsFound", "Waybill %s is already being used", v.Name) 199 | continue 200 | } 201 | unique[wb.Namespace] = wb 202 | } 203 | ret := make([]kubeapplierv1alpha1.Waybill, len(unique)) 204 | i := 0 205 | for _, wb := range unique { 206 | ret[i] = wb 207 | i++ 208 | } 209 | sort.Slice(ret, func(i, j int) bool { 210 | return ret[i].Namespace < ret[j].Namespace 211 | }) 212 | return ret, nil 213 | } 214 | 215 | // GetWaybill returns the Waybill resource specified by the namespace 216 | // and name. 217 | func (c *Client) GetWaybill(ctx context.Context, namespace, name string) (*kubeapplierv1alpha1.Waybill, error) { 218 | waybill := &kubeapplierv1alpha1.Waybill{} 219 | if err := c.GetClient().Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, waybill); err != nil { 220 | return nil, err 221 | } 222 | return waybill, nil 223 | } 224 | 225 | // UpdateWaybill updates the Waybill resource provided. 226 | func (c *Client) UpdateWaybill(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill) error { 227 | return c.GetClient().Update(ctx, waybill, defaultUpdateOptions) 228 | } 229 | 230 | // UpdateWaybillStatus updates the status of the Waybill resource 231 | // provided. 232 | func (c *Client) UpdateWaybillStatus(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill) error { 233 | return c.GetClient().SubResource("status").Update(ctx, waybill, &client.SubResourceUpdateOptions{UpdateOptions: *defaultUpdateOptions}) 234 | } 235 | 236 | // GetSecret returns the Secret resource specified by the namespace and name. 237 | func (c *Client) GetSecret(ctx context.Context, namespace, name string) (*corev1.Secret, error) { 238 | secret := &corev1.Secret{} 239 | // Use the APIReader to bypass the cache and get secrets directly from 240 | // the API server. 241 | // 242 | // If it used the cache then it would cache ALL secrets, 243 | // which is a bit of an overreach given that we're only interested in 244 | // specific secrets. 245 | if err := c.GetAPIReader().Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, secret); err != nil { 246 | return nil, err 247 | } 248 | return secret, nil 249 | } 250 | 251 | // PrunableResourceGVKs returns the cluster and namespaced resources that the 252 | // client can prune as two slices of strings of the format 253 | // //. 254 | func (c *Client) PrunableResourceGVKs(ctx context.Context, namespace string) ([]string, []string, error) { 255 | var cluster, namespaced []string 256 | 257 | _, resourceList, err := c.clientset.Discovery().ServerGroupsAndResources() 258 | if err != nil { 259 | return cluster, namespaced, err 260 | } 261 | 262 | srr := &authorizationv1.SelfSubjectRulesReview{ 263 | Spec: authorizationv1.SelfSubjectRulesReviewSpec{ 264 | Namespace: namespace, 265 | }, 266 | } 267 | reviewResp, err := c.clientset.AuthorizationV1().SelfSubjectRulesReviews().Create(ctx, srr, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}}) 268 | if err != nil { 269 | return cluster, namespaced, err 270 | } 271 | for _, l := range resourceList { 272 | groupVersion := l.GroupVersion 273 | if groupVersion == "v1" { 274 | groupVersion = "core/v1" 275 | } 276 | gv, err := schema.ParseGroupVersion(l.GroupVersion) 277 | if err != nil { 278 | return cluster, namespaced, err 279 | } 280 | 281 | for _, r := range l.APIResources { 282 | gvk := groupVersion + "/" + r.Kind 283 | if prunable(r) && rulesAllowPrune(reviewResp.Status.ResourceRules, gv.Group, r.Name) { 284 | if r.Namespaced { 285 | namespaced = append(namespaced, gvk) 286 | } else { 287 | cluster = append(cluster, gvk) 288 | } 289 | } 290 | } 291 | } 292 | 293 | // resourceList no longer sorted and its a randomly sorted list on each call 294 | // this behaviour affects all the test based on this list 295 | slices.Sort(cluster) 296 | slices.Sort(namespaced) 297 | 298 | return cluster, namespaced, nil 299 | } 300 | 301 | // rulesAllowPrune checks whether the given resource rules allow pruning the 302 | // given group/resource 303 | func rulesAllowPrune(rules []authorizationv1.ResourceRule, group, resource string) bool { 304 | for _, verb := range []string{"get", "list", "delete"} { 305 | if !rulesAllowVerb(rules, group, resource, verb) { 306 | return false 307 | } 308 | } 309 | 310 | return true 311 | } 312 | 313 | // rulesAllowVerb checks whether the given resource rules allow 'verb' on the 314 | // given group/resource 315 | func rulesAllowVerb(rules []authorizationv1.ResourceRule, group, resource, verb string) bool { 316 | for _, rule := range rules { 317 | if !match(rule.APIGroups, group) { 318 | continue 319 | } 320 | 321 | if !match(rule.Resources, resource) { 322 | continue 323 | } 324 | 325 | if !match(rule.Verbs, verb) { 326 | continue 327 | } 328 | 329 | if !resourceNamesAllowAll(rule.ResourceNames) { 330 | continue 331 | } 332 | 333 | return true 334 | } 335 | 336 | return false 337 | } 338 | 339 | // match checks that the item is in the list of items, or if one of the 340 | // items in the list is a '*' 341 | func match(items []string, item string) bool { 342 | for _, i := range items { 343 | if i == "*" { 344 | return true 345 | } 346 | if i == item { 347 | return true 348 | } 349 | } 350 | 351 | return false 352 | } 353 | 354 | // resourceNamesAllowAll ensures that the given resource names allow all names. 355 | // We can't filter out resources by name so we can only prune resources if we 356 | // have access regardless of name. 357 | func resourceNamesAllowAll(names []string) bool { 358 | for _, n := range names { 359 | if n == "*" { 360 | return true 361 | } 362 | } 363 | return len(names) == 0 364 | } 365 | 366 | // prunable returns true if a resource can be deleted and isn't a subresource 367 | func prunable(r metav1.APIResource) bool { 368 | if !strings.Contains(r.Name, "/") { 369 | for _, v := range r.Verbs { 370 | if v == "delete" { 371 | return true 372 | } 373 | } 374 | } 375 | return false 376 | } 377 | 378 | // pluralName returns the plural name of a resource, if found in the API 379 | // resources 380 | func (c *Client) pluralName(gvk schema.GroupVersionKind) (string, error) { 381 | ar, err := c.clientset.Discovery().ServerResourcesForGroupVersion(gvk.GroupVersion().String()) 382 | if err != nil { 383 | return "", err 384 | } 385 | for _, r := range ar.APIResources { 386 | if r.Kind == gvk.Kind { 387 | return r.Name, nil 388 | } 389 | } 390 | return "", fmt.Errorf("api resource %s not found", gvk.String()) 391 | } 392 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | . "github.com/onsi/gomega/gstruct" 11 | gomegatypes "github.com/onsi/gomega/types" 12 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 13 | corev1 "k8s.io/api/core/v1" 14 | rbacv1 "k8s.io/api/rbac/v1" 15 | "k8s.io/apimachinery/pkg/api/errors" 16 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 17 | "sigs.k8s.io/controller-runtime/pkg/envtest" 18 | ) 19 | 20 | var _ = Describe("Client", func() { 21 | Context("When retrieving prunable resources", func() { 22 | It("Should only return resources that support delete and that the client has permissions to get/list/delete", func() { 23 | // Create a user and a client that can auth as the user 24 | user, err := testEnv.AddUser(envtest.User{Name: "foobar"}, testConfig) 25 | Expect(err).NotTo(HaveOccurred()) 26 | userKubeClient, err := NewWithConfig(user.Config()) 27 | Expect(err).NotTo(HaveOccurred()) 28 | defer userKubeClient.Shutdown() 29 | Expect(userKubeClient).ToNot(BeNil()) 30 | 31 | // Create a namespace for the user to manage 32 | if err := testKubeClient.GetClient().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foobar"}}); err != nil { 33 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 34 | } 35 | 36 | // Create a clusterrole that gives the user access to 37 | // various cluster/namespaced resources 38 | if err := testKubeClient.GetClient().Create(context.TODO(), &rbacv1.ClusterRole{ 39 | ObjectMeta: metav1.ObjectMeta{Name: "foobar"}, 40 | Rules: []rbacv1.PolicyRule{ 41 | { 42 | Verbs: []string{"*"}, 43 | APIGroups: []string{""}, 44 | Resources: []string{"pods"}, 45 | }, 46 | { 47 | Verbs: []string{"*"}, 48 | APIGroups: []string{""}, 49 | Resources: []string{"namespaces"}, 50 | }, 51 | { 52 | Verbs: []string{"get", "list", "delete"}, 53 | APIGroups: []string{"storage.k8s.io"}, 54 | Resources: []string{"storageclasses"}, 55 | }, 56 | { 57 | Verbs: []string{"get", "list", "delete"}, 58 | APIGroups: []string{"apps"}, 59 | Resources: []string{"deployments"}, 60 | }, 61 | // Not prunable: get, list and delete 62 | // permissions are required to prune a 63 | // resource 64 | { 65 | Verbs: []string{"delete"}, 66 | APIGroups: []string{""}, 67 | Resources: []string{"serviceaccounts"}, 68 | }, 69 | // Not prunable: pruning individual 70 | // resources by name isn't possible, so 71 | // we can't support specific 72 | // ResourceNames 73 | { 74 | Verbs: []string{"*"}, 75 | APIGroups: []string{""}, 76 | Resources: []string{"validatingwebhookconfigurations"}, 77 | ResourceNames: []string{"foobar"}, 78 | }, 79 | // Not prunable: bindings don't support 80 | // the 'delete' verb 81 | { 82 | Verbs: []string{"*"}, 83 | APIGroups: []string{""}, 84 | Resources: []string{"bindings"}, 85 | }, 86 | }}); err != nil { 87 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 88 | } 89 | if err := testKubeClient.GetClient().Create(context.TODO(), &rbacv1.RoleBinding{ 90 | ObjectMeta: metav1.ObjectMeta{Name: "foobar", Namespace: "foobar"}, 91 | Subjects: []rbacv1.Subject{ 92 | { 93 | Kind: "User", 94 | Name: "foobar", 95 | }, 96 | }, 97 | RoleRef: rbacv1.RoleRef{ 98 | APIGroup: "rbac.authorization.k8s.io", 99 | Kind: "ClusterRole", 100 | Name: "foobar", 101 | }}); err != nil { 102 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 103 | } 104 | 105 | // Ensure that only prunable resources are returned 106 | cluster, namespaced, err := userKubeClient.PrunableResourceGVKs(context.TODO(), "foobar") 107 | Expect(err).NotTo(HaveOccurred()) 108 | Expect(cluster).To(Equal([]string{ 109 | "core/v1/Namespace", 110 | "storage.k8s.io/v1/StorageClass", 111 | })) 112 | Expect(namespaced).To(Equal([]string{ 113 | "apps/v1/Deployment", 114 | "core/v1/Pod", 115 | })) 116 | }) 117 | }) 118 | Context("When listing waybills", func() { 119 | It("Should return only one Waybill per namespace and emit events for the others", func() { 120 | wbList := []kubeapplierv1alpha1.Waybill{ 121 | { 122 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 123 | ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "ns-0"}, 124 | }, 125 | { 126 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 127 | ObjectMeta: metav1.ObjectMeta{Name: "beta", Namespace: "ns-0"}, 128 | }, 129 | { 130 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 131 | ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns-1"}, 132 | }, 133 | } 134 | 135 | for i := range wbList { 136 | err := testKubeClient.GetClient().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wbList[i].Namespace}}) 137 | if err != nil { 138 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 139 | } 140 | Expect(testKubeClient.GetClient().Create(context.TODO(), &wbList[i])).To(BeNil()) 141 | } 142 | 143 | Eventually( 144 | func() int { 145 | waybills, err := testKubeClient.ListWaybills(context.TODO()) 146 | if err != nil { 147 | return -1 148 | } 149 | return len(waybills) 150 | }, 151 | time.Second*15, 152 | time.Second, 153 | ).Should(Equal(2)) 154 | 155 | events := &corev1.EventList{} 156 | Eventually( 157 | func() int { 158 | if err := testKubeClient.GetAPIReader().List(context.TODO(), events); err != nil { 159 | return -1 160 | } 161 | return len(events.Items) 162 | }, 163 | time.Second*15, 164 | time.Second, 165 | ).Should(Equal(1)) 166 | for _, e := range events.Items { 167 | Expect(e).To(matchEvent(wbList[1], corev1.EventTypeWarning, "MultipleWaybillsFound", fmt.Sprintf("^.*%s.*$", wbList[0].Name))) 168 | } 169 | 170 | Expect(testKubeClient.GetClient().Delete(context.TODO(), &events.Items[0])).To(BeNil()) 171 | }) 172 | }) 173 | Context("When listing events", func() { 174 | It("Should return all the Waybill events, ordered by timestamp", func() { 175 | wb := kubeapplierv1alpha1.Waybill{ 176 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 177 | ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "ns-0"}, 178 | } 179 | eventMessages := []string{"test1", "test2"} 180 | if err := testKubeClient.GetClient().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-0"}}); err != nil { 181 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 182 | } 183 | if err := testKubeClient.GetClient().Create(context.TODO(), &wb); err != nil { 184 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 185 | } 186 | for _, msg := range eventMessages { 187 | testKubeClient.EmitWaybillEvent(&wb, corev1.EventTypeWarning, "TestWaybillEvent", msg) 188 | //metav1.Time has second level precision when marshalling/unmarshalling 189 | time.Sleep(1 * time.Second) 190 | } 191 | eventList := &corev1.EventList{} 192 | Eventually( 193 | func() int { 194 | if err := testKubeClient.GetAPIReader().List(context.TODO(), eventList); err != nil { 195 | return -1 196 | } 197 | return len(eventList.Items) 198 | }, 199 | time.Second*15, 200 | time.Second, 201 | ).Should(Equal(2)) 202 | 203 | events, err := testKubeClient.ListWaybillEvents(context.TODO()) 204 | Expect(err).To(BeNil()) 205 | for i, e := range events { 206 | Expect(e).To(matchEvent(wb, corev1.EventTypeWarning, "TestWaybillEvent", eventMessages[i])) 207 | Expect(testKubeClient.GetClient().Delete(context.TODO(), &e)).To(BeNil()) 208 | } 209 | }) 210 | }) 211 | }) 212 | 213 | func matchEvent(waybill kubeapplierv1alpha1.Waybill, eventType, reason, message string) gomegatypes.GomegaMatcher { 214 | return MatchFields(IgnoreExtras, Fields{ 215 | "TypeMeta": Ignore(), 216 | "ObjectMeta": MatchFields(IgnoreExtras, Fields{ 217 | "Namespace": Equal(waybill.ObjectMeta.Namespace), 218 | }), 219 | "InvolvedObject": MatchFields(IgnoreExtras, Fields{ 220 | "Kind": Equal("Waybill"), 221 | "Namespace": Equal(waybill.ObjectMeta.Namespace), 222 | "Name": Equal(waybill.ObjectMeta.Name), 223 | }), 224 | "Action": BeEmpty(), 225 | "Count": BeNumerically(">", 0), 226 | "Message": MatchRegexp(message), 227 | "Reason": Equal(reason), 228 | "Source": MatchFields(IgnoreExtras, Fields{ 229 | "Component": Equal(Name), 230 | }), 231 | "Type": Equal(eventType), 232 | }) 233 | } 234 | -------------------------------------------------------------------------------- /client/suite_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | kubescheme "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "sigs.k8s.io/controller-runtime/pkg/envtest" 12 | logf "sigs.k8s.io/controller-runtime/pkg/log" 13 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 14 | 15 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 16 | "github.com/utilitywarehouse/kube-applier/log" 17 | // +kubebuilder:scaffold:imports 18 | ) 19 | 20 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 21 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 22 | 23 | var ( 24 | testConfig *rest.Config 25 | testKubeClient *Client 26 | testEnv *envtest.Environment 27 | ) 28 | 29 | func TestClient(t *testing.T) { 30 | RegisterFailHandler(Fail) 31 | 32 | RunSpecs(t, "Run package suite") 33 | } 34 | 35 | var _ = BeforeSuite(func() { 36 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 37 | 38 | By("bootstrapping test environment") 39 | testEnv = &envtest.Environment{ 40 | CRDDirectoryPaths: []string{filepath.Join("..", "manifests", "base", "cluster")}, 41 | } 42 | 43 | var err error 44 | testConfig, err = testEnv.Start() 45 | Expect(err).ToNot(HaveOccurred()) 46 | Expect(testConfig).ToNot(BeNil()) 47 | 48 | err = kubeapplierv1alpha1.AddToScheme(kubescheme.Scheme) 49 | Expect(err).NotTo(HaveOccurred()) 50 | 51 | // +kubebuilder:scaffold:scheme 52 | 53 | testKubeClient, err = NewWithConfig(testConfig) 54 | Expect(err).ToNot(HaveOccurred()) 55 | Expect(testKubeClient).ToNot(BeNil()) 56 | }, 60) 57 | 58 | var _ = AfterSuite(func() { 59 | By("tearing down the test environment") 60 | testKubeClient.Shutdown() 61 | err := testEnv.Stop() 62 | Expect(err).ToNot(HaveOccurred()) 63 | }) 64 | 65 | func init() { 66 | log.SetLevel("off") 67 | } 68 | -------------------------------------------------------------------------------- /git/repository.go: -------------------------------------------------------------------------------- 1 | // Package git provides methods for manipulating and querying git repositories 2 | // on disk. 3 | package git 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "errors" 9 | "fmt" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | "github.com/utilitywarehouse/kube-applier/log" 19 | "github.com/utilitywarehouse/kube-applier/metrics" 20 | ) 21 | 22 | var ( 23 | gitExecutablePath string 24 | ) 25 | 26 | func init() { 27 | gitExecutablePath = exec.Command("git").String() 28 | } 29 | 30 | // RepositoryConfig defines a remote git repository. 31 | type RepositoryConfig struct { 32 | Remote string 33 | Branch string 34 | Revision string 35 | Depth int 36 | } 37 | 38 | // SyncOptions encapsulates options about how a Repository should be fetched 39 | // from the remote. 40 | type SyncOptions struct { 41 | GitSSHKeyPath string 42 | GitSSHKnownHostsPath string 43 | Interval time.Duration 44 | } 45 | 46 | // gitSSHCommand returns the environment variable to be used for configuring 47 | // git over ssh. 48 | func (so SyncOptions) gitSSHCommand() string { 49 | sshKeyPath := so.GitSSHKeyPath 50 | if sshKeyPath == "" { 51 | sshKeyPath = "/dev/null" 52 | } 53 | knownHostsOptions := "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" 54 | if so.GitSSHKeyPath != "" && so.GitSSHKnownHostsPath != "" { 55 | knownHostsOptions = fmt.Sprintf("-o UserKnownHostsFile=%s", so.GitSSHKnownHostsPath) 56 | } 57 | return fmt.Sprintf(`GIT_SSH_COMMAND=ssh -q -F none -o IdentitiesOnly=yes -o IdentityFile=%s %s`, sshKeyPath, knownHostsOptions) 58 | } 59 | 60 | // Repository defines a remote git repository that should be synced regularly 61 | // and is the source of truth for a cluster. Changes in this repository trigger 62 | // GitPolling type runs for namespaces. The implementation borrows heavily from 63 | // git-sync. 64 | type Repository struct { 65 | lock sync.RWMutex 66 | path string 67 | repositoryConfig RepositoryConfig 68 | running bool 69 | stop, stopped chan bool 70 | syncOptions SyncOptions 71 | } 72 | 73 | // NewRepository initialises a Repository struct. 74 | func NewRepository(path string, repositoryConfig RepositoryConfig, syncOptions SyncOptions) (*Repository, error) { 75 | if path == "" { 76 | return nil, fmt.Errorf("cannot create Repository with empty local path") 77 | } 78 | if !filepath.IsAbs(path) { 79 | return nil, fmt.Errorf("Repository path must be absolute") 80 | } 81 | if repositoryConfig.Remote == "" { 82 | return nil, fmt.Errorf("cannot create Repository with empty remote") 83 | } 84 | if repositoryConfig.Depth < 0 { 85 | return nil, fmt.Errorf("Repository depth cannot be negative") 86 | } 87 | if repositoryConfig.Branch == "" { 88 | log.Logger("repository").Info("Defaulting repository branch to 'master'") 89 | repositoryConfig.Branch = "master" 90 | } 91 | if repositoryConfig.Revision == "" { 92 | log.Logger("repository").Info("Defaulting repository revision to 'HEAD'") 93 | repositoryConfig.Revision = "HEAD" 94 | } 95 | if syncOptions.Interval == 0 { 96 | log.Logger("repository").Info("Defaulting Interval to 30 seconds") 97 | syncOptions.Interval = time.Second * 30 98 | } 99 | return &Repository{ 100 | path: path, 101 | repositoryConfig: repositoryConfig, 102 | syncOptions: syncOptions, 103 | lock: sync.RWMutex{}, 104 | }, nil 105 | } 106 | 107 | // StartSync begins syncing from the remote git repository. The provided context 108 | // is only used for the initial sync operation which is performed synchronously. 109 | func (r *Repository) StartSync(ctx context.Context) error { 110 | if r.running { 111 | return fmt.Errorf("sync has already been started") 112 | } 113 | r.stop = make(chan bool) 114 | r.running = true 115 | log.Logger("repository").Info("waiting for the repository to complete initial sync") 116 | // The first sync is done outside of the syncLoop (and a seperate timeout if 117 | // the provided context has a deadline). The first clone might take longer 118 | // than usual depending on the size of the repository. Additionally it 119 | // runs in the foreground which simplifies startup since kube-applier 120 | // requires a repository clone to exist before starting up properly. 121 | if err := r.sync(ctx); err != nil { 122 | return err 123 | } 124 | go r.syncLoop() 125 | return nil 126 | } 127 | 128 | func (r *Repository) syncLoop() { 129 | r.stopped = make(chan bool) 130 | defer close(r.stopped) 131 | 132 | ticker := time.NewTicker(r.syncOptions.Interval) 133 | defer ticker.Stop() 134 | 135 | gcTicker := time.NewTicker(24 * time.Hour) 136 | defer gcTicker.Stop() 137 | 138 | log.Logger("repository").Info("started repository sync loop", "interval", r.syncOptions.Interval) 139 | for { 140 | select { 141 | case <-ticker.C: 142 | start := time.Now() 143 | ctx, cancel := context.WithTimeout(context.Background(), r.syncOptions.Interval-time.Second) 144 | err := r.sync(ctx) 145 | if err != nil { 146 | log.Logger("repository").Error("could not sync git repository", "error", err) 147 | } 148 | metrics.RecordGitSync(err == nil, start) 149 | cancel() 150 | 151 | case <-gcTicker.C: 152 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 153 | err := r.gitCleanup(ctx) 154 | if err != nil { 155 | log.Logger("repository").Error("error cleaning repository", "error", err) 156 | } 157 | cancel() 158 | 159 | case <-r.stop: 160 | return 161 | } 162 | } 163 | } 164 | 165 | // StopSync stops the syncing process. 166 | func (r *Repository) StopSync() { 167 | if !r.running { 168 | log.Logger("repository").Info("Sync has not been started, will not do anything") 169 | return 170 | } 171 | close(r.stop) 172 | <-r.stopped 173 | r.running = false 174 | } 175 | 176 | func (r *Repository) runGitCommand(ctx context.Context, environment []string, cwd string, args ...string) (string, error) { 177 | cmdStr := gitExecutablePath + " " + strings.Join(args, " ") 178 | log.Logger("repository").Debug("running command", "cwd", cwd, "cmd", cmdStr) 179 | 180 | cmd := exec.CommandContext(ctx, gitExecutablePath, args...) 181 | if cwd != "" { 182 | cmd.Dir = cwd 183 | } 184 | // force kill git & child process 5 seconds after sending it sigterm (when ctx is cancelled/timed out) 185 | cmd.WaitDelay = 5 * time.Second 186 | 187 | outbuf := bytes.NewBuffer(nil) 188 | errbuf := bytes.NewBuffer(nil) 189 | cmd.Stdout = outbuf 190 | cmd.Stderr = errbuf 191 | cmd.Env = append(os.Environ(), r.syncOptions.gitSSHCommand()) 192 | if len(environment) > 0 { 193 | cmd.Env = append(cmd.Env, environment...) 194 | } 195 | start := time.Now() 196 | err := cmd.Run() 197 | stdout := outbuf.String() 198 | stderr := errbuf.String() 199 | if ctx.Err() == context.DeadlineExceeded { 200 | return "", fmt.Errorf("Run(%s): %w: { stdout: %q, stderr: %q }", cmdStr, ctx.Err(), stdout, stderr) 201 | } 202 | if err != nil { 203 | return "", fmt.Errorf("Run(%s): %w: { stdout: %q, stderr: %q }", cmdStr, err, stdout, stderr) 204 | } 205 | log.Logger("repository").Debug("command result", "cmd", cmdStr, "exec_time", time.Since(start).Seconds(), "stdout", stdout, "stderr", stderr) 206 | 207 | return stdout, nil 208 | } 209 | 210 | // localHash returns the locally known hash for the configured Revision. 211 | func (r *Repository) localHash(ctx context.Context) (string, error) { 212 | output, err := r.runGitCommand(ctx, nil, r.path, "rev-parse", r.repositoryConfig.Revision) 213 | if err != nil { 214 | return "", err 215 | } 216 | return strings.Trim(string(output), "\n"), nil 217 | } 218 | 219 | // localHashForPath returns the hash of the configured revision for the 220 | // specified path. 221 | func (r *Repository) localHashForPath(ctx context.Context, path string) (string, error) { 222 | output, err := r.runGitCommand(ctx, nil, r.path, "log", "--pretty=format:%h", "-n", "1", "--", path) 223 | if err != nil { 224 | return "", err 225 | } 226 | return strings.Trim(string(output), "\n"), nil 227 | } 228 | 229 | // remoteHash returns the upstream hash for the ref that corresponds to the 230 | // configured Revision. 231 | func (r *Repository) remoteHash(ctx context.Context) (string, error) { 232 | // Build a ref string, depending on whether the user asked to track HEAD or 233 | // a tag. 234 | ref := "" 235 | if r.repositoryConfig.Revision == "HEAD" { 236 | ref = "refs/heads/" + r.repositoryConfig.Branch 237 | } else { 238 | ref = "refs/tags/" + r.repositoryConfig.Revision 239 | } 240 | 241 | output, err := r.runGitCommand(ctx, nil, r.path, "ls-remote", "-q", "origin", ref) 242 | if err != nil { 243 | return "", err 244 | } 245 | parts := strings.Split(string(output), "\t") 246 | return parts[0], nil 247 | } 248 | 249 | func (r *Repository) sync(ctx context.Context) error { 250 | r.lock.Lock() 251 | defer r.lock.Unlock() 252 | 253 | gitRepoPath := filepath.Join(r.path, ".git") 254 | _, err := os.Stat(gitRepoPath) 255 | switch { 256 | case os.IsNotExist(err): 257 | // First time. Just clone it and get the hash. 258 | err = r.cloneRemote(ctx) 259 | if err != nil { 260 | return err 261 | } 262 | return nil 263 | case err != nil: 264 | return fmt.Errorf("error checking if repo exists %q: %v", gitRepoPath, err) 265 | default: 266 | // Not the first time. Figure out if the ref has changed. 267 | // Since this operation is read only no lock required 268 | local, err := r.localHash(ctx) 269 | if err != nil { 270 | return err 271 | } 272 | remote, err := r.remoteHash(ctx) 273 | if err != nil { 274 | return err 275 | } 276 | if local == remote { 277 | log.Logger("repository").Info("no update required", "rev", r.repositoryConfig.Revision, "local", local, "remote", remote) 278 | return nil 279 | } 280 | log.Logger("repository").Info("update required", "rev", r.repositoryConfig.Revision, "local", local, "remote", remote) 281 | } 282 | 283 | log.Logger("repository").Info("syncing git", "branch", r.repositoryConfig.Branch, "rev", r.repositoryConfig.Revision) 284 | args := []string{"fetch", "-f", "--tags"} 285 | if r.repositoryConfig.Depth != 0 { 286 | args = append(args, "--depth", strconv.Itoa(r.repositoryConfig.Depth)) 287 | } 288 | args = append(args, "origin", r.repositoryConfig.Branch) 289 | // Update from the remote. 290 | if _, err := r.runGitCommand(ctx, nil, r.path, args...); err != nil { 291 | return err 292 | } 293 | 294 | // Reset HEAD 295 | if _, err = r.runGitCommand(ctx, nil, r.path, "reset", "--soft", fmt.Sprintf("origin/%s", r.repositoryConfig.Branch)); err != nil { 296 | return err 297 | } 298 | return nil 299 | } 300 | 301 | func (r *Repository) gitCleanup(ctx context.Context) error { 302 | r.lock.Lock() 303 | defer r.lock.Unlock() 304 | 305 | gitRepoPath := filepath.Join(r.path, ".git") 306 | // GC clone 307 | if _, err := r.runGitCommand(ctx, nil, r.path, "gc", "--prune=all"); err != nil { 308 | commitGraphLock := filepath.Join(gitRepoPath, "objects/info/commit-graph.lock") 309 | if strings.Contains(err.Error(), fmt.Sprintf("Unable to create '%s': File exists.", commitGraphLock)) { 310 | if e := os.Remove(commitGraphLock); e != nil { 311 | log.Logger("repository").Error("possible git crash detected but could not remove commit graph lock", "path", commitGraphLock, "error", e) 312 | } else { 313 | log.Logger("repository").Error("possible git crash detected, commit graph lock removed and next attempt should succeed", "path", commitGraphLock) 314 | } 315 | } 316 | return err 317 | } 318 | return nil 319 | } 320 | 321 | func (r *Repository) cloneRemote(ctx context.Context) error { 322 | args := []string{"clone", "--no-checkout", "-b", r.repositoryConfig.Branch} 323 | if r.repositoryConfig.Depth != 0 { 324 | args = append(args, "--depth", strconv.Itoa(r.repositoryConfig.Depth)) 325 | } 326 | args = append(args, r.repositoryConfig.Remote, r.path) 327 | log.Logger("repository").Info("cloning repo", "origin", r.repositoryConfig.Remote, "path", r.path) 328 | 329 | _, err := r.runGitCommand(ctx, nil, "", args...) 330 | if err != nil { 331 | if strings.Contains(err.Error(), "already exists and is not an empty directory") { 332 | // Maybe a previous run crashed? Git won't use this dir. 333 | log.Logger("repository").Info("git root exists and is not empty (previous crash?), cleaning up", "path", r.path) 334 | err := os.RemoveAll(r.path) 335 | if err != nil { 336 | return err 337 | } 338 | _, err = r.runGitCommand(ctx, nil, "", args...) 339 | if err != nil { 340 | return err 341 | } 342 | } else { 343 | return err 344 | } 345 | } 346 | return nil 347 | } 348 | 349 | // CloneLocal creates a clone of the existing repository to a new location on 350 | // disk and only checkouts the specified subpath. On success, it returns the 351 | // hash of the new repository clone's HEAD. 352 | func (r *Repository) CloneLocal(ctx context.Context, environment []string, dst, subpath string) (string, error) { 353 | r.lock.RLock() 354 | defer r.lock.RUnlock() 355 | 356 | hash, err := r.localHashForPath(ctx, subpath) 357 | if err != nil { 358 | return "", err 359 | } 360 | 361 | // git clone --no-checkout src dst 362 | // 363 | // Uncomment and use the following if running a Docker build with rootless Docker 364 | //if _, err := r.runGitCommand(ctx, nil, "", "clone", "--no-checkout", "--no-hardlinks", r.path, dst); err != nil { 365 | if _, err := r.runGitCommand(ctx, nil, "", "clone", "--no-checkout", r.path, dst); err != nil { 366 | return "", err 367 | } 368 | 369 | // git checkout HEAD -- ./path 370 | if _, err := r.runGitCommand(ctx, environment, dst, "checkout", r.repositoryConfig.Revision, "--", subpath); err != nil { 371 | return "", err 372 | } 373 | return hash, nil 374 | } 375 | 376 | // HashForPath returns the hash of the configured revision for the specified 377 | // path. 378 | func (r *Repository) HashForPath(ctx context.Context, path string) (string, error) { 379 | r.lock.RLock() 380 | defer r.lock.RUnlock() 381 | return r.localHashForPath(ctx, path) 382 | } 383 | 384 | // HasChangesForPath returns true if there are changes that have been committed 385 | // since the commit hash provided, under the specified path. 386 | func (r *Repository) HasChangesForPath(ctx context.Context, path, sinceHash string) (bool, error) { 387 | r.lock.RLock() 388 | defer r.lock.RUnlock() 389 | 390 | cmd := []string{"diff", "--quiet", sinceHash, r.repositoryConfig.Revision, "--", path} 391 | _, err := r.runGitCommand(ctx, nil, r.path, cmd...) 392 | if err == nil { 393 | return false, nil 394 | } 395 | var e *exec.ExitError 396 | if errors.As(err, &e) && e.ExitCode() == 1 { 397 | return true, nil 398 | } 399 | return false, err 400 | } 401 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/utilitywarehouse/kube-applier 2 | 3 | go 1.24.1 4 | 5 | require ( 6 | github.com/go-logr/logr v1.4.2 7 | github.com/go-test/deep v1.0.5 8 | github.com/google/go-cmp v0.6.0 9 | github.com/gorilla/mux v1.8.1 10 | github.com/gorilla/securecookie v1.1.2 11 | github.com/hashicorp/go-hclog v1.6.3 12 | github.com/onsi/ginkgo v1.16.5 13 | github.com/onsi/gomega v1.33.1 14 | github.com/pkg/errors v0.9.1 15 | github.com/prometheus/client_golang v1.20.3 16 | github.com/stretchr/testify v1.9.0 17 | github.com/utilitywarehouse/go-operational v0.0.0-20220413104526-79ce40a50281 18 | golang.org/x/oauth2 v0.23.0 19 | k8s.io/api v0.31.1 20 | k8s.io/apimachinery v0.31.1 21 | k8s.io/client-go v0.31.1 22 | k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 23 | sigs.k8s.io/controller-runtime v0.19.0 24 | sigs.k8s.io/yaml v1.4.0 25 | ) 26 | 27 | require ( 28 | github.com/beorn7/perks v1.0.1 // indirect 29 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 30 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 31 | github.com/emicklei/go-restful/v3 v3.12.1 // indirect 32 | github.com/evanphx/json-patch/v5 v5.9.0 // indirect 33 | github.com/fatih/color v1.17.0 // indirect 34 | github.com/fsnotify/fsnotify v1.7.0 // indirect 35 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 36 | github.com/go-logr/zapr v1.3.0 // indirect 37 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 38 | github.com/go-openapi/jsonreference v0.21.0 // indirect 39 | github.com/go-openapi/swag v0.23.0 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 42 | github.com/golang/protobuf v1.5.4 // indirect 43 | github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect 44 | github.com/google/gofuzz v1.2.0 // indirect 45 | github.com/google/uuid v1.6.0 // indirect 46 | github.com/imdario/mergo v0.3.16 // indirect 47 | github.com/josharian/intern v1.0.0 // indirect 48 | github.com/json-iterator/go v1.1.12 // indirect 49 | github.com/klauspost/compress v1.17.9 // indirect 50 | github.com/mailru/easyjson v0.7.7 // indirect 51 | github.com/mattn/go-colorable v0.1.13 // indirect 52 | github.com/mattn/go-isatty v0.0.20 // indirect 53 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 54 | github.com/modern-go/reflect2 v1.0.2 // indirect 55 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 56 | github.com/nxadm/tail v1.4.8 // indirect 57 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 58 | github.com/prometheus/client_model v0.6.1 // indirect 59 | github.com/prometheus/common v0.59.1 // indirect 60 | github.com/prometheus/procfs v0.15.1 // indirect 61 | github.com/spf13/pflag v1.0.5 // indirect 62 | github.com/x448/float16 v0.8.4 // indirect 63 | go.uber.org/multierr v1.11.0 // indirect 64 | go.uber.org/zap v1.26.0 // indirect 65 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect 66 | golang.org/x/net v0.38.0 // indirect 67 | golang.org/x/sys v0.31.0 // indirect 68 | golang.org/x/term v0.30.0 // indirect 69 | golang.org/x/text v0.23.0 // indirect 70 | golang.org/x/time v0.6.0 // indirect 71 | gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect 72 | google.golang.org/protobuf v1.34.2 // indirect 73 | gopkg.in/inf.v0 v0.9.1 // indirect 74 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 75 | gopkg.in/yaml.v2 v2.4.0 // indirect 76 | gopkg.in/yaml.v3 v3.0.1 // indirect 77 | k8s.io/apiextensions-apiserver v0.31.0 // indirect 78 | k8s.io/klog/v2 v2.130.1 // indirect 79 | k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect 80 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 81 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 82 | ) 83 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/hack/boilerplate.go.txt -------------------------------------------------------------------------------- /kubectl/client.go: -------------------------------------------------------------------------------- 1 | // Package kubectl provides a kubectl Client for interacting with the kubernetes 2 | // apiserver. 3 | package kubectl 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "io" 10 | "os" 11 | "os/exec" 12 | "regexp" 13 | "strings" 14 | "time" 15 | 16 | "github.com/pkg/errors" 17 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 18 | "k8s.io/apimachinery/pkg/runtime" 19 | kubeyaml "k8s.io/apimachinery/pkg/util/yaml" 20 | "sigs.k8s.io/yaml" 21 | 22 | "github.com/utilitywarehouse/kube-applier/metrics" 23 | ) 24 | 25 | var ( 26 | // The output is omitted if it contains any of these terms when there is an 27 | // error running `kubectl apply -f ` 28 | omitErrOutputTerms = []string{"Secret", "base64"} 29 | omitErrOutputMessage = "Some error output has been omitted because it may contain sensitive data\n" 30 | 31 | // Used in sanitiseCmdStr 32 | sanitiseCmdStrRe = regexp.MustCompile(`--token=[\S]+`) 33 | ) 34 | 35 | func sanitiseCmdStr(cmdStr string) string { 36 | return sanitiseCmdStrRe.ReplaceAllString(cmdStr, "--token=") 37 | } 38 | 39 | // ApplyOptions configure kubectl apply 40 | type ApplyOptions struct { 41 | DryRunStrategy string 42 | Environment []string 43 | Namespace string 44 | PruneWhitelist []string 45 | ServerSide bool 46 | Token string 47 | } 48 | 49 | // Args returns the flags that should be passed to exec.Command 50 | func (o *ApplyOptions) Args() []string { 51 | args := []string{} 52 | 53 | if o.Token != "" { 54 | args = append(args, fmt.Sprintf("--token=%s", o.Token)) 55 | } 56 | 57 | if o.Namespace != "" { 58 | args = append(args, []string{"-n", o.Namespace}...) 59 | } 60 | 61 | if o.DryRunStrategy != "" { 62 | args = append(args, fmt.Sprintf("--dry-run=%s", o.DryRunStrategy)) 63 | } 64 | 65 | if o.ServerSide { 66 | args = append(args, []string{"--server-side", "--force-conflicts"}...) 67 | } 68 | 69 | if len(o.PruneWhitelist) > 0 { 70 | args = append(args, []string{"--prune", "--all"}...) 71 | for _, w := range o.PruneWhitelist { 72 | args = append(args, "--prune-allowlist="+w) 73 | } 74 | } 75 | 76 | return args 77 | } 78 | 79 | func (o *ApplyOptions) setCommandEnvironment(cmd *exec.Cmd) { 80 | if len(o.Environment) > 0 { 81 | cmd.Env = append(os.Environ(), o.Environment...) 82 | } 83 | } 84 | 85 | // Client enables communication with the Kubernetes API Server through kubectl commands. 86 | type Client struct { 87 | Host string 88 | Label string 89 | KubeCtlPath string 90 | KubeCtlOpts []string 91 | } 92 | 93 | func NewClient(host, label, kubeCtlPath string, kubeCtlOpts []string) *Client { 94 | if kubeCtlPath == "" { 95 | kubeCtlPath = exec.Command("kubectl").String() 96 | } 97 | return &Client{ 98 | Host: host, 99 | Label: label, 100 | KubeCtlPath: kubeCtlPath, 101 | KubeCtlOpts: kubeCtlOpts, 102 | } 103 | } 104 | 105 | // Apply attempts to "kubectl apply" the files located at path. It returns the 106 | // full apply command and its output. 107 | func (c *Client) Apply(ctx context.Context, path string, options ApplyOptions) (string, string, error) { 108 | var kustomize bool 109 | if _, err := os.Stat(path + "/kustomization.yaml"); err == nil { 110 | kustomize = true 111 | } else if _, err := os.Stat(path + "/kustomization.yml"); err == nil { 112 | kustomize = true 113 | } else if _, err := os.Stat(path + "/Kustomization"); err == nil { 114 | kustomize = true 115 | } 116 | if kustomize { 117 | cmd, out, err := c.applyKustomize(ctx, path, options) 118 | return sanitiseCmdStr(cmd), out, err 119 | } 120 | cmd, out, err := c.applyPath(ctx, path, options) 121 | return sanitiseCmdStr(cmd), out, err 122 | } 123 | 124 | // KubectlPath returns the filesystem path to the kubectl binary 125 | func (c *Client) KubectlPath() string { 126 | kubectlCmd := exec.Command(c.KubeCtlPath) 127 | return kubectlCmd.String() 128 | } 129 | 130 | // KustomizePath returns the filesystem path to the kustomize binary 131 | func (c *Client) KustomizePath() string { 132 | kustomizeCmd := exec.Command("kustomize") 133 | return kustomizeCmd.String() 134 | } 135 | 136 | // applyPath runs `kubectl apply -f ` 137 | func (c *Client) applyPath(ctx context.Context, path string, options ApplyOptions) (string, string, error) { 138 | cmdStr, out, err := c.apply(ctx, path, []byte{}, options) 139 | if err != nil { 140 | // Filter potential secret leaks out of the output 141 | return cmdStr, filterErrOutput(out), err 142 | } 143 | 144 | return cmdStr, out, nil 145 | } 146 | 147 | // applyKustomize does a `kustomize build | kubectl apply -f -` on the path 148 | func (c *Client) applyKustomize(ctx context.Context, path string, options ApplyOptions) (string, string, error) { 149 | var kustomizeStdout, kustomizeStderr bytes.Buffer 150 | 151 | kustomizeCmd := exec.CommandContext(ctx, "kustomize", "build", path) 152 | // force kill command 5 seconds after sending it sigterm (when ctx is cancelled/timed out) 153 | kustomizeCmd.WaitDelay = 5 * time.Second 154 | 155 | options.setCommandEnvironment(kustomizeCmd) 156 | kustomizeCmd.Stdout = &kustomizeStdout 157 | kustomizeCmd.Stderr = &kustomizeStderr 158 | 159 | err := kustomizeCmd.Run() 160 | if err != nil { 161 | if ctx.Err() == context.DeadlineExceeded { 162 | err = errors.Wrap(ctx.Err(), err.Error()) 163 | } 164 | return kustomizeCmd.String(), kustomizeStderr.String(), err 165 | } 166 | 167 | // Split the stdout into secrets and other resources 168 | stdout, err := io.ReadAll(&kustomizeStdout) 169 | if err != nil { 170 | return kustomizeCmd.String(), "error reading kustomize output", err 171 | } 172 | resources, secrets, err := splitSecrets(stdout) 173 | if err != nil { 174 | return kustomizeCmd.String(), "error extracting secrets from kustomize output", err 175 | } 176 | if len(resources) == 0 && len(secrets) == 0 { 177 | return kustomizeCmd.String(), "", fmt.Errorf("No resources were extracted from the kustomize output") 178 | } 179 | 180 | // This is the command we are effectively applying. In actuality we're splitting it into two 181 | // separate invocations of kubectl but we'll return this as the command 182 | // string. 183 | displayArgs := []string{} 184 | if c.Host != "" { 185 | displayArgs = append(displayArgs, "--server", c.Host) 186 | } 187 | displayArgs = append(displayArgs, "apply", "-f", "-") 188 | displayArgs = append(displayArgs, options.Args()...) 189 | // Add opts that are specific to this client 190 | displayArgs = append(c.KubeCtlOpts, displayArgs...) 191 | kubectlCmd := exec.Command(c.KubeCtlPath, displayArgs...) 192 | cmdStr := kustomizeCmd.String() + " | " + kubectlCmd.String() 193 | 194 | var kubectlOut string 195 | 196 | if len(resources) > 0 { 197 | // Don't prune secrets 198 | resourcesPruneWhitelist := []string{} 199 | for _, w := range options.PruneWhitelist { 200 | if w != "core/v1/Secret" { 201 | resourcesPruneWhitelist = append(resourcesPruneWhitelist, w) 202 | } 203 | } 204 | 205 | resourcesOptions := options 206 | resourcesOptions.PruneWhitelist = resourcesPruneWhitelist 207 | 208 | _, out, err := c.apply(ctx, "-", resources, resourcesOptions) 209 | kubectlOut = kubectlOut + out 210 | if err != nil { 211 | return cmdStr, kubectlOut, err 212 | } 213 | } 214 | 215 | if len(secrets) > 0 { 216 | // Only prune secrets 217 | var secretsPruneWhitelist []string 218 | for _, w := range options.PruneWhitelist { 219 | if w == "core/v1/Secret" { 220 | secretsPruneWhitelist = append(secretsPruneWhitelist, w) 221 | } 222 | } 223 | 224 | secretsOptions := options 225 | secretsOptions.PruneWhitelist = secretsPruneWhitelist 226 | 227 | _, out, err := c.apply(ctx, "-", secrets, secretsOptions) 228 | if err != nil { 229 | // Don't append the actual output, as the error output 230 | // from kubectl can leak the content of secrets 231 | kubectlOut = kubectlOut + omitErrOutputMessage 232 | return cmdStr, kubectlOut, err 233 | } 234 | kubectlOut = kubectlOut + out 235 | } 236 | 237 | return cmdStr, kubectlOut, nil 238 | } 239 | 240 | // apply runs `kubectl apply` 241 | func (c *Client) apply(ctx context.Context, path string, stdin []byte, options ApplyOptions) (string, string, error) { 242 | args := []string{} 243 | if c.Host != "" { 244 | args = append(args, "--server", c.Host) 245 | } 246 | args = append(args, "apply", "-f", path) 247 | if path != "-" { 248 | args = append(args, "-R") 249 | } 250 | args = append(args, options.Args()...) 251 | // Add opts that are specific to this client 252 | args = append(c.KubeCtlOpts, args...) 253 | 254 | kubectlCmd := exec.CommandContext(ctx, c.KubeCtlPath, args...) 255 | // force kill command 5 seconds after sending it sigterm (when ctx is cancelled/timed out) 256 | kubectlCmd.WaitDelay = 5 * time.Second 257 | options.setCommandEnvironment(kubectlCmd) 258 | if path == "-" { 259 | if len(stdin) == 0 { 260 | return "", "", fmt.Errorf("path can't be %s when stdin is empty", path) 261 | } 262 | kubectlCmd.Stdin = bytes.NewReader(stdin) 263 | } 264 | out, err := kubectlCmd.CombinedOutput() 265 | if err != nil { 266 | if e, ok := err.(*exec.ExitError); ok { 267 | metrics.UpdateKubectlExitCodeCount(options.Namespace, e.ExitCode()) 268 | } 269 | if ctx.Err() == context.DeadlineExceeded { 270 | err = errors.Wrap(ctx.Err(), err.Error()) 271 | } 272 | return kubectlCmd.String(), string(out), err 273 | } 274 | metrics.UpdateKubectlExitCodeCount(options.Namespace, 0) 275 | 276 | return kubectlCmd.String(), string(out), nil 277 | } 278 | 279 | // filterErrOutput squashes output that may contain potentially sensitive 280 | // information 281 | func filterErrOutput(out string) string { 282 | for _, term := range omitErrOutputTerms { 283 | if strings.Contains(out, term) { 284 | return omitErrOutputMessage 285 | } 286 | } 287 | 288 | return out 289 | } 290 | 291 | // splitSecrets will take a yaml file and separate the resources into Secrets 292 | // and other resources. This allows Secrets to be applied separately to other 293 | // resources. 294 | func splitSecrets(yamlData []byte) (resources, secrets []byte, err error) { 295 | objs, err := splitYAML(yamlData) 296 | if err != nil { 297 | return resources, secrets, err 298 | } 299 | 300 | secretsDocs := [][]byte{} 301 | resourcesDocs := [][]byte{} 302 | for _, obj := range objs { 303 | y, err := yaml.Marshal(obj) 304 | if err != nil { 305 | return resources, secrets, err 306 | } 307 | if obj.Object["kind"] == "Secret" { 308 | secretsDocs = append(secretsDocs, y) 309 | } else { 310 | resourcesDocs = append(resourcesDocs, y) 311 | } 312 | } 313 | 314 | secrets = bytes.Join(secretsDocs, []byte("---\n")) 315 | resources = bytes.Join(resourcesDocs, []byte("---\n")) 316 | 317 | return resources, secrets, nil 318 | } 319 | 320 | // splitYAML splits a YAML file into unstructured objects. Returns list of all unstructured objects 321 | // found in the yaml. If an error occurs, returns objects that have been parsed so far too. 322 | // 323 | // Taken from the gitops-engine: 324 | // - https://github.com/argoproj/gitops-engine/blob/cc0fb5531c29c193291a7f97a50921f544b2d3b9/pkg/utils/kube/kube.go#L282-L310 325 | func splitYAML(yamlData []byte) ([]*unstructured.Unstructured, error) { 326 | // Similar way to what kubectl does 327 | // https://github.com/kubernetes/cli-runtime/blob/master/pkg/resource/visitor.go#L573-L600 328 | // Ideally k8s.io/cli-runtime/pkg/resource.Builder should be used instead of this method. 329 | // E.g. Builder does list unpacking and flattening and this code does not. 330 | d := kubeyaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlData), 4096) 331 | var objs []*unstructured.Unstructured 332 | for { 333 | ext := runtime.RawExtension{} 334 | if err := d.Decode(&ext); err != nil { 335 | if err == io.EOF { 336 | break 337 | } 338 | return objs, fmt.Errorf("failed to unmarshal manifest: %v", err) 339 | } 340 | ext.Raw = bytes.TrimSpace(ext.Raw) 341 | if len(ext.Raw) == 0 || bytes.Equal(ext.Raw, []byte("null")) { 342 | continue 343 | } 344 | u := &unstructured.Unstructured{} 345 | if err := yaml.Unmarshal(ext.Raw, u); err != nil { 346 | return objs, fmt.Errorf("failed to unmarshal manifest: %v", err) 347 | } 348 | objs = append(objs, u) 349 | } 350 | return objs, nil 351 | } 352 | -------------------------------------------------------------------------------- /kubectl/client_test.go: -------------------------------------------------------------------------------- 1 | package kubectl 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-test/deep" 7 | ) 8 | 9 | func TestFilterErrOutput(t *testing.T) { 10 | testCases := []struct { 11 | output string 12 | filtered bool 13 | }{ 14 | { 15 | output: `Error from server (BadRequest): error when creating "secrets/test.yaml": Secret in version "v1" cannot be handled as a Secret: v1.Secret.ObjectMeta: v1.ObjectMeta.TypeMeta: Kind: Data: decode base64: illegal base64 data at input byte 4, error found in #10 byte of ...|:"invalid"},"kind":"|..., bigger context ...|{"apiVersion":"v1","data":{"something":"invalid"},"kind":"Secret","metadata":{"annotations":{"kube|...`, 16 | filtered: true, 17 | }, 18 | { 19 | output: `The request is invalid: patch: Invalid value: "map[data:map[something:invalid] metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{\"apiVersion\":\"v1\",\"data\":{\"something\":\"invalid\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"name\":\"test\",\"namespace\":\"sys-auth\"},\"type\":\"Opaque\"}\n]]]": error decoding from json: illegal base64 data at input byte 4`, 20 | filtered: true, 21 | }, 22 | { 23 | output: `The request is invalid: patch: Invalid value: "map[data:map[something:map[]] metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{\"apiVersion\":\"v1\",\"data\":{\"something\":{}},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"name\":\"test\",\"namespace\":\"sys-auth\"},\"type\":\"Opaque\"}\n]]]": cannot restore slice from map`, 24 | filtered: true, 25 | }, 26 | { 27 | output: `Error from server (BadRequest): error when creating "secrets/test.yaml": Secret in version "v1" cannot be handled as a Secret: v1.Secret.Data: base64Codec: invalid input, error found in #10 byte of ...|omething":{}},"kind"|..., bigger context ...|{"apiVersion":"v1","data":{"something":{}},"kind":"Secret","metadata":{"annotations":{"ku|...`, 28 | filtered: true, 29 | }, 30 | { 31 | output: `namespace/sys-auth configured 32 | rolebinding.rbac.authorization.k8s.io/vault-configmap-applier unchanged 33 | Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply 34 | configmap/vault-tls configured 35 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged 36 | secret/k8s-auth-conf unchanged 37 | Error from server (BadRequest): error when creating "secrets/test.yaml": Secret in version "v1" cannot be handled as a Secret: v1.Secret.ObjectMeta: v1.ObjectMeta.TypeMeta: Kind: Data: decode base64: illegal base64 data at input byte 4, error found in #10 byte of ...|:"invalid"},"kind":"|..., bigger context ...|{"apiVersion":"v1","data":{"something":"invalid"},"kind":"Secret","metadata":{"annotations":{"kube|... 38 | `, 39 | filtered: true, 40 | }, 41 | { 42 | output: `namespace/sys-auth configured (server dry run) 43 | rolebinding.rbac.authorization.k8s.io/vault-configmap-applier unchanged (server dry run) 44 | Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply 45 | configmap/vault-tls configured (server dry run) 46 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged (server dry run) 47 | secret/k8s-auth-conf unchanged (server dry run) 48 | The request is invalid: patch: Invalid value: "map[data:map[something:invalid] metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{\"apiVersion\":\"v1\",\"data\":{\"something\":\"invalid\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"name\":\"test\",\"namespace\":\"sys-auth\"},\"type\":\"Opaque\"}\n]]]": error decoding from json: illegal base64 data at input byte 4 49 | `, 50 | filtered: true, 51 | }, 52 | { 53 | output: `The Secret "test_" is invalid: metadata.name: Invalid value: "test_": a DNS-1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, 54 | filtered: true, 55 | }, 56 | { 57 | output: `The request is invalid: patch: Invalid value: "map[data:map[invalid:map[]] metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{\"apiVersion\":\"v1\",\"data\":{\"invalid\":{}},\"kind\":\"ConfigMap\",\"metadata\":{\"annotations\":{},\"name\":\"test\",\"namespace\":\"labs\"}}\n]]]": unrecognized type: string`, 58 | filtered: false, 59 | }, 60 | { 61 | output: `namespace/sys-auth configured (server dry run) 62 | rolebinding.rbac.authorization.k8s.io/vault-configmap-applier unchanged (server dry run) 63 | Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply 64 | configmap/vault-tls configured (server dry run) 65 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged (server dry run) 66 | secret/k8s-auth-conf unchanged (server dry run) 67 | `, 68 | filtered: false, 69 | }, 70 | { 71 | output: `namespace/sys-auth configured (server dry run) 72 | rolebinding.rbac.authorization.k8s.io/vault-configmap-applier unchanged (server dry run) 73 | Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply 74 | configmap/vault-tls configured (server dry run) 75 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged (server dry run) 76 | secret/k8s-auth-conf unchanged (server dry run) 77 | The request is invalid: patch: Invalid value: "map[data:map[invalid:map[]] metadata:map[annotations:map[kubectl.kubernetes.io/last-applied-configuration:{\"apiVersion\":\"v1\",\"data\":{\"invalid\":{}},\"kind\":\"ConfigMap\",\"metadata\":{\"annotations\":{},\"name\":\"test\",\"namespace\":\"labs\"}}\n]]]": unrecognized type: string 78 | `, 79 | filtered: false, 80 | }, 81 | } 82 | 83 | for _, tc := range testCases { 84 | var want string 85 | if tc.filtered { 86 | want = omitErrOutputMessage 87 | } else { 88 | want = tc.output 89 | } 90 | 91 | got := filterErrOutput(tc.output) 92 | 93 | if diff := deep.Equal(got, want); diff != nil { 94 | t.Error(diff) 95 | } 96 | } 97 | 98 | for _, term := range omitErrOutputTerms { 99 | want := omitErrOutputMessage 100 | got := filterErrOutput(term) 101 | 102 | if diff := deep.Equal(got, want); diff != nil { 103 | t.Error(diff) 104 | } 105 | } 106 | } 107 | 108 | func TestApplyOptionsArgs(t *testing.T) { 109 | testCases := []struct { 110 | options ApplyOptions 111 | want []string 112 | }{ 113 | { 114 | options: ApplyOptions{ 115 | Namespace: "example", 116 | DryRunStrategy: "server", 117 | PruneWhitelist: []string{ 118 | "core/v1/ConfigMap", 119 | "core/v1/Pod", 120 | "rbac.authorization.k8s.io/v1beta1/RoleBinding", 121 | }, 122 | ServerSide: true, 123 | }, 124 | want: []string{"-n", "example", "--dry-run=server", 125 | "--server-side", 126 | "--force-conflicts", 127 | "--prune", "--all", 128 | "--prune-allowlist=core/v1/ConfigMap", 129 | "--prune-allowlist=core/v1/Pod", 130 | "--prune-allowlist=rbac.authorization.k8s.io/v1beta1/RoleBinding", 131 | }, 132 | }, 133 | { 134 | options: ApplyOptions{ 135 | Namespace: "example", 136 | DryRunStrategy: "server", 137 | PruneWhitelist: []string{ 138 | "core/v1/ConfigMap", 139 | "core/v1/Pod", 140 | "rbac.authorization.k8s.io/v1beta1/RoleBinding", 141 | }, 142 | }, 143 | want: []string{"-n", "example", "--dry-run=server", 144 | "--prune", "--all", 145 | "--prune-allowlist=core/v1/ConfigMap", 146 | "--prune-allowlist=core/v1/Pod", 147 | "--prune-allowlist=rbac.authorization.k8s.io/v1beta1/RoleBinding", 148 | }, 149 | }, 150 | { 151 | options: ApplyOptions{ 152 | Namespace: "example", 153 | PruneWhitelist: []string{ 154 | "core/v1/ConfigMap", 155 | "core/v1/Pod", 156 | "rbac.authorization.k8s.io/v1beta1/RoleBinding", 157 | }, 158 | }, 159 | want: []string{"-n", "example", 160 | "--prune", "--all", 161 | "--prune-allowlist=core/v1/ConfigMap", 162 | "--prune-allowlist=core/v1/Pod", 163 | "--prune-allowlist=rbac.authorization.k8s.io/v1beta1/RoleBinding", 164 | }, 165 | }, 166 | { 167 | options: ApplyOptions{ 168 | Namespace: "example", 169 | DryRunStrategy: "server", 170 | }, 171 | want: []string{"-n", "example", "--dry-run=server"}, 172 | }, 173 | { 174 | options: ApplyOptions{ 175 | PruneWhitelist: []string{ 176 | "core/v1/ConfigMap", 177 | "core/v1/Pod", 178 | "rbac.authorization.k8s.io/v1beta1/RoleBinding", 179 | }, 180 | }, 181 | want: []string{"--prune", "--all", 182 | "--prune-allowlist=core/v1/ConfigMap", 183 | "--prune-allowlist=core/v1/Pod", 184 | "--prune-allowlist=rbac.authorization.k8s.io/v1beta1/RoleBinding", 185 | }, 186 | }, 187 | } 188 | 189 | for _, tc := range testCases { 190 | got := tc.options.Args() 191 | 192 | if diff := deep.Equal(got, tc.want); diff != nil { 193 | t.Error(diff) 194 | } 195 | } 196 | } 197 | 198 | func TestSplitSecrets(t *testing.T) { 199 | testCases := []struct { 200 | yamlData []byte 201 | resourcesData []byte 202 | secretsData []byte 203 | }{ 204 | { 205 | yamlData: []byte(`apiVersion: v1 206 | kind: Secret 207 | metadata: 208 | name: example 209 | namespace: example-ns 210 | stringData: 211 | some-key: some-value 212 | some-other-key: some-other-value 213 | --- 214 | apiVersion: v1 215 | kind: Namespace 216 | metadata: 217 | labels: 218 | name: example-ns 219 | some-label: some-value 220 | name: example-ns 221 | --- 222 | apiVersion: v1 223 | kind: ServiceAccount 224 | metadata: 225 | annotations: 226 | some-annotation: some-value 227 | name: example 228 | --- 229 | apiVersion: v1 230 | data: 231 | some-value: c2Vuc2l0aXZlCg== 232 | kind: Secret 233 | metadata: 234 | name: example-1 235 | namespace: example-ns 236 | type: Opaque 237 | --- 238 | apiVersion: rbac.authorization.k8s.io/v1 239 | kind: RoleBinding 240 | metadata: 241 | name: example 242 | roleRef: 243 | apiGroup: rbac.authorization.k8s.io 244 | kind: ClusterRole 245 | name: admin 246 | subjects: 247 | - kind: ServiceAccount 248 | name: kube-applier 249 | namespace: example-ns 250 | `), 251 | resourcesData: []byte(`apiVersion: v1 252 | kind: Namespace 253 | metadata: 254 | labels: 255 | name: example-ns 256 | some-label: some-value 257 | name: example-ns 258 | --- 259 | apiVersion: v1 260 | kind: ServiceAccount 261 | metadata: 262 | annotations: 263 | some-annotation: some-value 264 | name: example 265 | --- 266 | apiVersion: rbac.authorization.k8s.io/v1 267 | kind: RoleBinding 268 | metadata: 269 | name: example 270 | roleRef: 271 | apiGroup: rbac.authorization.k8s.io 272 | kind: ClusterRole 273 | name: admin 274 | subjects: 275 | - kind: ServiceAccount 276 | name: kube-applier 277 | namespace: example-ns 278 | `), 279 | secretsData: []byte(`apiVersion: v1 280 | kind: Secret 281 | metadata: 282 | name: example 283 | namespace: example-ns 284 | stringData: 285 | some-key: some-value 286 | some-other-key: some-other-value 287 | --- 288 | apiVersion: v1 289 | data: 290 | some-value: c2Vuc2l0aXZlCg== 291 | kind: Secret 292 | metadata: 293 | name: example-1 294 | namespace: example-ns 295 | type: Opaque 296 | `), 297 | }, 298 | { 299 | yamlData: []byte(`apiVersion: v1 300 | kind: Namespace 301 | metadata: 302 | labels: 303 | name: example-ns 304 | some-label: some-value 305 | name: example-ns 306 | --- 307 | apiVersion: v1 308 | kind: ServiceAccount 309 | metadata: 310 | annotations: 311 | some-annotation: some-value 312 | name: example 313 | --- 314 | apiVersion: rbac.authorization.k8s.io/v1 315 | kind: RoleBinding 316 | metadata: 317 | name: example 318 | roleRef: 319 | apiGroup: rbac.authorization.k8s.io 320 | kind: ClusterRole 321 | name: admin 322 | subjects: 323 | - kind: ServiceAccount 324 | name: kube-applier 325 | namespace: example-ns 326 | `), 327 | resourcesData: []byte(`apiVersion: v1 328 | kind: Namespace 329 | metadata: 330 | labels: 331 | name: example-ns 332 | some-label: some-value 333 | name: example-ns 334 | --- 335 | apiVersion: v1 336 | kind: ServiceAccount 337 | metadata: 338 | annotations: 339 | some-annotation: some-value 340 | name: example 341 | --- 342 | apiVersion: rbac.authorization.k8s.io/v1 343 | kind: RoleBinding 344 | metadata: 345 | name: example 346 | roleRef: 347 | apiGroup: rbac.authorization.k8s.io 348 | kind: ClusterRole 349 | name: admin 350 | subjects: 351 | - kind: ServiceAccount 352 | name: kube-applier 353 | namespace: example-ns 354 | `), 355 | secretsData: []byte{}, 356 | }, 357 | { 358 | yamlData: []byte(`apiVersion: v1 359 | kind: Secret 360 | metadata: 361 | name: example 362 | namespace: example-ns 363 | stringData: 364 | some-key: some-value 365 | some-other-key: some-other-value 366 | --- 367 | apiVersion: v1 368 | data: 369 | some-value: c2Vuc2l0aXZlCg== 370 | kind: Secret 371 | metadata: 372 | name: example-1 373 | namespace: example-ns 374 | type: Opaque 375 | `), 376 | resourcesData: []byte{}, 377 | secretsData: []byte(`apiVersion: v1 378 | kind: Secret 379 | metadata: 380 | name: example 381 | namespace: example-ns 382 | stringData: 383 | some-key: some-value 384 | some-other-key: some-other-value 385 | --- 386 | apiVersion: v1 387 | data: 388 | some-value: c2Vuc2l0aXZlCg== 389 | kind: Secret 390 | metadata: 391 | name: example-1 392 | namespace: example-ns 393 | type: Opaque 394 | `), 395 | }, 396 | { 397 | yamlData: []byte{}, 398 | resourcesData: []byte{}, 399 | secretsData: []byte{}, 400 | }, 401 | } 402 | 403 | for _, tc := range testCases { 404 | resources, secrets, err := splitSecrets(tc.yamlData) 405 | if err != nil { 406 | t.Error(err) 407 | continue 408 | } 409 | 410 | if diff := deep.Equal(string(resources), string(tc.resourcesData)); diff != nil { 411 | t.Error(diff) 412 | } 413 | 414 | if diff := deep.Equal(string(secrets), string(tc.secretsData)); diff != nil { 415 | t.Error(diff) 416 | } 417 | } 418 | } 419 | -------------------------------------------------------------------------------- /log/logger.go: -------------------------------------------------------------------------------- 1 | // Package log provides the Logger struct for logging based on go-hclog. 2 | package log 3 | 4 | import ( 5 | "sync" 6 | 7 | hclog "github.com/hashicorp/go-hclog" 8 | ) 9 | 10 | const ( 11 | defaultName = "kube-applier" 12 | ) 13 | 14 | var ( 15 | loggers = map[string]hclog.Logger{} 16 | m = sync.Mutex{} 17 | ) 18 | 19 | func init() { 20 | m.Lock() 21 | defer m.Unlock() 22 | loggers[defaultName] = hclog.New(&hclog.LoggerOptions{ 23 | Name: defaultName, 24 | Level: hclog.LevelFromString("warn"), 25 | IncludeLocation: true, 26 | }) 27 | } 28 | 29 | // SetLevel sets the global logging level. 30 | func SetLevel(logLevel string) { 31 | // Since the original logger with defaultName does not set IndependentLevels 32 | // changing its level will also change the level of sub-loggers. 33 | loggers[defaultName].SetLevel(hclog.LevelFromString(logLevel)) 34 | } 35 | 36 | // Logger returns an hclog.Logger with the specified name. 37 | func Logger(name string) hclog.Logger { 38 | m.Lock() 39 | defer m.Unlock() 40 | if _, ok := loggers[name]; !ok { 41 | loggers[name] = loggers[defaultName].ResetNamed(name) 42 | } 43 | return loggers[name] 44 | } 45 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log/slog" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | ctrl "sigs.k8s.io/controller-runtime" 14 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 15 | 16 | "github.com/go-logr/logr" 17 | "github.com/utilitywarehouse/kube-applier/client" 18 | "github.com/utilitywarehouse/kube-applier/git" 19 | "github.com/utilitywarehouse/kube-applier/kubectl" 20 | "github.com/utilitywarehouse/kube-applier/log" 21 | "github.com/utilitywarehouse/kube-applier/run" 22 | "github.com/utilitywarehouse/kube-applier/sysutil" 23 | "github.com/utilitywarehouse/kube-applier/webserver" 24 | "github.com/utilitywarehouse/kube-applier/webserver/oidc" 25 | ) 26 | 27 | var ( 28 | fDiffURLFormat = flag.String("diff-url-format", getStringEnv("DIFF_URL_FORMAT", ""), "Used to generate commit links in the status page") 29 | fDryRun = flag.Bool("dry-run", getBoolEnv("DRY_RUN", false), "Whether kube-applier operates in dry-run mode globally") 30 | fGitPollWait = flag.Duration("git-poll-wait", getDurationEnv("GIT_POLL_WAIT", time.Second*5), "How long kube-applier waits before checking for changes in the repository") 31 | fGitKnownHostsPath = flag.String("git-ssh-known-hosts-path", getStringEnv("GIT_KNOWN_HOSTS_PATH", ""), "Path to the known hosts file used for fetching the repository") 32 | fGitSSHKeyPath = flag.String("git-ssh-key-path", getStringEnv("GIT_SSH_KEY_PATH", ""), "Path to the SSH key file used for fetching the repository. This will also be used for any Kustomize bases fetched via ssh, unless overridden by Waybill.Spec.GitSSHSecretRef config") 33 | fListenPort = flag.Int("listen-port", getIntEnv("LISTEN_PORT", 8080), "Port that the http server is listening on") 34 | fLogLevel = flag.String("log-level", getStringEnv("LOG_LEVEL", "warn"), "Logging level: trace, debug, info, warn, error, off") 35 | fOidcCallbackURL = flag.String("oidc-callback-url", getStringEnv("OIDC_CALLBACK_URL", ""), "OIDC callback url should be the root URL where kube-applier is exposed") 36 | fOidcClientID = flag.String("oidc-client-id", getStringEnv("OIDC_CLIENT_ID", ""), "Client ID of the OIDC application") 37 | fOidcClientSecret = flag.String("oidc-client-secret", getStringEnv("OIDC_CLIENT_SECRET", ""), "Client secret of the OIDC application") 38 | fOidcIssuer = flag.String("oidc-issuer", getStringEnv("OIDC_ISSUER", ""), "OIDC issuer URL of the authentication server") 39 | fPruneBlacklist = flag.String("prune-blacklist", getStringEnv("PRUNE_BLACKLIST", ""), "Comma-seperated list of resources to add to the global prune blacklist, in the // format") 40 | fRepoBranch = flag.String("repo-branch", getStringEnv("REPO_BRANCH", "master"), "Branch of the git repository to use") 41 | fRepoDepth = flag.Int("repo-depth", getIntEnv("REPO_DEPTH", 0), "Depth of the git repository to fetch. Use zero to ignore") 42 | fRepoDest = flag.String("repo-dest", getStringEnv("REPO_DEST", "/src"), "Path under which the the git repository is fetched") 43 | fRepoPath = flag.String("repo-path", getStringEnv("REPO_PATH", ""), "Path relative to the repository root that kube-applier operates in") 44 | fRepoRemote = flag.String("repo-remote", getStringEnv("REPO_REMOTE", ""), "Remote URL of the git repository that kube-applier uses as a source") 45 | fRepoRevision = flag.String("repo-revision", getStringEnv("REPO_REVISION", "HEAD"), "Revision of the git repository to use") 46 | fRepoSyncInterval = flag.Duration("repo-sync-interval", getDurationEnv("REPO_SYNC_INTERVAL", time.Second*30), "How often kube-applier will try to sync the local repository clone to the remote") 47 | fRepoTimeout = flag.Duration("repo-timeout", getDurationEnv("REPO_TIMEOUT", time.Minute*3), "How long kube-applier will wait for the initial repository sync to complete") 48 | fStatusTimeout = flag.Duration("status-timeout", getDurationEnv("STATUS_TIMEOUT", time.Second*30), "Timeout for retrieving the status UI information from Kubernetes") 49 | fWaybillPollInterval = flag.Duration("waybill-poll-interval", getDurationEnv("WAYBILL_POLL_INTERVAL", time.Minute), "How often kube-applier updates the Waybills it tracks from the cluster") 50 | fWorkerCount = flag.Int("worker-count", getIntEnv("WORKER_COUNT", 2), "Number of apply worker goroutines that kube-applier uses") 51 | ) 52 | 53 | func getStringEnv(name, defaultValue string) string { 54 | if v, ok := os.LookupEnv(name); ok { 55 | return v 56 | } 57 | return defaultValue 58 | } 59 | 60 | func getBoolEnv(name string, defaultValue bool) bool { 61 | if v, ok := os.LookupEnv(name); ok { 62 | vv, err := strconv.ParseBool(v) 63 | if err != nil { 64 | fmt.Printf("%s must be a boolean, got %v\n", name, v) 65 | os.Exit(1) 66 | } 67 | return vv 68 | } 69 | return defaultValue 70 | } 71 | 72 | func getIntEnv(name string, defaultValue int) int { 73 | if v, ok := os.LookupEnv(name); ok { 74 | vv, err := strconv.Atoi(v) 75 | if err != nil { 76 | fmt.Printf("%s must be an integer, got %v\n", name, v) 77 | os.Exit(1) 78 | } 79 | return vv 80 | } 81 | return defaultValue 82 | } 83 | 84 | func getDurationEnv(name string, defaultValue time.Duration) time.Duration { 85 | if v, ok := os.LookupEnv(name); ok { 86 | vv, err := time.ParseDuration(v) 87 | if err != nil { 88 | fmt.Printf("%s must be a duration, got %v\n", name, v) 89 | os.Exit(1) 90 | } 91 | return vv 92 | } 93 | return defaultValue 94 | } 95 | 96 | func main() { 97 | flag.Parse() 98 | 99 | log.SetLevel(*fLogLevel) 100 | 101 | var slogLevel slog.Level 102 | slogLevel.UnmarshalText([]byte(*fLogLevel)) 103 | ctrl.SetLogger(logr.FromSlogHandler( 104 | slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ 105 | Level: slogLevel, 106 | }), 107 | )) 108 | 109 | clock := &sysutil.Clock{} 110 | 111 | var ( 112 | oidcAuthenticator *oidc.Authenticator 113 | err error 114 | ) 115 | if strings.Join([]string{*fOidcIssuer, *fOidcClientID, *fOidcClientSecret, *fOidcCallbackURL}, "") != "" { 116 | oidcAuthenticator, err = oidc.NewAuthenticator( 117 | *fOidcIssuer, 118 | *fOidcClientID, 119 | *fOidcClientSecret, 120 | *fOidcCallbackURL, 121 | ) 122 | if err != nil { 123 | log.Logger("kube-applier").Error("could not setup oidc authenticator", "error", err) 124 | os.Exit(1) 125 | } 126 | log.Logger("kube-applier").Info("OIDC authentication configured", "issuer", *fOidcIssuer, "clientID", *fOidcClientID) 127 | } 128 | 129 | repo, err := git.NewRepository( 130 | *fRepoDest, 131 | git.RepositoryConfig{ 132 | Remote: *fRepoRemote, 133 | Branch: *fRepoBranch, 134 | Revision: *fRepoRevision, 135 | Depth: *fRepoDepth, 136 | }, 137 | git.SyncOptions{ 138 | GitSSHKeyPath: *fGitSSHKeyPath, 139 | GitSSHKnownHostsPath: *fGitKnownHostsPath, 140 | Interval: *fRepoSyncInterval, 141 | }, 142 | ) 143 | if err != nil { 144 | log.Logger("kube-applier").Error("could not create git repository", "error", err) 145 | os.Exit(1) 146 | } 147 | ctx, cancel := context.WithTimeout(context.Background(), *fRepoTimeout) 148 | if err := repo.StartSync(ctx); err != nil { 149 | log.Logger("kube-applier").Error("could not sync git repository", "error", err) 150 | os.Exit(1) 151 | } 152 | cancel() 153 | 154 | kubeClient, err := client.New() 155 | if err != nil { 156 | log.Logger("kube-applier").Error("error creating kubernetes API client", "error", err) 157 | os.Exit(1) 158 | } 159 | defer kubeClient.Shutdown() 160 | 161 | kubeCtlClient := kubectl.NewClient("", "", "", []string{}) 162 | 163 | // Kubernetes copies annotations from StatefulSets, Deployments and 164 | // Daemonsets to the corresponding ControllerRevision, including 165 | // 'kubectl.kubernetes.io/last-applied-configuration', which will result 166 | // in kube-applier pruning ControllerRevisions that it shouldn't be 167 | // managing at all. This makes it unsuitable for pruning and a 168 | // reasonable default for blacklisting. 169 | pruneBlacklistSlice := []string{"apps/v1/ControllerRevision"} 170 | if *fPruneBlacklist != "" { 171 | pruneBlacklistSlice = append(pruneBlacklistSlice, strings.Split(*fPruneBlacklist, ",")...) 172 | } 173 | 174 | runner := &run.Runner{ 175 | Clock: clock, 176 | DefaultGitSSHKeyPath: *fGitSSHKeyPath, 177 | DryRun: *fDryRun, 178 | KubeClient: kubeClient, 179 | KubeCtlClient: kubeCtlClient, 180 | PruneBlacklist: pruneBlacklistSlice, 181 | Repository: repo, 182 | RepoPath: *fRepoPath, 183 | Strongbox: &run.Strongboxer{}, 184 | WorkerCount: *fWorkerCount, 185 | } 186 | 187 | runQueue := runner.Start() 188 | 189 | scheduler := &run.Scheduler{ 190 | Clock: clock, 191 | GitPollWait: *fGitPollWait, 192 | KubeClient: kubeClient, 193 | Repository: repo, 194 | RepoPath: *fRepoPath, 195 | RunQueue: runQueue, 196 | WaybillPollInterval: *fWaybillPollInterval, 197 | } 198 | scheduler.Start() 199 | 200 | webserver := &webserver.WebServer{ 201 | Authenticator: oidcAuthenticator, 202 | Clock: clock, 203 | DiffURLFormat: *fDiffURLFormat, 204 | KubeClient: kubeClient, 205 | ListenPort: *fListenPort, 206 | RunQueue: runQueue, 207 | StatusTimeout: *fStatusTimeout, 208 | } 209 | if err := webserver.Start(); err != nil { 210 | log.Logger("kube-applier").Error(fmt.Sprintf("Cannot start webserver: %v", err)) 211 | os.Exit(1) 212 | } 213 | 214 | ctx = signals.SetupSignalHandler() 215 | <-ctx.Done() 216 | log.Logger("kube-applier").Info("Interrupted, shutting down...") 217 | if err := webserver.Shutdown(); err != nil { 218 | log.Logger("kube-applier").Error(fmt.Sprintf("Cannot shutdown webserver: %v", err)) 219 | } 220 | repo.StopSync() 221 | scheduler.Stop() 222 | runner.Stop() 223 | } 224 | -------------------------------------------------------------------------------- /manifests/base/client/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - service-account.yaml 5 | -------------------------------------------------------------------------------- /manifests/base/client/service-account.yaml: -------------------------------------------------------------------------------- 1 | # Used by kube-applier to apply resources in this namespace 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: kube-applier-delegate 6 | --- 7 | # Creates a secret with fixed name, populated with the kube-applier-delegate SA 8 | # data by the token controller. 9 | # https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#manually-create-a-service-account-api-token 10 | apiVersion: v1 11 | kind: Secret 12 | type: kubernetes.io/service-account-token 13 | metadata: 14 | name: kube-applier-delegate-token 15 | annotations: 16 | kubernetes.io/service-account.name: kube-applier-delegate 17 | --- 18 | # The kube-applier-delegate SA should be an admin in this namespace 19 | kind: RoleBinding 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | metadata: 22 | name: kube-applier-delegate 23 | roleRef: 24 | kind: ClusterRole 25 | name: admin 26 | apiGroup: rbac.authorization.k8s.io 27 | subjects: 28 | - kind: ServiceAccount 29 | name: kube-applier-delegate 30 | -------------------------------------------------------------------------------- /manifests/base/cluster/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRole 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: kube-applier 5 | rules: 6 | - apiGroups: ["kube-applier.io"] 7 | resources: ["waybills"] 8 | verbs: ["list", "watch"] 9 | - apiGroups: ["kube-applier.io"] 10 | resources: ["waybills/status"] 11 | verbs: ["update"] 12 | - apiGroups: [""] 13 | resources: ["secrets"] 14 | resourceNames: 15 | - kube-applier-delegate-token 16 | - kube-applier-git-ssh 17 | - kube-applier-strongbox-keyring 18 | verbs: ["get"] 19 | - apiGroups: [""] 20 | resources: ["events"] 21 | verbs: 22 | - create 23 | - patch 24 | - list 25 | - watch 26 | - apiGroups: ["authorization.k8s.io"] 27 | resources: ["subjectaccessreviews"] 28 | verbs: ["create"] 29 | -------------------------------------------------------------------------------- /manifests/base/cluster/kube-applier.io_waybills.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.11.3 7 | creationTimestamp: null 8 | name: waybills.kube-applier.io 9 | spec: 10 | group: kube-applier.io 11 | names: 12 | kind: Waybill 13 | listKind: WaybillList 14 | plural: waybills 15 | shortNames: 16 | - wb 17 | - wbs 18 | singular: waybill 19 | scope: Namespaced 20 | versions: 21 | - additionalPrinterColumns: 22 | - jsonPath: .status.lastRun.success 23 | name: Success 24 | type: boolean 25 | - jsonPath: .status.lastRun.type 26 | name: Reason 27 | type: string 28 | - jsonPath: .status.lastRun.commit 29 | name: Commit 30 | type: string 31 | - jsonPath: .status.lastRun.finished 32 | name: Last Applied 33 | type: date 34 | - jsonPath: .spec.autoApply 35 | name: Auto Apply 36 | priority: 10 37 | type: boolean 38 | - jsonPath: .spec.dryRun 39 | name: Dry Run 40 | priority: 10 41 | type: boolean 42 | - jsonPath: .spec.prune 43 | name: Prune 44 | priority: 10 45 | type: boolean 46 | - jsonPath: .spec.runInterval 47 | name: Run Interval 48 | priority: 10 49 | type: number 50 | - jsonPath: .spec.repositoryPath 51 | name: Repository Path 52 | priority: 20 53 | type: string 54 | name: v1alpha1 55 | schema: 56 | openAPIV3Schema: 57 | description: Waybill is the Schema for the Waybills API of kube-applier. A 58 | Waybill is defined as a namespace associated with a path in a remote git 59 | repository where kubernetes configuration is stored. 60 | properties: 61 | apiVersion: 62 | description: 'APIVersion defines the versioned schema of this representation 63 | of an object. Servers should convert recognized schemas to the latest 64 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 65 | type: string 66 | kind: 67 | description: 'Kind is a string value representing the REST resource this 68 | object represents. Servers may infer this from the endpoint the client 69 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 70 | type: string 71 | metadata: 72 | type: object 73 | spec: 74 | default: 75 | autoApply: true 76 | description: WaybillSpec defines the desired state of Waybill 77 | properties: 78 | autoApply: 79 | default: true 80 | description: AutoApply determines whether this Waybill will be automatically 81 | applied by scheduled or polling runs. 82 | type: boolean 83 | delegateServiceAccountSecretRef: 84 | default: kube-applier-delegate-token 85 | description: DelegateServiceAccountSecretRef references a Secret of 86 | type kubernetes.io/service-account-token in the same namespace as 87 | the Waybill that will be passed by kube-applier to kubectl when 88 | performing apply runs. 89 | minLength: 1 90 | type: string 91 | dryRun: 92 | default: false 93 | description: DryRun enables the dry-run flag when applying this Waybill. 94 | type: boolean 95 | gitSSHSecretRef: 96 | description: GitSSHSecretRef will override the default Git SSH key 97 | passed as a flag. It references a Secret that contains an item named 98 | `key` and optionally an item named `known_hosts`. If present, these 99 | are passed to the apply runtime and are used by `kustomize` when 100 | cloning remote bases. This allows the use of bases from private 101 | repositories that the default key will not have access to. 102 | properties: 103 | name: 104 | description: Name of the resource being referred to. 105 | type: string 106 | namespace: 107 | description: Namespace of the resource being referred to. 108 | type: string 109 | required: 110 | - name 111 | type: object 112 | prune: 113 | default: true 114 | description: Prune determines whether pruning is enabled for this 115 | Waybill. 116 | type: boolean 117 | pruneBlacklist: 118 | description: PruneBlacklist can be used to specify a list of resources 119 | that are exempt from pruning. 120 | items: 121 | type: string 122 | type: array 123 | pruneClusterResources: 124 | default: false 125 | description: PruneClusterResources determines whether pruning is enabled 126 | for cluster resources, as part of this Waybill. 127 | type: boolean 128 | repositoryPath: 129 | description: 'RepositoryPath defines the relative path inside the 130 | Repository where the configuration for this Waybill is stored. Accepted 131 | values are absolute or relative paths (relative to the root of the 132 | repository), such as: ''foo'', ''/foo'', ''foo/bar'', ''/foo/bar'' 133 | etc., as well as an empty string. If not specified, it will default 134 | to the name of the namespace where the Waybill is created.' 135 | pattern: ^(\/?[a-zA-Z0-9.\_\-]+(\/[a-zA-Z0-9.\_\-]+)*\/?)?$ 136 | type: string 137 | runInterval: 138 | default: 3600 139 | description: RunInterval determines how often this Waybill is applied 140 | in seconds. 141 | type: integer 142 | runTimeout: 143 | default: 900 144 | description: RunTimeout specifies the timeout for performing an apply 145 | run. 146 | type: integer 147 | serverSideApply: 148 | default: false 149 | description: ServerSideApply determines whether the server-side apply 150 | flag is enabled for this Waybill. 151 | type: boolean 152 | strongboxKeyringSecretRef: 153 | description: StrongboxKeyringSecretRef references a Secret that contains 154 | an item named '.strongbox_keyring' with any strongbox keys required 155 | to decrypt the files before applying. See the strongbox documentation 156 | for the format of the keyring data. 157 | properties: 158 | name: 159 | description: Name of the resource being referred to. 160 | type: string 161 | namespace: 162 | description: Namespace of the resource being referred to. 163 | type: string 164 | required: 165 | - name 166 | type: object 167 | type: object 168 | status: 169 | description: WaybillStatus defines the observed state of Waybill 170 | properties: 171 | lastRun: 172 | description: LastRun contains the last apply run's information. 173 | nullable: true 174 | properties: 175 | command: 176 | description: Command is the command used during the apply run. 177 | type: string 178 | commit: 179 | description: Commit is the git commit hash on which this apply 180 | run operated. 181 | type: string 182 | errorMessage: 183 | description: ErrorMessage describes any errors that occured during 184 | the apply run. 185 | type: string 186 | finished: 187 | description: Finished is the time that the apply run finished 188 | applying this Waybill. 189 | format: date-time 190 | type: string 191 | output: 192 | description: Output is the stdout of the Command. 193 | type: string 194 | started: 195 | description: Started is the time that the apply run started applying 196 | this Waybill. 197 | format: date-time 198 | type: string 199 | success: 200 | description: Success denotes whether the apply run was successful 201 | or not. 202 | type: boolean 203 | type: 204 | default: unknown 205 | description: Type is a short description of the kind of apply 206 | run that was attempted. 207 | type: string 208 | required: 209 | - command 210 | - commit 211 | - errorMessage 212 | - finished 213 | - output 214 | - started 215 | - success 216 | - type 217 | type: object 218 | type: object 219 | type: object 220 | served: true 221 | storage: true 222 | subresources: 223 | status: {} 224 | -------------------------------------------------------------------------------- /manifests/base/cluster/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - clusterrole.yaml 5 | - kube-applier.io_waybills.yaml 6 | -------------------------------------------------------------------------------- /manifests/base/server/kube-applier.yaml: -------------------------------------------------------------------------------- 1 | kind: ServiceAccount 2 | apiVersion: v1 3 | metadata: 4 | name: kube-applier 5 | --- 6 | apiVersion: v1 7 | kind: Service 8 | metadata: 9 | annotations: 10 | prometheus.io/scrape: "true" 11 | prometheus.io/path: /__/metrics 12 | prometheus.io/port: "8080" 13 | name: kube-applier 14 | labels: 15 | app: kube-applier 16 | spec: 17 | ports: 18 | - name: web 19 | protocol: TCP 20 | port: 80 21 | targetPort: 8080 22 | selector: 23 | app: kube-applier 24 | --- 25 | apiVersion: apps/v1 26 | kind: Deployment 27 | metadata: 28 | name: kube-applier 29 | spec: 30 | replicas: 1 31 | selector: 32 | matchLabels: 33 | app: kube-applier 34 | template: 35 | metadata: 36 | labels: 37 | app: kube-applier 38 | spec: 39 | serviceAccountName: kube-applier 40 | containers: 41 | - name: kube-applier 42 | image: quay.io/utilitywarehouse/kube-applier:master 43 | env: 44 | - name: DIFF_URL_FORMAT 45 | value: "https://github.com/org/repo/commit/%s" 46 | - name: LOG_LEVEL 47 | value: "warn" 48 | - name: GIT_SSH_KEY_PATH 49 | value: "/etc/git-secret/ssh" 50 | - name: GIT_KNOWN_HOSTS_PATH 51 | value: "/etc/git-secret/known_hosts" 52 | - name: REPO_REMOTE 53 | value: "git@github.com:org/repo.git" 54 | volumeMounts: 55 | - name: git-secret 56 | mountPath: /etc/git-secret 57 | resources: 58 | requests: 59 | cpu: 250m 60 | memory: 256Mi 61 | limits: 62 | cpu: 4000m 63 | memory: 512Mi 64 | ports: 65 | - containerPort: 8080 66 | volumes: 67 | - name: git-secret 68 | secret: 69 | secretName: ssh 70 | defaultMode: 0400 71 | -------------------------------------------------------------------------------- /manifests/base/server/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - kube-applier.yaml 5 | -------------------------------------------------------------------------------- /manifests/example/kube-applier-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: Ingress 3 | metadata: 4 | name: kube-applier 5 | labels: 6 | kubernetes.io/ingress.class: private-example 7 | annotations: 8 | external-dns.alpha.kubernetes.io/target: example.com 9 | spec: 10 | rules: 11 | - host: example.com 12 | http: 13 | paths: 14 | - path: / 15 | pathType: ImplementationSpecific 16 | backend: 17 | service: 18 | name: kube-applier 19 | port: 80 20 | -------------------------------------------------------------------------------- /manifests/example/kube-applier-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: kube-applier 5 | spec: 6 | template: 7 | spec: 8 | containers: 9 | - name: kube-applier 10 | env: 11 | - name: DIFF_URL_FORMAT 12 | value: "https://github.com/org/repo/commit/%s" 13 | - name: REPO_PATH 14 | value: "example-env" 15 | - name: REPO_REMOTE 16 | value: "git@github.com:org/repo.git" 17 | -------------------------------------------------------------------------------- /manifests/example/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | bases: 4 | - ../base/server 5 | # - github.com/utilitywarehouse/kube-applier//manifests/base/server?ref= 6 | - ../base/client 7 | # - github.com/utilitywarehouse/kube-applier//manifests/base/client?ref= 8 | 9 | resources: 10 | - kube-applier-ingress.yaml 11 | - waybill.yaml 12 | 13 | patchesStrategicMerge: 14 | # generic patch to specify environment/namespaces 15 | - kube-applier-patch.yaml 16 | 17 | secretGenerator: 18 | # ssh key to clone the "root" kubernetes manifests repository, used by git-sync 19 | - name: ssh 20 | type: Opaque 21 | files: 22 | - ssh=secrets/ssh 23 | - known_hosts=resources/known_hosts 24 | 25 | # strongbox keyring is used to decrypt Secrets in this namespace 26 | - name: kube-applier-strongbox-keyring 27 | type: Opaque 28 | files: 29 | - .strongbox_keyring=secrets/strongbox-keyring 30 | options: 31 | disableNameSuffixHash: true 32 | -------------------------------------------------------------------------------- /manifests/example/resources/known_hosts: -------------------------------------------------------------------------------- 1 | github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== 2 | -------------------------------------------------------------------------------- /manifests/example/secrets/ssh: -------------------------------------------------------------------------------- 1 | -----BEGIN OPENSSH PRIVATE KEY----- 2 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 3 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 4 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 5 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 6 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 7 | -----END OPENSSH PRIVATE KEY----- 8 | -------------------------------------------------------------------------------- /manifests/example/secrets/strongbox-keyring: -------------------------------------------------------------------------------- 1 | keyentries: 2 | - description: name 3 | key-id: 4 | key: 5 | -------------------------------------------------------------------------------- /manifests/example/waybill.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kube-applier.io/v1alpha1 2 | kind: Waybill 3 | metadata: 4 | name: main 5 | spec: 6 | strongboxKeyringSecretRef: 7 | name: kube-applier-strongbox-keyring 8 | -------------------------------------------------------------------------------- /metrics/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/go-test/deep" 8 | "github.com/utilitywarehouse/kube-applier/log" 9 | ) 10 | 11 | func TestMain(m *testing.M) { 12 | log.SetLevel("off") 13 | os.Exit(m.Run()) 14 | } 15 | 16 | func TestParseKubectlOutput(t *testing.T) { 17 | output := `namespace/namespaceName configured 18 | limitrange/limit-range configured 19 | role.rbac.authorization.k8s.io/auth unchanged 20 | rolebinding.rbac.authorization.k8s.io/rolebinding unchanged 21 | serviceaccount/account unchanged 22 | networkpolicy.networking.k8s.io/default unchanged 23 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged 24 | service/serviceName unchanged 25 | deployment.apps/deploymentName unchanged 26 | ` 27 | 28 | want := []applyObjectResult{ 29 | {"namespace", "namespaceName", "configured"}, 30 | {"limitrange", "limit-range", "configured"}, 31 | {"role.rbac.authorization.k8s.io", "auth", "unchanged"}, 32 | {"rolebinding.rbac.authorization.k8s.io", "rolebinding", "unchanged"}, 33 | {"serviceaccount", "account", "unchanged"}, 34 | {"networkpolicy.networking.k8s.io", "default", "unchanged"}, 35 | {"clusterrole.rbac.authorization.k8s.io", "system:metrics-server", "unchanged"}, 36 | {"service", "serviceName", "unchanged"}, 37 | {"deployment.apps", "deploymentName", "unchanged"}, 38 | } 39 | 40 | got := parseKubectlOutput(output) 41 | 42 | if diff := deep.Equal(got, want); diff != nil { 43 | t.Error(diff) 44 | } 45 | } 46 | 47 | func TestParseKubectlOutputServerDryRun(t *testing.T) { 48 | output := `namespace/namespaceName configured (server dry run) 49 | limitrange/limit-range configured (server dry run) 50 | role.rbac.authorization.k8s.io/auth configured (server dry run) 51 | rolebinding.rbac.authorization.k8s.io/rolebinding configured (server dry run) 52 | serviceaccount/account configured (server dry run) 53 | networkpolicy.networking.k8s.io/default configured (server dry run) 54 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged (server dry run) 55 | service/serviceName configured (server dry run) 56 | deployment.apps/deploymentName configured (server dry run) 57 | ` 58 | 59 | want := []applyObjectResult{ 60 | {"namespace", "namespaceName", "configured"}, 61 | {"limitrange", "limit-range", "configured"}, 62 | {"role.rbac.authorization.k8s.io", "auth", "configured"}, 63 | {"rolebinding.rbac.authorization.k8s.io", "rolebinding", "configured"}, 64 | {"serviceaccount", "account", "configured"}, 65 | {"networkpolicy.networking.k8s.io", "default", "configured"}, 66 | {"clusterrole.rbac.authorization.k8s.io", "system:metrics-server", "unchanged"}, 67 | {"service", "serviceName", "configured"}, 68 | {"deployment.apps", "deploymentName", "configured"}, 69 | } 70 | 71 | got := parseKubectlOutput(output) 72 | 73 | if diff := deep.Equal(got, want); diff != nil { 74 | t.Error(diff) 75 | } 76 | } 77 | 78 | func TestParseKubectlOutputWarningLine(t *testing.T) { 79 | output := `namespace/sys-auth configured (server dry run) 80 | rolebinding.rbac.authorization.k8s.io/vault-configmap-applier unchanged (server dry run) 81 | Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply 82 | configmap/vault-tls configured (server dry run) 83 | clusterrole.rbac.authorization.k8s.io/system:metrics-server unchanged (server dry run) 84 | secret/k8s-auth-conf unchanged (server dry run) 85 | ` 86 | 87 | want := []applyObjectResult{ 88 | {"namespace", "sys-auth", "configured"}, 89 | {"rolebinding.rbac.authorization.k8s.io", "vault-configmap-applier", "unchanged"}, 90 | {"configmap", "vault-tls", "configured"}, 91 | {"clusterrole.rbac.authorization.k8s.io", "system:metrics-server", "unchanged"}, 92 | {"secret", "k8s-auth-conf", "unchanged"}, 93 | } 94 | 95 | got := parseKubectlOutput(output) 96 | 97 | if diff := deep.Equal(got, want); diff != nil { 98 | t.Error(diff) 99 | } 100 | } 101 | 102 | func TestParseKubectlServerSideApply(t *testing.T) { 103 | output := `namespace/namespaceName serverside-applied 104 | limitrange/limit-range serverside-applied 105 | role.rbac.authorization.k8s.io/auth serverside-applied 106 | rolebinding.rbac.authorization.k8s.io/rolebinding serverside-applied 107 | serviceaccount/account serverside-applied 108 | networkpolicy.networking.k8s.io/default serverside-applied 109 | clusterrole.rbac.authorization.k8s.io/system:metrics-server serverside-applied 110 | service/serviceName serverside-applied 111 | deployment.apps/deploymentName serverside-applied 112 | ` 113 | 114 | want := []applyObjectResult{ 115 | {"namespace", "namespaceName", "serverside-applied"}, 116 | {"limitrange", "limit-range", "serverside-applied"}, 117 | {"role.rbac.authorization.k8s.io", "auth", "serverside-applied"}, 118 | {"rolebinding.rbac.authorization.k8s.io", "rolebinding", "serverside-applied"}, 119 | {"serviceaccount", "account", "serverside-applied"}, 120 | {"networkpolicy.networking.k8s.io", "default", "serverside-applied"}, 121 | {"clusterrole.rbac.authorization.k8s.io", "system:metrics-server", "serverside-applied"}, 122 | {"service", "serviceName", "serverside-applied"}, 123 | {"deployment.apps", "deploymentName", "serverside-applied"}, 124 | } 125 | 126 | got := parseKubectlOutput(output) 127 | 128 | if diff := deep.Equal(got, want); diff != nil { 129 | t.Error(diff) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /run/scheduler.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path/filepath" 7 | "reflect" 8 | "sync" 9 | "time" 10 | 11 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 12 | "github.com/utilitywarehouse/kube-applier/client" 13 | "github.com/utilitywarehouse/kube-applier/git" 14 | "github.com/utilitywarehouse/kube-applier/log" 15 | "github.com/utilitywarehouse/kube-applier/metrics" 16 | "github.com/utilitywarehouse/kube-applier/sysutil" 17 | ) 18 | 19 | // Type defines what kind of apply run is performed. 20 | type Type int 21 | 22 | func typeFromString(s string) Type { 23 | for i, v := range typeToString { 24 | if s == v { 25 | return Type(i) 26 | } 27 | } 28 | return -1 29 | } 30 | 31 | func (t Type) String() string { 32 | if int(t) >= len(typeToString) || int(t) < 0 { 33 | return "Unknown run type" 34 | } 35 | return typeToString[int(t)] 36 | } 37 | 38 | var typeToString = []string{ 39 | "Scheduled run", // ScheduledRun 40 | "Forced run", // ForcedRun 41 | "Git polling run", // PollingRun 42 | "Failed run", // FailedRun 43 | } 44 | 45 | const ( 46 | // ScheduledRun indicates a scheduled, regular apply run. 47 | ScheduledRun Type = iota 48 | // ForcedRun indicates a forced (triggered on the UI) apply run. 49 | ForcedRun 50 | // PollingRun indicated a run triggered by changes in the git repository. 51 | PollingRun 52 | // FailedRun indicates an apply run, scheduled after a previous failure. 53 | FailedRun 54 | ) 55 | 56 | // Scheduler handles queueing apply runs. 57 | type Scheduler struct { 58 | Clock sysutil.ClockInterface 59 | GitPollWait time.Duration 60 | KubeClient *client.Client 61 | Repository *git.Repository 62 | RepoPath string 63 | RunQueue chan<- Request 64 | WaybillPollInterval time.Duration 65 | waybills map[string]*kubeapplierv1alpha1.Waybill 66 | waybillSchedulers map[string]func() 67 | waybillsMutex sync.Mutex 68 | gitLastQueuedHash string 69 | stop chan bool 70 | waitGroup *sync.WaitGroup 71 | } 72 | 73 | // Start runs two loops: one that keeps track of Waybills on apiserver and 74 | // maintains loops for applying namespaces on a schedule, and one that watches 75 | // the git repository for changes and queues runs for waybills that are affected 76 | // by commits. 77 | func (s *Scheduler) Start() { 78 | if s.waitGroup != nil { 79 | return 80 | } 81 | s.stop = make(chan bool) 82 | s.waitGroup = &sync.WaitGroup{} 83 | s.waybills = make(map[string]*kubeapplierv1alpha1.Waybill) 84 | s.waybillSchedulers = make(map[string]func()) 85 | 86 | s.waitGroup.Add(1) 87 | go s.updateWaybillsLoop() 88 | s.waitGroup.Add(1) 89 | go s.gitPollingLoop() 90 | } 91 | 92 | // Stop gracefully shuts down the Scheduler. 93 | func (s *Scheduler) Stop() { 94 | if s.waitGroup == nil { 95 | log.Logger("scheduler").Debug("already stopped or being stopped") 96 | return 97 | } 98 | close(s.stop) 99 | s.waitGroup.Wait() 100 | s.waitGroup = nil 101 | s.waybillsMutex.Lock() 102 | for _, cancel := range s.waybillSchedulers { 103 | cancel() 104 | } 105 | s.waybillSchedulers = nil 106 | s.waybills = nil 107 | s.waybillsMutex.Unlock() 108 | } 109 | 110 | func (s *Scheduler) updateWaybills() { 111 | ctx, cancel := context.WithTimeout(context.Background(), s.WaybillPollInterval-time.Second) 112 | defer cancel() 113 | 114 | waybills, err := s.KubeClient.ListWaybills(ctx) 115 | if err != nil { 116 | log.Logger("scheduler").Error("Could not list Waybills", "error", err) 117 | return 118 | } 119 | metrics.ReconcileFromWaybillList(waybills) 120 | metrics.UpdateResultSummary(waybills) 121 | s.waybillsMutex.Lock() 122 | for i := range waybills { 123 | wb := &waybills[i] 124 | if v, ok := s.waybills[wb.Namespace]; ok { 125 | if !reflect.DeepEqual(v, wb) { 126 | s.waybillSchedulers[wb.Namespace]() 127 | s.waybillSchedulers[wb.Namespace] = s.newWaybillLoop(wb) 128 | s.waybills[wb.Namespace] = wb 129 | log.Logger("scheduler").Debug("Waybill changed, updating schedulers", "waybill", fmt.Sprintf("%s/%s", wb.Namespace, wb.Name)) 130 | } 131 | } else { 132 | s.waybillSchedulers[wb.Namespace] = s.newWaybillLoop(wb) 133 | s.waybills[wb.Namespace] = wb 134 | } 135 | } 136 | for ns := range s.waybills { 137 | found := false 138 | for _, wb := range waybills { 139 | if ns == wb.Namespace { 140 | found = true 141 | break 142 | } 143 | } 144 | if !found { 145 | s.waybillSchedulers[ns]() 146 | delete(s.waybillSchedulers, ns) 147 | delete(s.waybills, ns) 148 | } 149 | } 150 | s.waybillsMutex.Unlock() 151 | } 152 | 153 | func (s *Scheduler) updateWaybillsLoop() { 154 | ticker := time.NewTicker(s.WaybillPollInterval) 155 | defer ticker.Stop() 156 | defer s.waitGroup.Done() 157 | s.updateWaybills() 158 | for { 159 | select { 160 | case <-ticker.C: 161 | s.updateWaybills() 162 | case <-s.stop: 163 | return 164 | } 165 | } 166 | } 167 | 168 | func (s *Scheduler) gitPollingLoop() { 169 | defer s.waitGroup.Done() 170 | for { 171 | select { 172 | case <-time.After(s.GitPollWait): 173 | s.processGitChanges() 174 | case <-s.stop: 175 | return 176 | } 177 | } 178 | } 179 | 180 | func (s *Scheduler) processGitChanges() { 181 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) 182 | defer cancel() 183 | 184 | hash, err := s.Repository.HashForPath(ctx, s.RepoPath) 185 | if err != nil { 186 | log.Logger("scheduler").Warn("Git polling could not get HEAD hash", "error", err) 187 | return 188 | } 189 | // This check prevents the Scheduler from queueing multiple runs for 190 | // a Waybill; without this check, when a new commit appears it will 191 | // be eligible for new a run until it finishes the run and its 192 | // status is updated. 193 | // Waybills that are not in the Scheduler's cache when a new commit 194 | // appears will not be retroactively checked against the latest 195 | // commit when they are acknowledged. This is acceptable, since they 196 | // will (eventually) trigger a scheduled run. 197 | s.waybillsMutex.Lock() 198 | defer s.waybillsMutex.Unlock() 199 | if hash == s.gitLastQueuedHash { 200 | return 201 | } 202 | log.Logger("scheduler").Debug("New HEAD hash detected, checking for Waybills that need to be applied", "hash", hash) 203 | for i := range s.waybills { 204 | // If LastRun is nil, we don't trigger the Polling run at all 205 | // and instead rely on the Scheduled run to kickstart things. 206 | if s.waybills[i].Status.LastRun != nil && s.waybills[i].Status.LastRun.Commit != hash { 207 | sinceHash := s.waybills[i].Status.LastRun.Commit 208 | path := s.waybills[i].Spec.RepositoryPath 209 | if path == "" { 210 | path = s.waybills[i].Namespace 211 | } 212 | wbId := fmt.Sprintf("%s/%s", s.waybills[i].Namespace, s.waybills[i].Name) 213 | changed, err := s.Repository.HasChangesForPath(ctx, filepath.Join(s.RepoPath, path), sinceHash) 214 | if err != nil { 215 | log.Logger("scheduler").Warn("Could not check path for changes, skipping polling run", "waybill", wbId, "path", path, "since", sinceHash, "error", err) 216 | continue 217 | } 218 | if !changed { 219 | continue 220 | } 221 | Enqueue(s.RunQueue, PollingRun, s.waybills[i]) 222 | } 223 | } 224 | s.gitLastQueuedHash = hash 225 | } 226 | 227 | func (s *Scheduler) newWaybillLoop(waybill *kubeapplierv1alpha1.Waybill) func() { 228 | stop := make(chan bool) 229 | stopped := make(chan bool) 230 | go func() { 231 | defer close(stopped) 232 | 233 | // Immediately trigger if there is no previous run recorded, otherwise 234 | // wait for the proper amount of time in order to maintain the period. 235 | // If it's been too long, it will still trigger immediately since the 236 | // wait duration is going to be negative. 237 | if waybill.Status.LastRun == nil { 238 | Enqueue(s.RunQueue, ScheduledRun, waybill) 239 | } else { 240 | runAt := waybill.Status.LastRun.Started.Add(time.Duration(waybill.Spec.RunInterval) * time.Second) 241 | select { 242 | case <-time.After(runAt.Sub(s.Clock.Now())): 243 | Enqueue(s.RunQueue, ScheduledRun, waybill) 244 | case <-stop: 245 | return 246 | } 247 | } 248 | ticker := time.NewTicker(time.Duration(waybill.Spec.RunInterval) * time.Second) 249 | defer ticker.Stop() 250 | for { 251 | select { 252 | case <-ticker.C: 253 | Enqueue(s.RunQueue, ScheduledRun, waybill) 254 | case <-stop: 255 | return 256 | } 257 | } 258 | }() 259 | return func() { 260 | close(stop) 261 | <-stopped 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /run/strongbox.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "time" 10 | 11 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 12 | "github.com/utilitywarehouse/kube-applier/client" 13 | ) 14 | 15 | // strongboxInterface holds functions to configure strongbox for waybill runs 16 | type StrongboxInterface interface { 17 | SetupGitConfigForStrongbox(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill, env []string) error 18 | SetupStrongboxKeyring(ctx context.Context, kubeClient *client.Client, waybill *kubeapplierv1alpha1.Waybill, homeDir string) error 19 | } 20 | 21 | type strongboxBase struct{} 22 | 23 | func (sb *strongboxBase) SetupStrongboxKeyring(ctx context.Context, kubeClient *client.Client, waybill *kubeapplierv1alpha1.Waybill, homeDir string) error { 24 | if waybill.Spec.StrongboxKeyringSecretRef == nil { 25 | return nil 26 | } 27 | sbNamespace := waybill.Spec.StrongboxKeyringSecretRef.Namespace 28 | if sbNamespace == "" { 29 | sbNamespace = waybill.Namespace 30 | } 31 | secret, err := kubeClient.GetSecret(ctx, sbNamespace, waybill.Spec.StrongboxKeyringSecretRef.Name) 32 | if err != nil { 33 | return err 34 | } 35 | if err := checkSecretIsAllowed(waybill, secret); err != nil { 36 | return err 37 | } 38 | keyring, ok1 := secret.Data[".strongbox_keyring"] 39 | if ok1 { 40 | if err := os.WriteFile(filepath.Join(homeDir, ".strongbox_keyring"), keyring, 0400); err != nil { 41 | return err 42 | } 43 | } 44 | identity, ok2 := secret.Data[".strongbox_identity"] 45 | if ok2 { 46 | if err := os.WriteFile(filepath.Join(homeDir, ".strongbox_identity"), identity, 0400); err != nil { 47 | return err 48 | } 49 | } 50 | if !ok1 && !ok2 { 51 | return fmt.Errorf(`secret "%s/%s" does not contain key '.strongbox_keyring' or '.strongbox_identity'`, secret.Namespace, secret.Name) 52 | } 53 | return nil 54 | } 55 | 56 | type Strongboxer struct { 57 | strongboxBase 58 | } 59 | 60 | func (s *Strongboxer) SetupGitConfigForStrongbox(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill, env []string) error { 61 | if waybill.Spec.StrongboxKeyringSecretRef == nil { 62 | return nil 63 | } 64 | 65 | cmd := exec.CommandContext(ctx, "strongbox", "-git-config") 66 | // force kill command 5 seconds after sending it sigterm (when ctx is cancelled/timed out) 67 | cmd.WaitDelay = 5 * time.Second 68 | // Set PATH so we can find strongbox bin 69 | cmd.Env = append(env, fmt.Sprintf("PATH=%s", os.Getenv("PATH"))) 70 | stderr, err := cmd.CombinedOutput() 71 | if err != nil { 72 | return fmt.Errorf("error running strongbox err:%s %w ", stderr, err) 73 | } 74 | 75 | return nil 76 | } 77 | 78 | // Mock Strongboxer for testing 79 | type mockStrongboxer struct { 80 | strongboxBase 81 | } 82 | 83 | func (m *mockStrongboxer) SetupGitConfigForStrongbox(ctx context.Context, waybill *kubeapplierv1alpha1.Waybill, env []string) error { 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /run/suite_test.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | . "github.com/onsi/ginkgo" 16 | . "github.com/onsi/gomega" 17 | gomegaformat "github.com/onsi/gomega/format" 18 | . "github.com/onsi/gomega/gstruct" 19 | gomegatypes "github.com/onsi/gomega/types" 20 | "github.com/prometheus/client_golang/prometheus/promhttp" 21 | corev1 "k8s.io/api/core/v1" 22 | "k8s.io/client-go/kubernetes/scheme" 23 | "k8s.io/client-go/rest" 24 | controllerruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" 25 | "sigs.k8s.io/controller-runtime/pkg/envtest" 26 | logf "sigs.k8s.io/controller-runtime/pkg/log" 27 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 28 | 29 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 30 | "github.com/utilitywarehouse/kube-applier/client" 31 | "github.com/utilitywarehouse/kube-applier/git" 32 | "github.com/utilitywarehouse/kube-applier/log" 33 | // +kubebuilder:scaffold:imports 34 | ) 35 | 36 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 37 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 38 | 39 | var ( 40 | cfg *rest.Config 41 | k8sClient *client.Client 42 | kubeCtlPath string 43 | kubeCtlOpts []string 44 | testEnv *envtest.Environment 45 | repo *git.Repository 46 | tokenAuthFile *os.File 47 | adminToken = "admintoken" 48 | ) 49 | 50 | func init() { 51 | repoPath, _ := filepath.Abs("..") 52 | repo, _ = git.NewRepository(repoPath, git.RepositoryConfig{Remote: "foo"}, git.SyncOptions{}) 53 | //Disable truncating of tests output 54 | gomegaformat.MaxLength = 0 55 | } 56 | 57 | func TestAPIs(t *testing.T) { 58 | RegisterFailHandler(Fail) 59 | 60 | RunSpecs(t, "Run package suite") 61 | } 62 | 63 | var _ = BeforeSuite(func() { 64 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 65 | 66 | By("bootstrapping test environment") 67 | 68 | var err error 69 | 70 | // Create a token file with cluster admin permissions. This will be used 71 | // as the delegate service account token when running tests. 72 | tokenAuthFile, err = ioutil.TempFile("", "token-") 73 | Expect(err).ToNot(HaveOccurred()) 74 | _, err = tokenAuthFile.Write([]byte(fmt.Sprintf("%s,admin-user,1123,system:masters", adminToken))) 75 | Expect(err).ToNot(HaveOccurred()) 76 | err = tokenAuthFile.Close() 77 | Expect(err).ToNot(HaveOccurred()) 78 | 79 | testEnv = &envtest.Environment{ 80 | CRDDirectoryPaths: []string{filepath.Join("..", "manifests", "base", "cluster")}, 81 | } 82 | testEnv.ControlPlane.GetAPIServer().Configure().Append("token-auth-file", tokenAuthFile.Name()) 83 | 84 | cfg, err = testEnv.Start() 85 | Expect(err).ToNot(HaveOccurred()) 86 | Expect(cfg).ToNot(BeNil()) 87 | 88 | user, err := testEnv.AddUser(envtest.User{Name: "ka-test", Groups: []string{"system:masters"}}, &rest.Config{}) 89 | Expect(err).NotTo(HaveOccurred()) 90 | kubeCtl, err := user.Kubectl() 91 | Expect(err).NotTo(HaveOccurred()) 92 | kubeCtlPath = kubeCtl.Path 93 | Expect(kubeCtlPath).ToNot(BeEmpty()) 94 | kubeCtlOpts = kubeCtl.Opts 95 | Expect(kubeCtlOpts).ToNot(BeEmpty()) 96 | 97 | err = kubeapplierv1alpha1.AddToScheme(scheme.Scheme) 98 | Expect(err).NotTo(HaveOccurred()) 99 | 100 | // +kubebuilder:scaffold:scheme 101 | 102 | k8sClient, err = client.NewWithConfig(cfg) 103 | Expect(err).ToNot(HaveOccurred()) 104 | Expect(k8sClient).ToNot(BeNil()) 105 | 106 | hostParts := strings.Split(cfg.Host, ":") 107 | os.Setenv("KUBERNETES_SERVICE_HOST", hostParts[0]) 108 | os.Setenv("KUBERNETES_SERVICE_PORT", hostParts[1]) 109 | }, 60) 110 | 111 | var _ = AfterSuite(func() { 112 | By("tearing down the test environment") 113 | k8sClient.Shutdown() 114 | err := testEnv.Stop() 115 | Expect(err).ToNot(HaveOccurred()) 116 | os.Remove(tokenAuthFile.Name()) 117 | }) 118 | 119 | func init() { 120 | log.SetLevel("off") 121 | } 122 | 123 | type zeroClock struct{} 124 | 125 | func (c *zeroClock) Now() time.Time { return time.Time{} } 126 | func (c *zeroClock) Since(t time.Time) time.Duration { return time.Duration(0) } 127 | func (c *zeroClock) Sleep(d time.Duration) {} 128 | 129 | // testMetrics spins up a temporary webserver that exports the metrics and 130 | // captures the response to be tested again regexes 131 | func testMetrics(regex []string) { 132 | server := &http.Server{ 133 | Addr: fmt.Sprintf(":12700"), 134 | Handler: promhttp.Handler(), 135 | } 136 | go server.ListenAndServe() 137 | defer server.Shutdown(context.TODO()) 138 | var output string 139 | Eventually( 140 | func() error { 141 | res, err := http.Get(fmt.Sprintf("http://%s", server.Addr)) 142 | if err != nil { 143 | return err 144 | } 145 | body, err := io.ReadAll(res.Body) 146 | if err != nil { 147 | return err 148 | } 149 | output = string(body) 150 | return nil 151 | }, 152 | time.Second*15, 153 | time.Second, 154 | ).Should(BeNil()) 155 | // remove any metrics that don't come from the metrics package to reduce 156 | // output length in case of failures 157 | metricsLines := []string{} 158 | for _, s := range strings.Split(output, "\n") { 159 | if strings.HasPrefix(s, "kube_applier") { 160 | metricsLines = append(metricsLines, s) 161 | } 162 | } 163 | output = strings.Join(metricsLines, "\n") 164 | matchers := make([]gomegatypes.GomegaMatcher, len(regex)) 165 | for i, r := range regex { 166 | matchers[i] = MatchRegexp(r) 167 | } 168 | Expect(output).To(And(matchers...)) 169 | } 170 | 171 | func testCleanupNamespaces() { 172 | // With the envtest package we cannot delete namespaces, however, deleting 173 | // the CRs should be enough to avoid test pollution. 174 | // See https://github.com/kubernetes-sigs/controller-runtime/issues/880 175 | testRemoveAllWaybills() 176 | } 177 | 178 | func testRemoveAllWaybills() { 179 | // Although we could in theory use DeleteAllOf() here, it returns with a 180 | // NotFound error that has proven hard to debug. Instead, we can manually 181 | // List and Delete Waybills one by one. There should not be too many of them 182 | // to significantly affect test duration. 183 | waybills := kubeapplierv1alpha1.WaybillList{} 184 | Expect(k8sClient.GetAPIReader().List( 185 | context.TODO(), 186 | &waybills, 187 | )).To(BeNil()) 188 | for _, wb := range waybills.Items { 189 | Expect(k8sClient.GetClient().Delete( 190 | context.TODO(), 191 | &wb, 192 | controllerruntimeclient.GracePeriodSeconds(0), 193 | )).To(BeNil()) 194 | } 195 | Eventually( 196 | func() int { 197 | waybills := kubeapplierv1alpha1.WaybillList{} 198 | Expect(k8sClient.GetAPIReader().List(context.TODO(), &waybills)).To(BeNil()) 199 | return len(waybills.Items) 200 | }, 201 | time.Second*60, 202 | time.Second, 203 | ).Should(Equal(0)) 204 | } 205 | 206 | func testMatchEvents(matchers []gomegatypes.GomegaMatcher) { 207 | elements := make([]interface{}, len(matchers)) 208 | for i := range matchers { 209 | elements[i] = matchers[i] 210 | } 211 | Eventually( 212 | func() ([]corev1.Event, error) { 213 | events := &corev1.EventList{} 214 | if err := k8sClient.GetAPIReader().List(context.TODO(), events); err != nil { 215 | return nil, err 216 | } 217 | return events.Items, nil 218 | }, 219 | time.Second*15, 220 | time.Second, 221 | ).Should(ContainElements(elements...)) 222 | } 223 | 224 | // matchEvent is duplicated from the client package. 225 | func matchEvent(waybill kubeapplierv1alpha1.Waybill, eventType, reason, message string) gomegatypes.GomegaMatcher { 226 | return MatchFields(IgnoreExtras, Fields{ 227 | "TypeMeta": Ignore(), 228 | "ObjectMeta": MatchFields(IgnoreExtras, Fields{ 229 | "Namespace": Equal(waybill.ObjectMeta.Namespace), 230 | }), 231 | "InvolvedObject": MatchFields(IgnoreExtras, Fields{ 232 | "Kind": Equal("Waybill"), 233 | "Namespace": Equal(waybill.ObjectMeta.Namespace), 234 | "Name": Equal(waybill.ObjectMeta.Name), 235 | }), 236 | "Action": BeEmpty(), 237 | "Count": BeNumerically(">", 0), 238 | "Message": MatchRegexp(message), 239 | "Reason": Equal(reason), 240 | "Source": MatchFields(IgnoreExtras, Fields{ 241 | "Component": Equal(client.Name), 242 | }), 243 | "Type": Equal(eventType), 244 | }) 245 | } 246 | -------------------------------------------------------------------------------- /static/bootstrap/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/static/bootstrap/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /static/bootstrap/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/static/bootstrap/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /static/bootstrap/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/static/bootstrap/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /static/bootstrap/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/static/bootstrap/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /static/bootstrap/js/npm.js: -------------------------------------------------------------------------------- 1 | // This file is autogenerated via the `commonjs` Grunt task. You can require() this file in a CommonJS environment. 2 | require('../../js/transition.js') 3 | require('../../js/alert.js') 4 | require('../../js/button.js') 5 | require('../../js/carousel.js') 6 | require('../../js/collapse.js') 7 | require('../../js/dropdown.js') 8 | require('../../js/modal.js') 9 | require('../../js/tooltip.js') 10 | require('../../js/popover.js') 11 | require('../../js/scrollspy.js') 12 | require('../../js/tab.js') 13 | require('../../js/affix.js') -------------------------------------------------------------------------------- /static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utilitywarehouse/kube-applier/335000341af42febab362533bad628f877d38cc8/static/img/favicon.ico -------------------------------------------------------------------------------- /static/js/main.js: -------------------------------------------------------------------------------- 1 | // On button click, sends empty POST request to API endpoint for forcing a run and shows a relevant alert when a response is received. 2 | $(document).ready(function() { 3 | $(".force-namespace-button").each(function(){ 4 | $(this).bind('click', function(){ 5 | // Disable the buttons and close existing alert 6 | $('.force-button').each(function(){ $(this).prop('disabled', true); }); 7 | $('#force-alert').alert('close') 8 | 9 | forceRun($(this).data('namespace')) 10 | }); 11 | }); 12 | }); 13 | 14 | // Send an XHR request to the server to force a run. 15 | function forceRun(namespace) { 16 | url = window.location.href + 'api/v1/forceRun'; 17 | $.ajax({ 18 | type: 'POST', 19 | url: url, 20 | data: {namespace: namespace}, 21 | dataType: "json", 22 | success: function(data) { 23 | showForceAlert(true, data.message); 24 | $('.force-button').each(function(){ $(this).prop('disabled', false); }); 25 | }, 26 | error: function(xhr) { 27 | showForceAlert(false, 'Error: ' + xhr.responseJSON.message + '
See container logs for more info.'); 28 | $('.force-button').each(function(){ $(this).prop('disabled', false); }); 29 | } 30 | }); 31 | } 32 | 33 | // Show a relevant alert message, styled based on the "success" of the associated response. 34 | function showForceAlert(success, message) { 35 | alertClass = success ? 'success' : 'warning'; 36 | $('#force-alert-container').empty(); 37 | $('#force-alert-container').append( 38 | ''); 43 | } 44 | -------------------------------------------------------------------------------- /static/stylesheets/main.css: -------------------------------------------------------------------------------- 1 | .file-output { 2 | display: block; 3 | padding: 10px; 4 | margin: 0 0 10px; 5 | color: #333; 6 | background-color: #f5f5f5; 7 | border: 1px solid #ccc; 8 | border-radius: 4px; 9 | font-family: Monaco; 10 | font-size: 10px; 11 | white-space: nowrap; 12 | overflow: scroll; 13 | } 14 | 15 | pre.commit { 16 | font-size: 12px; 17 | } 18 | 19 | 20 | .panel:nth-of-type(odd) { 21 | background-color: #f9f9f9; 22 | } 23 | 24 | .force-button-group { 25 | margin-bottom: 5px; 26 | } 27 | -------------------------------------------------------------------------------- /sysutil/clock.go: -------------------------------------------------------------------------------- 1 | // Package sysutil provides interfaces for working with the filesystem, go 2 | // templates and a Clock interface for time-related functions. 3 | package sysutil 4 | 5 | import "time" 6 | 7 | // ClockInterface allows for mocking out the functionality of the standard time library when testing. 8 | type ClockInterface interface { 9 | Now() time.Time 10 | Since(time.Time) time.Duration 11 | Sleep(time.Duration) 12 | } 13 | 14 | // Clock implements ClockInterface with the standard time library functions. 15 | type Clock struct{} 16 | 17 | // Now returns current time 18 | func (c *Clock) Now() time.Time { 19 | return time.Now() 20 | } 21 | 22 | // Since returns time since t 23 | func (c *Clock) Since(t time.Time) time.Duration { 24 | return time.Since(t) 25 | } 26 | 27 | // Sleep sleeps for d duration 28 | func (c *Clock) Sleep(d time.Duration) { 29 | time.Sleep(d) 30 | } 31 | -------------------------------------------------------------------------------- /templates/status.html: -------------------------------------------------------------------------------- 1 | {{define "index"}} 2 | 3 | 4 | 5 | 6 | kube-applier 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |

kube-applier

15 | {{ if . }} 16 |
17 |
18 |
19 |
20 | 21 | {{ if (filter . "pending").Namespaces }} 22 | {{template "section" (filter . "pending")}} 23 | {{ end }} 24 | 25 | {{ if (filter . "auto-apply-disabled").Namespaces }} 26 | {{template "section" (filter . "auto-apply-disabled")}} 27 | {{ end }} 28 | 29 | {{ if (filter . "dry-run").Namespaces }} 30 | {{template "section" (filter . "dry-run")}} 31 | {{ end }} 32 | 33 | {{ if (filter . "failure").Namespaces }} 34 | {{template "section" (filter . "failure")}} 35 | {{ end }} 36 | 37 | {{ if (filter . "warning").Namespaces }} 38 | {{template "section" (filter . "warning")}} 39 | {{ end }} 40 | 41 | {{ if (filter . "success").Namespaces }} 42 | {{template "section" (filter . "success")}} 43 | {{ end }} 44 | 45 | {{ else }} 46 |

Waiting for information to be collected...

47 |

Refresh for updates and check the status and logs for the kube-applier container to make sure it is running properly.

48 | {{ end }} 49 | 50 | 51 | {{ end }} 52 | 53 | 54 | {{define "section"}} 55 |
56 |
57 |
58 |
59 |
60 |
61 |

62 | 63 | {{if eq $.FilteredBy "pending"}} 64 | Pending Namespaces: {{ len .Namespaces }} / {{ .Total }} 65 | {{else if eq $.FilteredBy "auto-apply-disabled"}} 66 | Auto Apply Disabled Namespaces: {{ len .Namespaces }} / {{ .Total }} 67 | {{else if eq $.FilteredBy "dry-run"}} 68 | Dry Run Only Namespaces: {{ len .Namespaces }} / {{ .Total }} 69 | {{else if eq $.FilteredBy "pending"}} 70 | Pending Namespaces: {{ len .Namespaces }} / {{ .Total }} 71 | {{else if eq $.FilteredBy "failure"}} 72 | Failed Namespaces: {{ len .Namespaces }} / {{ .Total }} 73 | {{else if eq $.FilteredBy "warning"}} 74 | Applied Namespaces (with warning): {{ len .Namespaces }} / {{ .Total }} 75 | {{else if eq $.FilteredBy "success"}} 76 | Applied Namespaces: {{ len .Namespaces }} / {{ .Total }} 77 | {{end}} 78 | 79 |

80 |
81 |
82 | {{ range $wb := .Namespaces }} 83 | {{template "namespace" $wb}} 84 | {{ end }} 85 |
86 |
87 |
88 |
89 |
90 | {{ end }} 91 | 92 | 93 | {{define "namespace"}} 94 |
95 |
96 |
97 | {{ .Waybill.Namespace }} {{ status .Waybill }} 98 |
99 |
100 | {{if .Waybill.Status.LastRun }} 101 |
102 |
    103 |
  • 104 |
    105 |
    106 | Type: {{ .Waybill.Status.LastRun.Type }}
    107 | 108 | {{ if .Waybill.Status.LastRun.Commit }} 109 | Commit: {{ if commitLink .DiffURLFormat .Waybill.Status.LastRun.Commit }}{{ .Waybill.Status.LastRun.Commit }}{{ else }}{{ .Waybill.Status.LastRun.Commit }}{{ end }}
    110 | {{ end }} 111 | 112 | Started: {{ formattedTime .Waybill.Status.LastRun.Started }} (took {{ latency .Waybill.Status.LastRun.Started .Waybill.Status.LastRun.Finished }})
    113 | 114 | {{ if .Waybill.Status.LastRun.ErrorMessage}} 115 | Error Message: {{ .Waybill.Status.LastRun.ErrorMessage }} 116 | {{ end }} 117 |
    118 |
    119 |
    120 |
    • 121 | {{ if .Waybill.Status.LastRun.Command }} 122 |
  • 123 |
    124 |
    {{ printf "$ %s\n" .Waybill.Status.LastRun.Command }}
    125 | {{ range $l := splitByNewline .Waybill.Status.LastRun.Output }} 126 | {{$l}}
    127 | {{ end }} 128 |
129 | 130 | {{ end }} 131 | {{ if .Events }} 132 |
  • 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | {{ range $i, $e := .Events }} 144 | {{ if eq $e.Namespace $.Waybill.Namespace }} 145 | 146 | 148 | 149 | 150 | 151 | {{ end }} 152 | {{ end }} 153 | 154 |
    LAST SEENTYPEREASONMESSAGE
    {{ $e.LastTimestamp }} 147 | {{ $e.Type }}{{ $e.Reason }}{{ $e.Message }}
    155 |
    • 156 | {{ end }} 157 | 158 |
    159 | {{ end }} 160 | 161 | {{ end }} 162 | -------------------------------------------------------------------------------- /testdata/bases/simple-deployment/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: 17 | containers: 18 | - name: alpine 19 | image: alpine 20 | command: 21 | - /bin/sh 22 | - -c 23 | - sleep 3600 24 | -------------------------------------------------------------------------------- /testdata/bases/simple-deployment/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - deployment.yaml 3 | -------------------------------------------------------------------------------- /testdata/manifests/app-a-kustomize/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-a-kustomize 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-a-kustomize/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: 17 | containers: 18 | - name: alpine 19 | image: alpine 20 | command: 21 | - /bin/sh 22 | - -c 23 | - sleep 3600 24 | -------------------------------------------------------------------------------- /testdata/manifests/app-a-kustomize/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - 00-namespace.yaml 3 | - deployment.yaml 4 | - secret.yaml 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-a-kustomize/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: -invalid 5 | type: Opaque 6 | -------------------------------------------------------------------------------- /testdata/manifests/app-a/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-a 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-a/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: 17 | containers: 18 | - name: alpine 19 | image: alpine 20 | command: 21 | - /bin/sh 22 | - -c 23 | - sleep 3600 24 | -------------------------------------------------------------------------------- /testdata/manifests/app-b-kustomize/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-b-kustomize 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-b-kustomize/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | # kube-applier: key_deploy 3 | - ssh://github.com/utilitywarehouse/kube-applier//testdata/bases/simple-deployment?ref=master 4 | resources: 5 | - 00-namespace.yaml 6 | -------------------------------------------------------------------------------- /testdata/manifests/app-b/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-b 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-b/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: {} 17 | -------------------------------------------------------------------------------- /testdata/manifests/app-c-kustomize/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-c-kustomize 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-c-kustomize/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - github.com/utilitywarehouse/kube-applier//testdata/bases/simple-deployment?ref=master 3 | resources: 4 | - 00-namespace.yaml 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-c/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-c 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-c/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: 17 | containers: 18 | - name: alpine 19 | image: alpine 20 | command: 21 | - /bin/sh 22 | - -c 23 | - sleep 3600 24 | -------------------------------------------------------------------------------- /testdata/manifests/app-d-kustomize/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-d-kustomize 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-d-kustomize/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | # kube-applier: key_should_be_ignored 3 | - ssh://github.com/utilitywarehouse/kube-applier//testdata/bases/simple-deployment?ref=master 4 | resources: 5 | - 00-namespace.yaml 6 | -------------------------------------------------------------------------------- /testdata/manifests/app-d/.gitattributes: -------------------------------------------------------------------------------- 1 | deployment.yaml filter=strongbox diff=strongbox 2 | -------------------------------------------------------------------------------- /testdata/manifests/app-d/.strongbox-keyid: -------------------------------------------------------------------------------- 1 | G4M/cCqr+LZtEyQbAjSu5SMEcnVTj2IkWahrkOUq/J4= 2 | -------------------------------------------------------------------------------- /testdata/manifests/app-d/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-d 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-d/deployment.yaml: -------------------------------------------------------------------------------- 1 | # STRONGBOX ENCRYPTED RESOURCE ; See https://github.com/uw-labs/strongbox 2 | GZ2MI6Vgz4DUMavhuW1TKD8FGgVp0V33hUiPBtg4EZpciDBY8QKWLby8UwpthnS/5Cwa2/MddsFT 3 | dGMqTwLUXGJQgqzRLMCoa94F8HzJilTNXSyWw6l3FZWfX6Dp6mAiA6E09tSpV7MyPc/5U1YYN6F9 4 | lnFVTj43IFuXioSu9GRMLmnMnKfH4fuhIXKxCARWiInRR8LGQsdRcdR2kF7kr6NDfSxyHFYW0CZc 5 | dC5YbvP5BAL/axC0hP/CfIuvueYjxaYZsQvmHcaKMdynOSHL5chPJhWy1KJxESc= 6 | -------------------------------------------------------------------------------- /testdata/manifests/app-e/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: app-e 5 | -------------------------------------------------------------------------------- /testdata/manifests/app-e/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-deployment 5 | labels: 6 | app: test-app 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: test-app 12 | template: 13 | metadata: 14 | labels: 15 | app: test-app 16 | spec: 17 | containers: 18 | - name: alpine 19 | image: alpine 20 | command: 21 | - /bin/sh 22 | - -c 23 | - sleep 3600 24 | -------------------------------------------------------------------------------- /testdata/manifests/strongbox-age/.gitattributes: -------------------------------------------------------------------------------- 1 | deployment.yaml filter=strongbox diff=strongbox 2 | -------------------------------------------------------------------------------- /testdata/manifests/strongbox-age/.strongbox_recipient: -------------------------------------------------------------------------------- 1 | age1ex4ph3ryaathfac0xpjhxk50utn50mtprke7h0vsmdlh6j63q5dsafxehs 2 | -------------------------------------------------------------------------------- /testdata/manifests/strongbox-age/00-namespace.yaml: -------------------------------------------------------------------------------- 1 | kind: Namespace 2 | apiVersion: v1 3 | metadata: 4 | name: strongbox-age 5 | -------------------------------------------------------------------------------- /testdata/manifests/strongbox-age/deployment.yaml: -------------------------------------------------------------------------------- 1 | -----BEGIN AGE ENCRYPTED FILE----- 2 | YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5TXhTQlVLNWtBZ0F4ejJV 3 | eWIzdnFGbllmZ1M1Uk05bFhUYmdKYUxNQkdVClNwNkxmYnZYUnAxay83N2JyWWRK 4 | dUY1VENnOWxCS044SStzc1FqdzUvclEKLS0tIE9Ob3NHUm1ZKzBUcU12RkJka05Z 5 | ZWpRMjZiS2E0ZUxkTHJiaUR0VGF2dzAKUqQbb3PhczAZuM6emvAC1Xl55mGDlZEO 6 | l7uELZcM88CN0vCZiEi6YC8hPJi2AhP6/8BQydwoR7uTyYxOVAd/1yw3ZgF1kZTJ 7 | boK+LADYFma2iM77utRjKh9WmM8/2GDjEdHkTYXE/BPqXR405yx++MaDsx65J5U6 8 | KknsfVP/GwwkKQzdOkY60MH5kpPixNDQjzgeYnwWfoSLDdocRmH3qvLEiVjXp+eS 9 | weMhDO4V8HJ8ljpsorbMJaXVWzRDy6d1K+XKV5PqNdysaSmivsptlEGRV58fpidU 10 | g5DyEW/rrRR5IrXutgXEzCnnst/nEEPGV1H8UeFXIhYuAxcF+gXZjDTzTwWhSgUJ 11 | OYj1jOOTRj/c+wSMK3cUZvOLK4wV8maPJQFp6fd3KOUVnzT+4ANQA92o5il3nJG7 12 | dAlWd2ddiRJfrSydrwXTKvHP7FZgVCVClIsQ1P3b9Sj9domQ8X5eMObGiMl2h2x4 13 | 9cH+qZzrkEhr4ZqhG/jHBK07xLqDaX7DwVHUSKR54EXtxYFKNuhMIKOFtmsAvCon 14 | XJwuVDvvVvs8Lyj0 15 | -----END AGE ENCRYPTED FILE----- 16 | -------------------------------------------------------------------------------- /webserver/operational.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "fmt" 5 | "net/http/pprof" 6 | 7 | "github.com/gorilla/mux" 8 | 9 | "github.com/utilitywarehouse/go-operational/op" 10 | ) 11 | 12 | const appName = "kube-applier" 13 | const appDescription = "Continuous deployment of Kubernetes objects by applying declarative configuration files from a Git repository to a Kubernetes cluster" 14 | 15 | func addStatusEndpoints(m *mux.Router) *mux.Router { 16 | m.PathPrefix("/__/").Handler(op.NewHandler(op.NewStatus(appName, appDescription). 17 | AddOwner("Billing team", "#finance_billing"). 18 | AddLink("readme", fmt.Sprintf("https://github.com/utilitywarehouse/%s/blob/master/README.md", appName)). 19 | ReadyAlways())) 20 | m.PathPrefix("/debug/pprof/cmdline").HandlerFunc(pprof.Cmdline) 21 | m.PathPrefix("/debug/pprof/profile").HandlerFunc(pprof.Profile) 22 | m.PathPrefix("/debug/pprof/symbol").HandlerFunc(pprof.Symbol) 23 | m.PathPrefix("/debug/pprof/trace").HandlerFunc(pprof.Trace) 24 | // Index covers the following: 25 | // - goroutine 26 | // - threadcreate 27 | // - heap 28 | // - allocs 29 | // - block 30 | // - mutex 31 | m.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) 32 | return m 33 | } 34 | -------------------------------------------------------------------------------- /webserver/result.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | "time" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/utils/ptr" 12 | 13 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 14 | ) 15 | 16 | var warningCheckReg = regexp.MustCompile("^Warning:.*") 17 | 18 | // namespace stores the current state of the waybill and events of a namespace. 19 | type Namespace struct { 20 | Waybill kubeapplierv1alpha1.Waybill 21 | Events []corev1.Event 22 | DiffURLFormat string 23 | } 24 | 25 | // GetNamespaces will create Namespace object combining wayBill and its corresponding events 26 | func GetNamespaces(waybills []kubeapplierv1alpha1.Waybill, allEvents []corev1.Event, diffURL string) []Namespace { 27 | var ns []Namespace 28 | for _, wb := range waybills { 29 | ns = append(ns, Namespace{ 30 | Waybill: wb, 31 | DiffURLFormat: diffURL, 32 | Events: waybillEvents(&wb, allEvents), 33 | }) 34 | } 35 | 36 | return ns 37 | } 38 | 39 | // waybillEvents returns all the events for the given Waybill 40 | func waybillEvents(wb *kubeapplierv1alpha1.Waybill, allEvents []corev1.Event) []corev1.Event { 41 | var events []corev1.Event 42 | for _, e := range allEvents { 43 | if e.InvolvedObject.Name == wb.Name && e.InvolvedObject.Namespace == wb.Namespace { 44 | events = append(events, e) 45 | } 46 | } 47 | 48 | return events 49 | } 50 | 51 | // Filtered stores collections of Namespaces with same outsome 52 | type Filtered struct { 53 | FilteredBy string 54 | Total int 55 | Namespaces []Namespace 56 | } 57 | 58 | func filter(Namespaces []Namespace, filteredBy string) Filtered { 59 | filtered := Filtered{ 60 | FilteredBy: filteredBy, 61 | Total: len(Namespaces), 62 | } 63 | for _, ns := range Namespaces { 64 | 65 | // specs specific filters 66 | switch filteredBy { 67 | case "auto-apply-disabled": 68 | if !isAutoApplyEnabled(ns) { 69 | filtered.Namespaces = append(filtered.Namespaces, ns) 70 | } 71 | case "dry-run": 72 | if ns.Waybill.Spec.DryRun { 73 | filtered.Namespaces = append(filtered.Namespaces, ns) 74 | } 75 | } 76 | 77 | // Following outcome(filters) only applies if DryRun is Disabled && autoApply is Enabled. 78 | if !ns.Waybill.Spec.DryRun && isAutoApplyEnabled(ns) { 79 | switch filteredBy { 80 | case "pending": 81 | if ns.Waybill.Status.LastRun == nil { 82 | filtered.Namespaces = append(filtered.Namespaces, ns) 83 | } 84 | case "failure": 85 | if ns.Waybill.Status.LastRun != nil && !ns.Waybill.Status.LastRun.Success { 86 | filtered.Namespaces = append(filtered.Namespaces, ns) 87 | } 88 | case "warning": 89 | if ns.Waybill.Status.LastRun != nil && ns.Waybill.Status.LastRun.Success && 90 | isOutcomeHasWarnings(ns.Waybill.Status.LastRun.Output) { 91 | filtered.Namespaces = append(filtered.Namespaces, ns) 92 | } 93 | case "success": 94 | if ns.Waybill.Status.LastRun != nil && ns.Waybill.Status.LastRun.Success && 95 | !isOutcomeHasWarnings(ns.Waybill.Status.LastRun.Output) { 96 | filtered.Namespaces = append(filtered.Namespaces, ns) 97 | } 98 | } 99 | } 100 | } 101 | return filtered 102 | } 103 | 104 | func isAutoApplyEnabled(ns Namespace) bool { 105 | if ns.Waybill.Spec.AutoApply != nil { 106 | return *ns.Waybill.Spec.AutoApply 107 | } 108 | // default AutoApply value is true 109 | return true 110 | } 111 | 112 | func isOutcomeHasWarnings(output string) bool { 113 | for _, l := range strings.Split(output, "\n") { 114 | if warningCheckReg.MatchString(strings.TrimSpace(l)) { 115 | return true 116 | } 117 | } 118 | return false 119 | } 120 | 121 | // Helper functions used in templates 122 | 123 | // FormattedTime returns the Time in the format "YYYY-MM-DD hh:mm:ss -0000 GMT" 124 | func formattedTime(t metav1.Time) string { 125 | return t.Time.Truncate(time.Second).String() 126 | } 127 | 128 | // Latency returns the latency between the two Times in seconds. 129 | func latency(t1, t2 metav1.Time) string { 130 | return fmt.Sprintf("%.0f sec", t2.Time.Sub(t1.Time).Seconds()) 131 | } 132 | 133 | // CommitLink returns a URL for the commit most recently applied or it returns 134 | // an empty string if it cannot construct the URL. 135 | func commitLink(diffUrl, commit string) string { 136 | if commit == "" || diffUrl == "" || !strings.Contains(diffUrl, "%s") { 137 | return "" 138 | } 139 | return fmt.Sprintf(diffUrl, commit) 140 | } 141 | 142 | // Status returns a human-readable string that describes the Waybill in terms 143 | // of its autoApply and dryRun attributes. 144 | func status(wb kubeapplierv1alpha1.Waybill) string { 145 | ret := []string{} 146 | if !ptr.Deref(wb.Spec.AutoApply, true) { 147 | ret = append(ret, "auto-apply disabled") 148 | } 149 | if wb.Spec.DryRun { 150 | ret = append(ret, "dry-run") 151 | } 152 | if len(ret) == 0 { 153 | return "" 154 | } 155 | return fmt.Sprintf("(%s)", strings.Join(ret, ", ")) 156 | } 157 | 158 | // AppliedRecently checks whether the provided Waybill was applied in the last 159 | // 15 minutes. 160 | func appliedRecently(waybill kubeapplierv1alpha1.Waybill) bool { 161 | return waybill.Status.LastRun != nil && 162 | time.Since(waybill.Status.LastRun.Started.Time) < time.Minute*15 163 | } 164 | 165 | func splitByNewline(output string) []string { 166 | return strings.Split(output, "\n") 167 | } 168 | 169 | func getOutputClass(l string) string { 170 | l = strings.TrimSpace(l) 171 | if warningCheckReg.MatchString(l) { 172 | return "text-warning" 173 | } 174 | if strings.HasSuffix(l, "configured") || 175 | strings.HasSuffix(l, "configured (server dry run)") { 176 | return "text-primary" 177 | } 178 | if strings.Contains(l, "unable to recognize") || 179 | strings.HasPrefix(l, "error:") { 180 | return "text-danger" 181 | } 182 | return "" 183 | } 184 | -------------------------------------------------------------------------------- /webserver/suite_test.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | "time" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | "k8s.io/client-go/kubernetes/scheme" 11 | "k8s.io/client-go/rest" 12 | "sigs.k8s.io/controller-runtime/pkg/envtest" 13 | logf "sigs.k8s.io/controller-runtime/pkg/log" 14 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 15 | 16 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 17 | "github.com/utilitywarehouse/kube-applier/client" 18 | "github.com/utilitywarehouse/kube-applier/log" 19 | // +kubebuilder:scaffold:imports 20 | ) 21 | 22 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 23 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 24 | 25 | var ( 26 | testConfig *rest.Config 27 | testKubeClient *client.Client 28 | testEnv *envtest.Environment 29 | ) 30 | 31 | func TestAPIs(t *testing.T) { 32 | RegisterFailHandler(Fail) 33 | 34 | RunSpecs(t, "Run package suite") 35 | } 36 | 37 | var _ = BeforeSuite(func() { 38 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 39 | 40 | By("bootstrapping test environment") 41 | testEnv = &envtest.Environment{ 42 | CRDDirectoryPaths: []string{filepath.Join("..", "manifests", "base", "cluster")}, 43 | } 44 | 45 | var err error 46 | testConfig, err = testEnv.Start() 47 | Expect(err).ToNot(HaveOccurred()) 48 | Expect(testConfig).ToNot(BeNil()) 49 | 50 | err = kubeapplierv1alpha1.AddToScheme(scheme.Scheme) 51 | Expect(err).NotTo(HaveOccurred()) 52 | 53 | // +kubebuilder:scaffold:scheme 54 | 55 | testKubeClient, err = client.NewWithConfig(testConfig) 56 | Expect(err).ToNot(HaveOccurred()) 57 | Expect(testKubeClient).ToNot(BeNil()) 58 | }, 60) 59 | 60 | var _ = AfterSuite(func() { 61 | By("tearing down the test environment") 62 | testKubeClient.Shutdown() 63 | err := testEnv.Stop() 64 | Expect(err).ToNot(HaveOccurred()) 65 | }) 66 | 67 | func init() { 68 | log.SetLevel("off") 69 | } 70 | 71 | type zeroClock struct{} 72 | 73 | func (c *zeroClock) Now() time.Time { return time.Time{} } 74 | func (c *zeroClock) Since(t time.Time) time.Duration { return time.Duration(0) } 75 | func (c *zeroClock) Sleep(d time.Duration) {} 76 | -------------------------------------------------------------------------------- /webserver/template.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "os" 7 | ) 8 | 9 | // createTemplate takes in a path to a template file and parses the file to create a Template instance. 10 | func createTemplate(templatePath string) (*template.Template, error) { 11 | if _, err := os.Stat(templatePath); err != nil { 12 | return nil, fmt.Errorf("Error opening template file: %v", err) 13 | } 14 | tmpl, err := template.New("index"). 15 | Funcs(template.FuncMap{ 16 | "filter": filter, 17 | "commitLink": commitLink, 18 | "formattedTime": formattedTime, 19 | "latency": latency, 20 | "appliedRecently": appliedRecently, 21 | "status": status, 22 | "splitByNewline": splitByNewline, 23 | "getOutputClass": getOutputClass, 24 | }). 25 | ParseFiles(templatePath) 26 | if err != nil { 27 | return nil, fmt.Errorf("Error parsing template: %v", err) 28 | } 29 | return tmpl, nil 30 | } 31 | -------------------------------------------------------------------------------- /webserver/template_test.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "regexp" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/google/go-cmp/cmp" 12 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 13 | corev1 "k8s.io/api/core/v1" 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | ) 16 | 17 | var ( 18 | diffURL = "https://github.com/org/repo/commit/%s" 19 | fixedTime = time.Date(2022, time.April, 26, 13, 36, 05, 0, time.UTC) 20 | emptyLineReg = regexp.MustCompile(`[\t\r\n]+`) 21 | 22 | removeSpaceEmptyLine = cmp.Transformer("RemoveSpace", func(in string) string { 23 | t := strings.ReplaceAll(in, " ", "") 24 | return emptyLineReg.ReplaceAllString(strings.TrimSpace(t), "\n") 25 | }) 26 | ) 27 | 28 | func Test_ExecuteTemplate(t *testing.T) { 29 | wbList := []kubeapplierv1alpha1.Waybill{ 30 | { 31 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 32 | ObjectMeta: metav1.ObjectMeta{ 33 | Name: "main", 34 | Namespace: "foo", 35 | }, 36 | Spec: kubeapplierv1alpha1.WaybillSpec{ 37 | AutoApply: &varTrue, 38 | }, 39 | }, 40 | { 41 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 42 | ObjectMeta: metav1.ObjectMeta{ 43 | Name: "main", 44 | Namespace: "bar", 45 | }, 46 | Spec: kubeapplierv1alpha1.WaybillSpec{ 47 | AutoApply: &varFalse, 48 | }, 49 | }, 50 | { 51 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 52 | ObjectMeta: metav1.ObjectMeta{ 53 | Name: "main", 54 | Namespace: "biz", 55 | }, 56 | Spec: kubeapplierv1alpha1.WaybillSpec{DryRun: true}, 57 | Status: kubeapplierv1alpha1.WaybillStatus{ 58 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 59 | Command: "/usr/local/bin/kustomize build /tmp/run_biz_main_repo_2305192794/dev/biz | /usr/local/bin/kubectl apply -f - --token= -n biz --dry-run=none --prune --all --prune-allowlist=core/v1/ConfigMap serviceaccount/job-trigger unchanged serviceaccount/kube-applier-delegate unchanged", 60 | Commit: "22c815614b", 61 | ErrorMessage: `exit status 1`, 62 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 63 | Finished: metav1.Time{Time: fixedTime}, 64 | Type: "Scheduled run", 65 | Output: `namespace/biz unchanged (server dry run) 66 | serviceaccount/fluentd unchanged (server dry run) 67 | serviceaccount/forwarder unchanged (server dry run) 68 | serviceaccount/kube-applier-delegate unchanged (server dry run) 69 | serviceaccount/loki unchanged (server dry run) 70 | rolebinding.rbac.authorization.k8s.io/admin configured (server dry run) 71 | rolebinding.rbac.authorization.k8s.io/kube-applier-delegate unchanged (server dry run) 72 | Warning: batch/v1beta1 CronJob is deprecated in v1.21+, unavailable in v1.25+; use batch/v1 CronJob 73 | secret/kube-applier-delegate-token unchanged (server dry run) 74 | `, 75 | }, 76 | }, 77 | }, 78 | { 79 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 80 | ObjectMeta: metav1.ObjectMeta{ 81 | Name: "main", 82 | Namespace: "zot", 83 | }, 84 | Status: kubeapplierv1alpha1.WaybillStatus{ 85 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 86 | Commit: "22c815614b", 87 | ErrorMessage: `exit status 1`, 88 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 89 | Finished: metav1.Time{Time: fixedTime}, 90 | Type: "Git polling run", 91 | }, 92 | }, 93 | }, 94 | { 95 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 96 | ObjectMeta: metav1.ObjectMeta{ 97 | Name: "main", 98 | Namespace: "zoo", 99 | }, 100 | Status: kubeapplierv1alpha1.WaybillStatus{ 101 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 102 | Command: "/usr/local/bin/kustomize build /tmp/run_zoo_main_repo_2305192794/dev/zoo | /usr/local/bin/kubectl apply -f - --token= -n zoo --dry-run=none --prune --all --prune-allowlist=core/v1/ConfigMap serviceaccount/job-trigger unchanged serviceaccount/kube-applier-delegate unchanged", 103 | Commit: "22c815614b", 104 | ErrorMessage: `exit status 1`, 105 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 106 | Finished: metav1.Time{Time: fixedTime}, 107 | Type: "Scheduled run", 108 | Output: `namespace/zoo unchanged 109 | serviceaccount/kube-applier-delegate unchanged 110 | rolebinding.rbac.authorization.k8s.io/kube-applier-delegate unchanged 111 | configmap/postgres-init unchanged 112 | configmap/postgres-env unchanged 113 | service/postgres unchanged 114 | service/webapp unchanged 115 | limitrange/default configured 116 | persistentvolumeclaim/postgres unchanged 117 | deployment.apps/postgres unchanged 118 | deployment.apps/webapp configured 119 | deployment.apps/scheduler configured 120 | Warning: autoscaling/v2beta1 HorizontalPodAutoscaler is deprecated in v1.22+, unavailable in v1.25+; use autoscaling/v2beta2 HorizontalPodAutoscaler 121 | Warning: batch/v1beta1 CronJob is deprecated in v1.21+, unavailable in v1.25+; use batch/v1 CronJob 122 | waybill.kube-applier.io/main unchanged 123 | networkpolicy.networking.k8s.io/default unchanged 124 | error: error validating "/tmp/dev/secrets.yaml": 125 | unable to recognize "STDIN": no matches for kind "Ingress" in version "extensions/v1beta1" 126 | unable to recognize "STDIN": no matches for kind "Ingress" in version "extensions/v1beta1" 127 | `, 128 | }, 129 | }, 130 | }, { 131 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 132 | ObjectMeta: metav1.ObjectMeta{ 133 | Name: "main", 134 | Namespace: "buz", 135 | }, 136 | Status: kubeapplierv1alpha1.WaybillStatus{ 137 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 138 | Command: "/usr/local/bin/kustomize build /tmp/run_buz_main_repo_2305192794/dev/buz | /usr/local/bin/kubectl apply -f - --token= -n buz --dry-run=none --prune --all --prune-allowlist=core/v1/ConfigMap serviceaccount/job-trigger unchanged serviceaccount/kube-applier-delegate unchanged", 139 | Commit: "22c815614b", 140 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 141 | Finished: metav1.Time{Time: fixedTime}, 142 | Success: true, 143 | Type: "Scheduled run", 144 | Output: `namespace/buz unchanged 145 | serviceaccount/kube-applier-delegate unchanged 146 | rolebinding.rbac.authorization.k8s.io/kube-applier-delegate unchanged 147 | configmap/postgres-init unchanged 148 | configmap/postgres-env unchanged 149 | service/postgres unchanged 150 | service/webapp unchanged 151 | limitrange/default configured 152 | persistentvolumeclaim/postgres unchanged 153 | deployment.apps/postgres unchanged 154 | deployment.apps/webapp configured 155 | waybill.kube-applier.io/main unchanged 156 | networkpolicy.networking.k8s.io/default unchanged 157 | `, 158 | }, 159 | }, 160 | }, 161 | { 162 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 163 | ObjectMeta: metav1.ObjectMeta{ 164 | Name: "main", 165 | Namespace: "eng", 166 | }, 167 | Status: kubeapplierv1alpha1.WaybillStatus{ 168 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 169 | Command: "/usr/local/bin/kustomize build /tmp/run_eng_main_repo_2305192794/dev/eng | /usr/local/bin/kubectl apply -f - --token= -n eng --dry-run=none --prune --all --prune-allowlist=core/v1/ConfigMap serviceaccount/job-trigger unchanged serviceaccount/kube-applier-delegate unchanged", 170 | Commit: "22c815614b", 171 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 172 | Finished: metav1.Time{Time: fixedTime}, 173 | Success: true, 174 | Type: "Scheduled run", 175 | Output: `namespace/eng unchanged 176 | serviceaccount/kube-applier-delegate unchanged 177 | rolebinding.rbac.authorization.k8s.io/kube-applier-delegate unchanged 178 | waybill.kube-applier.io/main unchanged 179 | networkpolicy.networking.k8s.io/default unchanged 180 | `, 181 | }, 182 | }, 183 | }, 184 | { 185 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 186 | ObjectMeta: metav1.ObjectMeta{ 187 | Name: "main", 188 | Namespace: "fuz", 189 | }, 190 | Status: kubeapplierv1alpha1.WaybillStatus{ 191 | LastRun: &kubeapplierv1alpha1.WaybillStatusRun{ 192 | Command: "/usr/local/bin/kustomize build /tmp/run_fuz_main_repo_2305192794/dev/fuz | /usr/local/bin/kubectl apply -f - --token= -n fuz --dry-run=none --prune --all --prune-allowlist=core/v1/ConfigMap serviceaccount/job-trigger unchanged serviceaccount/kube-applier-delegate unchanged", 193 | Commit: "22c815614b", 194 | Started: metav1.Time{Time: fixedTime.Add(-time.Minute)}, 195 | Finished: metav1.Time{Time: fixedTime}, 196 | Success: true, 197 | Type: "Scheduled run", 198 | Output: `namespace/fuz unchanged 199 | serviceaccount/kube-applier-delegate unchanged 200 | rolebinding.rbac.authorization.k8s.io/kube-applier-delegate unchanged 201 | configmap/postgres-init unchanged 202 | configmap/postgres-env unchanged 203 | service/postgres unchanged 204 | service/webapp unchanged 205 | limitrange/default configured 206 | persistentvolumeclaim/postgres unchanged 207 | deployment.apps/postgres unchanged 208 | deployment.apps/webapp configured 209 | waybill.kube-applier.io/main unchanged 210 | networkpolicy.networking.k8s.io/default unchanged 211 | Warning: autoscaling/v2beta1 HorizontalPodAutoscaler is deprecated in v1.22+, unavailable in v1.25+; use autoscaling/v2beta2 HorizontalPodAutoscaler 212 | Warning: batch/v1beta1 CronJob is deprecated in v1.21+, unavailable in v1.25+; use batch/v1 CronJob 213 | Warning: policy/v1beta1 PodDisruptionBudget is deprecated in v1.21+, unavailable in v1.25+; use policy/v1 PodDisruptionBudget 214 | Warning: discovery.k8s.io/v1beta1 EndpointSlice is deprecated in v1.21+, unavailable in v1.25+; use discovery.k8s.io/v1 EndpointSlice 215 | `, 216 | }, 217 | }, 218 | }, 219 | } 220 | 221 | events := []corev1.Event{ 222 | { 223 | TypeMeta: metav1.TypeMeta{Kind: "Event", APIVersion: "v1"}, 224 | FirstTimestamp: metav1.Time{Time: fixedTime.Add(-4 * time.Minute)}, 225 | LastTimestamp: metav1.Time{Time: fixedTime.Add(4 * time.Minute)}, 226 | Source: corev1.EventSource{Component: "kube-applier"}, 227 | ObjectMeta: metav1.ObjectMeta{Name: "main", Namespace: "zot"}, 228 | Type: "Warning", 229 | InvolvedObject: corev1.ObjectReference{Kind: "Waybill", Namespace: "zot", Name: "main"}, 230 | Message: `failed fetching delegate token: secrets "kube-applier-delegate-token" not found`, 231 | Reason: "WaybillRunRequestFailed", 232 | }, 233 | } 234 | 235 | result := GetNamespaces(wbList, events, diffURL) 236 | 237 | templt, err := createTemplate("../templates/status.html") 238 | if err != nil { 239 | t.Errorf("error parsing template: %v\n", err) 240 | return 241 | } 242 | 243 | rendered := &bytes.Buffer{} 244 | err = templt.ExecuteTemplate(rendered, "index", result) 245 | if err != nil { 246 | t.Errorf("error executing template: %v\n", err) 247 | return 248 | } 249 | 250 | // read actual test html output file 251 | want, err := os.ReadFile("../testdata/web/testStatusPage.html") 252 | if err != nil { 253 | t.Errorf("error reading test file: %v\n", err) 254 | return 255 | } 256 | 257 | if diff := cmp.Diff(string(want), rendered.String(), removeSpaceEmptyLine); diff != "" { 258 | t.Errorf("ExecuteTemplate mismatch (-want +got):\n%s", diff) 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /webserver/webserver.go: -------------------------------------------------------------------------------- 1 | // Package webserver implements the Webserver struct which can serve the 2 | // kube-applier status page and prometheus metrics, as well as receive run 3 | // requests from users. 4 | package webserver 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "html/template" 13 | "net/http" 14 | "time" 15 | 16 | "github.com/gorilla/mux" 17 | 18 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 19 | "github.com/utilitywarehouse/kube-applier/client" 20 | "github.com/utilitywarehouse/kube-applier/log" 21 | "github.com/utilitywarehouse/kube-applier/run" 22 | "github.com/utilitywarehouse/kube-applier/sysutil" 23 | "github.com/utilitywarehouse/kube-applier/webserver/oidc" 24 | ) 25 | 26 | const ( 27 | defaultServerTemplatePath = "templates/status.html" 28 | ) 29 | 30 | // WebServer struct 31 | type WebServer struct { 32 | Authenticator *oidc.Authenticator 33 | Clock sysutil.ClockInterface 34 | DiffURLFormat string 35 | KubeClient *client.Client 36 | ListenPort int 37 | RunQueue chan<- run.Request 38 | StatusTimeout time.Duration 39 | TemplatePath string 40 | server *http.Server 41 | } 42 | 43 | // StatusPageHandler implements the http.Handler interface and serves a status 44 | // page with info about the most recent applier run. 45 | type StatusPageHandler struct { 46 | Authenticator *oidc.Authenticator 47 | Clock sysutil.ClockInterface 48 | DiffURLFormat string 49 | KubeClient *client.Client 50 | Template *template.Template 51 | Timeout time.Duration 52 | } 53 | 54 | // ServeHTTP populates the status page template with data and serves it when 55 | // there is a request. 56 | func (s *StatusPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 57 | if s.Authenticator != nil { 58 | _, err := s.Authenticator.Authenticate(r.Context(), w, r) 59 | if errors.Is(err, oidc.ErrRedirectRequired) { 60 | return 61 | } 62 | if err != nil { 63 | http.Error(w, "Error: Authentication failed", http.StatusInternalServerError) 64 | log.Logger("webserver").Error("Authentication failed", "error", err, "time", s.Clock.Now().String()) 65 | return 66 | } 67 | } 68 | 69 | log.Logger("webserver").Info("Applier status request", "time", s.Clock.Now().String()) 70 | if s.Template == nil { 71 | http.Error(w, "Error: Unable to load HTML template", http.StatusInternalServerError) 72 | log.Logger("webserver").Error("Request failed", "error", "No template found", "time", s.Clock.Now().String()) 73 | return 74 | } 75 | ctx, cancel := context.WithTimeout(context.Background(), s.Timeout) 76 | defer cancel() 77 | waybills, err := s.KubeClient.ListWaybills(ctx) 78 | if err != nil { 79 | http.Error(w, fmt.Sprintf("Error: Unable to list Waybill resources: %v", err), http.StatusInternalServerError) 80 | log.Logger("webserver").Error("Unable to list Waybill resources", "error", err, "time", s.Clock.Now().String()) 81 | return 82 | } 83 | events, err := s.KubeClient.ListWaybillEvents(ctx) 84 | if err != nil { 85 | http.Error(w, fmt.Sprintf("Error: Unable to list Waybill events: %v", err), http.StatusInternalServerError) 86 | log.Logger("webserver").Error("Unable to list Waybill events", "error", err, "time", s.Clock.Now().String()) 87 | return 88 | } 89 | result := GetNamespaces(waybills, events, s.DiffURLFormat) 90 | 91 | rendered := &bytes.Buffer{} 92 | if err := s.Template.ExecuteTemplate(rendered, "index", result); err != nil { 93 | http.Error(w, "Error: Unable to render HTML template", http.StatusInternalServerError) 94 | log.Logger("webserver").Error("Request failed", "error", http.StatusInternalServerError, "time", s.Clock.Now().String(), "err", err) 95 | return 96 | } 97 | w.WriteHeader(http.StatusOK) 98 | if _, err := rendered.WriteTo(w); err != nil { 99 | log.Logger("webserver").Error("Request failed", "error", http.StatusInternalServerError, "time", s.Clock.Now().String(), "err", err) 100 | } 101 | log.Logger("webserver").Info("Request completed successfully", "time", s.Clock.Now().String()) 102 | } 103 | 104 | // ForceRunHandler implements the http.Handle interface and serves an API 105 | // endpoint for forcing a new run. 106 | type ForceRunHandler struct { 107 | Authenticator *oidc.Authenticator 108 | KubeClient *client.Client 109 | RunQueue chan<- run.Request 110 | } 111 | 112 | // ServeHTTP handles requests for forcing a run by attempting to add to the 113 | // runQueue, and writes a response including the result and a relevant message. 114 | func (f *ForceRunHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 115 | log.Logger("webserver").Info("Force run requested") 116 | var data struct { 117 | Result string `json:"result"` 118 | Message string `json:"message"` 119 | } 120 | 121 | switch r.Method { 122 | case "POST": 123 | var ( 124 | userEmail string 125 | err error 126 | ) 127 | if f.Authenticator != nil { 128 | userEmail, err = f.Authenticator.UserEmail(r.Context(), r) 129 | if err != nil { 130 | data.Result = "error" 131 | data.Message = "not authenticated" 132 | log.Logger("webserver").Error(data.Message, "error", err) 133 | w.WriteHeader(http.StatusForbidden) 134 | break 135 | } 136 | } 137 | 138 | if err := r.ParseForm(); err != nil { 139 | data.Result = "error" 140 | data.Message = "could not parse form data" 141 | log.Logger("webserver").Error(data.Message, "error", err) 142 | w.WriteHeader(http.StatusBadRequest) 143 | break 144 | } 145 | 146 | ns := r.FormValue("namespace") 147 | if ns == "" { 148 | data.Result = "error" 149 | data.Message = "empty namespace value" 150 | log.Logger("webserver").Error(data.Message) 151 | w.WriteHeader(http.StatusBadRequest) 152 | break 153 | } 154 | 155 | waybills, err := f.KubeClient.ListWaybills(r.Context()) 156 | if err != nil { 157 | data.Result = "error" 158 | data.Message = "cannot list Waybills" 159 | log.Logger("webserver").Error(data.Message, "error", err) 160 | w.WriteHeader(http.StatusInternalServerError) 161 | break 162 | } 163 | 164 | var waybill *kubeapplierv1alpha1.Waybill 165 | for i := range waybills { 166 | if waybills[i].Namespace == ns { 167 | waybill = &waybills[i] 168 | break 169 | } 170 | } 171 | if waybill == nil { 172 | data.Result = "error" 173 | data.Message = fmt.Sprintf("cannot find Waybills in namespace '%s'", ns) 174 | w.WriteHeader(http.StatusBadRequest) 175 | break 176 | } 177 | 178 | if f.Authenticator != nil { 179 | // if the user can patch the Waybill, they are allowed to force a run 180 | hasAccess, err := f.KubeClient.HasAccess(r.Context(), waybill, userEmail, "patch") 181 | if !hasAccess { 182 | data.Result = "error" 183 | data.Message = fmt.Sprintf("user %s is not allowed to force a run on waybill %s/%s", userEmail, waybill.Namespace, waybill.Name) 184 | if err != nil { 185 | log.Logger("webserver").Error(data.Message, "error", err) 186 | } 187 | w.WriteHeader(http.StatusForbidden) 188 | break 189 | } 190 | } 191 | 192 | run.Enqueue(f.RunQueue, run.ForcedRun, waybill) 193 | data.Result = "success" 194 | data.Message = "Run queued" 195 | w.WriteHeader(http.StatusOK) 196 | default: 197 | data.Result = "error" 198 | data.Message = "must be a POST request" 199 | w.WriteHeader(http.StatusBadRequest) 200 | } 201 | 202 | w.Header().Set("Content-Type", "waybill/json; charset=UTF-8") 203 | json.NewEncoder(w).Encode(data) 204 | } 205 | 206 | // Start starts the webserver using the given port, and sets up handlers for: 207 | // 1. Status page 208 | // 2. Metrics 209 | // 3. Static content 210 | // 4. Endpoint for forcing a run 211 | func (ws *WebServer) Start() error { 212 | if ws.server != nil { 213 | return fmt.Errorf("WebServer already running") 214 | } 215 | 216 | log.Logger("webserver").Info("Launching") 217 | 218 | templatePath := ws.TemplatePath 219 | if templatePath == "" { 220 | templatePath = defaultServerTemplatePath 221 | } 222 | template, err := createTemplate(templatePath) 223 | if err != nil { 224 | return err 225 | } 226 | 227 | m := mux.NewRouter() 228 | addStatusEndpoints(m) 229 | statusPageHandler := &StatusPageHandler{ 230 | ws.Authenticator, 231 | ws.Clock, 232 | ws.DiffURLFormat, 233 | ws.KubeClient, 234 | template, 235 | ws.StatusTimeout, 236 | } 237 | forceRunHandler := &ForceRunHandler{ 238 | ws.Authenticator, 239 | ws.KubeClient, 240 | ws.RunQueue, 241 | } 242 | m.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("static")))) 243 | m.PathPrefix("/api/v1/forceRun").Handler(forceRunHandler) 244 | m.PathPrefix("/").Handler(statusPageHandler) 245 | 246 | ws.server = &http.Server{ 247 | Addr: fmt.Sprintf(":%v", ws.ListenPort), 248 | Handler: m, 249 | ErrorLog: log.Logger("http.Server").StandardLogger(nil), 250 | } 251 | 252 | go func() { 253 | if err = ws.server.ListenAndServe(); err != nil { 254 | if !errors.Is(err, http.ErrServerClosed) { 255 | log.Logger("webserver").Error("Shutdown", "error", err) 256 | } 257 | log.Logger("webserver").Info("Shutdown") 258 | } 259 | }() 260 | 261 | return nil 262 | } 263 | 264 | // Shutdown gracefully shuts the webserver down. 265 | func (ws *WebServer) Shutdown() error { 266 | err := ws.server.Shutdown(context.Background()) 267 | ws.server = nil 268 | return err 269 | } 270 | -------------------------------------------------------------------------------- /webserver/webserver_test.go: -------------------------------------------------------------------------------- 1 | package webserver 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | . "github.com/onsi/ginkgo" 12 | . "github.com/onsi/gomega" 13 | corev1 "k8s.io/api/core/v1" 14 | "k8s.io/apimachinery/pkg/api/errors" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | 17 | kubeapplierv1alpha1 "github.com/utilitywarehouse/kube-applier/apis/kubeapplier/v1alpha1" 18 | "github.com/utilitywarehouse/kube-applier/run" 19 | ) 20 | 21 | // TODO: this is essentially duplication from the run package, can we share? 22 | func testWebServerDrainRequests(requests <-chan run.Request) func() []run.Request { 23 | ret := []run.Request{} 24 | finished := make(chan bool) 25 | 26 | go func() { 27 | for r := range requests { 28 | ret = append(ret, r) 29 | } 30 | close(finished) 31 | }() 32 | 33 | return func() []run.Request { 34 | <-finished 35 | return ret 36 | } 37 | } 38 | 39 | var _ = Describe("WebServer", func() { 40 | var ( 41 | testRunQueue chan run.Request 42 | testWebServer WebServer 43 | testWebServerRequests func() []run.Request 44 | ) 45 | 46 | BeforeEach(func() { 47 | testRunQueue = make(chan run.Request) 48 | testWebServerRequests = testWebServerDrainRequests(testRunQueue) 49 | testWebServer = WebServer{ 50 | ListenPort: 35432, 51 | Clock: &zeroClock{}, 52 | DiffURLFormat: "http://foo.bar/diff/%s", 53 | KubeClient: testKubeClient, 54 | RunQueue: testRunQueue, 55 | StatusTimeout: time.Second * 5, 56 | TemplatePath: "../templates/status.html", 57 | } 58 | Expect(testWebServer.Start()).To(BeNil()) 59 | }) 60 | 61 | Context("When running", func() { 62 | wbList := []kubeapplierv1alpha1.Waybill{ 63 | { 64 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 65 | ObjectMeta: metav1.ObjectMeta{ 66 | Name: "main", 67 | Namespace: "foo", 68 | }, 69 | }, 70 | { 71 | TypeMeta: metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"}, 72 | ObjectMeta: metav1.ObjectMeta{ 73 | Name: "main", 74 | Namespace: "bar", 75 | }, 76 | }, 77 | } 78 | 79 | It("Should trigger a ForcedRun when a valid request is made", func() { 80 | testEnsureWaybills(wbList) 81 | 82 | v := url.Values{} 83 | res, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/forceRun", testWebServer.ListenPort)) 84 | Expect(err).To(BeNil()) 85 | body, err := io.ReadAll(res.Body) 86 | Expect(err).To(BeNil()) 87 | Expect(res.StatusCode).To(Equal(http.StatusBadRequest)) 88 | Expect(body).To(MatchJSON(`{"result": "error", "message": "must be a POST request"}`)) 89 | 90 | res, err = http.PostForm(fmt.Sprintf("http://localhost:%d/api/v1/forceRun", testWebServer.ListenPort), v) 91 | Expect(err).To(BeNil()) 92 | body, err = io.ReadAll(res.Body) 93 | Expect(err).To(BeNil()) 94 | Expect(res.StatusCode).To(Equal(http.StatusBadRequest)) 95 | Expect(body).To(MatchJSON(`{"result": "error", "message": "empty namespace value"}`)) 96 | 97 | v.Set("namespace", "invalid") 98 | res, err = http.PostForm(fmt.Sprintf("http://localhost:%d/api/v1/forceRun", testWebServer.ListenPort), v) 99 | Expect(err).To(BeNil()) 100 | body, err = io.ReadAll(res.Body) 101 | Expect(err).To(BeNil()) 102 | Expect(res.StatusCode).To(Equal(http.StatusBadRequest)) 103 | Expect(body).To(MatchJSON(`{"result": "error", "message": "cannot find Waybills in namespace 'invalid'"}`)) 104 | 105 | v.Set("namespace", wbList[0].Namespace) 106 | res, err = http.PostForm(fmt.Sprintf("http://localhost:%d/api/v1/forceRun", testWebServer.ListenPort), v) 107 | Expect(err).To(BeNil()) 108 | body, err = io.ReadAll(res.Body) 109 | Expect(err).To(BeNil()) 110 | Expect(res.StatusCode).To(Equal(http.StatusOK)) 111 | Expect(body).To(MatchJSON(`{"result": "success", "message": "Run queued"}`)) 112 | 113 | testWebServer.Shutdown() 114 | close(testRunQueue) 115 | 116 | Expect(testWebServerRequests()).To(Equal([]run.Request{{ 117 | Type: run.ForcedRun, 118 | Waybill: &wbList[0], 119 | }})) 120 | }) 121 | 122 | It("Should render HTML on the root page", func() { 123 | var res *http.Response 124 | Eventually( 125 | func() error { 126 | r, err := http.Get(fmt.Sprintf("http://localhost:%d/", testWebServer.ListenPort)) 127 | if err != nil { 128 | return err 129 | } 130 | res = r 131 | return nil 132 | }, 133 | time.Second*15, 134 | time.Second, 135 | ).Should(BeNil()) 136 | 137 | body, err := io.ReadAll(res.Body) 138 | Expect(err).To(BeNil()) 139 | Expect(res.StatusCode).To(Equal(http.StatusOK)) 140 | Expect(body).ToNot(BeEmpty()) 141 | 142 | testWebServer.Shutdown() 143 | close(testRunQueue) 144 | Expect(testWebServerRequests()).To(Equal([]run.Request{})) 145 | }) 146 | }) 147 | }) 148 | 149 | // TODO: this is essentially duplication from the run package (but with values 150 | // instead of pointers), can we share? 151 | func testEnsureWaybills(wbList []kubeapplierv1alpha1.Waybill) { 152 | for i := range wbList { 153 | err := testKubeClient.GetClient().Create(context.TODO(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wbList[i].Namespace}}) 154 | if err != nil { 155 | Expect(errors.IsAlreadyExists(err)).To(BeTrue()) 156 | } 157 | // The ResourceVersion swapping is to prevent the respective error from 158 | // Create() which makes it difficult to handle it below. 159 | rv := wbList[i].ResourceVersion 160 | wbList[i].ResourceVersion = "" 161 | err = testKubeClient.GetClient().Create(context.TODO(), &wbList[i]) 162 | if err != nil && errors.IsAlreadyExists(err) { 163 | wbList[i].ResourceVersion = rv 164 | Expect(testKubeClient.UpdateWaybill(context.TODO(), &wbList[i])).To(BeNil()) 165 | } else { 166 | Expect(err).To(BeNil()) 167 | } 168 | if wbList[i].Status.LastRun != nil { 169 | // UpdateStatus changes SelfLink to the status sub-resource but we 170 | // should revert the change for tests to pass 171 | selfLink := wbList[i].ObjectMeta.SelfLink 172 | Expect(testKubeClient.UpdateWaybillStatus(context.TODO(), &wbList[i])).To(BeNil()) 173 | wbList[i].ObjectMeta.SelfLink = selfLink 174 | } 175 | // This is a workaround for Equal checks to work below. 176 | // Apparently, List will return Waybills with TypeMeta but 177 | // Get and Create (which updates the struct) do not. 178 | wbList[i].TypeMeta = metav1.TypeMeta{APIVersion: "kube-applier.io/v1alpha1", Kind: "Waybill"} 179 | } 180 | } 181 | --------------------------------------------------------------------------------