├── .dockerignore ├── .github └── workflows │ ├── buildtest.yml │ ├── codeql.yml │ ├── e2e.yml │ ├── image-push-master.yml │ └── image-push-release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── installer │ └── main.go └── webhook │ └── main.go ├── deployments ├── auth.yaml ├── pdb.yaml ├── server.yaml ├── service.yaml └── webhook.yaml ├── docs └── installation.md ├── go.mod ├── go.sum ├── pkg ├── controlswitches │ ├── controlswitches.go │ ├── controlswitches_suite_test.go │ ├── controlswitches_test.go │ └── controlswitchesaccessors.go ├── installer │ └── installer.go ├── tools │ └── cache.go ├── types │ └── types.go ├── userdefinedinjections │ ├── userdefinedinjections.go │ ├── userdefinedinjections_suite_test.go │ └── userdefinedinjections_test.go └── webhook │ ├── tlsutils.go │ ├── webhook.go │ ├── webhook_suite_test.go │ └── webhook_test.go ├── scripts ├── build-image.sh ├── build.sh ├── control-plane-additions │ ├── ac.yaml │ └── mutatingkubeconfig.yaml ├── e2e_cleanup.sh ├── e2e_get_tools.sh ├── e2e_setup_cluster.sh ├── e2e_teardown_cluster.sh ├── test.sh ├── webhook-deployment.sh └── webhook-patch-ca-bundle.sh └── test ├── README.md ├── e2e ├── control_switches_test.go ├── e2e_tests_suite_test.go ├── hugepages_test.go ├── nodeselector_test.go ├── resourcename_test.go └── userdefinedinjections_test.go └── util ├── configmap.go ├── features.go ├── images.go ├── namespace.go ├── networkattachmentdefinition.go └── pod.go /.dockerignore: -------------------------------------------------------------------------------- 1 | */tmp/* 2 | bin 3 | -------------------------------------------------------------------------------- /.github/workflows/buildtest.yml: -------------------------------------------------------------------------------- 1 | #Originally from https://raw.githubusercontent.com/intel/multus-cni/master/.github/workflows/go-build-test-amd64.yml 2 | name: Go-build-and-test-amd64 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: "0 10 * * *" # everyday at 10 am 8 | jobs: 9 | build: 10 | strategy: 11 | matrix: 12 | go-version: [1.20.x, 1.21.x] 13 | os: [ubuntu-24.04] 14 | runs-on: ${{ matrix.os }} 15 | env: 16 | GO111MODULE: on 17 | TARGET: amd64 18 | steps: 19 | - name: Set up Go 20 | uses: actions/setup-go@v3 21 | with: 22 | go-version: ${{ matrix.go-version }} 23 | 24 | - name: Check out code into the Go module directory 25 | uses: actions/checkout@v2 26 | 27 | - name: Build 28 | run: GOARCH="${TARGET}" ./scripts/build.sh 29 | 30 | - name: Go test 31 | run: ./scripts/test.sh 32 | 33 | sriov-operator-e2e-test: 34 | name: SR-IOV operator e2e tests 35 | needs: [ build ] 36 | runs-on: [ sriov ] 37 | env: 38 | TEST_REPORT_PATH: k8s-artifacts 39 | steps: 40 | - name: Check out the repo 41 | uses: actions/checkout@v3 42 | 43 | - name: build network resource injector image 44 | run: podman build -f Dockerfile -t ghaction-network-resource-injector:pr-${{github.event.pull_request.number}} . 45 | 46 | - name: Check out sriov operator's code 47 | uses: actions/checkout@v2 48 | with: 49 | repository: k8snetworkplumbingwg/sriov-network-operator 50 | path: sriov-network-operator-wc 51 | 52 | - name: run test 53 | run: make test-e2e-conformance-virtual-k8s-cluster-ci 54 | working-directory: sriov-network-operator-wc 55 | env: 56 | LOCAL_NETWORK_RESOURCES_INJECTOR_IMAGE: ghaction-network-resource-injector:pr-${{github.event.pull_request.number}} 57 | 58 | - uses: actions/upload-artifact@v3 59 | if: always() 60 | with: 61 | name: ${{ env.TEST_REPORT_PATH }} 62 | path: ./sriov-network-operator-wc/${{ env.TEST_REPORT_PATH }} 63 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: "24 17 * * 5" 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-24.04 15 | permissions: 16 | actions: read 17 | contents: read 18 | security-events: write 19 | 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | language: [ go ] 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v3 28 | 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v2 31 | with: 32 | languages: ${{ matrix.language }} 33 | queries: +security-and-quality 34 | 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v2 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v2 40 | with: 41 | category: "/language:${{ matrix.language }}" 42 | -------------------------------------------------------------------------------- /.github/workflows/e2e.yml: -------------------------------------------------------------------------------- 1 | name: e2e 2 | on: [pull_request] 3 | jobs: 4 | e2e-test-cloud: 5 | name: E2E test cloud (GitHub VM) 6 | runs-on: ubuntu-24.04 7 | strategy: 8 | matrix: 9 | go-version: ['1.20', '1.21'] 10 | steps: 11 | - name: Set up Go version 12 | uses: actions/setup-go@v3 13 | with: 14 | go-version: ${{ matrix.go-version }} 15 | 16 | - name: Checkout code into the Go module directory 17 | uses: actions/checkout@v2 18 | 19 | - name: Get tools, setup KinD cluster test environment 20 | run: source scripts/e2e_get_tools.sh && ./scripts/e2e_setup_cluster.sh 21 | 22 | - name: Execute E2E tests 23 | run: go test -timeout 60m ./test/e2e/... 24 | 25 | # Disable for now because there is no dedicated server on which those tests can be run by GitHub. 26 | # e2e-test-self-hosted: 27 | # name: E2E test self-hosted 28 | # environment: nri-team 29 | # runs-on: [self-hosted, Linux, hugepages] 30 | # steps: 31 | # - name: Set up Go version 32 | # uses: actions/setup-go@v1 33 | # with: 34 | # go-version: 1.13 35 | 36 | # - name: Checkout code into the Go module directory 37 | # uses: actions/checkout@v2 38 | 39 | # - name: Get tools, setup KinD cluster test environment 40 | # run: source scripts/e2e_get_tools.sh && ./scripts/e2e_setup_cluster.sh 41 | 42 | # - name: Execute E2E tests 43 | # run: go test -timeout 60m ./test/e2e/... 44 | 45 | # - name: Tear down KinD cluster 46 | # if: always() 47 | # run: ./scripts/e2e_teardown_cluster.sh 48 | -------------------------------------------------------------------------------- /.github/workflows/image-push-master.yml: -------------------------------------------------------------------------------- 1 | name: "Push images on merge to master" 2 | 3 | env: 4 | IMAGE_NAME: ghcr.io/${{ github.repository }} 5 | BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/ppc64le 6 | 7 | on: 8 | push: 9 | branches: 10 | - master 11 | jobs: 12 | build-and-push-image-nri: 13 | runs-on: ubuntu-24.04 14 | steps: 15 | - name: Check out the repo 16 | uses: actions/checkout@v3 17 | 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v2 20 | 21 | # Add support for more platforms with QEMU (optional) 22 | # https://github.com/docker/setup-qemu-action 23 | - name: Set up QEMU 24 | uses: docker/setup-qemu-action@v3 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v2 28 | 29 | - name: Login to Docker 30 | uses: docker/login-action@v2 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.repository_owner }} 34 | password: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Docker meta 37 | id: docker_meta 38 | uses: docker/metadata-action@v4 39 | with: 40 | images: ${{ env.IMAGE_NAME }} 41 | 42 | - name: Build and push network-resources-injector 43 | uses: docker/build-push-action@v4 44 | with: 45 | context: . 46 | push: true 47 | platforms: ${{ env.BUILD_PLATFORMS }} 48 | tags: | 49 | ${{ env.IMAGE_NAME }}:latest 50 | ${{ env.IMAGE_NAME }}:${{ github.sha }} 51 | labels: ${{ steps.docker_meta.outputs.labels }} 52 | file: ./Dockerfile 53 | -------------------------------------------------------------------------------- /.github/workflows/image-push-release.yml: -------------------------------------------------------------------------------- 1 | name: "Push images on release" 2 | 3 | env: 4 | IMAGE_NAME: ghcr.io/${{ github.repository }} 5 | BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/ppc64le 6 | 7 | on: 8 | push: 9 | tags: 10 | - v* 11 | jobs: 12 | build-and-push-image-nri: 13 | runs-on: ubuntu-24.04 14 | steps: 15 | - name: Check out the repo 16 | uses: actions/checkout@v3 17 | 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v1 20 | 21 | # Add support for more platforms with QEMU (optional) 22 | # https://github.com/docker/setup-qemu-action 23 | - name: Set up QEMU 24 | uses: docker/setup-qemu-action@v3 25 | 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v2 28 | 29 | - name: Login to Docker 30 | uses: docker/login-action@v2 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.repository_owner }} 34 | password: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Docker meta 37 | id: docker_meta 38 | uses: docker/metadata-action@v4 39 | with: 40 | images: ${{ env.IMAGE_NAME }} 41 | flavor: | 42 | latest=false 43 | 44 | - name: Build and push network-resources-injector 45 | uses: docker/build-push-action@v4 46 | with: 47 | context: . 48 | push: true 49 | platforms: ${{ env.BUILD_PLATFORMS }} 50 | tags: | 51 | ${{ steps.docker_meta.outputs.tags }} 52 | labels: ${{ steps.docker_meta.outputs.labels }} 53 | file: ./Dockerfile 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/go,macos,linux,windows,visualstudiocode 2 | 3 | ### Go ### 4 | # Binaries for programs and plugins 5 | bin 6 | *.exe 7 | *.exe~ 8 | *.dll 9 | *.so 10 | *.dylib 11 | .gopath/* 12 | 13 | # Test binary, build with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | ### Linux ### 20 | *~ 21 | 22 | # temporary files which can be created if a process still has a handle open of a deleted file 23 | .fuse_hidden* 24 | 25 | # KDE directory preferences 26 | .directory 27 | 28 | # Linux trash folder which might appear on any partition or disk 29 | .Trash-* 30 | 31 | # .nfs files are created when an open file is removed but is still being accessed 32 | .nfs* 33 | 34 | ### macOS ### 35 | *.DS_Store 36 | .AppleDouble 37 | .LSOverride 38 | 39 | # Icon must end with two \r 40 | Icon 41 | 42 | # Thumbnails 43 | ._* 44 | 45 | # Files that might appear in the root of a volume 46 | .DocumentRevisions-V100 47 | .fseventsd 48 | .Spotlight-V100 49 | .TemporaryItems 50 | .Trashes 51 | .VolumeIcon.icns 52 | .com.apple.timemachine.donotpresent 53 | 54 | # Directories potentially created on remote AFP share 55 | .AppleDB 56 | .AppleDesktop 57 | Network Trash Folder 58 | Temporary Items 59 | .apdisk 60 | 61 | ### VisualStudioCode ### 62 | .vscode/* 63 | !.vscode/settings.json 64 | !.vscode/tasks.json 65 | !.vscode/launch.json 66 | !.vscode/extensions.json 67 | .history 68 | 69 | ### Windows ### 70 | # Windows thumbnail cache files 71 | Thumbs.db 72 | ehthumbs.db 73 | ehthumbs_vista.db 74 | 75 | # Folder config file 76 | Desktop.ini 77 | 78 | # Recycle Bin used on file shares 79 | $RECYCLE.BIN/ 80 | 81 | # Windows Installer files 82 | *.cab 83 | *.msi 84 | *.msm 85 | *.msp 86 | 87 | # Windows shortcuts 88 | *.lnk 89 | 90 | 91 | # End of https://www.gitignore.io/api/go,macos,linux,windows,visualstudiocode 92 | /test/tmp/* 93 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.21-alpine as builder 16 | COPY . /usr/src/network-resources-injector 17 | WORKDIR /usr/src/network-resources-injector 18 | RUN apk add --update --virtual build-dependencies build-base bash && \ 19 | make 20 | 21 | FROM alpine:3.11 22 | USER 1001 23 | COPY --from=builder /usr/src/network-resources-injector/bin/webhook /usr/bin/ 24 | COPY --from=builder /usr/src/network-resources-injector/bin/installer /usr/bin/ 25 | 26 | CMD ["webhook"] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 © Intel Corporation 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | SHELL := /usr/bin/env bash 16 | 17 | default : 18 | scripts/build.sh 19 | 20 | image : 21 | scripts/build-image.sh 22 | 23 | .PHONY: test 24 | test : 25 | scripts/test.sh 26 | 27 | vendor : 28 | go mod tidy && go mod vendor 29 | 30 | e2e: 31 | source scripts/e2e_get_tools.sh && scripts/e2e_setup_cluster.sh 32 | go test -timeout 40m -v ./test/e2e/... 33 | 34 | e2e-clean: 35 | source scripts/e2e_get_tools.sh && scripts/e2e_teardown_cluster.sh 36 | scripts/e2e_cleanup.sh 37 | 38 | deps-update: ; $(info Updating dependencies...) @ ## Update dependencies 39 | @go mod tidy 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | * [Network Resources Injector](#network-resources-injector) 2 | * [Getting started](#getting-started) 3 | * [Network resources injection example](#network-resources-injection-example) 4 | * [Vendoring](#vendoring) 5 | * [Security](#security) 6 | * [Disable adding client CAs to server TLS endpoint](#disable-adding-client-cas-to-server-tls-endpoint) 7 | * [Client CAs](#client-cas) 8 | * [Additional features](#additional-features) 9 | * [Features control switches](#features-control-switches) 10 | * [Expose Hugepages via Downward API](#expose-hugepages-via-downward-api) 11 | * [Node Selector](#node-selector) 12 | * [User Defined Injections](#user-defined-injections) 13 | * [Test](#test) 14 | * [Unit tests](#unit-tests) 15 | * [E2E tests using Kubernetes in Docker (KinD)](#e2e-tests-using-kubernetes-in-docker-kind) 16 | * [Contact Us](#contact-us) 17 | 18 | # Network Resources Injector 19 | 20 | [![Weekly minutes](https://img.shields.io/badge/Weekly%20Meeting%20Minutes-Mon%203pm%20GMT-blue.svg?style=plastic)](https://docs.google.com/document/d/1sJQMHbxZdeYJPgAWK1aSt6yzZ4K_8es7woVIrwinVwI) 21 | 22 | Network Resources Injector is a Kubernetes Dynamic Admission Controller application that provides functionality of patching Kubernetes pod specifications with requests and limits of custom network resources (managed by device plugins such as [k8snetworkplumbingwg/sriov-network-device-plugin](https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin)). 23 | 24 | ## Getting started 25 | 26 | To quickly build and deploy admission controller run: 27 | ``` 28 | make image 29 | kubectl apply -f deployments/auth.yaml \ 30 | -f deployments/server.yaml 31 | ``` 32 | For full installation and troubleshooting steps please see [Installation guide](docs/installation.md). 33 | 34 | ## Network resources injection example 35 | 36 | To see mutating webhook in action you're going to need to add custom resources to your Kubernetes node. In real life scenarios you're going to use network resources managed by network devices plugins, such as [k8snetworkplumbingwg/sriov-network-device-plugin](https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin). 37 | There should be [net-attach-def CRD](https://github.com/intel/multus-cni/blob/master/examples/crd.yml) already created before you start. 38 | In a terminal window start proxy, so that you can easily send HTTP requests to the Kubernetes API server: 39 | ``` 40 | kubectl proxy 41 | ``` 42 | In another terminal window, execute below command to add 4 `example.com/foo` resources. Remember to edit `` to match your cluster environment. 43 | 44 | ``` 45 | curl -s --header "Content-Type: application/json-patch+json" \ 46 | --request PATCH \ 47 | --data '[{"op": "add", "path": "/status/capacity/example.com~1foo", "value": "4"}]' \ 48 | http://localhost:8001/api/v1/nodes//status >/dev/null 49 | ``` 50 | Next, you need to create a net-attach-def linked to this `example.com/foo` resource. To achieve that execute below command: 51 | ``` 52 | cat </status 126 | kubectl delete net-attach-def foo-network 127 | kubectl delete pod webhook-demo 128 | ``` 129 | 130 | ## Vendoring 131 | To create the vendor folder invoke the following which will create a vendor folder. 132 | ```bash 133 | make vendor 134 | ``` 135 | 136 | ## Security 137 | ### Disable adding client CAs to server TLS endpoint 138 | If you wish to not add any client CAs to the servers TLS endpoint, add ```--insecure``` flag to webhook binary arguments (See [server.yaml](deployments/server.yaml)). 139 | 140 | ### Client CAs 141 | By default, we consume the client CA from the Kubernetes service account secrets directory ```/var/run/secrets/kubernetes.io/serviceaccount/```. 142 | If you wish to consume a client CA from a different location, please specify flag ```--client-ca``` with a valid path. If you wish to add more than one client CA, repeat this flag multiple times. If ```--client-ca``` is defined, the default client CA from the service account secrets directory will not be consumed. 143 | 144 | ## Additional features 145 | All additional Network Resource Injector features can be enabled by passing command line arguments to executable. It can be done by modification of arguments passed to webhook. Example yaml with deployment is here [server.yaml](deployments/server.yaml) 146 | 147 | Currently supported arguments are below. If needed, detailed description is available below in sub points. ConfigMap with runtime configuration is described below in point [Features control switches](#features-control-switches). 148 | 149 | |Argument|Default|Description|Can be set via ConfigMap| 150 | |---|---|---|---| 151 | |port|8443|The port on which to serve.|NO| 152 | |bind-address|0.0.0.0|The IP address on which to listen for the --port port.|NO| 153 | |tls-cert-file|cert.pem|File containing the default x509 Certificate for HTTPS.|NO| 154 | |tls-private-key-file|key.pem|File containing the default x509 private key matching --tls-cert-file.|NO| 155 | |insecure|false|Disable adding client CA to server TLS endpoint|NO| 156 | |client-ca|""|File containing client CA. This flag is repeatable if more than one client CA needs to be added to server|NO| 157 | |health-check-port|8444|The port to use for health check monitoring.|NO| 158 | |injectHugepageDownApi|false|Enable hugepage requests and limits into Downward API.|YES| 159 | |network-resource-name-keys|k8s.v1.cni.cncf.io/resourceName|comma separated resource name keys|YES| 160 | |honor-resources|false|Honor the existing requested resources requests & limits|YES| 161 | 162 | NOTE: Network Resource Injector would not mutate pods in kube-system namespace. 163 | 164 | ### Features control switches 165 | It is possible to control some features of Network Resource Injector with runtime configuration. NRI is watching for a ConfigMap with name **nri-control-switches** that should be available in the same namespace as NRI (default is kube-system). Below is example with full configuration that sets all features to disable state. Not all values have to be defined. User can toggle only one feature leaving others in default state. By default state, one should understand state set during webhook initialization. Could be a state set by CLI argument, default argument embedded in code or environment variable. 166 | 167 | ``` 168 | apiVersion: v1 169 | kind: ConfigMap 170 | metadata: 171 | name: nri-control-switches 172 | namespace: kube-system 173 | data: 174 | config.json: | 175 | { 176 | "features": { 177 | "enableHugePageDownApi": false, 178 | "enableHonorExistingResources": false 179 | } 180 | } 181 | 182 | ``` 183 | 184 | Set feature state is available as long as ConfigMap exists. Webhook checks for map update every 30 seconds. Please keep in mind that runtime configuration settings override all other settings. They have the highest priority. 185 | 186 | ### Expose Hugepages via Downward API 187 | In Kubernetes 1.20, an alpha feature was added to expose the requested hugepages to the container via the Downward API. 188 | Being alpha, this feature is disabled in Kubernetes by default. 189 | If enabled when Kubernetes is deployed via `FEATURE_GATES="DownwardAPIHugePages=true"`, then Network Resource Injector can be used to mutate the pod spec to publish the hugepage data to the container. To enable this functionality in Network Resource Injector, add ```--injectHugepageDownApi``` flag to webhook binary arguments (See [server.yaml](deployments/server.yaml)). 190 | 191 | > NOTE: Please note that the Network Resource Injector does not add hugepage resources to the POD specification. It means that user has to explicitly add it. This feature only exposes it to Downward API. More information about hugepages can be found within Kubernetes [specification](https://kubernetes.io/docs/tasks/manage-hugepages/scheduling-hugepages/). Snippet of how to request hugepage resources in pod spec: 192 | ``` 193 | spec: 194 | containers: 195 | - image: busybox 196 | resources: 197 | limits: 198 | hugepages-1Gi: 2Gi 199 | memory: 2Gi 200 | requests: 201 | hugepages-1Gi: 2Gi 202 | memory: 2Gi 203 | ``` 204 | 205 | Like the other Downward API provided data, hugepage information for a pod can be located by an application at the path `/etc/podnetinfo/` in the container's file system. 206 | This directory will contain the request and limit information for 1Gi/2Mb. 207 | 208 | 1Gi Hugepages: 209 | ``` 210 | Requests: /etc/podnetinfo/hugepages_1G_request_${CONTAINER_NAME} 211 | Limits: /etc/podnetinfo/hugepages_1G_limit_${CONTAINER_NAME} 212 | ``` 213 | 2Mb: Hugepages: 214 | ``` 215 | Requests: /etc/podnetinfo/hugepages_2M_request_${CONTAINER_NAME} 216 | Limits: /etc/podnetinfo/hugepages_2M_limit_${CONTAINER_NAME} 217 | ``` 218 | 219 | > NOTE: To aid the application, when hugepage fields are being requested via the Downward API, Network Resource Injector also mutates the pod spec to add the environment variable `CONTAINER_NAME` with the container's name applied. 220 | 221 | ### Node Selector 222 | If a ```NetworkAttachmentDefinition``` CR annotation ```k8s.v1.cni.cncf.io/nodeSelector``` is present and a pod utilizes this network, Network Resources Injector will add this node selection constraint into the pod spec field ```nodeSelector```. Injecting a single node selector label is currently supported. 223 | 224 | Example: 225 | ```yaml 226 | apiVersion: k8s.cni.cncf.io/v1 227 | kind: NetworkAttachmentDefinition 228 | metadata: 229 | name: test-network 230 | annotations: 231 | k8s.v1.cni.cncf.io/nodeSelector: master=eno3 232 | spec: 233 | config: '{ 234 | "cniVersion": "0.3.1", 235 | "type": "macvlan", 236 | "master": "eno3", 237 | "mode": "bridge", 238 | }' 239 | ... 240 | ``` 241 | Pod spec after modification by Network Resources Injector: 242 | ```yaml 243 | apiVersion: v1 244 | kind: Pod 245 | metadata: 246 | name: testpod 247 | annotations: 248 | k8s.v1.cni.cncf.io/networks: test-network 249 | spec: 250 | .. 251 | nodeSelector: 252 | master: eno3 253 | ``` 254 | 255 | ### User Defined Injections 256 | 257 | User Defined injections allows user to define additional injections (besides what's supported in NRI, such as ResourceName, Downward API volumes etc) in Kubernetes ConfigMap and request additional injection for individual pod based on pod label. Currently user defined injection only support injecting pod annotations. 258 | 259 | In order to use this feature, user needs to create the user defined injection ConfigMap with name `nri-control-switches` in the namespace where NRI was deployed in (`kube-system` namespace is used when there is no `NAMESPACE` environment variable passed to NRI). The ConfigMap is shared between control switches and user defined injections. The data entry in ConfigMap is in the format of key:value pair. Key is a user defined label that will be used to match with pod labels, Value is the actual injection in the format as defined by [RFC6902](https://tools.ietf.org/html/rfc6902) that will be applied to pod manifest. NRI would listen to the creation/update/deletion of this ConfigMap and update its internal data structure every 30 seconds so that subsequential creation of pods will be evaluated against the latest user defined injections. 260 | 261 | Metadata.Annotations in Pod definition is the only supported field for customization, whose `path` should be "/metadata/annotations". 262 | 263 | Below is an example of user defined injection ConfigMap: 264 | 265 | ```yaml 266 | apiVersion: v1 267 | kind: ConfigMap 268 | metadata: 269 | name: nri-control-switches 270 | namespace: kube-system 271 | data: 272 | config.json: | 273 | { 274 | "user-defined-injections": { 275 | "feature.pod.kubernetes.io_sriov-network": { 276 | "op": "add", 277 | "path": "/metadata/annotations", 278 | "value": { 279 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 280 | } 281 | } 282 | } 283 | } 284 | ``` 285 | 286 | `feature.pod.kubernetes.io/sriov-network` is a user defined label to request additional networks. Every pod that contains this label with a value set to `"true"` will be applied with the patch that's defined in the following json string. 287 | 288 | `'{"op": "add", "path": "/metadata/annotations", "value": {"k8s.v1.cni.cncf.io/networks": "sriov-net -attach-def"}}` defines how/where/what the patch shall be applied. 289 | 290 | `"op": "add"` is the action for user defined injection. In above case, it requests to add the `value` to `path` in pod manifests. 291 | 292 | `"path": "/metadata/annotations"` is the path in pod manifest to be updated. 293 | 294 | `"value": {"k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def"}}` is the value to be updated in the given `path`. 295 | 296 | > NOTE: Please be aware that current implementation supports only **add** type of JSON operation. Other types like _remove, replace, copy, move_ are not yet supported. 297 | 298 | For a pod to request user defined injection, one of its labels shall match with the labels defined in user defined injection ConfigMap. 299 | For example, with the below pod manifest: 300 | 301 | ```yaml 302 | apiVersion: v1 303 | kind: Pod 304 | metadata: 305 | name: testpod 306 | labels: 307 | feature.pod.kubernetes.io_sriov-network: "true" 308 | spec: 309 | containers: 310 | - name: app 311 | image: centos:7 312 | command: [ "/bin/bash", "-c", "--" ] 313 | args: [ "while true; do sleep 300000; done;" ] 314 | ``` 315 | 316 | NRI would update pod manifest to: 317 | 318 | ```yaml 319 | apiVersion: v1 320 | kind: Pod 321 | metadata: 322 | name: testpod 323 | labels: 324 | feature.pod.kubernetes.io_sriov-network: "true" 325 | annotations: 326 | k8s.v1.cni.cncf.io/networks: sriov-net-attach-def 327 | spec: 328 | containers: 329 | - name: app 330 | image: centos:7 331 | command: [ "/bin/bash", "-c", "--" ] 332 | args: [ "while true; do sleep 300000; done;" ] 333 | ``` 334 | 335 | > NOTE: It it worth to mention that every existing network defined in annotations.k8s.v1.cni.cncf.io/networks is going to be replaced by NRI with new value. 336 | 337 | > NOTE: NRI is only able to inject one custom definition. When user will define more key/values pairs within ConfigMap (nri-user-defined-injections), only one will be injected. 338 | 339 | ## Test 340 | ### Unit tests 341 | 342 | ``` 343 | $ make test 344 | ``` 345 | 346 | ### E2E tests using Kubernetes in Docker (KinD) 347 | Deploy KinD and run tests 348 | 349 | ``` 350 | $ make e2e 351 | ``` 352 | Note: For all tests to run you will need to provision your host with 3 Gi and 1024 Mi hugepages. 353 | 354 | Cleanup KinD deployment 355 | 356 | ``` 357 | make e2e-clean 358 | ``` 359 | 360 | ## Contact Us 361 | 362 | For any questions about Network Resources Injector, feel free to ask a question in #network-resources-injector in [NPWG slack](https://npwg-team.slack.com/), or open up a GitHub issue. Request an invite to NPWG slack [here](https://intel-corp.herokuapp.com/). 363 | -------------------------------------------------------------------------------- /cmd/installer/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | 20 | "github.com/golang/glog" 21 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/installer" 22 | ) 23 | 24 | func main() { 25 | namespace := flag.String("namespace", "kube-system", "Namespace in which all Kubernetes resources will be created.") 26 | prefix := flag.String("name", "network-resources-injector", "Prefix added to the names of all created resources.") 27 | failurePolicy := flag.String("failure-policy", "Fail", "K8 admission controller failure policy to handle unrecognized errors and timeout errors") 28 | flag.Parse() 29 | 30 | glog.Info("starting webhook installation") 31 | installer.Install(*namespace, *prefix, *failurePolicy) 32 | } 33 | -------------------------------------------------------------------------------- /cmd/webhook/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "crypto/tls" 20 | "flag" 21 | "fmt" 22 | "net/http" 23 | "os" 24 | "time" 25 | 26 | "github.com/fsnotify/fsnotify" 27 | "github.com/golang/glog" 28 | 29 | "k8s.io/apimachinery/pkg/api/errors" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | 32 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/controlswitches" 33 | netcache "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/tools" 34 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/userdefinedinjections" 35 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/webhook" 36 | ) 37 | 38 | const ( 39 | defaultClientCa = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" 40 | controlSwitchesConfigMap = "nri-control-switches" 41 | ) 42 | 43 | func main() { 44 | var namespace string 45 | var clientCAPaths webhook.ClientCAFlags 46 | 47 | /* load configuration */ 48 | port := flag.Int("port", 8443, "The port on which to serve.") 49 | address := flag.String("bind-address", "0.0.0.0", "The IP address on which to listen for the --port port.") 50 | cert := flag.String("tls-cert-file", "cert.pem", "File containing the default x509 Certificate for HTTPS.") 51 | key := flag.String("tls-private-key-file", "key.pem", "File containing the default x509 private key matching --tls-cert-file.") 52 | insecure := flag.Bool("insecure", false, "Disable adding client CA to server TLS endpoint --insecure") 53 | flag.Var(&clientCAPaths, "client-ca", "File containing client CA. This flag is repeatable if more than one client CA needs to be added to server") 54 | healthCheckPort := flag.Int("health-check-port", 8444, "The port to use for health check monitoring") 55 | enableHTTP2 := flag.Bool("enable-http2", false, "If HTTP/2 should be enabled for the webhook server.") 56 | 57 | // do initialization of control switches flags 58 | controlSwitches := controlswitches.SetupControlSwitchesFlags() 59 | 60 | // at the end when all flags are declared parse it 61 | flag.Parse() 62 | 63 | // initialize all control switches structures 64 | controlSwitches.InitControlSwitches() 65 | glog.Infof("controlSwitches: %+v", *controlSwitches) 66 | 67 | if !isValidPort(*port) { 68 | glog.Fatalf("invalid port number. Choose between 1024 and 65535") 69 | } 70 | 71 | if !controlSwitches.IsResourcesNameEnabled() { 72 | glog.Fatalf("Input argument for resourceName cannot be empty.") 73 | } 74 | 75 | if *address == "" || *cert == "" || *key == "" { 76 | glog.Fatalf("input argument(s) not defined correctly") 77 | } 78 | 79 | if len(clientCAPaths) == 0 { 80 | clientCAPaths = append(clientCAPaths, defaultClientCa) 81 | } 82 | 83 | if namespace = os.Getenv("NAMESPACE"); namespace == "" { 84 | namespace = "kube-system" 85 | } 86 | 87 | if !isValidPort(*healthCheckPort) { 88 | glog.Fatalf("Invalid health check port number. Choose between 1024 and 65535") 89 | } else if *healthCheckPort == *port { 90 | glog.Fatalf("Health check port should be different from port") 91 | } else { 92 | go func() { 93 | addr := fmt.Sprintf("%s:%d", *address, *healthCheckPort) 94 | mux := http.NewServeMux() 95 | 96 | mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { 97 | w.WriteHeader(http.StatusOK) 98 | }) 99 | err := http.ListenAndServe(addr, mux) 100 | if err != nil { 101 | glog.Fatalf("error starting health check server: %v", err) 102 | } 103 | }() 104 | } 105 | 106 | glog.Infof("starting mutating admission controller for network resources injection") 107 | 108 | keyPair, err := webhook.NewTlsKeypairReloader(*cert, *key) 109 | if err != nil { 110 | glog.Fatalf("error load certificate: %s", err.Error()) 111 | } 112 | 113 | clientCaPool, err := webhook.NewClientCertPool(&clientCAPaths, *insecure) 114 | if err != nil { 115 | glog.Fatalf("error loading client CA pool: '%s'", err.Error()) 116 | } 117 | 118 | /* init API client */ 119 | clientset := webhook.SetupInClusterClient() 120 | 121 | // initialize webhook with controlSwitches 122 | webhook.SetControlSwitches(controlSwitches) 123 | 124 | //initialize webhook with cache 125 | netAnnotationCache := netcache.Create() 126 | netAnnotationCache.Start() 127 | webhook.SetNetAttachDefCache(netAnnotationCache) 128 | 129 | userInjections := userdefinedinjections.CreateUserInjectionsStructure() 130 | webhook.SetUserInjectionStructure(userInjections) 131 | 132 | go func() { 133 | /* register handlers */ 134 | var httpServer *http.Server 135 | 136 | http.HandleFunc("/mutate", func(w http.ResponseWriter, r *http.Request) { 137 | if r.URL.Path != "/mutate" { 138 | http.NotFound(w, r) 139 | return 140 | } 141 | if r.Method != http.MethodPost { 142 | http.Error(w, "Invalid HTTP verb requested", 405) 143 | return 144 | } 145 | webhook.MutateHandler(w, r) 146 | }) 147 | 148 | /* start serving */ 149 | httpServer = &http.Server{ 150 | Addr: fmt.Sprintf("%s:%d", *address, *port), 151 | ReadTimeout: 5 * time.Second, 152 | WriteTimeout: 10 * time.Second, 153 | MaxHeaderBytes: 1 << 20, 154 | ReadHeaderTimeout: 1 * time.Second, 155 | TLSConfig: &tls.Config{ 156 | ClientAuth: webhook.GetClientAuth(*insecure), 157 | MinVersion: tls.VersionTLS12, 158 | CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384}, 159 | ClientCAs: clientCaPool.GetCertPool(), 160 | PreferServerCipherSuites: true, 161 | InsecureSkipVerify: false, 162 | CipherSuites: []uint16{ 163 | // tls 1.2 164 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 165 | tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, 166 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 167 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 168 | // tls 1.3 configuration not supported 169 | }, 170 | GetCertificate: keyPair.GetCertificateFunc(), 171 | }, 172 | // CVE-2023-39325 https://github.com/golang/go/issues/63417 173 | TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), 174 | } 175 | 176 | if *enableHTTP2 { 177 | httpServer.TLSNextProto = nil 178 | } 179 | 180 | err := httpServer.ListenAndServeTLS("", "") 181 | if err != nil { 182 | glog.Fatalf("error starting web server: %v", err) 183 | } 184 | }() 185 | 186 | /* watch the cert file and restart http sever if the file updated. */ 187 | watcher, err := fsnotify.NewWatcher() 188 | if err != nil { 189 | glog.Fatalf("error starting fsnotify watcher: %v", err) 190 | } 191 | defer watcher.Close() 192 | 193 | certUpdated := false 194 | keyUpdated := false 195 | 196 | for { 197 | watcher.Add(*cert) 198 | watcher.Add(*key) 199 | 200 | select { 201 | case event, ok := <-watcher.Events: 202 | if !ok { 203 | continue 204 | } 205 | glog.V(2).Infof("watcher event: %v", event) 206 | mask := fsnotify.Create | fsnotify.Rename | fsnotify.Remove | 207 | fsnotify.Write | fsnotify.Chmod 208 | if (event.Op & mask) != 0 { 209 | glog.V(2).Infof("modified file: %v", event.Name) 210 | if event.Name == *cert { 211 | certUpdated = true 212 | } 213 | if event.Name == *key { 214 | keyUpdated = true 215 | } 216 | if keyUpdated && certUpdated { 217 | if err := keyPair.Reload(); err != nil { 218 | glog.Fatalf("Failed to reload certificate: %v", err) 219 | } 220 | certUpdated = false 221 | keyUpdated = false 222 | } 223 | } 224 | case err, ok := <-watcher.Errors: 225 | if !ok { 226 | continue 227 | } 228 | glog.Infof("watcher error: %v", err) 229 | case <-time.After(30 * time.Second): 230 | cm, err := clientset.CoreV1().ConfigMaps(namespace).Get( 231 | context.Background(), controlSwitchesConfigMap, metav1.GetOptions{}) 232 | // only in case of API errors report an error and do not restore default values 233 | if err != nil && !errors.IsNotFound(err) { 234 | glog.Warningf("Error getting control switches configmap %s", err.Error()) 235 | continue 236 | } 237 | 238 | // to be called each time when map is present or not (in that case to restore default values) 239 | controlSwitches.ProcessControlSwitchesConfigMap(cm) 240 | userInjections.SetUserDefinedInjections(cm) 241 | } 242 | } 243 | 244 | // TODO: find a way to stop cache, should we run the above block in a go routine and make main module 245 | // to respond to terminate singal ? 246 | } 247 | 248 | func isValidPort(port int) bool { 249 | if port < 1024 || port > 65535 { 250 | return false 251 | } 252 | return true 253 | } 254 | -------------------------------------------------------------------------------- /deployments/auth.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | apiVersion: v1 16 | kind: ServiceAccount 17 | metadata: 18 | namespace: kube-system 19 | name: network-resources-injector-sa 20 | --- 21 | apiVersion: v1 22 | kind: Secret 23 | metadata: 24 | name: network-resources-injector-sa-secret 25 | namespace: kube-system 26 | annotations: 27 | kubernetes.io/service-account.name: network-resources-injector-sa 28 | type: kubernetes.io/service-account-token 29 | --- 30 | apiVersion: rbac.authorization.k8s.io/v1 31 | kind: ClusterRole 32 | metadata: 33 | name: network-resources-injector 34 | rules: 35 | - apiGroups: 36 | - "" 37 | - k8s.cni.cncf.io 38 | - extensions 39 | - apps 40 | resources: 41 | - replicationcontrollers 42 | - replicasets 43 | - daemonsets 44 | - statefulsets 45 | - pods 46 | - network-attachment-definitions 47 | verbs: 48 | - '*' 49 | --- 50 | apiVersion: rbac.authorization.k8s.io/v1 51 | kind: ClusterRole 52 | metadata: 53 | name: network-resources-injector-secrets 54 | rules: 55 | - apiGroups: 56 | - "" 57 | resources: 58 | - secrets 59 | verbs: 60 | - '*' 61 | --- 62 | apiVersion: rbac.authorization.k8s.io/v1 63 | kind: ClusterRole 64 | metadata: 65 | name: network-resources-injector-webhook-configs 66 | rules: 67 | - apiGroups: 68 | - admissionregistration.k8s.io 69 | resources: 70 | - mutatingwebhookconfigurations 71 | - validatingwebhookconfigurations 72 | verbs: 73 | - '*' 74 | --- 75 | apiVersion: rbac.authorization.k8s.io/v1 76 | kind: ClusterRole 77 | metadata: 78 | name: network-resources-injector-service 79 | rules: 80 | - apiGroups: 81 | - "" 82 | resources: 83 | - services 84 | verbs: 85 | - '*' 86 | --- 87 | apiVersion: rbac.authorization.k8s.io/v1 88 | kind: ClusterRole 89 | metadata: 90 | name: network-resources-injector-configmaps 91 | rules: 92 | - apiGroups: 93 | - "" 94 | resources: 95 | - configmaps 96 | verbs: 97 | - 'get' 98 | --- 99 | apiVersion: rbac.authorization.k8s.io/v1 100 | kind: ClusterRoleBinding 101 | metadata: 102 | name: network-resources-injector-role-binding 103 | roleRef: 104 | apiGroup: rbac.authorization.k8s.io 105 | kind: ClusterRole 106 | name: network-resources-injector 107 | subjects: 108 | - kind: ServiceAccount 109 | name: network-resources-injector-sa 110 | namespace: kube-system 111 | --- 112 | apiVersion: rbac.authorization.k8s.io/v1 113 | kind: ClusterRoleBinding 114 | metadata: 115 | name: network-resources-injector-secrets-role-binding 116 | roleRef: 117 | apiGroup: rbac.authorization.k8s.io 118 | kind: ClusterRole 119 | name: network-resources-injector-secrets 120 | subjects: 121 | - kind: ServiceAccount 122 | name: network-resources-injector-sa 123 | namespace: kube-system 124 | --- 125 | apiVersion: rbac.authorization.k8s.io/v1 126 | kind: ClusterRoleBinding 127 | metadata: 128 | name: network-resources-injector-webhook-configs-role-binding 129 | roleRef: 130 | apiGroup: rbac.authorization.k8s.io 131 | kind: ClusterRole 132 | name: network-resources-injector-webhook-configs 133 | subjects: 134 | - kind: ServiceAccount 135 | name: network-resources-injector-sa 136 | namespace: kube-system 137 | --- 138 | apiVersion: rbac.authorization.k8s.io/v1 139 | kind: ClusterRoleBinding 140 | metadata: 141 | name: network-resources-injector-service-role-binding 142 | roleRef: 143 | apiGroup: rbac.authorization.k8s.io 144 | kind: ClusterRole 145 | name: network-resources-injector-service 146 | subjects: 147 | - kind: ServiceAccount 148 | name: network-resources-injector-sa 149 | namespace: kube-system 150 | --- 151 | apiVersion: rbac.authorization.k8s.io/v1 152 | kind: ClusterRoleBinding 153 | metadata: 154 | name: network-resources-injector-configmaps-role-binding 155 | roleRef: 156 | apiGroup: rbac.authorization.k8s.io 157 | kind: ClusterRole 158 | name: network-resources-injector-configmaps 159 | subjects: 160 | - kind: ServiceAccount 161 | name: network-resources-injector-sa 162 | namespace: kube-system 163 | 164 | -------------------------------------------------------------------------------- /deployments/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: network-resources-injector-pdb 5 | namespace: kube-system 6 | spec: 7 | minAvailable: 1 8 | selector: 9 | matchLabels: 10 | app: network-resources-injector 11 | -------------------------------------------------------------------------------- /deployments/server.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | --- 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | labels: 19 | app: network-resources-injector 20 | name: network-resources-injector 21 | namespace: kube-system 22 | spec: 23 | replicas: 2 24 | selector: 25 | matchLabels: 26 | app: network-resources-injector 27 | template: 28 | metadata: 29 | labels: 30 | app: network-resources-injector 31 | spec: 32 | serviceAccount: network-resources-injector-sa 33 | containers: 34 | - name: webhook-server 35 | image: network-resources-injector:latest 36 | imagePullPolicy: IfNotPresent 37 | command: 38 | - webhook 39 | args: 40 | - -bind-address=0.0.0.0 41 | - -port=8443 42 | - -tls-private-key-file=/etc/tls/tls.key 43 | - -tls-cert-file=/etc/tls/tls.crt 44 | - -health-check-port=8444 45 | - -logtostderr 46 | env: 47 | - name: NAMESPACE 48 | valueFrom: 49 | fieldRef: 50 | fieldPath: metadata.namespace 51 | securityContext: 52 | runAsUser: 10000 53 | runAsGroup: 10000 54 | capabilities: 55 | drop: 56 | - ALL 57 | add: ["NET_BIND_SERVICE"] 58 | readOnlyRootFilesystem: true 59 | allowPrivilegeEscalation: false 60 | volumeMounts: 61 | - mountPath: /etc/tls 62 | name: tls 63 | resources: 64 | requests: 65 | memory: "50Mi" 66 | cpu: "250m" 67 | limits: 68 | memory: "200Mi" 69 | cpu: "500m" 70 | livenessProbe: 71 | httpGet: 72 | path: /healthz 73 | port: 8444 74 | initialDelaySeconds: 10 75 | periodSeconds: 5 76 | initContainers: 77 | - name: installer 78 | image: network-resources-injector:latest 79 | imagePullPolicy: IfNotPresent 80 | command: 81 | - installer 82 | args: 83 | - -name=network-resources-injector 84 | - -namespace=kube-system 85 | - -alsologtostderr 86 | securityContext: 87 | runAsUser: 10000 88 | runAsGroup: 10000 89 | volumeMounts: 90 | - name: tls 91 | mountPath: /etc/tls 92 | env: 93 | - name: POD_NAME 94 | valueFrom: 95 | fieldRef: 96 | fieldPath: metadata.name 97 | volumes: 98 | - name: tls 99 | emptyDir: {} 100 | 101 | # For third-party certificate, use secret resource 102 | # instead of self-generated one from installer as below: 103 | # 104 | # 1) Remove initContainers from Pod spec. 105 | # 2) Replace `emptyDir: {}` with below config 106 | # 107 | # secret: 108 | # secretName: network-resources-injector-secret 109 | -------------------------------------------------------------------------------- /deployments/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: network-resources-injector-service 5 | namespace: kube-system 6 | spec: 7 | ports: 8 | - port: 443 9 | targetPort: 8443 10 | selector: 11 | app: network-resources-injector 12 | -------------------------------------------------------------------------------- /deployments/webhook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: admissionregistration.k8s.io/v1 3 | kind: MutatingWebhookConfiguration 4 | metadata: 5 | name: network-resources-injector-config 6 | namespace: kube-system 7 | webhooks: 8 | - name: network-resources-injector-config.k8s.io 9 | sideEffects: None 10 | admissionReviewVersions: ["v1"] 11 | clientConfig: 12 | service: 13 | name: network-resources-injector-service 14 | namespace: ${NAMESPACE} 15 | path: "/mutate" 16 | caBundle: ${CA_BUNDLE} 17 | namespaceSelector: 18 | matchExpressions: 19 | - key: "kubernetes.io/metadata.name" 20 | operator: "NotIn" 21 | values: 22 | - "kube-system" 23 | rules: 24 | - operations: [ "CREATE" ] 25 | apiGroups: ["apps", ""] 26 | apiVersions: ["v1"] 27 | resources: ["pods"] 28 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation guide 2 | 3 | ## Building Docker image 4 | Go to the root directory of the Network Resources Injector and build image: 5 | ``` 6 | cd $GOPATH/src/github.com/k8snetworkplumbingwg/network-resources-injector 7 | make image 8 | ``` 9 | 10 | ## Deploying webhook application 11 | Create Service Account for network resources injector mutating admission webhook and webhook installer and apply RBAC rules to created account: 12 | ``` 13 | kubectl apply -f deployments/auth.yaml 14 | ``` 15 | 16 | > Note: If you want to use third party certificate, create secret resource with following command and attach it in network-resources-injector pod spec: 17 | 18 | ``` 19 | kubectl create secret generic network-resources-injector-secret \ 20 | --from-file=key.pem= \ 21 | --from-file=cert.pem= \ 22 | -n kube-system 23 | ./scripts/webhook-deployment.sh 24 | ``` 25 | 26 | Next step creates Kubernetes pod. Init container creates all resources required to run webhook: 27 | * TLS key and certificate signed with Kubernetes CA 28 | * mutating webhook configuration 29 | * service to expose webhook deployment to the API server 30 | 31 | After successful completion of the init container work, the actual webhook server application container is started. 32 | 33 | Execute command: 34 | ``` 35 | kubectl apply -f deployments/server.yaml 36 | ``` 37 | 38 | > Note: Verify that Kubernetes controller manager has --cluster-signing-cert-file and --cluster-signing-key-file parameters set to paths to your CA keypair to make sure that Certificates API is enabled in order to generate certificate signed by cluster CA. More details about TLS certificates management in a cluster available [here](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/).* 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/k8snetworkplumbingwg/network-resources-injector 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/cloudflare/cfssl v1.4.1 7 | github.com/fsnotify/fsnotify v1.6.0 8 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b 9 | github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 10 | github.com/onsi/ginkgo v1.16.5 11 | github.com/onsi/gomega v1.27.6 12 | github.com/pkg/errors v0.9.1 13 | gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2 14 | k8s.io/api v0.28.3 15 | k8s.io/apimachinery v0.28.3 16 | k8s.io/client-go v0.28.3 17 | k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 18 | ) 19 | 20 | require ( 21 | github.com/BurntSushi/toml v1.3.2 // indirect 22 | github.com/containernetworking/cni v1.1.2 // indirect 23 | github.com/davecgh/go-spew v1.1.1 // indirect 24 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 25 | github.com/go-logr/logr v1.2.4 // indirect 26 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 27 | github.com/go-openapi/jsonreference v0.20.2 // indirect 28 | github.com/go-openapi/swag v0.22.3 // indirect 29 | github.com/gogo/protobuf v1.3.2 // indirect 30 | github.com/golang/protobuf v1.5.3 // indirect 31 | github.com/google/certificate-transparency-go v1.0.21 // indirect 32 | github.com/google/gnostic-models v0.6.8 // indirect 33 | github.com/google/go-cmp v0.5.9 // indirect 34 | github.com/google/gofuzz v1.2.0 // indirect 35 | github.com/google/uuid v1.3.0 // indirect 36 | github.com/imdario/mergo v0.3.11 // indirect 37 | github.com/josharian/intern v1.0.0 // indirect 38 | github.com/json-iterator/go v1.1.12 // indirect 39 | github.com/mailru/easyjson v0.7.7 // indirect 40 | github.com/moby/spdystream v0.2.0 // indirect 41 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 42 | github.com/modern-go/reflect2 v1.0.2 // indirect 43 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 44 | github.com/nxadm/tail v1.4.8 // indirect 45 | github.com/spf13/pflag v1.0.5 // indirect 46 | github.com/weppos/publicsuffix-go v0.5.0 // indirect 47 | github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e // indirect 48 | github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb // indirect 49 | golang.org/x/crypto v0.31.0 // indirect 50 | golang.org/x/net v0.33.0 // indirect 51 | golang.org/x/oauth2 v0.8.0 // indirect 52 | golang.org/x/sys v0.28.0 // indirect 53 | golang.org/x/term v0.27.0 // indirect 54 | golang.org/x/text v0.21.0 // indirect 55 | golang.org/x/time v0.3.0 // indirect 56 | google.golang.org/appengine v1.6.7 // indirect 57 | google.golang.org/protobuf v1.33.0 // indirect 58 | gopkg.in/inf.v0 v0.9.1 // indirect 59 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 60 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 61 | gopkg.in/yaml.v2 v2.4.0 // indirect 62 | gopkg.in/yaml.v3 v3.0.1 // indirect 63 | k8s.io/klog/v2 v2.100.1 // indirect 64 | k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect 65 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 66 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 67 | sigs.k8s.io/yaml v1.3.0 // indirect 68 | ) 69 | -------------------------------------------------------------------------------- /pkg/controlswitches/controlswitches.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlswitches 16 | 17 | import ( 18 | "encoding/json" 19 | "flag" 20 | "fmt" 21 | "strings" 22 | 23 | "github.com/golang/glog" 24 | corev1 "k8s.io/api/core/v1" 25 | 26 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/types" 27 | ) 28 | 29 | const ( 30 | // control switch keys 31 | controlSwitchesMainKey = "features" 32 | 33 | // enableHugePageDownAPIKey feature name 34 | enableHugePageDownAPIKey = "enableHugePageDownApi" 35 | // enableHonorExistingResourcesKey feature name 36 | enableHonorExistingResourcesKey = "enableHonorExistingResources" 37 | ) 38 | 39 | // controlSwitchesStates - depicts possible feature states 40 | type controlSwitchesStates struct { 41 | active bool 42 | initial bool 43 | } 44 | 45 | // setActiveToInitialState - set active state to the initial state set during initialization 46 | func (state *controlSwitchesStates) setActiveToInitialState() { 47 | state.active = state.initial 48 | } 49 | 50 | // setActiveState - set active state to the passed value 51 | func (state *controlSwitchesStates) setActiveState(value bool) { 52 | state.active = value 53 | } 54 | 55 | type ControlSwitches struct { 56 | // pointers to command line arguments 57 | injectHugepageDownAPI *bool 58 | resourceNameKeysFlag *string 59 | resourcesHonorFlag *bool 60 | 61 | configuration map[string]controlSwitchesStates 62 | resourceNameKeys []string 63 | isValid bool 64 | } 65 | 66 | // SetupControlSwitchesFlags - setup all control switches flags that can be set as command line NRI arguments 67 | // :return pointer to the structure that should be initialized with InitControlSwitches 68 | func SetupControlSwitchesFlags() *ControlSwitches { 69 | var initFlags ControlSwitches 70 | 71 | initFlags.injectHugepageDownAPI = flag.Bool("injectHugepageDownApi", false, "Enable hugepage requests and limits into Downward API.") 72 | initFlags.resourceNameKeysFlag = flag.String("network-resource-name-keys", "k8s.v1.cni.cncf.io/resourceName", "comma separated resource name keys --network-resource-name-keys.") 73 | initFlags.resourcesHonorFlag = flag.Bool("honor-resources", false, "Honor the existing requested resources requests & limits --honor-resources") 74 | 75 | return &initFlags 76 | } 77 | 78 | // InitControlSwitches - initialize internal control switches structures based on command line arguments 79 | func (switches *ControlSwitches) InitControlSwitches() { 80 | switches.configuration = make(map[string]controlSwitchesStates) 81 | 82 | state := controlSwitchesStates{initial: *switches.injectHugepageDownAPI, active: *switches.injectHugepageDownAPI} 83 | switches.configuration[enableHugePageDownAPIKey] = state 84 | 85 | state = controlSwitchesStates{initial: *switches.resourcesHonorFlag, active: *switches.resourcesHonorFlag} 86 | switches.configuration[enableHonorExistingResourcesKey] = state 87 | 88 | switches.resourceNameKeys = setResourceNameKeys(*switches.resourceNameKeysFlag) 89 | 90 | switches.isValid = true 91 | } 92 | 93 | // setResourceNameKeys extracts resources from a string and add them to resourceNameKeys array 94 | func setResourceNameKeys(keys string) []string { 95 | var resourceNameKeys []string 96 | 97 | for _, resourceNameKey := range strings.Split(keys, ",") { 98 | resourceNameKey = strings.TrimSpace(resourceNameKey) 99 | resourceNameKeys = append(resourceNameKeys, resourceNameKey) 100 | } 101 | 102 | return resourceNameKeys 103 | } 104 | 105 | func (switches *ControlSwitches) GetResourceNameKeys() []string { 106 | return switches.resourceNameKeys 107 | } 108 | 109 | func (switches *ControlSwitches) IsHugePagedownAPIEnabled() bool { 110 | return switches.configuration[enableHugePageDownAPIKey].active 111 | } 112 | 113 | func (switches *ControlSwitches) IsHonorExistingResourcesEnabled() bool { 114 | return switches.configuration[enableHonorExistingResourcesKey].active 115 | } 116 | 117 | func (switches *ControlSwitches) IsResourcesNameEnabled() bool { 118 | return len(*switches.resourceNameKeysFlag) > 0 119 | } 120 | 121 | // IsValid returns true when ControlSwitches structure was initialized, false otherwise 122 | func (switches *ControlSwitches) IsValid() bool { 123 | return switches.isValid 124 | } 125 | 126 | // GetAllFeaturesState returns string with information if feature is active or not 127 | func (switches *ControlSwitches) GetAllFeaturesState() string { 128 | var output string 129 | 130 | output = fmt.Sprintf("HugePageInject: %t", switches.IsHugePagedownAPIEnabled()) 131 | output = output + " / " + fmt.Sprintf("HonorExistingResources: %t", switches.IsHonorExistingResourcesEnabled()) 132 | output = output + " / " + fmt.Sprintf("EnableResourceNames: %t", switches.IsResourcesNameEnabled()) 133 | 134 | return output 135 | } 136 | 137 | // setAllFeaturesToInitialState - reset feature state to initial one set during NRI initialization 138 | func (switches *ControlSwitches) setAllFeaturesToInitialState() { 139 | state := switches.configuration[enableHugePageDownAPIKey] 140 | state.setActiveToInitialState() 141 | switches.configuration[enableHugePageDownAPIKey] = state 142 | 143 | state = switches.configuration[enableHonorExistingResourcesKey] 144 | state.setActiveToInitialState() 145 | switches.configuration[enableHonorExistingResourcesKey] = state 146 | } 147 | 148 | // setFeatureToState set given feature to the state defined in the map object 149 | func (switches *ControlSwitches) setFeatureToState(featureName string, switchObj map[string]bool) { 150 | if featureState, available := switchObj[featureName]; available { 151 | state := switches.configuration[featureName] 152 | state.setActiveState(featureState) 153 | switches.configuration[featureName] = state 154 | } else { 155 | state := switches.configuration[featureName] 156 | state.setActiveToInitialState() 157 | switches.configuration[featureName] = state 158 | } 159 | } 160 | 161 | // ProcessControlSwitchesConfigMap sets on the fly control switches 162 | // :param controlSwitchesCm - Kubernetes ConfigMap with control switches definition 163 | func (switches *ControlSwitches) ProcessControlSwitchesConfigMap(controlSwitchesCm *corev1.ConfigMap) { 164 | var err error 165 | if v, fileExists := controlSwitchesCm.Data[types.ConfigMapMainFileKey]; fileExists { 166 | var obj map[string]json.RawMessage 167 | 168 | if err = json.Unmarshal([]byte(v), &obj); err != nil { 169 | glog.Warningf("Error during json unmarshal %v", err) 170 | switches.setAllFeaturesToInitialState() 171 | return 172 | } 173 | 174 | if controlSwitches, mainExists := obj[controlSwitchesMainKey]; mainExists { 175 | var switchObj map[string]bool 176 | 177 | if err = json.Unmarshal(controlSwitches, &switchObj); err != nil { 178 | glog.Warningf("Unable to unmarshal [%s] from configmap, err: %v", controlSwitchesMainKey, err) 179 | switches.setAllFeaturesToInitialState() 180 | return 181 | } 182 | 183 | switches.setFeatureToState(enableHugePageDownAPIKey, switchObj) 184 | switches.setFeatureToState(enableHonorExistingResourcesKey, switchObj) 185 | } else { 186 | glog.Warningf("Map does not contains [%s]", controlSwitchesMainKey) 187 | } 188 | } else { 189 | glog.Warningf("Map does not contains [%s]", types.ConfigMapMainFileKey) 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /pkg/controlswitches/controlswitches_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlswitches 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestControlSwitches(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Controlswitches Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/controlswitches/controlswitches_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controlswitches 16 | 17 | import ( 18 | corev1 "k8s.io/api/core/v1" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | // Helper functions 25 | 26 | func createBool(value bool) *bool { 27 | return &value 28 | } 29 | 30 | func createString(value string) *string { 31 | return &value 32 | } 33 | 34 | var _ = Describe("Verify controlswitches package", func() { 35 | var structure *ControlSwitches 36 | 37 | Describe("Common functions", func() { 38 | Context("Display features", func() { 39 | BeforeEach(func() { 40 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(false), createString("")) 41 | structure.InitControlSwitches() 42 | }) 43 | 44 | AfterEach(func() { 45 | structure = nil 46 | }) 47 | 48 | It("Setup structure", func() { 49 | structure = SetupControlSwitchesFlags() 50 | Expect(structure.IsValid()).Should(Equal(false)) 51 | 52 | structure.InitControlSwitches() 53 | Expect(structure.IsValid()).Should(Equal(true)) 54 | }) 55 | }) 56 | }) 57 | 58 | Describe("Verify state structure", func() { 59 | Context("Check", func() { 60 | It("Set explicit active state", func() { 61 | state := controlSwitchesStates{active: false, initial: false} 62 | 63 | state.setActiveState(true) 64 | Expect(state.active).Should(Equal(true)) 65 | Expect(state.initial).Should(Equal(false)) 66 | 67 | state.setActiveState(false) 68 | Expect(state.active).Should(Equal(false)) 69 | Expect(state.initial).Should(Equal(false)) 70 | }) 71 | 72 | It("Set active to initial when initial is true", func() { 73 | state := controlSwitchesStates{active: false, initial: true} 74 | 75 | state.setActiveToInitialState() 76 | Expect(state.active).Should(Equal(true)) 77 | Expect(state.initial).Should(Equal(true)) 78 | }) 79 | 80 | It("Set active to initial when initial is false", func() { 81 | state := controlSwitchesStates{active: true, initial: false} 82 | 83 | state.setActiveToInitialState() 84 | Expect(state.active).Should(Equal(false)) 85 | Expect(state.initial).Should(Equal(false)) 86 | }) 87 | }) 88 | }) 89 | 90 | Describe("Hugepages downward API", func() { 91 | Context("Feature configuration flags", func() { 92 | AfterEach(func() { 93 | structure = nil 94 | }) 95 | 96 | It("Feature set to false, other features set to true", func() { 97 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(true), createString("something")) 98 | structure.InitControlSwitches() 99 | 100 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 101 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(true)) 102 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(true)) 103 | 104 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 105 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(true)) 106 | }) 107 | 108 | It("Feature set to true, other features set to false", func() { 109 | structure = SetupControlSwitchesUnitTests(createBool(true), createBool(false), createString("")) 110 | structure.InitControlSwitches() 111 | 112 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 113 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 114 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 115 | 116 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(true)) 117 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 118 | }) 119 | }) 120 | }) 121 | 122 | Describe("Honor existing resources", func() { 123 | Context("Feature configuration flags", func() { 124 | AfterEach(func() { 125 | structure = nil 126 | }) 127 | 128 | It("Feature set to false, other features set to true", func() { 129 | structure = SetupControlSwitchesUnitTests(createBool(true), createBool(false), createString("something")) 130 | structure.InitControlSwitches() 131 | 132 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 133 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 134 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(true)) 135 | 136 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(true)) 137 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 138 | }) 139 | 140 | It("Feature set to true, other features set to false", func() { 141 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(true), createString("")) 142 | structure.InitControlSwitches() 143 | 144 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 145 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(true)) 146 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 147 | 148 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 149 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(true)) 150 | }) 151 | }) 152 | }) 153 | 154 | Describe("Inject resource name keys ", func() { 155 | Context("Feature configuration flags", func() { 156 | AfterEach(func() { 157 | structure = nil 158 | }) 159 | 160 | It("Feature set to false, other features set to true", func() { 161 | structure = SetupControlSwitchesUnitTests(createBool(true), createBool(true), createString("")) 162 | structure.InitControlSwitches() 163 | 164 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 165 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(true)) 166 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 167 | 168 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(true)) 169 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(true)) 170 | 171 | Expect(structure.GetResourceNameKeys()).Should(Equal([]string{""})) 172 | }) 173 | 174 | It("Feature set to true, other features set to false", func() { 175 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(false), createString("something")) 176 | structure.InitControlSwitches() 177 | 178 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 179 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 180 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(true)) 181 | 182 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 183 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 184 | 185 | Expect(structure.GetResourceNameKeys()).Should(Equal([]string{"something"})) 186 | }) 187 | }) 188 | }) 189 | 190 | Describe("User defined injections", func() { 191 | Context("Feature configuration flags", func() { 192 | AfterEach(func() { 193 | structure = nil 194 | }) 195 | 196 | It("Feature set to false, other features set to true", func() { 197 | structure = SetupControlSwitchesUnitTests(createBool(true), createBool(true), createString("something")) 198 | structure.InitControlSwitches() 199 | 200 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 201 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(true)) 202 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(true)) 203 | 204 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(true)) 205 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(true)) 206 | }) 207 | 208 | It("Feature set to true, other features set to false", func() { 209 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(false), createString("")) 210 | structure.InitControlSwitches() 211 | 212 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 213 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 214 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 215 | 216 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 217 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 218 | }) 219 | }) 220 | }) 221 | 222 | Describe("Process Control Switches config map", func() { 223 | Context("Map without [features]", func() { 224 | BeforeEach(func() { 225 | structure = SetupControlSwitchesUnitTests(createBool(false), createBool(false), createString("")) 226 | structure.InitControlSwitches() 227 | }) 228 | 229 | AfterEach(func() { 230 | structure = nil 231 | }) 232 | 233 | It("Missing key", func() { 234 | cm := corev1.ConfigMap{ 235 | Data: map[string]string{}, 236 | } 237 | 238 | structure.ProcessControlSwitchesConfigMap(&cm) 239 | 240 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 241 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 242 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 243 | 244 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 245 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 246 | }) 247 | 248 | It("Map without config.json key", func() { 249 | cm := corev1.ConfigMap{ 250 | Data: map[string]string{"nri-inject-annotation": "true"}, 251 | } 252 | 253 | structure.ProcessControlSwitchesConfigMap(&cm) 254 | 255 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 256 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 257 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 258 | 259 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 260 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 261 | }) 262 | 263 | It("Map with correct key, but without [features] inside", func() { 264 | const value = `{ 265 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 266 | "user-defined-injections": { 267 | "network-resource-injector-pod-annotation": { 268 | "op": "add", 269 | "path": "/metadata/annotations", 270 | "value": { 271 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 272 | } 273 | } 274 | } 275 | } 276 | ` 277 | 278 | cm := corev1.ConfigMap{ 279 | Data: map[string]string{"config.json": value}, 280 | } 281 | 282 | structure.ProcessControlSwitchesConfigMap(&cm) 283 | 284 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 285 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 286 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 287 | 288 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 289 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 290 | }) 291 | 292 | It("Map with correct key, with [features] inside, but features name are incorrect", func() { 293 | const value = `{ 294 | "features": { 295 | "enableHugePageDown": false, 296 | "enableHonorExisting": true, 297 | "enableCustomizedInje": false, 298 | "enableResourceNa": false 299 | }, 300 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 301 | "user-defined-injections": { 302 | "network-resource-injector-pod-annotation": { 303 | "op": "add", 304 | "path": "/metadata/annotations", 305 | "value": { 306 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 307 | } 308 | } 309 | } 310 | } 311 | ` 312 | cm := corev1.ConfigMap{ 313 | Data: map[string]string{"config.json": value}, 314 | } 315 | 316 | structure.ProcessControlSwitchesConfigMap(&cm) 317 | 318 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 319 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 320 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 321 | 322 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 323 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 324 | }) 325 | 326 | It("Map with correct key, with [features] inside - but JSON is invalid", func() { 327 | const value = `{ 328 | "features": { 329 | "enableHugePageDownApi": false, 330 | "enableHonorExistingResources": true 331 | }, 332 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 333 | "user-defined-injections": { 334 | "network-resource-injector-pod-annotation": { 335 | "op": "add", 336 | "path": "/metadata/annotations" 337 | "value": { 338 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 339 | } 340 | } 341 | } 342 | } 343 | ` 344 | cm := corev1.ConfigMap{ 345 | Data: map[string]string{"config.json": value}, 346 | } 347 | 348 | structure.ProcessControlSwitchesConfigMap(&cm) 349 | 350 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 351 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 352 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 353 | 354 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 355 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 356 | }) 357 | 358 | It("Map with correct key, with [features] inside - all set to false", func() { 359 | const value = `{ 360 | "features": { 361 | "enableHugePageDownApi": false, 362 | "enableHonorExistingResources": false 363 | }, 364 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 365 | "user-defined-injections": { 366 | "network-resource-injector-pod-annotation": { 367 | "op": "add", 368 | "path": "/metadata/annotations", 369 | "value": { 370 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 371 | } 372 | } 373 | } 374 | } 375 | ` 376 | cm := corev1.ConfigMap{ 377 | Data: map[string]string{"config.json": value}, 378 | } 379 | 380 | structure.ProcessControlSwitchesConfigMap(&cm) 381 | 382 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 383 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 384 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 385 | 386 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 387 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 388 | }) 389 | 390 | // set one by one, instead of all in one 391 | It("Map with correct key, with [features] inside - value of feature set to string instead of bool", func() { 392 | const value = `{ 393 | "features": { 394 | "enableHugePageDownApi": true, 395 | "enableHonorExistingResources": "isThisAnError" 396 | }, 397 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 398 | "user-defined-injections": { 399 | "network-resource-injector-pod-annotation": { 400 | "op": "add", 401 | "path": "/metadata/annotations", 402 | "value": { 403 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 404 | } 405 | } 406 | } 407 | } 408 | ` 409 | cm := corev1.ConfigMap{ 410 | Data: map[string]string{"config.json": value}, 411 | } 412 | 413 | structure.ProcessControlSwitchesConfigMap(&cm) 414 | 415 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(false)) 416 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 417 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 418 | 419 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 420 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 421 | }) 422 | 423 | It("Map with correct key, with [features] inside - all set to true", func() { 424 | const value = `{ 425 | "features": { 426 | "enableHugePageDownApi": true, 427 | "enableHonorExistingResources": true 428 | }, 429 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 430 | "user-defined-injections": { 431 | "network-resource-injector-pod-annotation": { 432 | "op": "add", 433 | "path": "/metadata/annotations", 434 | "value": { 435 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 436 | } 437 | } 438 | } 439 | } 440 | ` 441 | cm := corev1.ConfigMap{ 442 | Data: map[string]string{"config.json": value}, 443 | } 444 | 445 | structure.ProcessControlSwitchesConfigMap(&cm) 446 | 447 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 448 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(true)) 449 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 450 | 451 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 452 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 453 | }) 454 | 455 | It("Map with correct key, with [features] inside - mix with values true / false", func() { 456 | const value = `{ 457 | "features": { 458 | "enableHugePageDownApi": true, 459 | "enableHonorExistingResources": false 460 | }, 461 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"], 462 | "user-defined-injections": { 463 | "network-resource-injector-pod-annotation": { 464 | "op": "add", 465 | "path": "/metadata/annotations", 466 | "value": { 467 | "k8s.v1.cni.cncf.io/networks": "sriov-net-attach-def" 468 | } 469 | } 470 | } 471 | } 472 | ` 473 | cm := corev1.ConfigMap{ 474 | Data: map[string]string{"config.json": value}, 475 | } 476 | 477 | structure.ProcessControlSwitchesConfigMap(&cm) 478 | 479 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 480 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 481 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 482 | 483 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 484 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 485 | }) 486 | 487 | It("Map with correct key, with [features] inside - some features are missing", func() { 488 | // create map that does not have all features defined, expected to be removed 489 | const value = `{ 490 | "features": { 491 | "enableHugePageDownApi": true 492 | } 493 | } 494 | ` 495 | 496 | cm := corev1.ConfigMap{ 497 | Data: map[string]string{"config.json": value}, 498 | } 499 | 500 | structure.ProcessControlSwitchesConfigMap(&cm) 501 | 502 | Expect(structure.IsHugePagedownAPIEnabled()).Should(Equal(true)) 503 | Expect(structure.IsHonorExistingResourcesEnabled()).Should(Equal(false)) 504 | Expect(structure.IsResourcesNameEnabled()).Should(Equal(false)) 505 | 506 | Expect(structure.configuration[enableHugePageDownAPIKey].initial).Should(Equal(false)) 507 | Expect(structure.configuration[enableHonorExistingResourcesKey].initial).Should(Equal(false)) 508 | }) 509 | }) 510 | }) 511 | }) 512 | -------------------------------------------------------------------------------- /pkg/controlswitches/controlswitchesaccessors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build unittests 16 | 17 | // This file should be only include in build system during unit tests execution. 18 | // Special method allows to setup structure with values needed by test. 19 | 20 | package controlswitches 21 | 22 | func SetupControlSwitchesUnitTests(downAPI, honor *bool, name *string) *ControlSwitches { 23 | var initFlags ControlSwitches 24 | 25 | initFlags.injectHugepageDownAPI = downAPI 26 | initFlags.resourceNameKeysFlag = name 27 | initFlags.resourcesHonorFlag = honor 28 | 29 | return &initFlags 30 | } 31 | -------------------------------------------------------------------------------- /pkg/installer/installer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package installer 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "io/ioutil" 22 | "os" 23 | "strings" 24 | "time" 25 | 26 | "github.com/cloudflare/cfssl/csr" 27 | "github.com/cloudflare/cfssl/helpers" 28 | "github.com/cloudflare/cfssl/initca" 29 | cfsigner "github.com/cloudflare/cfssl/signer" 30 | "github.com/cloudflare/cfssl/signer/local" 31 | "github.com/golang/glog" 32 | "github.com/pkg/errors" 33 | 34 | arv1 "k8s.io/api/admissionregistration/v1" 35 | corev1 "k8s.io/api/core/v1" 36 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 37 | "k8s.io/apimachinery/pkg/util/intstr" 38 | "k8s.io/apimachinery/pkg/util/wait" 39 | "k8s.io/client-go/kubernetes" 40 | "k8s.io/client-go/rest" 41 | ) 42 | 43 | var ( 44 | clientset kubernetes.Interface 45 | namespace string 46 | prefix string 47 | podName string 48 | ) 49 | 50 | const keyBitLength = 3072 51 | const CAExpiration = "630720000s" 52 | const secretName = "network-resources-injector" 53 | 54 | func generateCSR() ([]byte, []byte, error) { 55 | glog.Infof("generating Certificate Signing Request") 56 | serviceName := strings.Join([]string{prefix, "service"}, "-") 57 | certRequest := csr.New() 58 | certRequest.KeyRequest = &csr.KeyRequest{A: "rsa", S: keyBitLength} 59 | certRequest.CN = strings.Join([]string{serviceName, namespace, "svc"}, ".") 60 | certRequest.Hosts = []string{ 61 | serviceName, 62 | strings.Join([]string{serviceName, namespace}, "."), 63 | strings.Join([]string{serviceName, namespace, "svc"}, "."), 64 | } 65 | return csr.ParseRequest(certRequest) 66 | } 67 | 68 | func generateCACertificate() (*local.Signer, []byte, error) { 69 | certRequest := csr.New() 70 | certRequest.KeyRequest = &csr.KeyRequest{A: "rsa", S: keyBitLength} 71 | certRequest.CN = "Kubernetes NRI" 72 | certRequest.CA = &csr.CAConfig{Expiry: CAExpiration} 73 | cert, _, key, err := initca.New(certRequest) 74 | if err != nil { 75 | return nil, nil, fmt.Errorf("creating CA certificate failed: %v", err) 76 | } 77 | parsedKey, err := helpers.ParsePrivateKeyPEM(key) 78 | if err != nil { 79 | return nil, nil, fmt.Errorf("parsing private key pem failed: %v", err) 80 | } 81 | parsedCert, err := helpers.ParseCertificatePEM(cert) 82 | if err != nil { 83 | return nil, nil, fmt.Errorf("parse certificate failed: %v", err) 84 | } 85 | signer, err := local.NewSigner(parsedKey, parsedCert, cfsigner.DefaultSigAlgo(parsedKey), nil) 86 | if err != nil { 87 | return nil, nil, fmt.Errorf("failed to create signer: %v", err) 88 | } 89 | return signer, cert, nil 90 | } 91 | 92 | func writeToFile(certificate, key []byte, certFilename, keyFilename string) error { 93 | if err := ioutil.WriteFile("/etc/tls/"+certFilename, certificate, 0400); err != nil { 94 | return err 95 | } 96 | if err := ioutil.WriteFile("/etc/tls/"+keyFilename, key, 0400); err != nil { 97 | return err 98 | } 99 | return nil 100 | } 101 | 102 | func createMutatingWebhookConfiguration(certificate []byte, failurePolicyStr string) error { 103 | configName := strings.Join([]string{prefix, "mutating-config"}, "-") 104 | serviceName := strings.Join([]string{prefix, "service"}, "-") 105 | removeMutatingWebhookIfExists(configName) 106 | var failurePolicy arv1.FailurePolicyType 107 | if strings.EqualFold(strings.TrimSpace(failurePolicyStr), "Ignore") { 108 | failurePolicy = arv1.Ignore 109 | } else if strings.EqualFold(strings.TrimSpace(failurePolicyStr), "Fail") { 110 | failurePolicy = arv1.Fail 111 | } else { 112 | return errors.New("unknown failure policy type") 113 | } 114 | sideEffects := arv1.SideEffectClassNone 115 | path := "/mutate" 116 | namespaces := []string{"kube-system"} 117 | if namespace != "kube-system" { 118 | namespaces = append(namespaces, namespace) 119 | } 120 | namespaceSelector := metav1.LabelSelector{ 121 | MatchExpressions: []metav1.LabelSelectorRequirement{ 122 | { 123 | Key: "kubernetes.io/metadata.name", 124 | Operator: metav1.LabelSelectorOpNotIn, 125 | Values: namespaces, 126 | }, 127 | }, 128 | } 129 | configuration := &arv1.MutatingWebhookConfiguration{ 130 | ObjectMeta: metav1.ObjectMeta{ 131 | Name: configName, 132 | Labels: map[string]string{ 133 | "app": prefix, 134 | }, 135 | }, 136 | Webhooks: []arv1.MutatingWebhook{ 137 | arv1.MutatingWebhook{ 138 | Name: configName + ".k8s.cni.cncf.io", 139 | ClientConfig: arv1.WebhookClientConfig{ 140 | CABundle: certificate, 141 | Service: &arv1.ServiceReference{ 142 | Namespace: namespace, 143 | Name: serviceName, 144 | Path: &path, 145 | }, 146 | }, 147 | FailurePolicy: &failurePolicy, 148 | AdmissionReviewVersions: []string{"v1"}, 149 | SideEffects: &sideEffects, 150 | NamespaceSelector: &namespaceSelector, 151 | Rules: []arv1.RuleWithOperations{ 152 | arv1.RuleWithOperations{ 153 | Operations: []arv1.OperationType{arv1.Create}, 154 | Rule: arv1.Rule{ 155 | APIGroups: []string{""}, 156 | APIVersions: []string{"v1"}, 157 | Resources: []string{"pods"}, 158 | }, 159 | }, 160 | }, 161 | }, 162 | }, 163 | } 164 | _, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), configuration, metav1.CreateOptions{}) 165 | return err 166 | } 167 | 168 | func createService() error { 169 | serviceName := strings.Join([]string{prefix, "service"}, "-") 170 | removeServiceIfExists(serviceName) 171 | service := &corev1.Service{ 172 | ObjectMeta: metav1.ObjectMeta{ 173 | Name: serviceName, 174 | Labels: map[string]string{ 175 | "app": prefix, 176 | }, 177 | }, 178 | Spec: corev1.ServiceSpec{ 179 | Ports: []corev1.ServicePort{ 180 | corev1.ServicePort{ 181 | Port: 443, 182 | TargetPort: intstr.FromInt(8443), 183 | }, 184 | }, 185 | Selector: map[string]string{ 186 | "app": prefix, 187 | }, 188 | }, 189 | } 190 | _, err := clientset.CoreV1().Services(namespace).Create(context.TODO(), service, metav1.CreateOptions{}) 191 | return err 192 | } 193 | 194 | func removeServiceIfExists(serviceName string) { 195 | service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) 196 | if service != nil && err == nil { 197 | glog.Infof("service %s already exists, removing it first", serviceName) 198 | err := clientset.CoreV1().Services(namespace).Delete(context.TODO(), serviceName, metav1.DeleteOptions{}) 199 | if err != nil { 200 | glog.Errorf("error trying to remove service: %s", err) 201 | } 202 | glog.Infof("service %s removed", serviceName) 203 | } 204 | } 205 | 206 | func removeMutatingWebhookIfExists(configName string) { 207 | config, err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), configName, metav1.GetOptions{}) 208 | if config != nil && err == nil { 209 | glog.Infof("mutating webhook %s already exists, removing it first", configName) 210 | err := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{}) 211 | if err != nil { 212 | glog.Errorf("error trying to remove mutating webhook configuration: %s", err) 213 | } 214 | glog.Infof("mutating webhook configuration %s removed", configName) 215 | } 216 | } 217 | 218 | func removeSecretIfExists(secretName string) { 219 | secret, err := clientset.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) 220 | if secret != nil && err == nil { 221 | glog.Infof("secret %s already exists, removing it first", secretName) 222 | err := clientset.CoreV1().Secrets(namespace).Delete(context.TODO(), secretName, metav1.DeleteOptions{}) 223 | if err != nil { 224 | glog.Errorf("error trying to remove secret: %s", err) 225 | } 226 | glog.Infof("secret %s removed", secretName) 227 | } 228 | } 229 | 230 | // Install creates resources required by mutating admission webhook 231 | func Install(k8sNamespace, namePrefix, failurePolicy string) { 232 | /* setup Kubernetes API client */ 233 | config, err := rest.InClusterConfig() 234 | if err != nil { 235 | glog.Fatalf("error loading Kubernetes in-cluster configuration: %s", err) 236 | } 237 | clientset, err = kubernetes.NewForConfig(config) 238 | if err != nil { 239 | glog.Fatalf("error setting up Kubernetes client: %s", err) 240 | } 241 | populatePodName() 242 | 243 | namespace = k8sNamespace 244 | prefix = namePrefix 245 | 246 | signer, caCertificate, err := generateCACertificate() 247 | if err != nil { 248 | glog.Fatalf("Error generating CA certificate and signer: %s", err) 249 | } 250 | 251 | /* generate CSR and private key */ 252 | csr, key, err := generateCSR() 253 | if err != nil { 254 | glog.Fatalf("error generating CSR and private key: %s", err) 255 | } 256 | glog.Infof("raw CSR and private key successfully created") 257 | 258 | certificate, err := signer.Sign(cfsigner.SignRequest{ 259 | Request: string(csr), 260 | }) 261 | if err != nil { 262 | glog.Fatalf("error getting signed certificate: %s", err) 263 | } 264 | glog.Infof("signed certificate successfully obtained") 265 | 266 | if err = createSecret(context.Background(), certificate, key, "tls.crt", "tls.key"); err != nil { 267 | // As expected only one initContainer will succeed in creating secret. 268 | glog.Errorf("Failed creating secret: %v", err) 269 | // Wait for the secret to be created by the other initContainer and write 270 | // key and certificate to file. 271 | err = waitForCertDetailsUpdate() 272 | if err != nil { 273 | glog.Fatalf("Error occured while waiting for secret creation: %s", err) 274 | } 275 | return 276 | } 277 | glog.Info("Secret created successfully!") 278 | 279 | err = writeToFile(certificate, key, "tls.crt", "tls.key") 280 | if err != nil { 281 | glog.Fatalf("error writing certificate and key to files: %s", err) 282 | } 283 | glog.Infof("certificate and key written to files") 284 | 285 | /* create webhook configurations */ 286 | err = createMutatingWebhookConfiguration(caCertificate, failurePolicy) 287 | if err != nil { 288 | glog.Fatalf("error creating mutating webhook configuration: %s", err) 289 | } 290 | glog.Infof("mutating webhook configuration successfully created") 291 | 292 | /* create service */ 293 | err = createService() 294 | if err != nil { 295 | glog.Fatalf("error creating service: %s", err) 296 | } 297 | glog.Infof("service successfully created") 298 | 299 | glog.Infof("all resources created successfully") 300 | } 301 | 302 | func createSecret(ctx context.Context, certificate, key []byte, certFilename, keyFilename string) error { 303 | ownerRef, err := getOwnerReference() 304 | if err != nil { 305 | glog.Fatalf("Failed fetching owner reference for the pod:%v", err) 306 | } 307 | // Set owner reference so that on deleting deployment the secret is also deleted, 308 | // with this every new installation will create a new certificate and webhook config. 309 | secret := &corev1.Secret{ 310 | ObjectMeta: metav1.ObjectMeta{ 311 | Name: secretName, 312 | OwnerReferences: []metav1.OwnerReference{*ownerRef}, 313 | }, 314 | Data: map[string][]byte{ 315 | certFilename: certificate, 316 | keyFilename: key, 317 | }, 318 | } 319 | _, err = clientset.CoreV1().Secrets("kube-system").Create(ctx, secret, metav1.CreateOptions{}) 320 | return err 321 | } 322 | 323 | func getOwnerReference() (*metav1.OwnerReference, error) { 324 | b, err := clientset.CoreV1().RESTClient().Get().Resource("pods"). 325 | Name(podName).Namespace("kube-system").DoRaw(context.Background()) 326 | if err != nil { 327 | return nil, err 328 | } 329 | var pod corev1.Pod 330 | err = json.Unmarshal(b, &pod) 331 | if err != nil { 332 | glog.Info(err) 333 | return nil, err 334 | } 335 | var ownerRef metav1.OwnerReference 336 | for _, owner := range pod.OwnerReferences { 337 | if owner.Kind == "ReplicaSet" { 338 | ownerRef = metav1.OwnerReference{ 339 | Kind: owner.Kind, 340 | APIVersion: owner.APIVersion, 341 | Name: owner.Name, 342 | UID: owner.UID, 343 | } 344 | } 345 | } 346 | return &ownerRef, nil 347 | } 348 | 349 | func populatePodName() { 350 | var isPodNameAvailable bool 351 | podName, isPodNameAvailable = os.LookupEnv("POD_NAME") 352 | if !isPodNameAvailable { 353 | glog.Fatal(errors.New("pod name not set as environment variable")) 354 | } 355 | glog.Info("Pod Name set:", podName) 356 | } 357 | 358 | func waitForCertDetailsUpdate() error { 359 | return wait.Poll(5*time.Second, 300*time.Second, writeCertDetailsFromSecret) 360 | } 361 | 362 | func writeCertDetailsFromSecret() (bool, error) { 363 | secret, err := clientset.CoreV1().Secrets("kube-system").Get(context.Background(), secretName, metav1.GetOptions{}) 364 | if err != nil { 365 | return false, err 366 | } 367 | var tlsKey, tlsCertificate []byte 368 | for key, element := range secret.Data { 369 | if key == "tls.key" { 370 | tlsKey = element 371 | } else if key == "tls.crt" { 372 | tlsCertificate = element 373 | } 374 | } 375 | writeToFile(tlsCertificate, tlsKey, "tls.crt", "tls.key") 376 | glog.Info("Certificate details written to file") 377 | return true, nil 378 | } 379 | 380 | -------------------------------------------------------------------------------- /pkg/tools/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Nordix Foundation. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "sync" 19 | "sync/atomic" 20 | "time" 21 | 22 | "github.com/golang/glog" 23 | cniv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" 24 | "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" 25 | "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions" 26 | "k8s.io/client-go/rest" 27 | "k8s.io/client-go/tools/cache" 28 | ) 29 | 30 | type NetAttachDefCache struct { 31 | networkAnnotationsMap map[string]map[string]string 32 | networkAnnotationsMapMutex *sync.Mutex 33 | stopper chan struct{} 34 | isRunning int32 35 | } 36 | 37 | type NetAttachDefCacheService interface { 38 | Start() 39 | Stop() 40 | Get(namespace string, networkName string) map[string]string 41 | } 42 | 43 | func Create() NetAttachDefCacheService { 44 | return &NetAttachDefCache{make(map[string]map[string]string), 45 | &sync.Mutex{}, make(chan struct{}), 0} 46 | } 47 | 48 | // Start creates informer for NetworkAttachmentDefinition events and populate the local cache 49 | func (nc *NetAttachDefCache) Start() { 50 | factory := externalversions.NewSharedInformerFactoryWithOptions(setupNetAttachDefClient(), 0, externalversions.WithNamespace("")) 51 | informer := factory.K8sCniCncfIo().V1().NetworkAttachmentDefinitions().Informer() 52 | // mutex to serialize the events. 53 | mutex := &sync.Mutex{} 54 | 55 | informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ 56 | AddFunc: func(obj interface{}) { 57 | mutex.Lock() 58 | defer mutex.Unlock() 59 | netAttachDef := obj.(*cniv1.NetworkAttachmentDefinition) 60 | nc.put(netAttachDef.Namespace, netAttachDef.Name, netAttachDef.Annotations) 61 | }, 62 | UpdateFunc: func(oldObj, newObj interface{}) { 63 | mutex.Lock() 64 | defer mutex.Unlock() 65 | oldNetAttachDef := oldObj.(*cniv1.NetworkAttachmentDefinition) 66 | newNetAttachDef := newObj.(*cniv1.NetworkAttachmentDefinition) 67 | if oldNetAttachDef.GetResourceVersion() == newNetAttachDef.GetResourceVersion() { 68 | glog.Infof("no change in net-attach-def %s, ignoring update event", nc.getKey(oldNetAttachDef.Namespace, newNetAttachDef.Name)) 69 | return 70 | } 71 | nc.remove(oldNetAttachDef.Namespace, oldNetAttachDef.Name) 72 | nc.put(newNetAttachDef.Namespace, newNetAttachDef.Name, newNetAttachDef.Annotations) 73 | }, 74 | DeleteFunc: func(obj interface{}) { 75 | mutex.Lock() 76 | defer mutex.Unlock() 77 | netAttachDef := obj.(*cniv1.NetworkAttachmentDefinition) 78 | nc.remove(netAttachDef.Namespace, netAttachDef.Name) 79 | }, 80 | }) 81 | go func() { 82 | atomic.StoreInt32(&(nc.isRunning), int32(1)) 83 | // informer Run blocks until informer is stopped 84 | glog.Infof("starting net-attach-def informer") 85 | informer.Run(nc.stopper) 86 | glog.Infof("net-attach-def informer is stopped") 87 | atomic.StoreInt32(&(nc.isRunning), int32(0)) 88 | }() 89 | } 90 | 91 | // Stop teardown the NetworkAttachmentDefinition informer 92 | func (nc *NetAttachDefCache) Stop() { 93 | close(nc.stopper) 94 | tEnd := time.Now().Add(3 * time.Second) 95 | for tEnd.After(time.Now()) { 96 | if atomic.LoadInt32(&nc.isRunning) == 0 { 97 | glog.Infof("net-attach-def informer is no longer running, proceed to clean up nad cache") 98 | break 99 | } 100 | time.Sleep(600 * time.Millisecond) 101 | } 102 | nc.networkAnnotationsMapMutex.Lock() 103 | nc.networkAnnotationsMap = nil 104 | nc.networkAnnotationsMapMutex.Unlock() 105 | } 106 | 107 | func (nc *NetAttachDefCache) put(namespace, networkName string, annotations map[string]string) { 108 | nc.networkAnnotationsMapMutex.Lock() 109 | nc.networkAnnotationsMap[nc.getKey(namespace, networkName)] = annotations 110 | nc.networkAnnotationsMapMutex.Unlock() 111 | } 112 | 113 | // Get returns annotations map for the given namespace and network name, if it's not available 114 | // return nil 115 | func (nc *NetAttachDefCache) Get(namespace, networkName string) map[string]string { 116 | nc.networkAnnotationsMapMutex.Lock() 117 | defer nc.networkAnnotationsMapMutex.Unlock() 118 | if annotationsMap, exists := nc.networkAnnotationsMap[nc.getKey(namespace, networkName)]; exists { 119 | return annotationsMap 120 | } 121 | return nil 122 | } 123 | 124 | func (nc *NetAttachDefCache) remove(namespace, networkName string) { 125 | nc.networkAnnotationsMapMutex.Lock() 126 | delete(nc.networkAnnotationsMap, nc.getKey(namespace, networkName)) 127 | nc.networkAnnotationsMapMutex.Unlock() 128 | } 129 | 130 | func (nc *NetAttachDefCache) getKey(namespace, networkName string) string { 131 | return namespace + "/" + networkName 132 | } 133 | 134 | // setupNetAttachDefClient creates K8s client for net-attach-def crd 135 | func setupNetAttachDefClient() versioned.Interface { 136 | config, err := rest.InClusterConfig() 137 | if err != nil { 138 | glog.Fatal(err) 139 | } 140 | clientset, err := versioned.NewForConfig(config) 141 | if err != nil { 142 | glog.Fatal(err) 143 | } 144 | return clientset 145 | } 146 | -------------------------------------------------------------------------------- /pkg/types/types.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 Red Hat, Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | const ( 18 | DownwardAPIMountPath = "/etc/podnetinfo" 19 | AnnotationsPath = "annotations" 20 | LabelsPath = "labels" 21 | EnvNameContainerName = "CONTAINER_NAME" 22 | Hugepages1GRequestPath = "hugepages_1G_request" 23 | Hugepages2MRequestPath = "hugepages_2M_request" 24 | Hugepages1GLimitPath = "hugepages_1G_limit" 25 | Hugepages2MLimitPath = "hugepages_2M_limit" 26 | ConfigMapMainFileKey = "config.json" 27 | ) 28 | 29 | // JsonPatchOperation the JSON path operation 30 | type JsonPatchOperation struct { 31 | Operation string `json:"op"` 32 | Path string `json:"path"` 33 | Value interface{} `json:"value,omitempty"` 34 | } 35 | -------------------------------------------------------------------------------- /pkg/userdefinedinjections/userdefinedinjections.go: -------------------------------------------------------------------------------- 1 | package userdefinedinjections 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/golang/glog" 10 | corev1 "k8s.io/api/core/v1" 11 | 12 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/types" 13 | ) 14 | 15 | const ( 16 | userDefinedInjectionsMainKey = "user-defined-injections" 17 | ) 18 | 19 | // UserDefinedInjections user defined injections 20 | type UserDefinedInjections struct { 21 | sync.Mutex 22 | Patchs map[string]types.JsonPatchOperation 23 | } 24 | 25 | // CreateUserInjectionsStructure returns empty UserDefinedInjections structure 26 | func CreateUserInjectionsStructure() *UserDefinedInjections { 27 | var userDefinedInjects = UserDefinedInjections{Patchs: make(map[string]types.JsonPatchOperation)} 28 | return &userDefinedInjects 29 | } 30 | 31 | // SetUserDefinedInjections sets additional injections to be applied in Pod spec 32 | func (userDefinedInjects *UserDefinedInjections) SetUserDefinedInjections(injectionsCm *corev1.ConfigMap) { 33 | if v, fileExists := injectionsCm.Data[types.ConfigMapMainFileKey]; fileExists { 34 | var obj map[string]json.RawMessage 35 | var err error 36 | if err = json.Unmarshal([]byte(v), &obj); err != nil { 37 | glog.Warningf("Error during json unmarshal of main: %v", err) 38 | return 39 | } 40 | 41 | if userDefinedInjections, mainExists := obj[userDefinedInjectionsMainKey]; mainExists { 42 | var userDefinedInjectionsObj map[string]json.RawMessage 43 | if err = json.Unmarshal([]byte(userDefinedInjections), &userDefinedInjectionsObj); err != nil { 44 | glog.Warningf("Error during json unmarshal of injections: %v", err) 45 | return 46 | } 47 | 48 | // lock for writing 49 | userDefinedInjects.Lock() 50 | defer userDefinedInjects.Unlock() 51 | 52 | var patch types.JsonPatchOperation 53 | var userDefinedPatchs = userDefinedInjects.Patchs 54 | 55 | for k, value := range userDefinedInjectionsObj { 56 | existValue, exists := userDefinedPatchs[k] 57 | // unmarshal userDefined injection to json patch 58 | err := json.Unmarshal([]byte(value), &patch) 59 | if err != nil { 60 | glog.Errorf("Failed to unmarshal user-defined injection: %v", v) 61 | continue 62 | } 63 | // metadata.Annotations is the only supported field for user definition 64 | // jsonPatchOperation.Path should be "/metadata/annotations" 65 | if patch.Path != "/metadata/annotations" { 66 | glog.Errorf("Path: %v is not supported, only /metadata/annotations can be defined by user", patch.Path) 67 | continue 68 | } 69 | 70 | if !exists || !reflect.DeepEqual(existValue, patch) { 71 | glog.Infof("Initializing user-defined injections with key: %v, value: %v", k, v) 72 | userDefinedPatchs[k] = patch 73 | } 74 | } 75 | 76 | // remove stale entries from userDefined configMap 77 | for k := range userDefinedPatchs { 78 | if _, ok := userDefinedInjectionsObj[k]; ok { 79 | continue 80 | } 81 | glog.Infof("Removing stale entry: %v from user-defined injections", k) 82 | delete(userDefinedPatchs, k) 83 | } 84 | } else { 85 | glog.Warningf("Map does not contains [%s]. Clear old entries.", userDefinedInjectionsMainKey) 86 | userDefinedInjects.Patchs = make(map[string]types.JsonPatchOperation) 87 | } 88 | } else { 89 | glog.Warningf("Map does not contains [%s]. Clear old entries", types.ConfigMapMainFileKey) 90 | userDefinedInjects.Patchs = make(map[string]types.JsonPatchOperation) 91 | } 92 | } 93 | 94 | // CreateUserDefinedPatch creates customized patch for the specified POD 95 | func (userDefinedInjects *UserDefinedInjections) CreateUserDefinedPatch(pod corev1.Pod) ([]types.JsonPatchOperation, error) { 96 | var userDefinedPatch []types.JsonPatchOperation 97 | 98 | // lock for reading 99 | userDefinedInjects.Lock() 100 | defer userDefinedInjects.Unlock() 101 | 102 | for k, v := range userDefinedInjects.Patchs { 103 | // The userDefinedInjects will be injected when: 104 | // 1. Pod labels contain the patch key defined in userDefinedInjects 105 | // 2. The value of patch key in pod labels(not in userDefinedInjects) is "true" 106 | if podValue, exists := pod.ObjectMeta.Labels[k]; exists && strings.ToLower(podValue) == "true" { 107 | userDefinedPatch = append(userDefinedPatch, v) 108 | } 109 | } 110 | 111 | return userDefinedPatch, nil 112 | } 113 | -------------------------------------------------------------------------------- /pkg/userdefinedinjections/userdefinedinjections_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package userdefinedinjections 16 | 17 | import ( 18 | "testing" 19 | 20 | . "github.com/onsi/ginkgo" 21 | . "github.com/onsi/gomega" 22 | ) 23 | 24 | func TestUserDefinedInjections(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "UserDefinedInjections Suite") 27 | } 28 | -------------------------------------------------------------------------------- /pkg/userdefinedinjections/userdefinedinjections_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Intel, Redhat Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package userdefinedinjections 16 | 17 | import ( 18 | corev1 "k8s.io/api/core/v1" 19 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 | 21 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/types" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/ginkgo/extensions/table" 25 | . "github.com/onsi/gomega" 26 | ) 27 | 28 | var _ = Describe("UserDefinedInjections", func() { 29 | DescribeTable("Create user-defined patchs", 30 | 31 | func(pod corev1.Pod, userDefinedInjectPatchs map[string]types.JsonPatchOperation, out []types.JsonPatchOperation) { 32 | userDefinedInjects := CreateUserInjectionsStructure() 33 | userDefinedInjects.Patchs = userDefinedInjectPatchs 34 | appliedPatchs, _ := userDefinedInjects.CreateUserDefinedPatch(pod) 35 | Expect(appliedPatchs).Should(Equal(out)) 36 | }, 37 | Entry( 38 | "match pod label", 39 | corev1.Pod{ 40 | ObjectMeta: metav1.ObjectMeta{ 41 | Name: "test", 42 | Labels: map[string]string{"nri-inject-annotation": "true"}, 43 | }, 44 | Spec: corev1.PodSpec{}, 45 | }, 46 | map[string]types.JsonPatchOperation{ 47 | "nri-inject-annotation": types.JsonPatchOperation{ 48 | Operation: "add", 49 | Path: "/metadata/annotations", 50 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 51 | }, 52 | }, 53 | []types.JsonPatchOperation{ 54 | { 55 | Operation: "add", 56 | Path: "/metadata/annotations", 57 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 58 | }, 59 | }, 60 | ), 61 | Entry( 62 | "doesn't match pod label value", 63 | corev1.Pod{ 64 | ObjectMeta: metav1.ObjectMeta{ 65 | Name: "test", 66 | Labels: map[string]string{"nri-inject-annotation": "false"}, 67 | }, 68 | Spec: corev1.PodSpec{}, 69 | }, 70 | map[string]types.JsonPatchOperation{ 71 | "nri-inject-annotation": types.JsonPatchOperation{ 72 | Operation: "add", 73 | Path: "/metadata/annotations", 74 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 75 | }, 76 | }, 77 | nil, 78 | ), 79 | Entry( 80 | "doesn't match pod label key", 81 | corev1.Pod{ 82 | ObjectMeta: metav1.ObjectMeta{ 83 | Name: "test", 84 | Labels: map[string]string{"nri-inject-labels": "true"}, 85 | }, 86 | Spec: corev1.PodSpec{}, 87 | }, 88 | map[string]types.JsonPatchOperation{ 89 | "nri-inject-annotation": types.JsonPatchOperation{ 90 | Operation: "add", 91 | Path: "/metadata/annotations", 92 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 93 | }, 94 | }, 95 | nil, 96 | ), 97 | ) 98 | 99 | DescribeTable("Setting user-defined injections", 100 | func(in *corev1.ConfigMap, existing map[string]types.JsonPatchOperation, out map[string]types.JsonPatchOperation) { 101 | userDefinedInjects := CreateUserInjectionsStructure() 102 | userDefinedInjects.Patchs = existing 103 | userDefinedInjects.SetUserDefinedInjections(in) 104 | Expect(userDefinedInjects.Patchs).Should(Equal(out)) 105 | }, 106 | Entry( 107 | "patch - empty config map", 108 | &corev1.ConfigMap{ 109 | Data: map[string]string{}, 110 | }, 111 | map[string]types.JsonPatchOperation{}, 112 | map[string]types.JsonPatchOperation{}, 113 | ), 114 | Entry( 115 | "patch - empty config map", 116 | &corev1.ConfigMap{ 117 | Data: map[string]string{}, 118 | }, 119 | map[string]types.JsonPatchOperation{ 120 | "nri-inject-annotation": types.JsonPatchOperation{ 121 | Operation: "add", 122 | Path: "/metadata/annotations", 123 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net"}, 124 | }, 125 | }, 126 | map[string]types.JsonPatchOperation{}, 127 | ), 128 | Entry( 129 | "patch - config map without main config.json key", 130 | &corev1.ConfigMap{ 131 | Data: map[string]string{ 132 | "config": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 133 | }, 134 | map[string]types.JsonPatchOperation{}, 135 | map[string]types.JsonPatchOperation{}, 136 | ), 137 | Entry( 138 | "patch - config map without main config.json key", 139 | &corev1.ConfigMap{ 140 | Data: map[string]string{ 141 | "config": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 142 | }, 143 | map[string]types.JsonPatchOperation{ 144 | "nri-inject-annotation": types.JsonPatchOperation{ 145 | Operation: "add", 146 | Path: "/metadata/annotations", 147 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net"}, 148 | }, 149 | }, 150 | map[string]types.JsonPatchOperation{}, 151 | ), 152 | Entry( 153 | "patch - config map without userdefinedinjections key", 154 | &corev1.ConfigMap{ 155 | Data: map[string]string{ 156 | "config.json": "{\"custom\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 157 | }, 158 | map[string]types.JsonPatchOperation{}, 159 | map[string]types.JsonPatchOperation{}, 160 | ), 161 | Entry( 162 | "patch - config map with errors in json path", 163 | &corev1.ConfigMap{ 164 | Data: map[string]string{ 165 | "config.json": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": 5, \"path\": \"/metadata/annotations\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 166 | }, 167 | map[string]types.JsonPatchOperation{}, 168 | map[string]types.JsonPatchOperation{}, 169 | ), 170 | Entry( 171 | "patch - config map with incorrect json path - not /metadata/annotations", 172 | &corev1.ConfigMap{ 173 | Data: map[string]string{ 174 | "config.json": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/supprise\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 175 | }, 176 | map[string]types.JsonPatchOperation{}, 177 | map[string]types.JsonPatchOperation{}, 178 | ), 179 | Entry( 180 | "patch - additional networks annotation", 181 | &corev1.ConfigMap{ 182 | Data: map[string]string{ 183 | "config.json": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": { \"k8s.v1.cni.cncf.io/networks\": \"sriov-net\" }}}}"}, 184 | }, 185 | map[string]types.JsonPatchOperation{}, 186 | map[string]types.JsonPatchOperation{ 187 | "nri-inject-annotation": types.JsonPatchOperation{ 188 | Operation: "add", 189 | Path: "/metadata/annotations", 190 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 191 | }, 192 | }, 193 | ), 194 | Entry( 195 | "patch - default network annotation", 196 | &corev1.ConfigMap{ 197 | Data: map[string]string{ 198 | "config.json": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": {\"v1.multus-cni.io/default-network\": \"sriov-net\" }}}}"}, 199 | }, 200 | map[string]types.JsonPatchOperation{}, 201 | map[string]types.JsonPatchOperation{ 202 | "nri-inject-annotation": types.JsonPatchOperation{ 203 | Operation: "add", 204 | Path: "/metadata/annotations", 205 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net"}, 206 | }, 207 | }, 208 | ), 209 | Entry( 210 | "patch - remove stale entry", 211 | &corev1.ConfigMap{ 212 | Data: map[string]string{ 213 | "config.json": "{\"user-defined-injections\": { }}"}, 214 | }, 215 | map[string]types.JsonPatchOperation{ 216 | "nri-inject-annotation": types.JsonPatchOperation{ 217 | Operation: "add", 218 | Path: "/metadata/annotations", 219 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net"}, 220 | }, 221 | }, 222 | map[string]types.JsonPatchOperation{}, 223 | ), 224 | Entry( 225 | "patch - overwrite existing userDefinedInjects", 226 | &corev1.ConfigMap{ 227 | Data: map[string]string{ 228 | "config.json": "{\"user-defined-injections\": { \"nri-inject-annotation\": {\"op\": \"add\", \"path\": \"/metadata/annotations\", \"value\": {\"v1.multus-cni.io/default-network\": \"sriov-net-new\"}}}}"}, 229 | }, 230 | map[string]types.JsonPatchOperation{ 231 | "nri-inject-annotation": types.JsonPatchOperation{ 232 | Operation: "add", 233 | Path: "/metadata/annotations", 234 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net-old"}, 235 | }, 236 | }, 237 | map[string]types.JsonPatchOperation{ 238 | "nri-inject-annotation": types.JsonPatchOperation{ 239 | Operation: "add", 240 | Path: "/metadata/annotations", 241 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net-new"}, 242 | }, 243 | }, 244 | ), 245 | ) 246 | }) 247 | -------------------------------------------------------------------------------- /pkg/webhook/tlsutils.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Multus Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package webhook 16 | 17 | import ( 18 | "crypto/tls" 19 | "crypto/x509" 20 | "fmt" 21 | "io/ioutil" 22 | "sync" 23 | 24 | "github.com/golang/glog" 25 | ) 26 | 27 | type tlsKeypairReloader struct { 28 | certMutex sync.RWMutex 29 | cert *tls.Certificate 30 | certPath string 31 | keyPath string 32 | } 33 | 34 | type clientCertPool struct { 35 | certPool *x509.CertPool 36 | certPaths *ClientCAFlags 37 | insecure bool 38 | } 39 | 40 | type ClientCAFlags []string 41 | 42 | func (i *ClientCAFlags) String() string { 43 | return "" 44 | } 45 | 46 | func (i *ClientCAFlags) Set(path string) error { 47 | *i = append(*i, path) 48 | return nil 49 | } 50 | 51 | func (keyPair *tlsKeypairReloader) Reload() error { 52 | newCert, err := tls.LoadX509KeyPair(keyPair.certPath, keyPair.keyPath) 53 | if err != nil { 54 | return err 55 | } 56 | glog.V(2).Infof("cetificate reloaded") 57 | keyPair.certMutex.Lock() 58 | defer keyPair.certMutex.Unlock() 59 | keyPair.cert = &newCert 60 | return nil 61 | } 62 | 63 | func (keyPair *tlsKeypairReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) { 64 | return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { 65 | keyPair.certMutex.RLock() 66 | defer keyPair.certMutex.RUnlock() 67 | return keyPair.cert, nil 68 | } 69 | } 70 | 71 | // NewTlsKeypairReloader reload tlsKeypairReloader struct 72 | func NewTlsKeypairReloader(certPath, keyPath string) (*tlsKeypairReloader, error) { 73 | result := &tlsKeypairReloader{ 74 | certPath: certPath, 75 | keyPath: keyPath, 76 | } 77 | cert, err := tls.LoadX509KeyPair(certPath, keyPath) 78 | if err != nil { 79 | return nil, err 80 | } 81 | result.cert = &cert 82 | 83 | return result, nil 84 | } 85 | 86 | //NewClientCertPool will load a single client CA 87 | func NewClientCertPool(clientCaPaths *ClientCAFlags, insecure bool) (*clientCertPool, error) { 88 | pool := &clientCertPool{ 89 | certPaths: clientCaPaths, 90 | insecure: insecure, 91 | } 92 | if !pool.insecure { 93 | if err := pool.Load(); err != nil { 94 | return nil, err 95 | } 96 | } 97 | return pool, nil 98 | } 99 | 100 | //Load a certificate into the client CA pool 101 | func (pool *clientCertPool) Load() error { 102 | if pool.insecure { 103 | glog.Infof("can not load client CA pool. Remove --insecure flag to enable.") 104 | return nil 105 | } 106 | 107 | if len(*pool.certPaths) == 0 { 108 | return fmt.Errorf("no client CA file path(s) found") 109 | } 110 | 111 | pool.certPool = x509.NewCertPool() 112 | for _, path := range *pool.certPaths { 113 | caCertPem, err := ioutil.ReadFile(path) 114 | if err != nil { 115 | return fmt.Errorf("failed to load client CA file from path '%s'", path) 116 | } 117 | if ok := pool.certPool.AppendCertsFromPEM(caCertPem); !ok { 118 | return fmt.Errorf("failed to parse client CA file from path '%s'", path) 119 | } 120 | glog.Infof("added client CA to cert pool from path '%s'", path) 121 | } 122 | glog.Infof("added '%d' client CA(s) to cert pool", len(*pool.certPaths)) 123 | return nil 124 | } 125 | 126 | //GetCertPool returns a client CA pool 127 | func (pool *clientCertPool) GetCertPool() *x509.CertPool { 128 | if pool.insecure { 129 | return nil 130 | } 131 | return pool.certPool 132 | } 133 | 134 | //GetClientAuth determines the policy the http server will follow for TLS Client Authentication 135 | func GetClientAuth(insecure bool) tls.ClientAuthType { 136 | if insecure { 137 | return tls.NoClientCert 138 | } 139 | return tls.RequireAndVerifyClientCert 140 | } 141 | -------------------------------------------------------------------------------- /pkg/webhook/webhook_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package webhook 16 | 17 | import ( 18 | "io/ioutil" 19 | "log" 20 | "testing" 21 | 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | ) 25 | 26 | func TestWebhook(t *testing.T) { 27 | log.SetOutput(ioutil.Discard) 28 | RegisterFailHandler(Fail) 29 | RunSpecs(t, "Webhook Suite") 30 | } 31 | -------------------------------------------------------------------------------- /pkg/webhook/webhook_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 Intel Corporation 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package webhook 16 | 17 | import ( 18 | "bytes" 19 | "net/http" 20 | "net/http/httptest" 21 | 22 | "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/types" 23 | admissionv1 "k8s.io/api/admission/v1" 24 | corev1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | 27 | . "github.com/onsi/ginkgo" 28 | . "github.com/onsi/ginkgo/extensions/table" 29 | . "github.com/onsi/gomega" 30 | 31 | "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/controlswitches" 32 | nritypes "github.com/k8snetworkplumbingwg/network-resources-injector/pkg/types" 33 | ) 34 | 35 | func createBool(value bool) *bool { 36 | return &value 37 | } 38 | 39 | func createString(value string) *string { 40 | return &value 41 | } 42 | 43 | var _ = Describe("Webhook", func() { 44 | Describe("Preparing Admission Review Response", func() { 45 | Context("Admission Review Request is nil", func() { 46 | It("should return error", func() { 47 | ar := &admissionv1.AdmissionReview{} 48 | ar.Request = nil 49 | Expect(prepareAdmissionReviewResponse(false, "", ar)).To(HaveOccurred()) 50 | }) 51 | }) 52 | Context("Message is not empty", func() { 53 | It("should set message in the response", func() { 54 | ar := &admissionv1.AdmissionReview{} 55 | ar.Request = &admissionv1.AdmissionRequest{ 56 | UID: "fake-uid", 57 | } 58 | err := prepareAdmissionReviewResponse(false, "some message", ar) 59 | Expect(err).NotTo(HaveOccurred()) 60 | Expect(ar.Response.Result.Message).To(Equal("some message")) 61 | }) 62 | }) 63 | }) 64 | 65 | Describe("Deserializing Admission Review", func() { 66 | Context("It's not an Admission Review", func() { 67 | It("should return an error", func() { 68 | body := []byte("some-invalid-body") 69 | _, err := deserializeAdmissionReview(body) 70 | Expect(err).To(HaveOccurred()) 71 | }) 72 | }) 73 | }) 74 | 75 | Describe("Deserializing Network Attachment Definition", func() { 76 | Context("It's not an Network Attachment Definition", func() { 77 | It("should return an error", func() { 78 | ar := &admissionv1.AdmissionReview{} 79 | ar.Request = &admissionv1.AdmissionRequest{} 80 | _, err := deserializeNetworkAttachmentDefinition(ar) 81 | Expect(err).To(HaveOccurred()) 82 | }) 83 | }) 84 | }) 85 | 86 | Describe("Deserializing Pod", func() { 87 | Context("It's not a Pod", func() { 88 | It("should return an error", func() { 89 | ar := &admissionv1.AdmissionReview{} 90 | ar.Request = &admissionv1.AdmissionRequest{} 91 | _, err := deserializePod(ar) 92 | Expect(err).To(HaveOccurred()) 93 | }) 94 | }) 95 | }) 96 | 97 | Describe("Writing a response", func() { 98 | Context("with an AdmissionReview", func() { 99 | It("should be marshalled and written to a HTTP Response Writer", func() { 100 | w := httptest.NewRecorder() 101 | ar := &admissionv1.AdmissionReview{} 102 | ar.Response = &admissionv1.AdmissionResponse{ 103 | UID: "fake-uid", 104 | Allowed: true, 105 | Result: &metav1.Status{ 106 | Message: "fake-msg", 107 | }, 108 | } 109 | expected := []byte(`{"response":{"uid":"fake-uid","allowed":true,"status":{"metadata":{},"message":"fake-msg"}}}`) 110 | writeResponse(w, ar) 111 | Expect(w.Body.Bytes()).To(Equal(expected)) 112 | }) 113 | }) 114 | }) 115 | 116 | Describe("Handling requests", func() { 117 | BeforeEach(func() { 118 | structure := controlswitches.SetupControlSwitchesUnitTests(createBool(false), createBool(false), createString("")) 119 | structure.InitControlSwitches() 120 | SetControlSwitches(structure) 121 | }) 122 | 123 | Context("Request body is empty", func() { 124 | It("mutate - should return an error", func() { 125 | req := httptest.NewRequest("POST", "https://fakewebhook/mutate", nil) 126 | w := httptest.NewRecorder() 127 | MutateHandler(w, req) 128 | resp := w.Result() 129 | Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) 130 | }) 131 | }) 132 | 133 | Context("Content type is not application/json", func() { 134 | It("mutate - should return an error", func() { 135 | req := httptest.NewRequest("POST", "https://fakewebhook/mutate", bytes.NewBufferString("fake-body")) 136 | req.Header.Set("Content-Type", "invalid-type") 137 | w := httptest.NewRecorder() 138 | MutateHandler(w, req) 139 | resp := w.Result() 140 | Expect(resp.StatusCode).To(Equal(http.StatusUnsupportedMediaType)) 141 | }) 142 | }) 143 | }) 144 | 145 | DescribeTable("Get network selections", 146 | 147 | func(annotateKey string, pod corev1.Pod, patchs []nritypes.JsonPatchOperation, out string, shouldExist bool) { 148 | nets, exist := getNetworkSelections(annotateKey, pod, patchs) 149 | Expect(exist).To(Equal(shouldExist)) 150 | Expect(nets).Should(Equal(out)) 151 | }, 152 | Entry( 153 | "get from pod original annotation", 154 | "k8s.v1.cni.cncf.io/networks", 155 | corev1.Pod{ 156 | ObjectMeta: metav1.ObjectMeta{ 157 | Name: "test", 158 | Annotations: map[string]string{"k8s.v1.cni.cncf.io/networks": "sriov-net"}, 159 | }, 160 | Spec: corev1.PodSpec{}, 161 | }, 162 | []nritypes.JsonPatchOperation{ 163 | { 164 | Operation: "add", 165 | Path: "/metadata/annotations", 166 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net-user-defined"}, 167 | }, 168 | }, 169 | "sriov-net", 170 | true, 171 | ), 172 | Entry( 173 | "get from pod user-defined annotation", 174 | "k8s.v1.cni.cncf.io/networks", 175 | corev1.Pod{ 176 | ObjectMeta: metav1.ObjectMeta{ 177 | Name: "test", 178 | Annotations: map[string]string{"v1.multus-cni.io/default-network": "sriov-net"}, 179 | }, 180 | Spec: corev1.PodSpec{}, 181 | }, 182 | []nritypes.JsonPatchOperation{ 183 | { 184 | Operation: "add", 185 | Path: "/metadata/annotations", 186 | Value: map[string]interface{}{"k8s.v1.cni.cncf.io/networks": "sriov-net-user-defined"}, 187 | }, 188 | }, 189 | "sriov-net-user-defined", 190 | true, 191 | ), 192 | Entry( 193 | "get from pod user-defined annotation", 194 | "k8s.v1.cni.cncf.io/networks", 195 | corev1.Pod{ 196 | ObjectMeta: metav1.ObjectMeta{ 197 | Name: "test", 198 | Annotations: map[string]string{"v1.multus-cni.io/default-network": "sriov-net"}, 199 | }, 200 | Spec: corev1.PodSpec{}, 201 | }, 202 | []nritypes.JsonPatchOperation{ 203 | { 204 | Operation: "add", 205 | Path: "/metadata/annotations", 206 | Value: map[string]interface{}{"v1.multus-cni.io/default-network": "sriov-net-user-defined"}, 207 | }, 208 | }, 209 | "", 210 | false, 211 | ), 212 | ) 213 | 214 | var emptyList []*types.NetworkSelectionElement 215 | DescribeTable("Network selection elements parsing", 216 | 217 | func(in string, out []*types.NetworkSelectionElement, shouldFail bool) { 218 | actualOut, err := parsePodNetworkSelections(in, "default") 219 | Expect(actualOut).To(ConsistOf(out)) 220 | if shouldFail { 221 | Expect(err).To(HaveOccurred()) 222 | } 223 | }, 224 | Entry( 225 | "empty config", 226 | "", 227 | emptyList, 228 | false, 229 | ), 230 | Entry( 231 | "csv - correct ns/net@if format", 232 | "ns1/net1@eth0", 233 | []*types.NetworkSelectionElement{ 234 | { 235 | Namespace: "ns1", 236 | Name: "net1", 237 | InterfaceRequest: "eth0", 238 | }, 239 | }, 240 | false, 241 | ), 242 | Entry( 243 | "csv - correct net@if format", 244 | "net1@eth0", 245 | []*types.NetworkSelectionElement{ 246 | { 247 | Namespace: "default", 248 | Name: "net1", 249 | InterfaceRequest: "eth0", 250 | }, 251 | }, 252 | false, 253 | ), 254 | Entry( 255 | "csv - correct *name-only* format", 256 | "net1", 257 | []*types.NetworkSelectionElement{ 258 | { 259 | Namespace: "default", 260 | Name: "net1", 261 | InterfaceRequest: "", 262 | }, 263 | }, 264 | false, 265 | ), 266 | Entry( 267 | "csv - correct ns/net format", 268 | "ns1/net1", 269 | []*types.NetworkSelectionElement{ 270 | { 271 | Namespace: "ns1", 272 | Name: "net1", 273 | InterfaceRequest: "", 274 | }, 275 | }, 276 | false, 277 | ), 278 | Entry( 279 | "csv - correct multiple networks format", 280 | "ns1/net1,net2", 281 | []*types.NetworkSelectionElement{ 282 | { 283 | Namespace: "ns1", 284 | Name: "net1", 285 | InterfaceRequest: "", 286 | }, 287 | { 288 | Namespace: "default", 289 | Name: "net2", 290 | InterfaceRequest: "", 291 | }, 292 | }, 293 | false, 294 | ), 295 | Entry( 296 | "csv - incorrect format forward slashes", 297 | "ns1/net1/if1", 298 | emptyList, 299 | true, 300 | ), 301 | Entry( 302 | "csv - incorrect format @'s", 303 | "net1@if1@if2", 304 | emptyList, 305 | true, 306 | ), 307 | Entry( 308 | "csv - incorrect mixed with correct", 309 | "ns/net1,net2,net3@if1@if2", 310 | emptyList, 311 | true, 312 | ), 313 | Entry( 314 | "json - not an array", 315 | `{"name": "net1"}`, 316 | emptyList, 317 | true, 318 | ), 319 | Entry( 320 | "json - correct example", 321 | `[{"name": "net1"},{"name": "net2", "namespace": "ns1"}]`, 322 | []*types.NetworkSelectionElement{ 323 | { 324 | Namespace: "default", 325 | Name: "net1", 326 | InterfaceRequest: "", 327 | }, 328 | { 329 | Namespace: "ns1", 330 | Name: "net2", 331 | InterfaceRequest: "", 332 | }, 333 | }, 334 | false, 335 | ), 336 | ) 337 | }) 338 | -------------------------------------------------------------------------------- /scripts/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2019 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http:#www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Note: Execute only from the package root directory or top-level Makefile! 18 | 19 | set -e 20 | 21 | # set proxy args 22 | BUILD_ARGS=() 23 | [ ! -z "$http_proxy" ] && BUILD_ARGS+=("--build-arg http_proxy=$http_proxy") 24 | [ ! -z "$HTTP_PROXY" ] && BUILD_ARGS+=("--build-arg HTTP_PROXY=$HTTP_PROXY") 25 | [ ! -z "$https_proxy" ] && BUILD_ARGS+=("--build-arg https_proxy=$https_proxy") 26 | [ ! -z "$HTTPS_PROXY" ] && BUILD_ARGS+=("--build-arg HTTPS_PROXY=$HTTPS_PROXY") 27 | [ ! -z "$no_proxy" ] && BUILD_ARGS+=("--build-arg no_proxy=$no_proxy") 28 | [ ! -z "$NO_PROXY" ] && BUILD_ARGS+=("--build-arg NO_PROXY=$NO_PROXY") 29 | 30 | # build admission controller Docker image 31 | docker build ${BUILD_ARGS[@]} -f Dockerfile -t network-resources-injector . 32 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http:#www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | ORG_PATH="github.com/k8snetworkplumbingwg" 20 | REPO_PATH="${ORG_PATH}/network-resources-injector" 21 | 22 | if [ ! -h .gopath/src/${REPO_PATH} ]; then 23 | mkdir -p .gopath/src/${ORG_PATH} 24 | ln -s ../../../.. .gopath/src/${REPO_PATH} || exit 255 25 | fi 26 | 27 | export GOPATH=${PWD}/.gopath 28 | export GOBIN=${PWD}/bin 29 | export CGO_ENABLED=0 30 | export GO15VENDOREXPERIMENT=1 31 | 32 | go install -ldflags "-s -w" -tags no_openssl "$@" ${REPO_PATH}/cmd/installer 33 | go install -ldflags "-s -w" -tags no_openssl "$@" ${REPO_PATH}/cmd/webhook 34 | -------------------------------------------------------------------------------- /scripts/control-plane-additions/ac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiserver.k8s.io/v1alpha1 2 | kind: AdmissionConfiguration 3 | plugins: 4 | - name: MutatingAdmissionWebhook 5 | configuration: 6 | apiVersion: apiserver.config.k8s.io/v1 7 | kind: WebhookAdmissionConfiguration 8 | kubeConfigFile: /etc/kubernetes/pki/mutatingkubeconfig.yaml 9 | -------------------------------------------------------------------------------- /scripts/control-plane-additions/mutatingkubeconfig.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | users: 4 | - name: network-resources-injector-service.kube-system.svc 5 | user: 6 | client-certificate-data: CERT 7 | client-key-data: KEY 8 | -------------------------------------------------------------------------------- /scripts/e2e_cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Remove any test artifacts created by tests 3 | set -o errexit 4 | 5 | here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" 6 | root="$(readlink --canonicalize "$here/..")" 7 | tmp_dir="${root}/test/tmp" 8 | 9 | echo "removing '${tmp_dir}' and '${root}/bin'" 10 | rm -rf --preserve-root "${tmp_dir:?}" "${root:?}bin" 11 | rm -f "${root}/deployments/server_huge.yaml" 12 | -------------------------------------------------------------------------------- /scripts/e2e_get_tools.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o errexit 3 | # ensure this file is sourced to add required components to PATH 4 | 5 | here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" 6 | root="$(readlink --canonicalize "$here/..")" 7 | VERSION="v0.26.0" 8 | KIND_BINARY_URL="https://github.com/kubernetes-sigs/kind/releases/download/${VERSION}/kind-$(uname)-amd64" 9 | K8_STABLE_RELEASE_URL="https://storage.googleapis.com/kubernetes-release/release/stable.txt" 10 | 11 | if [ ! -d "${root}/bin" ]; then 12 | mkdir "${root}/bin" 13 | fi 14 | 15 | echo "retrieving kind" 16 | curl --max-time 20 --retry 10 --retry-delay 5 --retry-max-time 300 -Lo "${root}/bin/kind" "${KIND_BINARY_URL}" 17 | chmod +x "${root}/bin/kind" 18 | 19 | echo "retrieving kubectl" 20 | curl -Lo "${root}/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/$(curl -s ${K8_STABLE_RELEASE_URL})/bin/linux/amd64/kubectl" 21 | chmod +x "${root}/bin/kubectl" 22 | 23 | export PATH="$PATH:$root/bin" 24 | -------------------------------------------------------------------------------- /scripts/e2e_setup_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | # Create CA cert and key which will be used as root CA for Kubernetes. 4 | # we also create 2 yaml artifacts to enable K8 API server flag 5 | # --admission-control-config. MutatatingAdmissionWebhook controller 6 | # will read its credentials stored in a kube config file attached to this flag. 7 | # For simplicity sake, we use the pre-generated CA cert and key as the 8 | # credentials for NRI. 9 | 10 | here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" 11 | root="$(readlink --canonicalize "$here/..")" 12 | RETRY_MAX=10 13 | INTERVAL=30 14 | TIMEOUT=300 15 | APP_NAME="network-resources-injector" 16 | APP_DOCKER_TAG="${APP_NAME}:latest" 17 | K8_ADDITIONS_PATH="${root}/scripts/control-plane-additions" 18 | TMP_DIR="${root}/test/tmp" 19 | MULTUS_DAEMONSET_URL="https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml" 20 | MULTUS_NAME="multus" 21 | CNIS_DAEMONSET_URL="https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/e2e/templates/cni-install.yml.j2" 22 | CNIS_NAME="cni-plugins" 23 | # array with the KinD workers 24 | KIND_WORKER_NAMES=( kind-worker kind-worker2 ) 25 | 26 | # create cluster CA and API server admission configuration 27 | # to force API server and NRI authentication. 28 | # CA cert & key along with supporting yamls will be mounted to control plane 29 | # path /etc/kubernetes/pki. Kubeadm will utilise generated CA cert/key as root 30 | # Kubernetes CA. Cert passed to NRI will be signed by this CA. 31 | generate_k8_api_data() { 32 | mkdir -p "${TMP_DIR}" 33 | mount_dir="$(mktemp -q -p "${TMP_DIR}" -d -t nri-e2e-k8-api-pki-XXXXXXXX)" 34 | echo "### creating K8 CA cert & private key" 35 | openssl req \ 36 | -nodes \ 37 | -subj "/C=IE/ST=None/L=None/O=None/CN=kubernetes" \ 38 | -new -x509 \ 39 | -days 1 \ 40 | -keyout "${mount_dir:?}/ca.key" \ 41 | -out "${mount_dir}/ca.crt" > /dev/null 2>&1 42 | echo "### add admission config for K8 API server" 43 | # add kube config file for NRI 44 | cp "${K8_ADDITIONS_PATH}/ac.yaml" "${K8_ADDITIONS_PATH}/mutatingkubeconfig.yaml" \ 45 | "${mount_dir}/" 46 | echo "### add cert & key data to kube config template" 47 | cert_data="$(base64 -w 0 < "${mount_dir}/ca.crt")" 48 | key_data="$(base64 -w 0 < "${mount_dir}/ca.key")" 49 | sed -i -e "s/CERT/${cert_data}/" -e "s/KEY/${key_data}/" \ 50 | "${mount_dir}/mutatingkubeconfig.yaml" 51 | } 52 | 53 | retry() { 54 | local status=0 55 | local retries=${RETRY_MAX:=5} 56 | local delay=${INTERVAL:=5} 57 | local to=${TIMEOUT:=20} 58 | cmd="$*" 59 | 60 | while [ $retries -gt 0 ] 61 | do 62 | status=0 63 | timeout $to bash -c "echo $cmd && $cmd" || status=$? 64 | if [ $status -eq 0 ]; then 65 | break; 66 | fi 67 | echo "Exit code: '$status'. Sleeping '$delay' seconds before retrying" 68 | sleep $delay 69 | (( retries-- )) || true 70 | done 71 | return $status 72 | } 73 | 74 | create_cluster() { 75 | [ -z "${mount_dir}" ] && echo "### no mount directory set" && exit 1 76 | 77 | # create list of worker nodes 78 | workers="$(for i in "${KIND_WORKER_NAMES[@]}"; do echo " - role: worker"; done)" 79 | 80 | # create KinD configuration file 81 | exec 3<> "${PWD}"/kindConfig.yaml 82 | 83 | # Let's print Kind configuration to file to fd 3 84 | echo "kind: Cluster" >&3 85 | echo "apiVersion: kind.x-k8s.io/v1alpha4" >&3 86 | echo "kubeadmConfigPatches:" >&3 87 | echo "- |" >&3 88 | echo " kind: ClusterConfiguration" >&3 89 | echo " apiServer:" >&3 90 | echo " extraArgs:" >&3 91 | echo " admission-control-config-file: /etc/kubernetes/pki/ac.yaml" >&3 92 | echo "nodes:" >&3 93 | echo " - role: control-plane" >&3 94 | echo " extraMounts:" >&3 95 | echo " - hostPath: \"${mount_dir:?}\"" >&3 96 | echo " containerPath: \"/etc/kubernetes/pki\"" >&3 97 | echo "${workers}" >&3 98 | 99 | # Close fd 3 100 | exec 3>&- 101 | 102 | # deploy cluster with kind 103 | retry kind delete cluster && kind create cluster --config="${PWD}"/kindConfig.yaml 104 | 105 | rm "${PWD}"/kindConfig.yaml 106 | } 107 | 108 | check_requirements() { 109 | for cmd in docker kind openssl kubectl base64; do 110 | if ! command -v "$cmd" &> /dev/null; then 111 | echo "$cmd is not available" 112 | exit 1 113 | fi 114 | done 115 | } 116 | 117 | patch_kind_node() { 118 | echo "## Adding capacity of example.com/foo to $1 node" 119 | curl -g --retry ${RETRY_MAX} --retry-delay ${INTERVAL} --connect-timeout ${TIMEOUT} --header "Content-Type: application/json-patch+json" \ 120 | --request PATCH --data '[{"op": "add", "path": "/status/capacity/example.com~1foo", "value": "100"}]' \ 121 | http://127.0.0.1:8001/api/v1/nodes/"$1"/status > /dev/null 122 | 123 | curl -g --retry ${RETRY_MAX} --retry-delay ${INTERVAL} --connect-timeout ${TIMEOUT} --header "Content-Type: application/json-patch+json" \ 124 | --request PATCH --data '[{"op": "add", "path": "/status/capacity/example.com~1boo", "value": "100"}]' \ 125 | http://127.0.0.1:8001/api/v1/nodes/"$1"/status > /dev/null 126 | } 127 | 128 | echo "## checking requirements" 129 | check_requirements 130 | # generate K8 API server CA key/cert and supporting files for mTLS with NRI 131 | echo "## generating K8 api flags files" 132 | generate_k8_api_data 133 | echo "## start Kind cluster with precreated CA key/cert" 134 | create_cluster 135 | echo "## remove taints from master node" 136 | kubectl taint nodes kind-control-plane node-role.kubernetes.io/control-plane:NoSchedule- 137 | echo "## build NRI" 138 | retry docker build -t "${APP_DOCKER_TAG}" "${root}" 139 | echo "## load NRI image into Kind" 140 | kind load docker-image "${APP_DOCKER_TAG}" 141 | echo "## export kube config for utilising locally" 142 | kind export kubeconfig 143 | echo "## install coreDNS" 144 | kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s 145 | echo "## install multus" 146 | retry kubectl create -f "${MULTUS_DAEMONSET_URL}" 147 | retry kubectl -n kube-system wait --for=condition=ready -l name="${MULTUS_NAME}" pod --timeout=300s 148 | echo "## install CNIs" 149 | retry kubectl create -f "${CNIS_DAEMONSET_URL}" 150 | retry kubectl -n kube-system wait --for=condition=ready -l name="${CNIS_NAME}" pod --timeout=300s 151 | echo "## install NRI" 152 | retry kubectl create -f "${root}/deployments/auth.yaml" 153 | # enable HugePage API at webhook 154 | awk '/-logtostderr/ { print; print " - -injectHugepageDownApi"; next }1' "${root}/deployments/server.yaml" > "${root}/deployments/server_huge.yaml" 155 | retry kubectl create -f "${root}/deployments/server_huge.yaml" 156 | retry kubectl -n kube-system wait --for=condition=ready -l app="${APP_NAME}" pod --timeout=300s 157 | sleep 5 158 | echo "## starting kube proxy" 159 | nohup kubectl proxy -p=8001 > /dev/null 2>&1 & 160 | proxy_pid=$! 161 | sleep 1 162 | 163 | echo "## adding capacity of 4 example.com/foo to kind-worker node" 164 | for (( i = 0; i < "${#KIND_WORKER_NAMES[@]}"; i++ )); do 165 | patch_kind_node "${KIND_WORKER_NAMES[${i}]}" || true 166 | done 167 | 168 | echo "## killing kube proxy" 169 | kill $proxy_pid 170 | -------------------------------------------------------------------------------- /scripts/e2e_teardown_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Teardown Kind cluster 3 | 4 | if ! command -v kind &> /dev/null; then 5 | echo "KinD is not available" 6 | exit 1 7 | fi 8 | 9 | kind delete cluster 10 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http:#www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Note: Execute only from the package root directory or top-level Makefile! 18 | 19 | set -ex 20 | root="$(dirname "$0")/.." 21 | time=$(date +'%Y-%m-%d_%H-%M-%S') 22 | filePath="/tmp/go-cover.$time.tmp" 23 | echo "Coverage profile file path: $filePath" 24 | go test --tags=unittests -race -coverprofile="$filePath" "./${root}/pkg/..." 25 | go tool cover -html="$filePath" 26 | -------------------------------------------------------------------------------- /scripts/webhook-deployment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http:#www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Always exit on errors. 18 | set -e 19 | 20 | # Set our known directories and parameters. 21 | BASE_DIR="$(cd "$(dirname "$0")"/..; pwd)" 22 | NAMESPACE="kube-system" 23 | 24 | # Give help text for parameters. 25 | function usage() 26 | { 27 | echo -e "./webhook-deployment.sh" 28 | echo -e "\t-h --help" 29 | echo -e "\t--namespace=${NAMESPACE}" 30 | } 31 | 32 | 33 | # Parse parameters given as arguments to this script. 34 | while [ "$1" != "" ]; do 35 | PARAM="$(echo "$1" | awk -F= '{print $1}')" 36 | VALUE="$(echo "$1" | awk -F= '{print $2}')" 37 | case $PARAM in 38 | -h | --help) 39 | usage 40 | exit 41 | ;; 42 | --namespace) 43 | NAMESPACE=$VALUE 44 | ;; 45 | *) 46 | echo "ERROR: unknown parameter \"$PARAM\"" 47 | usage 48 | exit 1 49 | ;; 50 | esac 51 | shift 52 | done 53 | 54 | kubectl -n "${NAMESPACE}" create -f "${BASE_DIR}/deployments/service.yaml" 55 | export NAMESPACE 56 | cat "${BASE_DIR}/deployments/webhook.yaml" | \ 57 | "${BASE_DIR}/scripts/webhook-patch-ca-bundle.sh" | \ 58 | sed -e "s|\${NAMESPACE}|${NAMESPACE}|g" | \ 59 | kubectl -n "${NAMESPACE}" create -f - 60 | 61 | kubectl -n "${NAMESPACE}" create -f "${BASE_DIR}/deployments/auth.yaml" 62 | kubectl -n "${NAMESPACE}" create -f "${BASE_DIR}/deployments/server.yaml" 63 | -------------------------------------------------------------------------------- /scripts/webhook-patch-ca-bundle.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Original script found at: https://github.com/morvencao/kube-mutating-webhook-tutorial/blob/master/deployment/webhook-patch-ca-bundle.sh 3 | 4 | set -o errexit 5 | set -o nounset 6 | set -o pipefail 7 | 8 | export CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 --w=0) 9 | 10 | if command -v envsubst >/dev/null 2>&1; then 11 | envsubst 12 | else 13 | sed -e "s|\${CA_BUNDLE}|${CA_BUNDLE}|g" 14 | fi 15 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | ## NRI e2e test with kind 2 | 3 | ### How to test e2e 4 | 5 | ``` 6 | $ git clone https://github.com/k8snetworkplumbingwg/network-resources-injector.git 7 | $ cd network-resources-injector/ 8 | $ source scripts/e2e_get_tools.sh 9 | $ ./scripts/e2e_setup_cluster.sh 10 | $ go test ./test/e2e/... 11 | ``` 12 | 13 | ### How to teardown cluster 14 | 15 | ``` 16 | $ ./scripts/e2e_teardown.sh 17 | ``` 18 | 19 | ### How to cleanup test artifacts 20 | 21 | ``` 22 | $ ./scripts/e2e_cleanup.sh 23 | ``` 24 | 25 | ### How to change default test image 26 | By default, ```alpline:latest``` image is used as a base test image. To override this 27 | set environment variables ```IMAGE_REGISTRY```and ```TEST_IMAGE```. For example: 28 | 29 | ``` 30 | $ export IMAGE_REGISTRY=localhost:5000 31 | $ export TEST_IMAGE=nginx:latest 32 | ``` 33 | 34 | ### Current test cases 35 | * Test injection of one network 36 | * Test injection of two networks 37 | -------------------------------------------------------------------------------- /test/e2e/control_switches_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | //a subset of tests require Hugepages enabled on the test node 4 | import ( 5 | "fmt" 6 | "time" 7 | 8 | "github.com/k8snetworkplumbingwg/network-resources-injector/test/util" 9 | 10 | cniv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" 11 | corev1 "k8s.io/api/core/v1" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/ginkgo/extensions/table" 15 | . "github.com/onsi/gomega" 16 | ) 17 | 18 | var _ = Describe("Verify configuration set by control switches", func() { 19 | var pod *corev1.Pod 20 | var configMap *corev1.ConfigMap 21 | var nad *cniv1.NetworkAttachmentDefinition 22 | var err error 23 | 24 | BeforeEach(hugepageOrSkip) 25 | 26 | Context("Incorrect nri-control-switches ConfigMap", func() { 27 | BeforeEach(func() { 28 | nad = util.GetResourceSelectorOnly(testNetworkName, *testNs, testNetworkResName) 29 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 30 | }) 31 | 32 | AfterEach(func() { 33 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 34 | _ = util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout) 35 | 36 | if nil != configMap { 37 | _ = util.DeleteConfigMap(cs.CoreV1Interface, configMap, timeout) 38 | configMap = nil 39 | } 40 | }) 41 | 42 | const secondValue = ` 43 | "user-defined-injections": { 44 | "customInjection": { 45 | "op": "add", 46 | "path": "/metadata/annotations", 47 | "value": { 48 | "top-secret": "password" 49 | } 50 | } 51 | }, 52 | ` 53 | 54 | DescribeTable("verify if ConfigMap was accepted", func(configValue string) { 55 | configMap = util.GetConfigMap("nri-control-switches", "kube-system") 56 | configMap = util.AddData(configMap, "config.json", "{"+secondValue+configValue+"}") 57 | Expect(util.ApplyConfigMap(cs.CoreV1Interface, configMap, timeout)).Should(BeNil()) 58 | 59 | // wait for configmap to be consumed by NRI 60 | time.Sleep(60 * time.Second) 61 | 62 | pod = util.GetOneNetwork(testNetworkName, *testNs, defaultPodName) 63 | pod = util.AddMetadataLabel(pod, "customInjection", "true") 64 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 65 | Expect(pod.Name).ShouldNot(BeNil()) 66 | 67 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 68 | fmt.Println(pod) 69 | Expect(err).Should(BeNil()) 70 | Expect(pod.Annotations["top-secret"]).Should(ContainSubstring("password")) 71 | }, 72 | Entry("without features definition, correct namespace", ` 73 | "networkResourceNameKeys": ["k8s.v1.cni.cncf.io/resourceName", "k8s.v1.cni.cncf.io/bridgeName"] 74 | `), 75 | Entry("all features disabled, correct namespace", ` 76 | "features": { 77 | "enableHugePageDownApi": false, 78 | "enableHonorExistingResources": false, 79 | "enableCustomizedInjection": false, 80 | "enableResourceName": false 81 | } 82 | `), 83 | Entry("unknown features or misspelled, correct namespace", ` 84 | "features": { 85 | "someSuperFeature": false, 86 | "resourceName": false 87 | } 88 | `), 89 | ) 90 | }) 91 | }) 92 | -------------------------------------------------------------------------------- /test/e2e/e2e_tests_suite_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "path/filepath" 7 | "testing" 8 | "time" 9 | 10 | networkCoreClient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" 11 | "github.com/k8snetworkplumbingwg/network-resources-injector/test/util" 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | coreclient "k8s.io/client-go/kubernetes/typed/core/v1" 15 | restclient "k8s.io/client-go/rest" 16 | "k8s.io/client-go/tools/clientcmd" 17 | "k8s.io/client-go/util/homedir" 18 | ) 19 | 20 | const ( 21 | defaultPodName = "nri-e2e-test" 22 | testNetworkName = "foo-network" 23 | pod1stContainerName = "test" 24 | pod2ndContainerName = "second" 25 | testNetworkResName = "example.com/foo" 26 | interval = time.Second * 10 27 | timeout = time.Second * 30 28 | minHugepages1Gi = 2 29 | minHugepages2Mi = 1024 30 | ) 31 | 32 | type ClientSet struct { 33 | coreclient.CoreV1Interface 34 | } 35 | 36 | type NetworkClientSet struct { 37 | networkCoreClient.K8sCniCncfIoV1Interface 38 | } 39 | 40 | var ( 41 | master *string 42 | kubeConfigPath *string 43 | testNs *string 44 | cs *ClientSet 45 | networkClient *NetworkClientSet 46 | kubeConfig *restclient.Config 47 | ) 48 | 49 | func init() { 50 | if home := homedir.HomeDir(); home != "" { 51 | kubeConfigPath = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "path to your kubeconfig file") 52 | } else { 53 | kubeConfigPath = flag.String("kubeconfig", "", "require absolute path to your kubeconfig file") 54 | } 55 | master = flag.String("master", "", "Address of Kubernetes API server") 56 | testNs = flag.String("testnamespace", "default", "namespace for testing") 57 | } 58 | 59 | func TestSriovTests(t *testing.T) { 60 | RegisterFailHandler(Fail) 61 | RunSpecs(t, "NRI E2E suite") 62 | } 63 | 64 | var _ = BeforeSuite(func(done Done) { 65 | cfg, err := clientcmd.BuildConfigFromFlags(*master, *kubeConfigPath) 66 | Expect(err).Should(BeNil()) 67 | 68 | kubeConfig = cfg 69 | 70 | cs = &ClientSet{} 71 | cs.CoreV1Interface = coreclient.NewForConfigOrDie(cfg) 72 | 73 | networkClient = &NetworkClientSet{} 74 | networkClient.K8sCniCncfIoV1Interface = networkCoreClient.NewForConfigOrDie(cfg) 75 | 76 | close(done) 77 | }, 60) 78 | 79 | func hugepageOrSkip() { 80 | available, err := util.IsMinHugepagesAvailable(cs.CoreV1Interface, minHugepages1Gi, minHugepages2Mi) 81 | Expect(err).To(BeNil()) 82 | if !available { 83 | Skip(fmt.Sprintf("minimum hugepages of %d Gi and %d Mi not found in any k8 worker nodes.", 84 | minHugepages1Gi, minHugepages2Mi)) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /test/e2e/nodeselector_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "github.com/k8snetworkplumbingwg/network-resources-injector/test/util" 5 | . "github.com/onsi/ginkgo" 6 | . "github.com/onsi/gomega" 7 | 8 | cniv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" 9 | corev1 "k8s.io/api/core/v1" 10 | ) 11 | 12 | var _ = Describe("Node selector test", func() { 13 | var pod *corev1.Pod 14 | var nad *cniv1.NetworkAttachmentDefinition 15 | var err error 16 | 17 | Context("Cluster node available, default namespace", func() { 18 | AfterEach(func() { 19 | _ = util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout) 20 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 21 | }) 22 | 23 | It("POD assigned to correct cluster node, only node specified without resource name", func() { 24 | nad = util.GetNodeSelectorOnly(testNetworkName, *testNs, "kubernetes.io/hostname=kind-worker2") 25 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 26 | 27 | podName := defaultPodName + "-1" 28 | pod = util.GetOneNetwork(testNetworkName, *testNs, podName) 29 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 30 | 31 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 32 | Expect(err).Should(BeNil()) 33 | 34 | Expect(pod.Name).Should(Equal("nri-e2e-test-1")) 35 | Expect(pod.Spec.NodeName).Should(Equal("kind-worker2")) 36 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker2"})) 37 | Expect(pod.ObjectMeta.Namespace).Should(Equal(*testNs)) 38 | }) 39 | 40 | It("POD assigned to correct cluster node, node specified with resource name", func() { 41 | nad = util.GetResourceAndNodeSelector(testNetworkName, *testNs, "kubernetes.io/hostname=kind-worker2") 42 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 43 | 44 | podName := defaultPodName + "-2" 45 | pod = util.GetOneNetwork(testNetworkName, *testNs, podName) 46 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 47 | 48 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 49 | Expect(err).Should(BeNil()) 50 | 51 | Expect(pod.Name).Should(Equal("nri-e2e-test-2")) 52 | Expect(pod.Spec.NodeName).Should(Equal("kind-worker2")) 53 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker2"})) 54 | Expect(pod.ObjectMeta.Namespace).Should(Equal(*testNs)) 55 | }) 56 | }) 57 | 58 | Context("Cluster node not available, default namespace", func() { 59 | AfterEach(func() { 60 | _ = util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout) 61 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 62 | }) 63 | 64 | It("POD in pending state, only node selector passed without resource name", func() { 65 | nad = util.GetNodeSelectorOnly(testNetworkName, *testNs, "kubernetes.io/hostname=kind-worker15") 66 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 67 | 68 | podName := defaultPodName + "-3" 69 | pod = util.GetOneNetwork(testNetworkName, *testNs, podName) 70 | err = util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval) 71 | Expect(err).ShouldNot(BeNil()) 72 | Expect(err.Error()).Should(HavePrefix("timed out waiting for the condition")) 73 | 74 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 75 | Expect(err).Should(BeNil()) 76 | Expect(pod.Status.Phase).Should(Equal(corev1.PodPending)) 77 | Expect(pod.Name).Should(Equal("nri-e2e-test-3")) 78 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker15"})) 79 | }) 80 | 81 | It("POD in pending state, node selector and resource name in CRD", func() { 82 | nad = util.GetResourceAndNodeSelector(testNetworkName, *testNs, "kubernetes.io/hostname=kind-worker10") 83 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 84 | 85 | podName := defaultPodName + "-4" 86 | pod = util.GetOneNetwork(testNetworkName, *testNs, podName) 87 | err = util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval) 88 | Expect(err).ShouldNot(BeNil()) 89 | Expect(err.Error()).Should(HavePrefix("timed out waiting for the condition")) 90 | 91 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 92 | Expect(err).Should(BeNil()) 93 | Expect(pod.Status.Phase).Should(Equal(corev1.PodPending)) 94 | Expect(pod.Name).Should(Equal("nri-e2e-test-4")) 95 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker10"})) 96 | }) 97 | }) 98 | 99 | Context("Cluster node available with custom namespace", func() { 100 | var testNamespace string 101 | 102 | BeforeEach(func() { 103 | testNamespace = "mysterious" 104 | Expect(util.CreateNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 105 | }) 106 | 107 | AfterEach(func() { 108 | Expect(util.DeleteNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 109 | }) 110 | 111 | It("POD assigned to correct cluster node, only node specified without resource name", func() { 112 | nad = util.GetNodeSelectorOnly(testNetworkName, testNamespace, "kubernetes.io/hostname=kind-worker2") 113 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 114 | 115 | podName := defaultPodName + "-5" 116 | pod = util.GetOneNetwork(testNetworkName, testNamespace, podName) 117 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 118 | 119 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 120 | Expect(err).Should(BeNil()) 121 | 122 | Expect(pod.Name).Should(Equal("nri-e2e-test-5")) 123 | Expect(pod.Spec.NodeName).Should(Equal("kind-worker2")) 124 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker2"})) 125 | Expect(pod.ObjectMeta.Namespace).Should(Equal(testNamespace)) 126 | }) 127 | 128 | It("POD assigned to correct cluster node, node specified with resource name", func() { 129 | nad = util.GetResourceAndNodeSelector(testNetworkName, testNamespace, "kubernetes.io/hostname=kind-worker2") 130 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 131 | 132 | podName := defaultPodName + "-6" 133 | pod = util.GetOneNetwork(testNetworkName, testNamespace, podName) 134 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 135 | 136 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 137 | Expect(err).Should(BeNil()) 138 | 139 | Expect(pod.Name).Should(Equal("nri-e2e-test-6")) 140 | Expect(pod.Spec.NodeName).Should(Equal("kind-worker2")) 141 | Expect(pod.Spec.NodeSelector).Should(Equal(map[string]string{"kubernetes.io/hostname": "kind-worker2"})) 142 | Expect(pod.ObjectMeta.Namespace).Should(Equal(testNamespace)) 143 | }) 144 | }) 145 | }) 146 | -------------------------------------------------------------------------------- /test/e2e/resourcename_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "github.com/k8snetworkplumbingwg/network-resources-injector/test/util" 5 | . "github.com/onsi/ginkgo" 6 | . "github.com/onsi/gomega" 7 | 8 | cniv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/utils/pointer" 11 | ) 12 | 13 | var _ = Describe("Verify that resource and POD which consumes resource cannot be in different namespaces", func() { 14 | var pod *corev1.Pod 15 | var nad *cniv1.NetworkAttachmentDefinition 16 | var err error 17 | 18 | Context("network attachment definition configuration error", func() { 19 | It("Missing network attachment definition, try to setup POD in default namespace", func() { 20 | pod = util.GetOneNetwork(testNetworkName, *testNs, defaultPodName) 21 | err = util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval) 22 | Expect(err).ShouldNot(BeNil()) 23 | Expect(err.Error()).Should(ContainSubstring("could not get Network Attachment Definition default/foo-network")) 24 | }) 25 | 26 | It("Correct network name in CRD, but the namespace if different than in POD specification", func() { 27 | testNamespace := "mysterious" 28 | Expect(util.CreateNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 29 | 30 | nad = util.GetResourceSelectorOnly(testNetworkName, testNamespace, testNetworkResName) 31 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 32 | 33 | pod = util.GetOneNetwork(testNetworkName, *testNs, defaultPodName) 34 | err = util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval) 35 | Expect(err).ShouldNot(BeNil()) 36 | Expect(err.Error()).Should(ContainSubstring("could not get Network Attachment Definition default/foo-network")) 37 | 38 | Expect(util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout)).Should(BeNil()) 39 | 40 | Expect(util.DeleteNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 41 | }) 42 | 43 | It("CRD in default namespace, and POD in custom namespace", func() { 44 | testNamespace := "mysterious" 45 | Expect(util.CreateNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 46 | 47 | nad = util.GetResourceSelectorOnly(testNetworkName, *testNs, testNetworkResName) 48 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 49 | 50 | pod = util.GetOneNetwork(testNetworkName, testNamespace, defaultPodName) 51 | err = util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval) 52 | Expect(err).ShouldNot(BeNil()) 53 | Expect(err.Error()).Should(ContainSubstring("could not get Network Attachment Definition mysterious/foo-network")) 54 | 55 | Expect(util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout)).Should(BeNil()) 56 | 57 | Expect(util.DeleteNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 58 | }) 59 | }) 60 | }) 61 | 62 | var _ = Describe("Network injection testing", func() { 63 | var pod *corev1.Pod 64 | var nad *cniv1.NetworkAttachmentDefinition 65 | var err error 66 | 67 | Context("one network request in default namespace", func() { 68 | BeforeEach(func() { 69 | nad = util.GetResourceSelectorOnly(testNetworkName, *testNs, testNetworkResName) 70 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 71 | 72 | pod = util.GetOneNetwork(testNetworkName, *testNs, defaultPodName) 73 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 74 | Expect(pod.Name).ShouldNot(BeNil()) 75 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 76 | Expect(err).Should(BeNil()) 77 | }) 78 | 79 | AfterEach(func() { 80 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 81 | Expect(util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout)).Should(BeNil()) 82 | }) 83 | 84 | It("should have one limit injected", func() { 85 | limNo, ok := pod.Spec.Containers[0].Resources.Limits[testNetworkResName] 86 | Expect(ok).Should(BeTrue()) 87 | Expect(limNo.String()).Should(Equal("1")) 88 | }) 89 | 90 | It("should have one request injected", func() { 91 | limNo, ok := pod.Spec.Containers[0].Resources.Requests[testNetworkResName] 92 | Expect(ok).Should(BeTrue()) 93 | Expect(limNo.String()).Should(Equal("1")) 94 | }) 95 | }) 96 | 97 | Context("two network requests in default namespace", func() { 98 | BeforeEach(func() { 99 | nad = util.GetResourceSelectorOnly(testNetworkName, *testNs, testNetworkResName) 100 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 101 | 102 | pod = util.GetMultiNetworks([]string{testNetworkName, testNetworkName}, *testNs, defaultPodName) 103 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 104 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 105 | Expect(err).Should(BeNil()) 106 | }) 107 | 108 | AfterEach(func() { 109 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 110 | _ = util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout) 111 | }) 112 | 113 | It("should have two limits injected", func() { 114 | limNo, ok := pod.Spec.Containers[0].Resources.Limits[testNetworkResName] 115 | Expect(ok).Should(BeTrue()) 116 | Expect(limNo.String()).Should(Equal("2")) 117 | }) 118 | 119 | It("should have two requests injected", func() { 120 | limNo, ok := pod.Spec.Containers[0].Resources.Requests[testNetworkResName] 121 | Expect(ok).Should(BeTrue()) 122 | Expect(limNo.String()).Should(Equal("2")) 123 | }) 124 | }) 125 | 126 | Context("one network request in custom namespace", func() { 127 | BeforeEach(func() { 128 | testNamespace := "mysterious" 129 | Expect(util.CreateNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 130 | 131 | nad = util.GetResourceSelectorOnly(testNetworkName, testNamespace, testNetworkResName) 132 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 133 | 134 | pod = util.GetOneNetwork(testNetworkName, testNamespace, defaultPodName) 135 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 136 | Expect(pod.Name).ShouldNot(BeNil()) 137 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 138 | Expect(err).Should(BeNil()) 139 | }) 140 | 141 | AfterEach(func() { 142 | testNamespace := "mysterious" 143 | Expect(util.DeleteNamespace(cs.CoreV1Interface, testNamespace, timeout)).Should(BeNil()) 144 | }) 145 | 146 | It("should have one limit injected", func() { 147 | limNo, ok := pod.Spec.Containers[0].Resources.Limits[testNetworkResName] 148 | Expect(ok).Should(BeTrue()) 149 | Expect(limNo.String()).Should(Equal("1")) 150 | }) 151 | 152 | It("should have one request injected", func() { 153 | limNo, ok := pod.Spec.Containers[0].Resources.Requests[testNetworkResName] 154 | Expect(ok).Should(BeTrue()) 155 | Expect(limNo.String()).Should(Equal("1")) 156 | }) 157 | }) 158 | 159 | Context("one network request and automountServiceAccountToken=false", func() { 160 | BeforeEach(func() { 161 | nad = util.GetResourceSelectorOnly(testNetworkName, *testNs, testNetworkResName) 162 | Expect(util.ApplyNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, nad, timeout)).Should(BeNil()) 163 | 164 | pod = util.GetOneNetwork(testNetworkName, *testNs, defaultPodName) 165 | pod.Spec.AutomountServiceAccountToken = pointer.Bool(false) 166 | 167 | Expect(util.CreateRunningPod(cs.CoreV1Interface, pod, timeout, interval)).Should(BeNil()) 168 | Expect(pod.Name).ShouldNot(BeNil()) 169 | pod, err = util.UpdatePodInfo(cs.CoreV1Interface, pod, timeout) 170 | Expect(err).Should(BeNil()) 171 | }) 172 | 173 | AfterEach(func() { 174 | _ = util.DeletePod(cs.CoreV1Interface, pod, timeout) 175 | Expect(util.DeleteNetworkAttachmentDefinition(networkClient.K8sCniCncfIoV1Interface, testNetworkName, nad, timeout)).Should(BeNil()) 176 | }) 177 | 178 | It("should have one limit injected", func() { 179 | limNo, ok := pod.Spec.Containers[0].Resources.Limits[testNetworkResName] 180 | Expect(ok).Should(BeTrue()) 181 | Expect(limNo.String()).Should(Equal("1")) 182 | }) 183 | 184 | It("should have one request injected", func() { 185 | limNo, ok := pod.Spec.Containers[0].Resources.Requests[testNetworkResName] 186 | Expect(ok).Should(BeTrue()) 187 | Expect(limNo.String()).Should(Equal("1")) 188 | }) 189 | }) 190 | }) 191 | -------------------------------------------------------------------------------- /test/util/configmap.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | coreclient "k8s.io/client-go/kubernetes/typed/core/v1" 10 | ) 11 | 12 | func GetConfigMap(confgiMapName, namespace string) *corev1.ConfigMap { 13 | return &corev1.ConfigMap{ 14 | ObjectMeta: metav1.ObjectMeta{ 15 | Name: confgiMapName, 16 | Namespace: namespace, 17 | }, 18 | } 19 | } 20 | 21 | func AddData(configMap *corev1.ConfigMap, dataKey, dataValue string) *corev1.ConfigMap { 22 | if nil == configMap.Data { 23 | configMap.Data = make(map[string]string) 24 | } 25 | 26 | configMap.Data[dataKey] = dataValue 27 | 28 | return configMap 29 | } 30 | 31 | func ApplyConfigMap(ci coreclient.CoreV1Interface, configMap *corev1.ConfigMap, timeout time.Duration) error { 32 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 33 | 34 | defer cancel() 35 | _, err := ci.ConfigMaps(configMap.Namespace).Create(ctx, configMap, metav1.CreateOptions{}) 36 | 37 | if err != nil { 38 | return err 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func DeleteConfigMap(ci coreclient.CoreV1Interface, configMap *corev1.ConfigMap, timeout time.Duration) error { 45 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 46 | defer cancel() 47 | err := ci.ConfigMaps(configMap.Namespace).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) 48 | return err 49 | } 50 | -------------------------------------------------------------------------------- /test/util/features.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | v1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | coreclient "k8s.io/client-go/kubernetes/typed/core/v1" 10 | ) 11 | 12 | const ( 13 | hugepagesResourceName2Mi = "hugepages-2Mi" 14 | hugepagesResouceName1Gi = "hugepages-1Gi" 15 | ) 16 | 17 | //IsMinHugepagesAvailable checks if a min Gi/Mi hugepage number is available on any nodes 18 | func IsMinHugepagesAvailable(ci coreclient.CoreV1Interface, minGi, minMi int) (bool, error) { 19 | hugepages := map[string]int{hugepagesResouceName1Gi: minGi, hugepagesResourceName2Mi: minMi} 20 | list, err := ci.Nodes().List(context.Background(), metav1.ListOptions{}) 21 | if err != nil { 22 | return false, err 23 | } 24 | 25 | if minGi == 0 && minMi == 0 { 26 | return false, fmt.Errorf("IsHugepagesAvailable(): define at least greater than zero hugepages") 27 | } 28 | 29 | if len(list.Items) == 0 { 30 | return false, fmt.Errorf("IsHugepagesAvailable(): no nodes available") 31 | } 32 | 33 | foundOnce := map[string]bool{hugepagesResouceName1Gi: false, hugepagesResourceName2Mi: false} 34 | 35 | for _, node := range list.Items { 36 | for resource, minHugepages := range hugepages { 37 | if minHugepages == 0 { 38 | foundOnce[resource] = true 39 | continue 40 | } 41 | capacity, ok := node.Status.Capacity[v1.ResourceName(resource)] 42 | if !ok { 43 | return false, fmt.Errorf("IsHugepagesAvailable(): cannot find hugepage resource %s via K8 API", resource) 44 | } 45 | size, ok := capacity.AsInt64() 46 | if !ok { 47 | return false, fmt.Errorf("IsHugepagesAvailable(): failed to convert hugepage capacity") 48 | } 49 | if int64(minHugepages) <= size { 50 | foundOnce[resource] = true 51 | } 52 | } 53 | } 54 | return foundOnce[hugepagesResourceName2Mi] && foundOnce[hugepagesResouceName1Gi], nil 55 | } 56 | -------------------------------------------------------------------------------- /test/util/images.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | var ( 9 | registry string 10 | testImage string 11 | ) 12 | 13 | func init() { 14 | registry = os.Getenv("IMAGE_REGISTRY") 15 | if registry == "" { 16 | registry = "docker.io" 17 | } 18 | testImage = os.Getenv("TEST_IMAGE") 19 | if testImage == "" { 20 | testImage = "alpine" 21 | } 22 | } 23 | 24 | //GetPodTestImage returns image to be used during testing 25 | func GetPodTestImage() string { 26 | return fmt.Sprintf("%s/%s", registry, testImage) 27 | } 28 | -------------------------------------------------------------------------------- /test/util/namespace.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/util/wait" 10 | coreclient "k8s.io/client-go/kubernetes/typed/core/v1" 11 | ) 12 | 13 | func CreateNamespace(ci coreclient.CoreV1Interface, name string, timeout time.Duration) error { 14 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 15 | defer cancel() 16 | 17 | nsSpec := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} 18 | _, err := ci.Namespaces().Create(ctx, nsSpec, metav1.CreateOptions{}) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | return nil 24 | } 25 | 26 | func DeleteNamespace(ci coreclient.CoreV1Interface, name string, timeout time.Duration) error { 27 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 28 | 29 | defer cancel() 30 | 31 | err := ci.Namespaces().Delete(ctx, name, metav1.DeleteOptions{}) 32 | if err == nil { 33 | _ = WaitForNamespaceDelete(ci, name, timeout, 10) 34 | } 35 | 36 | return err 37 | } 38 | 39 | func WaitForNamespaceDelete(core coreclient.CoreV1Interface, namespaceName string, timeout, interval time.Duration) error { 40 | return wait.PollImmediate(interval, timeout, func() (done bool, err error) { 41 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 42 | defer cancel() 43 | 44 | namespace, err := core.Namespaces().Get(ctx, namespaceName, metav1.GetOptions{}) 45 | 46 | if err != nil { 47 | return false, err 48 | } 49 | 50 | switch namespace.Status.Phase { 51 | case corev1.NamespaceActive, corev1.NamespaceTerminating: 52 | return false, nil 53 | default: 54 | return true, nil 55 | } 56 | 57 | return false, nil 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /test/util/networkattachmentdefinition.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | cniv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" 8 | networkCoreClient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" 9 | 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | ) 12 | 13 | func GetWithoutAnnotations(ns string, networkName string) *cniv1.NetworkAttachmentDefinition { 14 | nad := GetNetworkAttachmentDefinition(ns, networkName) 15 | 16 | return nad 17 | } 18 | 19 | func GetResourceSelectorOnly(ns string, networkName string, resourceName string) *cniv1.NetworkAttachmentDefinition { 20 | nad := GetNetworkAttachmentDefinition(ns, networkName) 21 | nad.Annotations = map[string]string{"k8s.v1.cni.cncf.io/resourceName": resourceName} 22 | 23 | return nad 24 | } 25 | 26 | func GetNodeSelectorOnly(ns string, networkName string, nodeName string) *cniv1.NetworkAttachmentDefinition { 27 | nad := GetNetworkAttachmentDefinition(ns, networkName) 28 | nad.Annotations = map[string]string{"k8s.v1.cni.cncf.io/nodeSelector": nodeName} 29 | 30 | return nad 31 | } 32 | 33 | func GetResourceAndNodeSelector(ns string, networkName string, nodeName string) *cniv1.NetworkAttachmentDefinition { 34 | nad := GetNetworkAttachmentDefinition(ns, networkName) 35 | nad.Annotations = map[string]string{ 36 | "k8s.v1.cni.cncf.io/nodeSelector": nodeName, 37 | "k8s.v1.cni.cncf.io/resourceName": "example.com/foo", 38 | } 39 | 40 | return nad 41 | } 42 | 43 | func GetNetworkAttachmentDefinition(networkName string, ns string) *cniv1.NetworkAttachmentDefinition { 44 | return &cniv1.NetworkAttachmentDefinition{ 45 | ObjectMeta: metav1.ObjectMeta{ 46 | Name: networkName, 47 | Namespace: ns, 48 | }, 49 | Spec: cniv1.NetworkAttachmentDefinitionSpec{ 50 | Config: GetNetworkSpecConfig(networkName), 51 | }, 52 | } 53 | } 54 | 55 | // { 56 | // "cniVersion": "0.3.0", 57 | // "name": networkName, 58 | // "type": "loopback", 59 | // }, 60 | func GetNetworkSpecConfig(networkName string) string { 61 | config := "{\"cniVersion\": \"0.3.0\", \"name\": \"" + networkName + "\", \"type\":\"loopback\"}" 62 | return config 63 | } 64 | 65 | func ApplyNetworkAttachmentDefinition(ci networkCoreClient.K8sCniCncfIoV1Interface, nad *cniv1.NetworkAttachmentDefinition, timeout time.Duration) error { 66 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 67 | 68 | defer cancel() 69 | _, err := ci.NetworkAttachmentDefinitions(nad.Namespace).Create(ctx, nad, metav1.CreateOptions{}) 70 | 71 | if err != nil { 72 | return err 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func DeleteNetworkAttachmentDefinition(ci networkCoreClient.K8sCniCncfIoV1Interface, testNetworkName string, nad *cniv1.NetworkAttachmentDefinition, timeout time.Duration) error { 79 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 80 | 81 | defer cancel() 82 | err := ci.NetworkAttachmentDefinitions(nad.Namespace).Delete(ctx, testNetworkName, metav1.DeleteOptions{}) 83 | 84 | if err != nil { 85 | return err 86 | } 87 | 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /test/util/pod.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "strings" 8 | "time" 9 | 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/api/resource" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/apimachinery/pkg/util/wait" 14 | "k8s.io/client-go/kubernetes/scheme" 15 | coreclient "k8s.io/client-go/kubernetes/typed/core/v1" 16 | restclient "k8s.io/client-go/rest" 17 | "k8s.io/client-go/tools/remotecommand" 18 | ) 19 | 20 | //CreateRunningPod create a pod and wait until it is running 21 | func CreateRunningPod(ci coreclient.CoreV1Interface, pod *corev1.Pod, timeout, interval time.Duration) error { 22 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 23 | defer cancel() 24 | pod, err := ci.Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) 25 | 26 | if err != nil { 27 | return err 28 | } 29 | 30 | err = WaitForPodStateRunning(ci, pod.ObjectMeta.Name, pod.ObjectMeta.Namespace, timeout, interval) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | return nil 36 | } 37 | 38 | //DeletePod will delete a pod 39 | func DeletePod(ci coreclient.CoreV1Interface, pod *corev1.Pod, timeout time.Duration) error { 40 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 41 | defer cancel() 42 | err := ci.Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) 43 | return err 44 | } 45 | 46 | //UpdatePodInfo will get the current pod state and return it 47 | func UpdatePodInfo(ci coreclient.CoreV1Interface, pod *corev1.Pod, timeout time.Duration) (*corev1.Pod, error) { 48 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 49 | defer cancel() 50 | pod, err := ci.Pods(pod.Namespace).Get(ctx, pod.ObjectMeta.Name, metav1.GetOptions{}) 51 | if err != nil { 52 | return nil, err 53 | } 54 | return pod, nil 55 | } 56 | 57 | //GetPodDefinition will return a test pod 58 | func GetPodDefinition(ns string, podName string) *corev1.Pod { 59 | var graceTime int64 = 0 60 | return &corev1.Pod{ 61 | ObjectMeta: metav1.ObjectMeta{ 62 | Name: podName, 63 | Namespace: ns, 64 | }, 65 | Spec: corev1.PodSpec{ 66 | TerminationGracePeriodSeconds: &graceTime, 67 | Containers: []corev1.Container{ 68 | { 69 | Name: "test", 70 | Image: GetPodTestImage(), 71 | Command: []string{"/bin/sh", "-c", "sleep INF"}, 72 | }, 73 | }, 74 | }, 75 | } 76 | } 77 | 78 | //AddMetadataLabel adds label to the POD metadata section 79 | // :param labelName - key in map, label name 80 | // :param labelContent - value, label content 81 | func AddMetadataLabel(pod *corev1.Pod, labelName, labelContent string) *corev1.Pod { 82 | if nil == pod.ObjectMeta.Labels { 83 | pod.ObjectMeta.Labels = make(map[string]string) 84 | } 85 | 86 | pod.ObjectMeta.Labels[labelName] = labelContent 87 | 88 | return pod 89 | } 90 | 91 | //AddToPodDefinitionVolumesWithDownwardAPI adds to the POD specification at the 'path' downwardAPI volumes that expose POD namespace 92 | // :param pod - POD object to be modified 93 | // :param mountPath - path of the folder in which file is going to be available 94 | // :param volumeName - name of the volume 95 | // :param containerNumber - number of the container to which volumes have to be added 96 | // :return updated POD object 97 | func AddToPodDefinitionVolumesWithDownwardAPI(pod *corev1.Pod, mountPath, volumeName string, containerNumber int64) *corev1.Pod { 98 | pod.Spec.Volumes = []corev1.Volume{ 99 | { 100 | Name: volumeName, 101 | VolumeSource: corev1.VolumeSource{ 102 | DownwardAPI: &corev1.DownwardAPIVolumeSource{ 103 | Items: []corev1.DownwardAPIVolumeFile{ 104 | { 105 | Path: "namespace", 106 | FieldRef: &corev1.ObjectFieldSelector{ 107 | FieldPath: "metadata.namespace", 108 | }, 109 | }, 110 | }, 111 | }, 112 | }, 113 | }, 114 | } 115 | 116 | pod.Spec.Containers[containerNumber].VolumeMounts = []corev1.VolumeMount{ 117 | { 118 | Name: volumeName, 119 | ReadOnly: false, 120 | MountPath: mountPath, 121 | }, 122 | } 123 | 124 | return pod 125 | } 126 | 127 | // AddToPodDefinitionHugePages1Gi adds Hugepages 1Gi limits and requirements to the POD spec 128 | func AddToPodDefinitionHugePages1Gi(pod *corev1.Pod, amountLimit, amountRequest, containerNumber int64) *corev1.Pod { 129 | if nil == pod.Spec.Containers[containerNumber].Resources.Limits { 130 | pod.Spec.Containers[containerNumber].Resources.Limits = make(map[corev1.ResourceName]resource.Quantity) 131 | } 132 | 133 | if nil == pod.Spec.Containers[containerNumber].Resources.Requests { 134 | pod.Spec.Containers[containerNumber].Resources.Requests = make(map[corev1.ResourceName]resource.Quantity) 135 | } 136 | 137 | pod.Spec.Containers[containerNumber].Resources.Limits["hugepages-1Gi"] = *resource.NewQuantity(amountLimit*1024*1024*1024, resource.BinarySI) 138 | pod.Spec.Containers[containerNumber].Resources.Requests["hugepages-1Gi"] = *resource.NewQuantity(amountRequest*1024*1024*1024, resource.BinarySI) 139 | 140 | return pod 141 | } 142 | 143 | // AddToPodDefinitionHugePages2Mi adds Hugepages 2Mi limits and requirements to the POD spec 144 | func AddToPodDefinitionHugePages2Mi(pod *corev1.Pod, amountLimit, amountRequest, containerNumber int64) *corev1.Pod { 145 | if nil == pod.Spec.Containers[containerNumber].Resources.Limits { 146 | pod.Spec.Containers[containerNumber].Resources.Limits = make(map[corev1.ResourceName]resource.Quantity) 147 | } 148 | 149 | if nil == pod.Spec.Containers[containerNumber].Resources.Requests { 150 | pod.Spec.Containers[containerNumber].Resources.Requests = make(map[corev1.ResourceName]resource.Quantity) 151 | } 152 | 153 | pod.Spec.Containers[containerNumber].Resources.Limits["hugepages-2Mi"] = *resource.NewQuantity(amountLimit*1024*1024, resource.BinarySI) 154 | pod.Spec.Containers[containerNumber].Resources.Requests["hugepages-2Mi"] = *resource.NewQuantity(amountRequest*1024*1024, resource.BinarySI) 155 | 156 | return pod 157 | } 158 | 159 | // AddToPodDefinitionMemory adds Memory constraints to the POD spec 160 | func AddToPodDefinitionMemory(pod *corev1.Pod, amountLimit, amountRequest, containerNumber int64) *corev1.Pod { 161 | if nil == pod.Spec.Containers[containerNumber].Resources.Limits { 162 | pod.Spec.Containers[containerNumber].Resources.Limits = make(map[corev1.ResourceName]resource.Quantity) 163 | } 164 | 165 | if nil == pod.Spec.Containers[containerNumber].Resources.Requests { 166 | pod.Spec.Containers[containerNumber].Resources.Requests = make(map[corev1.ResourceName]resource.Quantity) 167 | } 168 | 169 | pod.Spec.Containers[containerNumber].Resources.Limits["memory"] = *resource.NewQuantity(amountLimit*1024*1024*1024, resource.BinarySI) 170 | pod.Spec.Containers[containerNumber].Resources.Requests["memory"] = *resource.NewQuantity(amountRequest*1024*1024*1024, resource.BinarySI) 171 | 172 | return pod 173 | } 174 | 175 | // AddToPodDefinitionCpuLimits adds CPU limits and requests to the definition of POD 176 | func AddToPodDefinitionCpuLimits(pod *corev1.Pod, cpuNumber, containerNumber int64) *corev1.Pod { 177 | if nil == pod.Spec.Containers[containerNumber].Resources.Limits { 178 | pod.Spec.Containers[containerNumber].Resources.Limits = make(map[corev1.ResourceName]resource.Quantity) 179 | } 180 | 181 | if nil == pod.Spec.Containers[containerNumber].Resources.Requests { 182 | pod.Spec.Containers[containerNumber].Resources.Requests = make(map[corev1.ResourceName]resource.Quantity) 183 | } 184 | 185 | pod.Spec.Containers[containerNumber].Resources.Limits["cpu"] = *resource.NewQuantity(cpuNumber, resource.DecimalSI) 186 | pod.Spec.Containers[containerNumber].Resources.Requests["cpu"] = *resource.NewQuantity(cpuNumber, resource.DecimalSI) 187 | 188 | return pod 189 | } 190 | 191 | //GetOneNetwork add one network to pod 192 | func GetOneNetwork(nad, ns string, podName string) *corev1.Pod { 193 | pod := GetPodDefinition(ns, podName) 194 | pod.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": nad} 195 | return pod 196 | } 197 | 198 | //GetOneNetworkTwoContainers returns POD with two containers and one network 199 | func GetOneNetworkTwoContainers(nad, ns, podName, secondContainerName string) *corev1.Pod { 200 | pod := GetPodDefinition(ns, podName) 201 | pod.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": nad} 202 | 203 | pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ 204 | Name: secondContainerName, 205 | Image: GetPodTestImage(), 206 | Command: []string{"/bin/sh", "-c", "sleep INF"}, 207 | }) 208 | 209 | return pod 210 | } 211 | 212 | //GetMultiNetworks adds a network to annotation 213 | func GetMultiNetworks(nad []string, ns string, podName string) *corev1.Pod { 214 | pod := GetPodDefinition(ns, podName) 215 | pod.Annotations = map[string]string{"k8s.v1.cni.cncf.io/networks": strings.Join(nad, ",")} 216 | return pod 217 | } 218 | 219 | //WaitForPodStateRunning waits for pod to enter running state 220 | func WaitForPodStateRunning(core coreclient.CoreV1Interface, podName, ns string, timeout, interval time.Duration) error { 221 | return wait.PollImmediate(interval, timeout, func() (done bool, err error) { 222 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 223 | defer cancel() 224 | pod, err := core.Pods(ns).Get(ctx, podName, metav1.GetOptions{}) 225 | 226 | if err != nil { 227 | return false, err 228 | } 229 | switch pod.Status.Phase { 230 | case corev1.PodRunning: 231 | return true, nil 232 | case corev1.PodFailed, corev1.PodSucceeded: 233 | return false, errors.New("pod failed or succeeded but is not running") 234 | } 235 | return false, nil 236 | }) 237 | } 238 | 239 | // ExecuteCommand execute command on the POD 240 | // :param core - core V1 Interface 241 | // :param config - configuration used to establish REST connection with K8s node 242 | // :param podName - POD name on which command has to be executed 243 | // :param ns - namespace in which POD exists 244 | // :param containerName - container name on which command should be executed 245 | // :param command - command to be executed on POD 246 | // :return string output of the command (stdout) 247 | // string output of the command (stderr) 248 | // error Error object or when everthing is correct nil 249 | func ExecuteCommand(core coreclient.CoreV1Interface, config *restclient.Config, podName, ns, containerName, command string) (string, string, error) { 250 | shellCommand := []string{"/bin/sh", "-c", command} 251 | request := core.RESTClient().Post().Resource("pods").Name(podName).Namespace(ns).SubResource("exec") 252 | options := &corev1.PodExecOptions{ 253 | Command: shellCommand, 254 | Container: containerName, 255 | Stdin: false, 256 | Stdout: true, 257 | Stderr: true, 258 | TTY: false, 259 | } 260 | 261 | request.VersionedParams(options, scheme.ParameterCodec) 262 | exec, err := remotecommand.NewSPDYExecutor(config, "POST", request.URL()) 263 | if nil != err { 264 | return "", "", err 265 | } 266 | 267 | var stdout, stderr bytes.Buffer 268 | err = exec.Stream(remotecommand.StreamOptions{ 269 | Stdin: nil, 270 | Stdout: &stdout, 271 | Stderr: &stderr, 272 | Tty: false, 273 | }) 274 | 275 | if nil != err { 276 | return "", "", err 277 | } 278 | 279 | return stdout.String(), stderr.String(), nil 280 | } 281 | --------------------------------------------------------------------------------