├── .dockerignore ├── .github └── workflows │ ├── codeql.yml │ ├── module_controller_dev_with_debug.yml │ ├── module_controller_release.yml │ └── module_controller_unit_test.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README-zh_CN.md ├── README.md ├── cmd ├── module-controller │ └── main.go └── self-test │ ├── main.go │ └── predicate.go ├── common ├── model │ ├── consts.go │ └── model.go ├── utils │ ├── utils.go │ └── utils_test.go └── zaplogger │ └── logger.go ├── config └── default_tracker_config.yaml ├── controller └── module_deployment_controller │ ├── module_deployment_controller.go │ ├── module_deployment_controller_test.go │ ├── predicates.go │ └── predicates_test.go ├── debug.Dockerfile ├── example ├── quick-start │ ├── base.yaml │ ├── module-controller-test.yaml │ ├── module-controller.yaml │ └── module.yaml └── self-test │ ├── base.yaml │ └── module.yaml ├── go.mod ├── go.sum ├── module_tunnels ├── koupleless_http_tunnel │ ├── ark_service │ │ ├── ark_service.go │ │ └── model.go │ └── http_tunnel.go └── koupleless_mqtt_tunnel │ ├── config.go │ ├── mqtt │ └── mqtt.go │ └── mqtt_tunnel.go ├── report_server └── server.go ├── samples ├── e2e_test_rbac │ ├── create_e2e-test_kubeconfig.sh │ ├── secret.yaml │ ├── service_account.yaml │ ├── service_account_cluster_role.yaml │ └── service_account_cluster_role_binding.yaml ├── module │ ├── module_daemonset.yaml │ ├── module_deployment.yaml │ ├── multi_module_biz1_biz2.yaml │ ├── single_module_biz1.yaml │ ├── single_module_biz1_version2.yaml │ └── single_module_biz2.yaml ├── module_controller_pod.yaml └── rbac │ ├── base_service_account.yaml │ ├── base_service_account_cluster_role.yaml │ └── base_service_account_cluster_role_binding.yaml └── suite ├── http ├── base_http_lifecycle_test.go ├── mock_http_base.go ├── module_http_lifecycle_test.go └── suite_test.go └── mqtt ├── base_mqtt_lifecycle_test.go ├── mock_mqtt_base.go ├── module_deployment_controller_suite_test.go ├── module_mqtt_lifecycle_test.go └── suite_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore build and test binaries. 3 | bin/ 4 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main"] 17 | pull_request: 18 | branches: [ "main"] 19 | schedule: 20 | - cron: '22 21 * * 2' 21 | 22 | jobs: 23 | analyze: 24 | name: Analyze 25 | # Runner size impacts CodeQL analysis time. To learn more, please see: 26 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 27 | # - https://gh.io/supported-runners-and-hardware-resources 28 | # - https://gh.io/using-larger-runners 29 | # Consider using larger runners for possible analysis time improvements. 30 | runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} 31 | timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} 32 | permissions: 33 | # required for all workflows 34 | security-events: write 35 | 36 | # only required for workflows in private repositories 37 | actions: read 38 | contents: read 39 | 40 | strategy: 41 | fail-fast: false 42 | matrix: 43 | language: [ 'go' ] 44 | # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] 45 | # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both 46 | # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both 47 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 48 | 49 | steps: 50 | - name: Checkout repository 51 | uses: actions/checkout@v4 52 | 53 | # Initializes the CodeQL tools for scanning. 54 | - name: Initialize CodeQL 55 | uses: github/codeql-action/init@v3 56 | with: 57 | languages: ${{ matrix.language }} 58 | # If you wish to specify custom queries, you can do so here or in a config file. 59 | # By default, queries listed here will override any specified in a config file. 60 | # Prefix the list here with "+" to use these queries and those in the config file. 61 | 62 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 63 | # queries: security-extended,security-and-quality 64 | 65 | 66 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). 67 | # If this step fails, then you should remove it and run the build manually (see below) 68 | - name: Autobuild 69 | uses: github/codeql-action/autobuild@v3 70 | 71 | # ℹ️ Command-line programs to run using the OS shell. 72 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 73 | 74 | # If the Autobuild fails above, remove it and uncomment the following three lines. 75 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 76 | 77 | # - run: | 78 | # echo "Run, Build Application using script" 79 | # ./location_of_script_within_repo/buildscript.sh 80 | 81 | - name: Perform CodeQL Analysis 82 | uses: github/codeql-action/analyze@v3 83 | with: 84 | category: "/language:${{matrix.language}}" 85 | -------------------------------------------------------------------------------- /.github/workflows/module_controller_dev_with_debug.yml: -------------------------------------------------------------------------------- 1 | name: Module Controller Dev With Remote Debug Enabled 2 | run-name: ${{ github.actor }} pushed module-controller code 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v*.*.*' 8 | workflow_dispatch: 9 | 10 | env: 11 | CGO_ENABLED: 0 12 | GO_VERSION: 1.22.4 13 | GOOS: linux 14 | WORK_DIR: . 15 | DOCKERHUB_REGISTRY: serverless-registry.cn-shanghai.cr.aliyuncs.com 16 | MODULE_CONTROLLER_IMAGE_PATH: opensource/test/module-controller-v2 17 | DOCKER_GITHUB_REGISTRY: ghcr.io 18 | DOCKER_GITHUB_IMAGE_NAME: ${{ github.repository }} 19 | 20 | jobs: 21 | build-push-linux-amd64-image: 22 | name: "Build and push module-controller Docker images" 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v3 27 | - name: Set up QEMU 28 | uses: docker/setup-qemu-action@v1 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v1 31 | - name: Login to DockerHub 32 | uses: docker/login-action@v1 33 | with: 34 | registry: ${{ env.DOCKERHUB_REGISTRY }} 35 | username: ${{ secrets.DOCKERHUB_USERNAME }} 36 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 37 | - name: Login to GitHub Container Registry 38 | uses: docker/login-action@v3 39 | with: 40 | registry: ${{ env.DOCKER_GITHUB_REGISTRY }} 41 | username: ${{ github.repository_owner }} 42 | password: ${{ secrets.GITHUB_TOKEN }} 43 | - name: Get the version 44 | id: get_version 45 | run: | 46 | if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then 47 | echo ::set-output name=VERSION::latest 48 | else 49 | echo ::set-output name=VERSION::${GITHUB_REF#refs/*/} 50 | fi 51 | - name: Build and push module-controller Docker images 52 | uses: docker/build-push-action@v4.1.1 53 | with: 54 | context: ${{ env.WORK_DIR }} 55 | cache-from: type=local,src=/tmp/.buildx-cache 56 | cache-to: type=local,dest=/tmp/.buildx-cache 57 | file: ${{ env.WORK_DIR }}/debug.Dockerfile 58 | platforms: linux/amd64,linux/arm64 59 | push: true 60 | tags: | 61 | ${{ env.DOCKERHUB_REGISTRY }}/${{ env.MODULE_CONTROLLER_IMAGE_PATH }}:${{ steps.get_version.outputs.VERSION }} 62 | ${{ env.DOCKER_GITHUB_REGISTRY }}/${{ env.DOCKER_GITHUB_IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }} -------------------------------------------------------------------------------- /.github/workflows/module_controller_release.yml: -------------------------------------------------------------------------------- 1 | name: Module Controller Release 2 | run-name: ${{ github.actor }} pushed module-controller code 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v*.*.*' 8 | workflow_dispatch: 9 | 10 | env: 11 | CGO_ENABLED: 0 12 | GO_VERSION: 1.22.4 13 | GOOS: linux 14 | WORK_DIR: . 15 | DOCKERHUB_REGISTRY: serverless-registry.cn-shanghai.cr.aliyuncs.com 16 | MODULE_CONTROLLER_IMAGE_PATH: opensource/release/module-controller-v2 17 | DOCKER_GITHUB_REGISTRY: ghcr.io 18 | DOCKER_GITHUB_IMAGE_NAME: ${{ github.repository }} 19 | 20 | jobs: 21 | build-push-linux-amd64-image: 22 | name: "Build and push module-controller Docker images" 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v3 27 | - name: Set up QEMU 28 | uses: docker/setup-qemu-action@v1 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v1 31 | - name: Login to DockerHub 32 | uses: docker/login-action@v1 33 | with: 34 | registry: ${{ env.DOCKERHUB_REGISTRY }} 35 | username: ${{ secrets.DOCKERHUB_USERNAME }} 36 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 37 | - name: Login to GitHub Container Registry 38 | uses: docker/login-action@v3 39 | with: 40 | registry: ${{ env.DOCKER_GITHUB_REGISTRY }} 41 | username: ${{ github.repository_owner }} 42 | password: ${{ secrets.GITHUB_TOKEN }} 43 | - name: Get the version 44 | id: get_version 45 | run: | 46 | if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then 47 | echo ::set-output name=VERSION::latest 48 | else 49 | echo ::set-output name=VERSION::${GITHUB_REF#refs/*/} 50 | fi 51 | - name: Build and push module-controller Docker images 52 | uses: docker/build-push-action@v4.1.1 53 | with: 54 | context: ${{ env.WORK_DIR }} 55 | cache-from: type=local,src=/tmp/.buildx-cache 56 | cache-to: type=local,dest=/tmp/.buildx-cache 57 | file: ${{ env.WORK_DIR }}/Dockerfile 58 | platforms: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le 59 | push: true 60 | tags: | 61 | ${{ env.DOCKERHUB_REGISTRY }}/${{ env.MODULE_CONTROLLER_IMAGE_PATH }}:${{ steps.get_version.outputs.VERSION }} 62 | ${{ env.DOCKER_GITHUB_REGISTRY }}/${{ env.DOCKER_GITHUB_IMAGE_NAME }}:${{ steps.get_version.outputs.VERSION }} -------------------------------------------------------------------------------- /.github/workflows/module_controller_unit_test.yml: -------------------------------------------------------------------------------- 1 | name: Module Controller Unit Test 2 | run-name: ${{ github.actor }} pushed module-controller code 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | 9 | pull_request: 10 | branches: 11 | - main 12 | 13 | # enable manually running the workflow 14 | workflow_dispatch: 15 | 16 | env: 17 | CGO_ENABLED: 0 18 | GOOS: linux 19 | WORK_DIR: . 20 | 21 | jobs: 22 | unit-test: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v3 26 | 27 | - name: Set up go 28 | uses: actions/setup-go@v4 29 | with: 30 | go-version: '1.22.4' 31 | cache-dependency-path: ${{ env.WORK_DIR }}/go.sum 32 | 33 | - name: Run go mod 34 | run: go mod download 35 | 36 | - name: Test 37 | run: make test 38 | 39 | - name: Upload coverage reports to Codecov 40 | uses: codecov/codecov-action@v4.0.1 41 | with: 42 | token: ${{ secrets.CODECOV_TOKEN }} 43 | slug: koupleless/module-controller 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | target/ 3 | *.iml 4 | .idea/ 5 | pom.xml.bak 6 | .DS_Store 7 | .settings 8 | .classpath 9 | *.log 10 | logs/ 11 | logs*/ 12 | .flattened-pom.xml 13 | coverage.out 14 | */coverage.out 15 | 16 | # Binaries for programs and plugins 17 | *.exe 18 | *.exe~ 19 | *.dll 20 | *.so 21 | *.dylib 22 | bin/* 23 | Dockerfile.cross 24 | 25 | # Test binary, build with `go test -c` 26 | *.test 27 | 28 | # Output of the go coverage tool, specifically when used with LiteIDE 29 | *.out 30 | 31 | # Kubernetes Generated files - skip generated files, except for vendored files 32 | 33 | !vendor/**/zz_generated.* 34 | 35 | # editor and IDE paraphernalia 36 | .idea 37 | .vscode 38 | *.swp 39 | *.swo 40 | *~ 41 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.22 as builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | 6 | WORKDIR /workspace 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY cmd/ cmd/ 16 | COPY common/ common/ 17 | COPY controller/ controller/ 18 | COPY module_tunnels/ module_tunnels/ 19 | COPY report_server/ report_server/ 20 | 21 | # Build 22 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 23 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 24 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 25 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 26 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o module_controller cmd/module-controller/main.go 27 | 28 | # Use distroless as minimal base image to package the manager binary 29 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 30 | FROM golang:1.22.8 31 | WORKDIR / 32 | COPY config/ config/ 33 | COPY --from=builder /workspace/module_controller . 34 | 35 | EXPOSE 9090 36 | EXPOSE 8080 37 | EXPOSE 7777 38 | 39 | ENTRYPOINT ["./module_controller"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. 2 | ENVTEST_K8S_VERSION = 1.27.1 3 | 4 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 5 | ifeq (,$(shell go env GOBIN)) 6 | GOBIN=$(shell go env GOPATH)/bin 7 | else 8 | GOBIN=$(shell go env GOBIN) 9 | endif 10 | 11 | .PHONY: help 12 | help: ## Display this help. 13 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 14 | 15 | .PHONY: fmt 16 | fmt: ## Run go fmt against code. 17 | go fmt ./... 18 | 19 | .PHONY: vet 20 | vet: ## Run go vet against code. 21 | go vet ./... 22 | 23 | .PHONY: test 24 | test: fmt vet envtest ## Run tests. 25 | KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test `go list ./... | grep -v cmd` -coverpkg=./... -coverprofile=coverage.out 26 | 27 | .PHONY: buildx 28 | buildx: fmt vet 29 | docker buildx build --platform linux/amd64 -t serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/test/module_controller:latest . 30 | 31 | .PHONY: build 32 | build: fmt vet 33 | docker build -t serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/test/module_controller:latest . 34 | 35 | ##@ Deployment 36 | 37 | ifndef ignore-not-found 38 | ignore-not-found = false 39 | endif 40 | 41 | ##@ Build Dependencies 42 | 43 | ## Location to install dependencies to 44 | LOCALBIN ?= $(shell pwd)/bin 45 | $(LOCALBIN): 46 | mkdir -p $(LOCALBIN) 47 | 48 | ## Tool Binaries 49 | KUBECTL ?= kubectl 50 | KUSTOMIZE ?= $(LOCALBIN)/kustomize 51 | CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen 52 | ENVTEST ?= $(LOCALBIN)/setup-envtest 53 | 54 | ## Tool Versions 55 | KUSTOMIZE_VERSION ?= v5.0.1 56 | CONTROLLER_TOOLS_VERSION ?= v0.12.0 57 | 58 | .PHONY: kustomize 59 | kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. 60 | $(KUSTOMIZE): $(LOCALBIN) 61 | @if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \ 62 | echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \ 63 | rm -rf $(LOCALBIN)/kustomize; \ 64 | fi 65 | test -s $(LOCALBIN)/kustomize || GOBIN=$(LOCALBIN) GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v5@$(KUSTOMIZE_VERSION) 66 | 67 | .PHONY: controller-gen 68 | controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. 69 | $(CONTROLLER_GEN): $(LOCALBIN) 70 | test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ 71 | GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) 72 | 73 | .PHONY: envtest 74 | envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. 75 | $(ENVTEST): $(LOCALBIN) 76 | test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest 77 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | # Code generated by tool. DO NOT EDIT. 2 | # This file is used to track the info used to scaffold your project 3 | # and allow the plugins properly work. 4 | # More info: https://book.kubebuilder.io/reference/project-config.html 5 | domain: koupleless.io 6 | layout: 7 | - go.kubebuilder.io/v4 8 | projectName: module-controller 9 | repo: github.com/koupleless/module-controller 10 | resources: 11 | - api: 12 | crdVersion: v1 13 | namespaced: true 14 | controller: true 15 | domain: alipay.com 16 | group: serverless 17 | kind: ModuleDeployment 18 | path: github.com/koupleless/module-controller/api/v1alpha1 19 | version: v1alpha1 20 | - api: 21 | crdVersion: v1 22 | namespaced: true 23 | controller: true 24 | domain: alipay.com 25 | group: serverless 26 | kind: Module 27 | path: github.com/koupleless/module-controller/api/v1alpha1 28 | version: v1alpha1 29 | - api: 30 | crdVersion: v1 31 | namespaced: true 32 | controller: true 33 | domain: alipay.com 34 | group: serverless 35 | kind: ModuleReplicaSet 36 | path: github.com/koupleless/module-controller/api/v1alpha1 37 | version: v1alpha1 38 | - controller: true 39 | domain: koupleless.io 40 | kind: Pod 41 | version: v1 42 | version: "3" 43 | -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 | 低成本地让开源用户接入模块运维体系。 2 | 3 | # 架构设计 4 | 5 | ## 整体思路 6 | 整体思想是用 mock 的方式利用 k8s 现有的的运维调度能力，完成低成本的模块运维体系接入。 7 | 8 | ### 如何触发调度？ 9 | 把基座 pod mock 成一个 k8s 的 node 节点，把模块 mock 成 k8s 的一个 pod，由此 kube-scheduler 会触发一轮调度，并且给 pod 分配上一个合适的 node。
值的注意的是，mock 的 pod 只能被调度到 mock 的 node 上，否则会造成无法执行安装的异常。这是因为，正常的 node 节点上的 kubelet 只会执行 pod 的安装流程。只有 mock 的特殊 node 上才存在 virtual-kubelet，只有 virtual-kubelet 会识别到模块，并且发起模块安装。这个约束可以通过 k8s 原生的 taints 和 toleration 配合保证。 10 | 11 | ### 如何触发运维？ 12 | 用户可以使用正常的 deployment 或者是其他开源社区自定义的运维 operator，只要保证其 pod template 的定义符合我们的 mock pod 的定义规范即可。 13 | 14 | ### 如何执行模块安装？ 15 | 可以使用社区的 virtual-kubelet 框架，virtual-kubelet 定义了一套 kubelet 的交互生命周期，预留了一些具体的接口点，如 createPod 等。开发者通过实现这些预留的接口，便可接入 k8s 的正常的运维生命周期。 16 | 17 | ## 架构图 18 | ![](https://intranetproxy.alipay.com/skylark/lark/0/2024/jpeg/43656686/1717399526452-f12daf0a-a991-43b8-b715-157925893947.jpeg)
通过架构图我们不难发现，仅仅通过 virtual-kubelet 这一个简单的组件，我们就可以直接利用 k8s 实现三层调度。值得注意的事情是，POD-B 的调度会对应一个 Kube ApiServer，我们不妨称其为 ApiServerA。而触发 mock pod 调度到 mock node 上会对应另外一个 ApiServer，我们不妨称其为 ApiServerB。ApiServerA 不一定必须等于 ApiServerB，可以是彼此独立的。在一些云托管的 k8s 场景下，由于云厂商对 ApiServer 的权限限制，用户可能必须独立部署一套 ApiServer。 19 | 20 | 21 | # 详细设计 22 | 本章节要求用户对 k8s 的运维调度体系有一定的了解，k8s 基础的运维调度流程不做复述，只对重点实现细节进行讨论。 23 | 24 | ## VPod 定义规范 25 | 在 Koupleless 体系中，模块（模块组）是一个重要的抽象，其包含如下核心属性： 26 | 27 | - 模块名。 28 | - 模块版本。 29 | - 模块包地址。 30 | - 模块运行状态等。 31 | 32 | 由于在 ModuleController V2 设计方案中，我们常使用 VPod（底层为 K8S 的 Pod）承载模块模型或模块组，因此，我们需要预先定义清楚相关模块配置到 Pod 属性的映射关系，本小结将探讨有关映射关系。
我们从元数据的映射开始讨论，其中，模块的元数据映射到 V1Pod 的 containers 字段下，由于 1 个 Pod 可以有 N 个 Containers，所以 V1Pod 模型天然地支持模块组的描述： 33 | 34 | ```yaml 35 | containers: 36 | - name: ${MODULE_NAME} 37 | image: ${MODULE_URL} 38 | env: 39 | - name: MODULE_VERSION 40 | value: ${MODULE_VERSION} 41 | resource: 42 | requests: 43 | cpu: 800m 44 | mem: 1GI 45 | ``` 46 | 47 | 相应的，模块的安装情况也可以塞在对应的 containerStatus 中，映射关系如下： 48 | 49 | ```yaml 50 | - containerID: arklet://{ip}:{module}:{version} 51 | image: {module url} 52 | name: moduleName 53 | ready: true 54 | started: true 55 | state: 56 | running: 57 | startedAt: "2024-04-25T03:53:09Z" 58 | ``` 59 | 60 | 除此之外，为了方便的能通过 kubectl 通过简单的表达是把有关的 pod 筛选出来，我们还应该在 labels 里加上:
module.koupleless.io/${moduleName}:${version} 标签
模块或模块组的运行期整体状态的映射关系如下： 61 | 62 | - 所有模块调度但未安装：pod.status.phase = 'Pending' 63 | - 所有模块调度成功但是有几个安装失败：pod.status.phase = 'Failed'，并设置一个 condition，type 为 module.koupleless.io/installed，value 为 false。 64 | - 所有模块调度成功并且所有都安装成功: pod.status.phase = 'Running'，并设置一个 condition，type 为 module.koupleless.io/installed，value 为 true。 65 | 66 | 上述介绍了模块的属性的配置，除此之外，为了和 k8s 的调度和生命周期体系融合，我们还需要配置一些高阶的运行期配置。
我们从调度开始介绍，为了保证 VPod 只会被调度到 VNode 上，我们需要添加对应的 Affinity 配置，如下所示： 67 | 68 | ```yaml 69 | affinity: 70 | nodeAffinity: 71 | requiredDuringSchedulingIgnoredDuringExecution: 72 | nodeSelectorTerms: 73 | - matchExpressions: 74 | - key: basement.koupleless.io/stack 75 | operator: In 76 | values: 77 | - java # 多语言环境下可能有其他技术栈 78 | - key: basement.koupleless.io/version 79 | operator: In 80 | values: 81 | - ${compatiable_version} # 模块可能只能被调度到一些特殊版本的 node 上，如有这种限制，则必须有这个字段。 82 | ``` 83 | 84 | 除此之外，为了保证 VNode 只会被调度 VPod，所以 VNode 会有一些特殊的 Taints 标签，相应的，Pod 也必须添加上对应的 Tolerations，如下： 85 | 86 | ```yaml 87 | tolerations: 88 | - key: "schedule.koupleless.io/virtual-node" 89 | operator: "Equal" 90 | value: "true" 91 | effect: "NoExecute" 92 | ``` 93 | 94 | 通过上述的 Affinity 和 Tolerations，我们可以保证 VPod 只会被调度到 VNode 上。
当然，我们还必须考虑这套模式和 k8s 原生流量的兼容性问题，我们可以通过 k8s 的 readinessGate 机制达到目的，添加如下配置： 95 | 96 | ```yaml 97 | readinessGates: 98 | - conditionType: "module.koupleless.io/ready" # virtual-kubelet 会根据健康检查状况跟新这个值 99 | ``` 100 | 101 | 通过这些关键的规范，我们不仅能用 k8s 的 pod 模型描述模块或模块组，还能和 k8s 的调度和流量体系结合起来，一个完整的可能的样例 yaml 如下： 102 | 103 | ```yaml 104 | apiVersion: v1 105 | metadata: 106 | labels: 107 | module.koupleless.io/module0: 0.1.0 108 | module.koupleless.io/module1: 0.1.0 109 | name: custome-module-group-as-pod 110 | spec: 111 | affinity: 112 | nodeAffinity: 113 | requiredDuringSchedulingIgnoredDuringExecution: 114 | nodeSelectorTerms: 115 | - matchExpressions: 116 | - key: basement.koupleless.io/stack 117 | operator: In 118 | values: 119 | - java # 多语言环境下可能有其他技术栈 120 | - key: basement.koupleless.io/version 121 | operator: In 122 | values: 123 | - version0 124 | tolerations: 125 | - key: "schedule.koupleless.io/virtual-node" 126 | operator: "Equal" 127 | value: "true" 128 | effect: "NoExecute" 129 | readinessGates: 130 | - conditionType: "module.koupleless.io/ready" 131 | containers: 132 | - name: module0 133 | image: http://module_url_0 134 | env: 135 | - name: MODULE_VERSION 136 | value: 0.1.0 137 | resource: 138 | requests: 139 | cpu: 800m 140 | mem: 1GI 141 | - name: module1 142 | image: http://module_url_1 143 | env: 144 | - name: MODULE_VERSION 145 | value: 0.1.0 146 | resource: 147 | requests: 148 | cpu: 800m 149 | mem: 1GI 150 | status: 151 | phase: Running 152 | containerStatuses: 153 | - containerID: arklet://192.168.0.1:module0:0.1.0 154 | image: http://module_url_1 155 | name: module0 156 | ready: true 157 | started: true 158 | state: 159 | running: 160 | startedAt: "2024-04-25T03:53:09Z" 161 | - containerID: arklet://192.168.0.1:module0:0.1.0 162 | image: http://module_url_1 163 | name: module1 164 | ready: true 165 | started: true 166 | state: 167 | running: 168 | startedAt: "2024-04-25T03:53:09Z" 169 | conditions: 170 | - lastProbeTime: null 171 | lastTransitionTime: "2024-04-24T09:24:58Z" 172 | status: "True" 173 | type: basement.koupleless.io/installed 174 | - lastProbeTime: null 175 | lastTransitionTime: "2024-04-24T09:24:58Z" 176 | status: "True" 177 | type: basement.koupleless.io/ready 178 | ``` 179 | 180 | 181 | ## VNode 规范设计 182 | 第一小节我们完成了 VPod 的规范设计。接下来我们需要探讨 VNode 的细则设计。
首先，VNode 必须有特殊的 Taints，保证正常的 Pod 不可能被调度到对应的 VNode 上，对应配置如下： 183 | 184 | ```yaml 185 | taints: 186 | - effect: NoSchedule 187 | key: "schedule.koupleless.io/virtual-node" 188 | value: True 189 | ``` 190 | 191 | 除此之外，我们还必须保证 VPod 只会被调度到 VNode 上。为此，Node 必须提供对应 labels，保证 Pod 能够配置相应的亲和性调度，对应配置如下： 192 | 193 | ```yaml 194 | labels: 195 | basement.koupleless.io/stack: java 196 | basement.koupleless.io/version: ${some_version} 197 | ``` 198 | 199 | 除此之外，node 还需要上报一些资源属性，如 capacity： 200 | 201 | ```yaml 202 | capacity: 203 | pods: 1 # 一般来说，我们只希望一个模块被调度 1 个模块 204 | ``` 205 | 206 | 以及需要定期更新 allocatable 字段： 207 | 208 | ```yaml 209 | allocatable: 210 | pods: 1 211 | ``` 212 | 213 | 为了方便排障，即通过 vnode 直接找到对应的 pod，vnode 的命名规范如下：virtual-node-{stack}-{namspace}-{podname}
最后，vnode 的 ip 直接使用对应 pod 暴露的 ip。
一个可能的样例 VNode 如下: 214 | 215 | ```yaml 216 | apiVersion: v1 217 | kind: Node 218 | metadata: 219 | labels: 220 | basement.koupleless.io/stack: java 221 | basement.koupleless.io/version: version0 222 | creationTimestamp: "2023-07-25T13:00:00Z" 223 | name: virtual-node-java-example-pod-01 224 | spec: 225 | taints: 226 | - effect: NoExecute 227 | key: "schedule.koupleless.io/virtual-node" 228 | value: True 229 | status: 230 | allocatble: 231 | pod: 1 232 | capacity: 233 | pod: 1 234 | ``` 235 | 236 | 237 | ## 自愈体系设计 238 | 239 | VNode 需要自愈能力，原因如下。在 JVM 体系中，模块的反复安装会导致 metaspace 的使用率逐渐上涨。最终，metaspace 的使用率会达到某个阈值，过了这个阈值后模块再也无法被安装，会触发 OOM。由此，VNode 需要有一定的自愈能力，去应对这个情况。由于 Java 目前无法有效地通过 API 去完全清理干净 metaspace 中的类，因此我们将选择更简单的做法，对基座 pod 做替换，整体流程如下: 240 | 241 | - 基座打 "schedule.koupleless.io/metaspace-overload: True: NoExecute" 的驱逐标签。 242 | - 等待 VPod 被驱逐到别的节点。 243 | - 执行模块卸载逻辑。 244 | - 从基座 Pod 所对应的 ApiServer 中（有可能不是 VPod 对应的 ApiServer），删除掉基座 Pod。 245 | 246 | 如此，便可以保证 Node 的可用性。 247 | 248 | 249 | # 实现 250 | ## 重要组件 251 | ### DaemonEndpoints 252 | 用于 kubectl 的回掉，获取 metric、日志、pod 信息等。 253 | 254 | ### nodeutil.Provider 255 | 实现 virtual-kubelet 的核心逻辑，执行具体的 pod 运维的动作如：pod 安装、pod 卸载、pod 状态获取等。 256 | 257 | ### 同步 node 节点信息 258 | 定期向 apiserver 上报 node 的信息，如 CPU、MEM 的使用量、POD 的承载量等。 259 | 260 | ## 初始化流程 261 | virtual-kubelet 组建的初始化流程如下，按照先后顺序依次介绍。 262 | 263 | ### 初始化 K8S 证书 264 | virtual-kubelet 和 k8s 交互依赖证书。 265 | 266 | ### 初始化 APIServer 267 | 初始化一个 golang 原生的 http 的 Mux 实例。
http 服务必须是加密的，因此还需要初始化对应: 268 | 269 | - kubelet 的 ca：用于加密 kubelet 的信息。 270 | - kubelet 的 key：用于解密服务端发送过来的信息。 271 | - server.ca：用于加密对服务端的掉用。 272 | 273 | 这些证书不可以是任意的自签证书，具体维护逻辑可以参考：
[https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/) 274 | 275 | ### 初始化 Node 信息 276 | 初始化一个 virtual-node 的信息，并且上报给 ApiServer，一些关键的配置有： 277 | 278 | - node 的 labels 279 | - node 的 taints 280 | - node 的 capacity 281 | - node 的 ip 地址 282 | - node 的 dameonEndpoint 配置 283 | 284 | ### 初始化 tracing 采点逻辑 285 | 初始化一个符合 open-tracing 踩点的逻辑。 286 | 287 | ### 启动本地的 Informer 循环 288 | 初始化 pod 和 node 的 Informer 循环。 289 | 290 | ### 初始化 Controller 循环 291 | 初始化 pod 和 node 的 controller 循环。 292 | 293 | 294 | ## NodeController 循环 295 | 主要逻辑是： 296 | 297 | 1. 初始化 node 的基础信息，并在 apiserver 创建。 298 | 2. 通过 NotifyNodeStatus 方法更新 node 的状态。 299 | 300 | ## PodController 循环 301 | 有 3 个核心循环： 302 | 303 | 1. 基于 k8s 的 informer 机制，不断同步服务器的 pod 信息到本地，并且更新本地的状态 / 创建 pod 实例，最终会掉用 Provider.GetPod / Provider.UpdatePod 304 | 2. 基于 k8s 的 informer 机制，不断同步服务器的待删除的（DeletionTimestamp 不为 null）pod 到本地，并在本地删除对应的 pod 实例，最终会掉用 Provider.DeletePod 305 | 3. 不断同步本地的 pod status，如果不一样则更新服务端的 pod status，会掉用 Provider.GetPodStatus 方法。 306 | 307 | ## Provider 实现核心 308 | 309 | ### 运行时信息映射 310 | 在 virtual-kubelet 的抽象中，vpod 是 vk 与 apiserver 交互的最小单位，模块是 vk 与 arklet 交互的最小单位。
在用户的视角，其提交的运维单位是 vpod，vpod 会被 vk 翻译成 n 个可能的模块，并下发给 arklet。而 vpod 和模块的对应关系，在 pod 的 spec 定义时已经通过 container 字段进行映射和描述清楚了。
在 vk 的视角，其需要不断的查询模块的信息，并翻译成 vpod 的状态，然后同步给 apiserver，最终更新对应的 status 字段。可是目前的问题是，模块是属于哪个 vpod的？这个信息应该记录在哪里？应该如何聚和翻译？
首先需要解决的问题是，如何确定模块是属于哪个 vpod 的？
一个可以预期的方案是，利用 containerStatus 的 containerId 字段设置成 bizIdenetity 字段，进行模块 -> vpod 的信息查找。目前 bizIdentity 的格式是 bizName:bizVersion，由系统自动生成。未来，我们希望这个 identity 可以由运维管道强制制定，格式为 vpod_{namespace}_{podName}_{moduleName}:{version}。
现阶段暂时使用 bizName:bizVersion 作为 bizIdenetity 的字段，不过这会带来一个问题，如果用户在 2 个 pod 上都声明了同样的 bizName+bizVersion，那么实际上在 arklet 安装的时候，会报错，因为其不支持同名同版本重复装载。不过一方面暂时用户没有这个用法，所以暂时不解决。
因此，vpod 到模块的关联关系，是通过 bizIdentity 这个信息去关联的。只是目前，bizIdentity 的值是 bizName:version。未来会采用 vpod_{namespace}_{podName}_{moduleName}:{version}。
那么接下来的问题是，如何翻译 pod 的状态？
pod 的状态依赖 container 的状态，而 container 的状态就是模块的状态，模块的状态可以通过 arklet 一口气查出来。我们只需要根据 bizIdentity 进行模块状态的按照关联 pod 的维度聚合，就可以获取一个完整的 contaienrStatus 状态了。其中，如果对比 vpod 的 spec 发现有模块未安装 / 安装失败，则更新为安装失败。如果都安装成功，则更新 pod 状态为安装成功。 311 | 312 | ### 从接受 vpod 到安装 / 更新 / 卸载模块 313 | 当 vk 接受到一个 vpod 后，其首先需要通过翻译模块，解析出 vpod 需要安装的模块模型，然后进行创建或者更新的调和逻辑。
如果是进行模块创建逻辑，则 vk 会掉用 arklet 进行有关模块的安装流程。
如果是进行 pod 更新的流程，则 vk 需要先对比内存中老的 pod 对应的那些模块信息，并安装新增的和删除老的。
如果是进行 pod 的删除流程，则 vk 需要卸载对应的模块。
在这里，由于分布式容错或者网络延迟等因素，jvm 层面可能出现悬挂的模块，即 vk 发现这个模块不关联到任何的模块中。因此我们需要一个兜底的守护进程，定期的去查找到这些的悬挂模块，并进行删除操作。 314 | 315 | ### Node 的信息初始化 / 状态上报 / 自愈流程 316 | todo 317 | 318 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | English | [简体中文](./README-zh_CN.md) 4 | 5 |
6 | 7 | ModuleController is Koupleless operation and scheduling system. It is a kubernetes operator application which can be deployed to your kubernetes cluster. 8 | 9 | For ModuleController user documentation, please see [here](https://koupleless.io/docs/tutorials/module-operation/module-online-and-offline/). 10 | 11 | For ModuleController contributor and technical details documentation, please see [here](https://koupleless.io/docs/contribution-guidelines/module-controller/architecture/). 12 | -------------------------------------------------------------------------------- /cmd/module-controller/main.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "fmt" 21 | "github.com/koupleless/module_controller/common/zaplogger" 22 | "github.com/koupleless/module_controller/report_server" 23 | "github.com/koupleless/virtual-kubelet/vnode_controller" 24 | "os" 25 | "os/signal" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/healthz" 28 | "strconv" 29 | "syscall" 30 | 31 | "github.com/google/uuid" 32 | "github.com/koupleless/module_controller/common/model" 33 | "github.com/koupleless/module_controller/controller/module_deployment_controller" 34 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel" 35 | "github.com/koupleless/module_controller/module_tunnels/koupleless_mqtt_tunnel" 36 | "github.com/koupleless/virtual-kubelet/common/tracker" 37 | "github.com/koupleless/virtual-kubelet/common/utils" 38 | vkModel "github.com/koupleless/virtual-kubelet/model" 39 | "github.com/koupleless/virtual-kubelet/tunnel" 40 | "github.com/sirupsen/logrus" 41 | "github.com/virtual-kubelet/virtual-kubelet/log" 42 | logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" 43 | "github.com/virtual-kubelet/virtual-kubelet/trace" 44 | "github.com/virtual-kubelet/virtual-kubelet/trace/opencensus" 45 | "sigs.k8s.io/controller-runtime/pkg/cache" 46 | "sigs.k8s.io/controller-runtime/pkg/client/config" 47 | "sigs.k8s.io/controller-runtime/pkg/manager" 48 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 49 | "sigs.k8s.io/controller-runtime/pkg/metrics/server" 50 | ) 51 | 52 | // Main function for the module controller 53 | // Responsibilities: 54 | // 1. Sets up signal handling for graceful shutdown 55 | // 2. Initializes reporting server 56 | // 3. Configures logging and tracing 57 | // 4. Sets up controller manager and caches 58 | // 5. Initializes tunnels (MQTT and HTTP) based on env vars 59 | // 6. Creates and configures the VNode controller 60 | // 7. Optionally creates module deployment controller 61 | // 8. Starts all tunnels and the manager 62 | 63 | func main() { 64 | ctx, cancel := context.WithCancel(context.Background()) 65 | sig := make(chan os.Signal, 1) 66 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) 67 | go func() { 68 | <-sig 69 | cancel() 70 | }() 71 | 72 | go report_server.InitReportServer() 73 | 74 | log.L = logruslogger.FromLogrus(logrus.NewEntry(logrus.StandardLogger())) 75 | trace.T = opencensus.Adapter{} 76 | 77 | // Get configuration from environment variables 78 | clientID := utils.GetEnv("CLIENT_ID", uuid.New().String()) 79 | env := utils.GetEnv("ENV", "dev") 80 | 81 | zlogger := zaplogger.GetLogger() 82 | ctx = zaplogger.WithLogger(ctx, zlogger) 83 | 84 | // Parse configuration with defaults 85 | isCluster := utils.GetEnv("IS_CLUSTER", "") == "true" 86 | workloadMaxLevel, err := strconv.Atoi(utils.GetEnv("WORKLOAD_MAX_LEVEL", "3")) 87 | 88 | if err != nil { 89 | zlogger.Error(err, "failed to parse WORKLOAD_MAX_LEVEL, will be set to 3 default") 90 | workloadMaxLevel = 3 91 | } 92 | 93 | vnodeWorkerNum, err := strconv.Atoi(utils.GetEnv("VNODE_WORKER_NUM", "8")) 94 | if err != nil { 95 | zlogger.Error(err, "failed to parse VNODE_WORKER_NUM, will be set to 8 default") 96 | vnodeWorkerNum = 8 97 | } 98 | 99 | // Initialize controller manager 100 | kubeConfig := config.GetConfigOrDie() 101 | // TODO: should support to set from parameter 102 | kubeConfig.QPS = 100 103 | kubeConfig.Burst = 200 104 | 105 | zlogger.Info("start to start manager") 106 | ctrl.SetLogger(zlogger) 107 | k8sControllerManager, err := manager.New(kubeConfig, manager.Options{ 108 | Cache: cache.Options{}, 109 | HealthProbeBindAddress: ":8081", 110 | Metrics: server.Options{ 111 | BindAddress: ":9090", 112 | }, 113 | }) 114 | 115 | if err != nil { 116 | zlogger.Error(err, "unable to set up overall controller manager") 117 | os.Exit(1) 118 | } 119 | 120 | tracker.SetTracker(&tracker.DefaultTracker{}) 121 | 122 | // Configure and create VNode controller 123 | vNodeControllerConfig := vkModel.BuildVNodeControllerConfig{ 124 | ClientID: clientID, 125 | Env: env, 126 | VPodType: model.ComponentModule, 127 | IsCluster: isCluster, 128 | WorkloadMaxLevel: workloadMaxLevel, 129 | VNodeWorkerNum: vnodeWorkerNum, 130 | } 131 | 132 | moduleDeploymentController, err := module_deployment_controller.NewModuleDeploymentController(env) 133 | if err != nil { 134 | zlogger.Error(err, "unable to set up module_deployment_controller") 135 | return 136 | } 137 | 138 | err = moduleDeploymentController.SetupWithManager(ctx, k8sControllerManager) 139 | if err != nil { 140 | zlogger.Error(err, "unable to setup module_deployment_controller") 141 | return 142 | } 143 | 144 | tunnel := startTunnels(ctx, clientID, env, k8sControllerManager, moduleDeploymentController) 145 | 146 | vNodeController, err := vnode_controller.NewVNodeController(&vNodeControllerConfig, tunnel) 147 | if err != nil { 148 | zlogger.Error(err, "unable to set up VNodeController") 149 | return 150 | } 151 | 152 | err = vNodeController.SetupWithManager(ctx, k8sControllerManager) 153 | if err != nil { 154 | zlogger.Error(err, "unable to setup vnode controller") 155 | return 156 | } 157 | 158 | if err := k8sControllerManager.AddHealthzCheck("healthz", healthz.Ping); err != nil { 159 | zlogger.Error(err, "unable to set up health check") 160 | os.Exit(1) 161 | } 162 | if err := k8sControllerManager.AddReadyzCheck("readyz", healthz.Ping); err != nil { 163 | zlogger.Error(err, "unable to set up ready check") 164 | os.Exit(1) 165 | } 166 | 167 | zlogger.Info("Module controller running") 168 | err = k8sControllerManager.Start(signals.SetupSignalHandler()) 169 | if err != nil { 170 | log.G(ctx).WithError(err).Error("failed to start manager") 171 | } 172 | } 173 | 174 | func startTunnels(ctx context.Context, clientId string, env string, mgr manager.Manager, 175 | moduleDeploymentController *module_deployment_controller.ModuleDeploymentController) tunnel.Tunnel { 176 | zlogger := zaplogger.FromContext(ctx) 177 | // Initialize tunnels based on configuration 178 | tunnels := make([]tunnel.Tunnel, 0) 179 | 180 | mqttTunnelEnable := utils.GetEnv("ENABLE_MQTT_TUNNEL", "false") 181 | if mqttTunnelEnable == "true" { 182 | mqttTl := koupleless_mqtt_tunnel.NewMqttTunnel(ctx, env, mgr.GetClient(), moduleDeploymentController) 183 | tunnels = append(tunnels, &mqttTl) 184 | } 185 | 186 | httpTunnelEnable := utils.GetEnv("ENABLE_HTTP_TUNNEL", "false") 187 | if httpTunnelEnable == "true" { 188 | httpTunnelListenPort, err := strconv.Atoi(utils.GetEnv("HTTP_TUNNEL_LISTEN_PORT", "7777")) 189 | 190 | if err != nil { 191 | log.G(ctx).WithError(err).Error("failed to parse HTTP_TUNNEL_LISTEN_PORT, set default port 7777") 192 | httpTunnelListenPort = 7777 193 | } 194 | 195 | httpTl := koupleless_http_tunnel.NewHttpTunnel(ctx, env, mgr.GetClient(), moduleDeploymentController, httpTunnelListenPort) 196 | tunnels = append(tunnels, &httpTl) 197 | } 198 | 199 | // Start all tunnels 200 | successTunnelCount := 0 201 | startFailedCount := 0 202 | for _, t := range tunnels { 203 | err := t.Start(clientId, env) 204 | if err != nil { 205 | zlogger.Error(err, "failed to start tunnel "+t.Key()) 206 | startFailedCount++ 207 | } else { 208 | zlogger.Info("Tunnel started: " + t.Key()) 209 | successTunnelCount++ 210 | } 211 | } 212 | 213 | if startFailedCount > 0 { 214 | panic(errors.New(fmt.Sprintf("failed to start %d tunnels", startFailedCount))) 215 | } else if successTunnelCount == 0 { 216 | panic(errors.New(fmt.Sprintf("successfully started 0 tunnels"))) 217 | } 218 | // we only using one tunnel for now 219 | return tunnels[0] 220 | } 221 | -------------------------------------------------------------------------------- /cmd/self-test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | "path" 10 | "strconv" 11 | "strings" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/virtual-kubelet/virtual-kubelet/log" 16 | v1 "k8s.io/api/apps/v1" 17 | corev1 "k8s.io/api/core/v1" 18 | "k8s.io/apimachinery/pkg/labels" 19 | "k8s.io/apimachinery/pkg/selection" 20 | "k8s.io/apimachinery/pkg/types" 21 | "k8s.io/client-go/util/workqueue" 22 | "k8s.io/utils/ptr" 23 | "sigs.k8s.io/controller-runtime/pkg/cache" 24 | "sigs.k8s.io/controller-runtime/pkg/client" 25 | "sigs.k8s.io/controller-runtime/pkg/client/config" 26 | "sigs.k8s.io/controller-runtime/pkg/controller" 27 | "sigs.k8s.io/controller-runtime/pkg/event" 28 | "sigs.k8s.io/controller-runtime/pkg/handler" 29 | "sigs.k8s.io/controller-runtime/pkg/manager" 30 | "sigs.k8s.io/controller-runtime/pkg/manager/signals" 31 | "sigs.k8s.io/controller-runtime/pkg/metrics/server" 32 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 33 | "sigs.k8s.io/controller-runtime/pkg/source" 34 | "sigs.k8s.io/yaml" 35 | ) 36 | 37 | /** 38 | 1. apply examples self-test base.yaml 39 | 2. run main 40 | */ 41 | 42 | var k8sClient client.Client 43 | var k8sCache cache.Cache 44 | 45 | var moduleDeploymentTemplate string 46 | 47 | func init() { 48 | content, err := os.ReadFile("examples/self-test/module.yaml") 49 | if err != nil { 50 | fmt.Println("Error reading module yaml") 51 | os.Exit(1) 52 | } 53 | moduleDeploymentTemplate = string(content) 54 | } 55 | 56 | var podNameToCreateTime = make(map[string]time.Time) 57 | var podNameToReadyTime = make(map[string]time.Time) 58 | 59 | var deploymentNameToCreateTime = make(map[string]time.Time) 60 | var deploymentNameToReadyTime = make(map[string]time.Time) 61 | 62 | // base num list 63 | var baseNumList = []int{ 64 | 1, 65 | 5, 10, 20, 40, 60, 80, 100, 66 | } 67 | 68 | // module num list 69 | var moduleNumList = []int{ 70 | 1, 71 | 5, 10, 20, 50, 100, 200, 500, 1000, 72 | } 73 | 74 | type TestResult struct { 75 | // PodMin represents the minimum time (in milliseconds) for a pod to become ready 76 | PodMin int64 77 | // PodMax represents the maximum time (in milliseconds) for a pod to become ready 78 | PodMax int64 79 | // PodAvg represents the average time (in milliseconds) for pods to become ready 80 | PodAvg int64 81 | // DeployMin represents the minimum time (in milliseconds) for a deployment to become ready 82 | DeployMin int64 83 | // DeployMax represents the maximum time (in milliseconds) for a deployment to become ready 84 | DeployMax int64 85 | // DeployAvg represents the average time (in milliseconds) for deployments to become ready 86 | DeployAvg int64 87 | } 88 | 89 | func constructModuleDeployment(baseNum int, bizIndex int, bizVersion string) v1.Deployment { 90 | template := strings.ReplaceAll(moduleDeploymentTemplate, "{baseNum}", strconv.Itoa(baseNum)) 91 | template = strings.ReplaceAll(template, "{bizIndex}", strconv.Itoa(bizIndex)) 92 | template = strings.ReplaceAll(template, "{bizVersion}", bizVersion) 93 | ret := v1.Deployment{} 94 | err := yaml.Unmarshal([]byte(template), &ret) 95 | if err != nil { 96 | panic(err) 97 | } 98 | return ret 99 | } 100 | 101 | func updateReplicas(newReplicas int32) error { 102 | obj := &v1.Deployment{} 103 | err := k8sClient.Get(context.TODO(), types.NamespacedName{ 104 | Name: "base", 105 | Namespace: "default", 106 | }, obj) 107 | if err != nil { 108 | fmt.Println("Error getting base:", err) 109 | } else { 110 | if obj.Spec.Replicas != nil && *obj.Spec.Replicas == newReplicas { 111 | return nil 112 | } 113 | obj.Spec.Replicas = ptr.To[int32](newReplicas) 114 | err = k8sClient.Update(context.TODO(), obj) 115 | if err != nil { 116 | fmt.Println("Error update base:", err) 117 | return err 118 | } 119 | } 120 | 121 | allReady := false 122 | for !allReady { 123 | fmt.Println("check base ready: ", newReplicas) 124 | list := corev1.NodeList{} 125 | err = k8sClient.List(context.TODO(), &list) 126 | if err != nil { 127 | fmt.Println("Error listing nodes:", err.Error()) 128 | continue 129 | } 130 | readyNum := 0 131 | for _, node := range list.Items { 132 | if !strings.HasPrefix(node.Name, "vnode") { 133 | continue 134 | } 135 | nodeReady := false 136 | for _, condition := range node.Status.Conditions { 137 | if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { 138 | nodeReady = true 139 | break 140 | } 141 | } 142 | if nodeReady { 143 | readyNum++ 144 | } 145 | } 146 | if readyNum == int(newReplicas) { 147 | allReady = true 148 | } else { 149 | time.Sleep(2 * time.Second) 150 | } 151 | } 152 | return nil 153 | } 154 | 155 | func statistic() TestResult { 156 | ret := TestResult{} 157 | minMillis := int64(1000 * 60 * 100) 158 | maxMillis := int64(0) 159 | totalMillis := int64(0) 160 | totalValidNum := int64(0) 161 | for podName, readyTime := range podNameToReadyTime { 162 | createTime, has := podNameToCreateTime[podName] 163 | if !has { 164 | fmt.Println("podName: ", podName, ", not find create time") 165 | continue 166 | } 167 | totalValidNum++ 168 | readyCost := readyTime.Sub(createTime).Milliseconds() 169 | if readyCost > maxMillis { 170 | maxMillis = readyCost 171 | } 172 | if readyCost < minMillis { 173 | minMillis = readyCost 174 | } 175 | totalMillis += readyCost 176 | } 177 | ret.PodMin = minMillis 178 | ret.PodMax = maxMillis 179 | ret.PodAvg = totalMillis / totalValidNum 180 | 181 | minMillis = int64(1000 * 60 * 100) 182 | maxMillis = int64(0) 183 | totalMillis = int64(0) 184 | totalValidNum = int64(0) 185 | for deploymentName, readyTime := range deploymentNameToReadyTime { 186 | createTime, has := deploymentNameToCreateTime[deploymentName] 187 | if !has { 188 | fmt.Println("deploymentName: ", deploymentName, ", not find create time") 189 | continue 190 | } 191 | totalValidNum++ 192 | readyCost := readyTime.Sub(createTime).Milliseconds() 193 | if readyCost > maxMillis { 194 | maxMillis = readyCost 195 | } 196 | if readyCost < minMillis { 197 | minMillis = readyCost 198 | } 199 | totalMillis += readyCost 200 | } 201 | ret.DeployMin = minMillis 202 | ret.DeployMax = maxMillis 203 | ret.DeployAvg = totalMillis / totalValidNum 204 | return ret 205 | } 206 | 207 | func test(baseNum, moduleNum int) (create TestResult, update *TestResult) { 208 | 209 | fmt.Println("start test: ", baseNum, moduleNum) 210 | 211 | podNameToReadyTime = make(map[string]time.Time) 212 | podNameToCreateTime = make(map[string]time.Time) 213 | deploymentNameToCreateTime = make(map[string]time.Time) 214 | deploymentNameToReadyTime = make(map[string]time.Time) 215 | // 1. 更新基座deployment，等待所有的 node 上线 216 | err := updateReplicas(int32(baseNum)) 217 | if err != nil { 218 | os.Exit(2) 219 | } 220 | // 2. 按序发布模块deployment，replicas 设置为base数量 221 | for i := 1; i <= moduleNum; i++ { 222 | deployment := constructModuleDeployment(baseNum, i, "0.0.1") 223 | err = k8sClient.Create(context.TODO(), &deployment) 224 | if err != nil { 225 | fmt.Println("Error creating module deployment:", err.Error()) 226 | os.Exit(2) 227 | } 228 | deploymentNameToCreateTime[deployment.Name] = time.Now() 229 | } 230 | // 3. 等待所有的deployment都变成ready，对数据进行统计，存到create 231 | for len(deploymentNameToReadyTime) != moduleNum { 232 | fmt.Println("waiting for all module deployment ready in creating: ", len(deploymentNameToReadyTime), "/", moduleNum) 233 | time.Sleep(time.Second * 5) 234 | } 235 | 236 | time.Sleep(time.Second * 5) 237 | 238 | create = statistic() 239 | 240 | return 241 | } 242 | 243 | type Result struct { 244 | Create TestResult 245 | Update *TestResult `json:"Update,omitempty"` 246 | } 247 | 248 | func cleanResources() { 249 | 250 | // clean resources 251 | list := v1.DeploymentList{} 252 | err := k8sClient.List(context.TODO(), &list, client.InNamespace("module")) 253 | if err != nil { 254 | fmt.Println("Error listing deployments:", err.Error()) 255 | os.Exit(1) 256 | } 257 | 258 | for _, deployment := range list.Items { 259 | err = k8sClient.Delete(context.TODO(), &deployment) 260 | if err != nil { 261 | fmt.Println("Error listing deployments:", err.Error()) 262 | os.Exit(1) 263 | } 264 | } 265 | 266 | // check all pod has been released 267 | existModulePod := true 268 | for existModulePod { 269 | fmt.Println("waiting for module pod to delete") 270 | podList := &corev1.PodList{} 271 | err = k8sClient.List(context.TODO(), podList, client.InNamespace("module")) 272 | if err != nil { 273 | fmt.Println("Error listing pods:", err.Error()) 274 | os.Exit(1) 275 | } 276 | if len(podList.Items) == 0 { 277 | existModulePod = false 278 | } 279 | time.Sleep(time.Second * 5) 280 | } 281 | } 282 | 283 | func pressureTest() { 284 | cleanResources() 285 | 286 | res := make(map[int]map[int]Result) 287 | dir := "test_results" 288 | filename := "test_result_http_tunnel_worker8_netty8.json" 289 | 290 | filename = path.Join(dir, filename) 291 | 292 | _, err := os.Stat(filename) 293 | if err != nil { 294 | os.Create(filename) 295 | os.WriteFile(filename, []byte("{}"), 0644) 296 | } 297 | 298 | content, err := os.ReadFile(filename) 299 | if err != nil { 300 | fmt.Println(err.Error()) 301 | os.Exit(1) 302 | } 303 | 304 | err = json.Unmarshal(content, &res) 305 | if err != nil { 306 | fmt.Println(err.Error()) 307 | os.Exit(1) 308 | } 309 | 310 | for _, baseNum := range baseNumList { 311 | val, has := res[baseNum] 312 | if !has { 313 | val = make(map[int]Result) 314 | } 315 | for _, moduleNum := range moduleNumList { 316 | _, done := val[moduleNum] 317 | if done { 318 | continue 319 | } 320 | create, update := test(baseNum, moduleNum) 321 | val[moduleNum] = Result{ 322 | Create: create, 323 | Update: update, 324 | } 325 | res[baseNum] = val 326 | // save result 327 | marshal, err := json.MarshalIndent(res, "", " ") 328 | if err != nil { 329 | fmt.Println(err.Error()) 330 | return 331 | } 332 | err = os.WriteFile(filename, marshal, 0644) 333 | if err != nil { 334 | fmt.Println(err.Error()) 335 | return 336 | } 337 | cleanResources() 338 | } 339 | } 340 | 341 | } 342 | 343 | type Controller struct{} 344 | 345 | func (c Controller) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 346 | return reconcile.Result{}, nil 347 | } 348 | 349 | var _ reconcile.TypedReconciler[reconcile.Request] = Controller{} 350 | 351 | func main() { 352 | ctx, cancel := context.WithCancel(context.Background()) 353 | sig := make(chan os.Signal, 1) 354 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) 355 | go func() { 356 | <-sig 357 | cancel() 358 | }() 359 | 360 | kubeConfig := config.GetConfigOrDie() 361 | mgr, err := manager.New(kubeConfig, manager.Options{ 362 | Cache: cache.Options{}, 363 | Metrics: server.Options{ 364 | BindAddress: "0", 365 | }, 366 | }) 367 | 368 | if err != nil { 369 | fmt.Println(err.Error(), "unable to set up overall controller manager") 370 | os.Exit(1) 371 | } 372 | 373 | k8sClient = mgr.GetClient() 374 | k8sCache = mgr.GetCache() 375 | 376 | c, err := controller.New("vnode-controller", mgr, controller.Options{ 377 | Reconciler: Controller{}, 378 | }) 379 | if err != nil { 380 | fmt.Println(err.Error(), "unable to set up overall controller manager") 381 | os.Exit(1) 382 | } 383 | 384 | vpodRequirement, _ := labels.NewRequirement("virtual-kubelet.koupleless.io/component", selection.In, []string{"module"}) 385 | 386 | podHandler := handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ 387 | CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*corev1.Pod], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 388 | podNameToCreateTime[e.Object.Name] = time.Now() 389 | }, 390 | UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*corev1.Pod], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 391 | if e.ObjectNew.DeletionTimestamp != nil { 392 | // 已删除 393 | return 394 | } 395 | _, has := podNameToReadyTime[e.ObjectNew.Name] 396 | if has { 397 | return 398 | } 399 | isReady := false 400 | for _, condition := range e.ObjectNew.Status.Conditions { 401 | if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { 402 | isReady = true 403 | break 404 | } 405 | } 406 | if !isReady { 407 | return 408 | } 409 | podNameToReadyTime[e.ObjectNew.Name] = time.Now() 410 | }, 411 | } 412 | 413 | if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &podHandler, &VPodPredicates{ 414 | LabelSelector: labels.NewSelector().Add(*vpodRequirement), 415 | })); err != nil { 416 | fmt.Println(err.Error(), "unable to watch Pods") 417 | os.Exit(1) 418 | } 419 | 420 | nodeHandler := handler.TypedFuncs[*corev1.Node, reconcile.Request]{ 421 | CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*corev1.Node], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 422 | log.G(ctx).Info("vnode created ", e.Object.Name) 423 | }, 424 | } 425 | 426 | vnodeRequirement, _ := labels.NewRequirement("virtual-kubelet.koupleless.io/component", selection.In, []string{"vnode"}) 427 | envRequirement, _ := labels.NewRequirement("virtual-kubelet.koupleless.io/env", selection.In, []string{"test"}) 428 | 429 | if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}, &nodeHandler, &VNodePredicate{ 430 | VNodeLabelSelector: labels.NewSelector().Add(*vnodeRequirement, *envRequirement), 431 | })); err != nil { 432 | fmt.Println(err.Error(), "unable to watch vnodes") 433 | os.Exit(1) 434 | } 435 | 436 | // sync deployment cache 437 | deploymentRequirement, _ := labels.NewRequirement("virtual-kubelet.koupleless.io/component", selection.In, []string{"module-deployment"}) 438 | deploymentSelector := labels.NewSelector().Add(*deploymentRequirement) 439 | 440 | var deploymentEventHandler = handler.TypedFuncs[*v1.Deployment, reconcile.Request]{ 441 | CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*v1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 442 | }, 443 | UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*v1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 444 | if e.ObjectNew.DeletionTimestamp != nil { 445 | return 446 | } 447 | if e.ObjectNew.Status.ReadyReplicas == *e.ObjectNew.Spec.Replicas { 448 | deploymentNameToReadyTime[e.ObjectNew.Name] = time.Now() 449 | } 450 | }, 451 | } 452 | 453 | if err = c.Watch(source.Kind(mgr.GetCache(), &v1.Deployment{}, &deploymentEventHandler, &ModuleDeploymentPredicates{LabelSelector: deploymentSelector})); err != nil { 454 | fmt.Println(err.Error(), "unable to watch deployments") 455 | os.Exit(1) 456 | } 457 | 458 | go func() { 459 | err = mgr.Start(signals.SetupSignalHandler()) 460 | if err != nil { 461 | log.G(ctx).WithError(err).Error("failed to start manager") 462 | } 463 | }() 464 | 465 | sync := k8sCache.WaitForCacheSync(ctx) 466 | if !sync { 467 | fmt.Println("unable to sync cache") 468 | os.Exit(1) 469 | } 470 | 471 | pressureTest() 472 | } 473 | -------------------------------------------------------------------------------- /cmd/self-test/predicate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | v1 "k8s.io/api/apps/v1" 5 | corev1 "k8s.io/api/core/v1" 6 | "k8s.io/apimachinery/pkg/labels" 7 | "sigs.k8s.io/controller-runtime/pkg/event" 8 | "sigs.k8s.io/controller-runtime/pkg/predicate" 9 | ) 10 | 11 | var _ predicate.TypedPredicate[*corev1.Pod] = &VPodPredicates{} 12 | 13 | type VPodPredicates struct { 14 | LabelSelector labels.Selector 15 | } 16 | 17 | func (V *VPodPredicates) Create(e event.TypedCreateEvent[*corev1.Pod]) bool { 18 | return V.LabelSelector.Matches(labels.Set(e.Object.Labels)) 19 | } 20 | 21 | func (V *VPodPredicates) Delete(e event.TypedDeleteEvent[*corev1.Pod]) bool { 22 | return V.LabelSelector.Matches(labels.Set(e.Object.Labels)) 23 | } 24 | 25 | func (V *VPodPredicates) Update(e event.TypedUpdateEvent[*corev1.Pod]) bool { 26 | return V.LabelSelector.Matches(labels.Set(e.ObjectNew.Labels)) 27 | } 28 | 29 | func (V *VPodPredicates) Generic(e event.TypedGenericEvent[*corev1.Pod]) bool { 30 | return false 31 | } 32 | 33 | var _ predicate.TypedPredicate[*corev1.Node] = &VNodePredicate{} 34 | 35 | type VNodePredicate struct { 36 | VNodeLabelSelector labels.Selector 37 | } 38 | 39 | type ModuleDeploymentPredicates struct { 40 | LabelSelector labels.Selector 41 | } 42 | 43 | func (V *VNodePredicate) Create(e event.TypedCreateEvent[*corev1.Node]) bool { 44 | return V.VNodeLabelSelector.Matches(labels.Set(e.Object.Labels)) 45 | } 46 | 47 | func (V *VNodePredicate) Delete(e event.TypedDeleteEvent[*corev1.Node]) bool { 48 | return V.VNodeLabelSelector.Matches(labels.Set(e.Object.Labels)) 49 | } 50 | 51 | func (V *VNodePredicate) Update(e event.TypedUpdateEvent[*corev1.Node]) bool { 52 | return V.VNodeLabelSelector.Matches(labels.Set(e.ObjectNew.Labels)) 53 | } 54 | 55 | func (V *VNodePredicate) Generic(e event.TypedGenericEvent[*corev1.Node]) bool { 56 | return V.VNodeLabelSelector.Matches(labels.Set(e.Object.Labels)) 57 | } 58 | 59 | func (m *ModuleDeploymentPredicates) Create(e event.TypedCreateEvent[*v1.Deployment]) bool { 60 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 61 | } 62 | 63 | func (m *ModuleDeploymentPredicates) Delete(e event.TypedDeleteEvent[*v1.Deployment]) bool { 64 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 65 | } 66 | 67 | func (m *ModuleDeploymentPredicates) Update(e event.TypedUpdateEvent[*v1.Deployment]) bool { 68 | return m.LabelSelector.Matches(labels.Set(e.ObjectNew.Labels)) 69 | } 70 | 71 | func (m *ModuleDeploymentPredicates) Generic(e event.TypedGenericEvent[*v1.Deployment]) bool { 72 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 73 | } 74 | -------------------------------------------------------------------------------- /common/model/consts.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "github.com/koupleless/virtual-kubelet/model" 4 | 5 | // TrackEvent constants 6 | const ( 7 | // TrackEventVPodPeerDeploymentReplicaModify tracks when a peer deployment's replicas are modified 8 | TrackEventVPodPeerDeploymentReplicaModify = "PeerDeploymentReplicaModify" 9 | ) 10 | 11 | // Label keys for module controller 12 | const ( 13 | // LabelKeyOfSkipReplicasControl indicates whether to skip replicas control 14 | LabelKeyOfSkipReplicasControl = "virtual-kubelet.koupleless.io/replicas-control" 15 | // LabelKeyOfVPodDeploymentStrategy specifies the deployment strategy 16 | LabelKeyOfVPodDeploymentStrategy = "virtual-kubelet.koupleless.io/strategy" 17 | ) 18 | 19 | // Component types 20 | const ( 21 | // ComponentModule represents a module component 22 | ComponentModule = "module" 23 | // ComponentModuleDeployment represents a module deployment component 24 | ComponentModuleDeployment = "module-deployment" 25 | ) 26 | 27 | // VPodDeploymentStrategy defines deployment strategies for VPods 28 | type VPodDeploymentStrategy string 29 | 30 | // Available VPod deployment strategies 31 | const ( 32 | // VPodDeploymentStrategyPeer indicates peer deployment strategy 33 | VPodDeploymentStrategyPeer VPodDeploymentStrategy = "peer" 34 | ) 35 | 36 | // Error codes 37 | const ( 38 | // CodeKubernetesOperationFailed indicates a Kubernetes operation failure 39 | CodeKubernetesOperationFailed model.ErrorCode = "00003" 40 | ) 41 | 42 | // Command types for module operations 43 | const ( 44 | // CommandHealth checks module health 45 | CommandHealth = "health" 46 | // CommandQueryAllBiz queries all business modules 47 | CommandQueryAllBiz = "queryAllBiz" 48 | // CommandInstallBiz installs a business module 49 | CommandInstallBiz = "installBiz" 50 | // CommandUnInstallBiz uninstalls a business module 51 | CommandUnInstallBiz = "uninstallBiz" 52 | // CommandBatchInstallBiz batch install biz, since koupleless-runtime 1.4.1 53 | CommandBatchInstallBiz = "batchInstallBiz" 54 | ) 55 | 56 | // MQTT topic patterns for base communication 57 | const ( 58 | // BaseHeartBeatTopic for heartbeat messages, broadcast mode 59 | BaseHeartBeatTopic = "koupleless_%s/+/base/heart" 60 | // BaseQueryBaselineTopic for baseline queries, broadcast mode 61 | BaseQueryBaselineTopic = "koupleless_%s/+/base/queryBaseline" 62 | // BaseHealthTopic for health status, p2p mode 63 | BaseHealthTopic = "koupleless_%s/%s/base/health" 64 | // BaseSimpleBizTopic for simple business operations, p2p mode 65 | BaseSimpleBizTopic = "koupleless_%s/%s/base/simpleBiz" 66 | // BaseAllBizTopic for all business operations, p2p mode 67 | BaseAllBizTopic = "koupleless_%s/%s/base/biz" 68 | // BaseBizOperationResponseTopic for business operation responses, p2p mode 69 | BaseBizOperationResponseTopic = "koupleless_%s/%s/base/bizOperation" 70 | // BaseBatchInstallBizResponseTopic for response of batch install biz, p2p mode, since koupleless-runtime 1.4.1 71 | BaseBatchInstallBizResponseTopic = "koupleless_%s/%s/base/batchInstallBizResponse" 72 | // BaseBaselineResponseTopic for baseline responses, p2p mode 73 | BaseBaselineResponseTopic = "koupleless_%s/%s/base/baseline" 74 | ) 75 | 76 | // Base labels 77 | const ( 78 | // LabelKeyOfTunnelPort specifies the tunnel port 79 | LabelKeyOfTunnelPort = "base.koupleless.io/tunnel-port" 80 | ) 81 | -------------------------------------------------------------------------------- /common/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/koupleless/arkctl/v1/service/ark" 5 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel/ark_service" 6 | ) 7 | 8 | // ArkMqttMsg is the response of mqtt message payload. 9 | type ArkMqttMsg[T any] struct { 10 | PublishTimestamp int64 `json:"publishTimestamp"` 11 | Data T `json:"data"` 12 | } 13 | 14 | // BaseMetadata contains basic identifying information 15 | type BaseMetadata struct { 16 | Identity string `json:"identity"` 17 | Version string `json:"version"` // Version identifier 18 | ClusterName string `json:"clusterName"` // ClusterName of the resource communicate with base 19 | Name string `json:"name"` // Name of the base 20 | } 21 | 22 | // BaseStatus is the data of base heart beat. 23 | // Contains information about the base node's status and network details 24 | type BaseStatus struct { 25 | BaseMetadata BaseMetadata `json:"baseMetadata"` // Master business info metadata 26 | LocalIP string `json:"localIP"` // Local IP address 27 | LocalHostName string `json:"localHostName"` // Local hostname 28 | Port int `json:"port"` // Port number for arklet service 29 | State string `json:"state"` // Current state of the base 30 | } 31 | 32 | // BizOperationResponse represents the response from a business operation 33 | type BizOperationResponse struct { 34 | Command string `json:"command"` // Operation command executed 35 | BizName string `json:"bizName"` // ClusterName of the business 36 | BizVersion string `json:"bizVersion"` // Version of the business 37 | Response ark_service.ArkResponse[ark.ArkResponseData] `json:"response"` // Response from ark service 38 | } 39 | 40 | type BatchInstallBizResponse struct { 41 | Command string `json:"command"` // Operation command executed 42 | Response ark_service.ArkResponse[ark.ArkBatchInstallResponse] `json:"response"` // Response from ark service 43 | } 44 | 45 | // QueryBaselineRequest is the request parameters of query baseline func 46 | // Used to query baseline configuration with filters 47 | type QueryBaselineRequest struct { 48 | Identity string `json:"identity"` // Identity base to filter by 49 | ClusterName string `json:"clusterName"` // ClusterName to filter by 50 | Version string `json:"version"` // Version to filter by 51 | } 52 | 53 | // BuildModuleDeploymentControllerConfig contains controller configuration 54 | type BuildModuleDeploymentControllerConfig struct { 55 | Env string `json:"env"` // Environment setting 56 | } 57 | 58 | // ArkSimpleAllBizInfoData is a collection of business info data 59 | type ArkSimpleAllBizInfoData []ArkSimpleBizInfoData 60 | 61 | // ArkSimpleBizInfoData represents simplified business information 62 | type ArkSimpleBizInfoData struct { 63 | Name string `json:"name"` // Name of the biz 64 | Version string `json:"version"` // Version of the biz 65 | State string `json:"state"` // State of the biz 66 | LatestStateRecord ark.ArkBizStateRecord `json:"latestStateRecord"` // Latest state record of the biz 67 | } 68 | -------------------------------------------------------------------------------- /common/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/koupleless/module_controller/common/zaplogger" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/koupleless/arkctl/common/fileutil" 12 | "github.com/koupleless/arkctl/v1/service/ark" 13 | "github.com/koupleless/module_controller/common/model" 14 | "github.com/koupleless/virtual-kubelet/common/utils" 15 | vkModel "github.com/koupleless/virtual-kubelet/model" 16 | corev1 "k8s.io/api/core/v1" 17 | apiErrors "k8s.io/apimachinery/pkg/api/errors" 18 | "sigs.k8s.io/controller-runtime/pkg/client" 19 | ) 20 | 21 | // GetBaseIdentityFromTopic extracts the base ID from a topic string by splitting on '/' 22 | func GetBaseIdentityFromTopic(topic string) string { 23 | fileds := strings.Split(topic, "/") 24 | if len(fileds) < 2 { 25 | return "" 26 | } 27 | return fileds[1] 28 | } 29 | 30 | // Expired checks if a timestamp has expired based on a max lifetime 31 | func Expired(publishTimestamp int64, maxLiveMilliSec int64) bool { 32 | return publishTimestamp+maxLiveMilliSec <= time.Now().UnixMilli() 33 | } 34 | 35 | // FormatArkletCommandTopic formats a topic string for arklet commands 36 | func FormatArkletCommandTopic(env, nodeID, command string) string { 37 | return fmt.Sprintf("koupleless_%s/%s/%s", env, nodeID, command) 38 | } 39 | 40 | // FormatBaselineResponseTopic formats a topic string for baseline responses 41 | func FormatBaselineResponseTopic(env, nodeID string) string { 42 | return fmt.Sprintf(model.BaseBaselineResponseTopic, env, nodeID) 43 | } 44 | 45 | // GetBizVersionFromContainer extracts the biz version from a container's env vars 46 | func GetBizVersionFromContainer(container *corev1.Container) string { 47 | bizVersion := "" 48 | for _, env := range container.Env { 49 | if env.Name == "BIZ_VERSION" { 50 | bizVersion = env.Value 51 | break 52 | } 53 | } 54 | return bizVersion 55 | } 56 | 57 | // TranslateCoreV1ContainerToBizModel converts a k8s container to an ark biz model 58 | func TranslateCoreV1ContainerToBizModel(container *corev1.Container) ark.BizModel { 59 | return ark.BizModel{ 60 | BizName: container.Name, 61 | BizVersion: GetBizVersionFromContainer(container), 62 | BizUrl: fileutil.FileUrl(container.Image), 63 | } 64 | } 65 | 66 | // GetBizIdentity creates a unique identifier from biz name and version 67 | func GetBizIdentity(bizName, bizVersion string) string { 68 | return bizName + ":" + bizVersion 69 | } 70 | 71 | // ConvertBaseStatusToNodeInfo converts heartbeat data to node info 72 | func ConvertBaseStatusToNodeInfo(data model.BaseStatus, env string) vkModel.NodeInfo { 73 | state := vkModel.NodeStateDeactivated 74 | if strings.EqualFold(data.State, "ACTIVATED") { 75 | state = vkModel.NodeStateActivated 76 | } 77 | labels := map[string]string{} 78 | if data.Port != 0 { 79 | labels[model.LabelKeyOfTunnelPort] = strconv.Itoa(data.Port) 80 | } 81 | 82 | return vkModel.NodeInfo{ 83 | Metadata: vkModel.NodeMetadata{ 84 | Name: utils.FormatNodeName(data.BaseMetadata.Identity, env), 85 | BaseName: data.BaseMetadata.Name, 86 | ClusterName: data.BaseMetadata.ClusterName, 87 | Version: data.BaseMetadata.Version, 88 | }, 89 | NetworkInfo: vkModel.NetworkInfo{ 90 | NodeIP: data.LocalIP, 91 | HostName: data.LocalHostName, 92 | }, 93 | CustomLabels: labels, 94 | State: state, 95 | } 96 | } 97 | 98 | // ConvertHealthDataToNodeStatus converts health data to node status 99 | func ConvertHealthDataToNodeStatus(data ark.HealthData) vkModel.NodeStatusData { 100 | resourceMap := make(map[corev1.ResourceName]vkModel.NodeResource) 101 | memory := vkModel.NodeResource{} 102 | // Set memory capacity if JavaMaxMetaspace is valid (not -1) 103 | if data.Jvm.JavaMaxMetaspace != -1 { 104 | memory.Capacity = utils.ConvertByteNumToResourceQuantity(data.Jvm.JavaMaxMetaspace) 105 | } 106 | 107 | // Calculate allocatable memory as max metaspace minus committed metaspace 108 | // Only if both values are valid (not -1) 109 | if data.Jvm.JavaMaxMetaspace != -1 && data.Jvm.JavaCommittedMetaspace != -1 { 110 | memory.Allocatable = utils.ConvertByteNumToResourceQuantity(data.Jvm.JavaMaxMetaspace - data.Jvm.JavaCommittedMetaspace) 111 | } 112 | resourceMap[corev1.ResourceMemory] = memory 113 | return vkModel.NodeStatusData{ 114 | Resources: resourceMap, 115 | } 116 | } 117 | 118 | // ConvertBaseMetadataToBaselineQuery converts heartbeat metadata to a baseline query 119 | func ConvertBaseMetadataToBaselineQuery(data model.BaseMetadata) model.QueryBaselineRequest { 120 | return model.QueryBaselineRequest{ 121 | Identity: data.Identity, 122 | ClusterName: data.ClusterName, 123 | Version: data.Version, 124 | } 125 | } 126 | 127 | // TranslateBizInfosToContainerStatuses converts biz info to container statuses 128 | func TranslateBizInfosToContainerStatuses(data []ark.ArkBizInfo, changeTimestamp int64) []vkModel.BizStatusData { 129 | ret := make([]vkModel.BizStatusData, 0) 130 | for _, bizInfo := range data { 131 | updatedTime, reason, message := GetLatestState(bizInfo.BizStateRecords) 132 | statusData := vkModel.BizStatusData{ 133 | Key: GetBizIdentity(bizInfo.BizName, bizInfo.BizVersion), 134 | Name: bizInfo.BizName, 135 | // fille PodKey when using 136 | //PodKey: vkModel.PodKeyAll, 137 | State: bizInfo.BizState, 138 | // TODO: 需要使用实际 bizState 变化的时间，而非心跳时间 139 | ChangeTime: time.UnixMilli(changeTimestamp), 140 | } 141 | if updatedTime.UnixMilli() != 0 { 142 | statusData.ChangeTime = updatedTime 143 | statusData.Reason = reason 144 | statusData.Message = message 145 | } 146 | ret = append(ret, statusData) 147 | } 148 | return ret 149 | } 150 | 151 | // TranslateSimpleBizDataToBizInfos converts simple biz data to biz info 152 | func TranslateSimpleBizDataToBizInfos(data model.ArkSimpleAllBizInfoData) []ark.ArkBizInfo { 153 | ret := make([]ark.ArkBizInfo, 0) 154 | for _, simpleBizInfo := range data { 155 | bizInfo := TranslateSimpleBizDataToArkBizInfo(simpleBizInfo) 156 | if bizInfo == nil { 157 | continue 158 | } 159 | ret = append(ret, *bizInfo) 160 | } 161 | return ret 162 | } 163 | 164 | // TranslateSimpleBizDataToArkBizInfo converts simple biz data to ark biz info 165 | func TranslateSimpleBizDataToArkBizInfo(data model.ArkSimpleBizInfoData) *ark.ArkBizInfo { 166 | return &ark.ArkBizInfo{ 167 | BizName: data.Name, 168 | BizVersion: data.Version, 169 | BizState: data.State, 170 | BizStateRecords: []ark.ArkBizStateRecord{data.LatestStateRecord}, 171 | } 172 | } 173 | 174 | // GetLatestState finds the most recent state record and returns its details 175 | func GetLatestState(records []ark.ArkBizStateRecord) (time.Time, string, string) { 176 | latestStateTime := int64(0) 177 | reason := "" 178 | message := "" 179 | for _, record := range records { 180 | changeTime := record.ChangeTime 181 | if changeTime > latestStateTime { 182 | latestStateTime = changeTime 183 | reason = record.Reason 184 | message = record.Message 185 | } 186 | } 187 | return time.UnixMilli(latestStateTime), reason, message 188 | } 189 | 190 | // OnBaseUnreachable handles cleanup when a base becomes unreachable 191 | func OnBaseUnreachable(ctx context.Context, nodeName string, k8sClient client.Client) { 192 | // base not ready, delete from api server 193 | node := corev1.Node{} 194 | err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &node) 195 | logger := zaplogger.FromContext(ctx).WithValues("nodeName", nodeName, "func", "OnNodeNotReady") 196 | if err == nil { 197 | // delete node from api server 198 | logger.Info("DeleteBaseNode") 199 | deleteErr := k8sClient.Delete(ctx, &node) 200 | if deleteErr != nil && !apiErrors.IsNotFound(err) { 201 | logger.Error(deleteErr, "delete base node failed") 202 | } 203 | } else if apiErrors.IsNotFound(err) { 204 | logger.Info("Node not found, skipping delete operation") 205 | } else { 206 | logger.Error(err, "Failed to get node, cannot delete") 207 | } 208 | } 209 | 210 | // ConvertBaseStatusFromNodeInfo extracts network info from node info 211 | func ConvertBaseStatusFromNodeInfo(initData vkModel.NodeInfo) model.BaseStatus { 212 | portStr := initData.CustomLabels[model.LabelKeyOfTunnelPort] 213 | 214 | port, err := strconv.Atoi(portStr) 215 | if err != nil { 216 | zaplogger.GetLogger().Error(nil, fmt.Sprintf("failed to parse port %s from node info", portStr)) 217 | port = 1238 218 | } 219 | 220 | return model.BaseStatus{ 221 | BaseMetadata: model.BaseMetadata{ 222 | Identity: utils.ExtractNodeIDFromNodeName(initData.Metadata.Name), 223 | Version: initData.Metadata.Version, 224 | ClusterName: initData.Metadata.ClusterName, 225 | }, 226 | 227 | LocalIP: initData.NetworkInfo.NodeIP, 228 | LocalHostName: initData.NetworkInfo.HostName, 229 | Port: port, 230 | State: string(initData.State), 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /common/zaplogger/logger.go: -------------------------------------------------------------------------------- 1 | package zaplogger 2 | 3 | import ( 4 | "context" 5 | "github.com/go-logr/logr" 6 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 7 | ) 8 | 9 | var logger = zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) 10 | 11 | func GetLogger() logr.Logger { 12 | return logger 13 | } 14 | 15 | func WithLogger(ctx context.Context, logger logr.Logger) context.Context { 16 | return logr.NewContext(ctx, logger) 17 | } 18 | 19 | func FromContext(ctx context.Context) logr.Logger { 20 | l, err := logr.FromContext(ctx) 21 | if err != nil { 22 | return logger 23 | } 24 | 25 | return l 26 | } 27 | -------------------------------------------------------------------------------- /config/default_tracker_config.yaml: -------------------------------------------------------------------------------- 1 | logDir: "." 2 | reportLevel: "error" # value: debug/error, debug means success log will report too 3 | reportLinks: # default tracker will send post requests with report data to every link 4 | - "http://localhost:8080/log" # here is a test link -------------------------------------------------------------------------------- /controller/module_deployment_controller/module_deployment_controller.go: -------------------------------------------------------------------------------- 1 | package module_deployment_controller 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/koupleless/module_controller/common/zaplogger" 8 | "github.com/koupleless/virtual-kubelet/common/tracker" 9 | "github.com/koupleless/virtual-kubelet/common/utils" 10 | errors2 "k8s.io/apimachinery/pkg/api/errors" 11 | "sort" 12 | 13 | "github.com/koupleless/module_controller/common/model" 14 | vkModel "github.com/koupleless/virtual-kubelet/model" 15 | appsv1 "k8s.io/api/apps/v1" 16 | corev1 "k8s.io/api/core/v1" 17 | "k8s.io/apimachinery/pkg/labels" 18 | "k8s.io/apimachinery/pkg/selection" 19 | "k8s.io/client-go/util/workqueue" 20 | "k8s.io/utils/ptr" 21 | "sigs.k8s.io/controller-runtime/pkg/cache" 22 | "sigs.k8s.io/controller-runtime/pkg/client" 23 | "sigs.k8s.io/controller-runtime/pkg/controller" 24 | "sigs.k8s.io/controller-runtime/pkg/event" 25 | "sigs.k8s.io/controller-runtime/pkg/handler" 26 | "sigs.k8s.io/controller-runtime/pkg/manager" 27 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 28 | "sigs.k8s.io/controller-runtime/pkg/source" 29 | ) 30 | 31 | // ModuleDeploymentController is a controller that manages the deployment of modules within a specific environment. 32 | type ModuleDeploymentController struct { 33 | env string // The environment in which the controller operates. 34 | 35 | client client.Client // The client for interacting with the Kubernetes API. 36 | 37 | cache cache.Cache // The cache for storing and retrieving Kubernetes objects. 38 | 39 | updateToken chan interface{} // A channel for signaling updates. 40 | } 41 | 42 | // Reconcile is the main reconciliation function for the controller. 43 | func (moduleDeploymentController *ModuleDeploymentController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 44 | zaplogger.FromContext(ctx).Info("Reconciling module deployment", "request", request) 45 | // This function is a placeholder for actual reconciliation logic. 46 | return reconcile.Result{}, nil 47 | } 48 | 49 | // NewModuleDeploymentController creates a new instance of the controller. 50 | func NewModuleDeploymentController(env string) (*ModuleDeploymentController, error) { 51 | return &ModuleDeploymentController{ 52 | env: env, 53 | updateToken: make(chan interface{}, 1), 54 | }, nil 55 | } 56 | 57 | // SetupWithManager sets up the controller with a manager. 58 | func (moduleDeploymentController *ModuleDeploymentController) SetupWithManager(ctx context.Context, mgr manager.Manager) (err error) { 59 | logger := zaplogger.FromContext(ctx) 60 | moduleDeploymentController.updateToken <- nil 61 | moduleDeploymentController.client = mgr.GetClient() 62 | moduleDeploymentController.cache = mgr.GetCache() 63 | 64 | logger.Info("Setting up module deployment controller") 65 | 66 | customController, err := controller.New("module-deployment-controller", mgr, controller.Options{ 67 | Reconciler: moduleDeploymentController, 68 | }) 69 | if err != nil { 70 | logger.Error(err, "unable to set up module-deployment controller") 71 | return err 72 | } 73 | 74 | envRequirement, _ := labels.NewRequirement(vkModel.LabelKeyOfEnv, selection.In, []string{moduleDeploymentController.env}) 75 | 76 | // first sync node cache 77 | nodeRequirement, _ := labels.NewRequirement(vkModel.LabelKeyOfComponent, selection.In, []string{vkModel.ComponentVNode}) 78 | vnodeSelector := labels.NewSelector().Add(*nodeRequirement, *envRequirement) 79 | 80 | // sync deployment cache 81 | deploymentRequirement, _ := labels.NewRequirement(vkModel.LabelKeyOfComponent, selection.In, []string{model.ComponentModuleDeployment}) 82 | deploymentSelector := labels.NewSelector().Add(*deploymentRequirement, *envRequirement) 83 | 84 | go func() { 85 | syncd := moduleDeploymentController.cache.WaitForCacheSync(ctx) 86 | if !syncd { 87 | logger.Error(nil, "failed to wait for cache sync") 88 | return 89 | } 90 | // init 91 | vnodeList := &corev1.NodeList{} 92 | err = moduleDeploymentController.cache.List(ctx, vnodeList, &client.ListOptions{ 93 | LabelSelector: vnodeSelector, 94 | }) 95 | if err != nil { 96 | err = errors.New("failed to list vnode") 97 | return 98 | } 99 | 100 | // init deployments 101 | depList := &appsv1.DeploymentList{} 102 | err = moduleDeploymentController.cache.List(ctx, depList, &client.ListOptions{ 103 | LabelSelector: deploymentSelector, 104 | }) 105 | 106 | if err != nil { 107 | err = errors.New("failed to list deployments") 108 | return 109 | } 110 | 111 | moduleDeploymentController.updateDeploymentReplicas(ctx, depList.Items) 112 | }() 113 | 114 | var vnodeEventHandler = handler.TypedFuncs[*corev1.Node, reconcile.Request]{ 115 | CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*corev1.Node], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 116 | moduleDeploymentController.vnodeCreateHandler(ctx, e.Object) 117 | }, 118 | UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*corev1.Node], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 119 | moduleDeploymentController.vnodeUpdateHandler(ctx, e.ObjectOld, e.ObjectNew) 120 | }, 121 | DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[*corev1.Node], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 122 | moduleDeploymentController.vnodeDeleteHandler(ctx, e.Object) 123 | }, 124 | GenericFunc: func(ctx context.Context, e event.TypedGenericEvent[*corev1.Node], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 125 | logger.WithValues("node_name", e.Object.Name).Info("Generic func call") 126 | }, 127 | } 128 | 129 | if err = customController.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}, vnodeEventHandler, &VNodePredicates{LabelSelector: vnodeSelector})); err != nil { 130 | logger.Error(err, "unable to watch nodes") 131 | return err 132 | } 133 | 134 | var deploymentEventHandler = handler.TypedFuncs[*appsv1.Deployment, reconcile.Request]{ 135 | CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*appsv1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 136 | moduleDeploymentController.deploymentAddHandler(ctx, e.Object) 137 | }, 138 | UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*appsv1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 139 | moduleDeploymentController.deploymentUpdateHandler(ctx, e.ObjectOld, e.ObjectNew) 140 | }, 141 | DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[*appsv1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 142 | }, 143 | GenericFunc: func(ctx context.Context, e event.TypedGenericEvent[*appsv1.Deployment], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { 144 | logger.WithValues("deployment_name", e.Object.Name).Info("Generic func call") 145 | }, 146 | } 147 | 148 | if err = customController.Watch(source.Kind(mgr.GetCache(), &appsv1.Deployment{}, &deploymentEventHandler, &ModuleDeploymentPredicates{LabelSelector: deploymentSelector})); err != nil { 149 | logger.Error(err, "unable to watch module Deployments") 150 | return err 151 | } 152 | 153 | logger.Info("module-deployment controller ready") 154 | return nil 155 | } 156 | 157 | // QueryContainerBaseline queries the baseline for a given container. 158 | func (moduleDeploymentController *ModuleDeploymentController) QueryContainerBaseline(ctx context.Context, req model.QueryBaselineRequest) []corev1.Container { 159 | logger := zaplogger.FromContext(ctx) 160 | labelMap := map[string]string{ 161 | // TODO: should add those label to deployments by module controller 162 | vkModel.LabelKeyOfEnv: moduleDeploymentController.env, 163 | } 164 | allDeploymentList := appsv1.DeploymentList{} 165 | err := moduleDeploymentController.cache.List(context.Background(), &allDeploymentList, &client.ListOptions{ 166 | LabelSelector: labels.SelectorFromSet(labelMap), 167 | }) 168 | if err != nil { 169 | logger.Error(err, "failed to list deployments") 170 | return []corev1.Container{} 171 | } 172 | 173 | // get relate containers of related deployments 174 | sort.Slice(allDeploymentList.Items, func(i, j int) bool { 175 | return allDeploymentList.Items[i].CreationTimestamp.UnixMilli() < allDeploymentList.Items[j].CreationTimestamp.UnixMilli() 176 | }) 177 | // record last version of biz model with same name 178 | containers := make([]corev1.Container, 0) 179 | containerNames := []string{} 180 | for _, deployment := range allDeploymentList.Items { 181 | clusterName := getClusterNameFromDeployment(&deployment) 182 | if clusterName != "" && clusterName == req.ClusterName { 183 | for _, container := range deployment.Spec.Template.Spec.Containers { 184 | containers = append(containers, container) 185 | containerNames = append(containerNames, utils.GetBizUniqueKey(&container)) 186 | } 187 | } 188 | } 189 | logger.Info(fmt.Sprintf("query base line got: %s", containerNames)) 190 | return containers 191 | } 192 | 193 | // vnodeCreateHandler handles the creation of a new vnode. 194 | func (moduleDeploymentController *ModuleDeploymentController) vnodeCreateHandler(ctx context.Context, vnode *corev1.Node) { 195 | relatedDeploymentsByNode := moduleDeploymentController.GetRelatedDeploymentsByNode(ctx, vnode) 196 | moduleDeploymentController.updateDeploymentReplicas(ctx, relatedDeploymentsByNode) 197 | } 198 | 199 | // vnodeUpdateHandler handles the update of an existing vnode. 200 | func (moduleDeploymentController *ModuleDeploymentController) vnodeUpdateHandler(ctx context.Context, _, vnode *corev1.Node) { 201 | relatedDeploymentsByNode := moduleDeploymentController.GetRelatedDeploymentsByNode(ctx, vnode) 202 | moduleDeploymentController.updateDeploymentReplicas(ctx, relatedDeploymentsByNode) 203 | } 204 | 205 | // vnodeDeleteHandler handles the deletion of a vnode. 206 | func (moduleDeploymentController *ModuleDeploymentController) vnodeDeleteHandler(ctx context.Context, vnode *corev1.Node) { 207 | vnodeCopy := vnode.DeepCopy() 208 | relatedDeploymentsByNode := moduleDeploymentController.GetRelatedDeploymentsByNode(ctx, vnodeCopy) 209 | moduleDeploymentController.updateDeploymentReplicas(ctx, relatedDeploymentsByNode) 210 | } 211 | 212 | func (moduleDeploymentController *ModuleDeploymentController) GetRelatedDeploymentsByNode(ctx context.Context, node *corev1.Node) []appsv1.Deployment { 213 | logger := zaplogger.FromContext(ctx) 214 | matchedDeployments := make([]appsv1.Deployment, 0) 215 | 216 | clusterName := getClusterNameFromNode(node) 217 | if clusterName == "" { 218 | logger.Info(fmt.Sprintf("failed to get cluster name of node %s", node.Name)) 219 | return matchedDeployments 220 | } 221 | 222 | deploymentList := appsv1.DeploymentList{} 223 | err := moduleDeploymentController.cache.List(ctx, &deploymentList, &client.ListOptions{ 224 | LabelSelector: labels.SelectorFromSet(map[string]string{ 225 | model.LabelKeyOfVPodDeploymentStrategy: string(model.VPodDeploymentStrategyPeer), 226 | }), 227 | }) 228 | 229 | if err != nil { 230 | logger.Error(err, "failed to list deployments") 231 | return matchedDeployments 232 | } 233 | 234 | for _, deployment := range deploymentList.Items { 235 | clusterNameFromDeployment := getClusterNameFromDeployment(&deployment) 236 | if clusterName == clusterNameFromDeployment && clusterNameFromDeployment != "" { 237 | matchedDeployments = append(matchedDeployments, deployment) 238 | } 239 | } 240 | 241 | return matchedDeployments 242 | } 243 | 244 | // deploymentAddHandler handles the addition of a new deployment. 245 | func (moduleDeploymentController *ModuleDeploymentController) deploymentAddHandler(ctx context.Context, dep *appsv1.Deployment) { 246 | if dep == nil { 247 | return 248 | } 249 | 250 | moduleDeploymentController.updateDeploymentReplicas(ctx, []appsv1.Deployment{*dep}) 251 | } 252 | 253 | // deploymentUpdateHandler handles the update of an existing deployment. 254 | func (moduleDeploymentController *ModuleDeploymentController) deploymentUpdateHandler(ctx context.Context, _, newDep *appsv1.Deployment) { 255 | if newDep == nil { 256 | return 257 | } 258 | 259 | moduleDeploymentController.updateDeploymentReplicas(ctx, []appsv1.Deployment{*newDep}) 260 | } 261 | 262 | // updateDeploymentReplicas updates the replicas of deployments based on node count. 263 | func (moduleDeploymentController *ModuleDeploymentController) updateDeploymentReplicas(ctx context.Context, deployments []appsv1.Deployment) { 264 | logger := zaplogger.FromContext(ctx) 265 | 266 | // TODO Implement this function. 267 | <-moduleDeploymentController.updateToken 268 | defer func() { 269 | moduleDeploymentController.updateToken <- nil 270 | }() 271 | 272 | enableModuleReplicasSameWithBase := utils.GetEnv("ENABLE_MODULE_REPLICAS_SYNC_WITH_BASE", "false") 273 | if enableModuleReplicasSameWithBase != "true" { 274 | return 275 | } 276 | 277 | for _, deployment := range deployments { 278 | if deployment.Labels[model.LabelKeyOfVPodDeploymentStrategy] != string(model.VPodDeploymentStrategyPeer) || deployment.Labels[model.LabelKeyOfSkipReplicasControl] == "true" { 279 | continue 280 | } 281 | 282 | clusterName := getClusterNameFromDeployment(&deployment) 283 | if clusterName == "" { 284 | logger.Info(fmt.Sprintf("failed to get cluster name of deployment %s, skip to update replicas", deployment.Name)) 285 | continue 286 | } 287 | 288 | sameClusterNodeCount, err := moduleDeploymentController.getReadyNodeCount(ctx, clusterName) 289 | if err != nil { 290 | logger.Error(err, fmt.Sprintf("failed to get nodes of cluster %s, skip to update relicas", clusterName)) 291 | continue 292 | } 293 | 294 | if int32(sameClusterNodeCount) != *deployment.Spec.Replicas { 295 | err := tracker.G().FuncTrack(deployment.Labels[vkModel.LabelKeyOfTraceID], vkModel.TrackSceneVPodDeploy, model.TrackEventVPodPeerDeploymentReplicaModify, deployment.Labels, func() (error, vkModel.ErrorCode) { 296 | return moduleDeploymentController.updateDeploymentReplicasOfKubernetes(ctx, sameClusterNodeCount, deployment) 297 | }) 298 | if err != nil { 299 | logger.Error(err, fmt.Sprintf("failed to update deployment replicas of %s", deployment.Name)) 300 | } 301 | } 302 | } 303 | } 304 | 305 | func getClusterNameFromDeployment(deployment *appsv1.Deployment) string { 306 | if clusterName, has := deployment.Labels[vkModel.LabelKeyOfBaseClusterName]; has { 307 | return clusterName 308 | } 309 | 310 | if deployment.Spec.Template.Spec.NodeSelector != nil { 311 | if clusterName, has := deployment.Spec.Template.Spec.NodeSelector[vkModel.LabelKeyOfBaseClusterName]; has { 312 | return clusterName 313 | } 314 | } 315 | 316 | if deployment.Spec.Template.Spec.Affinity != nil && 317 | deployment.Spec.Template.Spec.Affinity.NodeAffinity != nil && 318 | deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil && 319 | deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms != nil { 320 | for _, term := range deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { 321 | for _, expr := range term.MatchExpressions { 322 | if expr.Key == vkModel.LabelKeyOfBaseClusterName { 323 | return expr.Values[0] 324 | } 325 | } 326 | } 327 | } 328 | 329 | return "" 330 | } 331 | 332 | func getClusterNameFromNode(node *corev1.Node) string { 333 | if clusterName, has := node.Labels[vkModel.LabelKeyOfBaseClusterName]; has { 334 | return clusterName 335 | } 336 | 337 | return "" 338 | } 339 | 340 | func (moduleDeploymentController *ModuleDeploymentController) getReadyNodeCount(ctx context.Context, clusterName string) (int, error) { 341 | logger := zaplogger.FromContext(ctx) 342 | nodeList := &corev1.NodeList{} 343 | err := moduleDeploymentController.cache.List(ctx, nodeList, &client.ListOptions{ 344 | LabelSelector: labels.SelectorFromSet(map[string]string{vkModel.LabelKeyOfBaseClusterName: clusterName}), 345 | }) 346 | 347 | if err != nil { 348 | logger.Error(err, fmt.Sprintf("failed to list nodes of cluster %s", clusterName)) 349 | return 0, err 350 | } 351 | 352 | readyNodeCnt := 0 353 | for _, node := range nodeList.Items { 354 | if node.Status.Phase == corev1.NodeRunning { 355 | for _, cond := range node.Status.Conditions { 356 | if cond.Type == corev1.NodeReady { 357 | if cond.Status == corev1.ConditionTrue { 358 | readyNodeCnt++ 359 | break 360 | } 361 | } 362 | } 363 | } 364 | } 365 | 366 | return readyNodeCnt, nil 367 | } 368 | 369 | // updateDeploymentReplicasOfKubernetes updates the replicas of a deployment in Kubernetes. 370 | func (moduleDeploymentController *ModuleDeploymentController) updateDeploymentReplicasOfKubernetes(ctx context.Context, replicas int, deployment appsv1.Deployment) (error, vkModel.ErrorCode) { 371 | old := deployment.DeepCopy() 372 | patch := client.MergeFrom(old) 373 | 374 | deployment.Spec.Replicas = ptr.To[int32](int32(replicas)) 375 | err := moduleDeploymentController.client.Patch(ctx, &deployment, patch) 376 | if err != nil && !errors2.IsNotFound(err) { 377 | return err, model.CodeKubernetesOperationFailed 378 | } 379 | return nil, vkModel.CodeSuccess 380 | } 381 | -------------------------------------------------------------------------------- /controller/module_deployment_controller/module_deployment_controller_test.go: -------------------------------------------------------------------------------- 1 | package module_deployment_controller 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestDeploymentHandler(t *testing.T) { 9 | c := &ModuleDeploymentController{ 10 | updateToken: make(chan interface{}), 11 | } 12 | 13 | ctx := context.Background() 14 | c.deploymentAddHandler(ctx, nil) 15 | c.deploymentUpdateHandler(ctx, nil, nil) 16 | } 17 | -------------------------------------------------------------------------------- /controller/module_deployment_controller/predicates.go: -------------------------------------------------------------------------------- 1 | package module_deployment_controller 2 | 3 | import ( 4 | appsv1 "k8s.io/api/apps/v1" 5 | corev1 "k8s.io/api/core/v1" 6 | "k8s.io/apimachinery/pkg/labels" 7 | "sigs.k8s.io/controller-runtime/pkg/event" 8 | "sigs.k8s.io/controller-runtime/pkg/predicate" 9 | ) 10 | 11 | var _ predicate.TypedPredicate[*appsv1.Deployment] = &ModuleDeploymentPredicates{} 12 | var _ predicate.TypedPredicate[*corev1.Node] = &VNodePredicates{} 13 | 14 | type ModuleDeploymentPredicates struct { 15 | LabelSelector labels.Selector 16 | } 17 | 18 | type VNodePredicates struct { 19 | LabelSelector labels.Selector 20 | } 21 | 22 | func (V *VNodePredicates) Create(e event.TypedCreateEvent[*corev1.Node]) bool { 23 | return V.LabelSelector.Matches(labels.Set(e.Object.Labels)) 24 | } 25 | 26 | func (V *VNodePredicates) Delete(e event.TypedDeleteEvent[*corev1.Node]) bool { 27 | return V.LabelSelector.Matches(labels.Set(e.Object.Labels)) 28 | } 29 | 30 | func (V *VNodePredicates) Update(e event.TypedUpdateEvent[*corev1.Node]) bool { 31 | return V.LabelSelector.Matches(labels.Set(e.ObjectNew.Labels)) 32 | } 33 | 34 | func (V *VNodePredicates) Generic(e event.TypedGenericEvent[*corev1.Node]) bool { 35 | return false 36 | } 37 | 38 | func (m *ModuleDeploymentPredicates) Create(e event.TypedCreateEvent[*appsv1.Deployment]) bool { 39 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 40 | } 41 | 42 | func (m *ModuleDeploymentPredicates) Delete(e event.TypedDeleteEvent[*appsv1.Deployment]) bool { 43 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 44 | } 45 | 46 | func (m *ModuleDeploymentPredicates) Update(e event.TypedUpdateEvent[*appsv1.Deployment]) bool { 47 | return m.LabelSelector.Matches(labels.Set(e.ObjectNew.Labels)) 48 | } 49 | 50 | func (m *ModuleDeploymentPredicates) Generic(e event.TypedGenericEvent[*appsv1.Deployment]) bool { 51 | return m.LabelSelector.Matches(labels.Set(e.Object.Labels)) 52 | } 53 | -------------------------------------------------------------------------------- /controller/module_deployment_controller/predicates_test.go: -------------------------------------------------------------------------------- 1 | package module_deployment_controller 2 | 3 | import ( 4 | appsv1 "k8s.io/api/apps/v1" 5 | corev1 "k8s.io/api/core/v1" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/labels" 8 | "sigs.k8s.io/controller-runtime/pkg/event" 9 | "testing" 10 | ) 11 | 12 | func TestVNodePredicates(t *testing.T) { 13 | labelSelector := labels.SelectorFromSet(labels.Set{"env": "test"}) 14 | vNodePredicates := &VNodePredicates{LabelSelector: labelSelector} 15 | 16 | nodeWithLabels := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"env": "test"}}} 17 | nodeWithoutLabels := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"env": "prod"}}} 18 | 19 | // Test Create 20 | createEvent := event.TypedCreateEvent[*corev1.Node]{Object: nodeWithLabels} 21 | if !vNodePredicates.Create(createEvent) { 22 | t.Errorf("Expected Create to return true, got false") 23 | } 24 | 25 | createEvent = event.TypedCreateEvent[*corev1.Node]{Object: nodeWithoutLabels} 26 | if vNodePredicates.Create(createEvent) { 27 | t.Errorf("Expected Create to return false, got true") 28 | } 29 | 30 | // Test Delete 31 | deleteEvent := event.TypedDeleteEvent[*corev1.Node]{Object: nodeWithLabels} 32 | if !vNodePredicates.Delete(deleteEvent) { 33 | t.Errorf("Expected Delete to return true, got false") 34 | } 35 | 36 | deleteEvent = event.TypedDeleteEvent[*corev1.Node]{Object: nodeWithoutLabels} 37 | if vNodePredicates.Delete(deleteEvent) { 38 | t.Errorf("Expected Delete to return false, got true") 39 | } 40 | 41 | // Test Update 42 | updateEvent := event.TypedUpdateEvent[*corev1.Node]{ObjectNew: nodeWithLabels} 43 | if !vNodePredicates.Update(updateEvent) { 44 | t.Errorf("Expected Update to return true, got false") 45 | } 46 | 47 | updateEvent = event.TypedUpdateEvent[*corev1.Node]{ObjectNew: nodeWithoutLabels} 48 | if vNodePredicates.Update(updateEvent) { 49 | t.Errorf("Expected Update to return false, got true") 50 | } 51 | 52 | // Test Generic 53 | genericEvent := event.TypedGenericEvent[*corev1.Node]{Object: nodeWithLabels} 54 | if vNodePredicates.Generic(genericEvent) { 55 | t.Errorf("Expected Generic to return false, got true") 56 | } 57 | } 58 | 59 | func TestModuleDeploymentPredicates(t *testing.T) { 60 | labelSelector := labels.SelectorFromSet(labels.Set{"env": "test"}) 61 | moduleDeploymentPredicates := &ModuleDeploymentPredicates{LabelSelector: labelSelector} 62 | 63 | deploymentWithLabels := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"env": "test"}}} 64 | deploymentWithoutLabels := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"env": "prod"}}} 65 | 66 | // Test Create 67 | createEvent := event.TypedCreateEvent[*appsv1.Deployment]{Object: deploymentWithLabels} 68 | if !moduleDeploymentPredicates.Create(createEvent) { 69 | t.Errorf("Expected Create to return true, got false") 70 | } 71 | 72 | createEvent = event.TypedCreateEvent[*appsv1.Deployment]{Object: deploymentWithoutLabels} 73 | if moduleDeploymentPredicates.Create(createEvent) { 74 | t.Errorf("Expected Create to return false, got true") 75 | } 76 | 77 | // Test Delete 78 | deleteEvent := event.TypedDeleteEvent[*appsv1.Deployment]{Object: deploymentWithLabels} 79 | if !moduleDeploymentPredicates.Delete(deleteEvent) { 80 | t.Errorf("Expected Delete to return true, got false") 81 | } 82 | 83 | deleteEvent = event.TypedDeleteEvent[*appsv1.Deployment]{Object: deploymentWithoutLabels} 84 | if moduleDeploymentPredicates.Delete(deleteEvent) { 85 | t.Errorf("Expected Delete to return false, got true") 86 | } 87 | 88 | // Test Update 89 | updateEvent := event.TypedUpdateEvent[*appsv1.Deployment]{ObjectNew: deploymentWithLabels} 90 | if !moduleDeploymentPredicates.Update(updateEvent) { 91 | t.Errorf("Expected Update to return true, got false") 92 | } 93 | 94 | updateEvent = event.TypedUpdateEvent[*appsv1.Deployment]{ObjectNew: deploymentWithoutLabels} 95 | if moduleDeploymentPredicates.Update(updateEvent) { 96 | t.Errorf("Expected Update to return false, got true") 97 | } 98 | 99 | // Test Generic 100 | genericEvent := event.TypedGenericEvent[*appsv1.Deployment]{Object: deploymentWithLabels} 101 | if !moduleDeploymentPredicates.Generic(genericEvent) { 102 | t.Errorf("Expected Generic to return true, got false") 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /debug.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.22 as builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | 6 | WORKDIR /workspace/module-controller 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY cmd/ cmd/ 16 | COPY common/ common/ 17 | COPY controller/ controller/ 18 | COPY module_tunnels/ module_tunnels/ 19 | COPY report_server/ report_server/ 20 | 21 | # Build 22 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 23 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 24 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 25 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 26 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o module_controller cmd/module-controller/main.go 27 | 28 | # Use distroless as minimal base image to package the manager binary 29 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 30 | FROM golang:1.22.8 31 | WORKDIR / 32 | COPY config/ config/ 33 | COPY --from=builder /workspace/module-controller/module_controller . 34 | 35 | RUN git clone -b v1.24.1 https://github.com/go-delve/delve 36 | RUN cd delve && go mod download 37 | RUN cd delve && CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o dlv cmd/dlv/main.go 38 | RUN cp delve/dlv /usr/local/bin/dlv 39 | 40 | EXPOSE 9090 41 | EXPOSE 8080 42 | EXPOSE 7777 43 | EXPOSE 2345 44 | 45 | #ENTRYPOINT ["dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./module_controller"] 46 | CMD ["/bin/sh", "-c", "tail -f /dev/null"] -------------------------------------------------------------------------------- /example/quick-start/base.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: base 5 | labels: 6 | app: base 7 | spec: 8 | containers: 9 | - name: base 10 | image: serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/test/base-web:1.3.3 # 已经打包好的镜像，源码在 https://github.com/koupleless/samples/blob/main/springboot-samples/web/tomcat/Dockerfile 11 | imagePullPolicy: Always 12 | ports: 13 | - name: base 14 | containerPort: 8080 15 | - name: arklet 16 | containerPort: 1238 17 | env: 18 | - name: MODULE_CONTROLLER_ADDRESS 19 | value: { YOUR_MODULE_CONTROLLER_IP } -------------------------------------------------------------------------------- /example/quick-start/module-controller-test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: module-controller 5 | labels: 6 | app: module-controller 7 | spec: 8 | serviceAccountName: virtual-kubelet # 上一步中配置好的 Service Account 9 | containers: 10 | - name: module-controller 11 | image: serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/test/module-controller-v2:v2.1.2 # 已经打包好的镜像 12 | imagePullPolicy: Always 13 | resources: 14 | limits: 15 | cpu: "1000m" 16 | memory: "400Mi" 17 | ports: 18 | - name: httptunnel 19 | containerPort: 7777 20 | - name: debug 21 | containerPort: 2345 22 | env: 23 | - name: ENABLE_HTTP_TUNNEL 24 | value: "true" -------------------------------------------------------------------------------- /example/quick-start/module-controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: module-controller 5 | labels: 6 | app: module-controller 7 | spec: 8 | serviceAccountName: virtual-kubelet # 上一步中配置好的 Service Account 9 | containers: 10 | - name: module-controller 11 | image: serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/release/module-controller-v2:v2.1.2 # 已经打包好的镜像 12 | imagePullPolicy: Always 13 | resources: 14 | limits: 15 | cpu: "1000m" 16 | memory: "400Mi" 17 | ports: 18 | - name: httptunnel 19 | containerPort: 7777 20 | env: 21 | - name: ENABLE_HTTP_TUNNEL 22 | value: "true" -------------------------------------------------------------------------------- /example/quick-start/module.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: biz1 5 | labels: 6 | virtual-kubelet.koupleless.io/component: module-deployment 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | module: biz1 12 | template: 13 | metadata: 14 | labels: 15 | module: biz1 16 | virtual-kubelet.koupleless.io/component: module 17 | spec: 18 | containers: 19 | - name: biz1 20 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz1-web-single-host-0.0.1-SNAPSHOT-ark-biz.jar 21 | env: 22 | - name: BIZ_VERSION 23 | value: 0.0.1-SNAPSHOT 24 | affinity: 25 | nodeAffinity: 26 | requiredDuringSchedulingIgnoredDuringExecution: 27 | nodeSelectorTerms: 28 | - matchExpressions: 29 | - key: base.koupleless.io/name 30 | operator: In 31 | values: 32 | - base 33 | - key: base.koupleless.io/version 34 | operator: In 35 | values: 36 | - 1.0.0 37 | - key: base.koupleless.io/cluster-name 38 | operator: In 39 | values: 40 | - default 41 | tolerations: 42 | - key: "schedule.koupleless.io/virtual-node" 43 | operator: "Equal" 44 | value: "True" 45 | effect: "NoExecute" 46 | - key: "schedule.koupleless.io/node-env" 47 | operator: "Equal" 48 | value: "dev" 49 | effect: "NoExecute" -------------------------------------------------------------------------------- /example/self-test/base.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: base 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: base 10 | template: 11 | metadata: 12 | labels: 13 | app: base 14 | spec: 15 | containers: 16 | - name: base 17 | image: serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/test/test-base:1.3.4 # 已经打包好的镜像 18 | imagePullPolicy: Always 19 | ports: 20 | - name: base 21 | containerPort: 8080 22 | - name: arklet 23 | containerPort: 1238 -------------------------------------------------------------------------------- /example/self-test/module.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: module-biz{bizIndex} 5 | namespace: module 6 | labels: 7 | virtual-kubelet.koupleless.io/component: module-deployment 8 | spec: 9 | replicas: {baseNum} 10 | selector: 11 | matchLabels: 12 | module: biz{bizIndex} 13 | template: 14 | metadata: 15 | namespace: module 16 | labels: 17 | module: biz{bizIndex} 18 | virtual-kubelet.koupleless.io/component: module 19 | spec: 20 | containers: 21 | - name: biz{bizIndex} 22 | image: https://koupleless.oss-cn-shanghai.aliyuncs.com/module-packages/test_modules/biz{bizIndex}-{bizVersion}-ark-biz.jar 23 | env: 24 | - name: BIZ_VERSION 25 | value: {bizVersion} 26 | affinity: 27 | nodeAffinity: 28 | requiredDuringSchedulingIgnoredDuringExecution: 29 | nodeSelectorTerms: 30 | - matchExpressions: 31 | - key: base.koupleless.io/name 32 | operator: In 33 | values: 34 | - base 35 | - key: base.koupleless.io/version 36 | operator: In 37 | values: 38 | - 1.0.0 39 | - key: base.koupleless.io/cluster-name 40 | operator: In 41 | values: 42 | - default 43 | tolerations: 44 | - key: "schedule.koupleless.io/virtual-node" 45 | operator: "Equal" 46 | value: "True" 47 | effect: "NoExecute" 48 | - key: "schedule.koupleless.io/node-env" 49 | operator: "Equal" 50 | value: "test" 51 | effect: "NoExecute" -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/koupleless/module_controller 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.4 6 | 7 | require ( 8 | github.com/eclipse/paho.mqtt.golang v1.5.0 9 | github.com/go-logr/logr v1.4.2 10 | github.com/go-resty/resty/v2 v2.11.0 11 | github.com/google/uuid v1.6.0 12 | github.com/koupleless/arkctl v0.2.4-0.20250106035535-5ed5cb871995 13 | github.com/koupleless/virtual-kubelet v0.3.8 14 | github.com/onsi/ginkgo/v2 v2.19.0 15 | github.com/onsi/gomega v1.33.1 16 | github.com/sirupsen/logrus v1.9.3 17 | github.com/stretchr/testify v1.9.0 18 | github.com/virtual-kubelet/virtual-kubelet v1.11.0 19 | github.com/wind-c/comqtt/v2 v2.6.0 20 | k8s.io/api v0.31.0 21 | k8s.io/apimachinery v0.31.0 22 | k8s.io/client-go v0.31.0 23 | k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 24 | sigs.k8s.io/controller-runtime v0.19.0 25 | sigs.k8s.io/yaml v1.4.0 26 | ) 27 | 28 | require ( 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 31 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 32 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 33 | github.com/evanphx/json-patch/v5 v5.9.0 // indirect 34 | github.com/fsnotify/fsnotify v1.7.0 // indirect 35 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 36 | github.com/go-logr/zapr v1.3.0 // indirect 37 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 38 | github.com/go-openapi/jsonreference v0.20.2 // indirect 39 | github.com/go-openapi/swag v0.22.4 // indirect 40 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 41 | github.com/gogo/protobuf v1.3.2 // indirect 42 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 43 | github.com/golang/protobuf v1.5.4 // indirect 44 | github.com/google/gnostic-models v0.6.8 // indirect 45 | github.com/google/go-cmp v0.6.0 // indirect 46 | github.com/google/gofuzz v1.2.0 // indirect 47 | github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect 48 | github.com/gorilla/websocket v1.5.3 // indirect 49 | github.com/imdario/mergo v0.3.12 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/json-iterator/go v1.1.12 // indirect 52 | github.com/mailru/easyjson v0.7.7 // indirect 53 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 54 | github.com/modern-go/reflect2 v1.0.2 // indirect 55 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 56 | github.com/pkg/errors v0.9.1 // indirect 57 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 58 | github.com/prometheus/client_golang v1.19.1 // indirect 59 | github.com/prometheus/client_model v0.6.1 // indirect 60 | github.com/prometheus/common v0.55.0 // indirect 61 | github.com/prometheus/procfs v0.15.1 // indirect 62 | github.com/rs/xid v1.5.0 // indirect 63 | github.com/spf13/pflag v1.0.5 // indirect 64 | github.com/x448/float16 v0.8.4 // indirect 65 | go.opencensus.io v0.24.0 // indirect 66 | go.uber.org/multierr v1.11.0 // indirect 67 | go.uber.org/zap v1.27.0 // indirect 68 | golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect 69 | golang.org/x/net v0.27.0 // indirect 70 | golang.org/x/oauth2 v0.21.0 // indirect 71 | golang.org/x/sync v0.7.0 // indirect 72 | golang.org/x/sys v0.22.0 // indirect 73 | golang.org/x/term v0.22.0 // indirect 74 | golang.org/x/text v0.16.0 // indirect 75 | golang.org/x/time v0.5.0 // indirect 76 | golang.org/x/tools v0.23.0 // indirect 77 | gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect 78 | google.golang.org/protobuf v1.34.2 // indirect 79 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 80 | gopkg.in/inf.v0 v0.9.1 // indirect 81 | gopkg.in/yaml.v2 v2.4.0 // indirect 82 | gopkg.in/yaml.v3 v3.0.1 // indirect 83 | k8s.io/apiextensions-apiserver v0.31.0 // indirect 84 | k8s.io/klog/v2 v2.130.1 // indirect 85 | k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect 86 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 87 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 88 | ) 89 | 90 | //replace github.com/koupleless/virtual-kubelet => /Users/youji.zzl/Documents/workspace/koupleless/virtual-kubelet 91 | -------------------------------------------------------------------------------- /module_tunnels/koupleless_http_tunnel/ark_service/ark_service.go: -------------------------------------------------------------------------------- 1 | package ark_service 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/koupleless/arkctl/v1/service/ark" 9 | "github.com/koupleless/module_controller/common/zaplogger" 10 | "net/http" 11 | 12 | "github.com/go-resty/resty/v2" 13 | ) 14 | 15 | type Service struct { 16 | client *resty.Client 17 | } 18 | 19 | func NewService() *Service { 20 | return &Service{client: resty.New()} 21 | } 22 | 23 | func (h *Service) InstallBiz(ctx context.Context, req InstallBizRequest, baseIP string, arkletPort int) (response ArkResponse[ark.ArkResponseData], err error) { 24 | logger := zaplogger.FromContext(ctx) 25 | 26 | resp, err := h.client.R(). 27 | SetContext(ctx). 28 | SetBody(req). 29 | Post(fmt.Sprintf("http://%s:%d/installBiz", baseIP, arkletPort)) 30 | 31 | if err != nil { 32 | logger.Error(err, "installBiz request failed") 33 | return 34 | } 35 | 36 | if resp == nil { 37 | err = errors.New("received nil response from the server") 38 | logger.Error(err, "installBiz request failed") 39 | return 40 | } 41 | 42 | if resp.StatusCode() != http.StatusOK { 43 | err = errors.New(fmt.Sprintf("response status: %d", resp.StatusCode())) 44 | logger.Error(err, "installBiz request failed") 45 | return 46 | } 47 | 48 | if err = json.Unmarshal(resp.Body(), &response); err != nil { 49 | logger.Error(err, fmt.Sprintf("Unmarshal InstallBiz response: %s", resp.Body())) 50 | return 51 | } 52 | 53 | return response, nil 54 | } 55 | 56 | func (h *Service) UninstallBiz(ctx context.Context, req UninstallBizRequest, baseIP string, arkletPort int) (response ArkResponse[ark.ArkResponseData], err error) { 57 | logger := zaplogger.FromContext(ctx) 58 | 59 | resp, err := h.client.R(). 60 | SetContext(ctx). 61 | SetBody(req). 62 | Post(fmt.Sprintf("http://%s:%d/uninstallBiz", baseIP, arkletPort)) 63 | 64 | if err != nil { 65 | logger.Error(err, "uninstall request failed") 66 | return 67 | } 68 | 69 | if resp == nil { 70 | err = errors.New("received nil response from the server") 71 | logger.Error(err, "uninstall request failed") 72 | return 73 | } 74 | 75 | if resp.StatusCode() != http.StatusOK { 76 | err = errors.New(fmt.Sprintf("response status: %d", resp.StatusCode())) 77 | logger.Error(err, "uninstall request failed") 78 | return 79 | } 80 | 81 | if err = json.Unmarshal(resp.Body(), &response); err != nil { 82 | logger.Error(err, fmt.Sprintf("Unmarshal UnInstallBiz response: %s", resp.Body())) 83 | return 84 | } 85 | 86 | return response, nil 87 | } 88 | 89 | func (h *Service) QueryAllBiz(ctx context.Context, baseIP string, port int) (response QueryAllBizResponse, err error) { 90 | logger := zaplogger.FromContext(ctx) 91 | 92 | resp, err := h.client.R(). 93 | SetContext(ctx). 94 | SetBody(`{}`). 95 | Post(fmt.Sprintf("http://%s:%d/queryAllBiz", baseIP, port)) 96 | 97 | if err != nil { 98 | logger.Error(err, "queryAllBiz request failed") 99 | return 100 | } 101 | 102 | if resp == nil { 103 | err = errors.New("received nil response from the server") 104 | logger.Error(err, "queryAllBiz request failed") 105 | return 106 | } 107 | 108 | if resp.StatusCode() != http.StatusOK { 109 | err = errors.New(fmt.Sprintf("response status: %d", resp.StatusCode())) 110 | logger.Error(err, "queryAllBiz request failed") 111 | return 112 | } 113 | 114 | if err = json.Unmarshal(resp.Body(), &response); err != nil { 115 | logger.Error(err, fmt.Sprintf("Unmarshal QueryAllBiz response: %s", resp.Body())) 116 | return 117 | } 118 | 119 | return 120 | } 121 | 122 | func (h *Service) Health(ctx context.Context, baseIP string, arkletPort int) (response HealthResponse, err error) { 123 | logger := zaplogger.FromContext(ctx) 124 | 125 | resp, err := h.client.R(). 126 | SetContext(ctx). 127 | SetBody(`{}`). 128 | Post(fmt.Sprintf("http://%s:%d/health", baseIP, arkletPort)) 129 | 130 | if err != nil { 131 | logger.Error(err, "health request failed") 132 | return 133 | } 134 | 135 | if resp == nil { 136 | err = errors.New("received nil response from the server") 137 | logger.Error(err, "health request failed") 138 | return 139 | } 140 | 141 | if resp.StatusCode() != http.StatusOK { 142 | err = errors.New(fmt.Sprintf("response status: %d", resp.StatusCode())) 143 | logger.Error(err, "health request failed") 144 | return 145 | } 146 | 147 | if err = json.Unmarshal(resp.Body(), &response); err != nil { 148 | logger.Error(err, fmt.Sprintf("Unmarshal Health response: %s", resp.Body())) 149 | return 150 | } 151 | 152 | return 153 | } 154 | -------------------------------------------------------------------------------- /module_tunnels/koupleless_http_tunnel/ark_service/model.go: -------------------------------------------------------------------------------- 1 | package ark_service 2 | 3 | import "github.com/koupleless/arkctl/v1/service/ark" 4 | 5 | // InstallBizRequest represents a request to install a business service 6 | type InstallBizRequest struct { 7 | ark.BizModel `json:",inline"` 8 | } 9 | 10 | // UninstallBizRequest represents a request to uninstall a business service 11 | type UninstallBizRequest struct { 12 | ark.BizModel `json:",inline"` 13 | } 14 | 15 | // ArkResponse is a generic response structure for Ark service operations 16 | type ArkResponse[T any] struct { 17 | // Code is the response code indicating the outcome of the operation 18 | Code string `json:"code"` 19 | 20 | // Data is the response data, which can vary depending on the operation 21 | Data T `json:"data"` 22 | 23 | // Message is the error message in case of an error 24 | Message string `json:"message"` 25 | 26 | // ErrorStackTrace is the error stack trace in case of an error 27 | ErrorStackTrace string `json:"errorStackTrace"` 28 | 29 | // BaseIdentity is a unique identifier for the base service 30 | BaseIdentity string `json:"baseIdentity"` 31 | } 32 | 33 | // QueryAllBizResponse represents the response for querying all business services 34 | type QueryAllBizResponse struct { 35 | ark.GenericArkResponseBase[[]ark.ArkBizInfo] 36 | } 37 | 38 | // HealthResponse represents the response for health checks 39 | type HealthResponse struct { 40 | ark.GenericArkResponseBase[ark.HealthInfo] 41 | } 42 | -------------------------------------------------------------------------------- /module_tunnels/koupleless_http_tunnel/http_tunnel.go: -------------------------------------------------------------------------------- 1 | package koupleless_http_tunnel 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/koupleless/module_controller/common/zaplogger" 8 | "github.com/koupleless/module_controller/controller/module_deployment_controller" 9 | utils2 "github.com/koupleless/virtual-kubelet/common/utils" 10 | "net/http" 11 | "sync" 12 | "time" 13 | 14 | "errors" 15 | 16 | "github.com/koupleless/arkctl/v1/service/ark" 17 | "github.com/koupleless/module_controller/common/model" 18 | "github.com/koupleless/module_controller/common/utils" 19 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel/ark_service" 20 | vkModel "github.com/koupleless/virtual-kubelet/model" 21 | "github.com/koupleless/virtual-kubelet/tunnel" 22 | corev1 "k8s.io/api/core/v1" 23 | "sigs.k8s.io/controller-runtime/pkg/client" 24 | ) 25 | 26 | var _ tunnel.Tunnel = &HttpTunnel{} 27 | 28 | type HttpTunnel struct { 29 | ctx context.Context 30 | sync.Mutex 31 | 32 | arkService *ark_service.Service 33 | 34 | kubeClient client.Client 35 | env string 36 | port int 37 | 38 | ready bool 39 | 40 | onBaseDiscovered tunnel.OnBaseDiscovered 41 | onHealthDataArrived tunnel.OnBaseStatusArrived 42 | onQueryAllBizDataArrived tunnel.OnAllBizStatusArrived 43 | onOneBizDataArrived tunnel.OnSingleBizStatusArrived 44 | 45 | onlineNode map[string]bool 46 | 47 | nodeIdToBaseStatusMap map[string]model.BaseStatus 48 | 49 | queryAllBizLock sync.Mutex 50 | queryAllBizDataOutdated bool 51 | 52 | moduleDeploymentController *module_deployment_controller.ModuleDeploymentController 53 | } 54 | 55 | func NewHttpTunnel(ctx context.Context, env string, kubeClient client.Client, moduleDeploymentController *module_deployment_controller.ModuleDeploymentController, port int) HttpTunnel { 56 | return HttpTunnel{ 57 | ctx: ctx, 58 | env: env, 59 | kubeClient: kubeClient, 60 | moduleDeploymentController: moduleDeploymentController, 61 | port: port, 62 | } 63 | } 64 | 65 | // Ready returns the current status of the tunnel 66 | func (h *HttpTunnel) Ready() bool { 67 | return h.ready 68 | } 69 | 70 | // GetBizUniqueKey returns a unique key for the container 71 | func (h *HttpTunnel) GetBizUniqueKey(container *corev1.Container) string { 72 | return utils.GetBizIdentity(container.Name, utils.GetBizVersionFromContainer(container)) 73 | } 74 | 75 | // RegisterNode is called when a new node starts 76 | func (h *HttpTunnel) RegisterNode(initData vkModel.NodeInfo) error { 77 | h.Lock() 78 | defer h.Unlock() 79 | 80 | // check base network info, if not exist, extract from initData 81 | nodeID := utils2.ExtractNodeIDFromNodeName(initData.Metadata.Name) 82 | _, has := h.nodeIdToBaseStatusMap[nodeID] 83 | if !has { 84 | h.nodeIdToBaseStatusMap[nodeID] = utils.ConvertBaseStatusFromNodeInfo(initData) 85 | } 86 | return nil 87 | } 88 | 89 | // UnRegisterNode is called when a node stops 90 | func (h *HttpTunnel) UnRegisterNode(nodeName string) { 91 | h.Lock() 92 | defer h.Unlock() 93 | nodeID := utils2.ExtractNodeIDFromNodeName(nodeName) 94 | delete(h.nodeIdToBaseStatusMap, nodeID) 95 | } 96 | 97 | // OnNodeNotReady is called when a node is not ready 98 | func (h *HttpTunnel) OnNodeNotReady(nodeName string) { 99 | utils.OnBaseUnreachable(h.ctx, nodeName, h.kubeClient) 100 | } 101 | 102 | // Key returns the key of the tunnel 103 | func (h *HttpTunnel) Key() string { 104 | return "http_tunnel_provider" 105 | } 106 | 107 | // RegisterCallback registers the callback functions for the tunnel 108 | func (h *HttpTunnel) RegisterCallback(onBaseDiscovered tunnel.OnBaseDiscovered, onHealthDataArrived tunnel.OnBaseStatusArrived, onQueryAllBizDataArrived tunnel.OnAllBizStatusArrived, onOneBizDataArrived tunnel.OnSingleBizStatusArrived) { 109 | h.onBaseDiscovered = onBaseDiscovered 110 | 111 | h.onHealthDataArrived = onHealthDataArrived 112 | 113 | h.onQueryAllBizDataArrived = onQueryAllBizDataArrived 114 | 115 | h.onOneBizDataArrived = onOneBizDataArrived 116 | } 117 | 118 | // Start starts the tunnel 119 | func (h *HttpTunnel) Start(clientID, env string) (err error) { 120 | h.onlineNode = make(map[string]bool) 121 | h.nodeIdToBaseStatusMap = make(map[string]model.BaseStatus) 122 | h.env = env 123 | 124 | h.arkService = ark_service.NewService() 125 | 126 | h.ready = true 127 | 128 | // add base discovery 129 | go h.startBaseDiscovery(h.ctx) 130 | 131 | return 132 | } 133 | 134 | // startBaseDiscovery starts the base discovery server 135 | func (h *HttpTunnel) startBaseDiscovery(ctx context.Context) { 136 | logger := zaplogger.FromContext(ctx) 137 | // start a simple http server to handle base discovery, exit when ctx done 138 | mux := http.NewServeMux() 139 | server := &http.Server{ 140 | Addr: fmt.Sprintf(":%d", h.port), 141 | Handler: mux, 142 | } 143 | 144 | // handle heartbeat post request 145 | mux.HandleFunc("/heartbeat", func(w http.ResponseWriter, r *http.Request) { 146 | if r.Method != http.MethodPost { 147 | w.WriteHeader(http.StatusMethodNotAllowed) 148 | return 149 | } 150 | 151 | heartbeatData := model.BaseStatus{} 152 | err := json.NewDecoder(r.Body).Decode(&heartbeatData) 153 | if err != nil { 154 | logger.Error(err, "failed to unmarshal heartbeat data") 155 | w.WriteHeader(http.StatusBadRequest) 156 | w.Write([]byte(err.Error())) 157 | return 158 | } 159 | h.Lock() 160 | h.nodeIdToBaseStatusMap[heartbeatData.BaseMetadata.Identity] = heartbeatData 161 | h.Unlock() 162 | h.onBaseDiscovered(utils.ConvertBaseStatusToNodeInfo(heartbeatData, h.env)) 163 | 164 | w.WriteHeader(http.StatusOK) 165 | w.Write([]byte("SUCCESS")) 166 | }) 167 | 168 | mux.HandleFunc("/queryBaseline", func(w http.ResponseWriter, r *http.Request) { 169 | if r.Method != http.MethodPost { 170 | w.WriteHeader(http.StatusMethodNotAllowed) 171 | return 172 | } 173 | 174 | baseMetadata := model.BaseMetadata{} 175 | err := json.NewDecoder(r.Body).Decode(&baseMetadata) 176 | if err != nil { 177 | logger.Error(err, "failed to unmarshal baseMetadata data") 178 | w.WriteHeader(http.StatusBadRequest) 179 | w.Write([]byte(err.Error())) 180 | return 181 | } 182 | 183 | baselineBizs := make([]ark.BizModel, 0) 184 | baseline := h.moduleDeploymentController.QueryContainerBaseline(h.ctx, utils.ConvertBaseMetadataToBaselineQuery(baseMetadata)) 185 | for _, container := range baseline { 186 | baselineBizs = append(baselineBizs, utils.TranslateCoreV1ContainerToBizModel(&container)) 187 | } 188 | 189 | jsonData, _ := json.Marshal(baselineBizs) 190 | 191 | w.Header().Set("Content-Type", "application/json") 192 | w.WriteHeader(http.StatusOK) 193 | w.Write(jsonData) 194 | }) 195 | 196 | go func() { 197 | if err := server.ListenAndServe(); err != nil { 198 | logger.Error(err, "error starting http base discovery server") 199 | } 200 | }() 201 | 202 | logger.Info(fmt.Sprintf("http base discovery server started, listening on port %d", h.port)) 203 | 204 | defer func() { 205 | shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 206 | defer cancel() 207 | if err := server.Shutdown(shutdownCtx); err != nil { 208 | logger.Error(err, "error shutting down http server") 209 | } 210 | }() 211 | <-ctx.Done() 212 | } 213 | 214 | // healthMsgCallback is the callback function for health messages 215 | func (h *HttpTunnel) healthMsgCallback(nodeID string, data ark_service.HealthResponse) { 216 | if data.Code != "SUCCESS" { 217 | return 218 | } 219 | if h.onHealthDataArrived != nil { 220 | h.onHealthDataArrived(utils2.FormatNodeName(nodeID, h.env), utils.ConvertHealthDataToNodeStatus(data.Data.HealthData)) 221 | } 222 | } 223 | 224 | // allBizMsgCallback is the callback function for all business messages 225 | func (h *HttpTunnel) allBizMsgCallback(nodeID string, data ark_service.QueryAllBizResponse) { 226 | if data.Code != "SUCCESS" { 227 | return 228 | } 229 | if h.onQueryAllBizDataArrived != nil { 230 | h.onQueryAllBizDataArrived(utils2.FormatNodeName(nodeID, h.env), utils.TranslateBizInfosToContainerStatuses(data.GenericArkResponseBase.Data, time.Now().UnixMilli())) 231 | } 232 | } 233 | 234 | // bizOperationResponseCallback is the callback function for business operation responses 235 | func (httpTunnel *HttpTunnel) bizOperationResponseCallback(nodeID string, data model.BizOperationResponse) { 236 | logger := zaplogger.FromContext(httpTunnel.ctx) 237 | nodeName := utils2.FormatNodeName(nodeID, httpTunnel.env) 238 | if data.Response.Code == "SUCCESS" { 239 | if data.Command == model.CommandInstallBiz { 240 | logger.Info("install biz success: ", data.BizName, data.BizVersion) 241 | httpTunnel.onOneBizDataArrived(nodeName, vkModel.BizStatusData{ 242 | Key: utils.GetBizIdentity(data.BizName, data.BizVersion), 243 | Name: data.BizName, 244 | State: string(vkModel.BizStateActivated), 245 | ChangeTime: time.Now(), 246 | Reason: fmt.Sprintf("%s:%s %s succeed", data.BizName, data.BizVersion, data.Command), 247 | Message: data.Response.Data.Message, 248 | }) 249 | return 250 | } else if data.Command == model.CommandUnInstallBiz { 251 | logger.Info("uninstall biz success: ", data.BizName, data.BizVersion) 252 | return 253 | } else { 254 | logger.Error(nil, fmt.Sprintf("biz operation failed: %s:%s %s\n%s\n%s\n%s", data.BizName, data.BizVersion, data.Command, data.Response.Message, data.Response.ErrorStackTrace, data.Response.Data.Message)) 255 | } 256 | } 257 | httpTunnel.onOneBizDataArrived(nodeName, vkModel.BizStatusData{ 258 | Key: utils.GetBizIdentity(data.BizName, data.BizVersion), 259 | Name: data.BizName, 260 | // fille PodKey when using 261 | // PodKey: vkModel.PodKeyAll, 262 | State: string(vkModel.BizStateBroken), 263 | ChangeTime: time.Now(), 264 | Reason: data.Response.Code, 265 | Message: data.Response.Message, 266 | }) 267 | } 268 | 269 | // FetchHealthData fetches health data from the node 270 | func (h *HttpTunnel) FetchHealthData(nodeName string) error { 271 | h.Lock() 272 | nodeID := utils2.ExtractNodeIDFromNodeName(nodeName) 273 | baseStatus, ok := h.nodeIdToBaseStatusMap[nodeID] 274 | h.Unlock() 275 | if !ok { 276 | return errors.New("network info not found") 277 | } 278 | 279 | healthData, err := h.arkService.Health(h.ctx, baseStatus.LocalIP, baseStatus.Port) 280 | 281 | if err != nil { 282 | return err 283 | } 284 | 285 | h.healthMsgCallback(nodeID, healthData) 286 | 287 | return nil 288 | } 289 | 290 | // QueryAllBizStatusData queries all container status data from the node 291 | func (h *HttpTunnel) QueryAllBizStatusData(nodeName string) error { 292 | // add a signal to check 293 | success := h.queryAllBizLock.TryLock() 294 | if !success { 295 | // a query is processing 296 | h.queryAllBizDataOutdated = true 297 | return nil 298 | } 299 | h.queryAllBizDataOutdated = false 300 | defer func() { 301 | h.queryAllBizLock.Unlock() 302 | if h.queryAllBizDataOutdated { 303 | go h.QueryAllBizStatusData(nodeName) 304 | } 305 | }() 306 | 307 | h.Lock() 308 | nodeID := utils2.ExtractNodeIDFromNodeName(nodeName) 309 | baseStatus, ok := h.nodeIdToBaseStatusMap[nodeID] 310 | h.Unlock() 311 | if !ok { 312 | return errors.New("network info not found") 313 | } 314 | 315 | allBizData, err := h.arkService.QueryAllBiz(h.ctx, baseStatus.LocalIP, baseStatus.Port) 316 | 317 | if err != nil { 318 | return err 319 | } 320 | 321 | h.allBizMsgCallback(nodeID, allBizData) 322 | 323 | return nil 324 | } 325 | 326 | // StartBiz starts a container on the node 327 | func (h *HttpTunnel) StartBiz(nodeName, _ string, container *corev1.Container) error { 328 | 329 | nodeID := utils2.ExtractNodeIDFromNodeName(nodeName) 330 | h.Lock() 331 | baseStatus, ok := h.nodeIdToBaseStatusMap[nodeID] 332 | h.Unlock() 333 | if !ok { 334 | return errors.New("network info not found") 335 | } 336 | 337 | bizModel := utils.TranslateCoreV1ContainerToBizModel(container) 338 | logger := zaplogger.FromContext(h.ctx).WithValues("bizName", bizModel.BizName, "bizVersion", bizModel.BizVersion) 339 | logger.Info("InstallModule") 340 | 341 | // install current version 342 | bizOperationResponse := model.BizOperationResponse{ 343 | Command: model.CommandInstallBiz, 344 | BizName: bizModel.BizName, 345 | BizVersion: bizModel.BizVersion, 346 | } 347 | 348 | response, err := h.arkService.InstallBiz(h.ctx, ark_service.InstallBizRequest{ 349 | BizModel: bizModel, 350 | }, baseStatus.LocalIP, baseStatus.Port) 351 | 352 | bizOperationResponse.Response = response 353 | 354 | h.bizOperationResponseCallback(nodeID, bizOperationResponse) 355 | 356 | return err 357 | } 358 | 359 | // StopBiz shuts down a container on the node 360 | func (h *HttpTunnel) StopBiz(nodeName, _ string, container *corev1.Container) error { 361 | nodeID := utils2.ExtractNodeIDFromNodeName(nodeName) 362 | h.Lock() 363 | baseStatus, ok := h.nodeIdToBaseStatusMap[nodeID] 364 | h.Unlock() 365 | if !ok { 366 | return errors.New("network info not found") 367 | } 368 | 369 | bizModel := utils.TranslateCoreV1ContainerToBizModel(container) 370 | logger := zaplogger.FromContext(h.ctx).WithValues("bizName", bizModel.BizName, "bizVersion", bizModel.BizVersion) 371 | logger.Info("UninstallModule") 372 | 373 | bizOperationResponse := model.BizOperationResponse{ 374 | Command: model.CommandUnInstallBiz, 375 | BizName: bizModel.BizName, 376 | BizVersion: bizModel.BizVersion, 377 | } 378 | 379 | response, err := h.arkService.UninstallBiz(h.ctx, ark_service.UninstallBizRequest{ 380 | BizModel: bizModel, 381 | }, baseStatus.LocalIP, baseStatus.Port) 382 | 383 | bizOperationResponse.Response = response 384 | 385 | h.bizOperationResponseCallback(nodeID, bizOperationResponse) 386 | 387 | return err 388 | } 389 | -------------------------------------------------------------------------------- /module_tunnels/koupleless_mqtt_tunnel/config.go: -------------------------------------------------------------------------------- 1 | package koupleless_mqtt_tunnel 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | 7 | "github.com/koupleless/virtual-kubelet/common/utils" 8 | ) 9 | 10 | // Constants for default MQTT configurations 11 | const ( 12 | DefaultMQTTBroker = "test-broker" 13 | DefaultMQTTUsername = "test-username" 14 | DefaultMQTTClientPrefix = "koupleless" 15 | DefaultMQTTPort = "1883" 16 | ) 17 | 18 | // MqttConfig holds the configuration for an MQTT client 19 | type MqttConfig struct { 20 | MqttBroker string 21 | MqttPort int 22 | MqttUsername string 23 | MqttPassword string 24 | MqttClientPrefix string 25 | MqttCAPath string 26 | MqttClientCrtPath string 27 | MqttClientKeyPath string 28 | } 29 | 30 | // init initializes the MQTT configuration 31 | func (c *MqttConfig) init() { 32 | c.MqttBroker = utils.GetEnv("MQTT_BROKER", DefaultMQTTBroker) 33 | portStr := utils.GetEnv("MQTT_PORT", DefaultMQTTPort) 34 | port, err := strconv.Atoi(portStr) 35 | if err == nil { 36 | c.MqttPort = port 37 | } 38 | 39 | c.MqttUsername = utils.GetEnv("MQTT_USERNAME", DefaultMQTTUsername) 40 | c.MqttPassword = os.Getenv("MQTT_PASSWORD") 41 | c.MqttClientPrefix = utils.GetEnv("MQTT_CLIENT_PREFIX", DefaultMQTTClientPrefix) 42 | c.MqttCAPath = os.Getenv("MQTT_CA_PATH") 43 | c.MqttClientCrtPath = os.Getenv("MQTT_CLIENT_CRT_PATH") 44 | c.MqttClientKeyPath = os.Getenv("MQTT_CLIENT_KEY_PATH") 45 | } 46 | -------------------------------------------------------------------------------- /module_tunnels/koupleless_mqtt_tunnel/mqtt/mqtt.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "github.com/koupleless/module_controller/common/zaplogger" 8 | "os" 9 | "time" 10 | 11 | mqtt "github.com/eclipse/paho.mqtt.golang" 12 | ) 13 | 14 | type ClientInitFunc func(*mqtt.ClientOptions) mqtt.Client 15 | 16 | // Constants for MQTT QoS levels 17 | const ( 18 | // Qos0 means message only published once 19 | Qos0 = iota 20 | 21 | // Qos1 means message must be consumed 22 | Qos1 23 | 24 | // Qos2 means message must be consumed only once 25 | Qos2 26 | ) 27 | 28 | // Client represents an MQTT client 29 | type Client struct { 30 | client mqtt.Client 31 | } 32 | 33 | // ClientConfig holds configuration for an MQTT client 34 | type ClientConfig struct { 35 | Broker string 36 | Port int 37 | ClientID string 38 | Username string 39 | Password string 40 | CAPath string 41 | ClientCrtPath string 42 | ClientKeyPath string 43 | CleanSession bool 44 | KeepAlive time.Duration 45 | DefaultMessageHandler mqtt.MessageHandler 46 | OnConnectHandler mqtt.OnConnectHandler 47 | ConnectionLostHandler mqtt.ConnectionLostHandler 48 | ClientInitFunc ClientInitFunc 49 | } 50 | 51 | // DefaultMqttClientInitFunc is the default function to initialize an MQTT client 52 | var DefaultMqttClientInitFunc ClientInitFunc = mqtt.NewClient 53 | 54 | // Default message handlers 55 | var defaultMessageHandler mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) { 56 | zaplogger.GetLogger().Info(fmt.Sprintf("Received message: %s from topic: %s\n", msg.Payload(), msg.Topic())) 57 | } 58 | 59 | var defaultOnConnectHandler mqtt.OnConnectHandler = func(client mqtt.Client) { 60 | zaplogger.GetLogger().Info("Connected") 61 | } 62 | 63 | var defaultConnectionLostHandler mqtt.ConnectionLostHandler = func(client mqtt.Client, err error) { 64 | zaplogger.GetLogger().Info(fmt.Sprintf("Connection lost %v\n", err)) 65 | } 66 | 67 | // newTlsConfig creates a TLS configuration using the client configuration 68 | func newTlsConfig(cfg *ClientConfig) (*tls.Config, error) { 69 | config := tls.Config{ 70 | InsecureSkipVerify: true, 71 | } 72 | 73 | certpool := x509.NewCertPool() 74 | ca, err := os.ReadFile(cfg.CAPath) 75 | if err != nil { 76 | return nil, err 77 | } 78 | certpool.AppendCertsFromPEM(ca) 79 | config.RootCAs = certpool 80 | if cfg.ClientCrtPath != "" { 81 | // Import client certificate/key pair 82 | clientKeyPair, err := tls.LoadX509KeyPair(cfg.ClientCrtPath, cfg.ClientKeyPath) 83 | if err != nil { 84 | return nil, err 85 | } 86 | config.Certificates = []tls.Certificate{clientKeyPair} 87 | config.ClientAuth = tls.NoClientCert 88 | } 89 | 90 | return &config, nil 91 | } 92 | 93 | // NewMqttClient creates a new MQTT client using the provided configuration 94 | func NewMqttClient(cfg *ClientConfig) (*Client, error) { 95 | opts := mqtt.NewClientOptions() 96 | broker := "" 97 | opts.SetClientID(cfg.ClientID) 98 | opts.SetUsername(cfg.Username) 99 | opts.SetPassword(cfg.Password) 100 | if cfg.CAPath != "" { 101 | // TLS configuration 102 | tlsConfig, err := newTlsConfig(cfg) 103 | if err != nil { 104 | return nil, err 105 | } 106 | opts.SetTLSConfig(tlsConfig) 107 | broker = fmt.Sprintf("ssl://%s:%d", cfg.Broker, cfg.Port) 108 | } else { 109 | broker = fmt.Sprintf("tcp://%s:%d", cfg.Broker, cfg.Port) 110 | } 111 | 112 | opts.AddBroker(broker) 113 | 114 | if cfg.DefaultMessageHandler == nil { 115 | cfg.DefaultMessageHandler = defaultMessageHandler 116 | } 117 | 118 | if cfg.OnConnectHandler == nil { 119 | cfg.OnConnectHandler = defaultOnConnectHandler 120 | } 121 | 122 | if cfg.ConnectionLostHandler == nil { 123 | cfg.ConnectionLostHandler = defaultConnectionLostHandler 124 | } 125 | 126 | if cfg.ClientInitFunc == nil { 127 | cfg.ClientInitFunc = DefaultMqttClientInitFunc 128 | } 129 | 130 | if cfg.KeepAlive == 0 { 131 | cfg.KeepAlive = time.Minute * 3 132 | } 133 | 134 | opts.SetDefaultPublishHandler(cfg.DefaultMessageHandler) 135 | opts.SetAutoReconnect(true) 136 | opts.SetKeepAlive(cfg.KeepAlive) 137 | opts.SetCleanSession(cfg.CleanSession) 138 | opts.SetOnConnectHandler(cfg.OnConnectHandler) 139 | opts.SetConnectionLostHandler(cfg.ConnectionLostHandler) 140 | client := cfg.ClientInitFunc(opts) 141 | return &Client{ 142 | client: client, 143 | }, nil 144 | } 145 | 146 | // Connect attempts to connect the MQTT client 147 | func (c *Client) Connect() error { 148 | token := c.client.Connect() 149 | token.Wait() 150 | return token.Error() 151 | } 152 | 153 | // Pub publishes a message to a specified topic 154 | func (c *Client) Pub(topic string, qos byte, msg []byte) error { 155 | zaplogger.GetLogger().Info(fmt.Sprintf("Publishing message: %s to topic: %s\n", msg, topic)) 156 | token := c.client.Publish(topic, qos, true, msg) 157 | token.Wait() 158 | return token.Error() 159 | } 160 | 161 | // Sub subscribes to a topic with a callback 162 | func (c *Client) Sub(topic string, qos byte, callBack mqtt.MessageHandler) error { 163 | zaplogger.GetLogger().Info(fmt.Sprintf("Subscribing to topic: %s\n", topic)) 164 | token := c.client.Subscribe(topic, qos, callBack) 165 | token.Wait() 166 | return token.Error() 167 | } 168 | 169 | // UnSub unsubscribes from a topic 170 | func (c *Client) UnSub(topic string) error { 171 | zaplogger.GetLogger().Info(fmt.Sprintf("Unsubscribing from topic: %s\n", topic)) 172 | c.client.Unsubscribe(topic) 173 | return nil 174 | } 175 | 176 | // Disconnect disconnects the MQTT client 177 | func (c *Client) Disconnect() { 178 | c.client.Disconnect(200) 179 | } 180 | -------------------------------------------------------------------------------- /report_server/server.go: -------------------------------------------------------------------------------- 1 | package report_server 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/koupleless/module_controller/common/zaplogger" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "strings" 12 | 13 | "github.com/koupleless/virtual-kubelet/common/utils" 14 | ) 15 | 16 | // LogRequest represents a log request with various fields 17 | type LogRequest struct { 18 | TraceID string `json:"traceID"` // Unique identifier for the log request 19 | Scene string `json:"scene"` // Scene or context of the log request 20 | Event string `json:"event"` // Event that triggered the log request 21 | TimeUsed int64 `json:"timeUsed,omitempty"` // Time taken for the event to complete 22 | Result string `json:"result"` // Result of the event 23 | Message string `json:"message"` // Additional message for the log request 24 | Code string `json:"code"` // Error code if the event failed 25 | Labels map[string]string `json:"labels"` // Additional labels for the log request 26 | } 27 | 28 | // DingtalkMessage represents a message to be sent to Dingtalk 29 | type DingtalkMessage struct { 30 | MsgType string `json:"msgtype"` // Message type 31 | MarkDown struct { 32 | Title string `json:"title"` // Title of the message 33 | Text string `json:"text"` // Content of the message 34 | } `json:"markdown"` // Markdown content of the message 35 | } 36 | 37 | // formatMap converts a map to a formatted string 38 | func formatMap(input map[string]string) string { 39 | var sb strings.Builder 40 | sb.WriteString("[") 41 | for key, value := range input { 42 | sb.WriteString(fmt.Sprintf("%s: %s\n", key, value)) 43 | } 44 | sb.WriteString("]") 45 | return sb.String() 46 | } 47 | 48 | // webhooks stores the list of webhooks to send messages to 49 | var webhooks []string 50 | 51 | // logHandler handles incoming log requests 52 | func logHandler(w http.ResponseWriter, r *http.Request) { 53 | if r.Method != http.MethodPost { 54 | http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) 55 | return 56 | } 57 | var logRequest LogRequest 58 | body, err := ioutil.ReadAll(r.Body) 59 | if err != nil { 60 | http.Error(w, "Bad request", http.StatusBadRequest) 61 | return 62 | } 63 | defer r.Body.Close() 64 | if err := json.Unmarshal(body, &logRequest); err != nil { 65 | http.Error(w, "Bad request", http.StatusBadRequest) 66 | return 67 | } 68 | 69 | // Construct the message to be sent 70 | text := fmt.Sprintf("## MC告警\n\n- **Trace ID**: %s\n- **工单类型**: %s\n- **节点**: %s\n- **结果**: %s\n- **失败信息**: %s\n- **错误代码**: %s\n- **部署信息**: %s", 71 | logRequest.TraceID, logRequest.Scene, logRequest.Event, logRequest.Result, logRequest.Message, logRequest.Code, formatMap(logRequest.Labels)) 72 | 73 | // Prepare the Dingtalk message 74 | dingtalkMessage := DingtalkMessage{ 75 | MsgType: "markdown", 76 | MarkDown: struct { 77 | Title string `json:"title"` 78 | Text string `json:"text"` 79 | }{ 80 | Title: "Log Notification", 81 | Text: text, 82 | }, 83 | } 84 | jsonMessage, err := json.Marshal(dingtalkMessage) 85 | if err != nil { 86 | http.Error(w, "Internal server error", http.StatusInternalServerError) 87 | return 88 | } 89 | 90 | // Iterate over all webhooks and send messages 91 | for _, webhook := range webhooks { 92 | if webhook == "" { 93 | continue 94 | } 95 | _, err = http.Post(webhook, "application/json", bytes.NewBuffer(jsonMessage)) 96 | if err != nil { 97 | log.Println("Error sending to webhook:", err.Error()) 98 | } 99 | } 100 | w.WriteHeader(http.StatusOK) 101 | } 102 | 103 | // InitReportServer initializes the report server 104 | func InitReportServer() { 105 | reportHooks := utils.GetEnv("REPORT_HOOKS", "") 106 | 107 | webhooks = strings.Split(reportHooks, ";") 108 | 109 | http.HandleFunc("/log", logHandler) 110 | log.Println("Starting server on :8080") 111 | if err := http.ListenAndServe(":8080", nil); err != nil { 112 | zaplogger.GetLogger().Error(err, "Server failed: %v") 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /samples/e2e_test_rbac/create_e2e-test_kubeconfig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -x 4 | # 首先需要把设置好读写证书，如 export KUBECONFIG=kube-write.config 5 | 6 | kubectl apply -f service_account.yaml 7 | kubectl apply -f service_account_cluster_role.yaml 8 | kubectl apply -f service_account_cluster_role_binding.yaml 9 | kubectl apply -f secret.yaml 10 | 11 | # 请根据您的环境和需求进行调整 12 | NAMESPACE="default" # 指定命名空间 13 | SERVICE_ACCOUNT_NAME="e2e-test" # 替换为您的 ServiceAccount 名称 14 | SECRET_NAME="$SERVICE_ACCOUNT_NAME-secret" 15 | 16 | # 获取 Secret 的 CA 证书和 Token 17 | CA_CERT=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o=jsonpath='{.data.ca\.crt}' | base64 --decode) 18 | TOKEN=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o=jsonpath='{.data.token}' | base64 --decode) 19 | 20 | # 获取集群信息 21 | CLUSTER_NAME=$(kubectl config current-context) # 获取当前上下文名称 22 | CLUSTER_ENDPOINT=$(kubectl cluster-info | grep 'Kubernetes control plane' | awk '/https/ {print $NF}') 23 | 24 | # 生成 kubeconfig 文件 25 | KUBECONFIG_FILE="kubeconfig-${SERVICE_ACCOUNT_NAME}.yaml" 26 | 27 | cat < $KUBECONFIG_FILE 28 | apiVersion: v1 29 | kind: Config 30 | clusters: 31 | - cluster: 32 | certificate-authority-data: $(echo "$CA_CERT" | base64) 33 | server: $CLUSTER_ENDPOINT 34 | name: $CLUSTER_NAME 35 | contexts: 36 | - context: 37 | cluster: $CLUSTER_NAME 38 | user: $SERVICE_ACCOUNT_NAME 39 | name: ${SERVICE_ACCOUNT_NAME}-context 40 | current-context: ${SERVICE_ACCOUNT_NAME}-context 41 | users: 42 | - name: $SERVICE_ACCOUNT_NAME 43 | user: 44 | token: $TOKEN 45 | EOF 46 | 47 | echo "kubeconfig 文件已生成: $KUBECONFIG_FILE" 48 | -------------------------------------------------------------------------------- /samples/e2e_test_rbac/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: e2e-test-secret 5 | annotations: 6 | kubernetes.io/service-account.name: e2e-test 7 | type: kubernetes.io/service-account-token -------------------------------------------------------------------------------- /samples/e2e_test_rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: e2e-test 5 | -------------------------------------------------------------------------------- /samples/e2e_test_rbac/service_account_cluster_role.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRole 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: e2e-test-role 5 | rules: 6 | - apiGroups: [""] # "" indicates the core API group 7 | resources: ["pods" , "pods/status", "pods/spec","nodes", "nodes/status", "events"] 8 | verbs: ["get", "watch", "list"] 9 | 10 | - apiGroups: ["apps"] 11 | resources: ["deployments", "deployments/status", "deployments/spec", "daemonSets", "daemonSets/status", "daemonSets/spec"] 12 | verbs: ["get", "watch", "list"] 13 | -------------------------------------------------------------------------------- /samples/e2e_test_rbac/service_account_cluster_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: e2e-test-cluster-role-binding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: e2e-test-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: e2e-test 12 | namespace: default 13 | -------------------------------------------------------------------------------- /samples/module/module_daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # 指定api版本，此值必须在kubectl api-versions中 2 | kind: DaemonSet # 指定创建资源的角色/类型 3 | metadata: # 资源的元数据/属性 4 | name: test-module-daemon-set # 资源的名字，在同一个namespace中必须唯一 5 | namespace: default # 部署在哪个namespace中 6 | labels: # 设定资源的标签 7 | virtual-kubelet.koupleless.io/env: test 8 | spec: # 资源规范字段 9 | revisionHistoryLimit: 3 # 保留历史版本 10 | selector: # 选择器 11 | matchLabels: # 匹配标签 12 | module.koupleless.io/name: biz1 13 | module.koupleless.io/version: 0.0.1 14 | updateStrategy: # 策略 15 | type: RollingUpdate # 滚动更新策略 16 | rollingUpdate: # 滚动更新 17 | maxUnavailable: 30% # 示在更新过程中能够进入不可用状态的 Pod 的最大值，可以为百分比，也可以为整数 18 | template: # 模版 19 | metadata: # 资源的元数据/属性 20 | labels: # 设定资源的标签 21 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 22 | spec: # 资源规范字段 23 | containers: 24 | - name: biz1 25 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/test_modules/biz1-0.0.1-ark-biz.jar 26 | env: 27 | - name: BIZ_VERSION 28 | value: 0.0.1 29 | affinity: 30 | nodeAffinity: 31 | requiredDuringSchedulingIgnoredDuringExecution: 32 | nodeSelectorTerms: # 基座node选择 33 | - matchExpressions: 34 | - key: base.koupleless.io/name 35 | operator: In 36 | values: 37 | - base # 基座名 38 | - key: base.koupleless.io/version 39 | operator: In 40 | values: 41 | - 1.0.0 # 基座版本 42 | - key: base.koupleless.io/cluster-name 43 | operator: In 44 | values: 45 | - default # 基座集群名 46 | tolerations: 47 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 48 | operator: "Equal" 49 | value: "True" 50 | effect: "NoExecute" 51 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 52 | operator: "Equal" 53 | value: "test" 54 | effect: "NoExecute" -------------------------------------------------------------------------------- /samples/module/module_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 # 指定api版本，此值必须在kubectl api-versions中 2 | kind: Deployment # 指定创建资源的角色/类型 3 | metadata: # 资源的元数据/属性 4 | name: test-module-deployment # 资源的名字，在同一个namespace中必须唯一 5 | namespace: default # 部署在哪个namespace中 6 | labels: # 设定资源的标签 7 | virtual-kubelet.koupleless.io/component: module-deployment # 资源类型标记， 用于module controller管理 8 | virtual-kubelet.koupleless.io/env: test 9 | virtual-kubelet.koupleless.io/strategy: peer 10 | spec: # 资源规范字段 11 | replicas: 1 12 | revisionHistoryLimit: 3 # 保留历史版本 13 | selector: # 选择器 14 | matchLabels: # 匹配标签 15 | module.koupleless.io/name: biz1 16 | module.koupleless.io/version: 0.0.1 17 | strategy: # 策略 18 | rollingUpdate: # 滚动更新 19 | maxSurge: 30% # 最大额外可以存在的副本数，可以为百分比，也可以为整数 20 | maxUnavailable: 30% # 示在更新过程中能够进入不可用状态的 Pod 的最大值，可以为百分比，也可以为整数 21 | type: RollingUpdate # 滚动更新策略 22 | template: # 模版 23 | metadata: # 资源的元数据/属性 24 | labels: # 设定资源的标签 25 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 26 | module.koupleless.io/name: biz1 27 | module.koupleless.io/version: 0.0.1 28 | spec: # 资源规范字段 29 | containers: 30 | - name: biz1 31 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/test_modules/biz1-0.0.1-ark-biz.jar 32 | env: 33 | - name: BIZ_VERSION 34 | value: 0.0.1 35 | affinity: 36 | nodeAffinity: 37 | requiredDuringSchedulingIgnoredDuringExecution: 38 | nodeSelectorTerms: # 基座node选择 39 | - matchExpressions: 40 | - key: base.koupleless.io/name 41 | operator: In 42 | values: 43 | - base # 基座名 44 | - key: base.koupleless.io/version 45 | operator: In 46 | values: 47 | - 1.0.0 # 基座版本 48 | - key: base.koupleless.io/cluster-name 49 | operator: In 50 | values: 51 | - default # 基座集群名 52 | podAntiAffinity: 53 | requiredDuringSchedulingIgnoredDuringExecution: 54 | - labelSelector: 55 | matchExpressions: 56 | - key: module.koupleless.io/name 57 | operator: In 58 | values: 59 | - biz1 60 | - key: module.koupleless.io/version 61 | operator: In 62 | values: 63 | - 0.0.1 64 | topologyKey: kubernetes.io/hostname 65 | tolerations: 66 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 67 | operator: "Equal" 68 | value: "True" 69 | effect: "NoExecute" 70 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 71 | operator: "Equal" 72 | value: "test" 73 | effect: "NoExecute" -------------------------------------------------------------------------------- /samples/module/multi_module_biz1_biz2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-biz-combine-koupleless 5 | labels: 6 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 7 | spec: 8 | containers: 9 | - name: biz2 10 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz2-web-single-host-0.0.1-SNAPSHOT-ark-biz.jar 11 | env: 12 | - name: BIZ_VERSION 13 | value: 0.0.1-SNAPSHOT 14 | - name: biz1 15 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz1-web-single-host-0.0.1-SNAPSHOT-ark-biz.jar 16 | env: 17 | - name: BIZ_VERSION 18 | value: 0.0.1-SNAPSHOT 19 | affinity: 20 | nodeAffinity: 21 | requiredDuringSchedulingIgnoredDuringExecution: 22 | nodeSelectorTerms: # 基座node选择 23 | - matchExpressions: 24 | - key: base.koupleless.io/name 25 | operator: In 26 | values: 27 | - base # 基座名 28 | - key: base.koupleless.io/version 29 | operator: In 30 | values: 31 | - 1.0.0 # 基座版本 32 | - key: base.koupleless.io/cluster-name 33 | operator: In 34 | values: 35 | - default # 基座集群名 36 | podAntiAffinity: 37 | requiredDuringSchedulingIgnoredDuringExecution: 38 | - labelSelector: 39 | matchExpressions: 40 | - key: module.koupleless.io/name 41 | operator: In 42 | values: 43 | - biz1 44 | - key: module.koupleless.io/version 45 | operator: In 46 | values: 47 | - 0.0.1 48 | topologyKey: kubernetes.io/hostname 49 | tolerations: 50 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 51 | operator: "Equal" 52 | value: "True" 53 | effect: "NoExecute" 54 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 55 | operator: "Equal" 56 | value: "test" 57 | effect: "NoExecute" 58 | -------------------------------------------------------------------------------- /samples/module/single_module_biz1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-single-module-biz1 5 | labels: 6 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 7 | spec: 8 | containers: 9 | - name: biz1 10 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz1-web-single-host-0.0.1-SNAPSHOT-ark-biz.jar 11 | env: 12 | - name: BIZ_VERSION 13 | value: 0.0.1-SNAPSHOT 14 | affinity: 15 | nodeAffinity: 16 | requiredDuringSchedulingIgnoredDuringExecution: 17 | nodeSelectorTerms: # 基座node选择 18 | - matchExpressions: 19 | - key: base.koupleless.io/name 20 | operator: In 21 | values: 22 | - base # 基座名 23 | - key: base.koupleless.io/version 24 | operator: In 25 | values: 26 | - 1.0.0 # 基座版本 27 | - key: base.koupleless.io/cluster-name 28 | operator: In 29 | values: 30 | - default # 基座集群名 31 | tolerations: 32 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 33 | operator: "Equal" 34 | value: "True" 35 | effect: "NoExecute" 36 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 37 | operator: "Equal" 38 | value: "test" 39 | effect: "NoExecute" -------------------------------------------------------------------------------- /samples/module/single_module_biz1_version2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-single-module-biz1-version2 5 | labels: 6 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 7 | spec: 8 | containers: 9 | - name: biz1 10 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz1-web-single-host-0.0.2-SNAPSHOT-ark-biz.jar 11 | env: 12 | - name: BIZ_VERSION 13 | value: 0.0.2-SNAPSHOT 14 | affinity: 15 | nodeAffinity: 16 | requiredDuringSchedulingIgnoredDuringExecution: 17 | nodeSelectorTerms: # 基座node选择 18 | - matchExpressions: 19 | - key: base.koupleless.io/name 20 | operator: In 21 | values: 22 | - base # 基座名 23 | - key: base.koupleless.io/version 24 | operator: In 25 | values: 26 | - 1.0.0 # 基座版本 27 | - key: base.koupleless.io/cluster-name 28 | operator: In 29 | values: 30 | - default # 基座集群名 31 | tolerations: 32 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 33 | operator: "Equal" 34 | value: "True" 35 | effect: "NoExecute" 36 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 37 | operator: "Equal" 38 | value: "test" 39 | effect: "NoExecute" -------------------------------------------------------------------------------- /samples/module/single_module_biz2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-biz2-koupleless 5 | labels: 6 | virtual-kubelet.koupleless.io/component: module # 必要，声明pod的类型，用于module controller管理 7 | spec: 8 | containers: 9 | - name: biz2 10 | image: https://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/biz2-web-single-host-0.0.1-SNAPSHOT-ark-biz.jar 11 | env: 12 | - name: BIZ_VERSION 13 | value: 0.0.1-SNAPSHOT 14 | affinity: 15 | nodeAffinity: 16 | requiredDuringSchedulingIgnoredDuringExecution: 17 | nodeSelectorTerms: # 基座node选择 18 | - matchExpressions: 19 | - key: base.koupleless.io/name 20 | operator: In 21 | values: 22 | - base # 基座名 23 | - key: base.koupleless.io/version 24 | operator: In 25 | values: 26 | - 1.0.0 # 基座版本 27 | - key: base.koupleless.io/cluster-name 28 | operator: In 29 | values: 30 | - default # 基座集群名 31 | tolerations: 32 | - key: "schedule.koupleless.io/virtual-node" # 确保模块能够调度到基座node上 33 | operator: "Equal" 34 | value: "True" 35 | effect: "NoExecute" 36 | - key: "schedule.koupleless.io/node-env" # 确保模块能够调度到特定环境的基座node上，这里Virtual Kubelet使用taint对env做了强管控，以实现更强的隔离能力 37 | operator: "Equal" 38 | value: "test" 39 | effect: "NoExecute" 40 | -------------------------------------------------------------------------------- /samples/module_controller_pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: module-controller-pre 5 | namespace: default 6 | spec: 7 | replicas: 3 8 | selector: 9 | matchLabels: 10 | app: module-controller 11 | template: 12 | metadata: 13 | labels: 14 | app: module-controller 15 | spec: 16 | serviceAccountName: virtual-kubelet 17 | containers: 18 | - name: module-controller 19 | image: serverless-registry.cn-shanghai.cr.aliyuncs.com/opensource/release/module-controller-v2:v2.1.2 20 | imagePullPolicy: Always 21 | resources: 22 | limits: 23 | cpu: "1000m" 24 | memory: "400Mi" 25 | env: 26 | - name: ENABLE_MQTT_TUNNEL # mqtt tunnel flag 27 | value: "false" # true means enable, if enable mqtt tunnel, please configure env below 28 | - name: ENABLE_HTTP_TUNNEL # http tunnel flag 29 | value: "true" # true means enable, if enable mqtt tunnel, you can choose to configure env below 30 | - name: MQTT_BROKER # mqtt broker url 31 | value: YOUR_MQTT_BROKER_ENDPOINT 32 | - name: MQTT_PORT # mqtt port 33 | value: YOUR_MQTT_PORT 34 | - name: MQTT_USERNAME # mqtt username 35 | value: YOUR_MQTT_USERNAME 36 | - name: MQTT_PASSWORD # mqtt password 37 | value: YOUR_MQTT_PASSWORD 38 | - name: MQTT_CLIENT_PREFIX # mqtt client prefix 39 | value: YOUR_MQTT_CLIENT_PREFIX 40 | - name: HTTP_TUNNEL_LISTEN_PORT # Module Controller HTTP Tunnel listen port, use 7777 for default 41 | value: "7777" 42 | - name: REPORT_HOOKS # error report hooks, only support dingTalk robot webhook 43 | value: YOUR_REPORT_HOOKS 44 | - name: ENV # module controller env, will set to vnode labels 45 | value: YOUR_ENV 46 | - name: IS_CLUSTER # cluster flag, if true, will use cluster configure to start virtual kubelet 47 | value: "false" 48 | - name: WORKLOAD_MAX_LEVEL # cluster configure, means max workload level for workload calculation in virtual kubelet, default is 3 49 | value: "3" 50 | - name: ENABLE_MODULE_DEPLOYMENT_CONTROLLER # module deployment controller flag, if true, will start deployment controller to modify module deployment's replicas and baseline 51 | value: "true" 52 | - name: VNODE_WORKER_NUM # VNode concurrent module processing thread number, set to 1 to indicate single thread 53 | value: "8" -------------------------------------------------------------------------------- /samples/rbac/base_service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: virtual-kubelet 5 | -------------------------------------------------------------------------------- /samples/rbac/base_service_account_cluster_role.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRole 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: virtual-kubelet-role 5 | rules: 6 | - apiGroups: [""] # "" indicates the core API group 7 | resources: ["pods" , "pods/status", "pods/spec","nodes", "nodes/status", "events"] 8 | verbs: ["get", "watch", "list", "update", "patch", "create", "delete"] 9 | - apiGroups: [ "apps" ] 10 | resources: [ "deployments", "deployments/status", "deployments/spec", "daemonSets", "daemonSets/status", "daemonSets/spec" ] 11 | verbs: [ "get", "watch", "list" ] 12 | - apiGroups: [""] # "" indicates the core API group 13 | resources: ["configmaps", "secrets", "services"] 14 | verbs: ["get", "watch", "list"] 15 | - apiGroups: ["coordination.k8s.io"] # "" indicates the core API group 16 | resources: ["leases"] 17 | verbs: ["get", "watch", "list", "update", "patch", "create", "delete"] 18 | -------------------------------------------------------------------------------- /samples/rbac/base_service_account_cluster_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: virtual-kubelet-cluster-role-binding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: virtual-kubelet-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: virtual-kubelet 12 | namespace: default 13 | -------------------------------------------------------------------------------- /suite/http/base_http_lifecycle_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/koupleless/virtual-kubelet/common/utils" 6 | "github.com/koupleless/virtual-kubelet/model" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | v12 "k8s.io/api/coordination/v1" 10 | v1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/api/errors" 12 | "k8s.io/apimachinery/pkg/types" 13 | "time" 14 | ) 15 | 16 | var _ = Describe("Base Lifecycle Test", func() { 17 | 18 | ctx := context.Background() 19 | 20 | // NOTICE: nodeId should be unique in suite test to avoid incorrect vnode handling pod or deployment. 21 | nodeId := "http-base-for-base-test" 22 | clusterName := "test-cluster-name" 23 | // NOTICE: port should be unique in suite test to avoid incorrect base handling request. 24 | mockHttpBase := NewMockHttpBase(nodeId, clusterName, "1.0.0", env, 1237) 25 | 26 | // NOTICE: if test cases will contaminate each other, the cases should add `Serial` keyword in ginkgo 27 | Context("http base online and deactive finally", func() { 28 | It("base should become a ready vnode eventually", Serial, func() { 29 | time.Sleep(time.Second) 30 | 31 | go mockHttpBase.Start(ctx, clientID) 32 | 33 | Eventually(func() bool { 34 | lease := &v12.Lease{} 35 | err := k8sClient.Get(ctx, types.NamespacedName{ 36 | Name: utils.FormatNodeName(nodeId, env), 37 | Namespace: v1.NamespaceNodeLease, 38 | }, lease) 39 | 40 | isLeader := err == nil && 41 | *lease.Spec.HolderIdentity == clientID && 42 | !time.Now().After(lease.Spec.RenewTime.Time.Add(time.Second*model.NodeLeaseDurationSeconds)) 43 | 44 | return isLeader 45 | }, time.Second*50, time.Second).Should(BeTrue()) 46 | 47 | Eventually(func() bool { 48 | vnode := &v1.Node{} 49 | err := k8sClient.Get(ctx, types.NamespacedName{ 50 | Name: utils.FormatNodeName(nodeId, env), 51 | }, vnode) 52 | vnodeReady := false 53 | for _, cond := range vnode.Status.Conditions { 54 | if cond.Type == v1.NodeReady { 55 | vnodeReady = cond.Status == v1.ConditionTrue 56 | break 57 | } 58 | } 59 | return err == nil && vnodeReady 60 | }, time.Second*50, time.Second).Should(BeTrue()) 61 | }) 62 | 63 | It("base offline with deactive message and finally exit", Serial, func() { 64 | mockHttpBase.Exit() 65 | Eventually(func() bool { 66 | vnode := &v1.Node{} 67 | err := k8sClient.Get(ctx, types.NamespacedName{ 68 | Name: utils.FormatNodeName(nodeId, env), 69 | }, vnode) 70 | return errors.IsNotFound(err) 71 | }, time.Second*50, time.Second).Should(BeTrue()) 72 | }) 73 | 74 | It("base should become a ready vnode eventually", Serial, func() { 75 | time.Sleep(time.Second) 76 | 77 | go mockHttpBase.Start(ctx, clientID) 78 | vnode := &v1.Node{} 79 | Eventually(func() bool { 80 | err := k8sClient.Get(ctx, types.NamespacedName{ 81 | Name: utils.FormatNodeName(nodeId, env), 82 | }, vnode) 83 | vnodeReady := false 84 | for _, cond := range vnode.Status.Conditions { 85 | if cond.Type == v1.NodeReady { 86 | vnodeReady = cond.Status == v1.ConditionTrue 87 | break 88 | } 89 | } 90 | return err == nil && vnodeReady 91 | }, time.Second*50, time.Second).Should(BeTrue()) 92 | }) 93 | 94 | It("base unreachable finally exit", Serial, func() { 95 | mockHttpBase.reachable = false 96 | Eventually(func() bool { 97 | vnode := &v1.Node{} 98 | err := k8sClient.Get(ctx, types.NamespacedName{ 99 | Name: utils.FormatNodeName(nodeId, env), 100 | }, vnode) 101 | return errors.IsNotFound(err) 102 | }, time.Minute*2, time.Second).Should(BeTrue()) 103 | }) 104 | 105 | It("reachable base should become a ready vnode eventually", Serial, func() { 106 | time.Sleep(time.Second) 107 | mockHttpBase.reachable = true 108 | go mockHttpBase.Start(ctx, clientID) 109 | vnode := &v1.Node{} 110 | Eventually(func() bool { 111 | err := k8sClient.Get(ctx, types.NamespacedName{ 112 | Name: utils.FormatNodeName(nodeId, env), 113 | }, vnode) 114 | vnodeReady := false 115 | for _, cond := range vnode.Status.Conditions { 116 | if cond.Type == v1.NodeReady { 117 | vnodeReady = cond.Status == v1.ConditionTrue 118 | break 119 | } 120 | } 121 | return err == nil && vnodeReady 122 | }, time.Second*50, time.Second).Should(BeTrue()) 123 | }) 124 | 125 | It("base finally exit", Serial, func() { 126 | mockHttpBase.Exit() 127 | 128 | Eventually(func() bool { 129 | vnode := &v1.Node{} 130 | err := k8sClient.Get(ctx, types.NamespacedName{ 131 | Name: utils.FormatNodeName(nodeId, env), 132 | }, vnode) 133 | return errors.IsNotFound(err) 134 | }, time.Second*50, time.Second).Should(BeTrue()) 135 | 136 | }) 137 | }) 138 | }) 139 | -------------------------------------------------------------------------------- /suite/http/mock_http_base.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "github.com/koupleless/arkctl/v1/service/ark" 9 | "github.com/koupleless/module_controller/common/model" 10 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel/ark_service" 11 | "github.com/koupleless/virtual-kubelet/common/utils" 12 | "github.com/sirupsen/logrus" 13 | "github.com/virtual-kubelet/virtual-kubelet/log" 14 | "io" 15 | "net/http" 16 | "sync" 17 | "time" 18 | ) 19 | 20 | type MockHttpBase struct { 21 | sync.Mutex 22 | Env string 23 | CurrState string 24 | Metadata model.BaseMetadata 25 | port int 26 | BizInfos map[string]ark.ArkBizInfo 27 | Baseline []ark.BizModel 28 | 29 | exit chan struct{} 30 | reachable bool 31 | } 32 | 33 | func NewMockHttpBase(name, clusterName, version, env string, port int) *MockHttpBase { 34 | return &MockHttpBase{ 35 | Env: env, 36 | CurrState: "ACTIVATED", 37 | Metadata: model.BaseMetadata{ 38 | Identity: name, 39 | ClusterName: clusterName, 40 | Version: version, 41 | }, 42 | BizInfos: make(map[string]ark.ArkBizInfo), 43 | exit: make(chan struct{}), 44 | reachable: true, 45 | port: port, 46 | } 47 | } 48 | 49 | func (b *MockHttpBase) Exit() { 50 | select { 51 | case <-b.exit: 52 | default: 53 | close(b.exit) 54 | } 55 | } 56 | 57 | func (base *MockHttpBase) Start(ctx context.Context, clientID string) error { 58 | base.exit = make(chan struct{}) 59 | base.CurrState = "ACTIVATED" 60 | // start a http server to mock base 61 | mux := http.NewServeMux() 62 | 63 | server := http.Server{ 64 | Addr: fmt.Sprintf(":%d", base.port), 65 | Handler: mux, 66 | } 67 | 68 | mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { 69 | if base.reachable { 70 | w.WriteHeader(http.StatusOK) 71 | w.Header().Set("Content-Type", "application/json") 72 | w.Write(base.getHealthMsg()) 73 | } else { 74 | w.WriteHeader(http.StatusBadGateway) 75 | } 76 | }) 77 | 78 | mux.HandleFunc("/queryAllBiz", func(w http.ResponseWriter, r *http.Request) { 79 | if base.reachable { 80 | w.WriteHeader(http.StatusOK) 81 | w.Header().Set("Content-Type", "application/json") 82 | w.Write(base.getQueryAllBizMsg()) 83 | } else { 84 | w.WriteHeader(http.StatusBadGateway) 85 | } 86 | }) 87 | 88 | mux.HandleFunc("/installBiz", func(w http.ResponseWriter, r *http.Request) { 89 | if base.reachable { 90 | w.WriteHeader(http.StatusOK) 91 | w.Header().Set("Content-Type", "application/json") 92 | 93 | defer r.Body.Close() 94 | body, err := io.ReadAll(r.Body) 95 | if err != nil { 96 | logrus.Errorf("error reading body: %s", err) 97 | return 98 | } 99 | 100 | w.Write(base.processInstallBiz(body)) 101 | } else { 102 | w.WriteHeader(http.StatusBadGateway) 103 | } 104 | 105 | }) 106 | 107 | mux.HandleFunc("/uninstallBiz", func(w http.ResponseWriter, r *http.Request) { 108 | if base.reachable { 109 | w.WriteHeader(http.StatusOK) 110 | w.Header().Set("Content-Type", "application/json") 111 | 112 | defer r.Body.Close() 113 | body, err := io.ReadAll(r.Body) 114 | if err != nil { 115 | logrus.Errorf("error reading body: %s", err) 116 | return 117 | } 118 | 119 | w.Write(base.processUnInstallBiz(body)) 120 | } else { 121 | w.WriteHeader(http.StatusBadGateway) 122 | } 123 | }) 124 | 125 | go server.ListenAndServe() 126 | 127 | // Start a new goroutine to upload node heart beat data every 10 seconds 128 | go utils.TimedTaskWithInterval(ctx, time.Second*10, func(ctx context.Context) { 129 | if base.reachable { 130 | log.G(ctx).Info("upload node heart beat data from node ", base.Metadata.Identity) 131 | _, err := http.Post("http://127.0.0.1:7777/heartbeat", "application/json", bytes.NewBuffer(base.getHeartBeatMsg())) 132 | if err != nil { 133 | logrus.Errorf("error calling heartbeat: %s", err) 134 | } 135 | } 136 | }) 137 | 138 | _, err := http.Post("http://127.0.0.1:7777/heartbeat", "application/json", bytes.NewBuffer(base.getHeartBeatMsg())) 139 | if err != nil { 140 | logrus.Errorf("error calling heartbeat: %s", err) 141 | return err 142 | } 143 | 144 | go func() { 145 | select { 146 | case <-ctx.Done(): 147 | case <-base.exit: 148 | } 149 | base.CurrState = "DEACTIVATED" 150 | _, err = http.Post("http://127.0.0.1:7777/heartbeat", "application/json", bytes.NewBuffer(base.getHeartBeatMsg())) 151 | time.Sleep(2 * time.Second) 152 | server.Shutdown(ctx) 153 | if err != nil { 154 | logrus.Errorf("error calling heartbeat: %s", err) 155 | } 156 | }() 157 | 158 | return nil 159 | } 160 | 161 | func (b *MockHttpBase) getHeartBeatMsg() []byte { 162 | msg := model.BaseStatus{ 163 | BaseMetadata: b.Metadata, 164 | LocalIP: "127.0.0.1", 165 | LocalHostName: "localhost", 166 | Port: b.port, 167 | State: b.CurrState, 168 | } 169 | msgBytes, _ := json.Marshal(msg) 170 | return msgBytes 171 | } 172 | 173 | func (b *MockHttpBase) getHealthMsg() []byte { 174 | msg := ark_service.HealthResponse{ 175 | GenericArkResponseBase: ark.GenericArkResponseBase[ark.HealthInfo]{ 176 | Code: "SUCCESS", 177 | Data: ark.HealthInfo{ 178 | HealthData: ark.HealthData{ 179 | Jvm: ark.JvmInfo{ 180 | JavaUsedMetaspace: 10240, 181 | JavaMaxMetaspace: 1024, 182 | JavaCommittedMetaspace: 1024, 183 | }, 184 | MasterBizInfo: ark.MasterBizInfo{ 185 | BizName: b.Metadata.Identity, 186 | BizState: b.CurrState, 187 | BizVersion: b.Metadata.Version, 188 | }, 189 | Cpu: ark.CpuInfo{ 190 | Count: 1, 191 | TotalUsed: 20, 192 | Type: "intel", 193 | UserUsed: 2, 194 | Free: 80, 195 | SystemUsed: 13, 196 | }, 197 | }, 198 | }, 199 | Message: "", 200 | }, 201 | } 202 | msgBytes, _ := json.Marshal(msg) 203 | return msgBytes 204 | } 205 | 206 | func (b *MockHttpBase) getQueryAllBizMsg() []byte { 207 | 208 | arkBizInfos := make([]ark.ArkBizInfo, 0) 209 | 210 | for _, bizInfo := range b.BizInfos { 211 | arkBizInfos = append(arkBizInfos, bizInfo) 212 | } 213 | 214 | msg := ark_service.QueryAllBizResponse{ 215 | GenericArkResponseBase: ark.GenericArkResponseBase[[]ark.ArkBizInfo]{ 216 | Code: "SUCCESS", 217 | Data: arkBizInfos, 218 | Message: "", 219 | }, 220 | } 221 | msgBytes, _ := json.Marshal(msg) 222 | return msgBytes 223 | } 224 | 225 | func (b *MockHttpBase) processInstallBiz(msg []byte) []byte { 226 | b.Lock() 227 | defer b.Unlock() 228 | request := ark_service.InstallBizRequest{} 229 | json.Unmarshal(msg, &request) 230 | identity := getBizIdentity(request.BizModel) 231 | logrus.Infof("install biz %s from http base", identity) 232 | _, has := b.BizInfos[identity] 233 | if !has { 234 | b.BizInfos[identity] = ark.ArkBizInfo{ 235 | BizName: request.BizName, 236 | BizState: "ACTIVATED", 237 | BizVersion: request.BizVersion, 238 | BizStateRecords: []ark.ArkBizStateRecord{ 239 | { 240 | ChangeTime: 1234, 241 | State: "ACTIVATED", 242 | }, 243 | }, 244 | } 245 | } 246 | response := ark_service.ArkResponse[ark.ArkResponseData]{ 247 | Code: "SUCCESS", 248 | Data: ark.ArkResponseData{ 249 | ArkClientResponse: ark.ArkClientResponse{ 250 | Code: "SUCCESS", 251 | Message: "", 252 | BizInfos: nil, 253 | }, 254 | ElapsedSpace: 0, 255 | }, 256 | Message: "", 257 | ErrorStackTrace: "", 258 | BaseIdentity: b.Metadata.Identity, 259 | } 260 | respBytes, _ := json.Marshal(response) 261 | return respBytes 262 | } 263 | 264 | func (b *MockHttpBase) processUnInstallBiz(msg []byte) []byte { 265 | b.Lock() 266 | defer b.Unlock() 267 | request := ark_service.UninstallBizRequest{} 268 | json.Unmarshal(msg, &request) 269 | identity := getBizIdentity(request.BizModel) 270 | logrus.Infof("uninstall biz %s from http base", identity) 271 | delete(b.BizInfos, identity) 272 | // send to response 273 | response := ark_service.ArkResponse[ark.ArkResponseData]{ 274 | Code: "SUCCESS", 275 | Data: ark.ArkResponseData{ 276 | ArkClientResponse: ark.ArkClientResponse{ 277 | Code: "SUCCESS", 278 | Message: "", 279 | BizInfos: nil, 280 | }, 281 | ElapsedSpace: 0, 282 | }, 283 | Message: "", 284 | ErrorStackTrace: "", 285 | BaseIdentity: b.Metadata.Identity, 286 | } 287 | respBytes, _ := json.Marshal(response) 288 | return respBytes 289 | } 290 | 291 | func getBizIdentity(bizModel ark.BizModel) string { 292 | return fmt.Sprintf("%s:%s", bizModel.BizName, bizModel.BizVersion) 293 | } 294 | -------------------------------------------------------------------------------- /suite/http/module_http_lifecycle_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/koupleless/virtual-kubelet/common/utils" 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | v1 "k8s.io/api/core/v1" 9 | "k8s.io/apimachinery/pkg/api/errors" 10 | "k8s.io/apimachinery/pkg/types" 11 | "time" 12 | ) 13 | 14 | var _ = Describe("Module Lifecycle Test", func() { 15 | 16 | ctx := context.Background() 17 | 18 | // NOTICE: nodeId should be unique in suite test to avoid incorrect vnode handling pod or deployment. 19 | nodeID := "http-base-for-module-test" 20 | clusterName := "test-cluster-name" 21 | // NOTICE: port should be unique in suite test to avoid incorrect base handling request. 22 | mockBase := NewMockHttpBase(nodeID, clusterName, "1.0.0", env, 1238) 23 | 24 | mockModulePod := prepareModulePod("test-module", "default", utils.FormatNodeName(nodeID, env)) 25 | 26 | // NOTICE: if test cases will contaminate each other, the cases should add `Serial` keyword in ginkgo 27 | Context("pod install", Serial, func() { 28 | It("base should become a ready vnode eventually", Serial, func() { 29 | go mockBase.Start(ctx, clientID) 30 | Eventually(func() bool { 31 | vnode := &v1.Node{} 32 | err := k8sClient.Get(ctx, types.NamespacedName{ 33 | Name: utils.FormatNodeName(nodeID, env), 34 | }, vnode) 35 | vnodeReady := false 36 | for _, cond := range vnode.Status.Conditions { 37 | if cond.Type == v1.NodeReady { 38 | vnodeReady = cond.Status == v1.ConditionTrue 39 | break 40 | } 41 | } 42 | return err == nil && vnodeReady 43 | }, time.Second*20, time.Second).Should(BeTrue()) 44 | }) 45 | 46 | It("publish a module pod and it should be pending", Serial, func() { 47 | err := k8sClient.Create(ctx, &mockModulePod) 48 | Expect(err).To(BeNil()) 49 | Eventually(func() bool { 50 | podFromKubernetes := &v1.Pod{} 51 | err := k8sClient.Get(ctx, types.NamespacedName{ 52 | Namespace: mockModulePod.Namespace, 53 | Name: mockModulePod.Name, 54 | }, podFromKubernetes) 55 | return err == nil && podFromKubernetes.Status.Phase == v1.PodPending && podFromKubernetes.Spec.NodeName == utils.FormatNodeName(nodeID, env) 56 | }, time.Second*20, time.Second).Should(BeTrue()) 57 | Eventually(func() bool { 58 | return len(mockBase.BizInfos) == 1 59 | }, time.Second*20, time.Second).Should(BeTrue()) 60 | }) 61 | 62 | It("delete pod, all modules should deactivated, pod should finally deleted from k8s", Serial, func() { 63 | err := k8sClient.Delete(ctx, &mockModulePod) 64 | Expect(err).To(BeNil()) 65 | Eventually(func() bool { 66 | podFromKubernetes := &v1.Pod{} 67 | err := k8sClient.Get(ctx, types.NamespacedName{ 68 | Namespace: mockModulePod.Namespace, 69 | Name: mockModulePod.Name, 70 | }, podFromKubernetes) 71 | return errors.IsNotFound(err) 72 | }, time.Second*40, time.Second).Should(BeTrue()) 73 | }) 74 | 75 | It("base offline with deactive message and finally exit", Serial, func() { 76 | mockBase.Exit() 77 | Eventually(func() bool { 78 | vnode := &v1.Node{} 79 | err := k8sClient.Get(ctx, types.NamespacedName{ 80 | Name: utils.FormatNodeName(nodeID, env), 81 | }, vnode) 82 | return errors.IsNotFound(err) 83 | }, time.Second*30, time.Second).Should(BeTrue()) 84 | }) 85 | }) 86 | 87 | }) 88 | -------------------------------------------------------------------------------- /suite/http/suite_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | model2 "github.com/koupleless/module_controller/common/model" 7 | "github.com/koupleless/module_controller/controller/module_deployment_controller" 8 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel" 9 | "github.com/koupleless/virtual-kubelet/model" 10 | "github.com/koupleless/virtual-kubelet/vnode_controller" 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | "github.com/sirupsen/logrus" 14 | "github.com/virtual-kubelet/virtual-kubelet/log" 15 | logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" 16 | v1 "k8s.io/api/core/v1" 17 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 18 | "k8s.io/client-go/kubernetes/scheme" 19 | "k8s.io/client-go/rest" 20 | "os" 21 | ctrl "sigs.k8s.io/controller-runtime" 22 | "sigs.k8s.io/controller-runtime/pkg/client" 23 | "sigs.k8s.io/controller-runtime/pkg/envtest" 24 | logf "sigs.k8s.io/controller-runtime/pkg/log" 25 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 26 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 27 | "testing" 28 | "time" 29 | ) 30 | 31 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 32 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 33 | 34 | var cfg *rest.Config 35 | var testEnv *envtest.Environment 36 | var k8sClient client.Client 37 | var httpTunnel koupleless_http_tunnel.HttpTunnel 38 | var ctx, cancel = context.WithCancel(context.Background()) 39 | 40 | const ( 41 | clientID = "suite-test" 42 | env = "suite" 43 | ) 44 | 45 | func TestControllers(t *testing.T) { 46 | RegisterFailHandler(Fail) 47 | 48 | RunSpecs(t, "Module Controller Suite") 49 | } 50 | 51 | var _ = BeforeSuite(func() { 52 | 53 | os.Setenv("ENABLE_MODULE_REPLICAS_SYNC_WITH_BASE", "true") 54 | 55 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 56 | log.L = logruslogger.FromLogrus(logrus.NewEntry(logrus.StandardLogger())) 57 | 58 | By("bootstrapping suite environment") 59 | //usingExistingCluster := true 60 | testEnv = &envtest.Environment{ 61 | //UseExistingCluster: &usingExistingCluster, 62 | } 63 | 64 | var err error 65 | 66 | cfg, err = testEnv.Start() 67 | Expect(err).NotTo(HaveOccurred()) 68 | Expect(cfg).NotTo(BeNil()) 69 | 70 | // Allow all connections. 71 | err = scheme.AddToScheme(scheme.Scheme) 72 | Expect(err).NotTo(HaveOccurred()) 73 | 74 | k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ 75 | Scheme: scheme.Scheme, 76 | Metrics: metricsserver.Options{ 77 | BindAddress: ":8081", 78 | }, 79 | }) 80 | Expect(err).ToNot(HaveOccurred()) 81 | 82 | moduleDeploymentController, err := module_deployment_controller.NewModuleDeploymentController(env) 83 | Expect(err).ToNot(HaveOccurred()) 84 | 85 | err = moduleDeploymentController.SetupWithManager(ctx, k8sManager) 86 | 87 | Expect(err).ToNot(HaveOccurred()) 88 | 89 | httpTunnel = koupleless_http_tunnel.NewHttpTunnel(ctx, env, k8sManager.GetClient(), moduleDeploymentController, 7777) 90 | 91 | err = httpTunnel.Start(clientID, env) 92 | if err != nil { 93 | log.G(ctx).WithError(err).Error("failed to start tunnel", httpTunnel.Key()) 94 | panic(fmt.Sprintf("failed to start tunnel %s", httpTunnel.Key())) 95 | } else { 96 | log.G(ctx).Info("Tunnel started: ", httpTunnel.Key()) 97 | } 98 | 99 | vnodeController, err := vnode_controller.NewVNodeController(&model.BuildVNodeControllerConfig{ 100 | ClientID: clientID, 101 | Env: env, 102 | VPodType: model2.ComponentModule, 103 | VNodeWorkerNum: 4, 104 | }, &httpTunnel) 105 | Expect(err).ToNot(HaveOccurred()) 106 | 107 | err = vnodeController.SetupWithManager(ctx, k8sManager) 108 | 109 | k8sClient = k8sManager.GetClient() 110 | Expect(k8sClient).ToNot(BeNil()) 111 | 112 | go func() { 113 | err = k8sManager.Start(ctx) 114 | log.G(ctx).WithError(err).Error("k8sManager Start") 115 | Expect(err).ToNot(HaveOccurred()) 116 | }() 117 | 118 | k8sManager.GetCache().WaitForCacheSync(ctx) 119 | }) 120 | 121 | var _ = AfterSuite(func() { 122 | By("tearing down the suite environment") 123 | cancel() 124 | testEnv.Stop() 125 | time.Sleep(15 * time.Second) 126 | log.G(ctx).Info("suite for http stopped!") 127 | }) 128 | 129 | func prepareModulePod(name, namespace, nodeName string) v1.Pod { 130 | return v1.Pod{ 131 | ObjectMeta: metav1.ObjectMeta{ 132 | Name: name, 133 | Namespace: namespace, 134 | Labels: map[string]string{ 135 | model.LabelKeyOfComponent: model2.ComponentModule, 136 | }, 137 | }, 138 | Spec: v1.PodSpec{ 139 | NodeName: nodeName, 140 | Containers: []v1.Container{ 141 | { 142 | Name: "biz", 143 | Image: "suite-biz.jar", 144 | Env: []v1.EnvVar{ 145 | { 146 | Name: "BIZ_VERSION", 147 | Value: "0.0.1", 148 | }, 149 | }, 150 | }, 151 | }, 152 | Tolerations: []v1.Toleration{ 153 | { 154 | Key: model.TaintKeyOfVnode, 155 | Operator: v1.TolerationOpEqual, 156 | Value: "True", 157 | Effect: v1.TaintEffectNoExecute, 158 | }, 159 | { 160 | Key: model.TaintKeyOfEnv, 161 | Operator: v1.TolerationOpEqual, 162 | Value: env, 163 | Effect: v1.TaintEffectNoExecute, 164 | }, 165 | }, 166 | }, 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /suite/mqtt/base_mqtt_lifecycle_test.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "context" 5 | "github.com/koupleless/virtual-kubelet/common/utils" 6 | "github.com/koupleless/virtual-kubelet/model" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | v12 "k8s.io/api/coordination/v1" 10 | v1 "k8s.io/api/core/v1" 11 | "k8s.io/apimachinery/pkg/api/errors" 12 | "k8s.io/apimachinery/pkg/types" 13 | "time" 14 | ) 15 | 16 | var _ = Describe("Base Lifecycle Test", func() { 17 | 18 | ctx := context.Background() 19 | 20 | mqttNodeID := "test-mqtt-base" 21 | clusterName := "test-cluster-name" 22 | mockMqttBase := NewMockMqttBase(mqttNodeID, clusterName, "1.0.0", env) 23 | 24 | Context("mqtt base online and deactive finally", func() { 25 | It("base should become a ready vnode eventually", func() { 26 | time.Sleep(time.Second) 27 | 28 | go mockMqttBase.Start(ctx, clientID) 29 | 30 | Eventually(func() bool { 31 | lease := &v12.Lease{} 32 | err := k8sClient.Get(ctx, types.NamespacedName{ 33 | Name: utils.FormatNodeName(mqttNodeID, env), 34 | Namespace: v1.NamespaceNodeLease, 35 | }, lease) 36 | 37 | isLeader := err == nil && 38 | *lease.Spec.HolderIdentity == clientID && 39 | !time.Now().After(lease.Spec.RenewTime.Time.Add(time.Second*model.NodeLeaseDurationSeconds)) 40 | 41 | return isLeader 42 | }, time.Second*50, time.Second).Should(BeTrue()) 43 | 44 | Eventually(func() bool { 45 | vnode := &v1.Node{} 46 | err := k8sClient.Get(ctx, types.NamespacedName{ 47 | Name: utils.FormatNodeName(mqttNodeID, env), 48 | }, vnode) 49 | vnodeReady := false 50 | for _, cond := range vnode.Status.Conditions { 51 | if cond.Type == v1.NodeReady { 52 | vnodeReady = cond.Status == v1.ConditionTrue 53 | break 54 | } 55 | } 56 | return err == nil && vnodeReady 57 | }, time.Second*50, time.Second).Should(BeTrue()) 58 | }) 59 | 60 | It("base offline with deactive message and finally exit", func() { 61 | mockMqttBase.Exit() 62 | Eventually(func() bool { 63 | vnode := &v1.Node{} 64 | err := k8sClient.Get(ctx, types.NamespacedName{ 65 | Name: utils.FormatNodeName(mqttNodeID, env), 66 | }, vnode) 67 | return errors.IsNotFound(err) 68 | }, time.Second*50, time.Second).Should(BeTrue()) 69 | }) 70 | }) 71 | 72 | Context("mqtt base online and unreachable finally", func() { 73 | It("base should become a ready vnode eventually", func() { 74 | time.Sleep(time.Second) 75 | 76 | go mockMqttBase.Start(ctx, clientID) 77 | vnode := &v1.Node{} 78 | Eventually(func() bool { 79 | err := k8sClient.Get(ctx, types.NamespacedName{ 80 | Name: utils.FormatNodeName(mqttNodeID, env), 81 | }, vnode) 82 | vnodeReady := false 83 | for _, cond := range vnode.Status.Conditions { 84 | if cond.Type == v1.NodeReady { 85 | vnodeReady = cond.Status == v1.ConditionTrue 86 | break 87 | } 88 | } 89 | return err == nil && vnodeReady 90 | }, time.Second*50, time.Second).Should(BeTrue()) 91 | }) 92 | 93 | It("base unreachable finally exit", func() { 94 | mockMqttBase.reachable = false 95 | Eventually(func() bool { 96 | vnode := &v1.Node{} 97 | 98 | err := k8sClient.Get(ctx, types.NamespacedName{ 99 | Name: utils.FormatNodeName(mqttNodeID, env), 100 | }, vnode) 101 | return errors.IsNotFound(err) 102 | }, time.Minute*2, time.Second).Should(BeTrue()) 103 | }) 104 | 105 | It("reachable base should become a ready vnode eventually", func() { 106 | time.Sleep(time.Second) 107 | mockMqttBase.reachable = true 108 | go mockMqttBase.Start(ctx, clientID) 109 | vnode := &v1.Node{} 110 | Eventually(func() bool { 111 | err := k8sClient.Get(ctx, types.NamespacedName{ 112 | Name: utils.FormatNodeName(mqttNodeID, env), 113 | }, vnode) 114 | vnodeReady := false 115 | for _, cond := range vnode.Status.Conditions { 116 | if cond.Type == v1.NodeReady { 117 | vnodeReady = cond.Status == v1.ConditionTrue 118 | break 119 | } 120 | } 121 | return err == nil && vnodeReady 122 | }, time.Second*50, time.Second).Should(BeTrue()) 123 | }) 124 | 125 | It("base offline with deactive message and finally exit", func() { 126 | mockMqttBase.Exit() 127 | Eventually(func() bool { 128 | vnode := &v1.Node{} 129 | err := k8sClient.Get(ctx, types.NamespacedName{ 130 | Name: utils.FormatNodeName(mqttNodeID, env), 131 | }, vnode) 132 | return errors.IsNotFound(err) 133 | }, time.Second*50, time.Second).Should(BeTrue()) 134 | }) 135 | }) 136 | }) 137 | -------------------------------------------------------------------------------- /suite/mqtt/mock_mqtt_base.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | paho "github.com/eclipse/paho.mqtt.golang" 8 | "github.com/koupleless/arkctl/v1/service/ark" 9 | "github.com/koupleless/module_controller/common/model" 10 | "github.com/koupleless/module_controller/module_tunnels/koupleless_http_tunnel/ark_service" 11 | "github.com/koupleless/module_controller/module_tunnels/koupleless_mqtt_tunnel/mqtt" 12 | "github.com/koupleless/virtual-kubelet/common/utils" 13 | "github.com/virtual-kubelet/virtual-kubelet/log" 14 | "strings" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type MockMQTTBase struct { 20 | sync.Mutex 21 | Env string 22 | CurrState string 23 | BaseMetadata model.BaseMetadata 24 | BizInfos map[string]ark.ArkBizInfo 25 | Baseline []ark.BizModel 26 | client *mqtt.Client 27 | 28 | exit chan struct{} 29 | reachable bool 30 | } 31 | 32 | func NewMockMqttBase(baseName, clusterName, version, env string) *MockMQTTBase { 33 | return &MockMQTTBase{ 34 | Env: env, 35 | CurrState: "ACTIVATED", 36 | BaseMetadata: model.BaseMetadata{ 37 | Identity: baseName, 38 | ClusterName: clusterName, 39 | Version: version, 40 | }, 41 | BizInfos: make(map[string]ark.ArkBizInfo), 42 | exit: make(chan struct{}), 43 | reachable: true, 44 | } 45 | } 46 | 47 | func (b *MockMQTTBase) Exit() { 48 | select { 49 | case <-b.exit: 50 | default: 51 | close(b.exit) 52 | } 53 | } 54 | 55 | func (b *MockMQTTBase) Start(ctx context.Context, clientID string) error { 56 | b.exit = make(chan struct{}) 57 | b.CurrState = "ACTIVATED" 58 | var err error 59 | b.client, err = mqtt.NewMqttClient(&mqtt.ClientConfig{ 60 | Broker: "localhost", 61 | Port: 1883, 62 | ClientID: clientID, 63 | Username: "test", 64 | Password: "", 65 | OnConnectHandler: func(client paho.Client) { 66 | client.Subscribe(fmt.Sprintf("koupleless_%s/%s/+", b.Env, b.BaseMetadata.Identity), 1, b.processCommand) 67 | client.Subscribe(fmt.Sprintf("koupleless_%s/%s/base/baseline", b.Env, b.BaseMetadata.Identity), 1, b.processBaseline) 68 | }, 69 | }) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | b.client.Connect() 75 | 76 | // Start a new goroutine to upload node heart beat data every 10 seconds 77 | go utils.TimedTaskWithInterval(ctx, time.Second*10, func(ctx context.Context) { 78 | if b.reachable { 79 | log.G(ctx).Info("upload node heart beat data from node ", b.BaseMetadata.Identity) 80 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/heart", b.Env, b.BaseMetadata.Identity), 1, b.getHeartBeatMsg()) 81 | } 82 | }) 83 | 84 | // send heart beat message 85 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/heart", b.Env, b.BaseMetadata.Identity), 1, b.getHeartBeatMsg()) 86 | 87 | select { 88 | case <-b.exit: 89 | b.SetCurrState("DEACTIVATED") 90 | b.client.Disconnect() 91 | case <-ctx.Done(): 92 | } 93 | return nil 94 | } 95 | 96 | func (b *MockMQTTBase) SetCurrState(state string) { 97 | b.CurrState = state 98 | // send heart beat message 99 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/heart", b.Env, b.BaseMetadata.Identity), 1, b.getHeartBeatMsg()) 100 | } 101 | 102 | func (b *MockMQTTBase) getHeartBeatMsg() []byte { 103 | msg := model.ArkMqttMsg[model.BaseStatus]{ 104 | PublishTimestamp: time.Now().UnixMilli(), 105 | Data: model.BaseStatus{ 106 | BaseMetadata: b.BaseMetadata, 107 | LocalIP: "127.0.0.1", 108 | LocalHostName: "localhost", 109 | State: b.CurrState, 110 | }, 111 | } 112 | msgBytes, _ := json.Marshal(msg) 113 | return msgBytes 114 | } 115 | 116 | func (b *MockMQTTBase) getHealthMsg() []byte { 117 | msg := model.ArkMqttMsg[ark.HealthResponse]{ 118 | PublishTimestamp: time.Now().UnixMilli(), 119 | Data: ark.HealthResponse{ 120 | GenericArkResponseBase: ark.GenericArkResponseBase[ark.HealthInfo]{ 121 | Code: "SUCCESS", 122 | Data: ark.HealthInfo{ 123 | HealthData: ark.HealthData{ 124 | Jvm: ark.JvmInfo{ 125 | JavaUsedMetaspace: 10240, 126 | JavaMaxMetaspace: 1024, 127 | JavaCommittedMetaspace: 1024, 128 | }, 129 | MasterBizInfo: ark.MasterBizInfo{ 130 | BizName: b.BaseMetadata.Identity, 131 | BizState: b.CurrState, 132 | BizVersion: b.BaseMetadata.Version, 133 | }, 134 | Cpu: ark.CpuInfo{ 135 | Count: 1, 136 | TotalUsed: 20, 137 | Type: "intel", 138 | UserUsed: 2, 139 | Free: 80, 140 | SystemUsed: 13, 141 | }, 142 | }, 143 | }, 144 | Message: "", 145 | }, 146 | }, 147 | } 148 | msgBytes, _ := json.Marshal(msg) 149 | return msgBytes 150 | } 151 | 152 | func (b *MockMQTTBase) getQueryAllBizMsg() []byte { 153 | 154 | arkBizInfos := make([]ark.ArkBizInfo, 0) 155 | 156 | for _, bizInfo := range b.BizInfos { 157 | arkBizInfos = append(arkBizInfos, bizInfo) 158 | } 159 | 160 | msg := model.ArkMqttMsg[ark.QueryAllArkBizResponse]{ 161 | PublishTimestamp: time.Now().UnixMilli(), 162 | Data: ark.QueryAllArkBizResponse{ 163 | GenericArkResponseBase: ark.GenericArkResponseBase[[]ark.ArkBizInfo]{ 164 | Code: "SUCCESS", 165 | Data: arkBizInfos, 166 | Message: "", 167 | }, 168 | }, 169 | } 170 | msgBytes, _ := json.Marshal(msg) 171 | return msgBytes 172 | } 173 | 174 | func (b *MockMQTTBase) processCommand(_ paho.Client, msg paho.Message) { 175 | defer msg.Ack() 176 | if b.reachable { 177 | split := strings.Split(msg.Topic(), "/") 178 | command := split[len(split)-1] 179 | switch command { 180 | case model.CommandHealth: 181 | go b.processHealth() 182 | case model.CommandInstallBiz: 183 | go b.processInstallBiz(msg.Payload()) 184 | case model.CommandUnInstallBiz: 185 | go b.processUnInstallBiz(msg.Payload()) 186 | case model.CommandQueryAllBiz: 187 | go b.processQueryAllBiz() 188 | } 189 | } 190 | } 191 | 192 | func (b *MockMQTTBase) processBaseline(_ paho.Client, msg paho.Message) { 193 | defer msg.Ack() 194 | var data []ark.BizModel 195 | json.Unmarshal(msg.Payload(), &data) 196 | for _, bizModel := range data { 197 | identity := getBizIdentity(bizModel) 198 | b.BizInfos[identity] = ark.ArkBizInfo{ 199 | BizName: bizModel.BizName, 200 | BizState: "ACTIVATED", 201 | BizVersion: bizModel.BizVersion, 202 | BizStateRecords: []ark.ArkBizStateRecord{}, 203 | } 204 | } 205 | b.Baseline = data 206 | } 207 | 208 | func (b *MockMQTTBase) processHealth() { 209 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/health", b.Env, b.BaseMetadata.Identity), 1, b.getHealthMsg()) 210 | } 211 | 212 | func (b *MockMQTTBase) processInstallBiz(msg []byte) { 213 | b.Lock() 214 | defer b.Unlock() 215 | request := ark.BizModel{} 216 | json.Unmarshal(msg, &request) 217 | identity := getBizIdentity(request) 218 | _, has := b.BizInfos[identity] 219 | if !has { 220 | b.BizInfos[identity] = ark.ArkBizInfo{ 221 | BizName: request.BizName, 222 | BizState: "RESOLVED", 223 | BizVersion: request.BizVersion, 224 | BizStateRecords: []ark.ArkBizStateRecord{}, 225 | } 226 | } 227 | } 228 | 229 | func (b *MockMQTTBase) processUnInstallBiz(msg []byte) { 230 | b.Lock() 231 | defer b.Unlock() 232 | request := ark.BizModel{} 233 | json.Unmarshal(msg, &request) 234 | delete(b.BizInfos, getBizIdentity(request)) 235 | // send to response 236 | resp := model.ArkMqttMsg[model.BizOperationResponse]{ 237 | PublishTimestamp: time.Now().UnixMilli(), 238 | Data: model.BizOperationResponse{ 239 | Command: model.CommandUnInstallBiz, 240 | BizName: request.BizName, 241 | BizVersion: request.BizVersion, 242 | Response: ark_service.ArkResponse[ark.ArkResponseData]{ 243 | Code: "SUCCESS", 244 | Data: ark.ArkResponseData{ 245 | ArkClientResponse: ark.ArkClientResponse{ 246 | Code: "SUCCESS", 247 | Message: "", 248 | BizInfos: nil, 249 | }, 250 | ElapsedSpace: 0, 251 | }, 252 | Message: "", 253 | ErrorStackTrace: "", 254 | }, 255 | }, 256 | } 257 | respBytes, _ := json.Marshal(resp) 258 | b.client.Pub(fmt.Sprintf(model.BaseBizOperationResponseTopic, b.Env, b.BaseMetadata.Identity), 1, respBytes) 259 | } 260 | 261 | func (b *MockMQTTBase) SendInvalidMessage() { 262 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/health", b.Env, b.BaseMetadata.Identity), 1, []byte("")) 263 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/biz", b.Env, b.BaseMetadata.Identity), 1, []byte("")) 264 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/heart", b.Env, b.BaseMetadata.Identity), 1, []byte("")) 265 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/queryBaseline", b.Env, b.BaseMetadata.Identity), 1, []byte("")) 266 | } 267 | 268 | func (b *MockMQTTBase) SendTimeoutMessage() { 269 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/health", b.Env, b.BaseMetadata.Identity), 1, []byte("{\"publishTimestamp\":0}")) 270 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/biz", b.Env, b.BaseMetadata.Identity), 1, []byte("{\"publishTimestamp\":0}")) 271 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/heart", b.Env, b.BaseMetadata.Identity), 1, []byte("{\"publishTimestamp\":0}")) 272 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/queryBaseline", b.Env, b.BaseMetadata.Identity), 1, []byte("{\"publishTimestamp\":0}")) 273 | } 274 | 275 | func (b *MockMQTTBase) SendFailedMessage() { 276 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/health", b.Env, b.BaseMetadata.Identity), 1, []byte(fmt.Sprintf("{\"publishTimestamp\":%d, \"data\" : {\"code\":\"\"}}", time.Now().UnixMilli()))) 277 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/biz", b.Env, b.BaseMetadata.Identity), 1, []byte(fmt.Sprintf("{\"publishTimestamp\":%d, \"data\" : {\"code\":\"\"}}", time.Now().UnixMilli()))) 278 | } 279 | 280 | func (b *MockMQTTBase) QueryBaseline() { 281 | queryBaselineBytes, _ := json.Marshal(model.ArkMqttMsg[model.BaseMetadata]{ 282 | PublishTimestamp: time.Now().UnixMilli(), 283 | Data: b.BaseMetadata, 284 | }) 285 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/queryBaseline", b.Env, b.BaseMetadata.Identity), 1, queryBaselineBytes) 286 | } 287 | 288 | func (b *MockMQTTBase) SetBizState(bizIdentity, state, reason, message string) { 289 | b.Lock() 290 | defer b.Unlock() 291 | info := b.BizInfos[bizIdentity] 292 | info.BizState = state 293 | info.BizStateRecords = append(info.BizStateRecords, ark.ArkBizStateRecord{ 294 | ChangeTime: 1234, 295 | State: state, 296 | Reason: reason, 297 | Message: message, 298 | }) 299 | b.BizInfos[bizIdentity] = info 300 | if state == "ACTIVATED" { 301 | // send to response 302 | resp := model.ArkMqttMsg[model.BizOperationResponse]{ 303 | PublishTimestamp: time.Now().UnixMilli(), 304 | Data: model.BizOperationResponse{ 305 | Command: model.CommandInstallBiz, 306 | BizName: info.BizName, 307 | BizVersion: info.BizVersion, 308 | Response: ark_service.ArkResponse[ark.ArkResponseData]{ 309 | Code: "SUCCESS", 310 | Data: ark.ArkResponseData{ 311 | ArkClientResponse: ark.ArkClientResponse{ 312 | Code: "SUCCESS", 313 | Message: "", 314 | BizInfos: nil, 315 | }, 316 | ElapsedSpace: 0, 317 | }, 318 | Message: "", 319 | ErrorStackTrace: "", 320 | }, 321 | }, 322 | } 323 | respBytes, _ := json.Marshal(resp) 324 | b.client.Pub(fmt.Sprintf(model.BaseBizOperationResponseTopic, b.Env, b.BaseMetadata.Identity), 1, respBytes) 325 | } 326 | // send simple all biz data 327 | arkBizInfos := make(model.ArkSimpleAllBizInfoData, 0) 328 | 329 | for _, bizInfo := range b.BizInfos { 330 | stateIndex := "4" 331 | if state == "ACTIVATED" { 332 | stateIndex = "3" 333 | } 334 | arkBizInfos = append(arkBizInfos, model.ArkSimpleBizInfoData{ 335 | Name: bizInfo.BizName, 336 | Version: bizInfo.BizVersion, 337 | State: stateIndex, 338 | }) 339 | } 340 | 341 | msg := model.ArkMqttMsg[model.ArkSimpleAllBizInfoData]{ 342 | PublishTimestamp: time.Now().UnixMilli(), 343 | Data: arkBizInfos, 344 | } 345 | msgBytes, _ := json.Marshal(msg) 346 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/simpleBiz", b.Env, b.BaseMetadata.Identity), 1, msgBytes) 347 | } 348 | 349 | func (b *MockMQTTBase) processQueryAllBiz() { 350 | b.client.Pub(fmt.Sprintf("koupleless_%s/%s/base/biz", b.Env, b.BaseMetadata.Identity), 1, b.getQueryAllBizMsg()) 351 | } 352 | 353 | func getBizIdentity(bizModel ark.BizModel) string { 354 | return fmt.Sprintf("%s:%s", bizModel.BizName, bizModel.BizVersion) 355 | } 356 | -------------------------------------------------------------------------------- /suite/mqtt/module_deployment_controller_suite_test.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "context" 5 | "github.com/koupleless/module_controller/common/model" 6 | vkModel "github.com/koupleless/virtual-kubelet/model" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | appsv1 "k8s.io/api/apps/v1" 10 | v1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/types" 13 | "k8s.io/utils/ptr" 14 | "time" 15 | ) 16 | 17 | var _ = Describe("Module Deployment Controller Test", func() { 18 | 19 | ctx := context.Background() 20 | 21 | clusterName := "test-cluster-name" 22 | 23 | mockBase := NewMockMqttBase("test-base", clusterName, "1.0.0", env) 24 | mockBase2 := NewMockMqttBase("test-base-2", clusterName, "1.0.0", env) 25 | 26 | deployment1 := appsv1.Deployment{ 27 | ObjectMeta: metav1.ObjectMeta{ 28 | Name: "test-base-deployment", 29 | Namespace: "default", 30 | Labels: map[string]string{ 31 | vkModel.LabelKeyOfComponent: model.ComponentModuleDeployment, 32 | vkModel.LabelKeyOfEnv: env, 33 | model.LabelKeyOfVPodDeploymentStrategy: string(model.VPodDeploymentStrategyPeer), 34 | }, 35 | }, 36 | Spec: appsv1.DeploymentSpec{ 37 | Replicas: ptr.To[int32](1), 38 | Selector: &metav1.LabelSelector{ 39 | MatchLabels: map[string]string{ 40 | vkModel.LabelKeyOfComponent: model.ComponentModule, 41 | "app": "test-biz-deployment", 42 | }, 43 | }, 44 | Template: v1.PodTemplateSpec{ 45 | ObjectMeta: metav1.ObjectMeta{ 46 | Labels: map[string]string{ 47 | vkModel.LabelKeyOfComponent: model.ComponentModule, 48 | "app": "test-biz-deployment", 49 | }, 50 | }, 51 | Spec: v1.PodSpec{ 52 | Containers: []v1.Container{ 53 | { 54 | Name: "test-biz", 55 | Image: "test-biz.jar", 56 | Env: []v1.EnvVar{ 57 | { 58 | Name: "BIZ_VERSION", 59 | Value: "1.0.0", 60 | }, 61 | }, 62 | }, 63 | }, 64 | NodeSelector: map[string]string{ 65 | vkModel.LabelKeyOfBaseVersion: "1.0.0", 66 | vkModel.LabelKeyOfBaseClusterName: clusterName, 67 | }, 68 | Tolerations: []v1.Toleration{ 69 | { 70 | Key: vkModel.TaintKeyOfVnode, 71 | Operator: v1.TolerationOpEqual, 72 | Value: "True", 73 | Effect: v1.TaintEffectNoExecute, 74 | }, 75 | { 76 | Key: vkModel.TaintKeyOfEnv, 77 | Operator: v1.TolerationOpEqual, 78 | Value: env, 79 | Effect: v1.TaintEffectNoExecute, 80 | }, 81 | }, 82 | }, 83 | }, 84 | }, 85 | } 86 | 87 | deployment2 := appsv1.Deployment{ 88 | ObjectMeta: metav1.ObjectMeta{ 89 | Name: "test-base-deployment-2", 90 | Namespace: "default", 91 | Labels: map[string]string{ 92 | vkModel.LabelKeyOfComponent: model.ComponentModuleDeployment, 93 | vkModel.LabelKeyOfEnv: env, 94 | model.LabelKeyOfVPodDeploymentStrategy: string(model.VPodDeploymentStrategyPeer), 95 | }, 96 | }, 97 | Spec: appsv1.DeploymentSpec{ 98 | Replicas: ptr.To[int32](1), 99 | Selector: &metav1.LabelSelector{ 100 | MatchLabels: map[string]string{ 101 | vkModel.LabelKeyOfComponent: model.ComponentModule, 102 | "app": "test-biz-deployment", 103 | }, 104 | }, 105 | Template: v1.PodTemplateSpec{ 106 | ObjectMeta: metav1.ObjectMeta{ 107 | Labels: map[string]string{ 108 | vkModel.LabelKeyOfComponent: model.ComponentModule, 109 | "app": "test-biz-deployment", 110 | }, 111 | }, 112 | Spec: v1.PodSpec{ 113 | Containers: []v1.Container{ 114 | { 115 | Name: "test-biz-2", 116 | Image: "test-biz-2.jar", 117 | Env: []v1.EnvVar{ 118 | { 119 | Name: "BIZ_VERSION", 120 | Value: "1.0.0", 121 | }, 122 | }, 123 | }, 124 | }, 125 | NodeSelector: map[string]string{ 126 | vkModel.LabelKeyOfBaseVersion: "1.0.0", 127 | vkModel.LabelKeyOfBaseClusterName: clusterName + "-2", 128 | }, 129 | Tolerations: []v1.Toleration{ 130 | { 131 | Key: vkModel.TaintKeyOfVnode, 132 | Operator: v1.TolerationOpEqual, 133 | Value: "True", 134 | Effect: v1.TaintEffectNoExecute, 135 | }, 136 | { 137 | Key: vkModel.TaintKeyOfEnv, 138 | Operator: v1.TolerationOpEqual, 139 | Value: env, 140 | Effect: v1.TaintEffectNoExecute, 141 | }, 142 | }, 143 | }, 144 | }, 145 | }, 146 | } 147 | 148 | Context("deployment publish and query baseline", func() { 149 | It("publish deployment and replicas should be zero", func() { 150 | err := k8sClient.Create(ctx, &deployment1) 151 | Expect(err).ToNot(HaveOccurred()) 152 | Eventually(func() bool { 153 | depFromKubernetes := &appsv1.Deployment{} 154 | err = k8sClient.Get(ctx, types.NamespacedName{ 155 | Name: deployment1.Name, 156 | Namespace: deployment1.Namespace, 157 | }, depFromKubernetes) 158 | return err == nil && *depFromKubernetes.Spec.Replicas == 0 159 | }, time.Second*20, time.Second).Should(BeTrue()) 160 | }) 161 | 162 | It("one node online and deployment replicas should be 1", func() { 163 | go mockBase.Start(ctx, clientID) 164 | Eventually(func() bool { 165 | depFromKubernetes := &appsv1.Deployment{} 166 | err := k8sClient.Get(ctx, types.NamespacedName{ 167 | Name: deployment1.Name, 168 | Namespace: deployment1.Namespace, 169 | }, depFromKubernetes) 170 | return err == nil && *depFromKubernetes.Spec.Replicas == 1 171 | }, time.Second*20, time.Second).Should(BeTrue()) 172 | }) 173 | 174 | It("another node online and deployment replicas should be 2", func() { 175 | go mockBase2.Start(ctx, clientID) 176 | Eventually(func() bool { 177 | depFromKubernetes := &appsv1.Deployment{} 178 | err := k8sClient.Get(ctx, types.NamespacedName{ 179 | Name: deployment1.Name, 180 | Namespace: deployment1.Namespace, 181 | }, depFromKubernetes) 182 | return err == nil && *depFromKubernetes.Spec.Replicas == 2 183 | }, time.Second*20, time.Second).Should(BeTrue()) 184 | }) 185 | 186 | It("publish a new deployment and replicas should be 0", func() { 187 | err := k8sClient.Create(ctx, &deployment2) 188 | Expect(err).ToNot(HaveOccurred()) 189 | Eventually(func() bool { 190 | depFromKubernetes := &appsv1.Deployment{} 191 | err = k8sClient.Get(ctx, types.NamespacedName{ 192 | Name: deployment2.Name, 193 | Namespace: deployment2.Namespace, 194 | }, depFromKubernetes) 195 | return err == nil && *depFromKubernetes.Spec.Replicas == 0 196 | }, time.Second*20, time.Second).Should(BeTrue()) 197 | }) 198 | 199 | It("mock base 2 query baseline should not fetch deployment2 containers", func() { 200 | mockBase2.QueryBaseline() 201 | Eventually(func() bool { 202 | return len(mockBase2.Baseline) == 1 203 | }, time.Second*20, time.Second).Should(BeTrue()) 204 | }) 205 | 206 | It("replicas should be 2", func() { 207 | mockBase2.Exit() 208 | Eventually(func() bool { 209 | depFromKubernetes := &appsv1.Deployment{} 210 | err := k8sClient.Get(ctx, types.NamespacedName{ 211 | Name: deployment1.Name, 212 | Namespace: deployment1.Namespace, 213 | }, depFromKubernetes) 214 | return err == nil && *depFromKubernetes.Spec.Replicas == 2 215 | }, time.Second*20, time.Second).Should(BeTrue()) 216 | }) 217 | 218 | It("mock base 2 exit and replicas should be 1 finally", func() { 219 | mockBase2.Exit() 220 | Eventually(func() bool { 221 | depFromKubernetes := &appsv1.Deployment{} 222 | err := k8sClient.Get(ctx, types.NamespacedName{ 223 | Name: deployment1.Name, 224 | Namespace: deployment1.Namespace, 225 | }, depFromKubernetes) 226 | return err == nil && *depFromKubernetes.Spec.Replicas == 1 227 | }, time.Second*20, time.Second).Should(BeTrue()) 228 | }) 229 | 230 | It("mock base exit and replicas should be 0 finally", func() { 231 | mockBase.Exit() 232 | Eventually(func() bool { 233 | depFromKubernetes := &appsv1.Deployment{} 234 | err := k8sClient.Get(ctx, types.NamespacedName{ 235 | Name: deployment1.Name, 236 | Namespace: deployment1.Namespace, 237 | }, depFromKubernetes) 238 | return err == nil && *depFromKubernetes.Spec.Replicas == 0 239 | }, time.Minute*2, time.Second).Should(BeTrue()) 240 | }) 241 | }) 242 | }) 243 | -------------------------------------------------------------------------------- /suite/mqtt/module_mqtt_lifecycle_test.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "context" 5 | "github.com/koupleless/virtual-kubelet/common/utils" 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | "github.com/virtual-kubelet/virtual-kubelet/log" 9 | v1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/api/errors" 11 | "k8s.io/apimachinery/pkg/types" 12 | "time" 13 | ) 14 | 15 | var _ = Describe("Module Lifecycle Test", func() { 16 | 17 | ctx := context.Background() 18 | 19 | nodeID := "test-base" 20 | clusterName := "test-cluster-name" 21 | nodeName := utils.FormatNodeName(nodeID, env) 22 | mockBase := NewMockMqttBase(nodeID, clusterName, "1.0.0", env) 23 | 24 | mockModulePod := prepareModulePod("test-module", "default", nodeName) 25 | 26 | Context("pod install", func() { 27 | It("base should become a ready vnode eventually", func() { 28 | go mockBase.Start(ctx, clientID) 29 | vnode := &v1.Node{} 30 | Eventually(func() bool { 31 | err := k8sClient.Get(ctx, types.NamespacedName{ 32 | Name: nodeName, 33 | }, vnode) 34 | if err != nil { 35 | log.G(ctx).Error(err, "get vnode error") 36 | return false 37 | } 38 | vnodeReady := false 39 | for _, cond := range vnode.Status.Conditions { 40 | if cond.Type == v1.NodeReady { 41 | vnodeReady = cond.Status == v1.ConditionTrue 42 | break 43 | } 44 | } 45 | return err == nil && vnodeReady 46 | }, time.Second*20, time.Second).Should(BeTrue()) 47 | }) 48 | 49 | It("publish a module pod and it should be pending", func() { 50 | err := k8sClient.Create(ctx, &mockModulePod) 51 | Expect(err).To(BeNil()) 52 | Eventually(func() bool { 53 | podFromKubernetes := &v1.Pod{} 54 | err := k8sClient.Get(ctx, types.NamespacedName{ 55 | Namespace: mockModulePod.Namespace, 56 | Name: mockModulePod.Name, 57 | }, podFromKubernetes) 58 | return err == nil && podFromKubernetes.Status.Phase == v1.PodPending && podFromKubernetes.Spec.NodeName == utils.FormatNodeName(nodeID, env) 59 | }, time.Second*20, time.Second).Should(BeTrue()) 60 | Eventually(func() bool { 61 | return len(mockBase.BizInfos) == 1 62 | }, time.Second*20, time.Second).Should(BeTrue()) 63 | }) 64 | 65 | It("when all module's status changes to ACTIVATED, pod should become ready", func() { 66 | mockBase.SetBizState("biz:0.0.1", "ACTIVATED", "ACTIVATED", "ACTIVATED") 67 | Eventually(func() bool { 68 | podFromKubernetes := &v1.Pod{} 69 | err := k8sClient.Get(ctx, types.NamespacedName{ 70 | Namespace: mockModulePod.Namespace, 71 | Name: mockModulePod.Name, 72 | }, podFromKubernetes) 73 | return err == nil && podFromKubernetes.Status.Phase == v1.PodRunning 74 | }, time.Second*30, time.Second).Should(BeTrue()) 75 | }) 76 | 77 | It("when one module's status changes to deactived, pod should become unready", func() { 78 | mockBase.SetBizState("biz:0.0.1", "DEACTIVATED", "test", "test") 79 | 80 | Eventually(func() bool { 81 | podFromKubernetes := &v1.Pod{} 82 | err := k8sClient.Get(ctx, types.NamespacedName{ 83 | Namespace: mockModulePod.Namespace, 84 | Name: mockModulePod.Name, 85 | }, podFromKubernetes) 86 | return err == nil && podFromKubernetes.Status.Phase == v1.PodRunning && podFromKubernetes.Status.Conditions[0].Status == v1.ConditionFalse 87 | }, time.Second*20, time.Second).Should(BeTrue()) 88 | }) 89 | 90 | It("delete pod, all modules should deactivated, pod should finally deleted from k8s", func() { 91 | err := k8sClient.Delete(ctx, &mockModulePod) 92 | Expect(err).To(BeNil()) 93 | Eventually(func() bool { 94 | podFromKubernetes := &v1.Pod{} 95 | err := k8sClient.Get(ctx, types.NamespacedName{ 96 | Namespace: mockModulePod.Namespace, 97 | Name: mockModulePod.Name, 98 | }, podFromKubernetes) 99 | return errors.IsNotFound(err) 100 | }, time.Second*20, time.Second).Should(BeTrue()) 101 | }) 102 | 103 | It("invalid message should skip by mqtt tunnel", func() { 104 | // invalid msg payload 105 | mockBase.SendInvalidMessage() 106 | mockBase.SendFailedMessage() 107 | mockBase.SendTimeoutMessage() 108 | time.Sleep(time.Second) 109 | }) 110 | 111 | It("base offline with deactive message and finally exit", func() { 112 | mockBase.Exit() 113 | Eventually(func() bool { 114 | vnode := &v1.Node{} 115 | err := k8sClient.Get(ctx, types.NamespacedName{ 116 | Name: utils.FormatNodeName(nodeID, env), 117 | }, vnode) 118 | return errors.IsNotFound(err) 119 | }, time.Second*30, time.Second).Should(BeTrue()) 120 | }) 121 | }) 122 | 123 | }) 124 | -------------------------------------------------------------------------------- /suite/mqtt/suite_test.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | model2 "github.com/koupleless/module_controller/common/model" 7 | "github.com/koupleless/module_controller/controller/module_deployment_controller" 8 | "github.com/koupleless/module_controller/module_tunnels/koupleless_mqtt_tunnel" 9 | "github.com/koupleless/virtual-kubelet/model" 10 | "github.com/koupleless/virtual-kubelet/vnode_controller" 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | "github.com/sirupsen/logrus" 14 | "github.com/virtual-kubelet/virtual-kubelet/log" 15 | logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" 16 | "github.com/wind-c/comqtt/v2/mqtt" 17 | "github.com/wind-c/comqtt/v2/mqtt/hooks/auth" 18 | "github.com/wind-c/comqtt/v2/mqtt/listeners" 19 | v1 "k8s.io/api/core/v1" 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/client-go/kubernetes/scheme" 22 | "k8s.io/client-go/rest" 23 | "os" 24 | ctrl "sigs.k8s.io/controller-runtime" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | "sigs.k8s.io/controller-runtime/pkg/envtest" 27 | logf "sigs.k8s.io/controller-runtime/pkg/log" 28 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 29 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 30 | "testing" 31 | "time" 32 | ) 33 | 34 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 35 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 36 | 37 | var cfg *rest.Config 38 | var testEnv *envtest.Environment 39 | var k8sClient client.Client 40 | var mqttTunnel koupleless_mqtt_tunnel.MqttTunnel 41 | var mqttServer *mqtt.Server 42 | var ctx, cancel = context.WithCancel(context.Background()) 43 | 44 | const ( 45 | clientID = "suite-test" 46 | env = "suite" 47 | ) 48 | 49 | func TestControllers(t *testing.T) { 50 | RegisterFailHandler(Fail) 51 | 52 | RunSpecs(t, "Module Controller Suite") 53 | } 54 | 55 | var _ = BeforeSuite(func() { 56 | 57 | os.Setenv("MQTT_BROKER", "localhost") 58 | os.Setenv("MQTT_PORT", "1883") 59 | 60 | os.Setenv("MQTT_USERNAME", "test") 61 | os.Setenv("MQTT_PASSWORD", "") 62 | os.Setenv("MQTT_CLIENT_PREFIX", "suite-test") 63 | os.Setenv("ENABLE_MODULE_REPLICAS_SYNC_WITH_BASE", "true") 64 | 65 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 66 | log.L = logruslogger.FromLogrus(logrus.NewEntry(logrus.StandardLogger())) 67 | 68 | By("bootstrapping suite environment") 69 | //usingExistingCluster := true 70 | testEnv = &envtest.Environment{ 71 | //UseExistingCluster: &usingExistingCluster, 72 | } 73 | 74 | var err error 75 | 76 | cfg, err = testEnv.Start() 77 | Expect(err).NotTo(HaveOccurred()) 78 | Expect(cfg).NotTo(BeNil()) 79 | 80 | // start embedded mqtt broker 81 | // Create the new MQTT Server. 82 | mqttServer = mqtt.New(nil) 83 | 84 | // Allow all connections. 85 | _ = mqttServer.AddHook(new(auth.AllowHook), nil) 86 | 87 | // Create a TCP listener on a standard port. 88 | tcp := listeners.NewTCP("t1", ":1883", nil) 89 | err = mqttServer.AddListener(tcp) 90 | Expect(err).NotTo(HaveOccurred()) 91 | 92 | go func() { 93 | err := mqttServer.Serve() 94 | if err != nil { 95 | log.G(ctx).Error("failed to start mqtt server") 96 | panic(err) 97 | } 98 | }() 99 | 100 | err = scheme.AddToScheme(scheme.Scheme) 101 | Expect(err).NotTo(HaveOccurred()) 102 | 103 | k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ 104 | Scheme: scheme.Scheme, 105 | Metrics: metricsserver.Options{ 106 | BindAddress: ":8080", 107 | }, 108 | }) 109 | Expect(err).ToNot(HaveOccurred()) 110 | 111 | moduleDeploymentController, err := module_deployment_controller.NewModuleDeploymentController(env) 112 | Expect(err).ToNot(HaveOccurred()) 113 | 114 | err = moduleDeploymentController.SetupWithManager(ctx, k8sManager) 115 | 116 | Expect(err).ToNot(HaveOccurred()) 117 | 118 | mqttTunnel = koupleless_mqtt_tunnel.NewMqttTunnel(ctx, env, k8sManager.GetClient(), moduleDeploymentController) 119 | err = mqttTunnel.Start(clientID, env) 120 | if err != nil { 121 | log.G(ctx).WithError(err).Error("failed to start tunnel", mqttTunnel.Key()) 122 | panic(fmt.Sprintf("failed to start tunnel %s", mqttTunnel.Key())) 123 | } else { 124 | log.G(ctx).Info("Tunnel started: ", mqttTunnel.Key()) 125 | } 126 | 127 | vnodeController, err := vnode_controller.NewVNodeController(&model.BuildVNodeControllerConfig{ 128 | ClientID: clientID, 129 | Env: env, 130 | VPodType: model2.ComponentModule, 131 | VNodeWorkerNum: 4, 132 | }, &mqttTunnel) 133 | Expect(err).ToNot(HaveOccurred()) 134 | 135 | err = vnodeController.SetupWithManager(ctx, k8sManager) 136 | 137 | k8sClient = k8sManager.GetClient() 138 | Expect(k8sClient).ToNot(BeNil()) 139 | 140 | go func() { 141 | err = k8sManager.Start(ctx) 142 | log.G(ctx).WithError(err).Error("k8sManager Start") 143 | Expect(err).ToNot(HaveOccurred()) 144 | }() 145 | 146 | k8sManager.GetCache().WaitForCacheSync(ctx) 147 | }) 148 | 149 | var _ = AfterSuite(func() { 150 | By("tearing down the suite environment") 151 | mqttServer.Close() 152 | cancel() 153 | testEnv.Stop() 154 | time.Sleep(15 * time.Second) 155 | log.G(ctx).Info("suite for mqtt stopped!") 156 | }) 157 | 158 | func prepareModulePod(name, namespace, nodeName string) v1.Pod { 159 | return v1.Pod{ 160 | ObjectMeta: metav1.ObjectMeta{ 161 | Name: name, 162 | Namespace: namespace, 163 | Labels: map[string]string{ 164 | model.LabelKeyOfComponent: model2.ComponentModule, 165 | }, 166 | }, 167 | Spec: v1.PodSpec{ 168 | NodeName: nodeName, 169 | Containers: []v1.Container{ 170 | { 171 | Name: "biz", 172 | Image: "suite-biz.jar", 173 | Env: []v1.EnvVar{ 174 | { 175 | Name: "BIZ_VERSION", 176 | Value: "0.0.1", 177 | }, 178 | }, 179 | }, 180 | }, 181 | Tolerations: []v1.Toleration{ 182 | { 183 | Key: model.TaintKeyOfVnode, 184 | Operator: v1.TolerationOpEqual, 185 | Value: "True", 186 | Effect: v1.TaintEffectNoExecute, 187 | }, 188 | { 189 | Key: model.TaintKeyOfEnv, 190 | Operator: v1.TolerationOpEqual, 191 | Value: env, 192 | Effect: v1.TaintEffectNoExecute, 193 | }, 194 | }, 195 | }, 196 | } 197 | } 198 | --------------------------------------------------------------------------------