├── .github ├── gh-bot.yml └── workflows │ ├── bot.yml │ ├── pages.yml │ ├── pr.yml │ ├── push.yml │ └── release.yml ├── .gitignore ├── .goreleaser.yml ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── apis └── network │ └── v1beta1 │ ├── doc.go │ ├── register.go │ ├── types.go │ └── zz_generated.deepcopy.go ├── cmd ├── cepctl │ ├── app │ │ ├── client.go │ │ └── options │ │ │ └── options.go │ └── cepctl.go └── endpoints-operator │ ├── app │ ├── options │ │ └── options.go │ └── server.go │ └── endpoints-operator.go ├── config ├── charts │ └── endpoints-operator │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── crds │ │ └── sealos.io_clusterendpoints.yaml │ │ ├── templates │ │ ├── _helpers.tpl │ │ ├── deployment.yaml │ │ ├── rbac.yaml │ │ └── service.yaml │ │ └── values.yaml ├── crd │ └── sealos.io_clusterendpoints.yaml └── demo │ └── dmz-kube.yaml ├── controllers ├── controller.go ├── helper.go ├── install.go ├── predicate.go ├── run_probe.go ├── sync.go └── sync_test.go ├── dockerfiles ├── cepctl │ └── Dockerfile └── endpoints-operator │ └── Dockerfile ├── go.mod ├── go.sum ├── hack ├── LICENSE └── boilerplate.go.txt ├── helm.sh ├── main.go ├── test ├── concurrent_test.go ├── test.md └── testhelper │ └── kube.go └── utils ├── client ├── cep.go ├── kube.go └── utils.go ├── file └── file.go └── metrics ├── metrics.go └── types.go /.github/gh-bot.yml: -------------------------------------------------------------------------------- 1 | version: v1 2 | debug: true 3 | action: 4 | printConfig: false 5 | release: 6 | retry: 15s 7 | actionName: Release 8 | allowOps: 9 | - cuisongliu 10 | bot: 11 | prefix: / 12 | spe: _ 13 | allowOps: 14 | - sealos-ci-robot 15 | - sealos-release-robot 16 | email: sealos-ci-robot@sealos.io 17 | username: sealos-ci-robot 18 | repo: 19 | org: false 20 | 21 | message: 22 | success: | 23 | 🤖 says: Hooray! The action {{.Body}} has been completed successfully. 🎉 24 | format_error: | 25 | 🤖 says: ‼️ There is a formatting issue with the action, kindly verify the action's format. 26 | permission_error: | 27 | 🤖 says: ‼️ The action doesn't have permission to trigger. 28 | release_error: | 29 | 🤖 says: ‼️ Release action failed. 30 | Error details: {{.Error}} 31 | -------------------------------------------------------------------------------- /.github/workflows/bot.yml: -------------------------------------------------------------------------------- 1 | name: Github Robot 2 | on: 3 | issue_comment: 4 | types: 5 | - created 6 | env: 7 | GH_TOKEN: "${{ secrets.GH_PAT }}" 8 | jobs: 9 | comment: 10 | if: startswith(github.event.comment.body, '/') 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout the latest code 14 | uses: actions/checkout@v3 15 | - name: Gh Robot for Sealos 16 | uses: labring/robot@v2.0.0 17 | with: 18 | version: v2.0.0 19 | env: 20 | SEALOS_TYPE: "/comment" 21 | GH_TOKEN: "${{ secrets.GH_PAT }}" 22 | -------------------------------------------------------------------------------- /.github/workflows/pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Allows you to run this workflow manually from the Actions tab 6 | workflow_dispatch: 7 | repository_dispatch: 8 | types: [ release_success ] 9 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 10 | permissions: 11 | contents: read 12 | pages: write 13 | id-token: write 14 | 15 | # Allow one concurrent deployment 16 | concurrency: 17 | group: "pages" 18 | cancel-in-progress: true 19 | 20 | jobs: 21 | # Build job 22 | build: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v3 27 | - name: Setup Pages 28 | uses: actions/configure-pages@v1 29 | - name: Install Helm 30 | uses: azure/setup-helm@v1 31 | with: 32 | version: v3.8.1 33 | 34 | - name: Build with Jekyll 35 | uses: actions/jekyll-build-pages@v1 36 | with: 37 | source: ./ 38 | destination: ./_site 39 | - name: Helm package 40 | run: | 41 | if [[ "${{ github.event_name }}" == "repository_dispatch" ]]; then 42 | bash helm.sh ${{ github.event.client_payload.version }} 43 | elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then 44 | bash helm.sh 45 | fi 46 | sudo cp index.yaml _site/ 47 | - name: Upload artifact 48 | uses: actions/upload-pages-artifact@v1 49 | 50 | # Deployment job 51 | deploy: 52 | environment: 53 | name: github-pages 54 | url: ${{ steps.deployment.outputs.page_url }} 55 | runs-on: ubuntu-latest 56 | needs: build 57 | steps: 58 | - name: Deploy to GitHub Pages 59 | id: deployment 60 | uses: actions/deploy-pages@v1 61 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: Go for PR 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | goreleaser: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@master 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@master 16 | with: 17 | go-version: 1.20.x 18 | 19 | - name: Prepare 20 | id: prepare 21 | run: | 22 | TAG=${GITHUB_REF#refs/tags/} 23 | echo ::set-output name=tag_name::${TAG} 24 | - name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v1 26 | with: 27 | version: latest 28 | args: release --snapshot --rm-dist --timeout=1h 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} 31 | VERSION: latest 32 | -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: Go for Push 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | jobs: 8 | goreleaser: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@master 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@master 16 | with: 17 | go-version: 1.20.x 18 | 19 | - name: Prepare 20 | id: prepare 21 | run: | 22 | TAG=${GITHUB_REF#refs/tags/} 23 | echo ::set-output name=tag_name::${TAG} 24 | - name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v1 26 | with: 27 | version: latest 28 | args: release --snapshot --rm-dist --timeout=1h 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} 31 | VERSION: latest 32 | - name: set up buildx 33 | id: buildx 34 | uses: crazy-max/ghaction-docker-buildx@v1 35 | with: 36 | version: latest 37 | 38 | - name: Login to Docker Hub 39 | uses: docker/login-action@v1 40 | with: 41 | registry: ghcr.io 42 | username: ${{ github.repository_owner }} 43 | password: ${{ secrets.GITHUB_TOKEN }} 44 | - name: build (and publish) main image 45 | env: 46 | # fork friendly ^^ 47 | DOCKER_REPO: ghcr.io/${{ github.repository_owner }}/endpoints-operator 48 | run: | 49 | docker buildx build \ 50 | --platform linux/amd64,linux/arm64 \ 51 | --build-arg VERSION=latest \ 52 | --push \ 53 | -t ${DOCKER_REPO}:latest \ 54 | -f dockerfiles/endpoints-operator/Dockerfile \ 55 | . 56 | - name: build (and publish) cepctl image 57 | env: 58 | # fork friendly ^^ 59 | DOCKER_REPO: ghcr.io/${{ github.repository_owner }}/cepctl 60 | run: | 61 | docker buildx build \ 62 | --platform linux/amd64,linux/arm64 \ 63 | --build-arg VERSION=latest \ 64 | --push \ 65 | -t ${DOCKER_REPO}:latest \ 66 | -f dockerfiles/cepctl/Dockerfile \ 67 | . 68 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - '**' 7 | tags: 8 | - '*' 9 | 10 | jobs: 11 | goreleaser: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@master 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@master 19 | with: 20 | go-version: 1.20.x 21 | 22 | - name: Prepare 23 | id: prepare 24 | run: | 25 | TAG=${GITHUB_REF#refs/tags/} 26 | echo ::set-output name=tag_name::${TAG} 27 | 28 | - name: Install Helm 29 | uses: azure/setup-helm@v1 30 | with: 31 | version: v3.8.1 32 | 33 | - name: Helm package 34 | run: | 35 | latest_release=${{ steps.prepare.outputs.tag_name }} 36 | sed -i "s#latest#${latest_release}#g" config/charts/endpoints-operator/values.yaml 37 | sed -i "s#0.0.0#${latest_release#v}#g" config/charts/endpoints-operator/Chart.yaml 38 | helm package config/charts/endpoints-operator 39 | git checkout . 40 | 41 | - name: Run GoReleaser 42 | uses: goreleaser/goreleaser-action@v1 43 | with: 44 | version: latest 45 | args: release --rm-dist --timeout=1h 46 | env: 47 | VERSION: ${{ steps.prepare.outputs.tag_name }} 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | 50 | - name: set up buildx 51 | id: buildx 52 | uses: crazy-max/ghaction-docker-buildx@v1 53 | with: 54 | version: latest 55 | 56 | - name: Login to Docker Hub 57 | uses: docker/login-action@v1 58 | with: 59 | registry: ghcr.io 60 | username: ${{ github.repository_owner }} 61 | password: ${{ secrets.GITHUB_TOKEN }} 62 | - name: build (and publish) main image 63 | env: 64 | # fork friendly ^^ 65 | DOCKER_REPO: ghcr.io/${{ github.repository_owner }}/endpoints-operator 66 | run: | 67 | docker buildx build \ 68 | --platform linux/amd64,linux/arm64 \ 69 | --build-arg VERSION=${{ steps.prepare.outputs.tag_name }} \ 70 | --push \ 71 | -t ${DOCKER_REPO}:${{ steps.prepare.outputs.tag_name }} \ 72 | -f dockerfiles/endpoints-operator/Dockerfile \ 73 | . 74 | docker buildx build \ 75 | --platform linux/amd64,linux/arm64 \ 76 | --build-arg VERSION=latest \ 77 | --push \ 78 | -t ${DOCKER_REPO}:latest \ 79 | -f dockerfiles/endpoints-operator/Dockerfile \ 80 | . 81 | - name: build (and publish) cepctl image 82 | env: 83 | # fork friendly ^^ 84 | DOCKER_REPO: ghcr.io/${{ github.repository_owner }}/cepctl 85 | run: | 86 | docker buildx build \ 87 | --platform linux/amd64,linux/arm64 \ 88 | --build-arg VERSION=${{ steps.prepare.outputs.tag_name }} \ 89 | --push \ 90 | -t ${DOCKER_REPO}:${{ steps.prepare.outputs.tag_name }} \ 91 | -f dockerfiles/cepctl/Dockerfile \ 92 | . 93 | docker buildx build \ 94 | --platform linux/amd64,linux/arm64 \ 95 | --build-arg VERSION=latest \ 96 | --push \ 97 | -t ${DOCKER_REPO}:latest \ 98 | -f dockerfiles/cepctl/Dockerfile \ 99 | . 100 | 101 | trigger-workflow-build-helm-chart: 102 | needs: [ goreleaser ] 103 | runs-on: ubuntu-latest 104 | steps: 105 | - name: Set env 106 | run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 107 | - name: Trigger cluster image workflow 108 | uses: peter-evans/repository-dispatch@v2 109 | with: 110 | event-type: release_success 111 | client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "version": "${{ env.RELEASE_VERSION }}"}' 112 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | bin 17 | .idea 18 | dist 19 | endpoints-operator*.tgz 20 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | # You may remove this if you don't use go modules. 4 | - go mod download 5 | # you may remove this if you don't need go generate 6 | - go generate ./... 7 | builds: 8 | - env: 9 | - CGO_ENABLED=0 10 | goos: 11 | - linux 12 | - darwin 13 | goarch: 14 | - amd64 15 | - arm64 16 | main: ./cmd/endpoints-operator 17 | ldflags: 18 | - -X github.com/labring/operator-sdk/version.gitVersion={{.Version}} 19 | - -X github.com/labring/operator-sdk/version.gitCommit={{.ShortCommit}} 20 | - -X github.com/labring/operator-sdk/version.buildDate={{.Date}} 21 | - -s -w 22 | - env: 23 | - CGO_ENABLED=0 24 | id: cepctl 25 | binary: cepctl 26 | goos: 27 | - linux 28 | - darwin 29 | goarch: 30 | - amd64 31 | - arm64 32 | main: ./cmd/cepctl 33 | ldflags: 34 | - -X github.com/labring/operator-sdk/version.gitVersion={{.Version}} 35 | - -X github.com/labring/operator-sdk/version.gitCommit={{.ShortCommit}} 36 | - -X github.com/labring/operator-sdk/version.buildDate={{.Date}} 37 | - -s -w 38 | release: 39 | prerelease: auto 40 | extra_files: 41 | - glob: ./endpoints-operator*.tgz 42 | checksum: 43 | name_template: '{{ .ProjectName }}_checksums.txt' 44 | snapshot: 45 | name_template: "{{ .Tag }}-next" 46 | changelog: 47 | sort: asc 48 | filters: 49 | exclude: 50 | - '^docs:' 51 | - '^test:' 52 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # ChangeLog 2 | 3 | ## [v0.1.0](#v0.1.0) 4 | 5 | - 设计完成CRD `ClusterEndpoint` 来维护service和endpoint功能 6 | - 完成controller的探活功能，支持HTTP和TCP Port，使用controller维护CRD的status数据 7 | - 完善部署的helm chart 8 | 9 | ## [v0.1.1](#v0.1.1) 10 | 11 | - 支持GRPC功能 12 | - 支持UDP功能 13 | - 新增对应的监控数据 14 | - 新增cepctl工具做数据转换 15 | - 新增webhook证书维护代码 16 | 17 | ## [v0.2.0](#v0.2.0) 18 | 19 | - 所有的domain更正为sealos.io 20 | - FixBug #86 不支持clusterIP=None 21 | 22 | ## [v0.2.1](#v0.2.1) 23 | 24 | - FixBug #103 endpoint 无法更新的问题 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | Dirs=$(shell ls) 2 | COMMIT_ID ?= $(shell git rev-parse --short HEAD || echo "0.0.0") 3 | 4 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 5 | ifneq (,$(shell go env GOBIN)) 6 | GOBIN=$(shell go env GOPATH)/bin 7 | else 8 | GOBIN=$(shell go env GOBIN) 9 | endif 10 | GOPATH=$(shell go env GOPATH) 11 | SHELL = /usr/bin/env bash -o pipefail 12 | .SHELLFLAGS = -ec 13 | 14 | PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 15 | define go-get-tool 16 | @[ -f $(1) ] || { \ 17 | set -e ;\ 18 | TMP_DIR=$$(mktemp -d) ;\ 19 | cd $$TMP_DIR ;\ 20 | go mod init tmp ;\ 21 | echo "Downloading $(2)" ;\ 22 | GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ 23 | rm -rf $$TMP_DIR ;\ 24 | } 25 | endef 26 | 27 | default: build 28 | 29 | 30 | GORELEASER_BIN = $(shell pwd)/bin/goreleaser 31 | install-goreleaser: ## check license if not exist install go-lint tools 32 | $(call go-get-tool,$(GORELEASER_BIN),github.com/goreleaser/goreleaser@v1.6.3) 33 | 34 | 35 | build: SHELL:=/bin/bash 36 | build: install-goreleaser clean ## build binaries by default 37 | @echo "build endpoints-operator bin" 38 | $(GORELEASER_BIN) build --snapshot --rm-dist --timeout=1h 39 | 40 | help: ## this help 41 | @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) 42 | 43 | clean: ## clean 44 | rm -rf dist 45 | 46 | ADDLICENSE_BIN = $(shell pwd)/bin/addlicense 47 | install-addlicense: ## check license if not exist install go-lint tools 48 | $(call go-get-tool,$(ADDLICENSE_BIN),github.com/google/addlicense@latest) 49 | 50 | filelicense: 51 | filelicense: install-addlicense 52 | for file in ${Dirs} ; do \ 53 | if [[ $$file != '_output' && $$file != 'docs' && $$file != 'vendor' && $$file != 'logger' && $$file != 'applications' ]]; then \ 54 | $(ADDLICENSE_BIN) -y $(shell date +"%Y") -c "The sealos Authors." -f hack/LICENSE ./$$file ; \ 55 | fi \ 56 | done 57 | 58 | 59 | DEEPCOPY_BIN = $(shell pwd)/bin/deepcopy-gen 60 | install-deepcopy: ## check license if not exist install go-lint tools 61 | $(call go-get-tool,$(DEEPCOPY_BIN),k8s.io/code-generator/cmd/deepcopy-gen@v0.23.6) 62 | 63 | HEAD_FILE := hack/boilerplate.go.txt 64 | INPUT_DIR := github.com/labring/endpoints-operator/api/network/v1beta1 65 | deepcopy:install-deepcopy 66 | $(DEEPCOPY_BIN) \ 67 | --input-dirs="$(INPUT_DIR)" \ 68 | -O zz_generated.deepcopy \ 69 | --go-header-file "$(HEAD_FILE)" \ 70 | --output-base "${GOPATH}/src" 71 | CONTROLLER_GEN = $(shell pwd)/bin/controller-gen 72 | 73 | .PHONY: controller-gen 74 | controller-gen: ## Download controller-gen locally if necessary. 75 | $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) 76 | 77 | .PHONY: manifests 78 | manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. 79 | $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd 80 | $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/charts/endpoints-operator/crds 81 | 82 | .PHONY: generate 83 | generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 84 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 85 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: sealos.io 2 | layout: 3 | - go.kubebuilder.io/v3 4 | multigroup: true 5 | projectName: endpoints-operator 6 | repo: github.com/labring/endpoints-operator 7 | resources: 8 | - api: 9 | crdVersion: v1 10 | namespaced: true 11 | controller: true 12 | domain: sealos.io 13 | group: network 14 | kind: ClusterEndpoint 15 | path: github.com/labring/endpoints-operator/apis/network/v1beta1 16 | version: v1beta1 17 | version: "3" 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # endpoints-operator 2 | 3 | > 对于集群内访问集群外部服务场景使用固定的endpoint维护增加探活功能 4 | 5 | ### 注意事项: 6 | 7 | - v0.1.1 版本的数据是sealyun.com的domain 8 | - v0.2.0 之后所有的domain都是sealos.io 9 | - v0.2.1 调整了Hosts配置，升级需要注意一下 10 | 11 | 也可以手动执行一下脚本,namespace为xxx 12 | ```shell 13 | for cep in $(kubectl get cep -n xxx -o jsonpath={.items[*].metadata.name});do kubectl patch cep -n xxx --type='json' -p='[{"op": "replace", "path": "/metadata/finalizers", "value":[]}]' $cep;done 14 | ``` 15 | 16 | 升级资源之前最好先备份一下cr删除之后再重新创建即可。 17 | 18 | ## 背景 19 | 20 | - 在实际使用中，两个K8s集群之间的服务经常有互相访问和访问集群外部某些服务的需求，通常的解决方案为手动维护固定的Services和Endpoints或者直接在业务配置中写死IP，在这时候，是没有对外部服务进行探活的功能的，无法做到高可用。如果需要高可用一般是引入外部高可用LB来解决，但这样增加了复杂度，且好多公司不具备引入条件，不是最优解决方案。 21 | - 众所周知，Kube-Proxy的主要功能是维护集群内Services和Endpoints并在对应主机上创建对应的IPVS规则。从而达到我们可以在Pod里面通过ClusterIP访问的目的。 22 | 23 | 由此，新的想法诞生了: 写一个controller，维护一个CRD来自动创建需要访问的外部服务对应的Service和Endpoint，并对创建的Endpoint中的外部服务数据（IP:PORT列表）进行探活，探活失败则移除对应的外部服务数据。 24 | 25 | ## 介绍 26 | 27 | endpoints-operator是一个云原生、高可靠性、高性能、面向K8s内部服务访问外部服务的具备探活功能的4层LB。 28 | 29 | ### 特性 30 | 31 | - 更加贴近云原生 32 | - 声明式API：探活的定义方式与Kubelet保持一致，还是熟悉的语法、熟悉的味道 33 | - 高可靠性：原生Service、Endpoint资源，拒绝重复造轮子 34 | - 高性能、高稳定：原生IPVS高性能4层负载均衡 35 | 36 | 37 | 38 | ### 核心优势 39 | 40 | - 完全使用K8s原生的Service、Endpoint资源，无自定义IPVS策略，依托K8s的Service能力，高可靠。 41 | - 通过controller管理一个CRD资源ClusterEndpoint（缩写cep）即可，无需手动管理Service和Endpoint两个资源 42 | - 完全兼容已有的自定义Service、Endpoint资源，可无缝切换至endpoints-operator管理。 43 | - 原生的IPVS 4层负载，未引入Nginx、HAProxy等LB，降低了复杂度，满足高性能和高稳定性的需求 44 | 45 | ### 使用场景 46 | 47 | 主要使用在集群内部的Pod需要访问外部服务的场景，比如数据库、中间件等，通过endpoints-operator的探活能力，可及时将有问题的后端服务剔除，避免受单个宕机副本影响，并可查看status获取后端服务健康状态和探活失败的服务。 48 | 49 | ## helm 安装 50 | 51 | ```bash 52 | VERSION="0.2.1" 53 | wget https://github.com/labring/endpoints-operator/releases/download/v${VERSION}/endpoints-operator-${VERSION}.tgz 54 | helm install -n kube-system endpoints-operator ./endpoints-operator-${VERSION}.tgz 55 | ``` 56 | 57 | ## sealos 安装 58 | 59 | ```bash 60 | sealos run labring/endpoints-operator:v0.2.1 61 | ``` 62 | 63 | ## Usage 64 | 65 | ```yaml 66 | apiVersion: sealos.io/v1beta1 67 | kind: ClusterEndpoint 68 | metadata: 69 | name: wordpress 70 | namespace: default 71 | spec: 72 | periodSeconds: 10 73 | ports: 74 | - name: wp-https 75 | hosts: 76 | ## 端口相同的hosts 77 | - 10.33.40.151 78 | - 10.33.40.152 79 | protocol: TCP 80 | port: 38081 81 | targetPort: 443 82 | tcpSocket: 83 | enable: true 84 | timeoutSeconds: 1 85 | failureThreshold: 3 86 | successThreshold: 1 87 | - name: wp-http 88 | hosts: 89 | ## 端口相同的hosts 90 | - 10.33.40.151 91 | - 10.33.40.152 92 | protocol: TCP 93 | port: 38082 94 | targetPort: 80 95 | httpGet: 96 | path: /healthz 97 | scheme: http 98 | timeoutSeconds: 1 99 | failureThreshold: 3 100 | successThreshold: 1 101 | - name: wp-udp 102 | hosts: 103 | ## 端口相同的hosts 104 | - 10.33.40.151 105 | - 10.33.40.152 106 | protocol: UDP 107 | port: 38003 108 | targetPort: 1234 109 | udpSocket: 110 | enable: true 111 | data: "This is flag data for UDP svc test" 112 | timeoutSeconds: 1 113 | failureThreshold: 3 114 | successThreshold: 1 115 | - name: wp-grpc 116 | hosts: 117 | ## 端口相同的hosts 118 | - 10.33.40.151 119 | - 10.33.40.152 120 | protocol: TCP 121 | port: 38083 122 | targetPort: 8080 123 | grpc: 124 | enable: true 125 | timeoutSeconds: 1 126 | failureThreshold: 3 127 | successThreshold: 1 128 | ``` 129 | 130 | ## 总结 131 | "endpoints-operator” 的引入，对产品无侵入以及云原生等特性解决了在集群内部访问外部服务等问题。这个思路将会成为以后开发或者运维的标配，也是一个比较完善的项目，从开发的角度换个思路更优雅的去解决一些问题。 132 | -------------------------------------------------------------------------------- /apis/network/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Code generated by api-gen. EDIT THIS FILE! 16 | 17 | package v1beta1 18 | 19 | // +k8s:deepcopy-gen=package,register 20 | // +k8s:protobuf-gen=package 21 | // +k8s:openapi-gen=false 22 | // +k8s:defaulter-gen=TypeMeta 23 | 24 | // +groupName=sealos.io 25 | -------------------------------------------------------------------------------- /apis/network/v1beta1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by register-gen. DO NOT EDIT. 17 | 18 | package v1beta1 19 | 20 | import ( 21 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | ) 25 | 26 | // GroupName specifies the group name used to register the objects. 27 | const GroupName = "sealos.io" 28 | 29 | // GroupVersion specifies the group and the version used to register the objects. 30 | var GroupVersion = v1.GroupVersion{Group: GroupName, Version: "v1beta1"} 31 | 32 | // SchemeGroupVersion is group version used to register these objects 33 | // Deprecated: use GroupVersion instead. 34 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} 35 | 36 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 37 | func Resource(resource string) schema.GroupResource { 38 | return SchemeGroupVersion.WithResource(resource).GroupResource() 39 | } 40 | 41 | var ( 42 | // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. 43 | SchemeBuilder runtime.SchemeBuilder 44 | localSchemeBuilder = &SchemeBuilder 45 | // Depreciated: use Install instead 46 | AddToScheme = localSchemeBuilder.AddToScheme 47 | Install = localSchemeBuilder.AddToScheme 48 | ) 49 | 50 | func init() { 51 | // We only register manually written functions here. The registration of the 52 | // generated functions takes place in the generated files. The separation 53 | // makes the code compile even when the generated files are missing. 54 | localSchemeBuilder.Register(addKnownTypes) 55 | } 56 | 57 | // Adds the list of known types to Scheme. 58 | func addKnownTypes(scheme *runtime.Scheme) error { 59 | scheme.AddKnownTypes(SchemeGroupVersion, 60 | &ClusterEndpoint{}, 61 | &ClusterEndpointList{}, 62 | ) 63 | // AddToGroupVersion allows the serialization of client types like ListOptions. 64 | v1.AddToGroupVersion(scheme, SchemeGroupVersion) 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /apis/network/v1beta1/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by api-gen. EDIT THIS FILE! 18 | 19 | package v1beta1 20 | 21 | import ( 22 | v1 "k8s.io/api/core/v1" 23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | ) 25 | 26 | // ServicePort contains information on service's port. 27 | type ServicePort struct { 28 | Hosts []string `json:"hosts,omitempty" patchStrategy:"merge" patchMergeKey:"host" protobuf:"bytes,3,rep,name=hosts"` 29 | // The action taken to determine the health of a container 30 | Handler `json:",inline" protobuf:"bytes,1,opt,name=handler"` 31 | // Number of seconds after which the probe times out. 32 | // Defaults to 1 second. Minimum value is 1. 33 | // More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes 34 | // +optional 35 | TimeoutSeconds int32 `json:"timeoutSeconds,omitempty" protobuf:"varint,3,opt,name=timeoutSeconds"` 36 | // Minimum consecutive successes for the probe to be considered successful after having failed. 37 | // Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. 38 | // +optional 39 | SuccessThreshold int32 `json:"successThreshold,omitempty" protobuf:"varint,4,opt,name=successThreshold"` 40 | // Minimum consecutive failures for the probe to be considered failed after having succeeded. 41 | // Defaults to 3. Minimum value is 1. 42 | // +optional 43 | FailureThreshold int32 `json:"failureThreshold,omitempty" protobuf:"varint,5,opt,name=failureThreshold"` 44 | // The name of this port within the service. This must be a DNS_LABEL. 45 | // All ports within a ServiceSpec must have unique names. When considering 46 | // the endpoints for a Service, this must match the 'name' field in the 47 | // EndpointPort. 48 | // Optional if only one ServicePort is defined on this service. 49 | // +optional 50 | Name string `json:"name,omitempty" protobuf:"bytes,6,opt,name=name"` 51 | 52 | // The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". 53 | // Default is TCP. 54 | // +optional 55 | Protocol v1.Protocol `json:"protocol,omitempty" protobuf:"bytes,7,opt,name=protocol,casttype=Protocol"` 56 | 57 | // The port that will be exposed by this service. 58 | Port int32 `json:"port" protobuf:"varint,8,opt,name=port"` 59 | 60 | // Number or name of the port to access on the pods targeted by the service. 61 | // Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. 62 | // If this is a string, it will be looked up as a named port in the 63 | // target Pod's container ports. If this is not specified, the value 64 | // of the 'port' field is used (an identity map). 65 | TargetPort int32 `json:"targetPort" protobuf:"varint,10,opt,name=targetPort"` 66 | } 67 | 68 | func (sp *ServicePort) ToEndpointSubset(host string) v1.EndpointSubset { 69 | s := make([]v1.EndpointPort, 0) 70 | endPoint := v1.EndpointPort{ 71 | Name: sp.Name, 72 | Port: sp.TargetPort, 73 | Protocol: sp.Protocol, 74 | } 75 | s = append(s, endPoint) 76 | return v1.EndpointSubset{ 77 | Addresses: []v1.EndpointAddress{ 78 | { 79 | IP: host, 80 | }, 81 | }, 82 | Ports: s, 83 | } 84 | } 85 | 86 | // TCPSocketAction describes an action based on opening a socket 87 | type TCPSocketAction struct { 88 | Enable bool `json:"enable" protobuf:"bytes,1,opt,name=enable"` 89 | } 90 | 91 | // UDPSocketAction describes an action based on opening a socket 92 | type UDPSocketAction struct { 93 | Enable bool `json:"enable" protobuf:"bytes,1,opt,name=enable"` 94 | // UDP test data 95 | // +optional 96 | Data []uint8 `json:"data,omitempty" protobuf:"varint,2,rep,name=data"` 97 | } 98 | 99 | func Int8ArrToByteArr(data []uint8) []byte { 100 | r := make([]byte, len(data)) 101 | for i, d := range data { 102 | r[i] = d 103 | } 104 | return r 105 | } 106 | 107 | // HTTPGetAction describes an action based on HTTP Get requests. 108 | type HTTPGetAction struct { 109 | // Path to access on the HTTP server. 110 | // +optional 111 | Path string `json:"path,omitempty" protobuf:"bytes,1,opt,name=path"` 112 | // Scheme to use for connecting to the host. 113 | // Defaults to HTTP. 114 | // +optional 115 | Scheme v1.URIScheme `json:"scheme,omitempty" protobuf:"bytes,4,opt,name=scheme,casttype=URIScheme"` 116 | // Custom headers to set in the request. HTTP allows repeated headers. 117 | // +optional 118 | HTTPHeaders []v1.HTTPHeader `json:"httpHeaders,omitempty" protobuf:"bytes,5,rep,name=httpHeaders"` 119 | } 120 | 121 | type GRPCAction struct { 122 | Enable bool `json:"enable" protobuf:"bytes,1,opt,name=enable"` 123 | // Service is the name of the service to place in the gRPC HealthCheckRequest 124 | // (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). 125 | // 126 | // If this is not specified, the default behavior is defined by gRPC. 127 | // +optional 128 | // +default="" 129 | Service *string `json:"service" protobuf:"bytes,2,opt,name=service"` 130 | } 131 | 132 | // Handler defines a specific action that should be taken 133 | type Handler struct { 134 | // HTTPGet specifies the http request to perform. 135 | // +optional 136 | HTTPGet *HTTPGetAction `json:"httpGet,omitempty" protobuf:"bytes,2,opt,name=httpGet"` 137 | // TCPSocket specifies an action involving a TCP port. 138 | // TCP hooks not yet supported 139 | // +optional 140 | TCPSocket *TCPSocketAction `json:"tcpSocket,omitempty" protobuf:"bytes,3,opt,name=tcpSocket"` 141 | // UDPSocketAction specifies an action involving a UDP port. 142 | // UDP hooks not yet supported 143 | // +optional 144 | UDPSocket *UDPSocketAction `json:"udpSocket,omitempty" protobuf:"bytes,4,opt,name=udpSocket"` 145 | 146 | // GRPC specifies an action involving a GRPC port. 147 | // This is an alpha field and requires enabling GRPCContainerProbe feature gate. 148 | // +optional 149 | GRPC *GRPCAction `json:"grpc,omitempty" protobuf:"bytes,5,opt,name=grpc"` 150 | } 151 | 152 | // ClusterEndpointSpec defines the desired state of ClusterEndpoint 153 | type ClusterEndpointSpec struct { 154 | ClusterIP string `json:"clusterIP,omitempty" protobuf:"bytes,1,opt,name=clusterIP"` 155 | Ports []ServicePort `json:"ports,omitempty" patchStrategy:"merge" patchMergeKey:"port" protobuf:"bytes,2,rep,name=ports"` 156 | // How often (in seconds) to perform the probe. 157 | // Default to 10 seconds. Minimum value is 1. 158 | // +optional 159 | PeriodSeconds int32 `json:"periodSeconds,omitempty" protobuf:"varint,4,opt,name=periodSeconds"` 160 | } 161 | 162 | type Phase string 163 | 164 | // These are the valid phases of node. 165 | const ( 166 | // Pending means the node has been created/added by the system. 167 | Pending Phase = "Pending" 168 | // Healthy means the cluster service is healthy. 169 | Healthy Phase = "Healthy" 170 | // UnHealthy means the cluster service is not healthy. 171 | UnHealthy Phase = "UnHealthy" 172 | ) 173 | 174 | type ConditionType string 175 | 176 | const ( 177 | SyncServiceReady ConditionType = "SyncServiceReady" 178 | SyncEndpointReady ConditionType = "SyncEndpointReady" 179 | Initialized ConditionType = "Initialized" 180 | Ready ConditionType = "Ready" 181 | ) 182 | 183 | type Condition struct { 184 | Type ConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=ConditionType"` 185 | // Status is the status of the condition. One of True, False, Unknown. 186 | Status v1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"` 187 | // LastHeartbeatTime is the last time this condition was updated. 188 | // +optional 189 | LastHeartbeatTime metav1.Time `json:"lastHeartbeatTime,omitempty" protobuf:"bytes,3,opt,name=lastHeartbeatTime"` 190 | // LastTransitionTime is the last time the condition changed from one status to another. 191 | // +optional 192 | LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"` 193 | // Reason is a (brief) reason for the condition's last status change. 194 | // +optional 195 | Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"` 196 | // Message is a human-readable message indicating details about the last status change. 197 | // +optional 198 | Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` 199 | } 200 | 201 | // ClusterEndpointStatus defines the observed state of ClusterEndpoint 202 | type ClusterEndpointStatus struct { 203 | // Phase is the recently observed lifecycle phase of the cluster endpoints. 204 | Phase Phase `json:"phase,omitempty" protobuf:"bytes,1,opt,name=phase,casttype=Phase"` 205 | // Conditions contains the different condition statuses for this workspace. 206 | Conditions []Condition `json:"conditions" protobuf:"bytes,3,rep,name=conditions"` 207 | } 208 | 209 | // +kubebuilder:object:root=true 210 | // +kubebuilder:subresource:status 211 | // +kubebuilder:resource:shortName=cep 212 | // +kubebuilder:printcolumn:name="Age",type=date,description="The creation date",JSONPath=`.metadata.creationTimestamp`,priority=0 213 | // +kubebuilder:printcolumn:name="Status",type=string,description="The status",JSONPath=`.status.phase`,priority=0 214 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 215 | 216 | // ClusterEndpoint is the Schema for the tests API 217 | type ClusterEndpoint struct { 218 | metav1.TypeMeta `json:",inline"` 219 | metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 220 | 221 | Spec ClusterEndpointSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` 222 | Status ClusterEndpointStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` 223 | } 224 | 225 | // +kubebuilder:object:root=true 226 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 227 | 228 | // ClusterEndpointList contains a list of ClusterEndpoint 229 | type ClusterEndpointList struct { 230 | metav1.TypeMeta `json:",inline"` 231 | metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 232 | Items []ClusterEndpoint `json:"items" protobuf:"bytes,2,opt,name=items"` 233 | } 234 | -------------------------------------------------------------------------------- /apis/network/v1beta1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright 2022 The sealos Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by controller-gen. DO NOT EDIT. 21 | 22 | package v1beta1 23 | 24 | import ( 25 | v1 "k8s.io/api/core/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | ) 28 | 29 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 30 | func (in *ClusterEndpoint) DeepCopyInto(out *ClusterEndpoint) { 31 | *out = *in 32 | out.TypeMeta = in.TypeMeta 33 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 34 | in.Spec.DeepCopyInto(&out.Spec) 35 | in.Status.DeepCopyInto(&out.Status) 36 | } 37 | 38 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterEndpoint. 39 | func (in *ClusterEndpoint) DeepCopy() *ClusterEndpoint { 40 | if in == nil { 41 | return nil 42 | } 43 | out := new(ClusterEndpoint) 44 | in.DeepCopyInto(out) 45 | return out 46 | } 47 | 48 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 49 | func (in *ClusterEndpoint) DeepCopyObject() runtime.Object { 50 | if c := in.DeepCopy(); c != nil { 51 | return c 52 | } 53 | return nil 54 | } 55 | 56 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 57 | func (in *ClusterEndpointList) DeepCopyInto(out *ClusterEndpointList) { 58 | *out = *in 59 | out.TypeMeta = in.TypeMeta 60 | in.ListMeta.DeepCopyInto(&out.ListMeta) 61 | if in.Items != nil { 62 | in, out := &in.Items, &out.Items 63 | *out = make([]ClusterEndpoint, len(*in)) 64 | for i := range *in { 65 | (*in)[i].DeepCopyInto(&(*out)[i]) 66 | } 67 | } 68 | } 69 | 70 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterEndpointList. 71 | func (in *ClusterEndpointList) DeepCopy() *ClusterEndpointList { 72 | if in == nil { 73 | return nil 74 | } 75 | out := new(ClusterEndpointList) 76 | in.DeepCopyInto(out) 77 | return out 78 | } 79 | 80 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 81 | func (in *ClusterEndpointList) DeepCopyObject() runtime.Object { 82 | if c := in.DeepCopy(); c != nil { 83 | return c 84 | } 85 | return nil 86 | } 87 | 88 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 89 | func (in *ClusterEndpointSpec) DeepCopyInto(out *ClusterEndpointSpec) { 90 | *out = *in 91 | if in.Ports != nil { 92 | in, out := &in.Ports, &out.Ports 93 | *out = make([]ServicePort, len(*in)) 94 | for i := range *in { 95 | (*in)[i].DeepCopyInto(&(*out)[i]) 96 | } 97 | } 98 | } 99 | 100 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterEndpointSpec. 101 | func (in *ClusterEndpointSpec) DeepCopy() *ClusterEndpointSpec { 102 | if in == nil { 103 | return nil 104 | } 105 | out := new(ClusterEndpointSpec) 106 | in.DeepCopyInto(out) 107 | return out 108 | } 109 | 110 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 111 | func (in *ClusterEndpointStatus) DeepCopyInto(out *ClusterEndpointStatus) { 112 | *out = *in 113 | if in.Conditions != nil { 114 | in, out := &in.Conditions, &out.Conditions 115 | *out = make([]Condition, len(*in)) 116 | for i := range *in { 117 | (*in)[i].DeepCopyInto(&(*out)[i]) 118 | } 119 | } 120 | } 121 | 122 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterEndpointStatus. 123 | func (in *ClusterEndpointStatus) DeepCopy() *ClusterEndpointStatus { 124 | if in == nil { 125 | return nil 126 | } 127 | out := new(ClusterEndpointStatus) 128 | in.DeepCopyInto(out) 129 | return out 130 | } 131 | 132 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 133 | func (in *Condition) DeepCopyInto(out *Condition) { 134 | *out = *in 135 | in.LastHeartbeatTime.DeepCopyInto(&out.LastHeartbeatTime) 136 | in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) 137 | } 138 | 139 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. 140 | func (in *Condition) DeepCopy() *Condition { 141 | if in == nil { 142 | return nil 143 | } 144 | out := new(Condition) 145 | in.DeepCopyInto(out) 146 | return out 147 | } 148 | 149 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 150 | func (in *GRPCAction) DeepCopyInto(out *GRPCAction) { 151 | *out = *in 152 | if in.Service != nil { 153 | in, out := &in.Service, &out.Service 154 | *out = new(string) 155 | **out = **in 156 | } 157 | } 158 | 159 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GRPCAction. 160 | func (in *GRPCAction) DeepCopy() *GRPCAction { 161 | if in == nil { 162 | return nil 163 | } 164 | out := new(GRPCAction) 165 | in.DeepCopyInto(out) 166 | return out 167 | } 168 | 169 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 170 | func (in *HTTPGetAction) DeepCopyInto(out *HTTPGetAction) { 171 | *out = *in 172 | if in.HTTPHeaders != nil { 173 | in, out := &in.HTTPHeaders, &out.HTTPHeaders 174 | *out = make([]v1.HTTPHeader, len(*in)) 175 | copy(*out, *in) 176 | } 177 | } 178 | 179 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPGetAction. 180 | func (in *HTTPGetAction) DeepCopy() *HTTPGetAction { 181 | if in == nil { 182 | return nil 183 | } 184 | out := new(HTTPGetAction) 185 | in.DeepCopyInto(out) 186 | return out 187 | } 188 | 189 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 190 | func (in *Handler) DeepCopyInto(out *Handler) { 191 | *out = *in 192 | if in.HTTPGet != nil { 193 | in, out := &in.HTTPGet, &out.HTTPGet 194 | *out = new(HTTPGetAction) 195 | (*in).DeepCopyInto(*out) 196 | } 197 | if in.TCPSocket != nil { 198 | in, out := &in.TCPSocket, &out.TCPSocket 199 | *out = new(TCPSocketAction) 200 | **out = **in 201 | } 202 | if in.UDPSocket != nil { 203 | in, out := &in.UDPSocket, &out.UDPSocket 204 | *out = new(UDPSocketAction) 205 | (*in).DeepCopyInto(*out) 206 | } 207 | if in.GRPC != nil { 208 | in, out := &in.GRPC, &out.GRPC 209 | *out = new(GRPCAction) 210 | (*in).DeepCopyInto(*out) 211 | } 212 | } 213 | 214 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Handler. 215 | func (in *Handler) DeepCopy() *Handler { 216 | if in == nil { 217 | return nil 218 | } 219 | out := new(Handler) 220 | in.DeepCopyInto(out) 221 | return out 222 | } 223 | 224 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 225 | func (in *ServicePort) DeepCopyInto(out *ServicePort) { 226 | *out = *in 227 | if in.Hosts != nil { 228 | in, out := &in.Hosts, &out.Hosts 229 | *out = make([]string, len(*in)) 230 | copy(*out, *in) 231 | } 232 | in.Handler.DeepCopyInto(&out.Handler) 233 | } 234 | 235 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServicePort. 236 | func (in *ServicePort) DeepCopy() *ServicePort { 237 | if in == nil { 238 | return nil 239 | } 240 | out := new(ServicePort) 241 | in.DeepCopyInto(out) 242 | return out 243 | } 244 | 245 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 246 | func (in *TCPSocketAction) DeepCopyInto(out *TCPSocketAction) { 247 | *out = *in 248 | } 249 | 250 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCPSocketAction. 251 | func (in *TCPSocketAction) DeepCopy() *TCPSocketAction { 252 | if in == nil { 253 | return nil 254 | } 255 | out := new(TCPSocketAction) 256 | in.DeepCopyInto(out) 257 | return out 258 | } 259 | 260 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 261 | func (in *UDPSocketAction) DeepCopyInto(out *UDPSocketAction) { 262 | *out = *in 263 | if in.Data != nil { 264 | in, out := &in.Data, &out.Data 265 | *out = make([]uint8, len(*in)) 266 | copy(*out, *in) 267 | } 268 | } 269 | 270 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UDPSocketAction. 271 | func (in *UDPSocketAction) DeepCopy() *UDPSocketAction { 272 | if in == nil { 273 | return nil 274 | } 275 | out := new(UDPSocketAction) 276 | in.DeepCopyInto(out) 277 | return out 278 | } 279 | -------------------------------------------------------------------------------- /cmd/cepctl/app/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | client2 "github.com/labring/endpoints-operator/utils/client" 24 | "os" 25 | "sort" 26 | 27 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 28 | "github.com/labring/endpoints-operator/cmd/cepctl/app/options" 29 | "github.com/labring/operator-sdk/version" 30 | "github.com/spf13/cobra" 31 | v1 "k8s.io/api/core/v1" 32 | v1opts "k8s.io/apimachinery/pkg/apis/meta/v1" 33 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 34 | "k8s.io/apimachinery/pkg/util/json" 35 | cliflag "k8s.io/component-base/cli/flag" 36 | "k8s.io/component-base/term" 37 | "k8s.io/klog/v2" 38 | "sigs.k8s.io/yaml" 39 | ) 40 | 41 | func NewCommand() *cobra.Command { 42 | s := options.NewOptions() 43 | 44 | cmd := &cobra.Command{ 45 | Use: "cepctl", 46 | Short: "cepctl is cli for cluster-endpoint", 47 | Run: func(cmd *cobra.Command, args []string) { 48 | if errs := s.Validate(); len(errs) != 0 { 49 | klog.Error(utilerrors.NewAggregate(errs)) 50 | os.Exit(1) 51 | } 52 | if err := run(s, context.Background()); err != nil { 53 | klog.Error(err) 54 | os.Exit(1) 55 | } 56 | }, 57 | SilenceUsage: true, 58 | } 59 | 60 | fs := cmd.Flags() 61 | namedFlagSets := s.Flags() 62 | 63 | for _, f := range namedFlagSets.FlagSets { 64 | fs.AddFlagSet(f) 65 | } 66 | 67 | usageFmt := "Usage:\n %s\n" 68 | cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) 69 | cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { 70 | _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) 71 | cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) 72 | }) 73 | return cmd 74 | } 75 | 76 | func run(s *options.Options, ctx context.Context) error { 77 | if s.Version { 78 | if s.Short { 79 | fmt.Printf("Version: %s\n", version.Get().GitVersion) 80 | } else { 81 | fmt.Printf("Version: %s\n", fmt.Sprintf("%#v", version.Get())) 82 | } 83 | return nil 84 | } 85 | cli := client2.NewKubernetesClient(client2.NewKubernetesOptions(s.KubeConfig, s.Master)) 86 | if cli == nil { 87 | return errors.New("build kube client error") 88 | } 89 | cep := &v1beta1.ClusterEndpoint{} 90 | cep.Namespace = s.Namespace 91 | cep.Name = s.Name 92 | cep.Spec.PeriodSeconds = s.PeriodSeconds 93 | svc, err := cli.Kubernetes().CoreV1().Services(s.Namespace).Get(ctx, s.Name, v1opts.GetOptions{}) 94 | if err != nil { 95 | return err 96 | } 97 | klog.V(4).InfoS("get service", "name", s.Name, "namespace", s.Namespace, "spec", svc.Spec) 98 | if svc.Spec.ClusterIP == v1.ClusterIPNone { 99 | return errors.New("not support clusterIP=None service") 100 | } 101 | cep.Spec.ClusterIP = svc.Spec.ClusterIP 102 | ep, _ := cli.Kubernetes().CoreV1().Endpoints(s.Namespace).Get(ctx, s.Name, v1opts.GetOptions{}) 103 | if ep != nil { 104 | klog.V(4).InfoS("get endpoint", "name", s.Name, "namespace", s.Namespace, "subsets", ep.Subsets) 105 | if len(ep.Subsets) > 1 { 106 | return errors.New("not support endpoint subsets length more than 1. Please spilt it") 107 | } 108 | ports := make([]v1beta1.ServicePort, 0) 109 | for _, subset := range ep.Subsets { 110 | enable := s.Probe 111 | 112 | ips := make([]string, 0) 113 | for _, addr := range subset.Addresses { 114 | ips = append(ips, addr.IP) 115 | } 116 | sort.Sort(sort.StringSlice(ips)) 117 | for _, port := range subset.Ports { 118 | ports = append(ports, v1beta1.ServicePort{ 119 | Handler: v1beta1.Handler{ 120 | TCPSocket: &v1beta1.TCPSocketAction{Enable: enable}, 121 | }, 122 | TimeoutSeconds: 1, 123 | SuccessThreshold: 1, 124 | FailureThreshold: 3, 125 | Name: port.Name, 126 | Protocol: port.Protocol, 127 | Port: findPortInSvc(svc, port.Name), 128 | TargetPort: port.Port, 129 | Hosts: ips, 130 | }) 131 | } 132 | 133 | } 134 | cep.Spec.Ports = ports 135 | } 136 | configJson, _ := json.Marshal(cep) 137 | configYaml, _ := yaml.Marshal(cep) 138 | klog.V(4).InfoS("generator cep", "name", s.Name, "namespace", s.Namespace, "config", string(configJson)) 139 | if s.Output != "" { 140 | if s.Output == "yaml" { 141 | println(string(configYaml)) 142 | return nil 143 | } 144 | if s.Output == "json" { 145 | println(string(configJson)) 146 | return nil 147 | } 148 | } 149 | c := client2.NewCep(cli.KubernetesDynamic()) 150 | return c.CreateCR(ctx, cep) 151 | } 152 | 153 | func findPortInSvc(svc *v1.Service, portName string) int32 { 154 | 155 | if svc != nil { 156 | for _, p := range svc.Spec.Ports { 157 | if p.Name == portName { 158 | return p.Port 159 | } 160 | } 161 | } 162 | return 0 163 | } 164 | -------------------------------------------------------------------------------- /cmd/cepctl/app/options/options.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package options 16 | 17 | import ( 18 | "errors" 19 | "flag" 20 | "fmt" 21 | "github.com/labring/endpoints-operator/utils/file" 22 | "k8s.io/klog/v2" 23 | "path" 24 | "strings" 25 | 26 | cliflag "k8s.io/component-base/cli/flag" 27 | ) 28 | 29 | type Options struct { 30 | // Master is used to override the kubeconfig's URL to the apiserver. 31 | Master string 32 | //KubeConfig is the path to a KubeConfig file. 33 | KubeConfig string 34 | Name string 35 | Namespace string 36 | PeriodSeconds int32 37 | Probe bool 38 | Output string 39 | Version bool 40 | Short bool 41 | } 42 | 43 | func NewOptions() *Options { 44 | s := &Options{ 45 | Master: "", 46 | KubeConfig: path.Join(file.GetUserHomeDir(), ".kube", "config"), 47 | PeriodSeconds: 10, 48 | Name: "", 49 | Probe: false, 50 | Namespace: "default", 51 | Output: "", 52 | Version: false, 53 | Short: false, 54 | } 55 | return s 56 | } 57 | 58 | func (s *Options) Flags() cliflag.NamedFlagSets { 59 | fss := cliflag.NamedFlagSets{} 60 | 61 | kube := fss.FlagSet("kube") 62 | kube.StringVar(&s.KubeConfig, "kubeconfig", s.KubeConfig, "Path to kubeconfig file with authorization information (the master location can be overridden by the master flag).") 63 | kube.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") 64 | 65 | cep := fss.FlagSet("cep") 66 | cep.StringVar(&s.Name, "service-name", s.Name, "Sync cap from service name.") 67 | cep.StringVar(&s.Namespace, "service-namespace", s.Namespace, "Sync cap from service namespace.") 68 | cep.StringVarP(&s.Output, "output", "o", s.Output, "output json|yaml. if not set,will create cep to kubernetes") 69 | 70 | probe := fss.FlagSet("probe") 71 | probe.Int32Var(&s.PeriodSeconds, "periodSeconds", s.PeriodSeconds, "How often (in seconds) to perform the probe.Default is 10.") 72 | probe.BoolVar(&s.Probe, "probe", s.Probe, "When set value is true,add default probe of tcpAction.") 73 | 74 | version := fss.FlagSet("version") 75 | version.BoolVar(&s.Version, "version", s.Version, "Print the client version information") 76 | version.BoolVar(&s.Short, "short", s.Short, "If true, print just the version number.") 77 | 78 | kfs := fss.FlagSet("klog") 79 | local := flag.NewFlagSet("klog", flag.ExitOnError) 80 | klog.InitFlags(local) 81 | local.VisitAll(func(fl *flag.Flag) { 82 | fl.Name = strings.Replace(fl.Name, "_", "-", -1) 83 | kfs.AddGoFlag(fl) 84 | }) 85 | 86 | return fss 87 | } 88 | 89 | func (s *Options) Validate() []error { 90 | var errs []error 91 | if s.Version { 92 | return errs 93 | } 94 | if s.PeriodSeconds < 0 { 95 | errs = append(errs, errors.New("param periodSeconds must more than zero")) 96 | } 97 | if !file.Exist(s.KubeConfig) { 98 | errs = append(errs, fmt.Errorf("kubeconfig path %s is not exist", s.KubeConfig)) 99 | } 100 | if len(s.Name) == 0 { 101 | errs = append(errs, errors.New("service name must not empty")) 102 | } 103 | if len(s.Namespace) == 0 { 104 | errs = append(errs, errors.New("service namespace must not empty")) 105 | } 106 | if len(s.Output) != 0 { 107 | if s.Output != "yaml" && s.Output != "json" { 108 | errs = append(errs, errors.New("output must be is yaml or json")) 109 | } 110 | } 111 | return errs 112 | } 113 | -------------------------------------------------------------------------------- /cmd/cepctl/cepctl.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | "github.com/labring/endpoints-operator/cmd/cepctl/app" 23 | ) 24 | 25 | func main() { 26 | 27 | command := app.NewCommand() 28 | 29 | if err := command.Execute(); err != nil { 30 | os.Exit(1) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cmd/endpoints-operator/app/options/options.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package options 16 | 17 | import ( 18 | "flag" 19 | "strings" 20 | "time" 21 | 22 | "k8s.io/client-go/tools/leaderelection/resourcelock" 23 | 24 | utilcontroller "github.com/labring/operator-sdk/controller" 25 | "github.com/spf13/pflag" 26 | "k8s.io/client-go/tools/leaderelection" 27 | cliflag "k8s.io/component-base/cli/flag" 28 | "k8s.io/klog/v2" 29 | ) 30 | 31 | type Options struct { 32 | LeaderElect bool 33 | LeaderElection *leaderelection.LeaderElectionConfig 34 | LeaderElectionResourceLock string 35 | MaxConcurrent int 36 | MaxRetry int 37 | RateLimiterOptions utilcontroller.RateLimiterOptions 38 | } 39 | 40 | func NewOptions() *Options { 41 | s := &Options{ 42 | LeaderElection: &leaderelection.LeaderElectionConfig{ 43 | LeaseDuration: 15 * time.Second, 44 | RenewDeadline: 10 * time.Second, 45 | RetryPeriod: 2 * time.Second, 46 | }, 47 | LeaderElect: false, 48 | } 49 | 50 | return s 51 | } 52 | 53 | func (s *Options) Flags() cliflag.NamedFlagSets { 54 | fss := cliflag.NamedFlagSets{} 55 | 56 | fs := fss.FlagSet("leaderelection") 57 | s.bindLeaderElectionFlags(s.LeaderElection, fs) 58 | fs.BoolVar(&s.LeaderElect, "leader-elect", s.LeaderElect, ""+ 59 | "Whether to enable leader election. This field should be enabled when controller manager"+ 60 | "deployed with multiple replicas.") 61 | 62 | kfs := fss.FlagSet("klog") 63 | local := flag.NewFlagSet("klog", flag.ExitOnError) 64 | klog.InitFlags(local) 65 | local.VisitAll(func(fl *flag.Flag) { 66 | fl.Name = strings.Replace(fl.Name, "_", "-", -1) 67 | kfs.AddGoFlag(fl) 68 | }) 69 | 70 | // add MaxConcurrent args 71 | // MaxConcurrent this is the maximum number of concurrent Reconciles which can be run. Defaults to 1. 72 | mc := fss.FlagSet("worker") 73 | mc.IntVar(&s.MaxConcurrent, "maxconcurrent", 1, "MaxConcurrent this is the maximum number of concurrent Reconciles "+ 74 | "which can be run. Defaults to 1.") 75 | mc.IntVar(&s.MaxRetry, "maxretry", 1, "MaxRetry this is the maximum number of retry liveliness "+ 76 | "which can be run. Defaults to 1.") 77 | s.RateLimiterOptions.BindFlags(flag.CommandLine) 78 | return fss 79 | } 80 | 81 | func (s *Options) Validate() []error { 82 | var errs []error 83 | return errs 84 | } 85 | 86 | func (s *Options) bindLeaderElectionFlags(l *leaderelection.LeaderElectionConfig, fs *pflag.FlagSet) { 87 | fs.DurationVar(&l.LeaseDuration, "leader-elect-lease-duration", l.LeaseDuration, ""+ 88 | "The duration that non-leader candidates will wait after observing a leadership "+ 89 | "renewal until attempting to acquire leadership of a led but unrenewed leader "+ 90 | "slot. This is effectively the maximum duration that a leader can be stopped "+ 91 | "before it is replaced by another candidate. This is only applicable if leader "+ 92 | "election is enabled.") 93 | fs.DurationVar(&l.RenewDeadline, "leader-elect-renew-deadline", l.RenewDeadline, ""+ 94 | "The interval between attempts by the acting master to renew a leadership slot "+ 95 | "before it stops leading. This must be less than or equal to the lease duration. "+ 96 | "This is only applicable if leader election is enabled.") 97 | fs.DurationVar(&l.RetryPeriod, "leader-elect-retry-period", l.RetryPeriod, ""+ 98 | "The duration the clients should wait between attempting acquisition and renewal "+ 99 | "of a leadership. This is only applicable if leader election is enabled.") 100 | fs.StringVar(&s.LeaderElectionResourceLock, "leader-elect-resource-lock", resourcelock.ConfigMapsLeasesResourceLock, 101 | "Leader election resource lock, support: endpoints,configmaps,leases,endpointsleases,configmapsleases") 102 | 103 | } 104 | -------------------------------------------------------------------------------- /cmd/endpoints-operator/app/server.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 cuisongliu@qq.com. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "context" 19 | "flag" 20 | "fmt" 21 | "github.com/labring/endpoints-operator/utils/metrics" 22 | "github.com/labring/operator-sdk/controller" 23 | "net/http" 24 | "os" 25 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 26 | "sync" 27 | 28 | "github.com/labring/endpoints-operator/cmd/endpoints-operator/app/options" 29 | "github.com/labring/endpoints-operator/controllers" 30 | "k8s.io/component-base/term" 31 | 32 | "github.com/spf13/cobra" 33 | "k8s.io/apimachinery/pkg/runtime" 34 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 35 | cliflag "k8s.io/component-base/cli/flag" 36 | "k8s.io/klog/v2" 37 | ctrl "sigs.k8s.io/controller-runtime" 38 | "sigs.k8s.io/controller-runtime/pkg/manager" 39 | ) 40 | 41 | var ( 42 | scheme = runtime.NewScheme() 43 | metricsInfo *metrics.MetricsInfo 44 | ) 45 | 46 | func NewCommand() *cobra.Command { 47 | s := options.NewOptions() 48 | // make sure LeaderElection is not nil 49 | s = &options.Options{ 50 | LeaderElection: s.LeaderElection, 51 | LeaderElect: s.LeaderElect, 52 | } 53 | 54 | cmd := &cobra.Command{ 55 | Use: "endpoints-operator", 56 | Long: `endpoints-operator controller manager is a daemon that`, 57 | Run: func(cmd *cobra.Command, args []string) { 58 | if errs := s.Validate(); len(errs) != 0 { 59 | klog.Error(utilerrors.NewAggregate(errs)) 60 | os.Exit(1) 61 | } 62 | ctx, cancel := context.WithCancel(context.TODO()) 63 | defer cancel() 64 | if err := run(s, ctx); err != nil { 65 | klog.Error(err) 66 | os.Exit(1) 67 | } 68 | }, 69 | SilenceUsage: true, 70 | } 71 | 72 | fs := cmd.Flags() 73 | namedFlagSets := s.Flags() 74 | 75 | for _, f := range namedFlagSets.FlagSets { 76 | fs.AddFlagSet(f) 77 | } 78 | 79 | usageFmt := "Usage:\n %s\n" 80 | cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) 81 | cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { 82 | _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) 83 | cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) 84 | }) 85 | return cmd 86 | } 87 | 88 | func run(s *options.Options, ctx context.Context) error { 89 | mgrOptions := manager.Options{} 90 | if s.LeaderElect { 91 | mgrOptions = manager.Options{ 92 | LeaderElection: s.LeaderElect, 93 | LeaderElectionNamespace: "kube-system", 94 | LeaderElectionID: "sealos-endpoints-operator-leader-election", 95 | LeaderElectionResourceLock: s.LeaderElectionResourceLock, 96 | LeaseDuration: &s.LeaderElection.LeaseDuration, 97 | RetryPeriod: &s.LeaderElection.RetryPeriod, 98 | RenewDeadline: &s.LeaderElection.RenewDeadline, 99 | } 100 | } 101 | 102 | metricsInfo = metrics.NewMetricsInfo() 103 | metricsInfo.RegisterAllMetrics() 104 | 105 | mgrOptions.Scheme = scheme 106 | mgrOptions.HealthProbeBindAddress = ":8080" 107 | mgrOptions.MetricsBindAddress = ":9090" 108 | klog.V(0).Info("setting up manager") 109 | opts := zap.Options{ 110 | Development: true, 111 | } 112 | opts.BindFlags(flag.CommandLine) 113 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) 114 | // Use 8443 instead of 443 cause we need root permission to bind port 443 115 | mgr, err := manager.New(ctrl.GetConfigOrDie(), mgrOptions) 116 | if err != nil { 117 | klog.Fatalf("unable to set up overall controller manager: %v", err) 118 | } 119 | klog.V(4).Info("[****] MaxConcurrent value is ", s.MaxConcurrent) 120 | klog.V(4).Info("[****] MaxRetry value is ", s.MaxRetry) 121 | 122 | controllers.Install(scheme) 123 | clusterReconciler := &controllers.Reconciler{} 124 | if s.MaxConcurrent > 0 { 125 | clusterReconciler.MaxConcurrent = s.MaxConcurrent 126 | } else { 127 | clusterReconciler.MaxConcurrent = 1 128 | } 129 | if s.MaxRetry > 0 { 130 | clusterReconciler.RetryCount = s.MaxRetry 131 | } else { 132 | clusterReconciler.RetryCount = 1 133 | } 134 | 135 | clusterReconciler.RateLimiter = controller.GetRateLimiter(s.RateLimiterOptions) 136 | 137 | clusterReconciler.MetricsInfo = metricsInfo 138 | 139 | if err = clusterReconciler.SetupWithManager(mgr); err != nil { 140 | klog.Fatal("Unable to create cluster controller ", err) 141 | } 142 | 143 | klog.V(0).Info("Starting the controllers.") 144 | 145 | //healthz Liveness 146 | if err = mgr.AddHealthzCheck("check", func(req *http.Request) error { 147 | return nil 148 | }); err != nil { 149 | klog.Fatal(err, "problem running manager liveness Check") 150 | } 151 | //readyz Readiness 152 | if err = mgr.AddReadyzCheck("check", func(req *http.Request) error { 153 | return nil 154 | }); err != nil { 155 | klog.Fatal(err, "problem running manager readiness check") 156 | } 157 | 158 | go func() { 159 | klog.Info("starting manager") 160 | if err = mgr.Start(ctx); err != nil { 161 | klog.Fatalf("unable to run the manager: %v", err) 162 | os.Exit(1) 163 | } 164 | }() 165 | done := make(chan struct{}) 166 | go func() { 167 | if mgr.GetCache().WaitForCacheSync(context.Background()) { 168 | done <- struct{}{} 169 | } 170 | }() 171 | <-done 172 | 173 | var wg sync.WaitGroup 174 | wg.Add(1) 175 | wg.Wait() 176 | return nil 177 | } 178 | -------------------------------------------------------------------------------- /cmd/endpoints-operator/endpoints-operator.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | "github.com/labring/endpoints-operator/cmd/endpoints-operator/app" 23 | ) 24 | 25 | func main() { 26 | command := app.NewCommand() 27 | 28 | if err := command.Execute(); err != nil { 29 | os.Exit(1) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v2 16 | name: endpoints-operator 17 | description: kubernetes endpoints balance for outsite apiserver 18 | engine: gotpl 19 | kubeVersion: ">=1.19.0-0" 20 | sources: 21 | - https://github.com/labring/endpoints-operator 22 | home: https://github.com/labring/endpoints-operator 23 | keywords: 24 | - endpoints-operator 25 | type: application 26 | version: 0.0.0 27 | appVersion: "0.0.0" 28 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/crds/sealos.io_clusterendpoints.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.8.0 7 | creationTimestamp: null 8 | name: clusterendpoints.sealos.io 9 | spec: 10 | group: sealos.io 11 | names: 12 | kind: ClusterEndpoint 13 | listKind: ClusterEndpointList 14 | plural: clusterendpoints 15 | shortNames: 16 | - cep 17 | singular: clusterendpoint 18 | scope: Namespaced 19 | versions: 20 | - additionalPrinterColumns: 21 | - description: The creation date 22 | jsonPath: .metadata.creationTimestamp 23 | name: Age 24 | type: date 25 | - description: The status 26 | jsonPath: .status.phase 27 | name: Status 28 | type: string 29 | name: v1beta1 30 | schema: 31 | openAPIV3Schema: 32 | description: ClusterEndpoint is the Schema for the tests API 33 | properties: 34 | apiVersion: 35 | description: 'APIVersion defines the versioned schema of this representation 36 | of an object. Servers should convert recognized schemas to the latest 37 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 38 | type: string 39 | kind: 40 | description: 'Kind is a string value representing the REST resource this 41 | object represents. Servers may infer this from the endpoint the client 42 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 43 | type: string 44 | metadata: 45 | type: object 46 | spec: 47 | description: ClusterEndpointSpec defines the desired state of ClusterEndpoint 48 | properties: 49 | clusterIP: 50 | type: string 51 | periodSeconds: 52 | description: How often (in seconds) to perform the probe. Default 53 | to 10 seconds. Minimum value is 1. 54 | format: int32 55 | type: integer 56 | ports: 57 | items: 58 | description: ServicePort contains information on service's port. 59 | properties: 60 | failureThreshold: 61 | description: Minimum consecutive failures for the probe to be 62 | considered failed after having succeeded. Defaults to 3. Minimum 63 | value is 1. 64 | format: int32 65 | type: integer 66 | grpc: 67 | description: GRPC specifies an action involving a GRPC port. 68 | This is an alpha field and requires enabling GRPCContainerProbe 69 | feature gate. 70 | properties: 71 | enable: 72 | type: boolean 73 | service: 74 | description: "Service is the name of the service to place 75 | in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). 76 | \n If this is not specified, the default behavior is defined 77 | by gRPC." 78 | type: string 79 | required: 80 | - enable 81 | type: object 82 | hosts: 83 | items: 84 | type: string 85 | type: array 86 | httpGet: 87 | description: HTTPGet specifies the http request to perform. 88 | properties: 89 | httpHeaders: 90 | description: Custom headers to set in the request. HTTP 91 | allows repeated headers. 92 | items: 93 | description: HTTPHeader describes a custom header to be 94 | used in HTTP probes 95 | properties: 96 | name: 97 | description: The header field name 98 | type: string 99 | value: 100 | description: The header field value 101 | type: string 102 | required: 103 | - name 104 | - value 105 | type: object 106 | type: array 107 | path: 108 | description: Path to access on the HTTP server. 109 | type: string 110 | scheme: 111 | description: Scheme to use for connecting to the host. Defaults 112 | to HTTP. 113 | type: string 114 | type: object 115 | name: 116 | description: The name of this port within the service. This 117 | must be a DNS_LABEL. All ports within a ServiceSpec must have 118 | unique names. When considering the endpoints for a Service, 119 | this must match the 'name' field in the EndpointPort. Optional 120 | if only one ServicePort is defined on this service. 121 | type: string 122 | port: 123 | description: The port that will be exposed by this service. 124 | format: int32 125 | type: integer 126 | protocol: 127 | default: TCP 128 | description: The IP protocol for this port. Supports "TCP", 129 | "UDP", and "SCTP". Default is TCP. 130 | type: string 131 | successThreshold: 132 | description: Minimum consecutive successes for the probe to 133 | be considered successful after having failed. Defaults to 134 | 1. Must be 1 for liveness and startup. Minimum value is 1. 135 | format: int32 136 | type: integer 137 | targetPort: 138 | description: Number or name of the port to access on the pods 139 | targeted by the service. Number must be in the range 1 to 140 | 65535. Name must be an IANA_SVC_NAME. If this is a string, 141 | it will be looked up as a named port in the target Pod's container 142 | ports. If this is not specified, the value of the 'port' field 143 | is used (an identity map). 144 | format: int32 145 | type: integer 146 | tcpSocket: 147 | description: TCPSocket specifies an action involving a TCP port. 148 | TCP hooks not yet supported 149 | properties: 150 | enable: 151 | type: boolean 152 | required: 153 | - enable 154 | type: object 155 | timeoutSeconds: 156 | description: 'Number of seconds after which the probe times 157 | out. Defaults to 1 second. Minimum value is 1. More info: 158 | https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' 159 | format: int32 160 | type: integer 161 | udpSocket: 162 | description: UDPSocketAction specifies an action involving a 163 | UDP port. UDP hooks not yet supported 164 | properties: 165 | data: 166 | description: UDP test data 167 | items: 168 | type: integer 169 | type: array 170 | enable: 171 | type: boolean 172 | required: 173 | - enable 174 | type: object 175 | required: 176 | - port 177 | - targetPort 178 | type: object 179 | type: array 180 | type: object 181 | status: 182 | description: ClusterEndpointStatus defines the observed state of ClusterEndpoint 183 | properties: 184 | conditions: 185 | description: Conditions contains the different condition statuses 186 | for this workspace. 187 | items: 188 | properties: 189 | lastHeartbeatTime: 190 | description: LastHeartbeatTime is the last time this condition 191 | was updated. 192 | format: date-time 193 | type: string 194 | lastTransitionTime: 195 | description: LastTransitionTime is the last time the condition 196 | changed from one status to another. 197 | format: date-time 198 | type: string 199 | message: 200 | description: Message is a human-readable message indicating 201 | details about the last status change. 202 | type: string 203 | reason: 204 | description: Reason is a (brief) reason for the condition's 205 | last status change. 206 | type: string 207 | status: 208 | description: Status is the status of the condition. One of True, 209 | False, Unknown. 210 | type: string 211 | type: 212 | type: string 213 | required: 214 | - status 215 | - type 216 | type: object 217 | type: array 218 | phase: 219 | description: Phase is the recently observed lifecycle phase of the 220 | cluster endpoints. 221 | type: string 222 | required: 223 | - conditions 224 | type: object 225 | type: object 226 | served: true 227 | storage: true 228 | subresources: 229 | status: {} 230 | status: 231 | acceptedNames: 232 | kind: "" 233 | plural: "" 234 | conditions: [] 235 | storedVersions: [] 236 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "endpoints-operator.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "endpoints-operator.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "endpoints-operator.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "endpoints-operator.labels" -}} 37 | helm.sh/chart: {{ include "endpoints-operator.chart" . }} 38 | {{ include "endpoints-operator.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "endpoints-operator.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "endpoints-operator.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: {{ include "endpoints-operator.fullname" . }} 19 | labels: 20 | {{- include "endpoints-operator.labels" . | nindent 4 }} 21 | spec: 22 | replicas: {{ .Values.replicaCount }} 23 | selector: 24 | matchLabels: 25 | {{- include "endpoints-operator.selectorLabels" . | nindent 6 }} 26 | template: 27 | metadata: 28 | {{- with .Values.podAnnotations }} 29 | annotations: 30 | {{- toYaml . | nindent 8 }} 31 | {{- end }} 32 | labels: 33 | {{- include "endpoints-operator.selectorLabels" . | nindent 8 }} 34 | spec: 35 | {{- with .Values.imagePullSecrets }} 36 | imagePullSecrets: 37 | {{- toYaml . | nindent 8 }} 38 | {{- end }} 39 | serviceAccountName: {{ include "endpoints-operator.fullname" . }} 40 | securityContext: 41 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 42 | containers: 43 | - name: endpoints-operator 44 | securityContext: 45 | {{- toYaml .Values.securityContext | nindent 12 }} 46 | image: "{{ .Values.image.repository }}/{{ .Values.image.image }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 47 | imagePullPolicy: {{ .Values.image.pullPolicy }} 48 | args: 49 | - --v 50 | - "{{ .Values.loglevel }}" 51 | - --leader-elect 52 | - --maxconcurrent 53 | - "{{ .Values.maxconcurrent }}" 54 | - --maxretry 55 | - "{{ .Values.maxretry }}" 56 | ports: 57 | - name: health 58 | containerPort: 8080 59 | protocol: TCP 60 | - name: metrics 61 | containerPort: 9090 62 | protocol: TCP 63 | readinessProbe: 64 | initialDelaySeconds: 10 65 | periodSeconds: 10 66 | timeoutSeconds: 1 67 | httpGet: 68 | port: health 69 | scheme: HTTP 70 | path: /readyz 71 | failureThreshold: 3 72 | successThreshold: 1 73 | livenessProbe: 74 | initialDelaySeconds: 10 75 | periodSeconds: 10 76 | timeoutSeconds: 1 77 | httpGet: 78 | port: health 79 | scheme: HTTP 80 | path: /healthz 81 | failureThreshold: 3 82 | successThreshold: 1 83 | resources: 84 | {{- toYaml .Values.resources | nindent 12 }} 85 | {{- with .Values.nodeSelector }} 86 | nodeSelector: 87 | {{- toYaml . | nindent 8 }} 88 | {{- end }} 89 | {{- with .Values.affinity }} 90 | affinity: 91 | {{- toYaml . | nindent 8 }} 92 | {{- end }} 93 | {{- with .Values.tolerations }} 94 | tolerations: 95 | {{- toYaml . | nindent 8 }} 96 | {{- end }} 97 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | name: {{ include "endpoints-operator.fullname" . }} 20 | labels: 21 | {{- include "endpoints-operator.labels" . | nindent 4 }} 22 | --- 23 | kind: ClusterRoleBinding 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | metadata: 26 | name: {{ include "endpoints-operator.fullname" . }} 27 | namespace: {{ .Release.Namespace }} 28 | # annotations: 29 | # rbac.authorization.kubernetes.io/autoupdate: "true" 30 | roleRef: 31 | kind: ClusterRole 32 | name: {{ include "endpoints-operator.fullname" . }} 33 | apiGroup: rbac.authorization.k8s.io 34 | subjects: 35 | - kind: ServiceAccount 36 | name: {{ include "endpoints-operator.fullname" . }} 37 | namespace: {{ .Release.Namespace }} 38 | --- 39 | kind: ClusterRole 40 | apiVersion: rbac.authorization.k8s.io/v1 41 | metadata: 42 | namespace: {{.Release.Namespace}} 43 | name: {{ include "endpoints-operator.fullname" . }} 44 | rules: 45 | - apiGroups: 46 | - '*' 47 | resources: 48 | - endpoints 49 | - services 50 | - configmaps 51 | - events 52 | verbs: 53 | - '*' 54 | - apiGroups: 55 | - 'sealos.io' 56 | resources: 57 | - clusterendpoints 58 | - clusterendpoints/status 59 | verbs: 60 | - '*' 61 | - apiGroups: 62 | - coordination.k8s.io 63 | resources: 64 | - leases 65 | verbs: 66 | - '*' 67 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: {{ include "endpoints-operator.fullname" . }} 19 | labels: 20 | {{- include "endpoints-operator.labels" . | nindent 4 }} 21 | spec: 22 | type: {{ .Values.service.type }} 23 | ports: 24 | - port: {{ .Values.service.port }} 25 | targetPort: metrics 26 | protocol: TCP 27 | name: metrics 28 | selector: 29 | {{- include "endpoints-operator.selectorLabels" . | nindent 4 }} 30 | -------------------------------------------------------------------------------- /config/charts/endpoints-operator/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Default values for endpoints-operator. 16 | # This is a YAML-formatted file. 17 | # Declare variables to be passed into your templates. 18 | 19 | replicaCount: 2 20 | 21 | image: 22 | repository: ghcr.io/labring 23 | image: endpoints-operator 24 | pullPolicy: IfNotPresent 25 | # Overrides the image tag whose default is the chart appVersion. 26 | tag: "latest" 27 | 28 | loglevel: 1 29 | imagePullSecrets: [] 30 | nameOverride: "" 31 | fullnameOverride: "" 32 | maxconcurrent: 1 33 | maxretry: 1 34 | 35 | podAnnotations: {} 36 | 37 | podSecurityContext: {} 38 | # fsGroup: 2000 39 | 40 | securityContext: {} 41 | # capabilities: 42 | # drop: 43 | # - ALL 44 | # readOnlyRootFilesystem: true 45 | # runAsNonRoot: true 46 | # runAsUser: 1000 47 | 48 | service: 49 | type: ClusterIP 50 | port: 80 51 | 52 | 53 | resources: 54 | limits: 55 | cpu: 500m 56 | memory: 758Mi 57 | requests: 58 | cpu: 100m 59 | memory: 128Mi 60 | 61 | 62 | nodeSelector: {} 63 | 64 | tolerations: [] 65 | 66 | affinity: {} 67 | -------------------------------------------------------------------------------- /config/crd/sealos.io_clusterendpoints.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.8.0 7 | creationTimestamp: null 8 | name: clusterendpoints.sealos.io 9 | spec: 10 | group: sealos.io 11 | names: 12 | kind: ClusterEndpoint 13 | listKind: ClusterEndpointList 14 | plural: clusterendpoints 15 | shortNames: 16 | - cep 17 | singular: clusterendpoint 18 | scope: Namespaced 19 | versions: 20 | - additionalPrinterColumns: 21 | - description: The creation date 22 | jsonPath: .metadata.creationTimestamp 23 | name: Age 24 | type: date 25 | - description: The status 26 | jsonPath: .status.phase 27 | name: Status 28 | type: string 29 | name: v1beta1 30 | schema: 31 | openAPIV3Schema: 32 | description: ClusterEndpoint is the Schema for the tests API 33 | properties: 34 | apiVersion: 35 | description: 'APIVersion defines the versioned schema of this representation 36 | of an object. Servers should convert recognized schemas to the latest 37 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 38 | type: string 39 | kind: 40 | description: 'Kind is a string value representing the REST resource this 41 | object represents. Servers may infer this from the endpoint the client 42 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 43 | type: string 44 | metadata: 45 | type: object 46 | spec: 47 | description: ClusterEndpointSpec defines the desired state of ClusterEndpoint 48 | properties: 49 | clusterIP: 50 | type: string 51 | periodSeconds: 52 | description: How often (in seconds) to perform the probe. Default 53 | to 10 seconds. Minimum value is 1. 54 | format: int32 55 | type: integer 56 | ports: 57 | items: 58 | description: ServicePort contains information on service's port. 59 | properties: 60 | failureThreshold: 61 | description: Minimum consecutive failures for the probe to be 62 | considered failed after having succeeded. Defaults to 3. Minimum 63 | value is 1. 64 | format: int32 65 | type: integer 66 | grpc: 67 | description: GRPC specifies an action involving a GRPC port. 68 | This is an alpha field and requires enabling GRPCContainerProbe 69 | feature gate. 70 | properties: 71 | enable: 72 | type: boolean 73 | service: 74 | description: "Service is the name of the service to place 75 | in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). 76 | \n If this is not specified, the default behavior is defined 77 | by gRPC." 78 | type: string 79 | required: 80 | - enable 81 | type: object 82 | hosts: 83 | items: 84 | type: string 85 | type: array 86 | httpGet: 87 | description: HTTPGet specifies the http request to perform. 88 | properties: 89 | httpHeaders: 90 | description: Custom headers to set in the request. HTTP 91 | allows repeated headers. 92 | items: 93 | description: HTTPHeader describes a custom header to be 94 | used in HTTP probes 95 | properties: 96 | name: 97 | description: The header field name 98 | type: string 99 | value: 100 | description: The header field value 101 | type: string 102 | required: 103 | - name 104 | - value 105 | type: object 106 | type: array 107 | path: 108 | description: Path to access on the HTTP server. 109 | type: string 110 | scheme: 111 | description: Scheme to use for connecting to the host. Defaults 112 | to HTTP. 113 | type: string 114 | type: object 115 | name: 116 | description: The name of this port within the service. This 117 | must be a DNS_LABEL. All ports within a ServiceSpec must have 118 | unique names. When considering the endpoints for a Service, 119 | this must match the 'name' field in the EndpointPort. Optional 120 | if only one ServicePort is defined on this service. 121 | type: string 122 | port: 123 | description: The port that will be exposed by this service. 124 | format: int32 125 | type: integer 126 | protocol: 127 | default: TCP 128 | description: The IP protocol for this port. Supports "TCP", 129 | "UDP", and "SCTP". Default is TCP. 130 | type: string 131 | successThreshold: 132 | description: Minimum consecutive successes for the probe to 133 | be considered successful after having failed. Defaults to 134 | 1. Must be 1 for liveness and startup. Minimum value is 1. 135 | format: int32 136 | type: integer 137 | targetPort: 138 | description: Number or name of the port to access on the pods 139 | targeted by the service. Number must be in the range 1 to 140 | 65535. Name must be an IANA_SVC_NAME. If this is a string, 141 | it will be looked up as a named port in the target Pod's container 142 | ports. If this is not specified, the value of the 'port' field 143 | is used (an identity map). 144 | format: int32 145 | type: integer 146 | tcpSocket: 147 | description: TCPSocket specifies an action involving a TCP port. 148 | TCP hooks not yet supported 149 | properties: 150 | enable: 151 | type: boolean 152 | required: 153 | - enable 154 | type: object 155 | timeoutSeconds: 156 | description: 'Number of seconds after which the probe times 157 | out. Defaults to 1 second. Minimum value is 1. More info: 158 | https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' 159 | format: int32 160 | type: integer 161 | udpSocket: 162 | description: UDPSocketAction specifies an action involving a 163 | UDP port. UDP hooks not yet supported 164 | properties: 165 | data: 166 | description: UDP test data 167 | items: 168 | type: integer 169 | type: array 170 | enable: 171 | type: boolean 172 | required: 173 | - enable 174 | type: object 175 | required: 176 | - port 177 | - targetPort 178 | type: object 179 | type: array 180 | type: object 181 | status: 182 | description: ClusterEndpointStatus defines the observed state of ClusterEndpoint 183 | properties: 184 | conditions: 185 | description: Conditions contains the different condition statuses 186 | for this workspace. 187 | items: 188 | properties: 189 | lastHeartbeatTime: 190 | description: LastHeartbeatTime is the last time this condition 191 | was updated. 192 | format: date-time 193 | type: string 194 | lastTransitionTime: 195 | description: LastTransitionTime is the last time the condition 196 | changed from one status to another. 197 | format: date-time 198 | type: string 199 | message: 200 | description: Message is a human-readable message indicating 201 | details about the last status change. 202 | type: string 203 | reason: 204 | description: Reason is a (brief) reason for the condition's 205 | last status change. 206 | type: string 207 | status: 208 | description: Status is the status of the condition. One of True, 209 | False, Unknown. 210 | type: string 211 | type: 212 | type: string 213 | required: 214 | - status 215 | - type 216 | type: object 217 | type: array 218 | phase: 219 | description: Phase is the recently observed lifecycle phase of the 220 | cluster endpoints. 221 | type: string 222 | required: 223 | - conditions 224 | type: object 225 | type: object 226 | served: true 227 | storage: true 228 | subresources: 229 | status: {} 230 | status: 231 | acceptedNames: 232 | kind: "" 233 | plural: "" 234 | conditions: [] 235 | storedVersions: [] 236 | -------------------------------------------------------------------------------- /config/demo/dmz-kube.yaml: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 sealos. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: sealos.io/v1beta1 16 | kind: ClusterEndpoint 17 | metadata: 18 | name: dmz-kube 19 | spec: 20 | clusterIP: None 21 | periodSeconds: 10 22 | ports: 23 | - name: https 24 | hosts: 25 | - 192.168.64.24 26 | port: 6443 27 | protocol: TCP 28 | targetPort: 6443 29 | timeoutSeconds: 1 30 | tcpSocket: 31 | enable: true 32 | -------------------------------------------------------------------------------- /controllers/controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "github.com/labring/endpoints-operator/utils/metrics" 23 | "github.com/labring/operator-sdk/controller" 24 | "sigs.k8s.io/controller-runtime/pkg/builder" 25 | "sigs.k8s.io/controller-runtime/pkg/predicate" 26 | "sigs.k8s.io/controller-runtime/pkg/ratelimiter" 27 | "time" 28 | 29 | "github.com/go-logr/logr" 30 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 31 | corev1 "k8s.io/api/core/v1" 32 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 | "k8s.io/apimachinery/pkg/runtime" 34 | "k8s.io/client-go/tools/record" 35 | ctrl "sigs.k8s.io/controller-runtime" 36 | "sigs.k8s.io/controller-runtime/pkg/client" 37 | runtimecontroller "sigs.k8s.io/controller-runtime/pkg/controller" 38 | "sigs.k8s.io/controller-runtime/pkg/handler" 39 | "sigs.k8s.io/controller-runtime/pkg/log" 40 | ) 41 | 42 | const ( 43 | controllerName = "cluster_endpoints_controller" 44 | ) 45 | 46 | // Reconciler reconciles a Service object 47 | type Reconciler struct { 48 | client.Client 49 | logger logr.Logger 50 | recorder record.EventRecorder 51 | scheme *runtime.Scheme 52 | finalizer *controller.Finalizer 53 | RetryCount int 54 | MaxConcurrent int 55 | MetricsInfo *metrics.MetricsInfo 56 | RateLimiter ratelimiter.RateLimiter 57 | } 58 | 59 | func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 60 | r.logger.V(4).Info("start reconcile for ceps") 61 | cep := &v1beta1.ClusterEndpoint{} 62 | if err := r.Get(ctx, req.NamespacedName, cep); err != nil { 63 | return ctrl.Result{}, client.IgnoreNotFound(err) 64 | } 65 | 66 | if ok, err := r.finalizer.RemoveFinalizer(ctx, cep, controller.DefaultFunc); ok { 67 | return ctrl.Result{}, err 68 | } 69 | 70 | if ok, err := r.finalizer.AddFinalizer(ctx, cep); ok { 71 | if err != nil { 72 | return ctrl.Result{}, err 73 | } else { 74 | return r.reconcile(ctx, cep) 75 | } 76 | } 77 | return ctrl.Result{}, errors.New("reconcile error from Finalizer") 78 | } 79 | 80 | func (c *Reconciler) SetupWithManager(mgr ctrl.Manager) error { 81 | if c.Client == nil { 82 | c.Client = mgr.GetClient() 83 | } 84 | c.logger = log.Log.WithName(controllerName) 85 | if c.recorder == nil { 86 | c.recorder = mgr.GetEventRecorderFor(controllerName) 87 | } 88 | if c.finalizer == nil { 89 | c.finalizer = controller.NewFinalizer(c.Client, "sealos.io/cluster-endpoints.finalizers") 90 | } 91 | c.scheme = mgr.GetScheme() 92 | c.logger.V(4).Info("init reconcile controller service") 93 | owner := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &v1beta1.ClusterEndpoint{}, handler.OnlyControllerOwner()) 94 | 95 | return ctrl.NewControllerManagedBy(mgr). 96 | For(&v1beta1.ClusterEndpoint{}, builder.WithPredicates( 97 | predicate.Or(predicate.GenerationChangedPredicate{}))). 98 | Watches(&corev1.Service{}, owner). 99 | WithOptions(runtimecontroller.Options{ 100 | MaxConcurrentReconciles: c.MaxConcurrent, 101 | RateLimiter: c.RateLimiter, 102 | }). 103 | Complete(c) 104 | } 105 | 106 | func (c *Reconciler) reconcile(ctx context.Context, obj client.Object) (ctrl.Result, error) { 107 | c.logger.V(4).Info("update reconcile controller service", "request", client.ObjectKeyFromObject(obj)) 108 | cep, ok := obj.(*v1beta1.ClusterEndpoint) 109 | if !ok { 110 | return ctrl.Result{}, errors.New("obj convert cep is error") 111 | } 112 | 113 | initializedCondition := v1beta1.Condition{ 114 | Type: v1beta1.Initialized, 115 | Status: corev1.ConditionTrue, 116 | Reason: string(v1beta1.Initialized), 117 | Message: "cluster endpoints has been initialized", 118 | LastHeartbeatTime: metav1.Now(), 119 | LastTransitionTime: metav1.Now(), 120 | } 121 | cep.Status.Phase = v1beta1.Pending 122 | if !isConditionTrue(cep, v1beta1.Initialized) { 123 | c.updateCondition(cep, initializedCondition) 124 | } 125 | 126 | c.syncService(ctx, cep) 127 | c.syncEndpoint(ctx, cep) 128 | 129 | c.logger.V(4).Info("update finished reconcile controller service", "request", client.ObjectKeyFromObject(cep)) 130 | c.syncFinalStatus(cep) 131 | err := c.updateStatus(ctx, client.ObjectKeyFromObject(cep), &cep.Status) 132 | if err != nil { 133 | c.recorder.Eventf(cep, corev1.EventTypeWarning, "SyncStatus", "Sync status %s is error: %v", cep.Name, err) 134 | return ctrl.Result{}, err 135 | } 136 | sec := time.Duration(cep.Spec.PeriodSeconds) * time.Second 137 | if cep.Spec.PeriodSeconds == 0 { 138 | return ctrl.Result{}, nil 139 | } 140 | return ctrl.Result{RequeueAfter: sec}, nil 141 | } 142 | -------------------------------------------------------------------------------- /controllers/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 24 | v1 "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/types" 27 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 28 | "k8s.io/apimachinery/pkg/util/intstr" 29 | "k8s.io/apimachinery/pkg/util/sets" 30 | "k8s.io/client-go/util/retry" 31 | ) 32 | 33 | func (c *Reconciler) updateStatus(ctx context.Context, nn types.NamespacedName, status *v1beta1.ClusterEndpointStatus) error { 34 | if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { 35 | original := &v1beta1.ClusterEndpoint{} 36 | if err := c.Get(ctx, nn, original); err != nil { 37 | return err 38 | } 39 | original.Status = *status 40 | if err := c.Client.Status().Update(ctx, original); err != nil { 41 | return err 42 | } 43 | return nil 44 | }); err != nil { 45 | return err 46 | } 47 | return nil 48 | } 49 | func (c *Reconciler) syncFinalStatus(cep *v1beta1.ClusterEndpoint) { 50 | clusterReadyCondition := v1beta1.Condition{ 51 | Type: v1beta1.Ready, 52 | Status: v1.ConditionTrue, 53 | LastHeartbeatTime: metav1.Now(), 54 | LastTransitionTime: metav1.Now(), 55 | Reason: string(v1beta1.Ready), 56 | Message: "ClusterEndpoint is available now", 57 | } 58 | if isConditionsTrue(cep) { 59 | cep.Status.Phase = v1beta1.Healthy 60 | } else { 61 | clusterReadyCondition.LastHeartbeatTime = metav1.Now() 62 | clusterReadyCondition.Status = v1.ConditionFalse 63 | clusterReadyCondition.Reason = "Not" + string(v1beta1.Ready) 64 | clusterReadyCondition.Message = "ClusterEndpoint is not available now" 65 | cep.Status.Phase = v1beta1.UnHealthy 66 | } 67 | c.updateCondition(cep, clusterReadyCondition) 68 | } 69 | 70 | type healthyHostAndPort struct { 71 | sps []v1beta1.ServicePort 72 | host string 73 | } 74 | 75 | func (hap *healthyHostAndPort) toEndpoint() v1.EndpointSubset { 76 | s := make([]v1.EndpointPort, 0) 77 | for _, sp := range hap.sps { 78 | endPoint := v1.EndpointPort{ 79 | Name: sp.Name, 80 | Port: sp.TargetPort, 81 | Protocol: sp.Protocol, 82 | } 83 | s = append(s, endPoint) 84 | } 85 | return v1.EndpointSubset{ 86 | Addresses: []v1.EndpointAddress{ 87 | { 88 | IP: hap.host, 89 | }, 90 | }, 91 | Ports: s, 92 | } 93 | } 94 | 95 | // ToAggregate converts the ErrorList into an errors.Aggregate. 96 | func ToAggregate(list []error) utilerrors.Aggregate { 97 | errs := make([]error, 0, len(list)) 98 | errorMsgs := sets.NewString() 99 | for _, err := range list { 100 | msg := fmt.Sprintf("%v", err) 101 | if errorMsgs.Has(msg) { 102 | continue 103 | } 104 | errorMsgs.Insert(msg) 105 | errs = append(errs, err) 106 | } 107 | return utilerrors.NewAggregate(errs) 108 | } 109 | 110 | func convertServicePorts(sps []v1beta1.ServicePort) []v1.ServicePort { 111 | s := make([]v1.ServicePort, 0) 112 | set := sets.NewString() 113 | for _, sp := range sps { 114 | if !set.Has(sp.Name) { 115 | set = set.Insert(sp.Name) 116 | endPoint := v1.ServicePort{ 117 | Name: sp.Name, 118 | Port: sp.Port, 119 | Protocol: sp.Protocol, 120 | TargetPort: intstr.FromString(sp.Name), 121 | } 122 | s = append(s, endPoint) 123 | } 124 | } 125 | return s 126 | } 127 | 128 | func isConditionTrue(ce *v1beta1.ClusterEndpoint, conditionType v1beta1.ConditionType) bool { 129 | for _, condition := range ce.Status.Conditions { 130 | if condition.Type == conditionType && condition.Status == v1.ConditionTrue { 131 | return true 132 | } 133 | } 134 | return false 135 | } 136 | func isConditionsTrue(ce *v1beta1.ClusterEndpoint) bool { 137 | if len(ce.Status.Conditions) == 0 { 138 | return false 139 | } 140 | for _, condition := range ce.Status.Conditions { 141 | if condition.Type == v1beta1.Ready { 142 | continue 143 | } 144 | if condition.Status != v1.ConditionTrue { 145 | return false 146 | } 147 | } 148 | return true 149 | } 150 | 151 | // updateCondition updates condition in cluster conditions using giving condition 152 | // adds condition if not existed 153 | func (c *Reconciler) updateCondition(cep *v1beta1.ClusterEndpoint, condition v1beta1.Condition) { 154 | if cep.Status.Conditions == nil { 155 | cep.Status.Conditions = make([]v1beta1.Condition, 0) 156 | } 157 | hasCondition := false 158 | for i, cond := range cep.Status.Conditions { 159 | if cond.Type == condition.Type { 160 | hasCondition = true 161 | if cond.Reason != condition.Reason || cond.Status != condition.Status || cond.Message != condition.Message { 162 | cep.Status.Conditions[i] = condition 163 | } 164 | } 165 | } 166 | if !hasCondition { 167 | cep.Status.Conditions = append(cep.Status.Conditions, condition) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /controllers/install.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 21 | v1 "k8s.io/api/core/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | k8sruntime "k8s.io/apimachinery/pkg/util/runtime" 24 | ) 25 | 26 | func Install(scheme *runtime.Scheme) { 27 | k8sruntime.Must(v1.AddToScheme(scheme)) 28 | k8sruntime.Must(v1beta1.Install(scheme)) 29 | } 30 | -------------------------------------------------------------------------------- /controllers/predicate.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 sealos. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package controllers 16 | 17 | import ( 18 | "sigs.k8s.io/controller-runtime/pkg/event" 19 | "sigs.k8s.io/controller-runtime/pkg/predicate" 20 | ) 21 | 22 | type ResourceChangedPredicate struct { 23 | predicate.Funcs 24 | } 25 | 26 | func (rl *ResourceChangedPredicate) Update(e event.UpdateEvent) bool { 27 | return true 28 | } 29 | 30 | func (rl *ResourceChangedPredicate) Create(e event.CreateEvent) bool { 31 | return true 32 | } 33 | 34 | // Delete returns true if the Delete event should be processed 35 | func (rl *ResourceChangedPredicate) Delete(e event.DeleteEvent) bool { 36 | return true 37 | } 38 | 39 | // Generic returns true if the Generic event should be processed 40 | func (rl *ResourceChangedPredicate) Generic(e event.GenericEvent) bool { 41 | return true 42 | } 43 | -------------------------------------------------------------------------------- /controllers/run_probe.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | Copyright 2014 The Kubernetes Authors. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | package controllers 19 | 20 | import ( 21 | "errors" 22 | "fmt" 23 | "net" 24 | "net/http" 25 | "net/url" 26 | "strconv" 27 | "strings" 28 | "time" 29 | 30 | libv1 "github.com/labring/operator-sdk/api/core/v1" 31 | "github.com/labring/operator-sdk/probe" 32 | execprobe "github.com/labring/operator-sdk/probe/exec" 33 | grpcprobe "github.com/labring/operator-sdk/probe/grpc" 34 | httpprobe "github.com/labring/operator-sdk/probe/http" 35 | tcpprobe "github.com/labring/operator-sdk/probe/tcp" 36 | udpprobe "github.com/labring/operator-sdk/probe/udp" 37 | v1 "k8s.io/api/core/v1" 38 | "k8s.io/apimachinery/pkg/util/intstr" 39 | urutime "k8s.io/apimachinery/pkg/util/runtime" 40 | "k8s.io/klog/v2" 41 | ) 42 | 43 | type work struct { 44 | p *libv1.Probe 45 | resultRun int 46 | lastResult probe.Result 47 | retry int 48 | err error 49 | } 50 | 51 | func (pb *prober) runProbeWithRetries(p *libv1.Probe, retries int) (probe.Result, string, error) { 52 | var err error 53 | var result probe.Result 54 | var output string 55 | for i := 0; i < retries; i++ { 56 | result, output, err = pb.runProbe(p) 57 | if err == nil { 58 | return result, output, nil 59 | } 60 | } 61 | return result, output, err 62 | } 63 | 64 | // nolint: errcheck 65 | func (w *work) doProbe() (keepGoing bool) { 66 | defer func() { recover() }() // Actually eat panics (HandleCrash takes care of logging) 67 | defer urutime.HandleCrash(func(_ interface{}) { keepGoing = true }) 68 | 69 | // the full container environment here, OR we must make a call to the CRI in order to get those environment 70 | // values from the running container. 71 | result, output, err := proberCheck.runProbeWithRetries(w.p, w.retry) 72 | if err != nil { 73 | w.err = err 74 | return false 75 | } 76 | 77 | if w.lastResult == result { 78 | w.resultRun++ 79 | } else { 80 | w.lastResult = result 81 | w.resultRun = 1 82 | } 83 | 84 | if (result == probe.Failure && w.resultRun < int(w.p.FailureThreshold)) || 85 | (result == probe.Success && w.resultRun < int(w.p.SuccessThreshold)) { 86 | // Success or failure is below threshold - leave the probe state unchanged. 87 | return true 88 | } 89 | 90 | if err != nil { 91 | w.err = err 92 | } else if result == probe.Failure && len(output) != 0 { 93 | w.err = errors.New(output) 94 | } 95 | return false 96 | } 97 | 98 | // Prober helps to check the liveness/readiness/startup of a container. 99 | type prober struct { 100 | exec execprobe.Prober 101 | http httpprobe.Prober 102 | tcp tcpprobe.Prober 103 | udp udpprobe.Prober 104 | grpc grpcprobe.Prober 105 | } 106 | 107 | var proberCheck = newProber() 108 | 109 | // NewProber creates a Prober, it takes a command runner and 110 | // several container info managers. 111 | func newProber() *prober { 112 | 113 | const followNonLocalRedirects = false 114 | return &prober{ 115 | exec: execprobe.New(), 116 | http: httpprobe.New(followNonLocalRedirects), 117 | tcp: tcpprobe.New(), 118 | udp: udpprobe.New(), 119 | grpc: grpcprobe.New(), 120 | } 121 | } 122 | 123 | func (pb *prober) runProbe(p *libv1.Probe) (probe.Result, string, error) { 124 | timeout := time.Duration(p.TimeoutSeconds) * time.Second 125 | if p.Exec != nil { 126 | klog.V(4).Infof("Exec-Probe Command: %v", p.Exec.Command) 127 | //command := "" 128 | return probe.Success, "", nil 129 | } 130 | if p.HTTPGet != nil { 131 | scheme := strings.ToLower(string(p.HTTPGet.Scheme)) 132 | host := p.HTTPGet.Host 133 | port, err := extractPort(p.HTTPGet.Port) 134 | if err != nil { 135 | return probe.Unknown, "", err 136 | } 137 | path := p.HTTPGet.Path 138 | klog.V(4).Infof("HTTP-Probe Host: %v://%v, Port: %v, Path: %v", scheme, host, port, path) 139 | url := formatURL(scheme, host, port, path) 140 | headers := buildHeader(p.HTTPGet.HTTPHeaders) 141 | klog.V(4).Infof("HTTP-Probe Headers: %v", headers) 142 | return pb.http.Probe(url, headers, timeout) 143 | } 144 | if p.TCPSocket != nil { 145 | port, err := extractPort(p.TCPSocket.Port) 146 | if err != nil { 147 | return probe.Unknown, "", err 148 | } 149 | host := p.TCPSocket.Host 150 | klog.V(4).Infof("TCP-Probe Host: %v, Port: %v, Timeout: %v", host, port, timeout) 151 | return pb.tcp.Probe(host, port, timeout) 152 | } 153 | if p.UDPSocket != nil { 154 | port, err := extractPort(p.UDPSocket.Port) 155 | if err != nil { 156 | return probe.Unknown, "", err 157 | } 158 | host := p.UDPSocket.Host 159 | klog.V(4).Infof("UDP-Probe Host: %v, Port: %v, Timeout: %v", host, port, timeout) 160 | return pb.udp.Probe(host, port, p.UDPSocket.Data, timeout) 161 | } 162 | if p.GRPC != nil { 163 | host := &(p.GRPC.Host) 164 | service := p.GRPC.Service 165 | klog.V(4).Info("GRPC-Probe Host: %v,Service: %v, Port: %v, Timeout: %v", host, service, p.GRPC.Port, timeout) 166 | return pb.grpc.Probe(*host, *service, int(p.GRPC.Port), timeout) 167 | } 168 | klog.Warning("failed to find probe builder") 169 | return probe.Warning, "", nil 170 | } 171 | 172 | func extractPort(param intstr.IntOrString) (int, error) { 173 | port := -1 174 | switch param.Type { 175 | case intstr.Int: 176 | port = param.IntValue() 177 | default: 178 | return port, fmt.Errorf("intOrString had no kind: %+v", param) 179 | } 180 | if port > 0 && port < 65536 { 181 | return port, nil 182 | } 183 | return port, fmt.Errorf("invalid port number: %v", port) 184 | } 185 | 186 | // formatURL formats a URL from args. For testability. 187 | func formatURL(scheme string, host string, port int, path string) *url.URL { 188 | u, err := url.Parse(path) 189 | // Something is busted with the path, but it's too late to reject it. Pass it along as is. 190 | if err != nil { 191 | u = &url.URL{ 192 | Path: path, 193 | } 194 | } 195 | u.Scheme = scheme 196 | u.Host = net.JoinHostPort(host, strconv.Itoa(port)) 197 | return u 198 | } 199 | 200 | // buildHeaderMap takes a list of HTTPHeader string 201 | // pairs and returns a populated string->[]string http.Header map. 202 | func buildHeader(headerList []v1.HTTPHeader) http.Header { 203 | headers := make(http.Header) 204 | for _, header := range headerList { 205 | headers[header.Name] = append(headers[header.Name], header.Value) 206 | } 207 | return headers 208 | } 209 | -------------------------------------------------------------------------------- /controllers/sync.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "github.com/labring/endpoints-operator/utils/metrics" 22 | "strconv" 23 | "sync" 24 | 25 | "k8s.io/klog" 26 | 27 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 28 | libv1 "github.com/labring/operator-sdk/api/core/v1" 29 | corev1 "k8s.io/api/core/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/util/intstr" 32 | "k8s.io/client-go/util/retry" 33 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 34 | ) 35 | 36 | func (c *Reconciler) syncService(ctx context.Context, cep *v1beta1.ClusterEndpoint) { 37 | serviceCondition := v1beta1.Condition{ 38 | Type: v1beta1.SyncServiceReady, 39 | Status: corev1.ConditionTrue, 40 | LastHeartbeatTime: metav1.Now(), 41 | LastTransitionTime: metav1.Now(), 42 | Reason: string(v1beta1.SyncServiceReady), 43 | Message: "sync service successfully", 44 | } 45 | 46 | if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { 47 | svc := &corev1.Service{} 48 | svc.SetName(cep.Name) 49 | svc.SetNamespace(cep.Namespace) 50 | _, err := controllerutil.CreateOrUpdate(ctx, c.Client, svc, func() error { 51 | svc.Labels = cep.Labels 52 | svc.Annotations = cep.Annotations 53 | if err := controllerutil.SetControllerReference(cep, svc, c.scheme); err != nil { 54 | return err 55 | } 56 | if cep.Spec.ClusterIP == corev1.ClusterIPNone { 57 | svc.Spec.ClusterIP = corev1.ClusterIPNone 58 | } 59 | svc.Spec.Type = corev1.ServiceTypeClusterIP 60 | svc.Spec.SessionAffinity = corev1.ServiceAffinityNone 61 | svc.Spec.Ports = convertServicePorts(cep.Spec.Ports) 62 | return nil 63 | }) 64 | return err 65 | }); err != nil { 66 | serviceCondition.LastHeartbeatTime = metav1.Now() 67 | serviceCondition.Status = corev1.ConditionFalse 68 | serviceCondition.Reason = "ServiceSyncError" 69 | serviceCondition.Message = err.Error() 70 | c.updateCondition(cep, serviceCondition) 71 | c.logger.V(4).Info("error updating service", "name", cep.Name, "msg", err.Error()) 72 | return 73 | } 74 | if !isConditionTrue(cep, v1beta1.SyncServiceReady) { 75 | c.updateCondition(cep, serviceCondition) 76 | } 77 | } 78 | func (c *Reconciler) syncEndpoint(ctx context.Context, cep *v1beta1.ClusterEndpoint) { 79 | endpointCondition := v1beta1.Condition{ 80 | Type: v1beta1.SyncEndpointReady, 81 | Status: corev1.ConditionTrue, 82 | LastHeartbeatTime: metav1.Now(), 83 | LastTransitionTime: metav1.Now(), 84 | Reason: string(v1beta1.SyncEndpointReady), 85 | Message: "sync endpoint successfully", 86 | } 87 | var syncError error = nil 88 | if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { 89 | 90 | subsets, convertError := clusterEndpointConvertEndpointSubset(cep, c.RetryCount, c.MetricsInfo) 91 | 92 | if convertError != nil && len(convertError) != 0 { 93 | syncError = ToAggregate(convertError) 94 | } 95 | ep := &corev1.Endpoints{} 96 | ep.SetName(cep.Name) 97 | ep.SetNamespace(cep.Namespace) 98 | 99 | _, err := controllerutil.CreateOrUpdate(ctx, c.Client, ep, func() error { 100 | ep.Labels = map[string]string{} 101 | if err := controllerutil.SetControllerReference(cep, ep, c.scheme); err != nil { 102 | return err 103 | } 104 | ep.Subsets = subsets 105 | return nil 106 | }) 107 | return err 108 | }); err != nil { 109 | endpointCondition.LastHeartbeatTime = metav1.Now() 110 | endpointCondition.Status = corev1.ConditionFalse 111 | endpointCondition.Reason = "EndpointSyncError" 112 | endpointCondition.Message = err.Error() 113 | c.updateCondition(cep, endpointCondition) 114 | c.logger.V(4).Info("error updating endpoint", "name", cep.Name, "msg", err.Error()) 115 | return 116 | } 117 | if syncError != nil { 118 | endpointCondition.LastHeartbeatTime = metav1.Now() 119 | endpointCondition.Status = corev1.ConditionFalse 120 | endpointCondition.Reason = "EndpointSyncPortError" 121 | endpointCondition.Message = syncError.Error() 122 | c.updateCondition(cep, endpointCondition) 123 | c.logger.V(4).Info("error healthy endpoint", "name", cep.Name, "msg", syncError.Error()) 124 | return 125 | } 126 | if !isConditionTrue(cep, v1beta1.SyncEndpointReady) { 127 | c.updateCondition(cep, endpointCondition) 128 | } 129 | } 130 | 131 | func clusterEndpointConvertEndpointSubset(cep *v1beta1.ClusterEndpoint, retry int, metricsinfo *metrics.MetricsInfo) ([]corev1.EndpointSubset, []error) { 132 | var wg sync.WaitGroup 133 | var mx sync.Mutex 134 | var data []corev1.EndpointSubset 135 | var errors []error 136 | var pointList []metrics.Point 137 | 138 | for _, p := range cep.Spec.Ports { 139 | for _, h := range p.Hosts { 140 | wg.Add(1) 141 | go func(port v1beta1.ServicePort, host string) { 142 | defer wg.Done() 143 | if port.TimeoutSeconds == 0 { 144 | port.TimeoutSeconds = 1 145 | } 146 | if port.SuccessThreshold == 0 { 147 | port.SuccessThreshold = 1 148 | } 149 | if port.FailureThreshold == 0 { 150 | port.FailureThreshold = 3 151 | } 152 | pro := &libv1.Probe{ 153 | TimeoutSeconds: port.TimeoutSeconds, 154 | SuccessThreshold: port.SuccessThreshold, 155 | FailureThreshold: port.FailureThreshold, 156 | } 157 | if port.HTTPGet != nil { 158 | // add metrics point 159 | pointList = append(pointList, metrics.Point{ 160 | Name: cep.Name, 161 | Namespace: cep.Namespace, 162 | TargetHostAndPort: host + ":" + strconv.Itoa(int(port.TargetPort)), 163 | ProbeType: metrics.HTTP, 164 | }) 165 | pro.HTTPGet = &libv1.HTTPGetAction{ 166 | Path: port.HTTPGet.Path, 167 | Port: intstr.FromInt(int(port.TargetPort)), 168 | Host: host, 169 | Scheme: port.HTTPGet.Scheme, 170 | HTTPHeaders: port.HTTPGet.HTTPHeaders, 171 | } 172 | } 173 | if port.TCPSocket != nil && port.TCPSocket.Enable { 174 | // add metrics point 175 | pointList = append(pointList, metrics.Point{ 176 | Name: cep.Name, 177 | Namespace: cep.Namespace, 178 | TargetHostAndPort: host + ":" + strconv.Itoa(int(port.TargetPort)), 179 | ProbeType: metrics.TCP, 180 | }) 181 | pro.TCPSocket = &libv1.TCPSocketAction{ 182 | Port: intstr.FromInt(int(port.TargetPort)), 183 | Host: host, 184 | } 185 | } 186 | if port.UDPSocket != nil && port.UDPSocket.Enable { 187 | // add metrics point 188 | pointList = append(pointList, metrics.Point{ 189 | Name: cep.Name, 190 | Namespace: cep.Namespace, 191 | TargetHostAndPort: host + ":" + strconv.Itoa(int(port.TargetPort)), 192 | ProbeType: metrics.UDP, 193 | }) 194 | pro.UDPSocket = &libv1.UDPSocketAction{ 195 | Port: intstr.FromInt(int(port.TargetPort)), 196 | Host: host, 197 | Data: v1beta1.Int8ArrToByteArr(port.UDPSocket.Data), 198 | } 199 | } 200 | if port.GRPC != nil && port.GRPC.Enable { 201 | // add metrics point 202 | pointList = append(pointList, metrics.Point{ 203 | Name: cep.Name, 204 | Namespace: cep.Namespace, 205 | TargetHostAndPort: host + ":" + strconv.Itoa(int(port.TargetPort)), 206 | ProbeType: metrics.GRPC, 207 | }) 208 | pro.GRPC = &libv1.GRPCAction{ 209 | Port: port.TargetPort, 210 | Host: host, 211 | Service: port.GRPC.Service, 212 | } 213 | } 214 | w := &work{p: pro, retry: retry} 215 | for w.doProbe() { 216 | } 217 | mx.Lock() 218 | defer mx.Unlock() 219 | err := w.err 220 | 221 | var probe metrics.ProbeType 222 | if w.p.ProbeHandler.Exec != nil { 223 | probe = metrics.EXEC 224 | } else if w.p.ProbeHandler.HTTPGet != nil { 225 | probe = metrics.HTTP 226 | } else if w.p.ProbeHandler.TCPSocket != nil { 227 | probe = metrics.TCP 228 | } else if w.p.ProbeHandler.UDPSocket != nil { 229 | probe = metrics.UDP 230 | } else if w.p.ProbeHandler.GRPC != nil { 231 | probe = metrics.GRPC 232 | } 233 | klog.V(4).Info("[****] Probe is ", probe) 234 | 235 | if err != nil { 236 | // add metrics point 237 | if metricsinfo != nil { 238 | metricsinfo.RecordFailedCheck(cep.Name, cep.Namespace, host+":"+strconv.Itoa(int(port.TargetPort)), string(probe)) 239 | } 240 | errors = append(errors, err) 241 | } else { 242 | // add metrics point 243 | if metricsinfo != nil { 244 | metricsinfo.RecordSuccessfulCheck(cep.Name, cep.Namespace, host+":"+strconv.Itoa(int(port.TargetPort)), string(probe)) 245 | } 246 | data = append(data, port.ToEndpointSubset(host)) 247 | } 248 | }(p, h) 249 | } 250 | } 251 | wg.Wait() 252 | for _, point := range pointList { 253 | if metricsinfo != nil { 254 | metricsinfo.RecordCheck(point.Name, point.Namespace, point.TargetHostAndPort, string(point.ProbeType)) 255 | } 256 | //metricsinfo.RecordCeps(checkdata.NsName) 257 | } 258 | return data, errors 259 | } 260 | -------------------------------------------------------------------------------- /controllers/sync_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 cuisongliu@qq.com. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "github.com/labring/endpoints-operator/utils/metrics" 21 | "reflect" 22 | "testing" 23 | 24 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 25 | corev1 "k8s.io/api/core/v1" 26 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | ) 28 | 29 | func Test_clusterEndpointConvertEndpointSubset(t *testing.T) { 30 | type args struct { 31 | cep *v1beta1.ClusterEndpoint 32 | retry int 33 | metricsinfo *metrics.MetricsInfo 34 | } 35 | tests := []struct { 36 | name string 37 | args args 38 | want []corev1.EndpointSubset 39 | want1 []error 40 | }{ 41 | { 42 | name: "default", 43 | args: args{ 44 | cep: &v1beta1.ClusterEndpoint{ 45 | TypeMeta: v1.TypeMeta{}, 46 | ObjectMeta: v1.ObjectMeta{ 47 | Name: "", 48 | Namespace: "", 49 | }, 50 | Spec: v1beta1.ClusterEndpointSpec{ 51 | Ports: []v1beta1.ServicePort{ 52 | { 53 | Hosts: []string{"172.18.1.38", "172.18.1.69", "172.18.2.18"}, 54 | Handler: v1beta1.Handler{ 55 | TCPSocket: &v1beta1.TCPSocketAction{Enable: true}, 56 | }, 57 | TimeoutSeconds: 1, 58 | SuccessThreshold: 1, 59 | FailureThreshold: 1, 60 | Name: "default", 61 | Protocol: "TCP", 62 | Port: 8080, 63 | TargetPort: 31381, 64 | }, 65 | }, 66 | PeriodSeconds: 0, 67 | }, 68 | }, 69 | retry: 3, 70 | metricsinfo: nil, 71 | }, 72 | want: nil, 73 | want1: nil, 74 | }, 75 | { 76 | name: "endpoint", 77 | args: args{ 78 | cep: &v1beta1.ClusterEndpoint{ 79 | TypeMeta: v1.TypeMeta{}, 80 | ObjectMeta: v1.ObjectMeta{ 81 | Name: "", 82 | Namespace: "", 83 | }, 84 | Spec: v1beta1.ClusterEndpointSpec{ 85 | Ports: []v1beta1.ServicePort{ 86 | { 87 | Hosts: []string{"172.31.13.241", "172.31.3.240", "172.31.4.233"}, 88 | Handler: v1beta1.Handler{ 89 | HTTPGet: &v1beta1.HTTPGetAction{ 90 | Path: "/", 91 | Scheme: "HTTP", 92 | }, 93 | }, 94 | TimeoutSeconds: 1, 95 | SuccessThreshold: 1, 96 | FailureThreshold: 3, 97 | Name: "default", 98 | Protocol: "TCP", 99 | Port: 8848, 100 | TargetPort: 8848, 101 | }, 102 | }, 103 | PeriodSeconds: 0, 104 | }, 105 | }, 106 | retry: 3, 107 | metricsinfo: nil, 108 | }, 109 | want: nil, 110 | want1: nil, 111 | }, 112 | } 113 | for _, tt := range tests { 114 | t.Run(tt.name, func(t *testing.T) { 115 | got, got1 := clusterEndpointConvertEndpointSubset(tt.args.cep, tt.args.retry, tt.args.metricsinfo) 116 | if !reflect.DeepEqual(got, tt.want) { 117 | t.Errorf("clusterEndpointConvertEndpointSubset() got = %v, want %v", got, tt.want) 118 | } 119 | if !reflect.DeepEqual(got1, tt.want1) { 120 | t.Errorf("clusterEndpointConvertEndpointSubset() got1 = %v, want %v", got1, tt.want1) 121 | } 122 | }) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /dockerfiles/cepctl/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM alpine:latest 16 | MAINTAINER "cuisongliu " 17 | 18 | ARG TARGETPLATFORM 19 | WORKDIR /root 20 | 21 | RUN --mount=target=/build tar xf /build/dist/endpoints-operator_*_$(echo ${TARGETPLATFORM} | tr '/' '_' | sed -e 's/arm_/arm/').tar.gz && cp cepctl /usr/bin && rm -rf endpoints-operator 22 | -------------------------------------------------------------------------------- /dockerfiles/endpoints-operator/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 The sealos Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM alpine:latest 16 | MAINTAINER "cuisongliu " 17 | 18 | ENTRYPOINT ["/root/endpoints-operator"] 19 | 20 | ARG TARGETPLATFORM 21 | WORKDIR /root 22 | 23 | RUN --mount=target=/build tar xf /build/dist/endpoints-operator_*_$(echo ${TARGETPLATFORM} | tr '/' '_' | sed -e 's/arm_/arm/').tar.gz && cp endpoints-operator /usr/bin && rm -rf cepctl 24 | 25 | CMD ["--help"] 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/labring/endpoints-operator 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/go-logr/logr v1.2.4 7 | github.com/labring/operator-sdk v1.0.5 8 | github.com/prometheus/client_golang v1.15.1 9 | github.com/spf13/cobra v1.6.0 10 | github.com/spf13/pflag v1.0.5 11 | k8s.io/api v0.27.2 12 | k8s.io/apimachinery v0.27.2 13 | k8s.io/client-go v0.27.2 14 | k8s.io/component-base v0.27.2 15 | k8s.io/klog v1.0.0 16 | k8s.io/klog/v2 v2.90.1 17 | sigs.k8s.io/controller-runtime v0.15.0 18 | sigs.k8s.io/yaml v1.3.0 19 | ) 20 | 21 | require ( 22 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 27 | github.com/evanphx/json-patch/v5 v5.6.0 // indirect 28 | github.com/fsnotify/fsnotify v1.6.0 // indirect 29 | github.com/go-logr/zapr v1.2.4 // indirect 30 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 31 | github.com/go-openapi/jsonreference v0.20.1 // indirect 32 | github.com/go-openapi/swag v0.22.3 // indirect 33 | github.com/gogo/protobuf v1.3.2 // indirect 34 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 35 | github.com/golang/protobuf v1.5.3 // indirect 36 | github.com/google/gnostic v0.5.7-v3refs // indirect 37 | github.com/google/go-cmp v0.5.9 // indirect 38 | github.com/google/gofuzz v1.1.0 // indirect 39 | github.com/google/uuid v1.3.0 // indirect 40 | github.com/imdario/mergo v0.3.12 // indirect 41 | github.com/inconshreveable/mousetrap v1.0.1 // indirect 42 | github.com/josharian/intern v1.0.0 // indirect 43 | github.com/json-iterator/go v1.1.12 // indirect 44 | github.com/mailru/easyjson v0.7.7 // indirect 45 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 46 | github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect 47 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 48 | github.com/modern-go/reflect2 v1.0.2 // indirect 49 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 50 | github.com/pkg/errors v0.9.1 // indirect 51 | github.com/prometheus/client_model v0.4.0 // indirect 52 | github.com/prometheus/common v0.42.0 // indirect 53 | github.com/prometheus/procfs v0.9.0 // indirect 54 | go.uber.org/atomic v1.7.0 // indirect 55 | go.uber.org/multierr v1.6.0 // indirect 56 | go.uber.org/zap v1.24.0 // indirect 57 | golang.org/x/net v0.10.0 // indirect 58 | golang.org/x/oauth2 v0.5.0 // indirect 59 | golang.org/x/sys v0.8.0 // indirect 60 | golang.org/x/term v0.8.0 // indirect 61 | golang.org/x/text v0.9.0 // indirect 62 | golang.org/x/time v0.3.0 // indirect 63 | gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect 64 | google.golang.org/appengine v1.6.7 // indirect 65 | google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect 66 | google.golang.org/grpc v1.51.0 // indirect 67 | google.golang.org/protobuf v1.30.0 // indirect 68 | gopkg.in/inf.v0 v0.9.1 // indirect 69 | gopkg.in/yaml.v2 v2.4.0 // indirect 70 | gopkg.in/yaml.v3 v3.0.1 // indirect 71 | k8s.io/apiextensions-apiserver v0.27.2 // indirect 72 | k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect 73 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 74 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 75 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 76 | ) 77 | -------------------------------------------------------------------------------- /hack/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © {{.Year}} {{.Holder}} 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The sealos Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /helm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 仓库名称 4 | repository="labring/endpoints-operator" 5 | 6 | # 版本号参数 7 | version=$1 8 | 9 | if [[ -z "$version" ]]; then 10 | # 获取最新release的版本号 11 | version=$(curl -s "https://api.github.com/repos/$repository/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') 12 | fi 13 | 14 | # 构建下载链接 15 | download_url="https://github.com/$repository/releases/download/$version/endpoints-operator-${version#v}.tgz" 16 | 17 | # 下载指定版本的release 18 | wget $download_url 19 | 20 | helm repo index . --url https://github.com/$repository/releases/download/$version 21 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 cuisongliu@qq.com. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // need using kubebuilder generator it. 18 | // DO NOT EDIT. 19 | 20 | package endpoints_operator 21 | -------------------------------------------------------------------------------- /test/concurrent_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 cuisongliu@qq.com. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package test 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 23 | "github.com/labring/endpoints-operator/test/testhelper" 24 | ) 25 | 26 | func TestAddCrs(t *testing.T) { 27 | cep := &v1beta1.ClusterEndpoint{} 28 | cep.Namespace = "default" 29 | testhelper.CreateTestCRs(10, "sealos", cep, t) 30 | } 31 | 32 | func TestDeleteCrs(t *testing.T) { 33 | testhelper.DeleteTestCRs("sealos", "default", t) 34 | } 35 | -------------------------------------------------------------------------------- /test/test.md: -------------------------------------------------------------------------------- 1 | # 功能点 2 | 3 | ## 前提条件 4 | 5 | ### 1.部署operator 6 | 7 | ### 2.准备clusterendpoints类型的yaml文件 8 | 9 | - HTTP协议 10 | 11 | - spec.ports部分配置文件： 12 | ```ports: 13 | - name: http 14 | port: 80 15 | protocol: TCP 16 | targetPort: 80 17 | httpGet: 18 | path: / 19 | scheme: http 20 | ``` 21 | - TCP协议 22 | 23 | - spec.ports部分配置文件： 24 | ```ports: 25 | - name: https 26 | port: 6443 27 | protocol: TCP 28 | targetPort: 6443 29 | timeoutSeconds: 1 30 | successThreshold: 1 31 | failureThreshold: 3 32 | tcpSocket: 33 | enable: true 34 | ``` 35 | - UDP协议 36 | 37 | - spec.ports部分配置文件： 38 | ```ports: 39 | - name: udp 40 | port: 80 41 | protocol: UDP 42 | targetPort: 80 43 | ``` 44 | - 多协议 45 | 46 | - spec.ports部分配置文件： 47 | ```ports: 48 | - name: https 49 | port: 6443 50 | protocol: TCP 51 | targetPort: 6443 52 | timeoutSeconds: 1 53 | successThreshold: 1 54 | failureThreshold: 3 55 | tcpSocket: 56 | enable: true 57 | - name: http 58 | port: 80 59 | protocol: TCP 60 | targetPort: 80 61 | httpGet: 62 | path: / 63 | scheme: http 64 | - name: udp 65 | port: 80 66 | protocol: UDP 67 | targetPort: 80 68 | ``` 69 | ## 功能测试 70 | 71 | ### 1.部署crd 72 | 73 | - 名称已存在 74 | 75 | - 预期：创建失败 76 | 77 | - 同名svc已存在 78 | 79 | - 预期： 80 | 1. 同一命名空间，创建成功，更新已有svc和ep 81 | 2. 不同命名空间，创建成功，且在当前命名空间创建svc和ep 82 | 83 | - 同名ep已存在 84 | - 预期： 85 | 1. 同一命名空间，创建成功，更新已有svc和ep 86 | 2. 不同命名空间，创建成功，且在当前命名空间创建svc和ep 87 | 88 | - 配置中port名称重复 89 | 90 | - 创建失败 91 | 92 | - 配置中port重复 93 | 94 | - 创建失败 95 | 96 | - 配置中nodeport重复 97 | 98 | - 创建失败 99 | 100 | ### 2.svc和ep创建功能点 101 | 102 | - 单port 103 | 104 | - 探活成功 105 | 106 | - 预期： 107 | 1. 同名ep创建成功，显示ENDPOINTS信息，值为IP:PORT 108 | 2. 同名svc创建成功， 109 | 110 | - 探活失败 111 | 112 | - 预期： 113 | 1. 同名ep创建成功，ENDPOINTS信息为 114 | 2. 同名svc创建成功 115 | 116 | - 多port 117 | 118 | - 全部探活成功 119 | 120 | - 预期： 121 | 1. 同名ep创建成功，显示ENDPOINTS信息，值为IP:PORT 122 | 2. 同名svc创建成功， 123 | 124 | - 部分探活失败 125 | 126 | - 预期： 127 | 1. 同名ep创建成功，显示探活成功的ENDPOINTS信息，值为IP:PORT 128 | 2. 同名svc创建成功， 129 | 130 | - 全部探活失败 131 | 132 | - 预期： 133 | 1. 同名ep创建成功，ENDPOINTS信息为 134 | 2. 同名svc创建成功， 135 | 136 | ### 3.删除crd创建的ep 137 | 138 | - 预期： 139 | 1. 删除成功 140 | 2. 同步自动创建出ep和更新svc？？？？ 141 | 142 | ### 4.删除crd创建的svc 143 | 144 | - 预期： 145 | 1. 删除成功 146 | 2. 同时自动创建出svc 147 | 148 | ### 5.删除crd 149 | 150 | - 预期：同步删除该crd创建的ep和svc资源 151 | 152 | ### 6.clusterip功能点 153 | 154 | - 场景一：空ip 155 | 156 | - 预期：创建crd成功后，clusterip被分配正确的ip 157 | 158 | - 场景二：指定ip-错误 159 | - 场景三：指定ip-正确且未存在 160 | 161 | - 预期：创建crd成功后，clusterip被分配指定的ip 162 | 163 | - 场景四：指定ip-正确且已存在 164 | 165 | - 预期：创建crd成功后，？？？ 166 | 167 | ### 7.探测间隔指标功能点 168 | 169 | - 默认10s 170 | - 修改指标小于10s 171 | 172 | - 预期：生效 173 | 174 | - 修改指标大于10s 175 | 176 | - 预期：生效 177 | 178 | ### 8.兼容已有的自定义svc、ep资源测试 179 | 180 | - svc和ep在同一命名空间下 181 | 182 | - 预期：可无缝切换至endpoints-operator管理 183 | 184 | -------------------------------------------------------------------------------- /test/testhelper/kube.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package testhelper 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "github.com/labring/endpoints-operator/utils/client" 21 | "sync" 22 | "testing" 23 | 24 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/labels" 27 | ) 28 | 29 | var once sync.Once 30 | var cep *client.Cep 31 | 32 | func cepClient() { 33 | once.Do(func() { 34 | op := client.NewKubernetesOptions("", "") 35 | c := client.NewKubernetesClient(op) 36 | cep = client.NewCep(c.KubernetesDynamic()) 37 | }) 38 | } 39 | 40 | func CreateTestCRs(num int, prefix string, ep *v1beta1.ClusterEndpoint, t *testing.T) { 41 | ctx := context.Background() 42 | cepClient() 43 | ep.Labels = map[string]string{"test": prefix} 44 | for i := 0; i < num; i++ { 45 | ep.Name = fmt.Sprintf("%s-%d", prefix, i) 46 | err := cep.CreateCR(ctx, ep.DeepCopy()) 47 | if err != nil { 48 | t.Errorf("%s create failed: %v", ep.Name, err) 49 | continue 50 | } 51 | t.Logf("%s create success.", ep.Name) 52 | } 53 | } 54 | 55 | func DeleteTestCRs(prefix, namespace string, t *testing.T) { 56 | m := map[string]string{"test": prefix} 57 | cepClient() 58 | ctx := context.Background() 59 | err := cep.DeleteCRs(ctx, namespace, v1.ListOptions{ 60 | LabelSelector: labels.FormatLabels(m), 61 | }) 62 | if err != nil { 63 | t.Errorf("delete failed: %v", err) 64 | return 65 | } 66 | t.Log("delete success.") 67 | } 68 | -------------------------------------------------------------------------------- /utils/client/cep.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/labring/endpoints-operator/apis/network/v1beta1" 21 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "k8s.io/client-go/dynamic" 24 | ) 25 | 26 | type Cep struct { 27 | gvr schema.GroupVersionResource 28 | client dynamic.Interface 29 | } 30 | 31 | func NewCep(client dynamic.Interface) *Cep { 32 | c := &Cep{} 33 | c.gvr = schema.GroupVersionResource{Group: v1beta1.GroupName, Version: v1beta1.GroupVersion.Version, Resource: "clusterendpoints"} 34 | //NewKubernetesClient(NewKubernetesOptions("", "")) 35 | c.client = client 36 | return c 37 | } 38 | 39 | func (c *Cep) CreateCR(ctx context.Context, endpoint *v1beta1.ClusterEndpoint) error { 40 | endpoint.APIVersion = v1beta1.GroupVersion.String() 41 | endpoint.Kind = "ClusterEndpoint" 42 | _, err := c.client.Resource(c.gvr).Namespace(endpoint.Namespace).Create(ctx, runtimeConvertUnstructured(endpoint), v1.CreateOptions{}) 43 | return err 44 | } 45 | 46 | func (c *Cep) DeleteCR(ctx context.Context, namespace, name string) error { 47 | return c.client.Resource(c.gvr).Namespace(namespace).Delete(ctx, name, v1.DeleteOptions{}) 48 | } 49 | 50 | func (c *Cep) DeleteCRs(ctx context.Context, namespace string, options v1.ListOptions) error { 51 | return c.client.Resource(c.gvr).Namespace(namespace).DeleteCollection(ctx, v1.DeleteOptions{}, options) 52 | } 53 | -------------------------------------------------------------------------------- /utils/client/kube.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "github.com/labring/endpoints-operator/utils/file" 19 | "os" 20 | "path" 21 | 22 | "k8s.io/client-go/dynamic" 23 | "k8s.io/client-go/kubernetes" 24 | "k8s.io/client-go/rest" 25 | "k8s.io/client-go/tools/clientcmd" 26 | ) 27 | 28 | type Client interface { 29 | Kubernetes() kubernetes.Interface 30 | KubernetesDynamic() dynamic.Interface 31 | Config() *rest.Config 32 | } 33 | 34 | type kubernetesClient struct { 35 | // kubernetes client interface 36 | k8s kubernetes.Interface 37 | k8sDynamic dynamic.Interface 38 | // discovery client 39 | config *rest.Config 40 | } 41 | 42 | type KubernetesOptions struct { 43 | // kubernetes clientset qps 44 | // +optional 45 | QPS float32 `json:"qps,omitempty" yaml:"qps"` 46 | // kubernetes clientset burst 47 | // +optional 48 | Burst int `json:"burst,omitempty" yaml:"burst"` 49 | Kubeconfig string 50 | Master string 51 | Config *rest.Config `json:"-" yaml:"-"` 52 | } 53 | 54 | // NewKubernetesOptions returns a `zero` instance 55 | func NewKubernetesOptions(kubeConfig, master string) *KubernetesOptions { 56 | kubeconfigPath := os.Getenv(clientcmd.RecommendedConfigPathEnvVar) 57 | if kubeConfig == "" && kubeconfigPath != "" { 58 | kubeConfig = kubeconfigPath 59 | } 60 | if kubeconfigPath == "" { 61 | kubeConfig = path.Join(file.GetUserHomeDir(), clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName) 62 | } 63 | return &KubernetesOptions{ 64 | QPS: 1e6, 65 | Burst: 1e6, 66 | Kubeconfig: kubeConfig, 67 | Master: master, 68 | } 69 | } 70 | 71 | // NewKubernetesClient creates a KubernetesClient 72 | func NewKubernetesClient(options *KubernetesOptions) Client { 73 | config := options.Config 74 | var err error 75 | if config == nil { 76 | config, err = clientcmd.BuildConfigFromFlags(options.Master, options.Kubeconfig) 77 | if err != nil { 78 | return nil 79 | } 80 | } 81 | config.QPS = options.QPS 82 | config.Burst = options.Burst 83 | var k kubernetesClient 84 | k.k8s = kubernetes.NewForConfigOrDie(config) 85 | k.k8sDynamic = dynamic.NewForConfigOrDie(config) 86 | k.config = config 87 | 88 | return &k 89 | } 90 | 91 | func (k *kubernetesClient) Kubernetes() kubernetes.Interface { 92 | return k.k8s 93 | } 94 | 95 | func (k *kubernetesClient) Config() *rest.Config { 96 | return k.config 97 | } 98 | 99 | func (k *kubernetesClient) KubernetesDynamic() dynamic.Interface { 100 | return k.k8sDynamic 101 | } 102 | -------------------------------------------------------------------------------- /utils/client/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package client 16 | 17 | import ( 18 | "github.com/labring/operator-sdk/convert" 19 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 20 | "k8s.io/apimachinery/pkg/runtime" 21 | ) 22 | 23 | func runtimeConvertUnstructured(from runtime.Object) *unstructured.Unstructured { 24 | to, ok := from.(*unstructured.Unstructured) 25 | if ok { 26 | return to 27 | } 28 | if to, err := convert.ResourceToUnstructured(from); err == nil { 29 | return to 30 | } 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /utils/file/file.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package file 16 | 17 | import ( 18 | "os" 19 | ) 20 | 21 | func GetUserHomeDir() string { 22 | home, _ := os.UserHomeDir() 23 | return home 24 | } 25 | 26 | func Exist(path string) bool { 27 | _, err := os.Stat(path) 28 | return err == nil || os.IsExist(err) 29 | } 30 | -------------------------------------------------------------------------------- /utils/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | import ( 18 | "time" 19 | 20 | "github.com/prometheus/client_golang/prometheus" 21 | crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" 22 | ) 23 | 24 | // MetricsInfo Metrics contains Prometheus metrics 25 | type MetricsInfo struct { 26 | metrics map[string]prometheus.Collector 27 | } 28 | 29 | const ( 30 | numCepsKey = "cep_num_cpes" 31 | numCheckedKey = "cep_num_checked" 32 | numCheckFailedKey = "cep_num_check_failed" 33 | numCheckSuccessfulKey = "cep_num_check_successful" 34 | checkDurationSecondsKey = "cep_check_duration_seconds" 35 | 36 | cepLabel = "name" 37 | nameSpaces = "namespaces" 38 | instance = "instance" 39 | probe = "probe" 40 | ) 41 | 42 | func NewMetricsInfo() *MetricsInfo { 43 | return &MetricsInfo{ 44 | metrics: map[string]prometheus.Collector{ 45 | numCepsKey: prometheus.NewCounterVec( 46 | prometheus.CounterOpts{ 47 | Name: numCepsKey, 48 | Help: "Total number of ceps", 49 | }, 50 | []string{"total_Ceps", nameSpaces, probe}, 51 | ), 52 | 53 | numCheckedKey: prometheus.NewCounterVec( 54 | prometheus.CounterOpts{ 55 | Name: numCheckedKey, 56 | Help: "Total number of check", 57 | }, 58 | []string{cepLabel, nameSpaces, instance, probe}, 59 | ), 60 | 61 | numCheckFailedKey: prometheus.NewCounterVec( 62 | prometheus.CounterOpts{ 63 | Name: numCheckFailedKey, 64 | Help: "Total number of failed check", 65 | }, 66 | []string{cepLabel, nameSpaces, instance, probe}, 67 | ), 68 | 69 | numCheckSuccessfulKey: prometheus.NewCounterVec( 70 | prometheus.CounterOpts{ 71 | Name: numCheckSuccessfulKey, 72 | Help: "Total number of successful check", 73 | }, 74 | []string{cepLabel, nameSpaces, instance, probe}, 75 | ), 76 | 77 | checkDurationSecondsKey: prometheus.NewHistogramVec( 78 | prometheus.HistogramOpts{ 79 | Name: checkDurationSecondsKey, 80 | Help: "Time taken to complete check, in seconds", 81 | Buckets: []float64{ 82 | 15.0, 83 | 30.0, 84 | toSeconds(1 * time.Minute), 85 | toSeconds(5 * time.Minute), 86 | toSeconds(10 * time.Minute), 87 | toSeconds(15 * time.Minute), 88 | toSeconds(30 * time.Minute), 89 | toSeconds(1 * time.Hour), 90 | toSeconds(2 * time.Hour), 91 | toSeconds(3 * time.Hour), 92 | toSeconds(4 * time.Hour), 93 | toSeconds(5 * time.Hour), 94 | toSeconds(6 * time.Hour), 95 | toSeconds(7 * time.Hour), 96 | toSeconds(8 * time.Hour), 97 | toSeconds(9 * time.Hour), 98 | toSeconds(10 * time.Hour), 99 | }, 100 | }, 101 | []string{cepLabel, nameSpaces, instance, probe}, 102 | ), 103 | }, 104 | } 105 | } 106 | 107 | func (m *MetricsInfo) RegisterAllMetrics() { 108 | for _, pm := range m.metrics { 109 | crmetrics.Registry.MustRegister(pm) 110 | } 111 | } 112 | 113 | // RecordCheck updates the total number of checked. 114 | func (m *MetricsInfo) RecordCeps(ns string) { 115 | if pm, ok := m.metrics[numCepsKey].(*prometheus.CounterVec); ok { 116 | pm.WithLabelValues(ns).Inc() 117 | } 118 | } 119 | 120 | // RecordCheck updates the total number of checked. 121 | func (m *MetricsInfo) RecordCheck(epname, ns, instance, probe string) { 122 | if pm, ok := m.metrics[numCheckedKey].(*prometheus.CounterVec); ok { 123 | pm.WithLabelValues(epname, ns, instance, probe).Inc() 124 | } 125 | } 126 | 127 | // RecordFailedCheck updates the total number of successful checked. 128 | func (m *MetricsInfo) RecordFailedCheck(epname, ns, instance, probe string) { 129 | if pm, ok := m.metrics[numCheckFailedKey].(*prometheus.CounterVec); ok { 130 | pm.WithLabelValues(epname, ns, instance, probe).Inc() 131 | } 132 | } 133 | 134 | // RecordSuccessfulCheck updates the total number of successful checked. 135 | func (m *MetricsInfo) RecordSuccessfulCheck(epname, ns, instance, probe string) { 136 | if pm, ok := m.metrics[numCheckSuccessfulKey].(*prometheus.CounterVec); ok { 137 | pm.WithLabelValues(epname, ns, instance, probe).Inc() 138 | } 139 | } 140 | 141 | // RecordCheckDuration records the number of seconds taken by a checked. 142 | func (m *MetricsInfo) RecordCheckDuration(epname, ns, instance, probe string, seconds float64) { 143 | if c, ok := m.metrics[checkDurationSecondsKey].(*prometheus.HistogramVec); ok { 144 | c.WithLabelValues(epname, ns, instance, probe).Observe(seconds) 145 | } 146 | } 147 | 148 | func toSeconds(d time.Duration) float64 { 149 | return float64(d / time.Second) 150 | } 151 | -------------------------------------------------------------------------------- /utils/metrics/types.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2022 The sealos Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package metrics 16 | 17 | type ProbeType string 18 | 19 | const ( 20 | GRPC ProbeType = "grpc" 21 | HTTP ProbeType = "http" 22 | TCP ProbeType = "tcp" 23 | EXEC ProbeType = "exec" 24 | UDP ProbeType = "udp" 25 | ) 26 | 27 | type Point struct { 28 | Name string 29 | Namespace string 30 | TargetHostAndPort string 31 | ProbeType ProbeType 32 | } 33 | --------------------------------------------------------------------------------