├── .github └── workflows │ └── build-images.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── controller │ ├── app │ │ ├── config │ │ │ └── config.go │ │ ├── options │ │ │ ├── factory.go │ │ │ └── options.go │ │ └── server.go │ └── main.go └── scheduler │ └── main.go ├── deploy ├── controller │ ├── deployment.yaml │ └── rbac.yaml ├── manifests │ ├── dynamic │ │ ├── policy.yaml │ │ └── scheduler-config.yaml │ └── noderesourcetopology │ │ ├── rbac.yaml │ │ └── scheduler-config.yaml └── scheduler │ ├── deployment.yaml │ └── rbac.yaml ├── doc ├── Dynamic-scheduler.png └── dynamic-scheduler.md ├── examples └── cpu_stress.yaml ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt ├── tools.go ├── update-all.sh ├── update-codegen.sh └── update-vendor.sh └── pkg ├── controller ├── annotator │ ├── binding.go │ ├── config │ │ └── types.go │ ├── controller.go │ ├── event.go │ ├── node.go │ └── utils.go └── prometheus │ └── prometheus.go ├── plugins ├── apis │ ├── config │ │ ├── doc.go │ │ ├── register.go │ │ ├── scheme │ │ │ └── scheme.go │ │ ├── types.go │ │ ├── v1beta2 │ │ │ ├── defaults.go │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ ├── zz_generated.conversion.go │ │ │ ├── zz_generated.deepcopy.go │ │ │ └── zz_generated.defaults.go │ │ ├── v1beta3 │ │ │ ├── defaults.go │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ ├── zz_generated.conversion.go │ │ │ ├── zz_generated.deepcopy.go │ │ │ └── zz_generated.defaults.go │ │ └── zz_generated.deepcopy.go │ └── policy │ │ ├── deepcopy_generated.go │ │ ├── doc.go │ │ ├── register.go │ │ ├── scheme │ │ └── scheme.go │ │ ├── types.go │ │ └── v1alpha1 │ │ ├── conversion_generated.go │ │ ├── deepcopy_generated.go │ │ ├── doc.go │ │ ├── register.go │ │ └── types.go ├── dynamic │ ├── plugins.go │ ├── policyfile.go │ └── stats.go └── noderesourcetopology │ ├── binder.go │ ├── cache.go │ ├── filter.go │ ├── filter_test.go │ ├── helper.go │ ├── plugin.go │ ├── reserver.go │ ├── scorer.go │ └── scorer_test.go └── utils └── utils.go /.github/workflows/build-images.yml: -------------------------------------------------------------------------------- 1 | name: build-images 2 | 3 | on: 4 | pull_request_target: 5 | types: [ opened, synchronize, reopened ] 6 | paths: 7 | - 'cmd/**' 8 | - 'pkg/**' 9 | - '**.go' 10 | - 'go.*' 11 | - '**.yml' 12 | push: 13 | paths: 14 | - 'cmd/**' 15 | - 'pkg/**' 16 | - '**.go' 17 | - 'go.*' 18 | - '**.yml' 19 | workflow_dispatch: 20 | 21 | jobs: 22 | build-scheduler: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v2 27 | with: 28 | ref: ${{github.event.pull_request.head.ref}} 29 | repository: ${{github.event.pull_request.head.repo.full_name}} 30 | - id: git-versions 31 | run: | 32 | echo "::set-output name=git-version::$(git describe --tags --always)" 33 | - id: git-branch 34 | run: | 35 | echo "::set-output name=git-branch::$(echo ${GITHUB_REF##*/} | tr '[A-Z]' '[a-z]')" 36 | - id: build-name-image 37 | run: | 38 | echo "::set-output name=build-name-image::crane-scheduler" 39 | - id: build-name-file 40 | run: | 41 | echo "::set-output name=build-name-file::$(echo "${{steps.build-name-image.outputs.build-name-image}}" | tr '/' '-')" 42 | - name: Set up QEMU 43 | uses: docker/setup-qemu-action@v2 44 | - name: Set up Docker Buildx 45 | uses: docker/setup-buildx-action@v1 46 | - name: Inspect builder 47 | run: | 48 | echo "Name: ${{ steps.buildx.outputs.name }}" 49 | echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" 50 | echo "Status: ${{ steps.buildx.outputs.status }}" 51 | echo "Flags: ${{ steps.buildx.outputs.flags }}" 52 | echo "Platforms: ${{ steps.buildx.outputs.platforms }}" 53 | 54 | - name: Login to Docker Hub 55 | uses: docker/login-action@v2 56 | with: 57 | username: ${{ secrets.DOCKERHUB_USERNAME }} 58 | password: ${{ secrets.DOCKERHUB_TOKEN }} 59 | 60 | - name: Login to Coding Container Registry 61 | uses: docker/login-action@v1 62 | with: 63 | registry: finops-docker.pkg.coding.net 64 | username: ${{ secrets.CODING_USERNAME }} 65 | password: ${{ secrets.CODING_PASSWORD }} 66 | 67 | - name: Login to GitHub Container Registry 68 | uses: docker/login-action@v2 69 | with: 70 | registry: ghcr.io 71 | username: ${{ github.actor }} 72 | password: ${{ secrets.GITHUB_TOKEN }} 73 | 74 | - id: build-ldflags 75 | run: | 76 | echo "::set-output name=build-ldflags::$(make echoLDFLAGS)" 77 | 78 | - name: Build craned and push - push 79 | if: ${{ github.event_name == 'push' }} 80 | uses: docker/build-push-action@v3 81 | with: 82 | context: . 83 | platforms: linux/amd64,linux/arm64 84 | push: true 85 | build-args: | 86 | PKGNAME=scheduler 87 | LDFLAGS=${{steps.build-ldflags.outputs.build-ldflags}} 88 | BUILD=CI 89 | tags: | 90 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 91 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 92 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 93 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 94 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 95 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 96 | 97 | - name: Build craned and push - pull_request_target 98 | if: ${{ github.event_name == 'pull_request_target' }} 99 | uses: docker/build-push-action@v3 100 | with: 101 | context: . 102 | platforms: linux/amd64,linux/arm64 103 | push: true 104 | build-args: | 105 | PKGNAME=scheduler 106 | LDFLAGS=${{steps.build-ldflags.outputs.build-ldflags}} 107 | BUILD=CI 108 | tags: | 109 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 110 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 111 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 112 | 113 | build-controller: 114 | runs-on: ubuntu-latest 115 | steps: 116 | - name: Checkout 117 | uses: actions/checkout@v2 118 | with: 119 | ref: ${{github.event.pull_request.head.ref}} 120 | repository: ${{github.event.pull_request.head.repo.full_name}} 121 | - id: git-versions 122 | run: | 123 | echo "::set-output name=git-version::$(git describe --tags --always)" 124 | - id: git-branch 125 | run: | 126 | echo "::set-output name=git-branch::$(echo ${GITHUB_REF##*/} | tr '[A-Z]' '[a-z]')" 127 | - id: build-name-image 128 | run: | 129 | echo "::set-output name=build-name-image::crane-scheduler-controller" 130 | - id: build-name-file 131 | run: | 132 | echo "::set-output name=build-name-file::$(echo "${{steps.build-name-image.outputs.build-name-image}}" | tr '/' '-')" 133 | - name: Set up QEMU 134 | uses: docker/setup-qemu-action@v2 135 | - name: Set up Docker Buildx 136 | uses: docker/setup-buildx-action@v1 137 | - name: Inspect builder 138 | run: | 139 | echo "Name: ${{ steps.buildx.outputs.name }}" 140 | echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" 141 | echo "Status: ${{ steps.buildx.outputs.status }}" 142 | echo "Flags: ${{ steps.buildx.outputs.flags }}" 143 | echo "Platforms: ${{ steps.buildx.outputs.platforms }}" 144 | 145 | - name: Login to Docker Hub 146 | uses: docker/login-action@v2 147 | with: 148 | username: ${{ secrets.DOCKERHUB_USERNAME }} 149 | password: ${{ secrets.DOCKERHUB_TOKEN }} 150 | 151 | - name: Login to Coding Container Registry 152 | uses: docker/login-action@v1 153 | with: 154 | registry: finops-docker.pkg.coding.net 155 | username: ${{ secrets.CODING_USERNAME }} 156 | password: ${{ secrets.CODING_PASSWORD }} 157 | 158 | - name: Login to GitHub Container Registry 159 | uses: docker/login-action@v2 160 | with: 161 | registry: ghcr.io 162 | username: ${{ github.actor }} 163 | password: ${{ secrets.GITHUB_TOKEN }} 164 | 165 | - id: build-ldflags 166 | run: | 167 | echo "::set-output name=build-ldflags::$(make echoLDFLAGS)" 168 | 169 | - name: Build crane-agent and push - push 170 | if: ${{ github.event_name == 'push' }} 171 | uses: docker/build-push-action@v3 172 | with: 173 | context: . 174 | platforms: linux/amd64,linux/arm64 175 | push: true 176 | build-args: | 177 | PKGNAME=controller 178 | LDFLAGS=${{steps.build-ldflags.outputs.build-ldflags}} 179 | BUILD=CI 180 | tags: | 181 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 182 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 183 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 184 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 185 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}}-${{steps.git-versions.outputs.git-version}} 186 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:${{steps.git-branch.outputs.git-branch}} 187 | 188 | - name: Build crane-agent and push - pull_request_target 189 | if: ${{ github.event_name == 'pull_request_target' }} 190 | uses: docker/build-push-action@v3 191 | with: 192 | context: . 193 | platforms: linux/amd64,linux/arm64 194 | push: true 195 | build-args: | 196 | PKGNAME=controller 197 | LDFLAGS=${{steps.build-ldflags.outputs.build-ldflags}} 198 | BUILD=CI 199 | tags: | 200 | gocrane/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 201 | ghcr.io/gocrane/crane-scheduler/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 202 | finops-docker.pkg.coding.net/gocrane/crane/${{steps.build-name-image.outputs.build-name-image}}:pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}} 203 | 204 | post-comment: 205 | runs-on: ubuntu-latest 206 | if: ${{ github.event_name == 'pull_request_target' }} 207 | needs: 208 | - build-scheduler 209 | - build-controller 210 | steps: 211 | - name: Checkout 212 | uses: actions/checkout@v2 213 | with: 214 | ref: ${{github.event.pull_request.head.ref}} 215 | repository: ${{github.event.pull_request.head.repo.full_name}} 216 | - id: git-versions 217 | run: | 218 | echo "::set-output name=git-version::$(git describe --tags --always)" 219 | - id: image-tags 220 | run: | 221 | echo "::set-output name=image-tags::pr-${{github.event.number}}-${{steps.git-versions.outputs.git-version}}" 222 | - name: Get current date 223 | id: date 224 | run: echo "::set-output name=date::$(TZ='Asia/Shanghai' date +'%Y-%m-%d %H:%M')" 225 | - name: maintain-comment 226 | uses: actions-cool/maintain-one-comment@v2 227 | with: 228 | token: ${{ secrets.GITHUB_TOKEN }} 229 | body: | 230 | 🎉 Successfully Build Images. 231 | **Now Support ARM Platforms.** 232 | Comment Post Time(CST): ${{ steps.date.outputs.date }} 233 | Git Version: ${{steps.git-versions.outputs.git-version}} 234 | 235 | #### Docker Registry 236 | 237 | Overview: https://hub.docker.com/u/gocrane 238 | 239 | | Image | Pull Command | 240 | | --------------------------------------------- | --------------------------------------------------------- | 241 | | crane-scheduler:${{steps.image-tags.outputs.image-tags}} | docker pull gocrane/crane-scheduler:${{steps.image-tags.outputs.image-tags}} | 242 | | crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | docker pull gocrane/crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | 243 | 244 | --- 245 | 246 | #### Coding Registry 247 | 248 | Overview: https://finops.coding.net/public-artifacts/gocrane/crane/packages 249 | 250 | | Image | Pull Command | 251 | | --------------------------------------------- | --------------------------------------------------------- | 252 | | crane-scheduler:${{steps.image-tags.outputs.image-tags}} | docker pull finops-docker.pkg.coding.net/gocrane/crane/crane-scheduler:${{steps.image-tags.outputs.image-tags}} | 253 | | crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | docker pull finops-docker.pkg.coding.net/gocrane/crane/crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | 254 | 255 | --- 256 | 257 | #### Ghcr Registry 258 | 259 | Overview: https://github.com/orgs/gocrane/packages?repo_name=crane 260 | 261 | | Image | Pull Command | 262 | | --------------------------------------------- | --------------------------------------------------------- | 263 | | crane-scheduler:${{steps.image-tags.outputs.image-tags}} | docker pull ghcr.io/gocrane/crane/crane-scheduler:${{steps.image-tags.outputs.image-tags}} | 264 | | crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | docker pull ghcr.io/gocrane/crane/crane-scheduler-controller:${{steps.image-tags.outputs.image-tags}} | 265 | 266 | 267 | body-include: '' 268 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | /vendor/ 12 | 13 | # macOS 14 | .DS_Store 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | .idea/ 23 | .vscode/ 24 | 25 | # binary 26 | bin/ 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PKGNAME 2 | 3 | # Build the manager binary 4 | FROM golang:1.17.2-alpine as builder 5 | 6 | ARG LDFLAGS 7 | ARG PKGNAME 8 | 9 | WORKDIR /go/src/github.com/gocrane/crane-scheduler 10 | 11 | # Add build deps 12 | RUN apk add build-base 13 | 14 | # Copy the Go Modules manifests 15 | COPY go.mod go.mod 16 | COPY go.sum go.sum 17 | # cache deps before building and copying source so that we don't need to re-download as much 18 | # and so that source changes don't invalidate our downloaded layer 19 | RUN unset https_proxy HTTPS_PROXY HTTP_PROXY http_proxy && go mod download 20 | 21 | # Copy the go source 22 | COPY pkg pkg/ 23 | COPY cmd cmd/ 24 | 25 | # Build 26 | RUN go build -ldflags="${LDFLAGS}" -a -o ${PKGNAME} /go/src/github.com/gocrane/crane-scheduler/cmd/${PKGNAME}/main.go 27 | 28 | FROM alpine:3.13.5 29 | RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories 30 | RUN unset https_proxy HTTPS_PROXY HTTP_PROXY http_proxy && apk add -U tzdata 31 | 32 | WORKDIR / 33 | ARG PKGNAME 34 | COPY --from=builder /go/src/github.com/gocrane/crane-scheduler/${PKGNAME} . 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (C) 2021 The Crane Authors & THL A29 Limited, a Tencent company. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | Other dependencies and licenses: Please refer to file LICENSES/OTHER_LICENSES 204 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOOS ?= $(shell go env GOOS) 2 | 3 | # Git information 4 | GIT_VERSION ?= $(shell git describe --tags --always) 5 | GIT_COMMIT_HASH ?= $(shell git rev-parse HEAD) 6 | GIT_TREESTATE = "clean" 7 | GIT_DIFF = $(shell git diff --quiet >/dev/null 2>&1; if [ $$? -eq 1 ]; then echo "1"; fi) 8 | ifeq ($(GIT_DIFF), 1) 9 | GIT_TREESTATE = "dirty" 10 | endif 11 | BUILDDATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') 12 | 13 | LDFLAGS = "-X github.com/gocrane/crane-scheduler/pkg/version.gitVersion=$(GIT_VERSION) \ 14 | -X github.com/gocrane/crane-scheduler/pkg/version.gitCommit=$(GIT_COMMIT_HASH) \ 15 | -X github.com/gocrane/crane-scheduler/pkg/version.gitTreeState=$(GIT_TREESTATE) \ 16 | -X github.com/gocrane/crane-scheduler/pkg/version.buildDate=$(BUILDDATE)" 17 | 18 | # Images management 19 | REGISTRY ?= docker.io 20 | REGISTRY_NAMESPACE ?= gocrane 21 | REGISTRY_USER_NAME?="" 22 | REGISTRY_PASSWORD?="" 23 | 24 | # Image URL to use all building/pushing image targets 25 | SCHEDULER_IMG ?= "${REGISTRY}/${REGISTRY_NAMESPACE}/crane-scheduler:${GIT_VERSION}" 26 | CONTROLLER_IMG ?= "${REGISTRY}/${REGISTRY_NAMESPACE}/crane-scheduler-controller:${GIT_VERSION}" 27 | 28 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 29 | ifeq (,$(shell go env GOBIN)) 30 | GOBIN=$(shell go env GOPATH)/bin 31 | else 32 | GOBIN=$(shell go env GOBIN) 33 | endif 34 | 35 | # Setting SHELL to bash allows bash commands to be executed by recipes. 36 | # This is a requirement for 'setup-envtest.sh' in the test target. 37 | # Options are set to exit when a recipe line exits non-zero or a piped command fails. 38 | SHELL = /usr/bin/env bash -o pipefail 39 | .SHELLFLAGS = -ec 40 | 41 | ##@ General 42 | 43 | # The help target prints out all targets with their descriptions organized 44 | # beneath their categories. The categories are represented by '##@' and the 45 | # target descriptions by '##'. The awk commands is responsible for reading the 46 | # entire set of makefiles included in this invocation, looking for lines of the 47 | # file as xyz: ## something, and then pretty-format the target and help. Then, 48 | # if there's a line with ##@ something, that gets pretty-printed as a category. 49 | # More info on the usage of ANSI control characters for terminal formatting: 50 | # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 51 | # More info on the awk command: 52 | # http://linuxcommand.org/lc3_adv_awk.php 53 | 54 | .PHONY: help 55 | help: ## Display this help. 56 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 57 | 58 | ##@ Development 59 | 60 | .PHONY: update 61 | update: ## Run code generator scripts. 62 | hack/update-all.sh 63 | 64 | .PHONY: fmt 65 | fmt: ## Run go fmt against code. 66 | go fmt ./... 67 | 68 | .PHONY: vet 69 | vet: ## Run go vet against code. 70 | @find . -type f -name '*.go'| grep -v "/vendor/" | xargs gofmt -w -s 71 | 72 | # Run mod tidy against code 73 | .PHONY: tidy 74 | tidy: 75 | @go mod tidy 76 | 77 | .PHONY: lint 78 | lint: golangci-lint ## Run golang lint against code 79 | @$(GOLANG_LINT) run \ 80 | --timeout 30m \ 81 | --disable-all \ 82 | -E deadcode \ 83 | -E unused \ 84 | -E varcheck \ 85 | -E ineffassign \ 86 | -E goimports \ 87 | -E gofmt \ 88 | -E misspell \ 89 | -E unparam \ 90 | -E unconvert \ 91 | -E govet \ 92 | -E errcheck \ 93 | -E structcheck 94 | 95 | .PHONY: test 96 | test: fmt vet lint ## Run tests. 97 | go test -coverprofile coverage.out -covermode=atomic ./... 98 | 99 | .PHONY: build 100 | build: scheduler controller 101 | 102 | .PHONY: all 103 | all: test scheduler controller 104 | 105 | .PHONY: scheduler 106 | scheduler: ## Build binary with the crane scheduler. 107 | CGO_ENABLED=0 GOOS=$(GOOS) go build -ldflags $(LDFLAGS) -o bin/scheduler cmd/scheduler/main.go 108 | 109 | .PHONY: controller 110 | controller: ## Build binary with the controller. 111 | CGO_ENABLED=0 GOOS=$(GOOS) go build -ldflags $(LDFLAGS) -o bin/controller cmd/controller/main.go 112 | 113 | .PHONY: images 114 | images: image-scheduler image-controller 115 | 116 | .PHONY: image-scheduler 117 | image-scheduler: ## Build docker image with the crane scheduler. 118 | docker build --build-arg LDFLAGS=$(LDFLAGS) --build-arg PKGNAME=scheduler -t ${SCHEDULER_IMG} . 119 | 120 | .PHONY: image-controller 121 | image-controller: ## Build docker image with the controller. 122 | docker build --build-arg LDFLAGS=$(LDFLAGS) --build-arg PKGNAME=controller -t ${CONTROLLER_IMG} . 123 | 124 | .PHONY: push-images 125 | push-images: push-image-scheduler push-image-controller 126 | 127 | .PHONY: push-image-scheduler 128 | push-image-scheduler: ## Push images. 129 | ifneq ($(REGISTRY_USER_NAME), "") 130 | docker login -u $(REGISTRY_USER_NAME) -p $(REGISTRY_PASSWORD) ${REGISTRY} 131 | endif 132 | docker push ${SCHEDULER_IMG} 133 | 134 | .PHONY: push-image-controller 135 | push-image-controller: ## Push images. 136 | ifneq ($(REGISTRY_USER_NAME), "") 137 | docker login -u $(REGISTRY_USER_NAME) -p $(REGISTRY_PASSWORD) ${REGISTRY} 138 | endif 139 | docker push ${CONTROLLER_IMG} 140 | 141 | .PHONY: echoLDFLAGS 142 | echoLDFLAGS: 143 | @echo $(LDFLAGS) 144 | 145 | # go-get-tool will 'go get' any package $2 and install it to $1. 146 | PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 147 | define go-get-tool 148 | @[ -f $(1) ] || { \ 149 | set -e ;\ 150 | TMP_DIR=$$(mktemp -d) ;\ 151 | cd $$TMP_DIR ;\ 152 | go mod init tmp ;\ 153 | echo "Downloading $(2)" ;\ 154 | GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ 155 | rm -rf $$TMP_DIR ;\ 156 | } 157 | endef 158 | 159 | golangci-lint: 160 | ifeq (, $(shell which golangci-lint)) 161 | @{ \ 162 | set -e ;\ 163 | export GO111MODULE=on; \ 164 | GOLANG_LINT_TMP_DIR=$$(mktemp -d) ;\ 165 | cd $$GOLANG_LINT_TMP_DIR ;\ 166 | go mod init tmp ;\ 167 | go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0 ;\ 168 | rm -rf $$GOLANG_LINT_TMP_DIR ;\ 169 | } 170 | GOLANG_LINT=$(shell go env GOPATH)/bin/golangci-lint 171 | else 172 | GOLANG_LINT=$(shell which golangci-lint) 173 | endif 174 | 175 | goimports: 176 | ifeq (, $(shell which goimports)) 177 | @{ \ 178 | set -e ;\ 179 | export GO111MODULE=on; \ 180 | GO_IMPORTS_TMP_DIR=$$(mktemp -d) ;\ 181 | cd $$GO_IMPORTS_TMP_DIR ;\ 182 | go mod init tmp ;\ 183 | go get golang.org/x/tools/cmd/goimports@v0.1.7 ;\ 184 | rm -rf $$GO_IMPORTS_TMP_DIR ;\ 185 | } 186 | GO_IMPORTS=$(shell go env GOPATH)/bin/goimports 187 | else 188 | GO_IMPORTS=$(shell which goimports) 189 | endif 190 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Crane-scheduler 2 | 3 | ## Overview 4 | Crane-scheduler is a collection of scheduler plugins based on [scheduler framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/), including: 5 | 6 | - [Dynamic scheduler: a load-aware scheduler plugin](doc/dynamic-scheduler.md) 7 | ## Get Started 8 | ### 1. Install Prometheus 9 | Make sure your kubernetes cluster has Prometheus installed. If not, please refer to [Install Prometheus](https://github.com/gocrane/fadvisor/blob/main/README.md#prerequests). 10 | ### 2. Configure Prometheus Rules 11 | Configure the rules of Prometheus to get expected aggregated data: 12 | ```yaml 13 | apiVersion: monitoring.coreos.com/v1 14 | kind: PrometheusRule 15 | metadata: 16 | name: example-record 17 | labels: 18 | prometheus: k8s 19 | role: alert-rules 20 | spec: 21 | groups: 22 | - name: cpu_mem_usage_active 23 | interval: 30s 24 | rules: 25 | - record: cpu_usage_active 26 | expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[30s])) * 100) 27 | - record: mem_usage_active 28 | expr: 100*(1-node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes) 29 | - name: cpu-usage-5m 30 | interval: 5m 31 | rules: 32 | - record: cpu_usage_max_avg_1h 33 | expr: max_over_time(cpu_usage_avg_5m[1h]) 34 | - record: cpu_usage_max_avg_1d 35 | expr: max_over_time(cpu_usage_avg_5m[1d]) 36 | - name: cpu-usage-1m 37 | interval: 1m 38 | rules: 39 | - record: cpu_usage_avg_5m 40 | expr: avg_over_time(cpu_usage_active[5m]) 41 | - name: mem-usage-5m 42 | interval: 5m 43 | rules: 44 | - record: mem_usage_max_avg_1h 45 | expr: max_over_time(mem_usage_avg_5m[1h]) 46 | - record: mem_usage_max_avg_1d 47 | expr: max_over_time(mem_usage_avg_5m[1d]) 48 | - name: mem-usage-1m 49 | interval: 1m 50 | rules: 51 | - record: mem_usage_avg_5m 52 | expr: avg_over_time(mem_usage_active[5m]) 53 | ``` 54 | >**⚠️Troubleshooting:** The sampling interval of Prometheus must be less than 30 seconds, otherwise the above rules(such as cpu_usage_active) may not take effect. 55 | 56 | ### 3. Install Crane-scheduler 57 | There are two options: 58 | 1) Install Crane-scheduler as a second scheduler: 59 | ```bash 60 | helm repo add crane https://gocrane.github.io/helm-charts 61 | helm install scheduler -n crane-system --create-namespace --set global.prometheusAddr="REPLACE_ME_WITH_PROMETHEUS_ADDR" crane/scheduler 62 | ``` 63 | 2) Replace native Kube-scheduler with Crane-scheduler: 64 | 1) Backup `/etc/kubernetes/manifests/kube-scheduler.yaml` 65 | ```bash 66 | cp /etc/kubernetes/manifests/kube-scheduler.yaml /etc/kubernetes/ 67 | ``` 68 | 2) Modify configfile of kube-scheduler(`scheduler-config.yaml`) to enable Dynamic scheduler plugin and configure plugin args: 69 | ```yaml 70 | apiVersion: kubescheduler.config.k8s.io/v1beta2 71 | kind: KubeSchedulerConfiguration 72 | ... 73 | profiles: 74 | - schedulerName: default-scheduler 75 | plugins: 76 | filter: 77 | enabled: 78 | - name: Dynamic 79 | score: 80 | enabled: 81 | - name: Dynamic 82 | weight: 3 83 | pluginConfig: 84 | - name: Dynamic 85 | args: 86 | policyConfigPath: /etc/kubernetes/policy.yaml 87 | ... 88 | ``` 89 | 3) Create `/etc/kubernetes/policy.yaml`, using as scheduler policy of Dynamic plugin: 90 | ```yaml 91 | apiVersion: scheduler.policy.crane.io/v1alpha1 92 | kind: DynamicSchedulerPolicy 93 | spec: 94 | syncPolicy: 95 | ##cpu usage 96 | - name: cpu_usage_avg_5m 97 | period: 3m 98 | - name: cpu_usage_max_avg_1h 99 | period: 15m 100 | - name: cpu_usage_max_avg_1d 101 | period: 3h 102 | ##memory usage 103 | - name: mem_usage_avg_5m 104 | period: 3m 105 | - name: mem_usage_max_avg_1h 106 | period: 15m 107 | - name: mem_usage_max_avg_1d 108 | period: 3h 109 | 110 | predicate: 111 | ##cpu usage 112 | - name: cpu_usage_avg_5m 113 | maxLimitPecent: 0.65 114 | - name: cpu_usage_max_avg_1h 115 | maxLimitPecent: 0.75 116 | ##memory usage 117 | - name: mem_usage_avg_5m 118 | maxLimitPecent: 0.65 119 | - name: mem_usage_max_avg_1h 120 | maxLimitPecent: 0.75 121 | 122 | priority: 123 | ##cpu usage 124 | - name: cpu_usage_avg_5m 125 | weight: 0.2 126 | - name: cpu_usage_max_avg_1h 127 | weight: 0.3 128 | - name: cpu_usage_max_avg_1d 129 | weight: 0.5 130 | ##memory usage 131 | - name: mem_usage_avg_5m 132 | weight: 0.2 133 | - name: mem_usage_max_avg_1h 134 | weight: 0.3 135 | - name: mem_usage_max_avg_1d 136 | weight: 0.5 137 | 138 | hotValue: 139 | - timeRange: 5m 140 | count: 5 141 | - timeRange: 1m 142 | count: 2 143 | ``` 144 | 4) Modify `kube-scheduler.yaml` and replace kube-scheduler image with Crane-scheduler： 145 | ```yaml 146 | ... 147 | image: docker.io/gocrane/crane-scheduler:0.0.23 148 | ... 149 | ``` 150 | 1) Install [crane-scheduler-controller](deploy/controller/deployment.yaml): 151 | ```bash 152 | kubectl apply ./deploy/controller/rbac.yaml && kubectl apply -f ./deploy/controller/deployment.yaml 153 | ``` 154 | 155 | ### 4. Schedule Pods With Crane-scheduler 156 | Test Crane-scheduler with following example: 157 | ```yaml 158 | apiVersion: apps/v1 159 | kind: Deployment 160 | metadata: 161 | name: cpu-stress 162 | spec: 163 | selector: 164 | matchLabels: 165 | app: cpu-stress 166 | replicas: 1 167 | template: 168 | metadata: 169 | labels: 170 | app: cpu-stress 171 | spec: 172 | schedulerName: crane-scheduler 173 | hostNetwork: true 174 | tolerations: 175 | - key: node.kubernetes.io/network-unavailable 176 | operator: Exists 177 | effect: NoSchedule 178 | containers: 179 | - name: stress 180 | image: docker.io/gocrane/stress:latest 181 | command: ["stress", "-c", "1"] 182 | resources: 183 | requests: 184 | memory: "1Gi" 185 | cpu: "1" 186 | limits: 187 | memory: "1Gi" 188 | cpu: "1" 189 | ``` 190 | >**Note:** Change `crane-scheduler` to `default-scheduler` if `crane-scheduler` is used as default. 191 | 192 | There will be the following event if the test pod is successfully scheduled: 193 | ```bash 194 | Type Reason Age From Message 195 | ---- ------ ---- ---- ------- 196 | Normal Scheduled 28s crane-scheduler Successfully assigned default/cpu-stress-7669499b57-zmrgb to vm-162-247-ubuntu 197 | ``` 198 | 199 | ## Compatibility Matrix 200 | 201 | | Scheduler Image Version | Supported Kubernetes Version | 202 | | ------------------------------ | :--------------------------: | 203 | | 0.0.23 | >=1.22.0 | 204 | | 0.0.20 | >=1.18.0 | 205 | 206 | The default scheudler image version is `0.0.23`, and you can run the following command for quick replacement: 207 | 208 | ```bash 209 | KUBE_EDITOR="sed -i 's/v1beta2/v1beta1/g'" kubectl edit cm scheduler-config -n crane-system && KUBE_EDITOR="sed -i 's/0.0.23/0.0.20/g'" kubectl edit deploy crane-scheduler -n crane-system 210 | ``` 211 | -------------------------------------------------------------------------------- /cmd/controller/app/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "k8s.io/client-go/informers" 5 | clientset "k8s.io/client-go/kubernetes" 6 | "k8s.io/client-go/tools/record" 7 | 8 | componentbaseconfig "k8s.io/component-base/config" 9 | 10 | policy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 11 | 12 | annotatorconfig "github.com/gocrane/crane-scheduler/pkg/controller/annotator/config" 13 | prom "github.com/gocrane/crane-scheduler/pkg/controller/prometheus" 14 | ) 15 | 16 | // Config is the main context object for crane scheduler controller. 17 | type Config struct { 18 | // AnnotatorConfig holds configuration for a node annotator. 19 | AnnotatorConfig *annotatorconfig.AnnotatorConfiguration 20 | // LeaderElection holds configuration for leader election. 21 | LeaderElection *componentbaseconfig.LeaderElectionConfiguration 22 | // KubeInformerFactory gives access to kubernetes informers for the controller. 23 | KubeInformerFactory informers.SharedInformerFactory 24 | // KubeClient is the general kube client. 25 | KubeClient clientset.Interface 26 | // PromClient is used for getting metric data from Prometheus. 27 | PromClient prom.PromClient 28 | // Policy is a collection of scheduler policies. 29 | Policy *policy.DynamicSchedulerPolicy 30 | // EventRecorder is the event sink 31 | EventRecorder record.EventRecorder 32 | // LeaderElectionClient is the client used for leader election 33 | LeaderElectionClient *clientset.Clientset 34 | // HealthPort is server port used for health check 35 | HealthPort string 36 | } 37 | 38 | type completedConfig struct { 39 | *Config 40 | } 41 | 42 | // CompletedConfig same as Config, just to swap private object. 43 | type CompletedConfig struct { 44 | // Embed a private pointer that cannot be instantiated outside of this package. 45 | *completedConfig 46 | } 47 | 48 | // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. 49 | func (c *Config) Complete() *CompletedConfig { 50 | cc := completedConfig{c} 51 | 52 | return &CompletedConfig{&cc} 53 | } 54 | -------------------------------------------------------------------------------- /cmd/controller/app/options/factory.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | v1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/client-go/informers" 10 | coreinformers "k8s.io/client-go/informers/core/v1" 11 | clientset "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/tools/cache" 13 | ) 14 | 15 | // NewInformerFactory creates a SharedInformerFactory and initializes an event informer that returns specified events. 16 | func NewInformerFactory(cs clientset.Interface, resyncPeriod time.Duration) informers.SharedInformerFactory { 17 | informerFactory := informers.NewSharedInformerFactory(cs, resyncPeriod) 18 | 19 | informerFactory.InformerFor(&v1.Event{}, newEventInformer) 20 | 21 | return informerFactory 22 | } 23 | 24 | // newEventInformer creates a shared index informer that returns only scheduled and normal event. 25 | func newEventInformer(cs clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 26 | selector := fmt.Sprintf("type=%s,reason=Scheduled", v1.EventTypeNormal) 27 | 28 | tweakListOptions := func(options *metav1.ListOptions) { 29 | options.FieldSelector = selector 30 | } 31 | 32 | return coreinformers.NewFilteredEventInformer(cs, metav1.NamespaceAll, resyncPeriod, nil, tweakListOptions) 33 | } 34 | -------------------------------------------------------------------------------- /cmd/controller/app/options/options.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/spf13/pflag" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | clientset "k8s.io/client-go/kubernetes" 10 | "k8s.io/client-go/rest" 11 | "k8s.io/client-go/tools/clientcmd" 12 | componentbaseconfig "k8s.io/component-base/config" 13 | options "k8s.io/component-base/config/options" 14 | 15 | controllerappconfig "github.com/gocrane/crane-scheduler/cmd/controller/app/config" 16 | annotatorconfig "github.com/gocrane/crane-scheduler/pkg/controller/annotator/config" 17 | "github.com/gocrane/crane-scheduler/pkg/controller/prometheus" 18 | dynamicscheduler "github.com/gocrane/crane-scheduler/pkg/plugins/dynamic" 19 | utils "github.com/gocrane/crane-scheduler/pkg/utils" 20 | ) 21 | 22 | const ( 23 | ControllerUserAgent = "crane-scheduler-controller" 24 | ) 25 | 26 | // Options has all the params needed to run a Annotator. 27 | type Options struct { 28 | *annotatorconfig.AnnotatorConfiguration 29 | 30 | LeaderElection *componentbaseconfig.LeaderElectionConfiguration 31 | 32 | master string 33 | kubeconfig string 34 | healthPort string 35 | } 36 | 37 | // NewOptions returns default annotator app options. 38 | func NewOptions() (*Options, error) { 39 | o := &Options{ 40 | AnnotatorConfiguration: &annotatorconfig.AnnotatorConfiguration{ 41 | BindingHeapSize: 1024, 42 | ConcurrentSyncs: 1, 43 | PolicyConfigPath: "/etc/kubernetes/policy.yaml", 44 | }, 45 | LeaderElection: &componentbaseconfig.LeaderElectionConfiguration{ 46 | LeaderElect: true, 47 | LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, 48 | RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, 49 | RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, 50 | ResourceLock: "leases", 51 | ResourceName: "crane-scheduler-controller", 52 | ResourceNamespace: utils.GetSystemNamespace(), 53 | }, 54 | healthPort: "8090", 55 | } 56 | 57 | return o, nil 58 | } 59 | 60 | // Flags returns flags for a specific Annotator by section name. 61 | func (o *Options) Flags(flag *pflag.FlagSet) error { 62 | if flag == nil { 63 | return fmt.Errorf("nil pointer") 64 | } 65 | 66 | flag.StringVar(&o.PolicyConfigPath, "policy-config-path", o.PolicyConfigPath, "Path to annotator policy config") 67 | flag.StringVar(&o.PrometheusAddr, "prometheus-address", o.PrometheusAddr, "The address of prometheus, from which we can pull metrics data.") 68 | flag.Int32Var(&o.BindingHeapSize, "binding-heap-size", o.BindingHeapSize, "Max size of binding heap size, used to store hot value data.") 69 | flag.Int32Var(&o.ConcurrentSyncs, "concurrent-syncs", o.ConcurrentSyncs, "The number of annotator controller workers that are allowed to sync concurrently.") 70 | flag.StringVar(&o.kubeconfig, "kubeconfig", o.kubeconfig, "Path to kubeconfig file with authorization information") 71 | flag.StringVar(&o.master, "master", o.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") 72 | flag.StringVar(&o.healthPort, "health-port", o.healthPort, "The port of health check") 73 | 74 | options.BindLeaderElectionFlags(o.LeaderElection, flag) 75 | return nil 76 | } 77 | 78 | // ApplyTo fills up Annotator config with options. 79 | func (o *Options) ApplyTo(c *controllerappconfig.Config) error { 80 | c.AnnotatorConfig = o.AnnotatorConfiguration 81 | c.LeaderElection = o.LeaderElection 82 | return nil 83 | } 84 | 85 | // Validate validates the options and config before launching Annotator. 86 | func (o *Options) Validate() error { 87 | return nil 88 | } 89 | 90 | // Config returns an Annotator config object. 91 | func (o *Options) Config() (*controllerappconfig.Config, error) { 92 | var kubeconfig *rest.Config 93 | var err error 94 | 95 | if err := o.Validate(); err != nil { 96 | return nil, err 97 | } 98 | 99 | c := &controllerappconfig.Config{} 100 | if err := o.ApplyTo(c); err != nil { 101 | return nil, err 102 | } 103 | 104 | c.Policy, err = dynamicscheduler.LoadPolicyFromFile(o.PolicyConfigPath) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | if o.kubeconfig == "" { 110 | kubeconfig, err = rest.InClusterConfig() 111 | } else { 112 | // Build config from configfile 113 | kubeconfig, err = clientcmd.BuildConfigFromFlags(o.master, o.kubeconfig) 114 | } 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | c.KubeClient, err = clientset.NewForConfig(rest.AddUserAgent(kubeconfig, ControllerUserAgent)) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | c.LeaderElectionClient = clientset.NewForConfigOrDie(rest.AddUserAgent(kubeconfig, "leader-election")) 125 | 126 | c.PromClient, err = prometheus.NewPromClient(o.PrometheusAddr) 127 | if err != nil { 128 | return nil, err 129 | } 130 | 131 | c.KubeInformerFactory = NewInformerFactory(c.KubeClient, 0) 132 | 133 | c.HealthPort = o.healthPort 134 | 135 | return c, nil 136 | } 137 | -------------------------------------------------------------------------------- /cmd/controller/app/server.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "time" 9 | 10 | "github.com/spf13/cobra" 11 | "k8s.io/apimachinery/pkg/util/uuid" 12 | "k8s.io/apiserver/pkg/server/healthz" 13 | "k8s.io/client-go/tools/leaderelection" 14 | "k8s.io/client-go/tools/leaderelection/resourcelock" 15 | "k8s.io/component-base/version" 16 | "k8s.io/klog/v2" 17 | 18 | "github.com/gocrane/crane-scheduler/cmd/controller/app/config" 19 | "github.com/gocrane/crane-scheduler/cmd/controller/app/options" 20 | "github.com/gocrane/crane-scheduler/pkg/controller/annotator" 21 | ) 22 | 23 | // NewControllerCommand creates a *cobra.Command object with default parameters 24 | func NewControllerCommand() *cobra.Command { 25 | o, err := options.NewOptions() 26 | if err != nil { 27 | klog.Fatalf("unable to initialize command options: %v", err) 28 | } 29 | 30 | cmd := &cobra.Command{ 31 | Use: "controller", 32 | Long: `The Crane Scheduler Controller is a kubernetes controller, which is used for annotating 33 | nodes with real load imformation sourced from Prometheus defaultly. `, 34 | Run: func(cmd *cobra.Command, args []string) { 35 | 36 | c, err := o.Config() 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "%v\n", err) 39 | os.Exit(1) 40 | } 41 | 42 | stopCh := make(chan struct{}) 43 | if err := Run(c.Complete(), stopCh); err != nil { 44 | fmt.Fprintf(os.Stderr, "%v\n", err) 45 | os.Exit(1) 46 | } 47 | }, 48 | } 49 | 50 | err = o.Flags(cmd.Flags()) 51 | if err != nil { 52 | fmt.Fprintf(os.Stderr, "%v\n", err) 53 | } 54 | 55 | return cmd 56 | } 57 | 58 | // Run executes controller based on the given configuration. 59 | func Run(cc *config.CompletedConfig, stopCh <-chan struct{}) error { 60 | 61 | klog.Infof("Starting Controller version %+v", version.Get()) 62 | 63 | run := func(ctx context.Context) { 64 | annotatorController := annotator.NewNodeAnnotator( 65 | cc.KubeInformerFactory.Core().V1().Nodes(), 66 | cc.KubeInformerFactory.Core().V1().Events(), 67 | cc.KubeClient, 68 | cc.PromClient, 69 | *cc.Policy, 70 | cc.AnnotatorConfig.BindingHeapSize, 71 | ) 72 | 73 | cc.KubeInformerFactory.Start(stopCh) 74 | 75 | panic(annotatorController.Run(int(cc.AnnotatorConfig.ConcurrentSyncs), stopCh)) 76 | } 77 | 78 | healthMux := http.NewServeMux() 79 | healthz.InstallHandler(healthMux, healthz.NamedCheck("crane-scheduler-controller", healthz.PingHealthz.Check)) 80 | go func() { 81 | if err := http.ListenAndServe(fmt.Sprintf(":%s", cc.HealthPort), healthMux); err != nil { 82 | klog.Fatal("failed to listen & server health server from port %s: %v", cc.HealthPort, err) 83 | } 84 | }() 85 | 86 | if !cc.LeaderElection.LeaderElect { 87 | run(context.TODO()) 88 | panic("unreachable") 89 | } 90 | 91 | id, err := os.Hostname() 92 | if err != nil { 93 | return err 94 | } 95 | 96 | // add a uniquifier so that two processes on the same host don't accidentally both become active 97 | id = id + "_" + string(uuid.NewUUID()) 98 | rl, err := resourcelock.New(cc.LeaderElection.ResourceLock, 99 | cc.LeaderElection.ResourceNamespace, 100 | cc.LeaderElection.ResourceName, 101 | cc.LeaderElectionClient.CoreV1(), 102 | cc.LeaderElectionClient.CoordinationV1(), 103 | resourcelock.ResourceLockConfig{ 104 | Identity: id, 105 | EventRecorder: cc.EventRecorder, 106 | }) 107 | if err != nil { 108 | panic(err) 109 | } 110 | 111 | electionChecker := leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) 112 | leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ 113 | Lock: rl, 114 | LeaseDuration: cc.LeaderElection.LeaseDuration.Duration, 115 | RenewDeadline: cc.LeaderElection.RenewDeadline.Duration, 116 | RetryPeriod: cc.LeaderElection.RetryPeriod.Duration, 117 | Callbacks: leaderelection.LeaderCallbacks{ 118 | OnStartedLeading: run, 119 | OnStoppedLeading: func() { 120 | panic("leaderelection lost") 121 | }, 122 | }, 123 | WatchDog: electionChecker, 124 | Name: "crane-scheduler-controller", 125 | }) 126 | 127 | panic("unreachable") 128 | } 129 | -------------------------------------------------------------------------------- /cmd/controller/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | goflag "flag" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "time" 9 | 10 | "github.com/spf13/pflag" 11 | cliflag "k8s.io/component-base/cli/flag" 12 | "k8s.io/component-base/logs" 13 | "k8s.io/klog/v2" 14 | 15 | "github.com/gocrane/crane-scheduler/cmd/controller/app" 16 | ) 17 | 18 | func main() { 19 | rand.Seed(time.Now().UTC().UnixNano()) 20 | cmd := app.NewControllerCommand() 21 | 22 | klog.InitFlags(nil) 23 | 24 | pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) 25 | pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) 26 | logs.InitLogs() 27 | defer logs.FlushLogs() 28 | 29 | if err := cmd.Execute(); err != nil { 30 | _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) 31 | os.Exit(1) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /cmd/scheduler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "time" 8 | 9 | "k8s.io/component-base/logs" 10 | "k8s.io/kubernetes/cmd/kube-scheduler/app" 11 | 12 | _ "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config/scheme" 13 | 14 | "github.com/gocrane/crane-scheduler/pkg/plugins/dynamic" 15 | "github.com/gocrane/crane-scheduler/pkg/plugins/noderesourcetopology" 16 | ) 17 | 18 | func main() { 19 | rand.Seed(time.Now().UTC().UnixNano()) 20 | cmd := app.NewSchedulerCommand( 21 | app.WithPlugin(dynamic.Name, dynamic.NewDynamicScheduler), 22 | app.WithPlugin(noderesourcetopology.Name, noderesourcetopology.New), 23 | ) 24 | 25 | logs.InitLogs() 26 | defer logs.FlushLogs() 27 | 28 | if err := cmd.Execute(); err != nil { 29 | _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) 30 | os.Exit(1) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /deploy/controller/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | labels: 6 | app: crane-scheduler-controller 7 | name: crane-scheduler-controller 8 | namespace: crane-system 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: crane-scheduler-controller 14 | template: 15 | metadata: 16 | labels: 17 | app: crane-scheduler-controller 18 | spec: 19 | serviceAccountName: crane-scheduler-controller 20 | containers: 21 | - name: controller 22 | env: 23 | - name: TZ 24 | value: Asia/Shanghai 25 | - name: CRANE_SYSTEM_NAMESPACE 26 | valueFrom: 27 | fieldRef: 28 | fieldPath: metadata.namespace 29 | command: 30 | - /controller 31 | - --policy-config-path=/data/policy.yaml 32 | - --prometheus-address=PROMETHEUS_ADDRESS 33 | image: docker.io/gocrane/crane-scheduler-controller:0.0.23 34 | imagePullPolicy: IfNotPresent 35 | volumeMounts: 36 | - mountPath: /data 37 | name: dynamic-scheduler-policy 38 | resources: 39 | requests: 40 | cpu: 200m 41 | memory: 200Mi 42 | livenessProbe: 43 | failureThreshold: 3 44 | httpGet: 45 | path: /healthz 46 | port: 8090 47 | scheme: HTTP 48 | initialDelaySeconds: 15 49 | periodSeconds: 10 50 | readinessProbe: 51 | httpGet: 52 | path: /healthz 53 | port: 8090 54 | scheme: HTTP 55 | dnsPolicy: ClusterFirst 56 | restartPolicy: Always 57 | volumes: 58 | - configMap: 59 | defaultMode: 420 60 | name: dynamic-scheduler-policy 61 | name: dynamic-scheduler-policy 62 | 63 | --- 64 | apiVersion: v1 65 | kind: ConfigMap 66 | metadata: 67 | name: dynamic-scheduler-policy 68 | namespace: crane-system 69 | data: 70 | policy.yaml: | 71 | apiVersion: scheduler.policy.crane.io/v1alpha1 72 | kind: DynamicSchedulerPolicy 73 | spec: 74 | syncPolicy: 75 | ##cpu usage 76 | - name: cpu_usage_avg_5m 77 | period: 3m 78 | - name: cpu_usage_max_avg_1h 79 | period: 15m 80 | - name: cpu_usage_max_avg_1d 81 | period: 3h 82 | ##memory usage 83 | - name: mem_usage_avg_5m 84 | period: 3m 85 | - name: mem_usage_max_avg_1h 86 | period: 15m 87 | - name: mem_usage_max_avg_1d 88 | period: 3h 89 | 90 | predicate: 91 | ##cpu usage 92 | - name: cpu_usage_avg_5m 93 | maxLimitPecent: 0.65 94 | - name: cpu_usage_max_avg_1h 95 | maxLimitPecent: 0.75 96 | ##memory usage 97 | - name: mem_usage_avg_5m 98 | maxLimitPecent: 0.65 99 | - name: mem_usage_max_avg_1h 100 | maxLimitPecent: 0.75 101 | 102 | priority: 103 | ###score = sum(() * weight) / len, 0 <= score <= 10 104 | ##cpu usage 105 | - name: cpu_usage_avg_5m 106 | weight: 0.2 107 | - name: cpu_usage_max_avg_1h 108 | weight: 0.3 109 | - name: cpu_usage_max_avg_1d 110 | weight: 0.5 111 | ##memory usage 112 | - name: mem_usage_avg_5m 113 | weight: 0.2 114 | - name: mem_usage_max_avg_1h 115 | weight: 0.3 116 | - name: mem_usage_max_avg_1d 117 | weight: 0.5 118 | 119 | hotValue: 120 | - timeRange: 5m 121 | count: 5 122 | - timeRange: 1m 123 | count: 2 124 | -------------------------------------------------------------------------------- /deploy/controller/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: crane-system 5 | 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: ClusterRole 9 | metadata: 10 | name: crane-scheduler-controller 11 | rules: 12 | - apiGroups: 13 | - "" 14 | resources: 15 | - pods 16 | - nodes 17 | - namespaces 18 | verbs: 19 | - list 20 | - watch 21 | - get 22 | - patch 23 | - apiGroups: 24 | - "" 25 | resources: 26 | - configmaps 27 | - endpoints 28 | - events 29 | verbs: 30 | - get 31 | - list 32 | - watch 33 | - update 34 | - create 35 | - patch 36 | - apiGroups: 37 | - coordination.k8s.io 38 | resources: 39 | - leases 40 | verbs: 41 | - create 42 | - get 43 | - update 44 | 45 | --- 46 | apiVersion: v1 47 | kind: ServiceAccount 48 | metadata: 49 | name: crane-scheduler-controller 50 | namespace: crane-system 51 | 52 | --- 53 | apiVersion: rbac.authorization.k8s.io/v1 54 | kind: ClusterRoleBinding 55 | metadata: 56 | name: crane-scheduler-controller 57 | roleRef: 58 | apiGroup: rbac.authorization.k8s.io 59 | kind: ClusterRole 60 | name: crane-scheduler-controller 61 | subjects: 62 | - kind: ServiceAccount 63 | name: crane-scheduler-controller 64 | namespace: crane-system -------------------------------------------------------------------------------- /deploy/manifests/dynamic/policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduler.policy.crane.io/v1alpha1 2 | kind: DynamicSchedulerPolicy 3 | spec: 4 | syncPolicy: 5 | ##cpu usage 6 | - name: cpu_usage_avg_5m 7 | period: 3m 8 | - name: cpu_usage_max_avg_1h 9 | period: 15m 10 | - name: cpu_usage_max_avg_1d 11 | period: 3h 12 | ##memory usage 13 | - name: mem_usage_avg_5m 14 | period: 3m 15 | - name: mem_usage_max_avg_1h 16 | period: 15m 17 | - name: mem_usage_max_avg_1d 18 | period: 3h 19 | 20 | predicate: 21 | ##cpu usage 22 | - name: cpu_usage_avg_5m 23 | maxLimitPecent: 0.65 24 | - name: cpu_usage_max_avg_1h 25 | maxLimitPecent: 0.75 26 | ##memory usage 27 | - name: mem_usage_avg_5m 28 | maxLimitPecent: 0.65 29 | - name: mem_usage_max_avg_1h 30 | maxLimitPecent: 0.75 31 | 32 | priority: 33 | ##cpu usage 34 | - name: cpu_usage_avg_5m 35 | weight: 0.2 36 | - name: cpu_usage_max_avg_1h 37 | weight: 0.3 38 | - name: cpu_usage_max_avg_1d 39 | weight: 0.5 40 | ##memory usage 41 | - name: mem_usage_avg_5m 42 | weight: 0.2 43 | - name: mem_usage_max_avg_1h 44 | weight: 0.3 45 | - name: mem_usage_max_avg_1d 46 | weight: 0.5 47 | 48 | hotValue: 49 | - timeRange: 5m 50 | count: 5 51 | - timeRange: 1m 52 | count: 2 53 | -------------------------------------------------------------------------------- /deploy/manifests/dynamic/scheduler-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubescheduler.config.k8s.io/v1beta2 2 | kind: KubeSchedulerConfiguration 3 | leaderElection: 4 | leaderElect: true 5 | clientConnection: 6 | kubeconfig: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 7 | profiles: 8 | - schedulerName: default-scheduler 9 | plugins: 10 | filter: 11 | enabled: 12 | - name: Dynamic 13 | score: 14 | enabled: 15 | - name: Dynamic 16 | weight: 3 17 | pluginConfig: 18 | - name: Dynamic 19 | args: 20 | policyConfigPath: /etc/kubernetes/policy.yaml 21 | -------------------------------------------------------------------------------- /deploy/manifests/noderesourcetopology/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: system:kube-scheduler:plugins 5 | rules: 6 | - apiGroups: 7 | - topology.crane.io 8 | resources: 9 | - "*" 10 | verbs: 11 | - get 12 | - list 13 | - watch 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - pods 18 | verbs: 19 | - patch 20 | --- 21 | kind: ClusterRoleBinding 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | metadata: 24 | name: system:kube-scheduler:plugins 25 | roleRef: 26 | apiGroup: rbac.authorization.k8s.io 27 | kind: ClusterRole 28 | name: system:kube-scheduler:plugins 29 | subjects: 30 | - kind: User 31 | apiGroup: rbac.authorization.k8s.io 32 | name: system:kube-scheduler 33 | -------------------------------------------------------------------------------- /deploy/manifests/noderesourcetopology/scheduler-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubescheduler.config.k8s.io/v1beta2 2 | kind: KubeSchedulerConfiguration 3 | leaderElection: 4 | leaderElect: true 5 | clientConnection: 6 | kubeconfig: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 7 | profiles: 8 | - schedulerName: default-scheduler 9 | plugins: 10 | preFilter: 11 | enabled: 12 | - name: NodeResourceTopologyMatch 13 | filter: 14 | enabled: 15 | - name: NodeResourceTopologyMatch 16 | score: 17 | enabled: 18 | - name: NodeResourceTopologyMatch 19 | weight: 2 20 | reserve: 21 | enabled: 22 | - name: NodeResourceTopologyMatch 23 | preBind: 24 | enabled: 25 | - name: NodeResourceTopologyMatch 26 | -------------------------------------------------------------------------------- /deploy/scheduler/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: crane-scheduler 6 | namespace: crane-system 7 | labels: 8 | component: scheduler 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | component: scheduler 14 | template: 15 | metadata: 16 | labels: 17 | component: scheduler 18 | spec: 19 | volumes: 20 | - name: scheduler-config 21 | configMap: 22 | name: scheduler-config 23 | defaultMode: 420 24 | - name: dynamic-scheduler-policy 25 | configMap: 26 | name: dynamic-scheduler-policy 27 | defaultMode: 420 28 | containers: 29 | - name: crane-scheduler 30 | image: docker.io/gocrane/crane-scheduler:0.0.23 31 | command: 32 | - /scheduler 33 | - --leader-elect=true 34 | - --config=/etc/kubernetes/kube-scheduler/scheduler-config.yaml 35 | resources: 36 | requests: 37 | cpu: 200m 38 | volumeMounts: 39 | - name: scheduler-config 40 | readOnly: true 41 | mountPath: /etc/kubernetes/kube-scheduler 42 | - name: dynamic-scheduler-policy 43 | readOnly: true 44 | mountPath: /etc/kubernetes 45 | livenessProbe: 46 | httpGet: 47 | path: /healthz 48 | port: 10259 49 | scheme: HTTPS 50 | initialDelaySeconds: 15 51 | periodSeconds: 10 52 | failureThreshold: 3 53 | readinessProbe: 54 | httpGet: 55 | path: /healthz 56 | port: 10259 57 | scheme: HTTPS 58 | restartPolicy: Always 59 | dnsPolicy: ClusterFirst 60 | serviceAccountName: scheduler 61 | serviceAccount: scheduler 62 | 63 | --- 64 | apiVersion: v1 65 | kind: ConfigMap 66 | metadata: 67 | name: scheduler-config 68 | namespace: crane-system 69 | data: 70 | scheduler-config.yaml: | 71 | apiVersion: kubescheduler.config.k8s.io/v1beta2 72 | kind: KubeSchedulerConfiguration 73 | leaderElection: 74 | leaderElect: true 75 | profiles: 76 | - schedulerName: default-scheduler 77 | plugins: 78 | filter: 79 | enabled: 80 | - name: Dynamic 81 | score: 82 | enabled: 83 | - name: Dynamic 84 | weight: 3 85 | pluginConfig: 86 | - name: Dynamic 87 | args: 88 | policyConfigPath: /etc/kubernetes/policy.yaml 89 | -------------------------------------------------------------------------------- /deploy/scheduler/rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: scheduler 6 | namespace: crane-system 7 | 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | kind: ClusterRoleBinding 11 | metadata: 12 | name: scheduler 13 | subjects: 14 | - kind: ServiceAccount 15 | name: scheduler 16 | namespace: crane-system 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: ClusterRole 20 | name: scheduler 21 | 22 | --- 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | kind: ClusterRole 25 | metadata: 26 | name: scheduler 27 | rules: 28 | - verbs: 29 | - create 30 | - patch 31 | - update 32 | apiGroups: 33 | - '' 34 | - events.k8s.io 35 | resources: 36 | - events 37 | - verbs: 38 | - create 39 | apiGroups: 40 | - coordination.k8s.io 41 | resources: 42 | - leases 43 | - verbs: 44 | - get 45 | - update 46 | apiGroups: 47 | - coordination.k8s.io 48 | resources: 49 | - leases 50 | resourceNames: 51 | - kube-scheduler 52 | - verbs: 53 | - create 54 | apiGroups: 55 | - '' 56 | resources: 57 | - endpoints 58 | - verbs: 59 | - get 60 | - update 61 | apiGroups: 62 | - '' 63 | resources: 64 | - endpoints 65 | resourceNames: 66 | - kube-scheduler 67 | - verbs: 68 | - get 69 | - list 70 | - watch 71 | apiGroups: 72 | - '' 73 | resources: 74 | - nodes 75 | - verbs: 76 | - delete 77 | - get 78 | - list 79 | - watch 80 | apiGroups: 81 | - '' 82 | resources: 83 | - pods 84 | - verbs: 85 | - create 86 | apiGroups: 87 | - '' 88 | resources: 89 | - bindings 90 | - pods/binding 91 | - verbs: 92 | - patch 93 | - update 94 | apiGroups: 95 | - '' 96 | resources: 97 | - pods/status 98 | - verbs: 99 | - get 100 | - list 101 | - watch 102 | apiGroups: 103 | - '' 104 | resources: 105 | - replicationcontrollers 106 | - services 107 | - verbs: 108 | - get 109 | - list 110 | - watch 111 | apiGroups: 112 | - apps 113 | - extensions 114 | resources: 115 | - replicasets 116 | - verbs: 117 | - get 118 | - list 119 | - watch 120 | apiGroups: 121 | - apps 122 | resources: 123 | - statefulsets 124 | - verbs: 125 | - get 126 | - list 127 | - watch 128 | apiGroups: 129 | - policy 130 | resources: 131 | - poddisruptionbudgets 132 | - verbs: 133 | - get 134 | - list 135 | - watch 136 | - update 137 | apiGroups: 138 | - '' 139 | resources: 140 | - persistentvolumeclaims 141 | - persistentvolumes 142 | - verbs: 143 | - create 144 | apiGroups: 145 | - authentication.k8s.io 146 | resources: 147 | - tokenreviews 148 | - verbs: 149 | - create 150 | apiGroups: 151 | - authorization.k8s.io 152 | resources: 153 | - subjectaccessreviews 154 | - verbs: 155 | - get 156 | - list 157 | - watch 158 | apiGroups: 159 | - storage.k8s.io 160 | resources: 161 | - '*' 162 | - verbs: 163 | - get 164 | - list 165 | - watch 166 | apiGroups: 167 | - '' 168 | resources: 169 | - configmaps 170 | - namespaces -------------------------------------------------------------------------------- /doc/Dynamic-scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gocrane/crane-scheduler/c2c05338a5d75c0a6d92bd16a1cf257b48b30ef8/doc/Dynamic-scheduler.png -------------------------------------------------------------------------------- /doc/dynamic-scheduler.md: -------------------------------------------------------------------------------- 1 | # Dynamic-scheduler: a load-aware scheduler plugin 2 | 3 | ## Introduction 4 | Native scheduler of kubernetes can only schedule pods by resource request, which can easily cause a series of load uneven problems: 5 | - for some nodes, the actual load is not much different from the resource request, which will lead to a very high probability of stability problems. 6 | - for others, the actual load is much smaller than the resource request, which will lead to a huge waste of resources. 7 | 8 | To solve these problems, Dynamic scheduler builds a simple but efficient model based on actual node utilization data，and filters out those nodes with high load to balance the cluster. 9 | ## Design Details 10 | ### Architecture 11 | 12 | 13 | 14 | As shown above, Dynamic scheduler relies on `Prometheus` and `Node-exporter` to collect and aggregate metrics data, and it consists of two components: 15 | - `Node-annotator` periodically pulls data from Prometheus and marks them with timestamp on the node in the form of annotations. 16 | >**Note:** `Node-annotator` is currently a module of `Crane-scheduler-controller`. 17 | - `Dynamic plugin` reads the load data directly from the node's annotation, filters and scores candidates based on a simple algorithm. 18 | 19 | ### Scheduler Policy 20 | Dynamic provides a default [scheduler policy](../deploy/manifests/dynamic/policy.yaml) and supports user-defined policies. The default policy reies on following metrics: 21 | - `cpu_usage_avg_5m` 22 | - `cpu_usage_max_avg_1h` 23 | - `cpu_usage_max_avg_1d` 24 | - `mem_usage_avg_5m` 25 | - `mem_usage_max_avg_1h` 26 | - `mem_usage_max_avg_1d` 27 | 28 | At the scheduling `Filter` stage, the node will be filtered if the actual usage rate of this node is greater than the threshold of any the above metrics. And at the `Score` stage, the final score is the weighted sum of these metrics' values. 29 | 30 | ### Hot Value 31 | In the production cluster, scheduling hotspots may occur frequently because the load of the nodes can not increase immediately after the pod is created. Therefore, we define an extra metrics named `Hot Value`, which represents the scheduling frequency of the node in recent times. And the final priority of the node is the final score minus the `Hot Value`. 32 | 33 | -------------------------------------------------------------------------------- /examples/cpu_stress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cpu-stress 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: cpu-stress 9 | replicas: 2 10 | template: 11 | metadata: 12 | labels: 13 | app: cpu-stress 14 | spec: 15 | schedulerName: crane-scheduler 16 | hostNetwork: true 17 | tolerations: 18 | - key: node.kubernetes.io/network-unavailable 19 | operator: Exists 20 | effect: NoSchedule 21 | containers: 22 | - name: stress 23 | image: docker.io/gocrane/stress:latest 24 | command: ["stress", "-c", "1"] 25 | resources: 26 | requests: 27 | memory: "1Gi" 28 | cpu: "1" 29 | limits: 30 | memory: "1Gi" 31 | cpu: "1" 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gocrane/crane-scheduler 2 | 3 | go 1.17 4 | 5 | replace ( 6 | k8s.io/api => k8s.io/api v0.23.3 7 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.3 8 | k8s.io/apimachinery => k8s.io/apimachinery v0.23.3 9 | k8s.io/apiserver => k8s.io/apiserver v0.23.3 10 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.3 11 | k8s.io/client-go => k8s.io/client-go v0.23.3 12 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.3 13 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.3 14 | k8s.io/code-generator => k8s.io/code-generator v0.23.3 15 | k8s.io/component-base => k8s.io/component-base v0.23.3 16 | k8s.io/component-helpers => k8s.io/component-helpers v0.23.3 17 | k8s.io/controller-manager => k8s.io/controller-manager v0.23.3 18 | k8s.io/cri-api => k8s.io/cri-api v0.23.3 19 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.3 20 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.3 21 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.3 22 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.3 23 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.3 24 | k8s.io/kubectl => k8s.io/kubectl v0.23.3 25 | k8s.io/kubelet => k8s.io/kubelet v0.23.3 26 | k8s.io/kubernetes => k8s.io/kubernetes v1.23.3 27 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.3 28 | k8s.io/metrics => k8s.io/metrics v0.23.3 29 | k8s.io/mount-utils => k8s.io/mount-utils v0.23.3 30 | k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.3 31 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.3 32 | ) 33 | 34 | require ( 35 | github.com/evanphx/json-patch v4.12.0+incompatible 36 | github.com/gocrane/api v0.7.1-0.20220819080332-e4c0d60e812d 37 | github.com/prometheus/client_golang v1.12.1 38 | github.com/prometheus/common v0.33.0 39 | github.com/spf13/cobra v1.4.0 40 | github.com/spf13/pflag v1.0.5 41 | k8s.io/api v0.23.3 42 | k8s.io/apimachinery v0.23.3 43 | k8s.io/apiserver v0.23.3 44 | k8s.io/client-go v0.23.3 45 | k8s.io/code-generator v0.23.3 46 | k8s.io/component-base v0.23.3 47 | k8s.io/klog/v2 v2.60.1 48 | k8s.io/kube-scheduler v0.23.3 49 | k8s.io/kubernetes v1.23.3 50 | ) 51 | 52 | require ( 53 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 54 | github.com/NYTimes/gziphandler v1.1.1 // indirect 55 | github.com/PuerkitoBio/purell v1.1.1 // indirect 56 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect 57 | github.com/beorn7/perks v1.0.1 // indirect 58 | github.com/blang/semver v3.5.1+incompatible // indirect 59 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 60 | github.com/coreos/go-semver v0.3.0 // indirect 61 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 62 | github.com/cyphar/filepath-securejoin v0.2.2 // indirect 63 | github.com/davecgh/go-spew v1.1.1 // indirect 64 | github.com/docker/distribution v2.8.1+incompatible // indirect 65 | github.com/emicklei/go-restful v2.9.5+incompatible // indirect 66 | github.com/felixge/httpsnoop v1.0.1 // indirect 67 | github.com/fsnotify/fsnotify v1.5.1 // indirect 68 | github.com/go-logr/logr v1.2.0 // indirect 69 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 70 | github.com/go-openapi/jsonreference v0.19.5 // indirect 71 | github.com/go-openapi/swag v0.19.14 // indirect 72 | github.com/gogo/protobuf v1.3.2 // indirect 73 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 74 | github.com/golang/protobuf v1.5.2 // indirect 75 | github.com/google/go-cmp v0.5.5 // indirect 76 | github.com/google/gofuzz v1.1.0 // indirect 77 | github.com/google/uuid v1.1.2 // indirect 78 | github.com/googleapis/gnostic v0.5.5 // indirect 79 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 80 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 81 | github.com/imdario/mergo v0.3.12 // indirect 82 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 83 | github.com/josharian/intern v1.0.0 // indirect 84 | github.com/json-iterator/go v1.1.12 // indirect 85 | github.com/mailru/easyjson v0.7.6 // indirect 86 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 87 | github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect 88 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 89 | github.com/modern-go/reflect2 v1.0.2 // indirect 90 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 91 | github.com/onsi/gomega v1.18.1 // indirect 92 | github.com/opencontainers/go-digest v1.0.0 // indirect 93 | github.com/opencontainers/runc v1.0.2 // indirect 94 | github.com/opencontainers/selinux v1.10.0 // indirect 95 | github.com/pkg/errors v0.9.1 // indirect 96 | github.com/prometheus/client_model v0.2.0 // indirect 97 | github.com/prometheus/procfs v0.7.3 // indirect 98 | go.etcd.io/etcd/api/v3 v3.5.1 // indirect 99 | go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect 100 | go.etcd.io/etcd/client/v3 v3.5.1 // indirect 101 | go.opentelemetry.io/contrib v0.20.0 // indirect 102 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect 103 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect 104 | go.opentelemetry.io/otel v0.20.0 // indirect 105 | go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect 106 | go.opentelemetry.io/otel/metric v0.20.0 // indirect 107 | go.opentelemetry.io/otel/sdk v0.20.0 // indirect 108 | go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect 109 | go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect 110 | go.opentelemetry.io/otel/trace v0.20.0 // indirect 111 | go.opentelemetry.io/proto/otlp v0.7.0 // indirect 112 | go.uber.org/atomic v1.7.0 // indirect 113 | go.uber.org/goleak v1.1.12 // indirect 114 | go.uber.org/multierr v1.6.0 // indirect 115 | go.uber.org/zap v1.19.1 // indirect 116 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect 117 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect 118 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect 119 | golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect 120 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 121 | golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect 122 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect 123 | golang.org/x/text v0.3.7 // indirect 124 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect 125 | golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 // indirect 126 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 127 | google.golang.org/appengine v1.6.7 // indirect 128 | google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect 129 | google.golang.org/grpc v1.40.0 // indirect 130 | google.golang.org/protobuf v1.27.1 // indirect 131 | gopkg.in/inf.v0 v0.9.1 // indirect 132 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 133 | gopkg.in/yaml.v2 v2.4.0 // indirect 134 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 135 | k8s.io/autoscaler/vertical-pod-autoscaler v0.10.0 // indirect 136 | k8s.io/cloud-provider v0.23.3 // indirect 137 | k8s.io/component-helpers v0.23.3 // indirect 138 | k8s.io/csi-translation-lib v0.23.3 // indirect 139 | k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect 140 | k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect 141 | k8s.io/metrics v0.23.3 // indirect 142 | k8s.io/mount-utils v0.23.3 // indirect 143 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect 144 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect 145 | sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect 146 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect 147 | sigs.k8s.io/yaml v1.3.0 // indirect 148 | ) 149 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gocrane/crane-scheduler/c2c05338a5d75c0a6d92bd16a1cf257b48b30ef8/hack/boilerplate.go.txt -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | // Copied from https://github.com/kubernetes/sample-controller/blob/master/hack/tools.go 5 | 6 | // This package imports things required by build scripts, to force `go mod` to see them as dependencies 7 | package tools 8 | 9 | import ( 10 | _ "k8s.io/code-generator" 11 | ) 12 | -------------------------------------------------------------------------------- /hack/update-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 8 | 9 | bash "$REPO_ROOT/hack/update-vendor.sh" 10 | bash "$REPO_ROOT/hack/update-codegen.sh" 11 | -------------------------------------------------------------------------------- /hack/update-codegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[@]}")/.. 8 | 9 | CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)} 10 | 11 | bash "${CODEGEN_PKG}"/generate-internal-groups.sh \ 12 | "deepcopy,conversion,defaulter" \ 13 | github.com/gocrane/crane-scheduler/pkg/generated \ 14 | github.com/gocrane/crane-scheduler/pkg/plugins/apis \ 15 | github.com/gocrane/crane-scheduler/pkg/plugins/apis \ 16 | "config:v1beta2,v1beta3" \ 17 | --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate.go.txt 18 | -------------------------------------------------------------------------------- /hack/update-vendor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | go mod tidy 8 | go mod vendor 9 | -------------------------------------------------------------------------------- /pkg/controller/annotator/binding.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "container/heap" 5 | "sync" 6 | "time" 7 | 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | // Binding is a concise struction of pod binding records, 12 | // which consists of pod name, namespace name, node name and accurate timestamp. 13 | // Note that we only record temporary Binding imformation. 14 | type Binding struct { 15 | Node string 16 | Namespace string 17 | PodName string 18 | Timestamp int64 19 | } 20 | 21 | // BindingHeap is a Heap struction storing Binding imfromation. 22 | type BindingHeap []*Binding 23 | 24 | func (b BindingHeap) Len() int { 25 | return len(b) 26 | } 27 | 28 | func (b BindingHeap) Less(i, j int) bool { 29 | return b[i].Timestamp < b[j].Timestamp 30 | } 31 | 32 | func (b BindingHeap) Swap(i, j int) { 33 | b[i], b[j] = b[j], b[i] 34 | } 35 | 36 | func (b *BindingHeap) Pop() interface{} { 37 | old := *b 38 | n := len(old) 39 | x := old[n-1] 40 | *b = old[0 : n-1] 41 | return x 42 | } 43 | 44 | func (b *BindingHeap) Push(x interface{}) { 45 | *b = append(*b, x.(*Binding)) 46 | } 47 | 48 | // BindingRecords relizes an Binding heap, which limits heap size, 49 | // and recycles automatically, contributing to store latest and least imformation. 50 | type BindingRecords struct { 51 | size int32 52 | bindings *BindingHeap 53 | gcTimeRange time.Duration 54 | rw sync.RWMutex 55 | } 56 | 57 | // NewBindingRecords returns an BindingRecords object. 58 | func NewBindingRecords(size int32, tr time.Duration) *BindingRecords { 59 | bh := BindingHeap{} 60 | heap.Init(&bh) 61 | return &BindingRecords{ 62 | size: size, 63 | bindings: &bh, 64 | gcTimeRange: tr, 65 | } 66 | } 67 | 68 | // AddBinding add new Binding to BindingHeap. 69 | func (br *BindingRecords) AddBinding(b *Binding) { 70 | br.rw.Lock() 71 | defer br.rw.Unlock() 72 | 73 | if br.bindings.Len() == int(br.size) { 74 | heap.Pop(br.bindings) 75 | } 76 | 77 | heap.Push(br.bindings, b) 78 | } 79 | 80 | // GetLastNodeBindingCount caculates how many pods scheduled on specified node recently. 81 | func (br *BindingRecords) GetLastNodeBindingCount(node string, timeRange time.Duration) int { 82 | br.rw.RLock() 83 | defer br.rw.RUnlock() 84 | 85 | cnt, timeline := 0, time.Now().UTC().Unix()-int64(timeRange.Seconds()) 86 | 87 | for _, binding := range *br.bindings { 88 | if binding.Timestamp > timeline && binding.Node == node { 89 | cnt++ 90 | } 91 | } 92 | 93 | klog.V(4).Infof("The total Binding count is %d, while node[%s] count is %d", 94 | len(*br.bindings), node, cnt) 95 | 96 | return cnt 97 | } 98 | 99 | // BindingsGC recycles expired Bindings. 100 | func (br *BindingRecords) BindingsGC() { 101 | br.rw.Lock() 102 | defer br.rw.Unlock() 103 | 104 | klog.V(4).Infof("GC period is %f", br.gcTimeRange.Seconds()) 105 | 106 | if br.gcTimeRange == 0 { 107 | return 108 | } 109 | 110 | timeline := time.Now().UTC().Unix() - int64(br.gcTimeRange.Seconds()) 111 | for br.bindings.Len() > 0 { 112 | binding := heap.Pop(br.bindings).(*Binding) 113 | 114 | klog.V(6).Infof("Try to recycle Binding(%v) with timeline %d", binding, timeline) 115 | 116 | if binding.Timestamp > timeline { 117 | heap.Push(br.bindings, binding) 118 | return 119 | } 120 | } 121 | 122 | return 123 | } 124 | -------------------------------------------------------------------------------- /pkg/controller/annotator/config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // AnnotatorConfiguration holds configuration for a node annotator. 4 | type AnnotatorConfiguration struct { 5 | // BindingHeapSize limits the size of Binding Heap, which stores the lastest 6 | // pod scheduled imformation. 7 | BindingHeapSize int32 8 | // ConcurrentSyncs specified the number of annotator controller workers. 9 | ConcurrentSyncs int32 10 | // PolicyConfigPath specified the path of Scheduler Policy File. 11 | PolicyConfigPath string 12 | // PrometheusAddr is the address of Prometheus Service. 13 | PrometheusAddr string 14 | } 15 | -------------------------------------------------------------------------------- /pkg/controller/annotator/controller.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 8 | "k8s.io/apimachinery/pkg/util/wait" 9 | coreinformers "k8s.io/client-go/informers/core/v1" 10 | clientset "k8s.io/client-go/kubernetes" 11 | corelisters "k8s.io/client-go/listers/core/v1" 12 | "k8s.io/client-go/tools/cache" 13 | "k8s.io/klog/v2" 14 | 15 | policy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 16 | 17 | prom "github.com/gocrane/crane-scheduler/pkg/controller/prometheus" 18 | ) 19 | 20 | // Controller is Controller for node annotator. 21 | type Controller struct { 22 | nodeInformer coreinformers.NodeInformer 23 | nodeInformerSynced cache.InformerSynced 24 | nodeLister corelisters.NodeLister 25 | 26 | eventInformer coreinformers.EventInformer 27 | eventInformerSynced cache.InformerSynced 28 | eventLister corelisters.EventLister 29 | 30 | kubeClient clientset.Interface 31 | promClient prom.PromClient 32 | 33 | policy policy.DynamicSchedulerPolicy 34 | bindingRecords *BindingRecords 35 | } 36 | 37 | // NewController returns a Node Annotator object. 38 | func NewNodeAnnotator( 39 | nodeInformer coreinformers.NodeInformer, 40 | eventInformer coreinformers.EventInformer, 41 | kubeClient clientset.Interface, 42 | promClient prom.PromClient, 43 | policy policy.DynamicSchedulerPolicy, 44 | bingdingHeapSize int32, 45 | ) *Controller { 46 | return &Controller{ 47 | nodeInformer: nodeInformer, 48 | nodeInformerSynced: nodeInformer.Informer().HasSynced, 49 | nodeLister: nodeInformer.Lister(), 50 | eventInformer: eventInformer, 51 | eventInformerSynced: eventInformer.Informer().HasSynced, 52 | eventLister: eventInformer.Lister(), 53 | kubeClient: kubeClient, 54 | promClient: promClient, 55 | policy: policy, 56 | bindingRecords: NewBindingRecords(bingdingHeapSize, getMaxHotVauleTimeRange(policy.Spec.HotValue)), 57 | } 58 | } 59 | 60 | // Run runs node annotator. 61 | func (c *Controller) Run(worker int, stopCh <-chan struct{}) error { 62 | defer utilruntime.HandleCrash() 63 | 64 | eventController := newEventController(c) 65 | c.eventInformer.Informer().AddEventHandler(eventController.handles()) 66 | 67 | nodeController := newNodeController(c) 68 | 69 | if !cache.WaitForCacheSync(stopCh, c.nodeInformerSynced, c.eventInformerSynced) { 70 | return fmt.Errorf("failed to wait for cache sync for annotator") 71 | } 72 | klog.Info("Caches are synced for controller") 73 | 74 | for i := 0; i < worker; i++ { 75 | go wait.Until(nodeController.Run, time.Second, stopCh) 76 | go wait.Until(eventController.Run, time.Second, stopCh) 77 | } 78 | 79 | go wait.Until(c.bindingRecords.BindingsGC, time.Minute, stopCh) 80 | 81 | nodeController.CreateMetricSyncTicker(stopCh) 82 | 83 | <-stopCh 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/controller/annotator/event.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | v1 "k8s.io/api/core/v1" 9 | "k8s.io/client-go/tools/cache" 10 | "k8s.io/client-go/util/workqueue" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | type eventController struct { 15 | *Controller 16 | queue workqueue.RateLimitingInterface 17 | } 18 | 19 | func newEventController(c *Controller) *eventController { 20 | eventRateLimiter := workqueue.DefaultControllerRateLimiter() 21 | 22 | return &eventController{ 23 | Controller: c, 24 | queue: workqueue.NewNamedRateLimitingQueue(eventRateLimiter, "EVENT_event_queue"), 25 | } 26 | } 27 | 28 | func (e *eventController) Run() { 29 | defer e.queue.ShutDown() 30 | klog.Infof("Start to reconcile EVENT events") 31 | 32 | for e.processNextWorkItem() { 33 | } 34 | } 35 | 36 | func (e *eventController) processNextWorkItem() bool { 37 | key, quit := e.queue.Get() 38 | if quit { 39 | return false 40 | } 41 | defer e.queue.Done(key) 42 | 43 | err := e.reconcile(key.(string)) 44 | if err != nil { 45 | klog.Warningf("failed to sync this EVENT [%q]: %v", key.(string), err) 46 | } 47 | 48 | return true 49 | } 50 | 51 | func (e *eventController) handles() cache.ResourceEventHandlerFuncs { 52 | return cache.ResourceEventHandlerFuncs{ 53 | AddFunc: e.handleAddEvent, 54 | UpdateFunc: e.handleUpdateEvent, 55 | } 56 | } 57 | 58 | func (e *eventController) handleAddEvent(obj interface{}) { 59 | event := obj.(*v1.Event) 60 | 61 | if event.Type != v1.EventTypeNormal || event.Reason != "Scheduled" { 62 | return 63 | } 64 | 65 | e.enqueue(obj, cache.Added) 66 | } 67 | 68 | func (e *eventController) handleUpdateEvent(old, new interface{}) { 69 | oldEvent, curEvent := old.(*v1.Event), new.(*v1.Event) 70 | 71 | if oldEvent.ResourceVersion == curEvent.ResourceVersion { 72 | return 73 | } 74 | 75 | if curEvent.Type != v1.EventTypeNormal || curEvent.Reason != "Scheduled" { 76 | return 77 | } 78 | 79 | e.enqueue(new, cache.Updated) 80 | } 81 | 82 | func (e *eventController) enqueue(obj interface{}, action cache.DeltaType) { 83 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 84 | if err != nil { 85 | return 86 | } 87 | 88 | klog.V(5).Infof("enqueue EVENT %s %s event", key, action) 89 | e.queue.Add(key) 90 | } 91 | 92 | func (e *eventController) reconcile(key string) error { 93 | startTime := time.Now() 94 | defer func() { 95 | klog.V(5).Infof("Finished syncing EVENT event %q (%v)", key, time.Since(startTime)) 96 | }() 97 | 98 | namespace, name, err := cache.SplitMetaNamespaceKey(key) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | event, err := e.eventLister.Events(namespace).Get(name) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | binding, err := translateEventToBinding(event) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | e.bindingRecords.AddBinding(binding) 114 | 115 | return nil 116 | } 117 | 118 | func translateEventToBinding(event *v1.Event) (*Binding, error) { 119 | var metaKey, nodeName string 120 | 121 | _, err := fmt.Fscanf(strings.NewReader(event.Message), "Successfully assigned %s to %s", &metaKey, &nodeName) 122 | if err != nil { 123 | return nil, fmt.Errorf("failed to extact imformations from event message[%s]: %v", event.Message, err) 124 | } 125 | 126 | namespace, name, err := cache.SplitMetaNamespaceKey(metaKey) 127 | if err != nil { 128 | return nil, err 129 | } 130 | 131 | var lasteOccuredTime int64 132 | 133 | if event.Count == 0 { 134 | lasteOccuredTime = event.EventTime.Unix() 135 | } else { 136 | lasteOccuredTime = event.LastTimestamp.Unix() 137 | } 138 | 139 | return &Binding{ 140 | Node: nodeName, 141 | Namespace: namespace, 142 | PodName: name, 143 | Timestamp: lasteOccuredTime, 144 | }, nil 145 | } 146 | -------------------------------------------------------------------------------- /pkg/controller/annotator/node.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/labels" 12 | "k8s.io/apimachinery/pkg/types" 13 | clientset "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/util/workqueue" 15 | "k8s.io/klog/v2" 16 | 17 | policy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 18 | 19 | prom "github.com/gocrane/crane-scheduler/pkg/controller/prometheus" 20 | utils "github.com/gocrane/crane-scheduler/pkg/utils" 21 | ) 22 | 23 | const ( 24 | HotValueKey = "node_hot_value" 25 | DefaultBackOff = 10 * time.Second 26 | MaxBackOff = 360 * time.Second 27 | ) 28 | 29 | type nodeController struct { 30 | *Controller 31 | queue workqueue.RateLimitingInterface 32 | } 33 | 34 | func newNodeController(c *Controller) *nodeController { 35 | nodeRateLimiter := workqueue.NewItemExponentialFailureRateLimiter(DefaultBackOff, 36 | MaxBackOff) 37 | 38 | return &nodeController{ 39 | Controller: c, 40 | queue: workqueue.NewNamedRateLimitingQueue(nodeRateLimiter, "node_event_queue"), 41 | } 42 | } 43 | 44 | func (n *nodeController) Run() { 45 | defer n.queue.ShutDown() 46 | klog.Infof("Start to reconcile node events") 47 | 48 | for n.processNextWorkItem() { 49 | } 50 | } 51 | 52 | func (n *nodeController) processNextWorkItem() bool { 53 | key, quit := n.queue.Get() 54 | if quit { 55 | return false 56 | } 57 | defer n.queue.Done(key) 58 | 59 | forget, err := n.syncNode(key.(string)) 60 | if err != nil { 61 | klog.Warningf("failed to sync this node [%q]: %v", key.(string), err) 62 | } 63 | if forget { 64 | n.queue.Forget(key) 65 | return true 66 | } 67 | 68 | n.queue.AddRateLimited(key) 69 | return true 70 | } 71 | 72 | func (n *nodeController) syncNode(key string) (bool, error) { 73 | startTime := time.Now() 74 | defer func() { 75 | klog.Infof("Finished syncing node event %q (%v)", key, time.Since(startTime)) 76 | }() 77 | 78 | nodeName, metricName, err := splitMetaKeyWithMetricName(key) 79 | if err != nil { 80 | return true, fmt.Errorf("invalid resource key: %s", key) 81 | } 82 | 83 | node, err := n.nodeLister.Get(nodeName) 84 | if err != nil { 85 | return true, fmt.Errorf("can not find node[%s]: %v", node, err) 86 | } 87 | 88 | err = annotateNodeLoad(n.promClient, n.kubeClient, node, metricName) 89 | if err != nil { 90 | return false, fmt.Errorf("can not annotate node[%s]: %v", node.Name, err) 91 | } 92 | 93 | err = annotateNodeHotValue(n.kubeClient, n.bindingRecords, node, n.policy) 94 | if err != nil { 95 | return false, err 96 | } 97 | 98 | return true, nil 99 | } 100 | 101 | func annotateNodeLoad(promClient prom.PromClient, kubeClient clientset.Interface, node *v1.Node, key string) error { 102 | value, err := promClient.QueryByNodeIP(key, getNodeInternalIP(node)) 103 | if err == nil && len(value) > 0 { 104 | return patchNodeAnnotation(kubeClient, node, key, value) 105 | } 106 | value, err = promClient.QueryByNodeName(key, getNodeName(node)) 107 | if err == nil && len(value) > 0 { 108 | return patchNodeAnnotation(kubeClient, node, key, value) 109 | } 110 | return fmt.Errorf("failed to get data %s{%s=%s}: %v", key, node.Name, value, err) 111 | } 112 | 113 | func annotateNodeHotValue(kubeClient clientset.Interface, br *BindingRecords, node *v1.Node, policy policy.DynamicSchedulerPolicy) error { 114 | var value int 115 | 116 | for _, p := range policy.Spec.HotValue { 117 | value += br.GetLastNodeBindingCount(node.Name, p.TimeRange.Duration) / p.Count 118 | } 119 | 120 | return patchNodeAnnotation(kubeClient, node, HotValueKey, strconv.Itoa(value)) 121 | } 122 | 123 | func patchNodeAnnotation(kubeClient clientset.Interface, node *v1.Node, key, value string) error { 124 | annotation := node.GetAnnotations() 125 | if annotation == nil { 126 | annotation = map[string]string{} 127 | } 128 | 129 | operator := "add" 130 | _, exist := annotation[key] 131 | if exist { 132 | operator = "replace" 133 | } 134 | 135 | patchAnnotationTemplate := 136 | `[{ 137 | "op": "%s", 138 | "path": "/metadata/annotations/%s", 139 | "value": "%s" 140 | }]` 141 | 142 | patchData := fmt.Sprintf(patchAnnotationTemplate, operator, key, value+","+utils.GetLocalTime()) 143 | 144 | _, err := kubeClient.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.JSONPatchType, []byte(patchData), metav1.PatchOptions{}) 145 | return err 146 | } 147 | 148 | func (n *nodeController) CreateMetricSyncTicker(stopCh <-chan struct{}) { 149 | 150 | for _, p := range n.policy.Spec.SyncPeriod { 151 | enqueueFunc := func(policy policy.SyncPolicy) { 152 | nodes, err := n.nodeLister.List(labels.Everything()) 153 | if err != nil { 154 | panic(fmt.Errorf("failed to list nodes: %v", err)) 155 | } 156 | 157 | for _, node := range nodes { 158 | n.queue.Add(handlingMetaKeyWithMetricName(node.Name, policy.Name)) 159 | } 160 | } 161 | 162 | enqueueFunc(p) 163 | 164 | go func(policy policy.SyncPolicy) { 165 | ticker := time.NewTicker(policy.Period.Duration) 166 | defer ticker.Stop() 167 | for { 168 | select { 169 | case <-ticker.C: 170 | enqueueFunc(policy) 171 | case <-stopCh: 172 | return 173 | } 174 | } 175 | }(p) 176 | } 177 | } 178 | 179 | func getNodeInternalIP(node *v1.Node) string { 180 | for _, addr := range node.Status.Addresses { 181 | if addr.Type == v1.NodeInternalIP { 182 | return addr.Address 183 | } 184 | } 185 | 186 | return node.Name 187 | } 188 | 189 | func getNodeName(node *v1.Node) string { 190 | return node.Name 191 | } 192 | -------------------------------------------------------------------------------- /pkg/controller/annotator/utils.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | policy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 9 | ) 10 | 11 | func splitMetaKeyWithMetricName(key string) (string, string, error) { 12 | parts := strings.Split(key, "/") 13 | 14 | if len(parts) != 2 { 15 | return "", "", fmt.Errorf("unexpected key format: %q", key) 16 | } 17 | 18 | return parts[0], parts[1], nil 19 | } 20 | 21 | func handlingMetaKeyWithMetricName(nodeName, metricName string) string { 22 | return nodeName + "/" + metricName 23 | } 24 | 25 | func getMaxHotVauleTimeRange(hotValues []policy.HotValuePolicy) time.Duration { 26 | var max time.Duration 27 | 28 | if hotValues == nil { 29 | return max 30 | } 31 | 32 | for _, tr := range hotValues { 33 | if max < tr.TimeRange.Duration { 34 | max = tr.TimeRange.Duration 35 | } 36 | } 37 | 38 | return max 39 | } 40 | -------------------------------------------------------------------------------- /pkg/controller/prometheus/prometheus.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/api" 11 | v1 "github.com/prometheus/client_golang/api/prometheus/v1" 12 | "github.com/prometheus/common/model" 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | const ( 17 | DefaultPrometheusQueryTimeout = 10 * time.Second 18 | ) 19 | 20 | // PromClient provides client to interact with Prometheus. 21 | type PromClient interface { 22 | // QueryByNodeIP queries data by node IP. 23 | QueryByNodeIP(string, string) (string, error) 24 | // QueryByNodeName queries data by node IP. 25 | QueryByNodeName(string, string) (string, error) 26 | // QueryByNodeIPWithOffset queries data by node IP with offset. 27 | QueryByNodeIPWithOffset(string, string, string) (string, error) 28 | } 29 | 30 | type promClient struct { 31 | API v1.API 32 | } 33 | 34 | // NewPromClient returns PromClient interface. 35 | func NewPromClient(addr string) (PromClient, error) { 36 | config := api.Config{ 37 | Address: addr, 38 | } 39 | 40 | client, err := api.NewClient(config) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return &promClient{ 46 | API: v1.NewAPI(client), 47 | }, nil 48 | } 49 | 50 | func (p *promClient) QueryByNodeIP(metricName, ip string) (string, error) { 51 | klog.V(4).Infof("Try to query %s by node IP[%s]", metricName, ip) 52 | 53 | querySelector := fmt.Sprintf("%s{instance=~\"%s\"} /100", metricName, ip) 54 | 55 | result, err := p.query(querySelector) 56 | if result != "" && err == nil { 57 | return result, nil 58 | } 59 | 60 | querySelector = fmt.Sprintf("%s{instance=~\"%s:.+\"} /100", metricName, ip) 61 | result, err = p.query(querySelector) 62 | if result != "" && err == nil { 63 | return result, nil 64 | } 65 | 66 | return "", err 67 | } 68 | 69 | func (p *promClient) QueryByNodeName(metricName, name string) (string, error) { 70 | klog.V(4).Infof("Try to query %s by node name[%s]", metricName, name) 71 | 72 | querySelector := fmt.Sprintf("%s{instance=~\"%s\"} /100", metricName, name) 73 | 74 | result, err := p.query(querySelector) 75 | if result != "" && err == nil { 76 | return result, nil 77 | } 78 | 79 | return "", err 80 | } 81 | 82 | func (p *promClient) QueryByNodeIPWithOffset(metricName, ip, offset string) (string, error) { 83 | klog.V(4).Info("Try to query %s with offset %s by node IP[%s]", metricName, offset, ip) 84 | 85 | querySelector := fmt.Sprintf("%s{instance=~\"%s\"} offset %s /100", metricName, ip, offset) 86 | result, err := p.query(querySelector) 87 | if result != "" && err == nil { 88 | return result, nil 89 | } 90 | 91 | querySelector = fmt.Sprintf("%s{instance=~\"%s:.+\"} offset %s /100", metricName, ip, offset) 92 | result, err = p.query(querySelector) 93 | if result != "" && err == nil { 94 | return result, nil 95 | } 96 | 97 | return "", err 98 | } 99 | 100 | func (p *promClient) query(query string) (string, error) { 101 | klog.V(4).Infof("Begin to query prometheus by promQL [%s]...", query) 102 | 103 | ctx, cancel := context.WithTimeout(context.Background(), DefaultPrometheusQueryTimeout) 104 | defer cancel() 105 | 106 | result, warnings, err := p.API.Query(ctx, query, time.Now()) 107 | if err != nil { 108 | return "", err 109 | } 110 | 111 | if len(warnings) > 0 { 112 | return "", fmt.Errorf("unexpected warnings: %v", warnings) 113 | } 114 | 115 | if result.Type() != model.ValVector { 116 | return "", fmt.Errorf("illege result type: %v", result.Type()) 117 | } 118 | 119 | var metricValue string 120 | for _, elem := range result.(model.Vector) { 121 | if float64(elem.Value) < float64(0) || math.IsNaN(float64(elem.Value)) { 122 | elem.Value = 0 123 | } 124 | metricValue = strconv.FormatFloat(float64(elem.Value), 'f', 5, 64) 125 | } 126 | 127 | return metricValue, nil 128 | } 129 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | // +groupName=kubescheduler.config.k8s.io 3 | 4 | package config // import "crane.io/crane-scheduler/pkg/plugins/apis/config" 5 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/register.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" 7 | ) 8 | 9 | // SchemeGroupVersion is group version used to register these objects 10 | var SchemeGroupVersion = schema.GroupVersion{Group: kubeschedulerconfig.GroupName, Version: runtime.APIVersionInternal} 11 | 12 | var ( 13 | localSchemeBuilder = &kubeschedulerconfig.SchemeBuilder 14 | // AddToScheme is a global function that registers this API group & version to a scheme 15 | AddToScheme = localSchemeBuilder.AddToScheme 16 | ) 17 | 18 | // addKnownTypes registers known types to the given scheme 19 | func addKnownTypes(scheme *runtime.Scheme) error { 20 | scheme.AddKnownTypes(SchemeGroupVersion, 21 | &DynamicArgs{}, 22 | &NodeResourceTopologyMatchArgs{}, 23 | ) 24 | return nil 25 | } 26 | 27 | func init() { 28 | // We only register manually written functions here. The registration of the 29 | // generated functions takes place in the generated files. The separation 30 | // makes the code compile even when the generated files are missing. 31 | localSchemeBuilder.Register(addKnownTypes) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | package scheme 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/serializer" 6 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 7 | kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" 8 | 9 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config" 10 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config/v1beta2" 11 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config/v1beta3" 12 | ) 13 | 14 | var ( 15 | // Re-use the in-tree Scheme. 16 | Scheme = kubeschedulerscheme.Scheme 17 | 18 | // Codecs provides access to encoding and decoding for the scheme. 19 | Codecs = serializer.NewCodecFactory(Scheme, serializer.EnableStrict) 20 | ) 21 | 22 | func init() { 23 | AddToScheme(Scheme) 24 | } 25 | 26 | // AddToScheme builds the kubescheduler scheme using all known versions of the kubescheduler api. 27 | func AddToScheme(scheme *runtime.Scheme) { 28 | utilruntime.Must(config.AddToScheme(scheme)) 29 | utilruntime.Must(v1beta2.AddToScheme(scheme)) 30 | utilruntime.Must(v1beta3.AddToScheme(scheme)) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 8 | 9 | // DynamicArgs is the args struction of Dynamic scheduler plugin. 10 | type DynamicArgs struct { 11 | metav1.TypeMeta 12 | // PolicyConfigPath specified the path of policy config. 13 | PolicyConfigPath string 14 | } 15 | 16 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 17 | 18 | // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin. 19 | type NodeResourceTopologyMatchArgs struct { 20 | metav1.TypeMeta 21 | // TopologyAwareResources represents the resource names of topology. 22 | TopologyAwareResources []string 23 | } 24 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/defaults.go: -------------------------------------------------------------------------------- 1 | package v1beta2 2 | 3 | var ( 4 | defaultNodeResource = []string{"cpu"} 5 | ) 6 | 7 | func SetDefaults_DynamicArgs(obj *DynamicArgs) { 8 | if obj.PolicyConfigPath == "" { 9 | obj.PolicyConfigPath = "/etc/kubernetes/dynamic-scheduler-policy.yaml" 10 | } 11 | return 12 | } 13 | 14 | func SetDefaults_NodeResourceTopologyMatchArgs(obj *NodeResourceTopologyMatchArgs) { 15 | if len(obj.TopologyAwareResources) == 0 { 16 | obj.TopologyAwareResources = defaultNodeResource 17 | } 18 | return 19 | } 20 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package,register 2 | // +k8s:conversion-gen=github.com/gocrane/crane-scheduler/pkg/plugins/apis/config 3 | // +k8s:defaulter-gen=TypeMeta 4 | // +k8s:defaulter-gen-input=github.com/gocrane/crane-scheduler/pkg/plugins/apis/config/v1beta2 5 | 6 | // +groupName=kubescheduler.config.k8s.io 7 | 8 | // Package v1beta2 is the v1beta2 version of the API. 9 | package v1beta2 // import "crane.io/scheduler-plugins/apis/config/v1beta2" 10 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/register.go: -------------------------------------------------------------------------------- 1 | package v1beta2 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | kubeschedulerschemev1beta2 "k8s.io/kube-scheduler/config/v1beta2" 7 | kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" 8 | ) 9 | 10 | // SchemeGroupVersion is group version used to register these objects 11 | var SchemeGroupVersion = schema.GroupVersion{Group: kubeschedulerconfig.GroupName, Version: "v1beta2"} 12 | 13 | var ( 14 | // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. 15 | localSchemeBuilder = &kubeschedulerschemev1beta2.SchemeBuilder 16 | // AddToScheme is a global function that registers this API group & version to a scheme 17 | AddToScheme = localSchemeBuilder.AddToScheme 18 | ) 19 | 20 | // addKnownTypes registers known types to the given scheme 21 | func addKnownTypes(scheme *runtime.Scheme) error { 22 | scheme.AddKnownTypes(SchemeGroupVersion, 23 | &DynamicArgs{}, 24 | &NodeResourceTopologyMatchArgs{}, 25 | ) 26 | return nil 27 | } 28 | 29 | func init() { 30 | // We only register manually written functions here. The registration of the 31 | // generated functions takes place in the generated files. The separation 32 | // makes the code compile even when the generated files are missing. 33 | localSchemeBuilder.Register(addKnownTypes) 34 | localSchemeBuilder.Register(RegisterDefaults) 35 | localSchemeBuilder.Register(RegisterConversions) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/types.go: -------------------------------------------------------------------------------- 1 | package v1beta2 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 8 | 9 | // DynamicArgs is the args struction of Dynamic scheduler plugin. 10 | type DynamicArgs struct { 11 | metav1.TypeMeta `json:",inline"` 12 | // PolicyConfigPath specified the path of policy config. 13 | PolicyConfigPath string `json:"policyConfigPath"` 14 | } 15 | 16 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 17 | 18 | // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin. 19 | type NodeResourceTopologyMatchArgs struct { 20 | metav1.TypeMeta `json:",inline"` 21 | // TopologyAwareResources represents the resource names of topology. 22 | TopologyAwareResources []string `json:"topologyAwareResources,omitempty"` 23 | } 24 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/zz_generated.conversion.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by conversion-gen. DO NOT EDIT. 5 | 6 | package v1beta2 7 | 8 | import ( 9 | unsafe "unsafe" 10 | 11 | config "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config" 12 | conversion "k8s.io/apimachinery/pkg/conversion" 13 | runtime "k8s.io/apimachinery/pkg/runtime" 14 | ) 15 | 16 | func init() { 17 | localSchemeBuilder.Register(RegisterConversions) 18 | } 19 | 20 | // RegisterConversions adds conversion functions to the given scheme. 21 | // Public to allow building arbitrary schemes. 22 | func RegisterConversions(s *runtime.Scheme) error { 23 | if err := s.AddGeneratedConversionFunc((*DynamicArgs)(nil), (*config.DynamicArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 24 | return Convert_v1beta2_DynamicArgs_To_config_DynamicArgs(a.(*DynamicArgs), b.(*config.DynamicArgs), scope) 25 | }); err != nil { 26 | return err 27 | } 28 | if err := s.AddGeneratedConversionFunc((*config.DynamicArgs)(nil), (*DynamicArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 29 | return Convert_config_DynamicArgs_To_v1beta2_DynamicArgs(a.(*config.DynamicArgs), b.(*DynamicArgs), scope) 30 | }); err != nil { 31 | return err 32 | } 33 | if err := s.AddGeneratedConversionFunc((*NodeResourceTopologyMatchArgs)(nil), (*config.NodeResourceTopologyMatchArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 34 | return Convert_v1beta2_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(a.(*NodeResourceTopologyMatchArgs), b.(*config.NodeResourceTopologyMatchArgs), scope) 35 | }); err != nil { 36 | return err 37 | } 38 | if err := s.AddGeneratedConversionFunc((*config.NodeResourceTopologyMatchArgs)(nil), (*NodeResourceTopologyMatchArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 39 | return Convert_config_NodeResourceTopologyMatchArgs_To_v1beta2_NodeResourceTopologyMatchArgs(a.(*config.NodeResourceTopologyMatchArgs), b.(*NodeResourceTopologyMatchArgs), scope) 40 | }); err != nil { 41 | return err 42 | } 43 | return nil 44 | } 45 | 46 | func autoConvert_v1beta2_DynamicArgs_To_config_DynamicArgs(in *DynamicArgs, out *config.DynamicArgs, s conversion.Scope) error { 47 | out.PolicyConfigPath = in.PolicyConfigPath 48 | return nil 49 | } 50 | 51 | // Convert_v1beta2_DynamicArgs_To_config_DynamicArgs is an autogenerated conversion function. 52 | func Convert_v1beta2_DynamicArgs_To_config_DynamicArgs(in *DynamicArgs, out *config.DynamicArgs, s conversion.Scope) error { 53 | return autoConvert_v1beta2_DynamicArgs_To_config_DynamicArgs(in, out, s) 54 | } 55 | 56 | func autoConvert_config_DynamicArgs_To_v1beta2_DynamicArgs(in *config.DynamicArgs, out *DynamicArgs, s conversion.Scope) error { 57 | out.PolicyConfigPath = in.PolicyConfigPath 58 | return nil 59 | } 60 | 61 | // Convert_config_DynamicArgs_To_v1beta2_DynamicArgs is an autogenerated conversion function. 62 | func Convert_config_DynamicArgs_To_v1beta2_DynamicArgs(in *config.DynamicArgs, out *DynamicArgs, s conversion.Scope) error { 63 | return autoConvert_config_DynamicArgs_To_v1beta2_DynamicArgs(in, out, s) 64 | } 65 | 66 | func autoConvert_v1beta2_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs, out *config.NodeResourceTopologyMatchArgs, s conversion.Scope) error { 67 | out.TopologyAwareResources = *(*[]string)(unsafe.Pointer(&in.TopologyAwareResources)) 68 | return nil 69 | } 70 | 71 | // Convert_v1beta2_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs is an autogenerated conversion function. 72 | func Convert_v1beta2_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs, out *config.NodeResourceTopologyMatchArgs, s conversion.Scope) error { 73 | return autoConvert_v1beta2_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in, out, s) 74 | } 75 | 76 | func autoConvert_config_NodeResourceTopologyMatchArgs_To_v1beta2_NodeResourceTopologyMatchArgs(in *config.NodeResourceTopologyMatchArgs, out *NodeResourceTopologyMatchArgs, s conversion.Scope) error { 77 | out.TopologyAwareResources = *(*[]string)(unsafe.Pointer(&in.TopologyAwareResources)) 78 | return nil 79 | } 80 | 81 | // Convert_config_NodeResourceTopologyMatchArgs_To_v1beta2_NodeResourceTopologyMatchArgs is an autogenerated conversion function. 82 | func Convert_config_NodeResourceTopologyMatchArgs_To_v1beta2_NodeResourceTopologyMatchArgs(in *config.NodeResourceTopologyMatchArgs, out *NodeResourceTopologyMatchArgs, s conversion.Scope) error { 83 | return autoConvert_config_NodeResourceTopologyMatchArgs_To_v1beta2_NodeResourceTopologyMatchArgs(in, out, s) 84 | } 85 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by deepcopy-gen. DO NOT EDIT. 5 | 6 | package v1beta2 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *DynamicArgs) DeepCopyInto(out *DynamicArgs) { 14 | *out = *in 15 | out.TypeMeta = in.TypeMeta 16 | return 17 | } 18 | 19 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicArgs. 20 | func (in *DynamicArgs) DeepCopy() *DynamicArgs { 21 | if in == nil { 22 | return nil 23 | } 24 | out := new(DynamicArgs) 25 | in.DeepCopyInto(out) 26 | return out 27 | } 28 | 29 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 30 | func (in *DynamicArgs) DeepCopyObject() runtime.Object { 31 | if c := in.DeepCopy(); c != nil { 32 | return c 33 | } 34 | return nil 35 | } 36 | 37 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 38 | func (in *NodeResourceTopologyMatchArgs) DeepCopyInto(out *NodeResourceTopologyMatchArgs) { 39 | *out = *in 40 | out.TypeMeta = in.TypeMeta 41 | if in.TopologyAwareResources != nil { 42 | in, out := &in.TopologyAwareResources, &out.TopologyAwareResources 43 | *out = make([]string, len(*in)) 44 | copy(*out, *in) 45 | } 46 | return 47 | } 48 | 49 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourceTopologyMatchArgs. 50 | func (in *NodeResourceTopologyMatchArgs) DeepCopy() *NodeResourceTopologyMatchArgs { 51 | if in == nil { 52 | return nil 53 | } 54 | out := new(NodeResourceTopologyMatchArgs) 55 | in.DeepCopyInto(out) 56 | return out 57 | } 58 | 59 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 60 | func (in *NodeResourceTopologyMatchArgs) DeepCopyObject() runtime.Object { 61 | if c := in.DeepCopy(); c != nil { 62 | return c 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta2/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by defaulter-gen. DO NOT EDIT. 5 | 6 | package v1beta2 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // RegisterDefaults adds defaulters functions to the given scheme. 13 | // Public to allow building arbitrary schemes. 14 | // All generated defaulters are covering - they call all nested defaulters. 15 | func RegisterDefaults(scheme *runtime.Scheme) error { 16 | scheme.AddTypeDefaultingFunc(&DynamicArgs{}, func(obj interface{}) { SetObjectDefaults_DynamicArgs(obj.(*DynamicArgs)) }) 17 | scheme.AddTypeDefaultingFunc(&NodeResourceTopologyMatchArgs{}, func(obj interface{}) { 18 | SetObjectDefaults_NodeResourceTopologyMatchArgs(obj.(*NodeResourceTopologyMatchArgs)) 19 | }) 20 | return nil 21 | } 22 | 23 | func SetObjectDefaults_DynamicArgs(in *DynamicArgs) { 24 | SetDefaults_DynamicArgs(in) 25 | } 26 | 27 | func SetObjectDefaults_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs) { 28 | SetDefaults_NodeResourceTopologyMatchArgs(in) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/defaults.go: -------------------------------------------------------------------------------- 1 | package v1beta3 2 | 3 | var ( 4 | defaultNodeResource = []string{"cpu"} 5 | ) 6 | 7 | func SetDefaults_DynamicArgs(obj *DynamicArgs) { 8 | if obj.PolicyConfigPath == nil { 9 | path := "/etc/kubernetes/dynamic-scheduler-policy.yaml" 10 | obj.PolicyConfigPath = &path 11 | } 12 | return 13 | } 14 | 15 | func SetDefaults_NodeResourceTopologyMatchArgs(obj *NodeResourceTopologyMatchArgs) { 16 | if len(obj.TopologyAwareResources) == 0 { 17 | obj.TopologyAwareResources = defaultNodeResource 18 | } 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package,register 2 | // +k8s:conversion-gen=github.com/gocrane/crane-scheduler/pkg/plugins/apis/config 3 | // +k8s:defaulter-gen=TypeMeta 4 | // +k8s:defaulter-gen-input=github.com/gocrane/crane-scheduler/pkg/plugins/apis/config/v1beta3 5 | 6 | // +groupName=kubescheduler.config.k8s.io 7 | 8 | // Package v1beta3 is the v1beta3 version of the API. 9 | package v1beta3 // import "crane.io/scheduler-plugins/apis/config/v1beta3" 10 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/register.go: -------------------------------------------------------------------------------- 1 | package v1beta3 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | kubeschedulerschemev1beta3 "k8s.io/kube-scheduler/config/v1beta3" 7 | kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" 8 | ) 9 | 10 | // SchemeGroupVersion is group version used to register these objects 11 | var SchemeGroupVersion = schema.GroupVersion{Group: kubeschedulerconfig.GroupName, Version: "v1beta3"} 12 | 13 | var ( 14 | // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. 15 | localSchemeBuilder = &kubeschedulerschemev1beta3.SchemeBuilder 16 | // AddToScheme is a global function that registers this API group & version to a scheme 17 | AddToScheme = localSchemeBuilder.AddToScheme 18 | ) 19 | 20 | // addKnownTypes registers known types to the given scheme 21 | func addKnownTypes(scheme *runtime.Scheme) error { 22 | scheme.AddKnownTypes(SchemeGroupVersion, 23 | &DynamicArgs{}, 24 | &NodeResourceTopologyMatchArgs{}, 25 | ) 26 | return nil 27 | } 28 | 29 | func init() { 30 | // We only register manually written functions here. The registration of the 31 | // generated functions takes place in the generated files. The separation 32 | // makes the code compile even when the generated files are missing. 33 | localSchemeBuilder.Register(addKnownTypes) 34 | localSchemeBuilder.Register(RegisterDefaults) 35 | localSchemeBuilder.Register(RegisterConversions) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/types.go: -------------------------------------------------------------------------------- 1 | package v1beta3 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 8 | 9 | // DynamicArgs is the args struction of Dynamic scheduler plugin. 10 | type DynamicArgs struct { 11 | metav1.TypeMeta `json:",inline"` 12 | // PolicyConfigPath specified the path of policy config. 13 | PolicyConfigPath *string `json:"policyConfigPath,omitempty"` 14 | } 15 | 16 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 17 | 18 | // NodeResourceTopologyMatchArgs holds arguments used to configure the NodeResourceTopologyMatch plugin. 19 | type NodeResourceTopologyMatchArgs struct { 20 | metav1.TypeMeta `json:",inline"` 21 | // TopologyAwareResources represents the resource names of topology. 22 | TopologyAwareResources []string `json:"topologyAwareResources,omitempty"` 23 | } 24 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/zz_generated.conversion.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by conversion-gen. DO NOT EDIT. 5 | 6 | package v1beta3 7 | 8 | import ( 9 | unsafe "unsafe" 10 | 11 | config "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config" 12 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | conversion "k8s.io/apimachinery/pkg/conversion" 14 | runtime "k8s.io/apimachinery/pkg/runtime" 15 | ) 16 | 17 | func init() { 18 | localSchemeBuilder.Register(RegisterConversions) 19 | } 20 | 21 | // RegisterConversions adds conversion functions to the given scheme. 22 | // Public to allow building arbitrary schemes. 23 | func RegisterConversions(s *runtime.Scheme) error { 24 | if err := s.AddGeneratedConversionFunc((*DynamicArgs)(nil), (*config.DynamicArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 25 | return Convert_v1beta3_DynamicArgs_To_config_DynamicArgs(a.(*DynamicArgs), b.(*config.DynamicArgs), scope) 26 | }); err != nil { 27 | return err 28 | } 29 | if err := s.AddGeneratedConversionFunc((*config.DynamicArgs)(nil), (*DynamicArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 30 | return Convert_config_DynamicArgs_To_v1beta3_DynamicArgs(a.(*config.DynamicArgs), b.(*DynamicArgs), scope) 31 | }); err != nil { 32 | return err 33 | } 34 | if err := s.AddGeneratedConversionFunc((*NodeResourceTopologyMatchArgs)(nil), (*config.NodeResourceTopologyMatchArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 35 | return Convert_v1beta3_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(a.(*NodeResourceTopologyMatchArgs), b.(*config.NodeResourceTopologyMatchArgs), scope) 36 | }); err != nil { 37 | return err 38 | } 39 | if err := s.AddGeneratedConversionFunc((*config.NodeResourceTopologyMatchArgs)(nil), (*NodeResourceTopologyMatchArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 40 | return Convert_config_NodeResourceTopologyMatchArgs_To_v1beta3_NodeResourceTopologyMatchArgs(a.(*config.NodeResourceTopologyMatchArgs), b.(*NodeResourceTopologyMatchArgs), scope) 41 | }); err != nil { 42 | return err 43 | } 44 | return nil 45 | } 46 | 47 | func autoConvert_v1beta3_DynamicArgs_To_config_DynamicArgs(in *DynamicArgs, out *config.DynamicArgs, s conversion.Scope) error { 48 | if err := v1.Convert_Pointer_string_To_string(&in.PolicyConfigPath, &out.PolicyConfigPath, s); err != nil { 49 | return err 50 | } 51 | return nil 52 | } 53 | 54 | // Convert_v1beta3_DynamicArgs_To_config_DynamicArgs is an autogenerated conversion function. 55 | func Convert_v1beta3_DynamicArgs_To_config_DynamicArgs(in *DynamicArgs, out *config.DynamicArgs, s conversion.Scope) error { 56 | return autoConvert_v1beta3_DynamicArgs_To_config_DynamicArgs(in, out, s) 57 | } 58 | 59 | func autoConvert_config_DynamicArgs_To_v1beta3_DynamicArgs(in *config.DynamicArgs, out *DynamicArgs, s conversion.Scope) error { 60 | if err := v1.Convert_string_To_Pointer_string(&in.PolicyConfigPath, &out.PolicyConfigPath, s); err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | // Convert_config_DynamicArgs_To_v1beta3_DynamicArgs is an autogenerated conversion function. 67 | func Convert_config_DynamicArgs_To_v1beta3_DynamicArgs(in *config.DynamicArgs, out *DynamicArgs, s conversion.Scope) error { 68 | return autoConvert_config_DynamicArgs_To_v1beta3_DynamicArgs(in, out, s) 69 | } 70 | 71 | func autoConvert_v1beta3_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs, out *config.NodeResourceTopologyMatchArgs, s conversion.Scope) error { 72 | out.TopologyAwareResources = *(*[]string)(unsafe.Pointer(&in.TopologyAwareResources)) 73 | return nil 74 | } 75 | 76 | // Convert_v1beta3_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs is an autogenerated conversion function. 77 | func Convert_v1beta3_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs, out *config.NodeResourceTopologyMatchArgs, s conversion.Scope) error { 78 | return autoConvert_v1beta3_NodeResourceTopologyMatchArgs_To_config_NodeResourceTopologyMatchArgs(in, out, s) 79 | } 80 | 81 | func autoConvert_config_NodeResourceTopologyMatchArgs_To_v1beta3_NodeResourceTopologyMatchArgs(in *config.NodeResourceTopologyMatchArgs, out *NodeResourceTopologyMatchArgs, s conversion.Scope) error { 82 | out.TopologyAwareResources = *(*[]string)(unsafe.Pointer(&in.TopologyAwareResources)) 83 | return nil 84 | } 85 | 86 | // Convert_config_NodeResourceTopologyMatchArgs_To_v1beta3_NodeResourceTopologyMatchArgs is an autogenerated conversion function. 87 | func Convert_config_NodeResourceTopologyMatchArgs_To_v1beta3_NodeResourceTopologyMatchArgs(in *config.NodeResourceTopologyMatchArgs, out *NodeResourceTopologyMatchArgs, s conversion.Scope) error { 88 | return autoConvert_config_NodeResourceTopologyMatchArgs_To_v1beta3_NodeResourceTopologyMatchArgs(in, out, s) 89 | } 90 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by deepcopy-gen. DO NOT EDIT. 5 | 6 | package v1beta3 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *DynamicArgs) DeepCopyInto(out *DynamicArgs) { 14 | *out = *in 15 | out.TypeMeta = in.TypeMeta 16 | if in.PolicyConfigPath != nil { 17 | in, out := &in.PolicyConfigPath, &out.PolicyConfigPath 18 | *out = new(string) 19 | **out = **in 20 | } 21 | return 22 | } 23 | 24 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicArgs. 25 | func (in *DynamicArgs) DeepCopy() *DynamicArgs { 26 | if in == nil { 27 | return nil 28 | } 29 | out := new(DynamicArgs) 30 | in.DeepCopyInto(out) 31 | return out 32 | } 33 | 34 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 35 | func (in *DynamicArgs) DeepCopyObject() runtime.Object { 36 | if c := in.DeepCopy(); c != nil { 37 | return c 38 | } 39 | return nil 40 | } 41 | 42 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 43 | func (in *NodeResourceTopologyMatchArgs) DeepCopyInto(out *NodeResourceTopologyMatchArgs) { 44 | *out = *in 45 | out.TypeMeta = in.TypeMeta 46 | if in.TopologyAwareResources != nil { 47 | in, out := &in.TopologyAwareResources, &out.TopologyAwareResources 48 | *out = make([]string, len(*in)) 49 | copy(*out, *in) 50 | } 51 | return 52 | } 53 | 54 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourceTopologyMatchArgs. 55 | func (in *NodeResourceTopologyMatchArgs) DeepCopy() *NodeResourceTopologyMatchArgs { 56 | if in == nil { 57 | return nil 58 | } 59 | out := new(NodeResourceTopologyMatchArgs) 60 | in.DeepCopyInto(out) 61 | return out 62 | } 63 | 64 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 65 | func (in *NodeResourceTopologyMatchArgs) DeepCopyObject() runtime.Object { 66 | if c := in.DeepCopy(); c != nil { 67 | return c 68 | } 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/v1beta3/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by defaulter-gen. DO NOT EDIT. 5 | 6 | package v1beta3 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // RegisterDefaults adds defaulters functions to the given scheme. 13 | // Public to allow building arbitrary schemes. 14 | // All generated defaulters are covering - they call all nested defaulters. 15 | func RegisterDefaults(scheme *runtime.Scheme) error { 16 | scheme.AddTypeDefaultingFunc(&DynamicArgs{}, func(obj interface{}) { SetObjectDefaults_DynamicArgs(obj.(*DynamicArgs)) }) 17 | scheme.AddTypeDefaultingFunc(&NodeResourceTopologyMatchArgs{}, func(obj interface{}) { 18 | SetObjectDefaults_NodeResourceTopologyMatchArgs(obj.(*NodeResourceTopologyMatchArgs)) 19 | }) 20 | return nil 21 | } 22 | 23 | func SetObjectDefaults_DynamicArgs(in *DynamicArgs) { 24 | SetDefaults_DynamicArgs(in) 25 | } 26 | 27 | func SetObjectDefaults_NodeResourceTopologyMatchArgs(in *NodeResourceTopologyMatchArgs) { 28 | SetDefaults_NodeResourceTopologyMatchArgs(in) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/plugins/apis/config/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by deepcopy-gen. DO NOT EDIT. 5 | 6 | package config 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *DynamicArgs) DeepCopyInto(out *DynamicArgs) { 14 | *out = *in 15 | out.TypeMeta = in.TypeMeta 16 | return 17 | } 18 | 19 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicArgs. 20 | func (in *DynamicArgs) DeepCopy() *DynamicArgs { 21 | if in == nil { 22 | return nil 23 | } 24 | out := new(DynamicArgs) 25 | in.DeepCopyInto(out) 26 | return out 27 | } 28 | 29 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 30 | func (in *DynamicArgs) DeepCopyObject() runtime.Object { 31 | if c := in.DeepCopy(); c != nil { 32 | return c 33 | } 34 | return nil 35 | } 36 | 37 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 38 | func (in *NodeResourceTopologyMatchArgs) DeepCopyInto(out *NodeResourceTopologyMatchArgs) { 39 | *out = *in 40 | out.TypeMeta = in.TypeMeta 41 | if in.TopologyAwareResources != nil { 42 | in, out := &in.TopologyAwareResources, &out.TopologyAwareResources 43 | *out = make([]string, len(*in)) 44 | copy(*out, *in) 45 | } 46 | return 47 | } 48 | 49 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeResourceTopologyMatchArgs. 50 | func (in *NodeResourceTopologyMatchArgs) DeepCopy() *NodeResourceTopologyMatchArgs { 51 | if in == nil { 52 | return nil 53 | } 54 | out := new(NodeResourceTopologyMatchArgs) 55 | in.DeepCopyInto(out) 56 | return out 57 | } 58 | 59 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 60 | func (in *NodeResourceTopologyMatchArgs) DeepCopyObject() runtime.Object { 61 | if c := in.DeepCopy(); c != nil { 62 | return c 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/deepcopy_generated.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by deepcopy-gen. DO NOT EDIT. 5 | 6 | package policy 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *DynamicSchedulerPolicy) DeepCopyInto(out *DynamicSchedulerPolicy) { 14 | *out = *in 15 | out.TypeMeta = in.TypeMeta 16 | in.Spec.DeepCopyInto(&out.Spec) 17 | return 18 | } 19 | 20 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicSchedulerPolicy. 21 | func (in *DynamicSchedulerPolicy) DeepCopy() *DynamicSchedulerPolicy { 22 | if in == nil { 23 | return nil 24 | } 25 | out := new(DynamicSchedulerPolicy) 26 | in.DeepCopyInto(out) 27 | return out 28 | } 29 | 30 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 31 | func (in *DynamicSchedulerPolicy) DeepCopyObject() runtime.Object { 32 | if c := in.DeepCopy(); c != nil { 33 | return c 34 | } 35 | return nil 36 | } 37 | 38 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 39 | func (in *HotValuePolicy) DeepCopyInto(out *HotValuePolicy) { 40 | *out = *in 41 | in.TimeRange.DeepCopyInto(&out.TimeRange) 42 | return 43 | } 44 | 45 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HotValuePolicy. 46 | func (in *HotValuePolicy) DeepCopy() *HotValuePolicy { 47 | if in == nil { 48 | return nil 49 | } 50 | out := new(HotValuePolicy) 51 | in.DeepCopyInto(out) 52 | return out 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *PolicySpec) DeepCopyInto(out *PolicySpec) { 57 | *out = *in 58 | if in.SyncPeriod != nil { 59 | in, out := &in.SyncPeriod, &out.SyncPeriod 60 | *out = make([]SyncPolicy, len(*in)) 61 | for i := range *in { 62 | (*in)[i].DeepCopyInto(&(*out)[i]) 63 | } 64 | } 65 | if in.Predicate != nil { 66 | in, out := &in.Predicate, &out.Predicate 67 | *out = make([]PredicatePolicy, len(*in)) 68 | copy(*out, *in) 69 | } 70 | if in.Priority != nil { 71 | in, out := &in.Priority, &out.Priority 72 | *out = make([]PriorityPolicy, len(*in)) 73 | copy(*out, *in) 74 | } 75 | if in.HotValue != nil { 76 | in, out := &in.HotValue, &out.HotValue 77 | *out = make([]HotValuePolicy, len(*in)) 78 | for i := range *in { 79 | (*in)[i].DeepCopyInto(&(*out)[i]) 80 | } 81 | } 82 | return 83 | } 84 | 85 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicySpec. 86 | func (in *PolicySpec) DeepCopy() *PolicySpec { 87 | if in == nil { 88 | return nil 89 | } 90 | out := new(PolicySpec) 91 | in.DeepCopyInto(out) 92 | return out 93 | } 94 | 95 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 96 | func (in *PredicatePolicy) DeepCopyInto(out *PredicatePolicy) { 97 | *out = *in 98 | return 99 | } 100 | 101 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PredicatePolicy. 102 | func (in *PredicatePolicy) DeepCopy() *PredicatePolicy { 103 | if in == nil { 104 | return nil 105 | } 106 | out := new(PredicatePolicy) 107 | in.DeepCopyInto(out) 108 | return out 109 | } 110 | 111 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 112 | func (in *PriorityPolicy) DeepCopyInto(out *PriorityPolicy) { 113 | *out = *in 114 | return 115 | } 116 | 117 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PriorityPolicy. 118 | func (in *PriorityPolicy) DeepCopy() *PriorityPolicy { 119 | if in == nil { 120 | return nil 121 | } 122 | out := new(PriorityPolicy) 123 | in.DeepCopyInto(out) 124 | return out 125 | } 126 | 127 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 128 | func (in *SyncPolicy) DeepCopyInto(out *SyncPolicy) { 129 | *out = *in 130 | in.Period.DeepCopyInto(&out.Period) 131 | return 132 | } 133 | 134 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncPolicy. 135 | func (in *SyncPolicy) DeepCopy() *SyncPolicy { 136 | if in == nil { 137 | return nil 138 | } 139 | out := new(SyncPolicy) 140 | in.DeepCopyInto(out) 141 | return out 142 | } 143 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | 3 | package policy // import "crane.io/crane-scheduler/pkg/plugins/apis/policy" 4 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/register.go: -------------------------------------------------------------------------------- 1 | package policy 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | // GroupName is the group name used in this package 9 | const GroupName = "scheduler.policy.crane.io" 10 | 11 | // SchemeGroupVersion is group version used to register these objects 12 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} 13 | 14 | var ( 15 | // SchemeBuilder is the scheme builder with scheme init functions to run for this API package 16 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 17 | // AddToScheme is a global function that registers this API group & version to a scheme 18 | AddToScheme = SchemeBuilder.AddToScheme 19 | ) 20 | 21 | // addKnownTypes registers known types to the given scheme 22 | func addKnownTypes(scheme *runtime.Scheme) error { 23 | scheme.AddKnownTypes(SchemeGroupVersion, 24 | &DynamicSchedulerPolicy{}, 25 | ) 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | package scheme 2 | 3 | import ( 4 | dynamicpolicy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 5 | dynamicpolicyv1alpha1 "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy/v1alpha1" 6 | 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/apimachinery/pkg/runtime/serializer" 9 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 10 | ) 11 | 12 | var ( 13 | // Scheme is the runtime.Scheme to which all kubescheduler api types are registered. 14 | Scheme = runtime.NewScheme() 15 | 16 | // Codecs provides access to encoding and decoding for the scheme. 17 | Codecs = serializer.NewCodecFactory(Scheme, serializer.EnableStrict) 18 | ) 19 | 20 | func init() { 21 | AddToScheme(Scheme) 22 | } 23 | 24 | // AddToScheme builds the kubescheduler scheme using all known versions of the kubescheduler api. 25 | func AddToScheme(scheme *runtime.Scheme) { 26 | utilruntime.Must(dynamicpolicy.AddToScheme(scheme)) 27 | utilruntime.Must(dynamicpolicyv1alpha1.AddToScheme(scheme)) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/types.go: -------------------------------------------------------------------------------- 1 | package policy 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 8 | 9 | type DynamicSchedulerPolicy struct { 10 | metav1.TypeMeta 11 | Spec PolicySpec 12 | } 13 | 14 | type PolicySpec struct { 15 | SyncPeriod []SyncPolicy 16 | Predicate []PredicatePolicy 17 | Priority []PriorityPolicy 18 | HotValue []HotValuePolicy 19 | } 20 | 21 | type SyncPolicy struct { 22 | Name string 23 | Period metav1.Duration 24 | } 25 | 26 | type PredicatePolicy struct { 27 | Name string 28 | MaxLimitPecent float64 29 | } 30 | 31 | type PriorityPolicy struct { 32 | Name string 33 | Weight float64 34 | } 35 | 36 | type HotValuePolicy struct { 37 | TimeRange metav1.Duration 38 | Count int 39 | } 40 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/v1alpha1/conversion_generated.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by conversion-gen. DO NOT EDIT. 5 | 6 | package v1alpha1 7 | 8 | import ( 9 | unsafe "unsafe" 10 | 11 | policy "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 12 | conversion "k8s.io/apimachinery/pkg/conversion" 13 | runtime "k8s.io/apimachinery/pkg/runtime" 14 | ) 15 | 16 | func init() { 17 | localSchemeBuilder.Register(RegisterConversions) 18 | } 19 | 20 | // RegisterConversions adds conversion functions to the given scheme. 21 | // Public to allow building arbitrary schemes. 22 | func RegisterConversions(s *runtime.Scheme) error { 23 | if err := s.AddGeneratedConversionFunc((*DynamicSchedulerPolicy)(nil), (*policy.DynamicSchedulerPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 24 | return Convert_v1alpha1_DynamicSchedulerPolicy_To_policy_DynamicSchedulerPolicy(a.(*DynamicSchedulerPolicy), b.(*policy.DynamicSchedulerPolicy), scope) 25 | }); err != nil { 26 | return err 27 | } 28 | if err := s.AddGeneratedConversionFunc((*policy.DynamicSchedulerPolicy)(nil), (*DynamicSchedulerPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 29 | return Convert_policy_DynamicSchedulerPolicy_To_v1alpha1_DynamicSchedulerPolicy(a.(*policy.DynamicSchedulerPolicy), b.(*DynamicSchedulerPolicy), scope) 30 | }); err != nil { 31 | return err 32 | } 33 | if err := s.AddGeneratedConversionFunc((*HotValuePolicy)(nil), (*policy.HotValuePolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 34 | return Convert_v1alpha1_HotValuePolicy_To_policy_HotValuePolicy(a.(*HotValuePolicy), b.(*policy.HotValuePolicy), scope) 35 | }); err != nil { 36 | return err 37 | } 38 | if err := s.AddGeneratedConversionFunc((*policy.HotValuePolicy)(nil), (*HotValuePolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 39 | return Convert_policy_HotValuePolicy_To_v1alpha1_HotValuePolicy(a.(*policy.HotValuePolicy), b.(*HotValuePolicy), scope) 40 | }); err != nil { 41 | return err 42 | } 43 | if err := s.AddGeneratedConversionFunc((*PolicySpec)(nil), (*policy.PolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { 44 | return Convert_v1alpha1_PolicySpec_To_policy_PolicySpec(a.(*PolicySpec), b.(*policy.PolicySpec), scope) 45 | }); err != nil { 46 | return err 47 | } 48 | if err := s.AddGeneratedConversionFunc((*policy.PolicySpec)(nil), (*PolicySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { 49 | return Convert_policy_PolicySpec_To_v1alpha1_PolicySpec(a.(*policy.PolicySpec), b.(*PolicySpec), scope) 50 | }); err != nil { 51 | return err 52 | } 53 | if err := s.AddGeneratedConversionFunc((*PredicatePolicy)(nil), (*policy.PredicatePolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 54 | return Convert_v1alpha1_PredicatePolicy_To_policy_PredicatePolicy(a.(*PredicatePolicy), b.(*policy.PredicatePolicy), scope) 55 | }); err != nil { 56 | return err 57 | } 58 | if err := s.AddGeneratedConversionFunc((*policy.PredicatePolicy)(nil), (*PredicatePolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 59 | return Convert_policy_PredicatePolicy_To_v1alpha1_PredicatePolicy(a.(*policy.PredicatePolicy), b.(*PredicatePolicy), scope) 60 | }); err != nil { 61 | return err 62 | } 63 | if err := s.AddGeneratedConversionFunc((*PriorityPolicy)(nil), (*policy.PriorityPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 64 | return Convert_v1alpha1_PriorityPolicy_To_policy_PriorityPolicy(a.(*PriorityPolicy), b.(*policy.PriorityPolicy), scope) 65 | }); err != nil { 66 | return err 67 | } 68 | if err := s.AddGeneratedConversionFunc((*policy.PriorityPolicy)(nil), (*PriorityPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 69 | return Convert_policy_PriorityPolicy_To_v1alpha1_PriorityPolicy(a.(*policy.PriorityPolicy), b.(*PriorityPolicy), scope) 70 | }); err != nil { 71 | return err 72 | } 73 | if err := s.AddGeneratedConversionFunc((*SyncPolicy)(nil), (*policy.SyncPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 74 | return Convert_v1alpha1_SyncPolicy_To_policy_SyncPolicy(a.(*SyncPolicy), b.(*policy.SyncPolicy), scope) 75 | }); err != nil { 76 | return err 77 | } 78 | if err := s.AddGeneratedConversionFunc((*policy.SyncPolicy)(nil), (*SyncPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { 79 | return Convert_policy_SyncPolicy_To_v1alpha1_SyncPolicy(a.(*policy.SyncPolicy), b.(*SyncPolicy), scope) 80 | }); err != nil { 81 | return err 82 | } 83 | return nil 84 | } 85 | 86 | func autoConvert_v1alpha1_DynamicSchedulerPolicy_To_policy_DynamicSchedulerPolicy(in *DynamicSchedulerPolicy, out *policy.DynamicSchedulerPolicy, s conversion.Scope) error { 87 | if err := Convert_v1alpha1_PolicySpec_To_policy_PolicySpec(&in.Spec, &out.Spec, s); err != nil { 88 | return err 89 | } 90 | return nil 91 | } 92 | 93 | // Convert_v1alpha1_DynamicSchedulerPolicy_To_policy_DynamicSchedulerPolicy is an autogenerated conversion function. 94 | func Convert_v1alpha1_DynamicSchedulerPolicy_To_policy_DynamicSchedulerPolicy(in *DynamicSchedulerPolicy, out *policy.DynamicSchedulerPolicy, s conversion.Scope) error { 95 | return autoConvert_v1alpha1_DynamicSchedulerPolicy_To_policy_DynamicSchedulerPolicy(in, out, s) 96 | } 97 | 98 | func autoConvert_policy_DynamicSchedulerPolicy_To_v1alpha1_DynamicSchedulerPolicy(in *policy.DynamicSchedulerPolicy, out *DynamicSchedulerPolicy, s conversion.Scope) error { 99 | if err := Convert_policy_PolicySpec_To_v1alpha1_PolicySpec(&in.Spec, &out.Spec, s); err != nil { 100 | return err 101 | } 102 | return nil 103 | } 104 | 105 | // Convert_policy_DynamicSchedulerPolicy_To_v1alpha1_DynamicSchedulerPolicy is an autogenerated conversion function. 106 | func Convert_policy_DynamicSchedulerPolicy_To_v1alpha1_DynamicSchedulerPolicy(in *policy.DynamicSchedulerPolicy, out *DynamicSchedulerPolicy, s conversion.Scope) error { 107 | return autoConvert_policy_DynamicSchedulerPolicy_To_v1alpha1_DynamicSchedulerPolicy(in, out, s) 108 | } 109 | 110 | func autoConvert_v1alpha1_HotValuePolicy_To_policy_HotValuePolicy(in *HotValuePolicy, out *policy.HotValuePolicy, s conversion.Scope) error { 111 | out.TimeRange = in.TimeRange 112 | out.Count = in.Count 113 | return nil 114 | } 115 | 116 | // Convert_v1alpha1_HotValuePolicy_To_policy_HotValuePolicy is an autogenerated conversion function. 117 | func Convert_v1alpha1_HotValuePolicy_To_policy_HotValuePolicy(in *HotValuePolicy, out *policy.HotValuePolicy, s conversion.Scope) error { 118 | return autoConvert_v1alpha1_HotValuePolicy_To_policy_HotValuePolicy(in, out, s) 119 | } 120 | 121 | func autoConvert_policy_HotValuePolicy_To_v1alpha1_HotValuePolicy(in *policy.HotValuePolicy, out *HotValuePolicy, s conversion.Scope) error { 122 | out.TimeRange = in.TimeRange 123 | out.Count = in.Count 124 | return nil 125 | } 126 | 127 | // Convert_policy_HotValuePolicy_To_v1alpha1_HotValuePolicy is an autogenerated conversion function. 128 | func Convert_policy_HotValuePolicy_To_v1alpha1_HotValuePolicy(in *policy.HotValuePolicy, out *HotValuePolicy, s conversion.Scope) error { 129 | return autoConvert_policy_HotValuePolicy_To_v1alpha1_HotValuePolicy(in, out, s) 130 | } 131 | 132 | func autoConvert_v1alpha1_PolicySpec_To_policy_PolicySpec(in *PolicySpec, out *policy.PolicySpec, s conversion.Scope) error { 133 | out.SyncPeriod = *(*[]policy.SyncPolicy)(unsafe.Pointer(&in.SyncPeriod)) 134 | out.Predicate = *(*[]policy.PredicatePolicy)(unsafe.Pointer(&in.Predicate)) 135 | out.Priority = *(*[]policy.PriorityPolicy)(unsafe.Pointer(&in.Priority)) 136 | out.HotValue = *(*[]policy.HotValuePolicy)(unsafe.Pointer(&in.HotValue)) 137 | return nil 138 | } 139 | 140 | // Convert_v1alpha1_PolicySpec_To_policy_PolicySpec is an autogenerated conversion function. 141 | func Convert_v1alpha1_PolicySpec_To_policy_PolicySpec(in *PolicySpec, out *policy.PolicySpec, s conversion.Scope) error { 142 | return autoConvert_v1alpha1_PolicySpec_To_policy_PolicySpec(in, out, s) 143 | } 144 | 145 | func autoConvert_policy_PolicySpec_To_v1alpha1_PolicySpec(in *policy.PolicySpec, out *PolicySpec, s conversion.Scope) error { 146 | out.SyncPeriod = *(*[]SyncPolicy)(unsafe.Pointer(&in.SyncPeriod)) 147 | out.Predicate = *(*[]PredicatePolicy)(unsafe.Pointer(&in.Predicate)) 148 | out.Priority = *(*[]PriorityPolicy)(unsafe.Pointer(&in.Priority)) 149 | out.HotValue = *(*[]HotValuePolicy)(unsafe.Pointer(&in.HotValue)) 150 | return nil 151 | } 152 | 153 | // Convert_policy_PolicySpec_To_v1alpha1_PolicySpec is an autogenerated conversion function. 154 | func Convert_policy_PolicySpec_To_v1alpha1_PolicySpec(in *policy.PolicySpec, out *PolicySpec, s conversion.Scope) error { 155 | return autoConvert_policy_PolicySpec_To_v1alpha1_PolicySpec(in, out, s) 156 | } 157 | 158 | func autoConvert_v1alpha1_PredicatePolicy_To_policy_PredicatePolicy(in *PredicatePolicy, out *policy.PredicatePolicy, s conversion.Scope) error { 159 | out.Name = in.Name 160 | out.MaxLimitPecent = in.MaxLimitPecent 161 | return nil 162 | } 163 | 164 | // Convert_v1alpha1_PredicatePolicy_To_policy_PredicatePolicy is an autogenerated conversion function. 165 | func Convert_v1alpha1_PredicatePolicy_To_policy_PredicatePolicy(in *PredicatePolicy, out *policy.PredicatePolicy, s conversion.Scope) error { 166 | return autoConvert_v1alpha1_PredicatePolicy_To_policy_PredicatePolicy(in, out, s) 167 | } 168 | 169 | func autoConvert_policy_PredicatePolicy_To_v1alpha1_PredicatePolicy(in *policy.PredicatePolicy, out *PredicatePolicy, s conversion.Scope) error { 170 | out.Name = in.Name 171 | out.MaxLimitPecent = in.MaxLimitPecent 172 | return nil 173 | } 174 | 175 | // Convert_policy_PredicatePolicy_To_v1alpha1_PredicatePolicy is an autogenerated conversion function. 176 | func Convert_policy_PredicatePolicy_To_v1alpha1_PredicatePolicy(in *policy.PredicatePolicy, out *PredicatePolicy, s conversion.Scope) error { 177 | return autoConvert_policy_PredicatePolicy_To_v1alpha1_PredicatePolicy(in, out, s) 178 | } 179 | 180 | func autoConvert_v1alpha1_PriorityPolicy_To_policy_PriorityPolicy(in *PriorityPolicy, out *policy.PriorityPolicy, s conversion.Scope) error { 181 | out.Name = in.Name 182 | out.Weight = in.Weight 183 | return nil 184 | } 185 | 186 | // Convert_v1alpha1_PriorityPolicy_To_policy_PriorityPolicy is an autogenerated conversion function. 187 | func Convert_v1alpha1_PriorityPolicy_To_policy_PriorityPolicy(in *PriorityPolicy, out *policy.PriorityPolicy, s conversion.Scope) error { 188 | return autoConvert_v1alpha1_PriorityPolicy_To_policy_PriorityPolicy(in, out, s) 189 | } 190 | 191 | func autoConvert_policy_PriorityPolicy_To_v1alpha1_PriorityPolicy(in *policy.PriorityPolicy, out *PriorityPolicy, s conversion.Scope) error { 192 | out.Name = in.Name 193 | out.Weight = in.Weight 194 | return nil 195 | } 196 | 197 | // Convert_policy_PriorityPolicy_To_v1alpha1_PriorityPolicy is an autogenerated conversion function. 198 | func Convert_policy_PriorityPolicy_To_v1alpha1_PriorityPolicy(in *policy.PriorityPolicy, out *PriorityPolicy, s conversion.Scope) error { 199 | return autoConvert_policy_PriorityPolicy_To_v1alpha1_PriorityPolicy(in, out, s) 200 | } 201 | 202 | func autoConvert_v1alpha1_SyncPolicy_To_policy_SyncPolicy(in *SyncPolicy, out *policy.SyncPolicy, s conversion.Scope) error { 203 | out.Name = in.Name 204 | out.Period = in.Period 205 | return nil 206 | } 207 | 208 | // Convert_v1alpha1_SyncPolicy_To_policy_SyncPolicy is an autogenerated conversion function. 209 | func Convert_v1alpha1_SyncPolicy_To_policy_SyncPolicy(in *SyncPolicy, out *policy.SyncPolicy, s conversion.Scope) error { 210 | return autoConvert_v1alpha1_SyncPolicy_To_policy_SyncPolicy(in, out, s) 211 | } 212 | 213 | func autoConvert_policy_SyncPolicy_To_v1alpha1_SyncPolicy(in *policy.SyncPolicy, out *SyncPolicy, s conversion.Scope) error { 214 | out.Name = in.Name 215 | out.Period = in.Period 216 | return nil 217 | } 218 | 219 | // Convert_policy_SyncPolicy_To_v1alpha1_SyncPolicy is an autogenerated conversion function. 220 | func Convert_policy_SyncPolicy_To_v1alpha1_SyncPolicy(in *policy.SyncPolicy, out *SyncPolicy, s conversion.Scope) error { 221 | return autoConvert_policy_SyncPolicy_To_v1alpha1_SyncPolicy(in, out, s) 222 | } 223 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/v1alpha1/deepcopy_generated.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | // Code generated by deepcopy-gen. DO NOT EDIT. 5 | 6 | package v1alpha1 7 | 8 | import ( 9 | runtime "k8s.io/apimachinery/pkg/runtime" 10 | ) 11 | 12 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 13 | func (in *DynamicSchedulerPolicy) DeepCopyInto(out *DynamicSchedulerPolicy) { 14 | *out = *in 15 | out.TypeMeta = in.TypeMeta 16 | in.Spec.DeepCopyInto(&out.Spec) 17 | return 18 | } 19 | 20 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicSchedulerPolicy. 21 | func (in *DynamicSchedulerPolicy) DeepCopy() *DynamicSchedulerPolicy { 22 | if in == nil { 23 | return nil 24 | } 25 | out := new(DynamicSchedulerPolicy) 26 | in.DeepCopyInto(out) 27 | return out 28 | } 29 | 30 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 31 | func (in *DynamicSchedulerPolicy) DeepCopyObject() runtime.Object { 32 | if c := in.DeepCopy(); c != nil { 33 | return c 34 | } 35 | return nil 36 | } 37 | 38 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 39 | func (in *HotValuePolicy) DeepCopyInto(out *HotValuePolicy) { 40 | *out = *in 41 | in.TimeRange.DeepCopyInto(&out.TimeRange) 42 | return 43 | } 44 | 45 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HotValuePolicy. 46 | func (in *HotValuePolicy) DeepCopy() *HotValuePolicy { 47 | if in == nil { 48 | return nil 49 | } 50 | out := new(HotValuePolicy) 51 | in.DeepCopyInto(out) 52 | return out 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *PolicySpec) DeepCopyInto(out *PolicySpec) { 57 | *out = *in 58 | if in.SyncPeriod != nil { 59 | in, out := &in.SyncPeriod, &out.SyncPeriod 60 | *out = make([]SyncPolicy, len(*in)) 61 | for i := range *in { 62 | (*in)[i].DeepCopyInto(&(*out)[i]) 63 | } 64 | } 65 | if in.Predicate != nil { 66 | in, out := &in.Predicate, &out.Predicate 67 | *out = make([]PredicatePolicy, len(*in)) 68 | copy(*out, *in) 69 | } 70 | if in.Priority != nil { 71 | in, out := &in.Priority, &out.Priority 72 | *out = make([]PriorityPolicy, len(*in)) 73 | copy(*out, *in) 74 | } 75 | if in.HotValue != nil { 76 | in, out := &in.HotValue, &out.HotValue 77 | *out = make([]HotValuePolicy, len(*in)) 78 | for i := range *in { 79 | (*in)[i].DeepCopyInto(&(*out)[i]) 80 | } 81 | } 82 | return 83 | } 84 | 85 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicySpec. 86 | func (in *PolicySpec) DeepCopy() *PolicySpec { 87 | if in == nil { 88 | return nil 89 | } 90 | out := new(PolicySpec) 91 | in.DeepCopyInto(out) 92 | return out 93 | } 94 | 95 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 96 | func (in *PredicatePolicy) DeepCopyInto(out *PredicatePolicy) { 97 | *out = *in 98 | return 99 | } 100 | 101 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PredicatePolicy. 102 | func (in *PredicatePolicy) DeepCopy() *PredicatePolicy { 103 | if in == nil { 104 | return nil 105 | } 106 | out := new(PredicatePolicy) 107 | in.DeepCopyInto(out) 108 | return out 109 | } 110 | 111 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 112 | func (in *PriorityPolicy) DeepCopyInto(out *PriorityPolicy) { 113 | *out = *in 114 | return 115 | } 116 | 117 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PriorityPolicy. 118 | func (in *PriorityPolicy) DeepCopy() *PriorityPolicy { 119 | if in == nil { 120 | return nil 121 | } 122 | out := new(PriorityPolicy) 123 | in.DeepCopyInto(out) 124 | return out 125 | } 126 | 127 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 128 | func (in *SyncPolicy) DeepCopyInto(out *SyncPolicy) { 129 | *out = *in 130 | in.Period.DeepCopyInto(&out.Period) 131 | return 132 | } 133 | 134 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncPolicy. 135 | func (in *SyncPolicy) DeepCopy() *SyncPolicy { 136 | if in == nil { 137 | return nil 138 | } 139 | out := new(SyncPolicy) 140 | in.DeepCopyInto(out) 141 | return out 142 | } 143 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package,register 2 | // +k8s:conversion-gen=github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy 3 | 4 | package v1alpha1 // import "crane.io/crane-scheduler/pkg/plugins/apis/policy/v1alpha1" 5 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | // GroupName is the group name used in this package 9 | const GroupName = "scheduler.policy.crane.io" 10 | 11 | // SchemeGroupVersion is group version used to register these objects 12 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} 13 | 14 | var ( 15 | // SchemeBuilder is the scheme builder with scheme init functions to run for this API package 16 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 17 | // AddToScheme is a global function that registers this API group & version to a scheme 18 | AddToScheme = SchemeBuilder.AddToScheme 19 | // localSchemeBuilder extends the SchemeBuilder instance with the external types 20 | localSchemeBuilder = &SchemeBuilder 21 | ) 22 | 23 | // addKnownTypes registers known types to the given scheme 24 | func addKnownTypes(scheme *runtime.Scheme) error { 25 | scheme.AddKnownTypes(SchemeGroupVersion, 26 | &DynamicSchedulerPolicy{}, 27 | ) 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/plugins/apis/policy/v1alpha1/types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 8 | 9 | type DynamicSchedulerPolicy struct { 10 | metav1.TypeMeta `json:",inline"` 11 | Spec PolicySpec `json:"spec"` 12 | } 13 | 14 | type PolicySpec struct { 15 | SyncPeriod []SyncPolicy `json:"syncPolicy"` 16 | Predicate []PredicatePolicy `json:"predicate"` 17 | Priority []PriorityPolicy `json:"priority"` 18 | HotValue []HotValuePolicy `json:"hotValue"` 19 | } 20 | 21 | type SyncPolicy struct { 22 | Name string `json:"name"` 23 | Period metav1.Duration `json:"period"` 24 | } 25 | 26 | type PredicatePolicy struct { 27 | Name string `json:"name"` 28 | MaxLimitPecent float64 `json:"maxLimitPecent"` 29 | } 30 | 31 | type PriorityPolicy struct { 32 | Name string `json:"name"` 33 | Weight float64 `json:"weight"` 34 | } 35 | 36 | type HotValuePolicy struct { 37 | TimeRange metav1.Duration `json:"timeRange"` 38 | Count int `json:"count"` 39 | } 40 | -------------------------------------------------------------------------------- /pkg/plugins/dynamic/plugins.go: -------------------------------------------------------------------------------- 1 | package dynamic 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | v1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | "k8s.io/klog/v2" 10 | "k8s.io/kubernetes/pkg/scheduler/framework" 11 | 12 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config" 13 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 14 | "github.com/gocrane/crane-scheduler/pkg/utils" 15 | ) 16 | 17 | var _ framework.FilterPlugin = &DynamicScheduler{} 18 | var _ framework.ScorePlugin = &DynamicScheduler{} 19 | 20 | const ( 21 | // Name is the name of the plugin used in the plugin registry and configurations. 22 | Name = "Dynamic" 23 | ) 24 | 25 | // Dynamic-scheduler is a real load-aware scheduler plugin. 26 | type DynamicScheduler struct { 27 | handle framework.Handle 28 | schedulerPolicy *policy.DynamicSchedulerPolicy 29 | } 30 | 31 | // Name returns name of the plugin. 32 | func (ds *DynamicScheduler) Name() string { 33 | return Name 34 | } 35 | 36 | // Filter invoked at the filter extension point. 37 | // checkes if the real load of one node is too high. 38 | // It returns a list of failure reasons if the node is overload. 39 | func (ds *DynamicScheduler) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 40 | // ignore daemonset pod 41 | if utils.IsDaemonsetPod(pod) { 42 | return framework.NewStatus(framework.Success, "") 43 | } 44 | 45 | node := nodeInfo.Node() 46 | if node == nil { 47 | return framework.NewStatus(framework.Error, "node not found") 48 | } 49 | 50 | nodeAnnotations, nodeName := nodeInfo.Node().Annotations, nodeInfo.Node().Name 51 | if nodeAnnotations == nil { 52 | nodeAnnotations = map[string]string{} 53 | } 54 | 55 | for _, policy := range ds.schedulerPolicy.Spec.Predicate { 56 | activeDuration, err := getActiveDuration(ds.schedulerPolicy.Spec.SyncPeriod, policy.Name) 57 | 58 | if err != nil || activeDuration == 0 { 59 | klog.Warningf("[crane] failed to get active duration: %v", err) 60 | continue 61 | } 62 | 63 | if isOverLoad(nodeName, nodeAnnotations, policy, activeDuration) { 64 | return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Load[%s] of node[%s] is too high", policy.Name, nodeName)) 65 | } 66 | 67 | } 68 | return framework.NewStatus(framework.Success, "") 69 | } 70 | 71 | // Score invoked at the Score extension point. 72 | // It gets metric data from node annotation, and favors nodes with the least real resource usage. 73 | func (ds *DynamicScheduler) Score(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) { 74 | nodeInfo, err := ds.handle.SnapshotSharedLister().NodeInfos().Get(nodeName) 75 | if err != nil { 76 | return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v", nodeName, err)) 77 | } 78 | 79 | node := nodeInfo.Node() 80 | if node == nil { 81 | return 0, framework.NewStatus(framework.Error, "node not found") 82 | } 83 | 84 | nodeAnnotations := node.Annotations 85 | if nodeAnnotations == nil { 86 | nodeAnnotations = map[string]string{} 87 | } 88 | 89 | score, hotValue := getNodeScore(node.Name, nodeAnnotations, ds.schedulerPolicy.Spec), getNodeHotValue(node) 90 | 91 | score = score - int(hotValue*10) 92 | 93 | finalScore := utils.NormalizeScore(int64(score),framework.MaxNodeScore,framework.MinNodeScore) 94 | 95 | klog.V(4).Infof("[crane] Node[%s]'s final score is %d, while score is %d and hot value is %f", node.Name, finalScore, score, hotValue) 96 | 97 | return finalScore, nil 98 | } 99 | 100 | func (ds *DynamicScheduler) ScoreExtensions() framework.ScoreExtensions { 101 | return nil 102 | } 103 | 104 | // NewDynamicScheduler returns a Crane Scheduler object. 105 | func NewDynamicScheduler(plArgs runtime.Object, h framework.Handle) (framework.Plugin, error) { 106 | args, ok := plArgs.(*config.DynamicArgs) 107 | if !ok { 108 | return nil, fmt.Errorf("want args to be of type DynamicArgs, got %T.", plArgs) 109 | } 110 | 111 | schedulerPolicy, err := LoadPolicyFromFile(args.PolicyConfigPath) 112 | if err != nil { 113 | return nil, fmt.Errorf("failed to get scheduler policy from config file: %v", err) 114 | } 115 | 116 | return &DynamicScheduler{ 117 | schedulerPolicy: schedulerPolicy, 118 | handle: h, 119 | }, nil 120 | } 121 | -------------------------------------------------------------------------------- /pkg/plugins/dynamic/policyfile.go: -------------------------------------------------------------------------------- 1 | package dynamic 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | 7 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 8 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy/scheme" 9 | ) 10 | 11 | func LoadPolicyFromFile(file string) (*policy.DynamicSchedulerPolicy, error) { 12 | data, err := ioutil.ReadFile(file) 13 | if err != nil { 14 | return nil, err 15 | } 16 | 17 | return loadPolicy(data) 18 | } 19 | 20 | func loadPolicy(data []byte) (*policy.DynamicSchedulerPolicy, error) { 21 | // The UniversalDecoder runs defaulting and returns the internal type by default. 22 | obj, gvk, err := scheme.Codecs.UniversalDecoder().Decode(data, nil, nil) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | if policyObj, ok := obj.(*policy.DynamicSchedulerPolicy); ok { 28 | policyObj.TypeMeta.APIVersion = gvk.GroupVersion().String() 29 | return policyObj, nil 30 | } 31 | 32 | return nil, fmt.Errorf("couldn't decode as DynamicSchedulerPolicy, got %s: ", gvk) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/plugins/dynamic/stats.go: -------------------------------------------------------------------------------- 1 | package dynamic 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | "k8s.io/klog/v2" 11 | "k8s.io/kubernetes/pkg/scheduler/framework" 12 | 13 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/policy" 14 | 15 | "github.com/gocrane/crane-scheduler/pkg/utils" 16 | ) 17 | 18 | const ( 19 | // MinTimestampStrLength defines the min length of timestamp string. 20 | MinTimestampStrLength = 5 21 | // NodeHotValue is the key of hot value annotation. 22 | NodeHotValue = "node_hot_value" 23 | // DefautlHotVauleActivePeriod defines the validity period of nodes' hotvalue. 24 | DefautlHotVauleActivePeriod = 5 * time.Minute 25 | // ExtraActivePeriod gives extra active time to the annotation. 26 | ExtraActivePeriod = 5 * time.Minute 27 | ) 28 | 29 | // inActivePeriod judges if node annotation with this timestamp is effective. 30 | func inActivePeriod(updatetimeStr string, activeDuration time.Duration) bool { 31 | if len(updatetimeStr) < MinTimestampStrLength { 32 | klog.Errorf("[crane] illegel timestamp: %s", updatetimeStr) 33 | return false 34 | } 35 | 36 | originUpdateTime, err := time.ParseInLocation(utils.TimeFormat, updatetimeStr, utils.GetLocation()) 37 | if err != nil { 38 | klog.Errorf("[crane] failed to parse timestamp: %v", err) 39 | return false 40 | } 41 | 42 | now, updatetime := time.Now(), originUpdateTime.Add(activeDuration) 43 | 44 | if now.Before(updatetime) { 45 | return true 46 | } 47 | 48 | return false 49 | } 50 | 51 | func getResourceUsage(anno map[string]string, key string, activeDuration time.Duration) (float64, error) { 52 | usedstr, ok := anno[key] 53 | if !ok { 54 | return 0, fmt.Errorf("key[%s] not found", usedstr) 55 | } 56 | 57 | usedSlice := strings.Split(usedstr, ",") 58 | if len(usedSlice) != 2 { 59 | return 0, fmt.Errorf("illegel value: %s", usedstr) 60 | } 61 | 62 | if !inActivePeriod(usedSlice[1], activeDuration) { 63 | return 0, fmt.Errorf("timestamp[%s] is expired", usedstr) 64 | } 65 | 66 | UsedValue, err := strconv.ParseFloat(usedSlice[0], 64) 67 | if err != nil { 68 | return 0, fmt.Errorf("failed to parse float[%s]", usedSlice[0]) 69 | } 70 | 71 | if UsedValue < 0 { 72 | return 0, fmt.Errorf("illegel value: %s", usedstr) 73 | } 74 | 75 | return UsedValue, nil 76 | } 77 | 78 | func getScore(anno map[string]string, priorityPolicy policy.PriorityPolicy, syncPeriod []policy.SyncPolicy) (float64, error) { 79 | activeDuration, err := getActiveDuration(syncPeriod, priorityPolicy.Name) 80 | if err != nil || activeDuration == 0 { 81 | return 0, fmt.Errorf("failed to get the active duration of resource[%s]: %v, while the actual value is %v", priorityPolicy.Name, err, activeDuration) 82 | } 83 | 84 | usage, err := getResourceUsage(anno, priorityPolicy.Name, activeDuration) 85 | if err != nil { 86 | return 0, err 87 | } 88 | 89 | score := (1. - usage) * priorityPolicy.Weight * float64(framework.MaxNodeScore) 90 | 91 | return score, nil 92 | } 93 | 94 | func isOverLoad(name string, anno map[string]string, predicatePolicy policy.PredicatePolicy, activeDuration time.Duration) bool { 95 | usage, err := getResourceUsage(anno, predicatePolicy.Name, activeDuration) 96 | if err != nil { 97 | klog.Errorf("[crane] can not get the usage of resource[%s] from node[%s]'s annotation: %v", predicatePolicy.Name, name, err) 98 | return false 99 | } 100 | 101 | // threshold was set as 0 means that the filter according to this metric is useless. 102 | if predicatePolicy.MaxLimitPecent == 0 { 103 | klog.V(4).Info("[crane] ignore the filter of resource[%s] for MaxLimitPecent was set as 0") 104 | return false 105 | } 106 | 107 | if usage > predicatePolicy.MaxLimitPecent { 108 | return true 109 | } 110 | 111 | return false 112 | } 113 | 114 | func getNodeScore(name string, anno map[string]string, policySpec policy.PolicySpec) int { 115 | 116 | lenPriorityPolicyList := len(policySpec.Priority) 117 | if lenPriorityPolicyList == 0 { 118 | klog.Warningf("[crane] no priority policy exists, all nodes scores 0.") 119 | return 0 120 | } 121 | 122 | var score, weight float64 123 | 124 | for _, priorityPolicy := range policySpec.Priority { 125 | 126 | priorityScore, err := getScore(anno, priorityPolicy, policySpec.SyncPeriod) 127 | if err != nil { 128 | klog.Errorf("[crane] failed to get node 's score: %v", name, priorityPolicy.Name, score) 129 | } 130 | 131 | weight += priorityPolicy.Weight 132 | score += priorityScore 133 | } 134 | 135 | finnalScore := int(score / weight) 136 | 137 | return finnalScore 138 | } 139 | 140 | func getActiveDuration(syncPeriodList []policy.SyncPolicy, name string) (time.Duration, error) { 141 | for _, period := range syncPeriodList { 142 | if period.Name == name { 143 | if period.Period.Duration != 0 { 144 | return period.Period.Duration + ExtraActivePeriod, nil 145 | } 146 | } 147 | } 148 | 149 | return 0, fmt.Errorf("failed to get the active duration") 150 | } 151 | 152 | func getNodeHotValue(node *v1.Node) float64 { 153 | anno := node.ObjectMeta.Annotations 154 | if anno == nil { 155 | return 0 156 | } 157 | 158 | hotvalue, err := getResourceUsage(anno, NodeHotValue, DefautlHotVauleActivePeriod) 159 | if err != nil { 160 | return 0 161 | } 162 | 163 | klog.V(4).Infof("[crane] Node[%s]'s hotvalue is %f\n", node.Name, hotvalue) 164 | 165 | return hotvalue 166 | } 167 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/binder.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | jsonpatch "github.com/evanphx/json-patch" 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | "k8s.io/klog/v2" 13 | "k8s.io/kubernetes/pkg/scheduler/framework" 14 | 15 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 16 | ) 17 | 18 | // PreBind writes pod topology result annotations using the k8s client. 19 | func (tm *TopologyMatch) PreBind( 20 | ctx context.Context, 21 | state *framework.CycleState, 22 | pod *corev1.Pod, 23 | nodeName string, 24 | ) *framework.Status { 25 | klog.V(4).InfoS("Attempting to prebind pod to node", "pod", klog.KObj(pod), "node", nodeName) 26 | s, err := getStateData(state) 27 | if err != nil { 28 | return framework.AsStatus(err) 29 | } 30 | 31 | if len(s.topologyResult) == 0 { 32 | return nil 33 | } 34 | 35 | result, err := json.Marshal(s.topologyResult) 36 | if err != nil { 37 | return framework.AsStatus(err) 38 | } 39 | newObj := pod.DeepCopy() 40 | if newObj.Annotations == nil { 41 | newObj.Annotations = make(map[string]string) 42 | } 43 | newObj.Annotations[topologyv1alpha1.AnnotationPodTopologyResultKey] = string(result) 44 | 45 | oldData, err := json.Marshal(pod) 46 | if err != nil { 47 | return framework.AsStatus(err) 48 | } 49 | newData, err := json.Marshal(newObj) 50 | if err != nil { 51 | return framework.AsStatus(err) 52 | } 53 | 54 | patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData) 55 | if err != nil { 56 | return framework.AsStatus(fmt.Errorf("failed to create merge patch: %v", err)) 57 | } 58 | 59 | _, err = tm.handle.ClientSet().CoreV1().Pods(pod.Namespace).Patch(ctx, pod.Name, 60 | types.MergePatchType, patchBytes, metav1.PatchOptions{}) 61 | if err != nil { 62 | return framework.AsStatus(err) 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/cache.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/util/wait" 11 | "k8s.io/klog/v2" 12 | "k8s.io/kubernetes/pkg/scheduler/framework" 13 | 14 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 15 | ) 16 | 17 | var ( 18 | cleanAssumedPeriod = 1 * time.Second 19 | ) 20 | 21 | // PodTopologyCache is a cache which stores the pod topology scheduling result. 22 | // It is used before the pod bound since the result has not been recorded into 23 | // annotations yet. 24 | type PodTopologyCache interface { 25 | AssumePod(pod *corev1.Pod, zone topologyv1alpha1.ZoneList) error 26 | ForgetPod(pod *corev1.Pod) error 27 | PodCount() int 28 | GetPodTopology(pod *corev1.Pod) (topologyv1alpha1.ZoneList, error) 29 | } 30 | 31 | type podTopologyCacheImpl struct { 32 | // This mutex guards all fields within this cache struct. 33 | sync.RWMutex 34 | ttl time.Duration 35 | period time.Duration 36 | podTopology map[string]topologyv1alpha1.ZoneList 37 | podTopologyTTL map[string]*time.Time 38 | } 39 | 40 | // NewPodTopologyCache returns a PodTopologyCache. 41 | func NewPodTopologyCache(ctx context.Context, ttl time.Duration) PodTopologyCache { 42 | cache := &podTopologyCacheImpl{ 43 | ttl: ttl, 44 | period: cleanAssumedPeriod, 45 | podTopology: make(map[string]topologyv1alpha1.ZoneList), 46 | podTopologyTTL: make(map[string]*time.Time), 47 | } 48 | cache.run(ctx.Done()) 49 | return cache 50 | } 51 | 52 | // AssumePod adds a pod and its topology result into cache. 53 | func (c *podTopologyCacheImpl) AssumePod(pod *corev1.Pod, zone topologyv1alpha1.ZoneList) error { 54 | key, err := framework.GetPodKey(pod) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | c.Lock() 60 | defer c.Unlock() 61 | 62 | if _, ok := c.podTopology[key]; ok { 63 | return fmt.Errorf("pod %v is in the podTopologyCache, so can't be assumed", key) 64 | } 65 | dl := time.Now().Add(c.ttl) 66 | c.podTopology[key] = zone 67 | c.podTopologyTTL[key] = &dl 68 | return nil 69 | } 70 | 71 | // ForgetPod removes the pod topology result from cache. It is called after binding or scheduling failure. 72 | func (c *podTopologyCacheImpl) ForgetPod(pod *corev1.Pod) error { 73 | key, err := framework.GetPodKey(pod) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | c.Lock() 79 | defer c.Unlock() 80 | 81 | c.removePod(key) 82 | return nil 83 | } 84 | 85 | // PodCount returns the number of pods in the cache. 86 | func (c *podTopologyCacheImpl) PodCount() int { 87 | c.RLock() 88 | defer c.RUnlock() 89 | 90 | return len(c.podTopology) 91 | } 92 | 93 | // GetPodTopology returns the pod zone from the cache with the same namespace and the same name of the specified pod. 94 | func (c *podTopologyCacheImpl) GetPodTopology(pod *corev1.Pod) (topologyv1alpha1.ZoneList, error) { 95 | key, err := framework.GetPodKey(pod) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | c.RLock() 101 | defer c.RUnlock() 102 | 103 | topology, ok := c.podTopology[key] 104 | if !ok { 105 | return nil, fmt.Errorf("pod topology %v does not exist in cache", key) 106 | } 107 | 108 | return topology, nil 109 | } 110 | 111 | func (c *podTopologyCacheImpl) run(stopCh <-chan struct{}) { 112 | go wait.Until(c.cleanupExpiredAssumedPods, c.period, stopCh) 113 | } 114 | 115 | func (c *podTopologyCacheImpl) cleanupExpiredAssumedPods() { 116 | c.cleanupAssumedPods(time.Now()) 117 | } 118 | 119 | // cleanupAssumedPods exists for making test deterministic by taking time as input argument. 120 | func (c *podTopologyCacheImpl) cleanupAssumedPods(now time.Time) { 121 | c.Lock() 122 | defer c.Unlock() 123 | 124 | for key, dl := range c.podTopologyTTL { 125 | if now.After(*dl) { 126 | c.removePod(key) 127 | } 128 | } 129 | } 130 | 131 | func (c *podTopologyCacheImpl) removePod(key string) { 132 | delete(c.podTopology, key) 133 | delete(c.podTopologyTTL, key) 134 | klog.V(4).Infof("Finished binding for pod %v. Can be expired.", key) 135 | } 136 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/filter.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/kubernetes/pkg/scheduler/framework" 8 | 9 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 10 | 11 | "github.com/gocrane/crane-scheduler/pkg/utils" 12 | ) 13 | 14 | const ( 15 | ErrReasonNUMAResourceNotEnough = "node(s) had insufficient resource of NUMA node" 16 | ErrReasonFailedToGetNRT = "node(s) failed to get NRT" 17 | ) 18 | 19 | // PreFilter invoked at the prefilter extension point. 20 | func (tm *TopologyMatch) PreFilter( 21 | ctx context.Context, 22 | state *framework.CycleState, 23 | pod *corev1.Pod, 24 | ) *framework.Status { 25 | var indices []int 26 | if tm.topologyAwareResources.Has(string(corev1.ResourceCPU)) { 27 | indices = GetPodTargetContainerIndices(pod) 28 | } 29 | resources := computeContainerSpecifiedResourceRequest(pod, indices, tm.topologyAwareResources) 30 | state.Write(stateKey, &stateData{ 31 | aware: IsPodAwareOfTopology(pod.Annotations), 32 | targetContainerIndices: indices, 33 | targetContainerResource: resources, 34 | podTopologyByNode: make(map[string]*nodeWrapper), 35 | }) 36 | return nil 37 | } 38 | 39 | // PreFilterExtensions returns prefilter extensions, pod add and remove. 40 | func (tm *TopologyMatch) PreFilterExtensions() framework.PreFilterExtensions { 41 | return nil 42 | } 43 | 44 | // Filter will check if there exists numa node which has sufficient resource for the pod. 45 | func (tm *TopologyMatch) Filter( 46 | ctx context.Context, 47 | state *framework.CycleState, 48 | pod *corev1.Pod, 49 | nodeInfo *framework.NodeInfo, 50 | ) *framework.Status { 51 | s, err := getStateData(state) 52 | if err != nil { 53 | return framework.AsStatus(err) 54 | } 55 | 56 | if nodeInfo.Node() == nil { 57 | return framework.NewStatus(framework.Error, "node(s) not found") 58 | } 59 | 60 | if utils.IsDaemonsetPod(pod) || len(s.targetContainerIndices) == 0 { 61 | return nil 62 | } 63 | 64 | nrt, err := tm.lister.Get(nodeInfo.Node().Name) 65 | if err != nil { 66 | return framework.NewStatus(framework.Unschedulable, ErrReasonFailedToGetNRT) 67 | } 68 | // let kubelet handle cpuset 69 | if nrt.CraneManagerPolicy.CPUManagerPolicy != topologyv1alpha1.CPUManagerPolicyStatic { 70 | return nil 71 | } 72 | 73 | nw := tm.initializeNodeWrapper(s, nodeInfo, nrt) 74 | if nw.aware { 75 | if status := tm.filterNUMANodeResource(s, nw); status != nil { 76 | return status 77 | } 78 | } 79 | assignTopologyResult(nw, s.targetContainerResource.Clone()) 80 | 81 | s.Lock() 82 | defer s.Unlock() 83 | s.podTopologyByNode[nw.node] = nw 84 | 85 | return nil 86 | } 87 | 88 | func (tm *TopologyMatch) initializeNodeWrapper( 89 | state *stateData, 90 | nodeInfo *framework.NodeInfo, 91 | nrt *topologyv1alpha1.NodeResourceTopology, 92 | ) *nodeWrapper { 93 | node := nodeInfo.Node() 94 | nw := newNodeWrapper(node.Name, tm.topologyAwareResources, nrt.Zones, tm.GetPodTopology) 95 | for _, pod := range nodeInfo.Pods { 96 | nw.addPod(pod.Pod) 97 | } 98 | // If pod has specified awareness, ignore the awareness of node. 99 | if state.aware != nil { 100 | nw.aware = *state.aware 101 | } else { 102 | nw.aware = isNodeAwareOfTopology(nrt) 103 | } 104 | return nw 105 | } 106 | 107 | func (tm *TopologyMatch) filterNUMANodeResource(state *stateData, nw *nodeWrapper) *framework.Status { 108 | var res []*numaNode 109 | for _, numaNode := range nw.numaNodes { 110 | // Check resource 111 | insufficientResources := fitsRequestForNUMANode(state.targetContainerResource, numaNode) 112 | if len(insufficientResources) != 0 { 113 | continue 114 | } 115 | res = append(res, numaNode) 116 | } 117 | 118 | if len(res) == 0 { 119 | return framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough) 120 | } 121 | nw.numaNodes = res 122 | return nil 123 | } 124 | 125 | func isNodeAwareOfTopology(nrt *topologyv1alpha1.NodeResourceTopology) bool { 126 | return nrt.CraneManagerPolicy.TopologyManagerPolicy == topologyv1alpha1.TopologyManagerPolicySingleNUMANodePodLevel 127 | } 128 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/filter_test.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "math/rand" 7 | "reflect" 8 | "strconv" 9 | "testing" 10 | "time" 11 | 12 | corev1 "k8s.io/api/core/v1" 13 | "k8s.io/apimachinery/pkg/api/resource" 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | "k8s.io/apimachinery/pkg/types" 16 | "k8s.io/apimachinery/pkg/util/sets" 17 | "k8s.io/kubernetes/pkg/scheduler/framework" 18 | 19 | "github.com/gocrane/api/pkg/generated/clientset/versioned/fake" 20 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 21 | ) 22 | 23 | var ( 24 | nodeName = "master" 25 | hugePageResourceA = corev1.ResourceName(corev1.ResourceHugePagesPrefix + "2Mi") 26 | nrt = &topologyv1alpha1.NodeResourceTopology{ 27 | ObjectMeta: metav1.ObjectMeta{Name: nodeName}, 28 | CraneManagerPolicy: topologyv1alpha1.ManagerPolicy{ 29 | CPUManagerPolicy: topologyv1alpha1.CPUManagerPolicyStatic, 30 | TopologyManagerPolicy: topologyv1alpha1.TopologyManagerPolicySingleNUMANodePodLevel, 31 | }, 32 | Reserved: map[corev1.ResourceName]resource.Quantity{ 33 | corev1.ResourceCPU: resource.MustParse("1"), 34 | corev1.ResourceMemory: resource.MustParse("1Gi"), 35 | }, 36 | // node1: 2.5 cpu, 4GiB memory 37 | // node2: 3.9 cpu, 4GiB memory 38 | Zones: topologyv1alpha1.ZoneList{ 39 | topologyv1alpha1.Zone{ 40 | Name: "node1", 41 | Type: topologyv1alpha1.ZoneTypeNode, 42 | Resources: &topologyv1alpha1.ResourceInfo{ 43 | Allocatable: map[corev1.ResourceName]resource.Quantity{ 44 | corev1.ResourceCPU: resource.MustParse("2.5"), 45 | corev1.ResourceMemory: resource.MustParse("4Gi"), 46 | }, 47 | }, 48 | }, 49 | topologyv1alpha1.Zone{ 50 | Name: "node2", 51 | Type: topologyv1alpha1.ZoneTypeNode, 52 | Resources: &topologyv1alpha1.ResourceInfo{ 53 | Allocatable: map[corev1.ResourceName]resource.Quantity{ 54 | corev1.ResourceCPU: resource.MustParse("3.9"), 55 | corev1.ResourceMemory: resource.MustParse("4Gi"), 56 | }, 57 | }, 58 | }, 59 | }, 60 | } 61 | ) 62 | 63 | const ( 64 | // CPUTestUnit is 1 CPU 65 | CPUTestUnit = 1000 66 | // MemTestUnit is 1 GiB 67 | MemTestUnit = 1024 * 1024 * 1024 68 | ) 69 | 70 | type assumedPod struct { 71 | pod *corev1.Pod 72 | zone topologyv1alpha1.ZoneList 73 | } 74 | 75 | func newResourcePod(aware bool, result topologyv1alpha1.ZoneList, usage ...framework.Resource) *corev1.Pod { 76 | pod := newPod(usage...) 77 | if aware { 78 | pod.Annotations = map[string]string{ 79 | topologyv1alpha1.AnnotationPodTopologyAwarenessKey: "true", 80 | } 81 | } 82 | if len(result) != 0 { 83 | podTopologyResultBytes, err := json.Marshal(result) 84 | if err != nil { 85 | return pod 86 | } 87 | pod.Annotations[topologyv1alpha1.AnnotationPodTopologyResultKey] = string(podTopologyResultBytes) 88 | } 89 | return pod 90 | } 91 | 92 | func newResourceAssumedPod(result topologyv1alpha1.ZoneList, usage ...framework.Resource) *assumedPod { 93 | return &assumedPod{ 94 | pod: newPod(usage...), 95 | zone: result, 96 | } 97 | } 98 | 99 | type zone struct { 100 | name string 101 | cpu int64 102 | memory int64 103 | } 104 | 105 | func newZoneList(zones []zone) topologyv1alpha1.ZoneList { 106 | zoneList := make(topologyv1alpha1.ZoneList, 0, len(zones)) 107 | for _, z := range zones { 108 | elem := topologyv1alpha1.Zone{ 109 | Name: z.name, 110 | Type: topologyv1alpha1.ZoneTypeNode, 111 | Resources: &topologyv1alpha1.ResourceInfo{ 112 | Capacity: make(corev1.ResourceList), 113 | }, 114 | } 115 | if z.cpu != 0 { 116 | elem.Resources.Capacity[corev1.ResourceCPU] = *resource.NewMilliQuantity(z.cpu, resource.DecimalSI) 117 | } 118 | if z.memory != 0 { 119 | elem.Resources.Capacity[corev1.ResourceMemory] = *resource.NewQuantity(z.memory, resource.BinarySI) 120 | } 121 | zoneList = append(zoneList, elem) 122 | } 123 | return zoneList 124 | } 125 | 126 | func newPod(usage ...framework.Resource) *corev1.Pod { 127 | var containers []corev1.Container 128 | for _, req := range usage { 129 | rl := corev1.ResourceList{ 130 | corev1.ResourceCPU: *resource.NewMilliQuantity(req.MilliCPU, resource.DecimalSI), 131 | corev1.ResourceMemory: *resource.NewQuantity(req.Memory, resource.BinarySI), 132 | corev1.ResourceEphemeralStorage: *resource.NewQuantity(req.EphemeralStorage, resource.BinarySI), 133 | } 134 | for rName, rQuant := range req.ScalarResources { 135 | if rName == hugePageResourceA { 136 | rl[rName] = *resource.NewQuantity(rQuant, resource.BinarySI) 137 | } else { 138 | rl[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI) 139 | } 140 | } 141 | containers = append(containers, corev1.Container{ 142 | Resources: corev1.ResourceRequirements{Requests: rl, Limits: rl}, 143 | }) 144 | } 145 | pod := &corev1.Pod{ 146 | ObjectMeta: metav1.ObjectMeta{UID: types.UID(strconv.Itoa(rand.Int()))}, 147 | Spec: corev1.PodSpec{ 148 | Containers: containers, 149 | }, 150 | } 151 | return pod 152 | } 153 | 154 | func TestTopologyMatch_Filter(t *testing.T) { 155 | type args struct { 156 | pod *corev1.Pod 157 | nodeInfo *framework.NodeInfo 158 | nrt *topologyv1alpha1.NodeResourceTopology 159 | assumedPods []*assumedPod 160 | topologyAwareResources sets.String 161 | } 162 | tests := []struct { 163 | name string 164 | args args 165 | want *framework.Status 166 | }{ 167 | { 168 | name: "enough resource of node1 and node2", 169 | args: args{ 170 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 171 | nodeInfo: framework.NewNodeInfo( 172 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 173 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 174 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 175 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 176 | ), 177 | nrt: nrt, 178 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 179 | }, 180 | want: nil, 181 | }, 182 | { 183 | name: "enough resource of node1 and node2 with assumed pods", 184 | args: args{ 185 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 186 | nodeInfo: framework.NewNodeInfo(), 187 | nrt: nrt, 188 | assumedPods: []*assumedPod{ 189 | newResourceAssumedPod(newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 190 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 191 | newResourceAssumedPod(newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 192 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 193 | }, 194 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 195 | }, 196 | want: nil, 197 | }, 198 | { 199 | name: "no enough cpu resource", 200 | args: args{ 201 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 202 | nodeInfo: framework.NewNodeInfo( 203 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 2 * CPUTestUnit}}), 204 | framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: 2 * MemTestUnit}), 205 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 4 * CPUTestUnit}}), 206 | framework.Resource{MilliCPU: 4 * CPUTestUnit, Memory: 1 * MemTestUnit}), 207 | ), 208 | nrt: nrt, 209 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 210 | }, 211 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 212 | }, 213 | { 214 | name: "no enough cpu resource in one NUMA node", 215 | args: args{ 216 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 217 | nodeInfo: framework.NewNodeInfo( 218 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 219 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 220 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 3 * CPUTestUnit}}), 221 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 222 | ), 223 | nrt: nrt, 224 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 225 | }, 226 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 227 | }, 228 | { 229 | name: "no enough cpu resource in one NUMA node consider assumed pods", 230 | args: args{ 231 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 232 | nodeInfo: framework.NewNodeInfo( 233 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 234 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 235 | ), 236 | assumedPods: []*assumedPod{ 237 | newResourceAssumedPod(newZoneList([]zone{{name: "node2", cpu: 3 * CPUTestUnit}}), 238 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 239 | }, 240 | nrt: nrt, 241 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 242 | }, 243 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 244 | }, 245 | { 246 | name: "no enough memory resource in one NUMA node", 247 | args: args{ 248 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: 2 * MemTestUnit}), 249 | nodeInfo: framework.NewNodeInfo( 250 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit, memory: 3 * MemTestUnit}}), 251 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 3 * MemTestUnit}), 252 | ), 253 | assumedPods: []*assumedPod{ 254 | newResourceAssumedPod(newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit, memory: 3 * MemTestUnit}}), 255 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 3 * MemTestUnit}), 256 | }, 257 | nrt: nrt, 258 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU), string(corev1.ResourceMemory)), 259 | }, 260 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 261 | }, 262 | { 263 | name: "crane agent policy is not static", 264 | args: args{ 265 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 266 | nodeInfo: framework.NewNodeInfo( 267 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 268 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 269 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 270 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 271 | ), 272 | nrt: func() *topologyv1alpha1.NodeResourceTopology { 273 | nrtCopy := nrt.DeepCopy() 274 | nrtCopy.CraneManagerPolicy.CPUManagerPolicy = topologyv1alpha1.CPUManagerPolicyNone 275 | return nrtCopy 276 | }(), 277 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 278 | }, 279 | want: nil, 280 | }, 281 | { 282 | name: "no enough cpu resource in one NUMA node with default single numa topology manager policy", 283 | args: args{ 284 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 285 | nodeInfo: framework.NewNodeInfo( 286 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 287 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 288 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 3 * CPUTestUnit}}), 289 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 290 | ), 291 | nrt: nrt, 292 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 293 | }, 294 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 295 | }, 296 | { 297 | name: "enough cpu resource in node with default none topology manager policy", 298 | args: args{ 299 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 300 | nodeInfo: framework.NewNodeInfo( 301 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 302 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 303 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 3 * CPUTestUnit}}), 304 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 305 | ), 306 | nrt: func() *topologyv1alpha1.NodeResourceTopology { 307 | nrtCopy := nrt.DeepCopy() 308 | nrtCopy.CraneManagerPolicy.TopologyManagerPolicy = topologyv1alpha1.TopologyManagerPolicyNone 309 | return nrtCopy 310 | }(), 311 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 312 | }, 313 | want: nil, 314 | }, 315 | { 316 | name: "no enough cpu resource in one NUMA node with default single numa topology manager policy", 317 | args: args{ 318 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 319 | nodeInfo: framework.NewNodeInfo( 320 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 321 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 322 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 3 * CPUTestUnit}}), 323 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 324 | ), 325 | nrt: nrt, 326 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 327 | }, 328 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 329 | }, 330 | { 331 | name: "enough cpu resource in one NUMA node with cross numa pods", 332 | args: args{ 333 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 334 | nodeInfo: framework.NewNodeInfo( 335 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 336 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 337 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}, {name: "node2", cpu: 1 * CPUTestUnit}}), 338 | framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: 1 * MemTestUnit}), 339 | ), 340 | nrt: nrt, 341 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 342 | }, 343 | want: nil, 344 | }, 345 | { 346 | name: "no enough cpu resource in one NUMA node with cross numa pods", 347 | args: args{ 348 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 349 | nodeInfo: framework.NewNodeInfo( 350 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 351 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 352 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}, {name: "node2", cpu: 2 * CPUTestUnit}}), 353 | framework.Resource{MilliCPU: 3 * CPUTestUnit, Memory: 1 * MemTestUnit}), 354 | ), 355 | nrt: nrt, 356 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 357 | }, 358 | want: framework.NewStatus(framework.Unschedulable, ErrReasonNUMAResourceNotEnough), 359 | }, 360 | } 361 | for _, tt := range tests { 362 | t.Run(tt.name, func(t *testing.T) { 363 | ctx, cancel := context.WithCancel(context.Background()) 364 | defer cancel() 365 | 366 | fakeClient := fake.NewSimpleClientset(tt.args.nrt) 367 | lister, err := initTopologyInformer(ctx, fakeClient) 368 | if err != nil { 369 | t.Fatalf("initTopologyInformer function error: %v", err) 370 | } 371 | node := corev1.Node{ 372 | ObjectMeta: metav1.ObjectMeta{Name: nodeName}, 373 | } 374 | tt.args.nodeInfo.SetNode(&node) 375 | 376 | cache := NewPodTopologyCache(ctx, 30*time.Second) 377 | for _, aps := range tt.args.assumedPods { 378 | tt.args.nodeInfo.AddPod(aps.pod) 379 | if err := cache.AssumePod(aps.pod, aps.zone); err != nil { 380 | t.Errorf("assume pod error: %v", err) 381 | } 382 | } 383 | 384 | var p framework.Plugin = &TopologyMatch{ 385 | lister: lister, 386 | PodTopologyCache: cache, 387 | topologyAwareResources: tt.args.topologyAwareResources, 388 | } 389 | cycleState := framework.NewCycleState() 390 | preFilterStatus := p.(framework.PreFilterPlugin).PreFilter(ctx, cycleState, tt.args.pod) 391 | if !preFilterStatus.IsSuccess() { 392 | t.Errorf("prefilter failed with status: %v", preFilterStatus) 393 | } 394 | gotStatus := p.(framework.FilterPlugin).Filter(ctx, cycleState, tt.args.pod, tt.args.nodeInfo) 395 | if !reflect.DeepEqual(gotStatus, tt.want) { 396 | t.Errorf("status does not match: %v, want: %v", gotStatus, tt.want) 397 | } 398 | 399 | }) 400 | } 401 | } 402 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/helper.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | "strconv" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | "k8s.io/apimachinery/pkg/api/resource" 11 | "k8s.io/apimachinery/pkg/util/sets" 12 | v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" 13 | "k8s.io/kubernetes/pkg/scheduler/framework" 14 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" 15 | 16 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 17 | ) 18 | 19 | var ( 20 | // SupportedPolicy is the valid cpu policy. 21 | SupportedPolicy = sets.NewString( 22 | topologyv1alpha1.AnnotationPodCPUPolicyNone, topologyv1alpha1.AnnotationPodCPUPolicyExclusive, 23 | topologyv1alpha1.AnnotationPodCPUPolicyNUMA, topologyv1alpha1.AnnotationPodCPUPolicyImmovable, 24 | ) 25 | ) 26 | 27 | // IsPodAwareOfTopology returns if the pod needs to be scheduled based on topology awareness. 28 | func IsPodAwareOfTopology(attr map[string]string) *bool { 29 | if val, exist := attr[topologyv1alpha1.AnnotationPodTopologyAwarenessKey]; exist { 30 | if awareness, err := strconv.ParseBool(val); err == nil { 31 | return &awareness 32 | } 33 | } 34 | return nil 35 | } 36 | 37 | // GetPodTargetContainerIndices returns all pod whose cpus could be allocated. 38 | func GetPodTargetContainerIndices(pod *corev1.Pod) []int { 39 | if policy := GetPodCPUPolicy(pod.Annotations); policy == topologyv1alpha1.AnnotationPodCPUPolicyNone { 40 | return nil 41 | } 42 | var idx []int 43 | for i := range pod.Spec.Containers { 44 | if GuaranteedCPUs(&pod.Spec.Containers[i]) > 0 { 45 | idx = append(idx, i) 46 | } 47 | } 48 | return idx 49 | } 50 | 51 | // GetPodCPUPolicy returns the cpu policy of pod, only supports none, exclusive, numa and immovable. 52 | func GetPodCPUPolicy(attr map[string]string) string { 53 | policy, ok := attr[topologyv1alpha1.AnnotationPodCPUPolicyKey] 54 | if ok && SupportedPolicy.Has(policy) { 55 | return policy 56 | } 57 | return "" 58 | } 59 | 60 | // GuaranteedCPUs returns CPUs for guaranteed container. 61 | func GuaranteedCPUs(container *corev1.Container) int { 62 | cpuQuantity := container.Resources.Requests[corev1.ResourceCPU] 63 | cpuQuantityLimit := container.Resources.Limits[corev1.ResourceCPU] 64 | 65 | // If requests.cpu != limits.cpu or cpu is not an integer, there are no guaranteed cpus. 66 | if cpuQuantity.Cmp(cpuQuantityLimit) != 0 || cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() { 67 | return 0 68 | } 69 | // Safe downcast to do for all systems with < 2.1 billion CPUs. 70 | // Per the language spec, `int` is guaranteed to be at least 32 bits wide. 71 | // https://golang.org/ref/spec#Numeric_types 72 | return int(cpuQuantity.Value()) 73 | } 74 | 75 | // GetPodTopologyResult returns the Topology scheduling result of a pod. 76 | func GetPodTopologyResult(pod *corev1.Pod) topologyv1alpha1.ZoneList { 77 | raw, exist := pod.Annotations[topologyv1alpha1.AnnotationPodTopologyResultKey] 78 | if !exist { 79 | return nil 80 | } 81 | var zones topologyv1alpha1.ZoneList 82 | if err := json.Unmarshal([]byte(raw), &zones); err != nil { 83 | return nil 84 | } 85 | return zones 86 | } 87 | 88 | // GetPodNUMANodeResult returns the NUMA node scheduling result of a pod. 89 | func GetPodNUMANodeResult(pod *corev1.Pod) topologyv1alpha1.ZoneList { 90 | zones := GetPodTopologyResult(pod) 91 | var numaZones topologyv1alpha1.ZoneList 92 | for i := range zones { 93 | if zones[i].Type == topologyv1alpha1.ZoneTypeNode { 94 | numaZones = append(numaZones, zones[i]) 95 | } 96 | } 97 | return numaZones 98 | } 99 | 100 | type getAssumedPodTopologyFunc func(pod *corev1.Pod) (topologyv1alpha1.ZoneList, error) 101 | 102 | type numaNode struct { 103 | name string 104 | allocatable *framework.Resource 105 | requested *framework.Resource 106 | } 107 | 108 | func newNumaNode(zone *topologyv1alpha1.Zone) *numaNode { 109 | var allocatable corev1.ResourceList 110 | if zone.Resources != nil { 111 | allocatable = zone.Resources.Allocatable 112 | } 113 | return &numaNode{ 114 | name: zone.Name, 115 | allocatable: framework.NewResource(allocatable), 116 | requested: &framework.Resource{}, 117 | } 118 | } 119 | 120 | func (nn *numaNode) addResource(info *topologyv1alpha1.ResourceInfo) { 121 | if info == nil { 122 | return 123 | } 124 | nn.requested.Add(info.Capacity) 125 | } 126 | 127 | type nodeWrapper struct { 128 | aware bool 129 | node string 130 | numaNodes []*numaNode 131 | getAssumedPodTopology getAssumedPodTopologyFunc 132 | // we only care about the specified resources. 133 | topologyAwareResources sets.String 134 | result topologyv1alpha1.ZoneList 135 | } 136 | 137 | func newNodeWrapper( 138 | node string, 139 | resourceNames sets.String, 140 | zones topologyv1alpha1.ZoneList, 141 | f getAssumedPodTopologyFunc, 142 | ) *nodeWrapper { 143 | nw := &nodeWrapper{node: node, getAssumedPodTopology: f, topologyAwareResources: resourceNames} 144 | for i := range zones { 145 | nw.numaNodes = append(nw.numaNodes, newNumaNode(&zones[i])) 146 | } 147 | return nw 148 | } 149 | 150 | func (nw *nodeWrapper) addPod(pod *corev1.Pod) { 151 | numaNodeResult := GetPodNUMANodeResult(pod) 152 | // If result not found, we check the assumed cache because pod may not be bound. 153 | if len(numaNodeResult) == 0 { 154 | var err error 155 | if numaNodeResult, err = nw.getAssumedPodTopology(pod); err != nil { 156 | return 157 | } 158 | } 159 | nw.addNUMAResources(numaNodeResult) 160 | } 161 | 162 | func (nw *nodeWrapper) addNUMAResources(numaNodeResult topologyv1alpha1.ZoneList) { 163 | for i := range numaNodeResult { 164 | result := &numaNodeResult[i] 165 | for _, node := range nw.numaNodes { 166 | if node.name == result.Name { 167 | node.addResource(result.Resources) 168 | } 169 | } 170 | } 171 | } 172 | 173 | func assignTopologyResult(nw *nodeWrapper, request *framework.Resource) { 174 | // sort by free CPU resource 175 | sort.Slice(nw.numaNodes, func(i, j int) bool { 176 | nodeI, nodeJ := nw.numaNodes[i], nw.numaNodes[j] 177 | return nodeI.allocatable.MilliCPU-nodeI.requested.MilliCPU > nodeJ.allocatable.MilliCPU-nodeJ.requested.MilliCPU 178 | }) 179 | 180 | if nw.aware { 181 | nw.result = []topologyv1alpha1.Zone{ 182 | { 183 | Name: nw.numaNodes[0].name, 184 | Type: topologyv1alpha1.ZoneTypeNode, 185 | Resources: &topologyv1alpha1.ResourceInfo{ 186 | Capacity: ResourceListIgnoreZeroResources(request), 187 | }, 188 | }, 189 | } 190 | return 191 | } 192 | 193 | for _, node := range nw.numaNodes { 194 | node.allocatable.MilliCPU = node.allocatable.MilliCPU / 1000 * 1000 195 | res, finished := assignRequestForNUMANode(request, node) 196 | if capacity := ResourceListIgnoreZeroResources(res); len(capacity) != 0 { 197 | nw.result = append(nw.result, topologyv1alpha1.Zone{ 198 | Name: node.name, 199 | Type: topologyv1alpha1.ZoneTypeNode, 200 | Resources: &topologyv1alpha1.ResourceInfo{ 201 | Capacity: ResourceListIgnoreZeroResources(res), 202 | }, 203 | }) 204 | } 205 | if finished { 206 | break 207 | } 208 | } 209 | sort.Slice(nw.result, func(i, j int) bool { 210 | return nw.result[i].Name < nw.result[j].Name 211 | }) 212 | } 213 | 214 | func computeContainerSpecifiedResourceRequest(pod *corev1.Pod, indices []int, names sets.String) *framework.Resource { 215 | result := &framework.Resource{} 216 | for _, idx := range indices { 217 | container := &pod.Spec.Containers[idx] 218 | resources := make(corev1.ResourceList) 219 | for resourceName := range container.Resources.Requests { 220 | if names.Has(string(resourceName)) { 221 | resources[resourceName] = container.Resources.Requests[resourceName] 222 | } 223 | } 224 | result.Add(resources) 225 | } 226 | 227 | return result 228 | } 229 | 230 | func fitsRequestForNUMANode(podRequest *framework.Resource, numaNode *numaNode) []noderesources.InsufficientResource { 231 | insufficientResources := make([]noderesources.InsufficientResource, 0, 3) 232 | allocatable := numaNode.allocatable 233 | requested := numaNode.requested 234 | if podRequest.MilliCPU == 0 && 235 | podRequest.Memory == 0 && 236 | podRequest.EphemeralStorage == 0 && 237 | len(podRequest.ScalarResources) == 0 { 238 | return insufficientResources 239 | } 240 | 241 | if podRequest.MilliCPU > (allocatable.MilliCPU - requested.MilliCPU) { 242 | insufficientResources = append(insufficientResources, noderesources.InsufficientResource{ 243 | ResourceName: corev1.ResourceCPU, 244 | Reason: "Insufficient cpu of NUMA node", 245 | Requested: podRequest.MilliCPU, 246 | Used: requested.MilliCPU, 247 | Capacity: allocatable.MilliCPU, 248 | }) 249 | } 250 | if podRequest.Memory > (allocatable.Memory - requested.Memory) { 251 | insufficientResources = append(insufficientResources, noderesources.InsufficientResource{ 252 | ResourceName: corev1.ResourceMemory, 253 | Reason: "Insufficient memory of NUMA node", 254 | Requested: podRequest.Memory, 255 | Used: requested.Memory, 256 | Capacity: allocatable.Memory, 257 | }) 258 | } 259 | if podRequest.EphemeralStorage > (allocatable.EphemeralStorage - requested.EphemeralStorage) { 260 | insufficientResources = append(insufficientResources, noderesources.InsufficientResource{ 261 | ResourceName: corev1.ResourceEphemeralStorage, 262 | Reason: "Insufficient ephemeral-storage of NUMA node", 263 | Requested: podRequest.EphemeralStorage, 264 | Used: requested.EphemeralStorage, 265 | Capacity: allocatable.EphemeralStorage, 266 | }) 267 | } 268 | 269 | for rName, rQuant := range podRequest.ScalarResources { 270 | if rQuant > (allocatable.ScalarResources[rName] - requested.ScalarResources[rName]) { 271 | insufficientResources = append(insufficientResources, noderesources.InsufficientResource{ 272 | ResourceName: rName, 273 | Reason: fmt.Sprintf("Insufficient %v of NUMA node", rName), 274 | Requested: podRequest.ScalarResources[rName], 275 | Used: requested.ScalarResources[rName], 276 | Capacity: allocatable.ScalarResources[rName], 277 | }) 278 | } 279 | } 280 | 281 | return insufficientResources 282 | } 283 | 284 | func assignRequestForNUMANode(podRequest *framework.Resource, numaNode *numaNode) (*framework.Resource, bool) { 285 | allocatable := numaNode.allocatable 286 | requested := numaNode.requested 287 | if podRequest.MilliCPU == 0 && 288 | podRequest.Memory == 0 && 289 | podRequest.EphemeralStorage == 0 && 290 | len(podRequest.ScalarResources) == 0 { 291 | return nil, false 292 | } 293 | 294 | res := &framework.Resource{} 295 | finished := true 296 | 297 | assigned := min(podRequest.MilliCPU, allocatable.MilliCPU-requested.MilliCPU) 298 | podRequest.MilliCPU -= assigned 299 | res.MilliCPU = assigned 300 | if podRequest.MilliCPU > 0 { 301 | finished = false 302 | } 303 | 304 | assigned = min(podRequest.Memory, allocatable.Memory-requested.Memory) 305 | podRequest.Memory -= assigned 306 | res.Memory = assigned 307 | if podRequest.Memory > 0 { 308 | finished = false 309 | } 310 | 311 | assigned = min(podRequest.EphemeralStorage, allocatable.EphemeralStorage-requested.EphemeralStorage) 312 | podRequest.EphemeralStorage -= assigned 313 | res.EphemeralStorage = assigned 314 | if podRequest.EphemeralStorage > 0 { 315 | finished = false 316 | } 317 | 318 | for rName, rQuant := range podRequest.ScalarResources { 319 | assigned = min(rQuant, allocatable.ScalarResources[rName]-requested.ScalarResources[rName]) 320 | podRequest.ScalarResources[rName] -= assigned 321 | res.ScalarResources[rName] = assigned 322 | if podRequest.ScalarResources[rName] > 0 { 323 | finished = false 324 | } 325 | } 326 | 327 | return res, finished 328 | } 329 | 330 | // ResourceListIgnoreZeroResources returns non-zero ResourceList from a framework.Resource. 331 | func ResourceListIgnoreZeroResources(r *framework.Resource) corev1.ResourceList { 332 | if r == nil { 333 | return nil 334 | } 335 | result := make(corev1.ResourceList) 336 | if r.MilliCPU > 0 { 337 | result[corev1.ResourceCPU] = *resource.NewMilliQuantity(r.MilliCPU, resource.DecimalSI) 338 | } 339 | if r.Memory > 0 { 340 | result[corev1.ResourceMemory] = *resource.NewQuantity(r.MilliCPU, resource.BinarySI) 341 | } 342 | if r.AllowedPodNumber > 0 { 343 | result[corev1.ResourcePods] = *resource.NewQuantity(int64(r.AllowedPodNumber), resource.BinarySI) 344 | } 345 | if r.EphemeralStorage > 0 { 346 | result[corev1.ResourceEphemeralStorage] = *resource.NewQuantity(r.EphemeralStorage, resource.BinarySI) 347 | } 348 | for rName, rQuant := range r.ScalarResources { 349 | if rQuant > 0 { 350 | if v1helper.IsHugePageResourceName(rName) { 351 | result[rName] = *resource.NewQuantity(rQuant, resource.BinarySI) 352 | } else { 353 | result[rName] = *resource.NewQuantity(rQuant, resource.DecimalSI) 354 | } 355 | } 356 | } 357 | return result 358 | } 359 | 360 | func min(a, b int64) int64 { 361 | if a < b { 362 | return a 363 | } 364 | return b 365 | } 366 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/plugin.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/apimachinery/pkg/util/sets" 11 | "k8s.io/klog/v2" 12 | "k8s.io/kubernetes/pkg/scheduler/framework" 13 | 14 | topologyclientset "github.com/gocrane/api/pkg/generated/clientset/versioned" 15 | informers "github.com/gocrane/api/pkg/generated/informers/externalversions" 16 | listerv1alpha1 "github.com/gocrane/api/pkg/generated/listers/topology/v1alpha1" 17 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 18 | 19 | "github.com/gocrane/crane-scheduler/pkg/plugins/apis/config" 20 | ) 21 | 22 | const ( 23 | // Name is the name of the plugin used in the plugin registry and configurations. 24 | Name = "NodeResourceTopologyMatch" 25 | 26 | // stateKey is the key in CycleState to NodeResourcesTopology. 27 | stateKey framework.StateKey = Name 28 | ) 29 | 30 | // New initializes a new plugin and returns it. 31 | func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error) { 32 | klog.V(2).InfoS("Creating new TopologyMatch plugin") 33 | cfg, ok := args.(*config.NodeResourceTopologyMatchArgs) 34 | if !ok { 35 | return nil, fmt.Errorf("want args to be of type NodeResourceTopologyMatchArgs, got %T", args) 36 | } 37 | 38 | ctx := context.TODO() 39 | client, err := topologyclientset.NewForConfig(handle.KubeConfig()) 40 | if err != nil { 41 | klog.ErrorS(err, "Failed to create clientSet for NodeTopologyResource", "kubeConfig", handle.KubeConfig()) 42 | return nil, err 43 | } 44 | 45 | lister, err := initTopologyInformer(ctx, client) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | topologyMatch := &TopologyMatch{ 51 | PodTopologyCache: NewPodTopologyCache(ctx, 30*time.Minute), 52 | handle: handle, 53 | lister: lister, 54 | topologyAwareResources: sets.NewString(cfg.TopologyAwareResources...), 55 | } 56 | 57 | return topologyMatch, nil 58 | } 59 | 60 | func initTopologyInformer( 61 | ctx context.Context, 62 | client topologyclientset.Interface, 63 | ) (listerv1alpha1.NodeResourceTopologyLister, error) { 64 | topologyInformerFactory := informers.NewSharedInformerFactory(client, 0) 65 | nrtLister := topologyInformerFactory.Topology().V1alpha1().NodeResourceTopologies().Lister() 66 | 67 | klog.V(4).InfoS("Start nodeTopologyInformer") 68 | topologyInformerFactory.Start(ctx.Done()) 69 | topologyInformerFactory.WaitForCacheSync(ctx.Done()) 70 | return nrtLister, nil 71 | } 72 | 73 | var _ framework.PreFilterPlugin = &TopologyMatch{} 74 | var _ framework.FilterPlugin = &TopologyMatch{} 75 | var _ framework.ScorePlugin = &TopologyMatch{} 76 | var _ framework.ReservePlugin = &TopologyMatch{} 77 | var _ framework.PreBindPlugin = &TopologyMatch{} 78 | 79 | // TopologyMatch plugin which run simplified version of TopologyManager's admit handler 80 | type TopologyMatch struct { 81 | PodTopologyCache 82 | handle framework.Handle 83 | lister listerv1alpha1.NodeResourceTopologyLister 84 | topologyAwareResources sets.String 85 | } 86 | 87 | // Name returns name of the plugin. It is used in logs, etc. 88 | func (tm *TopologyMatch) Name() string { 89 | return Name 90 | } 91 | 92 | // stateData computed at PreFilter and used at Filter. 93 | type stateData struct { 94 | sync.Mutex 95 | 96 | aware *bool 97 | // If not empty, there are containers need to be bound. 98 | targetContainerIndices []int 99 | targetContainerResource *framework.Resource 100 | // all available NUMA node will be recorded into this map 101 | podTopologyByNode map[string]*nodeWrapper 102 | 103 | topologyResult topologyv1alpha1.ZoneList 104 | } 105 | 106 | // Clone the prefilter stateData. 107 | func (s *stateData) Clone() framework.StateData { 108 | return s 109 | } 110 | 111 | func getStateData(state *framework.CycleState) (*stateData, error) { 112 | c, err := state.Read(stateKey) 113 | if err != nil { 114 | return nil, fmt.Errorf("failed to read %q from cycleState: %w", stateKey, err) 115 | } 116 | 117 | s, ok := c.(*stateData) 118 | if !ok { 119 | return nil, fmt.Errorf("%+v convert to NodeResourcesTopology.stateData error", c) 120 | } 121 | return s, nil 122 | } 123 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/reserver.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/kubernetes/pkg/scheduler/framework" 8 | ) 9 | 10 | // Reserve reserves topology of pod and saves status in cycle stateData. 11 | func (tm *TopologyMatch) Reserve( 12 | ctx context.Context, 13 | state *framework.CycleState, 14 | pod *corev1.Pod, 15 | nodeName string, 16 | ) *framework.Status { 17 | s, err := getStateData(state) 18 | if err != nil { 19 | return framework.AsStatus(err) 20 | } 21 | nw, exist := s.podTopologyByNode[nodeName] 22 | if !exist { 23 | return nil 24 | } 25 | if len(nw.result) == 0 { 26 | // Should never happen 27 | return framework.NewStatus(framework.Error, "node(s) topology result is empty") 28 | } 29 | s.topologyResult = nw.result 30 | // Assume pod 31 | if err = tm.AssumePod(pod, s.topologyResult); err != nil { 32 | return framework.AsStatus(err) 33 | } 34 | return nil 35 | } 36 | 37 | // Unreserve clears assumed Pod topology cache. 38 | // It's idempotent, and does nothing if no cache found for the given pod. 39 | func (tm *TopologyMatch) Unreserve(ctx context.Context, state *framework.CycleState, pod *corev1.Pod, nodeName string) { 40 | s, err := getStateData(state) 41 | if err != nil { 42 | return 43 | } 44 | _, exist := s.podTopologyByNode[nodeName] 45 | if !exist { 46 | return 47 | } 48 | if err = tm.ForgetPod(pod); err != nil { 49 | return 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/scorer.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | "k8s.io/kubernetes/pkg/scheduler/framework" 8 | ) 9 | 10 | // Score invoked at the Score extension point. 11 | func (tm *TopologyMatch) Score( 12 | ctx context.Context, 13 | state *framework.CycleState, 14 | pod *corev1.Pod, 15 | nodeName string, 16 | ) (int64, *framework.Status) { 17 | s, err := getStateData(state) 18 | if err != nil { 19 | return 0, framework.AsStatus(err) 20 | } 21 | 22 | nw, exist := s.podTopologyByNode[nodeName] 23 | if !exist { 24 | return 0, nil 25 | } 26 | 27 | // TODO(Garrybest): make this plugin as configurable. 28 | return framework.MaxNodeScore / int64(len(nw.result)), nil 29 | } 30 | 31 | // ScoreExtensions of the Score plugin. 32 | func (tm *TopologyMatch) ScoreExtensions() framework.ScoreExtensions { 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/plugins/noderesourcetopology/scorer_test.go: -------------------------------------------------------------------------------- 1 | package noderesourcetopology 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/util/sets" 12 | "k8s.io/kubernetes/pkg/scheduler/framework" 13 | 14 | "github.com/gocrane/api/pkg/generated/clientset/versioned/fake" 15 | topologyv1alpha1 "github.com/gocrane/api/topology/v1alpha1" 16 | ) 17 | 18 | func TestTopologyMatch_Score(t *testing.T) { 19 | type args struct { 20 | pod *corev1.Pod 21 | nodeInfo *framework.NodeInfo 22 | nrt *topologyv1alpha1.NodeResourceTopology 23 | assumedPods []*assumedPod 24 | topologyAwareResources sets.String 25 | } 26 | type res struct { 27 | score int64 28 | status *framework.Status 29 | } 30 | tests := []struct { 31 | name string 32 | args args 33 | want res 34 | }{ 35 | { 36 | name: "enough resource of node1 and node2", 37 | args: args{ 38 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 39 | nodeInfo: framework.NewNodeInfo( 40 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 41 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 42 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 43 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 44 | ), 45 | nrt: nrt, 46 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 47 | }, 48 | want: res{ 49 | score: 100, 50 | status: nil, 51 | }, 52 | }, 53 | { 54 | name: "enough resource of node1 and node2 with assumed pods", 55 | args: args{ 56 | pod: newResourcePod(true, nil, framework.Resource{MilliCPU: CPUTestUnit, Memory: MemTestUnit}), 57 | nodeInfo: framework.NewNodeInfo(), 58 | nrt: nrt, 59 | assumedPods: []*assumedPod{ 60 | newResourceAssumedPod(newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}}), 61 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 2 * MemTestUnit}), 62 | newResourceAssumedPod(newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 63 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 64 | }, 65 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 66 | }, 67 | want: res{ 68 | score: 100, 69 | status: nil, 70 | }, 71 | }, 72 | { 73 | name: "enough cpu resource in one NUMA node with cross numa pods", 74 | args: args{ 75 | pod: newResourcePod(false, nil, framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: MemTestUnit}), 76 | nodeInfo: framework.NewNodeInfo( 77 | newResourcePod(true, newZoneList([]zone{{name: "node1", cpu: 1 * CPUTestUnit}, {name: "node2", cpu: 1 * CPUTestUnit}}), 78 | framework.Resource{MilliCPU: 2 * CPUTestUnit, Memory: 2 * MemTestUnit}), 79 | newResourcePod(true, newZoneList([]zone{{name: "node2", cpu: 1 * CPUTestUnit}}), 80 | framework.Resource{MilliCPU: 1 * CPUTestUnit, Memory: 1 * MemTestUnit}), 81 | ), 82 | nrt: func() *topologyv1alpha1.NodeResourceTopology { 83 | nrtCopy := nrt.DeepCopy() 84 | nrtCopy.CraneManagerPolicy.TopologyManagerPolicy = topologyv1alpha1.TopologyManagerPolicyNone 85 | return nrtCopy 86 | }(), 87 | topologyAwareResources: sets.NewString(string(corev1.ResourceCPU)), 88 | }, 89 | want: res{ 90 | score: 50, 91 | status: nil, 92 | }, 93 | }, 94 | } 95 | for _, tt := range tests { 96 | t.Run(tt.name, func(t *testing.T) { 97 | ctx, cancel := context.WithCancel(context.Background()) 98 | defer cancel() 99 | 100 | fakeClient := fake.NewSimpleClientset(tt.args.nrt) 101 | lister, err := initTopologyInformer(ctx, fakeClient) 102 | if err != nil { 103 | t.Fatalf("initTopologyInformer function error: %v", err) 104 | } 105 | node := corev1.Node{ 106 | ObjectMeta: metav1.ObjectMeta{Name: nodeName}, 107 | } 108 | tt.args.nodeInfo.SetNode(&node) 109 | 110 | cache := NewPodTopologyCache(ctx, 30*time.Second) 111 | for _, aps := range tt.args.assumedPods { 112 | tt.args.nodeInfo.AddPod(aps.pod) 113 | if err := cache.AssumePod(aps.pod, aps.zone); err != nil { 114 | t.Errorf("assume pod error: %v", err) 115 | } 116 | } 117 | 118 | var p framework.Plugin = &TopologyMatch{ 119 | lister: lister, 120 | PodTopologyCache: cache, 121 | topologyAwareResources: tt.args.topologyAwareResources, 122 | } 123 | cycleState := framework.NewCycleState() 124 | preFilterStatus := p.(framework.PreFilterPlugin).PreFilter(ctx, cycleState, tt.args.pod) 125 | if !preFilterStatus.IsSuccess() { 126 | t.Errorf("prefilter failed with status: %v", preFilterStatus) 127 | } 128 | filterStatus := p.(framework.FilterPlugin).Filter(ctx, cycleState, tt.args.pod, tt.args.nodeInfo) 129 | if !filterStatus.IsSuccess() { 130 | t.Errorf("filter failed with status: %v", preFilterStatus) 131 | } 132 | score, gotStatus := p.(framework.ScorePlugin).Score(ctx, cycleState, tt.args.pod, tt.args.nodeInfo.Node().Name) 133 | if !reflect.DeepEqual(res{score, gotStatus}, tt.want) { 134 | t.Errorf("status does not match: %v, want: %v", gotStatus, tt.want) 135 | } 136 | }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | corev1 "k8s.io/api/core/v1" 8 | ) 9 | 10 | const ( 11 | TimeFormat = "2006-01-02T15:04:05Z" 12 | DefaultTimeZone = "Asia/Shanghai" 13 | DefaultNamespace = "crane-system" 14 | ) 15 | 16 | // IsDaemonsetPod judges if this pod belongs to one daemonset workload. 17 | func IsDaemonsetPod(pod *corev1.Pod) bool { 18 | for _, ownerRef := range pod.GetOwnerReferences() { 19 | if ownerRef.Kind == "DaemonSet" { 20 | return true 21 | } 22 | } 23 | return false 24 | } 25 | 26 | func GetLocalTime() string { 27 | loc := GetLocation() 28 | if loc == nil { 29 | time.Now().Format(TimeFormat) 30 | } 31 | 32 | return time.Now().In(loc).Format(TimeFormat) 33 | } 34 | 35 | func GetLocation() *time.Location { 36 | zone := os.Getenv("TZ") 37 | 38 | if zone == "" { 39 | zone = DefaultTimeZone 40 | } 41 | 42 | loc, _ := time.LoadLocation(zone) 43 | 44 | return loc 45 | } 46 | 47 | func GetSystemNamespace() string { 48 | ns := os.Getenv("CRANE_SYSTEM_NAMESPACE") 49 | 50 | if ns == "" { 51 | ns = DefaultNamespace 52 | } 53 | 54 | return ns 55 | } 56 | 57 | // NormalizaScore nornalize the score in range [min, max] 58 | func NormalizeScore(value, max, min int64) int64 { 59 | if value < min { 60 | value = min 61 | } 62 | 63 | if value > max { 64 | value = max 65 | } 66 | 67 | return value 68 | } 69 | --------------------------------------------------------------------------------