├── manifests └── toposcheduling │ ├── topopolicy-example.yaml │ ├── crd.yaml │ └── scheduler-config.yaml ├── .gitignore ├── hack ├── boilerplate │ └── boilerplate.generatego.txt ├── update-vendor.sh ├── lib │ ├── init.sh │ ├── golang.sh │ ├── logging.sh │ ├── util.sh │ └── etcd.sh ├── tools.go ├── unit-test.sh ├── install-etcd.sh ├── update-gofmt.sh ├── update-codegen.sh ├── update-generated-openapi.sh ├── integration-test.sh └── verify-gofmt.sh ├── pkg ├── apis │ ├── scheduling │ │ ├── register.go │ │ └── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ └── zz_generated.deepcopy.go │ └── config │ │ ├── doc.go │ │ ├── v1beta1 │ │ ├── doc.go │ │ ├── zz_generated.defaults.go │ │ ├── types.go │ │ ├── register.go │ │ ├── zz_generated.deepcopy.go │ │ └── zz_generated.conversion.go │ │ ├── types.go │ │ ├── scheme │ │ └── scheme.go │ │ ├── register.go │ │ └── zz_generated.deepcopy.go ├── generated │ ├── clientset │ │ └── versioned │ │ │ ├── doc.go │ │ │ ├── fake │ │ │ ├── doc.go │ │ │ ├── register.go │ │ │ └── clientset_generated.go │ │ │ ├── scheme │ │ │ ├── doc.go │ │ │ └── register.go │ │ │ ├── typed │ │ │ └── scheduling │ │ │ │ └── v1alpha1 │ │ │ │ ├── fake │ │ │ │ ├── doc.go │ │ │ │ ├── fake_scheduling_client.go │ │ │ │ └── fake_topologyschedulingpolicy.go │ │ │ │ ├── doc.go │ │ │ │ ├── generated_expansion.go │ │ │ │ ├── scheduling_client.go │ │ │ │ └── topologyschedulingpolicy.go │ │ │ └── clientset.go │ ├── listers │ │ └── scheduling │ │ │ └── v1alpha1 │ │ │ ├── expansion_generated.go │ │ │ └── topologyschedulingpolicy.go │ └── informers │ │ └── externalversions │ │ ├── internalinterfaces │ │ └── factory_interfaces.go │ │ ├── scheduling │ │ ├── interface.go │ │ └── v1alpha1 │ │ │ ├── interface.go │ │ │ └── topologyschedulingpolicy.go │ │ ├── generic.go │ │ └── factory.go ├── common │ └── constraint.go ├── util │ ├── constants.go │ ├── parallelism.go │ └── utils.go ├── multicluster │ ├── scheduler │ │ ├── generic.go │ │ └── scheduler.go │ └── config │ │ └── config.go ├── descheduler │ ├── strategies │ │ └── topologyscheduling.go │ └── descheduler.go └── topologyscheduling │ └── topology_scheduling.go ├── cmd ├── descheduler │ ├── descheduler.go │ └── app │ │ ├── server.go │ │ ├── version.go │ │ └── options │ │ └── options.go └── scheduler │ └── main.go ├── go.mod ├── Makefile ├── README.md └── LICENSE /manifests/toposcheduling/topopolicy-example.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.sigs.k8s.io/v1alpha1 2 | kind: TopologySchedulingPolicy 3 | metadata: 4 | name: policy-test 5 | namespace: default 6 | spec: 7 | deployPlacement: 8 | - name: test1 9 | replicas: 6 10 | - name: test2 11 | replicas: 3 12 | labelSelector: 13 | matchLabels: 14 | cluster-test: "true" 15 | topologyKey: cluster-test -------------------------------------------------------------------------------- /manifests/toposcheduling/crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: topologyschedulingpolicies.scheduling.sigs.k8s.io 5 | spec: 6 | conversion: 7 | strategy: None 8 | group: scheduling.sigs.k8s.io 9 | names: 10 | kind: TopologySchedulingPolicy 11 | listKind: TopologySchedulingPolicyList 12 | plural: topologyschedulingpolicies 13 | shortNames: 14 | - tsp 15 | - tsps 16 | singular: topologyschedulingpolicy 17 | scope: Namespaced 18 | version: v1alpha1 19 | versions: 20 | - name: v1alpha1 21 | served: true 22 | storage: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | # build and test outputs 17 | /bin/ 18 | /_output/ 19 | /_artifacts/ 20 | /build/kube-scheduler 21 | 22 | # used for the code generators only 23 | /vendor/ 24 | 25 | # macOS 26 | .DS_Store 27 | 28 | # files generated by editors 29 | .idea/ 30 | *.iml 31 | .vscode/ 32 | *.swp 33 | *.sublime-project 34 | *.sublime-workspace 35 | *~ -------------------------------------------------------------------------------- /hack/boilerplate/boilerplate.generatego.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | -------------------------------------------------------------------------------- /pkg/apis/scheduling/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package scheduling 18 | 19 | // GroupName is the group name used in this package 20 | const ( 21 | GroupName = "scheduling.sigs.k8s.io" 22 | ) 23 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated clientset. 20 | package versioned 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated fake clientset. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/apis/config/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +groupName=kubescheduler.config.k8s.io 19 | 20 | package config // import "github.com/cwdsuzhou/super-scheduling/pkg/apis/config" 21 | -------------------------------------------------------------------------------- /pkg/apis/scheduling/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +groupName=scheduling.sigs.k8s.io 19 | 20 | // Package v1alpha1 is the v1alpha1 version of the API. 21 | package v1alpha1 22 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package contains the scheme of the automatically generated clientset. 20 | package scheme 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // Package fake has the automatically generated clients. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated typed clients. 20 | package v1alpha1 21 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | type TopologySchedulingPolicyExpansion interface{} 22 | -------------------------------------------------------------------------------- /manifests/toposcheduling/scheduler-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubescheduler.config.k8s.io/v1beta1 2 | kind: KubeSchedulerConfiguration 3 | leaderElection: 4 | leaderElect: true 5 | clientConnection: 6 | kubeconfig: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 7 | profiles: 8 | - schedulerName: default-multicluster 9 | plugins: 10 | preFilter: 11 | enabled: 12 | - name: TopologyScheduling 13 | filter: 14 | enabled: 15 | - name: TopologyScheduling 16 | - name: MultiClusterScheduling 17 | disabled: 18 | - name: "*" 19 | score: 20 | enabled: 21 | - name: TopologyScheduling 22 | disabled: 23 | - name: "*" 24 | reserve: 25 | enabled: 26 | - name: TopologyScheduling 27 | pluginConfig: 28 | - name: TopologyScheduling 29 | args: 30 | kubeConfigPath: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 31 | -------------------------------------------------------------------------------- /hack/update-vendor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCIPRT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 22 | source "${SCIPRT_ROOT}/hack/lib/init.sh" 23 | 24 | kube::golang::verify_go_version 25 | 26 | go mod tidy 27 | go mod vendor -------------------------------------------------------------------------------- /hack/lib/init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | unset CDPATH 18 | 19 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/../.. 20 | 21 | source "${SCRIPT_ROOT}/hack/lib/logging.sh" 22 | source "${SCRIPT_ROOT}/hack/lib/etcd.sh" 23 | source "${SCRIPT_ROOT}/hack/lib/util.sh" 24 | source "${SCRIPT_ROOT}/hack/lib/golang.sh" -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | /* 4 | Copyright 2019 The Kubernetes Authors. 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | // Copied from https://github.com/kubernetes/sample-controller/blob/master/hack/tools.go 17 | 18 | // This package imports things required by build scripts, to force `go mod` to see them as dependencies 19 | package tools 20 | 21 | import ( 22 | _ "k8s.io/code-generator" 23 | _ "k8s.io/kube-openapi/cmd/openapi-gen" 24 | ) 25 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // +k8s:deepcopy-gen=package 18 | // +k8s:conversion-gen=github.com/cwdsuzhou/super-scheduling/pkg/apis/config 19 | // +k8s:defaulter-gen=TypeMeta 20 | // +groupName=kubescheduler.config.k8s.io 21 | 22 | // Package v1beta1 is the v1beta1 version of the API. 23 | package v1beta1 // import "github.com/cwdsuzhou/super-scheduling/pkg/apis/config/v1beta1" 24 | -------------------------------------------------------------------------------- /hack/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. 22 | source "${SCRIPT_ROOT}/hack/lib/init.sh" 23 | 24 | # TODO: make args customizable. 25 | go test -mod=vendor \ 26 | github.com/cwdsuzhou/super-scheduling/cmd/... \ 27 | github.com/cwdsuzhou/super-scheduling/pkg/... 28 | -------------------------------------------------------------------------------- /hack/install-etcd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2014 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script is convenience to download and install etcd in third_party. 18 | # Mostly just used by CI. 19 | # Usage: `hack/install-etcd.sh`. 20 | 21 | set -o errexit 22 | set -o nounset 23 | set -o pipefail 24 | 25 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 26 | source "${SCRIPT_ROOT}/hack/lib/init.sh" 27 | 28 | kube::etcd::install 29 | -------------------------------------------------------------------------------- /cmd/descheduler/descheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "k8s.io/component-base/logs" 24 | 25 | "github.com/cwdsuzhou/super-scheduling/cmd/descheduler/app" 26 | ) 27 | 28 | func main() { 29 | out := os.Stdout 30 | cmd := app.NewDeschedulerCommand(out) 31 | cmd.AddCommand(app.NewVersionCommand()) 32 | 33 | logs.InitLogs() 34 | defer logs.FlushLogs() 35 | 36 | if err := cmd.Execute(); err != nil { 37 | fmt.Println(err) 38 | os.Exit(1) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by defaulter-gen. DO NOT EDIT. 20 | 21 | package v1beta1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // RegisterDefaults adds defaulters functions to the given scheme. 28 | // Public to allow building arbitrary schemes. 29 | // All generated defaulters are covering - they call all nested defaulters. 30 | func RegisterDefaults(scheme *runtime.Scheme) error { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /pkg/generated/listers/scheduling/v1alpha1/expansion_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | // TopologySchedulingPolicyListerExpansion allows custom methods to be added to 22 | // TopologySchedulingPolicyLister. 23 | type TopologySchedulingPolicyListerExpansion interface{} 24 | 25 | // TopologySchedulingPolicyNamespaceListerExpansion allows custom methods to be added to 26 | // TopologySchedulingPolicyNamespaceLister. 27 | type TopologySchedulingPolicyNamespaceListerExpansion interface{} 28 | -------------------------------------------------------------------------------- /pkg/common/constraint.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import "k8s.io/apimachinery/pkg/labels" 20 | 21 | // TopologySchedulingConstraint is an internal version for v1.TopologySchedulingPolicy 22 | // and where the selector is parsed. 23 | // Fields are exported for comparison during testing. 24 | type TopologySchedulingConstraint struct { 25 | TopologyKey string 26 | SchedulePolicy map[string]int32 27 | Selector labels.Selector 28 | } 29 | 30 | // TopologyPair stores the Key Value info 31 | type TopologyPair struct { 32 | Key string 33 | Value string 34 | } 35 | -------------------------------------------------------------------------------- /pkg/util/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import "fmt" 20 | 21 | const ( 22 | TopologySchedulingLabelKey = "topology-scheduling-policy.scheduling.sigs.k8s.io" 23 | ) 24 | 25 | var ( 26 | // ErrorNotMatched means pod does not match coscheduling 27 | ErrorNotMatched = fmt.Errorf("not match coscheduling") 28 | // ErrorWaiting means pod number does not match the min pods required 29 | ErrorWaiting = fmt.Errorf("waiting") 30 | // ErrorResourceNotEnough means cluster resource is not enough, mainly used in Pre-Filter 31 | ErrorResourceNotEnough = fmt.Errorf("resource not enough") 32 | ) 33 | -------------------------------------------------------------------------------- /hack/update-gofmt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCIPRT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 22 | source "${SCIPRT_ROOT}/hack/lib/init.sh" 23 | 24 | kube::golang::verify_go_version 25 | 26 | cd "${SCIPRT_ROOT}" 27 | 28 | find_files() { 29 | find . -not \( \ 30 | \( \ 31 | -wholename './output' \ 32 | -o -wholename './.git' \ 33 | -o -wholename './_output' \ 34 | -o -wholename './release' \ 35 | -o -wholename './target' \ 36 | -o -wholename '*/third_party/*' \ 37 | -o -wholename '*/vendor/*' \ 38 | -o -wholename './staging/src/k8s.io/client-go/*vendor/*' \ 39 | \) -prune \ 40 | \) -name '*.go' 41 | } 42 | 43 | find_files | xargs gofmt -s -w 44 | -------------------------------------------------------------------------------- /pkg/util/parallelism.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "context" 21 | "math" 22 | 23 | "k8s.io/client-go/util/workqueue" 24 | ) 25 | 26 | const parallelism = 16 27 | 28 | // chunkSizeFor returns a chunk size for the given number of items to use for 29 | // parallel work. The size aims to produce good CPU utilization. 30 | func chunkSizeFor(n int) workqueue.Options { 31 | s := int(math.Sqrt(float64(n))) 32 | if r := n/parallelism + 1; s > r { 33 | s = r 34 | } else if s < 1 { 35 | s = 1 36 | } 37 | return workqueue.WithChunkSize(s) 38 | } 39 | 40 | // Until is a wrapper around workqueue.ParallelizeUntil to use in scheduling algorithms. 41 | func Until(ctx context.Context, pieces int, doWorkPiece workqueue.DoWorkPieceFunc) { 42 | workqueue.ParallelizeUntil(ctx, parallelism, pieces, doWorkPiece, chunkSizeFor(pieces)) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/fake/fake_scheduling_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/typed/scheduling/v1alpha1" 23 | rest "k8s.io/client-go/rest" 24 | testing "k8s.io/client-go/testing" 25 | ) 26 | 27 | type FakeSchedulingV1alpha1 struct { 28 | *testing.Fake 29 | } 30 | 31 | func (c *FakeSchedulingV1alpha1) TopologySchedulingPolicies(namespace string) v1alpha1.TopologySchedulingPolicyInterface { 32 | return &FakeTopologySchedulingPolicies{c, namespace} 33 | } 34 | 35 | // RESTClient returns a RESTClient that is used to communicate 36 | // with API server by this client implementation. 37 | func (c *FakeSchedulingV1alpha1) RESTClient() rest.Interface { 38 | var ret *rest.RESTClient 39 | return ret 40 | } 41 | -------------------------------------------------------------------------------- /pkg/apis/config/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 24 | 25 | // SchedulingArgs defines the scheduling parameters for CapacityScheduling plugin. 26 | type SchedulingArgs struct { 27 | metav1.TypeMeta 28 | 29 | // KubeConfigPath is the path of kubeconfig. 30 | KubeConfigPath string 31 | 32 | // KubeMaster is the url of kubernetes master. 33 | KubeMaster string 34 | 35 | // ClusterConfiguration is a key-value map to store configuration 36 | ClusterConfiguration map[string]Configuration 37 | } 38 | 39 | // Configuration defines the lower cluster configuration 40 | type Configuration struct { 41 | // clusterName 42 | Name string 43 | // Master URL 44 | KubeMaster string 45 | // KubeConfig of the cluster 46 | KubeConfig string 47 | } 48 | -------------------------------------------------------------------------------- /hack/lib/golang.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Ensure the go tool exists and is a viable version. 18 | kube::golang::verify_go_version() { 19 | if [[ -z "$(command -v go)" ]]; then 20 | kube::log::usage_from_stdin </dev/null || echo ../code-generator)} 23 | 24 | bash "${CODEGEN_PKG}"/generate-internal-groups.sh \ 25 | "deepcopy,defaulter,conversion" \ 26 | github.com/cwdsuzhou/super-scheduling/pkg/generated \ 27 | github.com/cwdsuzhou/super-scheduling/pkg/apis \ 28 | github.com/cwdsuzhou/super-scheduling/pkg/apis \ 29 | "config:v1beta1" \ 30 | --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate/boilerplate.generatego.txt 31 | 32 | bash "${CODEGEN_PKG}"/generate-groups.sh \ 33 | all \ 34 | github.com/cwdsuzhou/super-scheduling/pkg/generated \ 35 | github.com/cwdsuzhou/super-scheduling/pkg/apis \ 36 | "scheduling:v1alpha1" \ 37 | --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate/boilerplate.generatego.txt 38 | -------------------------------------------------------------------------------- /pkg/apis/config/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package scheme 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "k8s.io/apimachinery/pkg/runtime/serializer" 22 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 23 | kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" 24 | 25 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/config" 26 | configv1beta1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/config/v1beta1" 27 | ) 28 | 29 | var ( 30 | // Re-use the in-tree Scheme. 31 | Scheme = kubeschedulerscheme.Scheme 32 | 33 | // Codecs provides access to encoding and decoding for the scheme. 34 | Codecs = serializer.NewCodecFactory(Scheme, serializer.EnableStrict) 35 | ) 36 | 37 | func init() { 38 | AddToScheme(Scheme) 39 | } 40 | 41 | // AddToScheme builds the kubescheduler scheme using all known versions of the kubescheduler api. 42 | func AddToScheme(scheme *runtime.Scheme) { 43 | utilruntime.Must(config.AddToScheme(scheme)) 44 | utilruntime.Must(configv1beta1.AddToScheme(scheme)) 45 | } 46 | -------------------------------------------------------------------------------- /hack/lib/logging.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | kube::log::status() { 18 | local timestamp=$(date +"[%m%d %H:%M:%S]") 19 | echo "+++ ${timestamp} ${1}" 20 | shift 21 | for message; do 22 | echo " ${message}" 23 | done 24 | } 25 | 26 | kube::log::info() { 27 | for message; do 28 | echo "${message}" 29 | done 30 | } 31 | 32 | # Log an error but keep going. Don't dump the stack or exit. 33 | kube::log::error() { 34 | timestamp=$(date +"[%m%d %H:%M:%S]") 35 | echo "!!! ${timestamp} ${1-}" >&2 36 | shift 37 | for message; do 38 | echo " ${message}" >&2 39 | done 40 | } 41 | 42 | # Print an usage message to stderr. The arguments are printed directly. 43 | kube::log::usage() { 44 | echo >&2 45 | local message 46 | for message; do 47 | echo "${message}" >&2 48 | done 49 | echo >&2 50 | } 51 | 52 | kube::log::usage_from_stdin() { 53 | local messages=() 54 | while read -r line; do 55 | messages+=("${line}") 56 | done 57 | 58 | kube::log::usage "${messages[@]}" 59 | } -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package internalinterfaces 20 | 21 | import ( 22 | time "time" 23 | 24 | versioned "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | runtime "k8s.io/apimachinery/pkg/runtime" 27 | cache "k8s.io/client-go/tools/cache" 28 | ) 29 | 30 | // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. 31 | type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer 32 | 33 | // SharedInformerFactory a small interface to allow for adding an informer without an import cycle 34 | type SharedInformerFactory interface { 35 | Start(stopCh <-chan struct{}) 36 | InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer 37 | } 38 | 39 | // TweakListOptionsFunc is a function that transforms a v1.ListOptions. 40 | type TweakListOptionsFunc func(*v1.ListOptions) 41 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 24 | 25 | // SchedulingArgs defines the scheduling parameters for CapacityScheduling plugin. 26 | type SchedulingArgs struct { 27 | metav1.TypeMeta `json:",inline"` 28 | 29 | // KubeConfigPath is the path of kubeconfig. 30 | KubeConfigPath *string `json:"kubeConfigPath,omitempty"` 31 | 32 | // KubeMaster is the url of kubernetes master. 33 | KubeMaster *string `json:"kubeMaster,omitempty"` 34 | 35 | // ClusterConfiguration is a key-value map to store configuration 36 | ClusterConfiguration map[string]Configuration `json:"clusterConfiguration"` 37 | } 38 | 39 | // Configuration defines the lower cluster configuration 40 | type Configuration struct { 41 | // clusterName 42 | Name *string `json:"name"` 43 | // Master URL 44 | KubeMaster *string `json:"kubeMaster,omitempty"` 45 | // KubeConfig of the cluster 46 | KubeConfig *string `json:"kubeConfig,omitempty"` 47 | } 48 | -------------------------------------------------------------------------------- /pkg/apis/config/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "k8s.io/apimachinery/pkg/runtime/schema" 22 | schedscheme "k8s.io/kubernetes/pkg/scheduler/apis/config" 23 | ) 24 | 25 | // GroupName is the group name used in this package 26 | const GroupName = "kubescheduler.config.k8s.io" 27 | 28 | // SchemeGroupVersion is group version used to register these objects 29 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} 30 | 31 | var ( 32 | localSchemeBuilder = &schedscheme.SchemeBuilder 33 | // AddToScheme is a global function that registers this API group & version to a scheme 34 | AddToScheme = localSchemeBuilder.AddToScheme 35 | ) 36 | 37 | // addKnownTypes registers known types to the given scheme 38 | func addKnownTypes(scheme *runtime.Scheme) error { 39 | scheme.AddKnownTypes(SchemeGroupVersion, 40 | &SchedulingArgs{}, 41 | ) 42 | return nil 43 | } 44 | 45 | func init() { 46 | // We only register manually written functions here. The registration of the 47 | // generated functions takes place in the generated files. The separation 48 | // makes the code compile even when the generated files are missing. 49 | localSchemeBuilder.Register(addKnownTypes) 50 | } 51 | -------------------------------------------------------------------------------- /cmd/scheduler/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "math/rand" 21 | "os" 22 | "time" 23 | 24 | "k8s.io/component-base/logs" 25 | "k8s.io/kubernetes/cmd/kube-scheduler/app" 26 | 27 | "github.com/cwdsuzhou/super-scheduling/pkg/multicluster/scheduler" 28 | "github.com/cwdsuzhou/super-scheduling/pkg/topologyscheduling" 29 | // Ensure scheme package is initialized. 30 | _ "github.com/cwdsuzhou/super-scheduling/pkg/apis/config/scheme" 31 | ) 32 | 33 | func main() { 34 | rand.Seed(time.Now().UnixNano()) 35 | 36 | // Register custom plugins to the multicluster framework. 37 | // Later they can consist of multicluster profile(s) and hence 38 | // used by various kinds of workloads. 39 | command := app.NewSchedulerCommand( 40 | app.WithPlugin(topologyscheduling.Name, topologyscheduling.New), 41 | app.WithPlugin(scheduler.Name, scheduler.New), 42 | ) 43 | 44 | // TODO: once we switch everything over to Cobra commands, we can go back to calling 45 | // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the 46 | // normalize func and add the go flag set by hand. 47 | // utilflag.InitFlags() 48 | logs.InitLogs() 49 | defer logs.FlushLogs() 50 | 51 | if err := command.Execute(); err != nil { 52 | os.Exit(1) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/scheduling/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package scheduling 20 | 21 | import ( 22 | internalinterfaces "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/internalinterfaces" 23 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/scheduling/v1alpha1" 24 | ) 25 | 26 | // Interface provides access to each of this group's versions. 27 | type Interface interface { 28 | // V1alpha1 provides access to shared informers for resources in V1alpha1. 29 | V1alpha1() v1alpha1.Interface 30 | } 31 | 32 | type group struct { 33 | factory internalinterfaces.SharedInformerFactory 34 | namespace string 35 | tweakListOptions internalinterfaces.TweakListOptionsFunc 36 | } 37 | 38 | // New returns a new Interface. 39 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 40 | return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 41 | } 42 | 43 | // V1alpha1 returns a new v1alpha1.Interface. 44 | func (g *group) V1alpha1() v1alpha1.Interface { 45 | return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/scheduling/v1alpha1/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | internalinterfaces "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/internalinterfaces" 23 | ) 24 | 25 | // Interface provides access to all the informers in this group version. 26 | type Interface interface { 27 | // TopologySchedulingPolicies returns a TopologySchedulingPolicyInformer. 28 | TopologySchedulingPolicies() TopologySchedulingPolicyInformer 29 | } 30 | 31 | type version struct { 32 | factory internalinterfaces.SharedInformerFactory 33 | namespace string 34 | tweakListOptions internalinterfaces.TweakListOptionsFunc 35 | } 36 | 37 | // New returns a new Interface. 38 | func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { 39 | return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} 40 | } 41 | 42 | // TopologySchedulingPolicies returns a TopologySchedulingPolicyInformer. 43 | func (v *version) TopologySchedulingPolicies() TopologySchedulingPolicyInformer { 44 | return &topologySchedulingPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} 45 | } 46 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1beta1 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/runtime" 21 | "k8s.io/apimachinery/pkg/runtime/schema" 22 | schedschemev1beta1 "k8s.io/kube-scheduler/config/v1beta1" 23 | ) 24 | 25 | // GroupName is the group name used in this package 26 | const GroupName = "kubescheduler.config.k8s.io" 27 | 28 | // SchemeGroupVersion is group version used to register these objects 29 | var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} 30 | 31 | var ( 32 | localSchemeBuilder = &schedschemev1beta1.SchemeBuilder 33 | // AddToScheme is a global function that registers this API group & version to a scheme 34 | AddToScheme = localSchemeBuilder.AddToScheme 35 | ) 36 | 37 | // addKnownTypes registers known types to the given scheme 38 | func addKnownTypes(scheme *runtime.Scheme) error { 39 | scheme.AddKnownTypes(SchemeGroupVersion, 40 | &SchedulingArgs{}, 41 | ) 42 | return nil 43 | } 44 | 45 | func init() { 46 | // We only register manually written functions here. The registration of the 47 | // generated functions takes place in the generated files. The separation 48 | // makes the code compile even when the generated files are missing. 49 | localSchemeBuilder.Register(addKnownTypes) 50 | localSchemeBuilder.Register(RegisterDefaults) 51 | localSchemeBuilder.Register(RegisterConversions) 52 | } 53 | -------------------------------------------------------------------------------- /hack/update-generated-openapi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | # TODO: make this script run faster. 22 | 23 | SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. 24 | 25 | go install k8s.io/kube-openapi/cmd/openapi-gen 26 | 27 | KUBE_INPUT_DIRS=( 28 | $( 29 | grep --color=never -rl '+k8s:openapi-gen=' vendor/k8s.io | \ 30 | xargs -n1 dirname | \ 31 | sed "s,^vendor/,," | \ 32 | sort -u | \ 33 | sed '/^k8s\.io\/kubernetes\/build\/root$/d' | \ 34 | sed '/^k8s\.io\/kubernetes$/d' | \ 35 | sed '/^k8s\.io\/kubernetes\/staging$/d' | \ 36 | sed 's,k8s\.io/kubernetes/staging/src/,,' | \ 37 | grep -v 'k8s.io/code-generator' | \ 38 | grep -v 'k8s.io/sample-apiserver' 39 | ) 40 | ) 41 | 42 | KUBE_INPUT_DIRS=$(IFS=,; echo "${KUBE_INPUT_DIRS[*]}") 43 | 44 | function join { local IFS="$1"; shift; echo "$*"; } 45 | 46 | echo "Generating Kubernetes OpenAPI" 47 | 48 | $GOPATH/bin/openapi-gen \ 49 | --output-file-base zz_generated.openapi \ 50 | --output-base="${GOPATH}/src" \ 51 | --go-header-file ${SCRIPT_ROOT}/hack/boilerplate/boilerplate.generatego.txt \ 52 | --output-base="./" \ 53 | --input-dirs $(join , "${KUBE_INPUT_DIRS[@]}") \ 54 | --output-package "vendor/k8s.io/kubernetes/pkg/generated/openapi" \ 55 | --report-filename "${SCRIPT_ROOT}/hack/openapi-violation.list" \ 56 | "$@" 57 | 58 | # TODO: verify hack/openapi-violation.list -------------------------------------------------------------------------------- /pkg/apis/scheduling/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | 24 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling" 25 | ) 26 | 27 | // SchemeGroupVersion is group version used to register these objects 28 | var SchemeGroupVersion = schema.GroupVersion{Group: scheduling.GroupName, Version: "v1alpha1"} 29 | 30 | // Kind takes an unqualified kind and returns back a Group qualified GroupKind 31 | func Kind(kind string) schema.GroupKind { 32 | return SchemeGroupVersion.WithKind(kind).GroupKind() 33 | } 34 | 35 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 36 | func Resource(resource string) schema.GroupResource { 37 | return SchemeGroupVersion.WithResource(resource).GroupResource() 38 | } 39 | 40 | var ( 41 | // SchemeBuilder initializes a scheme builder 42 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 43 | // AddToScheme is a global function that registers this API group & version to a scheme 44 | AddToScheme = SchemeBuilder.AddToScheme 45 | ) 46 | 47 | // Adds the list of known types to Scheme. 48 | func addKnownTypes(scheme *runtime.Scheme) error { 49 | scheme.AddKnownTypes(SchemeGroupVersion, 50 | &TopologySchedulingPolicy{}, 51 | &TopologySchedulingPolicyList{}, 52 | ) 53 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cwdsuzhou/super-scheduling 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/prometheus/client_golang v1.9.0 // indirect 7 | github.com/spf13/cobra v1.1.1 8 | github.com/spf13/pflag v1.0.5 9 | k8s.io/api v0.20.5 10 | k8s.io/apimachinery v0.20.5 11 | k8s.io/client-go v0.20.5 12 | k8s.io/code-generator v0.20.5 13 | k8s.io/component-base v0.20.5 14 | k8s.io/klog/v2 v2.4.0 15 | k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd 16 | k8s.io/kube-scheduler v0.20.5 17 | k8s.io/kubernetes v1.20.5 18 | sigs.k8s.io/descheduler v0.20.0 19 | ) 20 | 21 | replace ( 22 | k8s.io/api => k8s.io/api v0.20.5 23 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.5 24 | k8s.io/apimachinery => k8s.io/apimachinery v0.20.5 25 | k8s.io/apiserver => k8s.io/apiserver v0.20.5 26 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.5 27 | k8s.io/client-go => k8s.io/client-go v0.20.5 28 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.5 29 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.5 30 | k8s.io/code-generator => k8s.io/code-generator v0.20.5 31 | k8s.io/component-base => k8s.io/component-base v0.20.5 32 | k8s.io/component-helpers => k8s.io/component-helpers v0.20.5 33 | k8s.io/controller-manager => k8s.io/controller-manager v0.20.5 34 | k8s.io/cri-api => k8s.io/cri-api v0.20.5 35 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.5 36 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.5 37 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.5 38 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.5 39 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.5 40 | k8s.io/kubectl => k8s.io/kubectl v0.20.5 41 | k8s.io/kubelet => k8s.io/kubelet v0.20.5 42 | k8s.io/kubernetes => github.com/cwdsuzhou/kubernetes v1.20.6-0.20210902113229-e2b1d75d8057 43 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.5 44 | k8s.io/metrics => k8s.io/metrics v0.20.5 45 | k8s.io/mount-utils => k8s.io/mount-utils v0.20.5 46 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.5 47 | sigs.k8s.io/descheduler => sigs.k8s.io/descheduler v0.20.0 48 | ) 49 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | schedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var scheme = runtime.NewScheme() 31 | var codecs = serializer.NewCodecFactory(scheme) 32 | 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | schedulingv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /cmd/descheduler/app/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package app implements a Server object for running the descheduler. 18 | package app 19 | 20 | import ( 21 | "flag" 22 | "io" 23 | 24 | "github.com/spf13/cobra" 25 | 26 | aflag "k8s.io/component-base/cli/flag" 27 | "k8s.io/klog/v2" 28 | 29 | "github.com/cwdsuzhou/super-scheduling/cmd/descheduler/app/options" 30 | "github.com/cwdsuzhou/super-scheduling/pkg/descheduler" 31 | ) 32 | 33 | // NewDeschedulerCommand creates a *cobra.Command object with default parameters 34 | func NewDeschedulerCommand(out io.Writer) *cobra.Command { 35 | s, err := options.NewDeschedulerServer() 36 | 37 | if err != nil { 38 | klog.ErrorS(err, "unable to initialize server") 39 | } 40 | 41 | cmd := &cobra.Command{ 42 | Use: "descheduler", 43 | Short: "descheduler", 44 | Long: `The descheduler evicts pods which may be bound to less desired nodes`, 45 | Run: func(cmd *cobra.Command, args []string) { 46 | s.Logs.LogFormat = s.Logging.Format 47 | s.Logs.Apply() 48 | 49 | if err := s.Validate(); err != nil { 50 | klog.ErrorS(err, "failed to validate server configuration") 51 | } 52 | err := Run(s) 53 | if err != nil { 54 | klog.ErrorS(err, "descheduler server") 55 | } 56 | }, 57 | } 58 | cmd.SetOut(out) 59 | flags := cmd.Flags() 60 | flags.SetNormalizeFunc(aflag.WordSepNormalizeFunc) 61 | flags.AddGoFlagSet(flag.CommandLine) 62 | s.AddFlags(flags) 63 | return cmd 64 | } 65 | 66 | func Run(rs *options.DeschedulerServer) error { 67 | return descheduler.Run(rs) 68 | } 69 | -------------------------------------------------------------------------------- /hack/integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o nounset 19 | set -o pipefail 20 | 21 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. 22 | source "${SCRIPT_ROOT}/hack/lib/init.sh" 23 | 24 | checkEtcdOnPath() { 25 | # If it's in a prow CI env, add etcd to path. 26 | [[ ${CI:-} == "true" ]] && export PATH="$(pwd)/etcd:${PATH}" 27 | kube::log::status "Checking etcd is on PATH" 28 | command -v etcd >/dev/null && return 29 | kube::log::status "Cannot find etcd, cannot run integration tests." 30 | kube::log::status "Please see https://git.k8s.io/community/contributors/devel/sig-testing/integration-tests.md#install-etcd-dependency for instructions." 31 | # kube::log::usage "You can use 'hack/install-etcd.sh' to install a copy in third_party/." 32 | return 1 33 | } 34 | 35 | CLEANUP_REQUIRED= 36 | cleanup() { 37 | [[ -z "${CLEANUP_REQUIRED}" ]] && return 38 | kube::log::status "Cleaning up etcd" 39 | kube::etcd::cleanup 40 | CLEANUP_REQUIRED= 41 | kube::log::status "Integration test cleanup complete" 42 | } 43 | 44 | runTests() { 45 | kube::log::status "Starting etcd instance" 46 | CLEANUP_REQUIRED=1 47 | kube::etcd::start 48 | kube::log::status "Running integration test cases" 49 | 50 | # TODO: make args customizable. 51 | go test -timeout=40m -mod=vendor github.com/cwdsuzhou/super-scheduling/test/integration/... 52 | 53 | cleanup 54 | } 55 | 56 | checkEtcdOnPath 57 | 58 | # Run cleanup to stop etcd on interrupt or other kill signal. 59 | trap cleanup EXIT 60 | 61 | runTests 62 | -------------------------------------------------------------------------------- /hack/verify-gofmt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script checks whether the source codes need to be formatted or not by 18 | # `gofmt`. We should run `hack/update-gofmt.sh` if actually formats them. 19 | # Usage: `hack/verify-gofmt.sh`. 20 | # Note: GoFmt apparently is changing @ head... 21 | 22 | set -o errexit 23 | set -o nounset 24 | set -o pipefail 25 | 26 | SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. 27 | source "${SCRIPT_ROOT}/hack/lib/init.sh" 28 | 29 | kube::golang::verify_go_version 30 | 31 | cd "${SCRIPT_ROOT}" 32 | 33 | find_files() { 34 | find . -not \( \ 35 | \( \ 36 | -wholename './output' \ 37 | -o -wholename './.git' \ 38 | -o -wholename './_output' \ 39 | -o -wholename './release' \ 40 | -o -wholename './target' \ 41 | -o -wholename '*/third_party/*' \ 42 | -o -wholename '*/vendor/*' \ 43 | -o -wholename './staging/src/k8s.io/client-go/*vendor/*' \ 44 | -o -wholename '*/bindata.go' \ 45 | \) -prune \ 46 | \) -name '*.go' 47 | } 48 | 49 | 50 | # gofmt exits with non-zero exit code if it finds a problem unrelated to 51 | # formatting (e.g., a file does not parse correctly). Without "|| true" this 52 | # would have led to no useful error message from gofmt, because the script would 53 | # have failed before getting to the "echo" in the block below. 54 | diff=$(find_files | xargs gofmt -d -s 2>&1) || true 55 | if [[ -n "${diff}" ]]; then 56 | echo "${diff}" >&2 57 | echo >&2 58 | echo "Run ./hack/update-gofmt.sh" >&2 59 | exit 1 60 | fi 61 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package scheme 20 | 21 | import ( 22 | schedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var Scheme = runtime.NewScheme() 31 | var Codecs = serializer.NewCodecFactory(Scheme) 32 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | schedulingv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(Scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/generic.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package externalversions 20 | 21 | import ( 22 | "fmt" 23 | 24 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | cache "k8s.io/client-go/tools/cache" 27 | ) 28 | 29 | // GenericInformer is type of SharedIndexInformer which will locate and delegate to other 30 | // sharedInformers based on type 31 | type GenericInformer interface { 32 | Informer() cache.SharedIndexInformer 33 | Lister() cache.GenericLister 34 | } 35 | 36 | type genericInformer struct { 37 | informer cache.SharedIndexInformer 38 | resource schema.GroupResource 39 | } 40 | 41 | // Informer returns the SharedIndexInformer. 42 | func (f *genericInformer) Informer() cache.SharedIndexInformer { 43 | return f.informer 44 | } 45 | 46 | // Lister returns the GenericLister. 47 | func (f *genericInformer) Lister() cache.GenericLister { 48 | return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) 49 | } 50 | 51 | // ForResource gives generic access to a shared informer of the matching type 52 | // TODO extend this to unknown resources with a client pool 53 | func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { 54 | switch resource { 55 | // Group=scheduling.sigs.k8s.io, Version=v1alpha1 56 | case v1alpha1.SchemeGroupVersion.WithResource("topologyschedulingpolicies"): 57 | return &genericInformer{resource: resource.GroupResource(), informer: f.Scheduling().V1alpha1().TopologySchedulingPolicies().Informer()}, nil 58 | 59 | } 60 | 61 | return nil, fmt.Errorf("no informer found for %v", resource) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/apis/config/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package config 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Configuration) DeepCopyInto(out *Configuration) { 29 | *out = *in 30 | return 31 | } 32 | 33 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. 34 | func (in *Configuration) DeepCopy() *Configuration { 35 | if in == nil { 36 | return nil 37 | } 38 | out := new(Configuration) 39 | in.DeepCopyInto(out) 40 | return out 41 | } 42 | 43 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 44 | func (in *SchedulingArgs) DeepCopyInto(out *SchedulingArgs) { 45 | *out = *in 46 | out.TypeMeta = in.TypeMeta 47 | if in.ClusterConfiguration != nil { 48 | in, out := &in.ClusterConfiguration, &out.ClusterConfiguration 49 | *out = make(map[string]Configuration, len(*in)) 50 | for key, val := range *in { 51 | (*out)[key] = val 52 | } 53 | } 54 | return 55 | } 56 | 57 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingArgs. 58 | func (in *SchedulingArgs) DeepCopy() *SchedulingArgs { 59 | if in == nil { 60 | return nil 61 | } 62 | out := new(SchedulingArgs) 63 | in.DeepCopyInto(out) 64 | return out 65 | } 66 | 67 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 68 | func (in *SchedulingArgs) DeepCopyObject() runtime.Object { 69 | if c := in.DeepCopy(); c != nil { 70 | return c 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/util/utils.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | 8 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 9 | ) 10 | 11 | // ClustersNodeSelection is a struct including some scheduling parameters 12 | type ClustersNodeSelection struct { 13 | NodeSelector map[string]string `json:"nodeSelector,omitempty"` 14 | Affinity *corev1.Affinity `json:"affinity,omitempty"` 15 | Tolerations []corev1.Toleration `json:"tolerations,omitempty"` 16 | } 17 | 18 | const ( 19 | // NodeType is define the node type key 20 | NodeType = "type" 21 | // VirtualPodLabel is the label of virtual pod 22 | VirtualPodLabel = "virtual-pod" 23 | // VirtualKubeletLabel is the label of virtual kubelet 24 | VirtualKubeletLabel = "virtual-kubelet" 25 | // TrippedLabels is the label of tripped labels 26 | TrippedLabels = "tripped-labels" 27 | // ClusterID marks the id of a cluster 28 | ClusterID = "clusterID" 29 | // SelectorKey is the key of ClusterSelector 30 | SelectorKey = "clusterSelector" 31 | ) 32 | 33 | func CovertPolicy(dp []v1alpha1.SchedulePolicy) map[string]int32 { 34 | policy := make(map[string]int32) 35 | for _, p := range dp { 36 | policy[p.Name] = p.Replicas 37 | } 38 | return policy 39 | } 40 | 41 | // IsVirtualNode defines if a node is virtual node 42 | func IsVirtualNode(node *corev1.Node) bool { 43 | if node == nil { 44 | return false 45 | } 46 | valStr, exist := node.ObjectMeta.Labels[NodeType] 47 | if !exist { 48 | return false 49 | } 50 | return valStr == VirtualKubeletLabel 51 | } 52 | 53 | // IsVirtualPod defines if a pod is virtual pod 54 | func IsVirtualPod(pod *corev1.Pod) bool { 55 | if pod.Labels != nil && pod.Labels[VirtualPodLabel] == "true" { 56 | return true 57 | } 58 | return false 59 | } 60 | 61 | // GetClusterID return the cluster in node label 62 | func GetClusterID(node *corev1.Node) string { 63 | if node == nil { 64 | return "" 65 | } 66 | clusterName, exist := node.ObjectMeta.Labels[ClusterID] 67 | if !exist { 68 | return "" 69 | } 70 | return clusterName 71 | } 72 | 73 | // ConvertAnnotations converts annotations to ClustersNodeSelection 74 | func ConvertAnnotations(annotation map[string]string) *ClustersNodeSelection { 75 | if annotation == nil { 76 | return nil 77 | } 78 | val := annotation[SelectorKey] 79 | if len(val) == 0 { 80 | return nil 81 | } 82 | 83 | var cns ClustersNodeSelection 84 | err := json.Unmarshal([]byte(val), &cns) 85 | if err != nil { 86 | return nil 87 | } 88 | return &cns 89 | } 90 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/scheduling_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 23 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/scheme" 24 | rest "k8s.io/client-go/rest" 25 | ) 26 | 27 | type SchedulingV1alpha1Interface interface { 28 | RESTClient() rest.Interface 29 | TopologySchedulingPoliciesGetter 30 | } 31 | 32 | // SchedulingV1alpha1Client is used to interact with features provided by the scheduling.sigs.k8s.io group. 33 | type SchedulingV1alpha1Client struct { 34 | restClient rest.Interface 35 | } 36 | 37 | func (c *SchedulingV1alpha1Client) TopologySchedulingPolicies(namespace string) TopologySchedulingPolicyInterface { 38 | return newTopologySchedulingPolicies(c, namespace) 39 | } 40 | 41 | // NewForConfig creates a new SchedulingV1alpha1Client for the given config. 42 | func NewForConfig(c *rest.Config) (*SchedulingV1alpha1Client, error) { 43 | config := *c 44 | if err := setConfigDefaults(&config); err != nil { 45 | return nil, err 46 | } 47 | client, err := rest.RESTClientFor(&config) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return &SchedulingV1alpha1Client{client}, nil 52 | } 53 | 54 | // NewForConfigOrDie creates a new SchedulingV1alpha1Client for the given config and 55 | // panics if there is an error in the config. 56 | func NewForConfigOrDie(c *rest.Config) *SchedulingV1alpha1Client { 57 | client, err := NewForConfig(c) 58 | if err != nil { 59 | panic(err) 60 | } 61 | return client 62 | } 63 | 64 | // New creates a new SchedulingV1alpha1Client for the given RESTClient. 65 | func New(c rest.Interface) *SchedulingV1alpha1Client { 66 | return &SchedulingV1alpha1Client{c} 67 | } 68 | 69 | func setConfigDefaults(config *rest.Config) error { 70 | gv := v1alpha1.SchemeGroupVersion 71 | config.GroupVersion = &gv 72 | config.APIPath = "/apis" 73 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 74 | 75 | if config.UserAgent == "" { 76 | config.UserAgent = rest.DefaultKubernetesUserAgent() 77 | } 78 | 79 | return nil 80 | } 81 | 82 | // RESTClient returns a RESTClient that is used to communicate 83 | // with API server by this client implementation. 84 | func (c *SchedulingV1alpha1Client) RESTClient() rest.Interface { 85 | if c == nil { 86 | return nil 87 | } 88 | return c.restClient 89 | } 90 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package v1beta1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Configuration) DeepCopyInto(out *Configuration) { 29 | *out = *in 30 | if in.Name != nil { 31 | in, out := &in.Name, &out.Name 32 | *out = new(string) 33 | **out = **in 34 | } 35 | if in.KubeMaster != nil { 36 | in, out := &in.KubeMaster, &out.KubeMaster 37 | *out = new(string) 38 | **out = **in 39 | } 40 | if in.KubeConfig != nil { 41 | in, out := &in.KubeConfig, &out.KubeConfig 42 | *out = new(string) 43 | **out = **in 44 | } 45 | return 46 | } 47 | 48 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. 49 | func (in *Configuration) DeepCopy() *Configuration { 50 | if in == nil { 51 | return nil 52 | } 53 | out := new(Configuration) 54 | in.DeepCopyInto(out) 55 | return out 56 | } 57 | 58 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 59 | func (in *SchedulingArgs) DeepCopyInto(out *SchedulingArgs) { 60 | *out = *in 61 | out.TypeMeta = in.TypeMeta 62 | if in.KubeConfigPath != nil { 63 | in, out := &in.KubeConfigPath, &out.KubeConfigPath 64 | *out = new(string) 65 | **out = **in 66 | } 67 | if in.KubeMaster != nil { 68 | in, out := &in.KubeMaster, &out.KubeMaster 69 | *out = new(string) 70 | **out = **in 71 | } 72 | if in.ClusterConfiguration != nil { 73 | in, out := &in.ClusterConfiguration, &out.ClusterConfiguration 74 | *out = make(map[string]Configuration, len(*in)) 75 | for key, val := range *in { 76 | (*out)[key] = *val.DeepCopy() 77 | } 78 | } 79 | return 80 | } 81 | 82 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingArgs. 83 | func (in *SchedulingArgs) DeepCopy() *SchedulingArgs { 84 | if in == nil { 85 | return nil 86 | } 87 | out := new(SchedulingArgs) 88 | in.DeepCopyInto(out) 89 | return out 90 | } 91 | 92 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 93 | func (in *SchedulingArgs) DeepCopyObject() runtime.Object { 94 | if c := in.DeepCopy(); c != nil { 95 | return c 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /hack/lib/util.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | kube::util::host_os() { 18 | local host_os 19 | case "$(uname -s)" in 20 | Darwin) 21 | host_os=darwin 22 | ;; 23 | Linux) 24 | host_os=linux 25 | ;; 26 | *) 27 | kube::log::error "Unsupported host OS. Must be Linux or Mac OS X." 28 | exit 1 29 | ;; 30 | esac 31 | echo "${host_os}" 32 | } 33 | 34 | kube::util::host_arch() { 35 | local host_arch 36 | case "$(uname -m)" in 37 | x86_64*) 38 | host_arch=amd64 39 | ;; 40 | i?86_64*) 41 | host_arch=amd64 42 | ;; 43 | amd64*) 44 | host_arch=amd64 45 | ;; 46 | aarch64*) 47 | host_arch=arm64 48 | ;; 49 | arm64*) 50 | host_arch=arm64 51 | ;; 52 | arm*) 53 | host_arch=arm 54 | ;; 55 | i?86*) 56 | host_arch=x86 57 | ;; 58 | s390x*) 59 | host_arch=s390x 60 | ;; 61 | ppc64le*) 62 | host_arch=ppc64le 63 | ;; 64 | *) 65 | kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." 66 | exit 1 67 | ;; 68 | esac 69 | echo "${host_arch}" 70 | } 71 | 72 | kube::util::wait_for_url() { 73 | local url=$1 74 | local prefix=${2:-} 75 | local wait=${3:-1} 76 | local times=${4:-30} 77 | local maxtime=${5:-1} 78 | 79 | command -v curl >/dev/null || { 80 | kube::log::usage "curl must be installed" 81 | exit 1 82 | } 83 | 84 | local i 85 | for i in $(seq 1 "${times}"); do 86 | local out 87 | if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then 88 | kube::log::status "On try ${i}, ${prefix}: ${out}" 89 | return 0 90 | fi 91 | sleep "${wait}" 92 | done 93 | kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each" 94 | return 1 95 | } 96 | 97 | kube::util::download_file() { 98 | local -r url=$1 99 | local -r destination_file=$2 100 | 101 | rm "${destination_file}" 2&> /dev/null || true 102 | 103 | for i in $(seq 5) 104 | do 105 | if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then 106 | echo "Downloading ${url} failed. $((5-i)) retries left." 107 | sleep 1 108 | else 109 | echo "Downloading ${url} succeed" 110 | return 0 111 | fi 112 | done 113 | return 1 114 | } 115 | -------------------------------------------------------------------------------- /cmd/descheduler/app/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package app 18 | 19 | import ( 20 | "fmt" 21 | "regexp" 22 | "runtime" 23 | "strings" 24 | 25 | "github.com/spf13/cobra" 26 | ) 27 | 28 | var ( 29 | // version is a constant representing the version tag that 30 | // generated this build. It should be set during build via -ldflags. 31 | version string 32 | // buildDate in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') 33 | //It should be set during build via -ldflags. 34 | buildDate string 35 | ) 36 | 37 | // Info holds the information related to descheduler app version. 38 | type Info struct { 39 | Major string `json:"major"` 40 | Minor string `json:"minor"` 41 | GitVersion string `json:"gitVersion"` 42 | BuildDate string `json:"buildDate"` 43 | GoVersion string `json:"goVersion"` 44 | Compiler string `json:"compiler"` 45 | Platform string `json:"platform"` 46 | } 47 | 48 | // Get returns the overall codebase version. It's for detecting 49 | // what code a binary was built from. 50 | func Get() Info { 51 | majorVersion, minorVersion := splitVersion(version) 52 | return Info{ 53 | Major: majorVersion, 54 | Minor: minorVersion, 55 | GitVersion: version, 56 | BuildDate: buildDate, 57 | GoVersion: runtime.Version(), 58 | Compiler: runtime.Compiler, 59 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 60 | } 61 | } 62 | 63 | func NewVersionCommand() *cobra.Command { 64 | var versionCmd = &cobra.Command{ 65 | Use: "version", 66 | Short: "Version of descheduler", 67 | Long: `Prints the version of descheduler.`, 68 | Run: func(cmd *cobra.Command, args []string) { 69 | fmt.Printf("Descheduler version %+v\n", Get()) 70 | }, 71 | } 72 | return versionCmd 73 | } 74 | 75 | // splitVersion splits the git version to generate major and minor versions needed. 76 | func splitVersion(version string) (string, string) { 77 | if version == "" { 78 | return "", "" 79 | } 80 | 81 | // Version from an automated container build environment for a tag. For example v20200521-v0.18.0. 82 | m1, _ := regexp.MatchString(`^v\d{8}-v\d+\.\d+\.\d+$`, version) 83 | 84 | // Version from an automated container build environment(not a tag) or a local build. For example v20201009-v0.18.0-46-g939c1c0. 85 | m2, _ := regexp.MatchString(`^v\d{8}-v\d+\.\d+\.\d+-\w+-\w+$`, version) 86 | 87 | if m1 || m2 { 88 | semVer := strings.Split(version, "-")[1] 89 | return strings.Trim(strings.Split(semVer, ".")[0], "v"), strings.Split(semVer, ".")[1] + "+" 90 | } 91 | 92 | // Something went wrong 93 | return "", "" 94 | } 95 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/fake/clientset_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | clientset "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 23 | schedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/typed/scheduling/v1alpha1" 24 | fakeschedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/fake" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/watch" 27 | "k8s.io/client-go/discovery" 28 | fakediscovery "k8s.io/client-go/discovery/fake" 29 | "k8s.io/client-go/testing" 30 | ) 31 | 32 | // NewSimpleClientset returns a clientset that will respond with the provided objects. 33 | // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, 34 | // without applying any validations and/or defaults. It shouldn't be considered a replacement 35 | // for a real clientset and is mostly useful in simple unit tests. 36 | func NewSimpleClientset(objects ...runtime.Object) *Clientset { 37 | o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) 38 | for _, obj := range objects { 39 | if err := o.Add(obj); err != nil { 40 | panic(err) 41 | } 42 | } 43 | 44 | cs := &Clientset{tracker: o} 45 | cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} 46 | cs.AddReactor("*", "*", testing.ObjectReaction(o)) 47 | cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { 48 | gvr := action.GetResource() 49 | ns := action.GetNamespace() 50 | watch, err := o.Watch(gvr, ns) 51 | if err != nil { 52 | return false, nil, err 53 | } 54 | return true, watch, nil 55 | }) 56 | 57 | return cs 58 | } 59 | 60 | // Clientset implements clientset.Interface. Meant to be embedded into a 61 | // struct to get a default implementation. This makes faking out just the method 62 | // you want to test easier. 63 | type Clientset struct { 64 | testing.Fake 65 | discovery *fakediscovery.FakeDiscovery 66 | tracker testing.ObjectTracker 67 | } 68 | 69 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 70 | return c.discovery 71 | } 72 | 73 | func (c *Clientset) Tracker() testing.ObjectTracker { 74 | return c.tracker 75 | } 76 | 77 | var _ clientset.Interface = &Clientset{} 78 | 79 | // SchedulingV1alpha1 retrieves the SchedulingV1alpha1Client 80 | func (c *Clientset) SchedulingV1alpha1() schedulingv1alpha1.SchedulingV1alpha1Interface { 81 | return &fakeschedulingv1alpha1.FakeSchedulingV1alpha1{Fake: &c.Fake} 82 | } 83 | -------------------------------------------------------------------------------- /pkg/multicluster/scheduler/generic.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package scheduler 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "k8s.io/client-go/tools/cache" 24 | "k8s.io/client-go/tools/events" 25 | schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" 26 | schedulerserverconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" 27 | "k8s.io/kubernetes/pkg/scheduler" 28 | "k8s.io/kubernetes/pkg/scheduler/profile" 29 | ) 30 | 31 | // Scheduler define the multicluster struct 32 | type Scheduler struct { 33 | *scheduler.Scheduler 34 | // configuration of multicluster 35 | Config schedulerappconfig.Config 36 | // stop signal 37 | StopCh <-chan struct{} 38 | } 39 | 40 | // NewScheduler executes the multicluster based on the given configuration. It only return on error or when stopCh is closed. 41 | func NewScheduler(ctx context.Context, cc schedulerappconfig.Config, stopCh <-chan struct{}) (*Scheduler, error) { 42 | // To help debugging, immediately log version 43 | 44 | completedConfig := cc.Complete() 45 | recordFactory := getRecorderFactory(&completedConfig) 46 | 47 | // Create the multicluster. 48 | sched, err := scheduler.New(cc.Client, 49 | cc.InformerFactory, 50 | recordFactory, 51 | stopCh, 52 | scheduler.WithProfiles(cc.ComponentConfig.Profiles...), 53 | scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), 54 | scheduler.WithPodMaxBackoffSeconds(cc.ComponentConfig.PodMaxBackoffSeconds), 55 | scheduler.WithPodInitialBackoffSeconds(cc.ComponentConfig.PodInitialBackoffSeconds), 56 | scheduler.WithExtenders(cc.ComponentConfig.Extenders...), 57 | ) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return &Scheduler{ 62 | Config: cc, Scheduler: sched, StopCh: stopCh, 63 | }, nil 64 | } 65 | 66 | // Run executes the multicluster based on the given configuration. It only return on error or when stopCh is closed. 67 | func (sched *Scheduler) Run(ctx context.Context) error { 68 | // Prepare the event broadcaster. 69 | if sched.Config.EventBroadcaster != nil && sched.Config.Client != nil { 70 | sched.Config.EventBroadcaster.StartRecordingToSink(sched.StopCh) 71 | } 72 | 73 | // Start all informers. 74 | go sched.Config.InformerFactory.Core().V1().Pods().Informer().Run(sched.StopCh) 75 | sched.Config.InformerFactory.Start(sched.StopCh) 76 | 77 | // Wait for all caches to sync before scheduling. 78 | sched.Config.InformerFactory.WaitForCacheSync(sched.StopCh) 79 | 80 | if !cache.WaitForCacheSync(ctx.Done()) { 81 | return fmt.Errorf("failed to wait cache sync") 82 | } 83 | <-sched.StopCh 84 | return nil 85 | } 86 | 87 | func getRecorderFactory(cc *schedulerserverconfig.CompletedConfig) profile.RecorderFactory { 88 | return func(name string) events.EventRecorder { 89 | return cc.EventBroadcaster.NewRecorder(name) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package versioned 20 | 21 | import ( 22 | "fmt" 23 | 24 | schedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/typed/scheduling/v1alpha1" 25 | discovery "k8s.io/client-go/discovery" 26 | rest "k8s.io/client-go/rest" 27 | flowcontrol "k8s.io/client-go/util/flowcontrol" 28 | ) 29 | 30 | type Interface interface { 31 | Discovery() discovery.DiscoveryInterface 32 | SchedulingV1alpha1() schedulingv1alpha1.SchedulingV1alpha1Interface 33 | } 34 | 35 | // Clientset contains the clients for groups. Each group has exactly one 36 | // version included in a Clientset. 37 | type Clientset struct { 38 | *discovery.DiscoveryClient 39 | schedulingV1alpha1 *schedulingv1alpha1.SchedulingV1alpha1Client 40 | } 41 | 42 | // SchedulingV1alpha1 retrieves the SchedulingV1alpha1Client 43 | func (c *Clientset) SchedulingV1alpha1() schedulingv1alpha1.SchedulingV1alpha1Interface { 44 | return c.schedulingV1alpha1 45 | } 46 | 47 | // Discovery retrieves the DiscoveryClient 48 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 49 | if c == nil { 50 | return nil 51 | } 52 | return c.DiscoveryClient 53 | } 54 | 55 | // NewForConfig creates a new Clientset for the given config. 56 | // If config's RateLimiter is not set and QPS and Burst are acceptable, 57 | // NewForConfig will generate a rate-limiter in configShallowCopy. 58 | func NewForConfig(c *rest.Config) (*Clientset, error) { 59 | configShallowCopy := *c 60 | if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { 61 | if configShallowCopy.Burst <= 0 { 62 | return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") 63 | } 64 | configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) 65 | } 66 | var cs Clientset 67 | var err error 68 | cs.schedulingV1alpha1, err = schedulingv1alpha1.NewForConfig(&configShallowCopy) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) 74 | if err != nil { 75 | return nil, err 76 | } 77 | return &cs, nil 78 | } 79 | 80 | // NewForConfigOrDie creates a new Clientset for the given config and 81 | // panics if there is an error in the config. 82 | func NewForConfigOrDie(c *rest.Config) *Clientset { 83 | var cs Clientset 84 | cs.schedulingV1alpha1 = schedulingv1alpha1.NewForConfigOrDie(c) 85 | 86 | cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) 87 | return &cs 88 | } 89 | 90 | // New creates a new Clientset for the given RESTClient. 91 | func New(c rest.Interface) *Clientset { 92 | var cs Clientset 93 | cs.schedulingV1alpha1 = schedulingv1alpha1.New(c) 94 | 95 | cs.DiscoveryClient = discovery.NewDiscoveryClient(c) 96 | return &cs 97 | } 98 | -------------------------------------------------------------------------------- /pkg/multicluster/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "time" 21 | 22 | "k8s.io/client-go/informers" 23 | clientset "k8s.io/client-go/kubernetes" 24 | restclient "k8s.io/client-go/rest" 25 | "k8s.io/client-go/tools/clientcmd" 26 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 27 | "k8s.io/client-go/tools/events" 28 | componentbaseconfig "k8s.io/component-base/config" 29 | "k8s.io/klog/v2" 30 | kubeschedulerconfigv1alpha1 "k8s.io/kube-scheduler/config/v1beta1" 31 | schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" 32 | kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" 33 | kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" 34 | ) 35 | 36 | // Config return a multicluster config object 37 | func Config(c schedulerappconfig.Config, master string) (*schedulerappconfig.Config, error) { 38 | // Prepare kube clients. 39 | client, _, eventClient, err := createClients(c.ComponentConfig.ClientConnection, master, 40 | c.ComponentConfig.LeaderElection.RenewDeadline.Duration) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | c.Client = client 46 | c.InformerFactory = informers.NewSharedInformerFactory(client, 0) 47 | c.EventBroadcaster = events.NewEventBroadcasterAdapter(eventClient) 48 | 49 | return &c, nil 50 | } 51 | 52 | // createClients creates a kube client and an event client from the given config and masterOverride. 53 | func createClients(config componentbaseconfig.ClientConnectionConfiguration, 54 | masterOverride string, timeout time.Duration) (clientset.Interface, clientset.Interface, clientset.Interface, error) { 55 | if len(config.Kubeconfig) == 0 && len(masterOverride) == 0 { 56 | klog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") 57 | } 58 | 59 | // This creates a client, first loading any specified kubeconfig 60 | // file, and then overriding the Master flag, if non-empty. 61 | kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 62 | &clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig}, 63 | &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig() 64 | if err != nil { 65 | return nil, nil, nil, err 66 | } 67 | 68 | kubeConfig.AcceptContentTypes = config.AcceptContentTypes 69 | kubeConfig.ContentType = config.ContentType 70 | kubeConfig.QPS = config.QPS 71 | kubeConfig.Burst = int(config.Burst) 72 | 73 | client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "multicluster")) 74 | if err != nil { 75 | return nil, nil, nil, err 76 | } 77 | 78 | // shallow copy, do not modify the kubeConfig.Timeout. 79 | restConfig := *kubeConfig 80 | restConfig.Timeout = timeout 81 | leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(&restConfig, "leader-election")) 82 | if err != nil { 83 | return nil, nil, nil, err 84 | } 85 | 86 | eventClient, err := clientset.NewForConfig(kubeConfig) 87 | if err != nil { 88 | return nil, nil, nil, err 89 | } 90 | 91 | return client, leaderElectionClient, eventClient, nil 92 | } 93 | 94 | // NewDefaultComponentConfig return the default configuration of multi multicluster 95 | func NewDefaultComponentConfig() (*kubeschedulerconfig.KubeSchedulerConfiguration, error) { 96 | cfgv1alpha1 := kubeschedulerconfigv1alpha1.KubeSchedulerConfiguration{} 97 | kubeschedulerscheme.Scheme.Default(&cfgv1alpha1) 98 | cfg := kubeschedulerconfig.KubeSchedulerConfiguration{} 99 | if err := kubeschedulerscheme.Scheme.Convert(&cfgv1alpha1, &cfg, nil); err != nil { 100 | return nil, err 101 | } 102 | return &cfg, nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/scheduling/v1alpha1/topologyschedulingpolicy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "context" 23 | time "time" 24 | 25 | schedulingv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 26 | versioned "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 27 | internalinterfaces "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/internalinterfaces" 28 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/listers/scheduling/v1alpha1" 29 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | runtime "k8s.io/apimachinery/pkg/runtime" 31 | watch "k8s.io/apimachinery/pkg/watch" 32 | cache "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | // TopologySchedulingPolicyInformer provides access to a shared informer and lister for 36 | // TopologySchedulingPolicies. 37 | type TopologySchedulingPolicyInformer interface { 38 | Informer() cache.SharedIndexInformer 39 | Lister() v1alpha1.TopologySchedulingPolicyLister 40 | } 41 | 42 | type topologySchedulingPolicyInformer struct { 43 | factory internalinterfaces.SharedInformerFactory 44 | tweakListOptions internalinterfaces.TweakListOptionsFunc 45 | namespace string 46 | } 47 | 48 | // NewTopologySchedulingPolicyInformer constructs a new informer for TopologySchedulingPolicy type. 49 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 50 | // one. This reduces memory footprint and number of connections to the server. 51 | func NewTopologySchedulingPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { 52 | return NewFilteredTopologySchedulingPolicyInformer(client, namespace, resyncPeriod, indexers, nil) 53 | } 54 | 55 | // NewFilteredTopologySchedulingPolicyInformer constructs a new informer for TopologySchedulingPolicy type. 56 | // Always prefer using an informer factory to get a shared informer instead of getting an independent 57 | // one. This reduces memory footprint and number of connections to the server. 58 | func NewFilteredTopologySchedulingPolicyInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { 59 | return cache.NewSharedIndexInformer( 60 | &cache.ListWatch{ 61 | ListFunc: func(options v1.ListOptions) (runtime.Object, error) { 62 | if tweakListOptions != nil { 63 | tweakListOptions(&options) 64 | } 65 | return client.SchedulingV1alpha1().TopologySchedulingPolicies(namespace).List(context.TODO(), options) 66 | }, 67 | WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { 68 | if tweakListOptions != nil { 69 | tweakListOptions(&options) 70 | } 71 | return client.SchedulingV1alpha1().TopologySchedulingPolicies(namespace).Watch(context.TODO(), options) 72 | }, 73 | }, 74 | &schedulingv1alpha1.TopologySchedulingPolicy{}, 75 | resyncPeriod, 76 | indexers, 77 | ) 78 | } 79 | 80 | func (f *topologySchedulingPolicyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { 81 | return NewFilteredTopologySchedulingPolicyInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) 82 | } 83 | 84 | func (f *topologySchedulingPolicyInformer) Informer() cache.SharedIndexInformer { 85 | return f.factory.InformerFor(&schedulingv1alpha1.TopologySchedulingPolicy{}, f.defaultInformer) 86 | } 87 | 88 | func (f *topologySchedulingPolicyInformer) Lister() v1alpha1.TopologySchedulingPolicyLister { 89 | return v1alpha1.NewTopologySchedulingPolicyLister(f.Informer().GetIndexer()) 90 | } 91 | -------------------------------------------------------------------------------- /cmd/descheduler/app/options/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package options provides the descheduler flags 18 | package options 19 | 20 | import ( 21 | "github.com/spf13/pflag" 22 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 23 | clientset "k8s.io/client-go/kubernetes" 24 | 25 | "k8s.io/component-base/logs" 26 | "sigs.k8s.io/descheduler/pkg/apis/componentconfig" 27 | "sigs.k8s.io/descheduler/pkg/apis/componentconfig/v1alpha1" 28 | deschedulerscheme "sigs.k8s.io/descheduler/pkg/descheduler/scheme" 29 | 30 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 31 | ) 32 | 33 | // DeschedulerServer configuration 34 | type DeschedulerServer struct { 35 | componentconfig.DeschedulerConfiguration 36 | Client clientset.Interface 37 | TopologySchedulingClient versioned.Interface 38 | Logs *logs.Options 39 | } 40 | 41 | // NewDeschedulerServer creates a new DeschedulerServer with default parameters 42 | func NewDeschedulerServer() (*DeschedulerServer, error) { 43 | cfg, err := newDefaultComponentConfig() 44 | if err != nil { 45 | return nil, err 46 | } 47 | return &DeschedulerServer{ 48 | DeschedulerConfiguration: *cfg, 49 | Logs: logs.NewOptions(), 50 | }, nil 51 | } 52 | 53 | // Validation checks for DeschedulerServer. 54 | func (s *DeschedulerServer) Validate() error { 55 | var errs []error 56 | errs = append(errs, s.Logs.Validate()...) 57 | return utilerrors.NewAggregate(errs) 58 | } 59 | 60 | func newDefaultComponentConfig() (*componentconfig.DeschedulerConfiguration, error) { 61 | versionedCfg := v1alpha1.DeschedulerConfiguration{} 62 | deschedulerscheme.Scheme.Default(&versionedCfg) 63 | cfg := componentconfig.DeschedulerConfiguration{} 64 | if err := deschedulerscheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { 65 | return nil, err 66 | } 67 | return &cfg, nil 68 | } 69 | 70 | // AddFlags adds flags for a specific SchedulerServer to the specified FlagSet 71 | func (rs *DeschedulerServer) AddFlags(fs *pflag.FlagSet) { 72 | fs.StringVar(&rs.Logging.Format, "logging-format", rs.Logging.Format, `Sets the log format. Permitted formats: "text", "json". Non-default formats don't honor these flags: --add-dir-header, --alsologtostderr, --log-backtrace-at, --log-dir, --log-file, --log-file-max-size, --logtostderr, --skip-headers, --skip-log-headers, --stderrthreshold, --log-flush-frequency.\nNon-default choices are currently alpha and subject to change without warning.`) 73 | fs.DurationVar(&rs.DeschedulingInterval, "descheduling-interval", rs.DeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") 74 | fs.StringVar(&rs.KubeconfigFile, "kubeconfig", rs.KubeconfigFile, "File with kube configuration.") 75 | fs.StringVar(&rs.PolicyConfigFile, "policy-config-file", rs.PolicyConfigFile, "File with descheduler policy configuration.") 76 | fs.BoolVar(&rs.DryRun, "dry-run", rs.DryRun, "execute descheduler in dry run mode.") 77 | // node-selector query causes descheduler to run only on nodes that matches the node labels in the query 78 | fs.StringVar(&rs.NodeSelector, "node-selector", rs.NodeSelector, "DEPRECATED: selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") 79 | // max-no-pods-to-evict limits the maximum number of pods to be evicted per node by descheduler. 80 | fs.IntVar(&rs.MaxNoOfPodsToEvictPerNode, "max-pods-to-evict-per-node", rs.MaxNoOfPodsToEvictPerNode, "DEPRECATED: limits the maximum number of pods to be evicted per node by descheduler") 81 | // evict-local-storage-pods allows eviction of pods that are using local storage. This is false by default. 82 | fs.BoolVar(&rs.EvictLocalStoragePods, "evict-local-storage-pods", rs.EvictLocalStoragePods, "DEPRECATED: enables evicting pods using local storage by descheduler") 83 | } 84 | -------------------------------------------------------------------------------- /pkg/apis/scheduling/v1alpha1/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | ) 22 | 23 | // +genclient 24 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 25 | 26 | // TopologySchedulingPolicy describes multi clusters/regions/zones scheduling policy 27 | type TopologySchedulingPolicy struct { 28 | metav1.TypeMeta 29 | 30 | // Standard object's metadata. 31 | // +optional 32 | metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` 33 | 34 | // Specification of the desired behavior of the multi cluster. 35 | // +optional 36 | Spec TopologySchedulingPolicySpec `json:"spec,omitempty"` 37 | 38 | // Status represents the current information about multi cluster. 39 | // This data may not be up to date. 40 | // +optional 41 | Status TopologySchedulingPolicyStatus `json:"status,omitempty"` 42 | } 43 | 44 | type TopologySchedulingPolicySpec struct { 45 | // TopologyKey is used when match node topoKey 46 | TopologyKey string `json:"topologyKey"` 47 | 48 | // ScheduleStrategy is used when schedule pods for a topoKey. 49 | // It can be Fill or Balance. 50 | // Fill will schedule pods to satisfy a topo requirements first, then start another. 51 | // Balance will try to keep Balance across different topo when scheduling pods. 52 | ScheduleStrategy string `json:"scheduleStrategy"` 53 | 54 | // DeployPlacement describes the topology requirement when scheduling pods. 55 | DeployPlacement []SchedulePolicy `json:"deployPlacement"` 56 | 57 | // UpdatePlacement describes topology requirements when updating pods. 58 | // Notice: This is not implemented now. 59 | UpdatePlacement []UpdatePolicy `json:"updatePlacement"` 60 | 61 | // LabelSelector is used to find matching pods. 62 | // Pods that match this label selector are counted to determine the number of pods 63 | // in their corresponding topology domain. 64 | // +optional 65 | LabelSelector *metav1.LabelSelector `json:"labelSelector"` 66 | } 67 | 68 | type DeployPlacement struct { 69 | WhenUnsatisfiable UnsatisfiableConstraintAction `json:"whenUnsatisfiable"` 70 | Policy []SchedulePolicy `json:"policy"` 71 | } 72 | 73 | const ( 74 | ScheduleStrategyFill = "Fill" 75 | ScheduleStrategyBalance = "Balance" 76 | ) 77 | 78 | type UnsatisfiableConstraintAction string 79 | 80 | const ( 81 | // DoNotSchedule instructs the multicluster not to schedule the pod 82 | // when constraints are not satisfied. 83 | DoNotSchedule UnsatisfiableConstraintAction = "DoNotSchedule" 84 | // ScheduleAnyway instructs the multicluster to schedule the pod 85 | // even if constraints are not satisfied. 86 | ScheduleAnyway UnsatisfiableConstraintAction = "ScheduleAnyway" 87 | ) 88 | 89 | // SchedulePolicy describe the topo value required replicas 90 | type SchedulePolicy struct { 91 | // Name is the topo value 92 | Name string `json:"name"` 93 | 94 | // Replicas is the desired the replicas for the topo value 95 | Replicas int32 `json:"replicas"` 96 | } 97 | 98 | type UpdatePolicy struct { 99 | MaxSkew int32 `json:"maxSkew"` 100 | 101 | UpdateSequence []string `json:"updateSequence"` 102 | } 103 | 104 | // TopologySchedulingPolicyStatus describe the placement results. 105 | type TopologySchedulingPolicyStatus struct { 106 | Placement []SchedulePolicy `json:"placement"` 107 | } 108 | 109 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 110 | 111 | // TopologySchedulingPolicyList is a collection of policy. 112 | type TopologySchedulingPolicyList struct { 113 | metav1.TypeMeta `json:",inline"` 114 | // Standard list metadata 115 | // +optional 116 | metav1.ListMeta `json:"metadata,omitempty"` 117 | 118 | // Items is the list of MultiClusterPolicy 119 | Items []TopologySchedulingPolicy `json:"items"` 120 | } 121 | -------------------------------------------------------------------------------- /pkg/generated/listers/scheduling/v1alpha1/topologyschedulingpolicy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by lister-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/client-go/tools/cache" 26 | ) 27 | 28 | // TopologySchedulingPolicyLister helps list TopologySchedulingPolicies. 29 | // All objects returned here must be treated as read-only. 30 | type TopologySchedulingPolicyLister interface { 31 | // List lists all TopologySchedulingPolicies in the indexer. 32 | // Objects returned here must be treated as read-only. 33 | List(selector labels.Selector) (ret []*v1alpha1.TopologySchedulingPolicy, err error) 34 | // TopologySchedulingPolicies returns an object that can list and get TopologySchedulingPolicies. 35 | TopologySchedulingPolicies(namespace string) TopologySchedulingPolicyNamespaceLister 36 | TopologySchedulingPolicyListerExpansion 37 | } 38 | 39 | // topologySchedulingPolicyLister implements the TopologySchedulingPolicyLister interface. 40 | type topologySchedulingPolicyLister struct { 41 | indexer cache.Indexer 42 | } 43 | 44 | // NewTopologySchedulingPolicyLister returns a new TopologySchedulingPolicyLister. 45 | func NewTopologySchedulingPolicyLister(indexer cache.Indexer) TopologySchedulingPolicyLister { 46 | return &topologySchedulingPolicyLister{indexer: indexer} 47 | } 48 | 49 | // List lists all TopologySchedulingPolicies in the indexer. 50 | func (s *topologySchedulingPolicyLister) List(selector labels.Selector) (ret []*v1alpha1.TopologySchedulingPolicy, err error) { 51 | err = cache.ListAll(s.indexer, selector, func(m interface{}) { 52 | ret = append(ret, m.(*v1alpha1.TopologySchedulingPolicy)) 53 | }) 54 | return ret, err 55 | } 56 | 57 | // TopologySchedulingPolicies returns an object that can list and get TopologySchedulingPolicies. 58 | func (s *topologySchedulingPolicyLister) TopologySchedulingPolicies(namespace string) TopologySchedulingPolicyNamespaceLister { 59 | return topologySchedulingPolicyNamespaceLister{indexer: s.indexer, namespace: namespace} 60 | } 61 | 62 | // TopologySchedulingPolicyNamespaceLister helps list and get TopologySchedulingPolicies. 63 | // All objects returned here must be treated as read-only. 64 | type TopologySchedulingPolicyNamespaceLister interface { 65 | // List lists all TopologySchedulingPolicies in the indexer for a given namespace. 66 | // Objects returned here must be treated as read-only. 67 | List(selector labels.Selector) (ret []*v1alpha1.TopologySchedulingPolicy, err error) 68 | // Get retrieves the TopologySchedulingPolicy from the indexer for a given namespace and name. 69 | // Objects returned here must be treated as read-only. 70 | Get(name string) (*v1alpha1.TopologySchedulingPolicy, error) 71 | TopologySchedulingPolicyNamespaceListerExpansion 72 | } 73 | 74 | // topologySchedulingPolicyNamespaceLister implements the TopologySchedulingPolicyNamespaceLister 75 | // interface. 76 | type topologySchedulingPolicyNamespaceLister struct { 77 | indexer cache.Indexer 78 | namespace string 79 | } 80 | 81 | // List lists all TopologySchedulingPolicies in the indexer for a given namespace. 82 | func (s topologySchedulingPolicyNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.TopologySchedulingPolicy, err error) { 83 | err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { 84 | ret = append(ret, m.(*v1alpha1.TopologySchedulingPolicy)) 85 | }) 86 | return ret, err 87 | } 88 | 89 | // Get retrieves the TopologySchedulingPolicy from the indexer for a given namespace and name. 90 | func (s topologySchedulingPolicyNamespaceLister) Get(name string) (*v1alpha1.TopologySchedulingPolicy, error) { 91 | obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) 92 | if err != nil { 93 | return nil, err 94 | } 95 | if !exists { 96 | return nil, errors.NewNotFound(v1alpha1.Resource("topologyschedulingpolicy"), name) 97 | } 98 | return obj.(*v1alpha1.TopologySchedulingPolicy), nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/descheduler/strategies/topologyscheduling.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package strategies 18 | 19 | import ( 20 | "context" 21 | "sort" 22 | 23 | v1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/labels" 26 | clientset "k8s.io/client-go/kubernetes" 27 | "k8s.io/klog/v2" 28 | "sigs.k8s.io/descheduler/pkg/api" 29 | "sigs.k8s.io/descheduler/pkg/descheduler/evictions" 30 | 31 | "github.com/cwdsuzhou/super-scheduling/pkg/common" 32 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/listers/scheduling/v1alpha1" 33 | "github.com/cwdsuzhou/super-scheduling/pkg/util" 34 | ) 35 | 36 | type TopoPairCount struct { 37 | count int32 38 | pods []*v1.Pod 39 | } 40 | 41 | func RemovePodsViolatingTopologySchedulingPolicy( 42 | ctx context.Context, 43 | client clientset.Interface, 44 | strategy api.DeschedulerStrategy, 45 | nodes []*v1.Node, 46 | podEvictor *evictions.PodEvictor, 47 | tspLister v1alpha1.TopologySchedulingPolicyLister, 48 | ) { 49 | if tspLister == nil { 50 | return 51 | } 52 | nodeMap := make(map[string]*v1.Node) 53 | for _, node := range nodes { 54 | nodeMap[node.Name] = node 55 | } 56 | tsps, err := tspLister.TopologySchedulingPolicies(v1.NamespaceAll).List(labels.Everything()) 57 | if err != nil { 58 | klog.Error(err) 59 | return 60 | } 61 | for _, tsp := range tsps { 62 | selector, err := metav1.LabelSelectorAsSelector(tsp.Spec.LabelSelector) 63 | if err != nil { 64 | klog.Error(err) 65 | continue 66 | } 67 | constraint := common.TopologySchedulingConstraint{ 68 | TopologyKey: tsp.Spec.TopologyKey, 69 | SchedulePolicy: util.CovertPolicy(tsp.Spec.DeployPlacement), 70 | Selector: selector, 71 | } 72 | TpPairToMatchNum := make(map[common.TopologyPair]*TopoPairCount) 73 | pods, err := client.CoreV1().Pods(tsp.Namespace).List(ctx, metav1.ListOptions{ 74 | LabelSelector: metav1.FormatLabelSelector(tsp.Spec.LabelSelector), 75 | }) 76 | if err != nil { 77 | klog.Error(err) 78 | continue 79 | } 80 | for _, p := range pods.Items { 81 | pod := p.DeepCopy() 82 | if pod.DeletionTimestamp != nil { 83 | continue 84 | } 85 | node := nodeMap[pod.Spec.NodeName] 86 | if node == nil { 87 | continue 88 | } 89 | pair := common.TopologyPair{Key: tsp.Spec.TopologyKey, Value: node.Labels[tsp.Spec.TopologyKey]} 90 | counter := TpPairToMatchNum[pair] 91 | if counter == nil { 92 | counter = &TopoPairCount{ 93 | count: 1, 94 | pods: []*v1.Pod{pod}, 95 | } 96 | TpPairToMatchNum[pair] = counter 97 | } else { 98 | counter.count++ 99 | counter.pods = append(counter.pods, pod) 100 | } 101 | } 102 | if checkTopoReached(TpPairToMatchNum, constraint) { 103 | continue 104 | } 105 | for pair, counter := range TpPairToMatchNum { 106 | diff := constraint.SchedulePolicy[pair.Value] - counter.count 107 | if diff >= 0 { 108 | continue 109 | } 110 | // sort pods 111 | toDeleteCount := -diff 112 | sort.Sort(ByCostom(counter.pods)) 113 | for i := 0; i < int(toDeleteCount); i++ { 114 | podEvictor.EvictPod(ctx, counter.pods[i], nodeMap[counter.pods[i].Spec.NodeName], 115 | "Exceeded the policy desired") 116 | } 117 | } 118 | } 119 | } 120 | 121 | func checkTopoReached(pair map[common.TopologyPair]*TopoPairCount, 122 | constraint common.TopologySchedulingConstraint) bool { 123 | satisfied := true 124 | notSatisfied := false 125 | for pair, counter := range pair { 126 | diff := constraint.SchedulePolicy[pair.Value] - counter.count 127 | if diff > 0 { 128 | notSatisfied = true 129 | continue 130 | } else if diff < 0 { 131 | satisfied = true 132 | } 133 | } 134 | // some keys exceed, some keys not exceed => not reached 135 | // all keys exceed => reached 136 | // all key not exceed => not reached 137 | if satisfied && !notSatisfied { 138 | return true 139 | } 140 | return false 141 | } 142 | 143 | type ByCostom []*v1.Pod 144 | 145 | var _ sort.Interface = &ByCostom{} 146 | 147 | func (p ByCostom) Len() int { 148 | return len(p) 149 | } 150 | 151 | func (p ByCostom) Less(i, j int) bool { 152 | if p[i].Status.Phase == v1.PodFailed || p[i].Status.Phase == v1.PodSucceeded { 153 | return true 154 | } 155 | if p[i].Status.Phase == v1.PodPending { 156 | return true 157 | } 158 | if p[j].Status.Phase == v1.PodFailed || p[j].Status.Phase == v1.PodSucceeded { 159 | return false 160 | } 161 | if p[j].Status.Phase == v1.PodPending { 162 | return false 163 | } 164 | return p[i].Name < p[j].Name 165 | } 166 | 167 | func (p ByCostom) Swap(i, j int) { 168 | p[i], p[j] = p[j], p[i] 169 | } 170 | -------------------------------------------------------------------------------- /hack/lib/etcd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2020 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # A set of helpers for starting/running etcd for tests 18 | 19 | ETCD_VERSION=${ETCD_VERSION:-3.4.9} 20 | ETCD_HOST=${ETCD_HOST:-127.0.0.1} 21 | ETCD_PORT=${ETCD_PORT:-2379} 22 | export KUBE_INTEGRATION_ETCD_URL="http://${ETCD_HOST}:${ETCD_PORT}" 23 | 24 | kube::etcd::validate() { 25 | # validate if in path 26 | command -v etcd >/dev/null || { 27 | kube::log::usage "etcd must be in your PATH" 28 | kube::log::info "You can use 'hack/install-etcd.sh' to install a copy in third_party/." 29 | exit 1 30 | } 31 | 32 | # validate etcd port is free 33 | local port_check_command 34 | if command -v ss &> /dev/null && ss -Version | grep 'iproute2' &> /dev/null; then 35 | port_check_command="ss" 36 | elif command -v netstat &>/dev/null; then 37 | port_check_command="netstat" 38 | else 39 | kube::log::usage "unable to identify if etcd is bound to port ${ETCD_PORT}. unable to find ss or netstat utilities." 40 | exit 1 41 | fi 42 | if ${port_check_command} -nat | grep "LISTEN" | grep "[\.:]${ETCD_PORT:?}" >/dev/null 2>&1; then 43 | kube::log::usage "unable to start etcd as port ${ETCD_PORT} is in use. please stop the process listening on this port and retry." 44 | kube::log::usage "$(netstat -nat | grep "[\.:]${ETCD_PORT:?} .*LISTEN")" 45 | exit 1 46 | fi 47 | 48 | # need set the env of "ETCD_UNSUPPORTED_ARCH" on unstable arch. 49 | arch=$(uname -m) 50 | if [[ $arch =~ aarch* ]]; then 51 | export ETCD_UNSUPPORTED_ARCH=arm64 52 | elif [[ $arch =~ arm* ]]; then 53 | export ETCD_UNSUPPORTED_ARCH=arm 54 | fi 55 | # validate installed version is at least equal to minimum 56 | version=$(etcd --version | grep Version | head -n 1 | cut -d " " -f 3) 57 | if [[ $(kube::etcd::version "${ETCD_VERSION}") -gt $(kube::etcd::version "${version}") ]]; then 58 | kube::log::usage "etcd version ${ETCD_VERSION} or greater required." 59 | kube::log::info "You can use 'hack/install-etcd.sh' to install." 60 | exit 1 61 | fi 62 | } 63 | 64 | kube::etcd::version() { 65 | printf '%s\n' "${@}" | awk -F . '{ printf("%d%03d%03d\n", $1, $2, $3) }' 66 | } 67 | 68 | kube::etcd::start() { 69 | # validate before running 70 | kube::etcd::validate 71 | 72 | # Start etcd 73 | ETCD_DIR=${ETCD_DIR:-$(mktemp -d 2>/dev/null || mktemp -d -t test-etcd.XXXXXX)} 74 | if [[ -d "${ARTIFACTS:-}" ]]; then 75 | ETCD_LOGFILE="${ARTIFACTS}/etcd.$(uname -n).$(id -un).log.DEBUG.$(date +%Y%m%d-%H%M%S).$$" 76 | else 77 | ETCD_LOGFILE=${ETCD_LOGFILE:-"/dev/null"} 78 | fi 79 | kube::log::info "etcd --advertise-client-urls ${KUBE_INTEGRATION_ETCD_URL} --data-dir ${ETCD_DIR} --listen-client-urls http://${ETCD_HOST}:${ETCD_PORT} --debug > \"${ETCD_LOGFILE}\" 2>/dev/null" 80 | etcd --advertise-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --data-dir "${ETCD_DIR}" --listen-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --debug 2> "${ETCD_LOGFILE}" >/dev/null & 81 | ETCD_PID=$! 82 | 83 | echo "Waiting for etcd to come up." 84 | kube::util::wait_for_url "${KUBE_INTEGRATION_ETCD_URL}/health" "etcd: " 0.25 80 85 | curl -fs -X POST "${KUBE_INTEGRATION_ETCD_URL}/v3/kv/put" -d '{"key": "X3Rlc3Q=", "value": ""}' 86 | } 87 | 88 | kube::etcd::stop() { 89 | if [[ -n "${ETCD_PID-}" ]]; then 90 | kill "${ETCD_PID}" &>/dev/null || : 91 | wait "${ETCD_PID}" &>/dev/null || : 92 | fi 93 | } 94 | 95 | kube::etcd::clean_etcd_dir() { 96 | if [[ -n "${ETCD_DIR-}" ]]; then 97 | rm -rf "${ETCD_DIR}" 98 | fi 99 | } 100 | 101 | kube::etcd::cleanup() { 102 | kube::etcd::stop 103 | kube::etcd::clean_etcd_dir 104 | } 105 | 106 | kube::etcd::install() { 107 | ( 108 | local os 109 | local arch 110 | 111 | os=$(kube::util::host_os) 112 | arch=$(kube::util::host_arch) 113 | 114 | if [[ $(readlink etcd) == etcd-v${ETCD_VERSION}-${os}-* ]]; then 115 | kube::log::info "etcd v${ETCD_VERSION} already installed. To use:" 116 | kube::log::info "export PATH=\"$(pwd)/etcd:\${PATH}\"" 117 | return #already installed 118 | fi 119 | 120 | if [[ ${os} == "darwin" ]]; then 121 | download_file="etcd-v${ETCD_VERSION}-darwin-amd64.zip" 122 | url="https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/${download_file}" 123 | kube::util::download_file "${url}" "${download_file}" 124 | unzip -o "${download_file}" 125 | ln -fns "etcd-v${ETCD_VERSION}-darwin-amd64" etcd 126 | rm "${download_file}" 127 | else 128 | url="https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-${arch}.tar.gz" 129 | download_file="etcd-v${ETCD_VERSION}-linux-${arch}.tar.gz" 130 | kube::util::download_file "${url}" "${download_file}" 131 | tar xzf "${download_file}" 132 | ln -fns "etcd-v${ETCD_VERSION}-linux-${arch}" etcd 133 | rm "${download_file}" 134 | fi 135 | kube::log::info "etcd v${ETCD_VERSION} installed. To use:" 136 | kube::log::info "export PATH=\"$(pwd)/etcd:\${PATH}\"" 137 | ) 138 | } 139 | -------------------------------------------------------------------------------- /pkg/multicluster/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright ©2020. The virtual-kubelet authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package scheduler 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/client-go/tools/clientcmd" 26 | componentbaseconfig "k8s.io/component-base/config" 27 | "k8s.io/klog/v2" 28 | schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" 29 | "k8s.io/kubernetes/pkg/scheduler/framework" 30 | frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 31 | 32 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/config" 33 | clusterconfig "github.com/cwdsuzhou/super-scheduling/pkg/multicluster/config" 34 | "github.com/cwdsuzhou/super-scheduling/pkg/util" 35 | ) 36 | 37 | // Name is MultiClusterScheduling plugin name, will use in configuration file 38 | const Name = "MultiClusterScheduling" 39 | 40 | // MultiSchedulingPlugin is plugin implemented scheduling framework 41 | type MultiSchedulingPlugin struct { 42 | frameworkHandler framework.Handle 43 | schedulers map[string]*Scheduler 44 | } 45 | 46 | // Name returns the plugin name 47 | func (m MultiSchedulingPlugin) Name() string { 48 | return Name 49 | } 50 | 51 | var _ framework.FilterPlugin = &MultiSchedulingPlugin{} 52 | 53 | // Filter check if a pod can run on node 54 | func (m MultiSchedulingPlugin) Filter(pc context.Context, state *framework.CycleState, 55 | pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 56 | snapshot := m.frameworkHandler.SnapshotSharedLister().NodeInfos() 57 | if snapshot == nil { 58 | return framework.NewStatus(framework.Success, "") 59 | } 60 | 61 | if nodeInfo == nil || nodeInfo.Node() == nil { 62 | return framework.NewStatus(framework.Unschedulable, "node not exit") 63 | } 64 | if !util.IsVirtualNode(nodeInfo.Node()) { 65 | klog.V(5).Infof("node %v is not virtual node", nodeInfo.Node().Name) 66 | return framework.NewStatus(framework.Success, "") 67 | } 68 | 69 | schedulerName := util.GetClusterID(nodeInfo.Node()) 70 | if len(schedulerName) == 0 { 71 | klog.V(5).Infof("Can not found multicluster %v", schedulerName) 72 | return framework.NewStatus(framework.Success, "") 73 | } 74 | 75 | scheduler := m.schedulers[schedulerName] 76 | 77 | if scheduler == nil { 78 | klog.V(5).Infof("Can not found scheduling %v", schedulerName) 79 | return framework.NewStatus(framework.Success, "") 80 | } 81 | 82 | podCopy := pod.DeepCopy() 83 | klog.V(5).Infof("Matching node for pod: %v", pod.Name) 84 | cns := util.ConvertAnnotations(podCopy.Annotations) 85 | // remove selector 86 | if cns != nil { 87 | podCopy.Spec.NodeSelector = cns.NodeSelector 88 | podCopy.Spec.Affinity = cns.Affinity 89 | podCopy.Spec.Tolerations = cns.Tolerations 90 | } else { 91 | podCopy.Spec.NodeSelector = nil 92 | podCopy.Spec.Affinity = nil 93 | podCopy.Spec.Tolerations = nil 94 | } 95 | 96 | result, err := scheduler.Algorithm.Filter(pc, scheduler.Profiles["default-scheduler"], state, podCopy) 97 | klog.V(5).Infof("%v Nodes, Node %s can be scheduled to run pod", result.FeasibleNodes, result.SuggestedHost) 98 | if err != nil { 99 | klog.Infof("Pod selector: %+v, affinity: %+v", pod.Spec.NodeSelector, pod.Spec.Affinity) 100 | return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Can not found nodes: %s", err)) 101 | } 102 | if result.FeasibleNodes == 0 || result.SuggestedHost == "" { 103 | return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Can not found feasible nodes from cluster: %v/%v", 104 | result.FeasibleNodes, result.EvaluatedNodes)) 105 | } 106 | return framework.NewStatus(framework.Success, "") 107 | } 108 | 109 | // New initializes a new plugin and returns it. 110 | func New(obj runtime.Object, f framework.Handle) (framework.Plugin, error) { 111 | var configs config.SchedulingArgs 112 | err := frameworkruntime.DecodeInto(obj, &configs) 113 | if err != nil { 114 | return nil, fmt.Errorf("want args to be of type SchedulingArgs, got %T\n", obj) 115 | } 116 | ctx := context.TODO() 117 | schedulers := make(map[string]*Scheduler) 118 | for name, config := range configs.ClusterConfiguration { 119 | klog.V(4).Infof("cluster %s's config: master(%s), kube-config(%s)", name, config.KubeMaster, config.KubeConfig) 120 | // Init client and Informer 121 | c, err := clientcmd.BuildConfigFromFlags(config.KubeMaster, config.KubeConfig) 122 | if err != nil { 123 | return nil, fmt.Errorf("failed to init rest.Config: %v", err) 124 | } 125 | c.QPS = 10 126 | c.Burst = 20 127 | 128 | componentConfig, err := clusterconfig.NewDefaultComponentConfig() 129 | if err != nil { 130 | klog.Fatal(err) 131 | } 132 | 133 | componentConfig.ClientConnection = componentbaseconfig.ClientConnectionConfiguration{ 134 | Kubeconfig: config.KubeConfig, 135 | } 136 | // componentConfig.DisablePreemption = true 137 | 138 | cfg := schedulerappconfig.Config{ 139 | ComponentConfig: *componentConfig, 140 | } 141 | newConfig, err := clusterconfig.Config(cfg, config.KubeMaster) 142 | if err != nil { 143 | klog.Fatal(err) 144 | } 145 | 146 | scheduler, err := NewScheduler(ctx, *newConfig, ctx.Done()) 147 | if err != nil { 148 | klog.Fatal(err) 149 | } 150 | go scheduler.Run(ctx) 151 | schedulers[name] = scheduler 152 | } 153 | klog.Infof("MultiScheduling start") 154 | return MultiSchedulingPlugin{ 155 | frameworkHandler: f, 156 | schedulers: schedulers, 157 | }, nil 158 | } 159 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARCHS = amd64 arm64 16 | COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z) 17 | BUILDENVVAR=CGO_ENABLED=0 18 | 19 | LOCAL_REGISTRY=localhost:5000/scheduler-plugins 20 | LOCAL_IMAGE=kube-scheduler:latest 21 | 22 | # RELEASE_REGISTRY is the container registry to push 23 | # into. The default is to push to the staging 24 | # registry, not production(k8s.gcr.io). 25 | RELEASE_REGISTRY?=gcr.io/k8s-staging-scheduler-plugins 26 | RELEASE_VERSION?=v$(shell date +%Y%m%d)-$(shell git describe --tags --match "v*") 27 | RELEASE_IMAGE:=kube-scheduler:$(RELEASE_VERSION) 28 | RELEASE_DESCHEDULER_IMAGE:=descheduler:$(RELEASE_VERSION) 29 | 30 | # VERSION is the multicluster's version 31 | # 32 | # The RELEASE_VERSION variable can have one of two formats: 33 | # v20201009-v0.18.800-46-g939c1c0 - automated build for a commit(not a tag) and also a local build 34 | # v20200521-v0.18.800 - automated build for a tag 35 | VERSION=$(shell echo $(RELEASE_VERSION) | awk -F - '{print $$2}') 36 | 37 | .PHONY: all 38 | all: build 39 | 40 | .PHONY: build 41 | build: build-scheduler 42 | 43 | .PHONY: build.amd64 44 | build.amd64: build-scheduler.amd64 45 | 46 | .PHONY: build.arm64v8 47 | build.arm64v8: build-scheduler.arm64v8 48 | 49 | .PHONY: build-scheduler build-descheduler 50 | build-scheduler: autogen 51 | $(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go 52 | 53 | build-descheduler: autogen 54 | $(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/descheduler cmd/descheduler/descheduler.go 55 | 56 | .PHONY: build-scheduler.amd64 build-descheduler.amd64 57 | build-scheduler.amd64: autogen 58 | $(COMMONENVVAR) $(BUILDENVVAR) GOARCH=amd64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go 59 | 60 | build-descheduler.amd64: autogen 61 | $(COMMONENVVAR) $(BUILDENVVAR) GOARCH=amd64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/descheduler cmd/descheduler/descheduler.go 62 | 63 | .PHONY: build-scheduler.arm64v8 build-descheduler.arm64v8 64 | build-scheduler.arm64v8: autogen 65 | GOOS=linux $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go 66 | 67 | build-descheduler.arm64v8: autogen 68 | GOOS=linux $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/descheduler cmd/descheduler/descheduler.go 69 | 70 | .PHONY: local-image 71 | local-image: clean 72 | docker build -f ./build/scheduler/Dockerfile --build-arg ARCH="amd64" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(LOCAL_REGISTRY)/$(LOCAL_IMAGE) . 73 | docker build -f ./build/descheduler/Dockerfile --build-arg ARCH="amd64" -t $(LOCAL_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE) . 74 | 75 | .PHONY: release-image.amd64 76 | release-image.amd64: clean 77 | docker build -f ./build/scheduler/Dockerfile --build-arg ARCH="amd64" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-amd64 . 78 | docker build -f ./build/descheduler/Dockerfile --build-arg ARCH="amd64" -t $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE)-amd64 . 79 | 80 | .PHONY: release-image.arm64v8 81 | release-image.arm64v8: clean 82 | docker build -f ./build/scheduler/Dockerfile --build-arg ARCH="arm64v8" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-arm64 . 83 | docker build -f ./build/descheduler/Dockerfile --build-arg ARCH="arm64v8" -t $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE)-arm64 . 84 | 85 | .PHONY: push-release-images 86 | push-release-images: release-image.amd64 release-image.arm64v8 87 | gcloud auth configure-docker 88 | for arch in $(ARCHS); do \ 89 | docker push $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-$${arch} ;\ 90 | docker push $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE)-$${arch} ;\ 91 | done 92 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) $(addprefix --amend $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-, $(ARCHS)) 93 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE) $(addprefix --amend $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE)-, $(ARCHS)) 94 | for arch in $(ARCHS); do \ 95 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate --arch $${arch} $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-$${arch} ;\ 96 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate --arch $${arch} $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE) $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE)-$${arch} ;\ 97 | done 98 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) ;\ 99 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push $(RELEASE_REGISTRY)/$(RELEASE_DESCHEDULER_IMAGE) ;\ 100 | 101 | .PHONY: update-vendor 102 | update-vendor: 103 | hack/update-vendor.sh 104 | 105 | .PHONY: unit-test 106 | unit-test: autogen 107 | hack/unit-test.sh 108 | 109 | .PHONY: install-etcd 110 | install-etcd: 111 | hack/install-etcd.sh 112 | 113 | .PHONY: autogen 114 | autogen: update-vendor 115 | hack/update-generated-openapi.sh 116 | 117 | .PHONY: integration-test 118 | integration-test: install-etcd autogen 119 | hack/integration-test.sh 120 | 121 | .PHONY: verify-gofmt 122 | verify-gofmt: 123 | hack/verify-gofmt.sh 124 | 125 | .PHONY: clean 126 | clean: 127 | rm -rf ./bin 128 | -------------------------------------------------------------------------------- /pkg/apis/config/v1beta1/zz_generated.conversion.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by conversion-gen. DO NOT EDIT. 20 | 21 | package v1beta1 22 | 23 | import ( 24 | config "github.com/cwdsuzhou/super-scheduling/pkg/apis/config" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | conversion "k8s.io/apimachinery/pkg/conversion" 27 | runtime "k8s.io/apimachinery/pkg/runtime" 28 | ) 29 | 30 | func init() { 31 | localSchemeBuilder.Register(RegisterConversions) 32 | } 33 | 34 | // RegisterConversions adds conversion functions to the given scheme. 35 | // Public to allow building arbitrary schemes. 36 | func RegisterConversions(s *runtime.Scheme) error { 37 | if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*config.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { 38 | return Convert_v1beta1_Configuration_To_config_Configuration(a.(*Configuration), b.(*config.Configuration), scope) 39 | }); err != nil { 40 | return err 41 | } 42 | if err := s.AddGeneratedConversionFunc((*config.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { 43 | return Convert_config_Configuration_To_v1beta1_Configuration(a.(*config.Configuration), b.(*Configuration), scope) 44 | }); err != nil { 45 | return err 46 | } 47 | if err := s.AddGeneratedConversionFunc((*SchedulingArgs)(nil), (*config.SchedulingArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 48 | return Convert_v1beta1_SchedulingArgs_To_config_SchedulingArgs(a.(*SchedulingArgs), b.(*config.SchedulingArgs), scope) 49 | }); err != nil { 50 | return err 51 | } 52 | if err := s.AddGeneratedConversionFunc((*config.SchedulingArgs)(nil), (*SchedulingArgs)(nil), func(a, b interface{}, scope conversion.Scope) error { 53 | return Convert_config_SchedulingArgs_To_v1beta1_SchedulingArgs(a.(*config.SchedulingArgs), b.(*SchedulingArgs), scope) 54 | }); err != nil { 55 | return err 56 | } 57 | return nil 58 | } 59 | 60 | func autoConvert_v1beta1_Configuration_To_config_Configuration(in *Configuration, out *config.Configuration, s conversion.Scope) error { 61 | if err := v1.Convert_Pointer_string_To_string(&in.Name, &out.Name, s); err != nil { 62 | return err 63 | } 64 | if err := v1.Convert_Pointer_string_To_string(&in.KubeMaster, &out.KubeMaster, s); err != nil { 65 | return err 66 | } 67 | if err := v1.Convert_Pointer_string_To_string(&in.KubeConfig, &out.KubeConfig, s); err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | 73 | // Convert_v1beta1_Configuration_To_config_Configuration is an autogenerated conversion function. 74 | func Convert_v1beta1_Configuration_To_config_Configuration(in *Configuration, out *config.Configuration, s conversion.Scope) error { 75 | return autoConvert_v1beta1_Configuration_To_config_Configuration(in, out, s) 76 | } 77 | 78 | func autoConvert_config_Configuration_To_v1beta1_Configuration(in *config.Configuration, out *Configuration, s conversion.Scope) error { 79 | if err := v1.Convert_string_To_Pointer_string(&in.Name, &out.Name, s); err != nil { 80 | return err 81 | } 82 | if err := v1.Convert_string_To_Pointer_string(&in.KubeMaster, &out.KubeMaster, s); err != nil { 83 | return err 84 | } 85 | if err := v1.Convert_string_To_Pointer_string(&in.KubeConfig, &out.KubeConfig, s); err != nil { 86 | return err 87 | } 88 | return nil 89 | } 90 | 91 | // Convert_config_Configuration_To_v1beta1_Configuration is an autogenerated conversion function. 92 | func Convert_config_Configuration_To_v1beta1_Configuration(in *config.Configuration, out *Configuration, s conversion.Scope) error { 93 | return autoConvert_config_Configuration_To_v1beta1_Configuration(in, out, s) 94 | } 95 | 96 | func autoConvert_v1beta1_SchedulingArgs_To_config_SchedulingArgs(in *SchedulingArgs, out *config.SchedulingArgs, s conversion.Scope) error { 97 | if err := v1.Convert_Pointer_string_To_string(&in.KubeConfigPath, &out.KubeConfigPath, s); err != nil { 98 | return err 99 | } 100 | if err := v1.Convert_Pointer_string_To_string(&in.KubeMaster, &out.KubeMaster, s); err != nil { 101 | return err 102 | } 103 | if in.ClusterConfiguration != nil { 104 | in, out := &in.ClusterConfiguration, &out.ClusterConfiguration 105 | *out = make(map[string]config.Configuration, len(*in)) 106 | for key, val := range *in { 107 | newVal := new(config.Configuration) 108 | if err := Convert_v1beta1_Configuration_To_config_Configuration(&val, newVal, s); err != nil { 109 | return err 110 | } 111 | (*out)[key] = *newVal 112 | } 113 | } else { 114 | out.ClusterConfiguration = nil 115 | } 116 | return nil 117 | } 118 | 119 | // Convert_v1beta1_SchedulingArgs_To_config_SchedulingArgs is an autogenerated conversion function. 120 | func Convert_v1beta1_SchedulingArgs_To_config_SchedulingArgs(in *SchedulingArgs, out *config.SchedulingArgs, s conversion.Scope) error { 121 | return autoConvert_v1beta1_SchedulingArgs_To_config_SchedulingArgs(in, out, s) 122 | } 123 | 124 | func autoConvert_config_SchedulingArgs_To_v1beta1_SchedulingArgs(in *config.SchedulingArgs, out *SchedulingArgs, s conversion.Scope) error { 125 | if err := v1.Convert_string_To_Pointer_string(&in.KubeConfigPath, &out.KubeConfigPath, s); err != nil { 126 | return err 127 | } 128 | if err := v1.Convert_string_To_Pointer_string(&in.KubeMaster, &out.KubeMaster, s); err != nil { 129 | return err 130 | } 131 | if in.ClusterConfiguration != nil { 132 | in, out := &in.ClusterConfiguration, &out.ClusterConfiguration 133 | *out = make(map[string]Configuration, len(*in)) 134 | for key, val := range *in { 135 | newVal := new(Configuration) 136 | if err := Convert_config_Configuration_To_v1beta1_Configuration(&val, newVal, s); err != nil { 137 | return err 138 | } 139 | (*out)[key] = *newVal 140 | } 141 | } else { 142 | out.ClusterConfiguration = nil 143 | } 144 | return nil 145 | } 146 | 147 | // Convert_config_SchedulingArgs_To_v1beta1_SchedulingArgs is an autogenerated conversion function. 148 | func Convert_config_SchedulingArgs_To_v1beta1_SchedulingArgs(in *config.SchedulingArgs, out *SchedulingArgs, s conversion.Scope) error { 149 | return autoConvert_config_SchedulingArgs_To_v1beta1_SchedulingArgs(in, out, s) 150 | } 151 | -------------------------------------------------------------------------------- /pkg/apis/scheduling/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | runtime "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 29 | func (in *DeployPlacement) DeepCopyInto(out *DeployPlacement) { 30 | *out = *in 31 | if in.Policy != nil { 32 | in, out := &in.Policy, &out.Policy 33 | *out = make([]SchedulePolicy, len(*in)) 34 | copy(*out, *in) 35 | } 36 | return 37 | } 38 | 39 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeployPlacement. 40 | func (in *DeployPlacement) DeepCopy() *DeployPlacement { 41 | if in == nil { 42 | return nil 43 | } 44 | out := new(DeployPlacement) 45 | in.DeepCopyInto(out) 46 | return out 47 | } 48 | 49 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 50 | func (in *SchedulePolicy) DeepCopyInto(out *SchedulePolicy) { 51 | *out = *in 52 | return 53 | } 54 | 55 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulePolicy. 56 | func (in *SchedulePolicy) DeepCopy() *SchedulePolicy { 57 | if in == nil { 58 | return nil 59 | } 60 | out := new(SchedulePolicy) 61 | in.DeepCopyInto(out) 62 | return out 63 | } 64 | 65 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 66 | func (in *TopologySchedulingPolicy) DeepCopyInto(out *TopologySchedulingPolicy) { 67 | *out = *in 68 | out.TypeMeta = in.TypeMeta 69 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 70 | in.Spec.DeepCopyInto(&out.Spec) 71 | in.Status.DeepCopyInto(&out.Status) 72 | return 73 | } 74 | 75 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySchedulingPolicy. 76 | func (in *TopologySchedulingPolicy) DeepCopy() *TopologySchedulingPolicy { 77 | if in == nil { 78 | return nil 79 | } 80 | out := new(TopologySchedulingPolicy) 81 | in.DeepCopyInto(out) 82 | return out 83 | } 84 | 85 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 86 | func (in *TopologySchedulingPolicy) DeepCopyObject() runtime.Object { 87 | if c := in.DeepCopy(); c != nil { 88 | return c 89 | } 90 | return nil 91 | } 92 | 93 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 94 | func (in *TopologySchedulingPolicyList) DeepCopyInto(out *TopologySchedulingPolicyList) { 95 | *out = *in 96 | out.TypeMeta = in.TypeMeta 97 | in.ListMeta.DeepCopyInto(&out.ListMeta) 98 | if in.Items != nil { 99 | in, out := &in.Items, &out.Items 100 | *out = make([]TopologySchedulingPolicy, len(*in)) 101 | for i := range *in { 102 | (*in)[i].DeepCopyInto(&(*out)[i]) 103 | } 104 | } 105 | return 106 | } 107 | 108 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySchedulingPolicyList. 109 | func (in *TopologySchedulingPolicyList) DeepCopy() *TopologySchedulingPolicyList { 110 | if in == nil { 111 | return nil 112 | } 113 | out := new(TopologySchedulingPolicyList) 114 | in.DeepCopyInto(out) 115 | return out 116 | } 117 | 118 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 119 | func (in *TopologySchedulingPolicyList) DeepCopyObject() runtime.Object { 120 | if c := in.DeepCopy(); c != nil { 121 | return c 122 | } 123 | return nil 124 | } 125 | 126 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 127 | func (in *TopologySchedulingPolicySpec) DeepCopyInto(out *TopologySchedulingPolicySpec) { 128 | *out = *in 129 | if in.DeployPlacement != nil { 130 | in, out := &in.DeployPlacement, &out.DeployPlacement 131 | *out = make([]SchedulePolicy, len(*in)) 132 | copy(*out, *in) 133 | } 134 | if in.UpdatePlacement != nil { 135 | in, out := &in.UpdatePlacement, &out.UpdatePlacement 136 | *out = make([]UpdatePolicy, len(*in)) 137 | for i := range *in { 138 | (*in)[i].DeepCopyInto(&(*out)[i]) 139 | } 140 | } 141 | if in.LabelSelector != nil { 142 | in, out := &in.LabelSelector, &out.LabelSelector 143 | *out = new(v1.LabelSelector) 144 | (*in).DeepCopyInto(*out) 145 | } 146 | return 147 | } 148 | 149 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySchedulingPolicySpec. 150 | func (in *TopologySchedulingPolicySpec) DeepCopy() *TopologySchedulingPolicySpec { 151 | if in == nil { 152 | return nil 153 | } 154 | out := new(TopologySchedulingPolicySpec) 155 | in.DeepCopyInto(out) 156 | return out 157 | } 158 | 159 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 160 | func (in *TopologySchedulingPolicyStatus) DeepCopyInto(out *TopologySchedulingPolicyStatus) { 161 | *out = *in 162 | if in.Placement != nil { 163 | in, out := &in.Placement, &out.Placement 164 | *out = make([]SchedulePolicy, len(*in)) 165 | copy(*out, *in) 166 | } 167 | return 168 | } 169 | 170 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologySchedulingPolicyStatus. 171 | func (in *TopologySchedulingPolicyStatus) DeepCopy() *TopologySchedulingPolicyStatus { 172 | if in == nil { 173 | return nil 174 | } 175 | out := new(TopologySchedulingPolicyStatus) 176 | in.DeepCopyInto(out) 177 | return out 178 | } 179 | 180 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 181 | func (in *UpdatePolicy) DeepCopyInto(out *UpdatePolicy) { 182 | *out = *in 183 | if in.UpdateSequence != nil { 184 | in, out := &in.UpdateSequence, &out.UpdateSequence 185 | *out = make([]string, len(*in)) 186 | copy(*out, *in) 187 | } 188 | return 189 | } 190 | 191 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdatePolicy. 192 | func (in *UpdatePolicy) DeepCopy() *UpdatePolicy { 193 | if in == nil { 194 | return nil 195 | } 196 | out := new(UpdatePolicy) 197 | in.DeepCopyInto(out) 198 | return out 199 | } 200 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/fake/fake_topologyschedulingpolicy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | "context" 23 | 24 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 25 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | labels "k8s.io/apimachinery/pkg/labels" 27 | schema "k8s.io/apimachinery/pkg/runtime/schema" 28 | types "k8s.io/apimachinery/pkg/types" 29 | watch "k8s.io/apimachinery/pkg/watch" 30 | testing "k8s.io/client-go/testing" 31 | ) 32 | 33 | // FakeTopologySchedulingPolicies implements TopologySchedulingPolicyInterface 34 | type FakeTopologySchedulingPolicies struct { 35 | Fake *FakeSchedulingV1alpha1 36 | ns string 37 | } 38 | 39 | var topologyschedulingpoliciesResource = schema.GroupVersionResource{Group: "scheduling.sigs.k8s.io", Version: "v1alpha1", Resource: "topologyschedulingpolicies"} 40 | 41 | var topologyschedulingpoliciesKind = schema.GroupVersionKind{Group: "scheduling.sigs.k8s.io", Version: "v1alpha1", Kind: "TopologySchedulingPolicy"} 42 | 43 | // Get takes name of the topologySchedulingPolicy, and returns the corresponding topologySchedulingPolicy object, and an error if there is any. 44 | func (c *FakeTopologySchedulingPolicies) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 45 | obj, err := c.Fake. 46 | Invokes(testing.NewGetAction(topologyschedulingpoliciesResource, c.ns, name), &v1alpha1.TopologySchedulingPolicy{}) 47 | 48 | if obj == nil { 49 | return nil, err 50 | } 51 | return obj.(*v1alpha1.TopologySchedulingPolicy), err 52 | } 53 | 54 | // List takes label and field selectors, and returns the list of TopologySchedulingPolicies that match those selectors. 55 | func (c *FakeTopologySchedulingPolicies) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.TopologySchedulingPolicyList, err error) { 56 | obj, err := c.Fake. 57 | Invokes(testing.NewListAction(topologyschedulingpoliciesResource, topologyschedulingpoliciesKind, c.ns, opts), &v1alpha1.TopologySchedulingPolicyList{}) 58 | 59 | if obj == nil { 60 | return nil, err 61 | } 62 | 63 | label, _, _ := testing.ExtractFromListOptions(opts) 64 | if label == nil { 65 | label = labels.Everything() 66 | } 67 | list := &v1alpha1.TopologySchedulingPolicyList{ListMeta: obj.(*v1alpha1.TopologySchedulingPolicyList).ListMeta} 68 | for _, item := range obj.(*v1alpha1.TopologySchedulingPolicyList).Items { 69 | if label.Matches(labels.Set(item.Labels)) { 70 | list.Items = append(list.Items, item) 71 | } 72 | } 73 | return list, err 74 | } 75 | 76 | // Watch returns a watch.Interface that watches the requested topologySchedulingPolicies. 77 | func (c *FakeTopologySchedulingPolicies) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { 78 | return c.Fake. 79 | InvokesWatch(testing.NewWatchAction(topologyschedulingpoliciesResource, c.ns, opts)) 80 | 81 | } 82 | 83 | // Create takes the representation of a topologySchedulingPolicy and creates it. Returns the server's representation of the topologySchedulingPolicy, and an error, if there is any. 84 | func (c *FakeTopologySchedulingPolicies) Create(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.CreateOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 85 | obj, err := c.Fake. 86 | Invokes(testing.NewCreateAction(topologyschedulingpoliciesResource, c.ns, topologySchedulingPolicy), &v1alpha1.TopologySchedulingPolicy{}) 87 | 88 | if obj == nil { 89 | return nil, err 90 | } 91 | return obj.(*v1alpha1.TopologySchedulingPolicy), err 92 | } 93 | 94 | // Update takes the representation of a topologySchedulingPolicy and updates it. Returns the server's representation of the topologySchedulingPolicy, and an error, if there is any. 95 | func (c *FakeTopologySchedulingPolicies) Update(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 96 | obj, err := c.Fake. 97 | Invokes(testing.NewUpdateAction(topologyschedulingpoliciesResource, c.ns, topologySchedulingPolicy), &v1alpha1.TopologySchedulingPolicy{}) 98 | 99 | if obj == nil { 100 | return nil, err 101 | } 102 | return obj.(*v1alpha1.TopologySchedulingPolicy), err 103 | } 104 | 105 | // UpdateStatus was generated because the type contains a Status member. 106 | // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). 107 | func (c *FakeTopologySchedulingPolicies) UpdateStatus(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (*v1alpha1.TopologySchedulingPolicy, error) { 108 | obj, err := c.Fake. 109 | Invokes(testing.NewUpdateSubresourceAction(topologyschedulingpoliciesResource, "status", c.ns, topologySchedulingPolicy), &v1alpha1.TopologySchedulingPolicy{}) 110 | 111 | if obj == nil { 112 | return nil, err 113 | } 114 | return obj.(*v1alpha1.TopologySchedulingPolicy), err 115 | } 116 | 117 | // Delete takes name of the topologySchedulingPolicy and deletes it. Returns an error if one occurs. 118 | func (c *FakeTopologySchedulingPolicies) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { 119 | _, err := c.Fake. 120 | Invokes(testing.NewDeleteAction(topologyschedulingpoliciesResource, c.ns, name), &v1alpha1.TopologySchedulingPolicy{}) 121 | 122 | return err 123 | } 124 | 125 | // DeleteCollection deletes a collection of objects. 126 | func (c *FakeTopologySchedulingPolicies) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { 127 | action := testing.NewDeleteCollectionAction(topologyschedulingpoliciesResource, c.ns, listOpts) 128 | 129 | _, err := c.Fake.Invokes(action, &v1alpha1.TopologySchedulingPolicyList{}) 130 | return err 131 | } 132 | 133 | // Patch applies the patch and returns the patched topologySchedulingPolicy. 134 | func (c *FakeTopologySchedulingPolicies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.TopologySchedulingPolicy, err error) { 135 | obj, err := c.Fake. 136 | Invokes(testing.NewPatchSubresourceAction(topologyschedulingpoliciesResource, c.ns, name, pt, data, subresources...), &v1alpha1.TopologySchedulingPolicy{}) 137 | 138 | if obj == nil { 139 | return nil, err 140 | } 141 | return obj.(*v1alpha1.TopologySchedulingPolicy), err 142 | } 143 | -------------------------------------------------------------------------------- /pkg/descheduler/descheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package descheduler 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "k8s.io/client-go/rest" 23 | "k8s.io/client-go/tools/clientcmd" 24 | 25 | v1 "k8s.io/api/core/v1" 26 | clientset "k8s.io/client-go/kubernetes" 27 | "k8s.io/klog/v2" 28 | 29 | "k8s.io/apimachinery/pkg/util/wait" 30 | "k8s.io/client-go/informers" 31 | "sigs.k8s.io/descheduler/pkg/api" 32 | de "sigs.k8s.io/descheduler/pkg/descheduler" 33 | "sigs.k8s.io/descheduler/pkg/descheduler/client" 34 | "sigs.k8s.io/descheduler/pkg/descheduler/evictions" 35 | eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" 36 | nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" 37 | "sigs.k8s.io/descheduler/pkg/descheduler/strategies" 38 | 39 | "github.com/cwdsuzhou/super-scheduling/cmd/descheduler/app/options" 40 | sgs "github.com/cwdsuzhou/super-scheduling/pkg/descheduler/strategies" 41 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 42 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions" 43 | ) 44 | 45 | func CreateClient(kubeconfig string) (versioned.Interface, error) { 46 | var cfg *rest.Config 47 | if len(kubeconfig) != 0 { 48 | master, err := client.GetMasterFromKubeconfig(kubeconfig) 49 | if err != nil { 50 | return nil, fmt.Errorf("Failed to parse kubeconfig file: %v ", err) 51 | } 52 | 53 | cfg, err = clientcmd.BuildConfigFromFlags(master, kubeconfig) 54 | if err != nil { 55 | return nil, fmt.Errorf("Unable to build config: %v", err) 56 | } 57 | 58 | } else { 59 | var err error 60 | cfg, err = rest.InClusterConfig() 61 | if err != nil { 62 | return nil, fmt.Errorf("Unable to build in cluster config: %v", err) 63 | } 64 | } 65 | 66 | return versioned.NewForConfig(cfg) 67 | } 68 | 69 | func Run(rs *options.DeschedulerServer) error { 70 | ctx := context.Background() 71 | rsclient, err := client.CreateClient(rs.KubeconfigFile) 72 | if err != nil { 73 | return err 74 | } 75 | rs.Client = rsclient 76 | 77 | tspclient, err := CreateClient(rs.KubeconfigFile) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | rs.TopologySchedulingClient = tspclient 83 | 84 | deschedulerPolicy, err := de.LoadPolicyConfig(rs.PolicyConfigFile) 85 | if err != nil { 86 | return err 87 | } 88 | if deschedulerPolicy == nil { 89 | return fmt.Errorf("deschedulerPolicy is nil") 90 | } 91 | 92 | evictionPolicyGroupVersion, err := eutils.SupportEviction(rs.Client) 93 | if err != nil || len(evictionPolicyGroupVersion) == 0 { 94 | return err 95 | } 96 | 97 | stopChannel := make(chan struct{}) 98 | return RunDeschedulerStrategies(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, stopChannel) 99 | } 100 | 101 | type strategyFunction func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor) 102 | 103 | func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, stopChannel chan struct{}) error { 104 | sharedInformerFactory := informers.NewSharedInformerFactory(rs.Client, 0) 105 | nodeInformer := sharedInformerFactory.Core().V1().Nodes() 106 | 107 | tspInformerFactory := externalversions.NewSharedInformerFactory(rs.TopologySchedulingClient, 0) 108 | tspInformer := tspInformerFactory.Scheduling().V1alpha1().TopologySchedulingPolicies() 109 | _ = tspInformer.Informer() 110 | tspLister := tspInformer.Lister() 111 | 112 | sharedInformerFactory.Start(stopChannel) 113 | sharedInformerFactory.WaitForCacheSync(stopChannel) 114 | 115 | tspInformerFactory.Start(stopChannel) 116 | tspInformerFactory.WaitForCacheSync(stopChannel) 117 | 118 | strategyFuncs := map[string]strategyFunction{ 119 | "RemoveDuplicates": strategies.RemoveDuplicatePods, 120 | "LowNodeUtilization": strategies.LowNodeUtilization, 121 | "RemovePodsViolatingInterPodAntiAffinity": strategies.RemovePodsViolatingInterPodAntiAffinity, 122 | "RemovePodsViolatingNodeAffinity": strategies.RemovePodsViolatingNodeAffinity, 123 | "RemovePodsViolatingNodeTaints": strategies.RemovePodsViolatingNodeTaints, 124 | "RemovePodsHavingTooManyRestarts": strategies.RemovePodsHavingTooManyRestarts, 125 | "PodLifeTime": strategies.PodLifeTime, 126 | "RemovePodsViolatingTopologySpreadConstraint": strategies.RemovePodsViolatingTopologySpreadConstraint, 127 | "RemovePodsViolatingTopologySchedulingPolicy": func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor) { 128 | sgs.RemovePodsViolatingTopologySchedulingPolicy(ctx, client, strategy, nodes, podEvictor, tspLister) 129 | }, 130 | } 131 | 132 | nodeSelector := rs.NodeSelector 133 | if deschedulerPolicy.NodeSelector != nil { 134 | nodeSelector = *deschedulerPolicy.NodeSelector 135 | } 136 | 137 | evictLocalStoragePods := rs.EvictLocalStoragePods 138 | if deschedulerPolicy.EvictLocalStoragePods != nil { 139 | evictLocalStoragePods = *deschedulerPolicy.EvictLocalStoragePods 140 | } 141 | 142 | maxNoOfPodsToEvictPerNode := rs.MaxNoOfPodsToEvictPerNode 143 | if deschedulerPolicy.MaxNoOfPodsToEvictPerNode != nil { 144 | maxNoOfPodsToEvictPerNode = *deschedulerPolicy.MaxNoOfPodsToEvictPerNode 145 | } 146 | 147 | wait.Until(func() { 148 | nodes, err := nodeutil.ReadyNodes(ctx, rs.Client, nodeInformer, nodeSelector, stopChannel) 149 | if err != nil { 150 | klog.V(1).InfoS("Unable to get ready nodes", "err", err) 151 | close(stopChannel) 152 | return 153 | } 154 | 155 | if len(nodes) <= 1 { 156 | klog.V(1).InfoS("The cluster size is 0 or 1 meaning eviction causes service disruption or degradation. So aborting..") 157 | close(stopChannel) 158 | return 159 | } 160 | 161 | podEvictor := evictions.NewPodEvictor( 162 | rs.Client, 163 | evictionPolicyGroupVersion, 164 | rs.DryRun, 165 | maxNoOfPodsToEvictPerNode, 166 | nodes, 167 | evictLocalStoragePods, 168 | ) 169 | 170 | for name, f := range strategyFuncs { 171 | if strategy := deschedulerPolicy.Strategies[api.StrategyName(name)]; strategy.Enabled { 172 | f(ctx, rs.Client, strategy, nodes, podEvictor) 173 | } 174 | } 175 | 176 | // If there was no interval specified, send a signal to the stopChannel to end the wait.Until loop after 1 iteration 177 | if rs.DeschedulingInterval.Seconds() == 0 { 178 | close(stopChannel) 179 | } 180 | }, rs.DeschedulingInterval, stopChannel) 181 | 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /pkg/generated/informers/externalversions/factory.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by informer-gen. DO NOT EDIT. 18 | 19 | package externalversions 20 | 21 | import ( 22 | reflect "reflect" 23 | sync "sync" 24 | time "time" 25 | 26 | versioned "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 27 | internalinterfaces "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/internalinterfaces" 28 | scheduling "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions/scheduling" 29 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 | runtime "k8s.io/apimachinery/pkg/runtime" 31 | schema "k8s.io/apimachinery/pkg/runtime/schema" 32 | cache "k8s.io/client-go/tools/cache" 33 | ) 34 | 35 | // SharedInformerOption defines the functional option type for SharedInformerFactory. 36 | type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory 37 | 38 | type sharedInformerFactory struct { 39 | client versioned.Interface 40 | namespace string 41 | tweakListOptions internalinterfaces.TweakListOptionsFunc 42 | lock sync.Mutex 43 | defaultResync time.Duration 44 | customResync map[reflect.Type]time.Duration 45 | 46 | informers map[reflect.Type]cache.SharedIndexInformer 47 | // startedInformers is used for tracking which informers have been started. 48 | // This allows Start() to be called multiple times safely. 49 | startedInformers map[reflect.Type]bool 50 | } 51 | 52 | // WithCustomResyncConfig sets a custom resync period for the specified informer types. 53 | func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { 54 | return func(factory *sharedInformerFactory) *sharedInformerFactory { 55 | for k, v := range resyncConfig { 56 | factory.customResync[reflect.TypeOf(k)] = v 57 | } 58 | return factory 59 | } 60 | } 61 | 62 | // WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. 63 | func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { 64 | return func(factory *sharedInformerFactory) *sharedInformerFactory { 65 | factory.tweakListOptions = tweakListOptions 66 | return factory 67 | } 68 | } 69 | 70 | // WithNamespace limits the SharedInformerFactory to the specified namespace. 71 | func WithNamespace(namespace string) SharedInformerOption { 72 | return func(factory *sharedInformerFactory) *sharedInformerFactory { 73 | factory.namespace = namespace 74 | return factory 75 | } 76 | } 77 | 78 | // NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. 79 | func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { 80 | return NewSharedInformerFactoryWithOptions(client, defaultResync) 81 | } 82 | 83 | // NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. 84 | // Listers obtained via this SharedInformerFactory will be subject to the same filters 85 | // as specified here. 86 | // Deprecated: Please use NewSharedInformerFactoryWithOptions instead 87 | func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { 88 | return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) 89 | } 90 | 91 | // NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. 92 | func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { 93 | factory := &sharedInformerFactory{ 94 | client: client, 95 | namespace: v1.NamespaceAll, 96 | defaultResync: defaultResync, 97 | informers: make(map[reflect.Type]cache.SharedIndexInformer), 98 | startedInformers: make(map[reflect.Type]bool), 99 | customResync: make(map[reflect.Type]time.Duration), 100 | } 101 | 102 | // Apply all options 103 | for _, opt := range options { 104 | factory = opt(factory) 105 | } 106 | 107 | return factory 108 | } 109 | 110 | // Start initializes all requested informers. 111 | func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { 112 | f.lock.Lock() 113 | defer f.lock.Unlock() 114 | 115 | for informerType, informer := range f.informers { 116 | if !f.startedInformers[informerType] { 117 | go informer.Run(stopCh) 118 | f.startedInformers[informerType] = true 119 | } 120 | } 121 | } 122 | 123 | // WaitForCacheSync waits for all started informers' cache were synced. 124 | func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { 125 | informers := func() map[reflect.Type]cache.SharedIndexInformer { 126 | f.lock.Lock() 127 | defer f.lock.Unlock() 128 | 129 | informers := map[reflect.Type]cache.SharedIndexInformer{} 130 | for informerType, informer := range f.informers { 131 | if f.startedInformers[informerType] { 132 | informers[informerType] = informer 133 | } 134 | } 135 | return informers 136 | }() 137 | 138 | res := map[reflect.Type]bool{} 139 | for informType, informer := range informers { 140 | res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) 141 | } 142 | return res 143 | } 144 | 145 | // InternalInformerFor returns the SharedIndexInformer for obj using an internal 146 | // client. 147 | func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { 148 | f.lock.Lock() 149 | defer f.lock.Unlock() 150 | 151 | informerType := reflect.TypeOf(obj) 152 | informer, exists := f.informers[informerType] 153 | if exists { 154 | return informer 155 | } 156 | 157 | resyncPeriod, exists := f.customResync[informerType] 158 | if !exists { 159 | resyncPeriod = f.defaultResync 160 | } 161 | 162 | informer = newFunc(f.client, resyncPeriod) 163 | f.informers[informerType] = informer 164 | 165 | return informer 166 | } 167 | 168 | // SharedInformerFactory provides shared informers for resources in all known 169 | // API group versions. 170 | type SharedInformerFactory interface { 171 | internalinterfaces.SharedInformerFactory 172 | ForResource(resource schema.GroupVersionResource) (GenericInformer, error) 173 | WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool 174 | 175 | Scheduling() scheduling.Interface 176 | } 177 | 178 | func (f *sharedInformerFactory) Scheduling() scheduling.Interface { 179 | return scheduling.New(f, f.namespace, f.tweakListOptions) 180 | } 181 | -------------------------------------------------------------------------------- /pkg/generated/clientset/versioned/typed/scheduling/v1alpha1/topologyschedulingpolicy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "context" 23 | "time" 24 | 25 | v1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 26 | scheme "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned/scheme" 27 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | types "k8s.io/apimachinery/pkg/types" 29 | watch "k8s.io/apimachinery/pkg/watch" 30 | rest "k8s.io/client-go/rest" 31 | ) 32 | 33 | // TopologySchedulingPoliciesGetter has a method to return a TopologySchedulingPolicyInterface. 34 | // A group's client should implement this interface. 35 | type TopologySchedulingPoliciesGetter interface { 36 | TopologySchedulingPolicies(namespace string) TopologySchedulingPolicyInterface 37 | } 38 | 39 | // TopologySchedulingPolicyInterface has methods to work with TopologySchedulingPolicy resources. 40 | type TopologySchedulingPolicyInterface interface { 41 | Create(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.CreateOptions) (*v1alpha1.TopologySchedulingPolicy, error) 42 | Update(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (*v1alpha1.TopologySchedulingPolicy, error) 43 | UpdateStatus(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (*v1alpha1.TopologySchedulingPolicy, error) 44 | Delete(ctx context.Context, name string, opts v1.DeleteOptions) error 45 | DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error 46 | Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.TopologySchedulingPolicy, error) 47 | List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.TopologySchedulingPolicyList, error) 48 | Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) 49 | Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.TopologySchedulingPolicy, err error) 50 | TopologySchedulingPolicyExpansion 51 | } 52 | 53 | // topologySchedulingPolicies implements TopologySchedulingPolicyInterface 54 | type topologySchedulingPolicies struct { 55 | client rest.Interface 56 | ns string 57 | } 58 | 59 | // newTopologySchedulingPolicies returns a TopologySchedulingPolicies 60 | func newTopologySchedulingPolicies(c *SchedulingV1alpha1Client, namespace string) *topologySchedulingPolicies { 61 | return &topologySchedulingPolicies{ 62 | client: c.RESTClient(), 63 | ns: namespace, 64 | } 65 | } 66 | 67 | // Get takes name of the topologySchedulingPolicy, and returns the corresponding topologySchedulingPolicy object, and an error if there is any. 68 | func (c *topologySchedulingPolicies) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 69 | result = &v1alpha1.TopologySchedulingPolicy{} 70 | err = c.client.Get(). 71 | Namespace(c.ns). 72 | Resource("topologyschedulingpolicies"). 73 | Name(name). 74 | VersionedParams(&options, scheme.ParameterCodec). 75 | Do(ctx). 76 | Into(result) 77 | return 78 | } 79 | 80 | // List takes label and field selectors, and returns the list of TopologySchedulingPolicies that match those selectors. 81 | func (c *topologySchedulingPolicies) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.TopologySchedulingPolicyList, err error) { 82 | var timeout time.Duration 83 | if opts.TimeoutSeconds != nil { 84 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 85 | } 86 | result = &v1alpha1.TopologySchedulingPolicyList{} 87 | err = c.client.Get(). 88 | Namespace(c.ns). 89 | Resource("topologyschedulingpolicies"). 90 | VersionedParams(&opts, scheme.ParameterCodec). 91 | Timeout(timeout). 92 | Do(ctx). 93 | Into(result) 94 | return 95 | } 96 | 97 | // Watch returns a watch.Interface that watches the requested topologySchedulingPolicies. 98 | func (c *topologySchedulingPolicies) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { 99 | var timeout time.Duration 100 | if opts.TimeoutSeconds != nil { 101 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 102 | } 103 | opts.Watch = true 104 | return c.client.Get(). 105 | Namespace(c.ns). 106 | Resource("topologyschedulingpolicies"). 107 | VersionedParams(&opts, scheme.ParameterCodec). 108 | Timeout(timeout). 109 | Watch(ctx) 110 | } 111 | 112 | // Create takes the representation of a topologySchedulingPolicy and creates it. Returns the server's representation of the topologySchedulingPolicy, and an error, if there is any. 113 | func (c *topologySchedulingPolicies) Create(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.CreateOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 114 | result = &v1alpha1.TopologySchedulingPolicy{} 115 | err = c.client.Post(). 116 | Namespace(c.ns). 117 | Resource("topologyschedulingpolicies"). 118 | VersionedParams(&opts, scheme.ParameterCodec). 119 | Body(topologySchedulingPolicy). 120 | Do(ctx). 121 | Into(result) 122 | return 123 | } 124 | 125 | // Update takes the representation of a topologySchedulingPolicy and updates it. Returns the server's representation of the topologySchedulingPolicy, and an error, if there is any. 126 | func (c *topologySchedulingPolicies) Update(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 127 | result = &v1alpha1.TopologySchedulingPolicy{} 128 | err = c.client.Put(). 129 | Namespace(c.ns). 130 | Resource("topologyschedulingpolicies"). 131 | Name(topologySchedulingPolicy.Name). 132 | VersionedParams(&opts, scheme.ParameterCodec). 133 | Body(topologySchedulingPolicy). 134 | Do(ctx). 135 | Into(result) 136 | return 137 | } 138 | 139 | // UpdateStatus was generated because the type contains a Status member. 140 | // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). 141 | func (c *topologySchedulingPolicies) UpdateStatus(ctx context.Context, topologySchedulingPolicy *v1alpha1.TopologySchedulingPolicy, opts v1.UpdateOptions) (result *v1alpha1.TopologySchedulingPolicy, err error) { 142 | result = &v1alpha1.TopologySchedulingPolicy{} 143 | err = c.client.Put(). 144 | Namespace(c.ns). 145 | Resource("topologyschedulingpolicies"). 146 | Name(topologySchedulingPolicy.Name). 147 | SubResource("status"). 148 | VersionedParams(&opts, scheme.ParameterCodec). 149 | Body(topologySchedulingPolicy). 150 | Do(ctx). 151 | Into(result) 152 | return 153 | } 154 | 155 | // Delete takes name of the topologySchedulingPolicy and deletes it. Returns an error if one occurs. 156 | func (c *topologySchedulingPolicies) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { 157 | return c.client.Delete(). 158 | Namespace(c.ns). 159 | Resource("topologyschedulingpolicies"). 160 | Name(name). 161 | Body(&opts). 162 | Do(ctx). 163 | Error() 164 | } 165 | 166 | // DeleteCollection deletes a collection of objects. 167 | func (c *topologySchedulingPolicies) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { 168 | var timeout time.Duration 169 | if listOpts.TimeoutSeconds != nil { 170 | timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second 171 | } 172 | return c.client.Delete(). 173 | Namespace(c.ns). 174 | Resource("topologyschedulingpolicies"). 175 | VersionedParams(&listOpts, scheme.ParameterCodec). 176 | Timeout(timeout). 177 | Body(&opts). 178 | Do(ctx). 179 | Error() 180 | } 181 | 182 | // Patch applies the patch and returns the patched topologySchedulingPolicy. 183 | func (c *topologySchedulingPolicies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.TopologySchedulingPolicy, err error) { 184 | result = &v1alpha1.TopologySchedulingPolicy{} 185 | err = c.client.Patch(pt). 186 | Namespace(c.ns). 187 | Resource("topologyschedulingpolicies"). 188 | Name(name). 189 | SubResource(subresources...). 190 | VersionedParams(&opts, scheme.ParameterCodec). 191 | Body(data). 192 | Do(ctx). 193 | Into(result) 194 | return 195 | } 196 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Super scheduling 2 | 3 | ## Introduction 4 | 5 | This project includes a topology-scheduler and a descheduler extened 6 | from [descheduler](https://github.com/kubernetes-sigs/descheduler.git). 7 | 8 | topology-scheduler will help scheduling pods cross zones, regions or clusters. 9 | 10 | > We would like this project be merged by upstream in the future, so crd and codes includes xxx.scheduling.sigs.k8s.io 11 | 12 | ## Why we need this 13 | 14 | `TopologySpreadConstraint` helps schedule pods with desired skew, but it can not solve the issue: schedule desired 15 | replicas to a `zone`, `region` or `cluster`, e.g. 16 | 17 | ```yaml 18 | zoneA: 6 Pods 19 | zoneB: 1 Pods 20 | zoneC: 2 Pods 21 | ``` 22 | 23 | ## Install 24 | 25 | ### kube-scheduler 26 | 27 | *1* Apply crd 28 | 29 | ```yaml 30 | apiVersion: apiextensions.k8s.io/v1beta1 31 | kind: CustomResourceDefinition 32 | metadata: 33 | name: topologyschedulingpolicies.scheduling.sigs.k8s.io 34 | spec: 35 | conversion: 36 | strategy: None 37 | group: scheduling.sigs.k8s.io 38 | names: 39 | kind: TopologySchedulingPolicy 40 | listKind: TopologySchedulingPolicyList 41 | plural: topologyschedulingpolicies 42 | shortNames: 43 | - tsp 44 | - tsps 45 | singular: topologyschedulingpolicy 46 | scope: Namespaced 47 | version: v1alpha1 48 | versions: 49 | - name: v1alpha1 50 | served: true 51 | storage: true 52 | ``` 53 | 54 | > if your cluster only support kubescheduler.config.k8s.io/v1, please replace this with v1. 55 | 56 | *2* deploy scheduler 57 | 58 | Replace the kube-scheudler with this one, and add a config like this when starting scheduler. 59 | 60 | ```yaml 61 | apiVersion: kubescheduler.config.k8s.io/v1beta1 62 | kind: KubeSchedulerConfiguration 63 | leaderElection: 64 | leaderElect: true 65 | clientConnection: 66 | kubeconfig: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 67 | profiles: 68 | - schedulerName: default-multicluster 69 | plugins: 70 | preFilter: 71 | enabled: 72 | - name: TopologyScheduling 73 | filter: 74 | enabled: 75 | - name: TopologyScheduling 76 | disabled: 77 | - name: "*" 78 | score: 79 | enabled: 80 | - name: TopologyScheduling 81 | disabled: 82 | - name: "*" 83 | reserve: 84 | enabled: 85 | - name: TopologyScheduling 86 | pluginConfig: 87 | - name: TopologyScheduling 88 | args: 89 | kubeConfigPath: "REPLACE_ME_WITH_KUBE_CONFIG_PATH" 90 | ``` 91 | > If you want to enable multi-cluster, enable the MultiClusterScheduling in the config, as follow: 92 | ``` 93 | filter: 94 | enabled: 95 | - name: MultiClusterScheduling 96 | ``` 97 | 98 | *3* deploy descheduler 99 | 100 | descheduler should be deployed as deployment in cluster 101 | 102 | ```yaml 103 | --- 104 | apiVersion: v1 105 | kind: ConfigMap 106 | metadata: 107 | name: descheduler-policy-configmap 108 | namespace: kube-system 109 | data: 110 | policy.yaml: | 111 | apiVersion: "descheduler/v1alpha1" 112 | kind: "DeschedulerPolicy" 113 | strategies: 114 | RemovePodsViolatingTopologySchedulingPolicy: 115 | enabled: true 116 | --- 117 | kind: ClusterRole 118 | apiVersion: rbac.authorization.k8s.io/v1 119 | metadata: 120 | name: descheduler-cluster-role 121 | namespace: kube-system 122 | rules: 123 | - apiGroups: [""] 124 | resources: ["events"] 125 | verbs: ["create", "update"] 126 | - apiGroups: [""] 127 | resources: ["nodes"] 128 | verbs: ["get", "watch", "list"] 129 | - apiGroups: [""] 130 | resources: ["pods"] 131 | verbs: ["create", "get", "watch", "list", "delete", "patch"] 132 | - apiGroups: [""] 133 | resources: ["pods/eviction"] 134 | verbs: ["create"] 135 | --- 136 | apiVersion: v1 137 | kind: ServiceAccount 138 | metadata: 139 | name: descheduler-sa 140 | namespace: kube-system 141 | --- 142 | apiVersion: rbac.authorization.k8s.io/v1 143 | kind: ClusterRoleBinding 144 | metadata: 145 | name: descheduler-cluster-role-binding 146 | namespace: kube-system 147 | roleRef: 148 | apiGroup: rbac.authorization.k8s.io 149 | kind: ClusterRole 150 | name: descheduler-cluster-role 151 | subjects: 152 | - name: descheduler-sa 153 | kind: ServiceAccount 154 | namespace: kube-system 155 | --- 156 | apiVersion: apps/v1 157 | kind: Deployment 158 | metadata: 159 | name: descheduler 160 | namespace: kube-system 161 | spec: 162 | selector: 163 | matchLabels: 164 | app: descheduler 165 | replicas: 1 166 | template: 167 | metadata: 168 | labels: 169 | app: descheduler 170 | spec: 171 | affinity: 172 | nodeAffinity: 173 | requiredDuringSchedulingIgnoredDuringExecution: 174 | nodeSelectorTerms: 175 | - matchExpressions: 176 | - key: type 177 | operator: NotIn 178 | values: 179 | - virtual-kubelet 180 | tolerations: 181 | - effect: NoSchedule 182 | key: role 183 | value: not-vk 184 | operator: Equal 185 | priorityClassName: system-cluster-critical 186 | containers: 187 | - name: descheduler 188 | image: ${you image} 189 | volumeMounts: 190 | - mountPath: /policy-dir 191 | name: policy-volume 192 | command: 193 | - "/bin/descheduler" 194 | args: 195 | - "--policy-config-file=/policy-dir/policy.yaml" 196 | - "--v=3" 197 | restartPolicy: "Always" 198 | serviceAccountName: descheduler-sa 199 | volumes: 200 | - name: policy-volume 201 | configMap: 202 | name: descheduler-policy-configmap 203 | ``` 204 | 205 | ## Use Case 206 | 207 | ### multi zone 208 | 209 | ```yaml 210 | apiVersion: scheduling.sigs.k8s.io/v1alpha1 211 | kind: TopologySchedulingPolicy 212 | metadata: 213 | name: policy-zone 214 | spec: 215 | deployPlacement: 216 | - name: sh-1 217 | replicas: 6 218 | - name: nj-2 219 | replicas: 3 220 | labelSelector: 221 | matchLabels: 222 | cluster-test: "true" 223 | topologyKey: failure-domain.beta.kubernetes.io/zone 224 | ``` 225 | 226 | ```yaml 227 | apiVersion: apps/v1 228 | kind: Deployment 229 | metadata: 230 | labels: 231 | app: env 232 | name: test 233 | namespace: default 234 | spec: 235 | replicas: 9 236 | selector: 237 | matchLabels: 238 | app: env 239 | template: 240 | labels: 241 | app: env 242 | cluster-test: "true" 243 | topology-scheduling-policy.scheduling.sigs.k8s.io: policy-zone 244 | spec: 245 | containers: 246 | - image: nginx:latest 247 | imagePullPolicy: Always 248 | name: nginx 249 | resources: { } 250 | ``` 251 | 252 | ### multi region 253 | 254 | ```yaml 255 | apiVersion: scheduling.sigs.k8s.io/v1alpha1 256 | kind: TopologySchedulingPolicy 257 | metadata: 258 | name: policy-region 259 | spec: 260 | deployPlacement: 261 | - name: nj 262 | replicas: 6 263 | - name: sh 264 | replicas: 3 265 | labelSelector: 266 | matchLabels: 267 | cluster-test: "true" 268 | topologyKey: failure-domain.beta.kubernetes.io/region 269 | ``` 270 | 271 | ```yaml 272 | apiVersion: apps/v1 273 | kind: Deployment 274 | metadata: 275 | labels: 276 | app: env 277 | name: test 278 | namespace: default 279 | spec: 280 | replicas: 9 281 | selector: 282 | matchLabels: 283 | app: env 284 | template: 285 | metadata: 286 | creationTimestamp: null 287 | labels: 288 | app: env 289 | cluster-test: "true" 290 | topology-scheduling-policy.scheduling.sigs.k8s.io: policy-region 291 | spec: 292 | containers: 293 | - image: nginx:latest 294 | imagePullPolicy: Always 295 | name: nginx 296 | resources: { } 297 | ``` 298 | 299 | ### multi cluster 300 | 301 | This project also can be used in multi-cluster scene by deploy the 302 | [tensile-kube](https://github.com/virtual-kubelet/tensile-kube), with `descheduler` in `tensile-kube` not deployed. 303 | 304 | For example, we add a label `cluster-name..scheduling.sigs.k8s.io: cluster1` to a virtual node. 305 | 306 | ```yaml 307 | apiVersion: scheduling.sigs.k8s.io/v1alpha1 308 | kind: TopologySchedulingPolicy 309 | metadata: 310 | name: policy-cluster 311 | spec: 312 | deployPlacement: 313 | - name: cluster1 314 | replicas: 6 315 | - name: cluster2 316 | replicas: 3 317 | labelSelector: 318 | matchLabels: 319 | cluster-test: "true" 320 | topologyKey: cluster-name..scheduling.sigs.k8s.io 321 | ``` 322 | 323 | ```yaml 324 | apiVersion: apps/v1 325 | kind: Deployment 326 | metadata: 327 | labels: 328 | app: env 329 | name: test 330 | namespace: default 331 | spec: 332 | replicas: 9 333 | selector: 334 | matchLabels: 335 | app: env 336 | template: 337 | metadata: 338 | creationTimestamp: null 339 | labels: 340 | app: env 341 | cluster-test: "true" 342 | cluster-name..scheduling.sigs.k8s.io: policy-cluster 343 | spec: 344 | containers: 345 | - image: nginx:latest 346 | imagePullPolicy: Always 347 | name: nginx 348 | resources: { } 349 | ``` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /pkg/topologyscheduling/topology_scheduling.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package topologyscheduling 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "sync/atomic" 23 | 24 | "k8s.io/api/core/v1" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/labels" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | corelisters "k8s.io/client-go/listers/core/v1" 29 | "k8s.io/client-go/tools/cache" 30 | "k8s.io/client-go/tools/clientcmd" 31 | "k8s.io/client-go/util/workqueue" 32 | "k8s.io/klog/v2" 33 | "k8s.io/kubernetes/pkg/scheduler/framework" 34 | "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" 35 | frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 36 | 37 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/config" 38 | "github.com/cwdsuzhou/super-scheduling/pkg/apis/scheduling/v1alpha1" 39 | "github.com/cwdsuzhou/super-scheduling/pkg/common" 40 | "github.com/cwdsuzhou/super-scheduling/pkg/generated/clientset/versioned" 41 | schedinformer "github.com/cwdsuzhou/super-scheduling/pkg/generated/informers/externalversions" 42 | externalv1alpha1 "github.com/cwdsuzhou/super-scheduling/pkg/generated/listers/scheduling/v1alpha1" 43 | "github.com/cwdsuzhou/super-scheduling/pkg/util" 44 | ) 45 | 46 | const ( 47 | // ErrReasonConstraintsNotMatch is used for PodTopologySpread filter error. 48 | ErrReasonConstraintsNotMatch = "node(s) didn't match pod topology spread constraints" 49 | // ErrReasonNodeLabelNotMatch is used when the node doesn't hold the required label. 50 | ErrReasonNodeLabelNotMatch = ErrReasonConstraintsNotMatch + " (missing required label)" 51 | ) 52 | 53 | // TopologyScheduling is a plugin that implements the mechanism of capacity scheduling. 54 | type TopologyScheduling struct { 55 | frameworkHandle framework.Handle 56 | podLister corelisters.PodLister 57 | topologyPolicyLister externalv1alpha1.TopologySchedulingPolicyLister 58 | } 59 | 60 | // PreFilterState computed at PreFilter and used at PostFilter or Reserve. 61 | type PreFilterState struct { 62 | Constraint common.TopologySchedulingConstraint 63 | // TpPairToMatchNum is keyed with TopologyPair, and valued with the number of matching pods. 64 | TpPairToMatchNum map[common.TopologyPair]*int32 65 | // ScheduleStrategy can be Balance or Fill 66 | ScheduleStrategy string 67 | } 68 | 69 | func (s *PreFilterState) updateWithPod(updatedPod, preemptorPod *v1.Pod, node *v1.Node, delta int32) { 70 | if s == nil || updatedPod.Namespace != preemptorPod.Namespace || node == nil { 71 | return 72 | } 73 | if _, ok := node.Labels[s.Constraint.TopologyKey]; !ok { 74 | return 75 | } 76 | 77 | podLabelSet := labels.Set(updatedPod.Labels) 78 | if !s.Constraint.Selector.Matches(podLabelSet) { 79 | return 80 | } 81 | 82 | k, v := s.Constraint.TopologyKey, node.Labels[s.Constraint.TopologyKey] 83 | pair := common.TopologyPair{Key: k, Value: v} 84 | *s.TpPairToMatchNum[pair] += delta 85 | } 86 | 87 | // Clone the preFilter state. 88 | func (s *PreFilterState) Clone() framework.StateData { 89 | if s == nil { 90 | return nil 91 | } 92 | stateCopy := PreFilterState{ 93 | ScheduleStrategy: s.ScheduleStrategy, 94 | // Constraints are shared because they don't change. 95 | Constraint: s.Constraint, 96 | TpPairToMatchNum: make(map[common.TopologyPair]*int32, len(s.TpPairToMatchNum)), 97 | } 98 | for tpPair, matchNum := range s.TpPairToMatchNum { 99 | copyPair := common.TopologyPair{Key: tpPair.Key, Value: tpPair.Value} 100 | copyCount := *matchNum 101 | stateCopy.TpPairToMatchNum[copyPair] = ©Count 102 | } 103 | return &stateCopy 104 | } 105 | 106 | // Prefilter build cache for scheduling cycle 107 | // Filter Checks node filter. 108 | // Score the nodes according to topology 109 | // Reserve reserves the node/pod to cache, if failed run unreserve to clean up cache 110 | var _ framework.PreFilterPlugin = &TopologyScheduling{} 111 | var _ framework.FilterPlugin = &TopologyScheduling{} 112 | var _ framework.ScorePlugin = &TopologyScheduling{} 113 | var _ framework.ReservePlugin = &TopologyScheduling{} 114 | 115 | const ( 116 | // Name is the name of the plugin used in Registry and configurations. 117 | Name = "TopologyScheduling" 118 | 119 | // preFilterStateKey is the key in CycleState to NodeResourcesFit pre-computed data. 120 | preFilterStateKey = "PreFilter" + Name 121 | ) 122 | 123 | // Name returns name of the plugin. It is used in logs, etc. 124 | func (ts *TopologyScheduling) Name() string { 125 | return Name 126 | } 127 | 128 | // New initializes a new plugin and returns it. 129 | func New(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { 130 | var args config.SchedulingArgs 131 | err := frameworkruntime.DecodeInto(obj, &args) 132 | if err != nil { 133 | return nil, fmt.Errorf("want args to be of type SchedulingArgs, got %T\n", obj) 134 | } 135 | c := &TopologyScheduling{ 136 | frameworkHandle: handle, 137 | podLister: handle.SharedInformerFactory().Core().V1().Pods().Lister(), 138 | } 139 | 140 | restConfig, err := clientcmd.BuildConfigFromFlags(args.KubeMaster, args.KubeConfigPath) 141 | if err != nil { 142 | return nil, err 143 | } 144 | client, err := versioned.NewForConfig(restConfig) 145 | if err != nil { 146 | return nil, err 147 | } 148 | 149 | schedSharedInformerFactory := schedinformer.NewSharedInformerFactory(client, 0) 150 | c.topologyPolicyLister = schedSharedInformerFactory.Scheduling().V1alpha1().TopologySchedulingPolicies().Lister() 151 | topologyPolicyInformer := schedSharedInformerFactory.Scheduling().V1alpha1().TopologySchedulingPolicies().Informer() 152 | 153 | schedSharedInformerFactory.Start(nil) 154 | if !cache.WaitForCacheSync(nil, topologyPolicyInformer.HasSynced) { 155 | return nil, fmt.Errorf("timed out waiting for caches to sync %v", Name) 156 | } 157 | klog.Infof("TopologyScheduling start") 158 | return c, nil 159 | } 160 | 161 | // PreFilter performs the following validations. 162 | // 1. Check if the (pod.request + eq.allocated) is less than eq.max. 163 | // 2. Check if the sum(eq's usage) > sum(eq's min). 164 | func (ts *TopologyScheduling) PreFilter(ctx context.Context, state *framework.CycleState, 165 | pod *v1.Pod) *framework.Status { 166 | if !podRequireTopologyScheduling(pod) { 167 | return nil 168 | } 169 | s, err := ts.calculatePreFilterStateCache(pod) 170 | if err != nil { 171 | return framework.AsStatus(err) 172 | } 173 | state.Write(preFilterStateKey, s) 174 | return nil 175 | } 176 | 177 | // getPreFilterState fetches a pre-computed preFilterState. 178 | func getPreFilterState(cycleState *framework.CycleState) (*PreFilterState, error) { 179 | c, err := cycleState.Read(preFilterStateKey) 180 | if err != nil { 181 | // preFilterState doesn't exist, likely PreFilter wasn't invoked. 182 | return nil, fmt.Errorf("reading %q from cycleState: %v", preFilterStateKey, err) 183 | } 184 | 185 | s, ok := c.(*PreFilterState) 186 | if !ok { 187 | return nil, fmt.Errorf("%+v convert to podtopologyspread.preFilterState error", c) 188 | } 189 | return s, nil 190 | } 191 | 192 | func (ts *TopologyScheduling) calculatePreFilterStateCache(pod *v1.Pod) (*PreFilterState, error) { 193 | policyName, ok := pod.Labels[util.TopologySchedulingLabelKey] 194 | if !ok { 195 | return nil, nil 196 | } 197 | 198 | tsp, err := ts.topologyPolicyLister.TopologySchedulingPolicies(pod.Namespace).Get(policyName) 199 | if err != nil { 200 | return nil, fmt.Errorf("obtaining pod's topology policy: %v", err) 201 | } 202 | allNodes, err := ts.frameworkHandle.SnapshotSharedLister().NodeInfos().List() 203 | if err != nil { 204 | return nil, fmt.Errorf("listing NodeInfos: %w", err) 205 | } 206 | selector, err := metav1.LabelSelectorAsSelector(tsp.Spec.LabelSelector) 207 | if err != nil { 208 | return nil, err 209 | } 210 | constraint := common.TopologySchedulingConstraint{ 211 | TopologyKey: tsp.Spec.TopologyKey, 212 | SchedulePolicy: util.CovertPolicy(tsp.Spec.DeployPlacement), 213 | Selector: selector, 214 | } 215 | s := PreFilterState{ 216 | Constraint: constraint, 217 | TpPairToMatchNum: make(map[common.TopologyPair]*int32), 218 | ScheduleStrategy: tsp.Spec.ScheduleStrategy, 219 | } 220 | for _, n := range allNodes { 221 | node := n.Node() 222 | if node == nil { 223 | klog.Error("node not found") 224 | continue 225 | } 226 | // In accordance to design, if NodeAffinity or NodeSelector is defined, 227 | // spreading is applied to nodes that pass those filters. 228 | if !helper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) { 229 | continue 230 | } 231 | // check node labels 232 | tpval, ok := node.Labels[tsp.Spec.TopologyKey] 233 | if !ok { 234 | continue 235 | } 236 | pair := common.TopologyPair{Key: tsp.Spec.TopologyKey, Value: tpval} 237 | s.TpPairToMatchNum[pair] = new(int32) 238 | } 239 | 240 | processNode := func(i int) { 241 | nodeInfo := allNodes[i] 242 | node := nodeInfo.Node() 243 | 244 | pair := common.TopologyPair{Key: tsp.Spec.TopologyKey, Value: node.Labels[tsp.Spec.TopologyKey]} 245 | tpCount := s.TpPairToMatchNum[pair] 246 | if tpCount == nil { 247 | return 248 | } 249 | count := countPodsMatchSelector(nodeInfo.Pods, s.Constraint.Selector, pod.Namespace) 250 | atomic.AddInt32(tpCount, int32(count)) 251 | } 252 | workqueue.ParallelizeUntil(context.Background(), 32, len(allNodes), processNode) 253 | return &s, nil 254 | } 255 | 256 | func (ts *TopologyScheduling) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, 257 | nodeInfo *framework.NodeInfo) *framework.Status { 258 | if !podRequireTopologyScheduling(pod) { 259 | return nil 260 | } 261 | node := nodeInfo.Node() 262 | if node == nil { 263 | return framework.AsStatus(fmt.Errorf("node not found")) 264 | } 265 | 266 | s, err := getPreFilterState(state) 267 | if err != nil { 268 | return framework.AsStatus(err) 269 | } 270 | 271 | policyName, ok := pod.Labels[util.TopologySchedulingLabelKey] 272 | if !ok { 273 | return nil 274 | } 275 | 276 | tsp, err := ts.topologyPolicyLister.TopologySchedulingPolicies(pod.Namespace).Get(policyName) 277 | if err != nil { 278 | return framework.AsStatus(fmt.Errorf("obtaining pod's topology policy: %v", err)) 279 | } 280 | 281 | podLabelSet := labels.Set(pod.Labels) 282 | if !s.Constraint.Selector.Matches(podLabelSet) { 283 | return framework.NewStatus(framework.Unschedulable, ErrReasonConstraintsNotMatch) 284 | } 285 | tpKey := tsp.Spec.TopologyKey 286 | tpVal, ok := node.Labels[tpKey] 287 | if !ok { 288 | klog.V(5).Infof("node '%s' doesn't have required label '%s'", node.Name, tpKey) 289 | return framework.NewStatus(framework.UnschedulableAndUnresolvable, ErrReasonNodeLabelNotMatch) 290 | } 291 | 292 | if checkPolicyRequirementReached(tpKey, 293 | s.Constraint.SchedulePolicy, s.TpPairToMatchNum) { 294 | return nil 295 | } 296 | 297 | if !checkTopoValueReached(tpKey, tpVal, s.Constraint.SchedulePolicy, s.TpPairToMatchNum) { 298 | return nil 299 | 300 | } 301 | return framework.NewStatus(framework.Unschedulable, ErrReasonConstraintsNotMatch) 302 | } 303 | 304 | // PreFilterExtensions returns prefilter extensions, pod add and remove. 305 | func (ts *TopologyScheduling) PreFilterExtensions() framework.PreFilterExtensions { 306 | return ts 307 | } 308 | 309 | // AddPod from pre-computed data in cycleState. 310 | func (ts *TopologyScheduling) AddPod(ctx context.Context, cycleState *framework.CycleState, podToSchedule *v1.Pod, 311 | podToAdd *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 312 | if !podRequireTopologyScheduling(podToSchedule) { 313 | return framework.NewStatus(framework.Success, "") 314 | } 315 | s, err := getPreFilterState(cycleState) 316 | if err != nil { 317 | return framework.AsStatus(err) 318 | } 319 | 320 | s.updateWithPod(podToAdd, podToSchedule, nodeInfo.Node(), 1) 321 | 322 | return framework.NewStatus(framework.Success, "") 323 | } 324 | 325 | // RemovePod from pre-computed data in cycleState. 326 | func (ts *TopologyScheduling) RemovePod(ctx context.Context, cycleState *framework.CycleState, podToSchedule *v1.Pod, 327 | podToRemove *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { 328 | if !podRequireTopologyScheduling(podToSchedule) { 329 | return framework.NewStatus(framework.Success, "") 330 | } 331 | s, err := getPreFilterState(cycleState) 332 | if err != nil { 333 | return framework.AsStatus(err) 334 | } 335 | 336 | s.updateWithPod(podToRemove, podToSchedule, nodeInfo.Node(), -1) 337 | 338 | return framework.NewStatus(framework.Success, "") 339 | } 340 | 341 | func (ts *TopologyScheduling) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, 342 | nodeName string) (int64, *framework.Status) { 343 | if !podRequireTopologyScheduling(pod) { 344 | return 0, framework.NewStatus(framework.Success, "") 345 | } 346 | s, err := getPreFilterState(state) 347 | if s == nil { 348 | return 0, framework.NewStatus(framework.Success, "") 349 | } 350 | if err != nil { 351 | return 0, framework.NewStatus(framework.Success, "") 352 | } 353 | node, err := ts.frameworkHandle.SnapshotSharedLister().NodeInfos().Get(nodeName) 354 | if err != nil || node == nil || node.Node() == nil { 355 | return 0, framework.NewStatus(framework.Error, "node not exist") 356 | } 357 | // check node labels 358 | tpval, ok := node.Node().Labels[s.Constraint.TopologyKey] 359 | if !ok { 360 | return 0, framework.NewStatus(framework.Success, "") 361 | } 362 | pair := common.TopologyPair{ 363 | Key: s.Constraint.TopologyKey, 364 | Value: tpval, 365 | } 366 | desired, ok := s.Constraint.SchedulePolicy[tpval] 367 | if !ok { 368 | return 0, framework.NewStatus(framework.Success, "") 369 | } 370 | count, ok := s.TpPairToMatchNum[pair] 371 | if count == nil || !ok { 372 | return 0, framework.NewStatus(framework.Success, "") 373 | } 374 | // when count >= desired, return directly 375 | if *count >= desired { 376 | return 0, framework.NewStatus(framework.Success, "") 377 | } 378 | return score(float64(*count), float64(desired), s.ScheduleStrategy), 379 | framework.NewStatus(framework.Success, "") 380 | } 381 | 382 | func (ts *TopologyScheduling) ScoreExtensions() framework.ScoreExtensions { 383 | return ts 384 | } 385 | 386 | func (ts *TopologyScheduling) NormalizeScore(ctx context.Context, state *framework.CycleState, p *v1.Pod, 387 | scores framework.NodeScoreList) *framework.Status { 388 | return framework.NewStatus(framework.Success) 389 | } 390 | 391 | func (ts *TopologyScheduling) Reserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, 392 | nodeName string) *framework.Status { 393 | if !podRequireTopologyScheduling(pod) { 394 | return framework.NewStatus(framework.Success, "") 395 | } 396 | s, err := getPreFilterState(state) 397 | if err != nil { 398 | return framework.AsStatus(err) 399 | } 400 | node, err := ts.frameworkHandle.SnapshotSharedLister().NodeInfos().Get(nodeName) 401 | if err != nil { 402 | return framework.AsStatus(err) 403 | } 404 | s.updateWithPod(pod, pod, node.Node(), 1) 405 | return framework.NewStatus(framework.Success, "") 406 | } 407 | 408 | func (ts *TopologyScheduling) Unreserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) { 409 | if !podRequireTopologyScheduling(pod) { 410 | return 411 | } 412 | s, err := getPreFilterState(state) 413 | if err != nil { 414 | return 415 | } 416 | node, err := ts.frameworkHandle.SnapshotSharedLister().NodeInfos().Get(nodeName) 417 | if err != nil { 418 | return 419 | } 420 | s.updateWithPod(pod, pod, node.Node(), -1) 421 | } 422 | 423 | func countPodsMatchSelector(podInfos []*framework.PodInfo, selector labels.Selector, ns string) int { 424 | count := 0 425 | for _, p := range podInfos { 426 | if p == nil { 427 | continue 428 | } 429 | pod := p.Pod 430 | // Bypass terminating Pod (see #87621). 431 | if pod.DeletionTimestamp != nil || pod.Namespace != ns { 432 | continue 433 | } 434 | if selector.Matches(labels.Set(pod.Labels)) { 435 | count++ 436 | } 437 | } 438 | return count 439 | } 440 | 441 | func checkPolicyRequirementReached(topoKey string, policy map[string]int32, 442 | tpPairToMatchNum map[common.TopologyPair]*int32) bool { 443 | for value, replicas := range policy { 444 | pair := common.TopologyPair{ 445 | Key: topoKey, 446 | Value: value, 447 | } 448 | count, ok := tpPairToMatchNum[pair] 449 | if count == nil || !ok { 450 | return false 451 | } 452 | if *count < replicas { 453 | return false 454 | } 455 | } 456 | return true 457 | } 458 | 459 | func checkTopoValueReached(topoKey, topoValue string, policy map[string]int32, 460 | tpPairToMatchNum map[common.TopologyPair]*int32) bool { 461 | pair := common.TopologyPair{ 462 | Key: topoKey, 463 | Value: topoValue, 464 | } 465 | desired, ok := policy[topoValue] 466 | if !ok { 467 | return true 468 | } 469 | count, ok := tpPairToMatchNum[pair] 470 | if count == nil || !ok { 471 | return false 472 | } 473 | if *count < desired { 474 | return false 475 | } 476 | return true 477 | } 478 | 479 | func podRequireTopologyScheduling(pod *v1.Pod) bool { 480 | _, ok := pod.Labels[util.TopologySchedulingLabelKey] 481 | if !ok { 482 | return false 483 | } 484 | return true 485 | } 486 | 487 | func score(count, desired float64, policy string) int64 { 488 | if policy == v1alpha1.ScheduleStrategyBalance { 489 | return int64(1 - count/desired*100) 490 | } 491 | if policy == v1alpha1.ScheduleStrategyFill { 492 | return int64(count / desired * 100) 493 | } 494 | return 0 495 | } 496 | --------------------------------------------------------------------------------